1/*
2 * Copyright (C) 2018, 2019 Apple Inc. All rights reserved.
3 *
4 * Redistribution and use in source and binary forms, with or without
5 * modification, are permitted provided that the following conditions
6 * are met:
7 * 1. Redistributions of source code must retain the above copyright
8 * notice, this list of conditions and the following disclaimer.
9 * 2. Redistributions in binary form must reproduce the above copyright
10 * notice, this list of conditions and the following disclaimer in the
11 * documentation and/or other materials provided with the distribution.
12 *
13 * THIS SOFTWARE IS PROVIDED BY APPLE INC. AND ITS CONTRIBUTORS ``AS IS''
14 * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO,
15 * THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
16 * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL APPLE INC. OR ITS CONTRIBUTORS
17 * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
18 * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
19 * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
20 * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
21 * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
22 * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF
23 * THE POSSIBILITY OF SUCH DAMAGE.
24 */
25
26#include "config.h"
27#include "SimulatedInputDispatcher.h"
28
29#if ENABLE(WEBDRIVER_ACTIONS_API)
30
31#include "AutomationProtocolObjects.h"
32#include "Logging.h"
33#include "WebAutomationSession.h"
34#include "WebAutomationSessionMacros.h"
35
36namespace WebKit {
37
38SimulatedInputSourceState SimulatedInputSourceState::emptyStateForSourceType(SimulatedInputSourceType type)
39{
40 SimulatedInputSourceState result { };
41 switch (type) {
42 case SimulatedInputSourceType::Null:
43 case SimulatedInputSourceType::Keyboard:
44 break;
45 case SimulatedInputSourceType::Mouse:
46 case SimulatedInputSourceType::Touch:
47 result.location = WebCore::IntPoint();
48 }
49
50 return result;
51}
52
53
54SimulatedInputKeyFrame::SimulatedInputKeyFrame(Vector<StateEntry>&& entries)
55 : states(WTFMove(entries))
56{
57}
58
59Seconds SimulatedInputKeyFrame::maximumDuration() const
60{
61 // The "compute the tick duration" algorithm (§17.4 Dispatching Actions).
62 Seconds result;
63 for (auto& entry : states)
64 result = std::max(result, entry.second.duration.valueOr(Seconds(0)));
65
66 return result;
67}
68
69SimulatedInputKeyFrame SimulatedInputKeyFrame::keyFrameFromStateOfInputSources(HashSet<Ref<SimulatedInputSource>>& inputSources)
70{
71 // The client of this class is required to intern SimulatedInputSource instances if the last state
72 // from the previous command should be used as the inital state for the next command. This is the
73 // case for Perform Actions and Release Actions, but not Element Click or Element Send Keys.
74 Vector<SimulatedInputKeyFrame::StateEntry> entries;
75 entries.reserveCapacity(inputSources.size());
76
77 for (auto& inputSource : inputSources)
78 entries.uncheckedAppend(std::pair<SimulatedInputSource&, SimulatedInputSourceState> { inputSource.get(), inputSource->state });
79
80 return SimulatedInputKeyFrame(WTFMove(entries));
81}
82
83SimulatedInputKeyFrame SimulatedInputKeyFrame::keyFrameToResetInputSources(HashSet<Ref<SimulatedInputSource>>& inputSources)
84{
85 Vector<SimulatedInputKeyFrame::StateEntry> entries;
86 entries.reserveCapacity(inputSources.size());
87
88 for (auto& inputSource : inputSources)
89 entries.uncheckedAppend(std::pair<SimulatedInputSource&, SimulatedInputSourceState> { inputSource.get(), SimulatedInputSourceState::emptyStateForSourceType(inputSource->type) });
90
91 return SimulatedInputKeyFrame(WTFMove(entries));
92}
93
94SimulatedInputDispatcher::SimulatedInputDispatcher(WebPageProxy& page, SimulatedInputDispatcher::Client& client)
95 : m_page(page)
96 , m_client(client)
97 , m_keyFrameTransitionDurationTimer(RunLoop::current(), this, &SimulatedInputDispatcher::keyFrameTransitionDurationTimerFired)
98{
99}
100
101SimulatedInputDispatcher::~SimulatedInputDispatcher()
102{
103 ASSERT(!m_runCompletionHandler);
104 ASSERT(!m_keyFrameTransitionDurationTimer.isActive());
105}
106
107bool SimulatedInputDispatcher::isActive() const
108{
109 return !!m_runCompletionHandler;
110}
111
112void SimulatedInputDispatcher::keyFrameTransitionDurationTimerFired()
113{
114 ASSERT(m_keyFrameTransitionCompletionHandler);
115
116 m_keyFrameTransitionDurationTimer.stop();
117
118 LOG(Automation, "SimulatedInputDispatcher[%p]: timer finished for transition between keyframes: %d --> %d", this, m_keyframeIndex - 1, m_keyframeIndex);
119
120 if (isKeyFrameTransitionComplete()) {
121 auto finish = std::exchange(m_keyFrameTransitionCompletionHandler, nullptr);
122 finish(WTF::nullopt);
123 }
124}
125
126bool SimulatedInputDispatcher::isKeyFrameTransitionComplete() const
127{
128 ASSERT(m_keyframeIndex < m_keyframes.size());
129
130 if (m_inputSourceStateIndex < m_keyframes[m_keyframeIndex].states.size())
131 return false;
132
133 if (m_keyFrameTransitionDurationTimer.isActive())
134 return false;
135
136 return true;
137}
138
139void SimulatedInputDispatcher::transitionToNextKeyFrame()
140{
141 ++m_keyframeIndex;
142 if (m_keyframeIndex == m_keyframes.size()) {
143 finishDispatching(WTF::nullopt);
144 return;
145 }
146
147 transitionBetweenKeyFrames(m_keyframes[m_keyframeIndex - 1], m_keyframes[m_keyframeIndex], [this, protectedThis = makeRef(*this)](Optional<AutomationCommandError> error) {
148 if (error) {
149 finishDispatching(error);
150 return;
151 }
152
153 transitionToNextKeyFrame();
154 });
155}
156
157void SimulatedInputDispatcher::transitionToNextInputSourceState()
158{
159 if (isKeyFrameTransitionComplete()) {
160 auto finish = std::exchange(m_keyFrameTransitionCompletionHandler, nullptr);
161 finish(WTF::nullopt);
162 return;
163 }
164
165 // In this case, transitions are done but we need to wait for the tick timer.
166 if (m_inputSourceStateIndex == m_keyframes[m_keyframeIndex].states.size())
167 return;
168
169 auto& nextKeyFrame = m_keyframes[m_keyframeIndex];
170 auto& postStateEntry = nextKeyFrame.states[m_inputSourceStateIndex];
171 SimulatedInputSource& inputSource = postStateEntry.first;
172
173 transitionInputSourceToState(inputSource, postStateEntry.second, [this, protectedThis = makeRef(*this)](Optional<AutomationCommandError> error) {
174 if (error) {
175 auto finish = std::exchange(m_keyFrameTransitionCompletionHandler, nullptr);
176 finish(error);
177 return;
178 }
179
180 // Perform state transitions in the order specified by the currentKeyFrame.
181 ++m_inputSourceStateIndex;
182
183 transitionToNextInputSourceState();
184 });
185}
186
187void SimulatedInputDispatcher::transitionBetweenKeyFrames(const SimulatedInputKeyFrame& a, const SimulatedInputKeyFrame& b, AutomationCompletionHandler&& completionHandler)
188{
189 m_inputSourceStateIndex = 0;
190
191 // The "dispatch tick actions" algorithm (§17.4 Dispatching Actions).
192 m_keyFrameTransitionCompletionHandler = WTFMove(completionHandler);
193 m_keyFrameTransitionDurationTimer.startOneShot(b.maximumDuration());
194
195 LOG(Automation, "SimulatedInputDispatcher[%p]: started transition between keyframes: %d --> %d", this, m_keyframeIndex - 1, m_keyframeIndex);
196 LOG(Automation, "SimulatedInputDispatcher[%p]: timer started to ensure minimum duration of %.2f seconds for transition %d --> %d", this, b.maximumDuration().value(), m_keyframeIndex - 1, m_keyframeIndex);
197
198 transitionToNextInputSourceState();
199}
200
201void SimulatedInputDispatcher::resolveLocation(const WebCore::IntPoint& currentLocation, Optional<WebCore::IntPoint> location, MouseMoveOrigin origin, Optional<String> nodeHandle, Function<void (Optional<WebCore::IntPoint>, Optional<AutomationCommandError>)>&& completionHandler)
202{
203 if (!location) {
204 completionHandler(currentLocation, WTF::nullopt);
205 return;
206 }
207
208 switch (origin) {
209 case MouseMoveOrigin::Viewport:
210 completionHandler(location.value(), WTF::nullopt);
211 break;
212 case MouseMoveOrigin::Pointer: {
213 WebCore::IntPoint destination(currentLocation);
214 destination.moveBy(location.value());
215 completionHandler(destination, WTF::nullopt);
216 break;
217 }
218 case MouseMoveOrigin::Element: {
219 m_client.viewportInViewCenterPointOfElement(m_page, m_frameID.value(), nodeHandle.value(), [destination = location.value(), completionHandler = WTFMove(completionHandler)](Optional<WebCore::IntPoint> inViewCenterPoint, Optional<AutomationCommandError> error) mutable {
220 if (error) {
221 completionHandler(WTF::nullopt, error);
222 return;
223 }
224
225 if (!inViewCenterPoint) {
226 completionHandler(WTF::nullopt, AUTOMATION_COMMAND_ERROR_WITH_NAME(ElementNotInteractable));
227 return;
228 }
229
230 destination.moveBy(inViewCenterPoint.value());
231 completionHandler(destination, WTF::nullopt);
232 });
233 break;
234 }
235 }
236}
237
238void SimulatedInputDispatcher::transitionInputSourceToState(SimulatedInputSource& inputSource, SimulatedInputSourceState& newState, AutomationCompletionHandler&& completionHandler)
239{
240 // Make cases and conditionals more readable by aliasing pre/post states as 'a' and 'b'.
241 SimulatedInputSourceState& a = inputSource.state;
242 SimulatedInputSourceState& b = newState;
243
244 LOG(Automation, "SimulatedInputDispatcher[%p]: transition started between input source states: [%d.%d] --> %d.%d", this, m_keyframeIndex - 1, m_inputSourceStateIndex, m_keyframeIndex, m_inputSourceStateIndex);
245
246 AutomationCompletionHandler eventDispatchFinished = [this, &inputSource, &newState, completionHandler = WTFMove(completionHandler)](Optional<AutomationCommandError> error) mutable {
247 if (error) {
248 completionHandler(error);
249 return;
250 }
251
252#if !LOG_DISABLED
253 LOG(Automation, "SimulatedInputDispatcher[%p]: transition finished between input source states: %d.%d --> [%d.%d]", this, m_keyframeIndex - 1, m_inputSourceStateIndex, m_keyframeIndex, m_inputSourceStateIndex);
254#else
255 UNUSED_PARAM(this);
256#endif
257
258 inputSource.state = newState;
259 completionHandler(WTF::nullopt);
260 };
261
262 switch (inputSource.type) {
263 case SimulatedInputSourceType::Null:
264 // The maximum duration is handled at the keyframe level by m_keyFrameTransitionDurationTimer.
265 eventDispatchFinished(WTF::nullopt);
266 break;
267 case SimulatedInputSourceType::Mouse: {
268#if !ENABLE(WEBDRIVER_MOUSE_INTERACTIONS)
269 RELEASE_ASSERT_NOT_REACHED();
270#else
271 resolveLocation(a.location.valueOr(WebCore::IntPoint()), b.location, b.origin.valueOr(MouseMoveOrigin::Viewport), b.nodeHandle, [this, &a, &b, eventDispatchFinished = WTFMove(eventDispatchFinished)](Optional<WebCore::IntPoint> location, Optional<AutomationCommandError> error) mutable {
272 if (error) {
273 eventDispatchFinished(error);
274 return;
275 }
276
277 if (!location) {
278 eventDispatchFinished(AUTOMATION_COMMAND_ERROR_WITH_NAME(ElementNotInteractable));
279 return;
280 }
281
282 b.location = location;
283 // The "dispatch a pointer{Down,Up,Move} action" algorithms (§17.4 Dispatching Actions).
284 if (!a.pressedMouseButton && b.pressedMouseButton) {
285#if !LOG_DISABLED
286 String mouseButtonName = Inspector::Protocol::AutomationHelpers::getEnumConstantValue(b.pressedMouseButton.value());
287 LOG(Automation, "SimulatedInputDispatcher[%p]: simulating MouseDown[button=%s] @ (%d, %d) for transition to %d.%d", this, mouseButtonName.utf8().data(), b.location.value().x(), b.location.value().y(), m_keyframeIndex, m_inputSourceStateIndex);
288#endif
289 m_client.simulateMouseInteraction(m_page, MouseInteraction::Down, b.pressedMouseButton.value(), b.location.value(), WTFMove(eventDispatchFinished));
290 } else if (a.pressedMouseButton && !b.pressedMouseButton) {
291#if !LOG_DISABLED
292 String mouseButtonName = Inspector::Protocol::AutomationHelpers::getEnumConstantValue(a.pressedMouseButton.value());
293 LOG(Automation, "SimulatedInputDispatcher[%p]: simulating MouseUp[button=%s] @ (%d, %d) for transition to %d.%d", this, mouseButtonName.utf8().data(), b.location.value().x(), b.location.value().y(), m_keyframeIndex, m_inputSourceStateIndex);
294#endif
295 m_client.simulateMouseInteraction(m_page, MouseInteraction::Up, a.pressedMouseButton.value(), b.location.value(), WTFMove(eventDispatchFinished));
296 } else if (a.location != b.location) {
297 LOG(Automation, "SimulatedInputDispatcher[%p]: simulating MouseMove from (%d, %d) to (%d, %d) for transition to %d.%d", this, a.location.value().x(), a.location.value().y(), b.location.value().x(), b.location.value().y(), m_keyframeIndex, m_inputSourceStateIndex);
298 // FIXME: This does not interpolate mousemoves per the "perform a pointer move" algorithm (§17.4 Dispatching Actions).
299 m_client.simulateMouseInteraction(m_page, MouseInteraction::Move, b.pressedMouseButton.valueOr(MouseButton::NoButton), b.location.value(), WTFMove(eventDispatchFinished));
300 } else
301 eventDispatchFinished(WTF::nullopt);
302 });
303#endif // ENABLE(WEBDRIVER_MOUSE_INTERACTIONS)
304 break;
305 }
306 case SimulatedInputSourceType::Touch: {
307#if !ENABLE(WEBDRIVER_TOUCH_INTERACTIONS)
308 RELEASE_ASSERT_NOT_REACHED();
309#else
310 resolveLocation(a.location.valueOr(WebCore::IntPoint()), b.location, b.origin.valueOr(MouseMoveOrigin::Viewport), b.nodeHandle, [this, &a, &b, eventDispatchFinished = WTFMove(eventDispatchFinished)](Optional<WebCore::IntPoint> location, Optional<AutomationCommandError> error) mutable {
311 if (error) {
312 eventDispatchFinished(error);
313 return;
314 }
315
316 if (!location) {
317 eventDispatchFinished(AUTOMATION_COMMAND_ERROR_WITH_NAME(ElementNotInteractable));
318 return;
319 }
320
321 b.location = location;
322 // The "dispatch a pointer{Down,Up,Move} action" algorithms (§17.4 Dispatching Actions).
323 if (!a.pressedMouseButton && b.pressedMouseButton) {
324 LOG(Automation, "SimulatedInputDispatcher[%p]: simulating TouchDown @ (%d, %d) for transition to %d.%d", this, b.location.value().x(), b.location.value().y(), m_keyframeIndex, m_inputSourceStateIndex);
325 m_client.simulateTouchInteraction(m_page, TouchInteraction::TouchDown, b.location.value(), WTF::nullopt, WTFMove(eventDispatchFinished));
326 } else if (a.pressedMouseButton && !b.pressedMouseButton) {
327 LOG(Automation, "SimulatedInputDispatcher[%p]: simulating LiftUp @ (%d, %d) for transition to %d.%d", this, b.location.value().x(), b.location.value().y(), m_keyframeIndex, m_inputSourceStateIndex);
328 m_client.simulateTouchInteraction(m_page, TouchInteraction::LiftUp, b.location.value(), WTF::nullopt, WTFMove(eventDispatchFinished));
329 } else if (a.location != b.location) {
330 LOG(Automation, "SimulatedInputDispatcher[%p]: simulating MoveTo from (%d, %d) to (%d, %d) for transition to %d.%d", this, a.location.value().x(), a.location.value().y(), b.location.value().x(), b.location.value().y(), m_keyframeIndex, m_inputSourceStateIndex);
331 m_client.simulateTouchInteraction(m_page, TouchInteraction::MoveTo, b.location.value(), a.duration.valueOr(0_s), WTFMove(eventDispatchFinished));
332 } else
333 eventDispatchFinished(WTF::nullopt);
334 });
335#endif // !ENABLE(WEBDRIVER_TOUCH_INTERACTIONS)
336 break;
337 }
338 case SimulatedInputSourceType::Keyboard:
339#if !ENABLE(WEBDRIVER_KEYBOARD_INTERACTIONS)
340 RELEASE_ASSERT_NOT_REACHED();
341#else
342 // The "dispatch a key{Down,Up} action" algorithms (§17.4 Dispatching Actions).
343 if (!a.pressedCharKey && b.pressedCharKey) {
344 LOG(Automation, "SimulatedInputDispatcher[%p]: simulating KeyPress[key=%c] for transition to %d.%d", this, b.pressedCharKey.value(), m_keyframeIndex, m_inputSourceStateIndex);
345 m_client.simulateKeyboardInteraction(m_page, KeyboardInteraction::KeyPress, b.pressedCharKey.value(), WTFMove(eventDispatchFinished));
346 } else if (a.pressedCharKey && !b.pressedCharKey) {
347 LOG(Automation, "SimulatedInputDispatcher[%p]: simulating KeyRelease[key=%c] for transition to %d.%d", this, a.pressedCharKey.value(), m_keyframeIndex, m_inputSourceStateIndex);
348 m_client.simulateKeyboardInteraction(m_page, KeyboardInteraction::KeyRelease, a.pressedCharKey.value(), WTFMove(eventDispatchFinished));
349 } else if (a.pressedVirtualKeys != b.pressedVirtualKeys) {
350 bool simulatedAnInteraction = false;
351 for (VirtualKey key : b.pressedVirtualKeys) {
352 if (!a.pressedVirtualKeys.contains(key)) {
353 ASSERT_WITH_MESSAGE(!simulatedAnInteraction, "Only one VirtualKey may differ at a time between two input source states.");
354 if (simulatedAnInteraction)
355 continue;
356 simulatedAnInteraction = true;
357#if !LOG_DISABLED
358 String virtualKeyName = Inspector::Protocol::AutomationHelpers::getEnumConstantValue(key);
359 LOG(Automation, "SimulatedInputDispatcher[%p]: simulating KeyPress[key=%s] for transition to %d.%d", this, virtualKeyName.utf8().data(), m_keyframeIndex, m_inputSourceStateIndex);
360#endif
361 m_client.simulateKeyboardInteraction(m_page, KeyboardInteraction::KeyPress, key, WTFMove(eventDispatchFinished));
362 }
363 }
364
365 for (VirtualKey key : a.pressedVirtualKeys) {
366 if (!b.pressedVirtualKeys.contains(key)) {
367 ASSERT_WITH_MESSAGE(!simulatedAnInteraction, "Only one VirtualKey may differ at a time between two input source states.");
368 if (simulatedAnInteraction)
369 continue;
370 simulatedAnInteraction = true;
371#if !LOG_DISABLED
372 String virtualKeyName = Inspector::Protocol::AutomationHelpers::getEnumConstantValue(key);
373 LOG(Automation, "SimulatedInputDispatcher[%p]: simulating KeyRelease[key=%s] for transition to %d.%d", this, virtualKeyName.utf8().data(), m_keyframeIndex, m_inputSourceStateIndex);
374#endif
375 m_client.simulateKeyboardInteraction(m_page, KeyboardInteraction::KeyRelease, key, WTFMove(eventDispatchFinished));
376 }
377 }
378 } else
379 eventDispatchFinished(WTF::nullopt);
380#endif // !ENABLE(WEBDRIVER_KEYBOARD_INTERACTIONS)
381 break;
382 }
383}
384
385void SimulatedInputDispatcher::run(uint64_t frameID, Vector<SimulatedInputKeyFrame>&& keyFrames, HashSet<Ref<SimulatedInputSource>>& inputSources, AutomationCompletionHandler&& completionHandler)
386{
387 ASSERT(!isActive());
388 if (isActive()) {
389 completionHandler(AUTOMATION_COMMAND_ERROR_WITH_NAME(InternalError));
390 return;
391 }
392
393 m_frameID = frameID;
394 m_runCompletionHandler = WTFMove(completionHandler);
395 for (const Ref<SimulatedInputSource>& inputSource : inputSources)
396 m_inputSources.add(inputSource.copyRef());
397
398 // The "dispatch actions" algorithm (§17.4 Dispatching Actions).
399
400 m_keyframes.reserveCapacity(keyFrames.size() + 1);
401 m_keyframes.append(SimulatedInputKeyFrame::keyFrameFromStateOfInputSources(m_inputSources));
402 m_keyframes.appendVector(WTFMove(keyFrames));
403
404 LOG(Automation, "SimulatedInputDispatcher[%p]: starting input simulation using %d keyframes", this, m_keyframeIndex);
405
406 transitionToNextKeyFrame();
407}
408
409void SimulatedInputDispatcher::cancel()
410{
411 // If we were waiting for m_client to finish an interaction and the interaction had an error,
412 // then the rest of the async chain will have been torn down. If we are just waiting on a
413 // dispatch timer, then this will cancel the timer and clear
414
415 if (isActive())
416 finishDispatching(AUTOMATION_COMMAND_ERROR_WITH_NAME(InternalError));
417}
418
419void SimulatedInputDispatcher::finishDispatching(Optional<AutomationCommandError> error)
420{
421 m_keyFrameTransitionDurationTimer.stop();
422
423 LOG(Automation, "SimulatedInputDispatcher[%p]: finished all input simulation at [%u.%u]", this, m_keyframeIndex, m_inputSourceStateIndex);
424
425 auto finish = std::exchange(m_runCompletionHandler, nullptr);
426 m_frameID = WTF::nullopt;
427 m_keyframes.clear();
428 m_inputSources.clear();
429 m_keyframeIndex = 0;
430 m_inputSourceStateIndex = 0;
431
432 finish(error);
433}
434
435} // namespace Webkit
436
437#endif // ENABLE(WEBDRIVER_ACTIONS_API)
438