blob: e07181abe19c8044083b8f7007d719500cbbd37d [file] [log] [blame]
Alan Viverette3da604b2020-06-10 18:34:39 +00001/*
2 * Copyright (C) 2017 The Android Open Source Project
3 *
4 * Licensed under the Apache License, Version 2.0 (the "License");
5 * you may not use this file except in compliance with the License.
6 * You may obtain a copy of the License at
7 *
8 * http://www.apache.org/licenses/LICENSE-2.0
9 *
10 * Unless required by applicable law or agreed to in writing, software
11 * distributed under the License is distributed on an "AS IS" BASIS,
12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 * See the License for the specific language governing permissions and
14 * limitations under the License.
15 */
16
17package android.widget;
18
19import android.annotation.NonNull;
20import android.annotation.Nullable;
21import android.annotation.UiThread;
22import android.annotation.WorkerThread;
23import android.app.RemoteAction;
24import android.content.Context;
25import android.graphics.Canvas;
26import android.graphics.PointF;
27import android.graphics.RectF;
28import android.os.AsyncTask;
29import android.os.Build;
30import android.os.Bundle;
31import android.os.LocaleList;
32import android.text.Layout;
33import android.text.Selection;
34import android.text.Spannable;
35import android.text.TextUtils;
36import android.text.util.Linkify;
37import android.util.Log;
38import android.view.ActionMode;
39import android.view.textclassifier.ExtrasUtils;
40import android.view.textclassifier.SelectionEvent;
41import android.view.textclassifier.SelectionEvent.InvocationMethod;
42import android.view.textclassifier.TextClassification;
43import android.view.textclassifier.TextClassificationConstants;
44import android.view.textclassifier.TextClassificationContext;
45import android.view.textclassifier.TextClassificationManager;
46import android.view.textclassifier.TextClassifier;
47import android.view.textclassifier.TextClassifierEvent;
48import android.view.textclassifier.TextSelection;
49import android.widget.Editor.SelectionModifierCursorController;
50
51import com.android.internal.annotations.VisibleForTesting;
52import com.android.internal.util.Preconditions;
53
54import java.text.BreakIterator;
55import java.util.ArrayList;
56import java.util.Comparator;
57import java.util.List;
58import java.util.Objects;
59import java.util.function.Consumer;
60import java.util.function.Function;
61import java.util.function.Supplier;
62import java.util.regex.Pattern;
63
64/**
65 * Helper class for starting selection action mode
66 * (synchronously without the TextClassifier, asynchronously with the TextClassifier).
67 * @hide
68 */
69@UiThread
70@VisibleForTesting(visibility = VisibleForTesting.Visibility.PACKAGE)
71public final class SelectionActionModeHelper {
72
73 private static final String LOG_TAG = "SelectActionModeHelper";
74
75 private final Editor mEditor;
76 private final TextView mTextView;
77 private final TextClassificationHelper mTextClassificationHelper;
78
79 @Nullable private TextClassification mTextClassification;
80 private AsyncTask mTextClassificationAsyncTask;
81
82 private final SelectionTracker mSelectionTracker;
83
84 // TODO remove nullable marker once the switch gating the feature gets removed
85 @Nullable
86 private final SmartSelectSprite mSmartSelectSprite;
87
88 SelectionActionModeHelper(@NonNull Editor editor) {
89 mEditor = Objects.requireNonNull(editor);
90 mTextView = mEditor.getTextView();
91 mTextClassificationHelper = new TextClassificationHelper(
92 mTextView.getContext(),
93 mTextView::getTextClassificationSession,
94 getText(mTextView),
95 0, 1, mTextView.getTextLocales());
96 mSelectionTracker = new SelectionTracker(mTextView);
97
98 if (getTextClassificationSettings().isSmartSelectionAnimationEnabled()) {
99 mSmartSelectSprite = new SmartSelectSprite(mTextView.getContext(),
100 editor.getTextView().mHighlightColor, mTextView::invalidate);
101 } else {
102 mSmartSelectSprite = null;
103 }
104 }
105
106 /**
107 * Swap the selection index if the start index is greater than end index.
108 *
109 * @return the swap result, index 0 is the start index and index 1 is the end index.
110 */
111 private static int[] sortSelctionIndices(int selectionStart, int selectionEnd) {
112 if (selectionStart < selectionEnd) {
113 return new int[]{selectionStart, selectionEnd};
114 }
115 return new int[]{selectionEnd, selectionStart};
116 }
117
118 /**
119 * The {@link TextView} selection start and end index may not be sorted, this method will swap
120 * the {@link TextView} selection index if the start index is greater than end index.
121 *
122 * @param textView the selected TextView.
123 * @return the swap result, index 0 is the start index and index 1 is the end index.
124 */
125 private static int[] sortSelctionIndicesFromTextView(TextView textView) {
126 int selectionStart = textView.getSelectionStart();
127 int selectionEnd = textView.getSelectionEnd();
128
129 return sortSelctionIndices(selectionStart, selectionEnd);
130 }
131
132 /**
133 * Starts Selection ActionMode.
134 */
135 public void startSelectionActionModeAsync(boolean adjustSelection) {
136 // Check if the smart selection should run for editable text.
137 adjustSelection &= getTextClassificationSettings().isSmartSelectionEnabled();
138 int[] sortedSelectionIndices = sortSelctionIndicesFromTextView(mTextView);
139
140 mSelectionTracker.onOriginalSelection(
141 getText(mTextView),
142 sortedSelectionIndices[0],
143 sortedSelectionIndices[1],
144 false /*isLink*/);
145 cancelAsyncTask();
146 if (skipTextClassification()) {
147 startSelectionActionMode(null);
148 } else {
149 resetTextClassificationHelper();
150 mTextClassificationAsyncTask = new TextClassificationAsyncTask(
151 mTextView,
152 mTextClassificationHelper.getTimeoutDuration(),
153 adjustSelection
154 ? mTextClassificationHelper::suggestSelection
155 : mTextClassificationHelper::classifyText,
156 mSmartSelectSprite != null
157 ? this::startSelectionActionModeWithSmartSelectAnimation
158 : this::startSelectionActionMode,
159 mTextClassificationHelper::getOriginalSelection)
160 .execute();
161 }
162 }
163
164 /**
165 * Starts Link ActionMode.
166 */
167 public void startLinkActionModeAsync(int start, int end) {
168 int[] indexResult = sortSelctionIndices(start, end);
169 mSelectionTracker.onOriginalSelection(getText(mTextView), indexResult[0], indexResult[1],
170 true /*isLink*/);
171 cancelAsyncTask();
172 if (skipTextClassification()) {
173 startLinkActionMode(null);
174 } else {
175 resetTextClassificationHelper(indexResult[0], indexResult[1]);
176 mTextClassificationAsyncTask = new TextClassificationAsyncTask(
177 mTextView,
178 mTextClassificationHelper.getTimeoutDuration(),
179 mTextClassificationHelper::classifyText,
180 this::startLinkActionMode,
181 mTextClassificationHelper::getOriginalSelection)
182 .execute();
183 }
184 }
185
186 public void invalidateActionModeAsync() {
187 cancelAsyncTask();
188 if (skipTextClassification()) {
189 invalidateActionMode(null);
190 } else {
191 resetTextClassificationHelper();
192 mTextClassificationAsyncTask = new TextClassificationAsyncTask(
193 mTextView,
194 mTextClassificationHelper.getTimeoutDuration(),
195 mTextClassificationHelper::classifyText,
196 this::invalidateActionMode,
197 mTextClassificationHelper::getOriginalSelection)
198 .execute();
199 }
200 }
201
202 /** Reports a selection action event. */
203 public void onSelectionAction(int menuItemId, @Nullable String actionLabel) {
204 int[] sortedSelectionIndices = sortSelctionIndicesFromTextView(mTextView);
205 mSelectionTracker.onSelectionAction(
206 sortedSelectionIndices[0], sortedSelectionIndices[1],
207 getActionType(menuItemId), actionLabel, mTextClassification);
208 }
209
210 public void onSelectionDrag() {
211 int[] sortedSelectionIndices = sortSelctionIndicesFromTextView(mTextView);
212 mSelectionTracker.onSelectionAction(
213 sortedSelectionIndices[0], sortedSelectionIndices[1],
214 SelectionEvent.ACTION_DRAG, /* actionLabel= */ null, mTextClassification);
215 }
216
217 public void onTextChanged(int start, int end) {
218 int[] sortedSelectionIndices = sortSelctionIndices(start, end);
219 mSelectionTracker.onTextChanged(sortedSelectionIndices[0], sortedSelectionIndices[1],
220 mTextClassification);
221 }
222
223 public boolean resetSelection(int textIndex) {
224 if (mSelectionTracker.resetSelection(textIndex, mEditor)) {
225 invalidateActionModeAsync();
226 return true;
227 }
228 return false;
229 }
230
231 @Nullable
232 public TextClassification getTextClassification() {
233 return mTextClassification;
234 }
235
236 public void onDestroyActionMode() {
237 cancelSmartSelectAnimation();
238 mSelectionTracker.onSelectionDestroyed();
239 cancelAsyncTask();
240 }
241
242 public void onDraw(final Canvas canvas) {
243 if (isDrawingHighlight() && mSmartSelectSprite != null) {
244 mSmartSelectSprite.draw(canvas);
245 }
246 }
247
248 public boolean isDrawingHighlight() {
249 return mSmartSelectSprite != null && mSmartSelectSprite.isAnimationActive();
250 }
251
252 private TextClassificationConstants getTextClassificationSettings() {
253 return TextClassificationManager.getSettings(mTextView.getContext());
254 }
255
256 private void cancelAsyncTask() {
257 if (mTextClassificationAsyncTask != null) {
258 mTextClassificationAsyncTask.cancel(true);
259 mTextClassificationAsyncTask = null;
260 }
261 mTextClassification = null;
262 }
263
264 private boolean skipTextClassification() {
265 // No need to make an async call for a no-op TextClassifier.
266 final boolean noOpTextClassifier = mTextView.usesNoOpTextClassifier();
267 // Do not call the TextClassifier if there is no selection.
268 final boolean noSelection = mTextView.getSelectionEnd() == mTextView.getSelectionStart();
269 // Do not call the TextClassifier if this is a password field.
270 final boolean password = mTextView.hasPasswordTransformationMethod()
271 || TextView.isPasswordInputType(mTextView.getInputType());
272 return noOpTextClassifier || noSelection || password;
273 }
274
275 private void startLinkActionMode(@Nullable SelectionResult result) {
276 startActionMode(Editor.TextActionMode.TEXT_LINK, result);
277 }
278
279 private void startSelectionActionMode(@Nullable SelectionResult result) {
280 startActionMode(Editor.TextActionMode.SELECTION, result);
281 }
282
283 private void startActionMode(
284 @Editor.TextActionMode int actionMode, @Nullable SelectionResult result) {
285 final CharSequence text = getText(mTextView);
286 if (result != null && text instanceof Spannable
287 && (mTextView.isTextSelectable() || mTextView.isTextEditable())) {
288 // Do not change the selection if TextClassifier should be dark launched.
289 if (!getTextClassificationSettings().isModelDarkLaunchEnabled()) {
290 Selection.setSelection((Spannable) text, result.mStart, result.mEnd);
291 mTextView.invalidate();
292 }
293 mTextClassification = result.mClassification;
294 } else if (result != null && actionMode == Editor.TextActionMode.TEXT_LINK) {
295 mTextClassification = result.mClassification;
296 } else {
297 mTextClassification = null;
298 }
299 if (mEditor.startActionModeInternal(actionMode)) {
300 final SelectionModifierCursorController controller = mEditor.getSelectionController();
301 if (controller != null
302 && (mTextView.isTextSelectable() || mTextView.isTextEditable())) {
303 controller.show();
304 }
305 if (result != null) {
306 switch (actionMode) {
307 case Editor.TextActionMode.SELECTION:
308 mSelectionTracker.onSmartSelection(result);
309 break;
310 case Editor.TextActionMode.TEXT_LINK:
311 mSelectionTracker.onLinkSelected(result);
312 break;
313 default:
314 break;
315 }
316 }
317 }
318 mEditor.setRestartActionModeOnNextRefresh(false);
319 mTextClassificationAsyncTask = null;
320 }
321
322 private void startSelectionActionModeWithSmartSelectAnimation(
323 @Nullable SelectionResult result) {
324 final Layout layout = mTextView.getLayout();
325
326 final Runnable onAnimationEndCallback = () -> {
327 final SelectionResult startSelectionResult;
328 if (result != null && result.mStart >= 0 && result.mEnd <= getText(mTextView).length()
329 && result.mStart <= result.mEnd) {
330 startSelectionResult = result;
331 } else {
332 startSelectionResult = null;
333 }
334 startSelectionActionMode(startSelectionResult);
335 };
336 // TODO do not trigger the animation if the change included only non-printable characters
337 int[] sortedSelectionIndices = sortSelctionIndicesFromTextView(mTextView);
338 final boolean didSelectionChange =
339 result != null && (sortedSelectionIndices[0] != result.mStart
340 || sortedSelectionIndices[1] != result.mEnd);
341 if (!didSelectionChange) {
342 onAnimationEndCallback.run();
343 return;
344 }
345
346 final List<SmartSelectSprite.RectangleWithTextSelectionLayout> selectionRectangles =
347 convertSelectionToRectangles(layout, result.mStart, result.mEnd);
348
349 final PointF touchPoint = new PointF(
350 mEditor.getLastUpPositionX(),
351 mEditor.getLastUpPositionY());
352
353 final PointF animationStartPoint =
354 movePointInsideNearestRectangle(touchPoint, selectionRectangles,
355 SmartSelectSprite.RectangleWithTextSelectionLayout::getRectangle);
356
357 mSmartSelectSprite.startAnimation(
358 animationStartPoint,
359 selectionRectangles,
360 onAnimationEndCallback);
361 }
362
363 private List<SmartSelectSprite.RectangleWithTextSelectionLayout> convertSelectionToRectangles(
364 final Layout layout, final int start, final int end) {
365 final List<SmartSelectSprite.RectangleWithTextSelectionLayout> result = new ArrayList<>();
366
367 final Layout.SelectionRectangleConsumer consumer =
368 (left, top, right, bottom, textSelectionLayout) -> mergeRectangleIntoList(
369 result,
370 new RectF(left, top, right, bottom),
371 SmartSelectSprite.RectangleWithTextSelectionLayout::getRectangle,
372 r -> new SmartSelectSprite.RectangleWithTextSelectionLayout(r,
373 textSelectionLayout)
374 );
375
376 layout.getSelection(start, end, consumer);
377
378 result.sort(Comparator.comparing(
379 SmartSelectSprite.RectangleWithTextSelectionLayout::getRectangle,
380 SmartSelectSprite.RECTANGLE_COMPARATOR));
381
382 return result;
383 }
384
385 // TODO: Move public pure functions out of this class and make it package-private.
386 /**
387 * Merges a {@link RectF} into an existing list of any objects which contain a rectangle.
388 * While merging, this method makes sure that:
389 *
390 * <ol>
391 * <li>No rectangle is redundant (contained within a bigger rectangle)</li>
392 * <li>Rectangles of the same height and vertical position that intersect get merged</li>
393 * </ol>
394 *
395 * @param list the list of rectangles (or other rectangle containers) to merge the new
396 * rectangle into
397 * @param candidate the {@link RectF} to merge into the list
398 * @param extractor a function that can extract a {@link RectF} from an element of the given
399 * list
400 * @param packer a function that can wrap the resulting {@link RectF} into an element that
401 * the list contains
402 * @hide
403 */
404 @VisibleForTesting
405 public static <T> void mergeRectangleIntoList(final List<T> list,
406 final RectF candidate, final Function<T, RectF> extractor,
407 final Function<RectF, T> packer) {
408 if (candidate.isEmpty()) {
409 return;
410 }
411
412 final int elementCount = list.size();
413 for (int index = 0; index < elementCount; ++index) {
414 final RectF existingRectangle = extractor.apply(list.get(index));
415 if (existingRectangle.contains(candidate)) {
416 return;
417 }
418 if (candidate.contains(existingRectangle)) {
419 existingRectangle.setEmpty();
420 continue;
421 }
422
423 final boolean rectanglesContinueEachOther = candidate.left == existingRectangle.right
424 || candidate.right == existingRectangle.left;
425 final boolean canMerge = candidate.top == existingRectangle.top
426 && candidate.bottom == existingRectangle.bottom
427 && (RectF.intersects(candidate, existingRectangle)
428 || rectanglesContinueEachOther);
429
430 if (canMerge) {
431 candidate.union(existingRectangle);
432 existingRectangle.setEmpty();
433 }
434 }
435
436 for (int index = elementCount - 1; index >= 0; --index) {
437 final RectF rectangle = extractor.apply(list.get(index));
438 if (rectangle.isEmpty()) {
439 list.remove(index);
440 }
441 }
442
443 list.add(packer.apply(candidate));
444 }
445
446
447 /** @hide */
448 @VisibleForTesting
449 public static <T> PointF movePointInsideNearestRectangle(final PointF point,
450 final List<T> list, final Function<T, RectF> extractor) {
451 float bestX = -1;
452 float bestY = -1;
453 double bestDistance = Double.MAX_VALUE;
454
455 final int elementCount = list.size();
456 for (int index = 0; index < elementCount; ++index) {
457 final RectF rectangle = extractor.apply(list.get(index));
458 final float candidateY = rectangle.centerY();
459 final float candidateX;
460
461 if (point.x > rectangle.right) {
462 candidateX = rectangle.right;
463 } else if (point.x < rectangle.left) {
464 candidateX = rectangle.left;
465 } else {
466 candidateX = point.x;
467 }
468
469 final double candidateDistance = Math.pow(point.x - candidateX, 2)
470 + Math.pow(point.y - candidateY, 2);
471
472 if (candidateDistance < bestDistance) {
473 bestX = candidateX;
474 bestY = candidateY;
475 bestDistance = candidateDistance;
476 }
477 }
478
479 return new PointF(bestX, bestY);
480 }
481
482 private void invalidateActionMode(@Nullable SelectionResult result) {
483 cancelSmartSelectAnimation();
484 mTextClassification = result != null ? result.mClassification : null;
485 final ActionMode actionMode = mEditor.getTextActionMode();
486 if (actionMode != null) {
487 actionMode.invalidate();
488 }
489 final int[] sortedSelectionIndices = sortSelctionIndicesFromTextView(mTextView);
490 mSelectionTracker.onSelectionUpdated(
491 sortedSelectionIndices[0], sortedSelectionIndices[1], mTextClassification);
492 mTextClassificationAsyncTask = null;
493 }
494
495 private void resetTextClassificationHelper(int selectionStart, int selectionEnd) {
496 if (selectionStart < 0 || selectionEnd < 0) {
497 // Use selection indices
498 int[] sortedSelectionIndices = sortSelctionIndicesFromTextView(mTextView);
499 selectionStart = sortedSelectionIndices[0];
500 selectionEnd = sortedSelectionIndices[1];
501 }
502 mTextClassificationHelper.init(
503 mTextView::getTextClassificationSession,
504 getText(mTextView),
505 selectionStart, selectionEnd,
506 mTextView.getTextLocales());
507 }
508
509 private void resetTextClassificationHelper() {
510 resetTextClassificationHelper(-1, -1);
511 }
512
513 private void cancelSmartSelectAnimation() {
514 if (mSmartSelectSprite != null) {
515 mSmartSelectSprite.cancelAnimation();
516 }
517 }
518
519 /**
520 * Tracks and logs smart selection changes.
521 * It is important to trigger this object's methods at the appropriate event so that it tracks
522 * smart selection events appropriately.
523 */
524 private static final class SelectionTracker {
525
526 private final TextView mTextView;
527 private SelectionMetricsLogger mLogger;
528
529 private int mOriginalStart;
530 private int mOriginalEnd;
531 private int mSelectionStart;
532 private int mSelectionEnd;
533 private boolean mAllowReset;
534 private final LogAbandonRunnable mDelayedLogAbandon = new LogAbandonRunnable();
535
536 SelectionTracker(TextView textView) {
537 mTextView = Objects.requireNonNull(textView);
538 mLogger = new SelectionMetricsLogger(textView);
539 }
540
541 /**
542 * Called when the original selection happens, before smart selection is triggered.
543 */
544 public void onOriginalSelection(
545 CharSequence text, int selectionStart, int selectionEnd, boolean isLink) {
546 // If we abandoned a selection and created a new one very shortly after, we may still
547 // have a pending request to log ABANDON, which we flush here.
548 mDelayedLogAbandon.flush();
549
550 mOriginalStart = mSelectionStart = selectionStart;
551 mOriginalEnd = mSelectionEnd = selectionEnd;
552 mAllowReset = false;
553 maybeInvalidateLogger();
554 mLogger.logSelectionStarted(
555 mTextView.getTextClassificationSession(),
556 mTextView.getTextClassificationContext(),
557 text,
558 selectionStart,
559 isLink ? SelectionEvent.INVOCATION_LINK : SelectionEvent.INVOCATION_MANUAL);
560 }
561
562 /**
563 * Called when selection action mode is started and the results come from a classifier.
564 */
565 public void onSmartSelection(SelectionResult result) {
566 onClassifiedSelection(result);
567 mLogger.logSelectionModified(
568 result.mStart, result.mEnd, result.mClassification, result.mSelection);
569 }
570
571 /**
572 * Called when link action mode is started and the classification comes from a classifier.
573 */
574 public void onLinkSelected(SelectionResult result) {
575 onClassifiedSelection(result);
576 // TODO: log (b/70246800)
577 }
578
579 private void onClassifiedSelection(SelectionResult result) {
580 if (isSelectionStarted()) {
581 mSelectionStart = result.mStart;
582 mSelectionEnd = result.mEnd;
583 mAllowReset = mSelectionStart != mOriginalStart || mSelectionEnd != mOriginalEnd;
584 }
585 }
586
587 /**
588 * Called when selection bounds change.
589 */
590 public void onSelectionUpdated(
591 int selectionStart, int selectionEnd,
592 @Nullable TextClassification classification) {
593 if (isSelectionStarted()) {
594 mSelectionStart = selectionStart;
595 mSelectionEnd = selectionEnd;
596 mAllowReset = false;
597 mLogger.logSelectionModified(selectionStart, selectionEnd, classification, null);
598 }
599 }
600
601 /**
602 * Called when the selection action mode is destroyed.
603 */
604 public void onSelectionDestroyed() {
605 mAllowReset = false;
606 // Wait a few ms to see if the selection was destroyed because of a text change event.
607 mDelayedLogAbandon.schedule(100 /* ms */);
608 }
609
610 /**
611 * Called when an action is taken on a smart selection.
612 */
613 public void onSelectionAction(
614 int selectionStart, int selectionEnd,
615 @SelectionEvent.ActionType int action,
616 @Nullable String actionLabel,
617 @Nullable TextClassification classification) {
618 if (isSelectionStarted()) {
619 mAllowReset = false;
620 mLogger.logSelectionAction(
621 selectionStart, selectionEnd, action, actionLabel, classification);
622 }
623 }
624
625 /**
626 * Returns true if the current smart selection should be reset to normal selection based on
627 * information that has been recorded about the original selection and the smart selection.
628 * The expected UX here is to allow the user to select a word inside of the smart selection
629 * on a single tap.
630 */
631 public boolean resetSelection(int textIndex, Editor editor) {
632 final TextView textView = editor.getTextView();
633 if (isSelectionStarted()
634 && mAllowReset
635 && textIndex >= mSelectionStart && textIndex <= mSelectionEnd
636 && getText(textView) instanceof Spannable) {
637 mAllowReset = false;
638 boolean selected = editor.selectCurrentWord();
639 if (selected) {
640 final int[] sortedSelectionIndices = sortSelctionIndicesFromTextView(textView);
641 mSelectionStart = sortedSelectionIndices[0];
642 mSelectionEnd = sortedSelectionIndices[1];
643 mLogger.logSelectionAction(
644 sortedSelectionIndices[0], sortedSelectionIndices[1],
645 SelectionEvent.ACTION_RESET,
646 /* actionLabel= */ null, /* classification= */ null);
647 }
648 return selected;
649 }
650 return false;
651 }
652
653 public void onTextChanged(int start, int end, TextClassification classification) {
654 if (isSelectionStarted() && start == mSelectionStart && end == mSelectionEnd) {
655 onSelectionAction(
656 start, end, SelectionEvent.ACTION_OVERTYPE,
657 /* actionLabel= */ null, classification);
658 }
659 }
660
661 private void maybeInvalidateLogger() {
662 if (mLogger.isEditTextLogger() != mTextView.isTextEditable()) {
663 mLogger = new SelectionMetricsLogger(mTextView);
664 }
665 }
666
667 private boolean isSelectionStarted() {
668 return mSelectionStart >= 0 && mSelectionEnd >= 0 && mSelectionStart != mSelectionEnd;
669 }
670
671 /** A helper for keeping track of pending abandon logging requests. */
672 private final class LogAbandonRunnable implements Runnable {
673 private boolean mIsPending;
674
675 /** Schedules an abandon to be logged with the given delay. Flush if necessary. */
676 void schedule(int delayMillis) {
677 if (mIsPending) {
678 Log.e(LOG_TAG, "Force flushing abandon due to new scheduling request");
679 flush();
680 }
681 mIsPending = true;
682 mTextView.postDelayed(this, delayMillis);
683 }
684
685 /** If there is a pending log request, execute it now. */
686 void flush() {
687 mTextView.removeCallbacks(this);
688 run();
689 }
690
691 @Override
692 public void run() {
693 if (mIsPending) {
694 mLogger.logSelectionAction(
695 mSelectionStart, mSelectionEnd,
696 SelectionEvent.ACTION_ABANDON,
697 /* actionLabel= */ null, /* classification= */ null);
698 mSelectionStart = mSelectionEnd = -1;
699 mLogger.endTextClassificationSession();
700 mIsPending = false;
701 }
702 }
703 }
704 }
705
706 // TODO: Write tests
707 /**
708 * Metrics logging helper.
709 *
710 * This logger logs selection by word indices. The initial (start) single word selection is
711 * logged at [0, 1) -- end index is exclusive. Other word indices are logged relative to the
712 * initial single word selection.
713 * e.g. New York city, NY. Suppose the initial selection is "York" in
714 * "New York city, NY", then "York" is at [0, 1), "New" is at [-1, 0], and "city" is at [1, 2).
715 * "New York" is at [-1, 1).
716 * Part selection of a word e.g. "or" is counted as selecting the
717 * entire word i.e. equivalent to "York", and each special character is counted as a word, e.g.
718 * "," is at [2, 3). Whitespaces are ignored.
719 *
720 * NOTE that the definition of a word is defined by the TextClassifier's Logger's token
721 * iterator.
722 */
723 private static final class SelectionMetricsLogger {
724
725 private static final String LOG_TAG = "SelectionMetricsLogger";
726 private static final Pattern PATTERN_WHITESPACE = Pattern.compile("\\s+");
727
728 private final boolean mEditTextLogger;
729 private final BreakIterator mTokenIterator;
730
731 @Nullable private TextClassifier mClassificationSession;
732 @Nullable private TextClassificationContext mClassificationContext;
733
734 @Nullable private TextClassifierEvent mTranslateViewEvent;
735 @Nullable private TextClassifierEvent mTranslateClickEvent;
736
737 private int mStartIndex;
738 private String mText;
739
740 SelectionMetricsLogger(TextView textView) {
741 Objects.requireNonNull(textView);
742 mEditTextLogger = textView.isTextEditable();
743 mTokenIterator = BreakIterator.getWordInstance(textView.getTextLocale());
744 }
745
746 public void logSelectionStarted(
747 TextClassifier classificationSession,
748 TextClassificationContext classificationContext,
749 CharSequence text, int index,
750 @InvocationMethod int invocationMethod) {
751 try {
752 Objects.requireNonNull(text);
753 Preconditions.checkArgumentInRange(index, 0, text.length(), "index");
754 if (mText == null || !mText.contentEquals(text)) {
755 mText = text.toString();
756 }
757 mTokenIterator.setText(mText);
758 mStartIndex = index;
759 mClassificationSession = classificationSession;
760 mClassificationContext = classificationContext;
761 if (hasActiveClassificationSession()) {
762 mClassificationSession.onSelectionEvent(
763 SelectionEvent.createSelectionStartedEvent(invocationMethod, 0));
764 }
765 } catch (Exception e) {
766 // Avoid crashes due to logging.
767 Log.e(LOG_TAG, "" + e.getMessage(), e);
768 }
769 }
770
771 public void logSelectionModified(int start, int end,
772 @Nullable TextClassification classification, @Nullable TextSelection selection) {
773 try {
774 if (hasActiveClassificationSession()) {
775 Preconditions.checkArgumentInRange(start, 0, mText.length(), "start");
776 Preconditions.checkArgumentInRange(end, start, mText.length(), "end");
777 int[] wordIndices = getWordDelta(start, end);
778 if (selection != null) {
779 mClassificationSession.onSelectionEvent(
780 SelectionEvent.createSelectionModifiedEvent(
781 wordIndices[0], wordIndices[1], selection));
782 } else if (classification != null) {
783 mClassificationSession.onSelectionEvent(
784 SelectionEvent.createSelectionModifiedEvent(
785 wordIndices[0], wordIndices[1], classification));
786 } else {
787 mClassificationSession.onSelectionEvent(
788 SelectionEvent.createSelectionModifiedEvent(
789 wordIndices[0], wordIndices[1]));
790 }
791 maybeGenerateTranslateViewEvent(classification);
792 }
793 } catch (Exception e) {
794 // Avoid crashes due to logging.
795 Log.e(LOG_TAG, "" + e.getMessage(), e);
796 }
797 }
798
799 public void logSelectionAction(
800 int start, int end,
801 @SelectionEvent.ActionType int action,
802 @Nullable String actionLabel,
803 @Nullable TextClassification classification) {
804 try {
805 if (hasActiveClassificationSession()) {
806 Preconditions.checkArgumentInRange(start, 0, mText.length(), "start");
807 Preconditions.checkArgumentInRange(end, start, mText.length(), "end");
808 int[] wordIndices = getWordDelta(start, end);
809 if (classification != null) {
810 mClassificationSession.onSelectionEvent(
811 SelectionEvent.createSelectionActionEvent(
812 wordIndices[0], wordIndices[1], action,
813 classification));
814 } else {
815 mClassificationSession.onSelectionEvent(
816 SelectionEvent.createSelectionActionEvent(
817 wordIndices[0], wordIndices[1], action));
818 }
819
820 maybeGenerateTranslateClickEvent(classification, actionLabel);
821
822 if (SelectionEvent.isTerminal(action)) {
823 endTextClassificationSession();
824 }
825 }
826 } catch (Exception e) {
827 // Avoid crashes due to logging.
828 Log.e(LOG_TAG, "" + e.getMessage(), e);
829 }
830 }
831
832 public boolean isEditTextLogger() {
833 return mEditTextLogger;
834 }
835
836 public void endTextClassificationSession() {
837 if (hasActiveClassificationSession()) {
838 maybeReportTranslateEvents();
839 mClassificationSession.destroy();
840 }
841 }
842
843 private boolean hasActiveClassificationSession() {
844 return mClassificationSession != null && !mClassificationSession.isDestroyed();
845 }
846
847 private int[] getWordDelta(int start, int end) {
848 int[] wordIndices = new int[2];
849
850 if (start == mStartIndex) {
851 wordIndices[0] = 0;
852 } else if (start < mStartIndex) {
853 wordIndices[0] = -countWordsForward(start);
854 } else { // start > mStartIndex
855 wordIndices[0] = countWordsBackward(start);
856
857 // For the selection start index, avoid counting a partial word backwards.
858 if (!mTokenIterator.isBoundary(start)
859 && !isWhitespace(
860 mTokenIterator.preceding(start),
861 mTokenIterator.following(start))) {
862 // We counted a partial word. Remove it.
863 wordIndices[0]--;
864 }
865 }
866
867 if (end == mStartIndex) {
868 wordIndices[1] = 0;
869 } else if (end < mStartIndex) {
870 wordIndices[1] = -countWordsForward(end);
871 } else { // end > mStartIndex
872 wordIndices[1] = countWordsBackward(end);
873 }
874
875 return wordIndices;
876 }
877
878 private int countWordsBackward(int from) {
879 Preconditions.checkArgument(from >= mStartIndex);
880 int wordCount = 0;
881 int offset = from;
882 while (offset > mStartIndex) {
883 int start = mTokenIterator.preceding(offset);
884 if (!isWhitespace(start, offset)) {
885 wordCount++;
886 }
887 offset = start;
888 }
889 return wordCount;
890 }
891
892 private int countWordsForward(int from) {
893 Preconditions.checkArgument(from <= mStartIndex);
894 int wordCount = 0;
895 int offset = from;
896 while (offset < mStartIndex) {
897 int end = mTokenIterator.following(offset);
898 if (!isWhitespace(offset, end)) {
899 wordCount++;
900 }
901 offset = end;
902 }
903 return wordCount;
904 }
905
906 private boolean isWhitespace(int start, int end) {
907 return PATTERN_WHITESPACE.matcher(mText.substring(start, end)).matches();
908 }
909
910 private void maybeGenerateTranslateViewEvent(@Nullable TextClassification classification) {
911 if (classification != null) {
912 final TextClassifierEvent event = generateTranslateEvent(
913 TextClassifierEvent.TYPE_ACTIONS_SHOWN,
914 classification, mClassificationContext, /* actionLabel= */null);
915 mTranslateViewEvent = (event != null) ? event : mTranslateViewEvent;
916 }
917 }
918
919 private void maybeGenerateTranslateClickEvent(
920 @Nullable TextClassification classification, String actionLabel) {
921 if (classification != null) {
922 mTranslateClickEvent = generateTranslateEvent(
923 TextClassifierEvent.TYPE_SMART_ACTION,
924 classification, mClassificationContext, actionLabel);
925 }
926 }
927
928 private void maybeReportTranslateEvents() {
929 // Translate view and click events should only be logged once per selection session.
930 if (mTranslateViewEvent != null) {
931 mClassificationSession.onTextClassifierEvent(mTranslateViewEvent);
932 mTranslateViewEvent = null;
933 }
934 if (mTranslateClickEvent != null) {
935 mClassificationSession.onTextClassifierEvent(mTranslateClickEvent);
936 mTranslateClickEvent = null;
937 }
938 }
939
940 @Nullable
941 private static TextClassifierEvent generateTranslateEvent(
942 int eventType, TextClassification classification,
943 TextClassificationContext classificationContext, @Nullable String actionLabel) {
944
945 // The platform attempts to log "views" and "clicks" of the "Translate" action.
946 // Views are logged if a user is presented with the translate action during a selection
947 // session.
948 // Clicks are logged if the user clicks on the translate action.
949 // The index of the translate action is also logged to indicate whether it might have
950 // been in the main panel or overflow panel of the selection toolbar.
951 // NOTE that the "views" metric may be flawed if a TextView removes the translate menu
952 // item via a custom action mode callback or does not show a selection menu item.
953
954 final RemoteAction translateAction = ExtrasUtils.findTranslateAction(classification);
955 if (translateAction == null) {
956 // No translate action present. Nothing to log. Exit.
957 return null;
958 }
959
960 if (eventType == TextClassifierEvent.TYPE_SMART_ACTION
961 && !translateAction.getTitle().toString().equals(actionLabel)) {
962 // Clicked action is not a translate action. Nothing to log. Exit.
963 // Note that we don't expect an actionLabel for "view" events.
964 return null;
965 }
966
967 final Bundle foreignLanguageExtra = ExtrasUtils.getForeignLanguageExtra(classification);
968 final String language = ExtrasUtils.getEntityType(foreignLanguageExtra);
969 final float score = ExtrasUtils.getScore(foreignLanguageExtra);
970 final String model = ExtrasUtils.getModelName(foreignLanguageExtra);
971 return new TextClassifierEvent.LanguageDetectionEvent.Builder(eventType)
972 .setEventContext(classificationContext)
973 .setResultId(classification.getId())
974 .setEntityTypes(language)
975 .setScores(score)
976 .setActionIndices(classification.getActions().indexOf(translateAction))
977 .setModelName(model)
978 .build();
979 }
980 }
981
982 /**
983 * AsyncTask for running a query on a background thread and returning the result on the
984 * UiThread. The AsyncTask times out after a specified time, returning a null result if the
985 * query has not yet returned.
986 */
987 private static final class TextClassificationAsyncTask
988 extends AsyncTask<Void, Void, SelectionResult> {
989
990 private final int mTimeOutDuration;
991 private final Supplier<SelectionResult> mSelectionResultSupplier;
992 private final Consumer<SelectionResult> mSelectionResultCallback;
993 private final Supplier<SelectionResult> mTimeOutResultSupplier;
994 private final TextView mTextView;
995 private final String mOriginalText;
996
997 /**
998 * @param textView the TextView
999 * @param timeOut time in milliseconds to timeout the query if it has not completed
1000 * @param selectionResultSupplier fetches the selection results. Runs on a background thread
1001 * @param selectionResultCallback receives the selection results. Runs on the UiThread
1002 * @param timeOutResultSupplier default result if the task times out
1003 */
1004 TextClassificationAsyncTask(
1005 @NonNull TextView textView, int timeOut,
1006 @NonNull Supplier<SelectionResult> selectionResultSupplier,
1007 @NonNull Consumer<SelectionResult> selectionResultCallback,
1008 @NonNull Supplier<SelectionResult> timeOutResultSupplier) {
1009 super(textView != null ? textView.getHandler() : null);
1010 mTextView = Objects.requireNonNull(textView);
1011 mTimeOutDuration = timeOut;
1012 mSelectionResultSupplier = Objects.requireNonNull(selectionResultSupplier);
1013 mSelectionResultCallback = Objects.requireNonNull(selectionResultCallback);
1014 mTimeOutResultSupplier = Objects.requireNonNull(timeOutResultSupplier);
1015 // Make a copy of the original text.
1016 mOriginalText = getText(mTextView).toString();
1017 }
1018
1019 @Override
1020 @WorkerThread
1021 protected SelectionResult doInBackground(Void... params) {
1022 final Runnable onTimeOut = this::onTimeOut;
1023 mTextView.postDelayed(onTimeOut, mTimeOutDuration);
1024 final SelectionResult result = mSelectionResultSupplier.get();
1025 mTextView.removeCallbacks(onTimeOut);
1026 return result;
1027 }
1028
1029 @Override
1030 @UiThread
1031 protected void onPostExecute(SelectionResult result) {
1032 result = TextUtils.equals(mOriginalText, getText(mTextView)) ? result : null;
1033 mSelectionResultCallback.accept(result);
1034 }
1035
1036 private void onTimeOut() {
1037 Log.d(LOG_TAG, "Timeout in TextClassificationAsyncTask");
1038 if (getStatus() == Status.RUNNING) {
1039 onPostExecute(mTimeOutResultSupplier.get());
1040 }
1041 cancel(true);
1042 }
1043 }
1044
1045 /**
1046 * Helper class for querying the TextClassifier.
1047 * It trims text so that only text necessary to provide context of the selected text is
1048 * sent to the TextClassifier.
1049 */
1050 private static final class TextClassificationHelper {
1051
1052 private static final int TRIM_DELTA = 120; // characters
1053
1054 private final Context mContext;
1055 private Supplier<TextClassifier> mTextClassifier;
1056
1057 /** The original TextView text. **/
1058 private String mText;
1059 /** Start index relative to mText. */
1060 private int mSelectionStart;
1061 /** End index relative to mText. */
1062 private int mSelectionEnd;
1063
1064 @Nullable
1065 private LocaleList mDefaultLocales;
1066
1067 /** Trimmed text starting from mTrimStart in mText. */
1068 private CharSequence mTrimmedText;
1069 /** Index indicating the start of mTrimmedText in mText. */
1070 private int mTrimStart;
1071 /** Start index relative to mTrimmedText */
1072 private int mRelativeStart;
1073 /** End index relative to mTrimmedText */
1074 private int mRelativeEnd;
1075
1076 /** Information about the last classified text to avoid re-running a query. */
1077 private CharSequence mLastClassificationText;
1078 private int mLastClassificationSelectionStart;
1079 private int mLastClassificationSelectionEnd;
1080 private LocaleList mLastClassificationLocales;
1081 private SelectionResult mLastClassificationResult;
1082
1083 /** Whether the TextClassifier has been initialized. */
1084 private boolean mHot;
1085
1086 TextClassificationHelper(Context context, Supplier<TextClassifier> textClassifier,
1087 CharSequence text, int selectionStart, int selectionEnd, LocaleList locales) {
1088 init(textClassifier, text, selectionStart, selectionEnd, locales);
1089 mContext = Objects.requireNonNull(context);
1090 }
1091
1092 @UiThread
1093 public void init(Supplier<TextClassifier> textClassifier, CharSequence text,
1094 int selectionStart, int selectionEnd, LocaleList locales) {
1095 mTextClassifier = Objects.requireNonNull(textClassifier);
1096 mText = Objects.requireNonNull(text).toString();
1097 mLastClassificationText = null; // invalidate.
1098 Preconditions.checkArgument(selectionEnd > selectionStart);
1099 mSelectionStart = selectionStart;
1100 mSelectionEnd = selectionEnd;
1101 mDefaultLocales = locales;
1102 }
1103
1104 @WorkerThread
1105 public SelectionResult classifyText() {
1106 mHot = true;
1107 return performClassification(null /* selection */);
1108 }
1109
1110 @WorkerThread
1111 public SelectionResult suggestSelection() {
1112 mHot = true;
1113 trimText();
1114 final TextSelection selection;
1115 if (mContext.getApplicationInfo().targetSdkVersion >= Build.VERSION_CODES.P) {
1116 final TextSelection.Request request = new TextSelection.Request.Builder(
1117 mTrimmedText, mRelativeStart, mRelativeEnd)
1118 .setDefaultLocales(mDefaultLocales)
1119 .setDarkLaunchAllowed(true)
1120 .build();
1121 selection = mTextClassifier.get().suggestSelection(request);
1122 } else {
1123 // Use old APIs.
1124 selection = mTextClassifier.get().suggestSelection(
1125 mTrimmedText, mRelativeStart, mRelativeEnd, mDefaultLocales);
1126 }
1127 // Do not classify new selection boundaries if TextClassifier should be dark launched.
1128 if (!isDarkLaunchEnabled()) {
1129 mSelectionStart = Math.max(0, selection.getSelectionStartIndex() + mTrimStart);
1130 mSelectionEnd = Math.min(
1131 mText.length(), selection.getSelectionEndIndex() + mTrimStart);
1132 }
1133 return performClassification(selection);
1134 }
1135
1136 public SelectionResult getOriginalSelection() {
1137 return new SelectionResult(mSelectionStart, mSelectionEnd, null, null);
1138 }
1139
1140 /**
1141 * Maximum time (in milliseconds) to wait for a textclassifier result before timing out.
1142 */
1143 // TODO: Consider making this a ViewConfiguration.
1144 public int getTimeoutDuration() {
1145 if (mHot) {
1146 return 200;
1147 } else {
1148 // Return a slightly larger number than usual when the TextClassifier is first
1149 // initialized. Initialization would usually take longer than subsequent calls to
1150 // the TextClassifier. The impact of this on the UI is that we do not show the
1151 // selection handles or toolbar until after this timeout.
1152 return 500;
1153 }
1154 }
1155
1156 private boolean isDarkLaunchEnabled() {
1157 return TextClassificationManager.getSettings(mContext).isModelDarkLaunchEnabled();
1158 }
1159
1160 private SelectionResult performClassification(@Nullable TextSelection selection) {
1161 if (!Objects.equals(mText, mLastClassificationText)
1162 || mSelectionStart != mLastClassificationSelectionStart
1163 || mSelectionEnd != mLastClassificationSelectionEnd
1164 || !Objects.equals(mDefaultLocales, mLastClassificationLocales)) {
1165
1166 mLastClassificationText = mText;
1167 mLastClassificationSelectionStart = mSelectionStart;
1168 mLastClassificationSelectionEnd = mSelectionEnd;
1169 mLastClassificationLocales = mDefaultLocales;
1170
1171 trimText();
1172 final TextClassification classification;
1173 if (Linkify.containsUnsupportedCharacters(mText)) {
1174 // Do not show smart actions for text containing unsupported characters.
1175 android.util.EventLog.writeEvent(0x534e4554, "116321860", -1, "");
1176 classification = TextClassification.EMPTY;
1177 } else if (mContext.getApplicationInfo().targetSdkVersion
1178 >= Build.VERSION_CODES.P) {
1179 final TextClassification.Request request =
1180 new TextClassification.Request.Builder(
1181 mTrimmedText, mRelativeStart, mRelativeEnd)
1182 .setDefaultLocales(mDefaultLocales)
1183 .build();
1184 classification = mTextClassifier.get().classifyText(request);
1185 } else {
1186 // Use old APIs.
1187 classification = mTextClassifier.get().classifyText(
1188 mTrimmedText, mRelativeStart, mRelativeEnd, mDefaultLocales);
1189 }
1190 mLastClassificationResult = new SelectionResult(
1191 mSelectionStart, mSelectionEnd, classification, selection);
1192
1193 }
1194 return mLastClassificationResult;
1195 }
1196
1197 private void trimText() {
1198 mTrimStart = Math.max(0, mSelectionStart - TRIM_DELTA);
1199 final int referenceEnd = Math.min(mText.length(), mSelectionEnd + TRIM_DELTA);
1200 mTrimmedText = mText.subSequence(mTrimStart, referenceEnd);
1201 mRelativeStart = mSelectionStart - mTrimStart;
1202 mRelativeEnd = mSelectionEnd - mTrimStart;
1203 }
1204 }
1205
1206 /**
1207 * Selection result.
1208 */
1209 private static final class SelectionResult {
1210 private final int mStart;
1211 private final int mEnd;
1212 @Nullable private final TextClassification mClassification;
1213 @Nullable private final TextSelection mSelection;
1214
1215 SelectionResult(int start, int end,
1216 @Nullable TextClassification classification, @Nullable TextSelection selection) {
1217 int[] sortedIndices = sortSelctionIndices(start, end);
1218 mStart = sortedIndices[0];
1219 mEnd = sortedIndices[1];
1220 mClassification = classification;
1221 mSelection = selection;
1222 }
1223 }
1224
1225 @SelectionEvent.ActionType
1226 private static int getActionType(int menuItemId) {
1227 switch (menuItemId) {
1228 case TextView.ID_SELECT_ALL:
1229 return SelectionEvent.ACTION_SELECT_ALL;
1230 case TextView.ID_CUT:
1231 return SelectionEvent.ACTION_CUT;
1232 case TextView.ID_COPY:
1233 return SelectionEvent.ACTION_COPY;
1234 case TextView.ID_PASTE: // fall through
1235 case TextView.ID_PASTE_AS_PLAIN_TEXT:
1236 return SelectionEvent.ACTION_PASTE;
1237 case TextView.ID_SHARE:
1238 return SelectionEvent.ACTION_SHARE;
1239 case TextView.ID_ASSIST:
1240 return SelectionEvent.ACTION_SMART_SHARE;
1241 default:
1242 return SelectionEvent.ACTION_OTHER;
1243 }
1244 }
1245
1246 private static CharSequence getText(TextView textView) {
1247 // Extracts the textView's text.
1248 // TODO: Investigate why/when TextView.getText() is null.
1249 final CharSequence text = textView.getText();
1250 if (text != null) {
1251 return text;
1252 }
1253 return "";
1254 }
1255}