From 96145452a4d5f447becd1c8679a509fb1d9dd563 Mon Sep 17 00:00:00 2001 From: 4di-hello <4di-hello@users.noreply.github.com> Date: Wed, 14 Oct 2026 22:38:39 +0200 Subject: [PATCH 1/2] Add "Adapt to my typing": adaptive key zones learned from the user's touches One switch in Text correction, off by default. When on, the keyboard learns where the user touches each letter key, from the words finally committed (autocorrected word, picked suggestion, reverted correction, or a letter retyped after deleting a neighboring one). Once a key has 30 touches, a touch near the border to a neighboring key is assigned to the key the user most likely meant, and the dictionary gets the corrected coordinates too. The shift is capped to a third of a key. With the switch off, nothing is recorded and key detection runs the unchanged code path. Never active in password fields or incognito mode. All logic is in the heatmap package (TypingAdaptation, TypingHeatmap, KeyboardGeometry); InputLogic, KeyDetector and PointerTracker only call single hooks. Data is stored locally in no-backup storage and can be included in the keyboard's own backup. A statistics screen shows the heatmap, the most frequent misses and the weekly miss rate. Discussed in #676. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01VuiaMhLXwhErf9shhrr1uD --- .../keyboard/keyboard/KeyDetector.kt | 4 +- .../keyboard/keyboard/PointerTracker.kt | 6 +- .../latin/heatmap/KeyboardGeometry.kt | 72 ++ .../latin/heatmap/TypingAdaptation.kt | 143 ++++ .../keyboard/latin/heatmap/TypingHeatmap.kt | 789 ++++++++++++++++++ .../keyboard/latin/inputlogic/InputLogic.kt | 9 + .../keyboard/latin/settings/Defaults.kt | 1 + .../keyboard/latin/settings/Settings.kt | 1 + .../keyboard/latin/settings/SettingsValues.kt | 2 + .../keyboard/settings/SettingsContainer.kt | 2 + .../keyboard/settings/SettingsNavHost.kt | 4 + .../preferences/BackupRestorePreference.kt | 28 +- .../settings/screens/TextCorrectionScreen.kt | 11 +- .../settings/screens/TypingHeatmapScreen.kt | 315 +++++++ app/src/main/res/values-de/strings.xml | 26 + app/src/main/res/values/strings.xml | 27 + .../latin/heatmap/TypingHeatmapTest.kt | 217 +++++ 17 files changed, 1651 insertions(+), 6 deletions(-) create mode 100644 app/src/main/java/helium314/keyboard/latin/heatmap/KeyboardGeometry.kt create mode 100644 app/src/main/java/helium314/keyboard/latin/heatmap/TypingAdaptation.kt create mode 100644 app/src/main/java/helium314/keyboard/latin/heatmap/TypingHeatmap.kt create mode 100644 app/src/main/java/helium314/keyboard/settings/screens/TypingHeatmapScreen.kt create mode 100644 app/src/test/java/helium314/keyboard/latin/heatmap/TypingHeatmapTest.kt diff --git a/app/src/main/java/helium314/keyboard/keyboard/KeyDetector.kt b/app/src/main/java/helium314/keyboard/keyboard/KeyDetector.kt index 07293a14a..49c878fce 100644 --- a/app/src/main/java/helium314/keyboard/keyboard/KeyDetector.kt +++ b/app/src/main/java/helium314/keyboard/keyboard/KeyDetector.kt @@ -6,6 +6,8 @@ package helium314.keyboard.keyboard +import helium314.keyboard.latin.heatmap.TouchAdaptation + /** * This class handles key detection. */ @@ -110,6 +112,6 @@ open class KeyDetector { primaryKey = key } } - return primaryKey + return TouchAdaptation.adaptedKey(keyboard, touchX, touchY, primaryKey) ?: primaryKey } } diff --git a/app/src/main/java/helium314/keyboard/keyboard/PointerTracker.kt b/app/src/main/java/helium314/keyboard/keyboard/PointerTracker.kt index 9931008c1..2d403c4e6 100644 --- a/app/src/main/java/helium314/keyboard/keyboard/PointerTracker.kt +++ b/app/src/main/java/helium314/keyboard/keyboard/PointerTracker.kt @@ -30,6 +30,7 @@ import helium314.keyboard.latin.common.Constants import helium314.keyboard.latin.common.CoordinateUtils import helium314.keyboard.latin.common.InputPointers import helium314.keyboard.latin.define.DebugFlags +import helium314.keyboard.latin.heatmap.TouchAdaptation import helium314.keyboard.latin.settings.Settings import helium314.keyboard.latin.settings.SettingsValues import helium314.keyboard.latin.utils.Log @@ -195,7 +196,10 @@ class PointerTracker private constructor( sListener.onTextInput(key.outputText) } else if (code != KeyCode.NOT_SPECIFIED) { if (mKeyboard?.hasProximityCharsCorrection(code) == true) { - sListener.onCodeInput(code, x, y, isKeyRepeat) + val adjusted = mKeyboard?.let { + TouchAdaptation.adjustedTouch(it, key, code, x, y, mKeyDetector.getTouchX(0), mKeyDetector.getTouchY(0)) + } ?: intArrayOf(x, y) + sListener.onCodeInput(code, adjusted[0], adjusted[1], isKeyRepeat) } else { sListener.onCodeInput(code, Constants.NOT_A_COORDINATE, Constants.NOT_A_COORDINATE, isKeyRepeat) } diff --git a/app/src/main/java/helium314/keyboard/latin/heatmap/KeyboardGeometry.kt b/app/src/main/java/helium314/keyboard/latin/heatmap/KeyboardGeometry.kt new file mode 100644 index 000000000..49f5b9b66 --- /dev/null +++ b/app/src/main/java/helium314/keyboard/latin/heatmap/KeyboardGeometry.kt @@ -0,0 +1,72 @@ +// SPDX-License-Identifier: GPL-3.0-only +package helium314.keyboard.latin.heatmap + +import android.content.res.Configuration +import android.content.res.Resources +import helium314.keyboard.keyboard.Keyboard +import helium314.keyboard.keyboard.internal.keyboard_parser.floris.KeyCode +import helium314.keyboard.latin.LatinIME +import helium314.keyboard.latin.common.Constants +import helium314.keyboard.latin.settings.Settings + +/** [TypingHeatmap.KeyGeometry] for a [Keyboard] */ +class KeyboardGeometry(private val keyboard: Keyboard, landscape: Boolean = isLandscape()) : TypingHeatmap.KeyGeometry { + override val layoutId = layoutId(keyboard, landscape) + + companion object { + fun isLandscape() = Resources.getSystem().configuration.orientation == Configuration.ORIENTATION_LANDSCAPE + + /** floating and one-handed keyboards are held differently, so they get their own data */ + private fun modeSuffix(): String = when { + LatinIME.getInstance()?.floatingKeyboardManager?.isFloating == true -> " (floating)" + Settings.getValues().mOneHandedModeEnabled -> " (one-handed)" + else -> "" + } + + private class IdCache(val keyboard: Keyboard, val landscape: Boolean, val mode: String, val id: String) + @Volatile private var idCache: IdCache? = null + + /** Id for the data of this keyboard; cached, as it is used for every touch event */ + fun layoutId(keyboard: Keyboard, landscape: Boolean = isLandscape()): String { + val mode = modeSuffix() + val cache = idCache + if (cache != null && cache.keyboard === keyboard && cache.landscape == landscape && cache.mode == mode) + return cache.id + val id = keyboard.mId.mSubtype.mainLayoutName + (if (landscape) " (landscape)" else "") + mode + idCache = IdCache(keyboard, landscape, mode, id) + return id + } + } + + override val source: Any get() = keyboard + + override val aspectRatio: Float + get() = if (keyboard.mOccupiedWidth > 0) keyboard.mOccupiedHeight.toFloat() / keyboard.mOccupiedWidth else 0.35f + + override fun keyRect(codePoint: Int): IntArray? { + val key = keyboard.getKey(codePoint) + ?: keyboard.getKey(Character.toUpperCase(codePoint)) + ?: keyboard.getKey(Character.toTitleCase(codePoint)) + ?: return null + return intArrayOf(key.x, key.y, key.width, key.height) + } + + override fun snapshot(): List { + val w = keyboard.mOccupiedWidth.toFloat() + val h = keyboard.mOccupiedHeight.toFloat() + if (w <= 0 || h <= 0) return emptyList() + return keyboard.sortedKeys.filter { it.width > 0 && it.height > 0 && !it.isSpacer }.map { key -> + val label = when (key.code) { + Constants.CODE_SPACE -> "␣" + Constants.CODE_ENTER -> "⏎" + KeyCode.DELETE -> "⌫" + KeyCode.SHIFT -> "⇧" + else -> key.label ?: "" + } + TypingHeatmap.KeyShape( + Character.toLowerCase(key.code), label.lowercase(), + key.x / w, key.y / h, key.width / w, key.height / h + ) + } + } +} diff --git a/app/src/main/java/helium314/keyboard/latin/heatmap/TypingAdaptation.kt b/app/src/main/java/helium314/keyboard/latin/heatmap/TypingAdaptation.kt new file mode 100644 index 000000000..0115461bf --- /dev/null +++ b/app/src/main/java/helium314/keyboard/latin/heatmap/TypingAdaptation.kt @@ -0,0 +1,143 @@ +// SPDX-License-Identifier: GPL-3.0-only +package helium314.keyboard.latin.heatmap + +import helium314.keyboard.event.Event +import helium314.keyboard.keyboard.Key +import helium314.keyboard.keyboard.Keyboard +import helium314.keyboard.keyboard.internal.keyboard_parser.floris.KeyCode +import helium314.keyboard.latin.LatinIME +import helium314.keyboard.latin.WordComposer +import helium314.keyboard.latin.common.StringUtils +import helium314.keyboard.latin.settings.Settings +import helium314.keyboard.latin.settings.SettingsValues + +/** + * Hooks for "Adapt to my typing", so the core classes only need single calls. + * + * [TypingRecorder] is used by InputLogic: it learns where the user touches each key, from the + * words finally committed. [TouchAdaptation] is used by KeyDetector and PointerTracker: it picks + * the key the user most likely meant near key borders. With the setting off, nothing is recorded + * and key detection is unchanged. The data is in [TypingHeatmap]. + */ +class TypingRecorder(private val latinIME: LatinIME, private val wordComposer: WordComposer) { + // last letter of the composing word deleted with a single backspace press, for learning from self-corrections + private var deletedLetterIndex = -1 + private var deletedLetter = 0 + private var deletedLetterX = 0 + private var deletedLetterY = 0 + + fun onStartInput(settingsValues: SettingsValues) { + deletedLetterIndex = -1 + // load the data in background, so it is ready for key detection; nothing at all when off + if (settingsValues.mAdaptToTyping) TypingHeatmap.init(latinIME) + } + + fun onFinishInput() = TypingHeatmap.onFinishInput() + + /** Called for every event before the word composer processes it */ + fun onEvent(event: Event, settingsValues: SettingsValues) { + if (event.keyCode != KeyCode.DELETE) TypingHeatmap.finalizePending() + trackLetterRetype(event, settingsValues) + } + + /** + * Unless this backspace reverts the autocorrection ([onCommitReverted]), the user edits the + * committed word, so we don't know which word was finally used. + */ + fun onBackspace() = TypingHeatmap.onBackspace() + + fun onCommitReverted() = TypingHeatmap.onCommitReverted() + + /** A word is committed: remember where its keys were touched. Call before the word composer is reset. */ + fun onWordCommitted(settingsValues: SettingsValues, finalWord: String) { + if (!isRecording(settingsValues)) return + val keyboard = alphabetKeyboard() ?: return + val pointers = wordComposer.getInputPointers() + val size = pointers.pointerSize + val geometry = KeyboardGeometry(keyboard, KeyboardGeometry.isLandscape()) + TypingHeatmap.onWordCommitted( + latinIME, geometry, wordComposer.getTypedWord(), finalWord, + pointers.xCoordinates.copyOf(size), pointers.yCoordinates.copyOf(size), isAdapted(geometry) + ) + } + + /** + * If the user deletes the last letter and types a neighboring letter instead, the touch of the + * deleted letter was meant for the new one. See [TypingHeatmap.onLetterRetyped]. + */ + private fun trackLetterRetype(event: Event, settingsValues: SettingsValues) { + val deletedIndex = deletedLetterIndex + deletedLetterIndex = -1 + if (!isRecording(settingsValues)) return + if (event.keyCode == KeyCode.DELETE) { + if (event.isKeyRepeat || !wordComposer.isComposingWord() || wordComposer.isCursorFrontOrMiddleOfComposingWord()) return + val codePoints = StringUtils.toCodePointArray(wordComposer.getTypedWord()) + val index = codePoints.size - 1 + val pointers = wordComposer.getInputPointers() + if (index < 0 || index >= pointers.pointerSize) return + deletedLetterIndex = index + deletedLetter = codePoints[index] + deletedLetterX = pointers.xCoordinates[index] + deletedLetterY = pointers.yCoordinates[index] + return + } + if (deletedIndex < 0) return + val codePoint = event.codePoint + if (codePoint <= 0 || !Character.isLetter(codePoint) || event.x < 0 || event.y < 0) return + if (wordComposer.size() != deletedIndex) return + val keyboard = alphabetKeyboard() ?: return + val geometry = KeyboardGeometry(keyboard) + TypingHeatmap.onLetterRetyped( + latinIME, geometry, deletedLetter, deletedLetterX, deletedLetterY, codePoint, isAdapted(geometry) + ) + } + + // no real touch positions for gestures or words restarted from existing text + private fun isRecording(settingsValues: SettingsValues) = settingsValues.mAdaptToTyping + && !settingsValues.mIncognitoModeEnabled && !wordComposer.isBatchMode() && !wordComposer.isResumed() + + private fun alphabetKeyboard(): Keyboard? = + latinIME.keyboardSwitcher.keyboard?.takeIf { it.mId.isAlphabetKeyboard } + + /** whether adaptation was already active for this layout, for comparing misses before and after */ + private fun isAdapted(geometry: KeyboardGeometry) = + TypingHeatmap.getOffsets(null, geometry.layoutId, TypingHeatmap.DEFAULT_STRENGTH).isNotEmpty() +} + +object TouchAdaptation { + private fun isActive(keyboard: Keyboard, code: Int): Boolean { + val settingsValues = Settings.getValues() + return settingsValues.mAdaptToTyping && !settingsValues.mInputAttributes.mIsPasswordField + && Character.isLetter(code) && keyboard.mId.isAlphabetKeyboard + } + + private fun offsets(keyboard: Keyboard) = + TypingHeatmap.getOffsets(null, KeyboardGeometry.layoutId(keyboard), TypingHeatmap.DEFAULT_STRENGTH) + + /** + * Key chosen with the learned typing offsets of the user, if it differs from [primaryKey]. + * Only letter keys on alphabet keyboards are affected, never in password fields. + * [touchX] and [touchY] are keyboard coordinates, like the learned data. + */ + fun adaptedKey(keyboard: Keyboard, touchX: Int, touchY: Int, primaryKey: Key?): Key? { + if (primaryKey == null || !isActive(keyboard, primaryKey.code)) return null + val offsets = offsets(keyboard) + if (offsets.isEmpty()) return null + val keys = keyboard.getNearestKeys(touchX, touchY).filter { Character.isLetter(it.code) && it.width > 0 && it.height > 0 } + val candidates = keys.map { + TypingHeatmap.Candidate(Character.toLowerCase(it.code), it.x.toFloat(), it.y.toFloat(), it.width.toFloat(), it.height.toFloat()) + } + val code = TypingHeatmap.adaptedKeyCode(touchX.toFloat(), touchY.toFloat(), candidates, offsets) ?: return null + return keys.firstOrNull { Character.toLowerCase(it.code) == code } + } + + /** + * Touch moved by the learned offset of the key, so the dictionary sees a typical touch. + * [x] and [y] are view coordinates, [correctionX] and [correctionY] convert them to keyboard coordinates. + */ + fun adjustedTouch(keyboard: Keyboard, key: Key, code: Int, x: Int, y: Int, correctionX: Int, correctionY: Int): IntArray { + if (!isActive(keyboard, code)) return intArrayOf(x, y) + val offset = offsets(keyboard)[Character.toLowerCase(code)] ?: return intArrayOf(x, y) + return TypingHeatmap.adjustForDictionary(x, y, correctionX, correctionY, key.width, key.height, offset) + } +} diff --git a/app/src/main/java/helium314/keyboard/latin/heatmap/TypingHeatmap.kt b/app/src/main/java/helium314/keyboard/latin/heatmap/TypingHeatmap.kt new file mode 100644 index 000000000..2a9adee06 --- /dev/null +++ b/app/src/main/java/helium314/keyboard/latin/heatmap/TypingHeatmap.kt @@ -0,0 +1,789 @@ +// SPDX-License-Identifier: GPL-3.0-only +package helium314.keyboard.latin.heatmap + +import android.content.Context +import android.os.Build +import helium314.keyboard.compat.isUserLocked +import helium314.keyboard.latin.common.StringUtils +import helium314.keyboard.latin.utils.DeviceProtectedUtils +import helium314.keyboard.latin.utils.Log +import org.json.JSONArray +import org.json.JSONObject +import java.io.File +import java.io.InputStream +import java.util.concurrent.Executors +import kotlin.math.abs +import kotlin.math.hypot +import kotlin.math.roundToLong + +/** + * Learns where the user actually touches keys, and adapts key detection to it. + * + * The intended key is taken from the text that was finally used: the committed word (after + * autocorrection or a picked suggestion), the typed word if the autocorrection was reverted, + * or the retyped letter if the user deleted a letter and typed a neighboring one instead. + * Touches are stored per key, relative to the key center and in units of key width / height. + * + * From the touches, each key gets an average offset, which is used to decide which key a touch + * was meant for ([adaptedKeyCode]) and to move the touch coordinates passed to the dictionary + * ([adjustForDictionary]). Learning always uses the raw touch positions, so adaptation does not + * feed back into what is learned. All coordinates are keyboard coordinates (view coordinates + * corrected by the key detector, like the coordinates in the word composer). + * + * Privacy and security: + * - data stays on the device: no network access, stored in credential encrypted storage in the + * no-backup folder (excluded from Android cloud backup and device-to-device transfer), only + * available after the user unlocked the device + * - samples are stored grouped by key without any order across keys, so the typed text cannot + * be reconstructed from the stored data + * - nothing is recorded in password fields and incognito mode, and adaptation is not used in + * password fields + * - stored data and restored backups are validated, broken data is ignored + */ +object TypingHeatmap { + private const val TAG = "TypingHeatmap" + const val FILE_NAME = "typing_heatmap.json" + private const val MAX_SAMPLES_PER_KEY = 400 + /** Touches further away from the intended key center than this (in key sizes) are ignored */ + private const val MAX_OFFSET = 1.6f + private const val SAVE_EVERY_SAMPLES = 40 + /** Limits for reading stored or restored data */ + /** must be larger than the largest file we write: 16 layouts * ~40 keys * 400 samples * ~25 bytes */ + private const val MAX_FILE_SIZE = 10L * 1024 * 1024 + private const val MAX_LAYOUTS = 16 + private const val MAX_KEY_SHAPES = 300 + private const val MAX_SAMPLE_KEYS = 150 + private const val MAX_LABEL_LENGTH = 8 + /** a key is adapted only with at least this many samples ... */ + const val MIN_SAMPLES_FOR_ADAPTATION = 30 + /** ... and fully from this number of samples on */ + const val FULL_SAMPLES_FOR_ADAPTATION = 100 + /** maximum shift of a key center for strength 1..5, in key sizes */ + private val MAX_SHIFT_FOR_STRENGTH = floatArrayOf(0.2f, 0.27f, 0.33f, 0.42f, 0.5f) + const val DEFAULT_STRENGTH = 3 + + /** Position of a key, normalized to the keyboard size (0..1) */ + data class KeyShape(val code: Int, val label: String, val x: Float, val y: Float, val width: Float, val height: Float) { + val centerX get() = x + width / 2 + val centerY get() = y + height / 2 + } + + /** + * Touch relative to the intended key center, in units of key width / height. + * [adapted]: whether adaptation was active when the touch was made. + */ + data class Sample(val dx: Float, val dy: Float, val touchedCode: Int, val adapted: Boolean = false) + + class LayoutData(val id: String) { + var shapes: List = emptyList() + /** keyboard height / width, for drawing */ + var aspectRatio: Float = 0.35f + val samples = HashMap>() + val sampleCount get() = samples.values.sumOf { it.size } + /** keyboard the shapes were taken from (not persisted) */ + internal var shapesSource: Any? = null + } + + /** Geometry of the currently shown keyboard, in keyboard pixels */ + interface KeyGeometry { + val layoutId: String + /** key that produces this (lowercase) code point, as x, y, width, height in keyboard pixels */ + fun keyRect(codePoint: Int): IntArray? + fun snapshot(): List + val aspectRatio: Float + /** identity of the keyboard, to avoid taking a new snapshot for every word */ + val source: Any + } + + /** A key considered for adapted detection: code and x, y, width, height in any unit */ + class Candidate(val code: Int, val x: Float, val y: Float, val width: Float, val height: Float) + + private class PendingWord( + val geometry: KeyGeometry, + val typedWord: String, + val finalWord: String, + val xs: IntArray, + val ys: IntArray, + val adapted: Boolean, + ) + + /** Result of reading stored data */ + class ParsedData(val layouts: Map, val device: String?, val weeks: Map = emptyMap()) + + /** touches and misses of one week, for showing whether typing gets better */ + class WeekStats(val week: Int, val touches: Int, val misses: Int) + + private val layouts = LinkedHashMap() + /** week (days since 1970 / 7) -> [touches, misses]; only counts, nothing about the text */ + private val weeks = java.util.TreeMap() + private const val MAX_WEEKS = 26 + @Volatile private var loaded = false + private var pending: PendingWord? = null + /** committed word at the time of a backspace press, only kept for reverting the autocorrection */ + private var revertCandidate: PendingWord? = null + private var unsavedSamples = 0 + private var appContext: Context? = null + private val ioExecutor = Executors.newSingleThreadExecutor() + /** device the data was learned on, null if unknown (no data yet) */ + private var learnedOnDevice: String? = null + val currentDevice: String = "${Build.MANUFACTURER} ${Build.MODEL}".trim() + + /** cache: layout id + strength -> offsets per code as [dx, dy]; invalidated when data is saved or replaced */ + @Volatile private var offsetCache = HashMap>() + + /** adjusted dictionary coordinates -> raw touch coordinates (keyboard coordinates), for the last touches */ + private val rawTouches = object : LinkedHashMap(64, 0.75f, true) { + override fun removeEldestEntry(eldest: MutableMap.MutableEntry?) = size > 128 + } + + /** Make sure stored data can be loaded, e.g. for adapting key detection. Loads in background. */ + @Synchronized + fun init(context: Context) { + if (appContext == null) appContext = context.applicationContext + if (!loaded) ioExecutor.execute { synchronized(this) { ensureLoaded() } } + } + + // ---------------- recording ---------------- + + /** A word was committed. It is only used once we know it was not reverted. */ + @Synchronized + fun onWordCommitted(context: Context, geometry: KeyGeometry, typedWord: String, finalWord: String, + xs: IntArray, ys: IntArray, adapted: Boolean = false) { + appContext = context.applicationContext + finalizePending() + revertCandidate = null + if (typedWord.isEmpty() || finalWord.isEmpty()) return + val rawXs = xs.copyOf() + val rawYs = ys.copyOf() + for (i in rawXs.indices) { + if (i >= rawYs.size) break + val raw = rawTouches[pack(rawXs[i], rawYs[i])] ?: continue + rawXs[i] = unpackX(raw) + rawYs[i] = unpackY(raw) + } + pending = PendingWord(geometry, typedWord, finalWord, rawXs, rawYs, adapted) + } + + /** + * Backspace was pressed. Unless it reverts the autocorrection ([onCommitReverted]), the user + * edits the committed word, so we don't know which word was finally used: it is dropped. + */ + @Synchronized + fun onBackspace() { + revertCandidate = pending + pending = null + } + + /** The user reverted the autocorrection: the typed word was what they wanted */ + @Synchronized + fun onCommitReverted() { + val p = revertCandidate ?: pending ?: return + revertCandidate = null + pending = PendingWord(p.geometry, p.typedWord, p.typedWord, p.xs, p.ys, p.adapted) + finalizePending() + } + + /** The user deletes into the committed word, so we don't know the final word */ + @Synchronized + fun discardPending() { + pending = null + } + + /** Called when other input follows, i.e. the last committed word is final */ + @Synchronized + fun finalizePending() { + revertCandidate = null + val p = pending ?: return + pending = null + val newSamples = computeSamples(p.typedWord, p.finalWord, p.xs, p.ys, p.adapted) { p.geometry.keyRect(it) } + addSamples(p.geometry, newSamples) + } + + /** + * The user deleted [deletedCodePoint] (touched at [x], [y]) and typed [newCodePoint] instead. + * If the keys are neighbors, the touch was a miss for the new letter. + */ + @Synchronized + fun onLetterRetyped(context: Context, geometry: KeyGeometry, deletedCodePoint: Int, x: Int, y: Int, + newCodePoint: Int, adapted: Boolean = false) { + appContext = context.applicationContext + val raw = rawTouches[pack(x, y)] + val rawX = if (raw == null) x else unpackX(raw) + val rawY = if (raw == null) y else unpackY(raw) + val sample = selfCorrectionSample(deletedCodePoint, rawX, rawY, newCodePoint, adapted) { geometry.keyRect(it) } ?: return + addSamples(geometry, listOf(sample)) + } + + private fun addSamples(geometry: KeyGeometry, newSamples: List>) { + if (newSamples.isEmpty()) return + ensureLoaded() + if (!loaded) return // user storage not available (device not unlocked yet) + if (learnedOnDevice == null) learnedOnDevice = currentDevice + val layout = layouts.getOrPut(geometry.layoutId) { LayoutData(geometry.layoutId) } + // same limit as when reading, so nothing is lost when reading again: drop the layout with least data + while (layouts.size > MAX_LAYOUTS) { + val smallest = layouts.values.filter { it !== layout }.minByOrNull { it.sampleCount } ?: break + layouts.remove(smallest.id) + } + if (layout.shapesSource !== geometry.source) { + layout.shapes = geometry.snapshot() + layout.aspectRatio = geometry.aspectRatio + layout.shapesSource = geometry.source + } + for ((code, sample) in newSamples) { + val list = layout.samples.getOrPut(code) { ArrayDeque() } + list.addLast(sample) + while (list.size > MAX_SAMPLES_PER_KEY) list.removeFirst() + } + val week = currentWeek() + val stats = weeks.getOrPut(week) { IntArray(2) } + stats[0] += newSamples.size + stats[1] += newSamples.count { (code, sample) -> sample.touchedCode != code } + while (weeks.size > MAX_WEEKS) weeks.remove(weeks.firstKey()) + unsavedSamples += newSamples.size + if (unsavedSamples >= SAVE_EVERY_SAMPLES) save() + } + + /** Input finished (keyboard closed or other field): finalize and write to disk */ + @Synchronized + fun onFinishInput() { + finalizePending() + if (unsavedSamples > 0) save() + } + + // ---------------- adaptation ---------------- + + /** Effective offsets per key code for a layout, empty if there is not enough data */ + fun getOffsets(context: Context?, layoutId: String, strength: Int): Map { + val cacheKey = cacheKey(layoutId, strength) + offsetCache[cacheKey]?.let { return it } + if (context == null && !loaded) return emptyMap() // touch events: don't wait for loading (started by init) + synchronized(this) { + if (context != null && appContext == null) appContext = context.applicationContext + ensureLoaded() + if (!loaded) return emptyMap() // not available yet + val maxShift = maxShift(strength) + val result = HashMap() + layouts[layoutId]?.samples?.forEach { (code, list) -> + effectiveOffset(list, maxShift)?.let { result[code] = it } + } + val newCache = HashMap(offsetCache) + newCache[cacheKey] = result + offsetCache = newCache + return result + } + } + + // avoids building a new string for every touch event (layout ids are cached per keyboard) + private class CacheKey(val layoutId: String, val strength: Int, val key: String) + @Volatile private var lastCacheKey = CacheKey("", 0, "") + private fun cacheKey(layoutId: String, strength: Int): String { + val last = lastCacheKey + if (last.layoutId == layoutId && last.strength == strength) return last.key + val key = CacheKey(layoutId, strength, "$layoutId|$strength") + lastCacheKey = key + return key.key + } + + fun maxShift(strength: Int) = MAX_SHIFT_FOR_STRENGTH[(strength - 1).coerceIn(0, MAX_SHIFT_FOR_STRENGTH.size - 1)] + + /** + * Average offset of the touches meant for a key, weighted by the amount of data and limited + * to [maxShift]. Null if there is not enough data. + */ + fun effectiveOffset(samples: Collection, maxShift: Float): FloatArray? { + val n = samples.size + if (n < MIN_SAMPLES_FOR_ADAPTATION) return null + val weight = ((n - MIN_SAMPLES_FOR_ADAPTATION + 1).toFloat() / + (FULL_SAMPLES_FOR_ADAPTATION - MIN_SAMPLES_FOR_ADAPTATION + 1)).coerceIn(0f, 1f) + // limit single outliers + var dx = samples.sumOf { it.dx.coerceIn(-1f, 1f).toDouble() }.toFloat() / n * weight + var dy = samples.sumOf { it.dy.coerceIn(-1f, 1f).toDouble() }.toFloat() / n * weight + if (!dx.isFinite() || !dy.isFinite()) return null + val length = hypot(dx, dy) + if (length > maxShift) { + dx *= maxShift / length + dy *= maxShift / length + } + return floatArrayOf(dx, dy) + } + + /** + * Code of the key a touch at [x], [y] is meant for, if adaptation decides differently than + * plain key centers would. Null means: keep the normal detection. + * Only the candidates are considered, distances are relative to the key size. + */ + fun adaptedKeyCode(x: Float, y: Float, candidates: List, offsets: Map): Int? { + if (offsets.isEmpty() || candidates.size < 2) return null + var bestPlain: Candidate? = null + var bestPlainDistance = Float.MAX_VALUE + var bestAdapted: Candidate? = null + var bestAdaptedDistance = Float.MAX_VALUE + for (c in candidates) { + val cx = c.x + c.width / 2 + val cy = c.y + c.height / 2 + val plain = sq((x - cx) / c.width) + sq((y - cy) / c.height) + val offset = offsets[c.code] + val adapted = if (offset == null) plain + else sq((x - cx - offset[0] * c.width) / c.width) + sq((y - cy - offset[1] * c.height) / c.height) + if (plain < bestPlainDistance) { bestPlainDistance = plain; bestPlain = c } + if (adapted < bestAdaptedDistance) { bestAdaptedDistance = adapted; bestAdapted = c } + } + if (bestAdapted == null || bestAdapted === bestPlain) return null + // don't take touches that are far away from the adapted key + if (bestAdaptedDistance > 1f) return null + return bestAdapted.code + } + + /** + * Touch coordinates for the dictionary: moved by the learned offset of the detected key, so + * the spatial model of the dictionary sees the touch where an average touch for the key is. + * [x] and [y] are view coordinates, [correctionX] and [correctionY] convert them to keyboard + * coordinates. Remembers the raw position (keyboard coordinates), so learning is not affected. + */ + @Synchronized + fun adjustForDictionary(x: Int, y: Int, correctionX: Int, correctionY: Int, + keyWidth: Int, keyHeight: Int, offset: FloatArray?): IntArray { + if (offset == null || !offset[0].isFinite() || !offset[1].isFinite()) return intArrayOf(x, y) + val ax = (x - offset[0] * keyWidth).toInt().coerceAtLeast(0) + val ay = (y - offset[1] * keyHeight).toInt().coerceAtLeast(0) + if (ax != x || ay != y) rawTouches[pack(ax + correctionX, ay + correctionY)] = pack(x + correctionX, y + correctionY) + return intArrayOf(ax, ay) + } + + private fun sq(f: Float) = f * f + private fun pack(x: Int, y: Int) = (x.toLong() shl 32) or (y.toLong() and 0xffffffffL) + private fun unpackX(p: Long) = (p shr 32).toInt() + private fun unpackY(p: Long) = p.toInt() + + // ---------------- reading (settings screen) ---------------- + + @Synchronized + fun getLayouts(context: Context): List { + appContext = context.applicationContext + ensureLoaded() + return layouts.values.filter { it.sampleCount > 0 }.sortedByDescending { it.sampleCount } + } + + /** Touches and misses per week, oldest first */ + @Synchronized + fun getWeeklyStats(context: Context): List { + appContext = context.applicationContext + ensureLoaded() + return weeks.map { (week, stats) -> WeekStats(week, stats[0], stats[1]) } + } + + private fun currentWeek() = (System.currentTimeMillis() / 86_400_000L / 7).toInt() + + /** Device the data was learned on, if it is not this device */ + @Synchronized + fun getOtherLearnedOnDevice(context: Context): String? { + appContext = context.applicationContext + ensureLoaded() + val device = learnedOnDevice ?: return null + return if (device == currentDevice || layouts.isEmpty()) null else device + } + + /** Keep data that was learned on another device, and continue learning on this one */ + @Synchronized + fun acceptDataFromOtherDevice(context: Context) { + appContext = context.applicationContext + learnedOnDevice = currentDevice + save() + } + + @Synchronized + fun clear(context: Context) { + appContext = context.applicationContext + layouts.clear() + weeks.clear() + pending = null + revertCandidate = null + unsavedSamples = 0 + loaded = !isUserLocked(context) + learnedOnDevice = null + offsetCache = HashMap() + rawTouches.clear() + val file = getFile(context) + val legacyFile = getLegacyFile(context) + ioExecutor.execute { + file.delete() + legacyFile.delete() + } + } + + /** Data file was replaced, e.g. by restoring a backup */ + @Synchronized + fun reload() { + layouts.clear() + weeks.clear() + pending = null + revertCandidate = null + unsavedSamples = 0 + loaded = false + learnedOnDevice = null + offsetCache = HashMap() + } + + // ---------------- backup ---------------- + + /** File to put into a backup, null if there is no data. Must not be called on the main thread. */ + fun getFileForBackup(context: Context): File? { + synchronized(this) { + appContext = context.applicationContext + if (isUserLocked(context)) return null + ensureLoaded() + onFinishInput() // write unsaved data... + } + // ... and wait until it is written (without holding the lock, background tasks may need it) + try { ioExecutor.submit(Runnable { }).get() } catch (_: Exception) { } + return getFile(context).takeIf { it.isFile } + } + + /** + * Restore data from a backup. The data is validated and written in a clean form, invalid data + * is rejected. Returns whether data was restored. + */ + fun restore(context: Context, input: InputStream): Boolean { + val text = readLimited(input) ?: return false + val parsed = try { + parse(JSONObject(text)) + } catch (t: Throwable) { // also catch errors like StackOverflowError from malicious input + Log.w(TAG, "could not restore typing heatmap: ${t.javaClass.simpleName}") + return false + } + val json = toJson(parsed.layouts.values.map { it.copyForSaving() }, parsed.device ?: currentDevice, parsed.weeks).toString() + val file = getFile(context) + // forget the current data first, so it can't be saved over the restored data + synchronized(this) { reload() } + try { + // written in the same queue as normal saves, so earlier saves can't overwrite it + ioExecutor.submit(Runnable { writeAtomically(file, json) }).get() + } catch (e: Exception) { + Log.w(TAG, "could not restore typing heatmap: ${e.javaClass.simpleName}") + return false + } + synchronized(this) { + getLegacyFile(context).delete() + reload() + } + return true + } + + private fun readLimited(input: InputStream): String? { + val bytes = input.readNBytesCompat(MAX_FILE_SIZE + 1) + if (bytes.size > MAX_FILE_SIZE) { + Log.w(TAG, "typing heatmap too large, ignoring it") + return null + } + return String(bytes, Charsets.UTF_8) + } + + private fun InputStream.readNBytesCompat(limit: Long): ByteArray { + val out = java.io.ByteArrayOutputStream() + val buffer = ByteArray(8192) + var total = 0L + while (total < limit) { + val read = read(buffer, 0, minOf(buffer.size.toLong(), limit - total).toInt()) + if (read < 0) break + out.write(buffer, 0, read) + total += read + } + return out.toByteArray() + } + + // ---------------- pure logic (tested) ---------------- + + /** + * Pairs of (index in typed, index in final) for code points that correspond to each other, + * i.e. matches and substitutions of a minimal edit alignment (case insensitive). + */ + fun align(typed: IntArray, final: IntArray): List> { + val n = typed.size + val m = final.size + val d = Array(n + 1) { IntArray(m + 1) } + for (i in 0..n) d[i][0] = i + for (j in 0..m) d[0][j] = j + for (i in 1..n) for (j in 1..m) { + val cost = if (sameLetter(typed[i - 1], final[j - 1])) 0 else 1 + d[i][j] = minOf(d[i - 1][j - 1] + cost, d[i - 1][j] + 1, d[i][j - 1] + 1) + } + val pairs = ArrayList>() + var i = n + var j = m + while (i > 0 && j > 0) { + val cost = if (sameLetter(typed[i - 1], final[j - 1])) 0 else 1 + when { + d[i][j] == d[i - 1][j - 1] + cost -> { pairs.add(i - 1 to j - 1); i--; j-- } + d[i][j] == d[i - 1][j] + 1 -> i-- + else -> j-- + } + } + pairs.reverse() + return pairs + } + + /** + * Samples (intended code to sample) from the touches of a typed word, using the final word + * as ground truth. [keyRect] gives x, y, width, height for a lowercase code point. + */ + fun computeSamples(typedWord: String, finalWord: String, xs: IntArray, ys: IntArray, + adapted: Boolean = false, keyRect: (Int) -> IntArray?): List> { + val typed = StringUtils.toCodePointArray(typedWord) + val final = StringUtils.toCodePointArray(finalWord) + val pairs = align(typed, final) + val substitutions = pairs.count { !sameLetter(typed[it.first], final[it.second]) } + // unrelated word (e.g. a picked prediction): don't learn from it + if (pairs.isEmpty() || substitutions > maxOf(1, pairs.size / 3)) return emptyList() + val result = ArrayList>() + for ((ti, fi) in pairs) { + if (ti >= xs.size || ti >= ys.size) continue + // only letters: digits and symbols may have been typed on another keyboard (symbols) + if (!Character.isLetter(typed[ti])) continue + val sample = sampleFor(lower(final[fi]), lower(typed[ti]), xs[ti], ys[ti], adapted, keyRect) ?: continue + result.add(sample) + } + return result + } + + /** Sample for a touch at [x], [y] that was meant for [newCodePoint], if the keys are neighbors */ + fun selfCorrectionSample(deletedCodePoint: Int, x: Int, y: Int, newCodePoint: Int, adapted: Boolean = false, + keyRect: (Int) -> IntArray?): Pair? { + val deleted = lower(deletedCodePoint) + val intended = lower(newCodePoint) + if (deleted == intended || !Character.isLetter(deleted)) return null + val deletedRect = keyRect(deleted) ?: return null + val intendedRect = keyRect(intended) ?: return null + // neighbor keys: centers at most about one key apart (also diagonally) + val distX = abs((deletedRect[0] + deletedRect[2] / 2f) - (intendedRect[0] + intendedRect[2] / 2f)) / intendedRect[2] + val distY = abs((deletedRect[1] + deletedRect[3] / 2f) - (intendedRect[1] + intendedRect[3] / 2f)) / intendedRect[3] + if (distX > 1.1f || distY > 1.1f) return null + return sampleFor(intended, deleted, x, y, adapted, keyRect) + } + + private fun sampleFor(intended: Int, touched: Int, x: Int, y: Int, adapted: Boolean, keyRect: (Int) -> IntArray?): Pair? { + if (x < 0 || y < 0) return null // no coordinates + if (!Character.isLetter(intended)) return null + val rect = keyRect(intended) ?: return null + if (rect[2] <= 0 || rect[3] <= 0) return null + val dx = (x - (rect[0] + rect[2] / 2f)) / rect[2] + val dy = (y - (rect[1] + rect[3] / 2f)) / rect[3] + if (abs(dx) > MAX_OFFSET || abs(dy) > MAX_OFFSET) return null + return intended to Sample(dx, dy, touched, adapted) + } + + private fun lower(cp: Int) = Character.toLowerCase(cp) + private fun sameLetter(a: Int, b: Int) = lower(a) == lower(b) + + // ---------------- persistence ---------------- + + /** credential encrypted storage, excluded from Android backup and device-to-device transfer */ + private fun getFile(context: Context) = File(context.applicationContext.noBackupFilesDir, FILE_NAME) + + /** location used by earlier versions (device protected storage) */ + private fun getLegacyFile(context: Context) = File(DeviceProtectedUtils.getFilesDir(context), FILE_NAME) + + private fun ensureLoaded() { + if (loaded) return + val context = appContext ?: return + if (isUserLocked(context)) return // credential encrypted storage is not available yet + loaded = true + val file = getFile(context) + migrateLegacyFile(context, file) + if (!file.isFile) return + try { + if (file.length() > MAX_FILE_SIZE) throw IllegalStateException("file too large") + val parsed = parse(JSONObject(file.readText())) + layouts.clear() + layouts.putAll(parsed.layouts) + weeks.clear() + weeks.putAll(parsed.weeks) + learnedOnDevice = parsed.device + offsetCache = HashMap() + } catch (t: Throwable) { // also catch errors like StackOverflowError from broken data + // log only the type: the message may contain the file content + Log.w(TAG, "could not read typing heatmap: ${t.javaClass.simpleName}") + layouts.clear() + weeks.clear() + learnedOnDevice = null + // keep the unreadable file instead of overwriting it with new data + file.renameTo(File(file.parentFile, "$FILE_NAME.broken")) + } + } + + /** Move the file from the location of earlier versions, without risking to lose it */ + private fun migrateLegacyFile(context: Context, file: File) { + val legacyFile = getLegacyFile(context) + if (!legacyFile.isFile) return + try { + if (!file.exists()) { + file.parentFile?.mkdirs() + val tmp = File(file.parentFile, "$FILE_NAME.tmp") + legacyFile.copyTo(tmp, overwrite = true) + if (!tmp.renameTo(file)) return // keep the old file, try again next time + } + legacyFile.delete() + } catch (e: Exception) { + Log.w(TAG, "could not move typing heatmap: ${e.javaClass.simpleName}") + } + } + + private fun save() { + val context = appContext ?: return + if (!loaded) return // never overwrite data that was not loaded + unsavedSamples = 0 + offsetCache = HashMap() // new data is used for adaptation from now on + // copy the data, and do the expensive conversion in background + val copy = layouts.values.map { it.copyForSaving() } + val weeksCopy = weeks.mapValues { it.value.copyOf() } + val device = learnedOnDevice ?: currentDevice + val file = getFile(context) + ioExecutor.execute { + try { + writeAtomically(file, toJson(copy, device, weeksCopy).toString()) + } catch (t: Throwable) { + Log.w(TAG, "could not write typing heatmap: ${t.javaClass.simpleName}") + } + } + } + + private fun writeAtomically(file: File, text: String) { + file.parentFile?.mkdirs() + val tmp = File(file.parentFile, "$FILE_NAME.tmp") + tmp.writeText(text) + if (!tmp.renameTo(file)) { + tmp.delete() + throw IllegalStateException("rename failed") + } + } + + /** Copy whose lists are not changed while writing in background */ + private class LayoutCopy(val id: String, val aspectRatio: Float, val shapes: List, val samples: Map>) + + private fun LayoutData.copyForSaving() = LayoutCopy(id, aspectRatio, shapes, samples.mapValues { it.value.toList() }) + + private fun toJson(layouts: List, device: String, weeks: Map): JSONObject { + val root = JSONObject() + root.put("version", 2) + root.put("device", device) + val layoutArray = JSONArray() + for (layout in layouts) { + val l = JSONObject() + l.put("id", layout.id) + l.put("aspect", r(if (layout.aspectRatio.isFinite()) layout.aspectRatio else 0.35f)) + val shapes = JSONArray() + layout.shapes.forEach { + if (!it.x.isFinite() || !it.y.isFinite() || !it.width.isFinite() || !it.height.isFinite()) return@forEach + shapes.put(JSONArray().put(it.code).put(it.label).put(r(it.x)).put(r(it.y)).put(r(it.width)).put(r(it.height))) + } + l.put("keys", shapes) + val samples = JSONObject() + for ((code, list) in layout.samples) { + val flat = JSONArray() + list.forEach { + if (!it.dx.isFinite() || !it.dy.isFinite()) return@forEach + flat.put(r(it.dx)).put(r(it.dy)).put(it.touchedCode).put(if (it.adapted) 1 else 0) + } + samples.put(code.toString(), flat) + } + l.put("samples", samples) + layoutArray.put(l) + } + root.put("layouts", layoutArray) + val weeksJson = JSONObject() + weeks.forEach { (week, stats) -> weeksJson.put(week.toString(), JSONArray().put(stats[0]).put(stats[1])) } + root.put("weeks", weeksJson) + return root + } + + /** Read stored data, ignoring anything invalid or out of range */ + fun parse(root: JSONObject): ParsedData { + val version = root.optInt("version", 1) + val valuesPerSample = if (version >= 2) 4 else 3 + val device = root.optString("device", "").take(100).ifEmpty { null } + val result = LinkedHashMap() + val weeks = java.util.TreeMap() + root.optJSONObject("weeks")?.let { w -> + for (key in w.keys()) { + val week = key.toIntOrNull() ?: continue + val a = w.optJSONArray(key) ?: continue + val touches = a.optInt(0, -1) + val misses = a.optInt(1, -1) + if (week < 0 || touches < 0 || misses < 0 || misses > touches) continue + weeks[week] = intArrayOf(touches, misses) + } + while (weeks.size > MAX_WEEKS) weeks.remove(weeks.firstKey()) + } + val layoutArray = root.optJSONArray("layouts") ?: return ParsedData(result, device, weeks) + for (i in 0 until minOf(layoutArray.length(), MAX_LAYOUTS)) { + val l = layoutArray.optJSONObject(i) ?: continue + val id = l.optString("id", "").take(100) + if (id.isEmpty()) continue + val layout = LayoutData(id) + val aspect = l.optDouble("aspect", 0.35).toFloat() + layout.aspectRatio = if (aspect.isFinite() && aspect in 0.05f..3f) aspect else 0.35f + val shapes = l.optJSONArray("keys") ?: JSONArray() + layout.shapes = (0 until minOf(shapes.length(), MAX_KEY_SHAPES)).mapNotNull { + val s = shapes.optJSONArray(it) ?: return@mapNotNull null + if (s.length() < 6) return@mapNotNull null + val values = (2..5).map { index -> s.optDouble(index, Double.NaN).toFloat() } + if (values.any { v -> !v.isFinite() || v < -1f || v > 2f } || values[2] <= 0f || values[3] <= 0f) + return@mapNotNull null + KeyShape(s.optInt(0), s.optString(1, "").take(MAX_LABEL_LENGTH), values[0], values[1], values[2], values[3]) + } + val samples = l.optJSONObject("samples") ?: JSONObject() + var keyCount = 0 + for (key in samples.keys()) { + if (keyCount >= MAX_SAMPLE_KEYS) break + val code = key.toIntOrNull() ?: continue + if (!Character.isValidCodePoint(code) || !Character.isLetter(code)) continue + val flat = samples.optJSONArray(key) ?: continue + val count = flat.length() / valuesPerSample + val list = ArrayDeque() + // keep only the newest samples + for (index in maxOf(0, count - MAX_SAMPLES_PER_KEY) until count) { + val j = index * valuesPerSample + val dx = flat.optDouble(j, Double.NaN).toFloat() + val dy = flat.optDouble(j + 1, Double.NaN).toFloat() + val touched = flat.optInt(j + 2, -1) + if (!dx.isFinite() || !dy.isFinite() || abs(dx) > MAX_OFFSET || abs(dy) > MAX_OFFSET) continue + if (!Character.isValidCodePoint(touched) || touched == 0) continue + val adapted = valuesPerSample == 4 && flat.optInt(j + 3, 0) == 1 + list.addLast(Sample(dx, dy, touched, adapted)) + } + if (list.isNotEmpty()) { + layout.samples[code] = list + keyCount++ + } + } + result[layout.id] = layout + } + return ParsedData(result, device, weeks) + } + + private fun r(f: Float): Double = (f * 1000.0).roundToLong() / 1000.0 + + /** for tests */ + @Synchronized + internal fun resetForTest() { + layouts.clear() + weeks.clear() + pending = null + revertCandidate = null + unsavedSamples = 0 + loaded = true + appContext = null + learnedOnDevice = null + offsetCache = HashMap() + rawTouches.clear() + } + + @Synchronized + internal fun layoutForTest(id: String): LayoutData? = layouts[id] + + /** for tests: invalidate cached offsets without saving */ + @Synchronized + internal fun clearOffsetCacheForTest() { + offsetCache = HashMap() + } +} diff --git a/app/src/main/java/helium314/keyboard/latin/inputlogic/InputLogic.kt b/app/src/main/java/helium314/keyboard/latin/inputlogic/InputLogic.kt index cc49b0980..96aced552 100644 --- a/app/src/main/java/helium314/keyboard/latin/inputlogic/InputLogic.kt +++ b/app/src/main/java/helium314/keyboard/latin/inputlogic/InputLogic.kt @@ -19,6 +19,7 @@ import android.view.inputmethod.CorrectionInfo import android.view.inputmethod.EditorInfo import helium314.keyboard.compat.AppQuirksManager import helium314.keyboard.event.Event +import helium314.keyboard.latin.heatmap.TypingRecorder import helium314.keyboard.event.InputTransaction import helium314.keyboard.latin.cjk.CjkInputManager import helium314.keyboard.keyboard.Keyboard @@ -83,6 +84,7 @@ class InputLogic( private var mLastComposedWord: LastComposedWord = LastComposedWord.NOT_A_COMPOSED_WORD private val mWordComposer = WordComposer() + private val mTypingRecorder = TypingRecorder(mLatinIME, mWordComposer) private val mConnection = RichInputConnection(mLatinIME) private val mRecapitalizeStatus = RecapitalizeStatus() // the composing word at the time the shift key was pressed down, null if there was none @@ -134,6 +136,7 @@ class InputLogic( fun startInput(combiningSpec: String?, settingsValues: SettingsValues) { mCapsLockFromRecapitalize = false + mTypingRecorder.onStartInput(settingsValues) mEnteredText = null mWordBeingCorrectedByCursor = null mConnection.onStartInput() @@ -168,6 +171,7 @@ class InputLogic( } fun finishInput() { + mTypingRecorder.onFinishInput() if (mWordComposer.isComposingWord()) { mConnection.finishComposingText() StatsUtils.onWordCommitUserTyped(mWordComposer.getTypedWord(), mWordComposer.isBatchMode()) @@ -382,6 +386,7 @@ class InputLogic( mResumedWordStart = -1 mResumedWordEnd = -1 mJustRevertedACommit = false + mTypingRecorder.onEvent(event, settingsValues) val processedEvent = mWordComposer.processEvent(event) val inputTransaction = InputTransaction( settingsValues, @@ -1429,6 +1434,7 @@ class InputLogic( } private fun handleBackspaceEvent(event: Event, inputTransaction: InputTransaction) { + mTypingRecorder.onBackspace() val currentKeyboardScript = inputTransaction.settingsValues.mCurrentKeyboardScript mSpaceState = SpaceState.NONE mDeleteCount++ @@ -2169,6 +2175,7 @@ class InputLogic( } private fun revertCommit(inputTransaction: InputTransaction) { + mTypingRecorder.onCommitReverted() val originallyTypedWord = mLastComposedWord.mTypedWord val committedWord = mLastComposedWord.mCommittedWord val committedWordString = committedWord.toString() @@ -2310,6 +2317,7 @@ class InputLogic( val typedWord = mWordComposer.getTypedWord() val ngramContext = mConnection.getNgramContextFromNthPreviousWord(settingsValues.mSpacingAndPunctuations, 1) performAdditionToUserHistoryDictionary(settingsValues, typedWord, ngramContext) + mTypingRecorder.onWordCommitted(settingsValues, typedWord) mLastComposedWord = mWordComposer.commitWord( LastComposedWord.COMMIT_TYPE_USER_TYPED_WORD, typedWord, LastComposedWord.NOT_A_SEPARATOR, ngramContext @@ -2577,6 +2585,7 @@ class InputLogic( Log.d(TAG, "commitChosenWord() : $runTimeMillis ms to run performAdditionToUserHistoryDictionary()") startTimeMillis = System.currentTimeMillis() } + mTypingRecorder.onWordCommitted(settingsValues, chosenWord) mLastComposedWord = mWordComposer.commitWord(commitType, chosenWord, separatorString, ngramContext) if (DebugFlags.DEBUG_ENABLED) { val runTimeMillis = System.currentTimeMillis() - startTimeMillis diff --git a/app/src/main/java/helium314/keyboard/latin/settings/Defaults.kt b/app/src/main/java/helium314/keyboard/latin/settings/Defaults.kt index 60b16b89f..e29da4442 100644 --- a/app/src/main/java/helium314/keyboard/latin/settings/Defaults.kt +++ b/app/src/main/java/helium314/keyboard/latin/settings/Defaults.kt @@ -95,6 +95,7 @@ object Defaults { const val PREF_AUTOCORRECT_SHORTCUTS = true const val PREF_BACKSPACE_REVERTS_AUTOCORRECT = true const val PREF_BACKSPACE_REPEAT_DELETES_WORDS = false + const val PREF_ADAPT_TO_TYPING = false const val PREF_CENTER_SUGGESTION_TEXT_TO_ENTER = false const val PREF_CENTER_TYPED_WORD = false const val PREF_SHOW_SUGGESTIONS = true diff --git a/app/src/main/java/helium314/keyboard/latin/settings/Settings.kt b/app/src/main/java/helium314/keyboard/latin/settings/Settings.kt index 94e3509e7..40868591a 100644 --- a/app/src/main/java/helium314/keyboard/latin/settings/Settings.kt +++ b/app/src/main/java/helium314/keyboard/latin/settings/Settings.kt @@ -344,6 +344,7 @@ class Settings private constructor() : SharedPreferences.OnSharedPreferenceChang const val PREF_AUTOCORRECT_SHORTCUTS = "autocorrect_shortcuts" const val PREF_BACKSPACE_REVERTS_AUTOCORRECT = "backspace_reverts_autocorrect" const val PREF_BACKSPACE_REPEAT_DELETES_WORDS = "backspace_repeat_deletes_words" + const val PREF_ADAPT_TO_TYPING = "adapt_to_typing" const val PREF_CENTER_SUGGESTION_TEXT_TO_ENTER = "center_suggestion_text_to_enter" const val PREF_CENTER_TYPED_WORD = "center_typed_word" const val PREF_SHOW_SUGGESTIONS = "show_suggestions" diff --git a/app/src/main/java/helium314/keyboard/latin/settings/SettingsValues.kt b/app/src/main/java/helium314/keyboard/latin/settings/SettingsValues.kt index 4d91747f4..becc9266b 100644 --- a/app/src/main/java/helium314/keyboard/latin/settings/SettingsValues.kt +++ b/app/src/main/java/helium314/keyboard/latin/settings/SettingsValues.kt @@ -221,6 +221,7 @@ open class SettingsValues( val mPersistTextEditMode: Boolean val mBackspaceRevertsAutocorrect: Boolean val mBackspaceRepeatDeletesWords: Boolean + val mAdaptToTyping: Boolean val mDisableMultiWordSuggestions: Boolean val mPrioritizePersonalSuggestions: Boolean val mSuggestionBalance: Int @@ -363,6 +364,7 @@ open class SettingsValues( mPersistTextEditMode = prefs.getBoolean(Settings.PREF_PERSIST_TEXT_EDIT_MODE, Defaults.PREF_PERSIST_TEXT_EDIT_MODE) mBackspaceRevertsAutocorrect = prefs.getBoolean(Settings.PREF_BACKSPACE_REVERTS_AUTOCORRECT, Defaults.PREF_BACKSPACE_REVERTS_AUTOCORRECT) mBackspaceRepeatDeletesWords = prefs.getBoolean(Settings.PREF_BACKSPACE_REPEAT_DELETES_WORDS, Defaults.PREF_BACKSPACE_REPEAT_DELETES_WORDS) + mAdaptToTyping = prefs.getBoolean(Settings.PREF_ADAPT_TO_TYPING, Defaults.PREF_ADAPT_TO_TYPING) mDisableMultiWordSuggestions = prefs.getBoolean(Settings.PREF_DISABLE_MULTI_WORD_SUGGESTIONS, Defaults.PREF_DISABLE_MULTI_WORD_SUGGESTIONS) mBigramPredictionEnabled = prefs.getBoolean(Settings.PREF_BIGRAM_PREDICTIONS, Defaults.PREF_BIGRAM_PREDICTIONS) mPrioritizePersonalSuggestions = prefs.getBoolean(Settings.PREF_PRIORITIZE_PERSONAL_SUGGESTIONS, Defaults.PREF_PRIORITIZE_PERSONAL_SUGGESTIONS) diff --git a/app/src/main/java/helium314/keyboard/settings/SettingsContainer.kt b/app/src/main/java/helium314/keyboard/settings/SettingsContainer.kt index 92a7269d7..1a7131049 100644 --- a/app/src/main/java/helium314/keyboard/settings/SettingsContainer.kt +++ b/app/src/main/java/helium314/keyboard/settings/SettingsContainer.kt @@ -148,6 +148,7 @@ private val modules = listOf( SettingsModule(SettingsWithoutKey.SCREEN_NAV_TRANSLATION, SettingsDestination.Translation, R.string.translation_settings_title, iconRes = R.drawable.ic_translate), SettingsModule(SettingsWithoutKey.SCREEN_NAV_HANDWRITING, SettingsDestination.Handwriting, R.string.libraries_hub_handwriting_title, iconRes = R.drawable.ic_edit), SettingsModule(SettingsWithoutKey.SCREEN_NAV_CJK, SettingsDestination.Cjk, R.string.cjk_settings_title, iconRes = R.drawable.ic_settings_languages), + SettingsModule(SettingsWithoutKey.SCREEN_NAV_TYPING_HEATMAP, SettingsDestination.TypingHeatmap, R.string.typing_heatmap_title, iconRes = R.drawable.ic_settings_gesture), SettingsModule(SettingsWithoutKey.SCREEN_NAV_UPDATES, SettingsDestination.Updates, R.string.settings_screen_updates, iconRes = R.drawable.ic_settings_updates) ) @@ -228,6 +229,7 @@ object SettingsWithoutKey { const val SCREEN_NAV_HANDWRITING = "screen_nav_handwriting" const val SCREEN_NAV_CJK = "screen_nav_cjk" const val SCREEN_NAV_UPDATES = "screen_nav_updates" + const val SCREEN_NAV_TYPING_HEATMAP = "screen_nav_typing_heatmap" const val SCREEN_NAV_PERSONAL_DICTIONARIES = "screen_nav_personal_dictionaries" const val SCREEN_NAV_BLOCKED_WORDS = "screen_nav_blocked_words" } diff --git a/app/src/main/java/helium314/keyboard/settings/SettingsNavHost.kt b/app/src/main/java/helium314/keyboard/settings/SettingsNavHost.kt index 7217a0afa..aaeb679c4 100644 --- a/app/src/main/java/helium314/keyboard/settings/SettingsNavHost.kt +++ b/app/src/main/java/helium314/keyboard/settings/SettingsNavHost.kt @@ -214,6 +214,9 @@ fun SettingsNavHost( composable(SettingsDestination.AppQuirks) { helium314.keyboard.settings.screens.AppQuirksScreen(onClickBack = ::goBack) } + composable(SettingsDestination.TypingHeatmap) { + helium314.keyboard.settings.screens.TypingHeatmapScreen(onClickBack = ::goBack) + } composable(SettingsDestination.Storage) { helium314.keyboard.settings.screens.StorageScreen(onClickBack = ::goBack) } @@ -265,6 +268,7 @@ object SettingsDestination { const val AppQuirks = "app_quirks" const val Gif = "gif" const val Cjk = "cjk" + const val TypingHeatmap = "typing_heatmap" val navTarget = MutableStateFlow(Settings) // Use SupervisorJob so a cancellation in one navigation hop diff --git a/app/src/main/java/helium314/keyboard/settings/preferences/BackupRestorePreference.kt b/app/src/main/java/helium314/keyboard/settings/preferences/BackupRestorePreference.kt index d9eba99ce..ff28a8678 100644 --- a/app/src/main/java/helium314/keyboard/settings/preferences/BackupRestorePreference.kt +++ b/app/src/main/java/helium314/keyboard/settings/preferences/BackupRestorePreference.kt @@ -62,6 +62,7 @@ import java.util.Locale import java.util.zip.ZipEntry import java.util.zip.ZipInputStream import java.util.zip.ZipOutputStream +import helium314.keyboard.latin.heatmap.TypingHeatmap import helium314.keyboard.settings.FeedbackManager @Composable @@ -78,7 +79,8 @@ fun BackupRestorePreference(setting: Setting) { BackupCategory.DICTIONARIES, BackupCategory.TYPING_HISTORY, BackupCategory.CLIPBOARD, - BackupCategory.GENERAL_SETTINGS + BackupCategory.GENERAL_SETTINGS, + BackupCategory.TYPING_PROFILE ) ) } @@ -102,6 +104,7 @@ fun BackupRestorePreference(setting: Setting) { BackupCategory.TYPING_HISTORY to R.string.backup_category_typing_history, BackupCategory.CLIPBOARD to R.string.backup_category_clipboard, BackupCategory.API_KEYS to R.string.backup_category_api_keys, + BackupCategory.TYPING_PROFILE to R.string.backup_category_typing_profile, BackupCategory.GENERAL_SETTINGS to R.string.backup_category_general ) categories.forEach { (category, stringResId) -> @@ -242,6 +245,16 @@ private fun backupLauncher( fileStream.close() zipStream.closeEntry() } + if (selectedCategories.contains(BackupCategory.TYPING_PROFILE)) { + // stored outside the normal folders (excluded from Android backup), see TypingHeatmap + TypingHeatmap.getFileForBackup(ctx)?.let { file -> + val fileStream = FileInputStream(file).buffered() + zipStream.putNextEntry(ZipEntry(TypingHeatmap.FILE_NAME)) + fileStream.copyTo(zipStream, 1024) + fileStream.close() + zipStream.closeEntry() + } + } if (selectedCategories.contains(BackupCategory.CLIPBOARD)) { val dbFile = ctx.getDatabasePath(Database.NAME) if (dbFile.exists()) { @@ -348,7 +361,12 @@ private fun restoreLauncher( LayoutUtilsCustom.onLayoutFileChanged() Settings.getInstance().stopListener() while (entry != null) { - if (entry.name.startsWith("unprotected${File.separator}")) { + if (entry.name == TypingHeatmap.FILE_NAME || entry.name == "unprotected${File.separator}${TypingHeatmap.FILE_NAME}") { + // validated and only replaced if the backup contains it; older backups have it in "unprotected" + if (selectedCategories.contains(BackupCategory.TYPING_PROFILE)) { + TypingHeatmap.restore(ctx, zip) + } + } else if (entry.name.startsWith("unprotected${File.separator}")) { val adjustedName = entry.name.substringAfter("unprotected${File.separator}") if (backupFilePatterns.any { adjustedName.matches(it) }) { val cat = getCategoryForFilePath(adjustedName) @@ -584,7 +602,8 @@ enum class BackupCategory { TYPING_HISTORY, CLIPBOARD, API_KEYS, - GENERAL_SETTINGS + GENERAL_SETTINGS, + TYPING_PROFILE } private fun getCategoryForPrefKey(key: String): BackupCategory { @@ -649,5 +668,8 @@ private fun getCategoryForFilePath(path: String): BackupCategory? { if (path == API_KEYS_FILE_NAME) { return BackupCategory.API_KEYS } + if (path == TypingHeatmap.FILE_NAME) { + return BackupCategory.TYPING_PROFILE + } return null } diff --git a/app/src/main/java/helium314/keyboard/settings/screens/TextCorrectionScreen.kt b/app/src/main/java/helium314/keyboard/settings/screens/TextCorrectionScreen.kt index f804aed89..dbe1e82d5 100644 --- a/app/src/main/java/helium314/keyboard/settings/screens/TextCorrectionScreen.kt +++ b/app/src/main/java/helium314/keyboard/settings/screens/TextCorrectionScreen.kt @@ -28,6 +28,7 @@ import helium314.keyboard.latin.utils.getActivity import helium314.keyboard.latin.utils.prefs import helium314.keyboard.settings.SearchSettingsScreen import helium314.keyboard.settings.Setting +import helium314.keyboard.settings.SettingsWithoutKey import helium314.keyboard.settings.SettingsActivity import helium314.keyboard.settings.Theme import helium314.keyboard.settings.initPreview @@ -85,7 +86,12 @@ fun TextCorrectionScreen( Settings.PREF_ABC_AFTER_SYMBOL_SPACE, Settings.PREF_ABC_AFTER_NUMPAD_SPACE, Settings.PREF_ABC_AFTER_EMOJI, - Settings.PREF_ABC_AFTER_CLIP + Settings.PREF_ABC_AFTER_CLIP, + + // Typing heatmap + R.string.typing_heatmap_category, + Settings.PREF_ADAPT_TO_TYPING, + SettingsWithoutKey.SCREEN_NAV_TYPING_HEATMAP ) } @@ -97,6 +103,9 @@ fun TextCorrectionScreen( } fun createCorrectionSettings(context: Context) = listOf( + Setting(context, Settings.PREF_ADAPT_TO_TYPING, R.string.adapt_to_typing, R.string.adapt_to_typing_summary) { + SwitchPreference(it, Defaults.PREF_ADAPT_TO_TYPING) + }, Setting(context, Settings.PREF_BLOCK_POTENTIALLY_OFFENSIVE, R.string.prefs_block_potentially_offensive_title, R.string.prefs_block_potentially_offensive_summary diff --git a/app/src/main/java/helium314/keyboard/settings/screens/TypingHeatmapScreen.kt b/app/src/main/java/helium314/keyboard/settings/screens/TypingHeatmapScreen.kt new file mode 100644 index 000000000..612dbb795 --- /dev/null +++ b/app/src/main/java/helium314/keyboard/settings/screens/TypingHeatmapScreen.kt @@ -0,0 +1,315 @@ +// SPDX-License-Identifier: GPL-3.0-only +package helium314.keyboard.settings.screens + +import androidx.compose.foundation.Canvas +import androidx.compose.ui.Alignment +import androidx.compose.foundation.shape.RoundedCornerShape +import androidx.compose.foundation.layout.width +import androidx.compose.foundation.layout.height +import androidx.compose.foundation.layout.fillMaxHeight +import androidx.compose.foundation.layout.Box +import androidx.compose.foundation.background +import androidx.compose.foundation.layout.Arrangement +import androidx.compose.foundation.layout.Column +import androidx.compose.foundation.layout.Row +import androidx.compose.foundation.layout.aspectRatio +import androidx.compose.foundation.layout.fillMaxWidth +import androidx.compose.foundation.layout.padding +import androidx.compose.foundation.horizontalScroll +import androidx.compose.foundation.rememberScrollState +import androidx.compose.foundation.verticalScroll +import androidx.compose.material3.Card +import androidx.compose.material3.CardDefaults +import androidx.compose.material3.FilterChip +import androidx.compose.material3.MaterialTheme +import androidx.compose.material3.OutlinedButton +import androidx.compose.material3.Text +import androidx.compose.runtime.Composable +import androidx.compose.runtime.getValue +import androidx.compose.runtime.mutableIntStateOf +import androidx.compose.runtime.mutableStateOf +import androidx.compose.runtime.remember +import androidx.compose.runtime.setValue +import androidx.compose.ui.Modifier +import androidx.compose.ui.geometry.CornerRadius +import androidx.compose.ui.geometry.Offset +import androidx.compose.ui.geometry.Size +import androidx.compose.ui.graphics.Color +import androidx.compose.ui.graphics.drawscope.Stroke +import androidx.compose.ui.platform.LocalContext +import androidx.compose.ui.res.stringResource +import androidx.compose.ui.text.TextStyle +import androidx.compose.ui.text.drawText +import androidx.compose.ui.text.font.FontWeight +import androidx.compose.ui.text.rememberTextMeasurer +import androidx.compose.ui.unit.dp +import androidx.compose.ui.unit.sp +import helium314.keyboard.latin.R +import helium314.keyboard.latin.heatmap.TypingHeatmap +import helium314.keyboard.latin.settings.Defaults +import helium314.keyboard.latin.settings.Settings +import helium314.keyboard.latin.utils.prefs +import helium314.keyboard.settings.SearchSettingsScreen +import helium314.keyboard.settings.dialogs.ConfirmationDialog + +/** Number of samples a key needs before its average offset is shown */ +private const val MIN_SAMPLES_FOR_AVERAGE = 15 +/** Number of samples before and after enabling adaptation needed to compare them */ +private const val MIN_SAMPLES_FOR_COMPARISON = 200 +/** weeks with fewer touches are not shown in the history */ +private const val MIN_TOUCHES_PER_WEEK = 50 + +@Composable +fun TypingHeatmapScreen(onClickBack: () -> Unit) { + val context = LocalContext.current + var refresh by remember { mutableIntStateOf(0) } + val layouts = remember(refresh) { TypingHeatmap.getLayouts(context) } + var selectedLayoutId by remember { mutableStateOf(null) } + var onlyMisses by remember { mutableStateOf(false) } + var showClearDialog by remember { mutableStateOf(false) } + val layout = layouts.firstOrNull { it.id == selectedLayoutId } ?: layouts.firstOrNull() + val otherDevice = remember(refresh) { TypingHeatmap.getOtherLearnedOnDevice(context) } + val prefs = context.prefs() + val adaptEnabled = prefs.getBoolean(Settings.PREF_ADAPT_TO_TYPING, Defaults.PREF_ADAPT_TO_TYPING) + val strength = TypingHeatmap.DEFAULT_STRENGTH + + SearchSettingsScreen( + onClickBack = onClickBack, + title = stringResource(R.string.typing_heatmap_title), + settings = emptyList(), + ) { + Column( + Modifier.verticalScroll(rememberScrollState()).padding(horizontal = 16.dp, vertical = 8.dp), + verticalArrangement = Arrangement.spacedBy(12.dp) + ) { + if (otherDevice != null) { + Card( + Modifier.fillMaxWidth(), + colors = CardDefaults.cardColors(containerColor = MaterialTheme.colorScheme.errorContainer) + ) { + Column(Modifier.padding(16.dp), verticalArrangement = Arrangement.spacedBy(8.dp)) { + Text(stringResource(R.string.typing_heatmap_other_device, otherDevice, TypingHeatmap.currentDevice), + style = MaterialTheme.typography.bodyMedium) + Row(horizontalArrangement = Arrangement.spacedBy(8.dp)) { + OutlinedButton(onClick = { TypingHeatmap.clear(context); refresh++ }) { + Text(stringResource(R.string.typing_heatmap_start_over)) + } + OutlinedButton(onClick = { TypingHeatmap.acceptDataFromOtherDevice(context); refresh++ }) { + Text(stringResource(R.string.typing_heatmap_keep)) + } + } + } + } + } + if (layout == null) { + InfoCard(stringResource(R.string.typing_heatmap_no_data)) + InfoCard(stringResource(R.string.typing_heatmap_privacy)) + return@Column + } + if (layouts.size > 1) { + Row(Modifier.horizontalScroll(rememberScrollState()), horizontalArrangement = Arrangement.spacedBy(8.dp)) { + layouts.forEach { + FilterChip( + selected = it.id == layout.id, + onClick = { selectedLayoutId = it.id }, + label = { Text(it.id) } + ) + } + } + } + Row(horizontalArrangement = Arrangement.spacedBy(8.dp)) { + FilterChip(selected = !onlyMisses, onClick = { onlyMisses = false }, + label = { Text(stringResource(R.string.typing_heatmap_all_touches)) }) + FilterChip(selected = onlyMisses, onClick = { onlyMisses = true }, + label = { Text(stringResource(R.string.typing_heatmap_only_misses)) }) + } + + val offsets = remember(layout, strength, adaptEnabled) { + if (adaptEnabled) TypingHeatmap.getOffsets(context, layout.id, strength) else emptyMap() + } + HeatmapCanvas(layout, onlyMisses, offsets) + + val total = layout.sampleCount + val misses = layout.samples.entries.sumOf { (code, list) -> list.count { it.touchedCode != code } } + val percent = if (total == 0) 0f else misses * 100f / total + Text(stringResource(R.string.typing_heatmap_stats, total, misses, percent), style = MaterialTheme.typography.bodyMedium) + Text(stringResource(R.string.typing_heatmap_legend), style = MaterialTheme.typography.bodySmall, + color = MaterialTheme.colorScheme.onSurfaceVariant) + Text( + when { + !adaptEnabled -> stringResource(R.string.typing_heatmap_adapt_off) + offsets.isEmpty() -> stringResource(R.string.typing_heatmap_adapt_waiting, TypingHeatmap.MIN_SAMPLES_FOR_ADAPTATION) + else -> stringResource(R.string.typing_heatmap_adapt_on, offsets.size) + }, + style = MaterialTheme.typography.bodySmall, + color = MaterialTheme.colorScheme.onSurfaceVariant + ) + + // miss rate before and after enabling adaptation + val allSamples = layout.samples.entries.flatMap { (code, list) -> list.map { code to it } } + val before = allSamples.filter { !it.second.adapted } + val after = allSamples.filter { it.second.adapted } + if (after.isNotEmpty()) { + Text(stringResource(R.string.typing_heatmap_before_after), style = MaterialTheme.typography.titleSmall) + if (before.size < MIN_SAMPLES_FOR_COMPARISON || after.size < MIN_SAMPLES_FOR_COMPARISON) { + Text(stringResource(R.string.typing_heatmap_before_after_not_enough, MIN_SAMPLES_FOR_COMPARISON), + style = MaterialTheme.typography.bodyMedium) + } else { + fun rate(list: List>) = list.count { it.first != it.second.touchedCode } * 100f / list.size + Text(stringResource(R.string.typing_heatmap_before_after_values, rate(before), before.size, rate(after), after.size), + style = MaterialTheme.typography.bodyMedium) + } + } + + // miss rate per week, to see whether typing gets better + val weekly = remember(refresh) { + TypingHeatmap.getWeeklyStats(context).filter { it.touches >= MIN_TOUCHES_PER_WEEK }.takeLast(8) + } + if (weekly.isNotEmpty()) { + Text(stringResource(R.string.typing_heatmap_history), style = MaterialTheme.typography.titleSmall) + val maxRate = weekly.maxOf { it.misses * 100f / it.touches }.coerceAtLeast(1f) + val dateFormat = remember { java.text.SimpleDateFormat("dd.MM.", java.util.Locale.getDefault()) } + weekly.forEach { week -> + val rate = week.misses * 100f / week.touches + Row(verticalAlignment = Alignment.CenterVertically, horizontalArrangement = Arrangement.spacedBy(8.dp)) { + Text(dateFormat.format(java.util.Date(week.week * 7L * 86_400_000L)), + Modifier.width(52.dp), style = MaterialTheme.typography.bodySmall) + Box(Modifier.weight(1f).height(10.dp)) { + Box(Modifier.fillMaxWidth(rate / maxRate).fillMaxHeight() + .background(MaterialTheme.colorScheme.primary, RoundedCornerShape(5.dp))) + } + Text(stringResource(R.string.typing_heatmap_history_value, rate, week.touches), + Modifier.width(120.dp), style = MaterialTheme.typography.bodySmall) + } + } + } + + val confusions = layout.samples.entries + .flatMap { (code, list) -> list.filter { it.touchedCode != code }.map { code to it.touchedCode } } + .groupingBy { it }.eachCount() + .entries.sortedByDescending { it.value }.take(8) + if (confusions.isNotEmpty()) { + Text(stringResource(R.string.typing_heatmap_top_misses), style = MaterialTheme.typography.titleSmall) + confusions.forEach { (pair, count) -> + Text(stringResource(R.string.typing_heatmap_miss_line, cp(pair.first), cp(pair.second), count), + style = MaterialTheme.typography.bodyMedium) + } + } + + InfoCard(stringResource(R.string.typing_heatmap_privacy)) + OutlinedButton(onClick = { showClearDialog = true }) { + Text(stringResource(R.string.typing_heatmap_clear)) + } + } + } + if (showClearDialog) { + ConfirmationDialog( + onDismissRequest = { showClearDialog = false }, + onConfirmed = { + TypingHeatmap.clear(context) + refresh++ + }, + content = { Text(stringResource(R.string.typing_heatmap_clear_confirm)) } + ) + } +} + +private fun cp(code: Int) = if (Character.isValidCodePoint(code)) String(Character.toChars(code)) else "?" + +@Composable +private fun InfoCard(text: String) { + Card( + Modifier.fillMaxWidth(), + colors = CardDefaults.cardColors(containerColor = MaterialTheme.colorScheme.surfaceContainerHigh) + ) { + Text(text, Modifier.padding(16.dp), style = MaterialTheme.typography.bodyMedium) + } +} + +@Composable +private fun HeatmapCanvas(layout: TypingHeatmap.LayoutData, onlyMisses: Boolean, offsets: Map) { + val keyColor = MaterialTheme.colorScheme.surfaceContainerHighest + val keyBorder = MaterialTheme.colorScheme.outlineVariant + val labelColor = MaterialTheme.colorScheme.onSurface + val hitColor = Color(0xFF1E88E5) + val missColor = Color(0xFFE53935) + val averageColor = MaterialTheme.colorScheme.tertiary + val textMeasurer = rememberTextMeasurer() + val labelStyle = TextStyle(fontSize = 13.sp, fontWeight = FontWeight.Medium, color = labelColor) + val shapesByCode = remember(layout) { layout.shapes.associateBy { it.code } } + val aspect = layout.aspectRatio.takeIf { it > 0.05f } ?: 0.35f + val changedAreaColor = MaterialTheme.colorScheme.tertiary.copy(alpha = 0.3f) + // areas where adaptation picks another key than plain detection, in normalized coordinates + val changedAreas = remember(layout, offsets) { computeChangedAreas(layout, offsets, aspect) } + + Canvas(Modifier.fillMaxWidth().aspectRatio(1f / aspect)) { + val w = size.width + val h = size.height + val gap = 2.dp.toPx() + // keys + layout.shapes.forEach { key -> + val topLeft = Offset(key.x * w + gap, key.y * h + gap) + val keySize = Size(key.width * w - 2 * gap, key.height * h - 2 * gap) + if (keySize.width <= 0 || keySize.height <= 0) return@forEach + drawRoundRect(keyColor, topLeft, keySize, CornerRadius(6.dp.toPx())) + drawRoundRect(keyBorder, topLeft, keySize, CornerRadius(6.dp.toPx()), style = Stroke(1.dp.toPx())) + if (key.label.isNotEmpty() && key.label.length <= 3) { + val measured = textMeasurer.measure(key.label, labelStyle) + drawText(measured, topLeft = Offset( + key.centerX * w - measured.size.width / 2f, + key.centerY * h - measured.size.height / 2f + )) + } + } + // areas assigned to another key by adaptation + changedAreas.forEach { cell -> + drawRect(changedAreaColor, Offset(cell[0] * w, cell[1] * h), Size(cell[2] * w, cell[3] * h)) + } + // touches + val dotRadius = 2.5.dp.toPx() + layout.samples.forEach { (code, list) -> + val key = shapesByCode[code] ?: return@forEach + list.forEach { sample -> + val miss = sample.touchedCode != code + if (onlyMisses && !miss) return@forEach + val x = (key.centerX + sample.dx * key.width) * w + val y = (key.centerY + sample.dy * key.height) * h + drawCircle(if (miss) missColor.copy(alpha = 0.8f) else hitColor.copy(alpha = 0.45f), dotRadius, Offset(x, y)) + } + // average offset of all touches meant for this key + if (!onlyMisses && list.size >= MIN_SAMPLES_FOR_AVERAGE) { + val avgX = list.sumOf { it.dx.toDouble() }.toFloat() / list.size + val avgY = list.sumOf { it.dy.toDouble() }.toFloat() / list.size + val center = Offset(key.centerX * w, key.centerY * h) + val avg = Offset((key.centerX + avgX * key.width) * w, (key.centerY + avgY * key.height) * h) + drawLine(averageColor, center, avg, 2.dp.toPx()) + drawCircle(averageColor, 3.5.dp.toPx(), avg) + } + } + } +} + +/** Grid cells (x, y, width, height normalized) where adaptation assigns touches to another key */ +private fun computeChangedAreas(layout: TypingHeatmap.LayoutData, offsets: Map, aspect: Float): List { + if (offsets.isEmpty()) return emptyList() + val letters = layout.shapes.filter { Character.isLetter(it.code) && it.width > 0 && it.height > 0 } + if (letters.size < 2) return emptyList() + // use keyboard-proportional units, so distances are like on the real keyboard + val candidates = letters.map { TypingHeatmap.Candidate(it.code, it.x, it.y * aspect, it.width, it.height * aspect) } + val columns = 120 + val cellWidth = 1f / columns + val rows = (columns * aspect).toInt().coerceAtLeast(1) + val cellHeight = 1f / rows + val result = ArrayList() + for (row in 0 until rows) { + val y = (row + 0.5f) * cellHeight + for (column in 0 until columns) { + val x = (column + 0.5f) * cellWidth + // only inside the letter area + if (letters.none { x >= it.x && x < it.x + it.width && y >= it.y && y < it.y + it.height }) continue + if (TypingHeatmap.adaptedKeyCode(x, y * aspect, candidates, offsets) != null) + result.add(floatArrayOf(column * cellWidth, row * cellHeight, cellWidth, cellHeight)) + } + } + return result +} diff --git a/app/src/main/res/values-de/strings.xml b/app/src/main/res/values-de/strings.xml index dcfc0e668..895e0626e 100644 --- a/app/src/main/res/values-de/strings.xml +++ b/app/src/main/res/values-de/strings.xml @@ -1780,4 +1780,30 @@ Ermöglicht lokale GGUF / llama.cpp-Inferenz für Korrekturlesen und Umschreiben Löschen Sie den CJK-Benutzerverlauf Zurücksetzen der erlernten Kandidatenrankings und des Benutzerwörterbuch-Cache CJK-Benutzerverlauf gelöscht + Trefferzonen + An mein Tippen anpassen + Lernen, wo Du die Tasten triffst, und an Tastengrenzen die gemeinte Taste erkennen, auch für die Autokorrektur. Eine Taste wird angepasst, sobald sie 30 Anschläge hat. Nur lokal gespeichert, keine Cloud, keine Übertragung + Tipp-Heatmap + Noch keine Daten. Schalte \"An mein Tippen anpassen\" ein und tippe wie gewohnt ein paar Sätze. Mit Gesten oder im Inkognito-Modus getippte Wörter werden nicht aufgezeichnet. + Alle Anschläge + Nur Fehltreffer + %1$d Anschläge, %2$d Fehltreffer (%3$.1f %%) + Blau: Anschläge, die die gemeinte Taste getroffen haben. Rot: Anschläge auf einer anderen Taste. Linie und Punkt: mittlere Trefferstelle der Taste. + Häufigste Fehltreffer + %1$s gemeint, %2$s getroffen: %3$d× + Nur lokal auf diesem Gerät gespeichert: keine Cloud, keine Übertragung, nicht in Android-Sicherungen und nicht beim Umzug auf ein neues Gerät, und erst nach dem Entsperren lesbar. Die Daten verlassen das Gerät nur, wenn Du selbst eine Sicherung der Tastatur exportierst. In Passwortfeldern und im Inkognito-Modus wird nichts aufgezeichnet. Welche Taste gemeint war, ergibt sich aus dem Text, den Du am Ende verwendet hast: dem korrigierten Wort, dem gewählten Vorschlag, dem getippten Wort, wenn Du die Korrektur rückgängig gemacht hast, oder dem Buchstaben, den Du nach dem Löschen eines benachbarten getippt hast. Die Anschläge werden pro Taste ohne Reihenfolge gespeichert, Dein Text lässt sich daraus nicht rekonstruieren. „An mein Tippen anpassen“ auszuschalten beendet Sammeln und Anpassen, gelernte Daten bleiben, bis Du sie hier löschst. + Tipp-Daten löschen + Alle aufgezeichneten Tipp-Daten löschen? + „An mein Tippen anpassen“ ist aus. + Trefferzonen werden angepasst, sobald eine Taste %1$d Anschläge hat. + %1$d Tasten sind angepasst. Getönte Flächen: Hier wird ein Anschlag jetzt einer Nachbartaste zugeordnet. + Fehltreffer vor und nach der Anpassung + Noch zu wenig Daten für einen Vergleich (je mindestens %1$d Anschläge). + Vorher: %1$.1f %% (%2$d Anschläge), nachher: %3$.1f %% (%4$d Anschläge) + Diese Tipp-Daten wurden auf einem anderen Gerät gelernt (%1$s), dieses Gerät ist ein %2$s. Sie passen hier vielleicht nicht. Die Tastatur lernt in jedem Fall weiter und ersetzt alte Anschläge mit der Zeit. + Neu beginnen + Behalten + Tipp-Profil (Trefferzonen) + Fehltreffer pro Woche + %1$.1f %% (%2$d) diff --git a/app/src/main/res/values/strings.xml b/app/src/main/res/values/strings.xml index 7be95cae9..83c79ca74 100644 --- a/app/src/main/res/values/strings.xml +++ b/app/src/main/res/values/strings.xml @@ -2469,4 +2469,31 @@ New dictionary: Previous candidates Next candidates + + Key zones + Adapt to my typing + Learn where you touch the keys and recognize the key you meant near key borders, also for autocorrection. A key is adapted once it has 30 touches. Stored only locally, no cloud, no transfer + Typing heatmap + No data yet. Enable \"Adapt to my typing\" and type a few sentences as usual. Words typed with gestures or in incognito mode are not recorded. + All touches + Only misses + %1$d touches, %2$d misses (%3$.1f %%) + Blue: touches that hit the intended key. Red: touches that hit another key. Line and dot: average touch position for the key. + Most frequent misses + %1$s meant, %2$s hit: %3$d× + Stored only locally on this device: no cloud, no transfer, not included in Android backups or when moving to a new device, and only readable after unlocking the device. The data only leaves the device if you export a backup of the keyboard yourself. Nothing is recorded in password fields and incognito mode. The intended key is taken from the text you finally used: the autocorrected word or picked suggestion, what you typed if you reverted the correction, or the letter you typed after deleting a neighboring one. Touches are stored per key without their order, so your text cannot be reconstructed from them. Turning off \"Adapt to my typing\" stops collecting and adapting, learned data stays until you delete it here. + Delete typing data + Delete all recorded typing data? + \"Adapt to my typing\" is off. + Key zones are adapted as soon as a key has %1$d touches. + %1$d keys are adapted. Tinted areas: here a touch is now assigned to a neighboring key. + Misses before and after adaptation + Not enough data yet for a comparison (at least %1$d touches each). + Before: %1$.1f %% (%2$d touches), after: %3$.1f %% (%4$d touches) + This typing data was learned on another device (%1$s), this is %2$s. It may not fit here. The keyboard continues learning either way and replaces old touches over time. + Start over + Keep + Typing profile (key zones) + Misses per week + %1$.1f %% (%2$d) diff --git a/app/src/test/java/helium314/keyboard/latin/heatmap/TypingHeatmapTest.kt b/app/src/test/java/helium314/keyboard/latin/heatmap/TypingHeatmapTest.kt new file mode 100644 index 000000000..edffc4c61 --- /dev/null +++ b/app/src/test/java/helium314/keyboard/latin/heatmap/TypingHeatmapTest.kt @@ -0,0 +1,217 @@ +// SPDX-License-Identifier: GPL-3.0-only +package helium314.keyboard.latin.heatmap + +import org.junit.Assert.assertEquals +import org.junit.Assert.assertNull +import org.junit.Assert.assertTrue +import org.json.JSONArray +import org.json.JSONObject +import org.junit.Before +import org.junit.Test +import org.junit.runner.RunWith +import org.robolectric.RobolectricTestRunner +import org.robolectric.RuntimeEnvironment + +@RunWith(RobolectricTestRunner::class) +class TypingHeatmapTest { + // simple keyboard: one row "qwertzuiop", keys 100 px wide and 150 px high, row starts at y = 0 + private val row = "qwertzuiop" + private fun rect(cp: Int): IntArray? { + val i = row.indexOf(cp.toChar()) + return if (i < 0) null else intArrayOf(i * 100, 0, 100, 150) + } + private fun center(c: Char) = row.indexOf(c) * 100 + 50 + + private val geometry = object : TypingHeatmap.KeyGeometry { + override val layoutId = "test" + override fun keyRect(codePoint: Int) = rect(codePoint) + override fun snapshot() = row.mapIndexed { i, c -> TypingHeatmap.KeyShape(c.code, c.toString(), i / 10f, 0f, 0.1f, 1f) } + override val aspectRatio = 0.15f + override val source: Any = this + } + private val context get() = RuntimeEnvironment.getApplication() + + @Before fun setUp() = TypingHeatmap.resetForTest() + + @Test fun `align pairs matches and substitutions`() { + fun align(a: String, b: String) = TypingHeatmap.align(a.codePoints().toArray(), b.codePoints().toArray()) + assertEquals(listOf(0 to 0, 1 to 1, 2 to 2), align("tor", "tür")) + // missing letter in typed word: no pair for the inserted letter + assertEquals(listOf(0 to 0, 1 to 2), align("pt", "pot")) + // extra letter typed + assertEquals(listOf(0 to 0, 2 to 1), align("pqo", "po")) + // case does not matter + assertEquals(listOf(0 to 0, 1 to 1), align("Wo", "wO")) + } + + @Test fun `samples are relative to the intended key`() { + // "tir" typed, "tor" meant: the i touch is a miss for o + val xs = intArrayOf(center('t') + 10, center('i') + 30, center('r')) + val ys = intArrayOf(75, 75 + 15, 0) + val samples = TypingHeatmap.computeSamples("tir", "tor", xs, ys, keyRect = ::rect) + assertEquals(3, samples.size) + val (code, miss) = samples[1] + assertEquals('o'.code, code) + assertEquals('i'.code, miss.touchedCode) + assertEquals(-0.7f, miss.dx, 0.001f) // 70 px left of the o center + assertEquals(0.1f, miss.dy, 0.001f) + assertEquals(0.1f, samples[0].second.dx, 0.001f) + assertEquals(-0.5f, samples[2].second.dy, 0.001f) + } + + @Test fun `unrelated words and far touches are ignored`() { + val xs = intArrayOf(center('q'), center('w'), center('e')) + val ys = intArrayOf(75, 75, 75) + assertTrue(TypingHeatmap.computeSamples("qwe", "pou", xs, ys, keyRect = ::rect).isEmpty()) + // touch 3 keys away from intended key + val far = TypingHeatmap.computeSamples("ie", "te", intArrayOf(center('i'), center('e')), intArrayOf(75, 75), keyRect = ::rect) + assertEquals(1, far.size) + assertEquals('e'.code, far[0].first) + // no coordinates + assertTrue(TypingHeatmap.computeSamples("we", "we", intArrayOf(-1, -1), intArrayOf(-1, -1), keyRect = ::rect).isEmpty()) + } + + @Test fun `committed word is recorded when the next input follows`() { + val xs = intArrayOf(center('w'), center('o') - 80) // second touch hits i + TypingHeatmap.onWordCommitted(context, geometry, "wi", "wo", xs, intArrayOf(75, 75)) + assertNull(TypingHeatmap.layoutForTest("test")) + TypingHeatmap.finalizePending() + val layout = TypingHeatmap.layoutForTest("test")!! + assertEquals(2, layout.sampleCount) + assertEquals('i'.code, layout.samples['o'.code]!!.first().touchedCode) + } + + @Test fun `reverted autocorrection uses the typed word`() { + val xs = intArrayOf(center('w'), center('i')) + TypingHeatmap.onWordCommitted(context, geometry, "wi", "wo", xs, intArrayOf(75, 75)) + TypingHeatmap.onCommitReverted() + val layout = TypingHeatmap.layoutForTest("test")!! + assertNull(layout.samples['o'.code]) + assertEquals('i'.code, layout.samples['i'.code]!!.first().touchedCode) + } + + @Test fun `deleting into the committed word discards it`() { + TypingHeatmap.onWordCommitted(context, geometry, "wi", "wo", intArrayOf(center('w'), center('i')), intArrayOf(75, 75)) + TypingHeatmap.discardPending() + TypingHeatmap.finalizePending() + assertNull(TypingHeatmap.layoutForTest("test")) + } + + private fun samples(n: Int, dx: Float, dy: Float = 0f) = List(n) { TypingHeatmap.Sample(dx, dy, 'o'.code) } + + @Test fun `backspace without revert drops the committed word`() { + TypingHeatmap.onWordCommitted(context, geometry, "wi", "wo", intArrayOf(center('w'), center('i')), intArrayOf(75, 75)) + TypingHeatmap.onBackspace() // e.g. deleting the space and editing the word + TypingHeatmap.finalizePending() + assertNull(TypingHeatmap.layoutForTest("test")) + } + + @Test fun `backspace that reverts the autocorrection uses the typed word`() { + TypingHeatmap.onWordCommitted(context, geometry, "wi", "wo", intArrayOf(center('w'), center('i')), intArrayOf(75, 75)) + TypingHeatmap.onBackspace() + TypingHeatmap.onCommitReverted() + assertEquals('i'.code, TypingHeatmap.layoutForTest("test")!!.samples['i'.code]!!.first().touchedCode) + } + + @Test fun `touches and misses are counted per week`() { + TypingHeatmap.onWordCommitted(context, geometry, "wi", "wo", intArrayOf(center('w'), center('i')), intArrayOf(75, 75)) + TypingHeatmap.finalizePending() + val weeks = TypingHeatmap.getWeeklyStats(context) + assertEquals(1, weeks.size) + assertEquals(2, weeks[0].touches) + assertEquals(1, weeks[0].misses) + } + + @Test fun `offsets need enough data and are limited`() { + assertNull(TypingHeatmap.effectiveOffset(samples(29, 0.4f), 0.33f)) + val full = TypingHeatmap.effectiveOffset(samples(100, 0.4f), 0.33f)!! + assertEquals(0.33f, full[0], 0.001f) // limited to max shift + val small = TypingHeatmap.effectiveOffset(samples(100, 0.1f, -0.1f), 0.33f)!! + assertEquals(0.1f, small[0], 0.001f) + assertEquals(-0.1f, small[1], 0.001f) + // less weight with less data + val half = TypingHeatmap.effectiveOffset(samples(65, 0.2f), 0.33f)!! + assertTrue(half[0] > 0.05f && half[0] < 0.15f) + } + + @Test fun `adapted detection picks the key the user means`() { + val i = TypingHeatmap.Candidate('i'.code, 700f, 0f, 100f, 150f) + val o = TypingHeatmap.Candidate('o'.code, 800f, 0f, 100f, 150f) + // user hits o on average 30 % too far left + val offsets = mapOf('o'.code to floatArrayOf(-0.3f, 0f)) + // touch on i, close to the border: now meant for o + assertEquals('o'.code, TypingHeatmap.adaptedKeyCode(790f, 75f, listOf(i, o), offsets)) + // touch clearly on i: unchanged + assertNull(TypingHeatmap.adaptedKeyCode(760f, 75f, listOf(i, o), offsets)) + // no offsets: unchanged + assertNull(TypingHeatmap.adaptedKeyCode(790f, 75f, listOf(i, o), emptyMap())) + } + + @Test fun `retyped neighbor letter is a miss for the new letter`() { + // p deleted, o typed: the p touch was meant for o + val sample = TypingHeatmap.selfCorrectionSample('p'.code, center('p') - 40, 75, 'o'.code, keyRect = ::rect)!! + assertEquals('o'.code, sample.first) + assertEquals('p'.code, sample.second.touchedCode) + assertEquals(0.6f, sample.second.dx, 0.001f) + // keys far apart: not a typo + assertNull(TypingHeatmap.selfCorrectionSample('q'.code, center('q'), 75, 'p'.code, keyRect = ::rect)) + // same letter + assertNull(TypingHeatmap.selfCorrectionSample('o'.code, center('o'), 75, 'o'.code, keyRect = ::rect)) + } + + @Test fun `learning uses raw touches, not coordinates adjusted for the dictionary`() { + val raw = center('o') - 30 + val adjusted = TypingHeatmap.adjustForDictionary(raw, 75, 0, 0, 100, 150, floatArrayOf(-0.3f, 0f)) + assertEquals(center('o'), adjusted[0]) + TypingHeatmap.onWordCommitted(context, geometry, "wo", "wo", intArrayOf(center('w'), adjusted[0]), intArrayOf(75, adjusted[1]), true) + TypingHeatmap.finalizePending() + val sample = TypingHeatmap.layoutForTest("test")!!.samples['o'.code]!!.first() + assertEquals(-0.3f, sample.dx, 0.001f) + assertTrue(sample.adapted) + } + + @Test fun `raw touches are found with view padding`() { + // view coordinates are keyboard coordinates + 20 px padding; the word composer has keyboard coordinates + val padding = 20 + val rawKeyboardX = center('o') - 30 + val adjustedView = TypingHeatmap.adjustForDictionary(rawKeyboardX + padding, 75 + padding, -padding, -padding, 100, 150, floatArrayOf(-0.3f, 0f)) + val adjustedKeyboardX = adjustedView[0] - padding + val adjustedKeyboardY = adjustedView[1] - padding + TypingHeatmap.onWordCommitted(context, geometry, "wo", "wo", intArrayOf(center('w'), adjustedKeyboardX), intArrayOf(75, adjustedKeyboardY)) + TypingHeatmap.finalizePending() + assertEquals(-0.3f, TypingHeatmap.layoutForTest("test")!!.samples['o'.code]!!.first().dx, 0.001f) + } + + @Test fun `only letters are learned`() { + // "1" may have been typed on the symbols keyboard, its position says nothing about the letters + val samples = TypingHeatmap.computeSamples("a1", "a1", intArrayOf(10, 20), intArrayOf(75, 75)) { intArrayOf(0, 0, 100, 150) } + assertEquals(1, samples.size) + } + + @Test fun `broken or malicious data is ignored when reading`() { + val json = JSONObject(""" + {"version": 2, "device": "Test", + "layouts": [{"id": "qwertz", "aspect": "NaN", + "keys": [[111, "o", 0.8, 0, 0.1, 0.25], [112, "p", "NaN", 0, 0.1, 0.25], [113, "q", 0, 0, -1, 0.25]], + "samples": {"111": [0.1, 0.2, 111, 0, "NaN", 0, 105, 0, "Infinity", 0, 105, 0, 99, 0, 105, 0, 0.3, 0.1, -5, 0], + "-3": [0.1, 0.1, 111, 0], "abc": [0.1, 0.1, 111, 0]}}]} + """.trimIndent()) + val parsed = TypingHeatmap.parse(json) + val layout = parsed.layouts["qwertz"]!! + assertEquals(0.35f, layout.aspectRatio, 0.001f) + assertEquals(1, layout.shapes.size) // only the valid key + assertEquals(setOf('o'.code), layout.samples.keys) // invalid codes ignored + assertEquals(1, layout.samples['o'.code]!!.size) // NaN, Infinity, too far, invalid code point ignored + assertEquals("Test", parsed.device) + } + + @Test fun `too many samples are limited when reading`() { + val flat = JSONArray() + repeat(1000) { flat.put(0.1).put(0.1).put('o'.code).put(0) } + val layouts = JSONArray() + repeat(30) { layouts.put(JSONObject().put("id", "l$it").put("samples", JSONObject().put("111", flat))) } + val parsed = TypingHeatmap.parse(JSONObject().put("version", 2).put("layouts", layouts)) + assertTrue(parsed.layouts.size <= 16) + assertEquals(400, parsed.layouts.values.first().samples['o'.code]!!.size) + } +} From 1f187abf241be67d441f89c122894d43ca9277d9 Mon Sep 17 00:00:00 2001 From: 4di-hello <4di-hello@users.noreply.github.com> Date: Fri, 16 Oct 2026 12:41:50 +0200 Subject: [PATCH 2/2] Typing heatmap: readable layout names and a note on separate learning MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Show "Deutsch (Qwertz) · Landscape" instead of the internal id like "custom.Latn.1880uw1k6y. (landscape)", and explain that each way of holding the keyboard (normal, one-handed, floating, landscape) is learned separately. Display only, the stored data is unchanged. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01VuiaMhLXwhErf9shhrr1uD --- .../settings/screens/TypingHeatmapScreen.kt | 28 ++++++++++++++++++- app/src/main/res/values-de/strings.xml | 4 +++ app/src/main/res/values/strings.xml | 4 +++ 3 files changed, 35 insertions(+), 1 deletion(-) diff --git a/app/src/main/java/helium314/keyboard/settings/screens/TypingHeatmapScreen.kt b/app/src/main/java/helium314/keyboard/settings/screens/TypingHeatmapScreen.kt index 612dbb795..a30943710 100644 --- a/app/src/main/java/helium314/keyboard/settings/screens/TypingHeatmapScreen.kt +++ b/app/src/main/java/helium314/keyboard/settings/screens/TypingHeatmapScreen.kt @@ -48,6 +48,10 @@ import helium314.keyboard.latin.R import helium314.keyboard.latin.heatmap.TypingHeatmap import helium314.keyboard.latin.settings.Defaults import helium314.keyboard.latin.settings.Settings +import helium314.keyboard.latin.utils.SubtypeLocaleUtils +import helium314.keyboard.latin.utils.SubtypeSettings +import helium314.keyboard.latin.utils.locale +import helium314.keyboard.latin.utils.mainLayoutName import helium314.keyboard.latin.utils.prefs import helium314.keyboard.settings.SearchSettingsScreen import helium314.keyboard.settings.dialogs.ConfirmationDialog @@ -112,10 +116,11 @@ fun TypingHeatmapScreen(onClickBack: () -> Unit) { FilterChip( selected = it.id == layout.id, onClick = { selectedLayoutId = it.id }, - label = { Text(it.id) } + label = { Text(layoutLabel(it.id)) } ) } } + Text(stringResource(R.string.typing_heatmap_views), style = MaterialTheme.typography.bodySmall) } Row(horizontalArrangement = Arrangement.spacedBy(8.dp)) { FilterChip(selected = !onlyMisses, onClick = { onlyMisses = false }, @@ -214,6 +219,27 @@ fun TypingHeatmapScreen(onClickBack: () -> Unit) { } } +/** Readable name for a layout id like "custom.Latn.xyz. (landscape) (one-handed)", e.g. "Deutsch (Qwertz) · Querformat" */ +@Composable +private fun layoutLabel(id: String): String { + var main = id + val modes = mutableListOf() + for ((suffix, res) in listOf(" (floating)" to R.string.typing_heatmap_view_floating, + " (one-handed)" to R.string.typing_heatmap_view_one_handed, " (landscape)" to R.string.typing_heatmap_view_landscape)) { + if (main.endsWith(suffix)) { + main = main.removeSuffix(suffix) + modes.add(0, stringResource(res)) + } + } + val name = (SubtypeLocaleUtils.getMainLayoutDisplayName(main) ?: main).replaceFirstChar { it.uppercase() } + val language = remember(main) { + SubtypeSettings.getEnabledSubtypes().firstOrNull { it.mainLayoutName() == main } + ?.let { SubtypeLocaleUtils.getSubtypeLanguageDisplayName(it.locale()) } + } + val base = if (language == null || name.contains(language, ignoreCase = true)) name else "$language ($name)" + return (listOf(base) + modes).joinToString(" · ") +} + private fun cp(code: Int) = if (Character.isValidCodePoint(code)) String(Character.toChars(code)) else "?" @Composable diff --git a/app/src/main/res/values-de/strings.xml b/app/src/main/res/values-de/strings.xml index 895e0626e..717079c78 100644 --- a/app/src/main/res/values-de/strings.xml +++ b/app/src/main/res/values-de/strings.xml @@ -1785,6 +1785,10 @@ Ermöglicht lokale GGUF / llama.cpp-Inferenz für Korrekturlesen und Umschreiben Lernen, wo Du die Tasten triffst, und an Tastengrenzen die gemeinte Taste erkennen, auch für die Autokorrektur. Eine Taste wird angepasst, sobald sie 30 Anschläge hat. Nur lokal gespeichert, keine Cloud, keine Übertragung Tipp-Heatmap Noch keine Daten. Schalte \"An mein Tippen anpassen\" ein und tippe wie gewohnt ein paar Sätze. Mit Gesten oder im Inkognito-Modus getippte Wörter werden nicht aufgezeichnet. + Die Tastatur lernt für jede Ansicht getrennt, weil Du sie unterschiedlich hältst: normal, Einhand, schwebend und Querformat. + Einhand + Schwebend + Querformat Alle Anschläge Nur Fehltreffer %1$d Anschläge, %2$d Fehltreffer (%3$.1f %%) diff --git a/app/src/main/res/values/strings.xml b/app/src/main/res/values/strings.xml index 83c79ca74..17bd0028e 100644 --- a/app/src/main/res/values/strings.xml +++ b/app/src/main/res/values/strings.xml @@ -2475,6 +2475,10 @@ New dictionary: Learn where you touch the keys and recognize the key you meant near key borders, also for autocorrection. A key is adapted once it has 30 touches. Stored only locally, no cloud, no transfer Typing heatmap No data yet. Enable \"Adapt to my typing\" and type a few sentences as usual. Words typed with gestures or in incognito mode are not recorded. + The keyboard learns separately for each way you hold it: normal, one-handed, floating and landscape. + One-handed + Floating + Landscape All touches Only misses %1$d touches, %2$d misses (%3$.1f %%)