diff --git a/ios/Podfile.lock b/ios/Podfile.lock index 150f47b..285aef0 100644 --- a/ios/Podfile.lock +++ b/ios/Podfile.lock @@ -1,6 +1,8 @@ PODS: - app_links (0.0.2): - Flutter + - camera_avfoundation (0.0.1): + - Flutter - disk_space_plus (0.0.1): - Flutter - Firebase/CoreOnly (12.0.0): @@ -250,6 +252,7 @@ PODS: DEPENDENCIES: - app_links (from `.symlinks/plugins/app_links/ios`) + - camera_avfoundation (from `.symlinks/plugins/camera_avfoundation/ios`) - disk_space_plus (from `.symlinks/plugins/disk_space_plus/ios`) - firebase_analytics (from `.symlinks/plugins/firebase_analytics/ios`) - firebase_core (from `.symlinks/plugins/firebase_core/ios`) @@ -309,6 +312,8 @@ SPEC REPOS: EXTERNAL SOURCES: app_links: :path: ".symlinks/plugins/app_links/ios" + camera_avfoundation: + :path: ".symlinks/plugins/camera_avfoundation/ios" disk_space_plus: :path: ".symlinks/plugins/disk_space_plus/ios" firebase_analytics: @@ -356,6 +361,7 @@ EXTERNAL SOURCES: SPEC CHECKSUMS: app_links: 76b66b60cc809390ca1ad69bfd66b998d2387ac7 + camera_avfoundation: be3be85408cd4126f250386828e9b1dfa40ab436 disk_space_plus: a36391fb5f732dbdd29628b0bac5a1acbd43aaef Firebase: 800d487043c0557d9faed71477a38d9aafb08a41 firebase_analytics: cd56fc56f75c1df30a6ff5290cd56e230996a76d diff --git a/lib/ui/painters/ocr_painter.dart b/lib/ui/painters/ocr_painter.dart new file mode 100644 index 0000000..71a8a59 --- /dev/null +++ b/lib/ui/painters/ocr_painter.dart @@ -0,0 +1,49 @@ +import 'dart:ui'; + +import 'package:flutter/material.dart'; +import 'package:sagase/datamodels/recognized_text_block.dart'; + +class OcrPainter extends CustomPainter { + final List? recognizedTextBlocks; + final Size imageSize; + + OcrPainter(this.recognizedTextBlocks, this.imageSize); + + @override + void paint(Canvas canvas, Size size) { + if (recognizedTextBlocks != null) { + final unselectedPaint = Paint() + ..style = PaintingStyle.stroke + ..strokeWidth = 4 + ..color = Colors.lightBlueAccent; + + final selectedPaint = Paint() + ..style = PaintingStyle.stroke + ..strokeWidth = 4 + ..color = Colors.lightGreenAccent; + + for (final textBlock in recognizedTextBlocks!) { + final List cornerPoints = []; + for (final point in textBlock.points) { + double x = point.x.toDouble() * size.width / imageSize.width; + double y = point.y.toDouble() * size.height / imageSize.height; + cornerPoints.add(Offset(x, y)); + } + + textBlock.offsets = List.from(cornerPoints); + + cornerPoints.add(cornerPoints.first); + canvas.drawPoints( + PointMode.polygon, + cornerPoints, + textBlock.selected ? selectedPaint : unselectedPaint, + ); + } + } + } + + @override + bool shouldRepaint(OcrPainter oldDelegate) { + return oldDelegate.recognizedTextBlocks != recognizedTextBlocks; + } +} diff --git a/lib/ui/views/ocr/ocr_view.dart b/lib/ui/views/ocr/ocr_view.dart index f4dd94a..1570763 100644 --- a/lib/ui/views/ocr/ocr_view.dart +++ b/lib/ui/views/ocr/ocr_view.dart @@ -1,10 +1,13 @@ import 'package:flutter/material.dart'; +import 'package:flutter/services.dart'; +import 'package:flutter_hooks/flutter_hooks.dart'; import 'package:sagase/ui/widgets/list_item_loading.dart'; +import 'package:sagase/ui/widgets/ocr_image.dart'; import 'package:shimmer/shimmer.dart'; import 'package:stacked/stacked.dart'; +import 'package:stacked_hooks/stacked_hooks.dart'; import 'ocr_viewmodel.dart'; -import 'painters/text_detector_painter.dart'; class OcrView extends StackedView { final bool cameraStart; @@ -15,44 +18,80 @@ class OcrView extends StackedView { OcrViewModel viewModelBuilder(context) => OcrViewModel(cameraStart); @override - Widget builder(BuildContext context, OcrViewModel viewModel, Widget? child) { + Widget builder(context, viewModel, child) => const _Body(); +} + +class _Body extends StackedHookView { + const _Body(); + + @override + Widget builder(BuildContext context, OcrViewModel viewModel) { + final controller = useTextEditingController(); + return Scaffold( appBar: AppBar( title: Text('Character Detection'), - actions: viewModel.currentImageBytes == null + actions: viewModel.state == OcrState.waiting || + viewModel.state == OcrState.loading ? null : [ IconButton( - onPressed: viewModel.openCamera, + onPressed: () { + viewModel.openCamera(); + controller.clear(); + }, icon: Icon(Icons.camera), ), IconButton( - onPressed: viewModel.selectImage, + onPressed: () { + viewModel.selectImage(); + controller.clear(); + }, icon: Icon(Icons.photo), ), ], ), - body: Column( - children: [ - Expanded( - child: viewModel.currentImageBytes == null - ? _OcrImageLoading() - : _OcrImage(), - ), - const Divider(indent: 8, endIndent: 8), - Expanded( - child: viewModel.recognizedTextBlocks == null - ? Column( - children: [ - ListItemLoading(), - const SizedBox(height: 8), - ListItemLoading(), - ], - ) - : _SelectedText(), - ), - ], - ), + body: viewModel.state == OcrState.error + ? _Error(controller) + : Column( + children: [ + Expanded( + child: viewModel.image == null + ? _OcrImageLoading() + : OcrImage( + key: ValueKey(viewModel.image!.path.hashCode), + image: viewModel.image!, + onImageProcessed: viewModel.handleImageProcessed, + onImageError: viewModel.handleImageError, + onTextSelected: (text) { + controller.text = controller.text + text; + viewModel.handleTextSelected(); + }, + locked: false, + singleSelection: false, + ), + ), + const Divider(indent: 8, endIndent: 8), + Expanded( + child: switch (viewModel.state) { + OcrState.viewEmpty => const Center( + child: Text( + 'No text found in the image', + style: TextStyle(fontSize: 18), + ), + ), + OcrState.viewing => _SelectedText(controller), + _ => Column( + children: [ + ListItemLoading(), + const SizedBox(height: 8), + ListItemLoading(), + ], + ) + }, + ), + ], + ), ); } } @@ -80,97 +119,99 @@ class _OcrImageLoading extends StatelessWidget { } } -class _OcrImage extends ViewModelWidget { +class _SelectedText extends ViewModelWidget { + final TextEditingController controller; + + const _SelectedText(this.controller); + @override Widget build(BuildContext context, OcrViewModel viewModel) { - return Center( - child: Container( - padding: const EdgeInsets.all(8), - child: GestureDetector( - onTap: () => viewModel.rebuildUi(), - child: CustomPaint( - foregroundPainter: viewModel.recognizedTextBlocks == null - ? null - : TextRecognizerPainter( - viewModel.recognizedTextBlocks!, - viewModel.imageSize, + return GestureDetector( + onVerticalDragStart: (_) => FocusScope.of(context).unfocus(), + child: Column( + children: [ + Expanded( + child: SafeArea( + bottom: false, + child: Padding( + padding: const EdgeInsets.all(8), + child: TextField( + controller: controller, + maxLines: null, + style: const TextStyle(fontSize: 24), + decoration: const InputDecoration.collapsed( + hintText: 'Select text from the image...', ), - child: IgnorePointer( - child: Image.memory(viewModel.currentImageBytes!), + maxLength: 1000, + inputFormatters: [LengthLimitingTextInputFormatter(1000)], + ), + ), ), ), - ), + AnimatedCrossFade( + crossFadeState: controller.text.isEmpty + ? CrossFadeState.showFirst + : CrossFadeState.showSecond, + duration: const Duration(milliseconds: 200), + firstChild: SizedBox.shrink(), + secondChild: Container( + padding: EdgeInsets.only( + bottom: MediaQuery.of(context).padding.bottom, + ), + width: double.infinity, + color: Colors.deepPurple, + child: TextButton.icon( + icon: const Icon(Icons.text_snippet, color: Colors.white), + label: const Text( + 'Analyze', + style: TextStyle(color: Colors.white), + ), + onPressed: () => viewModel.analyzeText(controller.text), + ), + ), + ), + ], ), ); } } -class _SelectedText extends ViewModelWidget { +class _Error extends ViewModelWidget { + final TextEditingController controller; + + const _Error(this.controller); + @override Widget build(BuildContext context, OcrViewModel viewModel) { - if (viewModel.recognizedTextBlocks!.isEmpty) { - return Center(child: Text('No Japanese text was found')); - } - - late Widget textSection; - - if (viewModel.recognizedTextBlocks!.length == 1) { - textSection = SelectionArea( - child: SingleChildScrollView( - child: Container( - width: double.infinity, - padding: const EdgeInsets.all(8), - child: Text( - viewModel.recognizedTextBlocks![0].text, - textAlign: TextAlign.left, - style: TextStyle(fontSize: 16), - ), + return Center( + child: Column( + mainAxisSize: MainAxisSize.min, + children: [ + const Text( + 'Failed to analyze image', + style: TextStyle(fontSize: 18), ), - ), - ); - } else { - textSection = ReorderableListView.builder( - itemCount: viewModel.recognizedTextBlocks!.length, - onReorder: viewModel.reorderList, - itemBuilder: (context, index) { - final current = viewModel.recognizedTextBlocks![index]; - return ListTile( - key: ValueKey(current), - leading: Checkbox( - value: current.selected, - onChanged: (value) { - if (value == null) return; - viewModel.toggleCheckBox(index, value); - }, - ), - title: Text(current.text), - trailing: Icon(Icons.drag_indicator), - ); - }, - ); - } - - return Column( - children: [ - Expanded(child: textSection), - Container( - padding: EdgeInsets.only( - bottom: MediaQuery.of(context).padding.bottom, + const SizedBox(height: 8), + const Text('Please try again'), + const SizedBox(height: 16), + ElevatedButton.icon( + onPressed: () { + viewModel.openCamera(); + controller.clear(); + }, + icon: const Icon(Icons.camera), + label: const Text('Open camera'), ), - width: double.infinity, - color: Colors.deepPurple, - child: TextButton.icon( - icon: const Icon(Icons.text_snippet, color: Colors.white), - label: Text( - viewModel.recognizedTextBlocks!.length == 1 - ? 'Analyze text' - : 'Analyze selected text', - style: TextStyle(color: Colors.white), - ), - onPressed: viewModel.analyzeSelectedText, + ElevatedButton.icon( + onPressed: () { + viewModel.selectImage(); + controller.clear(); + }, + icon: const Icon(Icons.photo), + label: const Text('Pick from photos'), ), - ), - ], + ], + ), ); } } diff --git a/lib/ui/views/ocr/ocr_viewmodel.dart b/lib/ui/views/ocr/ocr_viewmodel.dart index c447e6d..d899179 100644 --- a/lib/ui/views/ocr/ocr_viewmodel.dart +++ b/lib/ui/views/ocr/ocr_viewmodel.dart @@ -1,36 +1,19 @@ -import 'dart:io'; - -import 'package:flutter/painting.dart'; -import 'package:flutter/services.dart'; -import 'package:google_mlkit_text_recognition/google_mlkit_text_recognition.dart'; import 'package:image_picker/image_picker.dart'; -import 'package:kana_kit/kana_kit.dart'; -import 'package:path/path.dart' as path; -import 'package:path_provider/path_provider.dart' as path_provider; import 'package:sagase/app/app.locator.dart'; -import 'package:sagase/datamodels/recognized_text_block.dart'; -import 'package:sagase/utils/constants.dart' as constants; import 'package:stacked/stacked.dart'; import 'package:stacked_services/stacked_services.dart'; -import 'package:flutter_exif_rotation/flutter_exif_rotation.dart'; class OcrViewModel extends BaseViewModel { final _navigationService = locator(); final _snackbarService = locator(); - final _kanaKit = const KanaKit(); - final ImagePicker _imagePicker = ImagePicker(); - final _textRecognizer = - TextRecognizer(script: TextRecognitionScript.japanese); - Uint8List? _currentImageBytes; - Uint8List? get currentImageBytes => _currentImageBytes; - late Size _imageSize; - Size get imageSize => _imageSize; + OcrState _state = OcrState.waiting; + OcrState get state => _state; - List? _recognizedTextBlocks; - List? get recognizedTextBlocks => _recognizedTextBlocks; + XFile? _image; + XFile? get image => _image; OcrViewModel(bool cameraStart) { if (cameraStart) { @@ -49,14 +32,12 @@ class OcrViewModel extends BaseViewModel { } Future _processImage(ImageSource imageSource) async { - _currentImageBytes = null; - _recognizedTextBlocks = null; + _image = null; + _state = OcrState.waiting; rebuildUi(); - XFile? image; - try { - image = await _imagePicker.pickImage(source: imageSource); + _image = await _imagePicker.pickImage(source: imageSource); } catch (_) { _snackbarService.showSnackbar( message: 'Failed to open camera or gallery', @@ -68,89 +49,41 @@ class OcrViewModel extends BaseViewModel { return; } - final ocrImageDir = path.join( - (await path_provider.getApplicationCacheDirectory()).path, - constants.ocrImagesDir, - ); - await Directory(ocrImageDir).create(); - - final imagePath = path.join(ocrImageDir, image.name); - try { - await File(image.path).rename(imagePath); - } catch (_) { - if (File(image.path).existsSync()) { - File(image.path).delete(); - } - _snackbarService.showSnackbar(message: 'Failed to open image'); - _navigationService.back(); - return; - } - - final rotatedImage = await FlutterExifRotation.rotateImage(path: imagePath); - - final inputImage = InputImage.fromFilePath(rotatedImage.path); + _state = OcrState.loading; - _currentImageBytes = await rotatedImage.readAsBytes(); rebuildUi(); + } - if (inputImage.metadata != null) { - _imageSize = inputImage.metadata!.size; + void handleImageProcessed(int length) { + if (length == 0) { + _state = OcrState.viewEmpty; } else { - final decodedImage = await decodeImageFromList(_currentImageBytes!); - _imageSize = Size( - decodedImage.width.toDouble(), - decodedImage.height.toDouble(), - ); - } - - final recognizedText = await _textRecognizer.processImage(inputImage); - - _recognizedTextBlocks = []; - for (final textBlock in recognizedText.blocks) { - if (_kanaKit.isRomaji(textBlock.text)) continue; - _recognizedTextBlocks!.add( - RecognizedTextBlock( - text: textBlock.text, - points: textBlock.cornerPoints, - ), - ); + _state = OcrState.viewing; } - rebuildUi(); - - await rotatedImage.delete(); } - void reorderList(int oldIndex, int newIndex) { - if (oldIndex < newIndex) newIndex -= 1; - - _recognizedTextBlocks!.insert( - newIndex, - _recognizedTextBlocks!.removeAt(oldIndex), - ); + void handleImageError() { + _image = null; + _state = OcrState.error; rebuildUi(); } - void toggleCheckBox(int index, bool value) { - _recognizedTextBlocks![index].selected = value; + void handleTextSelected() { rebuildUi(); } - void analyzeSelectedText() { - if (_recognizedTextBlocks == null) return; - if (_recognizedTextBlocks!.length == 1) { - _navigationService.back(result: _recognizedTextBlocks![0].text); - return; - } - - List lines = []; - - for (final textBlock in _recognizedTextBlocks!) { - if (textBlock.selected) lines.add(textBlock.text); - } + void analyzeText(String text) { + if (text.isEmpty) return; - if (lines.isEmpty) return; - - _navigationService.back(result: lines.join('\n')); + _navigationService.back(result: text); } } + +enum OcrState { + waiting, + loading, + viewing, + viewEmpty, + error, +} diff --git a/lib/ui/views/ocr/painters/text_detector_painter.dart b/lib/ui/views/ocr/painters/text_detector_painter.dart deleted file mode 100644 index d385cb6..0000000 --- a/lib/ui/views/ocr/painters/text_detector_painter.dart +++ /dev/null @@ -1,101 +0,0 @@ -import 'dart:math'; -import 'dart:ui'; - -import 'package:flutter/material.dart'; -import 'package:sagase/datamodels/recognized_text_block.dart'; - -class TextRecognizerPainter extends CustomPainter { - final List recognizedTextBlocks; - final Size imageSize; - - TextRecognizerPainter(this.recognizedTextBlocks, this.imageSize); - - @override - void paint(Canvas canvas, Size size) { - final unselectedPaint = Paint() - ..style = PaintingStyle.stroke - ..strokeWidth = 2 - ..color = Colors.blueGrey; - - final selectedPaint = Paint() - ..style = PaintingStyle.stroke - ..strokeWidth = 2 - ..color = Colors.lightGreenAccent; - - for (final textBlock in recognizedTextBlocks) { - final List cornerPoints = []; - for (final point in textBlock.points) { - double x = point.x.toDouble() * size.width / imageSize.width; - double y = point.y.toDouble() * size.height / imageSize.height; - cornerPoints.add(Offset(x, y)); - } - - textBlock.offsets = List.from(cornerPoints); - - cornerPoints.add(cornerPoints.first); - canvas.drawPoints( - PointMode.polygon, - cornerPoints, - textBlock.selected ? selectedPaint : unselectedPaint, - ); - } - } - - @override - bool shouldRepaint(TextRecognizerPainter oldDelegate) { - //TODO the painting isn't that intense so probably fine? - return true; - // return oldDelegate.recognizedTextBlocks != recognizedTextBlocks; - } - - @override - bool? hitTest(Offset position) { - for (final textBlock in recognizedTextBlocks) { - if (_pointInsideRectangle(position, textBlock.offsets)) { - textBlock.selected = true; - return true; - } - } - - return false; - } - - bool _pointInsideRectangle(Offset point, List rectCorners) { - double x1 = rectCorners[0].dx; - double x2 = rectCorners[1].dx; - double x3 = rectCorners[2].dx; - double x4 = rectCorners[3].dx; - - double y1 = rectCorners[0].dy; - double y2 = rectCorners[1].dy; - double y3 = rectCorners[2].dy; - double y4 = rectCorners[3].dy; - - double a1 = sqrt((x1 - x2) * (x1 - x2) + (y1 - y2) * (y1 - y2)); - double a2 = sqrt((x2 - x3) * (x2 - x3) + (y2 - y3) * (y2 - y3)); - double a3 = sqrt((x3 - x4) * (x3 - x4) + (y3 - y4) * (y3 - y4)); - double a4 = sqrt((x4 - x1) * (x4 - x1) + (y4 - y1) * (y4 - y1)); - - double b1 = sqrt( - (x1 - point.dx) * (x1 - point.dx) + (y1 - point.dy) * (y1 - point.dy)); - double b2 = sqrt( - (x2 - point.dx) * (x2 - point.dx) + (y2 - point.dy) * (y2 - point.dy)); - double b3 = sqrt( - (x3 - point.dx) * (x3 - point.dx) + (y3 - point.dy) * (y3 - point.dy)); - double b4 = sqrt( - (x4 - point.dx) * (x4 - point.dx) + (y4 - point.dy) * (y4 - point.dy)); - - double u1 = (a1 + b1 + b2) / 2; - double u2 = (a2 + b2 + b3) / 2; - double u3 = (a3 + b3 + b4) / 2; - double u4 = (a4 + b4 + b1) / 2; - - double area1 = sqrt(u1 * (u1 - a1) * (u1 - b1) * (u1 - b2)); - double area2 = sqrt(u2 * (u2 - a2) * (u2 - b2) * (u2 - b3)); - double area3 = sqrt(u3 * (u3 - a3) * (u3 - b3) * (u3 - b4)); - double area4 = sqrt(u4 * (u4 - a4) * (u4 - b4) * (u4 - b1)); - - double difference = 0.95 * (area1 + area2 + area3 + area4) - a1 * a2; - return difference < 1; - } -} diff --git a/lib/ui/views/search/search_view.dart b/lib/ui/views/search/search_view.dart index 98191d7..e08a74a 100644 --- a/lib/ui/views/search/search_view.dart +++ b/lib/ui/views/search/search_view.dart @@ -15,6 +15,8 @@ import 'package:stacked_hooks/stacked_hooks.dart'; import 'search_viewmodel.dart'; import 'widgets/analysis_prompt.dart'; +import 'widgets/hand_writing_input.dart'; +import 'widgets/ocr_widget.dart'; class SearchView extends StatelessWidget { const SearchView({super.key}); @@ -51,137 +53,17 @@ class _Body extends StackedHookView { viewModel.searchResult == null ? _SearchHistory(searchController) : const _SearchResults(), - if (viewModel.showHandWriting) - Expanded( - child: Column( - children: [ - const Divider(height: 1, thickness: 1), - SizedBox( - height: 40, - child: Row( - children: [ - Expanded( - child: ListView.builder( - scrollDirection: Axis.horizontal, - itemCount: viewModel.handWritingResult.length, - itemBuilder: (context, index) { - return GestureDetector( - onTap: () { - int cursorPosition = - searchController.selection.base.offset; - if (cursorPosition == 0) { - searchController.text = - viewModel.handWritingResult[index] + - searchController.text; - } else { - searchController.text = searchController - .text - .substring(0, cursorPosition) + - viewModel.handWritingResult[index] + - searchController.text - .substring(cursorPosition); - } - - searchController.selection = - TextSelection.fromPosition(TextPosition( - offset: cursorPosition + - viewModel - .handWritingResult[index].length, - )); - handWritingController.clear(); - viewModel - .searchOnChange(searchController.text); - }, - child: Container( - padding: const EdgeInsets.symmetric( - horizontal: 13, - ), - decoration: const BoxDecoration( - border: Border( - right: BorderSide(color: Colors.black), - ), - ), - child: Center( - child: Text( - viewModel.handWritingResult[index], - ), - ), - ), - ); - }, - ), - ), - const VerticalDivider(width: 1, thickness: 1), - IconButton( - onPressed: () { - viewModel.toggleHandWriting(); - keyboardFocusNode.requestFocus(); - }, - icon: const Icon(Icons.text_fields), - splashColor: Colors.transparent, - highlightColor: Colors.transparent, - ), - IconButton( - onPressed: viewModel.toggleHandWriting, - icon: const Icon(Icons.keyboard_hide), - splashColor: Colors.transparent, - highlightColor: Colors.transparent, - ), - ], - ), - ), - const Divider(height: 1, thickness: 1), - Expanded( - child: HandWritingCanvas( - onHandWritingChanged: viewModel.recognizeWriting, - controller: handWritingController, - ), - ), - const Divider(indent: 16, endIndent: 16, height: 1), - Padding( - padding: EdgeInsets.only( - bottom: MediaQuery.of(context).padding.bottom, - ), - child: Row( - mainAxisAlignment: MainAxisAlignment.spaceEvenly, - children: [ - IconButton( - onPressed: () => handWritingController.undo(), - icon: const Icon(Icons.undo), - splashColor: Colors.transparent, - highlightColor: Colors.transparent, - ), - IconButton( - onPressed: () => handWritingController.clear(), - icon: const Icon(Icons.delete), - splashColor: Colors.transparent, - highlightColor: Colors.transparent, - ), - IconButton( - onPressed: () { - int cursorPosition = - searchController.selection.base.offset; - if (cursorPosition != 0) { - searchController.text = searchController.text - .substring(0, cursorPosition - 1) + - searchController.text - .substring(cursorPosition); - - searchController.selection = - TextSelection.fromPosition( - TextPosition(offset: cursorPosition - 1)); - viewModel.searchOnChange(searchController.text); - } - }, - icon: const Icon(Icons.backspace), - splashColor: Colors.transparent, - highlightColor: Colors.transparent, - ), - ], - ), - ), - ], - ), + if (viewModel.inputMode == InputMode.handWriting) + HandWritingInput( + searchController: searchController, + handWritingController: handWritingController, + keyboardFocusNode: keyboardFocusNode, + ), + if (viewModel.inputMode == InputMode.ocr) + OcrWidget( + searchController: searchController, + handWritingController: handWritingController, + keyboardFocusNode: keyboardFocusNode, ), ], ), @@ -213,12 +95,23 @@ class _SearchTextField extends ViewModelWidget { focusNode: keyboardFocusNode, displayArrows: false, toolbarButtons: [ + // Open OCR and dismiss keyboard + (node) { + return IconButton( + onPressed: () { + node.unfocus(); + viewModel.setInputMode(InputMode.ocr); + handWritingFocusNode.requestFocus(); + }, + icon: const Icon(Icons.camera_alt), + ); + }, // Open hand writing and dismiss keyboard (node) { return IconButton( onPressed: () { node.unfocus(); - viewModel.toggleHandWriting(); + viewModel.setInputMode(InputMode.handWriting); handWritingFocusNode.requestFocus(); }, icon: const Icon(Icons.draw), @@ -228,7 +121,7 @@ class _SearchTextField extends ViewModelWidget { (node) { return IconButton( onPressed: node.unfocus, - icon: const Icon(Icons.keyboard_hide), + icon: const Icon(Icons.close), ); }, ], @@ -241,126 +134,72 @@ class _SearchTextField extends ViewModelWidget { child: Row( children: [ Expanded( - child: viewModel.showHandWriting - ? TextField( - autofocus: true, - readOnly: true, - showCursor: true, - autocorrect: false, - enableIMEPersonalizedLearning: false, - maxLines: 1, - focusNode: handWritingFocusNode, - controller: searchController, - inputFormatters: [ - LengthLimitingTextInputFormatter(1000), - FilteringTextInputFormatter.deny( - RegExp(r'‘|’'), - replacementString: '\'', - ), - FilteringTextInputFormatter.deny( - RegExp(r'“|”'), - replacementString: '"', - ), - ], - decoration: InputDecoration( - hintText: viewModel.searchFilter.displayTitle, - contentPadding: const EdgeInsets.symmetric( - horizontal: 24, - ), - enabledBorder: OutlineInputBorder( - borderRadius: BorderRadius.circular(24), - borderSide: const BorderSide( - color: Colors.transparent, - ), - ), - focusedBorder: OutlineInputBorder( - borderRadius: BorderRadius.circular(24), - borderSide: const BorderSide( - color: Colors.transparent, - ), - ), - border: OutlineInputBorder( - borderRadius: BorderRadius.circular(24), - borderSide: const BorderSide( - color: Colors.transparent, - ), - ), - filled: true, - fillColor: Theme.of(context).scaffoldBackgroundColor, - suffixIcon: IconButton( - onPressed: () { - viewModel.searchOnChange(''); - searchController.clear(); - handWritingFocusNode.requestFocus(); - }, - icon: Icon( - Icons.clear, - color: Theme.of(context).iconTheme.color, - ), - splashColor: Colors.transparent, - highlightColor: Colors.transparent, - ), - ), - ) - : TextField( - autocorrect: false, - enableIMEPersonalizedLearning: false, - maxLines: 1, - textInputAction: TextInputAction.done, - focusNode: keyboardFocusNode, - controller: searchController, - onChanged: viewModel.searchOnChange, - inputFormatters: [ - LengthLimitingTextInputFormatter(1000), - FilteringTextInputFormatter.deny( - RegExp(r'‘|’'), - replacementString: '\'', - ), - FilteringTextInputFormatter.deny( - RegExp(r'“|”'), - replacementString: '"', - ), - ], - decoration: InputDecoration( - hintText: viewModel.searchFilter.displayTitle, - contentPadding: const EdgeInsets.symmetric( - horizontal: 24, - ), - enabledBorder: OutlineInputBorder( - borderRadius: BorderRadius.circular(24), - borderSide: const BorderSide( - color: Colors.transparent, - ), - ), - focusedBorder: OutlineInputBorder( - borderRadius: BorderRadius.circular(24), - borderSide: const BorderSide( - color: Colors.transparent, - ), - ), - border: OutlineInputBorder( - borderRadius: BorderRadius.circular(24), - borderSide: const BorderSide( - color: Colors.transparent, - ), - ), - filled: true, - fillColor: Theme.of(context).scaffoldBackgroundColor, - suffixIcon: IconButton( - onPressed: () { - viewModel.searchOnChange(''); - searchController.clear(); - keyboardFocusNode.requestFocus(); - }, - icon: Icon( - Icons.clear, - color: Theme.of(context).iconTheme.color, - ), - splashColor: Colors.transparent, - highlightColor: Colors.transparent, - ), - ), + child: TextField( + autofocus: viewModel.inputMode != InputMode.text, + readOnly: viewModel.inputMode != InputMode.text, + showCursor: true, + autocorrect: false, + enableIMEPersonalizedLearning: false, + maxLines: 1, + textInputAction: viewModel.inputMode != InputMode.text + ? null + : TextInputAction.done, + focusNode: viewModel.inputMode != InputMode.text + ? handWritingFocusNode + : keyboardFocusNode, + controller: searchController, + onChanged: viewModel.searchOnChange, + inputFormatters: [ + LengthLimitingTextInputFormatter(1000), + FilteringTextInputFormatter.deny( + RegExp(r'‘|’'), + replacementString: '\'', + ), + FilteringTextInputFormatter.deny( + RegExp(r'“|”'), + replacementString: '"', + ), + ], + decoration: InputDecoration( + hintText: viewModel.searchFilter.displayTitle, + contentPadding: const EdgeInsets.symmetric( + horizontal: 24, + ), + enabledBorder: OutlineInputBorder( + borderRadius: BorderRadius.circular(24), + borderSide: const BorderSide( + color: Colors.transparent, + ), + ), + focusedBorder: OutlineInputBorder( + borderRadius: BorderRadius.circular(24), + borderSide: const BorderSide( + color: Colors.transparent, ), + ), + border: OutlineInputBorder( + borderRadius: BorderRadius.circular(24), + borderSide: const BorderSide( + color: Colors.transparent, + ), + ), + filled: true, + fillColor: Theme.of(context).scaffoldBackgroundColor, + suffixIcon: IconButton( + onPressed: () { + viewModel.searchOnChange(''); + searchController.clear(); + handWritingFocusNode.requestFocus(); + }, + icon: Icon( + Icons.clear, + color: Theme.of(context).iconTheme.color, + ), + splashColor: Colors.transparent, + highlightColor: Colors.transparent, + ), + ), + ), ), IconButton( onPressed: viewModel.setSearchFilter, @@ -393,7 +232,6 @@ class _SearchResults extends ViewModelWidget { ); } - // Show results return Expanded( child: ListView.separated( key: UniqueKey(), @@ -458,10 +296,11 @@ class _SearchHistory extends ViewModelWidget { onPressed: () async { final cdata = await Clipboard.getData(Clipboard.kTextPlain); if (cdata?.text != null) { - searchController.text = cdata!.text!; + final text = cdata!.text!.replaceAll('\n', ''); + searchController.text = text; searchController.selection = TextSelection.fromPosition( - TextPosition(offset: cdata.text!.length)); - viewModel.searchOnChange(cdata.text!); + TextPosition(offset: text.length)); + viewModel.searchOnChange(text); } }, ), diff --git a/lib/ui/views/search/search_viewmodel.dart b/lib/ui/views/search/search_viewmodel.dart index 1f8a132..aef704b 100644 --- a/lib/ui/views/search/search_viewmodel.dart +++ b/lib/ui/views/search/search_viewmodel.dart @@ -1,5 +1,7 @@ import 'package:async/async.dart'; +import 'package:camera/camera.dart'; import 'package:google_mlkit_digital_ink_recognition/google_mlkit_digital_ink_recognition.dart'; +import 'package:image_picker/image_picker.dart'; import 'package:kana_kit/kana_kit.dart'; import 'package:sagase/app/app.dialogs.dart'; import 'package:sagase/app/app.locator.dart'; @@ -30,8 +32,8 @@ class SearchViewModel extends FutureViewModel { CancelableOperation<(List, bool)>? _searchOperation; - bool _showHandWriting = false; - bool get showHandWriting => _showHandWriting; + InputMode _inputMode = InputMode.text; + InputMode get inputMode => _inputMode; List _handWritingResult = []; List get handWritingResult => _handWritingResult; @@ -45,6 +47,12 @@ class SearchViewModel extends FutureViewModel { bool _promptAnalysis = false; bool get promptAnalysis => _promptAnalysis; + XFile? _image; + XFile? get image => _image; + + bool _ocrError = false; + bool get ocrError => _ocrError; + @override Future futureToRun() async { await loadSearchHistory(); @@ -132,22 +140,6 @@ class SearchViewModel extends FutureViewModel { return (results, promptAnalysis); } - void toggleHandWriting() { - if (!_digitalInkService.ready) { - if (!_snackbarService.isSnackbarOpen) { - _snackbarService.showSnackbar( - message: - 'Hand writing detection is setting up. Please try again later.', - ); - } - return; - } - _showHandWriting = !showHandWriting; - handWritingResult.clear(); - locator().setShowNavigationBar(!_showHandWriting); - notifyListeners(); - } - Future recognizeWriting(Ink ink) async { _handWritingResult = await _digitalInkService.recognizeWriting(ink); notifyListeners(); @@ -210,4 +202,57 @@ class SearchViewModel extends FutureViewModel { arguments: ProperNounViewArguments(properNoun: properNoun), ); } + + void setInputMode(InputMode mode) { + if (_inputMode == mode) return; + + if (mode == InputMode.handWriting) { + if (!_digitalInkService.ready) { + if (!_snackbarService.isSnackbarOpen) { + _snackbarService.showSnackbar( + message: + 'Hand writing detection is setting up. Please try again later.', + ); + } + return; + } + + handWritingResult.clear(); + } + + _inputMode = mode; + _image = null; + _ocrError = false; + locator().setShowNavigationBar(mode == InputMode.text); + + rebuildUi(); + } + + Future handlePictureTaken(XFile image) async { + _image = image; + rebuildUi(); + } + + void handleImageError() { + _image = null; + _snackbarService.showSnackbar(message: 'Failed to process image'); + _ocrError = true; + rebuildUi(); + } + + void handleTextSelected(String text) { + searchOnChange(text); + } + + void resetImage() { + _image = null; + _ocrError = false; + rebuildUi(); + } +} + +enum InputMode { + text, + handWriting, + ocr, } diff --git a/lib/ui/views/search/widgets/hand_writing_input.dart b/lib/ui/views/search/widgets/hand_writing_input.dart new file mode 100644 index 0000000..f0880bb --- /dev/null +++ b/lib/ui/views/search/widgets/hand_writing_input.dart @@ -0,0 +1,152 @@ +import 'package:flutter/material.dart'; +import 'package:sagase/ui/widgets/hand_writing_canvas.dart'; +import 'package:stacked/stacked.dart'; + +import '../search_viewmodel.dart'; + +class HandWritingInput extends ViewModelWidget { + final TextEditingController searchController; + final HandWritingController handWritingController; + final FocusNode keyboardFocusNode; + + const HandWritingInput({ + super.key, + required this.searchController, + required this.handWritingController, + required this.keyboardFocusNode, + }); + + @override + Widget build(BuildContext context, SearchViewModel viewModel) { + return Expanded( + child: Column( + children: [ + const Divider(height: 1, thickness: 1), + SizedBox( + height: 40, + child: Row( + children: [ + Expanded( + child: ListView.builder( + scrollDirection: Axis.horizontal, + itemCount: viewModel.handWritingResult.length, + itemBuilder: (context, index) { + return GestureDetector( + onTap: () { + int cursorPosition = + searchController.selection.base.offset; + if (cursorPosition == 0) { + searchController.text = + viewModel.handWritingResult[index] + + searchController.text; + } else { + searchController.text = searchController.text + .substring(0, cursorPosition) + + viewModel.handWritingResult[index] + + searchController.text.substring(cursorPosition); + } + + searchController.selection = + TextSelection.fromPosition(TextPosition( + offset: cursorPosition + + viewModel.handWritingResult[index].length, + )); + handWritingController.clear(); + viewModel.searchOnChange(searchController.text); + }, + child: Container( + padding: const EdgeInsets.symmetric( + horizontal: 13, + ), + decoration: const BoxDecoration( + border: Border( + right: BorderSide(color: Colors.black), + ), + ), + child: Center( + child: Text( + viewModel.handWritingResult[index], + ), + ), + ), + ); + }, + ), + ), + const VerticalDivider(width: 1, thickness: 1), + IconButton( + onPressed: () => viewModel.setInputMode(InputMode.ocr), + icon: const Icon(Icons.camera_alt), + splashColor: Colors.transparent, + highlightColor: Colors.transparent, + ), + IconButton( + onPressed: () { + viewModel.setInputMode(InputMode.text); + keyboardFocusNode.requestFocus(); + }, + icon: const Icon(Icons.keyboard), + splashColor: Colors.transparent, + highlightColor: Colors.transparent, + ), + IconButton( + onPressed: () => viewModel.setInputMode(InputMode.text), + icon: const Icon(Icons.close), + splashColor: Colors.transparent, + highlightColor: Colors.transparent, + ), + ], + ), + ), + const Divider(height: 1, thickness: 1), + Expanded( + child: HandWritingCanvas( + onHandWritingChanged: viewModel.recognizeWriting, + controller: handWritingController, + ), + ), + const Divider(indent: 16, endIndent: 16, height: 1), + Padding( + padding: EdgeInsets.only( + bottom: MediaQuery.of(context).padding.bottom, + ), + child: Row( + mainAxisAlignment: MainAxisAlignment.spaceEvenly, + children: [ + IconButton( + onPressed: () => handWritingController.undo(), + icon: const Icon(Icons.undo), + splashColor: Colors.transparent, + highlightColor: Colors.transparent, + ), + IconButton( + onPressed: () => handWritingController.clear(), + icon: const Icon(Icons.delete), + splashColor: Colors.transparent, + highlightColor: Colors.transparent, + ), + IconButton( + onPressed: () { + int cursorPosition = searchController.selection.base.offset; + if (cursorPosition != 0) { + searchController.text = searchController.text + .substring(0, cursorPosition - 1) + + searchController.text.substring(cursorPosition); + + searchController.selection = TextSelection.fromPosition( + TextPosition(offset: cursorPosition - 1)); + viewModel.searchOnChange(searchController.text); + } + }, + icon: const Icon(Icons.backspace), + splashColor: Colors.transparent, + highlightColor: Colors.transparent, + ), + ], + ), + ), + ], + ), + ); + } +} diff --git a/lib/ui/views/search/widgets/ocr_widget.dart b/lib/ui/views/search/widgets/ocr_widget.dart new file mode 100644 index 0000000..b53e682 --- /dev/null +++ b/lib/ui/views/search/widgets/ocr_widget.dart @@ -0,0 +1,94 @@ +import 'package:flutter/material.dart'; +import 'package:sagase/ui/widgets/camera_viewfinder.dart'; +import 'package:sagase/ui/widgets/hand_writing_canvas.dart'; +import 'package:sagase/ui/widgets/ocr_image.dart'; +import 'package:stacked/stacked.dart'; + +import '../search_viewmodel.dart'; + +class OcrWidget extends ViewModelWidget { + final TextEditingController searchController; + final HandWritingController handWritingController; + final FocusNode keyboardFocusNode; + + const OcrWidget({ + super.key, + required this.searchController, + required this.handWritingController, + required this.keyboardFocusNode, + }); + + @override + Widget build(BuildContext context, SearchViewModel viewModel) { + return Expanded( + child: Column( + children: [ + const Divider(height: 1, thickness: 1), + SizedBox( + height: 40, + child: Row( + children: [ + Expanded( + child: Align( + alignment: Alignment.centerLeft, + child: TextButton( + onPressed: viewModel.resetImage, + child: const Text('Retake photo'), + ), + ), + ), + IconButton( + onPressed: () => + viewModel.setInputMode(InputMode.handWriting), + icon: const Icon(Icons.draw), + splashColor: Colors.transparent, + highlightColor: Colors.transparent, + ), + IconButton( + onPressed: () { + viewModel.setInputMode(InputMode.text); + keyboardFocusNode.requestFocus(); + }, + icon: const Icon(Icons.keyboard), + splashColor: Colors.transparent, + highlightColor: Colors.transparent, + ), + IconButton( + onPressed: () => viewModel.setInputMode(InputMode.text), + icon: const Icon(Icons.close), + splashColor: Colors.transparent, + highlightColor: Colors.transparent, + ), + ], + ), + ), + const Divider(height: 1, thickness: 1), + Expanded( + child: viewModel.ocrError + ? Center( + child: Text( + 'Failed to process image', + style: TextStyle(fontSize: 18), + ), + ) + : viewModel.image == null + ? CameraViewfinder( + onPictureTaken: viewModel.handlePictureTaken) + : OcrImage( + key: ValueKey(viewModel.image!.path.hashCode), + image: viewModel.image!, + onImageError: viewModel.handleImageError, + onTextSelected: (text) { + text = text.replaceAll('\n', ''); + searchController.text = text; + viewModel.handleTextSelected(text); + }, + locked: true, + singleSelection: true, + ), + ), + ], + ), + ); + } +} diff --git a/lib/ui/widgets/camera_viewfinder.dart b/lib/ui/widgets/camera_viewfinder.dart new file mode 100644 index 0000000..29aeef4 --- /dev/null +++ b/lib/ui/widgets/camera_viewfinder.dart @@ -0,0 +1,134 @@ +import 'package:camera/camera.dart'; +import 'package:flutter/material.dart'; + +class CameraViewfinder extends StatefulWidget { + final void Function(XFile) onPictureTaken; + + const CameraViewfinder({super.key, required this.onPictureTaken}); + + @override + State createState() => _CameraViewfinderState(); +} + +class _CameraViewfinderState extends State + with WidgetsBindingObserver { + CameraController? _controller; + List? _cameras; + bool _cameraInitialized = false; + bool _cameraError = false; + + @override + void initState() { + super.initState(); + _initCamera(); + } + + Future _initCamera() async { + _cameras = await availableCameras(); + if (_cameras!.isEmpty) { + setState(() { + _cameraError = true; + }); + } + CameraDescription cameraToUse = _cameras!.first; + for (final camera in _cameras!) { + if (camera.lensDirection == CameraLensDirection.back) { + cameraToUse = camera; + break; + } + } + _controller = CameraController( + cameraToUse, + ResolutionPreset.max, + enableAudio: false, + ); + + _controller!.initialize().then((_) { + if (mounted) { + setState(() => _cameraInitialized = true); + } + }).catchError((Object e) { + setState(() { + _cameraError = true; + }); + }); + } + + @override + void dispose() { + _controller?.dispose(); + super.dispose(); + } + + @override + void didChangeAppLifecycleState(AppLifecycleState state) { + if (_controller == null || !_controller!.value.isInitialized) { + return; + } + + if (state == AppLifecycleState.inactive) { + setState(() => _cameraInitialized = false); + _controller!.dispose(); + } else if (state == AppLifecycleState.resumed) { + _initCamera(); + } + } + + Future _takePhoto() async { + if (!_controller!.value.isInitialized) return; + final image = await _controller!.takePicture(); + widget.onPictureTaken(image); + } + + @override + Widget build(BuildContext context) { + if (_cameraError) { + return const Center( + child: Column( + mainAxisAlignment: MainAxisAlignment.center, + spacing: 4, + children: [ + Text( + 'Failed to start camera', + style: TextStyle(fontSize: 16), + ), + Text('Confirm camera permissions in system settings'), + ], + ), + ); + } + + if (_cameraInitialized) { + return Stack( + children: [ + SizedBox( + width: MediaQuery.of(context).size.width, + child: FittedBox( + clipBehavior: Clip.hardEdge, + fit: BoxFit.cover, + child: SizedBox( + width: MediaQuery.of(context).size.width, + child: CameraPreview(_controller!), + ), + ), + ), + Positioned( + bottom: MediaQuery.of(context).padding.bottom + 16, + left: 0, + right: 0, + child: Center( + child: FloatingActionButton( + onPressed: _takePhoto, + foregroundColor: Colors.white, + backgroundColor: Colors.deepPurple, + child: const Icon(Icons.camera_alt), + ), + ), + ), + ], + ); + } + + return const Center(child: CircularProgressIndicator()); + } +} diff --git a/lib/ui/widgets/ocr_image.dart b/lib/ui/widgets/ocr_image.dart new file mode 100644 index 0000000..bf9feb0 --- /dev/null +++ b/lib/ui/widgets/ocr_image.dart @@ -0,0 +1,282 @@ +import 'dart:io'; +import 'dart:math'; +import 'dart:typed_data'; + +import 'package:camera/camera.dart'; +import 'package:flutter/material.dart'; +import 'package:flutter_exif_rotation/flutter_exif_rotation.dart'; +import 'package:google_mlkit_text_recognition/google_mlkit_text_recognition.dart'; +import 'package:path/path.dart' as path; +import 'package:path_provider/path_provider.dart' as path_provider; +import 'package:sagase/datamodels/recognized_text_block.dart'; +import 'package:sagase/ui/painters/ocr_painter.dart'; +import 'package:sagase/utils/constants.dart' as constants; +import 'package:shimmer/shimmer.dart'; + +class OcrImage extends StatefulWidget { + final XFile image; + final void Function(int)? onImageProcessed; + final void Function() onImageError; + final void Function(String) onTextSelected; + final bool locked; + final bool singleSelection; + + const OcrImage({ + super.key, + required this.image, + this.onImageProcessed, + required this.onImageError, + required this.locked, + required this.onTextSelected, + required this.singleSelection, + }); + + @override + State createState() => _OcrImageState(); +} + +class _OcrImageState extends State { + final _textRecognizer = + TextRecognizer(script: TextRecognitionScript.japanese); + Uint8List? _currentImageBytes; + late Size _imageSize; + List? _recognizedTextBlocks; + + @override + void initState() { + super.initState(); + _loadImage(); + } + + Future _loadImage() async { + final ocrImageDir = path.join( + (await path_provider.getApplicationCacheDirectory()).path, + constants.ocrImagesDir, + ); + await Directory(ocrImageDir).create(); + + final imagePath = path.join(ocrImageDir, widget.image.name); + try { + await File(widget.image.path).rename(imagePath); + } catch (_) { + if (File(widget.image.path).existsSync()) { + File(widget.image.path).delete(); + } + + widget.onImageError(); + return; + } + + final rotatedImage = await FlutterExifRotation.rotateImage(path: imagePath); + + final inputImage = InputImage.fromFilePath(rotatedImage.path); + _currentImageBytes = await rotatedImage.readAsBytes(); + + if (inputImage.metadata != null) { + _imageSize = inputImage.metadata!.size; + } else { + final decodedImage = await decodeImageFromList(_currentImageBytes!); + _imageSize = Size( + decodedImage.width.toDouble(), + decodedImage.height.toDouble(), + ); + } + + setState(() {}); + + final recognizedText = await _textRecognizer.processImage(inputImage); + + _recognizedTextBlocks = []; + for (final textBlock in recognizedText.blocks) { + _recognizedTextBlocks!.add( + RecognizedTextBlock( + text: textBlock.text, + points: textBlock.cornerPoints, + ), + ); + } + + setState(() {}); + + if (widget.onImageProcessed != null) { + widget.onImageProcessed!(_recognizedTextBlocks!.length); + } + + await rotatedImage.delete(); + } + + void handleSelect(RecognizedTextBlock textBlock) { + setState(() { + if (!textBlock.selected) { + widget.onTextSelected(textBlock.text); + } + + if (widget.singleSelection) { + for (final textBlock in _recognizedTextBlocks!) { + textBlock.selected = false; + } + } + + textBlock.selected = true; + _recognizedTextBlocks = List.from(_recognizedTextBlocks!); + }); + } + + @override + Widget build(BuildContext context) { + return _currentImageBytes == null + ? _OcrImageLoading() + : _OcrImage( + recognizedTextBlocks: _recognizedTextBlocks, + imageSize: _imageSize, + image: _currentImageBytes!, + locked: widget.locked, + onSelect: handleSelect, + ); + } +} + +class _OcrImageLoading extends StatelessWidget { + const _OcrImageLoading(); + + @override + Widget build(BuildContext context) { + final isDark = Theme.of(context).brightness == Brightness.dark; + + return Padding( + padding: const EdgeInsets.all(16), + child: Shimmer.fromColors( + baseColor: isDark ? const Color(0xFF3a3a3a) : Colors.grey.shade300, + highlightColor: isDark ? const Color(0xFF4a4a4a) : Colors.grey.shade100, + child: Container( + decoration: const BoxDecoration( + color: Colors.white, + borderRadius: BorderRadius.all(Radius.circular(8)), + ), + ), + ), + ); + } +} + +class _OcrImage extends StatelessWidget { + final List? recognizedTextBlocks; + final Size imageSize; + final Uint8List image; + final bool locked; + final void Function(RecognizedTextBlock) onSelect; + + const _OcrImage({ + this.recognizedTextBlocks, + required this.imageSize, + required this.image, + required this.locked, + required this.onSelect, + }); + + @override + Widget build(BuildContext context) { + if (locked) { + return LayoutBuilder( + builder: (context, constraints) { + return SizedBox( + width: constraints.maxWidth, + height: constraints.maxHeight, + child: FittedBox( + clipBehavior: Clip.hardEdge, + fit: BoxFit.cover, + child: GestureDetector( + behavior: HitTestBehavior.opaque, + onTapUp: _handleTapUp, + child: CustomPaint( + foregroundPainter: OcrPainter( + recognizedTextBlocks, + imageSize, + ), + child: Image.memory(image), + ), + ), + ), + ); + }, + ); + } else { + return LayoutBuilder( + builder: (context, constraints) { + return InteractiveViewer( + constrained: false, + child: SizedBox( + width: constraints.maxWidth, + height: constraints.maxHeight, + child: FittedBox( + clipBehavior: Clip.hardEdge, + fit: BoxFit.contain, + child: GestureDetector( + behavior: HitTestBehavior.opaque, + onTapUp: _handleTapUp, + child: CustomPaint( + foregroundPainter: OcrPainter( + recognizedTextBlocks, + imageSize, + ), + child: Image.memory(image), + ), + ), + ), + ), + ); + }, + ); + } + } + + void _handleTapUp(TapUpDetails details) { + if (recognizedTextBlocks != null) { + for (final textBlock in recognizedTextBlocks!) { + if (_pointInsideRectangle(details.localPosition, textBlock.offsets)) { + onSelect(textBlock); + break; + } + } + } + } + + bool _pointInsideRectangle(Offset point, List rectCorners) { + double x1 = rectCorners[0].dx; + double x2 = rectCorners[1].dx; + double x3 = rectCorners[2].dx; + double x4 = rectCorners[3].dx; + + double y1 = rectCorners[0].dy; + double y2 = rectCorners[1].dy; + double y3 = rectCorners[2].dy; + double y4 = rectCorners[3].dy; + + double a1 = sqrt((x1 - x2) * (x1 - x2) + (y1 - y2) * (y1 - y2)); + double a2 = sqrt((x2 - x3) * (x2 - x3) + (y2 - y3) * (y2 - y3)); + double a3 = sqrt((x3 - x4) * (x3 - x4) + (y3 - y4) * (y3 - y4)); + double a4 = sqrt((x4 - x1) * (x4 - x1) + (y4 - y1) * (y4 - y1)); + + double b1 = sqrt( + (x1 - point.dx) * (x1 - point.dx) + (y1 - point.dy) * (y1 - point.dy)); + double b2 = sqrt( + (x2 - point.dx) * (x2 - point.dx) + (y2 - point.dy) * (y2 - point.dy)); + double b3 = sqrt( + (x3 - point.dx) * (x3 - point.dx) + (y3 - point.dy) * (y3 - point.dy)); + double b4 = sqrt( + (x4 - point.dx) * (x4 - point.dx) + (y4 - point.dy) * (y4 - point.dy)); + + double u1 = (a1 + b1 + b2) / 2; + double u2 = (a2 + b2 + b3) / 2; + double u3 = (a3 + b3 + b4) / 2; + double u4 = (a4 + b4 + b1) / 2; + + double area1 = sqrt(u1 * (u1 - a1) * (u1 - b1) * (u1 - b2)); + double area2 = sqrt(u2 * (u2 - a2) * (u2 - b2) * (u2 - b3)); + double area3 = sqrt(u3 * (u3 - a3) * (u3 - b3) * (u3 - b4)); + double area4 = sqrt(u4 * (u4 - a4) * (u4 - b4) * (u4 - b1)); + + double difference = 0.95 * (area1 + area2 + area3 + area4) - a1 * a2; + return difference < 1; + } +} diff --git a/pubspec.lock b/pubspec.lock index ff32da9..390d243 100644 --- a/pubspec.lock +++ b/pubspec.lock @@ -158,6 +158,46 @@ packages: url: "https://pub.dev" source: hosted version: "8.11.1" + camera: + dependency: "direct main" + description: + name: camera + sha256: d6ec2cbdbe2fa8f5e0d07d8c06368fe4effa985a4a5ddade9cc58a8cd849557d + url: "https://pub.dev" + source: hosted + version: "0.11.2" + camera_android_camerax: + dependency: transitive + description: + name: camera_android_camerax + sha256: "58b8fe843a3c83fd1273c00cb35f5a8ae507f6cc9b2029bcf7e2abba499e28d8" + url: "https://pub.dev" + source: hosted + version: "0.6.19+1" + camera_avfoundation: + dependency: transitive + description: + name: camera_avfoundation + sha256: "180c1b368cf3485a18c46e219e3ad80271a779a6d66a749376e31244fd927cde" + url: "https://pub.dev" + source: hosted + version: "0.9.20+7" + camera_platform_interface: + dependency: transitive + description: + name: camera_platform_interface + sha256: "2f757024a48696ff4814a789b0bd90f5660c0fb25f393ab4564fb483327930e2" + url: "https://pub.dev" + source: hosted + version: "2.10.0" + camera_web: + dependency: transitive + description: + name: camera_web + sha256: "595f28c89d1fb62d77c73c633193755b781c6d2e0ebcd8dc25b763b514e6ba8f" + url: "https://pub.dev" + source: hosted + version: "0.3.5" characters: dependency: transitive description: diff --git a/pubspec.yaml b/pubspec.yaml index 8818205..ce088ac 100644 --- a/pubspec.yaml +++ b/pubspec.yaml @@ -87,6 +87,7 @@ dependencies: ref: 'ddb5dff' security_scoped_resource: ^0.0.2 sanitize_filename: ^1.0.5 + camera: ^0.11.2 image_picker: ^1.1.2 flutter_exif_rotation: git: