From dba7fefd4dee0503e43eb082f691de0f417c32f8 Mon Sep 17 00:00:00 2001 From: Craig Labenz Date: Thu, 6 Jun 2024 16:46:29 -0700 Subject: [PATCH] fixes genai build.dart script (#63) --- packages/mediapipe-task-genai/build.dart | 122 ----------------- .../ios/Runner.xcodeproj/project.pbxproj | 6 +- packages/mediapipe-task-genai/hook/build.dart | 126 ++++++++++++++++++ packages/mediapipe-task-text/hook/build.dart | 2 +- 4 files changed, 130 insertions(+), 126 deletions(-) delete mode 100644 packages/mediapipe-task-genai/build.dart create mode 100644 packages/mediapipe-task-genai/hook/build.dart diff --git a/packages/mediapipe-task-genai/build.dart b/packages/mediapipe-task-genai/build.dart deleted file mode 100644 index fb63b000..00000000 --- a/packages/mediapipe-task-genai/build.dart +++ /dev/null @@ -1,122 +0,0 @@ -import 'dart:io'; -import 'package:native_assets_cli/native_assets_cli.dart'; -import 'package:http/http.dart' as http; -import 'package:path/path.dart' as path; - -import 'sdk_downloads.dart'; - -late File logFile; - -final logs = <(DateTime, String)>[]; -void log(String msg) { - logs.add((DateTime.now(), msg)); - if (!logFile.parent.existsSync()) { - logFile.parent.createSync(); - } - - if (logFile.existsSync()) { - logFile.deleteSync(); - } - logFile.createSync(); - logFile.writeAsStringSync(logs - .map((rec) => '[${rec.$1.toIso8601String()}] ${rec.$2}') - .toList() - .join('\n\n')); -} - -Future main(List args) async { - final buildConfig = await BuildConfig.fromArgs(args); - logFile = File( - path.joinAll([ - Directory.current.path, // root dir of app using `mediapipe-task-xyz` - 'build/${buildConfig.dryRun ? "dryrun" : "live-run"}-build-log.txt', - ]), - ); - - log(args.join(' ')); - final String targetOs = buildConfig.targetOs.toString(); - - log('dir.current: ${Directory.current.absolute.path}'); - - // Throw if target runtime is unsupported. - if (!sdkDownloadUrls.containsKey(targetOs)) { - throw Exception('Unsupported target OS: $targetOs. ' - 'Supported values are: ${sdkDownloadUrls.keys.toSet()}'); - } - - final buildOutput = BuildOutput(); - buildOutput.dependencies.dependencies - .add(buildConfig.packageRoot.resolve('build.dart')); - buildOutput.dependencies.dependencies - .add(buildConfig.packageRoot.resolve('sdk_downloads.dart')); - - final modelName = 'libllm_inference_engine'; - final Iterable archKeys; - if (buildConfig.dryRun) { - archKeys = sdkDownloadUrls[targetOs]![modelName]!.keys; - } else { - archKeys = [buildConfig.targetArchitecture.toString()]; - } - for (String arch in archKeys) { - arch = getArchAlias(arch); - log("arch: $arch"); - log("sdkDownloadUrls[targetOs]: ${sdkDownloadUrls[targetOs]}"); - log("sdkDownloadUrls[targetOs]['$modelName']: ${sdkDownloadUrls[targetOs]![modelName]}"); - log("sdkDownloadUrls[targetOs]['$modelName'][$arch]: ${sdkDownloadUrls[targetOs]![modelName]![arch]}"); - - if (!sdkDownloadUrls[targetOs]!['libllm_inference_engine']! - .containsKey(arch)) { - continue; - } - final assetUrl = - sdkDownloadUrls[targetOs]!['libllm_inference_engine']![arch]!; - final downloadFileLocation = buildConfig.outDir.resolve( - '${arch}_${assetUrl.split('/').last}', - ); - log('downloadFileLocation: $downloadFileLocation'); - buildOutput.assets.add( - Asset( - id: 'package:mediapipe_genai/src/io/third_party/mediapipe/generated/mediapipe_genai_bindings.dart', - linkMode: LinkMode.dynamic, - target: Target.fromArchitectureAndOs( - Architecture.fromString(arch), buildConfig.targetOs), - path: AssetAbsolutePath(downloadFileLocation), - ), - ); - if (!buildConfig.dryRun) { - downloadAsset(assetUrl, downloadFileLocation); - } - } - - await buildOutput.writeToFile(outDir: buildConfig.outDir); -} - -Future downloadAsset(String assetUrl, Uri destinationFile) async { - final downloadUri = Uri.parse(assetUrl); - final downloadedFile = File(destinationFile.toFilePath()); - log('Saving file to ${downloadedFile.absolute.path}'); - - final downloadResponse = await http.get(downloadUri); - log('Download response: ${downloadResponse.statusCode}'); - - if (downloadResponse.statusCode == 200) { - if (downloadedFile.existsSync()) { - downloadedFile.deleteSync(); - } - downloadedFile.createSync(); - log('Saved file to ${downloadedFile.absolute.path}\n'); - downloadedFile.writeAsBytes(downloadResponse.bodyBytes); - } else { - log('${downloadResponse.statusCode} :: ${downloadResponse.body}'); - throw Exception( - '${downloadResponse.statusCode} :: ${downloadResponse.body}'); - } -} - -/// Translates native-assets architecture names into MediaPipe architecture names -String getArchAlias(String arch) { - return { - 'arm': 'arm64', - }[arch] ?? - arch; -} diff --git a/packages/mediapipe-task-genai/example/ios/Runner.xcodeproj/project.pbxproj b/packages/mediapipe-task-genai/example/ios/Runner.xcodeproj/project.pbxproj index 4058dea0..587261af 100644 --- a/packages/mediapipe-task-genai/example/ios/Runner.xcodeproj/project.pbxproj +++ b/packages/mediapipe-task-genai/example/ios/Runner.xcodeproj/project.pbxproj @@ -470,7 +470,7 @@ ASSETCATALOG_COMPILER_APPICON_NAME = AppIcon; CLANG_ENABLE_MODULES = YES; CURRENT_PROJECT_VERSION = "$(FLUTTER_BUILD_NUMBER)"; - DEVELOPMENT_TEAM = 35JKZ342M3; + DEVELOPMENT_TEAM = S8QB4VV633; ENABLE_BITCODE = NO; INFOPLIST_FILE = Runner/Info.plist; LD_RUNPATH_SEARCH_PATHS = ( @@ -653,7 +653,7 @@ ASSETCATALOG_COMPILER_APPICON_NAME = AppIcon; CLANG_ENABLE_MODULES = YES; CURRENT_PROJECT_VERSION = "$(FLUTTER_BUILD_NUMBER)"; - DEVELOPMENT_TEAM = 35JKZ342M3; + DEVELOPMENT_TEAM = S8QB4VV633; ENABLE_BITCODE = NO; INFOPLIST_FILE = Runner/Info.plist; LD_RUNPATH_SEARCH_PATHS = ( @@ -676,7 +676,7 @@ ASSETCATALOG_COMPILER_APPICON_NAME = AppIcon; CLANG_ENABLE_MODULES = YES; CURRENT_PROJECT_VERSION = "$(FLUTTER_BUILD_NUMBER)"; - DEVELOPMENT_TEAM = 35JKZ342M3; + DEVELOPMENT_TEAM = S8QB4VV633; ENABLE_BITCODE = NO; INFOPLIST_FILE = Runner/Info.plist; LD_RUNPATH_SEARCH_PATHS = ( diff --git a/packages/mediapipe-task-genai/hook/build.dart b/packages/mediapipe-task-genai/hook/build.dart new file mode 100644 index 00000000..06d76f60 --- /dev/null +++ b/packages/mediapipe-task-genai/hook/build.dart @@ -0,0 +1,126 @@ +import 'dart:io'; +import 'package:native_assets_cli/native_assets_cli.dart'; +import 'package:http/http.dart' as http; +import 'package:path/path.dart' as path; + +import '../sdk_downloads.dart'; + +late File logFile; + +final logs = <(DateTime, String)>[]; +void log(String msg) { + logs.add((DateTime.now(), msg)); + if (!logFile.parent.existsSync()) { + logFile.parent.createSync(); + } + + if (logFile.existsSync()) { + logFile.deleteSync(); + } + logFile.createSync(); + logFile.writeAsStringSync(logs + .map((rec) => '[${rec.$1.toIso8601String()}] ${rec.$2}') + .toList() + .join('\n\n')); +} + +Future main(List args) async { + await build(args, (buildConfig, buildOutput) async { + logFile = File( + path.joinAll([ + Directory.current.path, // root dir of app using `mediapipe-task-xyz` + 'build/${buildConfig.dryRun ? "dryrun" : "live-run"}-build-log.txt', + ]), + ); + + log(args.join(' ')); + final String targetOs = buildConfig.targetOS.toString(); + + log('dir.current: ${Directory.current.absolute.path}'); + + // Throw if target runtime is unsupported. + if (!sdkDownloadUrls.containsKey(targetOs)) { + throw Exception('Unsupported target OS: $targetOs. ' + 'Supported values are: ${sdkDownloadUrls.keys.toSet()}'); + } + + buildOutput.addDependencies([ + buildConfig.packageRoot.resolve('build.dart'), + buildConfig.packageRoot.resolve('sdk_downloads.dart'), + ]); + + final modelName = 'libllm_inference_engine'; + final Iterable archKeys; + if (buildConfig.dryRun) { + archKeys = sdkDownloadUrls[targetOs]![modelName]!.keys; + } else { + archKeys = [buildConfig.targetArchitecture.toString()]; + } + for (String arch in archKeys) { + arch = getArchAlias(arch); + log('arch: $arch'); + log('sdkDownloadUrls[$targetOs]: ${sdkDownloadUrls[targetOs]}'); + log('sdkDownloadUrls[$targetOs][$modelName]: ${sdkDownloadUrls[targetOs]![modelName]}'); + log('sdkDownloadUrls[$targetOs][$modelName][$arch]: ${sdkDownloadUrls[targetOs]![modelName]![arch]}'); + + if (!sdkDownloadUrls[targetOs]!['libllm_inference_engine']! + .containsKey(arch)) { + continue; + } + final assetUrl = + sdkDownloadUrls[targetOs]!['libllm_inference_engine']![arch]!; + + // Take last chunk and drop file extension, e.g., + // "path/to/file.dylib" -> "file" + final fileName = assetUrl.split('/').last.split('.').first; + + final downloadFileLocation = buildConfig.outputDirectory.resolve( + buildConfig.targetOS.dylibFileName(fileName), + ); + buildOutput.addAsset( + NativeCodeAsset( + package: 'mediapipe_genai', + name: + 'src/io/third_party/mediapipe/generated/mediapipe_genai_bindings.dart', + linkMode: DynamicLoadingBundled(), + os: buildConfig.targetOS, + architecture: buildConfig.targetArchitecture, + file: downloadFileLocation, + ), + ); + if (!buildConfig.dryRun) { + log('downloadFileLocation: $downloadFileLocation'); + downloadAsset(assetUrl, downloadFileLocation); + } + } + }); +} + +Future downloadAsset(String assetUrl, Uri destinationFile) async { + final downloadUri = Uri.parse(assetUrl); + final downloadedFile = File(destinationFile.toFilePath()); + log('Saving file to ${downloadedFile.absolute.path}'); + + final downloadResponse = await http.get(downloadUri); + log('Download response: ${downloadResponse.statusCode}'); + + if (downloadResponse.statusCode == 200) { + if (downloadedFile.existsSync()) { + downloadedFile.deleteSync(); + } + downloadedFile.createSync(); + log('Saved file to ${downloadedFile.absolute.path}\n'); + downloadedFile.writeAsBytes(downloadResponse.bodyBytes); + } else { + log('${downloadResponse.statusCode} :: ${downloadResponse.body}'); + throw Exception( + '${downloadResponse.statusCode} :: ${downloadResponse.body}'); + } +} + +/// Translates native-assets architecture names into MediaPipe architecture names +String getArchAlias(String arch) => + { + 'arm': 'arm64', + }[arch] ?? + arch; diff --git a/packages/mediapipe-task-text/hook/build.dart b/packages/mediapipe-task-text/hook/build.dart index 59dbbfea..4cd71876 100644 --- a/packages/mediapipe-task-text/hook/build.dart +++ b/packages/mediapipe-task-text/hook/build.dart @@ -53,7 +53,6 @@ Future main(List args) async { final downloadFileLocation = buildConfig.outputDirectory.resolve( buildConfig.targetOS.dylibFileName('text'), ); - log('downloadFileLocation: $downloadFileLocation'); buildOutput.addAsset( NativeCodeAsset( package: 'mediapipe_text', @@ -66,6 +65,7 @@ Future main(List args) async { ), ); if (!buildConfig.dryRun) { + log('downloadFileLocation: $downloadFileLocation'); final arch = buildConfig.targetArchitecture.toString(); final assetUrl = sdkDownloadUrls[targetOS]!['libtext']![arch]!; downloadAsset(assetUrl, downloadFileLocation);