From d72d304f0d8bea62566e9c0dbc8767fd54bb5dc6 Mon Sep 17 00:00:00 2001 From: iree-copybara-bot Date: Wed, 18 Sep 2019 15:46:56 -0700 Subject: [PATCH] Internal change PiperOrigin-RevId: 269907991 --- .gitignore | 32 - CONTRIBUTING.md | 28 + LICENSE | 202 +++ README.md | 243 +++ base/arena.cc | 125 ++ base/arena.h | 121 ++ base/bitfield.h | 140 ++ base/file_io.h | 67 + base/flatbuffer_util.cc | 250 +++ base/flatbuffer_util.h | 280 ++++ base/init.h | 52 + base/internal/init_internal.cc | 100 ++ base/internal/init_internal.h | 99 ++ base/internal/logging.cc | 106 ++ base/internal/logging.h | 179 ++ base/internal/status.cc | 174 ++ base/internal/status.h | 125 ++ base/internal/status_builder.cc | 141 ++ base/internal/status_builder.h | 139 ++ base/internal/status_errno.cc | 175 ++ base/internal/status_errno.h | 41 + base/internal/status_errors.cc | 147 ++ base/internal/status_errors.h | 60 + base/internal/status_macros.h | 108 ++ base/internal/statusor.cc | 39 + base/internal/statusor.h | 699 ++++++++ base/intrusive_list.h | 766 +++++++++ base/intrusive_list_ref_ptr.inc | 160 ++ base/intrusive_list_unique_ptr.inc | 126 ++ base/logging.h | 63 + base/memory.h | 152 ++ base/ref_ptr.h | 364 +++++ base/shape.cc | 100 ++ base/shape.h | 156 ++ base/status.h | 29 + base/time.h | 37 + base/tracing.cc | 181 +++ base/tracing.h | 69 + base/tracing_disabled.cc | 42 + base/wait_handle.cc | 532 ++++++ base/wait_handle.h | 321 ++++ compiler/IR/ConfigOps.cpp | 111 ++ compiler/IR/ConfigOps.h | 38 + compiler/IR/ConfigOps.td | 44 + compiler/IR/Dialect.cpp | 90 ++ compiler/IR/Dialect.h | 38 + compiler/IR/Interpreter/HLDialect.cpp | 34 + compiler/IR/Interpreter/HLDialect.h | 32 + compiler/IR/Interpreter/HLOps.cpp | 249 +++ compiler/IR/Interpreter/HLOps.h | 38 + compiler/IR/Interpreter/HLOps.td | 643 ++++++++ compiler/IR/Interpreter/LLDialect.cpp | 34 + compiler/IR/Interpreter/LLDialect.h | 32 + compiler/IR/Interpreter/LLOps.cpp | 229 +++ compiler/IR/Interpreter/LLOps.h | 38 + compiler/IR/Interpreter/LLOps.td | 629 ++++++++ compiler/IR/Interpreter/OpWriters.cpp | 261 +++ compiler/IR/Interpreter/OpWriters.h | 30 + compiler/IR/Interpreter/test/concat.mlir | 68 + .../IR/Interpreter/test/invalid_types_hl.mlir | 55 + .../IR/Interpreter/test/invalid_types_ll.mlir | 103 ++ compiler/IR/OpBase.td | 167 ++ compiler/IR/Ops.cpp | 526 ++++++ compiler/IR/Ops.h | 36 + compiler/IR/Ops.td | 170 ++ compiler/IR/Sequencer/HLDialect.cpp | 34 + compiler/IR/Sequencer/HLDialect.h | 32 + compiler/IR/Sequencer/HLOps.cpp | 394 +++++ compiler/IR/Sequencer/HLOps.h | 39 + compiler/IR/Sequencer/HLOps.td | 416 +++++ compiler/IR/Sequencer/LLDialect.cpp | 34 + compiler/IR/Sequencer/LLDialect.h | 32 + compiler/IR/Sequencer/LLOps.cpp | 676 ++++++++ compiler/IR/Sequencer/LLOps.h | 38 + compiler/IR/Sequencer/LLOps.td | 578 +++++++ compiler/IR/Sequencer/OpWriters.cpp | 266 +++ compiler/IR/Sequencer/OpWriters.h | 30 + compiler/IR/Sequencer/test/concat.mlir | 68 + compiler/IR/StructureOps.cpp | 250 +++ compiler/IR/StructureOps.h | 40 + compiler/IR/StructureOps.td | 122 ++ compiler/IR/Traits.cpp | 23 + compiler/IR/Traits.h | 26 + compiler/IR/Types.cpp | 53 + compiler/IR/Types.h | 113 ++ compiler/IR/test/bindings.mlir | 10 + compiler/IR/test/dispatch_regions.mlir | 63 + compiler/IR/test/reduction_regions.mlir | 46 + compiler/Serialization/BytecodeTables.cpp | 73 + compiler/Serialization/BytecodeTables.h | 52 + compiler/Serialization/BytecodeWriter.cpp | 334 ++++ compiler/Serialization/BytecodeWriter.h | 96 ++ .../Serialization/VMDeviceTableBuilder.cpp | 46 + compiler/Serialization/VMDeviceTableBuilder.h | 45 + .../VMExecutableTableBuilder.cpp | 41 + .../Serialization/VMExecutableTableBuilder.h | 44 + compiler/Serialization/VMFunctionBuilder.cpp | 362 +++++ compiler/Serialization/VMFunctionBuilder.h | 77 + .../Serialization/VMFunctionTableBuilder.cpp | 87 + .../Serialization/VMFunctionTableBuilder.h | 75 + compiler/Serialization/VMModuleBuilder.cpp | 70 + compiler/Serialization/VMModuleBuilder.h | 57 + compiler/Serialization/VMSourceMapBuilder.cpp | 164 ++ compiler/Serialization/VMSourceMapBuilder.h | 63 + .../Transforms/AggressiveOpElimination.cpp | 130 ++ .../Transforms/AssignFunctionOrdinals.cpp | 47 + compiler/Transforms/ConversionUtils.h | 106 ++ .../ConvertFromTupleCallingConvention.cpp | 328 ++++ .../ConvertToMemRefCallingConvention.cpp | 402 +++++ .../Transforms/DropUnreachableFunctions.cpp | 67 + compiler/Transforms/DropUnusedExecutables.cpp | 62 + .../Interpreter/ExpandReductionsToOps.cpp | 205 +++ .../Interpreter/LegalizeInterpreterOps.cpp | 75 + .../Interpreter/LegalizeInterpreterOps.td | 6 + .../Interpreter/LoadStoreDataFlowOpt.cpp | 148 ++ .../Interpreter/LowerInterpreterDialect.cpp | 246 +++ .../LowerStdToInterpreterDialect.cpp | 328 ++++ .../LowerXLAToInterpreterDialect.cpp | 591 +++++++ .../Interpreter/MakeExecutableABI.cpp | 165 ++ compiler/Transforms/Interpreter/Passes.h | 47 + .../test/load_store_data_flow_opt.mlir | 127 ++ .../Interpreter/test/make_executable_abi.mlir | 154 ++ .../Interpreter/test/xla/concat.mlir | 47 + .../Interpreter/test/xla/const.mlir | 11 + .../test/xla/dynamic_update_slice.mlir | 139 ++ .../Interpreter/test/xla/gather.mlir | 121 ++ .../Interpreter/test/xla/slice.mlir | 31 + compiler/Transforms/LegalizeTensorMemRef.cpp | 56 + .../Transforms/LegalizeTupleElementAccess.cpp | 78 + compiler/Transforms/LegalizeTypeStorage.cpp | 145 ++ compiler/Transforms/Passes.h | 76 + .../Sequencer/AssignExecutableOrdinals.cpp | 75 + .../AssignExecutableWorkloadAttrs.cpp | 125 ++ .../FoldCompatibleDispatchRegions.cpp | 63 + .../Sequencer/IdentifyDispatchRegions.cpp | 259 +++ .../Sequencer/IdentifyReductionRegions.cpp | 163 ++ .../Sequencer/LoadStoreDataFlowOpt.cpp | 137 ++ .../Sequencer/LowerSequencerDialect.cpp | 255 +++ .../Sequencer/LowerStdToSequencerDialect.cpp | 262 +++ .../Sequencer/LowerXLAToSequencerDialect.cpp | 262 +++ .../Sequencer/OutlineDispatchRegions.cpp | 241 +++ .../Sequencer/OutlineReductionRegions.cpp | 308 ++++ compiler/Transforms/Sequencer/Passes.h | 80 + .../RematerializeDispatchConstants.cpp | 149 ++ .../drop_unreachable_module_functions.mlir | 49 + .../InterpreterExecutableTranslation.cpp | 293 ++++ .../InterpreterExecutableTranslation.h | 38 + .../Translation/SPIRV/AffineExprCodegen.h | 146 ++ .../SPIRV/IREEIndexComputation.cpp | 99 ++ .../Translation/SPIRV/IREEIndexComputation.h | 92 ++ compiler/Translation/SPIRV/IREEToSPIRV.cpp | 77 + compiler/Translation/SPIRV/IREEToSPIRV.h | 69 + .../Translation/SPIRV/IREEToSPIRVPass.cpp | 118 ++ compiler/Translation/SPIRV/IREEToSPIRVPass.h | 34 + .../Translation/SPIRV/IndexComputation.cpp | 269 +++ compiler/Translation/SPIRV/IndexComputation.h | 268 +++ .../Translation/SPIRV/Kernels/matmul.comp | 90 ++ .../SPIRV/SPIRVExecutableTranslation.cpp | 412 +++++ .../SPIRV/SPIRVExecutableTranslation.h | 37 + compiler/Translation/SPIRV/SPIRVLowering.cpp | 131 ++ compiler/Translation/SPIRV/SPIRVLowering.h | 593 +++++++ .../Translation/SPIRV/XLAIndexPropagation.cpp | 80 + .../Translation/SPIRV/XLAIndexPropagation.h | 102 ++ .../Translation/SPIRV/test/simple_test.mlir | 674 ++++++++ .../SequencerModuleTranslation.cpp | 519 ++++++ .../Translation/SequencerModuleTranslation.h | 36 + compiler/Utils/DispatchUtils.cpp | 733 +++++++++ compiler/Utils/DispatchUtils.h | 92 ++ compiler/Utils/Macros.h | 21 + compiler/Utils/MemRefUtils.cpp | 178 ++ compiler/Utils/MemRefUtils.h | 85 + compiler/Utils/ModuleUtils.cpp | 98 ++ compiler/Utils/ModuleUtils.h | 30 + compiler/Utils/OpUtils.cpp | 44 + compiler/Utils/OpUtils.h | 40 + compiler/Utils/TranslationUtils.cpp | 139 ++ compiler/Utils/TranslationUtils.h | 109 ++ hal/allocator.cc | 77 + hal/allocator.h | 138 ++ hal/buffer.cc | 549 +++++++ hal/buffer.h | 903 +++++++++++ hal/buffer_view.cc | 180 +++ hal/buffer_view.h | 108 ++ hal/buffer_view_string_util.cc | 487 ++++++ hal/buffer_view_string_util.h | 95 ++ hal/command_buffer.cc | 29 + hal/command_buffer.h | 383 +++++ hal/command_buffer_validation.cc | 403 +++++ hal/command_buffer_validation.h | 32 + hal/command_queue.h | 126 ++ hal/deferred_buffer.cc | 162 ++ hal/deferred_buffer.h | 106 ++ hal/device.h | 165 ++ hal/device_info.h | 90 ++ hal/device_manager.cc | 177 ++ hal/device_manager.h | 150 ++ hal/device_placement.h | 34 + hal/driver.h | 61 + hal/driver_registry.cc | 87 + hal/driver_registry.h | 83 + hal/event.h | 35 + hal/executable.h | 57 + hal/executable_cache.cc | 56 + hal/executable_cache.h | 145 ++ hal/executable_format.h | 60 + hal/executable_spec.h | 44 + hal/fence.h | 72 + hal/heap_buffer.cc | 190 +++ hal/heap_buffer.h | 117 ++ hal/host/async_command_queue.cc | 134 ++ hal/host/async_command_queue.h | 72 + hal/host/host_buffer.cc | 148 ++ hal/host/host_buffer.h | 67 + hal/host/host_event.cc | 25 + hal/host/host_event.h | 32 + hal/host/host_fence.cc | 110 ++ hal/host/host_fence.h | 63 + hal/host/host_local_allocator.cc | 111 ++ hal/host/host_local_allocator.h | 60 + hal/host/host_local_command_processor.cc | 120 ++ hal/host/host_local_command_processor.h | 85 + hal/host/host_submission_queue.cc | 295 ++++ hal/host/host_submission_queue.h | 162 ++ hal/host/inproc_command_buffer.cc | 264 +++ hal/host/inproc_command_buffer.h | 241 +++ hal/interpreter/bytecode_cache.cc | 54 + hal/interpreter/bytecode_cache.h | 42 + hal/interpreter/bytecode_dispatch.cc | 865 ++++++++++ hal/interpreter/bytecode_dispatch.h | 35 + .../bytecode_dispatch_conversion.h | 395 +++++ hal/interpreter/bytecode_dispatch_util.cc | 119 ++ hal/interpreter/bytecode_dispatch_util.h | 515 ++++++ hal/interpreter/bytecode_executable.cc | 85 + hal/interpreter/bytecode_executable.h | 68 + hal/interpreter/bytecode_kernels.h | 371 +++++ hal/interpreter/bytecode_kernels_generic.h | 696 ++++++++ hal/interpreter/bytecode_kernels_ruy.h | 81 + .../interpreter_command_processor.cc | 68 + .../interpreter_command_processor.h | 36 + hal/interpreter/interpreter_context.cc | 66 + hal/interpreter/interpreter_context.h | 50 + hal/interpreter/interpreter_device.cc | 177 ++ hal/interpreter/interpreter_device.h | 77 + hal/interpreter/interpreter_driver.cc | 62 + hal/interpreter/interpreter_driver.h | 39 + hal/interpreter/interpreter_driver_module.cc | 37 + hal/resource.h | 33 + hal/semaphore.h | 61 + hal/stack_trace.h | 39 + hal/testing/mock_allocator.h | 55 + hal/testing/mock_command_buffer.h | 80 + hal/testing/mock_command_queue.h | 44 + hal/vulkan/debug_reporter.cc | 160 ++ hal/vulkan/debug_reporter.h | 87 + hal/vulkan/direct_command_buffer.cc | 470 ++++++ hal/vulkan/direct_command_buffer.h | 95 ++ hal/vulkan/direct_command_queue.cc | 207 +++ hal/vulkan/direct_command_queue.h | 71 + hal/vulkan/dynamic_symbol_tables.h | 489 ++++++ hal/vulkan/dynamic_symbols.cc | 197 +++ hal/vulkan/dynamic_symbols.h | 129 ++ hal/vulkan/extensibility_util.cc | 217 +++ hal/vulkan/extensibility_util.h | 100 ++ hal/vulkan/handle_util.h | 136 ++ hal/vulkan/legacy_fence.cc | 396 +++++ hal/vulkan/legacy_fence.h | 200 +++ hal/vulkan/native_binary_semaphore.cc | 32 + hal/vulkan/native_binary_semaphore.h | 46 + hal/vulkan/native_event.cc | 31 + hal/vulkan/native_event.h | 44 + hal/vulkan/pipeline_cache.cc | 235 +++ hal/vulkan/pipeline_cache.h | 85 + hal/vulkan/pipeline_executable.cc | 138 ++ hal/vulkan/pipeline_executable.h | 90 ++ hal/vulkan/status_util.cc | 231 +++ hal/vulkan/status_util.h | 87 + hal/vulkan/vma_allocator.cc | 248 +++ hal/vulkan/vma_allocator.h | 110 ++ hal/vulkan/vma_buffer.cc | 163 ++ hal/vulkan/vma_buffer.h | 79 + hal/vulkan/vulkan_device.cc | 481 ++++++ hal/vulkan/vulkan_device.h | 117 ++ hal/vulkan/vulkan_driver.cc | 224 +++ hal/vulkan/vulkan_driver.h | 84 + hal/vulkan/vulkan_driver_module.cc | 100 ++ samples/hal/simple_compute_test.mlir | 5 + samples/vm/simple_module_test.mlir | 5 + schemas/archive_def.fbs | 14 + schemas/bytecode/bytecode_v0.h | 143 ++ schemas/bytecode/interpreter_bytecode_v0.h | 325 ++++ schemas/bytecode/sequencer_bytecode_v0.h | 313 ++++ schemas/bytecode_def.fbs | 8 + schemas/debug_service.fbs | 347 ++++ schemas/device_def.fbs | 11 + schemas/device_group_def.fbs | 9 + schemas/device_table_def.fbs | 13 + schemas/executable_def.fbs | 38 + schemas/executable_table_def.fbs | 28 + schemas/function_def.fbs | 18 + schemas/function_table_def.fbs | 9 + schemas/module_def.fbs | 20 + schemas/source_map_def.fbs | 46 + schemas/spirv_executable_def.fbs | 108 ++ schemas/type_def.fbs | 54 + test/e2e/cmpf.mlir | 20 + test/e2e/cmpi.mlir | 28 + test/e2e/extract_element.mlir | 10 + test/e2e/scalars.mlir | 8 + test/e2e/xla/compare.mlir | 163 ++ test/e2e/xla/convert_int.mlir | 16 + test/e2e/xla/exp.mlir | 39 + test/e2e/xla/floor.mlir | 39 + test/e2e/xla/fragment_000.mlir | 33 + test/e2e/xla/fullyconnected.mlir | 94 ++ test/e2e/xla/gemm.mlir | 10 + test/e2e/xla/log.mlir | 29 + test/e2e/xla/max_float.mlir | 43 + test/e2e/xla/max_int.mlir | 65 + test/e2e/xla/min_float.mlir | 43 + test/e2e/xla/min_int.mlir | 65 + test/e2e/xla/mnist.mlir | 48 + test/e2e/xla/pad.mlir | 12 + test/e2e/xla/reduce_float.mlir | 173 ++ test/e2e/xla/reduce_int.mlir | 171 ++ test/e2e/xla/reverse.mlir | 15 + test/e2e/xla/rsqrt.mlir | 29 + test/e2e/xla/select.mlir | 9 + test/e2e/xla/slice.mlir | 33 + test/e2e/xla/through_std.mlir | 11 + test/e2e/xla/unidirectional_lstm.mlir | 680 ++++++++ test/models/mnist.mlir | 54 + tools/debugger/debug_app.cc | 1435 +++++++++++++++++ tools/debugger/debug_app.h | 214 +++ tools/debugger/debug_app.html | 75 + tools/debugger/debug_app_embedded.cc | 151 ++ tools/debugger/debug_app_embedded.h | 52 + tools/debugger/debug_app_main_emscripten.cc | 69 + tools/debugger/debug_app_main_native.cc | 45 + tools/debugger/debug_cli_main.cc | 40 + tools/debugger/debug_prompt.cc | 90 ++ tools/debugger/debug_prompt.h | 35 + tools/run_mlir_main.cc | 343 ++++ tools/run_module_main.cc | 203 +++ tools/sanitizer_suppressions.txt | 1 + tools/web/run_module.html | 319 ++++ tools/web/run_module_emscripten.cc | 147 ++ vm/bytecode_printer.cc | 506 ++++++ vm/bytecode_printer.h | 68 + vm/bytecode_reader.cc | 296 ++++ vm/bytecode_reader.h | 169 ++ vm/bytecode_tables_interpreter.cc | 44 + vm/bytecode_tables_interpreter.h | 28 + vm/bytecode_tables_sequencer.cc | 44 + vm/bytecode_tables_sequencer.h | 28 + vm/bytecode_util.cc | 43 + vm/bytecode_util.h | 31 + vm/bytecode_validator.cc | 28 + vm/bytecode_validator.h | 38 + vm/context.cc | 111 ++ vm/context.h | 91 ++ vm/debug/debug_client.cc | 64 + vm/debug/debug_client.h | 280 ++++ vm/debug/debug_client_tcp.cc | 1121 +++++++++++++ vm/debug/debug_server.h | 86 + vm/debug/debug_server_disabled.cc | 28 + vm/debug/debug_server_flags.cc | 80 + vm/debug/debug_server_flags.h | 33 + vm/debug/debug_server_tcp.cc | 462 ++++++ vm/debug/debug_service.cc | 854 ++++++++++ vm/debug/debug_service.h | 178 ++ vm/debug/debug_session.cc | 49 + vm/debug/debug_session.h | 93 ++ vm/debug/debug_tcp_util.h | 217 +++ vm/executable_table.cc | 65 + vm/executable_table.h | 54 + vm/fiber_state.cc | 75 + vm/fiber_state.h | 113 ++ vm/function.cc | 121 ++ vm/function.h | 124 ++ vm/function_table.cc | 261 +++ vm/function_table.h | 125 ++ vm/instance.cc | 37 + vm/instance.h | 55 + vm/module.cc | 77 + vm/module.h | 63 + vm/module_printer.cc | 55 + vm/module_printer.h | 44 + vm/opcode_info.h | 45 + vm/sequencer_context.cc | 167 ++ vm/sequencer_context.h | 55 + vm/sequencer_dispatch.cc | 561 +++++++ vm/sequencer_dispatch.h | 35 + vm/source_map.cc | 206 +++ vm/source_map.h | 104 ++ vm/stack.cc | 82 + vm/stack.h | 73 + vm/stack_frame.cc | 46 + vm/stack_frame.h | 77 + vm/type.cc | 64 + vm/type.h | 65 + 400 files changed, 59625 insertions(+), 32 deletions(-) delete mode 100644 .gitignore create mode 100644 CONTRIBUTING.md create mode 100644 LICENSE create mode 100644 README.md create mode 100644 base/arena.cc create mode 100644 base/arena.h create mode 100644 base/bitfield.h create mode 100644 base/file_io.h create mode 100644 base/flatbuffer_util.cc create mode 100644 base/flatbuffer_util.h create mode 100644 base/init.h create mode 100644 base/internal/init_internal.cc create mode 100644 base/internal/init_internal.h create mode 100644 base/internal/logging.cc create mode 100644 base/internal/logging.h create mode 100644 base/internal/status.cc create mode 100644 base/internal/status.h create mode 100644 base/internal/status_builder.cc create mode 100644 base/internal/status_builder.h create mode 100644 base/internal/status_errno.cc create mode 100644 base/internal/status_errno.h create mode 100644 base/internal/status_errors.cc create mode 100644 base/internal/status_errors.h create mode 100644 base/internal/status_macros.h create mode 100644 base/internal/statusor.cc create mode 100644 base/internal/statusor.h create mode 100644 base/intrusive_list.h create mode 100644 base/intrusive_list_ref_ptr.inc create mode 100644 base/intrusive_list_unique_ptr.inc create mode 100644 base/logging.h create mode 100644 base/memory.h create mode 100644 base/ref_ptr.h create mode 100644 base/shape.cc create mode 100644 base/shape.h create mode 100644 base/status.h create mode 100644 base/time.h create mode 100644 base/tracing.cc create mode 100644 base/tracing.h create mode 100644 base/tracing_disabled.cc create mode 100644 base/wait_handle.cc create mode 100644 base/wait_handle.h create mode 100644 compiler/IR/ConfigOps.cpp create mode 100644 compiler/IR/ConfigOps.h create mode 100644 compiler/IR/ConfigOps.td create mode 100644 compiler/IR/Dialect.cpp create mode 100644 compiler/IR/Dialect.h create mode 100644 compiler/IR/Interpreter/HLDialect.cpp create mode 100644 compiler/IR/Interpreter/HLDialect.h create mode 100644 compiler/IR/Interpreter/HLOps.cpp create mode 100644 compiler/IR/Interpreter/HLOps.h create mode 100644 compiler/IR/Interpreter/HLOps.td create mode 100644 compiler/IR/Interpreter/LLDialect.cpp create mode 100644 compiler/IR/Interpreter/LLDialect.h create mode 100644 compiler/IR/Interpreter/LLOps.cpp create mode 100644 compiler/IR/Interpreter/LLOps.h create mode 100644 compiler/IR/Interpreter/LLOps.td create mode 100644 compiler/IR/Interpreter/OpWriters.cpp create mode 100644 compiler/IR/Interpreter/OpWriters.h create mode 100644 compiler/IR/Interpreter/test/concat.mlir create mode 100644 compiler/IR/Interpreter/test/invalid_types_hl.mlir create mode 100644 compiler/IR/Interpreter/test/invalid_types_ll.mlir create mode 100644 compiler/IR/OpBase.td create mode 100644 compiler/IR/Ops.cpp create mode 100644 compiler/IR/Ops.h create mode 100644 compiler/IR/Ops.td create mode 100644 compiler/IR/Sequencer/HLDialect.cpp create mode 100644 compiler/IR/Sequencer/HLDialect.h create mode 100644 compiler/IR/Sequencer/HLOps.cpp create mode 100644 compiler/IR/Sequencer/HLOps.h create mode 100644 compiler/IR/Sequencer/HLOps.td create mode 100644 compiler/IR/Sequencer/LLDialect.cpp create mode 100644 compiler/IR/Sequencer/LLDialect.h create mode 100644 compiler/IR/Sequencer/LLOps.cpp create mode 100644 compiler/IR/Sequencer/LLOps.h create mode 100644 compiler/IR/Sequencer/LLOps.td create mode 100644 compiler/IR/Sequencer/OpWriters.cpp create mode 100644 compiler/IR/Sequencer/OpWriters.h create mode 100644 compiler/IR/Sequencer/test/concat.mlir create mode 100644 compiler/IR/StructureOps.cpp create mode 100644 compiler/IR/StructureOps.h create mode 100644 compiler/IR/StructureOps.td create mode 100644 compiler/IR/Traits.cpp create mode 100644 compiler/IR/Traits.h create mode 100644 compiler/IR/Types.cpp create mode 100644 compiler/IR/Types.h create mode 100644 compiler/IR/test/bindings.mlir create mode 100644 compiler/IR/test/dispatch_regions.mlir create mode 100644 compiler/IR/test/reduction_regions.mlir create mode 100644 compiler/Serialization/BytecodeTables.cpp create mode 100644 compiler/Serialization/BytecodeTables.h create mode 100644 compiler/Serialization/BytecodeWriter.cpp create mode 100644 compiler/Serialization/BytecodeWriter.h create mode 100644 compiler/Serialization/VMDeviceTableBuilder.cpp create mode 100644 compiler/Serialization/VMDeviceTableBuilder.h create mode 100644 compiler/Serialization/VMExecutableTableBuilder.cpp create mode 100644 compiler/Serialization/VMExecutableTableBuilder.h create mode 100644 compiler/Serialization/VMFunctionBuilder.cpp create mode 100644 compiler/Serialization/VMFunctionBuilder.h create mode 100644 compiler/Serialization/VMFunctionTableBuilder.cpp create mode 100644 compiler/Serialization/VMFunctionTableBuilder.h create mode 100644 compiler/Serialization/VMModuleBuilder.cpp create mode 100644 compiler/Serialization/VMModuleBuilder.h create mode 100644 compiler/Serialization/VMSourceMapBuilder.cpp create mode 100644 compiler/Serialization/VMSourceMapBuilder.h create mode 100644 compiler/Transforms/AggressiveOpElimination.cpp create mode 100644 compiler/Transforms/AssignFunctionOrdinals.cpp create mode 100644 compiler/Transforms/ConversionUtils.h create mode 100644 compiler/Transforms/ConvertFromTupleCallingConvention.cpp create mode 100644 compiler/Transforms/ConvertToMemRefCallingConvention.cpp create mode 100644 compiler/Transforms/DropUnreachableFunctions.cpp create mode 100644 compiler/Transforms/DropUnusedExecutables.cpp create mode 100644 compiler/Transforms/Interpreter/ExpandReductionsToOps.cpp create mode 100644 compiler/Transforms/Interpreter/LegalizeInterpreterOps.cpp create mode 100644 compiler/Transforms/Interpreter/LegalizeInterpreterOps.td create mode 100644 compiler/Transforms/Interpreter/LoadStoreDataFlowOpt.cpp create mode 100644 compiler/Transforms/Interpreter/LowerInterpreterDialect.cpp create mode 100644 compiler/Transforms/Interpreter/LowerStdToInterpreterDialect.cpp create mode 100644 compiler/Transforms/Interpreter/LowerXLAToInterpreterDialect.cpp create mode 100644 compiler/Transforms/Interpreter/MakeExecutableABI.cpp create mode 100644 compiler/Transforms/Interpreter/Passes.h create mode 100644 compiler/Transforms/Interpreter/test/load_store_data_flow_opt.mlir create mode 100644 compiler/Transforms/Interpreter/test/make_executable_abi.mlir create mode 100644 compiler/Transforms/Interpreter/test/xla/concat.mlir create mode 100644 compiler/Transforms/Interpreter/test/xla/const.mlir create mode 100644 compiler/Transforms/Interpreter/test/xla/dynamic_update_slice.mlir create mode 100644 compiler/Transforms/Interpreter/test/xla/gather.mlir create mode 100644 compiler/Transforms/Interpreter/test/xla/slice.mlir create mode 100644 compiler/Transforms/LegalizeTensorMemRef.cpp create mode 100644 compiler/Transforms/LegalizeTupleElementAccess.cpp create mode 100644 compiler/Transforms/LegalizeTypeStorage.cpp create mode 100644 compiler/Transforms/Passes.h create mode 100644 compiler/Transforms/Sequencer/AssignExecutableOrdinals.cpp create mode 100644 compiler/Transforms/Sequencer/AssignExecutableWorkloadAttrs.cpp create mode 100644 compiler/Transforms/Sequencer/FoldCompatibleDispatchRegions.cpp create mode 100644 compiler/Transforms/Sequencer/IdentifyDispatchRegions.cpp create mode 100644 compiler/Transforms/Sequencer/IdentifyReductionRegions.cpp create mode 100644 compiler/Transforms/Sequencer/LoadStoreDataFlowOpt.cpp create mode 100644 compiler/Transforms/Sequencer/LowerSequencerDialect.cpp create mode 100644 compiler/Transforms/Sequencer/LowerStdToSequencerDialect.cpp create mode 100644 compiler/Transforms/Sequencer/LowerXLAToSequencerDialect.cpp create mode 100644 compiler/Transforms/Sequencer/OutlineDispatchRegions.cpp create mode 100644 compiler/Transforms/Sequencer/OutlineReductionRegions.cpp create mode 100644 compiler/Transforms/Sequencer/Passes.h create mode 100644 compiler/Transforms/Sequencer/RematerializeDispatchConstants.cpp create mode 100644 compiler/Transforms/test/drop_unreachable_module_functions.mlir create mode 100644 compiler/Translation/Interpreter/InterpreterExecutableTranslation.cpp create mode 100644 compiler/Translation/Interpreter/InterpreterExecutableTranslation.h create mode 100644 compiler/Translation/SPIRV/AffineExprCodegen.h create mode 100644 compiler/Translation/SPIRV/IREEIndexComputation.cpp create mode 100644 compiler/Translation/SPIRV/IREEIndexComputation.h create mode 100644 compiler/Translation/SPIRV/IREEToSPIRV.cpp create mode 100644 compiler/Translation/SPIRV/IREEToSPIRV.h create mode 100644 compiler/Translation/SPIRV/IREEToSPIRVPass.cpp create mode 100644 compiler/Translation/SPIRV/IREEToSPIRVPass.h create mode 100644 compiler/Translation/SPIRV/IndexComputation.cpp create mode 100644 compiler/Translation/SPIRV/IndexComputation.h create mode 100644 compiler/Translation/SPIRV/Kernels/matmul.comp create mode 100644 compiler/Translation/SPIRV/SPIRVExecutableTranslation.cpp create mode 100644 compiler/Translation/SPIRV/SPIRVExecutableTranslation.h create mode 100644 compiler/Translation/SPIRV/SPIRVLowering.cpp create mode 100644 compiler/Translation/SPIRV/SPIRVLowering.h create mode 100644 compiler/Translation/SPIRV/XLAIndexPropagation.cpp create mode 100644 compiler/Translation/SPIRV/XLAIndexPropagation.h create mode 100644 compiler/Translation/SPIRV/test/simple_test.mlir create mode 100644 compiler/Translation/SequencerModuleTranslation.cpp create mode 100644 compiler/Translation/SequencerModuleTranslation.h create mode 100644 compiler/Utils/DispatchUtils.cpp create mode 100644 compiler/Utils/DispatchUtils.h create mode 100644 compiler/Utils/Macros.h create mode 100644 compiler/Utils/MemRefUtils.cpp create mode 100644 compiler/Utils/MemRefUtils.h create mode 100644 compiler/Utils/ModuleUtils.cpp create mode 100644 compiler/Utils/ModuleUtils.h create mode 100644 compiler/Utils/OpUtils.cpp create mode 100644 compiler/Utils/OpUtils.h create mode 100644 compiler/Utils/TranslationUtils.cpp create mode 100644 compiler/Utils/TranslationUtils.h create mode 100644 hal/allocator.cc create mode 100644 hal/allocator.h create mode 100644 hal/buffer.cc create mode 100644 hal/buffer.h create mode 100644 hal/buffer_view.cc create mode 100644 hal/buffer_view.h create mode 100644 hal/buffer_view_string_util.cc create mode 100644 hal/buffer_view_string_util.h create mode 100644 hal/command_buffer.cc create mode 100644 hal/command_buffer.h create mode 100644 hal/command_buffer_validation.cc create mode 100644 hal/command_buffer_validation.h create mode 100644 hal/command_queue.h create mode 100644 hal/deferred_buffer.cc create mode 100644 hal/deferred_buffer.h create mode 100644 hal/device.h create mode 100644 hal/device_info.h create mode 100644 hal/device_manager.cc create mode 100644 hal/device_manager.h create mode 100644 hal/device_placement.h create mode 100644 hal/driver.h create mode 100644 hal/driver_registry.cc create mode 100644 hal/driver_registry.h create mode 100644 hal/event.h create mode 100644 hal/executable.h create mode 100644 hal/executable_cache.cc create mode 100644 hal/executable_cache.h create mode 100644 hal/executable_format.h create mode 100644 hal/executable_spec.h create mode 100644 hal/fence.h create mode 100644 hal/heap_buffer.cc create mode 100644 hal/heap_buffer.h create mode 100644 hal/host/async_command_queue.cc create mode 100644 hal/host/async_command_queue.h create mode 100644 hal/host/host_buffer.cc create mode 100644 hal/host/host_buffer.h create mode 100644 hal/host/host_event.cc create mode 100644 hal/host/host_event.h create mode 100644 hal/host/host_fence.cc create mode 100644 hal/host/host_fence.h create mode 100644 hal/host/host_local_allocator.cc create mode 100644 hal/host/host_local_allocator.h create mode 100644 hal/host/host_local_command_processor.cc create mode 100644 hal/host/host_local_command_processor.h create mode 100644 hal/host/host_submission_queue.cc create mode 100644 hal/host/host_submission_queue.h create mode 100644 hal/host/inproc_command_buffer.cc create mode 100644 hal/host/inproc_command_buffer.h create mode 100644 hal/interpreter/bytecode_cache.cc create mode 100644 hal/interpreter/bytecode_cache.h create mode 100644 hal/interpreter/bytecode_dispatch.cc create mode 100644 hal/interpreter/bytecode_dispatch.h create mode 100644 hal/interpreter/bytecode_dispatch_conversion.h create mode 100644 hal/interpreter/bytecode_dispatch_util.cc create mode 100644 hal/interpreter/bytecode_dispatch_util.h create mode 100644 hal/interpreter/bytecode_executable.cc create mode 100644 hal/interpreter/bytecode_executable.h create mode 100644 hal/interpreter/bytecode_kernels.h create mode 100644 hal/interpreter/bytecode_kernels_generic.h create mode 100644 hal/interpreter/bytecode_kernels_ruy.h create mode 100644 hal/interpreter/interpreter_command_processor.cc create mode 100644 hal/interpreter/interpreter_command_processor.h create mode 100644 hal/interpreter/interpreter_context.cc create mode 100644 hal/interpreter/interpreter_context.h create mode 100644 hal/interpreter/interpreter_device.cc create mode 100644 hal/interpreter/interpreter_device.h create mode 100644 hal/interpreter/interpreter_driver.cc create mode 100644 hal/interpreter/interpreter_driver.h create mode 100644 hal/interpreter/interpreter_driver_module.cc create mode 100644 hal/resource.h create mode 100644 hal/semaphore.h create mode 100644 hal/stack_trace.h create mode 100644 hal/testing/mock_allocator.h create mode 100644 hal/testing/mock_command_buffer.h create mode 100644 hal/testing/mock_command_queue.h create mode 100644 hal/vulkan/debug_reporter.cc create mode 100644 hal/vulkan/debug_reporter.h create mode 100644 hal/vulkan/direct_command_buffer.cc create mode 100644 hal/vulkan/direct_command_buffer.h create mode 100644 hal/vulkan/direct_command_queue.cc create mode 100644 hal/vulkan/direct_command_queue.h create mode 100644 hal/vulkan/dynamic_symbol_tables.h create mode 100644 hal/vulkan/dynamic_symbols.cc create mode 100644 hal/vulkan/dynamic_symbols.h create mode 100644 hal/vulkan/extensibility_util.cc create mode 100644 hal/vulkan/extensibility_util.h create mode 100644 hal/vulkan/handle_util.h create mode 100644 hal/vulkan/legacy_fence.cc create mode 100644 hal/vulkan/legacy_fence.h create mode 100644 hal/vulkan/native_binary_semaphore.cc create mode 100644 hal/vulkan/native_binary_semaphore.h create mode 100644 hal/vulkan/native_event.cc create mode 100644 hal/vulkan/native_event.h create mode 100644 hal/vulkan/pipeline_cache.cc create mode 100644 hal/vulkan/pipeline_cache.h create mode 100644 hal/vulkan/pipeline_executable.cc create mode 100644 hal/vulkan/pipeline_executable.h create mode 100644 hal/vulkan/status_util.cc create mode 100644 hal/vulkan/status_util.h create mode 100644 hal/vulkan/vma_allocator.cc create mode 100644 hal/vulkan/vma_allocator.h create mode 100644 hal/vulkan/vma_buffer.cc create mode 100644 hal/vulkan/vma_buffer.h create mode 100644 hal/vulkan/vulkan_device.cc create mode 100644 hal/vulkan/vulkan_device.h create mode 100644 hal/vulkan/vulkan_driver.cc create mode 100644 hal/vulkan/vulkan_driver.h create mode 100644 hal/vulkan/vulkan_driver_module.cc create mode 100644 samples/hal/simple_compute_test.mlir create mode 100644 samples/vm/simple_module_test.mlir create mode 100644 schemas/archive_def.fbs create mode 100644 schemas/bytecode/bytecode_v0.h create mode 100644 schemas/bytecode/interpreter_bytecode_v0.h create mode 100644 schemas/bytecode/sequencer_bytecode_v0.h create mode 100644 schemas/bytecode_def.fbs create mode 100644 schemas/debug_service.fbs create mode 100644 schemas/device_def.fbs create mode 100644 schemas/device_group_def.fbs create mode 100644 schemas/device_table_def.fbs create mode 100644 schemas/executable_def.fbs create mode 100644 schemas/executable_table_def.fbs create mode 100644 schemas/function_def.fbs create mode 100644 schemas/function_table_def.fbs create mode 100644 schemas/module_def.fbs create mode 100644 schemas/source_map_def.fbs create mode 100644 schemas/spirv_executable_def.fbs create mode 100644 schemas/type_def.fbs create mode 100644 test/e2e/cmpf.mlir create mode 100644 test/e2e/cmpi.mlir create mode 100644 test/e2e/extract_element.mlir create mode 100644 test/e2e/scalars.mlir create mode 100644 test/e2e/xla/compare.mlir create mode 100644 test/e2e/xla/convert_int.mlir create mode 100644 test/e2e/xla/exp.mlir create mode 100644 test/e2e/xla/floor.mlir create mode 100644 test/e2e/xla/fragment_000.mlir create mode 100644 test/e2e/xla/fullyconnected.mlir create mode 100644 test/e2e/xla/gemm.mlir create mode 100644 test/e2e/xla/log.mlir create mode 100644 test/e2e/xla/max_float.mlir create mode 100644 test/e2e/xla/max_int.mlir create mode 100644 test/e2e/xla/min_float.mlir create mode 100644 test/e2e/xla/min_int.mlir create mode 100644 test/e2e/xla/mnist.mlir create mode 100644 test/e2e/xla/pad.mlir create mode 100644 test/e2e/xla/reduce_float.mlir create mode 100644 test/e2e/xla/reduce_int.mlir create mode 100644 test/e2e/xla/reverse.mlir create mode 100644 test/e2e/xla/rsqrt.mlir create mode 100644 test/e2e/xla/select.mlir create mode 100644 test/e2e/xla/slice.mlir create mode 100644 test/e2e/xla/through_std.mlir create mode 100644 test/e2e/xla/unidirectional_lstm.mlir create mode 100644 test/models/mnist.mlir create mode 100644 tools/debugger/debug_app.cc create mode 100644 tools/debugger/debug_app.h create mode 100644 tools/debugger/debug_app.html create mode 100644 tools/debugger/debug_app_embedded.cc create mode 100644 tools/debugger/debug_app_embedded.h create mode 100644 tools/debugger/debug_app_main_emscripten.cc create mode 100644 tools/debugger/debug_app_main_native.cc create mode 100644 tools/debugger/debug_cli_main.cc create mode 100644 tools/debugger/debug_prompt.cc create mode 100644 tools/debugger/debug_prompt.h create mode 100644 tools/run_mlir_main.cc create mode 100644 tools/run_module_main.cc create mode 100644 tools/sanitizer_suppressions.txt create mode 100644 tools/web/run_module.html create mode 100644 tools/web/run_module_emscripten.cc create mode 100644 vm/bytecode_printer.cc create mode 100644 vm/bytecode_printer.h create mode 100644 vm/bytecode_reader.cc create mode 100644 vm/bytecode_reader.h create mode 100644 vm/bytecode_tables_interpreter.cc create mode 100644 vm/bytecode_tables_interpreter.h create mode 100644 vm/bytecode_tables_sequencer.cc create mode 100644 vm/bytecode_tables_sequencer.h create mode 100644 vm/bytecode_util.cc create mode 100644 vm/bytecode_util.h create mode 100644 vm/bytecode_validator.cc create mode 100644 vm/bytecode_validator.h create mode 100644 vm/context.cc create mode 100644 vm/context.h create mode 100644 vm/debug/debug_client.cc create mode 100644 vm/debug/debug_client.h create mode 100644 vm/debug/debug_client_tcp.cc create mode 100644 vm/debug/debug_server.h create mode 100644 vm/debug/debug_server_disabled.cc create mode 100644 vm/debug/debug_server_flags.cc create mode 100644 vm/debug/debug_server_flags.h create mode 100644 vm/debug/debug_server_tcp.cc create mode 100644 vm/debug/debug_service.cc create mode 100644 vm/debug/debug_service.h create mode 100644 vm/debug/debug_session.cc create mode 100644 vm/debug/debug_session.h create mode 100644 vm/debug/debug_tcp_util.h create mode 100644 vm/executable_table.cc create mode 100644 vm/executable_table.h create mode 100644 vm/fiber_state.cc create mode 100644 vm/fiber_state.h create mode 100644 vm/function.cc create mode 100644 vm/function.h create mode 100644 vm/function_table.cc create mode 100644 vm/function_table.h create mode 100644 vm/instance.cc create mode 100644 vm/instance.h create mode 100644 vm/module.cc create mode 100644 vm/module.h create mode 100644 vm/module_printer.cc create mode 100644 vm/module_printer.h create mode 100644 vm/opcode_info.h create mode 100644 vm/sequencer_context.cc create mode 100644 vm/sequencer_context.h create mode 100644 vm/sequencer_dispatch.cc create mode 100644 vm/sequencer_dispatch.h create mode 100644 vm/source_map.cc create mode 100644 vm/source_map.h create mode 100644 vm/stack.cc create mode 100644 vm/stack.h create mode 100644 vm/stack_frame.cc create mode 100644 vm/stack_frame.h create mode 100644 vm/type.cc create mode 100644 vm/type.h diff --git a/.gitignore b/.gitignore deleted file mode 100644 index 259148fa18f9..000000000000 --- a/.gitignore +++ /dev/null @@ -1,32 +0,0 @@ -# Prerequisites -*.d - -# Compiled Object files -*.slo -*.lo -*.o -*.obj - -# Precompiled Headers -*.gch -*.pch - -# Compiled Dynamic libraries -*.so -*.dylib -*.dll - -# Fortran module files -*.mod -*.smod - -# Compiled Static libraries -*.lai -*.la -*.a -*.lib - -# Executables -*.exe -*.out -*.app diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md new file mode 100644 index 000000000000..db177d4ac70f --- /dev/null +++ b/CONTRIBUTING.md @@ -0,0 +1,28 @@ +# How to Contribute + +We'd love to accept your patches and contributions to this project. There are +just a few small guidelines you need to follow. + +## Contributor License Agreement + +Contributions to this project must be accompanied by a Contributor License +Agreement. You (or your employer) retain the copyright to your contribution; +this simply gives us permission to use and redistribute your contributions as +part of the project. Head over to to see +your current agreements on file or to sign a new one. + +You generally only need to submit a CLA once, so if you've already submitted one +(even if it was for a different project), you probably don't need to do it +again. + +## Code reviews + +All submissions, including submissions by project members, require review. We +use GitHub pull requests for this purpose. Consult +[GitHub Help](https://help.github.com/articles/about-pull-requests/) for more +information on using pull requests. + +## Community Guidelines + +This project follows +[Google's Open Source Community Guidelines](https://opensource.google.com/conduct/). diff --git a/LICENSE b/LICENSE new file mode 100644 index 000000000000..d64569567334 --- /dev/null +++ b/LICENSE @@ -0,0 +1,202 @@ + + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "[]" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright [yyyy] [name of copyright owner] + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. diff --git a/README.md b/README.md new file mode 100644 index 000000000000..8823f241111f --- /dev/null +++ b/README.md @@ -0,0 +1,243 @@ +# IREE: An Experimental MLIR Execution Environment + +**DISCLAIMER**: This is not an officially supported Google product. It's an +experimental playground for low-level/tightly integrated machine learning +libraries that make use of modern hardware acceleration APIs and techniques (see +[non goals](#non-goals)). + +## Table of Contents + +- [Project Goals](#project-goals) +- [Milestones](#milestones) +- [Status](#status) +- [Dependencies](#dependencies) +- [Quickstart](#quickstart) +- [License](#license) + + + +## Project Goals + +IREE (**I**ntermediate **R**epresentation **E**xecution **E**nvironment, +pronounced as "eerie") is an experimental compiler backend for +[MLIR](https://github.com/tensorflow/mlir) that lowers ML models to an IR that +is optimized for real-time mobile/edge inference against heterogeneous hardware +accelerators. + +The IR produced contains the sequencing information required to communicate +pipelined data dependencies and parallelism to low-level hardware APIs like +Vulkan and embed hardware/API-specific binaries such as SPIR-V or compiled ARM +code. As the IR is specified against an abstract execution environment there are +many potential ways to run a compiled model, and one such way is included as an +example and testbed for runtime optimization experiments. + +The included layered runtime scales from generated code for a particular API +(such as emitting C code calling external DSP kernels), to a HAL (**H**ardware +**A**bstraction **L**ayer) that allows the same generated code to target +multiple APIs (like Vulkan and Direct3D 12), to a full VM allowing runtime model +loading for flexible deployment options and heterogenous execution. Consider +both the compiler and the included runtime a toolbox for making it easier - via +the versatility of MLIR - to take ML models from their source to some varying +degree of integration with your application. + +### Demonstrate MLIR + +IREE has been developed alongside MLIR and is used as an example of how +non-traditional ML compiler backends and runtimes can be built: it focuses more +on the math being performed and how that math is scheduled rather than graphs of +"ops" and in some cases allows doing away with a runtime entirely. It seeks to +show how more holistic approaches that exploit the MLIR framework and its +various dialects can be both easy to understand and powerful in the +optimizations to code size, runtime complexity, and performance they enable. + +### Demonstrate Advanced Models + +By using models with much greater complexity than the usual references (such as +MobileNet) we want to show how weird things can get when model authors are +allowed to get creative: dynamic shapes, dynamic flow control, dynamic +multi-model dispatch (including models that conditionally dispatch other +models), streaming models, tree-based search algorithms, etc. We are trying to +build IREE from the ground-up to enable these models and run them efficiently on +modern hardware. Many of our example models are sequence-to-sequence language +models from the [Lingvo](https://github.com/tensorflow/lingvo) project +representing cutting edge speech recognition and translation work. + +### Demonstrate ML-as-a-Game-Engine + +An observation that has driven the development of IREE is one of ML workloads +not being much different than traditional game rendering workloads: math is +performed on buffers with varying levels of concurrency and ordering in a +pipelined fashion against accelerators designed to make such operations fast. In +fact, most ML is performed on the same hardware that was designed for games! Our +approach is to use the compiler to transform ML workloads to ones that look +eerily _(pun intended)_ similar to what a game performs in per-frame render +workloads, optimize for low-latency and predictable execution, and integrate +well into existing systems both for batched and interactive usage. The IREE +runtime is designed to feel more like game engine middleware than a standalone +ML inference system, though we still have much work towards that goal. This +should make it easy to use existing tools for high-performance/low-power +optimization of GPU workloads, identify driver or system issues introducing +latency, and help to improve the ecosystem overall. + +### Demonstrate Standards-based ML via Vulkan and SPIR-V + +With the above observation that ML can look like games from the systems +perspective it follows that APIs and technologies good for games should probably +also be good for ML. In many cases we've identified only a few key differences +that exist and just as extensions have been added and API improvements have been +made to graphics/compute standards for decades we hope to demonstrate and +evaluate small, tactical changes that can have large impacts on ML performance +through these standard APIs. We would love to allow hardware vendors to be able +to make ML efficient on their hardware without the need for bespoke runtimes and +special access such that any ML workload produced by any tool runs well. We'd +consider the IREE experiment a success if what resulted was some worked examples +that help advance the entire ecosystem! + + + +## Non-Goals + +* Replace parts of the supported TensorFlow ecosystem of tools: The authors + within Google work closely with TensorFlow and contribute to it regularly. + However, IREE is exploring some different angles of the problem and is + experimental. We will seek to leverage anything of value that we learn in an + appropriate way to make TensorFlow better over time, but the two should not + be conflated. +* Providing an [SLA](https://en.wikipedia.org/wiki/Service-level_agreement) of + any kind: IREE is infrastructure research, not a supported product. If it + gains mind-share or traction, we would revisit that in conjunction with + finding a more permanent way to align it with the broader constellation of + ML tooling. + + + +## Milestones + +We are currently just at the starting line, with basic +[MNIST MLP](https://github.com/keras-team/keras/blob/master/examples/mnist_mlp.py) +running end-to-end on both a CPU interpreter and Vulkan. As we scale out the +compiler we will be increasing the complexity of the models that can run and +demonstrating more of the optimizations we've found useful in previous efforts +to run them efficiently on edge devices. + +We'll be setting up GitHub milestones with issues tracking major feature work we +are planning. For now, our areas of work are: + +* Allocation tracking and aliasing in the compiler +* Pipelined scheduler in the VM for issuing proper command buffers +* New CPU interpreter that enables lightweight execution on ARM and x86 +* C code generator and API to demonstrate "runtimeless" mode +* Quantization using the MLIR quantization framework +* High-level integration and examples when working with TensorFlow 2.0 +* Switching from IREE's XLA-to-SPIR-V backend to the general MLIR SPIR-V + backend + +Things we are interested in but don't yet have in-progress: + +* Ahead-of-time compiled ARM NEON backend (perhaps via + [SPIRV-LLVM](https://github.com/KhronosGroup/SPIRV-LLVM-Translator/), + [SPIRV-to-ISPC](https://github.com/GameTechDev/SPIRV-Cross), or some other + technique) +* HAL backends for Metal 2 and Direct3D 12 +* Profile-guided optimization support for scheduling feedback + + + +## Current Status + +### Documentation + +Coming soon :) + +### Build System and CI + +We are still working to port our build configuration to +[Bazel](https://bazel.build/) and will be open sourcing the BUILD files and +Starlark rules we use soon. We will also be maintaining a CMake build for +integration with the dependent projects +([Abseil](https://github.com/abseil/abseil-cpp) and +[MLIR](https://github.com/tensorflow/mlir)). + +### Code and Style + +The project is currently _very_ early and a mix of code written prior to a lot +of the more recent ergonomics improvements in MLIR and its tablegen. Future +changes will replace the legacy code style with prettier forms and simplify the +project structure to make it easier to separate the different components. Some +entire portions of the code (such as the CPU interpreter) will likely be dropped +or rewritten. For now, assume churn! + +The compiler portions of the code (almost exclusively under `iree/compiler/`) +follows the LLVM style guide and has the same system requirements as MLIR +itself. It general requires a more modern C++ compiler. + +The runtime portions vary but most are designed to work with C++11 and use +[Abseil](https://github.com/abseil/abseil-cpp) to bring in future C++14 and +C++17 features. + +### Hardware Support + +We are mostly targeting Vulkan and Metal on recent mobile devices and as such +have limited our usage of hardware features and vendor extensions to those we +have broad access to there. This is mainly just to keep our focus tight and does +not preclude usage of features outside the standard sets or for other hardware +types (in fact, we have a lot of fun ideas for +`VK_NVX_device_generated_commands` and Metal 2.1's Indirect Command Buffers!). + + + +## Dependencies + +NOTE: during the initial open source release we are still cleaning things up. If +there's weird dependencies/layering that makes life difficult for your +particular use case please file an issue so we can make sure to fix it. + +### Compiler + +The compiler has several layers that allow scaling the dependencies required +based on the source and target formats. In all cases +[MLIR](https://github.com/tensorflow/mlir) is required and for models not +originating from TensorFlow (or already in XLA HLO format) it is the only +dependency. When targeting the IREE Runtime VM and HAL +[FlatBuffers](https://google.github.io/flatbuffers/) is required for +serialization. Converting from TensorFlow models requires a dependency on +TensorFlow (however only those parts required for conversion). + +### Runtime VM + +The VM providing dynamic model deployment and advanced scheduling behavior +requires [Abseil](https://github.com/abseil/abseil-cpp) for its common types, +however contributions are welcome to make it possible to replace Abseil with +other libraries via shims/forwarding. The core types used by the runtime +(excluding command line flags and such in tools) are limited to types coming in +future C++ versions (variant, optional, string_view, etc), cheap types +(absl::Span), or simple standard containers (absl::InlinedVector). +[FlatBuffers](https://google.github.io/flatbuffers/) is used to load compiled +modules. + +### Runtime HAL + +The HAL and the provided implementations (Vulkan, etc) also use +[Abseil](https://github.com/abseil/abseil-cpp). Contributions are welcome to +allow other types to be swapped in. A C99 HAL API is planned for code generation +targets that will use no dependencies. + +### Testing and Tooling + +[Swiftshader](https://github.com/google/swiftshader) is used to provide fast +hardware-independent testing of the Vulkan and SPIR-V portions of the toolchain. + + + +## Quickstart + +Coming soon! Performing full model translation may require a few steps (such as +ensuring you have a working TensorFlow build), however we'll have pre-translated +example models that allow independent testing of the runtime portions. + + + +## License + +IREE is licensed under the terms of the Apache license. See [LICENSE](LICENSE) +for more information. diff --git a/base/arena.cc b/base/arena.cc new file mode 100644 index 000000000000..9e9776359e38 --- /dev/null +++ b/base/arena.cc @@ -0,0 +1,125 @@ +// Copyright 2019 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "third_party/mlir_edge/iree/base/arena.h" + +#include + +#include "third_party/absl/base/attributes.h" +#include "third_party/mlir_edge/iree/base/logging.h" + +namespace iree { + +namespace { + +// Rounds up to the next alignment value, if it is not already aligned. +template +ABSL_ATTRIBUTE_ALWAYS_INLINE constexpr T RoundToAlignment( + T value, T alignment) noexcept { + return ((value + alignment - 1) / alignment) * alignment; +} + +} // namespace + +Arena::Arena(size_t block_size) : block_size_(block_size) {} + +Arena::~Arena() { Clear(); } + +void Arena::Clear() { + // Deallocate all memory. + auto block_header = block_list_head_; + while (block_header) { + auto next_block = block_header->next_block; + std::free(block_header); + block_header = next_block; + } + block_list_head_ = nullptr; + block_header = unused_block_list_head_; + while (block_header) { + auto next_block = block_header->next_block; + std::free(block_header); + block_header = next_block; + } + unused_block_list_head_ = nullptr; + + bytes_allocated_ = 0; + block_bytes_allocated_ = 0; +} + +void Arena::Reset() { + // Move all blocks to the unused list and reset allocation count only. + auto block_header = block_list_head_; + while (block_header) { + auto next_block = block_header->next_block; + block_header->bytes_allocated = 0; + block_header->next_block = unused_block_list_head_; + unused_block_list_head_ = block_header; + block_header = next_block; + } + block_list_head_ = nullptr; + + bytes_allocated_ = 0; +} + +uint8_t* Arena::AllocateBytes(size_t length) { + if (!length) { + // Guarantee zero-length allocations return nullptr. + return nullptr; + } + + // Pad length allocated so we are machine word aligned. + // This ensures the next allocation starts at the right boundary. + size_t aligned_length = RoundToAlignment(length, sizeof(uintptr_t)); + + if (aligned_length > block_size_) { + // This allocation is larger than an entire block. That's bad. + // We could allocate this with malloc (and then keep track of those to free + // things), but for now let's just die. + CHECK(false); + return nullptr; + } + + if (!block_list_head_ || + block_list_head_->bytes_allocated + aligned_length > block_size_) { + // Check to see if we have an existing unused block we can use. + if (unused_block_list_head_) { + // Move block from unused list to main list. + auto block_header = unused_block_list_head_; + unused_block_list_head_ = block_header->next_block; + block_header->next_block = block_list_head_; + block_header->bytes_allocated = 0; + block_list_head_ = block_header; + } else { + // Allocate a new block. + auto block_ptr = reinterpret_cast( + std::malloc(sizeof(BlockHeader) + block_size_)); + auto block_header = reinterpret_cast(block_ptr); + block_header->next_block = block_list_head_; + block_header->bytes_allocated = 0; + block_list_head_ = block_header; + block_bytes_allocated_ += sizeof(BlockHeader) + block_size_; + } + } + + BlockHeader* target_block = block_list_head_; + auto data_ptr = reinterpret_cast(target_block) + + sizeof(BlockHeader) + target_block->bytes_allocated; + target_block->bytes_allocated += aligned_length; + + bytes_allocated_ += length; + + return data_ptr; +} + +} // namespace iree diff --git a/base/arena.h b/base/arena.h new file mode 100644 index 000000000000..c22ffa6a1705 --- /dev/null +++ b/base/arena.h @@ -0,0 +1,121 @@ +// Copyright 2019 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#ifndef THIRD_PARTY_MLIR_EDGE_IREE_BASE_ARENA_H_ +#define THIRD_PARTY_MLIR_EDGE_IREE_BASE_ARENA_H_ + +#include +#include + +#include "third_party/absl/types/span.h" + +namespace iree { + +// Arena allocator. +// Allocates memory from a cached block list grown at specified intervals. +// Individual allocations cannot be freed. +// Default constructors will be called when allocating but no destructors will +// ever be called. +// +// This should be used in places where extreme dynamic memory growth is required +// to ensure that the allocations stay close to each other in memory, are easy +// to account for, and can be released together. For example, proto or file +// parsing, per-batch write-once/read-once data buffers, etc. +// +// Usage: +// Arena arena; +// auto t0 = arena.Allocate(); +class Arena { + public: + static constexpr size_t kDefaultBlockSize = 32 * 1024; + static constexpr size_t kBlockOverhead = sizeof(void*) + sizeof(size_t); + + Arena() : Arena(kDefaultBlockSize) {} + explicit Arena(size_t block_size); + ~Arena(); + + // Clears all data in the arena and deallocates blocks. + // Use Reset to avoid reallocation. + void Clear(); + + // Resets data in the arena but does not deallocate blocks. + // Use Clear to reclaim memory. + void Reset(); + + // Block size, excluding the block header. + // This is the largest size of any allocation that can be made of the arena. + size_t block_size() const { return block_size_; } + + // Total number of bytes that have been allocated, excluding wasted space. + size_t bytes_allocated() const { return bytes_allocated_; } + // Total number of bytes as blocks allocated, including wasted space. + // If this number is much higher than bytes_allocated the block size requires + // tuning. + size_t block_bytes_allocated() const { return block_bytes_allocated_; } + + // Allocates an instance of the given type and calls its constructor. + template + T* Allocate() { + void* storage = AllocateBytes(sizeof(T)); + return new (storage) T(); + } + + // Allocates an instance of the given type and calls its constructor with + // arguments. + template + T* Allocate(Args&&... args) { + void* storage = AllocateBytes(sizeof(T)); + return new (storage) T(std::forward(args)...); + } + + // Allocates an array of items and returns a span pointing to them. + template + absl::Span AllocateSpan(size_t count) { + void* storage = AllocateBytes(count * sizeof(T)); + return absl::MakeSpan(reinterpret_cast(storage), count); + } + + // Allocates a block of raw bytes from the arena. + // Zero-byte allocations will return nullptr. + uint8_t* AllocateBytes(size_t length); + + private: + // Block size contains the BlockHeader, so a 1024b block size will result in + // 1024-sizeof(BlockHeader) usable bytes. + size_t block_size_ = kDefaultBlockSize; + size_t bytes_allocated_ = 0; + size_t block_bytes_allocated_ = 0; + + // Each block in the arena contains a prefixed header that lets us link the + // blocks together (to make freeing easier) as well as tracking current byte + // count to let us fill gaps. + // Immediately following the header is the actual arena data, up until the + // block size is reached. + struct BlockHeader { + BlockHeader* next_block; + size_t bytes_allocated; + }; + static_assert(sizeof(BlockHeader) == kBlockOverhead, "Block header mismatch"); + + // Singly-linked list of allocated blocks in reverse allocation order (so + // the most recently allocated block is first). + BlockHeader* block_list_head_ = nullptr; + + // Allocated but unused blocks. + BlockHeader* unused_block_list_head_ = nullptr; +}; + +} // namespace iree + +#endif // THIRD_PARTY_MLIR_EDGE_IREE_BASE_ARENA_H_ diff --git a/base/bitfield.h b/base/bitfield.h new file mode 100644 index 000000000000..c22f899adce3 --- /dev/null +++ b/base/bitfield.h @@ -0,0 +1,140 @@ +// Copyright 2019 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// Utility to enable bit operators on enum classes treated as bitfields. +// +// To use define an enum class with valid bitmask values and an underlying type +// then use the macro to enable support: +// enum class MyBitfield : uint32_t { +// kFoo = 1 << 0, +// kBar = 1 << 1, +// }; +// IREE_BITFIELD(MyBitfield); +// MyBitfield value = ~(MyBitfield::kFoo | MyBitfield::kBar); +// +// AnyBitSet is provided as a way to quickly test if any of the given bits are +// set: +// if (AnyBitSet(value)) { /* one or more bits are set */ } +// +// If testing for equality it's recommended that AllBitsSet is used to ensure +// that combined values are handled properly: +// if (AllBitsSet(value, MyBitfield::kSomeSetOfFlags)) { /* all bits set */ } + +#ifndef THIRD_PARTY_MLIR_EDGE_IREE_BASE_BITFIELD_H_ +#define THIRD_PARTY_MLIR_EDGE_IREE_BASE_BITFIELD_H_ + +#include +#include +#include +#include + +#include "third_party/absl/types/span.h" + +namespace iree { + +#define IREE_BITFIELD(enum_class) \ + inline enum_class operator|(enum_class lhs, enum_class rhs) { \ + using enum_type = typename std::underlying_type::type; \ + return static_cast(static_cast(lhs) | \ + static_cast(rhs)); \ + } \ + inline enum_class& operator|=(enum_class& lhs, enum_class rhs) { \ + using enum_type = typename std::underlying_type::type; \ + lhs = static_cast(static_cast(lhs) | \ + static_cast(rhs)); \ + return lhs; \ + } \ + inline enum_class operator&(enum_class lhs, enum_class rhs) { \ + using enum_type = typename std::underlying_type::type; \ + return static_cast(static_cast(lhs) & \ + static_cast(rhs)); \ + } \ + inline enum_class& operator&=(enum_class& lhs, enum_class rhs) { \ + using enum_type = typename std::underlying_type::type; \ + lhs = static_cast(static_cast(lhs) & \ + static_cast(rhs)); \ + return lhs; \ + } \ + inline enum_class operator^(enum_class lhs, enum_class rhs) { \ + using enum_type = typename std::underlying_type::type; \ + return static_cast(static_cast(lhs) ^ \ + static_cast(rhs)); \ + } \ + inline enum_class& operator^=(enum_class& lhs, enum_class rhs) { \ + using enum_type = typename std::underlying_type::type; \ + lhs = static_cast(static_cast(lhs) ^ \ + static_cast(rhs)); \ + return lhs; \ + } \ + inline enum_class operator~(enum_class lhs) { \ + using enum_type = typename std::underlying_type::type; \ + return static_cast(~static_cast(lhs)); \ + } \ + inline bool AnyBitSet(enum_class lhs) { \ + using enum_type = typename std::underlying_type::type; \ + return static_cast(lhs) != 0; \ + } \ + inline bool AllBitsSet(enum_class lhs, enum_class rhs) { \ + return (lhs & rhs) == rhs; \ + } + +// Appends the formatted contents of the given bitfield value to a stream. +// +// Processes values in the order of the mapping table provided and will only +// use each bit once. Use this to prioritize combined flags over split ones. +template +void FormatBitfieldValue( + std::ostringstream* stream, T value, + const absl::Span> mappings) { + T remaining_bits = value; + int i = 0; + for (const auto& mapping : mappings) { + if ((remaining_bits & mapping.first) == mapping.first) { + if (i > 0) { + *stream << "|"; + } + *stream << mapping.second; + remaining_bits &= ~mapping.first; + ++i; + } + } + using enum_type = typename std::underlying_type::type; + if (remaining_bits != static_cast(0)) { + if (i > 0) { + *stream << "|"; + } + *stream << std::hex << static_cast(remaining_bits) << "h"; + } +} + +// Returns a string with the formatted contents of the given bitfield value. +// +// Usage: +// MyValue my_value = MyValue::kA | MyValue::kB; +// std::string string_value = FormatBitfieldValue(my_value, { +// {MyValue::kA, "kA"}, +// {MyValue::kB, "kB"}, +// }); +// // string_value contains 'kA|kB' +template +std::string FormatBitfieldValue( + T value, absl::Span> mappings) { + std::ostringstream stream; + FormatBitfieldValue(&stream, value, mappings); + return stream.str(); +} + +} // namespace iree + +#endif // THIRD_PARTY_MLIR_EDGE_IREE_BASE_BITFIELD_H_ diff --git a/base/file_io.h b/base/file_io.h new file mode 100644 index 000000000000..a56c02d22c8f --- /dev/null +++ b/base/file_io.h @@ -0,0 +1,67 @@ +// Copyright 2019 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#ifndef THIRD_PARTY_MLIR_EDGE_IREE_BASE_FILE_IO_H_ +#define THIRD_PARTY_MLIR_EDGE_IREE_BASE_FILE_IO_H_ + +#include +#include +#include + +#include "third_party/absl/strings/string_view.h" +#include "third_party/mlir_edge/iree/base/status.h" + +namespace iree { +namespace file_io { + +// Deletes the file at the provided path. +Status DeleteFile(absl::string_view path); + +// Moves a file from 'source_path' to 'destination_path'. +// +// This may simply rename the file, but may fall back to a full copy and delete +// of the original if renaming is not possible (for example when moving between +// physical storage locations). +Status MoveFile(absl::string_view source_path, + absl::string_view destination_path); + +// Checks if a file exists at the provided path. +// +// Returns an OK status if the file definitely exists. +// Errors can include PermissionDeniedError, NotFoundError, etc. +Status FileExists(absl::string_view path); + +// Joins two paths together. +// +// For example: +// JoinFilePaths('foo', 'bar') --> 'foo/bar' +// JoinFilePaths('/foo/', '/bar') --> '/foo/bar' +std::string JoinFilePaths(absl::string_view path1, absl::string_view path2); + +// Gets the directory name component of a file path. +absl::string_view FileDirectoryName(absl::string_view path); + +// Returns the part of the path after the final "/". +absl::string_view FileBasename(absl::string_view path); + +// Returns the part of the basename of path prior to the final ".". +absl::string_view FileStem(absl::string_view path); + +// Synchronously reads a file's contents into a string. +StatusOr GetFileContents(absl::string_view path); + +} // namespace file_io +} // namespace iree + +#endif // THIRD_PARTY_MLIR_EDGE_IREE_BASE_FILE_IO_H_ diff --git a/base/flatbuffer_util.cc b/base/flatbuffer_util.cc new file mode 100644 index 000000000000..adede876c4bd --- /dev/null +++ b/base/flatbuffer_util.cc @@ -0,0 +1,250 @@ +// Copyright 2019 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "third_party/mlir_edge/iree/base/flatbuffer_util.h" + +#include +#include + +#include "third_party/absl/memory/memory.h" +#include "third_party/absl/types/source_location.h" +#include "third_party/mlir_edge/iree/base/memory.h" +#include "third_party/mlir_edge/iree/base/status.h" +#include "third_party/mlir_edge/iree/base/tracing.h" + +// Used for mmap: +#include +#include +#include +#include +#include + +namespace iree { + +FlatBufferFileBase::~FlatBufferFileBase() { + if (deleter_) { + deleter_(); + deleter_ = []() {}; + } +} + +Status FlatBufferFileBase::Create(const void* root_ptr, + std::function deleter) { + IREE_TRACE_SCOPE0("FlatBufferFileBase::Create"); + + root_ptr_ = root_ptr; + deleter_ = std::move(deleter); + + return OkStatus(); +} + +Status FlatBufferFileBase::CreateWithBackingBuffer( + const void* root_ptr, ::flatbuffers::DetachedBuffer backing_buffer) { + IREE_TRACE_SCOPE0("FlatBufferFileBase::Create"); + + root_ptr_ = root_ptr; + + // Pass along the buffer provided so we keep it alive until the + // FlatBufferFileBase is destructed. + auto backing_buffer_baton = MoveToLambda(backing_buffer); + deleter_ = [backing_buffer_baton]() { (void)backing_buffer_baton.value; }; + + return OkStatus(); +} + +Status FlatBufferFileBase::Wrap(const void* root_ptr) { + IREE_TRACE_SCOPE0("FlatBufferFileBase::Wrap"); + return Create(root_ptr, []() {}); +} + +Status FlatBufferFileBase::FromBuffer(Identifier identifier, + absl::Span buffer_data, + std::function deleter, + size_t root_type_size, + VerifierFn verifier_fn) { + IREE_TRACE_SCOPE("FlatBufferFileBase::FromBuffer:size", int) + (static_cast(buffer_data.size())); + + // Sanity check buffer for the minimum size as FlatBuffers doesn't. + if (buffer_data.size() < 16) { + return InvalidArgumentErrorBuilder(ABSL_LOC) + << "Provided serialized flatbuffer buffer is too small to be legit " + "at size=" + << buffer_data.size(); + } + + // Ensure the buffer has the BIPE magic bytes. + if (identifier.has_value() && !::flatbuffers::BufferHasIdentifier( + buffer_data.data(), identifier.value())) { + return InvalidArgumentErrorBuilder(ABSL_LOC) + << "Provided serialized buffer does not contain the expected type; " + "magic bytes mismatch (expected " + << identifier.value() << ")"; + } + + // Verify the FlatBuffer contains valid offsets and won't try to read out of + // bounds of the buffer. We inline a bit of VerifyBufferFromStart so this code + // can stay generic. + { + IREE_TRACE_SCOPE0("FlatBufferFileBase::FromBufferVerification"); + ::flatbuffers::Verifier verifier{buffer_data.data(), buffer_data.size()}; + if (!verifier_fn(identifier.value_or(nullptr), &verifier)) { + return InvalidArgumentErrorBuilder(ABSL_LOC) + << "FlatBuffer failed to verify as expected type; possibly " + "corrupt input"; + } + } + + // Resolve the root pointer in the buffer. + // This is GetMutableRoot such that we don't need to know T. + root_ptr_ = buffer_data.data() + + ::flatbuffers::EndianScalar( + *reinterpret_cast( + buffer_data.data())); + if (!root_ptr_) { + return FailedPreconditionErrorBuilder(ABSL_LOC) + << "Unable to resolve root table"; + } + deleter_ = std::move(deleter); + + return OkStatus(); +} + +Status FlatBufferFileBase::WrapBuffer(Identifier identifier, + absl::Span buffer_data, + size_t root_type_size, + VerifierFn verifier_fn) { + IREE_TRACE_SCOPE0("FlatBufferFileBase::WrapBuffer"); + return FromBuffer( + identifier, buffer_data, []() {}, root_type_size, verifier_fn); +} + +Status FlatBufferFileBase::FromString(Identifier identifier, + std::string buffer_data, + size_t root_type_size, + VerifierFn verifier_fn) { + IREE_TRACE_SCOPE0("FlatBufferFileBase::FromString"); + + // Reference right into the string buffer. + auto buffer_data_data = absl::MakeConstSpan( + reinterpret_cast(buffer_data.data()), buffer_data.size()); + + // Use a baton to keep the string alive until the FlatBufferFileBase is + // destroyed. + auto buffer_data_baton = MoveToLambda(buffer_data); + return FromBuffer( + identifier, buffer_data_data, + [buffer_data_baton]() { + // Keeping the string alive. + (void)buffer_data_baton.value; + }, + root_type_size, verifier_fn); +} + +namespace { + +class FileDescriptor { + public: + static StatusOr > OpenRead(std::string path) { + struct stat buf; + if (::lstat(path.c_str(), &buf) == -1) { + return NotFoundErrorBuilder(ABSL_LOC) + << "Unable to stat file " << path << ": " << ::strerror(errno); + } + + int fd = ::open(path.c_str(), O_RDONLY); + if (fd == -1) { + return UnavailableErrorBuilder(ABSL_LOC) + << "Unable to open file " << path << ": " << ::strerror(errno); + } + return absl::WrapUnique( + new FileDescriptor(fd, static_cast(buf.st_size))); + } + + ~FileDescriptor() { ::close(fd_); } + + int fd() const { return fd_; } + size_t size() const { return size_; } + + private: + FileDescriptor(int fd, size_t size) : fd_(fd), size_(size) {} + FileDescriptor(const FileDescriptor&) = delete; + FileDescriptor& operator=(const FileDescriptor&) = delete; + + int fd_; + size_t size_; +}; + +class MappedFile { + public: + static StatusOr > Open(absl::string_view path) { + // Open the file for reading. Note that we only need to keep it open long + // enough to map it and we can close the descriptor after that. + ASSIGN_OR_RETURN(auto file, FileDescriptor::OpenRead(std::string(path))); + + // Map the file from the file descriptor. + void* data = + ::mmap(nullptr, file->size(), PROT_READ, MAP_SHARED, file->fd(), 0); + if (data == MAP_FAILED) { + return UnavailableErrorBuilder(ABSL_LOC) + << "Mapping failed on file (ensure uncompressed): " << path; + } + + return absl::WrapUnique(new MappedFile(data, file->size())); + } + + ~MappedFile() { + if (::munmap(const_cast(data_), data_size_) != 0) { + LOG(WARNING) << "Unable to unmap file: " << strerror(errno); + } + } + + const void* data() const { return data_; } + size_t data_size() const { return data_size_; } + + private: + MappedFile(const void* data, size_t data_size) + : data_(data), data_size_(data_size) {} + MappedFile(const MappedFile&) = delete; + MappedFile& operator=(const MappedFile&) = delete; + + const void* data_; + size_t data_size_; +}; + +} // namespace + +Status FlatBufferFileBase::LoadFile(Identifier identifier, + absl::string_view path, + size_t root_type_size, + VerifierFn verifier_fn) { + IREE_TRACE_SCOPE0("FlatBufferFileBase::LoadFile"); + + ASSIGN_OR_RETURN(auto mapped_file, MappedFile::Open(path)); + + absl::Span buffer_data{ + reinterpret_cast(mapped_file->data()), + mapped_file->data_size()}; + + auto handle_baton = MoveToLambda(mapped_file); + return FromBuffer( + identifier, buffer_data, + [handle_baton]() { + // Keeping the mmap handle alive. + (void)handle_baton.value; + }, + root_type_size, verifier_fn); +} + +} // namespace iree diff --git a/base/flatbuffer_util.h b/base/flatbuffer_util.h new file mode 100644 index 000000000000..148d4569d4b0 --- /dev/null +++ b/base/flatbuffer_util.h @@ -0,0 +1,280 @@ +// Copyright 2019 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#ifndef THIRD_PARTY_MLIR_EDGE_IREE_BASE_FLATBUFFER_UTIL_H_ +#define THIRD_PARTY_MLIR_EDGE_IREE_BASE_FLATBUFFER_UTIL_H_ + +#include +#include +#include +#include +#include +#include +#include + +#include "third_party/absl/strings/string_view.h" +#include "third_party/absl/types/optional.h" +#include "third_party/absl/types/span.h" +#include "third_party/flatbuffers/include/flatbuffers/flatbuffers.h" +#include "third_party/mlir_edge/iree/base/status.h" + +namespace iree { + +// Wraps a FlatBuffer String in an absl::string_view. +// Returns empty-string ("") for nullptr values. +inline absl::string_view WrapString(const ::flatbuffers::String* value) { + return value ? absl::string_view{value->data(), value->size()} : ""; +} + +// Base type for FlatBufferFile. See below. +class FlatBufferFileBase { + public: + + using Identifier = absl::optional; + + virtual ~FlatBufferFileBase(); + + protected: + template + friend class FlatBufferFile; + + using VerifierFn = bool (*)(const char* identifier, + ::flatbuffers::Verifier* verifier); + + FlatBufferFileBase() = default; + + const void* root_ptr() const { return root_ptr_; } + + // Redirections of template static methods on FlatBufferFile so we can put the + // implementations in a shared compilation unit. + // See FlatBufferFile for doc comments. + Status Create(const void* root_ptr, std::function deleter); + Status CreateWithBackingBuffer(const void* root_ptr, + ::flatbuffers::DetachedBuffer backing_buffer); + Status Wrap(const void* root); + Status FromBuffer(Identifier identifier, + absl::Span buffer_data, + std::function deleter, size_t root_type_size, + VerifierFn verifier_fn); + Status WrapBuffer(Identifier identifier, + absl::Span buffer_data, + size_t root_type_size, VerifierFn verifier_fn); + Status FromString(Identifier identifier, std::string buffer_data, + size_t root_type_size, VerifierFn verifier_fn); + Status LoadFile(Identifier identifier, absl::string_view path, + size_t root_type_size, VerifierFn verifier_fn); + + private: + const void* root_ptr_ = nullptr; + std::function deleter_; +}; + +// Immutable root FlatBuffer type wrapper with support for loading and backing +// buffer management. +// +// Immutable and thread-safe. +template +class FlatBufferFile final : public FlatBufferFileBase { + public: + // Creates a FlatBufferFile from an in-memory root pointer. + // The provided |deleter| will be called when the FlatBufferFile is destructed + // and can be used to deallocate/clean up resources. + // + // This assumes that the root pointer has already been verified as valid. + // If verification is required instead use FromBuffer on the original buffer. + static StatusOr>> Create( + const T* root, std::function deleter); + + // Creates a FlatBufferFile from an in-memory root pointer and the detached + // backing buffer storing it. + // + // Example: + // FlatBufferBuilder fbb; + // MyTypeBuilder mtb(fbb); + // fbb.Finish(mtb.Finish()); + // auto my_type = FlatBufferFile::CreateWithBackingBuffer( + // fbb.Release()); + // my_type->foo(); + static StatusOr>> CreateWithBackingBuffer( + ::flatbuffers::DetachedBuffer backing_buffer); + + // Wraps a caller-owned in-memory root pointer. + // The provided |root| must remain valid for the lifetime of the returned + // FlatBufferFile. + // + // This assumes that the root pointer has already been verified as valid. + // If verification is required instead use FromBuffer on the original buffer. + static StatusOr>> Wrap(const T* root); + + // Creates a FlatBufferFile wrapping an external data buffer with a deleter + // function that will be called when the FlatBufferFile is destructed. + static StatusOr>> FromBuffer( + Identifier identifier, absl::Span buffer_data, + std::function deleter); + + // Creates a FlatBufferFile from a serialized data buffer. + // The FlatBufferFile takes ownership of the vector. + static StatusOr>> FromBuffer( + Identifier identifier, std::vector buffer_data); + + // Loads a FlatBufferFile from an external buffer owned by the caller. + // The buffer must remain valid until the Pipeline is destroyed. + static StatusOr>> WrapBuffer( + Identifier identifier, absl::Span buffer_data); + + // Loads a FlatBufferFile from a serialized string. + // The FlatBufferFile takes ownership of the string. + static StatusOr>> FromString( + Identifier identifier, std::string buffer_data); + + // Loads a FlatBufferFile from a serialized file on the file system. + // This will attempt to mmap the file and is the preferred way of loading as + // only those pages that contain requested tables will be read. + static StatusOr>> LoadFile( + Identifier identifier, absl::string_view path); + + // Returns a vector of file references that share the same underlying data + // buffer. The buffer will be kept alive until the last file is released. + static StatusOr>>> + CreateShareGroup(std::unique_ptr> file, int count); + + ~FlatBufferFile() override = default; + + // Typed root pointer of the file. + const T* root() const { return reinterpret_cast(root_ptr()); } + + private: + FlatBufferFile() = default; + + // Conforms to VerifierFn. + static bool VerifierFnT(const char* identifier, + ::flatbuffers::Verifier* verifier) { + return verifier->VerifyBuffer(identifier); + } +}; + +// static +template +StatusOr>> FlatBufferFile::Create( + const T* root, std::function deleter) { + std::unique_ptr> flat_buffer_file{new FlatBufferFile}; + auto* base_file = static_cast(flat_buffer_file.get()); + RETURN_IF_ERROR(base_file->Create(root, std::move(deleter))); + return std::move(flat_buffer_file); +} + +// static +template +StatusOr>> +FlatBufferFile::CreateWithBackingBuffer( + ::flatbuffers::DetachedBuffer backing_buffer) { + std::unique_ptr> flat_buffer_file{new FlatBufferFile}; + auto* base_file = static_cast(flat_buffer_file.get()); + auto* root_ptr = ::flatbuffers::GetRoot(backing_buffer.data()); + RETURN_IF_ERROR( + base_file->CreateWithBackingBuffer(root_ptr, std::move(backing_buffer))); + return std::move(flat_buffer_file); +} + +// static +template +StatusOr>> FlatBufferFile::Wrap( + const T* root) { + std::unique_ptr> flat_buffer_file{new FlatBufferFile}; + auto* base_file = static_cast(flat_buffer_file.get()); + RETURN_IF_ERROR(base_file->Wrap(root)); + return std::move(flat_buffer_file); +} + +// static +template +StatusOr>> FlatBufferFile::FromBuffer( + Identifier identifier, absl::Span buffer_data, + std::function deleter) { + std::unique_ptr> flat_buffer_file{new FlatBufferFile}; + auto* base_file = static_cast(flat_buffer_file.get()); + RETURN_IF_ERROR(base_file->FromBuffer( + identifier, buffer_data, std::move(deleter), sizeof(T), VerifierFnT)); + return std::move(flat_buffer_file); +} + +// static +template +StatusOr>> FlatBufferFile::FromBuffer( + Identifier identifier, std::vector buffer_data) { + auto* buffer_data_ptr = new decltype(buffer_data); + (*buffer_data_ptr) = std::move(buffer_data); + return FromBuffer(identifier, absl::MakeConstSpan(*buffer_data_ptr), + [buffer_data_ptr]() { delete buffer_data_ptr; }); +} + +// static +template +StatusOr>> FlatBufferFile::WrapBuffer( + Identifier identifier, absl::Span buffer_data) { + std::unique_ptr> flat_buffer_file{new FlatBufferFile}; + auto* base_file = static_cast(flat_buffer_file.get()); + RETURN_IF_ERROR( + base_file->WrapBuffer(identifier, buffer_data, sizeof(T), VerifierFnT)); + return std::move(flat_buffer_file); +} + +// static +template +StatusOr>> FlatBufferFile::FromString( + Identifier identifier, std::string buffer_data) { + std::unique_ptr> flat_buffer_file{new FlatBufferFile}; + auto* base_file = static_cast(flat_buffer_file.get()); + RETURN_IF_ERROR(base_file->FromString(identifier, std::move(buffer_data), + sizeof(T), VerifierFnT)); + return std::move(flat_buffer_file); +} + +// static +template +StatusOr>> FlatBufferFile::LoadFile( + Identifier identifier, absl::string_view path) { + std::unique_ptr> flat_buffer_file{new FlatBufferFile}; + auto* base_file = static_cast(flat_buffer_file.get()); + RETURN_IF_ERROR( + base_file->LoadFile(identifier, path, sizeof(T), VerifierFnT)); + return std::move(flat_buffer_file); +} + +// static +template +StatusOr>>> +FlatBufferFile::CreateShareGroup(std::unique_ptr> file, + int count) { + // Create a shared_ptr wrapper for the base file that will be. + std::shared_ptr> shared_file{file.release()}; + + // Create N files. We wrap and keep the shared_ptr alive in the deleter + // capture. By wrapping we avoid reverifying the entire buffer. + std::vector>> list; + for (int i = 0; i < count; ++i) { + ASSIGN_OR_RETURN(auto new_file, FlatBufferFile::Create( + shared_file->root(), [shared_file]() { + // Each new file keeps a reference to + // the shared file to keep it alive. + (void)shared_file; + })); + list.push_back(std::move(new_file)); + } + return std::move(list); +} + +} // namespace iree + +#endif // THIRD_PARTY_MLIR_EDGE_IREE_BASE_FLATBUFFER_UTIL_H_ diff --git a/base/init.h b/base/init.h new file mode 100644 index 000000000000..76f41d738523 --- /dev/null +++ b/base/init.h @@ -0,0 +1,52 @@ +// Copyright 2019 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#ifndef THIRD_PARTY_MLIR_EDGE_IREE_BASE_INIT_H_ +#define THIRD_PARTY_MLIR_EDGE_IREE_BASE_INIT_H_ + +// Initializer macros are defined in separate files: +// IREE_DECLARE_MODULE_INITIALIZER(name) +// IREE_REGISTER_MODULE_INITIALIZER(name, body) +// IREE_REGISTER_MODULE_INITIALIZER_SEQUENCE(name1, name2) +// IREE_REQUIRE_MODULE_INITIALIZED(name) +// IREE_RUN_MODULE_INITIALIZERS() +// IREE_REQUIRE_MODULE_LINKED(name) +// +// These macros allow for arranging pieces of initialization code to be +// executed at a well-defined time and in a well-defined order. +// +// Initialization happens automatically during InitializeEnvironment(), which +// should be called early in main(), before other code runs. + +#ifdef IREE_CONFIG_GOOGLE_INTERNAL +#include "third_party/mlir_edge/iree/base/google/init_google.h" +#else +#include "third_party/mlir_edge/iree/base/internal/init_internal.h" +#endif // IREE_CONFIG_GOOGLE_INTERNAL + +namespace iree { + +// Initializes the system environment in a binary. +// +// This first parses command line flags, then resolves module initializers +// by calling IREE_RUN_MODULE_INITIALIZERS(). +// +// 'argc' and 'argv' are the command line flags to parse. +// +// This should typically be called early in main(), before other code runs. +void InitializeEnvironment(int* argc, char*** argv); + +} // namespace iree + +#endif // THIRD_PARTY_MLIR_EDGE_IREE_BASE_INIT_H_ diff --git a/base/internal/init_internal.cc b/base/internal/init_internal.cc new file mode 100644 index 000000000000..160d09a91e7b --- /dev/null +++ b/base/internal/init_internal.cc @@ -0,0 +1,100 @@ +// Copyright 2019 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "third_party/mlir_edge/iree/base/internal/init_internal.h" + +#include + +#include "third_party/absl/flags/parse.h" + +namespace iree { + +static Initializer::NameMap* static_name_map = nullptr; + +struct Initializer::InitializerData { + Initializer* initializer_obj; + std::set dependency_names; + + InitializerData() : initializer_obj(nullptr) {} + explicit InitializerData(Initializer* i) : initializer_obj(i) {} +}; + +Initializer::DependencyRegisterer::DependencyRegisterer( + const char* name, Initializer* initializer, const Dependency& dependency) { + NameMap* name_map = InitializerNameMap(); + + // Insert 'dependency' into the 'dependency_names' set for 'initializer'. + InitializerData* initializer_data = &(*name_map)[name]; + initializer_data->dependency_names.insert(dependency.name); + + // Ensure that 'dependency' exists in the map. + InitializerData* dependency_data = &(*name_map)[dependency.name]; + dependency_data->initializer_obj = dependency.initializer; +} + +Initializer::Initializer(const char* name, InitializerFunc function) + : name_(name), function_(function), done_(false) { + // Register this Initializer instance (wrapped by an InitializerData) within + // the static name map. + NameMap* name_map = InitializerNameMap(); + InitializerData* initializer_data = &(*name_map)[name]; + initializer_data->initializer_obj = this; +} + +void Initializer::RunInitializers() { + // Run each registered Initializer, in lexicographic order of their names. + // Initializer dependencies will be run first as needed. + NameMap* name_map = InitializerNameMap(); + for (auto& p : *name_map) { + RunInitializer(&p.second); + } +} + +void Initializer::Require() { + NameMap* name_map = InitializerNameMap(); + InitializerData* initializer_data = &(name_map->find(name_)->second); + RunInitializer(initializer_data); +} + +Initializer::NameMap* Initializer::InitializerNameMap() { + if (static_name_map == nullptr) { + static_name_map = new Initializer::NameMap; + } + return static_name_map; +} + +void Initializer::RunInitializer(InitializerData* initializer_data) { + if (initializer_data->initializer_obj->done_) { + return; + } + + // Run Initializer dependencies first. + NameMap* name_map = InitializerNameMap(); + for (const auto& dependency_name : initializer_data->dependency_names) { + auto dep_init = name_map->find(dependency_name); + RunInitializer(&dep_init->second); + } + + // Finally run the Initializer itself. + initializer_data->initializer_obj->function_(); + initializer_data->initializer_obj->done_ = true; +} + +void InitializeEnvironment(int* argc, char*** argv) { + absl::ParseCommandLine(*argc, *argv); + + IREE_RUN_MODULE_INITIALIZERS(); +} + +} // namespace iree diff --git a/base/internal/init_internal.h b/base/internal/init_internal.h new file mode 100644 index 000000000000..109e3575b35d --- /dev/null +++ b/base/internal/init_internal.h @@ -0,0 +1,99 @@ +// Copyright 2019 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#ifndef THIRD_PARTY_MLIR_EDGE_IREE_BASE_INTERNAL_INIT_INTERNAL_H_ +#define THIRD_PARTY_MLIR_EDGE_IREE_BASE_INTERNAL_INIT_INTERNAL_H_ + +#include +#include + +namespace iree { + +// A static instance of this class is declared for each piece of initialization +// code using the initializer macros. +class Initializer { + public: + typedef void (*InitializerFunc)(); + + Initializer(const char* name, InitializerFunc function); + + // Runs all registered initializers that have not yet run. + // The initializers are invoked in lexicographically increasing order by name, + // except as necessary to satisfy dependencies. + // + // This is normally called by InitializeEnvironment(), so application code + // typically should not call it directly. + static void RunInitializers(); + + // Runs this initializer if it has not yet run, including any dependencies. + void Require(); + + struct Dependency { + Dependency(const char* n, Initializer* i) : name(n), initializer(i) {} + const char* const name; + Initializer* const initializer; + }; + + // A static instance of this class is declared for each piece of + // initializer ordering definition. + struct DependencyRegisterer { + DependencyRegisterer(const char* name, Initializer* initializer, + const Dependency& dependency); + }; + + struct InitializerData; + typedef std::map NameMap; + + private: + static NameMap* InitializerNameMap(); + static void RunInitializer(InitializerData* initializer_data); + + const std::string name_; + InitializerFunc function_; + bool done_; +}; + +} // namespace iree + +#define IREE_DECLARE_MODULE_INITIALIZER(name) \ + extern ::iree::Initializer iree_initializer_##name + +#define IREE_REGISTER_MODULE_INITIALIZER(name, body) \ + static void iree_init_##name() { body; } \ + ::iree::Initializer iree_initializer_##name(#name, iree_init_##name) + +#define IREE_REGISTER_MODULE_INITIALIZER_SEQUENCE(name1, name2) \ + namespace { \ + static ::iree::Initializer::DependencyRegisterer \ + iree_initializer_dependency_##name1##_##name2( \ + #name2, &iree_initializer_##name2, \ + ::iree::Initializer::Dependency(#name1, &iree_initializer_##name1)); \ + } + +#define IREE_REQUIRE_MODULE_INITIALIZED(name) \ + do { \ + IREE_DECLARE_MODULE_INITIALIZER(name); \ + iree_initializer_##name.Require(); \ + } while (0) + +#define IREE_RUN_MODULE_INITIALIZERS() \ + do { \ + ::iree::Initializer::RunInitializers(); \ + } while (0) + +#define IREE_REQUIRE_MODULE_LINKED(name) \ + __attribute__((used)) static ::iree::Initializer* iree_module_ref_##name = \ + &iree_initializer_##name + +#endif // THIRD_PARTY_MLIR_EDGE_IREE_BASE_INTERNAL_INIT_INTERNAL_H_ diff --git a/base/internal/logging.cc b/base/internal/logging.cc new file mode 100644 index 000000000000..fde231c69f64 --- /dev/null +++ b/base/internal/logging.cc @@ -0,0 +1,106 @@ +// Copyright 2019 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "third_party/mlir_edge/iree/base/internal/logging.h" + +#include + +#include "third_party/absl/flags/flag.h" + +ABSL_FLAG(int, iree_minloglevel, 0, + "Minimum logging level. 0 = INFO and above."); +ABSL_FLAG(int, iree_v, 0, + "Verbosity level maximum. 1 = VLOG(0-1), 2 = VLOG(0-2)."); +ABSL_FLAG(bool, iree_logtostderr, false, "Logs to stderr instead of stdout"); + +namespace iree { +namespace internal { + +namespace { + +// Parse log level (int64_t) from environment variable (char*). +// Returns true if the value was present and parsed successfully. +bool LogLevelStrToInt(const char* iree_env_var_val, int64_t* out_level) { + *out_level = 0; + if (iree_env_var_val == nullptr) { + return false; + } + + std::string min_log_level(iree_env_var_val); + std::istringstream ss(min_log_level); + int64_t level; + if (!(ss >> level)) { + // Invalid vlog level setting, set level to default (0). + return false; + } + + *out_level = level; + return true; +} + +int64_t MinLogLevelFromEnv() { + const char* iree_env_var_val = getenv("IREE_MIN_LOG_LEVEL"); + int64_t level = 0; + if (LogLevelStrToInt(iree_env_var_val, &level)) { + return level; + } + return absl::GetFlag(FLAGS_iree_minloglevel); +} + +int64_t MinVLogLevelFromEnv() { + const char* iree_env_var_val = getenv("IREE_MIN_VLOG_LEVEL"); + int64_t level = 0; + if (LogLevelStrToInt(iree_env_var_val, &level)) { + return level; + } + return absl::GetFlag(FLAGS_iree_v); +} + +} // namespace + +LogMessage::LogMessage(const char* file_name, int line, int severity) + : file_name_(file_name), line_(line), severity_(severity) {} + +LogMessage::~LogMessage() { + // Read the min log level once during the first call to logging. + static int64_t min_log_level = MinLogLevelFromEnv(); + if (ABSL_PREDICT_TRUE(severity_ >= min_log_level)) { + EmitLogMessage(); + } +} + +int64_t LogMessage::MinVLogLevel() { + static int64_t min_vlog_level = MinVLogLevelFromEnv(); + return min_vlog_level; +} + +void LogMessage::EmitLogMessage() { + // TODO(scotttodd): Include current system time + fprintf(absl::GetFlag(FLAGS_iree_logtostderr) ? stderr : stdout, + "%c %s:%d] %s\n", "IWEF"[severity_], file_name_, line_, + str().c_str()); +} + +LogMessageFatal::LogMessageFatal(const char* file, int line) + : LogMessage(file, line, FATAL) {} + +LogMessageFatal::~LogMessageFatal() { + EmitLogMessage(); + + // abort() ensures we don't return (as promised via ATTRIBUTE_NORETURN). + abort(); +} + +} // namespace internal +} // namespace iree diff --git a/base/internal/logging.h b/base/internal/logging.h new file mode 100644 index 000000000000..8f37e1be211d --- /dev/null +++ b/base/internal/logging.h @@ -0,0 +1,179 @@ +// Copyright 2019 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#ifndef THIRD_PARTY_MLIR_EDGE_IREE_BASE_INTERNAL_LOGGING_H_ +#define THIRD_PARTY_MLIR_EDGE_IREE_BASE_INTERNAL_LOGGING_H_ + +#include +#include + +#include "third_party/absl/base/attributes.h" +#include "third_party/absl/base/optimization.h" + +namespace iree { + +// ------------------------------------------------------------------------- // +// | LOG | // +// ------------------------------------------------------------------------- // + +// Severity levels for LOG(). +const int INFO = 0; // absl::LogSeverity::kInfo +const int WARNING = 1; // absl::LogSeverity::kWarning +const int ERROR = 2; // absl::LogSeverity::kError +const int FATAL = 3; // absl::LogSeverity::kFatal + +namespace internal { + +class LogMessage : public std::basic_ostringstream { + public: + LogMessage(const char* file_name, int line, int severity); + ~LogMessage(); + + const char* file_name() const { return file_name_; } + int line() const { return line_; } + int severity() const { return severity_; } + + // Returns the minimum log level for VLOG statements. + // E.g., if MinVLogLevel() is 2, then VLOG(2) statements will produce output, + // but VLOG(3) will not. Defaults to 0. + static int64_t MinVLogLevel(); + + protected: + void EmitLogMessage(); + + private: + const char* file_name_; + int line_; + int severity_; +}; + +// LogMessageFatal ensures the process exits in failure after logging a message. +class LogMessageFatal : public LogMessage { + public: + LogMessageFatal(const char* file, int line) ABSL_ATTRIBUTE_COLD; + ABSL_ATTRIBUTE_NORETURN ~LogMessageFatal(); +}; + +// NullStream implements operator<< but does nothing. +class NullStream { + public: + NullStream& stream() { return *this; } +}; +template +inline NullStream& operator<<(NullStream& str, const T&) { + return str; +} +inline NullStream& operator<<(NullStream& str, + std::ostream& (*)(std::ostream& os)) { + return str; +} +inline NullStream& operator<<(NullStream& str, + std::ios_base& (*)(std::ios_base& os)) { + return str; +} + +#define _IREE_LOG_INFO \ + ::iree::internal::LogMessage(__FILE__, __LINE__, ::iree::INFO) +#define _IREE_LOG_WARNING \ + ::iree::internal::LogMessage(__FILE__, __LINE__, ::iree::WARNING) +#define _IREE_LOG_ERROR \ + ::iree::internal::LogMessage(__FILE__, __LINE__, ::iree::ERROR) +#define _IREE_LOG_FATAL ::iree::internal::LogMessageFatal(__FILE__, __LINE__) + +#define LOG(severity) _IREE_LOG_##severity + +#define VLOG_IS_ON(lvl) ((lvl) <= ::iree::internal::LogMessage::MinVLogLevel()) + +#define VLOG(lvl) \ + if (ABSL_PREDICT_FALSE(VLOG_IS_ON(lvl))) \ + ::iree::internal::LogMessage(__FILE__, __LINE__, ::iree::INFO) + +// `DVLOG` behaves like `VLOG` in debug mode (i.e. `#ifndef NDEBUG`). +// Otherwise, it compiles away and does nothing. +#ifndef NDEBUG +#define DVLOG VLOG +#else +#define DVLOG(verbose_level) \ + while (false && (verbose_level) > 0) ::iree::internal::NullStream().stream() +#endif // !NDEBUG + +// ------------------------------------------------------------------------- // +// | CHECK | // +// ------------------------------------------------------------------------- // + +// CHECK dies with a fatal error if condition is not true. It is *not* +// controlled by NDEBUG, so the check will be executed regardless of +// compilation mode. Therefore, it is safe to do things like: +// CHECK(fp->Write(x) == 4) +#define CHECK(condition) \ + if (ABSL_PREDICT_FALSE(!(condition))) \ + LOG(FATAL) << "Check failed: " #condition " " + +// TODO(scotttodd): Log information about the check failure +#define CHECK_OP_LOG(name, op, val1, val2) LOG(FATAL) << "Check failed" + +#define CHECK_OP(name, op, val1, val2) CHECK_OP_LOG(name, op, val1, val2) + +// CHECK_EQ/NE/... +#define CHECK_EQ(val1, val2) CHECK_OP(Check_EQ, ==, val1, val2) +#define CHECK_NE(val1, val2) CHECK_OP(Check_NE, !=, val1, val2) +#define CHECK_LE(val1, val2) CHECK_OP(Check_LE, <=, val1, val2) +#define CHECK_LT(val1, val2) CHECK_OP(Check_LT, <, val1, val2) +#define CHECK_GE(val1, val2) CHECK_OP(Check_GE, >=, val1, val2) +#define CHECK_GT(val1, val2) CHECK_OP(Check_GT, >, val1, val2) + +#ifndef NDEBUG +#define DCHECK(condition) CHECK(condition) +#define DCHECK_EQ(val1, val2) CHECK_EQ(val1, val2) +#define DCHECK_NE(val1, val2) CHECK_NE(val1, val2) +#define DCHECK_LE(val1, val2) CHECK_LE(val1, val2) +#define DCHECK_LT(val1, val2) CHECK_LT(val1, val2) +#define DCHECK_GE(val1, val2) CHECK_GE(val1, val2) +#define DCHECK_GT(val1, val2) CHECK_GT(val1, val2) + +#else + +#define DCHECK(condition) \ + while (false && (condition)) LOG(FATAL) + +// NDEBUG is defined, so DCHECK_EQ(x, y) and so on do nothing. +// However, we still want the compiler to parse x and y, because +// we don't want to lose potentially useful errors and warnings. +// _DCHECK_NOP is a helper, and should not be used outside of this file. +#define _IREE_DCHECK_NOP(x, y) \ + while (false && ((void)(x), (void)(y), 0)) LOG(FATAL) + +#define DCHECK_EQ(x, y) _IREE_DCHECK_NOP(x, y) +#define DCHECK_NE(x, y) _IREE_DCHECK_NOP(x, y) +#define DCHECK_LE(x, y) _IREE_DCHECK_NOP(x, y) +#define DCHECK_LT(x, y) _IREE_DCHECK_NOP(x, y) +#define DCHECK_GE(x, y) _IREE_DCHECK_NOP(x, y) +#define DCHECK_GT(x, y) _IREE_DCHECK_NOP(x, y) + +#endif // !NDEBUG + +// These are for when you don't want a CHECK failure to print a verbose +// stack trace. The implementation of CHECK* in this file already doesn't. +#define QCHECK(condition) CHECK(condition) +#define QCHECK_EQ(x, y) CHECK_EQ(x, y) +#define QCHECK_NE(x, y) CHECK_NE(x, y) +#define QCHECK_LE(x, y) CHECK_LE(x, y) +#define QCHECK_LT(x, y) CHECK_LT(x, y) +#define QCHECK_GE(x, y) CHECK_GE(x, y) +#define QCHECK_GT(x, y) CHECK_GT(x, y) + +} // namespace internal +} // namespace iree + +#endif // THIRD_PARTY_MLIR_EDGE_IREE_BASE_INTERNAL_LOGGING_H_ diff --git a/base/internal/status.cc b/base/internal/status.cc new file mode 100644 index 000000000000..7d962049b717 --- /dev/null +++ b/base/internal/status.cc @@ -0,0 +1,174 @@ +// Copyright 2019 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "third_party/mlir_edge/iree/base/internal/status.h" + +#include +#include + +#include "third_party/absl/base/attributes.h" +#include "third_party/absl/debugging/stacktrace.h" +#include "third_party/absl/memory/memory.h" +#include "third_party/absl/strings/str_cat.h" + +ABSL_FLAG(bool, iree_status_save_stack_trace, false, + "Save and display the full stack trace of the point of error") + .OnUpdate([]() { + iree::StatusSavesStackTrace( + absl::GetFlag(FLAGS_iree_status_save_stack_trace)); + }); + +namespace iree { + +namespace status_internal { + +ABSL_CONST_INIT std::atomic iree_save_stack_trace{false}; + +std::string StatusCodeToString(StatusCode code) { + switch (code) { + case StatusCode::kOk: + return "OK"; + case StatusCode::kCancelled: + return "CANCELLED"; + case StatusCode::kUnknown: + return "UNKNOWN"; + case StatusCode::kInvalidArgument: + return "INVALID_ARGUMENT"; + case StatusCode::kDeadlineExceeded: + return "DEADLINE_EXCEEDED"; + case StatusCode::kNotFound: + return "NOT_FOUND"; + case StatusCode::kAlreadyExists: + return "ALREADY_EXISTS"; + case StatusCode::kPermissionDenied: + return "PERMISSION_DENIED"; + case StatusCode::kUnauthenticated: + return "UNAUTHENTICATED"; + case StatusCode::kResourceExhausted: + return "RESOURCE_EXHAUSTED"; + case StatusCode::kFailedPrecondition: + return "FAILED_PRECONDITION"; + case StatusCode::kAborted: + return "ABORTED"; + case StatusCode::kOutOfRange: + return "OUT_OF_RANGE"; + case StatusCode::kUnimplemented: + return "UNIMPLEMENTED"; + case StatusCode::kInternal: + return "INTERNAL"; + case StatusCode::kUnavailable: + return "UNAVAILABLE"; + case StatusCode::kDataLoss: + return "DATA_LOSS"; + default: + return ""; + } +} + +} // namespace status_internal + +bool DoesStatusSaveStackTrace() { + return status_internal::iree_save_stack_trace.load(std::memory_order_relaxed); +} +void StatusSavesStackTrace(bool on_off) { + status_internal::iree_save_stack_trace.store(on_off, + std::memory_order_relaxed); +} + +Status::Status() {} + +Status::Status(StatusCode code, absl::string_view message) { + state_ = absl::make_unique(); + state_->code = code; + state_->message = string(message); +} + +Status::Status(const Status& x) { + if (x.ok()) return; + + state_ = absl::make_unique(); + state_->code = x.state_->code; + state_->message = x.state_->message; +} + +Status& Status::operator=(const Status& x) { + state_ = absl::make_unique(); + state_->code = x.state_->code; + state_->message = x.state_->message; + return *this; +} + +Status::~Status() {} + +bool Status::ok() const { return state_ == nullptr; } + +StatusCode Status::code() const { + return ok() ? StatusCode::kOk : state_->code; +} + +absl::string_view Status::message() const { + return ok() ? absl::string_view() : absl::string_view(state_->message); +} + +std::string Status::ToString() const { + if (ok()) { + return "OK"; + } + + std::string text; + absl::StrAppend(&text, status_internal::StatusCodeToString(state_->code), + ": ", state_->message); + // TODO(scotttodd): Payloads (stack traces) + return text; +} + +void Status::IgnoreError() const { + // no-op +} + +bool Status::EqualsSlow(const Status& a, const Status& b) { + if (a.code() != b.code()) return false; + if (a.message() != b.message()) return false; + // TODO(scotttodd): Payloads + return true; +} + +bool operator==(const Status& lhs, const Status& rhs) { + return lhs.state_ == rhs.state_ || Status::EqualsSlow(lhs, rhs); +} + +bool operator!=(const Status& lhs, const Status& rhs) { return !(lhs == rhs); } + +std::ostream& operator<<(std::ostream& os, const Status& x) { + os << x.ToString(); + return os; +} + +Status OkStatus() { return Status(); } + +Status Annotate(const Status& s, absl::string_view msg) { + if (s.ok() || msg.empty()) return s; + + absl::string_view new_msg = msg; + std::string annotated; + if (!s.message().empty()) { + absl::StrAppend(&annotated, s.message(), "; ", msg); + new_msg = annotated; + } + Status result(s.code(), new_msg); + // TODO(scotttodd): Copy payload(s) into the new Status + return result; +} + +} // namespace iree diff --git a/base/internal/status.h b/base/internal/status.h new file mode 100644 index 000000000000..d948f2c36df7 --- /dev/null +++ b/base/internal/status.h @@ -0,0 +1,125 @@ +// Copyright 2019 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#ifndef THIRD_PARTY_MLIR_EDGE_IREE_BASE_INTERNAL_STATUS_H_ +#define THIRD_PARTY_MLIR_EDGE_IREE_BASE_INTERNAL_STATUS_H_ + +#include +#include + +#include "third_party/absl/base/attributes.h" +#include "third_party/absl/flags/flag.h" +#include "third_party/absl/strings/string_view.h" +#include "third_party/mlir_edge/iree/base/internal/logging.h" + +ABSL_DECLARE_FLAG(bool, iree_status_save_stack_trace); + +namespace iree { + +// True if Status objects will capture stack traces on init for non-ok Statuses. +bool DoesStatusSaveStackTrace(); + +// Enables/disables status stack trace saving. This is global for the process. +// While useful for debugging, stack traces can impact performance severely. +void StatusSavesStackTrace(bool on_off); + +enum class StatusCode : int { + kOk = 0, + kCancelled = 1, + kUnknown = 2, + kInvalidArgument = 3, + kDeadlineExceeded = 4, + kNotFound = 5, + kAlreadyExists = 6, + kPermissionDenied = 7, + kResourceExhausted = 8, + kFailedPrecondition = 9, + kAborted = 10, + kOutOfRange = 11, + kUnimplemented = 12, + kInternal = 13, + kUnavailable = 14, + kDataLoss = 15, + kUnauthenticated = 16, + kDoNotUseReservedForFutureExpansionUseDefaultInSwitchInstead_ = 20 +}; + +class ABSL_MUST_USE_RESULT Status; + +// A Status value can be either OK or not-OK +// * OK indicates that the operation succeeded. +// * A not-OK value indicates that the operation failed and contains details +// about the error. +class Status final { + public: + // Creates an OK status with no message. + Status(); + + // Creates a status with the specified code and error message. + Status(StatusCode code, absl::string_view message); + + Status(const Status&); + Status& operator=(const Status& x); + + ~Status(); + + // Returns true if the Status is OK. + ABSL_MUST_USE_RESULT bool ok() const; + + // Returns the error code. + StatusCode code() const; + + // Returns the error message. Note: prefer ToString() for debug logging. + // This message rarely describes the error code. It is not unusual for the + // error message to be the empty string. + absl::string_view message() const; + + // Return a combination of the error code name and message. + std::string ToString() const; + + friend bool operator==(const Status&, const Status&); + friend bool operator!=(const Status&, const Status&); + + // Ignores any errors, potentially suppressing complaints from any tools. + void IgnoreError() const; + + private: + static bool EqualsSlow(const Status& a, const Status& b); + + struct State { + StatusCode code; + std::string message; + }; + // OK status has a nullptr state_. Otherwise, 'state_' points to + // a 'State' structure containing the error code and message(s). + std::unique_ptr state_; +}; + +// Returns an OK status, equivalent to a default constructed instance. +Status OkStatus(); + +// Prints a human-readable representation of `x` to `os`. +std::ostream& operator<<(std::ostream& os, const Status& x); + +// Returns a Status that is identical to `s` except that the message() +// has been augmented by adding `msg` to the end of the original message. +Status Annotate(const Status& s, absl::string_view msg); + +#define CHECK_OK(val) CHECK_EQ(::iree::OkStatus(), (val)) +#define QCHECK_OK(val) QCHECK_EQ(::iree::OkStatus(), (val)) +#define DCHECK_OK(val) DCHECK_EQ(::iree::OkStatus(), (val)) + +} // namespace iree + +#endif // THIRD_PARTY_MLIR_EDGE_IREE_BASE_INTERNAL_STATUS_H_ diff --git a/base/internal/status_builder.cc b/base/internal/status_builder.cc new file mode 100644 index 000000000000..a853950662e9 --- /dev/null +++ b/base/internal/status_builder.cc @@ -0,0 +1,141 @@ +// Copyright 2019 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "third_party/mlir_edge/iree/base/internal/status_builder.h" + +#include + +#include "third_party/mlir_edge/iree/base/internal/status_errors.h" + +namespace iree { + +StatusBuilder::StatusBuilder(const Status& original_status, + absl::SourceLocation location) + : status_(original_status), loc_(location) {} + +StatusBuilder::StatusBuilder(Status&& original_status, + absl::SourceLocation location) + : status_(original_status), loc_(location) {} + +StatusBuilder::StatusBuilder(const StatusBuilder& sb) + : status_(sb.status_), loc_(sb.loc_), message_(sb.message_) {} + +StatusBuilder::StatusBuilder(StatusCode code, absl::SourceLocation location) + : status_(code, ""), loc_(location) {} + +StatusBuilder& StatusBuilder::operator=(const StatusBuilder& sb) { + status_ = sb.status_; + loc_ = sb.loc_; + message_ = sb.message_; + return *this; +} + +StatusBuilder::operator Status() const& { + return StatusBuilder(*this).CreateStatus(); +} +StatusBuilder::operator Status() && { return std::move(*this).CreateStatus(); } + +bool StatusBuilder::ok() const { return status_.ok(); } + +StatusCode StatusBuilder::code() const { return status_.code(); } + +absl::SourceLocation StatusBuilder::source_location() const { return loc_; } + +Status StatusBuilder::CreateStatus() && { + Status result = JoinMessageToStatus(status_, message_); + + // Reset the status after consuming it. + status_ = UnknownError(""); + message_ = ""; + return result; +} + +Status StatusBuilder::JoinMessageToStatus(Status s, absl::string_view msg) { + if (msg.empty()) return s; + return Annotate(s, msg); +} + +std::ostream& operator<<(std::ostream& os, const StatusBuilder& builder) { + return os << static_cast(builder); +} + +std::ostream& operator<<(std::ostream& os, StatusBuilder&& builder) { + return os << static_cast(std::move(builder)); +} + +StatusBuilder AbortedErrorBuilder(absl::SourceLocation location) { + return StatusBuilder(StatusCode::kAborted, location); +} + +StatusBuilder AlreadyExistsErrorBuilder(absl::SourceLocation location) { + return StatusBuilder(StatusCode::kAlreadyExists, location); +} + +StatusBuilder CancelledErrorBuilder(absl::SourceLocation location) { + return StatusBuilder(StatusCode::kCancelled, location); +} + +StatusBuilder DataLossErrorBuilder(absl::SourceLocation location) { + return StatusBuilder(StatusCode::kDataLoss, location); +} + +StatusBuilder DeadlineExceededErrorBuilder(absl::SourceLocation location) { + return StatusBuilder(StatusCode::kDeadlineExceeded, location); +} + +StatusBuilder FailedPreconditionErrorBuilder(absl::SourceLocation location) { + return StatusBuilder(StatusCode::kFailedPrecondition, location); +} + +StatusBuilder InternalErrorBuilder(absl::SourceLocation location) { + return StatusBuilder(StatusCode::kInternal, location); +} + +StatusBuilder InvalidArgumentErrorBuilder(absl::SourceLocation location) { + return StatusBuilder(StatusCode::kInvalidArgument, location); +} + +StatusBuilder NotFoundErrorBuilder(absl::SourceLocation location) { + return StatusBuilder(StatusCode::kNotFound, location); +} + +StatusBuilder OutOfRangeErrorBuilder(absl::SourceLocation location) { + return StatusBuilder(StatusCode::kOutOfRange, location); +} + +StatusBuilder PermissionDeniedErrorBuilder(absl::SourceLocation location) { + return StatusBuilder(StatusCode::kPermissionDenied, location); +} + +StatusBuilder UnauthenticatedErrorBuilder(absl::SourceLocation location) { + return StatusBuilder(StatusCode::kUnauthenticated, location); +} + +StatusBuilder ResourceExhaustedErrorBuilder(absl::SourceLocation location) { + return StatusBuilder(StatusCode::kResourceExhausted, location); +} + +StatusBuilder UnavailableErrorBuilder(absl::SourceLocation location) { + return StatusBuilder(StatusCode::kUnavailable, location); +} + +StatusBuilder UnimplementedErrorBuilder(absl::SourceLocation location) { + return StatusBuilder(StatusCode::kUnimplemented, location); +} + +StatusBuilder UnknownErrorBuilder(absl::SourceLocation location) { + return StatusBuilder(StatusCode::kUnknown, location); +} + +} // namespace iree diff --git a/base/internal/status_builder.h b/base/internal/status_builder.h new file mode 100644 index 000000000000..a22c2995e007 --- /dev/null +++ b/base/internal/status_builder.h @@ -0,0 +1,139 @@ +// Copyright 2019 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#ifndef THIRD_PARTY_MLIR_EDGE_IREE_BASE_INTERNAL_STATUS_BUILDER_H_ +#define THIRD_PARTY_MLIR_EDGE_IREE_BASE_INTERNAL_STATUS_BUILDER_H_ + +#include "third_party/absl/types/source_location.h" +#include "third_party/mlir_edge/iree/base/internal/status.h" + +namespace iree { + +// Creates a status based on an original_status, but enriched with additional +// information. The builder implicitly converts to Status and StatusOr +// allowing for it to be returned directly. +class ABSL_MUST_USE_RESULT StatusBuilder { + public: + // Creates a `StatusBuilder` based on an original status. + explicit StatusBuilder(const Status& original_status, + absl::SourceLocation location + ABSL_LOC_CURRENT_DEFAULT_ARG); + explicit StatusBuilder(Status&& original_status, + absl::SourceLocation location + ABSL_LOC_CURRENT_DEFAULT_ARG); + + // Creates a `StatusBuilder` from a status code. + // A typical user will not specify `location`, allowing it to default to the + // current location. + explicit StatusBuilder(StatusCode code, absl::SourceLocation location + ABSL_LOC_CURRENT_DEFAULT_ARG); + + StatusBuilder(const StatusBuilder& sb); + StatusBuilder& operator=(const StatusBuilder& sb); + StatusBuilder(StatusBuilder&&) = default; + StatusBuilder& operator=(StatusBuilder&&) = default; + + // Appends to the extra message that will be added to the original status. + template + StatusBuilder& operator<<(const T& value) &; + template + StatusBuilder&& operator<<(const T& value) &&; + + // No-op functions that may be added later. + StatusBuilder& LogError() & { return *this; } + StatusBuilder&& LogError() && { return std::move(LogError()); } + StatusBuilder& LogWarning() & { return *this; } + StatusBuilder&& LogWarning() && { return std::move(LogWarning()); } + StatusBuilder& LogInfo() & { return *this; } + StatusBuilder&& LogInfo() && { return std::move(LogInfo()); } + + // Returns true if the Status created by this builder will be ok(). + bool ok() const; + + // Returns the error code for the Status created by this builder. + StatusCode code() const; + + // Returns the source location used to create this builder. + absl::SourceLocation source_location() const; + + // Implicit conversion to Status. + operator Status() const&; + operator Status() &&; + + private: + Status CreateStatus() &&; + + static Status JoinMessageToStatus(Status s, absl::string_view msg); + + // The status that the result will be based on. + Status status_; + + // The location to record if this status is logged. + absl::SourceLocation loc_; + + // The message that will be added to the original status. + std::string message_; +}; + +template +StatusBuilder& StatusBuilder::operator<<(const T& value) & { + return *this; +} +template +StatusBuilder&& StatusBuilder::operator<<(const T& value) && { + return std::move(operator<<(value)); +} + +// Implicitly converts `builder` to `Status` and write it to `os`. +std::ostream& operator<<(std::ostream& os, const StatusBuilder& builder); +std::ostream& operator<<(std::ostream& os, StatusBuilder&& builder); + +// Each of the functions below creates StatusBuilder with a canonical error. +// The error code of the StatusBuilder matches the name of the function. +StatusBuilder AbortedErrorBuilder( + absl::SourceLocation location ABSL_LOC_CURRENT_DEFAULT_ARG); +StatusBuilder AlreadyExistsErrorBuilder( + absl::SourceLocation location ABSL_LOC_CURRENT_DEFAULT_ARG); +StatusBuilder CancelledErrorBuilder( + absl::SourceLocation location ABSL_LOC_CURRENT_DEFAULT_ARG); +StatusBuilder DataLossErrorBuilder( + absl::SourceLocation location ABSL_LOC_CURRENT_DEFAULT_ARG); +StatusBuilder DeadlineExceededErrorBuilder( + absl::SourceLocation location ABSL_LOC_CURRENT_DEFAULT_ARG); +StatusBuilder FailedPreconditionErrorBuilder( + absl::SourceLocation location ABSL_LOC_CURRENT_DEFAULT_ARG); +StatusBuilder InternalErrorBuilder( + absl::SourceLocation location ABSL_LOC_CURRENT_DEFAULT_ARG); +StatusBuilder InvalidArgumentErrorBuilder( + absl::SourceLocation location ABSL_LOC_CURRENT_DEFAULT_ARG); +StatusBuilder NotFoundErrorBuilder( + absl::SourceLocation location ABSL_LOC_CURRENT_DEFAULT_ARG); +StatusBuilder OutOfRangeErrorBuilder( + absl::SourceLocation location ABSL_LOC_CURRENT_DEFAULT_ARG); +StatusBuilder PermissionDeniedErrorBuilder( + absl::SourceLocation location ABSL_LOC_CURRENT_DEFAULT_ARG); +StatusBuilder UnauthenticatedErrorBuilder( + absl::SourceLocation location ABSL_LOC_CURRENT_DEFAULT_ARG); +StatusBuilder ResourceExhaustedErrorBuilder( + absl::SourceLocation location ABSL_LOC_CURRENT_DEFAULT_ARG); +StatusBuilder UnavailableErrorBuilder( + absl::SourceLocation location ABSL_LOC_CURRENT_DEFAULT_ARG); +StatusBuilder UnimplementedErrorBuilder( + absl::SourceLocation location ABSL_LOC_CURRENT_DEFAULT_ARG); +StatusBuilder UnknownErrorBuilder( + absl::SourceLocation location ABSL_LOC_CURRENT_DEFAULT_ARG); + +} // namespace iree + +#endif // THIRD_PARTY_MLIR_EDGE_IREE_BASE_INTERNAL_STATUS_BUILDER_H_ diff --git a/base/internal/status_errno.cc b/base/internal/status_errno.cc new file mode 100644 index 000000000000..3e28aa5afd0f --- /dev/null +++ b/base/internal/status_errno.cc @@ -0,0 +1,175 @@ +// Copyright 2019 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "third_party/mlir_edge/iree/base/internal/status_errno.h" + +#include + +#include "third_party/absl/strings/str_cat.h" + +namespace iree { + +StatusCode ErrnoToCanonicalCode(int error_number) { + switch (error_number) { + case 0: + return StatusCode::kOk; + case EINVAL: // Invalid argument + case ENAMETOOLONG: // Filename too long + case E2BIG: // Argument list too long + case EDESTADDRREQ: // Destination address required + case EDOM: // Mathematics argument out of domain of function + case EFAULT: // Bad address + case EILSEQ: // Illegal byte sequence + case ENOPROTOOPT: // Protocol not available + case ENOSTR: // Not a STREAM + case ENOTSOCK: // Not a socket + case ENOTTY: // Inappropriate I/O control operation + case EPROTOTYPE: // Protocol wrong type for socket + case ESPIPE: // Invalid seek + return StatusCode::kInvalidArgument; + case ETIMEDOUT: // Connection timed out + case ETIME: // Timer expired + return StatusCode::kDeadlineExceeded; + case ENODEV: // No such device + case ENOENT: // No such file or directory +#ifdef ENOMEDIUM + case ENOMEDIUM: // No medium found +#endif + case ENXIO: // No such device or address + case ESRCH: // No such process + return StatusCode::kNotFound; + case EEXIST: // File exists + case EADDRNOTAVAIL: // Address not available + case EALREADY: // Connection already in progress +#ifdef ENOTUNIQ + case ENOTUNIQ: // Name not unique on network +#endif + return StatusCode::kAlreadyExists; + case EPERM: // Operation not permitted + case EACCES: // Permission denied +#ifdef ENOKEY + case ENOKEY: // Required key not available +#endif + case EROFS: // Read only file system + return StatusCode::kPermissionDenied; + case ENOTEMPTY: // Directory not empty + case EISDIR: // Is a directory + case ENOTDIR: // Not a directory + case EADDRINUSE: // Address already in use + case EBADF: // Invalid file descriptor +#ifdef EBADFD + case EBADFD: // File descriptor in bad state +#endif + case EBUSY: // Device or resource busy + case ECHILD: // No child processes + case EISCONN: // Socket is connected +#ifdef EISNAM + case EISNAM: // Is a named type file +#endif +#ifdef ENOTBLK + case ENOTBLK: // Block device required +#endif + case ENOTCONN: // The socket is not connected + case EPIPE: // Broken pipe +#ifdef ESHUTDOWN + case ESHUTDOWN: // Cannot send after transport endpoint shutdown +#endif + case ETXTBSY: // Text file busy +#ifdef EUNATCH + case EUNATCH: // Protocol driver not attached +#endif + return StatusCode::kFailedPrecondition; + case ENOSPC: // No space left on device +#ifdef EDQUOT + case EDQUOT: // Disk quota exceeded +#endif + case EMFILE: // Too many open files + case EMLINK: // Too many links + case ENFILE: // Too many open files in system + case ENOBUFS: // No buffer space available + case ENODATA: // No message is available on the STREAM read queue + case ENOMEM: // Not enough space + case ENOSR: // No STREAM resources +#ifdef EUSERS + case EUSERS: // Too many users +#endif + return StatusCode::kResourceExhausted; +#ifdef ECHRNG + case ECHRNG: // Channel number out of range +#endif + case EFBIG: // File too large + case EOVERFLOW: // Value too large to be stored in data type + case ERANGE: // Result too large + return StatusCode::kOutOfRange; +#ifdef ENOPKG + case ENOPKG: // Package not installed +#endif + case ENOSYS: // Function not implemented + case ENOTSUP: // Operation not supported + case EAFNOSUPPORT: // Address family not supported +#ifdef EPFNOSUPPORT + case EPFNOSUPPORT: // Protocol family not supported +#endif + case EPROTONOSUPPORT: // Protocol not supported +#ifdef ESOCKTNOSUPPORT + case ESOCKTNOSUPPORT: // Socket type not supported +#endif + case EXDEV: // Improper link + return StatusCode::kUnimplemented; + case EAGAIN: // Resource temporarily unavailable +#ifdef ECOMM + case ECOMM: // Communication error on send +#endif + case ECONNREFUSED: // Connection refused + case ECONNABORTED: // Connection aborted + case ECONNRESET: // Connection reset + case EINTR: // Interrupted function call +#ifdef EHOSTDOWN + case EHOSTDOWN: // Host is down +#endif + case EHOSTUNREACH: // Host is unreachable + case ENETDOWN: // Network is down + case ENETRESET: // Connection aborted by network + case ENETUNREACH: // Network unreachable + case ENOLCK: // No locks available + case ENOLINK: // Link has been severed +#ifdef ENONET + case ENONET: // Machine is not on the network +#endif + return StatusCode::kUnavailable; + case EDEADLK: // Resource deadlock avoided +#ifdef ESTALE + case ESTALE: // Stale file handle +#endif + return StatusCode::kAborted; + case ECANCELED: // Operation cancelled + return StatusCode::kCancelled; + default: + return StatusCode::kUnknown; + } +} + +Status ErrnoToCanonicalStatus(int error_number, absl::string_view message) { + // TODO(scotttodd): convert error number to a string + return Status(ErrnoToCanonicalCode(error_number), + absl::StrCat(message, ": ", error_number)); +} + +StatusBuilder ErrnoToCanonicalStatusBuilder(int error_number, + absl::string_view message, + absl::SourceLocation location) { + return StatusBuilder(ErrnoToCanonicalStatus(error_number, message), location); +} + +} // namespace iree diff --git a/base/internal/status_errno.h b/base/internal/status_errno.h new file mode 100644 index 000000000000..d44ad3990a51 --- /dev/null +++ b/base/internal/status_errno.h @@ -0,0 +1,41 @@ +// Copyright 2019 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#ifndef THIRD_PARTY_MLIR_EDGE_IREE_BASE_INTERNAL_STATUS_ERRNO_H_ +#define THIRD_PARTY_MLIR_EDGE_IREE_BASE_INTERNAL_STATUS_ERRNO_H_ + +#include "third_party/absl/strings/string_view.h" +#include "third_party/absl/types/source_location.h" +#include "third_party/mlir_edge/iree/base/internal/status.h" +#include "third_party/mlir_edge/iree/base/internal/statusor.h" + +namespace iree { + +// Returns the code for `error_number`, which should be an `errno` value. +// See https://en.cppreference.com/w/cpp/error/errno_macros and similar refs. +StatusCode ErrnoToCanonicalCode(int error_number); + +// Returns a Status, using a code of `ErrnoToCode(error_number)`, and a +// `message` with the result of `StrError(error_number)` appended. +Status ErrnoToCanonicalStatus(int error_number, absl::string_view message); + +// Returns a StatusBuilder using a status of +// `ErrnoToCanonicalStatus(error_number, message)` and `location`. +StatusBuilder ErrnoToCanonicalStatusBuilder(int error_number, + absl::string_view message, + absl::SourceLocation location); + +} // namespace iree + +#endif // THIRD_PARTY_MLIR_EDGE_IREE_BASE_INTERNAL_STATUS_ERRNO_H_ diff --git a/base/internal/status_errors.cc b/base/internal/status_errors.cc new file mode 100644 index 000000000000..1316f3a3adad --- /dev/null +++ b/base/internal/status_errors.cc @@ -0,0 +1,147 @@ +// Copyright 2019 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "third_party/mlir_edge/iree/base/internal/status_errors.h" + +namespace iree { + +Status AbortedError(absl::string_view message) { + return Status(StatusCode::kAborted, message); +} + +Status AlreadyExistsError(absl::string_view message) { + return Status(StatusCode::kAlreadyExists, message); +} + +Status CancelledError(absl::string_view message) { + return Status(StatusCode::kCancelled, message); +} + +Status DataLossError(absl::string_view message) { + return Status(StatusCode::kDataLoss, message); +} + +Status DeadlineExceededError(absl::string_view message) { + return Status(StatusCode::kDeadlineExceeded, message); +} + +Status FailedPreconditionError(absl::string_view message) { + return Status(StatusCode::kFailedPrecondition, message); +} + +Status InternalError(absl::string_view message) { + return Status(StatusCode::kInternal, message); +} + +Status InvalidArgumentError(absl::string_view message) { + return Status(StatusCode::kInvalidArgument, message); +} + +Status NotFoundError(absl::string_view message) { + return Status(StatusCode::kNotFound, message); +} + +Status OutOfRangeError(absl::string_view message) { + return Status(StatusCode::kOutOfRange, message); +} + +Status PermissionDeniedError(absl::string_view message) { + return Status(StatusCode::kPermissionDenied, message); +} + +Status ResourceExhaustedError(absl::string_view message) { + return Status(StatusCode::kResourceExhausted, message); +} + +Status UnauthenticatedError(absl::string_view message) { + return Status(StatusCode::kUnauthenticated, message); +} + +Status UnavailableError(absl::string_view message) { + return Status(StatusCode::kUnavailable, message); +} + +Status UnimplementedError(absl::string_view message) { + return Status(StatusCode::kUnimplemented, message); +} + +Status UnknownError(absl::string_view message) { + return Status(StatusCode::kUnknown, message); +} + +bool IsAborted(const Status& status) { + return status.code() == StatusCode::kAborted; +} + +bool IsAlreadyExists(const Status& status) { + return status.code() == StatusCode::kAlreadyExists; +} + +bool IsCancelled(const Status& status) { + return status.code() == StatusCode::kCancelled; +} + +bool IsDataLoss(const Status& status) { + return status.code() == StatusCode::kDataLoss; +} + +bool IsDeadlineExceeded(const Status& status) { + return status.code() == StatusCode::kDeadlineExceeded; +} + +bool IsFailedPrecondition(const Status& status) { + return status.code() == StatusCode::kFailedPrecondition; +} + +bool IsInternal(const Status& status) { + return status.code() == StatusCode::kInternal; +} + +bool IsInvalidArgument(const Status& status) { + return status.code() == StatusCode::kInvalidArgument; +} + +bool IsNotFound(const Status& status) { + return status.code() == StatusCode::kNotFound; +} + +bool IsOutOfRange(const Status& status) { + return status.code() == StatusCode::kOutOfRange; +} + +bool IsPermissionDenied(const Status& status) { + return status.code() == StatusCode::kPermissionDenied; +} + +bool IsResourceExhausted(const Status& status) { + return status.code() == StatusCode::kResourceExhausted; +} + +bool IsUnauthenticated(const Status& status) { + return status.code() == StatusCode::kUnauthenticated; +} + +bool IsUnavailable(const Status& status) { + return status.code() == StatusCode::kUnavailable; +} + +bool IsUnimplemented(const Status& status) { + return status.code() == StatusCode::kUnimplemented; +} + +bool IsUnknown(const Status& status) { + return status.code() == StatusCode::kUnknown; +} + +} // namespace iree diff --git a/base/internal/status_errors.h b/base/internal/status_errors.h new file mode 100644 index 000000000000..de5c91798079 --- /dev/null +++ b/base/internal/status_errors.h @@ -0,0 +1,60 @@ +// Copyright 2019 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#ifndef THIRD_PARTY_MLIR_EDGE_IREE_BASE_INTERNAL_STATUS_ERRORS_H_ +#define THIRD_PARTY_MLIR_EDGE_IREE_BASE_INTERNAL_STATUS_ERRORS_H_ + +#include "third_party/absl/base/attributes.h" +#include "third_party/absl/strings/string_view.h" +#include "third_party/mlir_edge/iree/base/internal/status.h" + +namespace iree { + +Status AbortedError(absl::string_view message); +Status AlreadyExistsError(absl::string_view message); +Status CancelledError(absl::string_view message); +Status DataLossError(absl::string_view message); +Status DeadlineExceededError(absl::string_view message); +Status FailedPreconditionError(absl::string_view message); +Status InternalError(absl::string_view message); +Status InvalidArgumentError(absl::string_view message); +Status NotFoundError(absl::string_view message); +Status OutOfRangeError(absl::string_view message); +Status PermissionDeniedError(absl::string_view message); +Status ResourceExhaustedError(absl::string_view message); +Status UnauthenticatedError(absl::string_view message); +Status UnavailableError(absl::string_view message); +Status UnimplementedError(absl::string_view message); +Status UnknownError(absl::string_view message); + +ABSL_MUST_USE_RESULT bool IsAborted(const Status& status); +ABSL_MUST_USE_RESULT bool IsAlreadyExists(const Status& status); +ABSL_MUST_USE_RESULT bool IsCancelled(const Status& status); +ABSL_MUST_USE_RESULT bool IsDataLoss(const Status& status); +ABSL_MUST_USE_RESULT bool IsDeadlineExceeded(const Status& status); +ABSL_MUST_USE_RESULT bool IsFailedPrecondition(const Status& status); +ABSL_MUST_USE_RESULT bool IsInternal(const Status& status); +ABSL_MUST_USE_RESULT bool IsInvalidArgument(const Status& status); +ABSL_MUST_USE_RESULT bool IsNotFound(const Status& status); +ABSL_MUST_USE_RESULT bool IsOutOfRange(const Status& status); +ABSL_MUST_USE_RESULT bool IsPermissionDenied(const Status& status); +ABSL_MUST_USE_RESULT bool IsResourceExhausted(const Status& status); +ABSL_MUST_USE_RESULT bool IsUnauthenticated(const Status& status); +ABSL_MUST_USE_RESULT bool IsUnavailable(const Status& status); +ABSL_MUST_USE_RESULT bool IsUnimplemented(const Status& status); +ABSL_MUST_USE_RESULT bool IsUnknown(const Status& status); + +} // namespace iree + +#endif // THIRD_PARTY_MLIR_EDGE_IREE_BASE_INTERNAL_STATUS_ERRORS_H_ diff --git a/base/internal/status_macros.h b/base/internal/status_macros.h new file mode 100644 index 000000000000..b9492ede0cdd --- /dev/null +++ b/base/internal/status_macros.h @@ -0,0 +1,108 @@ +// Copyright 2019 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#ifndef THIRD_PARTY_MLIR_EDGE_IREE_BASE_INTERNAL_STATUS_MACROS_H_ +#define THIRD_PARTY_MLIR_EDGE_IREE_BASE_INTERNAL_STATUS_MACROS_H_ + +#include "third_party/absl/types/source_location.h" +#include "third_party/mlir_edge/iree/base/internal/status.h" +#include "third_party/mlir_edge/iree/base/internal/status_builder.h" +#include "third_party/mlir_edge/iree/base/internal/statusor.h" + +// Evaluates an expression that produces a `iree::Status`. If the status is not +// ok, returns it from the current function. +#define RETURN_IF_ERROR(expr) \ + STATUS_MACROS_IMPL_ELSE_BLOCKER_ \ + if (iree::status_macro_internal::StatusAdaptorForMacros \ + status_macro_internal_adaptor = {(expr), ABSL_LOC}) { \ + } else /* NOLINT */ \ + return status_macro_internal_adaptor.Consume() + +// Executes an expression `rexpr` that returns a `iree::StatusOr`. On OK, +// moves its value into the variable defined by `lhs`, otherwise returns +// from the current function. +#define ASSIGN_OR_RETURN(...) \ + STATUS_MACROS_IMPL_GET_VARIADIC_((__VA_ARGS__, \ + STATUS_MACROS_IMPL_ASSIGN_OR_RETURN_3_, \ + STATUS_MACROS_IMPL_ASSIGN_OR_RETURN_2_)) \ + (__VA_ARGS__) + +// ================================================================= +// == Implementation details, do not rely on anything below here. == +// ================================================================= + +// MSVC incorrectly expands variadic macros, splice together a macro call to +// work around the bug. +#define STATUS_MACROS_IMPL_GET_VARIADIC_HELPER_(_1, _2, _3, NAME, ...) NAME +#define STATUS_MACROS_IMPL_GET_VARIADIC_(args) \ + STATUS_MACROS_IMPL_GET_VARIADIC_HELPER_ args + +#define STATUS_MACROS_IMPL_ASSIGN_OR_RETURN_2_(lhs, rexpr) \ + STATUS_MACROS_IMPL_ASSIGN_OR_RETURN_3_(lhs, rexpr, std::move(_)) +#define STATUS_MACROS_IMPL_ASSIGN_OR_RETURN_3_(lhs, rexpr, error_expression) \ + STATUS_MACROS_IMPL_ASSIGN_OR_RETURN_( \ + STATUS_MACROS_IMPL_CONCAT_(_status_or_value, __LINE__), lhs, rexpr, \ + error_expression) +#define STATUS_MACROS_IMPL_ASSIGN_OR_RETURN_(statusor, lhs, rexpr, \ + error_expression) \ + auto statusor = (rexpr); \ + if (ABSL_PREDICT_FALSE(!statusor.ok())) { \ + iree::StatusBuilder _(std::move(statusor).status(), ABSL_LOC); \ + (void)_; /* error_expression is allowed to not use this variable */ \ + return (error_expression); \ + } \ + lhs = std::move(statusor).ValueOrDie() + +// Internal helper for concatenating macro values. +#define STATUS_MACROS_IMPL_CONCAT_INNER_(x, y) x##y +#define STATUS_MACROS_IMPL_CONCAT_(x, y) STATUS_MACROS_IMPL_CONCAT_INNER_(x, y) + +// clang-format off +#define STATUS_MACROS_IMPL_ELSE_BLOCKER_ switch (0) case 0: default: // NOLINT +// clang-format on + +namespace iree { +namespace status_macro_internal { + +// Provides a conversion to bool so that it can be used inside an if statement +// that declares a variable. +class StatusAdaptorForMacros { + public: + StatusAdaptorForMacros(const Status& status, absl::SourceLocation loc) + : builder_(status, loc) {} + + StatusAdaptorForMacros(Status&& status, absl::SourceLocation loc) + : builder_(std::move(status), loc) {} + + StatusAdaptorForMacros(const StatusBuilder& builder, absl::SourceLocation loc) + : builder_(builder) {} + + StatusAdaptorForMacros(StatusBuilder&& builder, absl::SourceLocation loc) + : builder_(std::move(builder)) {} + + StatusAdaptorForMacros(const StatusAdaptorForMacros&) = delete; + StatusAdaptorForMacros& operator=(const StatusAdaptorForMacros&) = delete; + + explicit operator bool() const { return ABSL_PREDICT_TRUE(builder_.ok()); } + + StatusBuilder&& Consume() { return std::move(builder_); } + + private: + StatusBuilder builder_; +}; + +} // namespace status_macro_internal +} // namespace iree + +#endif // THIRD_PARTY_MLIR_EDGE_IREE_BASE_INTERNAL_STATUS_MACROS_H_ diff --git a/base/internal/statusor.cc b/base/internal/statusor.cc new file mode 100644 index 000000000000..a74232dd1d57 --- /dev/null +++ b/base/internal/statusor.cc @@ -0,0 +1,39 @@ +// Copyright 2019 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "third_party/mlir_edge/iree/base/internal/statusor.h" + +#include "third_party/mlir_edge/iree/base/internal/status_errors.h" + +namespace iree { + +namespace internal_statusor { + +void Helper::HandleInvalidStatusCtorArg(Status* status) { + const char* kMessage = + "An OK status is not a valid constructor argument to StatusOr"; + LOG(ERROR) << kMessage; + *status = InternalError(kMessage); + abort(); +} + +void Helper::Crash(const Status& status) { + LOG(FATAL) << "Attempting to fetch value instead of handling error " + << status; + abort(); +} + +} // namespace internal_statusor + +} // namespace iree diff --git a/base/internal/statusor.h b/base/internal/statusor.h new file mode 100644 index 000000000000..342012612129 --- /dev/null +++ b/base/internal/statusor.h @@ -0,0 +1,699 @@ +// Copyright 2019 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#ifndef THIRD_PARTY_MLIR_EDGE_IREE_BASE_INTERNAL_STATUSOR_H_ +#define THIRD_PARTY_MLIR_EDGE_IREE_BASE_INTERNAL_STATUSOR_H_ + +#include "third_party/absl/base/attributes.h" +#include "third_party/mlir_edge/iree/base/internal/status.h" +#include "third_party/mlir_edge/iree/base/internal/status_builder.h" + +namespace iree { + +template +class ABSL_MUST_USE_RESULT StatusOr; + +namespace internal_statusor { + +template +using IsStatusOrConversionAmbiguous = + absl::disjunction&>, + std::is_constructible&>, + std::is_constructible&&>, + std::is_constructible&&>, + std::is_convertible&, T>, + std::is_convertible&, T>, + std::is_convertible&&, T>, + std::is_convertible&&, T>>; + +template +using IsStatusOrConversionAssigmentAmbiguous = + absl::disjunction, + std::is_assignable&>, + std::is_assignable&>, + std::is_assignable&&>, + std::is_assignable&&>>; + +template +struct IsAmbiguousStatusOrForInitialization + : // Strip const-value refs from type and check again, else false_type. + public absl::conditional_t< + std::is_same>, + U>::value, + std::false_type, + IsAmbiguousStatusOrForInitialization< + T, absl::remove_cv_t>>> {}; + +template +struct IsAmbiguousStatusOrForInitialization> + : public IsStatusOrConversionAmbiguous {}; + +template +using IsStatusOrDirectInitializationAmbiguous = absl::disjunction< + std::is_same, absl::remove_cv_t>>, + std::is_same>>, + std::is_same>>, + std::is_same>>, + IsAmbiguousStatusOrForInitialization>; + +template +using IsStatusOrDirectInitializationValid = absl::disjunction< + // The is_same allows nested status ors to ignore this check iff same type. + std::is_same>>, + absl::negation>>; + +class Helper { + public: + static void HandleInvalidStatusCtorArg(Status*); + ABSL_ATTRIBUTE_NORETURN static void Crash(const Status& status); +}; + +// Construct an instance of T in `p` through placement new, passing Args... to +// the constructor. +// This abstraction is here mostly for the gcc performance fix. +template +void PlacementNew(void* p, Args&&... args) { +#if defined(__GNUC__) && !defined(__clang__) + // Teach gcc that 'p' cannot be null, fixing code size issues. + if (p == nullptr) __builtin_unreachable(); +#endif + new (p) T(std::forward(args)...); +} + +// Helper base class to hold the data and all operations. +// We move all this to a base class to allow mixing with the appropriate +// TraitsBase specialization. +template +class StatusOrData { + template + friend class StatusOrData; + + public: + StatusOrData() = delete; + + StatusOrData(const StatusOrData& other) { + if (other.ok()) { + MakeValue(other.data_); + MakeStatus(); + } else { + MakeStatus(other.status_); + } + } + + StatusOrData(StatusOrData&& other) noexcept { + if (other.ok()) { + MakeValue(std::move(other.data_)); + MakeStatus(); + } else { + MakeStatus(std::move(other.status_)); + } + } + + template + explicit StatusOrData(const StatusOrData& other) { + if (other.ok()) { + MakeValue(other.data_); + MakeStatus(); + } else { + MakeStatus(other.status_); + } + } + + template + explicit StatusOrData(StatusOrData&& other) { + if (other.ok()) { + MakeValue(std::move(other.data_)); + MakeStatus(); + } else { + MakeStatus(std::move(other.status_)); + } + } + + template + explicit StatusOrData(absl::in_place_t, Args&&... args) + : data_(std::forward(args)...) { + MakeStatus(); + } + + explicit StatusOrData(const T& value) : data_(value) { MakeStatus(); } + explicit StatusOrData(T&& value) : data_(std::move(value)) { MakeStatus(); } + + explicit StatusOrData(const Status& status) : status_(status) { + EnsureNotOk(); + } + explicit StatusOrData(Status&& status) : status_(status) { EnsureNotOk(); } + + explicit StatusOrData(const StatusBuilder& builder) : status_(builder) { + EnsureNotOk(); + } + explicit StatusOrData(StatusBuilder&& builder) : status_(std::move(builder)) { + EnsureNotOk(); + } + + StatusOrData& operator=(const StatusOrData& other) { + if (this == &other) return *this; + if (other.ok()) + Assign(other.data_); + else + Assign(other.status_); + return *this; + } + + StatusOrData& operator=(StatusOrData&& other) { + if (this == &other) return *this; + if (other.ok()) + Assign(std::move(other.data_)); + else + Assign(std::move(other.status_)); + return *this; + } + + ~StatusOrData() { + if (ok()) { + status_.~Status(); + data_.~T(); + } else { + status_.~Status(); + } + } + + void Assign(const T& value) { + if (ok()) { + data_.~T(); + MakeValue(value); + } else { + MakeValue(value); + status_ = OkStatus(); + } + } + + void Assign(T&& value) { + if (ok()) { + data_.~T(); + MakeValue(std::move(value)); + } else { + MakeValue(std::move(value)); + status_ = OkStatus(); + } + } + + void Assign(const Status& status) { + Clear(); + status_ = status; + EnsureNotOk(); + } + + void Assign(Status&& status) { + Clear(); + status_ = std::move(status); + EnsureNotOk(); + } + + bool ok() const { return status_.ok(); } + + protected: + // status_ will always be active after the constructor. + // Union to be able to initialize exactly how we need without waste. + // Eg. in the copy constructor we use the default constructor of Status in + // the ok() path to avoid an extra Ref call. + union { + Status status_; + }; + + // data_ is active iff status_.ok()==true + struct Dummy {}; + union { + // When T is const, we need some non-const object we can cast to void* for + // the placement new. dummy_ is that object. + Dummy dummy_; + T data_; + }; + + void Clear() { + if (ok()) data_.~T(); + } + + void EnsureOk() const { + if (!ok()) Helper::Crash(status_); + } + + void EnsureNotOk() { + if (ok()) Helper::HandleInvalidStatusCtorArg(&status_); + } + + // Construct the value (data_) through placement new with the passed arg. + template + void MakeValue(Arg&& arg) { + internal_statusor::PlacementNew(&dummy_, std::forward(arg)); + } + + // Construct the status (status_) through placement new with the passed arg. + template + void MakeStatus(Args&&... args) { + internal_statusor::PlacementNew(&status_, + std::forward(args)...); + } +}; + +// Helper base class to allow implicitly deleted constructors and assignment +// operations in StatusOr. +// TraitsBase will explicitly delete what it can't support and StatusOr will +// inherit that behavior implicitly. +template +struct TraitsBase { + TraitsBase() = default; + TraitsBase(const TraitsBase&) = default; + TraitsBase(TraitsBase&&) = default; + TraitsBase& operator=(const TraitsBase&) = default; + TraitsBase& operator=(TraitsBase&&) = default; +}; + +template <> +struct TraitsBase { + TraitsBase() = default; + TraitsBase(const TraitsBase&) = delete; + TraitsBase(TraitsBase&&) = default; + TraitsBase& operator=(const TraitsBase&) = delete; + TraitsBase& operator=(TraitsBase&&) = default; +}; + +template <> +struct TraitsBase { + TraitsBase() = default; + TraitsBase(const TraitsBase&) = delete; + TraitsBase(TraitsBase&&) = delete; + TraitsBase& operator=(const TraitsBase&) = delete; + TraitsBase& operator=(TraitsBase&&) = delete; +}; + +} // namespace internal_statusor + +// StatusOr is the union of a Status object and a T object. +// +// A StatusOr object either holds a usable value, or an error Status explaining +// why such a value is not present. +template +class StatusOr : private internal_statusor::StatusOrData, + private internal_statusor::TraitsBase< + std::is_copy_constructible::value, + std::is_move_constructible::value> { + template + friend class StatusOr; + + typedef internal_statusor::StatusOrData Base; + + public: + typedef T element_type; + + // Constructs a new StatusOr with StatusCode::kUnknown status. + explicit StatusOr(); + + // StatusOr is copy constructible/assignable if T is copy constructible. + StatusOr(const StatusOr&) = default; + StatusOr& operator=(const StatusOr&) = default; + + // StatusOr is move constructible/assignable if T is move constructible. + StatusOr(StatusOr&&) = default; + StatusOr& operator=(StatusOr&&) = default; + + // Converting constructors from StatusOr, when T is constructible from U. + // To avoid ambiguity, they are disabled if T is also constructible from + // StatusOr. Explicit iff the corresponding construction of T from U is + // explicit. + template < + typename U, + absl::enable_if_t< + absl::conjunction< + absl::negation>, + std::is_constructible, + std::is_convertible, + absl::negation>>::value, + int> = 0> + StatusOr(const StatusOr& other) // NOLINT + : Base(static_cast::Base&>(other)) {} + template < + typename U, + absl::enable_if_t< + absl::conjunction< + absl::negation>, + std::is_constructible, + absl::negation>, + absl::negation>>::value, + int> = 0> + explicit StatusOr(const StatusOr& other) + : Base(static_cast::Base&>(other)) {} + + template < + typename U, + absl::enable_if_t< + absl::conjunction< + absl::negation>, std::is_constructible, + std::is_convertible, + absl::negation>>::value, + int> = 0> + StatusOr(StatusOr&& other) // NOLINT + : Base(static_cast::Base&&>(other)) {} + template < + typename U, + absl::enable_if_t< + absl::conjunction< + absl::negation>, std::is_constructible, + absl::negation>, + absl::negation>>::value, + int> = 0> + explicit StatusOr(StatusOr&& other) + : Base(static_cast::Base&&>(other)) {} + + // Conversion copy/move assignment operator, T must be constructible and + // assignable from U. Only enable if T cannot be directly assigned from + // StatusOr. + template < + typename U, + absl::enable_if_t< + absl::conjunction< + absl::negation>, + std::is_constructible, + std::is_assignable, + absl::negation< + internal_statusor::IsStatusOrConversionAssigmentAmbiguous< + T, U>>>::value, + int> = 0> + StatusOr& operator=(const StatusOr& other) { + this->Assign(other); + return *this; + } + template < + typename U, + absl::enable_if_t< + absl::conjunction< + absl::negation>, std::is_constructible, + std::is_assignable, + absl::negation< + internal_statusor::IsStatusOrConversionAssigmentAmbiguous< + T, U>>>::value, + int> = 0> + StatusOr& operator=(StatusOr&& other) { + this->Assign(std::move(other)); + return *this; + } + + // Constructs a new StatusOr with the given value. After calling this + // constructor, this->ok() will be true and the contained value may be + // retrieved with ValueOrDie(), operator*(), or operator->(). + StatusOr(const T& value); + + // Constructs a new StatusOr with the given non-ok status. After calling this + // constructor, this->ok() will be false and calls to ValueOrDie() will + // CHECK-fail. + StatusOr(const Status& status); + StatusOr& operator=(const Status& status); + StatusOr(const StatusBuilder& builder); + StatusOr& operator=(const StatusBuilder& builder); + + // Similar to the `const T&` overload. + // + // REQUIRES: T is move constructible. + StatusOr(T&& value); + + // RValue versions of the operations declared above. + StatusOr(Status&& status); + StatusOr& operator=(Status&& status); + StatusOr(StatusBuilder&& builder); + StatusOr& operator=(StatusBuilder&& builder); + + // Constructs the inner value T in-place using the provided args, using the + // T(args...) constructor. + template + explicit StatusOr(absl::in_place_t, Args&&... args); + template + explicit StatusOr(absl::in_place_t, std::initializer_list ilist, + Args&&... args); + + // Constructs the inner value T in-place using the provided args, using the + // T(U) (direct-initialization) constructor. Only valid if T can be + // constructed from a U. Can accept move or copy constructors. Explicit it + // U is not convertible to T. To avoid ambiguity, this is disabled if U is + // a StatusOr, where J is convertible to T. + template < + typename U = T, + absl::enable_if_t< + absl::conjunction< + internal_statusor::IsStatusOrDirectInitializationValid, + std::is_constructible, + std::is_convertible>::value, + int> = 0> + StatusOr(U&& u) // NOLINT + : StatusOr(absl::in_place, std::forward(u)) {} + + template < + typename U = T, + absl::enable_if_t< + absl::conjunction< + internal_statusor::IsStatusOrDirectInitializationValid, + std::is_constructible, + absl::negation>>::value, + int> = 0> + explicit StatusOr(U&& u) // NOLINT + : StatusOr(absl::in_place, std::forward(u)) {} + + // Returns this->ok() + explicit operator bool() const { return ok(); } + + // Returns this->status().ok() + ABSL_MUST_USE_RESULT bool ok() const { return this->status_.ok(); } + + // Returns a reference to our status. If this contains a T, then + // returns OkStatus(). + const Status& status() const&; + Status status() &&; + + // Returns a reference to our current value, or CHECK-fails if !this->ok(). If + // you have already checked the status using this->ok() or operator bool(), + // then you probably want to use operator*() or operator->() to access the + // current value instead of ValueOrDie(). + const T& ValueOrDie() const&; + T& ValueOrDie() &; + const T&& ValueOrDie() const&&; + T&& ValueOrDie() &&; + + // Returns a reference to the current value. + // + // REQUIRES: this->ok() == true, otherwise the behavior is undefined. + const T& operator*() const&; + T& operator*() &; + const T&& operator*() const&&; + T&& operator*() &&; + + // Returns a pointer to the current value. + // + // REQUIRES: this->ok() == true, otherwise the behavior is undefined. + const T* operator->() const; + T* operator->(); + + // Returns a copy of the current value if this->ok() == true. Otherwise + // returns a default value. + template + T value_or(U&& default_value) const&; + template + T value_or(U&& default_value) &&; + + // Ignores any errors. This method does nothing except potentially suppress + // complaints from any tools that are checking that errors are not dropped on + // the floor. + void IgnoreError() const; + + private: + using internal_statusor::StatusOrData::Assign; + template + void Assign(const StatusOr& other); + template + void Assign(StatusOr&& other); +}; + +//////////////////////////////////////////////////////////////////////////////// +// Implementation details for StatusOr + +template +StatusOr::StatusOr() : Base(Status(StatusCode::kUnknown, "")) {} + +template +StatusOr::StatusOr(const T& value) : Base(value) {} + +template +StatusOr::StatusOr(const Status& status) : Base(status) {} + +template +StatusOr::StatusOr(const StatusBuilder& builder) : Base(builder) {} + +template +StatusOr& StatusOr::operator=(const Status& status) { + this->Assign(status); + return *this; +} + +template +StatusOr& StatusOr::operator=(const StatusBuilder& builder) { + return *this = static_cast(builder); +} + +template +StatusOr::StatusOr(T&& value) : Base(std::move(value)) {} + +template +StatusOr::StatusOr(Status&& status) : Base(std::move(status)) {} + +template +StatusOr::StatusOr(StatusBuilder&& builder) : Base(std::move(builder)) {} + +template +StatusOr& StatusOr::operator=(Status&& status) { + this->Assign(std::move(status)); + return *this; +} + +template +StatusOr& StatusOr::operator=(StatusBuilder&& builder) { + return *this = static_cast(std::move(builder)); +} + +template +template +inline void StatusOr::Assign(const StatusOr& other) { + if (other.ok()) { + this->Assign(other.ValueOrDie()); + } else { + this->Assign(other.status()); + } +} + +template +template +inline void StatusOr::Assign(StatusOr&& other) { + if (other.ok()) { + this->Assign(std::move(other).ValueOrDie()); + } else { + this->Assign(std::move(other).status()); + } +} +template +template +StatusOr::StatusOr(absl::in_place_t, Args&&... args) + : Base(absl::in_place, std::forward(args)...) {} + +template +template +StatusOr::StatusOr(absl::in_place_t, std::initializer_list ilist, + Args&&... args) + : Base(absl::in_place, ilist, std::forward(args)...) {} + +template +const Status& StatusOr::status() const& { + return this->status_; +} +template +Status StatusOr::status() && { + return ok() ? OkStatus() : std::move(this->status_); +} + +template +const T& StatusOr::ValueOrDie() const& { + this->EnsureOk(); + return this->data_; +} + +template +T& StatusOr::ValueOrDie() & { + this->EnsureOk(); + return this->data_; +} + +template +const T&& StatusOr::ValueOrDie() const&& { + this->EnsureOk(); + return std::move(this->data_); +} + +template +T&& StatusOr::ValueOrDie() && { + this->EnsureOk(); + return std::move(this->data_); +} + +template +const T& StatusOr::operator*() const& { + this->EnsureOk(); + return this->data_; +} + +template +T& StatusOr::operator*() & { + this->EnsureOk(); + return this->data_; +} + +template +const T&& StatusOr::operator*() const&& { + this->EnsureOk(); + return std::move(this->data_); +} + +template +T&& StatusOr::operator*() && { + this->EnsureOk(); + return std::move(this->data_); +} + +template +const T* StatusOr::operator->() const { + this->EnsureOk(); + return &this->data_; +} + +template +T* StatusOr::operator->() { + this->EnsureOk(); + return &this->data_; +} + +template +template +T StatusOr::value_or(U&& default_value) const& { + if (ok()) { + return this->data_; + } + return std::forward(default_value); +} + +template +template +T StatusOr::value_or(U&& default_value) && { + if (ok()) { + return std::move(this->data_); + } + return std::forward(default_value); +} + +template +void StatusOr::IgnoreError() const { + // no-op +} + +} // namespace iree + +#endif // THIRD_PARTY_MLIR_EDGE_IREE_BASE_INTERNAL_STATUSOR_H_ diff --git a/base/intrusive_list.h b/base/intrusive_list.h new file mode 100644 index 000000000000..b3fbc88e48fe --- /dev/null +++ b/base/intrusive_list.h @@ -0,0 +1,766 @@ +// Copyright 2019 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// Doubly linked list using element interior storage. +// This has the performance of std::list (that means O(1) on insert and remove) +// but performs no allocations and has better caching behavior. +// +// Elements are maintained in lists by way of IntrusiveListLinks, with each link +// allowing the element to exist in one list simultaneously. In the most simple +// case subclassing IntrusiveLinkBase will let the type be added to a list with +// little boilerplate. If an element must be in more than one list +// simultaneously IntrusiveListLinks can be added as members. +// +// Usage (simple): +// class MySimpleElement : public IntrusiveLinkBase {}; +// IntrusiveList list; +// list.push_back(new MySimpleElement()); +// for (auto element : list) { ... } +// +// Usage (multiple lists): +// class MultiElement { +// public: +// IntrusiveListLink list_link_a; +// IntrusiveListLink list_link_b; +// }; +// IntrusiveList list_a; +// IntrusiveList list_b; +// +// By default elements in the list are not retained and must be kept alive +// externally. For automatic memory management there are specializations for +// std::unique_ptr. +// +// Usage (unique_ptr): +// IntrusiveList> list; +// list.push_back(absl::make_unique()); +// std::unique_ptr elm = list.take(list.front()); +// +// We use this instead of //util/gtl/intrusive_list.h as we need support for +// list items being in multiple lists simultaneously. Without some significant +// hackery this isn't possible with the gtl version as the intrusive_link must +// be subclassed instead of being a member variable. The gtl type also conforms +// more to STL (nice!) but unfortunately that means that iterators are used in +// most operations. We tend not to have iterators, and the act of mapping from +// an element to an iterator to then go back to elements is wasteful. +// +// This type is thread-unsafe. + +#ifndef THIRD_PARTY_MLIR_EDGE_IREE_BASE_INTRUSIVE_LIST_H_ +#define THIRD_PARTY_MLIR_EDGE_IREE_BASE_INTRUSIVE_LIST_H_ + +#include +#include +#include +#include +#include + +#include "third_party/mlir_edge/iree/base/logging.h" + +namespace iree { + +// Define to enable extensive checks after each mutation of the intrusive list. +// #define IREE_PARANOID_INTRUSIVE_LIST + +// Storage for the doubly-linked list. +// This is embedded within all elements in an intrusive list. +struct IntrusiveListLink { + IntrusiveListLink* prev = nullptr; + IntrusiveListLink* next = nullptr; + + IntrusiveListLink() = default; + + // Prevent copies. + IntrusiveListLink(const IntrusiveListLink&) = delete; + IntrusiveListLink& operator=(const IntrusiveListLink&) = delete; +}; + +template +struct IntrusiveLinkBase : public T { + public: + IntrusiveListLink link; +}; + +template <> +struct IntrusiveLinkBase { + public: + IntrusiveListLink link; +}; + +// Base type for intrusive lists. +// This is either used directly when the list is on naked pointers or +// specialized to std::unique_ptr. +template +class IntrusiveListBase { + public: + using self_type = IntrusiveListBase; + + IntrusiveListBase() = default; + virtual ~IntrusiveListBase() { clear(); } + + // Prevent copies. + IntrusiveListBase(const IntrusiveListBase&) = delete; + IntrusiveListBase& operator=(const IntrusiveListBase&) = delete; + + // Returns true if the list is empty. + // Performance: O(1) + constexpr bool empty() const { return head_ == nullptr; } + + // Returns the total number of items in the list. + // Performance: O(1) + constexpr size_t size() const { return count_; } + + // Returns true if the given item is contained within the list. + // Performance: O(n) + bool contains(T* value) const; + + // Appends the contents of the given list to this one. + // The |other_list| is cleared. + // Performance: O(1) + void merge_from(self_type* other_list); + + // Removes all items from the list. + // Performance: O(n) + void clear(); + + IteratorT begin() const { return IteratorT(head_); } + IteratorT end() const { return IteratorT(nullptr); } + ReverseIteratorT rbegin() const { return ReverseIteratorT(tail_); } + ReverseIteratorT rend() const { return ReverseIteratorT(nullptr); } + + // Returns the next item in the list relative to the given item. + // |value| must exist in the list. + // Performance: O(1) + T* next(T* value) const; + + // Returns the previous item in the list relative to the given item. + // |value| must exist in the list. + // Performance: O(1) + T* previous(T* value) const; + + // Returns the item at the front of the list, if any. + // Performance: O(1) + T* front() const; + + // Inserts an item at the front of the list. + // Performance: O(1) + void push_front(T* value); + + // Removes the item at the front of the list. + // Performance: O(1) + void pop_front(); + + // Returns the item at the back of the list, if any. + // Performance: O(1) + T* back() const; + + // Inserts an item at the back of the list. + // Performance: O(1) + void push_back(T* value); + + // Removes the item at the back of the list. + // Performance: O(1) + void pop_back(); + + // Inserts an item into the list before the given iterator. + // Performance: O(1) + void insert(const IteratorT& it, T* value) { return insert(*it, value); } + void insert(T* position, T* value); + + // Erases the given item from the list. + // Returns the item following the erased item, if any. + // Performance: O(1) + T* erase(T* value); + + // Erases the item from the list at the given iterator. + // Performance: O(1) + IteratorT erase(const IteratorT& it); + ReverseIteratorT erase(const ReverseIteratorT& it); + + // Replaces the item with a new item at the same position. + // |new_value| must not be contained in any list. + // Performance: O(1) + void replace(T* old_value, T* new_value); + + // Sorts the list with the given comparison function. + // The sort function is the same as used by std::sort. + // + // Uses merge sort O(N log N) using the algorithm described here: + // http://www.chiark.greenend.org.uk/~sgtatham/algorithms/listsort.html + void sort(bool (*compare_fn)(T* a, T* b)); + + protected: + // Called when an item is added to the list. + virtual void OnAdd(T* value) {} + // Called when an item is removed from the list. + virtual void OnRemove(T* value) {} + // Called when an item is removed and deallocated. + virtual void OnDeallocate(T* value) {} + + // Performs expensive correctness checks on the list structure. It's too slow + // to use in normal builds (even dbg), so it should only be used when there's + // a suspected issue with an intrusive list. Define + // IREE_PARANOID_INTRUSIVE_LIST to enable. + void CheckCorrectness() const; + + IntrusiveListLink* head_ = nullptr; + IntrusiveListLink* tail_ = nullptr; + size_t count_ = 0; +}; + +// Basic iterator for an IntrusiveList. +template +class IntrusiveListIterator + : public std::iterator { + public: + using self_type = IntrusiveListIterator; + + explicit IntrusiveListIterator(IntrusiveListLink* current) + : current_(current) {} + IntrusiveListIterator& operator++(); + self_type operator++(int); + self_type& operator--(); + self_type operator--(int); + bool operator==(const self_type& rhs) const; + bool operator!=(const self_type& rhs) const; + T* operator*() const; + + protected: + IntrusiveListLink* current_; +}; + +// Specialized IntrusiveListBase used for unreferenced naked pointers. +// This very thinly wraps the base type and does no special memory management. +template +class IntrusiveListUnrefBase + : public IntrusiveListBase, + IntrusiveListIterator, + kOffset> { + public: + using IteratorT = IntrusiveListIterator; + using ReverseIteratorT = IntrusiveListIterator; + using base_list = IntrusiveListBase; + + using base_list::clear; + + // Removes all items from the list and calls the given deleter function for + // each of them. The built-in OnDeallocate will not be used. + // Performance: O(n) + void clear(const std::function& deleter); + + private: + using base_list::count_; + using base_list::head_; + using base_list::tail_; +}; + +constexpr size_t kUseDefaultLinkOffset = std::numeric_limits::max(); + +// IntrusiveList for raw pointers with a specified offset. +// Use this if there are multiple links within a type. +// +// Usage: +// struct MyType { +// IntrusiveListLink link_a; +// IntrusiveListLink link_b; +// }; +// IntrusiveList list_a; +// IntrusiveList list_b; +template +class IntrusiveList : public IntrusiveListUnrefBase {}; + +// IntrusiveList for raw pointers. +// Items added to the list will not be owned by the list and must be freed by +// the caller. +// +// Usage: +// struct MyType : public IntrusiveListBase {}; +// IntrusiveList list; +// auto* p = new MyType(); +// list.push_back(p); // p is not retained and won't be freed! +// delete p; +template +class IntrusiveList + : public IntrusiveListUnrefBase {}; + +// -- implementation -- + +namespace impl { + +// Maps an IntrusiveListLink to its containing type T. +template +static T* LinkToT(IntrusiveListLink* link) { + if (link) { + return reinterpret_cast(reinterpret_cast(link) - kOffset); + } else { + return nullptr; + } +} + +// Maps a containing type T to its IntrusiveListLink. +template +static IntrusiveListLink* TToLink(T* value) { + if (value) { + return reinterpret_cast( + reinterpret_cast(value) + kOffset); + } else { + return nullptr; + } +} + +} // namespace impl + +template +IntrusiveListIterator& +IntrusiveListIterator::operator++() { + if (current_) { + current_ = kForward ? current_->next : current_->prev; + } + return *this; +} + +template +IntrusiveListIterator +IntrusiveListIterator::operator++(int) { + self_type tmp(current_); + operator++(); + return tmp; +} + +template +IntrusiveListIterator& +IntrusiveListIterator::operator--() { + if (current_) { + current_ = kForward ? current_->prev : current_->next; + } + return *this; +} + +template +IntrusiveListIterator +IntrusiveListIterator::operator--(int) { + self_type tmp(current_); + operator--(); + return tmp; +} + +template +bool IntrusiveListIterator::operator==( + const self_type& rhs) const { + return rhs.current_ == current_; +} + +template +bool IntrusiveListIterator::operator!=( + const self_type& rhs) const { + return !operator==(rhs); +} + +template +T* IntrusiveListIterator::operator*() const { + return impl::LinkToT(current_); +} + +template +void IntrusiveListUnrefBase::clear( + const std::function& deleter) { + auto* link = head_; + while (link) { + auto* next = link->next; + link->prev = link->next = nullptr; + deleter(impl::LinkToT(link)); + link = next; + } + head_ = tail_ = nullptr; + count_ = 0; +} + +template +void IntrusiveListBase::CheckCorrectness() const { +#if defined(IREE_PARANOID_INTRUSIVE_LIST) + auto* link = head_; + IntrusiveListLink* previous = nullptr; + size_t actual_count = 0; + while (link) { + ++actual_count; + if (!link->prev) { + DCHECK_EQ(link, head_); + } + if (!link->next) { + DCHECK_EQ(link, tail_); + } + DCHECK_EQ(link->prev, previous); + previous = link; + link = link->next; + } + DCHECK_EQ(actual_count, count_); +#endif // IREE_PARANOID_INTRUSIVE_LIST +} + +template +bool IntrusiveListBase::contains( + T* value) const { + if (!value) return false; + // TODO(benvanik): faster way of checking? requires list ptr in link? + auto* needle = impl::TToLink(value); + auto* link = head_; + while (link) { + if (link == needle) { + return true; + } + link = link->next; + } + return false; +} + +template +void IntrusiveListBase::merge_from( + self_type* other_list) { + if (tail_) { + tail_->next = other_list->head_; + } + if (other_list->head_) { + other_list->head_->prev = tail_; + } + if (!head_) { + head_ = other_list->head_; + } + tail_ = other_list->tail_; + + other_list->head_ = nullptr; + other_list->tail_ = nullptr; + + count_ += other_list->count_; + other_list->count_ = 0; +} + +template +void IntrusiveListBase::clear() { + auto* link = head_; + while (link) { + auto* next = link->next; + link->prev = link->next = nullptr; + OnDeallocate(impl::LinkToT(link)); + link = next; + } + head_ = tail_ = nullptr; + count_ = 0; +} + +template +inline T* IntrusiveListBase::next( + T* value) const { + if (!value) { + return nullptr; + } + auto* link = impl::TToLink(value); + return impl::LinkToT(link->next); +} + +template +inline T* IntrusiveListBase::previous( + T* value) const { + if (!value) { + return nullptr; + } + auto* link = impl::TToLink(value); + return impl::LinkToT(link->prev); +} + +template +inline T* IntrusiveListBase::front() + const { + return impl::LinkToT(head_); +} + +template +void IntrusiveListBase::push_front( + T* value) { + DCHECK(value); + auto* link = impl::TToLink(value); + DCHECK(!link->next); + DCHECK(!link->prev); + link->next = head_; + link->prev = nullptr; + head_ = link; + if (link->next) { + link->next->prev = link; + } + if (!tail_) { + tail_ = link; + } + ++count_; + OnAdd(value); + CheckCorrectness(); +} + +template +void IntrusiveListBase::pop_front() { + DCHECK(head_); + auto* link = head_; + if (link) { + head_ = head_->next; + link->next = link->prev = nullptr; + if (head_) { + head_->prev = nullptr; + } + if (link == tail_) { + tail_ = nullptr; + } + --count_; + OnDeallocate(impl::LinkToT(link)); + } + CheckCorrectness(); +} + +template +inline T* IntrusiveListBase::back() + const { + return impl::LinkToT(tail_); +} + +template +void IntrusiveListBase::push_back( + T* value) { + DCHECK(value); + auto* link = impl::TToLink(value); + DCHECK(!link->next); + DCHECK(!link->prev); + link->prev = tail_; + link->next = nullptr; + tail_ = link; + if (link->prev) { + link->prev->next = link; + } + if (!head_) { + head_ = link; + } + ++count_; + OnAdd(value); + CheckCorrectness(); +} + +template +void IntrusiveListBase::pop_back() { + DCHECK(tail_); + auto* link = tail_; + if (link) { + tail_ = tail_->prev; + link->next = link->prev = nullptr; + if (tail_) { + tail_->next = nullptr; + } + if (link == head_) { + head_ = nullptr; + } + --count_; + OnDeallocate(impl::LinkToT(link)); + } + CheckCorrectness(); +} + +template +void IntrusiveListBase::insert( + T* position, T* value) { + DCHECK(value); + auto* link = impl::TToLink(value); + auto* position_link = impl::TToLink(position); + DCHECK(!link->next); + DCHECK(!link->prev); + + if (position_link == head_) { + push_front(value); + } else if (position_link == nullptr) { + push_back(value); + } else { + link->next = position_link; + link->prev = position_link->prev; + position_link->prev->next = link; + position_link->prev = link; + ++count_; + OnAdd(value); + } + CheckCorrectness(); +} + +template +T* IntrusiveListBase::erase(T* value) { + if (!value) { + return nullptr; + } + auto* link = impl::TToLink(value); + if (link->prev) { + DCHECK_NE(link, head_); + link->prev->next = link->next; + } else { + DCHECK_EQ(link, head_); + head_ = link->next; + } + if (link->next) { + DCHECK_NE(link, tail_); + link->next->prev = link->prev; + } else { + DCHECK_EQ(link, tail_); + tail_ = link->prev; + } + auto* next = link->next; + link->next = link->prev = nullptr; + --count_; + OnDeallocate(value); + CheckCorrectness(); + return impl::LinkToT(next); +} + +template +IteratorT IntrusiveListBase::erase( + const IteratorT& it) { + return IteratorT(impl::TToLink(erase(*it))); +} + +template +ReverseIteratorT IntrusiveListBase::erase(const ReverseIteratorT& it) { + return ReverseIteratorT(impl::TToLink(erase(*it))); +} + +template +void IntrusiveListBase::replace( + T* old_value, T* new_value) { + DCHECK(old_value); + DCHECK(new_value); + DCHECK_NE(old_value, new_value); + auto* old_link = impl::TToLink(old_value); + auto* new_link = impl::TToLink(new_value); + new_link->next = old_link->next; + new_link->prev = old_link->prev; + if (new_link->prev) { + new_link->prev->next = new_link; + } else { + head_ = new_link; + } + if (new_link->next) { + new_link->next->prev = new_link; + } else { + tail_ = new_link; + } + old_link->next = old_link->prev = nullptr; + OnAdd(new_value); + OnDeallocate(old_value); + CheckCorrectness(); +} + +template +void IntrusiveListBase::sort( + bool (*compare_fn)(T* a, T* b)) { + if (empty()) { + // Empty list no-op. + return; + } + // Repeatedly run until the list is sorted. + int in_size = 1; + while (true) { + IntrusiveListLink* p = head_; + IntrusiveListLink* q = nullptr; + IntrusiveListLink* e = nullptr; + IntrusiveListLink* tail = nullptr; + head_ = nullptr; + tail_ = nullptr; + // Repeatedly merge sublists. + int merge_count = 0; + while (p) { + ++merge_count; + q = p; + // Determine the size of the first part and find the second. + int p_size = 0; + for (int i = 0; i < in_size; ++i) { + ++p_size; + q = q->next; + if (!q) { + break; + } + } + // Merge the two lists (if we have two). + int q_size = in_size; + while (p_size > 0 || (q_size > 0 && q)) { + if (p_size == 0) { + // p is empty; e must come from q. + e = q; + q = q->next; + --q_size; + } else if (q_size == 0 || !q) { + // q is empty; e must come from p. + e = p; + p = p->next; + --p_size; + } else if (compare_fn(impl::LinkToT(p), + impl::LinkToT(q))) { + // p <= q; e must come from p. + e = p; + p = p->next; + --p_size; + } else { + // q < p; e must come from q. + e = q; + q = q->next; + --q_size; + } + // Append e to the merged list. + if (tail) { + tail->next = e; + } else { + head_ = e; + } + e->prev = tail; + tail = e; + } + p = q; + } + tail->next = nullptr; + if (merge_count <= 1) { + // List is now sorted; stash and return. + tail_ = tail; + CheckCorrectness(); + return; + } + // Run merge again with larger lists. + in_size *= 2; + } +} + +} // namespace iree + +// Specializations: +#include "third_party/mlir_edge/iree/base/intrusive_list_ref_ptr.inc" +#include "third_party/mlir_edge/iree/base/intrusive_list_unique_ptr.inc" + +#endif // THIRD_PARTY_MLIR_EDGE_IREE_BASE_INTRUSIVE_LIST_H_ diff --git a/base/intrusive_list_ref_ptr.inc b/base/intrusive_list_ref_ptr.inc new file mode 100644 index 000000000000..cb0228f63f21 --- /dev/null +++ b/base/intrusive_list_ref_ptr.inc @@ -0,0 +1,160 @@ +// IWYU pragma: private, include "third_party/mlir_edge/iree/base/intrusive_list.h" + +#ifndef THIRD_PARTY_MLIR_EDGE_IREE_BASE_INTRUSIVE_LIST_REF_PTR_H_ +#define THIRD_PARTY_MLIR_EDGE_IREE_BASE_INTRUSIVE_LIST_REF_PTR_H_ + +#include +#include + +#include "third_party/mlir_edge/iree/base/intrusive_list.h" +#include "third_party/mlir_edge/iree/base/ref_ptr.h" + +namespace iree { + +// Iterator for an IntrusiveList specialized to ref_ptr. +template +class IntrusiveListRefPtrIterator + : public std::iterator { + public: + using self_type = IntrusiveListRefPtrIterator; + + explicit IntrusiveListRefPtrIterator(IntrusiveListLink* current) + : current_(current) {} + self_type& operator++() { + if (current_) { + current_ = kForward ? current_->next : current_->prev; + } + return *this; + } + self_type operator++(int) { + self_type tmp(current_); + operator++(); + return tmp; + } + self_type& operator--() { + if (current_) { + current_ = kForward ? current_->prev : current_->next; + } + return *this; + } + self_type operator--(int) { + self_type tmp(current_); + operator--(); + return tmp; + } + bool operator==(const self_type& rhs) const { + return rhs.current_ == current_; + } + bool operator!=(const self_type& rhs) const { return !operator==(rhs); } + ref_ptr operator*() const { + return add_ref(impl::LinkToT(current_)); + } + + protected: + IntrusiveListLink* current_; +}; + +// Specialized IntrusiveListBase for ref_ptr types. +// This makes the list methods accept/return ref_ptrs and iterate with +// a ref_ptr iterator. +template +class IntrusiveListRefPtrBase + : private IntrusiveListBase< + T, IntrusiveListRefPtrIterator, + IntrusiveListRefPtrIterator, kOffset> { + public: + using IteratorT = IntrusiveListRefPtrIterator; + using ReverseIteratorT = IntrusiveListRefPtrIterator; + using base_list = IntrusiveListBase; + + IntrusiveListRefPtrBase() = default; + + using base_list::empty; + using base_list::size; + + using base_list::contains; + bool contains(const ref_ptr& value) const { + return base_list::contains(value.get()); + } + + using base_list::clear; + + using base_list::begin; + using base_list::end; + using base_list::rbegin; + using base_list::rend; + + inline ref_ptr next(const ref_ptr& value) const { + return add_ref(base_list::next(value.get())); + } + inline ref_ptr next(T* value) const { + return add_ref(base_list::next(value)); + } + + inline ref_ptr previous(const ref_ptr& value) const { + return add_ref(base_list::previous(value.get())); + } + inline ref_ptr previous(T* value) const { + return add_ref(base_list::previous(value)); + } + + // Performance: O(1) + inline ref_ptr front() const { + return add_ref(impl::LinkToT(head_)); + } + + void push_front(const ref_ptr& value) { + base_list::push_front(value.get()); + } + + using base_list::pop_front; + + // Performance: O(1) + inline ref_ptr back() const { + return add_ref(impl::LinkToT(tail_)); + } + + void push_back(const ref_ptr& value) { base_list::push_back(value.get()); } + + using base_list::pop_back; + + void insert(const IteratorT& it, const ref_ptr& value) { + base_list::insert(it, value.get()); + } + + using base_list::erase; + + ref_ptr erase(const ref_ptr& value) { + return add_ref(base_list::erase(value.get())); + } + + void replace(const ref_ptr& old_value, const ref_ptr& new_value) { + base_list::replace(old_value.get(), new_value.get()); + } + void replace(T* old_value, const ref_ptr& new_value) { + base_list::replace(old_value, new_value.get()); + } + + using base_list::sort; + + private: + void OnAdd(T* value) override { value->AddReference(); } + void OnRemove(T* value) override { value->ReleaseReference(); } + void OnDeallocate(T* value) override { value->ReleaseReference(); } + + using base_list::count_; + using base_list::head_; + using base_list::tail_; +}; + +template +class IntrusiveList, kOffset> + : public IntrusiveListRefPtrBase {}; + +template +class IntrusiveList, kUseDefaultLinkOffset> + : public IntrusiveListRefPtrBase {}; + +} // namespace iree + +#endif // THIRD_PARTY_MLIR_EDGE_IREE_BASE_INTRUSIVE_LIST_REF_PTR_H_ diff --git a/base/intrusive_list_unique_ptr.inc b/base/intrusive_list_unique_ptr.inc new file mode 100644 index 000000000000..e7f6eebd8e7c --- /dev/null +++ b/base/intrusive_list_unique_ptr.inc @@ -0,0 +1,126 @@ +// IWYU pragma: private, include "third_party/mlir_edge/iree/base/intrusive_list.h" + +#ifndef THIRD_PARTY_MLIR_EDGE_IREE_BASE_INTRUSIVE_LIST_UNIQUE_PTR_H_ +#define THIRD_PARTY_MLIR_EDGE_IREE_BASE_INTRUSIVE_LIST_UNIQUE_PTR_H_ + +#include +#include + +#include "third_party/mlir_edge/iree/base/intrusive_list.h" +#include "third_party/mlir_edge/iree/base/logging.h" + +namespace iree { + +// Specialized IntrusiveListBase for std::unique_ptr types. +// This makes the list methods accept std::unique_ptrs and contains a special +// take() method that takes ownership of a list item. +template +class IntrusiveListUniquePtrBase + : private IntrusiveListBase, + IntrusiveListIterator, + kOffset> { + public: + using IteratorT = IntrusiveListIterator; + using ReverseIteratorT = IntrusiveListIterator; + using base_list = IntrusiveListBase; + + IntrusiveListUniquePtrBase() = default; + + using base_list::empty; + using base_list::size; + + using base_list::contains; + + using base_list::clear; + + using base_list::begin; + using base_list::end; + using base_list::rbegin; + using base_list::rend; + + using base_list::next; + + using base_list::previous; + + using base_list::front; + + void push_front(std::unique_ptr value) { + base_list::push_front(value.release()); + } + + using base_list::pop_front; + + using base_list::back; + + void push_back(std::unique_ptr value) { + base_list::push_back(value.release()); + } + + using base_list::pop_back; + + void insert(const IteratorT& it, std::unique_ptr value) { + base_list::insert(it, value.release()); + } + + using base_list::erase; + + // Removes an item from the list at the given iterator and transfers ownership + // to the caller. + // Performance: O(1) + std::unique_ptr take(IteratorT& it) { // NOLINT(runtime/references) + return take(*it); + } + + // Removes an item from the list and transfers ownership to the caller. + // Performance: O(1) + std::unique_ptr take(T* value) { + if (!value) { + return {nullptr}; + } + auto* link = impl::TToLink(value); + if (link->prev) { + DCHECK_NE(link, head_); + link->prev->next = link->next; + } else { + DCHECK_EQ(link, head_); + head_ = link->next; + } + if (link->next) { + DCHECK_NE(link, tail_); + link->next->prev = link->prev; + } else { + DCHECK_EQ(link, tail_); + tail_ = link->prev; + } + link->next = link->prev = nullptr; + --count_; + base_list::OnRemove(value); + base_list::CheckCorrectness(); + return std::unique_ptr(value); + } + + void replace(T* old_value, std::unique_ptr new_value) { + base_list::replace(old_value, new_value.release()); + } + + using base_list::sort; + + private: + void OnDeallocate(T* value) override { delete value; } + + using base_list::count_; + using base_list::head_; + using base_list::tail_; +}; + +template +class IntrusiveList, kOffset> + : public IntrusiveListUniquePtrBase {}; + +template +class IntrusiveList, kUseDefaultLinkOffset> + : public IntrusiveListUniquePtrBase {}; + +} // namespace iree + +#endif // THIRD_PARTY_MLIR_EDGE_IREE_BASE_INTRUSIVE_LIST_UNIQUE_PTR_H_ diff --git a/base/logging.h b/base/logging.h new file mode 100644 index 000000000000..c9697c527cf7 --- /dev/null +++ b/base/logging.h @@ -0,0 +1,63 @@ +// Copyright 2019 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#ifndef THIRD_PARTY_MLIR_EDGE_IREE_BASE_LOGGING_H_ +#define THIRD_PARTY_MLIR_EDGE_IREE_BASE_LOGGING_H_ + +// Logging macros live in their own file so that we can use external versions +// as required. +// +// LOG(severity) << ...; +// Logs a message at the given severity. +// Severity: +// INFO Logs information text. +// WARNING Logs a warning. +// ERROR Logs an error. +// FATAL Logs an error and exit(1). +// +// VLOG(level) << ...; +// Logs a verbose message at the given verbosity level. +// +// DVLOG(level) << ...; +// Behaves like `VLOG` in debug mode (i.e. `#ifndef NDEBUG`). +// Otherwise, it compiles away and does nothing. +// +// CHECK(condition) << ...; +// Runtime asserts that the given condition is true even in release builds. +// It's recommended that DCHECK is used instead as too many CHECKs +// can impact performance. +// +// CHECK_EQ|NE|LT|GT|LE|GE(val1, val2) << ...; +// Runtime assert the specified operation with the given values. +// +// DCHECK(condition) << ...; +// Runtime asserts that the given condition is true only in non-opt builds. +// +// DCHECK_EQ|NE|LT|GT|LE|GE(val1, val2) << ...; +// Runtime assert the specified operation with the given values in non-opt +// builds. +// +// QCHECK(condition) << ...; +// QCHECK_EQ|NE|LT|GT|LE|GE(val1, val2) << ...; +// These behave like `CHECK` but do not print a full stack trace. +// They are useful when problems are definitely unrelated to program flow, +// e.g. when validating user input. + +#ifdef IREE_CONFIG_GOOGLE_INTERNAL +#include "third_party/mlir_edge/iree/base/google/logging_google.h" +#else +#include "third_party/mlir_edge/iree/base/internal/logging.h" +#endif // IREE_CONFIG_GOOGLE_INTERNAL + +#endif // THIRD_PARTY_MLIR_EDGE_IREE_BASE_LOGGING_H_ diff --git a/base/memory.h b/base/memory.h new file mode 100644 index 000000000000..1f7ce9503fb8 --- /dev/null +++ b/base/memory.h @@ -0,0 +1,152 @@ +// Copyright 2019 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#ifndef THIRD_PARTY_MLIR_EDGE_IREE_BASE_MEMORY_H_ +#define THIRD_PARTY_MLIR_EDGE_IREE_BASE_MEMORY_H_ + +#include +#include +#include + +#include "third_party/absl/types/span.h" + +namespace iree { + +// reinterpret_cast for Spans, preserving byte size. +template +constexpr absl::Span ReinterpretSpan(absl::Span value) { + return absl::MakeSpan(reinterpret_cast(value.data()), + (value.size() * sizeof(U)) / sizeof(T)); +} +template +constexpr absl::Span ReinterpretSpan(absl::Span value) { + return absl::MakeSpan(reinterpret_cast(value.data()), + (value.size() * sizeof(U)) / sizeof(T)); +} + +// Cast a span of std::unique_ptr to a span of raw pointers. +template +inline absl::Span RawPtrSpan(absl::Span> value) { + return absl::MakeSpan(reinterpret_cast(value.data()), value.size()); +} + +// A helper wrapper that moves the wrapped object on copy. +// This is particularly handy for capturing unique_ptrs in lambdas. +// Usage example: +// +// std::unique_ptr foo_ptr(new Foo()); +// move_on_copy> foo(std::move(foo_ptr)); +// auto some_lambda = [bar]() { ... } +// +template +struct move_on_copy { + explicit move_on_copy(T&& t) : value(std::move(t)) {} + + move_on_copy(move_on_copy const& other) : value(std::move(other.value)) {} + + move_on_copy(move_on_copy&& other) : value(std::move(other.value)) {} + + move_on_copy& operator=(move_on_copy const& other) { + value = std::move(other.value); + return *this; + } + + move_on_copy& operator=(move_on_copy&& other) { + value = std::move(other.value); + return *this; + } + + mutable T value; +}; + +// Utility to aid in moving ref_ptr's into closures. +// +// Usage: +// auto baton = MoveToLambda(my_ref); +// DoSomething([baton] () { baton.value; }); +#define MoveToLambda(p) ::iree::move_on_copy(std::move(p)) + +// TODO(benvanik): replace with an absl version when it exists. +// A move-only RAII object that calls a stored cleanup functor when +// destroyed. Cleanup is the return type of iree::MakeCleanup(F). +template +class Cleanup { + public: + Cleanup() : released_(true), f_() {} + template + explicit Cleanup(G&& f) // NOLINT + : f_(std::forward(f)) {} // NOLINT(build/c++11) + Cleanup(Cleanup&& src) // NOLINT + : released_(src.is_released()), f_(src.release()) {} + + // Implicitly move-constructible from any compatible Cleanup. + // The source will be released as if src.release() were called. + // A moved-from Cleanup can be safely destroyed or reassigned. + template + Cleanup(Cleanup&& src) // NOLINT + : released_(src.is_released()), f_(src.release()) {} + + // Assignment to a Cleanup object behaves like destroying it + // and making a new one in its place, analogous to unique_ptr + // semantics. + Cleanup& operator=(Cleanup&& src) { // NOLINT + if (!released_) std::move(f_)(); + released_ = src.released_; + f_ = src.release(); + return *this; + } + + ~Cleanup() { + if (!released_) std::move(f_)(); + } + + // Releases the cleanup function instead of running it. + // Hint: use c.release()() to run early. + F release() { + released_ = true; + return std::move(f_); + } + + bool is_released() const { return released_; } + + private: + static_assert(!std::is_reference::value, "F must not be a reference"); + + bool released_ = false; + F f_; +}; + +// MakeCleanup(f) returns an RAII cleanup object that calls 'f' in its +// destructor. The easiest way to use MakeCleanup is with a lambda argument, +// capturing the return value in an 'auto' local variable. Most users will not +// need more sophisticated syntax than that. +// +// Example: +// void func() { +// FILE* fp = fopen("data.txt", "r"); +// if (fp == nullptr) return; +// auto fp_cleaner = MakeCleanup([fp] { fclose(fp); }); +// // No matter what, fclose(fp) will happen. +// } +// +// You can call 'release()' on a Cleanup object to cancel the cleanup. +template ::type> +ABSL_MUST_USE_RESULT Cleanup MakeCleanup(F&& f) { + return Cleanup(std::forward(f)); +} + +} // namespace iree + +#endif // THIRD_PARTY_MLIR_EDGE_IREE_BASE_MEMORY_H_ diff --git a/base/ref_ptr.h b/base/ref_ptr.h new file mode 100644 index 000000000000..f4435b91c828 --- /dev/null +++ b/base/ref_ptr.h @@ -0,0 +1,364 @@ +// Copyright 2019 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#ifndef THIRD_PARTY_MLIR_EDGE_IREE_BASE_REF_PTR_H_ +#define THIRD_PARTY_MLIR_EDGE_IREE_BASE_REF_PTR_H_ + +#include +#include +#include +#include + +#include "third_party/absl/base/attributes.h" +#include "third_party/mlir_edge/iree/base/logging.h" + +namespace iree { + +// Use this to get really verbose refptr logging: +// #define IREE_VERBOSE_REF_PTR + +template +class ref_ptr; + +// Allocates a new ref_ptr type. +// Like make_unique, but for ref_ptr. +// +// Usage: +// ref_ptr p = make_ref(1, 2, 3); +template +ref_ptr make_ref(Args&&... args) { + return ref_ptr(new T(std::forward(args)...)); +} + +// Assigns a raw pointer to a ref_ptr without adding a reference. +// +// Usage: +// ref_ptr p = assign_ref(new MyType()); +template +inline ref_ptr assign_ref(T* value) { + return ref_ptr(value); +} + +// Adds a reference to the given raw pointer. +// +// Usage: +// MyType* raw_ptr = AcquirePointerFromSomewhere(); +// ref_ptr p = add_ref(raw_ptr); +template +inline ref_ptr add_ref(T* value) { + if (value) ref_ptr_add_ref(value); + return ref_ptr(value); +} + +// Adds a reference to the given ref_ptr. +// +// Usage: +// ref_ptr a = make_ref(); +// ref_ptr p = add_ref(a); +template +inline ref_ptr add_ref(const ref_ptr& value) { + if (value.get()) ref_ptr_add_ref(value.get()); + return ref_ptr(value.get()); +} + +// Reference counted pointer container. +// This is modeled on boost::instrusive_ptr in that it requires no +// extra storage over the pointer type and should compile to almost +// no additional code. It also allows us to round-trip object pointers +// through regular pointers, which is critical when having to round-trip +// them through JNI/etc where we can't use things like unique_ptr/shared_ptr. +// +// ref_ptr p1(new Foo()); // ref count 1 +// ref_ptr p2(p1); // ref count 2 +// p1.reset(); // ref count 1 +// p2.reset(); // ref count 0, deleted +// +// When round-tripping the pointer through external APIs, use release(): +// ref_ptr p1(new Foo()); // ref count 1 +// Foo* raw_p = p1.release(); // ref count 1 +// // pass to API +// ref_ptr p2(raw_p); // ref count 1 (don't add ref) +// p2.reset(); // ref count 0, deleted +// +// See the boost intrusive_ptr docs for details of behavior: +// http://www.boost.org/doc/libs/1_55_0/libs/smart_ptr/intrusive_ptr.html +// +// ref_ptr manages the target objects in a thread-safe way, though you'll want +// to take care with objects that may have pinned threads for deallocation. If +// you release the last reference to an object on a thread other than what it +// was expecting you're gonna have a bad time. +// +// Compatible only with types that subclass RefObject or implement the following +// methods: +// ref_ptr_add_ref +// ref_ptr_release_ref +template +class ref_ptr { + private: + typedef ref_ptr this_type; + typedef T* this_type::*unspecified_bool_type; + + public: + // Initializes with nullptr. + ABSL_ATTRIBUTE_ALWAYS_INLINE ref_ptr() noexcept = default; + + // Initializes with nullptr so that there is no way to create an + // uninitialized ref_ptr. + ABSL_ATTRIBUTE_ALWAYS_INLINE ref_ptr(std::nullptr_t) noexcept {} // NOLINT + + // Initializes the pointer to the given value. + // The value will not have its reference count incremented (as it is with + // unique_ptr). Use Retain to add to the reference count. + ABSL_ATTRIBUTE_ALWAYS_INLINE explicit ref_ptr(T* p) noexcept : px_(p) {} + + // Decrements the reference count of the owned pointer. + ABSL_ATTRIBUTE_ALWAYS_INLINE ~ref_ptr() noexcept { + if (px_) ref_ptr_release_ref(px_); + } + + // No implicit ref_ptr copying allowed; use add_ref instead. + ref_ptr(const ref_ptr&) noexcept = delete; + ref_ptr& operator=(const ref_ptr&) noexcept = delete; + + // Move support to transfer ownership from one ref_ptr to another. + ref_ptr(ref_ptr&& rhs) noexcept : px_(rhs.release()) {} + ref_ptr& operator=(ref_ptr&& rhs) noexcept { + if (px_ != rhs.px_) { + if (px_) ref_ptr_release_ref(px_); + px_ = rhs.release(); + } + return *this; + } + + // Move support from another compatible type. + template + ref_ptr(ref_ptr&& rhs) noexcept : px_(rhs.release()) {} // NOLINT + template + ref_ptr& operator=(ref_ptr&& rhs) noexcept { + if (px_ != rhs.get()) { + if (px_) ref_ptr_release_ref(px_); + px_ = rhs.release(); + } + return *this; + } + + // Resets the object to nullptr and decrements the reference count, possibly + // deleting it. + void reset() noexcept { + if (px_) { + ref_ptr_release_ref(px_); + px_ = nullptr; + } + } + + // Releases a pointer. + // Returns the current pointer held by this object without having + // its reference count decremented and resets the ref_ptr to empty. + // Returns nullptr if the ref_ptr holds no value. + // To re-wrap in a ref_ptr use either ref_ptr(value) or assign(). + ABSL_ATTRIBUTE_ALWAYS_INLINE T* release() noexcept { + T* p = px_; + px_ = nullptr; + return p; + } + + // Assigns a pointer. + // The pointer will be accepted by the ref_ptr and its reference count will + // not be incremented. + ABSL_ATTRIBUTE_ALWAYS_INLINE void assign(T* value) noexcept { + reset(); + px_ = value; + } + + // Gets the pointer referenced by this instance. + // operator* and operator-> will assert() if there is no current object. + constexpr T* get() const noexcept { return px_; } + constexpr T& operator*() const noexcept { return *px_; } + constexpr T* operator->() const noexcept { return px_; } + + // Support boolean expression evaluation ala unique_ptr/shared_ptr: + // https://en.cppreference.com/w/cpp/memory/shared_ptr/operator_bool + constexpr operator unspecified_bool_type() const noexcept { + return px_ ? &this_type::px_ : nullptr; + } + // Supports unary expression evaluation. + constexpr bool operator!() const noexcept { return !px_; } + + // Swap support. + void swap(ref_ptr& rhs) { std::swap(px_, rhs.px_); } + + private: + T* px_ = nullptr; +}; + +// Base class for reference counted objects. +// Reference counted objects should be used with the ref_ptr pointer type. +// As reference counting can be tricky always prefer to use unique_ptr and +// avoid this type. Only use this when unique_ptr is not possible, such as +// when round-tripping objects through marshaling boundaries (v8/Java) or +// any objects that may have their lifetime tied to a garbage collected +// object. +// +// Subclasses should protect their dtor so that reference counting must +// be used. +// +// This is designed to avoid the need for extra vtable space or for adding +// methods to the vtable of subclasses. This differs from the boost Pointable +// version of this object. +// Inspiration for this comes from Peter Weinert's Dr. Dobb's article: +// http://www.drdobbs.com/cpp/a-base-class-for-intrusively-reference-c/229218807 +// +// RefObjects are thread safe and may be used with ref_ptrs from multiple +// threads. +// +// Subclasses may implement a custom Delete operator to handle their +// deallocation. It should be thread safe as it may be called from any thread. +// +// Usage: +// class MyRefObject : public RefObject { +// public: +// MyRefObject() = default; +// // Optional; can be used to return to pool/etc - must be public: +// static void Delete(MyRefObject* ptr) { +// ::operator delete(ptr); +// } +// }; +template +class RefObject { + static_assert(!std::is_array::value, "T must not be an array"); + + // value is true if a static Delete(T*) function is present. + struct has_custom_deleter { + template + static auto Test(C* p) -> decltype(C::Delete(nullptr), std::true_type()); + template + static std::false_type Test(...); + static constexpr bool value = + std::is_same(nullptr))>::value; + }; + + template + struct delete_thunk { + static void Delete(V* p) { + auto ref_obj = static_cast*>(p); + int previous_count = ref_obj->counter_.fetch_sub(1); +#ifdef IREE_VERBOSE_REF_PTR + LOG(INFO) << "ro-- " << typeid(V).name() << " " << p << " now " + << previous_count - 1 + << (previous_count == 1 ? " DEAD (CUSTOM)" : ""); +#endif // IREE_VERBOSE_REF_PTR + if (previous_count == 1) { + // We delete type T pointer here to avoid the need for a virtual dtor. + V::Delete(p); + } + } + }; + + template + struct delete_thunk { + static void Delete(V* p) { + auto ref_obj = static_cast*>(p); + int previous_count = ref_obj->counter_.fetch_sub(1); +#ifdef IREE_VERBOSE_REF_PTR + LOG(INFO) << "ro-- " << typeid(V).name() << " " << p << " now " + << previous_count - 1 << (previous_count == 1 ? " DEAD" : ""); +#endif // IREE_VERBOSE_REF_PTR + if (previous_count == 1) { + // We delete type T pointer here to avoid the need for a virtual dtor. + delete p; + } + } + }; + + public: + // Adds a reference; used by ref_ptr. + friend void ref_ptr_add_ref(T* p) { + auto ref_obj = static_cast(p); + ++ref_obj->counter_; + +#ifdef IREE_VERBOSE_REF_PTR + LOG(INFO) << "ro++ " << typeid(T).name() << " " << p << " now " + << ref_obj->counter_; +#endif // IREE_VERBOSE_REF_PTR + } + + // Releases a reference, potentially deleting the object; used by ref_ptr. + friend void ref_ptr_release_ref(T* p) { + delete_thunk::Delete(p); + } + + // Adds a reference. + // ref_ptr should be used instead of this in most cases. This is required + // for when interoperating with marshaling APIs. + void AddReference() { ref_ptr_add_ref(static_cast(this)); } + + // Releases a reference, potentially deleting the object. + // ref_ptr should be used instead of this in most cases. This is required + // for when interoperating with marshaling APIs. + void ReleaseReference() { ref_ptr_release_ref(static_cast(this)); } + + protected: + RefObject() { ref_ptr_add_ref(static_cast(this)); } + RefObject(const RefObject&) = default; + RefObject& operator=(const RefObject&) { return *this; } + + std::atomic counter_{0}; +}; + +// Various comparison operator overloads. + +template +inline bool operator==(ref_ptr const& a, ref_ptr const& b) { + return a.get() == b.get(); +} + +template +inline bool operator!=(ref_ptr const& a, ref_ptr const& b) { + return a.get() != b.get(); +} + +template +inline bool operator==(ref_ptr const& a, U* b) { + return a.get() == b; +} + +template +inline bool operator!=(ref_ptr const& a, U* b) { + return a.get() != b; +} + +template +inline bool operator==(T* a, ref_ptr const& b) { + return a == b.get(); +} + +template +inline bool operator!=(T* a, ref_ptr const& b) { + return a != b.get(); +} + +template +inline bool operator<(ref_ptr const& a, ref_ptr const& b) { + return a.get() < b.get(); +} + +// Swaps the pointers of two ref_ptrs. +template +void swap(ref_ptr& lhs, ref_ptr& rhs) { + lhs.swap(rhs); +} + +} // namespace iree + +#endif // THIRD_PARTY_MLIR_EDGE_IREE_BASE_REF_PTR_H_ diff --git a/base/shape.cc b/base/shape.cc new file mode 100644 index 000000000000..33b32391f94d --- /dev/null +++ b/base/shape.cc @@ -0,0 +1,100 @@ +// Copyright 2019 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "third_party/mlir_edge/iree/base/shape.h" + +#include + +#include "third_party/absl/strings/str_cat.h" +#include "third_party/absl/strings/str_join.h" +#include "third_party/absl/types/source_location.h" +#include "third_party/mlir_edge/iree/base/status.h" + +namespace iree { + +Shape::Shape(const int* values, int size) : rank_(size) { + QCHECK_LE(size, kMaxRank) + << "Max rank of " << kMaxRank << ", shape has " << size; + std::memcpy(value_, values, size * sizeof(int)); +} + +std::string Shape::DebugString() const { + return absl::StrCat("[", absl::StrJoin(subspan(), ","), "]"); +} + +absl::Span Shape::subspan(size_type pos, size_type len) const { + if (len == npos) { + len = rank_ - pos; + } + return absl::MakeConstSpan(&value_[pos], len); +} + +void Shape::push_back(int dim) { + DCHECK_LE(rank_ + 1, kMaxRank); + value_[rank_++] = dim; +} + +void Shape::insert(iterator pos, int dim) { + int axis = static_cast(pos - value_); + DCHECK_GE(axis, 0); + DCHECK_LE(axis, rank_); + DCHECK_LE(rank_ + 1, kMaxRank); + ++rank_; + for (int i = rank_ - 1; i > axis; --i) { + value_[i] = value_[i - 1]; + } + value_[axis] = dim; +} + +void Shape::erase(iterator pos) { + int axis = static_cast(pos - value_); + DCHECK_GE(axis, 0); + DCHECK_LE(axis, rank_); + for (int i = axis; i < rank_ - 1; ++i) { + value_[i] = value_[i + 1]; + } + --rank_; +} + +int Shape::element_count() const { + size_t element_count = 1; + for (int i = 0; i < rank_; ++i) { + int dim = value_[i]; + if (dim == -1) { + return 0; + } + element_count *= dim; + } + return element_count; +} + +StatusOr Shape::ResolveAxis(int axis) const { + if (rank_ == 0 && (axis == -1 || axis == 0)) { + // Scalar axes resolves to 0. + return 0; + } + + int new_axis = axis; + if (new_axis < 0) { + new_axis += rank_; + } + if (new_axis < 0 || new_axis >= rank_) { + return InvalidArgumentErrorBuilder(ABSL_LOC) + << "Axis " << new_axis << " (orig " << axis + << ") out of bounds of rank " << rank_; + } + return new_axis; +} + +} // namespace iree diff --git a/base/shape.h b/base/shape.h new file mode 100644 index 000000000000..df75b0e8b766 --- /dev/null +++ b/base/shape.h @@ -0,0 +1,156 @@ +// Copyright 2019 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#ifndef THIRD_PARTY_MLIR_EDGE_IREE_BASE_SHAPE_H_ +#define THIRD_PARTY_MLIR_EDGE_IREE_BASE_SHAPE_H_ + +#include +#include +#include +#include +#include +#include +#include + +#include "third_party/absl/meta/type_traits.h" +#include "third_party/absl/types/span.h" +#include "third_party/mlir_edge/iree/base/logging.h" +#include "third_party/mlir_edge/iree/base/status.h" + +namespace iree { + +// For simplicity we limit our shapes to a max of rank-N (shape.size() == N) as +// this prevents dynamic allocations and rarely are there greater ranks. +constexpr int kMaxRank = 5; + +// Represent indices and lengths of tensors. +using Index = std::array; +using Length = std::array; + +// Represents the number of elements in multiple dimensions. +// Can be rank-0 (scalar) to rank-kMaxRank. Tries to match the API of +// std::vector and can be converted to a Span via subspan(). +// +// https://www.tensorflow.org/guide/tensors#shape +class Shape { + public: + using size_type = int; + static constexpr size_type npos = ~(size_type(0)); // NOLINT + using iterator = int*; + using const_iterator = const int*; + + Shape() = default; + Shape(const int* values, int size); + Shape(std::initializer_list values) + : Shape(values.begin(), values.size()) {} + explicit Shape(absl::Span values) + : Shape(values.data(), values.size()) {} + + template + using EnableIfForwardIterator = absl::enable_if_t::iterator_category, + std::forward_iterator_tag>::value>; + template * = nullptr> + Shape(Iterator first, Iterator last) { + rank_ = std::distance(first, last); + QCHECK_LE(rank_, kMaxRank); + for (int i = 0; first != last; ++i, static_cast(++first)) { + value_[i] = *first; + } + } + + // Returns a string representation of the given shape. + std::string DebugString() const; + + // Size (aka 'rank') of the shape, counting the number of dimensions. + constexpr size_type size() const noexcept { return rank_; } + + // Whether the shape is rank-0 (scalar). + constexpr bool empty() const noexcept { return rank_ == 0; } + + // Returns the total elements in the tensor shape. + // Returns 0 if the tensor shape is not complete and 1 if the shape is a + // scalar value. + int element_count() const; + + // Resolves an axis in [-R,R) to the real axis value and verifies the range. + StatusOr ResolveAxis(int axis) const; + + // Compares two shapes for equality. + inline static bool Equal(const Shape& a, const Shape& b) { + return a.rank_ == b.rank_ && + std::memcmp(a.value_, b.value_, a.rank_ * sizeof(value_[0])) == 0; + } + + int& operator[](size_type i) noexcept { + DCHECK_GE(i, 0); + DCHECK_LT(i, rank_); + return value_[i]; + } + + const int& operator[](size_type i) const noexcept { + DCHECK_GE(i, 0); + DCHECK_LT(i, rank_); + return value_[i]; + } + + int front() const noexcept { + DCHECK_GE(rank_, 1); + return value_[0]; + } + + int back() const noexcept { + DCHECK_GE(rank_, 1); + return value_[rank_ - 1]; + } + + constexpr iterator begin() const noexcept { + return const_cast(&value_[0]); + } + constexpr iterator end() const noexcept { + return const_cast(&value_[rank_]); + } + constexpr const_iterator cbegin() const noexcept { return &value_[0]; } + constexpr const_iterator cend() const noexcept { return &value_[rank_]; } + + absl::Span subspan(size_type pos = 0, size_type len = npos) const; + absl::Span data() const { return subspan(); } + + void push_back(int dim); + + void insert(iterator pos, int dim); + + void erase(iterator pos); + + void clear() { rank_ = 0; } + + private: + size_type rank_ = 0; + int value_[kMaxRank]; +}; + +inline bool operator==(const Shape& a, const Shape& b) { + return Shape::Equal(a, b); +} + +inline bool operator!=(const Shape& a, const Shape& b) { return !(a == b); } + +inline std::ostream& operator<<(std::ostream& stream, const Shape& shape) { + stream << shape.DebugString(); + return stream; +} + +} // namespace iree + +#endif // THIRD_PARTY_MLIR_EDGE_IREE_BASE_SHAPE_H_ diff --git a/base/status.h b/base/status.h new file mode 100644 index 000000000000..2a3e205a58fc --- /dev/null +++ b/base/status.h @@ -0,0 +1,29 @@ +// Copyright 2019 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#ifndef THIRD_PARTY_MLIR_EDGE_IREE_BASE_STATUS_H_ +#define THIRD_PARTY_MLIR_EDGE_IREE_BASE_STATUS_H_ + +#ifdef IREE_CONFIG_GOOGLE_INTERNAL +#include "third_party/mlir_edge/iree/base/google/status_google.h" +#else +#include "third_party/mlir_edge/iree/base/internal/status.h" +#include "third_party/mlir_edge/iree/base/internal/status_builder.h" +#include "third_party/mlir_edge/iree/base/internal/status_errno.h" +#include "third_party/mlir_edge/iree/base/internal/status_errors.h" +#include "third_party/mlir_edge/iree/base/internal/status_macros.h" +#include "third_party/mlir_edge/iree/base/internal/statusor.h" +#endif // IREE_CONFIG_GOOGLE_INTERNAL + +#endif // THIRD_PARTY_MLIR_EDGE_IREE_BASE_STATUS_H_ diff --git a/base/time.h b/base/time.h new file mode 100644 index 000000000000..9499161144c0 --- /dev/null +++ b/base/time.h @@ -0,0 +1,37 @@ +// Copyright 2019 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#ifndef THIRD_PARTY_MLIR_EDGE_IREE_BASE_TIME_H_ +#define THIRD_PARTY_MLIR_EDGE_IREE_BASE_TIME_H_ + +#include "third_party/absl/time/clock.h" +#include "third_party/absl/time/time.h" + +namespace iree { + +// Converts a relative timeout duration to an absolute deadline time. +// This handles the special cases of absl::ZeroDuration and +// absl::InfiniteDuration to avoid extraneous time queries. +inline absl::Time RelativeTimeoutToDeadline(absl::Duration timeout) { + if (timeout == absl::ZeroDuration()) { + return absl::InfinitePast(); + } else if (timeout == absl::InfiniteDuration()) { + return absl::InfiniteFuture(); + } + return absl::Now() + timeout; +} + +} // namespace iree + +#endif // THIRD_PARTY_MLIR_EDGE_IREE_BASE_TIME_H_ diff --git a/base/tracing.cc b/base/tracing.cc new file mode 100644 index 000000000000..33a1e185fd2c --- /dev/null +++ b/base/tracing.cc @@ -0,0 +1,181 @@ +// Copyright 2019 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "third_party/mlir_edge/iree/base/tracing.h" + +#include // NOLINT: Fiber doesn't work during startup on Android. + +#include "third_party/absl/base/attributes.h" +#include "third_party/absl/base/const_init.h" +#include "third_party/absl/base/thread_annotations.h" +#include "third_party/absl/flags/flag.h" +#include "third_party/absl/strings/str_cat.h" +#include "third_party/absl/synchronization/mutex.h" +#include "third_party/absl/time/clock.h" +#include "third_party/absl/time/time.h" +#include "third_party/mlir_edge/iree/base/file_io.h" +#include "third_party/mlir_edge/iree/base/init.h" +#include "third_party/mlir_edge/iree/base/logging.h" +#include "third_party/mlir_edge/iree/base/status.h" + +ABSL_FLAG(int32_t, iree_trace_file_period, 5, + "Seconds between automatic flushing of WTF trace files. 0 to " + "disable auto-flush."); +ABSL_FLAG(std::string, iree_trace_file, "/dev/null", + "wtf-trace file to save if --define=GLOBAL_WTF_ENABLE=1 was used " + "when building."); + +namespace iree { +namespace { + +// Guards global WTF state (like the flush fiber and IO). +ABSL_CONST_INIT absl::Mutex global_tracing_mutex(absl::kConstInit); + +// True when tracing has been enabled and initialized. +bool global_tracing_initialized ABSL_GUARDED_BY(global_tracing_mutex) = false; + +// If there is an existing file at the given path back it up by moving it aside. +// Only kMaxBackups will be kept to avoid unbounded growth. +void RollTraceFiles(const std::string& path) { + std::string path_stem = file_io::JoinFilePaths( + file_io::FileDirectoryName(path), file_io::FileStem(path)); + const int kMaxBackups = 5; + for (int i = kMaxBackups; i >= 0; i--) { + std::string source_name; + if (i > 0) { + source_name = absl::StrCat(path_stem, ".", i, ".wtf-trace"); + } else { + source_name = path; + } + if (!file_io::FileExists(source_name).ok()) { + continue; + } + + Status status; + if (i == kMaxBackups) { + status = file_io::DeleteFile(source_name); + } else { + std::string backup_name = + absl::StrCat(path_stem, ".", (i + 1), ".wtf-trace"); + status = file_io::MoveFile(source_name, backup_name); + } + if (!status.ok()) { + LOG(WARNING) << "Could not remove backup trace file " << source_name + << ": " << status; + } + } +} + +// Flushes all recorded trace data since the last flush. +void FlushTraceFile() ABSL_EXCLUSIVE_LOCKS_REQUIRED(global_tracing_mutex) { + if (!global_tracing_initialized) return; + + const auto& trace_path = absl::GetFlag(FLAGS_iree_trace_file); + + static ::wtf::Runtime::SaveCheckpoint checkpoint; + static bool is_first_flush = true; + + if (is_first_flush && trace_path != "/dev/null") { + // Backup existing any existing trace files at the specified path. + RollTraceFiles(trace_path); + } + + auto save_options = + ::wtf::Runtime::SaveOptions::ForStreamingFile(&checkpoint); + if (is_first_flush) { + // On the first time, truncate the file. All subsequent flushes append. + save_options.open_mode = std::ios_base::trunc; + } + + is_first_flush = false; + + auto* runtime = ::wtf::Runtime::GetInstance(); + if (!runtime->SaveToFile(trace_path, save_options)) { + LOG(ERROR) << "Error saving WTF file: " << trace_path; + return; + } + + VLOG(1) << "Flushed WTF trace to: " << trace_path; +} + +} // namespace + +void InitializeTracing() { + if (!::wtf::kMasterEnable) { + if (!absl::GetFlag(FLAGS_iree_trace_file).empty()) { + LOG(WARNING) << "WTF trace save requested but WTF is not compiled in. " + << "Enable by building with --define=GLOBAL_WTF_ENABLE=1."; + } + return; + } + + absl::MutexLock lock(&global_tracing_mutex); + if (global_tracing_initialized) return; + global_tracing_initialized = true; + + LOG(INFO) << "Tracing enabled and streaming to: " + << absl::GetFlag(FLAGS_iree_trace_file); + + // Enable tracing on this thread, which we know is main. + IREE_TRACE_THREAD_ENABLE("main"); + + // Register atexit callback to stop tracking. + atexit(StopTracing); + + // Launch a thread to periodically flush the trace. + if (absl::GetFlag(FLAGS_iree_trace_file_period) > 0) { + auto flush_thread = std::thread(+[]() { + absl::Duration period = + absl::Seconds(absl::GetFlag(FLAGS_iree_trace_file_period)); + while (true) { + absl::SleepFor(period); + absl::MutexLock lock(&global_tracing_mutex); + if (!global_tracing_initialized) { + return; + } + FlushTraceFile(); + } + }); + flush_thread.detach(); + } +} + +// Stops tracing if currently initialized. +void StopTracing() { + if (!::wtf::kMasterEnable) return; + absl::MutexLock lock(&global_tracing_mutex); + if (!global_tracing_initialized) return; + + // Flush any pending trace data. + FlushTraceFile(); + + // Mark WTF as uninitialized to kill the flush thread. + global_tracing_initialized = false; + + LOG(INFO) << "Tracing stopped and flushed to file: " + << absl::GetFlag(FLAGS_iree_trace_file); +} + +void FlushTrace() { + if (!::wtf::kMasterEnable) return; + absl::MutexLock lock(&global_tracing_mutex); + if (!global_tracing_initialized) return; + FlushTraceFile(); +} + +} // namespace iree + +IREE_DECLARE_MODULE_INITIALIZER(iree_tracing); + +IREE_REGISTER_MODULE_INITIALIZER(iree_tracing, ::iree::InitializeTracing()); diff --git a/base/tracing.h b/base/tracing.h new file mode 100644 index 000000000000..ebb1cdb49c23 --- /dev/null +++ b/base/tracing.h @@ -0,0 +1,69 @@ +// Copyright 2019 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// Utilities for profiling and tracing. +// These attempt to support the various tools we use in a way that scales better +// than one annotation per tool per site and ensures things stay consistent and +// easy to correlate across tools. +// +// Tracing with WTF: +// - build with --define=GLOBAL_WTF_ENABLE=1 +// - pass --iree_trace_file=/tmp/foo.wtf-trace when running +// - view trace in WTF UI +// +// If GLOBAL_WTF_ENABLE=1 is specified WTF will automatically be initialized on +// startup and flushed on exit. + +#ifndef THIRD_PARTY_MLIR_EDGE_IREE_BASE_TRACING_H_ +#define THIRD_PARTY_MLIR_EDGE_IREE_BASE_TRACING_H_ + +#include "third_party/tracing_framework_bindings_cpp/event.h" // IWYU pragma: export +#include "third_party/tracing_framework_bindings_cpp/macros.h" // IWYU pragma: export + +namespace iree { + +// Initializes tracing if it is built into the binary. +// Does nothing if already initialized. +void InitializeTracing(); + +// Stops tracing and flushes any pending data. +void StopTracing(); + +// Flushes pending trace data to disk, if enabled. +void FlushTrace(); + +// Enables the current thread for WTF profiling/tracing. +#define IREE_TRACE_THREAD_ENABLE(name) WTF_THREAD_ENABLE(name); + +// Tracing scope that emits WTF tracing scopes depending on whether +// profiling/tracing are enabled. +// See WTF_SCOPE0 for more information. +#define IREE_TRACE_SCOPE0(name_spec) WTF_SCOPE0(name_spec); + +// Tracing scope that emits WTF tracing scopes with additional +// arguments depending on whether profiling/tracing is enabled. +// See WTF_SCOPE for more information. +#define IREE_TRACE_SCOPE(name_spec, ...) WTF_SCOPE(name_spec, __VA_ARGS__) + +// Tracing event that emits a WTF event. +// See WTF_EVENT0 for more information. +#define IREE_TRACE_EVENT0 WTF_EVENT0 + +// Tracing event that emits a WTF event with additional arguments. +// See WTF_EVENT for more information. +#define IREE_TRACE_EVENT WTF_EVENT + +} // namespace iree + +#endif // THIRD_PARTY_MLIR_EDGE_IREE_BASE_TRACING_H_ diff --git a/base/tracing_disabled.cc b/base/tracing_disabled.cc new file mode 100644 index 000000000000..e954ea331c91 --- /dev/null +++ b/base/tracing_disabled.cc @@ -0,0 +1,42 @@ +// Copyright 2019 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// This file is linked in only when WTF is not enabled. It allows us to keep the +// same flags and functions without needing to do a bunch of ifdef hackery or +// undefok mangling. + +#include +#include + +#include "third_party/absl/flags/flag.h" +#include "third_party/mlir_edge/iree/base/logging.h" +#include "third_party/mlir_edge/iree/base/tracing.h" + +ABSL_FLAG(int32_t, iree_trace_file_period, 0, + "Flag for tracing. Use --define=GLOBAL_WTF_ENABLE=1 to enable WTF."); +ABSL_FLAG(std::string, iree_trace_file, "", + "Flag for tracing. Use --define=GLOBAL_WTF_ENABLE=1 to enable WTF."); + +namespace iree { + +void InitializeTracing() { + if (!absl::GetFlag(FLAGS_iree_trace_file).empty()) { + LOG(WARNING) << "WTF trace save requested but WTF is not compiled in. " + << "Enable by building with --define=GLOBAL_WTF_ENABLE=1."; + } +} + +void FlushTrace() {} + +} // namespace iree diff --git a/base/wait_handle.cc b/base/wait_handle.cc new file mode 100644 index 000000000000..cf9d905e78f8 --- /dev/null +++ b/base/wait_handle.cc @@ -0,0 +1,532 @@ +// Copyright 2019 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "third_party/mlir_edge/iree/base/wait_handle.h" + +#include +#include +#include +#include +#include + +#include +#include + +#include "third_party/absl/container/fixed_array.h" +#include "third_party/absl/strings/str_cat.h" +#include "third_party/absl/time/clock.h" +#include "third_party/absl/time/time.h" +#include "third_party/absl/types/source_location.h" +#include "third_party/mlir_edge/iree/base/status.h" + +// TODO(benvanik): organize these macros - they are terrible. + +#if !defined(__ANDROID__) && !defined(OS_IOS) && !defined(__EMSCRIPTEN__) +#define IREE_HAS_PPOLL 1 +#endif // !__ANDROID__ && !__EMSCRIPTEN__ +#define IREE_HAS_POLL 1 + +#if !defined(OS_IOS) && !defined(OS_MACOSX) && !defined(__EMSCRIPTEN__) +#define IREE_HAS_EVENTFD 1 +#endif +#define IREE_HAS_PIPE 1 +// #define IREE_HAS_SYNC_FILE 1 + +#if defined(IREE_HAS_EVENTFD) +#include +#endif // IREE_HAS_EVENTFD + +namespace iree { + +namespace { + +constexpr int kInvalidFd = WaitableObject::kInvalidFd; +constexpr int kSignaledFd = WaitableObject::kSignaledFd; + +// Retries a syscall until it succeeds or fails for a real reason. +template +StatusOr::type> Syscall( + SyscallT syscall, ParamsT&&... params) { + while (true) { + const auto rv = syscall(std::forward(params)...); + if (rv >= 0) return rv; + if (errno == EINTR) { + // Retry on EINTR. + continue; + } else { + return ErrnoToCanonicalStatus(errno, ""); + } + } +} + +#if defined(IREE_HAS_PPOLL) + +// ppoll(), present on Linux. +// ppoll is preferred as it has a much better timing mechanism; poll can have a +// large slop on the deadline. +// Documentation: https://linux.die.net/man/2/poll +StatusOr SystemPoll(absl::Span poll_fds, absl::Time deadline) { + // Convert the deadline into a tmo_p struct for ppoll that controls whether + // the call is blocking or non-blocking. Note that we must do this every + // iteration of the loop as a previous ppoll may have taken some of the + // time. + // + // See the ppoll docs for more information as to what the expected value is: + // http://man7.org/linux/man-pages/man2/poll.2.html + timespec timeout_spec; + timespec* tmo_p; + if (deadline == absl::InfinitePast()) { + // 0 for non-blocking. + timeout_spec = {0}; + tmo_p = &timeout_spec; + } else if (deadline == absl::InfiniteFuture()) { + // nullptr to ppoll() to block forever. + tmo_p = nullptr; + } else { + // Wait only for as much time as we have before the deadline is exceeded. + absl::Duration remaining_time = deadline - absl::Now(); + if (remaining_time < absl::ZeroDuration()) { + // Note: we likely have already bailed before getting here with a negative + // duration. + return DeadlineExceededErrorBuilder(ABSL_LOC); + } + timeout_spec = absl::ToTimespec(remaining_time); + tmo_p = &timeout_spec; + } + return Syscall(::ppoll, poll_fds.data(), poll_fds.size(), tmo_p, nullptr); +} + +#elif defined(IREE_HAS_POLL) + +// poll(), present pretty much everywhere. +// Documentation: https://linux.die.net/man/2/poll +StatusOr SystemPoll(absl::Span poll_fds, absl::Time deadline) { + int timeout; + if (deadline == absl::InfinitePast()) { + // Don't block. + timeout = 0; + } else if (deadline == absl::InfiniteFuture()) { + // Block forever. + timeout = -1; + } else { + absl::Duration remaining_time = deadline - absl::Now(); + if (remaining_time < absl::ZeroDuration()) { + return DeadlineExceededErrorBuilder(ABSL_LOC); + } + timeout = static_cast(absl::ToInt64Milliseconds(remaining_time)); + } + return Syscall(::poll, poll_fds.data(), poll_fds.size(), timeout); +} + +#else +#error "No SystemPoll implementation" +#endif // IREE_HAS_PPOLL / IREE_HAS_POLL / etc + +// Builds the list of pollfds to for ppoll wait on and will perform any +// required wait handle callbacks. +// +// The provided deadline will be observed if any of the wait handles needs to +// block for acquiring an fd. +StatusOr> AcquireWaitHandles( + WaitHandle::WaitHandleSpan wait_handles, absl::Time deadline) { + absl::FixedArray poll_fds{wait_handles.size()}; + for (int i = 0; i < wait_handles.size(); ++i) { + poll_fds[i].events = POLLIN | POLLPRI | POLLERR | POLLHUP | POLLNVAL; + poll_fds[i].revents = 0; + // NOTE: poll will ignore any negative fds and our kInvalidFd == -1 so we + // can still put them in the list and it'll just skip them. + if (!wait_handles[i] || !wait_handles[i]->object()) { + poll_fds[i].fd = kInvalidFd; + continue; + } + + // Acquire the file descriptor for waiting. + // This may block (if |deadline| allows it) if the fd is not yet available. + // This is like a pre-wait for the actual poll operation. It can be bad with + // WaitAny, though we could handle that better here. + ASSIGN_OR_RETURN(auto fd_info, + wait_handles[i]->object()->AcquireFdForWait(deadline)); + poll_fds[i].fd = fd_info.second; + + // Abort if deadline exceeded. + if (deadline != absl::InfinitePast() && deadline < absl::Now()) { + return DeadlineExceededErrorBuilder(ABSL_LOC) + << "Deadline exceeded acquiring for fds"; + } + } + return poll_fds; +} + +Status ClearFd(WaitableObject::FdType fd_type, int fd) { + // Read in a loop until the read would block. + // Depending on how the users setup the fd the act of reading may reset the + // entire handle (such as with the default eventfd mode) or multiple reads + // may be required (such as with semaphores). + while (true) { +#if defined(IREE_HAS_EVENTFD) + eventfd_t val = 0; + int rv = ::eventfd_read(fd, &val); +#elif defined(IREE_HAS_PIPE) + char buf; + int rv = ::read(fd, &buf, 1); +#else + return UnimplementedErrorBuilder(ABSL_LOC) << "fd_type cannot be cleared"; +#endif // IREE_HAS_EVENTFD + if (rv != -1) { + // Success! Keep going. + continue; + } else { + if (errno == EWOULDBLOCK) { + // The read would have blocked meaning that we've hit the end and + // successfully cleared the fd. + return OkStatus(); + } else if (errno == EINTR) { + // Retry. + continue; + } else { + return ErrnoToCanonicalStatus(errno, "ClearFd failed"); + } + } + } +} + +// Performs a single poll on multiple fds and returns information about the +// signaled fds, if any. +Status MultiPoll(WaitHandle::WaitHandleSpan wait_handles, + absl::Span poll_fds, absl::Time deadline, + int* out_any_signaled_index, int* out_unsignaled_count) { + *out_any_signaled_index = -1; + *out_unsignaled_count = 0; + + // poll has a nasty behavior where it allows -1 for fds... except for at [0]. + // To keep the rest of the code sane we correct for that here as epoll doesn't + // have that behavior and we may want to special case this later. + bool any_valid_fds = true; + int swapped_zero_index = -1; + if (poll_fds[0].fd < 0) { + // Find a valid handle. + for (int i = 1; i < poll_fds.size(); ++i) { + if (poll_fds[i].fd > 0) { + swapped_zero_index = i; + std::swap(poll_fds[0], poll_fds[i]); + break; + } + } + if (swapped_zero_index == -1) { + // No valid handles found, meaning that all handles are invalid. + // We'll skip the wait below so we can share the processing code for any + // fds that may be kSignaledFd. + any_valid_fds = false; + } + } + + // Pass handles to ppoll. + // http://man7.org/linux/man-pages/man2/poll.2.html + if (any_valid_fds) { + ASSIGN_OR_RETURN(int rv, SystemPoll(poll_fds, deadline)); + if (rv == 0) { + // Call timed out and no descriptors were ready. + // If this was just a poll then that's fine. + return DeadlineExceededErrorBuilder(ABSL_LOC); + } + } + + // If we had swapped fds[0] above we need to correct for that now. + if (swapped_zero_index != -1) { + std::swap(poll_fds[0], poll_fds[swapped_zero_index]); + } + + // |rv| denotes the number of fds that were ready. Run through the list and + // find the ones that were ready and mark them as completed. + for (int i = 0; i < poll_fds.size(); ++i) { + if (poll_fds[i].fd == kSignaledFd || poll_fds[i].revents == POLLIN) { + // First attempt any resolve actions. If these fail we can't consider the + // fd as having been signaled. + ASSIGN_OR_RETURN( + bool resolved, + wait_handles[i]->object()->TryResolveWakeOnFd(poll_fds[i].fd)); + if (!resolved) { + ++(*out_unsignaled_count); + continue; + } + + // Successful wait. Kill the fd so it is ignored on the next poll. + poll_fds[i].fd = kInvalidFd; + *out_any_signaled_index = i; + } else if (poll_fds[i].revents) { + if (poll_fds[i].revents & POLLERR) { + return InternalErrorBuilder(ABSL_LOC); + } else if (poll_fds[i].revents & POLLHUP) { + return CancelledErrorBuilder(ABSL_LOC); + } else if (poll_fds[i].revents & POLLNVAL) { + return InvalidArgumentErrorBuilder(ABSL_LOC); + } else { + return UnknownErrorBuilder(ABSL_LOC); + } + } else if (poll_fds[i].fd != kInvalidFd) { + ++(*out_unsignaled_count); + } + } + + return OkStatus(); +} + +} // namespace + +// static +std::atomic WaitHandle::next_unique_id_{1}; + +// static +WaitHandle WaitHandle::AlwaysSignaling() { + class AlwaysSignalingObject : public WaitableObject { + public: + std::string DebugString() const override { return "signal"; } + StatusOr> AcquireFdForWait( + absl::Time deadline) override { + return std::make_pair(FdType::kPermanent, kSignaledFd); + } + StatusOr TryResolveWakeOnFd(int fd) override { return true; } + }; + static auto* obj = new AlwaysSignalingObject(); + return WaitHandle(add_ref(obj)); +} + +// static +WaitHandle WaitHandle::AlwaysFailing() { + class AlwaysFailingObject : public WaitableObject { + public: + std::string DebugString() const override { return "fail"; } + StatusOr> AcquireFdForWait( + absl::Time deadline) override { + return InternalErrorBuilder(ABSL_LOC) << "AlwaysFailingObject"; + } + StatusOr TryResolveWakeOnFd(int fd) override { + return InternalErrorBuilder(ABSL_LOC) << "AlwaysFailingObject"; + } + }; + static auto* obj = new AlwaysFailingObject(); + return WaitHandle(add_ref(obj)); +} + +// static +Status WaitHandle::WaitAll(WaitHandleSpan wait_handles, absl::Time deadline) { + if (wait_handles.empty()) return OkStatus(); + + // Build the list of pollfds to wait on. + ASSIGN_OR_RETURN(auto poll_fds, AcquireWaitHandles(wait_handles, deadline)); + + // Loop until all handles have been signaled or the deadline is exceeded. + int unsignaled_count = 0; + do { + int any_signaled_index = 0; + RETURN_IF_ERROR(MultiPoll(wait_handles, absl::MakeSpan(poll_fds), deadline, + &any_signaled_index, &unsignaled_count)); + } while (unsignaled_count > 0 && absl::Now() < deadline); + + if (unsignaled_count == 0) { + // All waits resolved. + return OkStatus(); + } else { + // One or more were unsignaled. + return DeadlineExceededErrorBuilder(ABSL_LOC); + } +} + +// static +StatusOr WaitHandle::TryWaitAll(WaitHandleSpan wait_handles) { + auto status = WaitAll(wait_handles, absl::InfinitePast()); + if (status.ok()) { + return true; + } else if (IsDeadlineExceeded(status)) { + return false; + } + return status; +} + +// static +StatusOr WaitHandle::WaitAny(WaitHandleSpan wait_handles, + absl::Time deadline) { + if (wait_handles.empty()) { + return InvalidArgumentErrorBuilder(ABSL_LOC) + << "At least one wait handle is required for WaitAny"; + } + + // Build the list of pollfds to wait on. + ASSIGN_OR_RETURN(auto poll_fds, AcquireWaitHandles(wait_handles, deadline)); + + // Poll once; this makes a WaitAny just a WaitMulti that doesn't loop. + int any_signaled_index = -1; + int unsignaled_count = 0; + RETURN_IF_ERROR(MultiPoll(wait_handles, absl::MakeSpan(poll_fds), deadline, + &any_signaled_index, &unsignaled_count)); + if (any_signaled_index == -1) { + // No wait handles were valid. Pretend 0 was signaled. + return 0; + } + return any_signaled_index; +} + +// static +StatusOr WaitHandle::TryWaitAny(WaitHandleSpan wait_handles) { + auto status_or = WaitAny(wait_handles, absl::InfinitePast()); + return IsDeadlineExceeded(status_or.status()) ? -1 : status_or; +} + +// Storage for static class variables; these won't be needed when we can use +// c++17 everywhere. +constexpr int WaitableObject::kInvalidFd; +constexpr int WaitableObject::kSignaledFd; + +WaitHandle::WaitHandle(ref_ptr object) + : unique_id_(++next_unique_id_), object_(std::move(object)) {} + +WaitHandle::~WaitHandle() { Dispose(); } + +void WaitHandle::Dispose() { object_.reset(); } + +WaitHandle::WaitHandle(WaitHandle&& other) + : unique_id_(other.unique_id_), object_(std::move(other.object_)) { + other.unique_id_ = 0; +} + +WaitHandle& WaitHandle::operator=(WaitHandle&& other) { + if (this != std::addressof(other)) { + // Close current handle. + Dispose(); + + // Take ownership of handle and resources. + object_ = std::move(other.object_); + + other.unique_id_ = ++next_unique_id_; + } + return *this; +} + +std::string WaitHandle::DebugString() const { + return object_ ? object_->DebugString() : absl::StrCat("wh_", unique_id_); +} + +StatusOr WaitHandle::TryWait() { + auto status = WaitAll({this}, absl::InfinitePast()); + if (status.ok()) { + return true; + } else if (IsDeadlineExceeded(status)) { + return false; + } + return status; +} + +ManualResetEvent::ManualResetEvent(const char* debug_name) + : debug_name_(debug_name) { + Initialize(); +} + +ManualResetEvent::~ManualResetEvent() { Dispose(); } + +void ManualResetEvent::Initialize() { +#if defined(IREE_HAS_EVENTFD) + // Create with an eventfd by default when we support it. + // eventfd has lower overhead than pipes (the syscalls are cheap). + // This usually will only fail if the system is completely out of handles. + // + // Docs: http://man7.org/linux/man-pages/man2/eventfd.2.html + fd_type_ = FdType::kEventFd; + fd_ = Syscall(::eventfd, 0, EFD_CLOEXEC | EFD_NONBLOCK).ValueOrDie(); +#elif defined(IREE_HAS_PIPE) + // Android/Linux/iOS-compatible POSIX pipe handle. + // Two handles are generated: one for transmitting and one for receiving. + // + // Docs: http://man7.org/linux/man-pages/man2/pipe.2.html + fd_type_ = FdType::kPipe; + int pipefd[2]; + Syscall(::pipe, pipefd).ValueOrDie(); + Syscall(::fcntl, pipefd[0], F_SETFL, O_NONBLOCK).ValueOrDie(); + fd_ = pipefd[0]; + write_fd_ = pipefd[1]; +#else +// NOTE: sync_file does not use Notifier as they come from the kernel. +#error "No fd-based sync primitive on this platform" +#endif // IREE_HAS_EVENTFD / IREE_HAS_PIPE / etc +} + +void ManualResetEvent::Dispose() { + if (fd_ != kInvalidFd) { + // Always signal, as we need to ensure waiters are woken. + CHECK_OK(Set()); + Syscall(::close, fd_).ValueOrDie(); + fd_ = kInvalidFd; + } + if (write_fd_ != kInvalidFd) { + Syscall(::close, write_fd_).ValueOrDie(); + write_fd_ = kInvalidFd; + } +} + +ManualResetEvent::ManualResetEvent(ManualResetEvent&& other) + : fd_type_(other.fd_type_), + fd_(other.fd_), + write_fd_(other.write_fd_), + debug_name_(other.debug_name_) { + other.fd_type_ = FdType::kPermanent; + other.fd_ = kInvalidFd; + other.write_fd_ = kInvalidFd; + other.debug_name_ = nullptr; +} + +ManualResetEvent& ManualResetEvent::operator=(ManualResetEvent&& other) { + if (this != std::addressof(other)) { + Dispose(); + fd_type_ = other.fd_type_; + fd_ = other.fd_; + write_fd_ = other.write_fd_; + debug_name_ = other.debug_name_; + other.fd_type_ = FdType::kPermanent; + other.fd_ = kInvalidFd; + other.write_fd_ = kInvalidFd; + other.debug_name_ = nullptr; + other.Initialize(); + } + return *this; +} + +std::string ManualResetEvent::DebugString() const { + if (debug_name_) { + return debug_name_; + } +#if defined(IREE_HAS_EVENTFD) + return absl::StrCat("eventfd_", fd_); +#elif defined(IREE_HAS_PIPE) + return absl::StrCat("pipe_", fd_, "_", write_fd_); +#else + return absl::StrCat("unknown_", fd_, "_", write_fd_); +#endif // IREE_HAS_EVENTFD / IREE_HAS_PIPE +} + +Status ManualResetEvent::Set() { +#if defined(IREE_HAS_EVENTFD) + return Syscall(::eventfd_write, fd_, 1ull).status(); +#elif defined(IREE_HAS_PIPE) + char buf = '\n'; + return Syscall(::write, write_fd_, &buf, 1).status(); +#else + return UnimplementedErrorBuilder(ABSL_LOC) + << "No fd-based sync primitive on this platform"; +#endif // IREE_HAS_EVENTFD / IREE_HAS_PIPE +} + +Status ManualResetEvent::Reset() { return ClearFd(fd_type_, fd_); } + +WaitHandle ManualResetEvent::OnSet() { return WaitHandle(add_ref(this)); } + +} // namespace iree diff --git a/base/wait_handle.h b/base/wait_handle.h new file mode 100644 index 000000000000..988590a2e04b --- /dev/null +++ b/base/wait_handle.h @@ -0,0 +1,321 @@ +// Copyright 2019 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#ifndef THIRD_PARTY_MLIR_EDGE_IREE_BASE_WAIT_HANDLE_H_ +#define THIRD_PARTY_MLIR_EDGE_IREE_BASE_WAIT_HANDLE_H_ + +#include +#include +#include +#include + +#include "third_party/absl/time/clock.h" +#include "third_party/absl/time/time.h" +#include "third_party/absl/types/span.h" +#include "third_party/mlir_edge/iree/base/ref_ptr.h" +#include "third_party/mlir_edge/iree/base/status.h" +#include "third_party/mlir_edge/iree/base/time.h" + +namespace iree { + +// Interfaces for waitable objects that can produce WaitHandles. +// WaitableObjects are much like ::thread::Selectable, only they support both +// the classic locking style as well as file descriptors for use with select(). +// +// Usage: +// class MyWaitableObject : public WaitableObject { +// public: +// std::string DebugString() const override { return "something useful"; } +// WaitHandle OnAsyncTask() { +// return WaitHandle(retain_ref(this)); +// } +// private: +// StatusOr> AcquireFdForWait( +// absl::Time deadline) override { +// // If blocking traditionally do so now and then return this: +// return std::make_pair(FdType::kPermanent, kSignaledFd); +// // Otherwise, see ManualResetEvent for an example using fds. +// } +// StatusOr TryResolveWakeOnFd(int fd) override { +// // Return true iff the object is really acquired, such as the semaphore +// // being decremented. +// return true; +// } +// }; +class WaitableObject : public RefObject { + public: + // Indicates that a file descriptor is invalid. It will not block when waited + // upon. + constexpr static int kInvalidFd = -1; + // Indicates that a file descriptor should be treated as signaled. + // Waiting on this fd should return as if it has already been signaled. + constexpr static int kSignaledFd = -2; + + // Defines the type of the native handle used for synchronization. + enum class FdType : uint16_t { + // Event has no handle and should be treated as permanently signaled. + kPermanent, + + // Android/Linux/iOS-compatible POSIX pipe handle. + // Two handles are generated: one for transmitting and one for receiving. + // + // More information: + // http://man7.org/linux/man-pages/man2/pipe.2.html + kPipe, + + // Android/Linux eventfd handle. + // These are akin to pipe() but require only a single handle and have + // significantly lower overhead (equivalent if not slightly better than + // pthreads condvars). + // + // eventfds support acting as both semaphores and auto reset events. + // + // More information: + // http://man7.org/linux/man-pages/man2/eventfd.2.html + kEventFd, + + // Android/Linux sync_file handle (aka 'sync fence'). + // The handle is allocated indirectly by the device driver via the + // API. It may be waited upon with poll(), select(), or + // epoll() and must be closed with close() when no longer required. If + // waiting on multiple sync_files the caller should first merge them + // together. + // + // A sync_file must only be used as fences (one-shot manual reset events). + // + // More information: + // https://www.kernel.org/doc/Documentation/sync_file.txt + // https://lwn.net/Articles/702339/ + // https://source.android.com/devices/graphics/implement-vsync#explicit_synchronization + kSyncFile, + }; + + virtual ~WaitableObject() = default; + + // Returns a string representing the object, either specified as a debug_name + // or a unique ID. + virtual std::string DebugString() const = 0; + + // Attempts to acquire a file descriptor for the waitable objects by the given + // |deadline|. In many cases this will return immediately with a valid fd. + // + // In cases where the file descriptor may not be available the call may block + // until either it is available or the |deadline| has elapsed. Use + // absl::InfinitePast() to prevent blocking. + // + // Returns a valid file descriptor or kInvalidFd as an indication that the + // object should not be waited on (already signaled, etc). Can return + // kSignaledFd to indicate that it's already known that the handle has been + // signaled and the caller should resolve as if it caused a wake normally. + virtual StatusOr> AcquireFdForWait( + absl::Time deadline) = 0; + + // Tries to resolve the object with the given |fd|. + // In many cases this will no-op, however some types may require additional + // checks to ensure that the wait operation succeeded (such as semaphores + // that may need to query a count). If resolution fails the waitable object + // must not be considered signaled. This call will never block. + virtual StatusOr TryResolveWakeOnFd(int fd) = 0; +}; + +// Handle to waitable objects. +// WaitHandles are created by a particular synchronization primitive, such as +// Fence, as a way for one or more observers to poll or wait for notification. +// +// External synchronization primitives can be wrapped in WaitHandles to enable +// other libraries or languages to be waited on alongside WaitHandles created +// by the IREE primitives like Fence. See the notes on WaitHandleType for a list +// of handle types that are supported. +// +// Wait handles are thread-safe in that multiple threads may be waiting on them +// concurrently. +class WaitHandle { + public: + // Returns a WaitHandle that when waited on will never block. + static WaitHandle AlwaysSignaling(); + + // Returns a WaitHandle that when waited on will always fail. + static WaitHandle AlwaysFailing(); + + using WaitHandleSpan = absl::Span; + + // Blocks the caller until all passed |wait_handles| are signaled or the + // |deadline| elapses. + // + // Returns success if the wait is successful and all events have been + // signaled. + // + // Returns DEADLINE_EXCEEDED if the |deadline| elapses without all handles + // having been signaled. Note that a subset of the |wait_handles| may have + // been signaled and each can be queried to see which one. + static Status WaitAll(WaitHandleSpan wait_handles, absl::Time deadline); + static Status WaitAll(WaitHandleSpan wait_handles, absl::Duration timeout) { + return WaitAll(wait_handles, RelativeTimeoutToDeadline(timeout)); + } + static Status WaitAll(WaitHandleSpan wait_handles) { + return WaitAll(wait_handles, absl::InfiniteFuture()); + } + + // Tries waiting on the handles and returns immediately if it would have + // blocked. The caller will not be blocked even if a handle has not yet been + // signaled. + // + // Returns true if all handles have been signaled. + static StatusOr TryWaitAll(WaitHandleSpan wait_handles); + + // Blocks the caller until at least one of the |wait_handles| is signaled or + // the |deadline| elapses. + // + // Returns the index into |wait_handles| of a handle that was signaled. Note + // that more than one handle may have been signaled and all of the other + // |wait_handles| should be queried or waited on again until waits for them + // succeed. + // + // Returns DEADLINE_EXCEEDED if the |deadline| elapses without any handles + // having been signaled. + static StatusOr WaitAny(WaitHandleSpan wait_handles, + absl::Time deadline); + static StatusOr WaitAny(WaitHandleSpan wait_handles, + absl::Duration timeout) { + return WaitAny(wait_handles, RelativeTimeoutToDeadline(timeout)); + } + static StatusOr WaitAny(WaitHandleSpan wait_handles) { + return WaitAny(wait_handles, absl::InfiniteFuture()); + } + + // Tries waiting for at least one handle to complete and returns immediately + // if none have been. The caller will not be blocked even if a handle has not + // yet been signaled. + // + // Returns the index into |wait_handles| of a handle that was signaled. Note + // that more than one handle may have been signaled and all of the other + // |wait_handles| should be queried or waited on again until waits for them + // succeed. + // + // Returns -1 if no handles were signaled. + static StatusOr TryWaitAny(WaitHandleSpan wait_handles); + + // Default constructor creates a permanently signaled handle. + // Waiting on this handle will never block. + WaitHandle() = default; + + // Wraps an existing sync file descriptor. + // Ownership of the file descriptor is transferred to the WaitHandle and must + // be duplicated by the caller if they want to continue using it. + explicit WaitHandle(ref_ptr object); + + ~WaitHandle(); + + // Copying not supported. Create a new WaitHandle from the source. + WaitHandle(const WaitHandle&) = delete; + WaitHandle& operator=(const WaitHandle&) = delete; + + // Moving supported; sync primitive ownership is transferred. + WaitHandle(WaitHandle&& other); + WaitHandle& operator=(WaitHandle&& other); + + // Unique ID for the WaitHandle instance. + // Two wait handles, even if waiting on the same underlying primitive, will + // have differing unique_ids. This can be used for deduping the handles or + // storing handles in a map. + uint64_t unique_id() const { return unique_id_; } + + // Returns a unique string representing the handle. + std::string DebugString() const; + + // Blocks the caller until the handle is signaled or the |deadline| elapses. + // + // If waiting on multiple wait handles use WaitAll or WaitAny instead of + // multiple calls to Wait as they can significantly reduce overhead. + // + // Returns success if the wait is successful and the |wait_handle| was + // signaled. Returns DEADLINE_EXCEEDED if the timeout elapses without the + // handle having been signaled. + Status Wait(absl::Time deadline) { return WaitAll({this}, deadline); } + Status Wait(absl::Duration timeout) { + return WaitAll({this}, RelativeTimeoutToDeadline(timeout)); + } + Status Wait() { return WaitAll({this}, absl::InfiniteFuture()); } + + // Tries waiting on the handle and returns immediately if it would have + // waited. The caller will not be blocked even if the handle has not yet been + // signaled. + // + // Returns true if the handle has been signaled. + StatusOr TryWait(); + + // These accessors should generally be considered opaque but may be useful to + // code trying to interop with other runtimes. + const ref_ptr& object() const { return object_; } + + private: + // Disposes the handle by closing the fd and issuing callbacks. + void Dispose(); + + static std::atomic next_unique_id_; + + uint64_t unique_id_ = 0; + ref_ptr object_; +}; + +// A manually-resettable event primitive. +// Effectively a binary semaphore with a maximum_count of 1 when running in +// auto-reset mode but also provides a sticky manual reset mode. +class ManualResetEvent : public WaitableObject { + public: + explicit ManualResetEvent(const char* debug_name = nullptr); + + ~ManualResetEvent() override; + + // Copying not supported. + ManualResetEvent(const ManualResetEvent&) = delete; + ManualResetEvent& operator=(const ManualResetEvent&) = delete; + + // Moving supported; sync primitive ownership is transferred. + ManualResetEvent(ManualResetEvent&& other); + ManualResetEvent& operator=(ManualResetEvent&& other); + + std::string DebugString() const override; + + // Sets the specified event object to the signaled state. + // The event stays signaled until Reset is called. Multiple waiters will be + // woken. + Status Set(); + + // Resets the specified event object to the nonsignaled state. + // Resetting an event that is already reset has no effect. + Status Reset(); + + // Returns a WaitHandle that will be signaled when the event is set. + WaitHandle OnSet(); + + protected: + void Initialize(); + void Dispose(); + + StatusOr> AcquireFdForWait( + absl::Time deadline) override { + return std::make_pair(fd_type_, fd_); + } + StatusOr TryResolveWakeOnFd(int fd) override { return true; } + + FdType fd_type_ = FdType::kPermanent; + int fd_ = kInvalidFd; + int write_fd_ = kInvalidFd; // Used only for fd_type_ == kPipe. + const char* debug_name_ = nullptr; +}; + +} // namespace iree + +#endif // THIRD_PARTY_MLIR_EDGE_IREE_BASE_WAIT_HANDLE_H_ diff --git a/compiler/IR/ConfigOps.cpp b/compiler/IR/ConfigOps.cpp new file mode 100644 index 000000000000..b23cc8555ec6 --- /dev/null +++ b/compiler/IR/ConfigOps.cpp @@ -0,0 +1,111 @@ +// Copyright 2019 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "third_party/mlir_edge/iree/compiler/IR/ConfigOps.h" + +#include "third_party/llvm/llvm/include/llvm/ADT/SmallString.h" +#include "third_party/llvm/llvm/include/llvm/ADT/SmallVector.h" +#include "third_party/llvm/llvm/projects/google_mlir/include/mlir/IR/Attributes.h" +#include "third_party/llvm/llvm/projects/google_mlir/include/mlir/IR/Builders.h" +#include "third_party/llvm/llvm/projects/google_mlir/include/mlir/IR/Diagnostics.h" +#include "third_party/llvm/llvm/projects/google_mlir/include/mlir/IR/OpImplementation.h" +#include "third_party/llvm/llvm/projects/google_mlir/include/mlir/IR/SymbolTable.h" +#include "third_party/llvm/llvm/projects/google_mlir/include/mlir/IR/Value.h" +#include "third_party/llvm/llvm/projects/google_mlir/include/mlir/Support/LLVM.h" +#include "third_party/llvm/llvm/projects/google_mlir/include/mlir/Support/STLExtras.h" +#include "third_party/mlir_edge/iree/compiler/IR/StructureOps.h" +#include "third_party/mlir_edge/iree/compiler/IR/Types.h" + +namespace mlir { +namespace iree_compiler { +namespace IREE { + +//===----------------------------------------------------------------------===// +// Generic printers and parsers. +//===----------------------------------------------------------------------===// + +// Parses an op that has no inputs and no outputs. +static ParseResult parseNoIOOp(OpAsmParser *parser, OperationState *state) { + if (failed(parser->parseOptionalAttributeDict(state->attributes))) { + return failure(); + } + return success(); +} + +// Prints an op that has no inputs and no outputs. +static void printNoIOOp(Operation *op, OpAsmPrinter *printer) { + *printer << op->getName(); + printer->printOptionalAttrDict(op->getAttrs()); +} + +//===----------------------------------------------------------------------===// +// iree.target_config +//===----------------------------------------------------------------------===// + +void ExecutableTargetConfigOp::build(Builder *builder, OperationState *state, + std::string backend) { + state->addAttribute("backend", builder->getStringAttr(backend)); + ensureTerminator(*state->addRegion(), *builder, state->location); +} + +static ParseResult parseExecutableTargetConfigOp(OpAsmParser *parser, + OperationState *state) { + llvm::SMLoc backendLoc; + StringAttr backendAttr; + if (failed(parser->parseLParen()) || + failed(parser->getCurrentLocation(&backendLoc)) || + failed( + parser->parseAttribute(backendAttr, "backend", state->attributes))) { + return failure(); + } + + Region *body = state->addRegion(); + if (failed(parser->parseRegion(*body, /*arguments=*/{}, /*argTypes=*/{}))) { + return failure(); + } + if (succeeded(parser->parseOptionalKeyword("attributes"))) { + if (failed(parser->parseOptionalAttributeDict(state->attributes))) { + return failure(); + } + } + + ExecutableTargetConfigOp::ensureTerminator(*body, parser->getBuilder(), + state->location); + + return success(); +} + +static void printExecutableTargetConfigOp(OpAsmPrinter *printer, + ExecutableTargetConfigOp op) { + *printer << op.getOperationName() << "(" << op.backend() << ")"; + + printer->printRegion(op.body(), /*printEntryBlockArgs=*/false, + /*printBlockTerminators=*/false); + + // Print out executable attributes, if present. + SmallVector ignoredAttrs = { + "backend", + }; + if (op.getAttrs().size() > ignoredAttrs.size()) { + *printer << "\n attributes "; + printer->printOptionalAttrDict(op.getAttrs(), ignoredAttrs); + } +} + +#define GET_OP_CLASSES +#include "third_party/mlir_edge/iree/compiler/IR/ConfigOps.cpp.inc" + +} // namespace IREE +} // namespace iree_compiler +} // namespace mlir diff --git a/compiler/IR/ConfigOps.h b/compiler/IR/ConfigOps.h new file mode 100644 index 000000000000..f6d597a838ea --- /dev/null +++ b/compiler/IR/ConfigOps.h @@ -0,0 +1,38 @@ +// Copyright 2019 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#ifndef THIRD_PARTY_MLIR_EDGE_IREE_COMPILER_IR_CONFIGOPS_H_ +#define THIRD_PARTY_MLIR_EDGE_IREE_COMPILER_IR_CONFIGOPS_H_ + +#include + +#include "third_party/llvm/llvm/projects/google_mlir/include/mlir/Dialect/StandardOps/Ops.h" +#include "third_party/llvm/llvm/projects/google_mlir/include/mlir/IR/Attributes.h" +#include "third_party/llvm/llvm/projects/google_mlir/include/mlir/IR/Dialect.h" +#include "third_party/llvm/llvm/projects/google_mlir/include/mlir/IR/OpDefinition.h" +#include "third_party/llvm/llvm/projects/google_mlir/include/mlir/IR/StandardTypes.h" +#include "third_party/mlir_edge/iree/compiler/IR/Types.h" + +namespace mlir { +namespace iree_compiler { +namespace IREE { + +#define GET_OP_CLASSES +#include "third_party/mlir_edge/iree/compiler/IR/ConfigOps.h.inc" + +} // namespace IREE +} // namespace iree_compiler +} // namespace mlir + +#endif // THIRD_PARTY_MLIR_EDGE_IREE_COMPILER_IR_CONFIGOPS_H_ diff --git a/compiler/IR/ConfigOps.td b/compiler/IR/ConfigOps.td new file mode 100644 index 000000000000..b2a153b10b27 --- /dev/null +++ b/compiler/IR/ConfigOps.td @@ -0,0 +1,44 @@ +// Ops used to declare configuration used by the IREE compiler. +// These allow inline config that follows along the IR they are associated with. +// Multiple config ops are allowed within a single scope to indicate that the +// parent IR node should be processed for multiple targets. + +#ifdef IREE_CONFIG_OPS +#else +#define IREE_CONFIG_OPS + +include "third_party/mlir_edge/iree/compiler/IR/OpBase.td" + +class IREE_ConfigOp traits = []> : + Op { + let parser = [{ return parse$cppClass(parser, result); }]; + let printer = [{ print$cppClass(p, *this); }]; +} + +//===----------------------------------------------------------------------===// +// iree.executable configuration +//===----------------------------------------------------------------------===// + +def IREE_ExecutableTargetConfigOp : IREE_ConfigOp<"target_config", [ + IREE_ExecutableOnly, + SingleBlockImplicitTerminator<"ExecutableTargetConfigEndOp"> +]> { + let arguments = (ins + StrAttr:$backend + ); + + let regions = (region SizedRegion<1>:$body); + + let skipDefaultBuilders = 1; + let builders = [ + OpBuilder<"Builder *builder, OperationState *state, std::string backend">, + ]; +} + +def IREE_ExecutableTargetConfigEndOp : + IREE_ConfigOp<"_target_config_end", [Terminator, IREE_ExecutableTargetConfigOnly]> { + let parser = [{ return parseNoIOOp(parser, result); }]; + let printer = [{ printNoIOOp(getOperation(), p); }]; +} + +#endif // IREE_CONFIG_OPS diff --git a/compiler/IR/Dialect.cpp b/compiler/IR/Dialect.cpp new file mode 100644 index 000000000000..597ee28a3ca8 --- /dev/null +++ b/compiler/IR/Dialect.cpp @@ -0,0 +1,90 @@ +// Copyright 2019 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "third_party/mlir_edge/iree/compiler/IR/Dialect.h" + +#include "third_party/llvm/llvm/include/llvm/Support/SourceMgr.h" +#include "third_party/mlir_edge/iree/compiler/IR/ConfigOps.h" +#include "third_party/mlir_edge/iree/compiler/IR/Ops.h" +#include "third_party/mlir_edge/iree/compiler/IR/StructureOps.h" +#include "third_party/mlir_edge/iree/compiler/IR/Types.h" + +namespace mlir { +namespace iree_compiler { + +static DialectRegistration iree_dialect; + +IREEDialect::IREEDialect(MLIRContext *context) + : Dialect(getDialectNamespace(), context) { +#define IREE_ADD_TYPE(NAME, KIND, TYPE) addTypes(); + IREE_TYPE_TABLE(IREE_ADD_TYPE); + +#define GET_OP_LIST + addOperations< +#include "third_party/mlir_edge/iree/compiler/IR/Ops.cpp.inc" + >(); +#define GET_OP_LIST + addOperations< +#include "third_party/mlir_edge/iree/compiler/IR/ConfigOps.cpp.inc" + >(); +#define GET_OP_LIST + addOperations< +#include "third_party/mlir_edge/iree/compiler/IR/StructureOps.cpp.inc" + >(); +} + +//===----------------------------------------------------------------------===// +// Type Parsing +//===----------------------------------------------------------------------===// + +#define IREE_TYPE_PARSER(NAME, KIND, TYPE) \ + static Type parse##TYPE(IREEDialect const &dialect, StringRef spec, \ + Location loc) { \ + spec.consume_front(NAME); \ + return TYPE::get(dialect.getContext()); \ + } +IREE_TYPE_TABLE(IREE_TYPE_PARSER); + +#define IREE_PARSE_TYPE(NAME, KIND, TYPE) \ + if (spec.startswith(NAME)) { \ + return parse##TYPE(*this, spec, loc); \ + } +Type IREEDialect::parseType(StringRef spec, Location loc) const { + IREE_TYPE_TABLE(IREE_PARSE_TYPE); + emitError(loc, "unknown IREE type: ") << spec; + return Type(); +} + +//===----------------------------------------------------------------------===// +// Type Printing +//===----------------------------------------------------------------------===// + +#define IREE_TYPE_PRINTER(NAME, KIND, TYPE) \ + static void print##TYPE(TYPE type, llvm::raw_ostream &os) { os << NAME; } +IREE_TYPE_TABLE(IREE_TYPE_PRINTER); + +#define IREE_PRINT_TYPE(NAME, KIND, TYPE) \ + case KIND: \ + print##TYPE(type.cast(), os); \ + return; +void IREEDialect::printType(Type type, llvm::raw_ostream &os) const { + switch (type.getKind()) { + IREE_TYPE_TABLE(IREE_PRINT_TYPE); + default: + llvm_unreachable("unhandled IREE type"); + } +} + +} // namespace iree_compiler +} // namespace mlir diff --git a/compiler/IR/Dialect.h b/compiler/IR/Dialect.h new file mode 100644 index 000000000000..97b292b63e0d --- /dev/null +++ b/compiler/IR/Dialect.h @@ -0,0 +1,38 @@ +// Copyright 2019 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#ifndef THIRD_PARTY_MLIR_EDGE_IREE_COMPILER_IR_DIALECT_H_ +#define THIRD_PARTY_MLIR_EDGE_IREE_COMPILER_IR_DIALECT_H_ + +#include "third_party/llvm/llvm/projects/google_mlir/include/mlir/IR/Dialect.h" + +namespace mlir { +namespace iree_compiler { + +class IREEDialect : public Dialect { + public: + explicit IREEDialect(MLIRContext* context); + static StringRef getDialectNamespace() { return "iree"; } + + /// Parses a type registered to this dialect. + Type parseType(llvm::StringRef spec, Location loc) const override; + + /// Prints a type registered to this dialect. + void printType(Type type, llvm::raw_ostream& os) const override; +}; + +} // namespace iree_compiler +} // namespace mlir + +#endif // THIRD_PARTY_MLIR_EDGE_IREE_COMPILER_IR_DIALECT_H_ diff --git a/compiler/IR/Interpreter/HLDialect.cpp b/compiler/IR/Interpreter/HLDialect.cpp new file mode 100644 index 000000000000..52a776d77be8 --- /dev/null +++ b/compiler/IR/Interpreter/HLDialect.cpp @@ -0,0 +1,34 @@ +// Copyright 2019 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "third_party/mlir_edge/iree/compiler/IR/Interpreter/HLDialect.h" + +#include "third_party/llvm/llvm/include/llvm/Support/SourceMgr.h" +#include "third_party/mlir_edge/iree/compiler/IR/Interpreter/HLOps.h" + +namespace mlir { +namespace iree_compiler { + +IREEHLInterpreterDialect::IREEHLInterpreterDialect(MLIRContext* context) + : Dialect(getDialectNamespace(), context) { +#define GET_OP_LIST + addOperations< +#include "third_party/mlir_edge/iree/compiler/IR/Interpreter/HLOps.cpp.inc" + >(); +} + +static DialectRegistration iree_hl_interp_dialect; + +} // namespace iree_compiler +} // namespace mlir diff --git a/compiler/IR/Interpreter/HLDialect.h b/compiler/IR/Interpreter/HLDialect.h new file mode 100644 index 000000000000..b398ecd108a5 --- /dev/null +++ b/compiler/IR/Interpreter/HLDialect.h @@ -0,0 +1,32 @@ +// Copyright 2019 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#ifndef THIRD_PARTY_MLIR_EDGE_IREE_COMPILER_IR_INTERPRETER_HLDIALECT_H_ +#define THIRD_PARTY_MLIR_EDGE_IREE_COMPILER_IR_INTERPRETER_HLDIALECT_H_ + +#include "third_party/llvm/llvm/projects/google_mlir/include/mlir/IR/Dialect.h" + +namespace mlir { +namespace iree_compiler { + +class IREEHLInterpreterDialect : public Dialect { + public: + explicit IREEHLInterpreterDialect(MLIRContext* context); + static StringRef getDialectNamespace() { return "iree_hl_interp"; } +}; + +} // namespace iree_compiler +} // namespace mlir + +#endif // THIRD_PARTY_MLIR_EDGE_IREE_COMPILER_IR_INTERPRETER_HLDIALECT_H_ diff --git a/compiler/IR/Interpreter/HLOps.cpp b/compiler/IR/Interpreter/HLOps.cpp new file mode 100644 index 000000000000..a93660dc1668 --- /dev/null +++ b/compiler/IR/Interpreter/HLOps.cpp @@ -0,0 +1,249 @@ +// Copyright 2019 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "third_party/mlir_edge/iree/compiler/IR/Interpreter/HLOps.h" + +#include "third_party/llvm/llvm/projects/google_mlir/include/mlir/IR/Builders.h" +#include "third_party/llvm/llvm/projects/google_mlir/include/mlir/IR/Function.h" +#include "third_party/llvm/llvm/projects/google_mlir/include/mlir/IR/OpImplementation.h" +#include "third_party/llvm/llvm/projects/google_mlir/include/mlir/IR/PatternMatch.h" +#include "third_party/mlir_edge/iree/compiler/IR/Ops.h" + +namespace mlir { +namespace iree_compiler { +namespace IREEInterp { +namespace HL { + +//===----------------------------------------------------------------------===// +// iree_hl_interp.call +//===----------------------------------------------------------------------===// + +static ParseResult parseCallOp(OpAsmParser *parser, OperationState *state) { + SymbolRefAttr calleeAttr; + FunctionType calleeType; + SmallVector operands; + auto calleeLoc = parser->getNameLoc(); + if (parser->parseAttribute(calleeAttr, "callee", state->attributes) || + parser->parseOperandList(operands, OpAsmParser::Delimiter::Paren) || + parser->parseOptionalAttributeDict(state->attributes) || + parser->parseColonType(calleeType) || + parser->addTypesToList(calleeType.getResults(), state->types) || + parser->resolveOperands(operands, calleeType.getInputs(), calleeLoc, + state->operands)) { + return failure(); + } + return success(); +} + +static void printCallOp(OpAsmPrinter *p, CallOp op) { + *p << "iree_hl_interp.call " << op.getAttr("callee") << '('; + p->printOperands(op.getOperands()); + *p << ')'; + p->printOptionalAttrDict(op.getAttrs(), /*elidedAttrs=*/{"callee"}); + *p << " : "; + p->printType(op.getCalleeType()); +} + +FunctionType CallOp::getCalleeType() { + SmallVector resultTypes(getResultTypes()); + SmallVector argTypes(getOperandTypes()); + return FunctionType::get(argTypes, resultTypes, getContext()); +} + +//===----------------------------------------------------------------------===// +// iree_hl_interp.call_indirect +//===----------------------------------------------------------------------===// + +static ParseResult parseCallIndirectOp(OpAsmParser *parser, + OperationState *result) { + FunctionType calleeType; + OpAsmParser::OperandType callee; + llvm::SMLoc operandsLoc; + SmallVector operands; + return failure( + parser->parseOperand(callee) || + parser->getCurrentLocation(&operandsLoc) || + parser->parseOperandList(operands, OpAsmParser::Delimiter::Paren) || + parser->parseOptionalAttributeDict(result->attributes) || + parser->parseColonType(calleeType) || + parser->resolveOperand(callee, calleeType, result->operands) || + parser->resolveOperands(operands, calleeType.getInputs(), operandsLoc, + result->operands) || + parser->addTypesToList(calleeType.getResults(), result->types)); +} + +static void printCallIndirectOp(OpAsmPrinter *p, CallIndirectOp op) { + *p << "iree_hl_interp.call_indirect "; + p->printOperand(op.getCallee()); + *p << '('; + auto operandRange = op.getOperands(); + p->printOperands(++operandRange.begin(), operandRange.end()); + *p << ')'; + p->printOptionalAttrDict(op.getAttrs(), /*elidedAttrs=*/{"callee"}); + *p << " : " << op.getCallee()->getType(); +} + +//===----------------------------------------------------------------------===// +// iree_hl_interp.return +//===----------------------------------------------------------------------===// + +static ParseResult parseReturnOp(OpAsmParser *parser, OperationState *state) { + SmallVector opInfo; + SmallVector types; + llvm::SMLoc loc = parser->getCurrentLocation(); + return failure(parser->parseOperandList(opInfo) || + (!opInfo.empty() && parser->parseColonTypeList(types)) || + parser->resolveOperands(opInfo, types, loc, state->operands)); +} + +static void printReturnOp(OpAsmPrinter *p, ReturnOp op) { + *p << "iree_hl_interp.return"; + if (op.getNumOperands() > 0) { + *p << ' '; + p->printOperands(op.operand_begin(), op.operand_end()); + *p << " : "; + interleaveComma(op.getOperandTypes(), *p); + } +} + +//===----------------------------------------------------------------------===// +// iree_hl_interp.br +//===----------------------------------------------------------------------===// + +static ParseResult parseBranchOp(OpAsmParser *parser, OperationState *result) { + Block *dest; + SmallVector destOperands; + if (parser->parseSuccessorAndUseList(dest, destOperands)) return failure(); + result->addSuccessor(dest, destOperands); + return success(); +} + +static void printBranchOp(OpAsmPrinter *p, BranchOp op) { + *p << "iree_hl_interp.br "; + p->printSuccessorAndUseList(op.getOperation(), 0); +} + +Block *BranchOp::getDest() { return getOperation()->getSuccessor(0); } + +void BranchOp::setDest(Block *block) { + return getOperation()->setSuccessor(block, 0); +} + +void BranchOp::eraseOperand(unsigned index) { + getOperation()->eraseSuccessorOperand(0, index); +} + +//===----------------------------------------------------------------------===// +// iree_hl_interp.cond_br +//===----------------------------------------------------------------------===// + +static ParseResult parseCondBranchOp(OpAsmParser *parser, + OperationState *result) { + SmallVector destOperands; + Block *dest; + OpAsmParser::OperandType condInfo; + + // Parse the condition. + Type int1Ty = parser->getBuilder().getI1Type(); + if (parser->parseOperand(condInfo) || parser->parseComma() || + parser->resolveOperand(condInfo, int1Ty, result->operands)) { + return parser->emitError(parser->getNameLoc(), + "expected condition type was boolean (i1)"); + } + + // Parse the true successor. + if (parser->parseSuccessorAndUseList(dest, destOperands)) return failure(); + result->addSuccessor(dest, destOperands); + + // Parse the false successor. + destOperands.clear(); + if (parser->parseComma() || + parser->parseSuccessorAndUseList(dest, destOperands)) + return failure(); + result->addSuccessor(dest, destOperands); + + return success(); +} + +static void printCondBranchOp(OpAsmPrinter *p, CondBranchOp op) { + *p << "iree_hl_interp.cond_br "; + p->printOperand(op.getCondition()); + *p << ", "; + p->printSuccessorAndUseList(op.getOperation(), CondBranchOp::trueIndex); + *p << ", "; + p->printSuccessorAndUseList(op.getOperation(), CondBranchOp::falseIndex); +} + +//===----------------------------------------------------------------------===// +// iree_hl_interp.concat +//===----------------------------------------------------------------------===// + +namespace { +static ElementsAttr elementsAttrFromArray(PatternRewriter &rewriter, + ArrayRef elements) { + return rewriter.getDenseIntElementsAttr( + rewriter.getTensorType(elements.size(), rewriter.getIntegerType(64)), + elements); +} + +static IREE::ConstantOp createArrayConstant(PatternRewriter &rewriter, + Location loc, + llvm::ArrayRef elements) { + auto elementsAttr = elementsAttrFromArray(rewriter, elements); + return rewriter.create(loc, elementsAttr); +} + +struct ConcatToCopies : public OpRewritePattern { + using OpRewritePattern::OpRewritePattern; + PatternMatchResult matchAndRewrite(ConcatOp concatOp, + PatternRewriter &rewriter) const override { + auto finalType = concatOp.getResult()->getType().cast(); + auto loc = concatOp.getLoc(); + std::vector dimPieces; + auto dst = + rewriter.create(loc, finalType, dimPieces); + + llvm::SmallVector zeroOffset(finalType.getRank(), 0); + auto srcIndices = createArrayConstant(rewriter, loc, zeroOffset); + + auto concatDimension = concatOp.dimension().getZExtValue(); + llvm::SmallVector dstIndices(finalType.getRank(), 0); + for (auto *src : concatOp.srcs()) { + auto srcShape = src->getType().cast().getShape(); + auto lengths = createArrayConstant(rewriter, loc, srcShape); + auto dstIndicesOp = createArrayConstant(rewriter, loc, dstIndices); + rewriter.create(loc, src, srcIndices, dst, + dstIndicesOp, lengths); + dstIndices[concatDimension] += srcShape[concatDimension]; + } + + concatOp.replaceAllUsesWith(dst.getResult()); + + return matchSuccess(); + } +}; +} // namespace + +void ConcatOp::getCanonicalizationPatterns(OwningRewritePatternList &results, + MLIRContext *context) { + results.insert(context); +} + +#define GET_OP_CLASSES +#include "third_party/mlir_edge/iree/compiler/IR/Interpreter/HLOps.cpp.inc" + +} // namespace HL +} // namespace IREEInterp +} // namespace iree_compiler +} // namespace mlir diff --git a/compiler/IR/Interpreter/HLOps.h b/compiler/IR/Interpreter/HLOps.h new file mode 100644 index 000000000000..9321dde3f98e --- /dev/null +++ b/compiler/IR/Interpreter/HLOps.h @@ -0,0 +1,38 @@ +// Copyright 2019 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#ifndef THIRD_PARTY_MLIR_EDGE_IREE_COMPILER_IR_INTERPRETER_HLOPS_H_ +#define THIRD_PARTY_MLIR_EDGE_IREE_COMPILER_IR_INTERPRETER_HLOPS_H_ + +#include "third_party/llvm/llvm/projects/google_mlir/include/mlir/Dialect/StandardOps/Ops.h" +#include "third_party/llvm/llvm/projects/google_mlir/include/mlir/IR/Attributes.h" +#include "third_party/llvm/llvm/projects/google_mlir/include/mlir/IR/Dialect.h" +#include "third_party/llvm/llvm/projects/google_mlir/include/mlir/IR/OpDefinition.h" +#include "third_party/llvm/llvm/projects/google_mlir/include/mlir/IR/StandardTypes.h" +#include "third_party/llvm/llvm/projects/google_mlir/include/mlir/IR/TypeUtilities.h" + +namespace mlir { +namespace iree_compiler { +namespace IREEInterp { +namespace HL { + +#define GET_OP_CLASSES +#include "third_party/mlir_edge/iree/compiler/IR/Interpreter/HLOps.h.inc" + +} // namespace HL +} // namespace IREEInterp +} // namespace iree_compiler +} // namespace mlir + +#endif // THIRD_PARTY_MLIR_EDGE_IREE_COMPILER_IR_INTERPRETER_HLOPS_H_ diff --git a/compiler/IR/Interpreter/HLOps.td b/compiler/IR/Interpreter/HLOps.td new file mode 100644 index 000000000000..1985bd19bbb5 --- /dev/null +++ b/compiler/IR/Interpreter/HLOps.td @@ -0,0 +1,643 @@ +// IREE high-level interpreter op definitions. +// This op set contains pseudo ops, ops that accept non-MemRef types, and ops in +// normal SSA form. +// +// Through lowering these high-level ops are converted to low-level ops in the +// LLOps.td (iree_ll_interp.*). These map 1:1 with the bytecode, +// accept only MemRef types, and generally use output parameters instead of +// return types. +// +// The source of truth for bytecode opcodes is: +// iree/schemas/bytecode/interpreter_bytecode_v0.h + +#ifdef IREE_INTERPRETER_HL_OPS +#else +#define IREE_INTERPRETER_HL_OPS + +#ifdef IREE_OP_BASE +#else +include "third_party/mlir_edge/iree/compiler/IR/OpBase.td" +#endif // IREE_OP_BASE + +def IREEInterpHL_Dialect : Dialect { + let name = "iree_hl_interp"; + let cppNamespace = "IREEInterp::HL"; +} + +//===----------------------------------------------------------------------===// +// Base op classes +//===----------------------------------------------------------------------===// + +class IREEInterpHL_Op traits = []> : + Op; + +class IREEInterpHL_PureOp traits = []> : + IREEInterpHL_Op; + +//===----------------------------------------------------------------------===// +// High-level interpreter ops +//===----------------------------------------------------------------------===// + +def IREEInterpHL_CallOp : IREEInterpHL_Op<"call"> { + let arguments = (ins SymbolRefAttr:$callee, Variadic); + let results = (outs Variadic); + + let builders = [OpBuilder< + "Builder *builder, OperationState *result, FuncOp callee," + "ArrayRef operands = {}", [{ + result->addOperands(operands); + result->addAttribute("callee", builder->getSymbolRefAttr(callee)); + result->addTypes(callee.getType().getResults()); + }]>, OpBuilder< + "Builder *builder, OperationState *result, StringRef callee," + "ArrayRef results, ArrayRef operands = {}", [{ + result->addOperands(operands); + result->addAttribute("callee", builder->getSymbolRefAttr(callee)); + result->addTypes(results); + }]>]; + + let extraClassDeclaration = [{ + // TODO(b/132296600): make tablegen follow the style guide. + StringRef getCallee() { return callee(); } + FunctionType getCalleeType(); + + // TODO(b/133879130): make tablegen support variadic operand accessors. + /// Get the argument operands to the called function. + operand_range getArgOperands() { + return {arg_operand_begin(), arg_operand_end()}; + } + operand_iterator arg_operand_begin() { return operand_begin(); } + operand_iterator arg_operand_end() { return operand_end(); } + }]; + + let parser = [{ return parse$cppClass(parser, result); }]; + let printer = [{ return print$cppClass(p, *this); }]; +} + +def IREEInterpHL_CallIndirectOp : IREEInterpHL_Op<"call_indirect"> { + let arguments = (ins FunctionType:$callee, Variadic:$operands); + let results = (outs Variadic); + + let builders = [OpBuilder< + "Builder *, OperationState *result, Value *callee," + "ArrayRef operands = {}", [{ + result->operands.push_back(callee); + result->addOperands(operands); + result->addTypes(callee->getType().cast().getResults()); + }]>]; + + let extraClassDeclaration = [{ + // TODO(b/132296600): make tablegen follow the style guide. + Value *getCallee() { return getOperand(0); } + + // TODO(b/133879130): make tablegen support variadic operand accessors. + /// Get the argument operands to the called function. + operand_range getArgOperands() { + return {arg_operand_begin(), arg_operand_end()}; + } + operand_iterator arg_operand_begin() { return ++operand_begin(); } + operand_iterator arg_operand_end() { return operand_end(); } + }]; + + let parser = [{ return parse$cppClass(parser, result); }]; + let printer = [{ return print$cppClass(p, *this); }]; +} + +def IREEInterpHL_ReturnOp : IREEInterpHL_Op<"return", [Terminator]> { + let arguments = (ins Variadic:$operands); + + let builders = [OpBuilder< + "Builder *b, OperationState *result", [{ build(b, result, llvm::None); }] + >]; + + let parser = [{ return parse$cppClass(parser, result); }]; + let printer = [{ return print$cppClass(p, *this); }]; +} + +def IREEInterpHL_BranchOp : IREEInterpHL_Op<"br", [Terminator]> { + let arguments = (ins Variadic:$operands); + + let builders = [OpBuilder< + "Builder *, OperationState *result, Block *dest," + "ArrayRef operands = {}", [{ + result->addSuccessor(dest, operands); + }]>]; + + let extraClassDeclaration = [{ + Block *getDest(); + void setDest(Block *block); + + /// Erase the operand at 'index' from the operand list. + void eraseOperand(unsigned index); + }]; + + let parser = [{ return parse$cppClass(parser, result); }]; + let printer = [{ return print$cppClass(p, *this); }]; +} + +def IREEInterpHL_CondBranchOp : IREEInterpHL_Op<"cond_br", [Terminator]> { + let arguments = (ins + IREEHL_BoolScalar:$condition, + Variadic:$branchOperands + ); + + let builders = [OpBuilder< + "Builder *, OperationState *result, Value *condition," + "Block *trueDest, ArrayRef trueOperands," + "Block *falseDest, ArrayRef falseOperands", [{ + result->addOperands(condition); + result->addSuccessor(trueDest, trueOperands); + result->addSuccessor(falseDest, falseOperands); + }]>]; + + let extraClassDeclaration = [{ + // These are the indices into the dests list. + enum { trueIndex = 0, falseIndex = 1 }; + + // The condition operand is the first operand in the list. + Value *getCondition() { return getOperand(0); } + + /// Return the destination if the condition is true. + Block *getTrueDest() { + return getOperation()->getSuccessor(trueIndex); + } + + /// Return the destination if the condition is false. + Block *getFalseDest() { + return getOperation()->getSuccessor(falseIndex); + } + + // Accessors for operands to the 'true' destination. + Value *getTrueOperand(unsigned idx) { + assert(idx < getNumTrueOperands()); + return getOperand(getTrueDestOperandIndex() + idx); + } + + void setTrueOperand(unsigned idx, Value *value) { + assert(idx < getNumTrueOperands()); + setOperand(getTrueDestOperandIndex() + idx, value); + } + + operand_iterator true_operand_begin() { + return operand_begin() + getTrueDestOperandIndex(); + } + operand_iterator true_operand_end() { + return true_operand_begin() + getNumTrueOperands(); + } + operand_range getTrueOperands() { + return {true_operand_begin(), true_operand_end()}; + } + + unsigned getNumTrueOperands() { + return getOperation()->getNumSuccessorOperands(trueIndex); + } + + /// Erase the operand at 'index' from the true operand list. + void eraseTrueOperand(unsigned index) { + getOperation()->eraseSuccessorOperand(trueIndex, index); + } + + // Accessors for operands to the 'false' destination. + Value *getFalseOperand(unsigned idx) { + assert(idx < getNumFalseOperands()); + return getOperand(getFalseDestOperandIndex() + idx); + } + void setFalseOperand(unsigned idx, Value *value) { + assert(idx < getNumFalseOperands()); + setOperand(getFalseDestOperandIndex() + idx, value); + } + + operand_iterator false_operand_begin() { return true_operand_end(); } + operand_iterator false_operand_end() { + return false_operand_begin() + getNumFalseOperands(); + } + operand_range getFalseOperands() { + return {false_operand_begin(), false_operand_end()}; + } + + unsigned getNumFalseOperands() { + return getOperation()->getNumSuccessorOperands(falseIndex); + } + + /// Erase the operand at 'index' from the false operand list. + void eraseFalseOperand(unsigned index) { + getOperation()->eraseSuccessorOperand(falseIndex, index); + } + + private: + /// Get the index of the first true destination operand. + unsigned getTrueDestOperandIndex() { return 1; } + + /// Get the index of the first false destination operand. + unsigned getFalseDestOperandIndex() { + return getTrueDestOperandIndex() + getNumTrueOperands(); + } + }]; + + let parser = [{ return parse$cppClass(parser, result); }]; + let printer = [{ return print$cppClass(p, *this); }]; +} + +def IREEInterpHL_CmpIOp : + IREEInterpHL_PureOp<"cmp_i", [SameOperandsAndResultShape, + AllTypesMatch<["lhs", "rhs"]>]> { + let arguments = (ins + I32Attr:$predicate, + IREEHL_IntMemRef:$lhs, + IREEHL_IntMemRef:$rhs + ); + let results = (outs IREEHL_BoolMemRef); +} + +def IREEInterpHL_CmpFOp : + IREEInterpHL_PureOp<"cmp_f", [SameOperandsAndResultShape, + AllTypesMatch<["lhs", "rhs"]>]> { + let arguments = (ins + I32Attr:$predicate, + IREEHL_FloatMemRef:$lhs, + IREEHL_FloatMemRef:$rhs + ); + let results = (outs IREEHL_BoolMemRef); +} + +// TODO(benvanik): make pure (when we can disable CSE). +def IREEInterpHL_AllocHeapOp : IREEInterpHL_Op<"alloc_heap"> { + // TODO(benvanik): attributes and args. + let arguments = (ins + Variadic:$dim_pieces + ); + let results = (outs + IREEHL_MemRef + ); +} + +def IREEInterpHL_DiscardOp : IREEInterpHL_Op<"discard"> { + let arguments = (ins IREEHL_MemRef); +} + +def IREEInterpHL_RankOp : IREEInterpHL_PureOp<"rank"> { + let arguments = (ins IREEHL_MemRef); + let results = (outs IREEHL_IntScalar); +} + +def IREEInterpHL_DimOp : IREEInterpHL_PureOp<"dim"> { + // TODO(benvanik) add dim attr (I32Attr:$dim) + let arguments = (ins IREEHL_MemRef); + let results = (outs IREEHL_IntScalar); +} + +def IREEInterpHL_ShapeOp : IREEInterpHL_PureOp<"shape"> { + let arguments = (ins IREEHL_MemRef); + let results = (outs IREEHL_1DIntMemRef); +} + +def IREEInterpHL_LengthOp : IREEInterpHL_PureOp<"length"> { + let arguments = (ins IREEHL_MemRef); + let results = (outs IREEHL_IndexScalar); +} + +def IREEInterpHL_SliceOp : + IREEInterpHL_PureOp<"slice", [AllElementTypesMatch<["src", "result"]>, + AllTypesMatch<["srcIndices", "lengths"]>]> { + let arguments = (ins + IREEHL_MemRef:$src, + IREEHL_1DIndexMemRef:$srcIndices, + IREEHL_1DIndexMemRef:$lengths + ); + let results = (outs IREEHL_MemRef:$result); +} + +def IREEInterpHL_CopyOp : IREEInterpHL_Op<"copy"> { + let arguments = (ins + IREEHL_MemRef:$src, + IREEHL_1DIndexMemRef:$srcIndices, + IREEHL_MemRef:$dst, + IREEHL_1DIndexMemRef:$dstIndices, + IREEHL_1DIndexMemRef:$lengths + ); +} + +def IREEInterpHL_CloneOp : + IREEInterpHL_PureOp<"clone", [SameOperandsAndResultType]> { + let arguments = (ins IREEHL_MemRef:$src); + let results = (outs IREEHL_MemRef); +} + +// A pseudo op provided for convenience. This gets canonicalized to a series of +// copies. +def IREEInterpHL_ConcatOp : IREEInterpHL_PureOp<"concat"> { + let arguments = (ins + Variadic:$srcs, + I32Attr:$dimension + ); + let results = (outs IREEHL_MemRef); + + let hasCanonicalizer = 1; +} + +// TODO(benvanik): add split dim/size/etc. Maybe make multiple ops? +def IREEInterpHL_SplitOp : + IREEInterpHL_PureOp<"split", [SameOperandsAndResultElementType]> { + let arguments = (ins IREEHL_MemRef:$src); + let results = (outs Variadic); +} + +def IREEInterpHL_AssignOp : + IREEInterpHL_PureOp<"assign", [SameOperandsAndResultType]> { + let arguments = (ins IREEHL_MemRef:$src); + let results = (outs IREEHL_MemRef:$result); +} + +def IREEInterpHL_CondAssignOp : + IREEInterpHL_PureOp<"cond_assign", + [AllTypesMatch<["lhs", "rhs", "result"]>]> { + let arguments = (ins + IREEHL_BoolScalar:$cond, + IREEHL_MemRef:$lhs, + IREEHL_MemRef:$rhs + ); + let results = (outs IREEHL_MemRef:$result); +} + +def IREEInterpHL_ReshapeOp : IREEInterpHL_PureOp<"reshape"> { + let arguments = (ins IREEHL_MemRef:$src, IREEHL_MemRef:$shape); + let results = (outs IREEHL_MemRef); +} + +def IREEInterpHL_SelectOp : + IREEInterpHL_PureOp<"select", [AllTypesMatch<["lhs", "rhs", "result"]>]> { + let arguments = (ins + IREEHL_BoolMemRef:$cond, + IREEHL_MemRef:$lhs, + IREEHL_MemRef:$rhs + ); + let results = (outs IREEHL_MemRef:$result); +} + +def IREEInterpHL_BroadcastOp : + IREEInterpHL_PureOp<"broadcast", + [AllElementTypesMatch<["operand", "result"]>]> { + let arguments = (ins + IREE_ScalarMemRefOf<[AnyType]>:$operand, + IREEHL_1DIntMemRef:$shape + ); + let results = (outs IREEHL_MemRef:$result); +} + +def IREEInterpHL_PadOp : + IREEInterpHL_PureOp< + "pad", [AllElementTypesMatch<["src", "result", "padding_value"]>]> { + let arguments = (ins + IREEHL_MemRef:$src, + IREEHL_AnyScalar:$padding_value, + IREEHL_1DIndexMemRef:$edge_padding_low, + IREEHL_1DIndexMemRef:$edge_padding_high, + IREEHL_1DIndexMemRef:$interior_padding + ); + + let results = (outs IREEHL_MemRef:$result); +} + +def IREEInterpHL_TileOp : + IREEInterpHL_PureOp<"tile", [AllElementTypesMatch<["operand", "result"]>]> { + let arguments = (ins + IREEHL_MemRef:$operand, + IREEHL_1DIntMemRef:$shape + ); + let results = (outs IREEHL_MemRef:$result); +} + +def IREEInterpHL_TransposeOp : + IREEInterpHL_PureOp<"transpose", [ + AllElementTypesMatch<["operand", "result"]>, + AllRanksMatch<["operand", "result"]>, + AllElementCountsMatch<["operand", "result"]> + ]> { + let arguments = (ins + IREEHL_MemRef:$operand, + IREEHL_1DIntMemRef:$permutation + ); + let results = (outs IREEHL_MemRef:$result); +} + +def IREEInterpHL_ReverseOp : + IREEInterpHL_PureOp<"reverse", [AllTypesMatch<["operand", "result"]>]> { + let arguments = (ins + IREEHL_MemRef:$operand, + IREEHL_1DIntMemRef:$dims + ); + let results = (outs IREEHL_MemRef:$result); +} + +class IREEInterpHL_UnaryElementwiseOp traits = []> : + IREEInterpHL_PureOp { + let arguments = (ins type); + let results = (outs type); +} + +class IREEInterpHL_UnaryElementwiseFloatOp traits = []> : + IREEInterpHL_UnaryElementwiseOp; + +class IREEInterpHL_UnaryElementwiseIntOp traits = []> : + IREEInterpHL_UnaryElementwiseOp; + +class IREEInterpHL_BinaryElementwiseOp traits> : + IREEInterpHL_PureOp { + let arguments = (ins type:$lhs, type:$rhs); + let results = (outs type); +} + +class IREEInterpHL_BinaryElementwiseFloatOp traits = []> : + IREEInterpHL_BinaryElementwiseOp; + +class IREEInterpHL_BinaryElementwiseIntOp traits = []> : + IREEInterpHL_BinaryElementwiseOp; + +class IREEInterpHL_TernaryOp traits = []> : + IREEInterpHL_PureOp { + let arguments = (ins type:$a, type:$b, type:$c); + let results = (outs type); +} + +// TODO(benvanik): add traits for broadcasting support. + +def IREEInterpHL_NotOp : IREEInterpHL_UnaryElementwiseIntOp<"not">; +def IREEInterpHL_AndOp : IREEInterpHL_BinaryElementwiseIntOp<"and">; +def IREEInterpHL_OrOp : IREEInterpHL_BinaryElementwiseIntOp<"or">; +def IREEInterpHL_XorOp : IREEInterpHL_BinaryElementwiseIntOp<"xor">; +def IREEInterpHL_ShiftLeftOp : IREEInterpHL_BinaryElementwiseIntOp<"sll">; +def IREEInterpHL_ShiftRightLogicalOp : IREEInterpHL_BinaryElementwiseIntOp<"srl">; +def IREEInterpHL_ShiftRightArithmeticOp : IREEInterpHL_BinaryElementwiseIntOp<"sra">; + +def IREEInterpHL_AddIOp : IREEInterpHL_BinaryElementwiseIntOp<"add_i">; +def IREEInterpHL_AddFOp : IREEInterpHL_BinaryElementwiseFloatOp<"add_f">; +def IREEInterpHL_SubIOp : IREEInterpHL_BinaryElementwiseIntOp<"sub_i">; +def IREEInterpHL_SubFOp : IREEInterpHL_BinaryElementwiseFloatOp<"sub_f">; +def IREEInterpHL_AbsIOp : IREEInterpHL_UnaryElementwiseIntOp<"abs_i">; +def IREEInterpHL_AbsFOp : IREEInterpHL_UnaryElementwiseFloatOp<"abs_f">; +def IREEInterpHL_MulIOp : IREEInterpHL_BinaryElementwiseIntOp<"mul_i">; +def IREEInterpHL_MulFOp : IREEInterpHL_BinaryElementwiseFloatOp<"mul_f">; +def IREEInterpHL_DivISOp : IREEInterpHL_BinaryElementwiseIntOp<"div_i_s">; +def IREEInterpHL_DivIUOp : IREEInterpHL_BinaryElementwiseIntOp<"div_i_u">; +def IREEInterpHL_DivFOp : IREEInterpHL_BinaryElementwiseFloatOp<"div_f">; +def IREEInterpHL_MulAddIOp : IREEInterpHL_TernaryOp<"madd_i", IREEHL_IntMemRef>; +def IREEInterpHL_MulAddFOp : IREEInterpHL_TernaryOp<"madd_f", IREEHL_FloatMemRef>; +def IREEInterpHL_ExpFOp : IREEInterpHL_UnaryElementwiseFloatOp<"exp_f">; +def IREEInterpHL_LogFOp : IREEInterpHL_UnaryElementwiseFloatOp<"log_f">; +def IREEInterpHL_RsqrtFOp : IREEInterpHL_UnaryElementwiseFloatOp<"rsqrt_f">; +def IREEInterpHL_CosFOp : IREEInterpHL_UnaryElementwiseFloatOp<"cos_f">; +def IREEInterpHL_SinFOp : IREEInterpHL_UnaryElementwiseFloatOp<"sin_f">; +def IREEInterpHL_TanhFOp : IREEInterpHL_UnaryElementwiseFloatOp<"tanh_f">; +def IREEInterpHL_Atan2FOp : IREEInterpHL_UnaryElementwiseFloatOp<"atan2_f">; + +def IREEInterpHL_MinISOp : IREEInterpHL_BinaryElementwiseIntOp<"min_i_s">; +def IREEInterpHL_MinIUOp : IREEInterpHL_BinaryElementwiseIntOp<"min_i_u">; +def IREEInterpHL_MinFOp : IREEInterpHL_BinaryElementwiseFloatOp<"min_f">; +def IREEInterpHL_MaxISOp : IREEInterpHL_BinaryElementwiseIntOp<"max_i_s">; +def IREEInterpHL_MaxIUOp : IREEInterpHL_BinaryElementwiseIntOp<"max_i_u">; +def IREEInterpHL_MaxFOp : IREEInterpHL_BinaryElementwiseFloatOp<"max_f">; +def IREEInterpHL_ClampFOp : IREEInterpHL_TernaryOp<"clamp_f", IREEHL_FloatMemRef>; +def IREEInterpHL_FloorFOp : IREEInterpHL_UnaryElementwiseFloatOp<"floor_f">; +def IREEInterpHL_CeilFOp : IREEInterpHL_UnaryElementwiseFloatOp<"ceil_f">; + +class IREEInterpHL_ConversionOp : + IREEInterpHL_PureOp { + let arguments = (ins inputType); + let results = (outs outputType); +} + +def IREEInterpHL_ConvertSSOp : + IREEInterpHL_ConversionOp<"convert_s_s", IREEHL_IntMemRef, + IREEHL_IntMemRef>; +def IREEInterpHL_ConvertSUOp : + IREEInterpHL_ConversionOp<"convert_s_u", IREEHL_IntMemRef, + IREEHL_IntMemRef>; +def IREEInterpHL_ConvertSFOp : + IREEInterpHL_ConversionOp<"convert_s_f", IREEHL_IntMemRef, + IREEHL_FloatMemRef>; + +def IREEInterpHL_ConvertUSOp : + IREEInterpHL_ConversionOp<"convert_u_s", IREEHL_IntMemRef, + IREEHL_IntMemRef>; +def IREEInterpHL_ConvertUUOp : + IREEInterpHL_ConversionOp<"convert_u_u", IREEHL_IntMemRef, + IREEHL_IntMemRef>; +def IREEInterpHL_ConvertUFOp : + IREEInterpHL_ConversionOp<"convert_u_f", IREEHL_IntMemRef, + IREEHL_FloatMemRef>; + +def IREEInterpHL_ConvertFSOp : + IREEInterpHL_ConversionOp<"convert_f_s", IREEHL_FloatMemRef, + IREEHL_IntMemRef>; +def IREEInterpHL_ConvertFUOp : + IREEInterpHL_ConversionOp<"convert_f_u", IREEHL_FloatMemRef, + IREEHL_IntMemRef>; +def IREEInterpHL_ConvertFFOp : + IREEInterpHL_ConversionOp<"convert_f_f", IREEHL_FloatMemRef, + IREEHL_FloatMemRef>; + +def IREEInterpHL_MatMulIOp : + IREEInterpHL_PureOp<"matmul_i", + [AllElementTypesMatch<["lhs", "rhs", "result"]>]> { + let arguments = (ins + IREEHL_IntMemRef:$lhs, + IREEHL_IntMemRef:$rhs, + IREEHL_IntMemRef:$multiplier_mantissa, + IREEHL_IntMemRef:$multiplier_exponent + ); + let results = (outs IREEHL_IntMemRef:$result); +} +def IREEInterpHL_MatMulFOp : + IREEInterpHL_PureOp<"matmul_f", [SameOperandsAndResultElementType]> { + let arguments = (ins + IREEHL_FloatMemRef:$lhs, + IREEHL_FloatMemRef:$rhs + ); + let results = (outs IREEHL_FloatMemRef); +} + +def IREEInterpHL_ReduceSumIOp : + IREEInterpHL_PureOp<"reduce_sum_i", + [AllElementTypesMatch<["src", "result", "init"]>]> { + let arguments = (ins + IREEHL_IntMemRef:$src, + IREEHL_IntMemRef:$init, + I32Attr:$dimension + ); + let results = (outs IREEHL_IntMemRef:$result); +} +def IREEInterpHL_ReduceSumFOp : + IREEInterpHL_PureOp<"reduce_sum_f", + [AllElementTypesMatch<["src", "result", "init"]>]> { + let arguments = (ins + IREEHL_FloatMemRef:$src, + IREEHL_FloatMemRef:$init, + I32Attr:$dimension + ); + let results = (outs IREEHL_FloatMemRef:$result); +} +def IREEInterpHL_ReduceMinIOp : + IREEInterpHL_PureOp<"reduce_min_i", + [AllElementTypesMatch<["src", "result", "init"]>]> { + let arguments = (ins + IREEHL_IntMemRef:$src, + IREEHL_IntMemRef:$init, + I32Attr:$dimension + ); + let results = (outs IREEHL_IntMemRef:$result); +} +def IREEInterpHL_ReduceMinFOp : + IREEInterpHL_PureOp<"reduce_min_f", + [AllElementTypesMatch<["src", "result", "init"]>]> { + let arguments = (ins + IREEHL_FloatMemRef:$src, + IREEHL_FloatMemRef:$init, + I32Attr:$dimension + ); + let results = (outs IREEHL_FloatMemRef:$result); +} +def IREEInterpHL_ReduceMaxIOp : + IREEInterpHL_PureOp<"reduce_max_i", + [AllElementTypesMatch<["src", "result", "init"]>]> { + let arguments = (ins + IREEHL_IntMemRef:$src, + IREEHL_IntMemRef:$init, + I32Attr:$dimension + ); + let results = (outs IREEHL_IntMemRef:$result); +} +def IREEInterpHL_ReduceMaxFOp : + IREEInterpHL_PureOp<"reduce_max_f", + [AllElementTypesMatch<["src", "result", "init"]>]> { + let arguments = (ins + IREEHL_FloatMemRef:$src, + IREEHL_FloatMemRef:$init, + I32Attr:$dimension + ); + let results = (outs IREEHL_FloatMemRef:$result); +} + +def IREEInterpHL_TraceOp : IREEInterpHL_Op<"trace"> { + let arguments = (ins Variadic:$srcs); +} + +def IREEInterpHL_CondBreakOp : IREEInterpHL_Op<"cond_break"> { + let arguments = (ins IREEHL_BoolScalar:$cond); +} + +def IREEInterpHL_BreakOp : IREEInterpHL_Op<"break">; + +#endif // IREE_INTERPRETER_HL_OPS diff --git a/compiler/IR/Interpreter/LLDialect.cpp b/compiler/IR/Interpreter/LLDialect.cpp new file mode 100644 index 000000000000..465f06c67815 --- /dev/null +++ b/compiler/IR/Interpreter/LLDialect.cpp @@ -0,0 +1,34 @@ +// Copyright 2019 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "third_party/mlir_edge/iree/compiler/IR/Interpreter/LLDialect.h" + +#include "third_party/llvm/llvm/include/llvm/Support/SourceMgr.h" +#include "third_party/mlir_edge/iree/compiler/IR/Interpreter/LLOps.h" + +namespace mlir { +namespace iree_compiler { + +IREELLInterpreterDialect::IREELLInterpreterDialect(MLIRContext* context) + : Dialect(getDialectNamespace(), context) { +#define GET_OP_LIST + addOperations< +#include "third_party/mlir_edge/iree/compiler/IR/Interpreter/LLOps.cpp.inc" + >(); +} + +static DialectRegistration iree_ll_interp_dialect; + +} // namespace iree_compiler +} // namespace mlir diff --git a/compiler/IR/Interpreter/LLDialect.h b/compiler/IR/Interpreter/LLDialect.h new file mode 100644 index 000000000000..e64728a3fd90 --- /dev/null +++ b/compiler/IR/Interpreter/LLDialect.h @@ -0,0 +1,32 @@ +// Copyright 2019 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#ifndef THIRD_PARTY_MLIR_EDGE_IREE_COMPILER_IR_INTERPRETER_LLDIALECT_H_ +#define THIRD_PARTY_MLIR_EDGE_IREE_COMPILER_IR_INTERPRETER_LLDIALECT_H_ + +#include "third_party/llvm/llvm/projects/google_mlir/include/mlir/IR/Dialect.h" + +namespace mlir { +namespace iree_compiler { + +class IREELLInterpreterDialect : public Dialect { + public: + explicit IREELLInterpreterDialect(MLIRContext* context); + static StringRef getDialectNamespace() { return "iree_ll_interp"; } +}; + +} // namespace iree_compiler +} // namespace mlir + +#endif // THIRD_PARTY_MLIR_EDGE_IREE_COMPILER_IR_INTERPRETER_LLDIALECT_H_ diff --git a/compiler/IR/Interpreter/LLOps.cpp b/compiler/IR/Interpreter/LLOps.cpp new file mode 100644 index 000000000000..3f645794fc16 --- /dev/null +++ b/compiler/IR/Interpreter/LLOps.cpp @@ -0,0 +1,229 @@ +// Copyright 2019 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "third_party/mlir_edge/iree/compiler/IR/Interpreter/LLOps.h" + +#include "third_party/llvm/llvm/projects/google_mlir/include/mlir/IR/Builders.h" +#include "third_party/llvm/llvm/projects/google_mlir/include/mlir/IR/Function.h" +#include "third_party/llvm/llvm/projects/google_mlir/include/mlir/IR/OpImplementation.h" + +namespace mlir { +namespace iree_compiler { +namespace IREEInterp { +namespace LL { + +//===----------------------------------------------------------------------===// +// iree_ll_interp.call +//===----------------------------------------------------------------------===// + +static ParseResult parseCallOp(OpAsmParser *parser, OperationState *state) { + SymbolRefAttr calleeAttr; + FunctionType calleeType; + SmallVector operands; + auto calleeLoc = parser->getNameLoc(); + if (parser->parseAttribute(calleeAttr, "callee", state->attributes) || + parser->parseOperandList(operands, OpAsmParser::Delimiter::Paren) || + parser->parseOptionalAttributeDict(state->attributes) || + parser->parseColonType(calleeType) || + parser->addTypesToList(calleeType.getResults(), state->types) || + parser->resolveOperands(operands, calleeType.getInputs(), calleeLoc, + state->operands)) { + return failure(); + } + return success(); +} + +static void printCallOp(OpAsmPrinter *p, CallOp op) { + *p << "iree_ll_interp.call " << op.getAttr("callee") << '('; + p->printOperands(op.getOperands()); + *p << ')'; + p->printOptionalAttrDict(op.getAttrs(), /*elidedAttrs=*/{"callee"}); + *p << " : "; + p->printType(op.getCalleeType()); +} + +FunctionType CallOp::getCalleeType() { + SmallVector resultTypes(getResultTypes()); + SmallVector argTypes(getOperandTypes()); + return FunctionType::get(argTypes, resultTypes, getContext()); +} + +//===----------------------------------------------------------------------===// +// iree_ll_interp.call_import +//===----------------------------------------------------------------------===// + +static ParseResult parseCallImportOp(OpAsmParser *parser, + OperationState *state) { + SymbolRefAttr calleeAttr; + FunctionType calleeType; + SmallVector operands; + auto calleeLoc = parser->getNameLoc(); + if (parser->parseAttribute(calleeAttr, "callee", state->attributes) || + parser->parseOperandList(operands, OpAsmParser::Delimiter::Paren) || + parser->parseOptionalAttributeDict(state->attributes) || + parser->parseColonType(calleeType) || + parser->addTypesToList(calleeType.getResults(), state->types) || + parser->resolveOperands(operands, calleeType.getInputs(), calleeLoc, + state->operands)) { + return failure(); + } + return success(); +} + +static void printCallImportOp(OpAsmPrinter *p, CallImportOp op) { + *p << "iree_ll_interp.call_import " << op.getAttr("callee") << '('; + p->printOperands(op.getOperands()); + *p << ')'; + p->printOptionalAttrDict(op.getAttrs(), /*elidedAttrs=*/{"callee"}); + *p << " : "; + p->printType(op.getCalleeType()); +} + +FunctionType CallImportOp::getCalleeType() { + SmallVector resultTypes(getResultTypes()); + SmallVector argTypes(getOperandTypes()); + return FunctionType::get(argTypes, resultTypes, getContext()); +} + +//===----------------------------------------------------------------------===// +// iree_ll_interp.call_indirect +//===----------------------------------------------------------------------===// + +static ParseResult parseCallIndirectOp(OpAsmParser *parser, + OperationState *result) { + FunctionType calleeType; + OpAsmParser::OperandType callee; + llvm::SMLoc operandsLoc; + SmallVector operands; + return failure( + parser->parseOperand(callee) || + parser->getCurrentLocation(&operandsLoc) || + parser->parseOperandList(operands, OpAsmParser::Delimiter::Paren) || + parser->parseOptionalAttributeDict(result->attributes) || + parser->parseColonType(calleeType) || + parser->resolveOperand(callee, calleeType, result->operands) || + parser->resolveOperands(operands, calleeType.getInputs(), operandsLoc, + result->operands) || + parser->addTypesToList(calleeType.getResults(), result->types)); +} + +static void printCallIndirectOp(OpAsmPrinter *p, CallIndirectOp op) { + *p << "iree_ll_interp.call_indirect "; + p->printOperand(op.getCallee()); + *p << '('; + auto operandRange = op.getOperands(); + p->printOperands(++operandRange.begin(), operandRange.end()); + *p << ')'; + p->printOptionalAttrDict(op.getAttrs(), /*elidedAttrs=*/{"callee"}); + *p << " : " << op.getCallee()->getType(); +} + +//===----------------------------------------------------------------------===// +// iree_ll_interp.return +//===----------------------------------------------------------------------===// + +static ParseResult parseReturnOp(OpAsmParser *parser, OperationState *state) { + SmallVector opInfo; + SmallVector types; + llvm::SMLoc loc = parser->getCurrentLocation(); + return failure(parser->parseOperandList(opInfo) || + (!opInfo.empty() && parser->parseColonTypeList(types)) || + parser->resolveOperands(opInfo, types, loc, state->operands)); +} + +static void printReturnOp(OpAsmPrinter *p, ReturnOp op) { + *p << "iree_ll_interp.return"; + if (op.getNumOperands() > 0) { + *p << ' '; + p->printOperands(op.operand_begin(), op.operand_end()); + *p << " : "; + interleaveComma(op.getOperandTypes(), *p); + } +} + +//===----------------------------------------------------------------------===// +// iree_ll_interp.br +//===----------------------------------------------------------------------===// + +static ParseResult parseBranchOp(OpAsmParser *parser, OperationState *result) { + Block *dest; + SmallVector destOperands; + if (parser->parseSuccessorAndUseList(dest, destOperands)) return failure(); + result->addSuccessor(dest, destOperands); + return success(); +} + +static void printBranchOp(OpAsmPrinter *p, BranchOp op) { + *p << "iree_ll_interp.br "; + p->printSuccessorAndUseList(op.getOperation(), 0); +} + +Block *BranchOp::getDest() { return getOperation()->getSuccessor(0); } + +void BranchOp::setDest(Block *block) { + return getOperation()->setSuccessor(block, 0); +} + +void BranchOp::eraseOperand(unsigned index) { + getOperation()->eraseSuccessorOperand(0, index); +} + +//===----------------------------------------------------------------------===// +// iree_ll_interp.cond_br +//===----------------------------------------------------------------------===// + +static ParseResult parseCondBranchOp(OpAsmParser *parser, + OperationState *result) { + SmallVector destOperands; + Block *dest; + OpAsmParser::OperandType condInfo; + + // Parse the condition. + Type int1Ty = parser->getBuilder().getI1Type(); + if (parser->parseOperand(condInfo) || parser->parseComma() || + parser->resolveOperand(condInfo, int1Ty, result->operands)) { + return parser->emitError(parser->getNameLoc(), + "expected condition type was boolean (i1)"); + } + + // Parse the true successor. + if (parser->parseSuccessorAndUseList(dest, destOperands)) return failure(); + result->addSuccessor(dest, destOperands); + + // Parse the false successor. + destOperands.clear(); + if (parser->parseComma() || + parser->parseSuccessorAndUseList(dest, destOperands)) + return failure(); + result->addSuccessor(dest, destOperands); + + return success(); +} + +static void printCondBranchOp(OpAsmPrinter *p, CondBranchOp op) { + *p << "iree_ll_interp.cond_br "; + p->printOperand(op.getCondition()); + *p << ", "; + p->printSuccessorAndUseList(op.getOperation(), CondBranchOp::trueIndex); + *p << ", "; + p->printSuccessorAndUseList(op.getOperation(), CondBranchOp::falseIndex); +} + +#define GET_OP_CLASSES +#include "third_party/mlir_edge/iree/compiler/IR/Interpreter/LLOps.cpp.inc" + +} // namespace LL +} // namespace IREEInterp +} // namespace iree_compiler +} // namespace mlir diff --git a/compiler/IR/Interpreter/LLOps.h b/compiler/IR/Interpreter/LLOps.h new file mode 100644 index 000000000000..eb410398fb1e --- /dev/null +++ b/compiler/IR/Interpreter/LLOps.h @@ -0,0 +1,38 @@ +// Copyright 2019 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#ifndef THIRD_PARTY_MLIR_EDGE_IREE_COMPILER_IR_INTERPRETER_LLOPS_H_ +#define THIRD_PARTY_MLIR_EDGE_IREE_COMPILER_IR_INTERPRETER_LLOPS_H_ + +#include "third_party/llvm/llvm/projects/google_mlir/include/mlir/Dialect/StandardOps/Ops.h" +#include "third_party/llvm/llvm/projects/google_mlir/include/mlir/IR/Attributes.h" +#include "third_party/llvm/llvm/projects/google_mlir/include/mlir/IR/Dialect.h" +#include "third_party/llvm/llvm/projects/google_mlir/include/mlir/IR/OpDefinition.h" +#include "third_party/llvm/llvm/projects/google_mlir/include/mlir/IR/StandardTypes.h" +#include "third_party/llvm/llvm/projects/google_mlir/include/mlir/IR/TypeUtilities.h" + +namespace mlir { +namespace iree_compiler { +namespace IREEInterp { +namespace LL { + +#define GET_OP_CLASSES +#include "third_party/mlir_edge/iree/compiler/IR/Interpreter/LLOps.h.inc" + +} // namespace LL +} // namespace IREEInterp +} // namespace iree_compiler +} // namespace mlir + +#endif // THIRD_PARTY_MLIR_EDGE_IREE_COMPILER_IR_INTERPRETER_LLOPS_H_ diff --git a/compiler/IR/Interpreter/LLOps.td b/compiler/IR/Interpreter/LLOps.td new file mode 100644 index 000000000000..fc2135e52307 --- /dev/null +++ b/compiler/IR/Interpreter/LLOps.td @@ -0,0 +1,629 @@ +// IREE low-level interpreter op definitions. +// These map 1:1 with the bytecode, accept only MemRef types and generally use +// output parameters instead of return types. +// +// The source of truth for bytecode opcodes is: +// iree/schemas/bytecode/interpreter_bytecode_v0.h + +#ifdef IREE_INTERPRETER_LL_OPS +#else +#define IREE_INTERPRETER_LL_OPS + +#ifdef IREE_OP_BASE +#else +include "third_party/mlir_edge/iree/compiler/IR/OpBase.td" +#endif // IREE_OP_BASE + +def IREEInterpLL_Dialect : Dialect { + let name = "iree_ll_interp"; + let cppNamespace = "IREEInterp::LL"; +} + +//===----------------------------------------------------------------------===// +// Base op classes +//===----------------------------------------------------------------------===// + +class IREEInterpLL_Op traits = []> : + Op; + +class IREEInterpLL_PureOp traits = []> : + IREEInterpLL_Op; + +class IREEInterpLL_UnaryOp traits = []> : IREEInterpLL_Op { + let arguments = (ins type:$input, type:$dst); +} + +class IREEInterpLL_BinaryOp traits = []> : IREEInterpLL_Op { + let arguments = (ins type:$lhs, type:$rhs, type:$dst); +} + +class IREEInterpLL_TernaryOp traits = []> + : IREEInterpLL_Op { + let arguments = (ins type : $a, type : $b, type : $c, type : $dst); +} + +//===----------------------------------------------------------------------===// +// Low-level interpreter ops +//===----------------------------------------------------------------------===// + +// TODO(benvanik): value attribute. +def IREEInterpLL_ConstantOp : IREEInterpLL_PureOp<"constant"> { + let results = (outs IREELL_MemRef); +} + +def IREEInterpLL_CallOp : IREEInterpLL_Op<"call"> { + let arguments = (ins SymbolRefAttr:$callee, Variadic); + let results = (outs Variadic); + + let builders = [OpBuilder< + "Builder *builder, OperationState *result, FuncOp callee," + "ArrayRef operands = {}", [{ + result->addOperands(operands); + result->addAttribute("callee", builder->getSymbolRefAttr(callee)); + result->addTypes(callee.getType().getResults()); + }]>, OpBuilder< + "Builder *builder, OperationState *result, StringRef callee," + "ArrayRef results, ArrayRef operands = {}", [{ + result->addOperands(operands); + result->addAttribute("callee", builder->getSymbolRefAttr(callee)); + result->addTypes(results); + }]>]; + + let extraClassDeclaration = [{ + // TODO(b/132296600): make tablegen follow the style guide. + StringRef getCallee() { return callee(); } + FunctionType getCalleeType(); + + // TODO(b/133879130): make tablegen support variadic operand accessors. + /// Get the argument operands to the called function. + operand_range getArgOperands() { + return {arg_operand_begin(), arg_operand_end()}; + } + + operand_iterator arg_operand_begin() { return operand_begin(); } + operand_iterator arg_operand_end() { return operand_end(); } + }]; + + let parser = [{ return parse$cppClass(parser, result); }]; + let printer = [{ return print$cppClass(p, *this); }]; +} + +// TODO(benvanik): add verifier that target isExternal. +def IREEInterpLL_CallImportOp : IREEInterpLL_Op<"call_import"> { + let arguments = (ins SymbolRefAttr:$callee, Variadic); + let results = (outs Variadic); + + let builders = [OpBuilder< + "Builder *builder, OperationState *result, FuncOp callee," + "ArrayRef operands = {}", [{ + result->addOperands(operands); + result->addAttribute("callee", builder->getSymbolRefAttr(callee)); + result->addTypes(callee.getType().getResults()); + }]>, OpBuilder< + "Builder *builder, OperationState *result, StringRef callee," + "ArrayRef results, ArrayRef operands = {}", [{ + result->addOperands(operands); + result->addAttribute("callee", builder->getSymbolRefAttr(callee)); + result->addTypes(results); + }]>]; + + let extraClassDeclaration = [{ + // TODO(b/132296600): make tablegen follow the style guide. + StringRef getCallee() { return callee(); } + FunctionType getCalleeType(); + + // TODO(b/133879130): make tablegen support variadic operand accessors. + /// Get the argument operands to the called function. + operand_range getArgOperands() { + return {arg_operand_begin(), arg_operand_end()}; + } + + operand_iterator arg_operand_begin() { return operand_begin(); } + operand_iterator arg_operand_end() { return operand_end(); } + }]; + + let parser = [{ return parse$cppClass(parser, result); }]; + let printer = [{ return print$cppClass(p, *this); }]; +} + +def IREEInterpLL_CallIndirectOp : IREEInterpLL_Op<"call_indirect"> { + let arguments = (ins FunctionType:$callee, Variadic:$operands); + let results = (outs Variadic); + + let builders = [OpBuilder< + "Builder *, OperationState *result, Value *callee," + "ArrayRef operands = {}", [{ + result->operands.push_back(callee); + result->addOperands(operands); + result->addTypes(callee->getType().cast().getResults()); + }]>]; + + let extraClassDeclaration = [{ + Value *getCallee() { return getOperand(0); } + + /// Get the argument operands to the called function. + operand_range getArgOperands() { + return {arg_operand_begin(), arg_operand_end()}; + } + + operand_iterator arg_operand_begin() { return ++operand_begin(); } + operand_iterator arg_operand_end() { return operand_end(); } + }]; + + let parser = [{ return parse$cppClass(parser, result); }]; + let printer = [{ return print$cppClass(p, *this); }]; +} + +def IREEInterpLL_ReturnOp : IREEInterpLL_Op<"return", [Terminator]> { + let arguments = (ins Variadic:$operands); + + let builders = [OpBuilder< + "Builder *b, OperationState *result", [{ build(b, result, llvm::None); }] + >]; + + let parser = [{ return parse$cppClass(parser, result); }]; + let printer = [{ return print$cppClass(p, *this); }]; +} + +def IREEInterpLL_BranchOp : IREEInterpLL_Op<"br", [Terminator]> { + let arguments = (ins Variadic:$operands); + + let builders = [OpBuilder< + "Builder *, OperationState *result, Block *dest," + "ArrayRef operands = {}", [{ + result->addSuccessor(dest, operands); + }]>]; + + let extraClassDeclaration = [{ + Block *getDest(); + void setDest(Block *block); + + /// Erase the operand at 'index' from the operand list. + void eraseOperand(unsigned index); + }]; + + let parser = [{ return parse$cppClass(parser, result); }]; + let printer = [{ return print$cppClass(p, *this); }]; +} + +def IREEInterpLL_CondBranchOp : IREEInterpLL_Op<"cond_br", [Terminator]> { + let arguments = (ins + IREELL_BoolScalar:$condition, + Variadic:$branchOperands + ); + + let builders = [OpBuilder< + "Builder *, OperationState *result, Value *condition," + "Block *trueDest, ArrayRef trueOperands," + "Block *falseDest, ArrayRef falseOperands", [{ + result->addOperands(condition); + result->addSuccessor(trueDest, trueOperands); + result->addSuccessor(falseDest, falseOperands); + }]>]; + + let extraClassDeclaration = [{ + // These are the indices into the dests list. + enum { trueIndex = 0, falseIndex = 1 }; + + // The condition operand is the first operand in the list. + Value *getCondition() { return getOperand(0); } + + /// Return the destination if the condition is true. + Block *getTrueDest() { + return getOperation()->getSuccessor(trueIndex); + } + + /// Return the destination if the condition is false. + Block *getFalseDest() { + return getOperation()->getSuccessor(falseIndex); + } + + // Accessors for operands to the 'true' destination. + Value *getTrueOperand(unsigned idx) { + assert(idx < getNumTrueOperands()); + return getOperand(getTrueDestOperandIndex() + idx); + } + + void setTrueOperand(unsigned idx, Value *value) { + assert(idx < getNumTrueOperands()); + setOperand(getTrueDestOperandIndex() + idx, value); + } + + operand_iterator true_operand_begin() { + return operand_begin() + getTrueDestOperandIndex(); + } + operand_iterator true_operand_end() { + return true_operand_begin() + getNumTrueOperands(); + } + operand_range getTrueOperands() { + return {true_operand_begin(), true_operand_end()}; + } + + unsigned getNumTrueOperands() { + return getOperation()->getNumSuccessorOperands(trueIndex); + } + + /// Erase the operand at 'index' from the true operand list. + void eraseTrueOperand(unsigned index) { + getOperation()->eraseSuccessorOperand(trueIndex, index); + } + + // Accessors for operands to the 'false' destination. + Value *getFalseOperand(unsigned idx) { + assert(idx < getNumFalseOperands()); + return getOperand(getFalseDestOperandIndex() + idx); + } + void setFalseOperand(unsigned idx, Value *value) { + assert(idx < getNumFalseOperands()); + setOperand(getFalseDestOperandIndex() + idx, value); + } + + operand_iterator false_operand_begin() { return true_operand_end(); } + operand_iterator false_operand_end() { + return false_operand_begin() + getNumFalseOperands(); + } + operand_range getFalseOperands() { + return {false_operand_begin(), false_operand_end()}; + } + + unsigned getNumFalseOperands() { + return getOperation()->getNumSuccessorOperands(falseIndex); + } + + /// Erase the operand at 'index' from the false operand list. + void eraseFalseOperand(unsigned index) { + getOperation()->eraseSuccessorOperand(falseIndex, index); + } + + private: + /// Get the index of the first true destination operand. + unsigned getTrueDestOperandIndex() { return 1; } + + /// Get the index of the first false destination operand. + unsigned getFalseDestOperandIndex() { + return getTrueDestOperandIndex() + getNumTrueOperands(); + } + }]; + + let parser = [{ return parse$cppClass(parser, result); }]; + let printer = [{ return print$cppClass(p, *this); }]; +} + +def IREEInterpLL_CmpIOp : IREEInterpLL_Op<"cmp_i"> { + let arguments = (ins + I32Attr:$predicate, + IREELL_IntMemRef:$lhs, + IREELL_IntMemRef:$rhs, + IREELL_BoolMemRef:$dst + ); +} + +def IREEInterpLL_CmpFOp : IREEInterpLL_Op<"cmp_f"> { + let arguments = (ins + I32Attr:$predicate, + IREELL_FloatMemRef:$lhs, + IREELL_FloatMemRef:$rhs, + IREELL_BoolMemRef:$dst + ); +} + +def IREEInterpLL_AllocStaticOp : IREEInterpLL_PureOp<"alloc_static"> { + // TODO(benvanik): attributes and args. + let results = (outs IREELL_MemRef); +} + +def IREEInterpLL_AllocStackOp : IREEInterpLL_PureOp<"alloc_stack"> { + // TODO(benvanik): atributes and args. + let arguments = (ins + Variadic:$dim_pieces + ); + let results = (outs + IREELL_MemRef + ); +} + +def IREEInterpLL_AllocStackInitOp : IREEInterpLL_PureOp<"alloc_stack_init"> { + // TODO(benvanik): attributes and args. + let arguments = (ins + Variadic:$dim_pieces + ); + let results = (outs + IREELL_MemRef + ); +} + +// TODO(benvanik): make pure (when we can disable CSE). +def IREEInterpLL_AllocHeapOp : IREEInterpLL_Op<"alloc_heap"> { + // TODO(benvanik): attributes and args. + let arguments = (ins + Variadic:$dim_pieces + ); + let results = (outs + IREELL_MemRef + ); +} + +def IREEInterpLL_DiscardOp : IREEInterpLL_Op<"discard"> { + let arguments = (ins IREELL_MemRef); +} + +def IREEInterpLL_RankOp : IREEInterpLL_Op<"rank"> { + let arguments = (ins + IREELL_MemRef:$input, + IREELL_I32Scalar:$dst + ); +} + +def IREEInterpLL_DimOp : IREEInterpLL_Op<"dim"> { + // TODO(benvanik) add dim attr (I32Attr:$dim) + let arguments = (ins + IREELL_MemRef:$input, + IREELL_I32Scalar:$dst + ); +} + +def IREEInterpLL_ShapeOp : IREEInterpLL_Op<"shape"> { + let arguments = (ins + IREELL_MemRef:$input, + IREELL_I32MemRef:$dst + ); +} + +def IREEInterpLL_LengthOp : IREEInterpLL_Op<"length"> { + let arguments = (ins + IREELL_MemRef:$input, + IREELL_I32Scalar:$dst + ); +} + + +def IREEInterpLL_DynamicSliceOp : IREEInterpLL_PureOp<"dynamic_slice"> { + let arguments = (ins + IREELL_MemRef:$src, + IREELL_1DIndexMemRef:$srcIndices, + IREELL_1DIndexMemRef:$lengths + ); + let results = (outs + IREELL_MemRef + ); +} + +// TODO(benvanik): add attribute requirements/types. +def IREEInterpLL_StaticSliceOp : + IREEInterpLL_PureOp<"static_slice", [SameOperandsAndResultElementType]> { + let arguments = (ins IREELL_MemRef:$src); + let results = (outs IREELL_MemRef); +} + +def IREEInterpLL_DynamicCopyOp : IREEInterpLL_Op<"dynamic_copy"> { + let arguments = (ins + IREELL_MemRef:$src, + IREELL_1DIndexMemRef:$srcIndices, + IREELL_MemRef:$dst, + IREELL_1DIndexMemRef:$dstIndices, + IREELL_1DIndexMemRef:$lengths + ); +} + +def IREEInterpLL_StaticCopyOp : IREEInterpLL_Op<"static_copy"> { + let arguments = (ins + IREELL_MemRef:$src, + I32ElementsAttr:$srcIndices, + IREELL_MemRef:$dst, + I32ElementsAttr:$dstIndices, + I32ElementsAttr:$lengths + ); +} + +def IREEInterpLL_CloneOp : + IREEInterpLL_PureOp<"clone", [SameOperandsAndResultType]> { + let arguments = (ins IREELL_MemRef:$src); + let results = (outs IREELL_MemRef); +} + +// TODO(benvanik): add split dim/size/etc. Maybe make multiple ops? +def IREEInterpLL_SplitOp : IREEInterpLL_PureOp<"split"> { + let arguments = (ins + IREELL_MemRef:$src + ); + let results = (outs + Variadic + ); +} + +def IREEInterpLL_AssignOp : + IREEInterpLL_Op<"assign", [SameOperandsAndResultType]> { + let arguments = (ins IREELL_MemRef:$src); + let results = (outs IREELL_MemRef); +} + +def IREEInterpLL_CondAssignOp : IREEInterpLL_Op<"cond_assign"> { + let arguments = (ins + IREELL_BoolScalar:$cond, + IREELL_MemRef:$lhs, + IREELL_MemRef:$rhs + ); + let results = (outs + IREELL_MemRef + ); +} + +def IREEInterpLL_ReshapeOp : IREEInterpLL_Op<"reshape"> { + let arguments = (ins + IREELL_MemRef:$input, + IREELL_1DIntMemRef:$shape + ); + let results = (outs + IREELL_MemRef + ); +} + +def IREEInterpLL_SelectOp : IREEInterpLL_Op<"select"> { + let arguments = (ins + IREELL_MemRef:$cond, + IREELL_MemRef:$lhs, + IREELL_MemRef:$rhs, + IREELL_MemRef:$dst + ); +} + +def IREEInterpLL_PadOp : + IREEInterpLL_Op< + "pad", [AllElementTypesMatch<["src", "dst", "padding_value"]>]> { + let arguments = (ins + IREELL_MemRef:$src, + IREELL_ElementScalar:$padding_value, + IREELL_1DIndexMemRef:$edge_padding_low, + IREELL_1DIndexMemRef:$edge_padding_high, + IREELL_1DIndexMemRef:$interior_padding, + IREELL_MemRef:$dst + ); +} + +def IREEInterpLL_TransposeOp : IREEInterpLL_BinaryOp<"transpose">; + +def IREEInterPLL_ReverseOp : IREEInterpLL_BinaryOp<"reverse">; + +def IREEInterpLL_BroadcastOp : IREEInterpLL_BinaryOp<"broadcast">; + +def IREEInterpLL_TileOp : IREEInterpLL_BinaryOp<"tile">; + +// TODO(benvanik): add traits for broadcasting support. + +def IREEInterpLL_NotOp : IREEInterpLL_UnaryOp<"not">; +def IREEInterpLL_AndOp : IREEInterpLL_BinaryOp<"and">; +def IREEInterpLL_OrOp : IREEInterpLL_BinaryOp<"or">; +def IREEInterpLL_XorOp : IREEInterpLL_BinaryOp<"xor">; +def IREEInterpLL_ShiftLeftOp : IREEInterpLL_BinaryOp<"sll">; +def IREEInterpLL_ShiftRightLogicalOp : IREEInterpLL_BinaryOp<"srl">; +def IREEInterpLL_ShiftRightArithmeticOp : IREEInterpLL_BinaryOp<"sra">; + +def IREEInterpLL_AddIOp : IREEInterpLL_BinaryOp<"add_i", IREELL_IntMemRef>; +def IREEInterpLL_AddFOp : IREEInterpLL_BinaryOp<"add_f", IREELL_FloatMemRef>; +def IREEInterpLL_SubIOp : IREEInterpLL_BinaryOp<"sub_i", IREELL_IntMemRef>; +def IREEInterpLL_SubFOp : IREEInterpLL_BinaryOp<"sub_f", IREELL_FloatMemRef>; +def IREEInterpLL_AbsIOp : IREEInterpLL_UnaryOp<"abs_i", IREELL_IntMemRef>; +def IREEInterpLL_AbsFOp : IREEInterpLL_UnaryOp<"abs_f", IREELL_FloatMemRef>; +def IREEInterpLL_MulIOp : IREEInterpLL_BinaryOp<"mul_i", IREELL_IntMemRef>; +def IREEInterpLL_MulFOp : IREEInterpLL_BinaryOp<"mul_f", IREELL_FloatMemRef>; +def IREEInterpLL_DivISOp : IREEInterpLL_BinaryOp<"div_i_s", IREELL_IntMemRef>; +def IREEInterpLL_DivIUOp : IREEInterpLL_BinaryOp<"div_i_u", IREELL_IntMemRef>; +def IREEInterpLL_DivFOp : IREEInterpLL_BinaryOp<"div_f", IREELL_FloatMemRef>; +def IREEInterpLL_MulAddIOp : IREEInterpLL_BinaryOp<"madd_i", IREELL_IntMemRef>; +def IREEInterpLL_MulAddFOp : IREEInterpLL_BinaryOp<"madd_f", IREELL_FloatMemRef>; +def IREEInterpLL_ExpFOp : IREEInterpLL_UnaryOp<"exp_f", IREELL_FloatMemRef>; +def IREEInterpLL_LogFOp : IREEInterpLL_UnaryOp<"log_f", IREELL_FloatMemRef>; +def IREEInterpLL_RsqrtFOp : IREEInterpLL_UnaryOp<"rsqrt_f", IREELL_FloatMemRef>; +def IREEInterpLL_CosFOp : IREEInterpLL_UnaryOp<"cos_f", IREELL_FloatMemRef>; +def IREEInterpLL_SinFOp : IREEInterpLL_UnaryOp<"sin_f", IREELL_FloatMemRef>; +def IREEInterpLL_TanhFOp : IREEInterpLL_UnaryOp<"tanh_f", IREELL_FloatMemRef>; +def IREEInterpLL_Atan2FOp : IREEInterpLL_UnaryOp<"atan2_f", IREELL_FloatMemRef>; + +def IREEInterpLL_MinISOp : IREEInterpLL_BinaryOp<"min_i_s", IREELL_IntMemRef>; +def IREEInterpLL_MinIUOp : IREEInterpLL_BinaryOp<"min_i_u", IREELL_IntMemRef>; +def IREEInterpLL_MinFOp : IREEInterpLL_BinaryOp<"min_f", IREELL_FloatMemRef>; +def IREEInterpLL_MaxISOp : IREEInterpLL_BinaryOp<"max_i_s", IREELL_IntMemRef>; +def IREEInterpLL_MaxIUOp : IREEInterpLL_BinaryOp<"max_i_u", IREELL_IntMemRef>; +def IREEInterpLL_MaxFOp : IREEInterpLL_BinaryOp<"max_f", IREELL_FloatMemRef>; +def IREEInterpLL_ClampFOp : IREEInterpLL_TernaryOp<"clamp_f", IREELL_FloatMemRef>; +def IREEInterpLL_FloorFOp : IREEInterpLL_UnaryOp<"floor_f", IREELL_FloatMemRef>; +def IREEInterpLL_CeilFOp : IREEInterpLL_UnaryOp<"ceil_f", IREELL_FloatMemRef>; + +def IREEInterpLL_ConvertSSOp : IREEInterpLL_UnaryOp<"convert_s_s", IREELL_MemRef>; +def IREEInterpLL_ConvertSUOp : IREEInterpLL_UnaryOp<"convert_s_u", IREELL_MemRef>; +def IREEInterpLL_ConvertSFOp : IREEInterpLL_UnaryOp<"convert_s_f", IREELL_MemRef>; + +def IREEInterpLL_ConvertUSOp : IREEInterpLL_UnaryOp<"convert_u_s", IREELL_MemRef>; +def IREEInterpLL_ConvertUUOp : IREEInterpLL_UnaryOp<"convert_u_u", IREELL_MemRef>; +def IREEInterpLL_ConvertUFOp : IREEInterpLL_UnaryOp<"convert_u_f", IREELL_MemRef>; + +def IREEInterpLL_ConvertFSOp : IREEInterpLL_UnaryOp<"convert_f_s", IREELL_MemRef>; +def IREEInterpLL_ConvertFUOp : IREEInterpLL_UnaryOp<"convert_f_u", IREELL_MemRef>; +def IREEInterpLL_ConvertFFOp : IREEInterpLL_UnaryOp<"convert_f_f", IREELL_MemRef>; + +def IREEInterpLL_MatMulIOp : IREEInterpLL_Op<"matmul_i"> { + let arguments = (ins + IREELL_IntMemRef:$lhs, + IREELL_IntMemRef:$rhs, + IREELL_IntMemRef:$multiplier_mantissa, + IREELL_IntMemRef:$multiplier_exponent, + IREELL_IntMemRef:$dst + ); +} +def IREEInterpLL_MatMulFOp : IREEInterpLL_Op<"matmul_f"> { + let arguments = (ins + IREELL_FloatMemRef:$lhs, + IREELL_FloatMemRef:$rhs, + IREELL_FloatMemRef:$dst + ); +} + +def IREEInterpLL_ReduceSumIOp : IREEInterpLL_Op<"reduce_sum_i"> { + let arguments = (ins + IREELL_IntMemRef:$src, + IREELL_IntMemRef:$init, + I32Attr:$dimension, + IREELL_IntMemRef:$dst + ); +} +def IREEInterpLL_ReduceSumFOp : IREEInterpLL_Op<"reduce_sum_f"> { + let arguments = (ins + IREELL_FloatMemRef:$src, + IREELL_FloatMemRef:$init, + I32Attr:$dimension, + IREELL_FloatMemRef:$dst + ); +} + +def IREEInterpLL_ReduceMinIOp : IREEInterpLL_Op<"reduce_min_i"> { + let arguments = (ins + IREELL_IntMemRef:$src, + IREELL_IntMemRef:$init, + I32Attr:$dimension, + IREELL_IntMemRef:$dst + ); +} +def IREEInterpLL_ReduceMinFOp : IREEInterpLL_Op<"reduce_min_f"> { + let arguments = (ins + IREELL_FloatMemRef:$src, + IREELL_FloatMemRef:$init, + I32Attr:$dimension, + IREELL_FloatMemRef:$dst + ); +} + +def IREEInterpLL_ReduceMaxIOp : IREEInterpLL_Op<"reduce_max_i"> { + let arguments = (ins + IREELL_IntMemRef:$src, + IREELL_IntMemRef:$init, + I32Attr:$dimension, + IREELL_IntMemRef:$dst + ); +} +def IREEInterpLL_ReduceMaxFOp : IREEInterpLL_Op<"reduce_max_f"> { + let arguments = (ins + IREELL_FloatMemRef:$src, + IREELL_FloatMemRef:$init, + I32Attr:$dimension, + IREELL_FloatMemRef:$dst + ); +} + +def IREEInterpLL_TraceOp : IREEInterpLL_Op<"trace"> { + let arguments = (ins + Variadic:$srcs + ); +} + +def IREEInterpLL_CondBreakOp : IREEInterpLL_Op<"cond_break"> { + let arguments = (ins + IREELL_BoolScalar:$cond + ); +} + +def IREEInterpLL_BreakOp : IREEInterpLL_Op<"break">; + +#endif // IREE_INTERPRETER_LL_OPS diff --git a/compiler/IR/Interpreter/OpWriters.cpp b/compiler/IR/Interpreter/OpWriters.cpp new file mode 100644 index 000000000000..ea4ba5a6d2ab --- /dev/null +++ b/compiler/IR/Interpreter/OpWriters.cpp @@ -0,0 +1,261 @@ +// Copyright 2019 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "third_party/mlir_edge/iree/compiler/IR/Interpreter/OpWriters.h" + +#include "third_party/llvm/llvm/projects/google_mlir/include/mlir/IR/Module.h" +#include "third_party/llvm/llvm/projects/google_mlir/include/mlir/IR/TypeUtilities.h" +#include "third_party/mlir_edge/iree/compiler/IR/Interpreter/LLOps.h" +#include "third_party/mlir_edge/iree/compiler/Serialization/BytecodeWriter.h" +#include "third_party/mlir_edge/iree/compiler/Utils/Macros.h" +#include "third_party/mlir_edge/iree/schemas/bytecode/interpreter_bytecode_v0.h" + +namespace mlir { +namespace iree_compiler { + +namespace { + +//===----------------------------------------------------------------------===// +// Sequencer ops +//===----------------------------------------------------------------------===// + +LogicalResult writeOp(IREEInterp::LL::ConstantOp op, BytecodeWriter *writer) { + RETURN_IF_FAILURE(writer->WriteOpcode(iree::InterpreterOpcode::kConstant)); + auto memrefType = op.getType().dyn_cast(); + if (!memrefType) { + return op.emitOpError() + << "Constant has an unsupported type; must be a memref: " + << op.getType(); + } + RETURN_IF_FAILURE(writer->WriteConstant(memrefType, op.getAttr("value"))); + RETURN_IF_FAILURE(writer->WriteLocal(op.getResult())); + return success(); +} + +LogicalResult writeOp(IREEInterp::LL::CallOp op, BytecodeWriter *writer) { + auto module = op.getOperation()->getParentOfType(); + auto callee = module.lookupSymbol(op.getCallee()); + // TODO(benvanik): transforms to convert Call->CallImport. + // TODO(benvanik): switch with kCallTail if attr exists. + if (callee.isExternal()) { + RETURN_IF_FAILURE( + writer->WriteOpcode(iree::InterpreterOpcode::kCallImport)); + } else { + RETURN_IF_FAILURE(writer->WriteOpcode(iree::InterpreterOpcode::kCall)); + } + RETURN_IF_FAILURE(writer->WriteFunctionOrdinal(callee)); + RETURN_IF_FAILURE(writer->WriteLocals(op.getArgOperands())); + RETURN_IF_FAILURE(writer->WriteLocals(op.getResults())); + return success(); +} + +LogicalResult writeOp(IREEInterp::LL::CallImportOp op, BytecodeWriter *writer) { + auto module = op.getOperation()->getParentOfType(); + auto callee = module.lookupSymbol(op.getCallee()); + // TODO(benvanik): switch with kCallTail if attr exists. + RETURN_IF_FAILURE(writer->WriteOpcode(iree::InterpreterOpcode::kCallImport)); + RETURN_IF_FAILURE(writer->WriteImportOrdinal(callee)); + RETURN_IF_FAILURE(writer->WriteLocals(op.getArgOperands())); + RETURN_IF_FAILURE(writer->WriteLocals(op.getResults())); + return success(); +} + +LogicalResult writeOp(IREEInterp::LL::CallIndirectOp op, + BytecodeWriter *writer) { + RETURN_IF_FAILURE( + writer->WriteOpcode(iree::InterpreterOpcode::kCallIndirect)); + RETURN_IF_FAILURE(writer->WriteTypeIndex(op.getCallee()->getType())); + RETURN_IF_FAILURE(writer->WriteLocal(op.getCallee())); + RETURN_IF_FAILURE(writer->WriteLocals(op.getArgOperands())); + RETURN_IF_FAILURE(writer->WriteLocals(op.getResults())); + return success(); +} + +LogicalResult WriteConvertOperands(Operation *op, BytecodeWriter *writer) { + auto *src = op->getOperand(0); + RETURN_IF_FAILURE( + writer->WriteTypeIndex(getElementTypeOrSelf(src->getType()))); + RETURN_IF_FAILURE(writer->WriteLocal(src)); + auto *dst = op->getOperand(1); + RETURN_IF_FAILURE( + writer->WriteTypeIndex(getElementTypeOrSelf(dst->getType()))); + RETURN_IF_FAILURE(writer->WriteLocal(dst)); + return success(); +} + +LogicalResult writeOp(IREEInterp::LL::ConvertSSOp op, BytecodeWriter *writer) { + RETURN_IF_FAILURE(writer->WriteOpcode(iree::InterpreterOpcode::kConvertSS)); + return WriteConvertOperands(op, writer); +} + +LogicalResult writeOp(IREEInterp::LL::ConvertUUOp op, BytecodeWriter *writer) { + RETURN_IF_FAILURE(writer->WriteOpcode(iree::InterpreterOpcode::kConvertUU)); + return WriteConvertOperands(op, writer); +} + +LogicalResult writeOp(IREEInterp::LL::ConvertSUOp op, BytecodeWriter *writer) { + RETURN_IF_FAILURE(writer->WriteOpcode(iree::InterpreterOpcode::kConvertSU)); + return WriteConvertOperands(op, writer); +} + +LogicalResult writeOp(IREEInterp::LL::ConvertUSOp op, BytecodeWriter *writer) { + RETURN_IF_FAILURE(writer->WriteOpcode(iree::InterpreterOpcode::kConvertUS)); + return WriteConvertOperands(op, writer); +} + +LogicalResult writeOp(IREEInterp::LL::BranchOp op, BytecodeWriter *writer) { + RETURN_IF_FAILURE(writer->WriteOpcode(iree::InterpreterOpcode::kBranch)); + RETURN_IF_FAILURE(writer->WriteBlockOffset(op.getDest())); + RETURN_IF_FAILURE(writer->WriteCount(op.getNumOperands())); + for (int i = 0; i < op.getNumOperands(); ++i) { + // Copy src->dst. + RETURN_IF_FAILURE(writer->WriteLocal(op.getOperand(i))); + RETURN_IF_FAILURE(writer->WriteLocal(op.getDest()->getArgument(i))); + } + return success(); +} + +LogicalResult writeOp(IREEInterp::LL::CondBranchOp op, BytecodeWriter *writer) { + RETURN_IF_FAILURE(writer->WriteOpcode(iree::InterpreterOpcode::kCondBranch)); + RETURN_IF_FAILURE(writer->WriteLocal(op.getCondition())); + RETURN_IF_FAILURE(writer->WriteBlockOffset(op.getTrueDest())); + RETURN_IF_FAILURE(writer->WriteCount(op.getNumTrueOperands())); + for (int i = 0; i < op.getNumTrueOperands(); ++i) { + // Copy src->dst. + RETURN_IF_FAILURE(writer->WriteLocal(op.getTrueOperand(i))); + RETURN_IF_FAILURE(writer->WriteLocal(op.getTrueDest()->getArgument(i))); + } + RETURN_IF_FAILURE(writer->WriteBlockOffset(op.getFalseDest())); + RETURN_IF_FAILURE(writer->WriteCount(op.getNumFalseOperands())); + for (int i = 0; i < op.getNumFalseOperands(); ++i) { + // Copy src->dst. + RETURN_IF_FAILURE(writer->WriteLocal(op.getFalseOperand(i))); + RETURN_IF_FAILURE(writer->WriteLocal(op.getFalseDest()->getArgument(i))); + } + return success(); +} + +LogicalResult writeOp(IREEInterp::LL::CmpIOp op, BytecodeWriter *writer) { + RETURN_IF_FAILURE(writer->WriteOpcode(iree::InterpreterOpcode::kCmpI)); + RETURN_IF_FAILURE( + writer->WriteUint8(static_cast(op.predicate().getZExtValue()))); + RETURN_IF_FAILURE(writer->WriteLocal(op.getOperand(0))); + RETURN_IF_FAILURE(writer->WriteLocal(op.getOperand(1))); + RETURN_IF_FAILURE(writer->WriteLocal(op.getOperand(2))); + return success(); +} + +LogicalResult writeOp(IREEInterp::LL::CmpFOp op, BytecodeWriter *writer) { + RETURN_IF_FAILURE(writer->WriteOpcode(iree::InterpreterOpcode::kCmpF)); + RETURN_IF_FAILURE( + writer->WriteUint8(static_cast(op.predicate().getZExtValue()))); + RETURN_IF_FAILURE(writer->WriteLocal(op.getOperand(0))); + RETURN_IF_FAILURE(writer->WriteLocal(op.getOperand(1))); + RETURN_IF_FAILURE(writer->WriteLocal(op.getOperand(2))); + return success(); +} + +LogicalResult writeOp(IREEInterp::LL::AllocHeapOp op, BytecodeWriter *writer) { + auto memrefType = op.getType().cast(); + RETURN_IF_FAILURE(writer->WriteOpcode(iree::InterpreterOpcode::kAllocHeap)); + RETURN_IF_FAILURE(writer->WriteInt32(0)); + RETURN_IF_FAILURE(writer->WriteTypeIndex(memrefType.getElementType())); + RETURN_IF_FAILURE(writer->WriteShapePieces(memrefType)); + RETURN_IF_FAILURE(writer->WriteLocals(op.getOperands())); + RETURN_IF_FAILURE(writer->WriteLocal(op.getResult())); + return success(); +} + +LogicalResult writeOp(IREEInterp::LL::StaticCopyOp op, BytecodeWriter *writer) { + RETURN_IF_FAILURE(writer->WriteOpcode(iree::InterpreterOpcode::kStaticCopy)); + RETURN_IF_FAILURE(writer->WriteLocal(op.src())); + RETURN_IF_FAILURE(writer->WriteShapePieces(op.srcIndices())); + RETURN_IF_FAILURE(writer->WriteLocal(op.dst())); + RETURN_IF_FAILURE(writer->WriteShapePieces(op.dstIndices())); + RETURN_IF_FAILURE(writer->WriteShapePieces(op.lengths())); + return success(); +} + +LogicalResult writeReduceOperands(Operation *op, BytecodeWriter *writer, + APInt dimension) { + RETURN_IF_FAILURE(writer->WriteLocal(op->getOperand(0))); + RETURN_IF_FAILURE(writer->WriteLocal(op->getOperand(1))); + RETURN_IF_FAILURE(writer->WriteInt32(dimension.getZExtValue())); + RETURN_IF_FAILURE(writer->WriteLocal(op->getOperand(2))); + return success(); +} + +LogicalResult writeOp(IREEInterp::LL::ReduceSumIOp op, BytecodeWriter *writer) { + RETURN_IF_FAILURE(writer->WriteOpcode(iree::InterpreterOpcode::kReduceSumI)); + return writeReduceOperands(op, writer, op.dimension()); +} + +LogicalResult writeOp(IREEInterp::LL::ReduceSumFOp op, BytecodeWriter *writer) { + RETURN_IF_FAILURE(writer->WriteOpcode(iree::InterpreterOpcode::kReduceSumF)); + return writeReduceOperands(op, writer, op.dimension()); +} + +LogicalResult writeOp(IREEInterp::LL::ReduceMinIOp op, BytecodeWriter *writer) { + RETURN_IF_FAILURE(writer->WriteOpcode(iree::InterpreterOpcode::kReduceMinI)); + return writeReduceOperands(op, writer, op.dimension()); +} + +LogicalResult writeOp(IREEInterp::LL::ReduceMinFOp op, BytecodeWriter *writer) { + RETURN_IF_FAILURE(writer->WriteOpcode(iree::InterpreterOpcode::kReduceMinF)); + return writeReduceOperands(op, writer, op.dimension()); +} + +LogicalResult writeOp(IREEInterp::LL::ReduceMaxIOp op, BytecodeWriter *writer) { + RETURN_IF_FAILURE(writer->WriteOpcode(iree::InterpreterOpcode::kReduceMaxI)); + return writeReduceOperands(op, writer, op.dimension()); +} + +LogicalResult writeOp(IREEInterp::LL::ReduceMaxFOp op, BytecodeWriter *writer) { + RETURN_IF_FAILURE(writer->WriteOpcode(iree::InterpreterOpcode::kReduceMaxF)); + return writeReduceOperands(op, writer, op.dimension()); +} + +} // namespace + +void registerInterpreterCustomWriters(VMFunctionBuilder *builder) { +#define REGISTER_CUSTOM_WRITER_IMPL(op_type) \ + builder->RegisterCustomWriter( \ + op_type::getOperationName(), \ + +[](Operation *op, BytecodeWriter *writer) { \ + return writeOp(cast(op), writer); \ + }); + REGISTER_CUSTOM_WRITER_IMPL(IREEInterp::LL::ConstantOp); + REGISTER_CUSTOM_WRITER_IMPL(IREEInterp::LL::CallOp); + REGISTER_CUSTOM_WRITER_IMPL(IREEInterp::LL::CallImportOp); + REGISTER_CUSTOM_WRITER_IMPL(IREEInterp::LL::CallIndirectOp); + REGISTER_CUSTOM_WRITER_IMPL(IREEInterp::LL::BranchOp); + REGISTER_CUSTOM_WRITER_IMPL(IREEInterp::LL::CondBranchOp); + REGISTER_CUSTOM_WRITER_IMPL(IREEInterp::LL::ConvertSSOp); + REGISTER_CUSTOM_WRITER_IMPL(IREEInterp::LL::ConvertUUOp); + REGISTER_CUSTOM_WRITER_IMPL(IREEInterp::LL::ConvertSUOp); + REGISTER_CUSTOM_WRITER_IMPL(IREEInterp::LL::ConvertUSOp); + REGISTER_CUSTOM_WRITER_IMPL(IREEInterp::LL::CmpIOp); + REGISTER_CUSTOM_WRITER_IMPL(IREEInterp::LL::CmpFOp); + REGISTER_CUSTOM_WRITER_IMPL(IREEInterp::LL::AllocHeapOp); + REGISTER_CUSTOM_WRITER_IMPL(IREEInterp::LL::StaticCopyOp); + REGISTER_CUSTOM_WRITER_IMPL(IREEInterp::LL::ReduceSumIOp); + REGISTER_CUSTOM_WRITER_IMPL(IREEInterp::LL::ReduceSumFOp); + REGISTER_CUSTOM_WRITER_IMPL(IREEInterp::LL::ReduceMinIOp); + REGISTER_CUSTOM_WRITER_IMPL(IREEInterp::LL::ReduceMinFOp); + REGISTER_CUSTOM_WRITER_IMPL(IREEInterp::LL::ReduceMaxIOp); + REGISTER_CUSTOM_WRITER_IMPL(IREEInterp::LL::ReduceMaxFOp); +} + +} // namespace iree_compiler +} // namespace mlir diff --git a/compiler/IR/Interpreter/OpWriters.h b/compiler/IR/Interpreter/OpWriters.h new file mode 100644 index 000000000000..7aa4bbaf4d27 --- /dev/null +++ b/compiler/IR/Interpreter/OpWriters.h @@ -0,0 +1,30 @@ +// Copyright 2019 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#ifndef THIRD_PARTY_MLIR_EDGE_IREE_COMPILER_IR_INTERPRETER_OPWRITERS_H_ +#define THIRD_PARTY_MLIR_EDGE_IREE_COMPILER_IR_INTERPRETER_OPWRITERS_H_ + +#include "third_party/mlir_edge/iree/compiler/Serialization/VMFunctionBuilder.h" + +namespace mlir { +namespace iree_compiler { + +// Registers custom op writers with the builder. +// Ops not registered will use the generic writer. +void registerInterpreterCustomWriters(VMFunctionBuilder *builder); + +} // namespace iree_compiler +} // namespace mlir + +#endif // THIRD_PARTY_MLIR_EDGE_IREE_COMPILER_IR_INTERPRETER_OPWRITERS_H_ diff --git a/compiler/IR/Interpreter/test/concat.mlir b/compiler/IR/Interpreter/test/concat.mlir new file mode 100644 index 000000000000..2ea2c734a869 --- /dev/null +++ b/compiler/IR/Interpreter/test/concat.mlir @@ -0,0 +1,68 @@ +// RUN: iree-opt -canonicalize %s --split-input-file | FileCheck %s --dump-input=fail + +// CHECK-LABEL: func @concat.1D +// CHECK-SAME: [[ARG0:%[a-zA-Z0-9]+]] +// CHECK-SAME: [[ARG1:%[a-zA-Z0-9]+]] +func @concat.1D(%arg0 : memref<4xi32>, %arg1 : memref<3xi32>) -> memref<7xi32> { + // CHECK-DAG: [[SRC_INDICES:%.+]] = iree.constant dense<0> : tensor<1x + // CHECK-DAG: [[DST:%.+]] = "iree_hl_interp.alloc_heap"() : () -> memref<7xi32> + + // CHECK-DAG: [[DST_INDICES0:%.+]] = iree.constant dense<0> : tensor<1x + // CHECK-DAG: [[LENGTHS0:%.+]] = iree.constant dense<4> : tensor<1x + // CHECK-DAG: "iree_hl_interp.copy"([[ARG0]], [[SRC_INDICES]], [[DST]], [[DST_INDICES0]], [[LENGTHS0]]) + + // CHECK-DAG: [[DST_INDICES1:%.+]] = iree.constant dense<4> : tensor<1x + // CHECK-DAG: [[LENGTHS1:%.+]] = iree.constant dense<3> : tensor<1x + // CHECK-DAG: "iree_hl_interp.copy"([[ARG1]], [[SRC_INDICES]], [[DST]], [[DST_INDICES1]], [[LENGTHS1]]) + + %0 = "iree_hl_interp.concat"(%arg0, %arg1) {dimension = 0 : i32} : (memref<4xi32>, memref<3xi32>) -> memref<7xi32> + + // CHECK: return [[DST]] + return %0 : memref<7xi32> +} + +// ----- + +// CHECK-LABEL: func @concat.2D.Dim0 +// CHECK-SAME: [[ARG0:%[a-zA-Z0-9]+]] +// CHECK-SAME: [[ARG1:%[a-zA-Z0-9]+]] +func @concat.2D.Dim0(%arg0 : memref<4x4xi32>, %arg1 : memref<3x4xi32>) -> memref<7x4xi32> { + // CHECK-DAG: [[SRC_INDICES:%.+]] = iree.constant dense<0> : tensor<2x + // CHECK-DAG: [[DST:%.+]] = "iree_hl_interp.alloc_heap"() : () -> memref<7x4xi32> + + // CHECK-DAG: [[DST_INDICES0:%.+]] = iree.constant dense<0> : tensor<2x + // CHECK-DAG: [[LENGTHS0:%.+]] = iree.constant dense<4> : tensor<2x + // CHECK-DAG: "iree_hl_interp.copy"([[ARG0]], [[SRC_INDICES]], [[DST]], [[DST_INDICES0]], [[LENGTHS0]]) + + // CHECK-DAG: [[DST_INDICES1:%.+]] = iree.constant dense<[4, 0]> + // CHECK-DAG: [[LENGTHS1:%.+]] = iree.constant dense<[3, 4]> + // CHECK-DAG: "iree_hl_interp.copy"([[ARG1]], [[SRC_INDICES]], [[DST]], [[DST_INDICES1]], [[LENGTHS1]]) + + %0 = "iree_hl_interp.concat"(%arg0, %arg1) {dimension = 0 : i32} : (memref<4x4xi32>, memref<3x4xi32>) -> memref<7x4xi32> + + // CHECK: return [[DST]] + return %0 : memref<7x4xi32> +} + +// ----- + +// CHECK-LABEL: func @concat.2D.Dim1 +// CHECK-SAME: [[ARG0:%[a-zA-Z0-9]+]] +// CHECK-SAME: [[ARG1:%[a-zA-Z0-9]+]] +func @concat.2D.Dim1(%arg0 : memref<4x4xi32>, %arg1 : memref<4x3xi32>) -> memref<4x7xi32> { + // CHECK-DAG: [[SRC_INDICES:%.+]] = iree.constant dense<0> : tensor<2x + // CHECK-DAG: [[DST:%.+]] = "iree_hl_interp.alloc_heap"() : () -> memref<4x7xi32> + + // CHECK-DAG: [[DST_INDICES0:%.+]] = iree.constant dense<0> : tensor<2x + // CHECK-DAG: [[LENGTHS0:%.+]] = iree.constant dense<4> : tensor<2x + // CHECK-DAG: "iree_hl_interp.copy"([[ARG0]], [[SRC_INDICES]], [[DST]], [[DST_INDICES0]], [[LENGTHS0]]) + + // CHECK-DAG: [[DST_INDICES1:%.+]] = iree.constant dense<[0, 4]> + // CHECK-DAG: [[LENGTHS1:%.+]] = iree.constant dense<[4, 3]> + // CHECK-DAG: "iree_hl_interp.copy"([[ARG1]], [[SRC_INDICES]], [[DST]], [[DST_INDICES1]], [[LENGTHS1]]) + + %0 = "iree_hl_interp.concat"(%arg0, %arg1) {dimension = 1 : i32} : (memref<4x4xi32>, memref<4x3xi32>) -> memref<4x7xi32> + + // CHECK: return [[DST]] + return %0 : memref<4x7xi32> +} diff --git a/compiler/IR/Interpreter/test/invalid_types_hl.mlir b/compiler/IR/Interpreter/test/invalid_types_hl.mlir new file mode 100644 index 000000000000..961f054746df --- /dev/null +++ b/compiler/IR/Interpreter/test/invalid_types_hl.mlir @@ -0,0 +1,55 @@ +// RUN: iree-opt %s -split-input-file -verify-diagnostics + +func @tensor(%arg0 : tensor<1xf32>) { + // expected-error@+1 {{must be memref}} + "iree_hl_interp.tanh_f"(%arg0) : (tensor<1xf32>) -> tensor<1xf32> + return +} + +// ----- + +func @scalar(%arg0 : f32) { + // expected-error@+1 {{must be memref}} + "iree_hl_interp.tanh_f"(%arg0) : (f32) -> f32 + return +} + +// ----- + +func @vector(%arg0 : vector<1xf32>) { + // expected-error@+1 {{must be memref}} + "iree_hl_interp.tanh_f"(%arg0) : (vector<1xf32>) -> vector<1xf32> + return +} + +// ----- + +func @bad_bool(%a : memref<1xf32>) { + // expected-error@+1 {{must be memref of boolean-storing type (1 or 8 -bit integer) values}} + "iree_hl_interp.cmp_f"(%a, %a) {predicate = 0 : i32} : (memref<1xf32>, memref<1xf32>) -> memref<1xi32> + return +} + +// ----- + +func @not_scalar(%a : memref<2xf32>) { + // expected-error@+1 {{0D memref of integer or index values}} + "iree_hl_interp.length"(%a) : (memref<2xf32>) -> memref<2xi32> + return +} + +// ----- + +func @not_scalar_int(%a : memref<1xf32>) { + // expected-error@+1 {{0D memref of integer or index values}} + "iree_hl_interp.length"(%a) : (memref<1xf32>) -> memref + return +} + +// ----- + +func @not_scalar_bool(%cond : memref, %a : memref<1xf32>) { + // expected-error@+1 {{0D memref of boolean-storing type (1 or 8 -bit integer) values}} + "iree_hl_interp.cond_assign"(%cond, %a, %a) : (memref, memref<1xf32>, memref<1xf32>) -> memref<1xf32> + return +} diff --git a/compiler/IR/Interpreter/test/invalid_types_ll.mlir b/compiler/IR/Interpreter/test/invalid_types_ll.mlir new file mode 100644 index 000000000000..0b6c112f2809 --- /dev/null +++ b/compiler/IR/Interpreter/test/invalid_types_ll.mlir @@ -0,0 +1,103 @@ +// RUN: iree-opt %s -split-input-file -verify-diagnostics + +func @tensor(%a: tensor<1xf32>, %dst: memref<1xf32>) { + // expected-error@+1 {{must be memref}} + "iree_ll_interp.tanh_f"(%a, %dst) : (tensor<1xf32>, memref<1xf32>) -> () + return +} + +// ----- + +func @scalar(%a: f32, %dst: memref<1xf32>) { + // expected-error@+1 {{must be memref}} + "iree_ll_interp.tanh_f"(%a, %dst) : (f32, memref<1xf32>) -> () + return +} + +// ----- + +func @vector(%a: vector<1xf32>, %dst: memref<1xf32>) { + // expected-error@+1 {{must be memref}} + "iree_ll_interp.tanh_f"(%a, %dst) : (vector<1xf32>, memref<1xf32>) -> () + return +} + +// ----- + +func @integer_float_op(%a: memref<1xi32>, %dst: memref<1xi32>) { + // expected-error@+1 {{must be memref of 32/64-bit float values}} + "iree_ll_interp.tanh_f"(%a, %dst) : (memref<1xi32>, memref<1xi32>) -> () + return +} + +// ----- + +func @big_float(%a: memref<1xbf16>, %dst: memref<1xbf16>) { + // expected-error@+1 {{must be memref of 32/64-bit float values}} + "iree_ll_interp.tanh_f"(%a, %dst) : (memref<1xbf16>, memref<1xbf16>) -> () + return +} + +// ----- + +func @index(%a : memref<1 x index>, %dst : memref<1 x index>) { + // expected-error@+1 {{must be memref of 8/16/32/64-bit integer values}} + "iree_ll_interp.add_i"(%a, %a, %dst) : (memref<1 x index>, memref<1 x index>, memref<1 x index>) -> () + return +} + +// ----- + +func @i1_cmp(%a: memref<1xf32>, %dst: memref<1xi1>) { + // expected-error@+1 {{must be memref of boolean-storing type (8-bit integer) values}} + "iree_ll_interp.cmp_f"(%a, %a, %dst) {predicate = 0 : i32} : (memref<1xf32>, memref<1xf32>, memref<1xi1>) -> () + return +} + +// ----- + +func @i1_cst() { + // expected-error@+1 {{must be memref of 8/16/32/64-bit integer or 32/64-bit float values}} + "iree_ll_interp.constant"() {value = dense<[1]> : tensor<1xi1>} : () -> memref<1xi1> + return +} + +// ----- + +func @i1_arg(%a : memref<1xi1>, %dst : memref<1xi1>) { + // expected-error@+1 {{must be memref of 8/16/32/64-bit integer values}} + "iree_ll_interp.add_i"(%a, %a, %dst) : (memref<1xi1>, memref<1xi1>, memref<1xi1>) -> () + return +} + +// ----- + +func @not_int(%a : memref<1xf32>, %dst : memref) { + // expected-error@+1 {{32-bit integer values}} + "iree_ll_interp.length"(%a, %dst) : (memref<1xf32>, memref) -> () + return +} + +// ----- + +func @wrong_int(%a : memref<1xf32>, %dst : memref) { + // expected-error@+1 {{32-bit integer values}} + "iree_ll_interp.length"(%a, %dst) : (memref<1xf32>, memref) -> () + return +} + +// ----- + +func @not_scalar_int(%a : memref<1xf32>, %dst : memref<1xi32>) { + // expected-error@+1 {{0D memref}} + "iree_ll_interp.length"(%a, %dst) : (memref<1xf32>, memref<1xi32>) -> () + return +} + +// ----- + +func @not_scalar_bool(%cond : memref, %a : memref<1xf32>) { + // expected-error@+1 {{0D memref of boolean-storing type (8-bit integer) values}} + "iree_ll_interp.cond_assign"(%cond, %a, %a) : (memref, memref<1xf32>, memref<1xf32>) -> memref<1xf32> + return +} diff --git a/compiler/IR/OpBase.td b/compiler/IR/OpBase.td new file mode 100644 index 000000000000..2e914eab8465 --- /dev/null +++ b/compiler/IR/OpBase.td @@ -0,0 +1,167 @@ +// Common IREE op definitions shared by the interpreter and sequencer dialects. + +#ifdef IREE_OP_BASE +#else +#define IREE_OP_BASE + +#ifdef OP_BASE +#else +include "mlir/IR/OpBase.td" +#endif // OP_BASE + +//===----------------------------------------------------------------------===// +// Top-level IREE dialect +//===----------------------------------------------------------------------===// + +def IREE_Dialect : Dialect { + let name = "iree"; + let cppNamespace = "IREE"; +} + +//===----------------------------------------------------------------------===// +// General types and helpers +//===----------------------------------------------------------------------===// + +// Type constraint for integer-like types: integers, indices. +def IntegerOrIndex : AnyTypeOf<[AnyInteger, Index], "integer or index">; + +class IREE_ScalarMemRefOf allowedTypes> : + MemRefRankOf; + +//===----------------------------------------------------------------------===// +// Op traits +//===----------------------------------------------------------------------===// + +def IREE_IsModuleOnlyPred : + CPred<"llvm::isa_and_nonnull($_op.getParentOp())">; +def IREE_ModuleOnly : + PredOpTrait<"op can only be used in an 'iree.module' block", + IREE_IsModuleOnlyPred>; + +def IREE_IsMultiArchExecutableOnlyPred : + CPred<"llvm::isa_and_nonnull($_op.getParentOp())">; +def IREE_MultiArchExecutableOnly : + PredOpTrait<"op can only be used in an 'iree.multi_arch_executable' block", + IREE_IsMultiArchExecutableOnlyPred>; + +def IREE_IsExecutableOnlyPred : + CPred<"llvm::isa_and_nonnull($_op.getParentOp())">; +def IREE_ExecutableOnly : + PredOpTrait<"op can only be used in an 'iree.executable' block", + IREE_IsExecutableOnlyPred>; + +def IREE_IsExecutableTargetConfigOnlyPred : + CPred<"llvm::isa_and_nonnull($_op.getParentOp())">; +def IREE_ExecutableTargetConfigOnly : + PredOpTrait<"op can only be used in an 'iree.target_config' block", + IREE_IsExecutableTargetConfigOnlyPred>; + +//===----------------------------------------------------------------------===// +// Structural types +//===----------------------------------------------------------------------===// + +def IREE_IsExecutableType : + CPred<"$_self.isa<::mlir::iree_compiler::ExecutableType>()">; + +def IREE_Executable : Type; + +//===----------------------------------------------------------------------===// +// High-level types +//===----------------------------------------------------------------------===// + +def IREEHL_Bool : + AnyTypeOf<[I1, I8], "boolean-storing type (1 or 8 -bit integer)">; + +def IREEHL_MemRef : MemRefOf<[AnyType]>; +def IREEHL_BoolMemRef : MemRefOf<[IREEHL_Bool]>; +def IREEHL_IntMemRef : MemRefOf<[AnyInteger]>; +def IREEHL_FloatMemRef : MemRefOf<[AnyFloat]>; +def IREEHL_IndexMemRef : MemRefOf<[IntegerOrIndex]>; + +def IREEHL_AnyScalar : IREE_ScalarMemRefOf<[AnyType]>; +def IREEHL_BoolScalar : IREE_ScalarMemRefOf<[IREEHL_Bool]>; +def IREEHL_IntScalar : IREE_ScalarMemRefOf<[AnyInteger]>; +def IREEHL_FloatScalar : IREE_ScalarMemRefOf<[AnyFloat]>; +def IREEHL_IndexScalar : IREE_ScalarMemRefOf<[IntegerOrIndex]>; +def IREEHL_I32Scalar : IREE_ScalarMemRefOf<[I32]>; + +def IREEHL_1DIntMemRef : MemRefRankOf<[AnyInteger], [1]>; +def IREEHL_1DIndexMemRef : MemRefRankOf<[IntegerOrIndex], [1]>; + + +//===----------------------------------------------------------------------===// +// Low-level types +//===----------------------------------------------------------------------===// + +def IREELL_Bool : TypeAlias; +def IREELL_Int: AnyTypeOf<[I8, I16, I32, I64], "8/16/32/64-bit integer">; +def IREELL_Float: AnyTypeOf<[F32, F64], "32/64-bit float">; +def IREELL_Index : AnyTypeOf<[I32, I64], "32/64-bit index integer">; +def IREELL_Element : AnyTypeOf<[IREELL_Int, IREELL_Float], + !strconcat(IREELL_Int.description, " or ", IREELL_Float.description)>; + +def IREELL_MemRef : MemRefOf<[IREELL_Element]>; +def IREELL_IntMemRef : MemRefOf<[IREELL_Int]>; +def IREELL_FloatMemRef : MemRefOf<[IREELL_Float]>; +def IREELL_BoolMemRef : MemRefOf<[IREELL_Bool]>; +def IREELL_IndexMemRef : MemRefOf<[IREELL_Index]>; +// For shape computation outputs, we want to consistently output I32 not I64 +// TODO(b/138851470) Iron out story for index types +def IREELL_I32MemRef : MemRefOf<[I32]>; + +def IREELL_ElementScalar : IREE_ScalarMemRefOf<[IREELL_Element]>; +def IREELL_IntScalar : IREE_ScalarMemRefOf<[IREELL_Int]>; +def IREELL_BoolScalar : IREE_ScalarMemRefOf<[IREELL_Bool]>; +def IREELL_FloatScalar : IREE_ScalarMemRefOf<[IREELL_Float]>; +def IREELL_IndexScalar : IREE_ScalarMemRefOf<[IREELL_Index]>; +// For shape computation outputs, we want to consistently output I32 not I64 +// TODO(b/138851470) Iron out story for index types +def IREELL_I32Scalar : IREE_ScalarMemRefOf<[I32]>; + +def IREELL_1DIntMemRef : MemRefRankOf<[IREELL_Int], [1]>; +def IREELL_1DIndexMemRef : MemRefRankOf<[IREELL_Index], [1]>; + + +//===----------------------------------------------------------------------===// +// Enums +//===----------------------------------------------------------------------===// + +// Values are FourCC codes. +// See iree/hal/executable_format.h for more info. +// https://ubershmekel.github.io/fourcc-to-text/ can be used to generate the +// integer values for inclusion here. +// +def IREE_EF_Unspecified : I32EnumAttrCase<"Unspecified", 538976288>; +def IREE_EF_MlirText : I32EnumAttrCase<"MlirText", 1296845138>; +def IREE_EF_IreeBytecode : I32EnumAttrCase<"IreeBytecode", 1230128453>; +def IREE_EF_SpirV : I32EnumAttrCase<"SpirV", 1397773893>; +def IREE_ExecutableFormatAttr : + I32EnumAttr<"ExecutableFormat", "IREE Executable format", [ + IREE_EF_Unspecified, + IREE_EF_MlirText, + IREE_EF_IreeBytecode, + IREE_EF_SpirV, + ]> { + let returnType = "uint32_t"; + let convertFromStorage = "static_cast($_self.getInt())"; + let cppNamespace = "::mlir::iree_compiler::IREE"; +} + +// Use no padding and clamp the window to the valid area, possibly stopping +// early prior to having covered all data. +def IREE_PM_ClampWindowToFit : I32EnumAttrCase<"ClampWindowToFit", 0>; +// Use initial values for padding when windows cross dimension boundaries. +def IREE_PM_PadBorder : I32EnumAttrCase<"PadBorder", 1>; +// Describes the padding applied for a windowed operation like convolution, +// where a window is placed inside a base area. +def IREE_PaddingModeAttr : + I32EnumAttr<"PaddingMode", "Padding mode", [ + IREE_PM_ClampWindowToFit, + IREE_PM_PadBorder, + ]> { + let returnType = "uint32_t"; + let convertFromStorage = "static_cast($_self.getInt())"; + let cppNamespace = "::mlir::iree_compiler::IREE"; +} + +#endif // IREE_OP_BASE diff --git a/compiler/IR/Ops.cpp b/compiler/IR/Ops.cpp new file mode 100644 index 000000000000..1b7f069ca06f --- /dev/null +++ b/compiler/IR/Ops.cpp @@ -0,0 +1,526 @@ +// Copyright 2019 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "third_party/mlir_edge/iree/compiler/IR/Ops.h" + +#include "third_party/llvm/llvm/include/llvm/ADT/SmallVector.h" +#include "third_party/llvm/llvm/include/llvm/Support/SMLoc.h" +#include "third_party/llvm/llvm/projects/google_mlir/include/mlir/IR/Attributes.h" +#include "third_party/llvm/llvm/projects/google_mlir/include/mlir/IR/Builders.h" +#include "third_party/llvm/llvm/projects/google_mlir/include/mlir/IR/Diagnostics.h" +#include "third_party/llvm/llvm/projects/google_mlir/include/mlir/IR/OpImplementation.h" +#include "third_party/llvm/llvm/projects/google_mlir/include/mlir/IR/Value.h" +#include "third_party/llvm/llvm/projects/google_mlir/include/mlir/Support/LogicalResult.h" +#include "third_party/llvm/llvm/projects/google_mlir/include/mlir/Support/STLExtras.h" + +namespace mlir { +namespace iree_compiler { +namespace IREE { + +//===----------------------------------------------------------------------===// +// iree.constant +//===----------------------------------------------------------------------===// + +static ParseResult parseConstantOp(OpAsmParser *parser, + OperationState *result) { + Attribute valueAttr; + if (parser->parseOptionalAttributeDict(result->attributes) || + parser->parseAttribute(valueAttr, "value", result->attributes)) + return failure(); + + // If the attribute is a symbol reference, then we expect a trailing type. + Type type; + if (!valueAttr.isa()) + type = valueAttr.getType(); + else if (parser->parseColonType(type)) + return failure(); + + // Add the attribute type to the list. + return parser->addTypeToList(type, result->types); +} + +static void printConstantOp(OpAsmPrinter *p, ConstantOp &op) { + *p << "iree.constant "; + p->printOptionalAttrDict(op.getAttrs(), /*elidedAttrs=*/{"value"}); + + if (op.getAttrs().size() > 1) *p << ' '; + p->printAttribute(op.getValue()); + + // If the value is a symbol reference, print a trailing type. + if (op.getValue().isa()) { + *p << " : "; + p->printType(op.getType()); + } +} + +namespace { + +MemRefType convertTypeToMemRef(Type type) { + if (type.isIntOrIndexOrFloat()) { + return MemRefType::get({}, type, {}, 0); + } else if (auto tensorType = type.dyn_cast()) { + return MemRefType::get(tensorType.getShape(), tensorType.getElementType()); + } else if (auto memRefType = type.dyn_cast()) { + return MemRefType::get(memRefType.getShape(), memRefType.getElementType()); + } else { + llvm_unreachable("Unconvertable type"); + } +} + +} // namespace + +void ConstantOp::build(Builder *builder, OperationState *state, + Attribute value) { + auto type = convertTypeToMemRef(value.getType()); + return build(builder, state, type, value); +} + +// TODO(b/134575149): enable folder when we store the correct type. +// OpFoldResult ConstantOp::fold(ArrayRef operands) { +// assert(operands.empty() && "constant has no operands"); +// return getValue(); +// } + +//===----------------------------------------------------------------------===// +// iree.tensor_to_memref +//===----------------------------------------------------------------------===// + +static ParseResult parseTensorToMemRefOp(OpAsmParser *parser, + OperationState *state) { + OpAsmParser::OperandType operand; + Type operandType; + Type resultType; + if (failed(parser->parseLParen()) || failed(parser->parseOperand(operand)) || + failed(parser->parseColonType(operandType)) || + failed(parser->resolveOperand(operand, operandType, state->operands)) || + failed(parser->parseRParen()) || + failed(parser->parseColonType(resultType)) || + failed(parser->addTypeToList(resultType, state->types))) { + return failure(); + } + return success(); +} + +static void printTensorToMemRefOp(OpAsmPrinter *p, TensorToMemRefOp &op) { + *p << "iree.tensor_to_memref("; + p->printOperand(op.getOperand()); + *p << " : "; + p->printType(op.getOperand()->getType()); + *p << ") : "; + p->printType(op.getType()); +} + +//===----------------------------------------------------------------------===// +// iree.memref_to_tensor +//===----------------------------------------------------------------------===// + +static ParseResult parseMemRefToTensorOp(OpAsmParser *parser, + OperationState *state) { + OpAsmParser::OperandType operand; + Type operandType; + Type resultType; + if (failed(parser->parseLParen()) || failed(parser->parseOperand(operand)) || + failed(parser->parseColonType(operandType)) || + failed(parser->resolveOperand(operand, operandType, state->operands)) || + failed(parser->parseRParen()) || + failed(parser->parseColonType(resultType)) || + failed(parser->addTypeToList(resultType, state->types))) { + return failure(); + } + return success(); +} + +static void printMemRefToTensorOp(OpAsmPrinter *p, MemRefToTensorOp &op) { + *p << "iree.memref_to_tensor("; + p->printOperand(op.getOperand()); + *p << " : "; + p->printType(op.getOperand()->getType()); + *p << ") : "; + p->printType(op.getType()); +} + +//===----------------------------------------------------------------------===// +// iree.dispatch_region +//===----------------------------------------------------------------------===// + +void DispatchRegionOp::build(Builder *builder, OperationState *state, + ArrayRef resultTypes, Value *workload, + ArrayRef operands, + ArrayRef attributes) { + state->addTypes(resultTypes); + state->addOperands({workload}); + state->addOperands(operands); + state->addAttributes(attributes); + state->addRegion(); + state->setOperandListToResizable(); +} + +ParseResult parseDispatchRegionOp(OpAsmParser *parser, OperationState *state) { + // Parse required workload. + OpAsmParser::OperandType workloadArg; + Type workloadArgType; + if (failed(parser->parseLSquare()) || + failed(parser->parseOperand(workloadArg)) || + failed(parser->parseColonType(workloadArgType)) || + failed(parser->parseRSquare()) || + failed(parser->resolveOperand(workloadArg, workloadArgType, + state->operands))) { + return failure(); + } + + // Parse (optional) args. + SmallVector regionArgs; + SmallVector regionArgTypes; + if (failed(parser->parseLParen())) { + return failure(); + } + if (failed(parser->parseOptionalRParen())) { + SmallVector regionOperands; + auto argsLoc = parser->getCurrentLocation(); + do { + // Reserve entries in the lists. + regionArgs.emplace_back(); + regionOperands.emplace_back(); + regionArgTypes.emplace_back(); + if (failed(parser->parseRegionArgument(regionArgs.back())) || + failed(parser->parseEqual()) || + failed(parser->parseOperand(regionOperands.back())) || + failed(parser->parseColonType(regionArgTypes.back()))) { + return failure(); + } + } while (succeeded(parser->parseOptionalComma())); + if (failed(parser->parseRParen()) || + failed(parser->resolveOperands(regionOperands, regionArgTypes, argsLoc, + state->operands))) { + return failure(); + } + } + state->setOperandListToResizable(); + + // Parse (optional) results. + if (failed(parser->parseOptionalColonTypeList(state->types))) { + return failure(); + } + + // Parse region body. + Region *body = state->addRegion(); + if (failed(parser->parseRegion(*body, regionArgs, regionArgTypes)) || + failed(parser->parseOptionalAttributeDict(state->attributes))) { + return failure(); + } + return success(); +} + +void printDispatchRegionOp(OpAsmPrinter *p, DispatchRegionOp op) { + *p << "iree.dispatch_region"; + + // Print the workload argument. + *p << "["; + p->printOperand(op.getWorkload()); + *p << " : "; + p->printType(op.getWorkload()->getType()); + *p << "]"; + + // Print the data argument remapping. + *p << "("; + interleaveComma( + llvm::zip(op.getBody().front().getArguments(), op.getArgOperands()), *p, + [&](std::tuple it) { + *p << *std::get<0>(it) << " = " << *std::get<1>(it); + *p << " : "; + *p << std::get<1>(it)->getType(); + }); + *p << ")"; + + // Print the result types, if any. + if (op.getNumResults() > 0) { + *p << " : "; + interleaveComma(op.getResultTypes(), *p); + } + + p->printRegion(op.getBody(), /*printEntryBlockArgs=*/false); + p->printOptionalAttrDict(op.getAttrs(), + /*elidedAttrs=*/{}); +} + +//===----------------------------------------------------------------------===// +// iree.reduction_region +//===----------------------------------------------------------------------===// + +void ReductionRegionOp::build(Builder *builder, OperationState *state, + ArrayRef resultTypes, Value *workload, + ArrayRef operands, + ArrayRef initialValues, + ArrayRef dimensions, + ArrayRef attributes) { + state->addTypes(resultTypes); + state->addOperands({workload}); + state->addOperands(operands); + state->addOperands(initialValues); + state->addAttribute( + "dimensions", + builder->getDenseIntElementsAttr( + builder->getTensorType({static_cast(dimensions.size())}, + builder->getIntegerType(64)), + dimensions)); + state->addAttributes(attributes); + state->addRegion(); + state->setOperandListToResizable(); +} + +void ReductionRegionOp::build( + Builder *builder, OperationState *state, ArrayRef resultTypes, + Value *workload, ArrayRef operands, + ArrayRef initialValues, ArrayRef windowDimensions, + ArrayRef windowStrides, ArrayRef baseDilations, + ArrayRef windowDilations, PaddingMode paddingMode, + ArrayRef attributes) { + state->addTypes(resultTypes); + state->addOperands({workload}); + state->addOperands(operands); + state->addOperands(initialValues); + state->addAttribute("window_dimensions", + builder->getDenseIntElementsAttr( + builder->getTensorType( + {static_cast(windowDimensions.size())}, + builder->getIntegerType(64)), + windowDimensions)); + state->addAttribute( + "window_strides", + builder->getDenseIntElementsAttr( + builder->getTensorType({static_cast(windowStrides.size())}, + builder->getIntegerType(64)), + windowStrides)); + state->addAttribute( + "base_dilations", + builder->getDenseIntElementsAttr( + builder->getTensorType({static_cast(baseDilations.size())}, + builder->getIntegerType(64)), + baseDilations)); + state->addAttribute( + "window_dilations", + builder->getDenseIntElementsAttr( + builder->getTensorType({static_cast(windowDilations.size())}, + builder->getIntegerType(64)), + windowDilations)); + state->addAttribute("padding_mode", builder->getI32IntegerAttr( + static_cast(paddingMode))); + state->addAttributes(attributes); + state->addRegion(); + state->setOperandListToResizable(); +} + +ParseResult parseReductionRegionOp(OpAsmParser *parser, OperationState *state) { + OpAsmParser::OperandType workloadArg; + Type workloadArgType; + if (failed(parser->parseLSquare()) || + failed(parser->parseOperand(workloadArg)) || + failed(parser->parseColonType(workloadArgType)) || + failed(parser->parseRSquare()) || + failed(parser->resolveOperand(workloadArg, workloadArgType, + state->operands))) { + return failure(); + } + + SmallVector reductionOperands; + Type reductionType; + auto operandsLoc = parser->getCurrentLocation(); + if (failed(parser->parseLParen()) || + failed(parser->parseOperandList(reductionOperands)) || + failed(parser->parseRParen()) || + failed(parser->parseColonType(reductionType)) || + failed(parser->resolveOperands( + reductionOperands, reductionType.cast().getInputs(), + operandsLoc, state->operands))) { + return failure(); + } + for (auto type : reductionType.cast().getResults()) { + state->types.push_back(type); + } + state->setOperandListToResizable(); + + SmallVector regionArgs; + SmallVector regionArgTypes; + if (failed(parser->parseKeyword("invocation")) || + failed(parser->parseLParen())) { + return failure(); + } + do { + Type argType; + SmallVector reductionRegionArgs; + OpAsmParser::OperandType initialValue; + if (failed(parser->parseLParen()) || + failed(parser->parseOperandList(reductionRegionArgs, 2)) || + failed(parser->parseRParen()) || failed(parser->parseEqual()) || + failed(parser->parseOperand(initialValue)) || + failed(parser->parseColonType(argType)) || + failed( + parser->resolveOperand(initialValue, argType, state->operands))) { + return failure(); + } + regionArgs.push_back(reductionRegionArgs[0]); + regionArgTypes.push_back(argType); + regionArgs.push_back(reductionRegionArgs[1]); + regionArgTypes.push_back(argType); + } while (succeeded(parser->parseOptionalComma())); + if (failed(parser->parseRParen())) { + return failure(); + } + + // Parse region body. + Region *body = state->addRegion(); + if (failed(parser->parseRegion(*body, regionArgs, regionArgTypes)) || + failed(parser->parseOptionalAttributeDict(state->attributes))) { + return failure(); + } + + return success(); +} + +void printReductionRegionOp(OpAsmPrinter *p, ReductionRegionOp op) { + *p << "iree.reduction_region"; + + // Print the workload argument. + *p << "["; + p->printOperand(op.getWorkload()); + *p << " : "; + p->printType(op.getWorkload()->getType()); + *p << "]"; + + *p << "("; + p->printOperands(op.getODSOperands(1)); + *p << ")"; + if (op.getNumResults() > 0) { + *p << " : ("; + interleaveComma(op.getODSOperands(1), *p, + [&](Value *operand) { p->printType(operand->getType()); }); + *p << ")"; + *p << " -> ("; + interleaveComma(op.getResultTypes(), *p); + *p << ")"; + } + *p << "\n"; + + *p << " invocation("; + auto &entryBlock = op.getBody().getBlocks().front(); + int regionArgIndex = 0; + interleaveComma(op.getODSOperands(2), *p, [&](Value *operand) { + *p << "("; + p->printOperand(entryBlock.getArgument(regionArgIndex++)); + *p << ", "; + p->printOperand(entryBlock.getArgument(regionArgIndex++)); + *p << ") = "; + p->printOperand(operand); + *p << " : "; + p->printType(operand->getType()); + }); + *p << ") "; + + p->printRegion(op.getBody(), /*printEntryBlockArgs=*/false); + p->printOptionalAttrDict(op.getAttrs(), + /*elidedAttrs=*/{}); +} + +//===----------------------------------------------------------------------===// +// iree.return +//===----------------------------------------------------------------------===// + +static ParseResult parseReturnOp(OpAsmParser *parser, OperationState *state) { + SmallVector opInfo; + SmallVector types; + llvm::SMLoc loc = parser->getCurrentLocation(); + return failure(parser->parseOperandList(opInfo) || + (!opInfo.empty() && parser->parseColonTypeList(types)) || + parser->resolveOperands(opInfo, types, loc, state->operands)); +} + +static void printReturnOp(OpAsmPrinter *p, ReturnOp op) { + *p << "iree.return"; + if (op.getNumOperands() > 0) { + *p << ' '; + p->printOperands(op.operand_begin(), op.operand_end()); + *p << " : "; + interleaveComma(op.getOperandTypes(), *p); + } +} + +//===----------------------------------------------------------------------===// +// iree.load_input +//===----------------------------------------------------------------------===// + +ParseResult parseLoadInputOp(OpAsmParser *parser, OperationState *state) { + OpAsmParser::OperandType operand; + Type argType; + if (parser->parseLParen() || parser->parseOperand(operand) || + parser->parseColonType(argType) || parser->parseRParen() || + parser->resolveOperand(operand, argType, state->operands)) { + return failure(); + } + Type outputType; + if (parser->parseColonType(outputType) || + parser->addTypeToList(outputType, state->types)) { + return failure(); + } + return success(); +} + +void printLoadInputOp(OpAsmPrinter *printer, Operation *op) { + auto *inputValue = op->getOperand(0); + auto *outputValue = op->getResult(0); + *printer << op->getName() << '('; + printer->printOperand(inputValue); + *printer << " : "; + printer->printType(inputValue->getType()); + *printer << ") : "; + printer->printType(outputValue->getType()); +} + +//===----------------------------------------------------------------------===// +// iree.store_output +//===----------------------------------------------------------------------===// + +ParseResult parseStoreOutputOp(OpAsmParser *parser, OperationState *state) { + OpAsmParser::OperandType op0, op1; + Type argType0, argType1; + if (parser->parseLParen() || parser->parseOperand(op0) || + parser->parseColonType(argType0) || parser->parseComma() || + parser->resolveOperand(op0, argType0, state->operands) || + parser->parseOperand(op1) || parser->parseColonType(argType1) || + parser->parseRParen() || + parser->resolveOperand(op1, argType1, state->operands)) { + return failure(); + } + return success(); +} + +void printStoreOutputOp(OpAsmPrinter *printer, Operation *op) { + auto *inputValue = op->getOperand(0); + auto *outputValue = op->getOperand(1); + *printer << op->getName() << '('; + printer->printOperand(inputValue); + *printer << " : "; + printer->printType(inputValue->getType()); + *printer << ", "; + printer->printOperand(outputValue); + *printer << " : "; + printer->printType(outputValue->getType()); + *printer << ")"; +} + +#define GET_OP_CLASSES +#include "third_party/mlir_edge/iree/compiler/IR/Ops.cpp.inc" + +} // namespace IREE +} // namespace iree_compiler +} // namespace mlir diff --git a/compiler/IR/Ops.h b/compiler/IR/Ops.h new file mode 100644 index 000000000000..bf71ba341510 --- /dev/null +++ b/compiler/IR/Ops.h @@ -0,0 +1,36 @@ +// Copyright 2019 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#ifndef THIRD_PARTY_MLIR_EDGE_IREE_COMPILER_IR_OPS_H_ +#define THIRD_PARTY_MLIR_EDGE_IREE_COMPILER_IR_OPS_H_ + +#include "third_party/llvm/llvm/projects/google_mlir/include/mlir/Dialect/StandardOps/Ops.h" +#include "third_party/llvm/llvm/projects/google_mlir/include/mlir/IR/Attributes.h" +#include "third_party/llvm/llvm/projects/google_mlir/include/mlir/IR/Dialect.h" +#include "third_party/llvm/llvm/projects/google_mlir/include/mlir/IR/OpDefinition.h" +#include "third_party/llvm/llvm/projects/google_mlir/include/mlir/IR/StandardTypes.h" +#include "third_party/mlir_edge/iree/compiler/IR/Types.h" + +namespace mlir { +namespace iree_compiler { +namespace IREE { + +#define GET_OP_CLASSES +#include "third_party/mlir_edge/iree/compiler/IR/Ops.h.inc" + +} // namespace IREE +} // namespace iree_compiler +} // namespace mlir + +#endif // THIRD_PARTY_MLIR_EDGE_IREE_COMPILER_IR_OPS_H_ diff --git a/compiler/IR/Ops.td b/compiler/IR/Ops.td new file mode 100644 index 000000000000..474cdd283266 --- /dev/null +++ b/compiler/IR/Ops.td @@ -0,0 +1,170 @@ +// IREE ops for working with buffers and buffer views. +// These are used by common transforms between the sequencer and interpreter and +// allow us to share some of the common lowering passes from other dialects. + +#ifdef IREE_OPS +#else +#define IREE_OPS + +#ifdef IREE_OP_BASE +#else +include "third_party/mlir_edge/iree/compiler/IR/OpBase.td" +#endif // IREE_OP_BASE + +class IREE_Op traits = []> : + Op { + let parser = [{ return parse$cppClass(parser, result); }]; + let printer = [{ print$cppClass(p, *this); }]; +} + +class IREE_PureOp traits = []> : + IREE_Op; + +// TODO(b/134575149): determine if we want multiple constant op types. +def IREE_ConstantOp : IREE_PureOp<"constant"> { + let arguments = (ins AnyAttr:$value); + let results = (outs IREEHL_MemRef); + + // TODO(b/132296600): make tablegen follow the style guide. + let extraClassDeclaration = [{ + Attribute getValue() { return value(); } + }]; + + let builders = [ + OpBuilder<"Builder *builder, OperationState *state, Attribute value">, + ]; + + // TODO(b/134575149): enable folder when we store the correct type. + // let hasFolder = 1; +} + +// TODO(b/134671482): remove/move tensor_to_memref/memref_to_tensor. +def IREE_TensorToMemRefOp : IREE_PureOp<"tensor_to_memref", [ + SameOperandsAndResultShape, SameOperandsAndResultElementType +]> { + let arguments = (ins AnyTensor:$src); + let results = (outs IREEHL_MemRef); +} + +// TODO(b/134671482): remove/move tensor_to_memref/memref_to_tensor. +def IREE_MemRefToTensorOp : IREE_PureOp<"memref_to_tensor", [ + SameOperandsAndResultShape, SameOperandsAndResultElementType +]> { + let arguments = (ins IREEHL_MemRef:$src); + let results = (outs AnyTensor); +} + +def IREE_Workload : TensorOf<[AnyInteger]>; + +def IREE_DispatchRegionOp : IREE_PureOp<"dispatch_region"> { + let arguments = (ins + IREE_Workload:$workload, + Variadic:$args + ); + let results = (outs Variadic); + let regions = (region AnyRegion:$body); + + let extraClassDeclaration = [{ + // TODO(b/132296600): make tablegen follow the style guide. + Value *getWorkload() { return workload(); } + Region& getBody() { return body(); } + + // TODO(b/133879130): make tablegen support variadic operand accessors. + /// Get the argument operands to the called function. + operand_range getArgOperands() { + return {arg_operand_begin(), arg_operand_end()}; + } + unsigned mapArgOperandToOpOperand(unsigned i) { return i + 1; } + unsigned getNumArgOperands() { return getNumOperands() - 1; } + Value *getArgOperand(unsigned i) { + return getOperand(mapArgOperandToOpOperand(i)); + } + void setArgOperand(unsigned i, Value *arg) { + setOperand(mapArgOperandToOpOperand(i), arg); + } + + operand_iterator arg_operand_begin() { + return operand_begin() + mapArgOperandToOpOperand(0); + } + operand_iterator arg_operand_end() { return operand_end(); } + }]; + + let skipDefaultBuilders = 1; + let builders = [ + OpBuilder<"Builder *builder, OperationState *state," + "ArrayRef resultTypes, Value *workload," + "ArrayRef args," + "ArrayRef attributes = {}">, + ]; +} + +def IREE_ReductionRegionOp : IREE_PureOp<"reduction_region", [ + SameVariadicOperandSize, +]> { + let arguments = (ins + IREE_Workload:$workload, + Variadic:$operands, + Variadic:$initial_values, + OptionalAttr:$dimensions, + OptionalAttr:$window_dimensions, + OptionalAttr:$window_strides, + OptionalAttr:$base_dilations, + OptionalAttr:$window_dilations, + OptionalAttr:$padding_mode + ); + let results = (outs Variadic); + let regions = (region AnyRegion:$body); + + let extraClassDeclaration = [{ + // TODO(b/132296600): make tablegen follow the style guide. + Value *getWorkload() { return workload(); } + Region& getBody() { return body(); } + + bool isWindowed() { + return window_dimensions().hasValue(); + } + + PaddingMode getPaddingMode() { + return static_cast(padding_mode().getValue()); + } + + unsigned getNumReductionOperands() { return (getNumOperands() - 1) / 2; } + operand_range getReductionOperands() { return getODSOperands(1); } + operand_range getInitialValueOperands() { return getODSOperands(2); } + }]; + + let skipDefaultBuilders = 1; + let builders = [ + OpBuilder<"Builder *builder, OperationState *state," + "ArrayRef resultTypes, Value *workload, ArrayRef operands," + "ArrayRef initialValues," + "ArrayRef dimensions," + "ArrayRef attributes = {}">, + OpBuilder<"Builder *builder, OperationState *state," + "ArrayRef resultTypes, Value *workload, ArrayRef operands," + "ArrayRef initialValues," + "ArrayRef windowDimensions, ArrayRef windowStrides," + "ArrayRef baseDilations, ArrayRef windowDilations," + "PaddingMode paddingMode," + "ArrayRef attributes = {}">, + ]; +} + +def IREE_ReturnOp : IREE_Op<"return", [Terminator]> { + let arguments = (ins Variadic:$operands); + + let builders = [OpBuilder< + "Builder *b, OperationState *result", [{ build(b, result, llvm::None); }] + >]; +} + +def IREE_LoadInputOp : IREE_PureOp<"load_input"> { + let arguments = (ins IREEHL_MemRef:$src); + let results = (outs AnyType); +} + +def IREE_StoreOutputOp : IREE_Op<"store_output"> { + let arguments = (ins AnyType:$src, IREEHL_MemRef:$dst); +} + +#endif // IREE_OPS diff --git a/compiler/IR/Sequencer/HLDialect.cpp b/compiler/IR/Sequencer/HLDialect.cpp new file mode 100644 index 000000000000..2085620aae41 --- /dev/null +++ b/compiler/IR/Sequencer/HLDialect.cpp @@ -0,0 +1,34 @@ +// Copyright 2019 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "third_party/mlir_edge/iree/compiler/IR/Sequencer/HLDialect.h" + +#include "third_party/llvm/llvm/include/llvm/Support/SourceMgr.h" +#include "third_party/mlir_edge/iree/compiler/IR/Sequencer/HLOps.h" + +namespace mlir { +namespace iree_compiler { + +IREEHLSequencerDialect::IREEHLSequencerDialect(MLIRContext* context) + : Dialect(getDialectNamespace(), context) { +#define GET_OP_LIST + addOperations< +#include "third_party/mlir_edge/iree/compiler/IR/Sequencer/HLOps.cpp.inc" + >(); +} + +static DialectRegistration iree_hl_seq_dialect; + +} // namespace iree_compiler +} // namespace mlir diff --git a/compiler/IR/Sequencer/HLDialect.h b/compiler/IR/Sequencer/HLDialect.h new file mode 100644 index 000000000000..5020c15cdee7 --- /dev/null +++ b/compiler/IR/Sequencer/HLDialect.h @@ -0,0 +1,32 @@ +// Copyright 2019 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#ifndef THIRD_PARTY_MLIR_EDGE_IREE_COMPILER_IR_SEQUENCER_HLDIALECT_H_ +#define THIRD_PARTY_MLIR_EDGE_IREE_COMPILER_IR_SEQUENCER_HLDIALECT_H_ + +#include "third_party/llvm/llvm/projects/google_mlir/include/mlir/IR/Dialect.h" + +namespace mlir { +namespace iree_compiler { + +class IREEHLSequencerDialect : public Dialect { + public: + explicit IREEHLSequencerDialect(MLIRContext* context); + static StringRef getDialectNamespace() { return "iree_hl_seq"; } +}; + +} // namespace iree_compiler +} // namespace mlir + +#endif // THIRD_PARTY_MLIR_EDGE_IREE_COMPILER_IR_SEQUENCER_HLDIALECT_H_ diff --git a/compiler/IR/Sequencer/HLOps.cpp b/compiler/IR/Sequencer/HLOps.cpp new file mode 100644 index 000000000000..6c60f9a77791 --- /dev/null +++ b/compiler/IR/Sequencer/HLOps.cpp @@ -0,0 +1,394 @@ +// Copyright 2019 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "third_party/mlir_edge/iree/compiler/IR/Sequencer/HLOps.h" + +#include "third_party/llvm/llvm/projects/google_mlir/include/mlir/IR/Attributes.h" +#include "third_party/llvm/llvm/projects/google_mlir/include/mlir/IR/Builders.h" +#include "third_party/llvm/llvm/projects/google_mlir/include/mlir/IR/Function.h" +#include "third_party/llvm/llvm/projects/google_mlir/include/mlir/IR/OpImplementation.h" +#include "third_party/llvm/llvm/projects/google_mlir/include/mlir/IR/PatternMatch.h" +#include "third_party/llvm/llvm/projects/google_mlir/include/mlir/IR/StandardTypes.h" +#include "third_party/mlir_edge/iree/compiler/IR/Ops.h" +#include "third_party/mlir_edge/iree/compiler/IR/Types.h" + +namespace mlir { +namespace iree_compiler { +namespace IREESeq { +namespace HL { + +namespace { + +static LogicalResult verifyWorkload(Operation *op, Value *workload) { + if (auto workloadType = workload->getType().dyn_cast()) { + if (workloadType.getNumElements() != 3) { + return op->emitOpError("workload must be specified as (x,y,z) but has ") + << workloadType.getNumElements() + << " elements (type=" << workload->getType() << ")"; + } + return success(); + } + return op->emitOpError( + "workload must be specified as an (x,y,z) memref but has type ") + << workload->getType(); +} + +} // namespace + +//===----------------------------------------------------------------------===// +// iree_hl_seq.call +//===----------------------------------------------------------------------===// + +static ParseResult parseCallOp(OpAsmParser *parser, OperationState *state) { + SymbolRefAttr calleeAttr; + FunctionType calleeType; + SmallVector operands; + auto calleeLoc = parser->getNameLoc(); + if (parser->parseAttribute(calleeAttr, "callee", state->attributes) || + parser->parseOperandList(operands, OpAsmParser::Delimiter::Paren) || + parser->parseOptionalAttributeDict(state->attributes) || + parser->parseColonType(calleeType) || + parser->addTypesToList(calleeType.getResults(), state->types) || + parser->resolveOperands(operands, calleeType.getInputs(), calleeLoc, + state->operands)) { + return failure(); + } + return success(); +} + +static void printCallOp(OpAsmPrinter *p, CallOp op) { + *p << "iree_hl_seq.call " << op.getAttr("callee") << '('; + p->printOperands(op.getOperands()); + *p << ')'; + p->printOptionalAttrDict(op.getAttrs(), /*elidedAttrs=*/{"callee"}); + *p << " : "; + p->printType(op.getCalleeType()); +} + +FunctionType CallOp::getCalleeType() { + SmallVector resultTypes(getResultTypes()); + SmallVector argTypes(getOperandTypes()); + return FunctionType::get(argTypes, resultTypes, getContext()); +} + +//===----------------------------------------------------------------------===// +// iree_hl_seq.call_indirect +//===----------------------------------------------------------------------===// + +static ParseResult parseCallIndirectOp(OpAsmParser *parser, + OperationState *result) { + FunctionType calleeType; + OpAsmParser::OperandType callee; + llvm::SMLoc operandsLoc; + SmallVector operands; + return failure( + parser->parseOperand(callee) || + parser->getCurrentLocation(&operandsLoc) || + parser->parseOperandList(operands, OpAsmParser::Delimiter::Paren) || + parser->parseOptionalAttributeDict(result->attributes) || + parser->parseColonType(calleeType) || + parser->resolveOperand(callee, calleeType, result->operands) || + parser->resolveOperands(operands, calleeType.getInputs(), operandsLoc, + result->operands) || + parser->addTypesToList(calleeType.getResults(), result->types)); +} + +static void printCallIndirectOp(OpAsmPrinter *p, CallIndirectOp op) { + *p << "iree_hl_seq.call_indirect "; + p->printOperand(op.getCallee()); + *p << '('; + auto operandRange = op.getOperands(); + p->printOperands(++operandRange.begin(), operandRange.end()); + *p << ')'; + p->printOptionalAttrDict(op.getAttrs(), /*elidedAttrs=*/{"callee"}); + *p << " : " << op.getCallee()->getType(); +} + +//===----------------------------------------------------------------------===// +// iree_hl_seq.return +//===----------------------------------------------------------------------===// + +static ParseResult parseReturnOp(OpAsmParser *parser, OperationState *state) { + SmallVector opInfo; + SmallVector types; + llvm::SMLoc loc = parser->getCurrentLocation(); + return failure(parser->parseOperandList(opInfo) || + (!opInfo.empty() && parser->parseColonTypeList(types)) || + parser->resolveOperands(opInfo, types, loc, state->operands)); +} + +static void printReturnOp(OpAsmPrinter *p, ReturnOp op) { + *p << "iree_hl_seq.return"; + if (op.getNumOperands() > 0) { + *p << ' '; + p->printOperands(op.operand_begin(), op.operand_end()); + *p << " : "; + interleaveComma(op.getOperandTypes(), *p); + } +} + +//===----------------------------------------------------------------------===// +// iree_hl_seq.br +//===----------------------------------------------------------------------===// + +static ParseResult parseBranchOp(OpAsmParser *parser, OperationState *result) { + Block *dest; + SmallVector destOperands; + if (parser->parseSuccessorAndUseList(dest, destOperands)) return failure(); + result->addSuccessor(dest, destOperands); + return success(); +} + +static void printBranchOp(OpAsmPrinter *p, BranchOp op) { + *p << "iree_hl_seq.br "; + p->printSuccessorAndUseList(op.getOperation(), 0); +} + +Block *BranchOp::getDest() { return getOperation()->getSuccessor(0); } + +void BranchOp::setDest(Block *block) { + return getOperation()->setSuccessor(block, 0); +} + +void BranchOp::eraseOperand(unsigned index) { + getOperation()->eraseSuccessorOperand(0, index); +} + +//===----------------------------------------------------------------------===// +// iree_hl_seq.cond_br +//===----------------------------------------------------------------------===// + +static ParseResult parseCondBranchOp(OpAsmParser *parser, + OperationState *result) { + SmallVector destOperands; + Block *dest; + OpAsmParser::OperandType condInfo; + + // Parse the condition. + Type int1Ty = parser->getBuilder().getI1Type(); + if (parser->parseOperand(condInfo) || parser->parseComma() || + parser->resolveOperand(condInfo, int1Ty, result->operands)) { + return parser->emitError(parser->getNameLoc(), + "expected condition type was boolean (i1)"); + } + + // Parse the true successor. + if (parser->parseSuccessorAndUseList(dest, destOperands)) return failure(); + result->addSuccessor(dest, destOperands); + + // Parse the false successor. + destOperands.clear(); + if (parser->parseComma() || + parser->parseSuccessorAndUseList(dest, destOperands)) + return failure(); + result->addSuccessor(dest, destOperands); + + return success(); +} + +static void printCondBranchOp(OpAsmPrinter *p, CondBranchOp op) { + *p << "iree_hl_seq.cond_br "; + p->printOperand(op.getCondition()); + *p << ", "; + p->printSuccessorAndUseList(op.getOperation(), CondBranchOp::trueIndex); + *p << ", "; + p->printSuccessorAndUseList(op.getOperation(), CondBranchOp::falseIndex); +} + +//===----------------------------------------------------------------------===// +// iree_hl_seq.dispatch +//===----------------------------------------------------------------------===// + +static ParseResult parseDispatchOp(OpAsmParser *parser, OperationState *state) { + auto executableLoc = parser->getNameLoc(); + + SymbolRefAttr executableAttr; + SymbolRefAttr entryPointAttr; + FunctionType entryPointType; + if (failed(parser->parseAttribute(executableAttr, "executable", + state->attributes)) || + failed(parser->parseColon()) || failed(parser->parseColon()) || + failed(parser->parseAttribute(entryPointAttr, "entry_point", + state->attributes))) { + return failure(); + } + + OpAsmParser::OperandType workloadArg; + Type workloadArgType; + if (failed(parser->parseLSquare()) || + failed(parser->parseOperand(workloadArg)) || + failed(parser->parseColonType(workloadArgType)) || + failed(parser->parseRSquare()) || + failed(parser->resolveOperand(workloadArg, workloadArgType, + state->operands))) { + return failure(); + } + + SmallVector operands; + if (failed( + parser->parseOperandList(operands, OpAsmParser::Delimiter::Paren)) || + failed(parser->parseOptionalAttributeDict(state->attributes)) || + failed(parser->parseColonType(entryPointType)) || + failed( + parser->addTypesToList(entryPointType.getResults(), state->types)) || + failed(parser->resolveOperands(operands, entryPointType.getInputs(), + executableLoc, state->operands))) { + return failure(); + } + return success(); +} + +static void printDispatchOp(OpAsmPrinter *p, DispatchOp op) { + *p << "iree_hl_seq.dispatch " << op.getExecutable() + << "::" << op.getEntryPoint(); + *p << "["; + p->printOperand(op.getWorkload()); + *p << " : "; + p->printType(op.getWorkload()->getType()); + *p << "]("; + p->printOperands(op.getArgOperands()); + *p << ')'; + p->printOptionalAttrDict(op.getAttrs(), /*elidedAttrs=*/{ + "executable", + "entry_point", + }); + *p << " : "; + p->printType(op.getEntryPointType()); +} + +static LogicalResult verifyDispatchOp(DispatchOp op) { + if (failed(verifyWorkload(op, op.getWorkload()))) { + return failure(); + } + return success(); +} + +FunctionType DispatchOp::getEntryPointType() { + SmallVector resultTypes(getResultTypes()); + SmallVector argTypes(getArgOperandTypes()); + return FunctionType::get(argTypes, resultTypes, getContext()); +} + +//===----------------------------------------------------------------------===// +// iree_hl_seq.rank +//===----------------------------------------------------------------------===// + +OpFoldResult RankOp::fold(ArrayRef operands) { + Builder builder(getContext()); + if (auto op0 = operands[0].dyn_cast_or_null()) { + return builder.getIntegerAttr(builder.getIntegerType(32), + op0.getType().getRank()); + } + return {}; +} + +//===----------------------------------------------------------------------===// +// iree_hl_seq.shape +//===----------------------------------------------------------------------===// + +void ShapeOp::build(Builder *builder, OperationState *state, Value *operand) { + state->addOperands(operand); + int64_t rank = 0; + if (auto shapedType = operand->getType().dyn_cast()) { + rank = shapedType.getRank(); + } + state->addTypes(builder->getMemRefType({rank}, builder->getIntegerType(32))); +} + +OpFoldResult ShapeOp::fold(ArrayRef operands) { + Builder builder(getContext()); + if (auto op0 = operands[0].dyn_cast_or_null()) { + return builder.getDenseIntElementsAttr( + builder.getTensorType({op0.getType().getRank()}, + builder.getIntegerType(32)), + op0.getType().getShape()); + } + return {}; +} + +//===----------------------------------------------------------------------===// +// iree_hl_seq.length +//===----------------------------------------------------------------------===// + +OpFoldResult LengthOp::fold(ArrayRef operands) { + Builder builder(getContext()); + if (auto op0 = operands[0].dyn_cast_or_null()) { + return builder.getIntegerAttr(builder.getIntegerType(32), + op0.getNumElements()); + } + return {}; +} + +//===----------------------------------------------------------------------===// +// iree_hl_seq.concat +//===----------------------------------------------------------------------===// + +namespace { +static ElementsAttr elementsAttrFromArray(PatternRewriter &rewriter, + ArrayRef elements) { + return rewriter.getDenseIntElementsAttr( + rewriter.getTensorType(elements.size(), rewriter.getIntegerType(64)), + elements); +} + +static IREE::ConstantOp createArrayConstant(PatternRewriter &rewriter, + Location loc, + llvm::ArrayRef elements) { + auto elementsAttr = elementsAttrFromArray(rewriter, elements); + return rewriter.create(loc, elementsAttr); +} + +struct ConcatToCopies : public OpRewritePattern { + using OpRewritePattern::OpRewritePattern; + PatternMatchResult matchAndRewrite(ConcatOp concatOp, + PatternRewriter &rewriter) const override { + auto finalType = concatOp.getResult()->getType().cast(); + auto loc = concatOp.getLoc(); + std::vector dimPieces; + auto dst = + rewriter.create(loc, finalType, dimPieces); + + llvm::SmallVector zeroOffset(finalType.getRank(), 0); + auto srcIndices = createArrayConstant(rewriter, loc, zeroOffset); + + auto concatDimension = concatOp.dimension().getZExtValue(); + llvm::SmallVector dstIndices(finalType.getRank(), 0); + for (auto *src : concatOp.srcs()) { + auto srcShape = src->getType().cast().getShape(); + auto lengths = createArrayConstant(rewriter, loc, srcShape); + auto dstIndicesOp = createArrayConstant(rewriter, loc, dstIndices); + rewriter.create(loc, src, srcIndices, dst, + dstIndicesOp, lengths); + dstIndices[concatDimension] += srcShape[concatDimension]; + } + + concatOp.replaceAllUsesWith(dst.getResult()); + + return matchSuccess(); + } +}; +} // namespace + +void ConcatOp::getCanonicalizationPatterns(OwningRewritePatternList &results, + MLIRContext *context) { + results.insert(context); +} + +#define GET_OP_CLASSES +#include "third_party/mlir_edge/iree/compiler/IR/Sequencer/HLOps.cpp.inc" + +} // namespace HL +} // namespace IREESeq +} // namespace iree_compiler +} // namespace mlir diff --git a/compiler/IR/Sequencer/HLOps.h b/compiler/IR/Sequencer/HLOps.h new file mode 100644 index 000000000000..7c3674761211 --- /dev/null +++ b/compiler/IR/Sequencer/HLOps.h @@ -0,0 +1,39 @@ +// Copyright 2019 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#ifndef THIRD_PARTY_MLIR_EDGE_IREE_COMPILER_IR_SEQUENCER_HLOPS_H_ +#define THIRD_PARTY_MLIR_EDGE_IREE_COMPILER_IR_SEQUENCER_HLOPS_H_ + +#include "third_party/llvm/llvm/projects/google_mlir/include/mlir/Dialect/StandardOps/Ops.h" +#include "third_party/llvm/llvm/projects/google_mlir/include/mlir/IR/Attributes.h" +#include "third_party/llvm/llvm/projects/google_mlir/include/mlir/IR/Dialect.h" +#include "third_party/llvm/llvm/projects/google_mlir/include/mlir/IR/OpDefinition.h" +#include "third_party/llvm/llvm/projects/google_mlir/include/mlir/IR/StandardTypes.h" +#include "third_party/llvm/llvm/projects/google_mlir/include/mlir/IR/TypeUtilities.h" +#include "third_party/mlir_edge/iree/compiler/IR/Types.h" + +namespace mlir { +namespace iree_compiler { +namespace IREESeq { +namespace HL { + +#define GET_OP_CLASSES +#include "third_party/mlir_edge/iree/compiler/IR/Sequencer/HLOps.h.inc" + +} // namespace HL +} // namespace IREESeq +} // namespace iree_compiler +} // namespace mlir + +#endif // THIRD_PARTY_MLIR_EDGE_IREE_COMPILER_IR_SEQUENCER_HLOPS_H_ diff --git a/compiler/IR/Sequencer/HLOps.td b/compiler/IR/Sequencer/HLOps.td new file mode 100644 index 000000000000..077352728820 --- /dev/null +++ b/compiler/IR/Sequencer/HLOps.td @@ -0,0 +1,416 @@ +// IREE high-level sequencer op definitions. +// This op set contains pseudo ops, ops that accept non-MemRef types, and ops in +// normal SSA form. +// +// Through lowering these high-level ops are converted to low-level ops in the +// LLOps.td (iree_ll_seq.*). These map 1:1 with the bytecode, accept +// only MemRef types, and generally use output parameters instead of return +// types. +// +// The source of truth for bytecode opcodes is: +// iree/schemas/bytecode/sequencer_bytecode_v0.h + +#ifdef IREE_SEQUENCER_HL_OPS +#else +#define IREE_SEQUENCER_HL_OPS + +#ifdef IREE_OP_BASE +#else +include "third_party/mlir_edge/iree/compiler/IR/OpBase.td" +#endif // IREE_OP_BASE + +def IREESeqHL_Dialect : Dialect { + let name = "iree_hl_seq"; + let cppNamespace = "IREESeq::HL"; +} + +//===----------------------------------------------------------------------===// +// Base op classes +//===----------------------------------------------------------------------===// + +class IREESeqHL_Op traits = []> : + Op; + +class IREESeqHL_PureOp traits = []> : + IREESeqHL_Op; + +//===----------------------------------------------------------------------===// +// High-level sequencer ops +//===----------------------------------------------------------------------===// + +def IREESeqHL_CallOp : IREESeqHL_PureOp<"call"> { + let arguments = (ins SymbolRefAttr:$callee, Variadic); + let results = (outs Variadic); + + let builders = [OpBuilder< + "Builder *builder, OperationState *result, FuncOp callee," + "ArrayRef operands = {}", [{ + result->addOperands(operands); + result->addAttribute("callee", builder->getSymbolRefAttr(callee)); + result->addTypes(callee.getType().getResults()); + }]>, OpBuilder< + "Builder *builder, OperationState *result, StringRef callee," + "ArrayRef results, ArrayRef operands = {}", [{ + result->addOperands(operands); + result->addAttribute("callee", builder->getSymbolRefAttr(callee)); + result->addTypes(results); + }]>]; + + let extraClassDeclaration = [{ + // TODO(b/132296600): make tablegen follow the style guide. + StringRef getCallee() { return callee(); } + FunctionType getCalleeType(); + + // TODO(b/133879130): make tablegen support variadic operand accessors. + /// Get the argument operands to the called function. + operand_range getArgOperands() { + return {arg_operand_begin(), arg_operand_end()}; + } + + operand_iterator arg_operand_begin() { return operand_begin(); } + operand_iterator arg_operand_end() { return operand_end(); } + }]; + + let parser = [{ return parse$cppClass(parser, result); }]; + let printer = [{ return print$cppClass(p, *this); }]; +} + +def IREESeqHL_CallIndirectOp : IREESeqHL_Op<"call_indirect"> { + let arguments = (ins FunctionType:$callee, Variadic:$operands); + let results = (outs Variadic); + + let builders = [OpBuilder< + "Builder *, OperationState *result, Value *callee," + "ArrayRef operands = {}", [{ + result->operands.push_back(callee); + result->addOperands(operands); + result->addTypes(callee->getType().cast().getResults()); + }]>]; + + let extraClassDeclaration = [{ + // TODO(b/132296600): make tablegen follow the style guide. + Value *getCallee() { return getOperand(0); } + + // TODO(b/133879130): make tablegen support variadic operand accessors. + /// Get the argument operands to the called function. + operand_range getArgOperands() { + return {arg_operand_begin(), arg_operand_end()}; + } + operand_iterator arg_operand_begin() { return ++operand_begin(); } + operand_iterator arg_operand_end() { return operand_end(); } + }]; + + let parser = [{ return parse$cppClass(parser, result); }]; + let printer = [{ return print$cppClass(p, *this); }]; +} + +def IREESeqHL_ReturnOp : IREESeqHL_Op<"return", [Terminator]> { + let arguments = (ins Variadic:$operands); + + let builders = [OpBuilder< + "Builder *b, OperationState *result", [{ build(b, result, llvm::None); }] + >]; + + let parser = [{ return parse$cppClass(parser, result); }]; + let printer = [{ return print$cppClass(p, *this); }]; +} + +def IREESeqHL_BranchOp : IREESeqHL_Op<"br", [Terminator]> { + let arguments = (ins Variadic:$operands); + + let skipDefaultBuilders = 1; + let builders = [OpBuilder< + "Builder *, OperationState *result, Block *dest," + "ArrayRef operands = {}", [{ + result->addSuccessor(dest, operands); + }]>]; + + let extraClassDeclaration = [{ + Block *getDest(); + void setDest(Block *block); + + /// Erase the operand at 'index' from the operand list. + void eraseOperand(unsigned index); + }]; + + let parser = [{ return parse$cppClass(parser, result); }]; + let printer = [{ return print$cppClass(p, *this); }]; +} + +def IREESeqHL_CondBranchOp : IREESeqHL_Op<"cond_br", [Terminator]> { + let arguments = (ins + IREEHL_BoolScalar:$condition, + Variadic:$branchOperands + ); + + let skipDefaultBuilders = 1; + let builders = [OpBuilder< + "Builder *, OperationState *result, Value *condition, " + "Block *trueDest, ArrayRef trueOperands, " + "Block *falseDest, ArrayRef falseOperands", [{ + result->addOperands(condition); + result->addSuccessor(trueDest, trueOperands); + result->addSuccessor(falseDest, falseOperands); + }]>]; + + let extraClassDeclaration = [{ + // These are the indices into the dests list. + enum { trueIndex = 0, falseIndex = 1 }; + + // The condition operand is the first operand in the list. + Value *getCondition() { return getOperand(0); } + + /// Return the destination if the condition is true. + Block *getTrueDest() { + return getOperation()->getSuccessor(trueIndex); + } + + /// Return the destination if the condition is false. + Block *getFalseDest() { + return getOperation()->getSuccessor(falseIndex); + } + + // Accessors for operands to the 'true' destination. + Value *getTrueOperand(unsigned idx) { + assert(idx < getNumTrueOperands()); + return getOperand(getTrueDestOperandIndex() + idx); + } + + void setTrueOperand(unsigned idx, Value *value) { + assert(idx < getNumTrueOperands()); + setOperand(getTrueDestOperandIndex() + idx, value); + } + + operand_iterator true_operand_begin() { + return operand_begin() + getTrueDestOperandIndex(); + } + operand_iterator true_operand_end() { + return true_operand_begin() + getNumTrueOperands(); + } + operand_range getTrueOperands() { + return {true_operand_begin(), true_operand_end()}; + } + + unsigned getNumTrueOperands() { + return getOperation()->getNumSuccessorOperands(trueIndex); + } + + /// Erase the operand at 'index' from the true operand list. + void eraseTrueOperand(unsigned index) { + getOperation()->eraseSuccessorOperand(trueIndex, index); + } + + // Accessors for operands to the 'false' destination. + Value *getFalseOperand(unsigned idx) { + assert(idx < getNumFalseOperands()); + return getOperand(getFalseDestOperandIndex() + idx); + } + void setFalseOperand(unsigned idx, Value *value) { + assert(idx < getNumFalseOperands()); + setOperand(getFalseDestOperandIndex() + idx, value); + } + + operand_iterator false_operand_begin() { return true_operand_end(); } + operand_iterator false_operand_end() { + return false_operand_begin() + getNumFalseOperands(); + } + operand_range getFalseOperands() { + return {false_operand_begin(), false_operand_end()}; + } + + unsigned getNumFalseOperands() { + return getOperation()->getNumSuccessorOperands(falseIndex); + } + + /// Erase the operand at 'index' from the false operand list. + void eraseFalseOperand(unsigned index) { + getOperation()->eraseSuccessorOperand(falseIndex, index); + } + + private: + /// Get the index of the first true destination operand. + unsigned getTrueDestOperandIndex() { return 1; } + + /// Get the index of the first false destination operand. + unsigned getFalseDestOperandIndex() { + return getTrueDestOperandIndex() + getNumTrueOperands(); + } + }]; + + let parser = [{ return parse$cppClass(parser, result); }]; + let printer = [{ return print$cppClass(p, *this); }]; +} + +def IREESeqHL_DispatchOp : IREESeqHL_Op<"dispatch"> { + let arguments = (ins + SymbolRefAttr:$executable, + SymbolRefAttr:$entry_point, + IREEHL_IntMemRef:$workload, + Variadic:$operands + ); + let results = (outs Variadic); + + let skipDefaultBuilders = 1; + let builders = [OpBuilder< + "Builder *builder, OperationState *result, StringRef executable," + "StringRef entry_point, Value *workload," + "ArrayRef results, ArrayRef operands = {}", [{ + result->addOperands({workload}); + result->addOperands(operands); + result->addAttribute("executable", builder->getSymbolRefAttr(executable)); + result->addAttribute("entry_point", builder->getSymbolRefAttr(entry_point)); + result->addTypes(results); + }]>]; + + let extraClassDeclaration = [{ + // TODO(b/132296600): make tablegen follow the style guide. + StringRef getExecutable() { return executable(); } + StringRef getEntryPoint() { return entry_point(); } + FunctionType getEntryPointType(); + + Value *getWorkload() { return getOperand(0); } + + // TODO(b/133879130): make tablegen support variadic operand accessors. + operand_range getArgOperands() { + return {arg_operand_begin(), arg_operand_end()}; + } + operand_iterator arg_operand_begin() { return operand_begin() + 1; } + operand_iterator arg_operand_end() { return operand_end(); } + + operand_type_range getArgOperandTypes() { + return {arg_operand_type_begin(), arg_operand_type_end()}; + } + operand_type_iterator arg_operand_type_begin() { + return operand_type_iterator(arg_operand_begin()); + } + operand_type_iterator arg_operand_type_end() { + return operand_type_iterator(arg_operand_end()); + } + }]; + + let parser = [{ return parse$cppClass(parser, result); }]; + let printer = [{ return print$cppClass(p, *this); }]; + let verifier = [{ return verify$cppClass(*this); }]; +} + +// TODO(benvanik): make pure (when we can disable CSE). +def IREESeqHL_AllocHeapOp : IREESeqHL_Op<"alloc_heap"> { + // TODO(benvanik): attributes and args. + let arguments = (ins Variadic:$dim_pieces); + let results = (outs IREEHL_MemRef); +} + +def IREESeqHL_DiscardOp : IREESeqHL_Op<"discard"> { + let arguments = (ins IREEHL_MemRef); +} + +def IREESeqHL_RankOp : IREESeqHL_PureOp<"rank"> { + let arguments = (ins IREEHL_MemRef); + let results = (outs IREEHL_IntScalar); + + let hasFolder = 1; +} + +def IREESeqHL_DimOp : IREESeqHL_PureOp<"dim"> { + // TODO(benvanik) add dim attr (I32Attr:$dim) + let arguments = (ins IREEHL_MemRef); + let results = (outs IREEHL_IntScalar); +} + +def IREESeqHL_ShapeOp : IREESeqHL_PureOp<"shape"> { + let arguments = (ins IREEHL_MemRef); + let results = (outs IREEHL_1DIntMemRef); + + let skipDefaultBuilders = 1; + let builders = [OpBuilder< + "Builder *builder, OperationState *result, Value *operand">]; + + let hasFolder = 1; +} + +def IREESeqHL_LengthOp : IREESeqHL_PureOp<"length"> { + let arguments = (ins IREEHL_MemRef); + let results = (outs IREEHL_IndexScalar); + + let hasFolder = 1; +} + +def IREESeqHL_SliceOp : + IREESeqHL_PureOp<"slice", [AllElementTypesMatch<["src", "result"]>, + AllTypesMatch<["indices", "lengths"]>]> { + let arguments = (ins + IREEHL_MemRef:$src, + IREEHL_1DIndexMemRef:$indices, + IREEHL_1DIndexMemRef:$lengths + ); + let results = (outs IREEHL_MemRef:$result); +} + +def IREESeqHL_CopyOp : IREESeqHL_Op<"copy"> { + let arguments = (ins + IREEHL_MemRef:$src, + IREEHL_1DIndexMemRef:$srcIndices, + IREEHL_MemRef:$dst, + IREEHL_1DIndexMemRef:$dstIndices, + IREEHL_1DIndexMemRef:$lengths + ); +} + +def IREESeqHL_FillOp : IREESeqHL_Op<"fill"> { + let arguments = (ins + IREEHL_I32Scalar:$value, + IREEHL_MemRef:$dst, + IREEHL_1DIndexMemRef:$dstIndices, + IREEHL_1DIndexMemRef:$lengths + ); +} + +def IREESeqHL_CloneOp : IREESeqHL_PureOp<"clone", [SameOperandsAndResultType]> { + let arguments = (ins IREEHL_MemRef:$src); + let results = (outs IREEHL_MemRef); +} + +// A pseudo op provided for convenience. This gets canonicalized to a series of +// copies. +def IREESeqHL_ConcatOp : IREESeqHL_PureOp<"concat"> { + // TODO(b/135032064) Add type constraints when they support variadic + let arguments = (ins + Variadic:$srcs, + I32Attr:$dimension + ); + let results = (outs IREEHL_MemRef); + + let hasCanonicalizer = 1; +} + +def IREESeqHL_AssignOp : + IREESeqHL_PureOp<"assign", [SameOperandsAndResultType]> { + let arguments = (ins IREEHL_MemRef:$src); + let results = (outs IREEHL_MemRef); +} + +def IREESeqHL_CondAssignOp : IREESeqHL_PureOp<"cond_assign"> { + let arguments = (ins + IREEHL_BoolScalar:$cond, + IREEHL_MemRef:$lhs, + IREEHL_MemRef:$rhs + ); + let results = (outs IREEHL_MemRef); +} + +def IREESeqHL_ReshapeOp : IREESeqHL_PureOp<"reshape"> { + let arguments = (ins IREEHL_MemRef:$src, IREEHL_MemRef:$shape); + let results = (outs IREEHL_MemRef); +} + +def IREESeqHL_TraceOp : IREESeqHL_Op<"trace"> { + let arguments = (ins Variadic:$srcs); +} + +def IREESeqHL_CondBreakOp : IREESeqHL_Op<"cond_break"> { + let arguments = (ins IREEHL_BoolScalar:$cond); +} + +def IREESeqHL_BreakOp : IREESeqHL_Op<"break">; + +#endif // IREE_INTERPRETER_HL_OPS diff --git a/compiler/IR/Sequencer/LLDialect.cpp b/compiler/IR/Sequencer/LLDialect.cpp new file mode 100644 index 000000000000..7f6d370ba73b --- /dev/null +++ b/compiler/IR/Sequencer/LLDialect.cpp @@ -0,0 +1,34 @@ +// Copyright 2019 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "third_party/mlir_edge/iree/compiler/IR/Sequencer/LLDialect.h" + +#include "third_party/llvm/llvm/include/llvm/Support/SourceMgr.h" +#include "third_party/mlir_edge/iree/compiler/IR/Sequencer/LLOps.h" + +namespace mlir { +namespace iree_compiler { + +IREELLSequencerDialect::IREELLSequencerDialect(MLIRContext* context) + : Dialect(getDialectNamespace(), context) { +#define GET_OP_LIST + addOperations< +#include "third_party/mlir_edge/iree/compiler/IR/Sequencer/LLOps.cpp.inc" + >(); +} + +static DialectRegistration iree_ll_seq_dialect; + +} // namespace iree_compiler +} // namespace mlir diff --git a/compiler/IR/Sequencer/LLDialect.h b/compiler/IR/Sequencer/LLDialect.h new file mode 100644 index 000000000000..6d9563101a09 --- /dev/null +++ b/compiler/IR/Sequencer/LLDialect.h @@ -0,0 +1,32 @@ +// Copyright 2019 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#ifndef THIRD_PARTY_MLIR_EDGE_IREE_COMPILER_IR_SEQUENCER_LLDIALECT_H_ +#define THIRD_PARTY_MLIR_EDGE_IREE_COMPILER_IR_SEQUENCER_LLDIALECT_H_ + +#include "third_party/llvm/llvm/projects/google_mlir/include/mlir/IR/Dialect.h" + +namespace mlir { +namespace iree_compiler { + +class IREELLSequencerDialect : public Dialect { + public: + explicit IREELLSequencerDialect(MLIRContext* context); + static StringRef getDialectNamespace() { return "iree_ll_seq"; } +}; + +} // namespace iree_compiler +} // namespace mlir + +#endif // THIRD_PARTY_MLIR_EDGE_IREE_COMPILER_IR_SEQUENCER_LLDIALECT_H_ diff --git a/compiler/IR/Sequencer/LLOps.cpp b/compiler/IR/Sequencer/LLOps.cpp new file mode 100644 index 000000000000..5d1916d81d3f --- /dev/null +++ b/compiler/IR/Sequencer/LLOps.cpp @@ -0,0 +1,676 @@ +// Copyright 2019 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "third_party/mlir_edge/iree/compiler/IR/Sequencer/LLOps.h" + +#include "third_party/llvm/llvm/projects/google_mlir/include/mlir/IR/Attributes.h" +#include "third_party/llvm/llvm/projects/google_mlir/include/mlir/IR/Builders.h" +#include "third_party/llvm/llvm/projects/google_mlir/include/mlir/IR/Function.h" +#include "third_party/llvm/llvm/projects/google_mlir/include/mlir/IR/Matchers.h" +#include "third_party/llvm/llvm/projects/google_mlir/include/mlir/IR/Module.h" +#include "third_party/llvm/llvm/projects/google_mlir/include/mlir/IR/OpImplementation.h" +#include "third_party/llvm/llvm/projects/google_mlir/include/mlir/IR/Operation.h" +#include "third_party/llvm/llvm/projects/google_mlir/include/mlir/IR/PatternMatch.h" +#include "third_party/llvm/llvm/projects/google_mlir/include/mlir/IR/TypeUtilities.h" +#include "third_party/llvm/llvm/projects/google_mlir/include/mlir/Support/STLExtras.h" +#include "third_party/mlir_edge/iree/compiler/IR/Ops.h" +#include "third_party/mlir_edge/iree/compiler/Utils/OpUtils.h" + +namespace mlir { +namespace iree_compiler { +namespace IREESeq { +namespace LL { + +namespace { + +static LogicalResult verifyWorkload(Operation *op, Value *workload) { + if (auto workloadType = workload->getType().dyn_cast()) { + if (workloadType.getNumElements() != 3) { + return op->emitOpError("workload must be specified as (x,y,z) but has ") + << workloadType.getNumElements() + << " elements (type=" << workload->getType() << ")"; + } + return success(); + } + return op->emitOpError( + "workload must be specified as an (x,y,z) memref but has type ") + << workload->getType(); +} + +static LogicalResult verifyWorkload(Operation *op, ElementsAttr workload) { + if (workload.getNumElements() != 3) { + return op->emitOpError("workload must be specified as (x,y,z) but has ") + << workload.getNumElements() << " elements (value=" << workload + << ")"; + } + return success(); +} + +} // namespace + +//===----------------------------------------------------------------------===// +// iree_ll_seq.constant +//===----------------------------------------------------------------------===// + +OpFoldResult ConstantOp::fold(ArrayRef operands) { + return getValue(); +} + +//===----------------------------------------------------------------------===// +// iree_ll_seq.call +//===----------------------------------------------------------------------===// + +static ParseResult parseCallOp(OpAsmParser *parser, OperationState *state) { + SymbolRefAttr calleeAttr; + FunctionType calleeType; + SmallVector operands; + auto calleeLoc = parser->getNameLoc(); + if (parser->parseAttribute(calleeAttr, "callee", state->attributes) || + parser->parseOperandList(operands, OpAsmParser::Delimiter::Paren) || + parser->parseOptionalAttributeDict(state->attributes) || + parser->parseColonType(calleeType) || + parser->addTypesToList(calleeType.getResults(), state->types) || + parser->resolveOperands(operands, calleeType.getInputs(), calleeLoc, + state->operands)) { + return failure(); + } + return success(); +} + +static void printCallOp(OpAsmPrinter *p, CallOp op) { + *p << "iree_ll_seq.call " << op.getAttr("callee") << '('; + p->printOperands(op.getOperands()); + *p << ')'; + p->printOptionalAttrDict(op.getAttrs(), /*elidedAttrs=*/{"callee"}); + *p << " : "; + p->printType(op.getCalleeType()); +} + +FunctionType CallOp::getCalleeType() { + SmallVector resultTypes(getResultTypes()); + SmallVector argTypes(getOperandTypes()); + return FunctionType::get(argTypes, resultTypes, getContext()); +} + +//===----------------------------------------------------------------------===// +// iree_ll_seq.call_import +//===----------------------------------------------------------------------===// + +static ParseResult parseCallImportOp(OpAsmParser *parser, + OperationState *state) { + SymbolRefAttr calleeAttr; + FunctionType calleeType; + SmallVector operands; + auto calleeLoc = parser->getNameLoc(); + if (parser->parseAttribute(calleeAttr, "callee", state->attributes) || + parser->parseOperandList(operands, OpAsmParser::Delimiter::Paren) || + parser->parseOptionalAttributeDict(state->attributes) || + parser->parseColonType(calleeType) || + parser->addTypesToList(calleeType.getResults(), state->types) || + parser->resolveOperands(operands, calleeType.getInputs(), calleeLoc, + state->operands)) { + return failure(); + } + return success(); +} + +static void printCallImportOp(OpAsmPrinter *p, CallImportOp op) { + *p << "iree_ll_seq.call_import " << op.getAttr("callee") << '('; + p->printOperands(op.getOperands()); + *p << ')'; + p->printOptionalAttrDict(op.getAttrs(), /*elidedAttrs=*/{"callee"}); + *p << " : "; + p->printType(op.getCalleeType()); +} + +FunctionType CallImportOp::getCalleeType() { + SmallVector resultTypes(getResultTypes()); + SmallVector argTypes(getOperandTypes()); + return FunctionType::get(argTypes, resultTypes, getContext()); +} + +//===----------------------------------------------------------------------===// +// iree_ll_seq.call_indirect +//===----------------------------------------------------------------------===// + +static ParseResult parseCallIndirectOp(OpAsmParser *parser, + OperationState *result) { + FunctionType calleeType; + OpAsmParser::OperandType callee; + llvm::SMLoc operandsLoc; + SmallVector operands; + return failure( + parser->parseOperand(callee) || + parser->getCurrentLocation(&operandsLoc) || + parser->parseOperandList(operands, OpAsmParser::Delimiter::Paren) || + parser->parseOptionalAttributeDict(result->attributes) || + parser->parseColonType(calleeType) || + parser->resolveOperand(callee, calleeType, result->operands) || + parser->resolveOperands(operands, calleeType.getInputs(), operandsLoc, + result->operands) || + parser->addTypesToList(calleeType.getResults(), result->types)); +} + +static void printCallIndirectOp(OpAsmPrinter *p, CallIndirectOp op) { + *p << "iree_ll_seq.call_indirect "; + p->printOperand(op.getCallee()); + *p << '('; + auto operandRange = op.getOperands(); + p->printOperands(++operandRange.begin(), operandRange.end()); + *p << ')'; + p->printOptionalAttrDict(op.getAttrs(), /*elidedAttrs=*/{"callee"}); + *p << " : " << op.getCallee()->getType(); +} + +//===----------------------------------------------------------------------===// +// iree_ll_seq.return +//===----------------------------------------------------------------------===// + +static ParseResult parseReturnOp(OpAsmParser *parser, OperationState *state) { + SmallVector opInfo; + SmallVector types; + llvm::SMLoc loc = parser->getCurrentLocation(); + return failure(parser->parseOperandList(opInfo) || + (!opInfo.empty() && parser->parseColonTypeList(types)) || + parser->resolveOperands(opInfo, types, loc, state->operands)); +} + +static void printReturnOp(OpAsmPrinter *p, ReturnOp op) { + *p << "iree_ll_seq.return"; + if (op.getNumOperands() > 0) { + *p << ' '; + p->printOperands(op.operand_begin(), op.operand_end()); + *p << " : "; + interleaveComma(op.getOperandTypes(), *p); + } +} + +//===----------------------------------------------------------------------===// +// iree_ll_seq.br +//===----------------------------------------------------------------------===// + +static ParseResult parseBranchOp(OpAsmParser *parser, OperationState *result) { + Block *dest; + SmallVector destOperands; + if (parser->parseSuccessorAndUseList(dest, destOperands)) return failure(); + result->addSuccessor(dest, destOperands); + return success(); +} + +static void printBranchOp(OpAsmPrinter *p, BranchOp op) { + *p << "iree_ll_seq.br "; + p->printSuccessorAndUseList(op.getOperation(), 0); +} + +Block *BranchOp::getDest() { return getOperation()->getSuccessor(0); } + +void BranchOp::setDest(Block *block) { + return getOperation()->setSuccessor(block, 0); +} + +void BranchOp::eraseOperand(unsigned index) { + getOperation()->eraseSuccessorOperand(0, index); +} + +//===----------------------------------------------------------------------===// +// iree_ll_seq.cond_br +//===----------------------------------------------------------------------===// + +static ParseResult parseCondBranchOp(OpAsmParser *parser, + OperationState *result) { + SmallVector destOperands; + Block *dest; + OpAsmParser::OperandType condInfo; + + // Parse the condition. + Type int1Ty = parser->getBuilder().getI1Type(); + if (parser->parseOperand(condInfo) || parser->parseComma() || + parser->resolveOperand(condInfo, int1Ty, result->operands)) { + return parser->emitError(parser->getNameLoc(), + "expected condition type was boolean (i1)"); + } + + // Parse the true successor. + if (parser->parseSuccessorAndUseList(dest, destOperands)) return failure(); + result->addSuccessor(dest, destOperands); + + // Parse the false successor. + destOperands.clear(); + if (parser->parseComma() || + parser->parseSuccessorAndUseList(dest, destOperands)) + return failure(); + result->addSuccessor(dest, destOperands); + + return success(); +} + +static void printCondBranchOp(OpAsmPrinter *p, CondBranchOp op) { + *p << "iree_ll_interp.cond_br "; + p->printOperand(op.getCondition()); + *p << ", "; + p->printSuccessorAndUseList(op.getOperation(), CondBranchOp::trueIndex); + *p << ", "; + p->printSuccessorAndUseList(op.getOperation(), CondBranchOp::falseIndex); +} + +//===----------------------------------------------------------------------===// +// iree_ll_seq.dynamic_dispatch +//===----------------------------------------------------------------------===// + +static ParseResult parseDynamicDispatchOp(OpAsmParser *parser, + OperationState *state) { + auto executableLoc = parser->getNameLoc(); + + SymbolRefAttr executableAttr; + SymbolRefAttr entryPointAttr; + FunctionType entryPointType; + if (failed(parser->parseAttribute(executableAttr, "executable", + state->attributes)) || + failed(parser->parseColon()) || failed(parser->parseColon()) || + failed(parser->parseAttribute(entryPointAttr, "entry_point", + state->attributes))) { + return failure(); + } + + OpAsmParser::OperandType workloadArg; + Type workloadArgType; + if (failed(parser->parseLSquare()) || + failed(parser->parseOperand(workloadArg)) || + failed(parser->parseColonType(workloadArgType)) || + failed(parser->parseRSquare()) || + failed(parser->resolveOperand(workloadArg, workloadArgType, + state->operands))) { + return failure(); + } + + SmallVector operands; + if (failed( + parser->parseOperandList(operands, OpAsmParser::Delimiter::Paren)) || + failed(parser->parseOptionalAttributeDict(state->attributes)) || + failed(parser->parseColonType(entryPointType)) || + failed( + parser->addTypesToList(entryPointType.getResults(), state->types)) || + failed(parser->resolveOperands(operands, entryPointType.getInputs(), + executableLoc, state->operands))) { + return failure(); + } + return success(); +} + +static void printDynamicDispatchOp(OpAsmPrinter *p, DynamicDispatchOp op) { + *p << "iree_ll_seq.dynamic_dispatch " << op.getExecutable() + << "::" << op.getEntryPoint(); + *p << "["; + p->printOperand(op.getWorkload()); + *p << " : "; + p->printType(op.getWorkload()->getType()); + *p << "]("; + p->printOperands(op.getArgOperands()); + *p << ')'; + p->printOptionalAttrDict(op.getAttrs(), /*elidedAttrs=*/{ + "executable", + "entry_point", + }); + *p << " : "; + p->printType(op.getEntryPointType()); +} + +static LogicalResult verifyDynamicDispatchOp(DynamicDispatchOp op) { + if (failed(verifyWorkload(op, op.getWorkload()))) { + return failure(); + } + return success(); +} + +FunctionType DynamicDispatchOp::getEntryPointType() { + SmallVector resultTypes(getResultTypes()); + SmallVector argTypes(getArgOperandTypes()); + return FunctionType::get(argTypes, resultTypes, getContext()); +} + +namespace { +struct MakeDynamicDispatchOpStatic + : public OpRewritePattern { + using OpRewritePattern::OpRewritePattern; + PatternMatchResult matchAndRewrite(DynamicDispatchOp dynamicDispatchOp, + PatternRewriter &rewriter) const override { + ElementsAttr workloadAttr; + if (!matchPattern(dynamicDispatchOp.getWorkload(), + m_Constant(&workloadAttr))) { + return matchFailure(); + } + + SmallVector resultTypes{dynamicDispatchOp.getResultTypes()}; + SmallVector operands{dynamicDispatchOp.getArgOperands()}; + rewriter.replaceOpWithNewOp( + dynamicDispatchOp, dynamicDispatchOp.getExecutable(), + dynamicDispatchOp.getEntryPoint(), workloadAttr, resultTypes, operands); + return matchSuccess(); + } +}; +} // namespace + +void DynamicDispatchOp::getCanonicalizationPatterns( + OwningRewritePatternList &results, MLIRContext *context) { + results.insert(context); +} + +//===----------------------------------------------------------------------===// +// iree_ll_seq.static_dispatch +//===----------------------------------------------------------------------===// + +static ParseResult parseStaticDispatchOp(OpAsmParser *parser, + OperationState *state) { + auto executableLoc = parser->getNameLoc(); + + SymbolRefAttr executableAttr; + SymbolRefAttr entryPointAttr; + FunctionType entryPointType; + if (failed(parser->parseAttribute(executableAttr, "executable", + state->attributes)) || + failed(parser->parseColon()) || failed(parser->parseColon()) || + failed(parser->parseAttribute(entryPointAttr, "entry_point", + state->attributes))) { + return failure(); + } + + ElementsAttr workloadAttr; + if (failed(parser->parseLSquare()) || + failed(parser->parseAttribute(workloadAttr, "workload", + state->attributes)) || + failed(parser->parseRSquare())) { + return failure(); + } + + SmallVector operands; + if (failed( + parser->parseOperandList(operands, OpAsmParser::Delimiter::Paren)) || + failed(parser->parseOptionalAttributeDict(state->attributes)) || + failed(parser->parseColonType(entryPointType)) || + failed( + parser->addTypesToList(entryPointType.getResults(), state->types)) || + failed(parser->resolveOperands(operands, entryPointType.getInputs(), + executableLoc, state->operands))) { + return failure(); + } + return success(); +} + +static void printStaticDispatchOp(OpAsmPrinter *p, StaticDispatchOp op) { + *p << "iree_ll_seq.static_dispatch " << op.getExecutable() + << "::" << op.getEntryPoint(); + *p << "["; + p->printAttribute(op.getWorkload()); + *p << "]("; + p->printOperands(op.getArgOperands()); + *p << ')'; + p->printOptionalAttrDict(op.getAttrs(), /*elidedAttrs=*/{ + "executable", + "entry_point", + "workload", + }); + *p << " : "; + p->printType(op.getEntryPointType()); +} + +static LogicalResult verifyStaticDispatchOp(StaticDispatchOp op) { + if (failed(verifyWorkload(op, op.getWorkload()))) { + return failure(); + } + return success(); +} + +FunctionType StaticDispatchOp::getEntryPointType() { + SmallVector resultTypes(getResultTypes()); + SmallVector argTypes(getArgOperandTypes()); + return FunctionType::get(argTypes, resultTypes, getContext()); +} + +//===----------------------------------------------------------------------===// +// iree_ll_seq.shape +//===----------------------------------------------------------------------===// + +namespace { +struct FoldShapeOp : public OpRewritePattern { + using OpRewritePattern::OpRewritePattern; + PatternMatchResult matchAndRewrite(ShapeOp shapeOp, + PatternRewriter &rewriter) const override { + auto memRefType = shapeOp.input()->getType().cast(); + if (memRefType.hasStaticShape()) { + auto constantOp = rewriter.create( + shapeOp.getLoc(), + rewriter.getMemRefType({memRefType.getRank()}, + rewriter.getIntegerType(64)), + rewriter.getDenseIntElementsAttr( + rewriter.getTensorType({memRefType.getRank()}, + rewriter.getIntegerType(64)), + memRefType.getShape())); + replaceSubsequentUses(shapeOp, shapeOp.dst(), constantOp.getResult()); + rewriter.replaceOp(shapeOp, {}); + return matchSuccess(); + } + return matchFailure(); + } +}; +} // namespace + +void ShapeOp::getCanonicalizationPatterns(OwningRewritePatternList &results, + MLIRContext *context) { + results.insert(context); +} + +//===----------------------------------------------------------------------===// +// iree_ll_seq.length +//===----------------------------------------------------------------------===// + +namespace { +struct FoldLengthOp : public OpRewritePattern { + using OpRewritePattern::OpRewritePattern; + PatternMatchResult matchAndRewrite(LengthOp lengthOp, + PatternRewriter &rewriter) const override { + auto memRefType = lengthOp.input()->getType().cast(); + if (memRefType.hasStaticShape()) { + auto constantOp = rewriter.create( + lengthOp.getLoc(), + rewriter.getMemRefType({}, rewriter.getIntegerType(64)), + rewriter.getDenseIntElementsAttr( + rewriter.getTensorType({}, rewriter.getIntegerType(64)), + {memRefType.getNumElements()})); + replaceSubsequentUses(lengthOp, lengthOp.dst(), constantOp.getResult()); + rewriter.replaceOp(lengthOp, {}); + return matchSuccess(); + } + return matchFailure(); + } +}; +} // namespace + +void LengthOp::getCanonicalizationPatterns(OwningRewritePatternList &results, + MLIRContext *context) { + results.insert(context); +} + +//===----------------------------------------------------------------------===// +// iree_ll_seq.compute_offset +//===----------------------------------------------------------------------===// + +namespace { +struct FoldComputeOffsetOp : public OpRewritePattern { + using OpRewritePattern::OpRewritePattern; + PatternMatchResult matchAndRewrite(ComputeOffsetOp computeOffsetOp, + PatternRewriter &rewriter) const override { + ElementsAttr shapeAttr; + ElementsAttr indicesAttr; + if (!matchPattern(computeOffsetOp.shape(), m_Constant(&shapeAttr)) || + !matchPattern(computeOffsetOp.indices(), m_Constant(&indicesAttr))) { + return matchFailure(); + } + + int64_t offset = 0; + for (unsigned i = 0; i < indicesAttr.getNumElements(); ++i) { + int64_t axisOffset = + indicesAttr.getValue({i}).cast().getInt(); + for (unsigned j = i + 1; j < shapeAttr.getNumElements(); ++j) { + axisOffset *= shapeAttr.getValue({j}).cast().getInt(); + } + offset += axisOffset; + } + offset *= computeOffsetOp.elementSize().getZExtValue(); + + auto constantOp = rewriter.create( + computeOffsetOp.getLoc(), + rewriter.getMemRefType({}, rewriter.getIntegerType(64)), + rewriter.getDenseIntElementsAttr( + rewriter.getTensorType({}, rewriter.getIntegerType(64)), {offset})); + replaceSubsequentUses(computeOffsetOp, computeOffsetOp.dst(), + constantOp.getResult()); + rewriter.replaceOp(computeOffsetOp, {}); + return matchSuccess(); + } +}; +} // namespace + +void ComputeOffsetOp::getCanonicalizationPatterns( + OwningRewritePatternList &results, MLIRContext *context) { + results.insert(context); +} + +//===----------------------------------------------------------------------===// +// iree_ll_seq.compute_range +//===----------------------------------------------------------------------===// + +namespace { +struct FoldComputeRangeOp : public OpRewritePattern { + using OpRewritePattern::OpRewritePattern; + PatternMatchResult matchAndRewrite(ComputeRangeOp computeRangeOp, + PatternRewriter &rewriter) const override { + ElementsAttr shapeAttr; + ElementsAttr indicesAttr; + ElementsAttr lengthsAttr; + if (!matchPattern(computeRangeOp.shape(), m_Constant(&shapeAttr)) || + !matchPattern(computeRangeOp.indices(), m_Constant(&indicesAttr)) || + !matchPattern(computeRangeOp.lengths(), m_Constant(&lengthsAttr))) { + return matchFailure(); + } + + int64_t offset = 0; + int64_t length = computeRangeOp.elementSize().getZExtValue(); + for (unsigned i = 0; i < indicesAttr.getNumElements(); ++i) { + int64_t axisOffset = + indicesAttr.getValue({i}).cast().getInt(); + for (unsigned j = i + 1; j < shapeAttr.getNumElements(); ++j) { + axisOffset *= shapeAttr.getValue({j}).cast().getInt(); + } + offset += axisOffset; + length *= lengthsAttr.getValue({i}).cast().getInt(); + } + offset *= computeRangeOp.elementSize().getZExtValue(); + + auto offsetConstantOp = rewriter.create( + computeRangeOp.getLoc(), + rewriter.getMemRefType({}, rewriter.getIntegerType(64)), + rewriter.getDenseIntElementsAttr( + rewriter.getTensorType({}, rewriter.getIntegerType(64)), {offset})); + replaceSubsequentUses(computeRangeOp, computeRangeOp.dstOffset(), + offsetConstantOp.getResult()); + auto lengthConstantOp = rewriter.create( + computeRangeOp.getLoc(), + rewriter.getMemRefType({}, rewriter.getIntegerType(64)), + rewriter.getDenseIntElementsAttr( + rewriter.getTensorType({}, rewriter.getIntegerType(64)), {length})); + replaceSubsequentUses(computeRangeOp, computeRangeOp.dstLength(), + lengthConstantOp.getResult()); + rewriter.replaceOp(computeRangeOp, {}); + return matchSuccess(); + } +}; +} // namespace + +void ComputeRangeOp::getCanonicalizationPatterns( + OwningRewritePatternList &results, MLIRContext *context) { + results.insert(context); +} + +//===----------------------------------------------------------------------===// +// iree_ll_seq.dynamic_copy +//===----------------------------------------------------------------------===// + +namespace { +struct MakeDynamicCopyOpStatic : public OpRewritePattern { + using OpRewritePattern::OpRewritePattern; + PatternMatchResult matchAndRewrite(DynamicCopyOp dynamicCopyOp, + PatternRewriter &rewriter) const override { + ElementsAttr srcOffsetAttr; + ElementsAttr dstOffsetAttr; + ElementsAttr lengthAttr; + if (!matchPattern(dynamicCopyOp.srcOffset(), m_Constant(&srcOffsetAttr)) || + !matchPattern(dynamicCopyOp.dstOffset(), m_Constant(&dstOffsetAttr)) || + !matchPattern(dynamicCopyOp.length(), m_Constant(&lengthAttr))) { + return matchFailure(); + } + + rewriter.replaceOpWithNewOp( + dynamicCopyOp, dynamicCopyOp.src(), + srcOffsetAttr.getValue({}).cast(), dynamicCopyOp.dst(), + dstOffsetAttr.getValue({}).cast(), + lengthAttr.getValue({}).cast()); + return matchSuccess(); + } +}; +} // namespace + +void DynamicCopyOp::getCanonicalizationPatterns( + OwningRewritePatternList &results, MLIRContext *context) { + results.insert(context); +} + +//===----------------------------------------------------------------------===// +// iree_ll_seq.dynamic_fill +//===----------------------------------------------------------------------===// + +namespace { +struct MakeDynamicFillOpStatic : public OpRewritePattern { + using OpRewritePattern::OpRewritePattern; + PatternMatchResult matchAndRewrite(DynamicFillOp dynamicFillOp, + PatternRewriter &rewriter) const override { + ElementsAttr valueAttr; + ElementsAttr dstOffsetAttr; + ElementsAttr lengthAttr; + if (!matchPattern(dynamicFillOp.value(), m_Constant(&valueAttr)) || + !matchPattern(dynamicFillOp.dstOffset(), m_Constant(&dstOffsetAttr)) || + !matchPattern(dynamicFillOp.length(), m_Constant(&lengthAttr))) { + return matchFailure(); + } + + rewriter.replaceOpWithNewOp( + dynamicFillOp, valueAttr.getValue({}).cast(), + dynamicFillOp.dst(), dstOffsetAttr.getValue({}).cast(), + lengthAttr.getValue({}).cast()); + return matchSuccess(); + } +}; +} // namespace + +void DynamicFillOp::getCanonicalizationPatterns( + OwningRewritePatternList &results, MLIRContext *context) { + results.insert(context); +} + +#define GET_OP_CLASSES +#include "third_party/mlir_edge/iree/compiler/IR/Sequencer/LLOps.cpp.inc" + +} // namespace LL +} // namespace IREESeq +} // namespace iree_compiler +} // namespace mlir diff --git a/compiler/IR/Sequencer/LLOps.h b/compiler/IR/Sequencer/LLOps.h new file mode 100644 index 000000000000..f343d50093d9 --- /dev/null +++ b/compiler/IR/Sequencer/LLOps.h @@ -0,0 +1,38 @@ +// Copyright 2019 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#ifndef THIRD_PARTY_MLIR_EDGE_IREE_COMPILER_IR_SEQUENCER_LLOPS_H_ +#define THIRD_PARTY_MLIR_EDGE_IREE_COMPILER_IR_SEQUENCER_LLOPS_H_ + +#include "third_party/llvm/llvm/projects/google_mlir/include/mlir/Dialect/StandardOps/Ops.h" +#include "third_party/llvm/llvm/projects/google_mlir/include/mlir/IR/Attributes.h" +#include "third_party/llvm/llvm/projects/google_mlir/include/mlir/IR/Dialect.h" +#include "third_party/llvm/llvm/projects/google_mlir/include/mlir/IR/OpDefinition.h" +#include "third_party/llvm/llvm/projects/google_mlir/include/mlir/IR/StandardTypes.h" +#include "third_party/mlir_edge/iree/compiler/IR/Types.h" + +namespace mlir { +namespace iree_compiler { +namespace IREESeq { +namespace LL { + +#define GET_OP_CLASSES +#include "third_party/mlir_edge/iree/compiler/IR/Sequencer/LLOps.h.inc" + +} // namespace LL +} // namespace IREESeq +} // namespace iree_compiler +} // namespace mlir + +#endif // THIRD_PARTY_MLIR_EDGE_IREE_COMPILER_IR_SEQUENCER_LLOPS_H_ diff --git a/compiler/IR/Sequencer/LLOps.td b/compiler/IR/Sequencer/LLOps.td new file mode 100644 index 000000000000..49dae69c3501 --- /dev/null +++ b/compiler/IR/Sequencer/LLOps.td @@ -0,0 +1,578 @@ +// IREE low-level sequencer op definitions. +// These map 1:1 with the bytecode, accept only MemRef types and generally use +// output parameters instead of return types. +// +// The source of truth for bytecode opcodes is: +// iree/schemas/bytecode/sequencer_bytecode_v0.h +// +// Note that in this dialect we cannot use folders: they require that all +// operands are possible to make constants where we use output arguments that +// will never be constant. Instead we can use canonicalization patterns to +// match constant input operands and do the folding by replacing output operands +// with the new values. + +#ifdef IREE_SEQUENCER_LL_OPS +#else +#define IREE_SEQUENCER_LL_OPS + +#ifdef IREE_OP_BASE +#else +include "third_party/mlir_edge/iree/compiler/IR/OpBase.td" +#endif // IREE_OP_BASE + +def IREESeqLL_Dialect : Dialect { + let name = "iree_ll_seq"; + let cppNamespace = "IREESeq::LL"; +} + +//===----------------------------------------------------------------------===// +// Base op classes +//===----------------------------------------------------------------------===// + +class IREESeqLL_Op traits = []> : + Op { + bit hasCustomSerializer = 0; +} + +class IREESeqLL_PureOp traits = []> : + IREESeqLL_Op; + +class IREESeqLL_UnaryOp traits = []> : IREESeqLL_Op { + let arguments = (ins type:$input, type:$dst); +} + +class IREESeqLL_BinaryOp traits = []> : IREESeqLL_Op { + let arguments = (ins type:$lhs, type:$rhs, type:$dst); +} + +class IREESeqLL_TernaryOp traits = []> + : IREESeqLL_Op { + let arguments = (ins type : $a, type : $b, type : $c, type : $dst); +} + +//===----------------------------------------------------------------------===// +// Low-level sequencer ops +//===----------------------------------------------------------------------===// + +def IREESeqLL_ConstantOp : IREESeqLL_PureOp<"constant"> { + let arguments = (ins ElementsAttr:$value); + let results = (outs IREELL_MemRef); + + // TODO(b/132296600): make tablegen follow the style guide. + let extraClassDeclaration = [{ + Attribute getValue() { return value(); } + }]; + + let hasFolder = 1; +} + +def IREESeqLL_CallOp : IREESeqLL_Op<"call"> { + let arguments = (ins SymbolRefAttr:$callee, Variadic); + let results = (outs Variadic); + + let builders = [OpBuilder< + "Builder *builder, OperationState *result, FuncOp callee," + "ArrayRef operands = {}", [{ + result->addOperands(operands); + result->addAttribute("callee", builder->getSymbolRefAttr(callee)); + result->addTypes(callee.getType().getResults()); + }]>, OpBuilder< + "Builder *builder, OperationState *result, StringRef callee," + "ArrayRef results, ArrayRef operands = {}", [{ + result->addOperands(operands); + result->addAttribute("callee", builder->getSymbolRefAttr(callee)); + result->addTypes(results); + }]>]; + + let extraClassDeclaration = [{ + // TODO(b/132296600): make tablegen follow the style guide. + StringRef getCallee() { return callee(); } + FunctionType getCalleeType(); + + // TODO(b/133879130): make tablegen support variadic operand accessors. + /// Get the argument operands to the called function. + operand_range getArgOperands() { + return {arg_operand_begin(), arg_operand_end()}; + } + + operand_iterator arg_operand_begin() { return operand_begin(); } + operand_iterator arg_operand_end() { return operand_end(); } + }]; + + let parser = [{ return parse$cppClass(parser, result); }]; + let printer = [{ return print$cppClass(p, *this); }]; +} + +// TODO(benvanik): add verifier that target isExternal. +def IREESeqLL_CallImportOp : IREESeqLL_Op<"call_import"> { + let arguments = (ins SymbolRefAttr:$callee, Variadic); + let results = (outs Variadic); + + let builders = [OpBuilder< + "Builder *builder, OperationState *result, FuncOp callee," + "ArrayRef operands = {}", [{ + result->addOperands(operands); + result->addAttribute("callee", builder->getSymbolRefAttr(callee)); + result->addTypes(callee.getType().getResults()); + }]>, OpBuilder< + "Builder *builder, OperationState *result, StringRef callee," + "ArrayRef results, ArrayRef operands = {}", [{ + result->addOperands(operands); + result->addAttribute("callee", builder->getSymbolRefAttr(callee)); + result->addTypes(results); + }]>]; + + let extraClassDeclaration = [{ + // TODO(b/132296600): make tablegen follow the style guide. + StringRef getCallee() { return callee(); } + FunctionType getCalleeType(); + + // TODO(b/133879130): make tablegen support variadic operand accessors. + /// Get the argument operands to the called function. + operand_range getArgOperands() { + return {arg_operand_begin(), arg_operand_end()}; + } + + operand_iterator arg_operand_begin() { return operand_begin(); } + operand_iterator arg_operand_end() { return operand_end(); } + }]; + + let parser = [{ return parse$cppClass(parser, result); }]; + let printer = [{ return print$cppClass(p, *this); }]; +} + +def IREESeqLL_CallIndirectOp : IREESeqLL_Op<"call_indirect"> { + let arguments = (ins FunctionType:$callee, Variadic:$operands); + let results = (outs Variadic); + + let builders = [OpBuilder< + "Builder *, OperationState *result, Value *callee," + "ArrayRef operands = {}", [{ + result->operands.push_back(callee); + result->addOperands(operands); + result->addTypes(callee->getType().cast().getResults()); + }]>]; + + let extraClassDeclaration = [{ + Value *getCallee() { return getOperand(0); } + + /// Get the argument operands to the called function. + operand_range getArgOperands() { + return {arg_operand_begin(), arg_operand_end()}; + } + + operand_iterator arg_operand_begin() { return ++operand_begin(); } + operand_iterator arg_operand_end() { return operand_end(); } + }]; + + let parser = [{ return parse$cppClass(parser, result); }]; + let printer = [{ return print$cppClass(p, *this); }]; +} + +def IREESeqLL_ReturnOp : IREESeqLL_Op<"return", [Terminator]> { + let arguments = (ins Variadic:$operands); + + let builders = [OpBuilder< + "Builder *b, OperationState *result", [{ build(b, result, llvm::None); }] + >]; + + let parser = [{ return parse$cppClass(parser, result); }]; + let printer = [{ return print$cppClass(p, *this); }]; +} + +def IREESeqLL_BranchOp : IREESeqLL_Op<"br", [Terminator]> { + let arguments = (ins Variadic:$operands); + + let skipDefaultBuilders = 1; + let builders = [OpBuilder< + "Builder *, OperationState *result, Block *dest, " + "ArrayRef operands = {}", [{ + result->addSuccessor(dest, operands); + }]>]; + + let extraClassDeclaration = [{ + Block *getDest(); + void setDest(Block *block); + + /// Erase the operand at 'index' from the operand list. + void eraseOperand(unsigned index); + }]; + + let parser = [{ return parse$cppClass(parser, result); }]; + let printer = [{ return print$cppClass(p, *this); }]; +} + +def IREESeqLL_CondBranchOp : IREESeqLL_Op<"cond_br", [Terminator]> { + let arguments = (ins + IREELL_BoolScalar:$condition, + Variadic:$branchOperands + ); + + let skipDefaultBuilders = 1; + let builders = [OpBuilder< + "Builder *, OperationState *result, Value *condition," + "Block *trueDest, ArrayRef trueOperands," + "Block *falseDest, ArrayRef falseOperands", [{ + result->addOperands(condition); + result->addSuccessor(trueDest, trueOperands); + result->addSuccessor(falseDest, falseOperands); + }]>]; + + let extraClassDeclaration = [{ + // These are the indices into the dests list. + enum { trueIndex = 0, falseIndex = 1 }; + + // The condition operand is the first operand in the list. + Value *getCondition() { return getOperand(0); } + + /// Return the destination if the condition is true. + Block *getTrueDest() { + return getOperation()->getSuccessor(trueIndex); + } + + /// Return the destination if the condition is false. + Block *getFalseDest() { + return getOperation()->getSuccessor(falseIndex); + } + + // Accessors for operands to the 'true' destination. + Value *getTrueOperand(unsigned idx) { + assert(idx < getNumTrueOperands()); + return getOperand(getTrueDestOperandIndex() + idx); + } + + void setTrueOperand(unsigned idx, Value *value) { + assert(idx < getNumTrueOperands()); + setOperand(getTrueDestOperandIndex() + idx, value); + } + + operand_iterator true_operand_begin() { + return operand_begin() + getTrueDestOperandIndex(); + } + operand_iterator true_operand_end() { + return true_operand_begin() + getNumTrueOperands(); + } + operand_range getTrueOperands() { + return {true_operand_begin(), true_operand_end()}; + } + + unsigned getNumTrueOperands() { + return getOperation()->getNumSuccessorOperands(trueIndex); + } + + /// Erase the operand at 'index' from the true operand list. + void eraseTrueOperand(unsigned index) { + getOperation()->eraseSuccessorOperand(trueIndex, index); + } + + // Accessors for operands to the 'false' destination. + Value *getFalseOperand(unsigned idx) { + assert(idx < getNumFalseOperands()); + return getOperand(getFalseDestOperandIndex() + idx); + } + void setFalseOperand(unsigned idx, Value *value) { + assert(idx < getNumFalseOperands()); + setOperand(getFalseDestOperandIndex() + idx, value); + } + + operand_iterator false_operand_begin() { return true_operand_end(); } + operand_iterator false_operand_end() { + return false_operand_begin() + getNumFalseOperands(); + } + operand_range getFalseOperands() { + return {false_operand_begin(), false_operand_end()}; + } + + unsigned getNumFalseOperands() { + return getOperation()->getNumSuccessorOperands(falseIndex); + } + + /// Erase the operand at 'index' from the false operand list. + void eraseFalseOperand(unsigned index) { + getOperation()->eraseSuccessorOperand(falseIndex, index); + } + + private: + /// Get the index of the first true destination operand. + unsigned getTrueDestOperandIndex() { return 1; } + + /// Get the index of the first false destination operand. + unsigned getFalseDestOperandIndex() { + return getTrueDestOperandIndex() + getNumTrueOperands(); + } + }]; + + let parser = [{ return parse$cppClass(parser, result); }]; + let printer = [{ return print$cppClass(p, *this); }]; +} + +def IREESeqLL_DynamicDispatchOp : IREESeqLL_Op<"dynamic_dispatch"> { + let arguments = (ins + SymbolRefAttr:$executable, + SymbolRefAttr:$entry_point, + IREELL_IntMemRef:$workload, + Variadic:$operands + ); + let results = (outs Variadic); + + let builders = [OpBuilder< + "Builder *builder, OperationState *result, StringRef executable," + "StringRef entry_point, Value *workload," + "ArrayRef results, ArrayRef operands = {}", [{ + result->addOperands({workload}); + result->addOperands(operands); + result->addAttribute("executable", builder->getSymbolRefAttr(executable)); + result->addAttribute("entry_point", builder->getSymbolRefAttr(entry_point)); + result->addTypes(results); + }]>]; + + let extraClassDeclaration = [{ + // TODO(b/132296600): make tablegen follow the style guide. + StringRef getExecutable() { return executable(); } + StringRef getEntryPoint() { return entry_point(); } + FunctionType getEntryPointType(); + + Value *getWorkload() { return getOperand(0); } + + // TODO(b/133879130): make tablegen support variadic operand accessors. + operand_range getArgOperands() { + return {arg_operand_begin(), arg_operand_end()}; + } + operand_iterator arg_operand_begin() { return operand_begin() + 1; } + operand_iterator arg_operand_end() { return operand_end(); } + + operand_type_range getArgOperandTypes() { + return {arg_operand_type_begin(), arg_operand_type_end()}; + } + operand_type_iterator arg_operand_type_begin() { + return operand_type_iterator(arg_operand_begin()); + } + operand_type_iterator arg_operand_type_end() { + return operand_type_iterator(arg_operand_end()); + } + }]; + + let parser = [{ return parse$cppClass(parser, result); }]; + let printer = [{ return print$cppClass(p, *this); }]; + let verifier = [{ return verify$cppClass(*this); }]; + let hasCanonicalizer = 1; +} + +def IREESeqLL_StaticDispatchOp : IREESeqLL_Op<"static_dispatch"> { + let arguments = (ins + SymbolRefAttr:$executable, + SymbolRefAttr:$entry_point, + I32ElementsAttr:$workload, + Variadic:$operands + ); + let results = (outs Variadic); + + let builders = [OpBuilder< + "Builder *builder, OperationState *result, StringRef executable," + "StringRef entry_point, ElementsAttr workload," + "ArrayRef results, ArrayRef operands = {}", [{ + result->addAttribute("workload", workload); + result->addOperands(operands); + result->addAttribute("executable", builder->getSymbolRefAttr(executable)); + result->addAttribute("entry_point", builder->getSymbolRefAttr(entry_point)); + result->addTypes(results); + }]>]; + + let extraClassDeclaration = [{ + // TODO(b/132296600): make tablegen follow the style guide. + StringRef getExecutable() { return executable(); } + StringRef getEntryPoint() { return entry_point(); } + FunctionType getEntryPointType(); + + ElementsAttr getWorkload() { return workload(); } + + // TODO(b/133879130): make tablegen support variadic operand accessors. + operand_range getArgOperands() { + return {arg_operand_begin(), arg_operand_end()}; + } + operand_iterator arg_operand_begin() { return operand_begin(); } + operand_iterator arg_operand_end() { return operand_end(); } + + operand_type_range getArgOperandTypes() { + return {arg_operand_type_begin(), arg_operand_type_end()}; + } + operand_type_iterator arg_operand_type_begin() { + return operand_type_iterator(arg_operand_begin()); + } + operand_type_iterator arg_operand_type_end() { + return operand_type_iterator(arg_operand_end()); + } + }]; + + let parser = [{ return parse$cppClass(parser, result); }]; + let printer = [{ return print$cppClass(p, *this); }]; + let verifier = [{ return verify$cppClass(*this); }]; +} + +def IREESeqLL_AllocStaticOp : IREESeqLL_PureOp<"alloc_static"> { + // TODO(benvanik): attributes and args. + let results = (outs IREELL_MemRef); +} + +def IREESeqLL_AllocStackOp : IREESeqLL_PureOp<"alloc_stack"> { + // TODO(benvanik): attributes and args. + let arguments = (ins Variadic:$dim_pieces); + let results = (outs IREELL_MemRef); +} + +def IREESeqLL_AllocStackInitOp : IREESeqLL_PureOp<"alloc_stack_init"> { + // TODO(benvanik): attributes and args. + let arguments = (ins Variadic:$dim_pieces); + let results = (outs IREELL_MemRef); +} + +def IREESeqLL_AllocHeapOp : IREESeqLL_PureOp<"alloc_heap"> { + // TODO(benvanik): attributes and args. + let arguments = (ins Variadic:$dim_pieces); + let results = (outs IREELL_MemRef); +} + +def IREESeqLL_DiscardOp : IREESeqLL_Op<"discard"> { + let arguments = (ins IREELL_MemRef); +} + +def IREESeqLL_ShapeOp : IREESeqLL_Op<"shape"> { + let arguments = (ins IREELL_MemRef:$input, IREELL_I32MemRef:$dst); + + let hasCanonicalizer = 1; +} + +def IREESeqLL_LengthOp : IREESeqLL_Op<"length"> { + let arguments = (ins IREELL_MemRef:$input, IREELL_I32Scalar:$dst); + + let hasCanonicalizer = 1; +} + +def IREESeqLL_ComputeOffsetOp : IREESeqLL_Op<"compute_offset"> { + let arguments = (ins + IREELL_1DIntMemRef:$shape, + I8Attr:$elementSize, + IREELL_1DIntMemRef:$indices, + IREELL_I32Scalar:$dst + ); + + let hasCanonicalizer = 1; +} + +def IREESeqLL_ComputeRangeOp : IREESeqLL_Op<"compute_range"> { + let arguments = (ins + IREELL_1DIntMemRef:$shape, + I8Attr:$elementSize, + IREELL_1DIntMemRef:$indices, + IREELL_1DIntMemRef:$lengths, + IREELL_I32Scalar:$dstOffset, + IREELL_I32Scalar:$dstLength + ); + + let hasCanonicalizer = 1; +} + +def IREESeqLL_DynamicSliceOp : IREESeqLL_PureOp<"dynamic_slice", [ + AllElementTypesMatch<["src", "result"]> +]> { + let arguments = (ins + IREELL_MemRef:$src, + IREELL_IntScalar:$offset, + IREELL_IntScalar:$length + ); + let results = (outs IREELL_MemRef:$result); +} + +def IREESeqLL_StaticSliceOp : IREESeqLL_PureOp<"static_slice", [ + AllElementTypesMatch<["src", "result"]> +]> { + let arguments = (ins + IREELL_MemRef:$src, + I64Attr:$offset, + I64Attr:$length + ); + let results = (outs IREELL_MemRef:$result); +} + +def IREESeqLL_DynamicCopyOp : IREESeqLL_Op<"dynamic_copy"> { + let arguments = (ins + IREELL_MemRef:$src, + IREELL_IndexScalar:$srcOffset, + IREELL_MemRef:$dst, + IREELL_IndexScalar:$dstOffset, + IREELL_IndexScalar:$length + ); + + let hasCanonicalizer = 1; +} + +def IREESeqLL_StaticCopyOp : IREESeqLL_Op<"static_copy"> { + let arguments = (ins + IREELL_MemRef:$src, + I64Attr:$srcOffset, + IREELL_MemRef:$dst, + I64Attr:$dstOffset, + I64Attr:$length + ); +} + +def IREESeqLL_DynamicFillOp : IREESeqLL_Op<"dynamic_fill"> { + let arguments = (ins + IREELL_I32Scalar:$value, + IREELL_MemRef:$dst, + IREELL_IndexScalar:$dstOffset, + IREELL_IndexScalar:$length + ); + + let hasCanonicalizer = 1; +} + +def IREESeqLL_StaticFillOp : IREESeqLL_Op<"static_fill"> { + let arguments = (ins + I32Attr:$value, + IREELL_MemRef:$dst, + I64Attr:$dstOffset, + I64Attr:$length + ); +} + +def IREESeqLL_CloneOp : + IREESeqLL_PureOp<"clone", [SameOperandsAndResultType]> { + let arguments = (ins IREELL_MemRef:$src); + let results = (outs IREELL_MemRef); +} + +def IREESeqLL_AssignOp : + IREESeqLL_Op<"assign", [SameOperandsAndResultType]> { + let arguments = (ins IREELL_MemRef:$src); + let results = (outs IREELL_MemRef); +} + +def IREESeqLL_CondAssignOp : IREESeqLL_Op<"cond_assign"> { + let arguments = (ins + IREELL_BoolScalar:$cond, + IREELL_MemRef:$lhs, + IREELL_MemRef:$rhs + ); + let results = (outs IREELL_MemRef); +} + +def IREESeqLL_ReshapeOp : IREESeqLL_Op<"reshape"> { + let arguments = (ins IREELL_MemRef:$input, IREELL_1DIntMemRef:$shape); + let results = (outs IREELL_MemRef); +} + +def IREESeqLL_TraceOp : IREESeqLL_Op<"trace"> { + let arguments = (ins Variadic:$srcs); +} + +def IREESeqLL_CondBreakOp : IREESeqLL_Op<"cond_break"> { + let arguments = (ins IREELL_BoolScalar:$cond); +} + +def IREESeqLL_BreakOp : IREESeqLL_Op<"break">; + +#endif // IREE_SEQUENCER_LL_OPS diff --git a/compiler/IR/Sequencer/OpWriters.cpp b/compiler/IR/Sequencer/OpWriters.cpp new file mode 100644 index 000000000000..78e9bf6f26b5 --- /dev/null +++ b/compiler/IR/Sequencer/OpWriters.cpp @@ -0,0 +1,266 @@ +// Copyright 2019 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "third_party/mlir_edge/iree/compiler/IR/Sequencer/OpWriters.h" + +#include "third_party/llvm/llvm/projects/google_mlir/include/mlir/IR/Module.h" +#include "third_party/llvm/llvm/projects/google_mlir/include/mlir/IR/OpImplementation.h" +#include "third_party/mlir_edge/iree/compiler/IR/Sequencer/LLOps.h" +#include "third_party/mlir_edge/iree/compiler/IR/StructureOps.h" +#include "third_party/mlir_edge/iree/compiler/Serialization/BytecodeWriter.h" +#include "third_party/mlir_edge/iree/compiler/Utils/Macros.h" +#include "third_party/mlir_edge/iree/schemas/bytecode/sequencer_bytecode_v0.h" + +namespace mlir { +namespace iree_compiler { + +namespace { + +//===----------------------------------------------------------------------===// +// Sequencer ops +//===----------------------------------------------------------------------===// + +LogicalResult writeOp(IREESeq::LL::ConstantOp op, BytecodeWriter *writer) { + RETURN_IF_FAILURE(writer->WriteOpcode(iree::SequencerOpcode::kConstant)); + auto memRefType = op.getType().dyn_cast(); + if (!memRefType) { + return op.emitError() + << "Constant has an unsupported type; must be a memref: " + << op.getType(); + } + RETURN_IF_FAILURE(writer->WriteConstant(memRefType, op.getAttr("value"))); + RETURN_IF_FAILURE(writer->WriteLocal(op.getResult())); + return success(); +} + +LogicalResult writeOp(IREESeq::LL::CallOp op, BytecodeWriter *writer) { + auto module = op.getOperation()->getParentOfType(); + auto callee = module.lookupSymbol(op.getCallee()); + // TODO(benvanik): switch with kCallTail if attr exists. + RETURN_IF_FAILURE(writer->WriteOpcode(iree::SequencerOpcode::kCall)); + RETURN_IF_FAILURE(writer->WriteFunctionOrdinal(callee)); + RETURN_IF_FAILURE(writer->WriteLocals(op.getArgOperands())); + RETURN_IF_FAILURE(writer->WriteLocals(op.getResults())); + return success(); +} + +LogicalResult writeOp(IREESeq::LL::CallImportOp op, BytecodeWriter *writer) { + auto module = op.getOperation()->getParentOfType(); + auto callee = module.lookupSymbol(op.getCallee()); + // TODO(benvanik): transforms to convert Call->CallImport. + // TODO(benvanik): switch with kCallTail if attr exists. + if (callee.isExternal()) { + RETURN_IF_FAILURE(writer->WriteOpcode(iree::SequencerOpcode::kCallImport)); + } else { + RETURN_IF_FAILURE(writer->WriteOpcode(iree::SequencerOpcode::kCall)); + } + RETURN_IF_FAILURE(writer->WriteImportOrdinal(callee)); + RETURN_IF_FAILURE(writer->WriteLocals(op.getArgOperands())); + RETURN_IF_FAILURE(writer->WriteLocals(op.getResults())); + return success(); +} + +LogicalResult writeOp(IREESeq::LL::CallIndirectOp op, BytecodeWriter *writer) { + RETURN_IF_FAILURE(writer->WriteOpcode(iree::SequencerOpcode::kCallIndirect)); + RETURN_IF_FAILURE(writer->WriteTypeIndex(op.getCallee()->getType())); + RETURN_IF_FAILURE(writer->WriteLocal(op.getCallee())); + RETURN_IF_FAILURE(writer->WriteLocals(op.getArgOperands())); + RETURN_IF_FAILURE(writer->WriteLocals(op.getResults())); + return success(); +} + +LogicalResult writeOp(IREESeq::LL::BranchOp op, BytecodeWriter *writer) { + RETURN_IF_FAILURE(writer->WriteOpcode(iree::SequencerOpcode::kBranch)); + RETURN_IF_FAILURE(writer->WriteBlockOffset(op.getDest())); + RETURN_IF_FAILURE(writer->WriteCount(op.getNumOperands())); + for (int i = 0; i < op.getNumOperands(); ++i) { + // Copy src->dst. + RETURN_IF_FAILURE(writer->WriteLocal(op.getOperand(i))); + RETURN_IF_FAILURE(writer->WriteLocal(op.getDest()->getArgument(i))); + } + return success(); +} + +LogicalResult writeOp(IREESeq::LL::CondBranchOp op, BytecodeWriter *writer) { + RETURN_IF_FAILURE(writer->WriteOpcode(iree::SequencerOpcode::kCondBranch)); + RETURN_IF_FAILURE(writer->WriteLocal(op.getCondition())); + RETURN_IF_FAILURE(writer->WriteBlockOffset(op.getTrueDest())); + RETURN_IF_FAILURE(writer->WriteCount(op.getNumTrueOperands())); + for (int i = 0; i < op.getNumTrueOperands(); ++i) { + // Copy src->dst. + RETURN_IF_FAILURE(writer->WriteLocal(op.getTrueOperand(i))); + RETURN_IF_FAILURE(writer->WriteLocal(op.getTrueDest()->getArgument(i))); + } + RETURN_IF_FAILURE(writer->WriteBlockOffset(op.getFalseDest())); + RETURN_IF_FAILURE(writer->WriteCount(op.getNumFalseOperands())); + for (int i = 0; i < op.getNumFalseOperands(); ++i) { + // Copy src->dst. + RETURN_IF_FAILURE(writer->WriteLocal(op.getFalseOperand(i))); + RETURN_IF_FAILURE(writer->WriteLocal(op.getFalseDest()->getArgument(i))); + } + return success(); +} + +LogicalResult writeDispatchOpExecutableRef(Operation *op, StringRef executable, + StringRef entryPoint, + BytecodeWriter *writer) { + auto module = op->getParentOfType(); + auto multiArchExecutableOp = + module.lookupSymbol(executable); + if (!multiArchExecutableOp) { + return op->emitError() << "Executable @" << executable.str() + << " not found in module"; + } + + auto executableOrdinalAttr = multiArchExecutableOp.getAttr("iree.ordinal") + .dyn_cast_or_null(); + if (!executableOrdinalAttr) { + return op->emitError() << "No ordinal assigned to executable"; + } + int executableOrdinal = executableOrdinalAttr.getInt(); + + // TODO(benvanik): move an export table to the MAE to make this cleaner. + auto executableOp = + cast(multiArchExecutableOp.getBlock().front()); + auto entryPointOp = + executableOp.getInnerModule().lookupSymbol(entryPoint); + if (!entryPointOp) { + return op->emitError() << "Entry point @" << entryPoint.str() + << " not found in executable @" << executable.str(); + } + if (!entryPointOp.getAttr("iree.ordinal")) { + return op->emitError() << "No ordinal assigned to entry point"; + } + int entryPointOrdinal = + entryPointOp.getAttr("iree.ordinal").cast().getInt(); + + RETURN_IF_FAILURE(writer->WriteUint32(executableOrdinal)); + RETURN_IF_FAILURE(writer->WriteUint16(entryPointOrdinal)); + + return success(); +} + +LogicalResult writeOp(IREESeq::LL::DynamicDispatchOp op, + BytecodeWriter *writer) { + RETURN_IF_FAILURE( + writer->WriteOpcode(iree::SequencerOpcode::kDynamicDispatch)); + RETURN_IF_FAILURE(writeDispatchOpExecutableRef(op, op.getExecutable(), + op.getEntryPoint(), writer)); + RETURN_IF_FAILURE(writer->WriteLocal(op.getWorkload())); + RETURN_IF_FAILURE(writer->WriteLocals(op.getArgOperands())); + // TODO(benvanik): support output arg group (or change to tags). + RETURN_IF_FAILURE(writer->WriteCount(/*output_arg_count*/ 0)); + RETURN_IF_FAILURE(writer->WriteLocals(op.getResults())); + return success(); +} + +LogicalResult writeOp(IREESeq::LL::StaticDispatchOp op, + BytecodeWriter *writer) { + RETURN_IF_FAILURE( + writer->WriteOpcode(iree::SequencerOpcode::kStaticDispatch)); + RETURN_IF_FAILURE(writeDispatchOpExecutableRef(op, op.getExecutable(), + op.getEntryPoint(), writer)); + auto workloadAttr = op.getWorkload(); + RETURN_IF_FAILURE( + writer->WriteInt32(workloadAttr.getValue({0}).getInt())); + RETURN_IF_FAILURE( + writer->WriteInt32(workloadAttr.getValue({1}).getInt())); + RETURN_IF_FAILURE( + writer->WriteInt32(workloadAttr.getValue({2}).getInt())); + RETURN_IF_FAILURE(writer->WriteLocals(op.getArgOperands())); + // TODO(benvanik): support output arg group (or change to tags). + RETURN_IF_FAILURE(writer->WriteCount(/*output_arg_count*/ 0)); + RETURN_IF_FAILURE(writer->WriteLocals(op.getResults())); + return success(); +} + +LogicalResult writeOp(IREESeq::LL::AllocHeapOp op, BytecodeWriter *writer) { + auto memRefType = op.getType().cast(); + RETURN_IF_FAILURE(writer->WriteOpcode(iree::SequencerOpcode::kAllocHeap)); + RETURN_IF_FAILURE(writer->WriteInt32(0)); + RETURN_IF_FAILURE(writer->WriteTypeIndex(memRefType.getElementType())); + RETURN_IF_FAILURE(writer->WriteShapePieces(memRefType)); + RETURN_IF_FAILURE(writer->WriteLocals(op.getOperands())); + RETURN_IF_FAILURE(writer->WriteLocal(op.getResult())); + return success(); +} + +LogicalResult writeOp(IREESeq::LL::ComputeRangeOp op, BytecodeWriter *writer) { + RETURN_IF_FAILURE(writer->WriteOpcode(iree::SequencerOpcode::kComputeRange)); + RETURN_IF_FAILURE(writer->WriteLocal(op.shape())); + RETURN_IF_FAILURE(writer->WriteUint8(op.elementSize().getZExtValue())); + RETURN_IF_FAILURE(writer->WriteLocal(op.indices())); + RETURN_IF_FAILURE(writer->WriteLocal(op.lengths())); + RETURN_IF_FAILURE(writer->WriteLocal(op.dstOffset())); + RETURN_IF_FAILURE(writer->WriteLocal(op.dstLength())); + return success(); +} + +LogicalResult writeOp(IREESeq::LL::StaticSliceOp op, BytecodeWriter *writer) { + RETURN_IF_FAILURE(writer->WriteOpcode(iree::SequencerOpcode::kStaticSlice)); + RETURN_IF_FAILURE(writer->WriteLocal(op.src())); + RETURN_IF_FAILURE(writer->WriteInt32(op.offset().getZExtValue())); + RETURN_IF_FAILURE(writer->WriteInt32(op.length().getZExtValue())); + RETURN_IF_FAILURE(writer->WriteTypeIndex(op.getResult()->getType())); + RETURN_IF_FAILURE( + writer->WriteShapePieces(op.getResult()->getType().cast())); + RETURN_IF_FAILURE(writer->WriteLocal(op.getResult())); + return success(); +} + +LogicalResult writeOp(IREESeq::LL::StaticCopyOp op, BytecodeWriter *writer) { + RETURN_IF_FAILURE(writer->WriteOpcode(iree::SequencerOpcode::kStaticCopy)); + RETURN_IF_FAILURE(writer->WriteLocal(op.src())); + RETURN_IF_FAILURE(writer->WriteInt32(op.srcOffset().getZExtValue())); + RETURN_IF_FAILURE(writer->WriteLocal(op.dst())); + RETURN_IF_FAILURE(writer->WriteInt32(op.dstOffset().getZExtValue())); + RETURN_IF_FAILURE(writer->WriteInt32(op.length().getZExtValue())); + return success(); +} + +LogicalResult writeOp(IREESeq::LL::StaticFillOp op, BytecodeWriter *writer) { + RETURN_IF_FAILURE(writer->WriteOpcode(iree::SequencerOpcode::kStaticFill)); + RETURN_IF_FAILURE(writer->WriteInt32(op.value().getZExtValue())); + RETURN_IF_FAILURE(writer->WriteLocal(op.dst())); + RETURN_IF_FAILURE(writer->WriteInt32(op.dstOffset().getZExtValue())); + RETURN_IF_FAILURE(writer->WriteInt32(op.length().getZExtValue())); + return success(); +} + +} // namespace + +void registerSequencerCustomWriters(VMFunctionBuilder *builder) { +#define REGISTER_CUSTOM_WRITER_IMPL(op_type) \ + builder->RegisterCustomWriter( \ + op_type::getOperationName(), \ + +[](Operation *op, BytecodeWriter *writer) { \ + return writeOp(cast(op), writer); \ + }); + REGISTER_CUSTOM_WRITER_IMPL(IREESeq::LL::ConstantOp); + REGISTER_CUSTOM_WRITER_IMPL(IREESeq::LL::CallOp); + REGISTER_CUSTOM_WRITER_IMPL(IREESeq::LL::CallImportOp); + REGISTER_CUSTOM_WRITER_IMPL(IREESeq::LL::CallIndirectOp); + REGISTER_CUSTOM_WRITER_IMPL(IREESeq::LL::BranchOp); + REGISTER_CUSTOM_WRITER_IMPL(IREESeq::LL::CondBranchOp); + REGISTER_CUSTOM_WRITER_IMPL(IREESeq::LL::DynamicDispatchOp); + REGISTER_CUSTOM_WRITER_IMPL(IREESeq::LL::StaticDispatchOp); + REGISTER_CUSTOM_WRITER_IMPL(IREESeq::LL::AllocHeapOp); + REGISTER_CUSTOM_WRITER_IMPL(IREESeq::LL::ComputeRangeOp); + REGISTER_CUSTOM_WRITER_IMPL(IREESeq::LL::StaticSliceOp); + REGISTER_CUSTOM_WRITER_IMPL(IREESeq::LL::StaticCopyOp); + REGISTER_CUSTOM_WRITER_IMPL(IREESeq::LL::StaticFillOp); +} + +} // namespace iree_compiler +} // namespace mlir diff --git a/compiler/IR/Sequencer/OpWriters.h b/compiler/IR/Sequencer/OpWriters.h new file mode 100644 index 000000000000..1177aa0eeb29 --- /dev/null +++ b/compiler/IR/Sequencer/OpWriters.h @@ -0,0 +1,30 @@ +// Copyright 2019 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#ifndef THIRD_PARTY_MLIR_EDGE_IREE_COMPILER_IR_SEQUENCER_OPWRITERS_H_ +#define THIRD_PARTY_MLIR_EDGE_IREE_COMPILER_IR_SEQUENCER_OPWRITERS_H_ + +#include "third_party/mlir_edge/iree/compiler/Serialization/VMFunctionBuilder.h" + +namespace mlir { +namespace iree_compiler { + +// Registers custom op writers with the builder. +// Ops not registered will use the generic writer. +void registerSequencerCustomWriters(VMFunctionBuilder *builder); + +} // namespace iree_compiler +} // namespace mlir + +#endif // THIRD_PARTY_MLIR_EDGE_IREE_COMPILER_IR_SEQUENCER_OPWRITERS_H_ diff --git a/compiler/IR/Sequencer/test/concat.mlir b/compiler/IR/Sequencer/test/concat.mlir new file mode 100644 index 000000000000..f208524b7cd3 --- /dev/null +++ b/compiler/IR/Sequencer/test/concat.mlir @@ -0,0 +1,68 @@ +// RUN: iree-opt -canonicalize %s --split-input-file | FileCheck %s --dump-input=fail + +// CHECK-LABEL: func @concat.1D +// CHECK-SAME: [[ARG0:%[a-zA-Z0-9]+]] +// CHECK-SAME: [[ARG1:%[a-zA-Z0-9]+]] +func @concat.1D(%arg0 : memref<4xi32>, %arg1 : memref<3xi32>) -> memref<7xi32> { + // CHECK-DAG: [[SRC_INDICES:%.+]] = iree.constant dense<0> : tensor<1x + // CHECK-DAG: [[DST:%.+]] = "iree_hl_seq.alloc_heap"() : () -> memref<7xi32> + + // CHECK-DAG: [[DST_INDICES0:%.+]] = iree.constant dense<0> : tensor<1x + // CHECK-DAG: [[LENGTHS0:%.+]] = iree.constant dense<4> : tensor<1x + // CHECK-DAG: "iree_hl_seq.copy"([[ARG0]], [[SRC_INDICES]], [[DST]], [[DST_INDICES0]], [[LENGTHS0]]) + + // CHECK-DAG: [[DST_INDICES1:%.+]] = iree.constant dense<4> : tensor<1x + // CHECK-DAG: [[LENGTHS1:%.+]] = iree.constant dense<3> : tensor<1x + // CHECK-DAG: "iree_hl_seq.copy"([[ARG1]], [[SRC_INDICES]], [[DST]], [[DST_INDICES1]], [[LENGTHS1]]) + + %0 = "iree_hl_seq.concat"(%arg0, %arg1) {dimension = 0 : i32} : (memref<4xi32>, memref<3xi32>) -> memref<7xi32> + + // CHECK: return [[DST]] + return %0 : memref<7xi32> +} + +// ----- + +// CHECK-LABEL: func @concat.2D.Dim0 +// CHECK-SAME: [[ARG0:%[a-zA-Z0-9]+]] +// CHECK-SAME: [[ARG1:%[a-zA-Z0-9]+]] +func @concat.2D.Dim0(%arg0 : memref<4x4xi32>, %arg1 : memref<3x4xi32>) -> memref<7x4xi32> { + // CHECK-DAG: [[SRC_INDICES:%.+]] = iree.constant dense<0> : tensor<2x + // CHECK-DAG: [[DST:%.+]] = "iree_hl_seq.alloc_heap"() : () -> memref<7x4xi32> + + // CHECK-DAG: [[DST_INDICES0:%.+]] = iree.constant dense<0> : tensor<2x + // CHECK-DAG: [[LENGTHS0:%.+]] = iree.constant dense<4> : tensor<2x + // CHECK-DAG: "iree_hl_seq.copy"([[ARG0]], [[SRC_INDICES]], [[DST]], [[DST_INDICES0]], [[LENGTHS0]]) + + // CHECK-DAG: [[DST_INDICES1:%.+]] = iree.constant dense<[4, 0]> + // CHECK-DAG: [[LENGTHS1:%.+]] = iree.constant dense<[3, 4]> + // CHECK-DAG: "iree_hl_seq.copy"([[ARG1]], [[SRC_INDICES]], [[DST]], [[DST_INDICES1]], [[LENGTHS1]]) + + %0 = "iree_hl_seq.concat"(%arg0, %arg1) {dimension = 0 : i32} : (memref<4x4xi32>, memref<3x4xi32>) -> memref<7x4xi32> + + // CHECK: return [[DST]] + return %0 : memref<7x4xi32> +} + +// ----- + +// CHECK-LABEL: func @concat.2D.Dim1 +// CHECK-SAME: [[ARG0:%[a-zA-Z0-9]+]] +// CHECK-SAME: [[ARG1:%[a-zA-Z0-9]+]] +func @concat.2D.Dim1(%arg0 : memref<4x4xi32>, %arg1 : memref<4x3xi32>) -> memref<4x7xi32> { + // CHECK-DAG: [[SRC_INDICES:%.+]] = iree.constant dense<0> : tensor<2x + // CHECK-DAG: [[DST:%.+]] = "iree_hl_seq.alloc_heap"() : () -> memref<4x7xi32> + + // CHECK-DAG: [[DST_INDICES0:%.+]] = iree.constant dense<0> : tensor<2x + // CHECK-DAG: [[LENGTHS0:%.+]] = iree.constant dense<4> : tensor<2x + // CHECK-DAG: "iree_hl_seq.copy"([[ARG0]], [[SRC_INDICES]], [[DST]], [[DST_INDICES0]], [[LENGTHS0]]) + + // CHECK-DAG: [[DST_INDICES1:%.+]] = iree.constant dense<[0, 4]> + // CHECK-DAG: [[LENGTHS1:%.+]] = iree.constant dense<[4, 3]> + // CHECK-DAG: "iree_hl_seq.copy"([[ARG1]], [[SRC_INDICES]], [[DST]], [[DST_INDICES1]], [[LENGTHS1]]) + + %0 = "iree_hl_seq.concat"(%arg0, %arg1) {dimension = 1 : i32} : (memref<4x4xi32>, memref<4x3xi32>) -> memref<4x7xi32> + + // CHECK: return [[DST]] + return %0 : memref<4x7xi32> +} diff --git a/compiler/IR/StructureOps.cpp b/compiler/IR/StructureOps.cpp new file mode 100644 index 000000000000..f84cbbc9f8c1 --- /dev/null +++ b/compiler/IR/StructureOps.cpp @@ -0,0 +1,250 @@ +// Copyright 2019 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "third_party/mlir_edge/iree/compiler/IR/StructureOps.h" + +#include "third_party/llvm/llvm/include/llvm/ADT/STLExtras.h" +#include "third_party/llvm/llvm/include/llvm/ADT/SmallString.h" +#include "third_party/llvm/llvm/include/llvm/ADT/SmallVector.h" +#include "third_party/llvm/llvm/projects/google_mlir/include/mlir/IR/Attributes.h" +#include "third_party/llvm/llvm/projects/google_mlir/include/mlir/IR/Builders.h" +#include "third_party/llvm/llvm/projects/google_mlir/include/mlir/IR/Diagnostics.h" +#include "third_party/llvm/llvm/projects/google_mlir/include/mlir/IR/OpImplementation.h" +#include "third_party/llvm/llvm/projects/google_mlir/include/mlir/IR/SymbolTable.h" +#include "third_party/llvm/llvm/projects/google_mlir/include/mlir/IR/Value.h" +#include "third_party/llvm/llvm/projects/google_mlir/include/mlir/Support/LLVM.h" +#include "third_party/llvm/llvm/projects/google_mlir/include/mlir/Support/STLExtras.h" +#include "third_party/mlir_edge/iree/compiler/IR/Types.h" + +namespace mlir { +namespace iree_compiler { +namespace IREE { + +//===----------------------------------------------------------------------===// +// Generic printers and parsers. +//===----------------------------------------------------------------------===// + +// Parses an op that has no inputs and no outputs. +static ParseResult parseNoIOOp(OpAsmParser *parser, OperationState *state) { + if (failed(parser->parseOptionalAttributeDict(state->attributes))) { + return failure(); + } + return success(); +} + +// Prints an op that has no inputs and no outputs. +static void printNoIOOp(Operation *op, OpAsmPrinter *printer) { + *printer << op->getName(); + printer->printOptionalAttrDict(op->getAttrs()); +} + +//===----------------------------------------------------------------------===// +// iree.module +//===----------------------------------------------------------------------===// + +void ModuleOp::build(Builder *builder, OperationState *state) { + ensureTerminator(*state->addRegion(), *builder, state->location); +} + +static ParseResult parseModuleOp(OpAsmParser *parser, OperationState *state) { + Region *body = state->addRegion(); + if (parser->parseRegion(*body, /*arguments=*/{}, /*argTypes=*/{})) { + return failure(); + } + if (parser->parseOptionalAttributeDict(state->attributes)) { + return failure(); + } + ModuleOp::ensureTerminator(*body, parser->getBuilder(), state->location); + return success(); +} + +static void printModuleOp(OpAsmPrinter *printer, Operation *op) { + *printer << op->getName(); + printer->printRegion(op->getRegion(0), /*printEntryBlockArgs=*/false, + /*printBlockTerminators=*/false); + printer->printOptionalAttrDict(op->getAttrs()); +} + +//===----------------------------------------------------------------------===// +// iree.multi_arch_executable +//===----------------------------------------------------------------------===// + +void MultiArchExecutableOp::build(Builder *builder, OperationState *state, + StringRef name) { + state->addAttribute(SymbolTable::getSymbolAttrName(), + builder->getStringAttr(name)); + ensureTerminator(*state->addRegion(), *builder, state->location); +} + +static ParseResult parseMultiArchExecutableOp(OpAsmParser *parser, + OperationState *state) { + auto &builder = parser->getBuilder(); + + // Parse the name as a symbol reference attr and then convert to a string. + SymbolRefAttr nameAttr; + if (failed(parser->parseAttribute(nameAttr, SymbolTable::getSymbolAttrName(), + state->attributes))) { + return failure(); + } + state->attributes.back().second = builder.getStringAttr(nameAttr.getValue()); + + if (succeeded(parser->parseOptionalLSquare())) { + IntegerAttr ordinalAttr; + if (failed(parser->parseAttribute(ordinalAttr, builder.getIntegerType(32), + "iree.ordinal", state->attributes)) || + failed(parser->parseRSquare())) { + return failure(); + } + } + + if (failed(parser->parseLParen()) || failed(parser->parseRParen())) { + return failure(); + } + + Region *body = state->addRegion(); + if (failed(parser->parseRegion(*body, /*arguments=*/{}, /*argTypes=*/{}))) { + return failure(); + } + if (succeeded(parser->parseOptionalKeyword("attributes"))) { + if (failed(parser->parseOptionalAttributeDict(state->attributes))) { + return failure(); + } + } + + MultiArchExecutableOp::ensureTerminator(*body, builder, state->location); + + return success(); +} + +static void printMultiArchExecutableOp(OpAsmPrinter *printer, + MultiArchExecutableOp op) { + *printer << op.getOperationName() << " @" << op.sym_name(); + if (auto ordinalAttr = + op.getAttr("iree.ordinal").dyn_cast_or_null()) { + *printer << "[" << ordinalAttr.getInt() << "]"; + } + *printer << "()"; + + printer->printRegion(op.body(), /*printEntryBlockArgs=*/false, + /*printBlockTerminators=*/false); + + // Print out executable attributes, if present. + SmallVector ignoredAttrs = { + SymbolTable::getSymbolAttrName(), + "iree.ordinal", + }; + SmallVector attrs( + llvm::make_filter_range(op.getAttrs(), [&](const NamedAttribute &attr) { + return llvm::count(ignoredAttrs, attr.first) == 0; + })); + if (!attrs.empty()) { + *printer << "\n attributes "; + printer->printOptionalAttrDict(attrs); + } +} + +//===----------------------------------------------------------------------===// +// iree.executable +//===----------------------------------------------------------------------===// + +void ExecutableOp::build(Builder *builder, OperationState *state, + IREE::ExecutableFormat format) { + state->addAttribute( + "format", builder->getI32IntegerAttr(static_cast(format))); + ensureTerminator(*state->addRegion(), *builder, state->location); +} + +static ParseResult parseExecutableOp(OpAsmParser *parser, + OperationState *state) { + auto &builder = parser->getBuilder(); + + if (succeeded(parser->parseOptionalLSquare())) { + IntegerAttr ordinalAttr; + if (failed(parser->parseAttribute(ordinalAttr, builder.getIntegerType(32), + "iree.ordinal", state->attributes)) || + failed(parser->parseRSquare())) { + return failure(); + } + } + + IntegerAttr executableOrdinalAttr; + StringAttr formatAttr; + llvm::SMLoc formatLoc; + if (failed(parser->parseLParen()) || + failed(parser->getCurrentLocation(&formatLoc)) || + failed(parser->parseAttribute(formatAttr, "format", state->attributes))) { + return failure(); + } + auto format = symbolizeExecutableFormat(formatAttr.getValue()); + if (!format.hasValue()) { + return parser->emitError(formatLoc) + << "Unknown executable format " << formatAttr.getValue(); + } + state->attributes.back().second = + builder.getI32IntegerAttr(static_cast(format.getValue())); + + Region *body = state->addRegion(); + if (failed(parser->parseRegion(*body, /*arguments=*/{}, /*argTypes=*/{}))) { + return failure(); + } + if (succeeded(parser->parseOptionalKeyword("attributes"))) { + if (failed(parser->parseOptionalAttributeDict(state->attributes))) { + return failure(); + } + } + + ExecutableOp::ensureTerminator(*body, parser->getBuilder(), state->location); + + return success(); +} + +static void printExecutableOp(OpAsmPrinter *printer, ExecutableOp op) { + *printer << op.getOperationName(); + if (auto ordinalAttr = + op.getAttr("iree.ordinal").dyn_cast_or_null()) { + *printer << "[" << ordinalAttr.getInt() << "]"; + } + *printer << "("; + auto format = symbolizeExecutableFormat(op.format()); + if (format.hasValue()) { + *printer << stringifyExecutableFormat(format.getValue()); + } else { + *printer << "INVALID FORMAT"; + } + *printer << ")"; + + printer->printRegion(op.body(), /*printEntryBlockArgs=*/false, + /*printBlockTerminators=*/false); + + // Print out executable attributes, if present. + SmallVector ignoredAttrs = { + "iree.ordinal", + "format", + }; + SmallVector attrs( + llvm::make_filter_range(op.getAttrs(), [&](const NamedAttribute &attr) { + return llvm::count(ignoredAttrs, attr.first) == 0; + })); + if (!attrs.empty()) { + *printer << "\n attributes "; + printer->printOptionalAttrDict(attrs); + } +} + +#define GET_OP_CLASSES +#include "third_party/mlir_edge/iree/compiler/IR/StructureOps.cpp.inc" + +} // namespace IREE +} // namespace iree_compiler +} // namespace mlir diff --git a/compiler/IR/StructureOps.h b/compiler/IR/StructureOps.h new file mode 100644 index 000000000000..6f71013f2bab --- /dev/null +++ b/compiler/IR/StructureOps.h @@ -0,0 +1,40 @@ +// Copyright 2019 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#ifndef THIRD_PARTY_MLIR_EDGE_IREE_COMPILER_IR_STRUCTUREOPS_H_ +#define THIRD_PARTY_MLIR_EDGE_IREE_COMPILER_IR_STRUCTUREOPS_H_ + +#include + +#include "third_party/llvm/llvm/projects/google_mlir/include/mlir/Dialect/StandardOps/Ops.h" +#include "third_party/llvm/llvm/projects/google_mlir/include/mlir/IR/Attributes.h" +#include "third_party/llvm/llvm/projects/google_mlir/include/mlir/IR/Dialect.h" +#include "third_party/llvm/llvm/projects/google_mlir/include/mlir/IR/FunctionSupport.h" +#include "third_party/llvm/llvm/projects/google_mlir/include/mlir/IR/Module.h" +#include "third_party/llvm/llvm/projects/google_mlir/include/mlir/IR/OpDefinition.h" +#include "third_party/llvm/llvm/projects/google_mlir/include/mlir/IR/StandardTypes.h" +#include "third_party/mlir_edge/iree/compiler/IR/Types.h" + +namespace mlir { +namespace iree_compiler { +namespace IREE { + +#define GET_OP_CLASSES +#include "third_party/mlir_edge/iree/compiler/IR/StructureOps.h.inc" + +} // namespace IREE +} // namespace iree_compiler +} // namespace mlir + +#endif // THIRD_PARTY_MLIR_EDGE_IREE_COMPILER_IR_STRUCTUREOPS_H_ diff --git a/compiler/IR/StructureOps.td b/compiler/IR/StructureOps.td new file mode 100644 index 000000000000..dc9b1966eda7 --- /dev/null +++ b/compiler/IR/StructureOps.td @@ -0,0 +1,122 @@ +// Structural ops such as 'module' and 'executable'. +// These are used to organize IREE IR into regions representing ops that act at +// the sequencer level (coarse control flow/scheduling) and ops that perform +// actual work (math/etc) on runtime execution backends. + +#ifdef IREE_STRUCTURE_OPS +#else +#define IREE_STRUCTURE_OPS + +#ifdef IREE_OP_BASE +#else +include "third_party/mlir_edge/iree/compiler/IR/OpBase.td" +#endif // IREE_OP_BASE + +class IREE_StructureOp traits = []> : + Op { + let parser = [{ return parse$cppClass(parser, result); }]; + let printer = [{ print$cppClass(p, *this); }]; +} + +def IREE_ModuleOp : + IREE_StructureOp<"module", [ + SingleBlockImplicitTerminator<"ModuleEndOp">, + NativeOpTrait<"SymbolTable"> + ]> { + let regions = (region SizedRegion<1>:$body); + let extraClassDeclaration = [{ + Block& getBlock() { + return this->getOperation()->getRegion(0).front(); + } + }]; + + let skipDefaultBuilders = 1; + let builders = [OpBuilder<"Builder *, OperationState *state">]; +} + +def IREE_ModuleEndOp : + IREE_StructureOp<"_module_end", [ + IREE_ModuleOnly, + Terminator + ]> { + let parser = [{ return parseNoIOOp(parser, result); }]; + let printer = [{ printNoIOOp(getOperation(), p); }]; +} + +def IREE_MultiArchExecutableOp : + IREE_StructureOp<"multi_arch_executable", [ + // TODO(benvanik): make iree.module work and make this IREE_ModuleOnly. + SingleBlockImplicitTerminator<"MultiArchExecutableEndOp"> + ]> { + let arguments = (ins + StrAttr:$sym_name, + OptionalAttr:$ordinal + ); + + let regions = (region SizedRegion<1>:$body); + let extraClassDeclaration = [{ + StringRef getName() { + return this->getOperation()->template getAttrOfType( + ::mlir::SymbolTable::getSymbolAttrName()).getValue(); + } + + Region& getBody() { + return this->getOperation()->getRegion(0); + } + Block& getBlock() { + return this->getOperation()->getRegion(0).front(); + } + }]; + + let skipDefaultBuilders = 1; + let builders = [ + OpBuilder<"Builder *builder, OperationState *state, StringRef name">, + ]; +} + +def IREE_MultiArchExecutableEndOp : + IREE_StructureOp<"_multi_arch_executable_end", [ + IREE_MultiArchExecutableOnly, + Terminator + ]> { + let parser = [{ return parseNoIOOp(parser, result); }]; + let printer = [{ printNoIOOp(getOperation(), p); }]; +} + +def IREE_ExecutableOp : + IREE_StructureOp<"executable", [ + SingleBlockImplicitTerminator<"ExecutableEndOp">, + NativeOpTrait<"SymbolTable"> + ]> { + let arguments = (ins + IREE_ExecutableFormatAttr:$format, + OptionalAttr:$ordinal + ); + + let regions = (region SizedRegion<1>:$body); + let extraClassDeclaration = [{ + Region& getBody() { + return this->getOperation()->getRegion(0); + } + Block& getBlock() { + return this->getOperation()->getRegion(0).front(); + } + ::mlir::ModuleOp getInnerModule() { + return *getBlock().getOps<::mlir::ModuleOp>().begin(); + } + }]; + + let skipDefaultBuilders = 1; + let builders = [ + OpBuilder<[{Builder *builder, OperationState *state, + ExecutableFormat executable_format}]>, + ]; +} + +def IREE_ExecutableEndOp : + IREE_StructureOp<"_executable_end", [Terminator, IREE_ExecutableOnly]> { + let parser = [{ return parseNoIOOp(parser, result); }]; + let printer = [{ printNoIOOp(getOperation(), p); }]; +} + +#endif // IREE_STRUCTURE_OPS diff --git a/compiler/IR/Traits.cpp b/compiler/IR/Traits.cpp new file mode 100644 index 000000000000..0fb99c099e90 --- /dev/null +++ b/compiler/IR/Traits.cpp @@ -0,0 +1,23 @@ +// Copyright 2019 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "third_party/mlir_edge/iree/compiler/IR/Traits.h" + +namespace mlir { +namespace iree_compiler { + +// TODO(benvanik): traits. + +} // namespace iree_compiler +} // namespace mlir diff --git a/compiler/IR/Traits.h b/compiler/IR/Traits.h new file mode 100644 index 000000000000..8477f7690f60 --- /dev/null +++ b/compiler/IR/Traits.h @@ -0,0 +1,26 @@ +// Copyright 2019 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#ifndef THIRD_PARTY_MLIR_EDGE_IREE_COMPILER_IR_TRAITS_H_ +#define THIRD_PARTY_MLIR_EDGE_IREE_COMPILER_IR_TRAITS_H_ + +namespace mlir { +namespace iree_compiler { + +// TODO(benvanik): traits. + +} // namespace iree_compiler +} // namespace mlir + +#endif // THIRD_PARTY_MLIR_EDGE_IREE_COMPILER_IR_TRAITS_H_ diff --git a/compiler/IR/Types.cpp b/compiler/IR/Types.cpp new file mode 100644 index 000000000000..737598bff98c --- /dev/null +++ b/compiler/IR/Types.cpp @@ -0,0 +1,53 @@ +// Copyright 2019 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "third_party/mlir_edge/iree/compiler/IR/Types.h" + +#include "third_party/mlir_edge/iree/compiler/IR/Enums.cpp.inc" + +namespace mlir { +namespace iree_compiler { + +// static +DeviceType DeviceType::get(MLIRContext *context) { + return Base::get(context, TypeKind::Device); +} + +// static +DeviceGroupType DeviceGroupType::get(MLIRContext *context) { + return Base::get(context, TypeKind::DeviceGroup); +} + +// static +CommandBufferType CommandBufferType::get(MLIRContext *context) { + return Base::get(context, TypeKind::CommandBuffer); +} + +// static +EventType EventType::get(MLIRContext *context) { + return Base::get(context, TypeKind::Event); +} + +// static +SemaphoreType SemaphoreType::get(MLIRContext *context) { + return Base::get(context, TypeKind::Semaphore); +} + +// static +FenceType FenceType::get(MLIRContext *context) { + return Base::get(context, TypeKind::Fence); +} + +} // namespace iree_compiler +} // namespace mlir diff --git a/compiler/IR/Types.h b/compiler/IR/Types.h new file mode 100644 index 000000000000..871c0156eeeb --- /dev/null +++ b/compiler/IR/Types.h @@ -0,0 +1,113 @@ +// Copyright 2019 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#ifndef THIRD_PARTY_MLIR_EDGE_IREE_COMPILER_IR_TYPES_H_ +#define THIRD_PARTY_MLIR_EDGE_IREE_COMPILER_IR_TYPES_H_ + +#include + +#include "third_party/llvm/llvm/include/llvm/ADT/DenseMapInfo.h" +#include "third_party/llvm/llvm/include/llvm/ADT/StringSwitch.h" +#include "third_party/llvm/llvm/projects/google_mlir/include/mlir/IR/Types.h" +#include "third_party/llvm/llvm/projects/google_mlir/include/mlir/Support/LLVM.h" +#include "third_party/mlir_edge/iree/compiler/IR/Enums.h.inc" + +namespace mlir { +namespace iree_compiler { + +namespace TypeKind { +enum Kind { + Device = Type::FIRST_IREE_TYPE, + DeviceGroup, + CommandBuffer, + Event, + Semaphore, + Fence, +}; +} // namespace TypeKind + +// clang-format off +#define IREE_TYPE_TABLE(map) \ + map("device", TypeKind::Device, DeviceType) \ + map("device_group", TypeKind::DeviceGroup, DeviceGroupType) \ + map("command_buffer", TypeKind::CommandBuffer, CommandBufferType) \ + map("event", TypeKind::Event, EventType) \ + map("semaphore", TypeKind::Semaphore, SemaphoreType) \ + map("fence", TypeKind::Fence, FenceType) +// clang-format on + +// iree.device mapping to a runtime-resolved device type. +class DeviceType : public Type::TypeBase { + public: + using Base::Base; + + static bool kindof(unsigned kind) { return kind == TypeKind::Device; } + + static DeviceType get(MLIRContext *context); +}; + +// iree.device_group relating multiple iree.device requirements with each other. +class DeviceGroupType : public Type::TypeBase { + public: + using Base::Base; + + static bool kindof(unsigned kind) { return kind == TypeKind::DeviceGroup; } + + static DeviceGroupType get(MLIRContext *context); +}; + +// iree.command_buffer mapping to an iree::hal::CommandBuffer. +class CommandBufferType : public Type::TypeBase { + public: + using Base::Base; + + static bool kindof(unsigned kind) { return kind == TypeKind::CommandBuffer; } + + static CommandBufferType get(MLIRContext *context); +}; + +// iree.event mapping to an iree::hal::Event. +class EventType : public Type::TypeBase { + public: + using Base::Base; + + static bool kindof(unsigned kind) { return kind == TypeKind::Event; } + + static EventType get(MLIRContext *context); +}; + +// iree.semaphore mapping to an iree::hal::Semaphore. +class SemaphoreType : public Type::TypeBase { + public: + using Base::Base; + + static bool kindof(unsigned kind) { return kind == TypeKind::Semaphore; } + + static SemaphoreType get(MLIRContext *context); +}; + +// iree.fence mapping to an iree::hal::Fence. +class FenceType : public Type::TypeBase { + public: + using Base::Base; + + static bool kindof(unsigned kind) { return kind == TypeKind::Fence; } + + static FenceType get(MLIRContext *context); +}; + +} // namespace iree_compiler +} // namespace mlir + +#endif // THIRD_PARTY_MLIR_EDGE_IREE_COMPILER_IR_TYPES_H_ diff --git a/compiler/IR/test/bindings.mlir b/compiler/IR/test/bindings.mlir new file mode 100644 index 000000000000..db3b85f4e9b4 --- /dev/null +++ b/compiler/IR/test/bindings.mlir @@ -0,0 +1,10 @@ +// RUN: iree-opt %s | FileCheck %s --dump-input=fail + +// CHECK-LABEL: @dispatch_entry +func @dispatch_entry(%arg0: memref<4x2xf32>, %arg1: memref<4x2xf32>) { + // CHECK-NEXT: %0 = iree.load_input(%arg0 : memref<4x2xf32>) : tensor<4x2xf32> + %0 = iree.load_input(%arg0 : memref<4x2xf32>) : tensor<4x2xf32> + // CHECK-NEXT: iree.store_output(%0 : tensor<4x2xf32>, %arg1 : memref<4x2xf32>) + iree.store_output(%0 : tensor<4x2xf32>, %arg1 : memref<4x2xf32>) + return +} diff --git a/compiler/IR/test/dispatch_regions.mlir b/compiler/IR/test/dispatch_regions.mlir new file mode 100644 index 000000000000..c3cb038dcb55 --- /dev/null +++ b/compiler/IR/test/dispatch_regions.mlir @@ -0,0 +1,63 @@ +// RUN: iree-opt %s -split-input-file | FileCheck %s --dump-input=fail + +// CHECK-LABEL: @singleArg +func @singleArg(%arg0 : tensor) { + // CHECK-NEXT: %0 = "some.shape" + // CHECK-NEXT: iree.dispatch_region[%0 : tensor<1xi32>](%arg1 = %arg0 : tensor) { + // CHECK-NEXT: iree.return + // CHECK-NEXT: } + %workload = "some.shape"(%arg0) : (tensor) -> tensor<1xi32> + iree.dispatch_region[%workload : tensor<1xi32>](%i0 = %arg0 : tensor) { + iree.return + } + // CHECK-NEXT: return + return +} + +// ----- + +// CHECK-LABEL: @multipleArgs +func @multipleArgs(%arg0 : tensor, %arg1 : tensor) { + // CHECK-NEXT: %0 = "some.shape" + // CHECK-NEXT: iree.dispatch_region[%0 : tensor<1xi32>](%arg2 = %arg0 : tensor, %arg3 = %arg1 : tensor) { + // CHECK-NEXT: iree.return + // CHECK-NEXT: } + %workload = "some.shape"(%arg0) : (tensor) -> tensor<1xi32> + iree.dispatch_region[%workload : tensor<1xi32>](%i0 = %arg0 : tensor, %i1 = %arg1 : tensor) { + iree.return + } + // CHECK-NEXT: return + return +} + +// ----- + +// CHECK-LABEL: @singleResult +func @singleResult(%arg0 : tensor) -> tensor { + // CHECK-NEXT: %0 = "some.shape" + // CHECK-NEXT: %1 = iree.dispatch_region[%0 : tensor<1xi32>](%arg1 = %arg0 : tensor) : tensor { + // CHECK-NEXT: iree.return %arg1 : tensor + // CHECK-NEXT: } + %workload = "some.shape"(%arg0) : (tensor) -> tensor<1xi32> + %ret0 = iree.dispatch_region[%workload : tensor<1xi32>](%i0 = %arg0 : tensor) : tensor { + iree.return %i0 : tensor + } + // CHECK-NEXT: return %1 : tensor + return %ret0 : tensor +} + +// ----- + +// CHECK-LABEL: @multipleResults +func @multipleResults(%arg0 : tensor) -> (tensor, tensor) { + // CHECK-NEXT: %0 = "some.shape" + // CHECK-NEXT: %1:2 = iree.dispatch_region[%0 : tensor<1xi32>](%arg1 = %arg0 : tensor) : tensor, tensor { + // CHECK-NEXT: iree.return %arg1, %arg1 : tensor, tensor + // CHECK-NEXT: } + %workload = "some.shape"(%arg0) : (tensor) -> tensor<1xi32> + %ret0, %ret1 = iree.dispatch_region[%workload : tensor<1xi32>](%i0 = %arg0 : tensor) : tensor, tensor { + iree.return %i0, %i0 : tensor, tensor + } + // CHECK-NEXT: return %1#0, %1#1 : tensor, tensor + return %ret0, %ret1 : tensor, tensor +} diff --git a/compiler/IR/test/reduction_regions.mlir b/compiler/IR/test/reduction_regions.mlir new file mode 100644 index 000000000000..404337b529fd --- /dev/null +++ b/compiler/IR/test/reduction_regions.mlir @@ -0,0 +1,46 @@ +// RUN: iree-opt %s -split-input-file | FileCheck %s --dump-input=fail + +// CHECK-LABEL: @singleReduction +func @singleReduction(%arg0 : tensor<5x1xf32>) { + // CHECK: %0 = "some.shape"(%arg0) : (tensor<5x1xf32>) -> tensor<1xi32> + %workload = "some.shape"(%arg0) : (tensor<5x1xf32>) -> tensor<1xi32> + // CHECK: %1 = "some.constant"() : () -> tensor + %initialValueF = "some.constant"() : () -> tensor + // CHECK: %2 = iree.reduction_region[%0 : tensor<1xi32>](%arg0) : (tensor<5x1xf32>) -> (tensor<1xf32>) + // CHECK-NEXT: invocation((%arg1, %arg2) = %1 : tensor) { + // CHECK-NEXT: %3 = "my.add"(%arg1, %arg2) : (tensor, tensor) -> tensor + // CHECK-NEXT: iree.return %3 : tensor + // CHECK-NEXT: } {dimensions = dense<[1, 2]> : tensor<2xi64>} + %ret = iree.reduction_region[%workload : tensor<1xi32>](%arg0) : (tensor<5x1xf32>) -> (tensor<1xf32>) + invocation((%i0, %i1) = %initialValueF : tensor) { + %resultF = "my.add"(%i0, %i1) : (tensor, tensor) -> tensor + iree.return %resultF : tensor + } {dimensions = dense<[1, 2]> : tensor<2xi64>} + return +} + +// ----- + +// CHECK-LABEL: @fusedReduction +func @fusedReduction(%arg0 : tensor<5x1xf32>, %arg1 : tensor<5x1xi32>) { + // CHECK: %0 = "some.shape"(%arg0) : (tensor<5x1xf32>) -> tensor<1xi32> + %workload = "some.shape"(%arg0) : (tensor<5x1xf32>) -> tensor<1xi32> + // CHECK: %1 = "some.constant"() : () -> tensor + // CHECK: %2 = "some.constant"() : () -> tensor + %initialValueF = "some.constant"() : () -> tensor + %initialValueI = "some.constant"() : () -> tensor + // CHECK: %3:2 = iree.reduction_region[%0 : tensor<1xi32>](%arg0, %arg1) : (tensor<5x1xf32>, tensor<5x1xi32>) -> (tensor<1xf32>, tensor<1xi32>) + // CHECK-NEXT: invocation((%arg2, %arg3) = %1 : tensor, (%arg4, %arg5) = %2 : tensor) { + // CHECK-NEXT: %4 = "my.add"(%arg2, %arg3) : (tensor, tensor) -> tensor + // CHECK-NEXT: %5 = "my.add"(%arg4, %arg5) : (tensor, tensor) -> tensor + // CHECK-NEXT: iree.return %4, %5 : tensor, tensor + // CHECK-NEXT: } {dimensions = dense<[1, 2]> : tensor<2xi64>} + %ret:2 = iree.reduction_region[%workload : tensor<1xi32>](%arg0, %arg1) : (tensor<5x1xf32>, tensor<5x1xi32>) -> (tensor<1xf32>, tensor<1xi32>) + invocation((%i0, %i1) = %initialValueF : tensor, + (%i2, %i3) = %initialValueI : tensor) { + %resultF = "my.add"(%i0, %i1) : (tensor, tensor) -> tensor + %resultI = "my.add"(%i2, %i3) : (tensor, tensor) -> tensor + iree.return %resultF, %resultI : tensor, tensor + } {dimensions = dense<[1, 2]> : tensor<2xi64>} + return +} diff --git a/compiler/Serialization/BytecodeTables.cpp b/compiler/Serialization/BytecodeTables.cpp new file mode 100644 index 000000000000..2281793ae8f2 --- /dev/null +++ b/compiler/Serialization/BytecodeTables.cpp @@ -0,0 +1,73 @@ +// Copyright 2019 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "third_party/mlir_edge/iree/compiler/Serialization/BytecodeTables.h" + +#include "third_party/llvm/llvm/include/llvm/ADT/STLExtras.h" + +namespace mlir { +namespace iree_compiler { + +namespace { + +// Info tables mapping 1:1 with bytecode ops. +// +// Note that we ensure the table is 256 elements long exactly to make sure +// that unused opcodes are handled gracefully. +#define DECLARE_INFO(ordinal, enum_value, name, flags, operand_encodings, ...) \ + { \ + name, \ + flags, \ + {operand_encodings}, \ + }, + +static const OpcodeInfo kInterpreterInfoTable[256] = { + IREE_INTERPRETER_OPCODE_LIST(DECLARE_INFO, DECLARE_INFO)}; + +static const OpcodeInfo kSequencerInfoTable[256] = { + IREE_SEQUENCER_OPCODE_LIST(DECLARE_INFO, DECLARE_INFO)}; + +#undef DECLARE_INFO + +} // namespace + +llvm::Optional GetInterpreterOpcodeByName( + StringRef name) { + for (int i = 0; i < llvm::array_lengthof(kInterpreterInfoTable); ++i) { + if (name == kInterpreterInfoTable[i].mnemonic) { + return static_cast(i); + } + } + return llvm::None; +} + +const OpcodeInfo& GetInterpreterOpcodeInfo(iree::InterpreterOpcode opcode) { + return kInterpreterInfoTable[static_cast(opcode)]; +} + +llvm::Optional GetSequencerOpcodeByName(StringRef name) { + for (int i = 0; i < llvm::array_lengthof(kSequencerInfoTable); ++i) { + if (name == kSequencerInfoTable[i].mnemonic) { + return static_cast(i); + } + } + return llvm::None; +} + +const OpcodeInfo& GetSequencerOpcodeInfo(iree::SequencerOpcode opcode) { + return kSequencerInfoTable[static_cast(opcode)]; +} + +} // namespace iree_compiler +} // namespace mlir diff --git a/compiler/Serialization/BytecodeTables.h b/compiler/Serialization/BytecodeTables.h new file mode 100644 index 000000000000..6aa1421d35e6 --- /dev/null +++ b/compiler/Serialization/BytecodeTables.h @@ -0,0 +1,52 @@ +// Copyright 2019 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#ifndef THIRD_PARTY_MLIR_EDGE_IREE_COMPILER_SERIALIZATION_BYTECODE_TABLES_H_ +#define THIRD_PARTY_MLIR_EDGE_IREE_COMPILER_SERIALIZATION_BYTECODE_TABLES_H_ + +#include "third_party/llvm/llvm/include/llvm/ADT/Optional.h" +#include "third_party/llvm/llvm/include/llvm/ADT/StringRef.h" +#include "third_party/llvm/llvm/projects/google_mlir/include/mlir/Support/LLVM.h" +#include "third_party/mlir_edge/iree/schemas/bytecode/interpreter_bytecode_v0.h" +#include "third_party/mlir_edge/iree/schemas/bytecode/sequencer_bytecode_v0.h" + +namespace mlir { +namespace iree_compiler { + +struct OpcodeInfo { + const char* mnemonic = nullptr; + iree::OpcodeFlagBitfield flags = iree::OpcodeFlagBitfield::kDefault; + union { + const char operands_value[8] = {0}; + const iree::OperandEncoding operands[8]; + }; +}; + +// Returns an opcode - if found - for the given interpreter op. +llvm::Optional GetInterpreterOpcodeByName( + StringRef name); + +// Returns the info for the given interpreter opcode. +const OpcodeInfo& GetInterpreterOpcodeInfo(iree::InterpreterOpcode opcode); + +// Returns an opcode - if found - for the given sequencer op. +llvm::Optional GetSequencerOpcodeByName(StringRef name); + +// Returns the info for the given sequencer opcode. +const OpcodeInfo& GetSequencerOpcodeInfo(iree::SequencerOpcode opcode); + +} // namespace iree_compiler +} // namespace mlir + +#endif // THIRD_PARTY_MLIR_EDGE_IREE_COMPILER_SERIALIZATION_BYTECODE_TABLES_H_ diff --git a/compiler/Serialization/BytecodeWriter.cpp b/compiler/Serialization/BytecodeWriter.cpp new file mode 100644 index 000000000000..162e774afe51 --- /dev/null +++ b/compiler/Serialization/BytecodeWriter.cpp @@ -0,0 +1,334 @@ +// Copyright 2019 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "third_party/mlir_edge/iree/compiler/Serialization/BytecodeWriter.h" + +#include + +#include "third_party/llvm/llvm/include/llvm/Support/raw_ostream.h" +#include "third_party/llvm/llvm/projects/google_mlir/include/mlir/IR/Attributes.h" +#include "third_party/llvm/llvm/projects/google_mlir/include/mlir/IR/Diagnostics.h" +#include "third_party/llvm/llvm/projects/google_mlir/include/mlir/Support/LLVM.h" +#include "third_party/mlir_edge/iree/compiler/Utils/Macros.h" + +namespace mlir { +namespace iree_compiler { + +LogicalResult BytecodeWriter::WriteCount(int count) { + if (count > UINT8_MAX) { + // TODO(benvanik): varints? + llvm::errs() << "Too many items: " << count + << "; only 0-UINT8_MAX are supported"; + return failure(); + } + return WriteUint8(static_cast(count)); +} + +LogicalResult BytecodeWriter::WriteTypeIndex(Type type) { + iree::BuiltinType type_index; + if (type.isInteger(8)) { + type_index = iree::BuiltinType::kI8; + } else if (type.isInteger(16)) { + type_index = iree::BuiltinType::kI16; + } else if (type.isInteger(32)) { + type_index = iree::BuiltinType::kI32; + } else if (type.isInteger(64)) { + type_index = iree::BuiltinType::kI64; + } else if (type.isF16()) { + type_index = iree::BuiltinType::kF16; + } else if (type.isF32()) { + type_index = iree::BuiltinType::kF32; + } else if (type.isF64()) { + type_index = iree::BuiltinType::kF64; + } else { + // TODO(benvanik): support unknown types as BuiltinType::kOpaque? + return emitError(UnknownLoc::get(type.getContext())) + << "Type " << type << " cannot be represented by a builtin type"; + } + return WriteUint8(static_cast(type_index)); +} + +LogicalResult BytecodeWriter::WriteFunctionOrdinal(FuncOp function) { + auto functionOrdinal = function.getAttrOfType("iree.ordinal"); + if (!functionOrdinal) { + return function.emitError() << "Ordinal not assigned to function"; + } + RETURN_IF_FAILURE(WriteUint32(functionOrdinal.getInt())); + return success(); +} + +LogicalResult BytecodeWriter::WriteImportOrdinal(FuncOp function) { + // For now this is the same as internal function ordinals, though we could + // probably shrink it. + return WriteFunctionOrdinal(function); +} + +LogicalResult BytecodeWriter::WriteConstant(MemRefType memRefType, + Attribute baseAttr) { + // All types are memrefs, so we only need the element type. + RETURN_IF_FAILURE(WriteTypeIndex(memRefType.getElementType())); + + // Write shape (we could optimize this for cases of scalars and such). + RETURN_IF_FAILURE(WriteCount(memRefType.getRank())); + for (int i = 0; i < memRefType.getRank(); ++i) { + RETURN_IF_FAILURE(WriteInt32(memRefType.getDimSize(i))); + } + + if (auto attr = baseAttr.dyn_cast()) { + RETURN_IF_FAILURE( + WriteUint8(static_cast(iree::ConstantEncoding::kSplat))); + return WriteAttributeData(attr.getSplatValue()); + } + RETURN_IF_FAILURE( + WriteUint8(static_cast(iree::ConstantEncoding::kDense))); + return WriteAttributeData(baseAttr); +} + +LogicalResult BytecodeWriter::WriteAttributeData(Attribute baseAttr) { + if (auto attr = baseAttr.dyn_cast()) { + return WriteUint8(attr.getValue() ? 1 : 0); + } else if (auto attr = baseAttr.dyn_cast()) { + if (attr.getType().isIndex()) { + int32_t value = static_cast(attr.getInt()); + return WriteBytes(&value, 4); + } else { + int bitWidth = attr.getValue().getBitWidth(); + switch (bitWidth) { + case 8: + case 16: + case 32: + case 64: + return WriteBytes(attr.getValue().getRawData(), bitWidth / 8); + default: + return emitError(UnknownLoc::get(baseAttr.getContext())) + << "Bit width for integers must be one of 8,16,32,64; others " + "not implemented: " + << bitWidth; + } + } + } else if (auto attr = baseAttr.dyn_cast()) { + int bitWidth = attr.getType().getIntOrFloatBitWidth(); + auto bitcastValue = attr.getValue().bitcastToAPInt(); + switch (bitWidth) { + case 16: + case 32: + case 64: + return WriteBytes(bitcastValue.getRawData(), bitWidth / 8); + default: + return emitError(UnknownLoc::get(baseAttr.getContext())) + << "Bit width for floats must be one of 16,32,64; others " + "not implemented: " + << bitWidth; + } + } else if (auto attr = baseAttr.dyn_cast()) { + // TODO(benvanik): other attribute encodings. + } else if (auto attr = baseAttr.dyn_cast()) { + // TODO(benvanik): other attribute encodings. + } else if (auto attr = baseAttr.dyn_cast()) { + // TODO(benvanik): other attribute encodings. + } else if (auto attr = baseAttr.dyn_cast()) { + // TODO(benvanik): other attribute encodings. + } else if (auto attr = baseAttr.dyn_cast()) { + // TODO(benvanik): other attribute encodings. + } else if (auto attr = baseAttr.dyn_cast()) { + // TODO(benvanik): other attribute encodings. + } else if (auto attr = baseAttr.dyn_cast()) { + return WriteAttributeData(attr.getSplatValue()); + } else if (auto attr = baseAttr.dyn_cast()) { + int elementCount = attr.getType().getNumElements(); + if (elementCount == 0) { + return success(); + } + int bitWidth = attr.getType().getElementTypeBitWidth(); + int byteWidth = bitWidth / 8; + auto dst = ReserveBytes(elementCount * byteWidth); + if (dst.empty()) return failure(); + uint8_t *dstPtr = dst.data(); + for (auto element : attr) { + assert(element.getBitWidth() == bitWidth); + std::memcpy(dstPtr, element.getRawData(), byteWidth); + dstPtr += byteWidth; + } + return success(); + } else if (auto attr = baseAttr.dyn_cast()) { + int elementCount = attr.getType().getNumElements(); + if (elementCount == 0) { + return success(); + } + int bitWidth = attr.getType().getElementTypeBitWidth(); + auto dst = ReserveBytes(elementCount * bitWidth / 8); + if (dst.empty()) return failure(); + uint8_t *dstPtr = dst.data(); + for (auto element : attr) { + auto bitcastValue = element.bitcastToAPInt(); + std::memcpy(dstPtr, bitcastValue.getRawData(), + bitcastValue.getBitWidth() / 8); + dstPtr += bitWidth / 8; + } + return success(); + } else if (auto attr = baseAttr.dyn_cast()) { + // TODO(benvanik): other attribute encodings. + } else if (auto attr = baseAttr.dyn_cast()) { + // TODO(benvanik): other attribute encodings. + } else if (auto attr = baseAttr.dyn_cast()) { + // TODO(benvanik): other attribute encodings. + } + return emitError(UnknownLoc::get(baseAttr.getContext())) + << "Serializer for attribute kind " + << static_cast(baseAttr.getKind()) << " not implemented"; +} + +Optional BytecodeWriter::LookupLocalOrdinal(Value *value) { + int ordinal; + auto it = localMap_.find(value); + if (it != localMap_.end()) { + ordinal = it->second; + } else { + ordinal = localMap_.size(); + localMap_.insert({value, ordinal}); + } + if (ordinal > UINT16_MAX) { + // TODO(benvanik): varints? + emitError(UnknownLoc::get(value->getContext())) + << "Too many ordinals: " << ordinal + << "; only 0-UINT16_MAX are supported"; + return llvm::None; + } + return ordinal; +} + +LogicalResult BytecodeWriter::PrepareLocal(Value *value) { + if (!LookupLocalOrdinal(value).hasValue()) return failure(); + return success(); +} + +LogicalResult BytecodeWriter::WriteLocal(Value *value) { + auto ordinal = LookupLocalOrdinal(value); + if (!ordinal.hasValue()) { + return failure(); + } + if (ordinal > UINT16_MAX) { + // TODO(benvanik): varints? + return emitError(UnknownLoc::get(value->getContext())) + << "Too many locals: " << ordinal.getValue() + << "; only 0-UINT16_MAX are supported"; + } + return WriteUint16(static_cast(ordinal.getValue())); +} + +LogicalResult BytecodeWriter::WriteLocals( + llvm::iterator_range values) { + int count = std::distance(values.begin(), values.end()); + RETURN_IF_FAILURE(WriteCount(count)); + for (auto *value : values) { + RETURN_IF_FAILURE(WriteLocal(value)); + } + return success(); +} + +LogicalResult BytecodeWriter::WriteLocals( + llvm::iterator_range values) { + int count = std::distance(values.begin(), values.end()); + RETURN_IF_FAILURE(WriteCount(count)); + for (auto *value : values) { + RETURN_IF_FAILURE(WriteLocal(value)); + } + return success(); +} + +MutableArrayRef BytecodeWriter::ReserveBytes(size_t dataLength) { + int offset = bytecode_.size(); + bytecode_.resize(offset + dataLength); + return MutableArrayRef( + reinterpret_cast(bytecode_.data()) + offset, dataLength); +} + +LogicalResult BytecodeWriter::WriteBytes(const void *data, size_t dataLength) { + auto dst = ReserveBytes(dataLength); + if (dataLength != dst.size()) { + return failure(); + } + std::memcpy(dst.data(), data, dst.size()); + return success(); +} + +LogicalResult BytecodeWriter::WriteUint8(uint8_t value) { + return WriteBytes(&value, sizeof(value)); +} + +LogicalResult BytecodeWriter::WriteUint16(uint16_t value) { + return WriteBytes(&value, sizeof(value)); +} + +LogicalResult BytecodeWriter::WriteInt32(int32_t value) { + return WriteBytes(&value, sizeof(value)); +} + +LogicalResult BytecodeWriter::WriteUint32(uint32_t value) { + return WriteBytes(&value, sizeof(value)); +} + +LogicalResult BytecodeWriter::WriteElementsAttrInt32(ElementsAttr attr) { + int elementCount = attr.getType().getNumElements(); + RETURN_IF_FAILURE(WriteCount(elementCount)); + for (auto value : attr.getValues()) { + RETURN_IF_FAILURE(WriteInt32(value)); + } + return success(); +} + +LogicalResult BytecodeWriter::WriteShapePieces(const ShapedType &type) { + RETURN_IF_FAILURE(WriteCount(type.getRank())); + for (int64_t dim : type.getShape()) { + RETURN_IF_FAILURE(WriteInt32(dim)); + } + return success(); +} + +LogicalResult BytecodeWriter::WriteShapePieces(ElementsAttr pieces) { + return WriteElementsAttrInt32(pieces); +} + +LogicalResult BytecodeWriter::MarkBlockOffset(Block *block) { + blockOffsets_[block] = bytecode_.size(); + return success(); +} + +LogicalResult BytecodeWriter::WriteBlockOffset(Block *targetBlock) { + // Reserve space for the offset and stash for later fixup. + blockOffsetFixups_.push_back({targetBlock, bytecode_.size()}); + bytecode_.resize(bytecode_.size() + sizeof(int32_t)); + return success(); +} + +LogicalResult BytecodeWriter::FixupOffsets() { + for (const auto &fixup : blockOffsetFixups_) { + auto it = blockOffsets_.find(fixup.first); + if (it == blockOffsets_.end()) { + llvm::errs() << "Block offset not found: " << fixup.first; + return failure(); + } + std::memcpy(bytecode_.data() + fixup.second, &it->second, sizeof(int32_t)); + } + blockOffsetFixups_.clear(); + return success(); +} + +std::vector BytecodeWriter::Finish() { + localMap_.clear(); + return std::move(bytecode_); +} + +} // namespace iree_compiler +} // namespace mlir diff --git a/compiler/Serialization/BytecodeWriter.h b/compiler/Serialization/BytecodeWriter.h new file mode 100644 index 000000000000..7b16ec675d1e --- /dev/null +++ b/compiler/Serialization/BytecodeWriter.h @@ -0,0 +1,96 @@ +// Copyright 2019 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#ifndef THIRD_PARTY_MLIR_EDGE_IREE_COMPILER_SERIALIZATION_BYTECODE_WRITER_H_ +#define THIRD_PARTY_MLIR_EDGE_IREE_COMPILER_SERIALIZATION_BYTECODE_WRITER_H_ + +#include +#include +#include + +#include "third_party/llvm/llvm/include/llvm/ADT/Optional.h" +#include "third_party/llvm/llvm/projects/google_mlir/include/mlir/IR/Attributes.h" +#include "third_party/llvm/llvm/projects/google_mlir/include/mlir/IR/Block.h" +#include "third_party/llvm/llvm/projects/google_mlir/include/mlir/IR/Function.h" +#include "third_party/llvm/llvm/projects/google_mlir/include/mlir/IR/Operation.h" +#include "third_party/llvm/llvm/projects/google_mlir/include/mlir/IR/StandardTypes.h" +#include "third_party/llvm/llvm/projects/google_mlir/include/mlir/IR/Types.h" +#include "third_party/llvm/llvm/projects/google_mlir/include/mlir/IR/Value.h" +#include "third_party/mlir_edge/iree/compiler/IR/StructureOps.h" +#include "third_party/mlir_edge/iree/schemas/bytecode/bytecode_v0.h" + +namespace mlir { +namespace iree_compiler { + +class BytecodeWriter { + public: + int offset() const { return bytecode_.size(); } + + int local_count() const { return localMap_.size(); } + + template + LogicalResult WriteOpcode(T value) { + static_assert(sizeof(T) == sizeof(uint8_t), "Opcode enum size mismatch"); + return WriteUint8(static_cast(value)); + } + + LogicalResult WriteCount(int count); + + LogicalResult WriteTypeIndex(Type type); + + LogicalResult WriteFunctionOrdinal(FuncOp function); + LogicalResult WriteImportOrdinal(FuncOp function); + + LogicalResult WriteConstant(MemRefType memRefType, Attribute baseAttr); + LogicalResult WriteAttributeData(Attribute baseAttr); + + llvm::Optional LookupLocalOrdinal(Value *value); + LogicalResult PrepareLocal(Value *value); + LogicalResult WriteLocal(Value *value); + LogicalResult WriteLocals( + llvm::iterator_range values); + LogicalResult WriteLocals( + llvm::iterator_range values); + + LogicalResult WriteBytes(const void *data, size_t dataLength); + MutableArrayRef ReserveBytes(size_t dataLength); + LogicalResult WriteUint8(uint8_t value); + LogicalResult WriteUint16(uint16_t value); + LogicalResult WriteInt32(int32_t value); + LogicalResult WriteUint32(uint32_t value); + + LogicalResult WriteElementsAttrInt32(ElementsAttr attr); + + LogicalResult WriteShapePieces(const ShapedType &type); + LogicalResult WriteShapePieces(ElementsAttr pieces); + + LogicalResult MarkBlockOffset(Block *block); + LogicalResult WriteBlockOffset(Block *targetBlock); + LogicalResult FixupOffsets(); + + std::vector Finish(); + + private: + std::vector bytecode_; + + llvm::DenseMap localMap_; + + llvm::DenseMap blockOffsets_; + std::vector> blockOffsetFixups_; +}; + +} // namespace iree_compiler +} // namespace mlir + +#endif // THIRD_PARTY_MLIR_EDGE_IREE_COMPILER_SERIALIZATION_BYTECODE_WRITER_H_ diff --git a/compiler/Serialization/VMDeviceTableBuilder.cpp b/compiler/Serialization/VMDeviceTableBuilder.cpp new file mode 100644 index 000000000000..c2d746c828a0 --- /dev/null +++ b/compiler/Serialization/VMDeviceTableBuilder.cpp @@ -0,0 +1,46 @@ +// Copyright 2019 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "third_party/mlir_edge/iree/compiler/Serialization/VMDeviceTableBuilder.h" + +namespace mlir { +namespace iree_compiler { + +VMDeviceTableBuilder::VMDeviceTableBuilder( + ::flatbuffers::FlatBufferBuilder *fbb) + : fbb_(fbb) {} + +LogicalResult VMDeviceTableBuilder::AddDevice( + ::flatbuffers::Offset deviceDef) { + deviceDefs_.push_back(deviceDef); + return success(); +} + +LogicalResult VMDeviceTableBuilder::AddDeviceGroup( + ::flatbuffers::Offset deviceGroupDef) { + deviceGroupDefs_.push_back(deviceGroupDef); + return success(); +} + +::flatbuffers::Offset VMDeviceTableBuilder::Finish() { + auto devicesOffset = fbb_->CreateVector(deviceDefs_); + auto deviceGroupsOffset = fbb_->CreateVector(deviceGroupDefs_); + iree::DeviceTableDefBuilder dtdb(*fbb_); + dtdb.add_devices(devicesOffset); + dtdb.add_device_groups(deviceGroupsOffset); + return dtdb.Finish(); +} + +} // namespace iree_compiler +} // namespace mlir diff --git a/compiler/Serialization/VMDeviceTableBuilder.h b/compiler/Serialization/VMDeviceTableBuilder.h new file mode 100644 index 000000000000..b793982f35b2 --- /dev/null +++ b/compiler/Serialization/VMDeviceTableBuilder.h @@ -0,0 +1,45 @@ +// Copyright 2019 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#ifndef THIRD_PARTY_MLIR_EDGE_IREE_COMPILER_SERIALIZATION_VMDEVICETABLEBUILDER_H_ +#define THIRD_PARTY_MLIR_EDGE_IREE_COMPILER_SERIALIZATION_VMDEVICETABLEBUILDER_H_ + +#include "third_party/flatbuffers/include/flatbuffers/flatbuffers.h" +#include "third_party/llvm/llvm/projects/google_mlir/include/mlir/Support/LogicalResult.h" +#include "third_party/mlir_edge/iree/schemas/device_table_def_generated.h" + +namespace mlir { +namespace iree_compiler { + +class VMDeviceTableBuilder { + public: + explicit VMDeviceTableBuilder(::flatbuffers::FlatBufferBuilder *fbb); + + LogicalResult AddDevice(::flatbuffers::Offset deviceDef); + + LogicalResult AddDeviceGroup( + ::flatbuffers::Offset deviceGroupDef); + + ::flatbuffers::Offset Finish(); + + private: + ::flatbuffers::FlatBufferBuilder *fbb_; + std::vector<::flatbuffers::Offset> deviceDefs_; + std::vector<::flatbuffers::Offset> deviceGroupDefs_; +}; + +} // namespace iree_compiler +} // namespace mlir + +#endif // THIRD_PARTY_MLIR_EDGE_IREE_COMPILER_SERIALIZATION_VMDEVICETABLEBUILDER_H_ diff --git a/compiler/Serialization/VMExecutableTableBuilder.cpp b/compiler/Serialization/VMExecutableTableBuilder.cpp new file mode 100644 index 000000000000..fffe1875cb27 --- /dev/null +++ b/compiler/Serialization/VMExecutableTableBuilder.cpp @@ -0,0 +1,41 @@ +// Copyright 2019 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "third_party/mlir_edge/iree/compiler/Serialization/VMExecutableTableBuilder.h" + +namespace mlir { +namespace iree_compiler { + +VMExecutableTableBuilder::VMExecutableTableBuilder( + ::flatbuffers::FlatBufferBuilder *fbb) + : fbb_(fbb) {} + +LogicalResult VMExecutableTableBuilder::AddMultiArchExecutable( + ::flatbuffers::Offset + multiArchExecutableDef) { + multiArchExecutableDefs_.push_back(multiArchExecutableDef); + return success(); +} + +::flatbuffers::Offset +VMExecutableTableBuilder::Finish() { + auto multiArchExecutablesOffset = + fbb_->CreateVector(multiArchExecutableDefs_); + iree::ExecutableTableDefBuilder etdb(*fbb_); + etdb.add_multi_arch_executables(multiArchExecutablesOffset); + return etdb.Finish(); +} + +} // namespace iree_compiler +} // namespace mlir diff --git a/compiler/Serialization/VMExecutableTableBuilder.h b/compiler/Serialization/VMExecutableTableBuilder.h new file mode 100644 index 000000000000..4bddf8e43af0 --- /dev/null +++ b/compiler/Serialization/VMExecutableTableBuilder.h @@ -0,0 +1,44 @@ +// Copyright 2019 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#ifndef THIRD_PARTY_MLIR_EDGE_IREE_COMPILER_SERIALIZATION_VM_EXECUTABLE_TABLE_BUILDER_H_ +#define THIRD_PARTY_MLIR_EDGE_IREE_COMPILER_SERIALIZATION_VM_EXECUTABLE_TABLE_BUILDER_H_ + +#include "third_party/flatbuffers/include/flatbuffers/flatbuffers.h" +#include "third_party/llvm/llvm/projects/google_mlir/include/mlir/Support/LogicalResult.h" +#include "third_party/mlir_edge/iree/schemas/executable_table_def_generated.h" + +namespace mlir { +namespace iree_compiler { + +class VMExecutableTableBuilder { + public: + explicit VMExecutableTableBuilder(::flatbuffers::FlatBufferBuilder *fbb); + + LogicalResult AddMultiArchExecutable( + ::flatbuffers::Offset + multiArchExecutableDef); + + ::flatbuffers::Offset Finish(); + + private: + ::flatbuffers::FlatBufferBuilder *fbb_; + std::vector<::flatbuffers::Offset> + multiArchExecutableDefs_; +}; + +} // namespace iree_compiler +} // namespace mlir + +#endif // THIRD_PARTY_MLIR_EDGE_IREE_COMPILER_SERIALIZATION_VM_EXECUTABLE_TABLE_BUILDER_H_ diff --git a/compiler/Serialization/VMFunctionBuilder.cpp b/compiler/Serialization/VMFunctionBuilder.cpp new file mode 100644 index 000000000000..d4d9f118efbd --- /dev/null +++ b/compiler/Serialization/VMFunctionBuilder.cpp @@ -0,0 +1,362 @@ +// Copyright 2019 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "third_party/mlir_edge/iree/compiler/Serialization/VMFunctionBuilder.h" + +#include "third_party/flatbuffers/include/flatbuffers/flatbuffers.h" +#include "third_party/llvm/llvm/include/llvm/ADT/STLExtras.h" +#include "third_party/llvm/llvm/include/llvm/Support/raw_ostream.h" +#include "third_party/llvm/llvm/projects/google_mlir/include/mlir/IR/Attributes.h" +#include "third_party/llvm/llvm/projects/google_mlir/include/mlir/IR/Location.h" +#include "third_party/llvm/llvm/projects/google_mlir/include/mlir/IR/Module.h" +#include "third_party/mlir_edge/iree/compiler/IR/Dialect.h" +#include "third_party/mlir_edge/iree/compiler/IR/Interpreter/LLDialect.h" +#include "third_party/mlir_edge/iree/compiler/IR/Sequencer/LLDialect.h" +#include "third_party/mlir_edge/iree/compiler/IR/Types.h" +#include "third_party/mlir_edge/iree/compiler/Serialization/BytecodeTables.h" +#include "third_party/mlir_edge/iree/compiler/Utils/Macros.h" +#include "third_party/mlir_edge/iree/schemas/bytecode/bytecode_v0.h" +#include "third_party/mlir_edge/iree/schemas/type_def_generated.h" + +namespace mlir { +namespace iree_compiler { + +namespace { + +LogicalResult WriteGenericIreeOp(Block *block, Operation *op, + BytecodeWriter *writer) { + // Strip the dialect name from the op name and lookup the opcode. + // TODO(benvanik): adjust for supporting sequencer opcodes. + + auto opName = op->getName().getStringRef(); + auto dialect = op->getDialect(); + if (!dialect) { + return op->emitOpError() << "Op does not belong to a registered dialect"; + } + + auto dialectNamespace = dialect->getNamespace(); + std::unique_ptr operandInfo; + auto strippedOpName = opName.substr(opName.find('.') + 1).str(); + if (dialectNamespace == IREELLSequencerDialect::getDialectNamespace()) { + auto opcode = GetSequencerOpcodeByName(strippedOpName); + if (!opcode.hasValue()) { + return op->emitOpError() + << "No sequencer opcode found for op; is it a pseudo op?"; + } + RETURN_IF_FAILURE(writer->WriteOpcode(opcode.getValue())); + operandInfo = + std::make_unique(GetSequencerOpcodeInfo(opcode.getValue())); + } else if (dialectNamespace == + IREELLInterpreterDialect::getDialectNamespace() || + // TODO(gcmn) remove special case for IREE dialect? + dialectNamespace == IREEDialect::getDialectNamespace()) { + auto opcode = GetInterpreterOpcodeByName(strippedOpName); + if (!opcode.hasValue()) { + return op->emitOpError() + << "No interpreter opcode found for op; is it a pseudo op?"; + } + RETURN_IF_FAILURE(writer->WriteOpcode(opcode.getValue())); + operandInfo = std::make_unique( + GetInterpreterOpcodeInfo(opcode.getValue())); + } else { + return op->emitOpError() + << "Op belongs to unknown dialect " << dialectNamespace.str(); + } + // Write inputs and outputs based on the bytecode encoding. + int operandIndex = 0; + int resultIndex = 0; + for (int i = 0; i < llvm::array_lengthof(operandInfo->operands); ++i) { + auto op_encoding = operandInfo->operands[i]; + if (op_encoding == iree::OperandEncoding::kNone) break; + switch (op_encoding) { + case iree::OperandEncoding::kInputSlot: + case iree::OperandEncoding::kOutputSlot: { + auto *value = op->getOperand(operandIndex++); + RETURN_IF_FAILURE(writer->WriteLocal(value)); + break; + } + case iree::OperandEncoding::kVariadicInputSlots: + case iree::OperandEncoding::kVariadicOutputSlots: { + int count = op->getNumOperands() - operandIndex; + RETURN_IF_FAILURE(writer->WriteCount(count)); + for (; count; --count) { + auto *value = op->getOperand(operandIndex++); + RETURN_IF_FAILURE(writer->WriteLocal(value)); + } + break; + } + case iree::OperandEncoding::kResultSlot: { + auto *value = op->getResult(resultIndex++); + RETURN_IF_FAILURE(writer->WriteLocal(value)); + break; + } + case iree::OperandEncoding::kVariadicResultSlots: { + int count = op->getNumResults() - resultIndex; + RETURN_IF_FAILURE(writer->WriteCount(count)); + for (; count; --count) { + auto *value = op->getResult(resultIndex++); + RETURN_IF_FAILURE(writer->WriteLocal(value)); + } + break; + } + case iree::OperandEncoding::kConstant: + case iree::OperandEncoding::kFunctionOrdinal: + case iree::OperandEncoding::kBlockOffset: + case iree::OperandEncoding::kTypeIndex: + case iree::OperandEncoding::kIndex: + case iree::OperandEncoding::kIndexList: + case iree::OperandEncoding::kCmpIPredicate: + case iree::OperandEncoding::kCmpFPredicate: + return op->emitOpError() + << "Operand encoding " << static_cast(op_encoding) + << " not supported by generic writer for " << opName.str(); + return failure(); + default: + return op->emitOpError() + << "Operand encoding " << static_cast(op_encoding) << " (" + << static_cast(op_encoding) << ") not recognized (typo?)"; + } + } + + return success(); +} + +} // namespace + +VMFunctionBuilder::VMFunctionBuilder(FuncOp function, + VMFunctionTableBuilder *functionTable, + ::flatbuffers::FlatBufferBuilder *fbb) + : context_(function.getContext()), + function_(function), + functionTable_(functionTable), + fbb_(fbb) {} + +void VMFunctionBuilder::RegisterCustomWriter(StringRef operationName, + CustomWriterFn writerFn) { + customWriters_.insert({operationName, writerFn}); +} + +LogicalResult VMFunctionBuilder::ConvertBytecode() { + BytecodeWriter writer; + sourceMap_ = {}; + + RETURN_IF_FAILURE(BeginFunction(function_, &writer)); + for (auto &block : function_.getBlocks()) { + RETURN_IF_FAILURE(BeginBlock(&block, &writer)); + for (auto &op : block.getOperations()) { + if (failed(WriteOperation(&block, &op, &writer))) { + op.emitError() << "Unable to serialize operation"; + return failure(); + } + } + RETURN_IF_FAILURE(EndBlock(&block, block.getTerminator(), &writer)); + } + RETURN_IF_FAILURE(EndFunction(function_, &writer)); + + int localCount = writer.local_count(); + auto bodyBytes = writer.Finish(); + auto bodyOffset = fbb_->CreateVector( + reinterpret_cast(bodyBytes.data()), bodyBytes.size()); + iree::BytecodeDefBuilder bdb(*fbb_); + bdb.add_local_count(localCount); + bdb.add_contents(bodyOffset); + bytecodeDef_ = bdb.Finish(); + + return success(); +} + +::flatbuffers::Offset VMFunctionBuilder::Finish() { + using TypeDefVector = + ::flatbuffers::Vector<::flatbuffers::Offset>; + + const auto &functionType = function_.getType(); + std::vector<::flatbuffers::Offset> inputs; + for (const auto &type : functionType.getInputs()) { + auto typeOffset = SerializeType(type, fbb_); + if (typeOffset.IsNull()) return {}; + inputs.push_back(typeOffset); + } + ::flatbuffers::Offset inputsOffset; + if (!inputs.empty()) { + inputsOffset = fbb_->CreateVector(inputs); + } + + std::vector<::flatbuffers::Offset> results; + for (const auto &type : functionType.getResults()) { + auto typeOffset = SerializeType(type, fbb_); + if (typeOffset.IsNull()) return {}; + results.push_back(typeOffset); + } + ::flatbuffers::Offset resultsOffset; + if (!results.empty()) { + resultsOffset = fbb_->CreateVector(results); + } + iree::FunctionTypeDefBuilder ftb(*fbb_); + ftb.add_inputs(inputsOffset); + ftb.add_results(resultsOffset); + auto functionTypeOffset = ftb.Finish(); + + // TODO(benvanik): strip names of internal functions. + auto nameOffset = fbb_->CreateString(function_.getName().str()); + iree::FunctionDefBuilder fdb(*fbb_); + fdb.add_name(nameOffset); + fdb.add_type(functionTypeOffset); + fdb.add_bytecode(bytecodeDef_); + return fdb.Finish(); +} + +LogicalResult VMFunctionBuilder::BeginFunction(FuncOp function, + BytecodeWriter *writer) { + // Assign value slots for all arguments and results. + // Keeping them at the front will make it easier to find during debugging + // and makes spans easier to compute at runtime. + for (auto argument : function.getArguments()) { + RETURN_IF_FAILURE(writer->PrepareLocal(argument)); + } + return success(); +} + +LogicalResult VMFunctionBuilder::EndFunction(FuncOp function, + BytecodeWriter *writer) { + RETURN_IF_FAILURE(writer->FixupOffsets()); + return success(); +} + +LogicalResult VMFunctionBuilder::BeginBlock(Block *block, + BytecodeWriter *writer) { + RETURN_IF_FAILURE(writer->MarkBlockOffset(block)); + return success(); +} + +LogicalResult VMFunctionBuilder::EndBlock(Block *block, Operation *op, + BytecodeWriter *writer) { + return success(); +} + +LogicalResult VMFunctionBuilder::WriteOperation(Block *block, Operation *baseOp, + BytecodeWriter *writer) { + if (!baseOp->getLoc().isa()) { + sourceMap_.locations.push_back({writer->offset(), baseOp->getLoc()}); + } + + // Check registered writers first to allow overrides. + auto writerIt = customWriters_.find(baseOp->getName().getStringRef()); + if (writerIt != customWriters_.end()) { + return writerIt->second(baseOp, writer); + } + + // Fallback to using the generic writer. + if (baseOp->getAbstractOperation()->dialect.getNamespace().startswith( + "iree")) { + RETURN_IF_FAILURE(WriteGenericIreeOp(block, baseOp, writer)); + } else { + return baseOp->emitError() + << "Unsupported op " << baseOp->getName().getStringRef().str() + << "; incorrectly outlined or not yet implemented"; + } + return success(); +} + +::flatbuffers::Offset VMFunctionBuilder::SerializeType( + Type type, ::flatbuffers::FlatBufferBuilder *fbb) { + ::flatbuffers::Offset typeDefUnion; + iree::TypeDefUnion typeUnionType; + if (auto memRefType = type.dyn_cast()) { + auto memRefTypeOffset = SerializeMemRefType(memRefType, fbb_); + if (memRefTypeOffset.IsNull()) return {}; + typeDefUnion = memRefTypeOffset.Union(); + typeUnionType = iree::TypeDefUnion::MemRefTypeDef; + } else if (auto deviceType = type.dyn_cast()) { + typeDefUnion = iree::CreateDeviceTypeDef(*fbb).Union(); + typeUnionType = iree::TypeDefUnion::DeviceTypeDef; + } else if (auto commandBufferType = type.dyn_cast()) { + typeDefUnion = iree::CreateCommandBufferTypeDef(*fbb).Union(); + typeUnionType = iree::TypeDefUnion::CommandBufferTypeDef; + } else if (auto eventType = type.dyn_cast()) { + typeDefUnion = iree::CreateEventTypeDef(*fbb).Union(); + typeUnionType = iree::TypeDefUnion::EventTypeDef; + } else if (auto semaphoreType = type.dyn_cast()) { + typeDefUnion = iree::CreateSemaphoreTypeDef(*fbb).Union(); + typeUnionType = iree::TypeDefUnion::SemaphoreTypeDef; + } else if (auto fenceType = type.dyn_cast()) { + typeDefUnion = iree::CreateFenceTypeDef(*fbb).Union(); + typeUnionType = iree::TypeDefUnion::FenceTypeDef; + } else { + function_.emitError() << "Function " << function_.getName().str() + << " has unsupported I/O with type " << type; + return {}; + } + + iree::TypeDefBuilder tdb(*fbb); + tdb.add_type_union_type(typeUnionType); + tdb.add_type_union(typeDefUnion); + return tdb.Finish(); +} + +::flatbuffers::Offset +VMFunctionBuilder::SerializeMemRefType(const MemRefType &type, + ::flatbuffers::FlatBufferBuilder *fbb) { + auto elementTypeOffset = SerializeElementType(type.getElementType(), fbb); + if (elementTypeOffset.IsNull()) return {}; + std::vector shape; + for (int dim : type.getShape()) { + shape.push_back(dim); + } + auto shapeOffset = fbb->CreateVector(shape); + iree::MemRefTypeDefBuilder tb(*fbb); + tb.add_element_type(elementTypeOffset); + tb.add_shape(shapeOffset); + tb.add_memory_space(type.getMemorySpace()); + return tb.Finish(); +} + +::flatbuffers::Offset +VMFunctionBuilder::SerializeElementType(const Type &genericType, + ::flatbuffers::FlatBufferBuilder *fbb) { + ::flatbuffers::Offset typeDefUnion; + iree::ElementTypeDefUnion typeUnionType; + if (auto type = genericType.dyn_cast()) { + iree::FloatTypeDefBuilder tb(*fbb); + tb.add_width(type.getWidth()); + typeDefUnion = tb.Finish().Union(); + typeUnionType = iree::ElementTypeDefUnion::FloatTypeDef; + } else if (auto type = genericType.dyn_cast()) { + iree::IntegerTypeDefBuilder tb(*fbb); + tb.add_width(type.getWidth()); + typeDefUnion = tb.Finish().Union(); + typeUnionType = iree::ElementTypeDefUnion::IntegerTypeDef; + } else if (auto type = genericType.dyn_cast()) { + auto dialectOffset = fbb->CreateString(type.getDialectNamespace().c_str()); + auto typeDataOffset = fbb->CreateString(type.getTypeData().data()); + iree::UnknownTypeDefBuilder tb(*fbb); + tb.add_dialect(dialectOffset); + tb.add_type_data(typeDataOffset); + typeDefUnion = tb.Finish().Union(); + typeUnionType = iree::ElementTypeDefUnion::UnknownTypeDef; + } else { + function_.emitError() + << "Unimplemented type encoding: " << genericType + << "; ensure IREE lowering passes are converting types to the IREE " + "set"; + return {}; + } + + iree::ElementTypeDefBuilder tdb(*fbb); + tdb.add_type_union_type(typeUnionType); + tdb.add_type_union(typeDefUnion); + return tdb.Finish(); +} + +} // namespace iree_compiler +} // namespace mlir diff --git a/compiler/Serialization/VMFunctionBuilder.h b/compiler/Serialization/VMFunctionBuilder.h new file mode 100644 index 000000000000..26c3535dde9b --- /dev/null +++ b/compiler/Serialization/VMFunctionBuilder.h @@ -0,0 +1,77 @@ +// Copyright 2019 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#ifndef THIRD_PARTY_MLIR_EDGE_IREE_COMPILER_SERIALIZATION_VM_FUNCTION_BUILDER_H_ +#define THIRD_PARTY_MLIR_EDGE_IREE_COMPILER_SERIALIZATION_VM_FUNCTION_BUILDER_H_ + +#include "third_party/llvm/llvm/projects/google_mlir/include/mlir/Dialect/StandardOps/Ops.h" +#include "third_party/llvm/llvm/projects/google_mlir/include/mlir/IR/Function.h" +#include "third_party/llvm/llvm/projects/google_mlir/include/mlir/IR/MLIRContext.h" +#include "third_party/llvm/llvm/projects/google_mlir/include/mlir/IR/StandardTypes.h" +#include "third_party/mlir_edge/iree/compiler/Serialization/BytecodeWriter.h" +#include "third_party/mlir_edge/iree/compiler/Serialization/VMFunctionTableBuilder.h" +#include "third_party/mlir_edge/iree/compiler/Serialization/VMSourceMapBuilder.h" +#include "third_party/mlir_edge/iree/schemas/bytecode_def_generated.h" +#include "third_party/mlir_edge/iree/schemas/function_def_generated.h" + +namespace mlir { +namespace iree_compiler { + +class VMFunctionBuilder { + public: + using CustomWriterFn = + std::function; + + VMFunctionBuilder(FuncOp function, VMFunctionTableBuilder *functionTable, + ::flatbuffers::FlatBufferBuilder *fbb); + ~VMFunctionBuilder() = default; + + void RegisterCustomWriter(StringRef operationName, CustomWriterFn writerFn); + + const VMFunctionSourceMap &source_map() const { return sourceMap_; } + + LogicalResult ConvertBytecode(); + + ::flatbuffers::Offset Finish(); + + ::flatbuffers::Offset SerializeType( + Type type, ::flatbuffers::FlatBufferBuilder *fbb); + ::flatbuffers::Offset SerializeMemRefType( + const MemRefType &genericType, ::flatbuffers::FlatBufferBuilder *fbb); + ::flatbuffers::Offset SerializeElementType( + const Type &genericType, ::flatbuffers::FlatBufferBuilder *fbb); + + private: + LogicalResult BeginFunction(FuncOp function, BytecodeWriter *writer); + LogicalResult EndFunction(FuncOp function, BytecodeWriter *writer); + LogicalResult BeginBlock(Block *block, BytecodeWriter *writer); + LogicalResult EndBlock(Block *block, Operation *op, BytecodeWriter *writer); + + LogicalResult WriteOperation(Block *block, Operation *baseOp, + BytecodeWriter *writer); + + llvm::StringMap customWriters_; + + MLIRContext *context_; + FuncOp function_; + VMFunctionTableBuilder *functionTable_; + ::flatbuffers::FlatBufferBuilder *fbb_; + ::flatbuffers::Offset bytecodeDef_; + VMFunctionSourceMap sourceMap_; +}; + +} // namespace iree_compiler +} // namespace mlir + +#endif // THIRD_PARTY_MLIR_EDGE_IREE_COMPILER_SERIALIZATION_VM_FUNCTION_BUILDER_H_ diff --git a/compiler/Serialization/VMFunctionTableBuilder.cpp b/compiler/Serialization/VMFunctionTableBuilder.cpp new file mode 100644 index 000000000000..97e42438cc92 --- /dev/null +++ b/compiler/Serialization/VMFunctionTableBuilder.cpp @@ -0,0 +1,87 @@ +// Copyright 2019 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "third_party/mlir_edge/iree/compiler/Serialization/VMFunctionTableBuilder.h" + +#include "third_party/llvm/llvm/include/llvm/Support/raw_ostream.h" +#include "third_party/mlir_edge/iree/compiler/Serialization/VMSourceMapBuilder.h" + +namespace mlir { +namespace iree_compiler { + +VMFunctionTableBuilder::VMFunctionTableBuilder( + ::flatbuffers::FlatBufferBuilder *fbb) + : fbb_(fbb) {} + +bool VMFunctionTableBuilder::IsFunctionDeclared(FuncOp funcOp) { + return functionSet_.count(funcOp.getName()) != 0; +} + +LogicalResult VMFunctionTableBuilder::DeclareFunction(FuncOp funcOp, + LinkageType linkageType) { + if (functionSet_.count(funcOp.getName())) { + return funcOp.emitError() << "Function has already been declared/defined"; + } + auto functionOrdinal = funcOp.getAttrOfType("iree.ordinal"); + if (!functionOrdinal) { + return funcOp.emitError() << "Ordinal not assigned to function"; + } + int ordinal = functionOrdinal.getInt(); + functionDefs_.resize( + std::max(functionDefs_.size(), static_cast(ordinal) + 1u)); + functionSourceMaps_.resize( + std::max(functionDefs_.size(), static_cast(ordinal) + 1u)); + functionSet_.insert({funcOp.getName()}); + switch (linkageType) { + case LinkageType::kInternal: + break; + case LinkageType::kImport: + importIndices_.push_back(ordinal); + break; + case LinkageType::kExport: + exportIndices_.push_back(ordinal); + break; + } + return success(); +} + +LogicalResult VMFunctionTableBuilder::DefineFunction( + FuncOp funcOp, ::flatbuffers::Offset functionDef, + VMFunctionSourceMap functionSourceMap) { + auto functionOrdinal = funcOp.getAttrOfType("iree.ordinal"); + if (!functionOrdinal) { + return funcOp.emitError() << "Ordinal not assigned to function"; + } + int ordinal = functionOrdinal.getInt(); + if (!functionDefs_[ordinal].IsNull()) { + return funcOp.emitOpError() << "Function has already been defined"; + } + functionDefs_[ordinal] = functionDef; + functionSourceMaps_[ordinal] = std::move(functionSourceMap); + return success(); +} + +::flatbuffers::Offset VMFunctionTableBuilder::Finish() { + auto functionsOffset = fbb_->CreateVector(functionDefs_); + auto importsOffset = fbb_->CreateVector(importIndices_); + auto exportsOffset = fbb_->CreateVector(exportIndices_); + iree::FunctionTableDefBuilder ftdb(*fbb_); + ftdb.add_functions(functionsOffset); + ftdb.add_imports(importsOffset); + ftdb.add_exports(exportsOffset); + return ftdb.Finish(); +} + +} // namespace iree_compiler +} // namespace mlir diff --git a/compiler/Serialization/VMFunctionTableBuilder.h b/compiler/Serialization/VMFunctionTableBuilder.h new file mode 100644 index 000000000000..101e62780c9d --- /dev/null +++ b/compiler/Serialization/VMFunctionTableBuilder.h @@ -0,0 +1,75 @@ +// Copyright 2019 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#ifndef THIRD_PARTY_MLIR_EDGE_IREE_COMPILER_SERIALIZATION_VM_FUNCTION_TABLE_BUILDER_H_ +#define THIRD_PARTY_MLIR_EDGE_IREE_COMPILER_SERIALIZATION_VM_FUNCTION_TABLE_BUILDER_H_ + +#include +#include + +#include "third_party/flatbuffers/include/flatbuffers/flatbuffers.h" +#include "third_party/llvm/llvm/include/llvm/ADT/StringSet.h" +#include "third_party/llvm/llvm/projects/google_mlir/include/mlir/IR/Function.h" +#include "third_party/llvm/llvm/projects/google_mlir/include/mlir/IR/MLIRContext.h" +#include "third_party/llvm/llvm/projects/google_mlir/include/mlir/IR/OperationSupport.h" +#include "third_party/mlir_edge/iree/compiler/Serialization/VMSourceMapBuilder.h" +#include "third_party/mlir_edge/iree/schemas/function_def_generated.h" +#include "third_party/mlir_edge/iree/schemas/function_table_def_generated.h" + +namespace mlir { +namespace iree_compiler { + +enum class LinkageType { + kInternal, + kImport, + kExport, +}; + +class VMFunctionTableBuilder { + public: + explicit VMFunctionTableBuilder(::flatbuffers::FlatBufferBuilder *fbb); + + int max_function_ordinal() const { return functionDefs_.size(); } + + ArrayRef function_source_maps() { + return llvm::makeArrayRef(functionSourceMaps_); + } + + // Returns true if |funcOp| has already been declared in the table. + bool IsFunctionDeclared(FuncOp funcOp); + + // Declares |funcOp| with the given |linkageType|. + // Fails if the function has already been declared or defined. + LogicalResult DeclareFunction(FuncOp funcOp, LinkageType linkageType); + + // Defines |funcOp| using the given |functionDef|. + LogicalResult DefineFunction( + FuncOp funcOp, ::flatbuffers::Offset functionDef, + VMFunctionSourceMap functionSourceMap); + + ::flatbuffers::Offset Finish(); + + private: + ::flatbuffers::FlatBufferBuilder *fbb_; + llvm::StringSet<> functionSet_; + std::vector<::flatbuffers::Offset> functionDefs_; + std::vector functionSourceMaps_; + std::vector importIndices_; + std::vector exportIndices_; +}; + +} // namespace iree_compiler +} // namespace mlir + +#endif // THIRD_PARTY_MLIR_EDGE_IREE_COMPILER_SERIALIZATION_VM_FUNCTION_TABLE_BUILDER_H_ diff --git a/compiler/Serialization/VMModuleBuilder.cpp b/compiler/Serialization/VMModuleBuilder.cpp new file mode 100644 index 000000000000..5df1efe5c952 --- /dev/null +++ b/compiler/Serialization/VMModuleBuilder.cpp @@ -0,0 +1,70 @@ +// Copyright 2019 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "third_party/mlir_edge/iree/compiler/Serialization/VMModuleBuilder.h" + +#include "third_party/mlir_edge/iree/schemas/executable_table_def_generated.h" + +namespace mlir { +namespace iree_compiler { + +VMModuleBuilder::VMModuleBuilder(::flatbuffers::FlatBufferBuilder *fbb) + : fbb_(fbb), + deviceTable_(fbb), + functionTable_(fbb), + executableTable_(fbb), + sourceMap_(fbb) {} + +::flatbuffers::Offset VMModuleBuilder::Finish() { + auto nameOffset = fbb_->CreateString("module"); + auto deviceTableOffset = deviceTable_.Finish(); + if (deviceTableOffset.IsNull()) return {}; + auto functionTableOffset = functionTable_.Finish(); + if (functionTableOffset.IsNull()) return {}; + auto executableTableOffset = executableTable_.Finish(); + if (executableTableOffset.IsNull()) return {}; + + for (int function_ordinal = 0; + function_ordinal < functionTable_.function_source_maps().size(); + ++function_ordinal) { + if (failed(sourceMap_.AddFunction( + function_ordinal, + functionTable_.function_source_maps()[function_ordinal]))) { + return {}; + } + } + auto sourceMapOffset = + sourceMap_.Finish(functionTable_.max_function_ordinal()); + if (sourceMapOffset.IsNull()) return {}; + + iree::ModuleDefBuilder mdb(*fbb_); + mdb.add_name(nameOffset); + mdb.add_device_table(deviceTableOffset); + mdb.add_function_table(functionTableOffset); + mdb.add_executable_table(executableTableOffset); + mdb.add_source_map(sourceMapOffset); + return mdb.Finish(); +} + +std::vector VMModuleBuilder::Serialize( + ::flatbuffers::Offset module_def) { + FinishModuleDefBuffer(*fbb_, module_def); + std::vector bytes; + bytes.resize(fbb_->GetSize()); + std::memcpy(bytes.data(), fbb_->GetBufferPointer(), bytes.size()); + return bytes; +} + +} // namespace iree_compiler +} // namespace mlir diff --git a/compiler/Serialization/VMModuleBuilder.h b/compiler/Serialization/VMModuleBuilder.h new file mode 100644 index 000000000000..e3b54cf62879 --- /dev/null +++ b/compiler/Serialization/VMModuleBuilder.h @@ -0,0 +1,57 @@ +// Copyright 2019 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#ifndef THIRD_PARTY_MLIR_EDGE_IREE_COMPILER_SERIALIZATION_VM_MODULE_BUILDER_H_ +#define THIRD_PARTY_MLIR_EDGE_IREE_COMPILER_SERIALIZATION_VM_MODULE_BUILDER_H_ + +#include + +#include "third_party/flatbuffers/include/flatbuffers/flatbuffers.h" +#include "third_party/mlir_edge/iree/compiler/Serialization/VMDeviceTableBuilder.h" +#include "third_party/mlir_edge/iree/compiler/Serialization/VMExecutableTableBuilder.h" +#include "third_party/mlir_edge/iree/compiler/Serialization/VMFunctionTableBuilder.h" +#include "third_party/mlir_edge/iree/compiler/Serialization/VMSourceMapBuilder.h" +#include "third_party/mlir_edge/iree/schemas/module_def_generated.h" + +namespace mlir { +namespace iree_compiler { + +class VMModuleBuilder { + public: + explicit VMModuleBuilder(::flatbuffers::FlatBufferBuilder *fbb); + + ::flatbuffers::FlatBufferBuilder *fbb() const { return fbb_; } + VMDeviceTableBuilder *device_table() { return &deviceTable_; } + VMFunctionTableBuilder *function_table() { return &functionTable_; } + VMExecutableTableBuilder *executable_table() { return &executableTable_; } + VMSourceMapBuilder *source_map() { return &sourceMap_; } + + ::flatbuffers::Offset Finish(); + + std::vector Serialize( + ::flatbuffers::Offset module_def); + + private: + ::flatbuffers::FlatBufferBuilder *fbb_; + + VMDeviceTableBuilder deviceTable_; + VMFunctionTableBuilder functionTable_; + VMExecutableTableBuilder executableTable_; + VMSourceMapBuilder sourceMap_; +}; + +} // namespace iree_compiler +} // namespace mlir + +#endif // THIRD_PARTY_MLIR_EDGE_IREE_COMPILER_SERIALIZATION_VM_MODULE_BUILDER_H_ diff --git a/compiler/Serialization/VMSourceMapBuilder.cpp b/compiler/Serialization/VMSourceMapBuilder.cpp new file mode 100644 index 000000000000..2b89a5091c79 --- /dev/null +++ b/compiler/Serialization/VMSourceMapBuilder.cpp @@ -0,0 +1,164 @@ +// Copyright 2019 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "third_party/mlir_edge/iree/compiler/Serialization/VMSourceMapBuilder.h" + +#include "third_party/flatbuffers/include/flatbuffers/flatbuffers.h" +#include "third_party/llvm/llvm/include/llvm/Support/raw_ostream.h" +#include "third_party/llvm/llvm/projects/google_mlir/include/mlir/IR/Identifier.h" +#include "third_party/llvm/llvm/projects/google_mlir/include/mlir/IR/Location.h" +#include "third_party/mlir_edge/iree/schemas/source_map_def_generated.h" + +namespace mlir { +namespace iree_compiler { + +VMSourceMapBuilder::VMSourceMapBuilder(::flatbuffers::FlatBufferBuilder *fbb) + : fbb_(fbb) {} + +int VMSourceMapBuilder::GetUniqueString(std::string value) { + auto it = stringTableMap_.find(value); + if (it != stringTableMap_.end()) { + return it->second; + } + int stringIndex = stringTable_.size(); + stringTableMap_.insert({value, stringIndex}); + stringTable_.push_back(std::move(value)); + return stringIndex; +} + +LogicalResult VMSourceMapBuilder::AddFunction( + int functionOrdinal, VMFunctionSourceMap functionSourceMap) { + if (functionMaps_.size() <= functionOrdinal) { + functionMaps_.resize(functionOrdinal + 1); + } + functionMaps_[functionOrdinal] = std::move(functionSourceMap); + return success(); +} + +::flatbuffers::Offset VMSourceMapBuilder::Finish( + int maxFunctionOrdinal) { + // NOTE: we always ensure the source map table is the same size as the + // function table so that lookups at runtime can be validated once at load + // time (ensuring the tables match up) instead of on each lookup. + if (maxFunctionOrdinal < functionMaps_.size()) { + llvm::errs() << "Max function ordinal defined as " << maxFunctionOrdinal + << " but there are " << functionMaps_.size() + << " function source maps present"; + return {}; + } + functionMaps_.resize(maxFunctionOrdinal); + + std::vector<::flatbuffers::Offset> functionDefs; + functionDefs.resize(maxFunctionOrdinal); + for (int i = 0; i < functionMaps_.size(); ++i) { + const auto &functionMap = functionMaps_[i]; + functionDefs[i] = SerializeVMFunctionSourceMap(functionMap); + if (functionDefs[i].IsNull()) return {}; + } + + auto functionTableOffset = fbb_->CreateVector(functionDefs); + auto stringTableOffset = fbb_->CreateVectorOfStrings(stringTable_); + iree::SourceMapDefBuilder smdb(*fbb_); + smdb.add_function_table(functionTableOffset); + smdb.add_string_table(stringTableOffset); + return smdb.Finish(); +} + +::flatbuffers::Offset +VMSourceMapBuilder::SerializeVMFunctionSourceMap( + const VMFunctionSourceMap &functionMap) { + if (functionMap.locations.empty()) { + // Empty table. This ensures that we still have a non-null value in the + // function table but doesn't waste much space. + iree::FunctionSourceMapDefBuilder fsmdb(*fbb_); + return fsmdb.Finish(); + } + + LocationOffsetTable locationOffsetTable; + std::vector bytecodeMap; + for (const auto &offset_location : functionMap.locations) { + int locationIndex = + SerializeLocation(offset_location.second, &locationOffsetTable); + bytecodeMap.push_back({offset_location.first, locationIndex}); + } + auto locationTableOffset = + fbb_->CreateVector(locationOffsetTable.locationDefs); + auto bytecodeMapOffset = fbb_->CreateVectorOfStructs(bytecodeMap); + + iree::FunctionSourceMapDefBuilder fsmdb(*fbb_); + fsmdb.add_location_table(locationTableOffset); + fsmdb.add_bytecode_map(bytecodeMapOffset); + return fsmdb.Finish(); +} + +int VMSourceMapBuilder::SerializeLocation( + const Location &location, LocationOffsetTable *locationOffsetTable) { + auto existingIt = locationOffsetTable->locationMap.find(location); + if (existingIt != locationOffsetTable->locationMap.end()) { + return existingIt->getSecond(); + } + + iree::LocationDefUnion locationUnionType; + ::flatbuffers::Offset locationUnionOffset; + if (auto fileLoc = location.dyn_cast()) { + locationUnionType = iree::LocationDefUnion::FileLocationDef; + int filenameIndex = GetUniqueString(fileLoc.getFilename().str()); + iree::FileLocationDefBuilder lb(*fbb_); + lb.add_filename(filenameIndex); + lb.add_line(fileLoc.getLine()); + lb.add_column(fileLoc.getColumn()); + locationUnionOffset = lb.Finish().Union(); + } else if (auto nameLoc = location.dyn_cast()) { + locationUnionType = iree::LocationDefUnion::NameLocationDef; + int nameIndex = GetUniqueString(nameLoc.getName().str()); + iree::NameLocationDefBuilder lb(*fbb_); + lb.add_name(nameIndex); + locationUnionOffset = lb.Finish().Union(); + } else if (auto callSiteLoc = location.dyn_cast()) { + locationUnionType = iree::LocationDefUnion::CallSiteLocationDef; + int calleeIndex = + SerializeLocation(callSiteLoc.getCallee(), locationOffsetTable); + int callerIndex = + SerializeLocation(callSiteLoc.getCaller(), locationOffsetTable); + iree::CallSiteLocationDefBuilder lb(*fbb_); + lb.add_callee_location(calleeIndex); + lb.add_caller_location(callerIndex); + locationUnionOffset = lb.Finish().Union(); + } else if (auto fusedLoc = location.dyn_cast()) { + locationUnionType = iree::LocationDefUnion::FusedLocationDef; + std::vector locationIndices; + locationIndices.reserve(fusedLoc.getLocations().size()); + for (const auto &child_loc : fusedLoc.getLocations()) { + int child_index = SerializeLocation(child_loc, locationOffsetTable); + locationIndices.push_back(child_index); + } + auto locationIndicesOffset = fbb_->CreateVector(locationIndices); + iree::FusedLocationDefBuilder lb(*fbb_); + lb.add_locations(locationIndicesOffset); + locationUnionOffset = lb.Finish().Union(); + } else { + llvm_unreachable("Unimplemented location kind"); + } + + iree::LocationDefBuilder ldb(*fbb_); + ldb.add_location_union_type(locationUnionType); + ldb.add_location_union(locationUnionOffset); + int locationIndex = locationOffsetTable->locationDefs.size(); + locationOffsetTable->locationDefs.push_back(ldb.Finish()); + locationOffsetTable->locationMap.insert({location, locationIndex}); + return locationIndex; +} + +} // namespace iree_compiler +} // namespace mlir diff --git a/compiler/Serialization/VMSourceMapBuilder.h b/compiler/Serialization/VMSourceMapBuilder.h new file mode 100644 index 000000000000..c587605599fb --- /dev/null +++ b/compiler/Serialization/VMSourceMapBuilder.h @@ -0,0 +1,63 @@ +// Copyright 2019 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#ifndef THIRD_PARTY_MLIR_EDGE_IREE_COMPILER_SERIALIZATION_VM_SOURCE_MAP_BUILDER_H_ +#define THIRD_PARTY_MLIR_EDGE_IREE_COMPILER_SERIALIZATION_VM_SOURCE_MAP_BUILDER_H_ + +#include + +#include "third_party/flatbuffers/include/flatbuffers/flatbuffers.h" +#include "third_party/llvm/llvm/projects/google_mlir/include/mlir/IR/Location.h" +#include "third_party/llvm/llvm/projects/google_mlir/include/mlir/IR/MLIRContext.h" +#include "third_party/mlir_edge/iree/schemas/source_map_def_generated.h" + +namespace mlir { +namespace iree_compiler { + +struct VMFunctionSourceMap { + std::vector> locations; +}; + +class VMSourceMapBuilder { + public: + explicit VMSourceMapBuilder(::flatbuffers::FlatBufferBuilder *fbb); + + LogicalResult AddFunction(int functionOrdinal, + VMFunctionSourceMap functionSourceMap); + + ::flatbuffers::Offset Finish(int maxFunctionOrdinal); + + private: + struct LocationOffsetTable { + std::vector<::flatbuffers::Offset> locationDefs; + llvm::DenseMap locationMap; + }; + + int GetUniqueString(std::string value); + + ::flatbuffers::Offset + SerializeVMFunctionSourceMap(const VMFunctionSourceMap &functionMap); + int SerializeLocation(const Location &location, + LocationOffsetTable *locationOffsetTable); + + ::flatbuffers::FlatBufferBuilder *fbb_; + std::vector stringTable_; + llvm::StringMap stringTableMap_; + std::vector functionMaps_; +}; + +} // namespace iree_compiler +} // namespace mlir + +#endif // THIRD_PARTY_MLIR_EDGE_IREE_COMPILER_SERIALIZATION_VM_SOURCE_MAP_BUILDER_H_ diff --git a/compiler/Transforms/AggressiveOpElimination.cpp b/compiler/Transforms/AggressiveOpElimination.cpp new file mode 100644 index 000000000000..b359e879f60b --- /dev/null +++ b/compiler/Transforms/AggressiveOpElimination.cpp @@ -0,0 +1,130 @@ +// Copyright 2019 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include +#include + +#include "third_party/llvm/llvm/projects/google_mlir/include/mlir/Analysis/Dominance.h" +#include "third_party/llvm/llvm/projects/google_mlir/include/mlir/Dialect/StandardOps/Ops.h" +#include "third_party/llvm/llvm/projects/google_mlir/include/mlir/IR/Block.h" +#include "third_party/llvm/llvm/projects/google_mlir/include/mlir/Pass/Pass.h" +#include "third_party/llvm/llvm/projects/google_mlir/include/mlir/Pass/PassRegistry.h" +#include "third_party/llvm/llvm/projects/google_mlir/include/mlir/Transforms/Utils.h" +#include "third_party/mlir_edge/iree/compiler/IR/StructureOps.h" + +namespace mlir { +namespace iree_compiler { + +namespace { + +bool IsDead(Operation *op, llvm::DenseSet *deadOpsList) { + for (auto *result : op->getResults()) { + for (auto *user : result->getUsers()) { + if (deadOpsList->count(user) == 0) { + return false; + } + } + } + return true; +} + +void dceBlock(Block *block, llvm::DenseSet *deadOpsList) { + // Iterate in reverse so that we can do this in one pass per block. + std::vector opsToEraseList; + for (auto &op : llvm::reverse(block->getOperations())) { + // Ignore terminators. + if (op.isKnownTerminator()) { + continue; + } + + // Ignore ops that have side effects as we can't prove they are dead + // (yet?). + // TODO(b/135053584): IREE awareness for out params. + bool shouldBeSideEffectFree = isa(op) || isa(op); + if (!shouldBeSideEffectFree && !op.hasNoSideEffect()) { + continue; + } + + // Ignore stores. + // TODO(benvanik): IREE awareness of last use/liveness. + if (isa(op)) { + continue; + } + + if (IsDead(&op, deadOpsList)) { + deadOpsList->insert(&op); + opsToEraseList.push_back(&op); + } + } + + for (auto *op : opsToEraseList) { + op->erase(); + } +} + +struct CFGStackNode { + explicit CFGStackNode(DominanceInfoNode *node) + : node(node), childIterator(node->begin()) {} + DominanceInfoNode *node; + DominanceInfoNode::iterator childIterator; + bool processed = false; +}; + +void dceRegion(DominanceInfo &domInfo, Region ®ion) { + if (region.empty()) { + return; + } + + llvm::DenseSet deadOpsList; + + // Depth-first post-order traversal so that we go backwards. + std::deque> stack; + stack.emplace_back( + std::make_unique(domInfo.getRootNode(®ion))); + while (!stack.empty()) { + auto ¤tNode = stack.back(); + if (currentNode->childIterator != currentNode->node->end()) { + auto *childNode = *(currentNode->childIterator++); + stack.emplace_back(std::make_unique(childNode)); + } else { + if (!currentNode->processed) { + currentNode->processed = true; + dceBlock(currentNode->node->getBlock(), &deadOpsList); + } + stack.pop_back(); + } + } +} + +} // namespace + +class AggressiveOpEliminationPass + : public FunctionPass { + public: + void runOnFunction() override { + dceRegion(getAnalysis(), getFunction().getBody()); + markAnalysesPreserved(); + } +}; + +std::unique_ptr> createAggressiveOpEliminationPass() { + return std::make_unique(); +} + +static PassRegistration pass( + "iree-aggressive-op-elimination", + "Eliminate ops that have no side-effects"); + +} // namespace iree_compiler +} // namespace mlir diff --git a/compiler/Transforms/AssignFunctionOrdinals.cpp b/compiler/Transforms/AssignFunctionOrdinals.cpp new file mode 100644 index 000000000000..c8fbe91583b2 --- /dev/null +++ b/compiler/Transforms/AssignFunctionOrdinals.cpp @@ -0,0 +1,47 @@ +// Copyright 2019 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "third_party/llvm/llvm/projects/google_mlir/include/mlir/IR/Attributes.h" +#include "third_party/llvm/llvm/projects/google_mlir/include/mlir/IR/MLIRContext.h" +#include "third_party/llvm/llvm/projects/google_mlir/include/mlir/Pass/Pass.h" +#include "third_party/llvm/llvm/projects/google_mlir/include/mlir/Pass/PassRegistry.h" +#include "third_party/llvm/llvm/projects/google_mlir/include/mlir/Support/LLVM.h" +#include "third_party/llvm/llvm/projects/google_mlir/include/mlir/Support/LogicalResult.h" +#include "third_party/llvm/llvm/projects/google_mlir/include/mlir/Transforms/Utils.h" + +namespace mlir { +namespace iree_compiler { + +class AssignFunctionOrdinalsPass + : public ModulePass { + public: + void runOnModule() override { + Builder builder(getModule()); + int nextFunctionOrdinal = 0; + for (auto funcOp : getModule().getOps()) { + funcOp.setAttr("iree.ordinal", + builder.getI32IntegerAttr(nextFunctionOrdinal++)); + } + } +}; + +std::unique_ptr> createAssignFunctionOrdinalsPass() { + return std::make_unique(); +} + +static PassRegistration pass( + "iree-assign-function-ordinals", "Assigns all functions ordinals"); + +} // namespace iree_compiler +} // namespace mlir diff --git a/compiler/Transforms/ConversionUtils.h b/compiler/Transforms/ConversionUtils.h new file mode 100644 index 000000000000..32e8ed953dea --- /dev/null +++ b/compiler/Transforms/ConversionUtils.h @@ -0,0 +1,106 @@ +// Copyright 2019 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#ifndef THIRD_PARTY_MLIR_EDGE_IREE_COMPILER_TRANSFORMS_CONVERSIONUTILS_H_ +#define THIRD_PARTY_MLIR_EDGE_IREE_COMPILER_TRANSFORMS_CONVERSIONUTILS_H_ + +#include "third_party/llvm/llvm/projects/google_mlir/include/mlir/Transforms/DialectConversion.h" +#include "third_party/mlir_edge/iree/compiler/Utils/MemRefUtils.h" + +namespace mlir { +namespace iree_compiler { + +template +struct UnaryOpLowering : public ConversionPattern { + explicit UnaryOpLowering(MLIRContext *context) + : ConversionPattern(SrcOp::getOperationName(), 1, context) {} + + PatternMatchResult matchAndRewrite( + Operation *op, ArrayRef operands, + ConversionPatternRewriter &rewriter) const override { + auto *value = loadAccessValue(op->getLoc(), operands[0], rewriter); + value = wrapAsMemRef(value, op, rewriter); + + auto dstType = getMemRefType(op->getResult(0), rewriter); + auto dstOp = rewriter.create(op->getLoc(), dstType, value); + auto result = dstOp.getResult(); + result = wrapAsTensor(result, op, rewriter); + + rewriter.replaceOp( + op, {loadResultValue(op->getLoc(), op->getResult(0)->getType(), result, + rewriter)}); + return matchSuccess(); + } +}; + +template +struct BinaryOpLowering : public ConversionPattern { + explicit BinaryOpLowering(MLIRContext *context) + : ConversionPattern(SrcOp::getOperationName(), 1, context) {} + + PatternMatchResult matchAndRewrite( + Operation *op, ArrayRef operands, + ConversionPatternRewriter &rewriter) const override { + auto *lhsValue = loadAccessValue(op->getLoc(), operands[0], rewriter); + auto *rhsValue = loadAccessValue(op->getLoc(), operands[1], rewriter); + auto dstType = getMemRefType(op->getResult(0), rewriter); + + lhsValue = wrapAsMemRef(lhsValue, op, rewriter); + rhsValue = wrapAsMemRef(rhsValue, op, rewriter); + + auto midOp = + rewriter.create(op->getLoc(), dstType, lhsValue, rhsValue); + auto result = midOp.getResult(); + result = wrapAsTensor(result, op, rewriter); + + rewriter.replaceOp( + op, {loadResultValue(op->getLoc(), op->getResult(0)->getType(), result, + rewriter)}); + return matchSuccess(); + } +}; + +template +struct TernaryOpLowering : public ConversionPattern { + explicit TernaryOpLowering(MLIRContext *context) + : ConversionPattern(SrcOp::getOperationName(), 1, context) {} + + PatternMatchResult matchAndRewrite( + Operation *op, ArrayRef operands, + ConversionPatternRewriter &rewriter) const override { + auto *aValue = loadAccessValue(op->getLoc(), operands[0], rewriter); + auto *bValue = loadAccessValue(op->getLoc(), operands[1], rewriter); + auto *cValue = loadAccessValue(op->getLoc(), operands[2], rewriter); + + aValue = wrapAsMemRef(aValue, op, rewriter); + bValue = wrapAsMemRef(bValue, op, rewriter); + cValue = wrapAsMemRef(cValue, op, rewriter); + + auto dstType = getMemRefType(op->getResult(0), rewriter); + auto dstOp = + rewriter.create(op->getLoc(), dstType, aValue, bValue, cValue); + auto result = dstOp.getResult(); + result = wrapAsTensor(result, op, rewriter); + + rewriter.replaceOp( + op, {loadResultValue(op->getLoc(), op->getResult(0)->getType(), result, + rewriter)}); + return matchSuccess(); + } +}; + +} // namespace iree_compiler +} // namespace mlir + +#endif // THIRD_PARTY_MLIR_EDGE_IREE_COMPILER_TRANSFORMS_CONVERSIONUTILS_H_ diff --git a/compiler/Transforms/ConvertFromTupleCallingConvention.cpp b/compiler/Transforms/ConvertFromTupleCallingConvention.cpp new file mode 100644 index 000000000000..985a8e7daa3b --- /dev/null +++ b/compiler/Transforms/ConvertFromTupleCallingConvention.cpp @@ -0,0 +1,328 @@ +// Copyright 2019 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "third_party/llvm/llvm/include/llvm/ADT/ArrayRef.h" +#include "third_party/llvm/llvm/include/llvm/ADT/SmallVector.h" +#include "third_party/llvm/llvm/include/llvm/ADT/iterator_range.h" +#include "third_party/llvm/llvm/projects/google_mlir/include/mlir/Dialect/StandardOps/Ops.h" +#include "third_party/llvm/llvm/projects/google_mlir/include/mlir/IR/BlockAndValueMapping.h" +#include "third_party/llvm/llvm/projects/google_mlir/include/mlir/IR/Builders.h" +#include "third_party/llvm/llvm/projects/google_mlir/include/mlir/IR/StandardTypes.h" +#include "third_party/llvm/llvm/projects/google_mlir/include/mlir/Pass/Pass.h" +#include "third_party/llvm/llvm/projects/google_mlir/include/mlir/Pass/PassRegistry.h" +#include "third_party/llvm/llvm/projects/google_mlir/include/mlir/Transforms/Utils.h" +#include "third_party/tensorflow/compiler/mlir/xla/ir/hlo_ops.h" + +namespace mlir { +namespace iree_compiler { + +namespace { + +// Given a set of types, unpack to a list of a types, removing all tuples. +void untupleTypes(llvm::ArrayRef types, + llvm::SmallVectorImpl *newTypes) { + for (auto &type : types) { + if (type.isa()) { + untupleTypes(type.dyn_cast().getTypes(), newTypes); + } else { + newTypes->push_back(type); + } + } +} + +Value *processTuple(Type type, Location loc, Block *block, OpBuilder *builder) { + if (!type.isa()) { + return block->addArgument(type); + } + + auto tupleType = type.dyn_cast(); + llvm::SmallVector values; + values.reserve(tupleType.size()); + for (auto subtype : tupleType.getTypes()) { + values.push_back(processTuple(subtype, loc, block, builder)); + } + + return builder->create(loc, tupleType, values); +} + +void copyOperationAttrs(Operation *oldOp, Operation *newOp) { + for (const auto &oldAttr : oldOp->getAttrs()) { + newOp->setAttr(oldAttr.first, oldAttr.second); + } +} + +bool recursiveUntuple(Value *value, Location loc, OpBuilder *builder, + BlockAndValueMapping *mapping, + llvm::SmallVectorImpl *newValues) { + Type type = value->getType(); + // We can return the value as is. + if (!type.isa()) { + newValues->push_back(value); + return false; + } + + TupleType tupleType = type.dyn_cast(); + for (int i = 0; i < tupleType.size(); i++) { + auto subType = tupleType.getType(i); + + auto elementOp = builder->create( + loc, subType, value, builder->getI32IntegerAttr(i)); + recursiveUntuple(elementOp.getResult(), loc, builder, mapping, newValues); + } + + return false; +} + +Value *recursiveRetuple( + Type oldType, llvm::iterator_range *values, + OpBuilder *builder, Location loc) { + if (!oldType.isa()) { + Value *returnValue = *values->begin(); + *values = llvm::iterator_range( + values->begin() + 1, values->end()); + return returnValue; + } + + TupleType tupleType = oldType.dyn_cast(); + llvm::SmallVector subValues; + for (auto subtype : tupleType.getTypes()) { + subValues.push_back(recursiveRetuple(subtype, values, builder, loc)); + } + + return builder->create(loc, tupleType, subValues) + .getResult(); +} + +template +bool untupleAndLookupValues(T values, llvm::SmallVectorImpl *newValues, + OpBuilder *builder, Location loc, + BlockAndValueMapping *mapping) { + for (auto operand : values) { + auto newValue = mapping->lookupOrNull(operand); + if (!newValue) { + return true; + } + + recursiveUntuple(newValue, loc, builder, mapping, newValues); + } + + return false; +} + +bool convertReturnOp(ReturnOp *op, OpBuilder *builder, + BlockAndValueMapping *mapping) { + llvm::SmallVector newOperands; + if (untupleAndLookupValues(op->getOperands(), &newOperands, builder, + op->getLoc(), mapping)) { + return true; + } + + builder->create(op->getLoc(), newOperands); + return false; +} + +bool convertCallOp(CallOp *oldOp, OpBuilder *builder, + BlockAndValueMapping *mapping) { + llvm::SmallVector newArgs; + if (untupleAndLookupValues(oldOp->getOperands(), &newArgs, builder, + oldOp->getLoc(), mapping)) { + return true; + } + + SmallVector originalTypes(oldOp->getOperation()->getResultTypes()); + SmallVector resultTypes; + untupleTypes(originalTypes, &resultTypes); + auto newOp = builder->create(oldOp->getLoc(), oldOp->getCallee(), + resultTypes, newArgs); + copyOperationAttrs(oldOp->getOperation(), newOp.getOperation()); + + auto newResults = newOp.getResults(); + for (auto oldResult : oldOp->getResults()) { + llvm::SmallVector subValues; + auto newResult = recursiveRetuple(oldResult->getType(), &newResults, + builder, oldOp->getLoc()); + mapping->map(oldResult, newResult); + } + + return false; +} + +bool convertIndirectCallOp(CallIndirectOp *oldOp, OpBuilder *builder, + BlockAndValueMapping *mapping) { + llvm::SmallVector newArgs; + if (untupleAndLookupValues(oldOp->getOperands(), &newArgs, builder, + oldOp->getLoc(), mapping)) { + return true; + } + + auto newOp = builder->create(oldOp->getLoc(), + oldOp->getCallee(), newArgs); + copyOperationAttrs(oldOp->getOperation(), newOp.getOperation()); + + for (int i = 0; i < newOp.getNumResults(); ++i) { + auto *oldResult = oldOp->getResult(i); + auto *newResult = newOp.getResult(i); + mapping->map(oldResult, newResult); + } + + return false; +} + +bool convertBranchOp(BranchOp *oldOp, OpBuilder *builder, + BlockAndValueMapping *mapping) { + llvm::SmallVector newArgs; + if (untupleAndLookupValues(oldOp->getOperands(), &newArgs, builder, + oldOp->getLoc(), mapping)) { + return true; + } + + auto newOp = builder->create( + oldOp->getLoc(), mapping->lookupOrNull(oldOp->getDest()), newArgs); + + copyOperationAttrs(oldOp->getOperation(), newOp.getOperation()); + + return false; +} + +bool convertCondBranchOp(CondBranchOp *oldOp, OpBuilder *builder, + BlockAndValueMapping *mapping) { + llvm::SmallVector trueArgs; + if (untupleAndLookupValues(oldOp->getTrueOperands(), &trueArgs, builder, + oldOp->getLoc(), mapping)) { + return true; + } + + llvm::SmallVector falseArgs; + if (untupleAndLookupValues(oldOp->getFalseOperands(), &falseArgs, builder, + oldOp->getLoc(), mapping)) { + return true; + } + + auto newOp = builder->create( + oldOp->getLoc(), mapping->lookupOrNull(oldOp->getCondition()), + mapping->lookupOrNull(oldOp->getTrueDest()), trueArgs, + mapping->lookupOrNull(oldOp->getFalseDest()), falseArgs); + + copyOperationAttrs(oldOp->getOperation(), newOp.getOperation()); + + return false; +} + +bool convertOperation(Operation *op, OpBuilder *builder, + BlockAndValueMapping *mapping) { + if (auto returnOp = dyn_cast(op)) { + return convertReturnOp(&returnOp, builder, mapping); + } else if (auto callOp = dyn_cast(op)) { + return convertCallOp(&callOp, builder, mapping); + } else if (auto callIndirectOp = dyn_cast(op)) { + return convertIndirectCallOp(&callIndirectOp, builder, mapping); + } else if (auto branchOp = dyn_cast(op)) { + return convertBranchOp(&branchOp, builder, mapping); + } else if (auto condBranchOp = dyn_cast(op)) { + return convertCondBranchOp(&condBranchOp, builder, mapping); + } + + builder->clone(*op, *mapping); + return false; +} + +bool convertFunction(FuncOp oldFunction, FuncOp newFunction) { + OpBuilder builder(newFunction.getBody()); + BlockAndValueMapping mapping; + + newFunction.getBlocks().clear(); + for (auto &oldBlock : oldFunction.getBlocks()) { + auto *newBlock = builder.createBlock(&newFunction.getBody()); + for (auto *oldArg : oldBlock.getArguments()) { + llvm::SmallVector newTypes; + untupleTypes(oldArg->getType(), &newTypes); + + Value *newTuple = processTuple(oldArg->getType(), oldFunction.getLoc(), + newBlock, &builder); + if (!newTuple) { + return true; + } + + mapping.map(oldArg, newTuple); + } + mapping.map(&oldBlock, newBlock); + } + + // Convert all ops in the blocks. + for (auto &oldBlock : oldFunction.getBlocks()) { + builder.setInsertionPointToEnd(mapping.lookupOrNull(&oldBlock)); + for (auto &oldOp : oldBlock.getOperations()) { + if (convertOperation(&oldOp, &builder, &mapping)) { + return true; + } + } + } + + return false; +} + +class ConvertFromTupleCallingConventionPass + : public ModulePass { + public: + void runOnModule() override { + auto module = getModule(); + Builder builder(module.getContext()); + + // Build a list of (oldFunction, newFunction) for all functions we need to + // replace. This will ensure that when we go to convert function bodies we + // have only new functions defined. + std::vector> convertedFunctions; + + for (auto oldFunction : module.getOps()) { + auto oldFunctionType = oldFunction.getType(); + llvm::SmallVector newInputTypes; + untupleTypes(oldFunctionType.getInputs(), &newInputTypes); + + llvm::SmallVector newResultTypes; + untupleTypes(oldFunctionType.getResults(), &newResultTypes); + + auto newFunctionType = + builder.getFunctionType(newInputTypes, newResultTypes); + auto newFunction = + FuncOp::create(oldFunction.getLoc(), oldFunction.getName(), + newFunctionType, oldFunction.getDialectAttrs()); + convertedFunctions.push_back({oldFunction, newFunction}); + + // Perform the actual body conversion now that we have proper signatures. + if (convertFunction(oldFunction, newFunction)) { + return signalPassFailure(); + } + } + + // Replace functions in the module. + for (auto &pair : convertedFunctions) { + pair.first.erase(); + module.push_back(pair.second); + } + } +}; + +} // namespace + +std::unique_ptr> +createConvertFromTupleCallingConventionPass() { + return std::make_unique(); +} + +static PassRegistration pass( + "convert-from-tuple-calling-convention", + "Convert functions to remove tuples from method signatures."); + +} // namespace iree_compiler +} // namespace mlir diff --git a/compiler/Transforms/ConvertToMemRefCallingConvention.cpp b/compiler/Transforms/ConvertToMemRefCallingConvention.cpp new file mode 100644 index 000000000000..14138975c6b5 --- /dev/null +++ b/compiler/Transforms/ConvertToMemRefCallingConvention.cpp @@ -0,0 +1,402 @@ +// Copyright 2019 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "third_party/llvm/llvm/include/llvm/ADT/ArrayRef.h" +#include "third_party/llvm/llvm/include/llvm/ADT/DenseMap.h" +#include "third_party/llvm/llvm/include/llvm/ADT/SmallVector.h" +#include "third_party/llvm/llvm/projects/google_mlir/include/mlir/Dialect/StandardOps/Ops.h" +#include "third_party/llvm/llvm/projects/google_mlir/include/mlir/IR/Attributes.h" +#include "third_party/llvm/llvm/projects/google_mlir/include/mlir/IR/BlockAndValueMapping.h" +#include "third_party/llvm/llvm/projects/google_mlir/include/mlir/IR/Builders.h" +#include "third_party/llvm/llvm/projects/google_mlir/include/mlir/IR/Location.h" +#include "third_party/llvm/llvm/projects/google_mlir/include/mlir/IR/MLIRContext.h" +#include "third_party/llvm/llvm/projects/google_mlir/include/mlir/IR/StandardTypes.h" +#include "third_party/llvm/llvm/projects/google_mlir/include/mlir/IR/SymbolTable.h" +#include "third_party/llvm/llvm/projects/google_mlir/include/mlir/Pass/Pass.h" +#include "third_party/llvm/llvm/projects/google_mlir/include/mlir/Pass/PassRegistry.h" +#include "third_party/llvm/llvm/projects/google_mlir/include/mlir/Transforms/Utils.h" +#include "third_party/mlir_edge/iree/compiler/IR/Ops.h" +#include "third_party/mlir_edge/iree/compiler/IR/StructureOps.h" +#include "third_party/mlir_edge/iree/compiler/Utils/MemRefUtils.h" + +namespace mlir { +namespace iree_compiler { + +namespace { + +void copyOperationAttrs(Operation *oldOp, Operation *newOp) { + for (const auto &oldAttr : oldOp->getAttrs()) { + newOp->setAttr(oldAttr.first, oldAttr.second); + } +} + +FunctionType getMemRefFunctionType(FunctionType type) { + Builder builder(type.getContext()); + llvm::SmallVector replacementInputs; + for (auto type : type.getInputs()) { + auto memRefType = convertTypeToMemRef(type); + if (!memRefType) { + return nullptr; + } + replacementInputs.push_back(memRefType); + } + llvm::SmallVector replacementResults; + for (auto type : type.getResults()) { + auto memRefType = convertTypeToMemRef(type); + if (!memRefType) { + return nullptr; + } + replacementResults.push_back(memRefType); + } + return builder.getFunctionType(replacementInputs, replacementResults); +} + +bool insertLoad(BlockArgument *oldArg, BlockArgument *newArg, + OpBuilder *builder, BlockAndValueMapping *mapping) { + auto loc = oldArg->getOwner()->getParent()->getLoc(); + + // If old arg was a memref we don't need to change anything. We still need + // to remap so that the use lists match through conversion, though. + if (oldArg->getType().isa()) { + mapping->map(oldArg, newArg); + return false; + } else if (oldArg->getType().isa()) { + auto castOp = + builder->create(loc, oldArg->getType(), newArg); + mapping->map(oldArg, castOp.getResult()); + return false; + } + + // Insert the load we'll use to unbox the value. + auto loadedValue = builder->create(loc, newArg, ArrayRef{}); + mapping->map(oldArg, loadedValue); + + return false; +} + +bool insertLoad(Operation *oldOp, Value *oldValue, Value *newValue, + OpBuilder *builder, BlockAndValueMapping *mapping) { + // If old value was a memref we don't need to change anything. + if (oldValue->getType().isa()) { + mapping->map(oldValue, newValue); + return false; + } else if (oldValue->getType().isa()) { + auto castOp = builder->create( + oldOp->getLoc(), oldValue->getType(), newValue); + mapping->map(oldValue, castOp.getResult()); + return false; + } + + assert(newValue->getType().isa()); + + // Insert the load we'll use to unbox the value. + auto loadedValue = + builder->create(oldOp->getLoc(), newValue, ArrayRef{}); + mapping->map(oldValue, loadedValue); + + return false; +} + +Value *insertStore(Operation *oldOp, Value *oldValue, OpBuilder *builder, + BlockAndValueMapping *mapping) { + auto *newValue = mapping->lookupOrNull(oldValue); + if (!newValue) { + return nullptr; + } + + auto memRefType = convertTypeToMemRef(oldValue->getType()); + if (!memRefType) { + return nullptr; + } + + // If the previous value was already a memref we don't need to change + // anything. + // TODO(benvanik): ensure indices make sense. + if (oldValue->getType().isa()) { + return newValue; + } else if (oldValue->getType().isa()) { + auto castOp = builder->create(oldOp->getLoc(), + memRefType, newValue); + return castOp.getResult(); + } + + // Look back up and see if we can find the memref the value was loaded from. + if (auto *sourceMemRef = resolveValueToSourceMemRef(oldValue, oldOp)) { + return mapping->lookupOrNull(sourceMemRef); + } + + // Allocate the memref to store the value. + auto newStorage = builder->create(oldOp->getLoc(), memRefType); + + // Insert the store we'll use to box the value. + builder->create(oldOp->getLoc(), newValue, newStorage, + ArrayRef{}); + + return newStorage; +} + +bool convertCallOp(CallOp *oldOp, OpBuilder *builder, + BlockAndValueMapping *mapping) { + llvm::SmallVector newArgs; + for (auto *oldArg : oldOp->getOperands()) { + auto *newArg = insertStore(oldOp->getOperation(), oldArg, builder, mapping); + if (!newArg) { + return true; + } + newArgs.push_back(newArg); + } + + SmallVector resultTypes; + for (auto oldType : oldOp->getOperation()->getResultTypes()) { + resultTypes.push_back(convertTypeToMemRef(oldType)); + } + auto newOp = builder->create(oldOp->getLoc(), oldOp->getCallee(), + resultTypes, newArgs); + copyOperationAttrs(oldOp->getOperation(), newOp.getOperation()); + + for (int i = 0; i < newOp.getNumResults(); ++i) { + auto *oldResult = oldOp->getResult(i); + auto *newResult = newOp.getResult(i); + if (insertLoad(oldOp->getOperation(), oldResult, newResult, builder, + mapping)) { + return true; + } + } + + return false; +} + +bool convertCallIndirectOp(CallIndirectOp *oldOp, OpBuilder *builder, + BlockAndValueMapping *mapping) { + // TODO(benvanik): support wrapping callee values. + oldOp->emitError("CallIndirectOp not yet supported"); + return true; +#if 0 + llvm::SmallVector newArgs; + for (auto *oldArg : oldOp->getArgOperands()) { + auto *newArg = insertStore(oldOp->getOperation(), oldArg, builder, mapping); + if (!newArg) { + return true; + } + newArgs.push_back(newArg); + } + + auto newOp = builder->create(oldOp->getLoc(), + oldOp->getCallee(), newArgs); + copyOperationAttrs(oldOp->getOperation(), newOp.getOperation()); + + for (int i = 0; i < newOp.getNumResults(); ++i) { + auto *oldResult = oldOp->getResult(i); + auto *newResult = newOp.getResult(i); + if (insertLoad(oldOp->getOperation(), oldResult, newResult, builder, + mapping)) { + return true; + } + } + + return false; +#endif // 0 +} + +bool convertReturnOp(Operation *oldOp, OpBuilder *builder, + BlockAndValueMapping *mapping) { + BlockAndValueMapping returnMapping; + for (auto *oldArg : oldOp->getOperands()) { + auto *newArg = insertStore(oldOp, oldArg, builder, mapping); + if (!newArg) { + return true; + } + returnMapping.map(oldArg, newArg); + } + + builder->clone(*oldOp, returnMapping); + return false; +} + +bool convertBranchOp(BranchOp *oldOp, OpBuilder *builder, + BlockAndValueMapping *mapping) { + llvm::SmallVector newArgs; + for (auto *oldArg : oldOp->getOperands()) { + auto *newArg = insertStore(oldOp->getOperation(), oldArg, builder, mapping); + if (!newArg) { + return true; + } + newArgs.push_back(newArg); + } + + auto *dest = mapping->lookupOrNull(oldOp->getDest()); + if (!dest) { + oldOp->emitError("Destination block mapping not found"); + return true; + } + + auto newOp = builder->create(oldOp->getLoc(), dest, newArgs); + copyOperationAttrs(oldOp->getOperation(), newOp.getOperation()); + + return false; +} + +bool convertCondBranchOp(CondBranchOp *oldOp, OpBuilder *builder, + BlockAndValueMapping *mapping) { + llvm::SmallVector trueArgs; + for (auto *oldArg : oldOp->getTrueOperands()) { + auto *newArg = insertStore(oldOp->getOperation(), oldArg, builder, mapping); + if (!newArg) { + return true; + } + trueArgs.push_back(newArg); + } + llvm::SmallVector falseArgs; + for (auto *oldArg : oldOp->getFalseOperands()) { + auto *newArg = insertStore(oldOp->getOperation(), oldArg, builder, mapping); + if (!newArg) { + return true; + } + falseArgs.push_back(newArg); + } + + auto *trueDest = mapping->lookupOrNull(oldOp->getTrueDest()); + if (!trueDest) { + oldOp->emitError("True destination block mapping not found"); + return true; + } + auto *falseDest = mapping->lookupOrNull(oldOp->getFalseDest()); + if (!falseDest) { + oldOp->emitError("False destination block mapping not found"); + return true; + } + + // Lowering will take care of the condition store. + auto *newCondition = mapping->lookupOrNull(oldOp->getCondition()); + if (!newCondition) { + oldOp->emitError("Condition value mapping not found"); + return false; + } + + auto newOp = builder->create( + oldOp->getLoc(), newCondition, trueDest, trueArgs, falseDest, falseArgs); + copyOperationAttrs(oldOp->getOperation(), newOp.getOperation()); + + return false; +} + +bool convertOperation(Operation *oldOp, OpBuilder *builder, + BlockAndValueMapping *mapping) { + if (isa(oldOp)) { + builder->clone(*oldOp, *mapping); + return false; + } else if (auto callOp = dyn_cast(oldOp)) { + return convertCallOp(&callOp, builder, mapping); + } else if (auto callIndirectOp = dyn_cast(oldOp)) { + return convertCallIndirectOp(&callIndirectOp, builder, mapping); + } else if (isa(oldOp) || isa(oldOp)) { + return convertReturnOp(oldOp, builder, mapping); + } else if (auto branchOp = dyn_cast(oldOp)) { + return convertBranchOp(&branchOp, builder, mapping); + } else if (auto condBranchOp = dyn_cast(oldOp)) { + return convertCondBranchOp(&condBranchOp, builder, mapping); + } else { + builder->clone(*oldOp, *mapping); + return false; + } +} + +bool convertFunction(FuncOp oldFunc, FuncOp newFunc) { + OpBuilder builder(newFunc.getBody()); + BlockAndValueMapping mapping; + + // Create new blocks matching the expected arguments of the old ones. + // This sets up the block mappings to enable us to reference blocks forward + // during conversion. + newFunc.getBlocks().clear(); + for (auto &oldBlock : oldFunc.getBlocks()) { + auto *newBlock = builder.createBlock(&newFunc.getBody()); + for (auto *oldArg : oldBlock.getArguments()) { + // Replace the block args with memrefs. + auto memRefType = convertTypeToMemRef(oldArg->getType()); + if (!memRefType) return true; + auto *newArg = newBlock->addArgument(memRefType); + + // Insert loads to preserve type, if needed. + // This will replace all uses of the oldArg with the loaded value from + // newArg so that the block contents are still using unwrapped values. + if (insertLoad(oldArg, newArg, &builder, &mapping)) { + return true; + } + } + mapping.map(&oldBlock, newBlock); + } + + // Convert all ops in the blocks. + for (auto &oldBlock : oldFunc.getBlocks()) { + builder.setInsertionPointToEnd(mapping.lookupOrNull(&oldBlock)); + for (auto &oldOp : oldBlock.getOperations()) { + if (convertOperation(&oldOp, &builder, &mapping)) { + return true; + } + } + } + + return false; +} + +} // namespace + +class ConvertToMemRefCallingConventionPass + : public ModulePass { + public: + void runOnModule() override { + auto module = getModule(); + + // Build a list of (oldFunc, newFunc) for all functions we need to + // replace. This will ensure that when we go to convert function bodies we + // have only new functions defined. + std::vector> convertedFunctions; + + for (auto oldFunc : module.getOps()) { + // Create the replacement function, ensuring that we copy attributes. + auto functionType = getMemRefFunctionType(oldFunc.getType()); + if (!functionType) { + return signalPassFailure(); + } + + auto newFunc = FuncOp::create(oldFunc.getLoc(), oldFunc.getName(), + functionType, oldFunc.getDialectAttrs()); + convertedFunctions.push_back({oldFunc, newFunc}); + + // Perform the actual body conversion now. + if (convertFunction(oldFunc, newFunc)) { + return signalPassFailure(); + } + } + + // Replace functions in the module. + for (auto &pair : convertedFunctions) { + pair.first.erase(); + module.push_back(pair.second); + } + } +}; + +std::unique_ptr> +createConvertToMemRefCallingConventionPass() { + return std::make_unique(); +} + +static PassRegistration pass( + "convert-to-memref-calling-convention", + "Convert functions to use a memref-based calling convention."); + +} // namespace iree_compiler +} // namespace mlir diff --git a/compiler/Transforms/DropUnreachableFunctions.cpp b/compiler/Transforms/DropUnreachableFunctions.cpp new file mode 100644 index 000000000000..949851e34aa7 --- /dev/null +++ b/compiler/Transforms/DropUnreachableFunctions.cpp @@ -0,0 +1,67 @@ +// Copyright 2019 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "third_party/llvm/llvm/include/llvm/ADT/SetVector.h" +#include "third_party/llvm/llvm/projects/google_mlir/include/mlir/Dialect/StandardOps/Ops.h" +#include "third_party/llvm/llvm/projects/google_mlir/include/mlir/IR/Attributes.h" +#include "third_party/llvm/llvm/projects/google_mlir/include/mlir/Pass/Pass.h" +#include "third_party/llvm/llvm/projects/google_mlir/include/mlir/Pass/PassRegistry.h" +#include "third_party/llvm/llvm/projects/google_mlir/include/mlir/Support/LogicalResult.h" +#include "third_party/llvm/llvm/projects/google_mlir/include/mlir/Transforms/Utils.h" +#include "third_party/mlir_edge/iree/compiler/Utils/ModuleUtils.h" + +namespace mlir { +namespace iree_compiler { + +// Drops all functions in a module that are not reachable by functions with the +// "iree.module.export" attribute. +class DropUnreachableModuleFunctionsPass + : public ModulePass { + public: + void runOnModule() override { + dropUnusedFunctions(getModule(), {"iree.module.export"}); + } +}; + +// Drops all functions in a module that are not reachable by functions with the +// "iree.executable.export" attribute. +class DropUnreachableExecutableFunctionsPass + : public ModulePass { + public: + void runOnModule() override { + dropUnusedFunctions(getModule(), {"iree.executable.export"}); + } +}; + +std::unique_ptr> +createDropUnreachableModuleFunctionsPass() { + return std::make_unique(); +} + +std::unique_ptr> +createDropUnreachableExecutableFunctionsPass() { + return std::make_unique(); +} + +static PassRegistration moduleFunctionsPass( + "iree-drop-unreachable-module-functions", + "Drop all functions not reachable from an exported function"); + +static PassRegistration + executableFunctionsPass( + "iree-drop-unreachable-executable-functions", + "Drop all functions not reachable from an exported function"); + +} // namespace iree_compiler +} // namespace mlir diff --git a/compiler/Transforms/DropUnusedExecutables.cpp b/compiler/Transforms/DropUnusedExecutables.cpp new file mode 100644 index 000000000000..612ce8a143bf --- /dev/null +++ b/compiler/Transforms/DropUnusedExecutables.cpp @@ -0,0 +1,62 @@ +// Copyright 2019 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "third_party/llvm/llvm/include/llvm/ADT/SetVector.h" +#include "third_party/llvm/llvm/projects/google_mlir/include/mlir/Dialect/StandardOps/Ops.h" +#include "third_party/llvm/llvm/projects/google_mlir/include/mlir/IR/Attributes.h" +#include "third_party/llvm/llvm/projects/google_mlir/include/mlir/Pass/Pass.h" +#include "third_party/llvm/llvm/projects/google_mlir/include/mlir/Pass/PassRegistry.h" +#include "third_party/llvm/llvm/projects/google_mlir/include/mlir/Support/LLVM.h" +#include "third_party/llvm/llvm/projects/google_mlir/include/mlir/Support/LogicalResult.h" +#include "third_party/llvm/llvm/projects/google_mlir/include/mlir/Transforms/Utils.h" +#include "third_party/mlir_edge/iree/compiler/IR/Sequencer/HLOps.h" +#include "third_party/mlir_edge/iree/compiler/IR/StructureOps.h" + +namespace mlir { +namespace iree_compiler { + +// Drops all executables in a module that are not used by any dispatch +// sequencer op. +class DropUnusedExecutablesPass : public ModulePass { + public: + void runOnModule() override { + DenseSet usedExecutableNames; + for (auto funcOp : getModule().getOps()) { + funcOp.walk([&](IREESeq::HL::DispatchOp op) { + usedExecutableNames.insert(op.getExecutable()); + }); + } + DenseSet deadExecutables; + for (auto executableOp : + getModule().getOps()) { + if (usedExecutableNames.count(executableOp.getName()) == 0) { + deadExecutables.insert(executableOp); + } + } + for (auto executableOp : deadExecutables) { + executableOp->erase(); + } + } +}; + +std::unique_ptr> createDropUnusedExecutablesPass() { + return std::make_unique(); // NOLINT +} + +static PassRegistration executableFunctionsPass( + "iree-drop-unused-executables", + "Drop all executables not reachable from a dispatch/reduce op."); + +} // namespace iree_compiler +} // namespace mlir diff --git a/compiler/Transforms/Interpreter/ExpandReductionsToOps.cpp b/compiler/Transforms/Interpreter/ExpandReductionsToOps.cpp new file mode 100644 index 000000000000..5a6136684cc5 --- /dev/null +++ b/compiler/Transforms/Interpreter/ExpandReductionsToOps.cpp @@ -0,0 +1,205 @@ +// Copyright 2019 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "third_party/llvm/llvm/include/llvm/ADT/ArrayRef.h" +#include "third_party/llvm/llvm/include/llvm/ADT/STLExtras.h" +#include "third_party/llvm/llvm/include/llvm/ADT/SmallVector.h" +#include "third_party/llvm/llvm/include/llvm/ADT/StringRef.h" +#include "third_party/llvm/llvm/projects/google_mlir/include/mlir/IR/Attributes.h" +#include "third_party/llvm/llvm/projects/google_mlir/include/mlir/IR/Builders.h" +#include "third_party/llvm/llvm/projects/google_mlir/include/mlir/IR/Function.h" +#include "third_party/llvm/llvm/projects/google_mlir/include/mlir/IR/Operation.h" +#include "third_party/llvm/llvm/projects/google_mlir/include/mlir/IR/PatternMatch.h" +#include "third_party/llvm/llvm/projects/google_mlir/include/mlir/IR/StandardTypes.h" +#include "third_party/llvm/llvm/projects/google_mlir/include/mlir/IR/TypeUtilities.h" +#include "third_party/llvm/llvm/projects/google_mlir/include/mlir/IR/Value.h" +#include "third_party/llvm/llvm/projects/google_mlir/include/mlir/Pass/Pass.h" +#include "third_party/mlir_edge/iree/compiler/IR/Dialect.h" +#include "third_party/mlir_edge/iree/compiler/IR/Interpreter/LLDialect.h" +#include "third_party/mlir_edge/iree/compiler/IR/Interpreter/LLOps.h" +#include "third_party/mlir_edge/iree/compiler/IR/Ops.h" +#include "third_party/mlir_edge/iree/compiler/Transforms/ConversionUtils.h" +#include "third_party/mlir_edge/iree/compiler/Utils/MemRefUtils.h" +#include "third_party/tensorflow/compiler/mlir/xla/ir/hlo_ops.h" + +namespace mlir { +namespace iree_compiler { + +namespace { + +LogicalResult convertReductionOp(FuncOp entryPoint, FuncOp applyFunc, + Operation *elementOp, OpBuilder *builder) { + // Ensure that this op is pass-through and does not interact with any other + // ops within the function. + // TODO(b/139313439): support fused reductions. + for (auto *operand : elementOp->getOperands()) { + if (operand->getDefiningOp() != nullptr) { + return elementOp->emitOpError() + << "Fused reductions are not supported (operand not sourced from " + "block args)"; + } + } + for (auto *result : elementOp->getResults()) { + for (auto *user : result->getUsers()) { + if (!user->isKnownTerminator()) { + return elementOp->emitOpError() << "Fused reductions are not supported " + "(result used by non-terminator)"; + } + } + } + + // Determine the index of the args we care about. We'll use these to match up + // the operands of the entry point with our application. + // Our arguments are expanded tuples like , , so this + // index gets the offset * 2. + auto &applyEntryBlock = applyFunc.getBlocks().front(); + int setIndex = std::distance(applyEntryBlock.args_begin(), + llvm::find(applyEntryBlock.getArguments(), + elementOp->getOperand(0))) / + 2; + + // Map to the args from the entry point. + auto &entryPointEntryBlock = entryPoint.getBlocks().front(); + Value *srcArg = entryPointEntryBlock.getArgument(setIndex); + Value *initArg = entryPointEntryBlock.getArgument( + applyFunc.getNumArguments() / 2 + setIndex); + Value *dstArg = + entryPointEntryBlock.getArgument(applyFunc.getNumArguments() + setIndex); + auto dstType = dstArg->getType().cast(); + Type elementType = dstType.getElementType(); + auto dimensionAttr = entryPoint.getAttrOfType( + "iree.executable.reduction.dimension"); + + Operation *expandedOp = nullptr; + if (isa(elementOp) || isa(elementOp) || + isa(elementOp)) { + if (elementType.isa()) { + expandedOp = builder->create( + elementOp->getLoc(), srcArg, initArg, dimensionAttr, dstArg); + } else { + expandedOp = builder->create( + elementOp->getLoc(), srcArg, initArg, dimensionAttr, dstArg); + } + } else if (isa(elementOp)) { + if (elementType.isa()) { + expandedOp = builder->create( + elementOp->getLoc(), srcArg, initArg, dimensionAttr, dstArg); + } else { + expandedOp = builder->create( + elementOp->getLoc(), srcArg, initArg, dimensionAttr, dstArg); + } + } else if (isa(elementOp)) { + if (elementType.isa()) { + expandedOp = builder->create( + elementOp->getLoc(), srcArg, initArg, dimensionAttr, dstArg); + } else { + expandedOp = builder->create( + elementOp->getLoc(), srcArg, initArg, dimensionAttr, dstArg); + } + } + if (!expandedOp) { + return elementOp->emitOpError() + << "No matching expanded reduction op for elemental op"; + } + + return success(); +} + +// Replaces the given elemental |funcOp| with a widened reduction. +LogicalResult expandReductionFunction(FuncOp entryFunc) { + if (!entryFunc.empty()) { + return entryFunc.emitError() + << "Function has already been expanded or has existing contents"; + } else if (!entryFunc.getAttr("iree.executable.reduction.dimension")) { + return entryFunc.emitError() << "Windowed reductions are not yet supported"; + } + auto applySym = + entryFunc.getAttrOfType("iree.executable.reduction.apply"); + if (!applySym) { + return entryFunc.emitError() << "No reduction application function defined"; + } + auto applyFunc = entryFunc.getParentOfType().lookupSymbol( + applySym.getValue()); + if (!applyFunc) { + return entryFunc.emitError() + << "Unable to find apply function " << applySym; + } + + auto *entryBlock = entryFunc.addEntryBlock(); + OpBuilder builder(entryBlock); + + if (applyFunc.getBlocks() + .front() + .walk([&](Operation *op) { + if (!op->isKnownTerminator()) { + if (failed( + convertReductionOp(entryFunc, applyFunc, op, &builder))) { + return WalkResult::interrupt(); + } + } + return WalkResult::advance(); + }) + .wasInterrupted()) { + return applyFunc.emitError() << "Unable to convert apply func"; + } + + builder.create(builder.getUnknownLoc()); + + // Remove the apply function as we have inlined it. + applyFunc.erase(); + entryFunc.removeAttr("iree.executable.reduction.apply"); + entryFunc.removeAttr("iree.executable.reduction.dimension"); + + return success(); +} + +// Limited lowering of reductions to fat reduce_* ops. +// +// The specific subset this supports is: +// * 'min', 'max', and 'add' computations, with function names matching the +// computation +// * one op per reduction (no fusions yet). +// Note: computations and shapes are not validated. +// +// TODO(b/139410773): Implement more generally, supporting custom computations. +class ExpandReductionsToOpsPass : public ModulePass { + public: + void runOnModule() override { + auto module = getModule(); + SmallVector reductionFuncs; + for (auto funcOp : module.getOps()) { + if (funcOp.getAttr("iree.executable.reduction.apply")) { + reductionFuncs.push_back(funcOp); + } + } + for (auto funcOp : reductionFuncs) { + if (failed(expandReductionFunction(funcOp))) { + return signalPassFailure(); + } + } + } +}; + +} // namespace + +std::unique_ptr> createExpandReductionsToOpsPass() { + return std::make_unique(); +} + +static PassRegistration pass( + "iree-expand-reductions-to-ops", + "Expands IREE reduction functions to their interpreter ops"); + +} // namespace iree_compiler +} // namespace mlir diff --git a/compiler/Transforms/Interpreter/LegalizeInterpreterOps.cpp b/compiler/Transforms/Interpreter/LegalizeInterpreterOps.cpp new file mode 100644 index 000000000000..12fe67969374 --- /dev/null +++ b/compiler/Transforms/Interpreter/LegalizeInterpreterOps.cpp @@ -0,0 +1,75 @@ +// Copyright 2019 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include +#include + +#include "third_party/llvm/llvm/projects/google_mlir/include/mlir/Dialect/StandardOps/Ops.h" +#include "third_party/llvm/llvm/projects/google_mlir/include/mlir/IR/BlockAndValueMapping.h" +#include "third_party/llvm/llvm/projects/google_mlir/include/mlir/IR/OperationSupport.h" +#include "third_party/llvm/llvm/projects/google_mlir/include/mlir/IR/PatternMatch.h" +#include "third_party/llvm/llvm/projects/google_mlir/include/mlir/Pass/Pass.h" +#include "third_party/llvm/llvm/projects/google_mlir/include/mlir/Pass/PassRegistry.h" +#include "third_party/mlir_edge/iree/compiler/IR/Interpreter/HLOps.h" +#include "third_party/mlir_edge/iree/compiler/IR/Ops.h" + +namespace mlir { +namespace iree_compiler { + +namespace optimize_patterns { +#include "third_party/mlir_edge/iree/compiler/Transforms/Interpreter/LegalizeInterpreterOps.inc" +} // namespace optimize_patterns + +void tryElideClone(IREEInterp::HL::CloneOp *cloneOp, + std::vector *deadOperations) { + cloneOp->replaceAllUsesWith(cloneOp->src()); + deadOperations->push_back(cloneOp->getOperation()); +} + +void removeIdentityClones(FuncOp function) { + std::vector deadOperations; + for (auto &block : function.getBlocks()) { + for (auto &op : block.getOperations()) { + if (auto cloneOp = dyn_cast(op)) { + tryElideClone(&cloneOp, &deadOperations); + } + } + } + for (auto *op : deadOperations) { + op->erase(); + } +} + +class LegalizeInterpreterOpsPass + : public FunctionPass { + public: + void runOnFunction() override { + OwningRewritePatternList patterns; + auto *context = getFunction().getContext(); + optimize_patterns::populateWithGenerated(context, &patterns); + applyPatternsGreedily(getFunction(), patterns); + + removeIdentityClones(getFunction()); + } +}; + +std::unique_ptr> createLegalizeInterpreterOpsPass() { + return std::make_unique(); +} + +static PassRegistration pass( + "iree-legalize-interpreter-ops", "Optimizes common IREE op patterns."); + +} // namespace iree_compiler +} // namespace mlir diff --git a/compiler/Transforms/Interpreter/LegalizeInterpreterOps.td b/compiler/Transforms/Interpreter/LegalizeInterpreterOps.td new file mode 100644 index 000000000000..f5f83e16be53 --- /dev/null +++ b/compiler/Transforms/Interpreter/LegalizeInterpreterOps.td @@ -0,0 +1,6 @@ +#ifdef IREE_INTERPRETER_HL_OPS +#else +include "third_party/mlir_edge/iree/compiler/IR/Interpreter/HLOps.td" +#endif // IREE_INTERPRETER_HL_OPS + +// TODO(benvanik): fun patterns. diff --git a/compiler/Transforms/Interpreter/LoadStoreDataFlowOpt.cpp b/compiler/Transforms/Interpreter/LoadStoreDataFlowOpt.cpp new file mode 100644 index 000000000000..1da2471c74e7 --- /dev/null +++ b/compiler/Transforms/Interpreter/LoadStoreDataFlowOpt.cpp @@ -0,0 +1,148 @@ +// Copyright 2019 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include + +#include "third_party/llvm/llvm/include/llvm/ADT/ArrayRef.h" +#include "third_party/llvm/llvm/include/llvm/ADT/SetVector.h" +#include "third_party/llvm/llvm/include/llvm/ADT/SmallVector.h" +#include "third_party/llvm/llvm/include/llvm/ADT/iterator_range.h" +#include "third_party/llvm/llvm/projects/google_mlir/include/mlir/Dialect/StandardOps/Ops.h" +#include "third_party/llvm/llvm/projects/google_mlir/include/mlir/IR/BlockAndValueMapping.h" +#include "third_party/llvm/llvm/projects/google_mlir/include/mlir/IR/Builders.h" +#include "third_party/llvm/llvm/projects/google_mlir/include/mlir/IR/MLIRContext.h" +#include "third_party/llvm/llvm/projects/google_mlir/include/mlir/IR/StandardTypes.h" +#include "third_party/llvm/llvm/projects/google_mlir/include/mlir/Pass/Pass.h" +#include "third_party/llvm/llvm/projects/google_mlir/include/mlir/Pass/PassRegistry.h" +#include "third_party/llvm/llvm/projects/google_mlir/include/mlir/Transforms/Utils.h" +#include "third_party/mlir_edge/iree/compiler/IR/Interpreter/HLOps.h" +#include "third_party/mlir_edge/iree/compiler/IR/Ops.h" +#include "third_party/mlir_edge/iree/compiler/Utils/MemRefUtils.h" +#include "third_party/mlir_edge/iree/compiler/Utils/OpUtils.h" + +namespace mlir { +namespace iree_compiler { + +namespace { + +// Returns a value containing the indices in the form of a memref with shape +// {|indices|.size()}. +Value *makeIndicesMemRef(const MemRefType &type, + Operation::operand_range indices, OpBuilder &builder) { + auto &useOp = *builder.getInsertionPoint(); + size_t indicesCount = std::distance(indices.begin(), indices.end()); + if (indicesCount == 0) { + return builder + .create( + useOp.getLoc(), builder.getMemRefType({1}, builder.getIndexType()), + builder.getIntegerAttr(builder.getIndexType(), 0)) + .getResult(); + } else if (indicesCount == 1) { + auto allocOp = builder.create( + useOp.getLoc(), builder.getMemRefType({1}, builder.getIndexType())); + auto storeIndex = builder.create( + useOp.getLoc(), builder.getIndexType(), + builder.getIntegerAttr(builder.getIndexType(), 0)); + builder.create(useOp.getLoc(), *indices.begin(), + allocOp.getResult(), ArrayRef{storeIndex}); + return allocOp; + } + + // TODO(benvanik): support arbitrary indices. + useOp.emitError() << "Multiple indices are not yet implemented"; + return nullptr; +} + +// Returns a value containing the lengths in the form of a memref with shape +// {|dims|.size()}. +Value *makeLengthsMemRef(Value *storedValue, OpBuilder &builder) { + Type valueType = storedValue->getType(); + if (auto shapedType = valueType.dyn_cast()) { + auto shapeType = + builder.getMemRefType({shapedType.getRank()}, builder.getIndexType()); + return builder.create(storedValue->getLoc(), + shapeType, storedValue); + } else { + return builder.create( + storedValue->getLoc(), + builder.getMemRefType({1}, builder.getIndexType()), + builder.getIntegerAttr(builder.getIndexType(), 1)); + } +} + +// Returns the origin operation of a value if it is a load. +LoadOp findOriginLoadOperation(Value *value) { + // TODO(benvanik): follow through identity ops or something? + if (auto loadOp = dyn_cast(value->getDefiningOp())) { + return loadOp; + } + return nullptr; +} + +// Inserts a copy operation performing the same work as a store. +// +// Example: +// %0 = ... : memref<4xf32> +// %1 = load %0[%offset] : memref<4xf32> +// %2 = ... : memref +// store %1, %2[] : memref +// -> +// %0 = ... : memref<4xf32> +// %2 = ... : memref +// iree_hl_interp.copy %0[%offset], %2[], [%length] +void insertCopyForStore(LoadOp &loadOp, StoreOp &storeOp) { + OpBuilder builder(storeOp); + auto *srcIndices = + makeIndicesMemRef(loadOp.getMemRefType(), loadOp.getIndices(), builder); + auto *dstIndices = + makeIndicesMemRef(storeOp.getMemRefType(), storeOp.getIndices(), builder); + auto *lengths = makeLengthsMemRef(storeOp.getValueToStore(), builder); + builder.create(storeOp.getLoc(), loadOp.getMemRef(), + srcIndices, storeOp.getMemRef(), + dstIndices, lengths); +} + +} // namespace + +class InterpreterLoadStoreDataFlowOptPass + : public FunctionPass { + public: + void runOnFunction() override { + auto func = getFunction(); + + // Find stores and attempt to optimize load+store pairs. + llvm::SetVector deadOperations; + func.walk([&](StoreOp storeOp) { + if (auto loadOp = findOriginLoadOperation(storeOp.getValueToStore())) { + insertCopyForStore(loadOp, storeOp); + deadOperations.insert(storeOp); + } + }); + + // Remove all the now-unused ops. + removeDeadOperations(deadOperations); + } +}; + +std::unique_ptr> +createInterpreterLoadStoreDataFlowOptPass() { + return std::make_unique(); +} + +static PassRegistration pass( + "iree-interpreter-load-store-data-flow-opt", + "Optimize local load and store data flow by removing redundant accesses"); + +} // namespace iree_compiler +} // namespace mlir diff --git a/compiler/Transforms/Interpreter/LowerInterpreterDialect.cpp b/compiler/Transforms/Interpreter/LowerInterpreterDialect.cpp new file mode 100644 index 000000000000..17447667697c --- /dev/null +++ b/compiler/Transforms/Interpreter/LowerInterpreterDialect.cpp @@ -0,0 +1,246 @@ +// Copyright 2019 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "third_party/llvm/llvm/include/llvm/ADT/DenseSet.h" +#include "third_party/llvm/llvm/include/llvm/ADT/SmallVector.h" +#include "third_party/llvm/llvm/include/llvm/Support/Allocator.h" +#include "third_party/llvm/llvm/include/llvm/Support/Casting.h" +#include "third_party/llvm/llvm/projects/google_mlir/include/mlir/Dialect/StandardOps/Ops.h" +#include "third_party/llvm/llvm/projects/google_mlir/include/mlir/IR/Attributes.h" +#include "third_party/llvm/llvm/projects/google_mlir/include/mlir/IR/BlockAndValueMapping.h" +#include "third_party/llvm/llvm/projects/google_mlir/include/mlir/IR/Location.h" +#include "third_party/llvm/llvm/projects/google_mlir/include/mlir/IR/MLIRContext.h" +#include "third_party/llvm/llvm/projects/google_mlir/include/mlir/IR/PatternMatch.h" +#include "third_party/llvm/llvm/projects/google_mlir/include/mlir/IR/StandardTypes.h" +#include "third_party/llvm/llvm/projects/google_mlir/include/mlir/Pass/Pass.h" +#include "third_party/llvm/llvm/projects/google_mlir/include/mlir/Pass/PassRegistry.h" +#include "third_party/llvm/llvm/projects/google_mlir/include/mlir/Transforms/DialectConversion.h" +#include "third_party/llvm/llvm/projects/google_mlir/include/mlir/Transforms/Utils.h" +#include "third_party/mlir_edge/iree/compiler/IR/Dialect.h" +#include "third_party/mlir_edge/iree/compiler/IR/Interpreter/HLDialect.h" +#include "third_party/mlir_edge/iree/compiler/IR/Interpreter/HLOps.h" +#include "third_party/mlir_edge/iree/compiler/IR/Interpreter/LLDialect.h" +#include "third_party/mlir_edge/iree/compiler/IR/Interpreter/LLOps.h" +#include "third_party/mlir_edge/iree/compiler/IR/Ops.h" +#include "third_party/mlir_edge/iree/compiler/Serialization/BytecodeTables.h" +#include "third_party/mlir_edge/iree/schemas/bytecode/interpreter_bytecode_v0.h" + +namespace mlir { +namespace iree_compiler { + +namespace { + +struct LowerBranchOpPattern + : public OpRewritePattern { + using OpRewritePattern::OpRewritePattern; + + PatternMatchResult matchAndRewrite(IREEInterp::HL::BranchOp op, + PatternRewriter &rewriter) const { + SmallVector operands{op.getOperation()->getOperands()}; + + rewriter.replaceOpWithNewOp(op, op.getDest(), + operands); + return matchSuccess(); + } +}; + +struct LowerCondCondBranchOpPattern + : public OpRewritePattern { + using OpRewritePattern::OpRewritePattern; + + PatternMatchResult matchAndRewrite(IREEInterp::HL::CondBranchOp op, + PatternRewriter &rewriter) const { + SmallVector trueOperands{op.getTrueOperands()}; + SmallVector falseOperands{op.getFalseOperands()}; + + rewriter.replaceOpWithNewOp( + op, op.getCondition(), op.getTrueDest(), trueOperands, + op.getFalseDest(), falseOperands); + return matchSuccess(); + } +}; + +// Returns true if the op defined by |opName| (like 'iree_ll_interp.reshape') +// uses output operands for results (like iree_ll_interp.add_i) or returns real +// results. +bool opTakesOutputOperands(llvm::StringRef opName) { + if (!opName.consume_front("iree_ll_interp.")) { + assert(false && "op not part of IREE LL Interpreter dialect"); + return false; + } + auto opcode = GetInterpreterOpcodeByName(opName.str()); + assert(opcode.hasValue() && "op has no corresponding opcode"); + const auto &info = GetInterpreterOpcodeInfo(opcode.getValue()); + for (auto &operand : info.operands) { + if (operand == iree::OperandEncoding::kOutputSlot || + operand == iree::OperandEncoding::kVariadicOutputSlots) { + return true; + } + } + return false; +} + +template +class SimpleOpLowering : public OpRewritePattern { + using OpRewritePattern::OpRewritePattern; + + PatternMatchResult matchAndRewrite(SrcOp op, + PatternRewriter &rewriter) const { + SmallVector operands{op.getOperation()->getOperands()}; + + // Most ops take results as output operands to populate during execution. + // Certain ops, like reshape, return references to existing memrefs and + // should still retain their results. + if (!opTakesOutputOperands(DstOp::getOperationName())) { + SmallVector resultTypes{op.getOperation()->getResultTypes()}; + + rewriter.replaceOpWithNewOp(op, resultTypes, operands, + op.getAttrs()); + return this->matchSuccess(); + } + + for (Value *result : op.getOperation()->getResults()) { + auto memRefType = result->getType().cast(); + if (!memRefType.hasStaticShape()) { + // TODO(benvanik): real thing here - dynamic shaping required. + // This should emit a shape calculation based on the operation. Most + // are likely simple and by running DCE after this we can clean up + // parts that are static or unused. + op.emitOpError() << "uses unsupported dynamic shapes"; + return this->matchFailure(); + } + ArrayRef dim_pieces; + auto allocOp = rewriter.create( + op.getLoc(), memRefType, dim_pieces); + operands.push_back(allocOp); + result->replaceAllUsesWith(allocOp); + } + ArrayRef resultTypes; + rewriter.create(op.getLoc(), resultTypes, operands, op.getAttrs()); + op.erase(); + return this->matchSuccess(); + } +}; + +} // namespace + +class LowerInterpreterDialectPass + : public FunctionPass { + public: + void runOnFunction() override { + OwningRewritePatternList patterns; + patterns.insert( + &getContext()); + patterns.insert< + SimpleOpLowering, + SimpleOpLowering, + SimpleOpLowering>(&getContext()); +#define SAME_NAME_SIMPLE_PATTERN(op_name) \ + SimpleOpLowering + // clang-format off + patterns.insert< + SAME_NAME_SIMPLE_PATTERN(AssignOp), + SAME_NAME_SIMPLE_PATTERN(AbsFOp), + SAME_NAME_SIMPLE_PATTERN(AbsIOp), + SAME_NAME_SIMPLE_PATTERN(AddFOp), + SAME_NAME_SIMPLE_PATTERN(AddIOp), + SAME_NAME_SIMPLE_PATTERN(AllocHeapOp), + SAME_NAME_SIMPLE_PATTERN(AndOp), + SAME_NAME_SIMPLE_PATTERN(Atan2FOp), + SAME_NAME_SIMPLE_PATTERN(BreakOp), + SAME_NAME_SIMPLE_PATTERN(BroadcastOp), + SAME_NAME_SIMPLE_PATTERN(CallOp), + SAME_NAME_SIMPLE_PATTERN(CallIndirectOp), + SAME_NAME_SIMPLE_PATTERN(CeilFOp), + SAME_NAME_SIMPLE_PATTERN(ClampFOp), + SAME_NAME_SIMPLE_PATTERN(CloneOp), + SAME_NAME_SIMPLE_PATTERN(CmpFOp), + SAME_NAME_SIMPLE_PATTERN(CmpIOp), + SAME_NAME_SIMPLE_PATTERN(CondAssignOp), + SAME_NAME_SIMPLE_PATTERN(ConvertSSOp), + SAME_NAME_SIMPLE_PATTERN(ConvertUUOp), + SAME_NAME_SIMPLE_PATTERN(ConvertSUOp), + SAME_NAME_SIMPLE_PATTERN(ConvertUSOp), + SAME_NAME_SIMPLE_PATTERN(CondBreakOp), + SAME_NAME_SIMPLE_PATTERN(CosFOp), + SAME_NAME_SIMPLE_PATTERN(DimOp), + SAME_NAME_SIMPLE_PATTERN(DivFOp), + SAME_NAME_SIMPLE_PATTERN(DivISOp), + SAME_NAME_SIMPLE_PATTERN(DivIUOp), + SAME_NAME_SIMPLE_PATTERN(ExpFOp), + SAME_NAME_SIMPLE_PATTERN(LogFOp), + SAME_NAME_SIMPLE_PATTERN(RsqrtFOp), + SAME_NAME_SIMPLE_PATTERN(FloorFOp), + SAME_NAME_SIMPLE_PATTERN(LengthOp), + SAME_NAME_SIMPLE_PATTERN(MatMulFOp), + SAME_NAME_SIMPLE_PATTERN(MatMulIOp), + SAME_NAME_SIMPLE_PATTERN(MaxFOp), + SAME_NAME_SIMPLE_PATTERN(MaxISOp), + SAME_NAME_SIMPLE_PATTERN(MaxIUOp), + SAME_NAME_SIMPLE_PATTERN(MinFOp), + SAME_NAME_SIMPLE_PATTERN(MinISOp), + SAME_NAME_SIMPLE_PATTERN(MinIUOp), + SAME_NAME_SIMPLE_PATTERN(MulAddFOp), + SAME_NAME_SIMPLE_PATTERN(MulAddIOp), + SAME_NAME_SIMPLE_PATTERN(MulFOp), + SAME_NAME_SIMPLE_PATTERN(MulIOp), + SAME_NAME_SIMPLE_PATTERN(NotOp), + SAME_NAME_SIMPLE_PATTERN(OrOp), + SAME_NAME_SIMPLE_PATTERN(PadOp), + SAME_NAME_SIMPLE_PATTERN(RankOp), + SAME_NAME_SIMPLE_PATTERN(ReduceSumIOp), + SAME_NAME_SIMPLE_PATTERN(ReduceSumFOp), + SAME_NAME_SIMPLE_PATTERN(ReduceMinIOp), + SAME_NAME_SIMPLE_PATTERN(ReduceMinFOp), + SAME_NAME_SIMPLE_PATTERN(ReduceMaxIOp), + SAME_NAME_SIMPLE_PATTERN(ReduceMaxFOp), + SAME_NAME_SIMPLE_PATTERN(ReshapeOp), + SAME_NAME_SIMPLE_PATTERN(ReturnOp), + SAME_NAME_SIMPLE_PATTERN(SelectOp), + SAME_NAME_SIMPLE_PATTERN(ShapeOp), + SAME_NAME_SIMPLE_PATTERN(ShiftLeftOp), + SAME_NAME_SIMPLE_PATTERN(ShiftRightArithmeticOp), + SAME_NAME_SIMPLE_PATTERN(ShiftRightLogicalOp), + SAME_NAME_SIMPLE_PATTERN(SinFOp), + SAME_NAME_SIMPLE_PATTERN(SplitOp), + SAME_NAME_SIMPLE_PATTERN(SubFOp), + SAME_NAME_SIMPLE_PATTERN(SubIOp), + SAME_NAME_SIMPLE_PATTERN(TanhFOp), + SAME_NAME_SIMPLE_PATTERN(TileOp), + SAME_NAME_SIMPLE_PATTERN(TraceOp), + SAME_NAME_SIMPLE_PATTERN(TransposeOp), + SAME_NAME_SIMPLE_PATTERN(ReverseOp), + SAME_NAME_SIMPLE_PATTERN(XorOp)>(&getContext()); + // clang-format on +#undef SAME_NAME_SIMPLE_PATTERN + + ConversionTarget target(getContext()); + target.addLegalDialect(); + target.addLegalOp(); + if (failed(applyFullConversion(getFunction(), target, patterns))) { + return signalPassFailure(); + } + } +}; + +std::unique_ptr> createLowerInterpreterDialectPass() { + return std::make_unique(); +} + +static PassRegistration pass( + "lower-iree-interpreter-hl-to-ll", "Lowers IREE HL ops to IREE LL ops"); + +} // namespace iree_compiler +} // namespace mlir diff --git a/compiler/Transforms/Interpreter/LowerStdToInterpreterDialect.cpp b/compiler/Transforms/Interpreter/LowerStdToInterpreterDialect.cpp new file mode 100644 index 000000000000..cc97379c20f9 --- /dev/null +++ b/compiler/Transforms/Interpreter/LowerStdToInterpreterDialect.cpp @@ -0,0 +1,328 @@ +// Copyright 2019 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "third_party/llvm/llvm/include/llvm/ADT/DenseSet.h" +#include "third_party/llvm/llvm/include/llvm/Support/Allocator.h" +#include "third_party/llvm/llvm/projects/google_mlir/include/mlir/Dialect/StandardOps/Ops.h" +#include "third_party/llvm/llvm/projects/google_mlir/include/mlir/IR/Attributes.h" +#include "third_party/llvm/llvm/projects/google_mlir/include/mlir/IR/BlockAndValueMapping.h" +#include "third_party/llvm/llvm/projects/google_mlir/include/mlir/IR/Function.h" +#include "third_party/llvm/llvm/projects/google_mlir/include/mlir/IR/Location.h" +#include "third_party/llvm/llvm/projects/google_mlir/include/mlir/IR/MLIRContext.h" +#include "third_party/llvm/llvm/projects/google_mlir/include/mlir/IR/Module.h" +#include "third_party/llvm/llvm/projects/google_mlir/include/mlir/IR/StandardTypes.h" +#include "third_party/llvm/llvm/projects/google_mlir/include/mlir/Pass/Pass.h" +#include "third_party/llvm/llvm/projects/google_mlir/include/mlir/Pass/PassRegistry.h" +#include "third_party/llvm/llvm/projects/google_mlir/include/mlir/Transforms/DialectConversion.h" +#include "third_party/mlir_edge/iree/compiler/IR/Dialect.h" +#include "third_party/mlir_edge/iree/compiler/IR/Interpreter/HLDialect.h" +#include "third_party/mlir_edge/iree/compiler/IR/Interpreter/HLOps.h" +#include "third_party/mlir_edge/iree/compiler/IR/Interpreter/LLDialect.h" +#include "third_party/mlir_edge/iree/compiler/IR/Ops.h" +#include "third_party/mlir_edge/iree/compiler/Transforms/ConversionUtils.h" +#include "third_party/mlir_edge/iree/compiler/Utils/MemRefUtils.h" + +namespace mlir { +namespace iree_compiler { + +struct ConstantOpLowering : public ConversionPattern { + explicit ConstantOpLowering(MLIRContext *context) + : ConversionPattern(ConstantOp::getOperationName(), 1, context) {} + + PatternMatchResult matchAndRewrite( + Operation *op, ArrayRef operands, + ConversionPatternRewriter &rewriter) const override { + auto midOp = rewriter.create( + op->getLoc(), cast(op).getValue()); + + auto result = wrapAsTensor(midOp.getResult(), op, rewriter); + rewriter.replaceOp( + op, {loadResultValue(op->getLoc(), op->getResult(0)->getType(), result, + rewriter)}); + return matchSuccess(); + } +}; + +class CallOpLowering : public ConversionPattern { + public: + explicit CallOpLowering(MLIRContext *context) + : ConversionPattern(CallOp::getOperationName(), 1, context) {} + + PatternMatchResult matchAndRewrite( + Operation *op, ArrayRef operands, + ConversionPatternRewriter &rewriter) const override { + auto callOp = cast(op); + auto calleeType = callOp.getCalleeType(); + rewriter.replaceOpWithNewOp( + op, callOp.getCallee(), calleeType.getResults(), operands); + return matchSuccess(); + } +}; + +class CallIndirectOpLowering : public ConversionPattern { + public: + explicit CallIndirectOpLowering(MLIRContext *context) + : ConversionPattern(CallIndirectOp::getOperationName(), 1, context) {} + + PatternMatchResult matchAndRewrite( + Operation *op, ArrayRef operands, + ConversionPatternRewriter &rewriter) const override { + auto callOp = cast(op); + rewriter.replaceOpWithNewOp( + op, callOp.getCallee(), operands); + return matchSuccess(); + } +}; + +struct ReturnOpLowering : public ConversionPattern { + explicit ReturnOpLowering(MLIRContext *context) + : ConversionPattern(ReturnOp::getOperationName(), 1, context) {} + + PatternMatchResult matchAndRewrite( + Operation *op, ArrayRef operands, + ConversionPatternRewriter &rewriter) const override { + rewriter.replaceOpWithNewOp(op, operands); + return matchSuccess(); + } +}; + +struct BranchOpLowering : public ConversionPattern { + explicit BranchOpLowering(MLIRContext *context) + : ConversionPattern(BranchOp::getOperationName(), 1, context) {} + + PatternMatchResult matchAndRewrite( + Operation *op, ArrayRef properOperands, + ArrayRef destinations, ArrayRef> operands, + ConversionPatternRewriter &rewriter) const override { + rewriter.replaceOpWithNewOp(op, destinations[0], + operands[0]); + return this->matchSuccess(); + } +}; + +struct CondBranchOpLowering : public ConversionPattern { + explicit CondBranchOpLowering(MLIRContext *context) + : ConversionPattern(CondBranchOp::getOperationName(), 1, context) {} + + PatternMatchResult matchAndRewrite( + Operation *op, ArrayRef properOperands, + ArrayRef destinations, ArrayRef> operands, + ConversionPatternRewriter &rewriter) const override { + auto *condValue = + loadAccessValue(op->getLoc(), properOperands[0], rewriter); + rewriter.replaceOpWithNewOp( + op, condValue, destinations[IREEInterp::HL::CondBranchOp::trueIndex], + operands[IREEInterp::HL::CondBranchOp::trueIndex], + destinations[IREEInterp::HL::CondBranchOp::falseIndex], + operands[IREEInterp::HL::CondBranchOp::falseIndex]); + return this->matchSuccess(); + } +}; + +template +struct CompareOpLowering : public ConversionPattern { + explicit CompareOpLowering(MLIRContext *context) + : ConversionPattern(SrcOp::getOperationName(), 1, context) {} + + PatternMatchResult matchAndRewrite( + Operation *op, ArrayRef operands, + ConversionPatternRewriter &rewriter) const override { + auto lhValue = loadAccessValue(op->getLoc(), operands[0], rewriter); + auto rhValue = loadAccessValue(op->getLoc(), operands[1], rewriter); + + lhValue = wrapAsMemRef(lhValue, op, rewriter); + rhValue = wrapAsMemRef(rhValue, op, rewriter); + + // TODO(benvanik): map predicate to stable value. + auto predicate = rewriter.getI32IntegerAttr( + static_cast(dyn_cast(op).getPredicate())); + + auto dstType = getMemRefType(op->getResult(0), rewriter); + auto midOp = rewriter.create(op->getLoc(), dstType, predicate, + lhValue, rhValue); + + auto result = wrapAsTensor(midOp.getResult(), op, rewriter); + rewriter.replaceOp( + op, {loadResultValue(op->getLoc(), op->getResult(0)->getType(), result, + rewriter)}); + return this->matchSuccess(); + } +}; + +struct CmpIOpLowering + : public CompareOpLowering { + using CompareOpLowering::CompareOpLowering; +}; + +struct CmpFOpLowering + : public CompareOpLowering { + using CompareOpLowering::CompareOpLowering; +}; + +struct AllocOpLowering : public ConversionPattern { + explicit AllocOpLowering(MLIRContext *context) + : ConversionPattern(AllocOp::getOperationName(), 1, context) {} + + PatternMatchResult matchAndRewrite( + Operation *op, ArrayRef operands, + ConversionPatternRewriter &rewriter) const override { + // TODO(benvanik): replace with length computation. + rewriter.replaceOpWithNewOp( + op, *op->getResultTypes().begin(), operands); + return matchSuccess(); + } +}; + +struct DeallocOpLowering : public ConversionPattern { + explicit DeallocOpLowering(MLIRContext *context) + : ConversionPattern(DeallocOp::getOperationName(), 1, context) {} + + PatternMatchResult matchAndRewrite( + Operation *op, ArrayRef operands, + ConversionPatternRewriter &rewriter) const override { + rewriter.replaceOpWithNewOp(op, operands[0]); + return matchSuccess(); + } +}; + +struct ExtractElementOpLowering : public ConversionPattern { + explicit ExtractElementOpLowering(MLIRContext *context) + : ConversionPattern(ExtractElementOp::getOperationName(), 1, context) {} + PatternMatchResult matchAndRewrite( + Operation *op, ArrayRef operands, + ConversionPatternRewriter &rewriter) const override { + auto extractOp = cast(op); + Value *memRefInput = wrapAsMemRef( + loadAccessValue(op->getLoc(), extractOp.getAggregate(), rewriter), op, + rewriter); + + SmallVector indices = {extractOp.indices().begin(), + extractOp.indices().end()}; + rewriter.replaceOpWithNewOp(op, memRefInput, indices); + return matchSuccess(); + } +}; + +#define UNARY_OP_LOWERING(StdOpType, IREEOpType) \ + struct StdOpType##Lowering : public UnaryOpLowering { \ + using UnaryOpLowering::UnaryOpLowering; \ + }; + +#define BINARY_OP_LOWERING(StdOpType, IREEOpType) \ + struct StdOpType##Lowering \ + : public BinaryOpLowering { \ + using BinaryOpLowering::BinaryOpLowering; \ + }; + +#define TERNARY_OP_LOWERING(StdOpType, IREEOpType) \ + struct StdOpType##Lowering \ + : public TernaryOpLowering { \ + using TernaryOpLowering::TernaryOpLowering; \ + }; + +// UNARY_OP_LOWERING(RankOp, IREEInterp::HL::RankOp); +UNARY_OP_LOWERING(DimOp, IREEInterp::HL::DimOp); +// UNARY_OP_LOWERING(ShapeOp, IREEInterp::HL::ShapeOp); +// UNARY_OP_LOWERING(LengthOp, IREEInterp::HL::LengthOp); + +// UNARY_OP_LOWERING(NotOp, IREEInterp::HL::NotOp); +BINARY_OP_LOWERING(AndOp, IREEInterp::HL::AndOp); +BINARY_OP_LOWERING(OrOp, IREEInterp::HL::OrOp); +// BINARY_OP_LOWERING(XorOp, IREEInterp::HL::XorOp); +// BINARY_OP_LOWERING(ShiftLeftOp, IREEInterp::HL::ShiftLeftOp); +// BINARY_OP_LOWERING(ShiftRightLogicalOp, IREEInterp::HL::ShiftRightLogicalOp); +// BINARY_OP_LOWERING(ShiftRightArithmeticOp, +// IREEInterp::HL::ShiftRightArithmeticOp); + +BINARY_OP_LOWERING(AddIOp, IREEInterp::HL::AddIOp); +BINARY_OP_LOWERING(AddFOp, IREEInterp::HL::AddFOp); +BINARY_OP_LOWERING(SubIOp, IREEInterp::HL::SubIOp); +BINARY_OP_LOWERING(SubFOp, IREEInterp::HL::SubFOp); +// UNARY_OP_LOWERING(AbsIOp, IREEInterp::HL::AbsIOp); +// UNARY_OP_LOWERING(AbsFOp, IREEInterp::HL::AbsFOp); +BINARY_OP_LOWERING(MulIOp, IREEInterp::HL::MulIOp); +BINARY_OP_LOWERING(MulFOp, IREEInterp::HL::MulFOp); +BINARY_OP_LOWERING(DivISOp, IREEInterp::HL::DivISOp); +BINARY_OP_LOWERING(DivIUOp, IREEInterp::HL::DivIUOp); +BINARY_OP_LOWERING(DivFOp, IREEInterp::HL::DivFOp); +// BINARY_OP_LOWERING(MulAddIOp, IREEInterp::HL::MulAddIOp); +// BINARY_OP_LOWERING(MulAddFOp, IREEInterp::HL::MulAddFOp); +// UNARY_OP_LOWERING(ExpFOp, IREEInterp::HL::ExpFOp); +// UNARY_OP_LOWERING(LogFOp, IREEInterp::HL::LogFOp); +// UNARY_OP_LOWERING(RsqrtFOp, IREEInterp::HL::RsqrtFOp); +// UNARY_OP_LOWERING(CosFOp, IREEInterp::HL::CosFOp); +// UNARY_OP_LOWERING(SinFOp, IREEInterp::HL::SinFOp); +// UNARY_OP_LOWERING(TanhFOp, IREEInterp::HL::TanhFOp); +// UNARY_OP_LOWERING(Atan2FOp, IREEInterp::HL::Atan2FOp); + +// BINARY_OP_LOWERING(MinISOp, IREEInterp::HL::MinISOp); +// BINARY_OP_LOWERING(MinIUOp, IREEInterp::HL::MinIUOp); +// BINARY_OP_LOWERING(MinFOp, IREEInterp::HL::MinFOp); +// BINARY_OP_LOWERING(MaxISOp, IREEInterp::HL::MaxISOp); +// BINARY_OP_LOWERING(MaxIUOp, IREEInterp::HL::MaxIUOp); +// BINARY_OP_LOWERING(MaxFOp, IREEInterp::HL::MaxFOp); +// TERNARY_OP_LOWERING(ClampFOp, IREEInterp::HL::ClampFOp); +// UNARY_OP_LOWERING(FloorFOp, IREEInterp::HL::FloorFOp); +// UNARY_OP_LOWERING(CeilFOp, IREEInterp::HL::CeilFOp); + +class LowerStdToInterpreterDialectPass + : public FunctionPass { + public: + void runOnFunction() override { + OwningRewritePatternList patterns; + patterns.insert< + ConstantOpLowering, + // Control flow. + CallOpLowering, CallIndirectOpLowering, ReturnOpLowering, + BranchOpLowering, CondBranchOpLowering, CmpIOpLowering, CmpFOpLowering, + // Memory management. + AllocOpLowering, DeallocOpLowering, ExtractElementOpLowering, + // Shape operations. + DimOpLowering, + // Logical ops. + AndOpLowering, OrOpLowering, + // Arithmetic ops. + AddIOpLowering, AddFOpLowering, SubIOpLowering, SubFOpLowering, + MulIOpLowering, MulFOpLowering, DivISOpLowering, DivIUOpLowering, + DivFOpLowering>(&getContext()); + + ConversionTarget target(getContext()); + target.addLegalDialect(); + target.addLegalOp(); + target.addDynamicallyLegalOp([](ConstantOp constOp) { + // std.constant is legal for index integers. + return constOp.getValue().isa() && + constOp.getType().isIndex(); + }); + // This is only able to be a full conversion because we know that we do it + // last. Other conversions (e.g. XLA->IREE) can't be full because we might + // still have e.g. std ops. + // TODO(b/139012931) Move fully to the conversion framework. + if (failed(applyFullConversion(getFunction(), target, patterns))) { + return signalPassFailure(); + } + } +}; + +std::unique_ptr> createLowerStdToInterpreterDialectPass() { + return std::make_unique(); +} + +static PassRegistration pass( + "lower-std-to-iree-interpreter", + "Convert all std functions to the IREE dialect"); + +} // namespace iree_compiler +} // namespace mlir diff --git a/compiler/Transforms/Interpreter/LowerXLAToInterpreterDialect.cpp b/compiler/Transforms/Interpreter/LowerXLAToInterpreterDialect.cpp new file mode 100644 index 000000000000..90ff7d6a7cbf --- /dev/null +++ b/compiler/Transforms/Interpreter/LowerXLAToInterpreterDialect.cpp @@ -0,0 +1,591 @@ +// Copyright 2019 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "third_party/llvm/llvm/include/llvm/ADT/ArrayRef.h" +#include "third_party/llvm/llvm/include/llvm/ADT/STLExtras.h" +#include "third_party/llvm/llvm/include/llvm/ADT/SmallVector.h" +#include "third_party/llvm/llvm/include/llvm/ADT/StringRef.h" +#include "third_party/llvm/llvm/projects/google_mlir/include/mlir/IR/Attributes.h" +#include "third_party/llvm/llvm/projects/google_mlir/include/mlir/IR/Builders.h" +#include "third_party/llvm/llvm/projects/google_mlir/include/mlir/IR/Function.h" +#include "third_party/llvm/llvm/projects/google_mlir/include/mlir/IR/Operation.h" +#include "third_party/llvm/llvm/projects/google_mlir/include/mlir/IR/PatternMatch.h" +#include "third_party/llvm/llvm/projects/google_mlir/include/mlir/IR/StandardTypes.h" +#include "third_party/llvm/llvm/projects/google_mlir/include/mlir/IR/TypeUtilities.h" +#include "third_party/llvm/llvm/projects/google_mlir/include/mlir/IR/Value.h" +#include "third_party/llvm/llvm/projects/google_mlir/include/mlir/Pass/Pass.h" +#include "third_party/llvm/llvm/projects/google_mlir/include/mlir/Transforms/DialectConversion.h" +#include "third_party/mlir_edge/iree/compiler/IR/Dialect.h" +#include "third_party/mlir_edge/iree/compiler/IR/Interpreter/HLDialect.h" +#include "third_party/mlir_edge/iree/compiler/IR/Interpreter/HLOps.h" +#include "third_party/mlir_edge/iree/compiler/IR/Ops.h" +#include "third_party/mlir_edge/iree/compiler/Transforms/ConversionUtils.h" +#include "third_party/mlir_edge/iree/compiler/Utils/MemRefUtils.h" +#include "third_party/tensorflow/compiler/mlir/xla/ir/hlo_ops.h" + +namespace mlir { +namespace iree_compiler { + +namespace { + +// TODO(suderman): tablegen this? or something a bit more flexible. + +#define UNARY_OP_LOWERING(XlaOpType, IREEOpType) \ + struct XlaOpType##Lowering \ + : public UnaryOpLowering { \ + using UnaryOpLowering::UnaryOpLowering; \ + }; + +#define TERNARY_OP_LOWERING(XlaOpType, IREEOpType) \ + struct XlaOpType##Lowering \ + : public TernaryOpLowering { \ + using TernaryOpLowering::TernaryOpLowering; \ + }; + +UNARY_OP_LOWERING(CopyOp, IREEInterp::HL::CloneOp); +UNARY_OP_LOWERING(ExpOp, IREEInterp::HL::ExpFOp); +UNARY_OP_LOWERING(LogOp, IREEInterp::HL::LogFOp); +UNARY_OP_LOWERING(FloorOp, IREEInterp::HL::FloorFOp); +UNARY_OP_LOWERING(RsqrtOp, IREEInterp::HL::RsqrtFOp); +UNARY_OP_LOWERING(TanhOp, IREEInterp::HL::TanhFOp); +TERNARY_OP_LOWERING(SelectOp, IREEInterp::HL::SelectOp); + +#undef UNARY_OP_LOWERING +#undef TERNARY_OP_LOWERING + +static ElementsAttr elementsAttrFromArray(ConversionPatternRewriter &rewriter, + ArrayRef elements) { + return rewriter.getDenseIntElementsAttr( + rewriter.getTensorType(elements.size(), rewriter.getIntegerType(64)), + elements); +} + +static IREE::ConstantOp createArrayConstant(ConversionPatternRewriter &rewriter, + Location loc, + llvm::ArrayRef elements) { + auto shapeAttr = elementsAttrFromArray(rewriter, elements); + return rewriter.create(loc, shapeAttr); +} + +template +static Operation *createShapeTargetingOp(ConversionPatternRewriter &rewriter, + Location loc, Value *input, + MemRefType targetType) { + auto shapeOp = createArrayConstant(rewriter, loc, targetType.getShape()); + return rewriter.create(loc, targetType, input, shapeOp); +} + +static Value *inputAsMemref(ConversionPatternRewriter &rewriter, Operation *op, + Value *tensor) { + return wrapAsMemRef(loadAccessValue(op->getLoc(), tensor, rewriter), op, + rewriter); +} + +static MemRefType getFinalType(ConversionPatternRewriter &rewriter, + Value *result) { + return getMemRefType(result, rewriter).cast(); +} + +template +class XlaOpLowering : public ConversionPattern { + public: + explicit XlaOpLowering(MLIRContext *context) + : ConversionPattern(SrcOp::getOperationName(), 1, context) {} + + PatternMatchResult matchAndRewrite( + Operation *op, ArrayRef operands, + ConversionPatternRewriter &rewriter) const override { + auto srcOp = cast(op); + + SmallVector memrefOperands; + for (auto operand : operands) { + memrefOperands.push_back(inputAsMemref(rewriter, op, operand)); + } + + if (auto dstOp = rewriteInternal(&srcOp, memrefOperands, rewriter)) { + rewriter.replaceOp(op, + wrapAsTensor(dstOp->getResult(0), srcOp, rewriter)); + return this->matchSuccess(); + } + return this->matchFailure(); + } + + protected: + virtual Operation *rewriteInternal( + SrcOp *op, ArrayRef operands, + ConversionPatternRewriter &rewriter) const { + llvm_unreachable("unimplemented rewrite, did you mean rewriteTerminator?"); + } +}; + +struct BroadcastInDimOpLowering + : public XlaOpLowering { + using XlaOpLowering::XlaOpLowering; + + Operation *rewriteInternal( + xla_hlo::BroadcastInDimOp *op, ArrayRef operands, + ConversionPatternRewriter &rewriter) const override { + auto *inputValue = operands[0]; + auto inputType = inputValue->getType().cast(); + auto finalType = getFinalType(rewriter, *op); + + // Reshape to scalar and broadcast. + auto createFinal = createShapeTargetingOp; + llvm::SmallVector intermediateShape{}; + + // Or reshape to final rank and tile. + if (getElementCount(inputType) != 1) { + createFinal = createShapeTargetingOp; + + intermediateShape = llvm::SmallVector(finalType.getRank(), 1); + auto inputShape = inputType.getShape(); + auto dimensions = op->broadcast_dimensions(); + for (size_t i = 0; i < inputType.getRank(); ++i) { + auto index = dimensions->getValue(i).cast().getInt(); + intermediateShape[index] = inputShape[i]; + } + } + + auto intermediateType = + rewriter.getMemRefType(intermediateShape, inputType.getElementType()); + auto reshapeOp = createShapeTargetingOp( + rewriter, op->getLoc(), inputValue, intermediateType); + return createFinal(rewriter, op->getLoc(), reshapeOp->getResult(0), + finalType); + } +}; + +struct ConcatOpLowering : public XlaOpLowering { + using XlaOpLowering::XlaOpLowering; + + Operation *rewriteInternal( + xla_hlo::ConcatenateOp *op, ArrayRef operands, + ConversionPatternRewriter &rewriter) const override { + auto finalType = getFinalType(rewriter, *op); + + return rewriter.create( + op->getLoc(), finalType, operands, + rewriter.getI32IntegerAttr(op->dimension().getZExtValue())); + } +}; + +struct ConstOpLowering : public XlaOpLowering { + using XlaOpLowering::XlaOpLowering; + + Operation *rewriteInternal( + xla_hlo::ConstOp *op, ArrayRef operands, + ConversionPatternRewriter &rewriter) const override { + return rewriter.create(op->getLoc(), op->value()); + } +}; + +struct DotOpLowering : public XlaOpLowering { + using XlaOpLowering::XlaOpLowering; + + Operation *rewriteInternal( + xla_hlo::DotOp *op, ArrayRef operands, + ConversionPatternRewriter &rewriter) const override { + auto *lhsValue = operands[0]; + auto *rhsValue = operands[1]; + + auto finalType = getFinalType(rewriter, *op); + auto elementType = finalType.getElementType(); + if (!elementType.isa()) { + op->emitOpError("xla_hlo.dot only supports floating point values"); + } + + Operation *matMulOp = rewriter + .create( + op->getLoc(), finalType, lhsValue, rhsValue) + .getOperation(); + return matMulOp; + } +}; + +struct DynamicUpdateSliceOpLowering + : public XlaOpLowering { + using XlaOpLowering::XlaOpLowering; + + Operation *rewriteInternal( + xla_hlo::DynamicUpdateSliceOp *op, ArrayRef operands, + ConversionPatternRewriter &rewriter) const override { + auto operand = operands[0]; + auto update = operands[1]; + + auto updateType = update->getType().cast(); + Value *lengthConstant = + createArrayConstant(rewriter, op->getLoc(), updateType.getShape()); + + auto startIndices = makeArrayRef(operands).drop_front(2); + const int rank = startIndices.size(); + llvm::SmallVector valuesToConcat; + valuesToConcat.reserve(startIndices.size()); + auto type = getElementTypeOrSelf(startIndices.front()); + + // To generate the offset matrix we need to convert the variadic tensors + // into a reshaped and concated value. + for (auto index : startIndices) { + auto reshapedIndex = rewriter.create( + op->getLoc(), rewriter.getMemRefType({1}, type), index, + createArrayConstant(rewriter, op->getLoc(), {1})); + valuesToConcat.push_back(reshapedIndex); + } + + auto dstOffset = rewriter + .create( + op->getLoc(), rewriter.getMemRefType({rank}, type), + valuesToConcat, rewriter.getI32IntegerAttr(0)) + .getResult(); + + llvm::SmallVector zero_offset; + zero_offset.resize(updateType.getRank(), 0); + auto srcOffset = createArrayConstant(rewriter, op->getLoc(), zero_offset); + + auto copiedOperand = rewriter.create( + op->getLoc(), operand->getType(), operand); + + rewriter + .create(op->getLoc(), update, srcOffset, + copiedOperand, dstOffset, + lengthConstant) + .getOperation(); + + return copiedOperand; + } +}; + +template +struct BinaryFloatIntOpLowering : public XlaOpLowering { + using XlaOpLowering::XlaOpLowering; + + Operation *rewriteInternal( + XlaOpType *op, ArrayRef operands, + ConversionPatternRewriter &rewriter) const override { + auto *lhs = operands[0]; + auto *rhs = operands[1]; + auto inputType = lhs->getType().cast(); + auto elementType = inputType.getElementType(); + + if (elementType.isa()) { + return rewriter.create(op->getLoc(), inputType, lhs, + rhs); + } + + return rewriter.create(op->getLoc(), inputType, lhs, rhs); + } +}; + +struct MaxOpLowering + : public BinaryFloatIntOpLowering { + using BinaryFloatIntOpLowering::BinaryFloatIntOpLowering; +}; + +struct MinOpLowering + : public BinaryFloatIntOpLowering { + using BinaryFloatIntOpLowering::BinaryFloatIntOpLowering; +}; + +struct ConvertLowering : public XlaOpLowering { + using XlaOpLowering::XlaOpLowering; + + Operation *rewriteInternal( + xla_hlo::ConvertOp *op, ArrayRef operands, + ConversionPatternRewriter &rewriter) const override { + auto *operand = operands[0]; + auto *result = op->getResult(); + + auto operandType = operand->getType().cast().getElementType(); + auto resultType = result->getType().cast().getElementType(); + + auto newResultType = getMemRefType(result, rewriter); + +#define ConvertCase(InType, OutType, NewOp) \ + { \ + if (operandType.isa() && resultType.isa()) { \ + return rewriter.create(op->getLoc(), newResultType, operand); \ + } \ + } + ConvertCase(IntegerType, IntegerType, IREEInterp::HL::ConvertSSOp); + ConvertCase(IntegerType, FloatType, IREEInterp::HL::ConvertSFOp); + ConvertCase(FloatType, IntegerType, IREEInterp::HL::ConvertFSOp); + ConvertCase(FloatType, FloatType, IREEInterp::HL::ConvertFFOp); +#undef ConvertCase + + return nullptr; + } +}; + +// Lowers a subset of gathers along axis 0 that are really just a slice and +// reshape. +struct GatherOpLowering : public ConversionPattern { + public: + explicit GatherOpLowering(MLIRContext *context) + : ConversionPattern(xla_hlo::GatherOp::getOperationName(), 1, context) {} + + // TODO(gcmn): This is a pile of hacks. When XLA redefines gather to be + // simpler, lower it properly. + PatternMatchResult matchAndRewrite( + Operation *op, ArrayRef operands, + ConversionPatternRewriter &rewriter) const override { + auto gatherOp = cast(op); + + if (gatherOp.index_vector_dim() != 0) { + op->emitRemark() << "Couldn't lower gather with index_vector_dim != 0"; + return matchFailure(); + } + if (gatherOp.start_index_map().getType().getRank() != 1 || + gatherOp.start_index_map().getValue(0).cast().getValue() != + 0) { + op->emitRemark() << "Couldn't lower gather with start_index_map != [0]"; + return matchFailure(); + } + if (gatherOp.collapsed_slice_dims().getType().getRank() != 1 || + gatherOp.collapsed_slice_dims() + .getValue(0) + .cast() + .getValue() != 0) { + op->emitRemark() << "Couldn't lower gather with collapsed_dims != [0]"; + return matchFailure(); + } + + auto resultType = gatherOp.getResult()->getType().cast(); + if (gatherOp.offset_dims().getType().getNumElements() != + resultType.getRank()) { + op->emitRemark() << "Couldn't lower gather with offset_dims != " + "[0,...,rank of output]"; + return matchFailure(); + } + for (auto it : llvm::enumerate(gatherOp.offset_dims())) { + if (it.index() != it.value()) { + op->emitRemark() << "Couldn't lower gather with offset_dims != " + "[0,...,rank of output]"; + return matchFailure(); + } + } + + for (auto it : llvm::enumerate(resultType.getShape())) { + if (gatherOp.slice_sizes() + .getValue(it.index() + 1) + .cast() + .getValue() != it.value()) { + op->emitRemark() + << "Couldn't lower gather with slice_sizes not [1] + final shape"; + return matchFailure(); + } + } + + auto inputType = gatherOp.operand()->getType().cast(); + + auto startIndices = inputAsMemref(rewriter, op, gatherOp.start_indices()); + auto startIndicesType = startIndices->getType().cast(); + if (startIndicesType.getNumElements() != inputType.getRank()) { + auto extraDims = inputType.getRank() - startIndicesType.getNumElements(); + auto elementType = startIndicesType.getElementType(); + + if (startIndicesType.getRank() != 1) { + startIndices = createShapeTargetingOp( + rewriter, op->getLoc(), startIndices, + rewriter.getMemRefType({1}, elementType)) + ->getResult(0); + } + + llvm::SmallVector zeroes; + zeroes.resize(extraDims, 0); + + auto elementsAttr = rewriter.getDenseIntElementsAttr( + rewriter.getTensorType(zeroes.size(), elementType), zeroes); + + auto extraStartIndices = + rewriter.create(op->getLoc(), elementsAttr); + + auto memrefOutputType = + rewriter.getMemRefType({inputType.getRank()}, elementType); + + SmallVector valuesToConcat = {startIndices, + extraStartIndices}; + startIndices = rewriter.create( + op->getLoc(), memrefOutputType, valuesToConcat, + rewriter.getI32IntegerAttr(0)); + } + + auto sliceSizeValues = gatherOp.slice_sizes().getValues(); + std::vector sliceSizes = {sliceSizeValues.begin(), + sliceSizeValues.end()}; + auto dstType = + rewriter.getMemRefType(sliceSizes, inputType.getElementType()); + + auto src = inputAsMemref(rewriter, op, gatherOp.operand()); + std::vector dim_pieces; + auto dst = rewriter.create( + op->getLoc(), dstType, dim_pieces); + auto lengths = + rewriter.create(op->getLoc(), gatherOp.slice_sizes()); + llvm::SmallVector zero_offset; + zero_offset.resize(dstType.getRank(), 0); + auto dstIndices = createArrayConstant(rewriter, op->getLoc(), zero_offset); + + rewriter.create(op->getLoc(), src, startIndices, + dst, dstIndices, lengths); + + auto reshaped = createShapeTargetingOp( + rewriter, op->getLoc(), dst, getFinalType(rewriter, gatherOp)); + rewriter.replaceOp( + op, wrapAsTensor(reshaped->getResult(0), gatherOp, rewriter)); + + return matchSuccess(); + } +}; + +struct SliceOpLowering : public XlaOpLowering { + using XlaOpLowering::XlaOpLowering; + + Operation *rewriteInternal( + xla_hlo::SliceOp *op, ArrayRef operands, + ConversionPatternRewriter &rewriter) const override { + // XLA slice has value semantics, whereas the IREE slice creates a view. We + // lower it to a copy if all strides are one which may be transformed to a + // slice by later optimizations. + auto isNotOne = [](APInt stride) { return stride != 1; }; + if (llvm::any_of(op->strides(), isNotOne)) { + op->emitRemark() << "Could not lower slice op with non-singular strides"; + return nullptr; + } + + auto finalType = getFinalType(rewriter, *op); + auto src = operands[0]; + std::vector dim_pieces; + auto dst = rewriter.create( + op->getLoc(), finalType, dim_pieces); + auto srcIndices = + rewriter.create(op->getLoc(), op->start_indices()); + auto lengths = + createArrayConstant(rewriter, op->getLoc(), finalType.getShape()); + + llvm::SmallVector zero_offset; + zero_offset.resize(finalType.getRank(), 0); + auto dstIndices = createArrayConstant(rewriter, op->getLoc(), zero_offset); + + rewriter.create(op->getLoc(), src, srcIndices, dst, + dstIndices, lengths); + return dst; + } +}; + +struct PadOpLowering : public XlaOpLowering { + using XlaOpLowering::XlaOpLowering; + + Operation *rewriteInternal( + xla_hlo::PadOp *op, ArrayRef operands, + ConversionPatternRewriter &rewriter) const override { + auto *src = operands[0]; + auto *paddingValue = operands[1]; + + // TODO(b/140836672) Support negative padding + for (int i = 0; i < op->edge_padding_high().getNumElements(); ++i) { + if (op->edge_padding_high().getValue(i).getInt() < 0 || + op->edge_padding_low().getValue(i).getInt() < 0) { + op->emitRemark() << "Could not lower pad op with negative padding"; + return nullptr; + } + } + + auto edgePaddingLowOp = + rewriter.create(op->getLoc(), op->edge_padding_low()); + auto edgePaddingHighOp = rewriter.create( + op->getLoc(), op->edge_padding_high()); + auto interiorPaddingOp = + rewriter.create(op->getLoc(), op->interior_padding()); + + return rewriter.create( + op->getLoc(), getFinalType(rewriter, *op), src, paddingValue, + edgePaddingLowOp, edgePaddingHighOp, interiorPaddingOp); + } +}; + +struct ReshapeOpLowering : public XlaOpLowering { + using XlaOpLowering::XlaOpLowering; + + Operation *rewriteInternal( + xla_hlo::ReshapeOp *op, ArrayRef operands, + ConversionPatternRewriter &rewriter) const override { + return createShapeTargetingOp( + rewriter, op->getLoc(), operands[0], getFinalType(rewriter, *op)); + } +}; + +struct TransposeOpLowering : public XlaOpLowering { + using XlaOpLowering::XlaOpLowering; + + Operation *rewriteInternal( + xla_hlo::TransposeOp *op, ArrayRef operands, + ConversionPatternRewriter &rewriter) const override { + auto permutationOp = + rewriter.create(op->getLoc(), op->permutation()); + + return rewriter.create( + op->getLoc(), getFinalType(rewriter, *op), operands[0], permutationOp); + } +}; + +struct ReverseOpLowering : public XlaOpLowering { + using XlaOpLowering::XlaOpLowering; + + Operation *rewriteInternal( + xla_hlo::ReverseOp *op, ArrayRef operands, + ConversionPatternRewriter &rewriter) const override { + auto reverseOp = + rewriter.create(op->getLoc(), op->dimensions()); + + return rewriter.create( + op->getLoc(), getFinalType(rewriter, *op), operands[0], reverseOp); + } +}; + +class LowerXLAToInterpreterDialectPass + : public FunctionPass { + public: + void runOnFunction() override { + OwningRewritePatternList patterns; + patterns + .insert(&getContext()); + + ConversionTarget target(getContext()); + target.addLegalDialect(); + target.addLegalOp(); + if (failed(applyPartialConversion(getFunction(), target, patterns))) { + return signalPassFailure(); + } + } +}; + +} // namespace + +std::unique_ptr> createLowerXLAToInterpreterDialectPass() { + return std::make_unique(); +} + +static PassRegistration pass( + "lower-xla-to-iree-interpreter", + "Convert all XLA functions to the IREE dialect"); + +} // namespace iree_compiler +} // namespace mlir diff --git a/compiler/Transforms/Interpreter/MakeExecutableABI.cpp b/compiler/Transforms/Interpreter/MakeExecutableABI.cpp new file mode 100644 index 000000000000..142bc19fae15 --- /dev/null +++ b/compiler/Transforms/Interpreter/MakeExecutableABI.cpp @@ -0,0 +1,165 @@ +// Copyright 2019 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "third_party/llvm/llvm/projects/google_mlir/include/mlir/Dialect/StandardOps/Ops.h" +#include "third_party/llvm/llvm/projects/google_mlir/include/mlir/IR/Attributes.h" +#include "third_party/llvm/llvm/projects/google_mlir/include/mlir/IR/BlockAndValueMapping.h" +#include "third_party/llvm/llvm/projects/google_mlir/include/mlir/IR/Builders.h" +#include "third_party/llvm/llvm/projects/google_mlir/include/mlir/IR/Location.h" +#include "third_party/llvm/llvm/projects/google_mlir/include/mlir/IR/MLIRContext.h" +#include "third_party/llvm/llvm/projects/google_mlir/include/mlir/IR/StandardTypes.h" +#include "third_party/llvm/llvm/projects/google_mlir/include/mlir/Pass/Pass.h" +#include "third_party/llvm/llvm/projects/google_mlir/include/mlir/Pass/PassRegistry.h" +#include "third_party/llvm/llvm/projects/google_mlir/include/mlir/Support/LogicalResult.h" +#include "third_party/mlir_edge/iree/compiler/IR/Interpreter/HLOps.h" +#include "third_party/mlir_edge/iree/compiler/IR/Ops.h" +#include "third_party/mlir_edge/iree/compiler/Utils/OpUtils.h" + +namespace mlir { +namespace iree_compiler { + +namespace { + +// Replaces a load_input op with valid IR that loads the input value. +LogicalResult replaceLoadInputOp(IREE::LoadInputOp bindOp) { + OpBuilder builder(bindOp); + + Value *newValue = nullptr; + auto dstType = bindOp.getResult()->getType(); + if (dstType.isa()) { + auto castOp = builder.create(bindOp.getLoc(), + dstType, bindOp.src()); + newValue = castOp.getResult(); + } else if (dstType.isIntOrIndexOrFloat()) { + auto loadOp = builder.create(bindOp.getLoc(), dstType, bindOp.src(), + ArrayRef{}); + newValue = loadOp.getResult(); + } else { + return bindOp.emitError() + << "Unsupported input destination type " << dstType; + } + + bindOp.replaceAllUsesWith(newValue); + bindOp.erase(); + + return success(); +} + +// Replaces a store_output op with valid IR that stores the output value. +LogicalResult replaceStoreOutputOp(IREE::StoreOutputOp bindOp) { + OpBuilder builder(bindOp); + + auto srcType = bindOp.src()->getType(); + if (srcType.isa()) { + // Already stored into the output. + } else if (srcType.isa()) { + auto castOp = builder.create( + bindOp.getLoc(), bindOp.dst()->getType(), bindOp.src()); + + // Insert a copy to our output parameter. + auto dst = bindOp.dst()->getType().cast(); + if (!dst.hasStaticShape()) { + return bindOp.emitError() + << "Dynamic output args are not yet implemented"; + } + + // TODO(b/134586626): decide if we want copy indices or byte offsets and + // support 0-rank natively. + int rank = dst.getRank() ? dst.getRank() : 1; + auto zeroValues = std::vector(rank); + auto shapeValues = std::vector(rank); + if (dst.getRank() > 0) { + for (int i = 0; i < dst.getRank(); ++i) { + shapeValues[i] = static_cast(dst.getDimSize(i)); + } + } else { + shapeValues[0] = 1; + } + auto zeros = builder.create( + bindOp.getLoc(), + DenseIntElementsAttr::get( + builder.getTensorType({rank}, builder.getIntegerType(32)), + zeroValues)); + auto lengths = builder.create( + bindOp.getLoc(), + DenseIntElementsAttr::get( + builder.getTensorType({rank}, builder.getIntegerType(32)), + shapeValues)); + + builder.create(bindOp.getLoc(), castOp.getResult(), + zeros, bindOp.dst(), zeros, lengths); + } else if (srcType.isIntOrIndexOrFloat()) { + builder.create(bindOp.getLoc(), bindOp.src(), bindOp.dst(), + ArrayRef{}); + } else { + return bindOp.emitError() << "Unsupported output src type " << srcType; + } + + bindOp.erase(); + + return success(); +} + +// Strips iree.bind_* ops from |func|. +LogicalResult stripBindingOps(FuncOp func) { + // Find iree.load_input ops to replace with memref_to_tensor if needed. + SmallVector bindInputOps; + func.walk([&](IREE::LoadInputOp bindOp) { bindInputOps.push_back(bindOp); }); + for (auto &bindOp : bindInputOps) { + if (failed(replaceLoadInputOp(bindOp))) { + return failure(); + } + } + + // Find iree.store_output ops and replace with tensor_to_memref if needed. + SmallVector bindOutputOps; + func.walk( + [&](IREE::StoreOutputOp bindOp) { bindOutputOps.push_back(bindOp); }); + for (auto &bindOp : bindOutputOps) { + if (failed(replaceStoreOutputOp(bindOp))) { + return failure(); + } + } + + return success(); +} + +} // namespace + +// Finds iree.executable.export functions and fixes up bindings. +// For the interpreter this really just means stripping the bind ops entirely. +class MakeExecutableABIPass : public ModulePass { + public: + void runOnModule() override { + auto module = getModule(); + for (auto func : module.getOps()) { + if (func.getAttr("iree.executable.export")) { + if (failed(stripBindingOps(func))) { + return signalPassFailure(); + } + } + } + } +}; + +std::unique_ptr> createMakeExecutableABIPass() { + return std::make_unique(); +} + +static PassRegistration pass( + "iree-make-executable-abi", + "Makes functions match the IREE dispatch executable ABI."); + +} // namespace iree_compiler +} // namespace mlir diff --git a/compiler/Transforms/Interpreter/Passes.h b/compiler/Transforms/Interpreter/Passes.h new file mode 100644 index 000000000000..47f80ff58280 --- /dev/null +++ b/compiler/Transforms/Interpreter/Passes.h @@ -0,0 +1,47 @@ +// Copyright 2019 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#ifndef THIRD_PARTY_MLIR_EDGE_IREE_COMPILER_TRANSFORMS_INTERPRETER_PASSES_H_ +#define THIRD_PARTY_MLIR_EDGE_IREE_COMPILER_TRANSFORMS_INTERPRETER_PASSES_H_ + +#include "third_party/llvm/llvm/projects/google_mlir/include/mlir/Pass/Pass.h" + +namespace mlir { +namespace iree_compiler { + +// Expands reduction functions to their interpreter ops. +std::unique_ptr> createExpandReductionsToOpsPass(); + +// Refactors entry points to match the IREE dispatch executable ABI. +std::unique_ptr> createMakeExecutableABIPass(); + +// Lowers Standard dialect (std.*) ops to IREE Interpreter HL ops. +std::unique_ptr> createLowerStdToInterpreterDialectPass(); + +// Lowers XLA dialect (xla_hlo.*) ops to IREE Interpreter HL ops. +std::unique_ptr> createLowerXLAToInterpreterDialectPass(); + +// Lowers IREE HL ops (iree_hl_interp.*) to LL ops (iree_ll_interp.*). +std::unique_ptr> createLowerInterpreterDialectPass(); + +// Optimizes std.load and std.store to remove unnessisary copies. +std::unique_ptr> createInterpreterLoadStoreDataFlowOptPass(); + +// Legalizes all ops to canonical/supported forms. +std::unique_ptr> createLegalizeInterpreterOpsPass(); + +} // namespace iree_compiler +} // namespace mlir + +#endif // THIRD_PARTY_MLIR_EDGE_IREE_COMPILER_TRANSFORMS_INTERPRETER_PASSES_H_ diff --git a/compiler/Transforms/Interpreter/test/load_store_data_flow_opt.mlir b/compiler/Transforms/Interpreter/test/load_store_data_flow_opt.mlir new file mode 100644 index 000000000000..0af5a1f6910d --- /dev/null +++ b/compiler/Transforms/Interpreter/test/load_store_data_flow_opt.mlir @@ -0,0 +1,127 @@ +// RUN: iree-opt %s -iree-interpreter-load-store-data-flow-opt -split-input-file | FileCheck %s --dump-input=fail + +// NOTE: There are no check statements for the constants created for specifying +// indices to store into because matching those with FileCheck is more trouble +// than its worth and anything other than a constant index 0 passed to the store +// would error in MLIR verification anyway. + +// CHECK-LABEL: func @scalarLoadStore +// CHECK-SAME: [[SRC:%[a-zA-Z0-9]+]] +// CHECK-SAME: [[DST:%[a-zA-Z0-9]+]] +func @scalarLoadStore(%src: memref, %dst: memref) { + // CHECK-DAG: [[SRC_INDICES:%.+]] = iree.constant 0 : index + %0 = load %src[] : memref + // CHECK-DAG: [[DST_INDICES:%.+]] = iree.constant 0 : index + // CHECK-DAG: [[LENGTHS:%.+]] = iree.constant 1 : index + store %0, %dst[] : memref + // CHECK-NEXT: "iree_hl_interp.copy"([[SRC]], [[SRC_INDICES]], [[DST]], [[DST_INDICES]], [[LENGTHS]]) + // CHECK-NEXT: return + return +} + +// ----- + +// CHECK-LABEL: func @replacementLoadStore +// CHECK-SAME: [[SRC:%[a-zA-Z0-9]+]] +// CHECK-SAME: [[DST:%[a-zA-Z0-9]+]] +func @replacementLoadStore(%src: memref<1xf32>, %dst: memref<1xf32>) { + // CHECK: [[C0:%.+]] = constant 0 : index + %c0 = constant 0 : index + // CHECK-DAG: [[SRC_INDICES:%.+]] = alloc() : memref<1xindex> + // Not checked: constant created for store index + // CHECK-DAG: store [[C0]], [[SRC_INDICES]] + %0 = load %src[%c0] : memref<1xf32> + // CHECK-DAG: [[DST_INDICES:%.+]] = alloc() : memref<1xindex> + // Not checked: constant created for store index + // CHECK-DAG: store [[C0]], [[DST_INDICES]] + store %0, %dst[%c0] : memref<1xf32> + // CHECK-DAG: [[LENGTHS:%.+]] = iree.constant 1 : index + // CHECK: "iree_hl_interp.copy"([[SRC]], [[SRC_INDICES]], [[DST]], [[DST_INDICES]], [[LENGTHS]]) + // CHECK-NEXT: return + return +} + +// ----- + +// CHECK-LABEL: func @offsetLoad +// CHECK-SAME: [[SRC:%[a-zA-Z0-9]+]] +// CHECK-SAME: [[DST:%[a-zA-Z0-9]+]] +func @offsetLoad(%src: memref<4xf32>, %dst: memref) { + // CHECK: [[C1:%.+]] = constant 1 : index + %c1 = constant 1 : index + // CHECK-DAG: [[SRC_INDICES:%.+]] = alloc() : memref<1xindex> + // CHECK-DAG: store [[C1]], [[SRC_INDICES]] + %1 = load %src[%c1] : memref<4xf32> + // CHECK-DAG: [[DST_INDICES:%.+]] = iree.constant 0 : index + store %1, %dst[] : memref + // CHECK-DAG: [[LENGTHS:%.+]] = iree.constant 1 : index + // CHECK: "iree_hl_interp.copy"([[SRC]], [[SRC_INDICES]], [[DST]], [[DST_INDICES]], [[LENGTHS]]) + // CHECK-NEXT: return + return +} + +// ----- + +// CHECK-LABEL: func @offsetStore +// CHECK-SAME: [[SRC:%[a-zA-Z0-9]+]] +// CHECK-SAME: [[DST:%[a-zA-Z0-9]+]] +func @offsetStore(%src: memref, %dst: memref<4xf32>) { + // CHECK-DAG: [[SRC_INDICES:%.+]] = iree.constant 0 : index + %0 = load %src[] : memref + // CHECK-DAG: [[C2:%.+]] = constant 2 : index + %c2 = constant 2 : index + // CHECK-DAG: [[DST_INDICES:%.+]] = alloc() : memref<1xindex> + // Not checked: constant created for store index + // CHECK-DAG: store [[C2]], [[DST_INDICES]] + store %0, %dst[%c2] : memref<4xf32> + // CHECK-DAG: [[LENGTHS:%.+]] = iree.constant 1 : index + // CHECK: "iree_hl_interp.copy"([[SRC]], [[SRC_INDICES]], [[DST]], [[DST_INDICES]], [[LENGTHS]]) + // CHECK-NEXT: return + return +} + +// ----- + +// CHECK-LABEL: func @offsetLoadStore +// CHECK-SAME: [[SRC:%[a-zA-Z0-9]+]] +// CHECK-SAME: [[DST:%[a-zA-Z0-9]+]] +func @offsetLoadStore(%src: memref<4xf32>, %dst: memref<4xf32>) { + // CHECK-DAG: [[C1:%.+]] = constant 1 : index + %c1 = constant 1 : index + // CHECK-DAG: [[SRC_INDICES:%.+]] = alloc() : memref<1xindex> + // Not checked: constant created for store index + // CHECK-DAG: store [[C1]], [[SRC_INDICES]] + %1 = load %src[%c1] : memref<4xf32> + // CHECK-DAG: [[C2:%.+]] = constant 2 : index + %c2 = constant 2 : index + // CHECK-DAG: [[DST_INDICES:%.+]] = alloc() : memref<1xindex> + // Not checked: constant created for store index + // CHECK-DAG: store [[C2]], [[DST_INDICES]] + store %1, %dst[%c2] : memref<4xf32> + // CHECK-DAG: [[LENGTHS:%.+]] = iree.constant 1 : index + // CHECK: "iree_hl_interp.copy"([[SRC]], [[SRC_INDICES]], [[DST]], [[DST_INDICES]], [[LENGTHS]]) + // CHECK-NEXT: return + return +} + +// ----- + +// CHECK-LABEL: func @offsetLoadStoreSameIndices +// CHECK-SAME: [[SRC:%[a-zA-Z0-9]+]] +// CHECK-SAME: [[DST:%[a-zA-Z0-9]+]] +func @offsetLoadStoreSameIndices(%src: memref<4xf32>, %dst: memref<4xf32>) { + // CHECK-DAG: [[C1:%.+]] = constant 1 : index + %c1 = constant 1 : index + // CHECK-DAG: [[SRC_INDICES:%.+]] = alloc() : memref<1xindex> + // Not checked: constant created for store index + // CHECK-DAG: store [[C1]], [[SRC_INDICES]] + %1 = load %src[%c1] : memref<4xf32> + // CHECK-DAG: [[DST_INDICES:%.+]] = alloc() : memref<1xindex> + // Not checked: constant created for store index + // CHECK-DAG: store [[C1]], [[DST_INDICES]] + store %1, %dst[%c1] : memref<4xf32> + // CHECK-DAG: [[LENGTHS:%.+]] = iree.constant 1 : index + // CHECK: "iree_hl_interp.copy"([[SRC]], [[SRC_INDICES]], [[DST]], [[DST_INDICES]], [[LENGTHS]]) + // CHECK-NEXT: return + return +} diff --git a/compiler/Transforms/Interpreter/test/make_executable_abi.mlir b/compiler/Transforms/Interpreter/test/make_executable_abi.mlir new file mode 100644 index 000000000000..a875407f6dc8 --- /dev/null +++ b/compiler/Transforms/Interpreter/test/make_executable_abi.mlir @@ -0,0 +1,154 @@ +// RUN: iree-opt %s -iree-make-executable-abi -split-input-file | FileCheck %s --dump-input=fail + +// CHECK-LABEL: func @staticOutputEntry +func @staticOutputEntry(%arg0: memref<4x2xf32>, %arg1: memref<4x2xf32>) + // CHECK-NEXT: attributes + attributes {iree.executable.export} { + // CHECK-NEXT: %0 = iree.memref_to_tensor(%arg0 : memref<4x2xf32>) : tensor<4x2xf32> + %0 = iree.load_input(%arg0 : memref<4x2xf32>) : tensor<4x2xf32> + // CHECK-NEXT: %1 = call @staticOutput(%0) : (tensor<4x2xf32>) -> tensor<4x2xf32> + %1 = call @staticOutput(%0) : (tensor<4x2xf32>) -> tensor<4x2xf32> + // CHECK-NEXT: %2 = iree.tensor_to_memref(%1 : tensor<4x2xf32>) : memref<4x2xf32> + // CHECK-NEXT: %3 = iree.constant dense<0> : tensor<2xi32> + // CHECK-NEXT: %4 = iree.constant dense<[4, 2]> : tensor<2xi32> + // CHECK-NEXT: "iree_hl_interp.copy"(%2, %3, %arg1, %3, %4) + iree.store_output(%1 : tensor<4x2xf32>, %arg1 : memref<4x2xf32>) + // CHECK-NEXT: return + return +} + +func @staticOutput(%arg0 : tensor<4x2xf32>) -> tensor<4x2xf32> + attributes {iree.dispatchable} { + return %arg0 : tensor<4x2xf32> +} + +// ----- + +// CHECK-LABEL: func @scalarFnEntry +func @scalarFnEntry(%arg0: memref, %arg1: memref) + // CHECK-NEXT: attributes + attributes {iree.executable.export} { + // CHECK-NEXT: %0 = load %arg0[] : memref + %0 = iree.load_input(%arg0 : memref) : f32 + // CHECK-NEXT: %1 = call @scalarFn(%0) : (f32) -> f32 + %1 = call @scalarFn(%0) : (f32) -> f32 + // CHECK-NEXT: store %1, %arg1[] : memref + iree.store_output(%1 : f32, %arg1 : memref) + // CHECK-NEXT: return + return +} + +func @scalarFn(%arg0 : f32) -> f32 + attributes {iree.dispatchable} { + return %arg0 : f32 +} + +// ----- + +// CHECK-LABEL: func @scalarTensorFnEntry +func @scalarTensorFnEntry(%arg0: memref, %arg1: memref) + // CHECK-NEXT: attributes + attributes {iree.executable.export} { + // CHECK-NEXT: %0 = iree.memref_to_tensor(%arg0 : memref) : tensor + %0 = iree.load_input(%arg0 : memref) : tensor + // CHECK-NEXT: %1 = call @scalarTensorFn(%0) : (tensor) -> tensor + %1 = call @scalarTensorFn(%0) : (tensor) -> tensor + // CHECK-NEXT: %2 = iree.tensor_to_memref(%1 : tensor) : memref + // CHECK-NEXT: %3 = iree.constant dense<0> : tensor<1xi32> + // CHECK-NEXT: %4 = iree.constant dense<1> : tensor<1xi32> + // CHECK-NEXT: "iree_hl_interp.copy"(%2, %3, %arg1, %3, %4) : (memref, memref<1xi32>, memref, memref<1xi32>, memref<1xi32>) -> () + iree.store_output(%1 : tensor, %arg1 : memref) + // CHECK-NEXT: return + return +} + +func @scalarTensorFn(%arg0 : tensor) -> tensor + attributes {iree.dispatchable} { + return %arg0 : tensor +} + +// ----- + +// CHECK-LABEL: func @returnValuesEntry +func @returnValuesEntry(%arg0: memref<4x2xf32>, %arg1: memref<4x2xf32>, %arg2: memref<4x2xf32>) -> (memref<4x2xf32>, memref<4x2xf32>) + // CHECK-NEXT: attributes + attributes {iree.executable.export} { + // CHECK-NEXT: %0 = iree.memref_to_tensor(%arg0 : memref<4x2xf32>) : tensor<4x2xf32> + %0 = iree.load_input(%arg0 : memref<4x2xf32>) : tensor<4x2xf32> + // CHECK-NEXT: call @returnValues + %1, %2 = call @returnValues(%0) : (tensor<4x2xf32>) -> (tensor<4x2xf32>, tensor<4x2xf32>) + // CHECK-NEXT: %2 = iree.tensor_to_memref(%1#0 : tensor<4x2xf32>) : memref<4x2xf32> + // CHECK-NEXT: %3 = iree.constant dense<0> : tensor<2xi32> + // CHECK-NEXT: %4 = iree.constant dense<[4, 2]> : tensor<2xi32> + // CHECK-NEXT: "iree_hl_interp.copy"(%2, %3, %arg1, %3, %4) + iree.store_output(%1 : tensor<4x2xf32>, %arg1 : memref<4x2xf32>) + // CHECK-NEXT: %5 = iree.tensor_to_memref(%1#1 : tensor<4x2xf32>) : memref<4x2xf32> + // CHECK-NEXT: %6 = iree.constant dense<0> : tensor<2xi32> + // CHECK-NEXT: %7 = iree.constant dense<[4, 2]> : tensor<2xi32> + // CHECK-NEXT: "iree_hl_interp.copy"(%5, %6, %arg2, %6, %7) + iree.store_output(%2 : tensor<4x2xf32>, %arg2 : memref<4x2xf32>) + %3 = iree.tensor_to_memref(%1 : tensor<4x2xf32>) : memref<4x2xf32> + %4 = iree.tensor_to_memref(%2 : tensor<4x2xf32>) : memref<4x2xf32> + // CHECK: return %8, %9 : memref<4x2xf32>, memref<4x2xf32> + return %3, %4 : memref<4x2xf32>, memref<4x2xf32> +} + +func @returnValues(%arg0 : tensor<4x2xf32>) -> (tensor<4x2xf32>, tensor<4x2xf32>) + attributes {iree.dispatchable} { + return %arg0, %arg0 : tensor<4x2xf32>, tensor<4x2xf32> +} + +// ----- + +// CHECK-LABEL: func @aliasInputsEntry +func @aliasInputsEntry(%arg0: memref<4x2xf32>, %arg1: memref<4x2xf32>) + // CHECK-NEXT: attributes + attributes {iree.executable.export} { + // CHECK-NEXT: %0 = iree.memref_to_tensor(%arg0 : memref<4x2xf32>) : tensor<4x2xf32> + %0 = iree.load_input(%arg0 : memref<4x2xf32>) : tensor<4x2xf32> + // CHECK-NEXT: %1 = iree.memref_to_tensor(%arg0 : memref<4x2xf32>) : tensor<4x2xf32> + %1 = iree.load_input(%arg0 : memref<4x2xf32>) : tensor<4x2xf32> + // CHECK-NEXT: call @aliasInputs + %2 = call @aliasInputs(%0, %1) : (tensor<4x2xf32>, tensor<4x2xf32>) -> tensor<4x2xf32> + // CHECK-NEXT: %3 = iree.tensor_to_memref(%2 : tensor<4x2xf32>) : memref<4x2xf32> + // CHECK-NEXT: %4 = iree.constant dense<0> : tensor<2xi32> + // CHECK-NEXT: %5 = iree.constant dense<[4, 2]> : tensor<2xi32> + // CHECK-NEXT: "iree_hl_interp.copy"(%3, %4, %arg1, %4, %5) + iree.store_output(%2 : tensor<4x2xf32>, %arg1 : memref<4x2xf32>) + // CHECK-NEXT: return + return +} + +func @aliasInputs(%arg0 : tensor<4x2xf32>, %arg1 : tensor<4x2xf32>) -> tensor<4x2xf32> + attributes {iree.dispatchable} { + return %arg0 : tensor<4x2xf32> +} + +// ----- + +// CHECK-LABEL: func @aliasOutputsEntry +func @aliasOutputsEntry(%arg0: memref<4x2xf32>, %arg1: memref<4x2xf32>) + // CHECK-NEXT: attributes + attributes {iree.executable.export} { + // CHECK-NEXT: %0 = iree.memref_to_tensor(%arg0 : memref<4x2xf32>) : tensor<4x2xf32> + %0 = iree.load_input(%arg0 : memref<4x2xf32>) : tensor<4x2xf32> + // CHECK-NEXT: call @aliasOutputs + %1 = call @aliasOutputs(%0) : (tensor<4x2xf32>) -> tensor<4x2xf32> + // CHECK-NEXT: %2 = iree.tensor_to_memref(%1 : tensor<4x2xf32>) : memref<4x2xf32> + // CHECK-NEXT: %3 = iree.constant dense<0> : tensor<2xi32> + // CHECK-NEXT: %4 = iree.constant dense<[4, 2]> : tensor<2xi32> + // CHECK-NEXT: "iree_hl_interp.copy"(%2, %3, %arg1, %3, %4) + iree.store_output(%1 : tensor<4x2xf32>, %arg1 : memref<4x2xf32>) + // CHECK-NEXT: %5 = iree.tensor_to_memref(%1 : tensor<4x2xf32>) : memref<4x2xf32> + // CHECK-NEXT: %6 = iree.constant dense<0> : tensor<2xi32> + // CHECK-NEXT: %7 = iree.constant dense<[4, 2]> : tensor<2xi32> + // CHECK-NEXT: "iree_hl_interp.copy"(%5, %6, %arg1, %6, %7) + iree.store_output(%1 : tensor<4x2xf32>, %arg1 : memref<4x2xf32>) + // CHECK-NEXT: return + return +} + +func @aliasOutputs(%arg0 : tensor<4x2xf32>) -> tensor<4x2xf32> + attributes {iree.dispatchable} { + return %arg0 : tensor<4x2xf32> +} diff --git a/compiler/Transforms/Interpreter/test/xla/concat.mlir b/compiler/Transforms/Interpreter/test/xla/concat.mlir new file mode 100644 index 000000000000..21eab819be86 --- /dev/null +++ b/compiler/Transforms/Interpreter/test/xla/concat.mlir @@ -0,0 +1,47 @@ +// RUN: iree-opt --lower-xla-to-iree-interpreter %s --split-input-file | FileCheck %s --dump-input=fail + +// CHECK-LABEL: func @concat.1D +// CHECK-SAME: [[ARG0:%[a-zA-Z0-9]+]] +// CHECK-SAME: [[ARG1:%[a-zA-Z0-9]+]] +func @concat.1D(%arg0 : tensor<4xi32>, %arg1 : tensor<4xi32>) -> tensor<8xi32> { + // CHECK-DAG: [[ARG0_MEMREF:%.+]] = iree.tensor_to_memref([[ARG0]] + // CHECK-DAG: [[ARG1_MEMREF:%.+]] = iree.tensor_to_memref([[ARG1]] + // CHECK: [[RES:%.+]] = "iree_hl_interp.concat"([[ARG0_MEMREF]], [[ARG1_MEMREF]]) {dimension = 0 : i32} + %0 = "xla_hlo.concatenate"(%arg0, %arg1) {dimension = 0 : i64} : (tensor<4xi32>, tensor<4xi32>) -> tensor<8xi32> + + // CHECK: [[RES_TENSOR:%.+]] = iree.memref_to_tensor([[RES]] + // CHECK: return [[RES_TENSOR]] + return %0 : tensor<8xi32> +} + +// ----- + +// CHECK-LABEL: func @concat.2D.Dim0 +// CHECK-SAME: [[ARG0:%[a-zA-Z0-9]+]] +// CHECK-SAME: [[ARG1:%[a-zA-Z0-9]+]] +func @concat.2D.Dim0(%arg0 : tensor<4x4xi32>, %arg1 : tensor<4x4xi32>) -> tensor<8x4xi32> { + // CHECK-DAG: [[ARG0_MEMREF:%.+]] = iree.tensor_to_memref([[ARG0]] + // CHECK-DAG: [[ARG1_MEMREF:%.+]] = iree.tensor_to_memref([[ARG1]] + // CHECK: [[RES:%.+]] = "iree_hl_interp.concat"([[ARG0_MEMREF]], [[ARG1_MEMREF]]) {dimension = 0 : i32} + %0 = "xla_hlo.concatenate"(%arg0, %arg1) {dimension = 0 : i64} : (tensor<4x4xi32>, tensor<4x4xi32>) -> tensor<8x4xi32> + + // CHECK: [[RES_TENSOR:%.+]] = iree.memref_to_tensor([[RES]] + // CHECK: return [[RES_TENSOR]] + return %0 : tensor<8x4xi32> +} + +// ----- + +// CHECK-LABEL: func @concat.2D.Dim1 +// CHECK-SAME: [[ARG0:%[a-zA-Z0-9]+]] +// CHECK-SAME: [[ARG1:%[a-zA-Z0-9]+]] +func @concat.2D.Dim1(%arg0 : tensor<4x4xi32>, %arg1 : tensor<4x4xi32>) -> tensor<4x8xi32> { + // CHECK-DAG: [[ARG0_MEMREF:%.+]] = iree.tensor_to_memref([[ARG0]] + // CHECK-DAG: [[ARG1_MEMREF:%.+]] = iree.tensor_to_memref([[ARG1]] + // CHECK: [[RES:%.+]] = "iree_hl_interp.concat"([[ARG0_MEMREF]], [[ARG1_MEMREF]]) {dimension = 1 : i32} + %0 = "xla_hlo.concatenate"(%arg0, %arg1) {dimension = 1 : i64} : (tensor<4x4xi32>, tensor<4x4xi32>) -> tensor<4x8xi32> + + // CHECK: [[RES_TENSOR:%.+]] = iree.memref_to_tensor([[RES]] + // CHECK: return [[RES_TENSOR]] + return %0 : tensor<4x8xi32> +} diff --git a/compiler/Transforms/Interpreter/test/xla/const.mlir b/compiler/Transforms/Interpreter/test/xla/const.mlir new file mode 100644 index 000000000000..a8b8e91bcd41 --- /dev/null +++ b/compiler/Transforms/Interpreter/test/xla/const.mlir @@ -0,0 +1,11 @@ +// RUN: iree-opt --lower-xla-to-iree-interpreter %s --split-input-file | FileCheck %s --dump-input=fail + +// CHECK-LABEL: func @const +func @const() -> tensor<3xi32> { + // CHECK: [[CONST:%.+]] = iree.constant dense<[1, 2, 3]> : tensor<3xi32> + %0 = "xla_hlo.constant"() {value = dense<[1, 2, 3]> : tensor<3xi32>} : () -> tensor<3xi32> + + // CHECK: [[RES_TENSOR:%.+]] = iree.memref_to_tensor([[CONST]] + // CHECK: return [[RES_TENSOR]] + return %0 : tensor<3xi32> +} diff --git a/compiler/Transforms/Interpreter/test/xla/dynamic_update_slice.mlir b/compiler/Transforms/Interpreter/test/xla/dynamic_update_slice.mlir new file mode 100644 index 000000000000..96eef9781fdd --- /dev/null +++ b/compiler/Transforms/Interpreter/test/xla/dynamic_update_slice.mlir @@ -0,0 +1,139 @@ +// RUN: iree-opt --lower-xla-to-iree-interpreter --mlir-print-op-generic %s --split-input-file | FileCheck %s --dump-input=fail + +// ----- + +// CHECK-LABEL: func @dynamic_update_slice.1D() -> tensor<4xi32> { +func @dynamic_update_slice.1D() -> tensor<4xi32> { + // CHECK-DAG: [[C:%[a-z_0-9]+]] = "std.constant"() {value = dense<5> : tensor<1xi32>} : () -> tensor<1xi32> + %cst = "std.constant"() {value = dense<5> : tensor<1xi32>} : () -> tensor<1xi32> + + // CHECK-DAG: [[C0:%[a-z_0-9]+]] = "std.constant"() {value = dense<[1, 2, 3, 4]> : tensor<4xi32>} : () -> tensor<4xi32> + %cst_0 = "std.constant"() {value = dense<[1, 2, 3, 4]> : tensor<4xi32>} : () -> tensor<4xi32> + + // CHECK-DAG: [[C1:%[a-z_0-9]+]] = "std.constant"() {value = dense<1> : tensor} : () -> tensor + %cst_1 = "std.constant"() {value = dense<1> : tensor} : () -> tensor + + // CHECK-DAG: [[R0:%[a-z_0-9]+]] = "iree.tensor_to_memref"([[C0]]) : (tensor<4xi32>) -> memref<4xi32> + // CHECK-DAG: [[R1:%[a-z_0-9]+]] = "iree.tensor_to_memref"([[C]]) : (tensor<1xi32>) -> memref<1xi32> + // CHECK-DAG: [[R2:%[a-z_0-9]+]] = "iree.tensor_to_memref"([[C1]]) : (tensor) -> memref + // CHECK-DAG: [[R3:%[a-z_0-9]+]] = "iree.constant"() {value = dense<1> : tensor<1xi64>} : () -> memref<1xi64> + // CHECK-DAG: [[R4:%[a-z_0-9]+]] = "iree.constant"() {value = dense<1> : tensor<1xi64>} : () -> memref<1xi64> + // CHECK-DAG: [[R5:%[a-z_0-9]+]] = "iree_hl_interp.reshape"([[R2]], [[R4]]) : (memref, memref<1xi64>) -> memref<1xi32> + // CHECK-DAG: [[R6:%[a-z_0-9]+]] = "iree_hl_interp.concat"([[R5]]) {dimension = 0 : i32} : (memref<1xi32>) -> memref<1xi32> + // CHECK-DAG: [[R7:%[a-z_0-9]+]] = "iree.constant"() {value = dense<0> : tensor<1xi64>} : () -> memref<1xi64> + // CHECK-DAG: [[R8:%[a-z_0-9]+]] = "iree_hl_interp.clone"([[R0]]) : (memref<4xi32>) -> memref<4xi32> + // CHECK-NEXT: "iree_hl_interp.copy"([[R1]], [[R7]], [[R8]], [[R6]], [[R3]]) : (memref<1xi32>, memref<1xi64>, memref<4xi32>, memref<1xi32>, memref<1xi64>) -> () + %0 = "xla_hlo.dynamic-update-slice"(%cst_0, %cst, %cst_1) : (tensor<4xi32>, tensor<1xi32>, tensor) -> tensor<4xi32> + + // CHECK-NEXT: [[R9:%[a-z_0-9]+]] = "iree.memref_to_tensor"([[R8]]) : (memref<4xi32>) -> tensor<4xi32> + // CHECK-NEXT: "std.return"([[R9]]) : (tensor<4xi32>) -> () + "std.return"(%0) : (tensor<4xi32>) -> () +} + +// ----- + +// CHECK-LABEL: func @dynamic_update_slice.2D() -> tensor<2x4xi32> { +func @dynamic_update_slice.2D() -> tensor<2x4xi32> { + // CHECK-DAG: [[C:%[a-z_0-9]+]] = "std.constant"() {value = dense<12> : tensor<1x1xi32>} : () -> tensor<1x1xi32> + %cst = "std.constant"() {value = dense<12> : tensor<1x1xi32>} : () -> tensor<1x1xi32> + + // CHECK-DAG: [[C0:%[a-z_0-9]+]] = "std.constant"() {value = dense<{{\[\[}}1, 2, 3, 4], [5, 6, 7, 8]]> : tensor<2x4xi32>} : () -> tensor<2x4xi32> + %cst_0 = "std.constant"() {value = dense<[[1, 2, 3, 4], [5, 6, 7, 8]]> : tensor<2x4xi32>} : () -> tensor<2x4xi32> + + // CHECK-DAG: [[C1:%[a-z_0-9]+]] = "std.constant"() {value = dense<1> : tensor} : () -> tensor + %cst_1 = "std.constant"() {value = dense<1> : tensor} : () -> tensor + + // CHECK-DAG: [[C2:%[a-z_0-9]+]] = "std.constant"() {value = dense<2> : tensor} : () -> tensor + %cst_2 = "std.constant"() {value = dense<2> : tensor} : () -> tensor + + // CHECK-DAG: [[R0:%[a-z_0-9]+]] = "iree.tensor_to_memref"([[C0]]) : (tensor<2x4xi32>) -> memref<2x4xi32> + // CHECK-DAG: [[R1:%[a-z_0-9]+]] = "iree.tensor_to_memref"([[C]]) : (tensor<1x1xi32>) -> memref<1x1xi32> + // CHECK-DAG: [[R2:%[a-z_0-9]+]] = "iree.tensor_to_memref"([[C1]]) : (tensor) -> memref + // CHECK-DAG: [[R3:%[a-z_0-9]+]] = "iree.tensor_to_memref"([[C2]]) : (tensor) -> memref + // CHECK-DAG: [[R4:%[a-z_0-9]+]] = "iree.constant"() {value = dense<1> : tensor<2xi64>} : () -> memref<2xi64> + // CHECK-DAG: [[R5:%[a-z_0-9]+]] = "iree.constant"() {value = dense<1> : tensor<1xi64>} : () -> memref<1xi64> + // CHECK-DAG: [[R6:%[a-z_0-9]+]] = "iree_hl_interp.reshape"([[R2]], [[R5]]) : (memref, memref<1xi64>) -> memref<1xi32> + // CHECK-DAG: [[R7:%[a-z_0-9]+]] = "iree.constant"() {value = dense<1> : tensor<1xi64>} : () -> memref<1xi64> + // CHECK-DAG: [[R8:%[a-z_0-9]+]] = "iree_hl_interp.reshape"([[R3]], [[R7]]) : (memref, memref<1xi64>) -> memref<1xi32> + // CHECK-DAG: [[R9:%[a-z_0-9]+]] = "iree_hl_interp.concat"([[R6]], [[R8]]) {dimension = 0 : i32} : (memref<1xi32>, memref<1xi32>) -> memref<2xi32> + // CHECK-DAG: [[R10:%[a-z_0-9]+]] = "iree.constant"() {value = dense<0> : tensor<2xi64>} : () -> memref<2xi64> + // CHECK-NEXT: [[R11:%[a-z_0-9]+]] = "iree_hl_interp.clone"([[R0]]) : (memref<2x4xi32>) -> memref<2x4xi32> + // CHECK-NEXT: "iree_hl_interp.copy"([[R1]], [[R10]], [[R11]], [[R9]], [[R4]]) : (memref<1x1xi32>, memref<2xi64>, memref<2x4xi32>, memref<2xi32>, memref<2xi64>) -> () + %0 = "xla_hlo.dynamic-update-slice"(%cst_0, %cst, %cst_1, %cst_2) : (tensor<2x4xi32>, tensor<1x1xi32>, tensor, tensor) -> tensor<2x4xi32> + + // CHECK-NEXT: [[R12:%[a-z_0-9]+]] = "iree.memref_to_tensor"([[R11]]) : (memref<2x4xi32>) -> tensor<2x4xi32> + // CHECK-NEXT: "std.return"([[R12]]) : (tensor<2x4xi32>) -> () + "std.return"(%0) : (tensor<2x4xi32>) -> () +} + +// ----- + +// CHECK-LABEL: func @dynamic_update_slice.1D.notlast() -> tensor<4xi32> { +func @dynamic_update_slice.1D.notlast() -> tensor<4xi32> { + // CHECK-DAG: [[C:%[a-z_0-9]+]] = "std.constant"() {value = dense<5> : tensor<1xi32>} : () -> tensor<1xi32> + %cst = "std.constant"() {value = dense<5> : tensor<1xi32>} : () -> tensor<1xi32> + + // CHECK-DAG: [[C0:%[a-z_0-9]+]] = "std.constant"() {value = dense<[1, 2, 3, 4]> : tensor<4xi32>} : () -> tensor<4xi32> + %cst_0 = "std.constant"() {value = dense<[1, 2, 3, 4]> : tensor<4xi32>} : () -> tensor<4xi32> + + // CHECK-DAG: [[C1:%[a-z_0-9]+]] = "std.constant"() {value = dense<1> : tensor} : () -> tensor + %cst_1 = "std.constant"() {value = dense<1> : tensor} : () -> tensor + + // CHECK-DAG: [[R0:%[a-z_0-9]+]] = "iree.tensor_to_memref"([[C0]]) : (tensor<4xi32>) -> memref<4xi32> + // CHECK-DAG: [[R1:%[a-z_0-9]+]] = "iree.tensor_to_memref"([[C]]) : (tensor<1xi32>) -> memref<1xi32> + // CHECK-DAG: [[R2:%[a-z_0-9]+]] = "iree.tensor_to_memref"([[C1]]) : (tensor) -> memref + // CHECK-DAG: [[R3:%[a-z_0-9]+]] = "iree.constant"() {value = dense<1> : tensor<1xi64>} : () -> memref<1xi64> + // CHECK-DAG: [[R4:%[a-z_0-9]+]] = "iree.constant"() {value = dense<1> : tensor<1xi64>} : () -> memref<1xi64> + // CHECK-DAG: [[R5:%[a-z_0-9]+]] = "iree_hl_interp.reshape"([[R2]], [[R4]]) : (memref, memref<1xi64>) -> memref<1xi32> + // CHECK-DAG: [[R6:%[a-z_0-9]+]] = "iree_hl_interp.concat"([[R5]]) {dimension = 0 : i32} : (memref<1xi32>) -> memref<1xi32> + // CHECK-DAG: [[R7:%[a-z_0-9]+]] = "iree.constant"() {value = dense<0> : tensor<1xi64>} : () -> memref<1xi64> + // CHECK-DAG: [[R8:%[a-z_0-9]+]] = "iree_hl_interp.clone"([[R0]]) : (memref<4xi32>) -> memref<4xi32> + // CHECK-NEXT: "iree_hl_interp.copy"([[R1]], [[R7]], [[R8]], [[R6]], [[R3]]) : (memref<1xi32>, memref<1xi64>, memref<4xi32>, memref<1xi32>, memref<1xi64>) -> () + %0 = "xla_hlo.dynamic-update-slice"(%cst_0, %cst, %cst_1) : (tensor<4xi32>, tensor<1xi32>, tensor) -> tensor<4xi32> + + // CHECK-NEXT: [[R9:%[a-z_0-9]+]] = "iree.memref_to_tensor"([[R8]]) : (memref<4xi32>) -> tensor<4xi32> + // CHECK-NEXT: [[R11:%[a-z_0-9]+]] = "xla_hlo.add"([[C0]], [[R9]]) : (tensor<4xi32>, tensor<4xi32>) -> tensor<4xi32> + %1 = "xla_hlo.add"(%cst_0, %0) : (tensor<4xi32>, tensor<4xi32>) -> tensor<4xi32> + + // CHECK-DAG: "std.return"([[R10]]) : (tensor<4xi32>) -> () + "std.return"(%1) : (tensor<4xi32>) -> () +} + +// ----- + +// CHECK-LABEL: func @dynamic_update_slice.2D.notlast() -> tensor<2x4xi32> { +func @dynamic_update_slice.2D.notlast() -> tensor<2x4xi32> { + // CHECK-DAG: [[C:%[a-z_0-9]+]] = "std.constant"() {value = dense<12> : tensor<1x1xi32>} : () -> tensor<1x1xi32> + %cst = "std.constant"() {value = dense<12> : tensor<1x1xi32>} : () -> tensor<1x1xi32> + + // CHECK-DAG: [[C0:%[a-z_0-9]+]] = "std.constant"() {value = dense<{{\[\[}}1, 2, 3, 4], [5, 6, 7, 8]]> : tensor<2x4xi32>} : () -> tensor<2x4xi32> + %cst_0 = "std.constant"() {value = dense<[[1, 2, 3, 4], [5, 6, 7, 8]]> : tensor<2x4xi32>} : () -> tensor<2x4xi32> + + // CHECK-DAG: [[C1:%[a-z_0-9]+]] = "std.constant"() {value = dense<1> : tensor} : () -> tensor + %cst_1 = "std.constant"() {value = dense<1> : tensor} : () -> tensor + + // CHECK-DAG: [[C2:%[a-z_0-9]+]] = "std.constant"() {value = dense<2> : tensor} : () -> tensor + %cst_2 = "std.constant"() {value = dense<2> : tensor} : () -> tensor + + // CHECK-DAG: [[R0:%[a-z_0-9]+]] = "iree.tensor_to_memref"([[C0]]) : (tensor<2x4xi32>) -> memref<2x4xi32> + // CHECK-DAG: [[R1:%[a-z_0-9]+]] = "iree.tensor_to_memref"([[C]]) : (tensor<1x1xi32>) -> memref<1x1xi32> + // CHECK-DAG: [[R2:%[a-z_0-9]+]] = "iree.tensor_to_memref"([[C1]]) : (tensor) -> memref + // CHECK-DAG: [[R3:%[a-z_0-9]+]] = "iree.tensor_to_memref"([[C2]]) : (tensor) -> memref + // CHECK-DAG: [[R4:%[a-z_0-9]+]] = "iree.constant"() {value = dense<1> : tensor<2xi64>} : () -> memref<2xi64> + // CHECK-DAG: [[R5:%[a-z_0-9]+]] = "iree.constant"() {value = dense<1> : tensor<1xi64>} : () -> memref<1xi64> + // CHECK-DAG: [[R6:%[a-z_0-9]+]] = "iree_hl_interp.reshape"([[R2]], [[R5]]) : (memref, memref<1xi64>) -> memref<1xi32> + // CHECK-DAG: [[R7:%[a-z_0-9]+]] = "iree.constant"() {value = dense<1> : tensor<1xi64>} : () -> memref<1xi64> + // CHECK-DAG: [[R8:%[a-z_0-9]+]] = "iree_hl_interp.reshape"([[R3]], [[R7]]) : (memref, memref<1xi64>) -> memref<1xi32> + // CHECK-DAG: [[R9:%[a-z_0-9]+]] = "iree_hl_interp.concat"([[R6]], [[R8]]) {dimension = 0 : i32} : (memref<1xi32>, memref<1xi32>) -> memref<2xi32> + // CHECK-DAG: [[R10:%[a-z_0-9]+]] = "iree.constant"() {value = dense<0> : tensor<2xi64>} : () -> memref<2xi64> + // CHECK-DAG: [[R11:%[a-z_0-9]+]] = "iree_hl_interp.clone"([[R0]]) : (memref<2x4xi32>) -> memref<2x4xi32> + // CHECK-NEXT: "iree_hl_interp.copy"([[R1]], [[R10]], [[R11]], [[R9]], [[R4]]) : (memref<1x1xi32>, memref<2xi64>, memref<2x4xi32>, memref<2xi32>, memref<2xi64>) -> () + // CHECK-NEXT: [[R12:%[a-z_0-9]+]] = "iree.memref_to_tensor"([[R11]]) : (memref<2x4xi32>) -> tensor<2x4xi32> + %0 = "xla_hlo.dynamic-update-slice"(%cst_0, %cst, %cst_1, %cst_2) : (tensor<2x4xi32>, tensor<1x1xi32>, tensor, tensor) -> tensor<2x4xi32> + + // CHECK-NEXT: [[R13:%[a-z_0-9]+]] = "xla_hlo.add"([[C0]], [[R12]]) : (tensor<2x4xi32>, tensor<2x4xi32>) -> tensor<2x4xi32> + %1 = "xla_hlo.add"(%cst_0, %0) : (tensor<2x4xi32>, tensor<2x4xi32>) -> tensor<2x4xi32> + + // CHECK-NEXT: "std.return"([[R13]]) : (tensor<2x4xi32>) -> () + "std.return"(%1) : (tensor<2x4xi32>) -> () +} diff --git a/compiler/Transforms/Interpreter/test/xla/gather.mlir b/compiler/Transforms/Interpreter/test/xla/gather.mlir new file mode 100644 index 000000000000..79f33e3639c3 --- /dev/null +++ b/compiler/Transforms/Interpreter/test/xla/gather.mlir @@ -0,0 +1,121 @@ +// RUN: iree-opt --lower-xla-to-iree-interpreter --print-ir-after-all %s | FileCheck %s --dump-input=fail + +// CHECK-LABEL: @gather +// CHECK-SAME: [[INPUT:%[a-zA-Z0-9]+]] +// CHECK-SAME: [[START_INDICES:%[a-zA-Z0-9]+]] +func @gather(%input : tensor<5x2x3xf32>, %start_indices : tensor) -> tensor<2x3xf32> { + // CHECK-DAG: [[SRC:%.+]] = iree.tensor_to_memref([[INPUT]] : tensor<5x2x3xf32>) + // CHECK-DAG: [[START_INDICES_MEMREF:%.+]] = iree.tensor_to_memref([[START_INDICES]] : tensor) + // CHECK-DAG: [[START_INDICES_NEW_SHAPE:%.+]] = iree.constant dense<1> : tensor<1xi64> + // CHECK-DAG: [[START_INDICES_RESHAPED:%.+]] = "iree_hl_interp.reshape"([[START_INDICES_MEMREF]], [[START_INDICES_NEW_SHAPE]]) + // CHECK-DAG: [[ZEROES:%.+]] = iree.constant dense<0> : tensor<2xi64> + // CHECK-DAG: [[START_INDICES_PADDED:%.+]] = "iree_hl_interp.concat"([[START_INDICES_RESHAPED]], [[ZEROES]]) + // CHECK-DAG: [[DST:%.+]] = "iree_hl_interp.alloc_heap"() : () -> memref<1x2x3xf32> + // CHECK-DAG: [[DST_INDICES:%.+]] = iree.constant dense<0> + // CHECK-DAG: [[LENGTHS:%.+]] = iree.constant dense<[1, 2, 3]> + // CHECK-NEXT: "iree_hl_interp.copy"([[SRC]], [[START_INDICES_PADDED]], [[DST]], [[DST_INDICES]], [[LENGTHS]]) + // CHECK-DAG: [[NEW_SHAPE:%.+]] = iree.constant dense<[2, 3]> + // CHECK-DAG: [[RESHAPED:%.+]] = "iree_hl_interp.reshape"([[DST]], [[NEW_SHAPE]]) + // CHECK-DAG: [[RESULT_TENSOR:%.+]] = iree.memref_to_tensor([[RESHAPED]] : memref<2x3xf32>) + %result = "xla_hlo.gather"(%input, %start_indices) { + collapsed_slice_dims = dense<0> : tensor<1xi64>, + index_vector_dim = 0 : i64, + offset_dims = dense<[0, 1]> : tensor<2xi64>, + slice_sizes = dense<[1, 2, 3]> : tensor<3xi64>, + start_index_map = dense<0> : tensor<1xi64> + } : (tensor<5x2x3xf32>, tensor) -> tensor<2x3xf32> + // CHECK-NEXT: return [[RESULT_TENSOR]] + return %result : tensor<2x3xf32> +} + +// CHECK-LABEL: @gather_nonscalar_indices +// CHECK-SAME: [[INPUT:%[a-zA-Z0-9]+]] +// CHECK-SAME: [[START_INDICES:%[a-zA-Z0-9]+]] +func @gather_nonscalar_indices(%input : tensor<5x2x3xf32>, %start_indices : tensor<1xi64>) -> tensor<2x3xf32> { + // CHECK-DAG: [[SRC:%.+]] = iree.tensor_to_memref([[INPUT]] : tensor<5x2x3xf32>) + // CHECK-DAG: [[START_INDICES_MEMREF:%.+]] = iree.tensor_to_memref([[START_INDICES]] : tensor<1xi64>) + // CHECK-DAG: [[ZEROES:%.+]] = iree.constant dense<0> : tensor<2xi64> + // CHECK-DAG: [[START_INDICES_PADDED:%.+]] = "iree_hl_interp.concat"([[START_INDICES_MEMREF]], [[ZEROES]]) + // CHECK-DAG: [[DST:%.+]] = "iree_hl_interp.alloc_heap"() : () -> memref<1x2x3xf32> + // CHECK-DAG: [[DST_INDICES:%.+]] = iree.constant dense<0> + // CHECK-DAG: [[LENGTHS:%.+]] = iree.constant dense<[1, 2, 3]> + // CHECK-NEXT: "iree_hl_interp.copy"([[SRC]], [[START_INDICES_PADDED]], [[DST]], [[DST_INDICES]], [[LENGTHS]]) + // CHECK-DAG: [[NEW_SHAPE:%.+]] = iree.constant dense<[2, 3]> + // CHECK-DAG: [[RESHAPED:%.+]] = "iree_hl_interp.reshape"([[DST]], [[NEW_SHAPE]]) + // CHECK-DAG: [[RESULT_TENSOR:%.+]] = iree.memref_to_tensor([[RESHAPED]] : memref<2x3xf32>) + %result = "xla_hlo.gather"(%input, %start_indices) { + collapsed_slice_dims = dense<0> : tensor<1xi64>, + index_vector_dim = 0 : i64, + offset_dims = dense<[0, 1]> : tensor<2xi64>, + slice_sizes = dense<[1, 2, 3]> : tensor<3xi64>, + start_index_map = dense<0> : tensor<1xi64> + } : (tensor<5x2x3xf32>, tensor<1xi64>) -> tensor<2x3xf32> + // CHECK-NEXT: return [[RESULT_TENSOR]] + return %result : tensor<2x3xf32> +} + +// CHECK-LABEL: @gather_fully_specified_indices +// CHECK-SAME: [[INPUT:%[a-zA-Z0-9]+]] +// CHECK-SAME: [[START_INDICES:%[a-zA-Z0-9]+]] +func @gather_fully_specified_indices(%input : tensor<5x2x3xf32>, %start_indices : tensor<3xi64>) -> tensor<2x3xf32> { + // CHECK-DAG: [[SRC:%.+]] = iree.tensor_to_memref([[INPUT]] : tensor<5x2x3xf32>) + // CHECK-DAG: [[START_INDICES_MEMREF:%.+]] = iree.tensor_to_memref([[START_INDICES]] : tensor<3xi64>) + // CHECK-DAG: [[DST:%.+]] = "iree_hl_interp.alloc_heap"() : () -> memref<1x2x3xf32> + // CHECK-DAG: [[DST_INDICES:%.+]] = iree.constant dense<0> + // CHECK-DAG: [[LENGTHS:%.+]] = iree.constant dense<[1, 2, 3]> + // CHECK-NEXT: "iree_hl_interp.copy"([[SRC]], [[START_INDICES_MEMREF]], [[DST]], [[DST_INDICES]], [[LENGTHS]]) + // CHECK-DAG: [[NEW_SHAPE:%.+]] = iree.constant dense<[2, 3]> + // CHECK-DAG: [[RESHAPED:%.+]] = "iree_hl_interp.reshape"([[DST]], [[NEW_SHAPE]]) + // CHECK-DAG: [[RESULT_TENSOR:%.+]] = iree.memref_to_tensor([[RESHAPED]] : memref<2x3xf32>) + %result = "xla_hlo.gather"(%input, %start_indices) { + collapsed_slice_dims = dense<0> : tensor<1xi64>, + index_vector_dim = 0 : i64, + offset_dims = dense<[0, 1]> : tensor<2xi64>, + slice_sizes = dense<[1, 2, 3]> : tensor<3xi64>, + start_index_map = dense<0> : tensor<1xi64> + } : (tensor<5x2x3xf32>, tensor<3xi64>) -> tensor<2x3xf32> + // CHECK-NEXT: return [[RESULT_TENSOR]] + return %result : tensor<2x3xf32> +} + +// CHECK-LABEL: @gather_not_lowered +// CHECK-SAME: [[INPUT:%[a-zA-Z0-9]+]] +// CHECK-SAME: [[START_INDICES:%[a-zA-Z0-9]+]] +func @gather_not_lowered(%input : tensor<5x2x3xf32>, %start_indices : tensor<2x2xi64>) { + // CHECK-NEXT "xla_hlo.gather" + %axis_1 = "xla_hlo.gather"(%input, %start_indices) { + collapsed_slice_dims = dense<0> : tensor<1xi64>, + index_vector_dim = 1 : i64, + offset_dims = dense<[0, 1, 2]> : tensor<3xi64>, + slice_sizes = dense<[1, 2, 3]> : tensor<3xi64>, + start_index_map = dense<0> : tensor<1xi64> + } : (tensor<5x2x3xf32>, tensor<2x2xi64>) -> tensor<2x3xf32> + + // CHECK-NEXT "xla_hlo.gather" + %collapse_1 = "xla_hlo.gather"(%input, %start_indices) { + collapsed_slice_dims = dense<1> : tensor<1xi64>, + index_vector_dim = 0 : i64, + offset_dims = dense<[0, 1, 2]> : tensor<3xi64>, + slice_sizes = dense<[1, 2, 3]> : tensor<3xi64>, + start_index_map = dense<0> : tensor<1xi64> + } : (tensor<5x2x3xf32>, tensor<2x2xi64>) -> tensor<2x3xf32> + + // CHECK-NEXT "xla_hlo.gather" + %transposes = "xla_hlo.gather"(%input, %start_indices) { + collapsed_slice_dims = dense<0> : tensor<1xi64>, + index_vector_dim = 0 : i64, + offset_dims = dense<[0, 1, 2]> : tensor<3xi64>, + slice_sizes = dense<[1, 2, 3]> : tensor<3xi64>, + start_index_map = dense<[1, 0]> : tensor<2xi64> + } : (tensor<5x2x3xf32>, tensor<2x2xi64>) -> tensor<2x3xf32> + + // CHECK-NEXT "xla_hlo.gather" + %has_batch_dims = "xla_hlo.gather"(%input, %start_indices) { + collapsed_slice_dims = dense<0> : tensor<1xi64>, + index_vector_dim = 0 : i64, + offset_dims = dense<1> : tensor<1xi64>, + slice_sizes = dense<[1, 2, 3]> : tensor<3xi64>, + start_index_map = dense<[1, 0]> : tensor<2xi64> + } : (tensor<5x2x3xf32>, tensor<2x2xi64>) -> tensor<2x3xf32> + return +} diff --git a/compiler/Transforms/Interpreter/test/xla/slice.mlir b/compiler/Transforms/Interpreter/test/xla/slice.mlir new file mode 100644 index 000000000000..cb29f5efdfc4 --- /dev/null +++ b/compiler/Transforms/Interpreter/test/xla/slice.mlir @@ -0,0 +1,31 @@ +// RUN: iree-opt --lower-xla-to-iree-interpreter %s | FileCheck %s --dump-input=fail + +// CHECK-LABEL: @slice +// CHECK-SAME: [[ARG:%[a-zA-Z0-9]+]] +func @slice(%arg : tensor<3x4xf32>) -> tensor<1x4xf32> { + // CHECK-DAG: [[SRC:%.+]] = iree.tensor_to_memref([[ARG]] + // CHECK-DAG: [[SRC_INDICES:%.+]] = iree.constant dense<[1, 0]> + // CHECK-DAG: [[DST:%.+]] = "iree_hl_interp.alloc_heap"() : () -> memref<1x4xf32> + // CHECK-DAG: [[DST_INDICES:%.+]] = iree.constant dense<0> + // CHECK-DAG: [[LENGTHS:%.+]] = iree.constant dense<[1, 4]> + // CHECK-NEXT: "iree_hl_interp.copy"([[SRC]], [[SRC_INDICES]], [[DST]], [[DST_INDICES]], [[LENGTHS]]) + // CHECK-NEXT: [[RESULT_TENSOR:%.+]] = iree.memref_to_tensor([[DST]] + %result = "xla_hlo.slice"(%arg) {start_indices = dense<[1, 0]> : tensor<2xi64>, limit_indices = dense<[2, 4]> : tensor<2xi64>, strides = dense<1> : tensor<2xi64>} : (tensor<3x4xf32>) -> tensor<1x4xf32> + // CHECK-NEXT: return [[RESULT_TENSOR]] + return %result : tensor<1x4xf32> +} + +// CHECK-LABEL: @slice_noncontiguous +// CHECK-SAME: [[ARG:%[a-zA-Z0-9]+]] +func @slice_noncontiguous(%arg : tensor<3x4xf32>) -> tensor<2x2xf32> { + // CHECK-DAG: [[SRC:%.+]] = iree.tensor_to_memref([[ARG]] + // CHECK-DAG: [[SRC_INDICES:%.+]] = iree.constant dense<1> + // CHECK-DAG: [[DST:%.+]] = "iree_hl_interp.alloc_heap"() : () -> memref<2x2xf32> + // CHECK-DAG: [[DST_INDICES:%.+]] = iree.constant dense<0> + // CHECK-DAG: [[LENGTHS:%.+]] = iree.constant dense<2> + // CHECK-NEXT: "iree_hl_interp.copy"([[SRC]], [[SRC_INDICES]], [[DST]], [[DST_INDICES]], [[LENGTHS]]) + // CHECK-NEXT: [[RESULT_TENSOR:%.+]] = iree.memref_to_tensor([[DST]] + %result = "xla_hlo.slice"(%arg) {start_indices = dense<1> : tensor<2xi64>, limit_indices = dense<3> : tensor<2xi64>, strides = dense<1> : tensor<2xi64>} : (tensor<3x4xf32>) -> tensor<2x2xf32> + // CHECK-NEXT: return [[RESULT_TENSOR]] + return %result : tensor<2x2xf32> +} diff --git a/compiler/Transforms/LegalizeTensorMemRef.cpp b/compiler/Transforms/LegalizeTensorMemRef.cpp new file mode 100644 index 000000000000..c572298d2cf9 --- /dev/null +++ b/compiler/Transforms/LegalizeTensorMemRef.cpp @@ -0,0 +1,56 @@ +// Copyright 2019 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "third_party/llvm/llvm/projects/google_mlir/include/mlir/IR/Builders.h" +#include "third_party/llvm/llvm/projects/google_mlir/include/mlir/IR/PatternMatch.h" +#include "third_party/llvm/llvm/projects/google_mlir/include/mlir/Pass/Pass.h" +#include "third_party/llvm/llvm/projects/google_mlir/include/mlir/Pass/PassRegistry.h" +#include "third_party/mlir_edge/iree/compiler/IR/Ops.h" +#include "third_party/mlir_edge/iree/compiler/IR/StructureOps.h" + +namespace mlir { +namespace iree_compiler { +namespace { + +struct LegalizeTensorMemRefPass + : public FunctionPass { + void runOnFunction() override { + for (auto& block : getFunction().getBlocks()) { + for (auto& op : block.getOperations()) { + if (isa(op) && + isa_and_nonnull( + op.getOperand(0)->getDefiningOp())) { + op.getResult(0)->replaceAllUsesWith( + op.getOperand(0)->getDefiningOp()->getOperand(0)); + } + } + + // Performs cleanup of ops removed above. + OwningRewritePatternList patterns; + applyPatternsGreedily(getFunction(), patterns); + } + } +}; + +} // namespace + +std::unique_ptr> createLegalizeTensorMemRefPass() { + return std::make_unique(); +} + +static PassRegistration legalize_tensor_memref_pass( + "iree-legalize-tensor-memref", "Remove extra tensor/memref operations."); + +} // namespace iree_compiler +} // namespace mlir diff --git a/compiler/Transforms/LegalizeTupleElementAccess.cpp b/compiler/Transforms/LegalizeTupleElementAccess.cpp new file mode 100644 index 000000000000..6d8076f0320f --- /dev/null +++ b/compiler/Transforms/LegalizeTupleElementAccess.cpp @@ -0,0 +1,78 @@ +// Copyright 2019 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "third_party/llvm/llvm/include/llvm/ADT/ArrayRef.h" +#include "third_party/llvm/llvm/include/llvm/ADT/SmallVector.h" +#include "third_party/llvm/llvm/include/llvm/ADT/iterator_range.h" +#include "third_party/llvm/llvm/projects/google_mlir/include/mlir/IR/BlockAndValueMapping.h" +#include "third_party/llvm/llvm/projects/google_mlir/include/mlir/IR/Builders.h" +#include "third_party/llvm/llvm/projects/google_mlir/include/mlir/IR/PatternMatch.h" +#include "third_party/llvm/llvm/projects/google_mlir/include/mlir/IR/StandardTypes.h" +#include "third_party/llvm/llvm/projects/google_mlir/include/mlir/Pass/Pass.h" +#include "third_party/llvm/llvm/projects/google_mlir/include/mlir/Pass/PassRegistry.h" +#include "third_party/llvm/llvm/projects/google_mlir/include/mlir/Transforms/Utils.h" +#include "third_party/tensorflow/compiler/mlir/xla/ir/hlo_ops.h" + +using mlir::PassRegistration; + +namespace mlir { +namespace iree_compiler { +namespace { + +struct LegalizeTupleElementAccessPass + : public FunctionPass { + void runOnFunction() override; +}; + +/// Legalizes XLA Tuple element access, mostly be removing them entirely. +void LegalizeTupleElementAccessPass::runOnFunction() { + auto func = getFunction(); + Builder builder(func.getContext()); + + llvm::SmallVector ops_to_remove; + func.walk([&](xla_hlo::TupleOp op) { + bool can_remove_tuple = true; + for (auto* user : op.getResult()->getUsers()) { + if (auto get_op = dyn_cast(user)) { + get_op.getResult()->replaceAllUsesWith( + op.getOperand(get_op.index().getLimitedValue())); + ops_to_remove.push_back(get_op.getOperation()); + } else { + can_remove_tuple = false; + } + } + + if (can_remove_tuple) { + ops_to_remove.push_back(op.getOperation()); + } + }); + + // Cleanup ops that have been bypassed. + for (auto op : ops_to_remove) { + op->erase(); + } +} + +} // namespace + +std::unique_ptr> createLegalizeTupleElementAccessPass() { + return std::make_unique(); +} + +static PassRegistration legalize_pass( + "legalize-tuple-element-access", + "Remove xla_hlo::GetTupleElementOp commands wherever possible."); + +} // namespace iree_compiler +} // namespace mlir diff --git a/compiler/Transforms/LegalizeTypeStorage.cpp b/compiler/Transforms/LegalizeTypeStorage.cpp new file mode 100644 index 000000000000..0e14aa1f05ff --- /dev/null +++ b/compiler/Transforms/LegalizeTypeStorage.cpp @@ -0,0 +1,145 @@ +// Copyright 2019 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "third_party/llvm/llvm/include/llvm/ADT/DenseSet.h" +#include "third_party/llvm/llvm/projects/google_mlir/include/mlir/Dialect/StandardOps/Ops.h" +#include "third_party/llvm/llvm/projects/google_mlir/include/mlir/IR/BlockAndValueMapping.h" +#include "third_party/llvm/llvm/projects/google_mlir/include/mlir/IR/Builders.h" +#include "third_party/llvm/llvm/projects/google_mlir/include/mlir/IR/Location.h" +#include "third_party/llvm/llvm/projects/google_mlir/include/mlir/IR/MLIRContext.h" +#include "third_party/llvm/llvm/projects/google_mlir/include/mlir/IR/StandardTypes.h" +#include "third_party/llvm/llvm/projects/google_mlir/include/mlir/Pass/Pass.h" +#include "third_party/llvm/llvm/projects/google_mlir/include/mlir/Pass/PassRegistry.h" +#include "third_party/llvm/llvm/projects/google_mlir/include/mlir/Transforms/Utils.h" +#include "third_party/mlir_edge/iree/compiler/Utils/MemRefUtils.h" + +namespace mlir { +namespace iree_compiler { + +namespace { + +bool convertOperation(Operation *oldOp, OpBuilder *builder, + BlockAndValueMapping *mapping) { + OperationState state(oldOp->getLoc(), oldOp->getName()); + if (oldOp->getNumSuccessors() == 0) { + // Non-branching operations can just add all the operands. + for (auto *oldOperand : oldOp->getOperands()) { + state.operands.push_back(mapping->lookupOrDefault(oldOperand)); + } + } else { + // We add the operands separated by nullptr's for each successor. + unsigned firstSuccOperand = oldOp->getNumSuccessors() + ? oldOp->getSuccessorOperandIndex(0) + : oldOp->getNumOperands(); + auto opOperands = oldOp->getOpOperands(); + unsigned i = 0; + for (; i != firstSuccOperand; ++i) { + state.operands.push_back(mapping->lookupOrDefault(opOperands[i].get())); + } + for (unsigned succ = 0, e = oldOp->getNumSuccessors(); succ != e; ++succ) { + state.successors.push_back( + mapping->lookupOrDefault(oldOp->getSuccessor(succ))); + // Add sentinel to delineate successor operands. + state.operands.push_back(nullptr); + // Remap the successors operands. + for (auto *operand : oldOp->getSuccessorOperands(succ)) { + state.operands.push_back(mapping->lookupOrDefault(operand)); + } + } + } + for (const auto &oldType : oldOp->getResultTypes()) { + state.types.push_back(legalizeType(oldType)); + } + state.attributes = {oldOp->getAttrs().begin(), oldOp->getAttrs().end()}; + auto newOp = builder->createOperation(state); + for (int i = 0; i < newOp->getNumResults(); ++i) { + mapping->map(oldOp->getResult(i), newOp->getResult(i)); + } + return false; +} + +bool convertFunction(FuncOp oldFunction, FuncOp newFunction) { + OpBuilder builder(newFunction.getBody()); + BlockAndValueMapping mapping; + + // Create new blocks matching the expected arguments of the old ones. + // This sets up the block mappings to enable us to reference blocks forward + // during conversion. + newFunction.getBlocks().clear(); + for (auto &oldBlock : oldFunction.getBlocks()) { + auto *newBlock = builder.createBlock(&newFunction.getBody()); + mapping.map(&oldBlock, newBlock); + for (auto *oldArg : oldBlock.getArguments()) { + auto *newArg = newBlock->addArgument(legalizeType(oldArg->getType())); + mapping.map(oldArg, newArg); + } + } + + // Convert all ops in the blocks. + for (auto &oldBlock : oldFunction.getBlocks()) { + builder.setInsertionPointToEnd(mapping.lookupOrNull(&oldBlock)); + for (auto &oldOp : oldBlock.getOperations()) { + if (convertOperation(&oldOp, &builder, &mapping)) { + return true; + } + } + } + + return false; +} + +} // namespace + +class LegalizeTypeStoragePass : public ModulePass { + public: + void runOnModule() override { + auto module = getModule(); + + // Build a list of (oldFunction, newFunction) for all functions we need to + // replace. This will ensure that when we go to convert function bodies we + // have only new functions defined. + std::vector> convertedFunctions; + + for (auto oldFunction : module.getOps()) { + // Create the replacement function, ensuring that we copy attributes. + auto newFunction = FuncOp::create( + oldFunction.getLoc(), oldFunction.getName(), + legalizeType(oldFunction.getType()).cast(), + oldFunction.getDialectAttrs()); + convertedFunctions.push_back({oldFunction, newFunction}); + + // Perform the actual body conversion now that we have proper signatures. + if (convertFunction(oldFunction, newFunction)) { + return signalPassFailure(); + } + } + + // Replace functions in the module. + for (auto &pair : convertedFunctions) { + pair.first.erase(); + module.push_back(pair.second); + } + } +}; + +std::unique_ptr> createLegalizeTypeStoragePass() { + return std::make_unique(); +} + +static PassRegistration pass( + "iree-legalize-type-storage", + "Legalizes types to ones supported by the IREE VM."); + +} // namespace iree_compiler +} // namespace mlir diff --git a/compiler/Transforms/Passes.h b/compiler/Transforms/Passes.h new file mode 100644 index 000000000000..6a5aae5e5e92 --- /dev/null +++ b/compiler/Transforms/Passes.h @@ -0,0 +1,76 @@ +// Copyright 2019 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#ifndef THIRD_PARTY_MLIR_EDGE_IREE_COMPILER_TRANSFORMS_TRANSFORMS_H_ +#define THIRD_PARTY_MLIR_EDGE_IREE_COMPILER_TRANSFORMS_TRANSFORMS_H_ + +#include "third_party/llvm/llvm/projects/google_mlir/include/mlir/Pass/Pass.h" + +namespace mlir { +namespace iree_compiler { + +//===----------------------------------------------------------------------===// +// Tensor <-> MemRef and Type Legalization +//===----------------------------------------------------------------------===// + +// Converts all function signatures to use memref's for args/results. +std::unique_ptr> +createConvertToMemRefCallingConventionPass(); + +// Legalizes tensor-to-memref conversions by deduping identity operations. +std::unique_ptr> createLegalizeTensorMemRefPass(); + +// Removes all uses of XLA Tuples as args/results from functions by expanding +// them (recursively) into individual args/results. +std::unique_ptr> +createConvertFromTupleCallingConventionPass(); + +// Removes all uses of XLA Tuples by evaluating their selection statically. +std::unique_ptr> createLegalizeTupleElementAccessPass(); + +// Legalizes all types to ones supported by the IREE VM. +std::unique_ptr> createLegalizeTypeStoragePass(); + +//===----------------------------------------------------------------------===// +// Cleanup and Dead Code Elimination +//===----------------------------------------------------------------------===// + +// Aggressively eliminates ops that have no side-effects. +std::unique_ptr> createAggressiveOpEliminationPass(); + +// Drops functions from the module that are unreachable from any exported +// functions (iree.module.export). +std::unique_ptr> +createDropUnreachableModuleFunctionsPass(); + +// Drops functions from the executable module that are unreachable from any +// exported functions (iree.executable.export). +std::unique_ptr> +createDropUnreachableExecutableFunctionsPass(); + +// Drops all executables in a module that are not reachable by any dispatch +// sequencer op. +std::unique_ptr> createDropUnusedExecutablesPass(); + +//===----------------------------------------------------------------------===// +// Module Analysis and Assignment +//===----------------------------------------------------------------------===// + +// Assigns module-unique ordinals to functions within the module. +std::unique_ptr> createAssignFunctionOrdinalsPass(); + +} // namespace iree_compiler +} // namespace mlir + +#endif // THIRD_PARTY_MLIR_EDGE_IREE_COMPILER_TRANSFORMS_TRANSFORMS_H_ diff --git a/compiler/Transforms/Sequencer/AssignExecutableOrdinals.cpp b/compiler/Transforms/Sequencer/AssignExecutableOrdinals.cpp new file mode 100644 index 000000000000..3d2f470f37e7 --- /dev/null +++ b/compiler/Transforms/Sequencer/AssignExecutableOrdinals.cpp @@ -0,0 +1,75 @@ +// Copyright 2019 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "third_party/llvm/llvm/include/llvm/ADT/DenseMap.h" +#include "third_party/llvm/llvm/projects/google_mlir/include/mlir/IR/Attributes.h" +#include "third_party/llvm/llvm/projects/google_mlir/include/mlir/IR/Builders.h" +#include "third_party/llvm/llvm/projects/google_mlir/include/mlir/IR/MLIRContext.h" +#include "third_party/llvm/llvm/projects/google_mlir/include/mlir/Pass/Pass.h" +#include "third_party/llvm/llvm/projects/google_mlir/include/mlir/Pass/PassRegistry.h" +#include "third_party/llvm/llvm/projects/google_mlir/include/mlir/Support/LLVM.h" +#include "third_party/llvm/llvm/projects/google_mlir/include/mlir/Support/LogicalResult.h" +#include "third_party/llvm/llvm/projects/google_mlir/include/mlir/Transforms/Utils.h" +#include "third_party/mlir_edge/iree/compiler/IR/StructureOps.h" +#include "third_party/mlir_edge/iree/compiler/Utils/OpUtils.h" + +namespace mlir { +namespace iree_compiler { + +class AssignExecutableOrdinalsPass + : public ModulePass { + public: + void runOnModule() override { + Builder builder(getModule()); + int nextExecutableOrdinal = 0; + for (auto multiArchExecutableOp : + getModule().getOps()) { + multiArchExecutableOp.setAttr( + "iree.ordinal", builder.getI32IntegerAttr(nextExecutableOrdinal++)); + + // We'll scan for all entry points in the first executable. Then on all + // other executables we can reuse the ordinals (ensuring that iteration + // order does not matter). + llvm::DenseMap entryPointMap; + for (auto executableOp : + multiArchExecutableOp.getBlock().getOps()) { + executableOp.setAttr("iree.ordinal", + multiArchExecutableOp.getAttr("iree.ordinal")); + int nextEntryPointOrdinal = 0; + for (auto funcOp : executableOp.getInnerModule().getOps()) { + if (!funcOp.getAttr("iree.executable.export")) continue; + auto it = entryPointMap.find(funcOp.getName()); + if (it == entryPointMap.end()) { + funcOp.setAttr("iree.ordinal", + builder.getI32IntegerAttr(nextEntryPointOrdinal++)); + entryPointMap.insert({funcOp.getName(), funcOp}); + } else { + funcOp.setAttr("iree.ordinal", it->second.getAttr("iree.ordinal")); + } + } + } + } + } +}; + +std::unique_ptr> createAssignExecutableOrdinalsPass() { + return std::make_unique(); +} + +static PassRegistration pass( + "iree-assign-executable-ordinals", + "Assigns executable and entry point ordinals"); + +} // namespace iree_compiler +} // namespace mlir diff --git a/compiler/Transforms/Sequencer/AssignExecutableWorkloadAttrs.cpp b/compiler/Transforms/Sequencer/AssignExecutableWorkloadAttrs.cpp new file mode 100644 index 000000000000..5eba4feb9e43 --- /dev/null +++ b/compiler/Transforms/Sequencer/AssignExecutableWorkloadAttrs.cpp @@ -0,0 +1,125 @@ +// Copyright 2019 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "third_party/llvm/llvm/include/llvm/ADT/StringMap.h" +#include "third_party/llvm/llvm/projects/google_mlir/include/mlir/IR/Attributes.h" +#include "third_party/llvm/llvm/projects/google_mlir/include/mlir/IR/Builders.h" +#include "third_party/llvm/llvm/projects/google_mlir/include/mlir/IR/MLIRContext.h" +#include "third_party/llvm/llvm/projects/google_mlir/include/mlir/Pass/Pass.h" +#include "third_party/llvm/llvm/projects/google_mlir/include/mlir/Pass/PassRegistry.h" +#include "third_party/llvm/llvm/projects/google_mlir/include/mlir/Support/LLVM.h" +#include "third_party/llvm/llvm/projects/google_mlir/include/mlir/Support/LogicalResult.h" +#include "third_party/llvm/llvm/projects/google_mlir/include/mlir/Transforms/Utils.h" +#include "third_party/mlir_edge/iree/compiler/IR/Sequencer/LLOps.h" +#include "third_party/mlir_edge/iree/compiler/IR/StructureOps.h" +#include "third_party/mlir_edge/iree/compiler/Utils/OpUtils.h" + +namespace mlir { +namespace iree_compiler { + +namespace { + +struct WorkloadInfo { + SmallVector staticWorkloads; + SmallVector dynamicWorkloads; +}; + +// Finds all dispatches and records their workload attributes mapped by +// (executable ordinal, entry point ordinal). +llvm::StringMap> gatherExecutableWorkloadInfos( + ModuleOp moduleOp) { + llvm::StringMap> workloadInfos; + for (auto funcOp : moduleOp.getOps()) { + funcOp.walk([&](IREESeq::LL::DynamicDispatchOp op) { + auto &workloadInfo = + workloadInfos[op.getExecutable()][op.getEntryPoint()]; + workloadInfo.dynamicWorkloads.push_back(op.getWorkload()); + }); + funcOp.walk([&](IREESeq::LL::StaticDispatchOp op) { + auto &workloadInfo = + workloadInfos[op.getExecutable()][op.getEntryPoint()]; + for (auto existingWorkloadAttr : workloadInfo.staticWorkloads) { + if (existingWorkloadAttr == op.getWorkload()) { + return; // Already present, ignore. + } + } + workloadInfo.staticWorkloads.push_back(op.getWorkload()); + }); + } + return workloadInfos; +} + +// Adds attributes to the given executable entry point describing the workload +// info to the backends that will be processing them. +LogicalResult attributeExecutableEntryPointWorkload( + FuncOp entryPointOp, const WorkloadInfo &workloadInfo) { + if (!workloadInfo.dynamicWorkloads.empty()) { + return entryPointOp.emitError() << "Dynamic workloads not yet supported"; + } + if (workloadInfo.staticWorkloads.size() != 1) { + return entryPointOp.emitError() << "Static workload sizes differ in shape"; + } + + // Easy because we just support static workloads now. + // When this code is adapted to support dynamic workloads we'll want to put + // a pair of attrs describing which dimensions may be static and which args + // have the dynamic values to reference. + entryPointOp.setAttr("iree.executable.workload", + workloadInfo.staticWorkloads.front()); + + return success(); +} + +} // namespace + +class AssignExecutableWorkloadAttrsPass + : public ModulePass { + public: + void runOnModule() override { + Builder builder(getModule()); + + // Find all dispatches and capture their workload information. + // We store this information by executable and then entry point ordinal. + auto executableWorkloadInfos = gatherExecutableWorkloadInfos(getModule()); + + // Process each executable with the workload information. + for (auto &executableIt : executableWorkloadInfos) { + auto multiArchExecutableOp = cast( + getModule().lookupSymbol(executableIt.first())); + for (auto executableOp : + multiArchExecutableOp.getBlock().getOps()) { + for (auto &entryPointIt : executableIt.second) { + auto funcOp = cast( + executableOp.getInnerModule().lookupSymbol(entryPointIt.first())); + if (failed(attributeExecutableEntryPointWorkload( + funcOp, entryPointIt.second))) { + return signalPassFailure(); + } + } + } + } + } +}; + +std::unique_ptr> +createAssignExecutableWorkloadAttrsPass() { + return std::make_unique(); +} + +static PassRegistration pass( + "iree-assign-executable-workload-attrs", + "Assigns executable entrypoint workload attributes"); + +} // namespace iree_compiler +} // namespace mlir diff --git a/compiler/Transforms/Sequencer/FoldCompatibleDispatchRegions.cpp b/compiler/Transforms/Sequencer/FoldCompatibleDispatchRegions.cpp new file mode 100644 index 000000000000..0114734a9341 --- /dev/null +++ b/compiler/Transforms/Sequencer/FoldCompatibleDispatchRegions.cpp @@ -0,0 +1,63 @@ +// Copyright 2019 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "third_party/llvm/llvm/include/llvm/ADT/ArrayRef.h" +#include "third_party/llvm/llvm/include/llvm/ADT/DenseMap.h" +#include "third_party/llvm/llvm/include/llvm/ADT/DenseSet.h" +#include "third_party/llvm/llvm/include/llvm/ADT/SetVector.h" +#include "third_party/llvm/llvm/include/llvm/ADT/SmallVector.h" +#include "third_party/llvm/llvm/projects/google_mlir/include/mlir/Dialect/StandardOps/Ops.h" +#include "third_party/llvm/llvm/projects/google_mlir/include/mlir/IR/Attributes.h" +#include "third_party/llvm/llvm/projects/google_mlir/include/mlir/IR/BlockAndValueMapping.h" +#include "third_party/llvm/llvm/projects/google_mlir/include/mlir/IR/Builders.h" +#include "third_party/llvm/llvm/projects/google_mlir/include/mlir/IR/Location.h" +#include "third_party/llvm/llvm/projects/google_mlir/include/mlir/IR/MLIRContext.h" +#include "third_party/llvm/llvm/projects/google_mlir/include/mlir/IR/StandardTypes.h" +#include "third_party/llvm/llvm/projects/google_mlir/include/mlir/Pass/Pass.h" +#include "third_party/llvm/llvm/projects/google_mlir/include/mlir/Pass/PassRegistry.h" +#include "third_party/llvm/llvm/projects/google_mlir/include/mlir/Support/LLVM.h" +#include "third_party/llvm/llvm/projects/google_mlir/include/mlir/Support/LogicalResult.h" +#include "third_party/llvm/llvm/projects/google_mlir/include/mlir/Transforms/Utils.h" +#include "third_party/mlir_edge/iree/compiler/IR/Ops.h" +#include "third_party/mlir_edge/iree/compiler/Utils/DispatchUtils.h" + +namespace mlir { +namespace iree_compiler { + +// Identifies dispatch regions that have compatible workloads and folds them. +// This relies on CSE having deduped workloads to simplify the logic to simply +// looking for dispatch regions using the same values. +class FoldCompatibleDispatchRegionsPass + : public FunctionPass { + public: + void runOnFunction() override { + auto func = getFunction(); + for (auto &block : func) { + if (failed(mergeBlockDispatchRegions(func, &block))) { + return signalPassFailure(); + } + } + } +}; + +std::unique_ptr> createFoldCompatibleDispatchRegionsPass() { + return std::make_unique(); +} + +static PassRegistration pass( + "iree-fold-compatible-dispatch-regions", + "Folds dispatch regions that have compatible workloads."); + +} // namespace iree_compiler +} // namespace mlir diff --git a/compiler/Transforms/Sequencer/IdentifyDispatchRegions.cpp b/compiler/Transforms/Sequencer/IdentifyDispatchRegions.cpp new file mode 100644 index 000000000000..a0a911ef0892 --- /dev/null +++ b/compiler/Transforms/Sequencer/IdentifyDispatchRegions.cpp @@ -0,0 +1,259 @@ +// Copyright 2019 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include + +#include "third_party/llvm/llvm/include/llvm/ADT/ArrayRef.h" +#include "third_party/llvm/llvm/include/llvm/ADT/DenseMap.h" +#include "third_party/llvm/llvm/include/llvm/ADT/DenseSet.h" +#include "third_party/llvm/llvm/include/llvm/ADT/STLExtras.h" +#include "third_party/llvm/llvm/include/llvm/ADT/SetVector.h" +#include "third_party/llvm/llvm/include/llvm/ADT/SmallVector.h" +#include "third_party/llvm/llvm/projects/google_mlir/include/mlir/Dialect/StandardOps/Ops.h" +#include "third_party/llvm/llvm/projects/google_mlir/include/mlir/IR/Attributes.h" +#include "third_party/llvm/llvm/projects/google_mlir/include/mlir/IR/BlockAndValueMapping.h" +#include "third_party/llvm/llvm/projects/google_mlir/include/mlir/IR/Builders.h" +#include "third_party/llvm/llvm/projects/google_mlir/include/mlir/IR/Location.h" +#include "third_party/llvm/llvm/projects/google_mlir/include/mlir/IR/MLIRContext.h" +#include "third_party/llvm/llvm/projects/google_mlir/include/mlir/IR/StandardTypes.h" +#include "third_party/llvm/llvm/projects/google_mlir/include/mlir/Pass/Pass.h" +#include "third_party/llvm/llvm/projects/google_mlir/include/mlir/Pass/PassRegistry.h" +#include "third_party/llvm/llvm/projects/google_mlir/include/mlir/Support/LLVM.h" +#include "third_party/llvm/llvm/projects/google_mlir/include/mlir/Support/LogicalResult.h" +#include "third_party/llvm/llvm/projects/google_mlir/include/mlir/Transforms/Utils.h" +#include "third_party/mlir_edge/iree/compiler/IR/Ops.h" +#include "third_party/mlir_edge/iree/compiler/Utils/DispatchUtils.h" +#include "third_party/tensorflow/compiler/mlir/xla/ir/hlo_ops.h" + +namespace mlir { +namespace iree_compiler { + +namespace { + +// Returns true if the given |op| can be dispatched in all cases. +// Other passes may handle special cases of these ops but this initial +// identification is conservative. +bool isDispatchableOp(Operation *op) { + if (op->getDialect() && op->getDialect()->getNamespace().startswith("iree")) { + // Ignore things we've already produced as they should only relate to + // sequencer operations. + return false; + } else if (op->isKnownTerminator()) { + // Currently we skip all terminators as we want to leave them in the block + // to keep it valid. Future folding passes may take care of them if they are + // worth bringing into the dispatch region. + return false; + } else if (isa(op)) { + // This may be handled by a control-flow folding pass later once we have + // done our initial analysis and know what functions are compatible. + return false; + } else if (isa(op)) { + // Indirect calls are not supported in dispatch code. + return false; + } else if (isa(op)) { + // Allocations are sequencer ops. + // Note that we could support static allocations (convert to stack/etc). + return false; + } else if (isa(op)) { + // Constants are handled in the RematerializeDispatchConstants pass. + // We do that independently so that we can more easily see the use of + // constants across all dispatches instead of just on an individual basis + // as we do here. + return false; + } else if (isa(op)) { + // TODO(benvanik): lower these to the sequencer dialect prior to ID'ing. + return false; + } + return true; +} + +// Returns true if the given |op| can have other ops fused into it. +// This is sketchy and it'd be nice to define this as an op property instead. +// +// What we are looking for in foldable ops is whether the execution of the op +// when fused has some possible benefit (or at least, a non-negative cost). +// Eventually we want to allow backends to vote on this and allow multiple +// folding strategies within the same executable. For now we just hardcode what +// we know for the ops we have. +// +// Preconditions: isDispatchableOp(op) == true. +bool isFusionRootOp(Operation *op) { + if (isa(op) || isa(op)) { + // We have hand-written kernels for these right now we want to stand alone. + // When we do a bit more magic we should allow these ops to fold. + return false; + } + return true; +} + +// Returns true if the given |op| can be fused into other ops. +// +// Ops that perform narrowing on shapes (such as reduction ops) should not +// generally be fused with other downstream ops (probably...). This avoids +// potential oversampling and indexing issues and allows backends to perform +// more efficient rooted cascading reduction dispatches. +// +// Preconditions: isDispatchableOp(op) == true. +bool isFusableOp(Operation *op) { + if (isa(op) || isa(op)) { + return false; + } else if (isa(op)) { + // Reduction is usually a dedicated root operation - we can shove things in + // the front of it but not behind. + return false; + } + return true; +} + +// Puts all of the |unsortedOps| into |sortedOps| in an arbitrary topological +// order. +// https://en.wikipedia.org/wiki/Topological_sorting#Depth-first_search +// +// Preconditions: |unsortedOps| has no cycles within the set of ops. +std::vector sortOpsTopologically( + const llvm::SetVector &unsortedOps) { + llvm::SetVector unmarkedOps; + unmarkedOps.insert(unsortedOps.begin(), unsortedOps.end()); + llvm::SetVector markedOps; + + using VisitFn = std::function; + VisitFn visit = [&](Operation *op) { + if (markedOps.count(op) > 0) return; + for (auto *result : op->getResults()) { + for (auto *user : result->getUsers()) { + // Don't visit ops not in our set. + if (unsortedOps.count(user) == 0) continue; + visit(user); + } + } + markedOps.insert(op); + }; + + while (!unmarkedOps.empty()) { + auto *op = unmarkedOps.pop_back_val(); + visit(op); + } + + auto sortedOps = markedOps.takeVector(); + std::reverse(sortedOps.begin(), sortedOps.end()); + return sortedOps; +} + +// Recursively traverses the IR DAG along the operand edges to find ops we are +// able to fuse and appends them to |subgraph|. +void gatherFusionOps(Operation *op, llvm::SetVector *subgraph) { + // Skip ops that are used outside of the subgraph we are building. + for (auto *result : op->getResults()) { + if (result->use_empty() || result->hasOneUse()) continue; + for (auto *user : result->getUsers()) { + if (subgraph->count(user) == 0) { + // Op that consumes the result is not (yet) in the subgraph. + // For now we'll ignore these as it may represent a fork that we don't + // want to join too early. + return; + } + } + } + + // Walk backward up to ops providing our input operands. + for (auto *operand : op->getOperands()) { + auto *sourceOp = operand->getDefiningOp(); + if (!sourceOp) continue; + if (subgraph->count(sourceOp) == 0) { + if (isDispatchableOp(sourceOp) && isFusableOp(sourceOp)) { + gatherFusionOps(sourceOp, subgraph); + } + } + } + + subgraph->insert(op); +} + +// Finds all ops that can be fused together with the given |rootOp| by searching +// backwards in the op order through input edges. +// Returns a topologically sorted list of all fused ops with |rootOp| at the +// end. +std::vector findFusionSubgraphFromRoot(Operation *rootOp) { + if (!isFusionRootOp(rootOp)) { + return {rootOp}; + } + llvm::SetVector subgraph; + subgraph.insert(rootOp); + gatherFusionOps(rootOp, &subgraph); + return sortOpsTopologically(subgraph); +} + +// Identifies ranges of dispatchable ops and moves them into dispatch regions. +LogicalResult identifyBlockDispatchRegions(FuncOp func, Block *block) { + // Fixed point iteration until we can no longer fuse anything. + bool didFindAnyNewRegions; + do { + // Iterate in reverse so we root further along in the op list. + didFindAnyNewRegions = false; + for (auto &rootOp : llvm::reverse(*block)) { + if (!isDispatchableOp(&rootOp)) { + // Op should remain at the sequencer level. + continue; + } + + // Attempt to find all operations, including rootOp, that can be fused. + // The ops will be sorted in topological order with rootOp as the last op. + // Worst case we may end up with a subgraph of only the rootOp. + auto fusedSubgraph = findFusionSubgraphFromRoot(&rootOp); + + // Compute the workload based on the output shape. + // When variadic all output shapes match so we can just take the first. + auto *workload = calculateWorkload(&rootOp, rootOp.getResult(0)); + + // Try to build a dispatch region from this root. + if (failed(buildDispatchRegion(func, block, workload, fusedSubgraph))) { + return failure(); + } + + // Successfully created a dispatch region from the ops and we must now + // start over again as we've likely trashed the whole block structure. + didFindAnyNewRegions = true; + break; + } + } while (didFindAnyNewRegions); + return success(); +} + +} // namespace + +// Identifies dispatchable ops and moves them into iree.dispatch_regions. +// Some ops, such as call, will be deferred until following passes. +class IdentifyDispatchRegionsPass + : public FunctionPass { + public: + void runOnFunction() override { + auto func = getFunction(); + for (auto &block : func) { + if (failed(identifyBlockDispatchRegions(func, &block))) { + return signalPassFailure(); + } + } + } +}; + +std::unique_ptr> createIdentifyDispatchRegionsPass() { + return std::make_unique(); +} + +static PassRegistration pass( + "iree-identify-dispatch-regions", + "Conservatively identifies dispatch regions in functions."); + +} // namespace iree_compiler +} // namespace mlir diff --git a/compiler/Transforms/Sequencer/IdentifyReductionRegions.cpp b/compiler/Transforms/Sequencer/IdentifyReductionRegions.cpp new file mode 100644 index 000000000000..1e6f5d772011 --- /dev/null +++ b/compiler/Transforms/Sequencer/IdentifyReductionRegions.cpp @@ -0,0 +1,163 @@ +// Copyright 2019 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include + +#include "third_party/llvm/llvm/include/llvm/ADT/ArrayRef.h" +#include "third_party/llvm/llvm/include/llvm/ADT/DenseMap.h" +#include "third_party/llvm/llvm/include/llvm/ADT/DenseSet.h" +#include "third_party/llvm/llvm/include/llvm/ADT/STLExtras.h" +#include "third_party/llvm/llvm/include/llvm/ADT/SetVector.h" +#include "third_party/llvm/llvm/include/llvm/ADT/SmallVector.h" +#include "third_party/llvm/llvm/projects/google_mlir/include/mlir/Dialect/StandardOps/Ops.h" +#include "third_party/llvm/llvm/projects/google_mlir/include/mlir/IR/Attributes.h" +#include "third_party/llvm/llvm/projects/google_mlir/include/mlir/IR/BlockAndValueMapping.h" +#include "third_party/llvm/llvm/projects/google_mlir/include/mlir/IR/Builders.h" +#include "third_party/llvm/llvm/projects/google_mlir/include/mlir/IR/Location.h" +#include "third_party/llvm/llvm/projects/google_mlir/include/mlir/IR/MLIRContext.h" +#include "third_party/llvm/llvm/projects/google_mlir/include/mlir/IR/StandardTypes.h" +#include "third_party/llvm/llvm/projects/google_mlir/include/mlir/Pass/Pass.h" +#include "third_party/llvm/llvm/projects/google_mlir/include/mlir/Pass/PassRegistry.h" +#include "third_party/llvm/llvm/projects/google_mlir/include/mlir/Support/LLVM.h" +#include "third_party/llvm/llvm/projects/google_mlir/include/mlir/Support/LogicalResult.h" +#include "third_party/llvm/llvm/projects/google_mlir/include/mlir/Transforms/Utils.h" +#include "third_party/mlir_edge/iree/compiler/IR/Ops.h" +#include "third_party/mlir_edge/iree/compiler/IR/Types.h" +#include "third_party/mlir_edge/iree/compiler/Utils/DispatchUtils.h" +#include "third_party/tensorflow/compiler/mlir/xla/ir/hlo_ops.h" + +namespace mlir { +namespace iree_compiler { + +namespace { + +// Builds a new iree.reduction_region with the given |invocationRegion|. +// The new region will be inserted after |originalOp|. +// +// All |invocationRegion| ops must be compatible with the |workload| specified +// as they will all be dispatched with the same workgroup structure. The +// |invocationRegion| will not be modified. +LogicalResult buildReductionRegion(Operation *originalOp, + ArrayRef operands, + ArrayRef initialValues, + ArrayRef dimensions, + Region &invocationRegion) { + OpBuilder parentBuilder(originalOp); + + // Compute the workload based on the output shape. + // When variadic all output shapes match so we can just take the first. + auto *workload = calculateWorkload(originalOp, originalOp->getResult(0)); + + // Build the region op and add it to the parent block. + SmallVector resultTypes{originalOp->getResultTypes()}; + auto reductionRegionOp = parentBuilder.create( + originalOp->getLoc(), resultTypes, workload, operands, initialValues, + dimensions); + + // Create the block and setup the arg mapping for captured values. + BlockAndValueMapping mapping; + invocationRegion.cloneInto(&reductionRegionOp.getBody(), mapping); + + // Replace xla_hlo.return -> iree.return. + OpBuilder regionBuilder(reductionRegionOp.getBody()); + reductionRegionOp.walk([&](xla_hlo::ReturnOp returnOp) { + regionBuilder.setInsertionPoint(returnOp); + SmallVector returnValues(returnOp.getOperands()); + regionBuilder.create(returnOp.getLoc(), returnValues); + returnOp.erase(); + }); + + // Replace usage of values with the results of the region. + for (int i = 0; i < originalOp->getNumResults(); ++i) { + originalOp->getResult(i)->replaceAllUsesWith( + reductionRegionOp.getResult(i)); + } + + return success(); +} + +// Converts an xla_hlo::ReduceOp to a reduction region and inlines the target +// computation into the region body. +LogicalResult buildReductionRegionFromXLAReduceOp(xla_hlo::ReduceOp reduceOp) { + SmallVector operands(reduceOp.getOperands()); + OperandAdaptor adaptor(operands); + + SmallVector dimensions; + for (auto dim : reduceOp.dimensions().getIntValues()) { + dimensions.push_back(dim.getSExtValue()); + } + + // Create the iree.reduction_region. + if (failed(buildReductionRegion(reduceOp, adaptor.operands(), + adaptor.init_values(), dimensions, + reduceOp.body()))) { + return failure(); + } + + // Remove original XLA reduction op. + reduceOp.erase(); + + return success(); +} + +// Identifies reduction ops and moves them into reduction regions. +LogicalResult identifyBlockReductionRegions(FuncOp funcOp, Block *block) { + // Fixed point iteration until we can no longer fuse anything. + bool didFindAnyNewRegions; + do { + // Iterate in reverse so we root further along in the op list. + didFindAnyNewRegions = false; + for (auto &rootOp : llvm::reverse(*block)) { + if (auto reduceOp = dyn_cast(rootOp)) { + if (failed(buildReductionRegionFromXLAReduceOp(reduceOp))) { + return failure(); + } + + // Successfully created a dispatch region from the ops and we must now + // start over again as we've likely trashed the whole block structure. + didFindAnyNewRegions = true; + break; + } + } + } while (didFindAnyNewRegions); + return success(); +} + +} // namespace + +// Identifies reduction ops and moves their targets into iree.reduction_regions. +class IdentifyReductionRegionsPass + : public ModulePass { + public: + void runOnModule() override { + for (auto funcOp : getModule().getOps()) { + for (auto &block : funcOp) { + if (failed(identifyBlockReductionRegions(funcOp, &block))) { + return signalPassFailure(); + } + } + } + } +}; + +std::unique_ptr> createIdentifyReductionRegionsPass() { + return std::make_unique(); // NOLINT +} + +static PassRegistration pass( + "iree-identify-reduction-regions", + "Identifies reduction regions based on input reduction ops."); + +} // namespace iree_compiler +} // namespace mlir diff --git a/compiler/Transforms/Sequencer/LoadStoreDataFlowOpt.cpp b/compiler/Transforms/Sequencer/LoadStoreDataFlowOpt.cpp new file mode 100644 index 000000000000..740a324db1b7 --- /dev/null +++ b/compiler/Transforms/Sequencer/LoadStoreDataFlowOpt.cpp @@ -0,0 +1,137 @@ +// Copyright 2019 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include + +#include "third_party/llvm/llvm/include/llvm/ADT/ArrayRef.h" +#include "third_party/llvm/llvm/include/llvm/ADT/SetVector.h" +#include "third_party/llvm/llvm/include/llvm/ADT/SmallVector.h" +#include "third_party/llvm/llvm/include/llvm/ADT/iterator_range.h" +#include "third_party/llvm/llvm/projects/google_mlir/include/mlir/Dialect/StandardOps/Ops.h" +#include "third_party/llvm/llvm/projects/google_mlir/include/mlir/IR/BlockAndValueMapping.h" +#include "third_party/llvm/llvm/projects/google_mlir/include/mlir/IR/Builders.h" +#include "third_party/llvm/llvm/projects/google_mlir/include/mlir/IR/MLIRContext.h" +#include "third_party/llvm/llvm/projects/google_mlir/include/mlir/IR/StandardTypes.h" +#include "third_party/llvm/llvm/projects/google_mlir/include/mlir/Pass/Pass.h" +#include "third_party/llvm/llvm/projects/google_mlir/include/mlir/Pass/PassRegistry.h" +#include "third_party/llvm/llvm/projects/google_mlir/include/mlir/Transforms/Utils.h" +#include "third_party/mlir_edge/iree/compiler/IR/Ops.h" +#include "third_party/mlir_edge/iree/compiler/IR/Sequencer/HLOps.h" +#include "third_party/mlir_edge/iree/compiler/Utils/MemRefUtils.h" +#include "third_party/mlir_edge/iree/compiler/Utils/OpUtils.h" + +namespace mlir { +namespace iree_compiler { + +namespace { + +// Returns a value containing the indices in the form of a memref with shape +// {|indices|.size()}. +Value *makeIndicesMemRef(const MemRefType &type, + Operation::operand_range indices, OpBuilder &builder) { + auto &useOp = *builder.getInsertionPoint(); + size_t indicesCount = std::distance(indices.begin(), indices.end()); + if (indicesCount == 0) { + return builder + .create( + useOp.getLoc(), builder.getMemRefType({1}, builder.getIndexType()), + builder.getIntegerAttr(builder.getIndexType(), 0)) + .getResult(); + } else if (indicesCount == 1) { + return loadAccessValue(useOp.getLoc(), *indices.begin(), builder); + } + + // TODO(benvanik): support arbitrary indices. + useOp.emitError() << "Multiple indices are not yet implemented"; + return nullptr; +} + +// Returns a value containing the lengths in the form of a memref with shape +// {|dims|.size()}. +Value *makeLengthsMemRef(Value *storedValue, OpBuilder &builder) { + Type valueType = storedValue->getType(); + if (auto shapedType = valueType.dyn_cast()) { + return builder.create(storedValue->getLoc(), + storedValue); + } else { + return builder.create( + storedValue->getLoc(), + builder.getMemRefType({1}, builder.getIndexType()), + builder.getIntegerAttr(builder.getIndexType(), 1)); + } +} + +// Returns the origin operation of a value if it is a load. +LoadOp findOriginLoadOperation(Value *value) { + // TODO(benvanik): follow through identity ops or something? + if (auto loadOp = dyn_cast(value->getDefiningOp())) { + return loadOp; + } + return nullptr; +} + +// Inserts a copy operation performing the same work as a store. +// +// Example: +// %0 = ... : memref<4xf32> +// %1 = load %0[%offset] : memref<4xf32> +// %2 = ... : memref +// store %1, %2[] : memref +// -> +// %0 = ... : memref<4xf32> +// %2 = ... : memref +// iree_hl_interp.copy %0[%offset], %2[], [%length] +void insertCopyForStore(LoadOp &loadOp, StoreOp &storeOp) { + OpBuilder builder(storeOp); + auto *srcIndices = + makeIndicesMemRef(loadOp.getMemRefType(), loadOp.getIndices(), builder); + auto *dstIndices = + makeIndicesMemRef(storeOp.getMemRefType(), storeOp.getIndices(), builder); + auto *lengths = makeLengthsMemRef(storeOp.getValueToStore(), builder); + builder.create(storeOp.getLoc(), loadOp.getMemRef(), + srcIndices, storeOp.getMemRef(), + dstIndices, lengths); +} + +} // namespace + +class LoadStoreDataFlowOptPass : public FunctionPass { + public: + void runOnFunction() override { + auto func = getFunction(); + + // Find stores and attempt to optimize load+store pairs. + llvm::SetVector deadOperations; + func.walk([&](StoreOp storeOp) { + if (auto loadOp = findOriginLoadOperation(storeOp.getValueToStore())) { + insertCopyForStore(loadOp, storeOp); + deadOperations.insert(storeOp); + } + }); + + // Remove all the now-unused ops. + removeDeadOperations(deadOperations); + } +}; + +std::unique_ptr> createLoadStoreDataFlowOptPass() { + return std::make_unique(); +} + +static PassRegistration pass( + "iree-load-store-data-flow-opt", + "Optimize local load and store data flow by removing redundant accesses"); + +} // namespace iree_compiler +} // namespace mlir diff --git a/compiler/Transforms/Sequencer/LowerSequencerDialect.cpp b/compiler/Transforms/Sequencer/LowerSequencerDialect.cpp new file mode 100644 index 000000000000..9c910e4f2828 --- /dev/null +++ b/compiler/Transforms/Sequencer/LowerSequencerDialect.cpp @@ -0,0 +1,255 @@ +// Copyright 2019 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "third_party/llvm/llvm/include/llvm/ADT/ArrayRef.h" +#include "third_party/llvm/llvm/include/llvm/ADT/DenseMap.h" +#include "third_party/llvm/llvm/include/llvm/ADT/SmallVector.h" +#include "third_party/llvm/llvm/projects/google_mlir/include/mlir/Dialect/StandardOps/Ops.h" +#include "third_party/llvm/llvm/projects/google_mlir/include/mlir/IR/Attributes.h" +#include "third_party/llvm/llvm/projects/google_mlir/include/mlir/IR/BlockAndValueMapping.h" +#include "third_party/llvm/llvm/projects/google_mlir/include/mlir/IR/Builders.h" +#include "third_party/llvm/llvm/projects/google_mlir/include/mlir/IR/Location.h" +#include "third_party/llvm/llvm/projects/google_mlir/include/mlir/IR/MLIRContext.h" +#include "third_party/llvm/llvm/projects/google_mlir/include/mlir/IR/OperationSupport.h" +#include "third_party/llvm/llvm/projects/google_mlir/include/mlir/IR/PatternMatch.h" +#include "third_party/llvm/llvm/projects/google_mlir/include/mlir/IR/StandardTypes.h" +#include "third_party/llvm/llvm/projects/google_mlir/include/mlir/Pass/Pass.h" +#include "third_party/llvm/llvm/projects/google_mlir/include/mlir/Pass/PassRegistry.h" +#include "third_party/llvm/llvm/projects/google_mlir/include/mlir/Support/LogicalResult.h" +#include "third_party/llvm/llvm/projects/google_mlir/include/mlir/Transforms/DialectConversion.h" +#include "third_party/llvm/llvm/projects/google_mlir/include/mlir/Transforms/Utils.h" +#include "third_party/mlir_edge/iree/compiler/IR/Dialect.h" +#include "third_party/mlir_edge/iree/compiler/IR/Ops.h" +#include "third_party/mlir_edge/iree/compiler/IR/Sequencer/HLDialect.h" +#include "third_party/mlir_edge/iree/compiler/IR/Sequencer/HLOps.h" +#include "third_party/mlir_edge/iree/compiler/IR/Sequencer/LLDialect.h" +#include "third_party/mlir_edge/iree/compiler/IR/Sequencer/LLOps.h" +#include "third_party/mlir_edge/iree/compiler/IR/StructureOps.h" + +namespace mlir { +namespace iree_compiler { + +namespace { + +// Returns an integer scalar memref containing the offset specified by |indices| +// within |type|. +Value *computeOffset(Location loc, Value *reference, Value *indices, + OpBuilder &builder) { + auto referenceType = reference->getType().cast(); + auto *shapeMemRef = builder + .create( + loc, + builder.getMemRefType({referenceType.getRank()}, + builder.getIntegerType(32)), + ArrayRef{}) + .getResult(); + builder.create(loc, reference, shapeMemRef); + auto *resultMemRef = + builder + .create( + loc, builder.getMemRefType({}, builder.getIntegerType(32)), + ArrayRef{}) + .getResult(); + auto elementSizeAttr = builder.getIntegerAttr( + builder.getIntegerType(8), referenceType.getElementTypeBitWidth() / 8); + builder.create( + loc, shapeMemRef, elementSizeAttr, indices, resultMemRef); + return resultMemRef; +} + +// Returns a tuple of (offset, length) integer scalar memrefs with the range +// specified by |indices| and |lengths| within |type|. +std::pair computeRange(Location loc, Value *reference, + Value *indices, Value *lengths, + OpBuilder &builder) { + auto referenceType = reference->getType().cast(); + auto *shapeMemRef = builder + .create( + loc, + builder.getMemRefType({referenceType.getRank()}, + builder.getIntegerType(32)), + ArrayRef{}) + .getResult(); + builder.create(loc, reference, shapeMemRef); + auto *offsetMemRef = + builder + .create( + loc, builder.getMemRefType({}, builder.getIntegerType(32)), + ArrayRef{}) + .getResult(); + auto *lengthMemRef = + builder + .create( + loc, builder.getMemRefType({}, builder.getIntegerType(32)), + ArrayRef{}) + .getResult(); + auto elementSizeAttr = builder.getIntegerAttr( + builder.getIntegerType(8), referenceType.getElementTypeBitWidth() / 8); + builder.create(loc, shapeMemRef, elementSizeAttr, + indices, lengths, offsetMemRef, + lengthMemRef); + return {offsetMemRef, lengthMemRef}; +} + +struct LowerSliceOpPattern : public OpRewritePattern { + using OpRewritePattern::OpRewritePattern; + + PatternMatchResult matchAndRewrite(IREESeq::HL::SliceOp op, + PatternRewriter &rewriter) const { + auto range = computeRange(op.getLoc(), op.src(), op.indices(), op.lengths(), + rewriter); + rewriter.replaceOpWithNewOp( + op, ArrayRef{op.getResult()->getType()}, + ArrayRef{op.src(), range.first, range.second}, op.getAttrs()); + return matchSuccess(); + } +}; + +struct LowerShapeOpPattern : public OpRewritePattern { + using OpRewritePattern::OpRewritePattern; + + PatternMatchResult matchAndRewrite(IREESeq::HL::ShapeOp op, + PatternRewriter &rewriter) const { + auto *shapeMemRef = + rewriter + .create( + op.getLoc(), + rewriter.getMemRefType( + {op.getResult()->getType().cast().getRank()}, + rewriter.getIntegerType(64)), + ArrayRef{}) + .getResult(); + op.replaceAllUsesWith(shapeMemRef); + rewriter.replaceOpWithNewOp(op, op.getOperand(), + shapeMemRef); + return matchSuccess(); + } +}; + +struct LowerCopyOpPattern : public OpRewritePattern { + using OpRewritePattern::OpRewritePattern; + + PatternMatchResult matchAndRewrite(IREESeq::HL::CopyOp op, + PatternRewriter &rewriter) const { + auto *srcOffsetMemRef = + computeOffset(op.getLoc(), op.src(), op.srcIndices(), rewriter); + auto dstRange = computeRange(op.getLoc(), op.dst(), op.dstIndices(), + op.lengths(), rewriter); + rewriter.replaceOpWithNewOp( + op, op.src(), srcOffsetMemRef, op.dst(), dstRange.first, + dstRange.second); + return matchSuccess(); + } +}; + +struct LowerFillOpPattern : public OpRewritePattern { + using OpRewritePattern::OpRewritePattern; + + PatternMatchResult matchAndRewrite(IREESeq::HL::FillOp op, + PatternRewriter &rewriter) const { + auto dstRange = computeRange(op.getLoc(), op.dst(), op.dstIndices(), + op.lengths(), rewriter); + rewriter.replaceOpWithNewOp( + op, op.value(), op.dst(), dstRange.first, dstRange.second); + return matchSuccess(); + } +}; + +struct LowerBranchOpPattern : public OpRewritePattern { + using OpRewritePattern::OpRewritePattern; + + PatternMatchResult matchAndRewrite(IREESeq::HL::BranchOp op, + PatternRewriter &rewriter) const { + SmallVector operands{op.getOperation()->getOperands()}; + + rewriter.replaceOpWithNewOp(op, op.getDest(), + operands); + return matchSuccess(); + } +}; + +struct LowerCondCondBranchOpPattern + : public OpRewritePattern { + using OpRewritePattern::OpRewritePattern; + + PatternMatchResult matchAndRewrite(IREESeq::HL::CondBranchOp op, + PatternRewriter &rewriter) const { + SmallVector trueOperands{op.getTrueOperands()}; + SmallVector falseOperands{op.getFalseOperands()}; + + rewriter.replaceOpWithNewOp( + op, op.getCondition(), op.getTrueDest(), trueOperands, + op.getFalseDest(), falseOperands); + return matchSuccess(); + } +}; + +// Rewrites an op into one with all the same operands, results, and attributes. +// Operands and results in the ops must have the same order and attributes must +// have the same name. They must also be constructed properly by the default +// builders. +template +struct LowerIdenticalOpPattern : public OpRewritePattern { + using OpRewritePattern::OpRewritePattern; + + PatternMatchResult matchAndRewrite(SRC op, PatternRewriter &rewriter) const { + SmallVector resultTypes{op.getOperation()->getResultTypes()}; + SmallVector operands{op.getOperation()->getOperands()}; + + rewriter.replaceOpWithNewOp(op, resultTypes, operands, op.getAttrs()); + return this->matchSuccess(); + } +}; + +} // namespace + +class LowerSequencerDialectPass + : public FunctionPass { + public: + void runOnFunction() override { + OwningRewritePatternList patterns; + patterns.insert< + LowerIdenticalOpPattern, + LowerIdenticalOpPattern, + LowerShapeOpPattern, LowerCopyOpPattern, LowerSliceOpPattern, + LowerBranchOpPattern, LowerCondCondBranchOpPattern>(&getContext()); +#define IDENTICAL_OP_LOWERING(op_name) \ + LowerIdenticalOpPattern + patterns.insert< + IDENTICAL_OP_LOWERING(AllocHeapOp), IDENTICAL_OP_LOWERING(CloneOp), + IDENTICAL_OP_LOWERING(ReshapeOp), IDENTICAL_OP_LOWERING(CallOp), + IDENTICAL_OP_LOWERING(ReturnOp)>(&getContext()); +#undef IDENTICAL_OP_LOWERING + + ConversionTarget target(getContext()); + target.addLegalDialect(); + target.addLegalOp(); + + if (failed(applyFullConversion(getFunction(), target, patterns))) { + return signalPassFailure(); + } + } +}; + +std::unique_ptr> createLowerSequencerDialectPass() { + return std::make_unique(); +} + +static PassRegistration pass( + "iree-lower-sequencer-dialect", + "Lowers the IREE HL sequencer dialect to the LL sequencer dialect."); + +} // namespace iree_compiler +} // namespace mlir diff --git a/compiler/Transforms/Sequencer/LowerStdToSequencerDialect.cpp b/compiler/Transforms/Sequencer/LowerStdToSequencerDialect.cpp new file mode 100644 index 000000000000..e2a81d87ad9f --- /dev/null +++ b/compiler/Transforms/Sequencer/LowerStdToSequencerDialect.cpp @@ -0,0 +1,262 @@ +// Copyright 2019 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "third_party/llvm/llvm/include/llvm/ADT/ArrayRef.h" +#include "third_party/llvm/llvm/include/llvm/ADT/DenseMap.h" +#include "third_party/llvm/llvm/include/llvm/ADT/SmallVector.h" +#include "third_party/llvm/llvm/projects/google_mlir/include/mlir/Dialect/StandardOps/Ops.h" +#include "third_party/llvm/llvm/projects/google_mlir/include/mlir/IR/Attributes.h" +#include "third_party/llvm/llvm/projects/google_mlir/include/mlir/IR/BlockAndValueMapping.h" +#include "third_party/llvm/llvm/projects/google_mlir/include/mlir/IR/Builders.h" +#include "third_party/llvm/llvm/projects/google_mlir/include/mlir/IR/Location.h" +#include "third_party/llvm/llvm/projects/google_mlir/include/mlir/IR/MLIRContext.h" +#include "third_party/llvm/llvm/projects/google_mlir/include/mlir/IR/OperationSupport.h" +#include "third_party/llvm/llvm/projects/google_mlir/include/mlir/IR/StandardTypes.h" +#include "third_party/llvm/llvm/projects/google_mlir/include/mlir/Pass/Pass.h" +#include "third_party/llvm/llvm/projects/google_mlir/include/mlir/Pass/PassRegistry.h" +#include "third_party/llvm/llvm/projects/google_mlir/include/mlir/Support/LogicalResult.h" +#include "third_party/llvm/llvm/projects/google_mlir/include/mlir/Transforms/DialectConversion.h" +#include "third_party/llvm/llvm/projects/google_mlir/include/mlir/Transforms/Utils.h" +#include "third_party/mlir_edge/iree/compiler/IR/Dialect.h" +#include "third_party/mlir_edge/iree/compiler/IR/Ops.h" +#include "third_party/mlir_edge/iree/compiler/IR/Sequencer/HLDialect.h" +#include "third_party/mlir_edge/iree/compiler/IR/Sequencer/HLOps.h" +#include "third_party/mlir_edge/iree/compiler/IR/StructureOps.h" +#include "third_party/mlir_edge/iree/compiler/Utils/MemRefUtils.h" + +namespace mlir { +namespace iree_compiler { + +namespace { + +class SequencerConversionPattern : public ConversionPattern { + public: + SequencerConversionPattern(StringRef operationName, int benefit, + MLIRContext *context, + MemRefTypeConverter &typeConverter) + : ConversionPattern(operationName, benefit, context), + typeConverter_(typeConverter) {} + + protected: + MemRefTypeConverter &typeConverter_; +}; + +struct ConstantOpLowering : public SequencerConversionPattern { + ConstantOpLowering(MLIRContext *context, MemRefTypeConverter &typeConverter) + : SequencerConversionPattern(ConstantOp::getOperationName(), 1, context, + typeConverter) {} + + PatternMatchResult matchAndRewrite( + Operation *op, ArrayRef operands, + ConversionPatternRewriter &rewriter) const override { + const auto &valueAttr = cast(op).getValue(); + auto midOp = rewriter.create(op->getLoc(), valueAttr); + + auto result = wrapAsTensor(midOp.getResult(), op, rewriter); + rewriter.replaceOp( + op, {loadResultValue(op->getLoc(), op->getResult(0)->getType(), result, + rewriter)}); + return matchSuccess(); + } +}; + +class CallOpLowering : public SequencerConversionPattern { + public: + CallOpLowering(MLIRContext *context, MemRefTypeConverter &typeConverter) + : SequencerConversionPattern(CallOp::getOperationName(), 1, context, + typeConverter) {} + + PatternMatchResult matchAndRewrite( + Operation *op, ArrayRef operands, + ConversionPatternRewriter &rewriter) const override { + auto callOp = cast(op); + + SmallVector convertedResults; + auto result = typeConverter_.convertTypes( + callOp.getCalleeType().getResults(), convertedResults); + (void)result; + assert(succeeded(result) && "expected valid callee type conversion"); + rewriter.replaceOpWithNewOp( + op, callOp.getCallee(), convertedResults, operands); + + return matchSuccess(); + } +}; + +class CallIndirectOpLowering : public SequencerConversionPattern { + public: + CallIndirectOpLowering(MLIRContext *context, + MemRefTypeConverter &typeConverter) + : SequencerConversionPattern(CallIndirectOp::getOperationName(), 1, + context, typeConverter) {} + + PatternMatchResult matchAndRewrite( + Operation *op, ArrayRef operands, + ConversionPatternRewriter &rewriter) const override { + auto callOp = cast(op); + rewriter.replaceOpWithNewOp( + op, callOp.getCallee(), operands); + return matchSuccess(); + } +}; + +struct ReturnOpLowering : public SequencerConversionPattern { + ReturnOpLowering(MLIRContext *context, MemRefTypeConverter &typeConverter) + : SequencerConversionPattern(ReturnOp::getOperationName(), 1, context, + typeConverter) {} + + PatternMatchResult matchAndRewrite( + Operation *op, ArrayRef operands, + ConversionPatternRewriter &rewriter) const override { + SmallVector newOperands; + newOperands.reserve(operands.size()); + for (auto *operand : operands) { + newOperands.push_back(wrapAsMemRef(operand, op, rewriter)); + } + rewriter.replaceOpWithNewOp(op, newOperands); + return matchSuccess(); + } +}; + +struct BranchOpLowering : public SequencerConversionPattern { + BranchOpLowering(MLIRContext *context, MemRefTypeConverter &typeConverter) + : SequencerConversionPattern(BranchOp::getOperationName(), 1, context, + typeConverter) {} + + PatternMatchResult matchAndRewrite( + Operation *op, ArrayRef properOperands, + ArrayRef destinations, ArrayRef> operands, + ConversionPatternRewriter &rewriter) const override { + rewriter.replaceOpWithNewOp(op, destinations[0], + operands[0]); + return this->matchSuccess(); + } +}; + +struct CondBranchOpLowering : public SequencerConversionPattern { + CondBranchOpLowering(MLIRContext *context, MemRefTypeConverter &typeConverter) + : SequencerConversionPattern(CondBranchOp::getOperationName(), 1, context, + typeConverter) {} + + PatternMatchResult matchAndRewrite( + Operation *op, ArrayRef properOperands, + ArrayRef destinations, ArrayRef> operands, + ConversionPatternRewriter &rewriter) const override { + auto *condValue = + loadAccessValue(op->getLoc(), properOperands[0], rewriter); + rewriter.replaceOpWithNewOp( + op, condValue, destinations[IREESeq::HL::CondBranchOp::trueIndex], + operands[IREESeq::HL::CondBranchOp::trueIndex], + destinations[IREESeq::HL::CondBranchOp::falseIndex], + operands[IREESeq::HL::CondBranchOp::falseIndex]); + return this->matchSuccess(); + } +}; + +class AllocOpLowering : public SequencerConversionPattern { + public: + AllocOpLowering(MLIRContext *context, MemRefTypeConverter &typeConverter) + : SequencerConversionPattern(AllocOp::getOperationName(), 1, context, + typeConverter) {} + + PatternMatchResult matchAndRewrite( + Operation *op, ArrayRef operands, + ConversionPatternRewriter &rewriter) const override { + // TODO(benvanik): replace with length computation. + rewriter.replaceOpWithNewOp( + op, *op->getResultTypes().begin(), operands); + return matchSuccess(); + } +}; + +class DeallocOpLowering : public SequencerConversionPattern { + public: + DeallocOpLowering(MLIRContext *context, MemRefTypeConverter &typeConverter) + : SequencerConversionPattern(DeallocOp::getOperationName(), 1, context, + typeConverter) {} + + PatternMatchResult matchAndRewrite( + Operation *op, ArrayRef operands, + ConversionPatternRewriter &rewriter) const override { + rewriter.replaceOpWithNewOp(op, operands[0]); + return matchSuccess(); + } +}; + +void populateStdToSequencerConversionPatterns( + MLIRContext *context, MemRefTypeConverter &converter, + OwningRewritePatternList &patterns) { + patterns.insert(context, converter); +} + +} // namespace + +// Lowers functions using std.* ops to the IREE HL sequencer dialect and buffer +// view types. +// FuncOp signatures will be updated to use the buffer view type and +// dispatch regions will get iree.bind_input where needed. +// +// Beyond bindings there will be no other changes within dispatchable regions. +// It is up to the downstream dialects to properly use the bindings to map their +// I/O to expected values. +// +// Note that output buffer allocation is required following this pass to either +// elide dispatch results entirely and provide output params or provide both +// while ensuring that the returned value is always sliced from an input. This +// should happen prior to outlining. +class LowerStdToSequencerDialectPass + : public ModulePass { + public: + void runOnModule() override { + auto module = getModule(); + + // Only convert top-level functions, not ones nested in executables. + std::vector toConvert; + for (auto funcOp : module.getOps()) { + toConvert.push_back(funcOp); + } + + // Convert the signature and body of all sequencer functions. + MemRefTypeConverter converter(&getContext()); + ConversionTarget target(getContext()); + target.addLegalDialect(); + target.addLegalOp(); + target.addDynamicallyLegalOp( + [&](FuncOp op) { return converter.isSignatureLegal(op.getType()); }); + + OwningRewritePatternList patterns; + populateStdToSequencerConversionPatterns(&getContext(), converter, + patterns); + if (failed( + applyPartialConversion(toConvert, target, patterns, &converter))) { + return signalPassFailure(); + } + } +}; + +std::unique_ptr> createLowerStdToSequencerDialectPass() { + return std::make_unique(); +} + +static PassRegistration pass( + "iree-lower-std-to-sequencer-dialect", + "Lowers std ops to the IREE HL sequencer dialect."); + +} // namespace iree_compiler +} // namespace mlir diff --git a/compiler/Transforms/Sequencer/LowerXLAToSequencerDialect.cpp b/compiler/Transforms/Sequencer/LowerXLAToSequencerDialect.cpp new file mode 100644 index 000000000000..4b3fae6b5199 --- /dev/null +++ b/compiler/Transforms/Sequencer/LowerXLAToSequencerDialect.cpp @@ -0,0 +1,262 @@ +// Copyright 2019 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "third_party/llvm/llvm/include/llvm/ADT/ArrayRef.h" +#include "third_party/llvm/llvm/include/llvm/ADT/SmallVector.h" +#include "third_party/llvm/llvm/projects/google_mlir/include/mlir/IR/Attributes.h" +#include "third_party/llvm/llvm/projects/google_mlir/include/mlir/IR/Builders.h" +#include "third_party/llvm/llvm/projects/google_mlir/include/mlir/IR/Function.h" +#include "third_party/llvm/llvm/projects/google_mlir/include/mlir/IR/Operation.h" +#include "third_party/llvm/llvm/projects/google_mlir/include/mlir/IR/PatternMatch.h" +#include "third_party/llvm/llvm/projects/google_mlir/include/mlir/IR/StandardTypes.h" +#include "third_party/llvm/llvm/projects/google_mlir/include/mlir/IR/TypeUtilities.h" +#include "third_party/llvm/llvm/projects/google_mlir/include/mlir/IR/Value.h" +#include "third_party/llvm/llvm/projects/google_mlir/include/mlir/Pass/Pass.h" +#include "third_party/llvm/llvm/projects/google_mlir/include/mlir/Transforms/DialectConversion.h" +#include "third_party/mlir_edge/iree/compiler/IR/Dialect.h" +#include "third_party/mlir_edge/iree/compiler/IR/Ops.h" +#include "third_party/mlir_edge/iree/compiler/IR/Sequencer/HLDialect.h" +#include "third_party/mlir_edge/iree/compiler/IR/Sequencer/HLOps.h" +#include "third_party/mlir_edge/iree/compiler/IR/StructureOps.h" +#include "third_party/mlir_edge/iree/compiler/Transforms/ConversionUtils.h" +#include "third_party/mlir_edge/iree/compiler/Utils/MemRefUtils.h" +#include "third_party/tensorflow/compiler/mlir/xla/ir/hlo_ops.h" + +namespace mlir { +namespace iree_compiler { + +namespace { + +// TODO(suderman): tablegen this? or something a bit more flexible. + +#define UNARY_OP_LOWERING(XlaOpType, IREEOpType) \ + struct XlaOpType##Lowering \ + : public UnaryOpLowering { \ + using UnaryOpLowering::UnaryOpLowering; \ + }; + +#define TERNARY_OP_LOWERING(XlaOpType, IREEOpType) \ + struct XlaOpType##Lowering \ + : public TernaryOpLowering { \ + using TernaryOpLowering::TernaryOpLowering; \ + }; + +UNARY_OP_LOWERING(CopyOp, IREESeq::HL::CloneOp); + +#undef UNARY_OP_LOWERING +#undef TERNARY_OP_LOWERING + +static ElementsAttr elementsAttrFromArray(ConversionPatternRewriter &rewriter, + ArrayRef elements) { + return rewriter.getDenseIntElementsAttr( + rewriter.getTensorType(elements.size(), rewriter.getIntegerType(64)), + elements); +} + +static IREE::ConstantOp createArrayConstant(ConversionPatternRewriter &rewriter, + Location loc, + llvm::ArrayRef elements) { + auto shapeAttr = elementsAttrFromArray(rewriter, elements); + return rewriter.create(loc, shapeAttr); +} + +template +static Operation *createShapeTargetingOp(ConversionPatternRewriter &rewriter, + Location loc, Value *input, + MemRefType targetType) { + auto shapeOp = createArrayConstant(rewriter, loc, targetType.getShape()); + return rewriter.create(loc, targetType, input, shapeOp); +} + +static Value *inputAsMemref(ConversionPatternRewriter &rewriter, Operation *op, + Value *tensor) { + return wrapAsMemRef(loadAccessValue(op->getLoc(), tensor, rewriter), op, + rewriter); +} + +static MemRefType getFinalType(ConversionPatternRewriter &rewriter, + Value *result) { + return getMemRefType(result, rewriter).cast(); +} + +template +class XlaOpLowering : public ConversionPattern { + public: + explicit XlaOpLowering(MLIRContext *context) + : ConversionPattern(SrcOp::getOperationName(), 1, context) {} + + PatternMatchResult matchAndRewrite( + Operation *op, ArrayRef operands, + ConversionPatternRewriter &rewriter) const override { + auto srcOp = cast(op); + + SmallVector memrefOperands; + for (auto operand : operands) { + memrefOperands.push_back(inputAsMemref(rewriter, op, operand)); + } + + auto dstOp = rewriteInternal(&srcOp, memrefOperands, rewriter); + rewriter.replaceOp(op, wrapAsTensor(dstOp->getResult(0), srcOp, rewriter)); + return this->matchSuccess(); + } + + protected: + virtual Operation *rewriteInternal( + SrcOp *op, ArrayRef operands, + ConversionPatternRewriter &rewriter) const { + llvm_unreachable("unimplemented rewrite, did you mean rewriteTerminator?"); + } +}; + +struct ConcatOpLowering : public XlaOpLowering { + using XlaOpLowering::XlaOpLowering; + + Operation *rewriteInternal( + xla_hlo::ConcatenateOp *op, ArrayRef operands, + ConversionPatternRewriter &rewriter) const override { + auto finalType = getFinalType(rewriter, *op); + + return rewriter.create( + op->getLoc(), finalType, operands, + rewriter.getI32IntegerAttr(op->dimension().getZExtValue())); + } +}; + +struct DynamicUpdateSliceLowering + : public XlaOpLowering { + using XlaOpLowering::XlaOpLowering; + + Operation *rewriteInternal( + xla_hlo::DynamicUpdateSliceOp *op, ArrayRef operands, + ConversionPatternRewriter &rewriter) const override { + auto operand = operands[0]; + auto update = operands[1]; + + auto updateType = update->getType().cast(); + Value *lengthConstant = + createArrayConstant(rewriter, op->getLoc(), updateType.getShape()); + + auto startIndices = makeArrayRef(operands).drop_front(2); + const int rank = startIndices.size(); + llvm::SmallVector valuesToConcat; + valuesToConcat.reserve(startIndices.size()); + auto type = getElementTypeOrSelf(startIndices.front()); + + // To generate the offset matrix we need to convert the variadic tensors + // into a reshaped and concated value. + for (auto index : startIndices) { + auto reshapedIndex = rewriter.create( + op->getLoc(), rewriter.getMemRefType({1}, type), index, + createArrayConstant(rewriter, op->getLoc(), {1})); + valuesToConcat.push_back(reshapedIndex); + } + + auto dstOffset = rewriter + .create( + op->getLoc(), rewriter.getMemRefType({rank}, type), + valuesToConcat, rewriter.getI32IntegerAttr(0)) + .getResult(); + + llvm::SmallVector zero_offset; + zero_offset.resize(updateType.getRank(), 0); + auto srcOffset = createArrayConstant(rewriter, op->getLoc(), zero_offset); + + auto copiedOperand = rewriter.create( + op->getLoc(), operand->getType(), operand); + + rewriter + .create(op->getLoc(), update, srcOffset, + copiedOperand, dstOffset, lengthConstant) + .getOperation(); + + return copiedOperand; + } +}; + +struct SliceLowering : public XlaOpLowering { + using XlaOpLowering::XlaOpLowering; + Operation *rewriteInternal( + xla_hlo::SliceOp *op, ArrayRef operands, + ConversionPatternRewriter &rewriter) const override { + // XLA slice has value semantics, whereas the IREE slice creates a view. We + // lower it to a copy if all strides are one which may be transformed to a + // slice by later optimizations. + auto isNotOne = [](APInt stride) { return stride != 1; }; + if (llvm::any_of(op->strides(), isNotOne)) { + op->emitRemark() << "Could not lower slice op with non-singular strides"; + return nullptr; + } + + auto finalType = getFinalType(rewriter, *op); + + auto src = operands[0]; + std::vector dim_pieces; + auto dst = rewriter.create(op->getLoc(), + finalType, dim_pieces); + auto srcIndices = + rewriter.create(op->getLoc(), op->start_indices()); + auto lengths = + createArrayConstant(rewriter, op->getLoc(), finalType.getShape()); + + llvm::SmallVector zero_offset; + zero_offset.resize(finalType.getRank(), 0); + auto dstIndices = createArrayConstant(rewriter, op->getLoc(), zero_offset); + + rewriter.create(op->getLoc(), src, srcIndices, dst, + dstIndices, lengths); + return dst; + } +}; + +struct ReshapeOpLowering : public XlaOpLowering { + using XlaOpLowering::XlaOpLowering; + + Operation *rewriteInternal( + xla_hlo::ReshapeOp *op, ArrayRef operands, + ConversionPatternRewriter &rewriter) const override { + return createShapeTargetingOp( + rewriter, op->getLoc(), operands[0], getFinalType(rewriter, *op)); + } +}; + +class LowerXLAToSequencerDialectPass + : public FunctionPass { + public: + void runOnFunction() override { + ConversionTarget target(getContext()); + target.addLegalDialect(); + target.addLegalOp(); + + OwningRewritePatternList patterns; + patterns + .insert(&getContext()); + if (failed(applyPartialConversion(getFunction(), target, patterns))) { + return signalPassFailure(); + } + } +}; + +} // namespace + +std::unique_ptr> createLowerXLAToSequencerDialectPass() { + return std::make_unique(); +} + +static PassRegistration pass( + "iree-lower-xla-to-iree-sequencer", + "Convert all supported XLA ops to the IREE sequencer dialect"); + +} // namespace iree_compiler +} // namespace mlir diff --git a/compiler/Transforms/Sequencer/OutlineDispatchRegions.cpp b/compiler/Transforms/Sequencer/OutlineDispatchRegions.cpp new file mode 100644 index 000000000000..e1005f167611 --- /dev/null +++ b/compiler/Transforms/Sequencer/OutlineDispatchRegions.cpp @@ -0,0 +1,241 @@ +// Copyright 2019 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include + +#include "third_party/llvm/llvm/include/llvm/ADT/SetVector.h" +#include "third_party/llvm/llvm/projects/google_mlir/include/mlir/IR/Attributes.h" +#include "third_party/llvm/llvm/projects/google_mlir/include/mlir/IR/Builders.h" +#include "third_party/llvm/llvm/projects/google_mlir/include/mlir/IR/MLIRContext.h" +#include "third_party/llvm/llvm/projects/google_mlir/include/mlir/Pass/Pass.h" +#include "third_party/llvm/llvm/projects/google_mlir/include/mlir/Pass/PassRegistry.h" +#include "third_party/llvm/llvm/projects/google_mlir/include/mlir/Support/LLVM.h" +#include "third_party/llvm/llvm/projects/google_mlir/include/mlir/Support/LogicalResult.h" +#include "third_party/llvm/llvm/projects/google_mlir/include/mlir/Transforms/Utils.h" +#include "third_party/mlir_edge/iree/compiler/IR/Ops.h" +#include "third_party/mlir_edge/iree/compiler/IR/Sequencer/HLOps.h" +#include "third_party/mlir_edge/iree/compiler/IR/StructureOps.h" +#include "third_party/mlir_edge/iree/compiler/IR/Types.h" +#include "third_party/mlir_edge/iree/compiler/Utils/DispatchUtils.h" +#include "third_party/mlir_edge/iree/compiler/Utils/MemRefUtils.h" + +namespace mlir { +namespace iree_compiler { + +namespace { + +// Inserts a load from a wrapped memref (as inserted via insertDispatcherStore). +// Returns the value in the original type. +Value *insertDispatcheeLoad(Operation *op, Type originalType, Value *value, + OpBuilder *builder) { + // If old value was a memref we don't need to change anything. + if (originalType.isa()) { + return value; + } + + auto loadInputOp = + builder->create(op->getLoc(), originalType, value); + value->replaceAllUsesWith(loadInputOp.getResult()); + loadInputOp.setOperand(value); + return loadInputOp.getResult(); +} + +// Marshals args and results as buffers for the given region. +// Beyond inserting the appropriate tensor-to-memref ops we avoid mutating the +// interior of the dispatch region as much as possible. +LogicalResult marshalDispatchSite(IREE::DispatchRegionOp regionOp) { + auto &entryBlock = regionOp.getBody().getBlocks().front(); + OpBuilder dispatcherBuilder(regionOp); + OpBuilder dispatcheeBuilder(&entryBlock, entryBlock.begin()); + + // Wrap input operands and unwrap in the entry block. + SmallVector newArgs; + for (int i = 0; i < regionOp.getNumArgOperands(); ++i) { + // Wrap the input outside of the region. + auto *blockArg = entryBlock.getArgument(i); + Type originalType = blockArg->getType(); + auto *originalArg = regionOp.getArgOperand(i); + auto *wrappedArg = + insertDispatcherStore(regionOp, originalArg, &dispatcherBuilder); + newArgs.push_back(wrappedArg); + blockArg->setType(wrappedArg->getType()); + + // Unwrap the block arg value and replace all of the uses with the newly + // unwrapped value. + insertDispatcheeLoad(regionOp, originalType, blockArg, &dispatcheeBuilder); + } + + // Allocate output arguments and replace the return values with those. + SmallVector newResults; + SmallVector, 8> resultIndicesToOutputArgs; + SmallVector deadResultIndices; + SmallVector, 8> replacedResults; + for (int i = 0; i < regionOp.getNumResults(); ++i) { + auto *result = regionOp.getResult(i); + auto convertedType = convertTypeToMemRef(result->getType()); + + // Allocate output buffer in the dispatcher to pass in to the region. + Value *allocatedValue = allocateDispatchOutputBuffer( + regionOp.getLoc(), convertedType, dispatcherBuilder); + if (!allocatedValue) { + regionOp.emitError("unable to allocate result value"); + return failure(); + } + newArgs.push_back(allocatedValue); + + auto *newBlockArg = entryBlock.addArgument(allocatedValue->getType()); + resultIndicesToOutputArgs.push_back({i, newBlockArg}); + + // NOTE: right now we always replace results. If we want to allow return + // values we can avoid killing them here. + deadResultIndices.push_back(i); + replacedResults.push_back({result, allocatedValue}); + } + + // Remove dead results from return statements. + regionOp.walk([&](IREE::ReturnOp returnOp) { + // Replace the results we were returning with stores to output arguments. + OpBuilder builder(returnOp); + for (auto resultToArg : resultIndicesToOutputArgs) { + auto *value = returnOp.getOperand(resultToArg.first); + auto *outputArg = resultToArg.second; + builder.create(returnOp.getLoc(), value, outputArg); + } + + // Filter out the results that are now dead. + SmallVector newOperands(returnOp.getOperands()); + for (int i = deadResultIndices.size() - 1; i >= 0; --i) { + newOperands.erase(newOperands.begin() + deadResultIndices[i]); + } + returnOp.getOperation()->setOperands(newOperands); + }); + + // Clone the region op with the new args/results. + auto newRegionOp = dispatcherBuilder.create( + regionOp.getLoc(), newResults, regionOp.getWorkload(), newArgs); + newRegionOp.getBody().takeBody(regionOp.getBody()); + + // Marshal back the results by replacing uses of the original with loads from + // the new output arg. + for (auto &it : replacedResults) { + insertDispatcherLoad(regionOp, it.first, it.second, &dispatcherBuilder); + } + + // Remove original region. + regionOp.erase(); + + return success(); +} + +// Converts a dispatch_region into a dispatch to the outlined region function. +LogicalResult convertToDispatchOp(IREE::DispatchRegionOp regionOp, + IREE::MultiArchExecutableOp executable, + FuncOp entryPoint) { + // Insert at the same place as the original region. + OpBuilder dispatcherBuilder(regionOp); + + // Ensure workload is a memref. + auto *workload = + wrapAsMemRef(regionOp.getWorkload(), regionOp, dispatcherBuilder); + + // Create the dispatch op to the executable function. + SmallVector operandValues(regionOp.getArgOperands()); + auto dispatchOp = dispatcherBuilder.create( + regionOp.getLoc(), executable.getName(), entryPoint.getName(), workload, + entryPoint.getType().getResults(), operandValues); + + // Replace uses of the existing results with the new results. + for (int i = 0; i < regionOp.getNumResults(); ++i) { + regionOp.getResult(i)->replaceAllUsesWith(dispatchOp.getResult(i)); + } + + // Erase original region. + regionOp.erase(); + + return success(); +} + +// Outlines a dispatch region into an iree.multi_arch_executable. +LogicalResult outlineDispatchRegion(IREE::DispatchRegionOp regionOp, + int outlinedRegionOrdinal) { + // Build function type matching 1:1 with the region signature. + SmallVector operandTypes; + for (auto *arg : regionOp.getArgOperands()) { + operandTypes.push_back(arg->getType()); + } + SmallVector resultTypes(regionOp.getResultTypes()); + auto functionType = + FunctionType::get(operandTypes, resultTypes, regionOp.getContext()); + + // Create the executable with the region cloned into it. + IREE::MultiArchExecutableOp multiArchExecutable; + FuncOp outlinedFunc; + std::tie(multiArchExecutable, outlinedFunc) = createRegionExecutable( + regionOp, functionType, + "_dispatch_" + std::to_string(outlinedRegionOrdinal)); + outlinedFunc.setAttr("iree.executable.export", + UnitAttr::get(regionOp.getContext())); + + // Finally convert the dispatch region into a dispatch to the outlined func. + return convertToDispatchOp(regionOp, multiArchExecutable, outlinedFunc); +} + +} // namespace + +class OutlineDispatchRegionsPass + : public ModulePass { + public: + void runOnModule() override { + auto module = getModule(); + + ModuleManager moduleManager(module); + std::vector funcOps(module.getOps().begin(), + module.getOps().end()); + for (auto func : funcOps) { + // Perform marshaling of the dispatcher and dispatchee I/O. + // This inserts the required stores and loads to make everything memrefs + // and adds the iree.load_input/iree.store_output ops to the dispatchee. + if (func.walk([&](IREE::DispatchRegionOp op) { + if (failed(marshalDispatchSite(op))) { + return WalkResult::interrupt(); + } + return WalkResult::advance(); + }) + .wasInterrupted()) { + return signalPassFailure(); + } + + // Outline all of the iree.dispatch_region ops in this function. + std::vector dispatchRegionOps; + func.walk( + [&](IREE::DispatchRegionOp op) { dispatchRegionOps.push_back(op); }); + for (int i = 0; i < dispatchRegionOps.size(); ++i) { + if (failed(outlineDispatchRegion(dispatchRegionOps[i], i))) { + return signalPassFailure(); + } + } + } + } +}; + +std::unique_ptr> createOutlineDispatchRegionsPass() { + return std::make_unique(); +} + +static PassRegistration pass( + "iree-outline-dispatch-regions", + "Outlines dispatch regions into standalone functions"); + +} // namespace iree_compiler +} // namespace mlir diff --git a/compiler/Transforms/Sequencer/OutlineReductionRegions.cpp b/compiler/Transforms/Sequencer/OutlineReductionRegions.cpp new file mode 100644 index 000000000000..0b91fd33fa3e --- /dev/null +++ b/compiler/Transforms/Sequencer/OutlineReductionRegions.cpp @@ -0,0 +1,308 @@ +// Copyright 2019 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include + +#include "third_party/llvm/llvm/include/llvm/ADT/STLExtras.h" +#include "third_party/llvm/llvm/include/llvm/ADT/SetVector.h" +#include "third_party/llvm/llvm/projects/google_mlir/include/mlir/Dialect/StandardOps/Ops.h" +#include "third_party/llvm/llvm/projects/google_mlir/include/mlir/IR/Attributes.h" +#include "third_party/llvm/llvm/projects/google_mlir/include/mlir/IR/Builders.h" +#include "third_party/llvm/llvm/projects/google_mlir/include/mlir/IR/MLIRContext.h" +#include "third_party/llvm/llvm/projects/google_mlir/include/mlir/Pass/Pass.h" +#include "third_party/llvm/llvm/projects/google_mlir/include/mlir/Pass/PassRegistry.h" +#include "third_party/llvm/llvm/projects/google_mlir/include/mlir/Support/LLVM.h" +#include "third_party/llvm/llvm/projects/google_mlir/include/mlir/Support/LogicalResult.h" +#include "third_party/llvm/llvm/projects/google_mlir/include/mlir/Transforms/Utils.h" +#include "third_party/mlir_edge/iree/compiler/IR/Ops.h" +#include "third_party/mlir_edge/iree/compiler/IR/Sequencer/HLOps.h" +#include "third_party/mlir_edge/iree/compiler/IR/StructureOps.h" +#include "third_party/mlir_edge/iree/compiler/IR/Types.h" +#include "third_party/mlir_edge/iree/compiler/Utils/DispatchUtils.h" +#include "third_party/mlir_edge/iree/compiler/Utils/MemRefUtils.h" + +namespace mlir { +namespace iree_compiler { + +namespace { + +// Determines the shapes involved with reducing this dimension. +SmallVector calculateResultShape(Value *input, + int windowDimension) { + SmallVector resultShape; + for (auto it : + llvm::enumerate(input->getType().cast().getShape())) { + if (it.index() != windowDimension) { + resultShape.push_back(it.value()); + } + } + return resultShape; +} + +// Creates an executable that holds the given elemental reduction region. +// The executable will have an entry point taking the specified reduction values +// and writing the results to output arguments. +std::pair createReductionExecutable( + IREE::ReductionRegionOp regionOp, int outlinedRegionOrdinal, + int separatedReductionIndex, int reductionDimension, + SmallVector initialValues, SmallVector inputs) { + Builder builder(regionOp.getContext()); + + // Build function type matching 1:1 with the region signature. + SmallVector elementalOperandTypes; + SmallVector elementalResultTypes; + for (auto *arg : regionOp.getInitialValueOperands()) { + // (in0, in1) -> out0 + elementalOperandTypes.push_back(arg->getType()); + elementalOperandTypes.push_back(arg->getType()); + elementalResultTypes.push_back(arg->getType()); + } + auto elementalFunctionType = FunctionType::get( + elementalOperandTypes, elementalResultTypes, regionOp.getContext()); + + // Create the executable with the region cloned into it. + IREE::MultiArchExecutableOp multiArchExecutable; + FuncOp elementalFunc; + std::tie(multiArchExecutable, elementalFunc) = createRegionExecutable( + regionOp, elementalFunctionType, + "_reduce_" + std::to_string(outlinedRegionOrdinal) + "_dim_" + + std::to_string(separatedReductionIndex)); + + // Create a new entry point that we can use with the signature for this + // dimension. + SmallVector allOperandTypes; + auto inputTypes = + llvm::map_range(inputs, [](Value *value) { return value->getType(); }); + allOperandTypes.append(inputTypes.begin(), inputTypes.end()); + auto initialValueTypes = llvm::map_range( + initialValues, [](Value *value) { return value->getType(); }); + allOperandTypes.append(initialValueTypes.begin(), initialValueTypes.end()); + for (auto resultType : llvm::enumerate(regionOp.getResultTypes())) { + auto shapedType = resultType.value().cast(); + allOperandTypes.push_back(builder.getMemRefType( + calculateResultShape(inputs[resultType.index()], reductionDimension), + shapedType.getElementType())); + } + auto entryFuncType = FunctionType::get(allOperandTypes, ArrayRef{}, + regionOp.getContext()); + auto entryFunc = + FuncOp::create(regionOp.getLoc(), + (elementalFunc.getName() + "_entry").str(), entryFuncType); + entryFunc.setAttr("iree.executable.export", + UnitAttr::get(regionOp.getContext())); + elementalFunc.getOperation()->getBlock()->push_back(entryFunc); + entryFunc.getOperation()->moveBefore(elementalFunc); + entryFunc.setAttr("iree.executable.reduction", + UnitAttr::get(regionOp.getContext())); + entryFunc.setAttr("iree.executable.reduction.apply", + builder.getSymbolRefAttr(elementalFunc)); + + return {multiArchExecutable, entryFunc}; +} + +// Converts a reduction_region into a dispatch to the outlined region function +// for a single reduction dimension. +// Returns the results of the reduction or empty if the construction fails. +SmallVector convertToDispatchOp( + IREE::ReductionRegionOp regionOp, IREE::MultiArchExecutableOp executable, + FuncOp entryFunc, int reductionDimension, + SmallVector initialValues, SmallVector inputs, + OpBuilder *dispatcherBuilder) { + // Allocate output args and replace the return values with those. + SmallVector resultValues; + for (auto resultType : llvm::enumerate(regionOp.getResultTypes())) { + // Allocate output buffer in the dispatcher to pass in to the region. + auto shapedType = resultType.value().cast(); + Value *allocatedValue = allocateDispatchOutputBuffer( + regionOp.getLoc(), + dispatcherBuilder->getMemRefType( + calculateResultShape(inputs[resultType.index()], + reductionDimension), + shapedType.getElementType()), + *dispatcherBuilder); + if (!allocatedValue) { + regionOp.emitError("unable to allocate result value"); + return {}; + } + resultValues.push_back(allocatedValue); + } + + // Calculate workload from the result shape. + auto *workload = + wrapAsMemRef(calculateWorkload(regionOp, resultValues.front()), regionOp, + *dispatcherBuilder); + + // Create the reduce op to the executable function. + std::vector allOperands; + allOperands.insert(allOperands.end(), inputs.begin(), inputs.end()); + allOperands.insert(allOperands.end(), initialValues.begin(), + initialValues.end()); + allOperands.insert(allOperands.end(), resultValues.begin(), + resultValues.end()); + dispatcherBuilder->create( + regionOp.getLoc(), executable.getName(), entryFunc.getName(), workload, + ArrayRef{}, allOperands); + + return resultValues; +} + +// Outlines a reduction region into one or more iree.multi_arch_executables. +// This separates the reduction into multiple dispatches, one for each reduction +// dimension (thankfully XLA's operation semantics state this is ok). We then +// special case the first dispatch such that it takes the constant initial +// values so that we don't have to materialize a buffer for them. +LogicalResult outlineReductionRegion(IREE::ReductionRegionOp regionOp, + int outlinedRegionOrdinal) { + // Insert at the same place as the original region. + OpBuilder dispatcherBuilder(regionOp); + + // Wrap input operands in memrefs. + SmallVector initialValues{llvm::map_range( + regionOp.getInitialValueOperands(), [&](Value *originalArg) { + return insertDispatcherStore(regionOp, originalArg, &dispatcherBuilder); + })}; + SmallVector temps{ + llvm::map_range(regionOp.getReductionOperands(), [&](Value *originalArg) { + return insertDispatcherStore(regionOp, originalArg, &dispatcherBuilder); + })}; + + // Create one dispatch per dimension being reduced. + // We'll do this by chaining the original input through with the temporary + // reduction results. The results we end up with will be the originally + // requested shape and we can just substitute them. + if (regionOp.isWindowed()) { + auto windowDimensions = regionOp.window_dimensions().getValue(); + auto windowStrides = regionOp.window_strides().getValue(); + auto baseDilations = regionOp.base_dilations().getValue(); + auto windowDilations = regionOp.window_dilations().getValue(); + SmallVector, 4> + sortedWindowAttrs; + for (uint64_t i = 0; i < windowDimensions.getNumElements(); ++i) { + int64_t windowDimension = + windowDimensions.getValue({i}).getInt(); + int64_t windowStride = windowStrides.getValue({i}).getInt(); + int64_t baseDilation = baseDilations.getValue({i}).getInt(); + int64_t windowDilation = + windowDilations.getValue({i}).getInt(); + sortedWindowAttrs.push_back( + {windowDimension, windowStride, baseDilation, windowDilation}); + } + llvm::sort(sortedWindowAttrs, + [](std::tuple a, + std::tuple b) { + return std::get<0>(a) - std::get<0>(b); + }); + for (auto windowAttrs : llvm::enumerate(sortedWindowAttrs)) { + int64_t windowDimension = std::get<0>(windowAttrs.value()); + int64_t windowStride = std::get<1>(windowAttrs.value()); + int64_t baseDilation = std::get<2>(windowAttrs.value()); + int64_t windowDilation = std::get<3>(windowAttrs.value()); + IREE::MultiArchExecutableOp multiArchExecutable; + FuncOp entryFunc; + std::tie(multiArchExecutable, entryFunc) = createReductionExecutable( + regionOp, outlinedRegionOrdinal, windowAttrs.index(), windowDimension, + initialValues, temps); + entryFunc.setAttr("iree.executable.reduction.padding_mode", + dispatcherBuilder.getI32IntegerAttr( + regionOp.padding_mode().getValue())); + entryFunc.setAttr("iree.executable.reduction.window_dimension", + dispatcherBuilder.getI32IntegerAttr(windowDimension)); + entryFunc.setAttr("iree.executable.reduction.window_stride", + dispatcherBuilder.getI32IntegerAttr(windowStride)); + entryFunc.setAttr("iree.executable.reduction.base_dilation", + dispatcherBuilder.getI32IntegerAttr(baseDilation)); + entryFunc.setAttr("iree.executable.reduction.window_dilation", + dispatcherBuilder.getI32IntegerAttr(windowDilation)); + temps = convertToDispatchOp(regionOp, multiArchExecutable, entryFunc, + windowDimension, initialValues, + std::move(temps), &dispatcherBuilder); + if (temps.empty()) { + return regionOp.emitOpError() + << "Failed to construct reduction for windowed dimension " + << windowDimension; + } + } + } else { + auto dimensions = regionOp.dimensions().getValue(); + SmallVector sortedDimensions; + for (uint64_t i = 0; i < dimensions.getNumElements(); ++i) { + sortedDimensions.push_back( + dimensions.getValue({i}).getInt()); + } + llvm::sort(sortedDimensions, [](int64_t a, int64_t b) { return a - b; }); + for (auto dimension : llvm::enumerate(sortedDimensions)) { + IREE::MultiArchExecutableOp multiArchExecutable; + FuncOp entryFunc; + std::tie(multiArchExecutable, entryFunc) = createReductionExecutable( + regionOp, outlinedRegionOrdinal, dimension.index(), dimension.value(), + initialValues, temps); + entryFunc.setAttr("iree.executable.reduction.dimension", + dispatcherBuilder.getI32IntegerAttr(dimension.value())); + temps = convertToDispatchOp(regionOp, multiArchExecutable, entryFunc, + dimension.value(), initialValues, + std::move(temps), &dispatcherBuilder); + if (temps.empty()) { + return regionOp.emitOpError() + << "Failed to construct reduction for dimension " + << dimension.value(); + } + } + } + for (auto it : llvm::enumerate(regionOp.getResults())) { + insertDispatcherLoad(regionOp, it.value(), temps[it.index()], + &dispatcherBuilder); + } + + // Erase original region. + regionOp.erase(); + + return success(); +} + +} // namespace + +class OutlineReductionRegionsPass + : public ModulePass { + public: + void runOnModule() override { + auto module = getModule(); + + ModuleManager moduleManager(module); + std::vector funcOps(module.getOps().begin(), + module.getOps().end()); + for (auto func : funcOps) { + // Outline all of the iree.reduction_region ops in this function. + std::vector reductionRegionOps; + func.walk([&](IREE::ReductionRegionOp op) { + reductionRegionOps.push_back(op); + }); + for (int i = 0; i < reductionRegionOps.size(); ++i) { + if (failed(outlineReductionRegion(reductionRegionOps[i], i))) { + return signalPassFailure(); + } + } + } + } +}; + +std::unique_ptr> createOutlineReductionRegionsPass() { + return std::make_unique(); // NOLINT +} + +static PassRegistration pass( + "iree-outline-reduction-regions", + "Outlines reduction regions into standalone functions"); + +} // namespace iree_compiler +} // namespace mlir diff --git a/compiler/Transforms/Sequencer/Passes.h b/compiler/Transforms/Sequencer/Passes.h new file mode 100644 index 000000000000..0a5bac57637d --- /dev/null +++ b/compiler/Transforms/Sequencer/Passes.h @@ -0,0 +1,80 @@ +// Copyright 2019 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#ifndef THIRD_PARTY_MLIR_EDGE_IREE_COMPILER_TRANSFORMS_SEQUENCER_PASSES_H_ +#define THIRD_PARTY_MLIR_EDGE_IREE_COMPILER_TRANSFORMS_SEQUENCER_PASSES_H_ + +#include "third_party/llvm/llvm/projects/google_mlir/include/mlir/Pass/Pass.h" + +namespace mlir { +namespace iree_compiler { + +//===----------------------------------------------------------------------===// +// Dispatches (iree.dispatch_region) +//===----------------------------------------------------------------------===// + +// Identifies dispatchable regions of functions and wraps them in +// iree.dispatch_regions. +std::unique_ptr> createIdentifyDispatchRegionsPass(); + +// Folds multiple dispatch regions together that have compatible workloads. +std::unique_ptr> createFoldCompatibleDispatchRegionsPass(); + +// Rematerializes small previously-CSE'd constants into dispatch regions. +std::unique_ptr> createRematerializeDispatchConstantsPass(); + +// Outlines dispatch regions into executables. +std::unique_ptr> createOutlineDispatchRegionsPass(); + +//===----------------------------------------------------------------------===// +// Reductions (iree.reduction_region) +//===----------------------------------------------------------------------===// + +// Identifies reduction regions and wraps them in iree.reduction_regions. +std::unique_ptr> createIdentifyReductionRegionsPass(); + +// Outlines dispatch regions into executables. +std::unique_ptr> createOutlineReductionRegionsPass(); + +//===----------------------------------------------------------------------===// +// Lowering/Conversion +//===----------------------------------------------------------------------===// + +// Lowers XLA dialect (xla.*) ops to IREE Sequencer HL ops. +std::unique_ptr> createLowerXLAToSequencerDialectPass(); + +// Lowers sequencer functions using std ops (call/cond_br/etc) to the IREE HL +// sequencer dialect and buffer view types. +std::unique_ptr> createLowerStdToSequencerDialectPass(); + +// Lowers the HL sequencer dialect to the LL sequencer dialect. +std::unique_ptr> createLowerSequencerDialectPass(); + +// Optimizes std.load and std.store to remove unnessisary copies. +std::unique_ptr> createLoadStoreDataFlowOptPass(); + +//===----------------------------------------------------------------------===// +// Module Analysis and Assignment +//===----------------------------------------------------------------------===// + +// Assigns module-unique ordinals to executables and their entry points. +std::unique_ptr> createAssignExecutableOrdinalsPass(); + +// Assigns workload attributes to executable entry points based on dispatches. +std::unique_ptr> createAssignExecutableWorkloadAttrsPass(); + +} // namespace iree_compiler +} // namespace mlir + +#endif // THIRD_PARTY_MLIR_EDGE_IREE_COMPILER_TRANSFORMS_SEQUENCER_PASSES_H_ diff --git a/compiler/Transforms/Sequencer/RematerializeDispatchConstants.cpp b/compiler/Transforms/Sequencer/RematerializeDispatchConstants.cpp new file mode 100644 index 000000000000..ac417e7a29f6 --- /dev/null +++ b/compiler/Transforms/Sequencer/RematerializeDispatchConstants.cpp @@ -0,0 +1,149 @@ +// Copyright 2019 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include + +#include "third_party/llvm/llvm/include/llvm/Support/Debug.h" +#include "third_party/llvm/llvm/projects/google_mlir/include/mlir/Dialect/StandardOps/Ops.h" +#include "third_party/llvm/llvm/projects/google_mlir/include/mlir/IR/Attributes.h" +#include "third_party/llvm/llvm/projects/google_mlir/include/mlir/IR/BlockAndValueMapping.h" +#include "third_party/llvm/llvm/projects/google_mlir/include/mlir/IR/Builders.h" +#include "third_party/llvm/llvm/projects/google_mlir/include/mlir/IR/Location.h" +#include "third_party/llvm/llvm/projects/google_mlir/include/mlir/IR/MLIRContext.h" +#include "third_party/llvm/llvm/projects/google_mlir/include/mlir/IR/StandardTypes.h" +#include "third_party/llvm/llvm/projects/google_mlir/include/mlir/Pass/Pass.h" +#include "third_party/llvm/llvm/projects/google_mlir/include/mlir/Pass/PassRegistry.h" +#include "third_party/llvm/llvm/projects/google_mlir/include/mlir/Support/LLVM.h" +#include "third_party/llvm/llvm/projects/google_mlir/include/mlir/Support/LogicalResult.h" +#include "third_party/llvm/llvm/projects/google_mlir/include/mlir/Transforms/Utils.h" +#include "third_party/mlir_edge/iree/compiler/IR/Ops.h" +#include "third_party/mlir_edge/iree/compiler/Utils/DispatchUtils.h" +#include "third_party/tensorflow/compiler/mlir/xla/ir/hlo_ops.h" + +namespace mlir { +namespace iree_compiler { + +namespace { + +// Chosen randomly for now. We can measure and see what makes sense. +constexpr int64_t kMaxRematerializedConstantSizeInBytes = 1 * 1024; + +// Returns true if the constant value is under a certain threshold. +// This threshold is fixed for all backends as a value that is assumed small +// enough to be worth inlining possibly several times (at the cost of binary +// bloat). +bool isConstantSmall(ConstantOp constantOp) { + if (auto shapedType = constantOp.getType().dyn_cast()) { + return shapedType.getSizeInBits() / 8 <= + kMaxRematerializedConstantSizeInBytes; + } + + // Assume anything unshaped is small. This may not always be true in custom + // dialects but is in std for now. + return true; +} + +// Returns true if the dispatch region is allowed to have constants inside. +// Certain regions that may get replaced or turned into kernel imports shouldn't +// have the constants moved into them as they'll just get lost. +bool canDispatchRegionContainConstants( + IREE::DispatchRegionOp dispatchRegionOp) { + for (auto &block : dispatchRegionOp.getBody()) { + for (auto &op : block) { + if (isa(&op)) { + return false; + } + } + } + return true; +} + +// Rematerializes a constant inside of all dispatch regions that use it. +// Afterward the constant is only removed if there are no other uses within the +// non-dispatch block (such as by sequencer ops). +LogicalResult rematerializeConstantInDispatchRegions(ConstantOp constantOp) { + Value *constantValue = constantOp.getResult(); + SmallVector usingRegionOps; + for (auto *user : constantValue->getUsers()) { + if (auto dispatchRegionOp = dyn_cast(user)) { + // Ensure this isn't just the workload and is used as an arg. + if (std::find(dispatchRegionOp.arg_operand_begin(), + dispatchRegionOp.arg_operand_end(), + constantValue) != dispatchRegionOp.arg_operand_end()) { + if (canDispatchRegionContainConstants(dispatchRegionOp)) { + usingRegionOps.push_back(dispatchRegionOp); + } + } + } + } + for (auto &dispatchRegionOp : usingRegionOps) { + if (failed(inlineDispatchRegionOperandsUsingValue(dispatchRegionOp, + constantValue))) { + return failure(); + } + } + + // Remove if there are no other uses within the block. + if (constantOp.use_empty()) { + constantOp.erase(); + } + + return success(); +} + +} // namespace + +// Finds constant arguments to dispatch regions that are too small to be worth +// putting into constant pools. This prevents things like a CSE'd scalar +// constant of 0.0 being passed by reference to a bunch of regions. Later +// backend-specific passes running on the dispatch regions may also be able to +// improve their constant propagation chances by having the full constant value +// available. +// +// Note that this currently only operates at the block level. Constants that are +// pushed across branches are assumed to have been rematerialized within blocks +// already, but if that isn't the case then this pass can be extended to do +// that. +class RematerializeDispatchConstantsPass + : public FunctionPass { + public: + void runOnFunction() override { + for (auto &block : getFunction()) { + SmallVector smallConstantOps; + for (auto constantOp : block.getOps()) { + if (isConstantSmall(constantOp)) { + smallConstantOps.push_back(constantOp); + } + } + // Note: we iterate in reverse so that the rematerialized constants appear + // in the same order they did originally (as insertion is at the top). + for (auto constantOp : llvm::reverse(smallConstantOps)) { + if (failed(rematerializeConstantInDispatchRegions(constantOp))) { + return signalPassFailure(); + } + } + } + } +}; + +std::unique_ptr> createRematerializeDispatchConstantsPass() { + return std::make_unique(); +} + +static PassRegistration pass( + "iree-rematerialize-dispatch-constants", + "Rematerializes small previously-CSE'd constants into dispatch regions."); + +} // namespace iree_compiler +} // namespace mlir diff --git a/compiler/Transforms/test/drop_unreachable_module_functions.mlir b/compiler/Transforms/test/drop_unreachable_module_functions.mlir new file mode 100644 index 000000000000..871b287822d6 --- /dev/null +++ b/compiler/Transforms/test/drop_unreachable_module_functions.mlir @@ -0,0 +1,49 @@ +// RUN: iree-opt %s -iree-drop-unreachable-module-functions -split-input-file | FileCheck %s --dump-input=fail --implicit-check-not @unused + +// CHECK-LABEL: @exportedModuleFn +func @exportedModuleFn(%arg0 : memref) -> memref + attributes {iree.module.export} { + // CHECK: iree_hl_seq.call @fn1 + %0 = iree_hl_seq.call @fn1(%arg0) : (memref) -> memref + return %0 : memref +} + +// CHECK: @fn1 +func @fn1(%arg0 : memref) -> memref { + // CHECK: iree_hl_seq.call @fn2 + %0 = iree_hl_seq.call @fn2(%arg0) : (memref) -> memref + return %0 : memref +} + +// CHECK: @fn2 +func @fn2(%arg0 : memref) -> memref { + return %arg0 : memref +} + +// CHECK-NOT: @unusedFn3 +func @unusedFn3(%arg0 : memref) -> memref { + return %arg0 : memref +} + +// ----- + +// CHECK-NOT: @unusedFn +func @unusedFn(%arg0 : memref) -> memref { + return %arg0 : memref +} + +// ----- + +// CHECK-LABEL: @exportedFnWithImports +func @exportedFnWithImports(%arg0 : memref) -> memref + attributes {iree.module.export} { + // CHECK: iree_hl_seq.call @usedImportFn + %0 = iree_hl_seq.call @usedImportFn(%arg0) : (memref) -> memref + return %0 : memref +} + +// CHECK: @usedImportFn +func @usedImportFn(%arg0 : memref) -> memref + +// CHECK-NOT: @unusedImportFn +func @unusedImportFn(%arg0 : memref) -> memref diff --git a/compiler/Translation/Interpreter/InterpreterExecutableTranslation.cpp b/compiler/Translation/Interpreter/InterpreterExecutableTranslation.cpp new file mode 100644 index 000000000000..e05ae42ae327 --- /dev/null +++ b/compiler/Translation/Interpreter/InterpreterExecutableTranslation.cpp @@ -0,0 +1,293 @@ +// Copyright 2019 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "third_party/mlir_edge/iree/compiler/Translation/Interpreter/InterpreterExecutableTranslation.h" + +#include +#include +#include + +#include "third_party/flatbuffers/include/flatbuffers/flatbuffers.h" +#include "third_party/flatbuffers/include/flatbuffers/minireflect.h" +#include "third_party/llvm/llvm/include/llvm/ADT/STLExtras.h" +#include "third_party/llvm/llvm/include/llvm/ADT/StringRef.h" +#include "third_party/llvm/llvm/include/llvm/Support/Debug.h" +#include "third_party/llvm/llvm/projects/google_mlir/include/mlir/IR/Attributes.h" +#include "third_party/llvm/llvm/projects/google_mlir/include/mlir/IR/Module.h" +#include "third_party/llvm/llvm/projects/google_mlir/include/mlir/Pass/Pass.h" +#include "third_party/llvm/llvm/projects/google_mlir/include/mlir/Pass/PassManager.h" +#include "third_party/llvm/llvm/projects/google_mlir/include/mlir/Support/LogicalResult.h" +#include "third_party/llvm/llvm/projects/google_mlir/include/mlir/Transforms/Passes.h" +#include "third_party/llvm/llvm/projects/google_mlir/include/mlir/Translation.h" +#include "third_party/mlir_edge/iree/compiler/IR/ConfigOps.h" +#include "third_party/mlir_edge/iree/compiler/IR/Interpreter/OpWriters.h" +#include "third_party/mlir_edge/iree/compiler/IR/Types.h" +#include "third_party/mlir_edge/iree/compiler/Serialization/VMFunctionBuilder.h" +#include "third_party/mlir_edge/iree/compiler/Serialization/VMFunctionTableBuilder.h" +#include "third_party/mlir_edge/iree/compiler/Serialization/VMModuleBuilder.h" +#include "third_party/mlir_edge/iree/compiler/Transforms/Interpreter/Passes.h" +#include "third_party/mlir_edge/iree/compiler/Transforms/Passes.h" +#include "third_party/mlir_edge/iree/compiler/Utils/Macros.h" +#include "third_party/mlir_edge/iree/compiler/Utils/OpUtils.h" +#include "third_party/mlir_edge/iree/compiler/Utils/TranslationUtils.h" +#include "third_party/mlir_edge/iree/schemas/executable_def_generated.h" +#include "third_party/mlir_edge/iree/schemas/module_def_generated.h" +#include "third_party/tensorflow/compiler/mlir/xla/transforms/passes.h" + +namespace mlir { +namespace iree_compiler { + +namespace { + +// Builds a pass pipeline that optimizes and legalizes the module to the form +// expected by translation. +void buildLegalizeInputPassPipeline(PassManager *passManager) { + // Standard passes that shake out a lot of garbage. + // Some may have been run prior to translation but this ensures we are always + // in a known state. + passManager->addPass(createCanonicalizerPass()); + passManager->addPass(createLoopFusionPass()); + passManager->addPass(createLoopInvariantCodeMotionPass()); + passManager->addPass(createMemRefDataFlowOptPass()); + passManager->addPass(createCanonicalizerPass()); + passManager->addPass(createSimplifyAffineStructuresPass()); + passManager->addPass(createCSEPass()); + passManager->addPass(createCanonicalizerPass()); + + // Eliminate ops we don't care about based on a lack of side-effects. + // IREE does not guarantee exception/error behavior of dead ops. + passManager->addPass(createAggressiveOpEliminationPass()); + + // Expand uses of tuples into independent args/results. + passManager->addPass(createConvertFromTupleCallingConventionPass()); + passManager->addPass(createLegalizeTupleElementAccessPass()); +} + +// Builds a pass pipeline that converts functions to the iree_hl_interp dialect. +void buildInterpreterConversionPassPipeline(PassManager *passManager) { + // Widen reduction functions (that have iree.executable.reduction attrs) to + // use their primitive IREE ops. + passManager->addPass(createExpandReductionsToOpsPass()); + + // We don't need the IREE binding ops anymore, as we match the calling + // convention exactly (we're the same VM). + passManager->addPass(createMakeExecutableABIPass()); + + // Convert to the memref calling convention and optimize away as many + // loads and stores as we can prior to progressing. + passManager->addPass(createConvertToMemRefCallingConventionPass()); + passManager->addPass(createCanonicalizerPass()); + passManager->addPass(createMemRefDataFlowOptPass()); + + // Convert various dialects to IREE opcodes and cleanup leftover conversions. + passManager->addPass(createLowerXLAToInterpreterDialectPass()); + passManager->addPass(mlir::xla_hlo::createLegalizeToStdPass()); + passManager->addPass(createLowerStdToInterpreterDialectPass()); + passManager->addPass(createLegalizeTensorMemRefPass()); + passManager->addPass(createInterpreterLoadStoreDataFlowOptPass()); + passManager->addPass(createAggressiveOpEliminationPass()); + + // Convert any uses of index to int32_t (as we explicitly don't want to + // support dynamic index width). + // This also looks for other weird types (i1, etc). + passManager->addPass(createLegalizeTypeStoragePass()); + + // Perform any last-minute optimizations to trim down the IR. + passManager->addPass(createLegalizeInterpreterOpsPass()); + passManager->addPass(createAggressiveOpEliminationPass()); + passManager->addPass(createCanonicalizerPass()); + passManager->addPass(createLoopFusionPass()); + passManager->addPass(createLoopInvariantCodeMotionPass()); + passManager->addPass(createMemRefDataFlowOptPass()); + passManager->addPass(createCanonicalizerPass()); + passManager->addPass(createCSEPass()); + passManager->addPass(createCanonicalizerPass()); + + // Drop all functions that are not reachable. + passManager->addPass(createDropUnreachableExecutableFunctionsPass()); +} + +// Builds a pass pipeline that lowers the iree_hl_interp dialect to the +// iree_ll_interp dialect and prepares for serialization. +void buildInterpreterLoweringPassPipeline(PassManager *passManager) { + // Lower iree_hl_interp -> iree_ll_interp. + passManager->addPass(createLowerInterpreterDialectPass()); + + // Assign ordinals used by the bytecode to reference executables and + // functions. + passManager->addPass(createAssignFunctionOrdinalsPass()); +} + +class InterpreterTranslator { + public: + explicit InterpreterTranslator(ExecutableTranslationOptions options) + : options_(options) {} + + const ExecutableTranslationOptions &options() const { return options_; } + + std::unique_ptr translateExecutable( + IREE::ExecutableOp executableOp); + + private: + LogicalResult translateExecutableModule(IREE::ExecutableOp executableOp, + ModuleOp moduleOp, + VMModuleBuilder *moduleBuilder); + LogicalResult declareFunction(FuncOp function, + VMModuleBuilder *moduleBuilder); + LogicalResult defineFunction(FuncOp function, VMModuleBuilder *moduleBuilder); + + ExecutableTranslationOptions options_; +}; + +std::unique_ptr +InterpreterTranslator::translateExecutable(IREE::ExecutableOp executableOp) { + auto moduleOp = executableOp.getInnerModule(); + + // Run all passes to go from input to the iree_ll_interp dialect. + auto executableConversionPasses = + createPassManager(moduleOp.getContext(), options()); + buildLegalizeInputPassPipeline(executableConversionPasses.get()); + buildInterpreterConversionPassPipeline(executableConversionPasses.get()); + buildInterpreterLoweringPassPipeline(executableConversionPasses.get()); + if (failed(runPassPipeline(options(), executableConversionPasses.get(), + moduleOp))) { + executableOp.emitError() << "Failed to run conversion passes"; + return {}; + } + + // Build the module bytecode. + ::flatbuffers::FlatBufferBuilder fbb; + VMModuleBuilder moduleBuilder(&fbb); + if (failed( + translateExecutableModule(executableOp, moduleOp, &moduleBuilder))) { + executableOp.emitError() << "Failed to translate executable module"; + return {}; + } + auto moduleDef = moduleBuilder.Finish(); + if (moduleDef.IsNull()) { + moduleOp.emitError() << "Failed to verify completed module def"; + return {}; + } + auto bytes = moduleBuilder.Serialize(moduleDef); + if (bytes.empty()) { + moduleOp.emitError() << "Failed to serialize final module def"; + return {}; + } + + OpBuilder builder(executableOp); + executableOp.setAttr("format", builder.getI32IntegerAttr(static_cast( + IREE::ExecutableFormat::IreeBytecode))); + + auto executableDef = std::make_unique(); + executableDef->format = + static_cast(IREE::ExecutableFormat::IreeBytecode); + executableDef->supported_features = iree::ExecutableFeature::kDebugging; + executableDef->contents = std::move(bytes); + return executableDef; +} + +LogicalResult InterpreterTranslator::translateExecutableModule( + IREE::ExecutableOp executableOp, ModuleOp moduleOp, + VMModuleBuilder *moduleBuilder) { + // Declare functions first so that we get stable indices during declaration + // (as call ops need to use the function table). + for (auto function : moduleOp.getOps()) { + RETURN_IF_FAILURE(declareFunction(function, moduleBuilder)); + } + + // Define functions now that all functions have been declared. + for (auto function : moduleOp.getOps()) { + RETURN_IF_FAILURE(defineFunction(function, moduleBuilder)); + } + + return success(); +} + +LogicalResult InterpreterTranslator::declareFunction( + FuncOp function, VMModuleBuilder *moduleBuilder) { + auto *functionTable = moduleBuilder->function_table(); + if (functionTable->IsFunctionDeclared(function)) { + // Already declared. + return success(); + } + + LinkageType linkageType; + if (function.isExternal()) { + linkageType = LinkageType::kImport; + } else if (function.getAttr("iree.executable.export")) { + linkageType = LinkageType::kExport; + } else { + linkageType = LinkageType::kInternal; + } + if (failed(functionTable->DeclareFunction(function, linkageType))) { + return function.emitError() << "Unable to declare function"; + } + + // Import functions must have their definition defined here so we get their + // type. Internal and export functions will be defined during conversion. + if (linkageType == LinkageType::kImport) { + VMFunctionBuilder functionBuilder(function, moduleBuilder->function_table(), + moduleBuilder->fbb()); + auto functionOffset = functionBuilder.Finish(); + if (functionOffset.IsNull()) { + return function.emitError() + << "Failed to create import function bytecode"; + } + RETURN_IF_FAILURE( + functionTable->DefineFunction(function, functionOffset, {})); + } + + return success(); +} + +LogicalResult InterpreterTranslator::defineFunction( + FuncOp function, VMModuleBuilder *moduleBuilder) { + VMFunctionBuilder functionBuilder(function, moduleBuilder->function_table(), + moduleBuilder->fbb()); + registerInterpreterCustomWriters(&functionBuilder); + RETURN_IF_FAILURE(functionBuilder.ConvertBytecode()); + auto functionOffset = functionBuilder.Finish(); + if (functionOffset.IsNull()) { + return function.emitError() << "Failed to serialize function"; + } + RETURN_IF_FAILURE(moduleBuilder->function_table()->DefineFunction( + function, functionOffset, functionBuilder.source_map())); + return success(); +} + +} // namespace + +llvm::Optional +translateExecutableToInterpreterExecutable( + ArrayRef executableOps, + ExecutableTranslationOptions options) { + InterpreterTranslator translator(options); + ExecutableTranslationResult translationResult; + for (auto executableOp : llvm::make_early_inc_range(executableOps)) { + auto executableDef = translator.translateExecutable(executableOp); + if (!executableDef) { + executableOp.emitError() << "Failed to translate one or more executables"; + return llvm::None; + } + translationResult.executable_defs.push_back(std::move(executableDef)); + } + return translationResult; +} + +static ExecutableTranslationRegistration + InterpreterExecutableTranslationRegistration( + "interpreter-bytecode", translateExecutableToInterpreterExecutable); + +} // namespace iree_compiler +} // namespace mlir diff --git a/compiler/Translation/Interpreter/InterpreterExecutableTranslation.h b/compiler/Translation/Interpreter/InterpreterExecutableTranslation.h new file mode 100644 index 000000000000..0ab67e72286b --- /dev/null +++ b/compiler/Translation/Interpreter/InterpreterExecutableTranslation.h @@ -0,0 +1,38 @@ +// Copyright 2019 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#ifndef THIRD_PARTY_MLIR_EDGE_IREE_COMPILER_TRANSLATION_INTERPRETER_INTERPRETEREXECUTABLETRANSLATION_H_ +#define THIRD_PARTY_MLIR_EDGE_IREE_COMPILER_TRANSLATION_INTERPRETER_INTERPRETEREXECUTABLETRANSLATION_H_ + +#include + +#include "third_party/llvm/llvm/projects/google_mlir/include/mlir/IR/Module.h" +#include "third_party/mlir_edge/iree/compiler/IR/StructureOps.h" +#include "third_party/mlir_edge/iree/compiler/Utils/TranslationUtils.h" + +namespace mlir { +namespace iree_compiler { + +// Translates an MLIR module into a bytecode interpreter executable. +// These executables are stored as IREE modules as defined in the +// iree/schemas/module_def.fbs schema. +llvm::Optional +translateExecutableToInterpreterExecutable( + ArrayRef executableOps, + ExecutableTranslationOptions options = {}); + +} // namespace iree_compiler +} // namespace mlir + +#endif // THIRD_PARTY_MLIR_EDGE_IREE_COMPILER_TRANSLATION_INTERPRETER_INTERPRETEREXECUTABLETRANSLATION_H_ diff --git a/compiler/Translation/SPIRV/AffineExprCodegen.h b/compiler/Translation/SPIRV/AffineExprCodegen.h new file mode 100644 index 000000000000..9b92abe0c05e --- /dev/null +++ b/compiler/Translation/SPIRV/AffineExprCodegen.h @@ -0,0 +1,146 @@ +// Copyright 2019 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +//===- AffineExprCodegen.h -------------------------------------*- C++//-*-===// +// +// Code-generation for Affine Expression. +// +//===----------------------------------------------------------------------===// +#ifndef IREE_COMPILER_TRANSLATION_SPIRV_AFFINEEXPRCODGEN_H +#define IREE_COMPILER_TRANSLATION_SPIRV_AFFINEEXPRCODGEN_H + +#include "third_party/llvm/llvm/projects/google_mlir/include/mlir/Dialect/SPIRV/SPIRVOps.h" +#include "third_party/llvm/llvm/projects/google_mlir/include/mlir/IR/AffineExprVisitor.h" +#include "third_party/mlir_edge/iree/compiler/Translation/SPIRV/XLAIndexPropagation.h" + +namespace mlir { +namespace iree_compiler { + +/// Codegenerator for affine expressions. +class AffineExprCodegen : public AffineExprVisitor { + public: + explicit AffineExprCodegen(spirv::ModuleOp module, + IndexComputationCache &tensorIndices) + : builder(module.getContext()), + location(module.getLoc()), + tensorIndices(tensorIndices) {} + + Value *visitAddExpr(AffineBinaryOpExpr expr) { + auto operand1 = getValueInternal(expr.getLHS()); + auto operand2 = getValueInternal(expr.getRHS()); + return builder.create(location, operand1, operand2); + } + Value *visitMulExpr(AffineBinaryOpExpr expr) { + auto operand1 = getValueInternal(expr.getLHS()); + auto operand2 = getValueInternal(expr.getRHS()); + return builder.create(location, operand1, operand2); + } + Value *visitModExpr(AffineBinaryOpExpr expr) { + auto operand1 = getValueInternal(expr.getLHS()); + auto operand2 = getValueInternal(expr.getRHS()); + return builder.create(location, operand1, operand2); + } + Value *visitFloorDivExpr(AffineBinaryOpExpr expr) { + auto operand1 = getValueInternal(expr.getLHS()); + auto operand2 = getValueInternal(expr.getRHS()); + return builder.create(location, operand1, operand2); + } + Value *visitCeilDivExpr(AffineBinaryOpExpr expr) { + // TODO(ravishankarm): Implement ceil div expr codegen. + llvm_unreachable("Unimplemented affine AffineCeilDivExpr codegen"); + return nullptr; + } + Value *visitConstantExpr(AffineConstantExpr expr) { + // TODO(ravishankarm): Implement constant expr codegen. + return builder.create( + location, builder.getIntegerType(32), + builder.getI32IntegerAttr(expr.getValue())); + llvm_unreachable("Unimplemented affine AffineConstantExpr codegen"); + return nullptr; + } + Value *visitDimExpr(AffineDimExpr expr) { + return threadDimToDstValue.lookup(expr.getPosition()); + } + Value *visitSymbolExpr(AffineSymbolExpr expr) { + // TODO(ravishankarm): Implement symbol expr codegen. + llvm_unreachable("Unimplemented affine AffineSymbolExpr codegen"); + return nullptr; + } + + /// Set the value that contains the workitem ID along a particular + /// dimension. 0 -> x-dimension, 1 -> y-dimension, etc. + void setDimDstValue(unsigned dimID, Value *value) { + threadDimToDstValue[dimID] = value; + } + + /// Generates the scalar value for a affine expression. + Value *getValue(AffineExpr expr, OpBuilder::InsertPoint ip, Location loc) { + auto &val = exprToDstValue[expr]; + if (!val) { + location = loc; + builder.restoreInsertionPoint(ip); + val = visit(expr); + } + return val; + } + + /// Returns a list of indices of a particular tensor in the source dialect + /// needed within the dispatch function (obtained from the + /// IndexComputationCache) + SmallVector getIndices(Value *value) { + SmallVector indices; + for (auto &index : tensorIndices[value]) { + indices.push_back(index.first); + } + return indices; + } + + /// For a given tensor in the source dialect and index, return the index of + /// all operands needed to compute the result. + ArrayRef getOperandIndices(Value *value, AffineMap index) { + return tensorIndices[value][index]; + } + + private: + /// Returns the Value corresponding to the AffineExpr `expr` by either + /// previously generated value for the same index, or by generating the value. + /// This version assumes the insertion point/Location has already been set. + Value *getValueInternal(AffineExpr expr) { + auto &val = exprToDstValue[expr]; + if (!val) { + val = visit(expr); + } + return val; + } + + OpBuilder builder; + + Location location; + + /// Map from launch dimension to scalar value. + DenseMap threadDimToDstValue; + + /// Cache of affine expression to scalar value. TODO(ravishankarm) : Might + /// need to be changed if we are handling control flow within the dispatch + /// function. + DenseMap exprToDstValue; + + /// Map from tensor value in source dialect to list of indices of the tensor + /// needed within a workitem to compute the results of the dispatch function. + IndexComputationCache &tensorIndices; +}; +} // namespace iree_compiler +} // namespace mlir + +#endif // IREE_COMPILER_TRANSLATION_SPIRV_AFFINEEXPRCODGEN_H diff --git a/compiler/Translation/SPIRV/IREEIndexComputation.cpp b/compiler/Translation/SPIRV/IREEIndexComputation.cpp new file mode 100644 index 000000000000..ce502a1efd34 --- /dev/null +++ b/compiler/Translation/SPIRV/IREEIndexComputation.cpp @@ -0,0 +1,99 @@ +// Copyright 2019 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +//===- IREEIndexComputation.cpp --------------------------------*- C++//-*-===// +// +// Implementaiton of Index Propagation for IREE statements that are used in +// dispatch functions. +// +//===----------------------------------------------------------------------===// +#include "third_party/mlir_edge/iree/compiler/Translation/SPIRV/IREEIndexComputation.h" + +namespace mlir { +namespace iree_compiler { + +//===----------------------------------------------------------------------===// +// IREELoadInputOp +//===----------------------------------------------------------------------===// + +LogicalResult IREELoadIndexPropagation::propagateIndexMap( + Operation *operation, IndexComputationCache &indexMap) const { + auto loadOp = cast(operation); + auto result = operation->getResult(0); + auto src = loadOp.src(); + auto resultType = result->getType().dyn_cast(); + auto srcType = src->getType().dyn_cast(); + if (!resultType || !srcType || resultType.getShape() != srcType.getShape()) { + return loadOp.emitError( + "mismatch in shape of the result tensor and source memref"); + } + // Initialize the storage for the src. + indexMap[src]; + for (auto &resultIndexMap : indexMap[operation->getResult(0)]) { + indexMap[src][resultIndexMap.first]; + resultIndexMap.second.push_back(resultIndexMap.first); + } + return success(); +} + +//===----------------------------------------------------------------------===// +// IREEStoreOutputOp +//===----------------------------------------------------------------------===// + +LogicalResult IREEStoreIndexPropagation::propagateIndexMap( + Operation *operation, IndexComputationCache &indexMap) const { + auto storeOp = cast(operation); + auto src = storeOp.src(); + auto srcType = src->getType().dyn_cast(); + if (!srcType || !srcType.hasStaticShape()) { + return storeOp.emitError( + "can only handle store with src being tensor of static shape"); + } + + SmallVector launchSize; + if (failed(getLaunchSize(operation, launchSize))) { + return failure(); + } + + // The launch dimensions are [x, y, z] co-ordinates. The reverse of this is + // used to determine the location of the tensor element computed by a + // workitem. The choice is failry arbitrary but is done to enable the common + // case where consecutive workitems compute "logically" adjacent tensor + // elements. + Builder builder(storeOp.getContext()); + SmallVector affineExprs; + int64_t numElements = 1; + for (size_t i = launchSize.size(); i > 0; --i) { + affineExprs.push_back(builder.getAffineDimExpr(i - 1)); + numElements *= launchSize[i - 1]; + } + auto launchMap = builder.getAffineMap(launchSize.size(), 0, affineExprs); + + // The stored tensor can be a reshape of the launch dimension. It still + // retains the requirement that each workitem is computing a single element + // of the stored tensor. + AffineMap srcMap; + SmallVector revLaunchSize(reverse(launchSize)); + if (numElements != srcType.getNumElements() || + failed(getReshapeOperandMap(builder, launchMap, revLaunchSize, + srcType.getShape(), srcMap))) { + return storeOp.emitError( + "unable to map from launch id to element to compute within a " + "workitem"); + } + indexMap[src][srcMap]; + return success(); +} +} // namespace iree_compiler +} // namespace mlir diff --git a/compiler/Translation/SPIRV/IREEIndexComputation.h b/compiler/Translation/SPIRV/IREEIndexComputation.h new file mode 100644 index 000000000000..2c91d190dd4d --- /dev/null +++ b/compiler/Translation/SPIRV/IREEIndexComputation.h @@ -0,0 +1,92 @@ +// Copyright 2019 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +//===- IREEIndexComputation.h ----------------------------------*- C++//-*-===// +// +// Index Propagation for IREE statements that are used in dispatch functions. +// +//===----------------------------------------------------------------------===// +#ifndef IREE_COMPILER_TRANSLATION_SPIRV_H +#define IREE_COMPILER_TRANSLATION_SPIRV_H + +#include "third_party/llvm/llvm/projects/google_mlir/include/mlir/IR/Function.h" +#include "third_party/mlir_edge/iree/compiler/IR/Ops.h" +#include "third_party/mlir_edge/iree/compiler/IR/StructureOps.h" +#include "third_party/mlir_edge/iree/compiler/Translation/SPIRV/XLAIndexPropagation.h" + +namespace mlir { +namespace iree_compiler { + +/// Gets the launch size associated with the dispatch function that this op is +/// part of. +inline LogicalResult getLaunchSize(Operation *op, + SmallVectorImpl &launchSize) { + auto funcOp = op->getParentOfType(); + if (!funcOp || !funcOp.getAttr("iree.executable.export")) { + return op->emitError( + "expected operation to be in dispatch function to get launch size"); + } + auto workloadAttr = + funcOp.getAttrOfType("iree.executable.workload"); + if (!workloadAttr) { + op->emitError( + "unable to find workload size, missing attribute " + "iree.executable.workload in dispatch function"); + } + launchSize.clear(); + for (auto value : workloadAttr.getValues()) { + launchSize.push_back(value.getSExtValue()); + } + // Drop trailing ones. + auto dropFrom = launchSize.size() - 1; + while (dropFrom > 0 && launchSize[dropFrom] == 1) { + --dropFrom; + } + if (dropFrom > 0) { + launchSize.erase(std::next(launchSize.begin(), dropFrom + 1), + launchSize.end()); + } + return success(); +} + +/// Index propagation for iree.load_input operation. This operation is +/// essentially a copy from a memref to a tensor. So just copy the index map to +/// the memref operand from the result tensor. +class IREELoadIndexPropagation final + : public IndexPropagationOp { + public: + using IndexPropagationOp::IndexPropagationOp; + + LogicalResult propagateIndexMap( + Operation *operation, IndexComputationCache &indexMap) const override; +}; + +/// Index propagation for iree.store_output operation. The launch size is +/// assumed to match the shape of the tensor that is being stored. This +/// operation acts as a seed for the index propogation. Each workitem is assumed +/// to compute a single element of this tensor. The range of the index map is +/// the reverse of the launch dimension. +class IREEStoreIndexPropagation final + : public IndexPropagationOp { + public: + using IndexPropagationOp::IndexPropagationOp; + + LogicalResult propagateIndexMap( + Operation *operation, IndexComputationCache &indexMap) const override; +}; + +} // namespace iree_compiler +} // namespace mlir + +#endif // IREE_COMPILER_TRANSLATION_SPIRV_H diff --git a/compiler/Translation/SPIRV/IREEToSPIRV.cpp b/compiler/Translation/SPIRV/IREEToSPIRV.cpp new file mode 100644 index 000000000000..b86fb2bab76d --- /dev/null +++ b/compiler/Translation/SPIRV/IREEToSPIRV.cpp @@ -0,0 +1,77 @@ +// Copyright 2019 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +//===- IREEToSPIRV.cpp -----------------------------------------*- C++//-*-===// +// +// Translation of IREE statements in dispatch functions to SPIR-V. +// +//===----------------------------------------------------------------------===// +#include "third_party/mlir_edge/iree/compiler/Translation/SPIRV/IREEToSPIRV.h" + +namespace mlir { +namespace iree_compiler { + +/// IREE::LoadInputOp is essentially a memcpy. Just update the `valueCache` with +/// the value of the operand. +LogicalResult IREELoadOpSPIRVLowering::lowerOperation( + Operation *op, OpBuilder &builder, AffineMap index, + ArrayRef operands, ValueCache &valueCache) const { + auto loadOp = cast(op); + auto result = loadOp.getResult(); + valueCache.setOperandDstValue(result, index, operands[0]); + return success(); +} + +/// IREE::StoreOp needs to write to the spv.globalVariable created for the +/// memref that holds the result of the dispatch function. +LogicalResult IREEStoreOpSPIRVLowering::lowerOperation( + Operation *op, OpBuilder &builder, AffineExprCodegen &affineExprCodegen, + ValueCache &valueCache, + DenseMap &inputBuffers, + ArrayRef outputBuffers) const { + auto storeOp = cast(op); + auto src = storeOp.src(); + auto indices = affineExprCodegen.getIndices(src); + if (indices.size() != 1) { + return storeOp.emitError( + "expected to compute a single element of the tensor that is stored " + "into the output memref"); + } + auto var = inputBuffers.lookup(storeOp.dst()); + if (!var) { + return storeOp.emitError( + "unable to find spv.globalVariable that corresponds to the dst memref"); + } + auto ptr = genPointerOffset(builder, storeOp.getLoc(), affineExprCodegen, + indices[0], var); + auto scalarValue = valueCache.getOperandDstValue(src, indices[0]); + builder.create(storeOp.getLoc(), ptr, scalarValue, + /*memory_access = */ nullptr, + /*alignment = */ nullptr); + return success(); +} + +/// IREE::ReturnOp in dispatch functions lowered to SPIR-V should have no +/// operands. +LogicalResult IREEReturnOpSPIRVLowering::lowerOperation( + Operation *op, OpBuilder &builder, AffineExprCodegen &affineExprCodegen, + ValueCache &valueCache, + DenseMap &inputBuffers, + ArrayRef outputBuffers) const { + builder.create(op->getLoc()); + return success(); +} + +} // namespace iree_compiler +} // namespace mlir diff --git a/compiler/Translation/SPIRV/IREEToSPIRV.h b/compiler/Translation/SPIRV/IREEToSPIRV.h new file mode 100644 index 000000000000..6b7944375f4b --- /dev/null +++ b/compiler/Translation/SPIRV/IREEToSPIRV.h @@ -0,0 +1,69 @@ +// Copyright 2019 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +//===- IREEToSPIRV.h -------------------------------------------*- C++//-*-===// +// +// Translation of IREE statements in dispatch functions to SPIR-V. +// +//===----------------------------------------------------------------------===// +#ifndef IREE_COMPILER_TRANSLATION_SPIRV_IREETOSPIRV_H +#define IREE_COMPILER_TRANSLATION_SPIRV_IREETOSPIRV_H + +#include "third_party/mlir_edge/iree/compiler/IR/Ops.h" +#include "third_party/mlir_edge/iree/compiler/IR/StructureOps.h" +#include "third_party/mlir_edge/iree/compiler/Translation/SPIRV/SPIRVLowering.h" + +namespace mlir { +namespace iree_compiler { + +/// Translation of iree.load_input operation. +class IREELoadOpSPIRVLowering final + : public SPIRVOpLowering { + public: + using SPIRVOpLowering::SPIRVOpLowering; + + LogicalResult lowerOperation(Operation *op, OpBuilder &builder, + AffineMap index, ArrayRef operands, + ValueCache &valueCache) const override; +}; + +/// Translation of iree.return operation. +class IREEReturnOpSPIRVLowering final : public SPIRVOpLowering { + public: + using SPIRVOpLowering::SPIRVOpLowering; + + LogicalResult lowerOperation( + Operation *op, OpBuilder &builder, AffineExprCodegen &codegen, + ValueCache &valueCache, + DenseMap &inputBuffers, + ArrayRef outputBuffers) const override; +}; + +/// Translation of iree.store_output operation. +class IREEStoreOpSPIRVLowering final + : public SPIRVOpLowering { + public: + using SPIRVOpLowering::SPIRVOpLowering; + + LogicalResult lowerOperation( + Operation *op, OpBuilder &builder, AffineExprCodegen &codegen, + ValueCache &valueCache, + DenseMap &inputBuffers, + ArrayRef outputBuffers) const override; +}; + +} // namespace iree_compiler +} // namespace mlir + +#endif // IREE_COMPILER_TRANSLATION_SPIRV_IREETOSPIRV_H diff --git a/compiler/Translation/SPIRV/IREEToSPIRVPass.cpp b/compiler/Translation/SPIRV/IREEToSPIRVPass.cpp new file mode 100644 index 000000000000..4e18f55a6778 --- /dev/null +++ b/compiler/Translation/SPIRV/IREEToSPIRVPass.cpp @@ -0,0 +1,118 @@ +// Copyright 2019 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +//===- IREEToSPIRVPass.cpp -------------------------------------*- C++//-*-===// +// +// Pass to translate iree executables for vulkan-spirv. +// +//===----------------------------------------------------------------------===// +#include "third_party/mlir_edge/iree/compiler/Translation/SPIRV/IREEToSPIRVPass.h" + +#include "third_party/llvm/llvm/projects/google_mlir/include/mlir/Dialect/SPIRV/SPIRVTypes.h" +#include "third_party/mlir_edge/iree/compiler/Translation/SPIRV/IREEIndexComputation.h" +#include "third_party/mlir_edge/iree/compiler/Translation/SPIRV/IREEToSPIRV.h" + +namespace mlir { +namespace iree_compiler { + +namespace { + +class IREEToSPIRVPass : public ModulePass { + void runOnModule() override; +}; + +} // namespace + +void IREEToSPIRVPass::runOnModule() { + auto module = getModule(); + OpBuilder builder(module.getBodyRegion()); + + // Initialize the index computation. + IndexPropagationList< + IndexPropagationOp, IndexPropagationOp, + IREELoadIndexPropagation, IREEStoreIndexPropagation, + NoBroadcastPwOpIndexPropagation, + NoBroadcastPwOpIndexPropagation, + NoBroadcastPwOpIndexPropagation, + NoBroadcastPwOpIndexPropagation, + NoBroadcastPwOpIndexPropagation, + NoBroadcastPwOpIndexPropagation, + NoBroadcastPwOpIndexPropagation, + NoBroadcastPwOpIndexPropagation, + ReshapeOpIndexPropagation, + NoBroadcastPwOpIndexPropagation, + XLABroadcastInDimOpIndexPropagation, XLATransposeOpIndexPropagation> + indexPropagation; + + // Initialize the spir-v codegenerator. + SPIRVCodegen, + IREELoadOpSPIRVLowering, IREEReturnOpSPIRVLowering, + IREEStoreOpSPIRVLowering, + SPIRVPwOpLowering, + SPIRVPwOpLowering, + SPIRVPwOpLowering, + SPIRVPwOpLowering, + SPIRVPwOpLowering, + SPIRVPwOpLowering, + SPIRVIndexOpLowering, + SPIRVIndexOpLowering, + SPIRVIndexOpLowering, + SPIRVIndexOpLowering> + spirvCodegen; + + // Create a spirv.module Op. + auto spvModule = builder.create( + module.getLoc(), + builder.getI32IntegerAttr( + static_cast(spirv::AddressingModel::Logical)), + builder.getI32IntegerAttr( + static_cast(spirv::MemoryModel::GLSL450))); + SmallVector caps; + caps.push_back(spirv::stringifyCapability(spirv::Capability::Shader)); + spvModule.setAttr("capabilities", builder.getStrArrayAttr(caps)); + SmallVector exts; + exts.push_back("SPV_KHR_storage_buffer_storage_class"); + spvModule.setAttr("extensions", builder.getStrArrayAttr(exts)); + + for (auto funcOp : module.getOps()) { + // TODO(ravishankarm): FuncOps in executable that are not dispatch functions + // are not lowered to SPIR-V. Fix this limitation. + if (!funcOp.getAttr("iree.executable.export")) continue; + + IndexComputationCache indexMap; + if (failed(indexPropagation.propagate(funcOp.getBody(), indexMap))) { + return signalPassFailure(); + } + // dumpIndexCache(indexMap); + + ValueCache valueCache; + AffineExprCodegen affineExprCodegen(spvModule, indexMap); + if (failed(spirvCodegen.codegen(spvModule, funcOp, affineExprCodegen, + valueCache))) { + return signalPassFailure(); + } + } +} + +std::unique_ptr> createIREEToSPIRVPass() { + return std::make_unique(); +} +static PassRegistration pass( + "convert-iree-to-spirv", + "Convert IREE dispatch functions to SPIR-V dialect"); + +} // namespace iree_compiler +} // namespace mlir diff --git a/compiler/Translation/SPIRV/IREEToSPIRVPass.h b/compiler/Translation/SPIRV/IREEToSPIRVPass.h new file mode 100644 index 000000000000..e20ca7274038 --- /dev/null +++ b/compiler/Translation/SPIRV/IREEToSPIRVPass.h @@ -0,0 +1,34 @@ +// Copyright 2019 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +//===- IREEToSPIRVPass.cpp -------------------------------------*- C++//-*-===// +// +// Pass to translate iree executables for vulkan-spirv. +// +//===----------------------------------------------------------------------===// +#ifndef IREE_COMPILER_TRANSLATION_SPIRV_IREETOSPIRVPASS_H +#define IREE_COMPILER_TRANSLATION_SPIRV_IREETOSPIRVPASS_H + +#include "third_party/llvm/llvm/projects/google_mlir/include/mlir/Pass/Pass.h" + +namespace mlir { +namespace iree_compiler { + +// Generates a spirv::ModuleOp from the module within an IREE Executable with +// target-config vulkan-spirv. +std::unique_ptr> createIREEToSPIRVPass(); +} // namespace iree_compiler +} // namespace mlir + +#endif // IREE_COMPILER_TRANSLATION_SPIRV_IREETOSPIRVPASS_H diff --git a/compiler/Translation/SPIRV/IndexComputation.cpp b/compiler/Translation/SPIRV/IndexComputation.cpp new file mode 100644 index 000000000000..c20b5b159d6b --- /dev/null +++ b/compiler/Translation/SPIRV/IndexComputation.cpp @@ -0,0 +1,269 @@ +// Copyright 2019 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +//===- IndexComputation.cpp ------------------------------------*- C++//-*-===// +// +// For an IREE dispatch function, compute the map from workitem ID to index of +// tensor computed within that workitem. +// +//===----------------------------------------------------------------------===// +#include "third_party/mlir_edge/iree/compiler/Translation/SPIRV/IndexComputation.h" + +#include "third_party/llvm/llvm/include/llvm/Support/CommandLine.h" +#include "third_party/llvm/llvm/include/llvm/Support/raw_ostream.h" + +static llvm::cl::opt doAffineExprSimplify( + "simplify-spirv-affine-exprs", + llvm::cl::desc("Simplify affine expressions during code-generation."), + llvm::cl::init(true)); + +namespace mlir { +namespace iree_compiler { + +//===----------------------------------------------------------------------===// +// Reshape Utility Functions +//===----------------------------------------------------------------------===// + +namespace { +/// Handles shapes for scalars. Shape of scalars are represented as empty vetor, +/// i.e. {}. Its easier to do index propogation to handle the scalar as vector +/// of size 1. +inline SmallVector handleIfScalar(ArrayRef shape) { + SmallVector resultShape; + if (shape.empty()) { + return {1}; + } + return SmallVector(shape.begin(), shape.end()); +} + +/// Reshapes are often used to either add a dimension of size 1 or remove a +/// dimension of size 1. Recognizing such cases can make the code-generation +/// easier. The AffineMap needs to either add a constant 0 in the range for such +/// added dimensions or drop those dimensions. +inline LogicalResult getAffineExprForAddOrRemoveDimension( + Builder &builder, ArrayRef resultExprs, + ArrayRef resultShape, ArrayRef operandShape, + SmallVectorImpl &operandExprs) { + auto resultIndex = resultShape.size(); + auto operandIndex = operandShape.size(); + operandExprs.resize(operandShape.size()); + // Try to match up the dimensions of the operand and result by ignoring any + // dimensions of size of 1 that are introduced. + while (resultIndex > 0 && operandIndex > 0) { + if (resultShape[resultIndex - 1] == -1 || + operandShape[operandIndex - 1] == -1) { + return failure(); + } + if (resultShape[resultIndex - 1] == operandShape[operandIndex - 1]) { + operandExprs[operandIndex - 1] = resultExprs[resultIndex - 1]; + resultIndex--; + operandIndex--; + continue; + } + if (resultShape[resultIndex - 1] == 1) { + // This is a dimension that is added on the operand. This affine + // expression corresponding to this dimension is dropped. + resultIndex--; + continue; + } + if (operandShape[operandIndex - 1] == 1) { + // This is a dimension of size 1 of the operand that is dropped. Add a + // constant expr 0. + operandExprs[operandIndex - 1] = builder.getAffineConstantExpr(0); + operandIndex--; + continue; + } + return failure(); + } + // Any remaining dimensions should be 1. + while (resultIndex > 0) { + if (resultShape[resultIndex - 1] != 1) { + return failure(); + } + resultIndex--; + } + while (operandIndex > 0) { + if (operandShape[operandIndex - 1] != 1) { + return failure(); + } + // This is a dimension of size 1 that is dropped. Add a constant expression + // 0. + operandExprs[operandIndex - 1] = builder.getAffineConstantExpr(0); + operandIndex--; + } + return success(); +} + +/// Constructs the strides of an array assuming a row-major packed layout. +// TODO(ravishankarm): This assumes the shape are static. When using dynamic +// shapes, parameters of each dimension can be used to construct AffineExpr for +// strides along each dimension. Note that multiplying two symbolic constants is +// technically not affine, but you could use another symbol to represent the +// product, so it should be still representable as affine exprs. +inline LogicalResult getRowMajorPackedStrides( + Builder &builder, ArrayRef shape, + SmallVectorImpl &strides) { + strides.resize(shape.size()); + int64_t stride = 1; + for (auto dim : enumerate(reverse(shape))) { + if (dim.value() < 0) { + // TODO(ravishankarm) : Better error message. + return failure(); + } + strides[shape.size() - 1 - dim.index()] = + builder.getAffineConstantExpr(stride); + stride *= dim.value(); + } + return success(); +} + +/// Linearizes the index of the result position accessed using the shape of the +/// result tensor and delinearizes it to get the position of the operand. +inline LogicalResult getAffineExprForReshape( + Builder &builder, unsigned numDims, unsigned numSymbols, + ArrayRef resultExprs, ArrayRef resultShape, + ArrayRef operandShape, SmallVectorImpl &operandExprs) { + // To linearize the index, assume that the memory is laid out in + // packed-row-major layout based on the shape. + // TODO(ravishankarm) : When there is stride information, use that to map from + // index to memory location. + SmallVector resultStrides; + if (failed(getRowMajorPackedStrides(builder, resultShape, resultStrides))) { + return failure(); + } + AffineExpr linearizedExpr; + for (auto index : enumerate(resultExprs)) { + auto val = getAffineBinaryOpExpr(AffineExprKind::Mul, index.value(), + resultStrides[index.index()]); + if (doAffineExprSimplify) { + val = simplifyAffineExpr(val, numDims, numSymbols); + } + linearizedExpr = (index.index() ? getAffineBinaryOpExpr(AffineExprKind::Add, + linearizedExpr, val) + : val); + if (doAffineExprSimplify) { + linearizedExpr = simplifyAffineExpr(val, numDims, numSymbols); + } + } + + // Unlinearize the index, assuming row-major-packed layout. + // TODO(ravishankarm) : When there is stride information, use that to map from + // memory location to index. + SmallVector operandStrides; + if (failed(getRowMajorPackedStrides(builder, operandShape, operandStrides))) { + return failure(); + } + operandExprs.resize(operandStrides.size()); + for (auto stride : enumerate(operandStrides)) { + if (stride.index() == operandStrides.size() - 1) { + operandExprs[stride.index()] = linearizedExpr; + break; + } + auto expr = getAffineBinaryOpExpr(AffineExprKind::FloorDiv, linearizedExpr, + stride.value()); + operandExprs[stride.index()] = + (doAffineExprSimplify ? simplifyAffineExpr(expr, numDims, numSymbols) + : expr); + + linearizedExpr = getAffineBinaryOpExpr(AffineExprKind::Mod, linearizedExpr, + stride.value()); + if (doAffineExprSimplify) { + linearizedExpr = simplifyAffineExpr(linearizedExpr, numDims, numSymbols); + } + } + return success(); +} +} // namespace + +LogicalResult getReshapeOperandMap(Builder &builder, AffineMap resultIndexMap, + ArrayRef resultShapeRef, + ArrayRef operandShapeRef, + AffineMap &operandIndexMap) { + auto resultShape = handleIfScalar(resultShapeRef); + auto operandShape = handleIfScalar(operandShapeRef); + auto resultExprs = resultIndexMap.getResults(); + assert(resultShape.size() == resultExprs.size() && + "Ranks of the Domain of index map and result must be the same"); + SmallVector operandExprs; + if (failed(getAffineExprForAddOrRemoveDimension( + builder, resultExprs, resultShape, operandShape, operandExprs)) && + failed(getAffineExprForReshape( + builder, resultIndexMap.getNumDims(), resultIndexMap.getNumSymbols(), + resultExprs, resultShape, operandShape, operandExprs))) { + return failure(); + } + assert(operandExprs.size() == operandShape.size() && + "expected as many exprs for the operand as the rank of the operand"); + operandIndexMap = + builder.getAffineMap(resultIndexMap.getNumDims(), + resultIndexMap.getNumSymbols(), operandExprs); + + return success(); +} + +LogicalResult IndexPropagation::propagateIndexMap( + Operation *op, IndexComputationCache &indexMap) const { + if (op->getNumResults() == 0) { + // Nothing to do for this op. + return success(); + } + if (op->getNumResults() != 1) { + return op->emitError( + "default index propagation handles case with a single-return value"); + } + // Initialize the storage for all the operands. + for (auto arg : op->getOperands()) { + indexMap[arg]; + } + for (auto &resultIndexMap : indexMap[op->getResult(0)]) { + SmallVector operandIndices; + if (failed(this->propagateIndexMap(op, resultIndexMap.first, + operandIndices))) { + return failure(); + } + assert(operandIndices.size() == op->getNumOperands() && + "Expected as many indices as operands"); + for (auto arg : enumerate(op->getOperands())) { + indexMap[arg.value()][operandIndices[arg.index()]]; + resultIndexMap.second.push_back(operandIndices[arg.index()]); + } + } + return success(); +} + +void dumpIndexCache(IndexComputationCache &indexMap) { + for (auto &el : indexMap) { + // llvm::errs() << "Value : " << *(el.first); + // llvm::errs().flush(); + if (el.first->getKind() == Value::Kind::OpResult) { + llvm::errs() << "Operation : " << el.first->getDefiningOp()->getName(); + } else if (el.first->getKind() == Value::Kind::BlockArgument) { + llvm::errs() << "BlockArgument"; + } + for (auto &used : el.second) { + llvm::errs() << "\n\t" << used.first << " : ["; + std::string sep = ""; + for (auto &operand : used.second) { + llvm::errs() << sep << operand; + sep = ", "; + } + llvm::errs() << "]"; + } + llvm::errs() << "\n"; + } + llvm::errs() << "\n"; +} + +} // namespace iree_compiler +} // namespace mlir diff --git a/compiler/Translation/SPIRV/IndexComputation.h b/compiler/Translation/SPIRV/IndexComputation.h new file mode 100644 index 000000000000..94472ed6e95f --- /dev/null +++ b/compiler/Translation/SPIRV/IndexComputation.h @@ -0,0 +1,268 @@ +// Copyright 2019 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +//===- IndexComputation.h ---------------------------------------*- C++ -*-===// +// +// For an IREE dispatch function, compute the map from workitem ID to index of +// tensor computed within that workitem. +// +//===----------------------------------------------------------------------===// +#ifndef IREE_COMPILER_TRANSLATION_SPIRV_INDEXCOMPUTATION_H +#define IREE_COMPILER_TRANSLATION_SPIRV_INDEXCOMPUTATION_H + +#include "third_party/llvm/llvm/include/llvm/ADT/ArrayRef.h" +#include "third_party/llvm/llvm/include/llvm/ADT/DenseMap.h" +#include "third_party/llvm/llvm/include/llvm/ADT/DenseSet.h" +#include "third_party/llvm/llvm/include/llvm/ADT/MapVector.h" +#include "third_party/llvm/llvm/include/llvm/ADT/SmallVector.h" +#include "third_party/llvm/llvm/projects/google_mlir/include/mlir/IR/AffineExpr.h" +#include "third_party/llvm/llvm/projects/google_mlir/include/mlir/IR/AffineMap.h" +#include "third_party/llvm/llvm/projects/google_mlir/include/mlir/IR/Builders.h" +#include "third_party/llvm/llvm/projects/google_mlir/include/mlir/IR/Operation.h" +#include "third_party/llvm/llvm/projects/google_mlir/include/mlir/IR/StandardTypes.h" +#include "third_party/llvm/llvm/projects/google_mlir/include/mlir/Support/LogicalResult.h" + +namespace mlir { +namespace iree_compiler { + +/// For each tensor Value* within the dispatch function, store the map +/// representing the index of that tensor needed for a workitem. For each index +/// also store the index of the operands needed to not recompute it later on. +/// TODO(ravishankarm): This sort of assumes each operation has a single result +/// value. Might need to be changed later on. +using IndexComputationCache = + DenseMap>>; + +/// Base class used to construct a map from opname to the +/// computation that propagates the index map from result to +/// operands. +class IndexPropagation { + public: + virtual ~IndexPropagation() = default; + virtual StringRef getOpName() const = 0; + + /// Propagates the index map from result to operands, i.e. given the index map + /// that represents the element(s) of result(s) used within a thread, compute + /// the indices of the operands needed. If overridden by inherited class the + /// implementation should evaluate the indices of the operands needed for all + /// indices of the result value needed. The dafult implementation only handles + /// operations with a zero or single-return value. + virtual LogicalResult propagateIndexMap( + Operation *operation, IndexComputationCache &indexMap) const; + + /// Propagates the index map from result to operands for a given index of the + /// result operand. + virtual LogicalResult propagateIndexMap( + Operation *operation, AffineMap resultIndex, + SmallVectorImpl &operandIndices) const { + // By default do-nothing. + return success(); + } +}; + +/// Base class that is templated on operation type to common +/// method to get the operation name. +template +class IndexPropagationOp : public IndexPropagation { + public: + using IndexPropagation::IndexPropagation; + virtual ~IndexPropagationOp() = default; + virtual StringRef getOpName() const { return OpTy::getOperationName(); } +}; + +// ===-------------------------------------------------------------------- ===// +// NoBroadCastPwOp +// ===-------------------------------------------------------------------- ===// + +/// Propagates the index map from result to operands for +/// operations that are point-wise and the operands are not +/// implicitly broadcasted. Just copies the index maps of the +/// result to the operands. +template +class NoBroadcastPwOpIndexPropagation : public IndexPropagationOp { + public: + using IndexPropagationOp::IndexPropagationOp; + + LogicalResult propagateIndexMap( + Operation *operation, AffineMap resultIndex, + SmallVectorImpl &operandIndices) const override { + // All operands must have the same type. + auto argRefType = + operation->getOperand(0)->getType().dyn_cast(); + if (!argRefType) { + return operation->emitError("expected operands to be of tensortype"); + } + for (auto arg : operation->getOperands()) { + auto argType = arg->getType().dyn_cast(); + if (!argType || argType.getShape() != argRefType.getShape()) { + return operation->emitError("expected operands to have same shape"); + } + operandIndices.push_back(resultIndex); + } + return success(); + } +}; + +// ===-------------------------------------------------------------------- ===// +// ReshapeOp +// ===-------------------------------------------------------------------- ===// + +/// Computes the index map representing the element of the operand accessed +/// given the index map of the result for a reshape operation. +LogicalResult getReshapeOperandMap(Builder &builder, AffineMap resultIndexMap, + ArrayRef resultShapeRef, + ArrayRef operandShapeRef, + AffineMap &operandIndexMap); + +/// Propagates the index map from result to operands of reshape type operations. +template +class ReshapeOpIndexPropagation final : public IndexPropagationOp { + public: + using IndexPropagationOp::IndexPropagationOp; + + LogicalResult propagateIndexMap( + Operation *op, AffineMap resultIndex, + SmallVectorImpl &operandIndices) const override { + Builder builder(op->getContext()); + auto resultType = + op->getResult(0)->getType().template dyn_cast(); + auto operandType = + op->getOperand(0)->getType().template dyn_cast(); + if (!resultType || !operandType) { + return op->emitError("expected result and operand to be shaped types"); + } + if (!resultType.hasStaticShape() || !operandType.hasStaticShape()) { + return op->emitError( + "unhandled non static shape of result/operands of reshape op"); + } + if (resultType.getNumElements() != operandType.getNumElements()) { + return op->emitError( + "invalid reshape operation, mismatch in number of elements in " + "operand and result"); + } + // Check if the reshape is a adding or removing a dimension of size 1 (and + // build the index for the operand accordingly). + AffineMap operandIndexMap; + if (failed(getReshapeOperandMap(builder, resultIndex, resultType.getShape(), + operandType.getShape(), operandIndexMap))) { + return op->emitError("unhandled reshape index propagation"); + } + operandIndices.push_back(operandIndexMap); + return success(); + } +}; + +/// Index map of the operand of a transpose op is obtained by composing the +/// affine map of the result with the affine map that represents the inverse of +/// the transpose permutation vector. The permutation vector must be supplied by +/// derived classes in the definition of the method `propagateIndexMap`. +template +class TransposeOpIndexPropagation : public IndexPropagationOp { + public: + using IndexPropagationOp::IndexPropagationOp; + virtual ~TransposeOpIndexPropagation() = default; + + protected: + LogicalResult propagateIndexMapImpl( + Operation *op, ArrayRef permutation, AffineMap resultIndex, + SmallVectorImpl &operandIndices) const { + Builder builder(op->getContext()); + SmallVector permutationExprs; + for (auto index : permutation) { + permutationExprs.push_back(builder.getAffineDimExpr(index)); + } + auto permutationAffineMap = + builder.getAffineMap(permutationExprs.size(), 0, permutationExprs); + // Compute the inverse of the permutation map. + auto invPermutationMap = inversePermutation(permutationAffineMap); + + // Compose the index map of the result with the index map + // for the inverse of the permutation. + auto operandMap = invPermutationMap.compose(resultIndex); + operandIndices.push_back(operandMap); + return success(); + } +}; + +/// Maintains list of IndexPropagation objects that propagate the index +/// information from result to the operands of instructions. +template +class IndexPropagationList { + using IndexPropagationListT = + llvm::StringMap>; + + public: + explicit IndexPropagationList() { insert(); } + + /// Performs the propogation. + LogicalResult propagate(Region ®ion, + IndexComputationCache &indexMap) const { + if (region.getBlocks().size() != 1) { + return emitError( + region.getLoc(), + "unimplemented handling multiple blocks within a region"); + } + // TODO(ravishankarm) : Need to process blocks in reverse topological order. + for (auto it = region.rbegin(), ie = region.rend(); it != ie; ++it) { + for (auto jt = it->rbegin(), je = it->rend(); jt != je; ++jt) { + auto &op = *jt; + auto opName = op.getName().getStringRef(); + if (!indexPropagationList.count(opName)) { + return op.emitError("unhandled index propogation"); + } + for (auto rt = op.result_begin(), re = op.result_end(); rt != re; + ++rt) { + auto resultValue = *rt; + auto type = resultValue->getType().dyn_cast(); + if (!type) { + return op.emitError("expected return value to be a tensor"); + } + if (!indexMap.count(*rt)) { + return op.emitError("missing index map of result"); + } + } + auto propagate = indexPropagationList.find(opName); + if (failed(propagate->getValue()->propagateIndexMap(&op, indexMap))) { + return failure(); + } + } + } + return success(); + } + + private: + /// Builds the list by unpacking the parameter pack. + void insert() { + std::vector> objs; + // TODO(ravishankarm) : This uses the fold logic from + // mlir/IR/PatternMatch.h. There might be a simpler/more efficient way of + // implementing this. + using dummy = int[]; + (void)dummy{0, (objs.emplace_back(std::make_unique()), 0)...}; + for (auto &elem : objs) { + StringRef opName = elem->getOpName(); + indexPropagationList.try_emplace(opName, std::move(elem)); + } + } + + /// List of methods for propogation indexed using opname. + IndexPropagationListT indexPropagationList; +}; + +/// Debug method to just dump the indexMap to llvm::errs. +void dumpIndexCache(IndexComputationCache &indexMap); + +} // namespace iree_compiler +} // namespace mlir +#endif // IREE_COMPILER_TRANSLATION_SPIRV_INDEXCOMPUTATION_H diff --git a/compiler/Translation/SPIRV/Kernels/matmul.comp b/compiler/Translation/SPIRV/Kernels/matmul.comp new file mode 100644 index 000000000000..33c3c74eea9e --- /dev/null +++ b/compiler/Translation/SPIRV/Kernels/matmul.comp @@ -0,0 +1,90 @@ +// Simple tiled GEMM. +// The compiler emits this handwritten kernel for GEMM operations. Right now +// this prevents fusion with the GEMM dispatch, however in the future we can +// do some SPIR-V module merging tricks to make fusing possible. +// +// Since this is a special well-known kernel we can use specialization constants +// if we want to create variants for GEMV. Alternative vendor-specific +// implementations (such as one using VK_NV_cooperative_matrix) will need to be +// in separate files as we need some special handing for required extensions. + +#version 450 + +layout(local_size_x = 16, local_size_y = 16, local_size_z = 1) in; + +layout(std430, binding = 0) buffer readonly arg0_binding { + float arg0[]; +}; +layout(std430, binding = 1) buffer readonly arg1_binding { + float arg1[]; +}; +layout(std430, binding = 2) buffer writeonly ret0_binding { + float ret0[]; +}; + +layout(push_constant) uniform io_shapes_push_constant { + // Shapes of [arg0, arg1, ret0]. + // arg0 = [b0, m, k] + // arg1 = [b0, k, n] + // ret0 = [b0, m, n] + layout(offset = 0) ivec4 io_shapes[3]; +}; + +// TODO(benvanik): only send these values - the rest are not needed. +uint kMatrixM = io_shapes[0].y; +uint kMatrixK = io_shapes[0].z; +uint kMatrixN = io_shapes[1].z; + +const uint kTileSize = gl_WorkGroupSize.x; // .x == .y +uint kTileCount = (kMatrixK - 1) / kTileSize + 1; + +shared float tile_lhs[kTileSize][kTileSize]; +shared float tile_rhs[kTileSize][kTileSize]; + +// TODO(benvanik): spec constants to remove the bounds checking. +// TODO(benvanik): rely on robustness to do the check for us. +// TODO(benvanik): treat as externs so the SPIR-V generator can plug in. +float ReadLHS(uint row, uint col) { + if (row < kMatrixM && col < kMatrixK) { + return arg0[row * kMatrixK + col]; + } else { + return 0.0; + } +} + +float ReadRHS(uint row, uint col) { + if (row < kMatrixK && col < kMatrixN) { + return arg1[row * kMatrixN + col]; + } else { + return 0.0; + } +} + +void WriteOut(uint row, uint col, float value) { + if (col < kMatrixN && row < kMatrixM) { + ret0[row * kMatrixN + col] = value; + } +} + +void main() { + uint matrix_row = gl_GlobalInvocationID.y; // 0..kMatrixM + uint matrix_col = gl_GlobalInvocationID.x; // 0..kMatrixK + uint tile_row = gl_LocalInvocationID.y; // 0..kTileSize + uint tile_col = gl_LocalInvocationID.x; // 0..kTileSize + float acc = 0.0; + for (uint t = 0; t < kTileCount; ++t) { + // Load one tile of the LHS and the RHS into local memory. + uint tiled_lhs_col = kTileSize * t + tile_col; + uint tiled_rhs_row = kTileSize * t + tile_row; + tile_lhs[tile_row][tile_col] = ReadLHS(matrix_row, tiled_lhs_col); + tile_rhs[tile_row][tile_col] = ReadRHS(tiled_rhs_row, matrix_col); + // Synchronize to make sure the LHS and RHS tiles are loaded. + barrier(); + for (uint k = 0; k < kTileSize; ++k) { + acc += tile_lhs[tile_row][k] * tile_rhs[k][tile_col]; + } + // Synchronize before loading the next tile to make sure acc is valid. + barrier(); + } + WriteOut(matrix_row, matrix_col, acc); +} diff --git a/compiler/Translation/SPIRV/SPIRVExecutableTranslation.cpp b/compiler/Translation/SPIRV/SPIRVExecutableTranslation.cpp new file mode 100644 index 000000000000..8c90b4da760d --- /dev/null +++ b/compiler/Translation/SPIRV/SPIRVExecutableTranslation.cpp @@ -0,0 +1,412 @@ +// Copyright 2019 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "third_party/mlir_edge/iree/compiler/Translation/SPIRV/SPIRVExecutableTranslation.h" + +#include +#include +#include +#include + +#include "third_party/flatbuffers/include/flatbuffers/flatbuffers.h" +#include "third_party/llvm/llvm/include/llvm/ADT/STLExtras.h" +#include "third_party/llvm/llvm/include/llvm/ADT/StringRef.h" +#include "third_party/llvm/llvm/include/llvm/Support/Debug.h" +#include "third_party/llvm/llvm/include/llvm/Support/ErrorHandling.h" +#include "third_party/llvm/llvm/projects/google_mlir/include/mlir/Dialect/SPIRV/SPIRVOps.h" +#include "third_party/llvm/llvm/projects/google_mlir/include/mlir/Dialect/SPIRV/Serialization.h" +#include "third_party/llvm/llvm/projects/google_mlir/include/mlir/IR/Attributes.h" +#include "third_party/llvm/llvm/projects/google_mlir/include/mlir/IR/Module.h" +#include "third_party/llvm/llvm/projects/google_mlir/include/mlir/Pass/Pass.h" +#include "third_party/llvm/llvm/projects/google_mlir/include/mlir/Pass/PassManager.h" +#include "third_party/llvm/llvm/projects/google_mlir/include/mlir/Support/LogicalResult.h" +#include "third_party/llvm/llvm/projects/google_mlir/include/mlir/Transforms/Passes.h" +#include "third_party/llvm/llvm/projects/google_mlir/include/mlir/Translation.h" +#include "third_party/mlir_edge/iree/compiler/Translation/SPIRV/IREEToSPIRVPass.h" +#include "third_party/mlir_edge/iree/compiler/Translation/SPIRV/Kernels/Kernels.h" +#include "third_party/mlir_edge/iree/compiler/Utils/OpUtils.h" +#include "third_party/mlir_edge/iree/compiler/Utils/TranslationUtils.h" +#include "third_party/mlir_edge/iree/schemas/executable_def_generated.h" +#include "third_party/mlir_edge/iree/schemas/spirv_executable_def_generated.h" +#include "third_party/tensorflow/compiler/mlir/xla/ir/hlo_ops.h" + +namespace mlir { +namespace iree_compiler { + +namespace { + +enum class KnownKernel { + kConv = 0, + kMatMul, +}; + +// Matches the given |executableOp| against a set of well-known kernels. +// Returns the KnownKernel the executable represents or None if no match occurs. +llvm::Optional matchKnownKernel(IREE::ExecutableOp executableOp) { + auto module = executableOp.getInnerModule(); + for (auto funcOp : module.getOps()) { + for (auto &block : funcOp) { + for (auto &op : block) { + if (isa(&op)) { + return KnownKernel::kConv; + } else if (isa(&op)) { + return KnownKernel::kMatMul; + } + } + } + } + return llvm::None; +} + +// Builds a SPIR-V executable from a well-known matmul executable. +// |out_def| will be populated with all required information for serialization. +LogicalResult buildMatMulExecutable(IREE::ExecutableOp executableOp, + iree::SpirVExecutableDefT *out_def) { + out_def->tag = "__matmul__"; + out_def->entry_points = {"main"}; + + auto *fileToc = spirv_kernels::Kernels_create(); + for (int i = 0; i < spirv_kernels::Kernels_size(); ++i) { + if (std::strcmp(fileToc[i].name, "matmul.spv") == 0) { + out_def->code.resize(fileToc[i].size / 4); + std::memcpy(out_def->code.data(), fileToc[i].data, fileToc[i].size); + break; + } + } + + auto pipelineLayoutDef = std::make_unique(); + pipelineLayoutDef->buffer_binding_set = 0; + + pipelineLayoutDef->descriptor_set_layouts.resize(1); + + auto dsl = std::make_unique<::iree::VkDescriptorSetLayoutDefT>(); + { + auto binding = std::make_unique<::iree::VkDescriptorSetLayoutBindingDefT>(); + binding->binding = 0; + binding->descriptor_count = 1; + binding->descriptor_type = 7; // VK_DESCRIPTOR_TYPE_STORAGE_BUFFER + binding->stage_flags = 0x00000020; // VK_SHADER_STAGE_COMPUTE_BIT + dsl->bindings.push_back(std::move(binding)); + } + { + auto binding = std::make_unique<::iree::VkDescriptorSetLayoutBindingDefT>(); + binding->binding = 1; + binding->descriptor_count = 1; + binding->descriptor_type = 7; // VK_DESCRIPTOR_TYPE_STORAGE_BUFFER + binding->stage_flags = 0x00000020; // VK_SHADER_STAGE_COMPUTE_BIT + dsl->bindings.push_back(std::move(binding)); + } + { + auto binding = std::make_unique<::iree::VkDescriptorSetLayoutBindingDefT>(); + binding->binding = 2; + binding->descriptor_count = 1; + binding->descriptor_type = 7; // VK_DESCRIPTOR_TYPE_STORAGE_BUFFER + binding->stage_flags = 0x00000020; // VK_SHADER_STAGE_COMPUTE_BIT + dsl->bindings.push_back(std::move(binding)); + } + pipelineLayoutDef->descriptor_set_layouts[0] = std::move(dsl); + + auto pushConstantRangeDef = + std::make_unique<::iree::VkPushConstantRangeDefT>(); + pushConstantRangeDef->offset = 0; + pushConstantRangeDef->size = sizeof(int32_t) * 4 * 3; + pushConstantRangeDef->stage_flags = + 0x00000020; // VK_SHADER_STAGE_COMPUTE_BIT; + pipelineLayoutDef->push_constant_ranges.push_back( + std::move(pushConstantRangeDef)); + + out_def->pipeline_layout = std::move(pipelineLayoutDef); + + return success(); +} + +class SPIRVTranslator { + public: + explicit SPIRVTranslator(ExecutableTranslationOptions options) + : options_(options) {} + + const ExecutableTranslationOptions &options() const { return options_; } + + // Returns a populated ExecutableDef or nullptr if translation is + // unsuccessful. + std::unique_ptr translateExecutable( + IREE::ExecutableOp executableOp); + + private: + // Returns a list of entry point names matching the expected export ordinals. + std::vector populateEntryPointNames( + IREE::ExecutableOp executableOp); + + // Translates the input module into the SPIR-V dialect and returns the + // serialized code words or empty if translation failed. + std::vector translateAndSerializeShaderModule( + IREE::ExecutableOp executableOp); + + // Returns a pipeline layout definition based on the bindings required. + std::unique_ptr<::iree::VkPipelineLayoutDefT> populatePipelineLayout( + spirv::ModuleOp spirvModuleOp); + + ExecutableTranslationOptions options_; +}; + +std::unique_ptr SPIRVTranslator::translateExecutable( + IREE::ExecutableOp executableOp) { + ::iree::SpirVExecutableDefT spirvExecutableDef; + if (auto knownKernel = matchKnownKernel(executableOp)) { + // This executable represents a well-known kernel. + switch (knownKernel.getValue()) { + case KnownKernel::kConv: + executableOp.emitOpError() << "Conv not yet implemented"; + return {}; + case KnownKernel::kMatMul: + if (failed(buildMatMulExecutable(executableOp, &spirvExecutableDef))) { + executableOp.emitOpError() << "Failed to splat in the matmul kernel"; + return {}; + } + break; + default: + llvm_unreachable("unhandled known kernel"); + break; + } + } else { + // The sequencer and runtime use ordinals instead of names. We provide the + // list of entry point names here that are then passed in + // VkShaderModuleCreateInfo. + spirvExecutableDef.entry_points = populateEntryPointNames(executableOp); + + // Translate the module and generate the SPIR-V code. + // The module is expected to be modified and must contain the metadata + // required to enable the following information needed for the + // SpirVExecutableDef to be extracted. + spirvExecutableDef.code = translateAndSerializeShaderModule(executableOp); + if (spirvExecutableDef.code.empty()) { + executableOp.emitError() + << "Failed to translate and serialize SPIR-V executable"; + return {}; + } + + // Reflect against the entry thunk to identify the required pipeline + // layout based on binding information. This is used by the runtime to + // create the VkPipelineLayout. + for (auto spirvModuleOp : + executableOp.getBlock().getOps()) { + spirvExecutableDef.pipeline_layout = + populatePipelineLayout(spirvModuleOp); + if (!spirvExecutableDef.pipeline_layout) { + spirvModuleOp.emitError() + << "Failed to generate pipeline for SPIR-V module"; + return {}; + } + break; + } + } + + // Pack the executable definition and get the bytes with the proper header. + // The header is used to verify the contents at runtime. + ::flatbuffers::FlatBufferBuilder fbb; + auto executableOffset = + ::iree::SpirVExecutableDef::Pack(fbb, &spirvExecutableDef); + ::iree::FinishSpirVExecutableDefBuffer(fbb, executableOffset); + std::vector bytes; + bytes.resize(fbb.GetSize()); + std::memcpy(bytes.data(), fbb.GetBufferPointer(), bytes.size()); + + OpBuilder builder(executableOp); + executableOp.setAttr("format", builder.getI32IntegerAttr(static_cast( + IREE::ExecutableFormat::SpirV))); + + auto executableDef = std::make_unique(); + executableDef->format = static_cast(IREE::ExecutableFormat::SpirV); + executableDef->contents = std::move(bytes); + return executableDef; +} + +std::vector SPIRVTranslator::populateEntryPointNames( + IREE::ExecutableOp executableOp) { + auto module = executableOp.getInnerModule(); + DenseMap entryPoints; + for (auto funcOp : module.getOps()) { + if (!funcOp.getAttr("iree.executable.export")) continue; + auto ordinalAttr = funcOp.getAttrOfType("iree.ordinal"); + entryPoints[ordinalAttr.getInt()] = funcOp.getName(); + } + std::vector entryPointNames(entryPoints.size()); + for (auto &entry : entryPoints) { + entryPointNames[entry.first] = entry.second.str(); + } + return entryPointNames; +} + +std::vector SPIRVTranslator::translateAndSerializeShaderModule( + IREE::ExecutableOp executableOp) { + auto module = executableOp.getInnerModule(); + + // We can use the workload hint to know what the expected dispatch workload + // is. If we want to remap this to make more sense for the operations we are + // performing we can do that here. + // + // Note that workloads are computed per entry point. There may be some + // dimensions of the workload that are static (in which case workloadAttr will + // have non-dynamic dims) and others that need to be taken from an argument + // shape (in which case workloadRef is the argument ordinal to take dynamic + // dimensions from). + // TODO(benvanik): make it just an arg instead? iree.workload special op? + // TODO(benvanik): instead of FuncOp have an iree.entry_point op with these. + for (auto funcOp : module.getOps()) { + // TODO(ravishankarm): FuncOps in executable that are not dispatch functions + // are not lowered to SPIR-V. Fix this limitation. + if (!funcOp.getAttr("iree.executable.export")) continue; + auto workloadAttr = + funcOp.getAttrOfType("iree.executable.workload"); + auto workloadRefAttr = + funcOp.getAttrOfType("iree.executable.workload_ref"); + std::array staticWorkloadDims = {-1, -1, -1}; + if (workloadAttr) { + for (unsigned i = 0; i < 3; ++i) { + if (auto dimAttr = + workloadAttr.getValue({i}).dyn_cast_or_null()) { + staticWorkloadDims[i] = dimAttr.getInt(); + } + } + } + std::array dynamicWorkloadDimRefs; + if (workloadRefAttr) { + for (unsigned i = 0; i < 3; ++i) { + if (staticWorkloadDims[i] == -1) { + dynamicWorkloadDimRefs[i] = + funcOp.getArgument(workloadRefAttr.getInt()); + } + } + } + + // Now staticWorkloadDims will have non-negative values for known dimensions + // and any dim with -1 will need to be pulled from the corresponding shape + // dimension of dynamicWorkloadDimRefs. + + // TODO(b/137868263): use this information to map from workgroup to + // invocation and perform indexing. + } + + // Lower module to spirv::ModuleOp. + auto spirvGenPasses = createPassManager(module.getContext(), options()); + spirvGenPasses->addPass(createIREEToSPIRVPass()); + if (failed(runPassPipeline(options(), spirvGenPasses.get(), module))) { + executableOp.emitError() << "Failed to generate spv.module"; + return {}; + } + + auto spvModules = module.getOps(); + if (std::distance(spvModules.begin(), spvModules.end()) != 1) { + executableOp.emitError() + << "Expected a single spv.module for an IREE executable op"; + return {}; + } + + // Serialize the spirv::ModuleOp into the binary that we will embed in the + // final flatbuffer. + std::vector spvBinaries; + for (auto spvModule : spvModules) { + SmallVector spvBinary; + if (failed(spirv::serialize(spvModule, spvBinary))) { + executableOp.emitError() << "Failed to serialize spv.module"; + return {}; + } + spvBinaries.insert(spvBinaries.end(), spvBinary.begin(), spvBinary.end()); + + // Clone the module into executableOp directly. + auto clonedModule = spvModule.clone(); + executableOp.getBlock().getOperations().insert( + std::prev(executableOp.getBlock().getOperations().end()), clonedModule); + } + // Remove the original code. + module.erase(); + + return spvBinaries; +} + +std::unique_ptr<::iree::VkPipelineLayoutDefT> +SPIRVTranslator::populatePipelineLayout(spirv::ModuleOp spirvModuleOp) { + // NOTE: we currently make some assumptions about this based on the expected + // ABI of the runtime. If we wanted to support more general shaders with more + // complex I/O we'd need to find a better way to communicate this through the + // VkPipelineLayoutDef. + auto pipelineLayoutDef = std::make_unique<::iree::VkPipelineLayoutDefT>(); + pipelineLayoutDef->buffer_binding_set = 0; + + // Build a set of descriptor_set -> binding -> variable. + // This makes it easier to write out the descriptor in a logical order, even + // though this is not strictly required. + int64_t maxDescriptorSetOrdinal = -1; + std::map> descriptorSets; + for (auto globalVar : + spirvModuleOp.getBlock().getOps()) { + auto descriptorSetAttr = + globalVar.getAttrOfType("descriptor_set"); + auto bindingAttr = globalVar.getAttrOfType("binding"); + if (!descriptorSetAttr || !bindingAttr) { + // Not something the runtime cares about. + continue; + } + maxDescriptorSetOrdinal = + std::max(descriptorSetAttr.getInt(), maxDescriptorSetOrdinal); + auto &descriptorSet = descriptorSets[descriptorSetAttr.getInt()]; + descriptorSet[bindingAttr.getInt()] = globalVar; + } + + // Create the individual layout and binding defs. + pipelineLayoutDef->descriptor_set_layouts.resize(maxDescriptorSetOrdinal + 1); + for (auto &descriptorSetBindings : descriptorSets) { + int32_t descriptorSet = descriptorSetBindings.first; + auto dsl = std::make_unique<::iree::VkDescriptorSetLayoutDefT>(); + + for (auto &globalVarBinding : descriptorSetBindings.second) { + auto binding = + std::make_unique<::iree::VkDescriptorSetLayoutBindingDefT>(); + binding->binding = globalVarBinding.first; + binding->descriptor_count = 1; + // TODO(benvanik): pull from type info. + binding->descriptor_type = 7; // VK_DESCRIPTOR_TYPE_STORAGE_BUFFER + binding->stage_flags = 0x00000020; // VK_SHADER_STAGE_COMPUTE_BIT + dsl->bindings.push_back(std::move(binding)); + } + + pipelineLayoutDef->descriptor_set_layouts[descriptorSet] = std::move(dsl); + } + + return pipelineLayoutDef; +} + +} // namespace + +llvm::Optional +translateExecutableToSPIRVExecutable(ArrayRef executableOps, + ExecutableTranslationOptions options) { + SPIRVTranslator translator(options); + ExecutableTranslationResult translationResult; + for (auto executableOp : llvm::make_early_inc_range(executableOps)) { + auto executableDef = translator.translateExecutable(executableOp); + if (!executableDef) { + executableOp.emitError() << "Failed to translate one or more executables"; + return llvm::None; + } + translationResult.executable_defs.push_back(std::move(executableDef)); + } + return translationResult; +} + +static ExecutableTranslationRegistration SPIRVExecutableTranslationRegistration( + "vulkan-spirv", translateExecutableToSPIRVExecutable); + +} // namespace iree_compiler +} // namespace mlir diff --git a/compiler/Translation/SPIRV/SPIRVExecutableTranslation.h b/compiler/Translation/SPIRV/SPIRVExecutableTranslation.h new file mode 100644 index 000000000000..e26da3079ac3 --- /dev/null +++ b/compiler/Translation/SPIRV/SPIRVExecutableTranslation.h @@ -0,0 +1,37 @@ +// Copyright 2019 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#ifndef THIRD_PARTY_MLIR_EDGE_IREE_COMPILER_TRANSLATION_SPIRV_SPIRVEXECUTABLETRANSLATION_H_ +#define THIRD_PARTY_MLIR_EDGE_IREE_COMPILER_TRANSLATION_SPIRV_SPIRVEXECUTABLETRANSLATION_H_ + +#include + +#include "third_party/llvm/llvm/projects/google_mlir/include/mlir/IR/Module.h" +#include "third_party/mlir_edge/iree/compiler/IR/StructureOps.h" +#include "third_party/mlir_edge/iree/compiler/Utils/TranslationUtils.h" + +namespace mlir { +namespace iree_compiler { + +// Translates an MLIR module into a SPIR-V executable. +// These executables are stored as FlatBuffers in the +// iree/schemas/spirv_executable_def.fbs schema. +llvm::Optional +translateExecutableToSPIRVExecutable(ArrayRef executableOps, + ExecutableTranslationOptions options = {}); + +} // namespace iree_compiler +} // namespace mlir + +#endif // THIRD_PARTY_MLIR_EDGE_IREE_COMPILER_TRANSLATION_SPIRV_SPIRVEXECUTABLETRANSLATION_H_ diff --git a/compiler/Translation/SPIRV/SPIRVLowering.cpp b/compiler/Translation/SPIRV/SPIRVLowering.cpp new file mode 100644 index 000000000000..224d99c96024 --- /dev/null +++ b/compiler/Translation/SPIRV/SPIRVLowering.cpp @@ -0,0 +1,131 @@ +// Copyright 2019 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +//===- SPIRVLowering.cpp ---------------------------------------*- C++//-*-===// +// +// SPIR-V Code-generation for XLA-HLO Ops within IREE Dispatch functions +// +//===----------------------------------------------------------------------===// +#include "third_party/mlir_edge/iree/compiler/Translation/SPIRV/SPIRVLowering.h" + +namespace mlir { +namespace iree_compiler { +//===----------------------------------------------------------------------===// +// ConstantOp +//===----------------------------------------------------------------------===// +LogicalResult ConstantOpSPIRVLowering::lowerOperation( + Operation *op, OpBuilder &builder, AffineMap index, ArrayRef, + ValueCache &valueCache) const { + auto constOp = cast(op); + auto attr = constOp.value().dyn_cast(); + if (!attr || !attr.isSplat()) { + return op->emitError( + "unhandled constant lowering unless value is a splat dense element " + "attribute"); + } + auto resultType = constOp.getResult()->getType(); + Type resultElemType; + if (resultType.isIntOrFloat()) { + resultElemType = resultType; + } else if (auto shapedType = resultType.dyn_cast()) { + resultElemType = shapedType.getElementType(); + } else { + return op->emitError("unhandled result type of constant : ") << resultType; + } + Attribute constVal = attr.getSplatValue(); + auto spirvConstOp = + builder.create(op->getLoc(), resultElemType, constVal); + valueCache.setOperandDstValue(constOp.getResult(), index, + spirvConstOp.getResult()); + return success(); +} + +//===----------------------------------------------------------------------===// +// CmpFOp +//===----------------------------------------------------------------------===// +LogicalResult CmpFOpSPIRVLowering::lowerOperation( + Operation *op, OpBuilder &builder, AffineMap index, + ArrayRef operands, ValueCache &valueCache) const { + if (operands.size() != 2) { + return op->emitError("expected two operands in spir-v lowering of CmpFOp"); + } + Operation *spirvOp = nullptr; + auto opInfo = op->getAttrOfType(CmpFOp::getPredicateAttrName()); + if (!opInfo) { + return op->emitError("expected CmpFOp to contain ") + << CmpFOp::getPredicateAttrName() << " attribute"; + } + auto boolType = builder.getI1Type(); + auto predicateVal = static_cast(opInfo.getInt()); + switch (predicateVal) { +#define DISPATCH(caseLabel, opName) \ + case caseLabel: \ + spirvOp = builder.create(op->getLoc(), boolType, operands[0], \ + operands[1]); \ + break; + + DISPATCH(CmpFPredicate::OEQ, spirv::FOrdEqualOp); + DISPATCH(CmpFPredicate::OGE, spirv::FOrdGreaterThanEqualOp); + DISPATCH(CmpFPredicate::OGT, spirv::FOrdGreaterThanOp); + DISPATCH(CmpFPredicate::OLE, spirv::FOrdLessThanEqualOp); + DISPATCH(CmpFPredicate::OLT, spirv::FOrdLessThanOp); + DISPATCH(CmpFPredicate::ONE, spirv::FOrdNotEqualOp); + DISPATCH(CmpFPredicate::UEQ, spirv::FUnordEqualOp); + DISPATCH(CmpFPredicate::UGE, spirv::FUnordGreaterThanEqualOp); + DISPATCH(CmpFPredicate::UGT, spirv::FUnordGreaterThanOp); + DISPATCH(CmpFPredicate::ULE, spirv::FUnordLessThanEqualOp); + DISPATCH(CmpFPredicate::ULT, spirv::FUnordLessThanOp); + DISPATCH(CmpFPredicate::UNE, spirv::FUnordNotEqualOp); + +#undef DISPATCH + + default: + return op->emitError("unhandled predicate attribute for SPIR-V lowering"); + } + valueCache.setOperandDstValue(op->getResult(0), index, spirvOp->getResult(0)); + return success(); +} + +//===----------------------------------------------------------------------===// +// ReturnOp +//===----------------------------------------------------------------------===// +LogicalResult ReturnOpSPIRVLowering::lowerOperation( + Operation *op, OpBuilder &builder, AffineExprCodegen &affineExprCodegen, + ValueCache &valueCache, + DenseMap &inputBuffers, + ArrayRef outputBuffers) const { + auto returnOp = cast(op); + if (returnOp.getNumOperands() != 1) { + return returnOp.emitError( + "unhandled lowering of return statement with multiple returns"); + } + auto returnTensor = returnOp.getOperand(0); + auto indices = affineExprCodegen.getIndices(returnTensor); + if (indices.size() != 1) { + return returnOp.emitError( + "expected to compute a single element of the return tensor"); + } + assert(outputBuffers.size() == 1 && "Expected a single output buffer"); + auto var = outputBuffers[0]; + auto ptr = genPointerOffset(builder, returnOp.getLoc(), affineExprCodegen, + indices[0], var); + auto scalarVal = valueCache.getOperandDstValue(returnTensor, indices[0]); + builder.create(returnOp.getLoc(), ptr, scalarVal, + /*memory_access = */ nullptr, + /*alignment = */ nullptr); + builder.create(returnOp.getLoc()); + return success(); +} +} // namespace iree_compiler +} // namespace mlir diff --git a/compiler/Translation/SPIRV/SPIRVLowering.h b/compiler/Translation/SPIRV/SPIRVLowering.h new file mode 100644 index 000000000000..4c663cb2f8e9 --- /dev/null +++ b/compiler/Translation/SPIRV/SPIRVLowering.h @@ -0,0 +1,593 @@ +// Copyright 2019 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +//===- SPIRVLowering.h -----------------------------------------*- C++//-*-===// +// +// SPIR-V Code-generation for tensor operations within IREE Dispatch functions +// +//===----------------------------------------------------------------------===// +#ifndef IREE_COMPILER_TRANSLATION_SPIRV_SPIRVLOWERING_H +#define IREE_COMPILER_TRANSLATION_SPIRV_SPIRVLOWERING_H + +#include "third_party/llvm/llvm/projects/google_mlir/include/mlir/Dialect/SPIRV/SPIRVDialect.h" +#include "third_party/llvm/llvm/projects/google_mlir/include/mlir/Dialect/SPIRV/SPIRVOps.h" +#include "third_party/llvm/llvm/projects/google_mlir/include/mlir/Support/StringExtras.h" +#include "third_party/mlir_edge/iree/compiler/Translation/SPIRV/AffineExprCodegen.h" + +namespace mlir { +namespace iree_compiler { + +class ValueCache { + public: + Value *getOperandDstValue(Value *value, AffineMap index) { + return convertedValueMap.lookup(value).lookup(index); + } + + void setOperandDstValue(Value *value, AffineMap index, Value *scalar) { + convertedValueMap[value][index] = scalar; + } + + private: + DenseMap> convertedValueMap; +}; + +/// Base class for lowering tensor operations in the dispatch function to SPIR-V +/// op. +class SPIRVLowering { + public: + virtual ~SPIRVLowering() = default; + virtual StringRef getOpName() = 0; + /// This method (in the derived class) should generate the scalar operation + /// corresponding the the tensor operation `op` to generate the value of the + /// result tensor at a particular `index`. The scalar value of the operands + /// needed to compute this value is passed in within `operands`. The methods + /// have to insert the scalar result value of the generated operation into the + /// `valueCache`. + virtual LogicalResult lowerOperation(Operation *op, OpBuilder &builder, + AffineMap index, + ArrayRef operands, + ValueCache &valueCache) const { + return failure(); + } + + /// This method (in the derived class) should generate the scalar operations + /// corresponding to the tensor operation `op`. This should be implemented + /// when the `op` has no result value, typically store operations and return + /// operations. + virtual LogicalResult lowerOperation( + Operation *op, OpBuilder &builder, AffineExprCodegen &affineExprCodegen, + ValueCache &valueCache, + DenseMap &inputBuffers, + ArrayRef outputBuffers) const { + return failure(); + } +}; + +/// Base class that gets the opName for the operation. +template +class SPIRVOpLowering : public SPIRVLowering { + public: + using SPIRVLowering::SPIRVLowering; + virtual ~SPIRVOpLowering() {} + StringRef getOpName() override { return OpTy::getOperationName(); } +}; + +/// SPIR-V lowering for ConstantOp. +class ConstantOpSPIRVLowering final : public SPIRVOpLowering { + public: + using SPIRVOpLowering::SPIRVOpLowering; + LogicalResult lowerOperation(Operation *op, OpBuilder &builder, + AffineMap index, ArrayRef operands, + ValueCache &valueCache) const override; +}; + +/// SPIR-V lowering for CmpFOp. +class CmpFOpSPIRVLowering final : public SPIRVOpLowering { + public: + using SPIRVOpLowering::SPIRVOpLowering; + + LogicalResult lowerOperation(Operation *op, OpBuilder &builder, + AffineMap index, ArrayRef operands, + ValueCache &valueCache) const override; +}; + +/// SPIR-V lowering for Min/Max operations. +template +class CmpSelectOpSPIRVLowering final : public SPIRVOpLowering { + public: + using SPIRVOpLowering::SPIRVOpLowering; + LogicalResult lowerOperation(Operation *op, OpBuilder &builder, + AffineMap index, ArrayRef operands, + ValueCache &valueCache) const override { + if (op->getNumOperands() != 2) { + return op->emitError( + "unhandled SPIR-V lowering for more than 2 operands"); + } + assert(operands.size() == op->getNumOperands() && + "expected as many operands for the replacement as the original " + "instruction"); + auto cmpSelectOp = cast(op); + auto result = cmpSelectOp.getResult(); + auto resultTy = result->getType().template dyn_cast(); + if (!resultTy) { + return op->emitError( + "unhandled lowering of operations that don't return a " + "ShapedType"); + } + auto elementTy = resultTy.getElementType(); + auto boolTy = builder.getI1Type(); + Operation *cmpOp = nullptr; + if (elementTy.template isa()) { + cmpOp = builder.create(op->getLoc(), boolTy, operands, + ArrayRef()); + } else { + cmpOp = builder.create(op->getLoc(), boolTy, operands, + ArrayRef()); + } + auto selectOp = builder.create( + op->getLoc(), operands[0]->getType(), cmpOp->getResult(0), operands[0], + operands[1]); + valueCache.setOperandDstValue(op->getResult(0), index, + selectOp.getResult()); + return success(); + } +}; + +/// This class is the general template used to emit scalar instruction +/// corresponding for point-wise operations. Assumes that the original +/// instruction has a single result value of type ShapedType. +/// TODO(ravishankarm) : In XLA-HLO, the same operations is used for +/// integer/float tensor operations. So allow this op to take an additional op +/// type as a template parameter to handle such cases. Find a better way to do +/// this. +template +class SPIRVPwOpLowering final : public SPIRVOpLowering { + public: + using SPIRVOpLowering::SPIRVOpLowering; + + LogicalResult lowerOperation(Operation *op, OpBuilder &builder, + AffineMap index, + ArrayRef scalarOperands, + ValueCache &valueCache) const override { + // TODO(ravishankarm) : This check should really be a static_assert. See if + // that can be changed. + if (op->getNumOperands() == 0) { + return op->emitError("expected op to have at least one operand"); + } + auto pwOp = cast(op); + auto result = pwOp.getResult(); + auto resultType = result->getType().template dyn_cast(); + if (!resultType) { + return op->emitError( + "unhandled lowering of operations that don't return a " + "ShapedType"); + } + auto elementType = resultType.getElementType(); + Operation *scalarOp = nullptr; + if (elementType.template isa()) { + scalarOp = builder + .create(op->getLoc(), elementType, + scalarOperands, + ArrayRef()) + .getOperation(); + } else { + scalarOp = + builder + .create(op->getLoc(), elementType, scalarOperands, + ArrayRef()) + .getOperation(); + } + if (!scalarOp) { + return op->emitError("unable to lower operation"); + } + valueCache.setOperandDstValue(pwOp.getResult(), index, + scalarOp->getResult(0)); + return success(); + } +}; + +/// This class is the general template used to emit scalar instruction for index +/// transformation instructions like transpose. Assumes a single result value +/// and a single operand +template +class SPIRVIndexOpLowering final : public SPIRVOpLowering { + public: + using SPIRVOpLowering::SPIRVOpLowering; + + LogicalResult lowerOperation(Operation *op, OpBuilder &builder, + AffineMap index, + ArrayRef scalarOperands, + ValueCache &valueCache) const override { + if (op->getNumOperands() != 1) { + return op->emitError( + "unhandled lowering of index transformation operation with multiple " + "operands"); + } + auto indexOp = cast(op); + valueCache.setOperandDstValue(indexOp.getResult(), index, + scalarOperands[0]); + return success(); + } +}; + +/// Ggenerates spv.AccessChain instruction to get the pointer value at a given +/// location of a spv.globalVariable. +inline Value *genPointerOffset(OpBuilder &builder, Location loc, + AffineExprCodegen &affineExprCodegen, + AffineMap indexMap, + spirv::GlobalVariableOp &var) { + auto basePtr = builder.create( + loc, var.type(), builder.getSymbolRefAttr(var.sym_name())); + auto varPtrType = var.type().cast().getPointeeType(); + // The variable has to be a struct type with a single element. + assert(varPtrType.isa() && + "expected variable type to be a spv.ptr>"); + auto varStructType = varPtrType.cast(); + assert(varStructType.getNumElements() == 1 && + "expected variable type to be a spv.ptr of spv.struct with a single " + "element"); + auto varType = varStructType.getElementType(0); + + SmallVector accessIndex; + /// For scalar values, the index-map computed with already map to the 0-th + /// element. For arrays, they map to the position accessed. So just for arrays + /// we need to add an extra 0 to index into the struct. + if (varType.isa() || + varType.isa()) { + auto i32Type = builder.getIntegerType(32); + auto zero = builder.create(loc, i32Type, + builder.getI32IntegerAttr(0)); + accessIndex.push_back(zero); + } + for (auto indexExpr : indexMap.getResults()) { + accessIndex.push_back(affineExprCodegen.getValue( + indexExpr, builder.saveInsertionPoint(), loc)); + } + return builder.create(loc, basePtr, accessIndex); +} + +/// Lower return statements during SPIR-V codegeneration. +class ReturnOpSPIRVLowering : public SPIRVOpLowering { + public: + using SPIRVOpLowering::SPIRVOpLowering; + + LogicalResult lowerOperation( + Operation *op, OpBuilder &builder, AffineExprCodegen &affineExprCodegen, + ValueCache &valueCache, + DenseMap &inputBuffers, + ArrayRef outputBuffers) const override; +}; + +/// Class to drive the SPIRV code-generation. +template +class SPIRVCodegen { + using OpCodegenListT = llvm::StringMap>; + + public: + explicit SPIRVCodegen() { insert(); } + + LogicalResult codegen(spirv::ModuleOp &spirvModule, FuncOp &fn, + AffineExprCodegen &affineExprCodegen, + ValueCache &valueCache) { + if (fn.getBlocks().size() != 1) { + return emitError( + fn.getLoc(), + "unimplemeneted handling multiple blocks within a function"); + } + + OpBuilder builder(spirvModule.body()); + // Create the entry function and generate global invocation ID. Creates a + // global variable for all inputs and output tensors. + return createEntryFn(builder, fn, affineExprCodegen, valueCache); + } + + private: + /// Helper method to create the entry function. Creates global variables for + /// all inputs and outputs. Inserts the spv.EntryPoint operations as well. + LogicalResult createEntryFn(OpBuilder &builder, FuncOp &fn, + AffineExprCodegen &affineExprCodegen, + ValueCache &valueCache) { + auto loc = fn.getLoc(); + // TODO(ravishankarm) : This should actually be part of the SPIR-V + // conversion framework in MLIR core. Move it there. + auto convertType = [&loc](Type t, + spirv::PointerType &varType) -> LogicalResult { + auto shapedType = t.dyn_cast(); + if (!shapedType) { + return emitError(loc, "expected ShapedType argument"); + } + auto elementType = shapedType.getElementType(); + if (!elementType.isIntOrFloat()) { + return emitError(loc, "unhandled element type ") + << elementType << " while lowering to SPIR-V"; + } + int64_t stride = elementType.getIntOrFloatBitWidth() / 8; + for (auto dim : reverse(shapedType.getShape())) { + if (dim <= 0) { + return emitError(loc, "expected tensor dimensions to be non-zero"); + } + elementType = spirv::ArrayType::get( + elementType, dim, + static_cast(stride)); + stride *= dim; + } + // TODO(ravishankarm): Verify that the type of the variable passes + // spirv-val. + varType = spirv::PointerType::get( + spirv::StructType::get(elementType, + static_cast(0)), + spirv::StorageClass::StorageBuffer); + return success(); + }; + + // Convert functions arguments and return values to + // spirv::GlobalVariables. All global variables are given a descriptor set + // of 0 and binding is the argument number. + auto fnType = fn.getType(); + auto descriptorSetAttrName = convertToSnakeCase( + stringifyDecoration(spirv::Decoration::DescriptorSet)); + auto bindingAttrName = + convertToSnakeCase(stringifyDecoration(spirv::Decoration::Binding)); + for (auto argType : enumerate(fnType.getInputs())) { + spirv::PointerType varType; + if (failed(convertType(argType.value(), varType))) { + return failure(); + } + auto varName = + fn.getName().str() + "_arg_" + std::to_string(argType.index()); + auto var = builder.create( + loc, builder.getTypeAttr(varType), builder.getStringAttr(varName), + nullptr); + // Set descriptor_set to 0. + var.setAttr(descriptorSetAttrName, builder.getI32IntegerAttr(0)); + // Set binding to argument number. + var.setAttr(bindingAttrName, builder.getI32IntegerAttr(argType.index())); + + inputArgToVariable[fn.getArgument(argType.index())] = var; + } + for (auto resType : enumerate(fnType.getResults())) { + spirv::PointerType varType; + if (failed(convertType(resType.value(), varType))) { + return failure(); + } + auto varName = + fn.getName().str() + "_res_" + std::to_string(resType.index()); + auto var = builder.create( + loc, builder.getTypeAttr(varType), builder.getStringAttr(varName), + nullptr); + // Set descriptor_set to 0. + var.setAttr(descriptorSetAttrName, builder.getI32IntegerAttr(0)); + // Set binding to (result number + num arguments) + var.setAttr( + bindingAttrName, + builder.getI32IntegerAttr(fnType.getNumInputs() + resType.index())); + + resultIndexToVariable.push_back(var); + } + + auto entryFnType = + builder.getFunctionType(ArrayRef(), ArrayRef()); + auto entryFn = builder.create(loc, fn.getName(), entryFnType, + ArrayRef()); + + // Start a scope to create an insertion guard to reset the builder once the + // function is lowered. + { + OpBuilder::InsertionGuard funcInsertGuard(builder); + builder.setInsertionPointToStart(entryFn.addEntryBlock()); + + // Create the Global invocation ID. + if (failed(createGlobalInvocationID(builder, fn.getLoc(), + affineExprCodegen))) { + return failure(); + } + + if (failed(lowerFunction(builder, fn, entryFn, affineExprCodegen, + valueCache))) { + return failure(); + } + } + + // Create the entry point instructions for the entry function. + if (failed(createEntryPoint(builder, loc, entryFn))) { + return failure(); + } + return success(); + } + + /// Creates the global variable for GlobalInvocationID, and gets the ID at x, + /// y and z dimensions. + LogicalResult createGlobalInvocationID(OpBuilder &builder, Location loc, + AffineExprCodegen &affineExprCodegen) { + auto moduleOp = builder.getInsertionBlock() + ->getParentOp() + ->getParentOfType(); + OpBuilder moduleBuilder(moduleOp.body()); + auto i32Type = builder.getIntegerType(32); + auto idType = builder.getVectorType(3, i32Type); + auto ptrIdType = + spirv::PointerType::get(idType, spirv::StorageClass::Input); + auto globalInvocationID = moduleBuilder.create( + loc, builder.getTypeAttr(ptrIdType), + builder.getStringAttr("globalInvocationID"), nullptr); + globalInvocationID.setAttr( + convertToSnakeCase(stringifyDecoration(spirv::Decoration::BuiltIn)), + builder.getStringAttr( + spirv::stringifyBuiltIn(spirv::BuiltIn::GlobalInvocationId))); + interface.push_back( + builder.getSymbolRefAttr(globalInvocationID.sym_name())); + + auto globalInvocationIDPtr = builder.create( + loc, ptrIdType, + builder.getSymbolRefAttr(globalInvocationID.getOperation())); + auto id = builder.create(loc, idType, globalInvocationIDPtr, + nullptr, nullptr); + auto id_x = builder.create( + loc, i32Type, id, builder.getArrayAttr(builder.getI32IntegerAttr(0))); + auto id_y = builder.create( + loc, i32Type, id, builder.getArrayAttr(builder.getI32IntegerAttr(1))); + auto id_z = builder.create( + loc, i32Type, id, builder.getArrayAttr(builder.getI32IntegerAttr(2))); + affineExprCodegen.setDimDstValue(0, id_x); + affineExprCodegen.setDimDstValue(1, id_y); + affineExprCodegen.setDimDstValue(2, id_z); + return success(); + } + + /// Method to load the values of globalVariables corresponding to the + /// arguments of the dispatch function at all indices needed within the + /// dispatch function. + LogicalResult initArgValues(OpBuilder &builder, Location loc, + AffineExprCodegen &affineExprCodegen, + ValueCache &valueCache, Value *origArg) { + for (auto indexMap : affineExprCodegen.getIndices(origArg)) { + auto var = inputArgToVariable.lookup(origArg); + if (!var) { + return emitError( + loc, "undefined SPIR-V global variable for tensor argument"); + } + auto ptr = + genPointerOffset(builder, loc, affineExprCodegen, indexMap, var); + auto elementType = + ptr->getType().template cast().getPointeeType(); + auto val = builder.create(loc, elementType, ptr, + /*memory_access =*/nullptr, + /*alignment = */ nullptr); + valueCache.setOperandDstValue(origArg, indexMap, val); + } + return success(); + } + + /// Adds the spv.EntryPointOp and records all the interface variables used in + /// the entryFn. + LogicalResult createEntryPoint(OpBuilder &builder, Location loc, + FuncOp entryFn) { + builder.create( + loc, + builder.getI32IntegerAttr( + static_cast(spirv::ExecutionModel::GLCompute)), + builder.getSymbolRefAttr(entryFn), builder.getArrayAttr(interface)); + builder.create( + loc, builder.getSymbolRefAttr(entryFn), + builder.getI32IntegerAttr( + static_cast(spirv::ExecutionMode::LocalSize)), + builder.getI32ArrayAttr({1, 1, 1})); + interface.clear(); + return success(); + } + + /// Lowers the body of the function in the original dialect to SPIR-V dialect. + LogicalResult lowerFunction(OpBuilder &builder, FuncOp fn, FuncOp entryFn, + AffineExprCodegen &affineExprCodegen, + ValueCache &valueCache) { + for (auto arg : fn.getArguments()) { + // Load values of the argument at all indices needed for computation + // within the dispatch function. + if (failed(initArgValues(builder, fn.getLoc(), affineExprCodegen, + valueCache, arg))) { + return failure(); + } + } + + for (auto &block : fn) { + for (auto &op : block) { + // Lower individual operations. + if (failed( + lowerOperation(builder, affineExprCodegen, valueCache, &op))) { + return failure(); + } + } + } + return success(); + } + + /// Dispatches the lowering of tensor operation to SPIR-V scalar + /// operation. + LogicalResult lowerOperation(OpBuilder &builder, + AffineExprCodegen &affineExprCodegen, + ValueCache &valueCache, Operation *op) { + auto opName = op->getName().getStringRef(); + if (!opCodegenList.count(opName)) { + return op->emitError("unhandled codegen"); + } + if (op->getNumResults() > 1) { + return op->emitError("unhandled codegen for multiple result values"); + } + + // Zero return case. + if (!op->getNumResults()) { + return opCodegenList[opName]->lowerOperation( + op, builder, affineExprCodegen, valueCache, inputArgToVariable, + resultIndexToVariable); + } + + // Single return case. + auto resultTensor = op->getResult(0); + auto indices = affineExprCodegen.getIndices(resultTensor); + for (auto &index : indices) { + auto operandIndices = + affineExprCodegen.getOperandIndices(resultTensor, index); + SmallVector scalarOperands; + for (auto arg : llvm::enumerate(op->getOperands())) { + auto scalarArg = valueCache.getOperandDstValue( + arg.value(), operandIndices[arg.index()]); + if (!scalarArg) { + return op->emitError("argument ") + << arg.index() << " has no scalar value"; + } + scalarOperands.push_back(scalarArg); + } + if (failed(opCodegenList[opName]->lowerOperation( + op, builder, index, scalarOperands, valueCache))) { + return failure(); + } + } + return success(); + } + + void insert() { + std::vector> objs; + using dummy = int[]; + (void)dummy{0, (objs.emplace_back(std::make_unique()), 0)...}; + for (auto &elem : objs) { + StringRef opName = elem->getOpName(); + opCodegenList.try_emplace(opName, std::move(elem)); + } + } + + /// List of classes that implement the operation lowering from tensor + /// operations to SPIR-V. + OpCodegenListT opCodegenList; + + /// I/O interface for the entry function containing global variables that are + /// used by the entire function call tree. + SmallVector interface; + + /// Mapping from argument of the dispatch function in tensor dialect to the + /// corresponding spv.globalVariable. + DenseMap inputArgToVariable; + + /// List of spv.globalVariables created for tensors returned by the dispatch + /// function in tensor dialects. + SmallVector resultIndexToVariable; + + /// GlobalInvocationID variable. + spirv::GlobalVariableOp globalInvocationID; +}; + +} // namespace iree_compiler +} // namespace mlir + +#endif // IREE_COMPILER_TRANSLATION_SPIRV_SPIRVLOWERING_H diff --git a/compiler/Translation/SPIRV/XLAIndexPropagation.cpp b/compiler/Translation/SPIRV/XLAIndexPropagation.cpp new file mode 100644 index 000000000000..1f43e28dd0b6 --- /dev/null +++ b/compiler/Translation/SPIRV/XLAIndexPropagation.cpp @@ -0,0 +1,80 @@ +// Copyright 2019 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +//===- XLAIndexPropagation.cpp ---------------------------------*- C++//-*-===// +// +// For an IREE dispatch function in XLA-HLO dialect, compute the indices of all +// tensors needed to produce the value of the result tensors at a particlar +// index. +// +//===----------------------------------------------------------------------===// +#include "third_party/mlir_edge/iree/compiler/Translation/SPIRV/XLAIndexPropagation.h" + +namespace mlir { +namespace iree_compiler { + +//===----------------------------------------------------------------------===// +// BroadcastInDimOp +//===----------------------------------------------------------------------===// + +LogicalResult XLABroadcastInDimOpIndexPropagation::propagateIndexMap( + Operation *op, AffineMap resultIndex, + SmallVectorImpl &operandIndices) const { + auto broadcastOp = cast(op); + auto broadcastDim = broadcastOp.broadcast_dimensions(); + + Builder builder(op->getContext()); + if (!broadcastDim) { + // This is a scalar. So all indices map to the same element. + AffineMap scalarMap = builder.getAffineMap( + resultIndex.getNumDims(), resultIndex.getNumSymbols(), + builder.getAffineConstantExpr(0)); + operandIndices.push_back(scalarMap); + return success(); + } + + // Handle non-scalar cases. + auto dimensions = broadcastDim->getValues(); + SmallVector exprs; + for (auto resultExpr : enumerate(resultIndex.getResults())) { + if (llvm::any_of(dimensions, [&resultExpr](int64_t dim) { + return dim == resultExpr.index(); + })) { + exprs.push_back(resultExpr.value()); + } + } + auto operandMap = builder.getAffineMap(resultIndex.getNumDims(), + resultIndex.getNumSymbols(), exprs); + operandIndices.push_back(operandMap); + return success(); +} + +//===----------------------------------------------------------------------===// +// TransposeOp +//===----------------------------------------------------------------------===// + +LogicalResult XLATransposeOpIndexPropagation::propagateIndexMap( + Operation *op, AffineMap resultIndex, + SmallVectorImpl &operandIndices) const { + auto transposeOp = cast(op); + // Compute the affine map that represents the permutation. + SmallVector permutation; + for (auto index : transposeOp.permutation()) { + permutation.push_back(index.getZExtValue()); + } + return propagateIndexMapImpl(op, permutation, resultIndex, operandIndices); +} + +} // namespace iree_compiler +} // namespace mlir diff --git a/compiler/Translation/SPIRV/XLAIndexPropagation.h b/compiler/Translation/SPIRV/XLAIndexPropagation.h new file mode 100644 index 000000000000..bcbaf6c6b54d --- /dev/null +++ b/compiler/Translation/SPIRV/XLAIndexPropagation.h @@ -0,0 +1,102 @@ +// Copyright 2019 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +//===- XLAIndexPropagation.h -----------------------------------*- C++//-*-===// +// +// For an IREE dispatch function in XLA-HLO dialect, compute the indices of all +// tensors needed to produce the value of the result tensors at a particlar +// index. +// +//===----------------------------------------------------------------------===// +#ifndef IREE_COMPILER_TRANSLATION_SPIRV_XLAINDEXPROPOGATION_H +#define IREE_COMPILER_TRANSLATION_SPIRV_XLAINDEXPROPOGATION_H + +#include "third_party/llvm/llvm/projects/google_mlir/include/mlir/Dialect/StandardOps/Ops.h" +#include "third_party/llvm/llvm/projects/google_mlir/include/mlir/IR/Function.h" +#include "third_party/mlir_edge/iree/compiler/Translation/SPIRV/IndexComputation.h" +#include "third_party/tensorflow/compiler/mlir/xla/ir/hlo_ops.h" + +namespace mlir { +namespace iree_compiler { + +class XLABroadcastInDimOpIndexPropagation final + : public IndexPropagationOp { + public: + using IndexPropagationOp::IndexPropagationOp; + + LogicalResult propagateIndexMap( + Operation *operation, AffineMap resultIndex, + SmallVectorImpl &operandIndices) const override; +}; + +/// For return ops, it is assumed that each thread is computing the value of one +/// element of the returned tensor. +template +class ReturnOpIndexPropagation : public IndexPropagationOp { + public: + using IndexPropagationOp::IndexPropagationOp; + + LogicalResult propagateIndexMap( + Operation *operation, IndexComputationCache &indexMap) const override { + if (operation->getNumOperands() != 1) { + return operation->emitError("unhandled multiple return values"); + } + auto returnValue = operation->getOperand(0); + auto returnType = returnValue->getType().cast(); + auto returnRank = returnType.getRank(); + if (returnRank > 3) { + return operation->emitError("unhandled return tensor of dimension ") + << returnType.getShape().size(); + } + // Have as many symbols as the rank of the input tensor. These symbols map + // to GlobalInvocationID along the three dimensions. + Builder builder(operation->getContext()); + SmallVector affineExprs; + for (size_t i = returnRank; i > 0; --i) { + affineExprs.push_back(builder.getAffineDimExpr(i - 1)); + } + indexMap[operation->getOperand(0)] + [builder.getAffineMap(returnRank, 0, affineExprs)]; + return success(); + } +}; + +/// Index propogation for XLA Transpose. +class XLATransposeOpIndexPropagation final + : public TransposeOpIndexPropagation { + public: + using TransposeOpIndexPropagation< + xla_hlo::TransposeOp>::TransposeOpIndexPropagation; + LogicalResult propagateIndexMap( + Operation *op, AffineMap resultIndex, + SmallVectorImpl &indexMap) const override; +}; + +/// Utility function that does index propagation for a fn that has only XLA-HLO +/// ops. +inline LogicalResult propagateIndexForXLAFn(FuncOp fn, + IndexComputationCache &map) { + IndexPropagationList, + NoBroadcastPwOpIndexPropagation, + ReturnOpIndexPropagation, + XLATransposeOpIndexPropagation> + indexList; + if (!fn.getOperation()->getNumRegions()) { + return success(); + } + return indexList.propagate(fn.getOperation()->getRegion(0), map); +} +} // namespace iree_compiler +} // namespace mlir +#endif // IREE_COMPILER_TRANSLATION_SPIRV_XLAINDEXPROPOGATION_H diff --git a/compiler/Translation/SPIRV/test/simple_test.mlir b/compiler/Translation/SPIRV/test/simple_test.mlir new file mode 100644 index 000000000000..b2c06c6a6e57 --- /dev/null +++ b/compiler/Translation/SPIRV/test/simple_test.mlir @@ -0,0 +1,674 @@ +// RUN: iree-opt -split-input-file -convert-iree-to-spirv -simplify-spirv-affine-exprs=false -verify-diagnostics -o - %s | FileCheck %s + +module { + // CHECK:spv.module "Logical" "GLSL450" + // CHECK: spv.globalVariable [[GLOBALIDVAR:@.*]] built_in("GlobalInvocationId") : !spv.ptr, Input> + // CHECK: spv.globalVariable [[ARG0VAR:@.*]] bind(0, 0) : !spv.ptr [168]> [0]>, StorageBuffer> + // CHECK: spv.globalVariable [[ARG1VAR:@.*]] bind(0, 1) : !spv.ptr [168]> [0]>, StorageBuffer> + // CHECK: func [[FN:@simple_load_store_entry_dispatch_0]]() + func @simple_load_store_entry_dispatch_0(%arg0: memref<12x42xi32>, %arg1: memref<12x42xi32>) + attributes {iree.executable.export, iree.executable.workload = dense<[42, 12, 1]> : tensor<3xi32>, iree.ordinal = 0 : i32} { + // CHECK: [[GLOBALIDPTR:%.*]] = spv._address_of [[GLOBALIDVAR]] + // CHECK: [[GLOBALID:%.*]] = spv.Load "Input" [[GLOBALIDPTR]] + // CHECK: [[GLOBALIDX:%.*]] = spv.CompositeExtract [[GLOBALID]][0 : i32] + // CHECK: [[GLOBALIDY:%.*]] = spv.CompositeExtract [[GLOBALID]][1 : i32] + // CHECK: [[ARG0PTR:%.*]] = spv._address_of [[ARG0VAR]] + // CHECK: [[ZERO1:%.*]] = spv.constant 0 : i32 + // CHECK: [[ARG0LOADPTR:%.*]] = spv.AccessChain [[ARG0PTR]]{{\[}}[[ZERO1]], [[GLOBALIDY]], [[GLOBALIDX]]{{\]}} + // CHECK: [[VAL:%.*]] = spv.Load "StorageBuffer" [[ARG0LOADPTR]] + %0 = iree.load_input(%arg0 : memref<12x42xi32>) : tensor<12x42xi32> + %1 = "xla_hlo.copy"(%0) : (tensor<12x42xi32>) -> tensor<12x42xi32> + // CHECK: [[ARG1PTR:%.*]] = spv._address_of [[ARG1VAR]] + // CHECK: [[ZERO2:%.*]] = spv.constant 0 : i32 + // CHECK: [[ARG1STOREPTR:%.*]] = spv.AccessChain [[ARG1PTR]]{{\[}}[[ZERO2]], [[GLOBALIDY]], [[GLOBALIDX]]{{\]}} + // CHECK: spv.Store "StorageBuffer" [[ARG1STOREPTR]], [[VAL]] + iree.store_output(%1 : tensor<12x42xi32>, %arg1 : memref<12x42xi32>) + // CHECK: spv.Return + iree.return + } + // CHECK: spv.EntryPoint "GLCompute" [[FN]], [[GLOBALIDVAR]] +} + +// ----- + +module { + // CHECK: spv.globalVariable [[GLOBALIDVAR:@.*]] built_in("GlobalInvocationId") : !spv.ptr, Input> + // CHECK: spv.globalVariable [[ARG0VAR:@.*]] bind(0, 0) + // CHECK: spv.globalVariable [[ARG1VAR:@.*]] bind(0, 1) + // CHECK: spv.globalVariable [[ARG2VAR:@.*]] bind(0, 2) + func @simple_mul_entry_dispatch_0(%arg0: memref<4xf32>, %arg1: memref<4xf32>, %arg2: memref<4xf32>) + attributes {iree.executable.export, iree.executable.workload = dense<[4, 1, 1]> : tensor<3xi32>, iree.ordinal = 0 : i32} { + // CHECK: [[GLOBALIDPTR:%.*]] = spv._address_of [[GLOBALIDVAR]] + // CHECK: [[GLOBALID:%.*]] = spv.Load "Input" [[GLOBALIDPTR]] + // CHECK: [[GLOBALIDX:%.*]] = spv.CompositeExtract [[GLOBALID]][0 : i32] + // CHECK: [[ARG0PTR:%.*]] = spv._address_of [[ARG0VAR]] + // CHECK: [[ZERO1:%.*]] = spv.constant 0 : i32 + // CHECK: [[ARG0LOADPTR:%.*]] = spv.AccessChain [[ARG0PTR]]{{\[}}[[ZERO1]], [[GLOBALIDX]]{{\]}} + // CHECK: [[VAL1:%.*]] = spv.Load "StorageBuffer" [[ARG0LOADPTR]] + %0 = iree.load_input(%arg0 : memref<4xf32>) : tensor<4xf32> + // CHECK: [[ARG1PTR:%.*]] = spv._address_of [[ARG1VAR]] + // CHECK: [[ZERO2:%.*]] = spv.constant 0 : i32 + // CHECK: [[ARG1LOADPTR:%.*]] = spv.AccessChain [[ARG1PTR]]{{\[}}[[ZERO2]], [[GLOBALIDX]]{{\]}} + // CHECK: [[VAL2:%.*]] = spv.Load "StorageBuffer" [[ARG1LOADPTR]] + %1 = iree.load_input(%arg1 : memref<4xf32>) : tensor<4xf32> + // CHECK: [[RESULT:%.*]] = spv.FMul [[VAL1]], [[VAL2]] + %2 = mulf %0, %1 : tensor<4xf32> + // CHECK: [[ARG2PTR:%.*]] = spv._address_of [[ARG2VAR]] + // CHECK: [[ZERO3:%.*]] = spv.constant 0 : i32 + // CHECK: [[ARG2STOREPTR:%.*]] = spv.AccessChain [[ARG2PTR]]{{\[}}[[ZERO3]], [[GLOBALIDX]]{{\]}} + // CHECK: spv.Store "StorageBuffer" [[ARG2STOREPTR]], [[RESULT]] + iree.store_output(%2 : tensor<4xf32>, %arg2 : memref<4xf32>) + iree.return + } +} + +// ----- + +module { + // CHECK: spv.globalVariable [[GLOBALIDVAR:@.*]] built_in("GlobalInvocationId") : !spv.ptr, Input> + // CHECK: spv.globalVariable [[ARG0VAR:@.*]] bind(0, 0) + // CHECK: spv.globalVariable [[ARG1VAR:@.*]] bind(0, 1) + func @simple_transpose_entry_dispatch_0(%arg0: memref<12x12xf32>, %arg1: memref<12x12xf32>) + attributes {iree.executable.export, iree.executable.workload = dense<[12, 12, 1]> : tensor<3xi32>, iree.ordinal = 0 : i32} { + // CHECK: [[GLOBALIDPTR:%.*]] = spv._address_of [[GLOBALIDVAR]] + // CHECK: [[GLOBALID:%.*]] = spv.Load "Input" [[GLOBALIDPTR]] + // CHECK: [[GLOBALIDX:%.*]] = spv.CompositeExtract [[GLOBALID]][0 : i32] + // CHECK: [[GLOBALIDY:%.*]] = spv.CompositeExtract [[GLOBALID]][1 : i32] + // CHECK: [[ARG0PTR:%.*]] = spv._address_of [[ARG0VAR]] + // CHECK: [[ZERO1:%.*]] = spv.constant 0 : i32 + // CHECK: [[ARG0LOADPTR:%.*]] = spv.AccessChain [[ARG0PTR]]{{\[}}[[ZERO1]], [[GLOBALIDY]], [[GLOBALIDX]]{{\]}} + // CHECK: [[VAL1:%.*]] = spv.Load "StorageBuffer" [[ARG0LOADPTR]] + // CHECK: [[ARG1PTR:%.*]] = spv._address_of [[ARG0VAR]] + // CHECK: [[ZERO2:%.*]] = spv.constant 0 : i32 + // CHECK: [[ARG1LOADPTR:%.*]] = spv.AccessChain [[ARG1PTR]]{{\[}}[[ZERO2]], [[GLOBALIDX]], [[GLOBALIDY]]{{\]}} + // CHECK: [[VAL2:%.*]] = spv.Load "StorageBuffer" [[ARG1LOADPTR]] + %0 = iree.load_input(%arg0 : memref<12x12xf32>) : tensor<12x12xf32> + %1 = "xla_hlo.transpose"(%0) {permutation = dense<[1, 0]> : tensor<2xi64>} : (tensor<12x12xf32>) -> tensor<12x12xf32> + // CHECK: [[RESULT:%.*]] = spv.FAdd [[VAL1]], [[VAL2]] + %2 = "xla_hlo.add"(%0, %1) : (tensor<12x12xf32>, tensor<12x12xf32>) -> tensor<12x12xf32> + iree.store_output(%2 : tensor<12x12xf32>, %arg1 : memref<12x12xf32>) + iree.return + } +} + +// ----- + +module { + func @simple_load_store_entry_dispatch_0(%arg0: memref<12x42xi32>, %arg1: memref<12x42xi32>) + attributes {iree.executable.export, iree.executable.workload = dense<[42, 1, 1]> : tensor<3xi32>, iree.ordinal = 0 : i32} { + %0 = iree.load_input(%arg0 : memref<12x42xi32>) : tensor<12x42xi32> + // expected-error @+1 {{unable to map from launch id to element to compute within a workitem}} + iree.store_output(%0 : tensor<12x42xi32>, %arg1 : memref<12x42xi32>) + iree.return + } +} + +// ----- + +module { + func @simple_load_store_entry_dispatch_0(%arg0: memref<12x42xi32>, %arg1: memref<12x42xi32>) + attributes {iree.executable.export, iree.executable.workload = dense<[42]> : tensor<1xi32>, iree.ordinal = 0 : i32} { + %0 = iree.load_input(%arg0 : memref<12x42xi32>) : tensor<12x42xi32> + // expected-error @+1 {{unable to map from launch id to element to compute within a workitem}} + iree.store_output(%0 : tensor<12x42xi32>, %arg1 : memref<12x42xi32>) + iree.return + } +} + +// ----- + +module { + func @simple_load_store_entry_dispatch_0(%arg0: memref<42xi32>, %arg1: memref<42xi32>) + attributes {iree.executable.export, iree.executable.workload = dense<[42, 12, 1]> : tensor<3xi32>, iree.ordinal = 0 : i32} { + %0 = iree.load_input(%arg0 : memref<42xi32>) : tensor<42xi32> + // expected-error @+1 {{unable to map from launch id to element to compute within a workitem}} + iree.store_output(%0 : tensor<42xi32>, %arg1 : memref<42xi32>) + iree.return + } +} + +// ----- + +module { + // CHECK:spv.module "Logical" "GLSL450" + // CHECK: spv.globalVariable [[GLOBALIDVAR:@.*]] built_in("GlobalInvocationId") : !spv.ptr, Input> + // CHECK: spv.globalVariable [[ARG0VAR:@.*]] bind(0, 0) + // CHECK: spv.globalVariable [[ARG1VAR:@.*]] bind(0, 1) + // CHECK: func [[FN:@simple_load_store_entry_dispatch_0]]() + func @simple_load_store_entry_dispatch_0(%arg0: memref<12x42x1xi32>, %arg1: memref<12x42xi32>) + attributes {iree.executable.export, iree.executable.workload = dense<[42, 12, 1]> : tensor<3xi32>, iree.ordinal = 0 : i32} { + // CHECK: [[GLOBALIDPTR:%.*]] = spv._address_of [[GLOBALIDVAR]] + // CHECK: [[GLOBALID:%.*]] = spv.Load "Input" [[GLOBALIDPTR]] + // CHECK: [[GLOBALIDX:%.*]] = spv.CompositeExtract [[GLOBALID]][0 : i32] + // CHECK: [[GLOBALIDY:%.*]] = spv.CompositeExtract [[GLOBALID]][1 : i32] + // CHECK: [[ARG0PTR:%.*]] = spv._address_of [[ARG0VAR]] + // CHECK: [[ZERO1:%.*]] = spv.constant 0 : i32 + // CHECK: [[ZERO2:%.*]] = spv.constant 0 : i32 + // CHECK: [[ARG0LOADPTR:%.*]] = spv.AccessChain [[ARG0PTR]]{{\[}}[[ZERO1]], [[GLOBALIDY]], [[GLOBALIDX]], [[ZERO2]]{{\]}} + %0 = iree.load_input(%arg0 : memref<12x42x1xi32>) : tensor<12x42x1xi32> + %1 = "xla_hlo.reshape"(%0) : (tensor<12x42x1xi32>) -> tensor<12x42xi32> + iree.store_output(%1 : tensor<12x42xi32>, %arg1 : memref<12x42xi32>) + iree.return + } +} + +// ----- + +module { + // CHECK:spv.module "Logical" "GLSL450" + // CHECK: spv.globalVariable [[GLOBALIDVAR:@.*]] built_in("GlobalInvocationId") : !spv.ptr, Input> + // CHECK: spv.globalVariable [[ARG0VAR:@.*]] bind(0, 0) + // CHECK: spv.globalVariable [[ARG1VAR:@.*]] bind(0, 1) + // CHECK: func [[FN:@simple_load_store_entry_dispatch_0]]() + func @simple_load_store_entry_dispatch_0(%arg0: memref<12x42x1x1xi32>, %arg1: memref<12x42xi32>) + attributes {iree.executable.export, iree.executable.workload = dense<[42, 12, 1]> : tensor<3xi32>, iree.ordinal = 0 : i32} { + // CHECK: [[GLOBALIDPTR:%.*]] = spv._address_of [[GLOBALIDVAR]] + // CHECK: [[GLOBALID:%.*]] = spv.Load "Input" [[GLOBALIDPTR]] + // CHECK: [[GLOBALIDX:%.*]] = spv.CompositeExtract [[GLOBALID]][0 : i32] + // CHECK: [[GLOBALIDY:%.*]] = spv.CompositeExtract [[GLOBALID]][1 : i32] + // CHECK: [[ARG0PTR:%.*]] = spv._address_of [[ARG0VAR]] + // CHECK: [[ZERO1:%.*]] = spv.constant 0 : i32 + // CHECK: [[ZERO2:%.*]] = spv.constant 0 : i32 + // CHECK: {{%.*}} = spv.AccessChain [[ARG0PTR]]{{\[}}[[ZERO1]], [[GLOBALIDY]], [[GLOBALIDX]], [[ZERO2]], [[ZERO2]]{{\]}} + %0 = iree.load_input(%arg0 : memref<12x42x1x1xi32>) : tensor<12x42x1x1xi32> + %1 = "xla_hlo.reshape"(%0) : (tensor<12x42x1x1xi32>) -> tensor<12x42xi32> + iree.store_output(%1 : tensor<12x42xi32>, %arg1 : memref<12x42xi32>) + iree.return + } +} + +// ----- + +module { + // CHECK:spv.module "Logical" "GLSL450" + // CHECK: spv.globalVariable [[GLOBALIDVAR:@.*]] built_in("GlobalInvocationId") : !spv.ptr, Input> + // CHECK: spv.globalVariable [[ARG0VAR:@.*]] bind(0, 0) + // CHECK: spv.globalVariable [[ARG1VAR:@.*]] bind(0, 1) + // CHECK: func [[FN:@simple_load_store_entry_dispatch_0]]() + func @simple_load_store_entry_dispatch_0(%arg0: memref<12x42xi32>, %arg1: memref<12x42x1x1xi32>) + attributes {iree.executable.export, iree.executable.workload = dense<[42, 12, 1]> : tensor<3xi32>, iree.ordinal = 0 : i32} { + // CHECK: [[GLOBALIDPTR:%.*]] = spv._address_of [[GLOBALIDVAR]] + // CHECK: [[GLOBALID:%.*]] = spv.Load "Input" [[GLOBALIDPTR]] + // CHECK: [[GLOBALIDX:%.*]] = spv.CompositeExtract [[GLOBALID]][0 : i32] + // CHECK: [[GLOBALIDY:%.*]] = spv.CompositeExtract [[GLOBALID]][1 : i32] + // CHECK: [[ARG0PTR:%.*]] = spv._address_of [[ARG0VAR]] + // CHECK: [[ZERO1:%.*]] = spv.constant 0 : i32 + // CHECK: {{%.*}} = spv.AccessChain [[ARG0PTR]]{{\[}}[[ZERO1]], [[GLOBALIDY]], [[GLOBALIDX]]{{\]}} + %0 = iree.load_input(%arg0 : memref<12x42xi32>) : tensor<12x42xi32> + %1 = "xla_hlo.reshape"(%0) : (tensor<12x42xi32>) -> tensor<12x42x1x1xi32> + // CHECK: [[ARG1PTR:%.*]] = spv._address_of [[ARG1VAR]] + // CHECK: [[ZERO2:%.*]] = spv.constant 0 : i32 + // CHECK: [[ZERO3:%.*]] = spv.constant 0 : i32 + // CHECK: {{%.*}} = spv.AccessChain [[ARG1PTR]]{{\[}}[[ZERO2]], [[GLOBALIDY]], [[GLOBALIDX]], [[ZERO3]], [[ZERO3]]{{\]}} + iree.store_output(%1 : tensor<12x42x1x1xi32>, %arg1 : memref<12x42x1x1xi32>) + iree.return + } +} + +// ----- + +module { + // CHECK:spv.module "Logical" "GLSL450" + // CHECK: spv.globalVariable [[GLOBALIDVAR:@.*]] built_in("GlobalInvocationId") : !spv.ptr, Input> + // CHECK: spv.globalVariable [[ARG0VAR:@.*]] bind(0, 0) + // CHECK: spv.globalVariable [[ARG1VAR:@.*]] bind(0, 1) + // CHECK: func [[FN:@simple_load_store_entry_dispatch_0]]() + func @simple_load_store_entry_dispatch_0(%arg0: memref<12x42xi32>, %arg1: memref<12x1x1x42xi32>) + attributes {iree.executable.export, iree.executable.workload = dense<[42, 12, 1]> : tensor<3xi32>, iree.ordinal = 0 : i32} { + %0 = iree.load_input(%arg0 : memref<12x42xi32>) : tensor<12x42xi32> + // CHECK: [[GLOBALIDPTR:%.*]] = spv._address_of [[GLOBALIDVAR]] + // CHECK: [[GLOBALID:%.*]] = spv.Load "Input" [[GLOBALIDPTR]] + // CHECK: [[GLOBALIDX:%.*]] = spv.CompositeExtract [[GLOBALID]][0 : i32] + // CHECK: [[GLOBALIDY:%.*]] = spv.CompositeExtract [[GLOBALID]][1 : i32] + // CHECK: [[ARG0PTR:%.*]] = spv._address_of [[ARG0VAR]] + // CHECK: [[ZERO1:%.*]] = spv.constant 0 : i32 + // CHECK: {{%.*}} = spv.AccessChain [[ARG0PTR]]{{\[}}[[ZERO1]], [[GLOBALIDY]], [[GLOBALIDX]]{{\]}} + %1 = "xla_hlo.reshape"(%0) : (tensor<12x42xi32>) -> tensor<12x1x1x42xi32> + // CHECK: [[ARG1PTR:%.*]] = spv._address_of [[ARG1VAR]] + // CHECK: [[ZERO2:%.*]] = spv.constant 0 : i32 + // CHECK: [[ZERO3:%.*]] = spv.constant 0 : i32 + // CHECK: {{%.*}} = spv.AccessChain [[ARG1PTR]]{{\[}}[[ZERO2]], [[GLOBALIDY]], [[ZERO3]], [[ZERO3]], [[GLOBALIDX]]{{\]}} + iree.store_output(%1 : tensor<12x1x1x42xi32>, %arg1 : memref<12x1x1x42xi32>) + iree.return + } +} + +// ----- + +module { + // CHECK:spv.module "Logical" "GLSL450" + // CHECK: spv.globalVariable [[GLOBALIDVAR:@.*]] built_in("GlobalInvocationId") : !spv.ptr, Input> + // CHECK: spv.globalVariable [[ARG0VAR:@.*]] bind(0, 0) + // CHECK: spv.globalVariable [[ARG1VAR:@.*]] bind(0, 1) + // CHECK: func [[FN:@simple_load_store_entry_dispatch_0]]() + func @simple_load_store_entry_dispatch_0(%arg0: memref<12x1x1x42xi32>, %arg1: memref<12x42xi32>) + attributes {iree.executable.export, iree.executable.workload = dense<[42, 12, 1]> : tensor<3xi32>, iree.ordinal = 0 : i32} { + // CHECK: [[GLOBALIDPTR:%.*]] = spv._address_of [[GLOBALIDVAR]] + // CHECK: [[GLOBALID:%.*]] = spv.Load "Input" [[GLOBALIDPTR]] + // CHECK: [[GLOBALIDX:%.*]] = spv.CompositeExtract [[GLOBALID]][0 : i32] + // CHECK: [[GLOBALIDY:%.*]] = spv.CompositeExtract [[GLOBALID]][1 : i32] + // CHECK: [[ARG0PTR:%.*]] = spv._address_of [[ARG0VAR]] + // CHECK: [[ZERO1:%.*]] = spv.constant 0 : i32 + // CHECK: [[ZERO2:%.*]] = spv.constant 0 : i32 + // CHECK: {{%.*}} = spv.AccessChain [[ARG0PTR]]{{\[}}[[ZERO1]], [[GLOBALIDY]], [[ZERO2]], [[ZERO2]], [[GLOBALIDX]]{{\]}} + %0 = iree.load_input(%arg0 : memref<12x1x1x42xi32>) : tensor<12x1x1x42xi32> + %1 = "xla_hlo.reshape"(%0) : (tensor<12x1x1x42xi32>) -> tensor<12x42xi32> + // CHECK: [[ARG1PTR:%.*]] = spv._address_of [[ARG1VAR]] + // CHECK: [[ZERO3:%.*]] = spv.constant 0 : i32 + // CHECK: {{%.*}} = spv.AccessChain [[ARG1PTR]]{{\[}}[[ZERO3]], [[GLOBALIDY]], [[GLOBALIDX]]{{\]}} + iree.store_output(%1 : tensor<12x42xi32>, %arg1 : memref<12x42xi32>) + iree.return + } +} + +// ----- + +module { + // CHECK:spv.module "Logical" "GLSL450" + // CHECK: spv.globalVariable [[GLOBALIDVAR:@.*]] built_in("GlobalInvocationId") : !spv.ptr, Input> + // CHECK: spv.globalVariable [[ARG0VAR:@.*]] bind(0, 0) + // CHECK: spv.globalVariable [[ARG1VAR:@.*]] bind(0, 1) + // CHECK: func [[FN:@simple_load_store_entry_dispatch_0]]() + func @simple_load_store_entry_dispatch_0(%arg0: memref<12x42xi32>, %arg1: memref<3x12x42xi32>) + attributes {iree.executable.export, iree.executable.workload = dense<[42, 12, 3]> : tensor<3xi32>, iree.ordinal = 0 : i32} { + // CHECK: [[GLOBALIDPTR:%.*]] = spv._address_of [[GLOBALIDVAR]] + // CHECK: [[GLOBALID:%.*]] = spv.Load "Input" [[GLOBALIDPTR]] + // CHECK: [[GLOBALIDX:%.*]] = spv.CompositeExtract [[GLOBALID]][0 : i32] + // CHECK: [[GLOBALIDY:%.*]] = spv.CompositeExtract [[GLOBALID]][1 : i32] + // CHECK: [[GLOBALIDZ:%.*]] = spv.CompositeExtract [[GLOBALID]][2 : i32] + // CHECK: [[ARG0PTR:%.*]] = spv._address_of [[ARG0VAR]] + // CHECK: [[ZERO1:%.*]] = spv.constant 0 : i32 + // CHECK: [[ARG0LOADPTR:%.*]] = spv.AccessChain [[ARG0PTR]]{{\[}}[[ZERO1]], [[GLOBALIDY]], [[GLOBALIDX]]{{\]}} + // CHECK: [[VAL:%.*]] = spv.Load "StorageBuffer" [[ARG0LOADPTR]] + %0 = iree.load_input(%arg0 : memref<12x42xi32>) : tensor<12x42xi32> + %1 = "xla_hlo.broadcast_in_dim"(%0) {broadcast_dimensions = dense<[1, 2]> : tensor<2xi64>} : (tensor<12x42xi32>) -> tensor<3x12x42xi32> + // CHECK: [[ARG1PTR:%.*]] = spv._address_of [[ARG1VAR]] + // CHECK: [[ZERO2:%.*]] = spv.constant 0 : i32 + // CHECK: [[ARG1STOREPTR:%.*]] = spv.AccessChain [[ARG1PTR]]{{\[}}[[ZERO2]], [[GLOBALIDZ]], [[GLOBALIDY]], [[GLOBALIDX]]{{\]}} + // CHECK: spv.Store "StorageBuffer" [[ARG1STOREPTR]], [[VAL]] + iree.store_output(%1 : tensor<3x12x42xi32>, %arg1 : memref<3x12x42xi32>) + iree.return + } +} + +// ----- + +module { + // CHECK:spv.module "Logical" "GLSL450" + // CHECK: spv.globalVariable [[GLOBALIDVAR:@.*]] built_in("GlobalInvocationId") : !spv.ptr, Input> + // CHECK: spv.globalVariable [[ARG0VAR:@.*]] bind(0, 0) : !spv.ptr, StorageBuffer> + // CHECK: spv.globalVariable [[ARG1VAR:@.*]] bind(0, 1) + // CHECK: func [[FN:@simple_load_store_entry_dispatch_0]]() + func @simple_load_store_entry_dispatch_0(%arg0: memref, %arg1: memref<3x12x42xi32>) + attributes {iree.executable.export, iree.executable.workload = dense<[42, 12, 3]> : tensor<3xi32>, iree.ordinal = 0 : i32} { + // CHECK: [[GLOBALIDPTR:%.*]] = spv._address_of [[GLOBALIDVAR]] + // CHECK: [[GLOBALID:%.*]] = spv.Load "Input" [[GLOBALIDPTR]] + // CHECK: [[GLOBALIDX:%.*]] = spv.CompositeExtract [[GLOBALID]][0 : i32] + // CHECK: [[GLOBALIDY:%.*]] = spv.CompositeExtract [[GLOBALID]][1 : i32] + // CHECK: [[GLOBALIDZ:%.*]] = spv.CompositeExtract [[GLOBALID]][2 : i32] + // CHECK: [[ARG0PTR:%.*]] = spv._address_of [[ARG0VAR]] + // CHECK: [[ZERO:%.*]] = spv.constant 0 : i32 + // CHECK: [[ARG0LOADPTR:%.*]] = spv.AccessChain [[ARG0PTR]]{{\[}}[[ZERO]]{{\]}} + // CHECK: [[VAL:%.*]] = spv.Load "StorageBuffer" [[ARG0LOADPTR]] + %0 = iree.load_input(%arg0 : memref) : tensor + %1 = "xla_hlo.broadcast_in_dim"(%0) : (tensor) -> tensor<3x12x42xi32> + // CHECK: [[ARG1PTR:%.*]] = spv._address_of [[ARG1VAR]] + // CHECK: [[ZERO1:%.*]] = spv.constant 0 : i32 + // CHECK: [[ARG1STOREPTR:%.*]] = spv.AccessChain [[ARG1PTR]]{{\[}}[[ZERO1]], [[GLOBALIDZ]], [[GLOBALIDY]], [[GLOBALIDX]]{{\]}} + // CHECK: spv.Store "StorageBuffer" [[ARG1STOREPTR]], [[VAL]] + iree.store_output(%1 : tensor<3x12x42xi32>, %arg1 : memref<3x12x42xi32>) + iree.return + } +} + +// ----- + +module { + func @simple_load_store_entry_dispatch_0(%arg0: memref<12x42xf32>, %arg1: memref<12x42xf32>, %arg2: memref<12x42xf32>) + attributes {iree.executable.export, iree.executable.workload = dense<[42, 12, 1]> : tensor<3xi32>, iree.ordinal = 0 : i32} { + %0 = iree.load_input(%arg0 : memref<12x42xf32>) : tensor<12x42xf32> + %1 = iree.load_input(%arg1 : memref<12x42xf32>) : tensor<12x42xf32> + //CHECK: [[COMPARE:%.*]] = spv.FOrdGreaterThanEqual {{%.*}}, {{%.*}} + %2 = cmpf "oge", %0, %1 : tensor<12x42xf32> + //CHECK: {{%.*}} = spv.Select [[COMPARE]], {{%.*}}, {{%.*}} + %3 = "xla_hlo.select"(%2, %0, %1) : (tensor<12x42xi1>, tensor<12x42xf32>, tensor<12x42xf32>) -> tensor<12x42xf32> + iree.store_output(%3 : tensor<12x42xf32>, %arg2 : memref<12x42xf32>) + iree.return + } +} + +// ----- + +module { + func @simple_load_store_entry_dispatch_0(%arg0: memref<12x42xf32>, %arg1: memref<12x42xf32>, %arg2: memref<12x42xf32>) + attributes {iree.executable.export, iree.executable.workload = dense<[42, 12, 1]> : tensor<3xi32>, iree.ordinal = 0 : i32} { + %0 = iree.load_input(%arg0 : memref<12x42xf32>) : tensor<12x42xf32> + %1 = iree.load_input(%arg1 : memref<12x42xf32>) : tensor<12x42xf32> + //CHECK: [[COMPARE:%.*]] = spv.FOrdEqual {{%.*}}, {{%.*}} + %2 = cmpf "oeq", %0, %1 : tensor<12x42xf32> + %3 = "xla_hlo.select"(%2, %0, %1) : (tensor<12x42xi1>, tensor<12x42xf32>, tensor<12x42xf32>) -> tensor<12x42xf32> + iree.store_output(%3 : tensor<12x42xf32>, %arg2 : memref<12x42xf32>) + iree.return + } +} + +// ----- + +module { + func @simple_load_store_entry_dispatch_0(%arg0: memref<12x42xf32>, %arg1: memref<12x42xf32>, %arg2: memref<12x42xf32>) + attributes {iree.executable.export, iree.executable.workload = dense<[42, 12, 1]> : tensor<3xi32>, iree.ordinal = 0 : i32} { + %0 = iree.load_input(%arg0 : memref<12x42xf32>) : tensor<12x42xf32> + %1 = iree.load_input(%arg1 : memref<12x42xf32>) : tensor<12x42xf32> + //CHECK: [[COMPARE:%.*]] = spv.FOrdGreaterThan {{%.*}}, {{%.*}} + %2 = cmpf "ogt", %0, %1 : tensor<12x42xf32> + %3 = "xla_hlo.select"(%2, %0, %1) : (tensor<12x42xi1>, tensor<12x42xf32>, tensor<12x42xf32>) -> tensor<12x42xf32> + iree.store_output(%3 : tensor<12x42xf32>, %arg2 : memref<12x42xf32>) + iree.return + } +} + +// ----- + +module { + func @simple_load_store_entry_dispatch_0(%arg0: memref<12x42xf32>, %arg1: memref<12x42xf32>, %arg2: memref<12x42xf32>) + attributes {iree.executable.export, iree.executable.workload = dense<[42, 12, 1]> : tensor<3xi32>, iree.ordinal = 0 : i32} { + %0 = iree.load_input(%arg0 : memref<12x42xf32>) : tensor<12x42xf32> + %1 = iree.load_input(%arg1 : memref<12x42xf32>) : tensor<12x42xf32> + //CHECK: [[COMPARE:%.*]] = spv.FOrdLessThan {{%.*}}, {{%.*}} + %2 = cmpf "olt", %0, %1 : tensor<12x42xf32> + %3 = "xla_hlo.select"(%2, %0, %1) : (tensor<12x42xi1>, tensor<12x42xf32>, tensor<12x42xf32>) -> tensor<12x42xf32> + iree.store_output(%3 : tensor<12x42xf32>, %arg2 : memref<12x42xf32>) + iree.return + } +} + +// ----- + +module { + func @simple_load_store_entry_dispatch_0(%arg0: memref<12x42xf32>, %arg1: memref<12x42xf32>, %arg2: memref<12x42xf32>) + attributes {iree.executable.export, iree.executable.workload = dense<[42, 12, 1]> : tensor<3xi32>, iree.ordinal = 0 : i32} { + %0 = iree.load_input(%arg0 : memref<12x42xf32>) : tensor<12x42xf32> + %1 = iree.load_input(%arg1 : memref<12x42xf32>) : tensor<12x42xf32> + //CHECK: [[COMPARE:%.*]] = spv.FOrdLessThanEqual {{%.*}}, {{%.*}} + %2 = cmpf "ole", %0, %1 : tensor<12x42xf32> + %3 = "xla_hlo.select"(%2, %0, %1) : (tensor<12x42xi1>, tensor<12x42xf32>, tensor<12x42xf32>) -> tensor<12x42xf32> + iree.store_output(%3 : tensor<12x42xf32>, %arg2 : memref<12x42xf32>) + iree.return + } +} + +// ----- + +module { + func @simple_load_store_entry_dispatch_0(%arg0: memref<12x42xf32>, %arg1: memref<12x42xf32>, %arg2: memref<12x42xf32>) + attributes {iree.executable.export, iree.executable.workload = dense<[42, 12, 1]> : tensor<3xi32>, iree.ordinal = 0 : i32} { + %0 = iree.load_input(%arg0 : memref<12x42xf32>) : tensor<12x42xf32> + %1 = iree.load_input(%arg1 : memref<12x42xf32>) : tensor<12x42xf32> + //CHECK: [[COMPARE:%.*]] = spv.FOrdNotEqual {{%.*}}, {{%.*}} + %2 = cmpf "one", %0, %1 : tensor<12x42xf32> + %3 = "xla_hlo.select"(%2, %0, %1) : (tensor<12x42xi1>, tensor<12x42xf32>, tensor<12x42xf32>) -> tensor<12x42xf32> + iree.store_output(%3 : tensor<12x42xf32>, %arg2 : memref<12x42xf32>) + iree.return + } +} + +// ----- + +module { + func @simple_load_store_entry_dispatch_0(%arg0: memref<12x42xf32>, %arg1: memref<12x42xf32>, %arg2: memref<12x42xf32>) + attributes {iree.executable.export, iree.executable.workload = dense<[42, 12, 1]> : tensor<3xi32>, iree.ordinal = 0 : i32} { + %0 = iree.load_input(%arg0 : memref<12x42xf32>) : tensor<12x42xf32> + %1 = iree.load_input(%arg1 : memref<12x42xf32>) : tensor<12x42xf32> + //CHECK: [[COMPARE:%.*]] = spv.FUnordEqual {{%.*}}, {{%.*}} + %2 = cmpf "ueq", %0, %1 : tensor<12x42xf32> + %3 = "xla_hlo.select"(%2, %0, %1) : (tensor<12x42xi1>, tensor<12x42xf32>, tensor<12x42xf32>) -> tensor<12x42xf32> + iree.store_output(%3 : tensor<12x42xf32>, %arg2 : memref<12x42xf32>) + iree.return + } +} + +// ----- + +module { + func @simple_load_store_entry_dispatch_0(%arg0: memref<12x42xf32>, %arg1: memref<12x42xf32>, %arg2: memref<12x42xf32>) + attributes {iree.executable.export, iree.executable.workload = dense<[42, 12, 1]> : tensor<3xi32>, iree.ordinal = 0 : i32} { + %0 = iree.load_input(%arg0 : memref<12x42xf32>) : tensor<12x42xf32> + %1 = iree.load_input(%arg1 : memref<12x42xf32>) : tensor<12x42xf32> + //CHECK: [[COMPARE:%.*]] = spv.FUnordGreaterThanEqual {{%.*}}, {{%.*}} + %2 = cmpf "uge", %0, %1 : tensor<12x42xf32> + %3 = "xla_hlo.select"(%2, %0, %1) : (tensor<12x42xi1>, tensor<12x42xf32>, tensor<12x42xf32>) -> tensor<12x42xf32> + iree.store_output(%3 : tensor<12x42xf32>, %arg2 : memref<12x42xf32>) + iree.return + } +} + +// ----- + +module { + func @simple_load_store_entry_dispatch_0(%arg0: memref<12x42xf32>, %arg1: memref<12x42xf32>, %arg2: memref<12x42xf32>) + attributes {iree.executable.export, iree.executable.workload = dense<[42, 12, 1]> : tensor<3xi32>, iree.ordinal = 0 : i32} { + %0 = iree.load_input(%arg0 : memref<12x42xf32>) : tensor<12x42xf32> + %1 = iree.load_input(%arg1 : memref<12x42xf32>) : tensor<12x42xf32> + //CHECK: [[COMPARE:%.*]] = spv.FUnordGreaterThan {{%.*}}, {{%.*}} + %2 = cmpf "ugt", %0, %1 : tensor<12x42xf32> + %3 = "xla_hlo.select"(%2, %0, %1) : (tensor<12x42xi1>, tensor<12x42xf32>, tensor<12x42xf32>) -> tensor<12x42xf32> + iree.store_output(%3 : tensor<12x42xf32>, %arg2 : memref<12x42xf32>) + iree.return + } +} + +// ----- + +module { + func @simple_load_store_entry_dispatch_0(%arg0: memref<12x42xf32>, %arg1: memref<12x42xf32>, %arg2: memref<12x42xf32>) + attributes {iree.executable.export, iree.executable.workload = dense<[42, 12, 1]> : tensor<3xi32>, iree.ordinal = 0 : i32} { + %0 = iree.load_input(%arg0 : memref<12x42xf32>) : tensor<12x42xf32> + %1 = iree.load_input(%arg1 : memref<12x42xf32>) : tensor<12x42xf32> + //CHECK: [[COMPARE:%.*]] = spv.FUnordLessThan {{%.*}}, {{%.*}} + %2 = cmpf "ult", %0, %1 : tensor<12x42xf32> + %3 = "xla_hlo.select"(%2, %0, %1) : (tensor<12x42xi1>, tensor<12x42xf32>, tensor<12x42xf32>) -> tensor<12x42xf32> + iree.store_output(%3 : tensor<12x42xf32>, %arg2 : memref<12x42xf32>) + iree.return + } +} + +// ----- + +module { + func @simple_load_store_entry_dispatch_0(%arg0: memref<12x42xf32>, %arg1: memref<12x42xf32>, %arg2: memref<12x42xf32>) + attributes {iree.executable.export, iree.executable.workload = dense<[42, 12, 1]> : tensor<3xi32>, iree.ordinal = 0 : i32} { + %0 = iree.load_input(%arg0 : memref<12x42xf32>) : tensor<12x42xf32> + %1 = iree.load_input(%arg1 : memref<12x42xf32>) : tensor<12x42xf32> + //CHECK: [[COMPARE:%.*]] = spv.FUnordLessThanEqual {{%.*}}, {{%.*}} + %2 = cmpf "ule", %0, %1 : tensor<12x42xf32> + %3 = "xla_hlo.select"(%2, %0, %1) : (tensor<12x42xi1>, tensor<12x42xf32>, tensor<12x42xf32>) -> tensor<12x42xf32> + iree.store_output(%3 : tensor<12x42xf32>, %arg2 : memref<12x42xf32>) + iree.return + } +} + +// ----- + +module { + func @simple_load_store_entry_dispatch_0(%arg0: memref<12x42xf32>, %arg1: memref<12x42xf32>, %arg2: memref<12x42xf32>) + attributes {iree.executable.export, iree.executable.workload = dense<[42, 12, 1]> : tensor<3xi32>, iree.ordinal = 0 : i32} { + %0 = iree.load_input(%arg0 : memref<12x42xf32>) : tensor<12x42xf32> + %1 = iree.load_input(%arg1 : memref<12x42xf32>) : tensor<12x42xf32> + //CHECK: [[COMPARE:%.*]] = spv.FUnordNotEqual {{%.*}}, {{%.*}} + %2 = cmpf "une", %0, %1 : tensor<12x42xf32> + %3 = "xla_hlo.select"(%2, %0, %1) : (tensor<12x42xi1>, tensor<12x42xf32>, tensor<12x42xf32>) -> tensor<12x42xf32> + iree.store_output(%3 : tensor<12x42xf32>, %arg2 : memref<12x42xf32>) + iree.return + } +} + +// ----- + +module { + func @const_float_splat(%arg0: memref<12x42xf32>) + attributes {iree.executable.export, iree.executable.workload = dense<[42, 12, 1]> : tensor<3xi32>, iree.ordinal = 0 : i32} { + // CHECK: spv.constant 1.000000e+00 : f32 + %0 = constant dense<1.0> : tensor<12xf32> + %1 = "xla_hlo.broadcast_in_dim"(%0) {broadcast_dimensions = dense<[0]> : tensor<1xi64>} : (tensor<12xf32>) -> tensor<12x42xf32> + iree.store_output(%1 : tensor<12x42xf32>, %arg0 : memref<12x42xf32>) + iree.return + } +} + +// ----- + +module { + func @const_int_splat(%arg0: memref<12x42xi64>) + attributes {iree.executable.export, iree.executable.workload = dense<[42, 12, 1]> : tensor<3xi32>, iree.ordinal = 0 : i32} { + // CHECK: spv.constant 42 : i64 + %0 = constant dense<42> : tensor<12xi64> + %1 = "xla_hlo.broadcast_in_dim"(%0) {broadcast_dimensions = dense<[0]> : tensor<1xi64>} : (tensor<12xi64>) -> tensor<12x42xi64> + iree.store_output(%1 : tensor<12x42xi64>, %arg0 : memref<12x42xi64>) + iree.return + } +} + +// ----- + +module { + func @const_int_splat(%arg0: memref<2x12x42xi64>) + attributes {iree.executable.export, iree.executable.workload = dense<[42, 12, 2]> : tensor<3xi32>, iree.ordinal = 0 : i32} { + // expected-error @+1{{unhandled constant lowering unless value is a splat dense element attribute}} + %0 = constant dense<[42, 21]> : tensor<2xi64> + %1 = "xla_hlo.broadcast_in_dim"(%0) {broadcast_dimensions = dense<[0]> : tensor<1xi64>} : (tensor<2xi64>) -> tensor<2x12x42xi64> + iree.store_output(%1 : tensor<2x12x42xi64>, %arg0 : memref<2x12x42xi64>) + iree.return + } +} + +// ----- + +module { + func @max(%arg0: memref<12x42xf32>, %arg1: memref<12x42xf32>, %arg2 : memref<12x42xf32>) + attributes {iree.executable.export, iree.executable.workload = dense<[42, 12, 1]> : tensor<3xi32>, iree.ordinal = 0 : i32} { + %0 = iree.load_input(%arg0 : memref<12x42xf32>) : tensor<12x42xf32> + %1 = iree.load_input(%arg1 : memref<12x42xf32>) : tensor<12x42xf32> + //CHECK: [[COMPARE:%.*]] = spv.FOrdGreaterThan [[VAL1:%.*]], [[VAL2:%.*]] : f32 + //CHECK: {{%.*}} = spv.Select [[COMPARE]], [[VAL1]], [[VAL2]] : i1, f32 + %2 = xla_hlo.max %0, %1 : tensor<12x42xf32> + iree.store_output(%2 : tensor<12x42xf32>, %arg2 : memref<12x42xf32>) + iree.return + } +} + +// ----- + +module { + func @max(%arg0: memref<12x42xi32>, %arg1: memref<12x42xi32>, %arg2 : memref<12x42xi32>) + attributes {iree.executable.export, iree.executable.workload = dense<[42, 12, 1]> : tensor<3xi32>, iree.ordinal = 0 : i32} { + %0 = iree.load_input(%arg0 : memref<12x42xi32>) : tensor<12x42xi32> + %1 = iree.load_input(%arg1 : memref<12x42xi32>) : tensor<12x42xi32> + //CHECK: [[COMPARE:%.*]] = spv.SGreaterThan [[VAL1:%.*]], [[VAL2:%.*]] : i32 + //CHECK: {{%.*}} = spv.Select [[COMPARE]], [[VAL1]], [[VAL2]] : i1, i32 + %2 = xla_hlo.max %0, %1 : tensor<12x42xi32> + iree.store_output(%2 : tensor<12x42xi32>, %arg2 : memref<12x42xi32>) + iree.return + } +} + +// ----- + +module { + // CHECK: spv.globalVariable [[GLOBALIDVAR:@.*]] built_in("GlobalInvocationId") : !spv.ptr, Input> + // CHECK: spv.globalVariable [[ARG0VAR:@.*]] bind(0, 0) : !spv.ptr [84]> [0]>, StorageBuffer> + func @simple_load_store_entry_dispatch_0(%arg0: memref<24x21xi32>, %arg1: memref<12x42xi32>) + attributes {iree.executable.export, iree.executable.workload = dense<[42, 12, 1]> : tensor<3xi32>, iree.ordinal = 0 : i32} { + // CHECK: [[GLOBALIDPTR:%.*]] = spv._address_of [[GLOBALIDVAR]] + // CHECK: [[GLOBALID:%.*]] = spv.Load "Input" [[GLOBALIDPTR]] + // CHECK: [[GLOBALIDX:%.*]] = spv.CompositeExtract [[GLOBALID]][0 : i32] + // CHECK: [[GLOBALIDY:%.*]] = spv.CompositeExtract [[GLOBALID]][1 : i32] + // CHECK-DAG: [[ARG0PTR:%.*]] = spv._address_of [[ARG0VAR]] + // CHECK-DAG: [[ZERO1:%.*]] = spv.constant 0 : i32 + // CHECK-DAG: [[STRIDE0:%.*]] = spv.constant 42 : i32 + // CHECK-DAG: [[L1:%.*]] = spv.IMul [[GLOBALIDY]], [[STRIDE0]] : i32 + // CHECK-DAG: [[L2:%.*]] = spv.IAdd [[L1]], [[GLOBALIDX]] : i32 + // CHECK-DAG: [[STRIDE2:%.*]] = spv.constant 21 : i32 + // CHECK-DAG: [[INDEX0:%.*]] = spv.UDiv [[L2]], [[STRIDE2]] : i32 + // CHECK-DAG: [[INDEX1:%.*]] = spv.UMod [[L2]], [[STRIDE2]] : i32 + // CHECK: [[ARG0LOADPTR:%.*]] = spv.AccessChain [[ARG0PTR]]{{\[}}[[ZERO1]], [[INDEX0]], [[INDEX1]]{{\]}} + %0 = iree.load_input(%arg0 : memref<24x21xi32>) : tensor<24x21xi32> + %1 = "xla_hlo.reshape"(%0) : (tensor<24x21xi32>) -> tensor<12x42xi32> + iree.store_output(%1 : tensor<12x42xi32>, %arg1 : memref<12x42xi32>) + iree.return + } +} + +// ----- + +module { + // CHECK: spv.globalVariable [[GLOBALIDVAR:@.*]] built_in("GlobalInvocationId") : !spv.ptr, Input> + // CHECK: spv.globalVariable [[ARG0VAR:@.*]] bind(0, 0) : !spv.ptr [84]> [504]> [0]>, StorageBuffer> + func @simple_load_store_entry_dispatch_0(%arg0: memref<4x6x21xi32>, %arg1: memref<12x42xi32>) + attributes {iree.executable.export, iree.executable.workload = dense<[42, 12, 1]> : tensor<3xi32>, iree.ordinal = 0 : i32} { + // CHECK-DAG: [[STRIDE1:%.*]] = spv.constant 126 : i32 + // CHECK-DAG: [[I0:%.*]] = spv.UDiv [[L1:%.*]], [[STRIDE2]] : i32 + // CHECK-DAG: [[I1:%.*]] = spv.UMod [[L1]], [[STRIDE2]] : i32 + // CHECK-DAG: [[STRIDE2:%.*]] = spv.constant 21 : i32 + // CHECK-DAG: [[I2:%.*]] = spv.UDiv [[I1]], [[STRIDE2]] : i32 + // CHECK-DAG: [[I3:%.*]] = spv.UMod [[I1]], [[STRIDE2]] : i32 + // CHECK-DAG: [[ARG0PTR:%.*]] = spv._address_of [[ARG0VAR]] + // CHECK-DAG: [[ARG0LOADPTR:%.*]] = spv.AccessChain [[ARG0PTR]]{{\[}}[[ZERO1]], [[I0]], [[I2]], [[I3]]{{\]}} + %0 = iree.load_input(%arg0 : memref<4x6x21xi32>) : tensor<4x6x21xi32> + %1 = "xla_hlo.reshape"(%0) : (tensor<4x6x21xi32>) -> tensor<12x42xi32> + iree.store_output(%1 : tensor<12x42xi32>, %arg1 : memref<12x42xi32>) + iree.return + } +} + +// ----- + +module { + // CHECK: spv.globalVariable [[GLOBALIDVAR:@.*]] built_in("GlobalInvocationId") : !spv.ptr, Input> + // CHECK: spv.globalVariable [[ARG0VAR:@.*]] bind(0, 0) : !spv.ptr [84]> [0]>, StorageBuffer> + // CHECK: spv.globalVariable [[ARG1VAR:@.*]] bind(0, 1) : !spv.ptr [28]> [168]> [0]>, StorageBuffer> + func @simple_load_store_entry_dispatch_0(%arg0: memref<24x21xi32>, %arg1: memref<12x6x7xi32>) + attributes {iree.executable.export, iree.executable.workload = dense<[42, 12, 1]> : tensor<3xi32>, iree.ordinal = 0 : i32} { + // CHECK: [[GLOBALIDPTR:%.*]] = spv._address_of [[GLOBALIDVAR]] + // CHECK: [[GLOBALID:%.*]] = spv.Load "Input" [[GLOBALIDPTR]] + // CHECK: [[GLOBALIDX:%.*]] = spv.CompositeExtract [[GLOBALID]][0 : i32] + // CHECK: [[GLOBALIDY:%.*]] = spv.CompositeExtract [[GLOBALID]][1 : i32] + // CHECK: [[ARG0PTR:%.*]] = spv._address_of [[ARG0VAR]] + // CHECK-DAG: [[ZERO:%.*]] = spv.constant 0 : i32 + // CHECK-DAG: [[FORTYTWO:%.*]] = spv.constant 42 : i32 + // CHECK-DAG: [[I1:%.*]] = spv.IMul [[GLOBALIDY]], [[FORTYTWO]] : i32 + // CHECK-DAG: [[I2:%.*]] = spv.IAdd [[I1]], [[GLOBALIDX]] : i32 + // CHECK-DAG: [[I3:%.*]] = spv.UDiv [[I2]], [[FORTYTWO]] : i32 + // CHECK-DAG: [[I4:%.*]] = spv.IMul [[I3]], [[FORTYTWO]] : i32 + // CHECK-DAG: [[I5:%.*]] = spv.UMod [[I2]], [[FORTYTWO]] : i32 + // CHECK: [[SEVEN:%.*]] = spv.constant 7 : i32 + // CHECK-DAG: [[I6:%.*]] = spv.UDiv [[I5]], [[SEVEN]] : i32 + // CHECK-DAG: [[I7:%.*]] = spv.IMul [[I6]], [[SEVEN]] : i32 + // CHECK-DAG: [[I8:%.*]] = spv.IAdd [[I4]], [[I7]] : i32 + // CHECK-DAG: [[I9:%.*]] = spv.UMod [[I5]], [[SEVEN]] : i32 + // CHECK-DAG: [[I10:%.*]] = spv.IAdd [[I8]], [[I9]] : i32 + // CHECK: [[TWENTYONE:%.*]] = spv.constant 21 : i32 + // CHECK-DAG: [[I11:%.*]] = spv.UDiv [[I10]], [[TWENTYONE]] : i32 + // CHECK-DAG: [[I12:%.*]] = spv.UMod [[I10]], [[TWENTYONE]] : i32 + // CHECK: {{%.*}} = spv.AccessChain [[ARG0PTR]]{{\[}}[[ZERO]], [[I11]], [[I12]] + // CHECK-DAG: [[ARG1PTR:%.*]] = spv._address_of [[ARG1VAR]] + // CHECK-DAG: [[ZERO2:%.*]] = spv.constant 0 : i32 + // CHECK: {{%.*}} = spv.AccessChain [[ARG1PTR]]{{\[}}[[ZERO2]], [[I3]], [[I6]], [[I9]]{{\]}} + %0 = iree.load_input(%arg0 : memref<24x21xi32>) : tensor<24x21xi32> + %1 = "xla_hlo.reshape"(%0) : (tensor<24x21xi32>) -> tensor<12x6x7xi32> + iree.store_output(%1 : tensor<12x6x7xi32>, %arg1 : memref<12x6x7xi32>) + iree.return + } +} + +// ----- + +module { + func @exp(%arg0: memref<12x42xf32>, %arg2 : memref<12x42xf32>) + attributes {iree.executable.export, iree.executable.workload = dense<[42, 12, 1]> : tensor<3xi32>, iree.ordinal = 0 : i32} { + %0 = iree.load_input(%arg0 : memref<12x42xf32>) : tensor<12x42xf32> + //CHECK: {{%.*}} = spv.GLSL.Exp {{%.*}} : f32 + %2 = "xla_hlo.exp"(%0) : (tensor<12x42xf32>) -> tensor<12x42xf32> + iree.store_output(%2 : tensor<12x42xf32>, %arg2 : memref<12x42xf32>) + iree.return + } +} diff --git a/compiler/Translation/SequencerModuleTranslation.cpp b/compiler/Translation/SequencerModuleTranslation.cpp new file mode 100644 index 000000000000..a7455d7bd50a --- /dev/null +++ b/compiler/Translation/SequencerModuleTranslation.cpp @@ -0,0 +1,519 @@ +// Copyright 2019 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "third_party/mlir_edge/iree/compiler/Translation/SequencerModuleTranslation.h" + +#include +#include +#include +#include +#include + +#include "third_party/flatbuffers/include/flatbuffers/flatbuffers.h" +#include "third_party/flatbuffers/include/flatbuffers/minireflect.h" +#include "third_party/llvm/llvm/include/llvm/ADT/STLExtras.h" +#include "third_party/llvm/llvm/include/llvm/ADT/Sequence.h" +#include "third_party/llvm/llvm/include/llvm/ADT/StringExtras.h" +#include "third_party/llvm/llvm/include/llvm/ADT/StringRef.h" +#include "third_party/llvm/llvm/include/llvm/ADT/StringSet.h" +#include "third_party/llvm/llvm/include/llvm/Support/Debug.h" +#include "third_party/llvm/llvm/include/llvm/Support/ToolOutputFile.h" +#include "third_party/llvm/llvm/projects/google_mlir/include/mlir/IR/Attributes.h" +#include "third_party/llvm/llvm/projects/google_mlir/include/mlir/IR/Module.h" +#include "third_party/llvm/llvm/projects/google_mlir/include/mlir/Pass/Pass.h" +#include "third_party/llvm/llvm/projects/google_mlir/include/mlir/Pass/PassManager.h" +#include "third_party/llvm/llvm/projects/google_mlir/include/mlir/Transforms/Passes.h" +#include "third_party/llvm/llvm/projects/google_mlir/include/mlir/Translation.h" +#include "third_party/mlir_edge/iree/base/status.h" +#include "third_party/mlir_edge/iree/compiler/IR/ConfigOps.h" +#include "third_party/mlir_edge/iree/compiler/IR/Sequencer/OpWriters.h" +#include "third_party/mlir_edge/iree/compiler/IR/StructureOps.h" +#include "third_party/mlir_edge/iree/compiler/IR/Types.h" +#include "third_party/mlir_edge/iree/compiler/Serialization/VMFunctionBuilder.h" +#include "third_party/mlir_edge/iree/compiler/Serialization/VMFunctionTableBuilder.h" +#include "third_party/mlir_edge/iree/compiler/Serialization/VMModuleBuilder.h" +#include "third_party/mlir_edge/iree/compiler/Transforms/Passes.h" +#include "third_party/mlir_edge/iree/compiler/Transforms/Sequencer/Passes.h" +#include "third_party/mlir_edge/iree/compiler/Utils/Macros.h" +#include "third_party/mlir_edge/iree/compiler/Utils/OpUtils.h" +#include "third_party/mlir_edge/iree/compiler/Utils/TranslationUtils.h" +#include "third_party/mlir_edge/iree/hal/executable_format.h" +#include "third_party/mlir_edge/iree/schemas/executable_def_generated.h" +#include "third_party/mlir_edge/iree/schemas/executable_table_def_generated.h" +#include "third_party/mlir_edge/iree/schemas/module_def_generated.h" +#include "third_party/tensorflow/compiler/mlir/xla/transforms/passes.h" + +namespace mlir { +namespace iree_compiler { + +namespace { + +// Builds a pass pipeline that optimizes and legalizes the module to the form +// expected by partitioning. +void buildLegalizeInputPassPipeline(PassManager *passManager) { + // Standard passes that shake out a lot of garbage. + // Some may have been run prior to translation but this ensures we are always + // in a known state. + passManager->addPass(createCanonicalizerPass()); + passManager->addPass(createLoopFusionPass()); + passManager->addPass(createLoopInvariantCodeMotionPass()); + passManager->addPass(createMemRefDataFlowOptPass()); + passManager->addPass(createCanonicalizerPass()); + passManager->addPass(createSimplifyAffineStructuresPass()); + passManager->addPass(createCSEPass()); + passManager->addPass(createCanonicalizerPass()); + + // Get out of TF ASAP and into a sane control flow representation. + // TODO(benvanik): do this externally prior to getting the input? + passManager->addPass(xla_hlo::createLegalizeTFPass()); + passManager->addPass(xla_hlo::createLegalizeToStdPass()); + passManager->addPass(xla_hlo::createLegalizeControlFlowPass()); + + // Expand uses of tuples into independent args/results. + passManager->addPass(createConvertFromTupleCallingConventionPass()); + passManager->addPass(createLegalizeTupleElementAccessPass()); +} + +// Builds a pass pipeline that partitions the module into sequencer functions +// and executables ready to be translated. +void buildPartitioningPassPipeline(PassManager *passManager) { + // Find reduction ops and create iree.reduction_regions. We do this prior to + // performing dispatch region identification so that we can build as big of + // fused reduction regions as possible. The remaining ops will be put into + // dispatch regions. + passManager->addPass(createIdentifyReductionRegionsPass()); + passManager->addPass(createCSEPass()); + + // Create all of the dispatch regions, CSE their workloads, and fold. + passManager->addPass(createIdentifyDispatchRegionsPass()); + passManager->addPass(createCSEPass()); + passManager->addPass(createFoldCompatibleDispatchRegionsPass()); + + // Note that as we are rematerializing things here it's critical we do not run + // the canonicalizer/CSE between now and when we outline - otherwise it'll + // undo all of our work! + passManager->addPass(createRematerializeDispatchConstantsPass()); + + // Outline the dispatch regions into their own functions. This separates the + // sequencer functions performing dispatches from the dispatchees. + passManager->addPass(createOutlineDispatchRegionsPass()); + passManager->addPass(createOutlineReductionRegionsPass()); + + // Cleanup identity sequencer tensor-to-memref ops that clutter up the IR. + // TODO(benvanik): implement as folder/canonicalizers instead. + passManager->addPass(createLegalizeTensorMemRefPass()); + + // Drop all functions that are no longer reachable. + // This is important as many of the functions remaining are probably + // dispatchable and unused now that we've outlined them executables. + passManager->addPass(createDropUnreachableModuleFunctionsPass()); + + // Drop all unused executables. + // Note that we need to have dropped unreachable functions first otherwise + // references could keep executables that are unreachable from exported + // functions alive. + passManager->addPass(createDropUnusedExecutablesPass()); +} + +// Builds a pass pipeline that converts sequencer functions to the iree_seq.hl +// dialect. +void buildSequencerConversionPassPipeline(PassManager *passManager) { + // Convert to the memref calling convention and optimize away as many + // loads and stores as we can prior to progressing. + passManager->addPass(createConvertToMemRefCallingConventionPass()); + + // Convert ops that are supported by the sequencer directly to the sequencer + // dialect. The ops that remain should be only those that can be moved into + // dispatch regions. + passManager->addPass(createLowerXLAToSequencerDialectPass()); + passManager->addPass(createLowerStdToSequencerDialectPass()); + + // Cleanup identity sequencer tensor-to-memref ops and other memory accesses + // that clutter up the IR. + passManager->addPass(createLegalizeTensorMemRefPass()); + passManager->addPass(createCanonicalizerPass()); + passManager->addPass(createMemRefDataFlowOptPass()); + passManager->addPass(createLoadStoreDataFlowOptPass()); + + // Convert any uses of index to int32_t (as we explicitly don't want to + // support dynamic index width). + // This also looks for other weird types (i1, etc). + passManager->addPass(createLegalizeTypeStoragePass()); + + // Eliminate ops we don't care about based on a lack of side-effects. + // IREE does not guarantee exception/error behavior of dead ops. + passManager->addPass(createAggressiveOpEliminationPass()); + + // Perform any last-minute optimizations to trim down the IR. + passManager->addPass(createCanonicalizerPass()); + passManager->addPass(createMemRefDataFlowOptPass()); + passManager->addPass(createCSEPass()); +} + +// Builds a pass pipeline that lowers the iree_seq.hl dialect to the iree_seq.ll +// dialect and prepares for serialization. +void buildSequencerLoweringPassPipeline(PassManager *passManager) { + // Lower iree_hl_seq -> iree_ll_seq. + passManager->addPass(createLowerSequencerDialectPass()); + passManager->addPass(createCanonicalizerPass()); + passManager->addPass(createMemRefDataFlowOptPass()); + passManager->addPass(createAggressiveOpEliminationPass()); + + // Assign ordinals used by the bytecode to reference executables and + // functions. + passManager->addPass(createAssignFunctionOrdinalsPass()); + passManager->addPass(createAssignExecutableOrdinalsPass()); + + // Plumb workload information down into executable entry points. This allows + // the backends to calculate their workgroup sizes, indexing, etc. + passManager->addPass(createAssignExecutableWorkloadAttrsPass()); +} + +// Inserts one or more iree.executable_target_config ops based on the +// translation options. +void insertTargetConfigOps(const ModuleTranslationOptions &options, + OpBuilder *builder) { + llvm::StringSet<> targetBackends; + if (options.target_backends.empty()) { + // Add all backends when none are explicitly provided. + targetBackends.insert(getExecutableTranslationRegistry().keys().begin(), + getExecutableTranslationRegistry().keys().end()); + } else { + for (auto &targetBackend : options.target_backends) { + for (auto &matchedBackend : + matchExecutableTranslationBackendNames(targetBackend)) { + targetBackends.insert(matchedBackend); + } + } + } + for (auto &targetBackend : targetBackends) { + builder->create(builder->getUnknownLoc(), + targetBackend.getKey()); + } +} + +class SequencerTranslator { + public: + explicit SequencerTranslator(ModuleTranslationOptions options) + : options_(options) {} + + const ModuleTranslationOptions &options() const { return options_; } + + std::vector translateModule(ModuleOp module); + + private: + LogicalResult translateMultiArchExecutable( + IREE::MultiArchExecutableOp executableOp, VMModuleBuilder *moduleBuilder); + + LogicalResult translateSequencerModule(ModuleOp module, + VMModuleBuilder *moduleBuilder); + LogicalResult declareFunction(FuncOp function, + VMModuleBuilder *moduleBuilder); + LogicalResult defineFunction(FuncOp function, VMModuleBuilder *moduleBuilder); + + ModuleTranslationOptions options_; +}; + +std::vector SequencerTranslator::translateModule(ModuleOp module) { + // Run one large set of passes to get to a partitioned module. + auto partitioningPasses = createPassManager(module.getContext(), options()); + buildLegalizeInputPassPipeline(partitioningPasses.get()); + buildPartitioningPassPipeline(partitioningPasses.get()); + if (failed(runPassPipeline(options(), partitioningPasses.get(), module))) { + module.emitError() << "Failed to run partitioning passes"; + return {}; + } + + // Run the sequencer-specific conversion passes on the module after we've + // removed the executables. We don't lower all the way yet as we need the + // executables to know some things. + auto sequencerConversionPasses = + createPassManager(module.getContext(), options()); + buildSequencerConversionPassPipeline(sequencerConversionPasses.get()); + if (failed(runPassPipeline(options(), sequencerConversionPasses.get(), + module))) { + module.emitError() << "Failed to run sequencer conversion passes"; + return {}; + } + + // Lower sequencer functions to their final form. + auto sequencerLoweringPasses = + createPassManager(module.getContext(), options()); + buildSequencerLoweringPassPipeline(sequencerLoweringPasses.get()); + if (failed( + runPassPipeline(options(), sequencerLoweringPasses.get(), module))) { + module.emitError() << "Failed to run sequencer lowering passes"; + return {}; + } + + // Perform translation on all executables. + // We then know exactly what executable formats we have and can query them to + // see if we need to do any additional processing (such as to support better + // types/etc). + ::flatbuffers::FlatBufferBuilder fbb; + VMModuleBuilder moduleBuilder(&fbb); + for (auto multiArchExecutableOp : + module.getOps()) { + if (failed(translateMultiArchExecutable(multiArchExecutableOp, + &moduleBuilder))) { + module.emitError() << "Failed to translate multi-arch-executable"; + return {}; + } + } + + // Build the module bytecode. + if (failed(translateSequencerModule(module, &moduleBuilder))) { + module.emitError() << "Unable to translate sequencer module"; + return {}; + } + auto moduleDef = moduleBuilder.Finish(); + if (moduleDef.IsNull()) { + module.emitError() << "Failed to verify completed module def"; + return {}; + } + auto bytes = moduleBuilder.Serialize(moduleDef); + if (bytes.empty()) { + module.emitError() << "Failed to serialize final module def"; + return {}; + } + return bytes; +} + +LogicalResult SequencerTranslator::translateMultiArchExecutable( + IREE::MultiArchExecutableOp multiArchExecutableOp, + VMModuleBuilder *moduleBuilder) { + auto &fbb = *moduleBuilder->fbb(); + + // Find the unspecified executable. This is the template from which we will + // translate to other targets. + IREE::ExecutableOp templateExecutableOp; + for (auto executableOp : + multiArchExecutableOp.getBlock().getOps()) { + if (executableOp.format() == + static_cast(IREE::ExecutableFormat::Unspecified)) { + templateExecutableOp = executableOp; + break; + } + } + if (!templateExecutableOp) { + // Fine for there to be no unspecified executable - just ignore. + return success(); + } + int entryPointCount = 0; + for (auto func : templateExecutableOp.getInnerModule().getOps()) { + if (func.getAttr("iree.executable.export")) { + ++entryPointCount; + } + } + + // For now we just add target config ops based on options. In the future we + // could do this earlier via an analysis pass determining which targets should + // be used for each executable. + OpBuilder configBuilder(templateExecutableOp); + configBuilder.setInsertionPointToStart(&templateExecutableOp.getBlock()); + insertTargetConfigOps(options(), &configBuilder); + + // Find all target configs and bucket them into the backends that will + // translate them. This way we can batch the translations and possibly enable + // backends to dedupe some things. + DenseMap> + backendTargetConfigOps; + for (auto targetConfigOp : templateExecutableOp.getBlock() + .getOps()) { + auto &targetConfigOps = backendTargetConfigOps[targetConfigOp.backend()]; + targetConfigOps.push_back(targetConfigOp); + } + if (backendTargetConfigOps.empty()) { + // There are no target configs - which likely means we've already translated + // this in a previous pass. + return success(); + } + + ExecutableTranslationOptions translationOptions; + translationOptions.CopyFrom(options()); + + // Invoke each backend translator on the template executables to produce new + // executables. The backends may produce any number of executables that we + // then merge back in to the iree.multi_arch_executable and the module + // flatbuffer. + std::vector> translatedExecutableDefs; + for (auto it : backendTargetConfigOps) { + const auto &backendKey = it.first; + const auto &targetConfigOps = it.second; + + // Find the translator to use in the registry. It must have been linked in + // and the name must match what is used in the registration macro. + auto translateExecutableFn = + getExecutableTranslationRegistry().lookup(backendKey); + if (!translateExecutableFn) { + return multiArchExecutableOp.emitError() + << "No registered backend found for target '" << backendKey.str() + << "'; ensure it is linked in to your binary (have: " + << llvm::join(getExecutableTranslationRegistry().keys(), ", ") + << ")"; + } + + // Clone the executable for each config so that the translator is allowed to + // modify it in-place. + // We also need to strip all of the other configs so that the translator + // backend only sees the one for each of its configs. + OpBuilder builder(&multiArchExecutableOp.getBlock()); + builder.setInsertionPoint(multiArchExecutableOp.getBlock().getTerminator()); + SmallVector clonedExecutableOps; + for (auto targetConfigOp : targetConfigOps) { + auto executableCloneOp = cast( + builder.clone(*templateExecutableOp.getOperation())); + for (auto existingTargetConfigOp : llvm::make_early_inc_range( + executableCloneOp.getBlock() + .getOps())) { + existingTargetConfigOp.erase(); + } + OpBuilder configBuilder(executableCloneOp); + configBuilder.setInsertionPointToStart(&executableCloneOp.getBlock()); + configBuilder.clone(*targetConfigOp.getOperation()); + clonedExecutableOps.push_back(executableCloneOp); + } + + // Perform translation on all of the backend-specific targets. + // Note that the results here may not have the same number of executables we + // started with if the backend either couldn't satisfy some of the requests + // or decided to dedupe or expand certain ones. + auto translationResults = + translateExecutableFn(clonedExecutableOps, translationOptions); + if (!translationResults.hasValue()) { + return multiArchExecutableOp.emitError() + << "Failed to translate executable with backend " << backendKey; + } + for (auto &executableDef : translationResults.getValue().executable_defs) { + translatedExecutableDefs.push_back(std::move(executableDef)); + } + } + + // Remove configs from the template executable so that if we are called again + // we don't re-translate. + for (auto targetConfigOp : llvm::make_early_inc_range( + templateExecutableOp.getBlock() + .getOps())) { + targetConfigOp.erase(); + } + + // Create multi-arch executable with all of the target-specific executables. + iree::MultiArchExecutableDefT maedf; + maedf.name = multiArchExecutableOp.getName(); + maedf.entry_point_count = entryPointCount; + maedf.executables = std::move(translatedExecutableDefs); + auto maedfOffset = iree::MultiArchExecutableDef::Pack(fbb, &maedf); + RETURN_IF_FAILURE( + moduleBuilder->executable_table()->AddMultiArchExecutable(maedfOffset)); + + return success(); +} + +LogicalResult SequencerTranslator::translateSequencerModule( + ModuleOp module, VMModuleBuilder *moduleBuilder) { + // Declare functions. This must happen first so that we get stable indices + // during declaration (as call ops need to use the function table). + for (auto function : module.getOps()) { + RETURN_IF_FAILURE(declareFunction(function, moduleBuilder)); + } + + // Define functions and convert their bodies to bytecode. + for (auto function : module.getOps()) { + RETURN_IF_FAILURE(defineFunction(function, moduleBuilder)); + } + + return success(); +} + +LogicalResult SequencerTranslator::declareFunction( + FuncOp function, VMModuleBuilder *moduleBuilder) { + auto *functionTable = moduleBuilder->function_table(); + if (functionTable->IsFunctionDeclared(function)) { + // Already declared. + return success(); + } + + LinkageType linkageType; + if (function.isExternal()) { + linkageType = LinkageType::kImport; + } else if (function.getAttr("iree.module.export")) { + linkageType = LinkageType::kExport; + } else { + linkageType = LinkageType::kInternal; + } + if (failed(functionTable->DeclareFunction(function, linkageType))) { + return function.emitError() + << "Unable to declare function " << function.getName(); + } + + // Import functions must have their definition defined here so we get their + // type. Internal and export functions will be defined during conversion. + if (linkageType == LinkageType::kImport) { + VMFunctionBuilder functionBuilder(function, moduleBuilder->function_table(), + moduleBuilder->fbb()); + auto functionOffset = functionBuilder.Finish(); + if (functionOffset.IsNull()) { + return function.emitError() + << "Failed to create import function bytecode"; + } + RETURN_IF_FAILURE( + functionTable->DefineFunction(function, functionOffset, {})); + } + + return success(); +} + +LogicalResult SequencerTranslator::defineFunction( + FuncOp function, VMModuleBuilder *moduleBuilder) { + VMFunctionBuilder functionBuilder(function, moduleBuilder->function_table(), + moduleBuilder->fbb()); + registerSequencerCustomWriters(&functionBuilder); + RETURN_IF_FAILURE(functionBuilder.ConvertBytecode()); + auto functionOffset = functionBuilder.Finish(); + if (functionOffset.IsNull()) { + return function.emitError() << "Failed to convert function to bytecode"; + } + RETURN_IF_FAILURE(moduleBuilder->function_table()->DefineFunction( + function, functionOffset, functionBuilder.source_map())); + return success(); +} + +} // namespace + +std::vector translateMlirToIreeSequencerModule( + ModuleOp module, ModuleTranslationOptions options) { + SequencerTranslator translator(options); + return translator.translateModule(module); +} + +LogicalResult translateMlirToIreeSequencerModuleFile( + ModuleOp module, llvm::raw_ostream &output) { + ModuleTranslationOptions options; + SequencerTranslator translator(options); + auto bytecodeModule = translator.translateModule(module); + if (bytecodeModule.empty()) { + return emitError(UnknownLoc::get(module.getContext()), + "failed to translate module"); + } + + output.write(reinterpret_cast(bytecodeModule.data()), + bytecodeModule.size()); + return success(); +} + +static TranslateFromMLIRRegistration MlirToIreeSequencerModuleTranslate( + "mlir-to-iree-module", translateMlirToIreeSequencerModuleFile); + +} // namespace iree_compiler +} // namespace mlir diff --git a/compiler/Translation/SequencerModuleTranslation.h b/compiler/Translation/SequencerModuleTranslation.h new file mode 100644 index 000000000000..7bc1e7d27894 --- /dev/null +++ b/compiler/Translation/SequencerModuleTranslation.h @@ -0,0 +1,36 @@ +// Copyright 2019 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#ifndef THIRD_PARTY_MLIR_EDGE_IREE_COMPILER_TRANSLATION_SEQUENCERMODULETRANSLATION_H_ +#define THIRD_PARTY_MLIR_EDGE_IREE_COMPILER_TRANSLATION_SEQUENCERMODULETRANSLATION_H_ + +#include + +#include "third_party/llvm/llvm/projects/google_mlir/include/mlir/IR/Module.h" +#include "third_party/mlir_edge/iree/compiler/Utils/TranslationUtils.h" + +namespace mlir { +namespace iree_compiler { + +// Translates an MLIR module in a compatible IREE input dialect (such as XLA HLO +// and/or Std) into an IREE Module. Executables will be lowered based on the +// provided configuration. +// Returns an empty vector on translation failure. +std::vector translateMlirToIreeSequencerModule( + ModuleOp module, ModuleTranslationOptions options = {}); + +} // namespace iree_compiler +} // namespace mlir + +#endif // THIRD_PARTY_MLIR_EDGE_IREE_COMPILER_TRANSLATION_SEQUENCERMODULETRANSLATION_H_ diff --git a/compiler/Utils/DispatchUtils.cpp b/compiler/Utils/DispatchUtils.cpp new file mode 100644 index 000000000000..fa829355dd8c --- /dev/null +++ b/compiler/Utils/DispatchUtils.cpp @@ -0,0 +1,733 @@ +// Copyright 2019 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "third_party/mlir_edge/iree/compiler/Utils/DispatchUtils.h" + +#include "third_party/llvm/llvm/include/llvm/ADT/ArrayRef.h" +#include "third_party/llvm/llvm/include/llvm/ADT/DenseSet.h" +#include "third_party/llvm/llvm/include/llvm/ADT/STLExtras.h" +#include "third_party/llvm/llvm/include/llvm/ADT/SetVector.h" +#include "third_party/llvm/llvm/include/llvm/ADT/SmallPtrSet.h" +#include "third_party/llvm/llvm/include/llvm/ADT/SmallVector.h" +#include "third_party/llvm/llvm/projects/google_mlir/include/mlir/Dialect/StandardOps/Ops.h" +#include "third_party/llvm/llvm/projects/google_mlir/include/mlir/IR/Attributes.h" +#include "third_party/llvm/llvm/projects/google_mlir/include/mlir/IR/BlockAndValueMapping.h" +#include "third_party/llvm/llvm/projects/google_mlir/include/mlir/IR/Builders.h" +#include "third_party/llvm/llvm/projects/google_mlir/include/mlir/IR/Location.h" +#include "third_party/llvm/llvm/projects/google_mlir/include/mlir/IR/MLIRContext.h" +#include "third_party/llvm/llvm/projects/google_mlir/include/mlir/IR/StandardTypes.h" +#include "third_party/llvm/llvm/projects/google_mlir/include/mlir/Support/LLVM.h" +#include "third_party/llvm/llvm/projects/google_mlir/include/mlir/Support/LogicalResult.h" +#include "third_party/llvm/llvm/projects/google_mlir/include/mlir/Transforms/Utils.h" +#include "third_party/mlir_edge/iree/compiler/IR/Ops.h" +#include "third_party/mlir_edge/iree/compiler/IR/Sequencer/HLOps.h" +#include "third_party/mlir_edge/iree/compiler/Utils/MemRefUtils.h" +#include "third_party/tensorflow/compiler/mlir/xla/ir/hlo_ops.h" + +namespace mlir { +namespace iree_compiler { + +Value *calculateWorkload(Operation *op, Value *baseOperand) { + OpBuilder builder(op); + + std::array workload = {1, 1, 1}; + + // TODO(b/139353314): lookup/calculate based on type/etc. + auto resultType = baseOperand->getType(); + if (auto shapedType = resultType.dyn_cast()) { + if (!shapedType.hasStaticShape()) { + op->emitOpError() << "Dynamic shapes not yet supported"; + return nullptr; + } + auto shape = shapedType.getShape(); + // Drop the trailing ones from the shape. + while (shape.size() > 1 && shape.back() == 1) { + shape = shape.drop_back(); + } + if (shape.size() <= 3) { + // Maps to XYZ (possibly with 1's for unused dimensions). + for (auto dim : enumerate(shape)) { + workload[shape.size() - 1 - dim.index()] = dim.value(); + } + } else { + // Need to flatten the shape to fit XYZ. For now we just squash from LHS. + workload[2] = 1; + for (int i = 0; i < shape.size(); ++i) { + workload[2] *= shape[i]; + } + workload[1] = shape[shape.size() - 2]; + workload[0] = shape.back(); + } + } + + // TODO(b/139353314): optimize workload layout. + + auto constantType = builder.getTensorType({3}, builder.getIntegerType(32)); + return builder.create( + op->getLoc(), constantType, + DenseIntElementsAttr::get(constantType, workload)); +} + +bool isTriviallyDispatchable(FuncOp func) { + if (func.empty()) return false; + auto &block = func.front(); + if (block.getOperations().size() != 2) return false; + auto &op0 = block.front(); + auto &op1 = block.back(); + auto regionOp = dyn_cast(op0); + auto returnOp = dyn_cast(op1); + if (!regionOp || !returnOp || + regionOp.getNumResults() != returnOp.getNumOperands()) { + return false; + } + for (int i = 0; i < regionOp.getNumResults(); ++i) { + if (regionOp.getResult(i) != returnOp.getOperand(i)) return false; + } + return true; +} + +namespace { + +// Returns the set of values that must be captured for use by |ops| and the +// set of values defined by |ops| that are used outside of the set. +LogicalResult analyzeOpRangeValues( + const llvm::SmallDenseSet &opSet, + llvm::SetVector *capturedValues, + llvm::SetVector *escapingValues) { + for (auto *op : opSet) { + for (auto *value : op->getOperands()) { + if (!llvm::is_contained(opSet, value->getDefiningOp())) { + // Op is using a value not in the ops set, ensure we capture it. + capturedValues->insert(value); + } + } + for (auto *value : op->getResults()) { + for (auto &use : value->getUses()) { + if (!llvm::is_contained(opSet, use.getOwner())) { + // An op outside of the ops set is using the value, needs to escape. + escapingValues->insert(value); + } + } + } + } + return success(); +} + +} // namespace + +LogicalResult buildDispatchRegion(FuncOp func, Block *parentBlock, + Value *workload, ArrayRef ops) { + // Fused location with all ops. + SmallVector opLocs; + for (auto *op : ops) { + opLocs.push_back(op->getLoc()); + } + auto regionLoc = FusedLoc::get(opLocs, func.getContext()); + + // Get a list of values that we need to capture and values that escape the + // region and need to be returned. + llvm::SmallDenseSet opSet; + opSet.reserve(ops.size()); + opSet.insert(ops.begin(), ops.end()); + llvm::SetVector capturedValues; + llvm::SetVector escapingValues; + if (failed(analyzeOpRangeValues(opSet, &capturedValues, &escapingValues))) { + return failure(); + } + SmallVector escapingTypes; + for (auto *value : escapingValues) escapingTypes.push_back(value->getType()); + + // Build the region op and add it to the parent block. + OpBuilder parentBuilder(parentBlock); + parentBuilder.setInsertionPoint(ops.back()); + auto dispatchRegionOp = parentBuilder.create( + regionLoc, escapingTypes, workload, capturedValues.getArrayRef()); + + // Create the block and setup the arg mapping for captured values. + auto *regionBlock = new Block(); + dispatchRegionOp.getBody().push_back(regionBlock); + OpBuilder regionBuilder(regionBlock); + BlockAndValueMapping mapping; + for (auto *capturedValue : capturedValues) { + auto *blockArg = regionBlock->addArgument(capturedValue->getType()); + mapping.map(capturedValue, blockArg); + } + + // Clone ops into the new region block. + for (auto *op : ops) { + // Note that this updates the mapping with the new values (so at the end + // we have those new values). + regionBuilder.clone(*op, mapping); + } + + // Return results (as we need a terminator in our block). + // These are all of the values that escape our region. + SmallVector resultValues; + for (auto *oldValue : escapingValues) { + resultValues.push_back(mapping.lookupOrDefault(oldValue)); + } + regionBuilder.create(opLocs.back(), resultValues); + + // Replace usage of values with the results of the region. + for (int i = 0; i < escapingValues.size(); ++i) { + escapingValues[i]->replaceAllUsesWith(dispatchRegionOp.getResult(i)); + } + + // Remove original ops from the parent region. + for (auto it = ops.rbegin(); it != ops.rend(); ++it) { + (*it)->erase(); + } + + return success(); +} + +namespace { + +// Replaces |returnOp| with a clone including |newOperands| appended. +LogicalResult appendReturnOperands(IREE::ReturnOp returnOp, + ArrayRef newOperands) { + // Insert prior to the original return. + OpBuilder builder(returnOp); + + // Clone with new args. + SmallVector operands; + operands.reserve(returnOp.getNumOperands() + newOperands.size()); + operands.append(returnOp.operand_begin(), returnOp.operand_end()); + operands.append(newOperands.begin(), newOperands.end()); + builder.create(returnOp.getLoc(), operands); + + // Remove original. + returnOp.erase(); + + return success(); +} + +// Replaces |regionOp| with a clone including |newArgs| and |newResults|. +IREE::DispatchRegionOp appendRegionArgsAndResults( + IREE::DispatchRegionOp ®ionOp, ArrayRef newArgs, + ArrayRef newResults, Location otherLoc) { + // Insert prior to the original region. + OpBuilder builder(regionOp); + + // Location is original region + new region location (both probably fused). + SmallVector fusedLocs = {regionOp.getLoc(), otherLoc}; + auto fusedLoc = FusedLoc::get(fusedLocs, regionOp.getContext()); + + // Clone with new results. + SmallVector operands; + operands.append(regionOp.getArgOperands().begin(), + regionOp.getArgOperands().end()); + operands.append(newArgs.begin(), newArgs.end()); + SmallVector resultTypes; + resultTypes.append(regionOp.result_type_begin(), regionOp.result_type_end()); + for (auto *newResult : newResults) { + resultTypes.push_back(newResult->getType()); + } + auto newRegionOp = builder.create( + fusedLoc, resultTypes, regionOp.getWorkload(), operands, + regionOp.getAttrs()); + newRegionOp.getBody().takeBody(regionOp.getBody()); + + // Replace uses of original values with the new values. + for (int i = 0; i < regionOp.getNumResults(); ++i) { + regionOp.getResult(i)->replaceAllUsesWith(newRegionOp.getResult(i)); + } + + // Erase the original region. + regionOp.erase(); + + return newRegionOp; +} + +// Removes results that are not used from the dispatch region. +// Returns the new operation. There may be unused ops in the region but DCE +// should take care of that later. +IREE::DispatchRegionOp removeUnusedResults(IREE::DispatchRegionOp regionOp) { + // Find return value within the region. + auto ®ionBlock = regionOp.getBody().getBlocks().front(); + auto returnOp = dyn_cast(regionBlock.getTerminator()); + if (!returnOp) { + regionBlock.getParent()->getParentOfType().emitError() + << "Block does not contain an iree.return op"; + } + + // Calculate new return values. + SmallVector newReturnTypes; + SmallVector newReturnValues; + SmallVector newRegionResults; + for (int i = 0; i < returnOp.getNumOperands(); ++i) { + auto *resultValue = regionOp.getResult(i); + if (!resultValue->use_empty()) { + // Still has uses so we will preserve it. + newReturnTypes.push_back(resultValue->getType()); + newReturnValues.push_back(returnOp.getOperand(i)); + newRegionResults.push_back(resultValue); + } + } + + // Update return op operands. We can do this in-place as we are only shrinking + // the list. + returnOp.getOperation()->setOperands(newReturnValues); + + // Insert prior to the original region. + OpBuilder builder(regionOp); + + // Clone with new results. + SmallVector operands(regionOp.getArgOperands()); + auto newRegionOp = builder.create( + regionOp.getLoc(), newReturnTypes, regionOp.getWorkload(), operands, + regionOp.getAttrs()); + newRegionOp.getBody().takeBody(regionOp.getBody()); + + // Replace uses of original values with the new values. + for (int i = 0; i < newRegionResults.size(); ++i) { + newRegionResults[i]->replaceAllUsesWith(newRegionOp.getResult(i)); + } + + // Erase the original region. + regionOp.erase(); + + return newRegionOp; +} + +// Returns true if |lhs| and |rhs| have either an identical workload or one that +// is compatible. +bool areDispatchRegionWorkloadsCompatible(IREE::DispatchRegionOp &lhs, + IREE::DispatchRegionOp &rhs) { + // TODO(benvanik): more sophisticated checking; right now it's just identical. + return lhs.getWorkload() == rhs.getWorkload(); +} + +// Returns true if |value| depends in any way on |op| through any path. +// Only works if the operations are within the same block. +bool doesValueDependOnOperation(Value *value, Operation *op) { + if (!value->getDefiningOp()) { + return false; + } else if (value->getDefiningOp() == op) { + return true; + } else if (value->getDefiningOp()->isBeforeInBlock(op)) { + // Can't depend on |op| as it is defined prior to it. + return false; + } + for (auto *operand : value->getDefiningOp()->getOperands()) { + if (doesValueDependOnOperation(operand, op)) { + return true; + } + } + return true; +} + +// Returns true if |rhs| transitively depends on any out of |lhs|. +// |rhs| may depend directly on the results of |lhs| but no other ops in the +// parent block will use the results prior to |rhs|. +bool areDispatchRegionsTransitivelyDependent(IREE::DispatchRegionOp &lhs, + IREE::DispatchRegionOp &rhs) { + for (auto *arg : rhs.getArgOperands()) { + if (arg->getDefiningOp() != lhs && doesValueDependOnOperation(arg, lhs)) { + // Transitively dependent - boo - can't merge yet. + return true; + } + } + return false; +} + +// Returns true if the dispatch region contains only a single block. +// This is because our merge isn't very smart and will not preserve the CFG +// right now. We can fix this when needed. +bool isDispatchRegionMergable(IREE::DispatchRegionOp ®ionOp) { + // Disallow merging of dispatch regions containing matmuls and other big ops. + // We do this to allow backends to lower the big op as entirely isolated such + // that substituting library calls is easier. + for (auto &block : regionOp.getBody().getBlocks()) { + for (auto &op : block) { + if (isa(op) || isa(op)) { + return false; + } + } + } + return regionOp.getBody().getBlocks().size() == 1; +} + +// Merges |rhs| into |lhs| and returns the new |lhs| op. +// Precondition: !areDispatchRegionsTransitivelyDependent +IREE::DispatchRegionOp mergeDispatchRegions(IREE::DispatchRegionOp &lhs, + IREE::DispatchRegionOp &rhs) { + auto &lhsBlock = lhs.getBody().front(); + auto &rhsBlock = rhs.getBody().front(); + + // Find the values used as return values in the lhs. + // We'll need to replace the uses in rhs with these. + auto lhsReturnOp = cast(lhsBlock.getTerminator()); + SmallVector lhsReturnValues; + lhsReturnValues.reserve(lhsReturnOp.getNumOperands()); + lhsReturnValues.append(lhsReturnOp.operand_begin(), + lhsReturnOp.operand_end()); + + // Find the values used as return values in the rhs. + // We'll add these to the results of the lhs region. + auto rhsReturnOp = cast(rhsBlock.getTerminator()); + SmallVector rhsReturnValues; + rhsReturnValues.reserve(rhsReturnOp.getNumOperands()); + rhsReturnValues.append(rhsReturnOp.operand_begin(), + rhsReturnOp.operand_end()); + + // Compute new args. + BlockAndValueMapping mapping; + SmallVector newArgs; + for (int rhsOpIdx = 0; rhsOpIdx < rhs.getNumArgOperands(); ++rhsOpIdx) { + bool didElide = false; + // Find if the rhs arg already exists on the lhs and dedupe. + for (int lhsOpIdx = 0; lhsOpIdx < lhs.getNumArgOperands(); ++lhsOpIdx) { + if (rhs.getArgOperand(rhsOpIdx) == lhs.getArgOperand(lhsOpIdx)) { + mapping.map(rhsBlock.getArgument(rhsOpIdx), + lhsBlock.getArgument(lhsOpIdx)); + didElide = true; + break; + } + } + // Find if the arg has a direct dependency on the results of the lhs. + for (int lhsResultIdx = 0; lhsResultIdx < lhs.getNumResults(); + ++lhsResultIdx) { + if (rhs.getArgOperand(rhsOpIdx) == lhs.getResult(lhsResultIdx)) { + // Direct dependency; can elide. We'll skip adding it to the new region + // args and instead just remap it later. + mapping.map(rhsBlock.getArgument(rhsOpIdx), + lhsReturnValues[lhsResultIdx]); + didElide = true; + break; + } + } + if (!didElide) { + // Add to the lhs block. + auto *oldArg = rhs.getOperand(rhsOpIdx + 1); + auto *newArg = lhsBlock.addArgument(oldArg->getType()); + mapping.map(rhsBlock.getArgument(rhsOpIdx), newArg); + newArgs.push_back(oldArg); + } + } + + OpBuilder regionBuilder(&lhsBlock); + + // Copy ops (replacing any args as needed). + // Note that we need to insert prior to the terminator. + regionBuilder.setInsertionPoint(lhsReturnOp); + for (auto &op : rhsBlock) { + // Note that this updates the mapping with the new values (so at the end + // we have those new values). + // + // We avoid the return op here as we have already merged it above. + if (!op.isKnownTerminator()) { + regionBuilder.clone(op, mapping); + } + } + + // Compute new results and add to both region and return op. + SmallVector newResults; + for (auto *rhsResult : rhsReturnValues) { + newResults.push_back(mapping.lookupOrDefault(rhsResult)); + } + if (failed(appendReturnOperands(lhsReturnOp, newResults))) { + return nullptr; + } + auto newRegionOp = + appendRegionArgsAndResults(lhs, newArgs, newResults, rhs.getLoc()); + + // Replace uses of original values with the new values. + for (int i = 0; i < rhs.getNumResults(); ++i) { + rhs.getResult(i)->replaceAllUsesWith( + newRegionOp.getResult(lhsReturnValues.size() + i)); + } + + // Remove rhs region. + rhs.erase(); + + // Remove results from the lhs that aren't used anymore as they may have been + // elided when we merged as only the rhs was using them. + newRegionOp = removeUnusedResults(newRegionOp); + + return newRegionOp; +} + +} // namespace + +LogicalResult mergeBlockDispatchRegions(FuncOp func, Block *parentBlock) { + SmallVector mergableRegions; + for (auto &op : *parentBlock) { + if (auto regionOp = dyn_cast(op)) { + if (isDispatchRegionMergable(regionOp)) { + mergableRegions.push_back(regionOp); + } else { + regionOp.emitRemark( + "Unable to merge into following iree.dispatch_regions; " + "contains non-trivial control flow"); + } + } + } + for (int i = 0; i < mergableRegions.size(); ++i) { + if (!mergableRegions[i]) continue; + auto &lhs = mergableRegions[i]; + for (int j = i + 1; j < mergableRegions.size(); ++j) { + if (!mergableRegions[j]) continue; + auto &rhs = mergableRegions[j]; + if (!areDispatchRegionWorkloadsCompatible(lhs, rhs) || + areDispatchRegionsTransitivelyDependent(lhs, rhs)) { + continue; + } + if (!isDispatchRegionMergable(rhs)) { + // TODO(b/134675461): support non-trivial control flow. + rhs.emitRemark( + "Unable to merge into previous iree.dispatch_region; " + "contains non-trivial control flow"); + } + mergableRegions[i] = mergeDispatchRegions(lhs, rhs); + if (!mergableRegions[i]) { + return failure(); + } + mergableRegions[j] = nullptr; + --i; // Try again to see if there are subsequent regions to merge. + break; + } + } + + return success(); +} + +namespace { + +// Recursively clones the given |sourceOp| and returns the newly cloned op. +Operation *recursivelyCloneOp(Operation *sourceOp, OpBuilder *builder, + BlockAndValueMapping *mapping) { + // Note that we dedupe required operands in the case of multiple arguments + // coming from the same source operation. + SmallPtrSet operandOps; + for (auto *operand : sourceOp->getOperands()) { + operandOps.insert(operand->getDefiningOp()); + } + for (auto *operandOp : operandOps) { + recursivelyCloneOp(operandOp, builder, mapping); + } + return builder->clone(*sourceOp, *mapping); +} + +// Clones the |sourceValue| op tree into |targetBlock|. +// |mapping| is used to lookup existing values that may be present in the block +// such as block arguments or already cloned ancestor ops. |mapping| will be +// updated as the tree is cloned. +Value *cloneOpTreeIntoBlock(Value *sourceValue, Block *targetBlock, + BlockAndValueMapping *mapping) { + // If the op has already been cloned we can just reuse that. + // This happens if multiple arguments reference the same trees. + if (auto *existingValue = mapping->lookupOrNull(sourceValue)) { + return existingValue; + } + + OpBuilder builder(targetBlock); + builder.setInsertionPointToStart(targetBlock); + auto *sourceOp = sourceValue->getDefiningOp(); + auto *clonedOp = recursivelyCloneOp(sourceOp, &builder, mapping); + + // Return only the result matching our source value (in the case of multiple + // results). + int resultIndex = std::distance( + sourceOp->result_begin(), + std::find(sourceOp->result_begin(), sourceOp->result_end(), sourceValue)); + return clonedOp->getResult(resultIndex); +} + +} // namespace + +LogicalResult inlineDispatchRegionOperandsUsingValue( + IREE::DispatchRegionOp dispatchRegionOp, Value *value) { + // Find all args that are using this value. + SmallVector argIndices; + for (auto arg : llvm::enumerate(dispatchRegionOp.getArgOperands())) { + if (arg.value() == value) { + argIndices.push_back(arg.index()); + } + } + if (argIndices.empty()) { + // Not used? Wasteful call! + return success(); + } + + // Clone the value (and the ops required to create it) into the entry block. + auto &entryBlock = dispatchRegionOp.getBody().getBlocks().front(); + BlockAndValueMapping mapping; + auto *clonedValue = cloneOpTreeIntoBlock(value, &entryBlock, &mapping); + + // Replace all uses of the inner operand with the new value. + for (unsigned argIndex : argIndices) { + entryBlock.getArgument(argIndex)->replaceAllUsesWith(clonedValue); + } + + // Remove the dispatch region args and the block args that have been + // replaced. + for (unsigned argIndex : llvm::reverse(argIndices)) { + dispatchRegionOp.getOperation()->eraseOperand( + dispatchRegionOp.mapArgOperandToOpOperand(argIndex)); + entryBlock.eraseArgument(argIndex); + } + + return success(); +} + +namespace { + +// Recursively finds all reachable functions from the given |rootFunc| and adds +// them to the |reachableFuncs| set. +// +// Note that indirect calls are not supported, however we don't allow those in +// dispatch regions anyway so they should not be present here. +LogicalResult findReachableFunctions(Operation *rootFunc, + llvm::SetVector &reachableFuncs) { + bool allCallsValid = true; + rootFunc->walk([&](CallOp op) { + auto callee = rootFunc->getParentOfType().lookupSymbol( + op.getCallee()); + if (!callee.getAttr("iree.dispatchable")) { + allCallsValid = false; + rootFunc->emitError() << callee.getName().str() << " is not dispatchable"; + return; + } + if (reachableFuncs.insert(callee)) { + findReachableFunctions(callee, reachableFuncs); + } + }); + return success(allCallsValid); +} + +} // namespace + +std::pair createRegionExecutable( + Operation *op, FunctionType functionType, StringRef symbolSuffix) { + // Create the function and take the region body directly. + // NOTE: this will get uniquified if we have multiple in the same block. + auto parentFunc = op->getParentOfType(); + std::string functionName = + (parentFunc.getName().str() + "_rgn" + symbolSuffix).str(); + auto outlinedFunc = FuncOp::create(op->getLoc(), functionName, functionType); + BlockAndValueMapping mapping; + op->getRegion(0).cloneInto(&outlinedFunc.getBody(), mapping); + + // Gather all reachable functions. + llvm::SetVector reachableFuncs; + findReachableFunctions(outlinedFunc, reachableFuncs); + + // Create the multi-arch executable that will contain the outlined region. + // NOTE: this will get uniquified if we have multiple in the same block. + auto parentModule = parentFunc.getParentOfType(); + OpBuilder parentModuleBuilder(parentModule); + parentModuleBuilder.setInsertionPoint(parentFunc); + std::string executableName = + (parentFunc.getName().str() + "_ex" + symbolSuffix).str(); + auto multiArchExecutable = + parentModuleBuilder.create( + outlinedFunc.getLoc(), executableName); + + // Create the executable op initially unspecified so that later + // transformations can compile it to various formats. + OpBuilder multiArchExecutableBuilder(multiArchExecutable); + multiArchExecutableBuilder.setInsertionPointToStart( + &multiArchExecutable.getBlock()); + auto executable = multiArchExecutableBuilder.create( + outlinedFunc.getLoc(), IREE::ExecutableFormat::Unspecified); + + // Create the inner ModuleOp that contains the original functions. We need + // to provide this shim as some ops (like std.call) look for the + // containing module to provide symbol resolution. + OpBuilder executableBuilder(executable); + executableBuilder.setInsertionPointToStart(&executable.getBlock()); + auto innerModule = executableBuilder.create(outlinedFunc.getLoc()); + + // TODO(b/137674142): make an ExecutableEntryPointOp and convert the + // entry thunk into that format. + innerModule.push_back(outlinedFunc); + + // Copy all reachable functions into the executable. + // Linker passes may dedupe these later on. + for (auto reachableFunc : reachableFuncs) { + auto clonedFunc = reachableFunc.clone(); + clonedFunc.removeAttr("iree.dispatchable"); + innerModule.push_back(clonedFunc); + } + + return std::make_pair(multiArchExecutable, outlinedFunc); +} + +Value *insertDispatcherStore(Operation *op, Value *value, OpBuilder *builder) { + if (!value) { + return nullptr; + } + + auto memRefType = convertTypeToMemRef(value->getType()); + if (!memRefType) { + return nullptr; + } + + // If the previous value was already a memref we don't need to change + // anything. + // TODO(benvanik): ensure indices make sense. + if (value->getType().isa()) { + return value; + } else if (value->getType().isa()) { + auto castOp = builder->create(op->getLoc(), + memRefType, value); + return castOp.getResult(); + } + + // Allocate the memref to store the value. + auto newStorage = builder->create(op->getLoc(), memRefType); + + // Insert the store we'll use to box the value. + builder->create(op->getLoc(), value, newStorage, + ArrayRef{}); + + return newStorage; +} + +Value *insertDispatcherLoad(Operation *op, Value *originalValue, + Value *allocatedValue, OpBuilder *builder) { + // If old value was a memref we don't need to change anything. + if (originalValue->getType().isa()) { + return allocatedValue; + } else if (originalValue->getType().isa()) { + auto castOp = builder->create( + op->getLoc(), originalValue->getType(), allocatedValue); + originalValue->replaceAllUsesWith(castOp.getResult()); + return castOp.getResult(); + } + + // Insert the load we'll use to unbox the value. + auto loadOp = builder->create(op->getLoc(), allocatedValue, + ArrayRef{}); + originalValue->replaceAllUsesWith(loadOp); + return loadOp; +} + +// TODO(benvanik): enough information to walk into dispatch region and compute +// shape when not static. +Value *allocateDispatchOutputBuffer(Location loc, MemRefType type, + OpBuilder &builder) { + // TODO(benvanik): allocation algorithm: + // - synthesize shape logic (magic) [[ for now assume fixed shapes ]] + // - insert shape logic above region + // - rely on folding to merge multiple calculations together + // - unranked = death, need to be able to alloc shape outputs + // - insert alloc + SmallVector dimPieces; + return builder.create(loc, type, dimPieces); +} + +} // namespace iree_compiler +} // namespace mlir diff --git a/compiler/Utils/DispatchUtils.h b/compiler/Utils/DispatchUtils.h new file mode 100644 index 000000000000..0581ce1a8455 --- /dev/null +++ b/compiler/Utils/DispatchUtils.h @@ -0,0 +1,92 @@ +// Copyright 2019 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// Utilities for dispatch region and function manipulation. +// These are shared between all dispatchable types such as the standard +// iree.dispatch_region as well as dispatch-related types like +// iree.reduction_region. + +#ifndef THIRD_PARTY_MLIR_EDGE_IREE_COMPILER_UTILS_DISPATCHUTILS_H_ +#define THIRD_PARTY_MLIR_EDGE_IREE_COMPILER_UTILS_DISPATCHUTILS_H_ + +#include + +#include "third_party/llvm/llvm/include/llvm/ADT/ArrayRef.h" +#include "third_party/llvm/llvm/projects/google_mlir/include/mlir/IR/Builders.h" +#include "third_party/llvm/llvm/projects/google_mlir/include/mlir/IR/Function.h" +#include "third_party/llvm/llvm/projects/google_mlir/include/mlir/IR/Operation.h" +#include "third_party/llvm/llvm/projects/google_mlir/include/mlir/IR/StandardTypes.h" +#include "third_party/llvm/llvm/projects/google_mlir/include/mlir/IR/Value.h" +#include "third_party/mlir_edge/iree/compiler/IR/Ops.h" +#include "third_party/mlir_edge/iree/compiler/IR/StructureOps.h" + +namespace mlir { +namespace iree_compiler { + +// Calculates the workload for |op| based on the op type. +Value *calculateWorkload(Operation *op, Value *baseOperand); + +// Returns true if the func is trivially dispatchable, meaning that: +// - it contains a single block +// - it contains a single dispatch region +// - it contains a return op directly returning the dispatch region results +bool isTriviallyDispatchable(FuncOp func); + +// Builds a new iree.dispatch_region with the given |ops|. +// The region will capture all required values and return all values used +// outside of the |ops| provided. The region will be inserted at the location of +// the last operation in the set. +// +// All |ops| must be compatible with the |workload| specified as they will all +// be dispatched with the same workgroup structure. +// TODO(benvanik): ensure we want to insert at end. Maybe front? +LogicalResult buildDispatchRegion(FuncOp func, Block *parentBlock, + Value *workload, ArrayRef ops); + +// Merges multiple dispatch regions within a block into the same region, +// if possible. Operations may be reordered if it's possible to merge more while +// still obeying data dependencies. +LogicalResult mergeBlockDispatchRegions(FuncOp func, Block *parentBlock); + +// Inlines use of the given |value| from outside of a dispatch region to inside +// of it and removes the argument. Supports multiple arguments that reference +// |value| and will clone the entire value tree. +LogicalResult inlineDispatchRegionOperandsUsingValue( + IREE::DispatchRegionOp dispatchRegionOp, Value *value); + +// Creates an iree.multi_arch_executable containing an iree.executable with an +// exported function containing the body region of |op|. Created executables +// will be named for their original function concatenated with |symbolSuffix|. +std::pair createRegionExecutable( + Operation *op, FunctionType functionType, StringRef symbolSuffix); + +// Inserts a conversion of an arbitrary |value| to a memref, possibly by way of +// wrapping in an allocation. +// Returns a new memref containing the value or an alias to |value|. +Value *insertDispatcherStore(Operation *op, Value *value, OpBuilder *builder); + +// Inserts a load from a wrapped memref. +// Returns the value in the original type or an alias to the |value| memref. +Value *insertDispatcherLoad(Operation *op, Value *originalValue, + Value *allocatedValue, OpBuilder *builder); + +// TODO(benvanik): enough information to walk into dispatch region and compute +// shape when not static. +Value *allocateDispatchOutputBuffer(Location loc, MemRefType type, + OpBuilder &builder); + +} // namespace iree_compiler +} // namespace mlir + +#endif // THIRD_PARTY_MLIR_EDGE_IREE_COMPILER_UTILS_DISPATCHUTILS_H_ diff --git a/compiler/Utils/Macros.h b/compiler/Utils/Macros.h new file mode 100644 index 000000000000..615fdd43fd9d --- /dev/null +++ b/compiler/Utils/Macros.h @@ -0,0 +1,21 @@ +// Copyright 2019 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#ifndef THIRD_PARTY_MLIR_EDGE_IREE_COMPILER_UTILS_HELPER_MACROS_H_ +#define THIRD_PARTY_MLIR_EDGE_IREE_COMPILER_UTILS_HELPER_MACROS_H_ + +#define RETURN_IF_FAILURE(expr) \ + if (failed(expr)) return failure(); + +#endif // THIRD_PARTY_MLIR_EDGE_IREE_COMPILER_UTILS_HELPER_MACROS_H_ diff --git a/compiler/Utils/MemRefUtils.cpp b/compiler/Utils/MemRefUtils.cpp new file mode 100644 index 000000000000..5a90da102633 --- /dev/null +++ b/compiler/Utils/MemRefUtils.cpp @@ -0,0 +1,178 @@ +// Copyright 2019 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "third_party/mlir_edge/iree/compiler/Utils/MemRefUtils.h" + +#include + +#include "third_party/llvm/llvm/include/llvm/ADT/SmallVector.h" +#include "third_party/llvm/llvm/include/llvm/Support/ErrorHandling.h" +#include "third_party/llvm/llvm/projects/google_mlir/include/mlir/Dialect/StandardOps/Ops.h" +#include "third_party/llvm/llvm/projects/google_mlir/include/mlir/IR/Builders.h" +#include "third_party/llvm/llvm/projects/google_mlir/include/mlir/IR/StandardTypes.h" +#include "third_party/mlir_edge/iree/compiler/IR/Ops.h" + +namespace mlir { +namespace iree_compiler { + +Type legalizeType(Type type) { + if (type.isIndex()) { + return IntegerType::get(kIndexBitWidth, type.getContext()); + } else if (type.isInteger(1)) { + return IntegerType::get(kBoolBitWidth, type.getContext()); + } else if (auto memRefType = type.dyn_cast()) { + return MemRefType::get(memRefType.getShape(), + legalizeType(memRefType.getElementType())); + } else if (auto functionType = type.dyn_cast()) { + llvm::SmallVector inputs; + for (const auto &oldType : functionType.getInputs()) { + inputs.push_back(legalizeType(oldType)); + } + llvm::SmallVector results; + for (const auto &oldType : functionType.getResults()) { + results.push_back(legalizeType(oldType)); + } + return FunctionType::get(inputs, results, type.getContext()); + } + return type; +} + +MemRefType legalizeMemRefType(MemRefType type) { + return MemRefType::get(type.getShape(), legalizeType(type.getElementType()), + type.getAffineMaps(), type.getMemorySpace()); +} + +MemRefType convertTypeToMemRef(Type type) { + if (type.isIntOrIndexOrFloat()) { + return MemRefType::get({}, type, {}, 0); + } else if (auto tensorType = type.dyn_cast()) { + return MemRefType::get(tensorType.getShape(), tensorType.getElementType()); + } else if (auto memRefType = type.dyn_cast()) { + return MemRefType::get(memRefType.getShape(), memRefType.getElementType()); + } else { + llvm_unreachable("Unconvertable type"); + } +} + +Type MemRefTypeConverter::convertType(Type type) { + return convertTypeToMemRef(type); +} + +int64_t getElementCount(const MemRefType &type) { + if (!type.hasStaticShape()) return -1; + int64_t count = 1; + for (int64_t dim : type.getShape()) { + count *= dim; + } + return count; +} + +Value *resolveMemRefSourceValue(Value *memRef, Operation *useOp, + llvm::ArrayRef indices) { + // TODO(benvanik): implement resolveMemRefSourceValue + return nullptr; +} + +Value *resolveValueToSourceMemRef(Value *value, Operation *useOp) { + // TODO(benvanik): implement this for real; this is naive but enough for our + // simple load patterns. + auto *defInstr = value->getDefiningOp(); + if (auto loadOp = dyn_cast_or_null(defInstr)) { + // TODO(benvanik): support views. + return loadOp.getMemRef(); + } + return nullptr; +} + +Type getTensorType(Value *value, OpBuilder &builder) { + Type resultType = value->getType(); + if (resultType.isa()) { + return resultType; + } else if (auto tensorType = resultType.dyn_cast()) { + return builder.getTensorType(tensorType.getShape(), + tensorType.getElementType()); + } + + return builder.getTensorType({}, resultType); +} + +Type getMemRefType(Value *value, OpBuilder &builder) { + Type resultType = value->getType(); + if (resultType.isa()) { + return resultType; + } else if (auto tensorType = resultType.dyn_cast()) { + return builder.getMemRefType(tensorType.getShape(), + tensorType.getElementType()); + } + return builder.getMemRefType({}, resultType); +} + +Value *wrapAsTensor(Value *value, Operation *srcOp, OpBuilder &builder) { + if (srcOp->getResult(0)->getType().isa()) { + if (isa_and_nonnull(value->getDefiningOp())) { + return value->getDefiningOp()->getOperand(0); + } + auto newOp = builder.create( + srcOp->getLoc(), getTensorType(value, builder), value); + value = newOp.getResult(); + } + return value; +} + +Value *wrapAsMemRef(Value *value, Operation *srcOp, OpBuilder &builder) { + if (value->getType().isa()) { + if (isa_and_nonnull(value->getDefiningOp())) { + return value->getDefiningOp()->getOperand(0); + } + auto newOp = builder.create( + srcOp->getLoc(), getMemRefType(value, builder), value); + value = newOp.getResult(); + } + return value; +} + +Value *loadAccessValue(Location location, Value *operand, OpBuilder &builder) { + if (operand->getType().isa() || + operand->getType().isa()) { + return operand; + } + + auto memRefType = builder.getMemRefType({}, operand->getType()); + if (auto loadOp = dyn_cast_or_null(operand->getDefiningOp())) { + // TODO(benvanik): handle creating views. + if (loadOp.getMemRefType() == memRefType) { + return loadOp.getMemRef(); + } + } + + auto allocOp = builder.create(location, memRefType); + builder.create(location, operand, allocOp.getResult(), + ArrayRef{}); + return allocOp.getResult(); +} + +Value *loadResultValue(Location location, const Type &originalType, + Value *result, OpBuilder &builder) { + if (originalType.isa()) { + return result; + } else if (auto tensorType = originalType.dyn_cast()) { + return result; + } + + auto loadOp = builder.create(location, result, ArrayRef{}); + return loadOp.getResult(); +} + +} // namespace iree_compiler +} // namespace mlir diff --git a/compiler/Utils/MemRefUtils.h b/compiler/Utils/MemRefUtils.h new file mode 100644 index 000000000000..8026252bb9b8 --- /dev/null +++ b/compiler/Utils/MemRefUtils.h @@ -0,0 +1,85 @@ +// Copyright 2019 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#ifndef THIRD_PARTY_MLIR_EDGE_IREE_COMPILER_UTILS_MEMREFUTILS_H_ +#define THIRD_PARTY_MLIR_EDGE_IREE_COMPILER_UTILS_MEMREFUTILS_H_ + +#include "third_party/llvm/llvm/include/llvm/ADT/ArrayRef.h" +#include "third_party/llvm/llvm/projects/google_mlir/include/mlir/IR/Builders.h" +#include "third_party/llvm/llvm/projects/google_mlir/include/mlir/IR/Operation.h" +#include "third_party/llvm/llvm/projects/google_mlir/include/mlir/IR/StandardTypes.h" +#include "third_party/llvm/llvm/projects/google_mlir/include/mlir/IR/Value.h" +#include "third_party/llvm/llvm/projects/google_mlir/include/mlir/Transforms/DialectConversion.h" + +namespace mlir { +namespace iree_compiler { + +static const int kBoolBitWidth = 8; +static const int kIndexBitWidth = 32; + +// Converts types to MemRefs using convertTypeToMemRef. +class MemRefTypeConverter : public TypeConverter { + public: + explicit MemRefTypeConverter(MLIRContext *context) {} + Type convertType(Type type) override; +}; + +Type legalizeType(Type type); + +MemRefType legalizeMemRefType(MemRefType type); + +// Converts a type (scalar, tensor, etc) to a MemRef-based type. +MemRefType convertTypeToMemRef(Type type); + +// Returns the number of elements within a MemRef, or -1 if the shape is +// dynamic and cannot be known at compile time. +int64_t getElementCount(const MemRefType &type); + +// Attempts to resolve the use of a MemRef back to the value stored into it. +// Returns either the value stored into the given index or nullptr if the value +// is unavailable (or possibly unknown). +Value *resolveMemRefSourceValue(Value *memRef, Operation *useOp, + llvm::ArrayRef indices = {}); + +// Attempts to resolve the use of a value back to the MemRef it was loaded from. +// Returns either a MemRef view containing the value or nullptr if the value was +// not loaded from a MemRef (or is possibly unknown). +Value *resolveValueToSourceMemRef(Value *value, Operation *useOp); + +// Returns an equivalent TensorType for a MemRef value or returns the values +// current type. +Type getTensorType(Value *value, OpBuilder &builder); + +// Returns an equivalent MemRefType for a Tensor value or returns the values +// current type. +Type getMemRefType(Value *value, OpBuilder &builder); + +// Wraps a memref with a MemRefToTensorOp, returning the resulting Tensor value. +Value *wrapAsTensor(Value *value, Operation *srcOp, OpBuilder &builder); + +// Wraps a tensor with a TensorToMemrefOp, returning the resulting MemRef value. +Value *wrapAsMemRef(Value *value, Operation *srcOp, OpBuilder &builder); + +// For non-{Tensor,MemRef} fetches either the associated MemRef if a LoadOp, +// otherwise append a Alloc and StoreOp. +Value *loadAccessValue(Location location, Value *operand, OpBuilder &builder); + +// Adds a LoadOp on a non-{Tensor,MemRef} type that returns the stored value. +Value *loadResultValue(Location location, const Type &originalType, + Value *result, OpBuilder &builder); + +} // namespace iree_compiler +} // namespace mlir + +#endif // THIRD_PARTY_MLIR_EDGE_IREE_COMPILER_UTILS_MEMREFUTILS_H_ diff --git a/compiler/Utils/ModuleUtils.cpp b/compiler/Utils/ModuleUtils.cpp new file mode 100644 index 000000000000..d6f672a35a95 --- /dev/null +++ b/compiler/Utils/ModuleUtils.cpp @@ -0,0 +1,98 @@ +// Copyright 2019 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "third_party/mlir_edge/iree/compiler/Utils/ModuleUtils.h" + +#include "third_party/llvm/llvm/include/llvm/ADT/SetVector.h" +#include "third_party/llvm/llvm/projects/google_mlir/include/mlir/IR/Function.h" + +namespace mlir { +namespace iree_compiler { + +namespace { + +// Finds a list of functions with the given |attrName| and adds them to |funcs|. +void findFunctionsWithAttr(ModuleOp module, const char *attrName, + llvm::SetVector &funcs) { + for (auto func : module.getOps()) { + if (func.getAttr(attrName)) { + funcs.insert(func); + } + } +} + +// Inserts functions reachable directly from |func| to |usedFuncs|. +void insertUsedFunctions(ModuleOp module, FuncOp func, + DenseSet *usedFuncs, + std::vector *toSearch) { + auto onCalledFunction = [&](StringRef calleeName) { + auto calleeFunc = module.lookupSymbol(calleeName); + if (usedFuncs->insert(calleeFunc).second) { + // New function found! Add to queue for searching. + toSearch->push_back(calleeFunc); + } + }; + for (auto &block : func) { + for (auto &op : block) { + // TODO(benvanik): replace with iree_hl.call check. + if (auto calleeAttr = op.getAttr("callee")) { + onCalledFunction(calleeAttr.cast().getValue()); + } + } + } +} + +// Returns a set containing the names of all functions used by the given +// |rootFuncs| list. +DenseSet findUsedFunctions(ModuleOp module, + ArrayRef rootFuncs) { + // Breadth-first search. + DenseSet usedFuncs; + usedFuncs.insert(rootFuncs.begin(), rootFuncs.end()); + std::vector toSearch = {rootFuncs.begin(), rootFuncs.end()}; + while (!toSearch.empty()) { + auto func = toSearch.back(); + toSearch.pop_back(); + insertUsedFunctions(module, func, &usedFuncs, &toSearch); + } + return usedFuncs; +} + +} // namespace + +void dropUnusedFunctions(ModuleOp module, ArrayRef keepAttrs) { + // Find all of the exported functions we'll treat as roots. + llvm::SetVector rootFuncs; + for (auto keepAttr : keepAttrs) { + findFunctionsWithAttr(module, keepAttr, rootFuncs); + } + + // Find the full set of all used functions reachable from the given rootFuncs. + // This set will contain the rootFuncs. + auto usedFuncs = findUsedFunctions(module, rootFuncs.getArrayRef()); + + // Drop all unused functions. + std::vector deadFuncs; + for (auto func : module.getOps()) { + if (!llvm::is_contained(usedFuncs, func)) { + deadFuncs.push_back(func); + } + } + for (auto func : deadFuncs) { + func.erase(); + } +} + +} // namespace iree_compiler +} // namespace mlir diff --git a/compiler/Utils/ModuleUtils.h b/compiler/Utils/ModuleUtils.h new file mode 100644 index 000000000000..086091b5f890 --- /dev/null +++ b/compiler/Utils/ModuleUtils.h @@ -0,0 +1,30 @@ +// Copyright 2019 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#ifndef THIRD_PARTY_MLIR_EDGE_IREE_COMPILER_UTILS_MODULEUTILS_H_ +#define THIRD_PARTY_MLIR_EDGE_IREE_COMPILER_UTILS_MODULEUTILS_H_ + +#include "third_party/llvm/llvm/projects/google_mlir/include/mlir/IR/Module.h" + +namespace mlir { +namespace iree_compiler { + +// Drops all functions unreachable from functions with at least one of the +// specified |keepAttrs| on them. +void dropUnusedFunctions(ModuleOp module, ArrayRef keepAttrs); + +} // namespace iree_compiler +} // namespace mlir + +#endif // THIRD_PARTY_MLIR_EDGE_IREE_COMPILER_UTILS_MODULEUTILS_H_ diff --git a/compiler/Utils/OpUtils.cpp b/compiler/Utils/OpUtils.cpp new file mode 100644 index 000000000000..3efcc108805d --- /dev/null +++ b/compiler/Utils/OpUtils.cpp @@ -0,0 +1,44 @@ +// Copyright 2019 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "third_party/mlir_edge/iree/compiler/Utils/OpUtils.h" + +namespace mlir { +namespace iree_compiler { + +void removeDeadOperations(llvm::SetVector &deadOperations) { + while (!deadOperations.empty()) { + auto *op = deadOperations.front(); + deadOperations.erase(deadOperations.begin()); + for (auto *operand : op->getOperands()) { + // TODO(benvanik): add check for op side effects. + if (operand->hasOneUse()) { + deadOperations.insert(operand->getDefiningOp()); + } + } + op->erase(); + } +} + +void replaceSubsequentUses(Operation *userOp, Value *oldValue, + Value *newValue) { + for (auto &use : oldValue->getUses()) { + if (userOp->isBeforeInBlock(use.getOwner())) { + use.set(newValue); + } + } +} + +} // namespace iree_compiler +} // namespace mlir diff --git a/compiler/Utils/OpUtils.h b/compiler/Utils/OpUtils.h new file mode 100644 index 000000000000..7be27201bb2e --- /dev/null +++ b/compiler/Utils/OpUtils.h @@ -0,0 +1,40 @@ +// Copyright 2019 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#ifndef THIRD_PARTY_MLIR_EDGE_IREE_COMPILER_UTILS_OPUTILS_H_ +#define THIRD_PARTY_MLIR_EDGE_IREE_COMPILER_UTILS_OPUTILS_H_ + +#include "third_party/llvm/llvm/include/llvm/ADT/None.h" +#include "third_party/llvm/llvm/include/llvm/ADT/Optional.h" +#include "third_party/llvm/llvm/include/llvm/ADT/SetVector.h" +#include "third_party/llvm/llvm/projects/google_mlir/include/mlir/IR/Builders.h" +#include "third_party/llvm/llvm/projects/google_mlir/include/mlir/IR/Operation.h" +#include "third_party/llvm/llvm/projects/google_mlir/include/mlir/IR/StandardTypes.h" +#include "third_party/llvm/llvm/projects/google_mlir/include/mlir/IR/Value.h" + +namespace mlir { +namespace iree_compiler { + +// Recursively removes the given operations and all of their inputs that become +// unused. +void removeDeadOperations(llvm::SetVector &deadOperations); + +// Replaces all uses of |oldValue| with |newValue| that are after |userOp| +// within the same block. +void replaceSubsequentUses(Operation *userOp, Value *oldValue, Value *newValue); + +} // namespace iree_compiler +} // namespace mlir + +#endif // THIRD_PARTY_MLIR_EDGE_IREE_COMPILER_UTILS_OPUTILS_H_ diff --git a/compiler/Utils/TranslationUtils.cpp b/compiler/Utils/TranslationUtils.cpp new file mode 100644 index 000000000000..e8f72d078a0a --- /dev/null +++ b/compiler/Utils/TranslationUtils.cpp @@ -0,0 +1,139 @@ +// Copyright 2019 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "third_party/mlir_edge/iree/compiler/Utils/TranslationUtils.h" + +#include "third_party/llvm/llvm/include/llvm/Support/Debug.h" +#include "third_party/llvm/llvm/include/llvm/Support/ErrorHandling.h" +#include "third_party/llvm/llvm/projects/google_mlir/include/mlir/Pass/Pass.h" +#include "third_party/llvm/llvm/projects/google_mlir/include/mlir/Support/LogicalResult.h" + +namespace mlir { +namespace iree_compiler { + +namespace { + +// Returns the static registry of translator names to translation functions. +llvm::StringMap + &getMutableExecutableTranslationRegistry() { + static llvm::StringMap registry; + return registry; +} + +// Returns true if the given |value| matches |pattern| (normal * and ? rules). +bool matchPattern(StringRef value, StringRef pattern) { + size_t nextCharIndex = pattern.find_first_of("*?"); + if (nextCharIndex == std::string::npos) { + return value == pattern; + } else if (nextCharIndex > 0) { + if (value.substr(0, nextCharIndex) != pattern.substr(0, nextCharIndex)) { + return false; + } + value = value.substr(nextCharIndex); + pattern = pattern.substr(nextCharIndex); + } + char patternChar = pattern[0]; + if (value.empty() && pattern.empty()) { + return true; + } else if (patternChar == '*' && pattern.size() > 1 && value.empty()) { + return false; + } else if (patternChar == '*' && pattern.size() == 1) { + return true; + } else if (patternChar == '?' || value[0] == patternChar) { + return matchPattern(value.substr(1), pattern.substr(1)); + } else if (patternChar == '*') { + return matchPattern(value, pattern.substr(1)) || + matchPattern(value.substr(1), pattern); + } + return false; +} + +// Force enables IR printing on the |passManager|. +void enableIRPrinting(PassManager *passManager) { + auto notVerifier = [](Pass *pass) { + return pass->getName() != "FunctionVerifier" && + pass->getName() != "ModuleVerifier"; + }; + bool printModuleScope = false; + passManager->enableIRPrinting(/*shouldPrintBeforePass=*/{}, + /*shouldPrintAfterPass=*/notVerifier, + printModuleScope, llvm::dbgs()); + passManager->disableMultithreading(); +} + +} // namespace + +ExecutableTranslationRegistration::ExecutableTranslationRegistration( + llvm::StringRef name, const TranslateExecutableFn &fn) { + auto ®istry = getMutableExecutableTranslationRegistry(); + if (registry.find(name) != registry.end()) { + llvm::report_fatal_error( + "Attempting to overwrite an existing translation function"); + } + assert(fn && "Attempting to register an empty translation function"); + registry[name] = fn; +} + +const llvm::StringMap + &getExecutableTranslationRegistry() { + return getMutableExecutableTranslationRegistry(); +} + +std::vector matchExecutableTranslationBackendNames( + llvm::StringRef pattern) { + std::vector matches; + for (auto &entry : getExecutableTranslationRegistry()) { + if (matchPattern(entry.getKey(), pattern)) { + matches.push_back(entry.getKey().str()); + } + } + return matches; +} + +std::unique_ptr createPassManager( + MLIRContext *ctx, const TranslationOptions &translationOptions) { + std::unique_ptr passManager(new PassManager(ctx)); + + // Enable IR printing/timing/etc from command line options. + registerPassManagerCLOptions(); + applyPassManagerCLOptions(*passManager); + + // Override with programmatic options. + if (translationOptions.print_mlir) { + enableIRPrinting(passManager.get()); + } + + return passManager; +} + +LogicalResult runPassPipeline(const TranslationOptions &translationOptions, + PassManager *passManager, ModuleOp module) { + if (translationOptions.print_mlir) { + module.dump(); + } + + // Run on the module. + if (failed(passManager->run(module))) { + return failure(); + } + + if (translationOptions.print_mlir) { + module.dump(); + } + + return success(); +} + +} // namespace iree_compiler +} // namespace mlir diff --git a/compiler/Utils/TranslationUtils.h b/compiler/Utils/TranslationUtils.h new file mode 100644 index 000000000000..f68d1846ebca --- /dev/null +++ b/compiler/Utils/TranslationUtils.h @@ -0,0 +1,109 @@ +// Copyright 2019 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#ifndef THIRD_PARTY_MLIR_EDGE_IREE_COMPILER_UTILS_TRANSLATIONUTILS_H_ +#define THIRD_PARTY_MLIR_EDGE_IREE_COMPILER_UTILS_TRANSLATIONUTILS_H_ + +#include +#include + +#include "third_party/llvm/llvm/include/llvm/ADT/StringMap.h" +#include "third_party/llvm/llvm/include/llvm/ADT/StringRef.h" +#include "third_party/llvm/llvm/projects/google_mlir/include/mlir/IR/Builders.h" +#include "third_party/llvm/llvm/projects/google_mlir/include/mlir/IR/Module.h" +#include "third_party/llvm/llvm/projects/google_mlir/include/mlir/IR/Operation.h" +#include "third_party/llvm/llvm/projects/google_mlir/include/mlir/IR/Value.h" +#include "third_party/llvm/llvm/projects/google_mlir/include/mlir/Pass/PassManager.h" +#include "third_party/mlir_edge/iree/compiler/IR/StructureOps.h" +#include "third_party/mlir_edge/iree/schemas/executable_def_generated.h" + +namespace mlir { +namespace iree_compiler { + +// Common translation options for diagnostics and debugging. +struct TranslationOptions { + // Enables MLIR IR printing during translation. + // This can be specified via the -print-ir-before-all and -print-ir-after-all + // command line flags or overridden programmatically via this flag. + bool print_mlir = false; + + void CopyFrom(const TranslationOptions &other) { + print_mlir = other.print_mlir; + } +}; + +// Options for iree.module translation for diagnostics and debugging. +struct ModuleTranslationOptions : public TranslationOptions { + // Defines which backend translators will be used to translate executables. + // If empty then all linked in translators will be used. + // TODO(benvanik): extend to allow specifying entire config blobs via mlir. + std::vector target_backends; +}; + +// Options for iree.executable translation for diagnostics and debugging. +// Target configuration is sourced from the iree.target_config op within the +// iree.executable. +struct ExecutableTranslationOptions : public TranslationOptions {}; + +// Results of a translation operation. +// May contain zero or more executable defs depending on translation options, +// defined target configs, and support. +struct ExecutableTranslationResult { + std::vector> executable_defs; +}; + +// Registered function that given a set of |executableOps| containing one +// or more iree.executables will produce zero or more serialized executables. +// +// Each iree.executable provided contains one iree.executable_target_config with +// backend-specific translation information. The translator can decide whether +// to translate each independently, group them together, etc. +// +// The provided |executableOps| can be mutated by the callee and will be +// preserved for debugging after translation. If any executable in +// |executableOps| is not used by the translator then it should be erased. +using TranslateExecutableFn = + std::function( + ArrayRef executableOps, + ExecutableTranslationOptions options)>; + +// Registers an executable translation function. +struct ExecutableTranslationRegistration { + ExecutableTranslationRegistration(llvm::StringRef name, + const TranslateExecutableFn &fn); +}; + +// Returns a read-only reference to the translator registry. +const llvm::StringMap + &getExecutableTranslationRegistry(); + +// Returns executable translation backend names matching the given pattern. +// This accepts wildcards for any delimited value. For example, 'foo-*-bar' will +// match 'foo-123-bar' and 'foo-456-bar' and 'foo-10?' will match 'foo-101' and +// 'foo-102'. +std::vector matchExecutableTranslationBackendNames( + llvm::StringRef pattern); + +// Creates a new pass manager initialized with the given options. +std::unique_ptr createPassManager( + MLIRContext *ctx, const TranslationOptions &translationOptions); + +// Runs an initialized set of passes on the given module. +LogicalResult runPassPipeline(const TranslationOptions &translationOptions, + PassManager *passManager, ModuleOp module); + +} // namespace iree_compiler +} // namespace mlir + +#endif // THIRD_PARTY_MLIR_EDGE_IREE_COMPILER_UTILS_TRANSLATIONUTILS_H_ diff --git a/hal/allocator.cc b/hal/allocator.cc new file mode 100644 index 000000000000..4f669007c190 --- /dev/null +++ b/hal/allocator.cc @@ -0,0 +1,77 @@ +// Copyright 2019 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "third_party/mlir_edge/iree/hal/allocator.h" + +#include +#include +#include +#include + +#include "third_party/absl/types/source_location.h" +#include "third_party/mlir_edge/iree/base/status.h" +#include "third_party/mlir_edge/iree/base/tracing.h" + +namespace iree { +namespace hal { + +bool Allocator::CanUseBuffer(Buffer* buffer, + BufferUsageBitfield intended_usage) const { + return CanUseBufferLike(buffer->allocator(), buffer->memory_type(), + buffer->usage(), intended_usage); +} + +StatusOr> Allocator::AllocateConstant( + BufferUsageBitfield buffer_usage, ref_ptr source_buffer) { + if (AnyBitSet(source_buffer->usage() & BufferUsage::kConstant) && + CanUseBuffer(source_buffer.get(), buffer_usage)) { + // Buffer can be used directly by the device. + return source_buffer; + } + + IREE_TRACE_SCOPE0("Allocator::AllocateConstant"); + + // We need to map so we can copy into it. + buffer_usage |= BufferUsage::kMapping; + // It will be constant after we write it. + buffer_usage |= BufferUsage::kConstant; + + MemoryTypeBitfield memory_type = + MemoryType::kDeviceLocal | MemoryType::kHostVisible; + ASSIGN_OR_RETURN(auto device_buffer, Allocate(memory_type, buffer_usage, + source_buffer->byte_length())); + ASSIGN_OR_RETURN(auto source_mapping, + source_buffer->MapMemory(MemoryAccess::kRead)); + RETURN_IF_ERROR(device_buffer->WriteData(0, source_mapping.data(), + source_mapping.byte_length())); + return device_buffer; +} + +StatusOr> Allocator::Wrap(MemoryTypeBitfield memory_type, + BufferUsageBitfield buffer_usage, + const void* data, + size_t data_length) { + return WrapMutable(memory_type, MemoryAccess::kRead, buffer_usage, + const_cast(data), data_length); +} + +StatusOr> Allocator::WrapMutable( + MemoryTypeBitfield memory_type, MemoryAccessBitfield allowed_access, + BufferUsageBitfield buffer_usage, void* data, size_t data_length) { + return UnimplementedErrorBuilder(ABSL_LOC) + << "Allocator does not support wrapping host memory"; +} + +} // namespace hal +} // namespace iree diff --git a/hal/allocator.h b/hal/allocator.h new file mode 100644 index 000000000000..4fe81c3e8162 --- /dev/null +++ b/hal/allocator.h @@ -0,0 +1,138 @@ +// Copyright 2019 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#ifndef THIRD_PARTY_MLIR_EDGE_IREE_HAL_ALLOCATOR_H_ +#define THIRD_PARTY_MLIR_EDGE_IREE_HAL_ALLOCATOR_H_ + +#include +#include + +#include "third_party/absl/types/span.h" +#include "third_party/mlir_edge/iree/base/status.h" +#include "third_party/mlir_edge/iree/hal/buffer.h" + +namespace iree { +namespace hal { + +// Allocates buffers for a particular device memory space. +// +// Buffers allocated are only guaranteed to work with the driver that the +// allocator services. Any attempt to use buffers on drivers they were not +// allocated from must first be checked with CanUseBuffer. +// +// Thread-safe. +class Allocator { + public: + virtual ~Allocator() = default; + + // Returns true if the device can use the given buffer for the provided usage. + // For buffers allocated from this allocator it's expected that the result + // will always be true. For buffers that originate from another allocator + // there may be limited support for cross-device usage. + // + // Returning false indicates that the buffer must be transferred externally + // into a buffer compatible with the device this allocator services. + bool CanUseBuffer(Buffer* buffer, BufferUsageBitfield intended_usage) const; + virtual bool CanUseBufferLike(Allocator* source_allocator, + MemoryTypeBitfield memory_type, + BufferUsageBitfield buffer_usage, + BufferUsageBitfield intended_usage) const = 0; + + // Returns true if the allocator can allocate a buffer with the given + // attributes. + virtual bool CanAllocate(MemoryTypeBitfield memory_type, + BufferUsageBitfield buffer_usage, + size_t allocation_size) const = 0; + + // Adjusts allocation parameters to be compatible with the allocator. + // Certain allocators may require particular memory types to function. By + // adjusting the parameters prior to allocation callers can be sure they are + // able to successfully Allocate a buffer later on with the same parameters. + virtual Status MakeCompatible(MemoryTypeBitfield* memory_type, + BufferUsageBitfield* buffer_usage) const { + return OkStatus(); + } + + // Allocates a buffer from the allocator. + // Fails if the memory type requested for the given usage cannot be serviced. + // Callers can use CanAllocate to decide their memory use strategy. + // + // The memory type of the buffer returned may differ from the requested value + // if the device can provide more functionality; for example, if requesting + // MemoryType::kHostVisible but the memory is really host cached you may get + // a buffer back with MemoryType::kHostVisible | MemoryType::kHostCached. The + // only requirement is that the buffer satisfy the required bits. + virtual StatusOr> Allocate(MemoryTypeBitfield memory_type, + BufferUsageBitfield buffer_usage, + size_t allocation_size) = 0; + + // Allocates a buffer from the allocator for use as a constant value. + // The provided |source_buffer| may be returned if the device can use it + // directly and otherwise will be copied. + virtual StatusOr> AllocateConstant( + BufferUsageBitfield buffer_usage, ref_ptr source_buffer); + + // Wraps an existing host heap allocation in a buffer. + // Ownership of the host allocation remains with the caller and the memory + // must remain valid for so long as the Buffer may be in use. + // Will have MemoryType::kHostLocal in most cases and may not be usable + // by the device. + // + // The inference optimizer makes assumptions about buffer aliasing based on + // Buffer instances and because of this wrapping the same host buffer in + // multiple Buffers will create potential memory aliasing issues that can be + // difficult to track down. There's no checking as to whether a host buffer + // has already been wrapped so it's best for callers to ensure this is never + // possible (the simplest way being to never use Wrap and always just allocate + // new Buffers). + // + // Fails if the allocator cannot access host memory in this way. + StatusOr> Wrap(MemoryTypeBitfield memory_type, + BufferUsageBitfield buffer_usage, + const void* data, size_t data_length); + virtual StatusOr> WrapMutable( + MemoryTypeBitfield memory_type, MemoryAccessBitfield allowed_access, + BufferUsageBitfield buffer_usage, void* data, size_t data_length); + template + StatusOr> Wrap(MemoryTypeBitfield memory_type, + BufferUsageBitfield buffer_usage, + absl::Span data); + template + StatusOr> WrapMutable(MemoryTypeBitfield memory_type, + MemoryAccessBitfield allowed_access, + BufferUsageBitfield buffer_usage, + absl::Span data); +}; + +// Inline functions and template definitions follow: + +template +StatusOr> Allocator::Wrap(MemoryTypeBitfield memory_type, + BufferUsageBitfield buffer_usage, + absl::Span data) { + return Wrap(memory_type, buffer_usage, data.data(), data.size() * sizeof(T)); +} + +template +StatusOr> Allocator::WrapMutable( + MemoryTypeBitfield memory_type, MemoryAccessBitfield allowed_access, + BufferUsageBitfield buffer_usage, absl::Span data) { + return WrapMutable(memory_type, allowed_access, buffer_usage, data.data(), + data.size() * sizeof(T)); +} + +} // namespace hal +} // namespace iree + +#endif // THIRD_PARTY_MLIR_EDGE_IREE_HAL_ALLOCATOR_H_ diff --git a/hal/buffer.cc b/hal/buffer.cc new file mode 100644 index 000000000000..3d7b351ad86c --- /dev/null +++ b/hal/buffer.cc @@ -0,0 +1,549 @@ +// Copyright 2019 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "third_party/mlir_edge/iree/hal/buffer.h" + +#include +#include +#include +#include +#include + +#include "third_party/absl/strings/str_cat.h" +#include "third_party/absl/strings/str_join.h" +#include "third_party/absl/types/variant.h" +#include "third_party/mlir_edge/iree/base/status.h" + +namespace iree { +namespace hal { + +#if HAS_IREE_BUFFER_DEBUG_NAME +namespace { +// Used for diagnostic purposes only as a default buffer name. +std::atomic next_buffer_id_{0}; +} // namespace +#endif // HAS_IREE_BUFFER_DEBUG_NAME + +std::string MemoryTypeString(MemoryTypeBitfield memory_type) { + return FormatBitfieldValue(memory_type, + { + // Combined: + {MemoryType::kHostLocal, "kHostLocal"}, + {MemoryType::kDeviceLocal, "kDeviceLocal"}, + // Separate: + {MemoryType::kTransient, "kTransient"}, + {MemoryType::kHostVisible, "kHostVisible"}, + {MemoryType::kHostCoherent, "kHostCoherent"}, + {MemoryType::kHostCached, "kHostCached"}, + {MemoryType::kDeviceVisible, "kDeviceVisible"}, + }); +} + +std::string MemoryAccessString(MemoryAccessBitfield memory_access) { + return FormatBitfieldValue(memory_access, + { + // Combined: + {MemoryAccess::kAll, "kAll"}, + {MemoryAccess::kDiscardWrite, "kDiscardWrite"}, + // Separate: + {MemoryAccess::kRead, "kRead"}, + {MemoryAccess::kWrite, "kWrite"}, + {MemoryAccess::kDiscard, "kDiscard"}, + }); +} + +std::string BufferUsageString(BufferUsageBitfield buffer_usage) { + return FormatBitfieldValue(buffer_usage, + { + // Combined: + {BufferUsage::kAll, "kAll"}, + // Separate: + {BufferUsage::kConstant, "kConstant"}, + {BufferUsage::kTransfer, "kTransfer"}, + {BufferUsage::kMapping, "kMapping"}, + {BufferUsage::kDispatch, "kDispatch"}, + }); +} + +// Special router for buffers that just reference other buffers. +// We keep this out of the base Buffer so that it's a bit easier to track +// delegation. +class SubspanBuffer : public Buffer { + public: + SubspanBuffer(ref_ptr parent_buffer, device_size_t byte_offset, + device_size_t byte_length) + : Buffer(parent_buffer->allocator(), parent_buffer->memory_type(), + parent_buffer->allowed_access(), parent_buffer->usage(), + parent_buffer->allocation_size(), byte_offset, byte_length) { + allocated_buffer_ = parent_buffer.get(); + parent_buffer_ = std::move(parent_buffer); + } + + protected: + Status FillImpl(device_size_t byte_offset, device_size_t byte_length, + const void* pattern, device_size_t pattern_length) override { + return parent_buffer_->FillImpl(byte_offset, byte_length, pattern, + pattern_length); + } + + Status ReadDataImpl(device_size_t source_offset, void* data, + device_size_t data_length) override { + return parent_buffer_->ReadDataImpl(source_offset, data, data_length); + } + + Status WriteDataImpl(device_size_t target_offset, const void* data, + device_size_t data_length) override { + return parent_buffer_->WriteDataImpl(target_offset, data, data_length); + } + + Status CopyDataImpl(device_size_t target_offset, Buffer* source_buffer, + device_size_t source_offset, + device_size_t data_length) override { + return parent_buffer_->CopyDataImpl(target_offset, source_buffer, + source_offset, data_length); + } + + Status MapMemoryImpl(MappingMode mapping_mode, + MemoryAccessBitfield memory_access, + device_size_t local_byte_offset, + device_size_t local_byte_length, + void** out_data) override { + return parent_buffer_->MapMemoryImpl(mapping_mode, memory_access, + local_byte_offset, local_byte_length, + out_data); + } + + Status UnmapMemoryImpl(device_size_t local_byte_offset, + device_size_t local_byte_length, void* data) override { + return parent_buffer_->UnmapMemoryImpl(local_byte_offset, local_byte_length, + data); + } + + Status InvalidateMappedMemoryImpl(device_size_t local_byte_offset, + device_size_t local_byte_length) override { + return parent_buffer_->InvalidateMappedMemoryImpl(local_byte_offset, + local_byte_length); + } + + Status FlushMappedMemoryImpl(device_size_t local_byte_offset, + device_size_t local_byte_length) override { + return parent_buffer_->FlushMappedMemoryImpl(local_byte_offset, + local_byte_length); + } +}; + +// static +StatusOr> Buffer::Subspan(const ref_ptr& buffer, + device_size_t byte_offset, + device_size_t byte_length) { + RETURN_IF_ERROR(buffer->CalculateRange(byte_offset, byte_length, &byte_offset, + &byte_length)); + if (byte_offset == 0 && byte_length == buffer->byte_length()) { + // Asking for the same buffer. + return add_ref(buffer); + } + + // To avoid heavy nesting of subspans that just add indirection we go to the + // parent buffer directly. If we wanted better accounting (to track where + // buffers came from) we'd want to avoid this but I'm not sure that's worth + // the super deep indirection that could arise. + if (buffer->allocated_buffer() != buffer.get()) { + CHECK(buffer->parent_buffer_); + return Buffer::Subspan(buffer->parent_buffer_, byte_offset, byte_length); + } else { + return {make_ref(add_ref(buffer), byte_offset, byte_length)}; + } +} + +// static +Buffer::Overlap Buffer::TestOverlap( + Buffer* lhs_buffer, device_size_t lhs_offset, device_size_t lhs_length, + Buffer* rhs_buffer, device_size_t rhs_offset, device_size_t rhs_length) { + if (lhs_buffer->allocated_buffer() != rhs_buffer->allocated_buffer()) { + // Not even the same buffers. + return Overlap::kDisjoint; + } + // Resolve offsets into the underlying allocation. + device_size_t lhs_alloc_offset = lhs_buffer->byte_offset() + lhs_offset; + device_size_t rhs_alloc_offset = rhs_buffer->byte_offset() + rhs_offset; + device_size_t lhs_alloc_length = lhs_length == kWholeBuffer + ? lhs_buffer->byte_length() - lhs_offset + : lhs_length; + device_size_t rhs_alloc_length = rhs_length == kWholeBuffer + ? rhs_buffer->byte_length() - rhs_offset + : rhs_length; + if (!lhs_alloc_length || !rhs_alloc_length) { + return Overlap::kDisjoint; + } + if (lhs_alloc_offset == rhs_alloc_offset && + lhs_alloc_length == rhs_alloc_length) { + return Overlap::kComplete; + } + return lhs_alloc_offset + lhs_alloc_length > rhs_alloc_offset && + rhs_alloc_offset + rhs_alloc_length > lhs_alloc_offset + ? Overlap::kPartial + : Overlap::kDisjoint; +} + +// static +bool Buffer::DoesOverlap(Buffer* lhs_buffer, device_size_t lhs_offset, + device_size_t lhs_length, Buffer* rhs_buffer, + device_size_t rhs_offset, device_size_t rhs_length) { + return TestOverlap(lhs_buffer, lhs_offset, lhs_length, rhs_buffer, rhs_offset, + rhs_length) != Overlap::kDisjoint; +} + +Buffer::Buffer(Allocator* allocator, MemoryTypeBitfield memory_type, + MemoryAccessBitfield allowed_access, BufferUsageBitfield usage, + device_size_t allocation_size, device_size_t byte_offset, + device_size_t byte_length) + : allocated_buffer_(const_cast(this)), + allocator_(allocator), + memory_type_(memory_type), + allowed_access_(allowed_access), + usage_(usage), + allocation_size_(allocation_size), + byte_offset_(byte_offset), + byte_length_(byte_length) { +#if HAS_IREE_BUFFER_DEBUG_NAME + // Default name for logging. + // It'd be nice to defer this until it's required but that would require + // synchronization or something. + const char* debug_name_prefix = ""; + if ((memory_type_ & MemoryType::kHostLocal) == MemoryType::kHostLocal) { + debug_name_prefix = "host_buffer_"; + } else if ((memory_type_ & MemoryType::kDeviceLocal) == + MemoryType::kDeviceLocal) { + // TODO(benvanik): include allocator ID to differentiate devices. + debug_name_prefix = "device_buffer_"; + } + debug_name_ = absl::StrCat(debug_name_prefix, next_buffer_id_++); +#endif // HAS_IREE_BUFFER_DEBUG_NAME +} + +Buffer* Buffer::allocated_buffer() const noexcept { + Buffer* allocated_buffer = allocated_buffer_; + while (allocated_buffer != this && + allocated_buffer != allocated_buffer->allocated_buffer()) { + allocated_buffer = allocated_buffer->allocated_buffer(); + } + return allocated_buffer; +} + +std::string Buffer::DebugString() const { + std::ostringstream stream; + stream << allocated_buffer()->debug_name() << "[" + << (allocation_size() == kWholeBuffer + ? "?" + : std::to_string(allocation_size())) + << "]."; + if (AnyBitSet(memory_type() & MemoryType::kTransient)) stream << "Z"; + if ((memory_type() & MemoryType::kHostLocal) == MemoryType::kHostLocal) { + stream << "h"; + } else { + if (AnyBitSet(memory_type() & MemoryType::kHostVisible)) stream << "v"; + if (AnyBitSet(memory_type() & MemoryType::kHostCoherent)) stream << "x"; + if (AnyBitSet(memory_type() & MemoryType::kHostCached)) stream << "c"; + } + if ((memory_type() & MemoryType::kDeviceLocal) == MemoryType::kDeviceLocal) { + stream << "D"; + } else { + if (AnyBitSet(memory_type() & MemoryType::kDeviceVisible)) stream << "V"; + } + stream << "."; + if (AnyBitSet(usage() & BufferUsage::kConstant)) stream << "c"; + if (AnyBitSet(usage() & BufferUsage::kTransfer)) stream << "t"; + if (AnyBitSet(usage() & BufferUsage::kMapping)) stream << "m"; + if (AnyBitSet(usage() & BufferUsage::kDispatch)) stream << "d"; + if (byte_offset_ || byte_length_ != allocation_size_) { + stream << "(" << byte_offset_ << "-" << (byte_offset_ + byte_length_ - 1) + << ")"; + } + return stream.str(); +} + +std::string Buffer::DebugStringShort() const { + // TODO(benvanik): figure out what's most useful here. Maybe a long variant? + std::ostringstream stream; + stream << allocated_buffer()->debug_name() << "[" + << (allocation_size() == kWholeBuffer + ? "?" + : std::to_string(allocation_size())) + << "]"; + if (byte_offset_ || byte_length_ != allocation_size_) { + stream << "(" << byte_offset_ << "-" << (byte_offset_ + byte_length_ - 1) + << ")"; + } + return stream.str(); +} + +Status Buffer::ValidateCompatibleMemoryType( + MemoryTypeBitfield memory_type) const { + if ((memory_type_ & memory_type) != memory_type) { + // Missing one or more bits. + return PermissionDeniedErrorBuilder(ABSL_LOC) + << "Buffer memory type is not compatible with the requested " + "operation; buffer has " + << MemoryTypeString(memory_type_) << ", operation requires " + << MemoryTypeString(memory_type); + } + return OkStatus(); +} + +Status Buffer::ValidateAccess(MemoryAccessBitfield memory_access) const { + if (!AnyBitSet(memory_access & + (MemoryAccess::kRead | MemoryAccess::kWrite))) { + // No actual access bits defined. + return InvalidArgumentErrorBuilder(ABSL_LOC) + << "Memory access must specify one or more of kRead or kWrite"; + } else if ((allowed_access_ & memory_access) != memory_access) { + // Bits must match exactly. + return PermissionDeniedErrorBuilder(ABSL_LOC) + << "The buffer does not support the requested access type; buffer " + "allows " + << MemoryAccessString(allowed_access_) << ", operation requires " + << MemoryAccessString(memory_access); + } + return OkStatus(); +} + +Status Buffer::ValidateUsage(BufferUsageBitfield usage) const { + if ((usage_ & usage) != usage) { + // Missing one or more bits. + return PermissionDeniedErrorBuilder(ABSL_LOC) + << "Requested usage was not specified when the buffer was " + "allocated; buffer allows " + << BufferUsageString(usage_) << ", operation requires " + << BufferUsageString(usage); + } + return OkStatus(); +} + +Status Buffer::CalculateRange(device_size_t base_offset, + device_size_t max_length, device_size_t offset, + device_size_t length, + device_size_t* out_adjusted_offset, + device_size_t* out_adjusted_length) { + // Check if the start of the range runs off the end of the buffer. + if (offset > max_length) { + *out_adjusted_offset = 0; + if (out_adjusted_length) *out_adjusted_length = 0; + return OutOfRangeErrorBuilder(ABSL_LOC) + << "Attempted to access an address off the end of the valid buffer " + "range (offset=" + << offset << ", length=" << length + << ", buffer byte_length=" << max_length << ")"; + } + + // Handle length as kWholeBuffer by adjusting it (if allowed). + if (length == kWholeBuffer && !out_adjusted_length) { + *out_adjusted_offset = 0; + return InvalidArgumentErrorBuilder(ABSL_LOC) + << "kWholeBuffer may only be used with buffer ranges, not external " + "pointer ranges"; + } + + // Calculate the real ranges adjusted for our region within the allocation. + device_size_t adjusted_offset = base_offset + offset; + device_size_t adjusted_length = + length == kWholeBuffer ? max_length - offset : length; + if (adjusted_length == 0) { + // Fine to have a zero length. + *out_adjusted_offset = adjusted_offset; + if (out_adjusted_length) *out_adjusted_length = adjusted_length; + return OkStatus(); + } + + // Check if the end runs over the allocation. + device_size_t end = offset + adjusted_length - 1; + if (end >= max_length) { + *out_adjusted_offset = 0; + if (out_adjusted_length) *out_adjusted_length = 0; + return OutOfRangeErrorBuilder(ABSL_LOC) + << "Attempted to access an address outside of the valid buffer " + "range (offset=" + << offset << ", adjusted_length=" << adjusted_length + << ", end=" << end << ", buffer byte_length=" << max_length << ")"; + } + + *out_adjusted_offset = adjusted_offset; + if (out_adjusted_length) *out_adjusted_length = adjusted_length; + return OkStatus(); +} + +Status Buffer::CalculateRange(device_size_t offset, device_size_t length, + device_size_t* out_adjusted_offset, + device_size_t* out_adjusted_length) const { + return CalculateRange(byte_offset_, byte_length_, offset, length, + out_adjusted_offset, out_adjusted_length); +} + +Status Buffer::CalculateLocalRange(device_size_t max_length, + device_size_t offset, device_size_t length, + device_size_t* out_adjusted_offset, + device_size_t* out_adjusted_length) { + return CalculateRange(0, max_length, offset, length, out_adjusted_offset, + out_adjusted_length); +} + +Status Buffer::Fill(device_size_t byte_offset, device_size_t byte_length, + const void* pattern, device_size_t pattern_length) { + // If not host visible we'll need to issue command buffers. + RETURN_IF_ERROR(ValidateCompatibleMemoryType(MemoryType::kHostVisible)); + RETURN_IF_ERROR(ValidateAccess(MemoryAccess::kWrite)); + RETURN_IF_ERROR(ValidateUsage(BufferUsage::kMapping)); + RETURN_IF_ERROR( + CalculateRange(byte_offset, byte_length, &byte_offset, &byte_length)); + if (pattern_length != 1 && pattern_length != 2 && pattern_length != 4) { + return InvalidArgumentErrorBuilder(ABSL_LOC) + << "Fill patterns must be 1, 2, or 4 bytes"; + } + if ((byte_offset % pattern_length) != 0 || + (byte_length % pattern_length) != 0) { + return InvalidArgumentErrorBuilder(ABSL_LOC) + << "Attempting to fill a range with " << pattern_length + << " byte values that is not " + "aligned (offset=" + << byte_offset << ", length=" << byte_length << ")"; + } + if (byte_length == 0) { + return OkStatus(); // No-op. + } + const uint32_t kZero = 0; + if (std::memcmp(pattern, &kZero, pattern_length) == 0) { + // We can turn all-zero values into single-byte fills as that can be much + // faster on devices (doing a fill8 vs fill32). + pattern_length = 1; + } + return FillImpl(byte_offset, byte_length, pattern, pattern_length); +} + +Status Buffer::ReadData(device_size_t source_offset, void* data, + device_size_t data_length) { + // If not host visible we'll need to issue command buffers. + RETURN_IF_ERROR(ValidateCompatibleMemoryType(MemoryType::kHostVisible)); + RETURN_IF_ERROR(ValidateAccess(MemoryAccess::kRead)); + RETURN_IF_ERROR(ValidateUsage(BufferUsage::kMapping)); + RETURN_IF_ERROR(CalculateRange(source_offset, data_length, &source_offset)); + if (data_length == 0) { + return OkStatus(); // No-op. + } + return ReadDataImpl(source_offset, data, data_length); +} + +Status Buffer::WriteData(device_size_t target_offset, const void* data, + device_size_t data_length) { + // If not host visible we'll need to issue command buffers. + RETURN_IF_ERROR(ValidateCompatibleMemoryType(MemoryType::kHostVisible)); + RETURN_IF_ERROR(ValidateAccess(MemoryAccess::kWrite)); + RETURN_IF_ERROR(ValidateUsage(BufferUsage::kMapping)); + RETURN_IF_ERROR(CalculateRange(target_offset, data_length, &target_offset)); + if (data_length == 0) { + return OkStatus(); // No-op. + } + return WriteDataImpl(target_offset, data, data_length); +} + +Status Buffer::CopyData(device_size_t target_offset, Buffer* source_buffer, + device_size_t source_offset, + device_size_t data_length) { + RETURN_IF_ERROR(ValidateCompatibleMemoryType(MemoryType::kHostVisible)); + RETURN_IF_ERROR(ValidateAccess(MemoryAccess::kWrite)); + RETURN_IF_ERROR(ValidateUsage(BufferUsage::kMapping)); + RETURN_IF_ERROR( + source_buffer->ValidateCompatibleMemoryType(MemoryType::kHostVisible)); + RETURN_IF_ERROR(source_buffer->ValidateAccess(MemoryAccess::kRead)); + RETURN_IF_ERROR(source_buffer->ValidateUsage(BufferUsage::kMapping)); + + // We need to validate both buffers. + device_size_t source_data_length = data_length; + device_size_t target_data_length = data_length; + device_size_t adjusted_source_offset; + RETURN_IF_ERROR(source_buffer->CalculateRange( + source_offset, source_data_length, &adjusted_source_offset, + &source_data_length)); + RETURN_IF_ERROR(CalculateRange(target_offset, target_data_length, + &target_offset, &target_data_length)); + device_size_t adjusted_data_length; + if (data_length == kWholeBuffer) { + // Whole buffer copy requested - that could mean either, so take the min. + adjusted_data_length = std::min(source_data_length, target_data_length); + } else { + // Specific length requested - validate that we have matching lengths. + CHECK_EQ(source_data_length, target_data_length); + adjusted_data_length = source_data_length; + } + + // Elide zero length copies. + if (adjusted_data_length == 0) { + return OkStatus(); + } + + // Check for overlap. + if (this == source_buffer && + adjusted_source_offset <= target_offset + adjusted_data_length && + target_offset <= adjusted_source_offset + adjusted_data_length) { + return InvalidArgumentErrorBuilder(ABSL_LOC) + << "Source and target ranges overlap within the same buffer"; + } + + return CopyDataImpl(target_offset, source_buffer, source_offset, + adjusted_data_length); +} + +Status Buffer::MapMemory(MappingMode mapping_mode, + MemoryAccessBitfield memory_access, + device_size_t* byte_offset, device_size_t* byte_length, + void** out_data) { + RETURN_IF_ERROR(ValidateCompatibleMemoryType(MemoryType::kHostVisible)); + RETURN_IF_ERROR(ValidateAccess(memory_access)); + RETURN_IF_ERROR(ValidateUsage(BufferUsage::kMapping)); + RETURN_IF_ERROR( + CalculateRange(*byte_offset, *byte_length, byte_offset, byte_length)); + *out_data = nullptr; + return MapMemoryImpl(mapping_mode, memory_access, *byte_offset, *byte_length, + out_data); +} + +Status Buffer::UnmapMemory(device_size_t local_byte_offset, + device_size_t local_byte_length, void* data) { + RETURN_IF_ERROR(ValidateCompatibleMemoryType(MemoryType::kHostVisible)); + RETURN_IF_ERROR(ValidateUsage(BufferUsage::kMapping)); + // NOTE: local_byte_offset/local_byte_length are already adjusted. + return UnmapMemoryImpl(local_byte_offset, local_byte_length, data); +} + +Status Buffer::InvalidateMappedMemory(device_size_t local_byte_offset, + device_size_t local_byte_length) { + RETURN_IF_ERROR(ValidateCompatibleMemoryType(MemoryType::kHostVisible)); + if (AnyBitSet(memory_type_ & MemoryType::kHostCoherent)) { + return PermissionDeniedErrorBuilder(ABSL_LOC) + << "Buffer memory type is coherent and invalidation is not required"; + } + RETURN_IF_ERROR(ValidateUsage(BufferUsage::kMapping)); + // NOTE: local_byte_offset/local_byte_length are already adjusted. + return InvalidateMappedMemoryImpl(local_byte_offset, local_byte_length); +} + +Status Buffer::FlushMappedMemory(device_size_t local_byte_offset, + device_size_t local_byte_length) { + RETURN_IF_ERROR(ValidateCompatibleMemoryType(MemoryType::kHostVisible | + MemoryType::kHostCached)); + RETURN_IF_ERROR(ValidateUsage(BufferUsage::kMapping)); + // NOTE: local_byte_offset/local_byte_length are already adjusted. + return FlushMappedMemoryImpl(local_byte_offset, local_byte_length); +} + +} // namespace hal +} // namespace iree diff --git a/hal/buffer.h b/hal/buffer.h new file mode 100644 index 000000000000..f2d1131fdf91 --- /dev/null +++ b/hal/buffer.h @@ -0,0 +1,903 @@ +// Copyright 2019 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// Allocated memory buffer wrapper type and utilities. +// +// Buffers are the basic unit of memory used by the inference system. They may +// be allocated such that they are accessible from the host (normal C++ code +// running on the main CPU), a particular device (such as an accelerator) or +// family of devices, or from some mix of all of those. +// +// The type of memory a buffer is allocated within has implications on it's +// performance and lifetime. For example if an application attempts to use a +// host-allocated buffer (MemoryType::kHostLocal) on an accelerator with +// discrete memory the accelerator may either be unable to access the memory or +// take a non-trivial performance hit when attempting to do so (involving +// setting up kernel mappings, doing DMA transfers, etc). Likewise, trying to +// access a device-allocated buffer (MemoryType::kDeviceLocal) may incur similar +// overhead or not be possible at all. This may be due to restrictions in the +// memory visibility, address spaces, mixed endianness or pointer widths, +// and other weirdness. +// +// The memory types (defined by a bitfield of MemoryType values) that a +// particular context (host or device) may use vary from device to device and +// must be queried by the application when allocating buffers. It's strongly +// recommended that the most specific memory type be set as possible. For +// example allocating a buffer with MemoryType::kHostCoherent even when it will +// never be used in a way that requires coherency may occupy address space +// reservations or memory mapping that would otherwise not be needed. +// +// As buffers may sometimes not be accessible from the host the base Buffer type +// does not allow for direct void* access and instead buffers must be either +// manipulated using utility functions (such as ReadData or WriteData) or by +// mapping them into a host-accessible address space via MapMemory. Buffer must +// be unmapped before any command may use it. +// +// Buffers may map (roughly) 1:1 with an allocation either from the host heap or +// a device. Buffer::Subspan can be used to reference subspans of buffers like +// absl::Span - though unlike absl::Span the returned Buffer holds a reference +// to the parent buffer. + +#ifndef THIRD_PARTY_MLIR_EDGE_IREE_HAL_BUFFER_H_ +#define THIRD_PARTY_MLIR_EDGE_IREE_HAL_BUFFER_H_ + +#include +#include +#include +#include +#include + +#include "third_party/absl/types/source_location.h" +#include "third_party/absl/types/span.h" +#include "third_party/absl/types/variant.h" +#include "third_party/mlir_edge/iree/base/bitfield.h" +#include "third_party/mlir_edge/iree/base/logging.h" +#include "third_party/mlir_edge/iree/base/status.h" +#include "third_party/mlir_edge/iree/hal/resource.h" + +// Only enable debug names in non-opt modes (unless the user forces it on). +#if !defined(NDEBUG) && !defined(HAS_IREE_BUFFER_DEBUG_NAME) +#define HAS_IREE_BUFFER_DEBUG_NAME 1 +#endif // !NDEBUG + +namespace iree { + +// std::size_t equivalent that is the size as used on device. +// As the device may have a larger memory address space than the host we treat +// all byte offsets as this type instead of the host-specified size_t. +using device_size_t = uint64_t; + +// When used as a length value in functions causes the length to be the entire +// remaining buffer from the specified offset. +constexpr device_size_t kWholeBuffer = ~0ull; + +} // namespace iree + +namespace iree { +namespace hal { + +class Allocator; +template +class MappedMemory; + +// A bitfield specifying properties for a memory type. +enum class MemoryType : uint32_t { + kNone = 0, + + // Memory is lazily allocated by the device and only exists transiently. + // This is the optimal mode for memory used only within a single command + // buffer. Transient buffers, even if they have kHostVisible set, should be + // treated as device-local and opaque as they may have no memory attached to + // them outside of the time they are being evaluated on devices. + // + // This flag can be treated as a hint in most cases; allocating a buffer with + // it set _may_ return the same as if it had not be set. Certain allocation + // routines may use the hint to more tightly control reuse or defer wiring the + // memory. + kTransient = 1 << 0, + + // Memory allocated with this type can be mapped for host access using + // Buffer::MapMemory. + kHostVisible = 1 << 1, + + // The host cache management commands MappedMemory::Flush and + // MappedMemory::Invalidate are not needed to flush host writes + // to the device or make device writes visible to the host, respectively. + kHostCoherent = 1 << 2, + + // Memory allocated with this type is cached on the host. Host memory + // accesses to uncached memory are slower than to cached memory, however + // uncached memory is always host coherent. MappedMemory::Flush must be used + // to ensure the device has visibility into any changes made on the host and + // Invalidate must be used to ensure the host has visibility into any changes + // made on the device. + kHostCached = 1 << 3, + + // Memory is accessible as normal host allocated memory. + kHostLocal = kHostVisible | kHostCoherent, + + // Memory allocated with this type is visible to the device for execution. + // Being device visible does not mean the same thing as kDeviceLocal. Though + // an allocation may be visible to the device and therefore useable for + // execution it may require expensive mapping or implicit transfers. + kDeviceVisible = 1 << 4, + + // Memory allocated with this type is the most efficient for device access. + // Devices may support using memory that is not device local via + // kDeviceVisible but doing so can incur non-trivial performance penalties. + // Device local memory, on the other hand, is guaranteed to be fast for all + // operations. + kDeviceLocal = kDeviceVisible | (1 << 5), +}; +IREE_BITFIELD(MemoryType); +using MemoryTypeBitfield = MemoryType; +std::string MemoryTypeString(MemoryTypeBitfield memory_type); + +// A bitfield specifying how memory will be accessed in a mapped memory region. +enum class MemoryAccess : uint32_t { + // Memory is not mapped. + kNone = 0, + + // Memory will be read. + // If a buffer is only mapped for reading it may still be possible to write to + // it but the results will be undefined (as it may present coherency issues). + kRead = 1 << 0, + + // Memory will be written. + // If a buffer is only mapped for writing it may still be possible to read + // from it but the results will be undefined or incredibly slow (as it may + // be mapped by the driver as uncached). + kWrite = 1 << 1, + + // Memory will be discarded prior to mapping. + // The existing contents will be undefined after mapping and must be written + // to ensure validity. + kDiscard = 1 << 2, + + // Memory will be discarded and completely overwritten in a single operation. + kDiscardWrite = kWrite | kDiscard, + + // Memory may have any operation performed on it. + kAll = kRead | kWrite | kDiscard, +}; +IREE_BITFIELD(MemoryAccess); +using MemoryAccessBitfield = MemoryAccess; +std::string MemoryAccessString(MemoryAccessBitfield memory_access); + +// Bitfield that defines how a buffer is intended to be used. +// Usage allows the driver to appropriately place the buffer for more +// efficient operations of the specified types. +enum class BufferUsage { + kNone = 0, + + // The buffer, once defined, will not be mapped or updated again. + // This should be used for uniform parameter values such as runtime + // constants for executables. Doing so may allow drivers to inline values or + // represent them in command buffers more efficiently (avoiding memory reads + // or swapping, etc). + kConstant = 1 << 0, + + // The buffer can be used as the source or target of a transfer command + // (CopyBuffer, UpdateBuffer, etc). + // + // If |kMapping| is not specified drivers may safely assume that the host + // may never need visibility of this buffer as all accesses will happen via + // command buffers. + kTransfer = 1 << 1, + + // The buffer can be mapped by the host application for reading and writing. + // + // As mapping may require placement in special address ranges or system + // calls to enable visibility the driver can use the presence (or lack of) + // this flag to perform allocation-type setup and avoid initial mapping + // overhead. + kMapping = 1 << 2, + + // The buffer can be provided as an input or output to an executable. + // Buffers of this type may be directly used by drivers during dispatch. + kDispatch = 1 << 3, + + // Buffer may be used for any operation. + kAll = kTransfer | kMapping | kDispatch, +}; +IREE_BITFIELD(BufferUsage); +using BufferUsageBitfield = BufferUsage; +std::string BufferUsageString(BufferUsageBitfield buffer_usage); + +// A memory buffer. +// Buffers have a specific memory_type that is used to describe the capabilities +// and behavior of the backing memory of the buffer. Buffers may be any mix of +// host-accessible, host-coherent, or device-accessible for various usages. +// Depending on these memory types the buffers may be mapped for access on the +// host as memory though certain restrictions may be imposed. +// +// See MemoryType for more information about the types and what operations they +// support. +class Buffer : public Resource { + public: + // Returns a reference to a subspan of the buffer. + // If |byte_length| is kWholeBuffer the remaining bytes in the buffer after + // |byte_offset| (possibly 0) will be selected. + // + // The parent buffer will remain alive for the lifetime of the subspan + // returned. If the subspan is a small portion this may cause additional + // memory to remain allocated longer than required. + // + // Returns the given |buffer| if the requested span covers the entire range. + static StatusOr> Subspan(const ref_ptr& buffer, + device_size_t byte_offset, + device_size_t byte_length); + + // Overlap test results. + enum class Overlap { + // No overlap between the two buffers. + kDisjoint, + // Partial overlap between the two buffers. + kPartial, + // Complete overlap between the two buffers (they are the same). + kComplete, + }; + + // Tests whether the given buffers overlap, including support for subspans. + // kWholeBuffer may be used for |lhs_length| and/or |rhs_length| to use the + // lengths of those buffers, respectively. + static Overlap TestOverlap(Buffer* lhs_buffer, device_size_t lhs_offset, + device_size_t lhs_length, Buffer* rhs_buffer, + device_size_t rhs_offset, + device_size_t rhs_length); + + // Returns true if the two buffer ranges overlap at all. + static bool DoesOverlap(Buffer* lhs_buffer, device_size_t lhs_offset, + device_size_t lhs_length, Buffer* rhs_buffer, + device_size_t rhs_offset, device_size_t rhs_length); + + // Disallow copies (as copying requires real work). + Buffer(const Buffer&) = delete; + Buffer& operator=(const Buffer&) = delete; + + ~Buffer() override = default; + +#if HAS_IREE_BUFFER_DEBUG_NAME + // Optionally populated name useful for logging a persistent name for the + // buffer. + absl::string_view debug_name() const { return debug_name_; } + void set_debug_name(std::string debug_name) { + debug_name_ = std::move(debug_name); + } +#else + absl::string_view debug_name() const { return ""; } + void set_debug_name(std::string debug_name) {} +#endif // HAS_IREE_BUFFER_DEBUG_NAME + + // Memory allocator this buffer was allocated from. + // May be nullptr if the buffer has no particular allocator and should be + // assumed to be allocated from the host heap. + constexpr Allocator* allocator() const { + return allocated_buffer_ == this ? allocator_ + : allocated_buffer_->allocator(); + } + + // Memory type this buffer is allocated from. + MemoryTypeBitfield memory_type() const { return memory_type_; } + + // Memory access operations allowed on the buffer. + MemoryAccessBitfield allowed_access() const { return allowed_access_; } + + // Bitfield describing how the buffer is to be used. + BufferUsageBitfield usage() const { return usage_; } + + // Returns the underlying buffer that represents the allocated memory for the + // Buffer. In most cases this is the buffer itself but for buffer subspan + // references it will point to the parent buffer. + Buffer* allocated_buffer() const noexcept; + + // Size of the resource memory allocation in bytes. + // This may be rounded up from the originally requested size or the ideal + // size for the resource based on device restrictions. + constexpr device_size_t allocation_size() const { + return allocated_buffer_ == this ? allocation_size_ + : allocated_buffer_->allocation_size(); + } + + // Range within the underlying allocation this buffer occupies. + // For buffers that map 1:1 with an allocation this should be + // [0, allocation_size()), however may still differ if the allocation needed + // to be aligned. + // + // The offset is most often manipulated by Subspan, however it's important to + // note that the offset may not be what was passed to Subspan as it refers to + // the offset in the original ancestor buffer, not the buffer from which the + // subspan was taken. + constexpr device_size_t byte_offset() const noexcept { return byte_offset_; } + constexpr device_size_t byte_length() const noexcept { return byte_length_; } + + // TODO(benvanik): add debug_name. + + // Returns a longer debug string describing the buffer and its attributes. + std::string DebugString() const; + // Returns a short debug string describing the buffer. + std::string DebugStringShort() const; + + // Sets a range of the buffer to the given value. + // This requires that the resource was allocated with + // MemoryType::kHostVisible and BufferUsage::kMapping. + // If |byte_length| is kWholeBuffer the remaining bytes in the buffer after + // |byte_offset| (possibly 0) will be filled. + // + // The |byte_offset| and |byte_length| must be aligned to the size of the fill + // value. Multi-byte values will be written in host order for host buffers and + // device order for device buffers. + // + // Only |pattern_length| values with 1, 2, or 4 bytes are supported. + // + // Fails if the write could not be performed; either the bounds are out of + // range or the memory type does not support writing in this way. + Status Fill(device_size_t byte_offset, device_size_t byte_length, + const void* pattern, device_size_t pattern_length); + template + Status Fill8(device_size_t byte_offset, device_size_t byte_length, T value); + template + Status Fill16(device_size_t byte_offset, device_size_t byte_length, T value); + template + Status Fill32(device_size_t byte_offset, device_size_t byte_length, T value); + template + Status Fill8(T value); + template + Status Fill16(T value); + template + Status Fill32(T value); + + // Reads a block of byte data from the resource at the given offset. + // This requires that the resource was allocated with + // MemoryType::kHostVisible and BufferUsage::kMapping. + // + // Fails if the read could not be performed; either the bounds are out of + // range or the memory type does not support reading in this way. + Status ReadData(device_size_t source_offset, void* data, + device_size_t data_length); + + // Writes a block of byte data into the resource at the given offset. + // This requires that the resource was allocated with + // MemoryType::kHostVisible and BufferUsage::kMapping. + // + // Fails if the write could not be performed; either the bounds are out of + // range or the memory type does not support writing in this way. + Status WriteData(device_size_t target_offset, const void* data, + device_size_t data_length); + + // Copies data from the provided source_buffer into the buffer. + // This requires that the resource was allocated with + // MemoryType::kHostVisible and BufferUsage::kMapping. + // The source and destination may be the same buffer but the ranges must not + // overlap (a la memcpy). + // + // Fails if the write could not be performed; either the bounds are out of + // range or the memory type does not support writing in this way. + Status CopyData(device_size_t target_offset, Buffer* source_buffer, + device_size_t source_offset, device_size_t data_length); + Status CopyData(device_size_t target_offset, Buffer* source_buffer) { + return CopyData(target_offset, source_buffer, 0, kWholeBuffer); + } + + // Maps the resource memory for direct access from the host. + // This requires that the resource was allocated with + // MemoryType::kHostVisible and BufferUsage::kMapping. + // + // If MemoryType::kHostCoherent was not specified then explicit + // Invalidate and Flush calls must be used to control visibility of the data + // on the device. If MemoryType::kHostCached is not set callers must not + // attempt to read from the mapped memory as doing so may produce undefined + // results and/or ultra slow reads. + // + // If the MemoryAccess::kDiscard bit is set when mapping for writes the caller + // guarantees that they will be overwriting all data in the mapped range. This + // is used as a hint to the device that the prior contents are no longer + // required and can enable optimizations that save on synchronization and + // readback. Note however that it is strictly a hint and the contents are not + // guaranteed to be zeroed during mapping. + // + // This allows mapping the memory as a C++ type. Care must be taken to ensure + // the data layout in C++ matches the expected data layout in the executables + // that consume this data. For simple primitives like uint8_t or float this is + // usually not a problem however struct packing may have many restrictions. + // + // The returned mapping should be unmapped when it is no longer required. + // Unmapping does not implicitly flush. + // + // Fails if the memory could not be mapped due to mapping exhaustion, invalid + // arguments, or unsupported memory types. + // + // Example: + // ASSIGN_OR_RETURN(auto mapping, buffer->MapForRead()); + // mapping[5].foo = 3; + // std::memcpy(mapping.data(), source_data, mapping.size()); + // mapping.reset(); + template + StatusOr> MapMemory( + MemoryAccessBitfield memory_access, device_size_t element_offset = 0, + device_size_t element_length = kWholeBuffer); + + protected: + template + friend class MappedMemory; + + // Defines the mode of a MapMemory operation. + enum class MappingMode { + // The call to MapMemory will always be matched with UnmapMemory. + kScoped, + }; + + Buffer(Allocator* allocator, MemoryTypeBitfield memory_type, + MemoryAccessBitfield allowed_access, BufferUsageBitfield usage, + device_size_t allocation_size, device_size_t byte_offset, + device_size_t byte_length); + + // Allows subclasses to override the allowed access bits. + // This should only be done when known safe by the allocation scheme. + void set_allowed_access(MemoryAccessBitfield allowed_access) { + allowed_access_ = allowed_access; + } + + // Sets a range of the buffer to the given value. + // State and parameters have already been validated. For the >8bit variants + // the offset and length have already been validated to be aligned to the + // natural alignment of the type. + virtual Status FillImpl(device_size_t byte_offset, device_size_t byte_length, + const void* pattern, + device_size_t pattern_length) = 0; + + // Reads a block of byte data from the resource at the given offset. + // State and parameters have already been validated. + virtual Status ReadDataImpl(device_size_t source_offset, void* data, + device_size_t data_length) = 0; + + // Writes a block of byte data into the resource at the given offset. + // State and parameters have already been validated. + virtual Status WriteDataImpl(device_size_t target_offset, const void* data, + device_size_t data_length) = 0; + + // Copies a block of byte data into the resource at the given offset. + // State and parameters have already been validated. + virtual Status CopyDataImpl(device_size_t target_offset, + Buffer* source_buffer, + device_size_t source_offset, + device_size_t data_length) = 0; + + // Maps memory directly. + // The output data pointer will be properly aligned to the start of the data. + // |local_byte_offset| and |local_byte_length| are the adjusted values that + // should map into the local space of the buffer. + // + // Fails if the memory could not be mapped (invalid access type, invalid + // range, or unsupported memory type). + // State and parameters have already been validated. + virtual Status MapMemoryImpl(MappingMode mapping_mode, + MemoryAccessBitfield memory_access, + device_size_t local_byte_offset, + device_size_t local_byte_length, + void** out_data) = 0; + + // Unmaps previously mapped memory. + // No-op if the memory is not mapped. As this is often used in destructors + // we can't rely on failures here propagating with anything but CHECK/DCHECK. + // State and parameters have already been validated. + virtual Status UnmapMemoryImpl(device_size_t local_byte_offset, + device_size_t local_byte_length, + void* data) = 0; + + // Invalidates ranges of non-coherent memory from the host caches. + // Use this before reading from non-coherent memory. + // This guarantees that device writes to the memory ranges provided are + // visible on the host. + // This is only required for memory types without kHostCoherent set. + // State and parameters have already been validated. + virtual Status InvalidateMappedMemoryImpl( + device_size_t local_byte_offset, device_size_t local_byte_length) = 0; + + // Flushes ranges of non-coherent memory from the host caches. + // Use this after writing to non-coherent memory. + // This guarantees that host writes to the memory ranges provided are made + // available for device access. + // This is only required for memory types without kHostCoherent set. + // State and parameters have already been validated. + virtual Status FlushMappedMemoryImpl(device_size_t local_byte_offset, + device_size_t local_byte_length) = 0; + + // Validates the given buffer range and adjusts the offset and length if the + // provided length is kWholeBuffer or the buffer is offset within its + // allocation. This calculates the range in the given domain without adjusting + // to any particular buffer base offsets. + static Status CalculateLocalRange(device_size_t max_length, + device_size_t offset, device_size_t length, + device_size_t* out_adjusted_offset, + device_size_t* out_adjusted_length); + + private: + friend class Allocator; + + // This is not great and deserves cleanup. + friend class DeferredBuffer; + friend class SubspanBuffer; + friend class HeapBuffer; + + // Maps memory directly. + // The byte offset and byte length may be adjusted for device alignment. + // The output data pointer will be properly aligned to the start of the data. + // Fails if the memory could not be mapped (invalid access type, invalid + // range, or unsupported memory type). + Status MapMemory(MappingMode mapping_mode, MemoryAccessBitfield memory_access, + device_size_t* byte_offset, device_size_t* byte_length, + void** out_data); + + // Unmaps previously mapped memory. + // No-op if the memory is not mapped. As this is often used in destructors + // we can't rely on failures here propagating with anything but CHECK/DCHECK. + Status UnmapMemory(device_size_t local_byte_offset, + device_size_t local_byte_length, void* data); + + // Invalidates ranges of non-coherent memory from the host caches. + // Use this before reading from non-coherent memory. + // This guarantees that device writes to the memory ranges provided are + // visible on the host. + // This is only required for memory types without kHostCoherent set. + Status InvalidateMappedMemory(device_size_t local_byte_offset, + device_size_t local_byte_length); + + // Flushes ranges of non-coherent memory from the host caches. + // Use this after writing to non-coherent memory. + // This guarantees that host writes to the memory ranges provided are made + // available for device access. + // This is only required for memory types without kHostCoherent set. + Status FlushMappedMemory(device_size_t local_byte_offset, + device_size_t local_byte_length); + + // Returns a failure if the memory type the buffer was allocated from is not + // compatible with the given type. + Status ValidateCompatibleMemoryType(MemoryTypeBitfield memory_type) const; + // Returns a failure if the buffer memory type or usage disallows the given + // access type. + Status ValidateAccess(MemoryAccessBitfield memory_access) const; + // Returns a failure if the buffer was not allocated for the given usage. + Status ValidateUsage(BufferUsageBitfield usage) const; + // Validates the given buffer range and optionally adjusts the offset and + // length if the provided length is kWholeBuffer or the buffer is offset + // within its allocation. + static Status CalculateRange(device_size_t base_offset, + device_size_t max_length, device_size_t offset, + device_size_t length, + device_size_t* out_adjusted_offset, + device_size_t* out_adjusted_length = nullptr); + Status CalculateRange(device_size_t offset, device_size_t length, + device_size_t* out_adjusted_offset, + device_size_t* out_adjusted_length = nullptr) const; + + // Points to either this or parent_buffer_.get(). + Buffer* allocated_buffer_ = nullptr; + + Allocator* allocator_ = nullptr; + MemoryTypeBitfield memory_type_ = MemoryType::kNone; + MemoryAccessBitfield allowed_access_ = MemoryAccess::kNone; + BufferUsageBitfield usage_ = BufferUsage::kNone; + + device_size_t allocation_size_ = 0; + device_size_t byte_offset_ = 0; + device_size_t byte_length_ = 0; + +#if HAS_IREE_BUFFER_DEBUG_NAME + // Friendly name for the buffer used in DebugString. May be set by the app or + // auto generated. + std::string debug_name_; +#endif // HAS_IREE_BUFFER_DEBUG_NAME + + // Defined when this buffer is a subspan of another buffer. + ref_ptr parent_buffer_; +}; + +// A memory mapping RAII object. +// The mapping will stay active until it is reset and will retain the buffer. +template +class MappedMemory { + public: + using unspecified_bool_type = const T* MappedMemory::*; + + MappedMemory() = default; + MappedMemory(MemoryAccessBitfield access, ref_ptr buffer, + device_size_t byte_offset, device_size_t byte_length, + device_size_t element_size, T* data); + + // Allow moving but disallow copying as the mapping is stateful. + MappedMemory(MappedMemory&& rhs) noexcept; + MappedMemory& operator=(MappedMemory&& rhs) noexcept; + MappedMemory(const MappedMemory&) = delete; + MappedMemory& operator=(const MappedMemory&) = delete; + + ~MappedMemory(); + + // The buffer resource that this mapping references. + const ref_ptr& buffer() const noexcept { return buffer_; } + // Offset, in bytes, into the resource allocation. + // This value is *informative only*, as it may vary from device to device. + device_size_t byte_offset() const noexcept { return byte_offset_; } + // Length, in bytes, of the resource mapping. + // This may be larger than the originally requested length due to alignment. + // This value is *informative only*, as it may vary from device to device. + device_size_t byte_length() const noexcept { return byte_length_; } + + // True if the mapping is empty. + bool empty() const noexcept { return element_size_ == 0; } + // The size of the mapping as requested in elements. + size_t size() const noexcept { return static_cast(element_size_); } + + // Returns a read-only pointer to the mapped memory. + // This will be nullptr if the mapping failed or the mapping is not readable. + const T* data() const noexcept; + absl::Span contents() const noexcept { return {data(), size()}; } + + // Returns a mutable pointer to the mapped memory. + // This will be nullptr if the mapping failed or the mapping is not writable. + // If the mapping was not made with read access it may still be possible to + // read from this memory but behavior is undefined. + T* mutable_data() noexcept; + absl::Span mutable_contents() noexcept { return {mutable_data(), size()}; } + + // Equivalent to absl::Span::subspan(). + // May return a 0-length span. + // Fails if the buffer is not mapped or not mapped for the requested access. + StatusOr> Subspan( + device_size_t element_offset = 0, + device_size_t element_length = kWholeBuffer) const noexcept; + StatusOr> MutableSubspan( + device_size_t element_offset = 0, + device_size_t element_length = kWholeBuffer) noexcept; + + // Accesses an element in the mapped memory. + // Must be called with a valid index in [0, size()). + const T& operator[](device_size_t i) const noexcept { return data_[i]; } + + // Invalidates a range of non-coherent elements from the host caches. + Status Invalidate(device_size_t element_offset = 0, + device_size_t element_length = kWholeBuffer) const; + + // Flushes a range of non-coherent elements from the host caches. + Status Flush(device_size_t element_offset = 0, + device_size_t element_length = kWholeBuffer); + + // Unmaps the mapped memory. + // The memory will not be implicitly flushed when unmapping. + void reset(); + + private: + Status ValidateAccess(MemoryAccessBitfield memory_access) const; + Status CalculateDataRange(device_size_t element_offset, + device_size_t element_length, + device_size_t* out_adjusted_element_offset, + device_size_t* out_adjusted_element_length) const; + + MemoryAccessBitfield access_ = MemoryAccess::kNone; + ref_ptr buffer_; + device_size_t byte_offset_ = 0; + device_size_t byte_length_ = 0; + device_size_t element_size_ = 0; + T* data_ = nullptr; +}; + +// Inline functions and template definitions follow: + +template +Status Buffer::Fill8(device_size_t byte_offset, device_size_t byte_length, + T value) { + auto sized_value = reinterpret_cast(&value); + return Fill(byte_offset, byte_length, sized_value, sizeof(*sized_value)); +} + +template +Status Buffer::Fill16(device_size_t byte_offset, device_size_t byte_length, + T value) { + auto sized_value = reinterpret_cast(&value); + return Fill(byte_offset, byte_length, sized_value, sizeof(*sized_value)); +} + +template +Status Buffer::Fill32(device_size_t byte_offset, device_size_t byte_length, + T value) { + auto sized_value = reinterpret_cast(&value); + return Fill(byte_offset, byte_length, sized_value, sizeof(*sized_value)); +} + +template +Status Buffer::Fill8(T value) { + return Fill8(0, kWholeBuffer, value); +} + +template +Status Buffer::Fill16(T value) { + return Fill16(0, kWholeBuffer, value); +} + +template +Status Buffer::Fill32(T value) { + return Fill32(0, kWholeBuffer, value); +} + +template +StatusOr> Buffer::MapMemory(MemoryAccessBitfield memory_access, + device_size_t element_offset, + device_size_t element_length) { + device_size_t byte_offset = element_offset * sizeof(T); + device_size_t byte_length = element_length == kWholeBuffer + ? kWholeBuffer + : element_length * sizeof(T); + void* data = nullptr; + RETURN_IF_ERROR(MapMemory(MappingMode::kScoped, memory_access, &byte_offset, + &byte_length, &data)); + return MappedMemory{ + memory_access, add_ref(this), byte_offset, + byte_length, byte_length / sizeof(T), static_cast(data)}; +} + +template +MappedMemory::MappedMemory(MemoryAccessBitfield access, + ref_ptr buffer, device_size_t byte_offset, + device_size_t byte_length, + device_size_t element_size, T* data) + : access_(access), + buffer_(std::move(buffer)), + byte_offset_(byte_offset), + byte_length_(byte_length), + element_size_(element_size), + data_(data) {} + +template +MappedMemory::MappedMemory(MappedMemory&& rhs) noexcept + : access_(rhs.access_), + buffer_(std::move(rhs.buffer_)), + byte_offset_(rhs.byte_offset_), + byte_length_(rhs.byte_length_), + element_size_(rhs.element_size_), + data_(rhs.data_) { + rhs.access_ = MemoryAccess::kNone; + rhs.buffer_.reset(); + rhs.byte_offset_ = 0; + rhs.byte_length_ = 0; + rhs.element_size_ = 0; + rhs.data_ = nullptr; +} + +template +MappedMemory& MappedMemory::operator=(MappedMemory&& rhs) noexcept { + if (this != &rhs) { + reset(); + access_ = rhs.access_; + buffer_ = std::move(rhs.buffer_); + byte_offset_ = rhs.byte_offset_; + byte_length_ = rhs.byte_length_; + element_size_ = rhs.element_size_; + data_ = rhs.data_; + + rhs.access_ = MemoryAccess::kNone; + rhs.buffer_.reset(); + rhs.byte_offset_ = 0; + rhs.byte_length_ = 0; + rhs.element_size_ = 0; + rhs.data_ = nullptr; + } + return *this; +} + +template +MappedMemory::~MappedMemory() { + // Unmap (if needed) - note that we can't fail gracefully here :( + reset(); +} + +template +const T* MappedMemory::data() const noexcept { + if (!data_ || !AnyBitSet(access_ & MemoryAccess::kRead)) { + return nullptr; + } + return data_; +} + +template +T* MappedMemory::mutable_data() noexcept { + if (!data_ || !AnyBitSet(access_ & MemoryAccess::kWrite)) { + return nullptr; + } + return data_; +} + +template +Status MappedMemory::ValidateAccess( + MemoryAccessBitfield memory_access) const { + if (!data_) { + return FailedPreconditionErrorBuilder(ABSL_LOC) << "Buffer is not mapped"; + } else if (!AnyBitSet(access_ & memory_access)) { + return PermissionDeniedErrorBuilder(ABSL_LOC) + << "Buffer is not mapped for the desired access"; + } + return OkStatus(); +} + +template +Status MappedMemory::CalculateDataRange( + device_size_t element_offset, device_size_t element_length, + device_size_t* out_adjusted_element_offset, + device_size_t* out_adjusted_element_length) const { + RETURN_IF_ERROR(Buffer::CalculateLocalRange( + element_size_ * sizeof(T), element_offset * sizeof(T), + element_length == kWholeBuffer ? kWholeBuffer + : element_length * sizeof(T), + out_adjusted_element_offset, out_adjusted_element_length)); + *out_adjusted_element_offset /= sizeof(T); + *out_adjusted_element_length /= sizeof(T); + return OkStatus(); +} + +template +inline StatusOr> MappedMemory::Subspan( + device_size_t element_offset, device_size_t element_length) const noexcept { + RETURN_IF_ERROR(ValidateAccess(MemoryAccess::kRead)); + RETURN_IF_ERROR(CalculateDataRange(element_offset, element_length, + &element_offset, &element_length)); + return absl::Span(data_ + element_offset, element_length); +} + +template +inline StatusOr> MappedMemory::MutableSubspan( + device_size_t element_offset, device_size_t element_length) noexcept { + RETURN_IF_ERROR(ValidateAccess(MemoryAccess::kWrite)); + RETURN_IF_ERROR(CalculateDataRange(element_offset, element_length, + &element_offset, &element_length)); + return absl::Span(data_ + element_offset, element_length); +} + +template +Status MappedMemory::Invalidate(device_size_t element_offset, + device_size_t element_length) const { + RETURN_IF_ERROR(ValidateAccess(MemoryAccess::kRead)); + RETURN_IF_ERROR(CalculateDataRange(element_offset, element_length, + &element_offset, &element_length)); + if (!element_length) return OkStatus(); + return buffer_->InvalidateMappedMemory( + byte_offset_ + element_offset * sizeof(T), element_length * sizeof(T)); +} + +template +Status MappedMemory::Flush(device_size_t element_offset, + device_size_t element_length) { + RETURN_IF_ERROR(ValidateAccess(MemoryAccess::kWrite)); + RETURN_IF_ERROR(CalculateDataRange(element_offset, element_length, + &element_offset, &element_length)); + if (!element_length) return OkStatus(); + return buffer_->FlushMappedMemory(byte_offset_ + element_offset * sizeof(T), + element_length * sizeof(T)); +} + +template +void MappedMemory::reset() { + if (!buffer_) return; + // TODO(benvanik): better handling of errors? may be fine to always warn. + buffer_->UnmapMemory(byte_offset_, byte_length_, data_).IgnoreError(); + buffer_.reset(); + access_ = MemoryAccess::kNone; + byte_offset_ = 0; + byte_length_ = 0; + element_size_ = 0; + data_ = nullptr; +} + +} // namespace hal +} // namespace iree + +#endif // THIRD_PARTY_MLIR_EDGE_IREE_HAL_BUFFER_H_ diff --git a/hal/buffer_view.cc b/hal/buffer_view.cc new file mode 100644 index 000000000000..3225ffe7850c --- /dev/null +++ b/hal/buffer_view.cc @@ -0,0 +1,180 @@ +// Copyright 2019 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "third_party/mlir_edge/iree/hal/buffer_view.h" + +#include "third_party/absl/container/inlined_vector.h" +#include "third_party/absl/strings/str_cat.h" +#include "third_party/absl/strings/str_join.h" +#include "third_party/absl/types/source_location.h" +#include "third_party/mlir_edge/iree/base/status.h" +#include "third_party/mlir_edge/iree/hal/buffer.h" + +namespace iree { +namespace hal { + +namespace { +// Pretty prints an array, e.g. [1, 2, 3, 4] +inline std::string PrettyPrint(absl::Span arr) { + return "[" + absl::StrJoin(arr, ",") + "]"; +} +} // namespace + +// static +bool BufferView::Equal(const BufferView& lhs, const BufferView& rhs) { + return lhs.buffer.get() == rhs.buffer.get() && + lhs.element_size == rhs.element_size && lhs.shape == rhs.shape; +} + +std::string BufferView::DebugStringShort() const { + if (element_size == 0) { + return "Ø"; + } + return shape.empty() ? std::to_string(element_size) + : absl::StrCat(absl::StrJoin(shape.subspan(), "x"), "x", + element_size); +} + +StatusOr BufferView::CalculateOffset( + absl::Span indices) const { + if (indices.empty()) { + return 0; + } else if (shape.empty() || indices.size() > shape.size()) { + return InvalidArgumentErrorBuilder(ABSL_LOC) + << "Indices " << PrettyPrint(indices) + << " out of bounds of the rank of buffer_view " + << DebugStringShort(); + } + device_size_t offset = 0; + for (int i = 0; i < indices.size(); ++i) { + if (indices[i] >= shape[i]) { + return InvalidArgumentErrorBuilder(ABSL_LOC) + << "Indices[" << i << "]=" << indices[i] + << " out of bounds of buffer_view " << DebugStringShort(); + } + device_size_t axis_offset = indices[i]; + for (int j = i + 1; j < shape.size(); ++j) { + axis_offset *= shape[j]; + } + offset += axis_offset; + } + offset *= element_size; + return offset; +} + +StatusOr BufferView::Slice( + absl::Span start_indices, + absl::Span lengths) const { + if (start_indices.size() != shape.size()) { + return InvalidArgumentErrorBuilder(ABSL_LOC) + << "Slice start_indices " << PrettyPrint(start_indices) + << " do not match rank of buffer_view " << DebugStringShort(); + } + if (start_indices.size() != lengths.size()) { + return InvalidArgumentErrorBuilder(ABSL_LOC) + << "Slice start_indices " << PrettyPrint(start_indices) + << " and lengths " << PrettyPrint(lengths) + << " are not the same size"; + } + + // Buffer::Subspan only support contiguous memory. To ensure that this slice + // only requests such, we validate that the offset in the buffer between the + // start and end indices is the same as the requested size of the slice. + absl::InlinedVector end_indices(lengths.size()); + device_size_t subspan_length = element_size; + for (int i = 0; i < lengths.size(); ++i) { + subspan_length *= lengths[i]; + end_indices[i] = start_indices[i] + lengths[i] - 1; + } + + ASSIGN_OR_RETURN(auto start_byte_offset, CalculateOffset(start_indices)); + // Also validates the ends are in bounds. + ASSIGN_OR_RETURN(auto end_byte_offset, CalculateOffset(end_indices)); + + auto offset_length = end_byte_offset - start_byte_offset + element_size; + if (subspan_length != offset_length) { + return UnimplementedErrorBuilder(ABSL_LOC) + << "Slice for non-contiguous region of memory unimplemented. " + "start_indices: " + << PrettyPrint(start_indices) << " lengths: " << PrettyPrint(lengths) + << " " << subspan_length << " " << offset_length << " " + << PrettyPrint(end_indices); + } + + ASSIGN_OR_RETURN(auto new_buffer, + Buffer::Subspan(buffer, start_byte_offset, subspan_length)); + return BufferView(std::move(new_buffer), Shape(lengths), element_size); +} + +// static +Status BufferView::Copy(BufferView* src, + absl::Span src_start_indices, + BufferView* dst, + absl::Span dst_start_indices, + absl::Span lengths) { + if (src_start_indices.size() != src->shape.size() || + dst_start_indices.size() != dst->shape.size() || + src_start_indices.size() != lengths.size() || + dst_start_indices.size() != lengths.size()) { + return InvalidArgumentErrorBuilder(ABSL_LOC) + << "Src/dst shape/size mismatch: src=" << src->DebugStringShort() + << ", dst=" << dst->DebugStringShort() + << ", src_indices=" << PrettyPrint(src_start_indices) + << ", dst_indices=" << PrettyPrint(dst_start_indices) + << ", lengths=" << PrettyPrint(lengths); + } + + // Copies only support contiguous memory. To ensure that this copy + // only requests such, we validate that the offset in the buffer between the + // start and end indices is the same as the requested size of the copy. + absl::InlinedVector src_end_indices(lengths.size()); + absl::InlinedVector dst_end_indices(lengths.size()); + device_size_t total_length = src->element_size; + for (int i = 0; i < lengths.size(); ++i) { + total_length *= lengths[i]; + src_end_indices[i] = src_start_indices[i] + lengths[i] - 1; + dst_end_indices[i] = dst_start_indices[i] + lengths[i] - 1; + } + + ASSIGN_OR_RETURN(auto src_start_byte_offset, + src->CalculateOffset(src_start_indices)); + ASSIGN_OR_RETURN(auto src_end_byte_offset, + src->CalculateOffset(src_end_indices)); + ASSIGN_OR_RETURN(auto dst_start_byte_offset, + dst->CalculateOffset(dst_start_indices)); + ASSIGN_OR_RETURN(auto dst_end_byte_offset, + dst->CalculateOffset(dst_end_indices)); + + auto src_length = + src_end_byte_offset - src_start_byte_offset + src->element_size; + auto dst_length = + dst_end_byte_offset - dst_start_byte_offset + dst->element_size; + if (src_length != dst_length || src_length != total_length) { + return UnimplementedErrorBuilder(ABSL_LOC) + << "Copy for non-contiguous region of memory unimplemented: " + << src->DebugStringShort() << ", dst=" << dst->DebugStringShort() + << ", src_indices=" << PrettyPrint(src_start_indices) + << ", dst_indices=" << PrettyPrint(dst_start_indices) + << ", lengths=" << PrettyPrint(lengths); + } + + RETURN_IF_ERROR(dst->buffer->CopyData(dst_start_byte_offset, + src->buffer.get(), + src_start_byte_offset, total_length)); + + return OkStatus(); +} + +} // namespace hal +} // namespace iree diff --git a/hal/buffer_view.h b/hal/buffer_view.h new file mode 100644 index 000000000000..7a5eb1fe55c1 --- /dev/null +++ b/hal/buffer_view.h @@ -0,0 +1,108 @@ +// Copyright 2019 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#ifndef THIRD_PARTY_MLIR_EDGE_IREE_HAL_BUFFER_VIEW_H_ +#define THIRD_PARTY_MLIR_EDGE_IREE_HAL_BUFFER_VIEW_H_ + +#include +#include + +#include "third_party/mlir_edge/iree/base/shape.h" +#include "third_party/mlir_edge/iree/hal/buffer.h" + +namespace iree { +namespace hal { + +struct BufferView { + // Returns true if the given buffer_views are exactly equal. + static bool Equal(const BufferView& lhs, const BufferView& rhs); + + BufferView() = default; + BufferView(ref_ptr buffer, Shape shape, int8_t element_size) noexcept + : buffer(std::move(buffer)), shape(shape), element_size(element_size) {} + + BufferView(const BufferView& other) noexcept + : buffer(add_ref(other.buffer)), + shape(other.shape), + element_size(other.element_size) {} + BufferView& operator=(const BufferView& other) noexcept { + buffer = add_ref(other.buffer); + shape = other.shape; + element_size = other.element_size; + return *this; + } + BufferView(BufferView&& other) noexcept + : buffer(std::move(other.buffer)), + shape(other.shape), + element_size(other.element_size) {} + BufferView& operator=(BufferView&& other) noexcept { + buffer = std::move(other.buffer); + shape = other.shape; + element_size = other.element_size; + return *this; + } + + // Returns a string useful for printing debug messages. + std::string DebugStringShort() const; + + // Total length of the valid view range in bytes. + device_size_t byte_length() const { + return shape.element_count() * element_size; + } + + // TODO(b/134586626): remove this when byte ranges are encoded in IR. + // Calculates a byte offset into the buffer_view at the given dimension + // indices. + StatusOr CalculateOffset( + absl::Span indices) const; + + // TODO(b/134586626): remove this when byte ranges are encoded in IR. + // Returns a view onto the given range of the buffer underlying this view. The + // returned view starts at the offset indicated by |start_indices| and has a + // shape of |lengths|. + // Only contiguous regions of memory are supported at the moment. + StatusOr Slice(absl::Span start_indices, + absl::Span lengths) const; + + // TODO(b/134586626): remove this when byte ranges are encoded in IR. + static Status Copy(BufferView* src, + absl::Span src_start_indices, + BufferView* dst, + absl::Span dst_start_indices, + absl::Span lengths); + + ref_ptr buffer; + Shape shape; + int8_t element_size; + // TODO(benvanik): strides. +}; + +inline bool operator==(const BufferView& a, const BufferView& b) { + return BufferView::Equal(a, b); +} + +inline bool operator!=(const BufferView& a, const BufferView& b) { + return !(a == b); +} + +inline std::ostream& operator<<(std::ostream& stream, + const BufferView& buffer_view) { + stream << buffer_view.DebugStringShort(); + return stream; +} + +} // namespace hal +} // namespace iree + +#endif // THIRD_PARTY_MLIR_EDGE_IREE_HAL_BUFFER_VIEW_H_ diff --git a/hal/buffer_view_string_util.cc b/hal/buffer_view_string_util.cc new file mode 100644 index 000000000000..2d4c4298f706 --- /dev/null +++ b/hal/buffer_view_string_util.cc @@ -0,0 +1,487 @@ +// Copyright 2019 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "third_party/mlir_edge/iree/hal/buffer_view_string_util.h" + +#include +#include +#include + +#include "third_party/absl/strings/ascii.h" +#include "third_party/absl/strings/escaping.h" +#include "third_party/absl/strings/numbers.h" +#include "third_party/absl/strings/str_join.h" +#include "third_party/absl/strings/str_split.h" +#include "third_party/absl/strings/strip.h" +#include "third_party/absl/types/optional.h" +#include "third_party/absl/types/source_location.h" +#include "third_party/mlir_edge/iree/base/status.h" +#include "third_party/mlir_edge/iree/hal/heap_buffer.h" + +namespace iree { +namespace hal { + +namespace { + +// Returns true if the given type is represented as binary hex data. +bool IsBinaryType(absl::string_view type_str) { + return !type_str.empty() && absl::ascii_isdigit(type_str[0]); +} + +// Parses binary hex data. +Status ParseBinaryData(absl::string_view data_str, Buffer* buffer) { + data_str = absl::StripAsciiWhitespace(data_str); + ASSIGN_OR_RETURN(auto mapping, + buffer->MapMemory(MemoryAccess::kDiscardWrite)); + auto contents = mapping.mutable_contents(); + size_t dst_i = 0; + size_t src_i = 0; + while (src_i < data_str.size() && dst_i < contents.size()) { + char c = data_str[src_i]; + if (absl::ascii_isspace(c) || c == ',') { + ++src_i; + continue; + } + if (src_i + 1 >= data_str.size()) { + return InvalidArgumentErrorBuilder(ABSL_LOC) + << "Invalid input hex data (offset=" << src_i << ")"; + } + strings::a2b_hex(data_str.data() + src_i, contents.data() + dst_i, 1); + src_i += 2; + ++dst_i; + } + if (dst_i < contents.size()) { + return InvalidArgumentErrorBuilder(ABSL_LOC) + << "Too few elements to fill type; expected " << contents.size() + << " but only read " << dst_i; + } else if (data_str.size() - src_i > 0) { + return InvalidArgumentErrorBuilder(ABSL_LOC) + << "Input data string contains more elements than the underlying " + "buffer (" + << contents.size() << ")"; + } + return OkStatus(); +} + +// Prints binary hex data. +Status PrintBinaryData(int element_size, Buffer* buffer, size_t max_entries, + std::ostream* stream) { + max_entries *= element_size; // Counting bytes, but treat them as elements. + ASSIGN_OR_RETURN(auto mapping, + buffer->MapMemory(MemoryAccess::kRead)); + auto contents = mapping.contents(); + char hex_buffer[8 * 2]; + for (size_t i = 0; i < std::min(max_entries, mapping.size()); + i += element_size) { + if (i > 0) *stream << " "; + strings::b2a_hex(contents.data() + i, hex_buffer, element_size); + *stream << hex_buffer; + } + if (mapping.size() > max_entries) *stream << "..."; + return OkStatus(); +} + +template +struct SimpleStrToValue { + absl::optional operator()(absl::string_view text) const = delete; +}; + +template +struct SimpleStrToValue< + IntegerType, + typename std::enable_if<(sizeof(IntegerType) < 4), void>::type> { + absl::optional operator()(absl::string_view text) const { + int32_t value; + return absl::SimpleAtoi(text, &value) ? absl::optional{value} + : absl::nullopt; + } +}; + +template +struct SimpleStrToValue< + IntegerType, + typename std::enable_if<(sizeof(IntegerType) >= 4), void>::type> { + absl::optional operator()(absl::string_view text) const { + IntegerType value; + return absl::SimpleAtoi(text, &value) ? absl::optional{value} + : absl::nullopt; + } +}; + +template <> +struct SimpleStrToValue { + absl::optional operator()(absl::string_view text) const { + float value; + return absl::SimpleAtof(text, &value) ? absl::optional{value} + : absl::nullopt; + } +}; + +template <> +struct SimpleStrToValue { + absl::optional operator()(absl::string_view text) const { + double value; + return absl::SimpleAtod(text, &value) ? absl::optional{value} + : absl::nullopt; + } +}; + +template +Status ParseNumericalDataElement(absl::string_view data_str, size_t token_start, + size_t token_end, absl::Span contents, + int dst_i) { + if (dst_i >= contents.size()) { + return InvalidArgumentErrorBuilder(ABSL_LOC) + << "Input data string contains more elements than the underlying " + "buffer (" + << contents.size() << ")"; + } + auto element_str = data_str.substr(token_start, token_end - token_start + 1); + auto element = SimpleStrToValue()(element_str); + if (!element.has_value()) { + return InvalidArgumentErrorBuilder(ABSL_LOC) + << "Unable to parse element " << dst_i << " = '" << element_str + << "'"; + } + contents[dst_i] = element.value(); + return OkStatus(); +} + +template +Status ParseNumericalDataAsType(absl::string_view data_str, Buffer* buffer) { + ASSIGN_OR_RETURN(auto mapping, + buffer->MapMemory(MemoryAccess::kDiscardWrite)); + auto contents = mapping.mutable_contents(); + size_t src_i = 0; + size_t dst_i = 0; + size_t token_start = std::string::npos; + while (src_i < data_str.size()) { + char c = data_str[src_i++]; + bool is_separator = + absl::ascii_isspace(c) || c == ',' || c == '[' || c == ']'; + if (token_start == std::string::npos) { + if (!is_separator) { + token_start = src_i - 1; + } + continue; + } else if (token_start != std::string::npos && !is_separator) { + continue; + } + RETURN_IF_ERROR(ParseNumericalDataElement(data_str, token_start, + src_i - 2, contents, dst_i++)); + token_start = std::string::npos; + } + if (token_start != std::string::npos) { + RETURN_IF_ERROR(ParseNumericalDataElement( + data_str, token_start, data_str.size() - 1, contents, dst_i++)); + } + if (dst_i < contents.size()) { + return InvalidArgumentErrorBuilder(ABSL_LOC) + << "Input data string contains fewer elements than the underlying " + "buffer (expected " + << contents.size() << ")"; + } + return OkStatus(); +} + +// Parses numerical data (ints, floats, etc) in some typed form. +Status ParseNumericalData(absl::string_view type_str, + absl::string_view data_str, Buffer* buffer) { + if (type_str == "i8") { + return ParseNumericalDataAsType(data_str, buffer); + } else if (type_str == "u8") { + return ParseNumericalDataAsType(data_str, buffer); + } else if (type_str == "i16") { + return ParseNumericalDataAsType(data_str, buffer); + } else if (type_str == "u16") { + return ParseNumericalDataAsType(data_str, buffer); + } else if (type_str == "i32") { + return ParseNumericalDataAsType(data_str, buffer); + } else if (type_str == "u32") { + return ParseNumericalDataAsType(data_str, buffer); + } else if (type_str == "i64") { + return ParseNumericalDataAsType(data_str, buffer); + } else if (type_str == "u64") { + return ParseNumericalDataAsType(data_str, buffer); + } else if (type_str == "f32") { + return ParseNumericalDataAsType(data_str, buffer); + } else if (type_str == "f64") { + return ParseNumericalDataAsType(data_str, buffer); + } else { + return InvalidArgumentErrorBuilder(ABSL_LOC) + << "Unsupported type: " << type_str; + } +} + +template +void PrintElementList(const Shape& shape, absl::Span data, + size_t* max_entries, std::ostream* stream) { + if (shape.empty()) { + // Scalar value. + PrintElementList({1}, data, max_entries, stream); + return; + } else if (shape.size() == 1) { + // Leaf dimension; output data. + size_t max_count = std::min(*max_entries, static_cast(shape[0])); + *stream << absl::StrJoin(data.subspan(0, max_count), " "); + if (max_count < shape[0]) { + *stream << "..."; + } + *max_entries -= max_count; + } else { + // Nested; recurse into next dimension. + Shape nested_shape = Shape(shape.subspan(1)); + size_t length = nested_shape.element_count(); + size_t offset = 0; + for (int i = 0; i < shape[0]; ++i) { + *stream << "["; + PrintElementList(nested_shape, data.subspan(offset, length), + max_entries, stream); + offset += length; + *stream << "]"; + } + } +} + +template +Status PrintNumericalDataAsType(const Shape& shape, Buffer* buffer, + size_t max_entries, std::ostream* stream) { + ASSIGN_OR_RETURN(auto mapping, buffer->MapMemory(MemoryAccess::kRead)); + PrintElementList(shape, mapping.contents(), &max_entries, stream); + return OkStatus(); +} + +// Prints numerical data (ints, floats, etc) from some typed form. +Status PrintNumericalData(const Shape& shape, absl::string_view type_str, + Buffer* buffer, size_t max_entries, + std::ostream* stream) { + if (type_str == "i8") { + return PrintNumericalDataAsType(shape, buffer, max_entries, stream); + } else if (type_str == "u8") { + return PrintNumericalDataAsType(shape, buffer, max_entries, + stream); + } else if (type_str == "i16") { + return PrintNumericalDataAsType(shape, buffer, max_entries, + stream); + } else if (type_str == "u16") { + return PrintNumericalDataAsType(shape, buffer, max_entries, + stream); + } else if (type_str == "i32") { + return PrintNumericalDataAsType(shape, buffer, max_entries, + stream); + } else if (type_str == "u32") { + return PrintNumericalDataAsType(shape, buffer, max_entries, + stream); + } else if (type_str == "i64") { + return PrintNumericalDataAsType(shape, buffer, max_entries, + stream); + } else if (type_str == "u64") { + return PrintNumericalDataAsType(shape, buffer, max_entries, + stream); + } else if (type_str == "f32") { + return PrintNumericalDataAsType(shape, buffer, max_entries, stream); + } else if (type_str == "f64") { + return PrintNumericalDataAsType(shape, buffer, max_entries, stream); + } else { + return InvalidArgumentErrorBuilder(ABSL_LOC) + << "Unsupported type: " << type_str; + } +} + +} // namespace + +StatusOr GetTypeElementSize(absl::string_view type_str) { + if (type_str.empty()) { + return InvalidArgumentErrorBuilder(ABSL_LOC) << "Type is empty"; + } else if (IsBinaryType(type_str)) { + // If the first character is a digit then we are dealign with binary data. + // The type is just the number of bytes per element. + int element_size = 0; + if (!absl::SimpleAtoi(type_str, &element_size)) { + return InvalidArgumentErrorBuilder(ABSL_LOC) + << "Unable to parse element size type '" << type_str << "'"; + } + return element_size; + } + // We know that our types are single characters followed by bit counts. + // If we start to support other types we may need to do something more clever. + int bit_count = 0; + if (!absl::SimpleAtoi(type_str.substr(1), &bit_count)) { + return InvalidArgumentErrorBuilder(ABSL_LOC) + << "Unable to parse type bit count from '" << type_str + << "'; expecting something like 'i32'"; + } + return bit_count / 8; +} + +StatusOr ParseShape(absl::string_view shape_str) { + std::vector dims; + for (auto dim_str : absl::StrSplit(shape_str, 'x', absl::SkipWhitespace())) { + int dim_value = 0; + if (!absl::SimpleAtoi(dim_str, &dim_value)) { + return InvalidArgumentErrorBuilder(ABSL_LOC) + << "Invalid shape dimension '" << dim_str + << "' while parsing shape '" << shape_str << "'"; + } + dims.push_back(dim_value); + } + return Shape{dims}; +} + +StatusOr ParseBufferViewFromString( + absl::string_view buffer_view_str, hal::Allocator* allocator) { + // Strip whitespace that may come along (linefeeds/etc). + buffer_view_str = absl::StripAsciiWhitespace(buffer_view_str); + if (buffer_view_str.empty()) { + // Empty lines denote empty buffer_views. + return BufferView{}; + } + + // Split into the components we can work with: shape, type, and data. + absl::string_view shape_and_type_str; + absl::string_view data_str; + auto equal_index = buffer_view_str.find('='); + if (equal_index == std::string::npos) { + // Treat a lack of = as defaulting the data to zeros. + shape_and_type_str = buffer_view_str; + } else { + shape_and_type_str = buffer_view_str.substr(0, equal_index); + data_str = buffer_view_str.substr(equal_index + 1); + } + absl::string_view shape_str; + absl::string_view type_str; + auto last_x_index = shape_and_type_str.rfind('x'); + if (last_x_index == std::string::npos) { + // Scalar. + type_str = shape_and_type_str; + } else { + // Has a shape. + shape_str = shape_and_type_str.substr(0, last_x_index); + type_str = shape_and_type_str.substr(last_x_index + 1); + } + + // Populate BufferView metadata required for allocation. + BufferView result; + ASSIGN_OR_RETURN(result.element_size, GetTypeElementSize(type_str)); + ASSIGN_OR_RETURN(result.shape, ParseShape(shape_str)); + + // Allocate the host buffer. + size_t allocation_size = result.shape.element_count() * result.element_size; + if (allocator) { + ASSIGN_OR_RETURN( + result.buffer, + allocator->Allocate(MemoryType::kHostLocal | MemoryType::kDeviceVisible, + BufferUsage::kAll | BufferUsage::kConstant, + allocation_size)); + } else { + result.buffer = HeapBuffer::Allocate( + MemoryType::kHostLocal, BufferUsage::kAll | BufferUsage::kConstant, + allocation_size); + } + + if (!data_str.empty()) { + // Parse the data from the string right into the buffer. + if (IsBinaryType(type_str)) { + // Parse as binary hex. + RETURN_IF_ERROR(ParseBinaryData(data_str, result.buffer.get())); + } else { + // Parse as some nicely formatted type. + RETURN_IF_ERROR( + ParseNumericalData(type_str, data_str, result.buffer.get())); + } + } + + return result; +} + +StatusOr ParseBufferViewPrintMode(absl::string_view str) { + char str_char = str.empty() ? '?' : str[0]; + switch (str_char) { + case 'b': + return BufferViewPrintMode::kBinary; + case 'i': + return BufferViewPrintMode::kSignedInteger; + case 'u': + return BufferViewPrintMode::kUnsignedInteger; + case 'f': + return BufferViewPrintMode::kFloatingPoint; + default: + return InvalidArgumentErrorBuilder(ABSL_LOC) + << "Unsupported output type '" << str << "'"; + } +} + +StatusOr PrintBufferViewToString(const BufferView& buffer_view, + BufferViewPrintMode print_mode, + size_t max_entries) { + std::string result; + RETURN_IF_ERROR( + PrintBufferViewToString(buffer_view, print_mode, max_entries, &result)); + return result; +} + +Status PrintBufferViewToString(const BufferView& buffer_view, + BufferViewPrintMode print_mode, + size_t max_entries, std::string* out_result) { + std::ostringstream stream; + RETURN_IF_ERROR( + PrintBufferViewToStream(buffer_view, print_mode, max_entries, &stream)); + *out_result = stream.str(); + return OkStatus(); +} + +Status PrintBufferViewToStream(const BufferView& buffer_view, + BufferViewPrintMode print_mode, + size_t max_entries, std::ostream* stream) { + if (!buffer_view.buffer) { + // No buffer means the buffer_view is empty. We use the empty string to + // denote this (as we have no useful information). + return OkStatus(); + } + + // Pick a type based on the element size and the printing mode. + std::string type_str; + switch (print_mode) { + case BufferViewPrintMode::kBinary: + type_str = std::to_string(buffer_view.element_size); + break; + case BufferViewPrintMode::kSignedInteger: + absl::StrAppend(&type_str, "i", buffer_view.element_size * 8); + break; + case BufferViewPrintMode::kUnsignedInteger: + absl::StrAppend(&type_str, "u", buffer_view.element_size * 8); + break; + case BufferViewPrintMode::kFloatingPoint: + absl::StrAppend(&type_str, "f", buffer_view.element_size * 8); + break; + } + + // [shape]x[type]= prefix (taking into account scalar values). + *stream << absl::StrJoin(buffer_view.shape.begin(), buffer_view.shape.end(), + "x"); + if (!buffer_view.shape.empty()) *stream << "x"; + *stream << type_str; + *stream << "="; + + if (IsBinaryType(type_str)) { + return PrintBinaryData(buffer_view.element_size, buffer_view.buffer.get(), + max_entries, stream); + } else { + return PrintNumericalData(buffer_view.shape, type_str, + buffer_view.buffer.get(), max_entries, stream); + } +} + +} // namespace hal +} // namespace iree diff --git a/hal/buffer_view_string_util.h b/hal/buffer_view_string_util.h new file mode 100644 index 000000000000..b2ce12cf88ff --- /dev/null +++ b/hal/buffer_view_string_util.h @@ -0,0 +1,95 @@ +// Copyright 2019 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// Utilities for working with BufferView data, mostly useful for testing. +// These functions allow for conversion between types, parsing and printing, and +// basic comparisons. +// +// The canonical BufferView string format is: +// [shape]x[type]=value,value,... +// For example: +// 2x2xi32=0,1,2,3 +// Characters like [] are optional and will be ignored during parsing: +// 2x2xi32=[[0 1][2 3]] +// +// The type may be one of the following: +// * 1/2/4/8 = 1/2/4/8 byte elements in binary hex format. +// * i8/u8 = signed/unsigned 8-bit integers. +// * i16/u16 = signed/unsigned 16-bit integers. +// * i32/u32 = signed/unsigned 32-bit integers. +// * i64/u64 = signed/unsigned 64-bit integers. +// * f32 = 32-bit floating-point number. +// * f64 = 64-bit floating-point number. + +#ifndef THIRD_PARTY_MLIR_EDGE_IREE_HAL_BUFFER_VIEW_STRING_UTIL_H_ +#define THIRD_PARTY_MLIR_EDGE_IREE_HAL_BUFFER_VIEW_STRING_UTIL_H_ + +#include +#include + +#include "third_party/absl/strings/string_view.h" +#include "third_party/mlir_edge/iree/base/status.h" +#include "third_party/mlir_edge/iree/hal/allocator.h" +#include "third_party/mlir_edge/iree/hal/buffer_view.h" + +namespace iree { +namespace hal { + +// Returns the size, in bytes, of the given type. +StatusOr GetTypeElementSize(absl::string_view type_str); + +// Returns a Shape parsed from the given NxMx... string. +StatusOr ParseShape(absl::string_view shape_str); + +// Parses a BufferView encoded in a string. +// If an |allocator| is provided the buffer will be allocated as host-local and +// device-visible. Otherwise, buffers will be host-local. +// The format accepted matches that produced by PrintBufferViewToString. +StatusOr ParseBufferViewFromString( + absl::string_view buffer_view_str, hal::Allocator* allocator = nullptr); + +// Defines how the elements within a BufferView are interpreted during printing. +enum class BufferViewPrintMode { + // Interpret the data as if it were serialized bytes. + // In this mode no conversion is performed and the bytes in memory are printed + // as hex in groupings based on the element size. Shortened to 'b'. + kBinary, + // Interpret elements as signed integers; shortened to 'i'. + kSignedInteger, + // Interpret elements as unsigned integers; shortened to 'u'. + kUnsignedInteger, + // Interpret elements as floating-point values; shortened to 'f'. + kFloatingPoint, +}; + +// Returns the BufferViewPrintMode based on the shortened char in |str|. +StatusOr ParseBufferViewPrintMode(absl::string_view str); + +// Prints a BufferView to a string encoded in the canonical format. +StatusOr PrintBufferViewToString(const BufferView& buffer_view, + BufferViewPrintMode print_mode, + size_t max_entries); +Status PrintBufferViewToString(const BufferView& buffer_view, + BufferViewPrintMode print_mode, + size_t max_entries, std::string* out_result); + +// Prints a BufferView to a string stream encoded in the canonical format. +Status PrintBufferViewToStream(const BufferView& buffer_view, + BufferViewPrintMode print_mode, + size_t max_entries, std::ostream* stream); + +} // namespace hal +} // namespace iree + +#endif // THIRD_PARTY_MLIR_EDGE_IREE_HAL_BUFFER_VIEW_STRING_UTIL_H_ diff --git a/hal/command_buffer.cc b/hal/command_buffer.cc new file mode 100644 index 000000000000..6856316de00e --- /dev/null +++ b/hal/command_buffer.cc @@ -0,0 +1,29 @@ +// Copyright 2019 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "third_party/mlir_edge/iree/hal/command_buffer.h" + +namespace iree { +namespace hal { + +std::string CommandCategoryString(CommandCategoryBitfield categories) { + return FormatBitfieldValue(categories, + { + {CommandCategory::kTransfer, "kTransfer"}, + {CommandCategory::kDispatch, "kDispatch"}, + }); +} + +} // namespace hal +} // namespace iree diff --git a/hal/command_buffer.h b/hal/command_buffer.h new file mode 100644 index 000000000000..c6cb2fe576b0 --- /dev/null +++ b/hal/command_buffer.h @@ -0,0 +1,383 @@ +// Copyright 2019 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#ifndef THIRD_PARTY_MLIR_EDGE_IREE_HAL_COMMAND_BUFFER_H_ +#define THIRD_PARTY_MLIR_EDGE_IREE_HAL_COMMAND_BUFFER_H_ + +#include + +#include "third_party/mlir_edge/iree/base/bitfield.h" +#include "third_party/mlir_edge/iree/base/shape.h" +#include "third_party/mlir_edge/iree/base/status.h" +#include "third_party/mlir_edge/iree/hal/allocator.h" +#include "third_party/mlir_edge/iree/hal/buffer.h" +#include "third_party/mlir_edge/iree/hal/buffer_view.h" +#include "third_party/mlir_edge/iree/hal/event.h" +#include "third_party/mlir_edge/iree/hal/executable.h" +#include "third_party/mlir_edge/iree/hal/resource.h" + +namespace iree { +namespace hal { + +// A bitfield specifying the mode of operation for a command buffer. +enum class CommandBufferMode : uint32_t { + // Command buffer will be submitted once and never used again. + // This may enable in-place patching of command buffers that reduce overhead + // when it's known that command buffers will not be reused. + kOneShot = 1 << 0, +}; +IREE_BITFIELD(CommandBufferMode); +using CommandBufferModeBitfield = CommandBufferMode; +std::string CommandBufferModeString(CommandBufferModeBitfield mode); + +// A bitfield specifying the category of commands in a command queue. +enum class CommandCategory : uint32_t { + // Command is considered a transfer operation (memcpy, etc). + kTransfer = 1 << 0, + // Command is considered a dispatch operation (dispatch/execute). + kDispatch = 1 << 1, +}; +IREE_BITFIELD(CommandCategory); +using CommandCategoryBitfield = CommandCategory; +std::string CommandCategoryString(CommandCategoryBitfield categories); + +// Bitfield specifying which execution stage a brarrier should start/end at. +// +// Maps to VkPipelineStageFlagBits. +enum class ExecutionStage : uint32_t { + // Top of the pipeline when commands are initially issued by the device. + kCommandIssue = 1 << 0, + // Stage of the pipeline when dispatch parameter data is consumed. + kCommandProcess = 1 << 1, + // Stage where dispatch commands execute. + kDispatch = 1 << 2, + // Stage where transfer (copy/clear/fill/etc) commands execute. + kTransfer = 1 << 3, + // Final stage in the pipeline when commands are retired on the device. + kCommandRetire = 1 << 4, + // Pseudo-stage for read/writes by the host. Not executed on device. + kHost = 1 << 5, +}; +IREE_BITFIELD(ExecutionStage); +using ExecutionStageBitfield = ExecutionStage; + +// Bitfield specifying which scopes will access memory and how. +// +// Maps to VkAccessFlagBits. +enum class AccessScope : uint32_t { + // Read access to indirect command data as part of an indirect dispatch. + kIndirectCommandRead = 1 << 0, + // Constant uniform buffer reads by the device. + kConstantRead = 1 << 1, + // Storage buffer reads by dispatch commands. + kDispatchRead = 1 << 2, + // Storage buffer writes by dispatch commands. + kDispatchWrite = 1 << 3, + // Source of a transfer operation. + kTransferRead = 1 << 4, + // Target of a transfer operation. + kTransferWrite = 1 << 5, + // Read operation by the host through mapped memory. + kHostRead = 1 << 6, + // Write operation by the host through mapped memory. + kHostWrite = 1 << 7, + // External/non-specific read. + kMemoryRead = 1 << 8, + // External/non-specific write. + kMemoryWrite = 1 << 9, +}; +IREE_BITFIELD(AccessScope); +using AccessScopeBitfield = AccessScope; + +// Defines a global memory barrier. +// These are cheaper to encode than buffer-specific barriers but may cause +// stalls and bubbles in device pipelines if applied too broadly. Prefer them +// over equivalently large sets of buffer-specific barriers (such as when +// completely changing execution contexts). +// +// Maps to VkMemoryBarrier. +struct MemoryBarrier { + // All access scopes prior-to the barrier (inclusive). + AccessScopeBitfield source_scope; + // All access scopes following the barrier (inclusive). + AccessScopeBitfield target_scope; +}; + +// Defines a memory barrier that applies to a range of a specific buffer. +// Use of these (vs. global memory barriers) provides fine-grained execution +// ordering to device command processors and allows for more aggressive +// reordering. +// +// Maps to VkBufferMemoryBarrier. +struct BufferBarrier { + // All access scopes prior-to the barrier (inclusive). + AccessScopeBitfield source_scope; + // All access scopes following the barrier (inclusive). + AccessScopeBitfield target_scope; + // Buffer the barrier is restricted to. + // The barrier will apply to the entire physical device allocation. + Buffer* buffer = nullptr; + // Relative offset/length within |buffer| (which may itself be mapped into the + // device allocation at an offset). + device_size_t offset = 0; + device_size_t length = kWholeBuffer; +}; + +// Represents a binding to a buffer with a set of attributes. +// This may be used by drivers to validate alignment. +struct BufferBinding { + // Access rights of the buffer contents by the executable. + MemoryAccessBitfield access = MemoryAccess::kAll; + + // The buffer this binding references. + // The buffer is not retained by the binding and must be kept alive externally + // for the duration it is in use by the queue. + Buffer* buffer = nullptr; + + // Shape of the buffer contents. + Shape shape; + + // Size of each element within the buffer, in bytes. + int8_t element_size = 0; + + BufferBinding() = default; + BufferBinding(MemoryAccessBitfield access, Buffer* buffer) + : access(access), buffer(buffer) {} + BufferBinding(MemoryAccessBitfield access, Buffer* buffer, Shape shape, + int8_t element_size) + : access(access), + buffer(buffer), + shape(shape), + element_size(element_size) {} + BufferBinding(MemoryAccessBitfield access, const BufferView& buffer_view) + : access(access), + buffer(buffer_view.buffer.get()), + shape(buffer_view.shape), + element_size(buffer_view.element_size) {} +}; + +// Wraps parameters for a Dispatch request. +struct DispatchRequest { + // Executable prepared for use on the device. + // The executable must remain alive until all in-flight dispatch requests + // that use it have completed. + Executable* executable = nullptr; + + // Executable entry point ordinal. + int entry_point = 0; + + // TODO(benvanik): predication. + + // Static workload parameters defining the X, Y, and Z workgroup counts. + std::array workload; + + // An optional buffer containing the dynamic workload to dispatch. + // The contents need not be available at the time of recording but must be + // made visible prior to execution of the dispatch command. + // + // Buffer contents are expected to be 3 int32 values defining the X, Y, and Z + // workgroup counts. + // + // The buffer must have been allocated with BufferUsage::kDispatch and be + // of MemoryType::kDeviceVisible. + Buffer* workload_buffer = nullptr; + + // A list of buffers that contain the execution inputs/outputs. + // Order is dependent on executable arg layout. + // + // Buffers must have been allocated with BufferUsage::kDispatch and be + // of MemoryType::kDeviceVisible. + absl::Span bindings; + + // TODO(benvanik): push-constant equivalent (uniforms, etc). +}; + +// Asynchronous command buffer recording interface. +// Commands are recorded by the implementation for later submission to command +// queues. +// +// Buffers and synchronization objects referenced must remain valid and not be +// modified or read while there are commands in-flight. The usual flow is to +// populate input buffers, Dispatch using those buffers, wait on a Fence until +// the buffers are guaranteed to no longer be in use, and then reuse or release +// the buffers. +// +// Errors that can be recognized when operations are enqueued will be returned +// immediately, such as invalid argument errors. Errors that can only be +// determined at execution time will be returned on fences. Once a failure +// occurs the device queue will enter an error state that invalidates all +// operations on the device queue (as ordering is not strict and any may still +// be in-flight). In this case the user of the device queue should treat all +// in-flight operations as cancelled and fully reset themselves. Other device +// queues that may be waiting on events from the device queue will also enter +// error states. Only once a user has acknowledged and cleared the error state +// with a Reset the queue will become usable, and otherwise all operations will +// return errors. +// +// Command buffers are thread-compatible. Use multiple command buffers if trying +// to record commands from multiple threads. Command buffers must not be mutated +// between when they have are submitted for execution on a queue and when the +// fence fires indicating the completion of their execution. +class CommandBuffer : public Resource { + public: + virtual CommandBuffer* impl() { return this; } + + // Device allocator that commands encoded into the buffer share compatibility + // with. + Allocator* allocator() const { return allocator_; } + + // Command buffer operation mode. + CommandBufferModeBitfield mode() const { return mode_; } + + // Command categories that may be recorded into the buffer. + CommandCategoryBitfield command_categories() const { + return command_categories_; + } + + // True if the command buffer is between a Begin/End recording block. + virtual bool is_recording() const = 0; + + // Resets and begins recording into the command buffer, clearing all + // previously recorded contents. + // The command buffer must not be in-flight. + virtual Status Begin() = 0; + + // Ends recording into the command buffer. + // This must be called prior to submitting the command buffer for execution. + virtual Status End() = 0; + + // TODO(benvanik): annotations for debugging and tracing: + // enter/exit + // stack frame manipulation + // explicit timers? or profiling buffer? + + // TODO(b/138719910): cross-queue and external acquire/release. + // virtual Status AcquireBuffer() = 0; + // virtual Status ReleaseBuffer() = 0; + + // Defines a memory dependency between commands recorded before and after the + // barrier. One or more memory or buffer barriers can be specified to indicate + // between which stages or buffers the dependencies exist. + virtual Status ExecutionBarrier( + ExecutionStageBitfield source_stage_mask, + ExecutionStageBitfield target_stage_mask, + absl::Span memory_barriers, + absl::Span buffer_barriers) = 0; + + // Sets an event to the signaled state. + // |source_stage_mask| specifies when the event is signaled. + // + // Events are only valid within a single command buffer. Events can only be + // used on non-transfer queues. + virtual Status SignalEvent(Event* event, + ExecutionStageBitfield source_stage_mask) = 0; + + // Resets an event to the non-signaled state. + // |source_stage_mask| specifies when the event is unsignaled. + // + // Events are only valid within a single command buffer. Events can only be + // used on non-transfer queues. + virtual Status ResetEvent(Event* event, + ExecutionStageBitfield source_stage_mask) = 0; + + // Waits for one or more events to be signaled and defines a memory dependency + // between the synchronization scope of the signal operations and the commands + // following the wait. + // + // |source_stage_mask| must include ExecutionStage::kHost for Event::Signal to + // be visibile. + // + // Events are only valid within a single command buffer. Events remain + // signaled even after waiting and must be reset to be reused. Events can only + // be used on non-transfer queues. + virtual Status WaitEvents( + absl::Span events, ExecutionStageBitfield source_stage_mask, + ExecutionStageBitfield target_stage_mask, + absl::Span memory_barriers, + absl::Span buffer_barriers) = 0; + + // Fills the target buffer with the given repeating value. + // Expects that value_length is one of 1, 2, or 4 and that the offset and + // length are aligned to the natural alignment of the value. + // The target buffer must be compatible with the devices owned by this + // device queue and be allocated with BufferUsage::kTransfer. + virtual Status FillBuffer(Buffer* target_buffer, device_size_t target_offset, + device_size_t length, const void* pattern, + size_t pattern_length) = 0; + + // Hints to the device queue that the given buffer will not be used again. + // After encoding a discard the buffer contents will be considered undefined. + // This is because the discard may be used to elide write backs to host memory + // or aggressively reuse the allocation for other purposes. + // + // For buffers allocated with MemoryType::kTransient this may allow + // the device queue to reclaim the memory used by the buffer earlier than + // otherwise possible. + virtual Status DiscardBuffer(Buffer* buffer) = 0; + + // Updates a range of the given target buffer from the source host memory. + // The source host memory is copied immediately into the command buffer and + // occupies command buffer space. It is strongly recommended that large buffer + // updates are performed via CopyBuffer where there is the possibility of a + // zero-copy path. + // The |source_buffer| may be releaed by the caller immediately after this + // call returns. + // The |target_buffer| must be compatible with the devices owned by this + // device queue and be allocated with BufferUsage::kTransfer. + virtual Status UpdateBuffer(const void* source_buffer, + device_size_t source_offset, + Buffer* target_buffer, + device_size_t target_offset, + device_size_t length) = 0; + + // Copies a range of one buffer to another. + // Both buffers must be compatible with the devices owned by this device + // queue and be allocated with BufferUsage::kTransfer. Though the source and + // target buffer may be the same the ranges must not overlap (as with memcpy). + // + // This can be used to perform device->host, host->device, and device->device + // copies. + virtual Status CopyBuffer(Buffer* source_buffer, device_size_t source_offset, + Buffer* target_buffer, device_size_t target_offset, + device_size_t length) = 0; + + // Dispatches an execution request. + // The request may execute overlapped with any other transfer operation or + // dispatch made within the same barrier-defined sequence. + // + // The executable specified must be registered for use with the device driver + // owning this queue. It must not be unregistered until all requests that use + // it have completed. + // + // Fails if the queue does not support dispatch operations (as indicated by + // can_dispatch). + virtual Status Dispatch(const DispatchRequest& dispatch_request) = 0; + + protected: + CommandBuffer(Allocator* allocator, CommandBufferModeBitfield mode, + CommandCategoryBitfield command_categories) + : allocator_(allocator), + mode_(mode), + command_categories_(command_categories) {} + + private: + Allocator* const allocator_; + const CommandBufferModeBitfield mode_; + const CommandCategoryBitfield command_categories_; +}; + +} // namespace hal +} // namespace iree + +#endif // THIRD_PARTY_MLIR_EDGE_IREE_HAL_COMMAND_BUFFER_H_ diff --git a/hal/command_buffer_validation.cc b/hal/command_buffer_validation.cc new file mode 100644 index 000000000000..701dd567e8b9 --- /dev/null +++ b/hal/command_buffer_validation.cc @@ -0,0 +1,403 @@ +// Copyright 2019 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "third_party/mlir_edge/iree/hal/command_buffer_validation.h" + +#include "third_party/mlir_edge/iree/base/logging.h" +#include "third_party/mlir_edge/iree/base/status.h" + +namespace iree { +namespace hal { + +namespace { + +// Command buffer validation shim. +// Wraps an existing command buffer to provide in-depth validation during +// recording. This should be enabled whenever the command buffer is being driven +// by unsafe code or when early and readable diagnostics are needed. +class ValidatingCommandBuffer : public CommandBuffer { + public: + explicit ValidatingCommandBuffer(ref_ptr impl); + ~ValidatingCommandBuffer() override; + + CommandBuffer* impl() { return impl_.get(); } + + bool is_recording() const override; + + Status Begin() override; + Status End() override; + + Status ExecutionBarrier( + ExecutionStageBitfield source_stage_mask, + ExecutionStageBitfield target_stage_mask, + absl::Span memory_barriers, + absl::Span buffer_barriers) override; + Status SignalEvent(Event* event, + ExecutionStageBitfield source_stage_mask) override; + Status ResetEvent(Event* event, + ExecutionStageBitfield source_stage_mask) override; + Status WaitEvents(absl::Span events, + ExecutionStageBitfield source_stage_mask, + ExecutionStageBitfield target_stage_mask, + absl::Span memory_barriers, + absl::Span buffer_barriers) override; + Status FillBuffer(Buffer* target_buffer, device_size_t target_offset, + device_size_t length, const void* pattern, + size_t pattern_length) override; + Status DiscardBuffer(Buffer* buffer) override; + Status UpdateBuffer(const void* source_buffer, device_size_t source_offset, + Buffer* target_buffer, device_size_t target_offset, + device_size_t length) override; + Status CopyBuffer(Buffer* source_buffer, device_size_t source_offset, + Buffer* target_buffer, device_size_t target_offset, + device_size_t length) override; + Status Dispatch(const DispatchRequest& dispatch_request) override; + + private: + // Returns a failure if the queue does not support the given caps. + Status ValidateCategories(CommandCategoryBitfield required_categories) const; + // Returns a failure if the memory type the buffer was allocated from is not + // compatible with the given type. + Status ValidateCompatibleMemoryType(Buffer* buffer, + MemoryTypeBitfield memory_type) const; + // Returns a failure if the buffer memory type or usage disallows the given + // access type. + Status ValidateAccess(Buffer* buffer, + MemoryAccessBitfield memory_access) const; + // Returns a failure if the buffer was not allocated for the given usage. + Status ValidateUsage(Buffer* buffer, BufferUsageBitfield usage) const; + // Validates that the range provided is within the given buffer. + Status ValidateRange(Buffer* buffer, device_size_t byte_offset, + device_size_t byte_length) const; + + ref_ptr impl_; +}; + +ValidatingCommandBuffer::ValidatingCommandBuffer(ref_ptr impl) + : CommandBuffer(impl->allocator(), impl->mode(), + impl->command_categories()), + impl_(std::move(impl)) {} + +ValidatingCommandBuffer::~ValidatingCommandBuffer() = default; + +bool ValidatingCommandBuffer::is_recording() const { + return impl_->is_recording(); +} + +Status ValidatingCommandBuffer::Begin() { + DVLOG(3) << "CommandBuffer::Begin()"; + if (impl_->is_recording()) { + return FailedPreconditionErrorBuilder(ABSL_LOC) + << "Command buffer is already recording"; + } + return impl_->Begin(); +} + +Status ValidatingCommandBuffer::End() { + DVLOG(3) << "CommandBuffer::End()"; + if (!impl_->is_recording()) { + return FailedPreconditionErrorBuilder(ABSL_LOC) + << "Command buffer is not recording"; + } + return impl_->End(); +} + +Status ValidatingCommandBuffer::ValidateCategories( + CommandCategoryBitfield required_categories) const { + if (!AllBitsSet(command_categories(), required_categories)) { + return FailedPreconditionErrorBuilder(ABSL_LOC) + << "Operation requires categories " + << CommandCategoryString(required_categories) + << " but buffer only supports " + << CommandCategoryString(command_categories()); + } + return OkStatus(); +} + +Status ValidatingCommandBuffer::ValidateCompatibleMemoryType( + Buffer* buffer, MemoryTypeBitfield memory_type) const { + if ((buffer->memory_type() & memory_type) != memory_type) { + // Missing one or more bits. + return PermissionDeniedErrorBuilder(ABSL_LOC) + << "Buffer memory type is not compatible with the requested " + "operation; buffer has " + << MemoryTypeString(buffer->memory_type()) << ", operation requires " + << MemoryTypeString(memory_type); + } + return OkStatus(); +} + +Status ValidatingCommandBuffer::ValidateAccess( + Buffer* buffer, MemoryAccessBitfield memory_access) const { + if ((buffer->allowed_access() & memory_access) != memory_access) { + // Bits must match exactly. + return PermissionDeniedErrorBuilder(ABSL_LOC) + << "The buffer does not support the requested access type; buffer " + "allows " + << MemoryAccessString(buffer->allowed_access()) + << ", operation requires " << MemoryAccessString(memory_access); + } + return OkStatus(); +} + +// Returns a failure if the buffer was not allocated for the given usage. +Status ValidatingCommandBuffer::ValidateUsage(Buffer* buffer, + BufferUsageBitfield usage) const { + if (!allocator()->CanUseBuffer(buffer, usage)) { + // Buffer cannot be used on the queue for the given usage. + return PermissionDeniedErrorBuilder(ABSL_LOC) + << "Requested usage of " << buffer->DebugString() + << " is not supported for the buffer on this queue; " + "buffer allows " + << BufferUsageString(buffer->usage()) << ", queue requires " + << BufferUsageString(usage); + } + + if ((buffer->usage() & usage) != usage) { + // Missing one or more bits. + return PermissionDeniedErrorBuilder(ABSL_LOC) + << "Requested usage was not specified when the buffer was " + "allocated; buffer allows " + << BufferUsageString(buffer->usage()) << ", operation requires " + << BufferUsageString(usage); + } + + return OkStatus(); +} + +// Validates that the range provided is within the given buffer. +Status ValidatingCommandBuffer::ValidateRange(Buffer* buffer, + device_size_t byte_offset, + device_size_t byte_length) const { + // Check if the start of the range runs off the end of the buffer. + if (byte_offset > buffer->byte_length()) { + return OutOfRangeErrorBuilder(ABSL_LOC) + << "Attempted to access an address off the end of the valid buffer " + "range (offset=" + << byte_offset << ", length=" << byte_length + << ", buffer byte_length=" << buffer->byte_length() << ")"; + } + + if (byte_length == 0) { + // Fine to have a zero length. + return OkStatus(); + } + + // Check if the end runs over the allocation. + device_size_t end = byte_offset + byte_length; + if (end > buffer->byte_length()) { + return OutOfRangeErrorBuilder(ABSL_LOC) + << "Attempted to access an address outside of the valid buffer " + "range (offset=" + << byte_offset << ", length=" << byte_length + << ", end(inc)=" << (end - 1) + << ", buffer byte_length=" << buffer->byte_length() << ")"; + } + + return OkStatus(); +} + +Status ValidatingCommandBuffer::ExecutionBarrier( + ExecutionStageBitfield source_stage_mask, + ExecutionStageBitfield target_stage_mask, + absl::Span memory_barriers, + absl::Span buffer_barriers) { + DVLOG(3) << "CommandBuffer::ExecutionBarrier(...)"; + + // TODO(benvanik): additional synchronization validation. + RETURN_IF_ERROR(ValidateCategories(CommandCategory::kTransfer | + CommandCategory::kDispatch)); + + return impl_->ExecutionBarrier(source_stage_mask, target_stage_mask, + memory_barriers, buffer_barriers); +} + +Status ValidatingCommandBuffer::SignalEvent( + Event* event, ExecutionStageBitfield source_stage_mask) { + DVLOG(3) << "CommandBuffer::SignalEvent(...)"; + + // TODO(benvanik): additional synchronization validation. + RETURN_IF_ERROR(ValidateCategories(CommandCategory::kDispatch)); + + return impl_->SignalEvent(event, source_stage_mask); +} + +Status ValidatingCommandBuffer::ResetEvent( + Event* event, ExecutionStageBitfield source_stage_mask) { + DVLOG(3) << "CommandBuffer::ResetEvent(...)"; + + // TODO(benvanik): additional synchronization validation. + RETURN_IF_ERROR(ValidateCategories(CommandCategory::kDispatch)); + + return impl_->ResetEvent(event, source_stage_mask); +} + +Status ValidatingCommandBuffer::WaitEvents( + absl::Span events, ExecutionStageBitfield source_stage_mask, + ExecutionStageBitfield target_stage_mask, + absl::Span memory_barriers, + absl::Span buffer_barriers) { + DVLOG(3) << "CommandBuffer::WaitEvents(...)"; + + // TODO(benvanik): additional synchronization validation. + RETURN_IF_ERROR(ValidateCategories(CommandCategory::kDispatch)); + + return impl_->WaitEvents(events, source_stage_mask, target_stage_mask, + memory_barriers, buffer_barriers); +} + +Status ValidatingCommandBuffer::FillBuffer(Buffer* target_buffer, + device_size_t target_offset, + device_size_t length, + const void* pattern, + size_t pattern_length) { + DVLOG(3) << "CommandBuffer::FillBuffer(" << target_buffer->DebugString() + << ", " << target_offset << ", " << length << ", ??, " + << pattern_length << ")"; + + RETURN_IF_ERROR(ValidateCategories(CommandCategory::kTransfer)); + RETURN_IF_ERROR( + ValidateCompatibleMemoryType(target_buffer, MemoryType::kDeviceVisible)); + RETURN_IF_ERROR(ValidateAccess(target_buffer, MemoryAccess::kWrite)); + RETURN_IF_ERROR(ValidateUsage(target_buffer, BufferUsage::kTransfer)); + RETURN_IF_ERROR(ValidateRange(target_buffer, target_offset, length)); + + // Ensure the value length is supported. + if (pattern_length != 1 && pattern_length != 2 && pattern_length != 4) { + return InvalidArgumentErrorBuilder(ABSL_LOC) + << "Fill value length is not one of the supported values " + "(pattern_length=" + << pattern_length << ")"; + } + + // Ensure the offset and length have an alignment matching the value length. + if ((target_offset % pattern_length) != 0 || (length % pattern_length) != 0) { + return InvalidArgumentErrorBuilder(ABSL_LOC) + << "Fill offset and/or length do not match the natural alignment of " + "the fill value (target_offset=" + << target_offset << ", length=" << length + << ", pattern_length=" << pattern_length << ")"; + } + + return impl_->FillBuffer(target_buffer, target_offset, length, pattern, + pattern_length); +} + +Status ValidatingCommandBuffer::DiscardBuffer(Buffer* buffer) { + DVLOG(3) << "CommandBuffer::DiscardBuffer(" << buffer->DebugString() << ")"; + + RETURN_IF_ERROR(ValidateCategories(CommandCategory::kTransfer)); + RETURN_IF_ERROR( + ValidateCompatibleMemoryType(buffer, MemoryType::kDeviceVisible)); + RETURN_IF_ERROR(ValidateUsage(buffer, BufferUsage::kNone)); + + return impl_->DiscardBuffer(buffer); +} + +Status ValidatingCommandBuffer::UpdateBuffer(const void* source_buffer, + device_size_t source_offset, + Buffer* target_buffer, + device_size_t target_offset, + device_size_t length) { + DVLOG(3) << "CommandBuffer::UpdateBuffer(" << source_buffer << ", " + << source_offset << ", " << target_buffer->DebugString() << ", " + << target_offset << ", " << length << ")"; + + RETURN_IF_ERROR(ValidateCategories(CommandCategory::kTransfer)); + RETURN_IF_ERROR( + ValidateCompatibleMemoryType(target_buffer, MemoryType::kDeviceVisible)); + RETURN_IF_ERROR(ValidateAccess(target_buffer, MemoryAccess::kWrite)); + RETURN_IF_ERROR(ValidateUsage(target_buffer, BufferUsage::kTransfer)); + RETURN_IF_ERROR(ValidateRange(target_buffer, target_offset, length)); + + return impl_->UpdateBuffer(source_buffer, source_offset, target_buffer, + target_offset, length); +} + +Status ValidatingCommandBuffer::CopyBuffer(Buffer* source_buffer, + device_size_t source_offset, + Buffer* target_buffer, + device_size_t target_offset, + device_size_t length) { + DVLOG(3) << "CommandBuffer::CopyBuffer(" << source_buffer->DebugString() + << ", " << source_offset << ", " << target_buffer->DebugString() + << ", " << target_offset << ", " << length << ")"; + + RETURN_IF_ERROR(ValidateCategories(CommandCategory::kTransfer)); + + // At least source or destination must be device-visible to enable + // host->device, device->host, and device->device. + // TODO(b/117338171): host->host copies. + if (!AnyBitSet(source_buffer->memory_type() & MemoryType::kDeviceVisible) && + !AnyBitSet(target_buffer->memory_type() & MemoryType::kDeviceVisible)) { + return PermissionDeniedErrorBuilder(ABSL_LOC) + << "At least one buffer must be device-visible for a copy; " + "source_buffer=" + << MemoryTypeString(source_buffer->memory_type()) + << ", target_buffer=" + << MemoryTypeString(target_buffer->memory_type()); + } + + RETURN_IF_ERROR(ValidateAccess(source_buffer, MemoryAccess::kRead)); + RETURN_IF_ERROR(ValidateAccess(target_buffer, MemoryAccess::kWrite)); + RETURN_IF_ERROR(ValidateUsage(source_buffer, BufferUsage::kTransfer)); + RETURN_IF_ERROR(ValidateUsage(target_buffer, BufferUsage::kTransfer)); + RETURN_IF_ERROR(ValidateRange(source_buffer, source_offset, length)); + RETURN_IF_ERROR(ValidateRange(target_buffer, target_offset, length)); + + // Check for overlap - just like memcpy we don't handle that. + if (Buffer::TestOverlap(source_buffer, source_offset, length, target_buffer, + target_offset, + length) != Buffer::Overlap::kDisjoint) { + return InvalidArgumentErrorBuilder(ABSL_LOC) + << "Source and target ranges overlap within the same buffer"; + } + + return impl_->CopyBuffer(source_buffer, source_offset, target_buffer, + target_offset, length); +} + +Status ValidatingCommandBuffer::Dispatch( + const DispatchRequest& dispatch_request) { + DVLOG(3) << "CommandBuffer::Dispatch(?)"; + + RETURN_IF_ERROR(ValidateCategories(CommandCategory::kDispatch)); + + // Validate all buffers referenced have compatible memory types, access + // rights, and usage. + for (const auto& binding : dispatch_request.bindings) { + RETURN_IF_ERROR(ValidateCompatibleMemoryType(binding.buffer, + MemoryType::kDeviceVisible)) + << "input buffer: " << MemoryAccessString(binding.access) << " " + << binding.buffer->DebugStringShort(); + RETURN_IF_ERROR(ValidateAccess(binding.buffer, binding.access)); + RETURN_IF_ERROR(ValidateUsage(binding.buffer, BufferUsage::kDispatch)); + // TODO(benvanik): validate it matches the executable expectations. + // TODO(benvanik): validate buffer contains enough data for shape+size. + } + + // TODO(benvanik): validate no aliasing? + + return impl_->Dispatch(dispatch_request); +} + +} // namespace + +ref_ptr WrapCommandBufferWithValidation( + ref_ptr impl) { + return make_ref(std::move(impl)); +} + +} // namespace hal +} // namespace iree diff --git a/hal/command_buffer_validation.h b/hal/command_buffer_validation.h new file mode 100644 index 000000000000..42a7d8dbffc8 --- /dev/null +++ b/hal/command_buffer_validation.h @@ -0,0 +1,32 @@ +// Copyright 2019 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#ifndef THIRD_PARTY_MLIR_EDGE_IREE_HAL_COMMAND_BUFFER_VALIDATION_H_ +#define THIRD_PARTY_MLIR_EDGE_IREE_HAL_COMMAND_BUFFER_VALIDATION_H_ + +#include "third_party/mlir_edge/iree/hal/command_buffer.h" + +namespace iree { +namespace hal { + +// Wraps an existing command buffer to provide in-depth validation during +// recording. This should be enabled whenever the command buffer is being driven +// by unsafe code or when early and readable diagnostics are needed. +ref_ptr WrapCommandBufferWithValidation( + ref_ptr impl); + +} // namespace hal +} // namespace iree + +#endif // THIRD_PARTY_MLIR_EDGE_IREE_HAL_COMMAND_BUFFER_VALIDATION_H_ diff --git a/hal/command_queue.h b/hal/command_queue.h new file mode 100644 index 000000000000..9df2a493abd2 --- /dev/null +++ b/hal/command_queue.h @@ -0,0 +1,126 @@ +// Copyright 2019 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#ifndef THIRD_PARTY_MLIR_EDGE_IREE_HAL_COMMAND_QUEUE_H_ +#define THIRD_PARTY_MLIR_EDGE_IREE_HAL_COMMAND_QUEUE_H_ + +#include +#include + +#include "third_party/absl/time/clock.h" +#include "third_party/absl/time/time.h" +#include "third_party/absl/types/span.h" +#include "third_party/mlir_edge/iree/base/bitfield.h" +#include "third_party/mlir_edge/iree/base/status.h" +#include "third_party/mlir_edge/iree/base/time.h" +#include "third_party/mlir_edge/iree/hal/command_buffer.h" +#include "third_party/mlir_edge/iree/hal/fence.h" +#include "third_party/mlir_edge/iree/hal/semaphore.h" + +namespace iree { +namespace hal { + +// A batch of command buffers with synchronization information for submission. +struct SubmissionBatch { + // Semaphores that must be signaled prior to the execution of any command + // buffer in this submission. For TimelineSemaphores the specified payload + // must be reached or exceeded. + absl::Span wait_semaphores; + + // Command buffers that will execute in this batch. + // The command buffers will begin execution in order but may complete out of + // order. + absl::Span command_buffers; + + // Semaphores to signal after execution of all command buffers complete. + // TimelineSemaphores will be set to the maximum of the specified payload or + // their current payload. + absl::Span signal_semaphores; +}; + +// Asynchronous command execution queue. +// +// CommandQueues may capture device status at Fence barriers, including +// information about device state such as thermal throttling. This information +// is a snapshot of the state at the time the fence was signaled and not +// necessarily live at the time of the application query. +// +// Command queues are thread-safe and submissions may occur from multiple +// threads. +class CommandQueue { + public: + virtual ~CommandQueue() = default; + + // Name of the queue used for logging purposes. + // Try to keep at 4 characters total for prettier logging. + const std::string& name() const { return name_; } + + // Capabilities of the command queue. + CommandCategoryBitfield supported_categories() const { + return supported_categories_; + } + + // Whether this queue may be used for transfer commands. + bool can_transfer() const { + return AllBitsSet(supported_categories_, CommandCategory::kTransfer); + } + + // Whether this queue may be used for dispatch commands. + bool can_dispatch() const { + return AllBitsSet(supported_categories_, CommandCategory::kDispatch); + } + + // Submits one or more command batches for execution on the queue. + // Dependencies between |batches| on BinarySemaphores must be sorted in order + // such that all semaphores are signaled prior to any waits on them. + // Dependencies between TimelineSemaphores may occur in any order. + // + // The provided |fence| will be signaled when all |batches| have retired. + virtual Status Submit(absl::Span batches, + FenceValue fence) = 0; + inline Status Submit(const SubmissionBatch& batch, FenceValue fence) { + return Submit(absl::MakeConstSpan(&batch, 1), std::move(fence)); + } + + // Flushes any requests that are pending with a queue and ensure they begin + // executing ASAP. May be a no-op. + // + // If the command queue has encountered an error during submission at any + // point it will be returned here (repeatedly). + virtual Status Flush() = 0; + + // Blocks until all outstanding requests have been completed. + // This is equivalent to having waited on all outstanding fences. + // Implicitly calls Flush to ensure delayed requests are scheduled. + // + // If the command queue has encountered an error during submission at any + // point it will be returned here (repeatedly). + virtual Status WaitIdle(absl::Time deadline) = 0; + inline Status WaitIdle(absl::Duration timeout) { + return WaitIdle(RelativeTimeoutToDeadline(timeout)); + } + inline Status WaitIdle() { return WaitIdle(absl::InfiniteFuture()); } + + protected: + CommandQueue(std::string name, CommandCategoryBitfield supported_categories) + : name_(std::move(name)), supported_categories_(supported_categories) {} + + const std::string name_; + const CommandCategoryBitfield supported_categories_; +}; + +} // namespace hal +} // namespace iree + +#endif // THIRD_PARTY_MLIR_EDGE_IREE_HAL_COMMAND_QUEUE_H_ diff --git a/hal/deferred_buffer.cc b/hal/deferred_buffer.cc new file mode 100644 index 000000000000..df9f2f0133f1 --- /dev/null +++ b/hal/deferred_buffer.cc @@ -0,0 +1,162 @@ +// Copyright 2019 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "third_party/mlir_edge/iree/hal/deferred_buffer.h" + +#include "third_party/mlir_edge/iree/base/status.h" + +namespace iree { +namespace hal { + +DeferredBuffer::DeferredBuffer(Allocator* allocator, + MemoryTypeBitfield memory_type, + MemoryAccessBitfield allowed_access, + BufferUsageBitfield usage, + device_size_t byte_length) + : Buffer(allocator, memory_type, allowed_access, usage, 0, 0, byte_length) { +} + +DeferredBuffer::~DeferredBuffer() = default; + +Status DeferredBuffer::GrowByteLength(device_size_t new_byte_length) { + if (parent_buffer_) { + return FailedPreconditionErrorBuilder(ABSL_LOC) + << "Attempting to set min allocation size while bound to an " + "allocation"; + } + if (byte_length_ != kWholeBuffer && new_byte_length < byte_length_) { + return InvalidArgumentErrorBuilder(ABSL_LOC) + << "Attempting to shrink a buffer to " << new_byte_length + << " when it has a minimum size of " << byte_length_; + } + byte_length_ = new_byte_length; + return OkStatus(); +} + +Status DeferredBuffer::BindAllocation(ref_ptr allocated_buffer, + device_size_t byte_offset, + device_size_t byte_length) { + // We can only be bound to allocations that are compatible with our specified + // allocator and usage. + if (!allocator_->CanUseBuffer(allocated_buffer.get(), usage())) { + return InvalidArgumentErrorBuilder(ABSL_LOC) + << "Allocation is not compatible with the allocator specified for " + "the deferred buffer"; + } + + // Calculate the range in the allocated_buffer that we are interested in. + RETURN_IF_ERROR(Buffer::CalculateRange(0, allocated_buffer->byte_length(), + byte_offset, byte_length, &byte_offset, + &byte_length)); + + // Verify that we have enough bytes for what we've promised. + if (byte_length < byte_length_) { + return OutOfRangeErrorBuilder(ABSL_LOC) + << "Allocation range is too small; min_allocation_size=" + << byte_length_ << " but the range of " << byte_offset << "-" + << (byte_offset + byte_length - 1) << " (" << byte_length + << "b) is too small"; + } + + allocated_buffer_ = allocated_buffer.get(); + parent_buffer_ = std::move(allocated_buffer); + byte_offset_ = byte_offset; + return OkStatus(); +} + +void DeferredBuffer::ResetAllocation() { + allocated_buffer_ = this; + parent_buffer_.reset(); + byte_offset_ = 0; +} + +StatusOr DeferredBuffer::ResolveAllocation() const { + // If you get errors here then someone allocated the buffer with + // MemoryType::kTransient and you are trying to use it outside of the time + // it is actually allocated (such as during CommandBuffer evaluation). If + // you need to use the buffer in non-transient ways then allocate the buffer + // without the MemoryType::kTransient flag. + if (!parent_buffer_) { + return FailedPreconditionErrorBuilder(ABSL_LOC) + << "Attempting to use a transient buffer prior to allocation: " + << DebugString(); + } + return parent_buffer_.get(); +} + +Status DeferredBuffer::FillImpl(device_size_t byte_offset, + device_size_t byte_length, const void* pattern, + device_size_t pattern_length) { + ASSIGN_OR_RETURN(auto* allocated_buffer, ResolveAllocation()); + return allocated_buffer->FillImpl(byte_offset, byte_length, pattern, + pattern_length); +} + +Status DeferredBuffer::ReadDataImpl(device_size_t source_offset, void* data, + device_size_t data_length) { + ASSIGN_OR_RETURN(auto* allocated_buffer, ResolveAllocation()); + return allocated_buffer->ReadDataImpl(source_offset, data, data_length); +} + +Status DeferredBuffer::WriteDataImpl(device_size_t target_offset, + const void* data, + device_size_t data_length) { + ASSIGN_OR_RETURN(auto* allocated_buffer, ResolveAllocation()); + return allocated_buffer->WriteDataImpl(target_offset, data, data_length); +} + +Status DeferredBuffer::CopyDataImpl(device_size_t target_offset, + Buffer* source_buffer, + device_size_t source_offset, + device_size_t data_length) { + ASSIGN_OR_RETURN(auto* allocated_buffer, ResolveAllocation()); + return allocated_buffer->CopyDataImpl(target_offset, source_buffer, + source_offset, data_length); +} + +Status DeferredBuffer::MapMemoryImpl(MappingMode mapping_mode, + MemoryAccessBitfield memory_access, + device_size_t local_byte_offset, + device_size_t local_byte_length, + void** out_data) { + ASSIGN_OR_RETURN(auto* allocated_buffer, ResolveAllocation()); + return allocated_buffer->MapMemoryImpl(mapping_mode, memory_access, + local_byte_offset, local_byte_length, + out_data); +} + +Status DeferredBuffer::UnmapMemoryImpl(device_size_t local_byte_offset, + device_size_t local_byte_length, + void* data) { + ASSIGN_OR_RETURN(auto* allocated_buffer, ResolveAllocation()); + return allocated_buffer->UnmapMemoryImpl(local_byte_offset, local_byte_length, + data); +} + +Status DeferredBuffer::InvalidateMappedMemoryImpl( + device_size_t local_byte_offset, device_size_t local_byte_length) { + ASSIGN_OR_RETURN(auto* allocated_buffer, ResolveAllocation()); + return allocated_buffer->InvalidateMappedMemoryImpl(local_byte_offset, + local_byte_length); +} + +Status DeferredBuffer::FlushMappedMemoryImpl(device_size_t local_byte_offset, + device_size_t local_byte_length) { + ASSIGN_OR_RETURN(auto* allocated_buffer, ResolveAllocation()); + return allocated_buffer->FlushMappedMemoryImpl(local_byte_offset, + local_byte_length); +} + +} // namespace hal +} // namespace iree diff --git a/hal/deferred_buffer.h b/hal/deferred_buffer.h new file mode 100644 index 000000000000..00d7c38cb1e6 --- /dev/null +++ b/hal/deferred_buffer.h @@ -0,0 +1,106 @@ +// Copyright 2019 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#ifndef THIRD_PARTY_MLIR_EDGE_IREE_HAL_DEFERRED_BUFFER_H_ +#define THIRD_PARTY_MLIR_EDGE_IREE_HAL_DEFERRED_BUFFER_H_ + +#include +#include +#include + +#include "third_party/mlir_edge/iree/base/status.h" +#include "third_party/mlir_edge/iree/hal/allocator.h" +#include "third_party/mlir_edge/iree/hal/buffer.h" + +namespace iree { +namespace hal { + +// A Buffer that can have its underlying allocation changed at runtime. +// Unbound buffers act as a way to logically group dependent ranges of memory +// without needing to have allocated that memory yet. +// +// Usage: +// // Setup two spans referencing ranges of a deferred buffer. +// auto deferred_buffer = std::make_shared(..., 200); +// ASSIGN_OR_RETURN(auto span0, Buffer::Subspan(deferred_buffer, 0, 100)); +// ASSIGN_OR_RETURN(auto span1, Buffer::Subspan(deferred_buffer, 100, 100)); +// +// // Attempting to access |deferred_buffer| or |span0| or |span1| will fail. +// // ERROR: span0->Fill(false); +// +// // Now allocate a real buffer to serve as storage for the data. +// ASSIGN_OR_RETURN(auto allocated_buffer, Buffer::Allocate(..., 200)); +// RETURN_IF_ERROR(deferred_buffer->BindAllocation( +// allocated_buffer, 0, kWholeBuffer)); +// +// // And now we can use the spans. +// RETURN_IF_ERROR(span0->Fill(false)); +// +// // If at some point we want to detach the buffer from the allocation (so we +// // can use a different allocation, reuse the memory, etc). +// deferred_buffer->ResetAllocation(); +// +// Thread-compatible. Attempting to rebind the allocation while other threads +// are using the buffer will lead to undefined behavior. +class DeferredBuffer : public Buffer { + public: + DeferredBuffer(Allocator* allocator, MemoryTypeBitfield memory_type, + MemoryAccessBitfield allowed_access, BufferUsageBitfield usage, + device_size_t byte_length); + ~DeferredBuffer() override; + + // Grows the minimum allocation size of the buffer to |new_byte_length|. + // Attempting to bind an allocation less than this size will fail. This must + // only be called when the buffer is not bound to an allocation. + Status GrowByteLength(device_size_t new_byte_length); + + // Binds or rebinds the deferred buffer to an allocated buffer. + Status BindAllocation(ref_ptr allocated_buffer, + device_size_t byte_offset, device_size_t byte_length); + + // Resets the deferred buffer to have no binding. + void ResetAllocation(); + + private: + // Resolves the allocated buffer that this subspan references into. + // This will fail if the buffer has not yet been bound to an allocation or + // the allocated buffer has not been committed. + StatusOr ResolveAllocation() const; + + Status FillImpl(device_size_t byte_offset, device_size_t byte_length, + const void* pattern, device_size_t pattern_length) override; + Status ReadDataImpl(device_size_t source_offset, void* data, + device_size_t data_length) override; + Status WriteDataImpl(device_size_t target_offset, const void* data, + device_size_t data_length) override; + Status CopyDataImpl(device_size_t target_offset, Buffer* source_buffer, + device_size_t source_offset, + device_size_t data_length) override; + Status MapMemoryImpl(MappingMode mapping_mode, + MemoryAccessBitfield memory_access, + device_size_t local_byte_offset, + device_size_t local_byte_length, + void** out_data) override; + Status UnmapMemoryImpl(device_size_t local_byte_offset, + device_size_t local_byte_length, void* data) override; + Status InvalidateMappedMemoryImpl(device_size_t local_byte_offset, + device_size_t local_byte_length) override; + Status FlushMappedMemoryImpl(device_size_t local_byte_offset, + device_size_t local_byte_length) override; +}; + +} // namespace hal +} // namespace iree + +#endif // THIRD_PARTY_MLIR_EDGE_IREE_HAL_DEFERRED_BUFFER_H_ diff --git a/hal/device.h b/hal/device.h new file mode 100644 index 000000000000..ec790c8986cc --- /dev/null +++ b/hal/device.h @@ -0,0 +1,165 @@ +// Copyright 2019 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#ifndef THIRD_PARTY_MLIR_EDGE_IREE_HAL_DEVICE_H_ +#define THIRD_PARTY_MLIR_EDGE_IREE_HAL_DEVICE_H_ + +#include + +#include "third_party/absl/time/clock.h" +#include "third_party/absl/time/time.h" +#include "third_party/mlir_edge/iree/base/status.h" +#include "third_party/mlir_edge/iree/base/time.h" +#include "third_party/mlir_edge/iree/hal/allocator.h" +#include "third_party/mlir_edge/iree/hal/buffer.h" +#include "third_party/mlir_edge/iree/hal/command_queue.h" +#include "third_party/mlir_edge/iree/hal/device_info.h" +#include "third_party/mlir_edge/iree/hal/event.h" +#include "third_party/mlir_edge/iree/hal/executable_cache.h" +#include "third_party/mlir_edge/iree/hal/semaphore.h" + +namespace iree { +namespace hal { + +class Device { + public: + virtual ~Device() = default; + + // Information about device capabilities. + const DeviceInfo& info() const { return device_info_; } + + // TODO(benvanik): status (thermal, power mode, etc). + + // TODO(benvanik): throttling adjustment/power profile. + + // TODO(benvanik): control (suspend/resume, delay, etc). + + // An allocator providing buffers usable by the device. + // This allocator may be shared with other devices in the same family. + virtual Allocator* allocator() const = 0; + + // Returns a list of all general-purpose dispatch queues provided by the + // device. In general these map 1:1 with independent execution contexts, + // though some devices may hide that and expose only a single queue that is + // scheduled internally. + virtual absl::Span dispatch_queues() const = 0; + + // Returns a list of transfer queues provided by the device. These queues may + // perform transfer operations asynchronously with respect to execution on the + // dispatch queues. For large sequences of transfer operations always prefer + // using one of these queues. + // Note that if the device does not support a dedicated transfer queue this + // list may be the same as (or a subset of) dispatch_queues. + virtual absl::Span transfer_queues() const = 0; + + // TODO(b/137153339): accept initial cache data. + // Creates a device-specific cache for executables prepared for dispatch. + // The cache manages executable compilation, caching (on disk or in memory), + // and lifetime. Users can decide to use one or more caches to allow differing + // lifetimes (such as unloading modules), persistent on disk caching of only + // specific hot executables, etc. + // + // Returns a thread-safe cache that must remain alive until all executables + // using the cache are no longer in-flight. + virtual std::shared_ptr CreateExecutableCache() = 0; + + // Creates a command buffer for recording commands to submit to queues owned + // by this device. The command buffer may come from a pool but will be reset + // prior to being returned to the caller. + virtual StatusOr> CreateCommandBuffer( + CommandBufferModeBitfield mode, + CommandCategoryBitfield command_categories) = 0; + + // Creates an event for recording into command buffers. + // The returned event object is only usable with this device and events must + // only be used to synchronize within the same queue. + virtual StatusOr> CreateEvent() = 0; + + // Creates a binary semaphore that can be used with command queues owned by + // this device. To use the semaphores with other devices or instances they + // must first be exported. + virtual StatusOr> CreateBinarySemaphore( + bool initial_value) = 0; + + // Creates a timeline semaphore that can be used with command queues owned by + // this device. To use the semaphores with other devices or instances they + // must first be exported. + virtual StatusOr> CreateTimelineSemaphore( + uint64_t initial_value) = 0; + + // Creates a fence that can be used with command queues owned by this device. + // To use the fences with other devices or instances they must first be + // exported. + virtual StatusOr> CreateFence(uint64_t initial_value) = 0; + + // TODO(benvanik): import/export semaphore utilities. + // TODO(benvanik): import/export fence utilities. + // TODO(benvanik): fences to wait handles. + + // Blocks the caller until all passed |fences| reach or exceed the specified + // payload values or the |deadline| elapses. All |fences| must be created from + // this device (or be imported into it). + // + // Returns success if the wait is successful and all fences have been + // signaled. + // + // Returns DEADLINE_EXCEEDED if the |deadline| elapses without all fences + // having been signaled. Note that a subset of the |fences| may have been + // signaled and each can be queried to see which ones. + virtual Status WaitAllFences(absl::Span fences, + absl::Time deadline) = 0; + inline Status WaitAllFences(absl::Span fences, + absl::Duration timeout) { + return WaitAllFences(fences, RelativeTimeoutToDeadline(timeout)); + } + + // Blocks the caller until at least one of the |fences| reaches or exceeds the + // specified payload value or the |deadline| elapses. All |fences| must be + // created from this device (or be imported into it). + // + // Returns an arbitrary index into |fences| of a fence that was signaled. Note + // that more than one fence may have been signaled and all of the other + // |fences| should be queried or waited on again until waits for them + // succeed. + // + // Returns DEADLINE_EXCEEDED if the |deadline| elapses without any fences + // having been signaled. + virtual StatusOr WaitAnyFence(absl::Span fences, + absl::Time deadline) = 0; + inline StatusOr WaitAnyFence(absl::Span fences, + absl::Duration timeout) { + return WaitAnyFence(fences, RelativeTimeoutToDeadline(timeout)); + } + + // Blocks until all outstanding requests on all queues have been + // completed. This is equivalent to having waited on all outstanding + // fences. + virtual Status WaitIdle(absl::Time deadline) = 0; + inline Status WaitIdle(absl::Duration timeout) { + return WaitIdle(RelativeTimeoutToDeadline(timeout)); + } + inline Status WaitIdle() { return WaitIdle(absl::InfiniteFuture()); } + + protected: + explicit Device(DeviceInfo device_info) + : device_info_(std::move(device_info)) {} + + private: + const DeviceInfo device_info_; +}; + +} // namespace hal +} // namespace iree + +#endif // THIRD_PARTY_MLIR_EDGE_IREE_HAL_DEVICE_H_ diff --git a/hal/device_info.h b/hal/device_info.h new file mode 100644 index 000000000000..ac5f4cc3cb05 --- /dev/null +++ b/hal/device_info.h @@ -0,0 +1,90 @@ +// Copyright 2019 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#ifndef THIRD_PARTY_MLIR_EDGE_IREE_HAL_DEVICE_INFO_H_ +#define THIRD_PARTY_MLIR_EDGE_IREE_HAL_DEVICE_INFO_H_ + +#include +#include +#include + +#include "third_party/mlir_edge/iree/base/bitfield.h" + +namespace iree { +namespace hal { + +// Describes features supported by the device. +// These flags indicate the availability of features that may be enabled at the +// request of the calling application. Note that certain features may disable +// runtime optimizations or require compilation flags to ensure the required +// metadata is present in executables. +enum class DeviceFeature : uint32_t { + kNone = 0, + + // Device supports executable debugging. + // When present executables *may* be compiled with + // ExecutableCachingMode::kEnableDebugging and will have usable debugging + // related methods. Note that if the input executables do not have embedded + // debugging information they still may not be able to perform disassembly or + // fine-grained breakpoint insertion. + kDebugging = 1 << 0, + + // Device supports executable coverage information. + // When present executables *may* be compiled with + // ExecutableCachingMode::kEnableCoverage and will produce coverage buffers + // during dispatch. Note that input executables must have partial embedded + // debug information to allow mapping back to source offsets. + kCoverage = 1 << 1, + + // Device supports executable and command queue profiling. + // When present executables *may* be compiled with + // ExecutableCachingMode::kEnableProfiling and will produce profiling buffers + // during dispatch. Note that input executables must have partial embedded + // debug information to allow mapping back to source offsets. + kProfiling = 1 << 2, +}; +IREE_BITFIELD(DeviceFeature); +using DeviceFeatureBitfield = DeviceFeature; + +// TODO(benvanik): device info (caps, physical mappings, etc). +class DeviceInfo { + public: + DeviceInfo(std::string name, DeviceFeatureBitfield supported_features, + void* driver_handle = nullptr) + : name_(std::move(name)), + supported_features_(supported_features), + driver_handle_(driver_handle) {} + + const std::string& name() const { return name_; } + + // Features supported by the device. + DeviceFeatureBitfield supported_features() const { + return supported_features_; + } + + // Opaque handle used by drivers to correlate this device with their internal + // listing. This handle will not be valid across driver instances or outside + // of the current process. + void* driver_handle() const { return driver_handle_; } + + private: + const std::string name_; + const DeviceFeatureBitfield supported_features_; + void* driver_handle_; +}; + +} // namespace hal +} // namespace iree + +#endif // THIRD_PARTY_MLIR_EDGE_IREE_HAL_DEVICE_INFO_H_ diff --git a/hal/device_manager.cc b/hal/device_manager.cc new file mode 100644 index 000000000000..cf645f331c0f --- /dev/null +++ b/hal/device_manager.cc @@ -0,0 +1,177 @@ +// Copyright 2019 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "third_party/mlir_edge/iree/hal/device_manager.h" + +#include + +#include "third_party/absl/types/source_location.h" +#include "third_party/mlir_edge/iree/base/status.h" +#include "third_party/mlir_edge/iree/base/tracing.h" +#include "third_party/mlir_edge/iree/hal/heap_buffer.h" + +namespace iree { +namespace hal { + +DeviceManager::DeviceManager() = default; + +DeviceManager::~DeviceManager() = default; + +Status DeviceManager::RegisterDevice(std::shared_ptr device) { + IREE_TRACE_SCOPE0("DeviceManager::RegisterDevice"); + absl::MutexLock lock(&device_mutex_); + if (std::find(devices_.begin(), devices_.end(), device) != devices_.end()) { + return FailedPreconditionErrorBuilder(ABSL_LOC) + << "Device already registered"; + } + devices_.push_back(std::move(device)); + return OkStatus(); +} + +Status DeviceManager::UnregisterDevice(Device* device) { + IREE_TRACE_SCOPE0("DeviceManager::UnregisterDevice"); + absl::MutexLock lock(&device_mutex_); + auto it = std::find_if(devices_.begin(), devices_.end(), + [device](const std::shared_ptr& other_device) { + return device == other_device.get(); + }); + if (it == devices_.end()) { + return NotFoundErrorBuilder(ABSL_LOC) << "Device not registered"; + } + devices_.erase(it); + return OkStatus(); +} + +StatusOr DeviceManager::ResolvePlacement( + const PlacementSpec& placement_spec) const { + IREE_TRACE_SCOPE0("DeviceManager::ResolvePlacement"); + absl::MutexLock lock(&device_mutex_); + if (devices_.empty()) { + return NotFoundErrorBuilder(ABSL_LOC) << "No devices registered"; + } + + // TODO(benvanik): multiple devices and placement. + QCHECK_EQ(devices_.size(), 1) + << "Multiple devices not yet supported (need placement)"; + DevicePlacement device_placement; + device_placement.device = devices_.front(); + + return device_placement; +} + +StatusOr DeviceManager::FindCompatibleAllocator( + MemoryTypeBitfield memory_type, BufferUsageBitfield buffer_usage, + absl::Span device_placements) const { + IREE_TRACE_SCOPE0("DeviceManager::FindCompatibleAllocator"); + if (device_placements.empty()) { + return InvalidArgumentErrorBuilder(ABSL_LOC) << "No placements provided"; + } + + // Find the first allocator. As we only return an allocator if all placements + // are compatible we'll compare allocator[0] against allocator[1,N]. + Allocator* some_allocator = nullptr; + for (const auto& device_placement : device_placements) { + auto* allocator = device_placement.device->allocator(); + if (!some_allocator) { + some_allocator = allocator; + continue; + } + // NOTE: as there can be asymmetry between usage restrictions (A can use B + // but B cannot use A) we have to compare both directions. + if (!some_allocator->CanUseBufferLike(allocator, memory_type, buffer_usage, + buffer_usage) || + !allocator->CanUseBufferLike(some_allocator, memory_type, buffer_usage, + buffer_usage)) { + // Allocators are not compatible. + return NotFoundErrorBuilder(ABSL_LOC) + << "No single allocator found that is compatible with all " + "placements"; + } + } + return some_allocator; +} + +StatusOr> DeviceManager::TryAllocateDeviceVisibleBuffer( + MemoryTypeBitfield memory_type, BufferUsageBitfield buffer_usage, + device_size_t allocation_size, + absl::Span device_placements) { + IREE_TRACE_SCOPE("DeviceManager::TryAllocateDeviceVisibleBuffer:size", int) + (static_cast(allocation_size)); + if (!AnyBitSet(memory_type & MemoryType::kHostLocal)) { + return InvalidArgumentErrorBuilder(ABSL_LOC) + << "Host-local buffers require the kHostLocal bit: " + << MemoryTypeString(memory_type); + } + + // Strip kDeviceVisible as we conditionally add it based on support. + memory_type &= ~MemoryType::kDeviceVisible; + + // Find an allocator that works for device-visible buffers. + // If this fails we'll fall back to allocation a non-device-visible buffer. + auto allocator_or = + FindCompatibleAllocator(memory_type | MemoryType::kDeviceVisible, + buffer_usage, device_placements); + if (allocator_or.ok()) { + return allocator_or.ValueOrDie()->Allocate( + memory_type | MemoryType::kDeviceVisible, buffer_usage, + allocation_size); + } + + // Fallback to allocating a host-local buffer. + return HeapBuffer::Allocate(memory_type, buffer_usage, allocation_size); +} + +StatusOr> DeviceManager::AllocateDeviceVisibleBuffer( + MemoryTypeBitfield memory_type, BufferUsageBitfield buffer_usage, + device_size_t allocation_size, + absl::Span device_placements) { + IREE_TRACE_SCOPE("DeviceManager::AllocateDeviceVisibleBuffer:size", int) + (static_cast(allocation_size)); + if (!AnyBitSet(memory_type & MemoryType::kHostLocal)) { + return InvalidArgumentErrorBuilder(ABSL_LOC) + << "Host-local buffers require the kHostLocal bit: " + << MemoryTypeString(memory_type); + } + + // Always use device-visible. + memory_type |= MemoryType::kDeviceVisible; + + // Find an allocator that works for device-visible buffers. + ASSIGN_OR_RETURN( + auto* allocator, + FindCompatibleAllocator(memory_type, buffer_usage, device_placements)); + return allocator->Allocate(memory_type, buffer_usage, allocation_size); +} + +StatusOr> DeviceManager::AllocateDeviceLocalBuffer( + MemoryTypeBitfield memory_type, BufferUsageBitfield buffer_usage, + device_size_t allocation_size, + absl::Span device_placements) { + IREE_TRACE_SCOPE("DeviceManager::AllocateDeviceLocalBuffer:size", int) + (static_cast(allocation_size)); + if (!AnyBitSet(memory_type & MemoryType::kDeviceLocal)) { + return InvalidArgumentErrorBuilder(ABSL_LOC) + << "Device-local buffers require the kDeviceLocal bit: " + << MemoryTypeString(memory_type); + } + + // Find an allocator that works for device-local buffers. + ASSIGN_OR_RETURN( + auto* allocator, + FindCompatibleAllocator(memory_type, buffer_usage, device_placements)); + return allocator->Allocate(memory_type, buffer_usage, allocation_size); +} + +} // namespace hal +} // namespace iree diff --git a/hal/device_manager.h b/hal/device_manager.h new file mode 100644 index 000000000000..c140697b6651 --- /dev/null +++ b/hal/device_manager.h @@ -0,0 +1,150 @@ +// Copyright 2019 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#ifndef THIRD_PARTY_MLIR_EDGE_IREE_HAL_DEVICE_MANAGER_H_ +#define THIRD_PARTY_MLIR_EDGE_IREE_HAL_DEVICE_MANAGER_H_ + +#include + +#include "third_party/absl/synchronization/mutex.h" +#include "third_party/absl/types/span.h" +#include "third_party/mlir_edge/iree/base/status.h" +#include "third_party/mlir_edge/iree/hal/allocator.h" +#include "third_party/mlir_edge/iree/hal/buffer.h" +#include "third_party/mlir_edge/iree/hal/device.h" +#include "third_party/mlir_edge/iree/hal/device_placement.h" +#include "third_party/mlir_edge/iree/hal/executable_format.h" + +namespace iree { +namespace hal { + +// Specifies how devices should be resolved to DevicePlacements. +// Most fields are optional and when not included will be ignored. +struct PlacementSpec { + // TODO(benvanik): other requirements (features/caps, power, etc). + + // A list of executable formats that the placement should support. + // If more than one format is provided any device satisfying at least one + // will be considered for placement. The formats can be sorted in descending + // priority order to prefer the first available format in the case of ties. + absl::Span available_formats; +}; + +// Manages device lifetime and placement resolution. +// +// Thread-safe. Note that callers must ensure that unregistered devices are kept +// alive for as long as any commands are in-flight that may be using them. +class DeviceManager final { + public: + DeviceManager(); + ~DeviceManager(); + + // Registers a device with the manager. + // The device will be used to resolve placements. Any placements resolved + // prior to the addition of the device will need to be refreshed by the caller + // if they want to make use of the new device. + Status RegisterDevice(std::shared_ptr device); + + // Unregisters a device with the manager. + // Placements that resolved to the device prior to unregistering will remain + // valid for that device. Callers will need to refresh the placements to + // ensure the device stops being used. + Status UnregisterDevice(Device* device); + + // TODO(benvanik): dispatch info + requirements + etc -> DevicePlacement. + + // Resolves a placement spec to a device placement based on the registered + // devices. + // If the placement is not fully specified the device and queue may be chosen + // at random. See PlacementSpec for more information about resolution and + // ranking. + StatusOr ResolvePlacement( + const PlacementSpec& placement_spec) const; + + // Finds an allocator that can allocate buffers of the given |memory_type| and + // |buffer_usage| such that the buffers can be used interchangebly. + // Fails if there is no Allocator that can satisfy that requirement. + StatusOr FindCompatibleAllocator( + MemoryTypeBitfield memory_type, BufferUsageBitfield buffer_usage, + absl::Span device_placements) const; + + // Tries to allocate a host-local buffer that _may_ be optimal for use with + // the given |device_placements| and _may_ be device-visible. The buffer can + // be used for staging uploads to device-local buffers and is useful for times + // when the buffer will be used more on the host than the device. If a buffer + // never needs to be used with a device prefer instead + // Allocator::host_local()::Allocate. + // + // Returns a buffer even if it's not possible to satisfy the requested + // |buffer_usage| for the |device_placements| at the cost of a run-time + // performance hit. + StatusOr> TryAllocateDeviceVisibleBuffer( + MemoryTypeBitfield memory_type, BufferUsageBitfield buffer_usage, + device_size_t allocation_size, + absl::Span device_placements); + StatusOr> TryAllocateDeviceVisibleBuffer( + BufferUsageBitfield buffer_usage, device_size_t allocation_size, + absl::Span device_placements) { + return TryAllocateDeviceVisibleBuffer( + MemoryType::kHostLocal | MemoryType::kDeviceVisible, buffer_usage, + allocation_size, device_placements); + } + + // Allocates a host-local buffer that is optimal for use on the host but is + // usable by the given |device_placements| (at a possible performance + // penalty). The buffer can be used for staging uploads to device-local + // buffers and is useful for times when the buffer will be used more on the + // host than the device. If a buffer never needs to be used with a device + // prefer instead HeapBuffer::Allocate. + // + // Fails if it is not possible to allocate and satisfy all |device_placements| + // for the requested |buffer_usage|. + StatusOr> AllocateDeviceVisibleBuffer( + MemoryTypeBitfield memory_type, BufferUsageBitfield buffer_usage, + device_size_t allocation_size, + absl::Span device_placements); + StatusOr> AllocateDeviceVisibleBuffer( + BufferUsageBitfield buffer_usage, device_size_t allocation_size, + absl::Span device_placements) { + return AllocateDeviceVisibleBuffer( + MemoryType::kHostLocal | MemoryType::kDeviceVisible, buffer_usage, + allocation_size, device_placements); + } + + // Allocates a device-local buffer that is optimal for use with the given + // |device_placements|. The buffer will not be host-visible and can only be + // used from compatible device queues. + // + // Fails if it is not possible to allocate and satisfy all |device_placements| + // for the requested |buffer_usage|. + StatusOr> AllocateDeviceLocalBuffer( + MemoryTypeBitfield memory_type, BufferUsageBitfield buffer_usage, + device_size_t allocation_size, + absl::Span device_placements); + StatusOr> AllocateDeviceLocalBuffer( + BufferUsageBitfield buffer_usage, device_size_t allocation_size, + absl::Span device_placements) { + return AllocateDeviceLocalBuffer(MemoryType::kDeviceLocal, buffer_usage, + allocation_size, device_placements); + } + + private: + mutable absl::Mutex device_mutex_; + std::vector> devices_ ABSL_GUARDED_BY(device_mutex_); +}; + +} // namespace hal +} // namespace iree + +#endif // THIRD_PARTY_MLIR_EDGE_IREE_HAL_DEVICE_MANAGER_H_ diff --git a/hal/device_placement.h b/hal/device_placement.h new file mode 100644 index 000000000000..4ffadc4647db --- /dev/null +++ b/hal/device_placement.h @@ -0,0 +1,34 @@ +// Copyright 2019 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#ifndef THIRD_PARTY_MLIR_EDGE_IREE_HAL_DEVICE_PLACEMENT_H_ +#define THIRD_PARTY_MLIR_EDGE_IREE_HAL_DEVICE_PLACEMENT_H_ + +#include + +namespace iree { +namespace hal { + +class Device; + +// TODO(benvanik): define device-specific placement info - possibly opaque. +struct DevicePlacement { + std::shared_ptr device; + int queue_id = 0; +}; + +} // namespace hal +} // namespace iree + +#endif // THIRD_PARTY_MLIR_EDGE_IREE_HAL_DEVICE_PLACEMENT_H_ diff --git a/hal/driver.h b/hal/driver.h new file mode 100644 index 000000000000..2cb9b997df7f --- /dev/null +++ b/hal/driver.h @@ -0,0 +1,61 @@ +// Copyright 2019 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#ifndef THIRD_PARTY_MLIR_EDGE_IREE_HAL_DRIVER_H_ +#define THIRD_PARTY_MLIR_EDGE_IREE_HAL_DRIVER_H_ + +#include +#include +#include + +#include "third_party/mlir_edge/iree/base/status.h" +#include "third_party/mlir_edge/iree/hal/device.h" +#include "third_party/mlir_edge/iree/hal/device_info.h" + +namespace iree { +namespace hal { + +class Driver { + public: + virtual ~Driver() = default; + + // Driver name used during registration. + const std::string& name() const { return name_; } + + // TODO(benvanik): info/query (version number, etc). + + // Enumerates devices available for creation from the driver. + // This may fail if the driver is in an invalid state but otherwise will + // return an empty list if no devices are available. + virtual StatusOr> EnumerateAvailableDevices() = 0; + + // Creates the driver-defined 'default' device. + // This may simply be the first device enumerated. + virtual StatusOr> CreateDefaultDevice() = 0; + + // Creates a device as queried with the given |device_info|. + virtual StatusOr> CreateDevice( + const DeviceInfo& device_info) = 0; + + protected: + explicit Driver(std::string name) : name_(std::move(name)) {} + + private: + const std::string name_; +}; + +} // namespace hal +} // namespace iree + +#endif // THIRD_PARTY_MLIR_EDGE_IREE_HAL_DRIVER_H_ diff --git a/hal/driver_registry.cc b/hal/driver_registry.cc new file mode 100644 index 000000000000..e608231d803e --- /dev/null +++ b/hal/driver_registry.cc @@ -0,0 +1,87 @@ +// Copyright 2019 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "third_party/mlir_edge/iree/hal/driver_registry.h" + +#include "third_party/mlir_edge/iree/base/status.h" + +namespace iree { +namespace hal { + +// static +DriverRegistry* DriverRegistry::shared_registry() { + static auto* singleton = new DriverRegistry(); + return singleton; +} + +DriverRegistry::DriverRegistry() = default; + +DriverRegistry::~DriverRegistry() = default; + +Status DriverRegistry::Register(std::string driver_name, FactoryFn factory_fn) { + absl::MutexLock lock(&mutex_); + for (const auto& pair : driver_factory_fns_) { + if (pair.first == driver_name) { + return AlreadyExistsErrorBuilder(ABSL_LOC) + << "Driver already registered: " << driver_name; + } + } + driver_factory_fns_.emplace_back(driver_name, std::move(factory_fn)); + return OkStatus(); +} + +bool DriverRegistry::HasDriver(absl::string_view driver_name) const { + absl::MutexLock lock(&mutex_); + for (const auto& pair : driver_factory_fns_) { + if (pair.first == driver_name) { + return true; + } + } + return false; +} + +std::vector DriverRegistry::EnumerateAvailableDrivers() const { + absl::MutexLock lock(&mutex_); + std::vector driver_names; + driver_names.reserve(driver_factory_fns_.size()); + for (const auto& pair : driver_factory_fns_) { + driver_names.push_back(pair.first); + } + return driver_names; +} + +StatusOr> DriverRegistry::Create( + absl::string_view driver_name) const { + FactoryFn factory_fn; + { + absl::MutexLock lock(&mutex_); + for (const auto& pair : driver_factory_fns_) { + if (pair.first == driver_name) { + factory_fn = pair.second; + break; + } + } + if (!factory_fn) { + return NotFoundErrorBuilder(ABSL_LOC) + << "Driver " << driver_name << " not found"; + } + } + return factory_fn(); +} + +} // namespace hal +} // namespace iree + +IREE_REGISTER_MODULE_INITIALIZER( + iree_hal, ::iree::hal::DriverRegistry::shared_registry()); diff --git a/hal/driver_registry.h b/hal/driver_registry.h new file mode 100644 index 000000000000..2f154e63953a --- /dev/null +++ b/hal/driver_registry.h @@ -0,0 +1,83 @@ +// Copyright 2019 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#ifndef THIRD_PARTY_MLIR_EDGE_IREE_HAL_DRIVER_REGISTRY_H_ +#define THIRD_PARTY_MLIR_EDGE_IREE_HAL_DRIVER_REGISTRY_H_ + +#include +#include + +#include "third_party/absl/base/thread_annotations.h" +#include "third_party/absl/synchronization/mutex.h" +#include "third_party/mlir_edge/iree/base/init.h" +#include "third_party/mlir_edge/iree/base/status.h" +#include "third_party/mlir_edge/iree/hal/driver.h" + +namespace iree { +namespace hal { + +// Driver registry and factory. +// Factory functions for available drivers are registered with a given name and +// can be invoked with a call to Create. The configuration of the drivers is +// generally contained within the factory function and consumers of the drivers +// don't need to fiddle with things. +// +// This is used for dynamic *safe* link-time driver module registration. +// Roughly: driver_registry provides the shared registry and a way to create +// drivers and *_driver_module.cc files register drivers when linked in. +// Remember to alwayslink=1 on cc_libraries providing modules. +// +// If link-time driver registration is not desired (or possible) it's also +// possible to explicitly register drivers via this registry. This is useful +// when programmatically enabling drivers. +// +// Thread-safe. +class DriverRegistry final { + public: + using FactoryFn = std::function>()>; + + // The shared driver registry singleton that modules use when linked in. + static DriverRegistry* shared_registry(); + + DriverRegistry(); + ~DriverRegistry(); + + // Registers a driver and its factory function. + // The function will be called to create a new driver whenever it is requested + // via Create. + Status Register(std::string driver_name, FactoryFn factory_fn); + + // Returns true if there is a driver registered with the given name. + bool HasDriver(absl::string_view driver_name) const; + + // Returns a list of registered drivers. + std::vector EnumerateAvailableDrivers() const; + + // TODO(benvanik): flags for enabling debug validation/control/etc. + // Creates a driver by name. + StatusOr> Create(absl::string_view driver_name) const; + + private: + mutable absl::Mutex mutex_; + std::vector> driver_factory_fns_ + ABSL_GUARDED_BY(mutex_); +}; + +} // namespace hal +} // namespace iree + +IREE_DECLARE_MODULE_INITIALIZER(iree_hal); +IREE_REQUIRE_MODULE_LINKED(iree_hal); + +#endif // THIRD_PARTY_MLIR_EDGE_IREE_HAL_DRIVER_REGISTRY_H_ diff --git a/hal/event.h b/hal/event.h new file mode 100644 index 000000000000..ae9a481bbd14 --- /dev/null +++ b/hal/event.h @@ -0,0 +1,35 @@ +// Copyright 2019 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#ifndef THIRD_PARTY_MLIR_EDGE_IREE_HAL_EVENT_H_ +#define THIRD_PARTY_MLIR_EDGE_IREE_HAL_EVENT_H_ + +#include "third_party/mlir_edge/iree/hal/resource.h" + +namespace iree { +namespace hal { + +// Events are used for defining synchronization scopes within CommandBuffers. +// An event only exists within a single CommandBuffer and must not be used +// across CommandBuffers from the same device or others. +// +// See CommandBuffer::SignalEvent and CommandBuffer::WaitEvents for more info. +class Event : public Resource { + public: +}; + +} // namespace hal +} // namespace iree + +#endif // THIRD_PARTY_MLIR_EDGE_IREE_HAL_EVENT_H_ diff --git a/hal/executable.h b/hal/executable.h new file mode 100644 index 000000000000..a0564abaa09d --- /dev/null +++ b/hal/executable.h @@ -0,0 +1,57 @@ +// Copyright 2019 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#ifndef THIRD_PARTY_MLIR_EDGE_IREE_HAL_EXECUTABLE_H_ +#define THIRD_PARTY_MLIR_EDGE_IREE_HAL_EXECUTABLE_H_ + +#include "third_party/mlir_edge/iree/hal/resource.h" + +namespace iree { +namespace hal { + +class Executable : public Resource { + public: + ~Executable() override = default; + + // True if the executable was prepared with debugging enabled and the device + // and input data support debugging (symbols present, etc). + virtual bool supports_debugging() const = 0; + + // TODO(benvanik): disassembly methods. + + // TODO(benvanik): relative offset calculation: + // - step once + // - step over + // - step out + + // TODO(benvanik): create executable split on breakpoint. + // Executable should return when the breakpoint is hit without any future + // modifications to output buffers. If the breakpoint is not hit the + // executable should run to completion as normal. + + // TODO(benvanik): retrieve coverage info. + // Returns a buffer containing offset -> coverage metrics. Note that depending + // on the device this may only contain a single coverage metric for the entire + // executable or some subset of the available offsets. + + // TODO(benvanik): retrieve profiling info. + + protected: + Executable() = default; +}; + +} // namespace hal +} // namespace iree + +#endif // THIRD_PARTY_MLIR_EDGE_IREE_HAL_EXECUTABLE_H_ diff --git a/hal/executable_cache.cc b/hal/executable_cache.cc new file mode 100644 index 000000000000..d36e2a0e2bfd --- /dev/null +++ b/hal/executable_cache.cc @@ -0,0 +1,56 @@ +// Copyright 2019 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "third_party/mlir_edge/iree/hal/executable_cache.h" + +#include "third_party/absl/types/source_location.h" +#include "third_party/mlir_edge/iree/base/status.h" +#include "third_party/mlir_edge/iree/base/tracing.h" + +namespace iree { +namespace hal { + +ExecutableCache::ExecutableCache() = default; + +ExecutableCache::~ExecutableCache() = default; + +StatusOr ExecutableCache::PrepareExecutables( + ExecutableCachingModeBitfield mode, absl::Span specs, + absl::Span> out_executables) { + IREE_TRACE_SCOPE0("ExecutableCache::PrepareExecutables"); + if (specs.size() != out_executables.size()) { + return InvalidArgumentErrorBuilder(ABSL_LOC) + << "1:1 specs:out_executables required"; + } + + ManualResetEvent fence("ExecutableCachePreparation"); + auto wait_handle = fence.OnSet(); + + // TODO(benvanik): make async (spin up thread, etc). + for (int i = 0; i < specs.size(); ++i) { + auto executable_or = PrepareExecutable(mode, specs[i]); + if (!executable_or.ok()) { + // TODO(benvanik): propagate executable error. + RETURN_IF_ERROR(fence.Set()); + return wait_handle; + } + out_executables[i] = add_ref(std::move(executable_or).ValueOrDie()); + } + + RETURN_IF_ERROR(fence.Set()); + return wait_handle; +} + +} // namespace hal +} // namespace iree diff --git a/hal/executable_cache.h b/hal/executable_cache.h new file mode 100644 index 000000000000..ae9b364d9348 --- /dev/null +++ b/hal/executable_cache.h @@ -0,0 +1,145 @@ +// Copyright 2019 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#ifndef THIRD_PARTY_MLIR_EDGE_IREE_HAL_EXECUTABLE_CACHE_H_ +#define THIRD_PARTY_MLIR_EDGE_IREE_HAL_EXECUTABLE_CACHE_H_ + +#include "third_party/mlir_edge/iree/base/bitfield.h" +#include "third_party/mlir_edge/iree/base/ref_ptr.h" +#include "third_party/mlir_edge/iree/base/status.h" +#include "third_party/mlir_edge/iree/base/wait_handle.h" +#include "third_party/mlir_edge/iree/hal/executable.h" +#include "third_party/mlir_edge/iree/hal/executable_format.h" +#include "third_party/mlir_edge/iree/hal/executable_spec.h" + +namespace iree { +namespace hal { + +// Defines how the executable cache performs preparation. +enum class ExecutableCachingMode : uint32_t { + // Allows the cache to reference the provided executable_data after it has + // prepared the executable. Callers must ensure the data remains valid for the + // lifetime of the cache. If memory mapping constant executable data from + // disk this can be used to avoid copies. + kAliasProvidedData = 1 << 0, + + // Allows the prepared executable to be cached persistently (on disk/etc). + // Enable for any executable that is likely to be used in future runs. + // Note that not all caches support persistent serialization and this is just + // a hint. + kAllowPersistentCaching = 1 << 1, + + // Allows the cache to optimize the executable as much as it can. + // This may cause preparation to take significantly longer while (hopefully) + // improving runtime performance. Avoid for one-shot executables. + kAllowOptimization = 1 << 2, + + // Enables Executable debugging methods if supported by the device and + // executable. This may disable certain optimizations or retain additional + // data to allow disassembly, stepping, etc. + // + // Device must support the DeviceFeature::kDebugging feature and executables + // must support the ExecutableFeature::kDebugging feature. + kEnableDebugging = 1 << 3, + + // Enables Executable coverage if supported by the device and executable. + // Depending on the optimization mode this may produce partial coverage + // results (for example, when certain source operations were optimized away). + // + // Device must support the DeviceFeature::kCoverage feature and executables + // must support the ExecutableFeature::kCoverage feature. + kEnableCoverage = 1 << 4, + + // Enables Executable profiling if supported by the device and executable. + // Depending on the optimization mode this may produce partial profiling + // results. Profiling attribution (whether to the entire executable or + // specific operations) depends on the implementation. + // + // Device must support the DeviceFeature::kProfiling feature and executables + // must support the ExecutableFeature::kProfiling feature. + kEnableProfiling = 1 << 5, + + // Default caching mode. + kDefault = kAllowPersistentCaching | kAllowOptimization, +}; +IREE_BITFIELD(ExecutableCachingMode); +using ExecutableCachingModeBitfield = ExecutableCachingMode; + +// A cache of prepared executables for a particular device. +// Caches may be shared across multiple devices from the same driver or specific +// to individual devices. Caches may persist prepared executables across process +// launches or reprepare them each run. Callers should assume that the cache is +// a no-op and the returned Executables only live for as long as the cache does. +// +// The term 'cache' here is rather optimistic - it's perfectly acceptable for +// implementations to not cache at all and return new Executables for each +// PrepareExecutable called (even for the same executable). Callers should +// expect such behavior and try to retain the results of the PrepareExecutable +// calls to reduce overhead in re-preparing executables. +// +// Thread-safe - multiple threads may prepare executables (including the *same* +// executable) simultaneously. +class ExecutableCache { + public: + virtual ~ExecutableCache(); + + // TODO(benvanik): status/queries (size, etc). + + // TODO(b/137153339): serialization/deserialization. + + // Returns true if the executable cache can prepare the given executable input + // format. Perparation may still fail if the particular version or features + // required by the executable are not supported. + virtual bool CanPrepareFormat(ExecutableFormat format) const = 0; + + // Prepares an executable for use. + // The provided |spec| and |executable_data| will be used to either lookup a + // previously prepared executable in the cache or prepare a new one. + // + // Depending on the driver preparation may take a non-trivial amount of time + // (such as when JITing/etc). As the cache is internally synchronized callers + // can issue preparation requests from multiple threads - even for the same + // executables - and calls will block until preparation completes. + // + // When preparing a large number of executables it's recommended to use the + // PrepareExecutables method to batch and wait on the results. + virtual StatusOr> PrepareExecutable( + ExecutableCachingModeBitfield mode, const ExecutableSpec& spec) = 0; + + // Prepares one or more executables asynchronously on a worker thread (maybe). + // When the WaitHandle is signaled successfully |out_executables| will contain + // one Executable for each ExecutableSpec provided in |specs|, in order. + // The backing memory of |out_executables| must remain valid until the + // WaitHandle resolves. Preparation errors will be returned on the WaitHandle. + // If more than one preparation errors occurs only one will be returned (from + // an undefined order). + // + // Note: not all implementations will actually perform preparation + // asynchronously. This method just allows drivers to do so as possible. + // + // If applications already have their own preparation threads it is better to + // use PrepareExecutable in a loop to avoid the creation of new threads. + virtual StatusOr PrepareExecutables( + ExecutableCachingModeBitfield mode, + absl::Span specs, + absl::Span> out_executables); + + protected: + ExecutableCache(); +}; + +} // namespace hal +} // namespace iree + +#endif // THIRD_PARTY_MLIR_EDGE_IREE_HAL_EXECUTABLE_CACHE_H_ diff --git a/hal/executable_format.h b/hal/executable_format.h new file mode 100644 index 000000000000..229ddecf64ca --- /dev/null +++ b/hal/executable_format.h @@ -0,0 +1,60 @@ +// Copyright 2019 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// Defines the ExecutableFormat 4cc type and a few well-known formats. +// Not all formats need to be defined here, however any format expected to be +// supported by debuggers/tooling will likely want to be here to ensure easier +// referencing. + +#ifndef THIRD_PARTY_MLIR_EDGE_IREE_HAL_EXECUTABLE_FORMAT_H_ +#define THIRD_PARTY_MLIR_EDGE_IREE_HAL_EXECUTABLE_FORMAT_H_ + +#include + +namespace iree { +namespace hal { + +// Executable format 4cc identifier. +using ExecutableFormat = uint32_t; + +// Constructs an ExecutableFormat 4cc at compile-time. +constexpr ExecutableFormat MakeExecutableFormatID(char const four_cc[5]) { + return (four_cc[0] << 24) | (four_cc[1] << 16) | (four_cc[2] << 8) | + four_cc[3]; +} + + +// Undefined (or unknown). The format may be derived from the executable +// contents (such as file magic bytes). +constexpr ExecutableFormat kExecutableFormatUnspecified = + MakeExecutableFormatID(" "); + +// MLIR text form. +constexpr ExecutableFormat kExecutableFormatMlir = + MakeExecutableFormatID("MLIR"); + +// IREE v0 bytecode. +constexpr ExecutableFormat kExecutableFormatIreeBytecode = + MakeExecutableFormatID("IREE"); + +// SPIR-V executable in FlatBuffer format using the +// iree/schemas/spirv_executable_def.fbs schema. +constexpr ExecutableFormat kExecutableFormatSpirV = + MakeExecutableFormatID("SPVE"); + + +} // namespace hal +} // namespace iree + +#endif // THIRD_PARTY_MLIR_EDGE_IREE_HAL_EXECUTABLE_FORMAT_H_ diff --git a/hal/executable_spec.h b/hal/executable_spec.h new file mode 100644 index 000000000000..34f4d02549dc --- /dev/null +++ b/hal/executable_spec.h @@ -0,0 +1,44 @@ +// Copyright 2019 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#ifndef THIRD_PARTY_MLIR_EDGE_IREE_HAL_EXECUTABLE_SPEC_H_ +#define THIRD_PARTY_MLIR_EDGE_IREE_HAL_EXECUTABLE_SPEC_H_ + +#include "third_party/absl/types/span.h" +#include "third_party/mlir_edge/iree/hal/executable_format.h" + +namespace iree { +namespace hal { + +// Defines an executable specification used by a cache to prepare an executable. +struct ExecutableSpec { + // TODO(benvanik): pre-populated hash_code/key to avoid calculation. + + // Format of the executable input data. + ExecutableFormat format = kExecutableFormatUnspecified; + + // A reference to the executable data as input to the cache. + // If ExecutableCachingMode::kAliasProvidedData is set then this reference + // may be retained by the cache and the backing buffer must be kept valid for + // the lifetime of the cache. + absl::Span executable_data; + + // TODO(benvanik): add specialization info (constants/defines). + // TODO(benvanik): add compiler flags? could treat as opaque. +}; + +} // namespace hal +} // namespace iree + +#endif // THIRD_PARTY_MLIR_EDGE_IREE_HAL_EXECUTABLE_SPEC_H_ diff --git a/hal/fence.h b/hal/fence.h new file mode 100644 index 000000000000..2ceda78442cf --- /dev/null +++ b/hal/fence.h @@ -0,0 +1,72 @@ +// Copyright 2019 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#ifndef THIRD_PARTY_MLIR_EDGE_IREE_HAL_FENCE_H_ +#define THIRD_PARTY_MLIR_EDGE_IREE_HAL_FENCE_H_ + +#include + +#include "third_party/mlir_edge/iree/base/status.h" +#include "third_party/mlir_edge/iree/hal/resource.h" + +namespace iree { +namespace hal { + +// Synchronization mechanism for device->host notification. +// Fences behave like timeline semaphores and contain a monotonically increasing +// uint64_t payload. They may be waited on any number of times - even if they +// have already been signaled. +// +// A fence is updated to its new value after all prior commands have completed +// but the delay between completion and the host being woken varies. Some +// implementations may coalesce fences to avoid spurious waking while others +// will immediately synchronize with the host. +// +// The primary use of fences is for resource lifetime management: all resources +// used by a set of submission batches must be considered live until the fence +// attached to the submission has signaled. +// +// Fences may be set to a permanently failed state by implementations when +// errors occur during asynchronous execution. Users are expected to propagate +// the failures and possibly reset the entire device that produced the error. +// +// For more information on fences see the following docs describing how +// timelines are generally used (specifically in the device->host case): +// https://www.youtube.com/watch?v=SpE--Rf516Y +// https://www.khronos.org/assets/uploads/developers/library/2018-xdc/Vulkan-Timeline-Semaphores-Part-1_Sep18.pdf +// https://docs.microsoft.com/en-us/windows/win32/direct3d12/user-mode-heap-synchronization +class Fence : public Resource { + public: + // Returns a permanent failure status if the fence is indicating an + // asynchronous failure. + // + // Returns the status at the time the method is called without blocking and as + // such is only valid after a fence has been signaled. The same failure status + // will be returned regardless of when in the timeline the error occurred. + virtual Status status() const = 0; + + // Queries the current payload of the fence. As the payload is monotonically + // increasing it is guaranteed that the value is at least equal to the + // previous result of a QueryValue call and coherent with any waits for a + // specified value via Device::WaitAllFences. + virtual StatusOr QueryValue() = 0; +}; + +// A reference to a fence and associated payload value. +using FenceValue = std::pair; + +} // namespace hal +} // namespace iree + +#endif // THIRD_PARTY_MLIR_EDGE_IREE_HAL_FENCE_H_ diff --git a/hal/heap_buffer.cc b/hal/heap_buffer.cc new file mode 100644 index 000000000000..3959a83211dd --- /dev/null +++ b/hal/heap_buffer.cc @@ -0,0 +1,190 @@ +// Copyright 2019 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "third_party/mlir_edge/iree/hal/heap_buffer.h" + +#include +#include +#include +#include + +#include "third_party/absl/types/source_location.h" +#include "third_party/mlir_edge/iree/base/status.h" +#include "third_party/mlir_edge/iree/base/tracing.h" +#include "third_party/mlir_edge/iree/hal/allocator.h" +#include "third_party/mlir_edge/iree/hal/host/host_buffer.h" + +namespace iree { +namespace hal { + +namespace { + +// An allocator that allocates or wraps host-only buffers. +// The resulting buffers are not usable by most devices without a copy and +// using a device allocator is strongly preferred. +class HeapAllocator : public Allocator { + public: + // Returns a singleton heap allocator that can provide buffers that have + // MemoryType::kHostLocal and are allocated with malloc/free. + // These buffers will not be usable by devices directly and may incur + // additional copies. + static Allocator* std_heap(); + + // TODO(benvanik): specify custom allocator (not malloc/free). + HeapAllocator(); + ~HeapAllocator() override; + + bool CanUseBufferLike(Allocator* source_allocator, + MemoryTypeBitfield memory_type, + BufferUsageBitfield buffer_usage, + BufferUsageBitfield intended_usage) const override; + + bool CanAllocate(MemoryTypeBitfield memory_type, + BufferUsageBitfield buffer_usage, + size_t allocation_size) const override; + + StatusOr> Allocate(MemoryTypeBitfield memory_type, + BufferUsageBitfield buffer_usage, + size_t allocation_size) override; + + StatusOr> WrapMutable(MemoryTypeBitfield memory_type, + MemoryAccessBitfield allowed_access, + BufferUsageBitfield buffer_usage, + void* data, + size_t data_length) override; +}; + +// static +Allocator* HeapAllocator::std_heap() { + static Allocator* std_heap_allocator = new HeapAllocator(); + return std_heap_allocator; +} + +HeapAllocator::HeapAllocator() = default; + +HeapAllocator::~HeapAllocator() = default; + +bool HeapAllocator::CanUseBufferLike(Allocator* source_allocator, + MemoryTypeBitfield memory_type, + BufferUsageBitfield buffer_usage, + BufferUsageBitfield intended_usage) const { + // The host can use anything with kHostVisible. + if (!AnyBitSet(memory_type & MemoryType::kHostVisible)) { + return false; + } + + // Host currently uses mapping to copy buffers, which is done a lot. + if (!AnyBitSet(buffer_usage & BufferUsage::kMapping)) { + return false; + } + + return true; +} + +bool HeapAllocator::CanAllocate(MemoryTypeBitfield memory_type, + BufferUsageBitfield buffer_usage, + size_t allocation_size) const { + // This host only allocator cannot serve device visible allocation as we + // can't know which devices these buffers will be used with. + return (memory_type & MemoryType::kHostLocal) == MemoryType::kHostLocal && + !AnyBitSet(memory_type & MemoryType::kDeviceLocal) && + !AnyBitSet(memory_type & MemoryType::kDeviceVisible); +} + +StatusOr> HeapAllocator::Allocate( + MemoryTypeBitfield memory_type, BufferUsageBitfield buffer_usage, + size_t allocation_size) { + IREE_TRACE_SCOPE0("HeapAllocator::Allocate"); + + if (!CanAllocate(memory_type, buffer_usage, allocation_size)) { + return FailedPreconditionErrorBuilder(ABSL_LOC) + << "Allocation not supported; memory_type=" + << MemoryTypeString(memory_type) + << ", buffer_usage=" << BufferUsageString(buffer_usage) + << ", allocation_size=" << allocation_size; + } + + void* malloced_data = std::calloc(1, allocation_size); + if (!malloced_data) { + return ResourceExhaustedErrorBuilder(ABSL_LOC) + << "Failed to malloc " << allocation_size << " bytes"; + } + + auto buffer = + make_ref(this, memory_type, MemoryAccess::kAll, buffer_usage, + allocation_size, malloced_data, true); + return buffer; +} + +StatusOr> HeapAllocator::WrapMutable( + MemoryTypeBitfield memory_type, MemoryAccessBitfield allowed_access, + BufferUsageBitfield buffer_usage, void* data, size_t data_length) { + auto buffer = make_ref(this, memory_type, allowed_access, + buffer_usage, data_length, data, false); + return buffer; +} + +} // namespace + +// static +ref_ptr HeapBuffer::Allocate(MemoryTypeBitfield memory_type, + BufferUsageBitfield usage, + size_t allocation_size) { + auto buffer_or = + HeapAllocator::std_heap()->Allocate(memory_type, usage, allocation_size); + return std::move(buffer_or.ValueOrDie()); +} + +// static +ref_ptr HeapBuffer::AllocateCopy(BufferUsageBitfield usage, + const void* data, size_t data_length) { + return AllocateCopy(usage, MemoryAccess::kAll, data, data_length); +} + +// static +ref_ptr HeapBuffer::AllocateCopy(BufferUsageBitfield usage, + MemoryAccessBitfield allowed_access, + const void* data, size_t data_length) { + IREE_TRACE_SCOPE0("HeapBuffer::AllocateCopy"); + // Ensure we can map so that we can copy into it. + usage |= BufferUsage::kMapping; + auto buffer_or = HeapAllocator::std_heap()->Allocate(MemoryType::kHostLocal, + usage, data_length); + auto buffer = std::move(buffer_or.ValueOrDie()); + buffer->WriteData(0, data, data_length).IgnoreError(); + buffer->set_allowed_access(allowed_access); + return buffer; +} + +// static +ref_ptr HeapBuffer::Wrap(MemoryTypeBitfield memory_type, + BufferUsageBitfield usage, const void* data, + size_t data_length) { + auto buffer_or = + HeapAllocator::std_heap()->Wrap(memory_type, usage, data, data_length); + return std::move(buffer_or.ValueOrDie()); +} + +// static +ref_ptr HeapBuffer::WrapMutable(MemoryTypeBitfield memory_type, + MemoryAccessBitfield allowed_access, + BufferUsageBitfield usage, void* data, + size_t data_length) { + auto buffer_or = HeapAllocator::std_heap()->WrapMutable( + memory_type, allowed_access, usage, data, data_length); + return std::move(buffer_or.ValueOrDie()); +} + +} // namespace hal +} // namespace iree diff --git a/hal/heap_buffer.h b/hal/heap_buffer.h new file mode 100644 index 000000000000..ead35fb3d1b4 --- /dev/null +++ b/hal/heap_buffer.h @@ -0,0 +1,117 @@ +// Copyright 2019 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#ifndef THIRD_PARTY_MLIR_EDGE_IREE_HAL_HEAP_BUFFER_H_ +#define THIRD_PARTY_MLIR_EDGE_IREE_HAL_HEAP_BUFFER_H_ + +#include + +#include "third_party/mlir_edge/iree/base/status.h" +#include "third_party/mlir_edge/iree/hal/buffer.h" + +namespace iree { +namespace hal { + +// Factory for buffers that are allocated from the host heap (malloc/free). +// These buffers cannot be used by devices and will incur copies/transfers when +// used. Prefer device-specific allocators instead. +class HeapBuffer { + public: + // Allocates a zeroed host heap buffer of the given size. + // Returns a buffer allocated with malloc and have MemoryType::kHostLocal + // and will not be usable by devices without copies. + static ref_ptr Allocate(MemoryTypeBitfield memory_type, + BufferUsageBitfield usage, + size_t allocation_size); + static ref_ptr Allocate(BufferUsageBitfield usage, + size_t allocation_size) { + return Allocate(MemoryType::kHostLocal, usage, allocation_size); + } + + // Allocates a host heap buffer with a copy of the given data. + // Returns a buffer allocated with malloc and have MemoryType::kHostLocal + // and will not be usable by devices without copies. + static ref_ptr AllocateCopy(BufferUsageBitfield usage, + const void* data, size_t data_length); + static ref_ptr AllocateCopy(BufferUsageBitfield usage, + MemoryAccessBitfield allowed_access, + const void* data, size_t data_length); + template + static ref_ptr AllocateCopy(BufferUsageBitfield usage, + absl::Span data); + template + static ref_ptr AllocateCopy(BufferUsageBitfield usage, + MemoryAccessBitfield allowed_access, + absl::Span data); + + // Wraps an existing host heap allocation in a buffer. + // Ownership of the host allocation remains with the caller and the memory + // must remain valid for so long as the Buffer may be in use. + // Will have MemoryType::kHostLocal in most cases and may not be usable + // by the device. + static ref_ptr Wrap(MemoryTypeBitfield memory_type, + BufferUsageBitfield usage, const void* data, + size_t data_length); + static ref_ptr WrapMutable(MemoryTypeBitfield memory_type, + MemoryAccessBitfield allowed_access, + BufferUsageBitfield usage, void* data, + size_t data_length); + template + static ref_ptr Wrap(MemoryTypeBitfield memory_type, + BufferUsageBitfield usage, + absl::Span data); + template + static ref_ptr WrapMutable(MemoryTypeBitfield memory_type, + MemoryAccessBitfield allowed_access, + BufferUsageBitfield usage, + absl::Span data); +}; + +// Inline functions and template definitions follow: + +template +ref_ptr HeapBuffer::AllocateCopy(BufferUsageBitfield usage, + absl::Span data) { + return HeapBuffer::AllocateCopy(usage, MemoryAccess::kAll, data); +} + +template +ref_ptr HeapBuffer::AllocateCopy(BufferUsageBitfield usage, + MemoryAccessBitfield allowed_access, + absl::Span data) { + return HeapBuffer::AllocateCopy(usage, allowed_access, data.data(), + data.size() * sizeof(T)); +} + +template +ref_ptr HeapBuffer::Wrap(MemoryTypeBitfield memory_type, + BufferUsageBitfield usage, + absl::Span data) { + return HeapBuffer::Wrap(memory_type, usage, data.data(), + data.size() * sizeof(T)); +} + +template +ref_ptr HeapBuffer::WrapMutable(MemoryTypeBitfield memory_type, + MemoryAccessBitfield allowed_access, + BufferUsageBitfield usage, + absl::Span data) { + return HeapBuffer::WrapMutable(memory_type, allowed_access, usage, + data.data(), data.size() * sizeof(T)); +} + +} // namespace hal +} // namespace iree + +#endif // THIRD_PARTY_MLIR_EDGE_IREE_HAL_HEAP_BUFFER_H_ diff --git a/hal/host/async_command_queue.cc b/hal/host/async_command_queue.cc new file mode 100644 index 000000000000..4b6a5c7f71ee --- /dev/null +++ b/hal/host/async_command_queue.cc @@ -0,0 +1,134 @@ +// Copyright 2019 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "third_party/mlir_edge/iree/hal/host/async_command_queue.h" + +#include "third_party/absl/base/thread_annotations.h" +#include "third_party/mlir_edge/iree/base/status.h" +#include "third_party/mlir_edge/iree/base/tracing.h" + +namespace iree { +namespace hal { + +AsyncCommandQueue::AsyncCommandQueue(std::unique_ptr target_queue) + : CommandQueue(target_queue->name(), target_queue->supported_categories()), + target_queue_(std::move(target_queue)) { + IREE_TRACE_SCOPE0("AsyncCommandQueue::ctor"); + thread_ = std::thread([this]() { ThreadMain(); }); +} + +AsyncCommandQueue::~AsyncCommandQueue() { + IREE_TRACE_SCOPE0("AsyncCommandQueue::dtor"); + { + // Signal to thread that we want to stop. Note that the thread may have + // already been stopped and that's ok (as we'll Join right away). + // The thread will finish processing any queued submissions. + absl::MutexLock lock(&submission_mutex_); + submission_queue_.SignalShutdown(); + } + thread_.join(); + + // Ensure we shut down OK. + { + absl::MutexLock lock(&submission_mutex_); + CHECK(submission_queue_.empty()) + << "Dirty shutdown of async queue (unexpected thread exit?)"; + } +} + +void AsyncCommandQueue::ThreadMain() { + // TODO(benvanik): make this safer (may die if trace is flushed late). + IREE_TRACE_THREAD_ENABLE(target_queue_->name().c_str()); + + bool is_exiting = false; + while (!is_exiting) { + // Block until we are either requested to exit or there are pending + // submissions. + submission_mutex_.Lock(); + submission_mutex_.Await(absl::Condition( + +[](HostSubmissionQueue* queue) { + return queue->has_shutdown() || !queue->empty(); + }, + &submission_queue_)); + if (!submission_queue_.empty()) { + // Run all ready submissions (this may be called many times). + submission_mutex_.AssertHeld(); + submission_queue_ + .ProcessBatches( + [this](absl::Span command_buffers) + ABSL_EXCLUSIVE_LOCKS_REQUIRED(submission_mutex_) { + // Release the lock while we perform the processing so that + // other threads can submit more work. + submission_mutex_.AssertHeld(); + submission_mutex_.Unlock(); + + // Relay the command buffers to the target queue. + // Since we are taking care of all synchronization they + // don't need any waiters or fences. + auto status = target_queue_->Submit( + {{}, command_buffers, {}}, {nullptr, 0u}); + + // Take back the lock so we can manipulate the queue safely. + submission_mutex_.Lock(); + submission_mutex_.AssertHeld(); + + return status; + }) + .IgnoreError(); + submission_mutex_.AssertHeld(); + } + if (submission_queue_.has_shutdown()) { + // Exit when there are no more submissions to process and an exit was + // requested (or we errored out). + is_exiting = true; + } + submission_mutex_.Unlock(); + } +} + +Status AsyncCommandQueue::Submit(absl::Span batches, + FenceValue fence) { + IREE_TRACE_SCOPE0("AsyncCommandQueue::Submit"); + absl::MutexLock lock(&submission_mutex_); + return submission_queue_.Enqueue(batches, fence); +} + +Status AsyncCommandQueue::Flush() { + IREE_TRACE_SCOPE0("AsyncCommandQueue::Flush"); + // No-op (as we don't currently delay). + absl::MutexLock lock(&submission_mutex_); + return submission_queue_.permanent_error(); +} + +Status AsyncCommandQueue::WaitIdle(absl::Time deadline) { + IREE_TRACE_SCOPE0("AsyncCommandQueue::WaitIdle"); + + // Wait until the deadline, the thread exits, or there are no more pending + // submissions. + absl::MutexLock lock(&submission_mutex_); + if (!submission_mutex_.AwaitWithDeadline( + absl::Condition( + +[](HostSubmissionQueue* queue) { + return queue->empty() || !queue->permanent_error().ok(); + }, + &submission_queue_), + deadline)) { + return DeadlineExceededErrorBuilder(ABSL_LOC) + << "Deadline exceeded waiting for submission thread to go idle"; + } + return submission_queue_.permanent_error(); +} + +} // namespace hal +} // namespace iree diff --git a/hal/host/async_command_queue.h b/hal/host/async_command_queue.h new file mode 100644 index 000000000000..7cc9712a8cce --- /dev/null +++ b/hal/host/async_command_queue.h @@ -0,0 +1,72 @@ +// Copyright 2019 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#ifndef THIRD_PARTY_MLIR_EDGE_IREE_HAL_HOST_ASYNC_COMMAND_QUEUE_H_ +#define THIRD_PARTY_MLIR_EDGE_IREE_HAL_HOST_ASYNC_COMMAND_QUEUE_H_ + +#include +#include // NOLINT + +#include "third_party/absl/base/thread_annotations.h" +#include "third_party/absl/synchronization/mutex.h" +#include "third_party/mlir_edge/iree/hal/command_queue.h" +#include "third_party/mlir_edge/iree/hal/fence.h" +#include "third_party/mlir_edge/iree/hal/host/host_submission_queue.h" + +namespace iree { +namespace hal { + +// Asynchronous command queue wrapper. +// This creates a single thread to perform all CommandQueue operations. Any +// submitted CommandBuffer is dispatched in FIFO order on the queue thread +// against the provided |target_queue|. +// +// Target queues will receive submissions containing only command buffers as +// all semaphore synchronization is handled by the wrapper. Fences will also be +// omitted and code should safely handle nullptr. +// +// AsyncCommandQueue (as with CommandQueue) is thread-safe. Multiple threads +// may submit command buffers concurrently, though the order of execution in +// such a case depends entirely on the synchronization primitives provided. +class AsyncCommandQueue final : public CommandQueue { + public: + explicit AsyncCommandQueue(std::unique_ptr target_queue); + ~AsyncCommandQueue() override; + + Status Submit(absl::Span batches, + FenceValue fence) override; + + Status Flush() override; + Status WaitIdle(absl::Time deadline) override; + + private: + // Thread entry point for the async worker thread. + // Waits for submissions to be queued up and processes them eagerly. + void ThreadMain(); + + // CommandQueue that the async queue relays submissions into. + std::unique_ptr target_queue_; + + // Thread that runs the ThreadMain() function and processes submissions. + std::thread thread_; + + // Queue that manages submission ordering. + mutable absl::Mutex submission_mutex_; + HostSubmissionQueue submission_queue_ ABSL_GUARDED_BY(submission_mutex_); +}; + +} // namespace hal +} // namespace iree + +#endif // THIRD_PARTY_MLIR_EDGE_IREE_HAL_HOST_ASYNC_COMMAND_QUEUE_H_ diff --git a/hal/host/host_buffer.cc b/hal/host/host_buffer.cc new file mode 100644 index 000000000000..9a3638edc9a4 --- /dev/null +++ b/hal/host/host_buffer.cc @@ -0,0 +1,148 @@ +// Copyright 2019 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "third_party/mlir_edge/iree/hal/host/host_buffer.h" + +#include +#include +#include + +#include "third_party/absl/types/source_location.h" +#include "third_party/mlir_edge/iree/base/logging.h" +#include "third_party/mlir_edge/iree/base/status.h" + +namespace iree { +namespace hal { + +class Allocator; + +HostBuffer::HostBuffer(Allocator* allocator, MemoryTypeBitfield memory_type, + MemoryAccessBitfield allowed_access, + BufferUsageBitfield usage, device_size_t allocation_size, + void* data, bool owns_data) + : Buffer(allocator, memory_type, allowed_access, usage, allocation_size, 0, + allocation_size), + data_(data), + owns_data_(owns_data) {} + +HostBuffer::~HostBuffer() { + if (owns_data_ && data_) { + std::free(data_); + data_ = nullptr; + } +} + +Status HostBuffer::FillImpl(device_size_t byte_offset, + device_size_t byte_length, const void* pattern, + device_size_t pattern_length) { + auto data_ptr = data_; + switch (pattern_length) { + case 1: { + uint8_t* data = static_cast(data_ptr); + uint8_t value_bits = *static_cast(pattern); + std::fill_n(data + byte_offset, byte_length, value_bits); + break; + } + case 2: { + uint16_t* data = static_cast(data_ptr); + uint16_t value_bits = *static_cast(pattern); + std::fill_n(data + byte_offset / sizeof(uint16_t), + byte_length / sizeof(uint16_t), value_bits); + break; + } + case 4: { + uint32_t* data = static_cast(data_ptr); + uint32_t value_bits = *static_cast(pattern); + std::fill_n(data + byte_offset / sizeof(uint32_t), + byte_length / sizeof(uint32_t), value_bits); + break; + } + default: + return InvalidArgumentErrorBuilder(ABSL_LOC) + << "Unsupported scalar data size: " << pattern_length; + } + return OkStatus(); +} + +Status HostBuffer::ReadDataImpl(device_size_t source_offset, void* data, + device_size_t data_length) { + auto data_ptr = static_cast(data_); + std::memcpy(data, data_ptr + source_offset, data_length); + return OkStatus(); +} + +Status HostBuffer::WriteDataImpl(device_size_t target_offset, const void* data, + device_size_t data_length) { + auto data_ptr = static_cast(data_); + std::memcpy(data_ptr + target_offset, data, data_length); + return OkStatus(); +} + +Status HostBuffer::CopyDataImpl(device_size_t target_offset, + Buffer* source_buffer, + device_size_t source_offset, + device_size_t data_length) { + // This is pretty terrible. Let's not do this. + // TODO(benvanik): a way for allocators to indicate transfer compat. + ASSIGN_OR_RETURN(auto source_data, + source_buffer->MapMemory( + MemoryAccess::kRead, source_offset, data_length)); + CHECK_EQ(data_length, source_data.size()); + auto data_ptr = static_cast(data_); + std::memcpy(data_ptr + target_offset, source_data.data(), data_length); + return OkStatus(); +} + +Status HostBuffer::MapMemoryImpl(MappingMode mapping_mode, + MemoryAccessBitfield memory_access, + device_size_t local_byte_offset, + device_size_t local_byte_length, + void** out_data) { + auto data_ptr = static_cast(data_); + *out_data = data_ptr + local_byte_offset; + + // If we mapped for discard scribble over the bytes. This is not a mandated + // behavior but it will make debugging issues easier. Alternatively for + // heap buffers we could reallocate them such that ASAN yells, but that + // would only work if the entire buffer was discarded. +#ifndef NDEBUG + if (AnyBitSet(memory_access & MemoryAccess::kDiscard)) { + std::memset(data_ptr + local_byte_offset, 0xCD, local_byte_length); + } +#endif // !NDEBUG + + return OkStatus(); +} + +Status HostBuffer::UnmapMemoryImpl(device_size_t local_byte_offset, + device_size_t local_byte_length, + void* data) { + // No-op? We still want error checking to make finding misuse easier. + return OkStatus(); +} + +Status HostBuffer::InvalidateMappedMemoryImpl(device_size_t local_byte_offset, + device_size_t local_byte_length) { + // No-op? We still want error checking to make finding misuse easier. + return OkStatus(); +} + +Status HostBuffer::FlushMappedMemoryImpl(device_size_t local_byte_offset, + device_size_t local_byte_length) { + // No-op? We still want error checking to make finding misuse easier. + return OkStatus(); +} + +} // namespace hal +} // namespace iree diff --git a/hal/host/host_buffer.h b/hal/host/host_buffer.h new file mode 100644 index 000000000000..37abaca55b9b --- /dev/null +++ b/hal/host/host_buffer.h @@ -0,0 +1,67 @@ +// Copyright 2019 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#ifndef THIRD_PARTY_MLIR_EDGE_IREE_HAL_HOST_BUFFER_H_ +#define THIRD_PARTY_MLIR_EDGE_IREE_HAL_HOST_BUFFER_H_ + +#include + +#include "third_party/mlir_edge/iree/base/status.h" +#include "third_party/mlir_edge/iree/hal/buffer.h" + +namespace iree { +namespace hal { + +// A buffer type that operates on host pointers. +// This can be used by Allocator implementations when they support operating +// on host memory (or mapping their memory to host memory). +class HostBuffer : public Buffer { + public: + HostBuffer(Allocator* allocator, MemoryTypeBitfield memory_type, + MemoryAccessBitfield allowed_access, BufferUsageBitfield usage, + device_size_t allocation_size, void* data, bool owns_data); + + ~HostBuffer() override; + + protected: + Status FillImpl(device_size_t byte_offset, device_size_t byte_length, + const void* pattern, device_size_t pattern_length) override; + Status ReadDataImpl(device_size_t source_offset, void* data, + device_size_t data_length) override; + Status WriteDataImpl(device_size_t target_offset, const void* data, + device_size_t data_length) override; + Status CopyDataImpl(device_size_t target_offset, Buffer* source_buffer, + device_size_t source_offset, + device_size_t data_length) override; + Status MapMemoryImpl(MappingMode mapping_mode, + MemoryAccessBitfield memory_access, + device_size_t local_byte_offset, + device_size_t local_byte_length, + void** out_data) override; + Status UnmapMemoryImpl(device_size_t local_byte_offset, + device_size_t local_byte_length, void* data) override; + Status InvalidateMappedMemoryImpl(device_size_t local_byte_offset, + device_size_t local_byte_length) override; + Status FlushMappedMemoryImpl(device_size_t local_byte_offset, + device_size_t local_byte_length) override; + + private: + void* data_ = nullptr; + bool owns_data_ = false; +}; + +} // namespace hal +} // namespace iree + +#endif // THIRD_PARTY_MLIR_EDGE_IREE_HAL_HOST_BUFFER_H_ diff --git a/hal/host/host_event.cc b/hal/host/host_event.cc new file mode 100644 index 000000000000..e669bd9bd460 --- /dev/null +++ b/hal/host/host_event.cc @@ -0,0 +1,25 @@ +// Copyright 2019 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "third_party/mlir_edge/iree/hal/host/host_event.h" + +namespace iree { +namespace hal { + +HostEvent::HostEvent() = default; + +HostEvent::~HostEvent() = default; + +} // namespace hal +} // namespace iree diff --git a/hal/host/host_event.h b/hal/host/host_event.h new file mode 100644 index 000000000000..bb751ee9c3e1 --- /dev/null +++ b/hal/host/host_event.h @@ -0,0 +1,32 @@ +// Copyright 2019 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#ifndef THIRD_PARTY_MLIR_EDGE_IREE_HAL_HOST_HOST_EVENT_H_ +#define THIRD_PARTY_MLIR_EDGE_IREE_HAL_HOST_HOST_EVENT_H_ + +#include "third_party/mlir_edge/iree/hal/event.h" + +namespace iree { +namespace hal { + +class HostEvent final : public Event { + public: + HostEvent(); + ~HostEvent() override; +}; + +} // namespace hal +} // namespace iree + +#endif // THIRD_PARTY_MLIR_EDGE_IREE_HAL_HOST_HOST_EVENT_H_ diff --git a/hal/host/host_fence.cc b/hal/host/host_fence.cc new file mode 100644 index 000000000000..9ba3b52c15a6 --- /dev/null +++ b/hal/host/host_fence.cc @@ -0,0 +1,110 @@ +// Copyright 2019 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "third_party/mlir_edge/iree/hal/host/host_fence.h" + +#include +#include + +#include "third_party/absl/container/inlined_vector.h" +#include "third_party/absl/synchronization/mutex.h" +#include "third_party/mlir_edge/iree/base/status.h" +#include "third_party/mlir_edge/iree/base/tracing.h" + +namespace iree { +namespace hal { + +HostFence::HostFence(uint64_t initial_value) : value_(initial_value) {} + +HostFence::~HostFence() = default; + +Status HostFence::status() const { + absl::MutexLock lock(&mutex_); + return status_; +} + +StatusOr HostFence::QueryValue() { + return value_.load(std::memory_order_acquire); +} + +Status HostFence::Signal(uint64_t value) { + absl::MutexLock lock(&mutex_); + if (!status_.ok()) { + return status_; + } + if (value_.exchange(value) >= value) { + return InvalidArgumentErrorBuilder(ABSL_LOC) + << "Fence values must be monotonically increasing"; + } + return OkStatus(); +} + +Status HostFence::Fail(Status status) { + absl::MutexLock lock(&mutex_); + status_ = status; + value_.store(UINT64_MAX, std::memory_order_release); + return OkStatus(); +} + +// static +Status HostFence::WaitForFences(absl::Span fences, + bool wait_all, absl::Time deadline) { + IREE_TRACE_SCOPE0("HostFence::WaitForFences"); + + // Some of the fences may already be signaled; we only need to wait for those + // that are not yet at the expected value. + using HostFenceValue = std::pair; + absl::InlinedVector waitable_fences; + waitable_fences.reserve(fences.size()); + for (auto& fence_value : fences) { + auto* fence = reinterpret_cast(fence_value.first); + ASSIGN_OR_RETURN(uint64_t current_value, fence->QueryValue()); + if (current_value == UINT64_MAX) { + // Fence has failed. Return the error. + return fence->status(); + } else if (current_value < fence_value.second) { + // Fence has not yet hit the required value; wait for it. + waitable_fences.push_back({fence, fence_value.second}); + } + } + + // TODO(benvanik): maybe sort fences by value in case we are waiting on + // multiple values from the same fence. + + // Loop over the fences and wait for them to complete. + // TODO(b/140026716): add WaitHandle support for !wait_all (wait any). + for (auto& fence_value : waitable_fences) { + auto* fence = fence_value.first; + absl::MutexLock lock(&fence->mutex_); + if (!fence->mutex_.AwaitWithDeadline( + absl::Condition( + +[](HostFenceValue* fence_value) { + return fence_value->first->value_.load( + std::memory_order_acquire) >= fence_value->second; + }, + &fence_value), + deadline)) { + return DeadlineExceededErrorBuilder(ABSL_LOC) + << "Deadline exceeded waiting for fences"; + } + if (!fence->status_.ok()) { + return fence->status_; + } + } + + return OkStatus(); +} + +} // namespace hal +} // namespace iree diff --git a/hal/host/host_fence.h b/hal/host/host_fence.h new file mode 100644 index 000000000000..306ef5ec7d73 --- /dev/null +++ b/hal/host/host_fence.h @@ -0,0 +1,63 @@ +// Copyright 2019 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#ifndef THIRD_PARTY_MLIR_EDGE_IREE_HAL_HOST_HOST_FENCE_H_ +#define THIRD_PARTY_MLIR_EDGE_IREE_HAL_HOST_HOST_FENCE_H_ + +#include +#include + +#include "third_party/absl/base/thread_annotations.h" +#include "third_party/absl/synchronization/mutex.h" +#include "third_party/mlir_edge/iree/base/status.h" +#include "third_party/mlir_edge/iree/hal/fence.h" + +namespace iree { +namespace hal { + +// TODO(b/140026716): add WaitHandle support for better multi-wait. +// Simple host-only fence semaphore implemented with a mutex. +// +// Thread-safe (as instances may be imported and used by others). +class HostFence final : public Fence { + public: + // Waits for one or more (or all) fences to reach or exceed the given values. + static Status WaitForFences(absl::Span fences, + bool wait_all, absl::Time deadline); + + explicit HostFence(uint64_t initial_value); + ~HostFence() override; + + Status status() const override; + StatusOr QueryValue() override; + + Status Signal(uint64_t value); + Status Fail(Status status); + + private: + // The mutex is not required to query the value; this lets us quickly check if + // a required value has been exceeded. The mutex is only used to update and + // notify waiters. + std::atomic value_{0}; + + // We have a full mutex here so that we can perform condvar waits on value + // changes. + mutable absl::Mutex mutex_; + Status status_ ABSL_GUARDED_BY(mutex_); +}; + +} // namespace hal +} // namespace iree + +#endif // THIRD_PARTY_MLIR_EDGE_IREE_HAL_HOST_HOST_FENCE_H_ diff --git a/hal/host/host_local_allocator.cc b/hal/host/host_local_allocator.cc new file mode 100644 index 000000000000..6e3e18806eee --- /dev/null +++ b/hal/host/host_local_allocator.cc @@ -0,0 +1,111 @@ +// Copyright 2019 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "third_party/mlir_edge/iree/hal/host/host_local_allocator.h" + +#include +#include +#include + +#include "third_party/absl/types/source_location.h" +#include "third_party/mlir_edge/iree/base/status.h" +#include "third_party/mlir_edge/iree/base/tracing.h" +#include "third_party/mlir_edge/iree/hal/host/host_buffer.h" + +namespace iree { +namespace hal { + +HostLocalAllocator::HostLocalAllocator() = default; + +HostLocalAllocator::~HostLocalAllocator() = default; + +bool HostLocalAllocator::CanUseBufferLike( + Allocator* source_allocator, MemoryTypeBitfield memory_type, + BufferUsageBitfield buffer_usage, + BufferUsageBitfield intended_usage) const { + // Must always have visibility to the device, which ensures we can test + // against the host but have things work on devices with separate address + // spaces. + if (!AnyBitSet(memory_type & MemoryType::kDeviceVisible)) { + return false; + } + + // kHostVisible is required for mapping. + if (AnyBitSet(intended_usage & BufferUsage::kMapping) && + !AnyBitSet(memory_type & MemoryType::kHostVisible)) { + return false; + } + + // Dispatch needs to be specified if we intend to dispatch. + if (AnyBitSet(intended_usage & BufferUsage::kDispatch) && + !AnyBitSet(buffer_usage & BufferUsage::kDispatch)) { + return false; + } + + return true; +} + +bool HostLocalAllocator::CanAllocate(MemoryTypeBitfield memory_type, + BufferUsageBitfield buffer_usage, + size_t allocation_size) const { + // Host allows everything, pretty much, so long as it is device-visible (as + // the host is the device here). + return AnyBitSet(memory_type & MemoryType::kDeviceVisible); +} + +Status HostLocalAllocator::MakeCompatible( + MemoryTypeBitfield* memory_type, BufferUsageBitfield* buffer_usage) const { + // Always ensure we are host-visible. + *memory_type |= MemoryType::kHostVisible; + + // Host currently uses mapping to copy buffers, which is done a lot. + // We could probably remove this restriction somehow. + *buffer_usage |= BufferUsage::kMapping; + + // TODO(b/111372612): tensorflow needs transfer too, but shouldn't. + *buffer_usage |= BufferUsage::kTransfer; + + return OkStatus(); +} + +StatusOr> HostLocalAllocator::Allocate( + MemoryTypeBitfield memory_type, BufferUsageBitfield buffer_usage, + size_t allocation_size) { + IREE_TRACE_SCOPE0("HostLocalAllocator::Allocate"); + + if (!CanAllocate(memory_type, buffer_usage, allocation_size)) { + return FailedPreconditionErrorBuilder(ABSL_LOC) + << "Allocation not supported; memory_type=" + << MemoryTypeString(memory_type) + << ", buffer_usage=" << BufferUsageString(buffer_usage) + << ", allocation_size=" << allocation_size; + } + + // Make compatible with our requirements. + RETURN_IF_ERROR(MakeCompatible(&memory_type, &buffer_usage)); + + void* malloced_data = std::calloc(1, allocation_size); + if (!malloced_data) { + return ResourceExhaustedErrorBuilder(ABSL_LOC) + << "Failed to malloc " << allocation_size << " bytes"; + } + + auto buffer = + make_ref(this, memory_type, MemoryAccess::kAll, buffer_usage, + allocation_size, malloced_data, true); + return buffer; +} + +} // namespace hal +} // namespace iree diff --git a/hal/host/host_local_allocator.h b/hal/host/host_local_allocator.h new file mode 100644 index 000000000000..a036b3b8deac --- /dev/null +++ b/hal/host/host_local_allocator.h @@ -0,0 +1,60 @@ +// Copyright 2019 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#ifndef THIRD_PARTY_MLIR_EDGE_IREE_HAL_HOST_LOCAL_ALLOCATOR_H_ +#define THIRD_PARTY_MLIR_EDGE_IREE_HAL_HOST_LOCAL_ALLOCATOR_H_ + +#include +#include + +#include "third_party/mlir_edge/iree/base/status.h" +#include "third_party/mlir_edge/iree/hal/allocator.h" +#include "third_party/mlir_edge/iree/hal/buffer.h" + +namespace iree { +namespace hal { + +// An allocator implementation that allocates buffers from host memory. +// This can be used for drivers that do not have a memory space of their own. +// +// Buffers allocated will have be MemoryType::kHostLocal | kDeviceVisible as +// the 'device' in the case of a host-local queue *is* the host. To keep code +// written initially for a host-local queue working when other queues are used +// the allocator only works with buffers that are kDeviceVisible. +class HostLocalAllocator : public Allocator { + public: + HostLocalAllocator(); + ~HostLocalAllocator() override; + + bool CanUseBufferLike(Allocator* source_allocator, + MemoryTypeBitfield memory_type, + BufferUsageBitfield buffer_usage, + BufferUsageBitfield intended_usage) const override; + + bool CanAllocate(MemoryTypeBitfield memory_type, + BufferUsageBitfield buffer_usage, + size_t allocation_size) const override; + + Status MakeCompatible(MemoryTypeBitfield* memory_type, + BufferUsageBitfield* buffer_usage) const override; + + StatusOr> Allocate(MemoryTypeBitfield memory_type, + BufferUsageBitfield buffer_usage, + size_t allocation_size) override; +}; + +} // namespace hal +} // namespace iree + +#endif // THIRD_PARTY_MLIR_EDGE_IREE_HAL_HOST_LOCAL_ALLOCATOR_H_ diff --git a/hal/host/host_local_command_processor.cc b/hal/host/host_local_command_processor.cc new file mode 100644 index 000000000000..dcc3bb9ca35a --- /dev/null +++ b/hal/host/host_local_command_processor.cc @@ -0,0 +1,120 @@ +// Copyright 2019 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "third_party/mlir_edge/iree/hal/host/host_local_command_processor.h" + +#include "third_party/absl/types/source_location.h" +#include "third_party/mlir_edge/iree/base/status.h" +#include "third_party/mlir_edge/iree/base/tracing.h" + +namespace iree { +namespace hal { + +HostLocalCommandProcessor::HostLocalCommandProcessor( + Allocator* allocator, CommandBufferModeBitfield mode, + CommandCategoryBitfield command_categories) + : CommandBuffer(allocator, mode, command_categories) {} + +HostLocalCommandProcessor::~HostLocalCommandProcessor() = default; + +Status HostLocalCommandProcessor::Begin() { + IREE_TRACE_SCOPE0("HostLocalCommandProcessor::Begin"); + is_recording_ = true; + return OkStatus(); +} + +Status HostLocalCommandProcessor::End() { + IREE_TRACE_SCOPE0("HostLocalCommandProcessor::End"); + is_recording_ = false; + return OkStatus(); +} + +Status HostLocalCommandProcessor::ExecutionBarrier( + ExecutionStageBitfield source_stage_mask, + ExecutionStageBitfield target_stage_mask, + absl::Span memory_barriers, + absl::Span buffer_barriers) { + IREE_TRACE_SCOPE0("HostLocalCommandProcessor::ExecutionBarrier"); + // No-op. + return OkStatus(); +} + +Status HostLocalCommandProcessor::SignalEvent( + Event* event, ExecutionStageBitfield source_stage_mask) { + IREE_TRACE_SCOPE0("HostLocalCommandProcessor::SignalEvent"); + // No-op. + return OkStatus(); +} + +Status HostLocalCommandProcessor::ResetEvent( + Event* event, ExecutionStageBitfield source_stage_mask) { + IREE_TRACE_SCOPE0("HostLocalCommandProcessor::ResetEvent"); + // No-op. + return OkStatus(); +} + +Status HostLocalCommandProcessor::WaitEvents( + absl::Span events, ExecutionStageBitfield source_stage_mask, + ExecutionStageBitfield target_stage_mask, + absl::Span memory_barriers, + absl::Span buffer_barriers) { + IREE_TRACE_SCOPE0("HostLocalCommandProcessor::WaitEvents"); + // No-op. + return OkStatus(); +} + +Status HostLocalCommandProcessor::FillBuffer(Buffer* target_buffer, + device_size_t target_offset, + device_size_t length, + const void* pattern, + size_t pattern_length) { + IREE_TRACE_SCOPE0("HostLocalCommandProcessor::FillBuffer"); + return target_buffer->Fill(target_offset, length, pattern, pattern_length); +} + +Status HostLocalCommandProcessor::DiscardBuffer(Buffer* buffer) { + IREE_TRACE_SCOPE0("HostLocalCommandProcessor::DiscardBuffer"); + // No-op as we don't support lazily allocated buffers. + return OkStatus(); +} + +Status HostLocalCommandProcessor::UpdateBuffer(const void* source_buffer, + device_size_t source_offset, + Buffer* target_buffer, + device_size_t target_offset, + device_size_t length) { + IREE_TRACE_SCOPE0("HostLocalCommandProcessor::UpdateBuffer"); + return target_buffer->WriteData( + target_offset, static_cast(source_buffer) + source_offset, + length); +} + +Status HostLocalCommandProcessor::CopyBuffer(Buffer* source_buffer, + device_size_t source_offset, + Buffer* target_buffer, + device_size_t target_offset, + device_size_t length) { + IREE_TRACE_SCOPE0("HostLocalCommandProcessor::CopyBuffer"); + return target_buffer->CopyData(target_offset, source_buffer, source_offset, + length); +} + +Status HostLocalCommandProcessor::Dispatch( + const DispatchRequest& dispatch_request) { + return FailedPreconditionErrorBuilder(ABSL_LOC) + << "Command processor does not support dispatch operations"; +} + +} // namespace hal +} // namespace iree diff --git a/hal/host/host_local_command_processor.h b/hal/host/host_local_command_processor.h new file mode 100644 index 000000000000..9b2e946457e1 --- /dev/null +++ b/hal/host/host_local_command_processor.h @@ -0,0 +1,85 @@ +// Copyright 2019 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#ifndef THIRD_PARTY_MLIR_EDGE_IREE_HAL_HOST_HOST_LOCAL_COMMAND_PROCESSOR_H_ +#define THIRD_PARTY_MLIR_EDGE_IREE_HAL_HOST_HOST_LOCAL_COMMAND_PROCESSOR_H_ + +#include "third_party/mlir_edge/iree/hal/command_buffer.h" + +namespace iree { +namespace hal { + +// Host-local command processor for dispatching transfer operations against +// buffers allocated from the HostLocalAllocator. +// This assumes that all buffers are host-visible (if not local) and that all +// buffers can be mapped for access. +// +// Subclasses may implement Dispatch, otherwise the default implementation just +// returns failure. +// +// Thread-compatible (as with CommandBuffer itself). +class HostLocalCommandProcessor : public CommandBuffer { + public: + HostLocalCommandProcessor(Allocator* allocator, + CommandBufferModeBitfield mode, + CommandCategoryBitfield command_categories); + ~HostLocalCommandProcessor() override; + + bool is_recording() const override { return is_recording_; } + + Status Begin() override; + Status End() override; + + Status ExecutionBarrier( + ExecutionStageBitfield source_stage_mask, + ExecutionStageBitfield target_stage_mask, + absl::Span memory_barriers, + absl::Span buffer_barriers) override; + + Status SignalEvent(Event* event, + ExecutionStageBitfield source_stage_mask) override; + + Status ResetEvent(Event* event, + ExecutionStageBitfield source_stage_mask) override; + + Status WaitEvents(absl::Span events, + ExecutionStageBitfield source_stage_mask, + ExecutionStageBitfield target_stage_mask, + absl::Span memory_barriers, + absl::Span buffer_barriers) override; + + Status FillBuffer(Buffer* target_buffer, device_size_t target_offset, + device_size_t length, const void* pattern, + size_t pattern_length) override; + + Status DiscardBuffer(Buffer* buffer) override; + + Status UpdateBuffer(const void* source_buffer, device_size_t source_offset, + Buffer* target_buffer, device_size_t target_offset, + device_size_t length) override; + + Status CopyBuffer(Buffer* source_buffer, device_size_t source_offset, + Buffer* target_buffer, device_size_t target_offset, + device_size_t length) override; + + Status Dispatch(const DispatchRequest& dispatch_request) override; + + private: + bool is_recording_ = false; +}; + +} // namespace hal +} // namespace iree + +#endif // THIRD_PARTY_MLIR_EDGE_IREE_HAL_HOST_HOST_LOCAL_COMMAND_PROCESSOR_H_ diff --git a/hal/host/host_submission_queue.cc b/hal/host/host_submission_queue.cc new file mode 100644 index 000000000000..b006ae82ab08 --- /dev/null +++ b/hal/host/host_submission_queue.cc @@ -0,0 +1,295 @@ +// Copyright 2019 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "third_party/mlir_edge/iree/hal/host/host_submission_queue.h" + +#include +#include + +#include "third_party/absl/synchronization/mutex.h" +#include "third_party/mlir_edge/iree/base/status.h" +#include "third_party/mlir_edge/iree/base/tracing.h" + +namespace iree { +namespace hal { + +HostBinarySemaphore::HostBinarySemaphore(bool initial_value) { + State state = {0}; + state.signaled = initial_value ? 1 : 0; + state_ = state; +} + +bool HostBinarySemaphore::is_signaled() const { + return state_.load(std::memory_order_acquire).signaled == 1; +} + +Status HostBinarySemaphore::BeginSignaling() { + State old_state = state_.load(std::memory_order_acquire); + if (old_state.signal_pending != 0) { + return FailedPreconditionErrorBuilder(ABSL_LOC) + << "A signal operation on a binary semaphore is already pending"; + } + State new_state = old_state; + new_state.signal_pending = 1; + state_.compare_exchange_strong(old_state, new_state); + return OkStatus(); +} + +Status HostBinarySemaphore::EndSignaling() { + State old_state = state_.load(std::memory_order_acquire); + DCHECK_EQ(old_state.signal_pending, 1) + << "A signal operation on a binary semaphore was not pending"; + if (old_state.signaled != 0) { + return FailedPreconditionErrorBuilder(ABSL_LOC) + << "A binary semaphore cannot be signaled multiple times"; + } + State new_state = old_state; + new_state.signal_pending = 0; + new_state.signaled = 1; + state_.compare_exchange_strong(old_state, new_state); + return OkStatus(); +} + +Status HostBinarySemaphore::BeginWaiting() { + State old_state = state_.load(std::memory_order_acquire); + if (old_state.wait_pending != 0) { + return FailedPreconditionErrorBuilder(ABSL_LOC) + << "A wait operation on a binary semaphore is already pending"; + } + State new_state = old_state; + new_state.wait_pending = 1; + state_.compare_exchange_strong(old_state, new_state); + return OkStatus(); +} + +Status HostBinarySemaphore::EndWaiting() { + State old_state = state_.load(std::memory_order_acquire); + DCHECK_EQ(old_state.wait_pending, 1) + << "A wait operation on a binary semaphore was not pending"; + if (old_state.signaled != 1) { + return FailedPreconditionErrorBuilder(ABSL_LOC) + << "A binary semaphore cannot be reset multiple times"; + } + State new_state = old_state; + new_state.wait_pending = 0; + new_state.signaled = 0; + state_.compare_exchange_strong(old_state, new_state); + return OkStatus(); +} + +HostSubmissionQueue::HostSubmissionQueue() = default; + +HostSubmissionQueue::~HostSubmissionQueue() = default; + +bool HostSubmissionQueue::IsBatchReady(const PendingBatch& batch) const { + for (auto& wait_point : batch.wait_semaphores) { + if (wait_point.index() == 0) { + auto* binary_semaphore = + reinterpret_cast(absl::get<0>(wait_point)); + if (!binary_semaphore->is_signaled()) { + return false; + } + } else { + // TODO(b/140141417): implement timeline semaphores. + return false; + } + } + return true; +} + +Status HostSubmissionQueue::Enqueue(absl::Span batches, + FenceValue fence) { + IREE_TRACE_SCOPE0("HostSubmissionQueue::Enqueue"); + + if (has_shutdown_) { + return FailedPreconditionErrorBuilder(ABSL_LOC) + << "Cannot enqueue new submissions; queue is exiting"; + } else if (!permanent_error_.ok()) { + return permanent_error_; + } + + // Verify waiting/signaling behavior on semaphores and prepare them all. + // We need to track this to ensure that we are modeling the Vulkan behavior + // and are consistent across HAL implementations. + for (auto& batch : batches) { + for (auto& semaphore_value : batch.wait_semaphores) { + if (semaphore_value.index() == 0) { + auto* binary_semaphore = reinterpret_cast( + absl::get<0>(semaphore_value)); + RETURN_IF_ERROR(binary_semaphore->BeginWaiting()); + } else { + // TODO(b/140141417): implement timeline semaphores. + return UnimplementedErrorBuilder(ABSL_LOC) << "Timeline semaphores NYI"; + } + } + for (auto& semaphore_value : batch.signal_semaphores) { + if (semaphore_value.index() == 0) { + auto* binary_semaphore = reinterpret_cast( + absl::get<0>(semaphore_value)); + RETURN_IF_ERROR(binary_semaphore->BeginSignaling()); + } else { + // TODO(b/140141417): implement timeline semaphores. + return UnimplementedErrorBuilder(ABSL_LOC) << "Timeline semaphores NYI"; + } + } + } + + // Add to list - order does not matter as Process evaluates semaphores. + auto submission = absl::make_unique(); + submission->fence = std::move(fence); + submission->pending_batches.resize(batches.size()); + for (int i = 0; i < batches.size(); ++i) { + submission->pending_batches[i] = PendingBatch{ + {batches[i].wait_semaphores.begin(), batches[i].wait_semaphores.end()}, + {batches[i].command_buffers.begin(), batches[i].command_buffers.end()}, + {batches[i].signal_semaphores.begin(), + batches[i].signal_semaphores.end()}, + }; + } + list_.push_back(std::move(submission)); + + return OkStatus(); +} + +Status HostSubmissionQueue::ProcessBatches(ExecuteFn execute_fn) { + IREE_TRACE_SCOPE0("HostSubmissionQueue::ProcessBatches"); + + if (!permanent_error_.ok()) { + // Sticky failure state. + return permanent_error_; + } + + // Repeated try to run things until we quiesce or are blocked. + while (permanent_error_.ok() && !list_.empty()) { + // NOTE: to support re-entrancy where |execute_fn| may modify the submission + // list we need to always start from the beginning. If we wanted we could + // track a list of ready submissions however that's a lot of bookkeeping and + // the list is usually short. + bool restart_iteration = false; + for (auto* submission : list_) { + for (int i = 0; i < submission->pending_batches.size(); ++i) { + auto& batch = submission->pending_batches[i]; + if (!IsBatchReady(batch)) { + // Try the next batch in the submission until we find one that is + // ready. If none are ready we'll return to the caller. + continue; + } + + // Batch can run! Process now and remove it from the list so we don't + // try to run it again. + auto batch_status = ProcessBatch(batch, execute_fn); + submission->pending_batches.erase(submission->pending_batches.begin() + + i); + if (batch_status.ok()) { + // Batch succeeded. Since we want to preserve submission order we'll + // break out of the loop and try from the first submission again. + if (submission->pending_batches.empty()) { + // All work for this submission completed successfully. Signal the + // fence and remove the submission from the list. + RETURN_IF_ERROR(CompleteSubmission(submission, OkStatus())); + list_.take(submission).reset(); + } + } else { + // Batch failed; set the permanent error flag and abort so we don't + // try to process anything else. + permanent_error_ = batch_status; + RETURN_IF_ERROR(CompleteSubmission(submission, batch_status)); + list_.take(submission).reset(); + } + restart_iteration = true; + break; + } + if (restart_iteration) break; + } + } + + if (!permanent_error_.ok()) { + // If the sticky error got set while processing we need to abort all + // remaining submissions (simulating a device loss). + FailAllPending(permanent_error_); + return permanent_error_; + } + + return OkStatus(); +} + +Status HostSubmissionQueue::ProcessBatch(const PendingBatch& batch, + const ExecuteFn& execute_fn) { + IREE_TRACE_SCOPE0("HostSubmissionQueue::ProcessBatch"); + + // Complete the waits on all semaphores and reset them. + for (auto& semaphore_value : batch.wait_semaphores) { + if (semaphore_value.index() == 0) { + auto* binary_semaphore = + reinterpret_cast(absl::get<0>(semaphore_value)); + RETURN_IF_ERROR(binary_semaphore->EndWaiting()); + } else { + // TODO(b/140141417): implement timeline semaphores. + return UnimplementedErrorBuilder(ABSL_LOC) << "Timeline semaphores NYI"; + } + } + + // Let the caller handle execution of the command buffers. + RETURN_IF_ERROR(execute_fn(batch.command_buffers)); + + // Signal all semaphores to allow them to unblock waiters. + for (auto& semaphore_value : batch.signal_semaphores) { + if (semaphore_value.index() == 0) { + auto* binary_semaphore = + reinterpret_cast(absl::get<0>(semaphore_value)); + RETURN_IF_ERROR(binary_semaphore->EndSignaling()); + } else { + // TODO(b/140141417): implement timeline semaphores. + return UnimplementedErrorBuilder(ABSL_LOC) << "Timeline semaphores NYI"; + } + } + + return OkStatus(); +} + +Status HostSubmissionQueue::CompleteSubmission(Submission* submission, + Status status) { + IREE_TRACE_SCOPE0("HostSubmissionQueue::CompleteSubmission"); + + // It's safe to drop any remaining batches - their semaphores will never be + // signaled but that's fine as we should be the only thing relying on them. + submission->pending_batches.clear(); + + // Signal the fence. + auto* fence = static_cast(submission->fence.first); + if (status.ok()) { + RETURN_IF_ERROR(fence->Signal(submission->fence.second)); + } else { + RETURN_IF_ERROR(fence->Fail(std::move(status))); + } + + return OkStatus(); +} + +void HostSubmissionQueue::FailAllPending(Status status) { + IREE_TRACE_SCOPE0("HostSubmissionQueue::FailAllPending"); + while (!list_.empty()) { + auto submission = list_.take(list_.front()); + CompleteSubmission(submission.get(), status).IgnoreError(); + submission.reset(); + } +} + +void HostSubmissionQueue::SignalShutdown() { + IREE_TRACE_SCOPE0("HostSubmissionQueue::SignalShutdown"); + has_shutdown_ = true; +} + +} // namespace hal +} // namespace iree diff --git a/hal/host/host_submission_queue.h b/hal/host/host_submission_queue.h new file mode 100644 index 000000000000..b77b87db0909 --- /dev/null +++ b/hal/host/host_submission_queue.h @@ -0,0 +1,162 @@ +// Copyright 2019 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#ifndef THIRD_PARTY_MLIR_EDGE_IREE_HAL_HOST_HOST_SUBMISSION_QUEUE_H_ +#define THIRD_PARTY_MLIR_EDGE_IREE_HAL_HOST_HOST_SUBMISSION_QUEUE_H_ + +#include "third_party/absl/base/thread_annotations.h" +#include "third_party/absl/synchronization/mutex.h" +#include "third_party/mlir_edge/iree/base/intrusive_list.h" +#include "third_party/mlir_edge/iree/base/status.h" +#include "third_party/mlir_edge/iree/hal/command_queue.h" +#include "third_party/mlir_edge/iree/hal/host/host_fence.h" +#include "third_party/mlir_edge/iree/hal/semaphore.h" + +namespace iree { +namespace hal { + +class HostSubmissionQueue; + +// Simple host-only binary semaphore implemented with a mutex. +// To match the expected HAL behavior (mostly dictated by Vulkan) we can only +// have a single waiter and waits can only occur once a signal has been +// enqueued. +// +// Thread-safe (as instances may be imported and used by others). +class HostBinarySemaphore final : public BinarySemaphore { + public: + explicit HostBinarySemaphore(bool initial_value); + + // Returns true if the semaphore has been signaled. + bool is_signaled() const; + + private: + friend class HostSubmissionQueue; + + // Begins a signal operation and ensures no other signal operation is pending. + Status BeginSignaling(); + // Ends a signal operation by setting the semaphore to the signaled state. + Status EndSignaling(); + + // Begins a wait operation and ensures no other wait operation is pending. + Status BeginWaiting(); + // Ends a wait operation by resetting the semaphore to the unsignaled state. + Status EndWaiting(); + + // A single 32-bit int for lock-free semaphore behavior. We need to do this + // extra tracking so that we get consistent behavior across HAL + // implementations that have strict semaphore semantics. + struct State { + uint32_t signal_pending : 1; + uint32_t wait_pending : 1; + uint32_t signaled : 1; + }; + std::atomic state_{{0}}; +}; + +// Simple host-only timeline semaphore implemented with a mutex. +// +// Thread-safe (as instances may be imported and used by others). +class HostTimelineSemaphore final : public TimelineSemaphore { + public: + // TODO(b/140141417): implement timeline semaphores. +}; + +// A queue managing CommandQueue submissions that uses host-local +// synchronization primitives. Evaluates submission order by respecting the +// wait and signal semaphores defined per batch and notifies fences upon +// submission completion. +// +// Note that it's possible for HAL users to deadlock themselves; we don't try to +// avoid that as in device backends it may not be possible and we want to have +// some kind of warning in the host implementation that TSAN can catch. +// +// Thread-compatible. Const methods may be called from any thread. +class HostSubmissionQueue { + public: + using ExecuteFn = + std::function command_buffers)>; + + HostSubmissionQueue(); + ~HostSubmissionQueue(); + + // Returns true if the queue is currently empty. + bool empty() const { return list_.empty(); } + // Returns true if SignalShutdown has been called. + bool has_shutdown() const { return has_shutdown_; } + // The sticky error status, if an error has occurred. + Status permanent_error() const { return permanent_error_; } + + // Enqueues a new submission. + // No work will be performed until Process is called. + Status Enqueue(absl::Span batches, FenceValue fence); + + // Processes all ready batches using the provided |execute_fn|. + // The function may be called several times if new batches become ready due to + // prior batches in the sequence completing during processing. + // + // Returns any errors returned by |execute_fn| (which will be the same as + // permanent_error()). When an error occurs all in-flight submissions are + // aborted, the permanent_error() is set, and the queue is shutdown. + Status ProcessBatches(ExecuteFn execute_fn); + + // Marks the queue as having shutdown. All pending submissions will be allowed + // to complete but future enqueues will fail. + void SignalShutdown(); + + private: + // A submitted command buffer batch and its synchronization information. + struct PendingBatch { + absl::InlinedVector wait_semaphores; + absl::InlinedVector command_buffers; + absl::InlinedVector signal_semaphores; + }; + struct Submission : public IntrusiveLinkBase { + absl::InlinedVector pending_batches; + FenceValue fence; + }; + + // Returns true if all wait semaphores in the |batch| are signaled. + bool IsBatchReady(const PendingBatch& batch) const; + + // Processes a batch by resetting semaphores, dispatching the command buffers + // to the specified |execute_fn|, and signaling semaphores. + // + // Preconditions: IsBatchReady(batch) == true + Status ProcessBatch(const PendingBatch& batch, const ExecuteFn& execute_fn); + + // Completes a submission by signaling the fence with the given |status|. + Status CompleteSubmission(Submission* submission, Status status); + + // Fails all pending submissions with the given status. + // Errors that occur during this process are silently ignored. + void FailAllPending(Status status); + + // True to exit the thread after all submissions complete. + bool has_shutdown_ = false; + + // A sticky error that is set on the first failed submit. All future + // submissions will be skipped except for fences, which will receive this + // error. + Status permanent_error_; + + // Pending submissions in submission order. + // Note that we may evaluate batches within the list out of order. + IntrusiveList> list_; +}; + +} // namespace hal +} // namespace iree + +#endif // THIRD_PARTY_MLIR_EDGE_IREE_HAL_HOST_HOST_SUBMISSION_QUEUE_H_ diff --git a/hal/host/inproc_command_buffer.cc b/hal/host/inproc_command_buffer.cc new file mode 100644 index 000000000000..7ed7269c2632 --- /dev/null +++ b/hal/host/inproc_command_buffer.cc @@ -0,0 +1,264 @@ +// Copyright 2019 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "third_party/mlir_edge/iree/hal/host/inproc_command_buffer.h" + +#include "third_party/mlir_edge/iree/base/tracing.h" + +namespace iree { +namespace hal { + +InProcCommandBuffer::InProcCommandBuffer( + Allocator* allocator, CommandBufferModeBitfield mode, + CommandCategoryBitfield command_categories) + : CommandBuffer(allocator, mode, command_categories) {} + +InProcCommandBuffer::~InProcCommandBuffer() { Reset(); } + +Status InProcCommandBuffer::Begin() { + IREE_TRACE_SCOPE0("InProcCommandBuffer::Begin"); + is_recording_ = true; + Reset(); + return OkStatus(); +} + +Status InProcCommandBuffer::End() { + IREE_TRACE_SCOPE0("InProcCommandBuffer::End"); + is_recording_ = false; + return OkStatus(); +} + +Status InProcCommandBuffer::ExecutionBarrier( + ExecutionStageBitfield source_stage_mask, + ExecutionStageBitfield target_stage_mask, + absl::Span memory_barriers, + absl::Span buffer_barriers) { + IREE_TRACE_SCOPE0("InProcCommandBuffer::ExecutionBarrier"); + ASSIGN_OR_RETURN(auto* cmd, AppendCmd()); + cmd->source_stage_mask = source_stage_mask; + cmd->target_stage_mask = target_stage_mask; + cmd->memory_barriers = AppendStructSpan(memory_barriers); + cmd->buffer_barriers = AppendStructSpan(buffer_barriers); + return OkStatus(); +} + +Status InProcCommandBuffer::SignalEvent( + Event* event, ExecutionStageBitfield source_stage_mask) { + IREE_TRACE_SCOPE0("InProcCommandBuffer::SignalEvent"); + ASSIGN_OR_RETURN(auto* cmd, AppendCmd()); + cmd->event = event; + cmd->source_stage_mask = source_stage_mask; + return OkStatus(); +} + +Status InProcCommandBuffer::ResetEvent( + Event* event, ExecutionStageBitfield source_stage_mask) { + IREE_TRACE_SCOPE0("InProcCommandBuffer::ResetEvent"); + ASSIGN_OR_RETURN(auto* cmd, AppendCmd()); + cmd->event = event; + cmd->source_stage_mask = source_stage_mask; + return OkStatus(); +} + +Status InProcCommandBuffer::WaitEvents( + absl::Span events, ExecutionStageBitfield source_stage_mask, + ExecutionStageBitfield target_stage_mask, + absl::Span memory_barriers, + absl::Span buffer_barriers) { + IREE_TRACE_SCOPE0("InProcCommandBuffer::WaitEvents"); + ASSIGN_OR_RETURN(auto* cmd, AppendCmd()); + cmd->events = AppendStructSpan(events); + cmd->source_stage_mask = source_stage_mask; + cmd->target_stage_mask = target_stage_mask; + cmd->memory_barriers = AppendStructSpan(memory_barriers); + cmd->buffer_barriers = AppendStructSpan(buffer_barriers); + return OkStatus(); +} + +Status InProcCommandBuffer::FillBuffer(Buffer* target_buffer, + device_size_t target_offset, + device_size_t length, + const void* pattern, + size_t pattern_length) { + IREE_TRACE_SCOPE0("InProcCommandBuffer::FillBuffer"); + ASSIGN_OR_RETURN(auto* cmd, AppendCmd()); + cmd->target_buffer = target_buffer; + cmd->target_offset = target_offset; + cmd->length = length; + std::memcpy(cmd->pattern, pattern, pattern_length); + cmd->pattern_length = pattern_length; + return OkStatus(); +} + +Status InProcCommandBuffer::DiscardBuffer(Buffer* buffer) { + IREE_TRACE_SCOPE0("InProcCommandBuffer::DiscardBuffer"); + ASSIGN_OR_RETURN(auto* cmd, AppendCmd()); + cmd->buffer = buffer; + return OkStatus(); +} + +Status InProcCommandBuffer::UpdateBuffer(const void* source_buffer, + device_size_t source_offset, + Buffer* target_buffer, + device_size_t target_offset, + device_size_t length) { + IREE_TRACE_SCOPE0("InProcCommandBuffer::UpdateBuffer"); + ASSIGN_OR_RETURN(auto* cmd, AppendCmd()); + cmd->source_buffer = AppendCmdData(source_buffer, source_offset, length); + cmd->target_buffer = target_buffer; + cmd->target_offset = target_offset; + cmd->length = length; + return OkStatus(); +} + +Status InProcCommandBuffer::CopyBuffer(Buffer* source_buffer, + device_size_t source_offset, + Buffer* target_buffer, + device_size_t target_offset, + device_size_t length) { + IREE_TRACE_SCOPE0("InProcCommandBuffer::CopyBuffer"); + ASSIGN_OR_RETURN(auto* cmd, AppendCmd()); + cmd->source_buffer = source_buffer; + cmd->source_offset = source_offset; + cmd->target_buffer = target_buffer; + cmd->target_offset = target_offset; + cmd->length = length; + return OkStatus(); +} + +Status InProcCommandBuffer::Dispatch(const DispatchRequest& dispatch_request) { + IREE_TRACE_SCOPE0("InProcCommandBuffer::Dispatch"); + ASSIGN_OR_RETURN(auto* cmd, AppendCmd()); + cmd->request.executable = dispatch_request.executable; + cmd->request.entry_point = dispatch_request.entry_point; + cmd->request.workload = dispatch_request.workload; + cmd->request.workload_buffer = dispatch_request.workload_buffer; + cmd->request.bindings = AppendStructSpan(dispatch_request.bindings); + return OkStatus(); +} + +void InProcCommandBuffer::Reset() { + auto* cmd_list = ¤t_cmd_list_; + cmd_list->head = cmd_list->tail = nullptr; + cmd_list->arena.Reset(); +} + +InProcCommandBuffer::CmdHeader* InProcCommandBuffer::AppendCmdHeader( + CmdType type, size_t cmd_size) { + auto* cmd_list = ¤t_cmd_list_; + auto* cmd_header = reinterpret_cast( + cmd_list->arena.AllocateBytes(sizeof(CmdHeader) + cmd_size)); + cmd_header->next = nullptr; + cmd_header->type = type; + if (!cmd_list->head) { + cmd_list->head = cmd_header; + } else if (cmd_list->tail) { + cmd_list->tail->next = cmd_header; + } + cmd_list->tail = cmd_header; + return cmd_header; +} + +void* InProcCommandBuffer::AppendCmdData(const void* source_buffer, + device_size_t source_offset, + device_size_t source_length) { + auto* cmd_list = ¤t_cmd_list_; + + uint8_t* allocated_bytes = cmd_list->arena.AllocateBytes(source_length); + std::memcpy(allocated_bytes, + static_cast(source_buffer) + source_offset, + source_length); + return allocated_bytes; +} + +Status InProcCommandBuffer::Process(CommandBuffer* command_processor) const { + IREE_TRACE_SCOPE0("InProcCommandBuffer::Process"); + + RETURN_IF_ERROR(command_processor->Begin()); + + // Process each command in the order they were recorded. + auto* cmd_list = ¤t_cmd_list_; + for (CmdHeader* cmd_header = cmd_list->head; cmd_header != nullptr; + cmd_header = cmd_header->next) { + auto command_status = ProcessCmd(cmd_header, command_processor); + if (!command_status.ok()) { + LOG(ERROR) << "DeviceQueue failure while executing command; permanently " + "failing all future commands: " + << command_status; + } + } + + RETURN_IF_ERROR(command_processor->End()); + + return OkStatus(); +} + +Status InProcCommandBuffer::ProcessCmd(CmdHeader* cmd_header, + CommandBuffer* command_processor) const { + switch (cmd_header->type) { + case CmdType::kExecutionBarrier: { + auto* cmd = reinterpret_cast(cmd_header + 1); + return command_processor->ExecutionBarrier( + cmd->source_stage_mask, cmd->target_stage_mask, cmd->memory_barriers, + cmd->buffer_barriers); + } + case CmdType::kSignalEvent: { + auto* cmd = reinterpret_cast(cmd_header + 1); + return command_processor->SignalEvent(cmd->event, cmd->source_stage_mask); + } + case CmdType::kResetEvent: { + auto* cmd = reinterpret_cast(cmd_header + 1); + return command_processor->ResetEvent(cmd->event, cmd->source_stage_mask); + } + case CmdType::kWaitEvents: { + auto* cmd = reinterpret_cast(cmd_header + 1); + return command_processor->WaitEvents( + cmd->events, cmd->source_stage_mask, cmd->target_stage_mask, + cmd->memory_barriers, cmd->buffer_barriers); + } + case CmdType::kFillBuffer: { + auto* cmd = reinterpret_cast(cmd_header + 1); + return command_processor->FillBuffer(cmd->target_buffer, + cmd->target_offset, cmd->length, + cmd->pattern, cmd->pattern_length); + } + case CmdType::kDiscardBuffer: { + auto* cmd = reinterpret_cast(cmd_header + 1); + return command_processor->DiscardBuffer(cmd->buffer); + } + case CmdType::kUpdateBuffer: { + auto* cmd = reinterpret_cast(cmd_header + 1); + return command_processor->UpdateBuffer(cmd->source_buffer, 0, + cmd->target_buffer, + cmd->target_offset, cmd->length); + } + case CmdType::kCopyBuffer: { + auto* cmd = reinterpret_cast(cmd_header + 1); + return command_processor->CopyBuffer( + cmd->source_buffer, cmd->source_offset, cmd->target_buffer, + cmd->target_offset, cmd->length); + } + case CmdType::kDispatch: { + auto* cmd = reinterpret_cast(cmd_header + 1); + return command_processor->Dispatch(cmd->request); + } + default: + return DataLossErrorBuilder(ABSL_LOC) + << "Unrecognized command type " + << static_cast(cmd_header->type) << "; corrupt buffer?"; + } +} + +} // namespace hal +} // namespace iree diff --git a/hal/host/inproc_command_buffer.h b/hal/host/inproc_command_buffer.h new file mode 100644 index 000000000000..0e5496a36afe --- /dev/null +++ b/hal/host/inproc_command_buffer.h @@ -0,0 +1,241 @@ +// Copyright 2019 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#ifndef THIRD_PARTY_MLIR_EDGE_IREE_HAL_HOST_INPROC_COMMAND_BUFFER_H_ +#define THIRD_PARTY_MLIR_EDGE_IREE_HAL_HOST_INPROC_COMMAND_BUFFER_H_ + +#include "third_party/mlir_edge/iree/base/arena.h" +#include "third_party/mlir_edge/iree/base/intrusive_list.h" +#include "third_party/mlir_edge/iree/base/status.h" +#include "third_party/mlir_edge/iree/hal/command_buffer.h" + +namespace iree { +namespace hal { + +// In-process command buffer with support for recording and playback. +// Commands are recorded into heap-allocated arenas with pointers to used +// resources (Buffer*, etc). To replay a command buffer against a real +// implementation use Process to call each command method as it was originally +// recorded. +// +// Thread-compatible (as with CommandBuffer itself). +class InProcCommandBuffer final : public CommandBuffer { + public: + InProcCommandBuffer(Allocator* allocator, CommandBufferModeBitfield mode, + CommandCategoryBitfield command_categories); + ~InProcCommandBuffer() override; + + bool is_recording() const override { return is_recording_; } + + Status Begin() override; + Status End() override; + + Status ExecutionBarrier( + ExecutionStageBitfield source_stage_mask, + ExecutionStageBitfield target_stage_mask, + absl::Span memory_barriers, + absl::Span buffer_barriers) override; + + Status SignalEvent(Event* event, + ExecutionStageBitfield source_stage_mask) override; + + Status ResetEvent(Event* event, + ExecutionStageBitfield source_stage_mask) override; + + Status WaitEvents(absl::Span events, + ExecutionStageBitfield source_stage_mask, + ExecutionStageBitfield target_stage_mask, + absl::Span memory_barriers, + absl::Span buffer_barriers) override; + + Status FillBuffer(Buffer* target_buffer, device_size_t target_offset, + device_size_t length, const void* pattern, + size_t pattern_length) override; + + Status DiscardBuffer(Buffer* buffer) override; + + Status UpdateBuffer(const void* source_buffer, device_size_t source_offset, + Buffer* target_buffer, device_size_t target_offset, + device_size_t length) override; + + Status CopyBuffer(Buffer* source_buffer, device_size_t source_offset, + Buffer* target_buffer, device_size_t target_offset, + device_size_t length) override; + + Status Dispatch(const DispatchRequest& dispatch_request) override; + + // Processes all commands in the buffer using the given |command_processor|. + // The commands are issued in the order they were recorded. + Status Process(CommandBuffer* command_processor) const; + + private: + // Type of Cmd, used by CmdHeader to identify the command payload. + enum class CmdType { + kExecutionBarrier, + kSignalEvent, + kResetEvent, + kWaitEvents, + kFillBuffer, + kDiscardBuffer, + kUpdateBuffer, + kCopyBuffer, + kDispatch, + }; + + // Prefix for commands encoded into the CmdList. + // This is used to identify the type of a command as well as connect commands + // in the list sequence. Command data immediately follows the header in + // memory. + struct CmdHeader { + // Optional next command in the list. + CmdHeader* next; + // Type of the command. + CmdType type; + }; + + // A lightweight linked list of commands and an arena that stores them. + // CmdLists are designed to be reused so that the arena allocations are + // amortized across multiple uses. + // + // Note that this and the CmdHeader/Cmd types include raw pointers and as + // such are *not* portable across processes. It'd be possible, though, to + // extend this for cross-process use if a shared-memory Buffer was also + // implemented. For YAGNI we avoid that here. + struct CmdList : public IntrusiveLinkBase { + static constexpr size_t kArenaBlockSize = 64 * 1024; + + Arena arena{kArenaBlockSize}; + CmdHeader* head = nullptr; + CmdHeader* tail = nullptr; + }; + + // Defines an execution barrier. + struct ExecutionBarrierCmd { + static constexpr CmdType kType = CmdType::kExecutionBarrier; + ExecutionStageBitfield source_stage_mask; + ExecutionStageBitfield target_stage_mask; + absl::Span memory_barriers; + absl::Span buffer_barriers; + }; + + // Signals an event. + struct SignalEventCmd { + static constexpr CmdType kType = CmdType::kSignalEvent; + Event* event; + ExecutionStageBitfield source_stage_mask; + }; + + // Resets an event. + struct ResetEventCmd { + static constexpr CmdType kType = CmdType::kResetEvent; + Event* event; + ExecutionStageBitfield source_stage_mask; + }; + + // Waits for one or more events. + struct WaitEventsCmd { + static constexpr CmdType kType = CmdType::kWaitEvents; + absl::Span events; + ExecutionStageBitfield source_stage_mask; + ExecutionStageBitfield target_stage_mask; + absl::Span memory_barriers; + absl::Span buffer_barriers; + }; + + // Fills the target buffer with the given repeating value. + struct FillBufferCmd { + static constexpr CmdType kType = CmdType::kFillBuffer; + Buffer* target_buffer; + device_size_t target_offset; + device_size_t length; + uint8_t pattern[4]; + size_t pattern_length; + }; + + // Hints to the device queue that the given buffer will not be used again. + struct DiscardBufferCmd { + static constexpr CmdType kType = CmdType::kDiscardBuffer; + Buffer* buffer; + }; + + // Writes a range of the given target buffer from the embedded memory. + // The source buffer contents immediately follow the command in the arena. + struct UpdateBufferCmd { + static constexpr CmdType kType = CmdType::kUpdateBuffer; + const void* source_buffer; + Buffer* target_buffer; + device_size_t target_offset; + device_size_t length; + }; + + // Copies a range of one buffer to another. + struct CopyBufferCmd { + static constexpr CmdType kType = CmdType::kCopyBuffer; + Buffer* source_buffer; + device_size_t source_offset; + Buffer* target_buffer; + device_size_t target_offset; + device_size_t length; + }; + + // Dispatches an execution request. + struct DispatchCmd { + static constexpr CmdType kType = CmdType::kDispatch; + DispatchRequest request; + }; + + // Resets the command list. + void Reset(); + + // Allocates a command and appends it to the current command list. + // The caller must populate the fields in the returned pointer. + template + StatusOr AppendCmd() { + return reinterpret_cast(AppendCmdHeader(T::kType, sizeof(T)) + 1); + } + + // Appends a command with the given |type| and payload |cmd_size| prefixed + // with a CmdHeader. Returns a pointer to the CmdHeader that is followed + // immediately by |cmd_size| zero bytes. + CmdHeader* AppendCmdHeader(CmdType type, size_t cmd_size); + + // Appends a byte buffer to the command buffer and returns a pointer to the + // copied data within the command buffer arena. + void* AppendCmdData(const void* source_buffer, device_size_t source_offset, + device_size_t source_length); + + // Appends a span of POD structs to the current CmdList and returns a span + // pointing into the CmdList arena. + template + absl::Span AppendStructSpan(absl::Span value) { + static_assert(std::is_standard_layout::value, + "Struct must be a POD type"); + void* data_ptr = AppendCmdData(value.data(), 0, value.size() * sizeof(T)); + return absl::MakeSpan(static_cast(data_ptr), value.size()); + } + + // Processes a single command. + Status ProcessCmd(CmdHeader* cmd_header, + CommandBuffer* command_processor) const; + + bool is_recording_ = false; + + // NOTE: not synchronized. Expected to be used from a single thread. + CmdList current_cmd_list_; +}; + +} // namespace hal +} // namespace iree + +#endif // THIRD_PARTY_MLIR_EDGE_IREE_HAL_HOST_INPROC_COMMAND_BUFFER_H_ diff --git a/hal/interpreter/bytecode_cache.cc b/hal/interpreter/bytecode_cache.cc new file mode 100644 index 000000000000..3396d442b103 --- /dev/null +++ b/hal/interpreter/bytecode_cache.cc @@ -0,0 +1,54 @@ +// Copyright 2019 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "third_party/mlir_edge/iree/hal/interpreter/bytecode_cache.h" + +#include "third_party/absl/types/source_location.h" +#include "third_party/mlir_edge/iree/base/status.h" +#include "third_party/mlir_edge/iree/base/tracing.h" +#include "third_party/mlir_edge/iree/hal/executable_format.h" +#include "third_party/mlir_edge/iree/hal/interpreter/bytecode_executable.h" + +namespace iree { +namespace hal { + +BytecodeCache::BytecodeCache(hal::Allocator* allocator) + : allocator_(allocator) {} + +BytecodeCache::~BytecodeCache() = default; + +bool BytecodeCache::CanPrepareFormat(ExecutableFormat format) const { + return format == kExecutableFormatIreeBytecode; +} + +StatusOr> BytecodeCache::PrepareExecutable( + ExecutableCachingModeBitfield mode, const ExecutableSpec& spec) { + IREE_TRACE_SCOPE0("BytecodeCache::PrepareExecutable"); + if (!CanPrepareFormat(spec.format)) { + return UnimplementedErrorBuilder(ABSL_LOC) + << "Unsupported format: " << spec.format; + } + + // Wrap the data (or copy it). + bool allow_aliasing_data = + AllBitsSet(mode, ExecutableCachingMode::kAliasProvidedData); + ASSIGN_OR_RETURN( + auto executable, + BytecodeExecutable::Load(allocator_, spec, !allow_aliasing_data)); + + return executable; +} + +} // namespace hal +} // namespace iree diff --git a/hal/interpreter/bytecode_cache.h b/hal/interpreter/bytecode_cache.h new file mode 100644 index 000000000000..30faa199caa1 --- /dev/null +++ b/hal/interpreter/bytecode_cache.h @@ -0,0 +1,42 @@ +// Copyright 2019 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#ifndef THIRD_PARTY_MLIR_EDGE_IREE_HAL_INTERPRETER_BYTECODE_CACHE_H_ +#define THIRD_PARTY_MLIR_EDGE_IREE_HAL_INTERPRETER_BYTECODE_CACHE_H_ + +#include "third_party/mlir_edge/iree/hal/allocator.h" +#include "third_party/mlir_edge/iree/hal/executable.h" +#include "third_party/mlir_edge/iree/hal/executable_cache.h" + +namespace iree { +namespace hal { + +class BytecodeCache final : public ExecutableCache { + public: + explicit BytecodeCache(hal::Allocator* allocator); + ~BytecodeCache() override; + + bool CanPrepareFormat(ExecutableFormat format) const override; + + StatusOr> PrepareExecutable( + ExecutableCachingModeBitfield mode, const ExecutableSpec& spec) override; + + private: + hal::Allocator* allocator_; +}; + +} // namespace hal +} // namespace iree + +#endif // THIRD_PARTY_MLIR_EDGE_IREE_HAL_INTERPRETER_BYTECODE_CACHE_H_ diff --git a/hal/interpreter/bytecode_dispatch.cc b/hal/interpreter/bytecode_dispatch.cc new file mode 100644 index 000000000000..232aee478f4f --- /dev/null +++ b/hal/interpreter/bytecode_dispatch.cc @@ -0,0 +1,865 @@ +// Copyright 2019 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// Implements a full bytecode dispatch system. +// Currently this is verbose and object oriented, but future revisions +// (once we have interesting benchmarks) will likely simplify and inline +// a lot of the checks to make things faster. Consider this to be as +// experimental an implementation as the entire rest of the project :) + +#include "third_party/mlir_edge/iree/hal/interpreter/bytecode_dispatch.h" + +#include + +#include "third_party/absl/base/attributes.h" +#include "third_party/absl/container/inlined_vector.h" +#include "third_party/absl/types/span.h" +#include "third_party/mlir_edge/iree/base/logging.h" +#include "third_party/mlir_edge/iree/base/memory.h" +#include "third_party/mlir_edge/iree/base/status.h" +#include "third_party/mlir_edge/iree/hal/buffer_view.h" +#include "third_party/mlir_edge/iree/hal/heap_buffer.h" +#include "third_party/mlir_edge/iree/hal/interpreter/bytecode_dispatch_conversion.h" +#include "third_party/mlir_edge/iree/hal/interpreter/bytecode_dispatch_util.h" +#include "third_party/mlir_edge/iree/hal/interpreter/bytecode_kernels.h" +#include "third_party/mlir_edge/iree/schemas/bytecode/interpreter_bytecode_v0.h" +#include "third_party/mlir_edge/iree/vm/bytecode_reader.h" +#include "third_party/mlir_edge/iree/vm/bytecode_tables_interpreter.h" +#include "third_party/mlir_edge/iree/vm/bytecode_util.h" +#include "third_party/mlir_edge/iree/vm/function.h" +#include "third_party/mlir_edge/iree/vm/opcode_info.h" + +namespace iree { +namespace hal { + +namespace { + +using ::iree::vm::BytecodeReader; +using ::iree::vm::ImportFunction; +using ::iree::vm::NativeFunction; +using ::iree::vm::Stack; +using ::iree::vm::StackFrame; + +} // namespace + +Status Dispatch(hal::Allocator* allocator, + kernels::RuntimeState* kernel_runtime_state, Stack* stack, + StackFrame* entry_stack_frame, + absl::Span entry_results) { + // Dispatch table mapping 1:1 with bytecode ops. + // Each entry is a label within this function that can be used for computed + // goto. You can find more information on computed goto here: + // https://eli.thegreenplace.net/2012/07/12/computed-goto-for-efficient-dispatch-tables + // + // Note that we ensure the table is 256 elements long exactly to make sure + // that unused opcodes are handled gracefully. + static const void* kDispatchTable[256] = { +#define DECLARE_DISPATCH(ordinal, name, ...) &&_dispatch_##name, +#define DECLARE_DISPATCH_RESERVED(ordinal, name, ...) &&_dispatch_unhandled, + IREE_INTERPRETER_OPCODE_LIST(DECLARE_DISPATCH, DECLARE_DISPATCH_RESERVED) +#undef DECLARE_DISPATCH +#undef DECLARE_DISPATCH_RESERVED + }; + + // Primary dispatch state. This is our 'native stack frame' and really just + // enough to make dereferencing common addresses (like the current offset) + // faster. You can think of this like CPU state (like PC). + // + // We hope that LLVM decides to keep these in registers (as they are touched + // for every instruction executed). The stack_frame will change as we call + // into different functions. + BytecodeReader reader(stack); + RETURN_IF_ERROR(reader.SwitchStackFrame(entry_stack_frame)); + +#define DISPATCH_NEXT() \ + { \ + uint8_t opcode = *reader.AdvanceOffset().ValueOrDie(); \ + DVLOG(1) \ + << "Interpreter dispatching op code: " \ + << GetOpcodeInfo(vm::interpreter_opcode_table(), opcode).mnemonic; \ + goto* kDispatchTable[opcode]; \ + } + +#define DISPATCH_CORE_OPCODE(opcode, body) \ + _dispatch_##opcode : {body} DISPATCH_NEXT() +#if defined(IREE_SUPPORT_F32) || defined(IREE_SUPPORT_F64) +#define DISPATCH_FLOAT_OPCODE(opcode, body) \ + _dispatch_##opcode : {body} DISPATCH_NEXT() +#else +#define DISPATCH_FLOAT_OPCODE(...) +#endif // IREE_SUPPORT_F32 || IREE_SUPPORT_F64 + + DISPATCH_NEXT(); + + DISPATCH_CORE_OPCODE(kConstant, { + ASSIGN_OR_RETURN(auto value, reader.ReadConstant()); + ASSIGN_OR_RETURN(auto* dst_local, reader.ReadLocal()); + *dst_local = std::move(value); + }); + + DISPATCH_CORE_OPCODE(kCall, { + auto* old_stack_frame = stack->current_frame(); + ASSIGN_OR_RETURN(const auto& target_function, reader.ReadFunction()); + ASSIGN_OR_RETURN(auto* new_stack_frame, stack->PushFrame(target_function)); + RETURN_IF_ERROR( + reader.CopyInputsAndSwitchStackFrame(old_stack_frame, new_stack_frame)); + DVLOG(1) << "Call; stack now: " << stack->DebugString(); + }); + + DISPATCH_CORE_OPCODE(kCallImport, { + auto* old_stack_frame = stack->current_frame(); + ASSIGN_OR_RETURN(const auto* target_function, reader.ReadImportFunction()); + switch (target_function->link_type()) { + case ImportFunction::LinkType::kModule: { + ASSIGN_OR_RETURN(auto* new_stack_frame, + stack->PushFrame(target_function->linked_function())); + RETURN_IF_ERROR(reader.CopyInputsAndSwitchStackFrame(old_stack_frame, + new_stack_frame)); + DVLOG(1) << "Call module import; stack now: " << stack->DebugString(); + break; + } + case ImportFunction::LinkType::kNativeFunction: { + ASSIGN_OR_RETURN(auto* new_stack_frame, + stack->PushFrame(*target_function)); + RETURN_IF_ERROR(reader.CopyInputsAndSwitchStackFrame(old_stack_frame, + new_stack_frame)); + DVLOG(1) << "Call native import; stack now: " << stack->DebugString(); + RETURN_IF_ERROR(CallNativeFunction(stack, *target_function)); + RETURN_IF_ERROR(reader.CopyResultsAndSwitchStackFrame(old_stack_frame, + new_stack_frame)); + RETURN_IF_ERROR(stack->PopFrame()); + DVLOG(1) << "Return from native; stack now: " << stack->DebugString(); + break; + } + } + }); + + DISPATCH_CORE_OPCODE(kCallIndirect, { + return UnimplementedErrorBuilder(ABSL_LOC) << "Unimplemented call_indirect"; + }); + + DISPATCH_CORE_OPCODE(kReturn, { + auto* old_stack_frame = stack->current_frame(); + auto* new_stack_frame = stack->caller_frame(); + if (old_stack_frame == entry_stack_frame) { + // Returning from entry function. Marshal results from the return stmt. + ASSIGN_OR_RETURN(int32_t src_count, reader.ReadCount()); + for (int i = 0; i < src_count; ++i) { + ASSIGN_OR_RETURN(auto* src_local, + reader.ReadLocal(old_stack_frame->mutable_locals())); + entry_results[i] = std::move(*src_local); + } + DVLOG(1) << "Returning to entry"; + return OkStatus(); + } else if (!new_stack_frame) { + return FailedPreconditionErrorBuilder(ABSL_LOC) << "Stack underflow"; + } + RETURN_IF_ERROR(reader.CopyResultsAndSwitchStackFrame(old_stack_frame, + new_stack_frame)); + RETURN_IF_ERROR(stack->PopFrame()); + DVLOG(1) << "Return; stack now: " << stack->DebugString(); + }); + + DISPATCH_CORE_OPCODE(kBranch, { + ASSIGN_OR_RETURN(int32_t offset, reader.ReadBlockOffset()); + RETURN_IF_ERROR(reader.CopySlots()); + RETURN_IF_ERROR(reader.BranchToOffset(offset)); + }); + + DISPATCH_CORE_OPCODE(kCondBranch, { + // Evaluate condition first so we can do the copies as we read them for + // which side of the branch we take. + ASSIGN_OR_RETURN(auto* cond_local, reader.ReadLocal()); + bool cond_value = BufferViewIsTrue(*cond_local); + ASSIGN_OR_RETURN(int32_t true_offset, reader.ReadBlockOffset()); + if (cond_value) { + RETURN_IF_ERROR(reader.CopySlots()); + RETURN_IF_ERROR(reader.BranchToOffset(true_offset)); + } else { + ASSIGN_OR_RETURN(int32_t true_op_count, reader.ReadCount()); + RETURN_IF_ERROR(reader.SkipLocals(2 * true_op_count)); + ASSIGN_OR_RETURN(int32_t false_offset, reader.ReadBlockOffset()); + RETURN_IF_ERROR(reader.CopySlots()); + RETURN_IF_ERROR(reader.BranchToOffset(false_offset)); + } + }); + + DISPATCH_CORE_OPCODE(kCmpI, { + ASSIGN_OR_RETURN(uint8_t predicate, reader.ReadUint8_t()); + ASSIGN_OR_RETURN(auto* lhs_local, reader.ReadLocal()); + ASSIGN_OR_RETURN(auto* rhs_local, reader.ReadLocal()); + ASSIGN_OR_RETURN(auto* dst_local, reader.ReadLocal()); + + switch (static_cast(predicate)) { + case CmpIPredicate::kEq: + RETURN_IF_ERROR(ApplyComparisonOpIS( + lhs_local, rhs_local, dst_local)); + break; + case CmpIPredicate::kNe: + RETURN_IF_ERROR(ApplyComparisonOpIS( + lhs_local, rhs_local, dst_local)); + break; + case CmpIPredicate::kSlt: + RETURN_IF_ERROR(ApplyComparisonOpIS( + lhs_local, rhs_local, dst_local)); + break; + case CmpIPredicate::kSle: + RETURN_IF_ERROR(ApplyComparisonOpIS( + lhs_local, rhs_local, dst_local)); + break; + case CmpIPredicate::kSgt: + RETURN_IF_ERROR(ApplyComparisonOpIS( + lhs_local, rhs_local, dst_local)); + break; + case CmpIPredicate::kSge: + RETURN_IF_ERROR(ApplyComparisonOpIS( + lhs_local, rhs_local, dst_local)); + break; + case CmpIPredicate::kUlt: + RETURN_IF_ERROR(ApplyComparisonOpIU( + lhs_local, rhs_local, dst_local)); + break; + case CmpIPredicate::kUle: + RETURN_IF_ERROR(ApplyComparisonOpIU( + lhs_local, rhs_local, dst_local)); + break; + case CmpIPredicate::kUgt: + RETURN_IF_ERROR(ApplyComparisonOpIU( + lhs_local, rhs_local, dst_local)); + break; + case CmpIPredicate::kUge: + RETURN_IF_ERROR(ApplyComparisonOpIU( + lhs_local, rhs_local, dst_local)); + break; + } + }); + + DISPATCH_FLOAT_OPCODE(kCmpF, { + ASSIGN_OR_RETURN(uint8_t p, reader.ReadUint8_t()); + ASSIGN_OR_RETURN(auto* lhs_local, reader.ReadLocal()); + ASSIGN_OR_RETURN(auto* rhs_local, reader.ReadLocal()); + ASSIGN_OR_RETURN(auto* dst_local, reader.ReadLocal()); + + auto predicate = static_cast(p); + switch (predicate) { + case CmpFPredicate::kOeq: + RETURN_IF_ERROR(ApplyComparisonOpF( + lhs_local, rhs_local, dst_local)); + break; + case CmpFPredicate::kUne: + RETURN_IF_ERROR(ApplyComparisonOpF( + lhs_local, rhs_local, dst_local)); + break; + case CmpFPredicate::kOlt: + RETURN_IF_ERROR(ApplyComparisonOpF( + lhs_local, rhs_local, dst_local)); + break; + case CmpFPredicate::kOle: + RETURN_IF_ERROR(ApplyComparisonOpF( + lhs_local, rhs_local, dst_local)); + break; + case CmpFPredicate::kOgt: + RETURN_IF_ERROR(ApplyComparisonOpF( + lhs_local, rhs_local, dst_local)); + break; + case CmpFPredicate::kOge: + RETURN_IF_ERROR(ApplyComparisonOpF( + lhs_local, rhs_local, dst_local)); + break; + case CmpFPredicate::kFalse: + case CmpFPredicate::kOne: + case CmpFPredicate::kOrd: + case CmpFPredicate::kUeq: + case CmpFPredicate::kUgt: + case CmpFPredicate::kUge: + case CmpFPredicate::kUlt: + case CmpFPredicate::kUle: + case CmpFPredicate::kUno: + case CmpFPredicate::kTrue: + // TODO(b/132183250) support these if we ever need them. + return UnimplementedErrorBuilder(ABSL_LOC) + << "Unsupported comparison predicate value " + << static_cast(p) << " (" + << vm::PredicateToString(predicate) << ")"; + } + }); + + DISPATCH_CORE_OPCODE(kAllocStatic, { + return UnimplementedErrorBuilder(ABSL_LOC) << "Unimplemented alloc_static"; + }); + + DISPATCH_CORE_OPCODE(kAllocStack, { + return UnimplementedErrorBuilder(ABSL_LOC) << "Unimplemented alloc_stack"; + }); + + DISPATCH_CORE_OPCODE(kAllocStackInit, { + return UnimplementedErrorBuilder(ABSL_LOC) + << "Unimplemented alloc_stack_init"; + }); + + DISPATCH_CORE_OPCODE(kAllocHeap, { + ASSIGN_OR_RETURN(auto heap_type, reader.ReadInt32()); + ASSIGN_OR_RETURN(auto type, reader.ReadType()); + size_t element_size = type.element_size(); + + // TODO(benvanik): more efficient reading and storage. + size_t element_count = 0; + ASSIGN_OR_RETURN(auto shape, reader.ReadShapePieces(&element_count)); + size_t allocation_size = element_size * element_count; + + ASSIGN_OR_RETURN(auto* dst_local, reader.ReadLocal()); + dst_local->element_size = element_size; + dst_local->shape = shape; + + // TODO(benvanik): properly allocate with attributes from op. + CHECK_EQ(heap_type, 0); + ASSIGN_OR_RETURN( + dst_local->buffer, + allocator->Allocate(MemoryType::kHostLocal | MemoryType::kDeviceVisible, + BufferUsage::kAll, allocation_size)); + }); + + DISPATCH_CORE_OPCODE(kDiscard, { + // NOTE: if we were an encoder we would actually discard the buffer. + ASSIGN_OR_RETURN(auto* local, reader.ReadLocal()); + *local = {}; + }); + + DISPATCH_CORE_OPCODE(kRank, { + ASSIGN_OR_RETURN(auto* src_local, reader.ReadLocal()); + ASSIGN_OR_RETURN(auto* dst_local, reader.ReadLocal()); + int32_t rank = src_local->shape.size(); + RETURN_IF_ERROR(dst_local->buffer->WriteData(0, &rank, sizeof(int32_t))); + }); + + DISPATCH_CORE_OPCODE(kDim, { + ASSIGN_OR_RETURN(int32_t axis, reader.ReadUint8_t()); + ASSIGN_OR_RETURN(auto* src_local, reader.ReadLocal()); + ASSIGN_OR_RETURN(auto* dst_local, reader.ReadLocal()); + ASSIGN_OR_RETURN(int32_t dim, src_local->shape.ResolveAxis(axis)); + RETURN_IF_ERROR(dst_local->buffer->WriteData(0, &dim, sizeof(int32_t))); + }); + + DISPATCH_CORE_OPCODE(kShape, { + ASSIGN_OR_RETURN(auto* src_local, reader.ReadLocal()); + ASSIGN_OR_RETURN(auto* dst_local, reader.ReadLocal()); + RETURN_IF_ERROR(dst_local->buffer->WriteData( + 0, src_local->shape.subspan().data(), + src_local->shape.subspan().size() * sizeof(int32_t))); + }); + + DISPATCH_CORE_OPCODE(kLength, { + ASSIGN_OR_RETURN(auto* src_local, reader.ReadLocal()); + ASSIGN_OR_RETURN(auto* dst_local, reader.ReadLocal()); + int32_t length = src_local->shape.element_count(); + RETURN_IF_ERROR(dst_local->buffer->WriteData(0, &length, sizeof(int32_t))); + }); + + DISPATCH_CORE_OPCODE(kDynamicSlice, { + ASSIGN_OR_RETURN(auto* src_local, reader.ReadLocal()); + ASSIGN_OR_RETURN(auto indices, reader.ReadSlotElements()); + ASSIGN_OR_RETURN(auto lengths, reader.ReadSlotElements()); + ASSIGN_OR_RETURN(auto* dst_local, reader.ReadLocal()); + ASSIGN_OR_RETURN(*dst_local, src_local->Slice(indices, lengths)); + }); + + DISPATCH_CORE_OPCODE(kStaticSlice, { + ASSIGN_OR_RETURN(auto* src_local, reader.ReadLocal()); + ASSIGN_OR_RETURN(auto indices, reader.ReadIndexList()); + ASSIGN_OR_RETURN(auto lengths, reader.ReadIndexList()); + ASSIGN_OR_RETURN(auto* dst_local, reader.ReadLocal()); + ASSIGN_OR_RETURN(*dst_local, src_local->Slice(indices, lengths)); + }); + + DISPATCH_CORE_OPCODE(kDynamicCopy, { + ASSIGN_OR_RETURN(auto* src_local, reader.ReadLocal()); + ASSIGN_OR_RETURN(auto src_indices, reader.ReadSlotElements()); + ASSIGN_OR_RETURN(auto* dst_local, reader.ReadLocal()); + ASSIGN_OR_RETURN(auto dst_indices, reader.ReadSlotElements()); + ASSIGN_OR_RETURN(auto lengths, reader.ReadSlotElements()); + RETURN_IF_ERROR( + ApplyCopy(src_local, src_indices, dst_local, dst_indices, lengths)); + }); + + DISPATCH_CORE_OPCODE(kStaticCopy, { + ASSIGN_OR_RETURN(auto* src_local, reader.ReadLocal()); + ASSIGN_OR_RETURN(auto src_indices, reader.ReadIndexList()); + ASSIGN_OR_RETURN(auto* dst_local, reader.ReadLocal()); + ASSIGN_OR_RETURN(auto dst_indices, reader.ReadIndexList()); + ASSIGN_OR_RETURN(auto lengths, reader.ReadIndexList()); + RETURN_IF_ERROR( + ApplyCopy(src_local, src_indices, dst_local, dst_indices, lengths)); + }); + + DISPATCH_CORE_OPCODE(kClone, { + ASSIGN_OR_RETURN(auto* src_local, reader.ReadLocal()); + ASSIGN_OR_RETURN(auto* dst_local, reader.ReadLocal()); + dst_local->element_size = src_local->element_size; + dst_local->shape = src_local->shape; + dst_local->buffer = HeapBuffer::Allocate(src_local->buffer->usage(), + src_local->buffer->byte_length()); + RETURN_IF_ERROR(dst_local->buffer->CopyData(0, src_local->buffer.get())); + }); + + DISPATCH_CORE_OPCODE(kSplit, { + return UnimplementedErrorBuilder(ABSL_LOC) << "Unimplemented split"; + }); + + DISPATCH_CORE_OPCODE(kAssign, { + ASSIGN_OR_RETURN(auto* src_local, reader.ReadLocal()); + ASSIGN_OR_RETURN(auto* dst_local, reader.ReadLocal()); + *dst_local = *src_local; + }); + + DISPATCH_CORE_OPCODE(kCondAssign, { + ASSIGN_OR_RETURN(auto* cond_local, reader.ReadLocal()); + ASSIGN_OR_RETURN(auto* lhs_local, reader.ReadLocal()); + ASSIGN_OR_RETURN(auto* rhs_local, reader.ReadLocal()); + ASSIGN_OR_RETURN(auto* dst_local, reader.ReadLocal()); + *dst_local = BufferViewIsTrue(*cond_local) ? *lhs_local : *rhs_local; + }); + + DISPATCH_CORE_OPCODE(kReshape, { + // TODO(benvanik): more logic required if strides differ. + ASSIGN_OR_RETURN(auto* src_local, reader.ReadLocal()); + ASSIGN_OR_RETURN(auto shape_data, reader.ReadSlotElements()); + ASSIGN_OR_RETURN(auto* dst_local, reader.ReadLocal()); + Shape new_shape = Shape{shape_data}; + if (src_local->shape.element_count() != new_shape.element_count()) { + return InvalidArgumentErrorBuilder(ABSL_LOC) + << "New element count " << new_shape.element_count() + << " != source element count " << src_local->shape.element_count(); + } + dst_local->shape = new_shape; + dst_local->buffer = add_ref(src_local->buffer); + dst_local->element_size = src_local->element_size; + }); + + DISPATCH_CORE_OPCODE(kSelect, { + ASSIGN_OR_RETURN(auto* cond_local, reader.ReadLocal()); + ASSIGN_OR_RETURN(auto* lhs_local, reader.ReadLocal()); + ASSIGN_OR_RETURN(auto* rhs_local, reader.ReadLocal()); + ASSIGN_OR_RETURN(auto* dst_local, reader.ReadLocal()); + ASSIGN_OR_RETURN(auto cond_buffer, cond_local->buffer->MapMemory( + MemoryAccess::kRead)); + ASSIGN_OR_RETURN(auto lhs_buffer, lhs_local->buffer->MapMemory( + MemoryAccess::kRead)); + ASSIGN_OR_RETURN(auto rhs_buffer, rhs_local->buffer->MapMemory( + MemoryAccess::kRead)); + ASSIGN_OR_RETURN(auto dst_buffer, dst_local->buffer->MapMemory( + MemoryAccess::kDiscardWrite)); + if (cond_local->element_size != 1) { + return InvalidArgumentErrorBuilder(ABSL_LOC) << "Select cond must be i8"; + } else if (lhs_buffer.size() != rhs_buffer.size()) { + return InvalidArgumentErrorBuilder(ABSL_LOC) + << "LHS " << lhs_buffer.size() << "b != RHS " << rhs_buffer.size() + << "b; both arguments must match"; + } else if (lhs_buffer.size() != dst_buffer.size()) { + return InvalidArgumentErrorBuilder(ABSL_LOC) + << "Dest " << dst_buffer.size() << "b != LHS/RHS " + << lhs_buffer.size() << "b; dest must match inputs"; + } + switch (lhs_local->element_size) { + case 1: + RETURN_IF_ERROR(kernels::Select::Execute( + cond_buffer.contents(), lhs_buffer.contents(), + rhs_buffer.contents(), dst_buffer.mutable_contents())); + break; + case 2: + RETURN_IF_ERROR(kernels::Select::Execute( + cond_buffer.contents(), + ReinterpretSpan(lhs_buffer.contents()), + ReinterpretSpan(rhs_buffer.contents()), + ReinterpretSpan(dst_buffer.mutable_contents()))); + break; + case 4: + RETURN_IF_ERROR(kernels::Select::Execute( + cond_buffer.contents(), + ReinterpretSpan(lhs_buffer.contents()), + ReinterpretSpan(rhs_buffer.contents()), + ReinterpretSpan(dst_buffer.mutable_contents()))); + break; + case 8: + RETURN_IF_ERROR(kernels::Select::Execute( + cond_buffer.contents(), + ReinterpretSpan(lhs_buffer.contents()), + ReinterpretSpan(rhs_buffer.contents()), + ReinterpretSpan(dst_buffer.mutable_contents()))); + break; + default: + return UnimplementedErrorBuilder(ABSL_LOC) + << "Unimplemented element size: " << lhs_local->element_size; + } + }); + + DISPATCH_CORE_OPCODE(kTranspose, { + ASSIGN_OR_RETURN(auto* src_local, reader.ReadLocal()); + ASSIGN_OR_RETURN(auto perm_data, reader.ReadSlotElements()); + ASSIGN_OR_RETURN(auto* dst_local, reader.ReadLocal()); + RETURN_IF_ERROR(ApplyUnaryOpIU( + src_local, dst_local, src_local->shape, + absl::MakeConstSpan(perm_data))); + }); + + DISPATCH_CORE_OPCODE(kReverse, { + ASSIGN_OR_RETURN(auto* src_local, reader.ReadLocal()); + ASSIGN_OR_RETURN(auto perm_data, reader.ReadSlotElements()); + ASSIGN_OR_RETURN(auto* dst_local, reader.ReadLocal()); + RETURN_IF_ERROR( + ApplyUnaryOpIU(src_local, dst_local, src_local->shape, + absl::MakeConstSpan(perm_data))); + }); + + DISPATCH_CORE_OPCODE(kPad, { + ASSIGN_OR_RETURN(auto* src_local, reader.ReadLocal()); + ASSIGN_OR_RETURN(auto* padding_value, reader.ReadLocal()); + ASSIGN_OR_RETURN(auto edge_padding_low, reader.ReadSlotElements()); + ASSIGN_OR_RETURN(auto edge_padding_high, + reader.ReadSlotElements()); + ASSIGN_OR_RETURN(auto interior_padding, reader.ReadSlotElements()); + ASSIGN_OR_RETURN(auto* dst_local, reader.ReadLocal()); + + RETURN_IF_ERROR(ApplyBinaryOpIU( + src_local, padding_value, dst_local, src_local->shape, dst_local->shape, + absl::MakeConstSpan(edge_padding_low), + absl::MakeConstSpan(edge_padding_high), + absl::MakeConstSpan(interior_padding))); + }); + + DISPATCH_CORE_OPCODE(kBroadcast, { + ASSIGN_OR_RETURN(auto* src_local, reader.ReadLocal()); + ASSIGN_OR_RETURN(auto shape_data, reader.ReadSlotElements()); + ASSIGN_OR_RETURN(auto* dst_local, reader.ReadLocal()); + dst_local->shape = Shape{shape_data}; + RETURN_IF_ERROR(ApplyUnaryOpIU(src_local, dst_local)); + }); + + DISPATCH_CORE_OPCODE(kTile, { + ASSIGN_OR_RETURN(auto* src_local, reader.ReadLocal()); + ASSIGN_OR_RETURN(auto shape_data, reader.ReadSlotElements()); + ASSIGN_OR_RETURN(auto* dst_local, reader.ReadLocal()); + dst_local->shape = Shape{shape_data}; + RETURN_IF_ERROR(ApplyUnaryOpIU( + src_local, dst_local, src_local->shape, dst_local->shape)); + }); + + DISPATCH_CORE_OPCODE(kNot, { + RETURN_IF_ERROR(DispatchElementwiseUnaryOpIU(&reader)); + }); + DISPATCH_CORE_OPCODE(kAnd, { + RETURN_IF_ERROR(DispatchElementwiseBinaryOpIU(&reader)); + }); + DISPATCH_CORE_OPCODE(kOr, { + RETURN_IF_ERROR(DispatchElementwiseBinaryOpIU(&reader)); + }); + DISPATCH_CORE_OPCODE(kXor, { + RETURN_IF_ERROR(DispatchElementwiseBinaryOpIU(&reader)); + }); + DISPATCH_CORE_OPCODE(kShiftLeft, { + RETURN_IF_ERROR(DispatchElementwiseBinaryOpIU(&reader)); + }); + DISPATCH_CORE_OPCODE(kShiftRightLogical, { + RETURN_IF_ERROR( + DispatchElementwiseBinaryOpIU(&reader)); + }); + DISPATCH_CORE_OPCODE(kShiftRightArithmetic, { + RETURN_IF_ERROR( + DispatchElementwiseBinaryOpIS(&reader)); + }); + + DISPATCH_CORE_OPCODE(kAddI, { + RETURN_IF_ERROR(DispatchElementwiseBinaryOpIU(&reader)); + }); + DISPATCH_FLOAT_OPCODE(kAddF, { + RETURN_IF_ERROR(DispatchElementwiseBinaryOpF(&reader)); + }); + + DISPATCH_CORE_OPCODE(kSubI, { + RETURN_IF_ERROR(DispatchElementwiseBinaryOpIU(&reader)); + }); + DISPATCH_FLOAT_OPCODE(kSubF, { + RETURN_IF_ERROR(DispatchElementwiseBinaryOpF(&reader)); + }); + + DISPATCH_CORE_OPCODE(kAbsI, { + RETURN_IF_ERROR(DispatchElementwiseUnaryOpIS(&reader)); + }); + DISPATCH_FLOAT_OPCODE(kAbsF, { + RETURN_IF_ERROR(DispatchElementwiseUnaryOpF(&reader)); + }); + + DISPATCH_CORE_OPCODE(kMulI, { + RETURN_IF_ERROR(DispatchElementwiseBinaryOpIU(&reader)); + }); + DISPATCH_FLOAT_OPCODE(kMulF, { + RETURN_IF_ERROR(DispatchElementwiseBinaryOpF(&reader)); + }); + + DISPATCH_CORE_OPCODE(kDivIS, { + RETURN_IF_ERROR(DispatchElementwiseBinaryOpIS(&reader)); + }); + DISPATCH_CORE_OPCODE(kDivIU, { + RETURN_IF_ERROR(DispatchElementwiseBinaryOpIU(&reader)); + }); + DISPATCH_FLOAT_OPCODE(kDivF, { + RETURN_IF_ERROR(DispatchElementwiseBinaryOpF(&reader)); + }); + + DISPATCH_CORE_OPCODE(kMulAddI, { + RETURN_IF_ERROR(DispatchElementwiseTernaryOpIU(&reader)); + }); + DISPATCH_FLOAT_OPCODE(kMulAddF, { + RETURN_IF_ERROR(DispatchElementwiseTernaryOpF(&reader)); + }); + DISPATCH_FLOAT_OPCODE(kExpF, { + RETURN_IF_ERROR(DispatchElementwiseUnaryOpF(&reader)); + }); + DISPATCH_FLOAT_OPCODE(kLogF, { + RETURN_IF_ERROR(DispatchElementwiseUnaryOpF(&reader)); + }); + DISPATCH_FLOAT_OPCODE(kRsqrtF, { + RETURN_IF_ERROR(DispatchElementwiseUnaryOpF(&reader)); + }); + DISPATCH_FLOAT_OPCODE(kCosF, { + RETURN_IF_ERROR(DispatchElementwiseUnaryOpF(&reader)); + }); + DISPATCH_FLOAT_OPCODE(kSinF, { + RETURN_IF_ERROR(DispatchElementwiseUnaryOpF(&reader)); + }); + DISPATCH_FLOAT_OPCODE(kTanhF, { + RETURN_IF_ERROR(DispatchElementwiseUnaryOpF(&reader)); + }); + DISPATCH_FLOAT_OPCODE(kAtan2F, { + RETURN_IF_ERROR(DispatchElementwiseBinaryOpF(&reader)); + }); + + DISPATCH_CORE_OPCODE(kMinIS, { + RETURN_IF_ERROR(DispatchElementwiseBinaryOpIS(&reader)); + }); + DISPATCH_CORE_OPCODE(kMinIU, { + RETURN_IF_ERROR(DispatchElementwiseBinaryOpIU(&reader)); + }); + DISPATCH_FLOAT_OPCODE(kMinF, { + RETURN_IF_ERROR(DispatchElementwiseBinaryOpF(&reader)); + }); + + DISPATCH_CORE_OPCODE(kMaxIS, { + RETURN_IF_ERROR(DispatchElementwiseBinaryOpIS(&reader)); + }); + DISPATCH_CORE_OPCODE(kMaxIU, { + RETURN_IF_ERROR(DispatchElementwiseBinaryOpIU(&reader)); + }); + DISPATCH_FLOAT_OPCODE(kMaxF, { + RETURN_IF_ERROR(DispatchElementwiseBinaryOpF(&reader)); + }); + + DISPATCH_CORE_OPCODE(kClampIS, { + RETURN_IF_ERROR(DispatchElementwiseTernaryOpIS(&reader)); + }); + DISPATCH_CORE_OPCODE(kClampIU, { + RETURN_IF_ERROR(DispatchElementwiseTernaryOpIS(&reader)); + }); + DISPATCH_FLOAT_OPCODE(kClampF, { + RETURN_IF_ERROR(DispatchElementwiseTernaryOpF(&reader)); + }); + + DISPATCH_FLOAT_OPCODE(kFloorF, { + RETURN_IF_ERROR(DispatchElementwiseUnaryOpF(&reader)); + }); + DISPATCH_FLOAT_OPCODE(kCeilF, { + RETURN_IF_ERROR(DispatchElementwiseUnaryOpF(&reader)); + }); + + DISPATCH_CORE_OPCODE(kConvertSS, { + ASSIGN_OR_RETURN(auto src_type, reader.ReadType()); + ASSIGN_OR_RETURN(auto* src_local, reader.ReadLocal()); + ASSIGN_OR_RETURN(auto dst_type, reader.ReadType()); + ASSIGN_OR_RETURN(auto* dst_local, reader.ReadLocal()); + RETURN_IF_ERROR( + ApplyConvertSS::Apply(src_type, src_local, dst_type, dst_local)); + }); + DISPATCH_CORE_OPCODE(kConvertUU, { + ASSIGN_OR_RETURN(auto src_type, reader.ReadType()); + ASSIGN_OR_RETURN(auto* src_local, reader.ReadLocal()); + ASSIGN_OR_RETURN(auto dst_type, reader.ReadType()); + ASSIGN_OR_RETURN(auto* dst_local, reader.ReadLocal()); + RETURN_IF_ERROR( + ApplyConvertUU::Apply(src_type, src_local, dst_type, dst_local)); + }); + DISPATCH_CORE_OPCODE(kConvertSU, { + ASSIGN_OR_RETURN(auto src_type, reader.ReadType()); + ASSIGN_OR_RETURN(auto* src_local, reader.ReadLocal()); + ASSIGN_OR_RETURN(auto dst_type, reader.ReadType()); + ASSIGN_OR_RETURN(auto* dst_local, reader.ReadLocal()); + RETURN_IF_ERROR( + ApplyConvertSU::Apply(src_type, src_local, dst_type, dst_local)); + }); + DISPATCH_CORE_OPCODE(kConvertUS, { + ASSIGN_OR_RETURN(auto src_type, reader.ReadType()); + ASSIGN_OR_RETURN(auto* src_local, reader.ReadLocal()); + ASSIGN_OR_RETURN(auto dst_type, reader.ReadType()); + ASSIGN_OR_RETURN(auto* dst_local, reader.ReadLocal()); + RETURN_IF_ERROR( + ApplyConvertUS::Apply(src_type, src_local, dst_type, dst_local)); + }); + + DISPATCH_CORE_OPCODE(kMatMulI, { + ASSIGN_OR_RETURN(auto* lhs_local, reader.ReadLocal()); + ASSIGN_OR_RETURN(auto* rhs_local, reader.ReadLocal()); + // TODO(benvanik): add fused matmul-with-bias op in MLIR and lower to this. + BufferView* bias_local = nullptr; + ASSIGN_OR_RETURN(auto* multiplier_mantissa_local, reader.ReadLocal()); + ASSIGN_OR_RETURN(auto* multiplier_exponent_local, reader.ReadLocal()); + ASSIGN_OR_RETURN(auto* dst_local, reader.ReadLocal()); + RETURN_IF_ERROR(ValidateMatMulOpI(lhs_local, rhs_local, bias_local, + multiplier_mantissa_local, + multiplier_exponent_local, dst_local)); + auto* mat_mul_state = kernel_runtime_state->mat_mul_state.get(); + // TODO(benvanik): define as a matrix of supported types to enable 8*8=16, + // accumulator options, and other precision modes. + switch (lhs_local->element_size) { + case 1: + RETURN_IF_ERROR(ApplyMatMulOpI( + mat_mul_state, lhs_local, rhs_local, bias_local, + multiplier_mantissa_local, multiplier_exponent_local, dst_local)); + break; + case 2: + RETURN_IF_ERROR(ApplyMatMulOpI( + mat_mul_state, lhs_local, rhs_local, bias_local, + multiplier_mantissa_local, multiplier_exponent_local, dst_local)); + break; + case 4: + RETURN_IF_ERROR(ApplyMatMulOpI( + mat_mul_state, lhs_local, rhs_local, bias_local, + multiplier_mantissa_local, multiplier_exponent_local, dst_local)); + break; + case 8: + RETURN_IF_ERROR(ApplyMatMulOpI( + mat_mul_state, lhs_local, rhs_local, bias_local, + multiplier_mantissa_local, multiplier_exponent_local, dst_local)); + break; + default: + return UnimplementedErrorBuilder(ABSL_LOC) + << "Unimplemented element size: " << lhs_local->element_size; + } + }); + + DISPATCH_FLOAT_OPCODE(kMatMulF, { + ASSIGN_OR_RETURN(auto* lhs_local, reader.ReadLocal()); + ASSIGN_OR_RETURN(auto* rhs_local, reader.ReadLocal()); + BufferView* bias_local = nullptr; + ASSIGN_OR_RETURN(auto* dst_local, reader.ReadLocal()); + RETURN_IF_ERROR( + ValidateMatMulOpF(lhs_local, rhs_local, bias_local, dst_local)); + auto* mat_mul_state = kernel_runtime_state->mat_mul_state.get(); + switch (lhs_local->element_size) { + case 4: + RETURN_IF_ERROR(ApplyMatMulOpF( + mat_mul_state, lhs_local, rhs_local, bias_local, dst_local)); + break; + case 8: + RETURN_IF_ERROR(ApplyMatMulOpF( + mat_mul_state, lhs_local, rhs_local, bias_local, dst_local)); + break; + default: + return UnimplementedErrorBuilder(ABSL_LOC) + << "Unimplemented element size: " << lhs_local->element_size; + } + }); + + DISPATCH_CORE_OPCODE(kReduceSumI, { + ASSIGN_OR_RETURN(auto* src_local, reader.ReadLocal()); + ASSIGN_OR_RETURN(auto* init_local, reader.ReadLocal()); + ASSIGN_OR_RETURN(auto dimension, reader.ReadInt32()); + ASSIGN_OR_RETURN(auto* dst_local, reader.ReadLocal()); + // TODO(scotttodd): validate + RETURN_IF_ERROR(ApplyBinaryOpIS( + src_local, init_local, dst_local, dimension, src_local->shape, + dst_local->shape)); + }); + + DISPATCH_FLOAT_OPCODE(kReduceSumF, { + ASSIGN_OR_RETURN(auto* src_local, reader.ReadLocal()); + ASSIGN_OR_RETURN(auto* init_local, reader.ReadLocal()); + ASSIGN_OR_RETURN(auto dimension, reader.ReadInt32()); + ASSIGN_OR_RETURN(auto* dst_local, reader.ReadLocal()); + // TODO(scotttodd): validate + RETURN_IF_ERROR(ApplyBinaryOpF( + src_local, init_local, dst_local, dimension, src_local->shape, + dst_local->shape)); + }); + + DISPATCH_CORE_OPCODE(kReduceMinI, { + ASSIGN_OR_RETURN(auto* src_local, reader.ReadLocal()); + ASSIGN_OR_RETURN(auto* init_local, reader.ReadLocal()); + ASSIGN_OR_RETURN(auto dimension, reader.ReadInt32()); + ASSIGN_OR_RETURN(auto* dst_local, reader.ReadLocal()); + // TODO(scotttodd): validate + RETURN_IF_ERROR(ApplyBinaryOpIS( + src_local, init_local, dst_local, dimension, src_local->shape, + dst_local->shape)); + }); + + DISPATCH_FLOAT_OPCODE(kReduceMinF, { + ASSIGN_OR_RETURN(auto* src_local, reader.ReadLocal()); + ASSIGN_OR_RETURN(auto* init_local, reader.ReadLocal()); + ASSIGN_OR_RETURN(auto dimension, reader.ReadInt32()); + ASSIGN_OR_RETURN(auto* dst_local, reader.ReadLocal()); + // TODO(scotttodd): validate + RETURN_IF_ERROR(ApplyBinaryOpF( + src_local, init_local, dst_local, dimension, src_local->shape, + dst_local->shape)); + }); + + DISPATCH_CORE_OPCODE(kReduceMaxI, { + ASSIGN_OR_RETURN(auto* src_local, reader.ReadLocal()); + ASSIGN_OR_RETURN(auto* init_local, reader.ReadLocal()); + ASSIGN_OR_RETURN(auto dimension, reader.ReadInt32()); + ASSIGN_OR_RETURN(auto* dst_local, reader.ReadLocal()); + // TODO(scotttodd): validate + RETURN_IF_ERROR(ApplyBinaryOpIS( + src_local, init_local, dst_local, dimension, src_local->shape, + dst_local->shape)); + }); + + DISPATCH_FLOAT_OPCODE(kReduceMaxF, { + ASSIGN_OR_RETURN(auto* src_local, reader.ReadLocal()); + ASSIGN_OR_RETURN(auto* init_local, reader.ReadLocal()); + ASSIGN_OR_RETURN(auto dimension, reader.ReadInt32()); + ASSIGN_OR_RETURN(auto* dst_local, reader.ReadLocal()); + // TODO(scotttodd): validate + RETURN_IF_ERROR(ApplyBinaryOpF( + src_local, init_local, dst_local, dimension, src_local->shape, + dst_local->shape)); + }); + + DISPATCH_CORE_OPCODE(kTrace, { + return UnimplementedErrorBuilder(ABSL_LOC) << "Unimplemented trace"; + }); + + DISPATCH_CORE_OPCODE(kBreak, { + return UnimplementedErrorBuilder(ABSL_LOC) << "Unimplemented break"; + }); + + DISPATCH_CORE_OPCODE(kCondBreak, { + return UnimplementedErrorBuilder(ABSL_LOC) << "Unimplemented cond_break"; + }); + +_dispatch_unhandled: + // TODO(benvanik): better tracing. + return UnimplementedErrorBuilder(ABSL_LOC) << "Unknown dispatch opcode"; +} // NOLINT(readability/fn_size) + +} // namespace hal +} // namespace iree diff --git a/hal/interpreter/bytecode_dispatch.h b/hal/interpreter/bytecode_dispatch.h new file mode 100644 index 000000000000..478a7fd6ddde --- /dev/null +++ b/hal/interpreter/bytecode_dispatch.h @@ -0,0 +1,35 @@ +// Copyright 2019 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#ifndef THIRD_PARTY_MLIR_EDGE_IREE_HAL_INTERPRETER_BYTECODE_DISPATCH_H_ +#define THIRD_PARTY_MLIR_EDGE_IREE_HAL_INTERPRETER_BYTECODE_DISPATCH_H_ + +#include "third_party/mlir_edge/iree/base/status.h" +#include "third_party/mlir_edge/iree/hal/allocator.h" +#include "third_party/mlir_edge/iree/hal/interpreter/bytecode_kernels.h" +#include "third_party/mlir_edge/iree/vm/stack.h" +#include "third_party/mlir_edge/iree/vm/stack_frame.h" + +namespace iree { +namespace hal { + +Status Dispatch(hal::Allocator* allocator, + kernels::RuntimeState* kernel_runtime_state, vm::Stack* stack, + vm::StackFrame* entry_stack_frame, + absl::Span entry_results); + +} // namespace hal +} // namespace iree + +#endif // THIRD_PARTY_MLIR_EDGE_IREE_HAL_INTERPRETER_BYTECODE_DISPATCH_H_ diff --git a/hal/interpreter/bytecode_dispatch_conversion.h b/hal/interpreter/bytecode_dispatch_conversion.h new file mode 100644 index 000000000000..cd3cbe16deae --- /dev/null +++ b/hal/interpreter/bytecode_dispatch_conversion.h @@ -0,0 +1,395 @@ +// Copyright 2019 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// Conversion helper tables. + +#ifndef THIRD_PARTY_MLIR_EDGE_IREE_HAL_INTERPRETER_BYTECODE_DISPATCH_CONVERSION_H_ +#define THIRD_PARTY_MLIR_EDGE_IREE_HAL_INTERPRETER_BYTECODE_DISPATCH_CONVERSION_H_ + +#include "third_party/mlir_edge/iree/base/status.h" +#include "third_party/mlir_edge/iree/hal/buffer_view.h" +#include "third_party/mlir_edge/iree/hal/interpreter/bytecode_dispatch_util.h" +#include "third_party/mlir_edge/iree/schemas/bytecode/interpreter_bytecode_v0.h" +#include "third_party/mlir_edge/iree/vm/type.h" + +namespace iree { +namespace hal { + +template +struct ApplyConversionOp { + static Status Apply(const vm::Type& src_type, BufferView* src_local, + const vm::Type& dst_type, BufferView* dst_local, + ARGS... args) { + // Validate ranges so that we cannot go out of bounds on thunk table. + int src_type_index = src_type.type_index(); + int dst_type_index = dst_type.type_index(); + if (src_type_index < 0 || src_type_index >= kBuiltinTypeCount) { + return InvalidArgumentErrorBuilder(ABSL_LOC) + << "Conversion from invalid source builtin type " + << src_type_index; + } else if (dst_type_index < 0 || dst_type_index >= kBuiltinTypeCount) { + return InvalidArgumentErrorBuilder(ABSL_LOC) + << "Conversion to invalid dest builtin type " << dst_type_index; + } + + // All possible combinations of conversions. + using KernelFn = Status (*)(BufferView * src_local, BufferView * dst_local, + ARGS... args); + KernelFn fn = nullptr; + if (src_signed && dst_signed) { + // Signed -> signed. + static const KernelFn + kConversionTable[kBuiltinTypeCount * kBuiltinTypeCount] = { + // src_type = kI8: + /* kI8 */ Thunk::Apply, + /* kI16 */ Thunk::Apply, + /* kI32 */ Thunk::Apply, + /* kI64 */ Thunk::Apply, + /* kF16 */ nullptr, + /* kF32 */ Thunk::Apply, + /* kF64 */ Thunk::Apply, + + // src_type = kI16: + /* kI8 */ Thunk::Apply, + /* kI16 */ Thunk::Apply, + /* kI32 */ Thunk::Apply, + /* kI64 */ Thunk::Apply, + /* kF16 */ nullptr, + /* kF32 */ Thunk::Apply, + /* kF64 */ Thunk::Apply, + + // src_type = kI32: + /* kI8 */ Thunk::Apply, + /* kI16 */ Thunk::Apply, + /* kI32 */ Thunk::Apply, + /* kI64 */ Thunk::Apply, + /* kF16 */ nullptr, + /* kF32 */ Thunk::Apply, + /* kF64 */ Thunk::Apply, + + // src_type = kI64: + /* kI8 */ Thunk::Apply, + /* kI16 */ Thunk::Apply, + /* kI32 */ Thunk::Apply, + /* kI64 */ Thunk::Apply, + /* kF16 */ nullptr, + /* kF32 */ Thunk::Apply, + /* kF64 */ Thunk::Apply, + + // src_type = kF16: + /* kI8 */ nullptr, + /* kI16 */ nullptr, + /* kI32 */ nullptr, + /* kI64 */ nullptr, + /* kF16 */ Thunk::Apply, + /* kF32 */ nullptr, + /* kF64 */ nullptr, + + // src_type = kF32: + /* kI8 */ Thunk::Apply, + /* kI16 */ Thunk::Apply, + /* kI32 */ Thunk::Apply, + /* kI64 */ Thunk::Apply, + /* kF16 */ nullptr, + /* kF32 */ Thunk::Apply, + /* kF64 */ Thunk::Apply, + + // src_type = kF64: + /* kI8 */ Thunk::Apply, + /* kI16 */ Thunk::Apply, + /* kI32 */ Thunk::Apply, + /* kI64 */ Thunk::Apply, + /* kF16 */ nullptr, + /* kF32 */ Thunk::Apply, + /* kF64 */ Thunk::Apply, + }; + fn = + kConversionTable[src_type_index * kBuiltinTypeCount + dst_type_index]; + } else if (src_signed && !dst_signed) { + // Signed -> unsigned. + static const KernelFn + kConversionTable[kBuiltinTypeCount * kBuiltinTypeCount] = { + // src_type = kI8: + /* kI8 */ Thunk::Apply, + /* kI16 */ Thunk::Apply, + /* kI32 */ Thunk::Apply, + /* kI64 */ Thunk::Apply, + /* kF16 */ nullptr, + /* kF32 */ nullptr, + /* kF64 */ nullptr, + + // src_type = kI16: + /* kI8 */ Thunk::Apply, + /* kI16 */ Thunk::Apply, + /* kI32 */ Thunk::Apply, + /* kI64 */ Thunk::Apply, + /* kF16 */ nullptr, + /* kF32 */ nullptr, + /* kF64 */ nullptr, + + // src_type = kI32: + /* kI8 */ Thunk::Apply, + /* kI16 */ Thunk::Apply, + /* kI32 */ Thunk::Apply, + /* kI64 */ Thunk::Apply, + /* kF16 */ nullptr, + /* kF32 */ nullptr, + /* kF64 */ nullptr, + + // src_type = kI64: + /* kI8 */ Thunk::Apply, + /* kI16 */ Thunk::Apply, + /* kI32 */ Thunk::Apply, + /* kI64 */ Thunk::Apply, + /* kF16 */ nullptr, + /* kF32 */ nullptr, + /* kF64 */ nullptr, + + // src_type = kF16: + /* kI8 */ nullptr, + /* kI16 */ nullptr, + /* kI32 */ nullptr, + /* kI64 */ nullptr, + /* kF16 */ nullptr, + /* kF32 */ nullptr, + /* kF64 */ nullptr, + + // src_type = kF32: + /* kI8 */ Thunk::Apply, + /* kI16 */ Thunk::Apply, + /* kI32 */ Thunk::Apply, + /* kI64 */ Thunk::Apply, + /* kF16 */ nullptr, + /* kF32 */ nullptr, + /* kF64 */ nullptr, + + // src_type = kF64: + /* kI8 */ Thunk::Apply, + /* kI16 */ Thunk::Apply, + /* kI32 */ Thunk::Apply, + /* kI64 */ Thunk::Apply, + /* kF16 */ nullptr, + /* kF32 */ nullptr, + /* kF64 */ nullptr, + }; + fn = + kConversionTable[src_type_index * kBuiltinTypeCount + dst_type_index]; + } else if (!src_signed && dst_signed) { + // Unsigned -> signed. + static const KernelFn + kConversionTable[kBuiltinTypeCount * kBuiltinTypeCount] = { + // src_type = kI8: + /* kI8 */ Thunk::Apply, + /* kI16 */ Thunk::Apply, + /* kI32 */ Thunk::Apply, + /* kI64 */ Thunk::Apply, + /* kF16 */ nullptr, + /* kF32 */ Thunk::Apply, + /* kF64 */ Thunk::Apply, + + // src_type = kI16: + /* kI8 */ Thunk::Apply, + /* kI16 */ Thunk::Apply, + /* kI32 */ Thunk::Apply, + /* kI64 */ Thunk::Apply, + /* kF16 */ nullptr, + /* kF32 */ Thunk::Apply, + /* kF64 */ Thunk::Apply, + + // src_type = kI32: + /* kI8 */ Thunk::Apply, + /* kI16 */ Thunk::Apply, + /* kI32 */ Thunk::Apply, + /* kI64 */ Thunk::Apply, + /* kF16 */ nullptr, + /* kF32 */ Thunk::Apply, + /* kF64 */ Thunk::Apply, + + // src_type = kI64: + /* kI8 */ Thunk::Apply, + /* kI16 */ Thunk::Apply, + /* kI32 */ Thunk::Apply, + /* kI64 */ Thunk::Apply, + /* kF16 */ nullptr, + /* kF32 */ Thunk::Apply, + /* kF64 */ Thunk::Apply, + + // src_type = kF16: + /* kI8 */ nullptr, + /* kI16 */ nullptr, + /* kI32 */ nullptr, + /* kI64 */ nullptr, + /* kF16 */ nullptr, + /* kF32 */ nullptr, + /* kF64 */ nullptr, + + // src_type = kF32: + /* kI8 */ nullptr, + /* kI16 */ nullptr, + /* kI32 */ nullptr, + /* kI64 */ nullptr, + /* kF16 */ nullptr, + /* kF32 */ nullptr, + /* kF64 */ nullptr, + + // src_type = kF64: + /* kI8 */ nullptr, + /* kI16 */ nullptr, + /* kI32 */ nullptr, + /* kI64 */ nullptr, + /* kF16 */ nullptr, + /* kF32 */ nullptr, + /* kF64 */ nullptr, + }; + fn = + kConversionTable[src_type_index * kBuiltinTypeCount + dst_type_index]; + } else if (!src_signed && !dst_signed) { + // Unsigned -> unsigned. + static const KernelFn + kConversionTable[kBuiltinTypeCount * kBuiltinTypeCount] = { + // src_type = kI8: + /* kI8 */ Thunk::Apply, + /* kI16 */ Thunk::Apply, + /* kI32 */ Thunk::Apply, + /* kI64 */ Thunk::Apply, + /* kF16 */ nullptr, + /* kF32 */ nullptr, + /* kF64 */ nullptr, + + // src_type = kI16: + /* kI8 */ Thunk::Apply, + /* kI16 */ Thunk::Apply, + /* kI32 */ Thunk::Apply, + /* kI64 */ Thunk::Apply, + /* kF16 */ nullptr, + /* kF32 */ nullptr, + /* kF64 */ nullptr, + + // src_type = kI32: + /* kI8 */ Thunk::Apply, + /* kI16 */ Thunk::Apply, + /* kI32 */ Thunk::Apply, + /* kI64 */ Thunk::Apply, + /* kF16 */ nullptr, + /* kF32 */ nullptr, + /* kF64 */ nullptr, + + // src_type = kI64: + /* kI8 */ Thunk::Apply, + /* kI16 */ Thunk::Apply, + /* kI32 */ Thunk::Apply, + /* kI64 */ Thunk::Apply, + /* kF16 */ nullptr, + /* kF32 */ nullptr, + /* kF64 */ nullptr, + + // src_type = kF16: + /* kI8 */ nullptr, + /* kI16 */ nullptr, + /* kI32 */ nullptr, + /* kI64 */ nullptr, + /* kF16 */ nullptr, + /* kF32 */ nullptr, + /* kF64 */ nullptr, + + // src_type = kF32: + /* kI8 */ nullptr, + /* kI16 */ nullptr, + /* kI32 */ nullptr, + /* kI64 */ nullptr, + /* kF16 */ nullptr, + /* kF32 */ nullptr, + /* kF64 */ nullptr, + + // src_type = kF64: + /* kI8 */ nullptr, + /* kI16 */ nullptr, + /* kI32 */ nullptr, + /* kI64 */ nullptr, + /* kF16 */ nullptr, + /* kF32 */ nullptr, + /* kF64 */ nullptr, + }; + fn = + kConversionTable[src_type_index * kBuiltinTypeCount + dst_type_index]; + } + if (!fn) { + return InvalidArgumentErrorBuilder(ABSL_LOC) + << "Unsupported conversion from " << src_type_index << " to " + << dst_type_index; + } + return fn(src_local, dst_local, args...); + } + + template + struct Thunk { + static Status Apply(BufferView* src_local, BufferView* dst_local, + ARGS... args) { + ASSIGN_OR_RETURN(auto src_buffer, + src_local->buffer->MapMemory(MemoryAccess::kRead)); + ASSIGN_OR_RETURN(auto dst_buffer, dst_local->buffer->MapMemory( + MemoryAccess::kDiscardWrite)); + return KERNEL::Execute(src_buffer.contents(), + dst_buffer.mutable_contents(), args...); + } + }; + +// Disable F32/F64 conversions if they are not supported. +#if !defined(IREE_SUPPORT_F32) + template + struct Thunk { + static Status Apply(BufferView* src_local, BufferView* dst_local, + ARGS... args) { + return UnimplementedErrorBuilder(ABSL_LOC) << "F32 not supported"; + } + }; + template + struct Thunk { + static Status Apply(BufferView* src_local, BufferView* dst_local, + ARGS... args) { + return UnimplementedErrorBuilder(ABSL_LOC) << "F32 not supported"; + } + }; +#endif // !IREE_SUPPORT_F32 +#if !defined(IREE_SUPPORT_F64) + template + struct Thunk { + static Status Apply(BufferView* src_local, BufferView* dst_local, + ARGS... args) { + return UnimplementedErrorBuilder(ABSL_LOC) << "F64 not supported"; + } + }; + template + struct Thunk { + static Status Apply(BufferView* src_local, BufferView* dst_local, + ARGS... args) { + return UnimplementedErrorBuilder(ABSL_LOC) << "F64 not supported"; + } + }; +#endif // !IREE_SUPPORT_F64 +}; + +using ApplyConvertSS = ApplyConversionOp; +using ApplyConvertUU = ApplyConversionOp; +using ApplyConvertSU = ApplyConversionOp; +using ApplyConvertUS = ApplyConversionOp; + +} // namespace hal +} // namespace iree + +#endif // THIRD_PARTY_MLIR_EDGE_IREE_HAL_INTERPRETER_BYTECODE_DISPATCH_CONVERSION_H_ diff --git a/hal/interpreter/bytecode_dispatch_util.cc b/hal/interpreter/bytecode_dispatch_util.cc new file mode 100644 index 000000000000..72bc313325de --- /dev/null +++ b/hal/interpreter/bytecode_dispatch_util.cc @@ -0,0 +1,119 @@ +// Copyright 2019 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "third_party/mlir_edge/iree/hal/interpreter/bytecode_dispatch_util.h" + +namespace iree { +namespace hal { + +bool BufferViewIsTrue(const BufferView& buffer_view) { + if (buffer_view.element_size == 0 || !buffer_view.buffer || + buffer_view.byte_length() == 0) { + return false; + } + // TODO(benvanik): map more efficiently (based on element size?). + auto mapping = + buffer_view.buffer->MapMemory(hal::MemoryAccess::kRead); + if (!mapping.ok()) { + return false; + } + for (uint8_t value : mapping.ValueOrDie().contents()) { + if (value) return true; + } + return false; +} + +Status CallNativeFunction(vm::Stack* stack, + const vm::ImportFunction& function) { + auto* stack_frame = stack->current_frame(); + + // Marshal inputs and outputs. + auto args = stack_frame->mutable_locals().subspan(0, function.input_count()); + auto results = stack_frame->mutable_locals().subspan(args.size()); + + const auto& fn = function.native_function(); + return fn(stack, args, results); +} + +Status ValidateElementwiseUnaryOp(BufferView* src_local, + BufferView* dst_local) { + // TODO(benvanik): validate shapes. + return OkStatus(); +} + +Status ValidateElementwiseBinaryOp(BufferView* lhs_local, BufferView* rhs_local, + BufferView* dst_local) { + // TODO(benvanik): validate shapes. + return OkStatus(); +} + +Status ValidateElementwiseTernaryOp(BufferView* a_local, BufferView* b_local, + BufferView* c_local, + BufferView* dst_local) { + // TODO(benvanik): validate shapes. + return OkStatus(); +} + +Status ValidateMatMulOpI(BufferView* lhs_local, BufferView* rhs_local, + BufferView* bias_local, + BufferView* multiplier_mantissa_local, + BufferView* multiplier_exponent_local, + BufferView* dst_local) { + // TODO(benvanik): validate shapes. + return OkStatus(); +} + +Status ValidateMatMulOpF(BufferView* lhs_local, BufferView* rhs_local, + BufferView* bias_local, BufferView* dst_local) { + // TODO(benvanik): validate shapes. + return OkStatus(); +} + +Status ApplyCopy(BufferView* src_local, absl::Span src_indices, + BufferView* dst_local, absl::Span dst_indices, + absl::Span lengths) { + ASSIGN_OR_RETURN(auto src_buffer, + src_local->buffer->MapMemory(MemoryAccess::kRead)); + // TODO(benvanik): discard if overwriting the entire buffer. + ASSIGN_OR_RETURN(auto dst_buffer, + dst_local->buffer->MapMemory(MemoryAccess::kWrite)); + switch (src_local->element_size) { + case 1: + return kernels::Copy::Execute<1>(src_buffer.contents(), src_local->shape, + src_indices, + dst_buffer.mutable_contents(), + dst_local->shape, dst_indices, lengths); + case 2: + return kernels::Copy::Execute<2>(src_buffer.contents(), src_local->shape, + src_indices, + dst_buffer.mutable_contents(), + dst_local->shape, dst_indices, lengths); + case 4: + return kernels::Copy::Execute<4>(src_buffer.contents(), src_local->shape, + src_indices, + dst_buffer.mutable_contents(), + dst_local->shape, dst_indices, lengths); + case 8: + return kernels::Copy::Execute<8>(src_buffer.contents(), src_local->shape, + src_indices, + dst_buffer.mutable_contents(), + dst_local->shape, dst_indices, lengths); + default: + return UnimplementedErrorBuilder(ABSL_LOC) + << "Unimplemented element size: " << src_local->element_size; + } +} + +} // namespace hal +} // namespace iree diff --git a/hal/interpreter/bytecode_dispatch_util.h b/hal/interpreter/bytecode_dispatch_util.h new file mode 100644 index 000000000000..477293ee0dc7 --- /dev/null +++ b/hal/interpreter/bytecode_dispatch_util.h @@ -0,0 +1,515 @@ +// Copyright 2019 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// Utilities used by the bytecode_dispatch routines to aid in working with the +// bytecode stream and kernel dispatch. + +#ifndef THIRD_PARTY_MLIR_EDGE_IREE_HAL_INTERPRETER_BYTECODE_DISPATCH_UTIL_H_ +#define THIRD_PARTY_MLIR_EDGE_IREE_HAL_INTERPRETER_BYTECODE_DISPATCH_UTIL_H_ + +#include "third_party/absl/base/attributes.h" +#include "third_party/absl/container/inlined_vector.h" +#include "third_party/mlir_edge/iree/base/status.h" +#include "third_party/mlir_edge/iree/hal/buffer_view.h" +#include "third_party/mlir_edge/iree/hal/heap_buffer.h" +#include "third_party/mlir_edge/iree/hal/interpreter/bytecode_kernels.h" +#include "third_party/mlir_edge/iree/schemas/bytecode/interpreter_bytecode_v0.h" +#include "third_party/mlir_edge/iree/vm/bytecode_reader.h" +#include "third_party/mlir_edge/iree/vm/function.h" +#include "third_party/mlir_edge/iree/vm/stack.h" +#include "third_party/mlir_edge/iree/vm/type.h" + +// TODO(benvanik): move to dedicated config file/build flags. +#define IREE_SUPPORT_F32 1 +#define IREE_SUPPORT_F64 1 + +namespace iree { +namespace hal { + +// Returns true if the contents of the BufferView are bitwise non-zero. +// Returns false if there is no buffer, the buffer is empty, or the contents are +// bitwise zero. +bool BufferViewIsTrue(const BufferView& buffer_view); + +Status CallNativeFunction(vm::Stack* stack, const vm::ImportFunction& function); + +Status ValidateElementwiseUnaryOp(BufferView* src_local, BufferView* dst_local); +Status ValidateElementwiseBinaryOp(BufferView* lhs_local, BufferView* rhs_local, + BufferView* dst_local); +Status ValidateElementwiseTernaryOp(BufferView* a_local, BufferView* b_local, + BufferView* c_local, BufferView* dst_local); +Status ValidateMatMulOpI(BufferView* lhs_local, BufferView* rhs_local, + BufferView* bias_local, + BufferView* multiplier_mantissa_local, + BufferView* multiplier_exponent_local, + BufferView* dst_local); +Status ValidateMatMulOpF(BufferView* lhs_local, BufferView* rhs_local, + BufferView* bias_local, BufferView* dst_local); + +template +Status ApplyUnaryOp(BufferView* src_local, BufferView* dst_local, + ARGS... args) { + // TODO(benvanik): avoid mapping by changing buffer type? + ASSIGN_OR_RETURN(auto src_buffer, + src_local->buffer->MapMemory(MemoryAccess::kRead)); + ASSIGN_OR_RETURN(auto dst_buffer, dst_local->buffer->MapMemory( + MemoryAccess::kDiscardWrite)); + return KERNEL::Execute(src_buffer.contents(), dst_buffer.mutable_contents(), + args...); +} + +template +Status ApplyBinaryOp(BufferView* lhs_local, BufferView* rhs_local, + BufferView* dst_local, ARGS... args) { + ASSIGN_OR_RETURN(auto lhs_buffer, + lhs_local->buffer->MapMemory(MemoryAccess::kRead)); + ASSIGN_OR_RETURN(auto rhs_buffer, + rhs_local->buffer->MapMemory(MemoryAccess::kRead)); + ASSIGN_OR_RETURN(auto dst_buffer, dst_local->buffer->MapMemory( + MemoryAccess::kDiscardWrite)); + return KERNEL::Execute(lhs_buffer.contents(), rhs_buffer.contents(), + dst_buffer.mutable_contents(), args...); +} + +template +Status ApplyTernaryOp(BufferView* a_local, BufferView* b_local, + BufferView* c_local, BufferView* dst_local, + ARGS... args) { + ASSIGN_OR_RETURN(auto a_buffer, + a_local->buffer->MapMemory(MemoryAccess::kRead)); + ASSIGN_OR_RETURN(auto b_buffer, + b_local->buffer->MapMemory(MemoryAccess::kRead)); + ASSIGN_OR_RETURN(auto c_buffer, + c_local->buffer->MapMemory(MemoryAccess::kRead)); + ASSIGN_OR_RETURN(auto dst_buffer, dst_local->buffer->MapMemory( + MemoryAccess::kDiscardWrite)); + return KERNEL::Execute(a_buffer.contents(), b_buffer.contents(), + c_buffer.contents(), dst_buffer.mutable_contents(), + args...); +} + +template +Status ApplyComparisonOp(BufferView* lhs_local, BufferView* rhs_local, + BufferView* dst_local) { + ASSIGN_OR_RETURN(auto lhs_buffer, + lhs_local->buffer->MapMemory(MemoryAccess::kRead)); + ASSIGN_OR_RETURN(auto rhs_buffer, + rhs_local->buffer->MapMemory(MemoryAccess::kRead)); + ASSIGN_OR_RETURN(auto dst_buffer, dst_local->buffer->MapMemory( + MemoryAccess::kDiscardWrite)); + return KERNEL::Execute(lhs_buffer.contents(), rhs_buffer.contents(), + dst_buffer.mutable_contents()); +} + +template +Status ApplyUnaryOpIS(BufferView* src_local, BufferView* dst_local, + ARGS... args) { + switch (src_local->element_size) { + case 1: + return ApplyUnaryOp(src_local, dst_local, args...); + case 2: + return ApplyUnaryOp(src_local, dst_local, args...); + case 4: + return ApplyUnaryOp(src_local, dst_local, args...); + case 8: + return ApplyUnaryOp(src_local, dst_local, args...); + default: + return UnimplementedErrorBuilder(ABSL_LOC) + << "Unimplemented element size: " << src_local->element_size; + } +} + +template +Status ApplyUnaryOpIU(BufferView* src_local, BufferView* dst_local, + ARGS... args) { + switch (src_local->element_size) { + case 1: + return ApplyUnaryOp(src_local, dst_local, args...); + case 2: + return ApplyUnaryOp(src_local, dst_local, args...); + case 4: + return ApplyUnaryOp(src_local, dst_local, args...); + case 8: + return ApplyUnaryOp(src_local, dst_local, args...); + default: + return UnimplementedErrorBuilder(ABSL_LOC) + << "Unimplemented element size: " << src_local->element_size; + } +} + +template +Status ApplyUnaryOpF(BufferView* src_local, BufferView* dst_local, + ARGS... args) { + switch (src_local->element_size) { +#if defined(IREE_SUPPORT_F32) + case 4: + return ApplyUnaryOp(src_local, dst_local, args...); +#endif // IREE_SUPPORT_F32 +#if defined(IREE_SUPPORT_F64) + case 8: + return ApplyUnaryOp(src_local, dst_local, args...); +#endif // IREE_SUPPORT_F64 + default: + return UnimplementedErrorBuilder(ABSL_LOC) + << "Unimplemented element size: " << src_local->element_size; + } +} + +template +Status ApplyBinaryOpIS(BufferView* lhs_local, BufferView* rhs_local, + BufferView* dst_local, ARGS... args) { + switch (lhs_local->element_size) { + case 1: + return ApplyBinaryOp(lhs_local, rhs_local, dst_local, + args...); + case 2: + return ApplyBinaryOp(lhs_local, rhs_local, dst_local, + args...); + case 4: + return ApplyBinaryOp(lhs_local, rhs_local, dst_local, + args...); + case 8: + return ApplyBinaryOp(lhs_local, rhs_local, dst_local, + args...); + default: + return UnimplementedErrorBuilder(ABSL_LOC) + << "Unimplemented element size: " << lhs_local->element_size; + } +} + +template +Status ApplyBinaryOpIU(BufferView* lhs_local, BufferView* rhs_local, + BufferView* dst_local, ARGS... args) { + switch (lhs_local->element_size) { + case 1: + return ApplyBinaryOp(lhs_local, rhs_local, dst_local, + args...); + case 2: + return ApplyBinaryOp(lhs_local, rhs_local, dst_local, + args...); + case 4: + return ApplyBinaryOp(lhs_local, rhs_local, dst_local, + args...); + case 8: + return ApplyBinaryOp(lhs_local, rhs_local, dst_local, + args...); + default: + return UnimplementedErrorBuilder(ABSL_LOC) + << "Unimplemented element size: " << lhs_local->element_size; + } +} + +template +Status ApplyBinaryOpF(BufferView* lhs_local, BufferView* rhs_local, + BufferView* dst_local, ARGS... args) { + switch (lhs_local->element_size) { +#if defined(IREE_SUPPORT_F32) + case 4: + return ApplyBinaryOp(lhs_local, rhs_local, dst_local, + args...); +#endif // IREE_SUPPORT_F32 +#if defined(IREE_SUPPORT_F64) + case 8: + return ApplyBinaryOp(lhs_local, rhs_local, dst_local, + args...); +#endif // IREE_SUPPORT_F64 + default: + return UnimplementedErrorBuilder(ABSL_LOC) + << "Unimplemented element size: " << lhs_local->element_size; + } +} + +template +Status ApplyTernaryOpIS(BufferView* a_local, BufferView* b_local, + BufferView* c_local, BufferView* dst_local, + ARGS... args) { + switch (a_local->element_size) { + case 1: + return ApplyTernaryOp(a_local, b_local, c_local, + dst_local, args...); + case 2: + return ApplyTernaryOp(a_local, b_local, c_local, + dst_local, args...); + case 4: + return ApplyTernaryOp(a_local, b_local, c_local, + dst_local, args...); + case 8: + return ApplyTernaryOp(a_local, b_local, c_local, + dst_local, args...); + default: + return UnimplementedErrorBuilder(ABSL_LOC) + << "Unimplemented element size: " << a_local->element_size; + } +} + +template +Status ApplyTernaryOpIU(BufferView* a_local, BufferView* b_local, + BufferView* c_local, BufferView* dst_local, + ARGS... args) { + switch (a_local->element_size) { + case 1: + return ApplyTernaryOp(a_local, b_local, c_local, + dst_local, args...); + case 2: + return ApplyTernaryOp(a_local, b_local, c_local, + dst_local, args...); + case 4: + return ApplyTernaryOp(a_local, b_local, c_local, + dst_local, args...); + case 8: + return ApplyTernaryOp(a_local, b_local, c_local, + dst_local, args...); + default: + return UnimplementedErrorBuilder(ABSL_LOC) + << "Unimplemented element size: " << a_local->element_size; + } +} + +template +Status ApplyTernaryOpF(BufferView* a_local, BufferView* b_local, + BufferView* c_local, BufferView* dst_local, + ARGS... args) { + switch (a_local->element_size) { +#if defined(IREE_SUPPORT_F32) + case 4: + return ApplyTernaryOp(a_local, b_local, c_local, dst_local, + args...); +#endif // IREE_SUPPORT_F32 +#if defined(IREE_SUPPORT_F64) + case 8: + return ApplyTernaryOp(a_local, b_local, c_local, + dst_local, args...); +#endif // IREE_SUPPORT_F64 + default: + return UnimplementedErrorBuilder(ABSL_LOC) + << "Unimplemented element size: " << a_local->element_size; + } +} + +template +Status ApplyComparisonOpIS(BufferView* lhs_local, BufferView* rhs_local, + BufferView* dst_local) { + switch (lhs_local->element_size) { + case 1: + return ApplyComparisonOp(lhs_local, rhs_local, dst_local); + case 2: + return ApplyComparisonOp(lhs_local, rhs_local, + dst_local); + case 4: + return ApplyComparisonOp(lhs_local, rhs_local, + dst_local); + case 8: + return ApplyComparisonOp(lhs_local, rhs_local, + dst_local); + default: + return UnimplementedErrorBuilder(ABSL_LOC) + << "Unimplemented element size: " << lhs_local->element_size; + } +} + +template +Status ApplyComparisonOpIU(BufferView* lhs_local, BufferView* rhs_local, + BufferView* dst_local) { + switch (lhs_local->element_size) { + case 1: + return ApplyComparisonOp(lhs_local, rhs_local, + dst_local); + case 2: + return ApplyComparisonOp(lhs_local, rhs_local, + dst_local); + case 4: + return ApplyComparisonOp(lhs_local, rhs_local, + dst_local); + case 8: + return ApplyComparisonOp(lhs_local, rhs_local, + dst_local); + default: + return UnimplementedErrorBuilder(ABSL_LOC) + << "Unimplemented element size: " << lhs_local->element_size; + } +} + +template +Status ApplyComparisonOpF(BufferView* lhs_local, BufferView* rhs_local, + BufferView* dst_local) { + switch (lhs_local->element_size) { + case 4: + return ApplyComparisonOp(lhs_local, rhs_local, dst_local); + case 8: + return ApplyComparisonOp(lhs_local, rhs_local, dst_local); + default: + return UnimplementedErrorBuilder(ABSL_LOC) + << "Unimplemented element size: " << lhs_local->element_size; + } +} + +template +Status ApplyMatMulOpI(kernels::MatMul::RuntimeState* runtime_state, + BufferView* lhs_local, BufferView* rhs_local, + BufferView* bias_local, + BufferView* multiplier_mantissa_local, + BufferView* multiplier_exponent_local, + BufferView* dst_local) { + kernels::MatMul::Buffers buffers; + ASSIGN_OR_RETURN(auto lhs_buffer, + lhs_local->buffer->MapMemory(MemoryAccess::kRead)); + buffers.lhs_buffer = lhs_buffer.contents(); + buffers.lhs_shape = lhs_local->shape; + ASSIGN_OR_RETURN(auto rhs_buffer, + rhs_local->buffer->MapMemory(MemoryAccess::kRead)); + buffers.rhs_buffer = rhs_buffer.contents(); + buffers.rhs_shape = rhs_local->shape; + MappedMemory bias_buffer; + if (bias_local && bias_local->buffer && !bias_local->shape.empty()) { + if (bias_local->element_size != sizeof(ACC)) { + return UnimplementedErrorBuilder(ABSL_LOC) + << "Only " << sizeof(ACC) << "b biases are supported right now"; + } + ASSIGN_OR_RETURN(bias_buffer, + bias_local->buffer->MapMemory(MemoryAccess::kRead)); + buffers.bias_buffer = bias_buffer.contents(); + } + ASSIGN_OR_RETURN( + auto multiplier_mantissa_buffer, + multiplier_mantissa_local->buffer->MapMemory(MemoryAccess::kRead)); + buffers.multiplier_mantissa_buffer = multiplier_mantissa_buffer.contents(); + ASSIGN_OR_RETURN(auto multiplier_exponent_buffer, + multiplier_exponent_local->buffer->MapMemory( + MemoryAccess::kRead)); + buffers.multiplier_exponent_buffer = multiplier_exponent_buffer.contents(); + ASSIGN_OR_RETURN(auto dst_buffer, dst_local->buffer->MapMemory( + MemoryAccess::kDiscardWrite)); + buffers.dst_buffer = dst_buffer.mutable_contents(); + buffers.dst_shape = dst_local->shape; + return kernels::MatMul::Execute(runtime_state, buffers); +} + +template +Status ApplyMatMulOpF(kernels::MatMul::RuntimeState* runtime_state, + BufferView* lhs_local, BufferView* rhs_local, + BufferView* bias_local, BufferView* dst_local) { + kernels::MatMul::Buffers buffers; + ASSIGN_OR_RETURN(auto lhs_buffer, + lhs_local->buffer->MapMemory(MemoryAccess::kRead)); + buffers.lhs_buffer = lhs_buffer.contents(); + buffers.lhs_shape = lhs_local->shape; + ASSIGN_OR_RETURN(auto rhs_buffer, + rhs_local->buffer->MapMemory(MemoryAccess::kRead)); + buffers.rhs_buffer = rhs_buffer.contents(); + buffers.rhs_shape = rhs_local->shape; + MappedMemory bias_buffer; + if (bias_local && bias_local->buffer && !bias_local->shape.empty()) { + ASSIGN_OR_RETURN(bias_buffer, + bias_local->buffer->MapMemory(MemoryAccess::kRead)); + buffers.bias_buffer = bias_buffer.contents(); + } + ASSIGN_OR_RETURN(auto dst_buffer, dst_local->buffer->MapMemory( + MemoryAccess::kDiscardWrite)); + buffers.dst_buffer = dst_buffer.mutable_contents(); + buffers.dst_shape = dst_local->shape; + return kernels::MatMul::Execute(runtime_state, buffers); +} + +template +Status DispatchElementwiseUnaryOpIS(vm::BytecodeReader* reader) { + ASSIGN_OR_RETURN(auto* src_local, reader->ReadLocal()); + ASSIGN_OR_RETURN(auto* dst_local, reader->ReadLocal()); + RETURN_IF_ERROR(ValidateElementwiseUnaryOp(src_local, dst_local)); + return ApplyUnaryOpIS(src_local, dst_local); +} + +template +Status DispatchElementwiseUnaryOpIU(vm::BytecodeReader* reader) { + ASSIGN_OR_RETURN(auto* src_local, reader->ReadLocal()); + ASSIGN_OR_RETURN(auto* dst_local, reader->ReadLocal()); + RETURN_IF_ERROR(ValidateElementwiseUnaryOp(src_local, dst_local)); + return ApplyUnaryOpIU(src_local, dst_local); +} + +template +Status DispatchElementwiseUnaryOpF(vm::BytecodeReader* reader) { + ASSIGN_OR_RETURN(auto* src_local, reader->ReadLocal()); + ASSIGN_OR_RETURN(auto* dst_local, reader->ReadLocal()); + RETURN_IF_ERROR(ValidateElementwiseUnaryOp(src_local, dst_local)); + return ApplyUnaryOpF(src_local, dst_local); +} + +template +Status DispatchElementwiseBinaryOpIS(vm::BytecodeReader* reader) { + ASSIGN_OR_RETURN(auto* lhs_local, reader->ReadLocal()); + ASSIGN_OR_RETURN(auto* rhs_local, reader->ReadLocal()); + ASSIGN_OR_RETURN(auto* dst_local, reader->ReadLocal()); + RETURN_IF_ERROR(ValidateElementwiseBinaryOp(lhs_local, rhs_local, dst_local)); + return ApplyBinaryOpIS(lhs_local, rhs_local, dst_local); +} + +template +Status DispatchElementwiseBinaryOpIU(vm::BytecodeReader* reader) { + ASSIGN_OR_RETURN(auto* lhs_local, reader->ReadLocal()); + ASSIGN_OR_RETURN(auto* rhs_local, reader->ReadLocal()); + ASSIGN_OR_RETURN(auto* dst_local, reader->ReadLocal()); + RETURN_IF_ERROR(ValidateElementwiseBinaryOp(lhs_local, rhs_local, dst_local)); + return ApplyBinaryOpIU(lhs_local, rhs_local, dst_local); +} + +template +Status DispatchElementwiseBinaryOpF(vm::BytecodeReader* reader) { + ASSIGN_OR_RETURN(auto* lhs_local, reader->ReadLocal()); + ASSIGN_OR_RETURN(auto* rhs_local, reader->ReadLocal()); + ASSIGN_OR_RETURN(auto* dst_local, reader->ReadLocal()); + RETURN_IF_ERROR(ValidateElementwiseBinaryOp(lhs_local, rhs_local, dst_local)); + return ApplyBinaryOpF(lhs_local, rhs_local, dst_local); +} + +template +Status DispatchElementwiseTernaryOpIS(vm::BytecodeReader* reader) { + ASSIGN_OR_RETURN(auto* a_local, reader->ReadLocal()); + ASSIGN_OR_RETURN(auto* b_local, reader->ReadLocal()); + ASSIGN_OR_RETURN(auto* c_local, reader->ReadLocal()); + ASSIGN_OR_RETURN(auto* dst_local, reader->ReadLocal()); + RETURN_IF_ERROR( + ValidateElementwiseTernaryOp(a_local, b_local, c_local, dst_local)); + return ApplyTernaryOpIS(a_local, b_local, c_local, dst_local); +} + +template +Status DispatchElementwiseTernaryOpIU(vm::BytecodeReader* reader) { + ASSIGN_OR_RETURN(auto* a_local, reader->ReadLocal()); + ASSIGN_OR_RETURN(auto* b_local, reader->ReadLocal()); + ASSIGN_OR_RETURN(auto* c_local, reader->ReadLocal()); + ASSIGN_OR_RETURN(auto* dst_local, reader->ReadLocal()); + RETURN_IF_ERROR( + ValidateElementwiseTernaryOp(a_local, b_local, c_local, dst_local)); + return ApplyTernaryOpIU(a_local, b_local, c_local, dst_local); +} + +template +Status DispatchElementwiseTernaryOpF(vm::BytecodeReader* reader) { + ASSIGN_OR_RETURN(auto* a_local, reader->ReadLocal()); + ASSIGN_OR_RETURN(auto* b_local, reader->ReadLocal()); + ASSIGN_OR_RETURN(auto* c_local, reader->ReadLocal()); + ASSIGN_OR_RETURN(auto* dst_local, reader->ReadLocal()); + RETURN_IF_ERROR( + ValidateElementwiseTernaryOp(a_local, b_local, c_local, dst_local)); + return ApplyTernaryOpF(a_local, b_local, c_local, dst_local); +} + +Status ApplyCopy(BufferView* src_local, absl::Span src_indices, + BufferView* dst_local, absl::Span dst_indices, + absl::Span lengths); + +} // namespace hal +} // namespace iree + +#endif // THIRD_PARTY_MLIR_EDGE_IREE_HAL_INTERPRETER_BYTECODE_DISPATCH_UTIL_H_ diff --git a/hal/interpreter/bytecode_executable.cc b/hal/interpreter/bytecode_executable.cc new file mode 100644 index 000000000000..12fc59dab80b --- /dev/null +++ b/hal/interpreter/bytecode_executable.cc @@ -0,0 +1,85 @@ +// Copyright 2019 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "third_party/mlir_edge/iree/hal/interpreter/bytecode_executable.h" + +#include "third_party/mlir_edge/iree/vm/bytecode_tables_interpreter.h" +#include "third_party/mlir_edge/iree/vm/bytecode_validator.h" +#include "third_party/mlir_edge/iree/vm/module.h" +#include "third_party/mlir_edge/iree/vm/module_printer.h" + +namespace iree { +namespace hal { + +namespace { +// TODO(benvanik): remove when debugger is wired up to the HAL. +const bool kEnableExecutablePrinting = false; +} // namespace + +// static +StatusOr> BytecodeExecutable::Load( + hal::Allocator* allocator, ExecutableSpec spec, bool allow_aliasing_data) { + // Allocate the executable now. + // We do this here so that if we need to clone the data we are passing that + // to the VM loader instead of the data we may not have access to later. + auto executable = + make_ref(allocator, spec, allow_aliasing_data); + auto* context = executable->mutable_context(); + + // Create the executable module. + auto module_def = + ::flatbuffers::GetRoot(executable->executable_data().data()); + ASSIGN_OR_RETURN(auto module, vm::Module::FromDef(*module_def)); + executable->module_ = module.get(); + RETURN_IF_ERROR(context->RegisterModule(std::move(module))); + + // Validate bytecode to ensure it will be usable for execution. + // We do this here so that we get a good stack immediately when the bytecode + // is provided instead of when we go to run it. This more closely mirrors how + // a backend that performed compilation (such as SPIR-V) would fail. + for (auto* function_def : + *executable->module().function_table().def().functions()) { + RETURN_IF_ERROR(vm::BytecodeValidator::Validate( + *context, executable->module(), *function_def->bytecode())); + } + + // Print the bytecode. + // TODO(benvanik): remove when debugger is wired up to the HAL. + if (kEnableExecutablePrinting) { + vm::PrintModuleFlagBitfield print_flags = vm::PrintModuleFlag::kNone; + for (const auto& module : context->modules()) { + RETURN_IF_ERROR(vm::PrintModuleToStream( + vm::interpreter_opcode_table(), *module, print_flags, &std::cout)); + } + } + + return executable; +} + +BytecodeExecutable::BytecodeExecutable(hal::Allocator* allocator, + ExecutableSpec spec, + bool allow_aliasing_data) + : spec_(spec), context_(allocator) { + if (!allow_aliasing_data) { + // Clone data. + cloned_executable_data_ = {spec.executable_data.begin(), + spec.executable_data.end()}; + spec_.executable_data = absl::MakeConstSpan(cloned_executable_data_); + } +} + +BytecodeExecutable::~BytecodeExecutable() = default; + +} // namespace hal +} // namespace iree diff --git a/hal/interpreter/bytecode_executable.h b/hal/interpreter/bytecode_executable.h new file mode 100644 index 000000000000..c39bbbd0e278 --- /dev/null +++ b/hal/interpreter/bytecode_executable.h @@ -0,0 +1,68 @@ +// Copyright 2019 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#ifndef THIRD_PARTY_MLIR_EDGE_IREE_HAL_INTERPRETER_BYTECODE_EXECUTABLE_H_ +#define THIRD_PARTY_MLIR_EDGE_IREE_HAL_INTERPRETER_BYTECODE_EXECUTABLE_H_ + +#include + +#include "third_party/absl/types/span.h" +#include "third_party/mlir_edge/iree/base/status.h" +#include "third_party/mlir_edge/iree/hal/allocator.h" +#include "third_party/mlir_edge/iree/hal/executable.h" +#include "third_party/mlir_edge/iree/hal/executable_spec.h" +#include "third_party/mlir_edge/iree/hal/interpreter/interpreter_context.h" +#include "third_party/mlir_edge/iree/vm/context.h" + +namespace iree { +namespace hal { + +class BytecodeExecutable final : public Executable { + public: + static StatusOr> Load(hal::Allocator* allocator, + ExecutableSpec spec, + bool allow_aliasing_data); + + BytecodeExecutable(hal::Allocator* allocator, ExecutableSpec spec, + bool allow_aliasing_data); + ~BytecodeExecutable() override; + + bool supports_debugging() const override { return false; } + + // Reference to the bytecode blob contents. + absl::Span executable_data() const { + return spec_.executable_data; + } + + // VM context with the executable registered. + const InterpreterContext& context() const { return context_; } + InterpreterContext* mutable_context() { return &context_; } + + // VM module representing the executable. + // Note that there may be more than one module in the Context and only this + // module can be used to lookup executable exports. + const vm::Module& module() const { return *module_; } + + private: + ExecutableSpec spec_; + std::vector cloned_executable_data_; + + InterpreterContext context_; + vm::Module* module_ = nullptr; +}; + +} // namespace hal +} // namespace iree + +#endif // THIRD_PARTY_MLIR_EDGE_IREE_HAL_INTERPRETER_BYTECODE_EXECUTABLE_H_ diff --git a/hal/interpreter/bytecode_kernels.h b/hal/interpreter/bytecode_kernels.h new file mode 100644 index 000000000000..d027af6c673b --- /dev/null +++ b/hal/interpreter/bytecode_kernels.h @@ -0,0 +1,371 @@ +// Copyright 2019 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// Defines kernel functions and provides their implementation via one (or more) +// included files. +// +// Kernels should do the simplest possible operation. Buffer validation is +// handled by the dispatch logic and need not be checked. Kernels may optionally +// accept arguments beyond just the buffers, depending on the required state +// and attributes. +// +// Kernels may optionally have runtime state. This is state that is allocated +// once for the entire Runtime (and stored on RuntimeState) and shared across +// all fibers. This enables kernels that may require thread pools or device +// handles to be shared while kernels that require transient storage to be safe +// to use from multiple fibers concurrently. +// +// All kernels are templated to enable specialization of particular types or +// type combinations. By default the bytecode_kernels_generic.h will provide C++ +// semantics as reference and platform-specific versions can be implemented +// as needed. + +#ifndef THIRD_PARTY_MLIR_EDGE_IREE_HAL_INTERPRETER_BYTECODE_KERNELS_H_ +#define THIRD_PARTY_MLIR_EDGE_IREE_HAL_INTERPRETER_BYTECODE_KERNELS_H_ + +#include + +#include "third_party/absl/types/span.h" +#include "third_party/mlir_edge/iree/base/shape.h" +#include "third_party/mlir_edge/iree/base/status.h" + +namespace iree { +namespace hal { +namespace kernels { + +struct CompareEQ { + template + static Status Execute(absl::Span lhs_buffer, + absl::Span rhs_buffer, + absl::Span dst_buffer); +}; +struct CompareNE { + template + static Status Execute(absl::Span lhs_buffer, + absl::Span rhs_buffer, + absl::Span dst_buffer); +}; +struct CompareLT { + template + static Status Execute(absl::Span lhs_buffer, + absl::Span rhs_buffer, + absl::Span dst_buffer); +}; +struct CompareLE { + template + static Status Execute(absl::Span lhs_buffer, + absl::Span rhs_buffer, + absl::Span dst_buffer); +}; +struct CompareGT { + template + static Status Execute(absl::Span lhs_buffer, + absl::Span rhs_buffer, + absl::Span dst_buffer); +}; +struct CompareGE { + template + static Status Execute(absl::Span lhs_buffer, + absl::Span rhs_buffer, + absl::Span dst_buffer); +}; + +struct Copy { + template + static Status Execute(absl::Span src_buffer, + const Shape& src_shape, + absl::Span src_indices, + absl::Span dst_buffer, const Shape& dst_shape, + absl::Span dst_indices, + absl::Span lengths); +}; + +struct Select { + template + static Status Execute(absl::Span cond_buffer, + absl::Span lhs_buffer, + absl::Span rhs_buffer, + absl::Span dst_buffer); +}; + +struct Transpose { + template + static Status Execute(absl::Span src_buffer, + absl::Span dst_buffer, const Shape& src_shape, + absl::Span perm); +}; + +struct Pad { + template + static Status Execute(absl::Span src_buffer, + absl::Span padding_value, + absl::Span dst_buffer, const Shape& src_shape, + const Shape& dst_shape, + absl::Span edge_padding_low, + absl::Span edge_padding_high, + absl::Span interior_padding); +}; + +struct Reverse { + template + static Status Execute(absl::Span src_buffer, + absl::Span dst_buffer, const Shape& src_shape, + absl::Span dimensions); +}; + +struct Broadcast { + template + static Status Execute(absl::Span src_buffer, + absl::Span dst_buffer); +}; + +struct Tile { + template + static Status Execute(absl::Span src_buffer, + absl::Span dst_buffer, const Shape& src_shape, + const Shape& dst_shape); +}; + +struct Not { + template + static Status Execute(absl::Span src_buffer, + absl::Span dst_buffer); +}; + +struct And { + template + static Status Execute(absl::Span lhs_buffer, + absl::Span rhs_buffer, + absl::Span dst_buffer); +}; + +struct Or { + template + static Status Execute(absl::Span lhs_buffer, + absl::Span rhs_buffer, + absl::Span dst_buffer); +}; + +struct Xor { + template + static Status Execute(absl::Span lhs_buffer, + absl::Span rhs_buffer, + absl::Span dst_buffer); +}; + +struct ShiftLeft { + template + static Status Execute(absl::Span lhs_buffer, + absl::Span rhs_buffer, + absl::Span dst_buffer); +}; + +struct ShiftRight { + template + static Status Execute(absl::Span lhs_buffer, + absl::Span rhs_buffer, + absl::Span dst_buffer); +}; + +struct Add { + template + static Status Execute(absl::Span lhs_buffer, + absl::Span rhs_buffer, + absl::Span dst_buffer); +}; + +struct Sub { + template + static Status Execute(absl::Span lhs_buffer, + absl::Span rhs_buffer, + absl::Span dst_buffer); +}; + +struct Abs { + template + static Status Execute(absl::Span src_buffer, + absl::Span dst_buffer); +}; + +struct Mul { + template + static Status Execute(absl::Span lhs_buffer, + absl::Span rhs_buffer, + absl::Span dst_buffer); +}; + +struct Div { + template + static Status Execute(absl::Span lhs_buffer, + absl::Span rhs_buffer, + absl::Span dst_buffer); +}; + +// a + (b * c) +struct MulAdd { + template + static Status Execute(absl::Span a_buffer, + absl::Span b_buffer, + absl::Span c_buffer, absl::Span dst_buffer); +}; + +struct Exp { + template + static Status Execute(absl::Span src_buffer, + absl::Span dst_buffer); +}; + +struct Log { + template + static Status Execute(absl::Span src_buffer, + absl::Span dst_buffer); +}; + +struct Rsqrt { + template + static Status Execute(absl::Span src_buffer, + absl::Span dst_buffer); +}; + +struct Cos { + template + static Status Execute(absl::Span src_buffer, + absl::Span dst_buffer); +}; + +struct Sin { + template + static Status Execute(absl::Span src_buffer, + absl::Span dst_buffer); +}; + +struct Tanh { + template + static Status Execute(absl::Span src_buffer, + absl::Span dst_buffer); +}; + +struct Atan2 { + template + static Status Execute(absl::Span lhs_buffer, + absl::Span rhs_buffer, + absl::Span dst_buffer); +}; + +struct Min { + template + static Status Execute(absl::Span lhs_buffer, + absl::Span rhs_buffer, + absl::Span dst_buffer); +}; + +struct Max { + template + static Status Execute(absl::Span lhs_buffer, + absl::Span rhs_buffer, + absl::Span dst_buffer); +}; + +struct Clamp { + template + static Status Execute(absl::Span src_buffer, + absl::Span min_buffer, + absl::Span max_buffer, + absl::Span dst_buffer); +}; + +struct Floor { + template + static Status Execute(absl::Span src_buffer, + absl::Span dst_buffer); +}; + +struct Ceil { + template + static Status Execute(absl::Span src_buffer, + absl::Span dst_buffer); +}; + +struct Convert { + template + static Status Execute(absl::Span src_buffer, + absl::Span dst_buffer); +}; + +struct MatMul { + struct RuntimeState; + + static std::unique_ptr CreateRuntimeState(); + + template + struct Buffers { + Shape lhs_shape; + absl::Span lhs_buffer; + Shape rhs_shape; + absl::Span rhs_buffer; + Shape dst_shape; + absl::Span dst_buffer; + + // Optional bias buffer. + absl::Span bias_buffer; + + // Fixed-point multiplier mantissa/exponent. May be a single value (for + // uniform quantization) or one element per row of the destination matrix + // for per-channel. + absl::Span multiplier_mantissa_buffer; + absl::Span multiplier_exponent_buffer; + }; + + template + static Status Execute(RuntimeState* runtime_state, + const Buffers& buffers); +}; + +struct RuntimeState { + std::unique_ptr mat_mul_state = + MatMul::CreateRuntimeState(); +}; + +struct ReduceSum { + template + static Status Execute(absl::Span src_buffer, + absl::Span init_buffer, + absl::Span dst_buffer, int32_t dimension, + const Shape& src_shape, const Shape& dst_shape); +}; + +struct ReduceMin { + template + static Status Execute(absl::Span src_buffer, + absl::Span init_buffer, + absl::Span dst_buffer, int32_t dimension, + const Shape& src_shape, const Shape& dst_shape); +}; + +struct ReduceMax { + template + static Status Execute(absl::Span src_buffer, + absl::Span init_buffer, + absl::Span dst_buffer, int32_t dimension, + const Shape& src_shape, const Shape& dst_shape); +}; + +} // namespace kernels +} // namespace hal +} // namespace iree + +#include "third_party/mlir_edge/iree/hal/interpreter/bytecode_kernels_generic.h" // IWYU pragma: export +#include "third_party/mlir_edge/iree/hal/interpreter/bytecode_kernels_ruy.h" // IWYU pragma: export + +#endif // THIRD_PARTY_MLIR_EDGE_IREE_HAL_INTERPRETER_BYTECODE_KERNELS_H_ diff --git a/hal/interpreter/bytecode_kernels_generic.h b/hal/interpreter/bytecode_kernels_generic.h new file mode 100644 index 000000000000..4cea884d4182 --- /dev/null +++ b/hal/interpreter/bytecode_kernels_generic.h @@ -0,0 +1,696 @@ +// Copyright 2019 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#ifndef THIRD_PARTY_MLIR_EDGE_IREE_HAL_INTERPRETER_BYTECODE_KERNELS_GENERIC_H_ +#define THIRD_PARTY_MLIR_EDGE_IREE_HAL_INTERPRETER_BYTECODE_KERNELS_GENERIC_H_ + +#include "third_party/absl/container/flat_hash_set.h" +#include "third_party/absl/container/inlined_vector.h" +#include "third_party/absl/types/span.h" +#include "third_party/mlir_edge/iree/base/status.h" + +namespace iree { +namespace hal { +namespace kernels { + +template +Status CompareEQ::Execute(absl::Span lhs_buffer, + absl::Span rhs_buffer, + absl::Span dst_buffer) { + for (size_t i = 0; i < dst_buffer.size(); ++i) { + dst_buffer[i] = lhs_buffer[i] == rhs_buffer[i]; + } + return OkStatus(); +} + +template +Status CompareNE::Execute(absl::Span lhs_buffer, + absl::Span rhs_buffer, + absl::Span dst_buffer) { + for (size_t i = 0; i < dst_buffer.size(); ++i) { + dst_buffer[i] = lhs_buffer[i] != rhs_buffer[i]; + } + return OkStatus(); +} + +template +Status CompareLT::Execute(absl::Span lhs_buffer, + absl::Span rhs_buffer, + absl::Span dst_buffer) { + for (size_t i = 0; i < dst_buffer.size(); ++i) { + dst_buffer[i] = lhs_buffer[i] < rhs_buffer[i]; + } + return OkStatus(); +} + +template +Status CompareLE::Execute(absl::Span lhs_buffer, + absl::Span rhs_buffer, + absl::Span dst_buffer) { + for (size_t i = 0; i < dst_buffer.size(); ++i) { + dst_buffer[i] = lhs_buffer[i] <= rhs_buffer[i]; + } + return OkStatus(); +} + +template +Status CompareGT::Execute(absl::Span lhs_buffer, + absl::Span rhs_buffer, + absl::Span dst_buffer) { + for (size_t i = 0; i < dst_buffer.size(); ++i) { + dst_buffer[i] = lhs_buffer[i] > rhs_buffer[i]; + } + return OkStatus(); +} + +template +Status CompareGE::Execute(absl::Span lhs_buffer, + absl::Span rhs_buffer, + absl::Span dst_buffer) { + for (size_t i = 0; i < dst_buffer.size(); ++i) { + dst_buffer[i] = lhs_buffer[i] >= rhs_buffer[i]; + } + return OkStatus(); +} + +namespace impl { +inline absl::InlinedVector ComputeCopyStrides(const Shape& shape, + size_t element_size) { + absl::InlinedVector strides(shape.empty() ? 1 : shape.size()); + strides.back() = element_size; + for (int i = shape.size() - 2; i >= 0; --i) { + strides[i] = strides[i + 1] * shape[i + 1]; + } + return strides; +} + +inline void CopyRegion(absl::Span src_buffer, + absl::Span src_strides, + absl::Span src_indices, + absl::Span dst_buffer, + absl::Span dst_strides, + absl::Span dst_indices, + absl::Span lengths) { + if (lengths.size() > 1) { + for (int i = 0; i < lengths[0]; ++i) { + size_t src_offset = src_strides[0] * (src_indices[0] + i); + size_t dst_offset = dst_strides[0] * (dst_indices[0] + i); + CopyRegion(src_buffer.subspan(src_offset), src_strides.subspan(1), + src_indices.subspan(1), dst_buffer.subspan(dst_offset), + dst_strides.subspan(1), dst_indices.subspan(1), + lengths.subspan(1)); + } + } else { + DCHECK_EQ(dst_strides.size(), 1); + DCHECK_EQ(src_strides.size(), 1); + DCHECK_EQ(src_indices.size(), 1); + DCHECK_EQ(dst_indices.size(), 1); + DCHECK_EQ(lengths.size(), 1); + auto src_offset = src_indices[0] * src_strides[0]; + auto dst_offset = dst_indices[0] * dst_strides[0]; + auto length = dst_strides[0] * lengths[0]; + std::memcpy(dst_buffer.data() + dst_offset, src_buffer.data() + src_offset, + length); + } +} +} // namespace impl + +// TODO(benvanik): replace with a real implementation once copy is defined. +template +Status Copy::Execute(absl::Span src_buffer, + const Shape& src_shape, + absl::Span src_indices, + absl::Span dst_buffer, const Shape& dst_shape, + absl::Span dst_indices, + absl::Span lengths) { + // TODO(gcmn) Maybe we can fast-path earlier if we detect contiguous memory + // across multiple rows. + auto src_strides = impl::ComputeCopyStrides(src_shape, element_size); + auto dst_strides = impl::ComputeCopyStrides(dst_shape, element_size); + impl::CopyRegion(src_buffer, src_strides, src_indices, dst_buffer, + dst_strides, dst_indices, lengths); + return OkStatus(); +} + +template +Status Select::Execute(absl::Span cond_buffer, + absl::Span lhs_buffer, + absl::Span rhs_buffer, + absl::Span dst_buffer) { + for (size_t i = 0; i < dst_buffer.size(); ++i) { + dst_buffer[i] = cond_buffer[i] ? lhs_buffer[i] : rhs_buffer[i]; + } + return OkStatus(); +} + +template +Status Transpose::Execute(absl::Span src_buffer, + absl::Span dst_buffer, const Shape& src_shape, + absl::Span perm) { + // This implementation is .... not fast. + int rank = src_shape.size(); + absl::InlinedVector src_strides(rank); + absl::InlinedVector dst_strides(rank); + size_t src_stride = 1; + size_t dst_stride = 1; + for (int dim_i = rank - 1; dim_i >= 0; --dim_i) { + src_strides[dim_i] = src_stride; + dst_strides[dim_i] = dst_stride; + src_stride *= src_shape[dim_i]; + dst_stride *= src_shape[perm[dim_i]]; + } + for (size_t dst_i = 0; dst_i < dst_buffer.size(); ++dst_i) { + size_t src_i = 0; + size_t t = dst_i; + for (int dim_i = 0; dim_i < rank; ++dim_i) { + size_t ratio = t / dst_strides[dim_i]; + t -= ratio * dst_strides[dim_i]; + src_i += ratio * src_strides[perm[dim_i]]; + } + dst_buffer[dst_i] = src_buffer[src_i]; + } + return OkStatus(); +} + +namespace impl { +inline void IncrementShapeIndex(absl::Span indices, + const Shape& shape) { + for (int i = indices.size() - 1; i >= 0; --i) { + if (++indices[i] < shape[i]) return; + indices[i] = 0; + } +} + +inline bool IsPadding(absl::Span indices, const Shape& shape, + absl::Span edge_padding_low, + absl::Span edge_padding_high, + absl::Span interior_padding) { + for (int i = 0; i < indices.size(); ++i) { + auto index = indices[i]; + if (index < edge_padding_low[i] || + index >= shape[i] - edge_padding_high[i] || + (index - edge_padding_low[i]) % (interior_padding[i] + 1) != 0) { + return true; + } + } + + return false; +} +} // namespace impl + +template +Status Pad::Execute(absl::Span src_buffer, + absl::Span padding_value_buffer, + absl::Span dst_buffer, const Shape& src_shape, + const Shape& dst_shape, + absl::Span edge_padding_low, + absl::Span edge_padding_high, + absl::Span interior_padding) { + // This implementation is not at all fast, as it iterates every index in the + // destination buffer individually. Potential improvements: + // 1. Fill the dst buffer with padded value initially. Only need to iterate + // through source buffer and can exit early. + // 2. Use striding to advance through larger swaths of the buffer with a + // memcpy from src and filling (or skipping) padded incides. Especially + // useful when e.g. entire rows are padded. + + // TODO(b/140836672) support negative padding + + if (padding_value_buffer.size() != 1) { + return InvalidArgumentErrorBuilder(ABSL_LOC) + << "Padding value buffer is larger than one element."; + } + auto padding_value = padding_value_buffer.front(); + + absl::InlinedVector dst_indices(src_shape.size(), 0); + + const T* src_ptr = src_buffer.begin(); + T* dst_ptr = dst_buffer.begin(); + while (dst_ptr != dst_buffer.end()) { + if (impl::IsPadding(dst_indices, dst_shape, edge_padding_low, + edge_padding_high, interior_padding)) { + *dst_ptr++ = padding_value; + } else { + DCHECK(src_ptr != src_buffer.end()); + *dst_ptr++ = *src_ptr++; + } + impl::IncrementShapeIndex(absl::MakeSpan(dst_indices), dst_shape); + } + + return OkStatus(); +} + +template +Status Reverse::Execute(absl::Span src_buffer, + absl::Span dst_buffer, const Shape& src_shape, + absl::Span dimensions) { + // This implementation is not fast either + int rank = src_shape.size(); + absl::InlinedVector strides(rank); + size_t stride = 1; + for (int dim_i = rank - 1; dim_i >= 0; --dim_i) { + strides[dim_i] = stride; + stride *= src_shape[dim_i]; + } + absl::flat_hash_set dims_set(dimensions.begin(), dimensions.end()); + for (size_t dst_i = 0; dst_i < dst_buffer.size(); ++dst_i) { + size_t src_i = 0; + size_t t = dst_i; + for (int dim_i = 0; dim_i < rank; ++dim_i) { + size_t ratio = t / strides[dim_i]; + t -= ratio * strides[dim_i]; + bool do_reverse = dims_set.contains(dim_i); + src_i += (do_reverse ? (src_shape[dim_i] - 1 - ratio) : ratio) * + strides[dim_i]; + } + dst_buffer[dst_i] = src_buffer[src_i]; + } + return OkStatus(); +} + +template +Status Broadcast::Execute(absl::Span src_buffer, + absl::Span dst_buffer) { + for (size_t i = 0; i < dst_buffer.size(); ++i) { + dst_buffer[i] = src_buffer[0]; + } + return OkStatus(); +} + +template +Status Tile::Execute(absl::Span src_buffer, absl::Span dst_buffer, + const Shape& src_shape, const Shape& dst_shape) { + // This implementation is .... not fast. + int rank = dst_shape.size(); + absl::InlinedVector src_strides(rank); + absl::InlinedVector dst_strides(rank); + size_t src_stride = 1; + size_t dst_stride = 1; + for (int dim_i = rank - 1; dim_i >= 0; --dim_i) { + src_strides[dim_i] = src_stride; + dst_strides[dim_i] = dst_stride; + src_stride *= src_shape[dim_i]; + dst_stride *= dst_shape[dim_i]; + } + for (size_t dst_i = 0; dst_i < dst_buffer.size(); ++dst_i) { + size_t src_i = 0; + size_t t = dst_i; + for (int dim_i = 0; dim_i < rank; ++dim_i) { + src_i += t / dst_strides[dim_i] % src_shape[dim_i] * src_strides[dim_i]; + t %= dst_strides[dim_i]; + } + dst_buffer[dst_i] = src_buffer[src_i]; + } + return OkStatus(); +} + +template +Status Not::Execute(absl::Span src_buffer, absl::Span dst_buffer) { + for (size_t i = 0; i < dst_buffer.size(); ++i) { + dst_buffer[i] = ~src_buffer[i]; + } + return OkStatus(); +} + +template +Status And::Execute(absl::Span lhs_buffer, + absl::Span rhs_buffer, absl::Span dst_buffer) { + for (size_t i = 0; i < dst_buffer.size(); ++i) { + dst_buffer[i] = lhs_buffer[i] & rhs_buffer[i]; + } + return OkStatus(); +} + +template +Status Or::Execute(absl::Span lhs_buffer, + absl::Span rhs_buffer, absl::Span dst_buffer) { + for (size_t i = 0; i < dst_buffer.size(); ++i) { + dst_buffer[i] = lhs_buffer[i] | rhs_buffer[i]; + } + return OkStatus(); +} + +template +Status Xor::Execute(absl::Span lhs_buffer, + absl::Span rhs_buffer, absl::Span dst_buffer) { + for (size_t i = 0; i < dst_buffer.size(); ++i) { + dst_buffer[i] = lhs_buffer[i] ^ rhs_buffer[i]; + } + return OkStatus(); +} + +template +Status ShiftLeft::Execute(absl::Span lhs_buffer, + absl::Span rhs_buffer, + absl::Span dst_buffer) { + for (size_t i = 0; i < dst_buffer.size(); ++i) { + dst_buffer[i] = lhs_buffer[i] << rhs_buffer[i]; + } + return OkStatus(); +} + +template +Status ShiftRight::Execute(absl::Span lhs_buffer, + absl::Span rhs_buffer, + absl::Span dst_buffer) { + for (size_t i = 0; i < dst_buffer.size(); ++i) { + dst_buffer[i] = lhs_buffer[i] >> rhs_buffer[i]; + } + return OkStatus(); +} + +template +Status Add::Execute(absl::Span lhs_buffer, + absl::Span rhs_buffer, absl::Span dst_buffer) { + for (size_t i = 0; i < dst_buffer.size(); ++i) { + dst_buffer[i] = lhs_buffer[i] + rhs_buffer[i]; + } + return OkStatus(); +} + +template +Status Sub::Execute(absl::Span lhs_buffer, + absl::Span rhs_buffer, absl::Span dst_buffer) { + for (size_t i = 0; i < dst_buffer.size(); ++i) { + dst_buffer[i] = lhs_buffer[i] - rhs_buffer[i]; + } + return OkStatus(); +} + +template +Status Abs::Execute(absl::Span src_buffer, absl::Span dst_buffer) { + for (size_t i = 0; i < dst_buffer.size(); ++i) { + dst_buffer[i] = std::abs(src_buffer[i]); + } + return OkStatus(); +} + +template +Status Mul::Execute(absl::Span lhs_buffer, + absl::Span rhs_buffer, absl::Span dst_buffer) { + for (size_t i = 0; i < dst_buffer.size(); ++i) { + dst_buffer[i] = lhs_buffer[i] * rhs_buffer[i]; + } + return OkStatus(); +} + +template +Status Div::Execute(absl::Span lhs_buffer, + absl::Span rhs_buffer, absl::Span dst_buffer) { + for (size_t i = 0; i < dst_buffer.size(); ++i) { + dst_buffer[i] = lhs_buffer[i] / rhs_buffer[i]; + } + return OkStatus(); +} + +template +Status MulAdd::Execute(absl::Span a_buffer, + absl::Span b_buffer, + absl::Span c_buffer, absl::Span dst_buffer) { + for (size_t i = 0; i < dst_buffer.size(); ++i) { + dst_buffer[i] = a_buffer[i] + (b_buffer[i] * c_buffer[i]); + } + return OkStatus(); +} + +template +Status Exp::Execute(absl::Span src_buffer, absl::Span dst_buffer) { + for (size_t i = 0; i < dst_buffer.size(); ++i) { + dst_buffer[i] = std::exp(src_buffer[i]); + } + return OkStatus(); +} + +template +Status Rsqrt::Execute(absl::Span src_buffer, + absl::Span dst_buffer) { + for (size_t i = 0; i < dst_buffer.size(); ++i) { + dst_buffer[i] = 1.0 / std::sqrt(src_buffer[i]); + } + return OkStatus(); +} + +template +Status Log::Execute(absl::Span src_buffer, absl::Span dst_buffer) { + for (size_t i = 0; i < dst_buffer.size(); ++i) { + dst_buffer[i] = std::log(src_buffer[i]); + } + return OkStatus(); +} + +template +Status Cos::Execute(absl::Span src_buffer, absl::Span dst_buffer) { + for (size_t i = 0; i < dst_buffer.size(); ++i) { + dst_buffer[i] = std::cos(src_buffer[i]); + } + return OkStatus(); +} + +template +Status Sin::Execute(absl::Span src_buffer, absl::Span dst_buffer) { + for (size_t i = 0; i < dst_buffer.size(); ++i) { + dst_buffer[i] = std::sin(src_buffer[i]); + } + return OkStatus(); +} + +template +Status Tanh::Execute(absl::Span src_buffer, absl::Span dst_buffer) { + for (size_t i = 0; i < dst_buffer.size(); ++i) { + dst_buffer[i] = std::tanh(src_buffer[i]); + } + return OkStatus(); +} + +template +Status Atan2::Execute(absl::Span lhs_buffer, + absl::Span rhs_buffer, + absl::Span dst_buffer) { + for (size_t i = 0; i < dst_buffer.size(); ++i) { + dst_buffer[i] = std::atan2(lhs_buffer[i], rhs_buffer[i]); + } + return OkStatus(); +} + +template +Status Min::Execute(absl::Span lhs_buffer, + absl::Span rhs_buffer, absl::Span dst_buffer) { + for (size_t i = 0; i < dst_buffer.size(); ++i) { + dst_buffer[i] = std::min(lhs_buffer[i], rhs_buffer[i]); + } + return OkStatus(); +} + +template +Status Max::Execute(absl::Span lhs_buffer, + absl::Span rhs_buffer, absl::Span dst_buffer) { + for (size_t i = 0; i < dst_buffer.size(); ++i) { + dst_buffer[i] = std::max(lhs_buffer[i], rhs_buffer[i]); + } + return OkStatus(); +} + +template +Status Clamp::Execute(absl::Span src_buffer, + absl::Span min_buffer, + absl::Span max_buffer, + absl::Span dst_buffer) { + for (size_t i = 0; i < dst_buffer.size(); ++i) { + T src = src_buffer[i]; + T min = min_buffer[i]; + T max = max_buffer[i]; + dst_buffer[i] = src <= min ? min : src >= max ? max : src; + } + return OkStatus(); +} + +template +Status Floor::Execute(absl::Span src_buffer, + absl::Span dst_buffer) { + for (size_t i = 0; i < dst_buffer.size(); ++i) { + dst_buffer[i] = std::floor(src_buffer[i]); + } + return OkStatus(); +} + +template +Status Ceil::Execute(absl::Span src_buffer, absl::Span dst_buffer) { + for (size_t i = 0; i < dst_buffer.size(); ++i) { + dst_buffer[i] = std::ceil(src_buffer[i]); + } + return OkStatus(); +} + +template +Status Convert::Execute(absl::Span src_buffer, + absl::Span dst_buffer) { + DCHECK_EQ(src_buffer.size(), dst_buffer.size()); + for (size_t i = 0; i < dst_buffer.size(); ++i) { + dst_buffer[i] = static_cast(src_buffer[i]); + } + return OkStatus(); +} + +namespace impl { + +struct SumKernel { + template + inline void operator()(T* value0, const T value1) { + *value0 += value1; + } +}; + +struct MinKernel { + template + inline void operator()(T* value0, const T value1) { + *value0 = std::min(*value0, value1); + } +}; + +struct MaxKernel { + template + inline void operator()(T* value0, const T value1) { + *value0 = std::max(*value0, value1); + } +}; + +template +inline void ReduceDimension(absl::Span src_buffer, + absl::Span dst_buffer, const Shape& src_shape, + absl::Span reduce_dims, + absl::Span dst_strides, int dim, + absl::Span src_indices, size_t flat_src_i, + size_t src_stride) { + if (dim < 0) { + // Base case of the recursion - figure out which elements should be acted + // upon and apply the reduction kernel to them. + + // Derive destination indices from source indices. + // For example, + // reduce_dims: [1, 2] + // src_indices: [2, 1, 3, 0] + // ^ ^ + // | | + // |----- remove these dimensions + // dst_indices: [2, 0] + // + // TODO(scotttodd): Clean this up somehow, share across recursion levels? + size_t dst_size = src_shape.size() - reduce_dims.size(); + absl::InlinedVector dst_indices; + for (size_t i = 0; i < src_indices.size(); ++i) { + if (std::find(std::begin(reduce_dims), std::end(reduce_dims), i) == + std::end(reduce_dims)) { + dst_indices.push_back(src_indices[i]); + } + } + // Compute the flattened index into dst_buffer at [dst_indices]. + size_t dst_i = 0; + for (size_t i = 0; i < dst_indices.size(); ++i) { + dst_i += dst_indices[i] * dst_strides[dst_size - 1 - i]; + } + + // Flattened src and dst indices have been computed, invoke the kernel. + KernelImpl()(&dst_buffer[dst_i], src_buffer[flat_src_i]); + return; + } + + // Iterate through the current dimension in the source shape, recursing + // down one dimension at a time. + // + // This touches each element in the source buffer once, tracking complete + // dimensions within the shaped source buffer and using them to compute + // the corresponding indices (shaped and flattened) within the destination + // buffer. Each element in the destination buffer will be touched multiple + // times. + // + // Note that cache coherency isn't considered here, and some computations + // are redundant, so this could be optimized substantially. + for (size_t dim_i = 0; dim_i < src_shape[dim]; ++dim_i) { + src_indices[dim] = dim_i; + + // Recurse down to the next dimension (e.g. 2 -> 1 -> 0 -> base case) + // * Add the current stride to flat_src_i + // * Multiply src_stride by this dimension's shape + ReduceDimension(src_buffer, dst_buffer, src_shape, + reduce_dims, dst_strides, dim - 1, + src_indices, flat_src_i + dim_i * src_stride, + src_stride * src_shape[dim]); + } +} + +template +Status GenericReduce(absl::Span src_buffer, + absl::Span init_buffer, absl::Span dst_buffer, + int32_t dimension, const Shape& src_shape, + const Shape& dst_shape) { + // Initialize using init_buffer, which is expected to be a scalar. + std::fill_n(dst_buffer.data(), dst_buffer.size(), init_buffer[0]); + + // Precompute destination strides. + int dst_rank = dst_shape.size(); + absl::InlinedVector dst_strides; + size_t dst_stride = 1; + for (int dim_i = dst_rank - 1; dim_i >= 0; --dim_i) { + dst_strides.push_back(dst_stride); + dst_stride *= dst_shape[dim_i]; + } + + // Call the helper (recursive) function, starting with: + // * source index [0, 0, ..., 0] + // * the innermost dimension (last in the shape) + // * flat_src_i of 0 (corresponds to [0, 0, ..., 0] above) + // * source stride 1 + absl::InlinedVector src_indices(src_shape.size(), 0); + ReduceDimension(src_buffer, dst_buffer, src_shape, {dimension}, + absl::MakeSpan(dst_strides), + src_shape.size() - 1, + absl::MakeSpan(src_indices), 0, 1); + + return OkStatus(); +} + +} // namespace impl + +template +Status ReduceSum::Execute(absl::Span src_buffer, + absl::Span init_buffer, + absl::Span dst_buffer, int32_t dimension, + const Shape& src_shape, const Shape& dst_shape) { + return impl::GenericReduce( + src_buffer, init_buffer, dst_buffer, dimension, src_shape, dst_shape); +} + +template +Status ReduceMin::Execute(absl::Span src_buffer, + absl::Span init_buffer, + absl::Span dst_buffer, int32_t dimension, + const Shape& src_shape, const Shape& dst_shape) { + return impl::GenericReduce( + src_buffer, init_buffer, dst_buffer, dimension, src_shape, dst_shape); +} + +template +Status ReduceMax::Execute(absl::Span src_buffer, + absl::Span init_buffer, + absl::Span dst_buffer, int32_t dimension, + const Shape& src_shape, const Shape& dst_shape) { + return impl::GenericReduce( + src_buffer, init_buffer, dst_buffer, dimension, src_shape, dst_shape); +} + +} // namespace kernels +} // namespace hal +} // namespace iree + +#endif // THIRD_PARTY_MLIR_EDGE_IREE_HAL_INTERPRETER_BYTECODE_KERNELS_GENERIC_H_ diff --git a/hal/interpreter/bytecode_kernels_ruy.h b/hal/interpreter/bytecode_kernels_ruy.h new file mode 100644 index 000000000000..48e30b5cbe8d --- /dev/null +++ b/hal/interpreter/bytecode_kernels_ruy.h @@ -0,0 +1,81 @@ +// Copyright 2019 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#ifndef THIRD_PARTY_MLIR_EDGE_IREE_HAL_INTERPRETER_BYTECODE_KERNELS_RUY_H_ +#define THIRD_PARTY_MLIR_EDGE_IREE_HAL_INTERPRETER_BYTECODE_KERNELS_RUY_H_ + +#include "third_party/absl/base/thread_annotations.h" +#include "third_party/absl/memory/memory.h" +#include "third_party/mlir_edge/iree/base/status.h" +#include "third_party/mlir_edge/iree/hal/buffer_view.h" +#include "third_party/tensorflow/lite/experimental/ruy/context.h" +#include "third_party/tensorflow/lite/experimental/ruy/ruy.h" + +namespace iree { +namespace hal { +namespace kernels { + +// TODO(benvanik): something more clever for making this shareable. +// Maybe a factory fn based on the impl selected? +struct MatMul::RuntimeState { + // TODO(benvanik): share the thread pool but keep context per-fiber? + ruy::Context context; +}; + +inline std::unique_ptr MatMul::CreateRuntimeState() { + return absl::make_unique(); +} + +template +Status MatMul::Execute(RuntimeState* runtime_state, + const Buffers& buffers) { + ruy::Matrix lhs_matrix; + ruy::MakeSimpleLayout(buffers.lhs_shape[0], buffers.lhs_shape[1], + ruy::Order::kRowMajor, &lhs_matrix.layout); + lhs_matrix.data.set(buffers.lhs_buffer.data()); + + ruy::Matrix rhs_matrix; + ruy::MakeSimpleLayout(buffers.rhs_shape[0], buffers.rhs_shape[1], + ruy::Order::kRowMajor, &rhs_matrix.layout); + rhs_matrix.data.set(buffers.rhs_buffer.data()); + + ruy::Matrix dst_matrix; + ruy::MakeSimpleLayout(buffers.dst_shape[0], buffers.dst_shape[1], + ruy::Order::kRowMajor, &dst_matrix.layout); + dst_matrix.data.set(buffers.dst_buffer.data()); + + ruy::BasicSpec spec; + spec.bias = buffers.bias_buffer.data(); + + if (buffers.multiplier_mantissa_buffer.size() == 1) { + spec.multiplier_fixedpoint = buffers.multiplier_mantissa_buffer[0]; + spec.multiplier_exponent = buffers.multiplier_exponent_buffer[0]; + } else { + spec.multiplier_fixedpoint_perchannel = + buffers.multiplier_mantissa_buffer.data(); + spec.multiplier_exponent_perchannel = + buffers.multiplier_exponent_buffer.data(); + } + + ruy::Mul(lhs_matrix, rhs_matrix, spec, + &runtime_state->context, &dst_matrix); + + return OkStatus(); +} + +} // namespace kernels +} // namespace hal +} // namespace iree + +#endif // THIRD_PARTY_MLIR_EDGE_IREE_HAL_INTERPRETER_BYTECODE_KERNELS_RUY_H_ diff --git a/hal/interpreter/interpreter_command_processor.cc b/hal/interpreter/interpreter_command_processor.cc new file mode 100644 index 000000000000..a3cd622e7f2e --- /dev/null +++ b/hal/interpreter/interpreter_command_processor.cc @@ -0,0 +1,68 @@ +// Copyright 2019 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "third_party/mlir_edge/iree/hal/interpreter/interpreter_command_processor.h" + +#include "third_party/absl/container/inlined_vector.h" +#include "third_party/absl/types/source_location.h" +#include "third_party/absl/types/span.h" +#include "third_party/mlir_edge/iree/base/status.h" +#include "third_party/mlir_edge/iree/base/tracing.h" +#include "third_party/mlir_edge/iree/hal/buffer_view.h" +#include "third_party/mlir_edge/iree/hal/interpreter/bytecode_executable.h" + +namespace iree { +namespace hal { + +InterpreterCommandProcessor::InterpreterCommandProcessor( + Allocator* allocator, CommandBufferModeBitfield mode, + CommandCategoryBitfield command_categories) + : HostLocalCommandProcessor(allocator, mode, command_categories) {} + +InterpreterCommandProcessor::~InterpreterCommandProcessor() = default; + +Status InterpreterCommandProcessor::Dispatch( + const DispatchRequest& dispatch_request) { + IREE_TRACE_SCOPE0("InterpreterCommandProcessor::Dispatch"); + + // Lookup the exported function. + auto* executable = + static_cast(dispatch_request.executable); + const auto& module = executable->module(); + ASSIGN_OR_RETURN(auto entry_function, module.function_table().LookupExport( + dispatch_request.entry_point)); + + vm::Stack stack; + + // TODO(benvanik): avoid this by directly referencing the bindings. + absl::InlinedVector args; + args.reserve(dispatch_request.bindings.size()); + for (auto& binding : dispatch_request.bindings) { + args.push_back(BufferView{add_ref(binding.buffer), binding.shape, + binding.element_size}); + } + absl::InlinedVector results; + if (entry_function.result_count() > 0) { + return UnimplementedErrorBuilder(ABSL_LOC) + << "Executable export results are not yet implemented"; + } + + RETURN_IF_ERROR(executable->context().Invoke( + &stack, entry_function, absl::MakeSpan(args), absl::MakeSpan(results))); + + return OkStatus(); +} + +} // namespace hal +} // namespace iree diff --git a/hal/interpreter/interpreter_command_processor.h b/hal/interpreter/interpreter_command_processor.h new file mode 100644 index 000000000000..642ad712e508 --- /dev/null +++ b/hal/interpreter/interpreter_command_processor.h @@ -0,0 +1,36 @@ +// Copyright 2019 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#ifndef THIRD_PARTY_MLIR_EDGE_IREE_HAL_INTERPRETER_INTERPRETER_COMMAND_PROCESSOR_H_ +#define THIRD_PARTY_MLIR_EDGE_IREE_HAL_INTERPRETER_INTERPRETER_COMMAND_PROCESSOR_H_ + +#include "third_party/mlir_edge/iree/hal/host/host_local_command_processor.h" + +namespace iree { +namespace hal { + +class InterpreterCommandProcessor final : public HostLocalCommandProcessor { + public: + InterpreterCommandProcessor(Allocator* allocator, + CommandBufferModeBitfield mode, + CommandCategoryBitfield command_categories); + ~InterpreterCommandProcessor() override; + + Status Dispatch(const DispatchRequest& dispatch_request) override; +}; + +} // namespace hal +} // namespace iree + +#endif // THIRD_PARTY_MLIR_EDGE_IREE_HAL_INTERPRETER_INTERPRETER_COMMAND_PROCESSOR_H_ diff --git a/hal/interpreter/interpreter_context.cc b/hal/interpreter/interpreter_context.cc new file mode 100644 index 000000000000..14935d353b8a --- /dev/null +++ b/hal/interpreter/interpreter_context.cc @@ -0,0 +1,66 @@ +// Copyright 2019 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "third_party/mlir_edge/iree/hal/interpreter/interpreter_context.h" + +#include "third_party/mlir_edge/iree/base/flatbuffer_util.h" +#include "third_party/mlir_edge/iree/base/status.h" +#include "third_party/mlir_edge/iree/hal/interpreter/bytecode_dispatch.h" + +namespace iree { +namespace hal { + +namespace { + +using ::iree::vm::Function; + +} // namespace + +Status InterpreterContext::Invoke(vm::Stack* stack, Function function, + absl::Span args, + absl::Span results) const { + // Verify arg/result counts. + if (args.size() != function.input_count()) { + return InvalidArgumentErrorBuilder(ABSL_LOC) + << "Function " << function.name() << " requires " + << function.input_count() << " inputs but only " << args.size() + << " provided"; + } + if (results.size() != function.result_count()) { + return InvalidArgumentErrorBuilder(ABSL_LOC) + << "Function " << function.name() << " requires " + << function.result_count() << " outputs but only " << results.size() + << " provided"; + } + + // Push stack frame for the function we are calling. + ASSIGN_OR_RETURN(auto* callee_stack_frame, stack->PushFrame(function)); + + // Marshal input arguments. + for (int i = 0; i < args.size(); ++i) { + *callee_stack_frame->mutable_local(i) = std::move(args[i]); + } + + // Run main dispatch loop until it exits (or errors). + RETURN_IF_ERROR(Dispatch(allocator_, &kernel_runtime_state_, stack, + callee_stack_frame, results)); + + // Pop the callee frame to balance out the stack. + RETURN_IF_ERROR(stack->PopFrame()); + + return OkStatus(); +} + +} // namespace hal +} // namespace iree diff --git a/hal/interpreter/interpreter_context.h b/hal/interpreter/interpreter_context.h new file mode 100644 index 000000000000..1cdb0c45d620 --- /dev/null +++ b/hal/interpreter/interpreter_context.h @@ -0,0 +1,50 @@ +// Copyright 2019 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#ifndef THIRD_PARTY_MLIR_EDGE_IREE_HAL_INTERPRETER_INTERPRETER_CONTEXT_H_ +#define THIRD_PARTY_MLIR_EDGE_IREE_HAL_INTERPRETER_INTERPRETER_CONTEXT_H_ + +#include + +#include "third_party/absl/types/span.h" +#include "third_party/mlir_edge/iree/base/status.h" +#include "third_party/mlir_edge/iree/hal/allocator.h" +#include "third_party/mlir_edge/iree/hal/buffer_view.h" +#include "third_party/mlir_edge/iree/hal/interpreter/bytecode_kernels.h" +#include "third_party/mlir_edge/iree/vm/context.h" +#include "third_party/mlir_edge/iree/vm/function.h" +#include "third_party/mlir_edge/iree/vm/stack.h" + +namespace iree { +namespace hal { + +class InterpreterContext final : public vm::Context { + public: + explicit InterpreterContext(hal::Allocator* allocator) + : allocator_(allocator) {} + + // TODO(benvanik): helpers to make passing args easier + Status Invoke(vm::Stack* stack, vm::Function function, + absl::Span args, + absl::Span results) const; + + private: + hal::Allocator* allocator_; + mutable kernels::RuntimeState kernel_runtime_state_; +}; + +} // namespace hal +} // namespace iree + +#endif // THIRD_PARTY_MLIR_EDGE_IREE_HAL_INTERPRETER_INTERPRETER_CONTEXT_H_ diff --git a/hal/interpreter/interpreter_device.cc b/hal/interpreter/interpreter_device.cc new file mode 100644 index 000000000000..3f225ee2782e --- /dev/null +++ b/hal/interpreter/interpreter_device.cc @@ -0,0 +1,177 @@ +// Copyright 2019 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "third_party/mlir_edge/iree/hal/interpreter/interpreter_device.h" + +#include + +#include "third_party/absl/memory/memory.h" +#include "third_party/mlir_edge/iree/base/status.h" +#include "third_party/mlir_edge/iree/base/tracing.h" +#include "third_party/mlir_edge/iree/hal/command_buffer_validation.h" +#include "third_party/mlir_edge/iree/hal/command_queue.h" +#include "third_party/mlir_edge/iree/hal/fence.h" +#include "third_party/mlir_edge/iree/hal/host/async_command_queue.h" +#include "third_party/mlir_edge/iree/hal/host/host_event.h" +#include "third_party/mlir_edge/iree/hal/host/host_submission_queue.h" +#include "third_party/mlir_edge/iree/hal/host/inproc_command_buffer.h" +#include "third_party/mlir_edge/iree/hal/interpreter/bytecode_cache.h" +#include "third_party/mlir_edge/iree/hal/interpreter/interpreter_command_processor.h" + +namespace iree { +namespace hal { + +namespace { + +// A CommandQueue that performs no synchronization (semaphores/fences) and just +// directly executes command buffers inline. +// +// This is meant to be wrapped by SyncCommandQueue or AsyncCommandQueue that +// themselves perform the synchronization/threading/etc. As such we ignore +// all semaphores in the provided batches under the assumption that if Submit is +// being called then all dependencies are valid. The wrapping queue is also +// responsible for signaling the fence as well as propagating errors in a way +// that is dependent on how it is performing its synchronization. +class UnsynchronizedCommandQueue final : public CommandQueue { + public: + UnsynchronizedCommandQueue(Allocator* allocator, std::string name, + CommandCategoryBitfield supported_categories) + : CommandQueue(std::move(name), supported_categories), + allocator_(allocator) {} + ~UnsynchronizedCommandQueue() override = default; + + Status Submit(absl::Span batches, + FenceValue fence) override { + IREE_TRACE_SCOPE0("UnsynchronizedCommandQueue::Submit"); + DCHECK_EQ(nullptr, fence.first) + << "Fences must be handled by the wrapping queue"; + + // Process command buffers and propagate errors asynchronously through the + // fence. This ensures that even if we are running synchronously we still + // get consistent failure behavior with drivers that are purely async. + for (auto& batch : batches) { + DCHECK(batch.wait_semaphores.empty() && batch.signal_semaphores.empty()) + << "Semaphores must be handled by the wrapping queue"; + RETURN_IF_ERROR(ProcessCommandBuffers(batch.command_buffers)); + } + + // NOTE: fence is ignored here. + return OkStatus(); + } + + Status Flush() override { + // No-op. + return OkStatus(); + } + + Status WaitIdle(absl::Time deadline) override { + // No-op. + return OkStatus(); + } + + private: + // Processes each command buffer in-turn with a fresh processor. + // This ensures we don't have any state that can carry across buffers. + Status ProcessCommandBuffers( + absl::Span command_buffers) { + IREE_TRACE_SCOPE0("UnsynchronizedCommandQueue::ProcessCommandBuffers"); + for (auto* command_buffer : command_buffers) { + auto* inproc_command_buffer = + static_cast(command_buffer->impl()); + InterpreterCommandProcessor command_processor( + allocator_, command_buffer->mode(), supported_categories()); + RETURN_IF_ERROR(inproc_command_buffer->Process(&command_processor)); + } + return OkStatus(); + } + + Allocator* const allocator_; +}; + +} // namespace + +InterpreterDevice::InterpreterDevice(DeviceInfo device_info) + : Device(std::move(device_info)) { + // We currently only expose a single command queue. + auto command_queue = absl::make_unique( + &allocator_, "cpu0", + CommandCategory::kTransfer | CommandCategory::kDispatch); + // TODO(benvanik): allow injection of the wrapper type to support + // SyncCommandQueue without always linking in both. + auto async_command_queue = + absl::make_unique(std::move(command_queue)); + command_queues_.push_back(std::move(async_command_queue)); +} + +InterpreterDevice::~InterpreterDevice() = default; + +std::shared_ptr InterpreterDevice::CreateExecutableCache() { + return std::make_shared(&allocator_); +} + +StatusOr> InterpreterDevice::CreateCommandBuffer( + CommandBufferModeBitfield mode, + CommandCategoryBitfield command_categories) { + // TODO(b/140026716): conditionally enable validation. + auto impl = + make_ref(&allocator_, mode, command_categories); + return WrapCommandBufferWithValidation(std::move(impl)); +} + +StatusOr> InterpreterDevice::CreateEvent() { + return make_ref(); +} + +StatusOr> InterpreterDevice::CreateBinarySemaphore( + bool initial_value) { + IREE_TRACE_SCOPE0("InterpreterDevice::CreateBinarySemaphore"); + return make_ref(initial_value); +} + +StatusOr> InterpreterDevice::CreateTimelineSemaphore( + uint64_t initial_value) { + IREE_TRACE_SCOPE0("InterpreterDevice::CreateTimelineSemaphore"); + + // TODO(b/140141417): implement timeline semaphores. + return UnimplementedErrorBuilder(ABSL_LOC) + << "Timeline semaphores not yet implemented"; +} + +StatusOr> InterpreterDevice::CreateFence( + uint64_t initial_value) { + IREE_TRACE_SCOPE0("InterpreterDevice::CreateFence"); + return make_ref(initial_value); +} + +Status InterpreterDevice::WaitAllFences(absl::Span fences, + absl::Time deadline) { + IREE_TRACE_SCOPE0("InterpreterDevice::WaitAllFences"); + return HostFence::WaitForFences(fences, /*wait_all=*/true, deadline); +} + +StatusOr InterpreterDevice::WaitAnyFence( + absl::Span fences, absl::Time deadline) { + IREE_TRACE_SCOPE0("InterpreterDevice::WaitAnyFence"); + return HostFence::WaitForFences(fences, /*wait_all=*/false, deadline); +} + +Status InterpreterDevice::WaitIdle(absl::Time deadline) { + for (auto& command_queue : command_queues_) { + RETURN_IF_ERROR(command_queue->WaitIdle(deadline)); + } + return OkStatus(); +} + +} // namespace hal +} // namespace iree diff --git a/hal/interpreter/interpreter_device.h b/hal/interpreter/interpreter_device.h new file mode 100644 index 000000000000..752fa7478cc0 --- /dev/null +++ b/hal/interpreter/interpreter_device.h @@ -0,0 +1,77 @@ +// Copyright 2019 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#ifndef THIRD_PARTY_MLIR_EDGE_IREE_HAL_INTERPRETER_INTERPRETER_DEVICE_H_ +#define THIRD_PARTY_MLIR_EDGE_IREE_HAL_INTERPRETER_INTERPRETER_DEVICE_H_ + +#include "third_party/absl/container/inlined_vector.h" +#include "third_party/absl/types/span.h" +#include "third_party/mlir_edge/iree/base/memory.h" +#include "third_party/mlir_edge/iree/hal/device.h" +#include "third_party/mlir_edge/iree/hal/host/host_local_allocator.h" +#include "third_party/mlir_edge/iree/hal/interpreter/bytecode_kernels.h" + +namespace iree { +namespace hal { + +class InterpreterDevice final : public Device { + public: + explicit InterpreterDevice(DeviceInfo device_info); + ~InterpreterDevice() override; + + kernels::RuntimeState* kernel_runtime_state() { + return &kernel_runtime_state_; + } + + Allocator* allocator() const override { return &allocator_; } + + absl::Span dispatch_queues() const override { + return RawPtrSpan(absl::MakeSpan(command_queues_)); + } + + absl::Span transfer_queues() const override { + return RawPtrSpan(absl::MakeSpan(command_queues_)); + } + + std::shared_ptr CreateExecutableCache() override; + + StatusOr> CreateCommandBuffer( + CommandBufferModeBitfield mode, + CommandCategoryBitfield command_categories) override; + + StatusOr> CreateEvent() override; + + StatusOr> CreateBinarySemaphore( + bool initial_value) override; + StatusOr> CreateTimelineSemaphore( + uint64_t initial_value) override; + + StatusOr> CreateFence(uint64_t initial_value) override; + Status WaitAllFences(absl::Span fences, + absl::Time deadline) override; + StatusOr WaitAnyFence(absl::Span fences, + absl::Time deadline) override; + + Status WaitIdle(absl::Time deadline) override; + + private: + kernels::RuntimeState kernel_runtime_state_; + mutable HostLocalAllocator allocator_; + mutable absl::InlinedVector, 1> command_queues_; +}; + +} // namespace hal +} // namespace iree + +#endif // THIRD_PARTY_MLIR_EDGE_IREE_HAL_INTERPRETER_INTERPRETER_DEVICE_H_ diff --git a/hal/interpreter/interpreter_driver.cc b/hal/interpreter/interpreter_driver.cc new file mode 100644 index 000000000000..a1af3959ca0e --- /dev/null +++ b/hal/interpreter/interpreter_driver.cc @@ -0,0 +1,62 @@ +// Copyright 2019 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "third_party/mlir_edge/iree/hal/interpreter/interpreter_driver.h" + +#include + +#include "third_party/mlir_edge/iree/hal/device_info.h" +#include "third_party/mlir_edge/iree/hal/interpreter/interpreter_device.h" + +namespace iree { +namespace hal { + +namespace { + +DeviceInfo GetDefaultDeviceInfo() { + DeviceFeatureBitfield supported_features = DeviceFeature::kNone; + // TODO(benvanik): implement debugging/profiling features. + // supported_features |= DeviceFeature::kDebugging; + // supported_features |= DeviceFeature::kCoverage; + // supported_features |= DeviceFeature::kProfiling; + DeviceInfo device_info("interpreter", supported_features); + // TODO(benvanik): device info. + return device_info; +} + +} // namespace + +InterpreterDriver::InterpreterDriver() : Driver("interpreter") {} + +InterpreterDriver::~InterpreterDriver() = default; + +StatusOr> +InterpreterDriver::EnumerateAvailableDevices() { + std::vector device_infos; + device_infos.push_back(GetDefaultDeviceInfo()); + return device_infos; +} + +StatusOr> InterpreterDriver::CreateDefaultDevice() { + return CreateDevice(GetDefaultDeviceInfo()); +} + +StatusOr> InterpreterDriver::CreateDevice( + const DeviceInfo& device_info) { + auto device = std::make_shared(device_info); + return device; +} + +} // namespace hal +} // namespace iree diff --git a/hal/interpreter/interpreter_driver.h b/hal/interpreter/interpreter_driver.h new file mode 100644 index 000000000000..9a760d515d9f --- /dev/null +++ b/hal/interpreter/interpreter_driver.h @@ -0,0 +1,39 @@ +// Copyright 2019 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#ifndef THIRD_PARTY_MLIR_EDGE_IREE_HAL_INTERPRETER_INTERPRETER_DRIVER_H_ +#define THIRD_PARTY_MLIR_EDGE_IREE_HAL_INTERPRETER_INTERPRETER_DRIVER_H_ + +#include "third_party/mlir_edge/iree/hal/driver.h" + +namespace iree { +namespace hal { + +class InterpreterDriver final : public Driver { + public: + InterpreterDriver(); + ~InterpreterDriver() override; + + StatusOr> EnumerateAvailableDevices() override; + + StatusOr> CreateDefaultDevice() override; + + StatusOr> CreateDevice( + const DeviceInfo& device_info) override; +}; + +} // namespace hal +} // namespace iree + +#endif // THIRD_PARTY_MLIR_EDGE_IREE_HAL_INTERPRETER_INTERPRETER_DRIVER_H_ diff --git a/hal/interpreter/interpreter_driver_module.cc b/hal/interpreter/interpreter_driver_module.cc new file mode 100644 index 000000000000..8c40d5ed7733 --- /dev/null +++ b/hal/interpreter/interpreter_driver_module.cc @@ -0,0 +1,37 @@ +// Copyright 2019 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include + +#include "third_party/mlir_edge/iree/base/init.h" +#include "third_party/mlir_edge/iree/base/status.h" +#include "third_party/mlir_edge/iree/hal/driver_registry.h" +#include "third_party/mlir_edge/iree/hal/interpreter/interpreter_driver.h" + +namespace iree { +namespace hal { + +StatusOr> CreateInterpreterDriver() { + return std::make_shared(); +} + +} // namespace hal +} // namespace iree + +IREE_REGISTER_MODULE_INITIALIZER(iree_hal_interpreter_driver, { + QCHECK_OK(::iree::hal::DriverRegistry::shared_registry()->Register( + "interpreter", ::iree::hal::CreateInterpreterDriver)); +}); +IREE_REGISTER_MODULE_INITIALIZER_SEQUENCE(iree_hal, + iree_hal_interpreter_driver); diff --git a/hal/resource.h b/hal/resource.h new file mode 100644 index 000000000000..ad46a6cafacc --- /dev/null +++ b/hal/resource.h @@ -0,0 +1,33 @@ +// Copyright 2019 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#ifndef THIRD_PARTY_MLIR_EDGE_IREE_HAL_RESOURCE_H_ +#define THIRD_PARTY_MLIR_EDGE_IREE_HAL_RESOURCE_H_ + +#include "third_party/mlir_edge/iree/base/ref_ptr.h" + +namespace iree { +namespace hal { + +// Abstract resource type whose lifetime is managed by a ResourceSet. +// Used mostly just to get a virtual dtor, though we could add nicer logging. +class Resource : public RefObject { + public: + virtual ~Resource() = default; +}; + +} // namespace hal +} // namespace iree + +#endif // THIRD_PARTY_MLIR_EDGE_IREE_HAL_RESOURCE_H_ diff --git a/hal/semaphore.h b/hal/semaphore.h new file mode 100644 index 000000000000..3a7e7bee0edf --- /dev/null +++ b/hal/semaphore.h @@ -0,0 +1,61 @@ +// Copyright 2019 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#ifndef THIRD_PARTY_MLIR_EDGE_IREE_HAL_SEMAPHORE_H_ +#define THIRD_PARTY_MLIR_EDGE_IREE_HAL_SEMAPHORE_H_ + +#include "third_party/absl/types/variant.h" +#include "third_party/mlir_edge/iree/hal/resource.h" + +namespace iree { +namespace hal { + +// A synchronization primitive used to indicate submission dependencies. +// Semaphores are either of type binary (signaled or unsignaled) or timeline +// (uint64 payload with >= semantics). +class Semaphore : public Resource { + public: +}; + +// Binary semaphores have strict ordering requirements and must be carefully +// balanced. Each binary semaphore must only be waited on after a signal +// operation has been issued and each wait requires exactly one signal. They +// are commonly used only when interacting with external handles that may +// cross device or process boundaries. +class BinarySemaphore : public Semaphore { + public: +}; + +// Timeline semaphores act as a fence along a per-semaphore timeline where +// signaling is done by setting the payload to a monotonically increasing +// 64-bit integer and waiting is done by blocking until the payload is set +// greater-than or equal-to the specified value. Timeline semaphores may be +// waited on or signaled in any order and can be significantly more +// efficient due to system-level coalescing. +class TimelineSemaphore : public Semaphore { + public: + // TODO(benvanik): add value query support. + // TODO(benvanik): add host-side signal/wait. +}; + +// A reference to a strongly-typed semaphore and associated information. +// For TimelineSemaphores the provided payload is used to specify either the +// payload to wait for or new payload value. +using SemaphoreValue = + absl::variant>; + +} // namespace hal +} // namespace iree + +#endif // THIRD_PARTY_MLIR_EDGE_IREE_HAL_SEMAPHORE_H_ diff --git a/hal/stack_trace.h b/hal/stack_trace.h new file mode 100644 index 000000000000..3b9243838b25 --- /dev/null +++ b/hal/stack_trace.h @@ -0,0 +1,39 @@ +// Copyright 2019 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#ifndef THIRD_PARTY_MLIR_EDGE_IREE_HAL_STACK_TRACE_H_ +#define THIRD_PARTY_MLIR_EDGE_IREE_HAL_STACK_TRACE_H_ + +namespace iree { +namespace hal { + +class StackTrace { + public: + // TODO(benvanik): define contents. + // frame: + // device type (cpu, etc) + // effective processor type (determines disasm/etc) <- r52, vliw, etc + // effective offset <- in disasm (abstract, could be op ordinal, byte + // offset) + // source offset <- used in source map lookup + // physical offset <- informative, void* (real memory address) + // physical_context (x86 registers, etc) + // effective_context (??) + // source_context (buffer views/etc?) +}; + +} // namespace hal +} // namespace iree + +#endif // THIRD_PARTY_MLIR_EDGE_IREE_HAL_STACK_TRACE_H_ diff --git a/hal/testing/mock_allocator.h b/hal/testing/mock_allocator.h new file mode 100644 index 000000000000..7cf42f755ede --- /dev/null +++ b/hal/testing/mock_allocator.h @@ -0,0 +1,55 @@ +// Copyright 2019 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#ifndef THIRD_PARTY_MLIR_EDGE_IREE_HAL_TESTING_MOCK_ALLOCATOR_H_ +#define THIRD_PARTY_MLIR_EDGE_IREE_HAL_TESTING_MOCK_ALLOCATOR_H_ + +#include "testing/base/public/gmock.h" +#include "third_party/mlir_edge/iree/hal/allocator.h" + +namespace iree { +namespace hal { +namespace testing { + +class MockAllocator : public ::testing::StrictMock { + public: + MockAllocator() : ::testing::StrictMock() {} + + MOCK_CONST_METHOD4(CanUseBufferLike, + bool(Allocator* source_allocator, + MemoryTypeBitfield memory_type, + BufferUsageBitfield buffer_usage, + BufferUsageBitfield intended_usage)); + + MOCK_CONST_METHOD3(CanAllocate, bool(MemoryTypeBitfield memory_type, + BufferUsageBitfield buffer_usage, + size_t allocation_size)); + + MOCK_METHOD3(Allocate, + StatusOr>(MemoryTypeBitfield memory_type, + BufferUsageBitfield buffer_usage, + size_t allocation_size)); + + MOCK_METHOD5(WrapMutable, + StatusOr>(MemoryTypeBitfield memory_type, + MemoryAccessBitfield allowed_access, + BufferUsageBitfield buffer_usage, + void* data, size_t data_length)); +}; + +} // namespace testing +} // namespace hal +} // namespace iree + +#endif // THIRD_PARTY_MLIR_EDGE_IREE_HAL_TESTING_MOCK_ALLOCATOR_H_ diff --git a/hal/testing/mock_command_buffer.h b/hal/testing/mock_command_buffer.h new file mode 100644 index 000000000000..5e479c78c664 --- /dev/null +++ b/hal/testing/mock_command_buffer.h @@ -0,0 +1,80 @@ +// Copyright 2019 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#ifndef THIRD_PARTY_MLIR_EDGE_IREE_HAL_TESTING_MOCK_COMMAND_BUFFER_H_ +#define THIRD_PARTY_MLIR_EDGE_IREE_HAL_TESTING_MOCK_COMMAND_BUFFER_H_ + +#include "testing/base/public/gmock.h" +#include "third_party/mlir_edge/iree/hal/command_buffer.h" + +namespace iree { +namespace hal { +namespace testing { + +class MockCommandBuffer : public ::testing::StrictMock { + public: + MockCommandBuffer(Allocator* allocator, CommandBufferModeBitfield mode, + CommandCategoryBitfield command_categories) + : ::testing::StrictMock(allocator, mode, + command_categories) {} + + bool is_recording() const override { return false; } + + MOCK_METHOD0(Begin, Status()); + MOCK_METHOD0(End, Status()); + + MOCK_METHOD4(ExecutionBarrier, + Status(ExecutionStageBitfield source_stage_mask, + ExecutionStageBitfield target_stage_mask, + absl::Span memory_barriers, + absl::Span buffer_barriers)); + + MOCK_METHOD2(SignalEvent, + Status(Event* event, ExecutionStageBitfield source_stage_mask)); + + MOCK_METHOD2(ResetEvent, + Status(Event* event, ExecutionStageBitfield source_stage_mask)); + + MOCK_METHOD5(WaitEvents, + Status(absl::Span events, + ExecutionStageBitfield source_stage_mask, + ExecutionStageBitfield target_stage_mask, + absl::Span memory_barriers, + absl::Span buffer_barriers)); + + MOCK_METHOD5(FillBuffer, + Status(Buffer* target_buffer, device_size_t target_offset, + device_size_t length, const void* pattern, + size_t pattern_length)); + + MOCK_METHOD1(DiscardBuffer, Status(Buffer* buffer)); + + MOCK_METHOD5(UpdateBuffer, + Status(const void* source_buffer, device_size_t source_offset, + Buffer* target_buffer, device_size_t target_offset, + device_size_t length)); + + MOCK_METHOD5(CopyBuffer, + Status(Buffer* source_buffer, device_size_t source_offset, + Buffer* target_buffer, device_size_t target_offset, + device_size_t length)); + + MOCK_METHOD1(Dispatch, Status(const DispatchRequest& dispatch_request)); +}; + +} // namespace testing +} // namespace hal +} // namespace iree + +#endif // THIRD_PARTY_MLIR_EDGE_IREE_HAL_TESTING_MOCK_COMMAND_BUFFER_H_ diff --git a/hal/testing/mock_command_queue.h b/hal/testing/mock_command_queue.h new file mode 100644 index 000000000000..ce152d525868 --- /dev/null +++ b/hal/testing/mock_command_queue.h @@ -0,0 +1,44 @@ +// Copyright 2019 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#ifndef THIRD_PARTY_MLIR_EDGE_IREE_HAL_TESTING_MOCK_COMMAND_QUEUE_H_ +#define THIRD_PARTY_MLIR_EDGE_IREE_HAL_TESTING_MOCK_COMMAND_QUEUE_H_ + +#include "testing/base/public/gmock.h" +#include "third_party/mlir_edge/iree/hal/command_queue.h" + +namespace iree { +namespace hal { +namespace testing { + +class MockCommandQueue : public ::testing::StrictMock { + public: + MockCommandQueue(std::string name, + CommandCategoryBitfield supported_categories) + : ::testing::StrictMock(std::move(name), + supported_categories) {} + + MOCK_METHOD2(Submit, Status(absl::Span batches, + FenceValue fence)); + + MOCK_METHOD0(Flush, Status()); + + MOCK_METHOD1(WaitIdle, Status(absl::Time deadline)); +}; + +} // namespace testing +} // namespace hal +} // namespace iree + +#endif // THIRD_PARTY_MLIR_EDGE_IREE_HAL_TESTING_MOCK_COMMAND_QUEUE_H_ diff --git a/hal/vulkan/debug_reporter.cc b/hal/vulkan/debug_reporter.cc new file mode 100644 index 000000000000..02e968455e6d --- /dev/null +++ b/hal/vulkan/debug_reporter.cc @@ -0,0 +1,160 @@ +// Copyright 2019 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "third_party/mlir_edge/iree/hal/vulkan/debug_reporter.h" + +#include "third_party/mlir_edge/iree/base/tracing.h" +#include "third_party/mlir_edge/iree/hal/vulkan/status_util.h" + +namespace iree { +namespace hal { +namespace vulkan { + +namespace { + +// NOTE: |user_data| may be nullptr if we are being called during instance +// creation. Otherwise it is a pointer to the DebugReporter instance. + +// NOTE: this callback must be thread safe and must be careful not to reach too +// far outside of the call - it is called in-context from arbitrary threads with +// some amount of Vulkan state on the stack. Assume that creating or deleting +// Vulkan objects, issuing most Vulkan commands, etc are off-limits. + +VKAPI_ATTR VkBool32 VKAPI_CALL DebugUtilsMessageCallback( + VkDebugUtilsMessageSeverityFlagBitsEXT message_severity, + VkDebugUtilsMessageTypeFlagsEXT message_type, + const VkDebugUtilsMessengerCallbackDataEXT* callback_data, + void* user_data) { + // TODO(benvanik): better logging once we have switched logging APIs. + LOG(ERROR) << callback_data->pMessage; + + return VK_FALSE; // VK_TRUE is reserved for future use. +} + +VKAPI_ATTR VkBool32 VKAPI_CALL DebugReportCallback( + VkDebugReportFlagsEXT flags, VkDebugReportObjectTypeEXT object_type, + uint64_t object, size_t location, int32_t message_code, + const char* layer_prefix, const char* message, void* user_data) { + // TODO(benvanik): better logging once we have switched logging APIs. + LOG(ERROR) << message; + + return VK_FALSE; // VK_TRUE is reserved for future use. +} + +} // namespace + +// static +void DebugReporter::PopulateStaticCreateInfo( + VkDebugUtilsMessengerCreateInfoEXT* create_info) { + create_info->sType = VK_STRUCTURE_TYPE_DEBUG_UTILS_MESSENGER_CREATE_INFO_EXT; + create_info->pNext = nullptr; + create_info->flags = 0; + + // TODO(benvanik): only enable the severities that logging has enabled. + create_info->messageSeverity = + VK_DEBUG_UTILS_MESSAGE_SEVERITY_VERBOSE_BIT_EXT | + VK_DEBUG_UTILS_MESSAGE_SEVERITY_INFO_BIT_EXT | + VK_DEBUG_UTILS_MESSAGE_SEVERITY_WARNING_BIT_EXT | + VK_DEBUG_UTILS_MESSAGE_SEVERITY_ERROR_BIT_EXT; + + // TODO(benvanik): allow filtering by category as a flag. + create_info->messageType = VK_DEBUG_UTILS_MESSAGE_TYPE_GENERAL_BIT_EXT | + VK_DEBUG_UTILS_MESSAGE_TYPE_VALIDATION_BIT_EXT | + VK_DEBUG_UTILS_MESSAGE_TYPE_PERFORMANCE_BIT_EXT; + + create_info->pfnUserCallback = DebugUtilsMessageCallback; + create_info->pUserData = nullptr; +} + +// static +void DebugReporter::PopulateStaticCreateInfo( + VkDebugReportCallbackCreateInfoEXT* create_info) { + create_info->sType = VK_STRUCTURE_TYPE_DEBUG_REPORT_CALLBACK_CREATE_INFO_EXT; + create_info->pNext = nullptr; + create_info->flags = 0; + + // TODO(benvanik): only enable the severities that logging has enabled. + create_info->flags |= + VK_DEBUG_REPORT_INFORMATION_BIT_EXT | VK_DEBUG_REPORT_WARNING_BIT_EXT | + VK_DEBUG_REPORT_PERFORMANCE_WARNING_BIT_EXT | + VK_DEBUG_REPORT_ERROR_BIT_EXT | VK_DEBUG_REPORT_DEBUG_BIT_EXT; + + create_info->pfnCallback = DebugReportCallback; + create_info->pUserData = nullptr; +} + +// static +StatusOr> +DebugReporter::CreateDebugUtilsMessenger( + VkInstance instance, const ref_ptr& syms, + const VkAllocationCallbacks* allocation_callbacks) { + IREE_TRACE_SCOPE0("DebugReporter::CreateDebugUtilsMessenger"); + + auto debug_reporter = + absl::WrapUnique(new DebugReporter(instance, syms, allocation_callbacks)); + + VkDebugUtilsMessengerCreateInfoEXT create_info; + PopulateStaticCreateInfo(&create_info); + create_info.pUserData = debug_reporter.get(); + + VK_RETURN_IF_ERROR(syms->vkCreateDebugUtilsMessengerEXT( + instance, &create_info, allocation_callbacks, + &debug_reporter->messenger_)); + + return debug_reporter; +} + +// static +StatusOr> +DebugReporter::CreateDebugReportCallback( + VkInstance instance, const ref_ptr& syms, + const VkAllocationCallbacks* allocation_callbacks) { + IREE_TRACE_SCOPE0("DebugReporter::CreateDebugReportCallback"); + + auto debug_reporter = + absl::WrapUnique(new DebugReporter(instance, syms, allocation_callbacks)); + + VkDebugReportCallbackCreateInfoEXT create_info; + PopulateStaticCreateInfo(&create_info); + create_info.pUserData = debug_reporter.get(); + + VK_RETURN_IF_ERROR(syms->vkCreateDebugReportCallbackEXT( + instance, &create_info, allocation_callbacks, + &debug_reporter->callback_)); + + return debug_reporter; +} + +DebugReporter::DebugReporter(VkInstance instance, + const ref_ptr& syms, + const VkAllocationCallbacks* allocation_callbacks) + : instance_(instance), + syms_(add_ref(syms)), + allocation_callbacks_(allocation_callbacks) {} + +DebugReporter::~DebugReporter() { + IREE_TRACE_SCOPE0("DebugReporter::dtor"); + if (messenger_ != VK_NULL_HANDLE) { + syms_->vkDestroyDebugUtilsMessengerEXT(instance_, messenger_, + allocation_callbacks_); + } + if (callback_ != VK_NULL_HANDLE) { + syms_->vkDestroyDebugReportCallbackEXT(instance_, callback_, + allocation_callbacks_); + } +} + +} // namespace vulkan +} // namespace hal +} // namespace iree diff --git a/hal/vulkan/debug_reporter.h b/hal/vulkan/debug_reporter.h new file mode 100644 index 000000000000..c95961d2d724 --- /dev/null +++ b/hal/vulkan/debug_reporter.h @@ -0,0 +1,87 @@ +// Copyright 2019 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#ifndef THIRD_PARTY_MLIR_EDGE_IREE_HAL_VULKAN_DEBUG_REPORTER_H_ +#define THIRD_PARTY_MLIR_EDGE_IREE_HAL_VULKAN_DEBUG_REPORTER_H_ + +#include + +#include "third_party/mlir_edge/iree/base/status.h" +#include "third_party/mlir_edge/iree/hal/vulkan/dynamic_symbols.h" + +namespace iree { +namespace hal { +namespace vulkan { + +// A debug reporter that works with the VK_EXT_debug_utils extension. +// One reporter should be created per VkInstance to receive callbacks from the +// API and route them to our logging systems. In general VK_EXT_debug_utils +// should be preferred if available as it provides a much cleaner interface and +// more plug-points than VK_EXT_debug_report. +// +// Since creating a reporter requires a VkInstance it's not possible to report +// on messages during instance creation. To work around this it's possible to +// pass a *CreateInfo struct to vkCreateInstance as part of the +// VkInstanceCreateInfo::pNext chain. The callback will only be used this way +// during the creation call after which users can create the real +// instance-specific reporter. +class DebugReporter final { + public: + // Populates |create_info| with an instance-agnostic callback. + // This can be used during instance creation by chaining the |create_info| to + // VkInstanceCreateInfo::pNext. + // + // Only use if VK_EXT_debug_utils is present. + static void PopulateStaticCreateInfo( + VkDebugUtilsMessengerCreateInfoEXT* create_info); + + // Populates |create_info| with an instance-agnostic callback. + // This can be used during instance creation by chaining the |create_info| to + // VkInstanceCreateInfo::pNext. + // + // Only use if VK_EXT_debug_report is present. + static void PopulateStaticCreateInfo( + VkDebugReportCallbackCreateInfoEXT* create_info); + + // Creates a debug messenger for the given Vulkan |instance| with + // VK_EXT_debug_utils enabled. + static StatusOr> CreateDebugUtilsMessenger( + VkInstance instance, const ref_ptr& syms, + const VkAllocationCallbacks* allocation_callbacks); + + // Creates a debug report callback for the given Vulkan |instance| with + // VK_EXT_debug_report enabled. + static StatusOr> CreateDebugReportCallback( + VkInstance instance, const ref_ptr& syms, + const VkAllocationCallbacks* allocation_callbacks); + + ~DebugReporter(); + + private: + DebugReporter(VkInstance instance, const ref_ptr& syms, + const VkAllocationCallbacks* allocation_callbacks); + + VkInstance instance_ = VK_NULL_HANDLE; + ref_ptr syms_; + const VkAllocationCallbacks* allocation_callbacks_ = nullptr; + + VkDebugUtilsMessengerEXT messenger_ = VK_NULL_HANDLE; + VkDebugReportCallbackEXT callback_ = VK_NULL_HANDLE; +}; + +} // namespace vulkan +} // namespace hal +} // namespace iree + +#endif // THIRD_PARTY_MLIR_EDGE_IREE_HAL_VULKAN_DEBUG_REPORTER_H_ diff --git a/hal/vulkan/direct_command_buffer.cc b/hal/vulkan/direct_command_buffer.cc new file mode 100644 index 000000000000..540b357da0d2 --- /dev/null +++ b/hal/vulkan/direct_command_buffer.cc @@ -0,0 +1,470 @@ +// Copyright 2019 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "third_party/mlir_edge/iree/hal/vulkan/direct_command_buffer.h" + +#include "third_party/absl/base/attributes.h" +#include "third_party/absl/container/inlined_vector.h" +#include "third_party/absl/synchronization/mutex.h" +#include "third_party/absl/types/source_location.h" +#include "third_party/mlir_edge/iree/base/status.h" +#include "third_party/mlir_edge/iree/base/tracing.h" +#include "third_party/mlir_edge/iree/hal/vulkan/status_util.h" + +namespace iree { +namespace hal { +namespace vulkan { + +namespace { + +VkPipelineStageFlags ConvertPipelineStageFlags( + ExecutionStageBitfield stage_mask) { + VkPipelineStageFlags flags = 0; + flags |= AnyBitSet(stage_mask & ExecutionStage::kCommandIssue) + ? VK_PIPELINE_STAGE_TOP_OF_PIPE_BIT + : 0; + flags |= AnyBitSet(stage_mask & ExecutionStage::kCommandProcess) + ? VK_PIPELINE_STAGE_DRAW_INDIRECT_BIT + : 0; + flags |= AnyBitSet(stage_mask & ExecutionStage::kDispatch) + ? VK_PIPELINE_STAGE_COMPUTE_SHADER_BIT + : 0; + flags |= AnyBitSet(stage_mask & ExecutionStage::kTransfer) + ? VK_PIPELINE_STAGE_TRANSFER_BIT + : 0; + flags |= AnyBitSet(stage_mask & ExecutionStage::kCommandRetire) + ? VK_PIPELINE_STAGE_BOTTOM_OF_PIPE_BIT + : 0; + flags |= AnyBitSet(stage_mask & ExecutionStage::kHost) + ? VK_PIPELINE_STAGE_HOST_BIT + : 0; + return flags; +} + +VkAccessFlags ConvertAccessMask(AccessScopeBitfield access_mask) { + VkAccessFlags flags = 0; + flags |= AnyBitSet(access_mask & AccessScope::kIndirectCommandRead) + ? VK_ACCESS_INDIRECT_COMMAND_READ_BIT + : 0; + flags |= AnyBitSet(access_mask & AccessScope::kConstantRead) + ? VK_ACCESS_UNIFORM_READ_BIT + : 0; + flags |= AnyBitSet(access_mask & AccessScope::kDispatchRead) + ? VK_ACCESS_SHADER_READ_BIT + : 0; + flags |= AnyBitSet(access_mask & AccessScope::kDispatchWrite) + ? VK_ACCESS_SHADER_WRITE_BIT + : 0; + flags |= AnyBitSet(access_mask & AccessScope::kTransferRead) + ? VK_ACCESS_TRANSFER_READ_BIT + : 0; + flags |= AnyBitSet(access_mask & AccessScope::kTransferWrite) + ? VK_ACCESS_TRANSFER_WRITE_BIT + : 0; + flags |= AnyBitSet(access_mask & AccessScope::kHostRead) + ? VK_ACCESS_HOST_READ_BIT + : 0; + flags |= AnyBitSet(access_mask & AccessScope::kHostWrite) + ? VK_ACCESS_HOST_WRITE_BIT + : 0; + flags |= AnyBitSet(access_mask & AccessScope::kMemoryRead) + ? VK_ACCESS_MEMORY_READ_BIT + : 0; + flags |= AnyBitSet(access_mask & AccessScope::kMemoryWrite) + ? VK_ACCESS_MEMORY_WRITE_BIT + : 0; + return flags; +} + +// Splats a pattern value of 1, 2, or 4 bytes out to a 4 byte value. +uint32_t SplatPattern(const void* pattern, size_t pattern_length) { + switch (pattern_length) { + case 1: { + uint32_t pattern_value = *static_cast(pattern); + return (pattern_value << 24) | (pattern_value << 16) | + (pattern_value << 8) | pattern_value; + } + case 2: { + uint32_t pattern_value = *static_cast(pattern); + return (pattern_value << 16) | pattern_value; + } + case 4: { + uint32_t pattern_value = *static_cast(pattern); + return pattern_value; + } + default: + return 0; // Already verified that this should not be possible. + } +} + +} // namespace + +DirectCommandBuffer::DirectCommandBuffer( + Allocator* allocator, CommandBufferModeBitfield mode, + CommandCategoryBitfield command_categories, + const ref_ptr& command_pool, + VkCommandBuffer command_buffer) + : CommandBuffer(allocator, mode, command_categories), + command_pool_(add_ref(command_pool)), + command_buffer_(command_buffer) {} + +DirectCommandBuffer::~DirectCommandBuffer() { + IREE_TRACE_SCOPE0("DirectCommandBuffer::dtor"); + absl::MutexLock lock(command_pool_->mutex()); + syms()->vkFreeCommandBuffers(*command_pool_->logical_device(), *command_pool_, + 1, &command_buffer_); +} + +StatusOr DirectCommandBuffer::CastEvent(Event* event) const { + // TODO(benvanik): assert the event is valid. + return static_cast(event); +} + +StatusOr DirectCommandBuffer::CastBuffer(Buffer* buffer) const { + // TODO(benvanik): assert that the buffer is from the right allocator and + // that it is compatible with our target queue family. + return static_cast(buffer->allocated_buffer()); +} + +Status DirectCommandBuffer::Begin() { + IREE_TRACE_SCOPE0("DirectCommandBuffer::Begin"); + + is_recording_ = true; + + VkCommandBufferBeginInfo begin_info; + begin_info.sType = VK_STRUCTURE_TYPE_COMMAND_BUFFER_BEGIN_INFO; + begin_info.pNext = nullptr; + begin_info.flags = AllBitsSet(mode(), CommandBufferMode::kOneShot) + ? VK_COMMAND_BUFFER_USAGE_ONE_TIME_SUBMIT_BIT + : 0; + begin_info.pInheritanceInfo = nullptr; + VK_RETURN_IF_ERROR( + syms()->vkBeginCommandBuffer(command_buffer_, &begin_info)); + + return OkStatus(); +} + +Status DirectCommandBuffer::End() { + IREE_TRACE_SCOPE0("DirectCommandBuffer::End"); + + VK_RETURN_IF_ERROR(syms()->vkEndCommandBuffer(command_buffer_)); + + is_recording_ = false; + + return OkStatus(); +} + +Status DirectCommandBuffer::ExecutionBarrier( + ExecutionStageBitfield source_stage_mask, + ExecutionStageBitfield target_stage_mask, + absl::Span memory_barriers, + absl::Span buffer_barriers) { + IREE_TRACE_SCOPE0("DirectCommandBuffer::ExecutionBarrier"); + + absl::InlinedVector memory_barrier_infos( + memory_barriers.size()); + for (int i = 0; i < memory_barriers.size(); ++i) { + const auto& memory_barrier = memory_barriers[i]; + auto& info = memory_barrier_infos[i]; + info.sType = VK_STRUCTURE_TYPE_MEMORY_BARRIER; + info.pNext = nullptr; + info.srcAccessMask = ConvertAccessMask(memory_barrier.source_scope); + info.dstAccessMask = ConvertAccessMask(memory_barrier.target_scope); + } + + absl::InlinedVector buffer_barrier_infos( + buffer_barriers.size()); + for (int i = 0; i < buffer_barriers.size(); ++i) { + const auto& buffer_barrier = buffer_barriers[i]; + auto& info = buffer_barrier_infos[i]; + info.sType = VK_STRUCTURE_TYPE_BUFFER_MEMORY_BARRIER; + info.pNext = nullptr; + info.srcAccessMask = ConvertAccessMask(buffer_barrier.source_scope); + info.dstAccessMask = ConvertAccessMask(buffer_barrier.target_scope); + info.srcQueueFamilyIndex = VK_QUEUE_FAMILY_IGNORED; + info.dstQueueFamilyIndex = VK_QUEUE_FAMILY_IGNORED; + ASSIGN_OR_RETURN(auto* device_buffer, CastBuffer(buffer_barrier.buffer)); + info.buffer = device_buffer->handle(); + info.offset = buffer_barrier.offset; + info.size = buffer_barrier.length; + } + + syms()->vkCmdPipelineBarrier( + command_buffer_, ConvertPipelineStageFlags(source_stage_mask), + ConvertPipelineStageFlags(target_stage_mask), /*dependencyFlags=*/0, + memory_barrier_infos.size(), memory_barrier_infos.data(), + buffer_barrier_infos.size(), buffer_barrier_infos.data(), 0, nullptr); + + return OkStatus(); +} + +Status DirectCommandBuffer::SignalEvent( + Event* event, ExecutionStageBitfield source_stage_mask) { + IREE_TRACE_SCOPE0("DirectCommandBuffer::SignalEvent"); + ASSIGN_OR_RETURN(auto* device_event, CastEvent(event)); + syms()->vkCmdSetEvent(command_buffer_, device_event->handle(), + ConvertPipelineStageFlags(source_stage_mask)); + return OkStatus(); +} + +Status DirectCommandBuffer::ResetEvent( + Event* event, ExecutionStageBitfield source_stage_mask) { + IREE_TRACE_SCOPE0("DirectCommandBuffer::ResetEvent"); + ASSIGN_OR_RETURN(auto* device_event, CastEvent(event)); + syms()->vkCmdResetEvent(command_buffer_, device_event->handle(), + ConvertPipelineStageFlags(source_stage_mask)); + return OkStatus(); +} + +Status DirectCommandBuffer::WaitEvents( + absl::Span events, ExecutionStageBitfield source_stage_mask, + ExecutionStageBitfield target_stage_mask, + absl::Span memory_barriers, + absl::Span buffer_barriers) { + IREE_TRACE_SCOPE0("DirectCommandBuffer::WaitEvents"); + + absl::InlinedVector event_handles(events.size()); + for (int i = 0; i < events.size(); ++i) { + ASSIGN_OR_RETURN(auto* device_event, CastEvent(events[i])); + event_handles[i] = device_event->handle(); + } + + absl::InlinedVector memory_barrier_infos( + memory_barriers.size()); + for (int i = 0; i < memory_barriers.size(); ++i) { + const auto& memory_barrier = memory_barriers[i]; + auto& info = memory_barrier_infos[i]; + info.sType = VK_STRUCTURE_TYPE_MEMORY_BARRIER; + info.pNext = nullptr; + info.srcAccessMask = ConvertAccessMask(memory_barrier.source_scope); + info.dstAccessMask = ConvertAccessMask(memory_barrier.target_scope); + } + + absl::InlinedVector buffer_barrier_infos( + buffer_barriers.size()); + for (int i = 0; i < buffer_barriers.size(); ++i) { + const auto& buffer_barrier = buffer_barriers[i]; + auto& info = buffer_barrier_infos[i]; + info.sType = VK_STRUCTURE_TYPE_BUFFER_MEMORY_BARRIER; + info.pNext = nullptr; + info.srcAccessMask = ConvertAccessMask(buffer_barrier.source_scope); + info.dstAccessMask = ConvertAccessMask(buffer_barrier.target_scope); + info.srcQueueFamilyIndex = VK_QUEUE_FAMILY_IGNORED; + info.dstQueueFamilyIndex = VK_QUEUE_FAMILY_IGNORED; + ASSIGN_OR_RETURN(auto* device_buffer, CastBuffer(buffer_barrier.buffer)); + info.buffer = device_buffer->handle(); + info.offset = buffer_barrier.offset; + info.size = buffer_barrier.length; + } + + syms()->vkCmdWaitEvents( + command_buffer_, event_handles.size(), event_handles.data(), + ConvertPipelineStageFlags(source_stage_mask), + ConvertPipelineStageFlags(target_stage_mask), memory_barrier_infos.size(), + memory_barrier_infos.data(), buffer_barrier_infos.size(), + buffer_barrier_infos.data(), 0, nullptr); + return OkStatus(); +} + +Status DirectCommandBuffer::FillBuffer(Buffer* target_buffer, + device_size_t target_offset, + device_size_t length, + const void* pattern, + size_t pattern_length) { + IREE_TRACE_SCOPE0("DirectCommandBuffer::FillBuffer"); + ASSIGN_OR_RETURN(auto* target_device_buffer, CastBuffer(target_buffer)); + + // Note that fill only accepts 4-byte aligned values so we need to splat out + // our variable-length pattern. + target_offset += target_buffer->byte_offset(); + uint32_t dword_pattern = SplatPattern(pattern, pattern_length); + syms()->vkCmdFillBuffer(command_buffer_, target_device_buffer->handle(), + target_offset, length, dword_pattern); + + return OkStatus(); +} + +Status DirectCommandBuffer::DiscardBuffer(Buffer* buffer) { + IREE_TRACE_SCOPE0("DirectCommandBuffer::DiscardBuffer"); + // NOTE: we could use this to prevent queue family transitions. + return OkStatus(); +} + +Status DirectCommandBuffer::UpdateBuffer(const void* source_buffer, + device_size_t source_offset, + Buffer* target_buffer, + device_size_t target_offset, + device_size_t length) { + IREE_TRACE_SCOPE0("DirectCommandBuffer::UpdateBuffer"); + ASSIGN_OR_RETURN(auto* target_device_buffer, CastBuffer(target_buffer)); + + // Vulkan only allows updates of <= 65536 because you really, really, really + // shouldn't do large updates like this (as it wastes command buffer space and + // may be slower than just using write-through mapped memory). The + // recommendation in the spec for larger updates is to split the single update + // into multiple updates over the entire desired range. + const auto* source_buffer_ptr = static_cast(source_buffer); + target_offset += target_buffer->byte_offset(); + while (length > 0) { + device_size_t chunk_length = + std::min(static_cast(65536u), length); + syms()->vkCmdUpdateBuffer(command_buffer_, target_device_buffer->handle(), + target_offset, chunk_length, source_buffer_ptr); + source_buffer_ptr += chunk_length; + target_offset += chunk_length; + length -= chunk_length; + } + + return OkStatus(); +} + +Status DirectCommandBuffer::CopyBuffer(Buffer* source_buffer, + device_size_t source_offset, + Buffer* target_buffer, + device_size_t target_offset, + device_size_t length) { + IREE_TRACE_SCOPE0("DirectCommandBuffer::CopyBuffer"); + ASSIGN_OR_RETURN(auto* source_device_buffer, CastBuffer(source_buffer)); + ASSIGN_OR_RETURN(auto* target_device_buffer, CastBuffer(target_buffer)); + + VkBufferCopy region; + region.srcOffset = source_buffer->byte_offset() + source_offset; + region.dstOffset = target_buffer->byte_offset() + target_offset; + region.size = length; + syms()->vkCmdCopyBuffer(command_buffer_, source_device_buffer->handle(), + target_device_buffer->handle(), 1, ®ion); + + return OkStatus(); +} + +Status DirectCommandBuffer::UpdateAndBindDescriptorSet( + PipelineExecutable* executable, absl::Span bindings) { + absl::InlinedVector buffer_infos; + buffer_infos.resize(bindings.size()); + for (int i = 0; i < bindings.size(); ++i) { + ASSIGN_OR_RETURN(auto buffer, CastBuffer(bindings[i].buffer)); + buffer_infos[i].buffer = buffer->handle(); + // TODO(benvanik): properly subrange (add to BufferBinding). + buffer_infos[i].offset = bindings[i].buffer->byte_offset(); + buffer_infos[i].range = bindings[i].buffer->byte_length(); + } + + const auto& descriptor_sets = executable->descriptor_sets(); + absl::InlinedVector write_infos; + write_infos.resize(bindings.size()); + for (int i = 0; i < bindings.size(); ++i) { + ASSIGN_OR_RETURN(auto buffer, CastBuffer(bindings[i].buffer)); + VkDescriptorBufferInfo buffer_info; + buffer_info.buffer = buffer->handle(); + // TODO(benvanik): properly subrange (add to BufferBinding). + buffer_info.offset = bindings[i].buffer->byte_offset(); + buffer_info.range = bindings[i].buffer->byte_length(); + auto& write_info = write_infos[i]; + write_info.sType = VK_STRUCTURE_TYPE_WRITE_DESCRIPTOR_SET; + write_info.pNext = nullptr; + write_info.dstSet = VK_NULL_HANDLE; + write_info.dstBinding = descriptor_sets.buffer_binding_set_map[i]; + write_info.dstArrayElement = 0; + write_info.descriptorCount = 1; + write_info.descriptorType = VK_DESCRIPTOR_TYPE_STORAGE_BUFFER; + write_info.pImageInfo = nullptr; + write_info.pBufferInfo = &buffer_infos[i]; + write_info.pTexelBufferView = nullptr; + } + + if (command_pool_->logical_device()->enabled_extensions().push_descriptors) { + // Fast path using push descriptors. These are pooled internally by the + // command buffer and prevent the need for our own pooling mechanisms. + syms()->vkCmdPushDescriptorSetKHR( + command_buffer_, VK_PIPELINE_BIND_POINT_COMPUTE, + executable->pipeline_layout(), descriptor_sets.buffer_binding_set, + write_infos.size(), write_infos.data()); + } else { + // TODO(benvanik): allocate from pool and update. + return UnimplementedErrorBuilder(ABSL_LOC) + << "Non-push descriptor set path not yet implemented"; + } + + return OkStatus(); +} + +Status DirectCommandBuffer::Dispatch(const DispatchRequest& dispatch_request) { + IREE_TRACE_SCOPE0("DirectCommandBuffer::Dispatch"); + + // Get the compiled and linked pipeline for the specified entry point and + // bind it to the command buffer. + auto* executable = + static_cast(dispatch_request.executable); + ASSIGN_OR_RETURN(VkPipeline pipeline, executable->GetPipelineForEntryPoint( + dispatch_request.entry_point)); + syms()->vkCmdBindPipeline(command_buffer_, VK_PIPELINE_BIND_POINT_COMPUTE, + pipeline); + + // Either allocate, update, and bind a descriptor set or use push descriptor + // sets to use the command buffer pool when supported. + RETURN_IF_ERROR( + UpdateAndBindDescriptorSet(executable, dispatch_request.bindings)); + + // TODO(benvanik): not this, /obviously/. Replace with semantic tags or just + // get SPIR-V roundtripping what we need to do this in proper IR. The infra + // for dynamic shapes is another route, with us being able to just pass shapes + // in via dynamically updated uniform buffers. + if (executable->is_matmul()) { + struct ABSL_ATTRIBUTE_PACKED { + int32_t dims[4]; + } shapes[3]; + for (int i = 0; i < 3; ++i) { + const auto& shape = dispatch_request.bindings[i].shape; + if (shape.size() == 3) { + shapes[i].dims[0] = shape[0]; + shapes[i].dims[1] = shape[1]; + shapes[i].dims[2] = shape[2]; + shapes[i].dims[3] = 1; + } else { + shapes[i].dims[0] = 1; + shapes[i].dims[1] = shape[0]; + shapes[i].dims[2] = shape[1]; + shapes[i].dims[3] = 1; + } + } + syms()->vkCmdPushConstants(command_buffer_, executable->pipeline_layout(), + VK_SHADER_STAGE_COMPUTE_BIT, 0, sizeof(shapes), + &shapes); + } + + // TODO(benvanik): divide workload by caps and issue multiple dispatches. + // TODO(benvanik): track local workgroup/subgroup size and divide into groups. + if (dispatch_request.workload_buffer) { + return UnimplementedErrorBuilder(ABSL_LOC) + << "Dynamic dispatches not yet implemented"; + } + uint32_t group_count_x = dispatch_request.workload[0]; + uint32_t group_count_y = dispatch_request.workload[1]; + uint32_t group_count_z = dispatch_request.workload[2]; + + if (executable->is_matmul()) { + group_count_x = (group_count_x + 16 - 1) / 16; + group_count_y = (group_count_y + 16 - 1) / 16; + group_count_z = 1; + } + + syms()->vkCmdDispatch(command_buffer_, group_count_x, group_count_y, + group_count_z); + + return OkStatus(); +} + +} // namespace vulkan +} // namespace hal +} // namespace iree diff --git a/hal/vulkan/direct_command_buffer.h b/hal/vulkan/direct_command_buffer.h new file mode 100644 index 000000000000..69978687d815 --- /dev/null +++ b/hal/vulkan/direct_command_buffer.h @@ -0,0 +1,95 @@ +// Copyright 2019 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#ifndef THIRD_PARTY_MLIR_EDGE_IREE_HAL_VULKAN_DIRECT_COMMAND_BUFFER_H_ +#define THIRD_PARTY_MLIR_EDGE_IREE_HAL_VULKAN_DIRECT_COMMAND_BUFFER_H_ + +#include + +#include "third_party/mlir_edge/iree/hal/command_buffer.h" +#include "third_party/mlir_edge/iree/hal/vulkan/dynamic_symbols.h" +#include "third_party/mlir_edge/iree/hal/vulkan/handle_util.h" +#include "third_party/mlir_edge/iree/hal/vulkan/native_event.h" +#include "third_party/mlir_edge/iree/hal/vulkan/pipeline_executable.h" +#include "third_party/mlir_edge/iree/hal/vulkan/vma_buffer.h" + +namespace iree { +namespace hal { +namespace vulkan { + +// Command buffer implementation that directly maps to VkCommandBuffer. +// This records the commands on the calling thread without additional threading +// indirection. +class DirectCommandBuffer final : public CommandBuffer { + public: + DirectCommandBuffer(Allocator* allocator, CommandBufferModeBitfield mode, + CommandCategoryBitfield command_categories, + const ref_ptr& command_pool, + VkCommandBuffer command_buffer); + ~DirectCommandBuffer() override; + + VkCommandBuffer handle() const { return command_buffer_; } + + bool is_recording() const override { return is_recording_; } + + Status Begin() override; + Status End() override; + + Status ExecutionBarrier( + ExecutionStageBitfield source_stage_mask, + ExecutionStageBitfield target_stage_mask, + absl::Span memory_barriers, + absl::Span buffer_barriers) override; + Status SignalEvent(Event* event, + ExecutionStageBitfield source_stage_mask) override; + Status ResetEvent(Event* event, + ExecutionStageBitfield source_stage_mask) override; + Status WaitEvents(absl::Span events, + ExecutionStageBitfield source_stage_mask, + ExecutionStageBitfield target_stage_mask, + absl::Span memory_barriers, + absl::Span buffer_barriers) override; + + Status FillBuffer(Buffer* target_buffer, device_size_t target_offset, + device_size_t length, const void* pattern, + size_t pattern_length) override; + Status DiscardBuffer(Buffer* buffer) override; + Status UpdateBuffer(const void* source_buffer, device_size_t source_offset, + Buffer* target_buffer, device_size_t target_offset, + device_size_t length) override; + Status CopyBuffer(Buffer* source_buffer, device_size_t source_offset, + Buffer* target_buffer, device_size_t target_offset, + device_size_t length) override; + + Status Dispatch(const DispatchRequest& dispatch_request) override; + + private: + const ref_ptr& syms() const { return command_pool_->syms(); } + + StatusOr CastEvent(Event* event) const; + StatusOr CastBuffer(Buffer* buffer) const; + + Status UpdateAndBindDescriptorSet(PipelineExecutable* executable, + absl::Span bindings); + + bool is_recording_ = false; + ref_ptr command_pool_; + VkCommandBuffer command_buffer_; +}; + +} // namespace vulkan +} // namespace hal +} // namespace iree + +#endif // THIRD_PARTY_MLIR_EDGE_IREE_HAL_VULKAN_DIRECT_COMMAND_BUFFER_H_ diff --git a/hal/vulkan/direct_command_queue.cc b/hal/vulkan/direct_command_queue.cc new file mode 100644 index 000000000000..08ca389907cd --- /dev/null +++ b/hal/vulkan/direct_command_queue.cc @@ -0,0 +1,207 @@ +// Copyright 2019 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "third_party/mlir_edge/iree/hal/vulkan/direct_command_queue.h" + +#include + +#include "third_party/absl/time/clock.h" +#include "third_party/absl/time/time.h" +#include "third_party/absl/types/source_location.h" +#include "third_party/mlir_edge/iree/base/memory.h" +#include "third_party/mlir_edge/iree/base/status.h" +#include "third_party/mlir_edge/iree/base/tracing.h" +#include "third_party/mlir_edge/iree/hal/vulkan/direct_command_buffer.h" +#include "third_party/mlir_edge/iree/hal/vulkan/legacy_fence.h" +#include "third_party/mlir_edge/iree/hal/vulkan/native_binary_semaphore.h" +#include "third_party/mlir_edge/iree/hal/vulkan/status_util.h" + +namespace iree { +namespace hal { +namespace vulkan { + +DirectCommandQueue::DirectCommandQueue( + std::string name, CommandCategoryBitfield supported_categories, + const ref_ptr& logical_device, VkQueue queue) + : CommandQueue(std::move(name), supported_categories), + logical_device_(add_ref(logical_device)), + queue_(queue) {} + +DirectCommandQueue::~DirectCommandQueue() { + IREE_TRACE_SCOPE0("DirectCommandQueue::dtor"); + absl::MutexLock lock(&queue_mutex_); + syms()->vkQueueWaitIdle(queue_); +} + +Status DirectCommandQueue::TranslateBatchInfo(const SubmissionBatch& batch, + VkSubmitInfo* submit_info, + Arena* arena) { + // TODO(benvanik): see if we can go to finer-grained stages. + // For example, if this was just queue ownership transfers then we can use + // the pseudo-stage of VK_PIPELINE_STAGE_BOTTOM_OF_PIPE_BIT. + VkPipelineStageFlags dst_stage_mask = + VK_PIPELINE_STAGE_TRANSFER_BIT | VK_PIPELINE_STAGE_COMPUTE_SHADER_BIT; + + auto wait_semaphore_handles = + arena->AllocateSpan(batch.wait_semaphores.size()); + auto wait_dst_stage_masks = + arena->AllocateSpan(batch.wait_semaphores.size()); + for (int i = 0; i < batch.wait_semaphores.size(); ++i) { + const auto& semaphore_value = batch.wait_semaphores[i]; + if (semaphore_value.index() == 0) { + const auto& binary_semaphore = + static_cast(absl::get<0>(semaphore_value)); + wait_semaphore_handles[i] = binary_semaphore->handle(); + } else { + // TODO(b/140141417): implement timeline semaphores. + return UnimplementedErrorBuilder(ABSL_LOC) + << "Timeline semaphores not yet implemented"; + } + wait_dst_stage_masks[i] = dst_stage_mask; + } + + auto signal_semaphore_handles = + arena->AllocateSpan(batch.signal_semaphores.size()); + for (int i = 0; i < batch.signal_semaphores.size(); ++i) { + const auto& semaphore_value = batch.signal_semaphores[i]; + if (semaphore_value.index() == 0) { + const auto& binary_semaphore = + static_cast(absl::get<0>(semaphore_value)); + signal_semaphore_handles[i] = binary_semaphore->handle(); + } else { + // TODO(b/140141417): implement timeline semaphores. + return UnimplementedErrorBuilder(ABSL_LOC) + << "Timeline semaphores not yet implemented"; + } + } + + auto command_buffer_handles = + arena->AllocateSpan(batch.command_buffers.size()); + for (int i = 0; i < batch.command_buffers.size(); ++i) { + const auto& command_buffer = batch.command_buffers[i]; + auto* direct_command_buffer = + static_cast(command_buffer->impl()); + command_buffer_handles[i] = direct_command_buffer->handle(); + } + + submit_info->sType = VK_STRUCTURE_TYPE_SUBMIT_INFO; + submit_info->pNext = nullptr; + submit_info->waitSemaphoreCount = wait_semaphore_handles.size(); + submit_info->pWaitSemaphores = wait_semaphore_handles.data(); + submit_info->pWaitDstStageMask = wait_dst_stage_masks.data(); + submit_info->commandBufferCount = command_buffer_handles.size(); + submit_info->pCommandBuffers = command_buffer_handles.data(); + submit_info->signalSemaphoreCount = signal_semaphore_handles.size(); + submit_info->pSignalSemaphores = signal_semaphore_handles.data(); + + return OkStatus(); +} + +Status DirectCommandQueue::Submit(absl::Span batches, + FenceValue fence) { + IREE_TRACE_SCOPE0("DirectCommandQueue::Submit"); + + // Map the submission batches to VkSubmitInfos. + // Note that we must keep all arrays referenced alive until submission + // completes and since there are a bunch of them we use an arena. + Arena arena(4 * 1024); + auto submit_infos = arena.AllocateSpan(batches.size()); + for (int i = 0; i < batches.size(); ++i) { + RETURN_IF_ERROR(TranslateBatchInfo(batches[i], &submit_infos[i], &arena)); + } + + // TODO(b/140141417): implement timeline semaphore fences and switch here. + auto legacy_fence = reinterpret_cast(fence.first); + ASSIGN_OR_RETURN(VkFence fence_handle, + legacy_fence->AcquireSignalFence(fence.second)); + + { + absl::MutexLock lock(&queue_mutex_); + VK_RETURN_IF_ERROR(syms()->vkQueueSubmit( + queue_, submit_infos.size(), submit_infos.data(), fence_handle)); + } + + return OkStatus(); +} + +Status DirectCommandQueue::Flush() { + IREE_TRACE_SCOPE0("DirectCommandQueue::Flush"); + // Nothing to do here as submit is not async. + return OkStatus(); +} + +Status DirectCommandQueue::WaitIdle(absl::Time deadline) { + if (deadline == absl::InfiniteFuture()) { + // Fast path for using vkQueueWaitIdle, which is usually cheaper (as it + // requires fewer calls into the driver). + IREE_TRACE_SCOPE0("DirectCommandQueue::WaitIdle#vkQueueWaitIdle"); + absl::MutexLock lock(&queue_mutex_); + VK_RETURN_IF_ERROR(syms()->vkQueueWaitIdle(queue_)); + return OkStatus(); + } + + IREE_TRACE_SCOPE0("DirectCommandQueue::WaitIdle#Fence"); + + // Create a new fence just for this wait. This keeps us thread-safe as the + // behavior of wait+reset is racey. + VkFenceCreateInfo create_info; + create_info.sType = VK_STRUCTURE_TYPE_FENCE_CREATE_INFO; + create_info.pNext = nullptr; + create_info.flags = 0; + VkFence fence = VK_NULL_HANDLE; + VK_RETURN_IF_ERROR(syms()->vkCreateFence( + *logical_device_, &create_info, logical_device_->allocator(), &fence)); + auto fence_cleanup = MakeCleanup([this, fence]() { + syms()->vkDestroyFence(*logical_device_, fence, + logical_device_->allocator()); + }); + + uint64_t timeout; + if (deadline == absl::InfinitePast()) { + // Do not wait. + timeout = 0; + } else if (deadline == absl::InfiniteFuture()) { + // Wait forever. + timeout = UINT64_MAX; + } else { + // Convert to relative time in nanoseconds. + // The implementation may not wait with this granularity (like, by 10000x). + absl::Time now = absl::Now(); + if (deadline < now) { + return DeadlineExceededErrorBuilder(ABSL_LOC) << "Deadline in the past"; + } + timeout = static_cast(absl::ToInt64Nanoseconds(deadline - now)); + } + + { + absl::MutexLock lock(&queue_mutex_); + VK_RETURN_IF_ERROR(syms()->vkQueueSubmit(queue_, 0, nullptr, fence)); + } + + VkResult result = + syms()->vkWaitForFences(*logical_device_, 1, &fence, VK_TRUE, timeout); + switch (result) { + case VK_SUCCESS: + return OkStatus(); + case VK_TIMEOUT: + return DeadlineExceededErrorBuilder(ABSL_LOC) + << "Deadline exceeded waiting for idle"; + default: + return VkResultToStatus(result); + } +} + +} // namespace vulkan +} // namespace hal +} // namespace iree diff --git a/hal/vulkan/direct_command_queue.h b/hal/vulkan/direct_command_queue.h new file mode 100644 index 000000000000..25d1908b52e1 --- /dev/null +++ b/hal/vulkan/direct_command_queue.h @@ -0,0 +1,71 @@ +// Copyright 2019 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#ifndef THIRD_PARTY_MLIR_EDGE_IREE_HAL_VULKAN_DIRECT_COMMAND_QUEUE_H_ +#define THIRD_PARTY_MLIR_EDGE_IREE_HAL_VULKAN_DIRECT_COMMAND_QUEUE_H_ + +#include + +#include +#include + +#include "third_party/absl/base/thread_annotations.h" +#include "third_party/absl/synchronization/mutex.h" +#include "third_party/absl/time/time.h" +#include "third_party/mlir_edge/iree/base/arena.h" +#include "third_party/mlir_edge/iree/base/status.h" +#include "third_party/mlir_edge/iree/hal/command_queue.h" +#include "third_party/mlir_edge/iree/hal/vulkan/dynamic_symbols.h" +#include "third_party/mlir_edge/iree/hal/vulkan/handle_util.h" + +namespace iree { +namespace hal { +namespace vulkan { + +// Command queue implementation directly maps to VkQueue. +class DirectCommandQueue final : public CommandQueue { + public: + DirectCommandQueue(std::string name, + CommandCategoryBitfield supported_categories, + const ref_ptr& logical_device, + VkQueue queue); + ~DirectCommandQueue() override; + + const ref_ptr& syms() const { + return logical_device_->syms(); + } + + Status Submit(absl::Span batches, + FenceValue fence) override; + + Status Flush() override; + + Status WaitIdle(absl::Time deadline) override; + + private: + Status TranslateBatchInfo(const SubmissionBatch& batch, + VkSubmitInfo* submit_info, Arena* arena); + + ref_ptr logical_device_; + + // VkQueue needs to be externally synchronized. + mutable absl::Mutex queue_mutex_; + VkQueue queue_ ABSL_GUARDED_BY(queue_mutex_); +}; + +} // namespace vulkan +} // namespace hal +} // namespace iree + +#endif // THIRD_PARTY_MLIR_EDGE_IREE_HAL_VULKAN_DIRECT_COMMAND_QUEUE_H_ diff --git a/hal/vulkan/dynamic_symbol_tables.h b/hal/vulkan/dynamic_symbol_tables.h new file mode 100644 index 000000000000..e05df3ec94f0 --- /dev/null +++ b/hal/vulkan/dynamic_symbol_tables.h @@ -0,0 +1,489 @@ +// Copyright 2019 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// Use these tables whenever enumerating all functions in the Vulkan API is +// required. In most cases IREE_VULKAN_DYNAMIC_SYMBOL_TABLES is the right +// choice (includes both common and enabled platform-specific functions). +// +// Table macros are designed to take two macros: one for each instance-specific +// function and one for each device-specific function. These macros are also +// passed a requirement flag that enables compile-time exclusion of methods that +// are not used in the binary. If you find yourself getting compilation errors +// on missing methods you probably need to change it in the tables below from +// EXCLUDED to REQUIRED or OPTIONAL. +// +// Define to get instance-specific functions: +// #define INS_PFN(requirement, function_name) +// +// Define to get device-specific functions: +// #define DEV_PFN(requirement, function_name) +// +// requirement is one of REQUIRED, OPTIONAL, or EXCLUDED. + +#ifndef THIRD_PARTY_MLIR_EDGE_IREE_HAL_VULKAN_DYNAMIC_SYMBOL_TABLES_H_ +#define THIRD_PARTY_MLIR_EDGE_IREE_HAL_VULKAN_DYNAMIC_SYMBOL_TABLES_H_ + +namespace iree { +namespace hal { +namespace vulkan { + +#define IREE_VULKAN_DYNAMIC_SYMBOL_COMMON_TABLE(INS_PFN, DEV_PFN) \ + INS_PFN(REQUIRED, vkCreateInstance) \ + INS_PFN(REQUIRED, vkEnumerateInstanceExtensionProperties) \ + INS_PFN(REQUIRED, vkEnumerateInstanceLayerProperties) \ + INS_PFN(REQUIRED, vkEnumerateInstanceVersion) \ + DEV_PFN(REQUIRED, vkBeginCommandBuffer) \ + DEV_PFN(EXCLUDED, vkCmdBeginConditionalRenderingEXT) \ + DEV_PFN(OPTIONAL, vkCmdBeginDebugUtilsLabelEXT) \ + DEV_PFN(EXCLUDED, vkCmdBeginQuery) \ + DEV_PFN(EXCLUDED, vkCmdBeginQueryIndexedEXT) \ + DEV_PFN(EXCLUDED, vkCmdBeginRenderPass) \ + DEV_PFN(EXCLUDED, vkCmdBeginRenderPass2KHR) \ + DEV_PFN(EXCLUDED, vkCmdBeginTransformFeedbackEXT) \ + DEV_PFN(REQUIRED, vkCmdBindDescriptorSets) \ + DEV_PFN(EXCLUDED, vkCmdBindIndexBuffer) \ + DEV_PFN(REQUIRED, vkCmdBindPipeline) \ + DEV_PFN(EXCLUDED, vkCmdBindShadingRateImageNV) \ + DEV_PFN(EXCLUDED, vkCmdBindTransformFeedbackBuffersEXT) \ + DEV_PFN(EXCLUDED, vkCmdBindVertexBuffers) \ + DEV_PFN(EXCLUDED, vkCmdBlitImage) \ + DEV_PFN(EXCLUDED, vkCmdBuildAccelerationStructureNV) \ + DEV_PFN(EXCLUDED, vkCmdClearAttachments) \ + DEV_PFN(EXCLUDED, vkCmdClearColorImage) \ + DEV_PFN(EXCLUDED, vkCmdClearDepthStencilImage) \ + DEV_PFN(EXCLUDED, vkCmdCopyAccelerationStructureNV) \ + DEV_PFN(REQUIRED, vkCmdCopyBuffer) \ + DEV_PFN(EXCLUDED, vkCmdCopyBufferToImage) \ + DEV_PFN(EXCLUDED, vkCmdCopyImage) \ + DEV_PFN(EXCLUDED, vkCmdCopyImageToBuffer) \ + DEV_PFN(EXCLUDED, vkCmdCopyQueryPoolResults) \ + DEV_PFN(EXCLUDED, vkCmdDebugMarkerBeginEXT) \ + DEV_PFN(EXCLUDED, vkCmdDebugMarkerEndEXT) \ + DEV_PFN(EXCLUDED, vkCmdDebugMarkerInsertEXT) \ + DEV_PFN(REQUIRED, vkCmdDispatch) \ + DEV_PFN(REQUIRED, vkCmdDispatchBase) \ + DEV_PFN(EXCLUDED, vkCmdDispatchBaseKHR) \ + DEV_PFN(REQUIRED, vkCmdDispatchIndirect) \ + DEV_PFN(EXCLUDED, vkCmdDraw) \ + DEV_PFN(EXCLUDED, vkCmdDrawIndexed) \ + DEV_PFN(EXCLUDED, vkCmdDrawIndexedIndirect) \ + DEV_PFN(EXCLUDED, vkCmdDrawIndexedIndirectCountAMD) \ + DEV_PFN(EXCLUDED, vkCmdDrawIndexedIndirectCountKHR) \ + DEV_PFN(EXCLUDED, vkCmdDrawIndirect) \ + DEV_PFN(EXCLUDED, vkCmdDrawIndirectByteCountEXT) \ + DEV_PFN(EXCLUDED, vkCmdDrawIndirectCountAMD) \ + DEV_PFN(EXCLUDED, vkCmdDrawIndirectCountKHR) \ + DEV_PFN(EXCLUDED, vkCmdDrawMeshTasksIndirectCountNV) \ + DEV_PFN(EXCLUDED, vkCmdDrawMeshTasksIndirectNV) \ + DEV_PFN(EXCLUDED, vkCmdDrawMeshTasksNV) \ + DEV_PFN(EXCLUDED, vkCmdEndConditionalRenderingEXT) \ + DEV_PFN(OPTIONAL, vkCmdEndDebugUtilsLabelEXT) \ + DEV_PFN(EXCLUDED, vkCmdEndQuery) \ + DEV_PFN(EXCLUDED, vkCmdEndQueryIndexedEXT) \ + DEV_PFN(EXCLUDED, vkCmdEndRenderPass) \ + DEV_PFN(EXCLUDED, vkCmdEndRenderPass2KHR) \ + DEV_PFN(EXCLUDED, vkCmdEndTransformFeedbackEXT) \ + DEV_PFN(REQUIRED, vkCmdExecuteCommands) \ + DEV_PFN(REQUIRED, vkCmdFillBuffer) \ + DEV_PFN(OPTIONAL, vkCmdInsertDebugUtilsLabelEXT) \ + DEV_PFN(EXCLUDED, vkCmdNextSubpass) \ + DEV_PFN(EXCLUDED, vkCmdNextSubpass2KHR) \ + DEV_PFN(REQUIRED, vkCmdPipelineBarrier) \ + DEV_PFN(EXCLUDED, vkCmdProcessCommandsNVX) \ + DEV_PFN(REQUIRED, vkCmdPushConstants) \ + DEV_PFN(OPTIONAL, vkCmdPushDescriptorSetKHR) \ + DEV_PFN(EXCLUDED, vkCmdPushDescriptorSetWithTemplateKHR) \ + DEV_PFN(EXCLUDED, vkCmdReserveSpaceForCommandsNVX) \ + DEV_PFN(REQUIRED, vkCmdResetEvent) \ + DEV_PFN(EXCLUDED, vkCmdResetQueryPool) \ + DEV_PFN(EXCLUDED, vkCmdResolveImage) \ + DEV_PFN(EXCLUDED, vkCmdSetBlendConstants) \ + DEV_PFN(EXCLUDED, vkCmdSetCheckpointNV) \ + DEV_PFN(EXCLUDED, vkCmdSetCoarseSampleOrderNV) \ + DEV_PFN(EXCLUDED, vkCmdSetDepthBias) \ + DEV_PFN(EXCLUDED, vkCmdSetDepthBounds) \ + DEV_PFN(EXCLUDED, vkCmdSetDeviceMask) \ + DEV_PFN(EXCLUDED, vkCmdSetDeviceMaskKHR) \ + DEV_PFN(EXCLUDED, vkCmdSetDiscardRectangleEXT) \ + DEV_PFN(REQUIRED, vkCmdSetEvent) \ + DEV_PFN(EXCLUDED, vkCmdSetExclusiveScissorNV) \ + DEV_PFN(EXCLUDED, vkCmdSetLineWidth) \ + DEV_PFN(EXCLUDED, vkCmdSetSampleLocationsEXT) \ + DEV_PFN(EXCLUDED, vkCmdSetScissor) \ + DEV_PFN(EXCLUDED, vkCmdSetStencilCompareMask) \ + DEV_PFN(EXCLUDED, vkCmdSetStencilReference) \ + DEV_PFN(EXCLUDED, vkCmdSetStencilWriteMask) \ + DEV_PFN(EXCLUDED, vkCmdSetViewport) \ + DEV_PFN(EXCLUDED, vkCmdSetViewportShadingRatePaletteNV) \ + DEV_PFN(EXCLUDED, vkCmdSetViewportWScalingNV) \ + DEV_PFN(EXCLUDED, vkCmdTraceRaysNV) \ + DEV_PFN(REQUIRED, vkCmdUpdateBuffer) \ + DEV_PFN(REQUIRED, vkCmdWaitEvents) \ + DEV_PFN(EXCLUDED, vkCmdWriteAccelerationStructuresPropertiesNV) \ + DEV_PFN(EXCLUDED, vkCmdWriteBufferMarkerAMD) \ + DEV_PFN(REQUIRED, vkCmdWriteTimestamp) \ + DEV_PFN(REQUIRED, vkEndCommandBuffer) \ + DEV_PFN(EXCLUDED, vkResetCommandBuffer) \ + DEV_PFN(EXCLUDED, vkAcquireNextImage2KHR) \ + DEV_PFN(EXCLUDED, vkAcquireNextImageKHR) \ + DEV_PFN(REQUIRED, vkAllocateCommandBuffers) \ + DEV_PFN(REQUIRED, vkAllocateDescriptorSets) \ + DEV_PFN(REQUIRED, vkAllocateMemory) \ + DEV_PFN(EXCLUDED, vkBindAccelerationStructureMemoryNV) \ + DEV_PFN(REQUIRED, vkBindBufferMemory) \ + DEV_PFN(EXCLUDED, vkBindBufferMemory2) \ + DEV_PFN(EXCLUDED, vkBindBufferMemory2KHR) \ + DEV_PFN(REQUIRED, vkBindImageMemory) \ + DEV_PFN(EXCLUDED, vkBindImageMemory2) \ + DEV_PFN(EXCLUDED, vkBindImageMemory2KHR) \ + DEV_PFN(EXCLUDED, vkCompileDeferredNV) \ + DEV_PFN(EXCLUDED, vkCreateAccelerationStructureNV) \ + DEV_PFN(REQUIRED, vkCreateBuffer) \ + DEV_PFN(REQUIRED, vkCreateBufferView) \ + DEV_PFN(REQUIRED, vkCreateCommandPool) \ + DEV_PFN(REQUIRED, vkCreateComputePipelines) \ + DEV_PFN(REQUIRED, vkCreateDescriptorPool) \ + DEV_PFN(REQUIRED, vkCreateDescriptorSetLayout) \ + DEV_PFN(EXCLUDED, vkCreateDescriptorUpdateTemplate) \ + DEV_PFN(EXCLUDED, vkCreateDescriptorUpdateTemplateKHR) \ + DEV_PFN(REQUIRED, vkCreateEvent) \ + DEV_PFN(REQUIRED, vkCreateFence) \ + DEV_PFN(EXCLUDED, vkCreateFramebuffer) \ + DEV_PFN(EXCLUDED, vkCreateGraphicsPipelines) \ + DEV_PFN(REQUIRED, vkCreateImage) \ + DEV_PFN(EXCLUDED, vkCreateImageView) \ + DEV_PFN(EXCLUDED, vkCreateIndirectCommandsLayoutNVX) \ + DEV_PFN(EXCLUDED, vkCreateObjectTableNVX) \ + DEV_PFN(REQUIRED, vkCreatePipelineCache) \ + DEV_PFN(REQUIRED, vkCreatePipelineLayout) \ + DEV_PFN(EXCLUDED, vkCreateQueryPool) \ + DEV_PFN(EXCLUDED, vkCreateRayTracingPipelinesNV) \ + DEV_PFN(EXCLUDED, vkCreateRenderPass) \ + DEV_PFN(EXCLUDED, vkCreateRenderPass2KHR) \ + DEV_PFN(EXCLUDED, vkCreateSampler) \ + DEV_PFN(EXCLUDED, vkCreateSamplerYcbcrConversion) \ + DEV_PFN(EXCLUDED, vkCreateSamplerYcbcrConversionKHR) \ + DEV_PFN(REQUIRED, vkCreateSemaphore) \ + DEV_PFN(REQUIRED, vkCreateShaderModule) \ + DEV_PFN(EXCLUDED, vkCreateSharedSwapchainsKHR) \ + DEV_PFN(EXCLUDED, vkCreateSwapchainKHR) \ + DEV_PFN(EXCLUDED, vkCreateValidationCacheEXT) \ + DEV_PFN(EXCLUDED, vkDebugMarkerSetObjectNameEXT) \ + DEV_PFN(EXCLUDED, vkDebugMarkerSetObjectTagEXT) \ + DEV_PFN(EXCLUDED, vkDestroyAccelerationStructureNV) \ + DEV_PFN(REQUIRED, vkDestroyBuffer) \ + DEV_PFN(REQUIRED, vkDestroyBufferView) \ + DEV_PFN(REQUIRED, vkDestroyCommandPool) \ + DEV_PFN(REQUIRED, vkDestroyDescriptorPool) \ + DEV_PFN(REQUIRED, vkDestroyDescriptorSetLayout) \ + DEV_PFN(EXCLUDED, vkDestroyDescriptorUpdateTemplate) \ + DEV_PFN(EXCLUDED, vkDestroyDescriptorUpdateTemplateKHR) \ + DEV_PFN(REQUIRED, vkDestroyDevice) \ + DEV_PFN(REQUIRED, vkDestroyEvent) \ + DEV_PFN(REQUIRED, vkDestroyFence) \ + DEV_PFN(EXCLUDED, vkDestroyFramebuffer) \ + DEV_PFN(REQUIRED, vkDestroyImage) \ + DEV_PFN(EXCLUDED, vkDestroyImageView) \ + DEV_PFN(EXCLUDED, vkDestroyIndirectCommandsLayoutNVX) \ + DEV_PFN(EXCLUDED, vkDestroyObjectTableNVX) \ + DEV_PFN(REQUIRED, vkDestroyPipeline) \ + DEV_PFN(REQUIRED, vkDestroyPipelineCache) \ + DEV_PFN(REQUIRED, vkDestroyPipelineLayout) \ + DEV_PFN(EXCLUDED, vkDestroyQueryPool) \ + DEV_PFN(EXCLUDED, vkDestroyRenderPass) \ + DEV_PFN(EXCLUDED, vkDestroySampler) \ + DEV_PFN(EXCLUDED, vkDestroySamplerYcbcrConversion) \ + DEV_PFN(EXCLUDED, vkDestroySamplerYcbcrConversionKHR) \ + DEV_PFN(REQUIRED, vkDestroySemaphore) \ + DEV_PFN(REQUIRED, vkDestroyShaderModule) \ + DEV_PFN(EXCLUDED, vkDestroySwapchainKHR) \ + DEV_PFN(EXCLUDED, vkDestroyValidationCacheEXT) \ + DEV_PFN(REQUIRED, vkDeviceWaitIdle) \ + DEV_PFN(EXCLUDED, vkDisplayPowerControlEXT) \ + DEV_PFN(REQUIRED, vkFlushMappedMemoryRanges) \ + DEV_PFN(REQUIRED, vkFreeCommandBuffers) \ + DEV_PFN(REQUIRED, vkFreeDescriptorSets) \ + DEV_PFN(REQUIRED, vkFreeMemory) \ + DEV_PFN(EXCLUDED, vkGetAccelerationStructureHandleNV) \ + DEV_PFN(EXCLUDED, vkGetAccelerationStructureMemoryRequirementsNV) \ + DEV_PFN(EXCLUDED, vkGetBufferDeviceAddressEXT) \ + DEV_PFN(REQUIRED, vkGetBufferMemoryRequirements) \ + DEV_PFN(EXCLUDED, vkGetBufferMemoryRequirements2) \ + DEV_PFN(EXCLUDED, vkGetBufferMemoryRequirements2KHR) \ + DEV_PFN(EXCLUDED, vkGetCalibratedTimestampsEXT) \ + DEV_PFN(REQUIRED, vkGetDescriptorSetLayoutSupport) \ + DEV_PFN(EXCLUDED, vkGetDescriptorSetLayoutSupportKHR) \ + DEV_PFN(EXCLUDED, vkGetDeviceGroupPeerMemoryFeatures) \ + DEV_PFN(EXCLUDED, vkGetDeviceGroupPeerMemoryFeaturesKHR) \ + DEV_PFN(EXCLUDED, vkGetDeviceGroupPresentCapabilitiesKHR) \ + DEV_PFN(EXCLUDED, vkGetDeviceGroupSurfacePresentModesKHR) \ + DEV_PFN(EXCLUDED, vkGetDeviceMemoryCommitment) \ + DEV_PFN(REQUIRED, vkGetDeviceQueue) \ + DEV_PFN(EXCLUDED, vkGetDeviceQueue2) \ + DEV_PFN(REQUIRED, vkGetEventStatus) \ + DEV_PFN(OPTIONAL, vkGetFenceFdKHR) \ + DEV_PFN(REQUIRED, vkGetFenceStatus) \ + DEV_PFN(EXCLUDED, vkGetImageDrmFormatModifierPropertiesEXT) \ + DEV_PFN(REQUIRED, vkGetImageMemoryRequirements) \ + DEV_PFN(EXCLUDED, vkGetImageMemoryRequirements2) \ + DEV_PFN(EXCLUDED, vkGetImageMemoryRequirements2KHR) \ + DEV_PFN(EXCLUDED, vkGetImageSparseMemoryRequirements) \ + DEV_PFN(EXCLUDED, vkGetImageSparseMemoryRequirements2) \ + DEV_PFN(EXCLUDED, vkGetImageSparseMemoryRequirements2KHR) \ + DEV_PFN(EXCLUDED, vkGetImageSubresourceLayout) \ + DEV_PFN(EXCLUDED, vkGetImageViewHandleNVX) \ + DEV_PFN(EXCLUDED, vkGetMemoryFdKHR) \ + DEV_PFN(EXCLUDED, vkGetMemoryFdPropertiesKHR) \ + DEV_PFN(EXCLUDED, vkGetMemoryHostPointerPropertiesEXT) \ + DEV_PFN(EXCLUDED, vkGetPastPresentationTimingGOOGLE) \ + DEV_PFN(REQUIRED, vkGetPipelineCacheData) \ + DEV_PFN(EXCLUDED, vkGetQueryPoolResults) \ + DEV_PFN(EXCLUDED, vkGetRayTracingShaderGroupHandlesNV) \ + DEV_PFN(EXCLUDED, vkGetRefreshCycleDurationGOOGLE) \ + DEV_PFN(EXCLUDED, vkGetRenderAreaGranularity) \ + DEV_PFN(OPTIONAL, vkGetSemaphoreFdKHR) \ + DEV_PFN(EXCLUDED, vkGetShaderInfoAMD) \ + DEV_PFN(EXCLUDED, vkGetSwapchainCounterEXT) \ + DEV_PFN(EXCLUDED, vkGetSwapchainImagesKHR) \ + DEV_PFN(EXCLUDED, vkGetSwapchainStatusKHR) \ + DEV_PFN(EXCLUDED, vkGetValidationCacheDataEXT) \ + DEV_PFN(OPTIONAL, vkImportFenceFdKHR) \ + DEV_PFN(OPTIONAL, vkImportSemaphoreFdKHR) \ + DEV_PFN(REQUIRED, vkInvalidateMappedMemoryRanges) \ + DEV_PFN(REQUIRED, vkMapMemory) \ + DEV_PFN(REQUIRED, vkMergePipelineCaches) \ + DEV_PFN(EXCLUDED, vkMergeValidationCachesEXT) \ + DEV_PFN(EXCLUDED, vkRegisterDeviceEventEXT) \ + DEV_PFN(EXCLUDED, vkRegisterDisplayEventEXT) \ + DEV_PFN(EXCLUDED, vkRegisterObjectsNVX) \ + DEV_PFN(EXCLUDED, vkResetCommandPool) \ + DEV_PFN(EXCLUDED, vkResetDescriptorPool) \ + DEV_PFN(REQUIRED, vkResetEvent) \ + DEV_PFN(REQUIRED, vkResetFences) \ + DEV_PFN(EXCLUDED, vkResetQueryPoolEXT) \ + DEV_PFN(OPTIONAL, vkSetDebugUtilsObjectNameEXT) \ + DEV_PFN(OPTIONAL, vkSetDebugUtilsObjectTagEXT) \ + DEV_PFN(REQUIRED, vkSetEvent) \ + DEV_PFN(EXCLUDED, vkSetHdrMetadataEXT) \ + DEV_PFN(EXCLUDED, vkSetLocalDimmingAMD) \ + DEV_PFN(REQUIRED, vkTrimCommandPool) \ + DEV_PFN(EXCLUDED, vkTrimCommandPoolKHR) \ + DEV_PFN(REQUIRED, vkUnmapMemory) \ + DEV_PFN(EXCLUDED, vkUnregisterObjectsNVX) \ + DEV_PFN(EXCLUDED, vkUpdateDescriptorSetWithTemplate) \ + DEV_PFN(EXCLUDED, vkUpdateDescriptorSetWithTemplateKHR) \ + DEV_PFN(REQUIRED, vkUpdateDescriptorSets) \ + DEV_PFN(REQUIRED, vkWaitForFences) \ + \ + INS_PFN(OPTIONAL, vkCreateDebugReportCallbackEXT) \ + INS_PFN(OPTIONAL, vkCreateDebugUtilsMessengerEXT) \ + INS_PFN(EXCLUDED, vkCreateDisplayPlaneSurfaceKHR) \ + INS_PFN(EXCLUDED, vkCreateHeadlessSurfaceEXT) \ + INS_PFN(EXCLUDED, vkDebugReportMessageEXT) \ + INS_PFN(OPTIONAL, vkDestroyDebugReportCallbackEXT) \ + INS_PFN(OPTIONAL, vkDestroyDebugUtilsMessengerEXT) \ + INS_PFN(REQUIRED, vkDestroyInstance) \ + INS_PFN(EXCLUDED, vkDestroySurfaceKHR) \ + INS_PFN(EXCLUDED, vkEnumeratePhysicalDeviceGroups) \ + INS_PFN(EXCLUDED, vkEnumeratePhysicalDeviceGroupsKHR) \ + INS_PFN(REQUIRED, vkEnumeratePhysicalDevices) \ + INS_PFN(EXCLUDED, vkSubmitDebugUtilsMessageEXT) \ + INS_PFN(REQUIRED, vkCreateDevice) \ + INS_PFN(EXCLUDED, vkCreateDisplayModeKHR) \ + INS_PFN(REQUIRED, vkEnumerateDeviceExtensionProperties) \ + INS_PFN(REQUIRED, vkEnumerateDeviceLayerProperties) \ + INS_PFN(EXCLUDED, vkGetDisplayModeProperties2KHR) \ + INS_PFN(EXCLUDED, vkGetDisplayModePropertiesKHR) \ + INS_PFN(EXCLUDED, vkGetDisplayPlaneCapabilities2KHR) \ + INS_PFN(EXCLUDED, vkGetDisplayPlaneCapabilitiesKHR) \ + INS_PFN(EXCLUDED, vkGetDisplayPlaneSupportedDisplaysKHR) \ + INS_PFN(EXCLUDED, vkGetPhysicalDeviceCalibrateableTimeDomainsEXT) \ + INS_PFN(EXCLUDED, vkGetPhysicalDeviceCooperativeMatrixPropertiesNV) \ + INS_PFN(EXCLUDED, vkGetPhysicalDeviceDisplayPlaneProperties2KHR) \ + INS_PFN(EXCLUDED, vkGetPhysicalDeviceDisplayPlanePropertiesKHR) \ + INS_PFN(EXCLUDED, vkGetPhysicalDeviceDisplayProperties2KHR) \ + INS_PFN(EXCLUDED, vkGetPhysicalDeviceDisplayPropertiesKHR) \ + INS_PFN(EXCLUDED, vkGetPhysicalDeviceExternalBufferProperties) \ + INS_PFN(EXCLUDED, vkGetPhysicalDeviceExternalBufferPropertiesKHR) \ + INS_PFN(EXCLUDED, vkGetPhysicalDeviceExternalFenceProperties) \ + INS_PFN(EXCLUDED, vkGetPhysicalDeviceExternalFencePropertiesKHR) \ + INS_PFN(EXCLUDED, vkGetPhysicalDeviceExternalImageFormatPropertiesNV) \ + INS_PFN(EXCLUDED, vkGetPhysicalDeviceExternalSemaphoreProperties) \ + INS_PFN(EXCLUDED, vkGetPhysicalDeviceExternalSemaphorePropertiesKHR) \ + INS_PFN(REQUIRED, vkGetPhysicalDeviceFeatures) \ + INS_PFN(EXCLUDED, vkGetPhysicalDeviceFeatures2) \ + INS_PFN(EXCLUDED, vkGetPhysicalDeviceFeatures2KHR) \ + INS_PFN(REQUIRED, vkGetPhysicalDeviceFormatProperties) \ + INS_PFN(EXCLUDED, vkGetPhysicalDeviceFormatProperties2) \ + INS_PFN(EXCLUDED, vkGetPhysicalDeviceFormatProperties2KHR) \ + INS_PFN(EXCLUDED, vkGetPhysicalDeviceGeneratedCommandsPropertiesNVX) \ + INS_PFN(EXCLUDED, vkGetPhysicalDeviceImageFormatProperties) \ + INS_PFN(EXCLUDED, vkGetPhysicalDeviceImageFormatProperties2) \ + INS_PFN(EXCLUDED, vkGetPhysicalDeviceImageFormatProperties2KHR) \ + INS_PFN(REQUIRED, vkGetPhysicalDeviceMemoryProperties) \ + INS_PFN(EXCLUDED, vkGetPhysicalDeviceMemoryProperties2) \ + INS_PFN(EXCLUDED, vkGetPhysicalDeviceMemoryProperties2KHR) \ + INS_PFN(EXCLUDED, vkGetPhysicalDeviceMultisamplePropertiesEXT) \ + INS_PFN(EXCLUDED, vkGetPhysicalDevicePresentRectanglesKHR) \ + INS_PFN(REQUIRED, vkGetPhysicalDeviceProperties) \ + INS_PFN(EXCLUDED, vkGetPhysicalDeviceProperties2) \ + INS_PFN(EXCLUDED, vkGetPhysicalDeviceProperties2KHR) \ + INS_PFN(REQUIRED, vkGetPhysicalDeviceQueueFamilyProperties) \ + INS_PFN(EXCLUDED, vkGetPhysicalDeviceQueueFamilyProperties2) \ + INS_PFN(EXCLUDED, vkGetPhysicalDeviceQueueFamilyProperties2KHR) \ + INS_PFN(EXCLUDED, vkGetPhysicalDeviceSparseImageFormatProperties) \ + INS_PFN(EXCLUDED, vkGetPhysicalDeviceSparseImageFormatProperties2) \ + INS_PFN(EXCLUDED, vkGetPhysicalDeviceSparseImageFormatProperties2KHR) \ + INS_PFN(EXCLUDED, vkGetPhysicalDeviceSurfaceCapabilities2EXT) \ + INS_PFN(EXCLUDED, vkGetPhysicalDeviceSurfaceCapabilities2KHR) \ + INS_PFN(EXCLUDED, vkGetPhysicalDeviceSurfaceCapabilitiesKHR) \ + INS_PFN(EXCLUDED, vkGetPhysicalDeviceSurfaceFormats2KHR) \ + INS_PFN(EXCLUDED, vkGetPhysicalDeviceSurfaceFormatsKHR) \ + INS_PFN(EXCLUDED, vkGetPhysicalDeviceSurfacePresentModesKHR) \ + INS_PFN(EXCLUDED, vkGetPhysicalDeviceSurfaceSupportKHR) \ + INS_PFN(EXCLUDED, vkReleaseDisplayEXT) \ + DEV_PFN(EXCLUDED, vkGetQueueCheckpointDataNV) \ + DEV_PFN(OPTIONAL, vkQueueBeginDebugUtilsLabelEXT) \ + DEV_PFN(EXCLUDED, vkQueueBindSparse) \ + DEV_PFN(OPTIONAL, vkQueueEndDebugUtilsLabelEXT) \ + DEV_PFN(OPTIONAL, vkQueueInsertDebugUtilsLabelEXT) \ + DEV_PFN(EXCLUDED, vkQueuePresentKHR) \ + DEV_PFN(REQUIRED, vkQueueSubmit) \ + DEV_PFN(REQUIRED, vkQueueWaitIdle) + +#ifdef VK_USE_PLATFORM_ANDROID_KHR +#define IREE_VULKAN_DYNAMIC_SYMBOL_TABLE_ANDROID_KHR(INS_PFN, DEV_PFN) \ + DEV_PFN(OPTIONAL, vkGetAndroidHardwareBufferPropertiesANDROID) \ + DEV_PFN(OPTIONAL, vkGetMemoryAndroidHardwareBufferANDROID) \ + INS_PFN(EXCLUDED, vkCreateAndroidSurfaceKHR) +#else +#define IREE_VULKAN_DYNAMIC_SYMBOL_TABLE_ANDROID_KHR(INS_PFN, DEV_PFN) +#endif // VK_USE_PLATFORM_ANDROID_KHR + +#ifdef VK_USE_PLATFORM_GGP +#define IREE_VULKAN_DYNAMIC_SYMBOL_TABLE_GGP(INS_PFN, DEV_PFN) \ + INS_PFN(EXCLUDED, vkCreateStreamDescriptorSurfaceGGP) +#else +#define IREE_VULKAN_DYNAMIC_SYMBOL_TABLE_GGP(INS_PFN, DEV_PFN) +#endif // VK_USE_PLATFORM_GGP + +#ifdef VK_USE_PLATFORM_IOS_MVK +#define IREE_VULKAN_DYNAMIC_SYMBOL_TABLE_IOS_MVK(INS_PFN, DEV_PFN) \ + INS_PFN(EXCLUDED, vkCreateIOSSurfaceMVK) +#else +#define IREE_VULKAN_DYNAMIC_SYMBOL_TABLE_IOS_MVK(INS_PFN, DEV_PFN) +#endif // VK_USE_PLATFORM_IOS_MVK + +#ifdef VK_USE_PLATFORM_FUCHSIA +#define IREE_VULKAN_DYNAMIC_SYMBOL_TABLE_FUSCHIA(INS_PFN, DEV_PFN) \ + INS_PFN(EXCLUDED, vkCreateImagePipeSurfaceFUCHSIA) +#else +#define IREE_VULKAN_DYNAMIC_SYMBOL_TABLE_FUSCHIA(INS_PFN, DEV_PFN) +#endif // VK_USE_PLATFORM_FUCHSIA + +#ifdef VK_USE_PLATFORM_MACOS_MVK +#define IREE_VULKAN_DYNAMIC_SYMBOL_TABLE_MACOS_MVK(INS_PFN, DEV_PFN) \ + INS_PFN(EXCLUDED, vkCreateMacOSSurfaceMVK) +#else +#define IREE_VULKAN_DYNAMIC_SYMBOL_TABLE_MACOS_MVK(INS_PFN, DEV_PFN) +#endif // VK_USE_PLATFORM_MACOS_MVK + +#ifdef VK_USE_PLATFORM_METAL_EXT +#define IREE_VULKAN_DYNAMIC_SYMBOL_TABLE_METAL_EXT(INS_PFN, DEV_PFN) \ + INS_PFN(EXCLUDED, vkCreateMetalSurfaceEXT) +#else +#define IREE_VULKAN_DYNAMIC_SYMBOL_TABLE_METAL_EXT(INS_PFN, DEV_PFN) +#endif // VK_USE_PLATFORM_METAL_EXT + +#ifdef VK_USE_PLATFORM_VI_NN +#define IREE_VULKAN_DYNAMIC_SYMBOL_TABLE_VI_NN(INS_PFN, DEV_PFN) \ + INS_PFN(EXCLUDED, vkCreateViSurfaceNN) +#else +#define IREE_VULKAN_DYNAMIC_SYMBOL_TABLE_VI_NN(INS_PFN, DEV_PFN) +#endif // VK_USE_PLATFORM_VI_NN + +#ifdef VK_USE_PLATFORM_WAYLAND_KHR +#define IREE_VULKAN_DYNAMIC_SYMBOL_TABLE_WAYLAND_KHR(INS_PFN, DEV_PFN) \ + INS_PFN(EXCLUDED, vkCreateWaylandSurfaceKHR) \ + INS_PFN(EXCLUDED, vkGetPhysicalDeviceWaylandPresentationSupportKHR) +#else +#define IREE_VULKAN_DYNAMIC_SYMBOL_TABLE_WAYLAND_KHR(INS_PFN, DEV_PFN) +#endif // VK_USE_PLATFORM_WAYLAND_KHR + +#ifdef VK_USE_PLATFORM_WIN32_KHR +#define IREE_VULKAN_DYNAMIC_SYMBOL_TABLE_WIN32_KHR(INS_PFN, DEV_PFN) \ + DEV_PFN(EXCLUDED, vkAcquireFullScreenExclusiveModeEXT) \ + DEV_PFN(EXCLUDED, vkGetDeviceGroupSurfacePresentModes2EXT) \ + DEV_PFN(REQUIRED, vkGetFenceWin32HandleKHR) \ + DEV_PFN(EXCLUDED, vkGetMemoryWin32HandleKHR) \ + DEV_PFN(EXCLUDED, vkGetMemoryWin32HandleNV) \ + DEV_PFN(EXCLUDED, vkGetMemoryWin32HandlePropertiesKHR) \ + DEV_PFN(REQUIRED, vkGetSemaphoreWin32HandleKHR) \ + DEV_PFN(REQUIRED, vkImportFenceWin32HandleKHR) \ + DEV_PFN(REQUIRED, vkImportSemaphoreWin32HandleKHR) \ + DEV_PFN(EXCLUDED, vkReleaseFullScreenExclusiveModeEXT) \ + INS_PFN(EXCLUDED, vkCreateWin32SurfaceKHR) \ + INS_PFN(EXCLUDED, vkGetPhysicalDeviceSurfacePresentModes2EXT) \ + INS_PFN(EXCLUDED, vkGetPhysicalDeviceWin32PresentationSupportKHR) +#else +#define IREE_VULKAN_DYNAMIC_SYMBOL_TABLE_WIN32_KHR(INS_PFN, DEV_PFN) +#endif // VK_USE_PLATFORM_WIN32_KHR + +#ifdef VK_USE_PLATFORM_XCB_KHR +#define IREE_VULKAN_DYNAMIC_SYMBOL_TABLE_XCB_KHR(INS_PFN, DEV_PFN) \ + INS_PFN(EXCLUDED, vkCreateXcbSurfaceKHR) \ + INS_PFN(EXCLUDED, vkGetPhysicalDeviceXcbPresentationSupportKHR) +#else +#define IREE_VULKAN_DYNAMIC_SYMBOL_TABLE_XCB_KHR(INS_PFN, DEV_PFN) +#endif // VK_USE_PLATFORM_XCB_KHR + +#ifdef VK_USE_PLATFORM_XLIB_KHR +#define IREE_VULKAN_DYNAMIC_SYMBOL_TABLE_XLIB_KHR(INS_PFN, DEV_PFN) \ + INS_PFN(EXCLUDED, vkCreateXlibSurfaceKHR) \ + INS_PFN(EXCLUDED, vkGetPhysicalDeviceXlibPresentationSupportKHR) +#else +#define IREE_VULKAN_DYNAMIC_SYMBOL_TABLE_XLIB_KHR(INS_PFN, DEV_PFN) +#endif // VK_USE_PLATFORM_XLIB_KHR + +#ifdef VK_USE_PLATFORM_XLIB_XRANDR_EXT +#define IREE_VULKAN_DYNAMIC_SYMBOL_TABLE_XLIB_XRANDR_EXT(INS_PFN, DEV_PFN) \ + INS_PFN(EXCLUDED, vkAcquireXlibDisplayEXT) \ + INS_PFN(EXCLUDED, vkGetRandROutputDisplayEXT) +#else +#define IREE_VULKAN_DYNAMIC_SYMBOL_TABLE_XLIB_XRANDR_EXT(INS_PFN, DEV_PFN) +#endif // VK_USE_PLATFORM_XLIB_XRANDR_EXT + +#define IREE_VULKAN_DYNAMIC_SYMBOL_PLATFORM_TABLES(INS_PFN, DEV_PFN) \ + IREE_VULKAN_DYNAMIC_SYMBOL_TABLE_ANDROID_KHR(INS_PFN, DEV_PFN) \ + IREE_VULKAN_DYNAMIC_SYMBOL_TABLE_GGP(INS_PFN, DEV_PFN) \ + IREE_VULKAN_DYNAMIC_SYMBOL_TABLE_IOS_MVK(INS_PFN, DEV_PFN) \ + IREE_VULKAN_DYNAMIC_SYMBOL_TABLE_FUSCHIA(INS_PFN, DEV_PFN) \ + IREE_VULKAN_DYNAMIC_SYMBOL_TABLE_MACOS_MVK(INS_PFN, DEV_PFN) \ + IREE_VULKAN_DYNAMIC_SYMBOL_TABLE_METAL_EXT(INS_PFN, DEV_PFN) \ + IREE_VULKAN_DYNAMIC_SYMBOL_TABLE_VI_NN(INS_PFN, DEV_PFN) \ + IREE_VULKAN_DYNAMIC_SYMBOL_TABLE_WAYLAND_KHR(INS_PFN, DEV_PFN) \ + IREE_VULKAN_DYNAMIC_SYMBOL_TABLE_WIN32_KHR(INS_PFN, DEV_PFN) \ + IREE_VULKAN_DYNAMIC_SYMBOL_TABLE_XCB_KHR(INS_PFN, DEV_PFN) \ + IREE_VULKAN_DYNAMIC_SYMBOL_TABLE_XLIB_KHR(INS_PFN, DEV_PFN) \ + IREE_VULKAN_DYNAMIC_SYMBOL_TABLE_XLIB_XRANDR_EXT(INS_PFN, DEV_PFN) + +#define IREE_VULKAN_DYNAMIC_SYMBOL_TABLES(INS_PFN, DEV_PFN) \ + IREE_VULKAN_DYNAMIC_SYMBOL_COMMON_TABLE(INS_PFN, DEV_PFN) \ + IREE_VULKAN_DYNAMIC_SYMBOL_PLATFORM_TABLES(INS_PFN, DEV_PFN) + +} // namespace vulkan +} // namespace hal +} // namespace iree + +#endif // THIRD_PARTY_MLIR_EDGE_IREE_HAL_VULKAN_DYNAMIC_SYMBOL_TABLES_H_ diff --git a/hal/vulkan/dynamic_symbols.cc b/hal/vulkan/dynamic_symbols.cc new file mode 100644 index 000000000000..8955eb23ae17 --- /dev/null +++ b/hal/vulkan/dynamic_symbols.cc @@ -0,0 +1,197 @@ +// Copyright 2019 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "third_party/mlir_edge/iree/hal/vulkan/dynamic_symbols.h" + +#include + +#include + +#include "third_party/absl/base/attributes.h" +#include "third_party/absl/base/macros.h" +#include "third_party/absl/memory/memory.h" +#include "third_party/absl/types/source_location.h" +#include "third_party/mlir_edge/iree/base/status.h" +#include "third_party/mlir_edge/iree/base/tracing.h" +#include "third_party/mlir_edge/iree/hal/vulkan/dynamic_symbol_tables.h" + +namespace iree { +namespace hal { +namespace vulkan { + +// Read-only table of function pointer information designed to be in .rdata. +// To reduce binary size this structure is packed (knowing that we won't have +// gigabytes of function pointers :). +struct FunctionPtrInfo { + // Name of the function (like 'vkSomeFunction'). + const char* function_name; + // 1 if the function pointer can be resolved via vkGetDeviceProcAddr. + uint32_t is_device : 1; + // 1 if the function is required and the loader should bail if not found. + uint32_t is_required : 1; + // TODO(benvanik): remove from table by manually walking sizeof(uintptr_t). + // An offset in bytes from the base of &syms to where the PFN_vkSomeFunction + // member is located. + uint32_t member_offset : 30; +} ABSL_ATTRIBUTE_PACKED; +static_assert(sizeof(FunctionPtrInfo) == sizeof(const char*) + sizeof(uint32_t), + "Alignment on FunctionPtrInfo struct is wrong"); + +namespace { + +#define REQUIRED_PFN_FUNCTION_PTR(function_name, is_device) \ + {#function_name, is_device, 1, offsetof(DynamicSymbols, function_name)}, +#define OPTIONAL_PFN_FUNCTION_PTR(function_name, is_device) \ + {#function_name, is_device, 0, offsetof(DynamicSymbols, function_name)}, +#define EXCLUDED_PFN_FUNCTION_PTR(function_name, is_device) +#define INS_PFN_FUNCTION_PTR(requirement, function_name) \ + requirement##_PFN_FUNCTION_PTR(function_name, 0) +#define DEV_PFN_FUNCTION_PTR(requirement, function_name) \ + requirement##_PFN_FUNCTION_PTR(function_name, 1) + +// Defines the table of mandatory FunctionPtrInfos resolved prior to instance +// creation. These are safe to call with no instance parameter and should be +// exported by all loaders/ICDs. +static constexpr const FunctionPtrInfo kInstancelessFunctionPtrInfos[] = { + REQUIRED_PFN_FUNCTION_PTR(vkCreateInstance, false) // + REQUIRED_PFN_FUNCTION_PTR(vkEnumerateInstanceLayerProperties, false) // + REQUIRED_PFN_FUNCTION_PTR(vkEnumerateInstanceExtensionProperties, false) // +}; + +// Defines the table of FunctionPtrInfos for dynamic loading that must wait +// until an instance has been created to be resolved. +static constexpr const FunctionPtrInfo kDynamicFunctionPtrInfos[] = { + IREE_VULKAN_DYNAMIC_SYMBOL_TABLES(INS_PFN_FUNCTION_PTR, + DEV_PFN_FUNCTION_PTR)}; + +} // namespace + +// static +StatusOr> DynamicSymbols::Create( + const GetProcAddrFn& get_proc_addr) { + IREE_TRACE_SCOPE0("DynamicSymbols::Create"); + + auto syms = make_ref(); + + // Resolve the method the shared object uses to resolve other functions. + // Some libraries will export all symbols while others will only export this + // single function. + syms->vkGetInstanceProcAddr = reinterpret_cast( + get_proc_addr("vkGetInstanceProcAddr")); + if (!syms->vkGetInstanceProcAddr) { + return UnavailableErrorBuilder(ABSL_LOC) + << "Required method vkGetInstanceProcAddr not " + "found in provided Vulkan library (did you pick the wrong file?)"; + } + + // Resolve the mandatory functions that we need to create instances. + // If the provided |get_proc_addr| cannot resolve these then it's not a loader + // or ICD we want to use, anyway. + for (int i = 0; i < ABSL_ARRAYSIZE(kInstancelessFunctionPtrInfos); ++i) { + const auto& function_ptr = kInstancelessFunctionPtrInfos[i]; + auto* member_ptr = reinterpret_cast( + reinterpret_cast(syms.get()) + function_ptr.member_offset); + *member_ptr = + syms->vkGetInstanceProcAddr(VK_NULL_HANDLE, function_ptr.function_name); + if (*member_ptr == nullptr) { + return UnavailableErrorBuilder(ABSL_LOC) + << "Mandatory Vulkan function " << function_ptr.function_name + << " not available; invalid loader/ICD?"; + } + } + + return syms; +} + +// static +StatusOr> DynamicSymbols::CreateFromSystemLoader() { + IREE_TRACE_SCOPE0("DynamicSymbols::CreateFromSystemLoader"); + + // TODO(benvanik): abstract out for other platforms. + void* library = ::dlopen("libvulkan.so.1", RTLD_LAZY | RTLD_LOCAL); + if (!library) { + return UnavailableErrorBuilder(ABSL_LOC) + << "Unable to open libvulkan.so; driver not installed/on " + "LD_LIBRARY_PATH"; + } + ASSIGN_OR_RETURN(auto syms, Create([library](const char* function_name) { + return reinterpret_cast( + ::dlsym(library, function_name)); + })); + syms->close_fn_ = [library]() { + // TODO(benvanik): disable if we want to get profiling results. Sometimes + // closing the library can prevent proper symbolization on crashes or + // in sampling profilers. + ::dlclose(library); + }; + return syms; +} + +Status DynamicSymbols::LoadFromInstance(VkInstance instance) { + IREE_TRACE_SCOPE0("DynamicSymbols::LoadFromInstance"); + return LoadFromDevice(instance, VK_NULL_HANDLE); +} + +Status DynamicSymbols::LoadFromDevice(VkInstance instance, VkDevice device) { + IREE_TRACE_SCOPE0("DynamicSymbols::LoadFromDevice"); + + if (!instance) { + return InvalidArgumentErrorBuilder(ABSL_LOC) + << "Instance must have been created and a default instance proc " + "lookup function is required"; + } + + // Setup the lookup methods first. The rest of the syms uses these to + // resolve function pointers. + this->vkGetDeviceProcAddr = reinterpret_cast( + this->vkGetInstanceProcAddr(instance, "vkGetDeviceProcAddr")); + if (!this->vkGetDeviceProcAddr) { + return UnavailableErrorBuilder(ABSL_LOC) + << "Required Vulkan function vkGetDeviceProcAddr not available; " + "invalid driver handle?"; + } + + // Load the rest of the functions. + for (int i = 0; i < ABSL_ARRAYSIZE(kDynamicFunctionPtrInfos); ++i) { + const auto& function_ptr = kDynamicFunctionPtrInfos[i]; + auto* member_ptr = reinterpret_cast( + reinterpret_cast(this) + function_ptr.member_offset); + if (function_ptr.is_device && device) { + *member_ptr = + this->vkGetDeviceProcAddr(device, function_ptr.function_name); + } else { + *member_ptr = + this->vkGetInstanceProcAddr(instance, function_ptr.function_name); + } + if (*member_ptr == nullptr && function_ptr.is_required) { + return UnavailableErrorBuilder(ABSL_LOC) + << "Required Vulkan function " << function_ptr.function_name + << " not available"; + } + } + + return OkStatus(); +} + +DynamicSymbols::DynamicSymbols() = default; + +DynamicSymbols::~DynamicSymbols() { + if (close_fn_) { + close_fn_(); + } +} + +} // namespace vulkan +} // namespace hal +} // namespace iree diff --git a/hal/vulkan/dynamic_symbols.h b/hal/vulkan/dynamic_symbols.h new file mode 100644 index 000000000000..77f63266685d --- /dev/null +++ b/hal/vulkan/dynamic_symbols.h @@ -0,0 +1,129 @@ +// Copyright 2019 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#ifndef THIRD_PARTY_MLIR_EDGE_IREE_HAL_VULKAN_DYNAMIC_SYMBOLS_H_ +#define THIRD_PARTY_MLIR_EDGE_IREE_HAL_VULKAN_DYNAMIC_SYMBOLS_H_ + +#include + +#include +#include +#include + +#include "third_party/mlir_edge/iree/base/ref_ptr.h" +#include "third_party/mlir_edge/iree/base/status.h" +#include "third_party/mlir_edge/iree/hal/vulkan/dynamic_symbol_tables.h" + +namespace iree { +namespace hal { +namespace vulkan { + +struct FunctionPtrInfo; + +// Dynamic Vulkan function loader for use with vulkan.hpp. +// This loader is a subset of the DispatchLoaderDynamic implementation that only +// loads functions we are interested in (a compute-specific subset) and avoids +// extensions we will never use. +// +// This exposes all Vulkan methods as function pointer members. Optional +// methods will be nullptr if not present. Excluded methods will be omitted. +// +// DynamicSymbols instances are designed to be passed to vulkan.hpp methods as +// the last argument, though they may also be called directly. +// **Always make sure to pass the loader to vulkan.hpp methods!** +// +// Loading is performed by walking a table of required and optional functions +// (defined in dynamic_symbol_tables.h) and populating the member function +// pointers exposed on this struct when available. For example, if the +// vkSomeFunction method is marked in the table as OPTIONAL the loader will +// attempt to lookup the function and if successful set the +// DynamicSymbols::vkSomeFunction pointer to the resolved address. If the +// function is not found then it will be set to nullptr so users can check for +// function availability. +// +// Documentation: +// https://github.com/KhronosGroup/Vulkan-Hpp#extensions--per-device-function-pointers +// +// Usage: +// ASSIGN_OR_RETURN(auto syms, DynamicSymbols::CreateFromSystemLoader()); +// VkInstance instance = VK_NULL_HANDLE; +// syms->vkCreateInstance(..., &instance); +// RETURN_IF_ERROR(syms->LoadFromInstance(instance)); +struct DynamicSymbols : public RefObject { + using GetProcAddrFn = + std::function; + + DynamicSymbols(); + ~DynamicSymbols(); + + // Creates the dynamic symbol table using the given |get_proc_addr| to resolve + // the vkCreateInstance function. + // + // After the instance is created the caller must use LoadFromInstance (or + // LoadFromDevice) to load the remaining symbols. + static StatusOr> Create( + const GetProcAddrFn& get_proc_addr); + + // Loads all required and optional Vulkan functions from the Vulkan loader. + // This will look for a Vulkan loader on the system (like libvulkan.so) and + // dlsym the functions from that. + // + // The loaded function pointers will point to thunks in the ICD. This may + // enable additional debug checking and more readable stack traces (as + // errors come from within the ICD, where we have symbols). + static StatusOr> CreateFromSystemLoader(); + + // Loads all required and optional Vulkan functions from the given instance. + // + // The loaded function pointers will point to thunks in the ICD. This may + // enable additional debug checking and more readable stack traces (as + // errors come from within the ICD, where we have symbols). + Status LoadFromInstance(VkInstance instance); + + // Loads all required and optional Vulkan functions from the given device, + // falling back to the instance when required. + // + // This attempts to directly query the methods from the device, bypassing any + // ICD or shim layers. These methods will generally have less overhead at + // runtime as they need not jump through the various trampolines. + Status LoadFromDevice(VkInstance instance, VkDevice device); + + // Define members for each function pointer. + // See dynamic_symbol_tables.h for the full list of methods. + // + // Each required and optional function in the loader tables will expand to + // the following member, such as for example 'vkSomeFunction': + // PFN_vkSomeFunction vkSomeFunction; +#define REQUIRED_PFN(function_name) PFN_##function_name function_name +#define OPTIONAL_PFN(function_name) PFN_##function_name function_name +#define EXCLUDED_PFN(function_name) +#define PFN_MEMBER(requirement, function_name) requirement##_PFN(function_name); + REQUIRED_PFN(vkGetInstanceProcAddr); + REQUIRED_PFN(vkGetDeviceProcAddr); + IREE_VULKAN_DYNAMIC_SYMBOL_TABLES(PFN_MEMBER, PFN_MEMBER); +#undef REQUIRED_PFN +#undef OPTIONAL_PFN +#undef EXCLUDED_PFN +#undef PFN_MEMBER + + private: + // Optional callback on loader destruction. + std::function close_fn_; +}; + +} // namespace vulkan +} // namespace hal +} // namespace iree + +#endif // THIRD_PARTY_MLIR_EDGE_IREE_HAL_VULKAN_DYNAMIC_SYMBOLS_H_ diff --git a/hal/vulkan/extensibility_util.cc b/hal/vulkan/extensibility_util.cc new file mode 100644 index 000000000000..e8c1ee7a9a68 --- /dev/null +++ b/hal/vulkan/extensibility_util.cc @@ -0,0 +1,217 @@ +// Copyright 2019 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "third_party/mlir_edge/iree/hal/vulkan/extensibility_util.h" + +#include "third_party/mlir_edge/iree/base/status.h" +#include "third_party/mlir_edge/iree/base/tracing.h" +#include "third_party/mlir_edge/iree/hal/vulkan/status_util.h" + +namespace iree { +namespace hal { +namespace vulkan { + +namespace { + +StatusOr> MatchAvailableLayers( + absl::Span required_layers, + absl::Span optional_layers, + absl::Span properties) { + IREE_TRACE_SCOPE0("MatchAvailableLayers"); + + std::vector enabled_layers; + enabled_layers.reserve(required_layers.size() + optional_layers.size()); + + for (const char* layer_name : required_layers) { + bool found = false; + for (const auto& layer_properties : properties) { + if (std::strcmp(layer_name, layer_properties.layerName) == 0) { + VLOG(1) << "Enabling required layer: " << layer_name; + found = true; + enabled_layers.push_back(layer_name); + break; + } + } + if (!found) { + return UnavailableErrorBuilder(ABSL_LOC) + << "Required layer " << layer_name << " not available"; + } + } + + for (const char* layer_name : optional_layers) { + bool found = false; + for (const auto& layer_properties : properties) { + if (std::strcmp(layer_name, layer_properties.layerName) == 0) { + VLOG(1) << "Enabling optional layer: " << layer_name; + found = true; + enabled_layers.push_back(layer_name); + break; + } + } + if (!found) { + VLOG(1) << "Optional layer " << layer_name << " not available"; + } + } + + return enabled_layers; +} + +StatusOr> MatchAvailableExtensions( + absl::Span required_extensions, + absl::Span optional_extensions, + absl::Span properties) { + IREE_TRACE_SCOPE0("MatchAvailableExtensions"); + + std::vector enabled_extensions; + enabled_extensions.reserve(required_extensions.size() + + optional_extensions.size()); + + for (const char* extension_name : required_extensions) { + bool found = false; + for (const auto& extension_properties : properties) { + if (std::strcmp(extension_name, extension_properties.extensionName) == + 0) { + VLOG(1) << "Enabling required extension: " << extension_name; + found = true; + enabled_extensions.push_back(extension_name); + break; + } + } + if (!found) { + return UnavailableErrorBuilder(ABSL_LOC) + << "Required extension " << extension_name << " not available"; + } + } + + for (const char* extension_name : optional_extensions) { + bool found = false; + for (const auto& extension_properties : properties) { + if (std::strcmp(extension_name, extension_properties.extensionName) == + 0) { + VLOG(1) << "Enabling optional extension: " << extension_name; + found = true; + enabled_extensions.push_back(extension_name); + break; + } + } + if (!found) { + VLOG(1) << "Optional extension " << extension_name << " not available"; + } + } + + return enabled_extensions; +} + +} // namespace + +StatusOr> MatchAvailableInstanceLayers( + const ExtensibilitySpec& extensibility_spec, const DynamicSymbols& syms) { + uint32_t layer_property_count = 0; + VK_RETURN_IF_ERROR( + syms.vkEnumerateInstanceLayerProperties(&layer_property_count, nullptr)); + std::vector layer_properties(layer_property_count); + VK_RETURN_IF_ERROR(syms.vkEnumerateInstanceLayerProperties( + &layer_property_count, layer_properties.data())); + ASSIGN_OR_RETURN(auto enabled_layers, + MatchAvailableLayers(extensibility_spec.required_layers, + extensibility_spec.optional_layers, + layer_properties), + _ << "Unable to find all required instance layers"); + return enabled_layers; +} + +StatusOr> MatchAvailableInstanceExtensions( + const ExtensibilitySpec& extensibility_spec, const DynamicSymbols& syms) { + uint32_t extension_property_count = 0; + VK_RETURN_IF_ERROR(syms.vkEnumerateInstanceExtensionProperties( + nullptr, &extension_property_count, nullptr)); + std::vector extension_properties( + extension_property_count); + VK_RETURN_IF_ERROR(syms.vkEnumerateInstanceExtensionProperties( + nullptr, &extension_property_count, extension_properties.data())); + ASSIGN_OR_RETURN( + auto enabled_extensions, + MatchAvailableExtensions(extensibility_spec.required_extensions, + extensibility_spec.optional_extensions, + extension_properties), + _ << "Unable to find all required instance extensions"); + return enabled_extensions; +} + +StatusOr> MatchAvailableDeviceLayers( + VkPhysicalDevice physical_device, + const ExtensibilitySpec& extensibility_spec, const DynamicSymbols& syms) { + uint32_t layer_property_count = 0; + VK_RETURN_IF_ERROR(syms.vkEnumerateDeviceLayerProperties( + physical_device, &layer_property_count, nullptr)); + std::vector layer_properties(layer_property_count); + VK_RETURN_IF_ERROR(syms.vkEnumerateDeviceLayerProperties( + physical_device, &layer_property_count, layer_properties.data())); + ASSIGN_OR_RETURN(auto enabled_layers, + MatchAvailableLayers(extensibility_spec.required_layers, + extensibility_spec.optional_layers, + layer_properties), + _ << "Unable to find all required device layers"); + return enabled_layers; +} + +StatusOr> MatchAvailableDeviceExtensions( + VkPhysicalDevice physical_device, + const ExtensibilitySpec& extensibility_spec, const DynamicSymbols& syms) { + uint32_t extension_property_count = 0; + VK_RETURN_IF_ERROR(syms.vkEnumerateDeviceExtensionProperties( + physical_device, nullptr, &extension_property_count, nullptr)); + std::vector extension_properties( + extension_property_count); + VK_RETURN_IF_ERROR(syms.vkEnumerateDeviceExtensionProperties( + physical_device, nullptr, &extension_property_count, + extension_properties.data())); + ASSIGN_OR_RETURN( + auto enabled_extensions, + MatchAvailableExtensions(extensibility_spec.required_extensions, + extensibility_spec.optional_extensions, + extension_properties), + _ << "Unable to find all required device extensions"); + return enabled_extensions; +} + +InstanceExtensions PopulateEnabledInstanceExtensions( + absl::Span extension_names) { + InstanceExtensions extensions = {0}; + for (const char* extension_name : extension_names) { + if (std::strcmp(extension_name, VK_EXT_DEBUG_REPORT_EXTENSION_NAME) == 0) { + extensions.debug_report = true; + } else if (std::strcmp(extension_name, VK_EXT_DEBUG_UTILS_EXTENSION_NAME) == + 0) { + extensions.debug_utils = true; + } + } + return extensions; +} + +DeviceExtensions PopulateEnabledDeviceExtensions( + absl::Span extension_names) { + DeviceExtensions extensions = {0}; + for (const char* extension_name : extension_names) { + if (std::strcmp(extension_name, VK_KHR_PUSH_DESCRIPTOR_EXTENSION_NAME) == + 0) { + extensions.push_descriptors = true; + } + } + return extensions; +} + +} // namespace vulkan +} // namespace hal +} // namespace iree diff --git a/hal/vulkan/extensibility_util.h b/hal/vulkan/extensibility_util.h new file mode 100644 index 000000000000..b7bd113b644c --- /dev/null +++ b/hal/vulkan/extensibility_util.h @@ -0,0 +1,100 @@ +// Copyright 2019 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// Utilities for working with layers and extensions. + +#ifndef THIRD_PARTY_MLIR_EDGE_IREE_HAL_VULKAN_EXTENSIBILITY_UTIL_H_ +#define THIRD_PARTY_MLIR_EDGE_IREE_HAL_VULKAN_EXTENSIBILITY_UTIL_H_ + +#include + +#include "third_party/absl/types/span.h" +#include "third_party/mlir_edge/iree/base/status.h" +#include "third_party/mlir_edge/iree/hal/vulkan/dynamic_symbols.h" + +namespace iree { +namespace hal { +namespace vulkan { + +// Describes required and optional extensibility points. +struct ExtensibilitySpec { + // A list of required and optional layers. + std::vector required_layers; + std::vector optional_layers; + + // A list of required and optional extensions. + // Prefer using the _EXTENSION_NAME macros to make tracking easier (such as + // 'VK_KHR_GET_PHYSICAL_DEVICE_PROPERTIES_2_EXTENSION_NAME'). + std::vector required_extensions; + std::vector optional_extensions; +}; + +// Returns a list of layer names available for instances. +// Fails if any required_layers are unavailable. +StatusOr> MatchAvailableInstanceLayers( + const ExtensibilitySpec& extensibility_spec, const DynamicSymbols& syms); + +// Returns a list of extension names available for instances. +// Fails if any required_extensions are unavailable. +StatusOr> MatchAvailableInstanceExtensions( + const ExtensibilitySpec& extensibility_spec, const DynamicSymbols& syms); + +// Returns a list of layer names available for the given |physical_device|. +// Fails if any required_layers are unavailable. +StatusOr> MatchAvailableDeviceLayers( + VkPhysicalDevice physical_device, + const ExtensibilitySpec& extensibility_spec, const DynamicSymbols& syms); + +// Returns a list of extension names available for the given |physical_device|. +// Fails if any required_extensions are unavailable. +StatusOr> MatchAvailableDeviceExtensions( + VkPhysicalDevice physical_device, + const ExtensibilitySpec& extensibility_spec, const DynamicSymbols& syms); + +// Bits for enabled instance extensions. +// We must use this to query support instead of just detecting symbol names as +// ICDs will resolve the functions sometimes even if they don't support the +// extension (or we didn't ask for it to be enabled). +struct InstanceExtensions { + // VK_EXT_debug_report is enabled and a callback is regsitered. + // https://www.khronos.org/registry/vulkan/specs/1.1-extensions/html/chap44.html#VK_EXT_debug_report + bool debug_report : 1; + + // VK_EXT_debug_utils is enabled and a debug messenger is registered. + // https://www.khronos.org/registry/vulkan/specs/1.1-extensions/html/chap44.html#VK_EXT_debug_utils + bool debug_utils : 1; +}; + +// Returns a bitfield with all of the provided extension names. +InstanceExtensions PopulateEnabledInstanceExtensions( + absl::Span extension_names); + +// Bits for enabled device extensions. +// We must use this to query support instead of just detecting symbol names as +// ICDs will resolve the functions sometimes even if they don't support the +// extension (or we didn't ask for it to be enabled). +struct DeviceExtensions { + // VK_KHR_push_descriptor is enabled and vkCmdPushDescriptorSetKHR is valid. + bool push_descriptors : 1; +}; + +// Returns a bitfield with all of the provided extension names. +DeviceExtensions PopulateEnabledDeviceExtensions( + absl::Span extension_names); + +} // namespace vulkan +} // namespace hal +} // namespace iree + +#endif // THIRD_PARTY_MLIR_EDGE_IREE_HAL_VULKAN_EXTENSIBILITY_UTIL_H_ diff --git a/hal/vulkan/handle_util.h b/hal/vulkan/handle_util.h new file mode 100644 index 000000000000..1b73fa19bef9 --- /dev/null +++ b/hal/vulkan/handle_util.h @@ -0,0 +1,136 @@ +// Copyright 2019 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// Helpers for wrapping Vulkan handles that don't require us to wrap every type. +// This keeps our compilation time reasonable (as the vulkancpp library is +// insane) while giving us nice safety around cleanup and ensuring we use +// dynamic symbols and consistent allocators. +// +// Do not add functionality beyond handle management to these types. Keep our +// Vulkan usage mostly functional and C-like to ensure minimal code size and +// readability. + +#ifndef THIRD_PARTY_MLIR_EDGE_IREE_HAL_VULKAN_HANDLE_UTIL_H_ +#define THIRD_PARTY_MLIR_EDGE_IREE_HAL_VULKAN_HANDLE_UTIL_H_ + +#include + +#include "third_party/absl/synchronization/mutex.h" +#include "third_party/absl/utility/utility.h" +#include "third_party/mlir_edge/iree/base/ref_ptr.h" +#include "third_party/mlir_edge/iree/hal/vulkan/dynamic_symbols.h" +#include "third_party/mlir_edge/iree/hal/vulkan/extensibility_util.h" + +namespace iree { +namespace hal { +namespace vulkan { + +class VkDeviceHandle : public RefObject { + public: + VkDeviceHandle(const ref_ptr& syms, + DeviceExtensions enabled_extensions, + const VkAllocationCallbacks* allocator = nullptr) + : syms_(add_ref(syms)), + enabled_extensions_(enabled_extensions), + allocator_(allocator) {} + ~VkDeviceHandle() { reset(); } + + VkDeviceHandle(const VkDeviceHandle&) = delete; + VkDeviceHandle& operator=(const VkDeviceHandle&) = delete; + VkDeviceHandle(VkDeviceHandle&& other) noexcept + : value_(absl::exchange(other.value_, + static_cast(VK_NULL_HANDLE))), + syms_(std::move(other.syms_)), + enabled_extensions_(other.enabled_extensions_), + allocator_(other.allocator_) {} + + void reset() { + if (value_ == VK_NULL_HANDLE) return; + syms_->vkDestroyDevice(value_, allocator_); + value_ = VK_NULL_HANDLE; + } + + VkDevice value() const noexcept { return value_; } + VkDevice* mutable_value() noexcept { return &value_; } + operator VkDevice() const noexcept { return value_; } + + const ref_ptr& syms() const noexcept { return syms_; } + const VkAllocationCallbacks* allocator() const noexcept { return allocator_; } + + const DeviceExtensions& enabled_extensions() const { + return enabled_extensions_; + } + + private: + VkDevice value_ = VK_NULL_HANDLE; + ref_ptr syms_; + DeviceExtensions enabled_extensions_; + const VkAllocationCallbacks* allocator_ = nullptr; +}; + +class VkCommandPoolHandle : public RefObject { + public: + explicit VkCommandPoolHandle(const ref_ptr& logical_device) + : logical_device_(add_ref(logical_device)) {} + ~VkCommandPoolHandle() { reset(); } + + VkCommandPoolHandle(const VkCommandPoolHandle&) = delete; + VkCommandPoolHandle& operator=(const VkCommandPoolHandle&) = delete; + VkCommandPoolHandle(VkCommandPoolHandle&& other) noexcept + : logical_device_(std::move(other.logical_device_)), + value_(absl::exchange(other.value_, + static_cast(VK_NULL_HANDLE))) {} + VkCommandPoolHandle& operator=(VkCommandPoolHandle&& other) { + std::swap(logical_device_, other.logical_device_); + std::swap(value_, other.value_); + return *this; + } + + void reset() { + if (value_ == VK_NULL_HANDLE) return; + syms()->vkDestroyCommandPool(*logical_device_, value_, allocator()); + value_ = VK_NULL_HANDLE; + } + + VkCommandPool value() const noexcept { return value_; } + VkCommandPool* mutable_value() noexcept { return &value_; } + operator VkCommandPool() const noexcept { return value_; } + + const ref_ptr& logical_device() const noexcept { + return logical_device_; + } + const ref_ptr& syms() const noexcept { + return logical_device_->syms(); + } + const VkAllocationCallbacks* allocator() const noexcept { + return logical_device_->allocator(); + } + + absl::Mutex* mutex() const { return &mutex_; } + + private: + ref_ptr logical_device_; + VkCommandPool value_ = VK_NULL_HANDLE; + + // Vulkan command pools are not thread safe and require external + // synchronization. Since we allow arbitrary threads to allocate and + // deallocate the HAL command buffers we need to externally synchronize. + mutable absl::Mutex mutex_; +}; + +} // namespace vulkan +} // namespace hal +} // namespace iree + +#endif // THIRD_PARTY_MLIR_EDGE_IREE_HAL_VULKAN_HANDLE_UTIL_H_ diff --git a/hal/vulkan/legacy_fence.cc b/hal/vulkan/legacy_fence.cc new file mode 100644 index 000000000000..87c7662ccddb --- /dev/null +++ b/hal/vulkan/legacy_fence.cc @@ -0,0 +1,396 @@ +// Copyright 2019 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "third_party/mlir_edge/iree/hal/vulkan/legacy_fence.h" + +#include + +#include "third_party/absl/container/inlined_vector.h" +#include "third_party/absl/synchronization/mutex.h" +#include "third_party/absl/time/time.h" +#include "third_party/absl/types/source_location.h" +#include "third_party/mlir_edge/iree/base/intrusive_list.h" +#include "third_party/mlir_edge/iree/base/status.h" +#include "third_party/mlir_edge/iree/base/tracing.h" +#include "third_party/mlir_edge/iree/hal/vulkan/status_util.h" + +namespace iree { +namespace hal { +namespace vulkan { + +namespace { + +// Inserts the given |fence_signal| into |list| in ascending order. +void InsertOutstandingFenceSignal(OutstandingFenceSignal* fence_signal, + IntrusiveList* list) { + for (auto existing_signal : *list) { + if (existing_signal->value > fence_signal->value) { + list->insert(existing_signal, fence_signal); + return; + } + } + list->push_back(fence_signal); +} + +} // namespace + +// static +StatusOr> LegacyFencePool::Create( + ref_ptr logical_device) { + IREE_TRACE_SCOPE0("LegacyFencePool::Create"); + ref_ptr fence_pool( + new LegacyFencePool(std::move(logical_device))); + RETURN_IF_ERROR(fence_pool->PreallocateFences()); + return fence_pool; +} + +LegacyFencePool::LegacyFencePool(ref_ptr logical_device) + : logical_device_(std::move(logical_device)) {} + +LegacyFencePool::~LegacyFencePool() { + IREE_TRACE_SCOPE0("LegacyFencePool::dtor"); + + absl::MutexLock lock(&mutex_); + for (auto& fence_signal : storage_) { + syms()->vkDestroyFence(*logical_device_, fence_signal.fence, + logical_device_->allocator()); + } + unused_fences_.clear(); + unresolved_fences_.clear(); +} + +Status LegacyFencePool::PreallocateFences() { + IREE_TRACE_SCOPE0("LegacyFencePool::PreallocateFences"); + + VkFenceCreateInfo create_info; + create_info.sType = VK_STRUCTURE_TYPE_FENCE_CREATE_INFO; + create_info.pNext = nullptr; + create_info.flags = 0; + + absl::MutexLock lock(&mutex_); + for (int i = 0; i < kMaxInFlightFenceCount; ++i) { + auto* fence_signal = &storage_[i]; + VK_RETURN_IF_ERROR(syms()->vkCreateFence(*logical_device_, &create_info, + logical_device_->allocator(), + &fence_signal->fence)); + unused_fences_.push_back(fence_signal); + } + + return OkStatus(); +} + +StatusOr LegacyFencePool::Acquire() { + IREE_TRACE_SCOPE0("LegacyFencePool::Acquire"); + + absl::MutexLock lock(&mutex_); + if (unused_fences_.empty()) { + return ResourceExhaustedErrorBuilder(ABSL_LOC) + << "Fence pool out of unused fences"; + } + + auto* fence_signal = unused_fences_.front(); + unused_fences_.pop_front(); + return fence_signal; +} + +void LegacyFencePool::ReleaseResolved( + IntrusiveList* fence_signals) { + IREE_TRACE_SCOPE0("LegacyFencePool::ReleaseResolved"); + + // Get a list of fences we need to reset. Note that not all fences may have + // been signaled and we can avoid resetting them. + absl::InlinedVector handles; + handles.reserve(fence_signals->size()); + for (auto* fence_signal : *fence_signals) { + if (fence_signal->is_pending) { + handles.push_back(fence_signal->fence); + } + } + if (!handles.empty()) { + syms()->vkResetFences(*logical_device_, handles.size(), handles.data()); + } + + absl::MutexLock lock(&mutex_); + unused_fences_.merge_from(fence_signals); +} + +void LegacyFencePool::ReleaseUnresolved( + IntrusiveList* fence_signals) { + IREE_TRACE_SCOPE0("LegacyFencePool::ReleaseUnresolved"); + + absl::MutexLock lock(&mutex_); + while (!fence_signals->empty()) { + auto* fence_signal = fence_signals->front(); + fence_signals->pop_front(); + if (fence_signal->is_pending) { + // Fence was submitted and may still have a pending signal on it. We can't + // reuse it until it has resolved. + // TODO(benvanik): fix these fences by reallocating? We aren't leaking + // here (technically) but we will exhaust the pool pretty quickly. + unresolved_fences_.push_back(fence_signal); + } else { + // Fence was never actually submitted so we can reuse it no problem. + unused_fences_.push_back(fence_signal); + } + } +} + +// static +Status LegacyFence::WaitForFences(VkDeviceHandle* logical_device, + absl::Span fences, + bool wait_all, absl::Time deadline) { + IREE_TRACE_SCOPE0("LegacyFence::WaitForFences"); + + // NOTE: we could pool this state too (probably right on the LegacyFencePool) + // or be smarter about using stack-allocated storage. The best idea is to use + // real timeline semaphores, though, so not much effort has been spent on + // optimizing this. + absl::InlinedVector handles; + handles.reserve(fences.size()); + + // Loop over the fences and wait for any/all to signal. In wait_all mode we + // perform the bookkeeping to remove fences that have already been signaled so + // that we only wait on ones we need to (and possibly avoid making the vk call + // entirely!). + while (true) { + // Grab handles and acquire fences for all fences not yet at the requested + // timeline value. + for (const auto& fence_value : fences) { + auto* fence = reinterpret_cast(fence_value.first); + // NOTE: this will return the sticky fence error if the fence has failed. + ASSIGN_OR_RETURN(VkFence handle, + fence->AcquireWaitFence(fence_value.second)); + if (handle != VK_NULL_HANDLE) { + // Fence is unresolved and we need to really wait for it. + handles.push_back(handle); + } + } + if (handles.empty()) { + // All fences resolved. + return OkStatus(); + } + + uint64_t timeout_nanos; + if (deadline == absl::InfiniteFuture()) { + timeout_nanos = UINT64_MAX; + } else if (deadline == absl::InfinitePast()) { + timeout_nanos = 0; + } else { + auto relative_nanos = absl::ToInt64Nanoseconds(deadline - absl::Now()); + timeout_nanos = relative_nanos < 0 ? 0 : relative_nanos; + } + + // Wait on the fences we still need. + // Note that waking does not actually indicate all fences were hit! We need + // to do another pass above on the next iteration to make sure that we don't + // need to wait again on another fence. + VK_RETURN_IF_ERROR(logical_device->syms()->vkWaitForFences( + *logical_device, handles.size(), handles.data(), wait_all, + timeout_nanos)); + handles.clear(); + } + + return OkStatus(); +} + +LegacyFence::LegacyFence(ref_ptr fence_pool, + uint64_t initial_value) + : fence_pool_(std::move(fence_pool)), value_(initial_value) {} + +LegacyFence::~LegacyFence() { + IREE_TRACE_SCOPE0("LegacyFence::dtor"); + CHECK_OK(TryResolveOutstandingFences(UINT64_MAX)); + absl::MutexLock lock(&mutex_); + CHECK(outstanding_signals_.empty()) + << "Destroying a fence without first waiting on outstanding signals"; +} + +Status LegacyFence::status() const { + if (value_.load() != UINT64_MAX) { + return OkStatus(); + } + absl::MutexLock lock(&mutex_); + return status_; +} + +StatusOr LegacyFence::QueryValue() { + RETURN_IF_ERROR(TryResolveOutstandingFences(UINT64_MAX)); + return value_.load(); +} + +StatusOr LegacyFence::AcquireSignalFence(uint64_t value) { + absl::MutexLock lock(&mutex_); + + // It's an error to signal out of order (as that requires a lot more + // tracking and magic to get right). + if (value_.load() >= value) { + return FailedPreconditionErrorBuilder(ABSL_LOC) + << "Attempting to signal a timeline fence out of order; value=" + << value_ << ", new_value=" << value; + } + + // Scan to see if there's waiters for this value (or values before it). + // We may be able to reuse a previously allocated fence in the case that a + // user is waiting prior to actually submitting the signal operation. + OutstandingFenceSignal* signal_state = nullptr; + for (auto* fence_signal : outstanding_signals_) { + if (fence_signal->value == value) { + // Fence is going to be signaled at exactly the required value. + if (fence_signal->is_pending) { + // Already have signaled to this value - that's a paddlin'. + return FailedPreconditionErrorBuilder(ABSL_LOC) + << "Duplicate signal of timeline fence for value=" << value; + } + signal_state = fence_signal; + break; + } + } + if (!signal_state) { + // Allocate a signal state entry and a VkFence to submit with. + // TODO(benvanik): check for RESOURCE_EXHAUSTED and force a flush. + ASSIGN_OR_RETURN(signal_state, fence_pool_->Acquire()); + signal_state->value = value; + InsertOutstandingFenceSignal(signal_state, &outstanding_signals_); + } + + signal_state->is_pending = true; + return signal_state->fence; +} + +StatusOr LegacyFence::AcquireWaitFence(uint64_t value) { + // If we've already resolved then we want to avoid doing any kind of wait. + // Since the value is monotonically increasing we can do a lock-free peek + // here to see if we need to bother taking a full lock. + if (value_.load() >= value) { + return VK_NULL_HANDLE; + } + + absl::MutexLock lock(&mutex_); + + // Try to resolve any outstanding fence signals. + RETURN_IF_ERROR(TryResolveOutstandingFencesLocked(value)); + if (value_.load() >= value) { + return VK_NULL_HANDLE; + } + + // Try to find an existing fence we can reuse based on the required value. + OutstandingFenceSignal* signal_state = nullptr; + for (auto* fence_signal : outstanding_signals_) { + if (fence_signal->value >= value) { + // Fence is going to be signaled at or above the required value. + signal_state = fence_signal; + break; // |outstanding_signals_| is in sorted order. + } + } + if (!signal_state) { + // Allocate a signal state entry and a VkFence that we will need to signal + // in the future. We can't yet insert it into the queue but it will go in + // when the user tries to signal a value >= the required value. + // TODO(benvanik): check for RESOURCE_EXHAUSTED and force a flush. + ASSIGN_OR_RETURN(signal_state, fence_pool_->Acquire()); + signal_state->value = value; + InsertOutstandingFenceSignal(signal_state, &outstanding_signals_); + } + + return signal_state->fence; +} + +Status LegacyFence::TryResolveOutstandingFences(uint64_t upper_value) { + absl::MutexLock lock(&mutex_); + return TryResolveOutstandingFencesLocked(upper_value); +} + +Status LegacyFence::TryResolveOutstandingFencesLocked(uint64_t upper_value) { + // Fast-path for when we have no outstanding fences. + // NOTE: we hold the lock during the entire resolve process so that any waiter + // will only be woken once we have resolved to the furthest possible value. + if (outstanding_signals_.empty() || value_ > upper_value) { + return OkStatus(); + } + + IREE_TRACE_SCOPE0("LegacyFence::TryResolveOutstandingFences"); + + IntrusiveList resolved_fences; + IntrusiveList unresolved_fences; + VkDevice device = *fence_pool_->logical_device(); + const auto& syms = fence_pool_->syms(); + bool keep_resolving = true; + while (keep_resolving && !outstanding_signals_.empty()) { + auto* fence_signal = outstanding_signals_.front(); + if (fence_signal->value > upper_value) { + // Signal is for a value beyond our upper limit - early exit so that we + // don't spend time dealing with signals we don't yet care about. This can + // prevent live lock where one thread is signaling fences as fast/faster + // than another thread can consume them. + keep_resolving = false; + break; + } + VkResult fence_status = syms->vkGetFenceStatus(device, fence_signal->fence); + switch (fence_status) { + case VK_SUCCESS: { + // Fence has signaled meaning that we have reached this point in the + // timeline and can advance the value. + value_.store(fence_signal->value); + outstanding_signals_.erase(fence_signal); + resolved_fences.push_back(fence_signal); + + // Run backwards and resolve any non-pending fences as they will never + // be used. + for (auto* it = fence_signal; it != nullptr;) { + auto* prev_fence_signal = it; + it = outstanding_signals_.previous(it); + if (!prev_fence_signal->is_pending) { + outstanding_signals_.erase(prev_fence_signal); + unresolved_fences.push_back(prev_fence_signal); + } + } + break; + } + case VK_NOT_READY: + if (fence_signal->is_pending) { + // Fence has not yet been signaled. We stop here and wait for future + // attempts at resolution. + keep_resolving = false; + } + // Fence is not even pending yet - we may have skipped it. Keep + // resolving to see if there's a higher value we can use. + break; + default: + // Fence indicates an error (device lost, out of memory, etc). + // Propagate this back to our status (and thus any waiters). + // Since we only take the first error we find we skip all remaining + // fences. + status_ = VkResultToStatus(fence_status); + value_.store(UINT64_MAX); + outstanding_signals_.erase(fence_signal); + resolved_fences.push_back(fence_signal); + break; + } + } + + // Release resolved fences back to the pool. Note that we can only do this + // to fences we know have actually completed: unresolved fences after an error + // may still be in-flight and we don't want to reuse them. + fence_pool_->ReleaseResolved(&resolved_fences); + fence_pool_->ReleaseUnresolved(&unresolved_fences); + if (!status_.ok()) { + fence_pool_->ReleaseUnresolved(&outstanding_signals_); + } + + return status_; +} + +} // namespace vulkan +} // namespace hal +} // namespace iree diff --git a/hal/vulkan/legacy_fence.h b/hal/vulkan/legacy_fence.h new file mode 100644 index 000000000000..9f4ceee4d443 --- /dev/null +++ b/hal/vulkan/legacy_fence.h @@ -0,0 +1,200 @@ +// Copyright 2019 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// TODO(b/140141417): share the pool (and possibly most of the fence impl) with +// the timeline semaphores fallback. + +#ifndef THIRD_PARTY_MLIR_EDGE_IREE_HAL_VULKAN_LEGACY_FENCE_H_ +#define THIRD_PARTY_MLIR_EDGE_IREE_HAL_VULKAN_LEGACY_FENCE_H_ + +#include + +#include +#include + +#include "third_party/absl/base/thread_annotations.h" +#include "third_party/absl/synchronization/mutex.h" +#include "third_party/mlir_edge/iree/base/intrusive_list.h" +#include "third_party/mlir_edge/iree/base/ref_ptr.h" +#include "third_party/mlir_edge/iree/base/status.h" +#include "third_party/mlir_edge/iree/hal/fence.h" +#include "third_party/mlir_edge/iree/hal/vulkan/handle_util.h" + +namespace iree { +namespace hal { +namespace vulkan { + +// An outstanding legacy fence signal for a particular timeline value. +// Each signal to a new value gets a new VkFence and these are stored in a +// LegacyFence to quickly scan and process signaled fences. +// +// Must be externally synchronized via the LegacyFence mutex. +struct OutstandingFenceSignal : public IntrusiveLinkBase { + // Allocated fence that is passed to vkQueueSubmit/vkWaitForFences. + // Represents a point in the timeline of value. + VkFence fence = VK_NULL_HANDLE; + + // Value that the fence payload should be when the fence is signaled. + // Note that since fences may resolve out of order we still need to check that + // we are only ever advancing the timeline and not just setting this value. + uint64_t value = UINT64_MAX; + + // True when the fence has been submitted and is pending on the device. + bool is_pending = false; +}; + +// A pool of VkFences that can be used by LegacyFence to simulate individual +// payload value signaling. Note that we prefer a pool instead of a ringbuffer +// as we want to allow out-of-order completion. +class LegacyFencePool final : public RefObject { + public: + static constexpr int kMaxInFlightFenceCount = 64; + + // Allocates a new fence pool and all fences. + static StatusOr> Create( + ref_ptr logical_device); + + ~LegacyFencePool(); + + const ref_ptr& logical_device() const { + return logical_device_; + } + const ref_ptr& syms() const { + return logical_device_->syms(); + } + + // Acquires a fence from the pool for use by the caller. + // The fence is guaranteed to not be in-flight and will have been reset to an + // unsignaled state. + // + // Returns RESOURCE_EXHAUSTED if the pool has no more available fences. + // Callers are expected to handle this by waiting on previous fences or for + // complete device idle. Yes, that's as bad as it sounds, and if we start + // seeing that we should bump up the max count. + StatusOr Acquire(); + + // Releases one or more fences back to the pool. + // The fences must either be signaled or not be in-flight. + void ReleaseResolved(IntrusiveList* fence_signals); + + // Releases one or more unresolved fences back to the pool. + // These may be in any state and will be assumed as untouchable. + void ReleaseUnresolved(IntrusiveList* fence_signals); + + private: + explicit LegacyFencePool(ref_ptr logical_device); + + Status PreallocateFences() ABSL_LOCKS_EXCLUDED(mutex_); + + ref_ptr logical_device_; + + absl::Mutex mutex_; + std::array storage_ + ABSL_GUARDED_BY(mutex_); + IntrusiveList unused_fences_ ABSL_GUARDED_BY(mutex_); + IntrusiveList unresolved_fences_ + ABSL_GUARDED_BY(mutex_); +}; + +// A fence implemented using a pool of native VkFences. +// This is supported unconditionally on all versions of Vulkan. When timeline +// semaphores are available we prefer using those instead and this is only +// present as a fallback. We keep this implementation separate so that it can be +// compiled out when the target is known to have the extension. +// +// Simulation of timeline semaphore-based fences is done via a pool of native +// VkFences that each represent a single signaled value. This means that worst +// case we are using one fence per submit however that's no different than if +// we did anything else. Though we can't cancel previously-queued fences when +// increasing values are signaled we can be clever when querying and releasing +// by always walking in reverse relying on the monotonically increasing values. +// +// Valid usage patterns we need to handle: +// 1. fence signaled and waited on (common case) +// 2. fence waited on before beginning signaling +// 3. fence signaled and never waited on +// +// Case 1 is fairly straightforward: we acquire a VkFence, pass that to the +// queue submit, and then vkWaitForFences/query it for completion. +// +// Case 2 requires that we reserve a fence during the wait so that we can pass +// it to vkWaitForFences and track it such that we can reuse it during a future +// signal operation. Since we don't know during signaling if the specific value +// we waited on will ever have its own dedicated signal operation we need to be +// conservative and try to coalesce for correctness. This means that if a wait +// for a value of 1 is performed and we get a signal for a value of 2 we need to +// combine the two. If a signal for a value of 1 is later performed it then +// becomes a no-op. This could lead to some additional latency however that's a +// risk (or benefit!) of using timelines. Rule of thumb: don't do out of order +// signaling. +// +// Case 3 is like case 2 where we need to reserve a fence to wait on, however +// since we don't know if it will ever be signaled we need to take care to +// properly release the VkFence back to the pool for reuse: we don't want to +// return it while there are still waiters for its original event. For this +// reason we track the waiters on a given fence during their wait operation and +// if a fence is released with waiters active we put them in a special +// unresolved until the waiters continue on. +class LegacyFence final : public Fence { + public: + // Waits for one or more (or all) fences to reach or exceed the given values. + static Status WaitForFences(VkDeviceHandle* logical_device, + absl::Span fences, + bool wait_all, absl::Time deadline); + + LegacyFence(ref_ptr fence_pool, uint64_t initial_value); + ~LegacyFence() override; + + Status status() const override; + + StatusOr QueryValue() override; + + // Acquires a new fence for signaling a specific value. + StatusOr AcquireSignalFence(uint64_t value); + + private: + // Acquires a new fence for waiting on a specific value. + // Returns VK_NULL_HANDLE if the fence already resolved and the sticky error + // if the fence is in an error state. + StatusOr AcquireWaitFence(uint64_t value); + + // Runs down the outstanding fences list and resolves to the latest signaled + // value. Will early exit if the value moves beyond |upper_value|. + Status TryResolveOutstandingFences(uint64_t upper_value) + ABSL_LOCKS_EXCLUDED(mutex_); + Status TryResolveOutstandingFencesLocked(uint64_t upper_value) + ABSL_EXCLUSIVE_LOCKS_REQUIRED(mutex_); + + ref_ptr fence_pool_; + + // The current highest value of the fence as verified during a wait or query. + // Kept outside of |mutex_| so that queries do not require a lock. + std::atomic value_; + + mutable absl::Mutex mutex_; + + // Sticky status failure value set on first failure. + Status status_ ABSL_GUARDED_BY(mutex_); + + // Outstanding VkFences representing signal values. + // Expected to be sorted in ascending order by value. + IntrusiveList outstanding_signals_ + ABSL_GUARDED_BY(mutex_); +}; + +} // namespace vulkan +} // namespace hal +} // namespace iree + +#endif // THIRD_PARTY_MLIR_EDGE_IREE_HAL_VULKAN_LEGACY_FENCE_H_ diff --git a/hal/vulkan/native_binary_semaphore.cc b/hal/vulkan/native_binary_semaphore.cc new file mode 100644 index 000000000000..79a4a5092c1b --- /dev/null +++ b/hal/vulkan/native_binary_semaphore.cc @@ -0,0 +1,32 @@ +// Copyright 2019 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "third_party/mlir_edge/iree/hal/vulkan/native_binary_semaphore.h" + +namespace iree { +namespace hal { +namespace vulkan { + +NativeBinarySemaphore::NativeBinarySemaphore( + ref_ptr logical_device, VkSemaphore handle) + : logical_device_(std::move(logical_device)), handle_(handle) {} + +NativeBinarySemaphore::~NativeBinarySemaphore() { + logical_device_->syms()->vkDestroySemaphore(*logical_device_, handle_, + logical_device_->allocator()); +} + +} // namespace vulkan +} // namespace hal +} // namespace iree diff --git a/hal/vulkan/native_binary_semaphore.h b/hal/vulkan/native_binary_semaphore.h new file mode 100644 index 000000000000..d1e22a88993b --- /dev/null +++ b/hal/vulkan/native_binary_semaphore.h @@ -0,0 +1,46 @@ +// Copyright 2019 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#ifndef THIRD_PARTY_MLIR_EDGE_IREE_HAL_VULKAN_NATIVE_BINARY_SEMAPHORE_H_ +#define THIRD_PARTY_MLIR_EDGE_IREE_HAL_VULKAN_NATIVE_BINARY_SEMAPHORE_H_ + +#include + +#include "third_party/mlir_edge/iree/hal/semaphore.h" +#include "third_party/mlir_edge/iree/hal/vulkan/handle_util.h" + +namespace iree { +namespace hal { +namespace vulkan { + +// A binary semaphore implemented using the native VkSemaphore type. +// This is supported unconditionally on all versions of Vulkan. +class NativeBinarySemaphore final : public BinarySemaphore { + public: + NativeBinarySemaphore(ref_ptr logical_device, + VkSemaphore handle); + ~NativeBinarySemaphore() override; + + VkSemaphore handle() const { return handle_; } + + private: + ref_ptr logical_device_; + VkSemaphore handle_; +}; + +} // namespace vulkan +} // namespace hal +} // namespace iree + +#endif // THIRD_PARTY_MLIR_EDGE_IREE_HAL_VULKAN_NATIVE_BINARY_SEMAPHORE_H_ diff --git a/hal/vulkan/native_event.cc b/hal/vulkan/native_event.cc new file mode 100644 index 000000000000..2defc6146808 --- /dev/null +++ b/hal/vulkan/native_event.cc @@ -0,0 +1,31 @@ +// Copyright 2019 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "third_party/mlir_edge/iree/hal/vulkan/native_event.h" + +namespace iree { +namespace hal { +namespace vulkan { + +NativeEvent::NativeEvent(ref_ptr logical_device, VkEvent handle) + : logical_device_(std::move(logical_device)), handle_(handle) {} + +NativeEvent::~NativeEvent() { + logical_device_->syms()->vkDestroyEvent(*logical_device_, handle_, + logical_device_->allocator()); +} + +} // namespace vulkan +} // namespace hal +} // namespace iree diff --git a/hal/vulkan/native_event.h b/hal/vulkan/native_event.h new file mode 100644 index 000000000000..62f836092f64 --- /dev/null +++ b/hal/vulkan/native_event.h @@ -0,0 +1,44 @@ +// Copyright 2019 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#ifndef THIRD_PARTY_MLIR_EDGE_IREE_HAL_VULKAN_NATIVE_EVENT_H_ +#define THIRD_PARTY_MLIR_EDGE_IREE_HAL_VULKAN_NATIVE_EVENT_H_ + +#include + +#include "third_party/mlir_edge/iree/hal/event.h" +#include "third_party/mlir_edge/iree/hal/vulkan/handle_util.h" + +namespace iree { +namespace hal { +namespace vulkan { + +// An event implemented with the native VkEvent type. +class NativeEvent final : public Event { + public: + NativeEvent(ref_ptr logical_device, VkEvent handle); + ~NativeEvent() override; + + VkEvent handle() const { return handle_; } + + private: + ref_ptr logical_device_; + VkEvent handle_; +}; + +} // namespace vulkan +} // namespace hal +} // namespace iree + +#endif // THIRD_PARTY_MLIR_EDGE_IREE_HAL_VULKAN_NATIVE_EVENT_H_ diff --git a/hal/vulkan/pipeline_cache.cc b/hal/vulkan/pipeline_cache.cc new file mode 100644 index 000000000000..ca5c998c2210 --- /dev/null +++ b/hal/vulkan/pipeline_cache.cc @@ -0,0 +1,235 @@ +// Copyright 2019 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "third_party/mlir_edge/iree/hal/vulkan/pipeline_cache.h" + +#include "third_party/absl/synchronization/mutex.h" +#include "third_party/absl/types/source_location.h" +#include "third_party/flatbuffers/include/flatbuffers/flatbuffers.h" +#include "third_party/mlir_edge/iree/base/status.h" +#include "third_party/mlir_edge/iree/base/tracing.h" +#include "third_party/mlir_edge/iree/hal/executable_format.h" +#include "third_party/mlir_edge/iree/hal/vulkan/status_util.h" +#include "third_party/mlir_edge/iree/schemas/spirv_executable_def_generated.h" + +namespace iree { +namespace hal { +namespace vulkan { + +PipelineCache::PipelineCache(const ref_ptr& logical_device) + : logical_device_(add_ref(logical_device)) {} + +PipelineCache::~PipelineCache() { + IREE_TRACE_SCOPE0("PipelineCache::dtor"); + ClearLayoutCaches(); +} + +bool PipelineCache::CanPrepareFormat(ExecutableFormat format) const { + return format == kExecutableFormatSpirV; +} + +StatusOr> PipelineCache::PrepareExecutable( + ExecutableCachingModeBitfield mode, const ExecutableSpec& spec) { + IREE_TRACE_SCOPE0("PipelineCache::PrepareExecutable"); + if (!CanPrepareFormat(spec.format)) { + return UnimplementedErrorBuilder(ABSL_LOC) + << "Unsupported 4CC format: 0x" << std::hex << spec.format; + } + if (spec.executable_data.size() <= 4 || + !SpirVExecutableDefBufferHasIdentifier(spec.executable_data.data())) { + return InvalidArgumentErrorBuilder(ABSL_LOC) + << "Supplied executable data does not contain a SpirVExecutableDef"; + } + + // Get the SPIR-V executable def flatbuffer. + const auto& spirv_executable_def = + *::flatbuffers::GetRoot(spec.executable_data.data()); + + // Create (or reuse) a pipeline layout. + if (!spirv_executable_def.pipeline_layout()) { + return InvalidArgumentErrorBuilder(ABSL_LOC) + << "Missing pipeline layout def"; + } + ASSIGN_OR_RETURN( + auto pipeline_layout_entry, + LookupOrInsertPipelineLayout(*spirv_executable_def.pipeline_layout())); + + // Create the executable (which may itself own many pipelines). + ASSIGN_OR_RETURN(auto executable, PipelineExecutable::Create( + logical_device_, + /*pipeline_cache=*/VK_NULL_HANDLE, + pipeline_layout_entry->pipeline_layout, + pipeline_layout_entry->descriptor_sets, + mode, spirv_executable_def)); + return executable; +} + +StatusOr +PipelineCache::LookupOrInsertPipelineLayout( + const VkPipelineLayoutDef& pipeline_layout_def) { + IREE_TRACE_SCOPE0("PipelineCache::LookupOrInsertPipelineLayout"); + absl::MutexLock lock(&mutex_); + + // Build a list of the required descriptor set layouts and push constants. + // If we were being fast about this we would just hash the def and directly + // look up the pipeline layout. + PipelineDescriptorSets descriptor_sets; + descriptor_sets.buffer_binding_set = pipeline_layout_def.buffer_binding_set(); + descriptor_sets.buffer_binding_set_layout = VK_NULL_HANDLE; + absl::InlinedVector descriptor_set_layouts; + if (pipeline_layout_def.descriptor_set_layouts()) { + const auto& layout_defs = *pipeline_layout_def.descriptor_set_layouts(); + descriptor_set_layouts.resize(layout_defs.size()); + for (int i = 0; i < descriptor_set_layouts.size(); ++i) { + if (!layout_defs[i]) { + return InvalidArgumentErrorBuilder(ABSL_LOC) << "Missing layout def"; + } + ASSIGN_OR_RETURN(descriptor_set_layouts[i], + LookupOrInsertDescriptorSetLayout(*layout_defs[i])); + if (i == pipeline_layout_def.buffer_binding_set()) { + descriptor_sets.buffer_binding_set_layout = descriptor_set_layouts[i]; + descriptor_sets.buffer_binding_set_map.resize( + layout_defs[i]->bindings()->size()); + for (int j = 0; j < layout_defs[i]->bindings()->size(); ++j) { + descriptor_sets.buffer_binding_set_map[j] = + layout_defs[i]->bindings()->Get(j)->binding(); + } + } + } + } + + absl::InlinedVector push_constant_ranges; + if (pipeline_layout_def.push_constant_ranges()) { + const auto& range_defs = *pipeline_layout_def.push_constant_ranges(); + push_constant_ranges.resize(range_defs.size()); + for (int i = 0; i < push_constant_ranges.size(); ++i) { + if (!range_defs[i]) { + return InvalidArgumentErrorBuilder(ABSL_LOC) + << "Missing push constant range def"; + } + push_constant_ranges[i].stageFlags = range_defs[i]->stage_flags(); + push_constant_ranges[i].offset = range_defs[i]->offset(); + push_constant_ranges[i].size = range_defs[i]->size(); + } + } + + // Scan for an existing pipeline layout that matches the descriptor sets. + for (auto& entry : pipeline_layout_cache_) { + if (entry.descriptor_set_layouts.size() != descriptor_set_layouts.size() || + entry.push_constant_ranges.size() != push_constant_ranges.size()) { + continue; + } + if (std::memcmp( + descriptor_set_layouts.data(), entry.descriptor_set_layouts.data(), + descriptor_set_layouts.size() * sizeof(VkDescriptorSetLayout)) == + 0 && + std::memcmp( + push_constant_ranges.data(), entry.push_constant_ranges.data(), + push_constant_ranges.size() * sizeof(VkPushConstantRange)) == 0) { + return &entry; + } + } + + VkPipelineLayoutCreateInfo create_info; + create_info.sType = VK_STRUCTURE_TYPE_PIPELINE_LAYOUT_CREATE_INFO; + create_info.pNext = nullptr; + create_info.flags = 0; + create_info.setLayoutCount = descriptor_set_layouts.size(); + create_info.pSetLayouts = descriptor_set_layouts.data(); + create_info.pushConstantRangeCount = push_constant_ranges.size(); + create_info.pPushConstantRanges = push_constant_ranges.data(); + + // Create and insert into the cache. + VkPipelineLayout pipeline_layout = VK_NULL_HANDLE; + VK_RETURN_IF_ERROR(syms()->vkCreatePipelineLayout( + *logical_device_, &create_info, logical_device_->allocator(), + &pipeline_layout)); + pipeline_layout_cache_.push_back({std::move(descriptor_set_layouts), + std::move(push_constant_ranges), + pipeline_layout, descriptor_sets}); + return &pipeline_layout_cache_.back(); +} + +StatusOr +PipelineCache::LookupOrInsertDescriptorSetLayout( + const VkDescriptorSetLayoutDef& descriptor_set_layout_def) { + // Build a list of bindings in the set. + // If we were being fast we would hash the bindings and directly lookup + // without doing this allocation. + absl::InlinedVector bindings; + if (descriptor_set_layout_def.bindings()) { + const auto& binding_defs = *descriptor_set_layout_def.bindings(); + bindings.resize(binding_defs.size()); + for (int i = 0; i < binding_defs.size(); ++i) { + bindings[i].binding = binding_defs[i]->binding(); + bindings[i].descriptorType = + static_cast(binding_defs[i]->descriptor_type()); + bindings[i].descriptorCount = binding_defs[i]->descriptor_count(); + bindings[i].stageFlags = binding_defs[i]->stage_flags(); + bindings[i].pImmutableSamplers = nullptr; + } + } + + // Scan for an existing descriptor set layout that matches the bindings. + for (auto& entry : descriptor_set_layout_cache_) { + if (entry.bindings.size() != bindings.size()) continue; + if (std::memcmp(bindings.data(), entry.bindings.data(), + bindings.size() * sizeof(VkDescriptorSetLayoutBinding)) == + 0) { + return entry.descriptor_set_layout; + } + } + + VkDescriptorSetLayoutCreateInfo create_info; + create_info.sType = VK_STRUCTURE_TYPE_DESCRIPTOR_SET_LAYOUT_CREATE_INFO; + create_info.pNext = nullptr; + create_info.flags = 0; + if (logical_device_->enabled_extensions().push_descriptors) { + // Note that we can *only* use push descriptor sets if we set this create + // flag. That's fine, though, as the command buffer recording logic always + // prefers the extension if available. + create_info.flags |= + VK_DESCRIPTOR_SET_LAYOUT_CREATE_PUSH_DESCRIPTOR_BIT_KHR; + } + create_info.bindingCount = bindings.size(); + create_info.pBindings = bindings.data(); + + // Create and insert into the cache. + VkDescriptorSetLayout descriptor_set_layout = VK_NULL_HANDLE; + VK_RETURN_IF_ERROR(syms()->vkCreateDescriptorSetLayout( + *logical_device_, &create_info, logical_device_->allocator(), + &descriptor_set_layout)); + descriptor_set_layout_cache_.push_back( + {std::move(bindings), descriptor_set_layout}); + return descriptor_set_layout; +} + +void PipelineCache::ClearLayoutCaches() { + absl::MutexLock lock(&mutex_); + for (auto& entry : pipeline_layout_cache_) { + syms()->vkDestroyPipelineLayout(*logical_device_, entry.pipeline_layout, + logical_device_->allocator()); + } + pipeline_layout_cache_.clear(); + for (auto& entry : descriptor_set_layout_cache_) { + syms()->vkDestroyDescriptorSetLayout(*logical_device_, + entry.descriptor_set_layout, + logical_device_->allocator()); + } + descriptor_set_layout_cache_.clear(); +} + +} // namespace vulkan +} // namespace hal +} // namespace iree diff --git a/hal/vulkan/pipeline_cache.h b/hal/vulkan/pipeline_cache.h new file mode 100644 index 000000000000..9d520601e0eb --- /dev/null +++ b/hal/vulkan/pipeline_cache.h @@ -0,0 +1,85 @@ +// Copyright 2019 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#ifndef THIRD_PARTY_MLIR_EDGE_IREE_HAL_VULKAN_PIPELINE_CACHE_H_ +#define THIRD_PARTY_MLIR_EDGE_IREE_HAL_VULKAN_PIPELINE_CACHE_H_ + +#include + +#include "third_party/absl/base/thread_annotations.h" +#include "third_party/absl/container/inlined_vector.h" +#include "third_party/absl/synchronization/mutex.h" +#include "third_party/mlir_edge/iree/hal/executable.h" +#include "third_party/mlir_edge/iree/hal/executable_cache.h" +#include "third_party/mlir_edge/iree/hal/vulkan/handle_util.h" +#include "third_party/mlir_edge/iree/hal/vulkan/pipeline_executable.h" +#include "third_party/mlir_edge/iree/schemas/spirv_executable_def_generated.h" + +namespace iree { +namespace hal { +namespace vulkan { + +class PipelineCache final : public ExecutableCache { + public: + explicit PipelineCache(const ref_ptr& logical_device); + ~PipelineCache() override; + + const ref_ptr& syms() const { + return logical_device_->syms(); + } + + bool CanPrepareFormat(ExecutableFormat format) const override; + + StatusOr> PrepareExecutable( + ExecutableCachingModeBitfield mode, const ExecutableSpec& spec) override; + + private: + struct CachedDescriptorSetLayout { + absl::InlinedVector bindings; + VkDescriptorSetLayout descriptor_set_layout; + }; + struct CachedPipelineLayout { + absl::InlinedVector descriptor_set_layouts; + absl::InlinedVector push_constant_ranges; + VkPipelineLayout pipeline_layout; + PipelineDescriptorSets descriptor_sets; + }; + + StatusOr LookupOrInsertPipelineLayout( + const VkPipelineLayoutDef& pipeline_layout_def) + ABSL_LOCKS_EXCLUDED(mutex_); + StatusOr LookupOrInsertDescriptorSetLayout( + const VkDescriptorSetLayoutDef& descriptor_set_layout_def) + ABSL_EXCLUSIVE_LOCKS_REQUIRED(mutex_); + void ClearLayoutCaches() ABSL_LOCKS_EXCLUDED(mutex_); + + ref_ptr logical_device_; + + // A "cache" of descriptor set and pipeline layouts for various values. + // We never evict and just do a simple linear scan on lookup. This is fine for + // now as we only support a single descriptor type and really we only need to + // check for binding count. As we go toward more general usage of descriptors + // (images/etc) we will likely want to change this to a real cache. + absl::Mutex mutex_; + std::vector descriptor_set_layout_cache_ + ABSL_GUARDED_BY(mutex_); + std::vector pipeline_layout_cache_ + ABSL_GUARDED_BY(mutex_); +}; + +} // namespace vulkan +} // namespace hal +} // namespace iree + +#endif // THIRD_PARTY_MLIR_EDGE_IREE_HAL_VULKAN_PIPELINE_CACHE_H_ diff --git a/hal/vulkan/pipeline_executable.cc b/hal/vulkan/pipeline_executable.cc new file mode 100644 index 000000000000..cee13e65e19c --- /dev/null +++ b/hal/vulkan/pipeline_executable.cc @@ -0,0 +1,138 @@ +// Copyright 2019 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "third_party/mlir_edge/iree/hal/vulkan/pipeline_executable.h" + +#include "third_party/absl/container/inlined_vector.h" +#include "third_party/absl/types/source_location.h" +#include "third_party/mlir_edge/iree/base/memory.h" +#include "third_party/mlir_edge/iree/base/status.h" +#include "third_party/mlir_edge/iree/base/tracing.h" +#include "third_party/mlir_edge/iree/hal/vulkan/status_util.h" + +namespace iree { +namespace hal { +namespace vulkan { + +// static +StatusOr> PipelineExecutable::Create( + const ref_ptr& logical_device, + VkPipelineCache pipeline_cache, VkPipelineLayout pipeline_layout, + PipelineDescriptorSets descriptor_sets, ExecutableCachingModeBitfield mode, + const SpirVExecutableDef& spirv_executable_def) { + IREE_TRACE_SCOPE0("PipelineExecutable::Create"); + const auto& syms = logical_device->syms(); + if (!spirv_executable_def.entry_points() || + spirv_executable_def.entry_points()->size() == 0) { + return InvalidArgumentErrorBuilder(ABSL_LOC) << "No entry points defined"; + } + if (!spirv_executable_def.code()) { + return InvalidArgumentErrorBuilder(ABSL_LOC) << "No SPIR-V code present"; + } + const auto& code = *spirv_executable_def.code(); + + // Create the shader module. + VkShaderModuleCreateInfo shader_module_create_info; + shader_module_create_info.sType = VK_STRUCTURE_TYPE_SHADER_MODULE_CREATE_INFO; + shader_module_create_info.pNext = nullptr; + shader_module_create_info.flags = 0; + shader_module_create_info.codeSize = code.size() * sizeof(uint32_t); + shader_module_create_info.pCode = code.data(); + VkShaderModule shader_module = VK_NULL_HANDLE; + VK_RETURN_IF_ERROR( + syms->vkCreateShaderModule(*logical_device, &shader_module_create_info, + logical_device->allocator(), &shader_module)); + + // We only need to keep this around during pipeline creation so ensure we + // always clean it up when we exit this function. + auto shader_module_cleanup = MakeCleanup([&logical_device, shader_module]() { + logical_device->syms()->vkDestroyShaderModule( + *logical_device, shader_module, logical_device->allocator()); + }); + + // Create pipelines for each entry point. + const auto& entry_points = *spirv_executable_def.entry_points(); + absl::InlinedVector pipeline_create_infos; + pipeline_create_infos.resize(entry_points.size()); + for (int entry_ordinal = 0; entry_ordinal < entry_points.size(); + ++entry_ordinal) { + auto& pipeline_create_info = pipeline_create_infos[entry_ordinal]; + pipeline_create_info.sType = VK_STRUCTURE_TYPE_COMPUTE_PIPELINE_CREATE_INFO; + pipeline_create_info.pNext = nullptr; + pipeline_create_info.flags = 0; + if (!AllBitsSet(mode, ExecutableCachingMode::kAllowOptimization)) { + pipeline_create_info.flags |= VK_PIPELINE_CREATE_DISABLE_OPTIMIZATION_BIT; + } + if (entry_ordinal == 0) { + pipeline_create_info.flags |= VK_PIPELINE_CREATE_ALLOW_DERIVATIVES_BIT; + } else { + pipeline_create_info.flags |= VK_PIPELINE_CREATE_DERIVATIVE_BIT; + } + pipeline_create_info.layout = pipeline_layout; + pipeline_create_info.basePipelineHandle = VK_NULL_HANDLE; + pipeline_create_info.basePipelineIndex = 0; + auto& stage_create_info = pipeline_create_info.stage; + stage_create_info.sType = + VK_STRUCTURE_TYPE_PIPELINE_SHADER_STAGE_CREATE_INFO; + stage_create_info.pNext = nullptr; + stage_create_info.flags = 0; + stage_create_info.stage = VK_SHADER_STAGE_COMPUTE_BIT; + stage_create_info.module = shader_module; + stage_create_info.pName = entry_points[entry_ordinal]->c_str(); + stage_create_info.pSpecializationInfo = nullptr; + } + absl::InlinedVector pipelines; + pipelines.resize(entry_points.size()); + VK_RETURN_IF_ERROR(syms->vkCreateComputePipelines( + *logical_device, pipeline_cache, pipeline_create_infos.size(), + pipeline_create_infos.data(), logical_device->allocator(), + pipelines.data())); + + auto executable = + make_ref(CtorKey{}, logical_device, pipeline_layout, + descriptor_sets, std::move(pipelines)); + executable->tag_ = + spirv_executable_def.tag() ? spirv_executable_def.tag()->str() : ""; + return executable; +} + +PipelineExecutable::PipelineExecutable( + CtorKey ctor_key, const ref_ptr& logical_device, + VkPipelineLayout pipeline_layout, PipelineDescriptorSets descriptor_sets, + absl::InlinedVector pipelines) + : logical_device_(add_ref(logical_device)), + pipeline_layout_(pipeline_layout), + descriptor_sets_(descriptor_sets), + pipelines_(std::move(pipelines)) {} + +PipelineExecutable::~PipelineExecutable() { + IREE_TRACE_SCOPE0("PipelineExecutable::dtor"); + for (auto pipeline : pipelines_) { + syms()->vkDestroyPipeline(*logical_device_, pipeline, + logical_device_->allocator()); + } + pipelines_.clear(); +} + +StatusOr PipelineExecutable::GetPipelineForEntryPoint( + int entry_ordinal) const { + if (entry_ordinal < 0 || entry_ordinal >= pipelines_.size()) { + return OutOfRangeErrorBuilder(ABSL_LOC) << "Invalid entry point ordinal"; + } + return pipelines_[entry_ordinal]; +} + +} // namespace vulkan +} // namespace hal +} // namespace iree diff --git a/hal/vulkan/pipeline_executable.h b/hal/vulkan/pipeline_executable.h new file mode 100644 index 000000000000..9cfd67134ee8 --- /dev/null +++ b/hal/vulkan/pipeline_executable.h @@ -0,0 +1,90 @@ +// Copyright 2019 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#ifndef THIRD_PARTY_MLIR_EDGE_IREE_HAL_VULKAN_PIPELINE_EXECUTABLE_H_ +#define THIRD_PARTY_MLIR_EDGE_IREE_HAL_VULKAN_PIPELINE_EXECUTABLE_H_ + +#include + +#include + +#include "third_party/mlir_edge/iree/base/status.h" +#include "third_party/mlir_edge/iree/hal/executable.h" +#include "third_party/mlir_edge/iree/hal/executable_cache.h" +#include "third_party/mlir_edge/iree/hal/executable_spec.h" +#include "third_party/mlir_edge/iree/hal/vulkan/handle_util.h" +#include "third_party/mlir_edge/iree/schemas/spirv_executable_def_generated.h" + +namespace iree { +namespace hal { +namespace vulkan { + +struct PipelineDescriptorSets { + uint32_t buffer_binding_set; + VkDescriptorSetLayout buffer_binding_set_layout; + absl::InlinedVector buffer_binding_set_map; +}; + +class PipelineExecutable final : public Executable { + public: + static StatusOr> Create( + const ref_ptr& logical_device, + VkPipelineCache pipeline_cache, VkPipelineLayout pipeline_layout, + PipelineDescriptorSets descriptor_sets, + ExecutableCachingModeBitfield mode, + const SpirVExecutableDef& spirv_executable_def); + + // Private constructor. + struct CtorKey { + private: + friend class PipelineExecutable; + CtorKey() = default; + }; + PipelineExecutable(CtorKey ctor_key, + const ref_ptr& logical_device, + VkPipelineLayout pipeline_layout, + PipelineDescriptorSets descriptor_sets, + absl::InlinedVector pipelines); + ~PipelineExecutable() override; + + const ref_ptr& syms() const { + return logical_device_->syms(); + } + + bool supports_debugging() const override { return false; } + + VkPipelineLayout pipeline_layout() const { return pipeline_layout_; } + const PipelineDescriptorSets& descriptor_sets() const { + return descriptor_sets_; + } + + bool is_matmul() const { return tag_ == "__matmul__"; } + + StatusOr GetPipelineForEntryPoint(int entry_ordinal) const; + + private: + ref_ptr logical_device_; + VkPipelineLayout pipeline_layout_; + PipelineDescriptorSets descriptor_sets_; + string tag_; + + // One pipeline per entry point. + absl::InlinedVector pipelines_; +}; + +} // namespace vulkan +} // namespace hal +} // namespace iree + +#endif // THIRD_PARTY_MLIR_EDGE_IREE_HAL_VULKAN_PIPELINE_EXECUTABLE_H_ diff --git a/hal/vulkan/status_util.cc b/hal/vulkan/status_util.cc new file mode 100644 index 000000000000..d48e8089e95e --- /dev/null +++ b/hal/vulkan/status_util.cc @@ -0,0 +1,231 @@ +// Copyright 2019 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "third_party/mlir_edge/iree/hal/vulkan/status_util.h" + +#include "third_party/mlir_edge/iree/base/status.h" + +namespace iree { +namespace hal { +namespace vulkan { + +Status VkResultToStatus(VkResult result) { + switch (result) { + // Success codes. + case VK_SUCCESS: + // Command successfully completed. + return OkStatus(); + case VK_NOT_READY: + // A fence or query has not yet completed. + return OkStatus(); + case VK_TIMEOUT: + // A wait operation has not completed in the specified time. + return OkStatus(); + case VK_EVENT_SET: + // An event is signaled. + return OkStatus(); + case VK_EVENT_RESET: + // An event is unsignaled. + return OkStatus(); + case VK_INCOMPLETE: + // A return array was too small for the result. + return OkStatus(); + case VK_SUBOPTIMAL_KHR: + // A swapchain no longer matches the surface properties exactly, but can + // still be used to present to the surface successfully. + return OkStatus(); + + // Error codes. + case VK_ERROR_OUT_OF_HOST_MEMORY: + // A host memory allocation has failed. + return ResourceExhaustedError("VK_ERROR_OUT_OF_HOST_MEMORY"); + case VK_ERROR_OUT_OF_DEVICE_MEMORY: + // A device memory allocation has failed. + return ResourceExhaustedError("VK_ERROR_OUT_OF_DEVICE_MEMORY"); + case VK_ERROR_INITIALIZATION_FAILED: + // Initialization of an object could not be completed for + // implementation-specific reasons. + return InternalError("VK_ERROR_INITIALIZATION_FAILED"); + case VK_ERROR_DEVICE_LOST: + // The logical or physical device has been lost. + // + // A logical device may become lost for a number of + // implementation-specific reasons, indicating that pending and future + // command execution may fail and cause resources and backing memory to + // become undefined. + // + // Typical reasons for device loss will include things like execution + // timing out (to prevent denial of service), power management events, + // platform resource management, or implementation errors. + // + // When this happens, certain commands will return + // VK_ERROR_DEVICE_LOST (see Error Codes for a list of such + // commands). After any such event, the logical device is considered lost. + // It is not possible to reset the logical device to a non-lost state, + // however the lost state is specific to a logical device (VkDevice), and + // the corresponding physical device (VkPhysicalDevice) may be otherwise + // unaffected. + // + // In some cases, the physical device may also be lost, and attempting to + // create a new logical device will fail, returning VK_ERROR_DEVICE_LOST. + // This is usually indicative of a problem with the underlying + // implementation, or its connection to the host. If the physical device + // has not been lost, and a new logical device is successfully created + // from that physical device, it must be in the non-lost state. + // + // Whilst logical device loss may be recoverable, in the case of physical + // device loss, it is unlikely that an application will be able to recover + // unless additional, unaffected physical devices exist on the system. The + // error is largely informational and intended only to inform the user + // that a platform issue has occurred, and should be investigated further. + // For example, underlying hardware may have developed a fault or become + // physically disconnected from the rest of the system. In many cases, + // physical device loss may cause other more serious issues such as the + // operating system crashing; in which case it may not be reported via the + // Vulkan API. + // + // Undefined behavior caused by an application error may cause a device to + // become lost. However, such undefined behavior may also cause + // unrecoverable damage to the process, and it is then not guaranteed that + // the API objects, including the VkPhysicalDevice or the VkInstance are + // still valid or that the error is recoverable. + // + // When a device is lost, its child objects are not implicitly destroyed + // and their handles are still valid. Those objects must still be + // destroyed before their parents or the device can be destroyed (see the + // Object Lifetime section). The host address space corresponding to + // device memory mapped using vkMapMemory is still valid, and host memory + // accesses to these mapped regions are still valid, but the contents are + // undefined. It is still legal to call any API command on the device and + // child objects. + // + // Once a device is lost, command execution may fail, and commands that + // return a VkResult may return VK_ERROR_DEVICE_LOST. + // Commands that do not allow run-time errors must still operate correctly + // for valid usage and, if applicable, return valid data. + // + // Commands that wait indefinitely for device execution (namely + // vkDeviceWaitIdle, vkQueueWaitIdle, vkWaitForFences with a maximum + // timeout, and vkGetQueryPoolResults with the VK_QUERY_RESULT_WAIT_BIT + // bit set in flags) must return in finite time even in the case + // of a lost device, and return either VK_SUCCESS or + // VK_ERROR_DEVICE_LOST. For any command that may return + // VK_ERROR_DEVICE_LOST, for the purpose of determining whether a + // command buffer is in the pending state, or whether resources are + // considered in-use by the device, a return value of + // VK_ERROR_DEVICE_LOST is equivalent to VK_SUCCESS. + return InternalError("VK_ERROR_DEVICE_LOST"); + case VK_ERROR_MEMORY_MAP_FAILED: + // Mapping of a memory object has failed. + return InternalError("VK_ERROR_MEMORY_MAP_FAILED"); + case VK_ERROR_LAYER_NOT_PRESENT: + // A requested layer is not present or could not be loaded. + return UnimplementedError("VK_ERROR_LAYER_NOT_PRESENT"); + case VK_ERROR_EXTENSION_NOT_PRESENT: + // A requested extension is not supported. + return UnimplementedError("VK_ERROR_EXTENSION_NOT_PRESENT"); + case VK_ERROR_FEATURE_NOT_PRESENT: + // A requested feature is not supported. + return UnimplementedError("VK_ERROR_FEATURE_NOT_PRESENT"); + case VK_ERROR_INCOMPATIBLE_DRIVER: + // The requested version of Vulkan is not supported by the driver or is + // otherwise incompatible for implementation-specific reasons. + return FailedPreconditionError("VK_ERROR_INCOMPATIBLE_DRIVER"); + case VK_ERROR_TOO_MANY_OBJECTS: + // Too many objects of the type have already been created. + return ResourceExhaustedError("VK_ERROR_TOO_MANY_OBJECTS"); + case VK_ERROR_FORMAT_NOT_SUPPORTED: + // A requested format is not supported on this device. + return UnimplementedError("VK_ERROR_FORMAT_NOT_SUPPORTED"); + case VK_ERROR_FRAGMENTED_POOL: + // A pool allocation has failed due to fragmentation of the pool’s memory. + // This must only be returned if no attempt to allocate host or device + // memory was made to accommodate the new allocation. + return ResourceExhaustedError("VK_ERROR_FRAGMENTED_POOL"); + case VK_ERROR_OUT_OF_POOL_MEMORY: + // A pool memory allocation has failed. This must only be returned if no + // attempt to allocate host or device memory was made to accommodate the + // new allocation. If the failure was definitely due to fragmentation of + // the pool, VK_ERROR_FRAGMENTED_POOL should be returned instead. + return ResourceExhaustedError("VK_ERROR_OUT_OF_POOL_MEMORY"); + case VK_ERROR_INVALID_EXTERNAL_HANDLE: + // An external handle is not a valid handle of the specified type. + return InvalidArgumentError("VK_ERROR_INVALID_EXTERNAL_HANDLE"); + case VK_ERROR_SURFACE_LOST_KHR: + // A surface is no longer available. + return UnavailableError("VK_ERROR_SURFACE_LOST_KHR"); + case VK_ERROR_NATIVE_WINDOW_IN_USE_KHR: + // The requested window is already in use by Vulkan or another API in a + // manner which prevents it from being used again. + return InvalidArgumentError("VK_ERROR_NATIVE_WINDOW_IN_USE_KHR"); + case VK_ERROR_OUT_OF_DATE_KHR: + // A surface has changed in such a way that it is no longer compatible + // with the swapchain, and further presentation requests using the + // swapchain will fail. Applications must query the new surface properties + // and recreate their swapchain if they wish to continue presenting to the + // surface. + return FailedPreconditionError("VK_ERROR_OUT_OF_DATE_KHR"); + case VK_ERROR_INCOMPATIBLE_DISPLAY_KHR: + // The display used by a swapchain does not use the same presentable image + // layout, or is incompatible in a way that prevents sharing an image. + return InvalidArgumentError("VK_ERROR_INCOMPATIBLE_DISPLAY_KHR"); + case VK_ERROR_VALIDATION_FAILED_EXT: + // Validation layer testing failed. It is not expected that an + // application would see this this error code during normal use of the + // validation layers. + return InvalidArgumentError("VK_ERROR_VALIDATION_FAILED_EXT"); + case VK_ERROR_INVALID_SHADER_NV: + // One or more shaders failed to compile or link. More details are + // reported back to the application when the validation layer is enabled + // using the extension VK_EXT_debug_report. + return InvalidArgumentError("VK_ERROR_INVALID_SHADER_NV"); + case VK_ERROR_INVALID_DRM_FORMAT_MODIFIER_PLANE_LAYOUT_EXT: + // When creating an image with + // VkImageDrmFormatModifierExplicitCreateInfoEXT, it is the application’s + // responsibility to satisfy all Valid Usage requirements. However, the + // implementation must validate that the provided pPlaneLayouts, when + // combined with the provided drmFormatModifier and other creation + // parameters in VkImageCreateInfo and its pNext chain, produce a valid + // image. (This validation is necessarily implementation-dependent and + // outside the scope of Vulkan, and therefore not described by Valid Usage + // requirements). If this validation fails, then vkCreateImage returns + // VK_ERROR_INVALID_DRM_FORMAT_MODIFIER_PLANE_LAYOUT_EXT. + return InvalidArgumentError( + "VK_ERROR_INVALID_DRM_FORMAT_MODIFIER_PLANE_LAYOUT_EXT"); + case VK_ERROR_FRAGMENTATION_EXT: + // A descriptor pool creation has failed due to fragmentation. + return ResourceExhaustedError("VK_ERROR_FRAGMENTATION_EXT"); + case VK_ERROR_NOT_PERMITTED_EXT: + // When creating a queue, the caller does not have sufficient privileges + // to request to acquire a priority above the default priority + // (VK_QUEUE_GLOBAL_PRIORITY_MEDIUM_EXT). + return PermissionDeniedError("VK_ERROR_NOT_PERMITTED_EXT"); + case VK_ERROR_INVALID_DEVICE_ADDRESS_EXT: + // A buffer creation failed because the requested address is not + // available. + return OutOfRangeError("VK_ERROR_INVALID_DEVICE_ADDRESS_EXT"); + case VK_ERROR_FULL_SCREEN_EXCLUSIVE_MODE_LOST_EXT: + // An operation on a swapchain created with + // VK_FULL_SCREEN_EXCLUSIVE_APPLICATION_CONTROLLED_EXT failed as it did + // not have exlusive full-screen access. This may occur due to + // implementation-dependent reasons, outside of the application’s control. + return UnavailableError("VK_ERROR_FULL_SCREEN_EXCLUSIVE_MODE_LOST_EXT"); + default: + return UnknownError(std::to_string(result)); + } +} + +} // namespace vulkan +} // namespace hal +} // namespace iree diff --git a/hal/vulkan/status_util.h b/hal/vulkan/status_util.h new file mode 100644 index 000000000000..7bec1791f291 --- /dev/null +++ b/hal/vulkan/status_util.h @@ -0,0 +1,87 @@ +// Copyright 2019 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#ifndef THIRD_PARTY_MLIR_EDGE_IREE_HAL_VULKAN_STATUS_UTIL_H_ +#define THIRD_PARTY_MLIR_EDGE_IREE_HAL_VULKAN_STATUS_UTIL_H_ + +#include + +#include "third_party/mlir_edge/iree/base/status.h" + +namespace iree { +namespace hal { +namespace vulkan { + +// RETURN_IF_ERROR but implicitly converts the VkResult return value to +// a Status. +// +// Usage: +// VK_RETURN_IF_ERROR(vkDoThing(...)); +#define VK_RETURN_IF_ERROR(expr) \ + RETURN_IF_ERROR(::iree::hal::vulkan::VkResultToStatus(expr)) + +// CHECK_OK but implicitly converts the VkResults return value to a +// Status and checks that it is OkStatus. +// +// Usage: +// VK_CHECK_OK(vkDoThing(...)); +#define VK_CHECK_OK(expr) CHECK_OK(::iree::hal::vulkan::VkResultToStatus(expr)) + +// Converts a VkResult to a Status object. +// +// Vulkan considers the following as "success codes" and users should ensure +// they first check the result prior to converting: +// +// - VK_SUCCESS -> OkStatus() +// - VK_NOT_READY -> OkStatus() +// - VK_TIMEOUT -> OkStatus() +// - VK_EVENT_SET -> OkStatus() +// - VK_EVENT_RESET -> OkStatus() +// - VK_INCOMPLETE -> OkStatus() +// - VK_SUBOPTIMAL_KHR -> OkStatus() +// +// The rest are considered as "error codes": +// +// - VK_ERROR_OUT_OF_HOST_MEMORY -> ResourceExhaustedError("VK...") +// - VK_ERROR_OUT_OF_DEVICE_MEMORY -> ResourceExhaustedError("VK...") +// - VK_ERROR_INITIALIZATION_FAILED -> InternalError("VK...") +// - VK_ERROR_DEVICE_LOST -> InternalError("VK...") +// - VK_ERROR_MEMORY_MAP_FAILED -> InternalError("VK...") +// - VK_ERROR_LAYER_NOT_PRESENT -> NotFoundError("VK...") +// - VK_ERROR_EXTENSION_NOT_PRESENT -> NotFoundError("VK...") +// - VK_ERROR_FEATURE_NOT_PRESENT -> NotFoundError("VK...") +// - VK_ERROR_INCOMPATIBLE_DRIVER -> FailedPreconditionError("VK...") +// - VK_ERROR_TOO_MANY_OBJECTS -> ResourceExhaustedError("VK...") +// - VK_ERROR_FORMAT_NOT_SUPPORTED -> UnimplementedError("VK...") +// - VK_ERROR_FRAGMENTED_POOL -> ResourceExhaustedError("VK...") +// - VK_ERROR_OUT_OF_POOL_MEMORY -> ResourceExhaustedError("VK...") +// - VK_ERROR_INVALID_EXTERNAL_HANDLE -> InvalidArgumentError("VK...") +// - VK_ERROR_SURFACE_LOST_KHR -> InternalError("VK...") +// - VK_ERROR_NATIVE_WINDOW_IN_USE_KHR -> InternalError("VK...") +// - VK_ERROR_OUT_OF_DATE_KHR -> InternalError("VK...") +// - VK_ERROR_INCOMPATIBLE_DISPLAY_KHR -> InternalError("VK...") +// - VK_ERROR_VALIDATION_FAILED_EXT -> InternalError("VK...") +// - VK_ERROR_INVALID_SHADER_NV -> InternalError("VK...") +// - VK_ERROR_INVALID_DRM_FORMAT_MODIFIER_PLANE_LAYOUT_EXT -> InternalError +// - VK_ERROR_FRAGMENTATION_EXT -> ResourceExhaustedError("VK...") +// - VK_ERROR_NOT_PERMITTED_EXT -> PermissionDeniedError("VK...") +// - VK_ERROR_INVALID_DEVICE_ADDRESS_EXT -> OutOfRangeError("VK...") +// - VK_ERROR_FULL_SCREEN_EXCLUSIVE_MODE_LOST_EXT -> InternalError("VK...") +Status VkResultToStatus(VkResult result); + +} // namespace vulkan +} // namespace hal +} // namespace iree + +#endif // THIRD_PARTY_MLIR_EDGE_IREE_HAL_VULKAN_STATUS_UTIL_H_ diff --git a/hal/vulkan/vma_allocator.cc b/hal/vulkan/vma_allocator.cc new file mode 100644 index 000000000000..eeaf2bdc6eef --- /dev/null +++ b/hal/vulkan/vma_allocator.cc @@ -0,0 +1,248 @@ +// Copyright 2019 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "third_party/mlir_edge/iree/hal/vulkan/vma_allocator.h" + +#include "third_party/absl/flags/flag.h" +#include "third_party/absl/memory/memory.h" +#include "third_party/absl/types/source_location.h" +#include "third_party/mlir_edge/iree/base/status.h" +#include "third_party/mlir_edge/iree/base/tracing.h" +#include "third_party/mlir_edge/iree/hal/buffer.h" +#include "third_party/mlir_edge/iree/hal/vulkan/status_util.h" +#include "third_party/mlir_edge/iree/hal/vulkan/vma_buffer.h" + +#if VMA_RECORDING_ENABLED +ABSL_FLAG(std::string, vma_recording_file, "", + "File path to write a CSV containing the VMA recording."); +ABSL_FLAG(bool, vma_recording_flush_after_call, false, + "Flush the VMA recording file after every call (useful if " + "crashing/not exiting cleanly)."); +#endif // VMA_RECORDING_ENABLED + +namespace iree { +namespace hal { +namespace vulkan { + +// static +StatusOr> VmaAllocator::Create( + VkPhysicalDevice physical_device, + const ref_ptr& logical_device) { + IREE_TRACE_SCOPE0("VmaAllocator::Create"); + + const auto& syms = logical_device->syms(); + VmaVulkanFunctions vulkan_fns; + vulkan_fns.vkGetPhysicalDeviceProperties = + syms->vkGetPhysicalDeviceProperties; + vulkan_fns.vkGetPhysicalDeviceMemoryProperties = + syms->vkGetPhysicalDeviceMemoryProperties; + vulkan_fns.vkAllocateMemory = syms->vkAllocateMemory; + vulkan_fns.vkFreeMemory = syms->vkFreeMemory; + vulkan_fns.vkMapMemory = syms->vkMapMemory; + vulkan_fns.vkUnmapMemory = syms->vkUnmapMemory; + vulkan_fns.vkFlushMappedMemoryRanges = syms->vkFlushMappedMemoryRanges; + vulkan_fns.vkInvalidateMappedMemoryRanges = + syms->vkInvalidateMappedMemoryRanges; + vulkan_fns.vkBindBufferMemory = syms->vkBindBufferMemory; + vulkan_fns.vkBindImageMemory = syms->vkBindImageMemory; + vulkan_fns.vkGetBufferMemoryRequirements = + syms->vkGetBufferMemoryRequirements; + vulkan_fns.vkGetImageMemoryRequirements = syms->vkGetImageMemoryRequirements; + vulkan_fns.vkCreateBuffer = syms->vkCreateBuffer; + vulkan_fns.vkDestroyBuffer = syms->vkDestroyBuffer; + vulkan_fns.vkCreateImage = syms->vkCreateImage; + vulkan_fns.vkDestroyImage = syms->vkDestroyImage; + vulkan_fns.vkCmdCopyBuffer = syms->vkCmdCopyBuffer; + + VmaRecordSettings record_settings; +#if VMA_RECORDING_ENABLED + record_settings.flags = absl::GetFlag(FLAGS_vma_recording_flush_after_call) + ? VMA_RECORD_FLUSH_AFTER_CALL_BIT + : 0; + record_settings.pFilePath = absl::GetFlag(FLAGS_vma_recording_file).c_str(); +#else + record_settings.flags = 0; + record_settings.pFilePath = nullptr; +#endif // VMA_RECORDING_ENABLED + + VmaAllocatorCreateInfo create_info; + create_info.flags = 0; + create_info.physicalDevice = physical_device; + create_info.device = *logical_device; + create_info.preferredLargeHeapBlockSize = 64 * 1024 * 1024; + create_info.pAllocationCallbacks = logical_device->allocator(); + create_info.pDeviceMemoryCallbacks = nullptr; + create_info.frameInUseCount = 0; + create_info.pHeapSizeLimit = nullptr; + create_info.pVulkanFunctions = &vulkan_fns; + create_info.pRecordSettings = &record_settings; + ::VmaAllocator vma = VK_NULL_HANDLE; + VK_RETURN_IF_ERROR(vmaCreateAllocator(&create_info, &vma)); + + auto allocator = + absl::WrapUnique(new VmaAllocator(physical_device, logical_device, vma)); + // TODO(benvanik): query memory properties/types. + return allocator; +} + +VmaAllocator::VmaAllocator(VkPhysicalDevice physical_device, + const ref_ptr& logical_device, + ::VmaAllocator vma) + : physical_device_(physical_device), + logical_device_(add_ref(logical_device)), + vma_(vma) {} + +VmaAllocator::~VmaAllocator() { + IREE_TRACE_SCOPE0("VmaAllocator::dtor"); + vmaDestroyAllocator(vma_); +} + +bool VmaAllocator::CanUseBufferLike(Allocator* source_allocator, + MemoryTypeBitfield memory_type, + BufferUsageBitfield buffer_usage, + BufferUsageBitfield intended_usage) const { + // TODO(benvanik): ensure there is a memory type that can satisfy the request. + return source_allocator == this; +} + +bool VmaAllocator::CanAllocate(MemoryTypeBitfield memory_type, + BufferUsageBitfield buffer_usage, + size_t allocation_size) const { + // TODO(benvnik): ensure there is a memory type that can satisfy the request. + return true; +} + +Status VmaAllocator::MakeCompatible(MemoryTypeBitfield* memory_type, + BufferUsageBitfield* buffer_usage) const { + // TODO(benvanik): mutate to match supported memory types. + return OkStatus(); +} + +StatusOr> VmaAllocator::AllocateInternal( + MemoryTypeBitfield memory_type, BufferUsageBitfield buffer_usage, + MemoryAccessBitfield allowed_access, size_t allocation_size, + VmaAllocationCreateFlags flags) { + IREE_TRACE_SCOPE0("VmaAllocator::AllocateInternal"); + + VkBufferCreateInfo buffer_create_info; + buffer_create_info.sType = VK_STRUCTURE_TYPE_BUFFER_CREATE_INFO; + buffer_create_info.pNext = nullptr; + buffer_create_info.flags = 0; + buffer_create_info.size = allocation_size; + buffer_create_info.usage = 0; + if (AllBitsSet(buffer_usage, BufferUsage::kTransfer)) { + buffer_create_info.usage |= VK_BUFFER_USAGE_TRANSFER_SRC_BIT; + buffer_create_info.usage |= VK_BUFFER_USAGE_TRANSFER_DST_BIT; + } + if (AllBitsSet(buffer_usage, BufferUsage::kDispatch)) { + buffer_create_info.usage |= VK_BUFFER_USAGE_UNIFORM_BUFFER_BIT; + buffer_create_info.usage |= VK_BUFFER_USAGE_STORAGE_BUFFER_BIT; + buffer_create_info.usage |= VK_BUFFER_USAGE_INDIRECT_BUFFER_BIT; + } + buffer_create_info.sharingMode = VK_SHARING_MODE_EXCLUSIVE; + buffer_create_info.queueFamilyIndexCount = 0; + buffer_create_info.pQueueFamilyIndices = nullptr; + + VmaAllocationCreateInfo allocation_create_info; + allocation_create_info.flags = flags; + allocation_create_info.usage = VMA_MEMORY_USAGE_UNKNOWN; + allocation_create_info.requiredFlags = 0; + allocation_create_info.preferredFlags = 0; + allocation_create_info.memoryTypeBits = 0; // Automatic selection. + allocation_create_info.pool = VK_NULL_HANDLE; + allocation_create_info.pUserData = nullptr; + if (AllBitsSet(memory_type, MemoryType::kDeviceLocal)) { + if (AllBitsSet(memory_type, MemoryType::kHostVisible)) { + // Device-local, host-visible. + allocation_create_info.usage = VMA_MEMORY_USAGE_CPU_TO_GPU; + allocation_create_info.preferredFlags |= + VK_MEMORY_PROPERTY_DEVICE_LOCAL_BIT; + } else { + // Device-local only. + allocation_create_info.usage = VMA_MEMORY_USAGE_GPU_ONLY; + allocation_create_info.requiredFlags |= + VK_MEMORY_PROPERTY_DEVICE_LOCAL_BIT; + } + } else { + if (AllBitsSet(memory_type, MemoryType::kDeviceVisible)) { + // Host-local, device-visible. + allocation_create_info.usage = VMA_MEMORY_USAGE_GPU_TO_CPU; + } else { + // Host-local only. + allocation_create_info.usage = VMA_MEMORY_USAGE_CPU_ONLY; + } + } + if (AllBitsSet(memory_type, MemoryType::kHostCached)) { + allocation_create_info.requiredFlags |= VK_MEMORY_PROPERTY_HOST_CACHED_BIT; + } + if (AllBitsSet(memory_type, MemoryType::kHostCoherent)) { + allocation_create_info.requiredFlags |= + VK_MEMORY_PROPERTY_HOST_COHERENT_BIT; + } + if (AllBitsSet(memory_type, MemoryType::kTransient)) { + allocation_create_info.preferredFlags |= + VK_MEMORY_PROPERTY_LAZILY_ALLOCATED_BIT; + } + if (AllBitsSet(buffer_usage, BufferUsage::kMapping)) { + allocation_create_info.requiredFlags |= VK_MEMORY_PROPERTY_HOST_VISIBLE_BIT; + } + + VkBuffer buffer = VK_NULL_HANDLE; + VmaAllocation allocation = VK_NULL_HANDLE; + VmaAllocationInfo allocation_info; + VK_RETURN_IF_ERROR(vmaCreateBuffer(vma_, &buffer_create_info, + &allocation_create_info, &buffer, + &allocation, &allocation_info)); + + return make_ref(this, memory_type, allowed_access, buffer_usage, + allocation_size, 0, allocation_size, buffer, + allocation, allocation_info); +} + +StatusOr> VmaAllocator::Allocate( + MemoryTypeBitfield memory_type, BufferUsageBitfield buffer_usage, + size_t allocation_size) { + IREE_TRACE_SCOPE0("VmaAllocator::Allocate"); + return AllocateInternal(memory_type, buffer_usage, MemoryAccess::kAll, + allocation_size, /*flags=*/0); +} + +StatusOr> VmaAllocator::AllocateConstant( + BufferUsageBitfield buffer_usage, ref_ptr source_buffer) { + IREE_TRACE_SCOPE0("VmaAllocator::AllocateConstant"); + // TODO(benvanik): import memory to avoid the copy. + ASSIGN_OR_RETURN( + auto buffer, + AllocateInternal(MemoryType::kDeviceLocal | MemoryType::kHostVisible, + buffer_usage, + MemoryAccess::kRead | MemoryAccess::kDiscardWrite, + source_buffer->byte_length(), + /*flags=*/0)); + RETURN_IF_ERROR(buffer->CopyData(0, source_buffer.get(), 0, kWholeBuffer)); + buffer->set_allowed_access(MemoryAccess::kRead); + return buffer; +} + +StatusOr> VmaAllocator::WrapMutable( + MemoryTypeBitfield memory_type, MemoryAccessBitfield allowed_access, + BufferUsageBitfield buffer_usage, void* data, size_t data_length) { + IREE_TRACE_SCOPE0("VmaAllocator::WrapMutable"); + // TODO(benvanik): import memory. + return UnimplementedErrorBuilder(ABSL_LOC) + << "Wrapping host memory is not yet implemented"; +} + +} // namespace vulkan +} // namespace hal +} // namespace iree diff --git a/hal/vulkan/vma_allocator.h b/hal/vulkan/vma_allocator.h new file mode 100644 index 000000000000..eead40183fdf --- /dev/null +++ b/hal/vulkan/vma_allocator.h @@ -0,0 +1,110 @@ +// Copyright 2019 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#ifndef THIRD_PARTY_MLIR_EDGE_IREE_HAL_VULKAN_VMA_ALLOCATOR_H_ +#define THIRD_PARTY_MLIR_EDGE_IREE_HAL_VULKAN_VMA_ALLOCATOR_H_ + +#include + +#include + +#include "third_party/mlir_edge/iree/base/status.h" +#include "third_party/mlir_edge/iree/hal/allocator.h" +#include "third_party/mlir_edge/iree/hal/vulkan/dynamic_symbols.h" +#include "third_party/mlir_edge/iree/hal/vulkan/handle_util.h" +#include "third_party/vulkan_memory_allocator/src/vk_mem_alloc.h" + +namespace iree { +namespace hal { +namespace vulkan { + +class VmaBuffer; + +// A HAL allocator using the Vulkan Memory Allocator (VMA) to manage memory. +// VMA (//third_party/vulkan_memory_allocator) provides dlmalloc-like behavior +// with suballocations made with various policies (best fit, first fit, etc). +// This reduces the number of allocations we need from the Vulkan implementation +// (which can sometimes be limited to as little as 4096 total allowed) and +// manages higher level allocation semantics like slab allocation and +// defragmentation. +// +// VMA is internally synchronized and the functionality exposed on the HAL +// interface is thread-safe. +// +// More information: +// https://github.com/GPUOpen-LibrariesAndSDKs/VulkanMemoryAllocator +// https://gpuopen-librariesandsdks.github.io/VulkanMemoryAllocator/html/ +class VmaAllocator final : public Allocator { + public: + static StatusOr> Create( + VkPhysicalDevice physical_device, + const ref_ptr& logical_device); + + ~VmaAllocator() override; + + const ref_ptr& syms() const { + return logical_device_->syms(); + } + + ::VmaAllocator vma() const { return vma_; } + + bool CanUseBufferLike(Allocator* source_allocator, + MemoryTypeBitfield memory_type, + BufferUsageBitfield buffer_usage, + BufferUsageBitfield intended_usage) const override; + + bool CanAllocate(MemoryTypeBitfield memory_type, + BufferUsageBitfield buffer_usage, + size_t allocation_size) const override; + + Status MakeCompatible(MemoryTypeBitfield* memory_type, + BufferUsageBitfield* buffer_usage) const override; + + StatusOr> Allocate(MemoryTypeBitfield memory_type, + BufferUsageBitfield buffer_usage, + size_t allocation_size) override; + + StatusOr> AllocateConstant( + BufferUsageBitfield buffer_usage, ref_ptr source_buffer) override; + + StatusOr> WrapMutable(MemoryTypeBitfield memory_type, + MemoryAccessBitfield allowed_access, + BufferUsageBitfield buffer_usage, + void* data, + size_t data_length) override; + + private: + VmaAllocator(VkPhysicalDevice physical_device, + const ref_ptr& logical_device, + ::VmaAllocator vma); + + StatusOr> AllocateInternal( + MemoryTypeBitfield memory_type, BufferUsageBitfield buffer_usage, + MemoryAccessBitfield allowed_access, size_t allocation_size, + VmaAllocationCreateFlags flags); + + VkPhysicalDevice physical_device_; + ref_ptr logical_device_; + + // Internally synchronized. We could externally synchronize if we thought it + // was worth it, however I'm not sure we'd be able to do much better with the + // current Allocator API. + ::VmaAllocator vma_; +}; + +} // namespace vulkan +} // namespace hal +} // namespace iree + +#endif // THIRD_PARTY_MLIR_EDGE_IREE_HAL_VULKAN_VMA_ALLOCATOR_H_ diff --git a/hal/vulkan/vma_buffer.cc b/hal/vulkan/vma_buffer.cc new file mode 100644 index 000000000000..8111e7d949e8 --- /dev/null +++ b/hal/vulkan/vma_buffer.cc @@ -0,0 +1,163 @@ +// Copyright 2019 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "third_party/mlir_edge/iree/hal/vulkan/vma_buffer.h" + +#include "third_party/absl/types/source_location.h" +#include "third_party/mlir_edge/iree/base/status.h" +#include "third_party/mlir_edge/iree/base/tracing.h" +#include "third_party/mlir_edge/iree/hal/vulkan/status_util.h" +#include "third_party/mlir_edge/iree/hal/vulkan/vma_allocator.h" + +namespace iree { +namespace hal { +namespace vulkan { + +VmaBuffer::VmaBuffer(VmaAllocator* allocator, MemoryTypeBitfield memory_type, + MemoryAccessBitfield allowed_access, + BufferUsageBitfield usage, device_size_t allocation_size, + device_size_t byte_offset, device_size_t byte_length, + VkBuffer buffer, VmaAllocation allocation, + VmaAllocationInfo allocation_info) + : Buffer(allocator, memory_type, allowed_access, usage, allocation_size, + byte_offset, byte_length), + vma_(allocator->vma()), + buffer_(buffer), + allocation_(allocation), + allocation_info_(allocation_info) { + // TODO(benvanik): set debug name instead and use the + // VMA_ALLOCATION_CREATE_USER_DATA_COPY_STRING_BIT flag. + vmaSetAllocationUserData(vma_, allocation_, this); +} + +VmaBuffer::~VmaBuffer() { + IREE_TRACE_SCOPE0("VmaBuffer::dtor"); + vmaDestroyBuffer(vma_, buffer_, allocation_); +} + +Status VmaBuffer::FillImpl(device_size_t byte_offset, device_size_t byte_length, + const void* pattern, device_size_t pattern_length) { + ASSIGN_OR_RETURN(auto mapping, MapMemory(MemoryAccess::kDiscardWrite, + byte_offset, byte_length)); + void* data_ptr = static_cast(mapping.mutable_data()); + switch (pattern_length) { + case 1: { + uint8_t* data = static_cast(data_ptr); + uint8_t value_bits = *static_cast(pattern); + std::fill_n(data + byte_offset, byte_length, value_bits); + break; + } + case 2: { + uint16_t* data = static_cast(data_ptr); + uint16_t value_bits = *static_cast(pattern); + std::fill_n(data + byte_offset / sizeof(uint16_t), + byte_length / sizeof(uint16_t), value_bits); + break; + } + case 4: { + uint32_t* data = static_cast(data_ptr); + uint32_t value_bits = *static_cast(pattern); + std::fill_n(data + byte_offset / sizeof(uint32_t), + byte_length / sizeof(uint32_t), value_bits); + break; + } + default: + return InvalidArgumentErrorBuilder(ABSL_LOC) + << "Unsupported scalar data size: " << pattern_length; + } + return OkStatus(); +} + +Status VmaBuffer::ReadDataImpl(device_size_t source_offset, void* data, + device_size_t data_length) { + ASSIGN_OR_RETURN( + auto mapping, + MapMemory(MemoryAccess::kRead, source_offset, data_length)); + std::memcpy(data, mapping.data(), mapping.byte_length()); + return OkStatus(); +} + +Status VmaBuffer::WriteDataImpl(device_size_t target_offset, const void* data, + device_size_t data_length) { + ASSIGN_OR_RETURN(auto mapping, + MapMemory(MemoryAccess::kDiscardWrite, + target_offset, data_length)); + std::memcpy(mapping.mutable_data(), data, mapping.byte_length()); + return OkStatus(); +} + +Status VmaBuffer::CopyDataImpl(device_size_t target_offset, + Buffer* source_buffer, + device_size_t source_offset, + device_size_t data_length) { + // This is pretty terrible. Let's not do this. + // TODO(benvanik): a way for allocators to indicate transfer compat. + ASSIGN_OR_RETURN(auto source_mapping, + source_buffer->MapMemory( + MemoryAccess::kRead, source_offset, data_length)); + CHECK_EQ(data_length, source_mapping.size()); + ASSIGN_OR_RETURN(auto target_mapping, + MapMemory(MemoryAccess::kDiscardWrite, + target_offset, data_length)); + CHECK_EQ(data_length, target_mapping.size()); + std::memcpy(target_mapping.mutable_data() + target_offset, + source_mapping.data(), data_length); + return OkStatus(); +} + +Status VmaBuffer::MapMemoryImpl(MappingMode mapping_mode, + MemoryAccessBitfield memory_access, + device_size_t local_byte_offset, + device_size_t local_byte_length, + void** out_data) { + uint8_t* data_ptr = nullptr; + VK_RETURN_IF_ERROR( + vmaMapMemory(vma_, allocation_, reinterpret_cast(&data_ptr))); + *out_data = data_ptr + local_byte_offset; + + // If we mapped for discard scribble over the bytes. This is not a mandated + // behavior but it will make debugging issues easier. Alternatively for + // heap buffers we could reallocate them such that ASAN yells, but that + // would only work if the entire buffer was discarded. +#ifndef NDEBUG + if (AnyBitSet(memory_access & MemoryAccess::kDiscard)) { + std::memset(data_ptr + local_byte_offset, 0xCD, local_byte_length); + } +#endif // !NDEBUG + + return OkStatus(); +} + +Status VmaBuffer::UnmapMemoryImpl(device_size_t local_byte_offset, + device_size_t local_byte_length, void* data) { + vmaUnmapMemory(vma_, allocation_); + return OkStatus(); +} + +Status VmaBuffer::InvalidateMappedMemoryImpl(device_size_t local_byte_offset, + device_size_t local_byte_length) { + vmaInvalidateAllocation(vma_, allocation_, local_byte_offset, + local_byte_length); + return OkStatus(); +} + +Status VmaBuffer::FlushMappedMemoryImpl(device_size_t local_byte_offset, + device_size_t local_byte_length) { + vmaFlushAllocation(vma_, allocation_, local_byte_offset, local_byte_length); + return OkStatus(); +} + +} // namespace vulkan +} // namespace hal +} // namespace iree diff --git a/hal/vulkan/vma_buffer.h b/hal/vulkan/vma_buffer.h new file mode 100644 index 000000000000..e20d5b9eca07 --- /dev/null +++ b/hal/vulkan/vma_buffer.h @@ -0,0 +1,79 @@ +// Copyright 2019 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#ifndef THIRD_PARTY_MLIR_EDGE_IREE_HAL_VULKAN_VMA_BUFFER_H_ +#define THIRD_PARTY_MLIR_EDGE_IREE_HAL_VULKAN_VMA_BUFFER_H_ + +#include + +#include "third_party/mlir_edge/iree/hal/buffer.h" +#include "third_party/vulkan_memory_allocator/src/vk_mem_alloc.h" + +namespace iree { +namespace hal { +namespace vulkan { + +class VmaAllocator; + +// A buffer implementation representing an allocation made from within a pool of +// a Vulkan Memory Allocator instance. See VmaAllocator for more information. +class VmaBuffer final : public Buffer { + public: + VmaBuffer(VmaAllocator* allocator, MemoryTypeBitfield memory_type, + MemoryAccessBitfield allowed_access, BufferUsageBitfield usage, + device_size_t allocation_size, device_size_t byte_offset, + device_size_t byte_length, VkBuffer buffer, + VmaAllocation allocation, VmaAllocationInfo allocation_info); + ~VmaBuffer() override; + + VkBuffer handle() const { return buffer_; } + VmaAllocation allocation() const { return allocation_; } + const VmaAllocationInfo& allocation_info() const { return allocation_info_; } + + // Exposed so that VmaAllocator can reset access after initial mapping. + using Buffer::set_allowed_access; + + private: + Status FillImpl(device_size_t byte_offset, device_size_t byte_length, + const void* pattern, device_size_t pattern_length) override; + Status ReadDataImpl(device_size_t source_offset, void* data, + device_size_t data_length) override; + Status WriteDataImpl(device_size_t target_offset, const void* data, + device_size_t data_length) override; + Status CopyDataImpl(device_size_t target_offset, Buffer* source_buffer, + device_size_t source_offset, + device_size_t data_length) override; + Status MapMemoryImpl(MappingMode mapping_mode, + MemoryAccessBitfield memory_access, + device_size_t local_byte_offset, + device_size_t local_byte_length, + void** out_data) override; + Status UnmapMemoryImpl(device_size_t local_byte_offset, + device_size_t local_byte_length, void* data) override; + Status InvalidateMappedMemoryImpl(device_size_t local_byte_offset, + device_size_t local_byte_length) override; + Status FlushMappedMemoryImpl(device_size_t local_byte_offset, + device_size_t local_byte_length) override; + + ::VmaAllocator vma_; + VkBuffer buffer_; + VmaAllocation allocation_; + VmaAllocationInfo allocation_info_; +}; + +} // namespace vulkan +} // namespace hal +} // namespace iree + +#endif // THIRD_PARTY_MLIR_EDGE_IREE_HAL_VULKAN_VMA_BUFFER_H_ diff --git a/hal/vulkan/vulkan_device.cc b/hal/vulkan/vulkan_device.cc new file mode 100644 index 000000000000..3c67b22cac86 --- /dev/null +++ b/hal/vulkan/vulkan_device.cc @@ -0,0 +1,481 @@ +// Copyright 2019 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "third_party/mlir_edge/iree/hal/vulkan/vulkan_device.h" + +#include +#include + +#include "third_party/absl/container/inlined_vector.h" +#include "third_party/absl/memory/memory.h" +#include "third_party/absl/strings/str_cat.h" +#include "third_party/absl/synchronization/mutex.h" +#include "third_party/mlir_edge/iree/base/status.h" +#include "third_party/mlir_edge/iree/base/tracing.h" +#include "third_party/mlir_edge/iree/hal/command_buffer_validation.h" +#include "third_party/mlir_edge/iree/hal/command_queue.h" +#include "third_party/mlir_edge/iree/hal/fence.h" +#include "third_party/mlir_edge/iree/hal/vulkan/direct_command_buffer.h" +#include "third_party/mlir_edge/iree/hal/vulkan/direct_command_queue.h" +#include "third_party/mlir_edge/iree/hal/vulkan/dynamic_symbols.h" +#include "third_party/mlir_edge/iree/hal/vulkan/extensibility_util.h" +#include "third_party/mlir_edge/iree/hal/vulkan/legacy_fence.h" +#include "third_party/mlir_edge/iree/hal/vulkan/native_binary_semaphore.h" +#include "third_party/mlir_edge/iree/hal/vulkan/native_event.h" +#include "third_party/mlir_edge/iree/hal/vulkan/pipeline_cache.h" +#include "third_party/mlir_edge/iree/hal/vulkan/status_util.h" +#include "third_party/mlir_edge/iree/hal/vulkan/vma_allocator.h" + +namespace iree { +namespace hal { +namespace vulkan { + +namespace { + +constexpr uint32_t kInvalidQueueFamilyIndex = -1; + +struct QueueFamilyInfo { + uint32_t dispatch_index = kInvalidQueueFamilyIndex; + uint32_t dispatch_queue_count = 0; + uint32_t transfer_index = kInvalidQueueFamilyIndex; + uint32_t transfer_queue_count = 0; +}; + +// Finds the first queue in the listing (which is usually the driver-preferred) +// that has all of the |required_queue_flags| and none of the +// |excluded_queue_flags|. +// Returns kInvalidQueueFamilyIndex if no matching queue is found. +uint32_t FindFirstQueueFamilyWithFlags( + absl::Span queue_family_properties, + uint32_t required_queue_flags, uint32_t excluded_queue_flags) { + for (int queue_family_index = 0; + queue_family_index < queue_family_properties.size(); + ++queue_family_index) { + const auto& properties = queue_family_properties[queue_family_index]; + if ((properties.queueFlags & required_queue_flags) == + required_queue_flags && + (properties.queueFlags & excluded_queue_flags) != 0) { + return queue_family_index; + } + } + return kInvalidQueueFamilyIndex; +} + +// Selects queue family indices for compute and transfer queues. +// Note that both queue families may be the same if there is only one family +// available. +StatusOr SelectQueueFamilies( + VkPhysicalDevice physical_device, const ref_ptr& syms) { + // Enumerate queue families available on the device. + uint32_t queue_family_count = 0; + syms->vkGetPhysicalDeviceQueueFamilyProperties(physical_device, + &queue_family_count, nullptr); + absl::InlinedVector queue_family_properties( + queue_family_count); + syms->vkGetPhysicalDeviceQueueFamilyProperties( + physical_device, &queue_family_count, queue_family_properties.data()); + + QueueFamilyInfo queue_family_info; + + // Try to find a dedicated compute queue (no graphics caps). + // Some may support both transfer and compute. If that fails then fallback to + // any queue that supports compute. + queue_family_info.dispatch_index = FindFirstQueueFamilyWithFlags( + queue_family_properties, VK_QUEUE_COMPUTE_BIT, VK_QUEUE_GRAPHICS_BIT); + if (queue_family_info.dispatch_index == kInvalidQueueFamilyIndex) { + queue_family_info.dispatch_index = FindFirstQueueFamilyWithFlags( + queue_family_properties, VK_QUEUE_COMPUTE_BIT, 0); + } + if (queue_family_info.dispatch_index == kInvalidQueueFamilyIndex) { + return NotFoundErrorBuilder(ABSL_LOC) + << "Unable to find any queue family support compute operations"; + } + queue_family_info.dispatch_queue_count = + queue_family_properties[queue_family_info.dispatch_index].queueCount; + + // Try to find a dedicated transfer queue (no compute or graphics caps). + // Not all devices have one, and some have only a queue family for everything + // and possibly a queue family just for compute/etc. If that fails then + // fallback to any queue that supports transfer. Finally, if /that/ fails then + // we just won't create a transfer queue and instead use the compute queue for + // all operations. + queue_family_info.transfer_index = FindFirstQueueFamilyWithFlags( + queue_family_properties, VK_QUEUE_TRANSFER_BIT, + VK_QUEUE_COMPUTE_BIT | VK_QUEUE_GRAPHICS_BIT); + if (queue_family_info.transfer_index == kInvalidQueueFamilyIndex) { + queue_family_info.transfer_index = FindFirstQueueFamilyWithFlags( + queue_family_properties, VK_QUEUE_TRANSFER_BIT, 0); + } + if (queue_family_info.transfer_index != kInvalidQueueFamilyIndex) { + queue_family_info.transfer_queue_count = + queue_family_properties[queue_family_info.transfer_index].queueCount; + } + + return queue_family_info; +} + +// Creates a transient command pool for the given queue family. +// Command buffers allocated from the pool must only be issued on queues +// belonging to the specified family. +StatusOr> CreateTransientCommandPool( + const ref_ptr& logical_device, + uint32_t queue_family_index) { + VkCommandPoolCreateInfo create_info; + create_info.sType = VK_STRUCTURE_TYPE_COMMAND_POOL_CREATE_INFO; + create_info.pNext = nullptr; + create_info.flags = VK_COMMAND_POOL_CREATE_TRANSIENT_BIT; + create_info.queueFamilyIndex = queue_family_index; + + auto command_pool = make_ref(logical_device); + VK_RETURN_IF_ERROR(logical_device->syms()->vkCreateCommandPool( + *logical_device, &create_info, logical_device->allocator(), + command_pool->mutable_value())); + return command_pool; +} + +} // namespace + +// static +StatusOr> VulkanDevice::Create( + const DeviceInfo& device_info, VkPhysicalDevice physical_device, + const ExtensibilitySpec& extensibility_spec, + const ref_ptr& syms) { + IREE_TRACE_SCOPE0("VulkanDevice::Create"); + + // Find the layers and extensions we need (or want) that are also available + // on the device. This will fail when required ones are not present. + ASSIGN_OR_RETURN( + auto enabled_layer_names, + MatchAvailableDeviceLayers(physical_device, extensibility_spec, *syms)); + ASSIGN_OR_RETURN(auto enabled_extension_names, + MatchAvailableDeviceExtensions(physical_device, + extensibility_spec, *syms)); + auto enabled_device_extensions = + PopulateEnabledDeviceExtensions(enabled_extension_names); + + // Find queue families we will expose as HAL queues. + ASSIGN_OR_RETURN(auto queue_family_info, + SelectQueueFamilies(physical_device, syms)); + + // Limit the number of queues we create (for now). + // We may want to allow this to grow, but each queue adds overhead and we need + // to measure to make sure we can effectively use them all. + queue_family_info.dispatch_queue_count = + std::min(2u, queue_family_info.dispatch_queue_count); + queue_family_info.transfer_queue_count = + std::min(1u, queue_family_info.transfer_queue_count); + bool has_dedicated_transfer_queues = + queue_family_info.transfer_queue_count > 0; + + // Setup the queue info we'll be using. + // Each queue here (created from within a family) will map to a HAL queue. + // + // Note that we need to handle the case where we have transfer queues that are + // of the same queue family as the dispatch queues: Vulkan requires that all + // queues created from the same family are done in the same + // VkDeviceQueueCreateInfo struct. + DVLOG(1) << "Creating " << queue_family_info.dispatch_queue_count + << " dispatch queue(s) in queue family " + << queue_family_info.dispatch_index; + absl::InlinedVector queue_create_info; + absl::InlinedVector dispatch_queue_priorities; + absl::InlinedVector transfer_queue_priorities; + queue_create_info.push_back({}); + auto& dispatch_queue_info = queue_create_info.back(); + dispatch_queue_info.sType = VK_STRUCTURE_TYPE_DEVICE_QUEUE_CREATE_INFO; + dispatch_queue_info.pNext = nullptr; + dispatch_queue_info.flags = 0; + dispatch_queue_info.queueFamilyIndex = queue_family_info.dispatch_index; + dispatch_queue_info.queueCount = queue_family_info.dispatch_queue_count; + if (has_dedicated_transfer_queues) { + if (queue_family_info.dispatch_index == queue_family_info.transfer_index) { + DVLOG(1) << "Creating " << queue_family_info.transfer_queue_count + << " dedicated transfer queue(s) in shared queue family " + << queue_family_info.transfer_index; + dispatch_queue_info.queueCount += queue_family_info.transfer_queue_count; + } else { + DVLOG(1) << "Creating " << queue_family_info.transfer_queue_count + << " dedicated transfer queue(s) in independent queue family " + << queue_family_info.transfer_index; + queue_create_info.push_back({}); + auto& transfer_queue_info = queue_create_info.back(); + transfer_queue_info.sType = VK_STRUCTURE_TYPE_DEVICE_QUEUE_CREATE_INFO; + transfer_queue_info.pNext = nullptr; + transfer_queue_info.queueFamilyIndex = queue_family_info.transfer_index; + transfer_queue_info.queueCount = queue_family_info.transfer_queue_count; + transfer_queue_info.flags = 0; + transfer_queue_priorities.resize(transfer_queue_info.queueCount); + transfer_queue_info.pQueuePriorities = transfer_queue_priorities.data(); + } + } + dispatch_queue_priorities.resize(dispatch_queue_info.queueCount); + dispatch_queue_info.pQueuePriorities = dispatch_queue_priorities.data(); + + // TODO(benvanik): specify features with VkPhysicalDeviceFeatures. + + // Create device and its queues. + VkDeviceCreateInfo device_create_info = {}; + device_create_info.sType = VK_STRUCTURE_TYPE_DEVICE_CREATE_INFO; + device_create_info.pNext = nullptr; + device_create_info.enabledLayerCount = enabled_layer_names.size(); + device_create_info.ppEnabledLayerNames = enabled_layer_names.data(); + device_create_info.enabledExtensionCount = enabled_extension_names.size(); + device_create_info.ppEnabledExtensionNames = enabled_extension_names.data(); + device_create_info.queueCreateInfoCount = queue_create_info.size(); + device_create_info.pQueueCreateInfos = queue_create_info.data(); + device_create_info.pEnabledFeatures = nullptr; + auto logical_device = make_ref( + syms, enabled_device_extensions, /*allocator=*/nullptr); + VK_RETURN_IF_ERROR(syms->vkCreateDevice(physical_device, &device_create_info, + logical_device->allocator(), + logical_device->mutable_value())); + + // Create the device memory allocator. + // TODO(benvanik): allow other types to be plugged in. + ASSIGN_OR_RETURN(auto allocator, + VmaAllocator::Create(physical_device, logical_device)); + + // Create command pools for each queue family. If we don't have a transfer + // queue then we'll ignore that one and just use the dispatch pool. + // If we wanted to expose the pools through the HAL to allow the VM to more + // effectively manage them (pool per fiber, etc) we could, however I doubt the + // overhead of locking the pool will be even a blip. + ASSIGN_OR_RETURN(auto dispatch_command_pool, + CreateTransientCommandPool( + logical_device, queue_family_info.dispatch_index)); + ref_ptr transfer_command_pool; + if (has_dedicated_transfer_queues) { + ASSIGN_OR_RETURN(transfer_command_pool, + CreateTransientCommandPool( + logical_device, queue_family_info.transfer_index)); + } + + // Get the queues and create the HAL wrappers. + absl::InlinedVector, 4> command_queues; + for (uint32_t i = 0; i < queue_family_info.dispatch_queue_count; ++i) { + VkQueue queue = VK_NULL_HANDLE; + syms->vkGetDeviceQueue(*logical_device, queue_family_info.dispatch_index, i, + &queue); + std::string queue_name = absl::StrCat(device_info.name(), ":d", i); + command_queues.push_back(absl::make_unique( + std::move(queue_name), + CommandCategory::kDispatch | CommandCategory::kTransfer, logical_device, + queue)); + } + if (has_dedicated_transfer_queues) { + uint32_t base_queue_index = 0; + if (queue_family_info.dispatch_index == queue_family_info.transfer_index) { + // Sharing a family, so transfer queues follow compute queues. + base_queue_index = queue_family_info.dispatch_index; + } + for (uint32_t i = 0; i < queue_family_info.transfer_queue_count; ++i) { + VkQueue queue = VK_NULL_HANDLE; + syms->vkGetDeviceQueue(*logical_device, queue_family_info.transfer_index, + base_queue_index + i, &queue); + std::string queue_name = absl::StrCat(device_info.name(), ":t", i); + command_queues.push_back(absl::make_unique( + std::move(queue_name), CommandCategory::kTransfer, logical_device, + queue)); + } + } + + // TODO(b/140141417): implement timeline semaphore fences and switch here. + ASSIGN_OR_RETURN(auto legacy_fence_pool, + LegacyFencePool::Create(add_ref(logical_device))); + + return std::make_shared( + CtorKey{}, device_info, physical_device, std::move(logical_device), + std::move(allocator), std::move(command_queues), + std::move(dispatch_command_pool), std::move(transfer_command_pool), + std::move(legacy_fence_pool)); +} + +VulkanDevice::VulkanDevice( + CtorKey ctor_key, const DeviceInfo& device_info, + VkPhysicalDevice physical_device, ref_ptr logical_device, + std::unique_ptr allocator, + absl::InlinedVector, 4> command_queues, + ref_ptr dispatch_command_pool, + ref_ptr transfer_command_pool, + ref_ptr legacy_fence_pool) + : Device(device_info), + physical_device_(physical_device), + logical_device_(std::move(logical_device)), + allocator_(std::move(allocator)), + command_queues_(std::move(command_queues)), + dispatch_command_pool_(std::move(dispatch_command_pool)), + transfer_command_pool_(std::move(transfer_command_pool)), + legacy_fence_pool_(std::move(legacy_fence_pool)) { + // Populate the queue lists based on queue capabilities. + for (auto& command_queue : command_queues_) { + if (command_queue->can_dispatch()) { + dispatch_queues_.push_back(command_queue.get()); + if (transfer_command_pool_ == VK_NULL_HANDLE) { + transfer_queues_.push_back(command_queue.get()); + } + } else { + transfer_queues_.push_back(command_queue.get()); + } + } +} + +VulkanDevice::~VulkanDevice() { + IREE_TRACE_SCOPE0("VulkanDevice::dtor"); + + // Drop all command queues. These may wait until idle. + command_queues_.clear(); + dispatch_queues_.clear(); + transfer_queues_.clear(); + + // Drop command pools now that we know there are no more outstanding command + // buffers. + dispatch_command_pool_.reset(); + transfer_command_pool_.reset(); + + // Finally, destroy the device. + logical_device_.reset(); +} + +std::shared_ptr VulkanDevice::CreateExecutableCache() { + IREE_TRACE_SCOPE0("VulkanDevice::CreateExecutableCache"); + return std::make_shared(logical_device_); +} + +StatusOr> VulkanDevice::CreateCommandBuffer( + CommandBufferModeBitfield mode, + CommandCategoryBitfield command_categories) { + IREE_TRACE_SCOPE0("VulkanDevice::CreateCommandBuffer"); + + // Select the command pool to used based on the types of commands used. + // Note that we may not have a dedicated transfer command pool if there are no + // dedicated transfer queues. + ref_ptr command_pool; + if (transfer_command_pool_ && + !AllBitsSet(command_categories, CommandCategory::kDispatch)) { + command_pool = add_ref(transfer_command_pool_); + } else { + command_pool = add_ref(dispatch_command_pool_); + } + + VkCommandBufferAllocateInfo allocate_info; + allocate_info.sType = VK_STRUCTURE_TYPE_COMMAND_BUFFER_ALLOCATE_INFO; + allocate_info.pNext = nullptr; + allocate_info.commandPool = *command_pool; + allocate_info.commandBufferCount = 1; + allocate_info.level = VK_COMMAND_BUFFER_LEVEL_PRIMARY; + + VkCommandBuffer command_buffer = VK_NULL_HANDLE; + { + absl::MutexLock lock(command_pool->mutex()); + VK_RETURN_IF_ERROR(syms()->vkAllocateCommandBuffers( + *logical_device_, &allocate_info, &command_buffer)); + } + + // TODO(b/140026716): conditionally enable validation. + auto impl = make_ref( + allocator(), mode, command_categories, command_pool, command_buffer); + return WrapCommandBufferWithValidation(std::move(impl)); +} + +StatusOr> VulkanDevice::CreateEvent() { + IREE_TRACE_SCOPE0("VulkanDevice::CreateEvent"); + + // TODO(b/138729892): pool events. + VkEventCreateInfo create_info; + create_info.sType = VK_STRUCTURE_TYPE_EVENT_CREATE_INFO; + create_info.pNext = nullptr; + create_info.flags = 0; + VkEvent event_handle = VK_NULL_HANDLE; + VK_RETURN_IF_ERROR(syms()->vkCreateEvent(*logical_device_, &create_info, + logical_device_->allocator(), + &event_handle)); + + return make_ref(add_ref(logical_device_), event_handle); +} + +StatusOr> VulkanDevice::CreateBinarySemaphore( + bool initial_value) { + IREE_TRACE_SCOPE0("VulkanDevice::CreateBinarySemaphore"); + + VkSemaphoreCreateInfo create_info; + create_info.sType = VK_STRUCTURE_TYPE_SEMAPHORE_CREATE_INFO; + create_info.pNext = nullptr; + create_info.flags = initial_value ? VK_FENCE_CREATE_SIGNALED_BIT : 0; + VkSemaphore semaphore_handle = VK_NULL_HANDLE; + VK_RETURN_IF_ERROR(syms()->vkCreateSemaphore(*logical_device_, &create_info, + logical_device_->allocator(), + &semaphore_handle)); + + return make_ref(add_ref(logical_device_), + semaphore_handle); +} + +StatusOr> VulkanDevice::CreateTimelineSemaphore( + uint64_t initial_value) { + IREE_TRACE_SCOPE0("VulkanDevice::CreateTimelineSemaphore"); + + // TODO(b/140141417): implement timeline semaphores. + return UnimplementedErrorBuilder(ABSL_LOC) + << "Timeline semaphores not yet implemented"; +} + +StatusOr> VulkanDevice::CreateFence(uint64_t initial_value) { + IREE_TRACE_SCOPE0("VulkanDevice::CreateFence"); + + // TODO(b/140141417): implement timeline semaphore fences and switch here. + // NOTE: we'll want some magic factory so that we can cleanly compile out the + // legacy implementation and pool. + + return make_ref(add_ref(legacy_fence_pool_), initial_value); +} + +Status VulkanDevice::WaitAllFences(absl::Span fences, + absl::Time deadline) { + IREE_TRACE_SCOPE0("VulkanDevice::WaitAllFences"); + + // TODO(b/140141417): implement timeline semaphore fences and switch here. + + return LegacyFence::WaitForFences(logical_device_.get(), fences, + /*wait_all=*/true, deadline); +} + +StatusOr VulkanDevice::WaitAnyFence(absl::Span fences, + absl::Time deadline) { + IREE_TRACE_SCOPE0("VulkanDevice::WaitAnyFence"); + + // TODO(b/140141417): implement timeline semaphore fences and switch here. + + return LegacyFence::WaitForFences(logical_device_.get(), fences, + /*wait_all=*/false, deadline); +} + +Status VulkanDevice::WaitIdle(absl::Time deadline) { + if (deadline == absl::InfiniteFuture()) { + // Fast path for using vkDeviceWaitIdle, which is usually cheaper (as it + // requires fewer calls into the driver). + IREE_TRACE_SCOPE0("VulkanDevice::WaitIdle#vkDeviceWaitIdle"); + VK_RETURN_IF_ERROR(syms()->vkDeviceWaitIdle(*logical_device_)); + return OkStatus(); + } + + IREE_TRACE_SCOPE0("VulkanDevice::WaitIdle#Fences"); + for (auto& command_queue : command_queues_) { + RETURN_IF_ERROR(command_queue->WaitIdle(deadline)); + } + return OkStatus(); +} + +} // namespace vulkan +} // namespace hal +} // namespace iree diff --git a/hal/vulkan/vulkan_device.h b/hal/vulkan/vulkan_device.h new file mode 100644 index 000000000000..755a9aaff77a --- /dev/null +++ b/hal/vulkan/vulkan_device.h @@ -0,0 +1,117 @@ +// Copyright 2019 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#ifndef THIRD_PARTY_MLIR_EDGE_IREE_HAL_VULKAN_VULKAN_DEVICE_H_ +#define THIRD_PARTY_MLIR_EDGE_IREE_HAL_VULKAN_VULKAN_DEVICE_H_ + +#include + +#include +#include + +#include "third_party/absl/container/inlined_vector.h" +#include "third_party/absl/types/span.h" +#include "third_party/mlir_edge/iree/base/memory.h" +#include "third_party/mlir_edge/iree/hal/allocator.h" +#include "third_party/mlir_edge/iree/hal/device.h" +#include "third_party/mlir_edge/iree/hal/vulkan/dynamic_symbols.h" +#include "third_party/mlir_edge/iree/hal/vulkan/extensibility_util.h" +#include "third_party/mlir_edge/iree/hal/vulkan/handle_util.h" +#include "third_party/mlir_edge/iree/hal/vulkan/legacy_fence.h" + +namespace iree { +namespace hal { +namespace vulkan { + +class VulkanDevice final : public Device { + public: + static StatusOr> Create( + const DeviceInfo& device_info, VkPhysicalDevice physical_device, + const ExtensibilitySpec& extensibility_spec, + const ref_ptr& syms); + + // Private constructor. + struct CtorKey { + private: + friend class VulkanDevice; + CtorKey() = default; + }; + VulkanDevice( + CtorKey ctor_key, const DeviceInfo& device_info, + VkPhysicalDevice physical_device, ref_ptr logical_device, + std::unique_ptr allocator, + absl::InlinedVector, 4> command_queues, + ref_ptr dispatch_command_pool, + ref_ptr transfer_command_pool, + ref_ptr legacy_fence_pool); + ~VulkanDevice() override; + + const ref_ptr& syms() const { + return logical_device_->syms(); + } + + Allocator* allocator() const override { return allocator_.get(); } + + absl::Span dispatch_queues() const override { + return absl::MakeSpan(dispatch_queues_); + } + + absl::Span transfer_queues() const override { + return absl::MakeSpan(transfer_queues_); + } + + std::shared_ptr CreateExecutableCache() override; + + StatusOr> CreateCommandBuffer( + CommandBufferModeBitfield mode, + CommandCategoryBitfield command_categories) override; + + StatusOr> CreateEvent() override; + + StatusOr> CreateBinarySemaphore( + bool initial_value) override; + StatusOr> CreateTimelineSemaphore( + uint64_t initial_value) override; + + StatusOr> CreateFence(uint64_t initial_value) override; + Status WaitAllFences(absl::Span fences, + absl::Time deadline) override; + StatusOr WaitAnyFence(absl::Span fences, + absl::Time deadline) override; + + Status WaitIdle(absl::Time deadline) override; + + private: + VkPhysicalDevice physical_device_; + ref_ptr logical_device_; + + std::unique_ptr allocator_; + + mutable absl::InlinedVector, 4> command_queues_; + mutable absl::InlinedVector dispatch_queues_; + mutable absl::InlinedVector transfer_queues_; + + ref_ptr dispatch_command_pool_; + ref_ptr transfer_command_pool_; + + // TODO(b/140141417): implement timeline semaphore fences and conditionally + // compile the legacy fence pool out. + ref_ptr legacy_fence_pool_; +}; + +} // namespace vulkan +} // namespace hal +} // namespace iree + +#endif // THIRD_PARTY_MLIR_EDGE_IREE_HAL_VULKAN_VULKAN_DEVICE_H_ diff --git a/hal/vulkan/vulkan_driver.cc b/hal/vulkan/vulkan_driver.cc new file mode 100644 index 000000000000..91ddcafe07de --- /dev/null +++ b/hal/vulkan/vulkan_driver.cc @@ -0,0 +1,224 @@ +// Copyright 2019 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "third_party/mlir_edge/iree/hal/vulkan/vulkan_driver.h" + +#include + +#include "third_party/absl/container/inlined_vector.h" +#include "third_party/mlir_edge/iree/base/status.h" +#include "third_party/mlir_edge/iree/base/tracing.h" +#include "third_party/mlir_edge/iree/hal/device_info.h" +#include "third_party/mlir_edge/iree/hal/vulkan/extensibility_util.h" +#include "third_party/mlir_edge/iree/hal/vulkan/status_util.h" +#include "third_party/mlir_edge/iree/hal/vulkan/vulkan_device.h" + +namespace iree { +namespace hal { +namespace vulkan { + +namespace { + +// Returns a VkApplicationInfo struct populated with the default app info. +// We may allow hosting applications to override this via weak-linkage if it's +// useful, otherwise this is enough to create the application. +VkApplicationInfo GetDefaultApplicationInfo() { + VkApplicationInfo info; + info.sType = VK_STRUCTURE_TYPE_APPLICATION_INFO; + info.pNext = nullptr; + info.pApplicationName = "IREE-ML"; + info.applicationVersion = 0; + info.pEngineName = "IREE"; + info.engineVersion = 0; + info.apiVersion = VK_API_VERSION_1_0; + return info; +} + +// Populates device information from the given Vulkan physical device handle. +StatusOr PopulateDeviceInfo(VkPhysicalDevice physical_device, + const ref_ptr& syms) { + VkPhysicalDeviceFeatures physical_device_features; + syms->vkGetPhysicalDeviceFeatures(physical_device, &physical_device_features); + // TODO(benvanik): check and optionally require these features: + // - physical_device_features.robustBufferAccess + // - physical_device_features.shaderInt16 + // - physical_device_features.shaderInt64 + // - physical_device_features.shaderFloat64 + + VkPhysicalDeviceProperties physical_device_properties; + syms->vkGetPhysicalDeviceProperties(physical_device, + &physical_device_properties); + // TODO(benvanik): check and optionally require reasonable limits. + + // TODO(benvanik): more clever/sanitized device naming. + std::string name = std::string(physical_device_properties.deviceName); + + DeviceFeatureBitfield supported_features = DeviceFeature::kNone; + // TODO(benvanik): implement debugging/profiling features. + // TODO(benvanik): use props to determine if we have timing info. + // supported_features |= DeviceFeature::kDebugging; + // supported_features |= DeviceFeature::kCoverage; + // supported_features |= DeviceFeature::kProfiling; + return DeviceInfo(std::move(name), supported_features, physical_device); +} + +} // namespace + +// static +StatusOr> VulkanDriver::Create( + Options options, ref_ptr syms) { + IREE_TRACE_SCOPE0("VulkanDriver::Create"); + + // Find the layers and extensions we need (or want) that are also available + // on the instance. This will fail when required ones are not present. + ASSIGN_OR_RETURN( + auto enabled_layer_names, + MatchAvailableInstanceLayers(options.instance_extensibility, *syms)); + ASSIGN_OR_RETURN( + auto enabled_extension_names, + MatchAvailableInstanceExtensions(options.instance_extensibility, *syms)); + auto instance_extensions = + PopulateEnabledInstanceExtensions(enabled_extension_names); + + // Create the instance this driver will use for all requests. + VkApplicationInfo app_info = GetDefaultApplicationInfo(); + app_info.apiVersion = options.api_version; + VkInstanceCreateInfo create_info; + create_info.sType = VK_STRUCTURE_TYPE_INSTANCE_CREATE_INFO; + create_info.pNext = nullptr; + create_info.flags = 0; + create_info.pApplicationInfo = &app_info; + create_info.enabledLayerCount = enabled_layer_names.size(); + create_info.ppEnabledLayerNames = enabled_layer_names.data(); + create_info.enabledExtensionCount = enabled_extension_names.size(); + create_info.ppEnabledExtensionNames = enabled_extension_names.data(); + + // If we have the debug_utils extension then we can chain a one-shot messenger + // callback that we can use to log out the instance creation errors. Once we + // have the real instance we can then register a real messenger. + union { + VkDebugUtilsMessengerCreateInfoEXT debug_utils_create_info; + VkDebugReportCallbackCreateInfoEXT debug_report_create_info; + }; + if (instance_extensions.debug_utils) { + create_info.pNext = &debug_utils_create_info; + DebugReporter::PopulateStaticCreateInfo(&debug_utils_create_info); + } else if (instance_extensions.debug_report) { + create_info.pNext = &debug_report_create_info; + DebugReporter::PopulateStaticCreateInfo(&debug_report_create_info); + } + + VkInstance instance = VK_NULL_HANDLE; + VK_RETURN_IF_ERROR( + syms->vkCreateInstance(&create_info, /*pAllocator=*/nullptr, &instance)) + << "Unable to create Vulkan instance"; + + // TODO(benvanik): enable validation layers if needed. + + // Now that the instance has been created we can fetch all of the instance + // symbols. + RETURN_IF_ERROR(syms->LoadFromInstance(instance)); + + // The real debug messenger (not just the static one used above) can now be + // created as we've loaded all the required symbols. + // TODO(benvanik): strip in release builds. + std::unique_ptr debug_reporter; + if (instance_extensions.debug_utils) { + ASSIGN_OR_RETURN(debug_reporter, DebugReporter::CreateDebugUtilsMessenger( + instance, syms, + /*allocation_callbacks=*/nullptr)); + } else if (instance_extensions.debug_report) { + ASSIGN_OR_RETURN(debug_reporter, + DebugReporter::CreateDebugReportCallback( + instance, syms, /*allocation_callbacks=*/nullptr)); + } + + return std::make_shared( + CtorKey{}, std::move(syms), instance, std::move(debug_reporter), + std::move(options.device_extensibility)); +} + +VulkanDriver::VulkanDriver(CtorKey ctor_key, ref_ptr syms, + VkInstance instance, + std::unique_ptr debug_reporter, + ExtensibilitySpec device_extensibility_spec) + : Driver("vulkan"), + syms_(std::move(syms)), + instance_(instance), + debug_reporter_(std::move(debug_reporter)), + device_extensibility_spec_(std::move(device_extensibility_spec)) {} + +VulkanDriver::~VulkanDriver() { + IREE_TRACE_SCOPE0("VulkanDriver::dtor"); + debug_reporter_.reset(); + syms()->vkDestroyInstance(instance_, /*pAllocator=*/nullptr); +} + +StatusOr> VulkanDriver::EnumerateAvailableDevices() { + IREE_TRACE_SCOPE0("VulkanDriver::EnumerateAvailableDevices"); + + // Query all available devices (at this moment, note that this may change!). + uint32_t physical_device_count = 0; + VK_RETURN_IF_ERROR(syms()->vkEnumeratePhysicalDevices( + instance_, &physical_device_count, nullptr)); + absl::InlinedVector physical_devices( + physical_device_count); + VK_RETURN_IF_ERROR(syms()->vkEnumeratePhysicalDevices( + instance_, &physical_device_count, physical_devices.data())); + + // Convert to our HAL structure. + std::vector device_infos; + device_infos.reserve(physical_device_count); + for (auto physical_device : physical_devices) { + // TODO(benvanik): if we fail should we just ignore the device in the list? + ASSIGN_OR_RETURN(auto device_info, + PopulateDeviceInfo(physical_device, syms())); + device_infos.push_back(std::move(device_info)); + } + return device_infos; +} + +StatusOr> VulkanDriver::CreateDefaultDevice() { + IREE_TRACE_SCOPE0("VulkanDriver::CreateDefaultDevice"); + + // Query available devices. + ASSIGN_OR_RETURN(auto available_devices, EnumerateAvailableDevices()); + if (available_devices.empty()) { + return NotFoundErrorBuilder(ABSL_LOC) << "No devices are available"; + } + + // Just create the first one we find. + return CreateDevice(available_devices.front()); +} + +StatusOr> VulkanDriver::CreateDevice( + const DeviceInfo& device_info) { + IREE_TRACE_SCOPE0("VulkanDriver::CreateDevice"); + + auto physical_device = + static_cast(device_info.driver_handle()); + + // Attempt to create the device. + // This may fail if the device was enumerated but is in exclusive use, + // disabled by the system, or permission is denied. + ASSIGN_OR_RETURN(auto device, + VulkanDevice::Create(device_info, physical_device, + device_extensibility_spec_, syms())); + + return device; +} + +} // namespace vulkan +} // namespace hal +} // namespace iree diff --git a/hal/vulkan/vulkan_driver.h b/hal/vulkan/vulkan_driver.h new file mode 100644 index 000000000000..6a8f0045a29f --- /dev/null +++ b/hal/vulkan/vulkan_driver.h @@ -0,0 +1,84 @@ +// Copyright 2019 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#ifndef THIRD_PARTY_MLIR_EDGE_IREE_HAL_VULKAN_VULKAN_DRIVER_H_ +#define THIRD_PARTY_MLIR_EDGE_IREE_HAL_VULKAN_VULKAN_DRIVER_H_ + +#include + +#include +#include + +#include "third_party/mlir_edge/iree/hal/driver.h" +#include "third_party/mlir_edge/iree/hal/vulkan/debug_reporter.h" +#include "third_party/mlir_edge/iree/hal/vulkan/dynamic_symbols.h" +#include "third_party/mlir_edge/iree/hal/vulkan/extensibility_util.h" + +namespace iree { +namespace hal { +namespace vulkan { + +class VulkanDriver final : public Driver { + public: + struct Options { + // Vulkan version that will be requested. + // Driver creation will fail if the required version is not available. + uint32_t api_version = VK_API_VERSION_1_0; + + // Extensibility descriptions for instances and devices. + // Device descriptions will be used for all devices created by the driver. + ExtensibilitySpec instance_extensibility; + ExtensibilitySpec device_extensibility; + }; + + static StatusOr> Create( + Options options, ref_ptr syms); + + // TODO(benvanik): method to wrap an existing instance/device (interop). + + // Private constructor. + struct CtorKey { + private: + friend class VulkanDriver; + CtorKey() = default; + }; + VulkanDriver(CtorKey ctor_key, ref_ptr syms, + VkInstance instance, + std::unique_ptr debug_reporter, + ExtensibilitySpec device_extensibility_spec); + ~VulkanDriver() override; + + const ref_ptr& syms() const { return syms_; } + + VkInstance instance() const { return instance_; } + + StatusOr> EnumerateAvailableDevices() override; + + StatusOr> CreateDefaultDevice() override; + + StatusOr> CreateDevice( + const DeviceInfo& device_info) override; + + private: + ref_ptr syms_; + VkInstance instance_; + std::unique_ptr debug_reporter_; + ExtensibilitySpec device_extensibility_spec_; +}; + +} // namespace vulkan +} // namespace hal +} // namespace iree + +#endif // THIRD_PARTY_MLIR_EDGE_IREE_HAL_VULKAN_VULKAN_DRIVER_H_ diff --git a/hal/vulkan/vulkan_driver_module.cc b/hal/vulkan/vulkan_driver_module.cc new file mode 100644 index 000000000000..30b085341434 --- /dev/null +++ b/hal/vulkan/vulkan_driver_module.cc @@ -0,0 +1,100 @@ +// Copyright 2019 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include + +#include "third_party/absl/flags/flag.h" +#include "third_party/mlir_edge/iree/base/init.h" +#include "third_party/mlir_edge/iree/base/status.h" +#include "third_party/mlir_edge/iree/base/tracing.h" +#include "third_party/mlir_edge/iree/hal/driver_registry.h" +#include "third_party/mlir_edge/iree/hal/vulkan/dynamic_symbols.h" +#include "third_party/mlir_edge/iree/hal/vulkan/vulkan_driver.h" + +ABSL_FLAG(bool, vulkan_validation_layers, true, + "Enables standard Vulkan validation layers."); +ABSL_FLAG(bool, vulkan_debug_utils, true, + "Enables VK_EXT_debug_utils, records markers, and logs errors."); +ABSL_FLAG(bool, vulkan_debug_report, false, + "Enables VK_EXT_debug_report and logs errors."); +ABSL_FLAG(bool, vulkan_push_descriptors, true, + "Enables use of vkCmdPushDescriptorSetKHR, if available."); + +namespace iree { +namespace hal { +namespace vulkan { + +StatusOr> CreateVulkanDriver() { + IREE_TRACE_SCOPE0("CreateVulkanDriver"); + + // Load the Vulkan library. This will fail if the library cannot be found or + // does not have the expected functions. + ASSIGN_OR_RETURN(auto syms, DynamicSymbols::CreateFromSystemLoader()); + + // Setup driver options from flags. We do this here as we want to enable other + // consumers that may not be using modules/command line flags to be able to + // set their options however they want. + VulkanDriver::Options options; + + // HACK: validation layers have bugs when using VK_EXT_debug_report, so if the + // user requested that we force them off with a warning. Prefer using + // VK_EXT_debug_utils when available. + if (absl::GetFlag(FLAGS_vulkan_debug_report) && + absl::GetFlag(FLAGS_vulkan_validation_layers)) { + LOG(WARNING) << "VK_EXT_debug_report has issues with modern validation " + "layers; disabling validation"; + absl::SetFlag(&FLAGS_vulkan_validation_layers, false); + } + + // REQUIRED: these are required extensions that must be present for IREE to + // work (such as those relied upon by SPIR-V kernels, etc). + options.device_extensibility.required_extensions.push_back( + VK_KHR_STORAGE_BUFFER_STORAGE_CLASS_EXTENSION_NAME); + + if (absl::GetFlag(FLAGS_vulkan_validation_layers)) { + options.instance_extensibility.optional_layers.push_back( + "VK_LAYER_LUNARG_standard_validation"); + } + + if (absl::GetFlag(FLAGS_vulkan_debug_report)) { + options.instance_extensibility.optional_extensions.push_back( + VK_EXT_DEBUG_REPORT_EXTENSION_NAME); + } + if (absl::GetFlag(FLAGS_vulkan_debug_utils)) { + options.instance_extensibility.optional_extensions.push_back( + VK_EXT_DEBUG_UTILS_EXTENSION_NAME); + } + + if (absl::GetFlag(FLAGS_vulkan_push_descriptors)) { + options.instance_extensibility.optional_extensions.push_back( + VK_KHR_GET_PHYSICAL_DEVICE_PROPERTIES_2_EXTENSION_NAME); + options.device_extensibility.optional_extensions.push_back( + VK_KHR_PUSH_DESCRIPTOR_EXTENSION_NAME); + } + + // Create the driver and VkInstance. + ASSIGN_OR_RETURN(auto driver, VulkanDriver::Create(options, std::move(syms))); + + return driver; +} + +} // namespace vulkan +} // namespace hal +} // namespace iree + +IREE_REGISTER_MODULE_INITIALIZER(iree_hal_vulkan_driver, { + QCHECK_OK(::iree::hal::DriverRegistry::shared_registry()->Register( + "vulkan", ::iree::hal::vulkan::CreateVulkanDriver)); +}); +IREE_REGISTER_MODULE_INITIALIZER_SEQUENCE(iree_hal, iree_hal_vulkan_driver); diff --git a/samples/hal/simple_compute_test.mlir b/samples/hal/simple_compute_test.mlir new file mode 100644 index 000000000000..ade3e2163629 --- /dev/null +++ b/samples/hal/simple_compute_test.mlir @@ -0,0 +1,5 @@ +func @simple_mul(%arg0: tensor<4xf32>, %arg1: tensor<4xf32>) -> tensor<4xf32> + attributes { iree.module.export } { + %0 = "xla_hlo.mul"(%arg0, %arg1) {name = "mul.1"} : (tensor<4xf32>, tensor<4xf32>) -> tensor<4xf32> + return %0 : tensor<4xf32> +} diff --git a/samples/vm/simple_module_test.mlir b/samples/vm/simple_module_test.mlir new file mode 100644 index 000000000000..ade3e2163629 --- /dev/null +++ b/samples/vm/simple_module_test.mlir @@ -0,0 +1,5 @@ +func @simple_mul(%arg0: tensor<4xf32>, %arg1: tensor<4xf32>) -> tensor<4xf32> + attributes { iree.module.export } { + %0 = "xla_hlo.mul"(%arg0, %arg1) {name = "mul.1"} : (tensor<4xf32>, tensor<4xf32>) -> tensor<4xf32> + return %0 : tensor<4xf32> +} diff --git a/schemas/archive_def.fbs b/schemas/archive_def.fbs new file mode 100644 index 000000000000..959a4a55d5e7 --- /dev/null +++ b/schemas/archive_def.fbs @@ -0,0 +1,14 @@ +include "third_party/mlir_edge/iree/schemas/module_def.fbs"; + +namespace iree; + +// 'Executable ARChive'. +file_identifier "EARC"; +file_extension "earc"; + +table ArchiveDef { + name:string; + modules:[ModuleDef]; +} + +root_type ArchiveDef; diff --git a/schemas/bytecode/bytecode_v0.h b/schemas/bytecode/bytecode_v0.h new file mode 100644 index 000000000000..b3b5f95b731e --- /dev/null +++ b/schemas/bytecode/bytecode_v0.h @@ -0,0 +1,143 @@ +// Copyright 2019 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// Opcode table for the V0 binary format. +// Additions are fine but changing the behavior or order of any opcodes will +// break pasring of existing files. +// +// Opcodes have been selected on frequency of use, general applicability, and +// relative stability. Experimental ops should be implemented via the FFI fisrt +// before graduating into the core set. Ops that may only be present on certain +// targets should also be kept as imports via the FFI. +// +// Opcodes may be specified for particular types (int32_t), categories of types +// (all floating-point types), or implicit types (output matches input). Saving +// opcode space by sharing a single opcode for multiple types is preferred +// except where hot operations are performed (for example, comparison used in +// loop iteratosr). + +#ifndef THIRD_PARTY_MLIR_EDGE_IREE_SCHEMAS_BYTECODE_BYTECODE_V0_H_ +#define THIRD_PARTY_MLIR_EDGE_IREE_SCHEMAS_BYTECODE_BYTECODE_V0_H_ + +#include + +#include "third_party/mlir_edge/iree/base/bitfield.h" + +namespace iree { + +#define IREE_CONSTANT_ENCODING_LIST(ENC) \ + ENC(0x00, kDense, "dense") \ + ENC(0x01, kSplat, "splat") + +#define IREE_TYPE_LIST(TYP) \ + TYP(0x00, kI8, "i8", 1) \ + TYP(0x01, kI16, "i16", 2) \ + TYP(0x02, kI32, "i32", 4) \ + TYP(0x03, kI64, "i64", 8) \ + TYP(0x04, kF16, "f16", 2) \ + TYP(0x05, kF32, "f32", 4) \ + TYP(0x06, kF64, "f64", 8) \ + TYP(0x80, kDevice, "device", 0) \ + TYP(0x81, kCommandBuffer, "command_buffer", 0) \ + TYP(0x82, kEvent, "event", 0) \ + TYP(0x83, kSemaphore, "semaphore", 0) \ + TYP(0x84, kFence, "fence", 0) \ + TYP(0xFF, kOpaque, "opaque", 0) + +#define IREE_CMPI_PREDICATE_LIST(PRED) \ + PRED(0, kEq, "eq") \ + PRED(1, kNe, "ne") \ + PRED(2, kSlt, "slt") \ + PRED(3, kSle, "sle") \ + PRED(4, kSgt, "sgt") \ + PRED(5, kSge, "sge") \ + PRED(6, kUlt, "ult") \ + PRED(7, kUle, "ule") \ + PRED(8, kUgt, "ugt") \ + PRED(9, kUge, "uge") + +#define IREE_CMPF_PREDICATE_LIST(PRED) \ + PRED(0, kFalse, "false") \ + PRED(1, kOeq, "oeq") \ + PRED(2, kOgt, "ogt") \ + PRED(3, kOge, "oge") \ + PRED(4, kOlt, "olt") \ + PRED(5, kOle, "ole") \ + PRED(6, kOne, "one") \ + PRED(7, kOrd, "ord") \ + PRED(8, kUeq, "ueq") \ + PRED(9, kUgt, "ugt") \ + PRED(10, kUge, "uge") \ + PRED(11, kUlt, "ult") \ + PRED(12, kUle, "ule") \ + PRED(13, kUne, "une") \ + PRED(14, kUno, "uno") \ + PRED(15, kTrue, "true") + +// NOTE: FF is a to-be-defined flag value for encoding/decoding. +#define FLAG(V) ::iree::OpcodeFlag::V + +#define RSV(opcode, RESERVED_OPC) \ + RESERVED_OPC(opcode, kReserved##opcode, "rsv." #opcode, FLAG(kDefault), "", \ + FF) + +#define DECLARE_ENUM(ordinal, enum_name, ...) enum_name = ordinal, + +enum class ConstantEncoding : uint8_t { + IREE_CONSTANT_ENCODING_LIST(DECLARE_ENUM) +}; + +enum class BuiltinType : uint8_t { IREE_TYPE_LIST(DECLARE_ENUM) }; + +enum class CmpIPredicate : uint8_t { IREE_CMPI_PREDICATE_LIST(DECLARE_ENUM) }; + +enum class CmpFPredicate : uint8_t { IREE_CMPF_PREDICATE_LIST(DECLARE_ENUM) }; + +#undef DECLARE_ENUM + +static constexpr uint8_t kBuiltinTypeCount = + static_cast(BuiltinType::kF64) + 1; + +enum class OpcodeFlag : uint8_t { + kDefault = 0, +}; +IREE_BITFIELD(OpcodeFlag); +using OpcodeFlagBitfield = OpcodeFlag; + +enum class OperandEncoding : char { + kNone = '\0', + kInputSlot = 's', + kVariadicInputSlots = 'S', + kOutputSlot = 'o', + kVariadicOutputSlots = 'O', + kResultSlot = 'r', + kVariadicResultSlots = 'R', + kVariadicTransferSlots = 'T', + kConstant = 'c', + kFunctionOrdinal = 'f', + kImportOrdinal = 'F', + kDispatchOrdinal = 'd', + kBlockOffset = 'b', + kTypeIndex = 't', + kIndex = 'i', + kIndexList = 'I', + kCmpIPredicate = 'p', + kCmpFPredicate = 'P', +}; +IREE_BITFIELD(OperandEncoding); +using OperandEncodingBitfield = OperandEncoding; + +} // namespace iree + +#endif // THIRD_PARTY_MLIR_EDGE_IREE_SCHEMAS_BYTECODE_BYTECODE_V0_H_ diff --git a/schemas/bytecode/interpreter_bytecode_v0.h b/schemas/bytecode/interpreter_bytecode_v0.h new file mode 100644 index 000000000000..59f8468718ae --- /dev/null +++ b/schemas/bytecode/interpreter_bytecode_v0.h @@ -0,0 +1,325 @@ +// Copyright 2019 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// Opcode table for the V0 binary format. +// Additions are fine but changing the behavior or order of any opcodes will +// break parsing of existing files. +// +// Opcodes have been selected on frequency of use, general applicability, and +// relative stability. Experimental ops should be implemented via the Foreign +// Function Interface (FFI) first before graduating into the core set. Ops that +// may only be present on certain targets should also be kept as imports via the +// FFI. +// +// Opcodes may be specified for particular types (int32_t), categories of types +// (all floating-point types), or implicit types (output matches input). Saving +// opcode space by sharing a single opcode for multiple types is preferred +// except where hot operations are performed (for example, comparison used in +// loop iterators). + +#ifndef THIRD_PARTY_MLIR_EDGE_IREE_SCHEMAS_BYTECODE_INTERPRETER_BYTECODE_V0_H_ +#define THIRD_PARTY_MLIR_EDGE_IREE_SCHEMAS_BYTECODE_INTERPRETER_BYTECODE_V0_H_ + +#include "third_party/mlir_edge/iree/schemas/bytecode/bytecode_v0.h" + +namespace iree { + +#define IREE_INTERPRETER_OPCODE_LIST(OPC, RESERVED_OPC) \ + OPC(0x00, kConstant, "constant", FLAG(kDefault), "cr", FF) \ + \ + OPC(0x01, kCall, "call", FLAG(kDefault), "fSR", FF) \ + OPC(0x02, kCallImport, "call_import", FLAG(kDefault), "FSR", FF) \ + OPC(0x03, kCallIndirect, "call_indirect", FLAG(kDefault), "tsSR", FF) \ + OPC(0x04, kReturn, "return", FLAG(kDefault), "S", FF) \ + OPC(0x05, kBranch, "br", FLAG(kDefault), "bT", FF) \ + OPC(0x06, kCondBranch, "cond_br", FLAG(kDefault), "sbTbT", FF) \ + OPC(0x07, kCmpI, "cmp_i", FLAG(kDefault), "psso", FF) \ + OPC(0x08, kCmpF, "cmp_f", FLAG(kDefault), "Psso", FF) \ + \ + RSV(0x09, RESERVED_OPC) \ + RSV(0x0A, RESERVED_OPC) \ + RSV(0x0B, RESERVED_OPC) \ + RSV(0x0C, RESERVED_OPC) \ + RSV(0x0D, RESERVED_OPC) \ + RSV(0x0E, RESERVED_OPC) \ + RSV(0x0F, RESERVED_OPC) \ + RSV(0x10, RESERVED_OPC) \ + RSV(0x11, RESERVED_OPC) \ + RSV(0x12, RESERVED_OPC) \ + RSV(0x13, RESERVED_OPC) \ + RSV(0x14, RESERVED_OPC) \ + RSV(0x15, RESERVED_OPC) \ + RSV(0x16, RESERVED_OPC) \ + RSV(0x17, RESERVED_OPC) \ + RSV(0x18, RESERVED_OPC) \ + RSV(0x19, RESERVED_OPC) \ + RSV(0x1A, RESERVED_OPC) \ + RSV(0x1B, RESERVED_OPC) \ + RSV(0x1C, RESERVED_OPC) \ + RSV(0x1D, RESERVED_OPC) \ + RSV(0x1E, RESERVED_OPC) \ + RSV(0x1F, RESERVED_OPC) \ + \ + OPC(0x20, kAllocStatic, "alloc_static", FLAG(kDefault), "Icr", FF) \ + OPC(0x21, kAllocStack, "alloc_stack", FLAG(kDefault), "itISr", FF) \ + OPC(0x22, kAllocStackInit, "alloc_stack_init", FLAG(kDefault), "tIScr", FF) \ + OPC(0x23, kAllocHeap, "alloc_heap", FLAG(kDefault), "itISr", FF) \ + OPC(0x24, kDiscard, "discard", FLAG(kDefault), "s", FF) \ + \ + RSV(0x25, RESERVED_OPC) \ + RSV(0x26, RESERVED_OPC) \ + RSV(0x27, RESERVED_OPC) \ + RSV(0x28, RESERVED_OPC) \ + RSV(0x29, RESERVED_OPC) \ + RSV(0x2A, RESERVED_OPC) \ + RSV(0x2B, RESERVED_OPC) \ + RSV(0x2C, RESERVED_OPC) \ + RSV(0x2D, RESERVED_OPC) \ + RSV(0x2E, RESERVED_OPC) \ + RSV(0x2F, RESERVED_OPC) \ + \ + OPC(0x30, kRank, "rank", FLAG(kDefault), "so", FF) \ + OPC(0x31, kDim, "dim", FLAG(kDefault), "iso", FF) \ + OPC(0x32, kShape, "shape", FLAG(kDefault), "so", FF) \ + OPC(0x33, kLength, "length", FLAG(kDefault), "so", FF) \ + OPC(0x34, kDynamicSlice, "dynamic_slice", FLAG(kDefault), "sssr", FF) \ + OPC(0x35, kStaticSlice, "static_slice", FLAG(kDefault), "sIIr", FF) \ + OPC(0x36, kDynamicCopy, "dynamic_copy", FLAG(kDefault), "ssoss", FF) \ + OPC(0x37, kStaticCopy, "static_copy", FLAG(kDefault), "sIoII", FF) \ + OPC(0x38, kClone, "clone", FLAG(kDefault), "sr", FF) \ + RSV(0x39, RESERVED_OPC) \ + OPC(0x3A, kSplit, "split", FLAG(kDefault), "isR", FF) \ + OPC(0x3B, kAssign, "assign", FLAG(kDefault), "sr", FF) \ + OPC(0x3C, kCondAssign, "cond_assign", FLAG(kDefault), "sssr", FF) \ + OPC(0x3D, kReshape, "reshape", FLAG(kDefault), "ssr", FF) \ + OPC(0x3E, kSelect, "select", FLAG(kDefault), "ssso", FF) \ + OPC(0x3F, kTranspose, "transpose", FLAG(kDefault), "sso", FF) \ + OPC(0x40, kBroadcast, "broadcast", FLAG(kDefault), "sso", FF) \ + OPC(0x41, kTile, "tile", FLAG(kDefault), "sso", FF) \ + OPC(0x42, kReverse, "reverse", FLAG(kDefault), "sso", FF) \ + OPC(0x43, kPad, "pad", FLAG(kDefault), "ssssso", FF) \ + \ + RSV(0x44, RESERVED_OPC) \ + RSV(0x45, RESERVED_OPC) \ + RSV(0x46, RESERVED_OPC) \ + RSV(0x47, RESERVED_OPC) \ + RSV(0x48, RESERVED_OPC) \ + RSV(0x49, RESERVED_OPC) \ + RSV(0x4A, RESERVED_OPC) \ + RSV(0x4B, RESERVED_OPC) \ + RSV(0x4C, RESERVED_OPC) \ + RSV(0x4D, RESERVED_OPC) \ + RSV(0x4E, RESERVED_OPC) \ + RSV(0x4F, RESERVED_OPC) \ + \ + OPC(0x50, kNot, "not", FLAG(kDefault), "so", FF) \ + OPC(0x51, kAnd, "and", FLAG(kDefault), "sso", FF) \ + OPC(0x52, kOr, "or", FLAG(kDefault), "sso", FF) \ + OPC(0x53, kXor, "xor", FLAG(kDefault), "sso", FF) \ + OPC(0x54, kShiftLeft, "sll", FLAG(kDefault), "sso", FF) \ + OPC(0x55, kShiftRightLogical, "srl", FLAG(kDefault), "sso", FF) \ + OPC(0x56, kShiftRightArithmetic, "sra", FLAG(kDefault), "sso", FF) \ + \ + RSV(0x57, RESERVED_OPC) \ + RSV(0x58, RESERVED_OPC) \ + RSV(0x59, RESERVED_OPC) \ + RSV(0x5A, RESERVED_OPC) \ + RSV(0x5B, RESERVED_OPC) \ + RSV(0x5C, RESERVED_OPC) \ + RSV(0x5D, RESERVED_OPC) \ + RSV(0x5E, RESERVED_OPC) \ + RSV(0x5F, RESERVED_OPC) \ + RSV(0x60, RESERVED_OPC) \ + RSV(0x61, RESERVED_OPC) \ + RSV(0x62, RESERVED_OPC) \ + RSV(0x63, RESERVED_OPC) \ + RSV(0x64, RESERVED_OPC) \ + RSV(0x65, RESERVED_OPC) \ + RSV(0x66, RESERVED_OPC) \ + RSV(0x67, RESERVED_OPC) \ + RSV(0x68, RESERVED_OPC) \ + RSV(0x69, RESERVED_OPC) \ + RSV(0x6A, RESERVED_OPC) \ + RSV(0x6B, RESERVED_OPC) \ + RSV(0x6C, RESERVED_OPC) \ + RSV(0x6D, RESERVED_OPC) \ + RSV(0x6E, RESERVED_OPC) \ + RSV(0x6F, RESERVED_OPC) \ + \ + /* TODO(benvanik): remove ones we don't need/can emulate */ \ + OPC(0x70, kAddI, "add_i", FLAG(kDefault), "sso", FF) \ + OPC(0x71, kAddF, "add_f", FLAG(kDefault), "sso", FF) \ + OPC(0x72, kSubI, "sub_i", FLAG(kDefault), "sso", FF) \ + OPC(0x73, kSubF, "sub_f", FLAG(kDefault), "sso", FF) \ + OPC(0x74, kAbsI, "abs_i", FLAG(kDefault), "so", FF) \ + OPC(0x75, kAbsF, "abs_f", FLAG(kDefault), "so", FF) \ + OPC(0x76, kMulI, "mul_i", FLAG(kDefault), "sso", FF) \ + OPC(0x77, kMulF, "mul_f", FLAG(kDefault), "sso", FF) \ + OPC(0x78, kDivIS, "div_i_s", FLAG(kDefault), "sso", FF) \ + OPC(0x79, kDivIU, "div_i_u", FLAG(kDefault), "sso", FF) \ + OPC(0x7A, kDivF, "div_f", FLAG(kDefault), "sso", FF) \ + OPC(0x7B, kMulAddI, "madd_i", FLAG(kDefault), "ssso", FF) \ + OPC(0x7C, kMulAddF, "madd_f", FLAG(kDefault), "ssso", FF) \ + OPC(0x7D, kCosF, "cos_f", FLAG(kDefault), "so", FF) \ + OPC(0x7E, kSinF, "sin_f", FLAG(kDefault), "so", FF) \ + OPC(0x7F, kTanhF, "tanh_f", FLAG(kDefault), "so", FF) \ + OPC(0x80, kAtan2F, "atan2_f", FLAG(kDefault), "sso", FF) \ + OPC(0x81, kExpF, "exp_f", FLAG(kDefault), "so", FF) \ + OPC(0x82, kLogF, "log_f", FLAG(kDefault), "so", FF) \ + OPC(0x83, kRsqrtF, "rsqrt_f", FLAG(kDefault), "so", FF) \ + \ + RSV(0x84, RESERVED_OPC) \ + RSV(0x85, RESERVED_OPC) \ + RSV(0x86, RESERVED_OPC) \ + RSV(0x87, RESERVED_OPC) \ + RSV(0x88, RESERVED_OPC) \ + RSV(0x89, RESERVED_OPC) \ + RSV(0x8A, RESERVED_OPC) \ + RSV(0x8B, RESERVED_OPC) \ + RSV(0x8C, RESERVED_OPC) \ + RSV(0x8D, RESERVED_OPC) \ + RSV(0x8E, RESERVED_OPC) \ + RSV(0x8F, RESERVED_OPC) \ + \ + OPC(0x90, kMinIS, "min_i_s", FLAG(kDefault), "sso", FF) \ + OPC(0x91, kMinIU, "min_i_u", FLAG(kDefault), "sso", FF) \ + OPC(0x92, kMinF, "min_f", FLAG(kDefault), "sso", FF) \ + OPC(0x93, kMaxIS, "max_i_s", FLAG(kDefault), "sso", FF) \ + OPC(0x94, kMaxIU, "max_i_u", FLAG(kDefault), "sso", FF) \ + OPC(0x95, kMaxF, "max_f", FLAG(kDefault), "sso", FF) \ + OPC(0x96, kClampIS, "clamp_i_s", FLAG(kDefault), "ssso", FF) \ + OPC(0x97, kClampIU, "clamp_i_u", FLAG(kDefault), "ssso", FF) \ + OPC(0x98, kClampF, "clamp_f", FLAG(kDefault), "ssso", FF) \ + OPC(0x99, kFloorF, "floor_f", FLAG(kDefault), "so", FF) \ + OPC(0x9A, kCeilF, "ceil_f", FLAG(kDefault), "so", FF) \ + \ + OPC(0x9B, kConvertSS, "convert_s_s", FLAG(kDefault), "tsto", FF) \ + OPC(0x9C, kConvertUU, "convert_u_u", FLAG(kDefault), "tsto", FF) \ + OPC(0x9D, kConvertSU, "convert_s_u", FLAG(kDefault), "tsto", FF) \ + OPC(0x9E, kConvertUS, "convert_u_s", FLAG(kDefault), "tsto", FF) \ + \ + RSV(0x9F, RESERVED_OPC) \ + \ + /* TODO(benvanik): reduction/sum/etc */ \ + /* TODO(benvanik): sort */ \ + \ + OPC(0xA0, kMatMulI, "matmul_i", FLAG(kDefault), "sssso", FF) \ + OPC(0xA1, kMatMulF, "matmul_f", FLAG(kDefault), "sso", FF) \ + /* TODO(benvanik): convolution */ \ + \ + OPC(0xA2, kReduceSumI, "reduce_sum_i", FLAG(kDefault), "ssio", FF) \ + OPC(0xA3, kReduceSumF, "reduce_sum_f", FLAG(kDefault), "ssio", FF) \ + OPC(0xA4, kReduceMinI, "reduce_min_i", FLAG(kDefault), "ssio", FF) \ + OPC(0xA5, kReduceMinF, "reduce_min_f", FLAG(kDefault), "ssio", FF) \ + OPC(0xA6, kReduceMaxI, "reduce_max_i", FLAG(kDefault), "ssio", FF) \ + OPC(0xA7, kReduceMaxF, "reduce_max_f", FLAG(kDefault), "ssio", FF) \ + RSV(0xA8, RESERVED_OPC) \ + RSV(0xA9, RESERVED_OPC) \ + RSV(0xAA, RESERVED_OPC) \ + RSV(0xAB, RESERVED_OPC) \ + RSV(0xAC, RESERVED_OPC) \ + RSV(0xAD, RESERVED_OPC) \ + RSV(0xAE, RESERVED_OPC) \ + RSV(0xAF, RESERVED_OPC) \ + RSV(0xB0, RESERVED_OPC) \ + RSV(0xB1, RESERVED_OPC) \ + RSV(0xB2, RESERVED_OPC) \ + RSV(0xB3, RESERVED_OPC) \ + RSV(0xB4, RESERVED_OPC) \ + RSV(0xB5, RESERVED_OPC) \ + RSV(0xB6, RESERVED_OPC) \ + RSV(0xB7, RESERVED_OPC) \ + RSV(0xB8, RESERVED_OPC) \ + RSV(0xB9, RESERVED_OPC) \ + RSV(0xBA, RESERVED_OPC) \ + RSV(0xBB, RESERVED_OPC) \ + RSV(0xBC, RESERVED_OPC) \ + RSV(0xBD, RESERVED_OPC) \ + RSV(0xBE, RESERVED_OPC) \ + RSV(0xBF, RESERVED_OPC) \ + RSV(0xC0, RESERVED_OPC) \ + RSV(0xC1, RESERVED_OPC) \ + RSV(0xC2, RESERVED_OPC) \ + RSV(0xC3, RESERVED_OPC) \ + RSV(0xC4, RESERVED_OPC) \ + RSV(0xC5, RESERVED_OPC) \ + RSV(0xC6, RESERVED_OPC) \ + RSV(0xC7, RESERVED_OPC) \ + RSV(0xC8, RESERVED_OPC) \ + RSV(0xC9, RESERVED_OPC) \ + RSV(0xCA, RESERVED_OPC) \ + RSV(0xCB, RESERVED_OPC) \ + RSV(0xCC, RESERVED_OPC) \ + RSV(0xCD, RESERVED_OPC) \ + RSV(0xCE, RESERVED_OPC) \ + RSV(0xCF, RESERVED_OPC) \ + RSV(0xD0, RESERVED_OPC) \ + RSV(0xD1, RESERVED_OPC) \ + RSV(0xD2, RESERVED_OPC) \ + RSV(0xD3, RESERVED_OPC) \ + RSV(0xD4, RESERVED_OPC) \ + RSV(0xD5, RESERVED_OPC) \ + RSV(0xD6, RESERVED_OPC) \ + RSV(0xD7, RESERVED_OPC) \ + RSV(0xD8, RESERVED_OPC) \ + RSV(0xD9, RESERVED_OPC) \ + RSV(0xDA, RESERVED_OPC) \ + RSV(0xDB, RESERVED_OPC) \ + RSV(0xDC, RESERVED_OPC) \ + RSV(0xDD, RESERVED_OPC) \ + RSV(0xDE, RESERVED_OPC) \ + RSV(0xDF, RESERVED_OPC) \ + RSV(0xE0, RESERVED_OPC) \ + RSV(0xE1, RESERVED_OPC) \ + RSV(0xE2, RESERVED_OPC) \ + RSV(0xE3, RESERVED_OPC) \ + RSV(0xE4, RESERVED_OPC) \ + RSV(0xE5, RESERVED_OPC) \ + RSV(0xE6, RESERVED_OPC) \ + RSV(0xE7, RESERVED_OPC) \ + RSV(0xE8, RESERVED_OPC) \ + RSV(0xE9, RESERVED_OPC) \ + RSV(0xEA, RESERVED_OPC) \ + RSV(0xEB, RESERVED_OPC) \ + RSV(0xEC, RESERVED_OPC) \ + RSV(0xED, RESERVED_OPC) \ + RSV(0xEE, RESERVED_OPC) \ + RSV(0xEF, RESERVED_OPC) \ + RSV(0xF0, RESERVED_OPC) \ + RSV(0xF1, RESERVED_OPC) \ + RSV(0xF2, RESERVED_OPC) \ + RSV(0xF3, RESERVED_OPC) \ + RSV(0xF4, RESERVED_OPC) \ + RSV(0xF5, RESERVED_OPC) \ + RSV(0xF6, RESERVED_OPC) \ + RSV(0xF7, RESERVED_OPC) \ + RSV(0xF8, RESERVED_OPC) \ + RSV(0xF9, RESERVED_OPC) \ + RSV(0xFA, RESERVED_OPC) \ + RSV(0xFB, RESERVED_OPC) \ + RSV(0xFC, RESERVED_OPC) \ + \ + OPC(0xFD, kTrace, "trace", FLAG(kDefault), "s", FF) \ + OPC(0xFE, kCondBreak, "cond_break", FLAG(kDefault), "s", FF) \ + OPC(0xFF, kBreak, "break", FLAG(kDefault), "", FF) + +#define DECLARE_ENUM(ordinal, enum_name, ...) enum_name = ordinal, +enum class InterpreterOpcode : uint8_t { + IREE_INTERPRETER_OPCODE_LIST(DECLARE_ENUM, DECLARE_ENUM) +}; +#undef DECLARE_ENUM + +} // namespace iree + +#endif // THIRD_PARTY_MLIR_EDGE_IREE_SCHEMAS_BYTECODE_INTERPRETER_BYTECODE_V0_H_ diff --git a/schemas/bytecode/sequencer_bytecode_v0.h b/schemas/bytecode/sequencer_bytecode_v0.h new file mode 100644 index 000000000000..609cb65770f1 --- /dev/null +++ b/schemas/bytecode/sequencer_bytecode_v0.h @@ -0,0 +1,313 @@ +// Copyright 2019 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// Opcode table for the V0 binary format. +// Additions are fine but changing the behavior or order of any opcodes will +// break parsing of existing files. +// +// Opcodes have been selected on frequency of use, general applicability, and +// relative stability. Experimental ops should be implemented via the Foreign +// Function Interface (FFI) first before graduating into the core set. Ops that +// may only be present on certain targets should also be kept as imports via the +// FFI. +// +// Opcodes may be specified for particular types (int32_t), categories of types +// (all floating-point types), or implicit types (output matches input). Saving +// opcode space by sharing a single opcode for multiple types is preferred +// except where hot operations are performed (for example, comparison used in +// loop iterators). + +#ifndef THIRD_PARTY_MLIR_EDGE_IREE_SCHEMAS_BYTECODE_SEQUENCER_BYTECODE_V0_H_ +#define THIRD_PARTY_MLIR_EDGE_IREE_SCHEMAS_BYTECODE_SEQUENCER_BYTECODE_V0_H_ + +#include "third_party/mlir_edge/iree/schemas/bytecode/bytecode_v0.h" + +namespace iree { + +#define IREE_SEQUENCER_OPCODE_LIST(OPC, RESERVED_OPC) \ + OPC(0x00, kConstant, "constant", FLAG(kDefault), "cr", FF) \ + \ + OPC(0x01, kCall, "call", FLAG(kDefault), "fSR", FF) \ + OPC(0x02, kCallImport, "call_import", FLAG(kDefault), "FSR", FF) \ + OPC(0x03, kCallIndirect, "call_indirect", FLAG(kDefault), "tsSR", FF) \ + OPC(0x04, kReturn, "return", FLAG(kDefault), "S", FF) \ + OPC(0x05, kBranch, "br", FLAG(kDefault), "bT", FF) \ + OPC(0x06, kCondBranch, "cond_br", FLAG(kDefault), "sbTbT", FF) \ + \ + RSV(0x07, RESERVED_OPC) \ + RSV(0x08, RESERVED_OPC) \ + RSV(0x09, RESERVED_OPC) \ + RSV(0x0A, RESERVED_OPC) \ + RSV(0x0B, RESERVED_OPC) \ + RSV(0x0C, RESERVED_OPC) \ + RSV(0x0D, RESERVED_OPC) \ + RSV(0x0E, RESERVED_OPC) \ + RSV(0x0F, RESERVED_OPC) \ + \ + OPC(0x10, kDynamicDispatch, "dynamic_dispatch", FLAG(kDefault), "dsSOR", FF) \ + OPC(0x11, kStaticDispatch, "static_dispatch", FLAG(kDefault), "diiiSOR", FF) \ + \ + RSV(0x12, RESERVED_OPC) \ + RSV(0x13, RESERVED_OPC) \ + RSV(0x14, RESERVED_OPC) \ + RSV(0x15, RESERVED_OPC) \ + RSV(0x16, RESERVED_OPC) \ + RSV(0x17, RESERVED_OPC) \ + RSV(0x18, RESERVED_OPC) \ + RSV(0x19, RESERVED_OPC) \ + RSV(0x1A, RESERVED_OPC) \ + RSV(0x1B, RESERVED_OPC) \ + RSV(0x1C, RESERVED_OPC) \ + RSV(0x1D, RESERVED_OPC) \ + RSV(0x1E, RESERVED_OPC) \ + RSV(0x1F, RESERVED_OPC) \ + \ + OPC(0x20, kAllocStatic, "alloc_static", FLAG(kDefault), "Icr", FF) \ + OPC(0x21, kAllocStack, "alloc_stack", FLAG(kDefault), "itISr", FF) \ + OPC(0x22, kAllocStackInit, "alloc_stack_init", FLAG(kDefault), "tIScr", FF) \ + OPC(0x23, kAllocHeap, "alloc_heap", FLAG(kDefault), "itISr", FF) \ + OPC(0x24, kDiscard, "discard", FLAG(kDefault), "s", FF) \ + \ + RSV(0x25, RESERVED_OPC) \ + RSV(0x26, RESERVED_OPC) \ + RSV(0x27, RESERVED_OPC) \ + RSV(0x28, RESERVED_OPC) \ + RSV(0x29, RESERVED_OPC) \ + RSV(0x2A, RESERVED_OPC) \ + RSV(0x2B, RESERVED_OPC) \ + RSV(0x2C, RESERVED_OPC) \ + RSV(0x2D, RESERVED_OPC) \ + RSV(0x2E, RESERVED_OPC) \ + RSV(0x2F, RESERVED_OPC) \ + RSV(0x30, RESERVED_OPC) \ + \ + OPC(0x31, kComputeRange, "compute_range", FLAG(kDefault), "sissoo", FF) \ + OPC(0x32, kShape, "shape", FLAG(kDefault), "so", FF) \ + OPC(0x33, kLength, "length", FLAG(kDefault), "so", FF) \ + OPC(0x34, kDynamicSlice, "dynamic_slice", FLAG(kDefault), "ssstsr", FF) \ + OPC(0x35, kStaticSlice, "static_slice", FLAG(kDefault), "siitIr", FF) \ + OPC(0x36, kDynamicCopy, "dynamic_copy", FLAG(kDefault), "ssoss", FF) \ + OPC(0x37, kStaticCopy, "static_copy", FLAG(kDefault), "sioii", FF) \ + OPC(0x38, kDynamicFill, "dynamic_fill", FLAG(kDefault), "soss", FF) \ + OPC(0x39, kStaticFill, "static_fill", FLAG(kDefault), "ioii", FF) \ + OPC(0x3A, kClone, "clone", FLAG(kDefault), "sr", FF) \ + OPC(0x3B, kAssign, "assign", FLAG(kDefault), "sr", FF) \ + OPC(0x3C, kCondAssign, "cond_assign", FLAG(kDefault), "sssr", FF) \ + OPC(0x3D, kReshape, "reshape", FLAG(kDefault), "ssr", FF) \ + \ + RSV(0x3E, RESERVED_OPC) \ + RSV(0x3F, RESERVED_OPC) \ + RSV(0x40, RESERVED_OPC) \ + RSV(0x41, RESERVED_OPC) \ + RSV(0x42, RESERVED_OPC) \ + RSV(0x43, RESERVED_OPC) \ + RSV(0x44, RESERVED_OPC) \ + RSV(0x45, RESERVED_OPC) \ + RSV(0x46, RESERVED_OPC) \ + RSV(0x47, RESERVED_OPC) \ + RSV(0x48, RESERVED_OPC) \ + RSV(0x49, RESERVED_OPC) \ + RSV(0x4A, RESERVED_OPC) \ + RSV(0x4B, RESERVED_OPC) \ + RSV(0x4C, RESERVED_OPC) \ + RSV(0x4D, RESERVED_OPC) \ + RSV(0x4E, RESERVED_OPC) \ + RSV(0x4F, RESERVED_OPC) \ + RSV(0x50, RESERVED_OPC) \ + RSV(0x51, RESERVED_OPC) \ + RSV(0x52, RESERVED_OPC) \ + RSV(0x53, RESERVED_OPC) \ + RSV(0x54, RESERVED_OPC) \ + RSV(0x55, RESERVED_OPC) \ + RSV(0x56, RESERVED_OPC) \ + RSV(0x57, RESERVED_OPC) \ + RSV(0x58, RESERVED_OPC) \ + RSV(0x59, RESERVED_OPC) \ + RSV(0x5A, RESERVED_OPC) \ + RSV(0x5B, RESERVED_OPC) \ + RSV(0x5C, RESERVED_OPC) \ + RSV(0x5D, RESERVED_OPC) \ + RSV(0x5E, RESERVED_OPC) \ + RSV(0x5F, RESERVED_OPC) \ + RSV(0x60, RESERVED_OPC) \ + RSV(0x61, RESERVED_OPC) \ + RSV(0x62, RESERVED_OPC) \ + RSV(0x63, RESERVED_OPC) \ + RSV(0x64, RESERVED_OPC) \ + RSV(0x65, RESERVED_OPC) \ + RSV(0x66, RESERVED_OPC) \ + RSV(0x67, RESERVED_OPC) \ + RSV(0x68, RESERVED_OPC) \ + RSV(0x69, RESERVED_OPC) \ + RSV(0x6A, RESERVED_OPC) \ + RSV(0x6B, RESERVED_OPC) \ + RSV(0x6C, RESERVED_OPC) \ + RSV(0x6D, RESERVED_OPC) \ + RSV(0x6E, RESERVED_OPC) \ + RSV(0x6F, RESERVED_OPC) \ + RSV(0x70, RESERVED_OPC) \ + RSV(0x71, RESERVED_OPC) \ + RSV(0x72, RESERVED_OPC) \ + RSV(0x73, RESERVED_OPC) \ + RSV(0x74, RESERVED_OPC) \ + RSV(0x75, RESERVED_OPC) \ + RSV(0x76, RESERVED_OPC) \ + RSV(0x77, RESERVED_OPC) \ + RSV(0x78, RESERVED_OPC) \ + RSV(0x79, RESERVED_OPC) \ + RSV(0x7A, RESERVED_OPC) \ + RSV(0x7B, RESERVED_OPC) \ + RSV(0x7C, RESERVED_OPC) \ + RSV(0x7D, RESERVED_OPC) \ + RSV(0x7E, RESERVED_OPC) \ + RSV(0x7F, RESERVED_OPC) \ + RSV(0x80, RESERVED_OPC) \ + RSV(0x81, RESERVED_OPC) \ + RSV(0x82, RESERVED_OPC) \ + RSV(0x83, RESERVED_OPC) \ + RSV(0x84, RESERVED_OPC) \ + RSV(0x85, RESERVED_OPC) \ + RSV(0x86, RESERVED_OPC) \ + RSV(0x87, RESERVED_OPC) \ + RSV(0x88, RESERVED_OPC) \ + RSV(0x89, RESERVED_OPC) \ + RSV(0x8A, RESERVED_OPC) \ + RSV(0x8B, RESERVED_OPC) \ + RSV(0x8C, RESERVED_OPC) \ + RSV(0x8D, RESERVED_OPC) \ + RSV(0x8E, RESERVED_OPC) \ + RSV(0x8F, RESERVED_OPC) \ + RSV(0x90, RESERVED_OPC) \ + RSV(0x91, RESERVED_OPC) \ + RSV(0x92, RESERVED_OPC) \ + RSV(0x93, RESERVED_OPC) \ + RSV(0x94, RESERVED_OPC) \ + RSV(0x95, RESERVED_OPC) \ + RSV(0x96, RESERVED_OPC) \ + RSV(0x97, RESERVED_OPC) \ + RSV(0x98, RESERVED_OPC) \ + RSV(0x99, RESERVED_OPC) \ + RSV(0x9A, RESERVED_OPC) \ + RSV(0x9B, RESERVED_OPC) \ + RSV(0x9C, RESERVED_OPC) \ + RSV(0x9D, RESERVED_OPC) \ + RSV(0x9E, RESERVED_OPC) \ + RSV(0x9F, RESERVED_OPC) \ + RSV(0xA0, RESERVED_OPC) \ + RSV(0xA1, RESERVED_OPC) \ + RSV(0xA2, RESERVED_OPC) \ + RSV(0xA3, RESERVED_OPC) \ + RSV(0xA4, RESERVED_OPC) \ + RSV(0xA5, RESERVED_OPC) \ + RSV(0xA6, RESERVED_OPC) \ + RSV(0xA7, RESERVED_OPC) \ + RSV(0xA8, RESERVED_OPC) \ + RSV(0xA9, RESERVED_OPC) \ + RSV(0xAA, RESERVED_OPC) \ + RSV(0xAB, RESERVED_OPC) \ + RSV(0xAC, RESERVED_OPC) \ + RSV(0xAD, RESERVED_OPC) \ + RSV(0xAE, RESERVED_OPC) \ + RSV(0xAF, RESERVED_OPC) \ + RSV(0xB0, RESERVED_OPC) \ + RSV(0xB1, RESERVED_OPC) \ + RSV(0xB2, RESERVED_OPC) \ + RSV(0xB3, RESERVED_OPC) \ + RSV(0xB4, RESERVED_OPC) \ + RSV(0xB5, RESERVED_OPC) \ + RSV(0xB6, RESERVED_OPC) \ + RSV(0xB7, RESERVED_OPC) \ + RSV(0xB8, RESERVED_OPC) \ + RSV(0xB9, RESERVED_OPC) \ + RSV(0xBA, RESERVED_OPC) \ + RSV(0xBB, RESERVED_OPC) \ + RSV(0xBC, RESERVED_OPC) \ + RSV(0xBD, RESERVED_OPC) \ + RSV(0xBE, RESERVED_OPC) \ + RSV(0xBF, RESERVED_OPC) \ + RSV(0xC0, RESERVED_OPC) \ + RSV(0xC1, RESERVED_OPC) \ + RSV(0xC2, RESERVED_OPC) \ + RSV(0xC3, RESERVED_OPC) \ + RSV(0xC4, RESERVED_OPC) \ + RSV(0xC5, RESERVED_OPC) \ + RSV(0xC6, RESERVED_OPC) \ + RSV(0xC7, RESERVED_OPC) \ + RSV(0xC8, RESERVED_OPC) \ + RSV(0xC9, RESERVED_OPC) \ + RSV(0xCA, RESERVED_OPC) \ + RSV(0xCB, RESERVED_OPC) \ + RSV(0xCC, RESERVED_OPC) \ + RSV(0xCD, RESERVED_OPC) \ + RSV(0xCE, RESERVED_OPC) \ + RSV(0xCF, RESERVED_OPC) \ + RSV(0xD0, RESERVED_OPC) \ + RSV(0xD1, RESERVED_OPC) \ + RSV(0xD2, RESERVED_OPC) \ + RSV(0xD3, RESERVED_OPC) \ + RSV(0xD4, RESERVED_OPC) \ + RSV(0xD5, RESERVED_OPC) \ + RSV(0xD6, RESERVED_OPC) \ + RSV(0xD7, RESERVED_OPC) \ + RSV(0xD8, RESERVED_OPC) \ + RSV(0xD9, RESERVED_OPC) \ + RSV(0xDA, RESERVED_OPC) \ + RSV(0xDB, RESERVED_OPC) \ + RSV(0xDC, RESERVED_OPC) \ + RSV(0xDD, RESERVED_OPC) \ + RSV(0xDE, RESERVED_OPC) \ + RSV(0xDF, RESERVED_OPC) \ + RSV(0xE0, RESERVED_OPC) \ + RSV(0xE1, RESERVED_OPC) \ + RSV(0xE2, RESERVED_OPC) \ + RSV(0xE3, RESERVED_OPC) \ + RSV(0xE4, RESERVED_OPC) \ + RSV(0xE5, RESERVED_OPC) \ + RSV(0xE6, RESERVED_OPC) \ + RSV(0xE7, RESERVED_OPC) \ + RSV(0xE8, RESERVED_OPC) \ + RSV(0xE9, RESERVED_OPC) \ + RSV(0xEA, RESERVED_OPC) \ + RSV(0xEB, RESERVED_OPC) \ + RSV(0xEC, RESERVED_OPC) \ + RSV(0xED, RESERVED_OPC) \ + RSV(0xEE, RESERVED_OPC) \ + RSV(0xEF, RESERVED_OPC) \ + RSV(0xF0, RESERVED_OPC) \ + RSV(0xF1, RESERVED_OPC) \ + RSV(0xF2, RESERVED_OPC) \ + RSV(0xF3, RESERVED_OPC) \ + RSV(0xF4, RESERVED_OPC) \ + RSV(0xF5, RESERVED_OPC) \ + RSV(0xF6, RESERVED_OPC) \ + RSV(0xF7, RESERVED_OPC) \ + RSV(0xF8, RESERVED_OPC) \ + RSV(0xF9, RESERVED_OPC) \ + RSV(0xFA, RESERVED_OPC) \ + RSV(0xFB, RESERVED_OPC) \ + RSV(0xFC, RESERVED_OPC) \ + \ + OPC(0xFD, kTrace, "trace", FLAG(kDefault), "s", FF) \ + OPC(0xFE, kCondBreak, "cond_break", FLAG(kDefault), "s", FF) \ + OPC(0xFF, kBreak, "break", FLAG(kDefault), "", FF) + +#define DECLARE_ENUM(ordinal, enum_name, ...) enum_name = ordinal, +enum class SequencerOpcode : uint8_t { + IREE_SEQUENCER_OPCODE_LIST(DECLARE_ENUM, DECLARE_ENUM) +}; +#undef DECLARE_ENUM + +} // namespace iree + +#endif // THIRD_PARTY_MLIR_EDGE_IREE_SCHEMAS_BYTECODE_SEQUENCER_BYTECODE_V0_H_ diff --git a/schemas/bytecode_def.fbs b/schemas/bytecode_def.fbs new file mode 100644 index 000000000000..1f3206b7134a --- /dev/null +++ b/schemas/bytecode_def.fbs @@ -0,0 +1,8 @@ +namespace iree; + +table BytecodeDef { + local_count:int; + contents:[byte]; +} + +root_type BytecodeDef; diff --git a/schemas/debug_service.fbs b/schemas/debug_service.fbs new file mode 100644 index 000000000000..16e6d44e6212 --- /dev/null +++ b/schemas/debug_service.fbs @@ -0,0 +1,347 @@ +include "third_party/mlir_edge/iree/schemas/function_def.fbs"; +include "third_party/mlir_edge/iree/schemas/module_def.fbs"; + +namespace iree.vm.debug.rpc; + +table Status { + code:int; + message:string; +} + +table CreateSessionRequest { +} +table CreateSessionResponse { + session_id:int; +} + +table MakeReadyRequest { + session_id:int; +} +table MakeReadyResponse { +} + +table GetStatusRequest { + session_id:int; + // TODO(benvanik): caps debugger supports? version expected? +} +table GetStatusResponse { + protocol:int; + // TODO(benvanik): run state. + // TODO(benvanik): profiling state. +} + +table NativeFunctionDef { + name:string; + // TODO(benvanik): more information about the fns (stack trace of registrant?) +} + +table ContextDef { + context_id:int; + native_functions:[NativeFunctionDef]; + module_names:[string]; +} + +table ListContextsRequest { + session_id:int; +} + +table ListContextsResponse { + contexts:[ContextDef]; +} + +table GetModuleRequest { + session_id:int; + context_id:int; + module_name:string; +} +table GetModuleResponse { + module:ModuleDef; +} + +table GetFunctionRequest { + session_id:int; + context_id:int; + module_name:string; + function_ordinal:int; +} +table GetFunctionResponse { + bytecode:BytecodeDef; + // TODO(benvanik): import info (linked module, etc). +} + +table ResolveFunctionRequest { + session_id:int; + module_name:string; + function_name:string; +} +table ResolveFunctionResponse { + context_ids:[int]; + function_ordinal:int; +} + +table BufferViewDef { + is_valid:bool; + shape:[int]; + element_size:int; + // TODO(benvanik): buffer attrs (type, access, usage). + // TODO(benvanik): buffer size/allocated_size. + // TODO(benvanik): buffer data (if accessible). +} + +table StackFrameDef { + module_name:string; + function_ordinal:int; + offset:int; + locals:[BufferViewDef]; +} + +table FiberStateDef { + fiber_id:int; + frames:[StackFrameDef]; +} + +table ListFibersRequest { + session_id:int; +} +table ListFibersResponse { + fiber_states:[FiberStateDef]; +} + +table SuspendFibersRequest { + session_id:int; + fiber_ids:[int]; +} +table SuspendFibersResponse { + fiber_states:[FiberStateDef]; +} + +table ResumeFibersRequest { + session_id:int; + fiber_ids:[int]; +} +table ResumeFibersResponse { +} + +enum StepMode : uint8 { + STEP_ONCE = 0, + STEP_TO_OFFSET = 1, +} + +table StepFiberRequest { + session_id:int; + step_id:int; + fiber_id:int; + step_mode:StepMode; + bytecode_offset:int; +} +table StepFiberResponse {} + +table GetFiberLocalRequest { + session_id:int; + fiber_id:int; + frame_index:int; + local_index:int; +} +table GetFiberLocalResponse { + value:BufferViewDef; +} + +table SetFiberLocalRequest { + session_id:int; + fiber_id:int; + frame_index:int; + local_index:int; + value:BufferViewDef; +} +table SetFiberLocalResponse { + value:BufferViewDef; +} + +enum BreakpointType : uint8 { + BYTECODE_FUNCTION = 0, + NATIVE_FUNCTION = 1, +} + +table BreakpointDef { + breakpoint_id:int; + breakpoint_type:BreakpointType; + + module_name:string; + function_name:string; + function_ordinal:int; + bytecode_offset:int; +} + +table ListBreakpointsRequest { + session_id:int; +} +table ListBreakpointsResponse { + breakpoints:[BreakpointDef]; +} + +table AddBreakpointRequest { + session_id:int; + breakpoint:BreakpointDef; +} +table AddBreakpointResponse { + breakpoint:BreakpointDef; +} + +table RemoveBreakpointRequest { + session_id:int; + breakpoint_id:int; +} +table RemoveBreakpointResponse { +} + +table StartProfilingRequest { + session_id:int; + context_id:int; + // TODO(benvanik): profiling mode. + // mode: sampling_timing, instrumented_coverage, instrumented_log, + // invoke_log +} +table StartProfilingResponse { + // TODO(benvanik): current/new mode. +} + +table StopProfilingRequest { + session_id:int; + context_id:int; +} +table StopProfilingResponse { + // TODO(benvanik): profiling data. +} + +// TODO(benvanik): streaming profiling data query. + +table ServiceShutdownEvent { +} + +table ContextRegisteredEvent { + context_id:int; +} +table ContextUnregisteredEvent { + context_id:int; +} + +table ModuleLoadedEvent { + context_id:int; + module_name:string; +} + +table FiberRegisteredEvent { + fiber_id:int; +} +table FiberUnregisteredEvent { + fiber_id:int; +} + +table BreakpointResolvedEvent { + breakpoint:BreakpointDef; + context_id:int; +} + +table BreakpointHitEvent { + breakpoint_id:int; + fiber_state:FiberStateDef; +} + +table StepCompletedEvent { + step_id:int; + fiber_states:[FiberStateDef]; +} + +union RequestUnion { + CreateSessionRequest, + MakeReadyRequest, + GetStatusRequest, + ListContextsRequest, + GetModuleRequest, + GetFunctionRequest, + ResolveFunctionRequest, + ListFibersRequest, + SuspendFibersRequest, + ResumeFibersRequest, + StepFiberRequest, + GetFiberLocalRequest, + SetFiberLocalRequest, + ListBreakpointsRequest, + AddBreakpointRequest, + RemoveBreakpointRequest, + StartProfilingRequest, + StopProfilingRequest, +} + +union ResponseUnion { + CreateSessionResponse, + MakeReadyResponse, + GetStatusResponse, + ListContextsResponse, + GetModuleResponse, + GetFunctionResponse, + ResolveFunctionResponse, + ListFibersResponse, + SuspendFibersResponse, + ResumeFibersResponse, + StepFiberResponse, + GetFiberLocalResponse, + SetFiberLocalResponse, + ListBreakpointsResponse, + AddBreakpointResponse, + RemoveBreakpointResponse, + StartProfilingResponse, + StopProfilingResponse, +} + +union EventUnion { + ServiceShutdownEvent, + ContextRegisteredEvent, + ContextUnregisteredEvent, + ModuleLoadedEvent, + FiberRegisteredEvent, + FiberUnregisteredEvent, + BreakpointResolvedEvent, + BreakpointHitEvent, + StepCompletedEvent, +} + +table Request { + message:RequestUnion; +} + +table Response { + status:Status; + message:ResponseUnion; +} + +table ServicePacket { + response:Response; + event:EventUnion; +} + +// NOTE: we aren't using this yet as the FlatBuffers gRPC code is... suspect. +rpc_service DebugServiceRpc { + MakeReady(MakeReadyRequest):MakeReadyResponse; + + GetStatus(GetStatusRequest):GetStatusResponse; + + ListContexts(ListContextsRequest):ListContextsResponse; + GetModule(GetModuleRequest):GetModuleResponse; + GetFunction(GetFunctionRequest):GetFunctionResponse; + ResolveFunction(ResolveFunctionRequest):ResolveFunctionResponse; + + ListFibers(ListFibersRequest):ListFibersResponse; + SuspendFibers(SuspendFibersRequest):SuspendFibersResponse; + ResumeFibers(ResumeFibersRequest):ResumeFibersResponse; + StepFiber(StepFiberRequest):StepFiberResponse; + GetFiberLocal(GetFiberLocalRequest):GetFiberLocalResponse; + SetFiberLocal(SetFiberLocalRequest):SetFiberLocalResponse; + + ListBreakpoints(ListBreakpointsRequest):ListBreakpointsResponse; + AddBreakpoint(AddBreakpointRequest):AddBreakpointResponse; + RemoveBreakpoint(RemoveBreakpointRequest):RemoveBreakpointResponse; + + StartProfiling(StartProfilingRequest):StartProfilingResponse; + StopProfiling(StopProfilingRequest):StopProfilingResponse; +} diff --git a/schemas/device_def.fbs b/schemas/device_def.fbs new file mode 100644 index 000000000000..efcc79fbffd1 --- /dev/null +++ b/schemas/device_def.fbs @@ -0,0 +1,11 @@ +namespace iree; + +// A specification for a runtime-selected logical device. +// Devices are specified by a set of rules that the runtime can evaluate when +// performing device enumeration and resolution. All sequencer ops within a +// module then reference these devices by ordinal within the DeviceTableDef and +// can assume the rules have all been met (such as device type, available +// memory, or other minimum limits). +table DeviceDef { + // TODO(benvanik): define device rules. +} diff --git a/schemas/device_group_def.fbs b/schemas/device_group_def.fbs new file mode 100644 index 000000000000..d451df29b920 --- /dev/null +++ b/schemas/device_group_def.fbs @@ -0,0 +1,9 @@ +namespace iree; + +// Specifies two or more devices that must have certain sharing properties. +// This is used by the runtime to, for example, ensure that devices which are +// selected can reference each other's memory, have certain performance +// characteristics, etc. +table DeviceGroupDef { + // TODO(benvanik): define device groups. +} diff --git a/schemas/device_table_def.fbs b/schemas/device_table_def.fbs new file mode 100644 index 000000000000..68a10dd4713f --- /dev/null +++ b/schemas/device_table_def.fbs @@ -0,0 +1,13 @@ +include "third_party/mlir_edge/iree/schemas/device_def.fbs"; +include "third_party/mlir_edge/iree/schemas/device_group_def.fbs"; + +namespace iree; + +// A table of devices used for runtime device resolution and referencing. +table DeviceTableDef { + // One or more virtual devices referenced by ordinal in the sequencer ops. + devices:[DeviceDef]; + + // Zero or more device groups that specify which devices must be compatible. + device_groups:[DeviceGroupDef]; +} diff --git a/schemas/executable_def.fbs b/schemas/executable_def.fbs new file mode 100644 index 000000000000..f2fd93745df3 --- /dev/null +++ b/schemas/executable_def.fbs @@ -0,0 +1,38 @@ +namespace iree; + +// Bitfield indicating which executable features are compiled into a particular +// executable. +enum ExecutableFeature : uint (bit_flags) { + // Executable supports debugging. + // This may not mean that it'll be the best debugging (for example an -O3 + // executable could still support debugging, just not well). + kDebugging = 0, + // Executable supports coverage recording. + kCoverage = 1, + // Executable supports profile recording. + kProfiling = 2, +} + +// A set of one or more executables that can be dispatched at runtime. +// +// Executables may contain multiple entry points to allow devices to more +// efficiently share preparation results. It is expected that all executables +// representing the same dispatchable code have the same number of entry points +// with the same meaning. +table ExecutableDef { + // ExecutableFormat 4CC used to match on device support. + format:uint; + + // Bitfield indicating which executable features are supported. + // Multiple executables with the same format but differing supported features + // may be present, for example to enable both optimized and debugger-friendly + // versions in the same module. + supported_features:ExecutableFeature; + + // Executable contents. + // Opaque and left to individual device implementations as to how this is + // loaded and processed at runtime. + contents:[ubyte]; +} + +root_type ExecutableDef; diff --git a/schemas/executable_table_def.fbs b/schemas/executable_table_def.fbs new file mode 100644 index 000000000000..9cd14517b6cb --- /dev/null +++ b/schemas/executable_table_def.fbs @@ -0,0 +1,28 @@ +include "third_party/mlir_edge/iree/schemas/executable_def.fbs"; + +namespace iree; + +// A fat executable containing multiple format variants for the same logical +// entry points. +table MultiArchExecutableDef { + // Friendly name of the executable used for diagnostics. + name:string; + + // Number of available entry points. + // This is used for bytecode verification even when the executable is not + // fully loaded into a device. All executables must have the same entry + // points. + entry_point_count:uint; + + // A set of executables of various formats and supported feature sets. + // The runtime will select the appropriate executable based on the dispatch + // requirements. + executables:[ExecutableDef]; +} + +// A table of executables used for runtime dispatch lookup. +table ExecutableTableDef { + // One or more top level executables referenced by sequencer dispatch ops. + // Ordinal is referenced by dispatch ops to index into the table. + multi_arch_executables:[MultiArchExecutableDef]; +} diff --git a/schemas/function_def.fbs b/schemas/function_def.fbs new file mode 100644 index 000000000000..b21c4f894b76 --- /dev/null +++ b/schemas/function_def.fbs @@ -0,0 +1,18 @@ +include "third_party/mlir_edge/iree/schemas/bytecode_def.fbs"; +include "third_party/mlir_edge/iree/schemas/type_def.fbs"; + +namespace iree; + +table FunctionAttributeDef { + key:string; + value:string; +} + +table FunctionDef { + name:string; + type:FunctionTypeDef; + + attrs:[FunctionAttributeDef]; + + bytecode:BytecodeDef; +} diff --git a/schemas/function_table_def.fbs b/schemas/function_table_def.fbs new file mode 100644 index 000000000000..218039f54110 --- /dev/null +++ b/schemas/function_table_def.fbs @@ -0,0 +1,9 @@ +include "third_party/mlir_edge/iree/schemas/function_def.fbs"; + +namespace iree; + +table FunctionTableDef { + functions:[FunctionDef]; + imports:[int]; + exports:[int]; +} diff --git a/schemas/module_def.fbs b/schemas/module_def.fbs new file mode 100644 index 000000000000..84822bc9d1f4 --- /dev/null +++ b/schemas/module_def.fbs @@ -0,0 +1,20 @@ +include "third_party/mlir_edge/iree/schemas/executable_table_def.fbs"; +include "third_party/mlir_edge/iree/schemas/device_table_def.fbs"; +include "third_party/mlir_edge/iree/schemas/function_table_def.fbs"; +include "third_party/mlir_edge/iree/schemas/source_map_def.fbs"; + +namespace iree; + +// 'Executable MODule'. +file_identifier "EMOD"; +file_extension "emod"; + +table ModuleDef { + name:string; + device_table:DeviceTableDef; + function_table:FunctionTableDef; + executable_table:ExecutableTableDef; + source_map:SourceMapDef; +} + +root_type ModuleDef; diff --git a/schemas/source_map_def.fbs b/schemas/source_map_def.fbs new file mode 100644 index 000000000000..54e0e39f7ca8 --- /dev/null +++ b/schemas/source_map_def.fbs @@ -0,0 +1,46 @@ +namespace iree; + +table SourceMapDef { + function_table:[FunctionSourceMapDef]; + string_table:[string]; +} + +struct BytecodeSourceLocation { + offset:int; + location:int; +} + +table FunctionSourceMapDef { + location_table:[LocationDef]; + bytecode_map:[BytecodeSourceLocation]; +} + +table FileLocationDef { + filename:int; + line:int; + column:int; +} + +table NameLocationDef { + name:int; +} + +table CallSiteLocationDef { + callee_location:int; + caller_location:int; +} + +table FusedLocationDef { + locations:[int]; +} + +union LocationDefUnion { + FileLocationDef, + NameLocationDef, + CallSiteLocationDef, + FusedLocationDef, +} + +table LocationDef { + location_union:LocationDefUnion; +} diff --git a/schemas/spirv_executable_def.fbs b/schemas/spirv_executable_def.fbs new file mode 100644 index 000000000000..cd91bb9778ca --- /dev/null +++ b/schemas/spirv_executable_def.fbs @@ -0,0 +1,108 @@ +namespace iree; + +// 'SPIR-V Executable'. +file_identifier "SPVE"; +file_extension "spve"; + +// Structure specifying a descriptor set layout binding. +// +// VkDescriptorSetLayoutBinding: +// https://www.khronos.org/registry/vulkan/specs/1.1-extensions/man/html/VkDescriptorSetLayoutBinding.html +table VkDescriptorSetLayoutBindingDef { + // The binding number of this entry. Corresponds to a resource of the same + // binding number in the shader stages. + binding:uint32; + + // VkDescriptorType specifying which type of resource descriptors are used for + // this binding. + // + // VkDescriptorType: + // https://www.khronos.org/registry/vulkan/specs/1.1-extensions/man/html/VkDescriptorType.html + descriptor_type:uint32; + + // The number of descriptors contained in the binding. Accessed in a shader as + // an array. + descriptor_count:uint32; + + // A bitmask of VkShaderStageFlagBits specifying which pipeline shader stages + // can access a resource for this binding. + // + // VkShaderStageFlagBits: + // https://www.khronos.org/registry/vulkan/specs/1.1-extensions/man/html/VkShaderStageFlagBits.html + // + // Assume 'VK_SHADER_STAGE_ALL' as a good default. + stage_flags:uint32; +} + +// A descriptor set layout defined by an array of zero or more descriptor +// bindings. +// +// VkDescriptorSetLayout: +// https://www.khronos.org/registry/vulkan/specs/1.1-extensions/man/html/VkDescriptorSetLayout.html +table VkDescriptorSetLayoutDef { + // A set of serialized VkDescriptorSetLayoutBindings. + bindings:[VkDescriptorSetLayoutBindingDef]; +} + +// Structure specifying a push constant range. +// +// VkPushConstantRange: +// https://www.khronos.org/registry/vulkan/specs/1.1-extensions/man/html/VkPushConstantRange.html +table VkPushConstantRangeDef { + // A set of stage flags describing the shader stages that will access a range + // of push constants. + // + // VkShaderStageFlagBits: + // https://www.khronos.org/registry/vulkan/specs/1.1-extensions/man/html/VkShaderStageFlagBits.html + // + // Assume 'VK_SHADER_STAGE_ALL' as a good default. + stage_flags:uint32; + + // The start offset and size, respectively, consumed by the range. Both offset + // and size are in units of bytes and must be a multiple of 4. + offset:uint32; + size:uint32; +} + +// Zero or more descriptor set layouts and zero or more push constant ranges are +// combined to form a pipeline layout object which describes the complete set of +// resources that can be accessed by a pipeline. +// +// VkPipelineLayout: +// https://www.khronos.org/registry/vulkan/specs/1.1-extensions/man/html/VkPipelineLayout.html +table VkPipelineLayoutDef { + // An array of serialized VkDescriptorSetLayout objects. + descriptor_set_layouts:[VkDescriptorSetLayoutDef]; + + // An array of serialized VkPushConstantRange structures defining a set of + // push constant ranges for use in a single pipeline layout. + // + // Note that some implementations support as few as 128 bytes of push + // constants: + // https://vulkan.gpuinfo.org/displaydevicelimit.php?name=maxPushConstantsSize + push_constant_ranges:[VkPushConstantRangeDef]; + + // Index into the pipeline layout descriptor sets used for I/O buffer binding. + buffer_binding_set:uint32; +} + +// A SPIR-V shader module and runtime pipeline layout description. +// This information is used to create the VkShaderModule, VkPipelineLayout, and +// any required VkDescriptorSetLayouts. +table SpirVExecutableDef { + // Reserved implementation-specific value that can be passed from compiler to + // runtime. + tag:string; + + // A map of entry point ordinals to string names as used in the shader module. + entry_points:[string]; + + // SPIR-V code words. + code:[uint32]; + + // A serialized VkPipelineLayout object describing the bindings and push + // constants used by the shader. + pipeline_layout:VkPipelineLayoutDef; +} + +root_type SpirVExecutableDef; diff --git a/schemas/type_def.fbs b/schemas/type_def.fbs new file mode 100644 index 000000000000..ba09911df3a0 --- /dev/null +++ b/schemas/type_def.fbs @@ -0,0 +1,54 @@ +namespace iree; + +table FloatTypeDef { + width:int; +} + +table IntegerTypeDef { + width:int; +} + +table UnknownTypeDef { + dialect:string; + type_data:string; +} + +union ElementTypeDefUnion { + FloatTypeDef, + IntegerTypeDef, + UnknownTypeDef, +} + +table ElementTypeDef { + type_union:ElementTypeDefUnion; +} + +table MemRefTypeDef { + element_type:ElementTypeDef; + shape:[int]; + memory_space:int; +} + +table DeviceTypeDef {} +table CommandBufferTypeDef {} +table EventTypeDef {} +table SemaphoreTypeDef {} +table FenceTypeDef {} + +union TypeDefUnion { + MemRefTypeDef, + DeviceTypeDef, + CommandBufferTypeDef, + EventTypeDef, + SemaphoreTypeDef, + FenceTypeDef, +} + +table TypeDef { + type_union:TypeDefUnion; +} + +table FunctionTypeDef { + inputs:[TypeDef]; + results:[TypeDef]; +} diff --git a/test/e2e/cmpf.mlir b/test/e2e/cmpf.mlir new file mode 100644 index 000000000000..b0f97dedf92e --- /dev/null +++ b/test/e2e/cmpf.mlir @@ -0,0 +1,20 @@ +// TODO(b/134586626): enable when scalar values are better supported. +// xxx: iree-run-mlir %s --input_values="f32=42.0" --output_types="i,i,i,i,i,i" | FileCheck %s + +// CHECK-LABEL: EXEC @cmpf +func @cmpf(%42 : f32) -> (i1, i1, i1, i1, i1, i1) { // need at least one arg to avoid constant folding + %cm1 = constant -1.0 : f32 + %oeq = cmpf "oeq", %42, %cm1 : f32 + %une = cmpf "une", %42, %cm1 : f32 + %olt = cmpf "olt", %42, %cm1 : f32 + %ole = cmpf "ole", %42, %cm1 : f32 + %ogt = cmpf "ogt", %42, %cm1 : f32 + %oge = cmpf "oge", %42, %cm1 : f32 + return %oeq, %une, %olt, %ole, %ogt, %oge : i1, i1, i1, i1, i1, i1 +} +// CHECK: i8=0 +// CHECK-NEXT: i8=1 +// CHECK-NEXT: i8=0 +// CHECK-NEXT: i8=0 +// CHECK-NEXT: i8=1 +// CHECK-NEXT: i8=1 diff --git a/test/e2e/cmpi.mlir b/test/e2e/cmpi.mlir new file mode 100644 index 000000000000..93e9eb5190ec --- /dev/null +++ b/test/e2e/cmpi.mlir @@ -0,0 +1,28 @@ +// TODO(b/134586626): enable when scalar values are better supported. +// xxx: iree-run-mlir %s --input_values="i32=42" --output_types="i,i,i,i,i,i,i,i,i,i" | FileCheck %s + +// CHECK-LABEL: EXEC @cmpi +func @cmpi(%42 : i32) -> (i1, i1, i1, i1, i1, i1, i1, i1, i1, i1) { // need at least one arg to avoid constant folding + %cm1 = constant -1 : i32 + %eq = cmpi "eq", %42, %cm1 : i32 + %ne = cmpi "ne", %42, %cm1 : i32 + %slt = cmpi "slt", %42, %cm1 : i32 + %sle = cmpi "sle", %42, %cm1 : i32 + %sgt = cmpi "sgt", %42, %cm1 : i32 + %sge = cmpi "sge", %42, %cm1 : i32 + %ult = cmpi "ult", %42, %cm1 : i32 + %ule = cmpi "ule", %42, %cm1 : i32 + %ugt = cmpi "ugt", %42, %cm1 : i32 + %uge = cmpi "uge", %42, %cm1 : i32 + return %eq, %ne, %slt, %sle, %sgt, %sge, %ult, %ule, %ugt, %uge : i1, i1, i1, i1, i1, i1, i1, i1, i1, i1 +} +// CHECK: i8=0 +// CHECK-NEXT: i8=1 +// CHECK-NEXT: i8=0 +// CHECK-NEXT: i8=0 +// CHECK-NEXT: i8=1 +// CHECK-NEXT: i8=1 +// CHECK-NEXT: i8=1 +// CHECK-NEXT: i8=1 +// CHECK-NEXT: i8=0 +// CHECK-NEXT: i8=0 diff --git a/test/e2e/extract_element.mlir b/test/e2e/extract_element.mlir new file mode 100644 index 000000000000..c262d65fc066 --- /dev/null +++ b/test/e2e/extract_element.mlir @@ -0,0 +1,10 @@ +// RUN: iree-run-mlir --target_backends=interpreter-bytecode %s --input_values="i8=4" --output_types=i | FileCheck %s + +// CHECK-LABEL: @extract_element +func @extract_element(%arg0: tensor) -> i8 { + %cst = constant dense<1> : tensor + %0 = addi %cst, %arg0 : tensor + %1 = extract_element %0[] : tensor + return %1 : i8 +} +// CHECK-NEXT: i8=5 diff --git a/test/e2e/scalars.mlir b/test/e2e/scalars.mlir new file mode 100644 index 000000000000..d7ee49ed6e08 --- /dev/null +++ b/test/e2e/scalars.mlir @@ -0,0 +1,8 @@ +// RUN: iree-run-mlir %s | FileCheck %s --dump-input=fail + +// CHECK-LABEL: EXEC @scalars +func @scalars() -> tensor { + %0 = constant dense<2.0> : tensor + return %0 : tensor +} +// CHECK: f32=2 diff --git a/test/e2e/xla/compare.mlir b/test/e2e/xla/compare.mlir new file mode 100644 index 000000000000..510b3303d578 --- /dev/null +++ b/test/e2e/xla/compare.mlir @@ -0,0 +1,163 @@ +// RUN: iree-run-mlir --target_backends=interpreter-bytecode %s --output_types=i | FileCheck %s + +// CHECK-LABEL: EXEC @compare_tensor +func @compare_tensor() -> tensor<4xi1> { + %lhs = constant dense<[1, 2, 7, 4]> : tensor<4xi32> + %rhs = constant dense<[5, 2, 3, 4]> : tensor<4xi32> + %result = "xla_hlo.compare"(%lhs, %rhs) {comparison_direction = "EQ"} : (tensor<4xi32>, tensor<4xi32>) -> tensor<4xi1> + return %result : tensor<4xi1> +} +// CHECK: 4xi8=0 1 0 1 + +// ----- + +// CHECK-LABEL: EXEC @compare_scalar +func @compare_scalar() -> tensor { + %lhs = constant dense<1> : tensor + %rhs = constant dense<5> : tensor + %result = "xla_hlo.compare"(%lhs, %rhs) {comparison_direction = "EQ"} : (tensor, tensor) -> tensor + return %result : tensor +} +// CHECK: i8=0 + +// ----- + +// CHECK-LABEL: EXEC @compare_i8 +func @compare_i8() -> tensor { + %lhs = constant dense<1> : tensor + %rhs = constant dense<5> : tensor + %result = "xla_hlo.compare"(%lhs, %rhs) {comparison_direction = "EQ"} : (tensor, tensor) -> tensor + return %result : tensor +} +// CHECK: i8=0 + +// ----- + +// CHECK-LABEL: EXEC @compare_i16 +func @compare_i16() -> tensor { + %lhs = constant dense<1> : tensor + %rhs = constant dense<5> : tensor + %result = "xla_hlo.compare"(%lhs, %rhs) {comparison_direction = "EQ"} : (tensor, tensor) -> tensor + return %result : tensor +} +// CHECK: i8=0 + +// ----- + +// CHECK-LABEL: EXEC @compare_i32 +func @compare_i32() -> tensor { + %lhs = constant dense<1> : tensor + %rhs = constant dense<5> : tensor + %result = "xla_hlo.compare"(%lhs, %rhs) {comparison_direction = "EQ"} : (tensor, tensor) -> tensor + return %result : tensor +} +// CHECK: i8=0 + +// ----- + +// CHECK-LABEL: EXEC @compare_i64 +func @compare_i64() -> tensor { + %lhs = constant dense<1> : tensor + %rhs = constant dense<5> : tensor + %result = "xla_hlo.compare"(%lhs, %rhs) {comparison_direction = "EQ"} : (tensor, tensor) -> tensor + return %result : tensor +} +// CHECK: i8=0 + +// ----- + +// CHECK-LABEL: EXEC @compare_f32 +func @compare_f32() -> tensor { + %lhs = constant dense<1.0> : tensor + %rhs = constant dense<5.0> : tensor + %result = "xla_hlo.compare"(%lhs, %rhs) {comparison_direction = "EQ"} : (tensor, tensor) -> tensor + return %result : tensor +} +// CHECK: i8=0 + +// ----- + +// CHECK-LABEL: EXEC @compare_f64 +func @compare_f64() -> tensor { + %lhs = constant dense<1.0> : tensor + %rhs = constant dense<5.0> : tensor + %result = "xla_hlo.compare"(%lhs, %rhs) {comparison_direction = "EQ"} : (tensor, tensor) -> tensor + return %result : tensor +} +// CHECK: i8=0 +// ----- + +// CHECK-LABEL: EXEC @compare_tensor_odd_length +func @compare_tensor_odd_length() -> tensor<3xi1> { + %lhs = constant dense<[1, 2, 7]> : tensor<3xi32> + %rhs = constant dense<[5, 2, 3]> : tensor<3xi32> + %result = "xla_hlo.compare"(%lhs, %rhs) {comparison_direction = "EQ"} : (tensor<3xi32>, tensor<3xi32>) -> tensor<3xi1> + return %result : tensor<3xi1> +} +// CHECK: 3xi8=0 1 0 + +// ----- + +// CHECK-LABEL: EXEC @compare_eq +func @compare_eq() -> tensor<4xi1> { + %lhs = constant dense<[1, 2, 7, 4]> : tensor<4xi32> + %rhs = constant dense<[5, 2, 3, 4]> : tensor<4xi32> + %result = "xla_hlo.compare"(%lhs, %rhs) {comparison_direction = "EQ"} : (tensor<4xi32>, tensor<4xi32>) -> tensor<4xi1> + return %result : tensor<4xi1> +} +// CHECK: 4xi8=0 1 0 1 + +// ----- + +// CHECK-LABEL: EXEC @compare_ne +func @compare_ne() -> tensor<4xi1> { + %lhs = constant dense<[1, 2, 7, 4]> : tensor<4xi32> + %rhs = constant dense<[5, 2, 3, 4]> : tensor<4xi32> + %result = "xla_hlo.compare"(%lhs, %rhs) {comparison_direction = "NE"} : (tensor<4xi32>, tensor<4xi32>) -> tensor<4xi1> + return %result : tensor<4xi1> +} +// CHECK: 4xi8=1 0 1 0 + +// ----- + +// CHECK-LABEL: EXEC @compare_lt +func @compare_lt() -> tensor<4xi1> { + %lhs = constant dense<[1, 2, 7, 4]> : tensor<4xi32> + %rhs = constant dense<[5, 2, 3, 4]> : tensor<4xi32> + %result = "xla_hlo.compare"(%lhs, %rhs) {comparison_direction = "LT"} : (tensor<4xi32>, tensor<4xi32>) -> tensor<4xi1> + return %result : tensor<4xi1> +} +// CHECK: 4xi8=1 0 0 0 + +// ----- + +// CHECK-LABEL: EXEC @compare_le +func @compare_le() -> tensor<4xi1> { + %lhs = constant dense<[1, 2, 7, 4]> : tensor<4xi32> + %rhs = constant dense<[5, 2, 3, 4]> : tensor<4xi32> + %result = "xla_hlo.compare"(%lhs, %rhs) {comparison_direction = "LE"} : (tensor<4xi32>, tensor<4xi32>) -> tensor<4xi1> + return %result : tensor<4xi1> +} +// CHECK: 4xi8=1 1 0 1 + +// ----- + +// CHECK-LABEL: EXEC @compare_gt +func @compare_gt() -> tensor<4xi1> { + %lhs = constant dense<[1, 2, 7, 4]> : tensor<4xi32> + %rhs = constant dense<[5, 2, 3, 4]> : tensor<4xi32> + %result = "xla_hlo.compare"(%lhs, %rhs) {comparison_direction = "GT"} : (tensor<4xi32>, tensor<4xi32>) -> tensor<4xi1> + return %result : tensor<4xi1> +} +// CHECK: 4xi8=0 0 1 0 + +// ----- + +// CHECK-LABEL: EXEC @compare_ge +func @compare_ge() -> tensor<4xi1> { + %lhs = constant dense<[1, 2, 7, 4]> : tensor<4xi32> + %rhs = constant dense<[5, 2, 3, 4]> : tensor<4xi32> + %result = "xla_hlo.compare"(%lhs, %rhs) {comparison_direction = "GE"} : (tensor<4xi32>, tensor<4xi32>) -> tensor<4xi1> + return %result : tensor<4xi1> +} +// CHECK: 4xi8=0 1 1 1 diff --git a/test/e2e/xla/convert_int.mlir b/test/e2e/xla/convert_int.mlir new file mode 100644 index 000000000000..e3f1c048470e --- /dev/null +++ b/test/e2e/xla/convert_int.mlir @@ -0,0 +1,16 @@ +// RUN: iree-run-mlir --target_backends=interpreter-bytecode %s --input_values=1xi32=42 --output_types=i | FileCheck %s --dump-input=fail + +// CHECK-LABEL: EXEC @narrow_int +func @narrow_int(%arg : tensor<1xi32>) -> tensor<1xi8> { + %0 = "xla_hlo.convert"(%arg) : (tensor<1xi32>) -> tensor<1xi8> + return %0 : tensor<1xi8> +} +// CHECK: 1xi8=42 + +// CHECK-LABEL: EXEC @widen_int +func @widen_int(%arg : tensor<1xi32>) -> tensor<1xi64> { + %0 = "xla_hlo.convert"(%arg) : (tensor<1xi32>) -> tensor<1xi64> + return %0 : tensor<1xi64> +} +// CHECK: 1xi64=42 + diff --git a/test/e2e/xla/exp.mlir b/test/e2e/xla/exp.mlir new file mode 100644 index 000000000000..a8110718cc58 --- /dev/null +++ b/test/e2e/xla/exp.mlir @@ -0,0 +1,39 @@ +// RUN: iree-run-mlir --target_backends=interpreter-bytecode %s --output_types=f | FileCheck %s + +// CHECK-LABEL: EXEC @tensor +func @tensor() -> tensor<4xf32> { + %input = constant dense<[0.0, 1.0, 2.0, 4.0]> : tensor<4xf32> + %result = "xla_hlo.exp"(%input) : (tensor<4xf32>) -> tensor<4xf32> + return %result : tensor<4xf32> +} +// CHECK: 4xf32=1 2.71828 7.38906 54.5981 + +// ----- + +// CHECK-LABEL: EXEC @scalar +func @scalar() -> tensor { + %input = constant dense<1.0> : tensor + %result = "xla_hlo.exp"(%input) : (tensor) -> tensor + return %result : tensor +} +// CHECK: f32=2.71828 + +// ----- + +// CHECK-LABEL: EXEC @double +func @double() -> tensor { + %input = constant dense<1.0> : tensor + %result = "xla_hlo.exp"(%input) : (tensor) -> tensor + return %result : tensor +} +// CHECK: f64=2.71828 + +// ----- + +// CHECK-LABEL: EXEC @negative +func @negative() -> tensor { + %input = constant dense<-1.0> : tensor + %result = "xla_hlo.exp"(%input) : (tensor) -> tensor + return %result : tensor +} +// CHECK: f32=0.367879 diff --git a/test/e2e/xla/floor.mlir b/test/e2e/xla/floor.mlir new file mode 100644 index 000000000000..a046bd8917e7 --- /dev/null +++ b/test/e2e/xla/floor.mlir @@ -0,0 +1,39 @@ +// RUN: iree-run-mlir --target_backends=interpreter-bytecode %s --output_types=f | FileCheck %s + +// CHECK-LABEL: EXEC @tensor +func @tensor() -> tensor<4xf32> { + %input = constant dense<[0.0, 1.1, 2.5, 4.9]> : tensor<4xf32> + %result = "xla_hlo.floor"(%input) : (tensor<4xf32>) -> tensor<4xf32> + return %result : tensor<4xf32> +} +// CHECK: 4xf32=0 1 2 4 + +// ----- + +// CHECK-LABEL: EXEC @scalar +func @scalar() -> tensor { + %input = constant dense<101.3> : tensor + %result = "xla_hlo.floor"(%input) : (tensor) -> tensor + return %result : tensor +} +// CHECK: f32=101 + +// ----- + +// CHECK-LABEL: EXEC @double +func @double() -> tensor { + %input = constant dense<11.2> : tensor + %result = "xla_hlo.floor"(%input) : (tensor) -> tensor + return %result : tensor +} +// CHECK: f64=11 + +// ----- + +// CHECK-LABEL: EXEC @negative +func @negative() -> tensor { + %input = constant dense<-1.1> : tensor + %result = "xla_hlo.floor"(%input) : (tensor) -> tensor + return %result : tensor +} +// CHECK: f32=-2 diff --git a/test/e2e/xla/fragment_000.mlir b/test/e2e/xla/fragment_000.mlir new file mode 100644 index 000000000000..b87c31909a55 --- /dev/null +++ b/test/e2e/xla/fragment_000.mlir @@ -0,0 +1,33 @@ +// RUN: iree-run-mlir %s --input_values="f32=0\n5x1xf32=[1][-2][-3][4][-5]\nf32=1\n5x5xf32=[3.46499 -7.64389 -5.72249 5.98053 17.6892][2.9707 -6.20734 -4.25962 4.76055 13.8784][2.47641 -4.77079 -2.79675 3.54056 10.0675][1.98212 -3.33424 -1.33388 2.32058 6.25666][1.48783 -1.8977 0.12899 1.1006 2.4458]\n5xf32=0 0 0 0 0" | FileCheck %s --implicit-check-not="[" --implicit-check-not="]" --dump-input=fail + +// CHECK-LABEL: EXEC @main_entry_dispatch_3 +func @main_entry_dispatch_3( + %0: tensor, + %1: tensor<5x1xf32>, + %2: tensor, + %3: tensor<5x5xf32>, + %4: tensor<5xf32>) -> tensor<5x5xf32> { + %5 = "xla_hlo.broadcast_in_dim"(%1) {broadcast_dimensions = dense<[0, 1]> : tensor<2xi64>, name = "broadcast.44"} : (tensor<5x1xf32>) -> tensor<5x1x5xf32> + %6 = "xla_hlo.broadcast_in_dim"(%2) {name = "broadcast.9"} : (tensor) -> tensor<5x1x5xf32> + %7 = mulf %5, %6 : tensor<5x1x5xf32> + %8 = "xla_hlo.broadcast_in_dim"(%0) {name = "broadcast.47"} : (tensor) -> tensor<5x1x5xf32> + %9 = cmpf "ogt", %7, %8 : tensor<5x1x5xf32> + %10 = "xla_hlo.broadcast_in_dim"(%0) {name = "broadcast.11"} : (tensor) -> tensor<5x1x5xf32> + %11 = "xla_hlo.broadcast_in_dim"(%0) {name = "broadcast.67"} : (tensor) -> tensor<5x5xf32> + %12 = "xla_hlo.broadcast_in_dim"(%4) {broadcast_dimensions = dense<1> : tensor<1xi64>, name = "broadcast.64"} : (tensor<5xf32>) -> tensor<5x5xf32> + %13 = addf %3, %12 : tensor<5x5xf32> + %14 = xla_hlo.max %11, %13 {name = "maximum.68"} : tensor<5x5xf32> + %15 = "xla_hlo.reshape"(%14) {name = "reshape.70"} : (tensor<5x5xf32>) -> tensor<5x1x5xf32> + %16 = "xla_hlo.select"(%9, %10, %15) {name = "select.71"} : (tensor<5x1x5xi1>, tensor<5x1x5xf32>, tensor<5x1x5xf32>) -> tensor<5x1x5xf32> + %17 = "xla_hlo.copy"(%16) {name = "copy.4"} : (tensor<5x1x5xf32>) -> tensor<5x1x5xf32> + %18 = "xla_hlo.reshape"(%17) {name = "reshape.72"} : (tensor<5x1x5xf32>) -> tensor<5x5xf32> + return %18 : tensor<5x5xf32> +} + +// On separate lines to avoid "[[" which FileCheck interprets as substitutions +// CHECK: 5x5xf32= +// CHECK-SAME: [0 0 0 0 0] +// CHECK-SAME: [2.97{{[0-9]+}} 0 0 4.76{{[0-9]+}} 13.87{{[0-9]+}}] +// CHECK-SAME: [2.47{{[0-9]+}} 0 0 3.54{{[0-9]+}} 10.06{{[0-9]+}}] +// CHECK-SAME: [0 0 0 0 0] +// CHECK-SAME: [1.48{{[0-9]+}} 0 0.12{{[0-9]+}} 1.10{{[0-9]+}} 2.44{{[0-9]+}}] diff --git a/test/e2e/xla/fullyconnected.mlir b/test/e2e/xla/fullyconnected.mlir new file mode 100644 index 000000000000..7fbd5aa0e8f8 --- /dev/null +++ b/test/e2e/xla/fullyconnected.mlir @@ -0,0 +1,94 @@ +// RUN: iree-run-mlir %s --input_values="1x5xf32=1,-2,-3,4,-5\n1x5x3x1xf32=15,14,13,12,11,10,9,8,7,6,5,4,3,2,1" | FileCheck %s --implicit-check-not="[" --implicit-check-not="]" --dump-input=fail + +// CHECK-LABEL: EXEC @main +func @main(%arg0: tensor<1x5xf32>, %arg1: tensor<1x5x3x1xf32>) -> tuple> + attributes {iree.module.export} { + %0 = "xla_hlo.reshape"(%arg0) {name = "reshape.3"} : (tensor<1x5xf32>) -> tensor<1x5xf32> + %1 = "xla_hlo.transpose"(%0) {name = "transpose.41", permutation = dense<[1, 0]> : tensor<2xi64>} : (tensor<1x5xf32>) -> tensor<5x1xf32> + %2 = "xla_hlo.reshape"(%1) {name = "reshape.42"} : (tensor<5x1xf32>) -> tensor<5x1x1xf32> + %3 = "xla_hlo.reshape"(%2) {name = "reshape.55"} : (tensor<5x1x1xf32>) -> tensor<5x1xf32> + %4 = "xla_hlo.broadcast_in_dim"(%3) {broadcast_dimensions = dense<[0, 1]> : tensor<2xi64>, name = "broadcast.56"} : (tensor<5x1xf32>) -> tensor<5x1x5xf32> + %cst = constant {name = "constant.22"} dense<1.000000e+00> : tensor + %5 = "xla_hlo.broadcast_in_dim"(%cst) {name = "broadcast.23"} : (tensor) -> tensor<5x1x5xf32> + %6 = xla_hlo.mul %4, %5 {name = "multiply.57"} : tensor<5x1x5xf32> + %cst_0 = constant {name = "constant.58"} dense<0.000000e+00> : tensor + %7 = "xla_hlo.broadcast_in_dim"(%cst_0) {name = "broadcast.59"} : (tensor) -> tensor<5x1x5xf32> + %8 = "xla_hlo.compare"(%6, %7) {comparison_direction = "GT", name = "compare.60"} : (tensor<5x1x5xf32>, tensor<5x1x5xf32>) -> tensor<5x1x5xi1> + %cst_1 = constant {name = "constant.24"} dense<0.000000e+00> : tensor + %9 = "xla_hlo.broadcast_in_dim"(%cst_1) {name = "broadcast.25"} : (tensor) -> tensor<5x1x5xf32> + %cst_2 = constant {name = "constant.90"} dense<0.000000e+00> : tensor + %10 = "xla_hlo.broadcast_in_dim"(%cst_2) {name = "broadcast.91"} : (tensor) -> tensor<5x5xf32> + %11 = "xla_hlo.reshape"(%2) {name = "reshape.49"} : (tensor<5x1x1xf32>) -> tensor<5x1xf32> + %12 = "xla_hlo.broadcast_in_dim"(%11) {broadcast_dimensions = dense<[0, 1]> : tensor<2xi64>, name = "broadcast.50"} : (tensor<5x1xf32>) -> tensor<5x1x5xf32> + %cst_3 = constant {name = "constant.15"} dense<1.000000e+00> : tensor + %13 = "xla_hlo.broadcast_in_dim"(%cst_3) {name = "broadcast.16"} : (tensor) -> tensor<5x1x5xf32> + %14 = xla_hlo.mul %12, %13 {name = "multiply.51"} : tensor<5x1x5xf32> + %cst_4 = constant {name = "constant.52"} dense<0.000000e+00> : tensor + %15 = "xla_hlo.broadcast_in_dim"(%cst_4) {name = "broadcast.53"} : (tensor) -> tensor<5x1x5xf32> + %16 = "xla_hlo.compare"(%14, %15) {comparison_direction = "GT", name = "compare.54"} : (tensor<5x1x5xf32>, tensor<5x1x5xf32>) -> tensor<5x1x5xi1> + %cst_5 = constant {name = "constant.17"} dense<0.000000e+00> : tensor + %17 = "xla_hlo.broadcast_in_dim"(%cst_5) {name = "broadcast.18"} : (tensor) -> tensor<5x1x5xf32> + %cst_6 = constant {name = "constant.78"} dense<0.000000e+00> : tensor + %18 = "xla_hlo.broadcast_in_dim"(%cst_6) {name = "broadcast.79"} : (tensor) -> tensor<5x5xf32> + %19 = "xla_hlo.reshape"(%2) {name = "reshape.43"} : (tensor<5x1x1xf32>) -> tensor<5x1xf32> + %20 = "xla_hlo.broadcast_in_dim"(%19) {broadcast_dimensions = dense<[0, 1]> : tensor<2xi64>, name = "broadcast.44"} : (tensor<5x1xf32>) -> tensor<5x1x5xf32> + %cst_7 = constant {name = "constant.8"} dense<1.000000e+00> : tensor + %21 = "xla_hlo.broadcast_in_dim"(%cst_7) {name = "broadcast.9"} : (tensor) -> tensor<5x1x5xf32> + %22 = xla_hlo.mul %20, %21 {name = "multiply.45"} : tensor<5x1x5xf32> + %cst_8 = constant {name = "constant.46"} dense<0.000000e+00> : tensor + %23 = "xla_hlo.broadcast_in_dim"(%cst_8) {name = "broadcast.47"} : (tensor) -> tensor<5x1x5xf32> + %24 = "xla_hlo.compare"(%22, %23) {comparison_direction = "GT", name = "compare.48"} : (tensor<5x1x5xf32>, tensor<5x1x5xf32>) -> tensor<5x1x5xi1> + %cst_9 = constant {name = "constant.10"} dense<0.000000e+00> : tensor + %25 = "xla_hlo.broadcast_in_dim"(%cst_9) {name = "broadcast.11"} : (tensor) -> tensor<5x1x5xf32> + %cst_10 = constant {name = "constant.66"} dense<0.000000e+00> : tensor + %26 = "xla_hlo.broadcast_in_dim"(%cst_10) {name = "broadcast.67"} : (tensor) -> tensor<5x5xf32> + %27 = "xla_hlo.copy"(%arg1) {name = "copy.3"} : (tensor<1x5x3x1xf32>) -> tensor<1x5x3x1xf32> + %28 = "xla_hlo.reshape"(%27) {name = "reshape.4"} : (tensor<1x5x3x1xf32>) -> tensor<1x5x3x1xf32> + %29 = "xla_hlo.reshape"(%28) {name = "reshape.38"} : (tensor<1x5x3x1xf32>) -> tensor<1x5x3xf32> + %30 = "xla_hlo.transpose"(%29) {name = "transpose.39", permutation = dense<[1, 0, 2]> : tensor<3xi64>} : (tensor<1x5x3xf32>) -> tensor<5x1x3xf32> + %31 = "xla_hlo.reshape"(%30) {name = "reshape.40"} : (tensor<5x1x3xf32>) -> tensor<5x3xf32> + %cst_11 = constant {name = "constant.61"} dense<[[0.706495285, -0.567672312, 0.483717591, 0.522725761, 0.7563259], [-0.0899272263, -0.283501834, -0.350822538, -0.351515919, -0.337136656], [-0.451804549, 0.372324884, -0.620518147, 0.235451385, 0.851095855]]> : tensor<3x5xf32> + %32 = "xla_hlo.dot"(%31, %cst_11) {name = "dot.62", precision_config = ["DEFAULT", "DEFAULT"]} : (tensor<5x3xf32>, tensor<3x5xf32>) -> tensor<5x5xf32> + %cst_12 = constant {name = "constant.63"} dense<[0.000000e+00, 0.000000e+00, 0.000000e+00, 0.000000e+00, 0.000000e+00]> : tensor<5xf32> + %33 = "xla_hlo.broadcast_in_dim"(%cst_12) {broadcast_dimensions = dense<[1]> : tensor<1xi64>, name = "broadcast.64"} : (tensor<5xf32>) -> tensor<5x5xf32> + %34 = xla_hlo.add %32, %33 {name = "add.65"} : tensor<5x5xf32> + %35 = xla_hlo.max %26, %34 {name = "maximum.68"} : tensor<5x5xf32> + %36 = "xla_hlo.reshape"(%35) {name = "reshape.70"} : (tensor<5x5xf32>) -> tensor<5x1x5xf32> + %37 = "xla_hlo.select"(%24, %25, %36) {name = "select.71"} : (tensor<5x1x5xi1>, tensor<5x1x5xf32>, tensor<5x1x5xf32>) -> tensor<5x1x5xf32> + %38 = "xla_hlo.copy"(%37) {name = "copy.4"} : (tensor<5x1x5xf32>) -> tensor<5x1x5xf32> + %39 = "xla_hlo.reshape"(%38) {name = "reshape.72"} : (tensor<5x1x5xf32>) -> tensor<5x5xf32> + %cst_13 = constant {name = "constant.73"} dense<[[-0.0118641369, -3.785000e-02, 0.489048243, 0.321015775, -0.702280283], [-0.280262798, -0.724645615, -0.00332254497, 0.392334729, 0.619746447], [-0.113318317, -0.180415511, -0.146743968, 0.250408649, -0.442881733], [0.115600757, 0.703136146, -0.00812680274, -0.225454301, -0.0835619792], [-0.136745885, -6.298570e-01, 0.43629986, -0.689790308, 0.230725273]]> : tensor<5x5xf32> + %40 = "xla_hlo.dot"(%39, %cst_13) {name = "dot.74", precision_config = ["DEFAULT", "DEFAULT"]} : (tensor<5x5xf32>, tensor<5x5xf32>) -> tensor<5x5xf32> + %cst_14 = constant {name = "constant.75"} dense<[0.000000e+00, 0.000000e+00, 0.000000e+00, 0.000000e+00, 0.000000e+00]> : tensor<5xf32> + %41 = "xla_hlo.broadcast_in_dim"(%cst_14) {broadcast_dimensions = dense<[1]> : tensor<1xi64>, name = "broadcast.76"} : (tensor<5xf32>) -> tensor<5x5xf32> + %42 = xla_hlo.add %40, %41 {name = "add.77"} : tensor<5x5xf32> + %43 = xla_hlo.max %18, %42 {name = "maximum.80"} : tensor<5x5xf32> + %44 = "xla_hlo.reshape"(%43) {name = "reshape.82"} : (tensor<5x5xf32>) -> tensor<5x1x5xf32> + %45 = "xla_hlo.select"(%16, %17, %44) {name = "select.83"} : (tensor<5x1x5xi1>, tensor<5x1x5xf32>, tensor<5x1x5xf32>) -> tensor<5x1x5xf32> + %46 = "xla_hlo.copy"(%45) {name = "copy.5"} : (tensor<5x1x5xf32>) -> tensor<5x1x5xf32> + %47 = "xla_hlo.reshape"(%46) {name = "reshape.84"} : (tensor<5x1x5xf32>) -> tensor<5x5xf32> + %cst_15 = constant {name = "constant.85"} dense<[[-0.136191264, -0.0401721969, 0.38497138, -5.850760e-01, 0.370910525], [-0.391011149, 0.0266356133, 0.309115469, -0.205079094, -0.559861302], [0.497760415, 0.689488232, 0.0759292394, -0.33134672, -0.237128958], [-0.53243047, 0.476418108, -0.371978909, 0.283265263, 0.63842845], [0.101761498, -0.218626946, 0.475128263, 0.042601984, 0.0988005772]]> : tensor<5x5xf32> + %48 = "xla_hlo.dot"(%47, %cst_15) {name = "dot.86", precision_config = ["DEFAULT", "DEFAULT"]} : (tensor<5x5xf32>, tensor<5x5xf32>) -> tensor<5x5xf32> + %cst_16 = constant {name = "constant.87"} dense<[0.000000e+00, 0.000000e+00, 0.000000e+00, 0.000000e+00, 0.000000e+00]> : tensor<5xf32> + %49 = "xla_hlo.broadcast_in_dim"(%cst_16) {broadcast_dimensions = dense<[1]> : tensor<1xi64>, name = "broadcast.88"} : (tensor<5xf32>) -> tensor<5x5xf32> + %50 = xla_hlo.add %48, %49 {name = "add.89"} : tensor<5x5xf32> + %51 = xla_hlo.max %10, %50 {name = "maximum.92"} : tensor<5x5xf32> + %52 = "xla_hlo.reshape"(%51) {name = "reshape.94"} : (tensor<5x5xf32>) -> tensor<5x1x5xf32> + %53 = "xla_hlo.select"(%8, %9, %52) {name = "select.95"} : (tensor<5x1x5xi1>, tensor<5x1x5xf32>, tensor<5x1x5xf32>) -> tensor<5x1x5xf32> + %54 = "xla_hlo.reshape"(%53) {name = "reshape.96"} : (tensor<5x1x5xf32>) -> tensor<5x1x5xf32> + %55 = "xla_hlo.tuple"(%54) {name = "tuple.97"} : (tensor<5x1x5xf32>) -> tuple> + return %55 : tuple> +} + +// On separate lines to avoid "[[" which FileCheck interprets as substitutions +// CHECK: 5x1x5xf32=[ +// CHECK-SAME: [0 0 0 0 0] +// CHECK-SAME: ][ +// CHECK-SAME: [3.79{{[0-9]+}} 4.99{{[0-9]+}} 0.90{{[0-9]+}} 0 0] +// CHECK-SAME: ][ +// CHECK-SAME: [2.80{{[0-9]+}} 3.78{{[0-9]+}} 0.56{{[0-9]+}} 0 0] +// CHECK-SAME: ][ +// CHECK-SAME: [0 0 0 0 0] +// CHECK-SAME: ][ +// CHECK-SAME: [0.87{{[0-9]+}} 1.21{{[0-9]+}} 0.13{{[0-9]+}} 0 0] +// CHECK-SAME: ] diff --git a/test/e2e/xla/gemm.mlir b/test/e2e/xla/gemm.mlir new file mode 100644 index 000000000000..91703bc59e15 --- /dev/null +++ b/test/e2e/xla/gemm.mlir @@ -0,0 +1,10 @@ +// RUN: iree-run-mlir %s --input_values="5x3xf32=15,14,13,12,11,10,9,8,7,6,5,4,3,2,1\n3x5xf32=15,14,13,12,11,10,9,8,7,6,5,4,3,2,1" | FileCheck %s --implicit-check-not="[" --implicit-check-not="]" --dump-input=fail + +// CHECK-LABEL: EXEC @main +func @main(%arg0: tensor<5x3xf32>, %arg1: tensor<3x5xf32>) -> tensor<5x5xf32> + attributes {iree.module.export} { + %0 = "xla_hlo.dot"(%arg0, %arg1) {precision_config = ["DEFAULT", "DEFAULT"]} : (tensor<5x3xf32>, tensor<3x5xf32>) -> tensor<5x5xf32> + return %0 : tensor<5x5xf32> +} + +// CHECK: 5x5xf32=[430 388 346 304 262][340 307 274 241 208][250 226 202 178 154][160 145 130 115 100][70 64 58 52 46] diff --git a/test/e2e/xla/log.mlir b/test/e2e/xla/log.mlir new file mode 100644 index 000000000000..902927b6a462 --- /dev/null +++ b/test/e2e/xla/log.mlir @@ -0,0 +1,29 @@ +// RUN: iree-run-mlir --target_backends=interpreter-bytecode %s --output_types=f | FileCheck %s + +// CHECK-LABEL: EXEC @tensor +func @tensor() -> tensor<4xf32> { + %input = constant dense<[1.0, 2.0, 3.0, 4.0]> : tensor<4xf32> + %result = "xla_hlo.log"(%input) : (tensor<4xf32>) -> tensor<4xf32> + return %result : tensor<4xf32> +} +// CHECK: 4xf32=0 0.693147 1.09861 1.38629 + +// ----- + +// CHECK-LABEL: EXEC @scalar +func @scalar() -> tensor { + %input = constant dense<4.0> : tensor + %result = "xla_hlo.log"(%input) : (tensor) -> tensor + return %result : tensor +} +// CHECK: f32=1.38629 + +// ----- + +// CHECK-LABEL: EXEC @double +func @double() -> tensor { + %input = constant dense<4.0> : tensor + %result = "xla_hlo.log"(%input) : (tensor) -> tensor + return %result : tensor +} +// CHECK: f64=1.38629 diff --git a/test/e2e/xla/max_float.mlir b/test/e2e/xla/max_float.mlir new file mode 100644 index 000000000000..b29da1ddaf30 --- /dev/null +++ b/test/e2e/xla/max_float.mlir @@ -0,0 +1,43 @@ +// RUN: iree-run-mlir --target_backends=interpreter-bytecode %s --output_types=f | FileCheck %s --dump-input=fail + +// CHECK-LABEL: EXEC @tensor +func @tensor() -> tensor<4xf32> { + %lhs = constant dense<[1.0, 2.0, 7.0, 4.0]> : tensor<4xf32> + %rhs = constant dense<[5.0, 2.0, 3.0, 4.0]> : tensor<4xf32> + %result = "xla_hlo.min"(%lhs, %rhs) : (tensor<4xf32>, tensor<4xf32>) -> tensor<4xf32> + return %result : tensor<4xf32> +} +// CHECK: 4xf32=1 2 3 4 + +// ----- + +// CHECK-LABEL: EXEC @scalar +func @scalar() -> tensor { + %lhs = constant dense<1.0> : tensor + %rhs = constant dense<2.0> : tensor + %result = "xla_hlo.min"(%lhs, %rhs) : (tensor, tensor) -> tensor + return %result : tensor +} +// CHECK: f32=1 + +// ----- + +// CHECK-LABEL: EXEC @double +func @double() -> tensor { + %lhs = constant dense<1.0> : tensor + %rhs = constant dense<2.0> : tensor + %result = "xla_hlo.min"(%lhs, %rhs) : (tensor, tensor) -> tensor + return %result : tensor +} +// CHECK: f64=1 + +// ----- + +// CHECK-LABEL: EXEC @negative +func @negative() -> tensor { + %lhs = constant dense<1.0> : tensor + %rhs = constant dense<-2.0> : tensor + %result = "xla_hlo.min"(%lhs, %rhs) : (tensor, tensor) -> tensor + return %result : tensor +} +// CHECK: f32=-2 diff --git a/test/e2e/xla/max_int.mlir b/test/e2e/xla/max_int.mlir new file mode 100644 index 000000000000..d36ddeaf0a9e --- /dev/null +++ b/test/e2e/xla/max_int.mlir @@ -0,0 +1,65 @@ +// RUN: iree-run-mlir --target_backends=interpreter-bytecode %s --output_types=i | FileCheck %s + +// CHECK-LABEL: EXEC @tensor +func @tensor() -> tensor<4xi32> { + %lhs = constant dense<[1, 6, 7, 8]> : tensor<4xi32> + %rhs = constant dense<[5, 6, 3, 8]> : tensor<4xi32> + %result = "xla_hlo.max"(%lhs, %rhs) : (tensor<4xi32>, tensor<4xi32>) -> tensor<4xi32> + return %result : tensor<4xi32> +} +// CHECK: 4xi32=5 6 7 8 + +// ----- + +// CHECK-LABEL: EXEC @tensor_odd_dim +func @tensor_odd_dim() -> tensor<3xi32> { + %lhs = constant dense<[1, 6, 7]> : tensor<3xi32> + %rhs = constant dense<[5, 6, 3]> : tensor<3xi32> + %result = "xla_hlo.max"(%lhs, %rhs) : (tensor<3xi32>, tensor<3xi32>) -> tensor<3xi32> + return %result : tensor<3xi32> +} +// CHECK: 3xi32=5 6 7 + +// ----- + +// CHECK-LABEL: EXEC @scalar +func @scalar() -> tensor { + %lhs = constant dense<1> : tensor + %rhs = constant dense<2> : tensor + %result = "xla_hlo.max"(%lhs, %rhs) : (tensor, tensor) -> tensor + return %result : tensor +} +// CHECK: i32=2 + +// ----- + +// CHECK-LABEL: EXEC @negative +func @negative() -> tensor { + %lhs = constant dense<1> : tensor + %rhs = constant dense<-2> : tensor + %result = "xla_hlo.max"(%lhs, %rhs) : (tensor, tensor) -> tensor + return %result : tensor +} +// CHECK: i32=1 + +// ----- + +// CHECK-LABEL: EXEC @i16 +func @i16() -> tensor { + %lhs = constant dense<1> : tensor + %rhs = constant dense<2> : tensor + %result = "xla_hlo.max"(%lhs, %rhs) : (tensor, tensor) -> tensor + return %result : tensor +} +// CHECK: i16=2 + +// ----- + +// CHECK-LABEL: EXEC @i64 +func @i64() -> tensor { + %lhs = constant dense<1> : tensor + %rhs = constant dense<2> : tensor + %result = "xla_hlo.max"(%lhs, %rhs) : (tensor, tensor) -> tensor + return %result : tensor +} +// CHECK: i64=2 diff --git a/test/e2e/xla/min_float.mlir b/test/e2e/xla/min_float.mlir new file mode 100644 index 000000000000..60734e1ad9c4 --- /dev/null +++ b/test/e2e/xla/min_float.mlir @@ -0,0 +1,43 @@ +// RUN: iree-run-mlir --target_backends=interpreter-bytecode %s --output_types=f | FileCheck %s + +// CHECK-LABEL: EXEC @tensor +func @tensor() -> tensor<4xf32> { + %lhs = constant dense<[1.0, 2.0, 7.0, 4.0]> : tensor<4xf32> + %rhs = constant dense<[5.0, 2.0, 3.0, 4.0]> : tensor<4xf32> + %result = "xla_hlo.min"(%lhs, %rhs) : (tensor<4xf32>, tensor<4xf32>) -> tensor<4xf32> + return %result : tensor<4xf32> +} +// CHECK: 4xf32=1 2 3 4 + +// ----- + +// CHECK-LABEL: EXEC @scalar +func @scalar() -> tensor { + %lhs = constant dense<1.0> : tensor + %rhs = constant dense<2.0> : tensor + %result = "xla_hlo.min"(%lhs, %rhs) : (tensor, tensor) -> tensor + return %result : tensor +} +// CHECK: f32=1 + +// ----- + +// CHECK-LABEL: EXEC @double +func @double() -> tensor { + %lhs = constant dense<1.0> : tensor + %rhs = constant dense<2.0> : tensor + %result = "xla_hlo.min"(%lhs, %rhs) : (tensor, tensor) -> tensor + return %result : tensor +} +// CHECK: f64=1 + +// ----- + +// CHECK-LABEL: EXEC @negative +func @negative() -> tensor { + %lhs = constant dense<1.0> : tensor + %rhs = constant dense<-2.0> : tensor + %result = "xla_hlo.min"(%lhs, %rhs) : (tensor, tensor) -> tensor + return %result : tensor +} +// CHECK: f32=-2 diff --git a/test/e2e/xla/min_int.mlir b/test/e2e/xla/min_int.mlir new file mode 100644 index 000000000000..99bcad25b8f7 --- /dev/null +++ b/test/e2e/xla/min_int.mlir @@ -0,0 +1,65 @@ +// RUN: iree-run-mlir --target_backends=interpreter-bytecode %s --output_types=i | FileCheck %s --dump-input=fail + +// CHECK-LABEL: EXEC @tensor +func @tensor() -> tensor<4xi32> { + %lhs = constant dense<[1, 2, 7, 4]> : tensor<4xi32> + %rhs = constant dense<[5, 2, 3, 4]> : tensor<4xi32> + %result = "xla_hlo.min"(%lhs, %rhs) : (tensor<4xi32>, tensor<4xi32>) -> tensor<4xi32> + return %result : tensor<4xi32> +} +// CHECK: 4xi32=1 2 3 4 + +// ----- + +// CHECK-LABEL: EXEC @tensor_odd_dim +func @tensor_odd_dim() -> tensor<3xi32> { + %lhs = constant dense<[1, 2, 7]> : tensor<3xi32> + %rhs = constant dense<[5, 2, 3]> : tensor<3xi32> + %result = "xla_hlo.min"(%lhs, %rhs) : (tensor<3xi32>, tensor<3xi32>) -> tensor<3xi32> + return %result : tensor<3xi32> +} +// CHECK: 3xi32=1 2 3 + +// ----- + +// CHECK-LABEL: EXEC @scalar +func @scalar() -> tensor { + %lhs = constant dense<1> : tensor + %rhs = constant dense<2> : tensor + %result = "xla_hlo.min"(%lhs, %rhs) : (tensor, tensor) -> tensor + return %result : tensor +} +// CHECK: i32=1 + +// ----- + +// CHECK-LABEL: EXEC @negative +func @negative() -> tensor { + %lhs = constant dense<1> : tensor + %rhs = constant dense<-2> : tensor + %result = "xla_hlo.min"(%lhs, %rhs) : (tensor, tensor) -> tensor + return %result : tensor +} +// CHECK: i32=-2 + +// ----- + +// CHECK-LABEL: EXEC @i16 +func @i16() -> tensor { + %lhs = constant dense<1> : tensor + %rhs = constant dense<2> : tensor + %result = "xla_hlo.min"(%lhs, %rhs) : (tensor, tensor) -> tensor + return %result : tensor +} +// CHECK: i16=1 + +// ----- + +// CHECK-LABEL: EXEC @i64 +func @i64() -> tensor { + %lhs = constant dense<1> : tensor + %rhs = constant dense<2> : tensor + %result = "xla_hlo.min"(%lhs, %rhs) : (tensor, tensor) -> tensor + return %result : tensor +} +// CHECK: i64=1 diff --git a/test/e2e/xla/mnist.mlir b/test/e2e/xla/mnist.mlir new file mode 100644 index 000000000000..f052fc28b0ad --- /dev/null +++ b/test/e2e/xla/mnist.mlir @@ -0,0 +1,48 @@ +// MNIST model with placeholder weights, for translation testing. + +// RUN: iree-run-mlir --target_backends=interpreter-bytecode %s --export_all=false --input_values="1x28x28x1xf32" | FileCheck %s --dump-input=fail + +module { + // CHECK-LABEL: EXEC @main + func @main(%arg0: tensor<1x28x28x1xf32>) -> tuple> + attributes {iree.module.export} { + %cst = constant {name = "constant.9"} dense<0.5> : tensor + %0 = "xla_hlo.broadcast_in_dim"(%cst) {name = "broadcast.10"} : (tensor) -> tensor<1x128xf32> + %1 = "xla_hlo.copy"(%arg0) {name = "copy.1"} : (tensor<1x28x28x1xf32>) -> tensor<1x28x28x1xf32> + %2 = "xla_hlo.reshape"(%1) {name = "reshape.2"} : (tensor<1x28x28x1xf32>) -> tensor<1x28x28x1xf32> + %3 = "xla_hlo.reshape"(%2) {name = "reshape.3"} : (tensor<1x28x28x1xf32>) -> tensor<1x784xf32> + %cst_0 = constant {name = "constant.4"} dense<0.5> : tensor<784x128xf32> + %4 = "xla_hlo.dot"(%3, %cst_0) {name = "dot.5", precision_config = ["DEFAULT", "DEFAULT"]} : (tensor<1x784xf32>, tensor<784x128xf32>) -> tensor<1x128xf32> + %cst_1 = constant {name = "constant.6"} dense<0.5> : tensor<128xf32> + %5 = "xla_hlo.broadcast_in_dim"(%cst_1) {broadcast_dimensions = dense<1> : tensor<1xi64>, name = "broadcast.7"} : (tensor<128xf32>) -> tensor<1x128xf32> + %6 = "xla_hlo.add"(%4, %5) {name = "add.8"} : (tensor<1x128xf32>, tensor<1x128xf32>) -> tensor<1x128xf32> + %7 = "xla_hlo.max"(%0, %6) {name = "maximum.11"} : (tensor<1x128xf32>, tensor<1x128xf32>) -> tensor<1x128xf32> + %cst_2 = constant {name = "constant.12"} dense<0.5> : tensor<128x10xf32> + %8 = "xla_hlo.dot"(%7, %cst_2) {name = "dot.13", precision_config = ["DEFAULT", "DEFAULT"]} : (tensor<1x128xf32>, tensor<128x10xf32>) -> tensor<1x10xf32> + %cst_3 = constant {name = "constant.14"} dense<0.5> : tensor<10xf32> + %9 = "xla_hlo.broadcast_in_dim"(%cst_3) {broadcast_dimensions = dense<1> : tensor<1xi64>, name = "broadcast.15"} : (tensor<10xf32>) -> tensor<1x10xf32> + %10 = "xla_hlo.add"(%8, %9) {name = "add.16"} : (tensor<1x10xf32>, tensor<1x10xf32>) -> tensor<1x10xf32> + %cst_4 = constant {name = "constant.17"} dense<0xFF800000> : tensor + %11 = "xla_hlo.reduce"(%10, %cst_4) ( { + ^bb0(%arg1: tensor, %arg2: tensor): // no predecessors + %20 = "xla_hlo.max"(%arg1, %arg2) {name = "maximum.21"} : (tensor, tensor) -> tensor + "xla_hlo.return"(%20) : (tensor) -> () + }) {dimensions = dense<1> : tensor<1xi64>} : (tensor<1x10xf32>, tensor) -> tensor<1xf32> + %12 = "xla_hlo.broadcast_in_dim"(%11) {broadcast_dimensions = dense<0> : tensor<1xi64>, name = "broadcast.23"} : (tensor<1xf32>) -> tensor<1x10xf32> + %13 = "xla_hlo.sub"(%10, %12) {name = "subtract.24"} : (tensor<1x10xf32>, tensor<1x10xf32>) -> tensor<1x10xf32> + %14 = "xla_hlo.exp"(%13) {name = "exponential.25"} : (tensor<1x10xf32>) -> tensor<1x10xf32> + %cst_5 = constant {name = "constant.27"} dense<0.5> : tensor + %15 = "xla_hlo.reduce"(%14, %cst_5) ( { + ^bb0(%arg3: tensor, %arg4: tensor): // no predecessors + %21 = "xla_hlo.add"(%arg3, %arg4) {name = "add.31"} : (tensor, tensor) -> tensor + "xla_hlo.return"(%21) : (tensor) -> () + }) {dimensions = dense<1> : tensor<1xi64>} : (tensor<1x10xf32>, tensor) -> tensor<1xf32> + %16 = "xla_hlo.broadcast_in_dim"(%15) {broadcast_dimensions = dense<0> : tensor<1xi64>, name = "broadcast.34"} : (tensor<1xf32>) -> tensor<1x10xf32> + %17 = "xla_hlo.div"(%14, %16) {name = "divide.35"} : (tensor<1x10xf32>, tensor<1x10xf32>) -> tensor<1x10xf32> + %18 = "xla_hlo.reshape"(%17) {name = "reshape.36"} : (tensor<1x10xf32>) -> tensor<1x10xf32> + %19 = "xla_hlo.tuple"(%18) {name = "tuple.37"} : (tensor<1x10xf32>) -> tuple> + return %19 : tuple> + } +} + +// CHECK: 1x10xf32=[ diff --git a/test/e2e/xla/pad.mlir b/test/e2e/xla/pad.mlir new file mode 100644 index 000000000000..e8c6340c62d0 --- /dev/null +++ b/test/e2e/xla/pad.mlir @@ -0,0 +1,12 @@ +// RUN: iree-run-mlir --target_backends=interpreter-bytecode %s --input_values="2x3xi32=[1 2 3 4 5 6]" --output_types=i | FileCheck %s --dump-input=fail + +// CHECK-LABEL: EXEC @pad +func @pad(%arg : tensor<2x3xi32>) -> tensor<3x6xi32> { + %pad_val = constant dense<0> : tensor + %result = "xla_hlo.pad"(%arg, %pad_val) {interior_padding = dense<0> : tensor<2xi64>, edge_padding_low = dense<[0,1]> : tensor<2xi64>, edge_padding_high = dense<[1, 2]> : tensor<2xi64>} : (tensor<2x3xi32>, tensor) -> tensor<3x6xi32> + return %result : tensor<3x6xi32> +} +// CHECK-NEXT: 3x6xi32= +// CHECK-SAME: [0 1 2 3 0 0] +// CHECK-SAME: [0 4 5 6 0 0] +// CHECK-SAME: [0 0 0 0 0 0] diff --git a/test/e2e/xla/reduce_float.mlir b/test/e2e/xla/reduce_float.mlir new file mode 100644 index 000000000000..82e8fd0474d5 --- /dev/null +++ b/test/e2e/xla/reduce_float.mlir @@ -0,0 +1,173 @@ +// RUN: iree-run-mlir --target_backends=interpreter-bytecode %s --output_types="f" | FileCheck %s --dump-input=fail + +// Float sum values from [1.0, 10.0] +// CHECK-LABEL: EXEC @reduce_sum_1x10xf32 +func @reduce_sum_1x10xf32() -> tensor<1xf32> { + %0 = constant dense<[[1.0, 2.0, 3.0, 4.0, 5.0, 6.0, 7.0, 8.0, 9.0, 10.0]]> : tensor<1x10xf32> + %1 = constant dense<0.0> : tensor + %2 = "xla_hlo.reduce"(%0, %1) ( { + ^bb0(%arg0: tensor, %arg1: tensor): // no predecessors + %3 = "xla_hlo.add"(%arg0, %arg1) : (tensor, tensor) -> tensor + "xla_hlo.return"(%3) : (tensor) -> () + }) {dimensions = dense<1> : tensor<1xi64>} : (tensor<1x10xf32>, tensor) -> tensor<1xf32> + return %2 : tensor<1xf32> +} +// CHECK: 1xf32=55 + +// ----- + +// Float max values from [1.0, 10.0] +// CHECK-LABEL: EXEC @reduce_max_1x10xf32 +func @reduce_max_1x10xf32() -> tensor<1xf32> { + %0 = constant dense<[[1.0, 2.0, 3.0, 4.0, 5.0, 6.0, 7.0, 8.0, 9.0, 10.0]]> : tensor<1x10xf32> + %1 = constant dense<0.0> : tensor + %2 = "xla_hlo.reduce"(%0, %1) + ( { + ^bb0(%arg0: tensor, %arg1: tensor): // no predecessors + %3 = "xla_hlo.max"(%arg0, %arg1) : (tensor, tensor) -> tensor + "xla_hlo.return"(%3) : (tensor) -> () + }) + {dimensions = dense<1> : tensor<1xi64>} : (tensor<1x10xf32>, tensor) -> tensor<1xf32> + return %2 : tensor<1xf32> +} +// CHECK: 1xf32=10 + +// ----- + +// Float min values, along multiple dimensions. Expected to just be a reshape in this case. +// CHECK-LABEL: EXEC @reduce_min_5x1x1xf32 +func @reduce_min_5x1x1xf32() -> tensor<5xf32> { + %0 = constant dense<[[[1.0]],[[2.0]],[[3.0]],[[4.0]],[[5.0]]]> : tensor<5x1x1xf32> + %1 = constant dense<999.0> : tensor + %2 = "xla_hlo.reduce"(%0, %1) ( { + ^bb0(%arg0: tensor, %arg1: tensor): // no predecessors + %3 = "xla_hlo.min"(%arg0, %arg1) : (tensor, tensor) -> tensor + "xla_hlo.return"(%3) : (tensor) -> () + }) {dimensions = dense<[1, 2]> : tensor<2xi64>} : (tensor<5x1x1xf32>, tensor) -> tensor<5xf32> + return %2 : tensor<5xf32> +} +// CHECK: 5xf32=1 2 3 4 5 + +// ----- + +// The following cases match the examples presented at +// https://www.tensorflow.org/xla/operation_semantics#reduce + +// CHECK-LABEL: EXEC @reduce_sum_2x3xf32_dim0 +func @reduce_sum_2x3xf32_dim0() -> tensor<3xf32> { + %0 = constant dense<[[1.0, 2.0, 3.0], + [4.0, 5.0, 6.0]]> : tensor<2x3xf32> + %1 = constant dense<0.0> : tensor + %2 = "xla_hlo.reduce"(%0, %1) ( { + ^bb0(%arg0: tensor, %arg1: tensor): // no predecessors + %3 = "xla_hlo.add"(%arg0, %arg1) : (tensor, tensor) -> tensor + "xla_hlo.return"(%3) : (tensor) -> () + }) {dimensions = dense<[0]> : tensor<1xi64>} : (tensor<2x3xf32>, tensor) -> tensor<3xf32> + return %2 : tensor<3xf32> +} +// CHECK: 3xf32=5 7 9 + +// ----- + +// CHECK-LABEL: EXEC @reduce_sum_2x3xf32_dim1 +func @reduce_sum_2x3xf32_dim1() -> tensor<2xf32> { + %0 = constant dense<[[1.0, 2.0, 3.0], + [4.0, 5.0, 6.0]]> : tensor<2x3xf32> + %1 = constant dense<0.0> : tensor + %2 = "xla_hlo.reduce"(%0, %1) ( { + ^bb0(%arg0: tensor, %arg1: tensor): // no predecessors + %3 = "xla_hlo.add"(%arg0, %arg1) : (tensor, tensor) -> tensor + "xla_hlo.return"(%3) : (tensor) -> () + }) {dimensions = dense<[1]> : tensor<1xi64>} : (tensor<2x3xf32>, tensor) -> tensor<2xf32> + return %2 : tensor<2xf32> +} +// CHECK: 2xf32=6 15 + +// ----- + +// CHECK-LABEL: EXEC @reduce_sum_4x2x3xf32_dim0 +func @reduce_sum_4x2x3xf32_dim0() -> tensor<2x3xf32> { + %0 = constant dense<[[[1.0, 2.0, 3.0], + [4.0, 5.0, 6.0]], + [[1.0, 2.0, 3.0], + [4.0, 5.0, 6.0]], + [[1.0, 2.0, 3.0], + [4.0, 5.0, 6.0]], + [[1.0, 2.0, 3.0], + [4.0, 5.0, 6.0]]]> : tensor<4x2x3xf32> + %1 = constant dense<0.0> : tensor + %2 = "xla_hlo.reduce"(%0, %1) ( { + ^bb0(%arg0: tensor, %arg1: tensor): // no predecessors + %3 = "xla_hlo.add"(%arg0, %arg1) : (tensor, tensor) -> tensor + "xla_hlo.return"(%3) : (tensor) -> () + }) {dimensions = dense<[0]> : tensor<1xi64>} : (tensor<4x2x3xf32>, tensor) -> tensor<2x3xf32> + return %2 : tensor<2x3xf32> +} +// CHECK: 2x3xf32=[4 8 12][16 20 24] + +// ----- + +// CHECK-LABEL: EXEC @reduce_sum_4x2x3xf32_dim2 +func @reduce_sum_4x2x3xf32_dim2() -> tensor<4x2xf32> { + %0 = constant dense<[[[1.0, 2.0, 3.0], + [4.0, 5.0, 6.0]], + [[1.0, 2.0, 3.0], + [4.0, 5.0, 6.0]], + [[1.0, 2.0, 3.0], + [4.0, 5.0, 6.0]], + [[1.0, 2.0, 3.0], + [4.0, 5.0, 6.0]]]> : tensor<4x2x3xf32> + %1 = constant dense<0.0> : tensor + %2 = "xla_hlo.reduce"(%0, %1) ( { + ^bb0(%arg0: tensor, %arg1: tensor): // no predecessors + %3 = "xla_hlo.add"(%arg0, %arg1) : (tensor, tensor) -> tensor + "xla_hlo.return"(%3) : (tensor) -> () + }) {dimensions = dense<[2]> : tensor<1xi64>} : (tensor<4x2x3xf32>, tensor) -> tensor<4x2xf32> + return %2 : tensor<4x2xf32> +} +// CHECK: 4x2xf32=[6 15][6 15][6 15][6 15] + +// ----- + +// CHECK-LABEL: EXEC @reduce_sum_4x2x3xf32_dims_0_1 +func @reduce_sum_4x2x3xf32_dims_0_1() -> tensor<3xf32> { + %0 = constant dense<[[[1.0, 2.0, 3.0], + [4.0, 5.0, 6.0]], + [[1.0, 2.0, 3.0], + [4.0, 5.0, 6.0]], + [[1.0, 2.0, 3.0], + [4.0, 5.0, 6.0]], + [[1.0, 2.0, 3.0], + [4.0, 5.0, 6.0]]]> : tensor<4x2x3xf32> + %1 = constant dense<0.0> : tensor + %2 = "xla_hlo.reduce"(%0, %1) ( { + ^bb0(%arg0: tensor, %arg1: tensor): // no predecessors + %3 = "xla_hlo.add"(%arg0, %arg1) : (tensor, tensor) -> tensor + "xla_hlo.return"(%3) : (tensor) -> () + }) {dimensions = dense<[0, 1]> : tensor<2xi64>} : (tensor<4x2x3xf32>, tensor) -> tensor<3xf32> + return %2 : tensor<3xf32> +} +// CHECK: 3xf32=20 28 36 + +// ----- + +// CHECK-LABEL: EXEC @reduce_sum_4x2x3xf32_dims_0_1_2 +func @reduce_sum_4x2x3xf32_dims_0_1_2() -> tensor { + %0 = constant dense<[[[1.0, 2.0, 3.0], + [4.0, 5.0, 6.0]], + [[1.0, 2.0, 3.0], + [4.0, 5.0, 6.0]], + [[1.0, 2.0, 3.0], + [4.0, 5.0, 6.0]], + [[1.0, 2.0, 3.0], + [4.0, 5.0, 6.0]]]> : tensor<4x2x3xf32> + %1 = constant dense<0.0> : tensor + %2 = "xla_hlo.reduce"(%0, %1) ( { + ^bb0(%arg0: tensor, %arg1: tensor): // no predecessors + %3 = "xla_hlo.add"(%arg0, %arg1) : (tensor, tensor) -> tensor + "xla_hlo.return"(%3) : (tensor) -> () + }) {dimensions = dense<[0, 1, 2]> : tensor<3xi64>} : (tensor<4x2x3xf32>, tensor) -> tensor + return %2 : tensor +} +// CHECK: f32=84 + diff --git a/test/e2e/xla/reduce_int.mlir b/test/e2e/xla/reduce_int.mlir new file mode 100644 index 000000000000..cce048c07fc1 --- /dev/null +++ b/test/e2e/xla/reduce_int.mlir @@ -0,0 +1,171 @@ +// RUN: iree-run-mlir --target_backends=interpreter-bytecode %s --output_types="i" | FileCheck %s --dump-input=fail + +// Int sum values from [1, 10] +// CHECK-LABEL: EXEC @reduce_sum_1x10xi32 +func @reduce_sum_1x10xi32() -> tensor<1xi32> { + %0 = constant dense<[[1, 2, 3, 4, 5, 6, 7, 8, 9, 10]]> : tensor<1x10xi32> + %1 = constant dense<0> : tensor + %2 = "xla_hlo.reduce"(%0, %1) ( { + ^bb0(%arg0: tensor, %arg1: tensor): // no predecessors + %3 = "xla_hlo.add"(%arg0, %arg1) : (tensor, tensor) -> tensor + "xla_hlo.return"(%3) : (tensor) -> () + }) {dimensions = dense<1> : tensor<1xi64>} : (tensor<1x10xi32>, tensor) -> tensor<1xi32> + return %2 : tensor<1xi32> +} +// CHECK: 1xi32=55 + +// ----- + +// Int max values from [1, 10] +// CHECK-LABEL: EXEC @reduce_max_1x10xi32 +func @reduce_max_1x10xi32() -> tensor<1xi32> { + %0 = constant dense<[[1, 2, 3, 4, 5, 6, 7, 8, 9, 10]]> : tensor<1x10xi32> + %1 = constant dense<0> : tensor + %2 = "xla_hlo.reduce"(%0, %1) ( { + ^bb0(%arg0: tensor, %arg1: tensor): // no predecessors + %3 = "xla_hlo.max"(%arg0, %arg1) : (tensor, tensor) -> tensor + "xla_hlo.return"(%3) : (tensor) -> () + }) {dimensions = dense<1> : tensor<1xi64>} : (tensor<1x10xi32>, tensor) -> tensor<1xi32> + return %2 : tensor<1xi32> +} +// CHECK: 1xi32=10 + +// ----- + +// Int min values, along multiple dimensions. Expected to just be a reshape in this case. +// CHECK-LABEL: EXEC @reduce_min_5x1x1xi32 +func @reduce_min_5x1x1xi32() -> tensor<5xi32> { + %0 = constant dense<[[[1]],[[2]],[[3]],[[4]],[[5]]]> : tensor<5x1x1xi32> + %1 = constant dense<999> : tensor + %2 = "xla_hlo.reduce"(%0, %1) ( { + ^bb0(%arg0: tensor, %arg1: tensor): // no predecessors + %3 = "xla_hlo.min"(%arg0, %arg1) : (tensor, tensor) -> tensor + "xla_hlo.return"(%3) : (tensor) -> () + }) {dimensions = dense<[1, 2]> : tensor<2xi64>} : (tensor<5x1x1xi32>, tensor) -> tensor<5xi32> + return %2 : tensor<5xi32> +} +// CHECK: 5xi32=1 2 3 4 5 + +// ----- + +// The following cases match the examples presented at +// https://www.tensorflow.org/xla/operation_semantics#reduce + +// CHECK-LABEL: EXEC @reduce_sum_2x3xi32_dim0 +func @reduce_sum_2x3xi32_dim0() -> tensor<3xi32> { + %0 = constant dense<[[1, 2, 3], + [4, 5, 6]]> : tensor<2x3xi32> + %1 = constant dense<0> : tensor + %2 = "xla_hlo.reduce"(%0, %1) ( { + ^bb0(%arg0: tensor, %arg1: tensor): // no predecessors + %3 = "xla_hlo.add"(%arg0, %arg1) : (tensor, tensor) -> tensor + "xla_hlo.return"(%3) : (tensor) -> () + }) {dimensions = dense<[0]> : tensor<1xi64>} : (tensor<2x3xi32>, tensor) -> tensor<3xi32> + return %2 : tensor<3xi32> +} +// CHECK: 3xi32=5 7 9 + +// ----- + +// CHECK-LABEL: EXEC @reduce_sum_2x3xi32_dim1 +func @reduce_sum_2x3xi32_dim1() -> tensor<2xi32> { + %0 = constant dense<[[1, 2, 3], + [4, 5, 6]]> : tensor<2x3xi32> + %1 = constant dense<0> : tensor + %2 = "xla_hlo.reduce"(%0, %1) ( { + ^bb0(%arg0: tensor, %arg1: tensor): // no predecessors + %3 = "xla_hlo.add"(%arg0, %arg1) : (tensor, tensor) -> tensor + "xla_hlo.return"(%3) : (tensor) -> () + }) {dimensions = dense<[1]> : tensor<1xi64>} : (tensor<2x3xi32>, tensor) -> tensor<2xi32> + return %2 : tensor<2xi32> +} +// CHECK: 2xi32=6 15 + +// ----- + +// CHECK-LABEL: EXEC @reduce_sum_4x2x3xi32_dim0 +func @reduce_sum_4x2x3xi32_dim0() -> tensor<2x3xi32> { + %0 = constant dense<[[[1, 2, 3], + [4, 5, 6]], + [[1, 2, 3], + [4, 5, 6]], + [[1, 2, 3], + [4, 5, 6]], + [[1, 2, 3], + [4, 5, 6]]]> : tensor<4x2x3xi32> + %1 = constant dense<0> : tensor + %2 = "xla_hlo.reduce"(%0, %1) ( { + ^bb0(%arg0: tensor, %arg1: tensor): // no predecessors + %3 = "xla_hlo.add"(%arg0, %arg1) : (tensor, tensor) -> tensor + "xla_hlo.return"(%3) : (tensor) -> () + }) {dimensions = dense<[0]> : tensor<1xi64>} : (tensor<4x2x3xi32>, tensor) -> tensor<2x3xi32> + return %2 : tensor<2x3xi32> +} +// CHECK: 2x3xi32=[4 8 12][16 20 24] + +// ----- + +// CHECK-LABEL: EXEC @reduce_sum_4x2x3xi32_dim2 +func @reduce_sum_4x2x3xi32_dim2() -> tensor<4x2xi32> { + %0 = constant dense<[[[1, 2, 3], + [4, 5, 6]], + [[1, 2, 3], + [4, 5, 6]], + [[1, 2, 3], + [4, 5, 6]], + [[1, 2, 3], + [4, 5, 6]]]> : tensor<4x2x3xi32> + %1 = constant dense<0> : tensor + %2 = "xla_hlo.reduce"(%0, %1) ( { + ^bb0(%arg0: tensor, %arg1: tensor): // no predecessors + %3 = "xla_hlo.add"(%arg0, %arg1) : (tensor, tensor) -> tensor + "xla_hlo.return"(%3) : (tensor) -> () + }) {dimensions = dense<[2]> : tensor<1xi64>} : (tensor<4x2x3xi32>, tensor) -> tensor<4x2xi32> + return %2 : tensor<4x2xi32> +} +// CHECK: 4x2xi32=[6 15][6 15][6 15][6 15] + +// ----- + +// CHECK-LABEL: EXEC @reduce_sum_4x2x3xi32_dims_0_1 +func @reduce_sum_4x2x3xi32_dims_0_1() -> tensor<3xi32> { + %0 = constant dense<[[[1, 2, 3], + [4, 5, 6]], + [[1, 2, 3], + [4, 5, 6]], + [[1, 2, 3], + [4, 5, 6]], + [[1, 2, 3], + [4, 5, 6]]]> : tensor<4x2x3xi32> + %1 = constant dense<0> : tensor + %2 = "xla_hlo.reduce"(%0, %1) ( { + ^bb0(%arg0: tensor, %arg1: tensor): // no predecessors + %3 = "xla_hlo.add"(%arg0, %arg1) : (tensor, tensor) -> tensor + "xla_hlo.return"(%3) : (tensor) -> () + }) {dimensions = dense<[0, 1]> : tensor<2xi64>} : (tensor<4x2x3xi32>, tensor) -> tensor<3xi32> + return %2 : tensor<3xi32> +} +// CHECK: 3xi32=20 28 36 + +// ----- + +// CHECK-LABEL: EXEC @reduce_sum_4x2x3xi32_dims_0_1_2 +func @reduce_sum_4x2x3xi32_dims_0_1_2() -> tensor { + %0 = constant dense<[[[1, 2, 3], + [4, 5, 6]], + [[1, 2, 3], + [4, 5, 6]], + [[1, 2, 3], + [4, 5, 6]], + [[1, 2, 3], + [4, 5, 6]]]> : tensor<4x2x3xi32> + %1 = constant dense<0> : tensor + %2 = "xla_hlo.reduce"(%0, %1) ( { + ^bb0(%arg0: tensor, %arg1: tensor): // no predecessors + %3 = "xla_hlo.add"(%arg0, %arg1) : (tensor, tensor) -> tensor + "xla_hlo.return"(%3) : (tensor) -> () + }) {dimensions = dense<[0, 1, 2]> : tensor<3xi64>} : (tensor<4x2x3xi32>, tensor) -> tensor + return %2 : tensor +} +// CHECK: i32=84 + diff --git a/test/e2e/xla/reverse.mlir b/test/e2e/xla/reverse.mlir new file mode 100644 index 000000000000..c0650bebefdc --- /dev/null +++ b/test/e2e/xla/reverse.mlir @@ -0,0 +1,15 @@ +// RUN: iree-run-mlir --target_backends=interpreter-bytecode %s | FileCheck %s --dump-input=fail + +// ----- + +// CHECK-LABEL: EXEC @xla_reverse +func @xla_reverse () -> (tensor<2x3xf32>, tensor <2x3xf32>, tensor <2x3xf32>) { + %t1 = "xla_hlo.constant"() {value = dense<[[1.0e0, 2.0e0, 3.0e0], [4.0e0, 5.0e0, 6.0e0]]> : tensor<2x3xf32>} : () -> tensor<2x3xf32> + %0 = "xla_hlo.reverse"(%t1) {dimensions = dense<[0]> : tensor<1xi64>} : (tensor<2x3xf32>) -> tensor<2x3xf32> + %1 = "xla_hlo.reverse"(%t1) {dimensions = dense<[1]> : tensor<1xi64>} : (tensor<2x3xf32>) -> tensor<2x3xf32> + %2 = "xla_hlo.reverse"(%t1) {dimensions = dense<[0, 1]> : tensor<2xi64>} : (tensor<2x3xf32>) -> tensor<2x3xf32> + return %0, %1, %2: tensor<2x3xf32>, tensor<2x3xf32>, tensor<2x3xf32> +} +// CHECK: 2x3xf32=[4 5 6][1 2 3] +// CHECK-NEXT: 2x3xf32=[3 2 1][6 5 4] +// CHECK_NEXT: 2x3xf32=[6 5 4][3 2 1] diff --git a/test/e2e/xla/rsqrt.mlir b/test/e2e/xla/rsqrt.mlir new file mode 100644 index 000000000000..893fe8f5ba31 --- /dev/null +++ b/test/e2e/xla/rsqrt.mlir @@ -0,0 +1,29 @@ +// RUN: iree-run-mlir --target_backends=interpreter-bytecode %s --output_types=f | FileCheck %s + +// CHECK-LABEL: EXEC @tensor +func @tensor() -> tensor<4xf32> { + %input = constant dense<[1.0, 2.0, 3.0, 4.0]> : tensor<4xf32> + %result = "xla_hlo.rsqrt"(%input) : (tensor<4xf32>) -> tensor<4xf32> + return %result : tensor<4xf32> +} +// CHECK: 4xf32=1 0.707107 0.57735 0.5 + +// ----- + +// CHECK-LABEL: EXEC @scalar +func @scalar() -> tensor { + %input = constant dense<16.0> : tensor + %result = "xla_hlo.rsqrt"(%input) : (tensor) -> tensor + return %result : tensor +} +// CHECK: f32=0.25 + +// ----- + +// CHECK-LABEL: EXEC @double +func @double() -> tensor { + %input = constant dense<16.0> : tensor + %result = "xla_hlo.rsqrt"(%input) : (tensor) -> tensor + return %result : tensor +} +// CHECK: f64=0.25 diff --git a/test/e2e/xla/select.mlir b/test/e2e/xla/select.mlir new file mode 100644 index 000000000000..aae902c1a8ce --- /dev/null +++ b/test/e2e/xla/select.mlir @@ -0,0 +1,9 @@ +// RUN: iree-run-mlir --target_backends=interpreter-bytecode %s --input_values="4xi8=[1 0 200 0]" | FileCheck %s +// CHECK-LABEL: EXEC @select +func @select(%cond : tensor<4xi1>) -> tensor<4xf32> { + %lhs = constant dense<[1.0, 2.0, 3.0, 4.0]> : tensor<4xf32> + %rhs = constant dense<[5.0, 6.0, 7.0, 8.0]> : tensor<4xf32> + %result = "xla_hlo.select"(%cond, %lhs, %rhs) : (tensor<4xi1>, tensor<4xf32>, tensor<4xf32>) -> tensor<4xf32> + return %result : tensor<4xf32> +} +// CHECK: 4xf32=1 6 3 8 diff --git a/test/e2e/xla/slice.mlir b/test/e2e/xla/slice.mlir new file mode 100644 index 000000000000..d793994f7ce7 --- /dev/null +++ b/test/e2e/xla/slice.mlir @@ -0,0 +1,33 @@ +// RUN: iree-run-mlir --target_backends=interpreter-bytecode %s --input_values="12xf32=[1 2 3 4 5 6 7 8 9 10 11 12]" | FileCheck %s + +// CHECK-LABEL: EXEC @slice_whole_buffer +func @slice_whole_buffer(%arg : tensor<12xf32>) -> tensor<3x4xf32> { + %reshaped = "xla_hlo.reshape"(%arg) : (tensor<12xf32>) -> tensor<3x4xf32> + %result = "xla_hlo.slice"(%reshaped) {start_indices = dense<[0, 0]> : tensor<2xi64>, limit_indices = dense<[3, 4]> : tensor<2xi64>, strides = dense<1> : tensor<2xi64>} : (tensor<3x4xf32>) -> tensor<3x4xf32> + return %result : tensor<3x4xf32> +} +// CHECK: 3x4xf32=[1 2 3 4][5 6 7 8][9 10 11 12] + +// CHECK-LABEL: EXEC @slice_whole_stride +func @slice_whole_stride(%arg : tensor<12xf32>) -> tensor<1x4xf32> { + %reshaped = "xla_hlo.reshape"(%arg) : (tensor<12xf32>) -> tensor<3x4xf32> + %result = "xla_hlo.slice"(%reshaped) {start_indices = dense<[1, 0]> : tensor<2xi64>, limit_indices = dense<[2, 4]> : tensor<2xi64>, strides = dense<1> : tensor<2xi64>} : (tensor<3x4xf32>) -> tensor<1x4xf32> + return %result : tensor<1x4xf32> +} +// CHECK: 1x4xf32=[5 6 7 8] + +// CHECK-LABEL: EXEC @slice_stride_part +func @slice_stride_part(%arg : tensor<12xf32>) -> tensor<1x2xf32> { + %reshaped = "xla_hlo.reshape"(%arg) : (tensor<12xf32>) -> tensor<3x4xf32> + %result = "xla_hlo.slice"(%reshaped) {start_indices = dense<[1, 1]> : tensor<2xi64>, limit_indices = dense<[2, 3]> : tensor<2xi64>, strides = dense<1> : tensor<2xi64>} : (tensor<3x4xf32>) -> tensor<1x2xf32> + return %result : tensor<1x2xf32> +} +// CHECK: 1x2xf32=[6 7] + +// CHECK-LABEL: EXEC @slice_multi_stride +func @slice_multi_stride(%arg : tensor<12xf32>) -> tensor<2x4xf32> { + %reshaped = "xla_hlo.reshape"(%arg) : (tensor<12xf32>) -> tensor<3x4xf32> + %result = "xla_hlo.slice"(%reshaped) {start_indices = dense<[1, 0]> : tensor<2xi64>, limit_indices = dense<[3, 4]> : tensor<2xi64>, strides = dense<1> : tensor<2xi64>} : (tensor<3x4xf32>) -> tensor<2x4xf32> + return %result : tensor<2x4xf32> +} +// CHECK: 2x4xf32=[5 6 7 8][9 10 11 12] diff --git a/test/e2e/xla/through_std.mlir b/test/e2e/xla/through_std.mlir new file mode 100644 index 000000000000..ebd1b642618a --- /dev/null +++ b/test/e2e/xla/through_std.mlir @@ -0,0 +1,11 @@ +// RUN: iree-run-mlir %s | FileCheck %s --dump-input=fail + +// CHECK-LABEL: EXEC @xla_through_stdops +func @xla_through_stdops () -> (tensor, tensor) { + %tf32 = constant dense<1.0> : tensor + %0 = "xla_hlo.add"(%tf32, %tf32) : (tensor, tensor) -> tensor + %1 = "xla_hlo.mul"(%tf32, %tf32) : (tensor, tensor) -> tensor + return %0, %1 : tensor, tensor +} +// CHECK: f32=2 +// CHECK-NEXT: f32=1 diff --git a/test/e2e/xla/unidirectional_lstm.mlir b/test/e2e/xla/unidirectional_lstm.mlir new file mode 100644 index 000000000000..d6ace11a1d1c --- /dev/null +++ b/test/e2e/xla/unidirectional_lstm.mlir @@ -0,0 +1,680 @@ +// An example LSTM exported from a python reference model with dummy weights. + +// RUN: iree-run-mlir %s --input_values="1x5xf32=[0 1 0 3 4]\n1x5x2x2xf32=[1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20]" --target_backends=interpreter-bytecode --noexport_all --noprint_mlir | FileCheck %s --implicit-check-not="[" --implicit-check-not="]" --dump-input=fail + +// Exported via the XLA HLO Importer +// The resulting MLIR was modified by hand by changing all large constants to be +// splats of 0.42, removing the leading "module" wrapper, removing "name" +// attributes, removing extraneous 0s from float constants, and cleaning up +// extra whitespace. + +func @Min_reduction.47(%arg0: tensor, %arg1: tensor) -> tensor { + %0 = xla_hlo.min %arg0, %arg1 : tensor + return %0 : tensor +} +func @Max_reduction.51(%arg0: tensor, %arg1: tensor) -> tensor { + %0 = xla_hlo.max %arg0, %arg1 : tensor + return %0 : tensor +} +func @Max_1_reduction.55(%arg0: tensor, %arg1: tensor) -> tensor { + %0 = xla_hlo.max %arg0, %arg1 : tensor + return %0 : tensor +} +func @ForwardLoopBody_o0Jnom3Cdxo__.59(%arg0: tuple, tensor, tensor<40xf32>, tensor, tensor<74x40xf32>, tensor, tensor<1x10xf32>, tensor<1x10xf32>, tensor<5x1x64xf32>, tensor<5x1x1xf32>, tensor<5x1x1xf32>, tensor<5xi64>, tensor<5x1x10xf32>, tensor<5x1x10xf32>>) -> tuple, tensor, tensor<40xf32>, tensor, tensor<74x40xf32>, tensor, tensor<1x10xf32>, tensor<1x10xf32>, tensor<5x1x64xf32>, tensor<5x1x1xf32>, tensor<5x1x1xf32>, tensor<5xi64>, tensor<5x1x10xf32>, tensor<5x1x10xf32>> { + %0 = "xla_hlo.get_tuple_element"(%arg0) {index = 0 : i32} : (tuple, tensor, tensor<40xf32>, tensor, tensor<74x40xf32>, tensor, tensor<1x10xf32>, tensor<1x10xf32>, tensor<5x1x64xf32>, tensor<5x1x1xf32>, tensor<5x1x1xf32>, tensor<5xi64>, tensor<5x1x10xf32>, tensor<5x1x10xf32>>) -> tensor + %cst = constant dense<1> : tensor + %1 = xla_hlo.add %0, %cst : tensor + %2 = "xla_hlo.get_tuple_element"(%arg0) {index = 1 : i32} : (tuple, tensor, tensor<40xf32>, tensor, tensor<74x40xf32>, tensor, tensor<1x10xf32>, tensor<1x10xf32>, tensor<5x1x64xf32>, tensor<5x1x1xf32>, tensor<5x1x1xf32>, tensor<5xi64>, tensor<5x1x10xf32>, tensor<5x1x10xf32>>) -> tensor + %3 = "xla_hlo.get_tuple_element"(%arg0) {index = 2 : i32} : (tuple, tensor, tensor<40xf32>, tensor, tensor<74x40xf32>, tensor, tensor<1x10xf32>, tensor<1x10xf32>, tensor<5x1x64xf32>, tensor<5x1x1xf32>, tensor<5x1x1xf32>, tensor<5xi64>, tensor<5x1x10xf32>, tensor<5x1x10xf32>>) -> tensor<40xf32> + %4 = "xla_hlo.get_tuple_element"(%arg0) {index = 3 : i32} : (tuple, tensor, tensor<40xf32>, tensor, tensor<74x40xf32>, tensor, tensor<1x10xf32>, tensor<1x10xf32>, tensor<5x1x64xf32>, tensor<5x1x1xf32>, tensor<5x1x1xf32>, tensor<5xi64>, tensor<5x1x10xf32>, tensor<5x1x10xf32>>) -> tensor + %5 = "xla_hlo.get_tuple_element"(%arg0) {index = 4 : i32} : (tuple, tensor, tensor<40xf32>, tensor, tensor<74x40xf32>, tensor, tensor<1x10xf32>, tensor<1x10xf32>, tensor<5x1x64xf32>, tensor<5x1x1xf32>, tensor<5x1x1xf32>, tensor<5xi64>, tensor<5x1x10xf32>, tensor<5x1x10xf32>>) -> tensor<74x40xf32> + %6 = "xla_hlo.get_tuple_element"(%arg0) {index = 5 : i32} : (tuple, tensor, tensor<40xf32>, tensor, tensor<74x40xf32>, tensor, tensor<1x10xf32>, tensor<1x10xf32>, tensor<5x1x64xf32>, tensor<5x1x1xf32>, tensor<5x1x1xf32>, tensor<5xi64>, tensor<5x1x10xf32>, tensor<5x1x10xf32>>) -> tensor + %7 = "xla_hlo.get_tuple_element"(%arg0) {index = 9 : i32} : (tuple, tensor, tensor<40xf32>, tensor, tensor<74x40xf32>, tensor, tensor<1x10xf32>, tensor<1x10xf32>, tensor<5x1x64xf32>, tensor<5x1x1xf32>, tensor<5x1x1xf32>, tensor<5xi64>, tensor<5x1x10xf32>, tensor<5x1x10xf32>>) -> tensor<5x1x1xf32> + %8 = "xla_hlo.gather"(%7, %0) {collapsed_slice_dims = dense<0> : tensor<1xi64>, index_vector_dim = 0 : i64, offset_dims = dense<[0, 1]> : tensor<2xi64>, slice_sizes = dense<1> : tensor<3xi64>, start_index_map = dense<0> : tensor<1xi64>} : (tensor<5x1x1xf32>, tensor) -> tensor<1x1xf32> + %9 = "xla_hlo.reshape"(%8) : (tensor<1x1xf32>) -> tensor<1xf32> + %10 = "xla_hlo.broadcast_in_dim"(%9) {broadcast_dimensions = dense<0> : tensor<1xi64>} : (tensor<1xf32>) -> tensor<1x10xf32> + %cst_0 = constant dense<1.0> : tensor + %11 = "xla_hlo.broadcast_in_dim"(%cst_0) : (tensor) -> tensor<1x10xf32> + %12 = xla_hlo.mul %10, %11 : tensor<1x10xf32> + %cst_1 = constant dense<0.0> : tensor + %13 = "xla_hlo.broadcast_in_dim"(%cst_1) : (tensor) -> tensor<1x10xf32> + %14 = "xla_hlo.compare"(%12, %13) {comparison_direction = "GT"} : (tensor<1x10xf32>, tensor<1x10xf32>) -> tensor<1x10xi1> + %15 = "xla_hlo.get_tuple_element"(%arg0) {index = 6 : i32} : (tuple, tensor, tensor<40xf32>, tensor, tensor<74x40xf32>, tensor, tensor<1x10xf32>, tensor<1x10xf32>, tensor<5x1x64xf32>, tensor<5x1x1xf32>, tensor<5x1x1xf32>, tensor<5xi64>, tensor<5x1x10xf32>, tensor<5x1x10xf32>>) -> tensor<1x10xf32> + %cst_2 = constant dense<5.0e-01> : tensor + %16 = "xla_hlo.broadcast_in_dim"(%cst_2) : (tensor) -> tensor<1x10xf32> + %17 = "xla_hlo.broadcast_in_dim"(%cst_2) : (tensor) -> tensor<1x10xf32> + %18 = "xla_hlo.broadcast_in_dim"(%cst_2) : (tensor) -> tensor<1x10xf32> + %19 = "xla_hlo.get_tuple_element"(%arg0) {index = 8 : i32} : (tuple, tensor, tensor<40xf32>, tensor, tensor<74x40xf32>, tensor, tensor<1x10xf32>, tensor<1x10xf32>, tensor<5x1x64xf32>, tensor<5x1x1xf32>, tensor<5x1x1xf32>, tensor<5xi64>, tensor<5x1x10xf32>, tensor<5x1x10xf32>>) -> tensor<5x1x64xf32> + %20 = "xla_hlo.gather"(%19, %0) {collapsed_slice_dims = dense<0> : tensor<1xi64>, index_vector_dim = 0 : i64, offset_dims = dense<[0, 1]> : tensor<2xi64>, slice_sizes = dense<[1, 1, 64]> : tensor<3xi64>, start_index_map = dense<0> : tensor<1xi64>} : (tensor<5x1x64xf32>, tensor) -> tensor<1x64xf32> + %21 = "xla_hlo.get_tuple_element"(%arg0) {index = 7 : i32} : (tuple, tensor, tensor<40xf32>, tensor, tensor<74x40xf32>, tensor, tensor<1x10xf32>, tensor<1x10xf32>, tensor<5x1x64xf32>, tensor<5x1x1xf32>, tensor<5x1x1xf32>, tensor<5xi64>, tensor<5x1x10xf32>, tensor<5x1x10xf32>>) -> tensor<1x10xf32> + %22 = "xla_hlo.concatenate"(%20, %21) {dimension = 1 : i64} : (tensor<1x64xf32>, tensor<1x10xf32>) -> tensor<1x74xf32> + %23 = "xla_hlo.dot"(%22, %5) {precision_config = ["DEFAULT", "DEFAULT"]} : (tensor<1x74xf32>, tensor<74x40xf32>) -> tensor<1x40xf32> + %24 = "xla_hlo.transpose"(%23) {permutation = dense<[0, 1]> : tensor<2xi64>} : (tensor<1x40xf32>) -> tensor<1x40xf32> + %25 = "xla_hlo.reshape"(%3) : (tensor<40xf32>) -> tensor<1x40xf32> + %26 = xla_hlo.add %24, %25 : tensor<1x40xf32> + %27 = "xla_hlo.slice"(%26) {limit_indices = dense<[1, 30]> : tensor<2xi64>, start_indices = dense<[0, 20]> : tensor<2xi64>, strides = dense<1> : tensor<2xi64>} : (tensor<1x40xf32>) -> tensor<1x10xf32> + %28 = xla_hlo.mul %18, %27 : tensor<1x10xf32> + %29 = "xla_hlo.tanh"(%28) : (tensor<1x10xf32>) -> tensor<1x10xf32> + %30 = xla_hlo.mul %17, %29 : tensor<1x10xf32> + %31 = xla_hlo.add %16, %30 : tensor<1x10xf32> + %32 = xla_hlo.mul %31, %15 : tensor<1x10xf32> + %cst_3 = constant dense<5.0e-01> : tensor + %33 = "xla_hlo.broadcast_in_dim"(%cst_3) : (tensor) -> tensor<1x10xf32> + %34 = "xla_hlo.broadcast_in_dim"(%cst_3) : (tensor) -> tensor<1x10xf32> + %35 = "xla_hlo.broadcast_in_dim"(%cst_3) : (tensor) -> tensor<1x10xf32> + %36 = "xla_hlo.slice"(%26) {limit_indices = dense<[1, 20]> : tensor<2xi64>, start_indices = dense<[0, 10]> : tensor<2xi64>, strides = dense<1> : tensor<2xi64>} : (tensor<1x40xf32>) -> tensor<1x10xf32> + %37 = xla_hlo.mul %35, %36 : tensor<1x10xf32> + %38 = "xla_hlo.tanh"(%37) : (tensor<1x10xf32>) -> tensor<1x10xf32> + %39 = xla_hlo.mul %34, %38 : tensor<1x10xf32> + %40 = xla_hlo.add %33, %39 : tensor<1x10xf32> + %41 = "xla_hlo.slice"(%26) {limit_indices = dense<[1, 10]> : tensor<2xi64>, start_indices = dense<0> : tensor<2xi64>, strides = dense<1> : tensor<2xi64>} : (tensor<1x40xf32>) -> tensor<1x10xf32> + %42 = "xla_hlo.tanh"(%41) : (tensor<1x10xf32>) -> tensor<1x10xf32> + %43 = xla_hlo.mul %40, %42 : tensor<1x10xf32> + %44 = xla_hlo.add %32, %43 : tensor<1x10xf32> + %cst_4 = constant dense<1.0e+01> : tensor + %45 = "xla_hlo.broadcast_in_dim"(%cst_4) : (tensor) -> tensor<1x10xf32> + %46 = xla_hlo.min %44, %45 : tensor<1x10xf32> + %cst_5 = constant dense<-1.0e+01> : tensor + %47 = "xla_hlo.broadcast_in_dim"(%cst_5) : (tensor) -> tensor<1x10xf32> + %48 = xla_hlo.max %46, %47 : tensor<1x10xf32> + %49 = "xla_hlo.select"(%14, %15, %48) : (tensor<1x10xi1>, tensor<1x10xf32>, tensor<1x10xf32>) -> tensor<1x10xf32> + %50 = "xla_hlo.reshape"(%8) : (tensor<1x1xf32>) -> tensor<1xf32> + %51 = "xla_hlo.broadcast_in_dim"(%50) {broadcast_dimensions = dense<0> : tensor<1xi64>} : (tensor<1xf32>) -> tensor<1x10xf32> + %cst_6 = constant dense<1.0> : tensor + %52 = "xla_hlo.broadcast_in_dim"(%cst_6) : (tensor) -> tensor<1x10xf32> + %53 = xla_hlo.mul %51, %52 : tensor<1x10xf32> + %cst_7 = constant dense<0.0> : tensor + %54 = "xla_hlo.broadcast_in_dim"(%cst_7) : (tensor) -> tensor<1x10xf32> + %55 = "xla_hlo.compare"(%53, %54) {comparison_direction = "GT"} : (tensor<1x10xf32>, tensor<1x10xf32>) -> tensor<1x10xi1> + %cst_8 = constant dense<5.0e-01> : tensor + %56 = "xla_hlo.broadcast_in_dim"(%cst_8) : (tensor) -> tensor<1x10xf32> + %57 = "xla_hlo.broadcast_in_dim"(%cst_8) : (tensor) -> tensor<1x10xf32> + %58 = "xla_hlo.broadcast_in_dim"(%cst_8) : (tensor) -> tensor<1x10xf32> + %59 = "xla_hlo.slice"(%26) {limit_indices = dense<[1, 40]> : tensor<2xi64>, start_indices = dense<[0, 30]> : tensor<2xi64>, strides = dense<1> : tensor<2xi64>} : (tensor<1x40xf32>) -> tensor<1x10xf32> + %60 = xla_hlo.mul %58, %59 : tensor<1x10xf32> + %61 = "xla_hlo.tanh"(%60) : (tensor<1x10xf32>) -> tensor<1x10xf32> + %62 = xla_hlo.mul %57, %61 : tensor<1x10xf32> + %63 = xla_hlo.add %56, %62 : tensor<1x10xf32> + %64 = "xla_hlo.tanh"(%48) : (tensor<1x10xf32>) -> tensor<1x10xf32> + %65 = xla_hlo.mul %63, %64 : tensor<1x10xf32> + %66 = "xla_hlo.select"(%55, %21, %65) : (tensor<1x10xi1>, tensor<1x10xf32>, tensor<1x10xf32>) -> tensor<1x10xf32> + %67 = "xla_hlo.get_tuple_element"(%arg0) {index = 10 : i32} : (tuple, tensor, tensor<40xf32>, tensor, tensor<74x40xf32>, tensor, tensor<1x10xf32>, tensor<1x10xf32>, tensor<5x1x64xf32>, tensor<5x1x1xf32>, tensor<5x1x1xf32>, tensor<5xi64>, tensor<5x1x10xf32>, tensor<5x1x10xf32>>) -> tensor<5x1x1xf32> + %68 = "xla_hlo.get_tuple_element"(%arg0) {index = 11 : i32} : (tuple, tensor, tensor<40xf32>, tensor, tensor<74x40xf32>, tensor, tensor<1x10xf32>, tensor<1x10xf32>, tensor<5x1x64xf32>, tensor<5x1x1xf32>, tensor<5x1x1xf32>, tensor<5xi64>, tensor<5x1x10xf32>, tensor<5x1x10xf32>>) -> tensor<5xi64> + %69 = "xla_hlo.reshape"(%6) : (tensor) -> tensor<1xi64> + %70 = "xla_hlo.slice"(%69) {limit_indices = dense<1> : tensor<1xi64>, start_indices = dense<0> : tensor<1xi64>, strides = dense<1> : tensor<1xi64>} : (tensor<1xi64>) -> tensor<1xi64> + %71 = "xla_hlo.reshape"(%0) : (tensor) -> tensor<1xi64> + %72 = "xla_hlo.concatenate"(%71) {dimension = 0 : i64} : (tensor<1xi64>) -> tensor<1xi64> + %73 = "xla_hlo.convert"(%72) : (tensor<1xi64>) -> tensor<1xi32> + %74 = "xla_hlo.slice"(%73) {limit_indices = dense<1> : tensor<1xi64>, start_indices = dense<0> : tensor<1xi64>, strides = dense<1> : tensor<1xi64>} : (tensor<1xi32>) -> tensor<1xi32> + %75 = "xla_hlo.reshape"(%74) : (tensor<1xi32>) -> tensor + %76 = "xla_hlo.dynamic-update-slice"(%68, %70, %75) : (tensor<5xi64>, tensor<1xi64>, tensor) -> tensor<5xi64> + %77 = "xla_hlo.get_tuple_element"(%arg0) {index = 12 : i32} : (tuple, tensor, tensor<40xf32>, tensor, tensor<74x40xf32>, tensor, tensor<1x10xf32>, tensor<1x10xf32>, tensor<5x1x64xf32>, tensor<5x1x1xf32>, tensor<5x1x1xf32>, tensor<5xi64>, tensor<5x1x10xf32>, tensor<5x1x10xf32>>) -> tensor<5x1x10xf32> + %78 = "xla_hlo.reshape"(%49) : (tensor<1x10xf32>) -> tensor<1x1x10xf32> + %79 = "xla_hlo.slice"(%78) {limit_indices = dense<[1, 1, 10]> : tensor<3xi64>, start_indices = dense<0> : tensor<3xi64>, strides = dense<1> : tensor<3xi64>} : (tensor<1x1x10xf32>) -> tensor<1x1x10xf32> + %80 = "xla_hlo.slice"(%73) {limit_indices = dense<1> : tensor<1xi64>, start_indices = dense<0> : tensor<1xi64>, strides = dense<1> : tensor<1xi64>} : (tensor<1xi32>) -> tensor<1xi32> + %81 = "xla_hlo.reshape"(%80) : (tensor<1xi32>) -> tensor + %cst_9 = constant dense<0> : tensor + %82 = "xla_hlo.dynamic-update-slice"(%77, %79, %81, %cst_9, %cst_9) : (tensor<5x1x10xf32>, tensor<1x1x10xf32>, tensor, tensor, tensor) -> tensor<5x1x10xf32> + %83 = "xla_hlo.get_tuple_element"(%arg0) {index = 13 : i32} : (tuple, tensor, tensor<40xf32>, tensor, tensor<74x40xf32>, tensor, tensor<1x10xf32>, tensor<1x10xf32>, tensor<5x1x64xf32>, tensor<5x1x1xf32>, tensor<5x1x1xf32>, tensor<5xi64>, tensor<5x1x10xf32>, tensor<5x1x10xf32>>) -> tensor<5x1x10xf32> + %84 = "xla_hlo.reshape"(%66) : (tensor<1x10xf32>) -> tensor<1x1x10xf32> + %85 = "xla_hlo.slice"(%84) {limit_indices = dense<[1, 1, 10]> : tensor<3xi64>, start_indices = dense<0> : tensor<3xi64>, strides = dense<1> : tensor<3xi64>} : (tensor<1x1x10xf32>) -> tensor<1x1x10xf32> + %86 = "xla_hlo.slice"(%73) {limit_indices = dense<1> : tensor<1xi64>, start_indices = dense<0> : tensor<1xi64>, strides = dense<1> : tensor<1xi64>} : (tensor<1xi32>) -> tensor<1xi32> + %87 = "xla_hlo.reshape"(%86) : (tensor<1xi32>) -> tensor + %cst_10 = constant dense<0> : tensor + %88 = "xla_hlo.dynamic-update-slice"(%83, %85, %87, %cst_10, %cst_10) : (tensor<5x1x10xf32>, tensor<1x1x10xf32>, tensor, tensor, tensor) -> tensor<5x1x10xf32> + %89 = "xla_hlo.tuple"(%1, %2, %3, %4, %5, %6, %49, %66, %19, %7, %67, %76, %82, %88) : (tensor, tensor, tensor<40xf32>, tensor, tensor<74x40xf32>, tensor, tensor<1x10xf32>, tensor<1x10xf32>, tensor<5x1x64xf32>, tensor<5x1x1xf32>, tensor<5x1x1xf32>, tensor<5xi64>, tensor<5x1x10xf32>, tensor<5x1x10xf32>) -> tuple, tensor, tensor<40xf32>, tensor, tensor<74x40xf32>, tensor, tensor<1x10xf32>, tensor<1x10xf32>, tensor<5x1x64xf32>, tensor<5x1x1xf32>, tensor<5x1x1xf32>, tensor<5xi64>, tensor<5x1x10xf32>, tensor<5x1x10xf32>> + return %89 : tuple, tensor, tensor<40xf32>, tensor, tensor<74x40xf32>, tensor, tensor<1x10xf32>, tensor<1x10xf32>, tensor<5x1x64xf32>, tensor<5x1x1xf32>, tensor<5x1x1xf32>, tensor<5xi64>, tensor<5x1x10xf32>, tensor<5x1x10xf32>> +} +func @ForwardLoopCond_gFAnjWGSoLs__.167(%arg0: tuple, tensor, tensor<40xf32>, tensor, tensor<74x40xf32>, tensor, tensor<1x10xf32>, tensor<1x10xf32>, tensor<5x1x64xf32>, tensor<5x1x1xf32>, tensor<5x1x1xf32>, tensor<5xi64>, tensor<5x1x10xf32>, tensor<5x1x10xf32>>) -> tuple> { + %0 = "xla_hlo.get_tuple_element"(%arg0) {index = 0 : i32} : (tuple, tensor, tensor<40xf32>, tensor, tensor<74x40xf32>, tensor, tensor<1x10xf32>, tensor<1x10xf32>, tensor<5x1x64xf32>, tensor<5x1x1xf32>, tensor<5x1x1xf32>, tensor<5xi64>, tensor<5x1x10xf32>, tensor<5x1x10xf32>>) -> tensor + %1 = "xla_hlo.get_tuple_element"(%arg0) {index = 1 : i32} : (tuple, tensor, tensor<40xf32>, tensor, tensor<74x40xf32>, tensor, tensor<1x10xf32>, tensor<1x10xf32>, tensor<5x1x64xf32>, tensor<5x1x1xf32>, tensor<5x1x1xf32>, tensor<5xi64>, tensor<5x1x10xf32>, tensor<5x1x10xf32>>) -> tensor + %2 = "xla_hlo.compare"(%0, %1) {comparison_direction = "LT"} : (tensor, tensor) -> tensor + %3 = "xla_hlo.tuple"(%2) : (tensor) -> tuple> + return %3 : tuple> +} +func @cond_wrapper.185(%arg0: tuple, tensor, tensor<40xf32>, tensor, tensor<74x40xf32>, tensor, tensor<1x10xf32>, tensor<1x10xf32>, tensor<5x1x64xf32>, tensor<5x1x1xf32>, tensor<5x1x1xf32>, tensor<5xi64>, tensor<5x1x10xf32>, tensor<5x1x10xf32>>) -> tensor { + %0 = call @ForwardLoopCond_gFAnjWGSoLs__.167(%arg0) : (tuple, tensor, tensor<40xf32>, tensor, tensor<74x40xf32>, tensor, tensor<1x10xf32>, tensor<1x10xf32>, tensor<5x1x64xf32>, tensor<5x1x1xf32>, tensor<5x1x1xf32>, tensor<5xi64>, tensor<5x1x10xf32>, tensor<5x1x10xf32>>) -> tuple> + %1 = "xla_hlo.get_tuple_element"(%0) {index = 0 : i32} : (tuple>) -> tensor + return %1 : tensor +} +func @Forward_DIZIAkooG44__disable_call_shape_inference_true_.189(%arg0: tensor<1x10xf32>, %arg1: tensor<1x10xf32>, %arg2: tensor<5x1x64xf32>, %arg3: tensor<5x1x1xf32>, %arg4: tensor<5x1x1xf32>) -> tuple, tensor<5xi64>, tensor<5x1x10xf32>, tensor<5x1x10xf32>, tensor, tensor<1x10xf32>, tensor<1x10xf32>> { + %cst = constant dense<5> : tensor + %0 = "xla_hlo.convert"(%arg3) : (tensor<5x1x1xf32>) -> tensor<5x1x1xf32> + %cst_0 = constant dense<0x7F800000> : tensor + %1 = "xla_hlo.convert"(%cst_0) : (tensor) -> tensor + %2 = "xla_hlo.reduce"(%0, %1) ( { + ^bb0(%arg5: tensor, %arg6: tensor): // no predecessors + %42 = xla_hlo.min %arg5, %arg6 : tensor + "xla_hlo.return"(%42) : (tensor) -> () + }) {dimensions = dense<[1, 2]> : tensor<2xi64>} : (tensor<5x1x1xf32>, tensor) -> tensor<5xf32> + %3 = "xla_hlo.convert"(%2) : (tensor<5xf32>) -> tensor<5xf32> + %cst_1 = constant dense<0.0> : tensor + %4 = "xla_hlo.broadcast_in_dim"(%cst_1) : (tensor) -> tensor<5xf32> + %5 = "xla_hlo.compare"(%3, %4) {comparison_direction = "EQ"} : (tensor<5xf32>, tensor<5xf32>) -> tensor<5xi1> + %6 = "xla_hlo.convert"(%5) : (tensor<5xi1>) -> tensor<5xi32> + %cst_2 = constant dense<[1, 2, 3, 4, 5]> : tensor<5xi32> + %7 = xla_hlo.mul %6, %cst_2 : tensor<5xi32> + %8 = "xla_hlo.convert"(%7) : (tensor<5xi32>) -> tensor<5xi32> + %cst_3 = constant dense<-2147483648> : tensor + %9 = "xla_hlo.convert"(%cst_3) : (tensor) -> tensor + %10 = "xla_hlo.reduce"(%8, %9) ( { + ^bb0(%arg5: tensor, %arg6: tensor): // no predecessors + %42 = xla_hlo.max %arg5, %arg6 : tensor + "xla_hlo.return"(%42) : (tensor) -> () + }) {dimensions = dense<0> : tensor<1xi64>} : (tensor<5xi32>, tensor) -> tensor + %11 = "xla_hlo.convert"(%10) : (tensor) -> tensor + %12 = xla_hlo.sub %cst, %11 : tensor + %cst_4 = constant dense<5> : tensor + %13 = "xla_hlo.compare"(%12, %cst_4) {comparison_direction = "EQ"} : (tensor, tensor) -> tensor + %cst_5 = constant dense<0> : tensor + %cst_6 = constant dense<5> : tensor + %14 = "xla_hlo.reverse"(%3) {dimensions = dense<0> : tensor<1xi64>} : (tensor<5xf32>) -> tensor<5xf32> + %cst_7 = constant dense<0.0> : tensor + %15 = "xla_hlo.broadcast_in_dim"(%cst_7) : (tensor) -> tensor<5xf32> + %16 = "xla_hlo.compare"(%14, %15) {comparison_direction = "EQ"} : (tensor<5xf32>, tensor<5xf32>) -> tensor<5xi1> + %17 = "xla_hlo.convert"(%16) : (tensor<5xi1>) -> tensor<5xi32> + %cst_8 = constant dense<[1, 2, 3, 4, 5]> : tensor<5xi32> + %18 = xla_hlo.mul %17, %cst_8 : tensor<5xi32> + %19 = "xla_hlo.convert"(%18) : (tensor<5xi32>) -> tensor<5xi32> + %cst_9 = constant dense<-2147483648> : tensor + %20 = "xla_hlo.convert"(%cst_9) : (tensor) -> tensor + %21 = "xla_hlo.reduce"(%19, %20) ( { + ^bb0(%arg5: tensor, %arg6: tensor): // no predecessors + %42 = xla_hlo.max %arg5, %arg6 : tensor + "xla_hlo.return"(%42) : (tensor) -> () + }) {dimensions = dense<0> : tensor<1xi64>} : (tensor<5xi32>, tensor) -> tensor + %22 = "xla_hlo.convert"(%21) : (tensor) -> tensor + %23 = xla_hlo.sub %cst_6, %22 : tensor + %24 = "xla_hlo.select"(%13, %cst_5, %23) : (tensor, tensor, tensor) -> tensor + %25 = "xla_hlo.convert"(%24) : (tensor) -> tensor + %cst_10 = constant dense<5> : tensor + %26 = xla_hlo.sub %cst_10, %12 : tensor + %27 = "xla_hlo.convert"(%26) : (tensor) -> tensor + %cst_11 = constant dense<0.0> : tensor + %28 = "xla_hlo.broadcast_in_dim"(%cst_11) : (tensor) -> tensor<40xf32> + %cst_12 = constant dense<0> : tensor + %cst_13 = constant dense<0.42> : tensor<74x40xf32> + %cst_14 = constant dense<0> : tensor + %cst_15 = constant dense<0> : tensor + %29 = "xla_hlo.broadcast_in_dim"(%cst_15) : (tensor) -> tensor<5xi64> + %cst_16 = constant dense<0.0> : tensor + %30 = "xla_hlo.broadcast_in_dim"(%cst_16) : (tensor) -> tensor<5x1x10xf32> + %cst_17 = constant dense<0.0> : tensor + %31 = "xla_hlo.broadcast_in_dim"(%cst_17) : (tensor) -> tensor<5x1x10xf32> + %32 = "xla_hlo.tuple"(%25, %27, %28, %cst_12, %cst_13, %cst_14, %arg0, %arg1, %arg2, %arg3, %arg4, %29, %30, %31) : (tensor, tensor, tensor<40xf32>, tensor, tensor<74x40xf32>, tensor, tensor<1x10xf32>, tensor<1x10xf32>, tensor<5x1x64xf32>, tensor<5x1x1xf32>, tensor<5x1x1xf32>, tensor<5xi64>, tensor<5x1x10xf32>, tensor<5x1x10xf32>) -> tuple, tensor, tensor<40xf32>, tensor, tensor<74x40xf32>, tensor, tensor<1x10xf32>, tensor<1x10xf32>, tensor<5x1x64xf32>, tensor<5x1x1xf32>, tensor<5x1x1xf32>, tensor<5xi64>, tensor<5x1x10xf32>, tensor<5x1x10xf32>> + %33 = "xla_hlo.while"(%32) {body = @ForwardLoopBody_o0Jnom3Cdxo__.59, cond = @cond_wrapper.185} : (tuple, tensor, tensor<40xf32>, tensor, tensor<74x40xf32>, tensor, tensor<1x10xf32>, tensor<1x10xf32>, tensor<5x1x64xf32>, tensor<5x1x1xf32>, tensor<5x1x1xf32>, tensor<5xi64>, tensor<5x1x10xf32>, tensor<5x1x10xf32>>) -> tuple, tensor, tensor<40xf32>, tensor, tensor<74x40xf32>, tensor, tensor<1x10xf32>, tensor<1x10xf32>, tensor<5x1x64xf32>, tensor<5x1x1xf32>, tensor<5x1x1xf32>, tensor<5xi64>, tensor<5x1x10xf32>, tensor<5x1x10xf32>> + %34 = "xla_hlo.get_tuple_element"(%33) {index = 0 : i32} : (tuple, tensor, tensor<40xf32>, tensor, tensor<74x40xf32>, tensor, tensor<1x10xf32>, tensor<1x10xf32>, tensor<5x1x64xf32>, tensor<5x1x1xf32>, tensor<5x1x1xf32>, tensor<5xi64>, tensor<5x1x10xf32>, tensor<5x1x10xf32>>) -> tensor + %35 = "xla_hlo.get_tuple_element"(%33) {index = 11 : i32} : (tuple, tensor, tensor<40xf32>, tensor, tensor<74x40xf32>, tensor, tensor<1x10xf32>, tensor<1x10xf32>, tensor<5x1x64xf32>, tensor<5x1x1xf32>, tensor<5x1x1xf32>, tensor<5xi64>, tensor<5x1x10xf32>, tensor<5x1x10xf32>>) -> tensor<5xi64> + %36 = "xla_hlo.get_tuple_element"(%33) {index = 12 : i32} : (tuple, tensor, tensor<40xf32>, tensor, tensor<74x40xf32>, tensor, tensor<1x10xf32>, tensor<1x10xf32>, tensor<5x1x64xf32>, tensor<5x1x1xf32>, tensor<5x1x1xf32>, tensor<5xi64>, tensor<5x1x10xf32>, tensor<5x1x10xf32>>) -> tensor<5x1x10xf32> + %37 = "xla_hlo.get_tuple_element"(%33) {index = 13 : i32} : (tuple, tensor, tensor<40xf32>, tensor, tensor<74x40xf32>, tensor, tensor<1x10xf32>, tensor<1x10xf32>, tensor<5x1x64xf32>, tensor<5x1x1xf32>, tensor<5x1x1xf32>, tensor<5xi64>, tensor<5x1x10xf32>, tensor<5x1x10xf32>>) -> tensor<5x1x10xf32> + %38 = "xla_hlo.get_tuple_element"(%33) {index = 5 : i32} : (tuple, tensor, tensor<40xf32>, tensor, tensor<74x40xf32>, tensor, tensor<1x10xf32>, tensor<1x10xf32>, tensor<5x1x64xf32>, tensor<5x1x1xf32>, tensor<5x1x1xf32>, tensor<5xi64>, tensor<5x1x10xf32>, tensor<5x1x10xf32>>) -> tensor + %39 = "xla_hlo.get_tuple_element"(%33) {index = 6 : i32} : (tuple, tensor, tensor<40xf32>, tensor, tensor<74x40xf32>, tensor, tensor<1x10xf32>, tensor<1x10xf32>, tensor<5x1x64xf32>, tensor<5x1x1xf32>, tensor<5x1x1xf32>, tensor<5xi64>, tensor<5x1x10xf32>, tensor<5x1x10xf32>>) -> tensor<1x10xf32> + %40 = "xla_hlo.get_tuple_element"(%33) {index = 7 : i32} : (tuple, tensor, tensor<40xf32>, tensor, tensor<74x40xf32>, tensor, tensor<1x10xf32>, tensor<1x10xf32>, tensor<5x1x64xf32>, tensor<5x1x1xf32>, tensor<5x1x1xf32>, tensor<5xi64>, tensor<5x1x10xf32>, tensor<5x1x10xf32>>) -> tensor<1x10xf32> + %41 = "xla_hlo.tuple"(%34, %35, %36, %37, %38, %39, %40) : (tensor, tensor<5xi64>, tensor<5x1x10xf32>, tensor<5x1x10xf32>, tensor, tensor<1x10xf32>, tensor<1x10xf32>) -> tuple, tensor<5xi64>, tensor<5x1x10xf32>, tensor<5x1x10xf32>, tensor, tensor<1x10xf32>, tensor<1x10xf32>> + return %41 : tuple, tensor<5xi64>, tensor<5x1x10xf32>, tensor<5x1x10xf32>, tensor, tensor<1x10xf32>, tensor<1x10xf32>> +} +func @Min_reduction.316(%arg0: tensor, %arg1: tensor) -> tensor { + %0 = xla_hlo.min %arg0, %arg1 : tensor + return %0 : tensor +} +func @Max_reduction.320(%arg0: tensor, %arg1: tensor) -> tensor { + %0 = xla_hlo.max %arg0, %arg1 : tensor + return %0 : tensor +} +func @Max_1_reduction.324(%arg0: tensor, %arg1: tensor) -> tensor { + %0 = xla_hlo.max %arg0, %arg1 : tensor + return %0 : tensor +} +func @ForwardLoopBody_o0Jnom3Cdxo__.328(%arg0: tuple, tensor, tensor<40xf32>, tensor, tensor<20x40xf32>, tensor, tensor<1x10xf32>, tensor<1x10xf32>, tensor<5x1x10xf32>, tensor<5x1x1xf32>, tensor<5x1x1xf32>, tensor<5xi64>, tensor<5x1x10xf32>, tensor<5x1x10xf32>>) -> tuple, tensor, tensor<40xf32>, tensor, tensor<20x40xf32>, tensor, tensor<1x10xf32>, tensor<1x10xf32>, tensor<5x1x10xf32>, tensor<5x1x1xf32>, tensor<5x1x1xf32>, tensor<5xi64>, tensor<5x1x10xf32>, tensor<5x1x10xf32>> { + %0 = "xla_hlo.get_tuple_element"(%arg0) {index = 0 : i32} : (tuple, tensor, tensor<40xf32>, tensor, tensor<20x40xf32>, tensor, tensor<1x10xf32>, tensor<1x10xf32>, tensor<5x1x10xf32>, tensor<5x1x1xf32>, tensor<5x1x1xf32>, tensor<5xi64>, tensor<5x1x10xf32>, tensor<5x1x10xf32>>) -> tensor + %cst = constant dense<1> : tensor + %1 = xla_hlo.add %0, %cst : tensor + %2 = "xla_hlo.get_tuple_element"(%arg0) {index = 1 : i32} : (tuple, tensor, tensor<40xf32>, tensor, tensor<20x40xf32>, tensor, tensor<1x10xf32>, tensor<1x10xf32>, tensor<5x1x10xf32>, tensor<5x1x1xf32>, tensor<5x1x1xf32>, tensor<5xi64>, tensor<5x1x10xf32>, tensor<5x1x10xf32>>) -> tensor + %3 = "xla_hlo.get_tuple_element"(%arg0) {index = 2 : i32} : (tuple, tensor, tensor<40xf32>, tensor, tensor<20x40xf32>, tensor, tensor<1x10xf32>, tensor<1x10xf32>, tensor<5x1x10xf32>, tensor<5x1x1xf32>, tensor<5x1x1xf32>, tensor<5xi64>, tensor<5x1x10xf32>, tensor<5x1x10xf32>>) -> tensor<40xf32> + %4 = "xla_hlo.get_tuple_element"(%arg0) {index = 3 : i32} : (tuple, tensor, tensor<40xf32>, tensor, tensor<20x40xf32>, tensor, tensor<1x10xf32>, tensor<1x10xf32>, tensor<5x1x10xf32>, tensor<5x1x1xf32>, tensor<5x1x1xf32>, tensor<5xi64>, tensor<5x1x10xf32>, tensor<5x1x10xf32>>) -> tensor + %5 = "xla_hlo.get_tuple_element"(%arg0) {index = 4 : i32} : (tuple, tensor, tensor<40xf32>, tensor, tensor<20x40xf32>, tensor, tensor<1x10xf32>, tensor<1x10xf32>, tensor<5x1x10xf32>, tensor<5x1x1xf32>, tensor<5x1x1xf32>, tensor<5xi64>, tensor<5x1x10xf32>, tensor<5x1x10xf32>>) -> tensor<20x40xf32> + %6 = "xla_hlo.get_tuple_element"(%arg0) {index = 5 : i32} : (tuple, tensor, tensor<40xf32>, tensor, tensor<20x40xf32>, tensor, tensor<1x10xf32>, tensor<1x10xf32>, tensor<5x1x10xf32>, tensor<5x1x1xf32>, tensor<5x1x1xf32>, tensor<5xi64>, tensor<5x1x10xf32>, tensor<5x1x10xf32>>) -> tensor + %7 = "xla_hlo.get_tuple_element"(%arg0) {index = 9 : i32} : (tuple, tensor, tensor<40xf32>, tensor, tensor<20x40xf32>, tensor, tensor<1x10xf32>, tensor<1x10xf32>, tensor<5x1x10xf32>, tensor<5x1x1xf32>, tensor<5x1x1xf32>, tensor<5xi64>, tensor<5x1x10xf32>, tensor<5x1x10xf32>>) -> tensor<5x1x1xf32> + %8 = "xla_hlo.gather"(%7, %0) {collapsed_slice_dims = dense<0> : tensor<1xi64>, index_vector_dim = 0 : i64, offset_dims = dense<[0, 1]> : tensor<2xi64>, slice_sizes = dense<1> : tensor<3xi64>, start_index_map = dense<0> : tensor<1xi64>} : (tensor<5x1x1xf32>, tensor) -> tensor<1x1xf32> + %9 = "xla_hlo.reshape"(%8) : (tensor<1x1xf32>) -> tensor<1xf32> + %10 = "xla_hlo.broadcast_in_dim"(%9) {broadcast_dimensions = dense<0> : tensor<1xi64>} : (tensor<1xf32>) -> tensor<1x10xf32> + %cst_0 = constant dense<1.0> : tensor + %11 = "xla_hlo.broadcast_in_dim"(%cst_0) : (tensor) -> tensor<1x10xf32> + %12 = xla_hlo.mul %10, %11 : tensor<1x10xf32> + %cst_1 = constant dense<0.0> : tensor + %13 = "xla_hlo.broadcast_in_dim"(%cst_1) : (tensor) -> tensor<1x10xf32> + %14 = "xla_hlo.compare"(%12, %13) {comparison_direction = "GT"} : (tensor<1x10xf32>, tensor<1x10xf32>) -> tensor<1x10xi1> + %15 = "xla_hlo.get_tuple_element"(%arg0) {index = 6 : i32} : (tuple, tensor, tensor<40xf32>, tensor, tensor<20x40xf32>, tensor, tensor<1x10xf32>, tensor<1x10xf32>, tensor<5x1x10xf32>, tensor<5x1x1xf32>, tensor<5x1x1xf32>, tensor<5xi64>, tensor<5x1x10xf32>, tensor<5x1x10xf32>>) -> tensor<1x10xf32> + %cst_2 = constant dense<5.0e-01> : tensor + %16 = "xla_hlo.broadcast_in_dim"(%cst_2) : (tensor) -> tensor<1x10xf32> + %17 = "xla_hlo.broadcast_in_dim"(%cst_2) : (tensor) -> tensor<1x10xf32> + %18 = "xla_hlo.broadcast_in_dim"(%cst_2) : (tensor) -> tensor<1x10xf32> + %19 = "xla_hlo.get_tuple_element"(%arg0) {index = 8 : i32} : (tuple, tensor, tensor<40xf32>, tensor, tensor<20x40xf32>, tensor, tensor<1x10xf32>, tensor<1x10xf32>, tensor<5x1x10xf32>, tensor<5x1x1xf32>, tensor<5x1x1xf32>, tensor<5xi64>, tensor<5x1x10xf32>, tensor<5x1x10xf32>>) -> tensor<5x1x10xf32> + %20 = "xla_hlo.gather"(%19, %0) {collapsed_slice_dims = dense<0> : tensor<1xi64>, index_vector_dim = 0 : i64, offset_dims = dense<[0, 1]> : tensor<2xi64>, slice_sizes = dense<[1, 1, 10]> : tensor<3xi64>, start_index_map = dense<0> : tensor<1xi64>} : (tensor<5x1x10xf32>, tensor) -> tensor<1x10xf32> + %21 = "xla_hlo.get_tuple_element"(%arg0) {index = 7 : i32} : (tuple, tensor, tensor<40xf32>, tensor, tensor<20x40xf32>, tensor, tensor<1x10xf32>, tensor<1x10xf32>, tensor<5x1x10xf32>, tensor<5x1x1xf32>, tensor<5x1x1xf32>, tensor<5xi64>, tensor<5x1x10xf32>, tensor<5x1x10xf32>>) -> tensor<1x10xf32> + %22 = "xla_hlo.concatenate"(%20, %21) {dimension = 1 : i64} : (tensor<1x10xf32>, tensor<1x10xf32>) -> tensor<1x20xf32> + %23 = "xla_hlo.dot"(%22, %5) {precision_config = ["DEFAULT", "DEFAULT"]} : (tensor<1x20xf32>, tensor<20x40xf32>) -> tensor<1x40xf32> + %24 = "xla_hlo.transpose"(%23) {permutation = dense<[0, 1]> : tensor<2xi64>} : (tensor<1x40xf32>) -> tensor<1x40xf32> + %25 = "xla_hlo.reshape"(%3) : (tensor<40xf32>) -> tensor<1x40xf32> + %26 = xla_hlo.add %24, %25 : tensor<1x40xf32> + %27 = "xla_hlo.slice"(%26) {limit_indices = dense<[1, 30]> : tensor<2xi64>, start_indices = dense<[0, 20]> : tensor<2xi64>, strides = dense<1> : tensor<2xi64>} : (tensor<1x40xf32>) -> tensor<1x10xf32> + %28 = xla_hlo.mul %18, %27 : tensor<1x10xf32> + %29 = "xla_hlo.tanh"(%28) : (tensor<1x10xf32>) -> tensor<1x10xf32> + %30 = xla_hlo.mul %17, %29 : tensor<1x10xf32> + %31 = xla_hlo.add %16, %30 : tensor<1x10xf32> + %32 = xla_hlo.mul %31, %15 : tensor<1x10xf32> + %cst_3 = constant dense<5.0e-01> : tensor + %33 = "xla_hlo.broadcast_in_dim"(%cst_3) : (tensor) -> tensor<1x10xf32> + %34 = "xla_hlo.broadcast_in_dim"(%cst_3) : (tensor) -> tensor<1x10xf32> + %35 = "xla_hlo.broadcast_in_dim"(%cst_3) : (tensor) -> tensor<1x10xf32> + %36 = "xla_hlo.slice"(%26) {limit_indices = dense<[1, 20]> : tensor<2xi64>, start_indices = dense<[0, 10]> : tensor<2xi64>, strides = dense<1> : tensor<2xi64>} : (tensor<1x40xf32>) -> tensor<1x10xf32> + %37 = xla_hlo.mul %35, %36 : tensor<1x10xf32> + %38 = "xla_hlo.tanh"(%37) : (tensor<1x10xf32>) -> tensor<1x10xf32> + %39 = xla_hlo.mul %34, %38 : tensor<1x10xf32> + %40 = xla_hlo.add %33, %39 : tensor<1x10xf32> + %41 = "xla_hlo.slice"(%26) {limit_indices = dense<[1, 10]> : tensor<2xi64>, start_indices = dense<0> : tensor<2xi64>, strides = dense<1> : tensor<2xi64>} : (tensor<1x40xf32>) -> tensor<1x10xf32> + %42 = "xla_hlo.tanh"(%41) : (tensor<1x10xf32>) -> tensor<1x10xf32> + %43 = xla_hlo.mul %40, %42 : tensor<1x10xf32> + %44 = xla_hlo.add %32, %43 : tensor<1x10xf32> + %cst_4 = constant dense<1.0e+01> : tensor + %45 = "xla_hlo.broadcast_in_dim"(%cst_4) : (tensor) -> tensor<1x10xf32> + %46 = xla_hlo.min %44, %45 : tensor<1x10xf32> + %cst_5 = constant dense<-1.0e+01> : tensor + %47 = "xla_hlo.broadcast_in_dim"(%cst_5) : (tensor) -> tensor<1x10xf32> + %48 = xla_hlo.max %46, %47 : tensor<1x10xf32> + %49 = "xla_hlo.select"(%14, %15, %48) : (tensor<1x10xi1>, tensor<1x10xf32>, tensor<1x10xf32>) -> tensor<1x10xf32> + %50 = "xla_hlo.reshape"(%8) : (tensor<1x1xf32>) -> tensor<1xf32> + %51 = "xla_hlo.broadcast_in_dim"(%50) {broadcast_dimensions = dense<0> : tensor<1xi64>} : (tensor<1xf32>) -> tensor<1x10xf32> + %cst_6 = constant dense<1.0> : tensor + %52 = "xla_hlo.broadcast_in_dim"(%cst_6) : (tensor) -> tensor<1x10xf32> + %53 = xla_hlo.mul %51, %52 : tensor<1x10xf32> + %cst_7 = constant dense<0.0> : tensor + %54 = "xla_hlo.broadcast_in_dim"(%cst_7) : (tensor) -> tensor<1x10xf32> + %55 = "xla_hlo.compare"(%53, %54) {comparison_direction = "GT"} : (tensor<1x10xf32>, tensor<1x10xf32>) -> tensor<1x10xi1> + %cst_8 = constant dense<5.0e-01> : tensor + %56 = "xla_hlo.broadcast_in_dim"(%cst_8) : (tensor) -> tensor<1x10xf32> + %57 = "xla_hlo.broadcast_in_dim"(%cst_8) : (tensor) -> tensor<1x10xf32> + %58 = "xla_hlo.broadcast_in_dim"(%cst_8) : (tensor) -> tensor<1x10xf32> + %59 = "xla_hlo.slice"(%26) {limit_indices = dense<[1, 40]> : tensor<2xi64>, start_indices = dense<[0, 30]> : tensor<2xi64>, strides = dense<1> : tensor<2xi64>} : (tensor<1x40xf32>) -> tensor<1x10xf32> + %60 = xla_hlo.mul %58, %59 : tensor<1x10xf32> + %61 = "xla_hlo.tanh"(%60) : (tensor<1x10xf32>) -> tensor<1x10xf32> + %62 = xla_hlo.mul %57, %61 : tensor<1x10xf32> + %63 = xla_hlo.add %56, %62 : tensor<1x10xf32> + %64 = "xla_hlo.tanh"(%48) : (tensor<1x10xf32>) -> tensor<1x10xf32> + %65 = xla_hlo.mul %63, %64 : tensor<1x10xf32> + %66 = "xla_hlo.select"(%55, %21, %65) : (tensor<1x10xi1>, tensor<1x10xf32>, tensor<1x10xf32>) -> tensor<1x10xf32> + %67 = "xla_hlo.get_tuple_element"(%arg0) {index = 10 : i32} : (tuple, tensor, tensor<40xf32>, tensor, tensor<20x40xf32>, tensor, tensor<1x10xf32>, tensor<1x10xf32>, tensor<5x1x10xf32>, tensor<5x1x1xf32>, tensor<5x1x1xf32>, tensor<5xi64>, tensor<5x1x10xf32>, tensor<5x1x10xf32>>) -> tensor<5x1x1xf32> + %68 = "xla_hlo.get_tuple_element"(%arg0) {index = 11 : i32} : (tuple, tensor, tensor<40xf32>, tensor, tensor<20x40xf32>, tensor, tensor<1x10xf32>, tensor<1x10xf32>, tensor<5x1x10xf32>, tensor<5x1x1xf32>, tensor<5x1x1xf32>, tensor<5xi64>, tensor<5x1x10xf32>, tensor<5x1x10xf32>>) -> tensor<5xi64> + %69 = "xla_hlo.reshape"(%6) : (tensor) -> tensor<1xi64> + %70 = "xla_hlo.slice"(%69) {limit_indices = dense<1> : tensor<1xi64>, start_indices = dense<0> : tensor<1xi64>, strides = dense<1> : tensor<1xi64>} : (tensor<1xi64>) -> tensor<1xi64> + %71 = "xla_hlo.reshape"(%0) : (tensor) -> tensor<1xi64> + %72 = "xla_hlo.concatenate"(%71) {dimension = 0 : i64} : (tensor<1xi64>) -> tensor<1xi64> + %73 = "xla_hlo.convert"(%72) : (tensor<1xi64>) -> tensor<1xi32> + %74 = "xla_hlo.slice"(%73) {limit_indices = dense<1> : tensor<1xi64>, start_indices = dense<0> : tensor<1xi64>, strides = dense<1> : tensor<1xi64>} : (tensor<1xi32>) -> tensor<1xi32> + %75 = "xla_hlo.reshape"(%74) : (tensor<1xi32>) -> tensor + %76 = "xla_hlo.dynamic-update-slice"(%68, %70, %75) : (tensor<5xi64>, tensor<1xi64>, tensor) -> tensor<5xi64> + %77 = "xla_hlo.get_tuple_element"(%arg0) {index = 12 : i32} : (tuple, tensor, tensor<40xf32>, tensor, tensor<20x40xf32>, tensor, tensor<1x10xf32>, tensor<1x10xf32>, tensor<5x1x10xf32>, tensor<5x1x1xf32>, tensor<5x1x1xf32>, tensor<5xi64>, tensor<5x1x10xf32>, tensor<5x1x10xf32>>) -> tensor<5x1x10xf32> + %78 = "xla_hlo.reshape"(%49) : (tensor<1x10xf32>) -> tensor<1x1x10xf32> + %79 = "xla_hlo.slice"(%78) {limit_indices = dense<[1, 1, 10]> : tensor<3xi64>, start_indices = dense<0> : tensor<3xi64>, strides = dense<1> : tensor<3xi64>} : (tensor<1x1x10xf32>) -> tensor<1x1x10xf32> + %80 = "xla_hlo.slice"(%73) {limit_indices = dense<1> : tensor<1xi64>, start_indices = dense<0> : tensor<1xi64>, strides = dense<1> : tensor<1xi64>} : (tensor<1xi32>) -> tensor<1xi32> + %81 = "xla_hlo.reshape"(%80) : (tensor<1xi32>) -> tensor + %cst_9 = constant dense<0> : tensor + %82 = "xla_hlo.dynamic-update-slice"(%77, %79, %81, %cst_9, %cst_9) : (tensor<5x1x10xf32>, tensor<1x1x10xf32>, tensor, tensor, tensor) -> tensor<5x1x10xf32> + %83 = "xla_hlo.get_tuple_element"(%arg0) {index = 13 : i32} : (tuple, tensor, tensor<40xf32>, tensor, tensor<20x40xf32>, tensor, tensor<1x10xf32>, tensor<1x10xf32>, tensor<5x1x10xf32>, tensor<5x1x1xf32>, tensor<5x1x1xf32>, tensor<5xi64>, tensor<5x1x10xf32>, tensor<5x1x10xf32>>) -> tensor<5x1x10xf32> + %84 = "xla_hlo.reshape"(%66) : (tensor<1x10xf32>) -> tensor<1x1x10xf32> + %85 = "xla_hlo.slice"(%84) {limit_indices = dense<[1, 1, 10]> : tensor<3xi64>, start_indices = dense<0> : tensor<3xi64>, strides = dense<1> : tensor<3xi64>} : (tensor<1x1x10xf32>) -> tensor<1x1x10xf32> + %86 = "xla_hlo.slice"(%73) {limit_indices = dense<1> : tensor<1xi64>, start_indices = dense<0> : tensor<1xi64>, strides = dense<1> : tensor<1xi64>} : (tensor<1xi32>) -> tensor<1xi32> + %87 = "xla_hlo.reshape"(%86) : (tensor<1xi32>) -> tensor + %cst_10 = constant dense<0> : tensor + %88 = "xla_hlo.dynamic-update-slice"(%83, %85, %87, %cst_10, %cst_10) : (tensor<5x1x10xf32>, tensor<1x1x10xf32>, tensor, tensor, tensor) -> tensor<5x1x10xf32> + %89 = "xla_hlo.tuple"(%1, %2, %3, %4, %5, %6, %49, %66, %19, %7, %67, %76, %82, %88) : (tensor, tensor, tensor<40xf32>, tensor, tensor<20x40xf32>, tensor, tensor<1x10xf32>, tensor<1x10xf32>, tensor<5x1x10xf32>, tensor<5x1x1xf32>, tensor<5x1x1xf32>, tensor<5xi64>, tensor<5x1x10xf32>, tensor<5x1x10xf32>) -> tuple, tensor, tensor<40xf32>, tensor, tensor<20x40xf32>, tensor, tensor<1x10xf32>, tensor<1x10xf32>, tensor<5x1x10xf32>, tensor<5x1x1xf32>, tensor<5x1x1xf32>, tensor<5xi64>, tensor<5x1x10xf32>, tensor<5x1x10xf32>> + return %89 : tuple, tensor, tensor<40xf32>, tensor, tensor<20x40xf32>, tensor, tensor<1x10xf32>, tensor<1x10xf32>, tensor<5x1x10xf32>, tensor<5x1x1xf32>, tensor<5x1x1xf32>, tensor<5xi64>, tensor<5x1x10xf32>, tensor<5x1x10xf32>> +} +func @ForwardLoopCond_gFAnjWGSoLs__.436(%arg0: tuple, tensor, tensor<40xf32>, tensor, tensor<20x40xf32>, tensor, tensor<1x10xf32>, tensor<1x10xf32>, tensor<5x1x10xf32>, tensor<5x1x1xf32>, tensor<5x1x1xf32>, tensor<5xi64>, tensor<5x1x10xf32>, tensor<5x1x10xf32>>) -> tuple> { + %0 = "xla_hlo.get_tuple_element"(%arg0) {index = 0 : i32} : (tuple, tensor, tensor<40xf32>, tensor, tensor<20x40xf32>, tensor, tensor<1x10xf32>, tensor<1x10xf32>, tensor<5x1x10xf32>, tensor<5x1x1xf32>, tensor<5x1x1xf32>, tensor<5xi64>, tensor<5x1x10xf32>, tensor<5x1x10xf32>>) -> tensor + %1 = "xla_hlo.get_tuple_element"(%arg0) {index = 1 : i32} : (tuple, tensor, tensor<40xf32>, tensor, tensor<20x40xf32>, tensor, tensor<1x10xf32>, tensor<1x10xf32>, tensor<5x1x10xf32>, tensor<5x1x1xf32>, tensor<5x1x1xf32>, tensor<5xi64>, tensor<5x1x10xf32>, tensor<5x1x10xf32>>) -> tensor + %2 = "xla_hlo.compare"(%0, %1) {comparison_direction = "LT"} : (tensor, tensor) -> tensor + %3 = "xla_hlo.tuple"(%2) : (tensor) -> tuple> + return %3 : tuple> +} +func @cond_wrapper.454(%arg0: tuple, tensor, tensor<40xf32>, tensor, tensor<20x40xf32>, tensor, tensor<1x10xf32>, tensor<1x10xf32>, tensor<5x1x10xf32>, tensor<5x1x1xf32>, tensor<5x1x1xf32>, tensor<5xi64>, tensor<5x1x10xf32>, tensor<5x1x10xf32>>) -> tensor { + %0 = call @ForwardLoopCond_gFAnjWGSoLs__.436(%arg0) : (tuple, tensor, tensor<40xf32>, tensor, tensor<20x40xf32>, tensor, tensor<1x10xf32>, tensor<1x10xf32>, tensor<5x1x10xf32>, tensor<5x1x1xf32>, tensor<5x1x1xf32>, tensor<5xi64>, tensor<5x1x10xf32>, tensor<5x1x10xf32>>) -> tuple> + %1 = "xla_hlo.get_tuple_element"(%0) {index = 0 : i32} : (tuple>) -> tensor + return %1 : tensor +} +func @Forward_DIZIAkooG44__disable_call_shape_inference_true_.458(%arg0: tensor<1x10xf32>, %arg1: tensor<1x10xf32>, %arg2: tensor<5x1x10xf32>, %arg3: tensor<5x1x1xf32>, %arg4: tensor<5x1x1xf32>) -> tuple, tensor<5xi64>, tensor<5x1x10xf32>, tensor<5x1x10xf32>, tensor, tensor<1x10xf32>, tensor<1x10xf32>> { + %cst = constant dense<5> : tensor + %0 = "xla_hlo.convert"(%arg3) : (tensor<5x1x1xf32>) -> tensor<5x1x1xf32> + %cst_0 = constant dense<0x7F800000> : tensor + %1 = "xla_hlo.convert"(%cst_0) : (tensor) -> tensor + %2 = "xla_hlo.reduce"(%0, %1) ( { + ^bb0(%arg5: tensor, %arg6: tensor): // no predecessors + %42 = xla_hlo.min %arg5, %arg6 : tensor + "xla_hlo.return"(%42) : (tensor) -> () + }) {dimensions = dense<[1, 2]> : tensor<2xi64>} : (tensor<5x1x1xf32>, tensor) -> tensor<5xf32> + %3 = "xla_hlo.convert"(%2) : (tensor<5xf32>) -> tensor<5xf32> + %cst_1 = constant dense<0.0> : tensor + %4 = "xla_hlo.broadcast_in_dim"(%cst_1) : (tensor) -> tensor<5xf32> + %5 = "xla_hlo.compare"(%3, %4) {comparison_direction = "EQ"} : (tensor<5xf32>, tensor<5xf32>) -> tensor<5xi1> + %6 = "xla_hlo.convert"(%5) : (tensor<5xi1>) -> tensor<5xi32> + %cst_2 = constant dense<[1, 2, 3, 4, 5]> : tensor<5xi32> + %7 = xla_hlo.mul %6, %cst_2 : tensor<5xi32> + %8 = "xla_hlo.convert"(%7) : (tensor<5xi32>) -> tensor<5xi32> + %cst_3 = constant dense<-2147483648> : tensor + %9 = "xla_hlo.convert"(%cst_3) : (tensor) -> tensor + %10 = "xla_hlo.reduce"(%8, %9) ( { + ^bb0(%arg5: tensor, %arg6: tensor): // no predecessors + %42 = xla_hlo.max %arg5, %arg6 : tensor + "xla_hlo.return"(%42) : (tensor) -> () + }) {dimensions = dense<0> : tensor<1xi64>} : (tensor<5xi32>, tensor) -> tensor + %11 = "xla_hlo.convert"(%10) : (tensor) -> tensor + %12 = xla_hlo.sub %cst, %11 : tensor + %cst_4 = constant dense<5> : tensor + %13 = "xla_hlo.compare"(%12, %cst_4) {comparison_direction = "EQ"} : (tensor, tensor) -> tensor + %cst_5 = constant dense<0> : tensor + %cst_6 = constant dense<5> : tensor + %14 = "xla_hlo.reverse"(%3) {dimensions = dense<0> : tensor<1xi64>} : (tensor<5xf32>) -> tensor<5xf32> + %cst_7 = constant dense<0.0> : tensor + %15 = "xla_hlo.broadcast_in_dim"(%cst_7) : (tensor) -> tensor<5xf32> + %16 = "xla_hlo.compare"(%14, %15) {comparison_direction = "EQ"} : (tensor<5xf32>, tensor<5xf32>) -> tensor<5xi1> + %17 = "xla_hlo.convert"(%16) : (tensor<5xi1>) -> tensor<5xi32> + %cst_8 = constant dense<[1, 2, 3, 4, 5]> : tensor<5xi32> + %18 = xla_hlo.mul %17, %cst_8 : tensor<5xi32> + %19 = "xla_hlo.convert"(%18) : (tensor<5xi32>) -> tensor<5xi32> + %cst_9 = constant dense<-2147483648> : tensor + %20 = "xla_hlo.convert"(%cst_9) : (tensor) -> tensor + %21 = "xla_hlo.reduce"(%19, %20) ( { + ^bb0(%arg5: tensor, %arg6: tensor): // no predecessors + %42 = xla_hlo.max %arg5, %arg6 : tensor + "xla_hlo.return"(%42) : (tensor) -> () + }) {dimensions = dense<0> : tensor<1xi64>} : (tensor<5xi32>, tensor) -> tensor + %22 = "xla_hlo.convert"(%21) : (tensor) -> tensor + %23 = xla_hlo.sub %cst_6, %22 : tensor + %24 = "xla_hlo.select"(%13, %cst_5, %23) : (tensor, tensor, tensor) -> tensor + %25 = "xla_hlo.convert"(%24) : (tensor) -> tensor + %cst_10 = constant dense<5> : tensor + %26 = xla_hlo.sub %cst_10, %12 : tensor + %27 = "xla_hlo.convert"(%26) : (tensor) -> tensor + %cst_11 = constant dense<0.0> : tensor + %28 = "xla_hlo.broadcast_in_dim"(%cst_11) : (tensor) -> tensor<40xf32> + %cst_12 = constant dense<0> : tensor + %cst_13 = constant dense<0.42> : tensor<20x40xf32> + %cst_14 = constant dense<-2130569576> : tensor + %cst_15 = constant dense<0> : tensor + %29 = "xla_hlo.broadcast_in_dim"(%cst_15) : (tensor) -> tensor<5xi64> + %cst_16 = constant dense<0.0> : tensor + %30 = "xla_hlo.broadcast_in_dim"(%cst_16) : (tensor) -> tensor<5x1x10xf32> + %cst_17 = constant dense<0.0> : tensor + %31 = "xla_hlo.broadcast_in_dim"(%cst_17) : (tensor) -> tensor<5x1x10xf32> + %32 = "xla_hlo.tuple"(%25, %27, %28, %cst_12, %cst_13, %cst_14, %arg0, %arg1, %arg2, %arg3, %arg4, %29, %30, %31) : (tensor, tensor, tensor<40xf32>, tensor, tensor<20x40xf32>, tensor, tensor<1x10xf32>, tensor<1x10xf32>, tensor<5x1x10xf32>, tensor<5x1x1xf32>, tensor<5x1x1xf32>, tensor<5xi64>, tensor<5x1x10xf32>, tensor<5x1x10xf32>) -> tuple, tensor, tensor<40xf32>, tensor, tensor<20x40xf32>, tensor, tensor<1x10xf32>, tensor<1x10xf32>, tensor<5x1x10xf32>, tensor<5x1x1xf32>, tensor<5x1x1xf32>, tensor<5xi64>, tensor<5x1x10xf32>, tensor<5x1x10xf32>> + %33 = "xla_hlo.while"(%32) {body = @ForwardLoopBody_o0Jnom3Cdxo__.328, cond = @cond_wrapper.454} : (tuple, tensor, tensor<40xf32>, tensor, tensor<20x40xf32>, tensor, tensor<1x10xf32>, tensor<1x10xf32>, tensor<5x1x10xf32>, tensor<5x1x1xf32>, tensor<5x1x1xf32>, tensor<5xi64>, tensor<5x1x10xf32>, tensor<5x1x10xf32>>) -> tuple, tensor, tensor<40xf32>, tensor, tensor<20x40xf32>, tensor, tensor<1x10xf32>, tensor<1x10xf32>, tensor<5x1x10xf32>, tensor<5x1x1xf32>, tensor<5x1x1xf32>, tensor<5xi64>, tensor<5x1x10xf32>, tensor<5x1x10xf32>> + %34 = "xla_hlo.get_tuple_element"(%33) {index = 0 : i32} : (tuple, tensor, tensor<40xf32>, tensor, tensor<20x40xf32>, tensor, tensor<1x10xf32>, tensor<1x10xf32>, tensor<5x1x10xf32>, tensor<5x1x1xf32>, tensor<5x1x1xf32>, tensor<5xi64>, tensor<5x1x10xf32>, tensor<5x1x10xf32>>) -> tensor + %35 = "xla_hlo.get_tuple_element"(%33) {index = 11 : i32} : (tuple, tensor, tensor<40xf32>, tensor, tensor<20x40xf32>, tensor, tensor<1x10xf32>, tensor<1x10xf32>, tensor<5x1x10xf32>, tensor<5x1x1xf32>, tensor<5x1x1xf32>, tensor<5xi64>, tensor<5x1x10xf32>, tensor<5x1x10xf32>>) -> tensor<5xi64> + %36 = "xla_hlo.get_tuple_element"(%33) {index = 12 : i32} : (tuple, tensor, tensor<40xf32>, tensor, tensor<20x40xf32>, tensor, tensor<1x10xf32>, tensor<1x10xf32>, tensor<5x1x10xf32>, tensor<5x1x1xf32>, tensor<5x1x1xf32>, tensor<5xi64>, tensor<5x1x10xf32>, tensor<5x1x10xf32>>) -> tensor<5x1x10xf32> + %37 = "xla_hlo.get_tuple_element"(%33) {index = 13 : i32} : (tuple, tensor, tensor<40xf32>, tensor, tensor<20x40xf32>, tensor, tensor<1x10xf32>, tensor<1x10xf32>, tensor<5x1x10xf32>, tensor<5x1x1xf32>, tensor<5x1x1xf32>, tensor<5xi64>, tensor<5x1x10xf32>, tensor<5x1x10xf32>>) -> tensor<5x1x10xf32> + %38 = "xla_hlo.get_tuple_element"(%33) {index = 5 : i32} : (tuple, tensor, tensor<40xf32>, tensor, tensor<20x40xf32>, tensor, tensor<1x10xf32>, tensor<1x10xf32>, tensor<5x1x10xf32>, tensor<5x1x1xf32>, tensor<5x1x1xf32>, tensor<5xi64>, tensor<5x1x10xf32>, tensor<5x1x10xf32>>) -> tensor + %39 = "xla_hlo.get_tuple_element"(%33) {index = 6 : i32} : (tuple, tensor, tensor<40xf32>, tensor, tensor<20x40xf32>, tensor, tensor<1x10xf32>, tensor<1x10xf32>, tensor<5x1x10xf32>, tensor<5x1x1xf32>, tensor<5x1x1xf32>, tensor<5xi64>, tensor<5x1x10xf32>, tensor<5x1x10xf32>>) -> tensor<1x10xf32> + %40 = "xla_hlo.get_tuple_element"(%33) {index = 7 : i32} : (tuple, tensor, tensor<40xf32>, tensor, tensor<20x40xf32>, tensor, tensor<1x10xf32>, tensor<1x10xf32>, tensor<5x1x10xf32>, tensor<5x1x1xf32>, tensor<5x1x1xf32>, tensor<5xi64>, tensor<5x1x10xf32>, tensor<5x1x10xf32>>) -> tensor<1x10xf32> + %41 = "xla_hlo.tuple"(%34, %35, %36, %37, %38, %39, %40) : (tensor, tensor<5xi64>, tensor<5x1x10xf32>, tensor<5x1x10xf32>, tensor, tensor<1x10xf32>, tensor<1x10xf32>) -> tuple, tensor<5xi64>, tensor<5x1x10xf32>, tensor<5x1x10xf32>, tensor, tensor<1x10xf32>, tensor<1x10xf32>> + return %41 : tuple, tensor<5xi64>, tensor<5x1x10xf32>, tensor<5x1x10xf32>, tensor, tensor<1x10xf32>, tensor<1x10xf32>> +} +func @Min_reduction.585(%arg0: tensor, %arg1: tensor) -> tensor { + %0 = xla_hlo.min %arg0, %arg1 : tensor + return %0 : tensor +} +func @Max_reduction.589(%arg0: tensor, %arg1: tensor) -> tensor { + %0 = xla_hlo.max %arg0, %arg1 : tensor + return %0 : tensor +} +func @Max_1_reduction.593(%arg0: tensor, %arg1: tensor) -> tensor { + %0 = xla_hlo.max %arg0, %arg1 : tensor + return %0 : tensor +} +func @ForwardLoopBody_o0Jnom3Cdxo__.597(%arg0: tuple, tensor, tensor<40xf32>, tensor, tensor<20x40xf32>, tensor, tensor<1x10xf32>, tensor<1x10xf32>, tensor<5x1x10xf32>, tensor<5x1x1xf32>, tensor<5x1x1xf32>, tensor<5xi64>, tensor<5x1x10xf32>, tensor<5x1x10xf32>>) -> tuple, tensor, tensor<40xf32>, tensor, tensor<20x40xf32>, tensor, tensor<1x10xf32>, tensor<1x10xf32>, tensor<5x1x10xf32>, tensor<5x1x1xf32>, tensor<5x1x1xf32>, tensor<5xi64>, tensor<5x1x10xf32>, tensor<5x1x10xf32>> { + %0 = "xla_hlo.get_tuple_element"(%arg0) {index = 0 : i32} : (tuple, tensor, tensor<40xf32>, tensor, tensor<20x40xf32>, tensor, tensor<1x10xf32>, tensor<1x10xf32>, tensor<5x1x10xf32>, tensor<5x1x1xf32>, tensor<5x1x1xf32>, tensor<5xi64>, tensor<5x1x10xf32>, tensor<5x1x10xf32>>) -> tensor + %cst = constant dense<1> : tensor + %1 = xla_hlo.add %0, %cst : tensor + %2 = "xla_hlo.get_tuple_element"(%arg0) {index = 1 : i32} : (tuple, tensor, tensor<40xf32>, tensor, tensor<20x40xf32>, tensor, tensor<1x10xf32>, tensor<1x10xf32>, tensor<5x1x10xf32>, tensor<5x1x1xf32>, tensor<5x1x1xf32>, tensor<5xi64>, tensor<5x1x10xf32>, tensor<5x1x10xf32>>) -> tensor + %3 = "xla_hlo.get_tuple_element"(%arg0) {index = 2 : i32} : (tuple, tensor, tensor<40xf32>, tensor, tensor<20x40xf32>, tensor, tensor<1x10xf32>, tensor<1x10xf32>, tensor<5x1x10xf32>, tensor<5x1x1xf32>, tensor<5x1x1xf32>, tensor<5xi64>, tensor<5x1x10xf32>, tensor<5x1x10xf32>>) -> tensor<40xf32> + %4 = "xla_hlo.get_tuple_element"(%arg0) {index = 3 : i32} : (tuple, tensor, tensor<40xf32>, tensor, tensor<20x40xf32>, tensor, tensor<1x10xf32>, tensor<1x10xf32>, tensor<5x1x10xf32>, tensor<5x1x1xf32>, tensor<5x1x1xf32>, tensor<5xi64>, tensor<5x1x10xf32>, tensor<5x1x10xf32>>) -> tensor + %5 = "xla_hlo.get_tuple_element"(%arg0) {index = 4 : i32} : (tuple, tensor, tensor<40xf32>, tensor, tensor<20x40xf32>, tensor, tensor<1x10xf32>, tensor<1x10xf32>, tensor<5x1x10xf32>, tensor<5x1x1xf32>, tensor<5x1x1xf32>, tensor<5xi64>, tensor<5x1x10xf32>, tensor<5x1x10xf32>>) -> tensor<20x40xf32> + %6 = "xla_hlo.get_tuple_element"(%arg0) {index = 5 : i32} : (tuple, tensor, tensor<40xf32>, tensor, tensor<20x40xf32>, tensor, tensor<1x10xf32>, tensor<1x10xf32>, tensor<5x1x10xf32>, tensor<5x1x1xf32>, tensor<5x1x1xf32>, tensor<5xi64>, tensor<5x1x10xf32>, tensor<5x1x10xf32>>) -> tensor + %7 = "xla_hlo.get_tuple_element"(%arg0) {index = 9 : i32} : (tuple, tensor, tensor<40xf32>, tensor, tensor<20x40xf32>, tensor, tensor<1x10xf32>, tensor<1x10xf32>, tensor<5x1x10xf32>, tensor<5x1x1xf32>, tensor<5x1x1xf32>, tensor<5xi64>, tensor<5x1x10xf32>, tensor<5x1x10xf32>>) -> tensor<5x1x1xf32> + %8 = "xla_hlo.gather"(%7, %0) {collapsed_slice_dims = dense<0> : tensor<1xi64>, index_vector_dim = 0 : i64, offset_dims = dense<[0, 1]> : tensor<2xi64>, slice_sizes = dense<1> : tensor<3xi64>, start_index_map = dense<0> : tensor<1xi64>} : (tensor<5x1x1xf32>, tensor) -> tensor<1x1xf32> + %9 = "xla_hlo.reshape"(%8) : (tensor<1x1xf32>) -> tensor<1xf32> + %10 = "xla_hlo.broadcast_in_dim"(%9) {broadcast_dimensions = dense<0> : tensor<1xi64>} : (tensor<1xf32>) -> tensor<1x10xf32> + %cst_0 = constant dense<1.0> : tensor + %11 = "xla_hlo.broadcast_in_dim"(%cst_0) : (tensor) -> tensor<1x10xf32> + %12 = xla_hlo.mul %10, %11 : tensor<1x10xf32> + %cst_1 = constant dense<0.0> : tensor + %13 = "xla_hlo.broadcast_in_dim"(%cst_1) : (tensor) -> tensor<1x10xf32> + %14 = "xla_hlo.compare"(%12, %13) {comparison_direction = "GT"} : (tensor<1x10xf32>, tensor<1x10xf32>) -> tensor<1x10xi1> + %15 = "xla_hlo.get_tuple_element"(%arg0) {index = 6 : i32} : (tuple, tensor, tensor<40xf32>, tensor, tensor<20x40xf32>, tensor, tensor<1x10xf32>, tensor<1x10xf32>, tensor<5x1x10xf32>, tensor<5x1x1xf32>, tensor<5x1x1xf32>, tensor<5xi64>, tensor<5x1x10xf32>, tensor<5x1x10xf32>>) -> tensor<1x10xf32> + %cst_2 = constant dense<5.0e-01> : tensor + %16 = "xla_hlo.broadcast_in_dim"(%cst_2) : (tensor) -> tensor<1x10xf32> + %17 = "xla_hlo.broadcast_in_dim"(%cst_2) : (tensor) -> tensor<1x10xf32> + %18 = "xla_hlo.broadcast_in_dim"(%cst_2) : (tensor) -> tensor<1x10xf32> + %19 = "xla_hlo.get_tuple_element"(%arg0) {index = 8 : i32} : (tuple, tensor, tensor<40xf32>, tensor, tensor<20x40xf32>, tensor, tensor<1x10xf32>, tensor<1x10xf32>, tensor<5x1x10xf32>, tensor<5x1x1xf32>, tensor<5x1x1xf32>, tensor<5xi64>, tensor<5x1x10xf32>, tensor<5x1x10xf32>>) -> tensor<5x1x10xf32> + %20 = "xla_hlo.gather"(%19, %0) {collapsed_slice_dims = dense<0> : tensor<1xi64>, index_vector_dim = 0 : i64, offset_dims = dense<[0, 1]> : tensor<2xi64>, slice_sizes = dense<[1, 1, 10]> : tensor<3xi64>, start_index_map = dense<0> : tensor<1xi64>} : (tensor<5x1x10xf32>, tensor) -> tensor<1x10xf32> + %21 = "xla_hlo.get_tuple_element"(%arg0) {index = 7 : i32} : (tuple, tensor, tensor<40xf32>, tensor, tensor<20x40xf32>, tensor, tensor<1x10xf32>, tensor<1x10xf32>, tensor<5x1x10xf32>, tensor<5x1x1xf32>, tensor<5x1x1xf32>, tensor<5xi64>, tensor<5x1x10xf32>, tensor<5x1x10xf32>>) -> tensor<1x10xf32> + %22 = "xla_hlo.concatenate"(%20, %21) {dimension = 1 : i64} : (tensor<1x10xf32>, tensor<1x10xf32>) -> tensor<1x20xf32> + %23 = "xla_hlo.dot"(%22, %5) {precision_config = ["DEFAULT", "DEFAULT"]} : (tensor<1x20xf32>, tensor<20x40xf32>) -> tensor<1x40xf32> + %24 = "xla_hlo.transpose"(%23) {permutation = dense<[0, 1]> : tensor<2xi64>} : (tensor<1x40xf32>) -> tensor<1x40xf32> + %25 = "xla_hlo.reshape"(%3) : (tensor<40xf32>) -> tensor<1x40xf32> + %26 = xla_hlo.add %24, %25 : tensor<1x40xf32> + %27 = "xla_hlo.slice"(%26) {limit_indices = dense<[1, 30]> : tensor<2xi64>, start_indices = dense<[0, 20]> : tensor<2xi64>, strides = dense<1> : tensor<2xi64>} : (tensor<1x40xf32>) -> tensor<1x10xf32> + %28 = xla_hlo.mul %18, %27 : tensor<1x10xf32> + %29 = "xla_hlo.tanh"(%28) : (tensor<1x10xf32>) -> tensor<1x10xf32> + %30 = xla_hlo.mul %17, %29 : tensor<1x10xf32> + %31 = xla_hlo.add %16, %30 : tensor<1x10xf32> + %32 = xla_hlo.mul %31, %15 : tensor<1x10xf32> + %cst_3 = constant dense<5.0e-01> : tensor + %33 = "xla_hlo.broadcast_in_dim"(%cst_3) : (tensor) -> tensor<1x10xf32> + %34 = "xla_hlo.broadcast_in_dim"(%cst_3) : (tensor) -> tensor<1x10xf32> + %35 = "xla_hlo.broadcast_in_dim"(%cst_3) : (tensor) -> tensor<1x10xf32> + %36 = "xla_hlo.slice"(%26) {limit_indices = dense<[1, 20]> : tensor<2xi64>, start_indices = dense<[0, 10]> : tensor<2xi64>, strides = dense<1> : tensor<2xi64>} : (tensor<1x40xf32>) -> tensor<1x10xf32> + %37 = xla_hlo.mul %35, %36 : tensor<1x10xf32> + %38 = "xla_hlo.tanh"(%37) : (tensor<1x10xf32>) -> tensor<1x10xf32> + %39 = xla_hlo.mul %34, %38 : tensor<1x10xf32> + %40 = xla_hlo.add %33, %39 : tensor<1x10xf32> + %41 = "xla_hlo.slice"(%26) {limit_indices = dense<[1, 10]> : tensor<2xi64>, start_indices = dense<0> : tensor<2xi64>, strides = dense<1> : tensor<2xi64>} : (tensor<1x40xf32>) -> tensor<1x10xf32> + %42 = "xla_hlo.tanh"(%41) : (tensor<1x10xf32>) -> tensor<1x10xf32> + %43 = xla_hlo.mul %40, %42 : tensor<1x10xf32> + %44 = xla_hlo.add %32, %43 : tensor<1x10xf32> + %cst_4 = constant dense<1.0e+01> : tensor + %45 = "xla_hlo.broadcast_in_dim"(%cst_4) : (tensor) -> tensor<1x10xf32> + %46 = xla_hlo.min %44, %45 : tensor<1x10xf32> + %cst_5 = constant dense<-1.0e+01> : tensor + %47 = "xla_hlo.broadcast_in_dim"(%cst_5) : (tensor) -> tensor<1x10xf32> + %48 = xla_hlo.max %46, %47 : tensor<1x10xf32> + %49 = "xla_hlo.select"(%14, %15, %48) : (tensor<1x10xi1>, tensor<1x10xf32>, tensor<1x10xf32>) -> tensor<1x10xf32> + %50 = "xla_hlo.reshape"(%8) : (tensor<1x1xf32>) -> tensor<1xf32> + %51 = "xla_hlo.broadcast_in_dim"(%50) {broadcast_dimensions = dense<0> : tensor<1xi64>} : (tensor<1xf32>) -> tensor<1x10xf32> + %cst_6 = constant dense<1.0> : tensor + %52 = "xla_hlo.broadcast_in_dim"(%cst_6) : (tensor) -> tensor<1x10xf32> + %53 = xla_hlo.mul %51, %52 : tensor<1x10xf32> + %cst_7 = constant dense<0.0> : tensor + %54 = "xla_hlo.broadcast_in_dim"(%cst_7) : (tensor) -> tensor<1x10xf32> + %55 = "xla_hlo.compare"(%53, %54) {comparison_direction = "GT"} : (tensor<1x10xf32>, tensor<1x10xf32>) -> tensor<1x10xi1> + %cst_8 = constant dense<5.0e-01> : tensor + %56 = "xla_hlo.broadcast_in_dim"(%cst_8) : (tensor) -> tensor<1x10xf32> + %57 = "xla_hlo.broadcast_in_dim"(%cst_8) : (tensor) -> tensor<1x10xf32> + %58 = "xla_hlo.broadcast_in_dim"(%cst_8) : (tensor) -> tensor<1x10xf32> + %59 = "xla_hlo.slice"(%26) {limit_indices = dense<[1, 40]> : tensor<2xi64>, start_indices = dense<[0, 30]> : tensor<2xi64>, strides = dense<1> : tensor<2xi64>} : (tensor<1x40xf32>) -> tensor<1x10xf32> + %60 = xla_hlo.mul %58, %59 : tensor<1x10xf32> + %61 = "xla_hlo.tanh"(%60) : (tensor<1x10xf32>) -> tensor<1x10xf32> + %62 = xla_hlo.mul %57, %61 : tensor<1x10xf32> + %63 = xla_hlo.add %56, %62 : tensor<1x10xf32> + %64 = "xla_hlo.tanh"(%48) : (tensor<1x10xf32>) -> tensor<1x10xf32> + %65 = xla_hlo.mul %63, %64 : tensor<1x10xf32> + %66 = "xla_hlo.select"(%55, %21, %65) : (tensor<1x10xi1>, tensor<1x10xf32>, tensor<1x10xf32>) -> tensor<1x10xf32> + %67 = "xla_hlo.get_tuple_element"(%arg0) {index = 10 : i32} : (tuple, tensor, tensor<40xf32>, tensor, tensor<20x40xf32>, tensor, tensor<1x10xf32>, tensor<1x10xf32>, tensor<5x1x10xf32>, tensor<5x1x1xf32>, tensor<5x1x1xf32>, tensor<5xi64>, tensor<5x1x10xf32>, tensor<5x1x10xf32>>) -> tensor<5x1x1xf32> + %68 = "xla_hlo.get_tuple_element"(%arg0) {index = 11 : i32} : (tuple, tensor, tensor<40xf32>, tensor, tensor<20x40xf32>, tensor, tensor<1x10xf32>, tensor<1x10xf32>, tensor<5x1x10xf32>, tensor<5x1x1xf32>, tensor<5x1x1xf32>, tensor<5xi64>, tensor<5x1x10xf32>, tensor<5x1x10xf32>>) -> tensor<5xi64> + %69 = "xla_hlo.reshape"(%6) : (tensor) -> tensor<1xi64> + %70 = "xla_hlo.slice"(%69) {limit_indices = dense<1> : tensor<1xi64>, start_indices = dense<0> : tensor<1xi64>, strides = dense<1> : tensor<1xi64>} : (tensor<1xi64>) -> tensor<1xi64> + %71 = "xla_hlo.reshape"(%0) : (tensor) -> tensor<1xi64> + %72 = "xla_hlo.concatenate"(%71) {dimension = 0 : i64} : (tensor<1xi64>) -> tensor<1xi64> + %73 = "xla_hlo.convert"(%72) : (tensor<1xi64>) -> tensor<1xi32> + %74 = "xla_hlo.slice"(%73) {limit_indices = dense<1> : tensor<1xi64>, start_indices = dense<0> : tensor<1xi64>, strides = dense<1> : tensor<1xi64>} : (tensor<1xi32>) -> tensor<1xi32> + %75 = "xla_hlo.reshape"(%74) : (tensor<1xi32>) -> tensor + %76 = "xla_hlo.dynamic-update-slice"(%68, %70, %75) : (tensor<5xi64>, tensor<1xi64>, tensor) -> tensor<5xi64> + %77 = "xla_hlo.get_tuple_element"(%arg0) {index = 12 : i32} : (tuple, tensor, tensor<40xf32>, tensor, tensor<20x40xf32>, tensor, tensor<1x10xf32>, tensor<1x10xf32>, tensor<5x1x10xf32>, tensor<5x1x1xf32>, tensor<5x1x1xf32>, tensor<5xi64>, tensor<5x1x10xf32>, tensor<5x1x10xf32>>) -> tensor<5x1x10xf32> + %78 = "xla_hlo.reshape"(%49) : (tensor<1x10xf32>) -> tensor<1x1x10xf32> + %79 = "xla_hlo.slice"(%78) {limit_indices = dense<[1, 1, 10]> : tensor<3xi64>, start_indices = dense<0> : tensor<3xi64>, strides = dense<1> : tensor<3xi64>} : (tensor<1x1x10xf32>) -> tensor<1x1x10xf32> + %80 = "xla_hlo.slice"(%73) {limit_indices = dense<1> : tensor<1xi64>, start_indices = dense<0> : tensor<1xi64>, strides = dense<1> : tensor<1xi64>} : (tensor<1xi32>) -> tensor<1xi32> + %81 = "xla_hlo.reshape"(%80) : (tensor<1xi32>) -> tensor + %cst_9 = constant dense<0> : tensor + %82 = "xla_hlo.dynamic-update-slice"(%77, %79, %81, %cst_9, %cst_9) : (tensor<5x1x10xf32>, tensor<1x1x10xf32>, tensor, tensor, tensor) -> tensor<5x1x10xf32> + %83 = "xla_hlo.get_tuple_element"(%arg0) {index = 13 : i32} : (tuple, tensor, tensor<40xf32>, tensor, tensor<20x40xf32>, tensor, tensor<1x10xf32>, tensor<1x10xf32>, tensor<5x1x10xf32>, tensor<5x1x1xf32>, tensor<5x1x1xf32>, tensor<5xi64>, tensor<5x1x10xf32>, tensor<5x1x10xf32>>) -> tensor<5x1x10xf32> + %84 = "xla_hlo.reshape"(%66) : (tensor<1x10xf32>) -> tensor<1x1x10xf32> + %85 = "xla_hlo.slice"(%84) {limit_indices = dense<[1, 1, 10]> : tensor<3xi64>, start_indices = dense<0> : tensor<3xi64>, strides = dense<1> : tensor<3xi64>} : (tensor<1x1x10xf32>) -> tensor<1x1x10xf32> + %86 = "xla_hlo.slice"(%73) {limit_indices = dense<1> : tensor<1xi64>, start_indices = dense<0> : tensor<1xi64>, strides = dense<1> : tensor<1xi64>} : (tensor<1xi32>) -> tensor<1xi32> + %87 = "xla_hlo.reshape"(%86) : (tensor<1xi32>) -> tensor + %cst_10 = constant dense<0> : tensor + %88 = "xla_hlo.dynamic-update-slice"(%83, %85, %87, %cst_10, %cst_10) : (tensor<5x1x10xf32>, tensor<1x1x10xf32>, tensor, tensor, tensor) -> tensor<5x1x10xf32> + %89 = "xla_hlo.tuple"(%1, %2, %3, %4, %5, %6, %49, %66, %19, %7, %67, %76, %82, %88) : (tensor, tensor, tensor<40xf32>, tensor, tensor<20x40xf32>, tensor, tensor<1x10xf32>, tensor<1x10xf32>, tensor<5x1x10xf32>, tensor<5x1x1xf32>, tensor<5x1x1xf32>, tensor<5xi64>, tensor<5x1x10xf32>, tensor<5x1x10xf32>) -> tuple, tensor, tensor<40xf32>, tensor, tensor<20x40xf32>, tensor, tensor<1x10xf32>, tensor<1x10xf32>, tensor<5x1x10xf32>, tensor<5x1x1xf32>, tensor<5x1x1xf32>, tensor<5xi64>, tensor<5x1x10xf32>, tensor<5x1x10xf32>> + return %89 : tuple, tensor, tensor<40xf32>, tensor, tensor<20x40xf32>, tensor, tensor<1x10xf32>, tensor<1x10xf32>, tensor<5x1x10xf32>, tensor<5x1x1xf32>, tensor<5x1x1xf32>, tensor<5xi64>, tensor<5x1x10xf32>, tensor<5x1x10xf32>> +} +func @ForwardLoopCond_gFAnjWGSoLs__.705(%arg0: tuple, tensor, tensor<40xf32>, tensor, tensor<20x40xf32>, tensor, tensor<1x10xf32>, tensor<1x10xf32>, tensor<5x1x10xf32>, tensor<5x1x1xf32>, tensor<5x1x1xf32>, tensor<5xi64>, tensor<5x1x10xf32>, tensor<5x1x10xf32>>) -> tuple> { + %0 = "xla_hlo.get_tuple_element"(%arg0) {index = 0 : i32} : (tuple, tensor, tensor<40xf32>, tensor, tensor<20x40xf32>, tensor, tensor<1x10xf32>, tensor<1x10xf32>, tensor<5x1x10xf32>, tensor<5x1x1xf32>, tensor<5x1x1xf32>, tensor<5xi64>, tensor<5x1x10xf32>, tensor<5x1x10xf32>>) -> tensor + %1 = "xla_hlo.get_tuple_element"(%arg0) {index = 1 : i32} : (tuple, tensor, tensor<40xf32>, tensor, tensor<20x40xf32>, tensor, tensor<1x10xf32>, tensor<1x10xf32>, tensor<5x1x10xf32>, tensor<5x1x1xf32>, tensor<5x1x1xf32>, tensor<5xi64>, tensor<5x1x10xf32>, tensor<5x1x10xf32>>) -> tensor + %2 = "xla_hlo.compare"(%0, %1) {comparison_direction = "LT"} : (tensor, tensor) -> tensor + %3 = "xla_hlo.tuple"(%2) : (tensor) -> tuple> + return %3 : tuple> +} +func @cond_wrapper.723(%arg0: tuple, tensor, tensor<40xf32>, tensor, tensor<20x40xf32>, tensor, tensor<1x10xf32>, tensor<1x10xf32>, tensor<5x1x10xf32>, tensor<5x1x1xf32>, tensor<5x1x1xf32>, tensor<5xi64>, tensor<5x1x10xf32>, tensor<5x1x10xf32>>) -> tensor { + %0 = call @ForwardLoopCond_gFAnjWGSoLs__.705(%arg0) : (tuple, tensor, tensor<40xf32>, tensor, tensor<20x40xf32>, tensor, tensor<1x10xf32>, tensor<1x10xf32>, tensor<5x1x10xf32>, tensor<5x1x1xf32>, tensor<5x1x1xf32>, tensor<5xi64>, tensor<5x1x10xf32>, tensor<5x1x10xf32>>) -> tuple> + %1 = "xla_hlo.get_tuple_element"(%0) {index = 0 : i32} : (tuple>) -> tensor + return %1 : tensor +} +func @Forward_DIZIAkooG44__disable_call_shape_inference_true_.727(%arg0: tensor<1x10xf32>, %arg1: tensor<1x10xf32>, %arg2: tensor<5x1x10xf32>, %arg3: tensor<5x1x1xf32>, %arg4: tensor<5x1x1xf32>) -> tuple, tensor<5xi64>, tensor<5x1x10xf32>, tensor<5x1x10xf32>, tensor, tensor<1x10xf32>, tensor<1x10xf32>> { + %cst = constant dense<5> : tensor + %0 = "xla_hlo.convert"(%arg3) : (tensor<5x1x1xf32>) -> tensor<5x1x1xf32> + %cst_0 = constant dense<0x7F800000> : tensor + %1 = "xla_hlo.convert"(%cst_0) : (tensor) -> tensor + %2 = "xla_hlo.reduce"(%0, %1) ( { + ^bb0(%arg5: tensor, %arg6: tensor): // no predecessors + %42 = xla_hlo.min %arg5, %arg6 : tensor + "xla_hlo.return"(%42) : (tensor) -> () + }) {dimensions = dense<[1, 2]> : tensor<2xi64>} : (tensor<5x1x1xf32>, tensor) -> tensor<5xf32> + %3 = "xla_hlo.convert"(%2) : (tensor<5xf32>) -> tensor<5xf32> + %cst_1 = constant dense<0.0> : tensor + %4 = "xla_hlo.broadcast_in_dim"(%cst_1) : (tensor) -> tensor<5xf32> + %5 = "xla_hlo.compare"(%3, %4) {comparison_direction = "EQ"} : (tensor<5xf32>, tensor<5xf32>) -> tensor<5xi1> + %6 = "xla_hlo.convert"(%5) : (tensor<5xi1>) -> tensor<5xi32> + %cst_2 = constant dense<[1, 2, 3, 4, 5]> : tensor<5xi32> + %7 = xla_hlo.mul %6, %cst_2 : tensor<5xi32> + %8 = "xla_hlo.convert"(%7) : (tensor<5xi32>) -> tensor<5xi32> + %cst_3 = constant dense<-2147483648> : tensor + %9 = "xla_hlo.convert"(%cst_3) : (tensor) -> tensor + %10 = "xla_hlo.reduce"(%8, %9) ( { + ^bb0(%arg5: tensor, %arg6: tensor): // no predecessors + %42 = xla_hlo.max %arg5, %arg6 : tensor + "xla_hlo.return"(%42) : (tensor) -> () + }) {dimensions = dense<0> : tensor<1xi64>} : (tensor<5xi32>, tensor) -> tensor + %11 = "xla_hlo.convert"(%10) : (tensor) -> tensor + %12 = xla_hlo.sub %cst, %11 : tensor + %cst_4 = constant dense<5> : tensor + %13 = "xla_hlo.compare"(%12, %cst_4) {comparison_direction = "EQ"} : (tensor, tensor) -> tensor + %cst_5 = constant dense<0> : tensor + %cst_6 = constant dense<5> : tensor + %14 = "xla_hlo.reverse"(%3) {dimensions = dense<0> : tensor<1xi64>} : (tensor<5xf32>) -> tensor<5xf32> + %cst_7 = constant dense<0.0> : tensor + %15 = "xla_hlo.broadcast_in_dim"(%cst_7) : (tensor) -> tensor<5xf32> + %16 = "xla_hlo.compare"(%14, %15) {comparison_direction = "EQ"} : (tensor<5xf32>, tensor<5xf32>) -> tensor<5xi1> + %17 = "xla_hlo.convert"(%16) : (tensor<5xi1>) -> tensor<5xi32> + %cst_8 = constant dense<[1, 2, 3, 4, 5]> : tensor<5xi32> + %18 = xla_hlo.mul %17, %cst_8 : tensor<5xi32> + %19 = "xla_hlo.convert"(%18) : (tensor<5xi32>) -> tensor<5xi32> + %cst_9 = constant dense<-2147483648> : tensor + %20 = "xla_hlo.convert"(%cst_9) : (tensor) -> tensor + %21 = "xla_hlo.reduce"(%19, %20) ( { + ^bb0(%arg5: tensor, %arg6: tensor): // no predecessors + %42 = xla_hlo.max %arg5, %arg6 : tensor + "xla_hlo.return"(%42) : (tensor) -> () + }) {dimensions = dense<0> : tensor<1xi64>} : (tensor<5xi32>, tensor) -> tensor + %22 = "xla_hlo.convert"(%21) : (tensor) -> tensor + %23 = xla_hlo.sub %cst_6, %22 : tensor + %24 = "xla_hlo.select"(%13, %cst_5, %23) : (tensor, tensor, tensor) -> tensor + %25 = "xla_hlo.convert"(%24) : (tensor) -> tensor + %cst_10 = constant dense<5> : tensor + %26 = xla_hlo.sub %cst_10, %12 : tensor + %27 = "xla_hlo.convert"(%26) : (tensor) -> tensor + %cst_11 = constant dense<0.0> : tensor + %28 = "xla_hlo.broadcast_in_dim"(%cst_11) : (tensor) -> tensor<40xf32> + %cst_12 = constant dense<0> : tensor + %cst_13 = constant dense<0.42> : tensor<20x40xf32> + %cst_14 = constant dense<-1356785880> : tensor + %cst_15 = constant dense<0> : tensor + %29 = "xla_hlo.broadcast_in_dim"(%cst_15) : (tensor) -> tensor<5xi64> + %cst_16 = constant dense<0.0> : tensor + %30 = "xla_hlo.broadcast_in_dim"(%cst_16) : (tensor) -> tensor<5x1x10xf32> + %cst_17 = constant dense<0.0> : tensor + %31 = "xla_hlo.broadcast_in_dim"(%cst_17) : (tensor) -> tensor<5x1x10xf32> + %32 = "xla_hlo.tuple"(%25, %27, %28, %cst_12, %cst_13, %cst_14, %arg0, %arg1, %arg2, %arg3, %arg4, %29, %30, %31) : (tensor, tensor, tensor<40xf32>, tensor, tensor<20x40xf32>, tensor, tensor<1x10xf32>, tensor<1x10xf32>, tensor<5x1x10xf32>, tensor<5x1x1xf32>, tensor<5x1x1xf32>, tensor<5xi64>, tensor<5x1x10xf32>, tensor<5x1x10xf32>) -> tuple, tensor, tensor<40xf32>, tensor, tensor<20x40xf32>, tensor, tensor<1x10xf32>, tensor<1x10xf32>, tensor<5x1x10xf32>, tensor<5x1x1xf32>, tensor<5x1x1xf32>, tensor<5xi64>, tensor<5x1x10xf32>, tensor<5x1x10xf32>> + %33 = "xla_hlo.while"(%32) {body = @ForwardLoopBody_o0Jnom3Cdxo__.597, cond = @cond_wrapper.723} : (tuple, tensor, tensor<40xf32>, tensor, tensor<20x40xf32>, tensor, tensor<1x10xf32>, tensor<1x10xf32>, tensor<5x1x10xf32>, tensor<5x1x1xf32>, tensor<5x1x1xf32>, tensor<5xi64>, tensor<5x1x10xf32>, tensor<5x1x10xf32>>) -> tuple, tensor, tensor<40xf32>, tensor, tensor<20x40xf32>, tensor, tensor<1x10xf32>, tensor<1x10xf32>, tensor<5x1x10xf32>, tensor<5x1x1xf32>, tensor<5x1x1xf32>, tensor<5xi64>, tensor<5x1x10xf32>, tensor<5x1x10xf32>> + %34 = "xla_hlo.get_tuple_element"(%33) {index = 0 : i32} : (tuple, tensor, tensor<40xf32>, tensor, tensor<20x40xf32>, tensor, tensor<1x10xf32>, tensor<1x10xf32>, tensor<5x1x10xf32>, tensor<5x1x1xf32>, tensor<5x1x1xf32>, tensor<5xi64>, tensor<5x1x10xf32>, tensor<5x1x10xf32>>) -> tensor + %35 = "xla_hlo.get_tuple_element"(%33) {index = 11 : i32} : (tuple, tensor, tensor<40xf32>, tensor, tensor<20x40xf32>, tensor, tensor<1x10xf32>, tensor<1x10xf32>, tensor<5x1x10xf32>, tensor<5x1x1xf32>, tensor<5x1x1xf32>, tensor<5xi64>, tensor<5x1x10xf32>, tensor<5x1x10xf32>>) -> tensor<5xi64> + %36 = "xla_hlo.get_tuple_element"(%33) {index = 12 : i32} : (tuple, tensor, tensor<40xf32>, tensor, tensor<20x40xf32>, tensor, tensor<1x10xf32>, tensor<1x10xf32>, tensor<5x1x10xf32>, tensor<5x1x1xf32>, tensor<5x1x1xf32>, tensor<5xi64>, tensor<5x1x10xf32>, tensor<5x1x10xf32>>) -> tensor<5x1x10xf32> + %37 = "xla_hlo.get_tuple_element"(%33) {index = 13 : i32} : (tuple, tensor, tensor<40xf32>, tensor, tensor<20x40xf32>, tensor, tensor<1x10xf32>, tensor<1x10xf32>, tensor<5x1x10xf32>, tensor<5x1x1xf32>, tensor<5x1x1xf32>, tensor<5xi64>, tensor<5x1x10xf32>, tensor<5x1x10xf32>>) -> tensor<5x1x10xf32> + %38 = "xla_hlo.get_tuple_element"(%33) {index = 5 : i32} : (tuple, tensor, tensor<40xf32>, tensor, tensor<20x40xf32>, tensor, tensor<1x10xf32>, tensor<1x10xf32>, tensor<5x1x10xf32>, tensor<5x1x1xf32>, tensor<5x1x1xf32>, tensor<5xi64>, tensor<5x1x10xf32>, tensor<5x1x10xf32>>) -> tensor + %39 = "xla_hlo.get_tuple_element"(%33) {index = 6 : i32} : (tuple, tensor, tensor<40xf32>, tensor, tensor<20x40xf32>, tensor, tensor<1x10xf32>, tensor<1x10xf32>, tensor<5x1x10xf32>, tensor<5x1x1xf32>, tensor<5x1x1xf32>, tensor<5xi64>, tensor<5x1x10xf32>, tensor<5x1x10xf32>>) -> tensor<1x10xf32> + %40 = "xla_hlo.get_tuple_element"(%33) {index = 7 : i32} : (tuple, tensor, tensor<40xf32>, tensor, tensor<20x40xf32>, tensor, tensor<1x10xf32>, tensor<1x10xf32>, tensor<5x1x10xf32>, tensor<5x1x1xf32>, tensor<5x1x1xf32>, tensor<5xi64>, tensor<5x1x10xf32>, tensor<5x1x10xf32>>) -> tensor<1x10xf32> + %41 = "xla_hlo.tuple"(%34, %35, %36, %37, %38, %39, %40) : (tensor, tensor<5xi64>, tensor<5x1x10xf32>, tensor<5x1x10xf32>, tensor, tensor<1x10xf32>, tensor<1x10xf32>) -> tuple, tensor<5xi64>, tensor<5x1x10xf32>, tensor<5x1x10xf32>, tensor, tensor<1x10xf32>, tensor<1x10xf32>> + return %41 : tuple, tensor<5xi64>, tensor<5x1x10xf32>, tensor<5x1x10xf32>, tensor, tensor<1x10xf32>, tensor<1x10xf32>> +} + +// CHECK-LABEL: EXEC @main +func @main(%arg0: tensor<1x5xf32>, %arg1: tensor<1x5x2x2xf32>) -> tuple> + attributes { iree.module.export } { + %cst = constant dense<0.0> : tensor + %0 = "xla_hlo.broadcast_in_dim"(%cst) : (tensor) -> tensor<1x10xf32> + %cst_0 = constant dense<0.0> : tensor + %1 = "xla_hlo.broadcast_in_dim"(%cst_0) : (tensor) -> tensor<1x10xf32> + %cst_1 = constant dense<0.0> : tensor + %2 = "xla_hlo.broadcast_in_dim"(%cst_1) : (tensor) -> tensor<1x10xf32> + %cst_2 = constant dense<0.0> : tensor + %3 = "xla_hlo.broadcast_in_dim"(%cst_2) : (tensor) -> tensor<1x10xf32> + %cst_3 = constant dense<0.0> : tensor + %4 = "xla_hlo.broadcast_in_dim"(%cst_3) : (tensor) -> tensor<1x10xf32> + %cst_4 = constant dense<0.0> : tensor + %5 = "xla_hlo.broadcast_in_dim"(%cst_4) : (tensor) -> tensor<1x10xf32> + %6 = "xla_hlo.reshape"(%arg1) : (tensor<1x5x2x2xf32>) -> tensor<1x5x2x2xf32> + %7 = "xla_hlo.reshape"(%6) : (tensor<1x5x2x2xf32>) -> tensor<1x5x4xf32> + %cst_5 = constant dense<0.0> : tensor + %8 = "xla_hlo.pad"(%7, %cst_5) {edge_padding_high = dense<[0, 0, 60]> : tensor<3xi64>, edge_padding_low = dense<0> : tensor<3xi64>, interior_padding = dense<0> : tensor<3xi64>} : (tensor<1x5x4xf32>, tensor) -> tensor<1x5x64xf32> + %9 = "xla_hlo.transpose"(%8) {permutation = dense<[1, 0, 2]> : tensor<3xi64>} : (tensor<1x5x64xf32>) -> tensor<5x1x64xf32> + %10 = "xla_hlo.reshape"(%arg0) : (tensor<1x5xf32>) -> tensor<1x5xf32> + %11 = "xla_hlo.transpose"(%10) {permutation = dense<[1, 0]> : tensor<2xi64>} : (tensor<1x5xf32>) -> tensor<5x1xf32> + %12 = "xla_hlo.reshape"(%11) : (tensor<5x1xf32>) -> tensor<5x1x1xf32> + %cst_6 = constant dense<0.0> : tensor + %13 = "xla_hlo.broadcast_in_dim"(%cst_6) : (tensor) -> tensor<5x1x1xf32> + %14 = call @Forward_DIZIAkooG44__disable_call_shape_inference_true_.189(%4, %5, %9, %12, %13) : (tensor<1x10xf32>, tensor<1x10xf32>, tensor<5x1x64xf32>, tensor<5x1x1xf32>, tensor<5x1x1xf32>) -> tuple, tensor<5xi64>, tensor<5x1x10xf32>, tensor<5x1x10xf32>, tensor, tensor<1x10xf32>, tensor<1x10xf32>> + %15 = "xla_hlo.get_tuple_element"(%14) {index = 3 : i32} : (tuple, tensor<5xi64>, tensor<5x1x10xf32>, tensor<5x1x10xf32>, tensor, tensor<1x10xf32>, tensor<1x10xf32>>) -> tensor<5x1x10xf32> + %cst_7 = constant dense<0.0> : tensor + %16 = "xla_hlo.broadcast_in_dim"(%cst_7) : (tensor) -> tensor<5x1x1xf32> + %17 = call @Forward_DIZIAkooG44__disable_call_shape_inference_true_.458(%2, %3, %15, %12, %16) : (tensor<1x10xf32>, tensor<1x10xf32>, tensor<5x1x10xf32>, tensor<5x1x1xf32>, tensor<5x1x1xf32>) -> tuple, tensor<5xi64>, tensor<5x1x10xf32>, tensor<5x1x10xf32>, tensor, tensor<1x10xf32>, tensor<1x10xf32>> + %18 = "xla_hlo.get_tuple_element"(%17) {index = 3 : i32} : (tuple, tensor<5xi64>, tensor<5x1x10xf32>, tensor<5x1x10xf32>, tensor, tensor<1x10xf32>, tensor<1x10xf32>>) -> tensor<5x1x10xf32> + %cst_8 = constant dense<0.0> : tensor + %19 = "xla_hlo.broadcast_in_dim"(%cst_8) : (tensor) -> tensor<5x1x1xf32> + %20 = call @Forward_DIZIAkooG44__disable_call_shape_inference_true_.727(%0, %1, %18, %12, %19) : (tensor<1x10xf32>, tensor<1x10xf32>, tensor<5x1x10xf32>, tensor<5x1x1xf32>, tensor<5x1x1xf32>) -> tuple, tensor<5xi64>, tensor<5x1x10xf32>, tensor<5x1x10xf32>, tensor, tensor<1x10xf32>, tensor<1x10xf32>> + %21 = "xla_hlo.get_tuple_element"(%20) {index = 3 : i32} : (tuple, tensor<5xi64>, tensor<5x1x10xf32>, tensor<5x1x10xf32>, tensor, tensor<1x10xf32>, tensor<1x10xf32>>) -> tensor<5x1x10xf32> + %22 = "xla_hlo.copy"(%21) : (tensor<5x1x10xf32>) -> tensor<5x1x10xf32> + %23 = "xla_hlo.reshape"(%22) : (tensor<5x1x10xf32>) -> tensor<5x1x10xf32> + %24 = "xla_hlo.tuple"(%23) : (tensor<5x1x10xf32>) -> tuple> + return %24 : tuple> +} + +// CHECK: 5x1x10xf32= +// CHECK-SAME: [ +// CHECK-SAME: [0.7{{[0-9]+}} 0.7{{[0-9]+}} 0.7{{[0-9]+}} 0.7{{[0-9]+}} 0.7{{[0-9]+}} 0.7{{[0-9]+}} 0.7{{[0-9]+}} 0.7{{[0-9]+}} 0.7{{[0-9]+}} 0.7{{[0-9]+}}] +// CHECK-SAME: ][ +// CHECK-SAME: [0.7{{[0-9]+}} 0.7{{[0-9]+}} 0.7{{[0-9]+}} 0.7{{[0-9]+}} 0.7{{[0-9]+}} 0.7{{[0-9]+}} 0.7{{[0-9]+}} 0.7{{[0-9]+}} 0.7{{[0-9]+}} 0.7{{[0-9]+}}] +// CHECK-SAME: ][ +// CHECK-SAME: [0.9{{[0-9]+}} 0.9{{[0-9]+}} 0.9{{[0-9]+}} 0.9{{[0-9]+}} 0.9{{[0-9]+}} 0.9{{[0-9]+}} 0.9{{[0-9]+}} 0.9{{[0-9]+}} 0.9{{[0-9]+}} 0.9{{[0-9]+}}] +// CHECK-SAME: ][ +// CHECK-SAME: [0 0 0 0 0 0 0 0 0 0] +// CHECK-SAME: ][ +// CHECK-SAME: [0 0 0 0 0 0 0 0 0 0] +// CHECK-SAME: ] + diff --git a/test/models/mnist.mlir b/test/models/mnist.mlir new file mode 100644 index 000000000000..2b6df739faf5 --- /dev/null +++ b/test/models/mnist.mlir @@ -0,0 +1,54 @@ +// Trained MNIST model generated from tf.keras: +// +// model = tf.keras.models.Sequential() +// model.add(tf.keras.layers.Flatten(input_shape=input_shape)) +// model.add(tf.keras.layers.Dense(128, activation='relu')) +// model.add(tf.keras.layers.Dense(10, activation='softmax')) +// +// Translated to MLIR via a pipeline of scripts and some manual adjustments. +// +// Input : a single 28x28 pixel image as a tensor<1x28x28x1xf32>, with pixels in [0.0, 1.0] +// Output: a tensor<1x10xf32> containing prediction confidence values for each digit in [0, 9] + +module { + // CHECK-LABEL: EXEC @main + func @main(%arg0: tensor<1x28x28x1xf32>) -> tuple> + attributes {iree.module.export} { + %cst = constant {name = "constant.9"} dense<0.000000e+00> : tensor + %0 = "xla_hlo.broadcast_in_dim"(%cst) {name = "broadcast.10"} : (tensor) -> tensor<1x128xf32> + %1 = "xla_hlo.copy"(%arg0) {name = "copy.1"} : (tensor<1x28x28x1xf32>) -> tensor<1x28x28x1xf32> + %2 = "xla_hlo.reshape"(%1) {name = "reshape.2"} : (tensor<1x28x28x1xf32>) -> tensor<1x28x28x1xf32> + %3 = "xla_hlo.reshape"(%2) {name = "reshape.3"} : (tensor<1x28x28x1xf32>) -> tensor<1x784xf32> + %cst_0 = constant {name = "constant.4"} dense<[[-0.0468098037, -0.0632946193, -0.0398983546, -0.0710294246, -0.023822289, 0.0473461524, 0.0391825512, 0.0148925036, 0.0365879834, 0.0320887342, -0.0181696862, 0.027583316, -0.00250811875, 0.0532226786, -0.0763869435, -0.0135650411, 0.016689375, -4.394180e-02, -0.0154540092, -0.0217981301, -0.0719985589, 0.0641570911, -0.0788092538, -0.0083970353, 0.058764033, -0.0800188109, 0.081016533, -0.00271687657, 0.0733955726, -0.034039218, -0.0662241429, 0.0219521597, -0.0665535107, -0.00395233929, 0.0589044169, -0.051932171, -0.0736572817, 0.0243073329, -0.0386644751, -0.0223476477, 0.0681681708, -0.0769664124, -0.0529585928, 0.029920876, 0.0557949021, 0.0337341726, 0.00623550266, 0.0112335831, 0.0737327412, -0.0345536731, -0.0340200141, -0.0185914934, 0.00830452144, 0.057347618, -0.0255274847, 0.0683817789, -0.0490058623, -0.0696000904, -0.0543702096, -0.0324145481, -0.0774229541, -0.0499921739, -0.0454356633, -0.00776191055, 0.0309250727, -0.0376766361, -0.070886746, 0.0808603391, 0.0379195437, -0.0680912361, 0.0133953914, -0.0193472728, 0.0729874447, -0.0587174967, -0.0144509673, -0.0181279704, 0.0648851618, 0.0712784752, 5.079300e-02, 0.00370504707, 0.0114036575, 0.0715220645, -0.0314652883, -0.0610471815, -0.0290173106, -0.0526435301, 0.00293905288, 0.063546963, 0.0258001164, -0.0483844839, 0.0323070884, 0.0588752404, -0.0595022067, 6.717830e-02, -0.0456992052, 0.0434630588, -5.44972718E-4, 0.0451680943, -0.0436398722, 0.035703145, -0.0662389547, -0.0213864781, -0.0180439875, -0.00188871473, 0.00790153443, -0.0561722107, 0.0556770787, -0.0610739663, -0.0262338929, -0.0339419842, -0.0182302892, 0.0696926787, -0.0122746751, 0.0697000101, 0.02137959, 2.97613442E-4, 0.0646877214, 0.0450728461, 0.0236926824, -0.0331303179, -0.0586530268, 0.0270998403, -0.00761820375, -0.00867640227, -0.0159053877, 0.03777536, -0.0427610241, 5.489990e-03], [-2.654700e-02, 0.0731033608, 0.0164419413, -0.0442713052, 0.0277543664, -0.0502333604, -0.0745885223, 0.0434903875, 0.0110496134, 0.0423537195, -0.0560086668, 0.00338432193, -0.0704438537, 0.0325655416, 0.00207322091, 0.0509534553, -0.0766826794, -0.0307514146, 0.00876168162, -0.0639666691, 0.0677317902, -0.0692902878, 0.0122272968, 0.00158297271, 8.042760e-03, -0.0689550564, 0.0800703391, -0.0616888665, -0.00999631732, -0.0316142514, 0.00947940349, -0.0122303367, -0.0707587451, 0.0514494702, 0.0299684629, 0.0180700719, 0.0727027282, -0.050394237, 0.0447321162, 0.0262792781, -0.00407519937, -0.0767310261, 0.0160740316, -0.0366263725, -0.0505889729, 0.0187743753, 6.49362803E-4, -0.0184327289, 0.0626570359, 8.034110e-04, 0.00741612166, -0.0516756475, 0.0328502804, -0.0391003229, 0.0723687485, 0.017062746, -0.0306136124, -0.0226897821, -0.0197826549, -0.0712656379, -0.0171659738, -0.0684045777, 0.0051895678, 0.03898862, -0.0804143399, -0.0287879407, 0.0264469832, 0.00271206349, -0.00905906409, 0.076090984, 0.00431114435, -0.0798260644, -0.02172691, -0.0722743794, 0.0420523286, -0.0563531406, 0.0289157704, 0.0666903928, -0.0254771449, -0.0795465484, -0.0320510082, -0.0218732804, 0.0777639672, 0.078025259, -0.0113189742, 0.07733614, 0.0563754216, 1.777120e-02, -0.0689545721, 0.0294427574, -0.00927700847, -0.0406460688, 0.0614332482, -0.0650770366, 0.0565626696, -0.0755974203, -0.0784944072, -0.0708998143, -0.0538957044, -0.0426259451, 0.0315886512, -0.05719807, -0.0451313779, -0.0195901245, -0.0630296692, 0.0604007021, -0.0544902422, -0.0649082512, 0.00754988193, 0.0439225957, -0.00446308404, -0.0392948054, 0.0102357268, -0.0671287477, -0.0319550335, -0.0399148874, 0.0335906297, -0.0571868531, 0.0799903646, -0.0384761393, 0.0270238221, 0.0264991149, -2.07908452E-4, 0.0728175119, -0.00347659737, 0.0174588487, 0.0277400538, -0.0434758626], [0.0749155805, 0.0373661593, 0.0493463203, 0.0219272375, 0.0204284564, -0.0248467959, 0.0120763406, 0.0573434904, 0.0603022948, -0.0245944113, 0.0496351048, -0.0336957499, 0.0598718151, -0.0154218897, -0.00563241541, 0.0728116259, 0.0320828557, 0.0525878146, -0.0558604598, 0.00712169706, -0.0726425647, 0.0565757975, 0.0642923191, -0.0655623302, -0.00335755199, -0.0530357137, 0.0607445762, -0.0796949937, 6.959460e-02, 0.0745885447, -0.0332475081, -0.0440803207, 0.0608265772, 0.0288691968, 0.049298875, 0.0130008906, -0.0638654903, 0.0542700216, -0.0649368316, -0.00775925815, -0.0410674326, 0.0183054209, 0.0122620463, 0.0775256231, 0.0728222355, -0.0174344629, -0.0794259384, 0.0466090962, 0.0175791904, 2.780430e-02, -0.0714612231, 0.0243250057, 0.0278386846, 0.0666381195, -0.0471636765, -0.00423011929, 0.0383188203, -0.0552441105, -0.0638696477, 0.02759289, -0.022636272, 0.0770376399, -0.0306863822, 0.0528739467, -0.0165115446, -0.0552818365, -0.0373335741, 0.0706240758, -0.0312193632, -0.0345825665, 0.038012892, -0.00915005058, -0.0610979274, 0.0487279966, 0.0535355285, -0.0358723737, 2.609820e-02, -0.0180400982, 0.0463426486, 0.0730041936, -0.043301817, -0.0586155877, 0.00315105915, 0.0599973425, 0.0673555657, 0.0188202485, -0.00173843652, -0.0443963073, 0.0436014459, -0.074987024, 0.0119154677, 0.0745643601, 0.0785492882, -8.567870e-03, 0.00102156401, -0.0788189843, -0.024346862, -0.0250319988, 0.0327541903, -0.0336458199, -0.0445024744, -0.0677098483, 0.0414213389, -0.0396930203, -0.0578291565, 4.76688147E-5, 0.06126789, 0.00688365847, -0.0507595949, 0.0435292572, 0.0267072394, -0.0579000302, 0.0510065034, -0.0383951701, 0.00143811107, 0.0073589161, -0.023732096, -0.0726551339, -0.029677134, 0.0618803874, -0.0795162469, 0.0571380034, -2.798130e-02, -0.0534737632, 0.0759749636, -0.0322444886, 7.262590e-02, 0.00815343112], [-0.00530507416, 0.00186378509, 0.050987877, 0.0558923408, -0.0531433299, 0.0183567107, -0.0127888992, -0.0440797769, 0.0516482219, -0.0446780473, 0.00214119256, -0.0112130418, -0.00677937269, -0.0550892651, -0.0410221405, -0.00583086163, 0.0554906651, -0.027102977, -0.0442918055, 0.00816126167, 0.0119438395, -0.0431418717, -0.00757831335, 0.0361299962, 0.0161707848, -0.0260612406, -0.00437734276, 0.0783582255, -0.0532071441, -0.0224624611, -0.0326958038, -0.0118192434, -0.00784301758, -0.0810388252, -0.0744383559, 0.00288692117, 0.0505069569, 0.0640772209, 0.0617495105, -0.0449145921, -0.010922946, 0.0778964683, 0.00717557222, 0.0558820739, 0.0353812352, 0.0673715398, 0.0772155449, 0.0305139422, 0.0150065944, 0.0523570403, -0.0451951362, 0.00300377607, 0.0449685529, 0.0306178629, 0.0664797947, 0.0396916494, -0.051951319, 0.069726564, 0.0319477618, -0.0463205278, -0.0319263339, 0.078187339, -0.0604070872, 0.0706043467, 0.069042258, -0.0221806802, 0.0153156817, 0.0710021332, -0.00874278694, -0.0759044289, -0.0605426878, -0.0215295218, 0.062916033, -0.0279298425, -0.0226687044, -0.0461406037, -7.107980e-02, -0.00237479806, -0.0588954687, -0.0145036653, -0.0111134499, 0.0545925424, -0.079839915, -0.036151249, -0.0689045116, 0.0117083937, -2.516590e-03, 0.0411223322, -0.0755546987, 0.0570268258, 0.0104290321, -0.0159679428, -0.0626602918, 0.0647717789, 0.0663732663, -0.0429211259, 0.0237527117, -0.0228204131, -0.0596786886, 0.0178867653, -0.0512370169, 0.0333520323, -0.028879296, 0.0312310085, -7.405560e-02, 0.029132396, 0.0371402279, 0.023530826, 0.0529574826, -0.0272530988, -0.0160889849, -4.565850e-03, -0.0403454565, 0.0420330167, -7.321420e-02, 0.0353073105, 0.0164906383, 0.036970146, 0.0190328732, 0.066137664, -0.0439626276, -0.0452193655, 0.0508075878, 0.0433169976, -0.075001955, -0.0687389373, 0.0749086067, 0.0262980014], [-0.0751940757, 0.038741596, -0.057708174, 0.0223464146, -7.369410e-02, 0.0203554556, -0.0626923293, -0.0503879339, -0.0455307104, -0.0155408978, -0.0025639087, 0.0442988798, -0.031890519, -0.0343939401, 0.0769701079, -0.0270926505, 0.0354114473, 0.0434476286, -0.0718310326, -0.0524769723, 0.0302854404, 0.0396752283, -0.0746791362, -0.0372915901, -0.0666130781, 0.0701694116, -4.414400e-02, 0.0492011532, 4.606480e-02, 0.0678318068, -0.0183414519, -0.0419363603, -0.0286052115, -0.0219182596, -0.0497395955, -0.0354207084, 0.0426520109, -0.00234099478, -0.00998519361, -0.00346716493, 0.0785633847, -0.0531716049, -0.0555451885, 0.0672869757, -0.0150749385, 0.0732959285, 0.0384311751, 0.0616362318, -0.0460516103, -0.0195678473, -0.0703980252, 0.0339349061, 0.0739620402, 0.0752128437, -0.0163593888, 0.0255144313, -0.00689508766, -0.0766597688, -0.0393048041, 0.0197858289, -0.0720228106, -0.0134525895, 0.0189665407, 0.0267702788, 0.0159463435, -0.00975865125, 0.0445440784, -0.075516507, -0.0358201191, 0.0285170898, -3.169010e-02, 0.0703091845, -0.0147938356, -0.0452152081, -0.0673763379, 0.0638188496, -0.0486467108, -0.0614536181, -0.0785825923, -0.0381793939, 0.029594928, 0.0496685728, -0.0517303385, 5.19424677E-4, -5.368270e-03, -0.0792409256, -0.02266673, 0.0607654974, 0.00463471562, -0.00278659165, 0.00636910647, -0.0652876496, 0.0191313773, -0.0720793902, 0.0207124576, 0.0809273794, -0.040968053, 0.035410881, -0.0770573467, 0.0402233899, 0.00353426486, -0.0494936705, -0.0373214483, 0.0238616616, -0.0520013869, -0.0461030304, -6.90028071E-4, 0.0546908453, 0.0101937428, 0.00552234054, 0.0380064473, -0.0443299785, 0.0447342023, -0.0116502792, 0.0246388093, -0.00144333392, 0.0701324269, 0.0803704634, 0.0120244548, 0.0132889524, 0.0683693811, -0.0518237613, 0.0421817228, -0.0672791451, 0.0185806081, 0.018719703, 0.0294601172, 0.0395380408], [0.0663282797, -0.0586814135, -0.0738754421, -0.039973367, 0.0306012332, -0.0510733388, 0.0552510694, 0.0751656517, -0.0324496292, 0.0121509135, -4.11711633E-4, 0.0210205019, -0.0443239249, 0.00631985068, -0.0784953162, 0.0611171052, 0.015183121, -0.0128994957, 0.00965698808, -0.0575427562, -0.0614000856, -0.0251935124, -0.0656464472, 0.00596671551, 0.0356344953, 0.00946824252, -0.00447499752, -0.0385973528, 0.0365130305, -0.017918773, 0.00251554698, 7.440930e-02, 0.0407413691, -0.0113813058, -0.00947663933, 0.0273248255, 0.0668159202, 0.00585526973, -0.0444188751, 0.0763439462, -0.0262874216, -0.0400151573, 0.00165537745, -0.00811481475, -0.0185774714, 0.0621987358, 4.520320e-02, -3.531930e-02, -0.00296498835, 0.0168564022, -0.0470667519, 0.041495584, 0.0101288632, 0.0233565867, 0.00985883921, 0.0741512254, -0.0172595084, -0.0360204838, 0.0213242099, 0.0111105293, -0.0565126613, 0.0460746661, 0.071520187, 0.0664171949, 0.00505698472, -0.0685730502, -0.0177672952, 0.0200691521, -0.0511094034, -0.0223835409, -4.89838421E-4, 0.0751272961, 0.0150672421, 0.0331726894, 0.0563972667, -0.0123503879, 0.0692702755, -7.337030e-02, -0.0448995866, -0.0409298204, -0.0387367234, -0.0444361456, 0.0481548384, 0.019518882, -0.0338945873, -0.00761799514, -0.0616314895, 0.0578126237, -0.0643256232, 0.021117948, -0.03748364, 0.0705390647, 0.0485471264, -0.0569788888, -0.0320282094, 0.0267617926, 0.0723109171, -0.0534940697, 1.268670e-03, 0.019470416, 0.0653114393, -0.00576411188, 0.0570309684, 0.0265444815, -0.0637794137, -0.0787889361, -0.0329145789, -0.044537805, -0.0307107083, 0.0437400043, -0.0274420157, 0.0530407056, 0.0697890297, 0.00167755783, -0.00565094501, 0.0320838243, -0.0619111434, -0.0204404853, 7.717550e-03, 0.0108780861, -0.0178495049, 0.0383282751, 0.00557958335, 0.0490775928, -0.0154399872, 0.0502532348, -0.0315033868, -0.0220204815], [0.00450333208, 0.0217451081, -0.0606031567, 0.0602856055, -0.002008304, 0.0211162642, -0.0766753927, 0.0691472217, 0.0522510931, 0.0191876516, 0.026108928, 0.0390524045, -0.0145055652, -0.0647673681, -0.0584993809, 0.052208595, -0.0211566426, -7.996760e-02, 0.0544743314, 0.0136151463, -0.0638420879, -0.065616183, -0.0784413293, 0.0345399082, -0.0700425878, -0.0327181779, 0.00438952446, -0.0719347447, 0.0172524303, 0.0311494023, -0.0114540383, 0.0757717118, -0.0504427776, 0.045548968, -0.0355993733, -0.00232042372, -0.062162444, 7.716500e-02, 0.0160261691, 0.0193145499, 0.0100043193, 7.073430e-02, 0.0236546099, -0.0133323446, 6.452330e-02, 0.0358861238, -0.0654319897, 0.0792858079, -0.0498435795, -0.0507842526, 0.0525484607, 0.0163656324, -7.097200e-02, -0.00514872372, -0.0392760858, -0.0784018412, -0.0373990722, -0.0422508195, 0.0603740886, -0.0703353658, 0.0586520359, -6.427550e-02, -0.0549977794, 0.0355815217, 5.995050e-03, -0.00257589668, 0.0371098518, 0.0778690353, 0.0679297075, -0.00434172153, 0.072967656, -0.079786092, -0.0560852848, 0.0426448956, 0.0326595083, -0.0261460394, 0.0610415116, -0.0751694962, -5.113530e-02, -0.0269462764, 0.069432877, -0.0617689081, 0.0709957704, -0.0247313455, 0.0202042311, -0.0660692826, -3.46139073E-4, -0.01670257, 0.0621551648, -0.0311582759, -0.0226594396, -0.0553078055, -0.0180063173, -0.0545022339, 0.0782145634, 0.032028459, 0.0592513308, -0.033873722, 0.0496887043, 0.0459820256, -0.0437587835, -0.0128922611, -0.0522635952, -0.00360235572, 0.068596147, -0.0529593453, -0.0491828844, -0.00753083825, -0.0315184295, 0.0810756459, -0.0531534851, 0.00918803364, 0.0130145773, 0.0720333681, 0.0348382145, -0.00791986286, -0.00102052093, 0.0563989952, 0.00589091331, -0.0580201782, 0.0717622414, 0.00333210826, -5.226630e-02, -0.0735816285, 0.0035667941, 0.0479231253, -0.071971409, -0.0765648559], [-0.00443895161, 0.0494671389, 0.0206336528, -6.948210e-02, -0.0649686232, -0.0501191691, -0.0130034387, -0.00703962147, -0.0731594935, 0.0725953951, 0.0654923394, -0.0149749964, -0.0699282438, -0.0448503718, 0.0179648772, 0.0729972497, -0.00997921824, -0.00595082343, 0.0549475774, -0.0529875979, 0.0357665494, 0.0399830565, 0.0377369896, 0.00553224236, -0.0646955296, -0.031194862, 8.087240e-03, 0.0423635617, 0.0462172553, -0.0311774388, -0.0685620084, 0.00601664931, 0.0242279321, 0.0069340989, 0.0217677131, -0.0609126277, 0.0126992464, 0.0665573105, -0.0581776313, 0.078155525, 0.0264820233, 0.0567328706, -0.0479412898, -0.0324403457, -0.0752620324, -0.0463478342, -0.0747569203, 0.0151240006, -9.89906489E-4, 4.64312732E-4, -0.0203652419, 0.0565778837, 0.0233891532, 0.0127989575, 0.067365922, -0.0561770461, -0.0367142856, -0.0785645321, 0.0537611768, -0.0663229227, -0.0718862042, 0.0740231201, 0.0773476437, -0.0127121881, 0.0768802091, 0.0192832798, -0.0652767196, -0.050641343, 0.0081737563, -3.442670e-02, 0.0190312043, 0.00790800899, -0.0229756013, 0.049864836, 0.0147043243, -0.0587343797, -0.0360263623, -0.0309422091, 0.0156733245, -0.0101349503, -0.0640957505, -0.0138536319, -0.0269175023, -0.05333969, -0.0284015983, 0.0236015022, 0.0123357102, -0.0700660273, -6.928180e-02, 0.024994947, -0.0581933334, -0.0768312588, 0.0306220651, -0.0586450398, 0.026932314, -0.0307277255, -0.0556352437, -0.0542861447, 0.0236075372, 0.0520871058, 0.0519828573, 0.042265594, -0.0373065583, 0.020297423, -0.0159387812, -0.054002203, 0.035502024, 0.0769894645, 0.0419814959, 0.0279981866, 0.0228345841, -0.0617587939, -0.05714982, -0.0317250602, 0.010277532, 0.0381745175, 0.0149690807, -0.075583823, 0.0497778133, -0.0372238308, 0.0617464408, -0.0691234395, 0.0490868464, 0.00809812546, -0.0213616453, -0.0155625567, 0.0686778203, 0.0447236821], [0.0646989122, 0.0641181096, -0.0742681623, -0.0594457202, -0.0421453305, -0.0767710581, 0.0808807238, 0.0345977098, 0.00104941428, -0.0264846347, 0.0761534944, 0.0231618881, 0.0514119342, 0.0774104967, 0.0767408684, 0.05982586, 0.052459456, 0.005707331, -0.037397895, -0.0223244801, -0.0147429556, -0.0438061208, 0.0645351037, -0.044281479, -0.0610467978, -0.0678903684, -0.00767697393, 0.0642537102, 0.0378694981, -0.0133228675, -0.0196680389, 4.262030e-02, 0.0740085468, 0.0804650858, 0.0422480181, -0.069481276, 0.0248533711, 0.0121939555, 0.0448357686, -0.00165239722, -0.0484093539, -0.0190080404, -0.0165894404, -0.060603179, 0.0679044649, -0.0466832556, -0.0118800625, 0.0212012529, 0.00197229534, -0.0267799869, 6.402410e-02, 0.0741344318, -0.0539021082, -0.00894363224, -0.019448664, 0.0594458804, -0.058011476, 0.0720781609, -0.0109658018, 0.018667534, -0.0732215121, 0.0210185498, 0.048999168, 0.0530139729, 0.0367631167, 0.0681197867, 0.00537316501, 0.0275480077, -0.047732763, 0.0130554587, -0.0338345394, 0.0191370249, -0.0086813867, -0.030968219, 0.0280723274, -0.0336590633, 9.5898658E-4, -0.0569133908, 0.00127711892, 8.103760e-02, -0.06477727, -0.0381747708, -7.447810e-02, -0.0108474493, -0.0782876089, -0.0329071134, 0.02053269, -0.00942486525, -0.0510230809, -0.0795926899, 0.0117250979, -0.02598029, 8.004660e-02, 0.0581176952, 0.024625428, -0.0102827176, 0.0315112025, -6.606110e-02, 0.0390307456, -0.00710754096, 0.0703301653, 0.0588545278, 0.0576194301, -0.0414955057, -0.0585171729, -7.998990e-02, 0.0674525872, -0.0246861503, 0.0787093117, -0.0391914248, 0.0315080434, 0.0629490092, 0.0783334597, -0.0547437891, 0.0755583718, -0.0432667583, -0.01754307, -6.345710e-02, 0.0209626555, 0.0297699422, 0.057111837, -0.0571822524, 0.0254371762, 0.0103272349, 0.0482208803, -0.0264418945, 0.0181815177, 0.0515126064], [-0.0474278964, 0.0385066345, -5.013170e-02, 0.0154831707, 0.0644894913, 0.0279744565, 0.0479718223, 0.0637608692, 4.316490e-02, 0.00422470272, -0.0708656237, -0.00655054301, -0.0618327633, 0.0689462647, 0.0544726625, 0.00415506214, 0.0622656271, -0.0448623598, 4.930570e-02, -0.0263858922, -0.0411039814, 0.0363525823, -0.012793988, -0.0649262964, -0.00952599198, -0.00258810073, 0.0057586357, 0.0767841712, -0.0415478535, -0.019298695, -0.0211380012, -0.028393846, 0.0682900175, 0.0311893299, 0.0435311347, -0.0360946842, 0.0076854676, 0.0488320366, -0.0534905307, -0.0080145821, 0.0512551442, -0.0204895474, 0.0695215389, 0.0738678053, -0.0576611832, 0.0263512582, -0.00806908309, -0.013840653, 0.0135321096, -0.0399144068, -0.0747085735, -0.0447627679, -0.0678876266, 0.0209475532, 0.0205930248, -0.0572828501, 6.755880e-02, 0.0243478641, 0.0796470716, -0.053691186, 0.0551133081, -0.0560676903, -0.0167089701, -0.044024162, 0.0718964562, 0.0243803188, 0.0676374212, -0.0567768216, -0.0153139979, -0.00518400222, 6.72787428E-5, 0.0143993944, -0.0770948231, -0.067770727, -0.0795035213, -0.0781975686, 0.014738977, 0.0449814126, 0.0591544136, 0.0118599832, -0.0342718177, -0.02632403, 0.0796398743, 0.0581703559, -0.0419339612, -0.0350686535, 0.0710702315, -0.0178267211, -0.0282851048, -0.0409650356, 0.0106030926, -0.0222779922, -0.0473558418, 0.0259544179, 0.0251022577, -0.0441437289, -0.0807652473, 0.0596372858, 0.074111633, -0.0566841923, 0.0745651796, 4.42825258E-4, 0.0340056866, 0.0728843734, 0.0222843364, -0.0337496661, -0.0571088642, -0.0276289396, 5.540020e-02, 0.0222735852, 0.0403614864, -0.0208463967, -0.0392276645, -3.27125192E-4, -0.00928395241, 0.0593126789, 0.0492200777, -0.0595576502, -0.0611597151, 0.0407754257, 0.059623234, 0.00969390571, -0.0735530704, -0.0772992074, 0.0328950286, 0.0744910315, -0.0487196185, 0.0801844671], [0.0386458486, 0.0394463837, 0.0584446862, 0.0481695458, 0.0538662151, -0.03874005, 0.0228637308, 0.0127208307, -0.0102698952, -0.0268956684, -0.0207675919, -0.0769006088, 0.00804179162, -0.0596519634, -0.0535459928, -0.0162847787, -0.0534655824, -0.0783785358, -0.0692430064, 0.0277859643, 0.0297691077, -0.0204233527, -0.0488472506, 0.0461756214, 0.00456377864, 0.0402370095, 0.0511036739, 0.0156612024, -0.0488238111, -0.0690129399, -0.00134128332, 0.0531742945, 0.0725646242, -0.0242974125, 0.0583606437, 0.01293502, -0.0295163356, 0.0490201786, 0.0390482247, -0.0199371502, -0.0127485842, 0.0634400621, 0.0102562606, 0.0128679574, 0.0251607522, -0.0506208241, 0.0232722163, -0.0459180176, -0.0755720288, -0.0553944036, 0.0416827798, -0.0688349306, 0.0113794655, -0.0214902647, 0.0283033028, 0.0319715887, -0.0259635039, -0.00169415027, 0.0755345896, -0.0781911313, 0.0677819178, 0.0214553401, -0.07260333, -0.0783567056, 0.0366809815, -0.070596382, -0.0395633951, 0.0250572935, 0.0362627581, -0.00917677581, 0.0400094166, -0.0305017382, -0.0299243741, 0.0616701022, -0.0208113752, 0.0592275038, 0.0586469546, 0.010219425, 0.0221510381, -0.0403954647, -0.0645550564, 0.0085664764, 0.0344753712, 0.0210700259, 0.0769049153, 0.0131760538, 0.0699870661, -0.0752895549, 0.0129908919, 0.0589485541, 0.00960794091, 0.0378998742, -0.0544659719, -0.0469897874, 0.0572472885, 0.0720740333, -0.0102461502, -0.0337744355, 8.926600e-03, -6.508800e-02, -0.0101283565, 0.0216901675, 0.0329623669, -0.0125541762, -0.077643469, -0.0202615298, 0.0246222615, -0.0318428129, 3.783080e-02, -0.0169602111, 0.0686737075, -0.0709367692, -0.0308094509, 0.060821332, -0.0565499663, -0.0427388214, -0.0227681398, -0.0430074707, 5.870100e-02, -0.0194492452, -2.9411912E-4, -0.0498593375, 0.021961309, 0.0020288229, 0.027497381, -0.0347284749, 0.0222473964, 0.0634164736], [0.040417321, -0.0751236454, 0.0137220323, -0.0488091521, 0.0371812284, -0.0439256728, -0.0583637618, 0.0400063023, 0.0557139441, 0.0773818716, -0.0563064367, -0.0348849595, -0.0443670303, 0.074465625, -0.0529208258, 0.0620803908, 0.0784387513, -0.0475140475, -0.0808783173, -0.0676963851, -0.0729034543, 0.0436816588, 0.0363102853, -0.0132102221, -0.00886046141, -0.0763458684, -0.0500150919, -0.0362293757, 0.0331178829, 8.960500e-03, 0.0410805047, 0.0436389595, 0.0111984983, -0.00464213639, 0.00883545727, 0.0302078202, 0.0295286924, -0.0407269225, 0.017214179, -0.0726674125, -0.0179670975, 0.0522369072, -0.0226152316, 0.0508196279, -0.0757914409, 0.0158704221, -0.00514781475, 4.352680e-02, -0.0126590654, 0.00148899108, 0.0286674425, 0.0181291178, 0.0781375542, -0.0346545242, -0.0442779176, 0.0160474256, -3.701900e-02, 0.0464453027, 0.0187453702, 0.0614731535, -0.0684383363, 0.015011318, 7.957190e-03, 0.0684067532, -6.876290e-03, -0.0367581435, 0.0141173825, 0.0763342306, 0.0565179661, 6.872020e-02, 0.0418856964, 0.0197653845, -0.024544131, -6.390260e-02, 0.0473061875, -0.0236257948, -0.0621977523, 0.0130874291, 1.33238733E-4, 0.0278715789, -0.00487841666, -9.98742878E-4, 0.00103649497, -0.0100817308, 0.0596359745, 0.00129409879, -0.0353315175, -0.0509603471, -0.0638212413, 0.0310695916, -0.0619603768, -0.0724476725, -0.0486528799, -0.0691726729, -0.0725461841, 0.0721845403, -0.0232971981, -0.00484902412, 0.0485825613, -0.0361100771, 0.010389924, 0.0427650437, -0.0750007331, -0.0359642282, 0.0802526846, 0.00855983049, -0.0545181297, -0.0186713226, -0.0769514889, 0.0573286191, 0.0809162482, 0.0663661584, -0.0401864387, 0.0321169719, -0.0384002365, 0.0289841294, -0.0457479768, 0.0572925583, 0.0326318368, -0.0768105313, 0.0159963295, 0.0480613783, -0.0118233189, -0.0154026821, 0.0363510326, -0.0582442731, 0.00113780797, -0.0232391842], [0.0623605773, -0.00916641391, 0.0711954311, 0.0391955636, -0.0455944426, -3.329930e-02, 0.0773576722, 0.0579499193, 0.0229692385, 0.0089473389, 0.0522380956, -0.05881235, -0.0243103299, 0.0619655773, 0.0162477829, -0.00265669962, 0.019371815, 3.8741625E-4, -0.0517928228, -0.0227963645, 0.0563595742, -0.021286983, -0.0476570763, 0.0278329607, 0.00990591663, -0.0320420936, 0.0603698716, 0.0663454234, 0.0334392115, -0.0173279718, 0.022369381, -0.00440520095, -0.0261881705, -0.043828506, -0.0675624236, -0.0335435942, 6.43701234E-4, 0.00465109805, -0.0776589885, -0.00380916754, -0.0468792617, 0.00994280725, 0.0723987296, -0.0153584611, -0.0350473784, 0.0468649082, 0.0292649101, 0.00448149629, 0.0682022497, 0.0368522145, 0.0231916308, 4.793460e-02, -8.66804854E-4, -9.2884693E-5, 0.0464559272, -0.0600873642, 0.0228115655, -0.00948877167, 0.0186279975, 0.0649977848, -0.049701415, -0.0735538825, -0.0572600067, -0.0513178445, 0.0462525561, 0.03880319, 0.0461011082, -0.0457300134, -2.08990066E-4, 0.0327962488, -0.0341961496, 0.0630327165, 0.0675826073, 0.0613558888, -0.0561816841, 0.0269272141, -0.0755301639, 0.0534376204, 0.0450117141, 5.9108471E-4, 0.048084151, 0.0612157322, -0.0653322935, -0.0743316785, 0.014027603, -0.0247348733, -0.0707445517, -0.0447193049, -0.0756549314, -0.0385935158, 0.0489459112, -0.0362501703, 0.0386578105, -0.0328066126, -0.0249806307, -0.0584580489, 6.863100e-02, -0.0228867624, 0.0425871797, 0.0246863775, 0.038524732, 0.00253012078, -3.84945277E-4, -7.496370e-02, 0.00397484144, -0.0784900113, -0.0770209655, -0.038423188, 0.0299057756, 0.0578271821, 7.952340e-02, 0.0692321733, 4.125210e-03, -0.0106577463, -0.0668134987, -0.0615890548, -0.0656086802, -0.0498580113, 0.0137553643, -0.0167214535, 0.054978434, -4.540020e-02, 0.073457323, -0.0521883853, -0.0416138284, 0.0159285143, 6.709170e-02, -0.0689931661], [0.025665693, 0.00829473324, 0.0599660277, -0.0174572561, 0.0792002379, -0.0172322057, 0.00113726628, -0.0102291303, -0.0252511576, -0.0191530418, -0.0754089952, -0.0600796044, 0.00892634689, -0.00937372446, 0.0253391452, -0.0282883756, -0.0241601318, -0.00231542368, -0.0228955783, -0.0315537192, -0.0247962344, -0.069287546, 0.0517971367, -0.0187808592, -0.0495574847, -0.0117858648, -0.070904173, -0.00921172089, 0.050173454, -0.0762792826, 3.574170e-02, 2.717250e-02, -0.0281237625, 0.0378358811, 0.0123933339, 0.0220163986, 0.0481312722, -0.0572820753, 0.058375068, -0.0240767132, -0.0435535051, 0.00903767813, -0.06448441, 0.039064765, 0.0506480262, 0.0421437435, 0.0153313763, -0.0169939026, -0.0266883653, -0.0536936074, 0.00795694813, 0.0416973121, -0.0643218085, -0.0316443928, 0.031736061, 0.0096342992, -0.00246241037, 0.0750445798, 0.0178995766, -0.0371479169, 0.0330590345, -0.00693297386, -0.0297834575, -0.0112843839, -0.0334641933, 0.0730702057, 0.0495407023, -0.0375614166, 0.0798372179, 0.0701334253, 0.0323812068, 0.0275138989, -0.0734091774, 0.0645099729, -3.828090e-02, 0.0325940251, -0.00883802306, -0.020188896, 0.0166573841, 0.031759657, -0.00160587451, 0.0539456755, 0.0409828126, -0.00511133391, 0.0376681499, -0.0175561141, 0.0241899434, -0.0679971054, 0.0551168062, 0.0441827439, 0.0434544124, -0.00609742803, 0.0265973751, 0.0382097363, -0.0762897283, -0.0246851984, -0.0680042729, 0.0514911786, 0.0727452412, -0.0723966956, 0.0269038826, -0.0717858374, -0.0758363232, 0.0187571589, 0.0411070809, -0.0267314799, -0.00769382482, -0.00568846194, 0.0668572113, -0.0362846889, -0.0341448262, 0.0587879978, 0.0611457713, 0.0575748049, -0.0677224472, 0.0338781849, -0.056898009, 0.0331666358, 0.0305282325, 0.0193697978, 0.0504498556, -0.0745366961, -0.0485531464, -0.024619706, -0.0387094393, -0.0547079742, 0.0123323882, -0.0606945641], [-0.0136789083, -0.0420855433, -0.0251880977, -0.0264269691, 0.0755589753, -0.0805115997, 0.0631259307, 0.0285262372, 0.0530532673, -0.0527979508, -0.0628431439, -0.0217278767, -0.0602695569, -0.0311232433, 0.035830602, -0.0230961163, -0.0776850655, 5.724310e-02, -0.0255779363, 0.0222460125, -0.0761334747, 0.0542588457, 0.00447221706, 0.0184538383, 0.0709338412, 0.0678498819, 1.869170e-02, -0.0608009547, -0.0173469409, 0.0142967803, 0.0350508019, -0.0711791888, 0.0117063494, -0.0741804764, -0.0202476047, -0.023445759, -0.0639386252, 0.0359404981, 0.0508955941, -0.025617497, -0.0573512577, -0.0507307649, 0.0323046148, -0.0150902085, 0.0563838407, 0.0461291745, -0.0500635654, 0.0774198994, -0.0280773025, 0.0207317099, -0.0545251109, 0.0570617244, 0.0681408718, 0.0443425216, 0.0233501866, 5.993760e-02, -2.950130e-02, -0.0589168705, 0.0141486842, 0.0707828626, 0.00153517525, -0.0740454644, -0.0740778223, 0.0311869904, -0.0122291148, -0.0200404525, 0.0439461209, -0.00314067304, -0.0471247844, -0.0440039895, 0.0488644764, 0.0771825239, 0.0420560464, -0.038383361, 0.0231011063, 0.0315069631, 0.00367358327, -0.0533756837, 0.0524641648, -0.0656011552, 0.0460039899, -0.0424727276, -0.0799872875, -0.075905256, -0.0629550442, 0.0215387456, -0.0363548435, 0.0565153658, -0.06190655, 0.0729535744, 7.006990e-02, -0.0229459684, -0.0286138467, 0.0694592074, 0.0513308421, 0.0262434874, -0.0577480309, 0.0680856258, 0.0671212077, 0.0718105957, -0.0186461732, -0.0454364493, 0.0674143285, -0.0645777732, -0.0375269502, -0.0268431921, 0.0223369319, -0.0520829931, 4.362220e-02, 0.0656669512, 0.00974486489, 0.0546918325, -0.0612372793, -0.0516095534, -0.0247072335, 0.011853613, -0.0354762711, -0.0232423898, 0.0733728856, 0.0164456535, 0.0457815267, 0.0763336942, -0.0454038642, 0.0534107611, -0.0285878349, -0.0464950949, -0.0373258255, 0.0483016111], [0.0174809322, -4.641460e-02, 0.0368817225, 0.020582566, -0.0305860341, -0.0400532149, 0.0331172794, 0.0375292115, 0.0350890905, -0.0179199502, -0.0354071893, -0.0243387613, -0.0206636284, 0.0178974606, -0.0166905783, 0.0568646789, -0.0729902684, 0.00281898212, -0.0764507577, 0.0114218676, 0.0789140835, -0.00211168081, -0.0624693222, -0.0237689856, -0.0155652752, -0.0405345857, 0.0777150914, 0.00761431875, 0.0638413951, 0.0445470437, 0.0782143846, 0.0556699336, -6.857720e-02, -0.0395829901, 0.0770234391, 0.0173529331, -0.0515137687, 0.0493760034, -0.0688974187, -0.0664432868, -0.0557469167, 9.25765547E-4, -0.031028688, -0.0507593453, 0.0606457293, 0.0598230734, 0.0345274098, -0.0469250716, 0.0364392176, -0.055186972, 0.0802121683, -0.048287116, 0.068618454, 0.0756933615, -0.0724203288, 0.00580669194, 0.035576418, -0.0797603726, -0.0548395067, -6.390860e-02, -0.0647087693, -6.16058707E-4, 0.0219484605, 0.0525705293, -0.0644821525, -0.0316505507, -0.042738013, 0.00126714259, -0.0476502031, -0.0143568059, -0.0176798478, 0.0262775179, 0.0393215157, 0.0779400169, 0.0295088515, 0.0053141606, 0.0320794955, -0.0151528371, -0.0478595793, -0.0279991664, -0.0222133249, 0.066738084, 0.0483898446, -0.041066017, -0.0135962702, 0.0650811121, 0.028107971, 0.0154005354, 0.0790980085, -0.0395566337, -0.0102699986, -0.0557044111, -0.0590576082, 0.0474861786, 0.0148890987, 0.0194549318, -0.051027216, 0.054104507, 0.0306875333, -0.0704476163, -0.041689191, 0.0194183085, 0.0305452384, 0.034931697, 0.0211782232, 0.0356653295, -0.00686625158, -0.0674295947, 0.0132854348, -0.0568038225, 0.0799620524, -0.0375926122, -0.0193575621, -0.0252075605, 0.00317442371, -0.0695716888, -0.0160600152, -0.0779699385, 0.0282930173, -0.0356751457, -0.0223692451, 0.0172876101, -0.029620979, -0.036267858, 0.0272545516, 7.408220e-02, -0.0725668073, 0.0650055781], [0.021892041, -0.0126680955, -0.00147359818, 0.047741197, 0.031547226, -0.0377574898, 0.00330939889, 0.0206527039, 0.00352600962, 2.673930e-02, -0.0606619269, -0.0772631764, -0.00260995328, -0.0747953802, 0.0301898345, -6.943840e-02, -0.00145678967, 0.0176524445, 0.0122473687, 0.0114033669, 0.0233899057, -0.0104068667, 0.036023058, -0.0584451184, 0.0645341948, -0.0201555565, -0.0201573335, -0.0421810858, -0.0491432026, 0.047521539, -0.00661601871, -0.0180796087, -0.0586038083, 0.0508693233, -0.0625083074, -0.0690297037, -0.0507396385, 0.0249411687, -0.0221686549, 0.00496564806, 0.0386692286, -0.0609357581, 0.0360824242, -0.0398408249, 0.0366005525, -0.0189370103, 0.06471885, -0.0773891136, -0.0359318778, 0.057644628, -0.067070581, -0.0716085657, -6.684660e-02, 0.0439104512, -0.0671808869, 0.0404395759, -0.0145552009, -0.00282155722, 0.0764590129, -0.0238432921, 0.0354475901, -6.125830e-02, -0.00456347317, 0.00900464505, -0.0315895975, 0.0702406093, -0.0607020333, 0.0579438731, 0.0436175168, 0.0538637266, -0.0656564832, 0.065661557, -0.0752777383, 0.0621961877, 0.0503570363, -0.0742234662, 0.00384484231, -0.0529940389, -0.0752007663, 0.042793937, -0.0800621659, -0.0193209723, 0.0482078269, -0.0017914772, -0.0644129216, 0.0165656358, 0.0750996247, -2.705140e-02, -0.0685643851, 0.0267664492, -0.0435040966, 0.015007548, 0.0143389255, 0.0590970442, -0.0791797414, -0.00491090119, -0.0182520896, -0.0606447347, -0.0164543763, 0.0623212233, 0.0364091247, 0.061517261, 0.0776326581, 0.032741949, -0.0807915479, 0.0157312974, 0.0465283021, -0.0748779923, -0.0456483476, -0.0497669801, -4.798430e-02, -0.0405016728, 0.0238100886, -0.0452651381, -7.9434365E-4, 0.0557902977, -0.0118417516, 0.028263934, -0.0660466775, 0.0703492537, -0.0809061676, 0.0435193181, -0.0100934505, 0.0281349048, 0.0133237243, 0.0238249823, -0.0530000143, -0.0673385337], [-0.0121777132, -0.0231328793, 0.0329843909, 0.042540431, -0.0242635906, -0.0486384146, 0.0166834816, 0.0749972537, 0.0477150306, -0.0290407687, -0.00994230061, 0.0587649122, -0.0292561203, -0.0164117217, 4.877340e-02, -0.0424159318, 0.0306797475, -0.0267787687, -0.0620097667, -0.00381569564, -0.034472473, -0.018015869, -0.0425591506, 0.071274586, -0.025286179, 0.0406974107, 0.0308265463, 0.00723464787, -0.00623275339, -0.0374569893, 0.0724436566, -0.0491393544, -0.0757122859, -0.0110010728, 0.0495365486, -0.0547886714, -0.0300692767, 0.00837104768, -0.0383762382, 0.0590508357, 0.0750711039, 0.0400286168, 0.0591898188, -0.00647947192, -0.079015173, -0.0249461159, 0.056922473, -0.067727752, -0.0590355359, 0.0496281758, 0.0307994559, -0.059187226, -0.0795960947, -0.061880298, 0.0458390042, 0.076244019, -0.0753058344, -0.0763067082, 0.0247494429, -0.0800801516, -0.0225090832, 6.10299408E-4, -0.0338655785, -0.0479834452, -0.0365578569, 0.0348608643, -0.0437868237, -0.0285359062, -0.0681928396, -0.0435932837, 0.0620994195, -6.030440e-02, -0.0465204641, 0.0422857627, 0.0361057669, 0.0326809138, 0.0196948797, -0.00831425189, 0.0173114538, 0.024945572, -4.273590e-02, 0.031249322, -0.0250912718, 0.0564930364, 0.0140238851, 0.0166749135, 0.0454459861, 0.0581909046, -0.0297922157, 0.010189876, 0.00583076477, 0.0168477967, 0.058604084, -0.0343600772, 0.0463371947, -0.0469192229, 0.0505526885, 0.00594364852, -0.0245156251, 0.0248388872, -0.0233997479, -0.0371786729, -0.0127433017, 0.0522078648, -5.705280e-02, -0.0192790292, 0.0473408923, 0.0445035174, 0.0628836229, -0.0536155514, -0.0748484284, 0.061974965, 0.00892981142, -0.00416477025, 0.00353980064, -0.0344370268, 0.00570443273, 0.00279656798, -0.0187362581, 0.0642822161, 0.0778870508, 0.0315378457, -0.0617624111, 0.012160331, -0.0394563787, -5.907640e-04, -0.0410396829, -0.0488282032], [0.0158314705, -0.0668158159, 0.0456365272, -0.0158644468, -0.0698845386, -0.014770925, 0.0712128058, -0.0157236978, 0.0099408552, -8.79116356E-4, -0.0434088185, -0.0234074853, 0.0369848236, 0.0349726379, -0.0809211358, -0.0438181125, -0.0406209901, -0.0219996162, 0.0594783202, -0.0425320975, -0.0502706245, -0.0175027102, 0.0472478047, -0.0523962528, -0.0553243421, -0.0175438225, -0.0575283691, -0.0388182923, -0.0738687292, -0.021946419, 0.0371833518, -0.044189021, 0.0191095844, -0.0331223123, 0.0799534693, 0.0530470386, 0.0184725821, 0.0800997093, -0.0314834677, 0.0383365378, -6.808520e-02, -4.604480e-02, -0.0425341651, -0.0108998418, -0.0295635425, 0.0507206544, 0.0752466395, -0.0404904149, 0.048702769, -0.073580429, -0.0265680589, 0.0593358949, -0.039249111, 0.0674163625, -0.0685557052, 0.0304642245, -0.00834234804, -0.00981898605, -0.0651470795, 0.0263840556, -0.0659398511, -0.0375732519, -0.0779802054, 5.248090e-02, -0.0180157349, -0.0669220835, -0.0252798945, 0.0781426206, -0.0199302249, -0.038519863, 0.0216252878, -0.0480195917, 0.078455843, -0.0556685254, -0.0346313752, -0.0749661177, 0.0371341929, 0.0255748034, 0.0626188144, 0.0681031868, 0.00621772557, 0.0646820739, -0.0650970116, -0.0766022727, 0.0411983505, -0.0748956501, 0.0234619826, -0.0349441133, 0.0581948683, 0.0805782452, -0.0259068646, -0.0118482113, 0.00740728527, -3.825040e-02, -0.0194972232, -0.011942409, -0.0627213195, 0.0526134297, 0.0132155642, -0.0735368803, -0.0807589665, -0.0736690387, 0.0525124595, 0.0731584951, 0.0574549362, 0.0745250359, -0.027489569, 0.0674981549, 0.0657284781, -0.0171950385, -0.0413438752, 0.0168541223, 0.0595969036, 0.0446855351, -0.0773633495, 0.0361504927, -0.0141982734, 0.0509119406, -0.0723340958, -0.0655310601, 0.00484254211, -0.0770967752, -7.513590e-03, -0.0511702634, 0.0681328848, 0.0452468023, -0.0742698609, -0.0568483546], [0.0460562631, -0.015241541, 0.0634051189, 5.319380e-02, 0.0313800499, -0.0770041048, 0.0530825183, -0.0180803016, 0.0186507627, 0.0481667742, 0.0331764966, -0.0264113396, -0.0301210843, 0.00135485828, -0.0537439771, 0.0687967613, -0.010611102, -0.0542483218, 0.00137593597, 0.0779009386, 0.0143792257, -0.0518665761, 0.011944361, 0.0509677306, -0.020877745, 0.00478048623, 0.0586699024, 0.0330022424, 0.0663707629, -0.0174508244, 0.00119066238, 0.056731455, -0.0398240201, -0.0276872627, -0.0486507341, 0.00443156809, 0.0472384766, 0.0456034467, 0.0690060332, 0.0179420188, -0.0161909908, -0.0750302821, 0.0178668499, -0.0654848367, 0.0436255783, -0.0629336387, -6.772060e-02, -0.0547307916, -0.0673184767, 0.0283536017, 0.0475271717, 0.00256279111, -0.00145644695, 0.0772815868, 0.0132994503, -6.195560e-02, 0.0794521346, 0.0775316879, -0.00302451104, -0.025062941, 0.0753882304, 0.0198363215, -0.0470852219, 0.00103273988, -0.0302237123, 0.0792328491, -0.0618642047, -0.0510829911, 0.00243053585, 0.0390227959, -0.00678621232, 0.0126989037, -0.0650197342, 0.0703525171, 0.0411827639, -2.096910e-02, 0.0246252343, 0.0251838267, 0.0284914672, 0.0802692547, -0.0455715172, -0.0570708029, -0.048629906, 0.0777473375, -0.0319932252, 0.0232114717, -0.0669527724, 0.0776922777, -0.0754522607, 0.0206220523, -0.0527444407, -0.0630040466, -0.0220401324, 0.0219354555, 0.0124831423, 0.0658475086, -0.0442348346, -0.0239963531, 0.00890145451, 0.0472257212, 0.0458733514, 0.0503139123, 0.0636279061, -2.791940e-02, 0.0272351354, -0.00420675427, 0.0604266599, -0.0699536875, -0.0286733434, 0.0378898755, 0.0302547291, 0.0154187754, 0.0471693948, -0.0373546332, 0.00842265784, -0.0242228061, 0.0618076101, -0.0374069065, -0.0221411735, -0.021166563, -0.0037619397, -0.00787399709, 0.0767449364, -0.0757666677, 0.0127781332, -0.0373151079, 0.030982025, -0.0476917662], [0.0513143316, 0.035593845, 0.0307156369, 0.0328322574, 0.0744690225, -0.0262031443, -0.0543507338, 0.0468384847, -0.0658857599, 0.0139158443, -0.0575331077, -0.0405399799, -0.0697829351, 0.060859777, 0.0788866952, -0.0564333163, -4.154650e-02, 0.00418894738, -0.0712000429, -0.027980838, 0.0554005876, 0.0492660925, 0.0729243234, 0.0136320889, -0.032990925, -0.0494985059, -2.306670e-03, 0.05257494, -0.0317989513, -0.0122517198, 0.0226971507, 0.00739692151, -0.00722645223, 0.0455635414, -0.0372277163, -0.03781268, 0.0255826786, -0.0684130266, 0.0612839833, -0.020078782, -0.0763723627, -0.00658176839, -0.070680134, -0.0310191549, 6.850540e-02, 0.0184101388, 0.0227946714, 0.0297272429, -0.0783394724, 0.044945322, -0.038309712, 0.00814813375, -0.0373152792, 0.0214968771, 0.0275496319, -0.0186824016, 0.00623373687, -0.0135731101, -0.0249601193, 0.062158592, 0.0154487491, -0.0722853243, 0.0125008598, 0.0138475373, -0.0306622684, -0.0676171779, 0.0623291954, 0.0576101318, 0.0794587954, 0.0417920575, 0.0307970345, -0.0233472064, 0.0367598236, -0.0445772968, -0.0578720495, -7.762270e-02, -0.076968886, -0.0395502262, -0.00514729321, 0.0255594924, 0.0290263817, 0.0683274493, -8.04439187E-5, 0.0170174912, -0.0438829139, 6.318580e-02, -0.0398576483, 0.0497049764, -0.0540244803, -0.0256490633, 0.0373665243, -0.0238512605, -0.0788771957, -0.0800801739, 0.0735132769, -2.125080e-02, 0.0316977575, 0.0398630425, 0.0153384432, 0.0599016622, 0.0197065398, -3.382400e-02, -0.0319343396, 0.0349508822, 0.0540789142, 0.0802756175, -0.0447428301, 0.0615991727, 0.0683684275, -0.0553965122, 0.0720240846, 0.0397217944, -0.0222696178, 0.0305190831, 0.0242281407, 0.00115501881, -0.0378805585, 0.052168794, -0.0549870245, 0.00114648789, 0.0729921833, 0.0585170612, 0.078520812, -0.00286398083, 0.0155549571, -0.0274847336, -0.00888767093, 0.0507066324], [0.00360746682, -0.0232176781, 1.790990e-02, -0.0539817624, -0.0786881819, -0.0487547554, -0.0121147111, -0.00671042501, 6.484900e-02, -0.0517261028, 0.0504849926, 0.0490739569, -0.0294621699, 0.0230340809, 7.959060e-03, 0.00168782473, -0.0300442539, -0.0454458743, 0.0772251859, 0.00106754899, -0.0687563419, -0.0449292921, 0.0225231647, 0.0356146321, 0.0640026107, -0.0759122223, 0.0624244735, 0.00983677059, 0.0548020378, 0.0129846632, 5.725210e-02, 0.0425605401, 0.0520728752, 0.0202839226, 0.012185894, -0.0091189593, 0.00387303531, 0.0142538175, -0.0177330077, 0.0347755253, 0.00380925834, 6.41956925E-4, -0.0590894893, 0.0490267053, 0.058987014, 0.0121833831, -0.00462400168, -0.0128983557, 0.069543995, 0.0763487294, -0.0128107145, -0.0667245388, -0.0102227852, -0.0264727771, -0.0750788227, 0.0727780089, 0.0477486029, 0.0461298898, -0.0472095273, -0.0604956374, -0.012331605, -0.07752572, -0.0369202755, 0.0764744654, 0.0735912248, -0.0796673745, 0.0579487309, 0.0666805133, -0.0330339745, -0.0623635203, -0.077176392, -0.0799832269, -0.0677007734, -0.0638707876, 0.0153321549, -0.0391282439, -4.486000e-02, -0.0488484278, 0.011817053, 0.0301378518, -0.0378195085, 0.0103608221, 0.034797743, 0.0750919953, -0.0112257451, -0.0271001905, -0.0163152963, -4.792520e-02, 0.0150561407, 0.016152449, 9.90137457E-4, 0.00769459456, 0.00430246443, 0.0809243693, -0.0502194166, -0.0640687197, -0.0156931654, 0.0286722407, -0.0525898114, 0.0445496216, -0.0787680447, 0.0182594135, 0.0345485881, -0.0234704316, -0.0527714342, -0.0671848282, -0.0324390121, 0.038716048, -0.0145723373, -0.0590739399, 0.0301160961, -0.025704138, -0.0434084684, -0.0637601167, -7.987350e-02, 0.0490977392, -0.0250484943, 0.079666756, 0.0632868633, 0.0384635851, -0.0796764865, -0.0164464712, -0.03731592, -0.0739921853, -0.0438828357, -0.0437074192, -0.0771215633, -0.0365388282], [-0.0267997719, 0.0529277697, 0.0558118448, -0.0312567838, -0.022881791, -0.0475537889, 0.0497104898, 0.0339821503, 0.0204115584, 0.0803403928, 0.0591197386, 0.0726199225, -0.0112313777, -0.0255391076, -0.00658987462, -0.00859322399, -0.0535765477, 0.0517485365, 0.0297474861, -0.00192587823, 0.0742135718, -0.0317851454, 0.0773697123, -0.0471559614, -0.0575816631, 0.024678573, 0.0173471868, -0.00535562634, 0.00534588099, 0.0765665248, 0.0734677836, -0.0265386067, 0.0680623427, -0.0288291126, 0.0563480482, -0.0714612231, 0.0689340159, 0.0502944067, 0.0358146653, -0.0740652978, 0.0758814439, -0.0391185582, 0.0332590118, 0.0503215119, -0.0733163133, 0.0031991154, -0.0516951457, 0.0366936103, 0.0755069181, -6.544000e-02, -0.0418363214, 0.00641810894, 0.029659383, 0.0700038299, -0.0466077588, 0.0383342132, -0.00807389616, -0.0164565444, 0.0615033284, -0.0152302831, -0.0531061627, 0.0330522507, -0.0787331983, 0.0489042625, -0.0237164535, 0.0653279051, -0.0250353068, -0.0589157939, 0.00145911425, 0.0468214527, 0.0658397749, 0.0330978855, 0.0580010787, -0.0574349463, -0.0793908759, -0.0803661644, -0.0349297449, -0.041995205, 0.0656168684, -0.0548642837, 0.0307448208, -0.016235061, -1.6348809E-4, 0.0752508566, -0.0597304404, -0.0749791562, -0.0573472641, -0.00480373204, 8.96096229E-4, 0.0223733485, 0.0641877428, 0.00617688149, 0.0279707462, -0.0374890529, -0.0344286524, 0.0477083251, 0.0479613468, -0.0621865168, 0.0368280932, -0.0171734169, -0.0233569928, -0.00330023468, 0.057968162, 0.0699248537, -0.0702413618, 0.0456837043, -0.0568313189, -0.0482171513, 0.0671506599, -0.0768865496, 0.0357165039, 0.0555250719, 0.0592291281, -0.0552519597, 0.0664143935, 0.0709855631, 0.0459399298, -0.0408408456, -0.0085471794, 0.0633512661, 0.0183049589, -0.00350485742, -2.219060e-02, 0.0183380842, -0.0593010485, 0.0491652414, -0.00888320059, 0.00720583647], [-0.0710644647, -0.0354879461, -0.0466161519, 0.0596870407, -0.0151634142, 0.0738424137, 0.0777263119, 0.0046178326, 0.0444288924, 0.0283696726, -0.0439791791, 0.0752908066, 0.0236053169, -0.0490088388, -0.0467454679, -0.0190929733, 0.0239557251, -0.0547192097, -0.0384211205, 0.00113216043, 0.0745771602, 0.00449035317, -0.0583053827, 0.0297093689, -0.0224425793, -0.0568786189, 0.016669631, -0.0675901249, 0.0200121403, 0.0106964558, 0.0253897384, 0.0525413081, -5.055720e-02, 0.0161912218, -0.0311796032, 0.0436843112, 0.0427863151, -0.0738267824, 0.0576934442, -0.0543049239, 8.052530e-02, 0.0646930262, -0.0632201135, 0.0102074891, -0.0277186893, 0.0227162763, 0.0633490756, 0.0724514648, 0.0445165858, -0.0468614772, 0.0796314701, 0.00694051385, 0.0505819395, 0.0746282563, 0.011250928, -0.0558811687, 0.0179945603, -0.0749581531, -0.0701648071, -0.032591261, 0.0155370831, 0.0427234098, -0.0497887358, -0.0590221547, -0.0172910914, -0.0330119468, 0.0559654906, 0.0544659719, -0.00497609377, -4.322060e-02, 0.0300056562, 4.829570e-02, -0.00123515725, -0.0436272435, -0.0674273148, -0.0170014203, -0.00519397855, 0.0479815379, -0.059399195, -0.00958922505, 0.043396689, 3.505850e-02, -0.0615264624, 0.0761832669, 0.0227393061, -0.00609268248, 0.062159054, -0.0635209977, -0.0039511025, 0.0722284541, 0.0142329708, -0.0322979763, 0.00417663157, -0.0331591107, 0.0694175139, 0.0524535403, -0.0584361851, 0.0711819455, 0.0236260667, -0.0711970627, -0.0220674165, 0.0356367752, -0.0309568457, 0.0587523207, 1.849670e-02, 0.0141497403, -0.0119422898, 0.0140391961, -0.0435373597, -0.0126213953, -0.0444754027, 1.6143918E-4, 0.0746151134, -0.0139076784, 0.0792838558, -0.0598799065, 0.00213697553, 0.0327250436, -0.0784735083, -0.0298774019, -0.0139016658, -0.0307681821, -0.0579526275, 0.0561740324, 0.044024758, 0.0706609264, -5.689680e-02, 0.0412498713], [0.0434301049, 0.066448994, 0.018126525, -1.000490e-02, -0.07103692, 0.079838477, 0.0235469714, -0.0809892043, -6.723620e-03, 0.0109627843, -0.0732200443, -0.0319556892, -0.0598010048, -0.0355415307, -0.017682381, -0.0714768693, 0.0552171096, 0.0549986139, -0.0255647302, 0.0758973584, 0.0237364694, -0.0342859738, -0.0322446637, 0.017799899, -0.0297678709, 0.0671020076, -0.046035964, 0.0677262321, -0.0486933179, -0.0774955078, -0.0629193782, 0.0375745073, -0.00122948736, 0.047371082, -0.0279431082, -5.071730e-02, -0.0279888064, -0.0610909462, 0.06026005, 0.0100442171, 0.0306770429, 0.0520082936, 0.0404964909, -0.0702275931, 0.0805662199, -0.0546475574, 0.040587686, 0.00938605517, -0.0165360048, -0.0684033185, 0.0148869306, -0.0228780396, -0.0184102133, -0.0158875361, -0.0068147555, -0.0374717079, -0.0390044414, -0.0430645943, 5.597850e-02, -0.0707690269, 0.032099627, 0.0120280907, 0.00470092893, 0.0552032068, -0.0804549307, -0.0304690376, 0.0335558578, 0.0115713403, 0.0283350199, -0.0641861781, 0.028611809, -9.3922019E-4, -0.0324258432, 0.0236358345, 0.0629628375, -0.0313322023, -0.0214364864, 0.0741459802, 0.0269661769, -0.0708575249, -0.0625887737, 6.932070e-02, 0.0543691888, 0.0715603605, -5.499950e-02, -0.0798603296, 0.0236089528, -0.0409409404, 0.0542705283, 0.0268585607, -0.0237596929, -0.0761118904, -0.0439750627, 4.81523573E-4, -0.0333110541, -6.620210e-02, -0.0366295055, -5.649980e-02, 0.0714975223, 0.029192403, 0.0407890156, -0.0385254137, -0.00276286155, -0.0347020961, 0.0684295818, 0.0798444226, -0.0130795762, -0.00867562741, -4.24012542E-4, -0.0747750774, 0.0127364174, 0.0576106384, -0.0225954317, -0.0026377216, 0.0417537689, 0.046165131, 6.096200e-02, 0.03290236, 0.041309081, -0.0660678744, -0.0111337155, -0.047307495, 0.0309392288, -0.00283644348, 0.0387914106, -0.0344649106, -0.0707609057, -0.0144959912], [0.0267277136, 0.0142065734, 0.00256735086, -0.0778853297, -0.0119148269, 0.00965046882, 0.0683022812, -0.02964627, -0.0784248709, -0.0670523271, 0.0584193394, -0.070148617, -0.0726420805, -0.0503699854, -0.0571109504, 0.0207223967, 6.876040e-02, -0.0457210205, -0.0209963284, -0.0741831288, 0.0418816507, 0.0104024038, 0.0286560133, 0.0456634685, -0.00205722451, 0.0355714485, 0.0486604199, 0.0143635198, -0.0561406128, 0.0161428228, -0.0414152108, 6.785140e-02, 0.0503018126, -0.0326178893, 0.0184782073, 0.0752559826, 0.0782385393, -0.0259675868, -6.805960e-02, 0.0411066115, -0.0451547168, 0.0318179652, 0.0736447498, -0.0726102292, 0.0533598587, 0.0607466474, 0.0319465846, 0.0433490425, 0.0778995379, 0.0690845475, 0.0564944074, -6.335980e-02, 0.0150835067, -0.0328350402, 0.018355757, -0.0275765508, 0.0203324631, -0.0691984519, -0.0136496499, -0.0135439485, 0.0207903162, 0.0630172864, 0.0368307978, 0.0571425632, -0.0767788514, -0.0277580619, -0.0534227863, -0.056261979, -0.0300902799, 0.00895166397, 8.56257975E-4, 0.0199515596, 0.0715691671, 0.00301545858, 0.0286021382, -0.00972596555, 0.0447541699, -0.0651780218, -0.0747511759, -0.0504187942, 0.0546945259, 0.0513339415, 0.0380852148, 0.070698075, -0.0652814955, 0.0638680384, -0.0453982651, 0.0678526387, -0.0468287356, -0.031573642, 0.0337464362, -0.0491197258, -0.0158930048, -0.0764242485, -0.0523517579, 0.0720332339, -7.785870e-02, -0.0743256733, 0.0218669176, 0.0205936655, 0.0325780362, -0.0176208466, -0.00908308476, 0.0199167505, 0.0517718568, -0.00182211399, -0.0384752676, -0.0569681562, -0.0526656732, 0.0480426624, -0.0783037319, 0.0356100127, -0.0198806226, -0.0166635588, 0.0108531192, 0.03584373, -3.862340e-02, 0.0146613717, -0.0555808097, 0.025425069, -0.0201935172, -0.053565681, 0.0342802107, 0.0183811337, -0.0292070024, -0.0192364268, -0.0453624092, -0.0122092739], [0.00779646635, -0.0765021369, 0.0183037966, -0.0380717553, -0.0387442447, 0.0246863663, -0.0632274225, 0.0222978517, -0.0624648184, -0.032133583, -0.0732190385, -0.0422314033, 0.0454909876, 0.0437588394, 0.0566993281, -0.0205247253, 0.0434534475, 0.0660348162, 0.00732828676, 0.036251232, -0.0713760182, -0.0809879302, 0.0276680961, 0.0317612812, 0.0292111188, -0.00981219857, -0.00243055075, -0.0574066713, 0.0466913804, 0.0738137439, 0.0711864308, -0.00158746541, 0.0220104679, 0.0418347791, 0.0432814956, -0.0678660199, 0.0747841224, -0.0561214313, 0.0371705145, -0.044906415, 0.0103961378, 0.0511863455, -0.0688610151, 0.0193349719, 0.0161310807, 0.0321609303, 0.00442195684, -0.0292671807, -0.0354069583, 0.0369686633, 0.0578918234, 0.0362951681, -0.0353302434, 0.0266827717, 0.0750099644, 0.0057547465, 0.0410894379, 0.0202035755, 0.0392746925, -0.0742286891, 5.290640e-02, 0.0445922688, -0.0566638671, -0.0716542229, -0.00645243376, 0.00641199946, -0.0377107859, 6.354920e-02, -0.042943228, -0.0598969236, -0.0538583621, -0.041321326, 6.297500e-02, -0.0583102927, -0.0739476457, -0.0497737862, 3.959360e-02, 0.0783971175, 0.0347915366, -0.0751156806, -0.0581318215, 0.0305041149, 0.0266666636, -0.0118427947, -0.02760024, -0.0677571297, -0.0463031605, 0.00118541718, 0.020286493, -0.00976833701, -0.050805077, -0.0596177727, -0.0189481117, 0.0698379353, -0.0776106715, 0.0504129753, -0.0639711171, 0.027722694, -0.0404095836, 0.0561268553, 0.0185658485, 0.035733752, 0.0473435298, 0.0656619444, 0.0186201334, -0.068807587, -0.0146028697, 2.268210e-03, 0.0254782513, 0.00186875463, -0.0504876412, -0.0190368146, 0.0100642517, 0.0407108366, -0.0305478796, -0.0487964302, -0.05053268, 0.00382778794, -0.0478851311, 0.00939046592, 0.0288649425, 0.018654421, 0.0185964406, 0.0324893892, 0.0228622034, -0.0717884675, -0.0245982409, -0.0686992704], [0.0253379866, -0.0659312233, 0.0483250245, 0.0636715814, 0.0743371621, -0.045603849, 0.03270033, 0.0227596536, -0.00547778606, 0.0578754321, -0.06981536, 0.00277189165, 0.0101546794, -0.0126724094, -0.0317887813, 0.0162132531, 0.055051215, 0.0501077548, 0.0784560814, -0.00620927662, 0.0797173604, -0.0356176272, -0.00474660844, -0.0251957551, -0.0147441402, -0.0334116891, -0.0650056377, -0.070841454, 0.0544561669, 0.0364892259, -0.059805721, -0.0114190355, -6.803210e-02, -0.0738505125, -0.02823212, 0.0475000665, 0.0633790419, -0.075959295, 0.0794819965, -0.0641422793, -0.0480877571, -0.0687717348, 0.0392865464, 0.0276142806, -0.0139361247, 0.0658091232, 0.028405413, 0.0116544962, -0.0204103366, -0.0295394063, -0.00166485459, -0.0420133062, -0.0795177146, 0.0788347795, 0.0536214337, 0.0662148222, -0.0183112398, 0.073588483, -0.0564868264, 0.0439992324, 0.0659275874, 0.0726867095, -0.0316926688, -0.00463997573, -9.6412748E-4, 0.0310522392, -0.00480221957, 0.0136254355, -0.0182963163, -0.0306493677, -0.00347188115, -0.00358097255, -0.0530931503, -0.0353935361, 0.043665126, 0.0366625935, 0.0234967694, -0.0594971627, -0.0753127933, -0.0784428343, 0.0705457702, -0.0209645927, -0.0263929293, 0.0316885114, 0.0689629242, -0.0564012341, -0.0693526566, 0.011307396, 0.0778343752, -0.0408611111, -0.0652514696, -0.0382318571, 0.0394062921, -0.0198042952, -0.0476022884, -0.0115253329, 0.0476323143, -0.0776547789, -0.0610329509, -0.0540966913, 0.0575843677, 0.0212581679, -0.0571514815, 0.0450773463, 0.0423435271, -0.0342469104, 0.0102152452, 0.0387678593, 0.0464796349, 0.0418901816, 7.536080e-02, -0.0613702871, -3.248930e-03, 0.0629957542, 0.0625108406, 0.0128978714, 0.0305918157, -0.0773793235, 0.023193039, 0.0156961828, -4.167860e-02, -0.0130586326, -0.0709837452, 0.0627812818, -0.0136915371, -4.949540e-03, -0.00230870396, -0.0666180477], [-0.0237542391, -0.0423029363, -0.0515226051, 0.0234193578, 0.0690749809, 0.063725926, 0.0105728507, 0.038189061, 0.0534764752, 0.0690470263, 6.360320e-02, 0.0337532461, 0.0485799685, -0.0368406214, 0.068385832, 0.0222337097, 0.00744608045, -0.0752419829, -0.0515501648, 0.013422057, 0.0620983914, 0.0141454861, -0.0372699127, 0.0731451884, -0.0490155704, 0.00264385343, -0.0407672822, 0.0183163658, -0.0700190738, -0.0226208605, 0.0328446105, -0.0724501311, -0.0484782159, -0.0652943775, -0.0363019742, 0.0759229586, 0.00248768181, 2.36876309E-4, 0.0619841591, 0.0511870608, 0.00919501483, -0.0638582185, 0.0561680868, -0.0261993557, 0.0410195291, -0.0216445066, 3.05324793E-5, 0.0606916174, 0.00966284424, 0.0362413079, -0.0692545473, 0.0653496608, -0.0602759123, -0.0446237661, -6.447480e-02, -0.0169030502, 0.0754348263, 0.0578211918, -0.0752221346, 0.0480545089, 0.0735204443, 0.0756190792, 0.0651287213, 0.0424593836, -2.906270e-02, 0.0469914451, -0.00947090983, 0.0377130881, -0.0147870332, -0.0405116118, 0.0730514452, -0.0281385593, -0.0264126547, -0.0526645146, 0.0638698563, 0.00159517676, 0.00921340286, -0.0581135228, 0.00681831688, -0.0139020681, -2.203520e-02, -0.0405495912, 0.0168216303, -0.0555878058, -0.0700701252, -0.0624342412, 0.07591445, 0.0339058414, 0.0293379053, 0.0409281775, 0.0642735437, -0.0597959757, -0.0184710324, 0.0731125101, -0.0770261287, -0.0346737057, 2.69904733E-4, -0.0106578395, 0.0136639029, 0.0207942054, -0.0539367422, -0.0708322451, -0.0669369921, 0.0765181109, 0.0123763978, 0.0487310216, 0.0523358211, 0.0407368466, -0.00588685274, 0.038185522, 0.0270975828, -0.0605276972, -0.0233536065, 0.025529556, 0.0468525067, -0.00331068039, -0.0426290594, 0.072878547, -7.812290e-02, -0.00478160381, 0.031189777, -0.0118730813, -0.0471217521, -0.0529317111, 0.0629719123, 0.0158714429, -0.0287901834, 0.0202849656], [-0.0765253305, 0.020577766, 0.0243412331, 0.0301926583, 0.0609879568, -0.0732739419, -0.0423150025, -0.0187586918, -0.026217185, 0.0754509494, 0.0405544266, 0.0699989423, 0.0038530007, -0.0159322843, 0.0118203238, -0.0110446066, 0.0522414073, -0.0286111683, 0.0372111648, -0.0222549029, -0.00353500247, 0.0719062313, 0.0799521133, -8.037140e-02, -0.0806282759, -0.00668644905, -0.0242776871, -0.014575392, -0.00590481609, -0.0756255761, -0.0114520863, 0.0612579957, -0.0105544552, -7.220020e-02, 0.0150824264, 0.00997246801, 0.0153680146, 0.020330064, 0.0208889246, -0.0802022368, -0.0186780915, 0.0143077895, 0.0260164738, -0.0210314058, 0.0421168804, 0.0404542312, 0.0756903961, -0.0388663076, -0.080183804, -0.0647566542, -0.079668343, -0.0545978807, -0.0404556841, -0.0578269698, -0.0615448356, -0.0105147734, 0.0123278573, 0.044525139, -0.031856969, 0.0652536824, -0.0314375386, -0.054607667, 0.0293322206, -0.033725705, -0.0332848094, -0.0745189562, -0.0134504288, 0.0509518608, 0.0746193901, -0.0526816845, 0.00519927591, -0.0268229358, -0.0763065368, -0.0178560987, -0.069552362, -0.00545531511, -0.0467026718, 0.0249839053, -0.00187209994, -0.0190789551, 0.0446410701, -0.010826081, 0.021568276, 0.0262073055, 0.0537095144, 0.0249737725, -0.0525524095, -0.0306347683, -0.0385993607, 0.0504605845, 0.035939686, -0.0551275536, 0.0144628063, -0.0379622243, 0.0530667827, -0.0175839663, -0.056794636, 0.0325052813, 0.00870339572, 0.00246323645, -0.0786475688, -0.0155597106, -0.00112415105, -0.00874435157, 0.0707026944, 0.0249262527, -0.0371686555, 0.0102801993, 0.0412464291, -7.56159424E-5, -0.0596050322, -0.0662232712, 0.0785772725, 0.0369450897, 2.167160e-02, -0.0593420863, -0.00932278484, -0.0514361449, 0.0716999546, -0.0669470504, 0.0572201684, -0.0565217882, 0.064502649, -7.676330e-02, 0.0085490346, 0.0375917777, 1.85146928E-4, 0.0119322315], [0.0309417248, -0.0480547845, 0.0549776033, -0.065106295, -0.0203878842, -0.078331545, 0.0236844122, 0.0461893901, -0.011960797, -0.0527071729, 0.00661930442, -0.0546880737, 0.0457982048, 0.0253852531, 0.0545989051, -0.0510043576, 0.0157192349, 0.0727728829, 0.062697269, -0.0732744634, 0.0770566836, -0.0294159502, 0.0181874037, -0.0473377407, 0.0446192548, 0.0280659273, 0.0313492417, 0.0696426257, 0.0301418528, 0.0306297764, 0.0404180139, -0.0238175131, -0.0280597769, -0.0273078643, 0.0431792364, -0.0119759217, 0.0107054338, -0.0102059394, 0.0231728926, -0.0180209354, -0.0467037745, 0.0757083073, -0.022852283, 0.00444885343, 0.0367253646, 0.0798475518, 0.0632155612, 0.0509558544, -0.0736482739, -0.0665107742, -0.0567055047, -0.059867315, -0.0550834462, -0.00739057362, -0.0260511637, -0.0606821738, 0.01423347, -0.012324959, 0.027057372, 0.0214725733, 0.00589804351, 0.00767809898, -0.0440057516, -0.0759687721, 0.0809124186, -0.0524847656, -0.0250317268, 0.052726306, -0.0487152077, -0.00388922542, -0.0166868865, -0.0308797061, 0.0491973236, 0.0587774441, 0.0241534561, 0.0428740755, -0.00320590287, -0.0654251575, 0.00958131253, -6.366170e-02, 0.0623069182, 0.0403254628, -0.0206797011, -0.019413352, -6.572800e-02, 0.0804360285, -0.0186660439, -0.06573686, 0.0640596673, 0.0146963522, -0.00435295701, 0.013023071, 6.136080e-02, 0.051857926, -0.026899226, 0.0256805643, 0.0430377126, -0.0155769065, 0.0422028229, 0.0568578914, 0.010397099, -0.034578871, 0.0777260735, -0.0349571109, -0.0378781222, -0.00778198242, 0.0164415017, -0.0512639359, -0.00944501907, -0.0746566504, 0.0486096963, 0.0175983533, 0.0346147418, -0.0343167409, 0.0630081818, 0.0752824172, -0.0611392334, 0.0561729595, 0.00294004381, -0.06805256, 0.0741778538, 0.035268575, 3.892900e-02, -0.0128229335, 0.0617452934, 0.0271580145, -0.00896316767, 0.0633022562], [-0.0713053197, -0.046072416, -0.0251994878, -0.0577850454, -0.0370956734, 0.0106397942, 0.0520577952, 2.968990e-02, -0.0622702538, 0.0502588227, -0.0392928347, 0.0275275856, 0.0739095882, 0.00881184637, -0.0375209227, 0.0237910375, -0.0672146082, 0.0457329974, -0.0139340013, -0.0134413391, -0.0766609684, 0.0456582233, -0.0297396183, 0.0441545174, 0.00413062423, 0.0187779143, 0.0475957319, 0.023636952, -0.033773914, 0.0063040331, 0.00510627776, 0.0122190192, 0.0491309837, 0.0679681823, -0.02025567, -0.0733583495, 0.0285223648, 0.0343666375, 0.0144643486, 0.0577761456, -0.0554381087, -0.016595915, -0.0143462121, -0.0139297843, -0.0539127067, -0.0160578638, -0.0179717764, -0.0118428543, 0.00201156735, -0.00142109394, -0.0457138829, 0.0690710768, 0.0117017627, -0.0611831695, -0.048451703, 0.0157753378, -0.0179436579, 0.0277998149, -0.0801605209, -0.0702770054, 0.0316354632, -0.066380918, 0.0376644358, 0.0271859542, 0.022500366, 0.0673423931, -0.0113851503, -0.068492122, 0.0502718762, 0.0639564171, -0.0325031579, 0.0635685548, 0.00932711363, -0.044226516, 0.0258613974, -0.0527431443, 0.0480395034, 0.0188027471, 0.0270649344, 0.0232070982, 0.0249682963, 1.863560e-02, 5.024270e-02, 0.0456708595, -0.0460819118, -0.0520582944, -0.00521814823, -0.0547008961, 7.37488269E-4, 0.0802157744, 2.41398811E-4, 0.0776657238, -0.00653280317, -0.0725055933, 0.0274756253, -0.0650250167, -0.035867285, -0.0390237607, 0.00283516943, 0.013446711, 0.0659811422, -0.0121594369, -0.0156034753, -0.0297826827, 0.0518954918, 0.0632864758, 0.0544084087, 0.0504891351, -0.0282749161, -0.0410538763, 5.029600e-02, 0.0424815491, -0.0585142337, 0.0384590626, 0.062678732, -0.0142115206, 0.0806857123, -0.0556417406, 0.0740640387, 0.0214552283, -0.0161712468, -0.0365068056, -0.00805395842, 0.0624124333, 0.0663357452, 0.0663696602, -0.078185752, -0.0521083437], [-0.0117153898, 0.0720385611, -0.00424954947, -0.043886207, 0.0320489854, 0.0804733559, -0.0280434899, 0.0184344966, -0.0153360935, 0.0235121511, 0.0588391796, -0.0264622774, -0.0416001864, 0.0472409539, 0.0639031231, 0.0569554195, -0.0312462263, 0.068562068, 0.0602318272, -0.0290993322, -1.90691731E-4, -0.0499265455, -0.0280150082, 0.0669901744, 0.0542183071, -0.0596841052, -0.0287026539, -0.028770335, 0.0724120066, -0.0596762151, -2.45884061E-4, 0.00458103558, -0.0772641376, 0.00361423218, -0.0638251752, -0.0618030354, 0.0545887798, 0.0557331294, -0.0378548801, -0.0407231785, -0.075122647, 0.0538854487, -0.0572594292, 0.0480417721, 0.010501381, -0.0383633859, -0.0357759111, -0.0397952199, -0.0774469823, 0.0428928919, -0.0597768314, 0.0333963074, 0.0162061416, -0.0776678919, -0.0407673195, -0.0459276065, -0.0568426698, -0.0333636329, 0.0419791564, 0.0571958646, -0.0331750177, -0.0600085407, -0.0208307076, -0.077133216, 0.0348607711, -0.0593847409, -0.0176550932, 0.0265807062, -2.350000e-02, -0.0531000569, 0.00498849247, -0.00154886546, -0.0352925546, 0.053040415, 0.0138749629, 0.075472258, 0.0432821512, -0.0717399418, -0.0447451435, 0.0672061071, 0.00708551704, 0.0501251705, -0.0439916141, 0.00168036215, 0.0578741208, -0.0260258559, 0.0602315404, -0.0757565945, -0.0541174896, -0.0470950827, 0.0741106346, -0.0373919867, -0.00124193751, -0.0428989232, -0.00134725869, -0.0459820107, 0.0430832431, -0.0147777926, 0.0198419522, -6.261780e-02, 5.74046338E-4, -0.0296138953, -0.0667065233, 0.0493606552, -0.0435005464, 0.0109864194, -0.0736760125, -0.0663842782, 0.0422871783, 0.0231865067, 0.0579480082, -0.0207496881, 0.0433615446, 0.0577101633, -0.0469471067, 0.0733617172, 0.0287193935, -0.0244162157, -0.0366531722, 0.0247567967, 0.00238203257, 0.0137247257, -0.0420490839, -0.0164373983, 0.0209685937, 0.0441238359, -0.0475347973, 0.0505634844], [0.05296316, -0.0398117676, 0.0746537744, 0.0611612685, -0.0379924662, 0.00312269921, -0.040799547, -0.0340013951, -0.0164810903, 0.0581890531, -0.0319439918, -0.0693680495, -0.013921259, 0.0330577604, -0.0330153815, 0.0473587438, -6.874220e-02, 0.0253047422, -0.0376502201, -0.0218790881, -0.00352773233, -0.0642566085, -0.0514830723, 0.0305149965, -0.00788840372, 0.0132782822, -0.036990542, 0.0559810661, 0.0707465038, -0.0139978994, 0.0810412988, 0.0602994896, -0.0508074313, -6.321620e-02, 0.00803605839, 0.0141397417, -0.0210375711, 0.0681038871, -0.0657349526, 0.0249489378, -0.0582703538, -9.10411181E-4, -0.0573415384, 0.0539651401, -0.0172865931, 0.0293986183, -0.0792233571, 0.0562053695, 0.00882869586, -0.0181577858, 9.12868068E-4, 0.0646871105, -0.0476605855, 0.0177030917, 0.0713635162, 0.0722009316, 0.0640913174, -0.0606769398, -0.052177906, -0.00867920369, -0.0512759164, -0.0484324917, 0.00711345347, 0.0280241016, 0.0515975915, -6.178120e-03, -0.0573438741, -0.0399115235, 0.0405861065, -0.0227043182, 0.0570675582, -0.0692851394, 0.0573233031, 0.0695203915, -0.0566493049, -0.00280940463, 0.0340522155, -0.0201733354, 0.0239323806, -0.0147870202, 0.0077356277, 0.0409141034, 0.0591301844, 0.0351615883, 0.0415944979, 0.061576806, -0.0187986642, -0.00288560195, -0.0789331645, -5.992380e-03, -0.0458672419, -0.0764353722, -0.0129566668, -0.0624593682, 0.00449278532, 0.0283632725, -6.774390e-02, -0.02653905, 0.0492935367, -0.0638302192, 7.862640e-02, -0.0490997024, -0.0531808287, -0.0189033896, -0.0431454554, -0.0253330804, -0.00958814938, 0.0710085332, 0.0526762754, -0.0199089702, -0.066706337, -0.0717912391, 0.0647336096, -0.0530668087, -0.0502543338, 0.00630649133, 0.0239793584, -0.0781333596, 0.0128825959, 0.0575421378, 0.0272670835, 0.0339120403, 0.00279668695, -0.0269566178, 0.0805966928, -0.0687877833, -0.0428953208, 0.0632181838], [0.060700275, -0.00963460467, -0.0109314872, 0.0226204805, -0.0332568586, 0.0521160699, 0.0574439242, 0.0392807126, -0.00915056467, 0.0515165664, 0.0133890547, 0.0559830405, -0.0566463433, 0.00448038522, 0.0627226308, 0.0574566461, 0.0573221073, 0.0650029257, 0.0805597677, 0.0442213304, -0.0356723592, -0.0421719067, 0.0367118083, -0.0422100835, -0.0285072364, 0.00267456914, 0.0688661411, -0.0699288771, -0.0753832459, 0.0239470489, -0.00184377981, 0.0237434451, 3.864270e-02, -0.0394119211, 0.0810462608, -0.0421678759, 0.0588073395, 0.0784924849, 0.0499630868, 0.00492314389, 0.0160748977, -0.0410256125, -0.069685854, -0.0619296543, -0.00959829241, -0.0462774709, -0.0503597036, 0.0568018816, 0.0121260295, 0.0209179912, 4.36423637E-4, -0.00316848629, -6.755410e-02, -0.0219847336, -0.00912000238, -0.0226698071, -0.0293056481, 0.0723913088, 0.0549803972, -0.0296224505, 0.0209934916, -0.0554407537, -0.0297016446, -0.00469676917, 0.0729916617, 0.0636828243, -0.0569759086, 0.0454941615, 0.0242109504, 0.0561091416, 0.0559182316, -0.0143487137, 4.21695033E-4, -0.0348930731, -0.0141915297, -7.635220e-02, 0.0363938473, -0.00253901281, 0.0756715462, 0.0185802672, 0.0106117371, -0.00288964924, -0.0564242676, 0.00982043147, -0.0782394632, -0.0133359469, 0.0738210455, -6.085150e-02, 0.045540154, 0.041997578, 0.0554698333, 0.0780961961, 0.0110659748, -1.87587284E-4, -0.0604635514, 0.0134691074, 0.0622661672, 0.056140136, 0.00846699346, 0.016634943, 0.0571425632, 0.0601437427, 0.0615147129, 0.058526203, -0.0152146863, 0.0232849643, 0.00368679478, 0.0722750276, -0.0270468835, -0.049649097, 0.0679210424, 0.0317571722, -0.0262856483, 0.0706475675, -0.0782714709, -0.00982067641, 0.0645450875, 0.074701339, 3.791430e-02, -0.0442585237, 0.0775321275, -0.0614174195, 0.0380616784, 0.029055737, -0.07379473, 0.00388121605, 0.0517619066, -0.0393376797], [-0.00634263549, 0.0584934354, -0.0486450419, 0.0219936837, 0.0679939687, -0.0778439939, -0.0035423662, -0.0155528598, -0.0343110524, -0.0292285513, 0.0026049884, 0.00967141613, 0.0322683677, -0.0518011265, 0.0672752261, 0.021153206, -0.0162995961, -0.0104558803, -0.0514430776, -0.00252284855, 0.078184925, -0.0204834938, 0.0519886203, 0.0635282695, 0.0801253766, 0.0462257639, 0.0174991712, 0.0625823811, 0.0220883191, -0.00634051487, 0.0678268224, -0.0431053862, 0.00590469548, -0.0310216956, 0.0532339029, 0.014621661, 0.0159434583, 0.0510691144, -0.00225198106, -0.0312963612, 0.0061162049, 3.170750e-02, -0.0116269803, -0.0738872587, 0.049290441, -0.0604466684, -0.049105268, 0.0735147521, 0.0454728343, 0.0385311432, -0.0661619157, -0.0310830455, -0.0448951423, 0.0627154782, -0.0472795926, 0.0138709182, -0.064482063, 0.0123519758, -0.0709089041, -0.0520546138, -0.0153506687, 0.0457991026, 0.00776980864, -0.0315550528, 0.0119248554, 0.0222066231, -0.0619561411, -0.0721987709, -0.00841960683, -0.0401327908, -0.00700219767, 0.00882081222, -0.0527133048, 0.0126304142, 0.0550363846, 0.00535053853, 0.00967556424, -0.0268697031, -0.0591913201, 0.0218756478, 0.0292291585, 0.0134313423, -0.035086859, 0.0619205571, -2.911330e-02, 0.0416563824, 0.0425836183, -0.0591331497, 5.123150e-02, -0.0536320135, 0.0440794453, -0.00122839632, -0.049395863, -0.00598555338, -0.0466344878, -0.0495581292, 0.0693121776, 0.0418097973, 0.0440978892, 0.0381551683, 0.0321800187, -6.541640e-02, 0.045699425, -0.0809728205, -0.00677064108, 0.0539861023, -0.0758429915, -0.00526069338, 0.00273849163, 0.0165730342, 0.0160393063, 0.00141105382, -0.0216324385, -0.00584777584, 0.0258725304, 0.0447974391, -0.0604038872, 0.0475597642, -0.0712100863, -0.071535036, -0.0785626545, -0.0418529809, -4.450240e-02, 0.0134775406, -0.0466304794, 0.0771532356, 9.46706918E-4, -8.086680e-02], [0.0477109179, 0.0122098234, 0.00908621307, 0.0767099485, -0.00825817417, 0.0582498163, -0.00500228954, -0.0494385213, -0.0373144187, -9.100520e-03, 0.069868885, -3.159680e-02, 0.0654592961, -0.0127069438, 0.0336152278, -0.00165768538, 0.045265343, -0.067027621, -0.0329812244, -0.0025811356, -0.0411101915, -0.0690569654, -0.0150693972, -0.00901012402, 0.0280272104, -0.06389568, 0.00981164258, -0.0349421389, -0.0644880235, -0.00981720816, 0.0554877073, 0.00719586154, -9.0855587E-4, -0.0787953287, -0.00362483342, -1.27705527E-4, -0.0693355575, -0.00255574984, -0.0214682352, -0.0127182333, 0.00679337233, 0.0598713309, 0.0386318117, 0.0531677492, -0.0469856523, -0.0115659256, 0.00322774472, 0.0037440192, 0.0514960065, 0.0612064227, 0.0333281495, 0.0637365431, -0.00811710767, 0.0196570642, 0.0776526108, 0.0654664636, -0.0305046085, 0.0794088393, -0.0273174029, -0.0229353774, -0.0714377612, -0.0138261616, 0.031283509, 0.0236163419, 0.00521409744, 0.050673075, -0.0528018437, -4.974700e-02, 0.0599129423, -0.0773340315, 0.0379904807, 0.0353282467, 0.0498711504, -0.0753066689, -0.0152367493, 6.660830e-02, 0.058814764, 0.0392833576, 0.0178741477, 0.0805430039, 0.05047803, -0.0411522724, 0.0302013345, -0.0414810292, -0.0636278391, -0.031325385, 0.0594957508, -0.0779689624, 0.0337770618, -0.0360521898, -0.074428454, -0.0516159199, 0.0421752818, 0.0318271928, -0.0492196158, 0.08003062, 0.0143721234, -0.0623286813, -0.0468424857, -0.00299955951, 0.00455167517, 0.0485471785, -0.00305411732, -7.558700e-02, -0.0263215974, -0.0424884111, -0.0633099824, -0.0402113162, -0.0267331135, 0.0347218849, -0.0706518888, -0.0595623814, 0.0219641384, 0.0599824972, 0.0502796359, -0.0704943091, 1.86170873E-5, 0.0764745325, -0.0563995279, -4.137850e-02, 0.0720886812, 0.0326823667, 0.0513620265, -0.0648592487, -0.0200602617, 0.0681103691, -0.0216717292, 0.0378616378], [0.00316815218, -5.410710e-02, 0.0457321741, 0.0304992963, 0.0591287166, 0.0039417604, -0.078353852, 0.015750682, 0.0274391435, 7.16423499E-4, -0.00957912859, -0.0296748187, -0.0426740237, 0.074467428, -0.0497264303, -0.0258895848, 0.0169776361, -0.0498714037, 0.0289769918, 9.824410e-03, -0.0695340708, -0.0802385658, -5.33565588E-4, -0.0385165066, 0.04938991, -0.0726995841, -0.0554923192, 0.042003721, 0.0612122938, 0.00566397561, 0.00218993751, 0.0193402041, -0.0144730685, 0.0698806792, 0.0754371583, -0.00475814939, -0.011011879, 0.0365960374, 0.0537692867, 0.070583269, 0.0470268205, -0.0242092349, -0.0153627442, 0.0527698658, -0.0270518139, 9.73254966E-4, -0.0300387871, 0.00899011269, 0.0646050423, 6.940610e-02, 0.0631658211, 0.00162798422, -2.627000e-02, -0.0158755798, -0.0404933877, 0.0622063875, -0.00416194322, 0.055430077, -0.0647653043, 0.0010292948, 0.0667934715, -0.0166810714, 0.0670618638, -0.0580297634, 0.00930756889, -0.051066272, 0.0777499229, -0.0672022477, 7.583430e-02, -0.019975828, -0.0169774722, -0.0249232817, 0.0318874754, 0.0675454885, -0.0166561678, -0.05984357, -0.0640418231, 0.0526572615, 0.0180163179, -0.0358818807, -0.0204229262, 0.0112632215, 0.00501798792, 0.0196146984, -0.0261462312, 0.0610846728, -0.0209620316, -0.035789527, -0.016534688, 0.00236019888, 0.0586084202, 0.012812715, -0.0272323731, 0.0592565425, 0.0321877524, 0.0495755933, 0.00625152699, -0.0480617546, -0.046241086, -0.0379608683, 0.0212225653, 0.0179443937, -0.0473802388, -0.0356587097, 0.00333144167, -0.0393349901, 0.0139216352, 0.0477169789, 0.00620736647, -0.0411673933, 0.0371403433, 0.0343280137, 0.0522192307, -0.0717091933, 0.0459268205, -0.031238405, -0.0660946369, 0.0231391788, -0.070248127, 0.0808056667, -0.0598967858, -0.00716865808, -0.0109610138, -0.0333718434, 0.046194803, 0.049027551, -0.0330906808, 6.118570e-03], [0.0808934047, -0.0371030346, -0.0101536065, -0.0714085475, -0.0716847181, -0.0374666676, 0.0114747882, 0.0183271207, -0.0352216288, 0.0678748861, -0.0137127824, -0.0234499313, -0.0250425022, 0.00230789953, 0.0508547202, 0.0335111469, 0.0673095435, -0.0139359878, -0.00728153856, 0.0642380267, -0.0125071732, 0.0410422236, -0.0284613464, 0.00105103315, 5.892940e-02, -0.0359517299, 0.0308290776, -0.0165757462, 0.0372095406, -0.0286607239, 0.0247662943, -0.0363768041, 0.0532225519, 0.0118737528, -0.00131681119, 0.0399303362, -0.0737008527, 0.0129801398, -0.00812094472, -0.02149285, 0.0294876266, 0.0478628613, -0.0551214777, 3.391710e-02, -0.00128528068, 0.0138201769, 0.0638617426, -0.0175779816, 0.0581541285, -0.0188298039, -0.0238789339, -0.0297757778, 0.0204693135, 0.0773603618, 0.019279914, 4.16099763E-4, -0.0250948109, -0.0520968102, -0.044210095, 0.0643982813, 0.0273056887, 0.061434567, -4.267750e-02, -0.0704102293, -0.029704757, 7.31819368E-4, -0.0523026399, -0.0376914963, -0.0104128197, -0.0641234741, 0.0479087532, 0.051193025, 0.0437616073, -7.228720e-02, -0.0694924518, 0.0478403494, -0.0759094879, 0.0595769063, -6.559230e-02, 0.0762775689, -2.13746345E-4, -0.0449482203, -0.0267022531, 0.0429455414, 0.0756426305, -0.0124424519, -0.0402056165, -0.0414346531, 0.070847474, -0.0612731203, -0.0737371668, -0.0424708277, 0.0497535616, 6.573930e-02, -0.0335622132, -0.0286341328, 0.0701824948, 0.0771856606, 0.0425265841, 0.0230820589, 0.069204472, 0.0751045272, -0.047303088, -0.0265760813, -0.0532593429, 0.0223561767, 0.041033268, -0.0309499819, 0.0232324526, -0.020461075, 0.0181708056, 0.0511066541, 0.0777347981, -0.0762246847, -0.0592118725, 0.0293698274, 0.042072095, 0.0424440466, -0.0471195653, -0.0238768794, -0.028307382, -0.00767077925, -0.068855688, 6.745940e-02, -0.043850828, -0.0664262548, -0.0202725045, 0.0072251386], [0.0350452177, -0.0499804951, -0.06047564, -0.0211201496, 0.00849552266, 0.0422966555, 0.00994448922, 7.825350e-02, -0.0368959606, -0.0610233843, 0.048726663, -0.0204786193, -0.0582741573, 0.0420213863, 0.0205420069, -0.0373144336, 0.00413084729, 0.0632839054, 0.0293394476, 0.00630530342, 0.0763058811, 0.0579670519, -0.0351511426, 0.00708733173, 0.00952158216, -0.0704570264, -0.0600163229, -0.0396279581, -0.0638583302, 0.0303085465, 0.0460986979, -0.0285531487, -0.0109708654, -0.040011622, 0.0796229243, -0.0480022356, 0.0274273474, 0.0480410121, -0.018218087, -0.0270681698, 0.0610833392, -0.057573691, 0.0673625246, 0.0522623472, 0.069647111, 0.0705709234, 0.0162508711, 0.0170502868, 0.0579564124, -0.0494242534, -0.0793388635, -0.0212146714, -0.0123330588, 0.0115957251, -0.0325849913, -0.0527718849, 0.00518119382, 0.00731440587, -0.0460534617, 0.0371405259, -0.0465270542, -0.0631478205, 0.078199692, -0.0178397615, 0.075122714, -0.0429925211, 0.0131029077, 0.0641400293, -0.0703650042, 0.0701936483, -0.0445429981, 0.012785539, 5.528920e-04, 0.0711351633, 0.0631135777, -0.0289442353, 0.0780344456, 0.00946327671, -0.0781876146, 0.00651740283, 0.0447436273, -0.0149327787, -0.0750377178, 0.0274526235, 0.0697235316, -0.00548748439, 0.0702112243, -0.0134356869, -0.0614663847, -0.0443611816, 0.0478329584, 0.0592850856, 0.00576445879, 0.0387810506, 0.0589556098, 0.00294701033, -0.0779888555, 0.0074747866, -0.0376832187, -0.0550655685, 0.054777842, -3.2272641E-5, -0.0353344232, 0.0517576262, 0.05705956, 0.0407638773, -0.0206943825, -0.0699271336, -0.0791579782, -6.544100e-02, -0.0583087429, -0.0579186343, -0.0321218558, -0.0475244336, 0.0268356446, -0.0253263041, -0.0532132201, 9.033030e-05, 0.00600073254, -0.0669909045, 0.0168085173, 0.0452007093, 0.0413544327, 0.0771834329, 0.0255741067, 0.00656236662, 0.0406812392, -0.0626366213], [-0.00268505374, 0.0124703497, 0.0473328419, -0.0685885772, 0.0126359183, -0.0793862566, -0.038687855, -0.0177605543, -0.0209013782, -0.0546755455, -0.0160936639, 0.0542474426, -6.379570e-03, -0.00630249083, 0.0617598668, 0.004854389, 0.0688877627, -0.0430330709, 0.0683509707, 0.0560050048, -0.0522279143, 0.0755217597, -2.305550e-02, -0.0500806496, 0.0761962532, 0.0182507262, 0.0801335349, 0.0226008352, 0.00120849162, -0.040003255, 3.645180e-02, -0.0105365971, 0.0218738709, -0.0388162658, -0.0405732393, 0.0782157555, 0.0597496405, 0.0244864598, 0.0719291865, 0.0316703282, 0.0051184115, -0.0405413881, -0.0424162969, -0.0703434721, 0.0693442672, 0.06606628, -0.0390833579, 0.0559594631, -0.0483287945, -0.015355329, 0.0549774878, 0.031963516, -0.0614453666, -0.025459189, -0.0374741256, -0.021610735, -0.0482399464, 6.320500e-02, -0.0748168528, -0.0787417442, -0.0443810932, -0.0734583139, 0.0553643219, 0.0151310433, 0.0634595305, -0.0739377663, -0.0723952204, 0.0513202474, -0.0331643485, 0.0629457384, -0.0740289688, -0.0448405184, 0.0222582836, -0.04638578, 0.0413301252, 0.00476286514, 0.0697041154, -0.0516804829, 0.0581138544, -0.0452535711, -0.0653664246, -6.537020e-02, 0.0553457364, -0.0186720956, 0.0712907612, -0.00558139384, 0.0552845411, -0.0709803924, -0.0186881982, -0.0321862474, -0.0621587447, -0.00343263056, -0.0496426895, 0.0517401956, 0.0300140977, 0.0238638353, 0.00361353229, -0.00938344467, -9.301650e-03, -0.0466221087, -0.0494145602, -0.0471953116, 0.0382782631, -0.00811553467, -0.0825327635, -0.0618084632, 0.0197720341, 4.907520e-02, -0.0152438823, -0.0207966398, -0.0307169799, 0.00743774185, 0.0780814886, -0.0700153783, 0.00134200684, -0.0635620579, -0.0303252246, 0.0782658234, 0.0391841568, -0.0437794067, -0.025788825, -0.0393831618, 0.0755276904, -0.0136895226, 0.0222759973, -0.0389743149, 0.0332256146, -0.0402226262], [0.00716232881, -0.0753126666, -0.0175464842, -0.0472379103, -0.0153300399, -0.0584933572, -0.0515103638, -0.036531385, 0.0337012559, -0.00876101292, -0.00244547683, -0.0622347891, 0.0523654781, 0.05303308, 0.00704237259, 0.042246934, -0.015535472, -0.0588174909, -0.0378940441, 0.0750902742, -0.0427735299, 0.075981535, -0.0732969791, -0.0410050191, -0.0395357721, -0.0291107111, -0.0691342279, 0.037805222, 0.025456205, 0.00120023452, -0.0672817156, 0.0268393289, 0.0456135571, -0.0817385167, 0.0402205884, 0.0582607649, -0.0618926883, -0.0477675274, 0.0349856131, 0.0350342542, -0.0166848712, -0.0358228385, 0.046350237, 0.023161836, 7.830590e-02, -0.0283588916, -0.0375126116, 0.0644737259, 0.0501563512, -0.0233920738, -0.0191605184, 0.00884593371, -0.0492862761, -0.0398975275, 5.884010e-02, 0.00501171546, 0.0440073907, 0.0194236729, 0.0681597516, 0.0517174639, 0.0577686056, 0.0398328379, 0.0671445206, 0.05873923, 0.0394569114, -0.0145306438, -0.0221893638, -0.0272459611, -0.0302091129, 0.00707538379, 0.013127842, -0.0194618907, 0.0586697087, 0.0548411421, -0.056019567, -0.0176407155, -0.0657045096, 0.0635526851, 0.0323844366, -0.067275621, -0.0576722771, -0.0736971498, -0.0511826426, -0.00132003822, -0.038096942, 0.063992247, -0.0301330574, 5.256540e-02, 0.0188492835, 0.0734523088, 0.0338389277, 0.0348902345, 0.0407360829, 0.0398462527, -0.0259853639, 0.0255692303, -0.0186829194, 8.139120e-02, 0.035990376, -0.071067363, 0.07411699, -0.00816843472, 0.0798631832, 0.0522484034, 0.0346655808, -0.0473645963, 0.0276889801, -0.0479913615, 0.00761620328, -0.0500854366, -0.0127689773, -0.0141541278, -0.0352863483, -0.0547954328, -0.00353415753, 0.0340588875, -0.0726587325, -0.0196245164, -0.0618234426, 0.0480685383, 0.0331299379, 0.0193118807, 0.0759841799, 0.0522449911, -0.0477721207, 0.0392501354, -0.0791557803, 0.0150022367], [0.0763663128, -0.00927543174, -0.0450837538, 0.075393863, -0.0648234263, 0.0547465645, -0.0351150744, -0.0720507503, -0.0079033291, -0.0258979294, 0.0641547218, -0.0526920259, -0.033761844, 0.0396322869, 0.0136177177, -2.77992891E-4, 0.0628117323, 0.0110555058, -0.0107893944, -0.0802481622, -0.0190283768, 5.066770e-03, -0.02561442, -0.021665765, 0.0655965805, -0.0319291092, 0.0755895525, -0.0381370857, -6.645050e-02, -0.0288146455, -0.0741508752, 0.0484617315, -0.0447441526, -0.0804686769, -0.00147860486, 0.0427959822, -0.0280283205, -0.0222710893, -0.0249423943, -0.0173569676, -6.569290e-02, -0.00865416694, -0.0415936336, 0.0661999583, 0.0082032457, -0.0635061413, 0.0671093091, 0.0451213941, -0.0440114513, -0.0698055103, 0.0421679951, -0.0226926617, 0.0382873192, 0.00412498461, -0.0508674309, -0.0568497889, -0.0256092884, 0.0258248094, -0.0127705215, 0.023421431, 0.00170432124, -0.0249181259, -0.0422122367, 0.0333166681, -0.0247829743, 0.07620579, 7.20023876E-4, -0.0318335481, 0.0114651099, -0.0600579567, 0.0749900714, -0.0187267289, 0.0594170876, 0.0794066786, -0.0479078405, 0.0363853574, 0.024133807, -0.0356327221, -0.0278993025, 0.005429212, -0.0279160459, -0.0392692648, 6.642760e-02, 0.0362864509, 0.0117147798, 0.06969551, -0.0640289411, -0.0586049743, -0.00281111128, 0.0753336176, 0.0637235567, 0.0130439345, -0.00864480808, 0.00475263828, 0.0746708289, -0.0560074709, 0.0127491467, 0.00471760193, -0.0598935038, 0.0335331261, -0.0589303225, -0.0317756273, 0.0303269662, 0.0440550596, -0.0284124408, 0.0428762436, -4.962080e-02, 0.0582150817, 0.06719172, -0.0194253456, 0.0316836834, -0.00456403149, -0.0722884759, 0.00339927571, -0.0158931818, 0.0787602365, 0.032879889, -0.00283800066, -0.0657932237, -0.0368593037, 0.07846421, -0.0741131753, -0.029700717, 0.0783022493, -0.0384871885, -0.011486846, 0.0216285437, 0.060573183], [-0.053564813, 0.0439014286, -0.0500947759, -0.0142138032, 0.0575123392, -0.0373187475, -0.0188797619, -0.0610291063, -0.0492860079, -0.0387370288, -0.0102376835, -0.0436874703, -0.0748898684, 0.0640534908, 0.0628616363, -0.0343118832, 0.0419668928, -0.0244131461, 2.47502117E-4, 0.0583201759, -0.0628471226, 0.0788726508, 0.0306587312, -0.0447739549, 0.0645291507, 0.0264609661, 0.00294195744, 0.0672124103, 0.0333978906, 0.076964058, -0.0537768081, 0.066825144, 0.0623981357, 0.00823123939, -0.0734884664, -0.0371366292, 0.0520379134, -0.0152580878, -0.0275830608, 0.00618132763, -0.0791825577, -0.0380331092, -0.080527924, 0.0198882986, 0.0351683684, -0.0722856373, 0.0290195532, 0.0167443845, -0.0328617096, -0.074908413, 0.026073182, 0.0615444966, 0.0364666209, -0.052754119, 0.0469008274, 0.00224230951, -0.0275361203, -0.0350897498, -0.0805909783, -0.0738512725, 0.0424574167, 0.0264523402, -0.0337005891, 0.0592786185, 0.0757884234, -0.0354387909, 0.0512510091, -0.0656975433, 0.0370358527, 0.0548632592, -0.0701467842, 0.0246196166, -0.0271360166, 0.0203201938, -0.0760709569, -0.020719653, -0.0773509219, 0.0704980865, 0.0465438738, 0.0146950064, 0.0349803194, 0.00857011973, -0.0102316802, 0.0150479246, -0.0164332204, 0.0100350119, -0.065346919, -0.0461673476, -4.076020e-03, 0.0723540708, 0.0516278259, -0.0148398299, -0.0653919503, 0.00481541548, -0.0778005048, 0.0243397653, -0.0415964201, -7.514630e-02, -0.0025740976, -6.565450e-02, -0.0501350053, -0.001106682, -0.0739801303, 0.0535284579, -7.548820e-02, 0.0720021501, 0.0239673648, -0.0342088155, -0.0316728875, -0.00783361215, 0.0138730267, -0.0598143302, -0.0745300352, 0.013978214, 0.0274804439, 0.0442923568, -0.0517894849, -7.759930e-02, 0.0781345888, -0.0758877769, 0.0752711073, -0.0700510144, 0.0645432845, -0.0103485547, 0.030420376, 0.0475196727, 0.0387806408, 0.0239329711], [-0.0242717136, -0.024906829, -0.00387625257, 0.077687107, -0.0597497299, 0.048459366, -0.0201348439, -0.0211757682, -0.0620791689, -0.0315908492, 0.0565815717, -0.0773581117, 0.0102454135, -0.0531957969, 0.0100945942, -0.0607852041, 0.0481664538, -0.0116098393, -0.0243072379, -0.0089657316, -0.0079818964, 0.00234484812, -0.0319988355, 0.031318564, -0.0516675077, -0.0211477708, 5.456230e-02, -0.0499854647, 0.0457300395, 0.0191166699, 0.0532520413, 0.0332510099, 0.00544422911, -0.0382818468, 0.0176336709, 0.0578946248, 0.0781985521, 0.0487044975, -0.0277209468, -0.0385249704, 0.059226945, 0.0293320809, -0.0358435623, -0.00346948858, 0.0756498128, 0.054223977, 0.0441700034, -0.0503514446, -6.576210e-02, -0.00882027484, -0.0263377856, -0.0320280083, -0.0480051488, -0.0521095507, -0.0548065826, -0.0463480726, 0.0345846862, -0.0791337564, 0.0716802478, -0.069639504, -0.075078018, -0.061167188, 0.0728463903, -0.0310791638, -0.0492666624, 0.0490908809, 0.0819469243, 0.0478712507, -0.0757170692, 0.0562442541, 0.0546966195, 0.0103310896, -0.0443996489, -0.0369239748, -0.00362596172, 0.0725511909, 0.0237638168, -0.0147742881, 0.0725206807, 0.0260993429, 0.0668385252, -0.074013032, 0.0323174857, 0.0303780548, -0.0501161404, -0.00238714926, -0.0112713873, -0.0310010649, -0.0654821917, -0.0650528669, 0.0495865904, -0.0350224599, 0.0741526708, -0.0642922595, -0.0365128331, 0.0167160053, -0.0102225346, 0.0477003977, 0.0288503896, -0.0595900677, 0.0661832839, -0.0571043119, 0.0440329649, 7.926370e-02, 0.0383101329, 0.02451564, -0.0198299084, 0.0609904825, -0.0593171529, -0.0261055268, 0.0339849927, 0.0530679524, -0.028792724, 0.0376426242, -0.0586162135, -0.027220061, 0.0418295525, 0.0491981804, 0.0202658251, -0.00646719104, -0.00867837761, -0.0130657982, 0.0121951243, 0.0315500833, 0.0604119897, 0.0731419622, -0.0788395404, 0.0256862435], [-0.0560515076, -0.0287477653, -0.0594557226, 0.0798766166, 0.0255173463, -0.0361986682, -0.0393096209, 0.0481830239, -0.0373203084, 0.0698953494, 3.062530e-02, 7.911150e-02, 0.0591651946, -0.0451406129, 0.0766223073, 0.0459299721, -0.0762514845, -3.262960e-02, -0.0365349613, -0.0246859733, 0.0423186086, 4.757660e-02, -3.395520e-02, -0.0344702229, 0.0519390516, 0.0417304188, 0.0582627207, -0.0177649502, 8.067140e-03, -0.00907266792, -0.0781021937, -0.0296214074, 0.00837433804, 0.0552393161, -0.0468232557, -0.0480565578, -0.0205860734, -0.0163142234, 0.0400330052, -0.0608687513, -0.0574329756, 6.041440e-02, 0.0193626918, 0.0443490967, 0.0158599578, -0.0207063034, 0.0542988591, -0.00513827708, 0.0595024899, 0.00866276119, 0.0264432505, -0.00902406126, 0.0508485772, -0.00969434343, 0.0669011771, 0.0576908365, 0.0171199646, -0.00926496461, 0.00268601812, 0.071273759, -0.00819504727, -0.0679610744, 0.0399204977, -0.0164282601, -0.0140218651, -0.0630865097, -0.0165849142, -0.0482930467, 0.0264804643, -0.0582198501, 0.0600587577, 0.0541869216, 0.0145315342, 0.0114666503, 0.00221930561, -0.0438293815, -0.0466291122, -0.0622255168, -0.015518873, -0.0275418498, 0.0450045541, 0.030535141, -0.011397684, -0.0744856894, -0.0664154664, 0.0638397709, -0.0422878079, -0.0721548945, -0.0128727006, -0.0100139761, -0.0795679539, 0.0802032351, 0.0647776201, 0.0303812157, -0.0193396956, -0.0103364885, 0.00259758183, 0.0110738836, -0.0494749211, 0.0347833224, -8.45868344E-5, -0.0681451038, 0.0287907142, 0.0732238069, -0.0592550598, 0.0218330566, 0.0703557953, 0.0415360369, 0.0168579854, 0.0035885293, 3.266860e-02, 0.00191594695, 0.0514851958, 0.0198469721, 0.0309552401, -0.0490360036, -0.00115188165, -0.0372474529, 0.00402358035, -0.0248689242, 0.0212394353, -0.0616777129, -6.241000e-02, 0.0505597368, 0.00521772029, -0.0653809384, 0.0385498218, -0.00887110456], [0.0580509789, -0.0110665774, -0.0205813944, 0.0247867778, 0.0263316948, 0.0795138776, -0.0599110425, 0.0170206819, 0.0804576575, -7.649950e-02, -0.0680613294, -0.0214805268, -0.0436589047, 0.0549500249, -0.029271422, -0.0620399266, 0.0414872095, -0.0775065347, 0.0532688536, -0.0772975683, -0.0154482545, 0.0671617389, -0.0409147553, -0.0111428015, -0.0366989896, 0.0057872124, -0.0467843264, 0.0544274636, -0.0629653856, -0.019445328, -0.0014226163, -0.0574949309, -0.0323577262, 0.0110116825, -0.027508771, 0.078442268, -0.00820690114, -0.0462600328, 0.00489702122, -0.0397615544, -4.89242084E-4, -4.695870e-02, 0.0568285882, -0.0328569859, 0.0215719454, 0.0453743376, -0.0651280656, 0.0711405352, -0.0365617797, -0.0242793486, 0.016551014, 0.0750639364, 0.0204782989, 0.00272700842, -0.0529962704, 0.0183450226, -0.00367675419, 0.0045985193, -0.0467584394, 0.0391691737, -0.0633653849, 0.0771948099, 0.047339987, -0.0408123881, 0.019009063, 0.0533547066, 0.0722025782, 0.0235418528, -0.0358975045, -0.0613036864, -0.0656465814, -0.0518662371, 0.0186140426, -0.0587354712, -0.0549117699, -0.0609641597, -0.0529113561, -0.0463278592, -0.034707725, 0.0475388467, 0.00634393329, -6.625810e-02, -0.0690236464, 0.0217836499, -0.0800291523, 0.0388775431, -0.0798476934, 0.0620031468, -0.0463523977, -0.0479571819, -0.0670428798, 0.0189536456, -0.0548016578, -0.0799074843, -0.0445603132, 0.0339867137, 0.00311838579, 0.0676074773, -0.0398606323, 0.0505768843, -0.0641110092, -0.00671840226, -0.0407918096, -0.0144690443, -0.0754565224, -0.0201666988, -0.0359357148, -0.0723240375, 0.0460784286, 0.00443832157, -0.0672446713, 0.0523132123, 0.0715955943, 0.0700317174, 0.014367912, 0.0642787814, 0.026905477, -3.558380e-02, 0.0167108048, 0.0168826636, 0.0786630064, 0.0271795932, 0.0756789446, -0.0124441218, 0.0213957541, -0.0269283336, 0.0100063654, 0.05994948], [0.0309737343, -0.0285206698, -0.059839569, 0.0138672926, -0.0115762241, 0.0239045154, -0.0313897841, -0.0306720287, 0.0318398587, 0.0350424834, -0.0646785051, 0.052266676, 0.00801315252, 0.05483336, 0.0284893103, -0.0251198858, 0.0622958019, -0.060296841, 0.0396341868, -0.0245251879, 9.67448359E-5, -0.0223675389, -0.00218978105, -0.0766845122, -0.067584388, -0.0170954242, -0.0627946779, -0.00320918392, 0.0553214215, -0.0185036734, -0.0177335218, -0.0476164035, -0.0313676447, -0.0175319854, -0.00864237826, 0.0130041363, -0.0325285047, -0.0493744314, 0.0364652164, 0.0282418393, 0.0417733565, -0.0644685403, -0.0589522533, 0.00925710238, 0.00910287909, 0.0504746735, -0.0192127619, 0.0304313134, 0.0676780492, 0.0733000711, 0.00891230627, -0.0735640526, -0.0799629315, 0.0221040398, 0.0375890732, -0.0658270121, 0.0136300521, -0.0492890477, 0.0466570109, -0.0210726783, 0.0673091859, -0.0119175222, -0.066635631, -0.0651836097, -0.0343201272, -0.0383152328, -0.0538931973, 0.00509305811, 0.066020593, 0.014662276, 0.04183596, -0.0238029268, -0.0163047761, 0.00989269837, 0.0497137681, -0.0171339866, 0.0352600627, 0.0677694753, -0.0293717552, 0.0462074168, -3.424750e-02, 0.0642857552, 0.0455914252, -0.0492312275, -0.0530384816, 0.0347817317, -0.0457455255, 0.0392736234, 0.00614285795, 0.0201452766, 6.721790e-03, 0.0677623525, -0.0213509519, -0.0268943477, 0.0369132236, -0.0801321268, -0.0347607397, -0.00523345592, -0.0500178933, 0.00436256407, -0.0160526168, 5.409960e-03, -0.00127825944, 0.00475594588, -0.066685535, -0.0216629878, 5.117000e-02, 0.0238920655, 0.00249515707, -0.0333312303, 0.0171311237, -0.0701868683, -0.0423861071, 0.0626128912, -0.0499690473, 1.76682894E-4, -0.0192131717, 0.0352787599, 0.0500199795, 6.326520e-02, 0.0360887349, -0.0614568628, -0.0499809459, 0.0241435934, -0.0575624779, -0.0372013859, -0.0565183684, -0.0306653585], [0.072737284, -0.0424681231, 0.0799834057, -5.05810953E-4, 0.0183753278, 2.77238898E-4, 0.0769044756, 0.0174468439, 0.072545521, -7.515450e-02, -0.0124902371, 0.0440028422, -0.0520738587, 0.0296914931, 0.0292655416, 0.0683509186, -0.0112867057, 0.072276853, 0.0230560582, -0.0456297696, 0.0488278754, 0.059115462, 0.0610912107, 0.0736235604, -0.0127344159, 0.0458095893, 0.0204227734, -0.00518937409, 0.0463501699, -0.00778652821, -0.00760606816, -0.00937715359, 0.0100444462, 0.0270468723, -0.0626091138, -0.0655692294, -0.0516739786, 0.0244127139, 0.0490928479, 0.0130349062, 0.0560305715, -0.0667365491, 0.00998640153, -0.00797563139, 0.0668912083, 0.0514411107, 0.0418141745, 0.0397205241, -0.0661854893, -0.0599790178, -0.00589125324, 0.0174258202, 0.0226652883, 0.0272518396, 0.00229900889, 0.0576990359, 0.0743423402, 0.0776961669, -0.0382633656, 0.0640774816, 0.00218974659, 0.04039637, -0.0452939719, 0.0246950965, 0.053453315, -0.0806830823, -0.055015713, 0.020408269, -0.0200760663, -0.0573467091, 0.0088351192, -0.0704358891, 0.009039931, -0.0441969521, 0.0315830894, 0.0295203589, -0.00527551817, -0.0117791174, 0.0458186157, 0.0486066341, 0.0663525909, -0.0313635282, -0.025820557, -0.0233147964, -0.0390163884, -0.00596396439, 0.0169973262, -0.0389200561, -0.00784569419, -0.023286514, 0.0240915027, -0.00199502264, -0.0343645886, 0.00922049395, 0.0446879938, -0.0542134941, -0.056144774, -0.0680991188, 0.0692017525, 0.0316490866, 0.01723052, 0.0555091389, -0.00593782542, 0.0462448038, -0.034130577, 0.0761037245, -0.0649478435, -0.0667298064, -0.0794450045, 0.0812600553, -0.052488625, 0.0457152799, -0.0416824743, -0.0343877114, 0.0806645825, 0.0592223965, -0.0635197833, -0.0118731819, 0.0694554746, 0.0241273232, -4.026130e-02, 7.239680e-02, 0.005058059, -0.0727990493, -0.0782048553, 0.0277128126, 0.00988908205, 0.0225262642], [0.0784178227, 0.0220581349, 0.0565770045, -0.0780781507, -0.00331932073, -0.0205534101, 0.0470047928, -0.0751262754, -0.0524502881, -0.0455912799, 0.080421567, -0.0345659778, -4.066990e-02, -0.0327568389, 0.0412527248, 0.0632019117, -0.00777570158, -0.0469623655, -0.0426742323, 0.0507817566, -0.00953112542, -0.0183757823, -0.0308452621, 0.00306301704, 0.0292929523, -0.0688963234, 0.00882519781, -0.0244266875, 0.0579908155, -0.00309497491, -0.0389899202, -0.078300789, 0.0236653034, -0.0343489759, -0.044013083, -0.0641329512, -0.00643951492, 0.0448870063, 0.0535028875, 0.0809979513, -0.0775334239, -0.0147116454, -0.0481208973, 0.0437069759, 0.0547541417, -0.0517656878, -0.0239221379, 0.0400486328, -0.0681331455, 0.0573088825, 4.734240e-02, -0.0292304829, -0.026292054, -0.0751624405, 0.0651971921, -0.0121876188, -0.0630812272, 0.0054181274, -0.0339708179, -0.0704680383, 0.0285366196, 0.058157552, -0.063013345, 0.00427784631, -0.0157185979, 0.00388667569, -0.0125396457, 0.0409015715, 0.0352373868, 0.0369340405, 0.0618350171, 0.00872511882, -0.00763846887, -0.0509614833, -0.0513430908, 0.0487181768, 0.0370652303, 0.0708321854, 0.0349463262, -0.0377597436, 0.0346501134, -7.120630e-02, 0.0494415052, 0.0466371439, 0.0115058143, 0.0551603548, 0.00584206637, -0.0605609714, 0.0317115895, 0.0517676584, -0.0479971804, 0.0587817281, 0.0671991482, 0.021880595, 0.0650592521, -0.0488940962, -0.0107703647, -0.039884463, -0.00967816263, -0.0740482733, 0.0776554867, 0.0348700657, 0.027678838, -0.0460277125, 0.0535973571, -0.0210177824, -0.0621663481, 0.0593053438, -0.0712253675, -6.519740e-02, -0.00363864913, 0.0488348529, 0.0503300689, -0.0598541349, -0.0259449091, 0.0430908762, -6.440340e-02, 0.0749952868, 0.0583273619, 0.0033243997, 0.0337997898, -0.0544164479, -0.0723841116, 0.0498959273, -0.0788055956, -0.024165906, -0.0225680564, -0.0802681371], [0.00707044778, -0.0588273108, 0.0767590403, 0.0530777164, -0.0219059661, -0.0370828025, -0.0719957501, -0.0188085102, -0.0780358985, -0.025919335, 5.3370872E-4, -0.0360985547, -0.00940490514, -0.0614941381, 0.062909469, -0.0349586532, -0.0660127774, 0.0313513279, 0.0648164153, -3.983610e-02, -4.520840e-02, 0.0727591291, 0.0494735278, 0.0140808625, 0.020422047, -0.0711183399, -0.00587209128, 0.0111952284, -0.0435808823, 0.0617286079, -0.0571643151, -0.0246208366, 0.0720285699, -0.0519742109, 0.00441260263, -0.013442101, -0.0607587621, -7.236270e-02, 0.0179292969, -0.0495571345, 0.00566163799, 0.0465728827, 0.0436229631, 0.0610813722, 0.0329287425, 0.0154535435, 0.0074730115, -0.0735591501, 0.0692976788, 0.0462270267, 5.246730e-02, 0.0578196757, -0.068907231, 0.0219238903, -0.0619032345, 0.0103769805, -0.0452541746, 0.0425260291, -0.0531116948, -0.00681095524, 0.0264632851, -0.0618138127, -0.0568934679, 0.0535867624, -0.0556662977, 0.0794874579, 0.0162766557, 0.0672467276, -0.031851612, -0.0616000369, 0.0607557893, 0.0746828094, 0.0289415978, -0.0157830287, -0.0725943297, 0.0481067449, -5.19616937E-4, 0.0287469346, -0.0217468739, 0.0460451953, -0.0502457358, 0.0117683271, -0.0303070191, -0.0347297378, -0.023878783, -0.034499228, 0.0736408904, 0.0658958852, -0.0581942871, 1.099683E-4, -0.0750201121, -0.0176852625, -0.075792633, 0.00467953272, -0.0451245271, -0.028987281, -0.0750840157, -0.0106958412, 0.00233941758, 0.0800826699, -0.00334998895, 0.0728903487, 0.0148700476, 0.0592622384, 9.47467342E-4, 0.0639151931, 0.0458910614, -0.0634861439, 0.0113877077, -0.0564559214, -0.0754654631, 0.0604788288, -0.0801434516, 0.0233268645, 0.0440132357, -0.0530138761, 0.0400711112, -0.0158263128, 0.023434028, -0.0572302751, -0.0359164178, 0.0290077291, 0.0514583923, 0.0518165454, -0.0508133024, -0.00486478209, -0.0462942906, -0.0529056191], [-0.0655162334, 0.0452359468, -0.0101992786, 0.00772491843, -0.0492079891, -9.17191966E-4, 0.0293013174, -0.0322183594, 3.943850e-02, 0.0237529129, -0.0379547663, 0.0526856296, -0.0437609777, 0.00958766601, -0.0510096438, 0.0675878226, 0.0771186277, -0.00107873231, -0.0169526022, -0.0282565355, -0.0191545933, 0.023257345, 0.0253811106, 0.0775694176, -0.0655017793, 0.0051503703, -0.0697573423, 0.0373890512, -0.0720573291, 0.0253694504, -0.0574088544, -0.00226980331, -0.0711988508, -0.0144179063, -0.0138938138, -0.0184654109, 0.0253450032, -0.0156557858, -0.0576806813, -0.0680370927, 0.00587042095, 0.00278163771, 0.0628338084, 0.0229326729, -0.00262230821, -0.0279498212, 0.00845723134, -0.0688405633, 0.0685371906, -0.0479188412, -7.228730e-02, -0.00969459582, 0.0152497739, 0.0750681534, -0.0517523289, 5.939890e-02, 0.0800589397, 0.0590252355, -0.00504000485, -0.0245018266, -0.0120485919, 0.0690197274, 0.0509147197, -0.0312109925, 0.0455483459, 5.395340e-02, 0.0220499784, 0.0562079325, -0.0286239125, -0.0068972013, -0.0808225125, -0.0402447283, -0.0157814492, -0.0677078068, 0.0559049621, -0.03089807, 0.0710021779, 0.00508707622, 0.0797159523, -0.00248996052, 7.413300e-02, -0.00971233193, 0.0214798637, 0.0613800101, 0.0258565955, -0.0772848651, 0.0726334453, 0.0517796166, 0.014012685, 0.0275705606, -0.00611401442, -0.0532864965, -0.0676275864, -0.0100247264, -0.0150205782, 0.0545555428, 7.558440e-02, -0.0442019366, 7.199390e-02, 0.00559311779, 0.0258111563, 0.0295849796, 0.018899994, -3.586800e-02, 0.033438351, -0.0662978217, 0.0787729695, 0.0608425774, -0.0514338166, 0.02072764, -0.0160723329, -0.0769602284, 0.0775335133, -0.0377890952, 0.0462350696, -0.0207345188, 0.0066212262, -0.0768821314, -5.824470e-02, 0.004502228, -0.0756493508, 0.0201348122, 0.0724710077, -0.0238349177, -0.00473320484, 0.00231198967, 0.0153360348, 0.0442158133], [-0.00828924775, 0.0412921235, 0.0772404447, 0.0316342711, -0.0255527981, -0.0425725132, -0.0792504251, -0.015932709, 0.0465776697, 0.0332471803, 0.0183933526, -0.0395826958, -0.0417260937, -0.0329450555, 0.0393096209, 0.014275901, 0.0478410944, 0.00506868213, 0.0430525467, -0.00986717641, 0.055148609, 0.0553309247, 0.0487433448, 0.0727443174, 0.00598408282, 0.0467475429, 5.96530735E-4, 0.0115748793, -0.0291726962, -0.00516881794, 0.0592721477, -0.0694887415, 0.0623203591, 0.0272807926, -5.940250e-02, -0.0423579551, 0.0735256299, 0.0605518147, 0.0746089593, 0.0394188613, 0.0604640022, 0.0192656815, 0.0540355816, -0.0599380545, -0.0339139067, -0.0280399173, -0.0162690058, -0.0536751524, -0.0673164129, -0.0177263543, 5.02757728E-4, -0.0235991068, 0.0296108648, 0.0300388783, 0.0320169926, -0.0533700734, 0.018252708, -0.0461395793, 0.0757020637, -0.0155437738, -0.0578224435, 0.0632905588, -0.0193983056, -0.0558963716, 0.0513027832, -0.0774384588, 0.0159399211, 0.0614580289, 0.0604395345, -0.0632193386, -0.0415436961, -0.0626591071, 5.164910e-02, 0.0141712055, 0.0490720347, -0.0302805491, 0.0551936552, -0.0615908429, 0.0527799651, 2.71916389E-4, -0.0696251318, -0.0506988168, 0.0698471889, 0.0521380678, -0.00583782792, 0.00709905475, 0.0297755077, 0.0350928828, 0.0804247185, -0.0233997665, 0.0251019076, 0.0765618607, -0.0439400785, -0.0373829044, -0.0249874629, 0.012601532, -0.0312789269, 0.0114225894, -0.00650977343, 0.00578354299, 0.045485504, -0.0758256689, 0.0548857525, 0.0042020157, 0.0751241073, -0.0263769589, -0.0184035078, 0.0508919135, -0.0242638215, 0.0477434918, -0.0608332455, -0.0426105894, 0.0253821537, 0.00575724244, -0.0774405896, 0.0403471589, 0.0473226532, -0.0228825472, 0.051877372, -0.0649064109, -0.0232353359, -0.00130231678, -0.0664313733, 0.0550490096, -4.038920e-02, 0.0216238722, -0.0441877842, -0.0345734395], [0.0532494262, -0.0402333513, 0.0607127026, -0.0640215874, -0.0674248785, 0.0574169978, -0.0312236398, 0.0638547316, -0.00387661159, 0.00951146334, 0.00701999664, 0.0680278614, 0.0448882952, -0.0792730078, 0.0219166949, 0.0540368035, -0.0527195334, 0.0335089192, -0.0662382767, -0.0428399406, 0.0740366802, 6.589820e-02, -0.0312778056, 0.0281505883, 0.00955876708, 0.00369564444, 0.0624825731, 0.036684081, -0.0448156409, 0.0426787362, 0.0614401922, -0.0794801414, -0.0770656615, -0.0409593508, 0.014419429, 0.057855539, -0.0657578781, -0.0433492735, -7.407110e-02, -0.079062879, 0.0188742802, -0.0490052067, 0.0564675704, 0.0648040697, -0.0596590787, 0.00464840233, 0.071262531, -0.0797966495, -0.067249611, 0.0805053785, 0.0399907902, -0.0537899286, -0.0272285379, 0.0567423031, -0.00572858751, 0.0585234836, 0.0587443337, 0.0636977479, 0.0193953663, 0.0484066531, 0.0327607393, 0.0247704685, 0.00286667049, 0.0428842679, -0.0589248054, -0.0585794598, 0.00774452835, 0.0235284865, 0.0399807915, 0.07052017, 0.0179417059, -0.0263272189, -0.0661030113, 0.0402220041, -0.07278499, 0.0121800378, 0.00187237561, 0.0368032381, 0.0168438107, 0.036729157, -0.0757095813, 0.0105267093, 0.0760272816, -0.0513121672, 0.0627970621, 0.0503564849, -0.0287786759, 0.0145435408, 0.0544506237, 0.0299624503, -0.0751333758, 0.0497663394, -0.0364542603, 3.910160e-02, -0.0231405012, 0.0417585224, 0.0158485696, -0.0592499971, 0.0544835553, -3.148600e-02, -0.0807837545, 0.0527886674, -0.00811766088, -0.0643262416, 0.0625703409, 0.0353855342, 0.0149496049, 0.0313619673, 0.00615930557, 0.0776732936, -0.0575346351, 0.0132508725, 0.0382607132, -0.0236513019, -0.0356785432, -0.00594767183, -0.04972459, 0.0279537439, 0.0656746849, -0.0311247632, 0.0407301933, -0.0318479761, 0.0609296635, -0.0347949192, -0.0357251689, 0.0293542445, 0.0587014481, -0.03441846], [-0.0755231604, -0.0412888564, 0.0378536806, 0.00283838063, 0.0163039863, -0.0090753287, 0.0382387787, -0.0684393495, -0.00585271418, 0.0785401687, -0.0778346285, -0.0581376404, -0.0652252808, 0.0693016276, -0.0238912888, 0.00811017305, -0.0614510253, 0.0352889523, 0.0417487398, -0.0719134509, 0.0521086976, -0.00182656199, 0.0742764398, -0.0317182913, -0.0652938932, 0.0735924169, -0.0355612561, -0.00544831157, 0.00248844922, -0.032787431, 0.053185232, -0.0156907514, -0.0575062856, -0.00339599699, -0.0279015116, 0.0243329927, 0.00370550901, 0.046633862, -0.0220735297, 0.012478441, 0.0354709253, 0.0575750247, -0.0724330544, -0.0169130415, 0.0294462144, -0.0376324281, -0.0198653862, -0.00771605968, 0.0577449575, 0.0229243785, -0.0448199511, 0.0199844539, 0.0636739209, -6.996510e-02, -0.0805689245, -0.0447574891, 0.0688127354, 0.0686566755, -0.00660002977, -0.0317856483, 0.0198262855, 0.0800375714, 0.0148979202, 0.0495064929, -0.0682734847, 0.00613103062, -0.0356661677, -7.193820e-02, 0.0233196467, -0.00332077593, 0.016837202, 0.0099472925, 0.057762973, -0.0466441326, -0.0451592244, -0.0782755613, -0.040074043, -0.00198097527, 0.0467950329, 0.0781413093, -0.0603467673, -0.050868392, 5.6707859E-4, -0.0685737431, -0.016608119, 0.0229959637, 0.0522454157, 0.0418726206, 0.0669682398, -0.0624085031, -0.00242096186, 0.0348493755, -0.0601994693, 0.0471636429, 0.0746263638, -0.0171248168, -0.0258472227, -0.0626906306, 0.0640361086, 0.0225648582, -0.0596321039, -0.0375628881, 0.0508037433, 0.0442692563, -0.0770809948, -0.0206146874, 0.0547460541, -0.0563672557, 0.037730664, -0.0609343648, -0.0215235278, -0.0579825863, -0.0563401841, 0.054940559, -0.0343974605, -0.0629401356, -0.0487177595, 0.0461444929, 0.0175287947, -0.0218415447, -0.0226778686, 6.759680e-02, 0.0491415039, 0.0021789223, 0.00479777157, 0.0685543641, 1.459150e-02, 0.0601975545], [0.0685752556, -0.0583486594, 0.0591800138, 0.00750061125, -0.0560401902, -3.233830e-02, -0.00889045745, -0.0751292184, 0.00208455324, 0.0283044279, 0.0122346282, 0.0154982582, -0.0175325051, 0.0418424755, -0.0106866509, -0.0796005427, -0.0152194127, -0.0418237522, -0.0573388115, -0.0126819983, 0.0558919236, -0.0434946977, -0.0510282032, -0.0754880756, 0.0231512338, 0.0725841746, -0.058783792, -0.0462590903, -0.0560242906, -0.0510367304, -0.0237003043, -0.0426365025, -0.0810102298, 0.0272957757, 0.0280127227, 0.0411694646, 0.0327766761, -0.0224336088, -0.0696994662, 0.0708919093, 0.0281477496, 0.0810872688, 0.0640544072, 0.0768543705, -0.0721087307, -0.0249995477, 0.045045875, 0.0200054497, 0.0190144032, 7.914700e-02, 0.0281598717, 0.0460188761, -0.0291424878, -0.0314818248, 0.0515950695, 0.0396436527, 0.0662078336, 0.0477320328, 0.0424237847, 0.0294557512, 0.0351852402, 0.0750620738, 0.0120767653, -0.0108262971, 0.0251487792, -6.785810e-03, -0.046657186, -0.0418836251, -0.0489282385, 0.0370590314, -0.0810356587, 0.00764537603, 0.0224677175, 0.062310122, 0.0100712106, -0.0280208476, -0.0724331886, 0.039102003, -0.0381064303, -0.0278800651, 0.0553046241, 0.07606069, -0.0499009751, 0.0159540176, -0.0795476138, 0.07554809, 0.00881835818, -0.0463004932, -0.0558686778, -0.0467987061, 0.0610805377, 0.0749517158, 0.0186223015, 0.0786460563, -0.0309491493, -0.0369262509, 0.0767164603, -0.054020226, -0.0724533796, -0.025868129, 0.0611229762, -0.0291995555, 0.00632277131, -0.0121317878, -0.0684109405, -0.0310799554, -0.0556141846, -0.0202654563, 0.0141570866, 0.05388733, 0.0170530379, 0.0083020851, -0.0706994683, -0.0114907175, -0.0142931864, -3.727970e-02, 0.0569850132, 0.0240653157, 0.0207202733, -0.0652435347, 1.72942877E-4, -2.70597637E-4, -0.0141454861, 0.0434293896, -0.0056463778, 0.0652083829, -0.0647064149, 0.020970881], [0.0183184147, 7.939250e-02, 0.0384471491, -0.0511079356, 0.015447028, -0.00127854943, 0.0572787449, -0.0715435892, 0.0696247593, -0.0809131264, -0.0797346159, 0.0549459234, 0.075097464, -0.0155500844, 0.069530271, 0.0603433028, -0.0721823275, 0.0327176154, -0.0589920431, 0.0766637698, -0.0606075674, 0.035703145, 0.0223879144, -0.065884389, -0.0119491145, 0.0417793319, -0.0488211252, 0.0663903579, -0.0188180432, -0.0311442167, -9.91493463E-4, 0.0599835441, -0.023446817, -0.00544135273, 0.0306745097, 0.0471114293, 0.0567230657, -0.0331538506, -0.0577680841, 0.00304303318, -0.043095421, -0.0782876834, 0.0672175959, -0.0260001905, -0.0344435237, 0.0807921811, -0.0317726731, 0.0273431167, -0.0677996575, 0.0807399228, -0.059500508, -0.0247660577, -0.068084158, 0.0678472295, 0.0502403453, -4.798360e-02, -4.633360e-02, -0.0272851028, 0.0280884951, -0.0593899116, 0.0153709874, 8.265190e-03, 0.0253396481, 0.0232237875, -0.00411520898, 0.0591387078, 0.0168219805, -0.00338924676, -0.0425764769, 0.0117219463, -0.0396760814, -0.0261439309, -0.0531448573, 0.00750109553, -0.0320358872, -0.0557561852, 0.0611763969, 6.78040087E-4, 0.0226815641, -0.00926271826, 0.0312409475, -0.0488980711, -0.0629594475, 0.00602544844, -0.0555597842, 0.0518523827, -0.025548216, 0.0466797426, 0.00197674334, 0.0361353904, -0.0704604089, 0.0740294233, 0.0779547914, 0.0597710833, 0.00271759182, -0.044299636, -0.048411034, 0.0398422927, -0.0771870241, -0.0428280309, 0.0185060948, -0.0785943865, 0.00469458848, -0.020567093, -0.0443649217, 0.054396607, 0.0241545439, 0.0783663466, 0.0654238686, 0.0722590312, -0.0230652168, -0.0270934403, -0.00537107885, -0.069913581, -0.0342511274, -0.0611617826, 0.0637038723, 0.0159550458, -0.0120669231, -0.0532413349, 0.0199396461, 0.0709514543, 0.0533500537, 0.0515936241, 0.0464142337, -2.89529562E-4, -0.0373725407, -0.0455457941], [-0.0264820606, -0.0119323134, -0.0439735912, -6.783340e-02, -0.0194349326, 0.075985454, -0.0761911049, 0.0501224175, -0.0805127099, -0.0405467488, -0.044970151, -0.0460944809, 0.0310345292, -0.0071932897, -0.0538349673, 0.0312611759, 0.0350137502, -0.057322491, -0.0732478127, -0.0216199085, -0.080858089, -0.0574666969, 0.0799650177, 0.0716475472, -0.0787949487, 0.0490871742, 0.00151616335, -0.0385150872, -0.0649172813, 0.0681062415, 0.00646519661, -0.00962176918, 0.0224665627, 0.0628145263, 0.0308288261, -0.0524689481, -0.0795748829, 0.0545477346, -0.0302173682, 0.0232597813, 0.0386965349, 0.0434579551, -0.0556551814, 0.0232614428, -0.0235618427, -0.0637127534, -0.0766806155, -0.0325165763, -0.0694182888, -0.0690255463, 0.006878227, -0.0688042566, 0.0460047498, 0.0407264978, 0.0106380582, 0.0725092813, 0.0150092468, -0.0719633251, -0.0760752633, 0.0732675418, 0.00687640905, -0.0442859642, -0.0300700702, 2.86921859E-4, 0.0116420612, -0.031242609, 0.0558262691, -0.00921986252, -0.0697150379, -0.0715834647, -0.020181451, 0.00748743862, -0.0198533945, 0.0612150952, 0.0693868771, 0.0246878341, -0.0739398152, -0.00326869637, 0.0503739938, 0.00886755437, 0.0580679402, 0.0585062876, 0.0603781119, 0.0477476493, -0.0792215914, 0.0371359736, -0.0160252824, -0.0193950012, 0.0215711594, 0.0157322064, -0.0262667686, -0.0111766309, 0.0130858421, -0.056498833, -0.0304591954, -0.0132260248, -0.00156837702, -0.0446219295, 0.0654105172, -0.0109410286, -0.02097084, 0.0464552119, 0.047928907, -0.0546057709, 0.0543311462, 0.0539891049, -0.0605184733, -0.00205442309, 0.0700975731, -0.00566589087, 0.052214615, 0.0333057716, 0.0336912423, -0.0136679411, -0.00790982693, 0.020728223, 0.0562100783, 0.0363979861, -0.0312461667, 0.0804333612, 0.0637323782, 0.0493514612, -0.0392895862, 0.0222901553, -0.0504665598, -0.00178999454, 0.067528747, 0.00911172479], [-0.0670812801, -0.0477054119, 0.0141867697, 4.442560e-02, 0.0630734935, 0.050638862, -0.0129119027, 0.0430484116, 0.0717177689, -2.605380e-02, 0.0807320997, 0.0092035178, -0.0763325915, -0.0596892238, 0.0552452691, -0.00352558307, -0.0342506431, -0.06217039, 0.0752827376, -0.0486552492, 0.0652384534, 0.0312752426, -4.935770e-02, -0.00132415444, -0.0422080867, -0.0638894662, -0.0192718692, 0.0241935775, 0.0559915975, -0.00760308793, -0.0238039196, 0.00507804239, 0.0605509356, 0.0456696227, -0.00597259635, 7.136440e-02, -5.94845915E-4, 0.0261039268, 6.133340e-02, 6.771870e-03, 0.00853764824, 0.0260281544, 0.0303016026, 0.0211835522, 0.010594069, 0.0318814069, -0.00639681658, -0.0668337941, 0.0809140354, -0.0244462583, 0.0645398795, -0.0541952252, -0.0554124787, 0.0782414898, -0.0325083584, 0.0435730368, 0.0536374375, -0.00881296396, 0.00293957442, 0.0684040338, 0.0308066681, -0.0678962469, 0.0735362247, -0.0698894411, 0.0718084797, 0.0092626959, -0.0453351811, -0.0265190154, 0.0171619263, -0.0709835216, -0.00696753804, 0.0383295454, 0.0546021722, 7.320110e-02, 1.091440e-02, -0.0470176376, 0.0298558492, 0.0383141115, -0.0257756636, 0.036392346, 0.026575312, 0.0233979821, 0.00236420333, -0.0309683532, 0.0619627386, -0.0555615313, -0.0243434571, -0.0123656448, -0.00559882028, -0.0176721215, 0.0523440875, -0.0615700148, -2.56444619E-4, -0.0717413798, 0.0190074816, 0.0680312738, -0.016617734, -0.0665009692, -0.0285934731, 0.0740298405, 0.0162876938, -0.00132888753, -0.00898269564, 0.0660519376, -0.0715301484, 0.0649668872, 0.0350179896, 0.0627217516, 0.0681391358, 0.0593972839, -0.0476879291, 0.0287300795, 0.0628919452, -0.0581379905, 0.0210483074, -0.0604133867, -0.0132358475, -0.00569198793, -0.0783057808, 0.0497144461, -0.00848662853, -0.0482618734, 0.056977991, -0.0462698415, -0.0525514819, 0.0315190852, -0.0788000524, -0.0498674586], [-0.0272930935, -0.0579511039, 0.0689755231, -0.0459674858, 0.0283928271, 0.00201150612, 0.0778815597, -0.0376669168, 0.0587763377, 0.0476159975, -0.0620944462, 0.00936759263, -0.0724318177, -0.0243929569, 0.0110397283, -0.00601250026, 0.0227958336, 0.0645620599, 0.0463763885, 0.0438654572, 0.0279065073, -0.00604932522, -0.0167091228, 0.00966611504, 0.0713328496, 0.0538915098, 0.0289013274, -0.012028547, -0.0725533739, 0.0340013206, 3.34613025E-4, 0.0523110069, 0.0589551292, 0.0494700335, 0.0577682853, 0.0270315409, -5.392410e-02, -0.0616786443, -0.016624175, 0.0790288671, -0.0652670413, 0.0759231523, -0.0343211144, 0.0383632779, -0.0177601166, 0.0193662159, 0.0544640757, 0.0481043234, 0.0364396647, -0.0111417491, 0.0383812115, 0.0438279472, -0.0617893711, 0.0232383795, 0.0788821801, 0.0445993394, -0.0111683784, -0.0217906386, -0.0423719361, 0.0156600084, -0.0301420465, -0.0492495634, -0.0262731798, -0.0292524062, 0.0512079485, -0.0750910044, -0.0729488358, 0.0201521739, -0.0728896111, 0.0320038423, 0.0545009114, -0.0622986294, -0.0572118759, -0.0152571127, -0.00781717896, -0.0493867137, -0.0181777254, -0.0413390957, 0.0526565425, -0.0732739269, 0.0654052793, 0.0399157554, 0.0236350223, -0.0102450624, -0.0104030827, 0.0114808939, -0.0773644373, 0.0588952973, -0.026305085, 3.695630e-03, -0.0153005579, -0.0305664744, -0.0317214839, -0.0058542965, 0.0367795154, -0.0794196799, -0.01040872, -0.0558405146, 0.0124639096, -0.0167815462, -0.0475909784, -0.0579397604, 0.0375385731, -0.0429754891, 0.0795978084, -0.0639640093, 0.0100121219, -0.0154977581, -0.0241054036, 0.00973017421, 0.0113609638, -0.0127323344, -0.0565063134, -0.0314200968, 0.0721839219, -0.0398521163, -5.586570e-02, -0.0727704763, 0.0426922217, -0.0256152228, -0.0460796617, 0.049332764, -0.00310355728, -0.0313650705, -0.077643983, -0.0435911417, 0.0730076879, -0.038429644], [0.0678343773, -0.0578807332, -0.0634698868, 0.0806674808, 2.50541343E-4, 0.0585242286, -0.0435844101, 0.00855880789, -0.0111356685, 0.0530158617, 0.031634897, 0.0383856259, 0.0255615842, 0.0270193182, 0.045404911, -0.0098827444, -0.0810943097, 0.0551822521, -0.0409433655, 0.0397342779, -0.0625922977, 0.0196613725, 7.238310e-02, -0.0471936576, 0.0288831871, -0.0124541447, -0.0754293799, -0.0106256912, -0.0166621543, 0.0581937954, 0.00983956363, 0.0749258623, -0.0114082694, 0.02496171, 0.0743522942, -0.0296353158, -0.00237601507, -0.0192439593, 0.0113636106, -0.0725198686, 0.0196067449, -0.0518345535, 1.66779617E-4, -0.0637149587, -0.0361376591, 0.0485523231, 0.0347986445, 0.0127573339, -0.0573318675, -0.0169027243, 0.0571380518, 0.038525857, 0.0566760674, -0.0133526092, -0.011497885, 0.0796404928, -0.0646690354, -0.00248907926, 0.00277978019, -0.0070224693, -0.0131116593, -0.024842523, -3.331860e-02, -0.0598739535, 0.0115676345, -0.0254023895, -0.0485129543, 7.427370e-02, 0.0777477771, -0.00876311865, -0.0785893127, 0.0109009612, 0.0431218408, -0.0516953617, 1.94735825E-4, 0.0156586785, 0.0186686274, 0.0378671326, 0.00752647221, 0.070248954, -0.0163749792, 0.0789905638, 0.0271239206, 0.0797152593, -0.064123325, -0.0063486197, 0.0132079488, 0.0436945558, -0.0795469507, -0.0621863417, -0.0663335249, -0.0575643405, -0.0547897629, -0.0144335534, -0.0668647513, 0.010223968, 0.052947145, 0.0214197859, 0.0783958211, -0.0229570922, -0.0735313147, 0.060084451, 1.002480e-02, 0.0736159384, -1.749840e-02, 0.0553716198, 0.0255106408, -0.0713509098, -0.0470821708, -9.22331878E-4, -0.044427827, -0.0125003532, -0.0300491694, 0.0664003938, -0.0485227332, 0.00776809407, -0.0121017024, 0.027552858, -0.00969560351, -0.0245279055, -0.0370902121, -0.00495796045, 0.0675458312, -0.0562532395, 0.0649377629, -0.0061221919, 0.029734116, 0.0641931891], [0.00537587795, 0.0211094804, -0.0749597102, -0.0277309399, 0.0357663371, -0.0102381269, 0.0390669405, 0.0506706834, 0.078385286, -0.00510889431, 0.0129903629, 0.00166149565, -0.0204357747, 0.0450139754, 0.0401588902, 0.0665906593, -0.0336863957, 0.0749845058, 0.0201530773, -0.00143453537, 0.023706723, 0.0284549668, 0.0241259634, 0.0621321648, 0.0288328882, 0.0210353974, 0.0745149329, 0.0289720632, -0.023276506, -0.0455467552, 0.0047361576, -0.040804103, 0.0650349557, 0.00305548985, 0.0698727295, 0.066495955, 0.067998439, -0.052143611, -0.0634134039, 0.0442901067, -0.0741689578, 0.0777112543, -0.0586899854, -0.0559675321, -0.00463227974, -0.041559007, 0.0218097083, -0.0606609248, -0.0532272682, 0.0760504752, 0.0500294082, 0.0230728295, 0.0599340163, -0.0620838627, 0.0551569574, -7.840060e-02, -0.0552918836, 0.0442379788, -0.0544889122, 0.0646871924, -0.0645563751, 0.0640536547, 0.0231896918, -0.0537085831, -0.0181500968, 0.0694559887, 0.0755256489, -6.917700e-02, 0.0753062665, 0.0118234055, 0.071165353, -0.0157130286, 0.0167413671, -2.759730e-02, -0.0612576231, -0.0659381673, -0.0652379691, 0.00777553208, 0.0688784048, 0.0190863926, 0.0598990284, -0.0751315802, 0.0352741778, -0.0111284973, -0.0117371706, 0.0804622545, -0.0712234825, 0.0726307631, 0.0481657274, -0.0567050688, -0.02392051, -0.0212993193, -0.0642801821, -0.0289423987, 0.024981562, -0.0670260488, 0.00878285896, 0.0505330861, -0.00175065268, 0.0550791025, -0.0271231271, -0.0688740611, 3.238020e-02, -0.0146729527, -0.0638064817, 0.0737972185, -0.06073853, 0.0127077941, 0.0301124398, -0.00845155213, -0.0198519733, 0.0234220438, 0.0740302131, -0.0706884637, -0.0733151659, -0.0190379284, 0.0755043402, 0.0198101494, 0.0346921906, 0.044938188, -0.0794649124, 0.00455183396, 0.00922346953, -0.0508982837, -0.0429610237, 0.0566115044, -0.0298683085, -0.0115195559], [-0.0291983578, 0.0347375534, -0.0665313303, 0.077243492, 0.062111102, 6.100240e-02, -0.0186406188, 0.0711123347, 0.0680981055, -0.0583191104, 4.394540e-02, -0.0191500615, -0.0419156514, 0.0159688871, 0.024320228, 0.0418022834, 0.0152223073, 0.0485882834, -0.0467320457, -0.0448675863, 0.0267530549, 8.086260e-02, 0.0471672267, -0.0663306341, -0.0744984149, 0.0492743216, -0.0741639808, 7.585230e-02, 0.0735567436, -0.0805912539, 0.0233291648, 0.0296104066, 0.0511365682, 0.0100615295, -0.00831545703, -0.055368308, 0.0774187967, 0.027745761, -0.0492698438, 0.0175166912, 0.0654721707, -0.00694464659, -0.0124460179, -0.0522533469, 3.383420e-02, 0.0598431639, 0.0471043363, 0.0616031401, 0.0450956263, -0.00433252426, 4.137410e-02, 0.0800341219, 0.044180911, 0.0469584353, -0.0784665569, -0.0612465292, -0.0239708275, -0.0595478155, -0.0584242493, 0.0255732574, 0.0117134275, 0.0531833321, -0.0673836395, 0.0780513286, 0.0276341513, 0.00191774569, 0.0041699945, 0.0552653447, 0.0614704452, -0.0266406909, 0.0588418618, -0.0569902286, 0.0728952512, -0.0599398874, -0.0806768834, -0.0589669235, -0.014336125, 0.02629718, 0.0744849071, -0.0216701981, 0.0188575126, 0.0155085186, -0.0211284049, -0.0294158328, -0.0402272642, 0.0177553687, 0.0321337916, 0.0809598043, 4.116260e-02, -0.0395678692, 0.020433262, 0.0316384658, 0.0577837862, 0.0760370269, 0.0300076939, -0.057019867, -0.0660176724, 0.0248746797, 6.613580e-02, -0.013205138, -0.0693391562, -0.0106207179, 0.00771707482, -0.0567451566, 0.0578956231, 0.07717181, 0.048812069, 0.0437301882, 0.051389046, 0.0747843161, -0.0493122153, -0.00885564927, 0.0151160341, 0.0572876893, -0.0609420128, 0.0713870376, 0.0606063232, -0.0107111232, -0.0424625687, 0.00542809861, -0.0304287914, 0.0473205782, -0.0431139655, 0.0472610071, -0.0498736091, -0.0274488982, -0.0776850357, -0.0329253562], [0.0663706362, 0.031028932, -0.00728455884, -0.0764136612, 0.0349043496, 0.0735297948, 0.0244218167, -0.00321477302, -0.0207026843, 0.0702339411, -0.0491336584, 0.0646906495, 0.057363838, -0.0534264073, -0.00264579826, 0.0612451471, -7.15258531E-4, -0.0351453573, 0.0119098527, -0.0170561951, 0.00667665433, 0.00308571942, -0.0519586056, -0.0253016837, -0.0427431092, -0.00993627123, -0.00288402988, -0.00230092811, 0.0168019775, 0.0486952625, -0.0032388703, -0.0150222806, -0.0409379303, 0.023860164, 0.0144286836, -0.00596596068, -0.0239151977, -0.0181697961, -0.0414541587, -6.733430e-03, -0.0124059869, 0.0798010826, 0.0186740048, -0.0397617221, -0.0337321162, -0.0282483604, 0.0163182616, 0.00419625128, -0.0663103834, 0.0726536289, -7.644490e-02, -0.0772605836, 0.060403347, -0.0561873317, -0.0463876352, -0.0484050699, -0.0761936306, -0.0779656469, 0.0158862025, -0.0396383107, 0.0587530062, 0.0510108545, -0.00610838179, -0.0788973048, 0.00548840268, 0.0335032381, 0.0757802352, -0.0758602545, 0.0171344597, 0.0390440077, -0.00621970464, -0.0473162234, -0.0391898453, -0.0185815152, 7.657870e-02, 0.0513728112, 0.0119684385, 0.0350006483, -0.0382024832, 0.0176110566, 0.0612946935, -0.0335643925, -0.0355706625, 6.947140e-02, 0.0436900631, -0.0425534621, 0.0306297876, 0.0104004117, 0.0156097403, 0.0289594941, 0.054641474, 0.0696687475, 0.0629895255, -0.0731672123, 0.0485063083, -0.039442908, -7.727120e-02, 0.0486506894, 0.075414829, -0.0692873299, -0.0394443758, 0.058596544, 0.00364485593, 0.00903516542, -0.054644376, 0.0596243441, -0.0648349747, -0.0297442824, 0.0595552325, 1.2570589E-4, 0.0447969362, 0.0779913738, 0.0480459295, -0.0470424853, -0.0226461776, 0.0360003449, -0.00101239234, -0.0326932818, 0.0534229167, -0.0550144836, 0.00492591597, 0.0609052666, -0.0692677274, -0.0802925601, 0.0442591719, -0.0566348806, 0.0661311373, 0.0179387312], [-0.0729977116, -0.0330243632, 0.0739061832, 7.052100e-02, -0.0548005067, -0.0609703921, -0.00475269416, -0.0061055487, 0.0287678838, -0.0526070222, 0.0309242141, 0.00102724507, -0.0760093629, -2.462110e-02, 7.520330e-02, -0.0659632608, 0.00389764877, 0.0777040123, 0.0679089725, -0.0162313916, 0.0209070966, -0.0613556318, -0.0411858112, 0.066838175, -0.00384139479, 0.06496232, 0.065656282, -0.0496144071, 6.93609647E-4, -0.045328252, 0.00602547359, -0.0296639428, -0.0753530189, 0.00395048922, 0.0182498507, 0.0518599637, -0.0317040756, -0.0376569703, 0.0208696835, -0.0390330441, -0.0307406709, -0.0236334912, -0.0261380281, 0.077656798, 0.00604362506, 0.00899451319, 7.642730e-02, 0.00795504171, -0.0377980731, 0.0410710275, -0.0720582604, -0.0716080591, 0.0538431779, -0.0427552052, -0.0387873799, 0.0709685162, -0.0570519902, -0.0233143382, 0.0211384874, -0.0566472709, -0.0582971834, 0.00335453614, 0.0765830651, 0.0470513478, 0.00969281141, 0.057454437, 0.0120848799, -0.0157534182, -0.0751515776, 0.0209590103, 0.0115150688, 0.0155730173, -0.0412533358, -0.0739861354, 0.018041959, -0.00589459576, 0.0386174731, -0.0168328304, -4.693410e-02, 0.0155682052, 0.0154734701, 0.0480086952, 0.0444714278, -0.0341265537, 0.00985967926, 0.0246204399, -4.501120e-02, 0.0508700311, 0.0439175107, -0.0441737697, -0.0782675966, 3.357680e-02, -0.0311519969, -0.0430816039, 0.019669706, 0.0696135238, -0.0743777305, 0.0585151054, -0.0320026539, -0.0542642251, -0.0720352158, -0.0110852821, 0.0278929453, 0.0646194592, 4.752330e-02, -0.0767641887, -0.0766115486, 0.0270807613, 0.0122792181, 0.0571142323, 0.0667796135, -0.0388900302, 0.0559234433, -0.0361178182, 0.0507067367, 0.037897449, 0.0107054319, 0.0176392868, -0.0576922297, -0.0618165694, 0.0787158086, 0.07064569, -0.0514623821, 0.00541220885, 0.0343738087, -0.0238404647, 0.0233575962, 0.0720319897], [-0.00315200025, -0.005224369, -0.00882916338, 4.412880e-02, -3.816290e-02, -0.0787819475, 0.0200103577, 0.0282256957, -6.932170e-02, -0.0120801646, 0.0601145364, -0.0330882631, -0.00203879736, 0.0181098748, 0.0588415414, 0.0788610056, -0.056122452, -0.0383131094, 0.00727072731, 0.0323893838, 0.0211329814, -0.0659178719, -0.0704888627, -0.0689928308, 0.0598158389, -0.0635470375, 0.0487471223, 0.0260199904, -0.0648032054, 0.0276070759, 7.066510e-02, -0.0116654336, 0.00502404058, 0.00966694299, 0.00210424676, -0.030892916, 8.942450e-03, 0.0282545369, -0.0457783826, 0.00638259063, -0.0656558871, -1.959040e-02, -0.0252630394, -0.0531628206, -0.0638696477, -0.0530332141, -0.0551357083, -0.0354029313, -0.0523820221, 0.0422695912, 6.423640e-02, 0.0684070736, -0.0375541151, -0.0201632455, 7.572100e-02, 0.070218496, 0.039399527, 0.0136743067, 0.0147871273, -0.0163321923, 0.0405242927, -0.0505402498, -0.0347434692, -0.0516848043, 0.0033730634, 0.0488708466, 0.0451664701, 0.0322556533, 0.0514721386, 4.227840e-02, 0.0280363411, 0.014169706, 0.00295367138, 0.0126820272, -0.010929713, -0.0255044214, -0.0567779467, 0.0218584016, -0.0754646956, -0.067497395, -0.00643266923, 0.0683359429, 0.036021594, -0.00328606018, 7.37442693E-4, -0.00471423101, -0.00326124555, -0.0107242158, -0.0703674256, 0.00679236604, -0.0630135089, -4.986300e-02, -6.421820e-02, 0.0413178578, -0.0745303705, -0.0411371626, 0.0347694717, -0.00493738847, 0.0752970129, -0.0198449474, -0.0367984064, -0.0321430974, 0.0479565263, -0.0587304644, -0.00470927963, 0.0591261386, -0.0221331734, 0.0711846649, -0.0523554422, 0.0227103047, 0.0460584052, -0.0330491327, 0.0260395333, 0.0154557135, 0.00177113549, 0.0322098248, 0.0653741657, -0.0391096361, -0.0371028297, 0.0112119801, -0.0838790759, 0.0423981436, -0.0106951157, 0.00552061619, 0.0180321839, -0.0238819662, -0.0217441507, -0.0139977178], [-0.0410334654, -0.00987553503, -0.0457412191, -0.0319327377, 0.00225837063, 0.00415087584, -0.0669846684, -0.0230482277, -0.0632767826, 0.0775615349, 0.00663429499, -0.0660117194, -0.00220082048, 0.0280534644, -0.0619674809, -0.0519676842, -0.00946800131, -0.0750084296, 0.00580944773, 0.0669619367, -3.356150e-02, -0.0583321042, -0.0340875313, 0.0219035801, 0.0655180141, -0.00889202207, -0.0133174742, -0.0584557205, -0.0155990152, -0.0313617736, -0.0334374905, -0.00898873247, -0.00475813961, -0.0374153033, -0.00301661715, -0.0344647802, 0.0362455137, -0.0162622016, 0.056298852, -0.0552590862, 0.0207931288, 0.0516335666, -6.855620e-03, -0.0359575525, 0.0769187137, -0.0485041663, 0.04396642, 0.0464078933, -0.0481153689, -0.0142797595, 0.0539568178, 0.032166142, -0.0493306369, 0.0516860932, 0.0741363317, -0.056340389, 0.0134491511, 3.47558758E-4, 0.058316052, -0.0389576815, 0.0116523579, 0.0188986938, -0.0301293023, 0.0540728569, -0.0672335774, 0.0598869063, -0.00879488233, 0.0126579283, 0.0616021901, 0.0837168917, 0.065845564, -0.0451010242, -0.00641694292, -0.0643106773, 0.0392221734, -0.0698946863, -0.0615742579, -0.0570733733, -0.0190457739, 0.0715597942, -0.0288300328, 0.0681405142, 0.0118813068, -0.0154466201, 0.0541923195, -0.0624082572, 0.050857611, -0.0417058766, -0.0529270954, 0.059330184, -0.0152276671, 0.0760593414, 0.0437895395, 0.0551990233, 0.0390080474, -0.0294839889, -0.006693264, 0.0734573826, 0.0321768895, -0.00113708095, 0.0541620106, -0.0636770129, 0.00462492602, -0.00971797481, 0.0550389662, -0.00125723681, 0.0389732271, 0.0558852404, -0.0256695431, -0.0153282005, -0.0584744662, -0.069148533, 0.0607147478, -0.0410036594, -0.0367029347, 0.017246319, -5.423720e-02, 0.0245059039, 0.0646050721, -0.0501327589, 0.0721088722, -0.0217468347, -0.0329347029, 0.0552436784, -0.0512270778, -5.869700e-02, 0.0258231554, -0.080357939], [0.026040839, 0.0686685144, 0.0191683155, 0.0139550772, -0.073547326, -0.0740110502, 0.0769732892, -0.0757101104, 0.0776014924, 0.0482947938, -0.0436493456, 0.00915520079, -0.0104503948, 0.0285647921, 0.00891598593, 0.0220351759, 0.0056264028, 0.0304156914, 0.00198572059, -7.439220e-03, -0.00122029113, 0.0532358363, -0.0394046456, 0.0658983886, -0.0734640657, -0.0747786537, -0.0664549171, 0.0243602283, 0.0805168077, -0.0259969942, 0.00880624074, 0.0680935532, 0.0405442081, -0.0876844152, 0.05557587, 0.0495573431, -0.00604227791, -0.0553589724, -0.0222458541, -0.0715181529, 0.0778030529, 9.45468782E-4, -0.0693576857, -0.012813638, 0.019354308, -0.0270274542, 0.0128888302, 0.0540477782, -0.0112516033, -0.0466526896, 0.0605801456, 0.065331839, -0.0181364417, -0.0309797581, -0.0685560852, -0.0599524453, 0.0172847696, 0.0417205505, -0.0301083233, 0.0580892228, 0.0791819393, -0.0496047623, 0.0285429657, -0.069297269, -0.0170989297, -0.0285982136, -0.00867167674, 0.0752235726, 0.019106511, -0.0357379019, 0.0345791616, 0.0139026102, -0.0379388519, -0.0523810908, 0.0215837955, 0.0384311117, -0.0674932897, -0.0100894067, -0.0186920483, 0.0337413475, 0.0780173838, 0.0498901568, 0.0428159274, -0.0399247706, -0.00318983826, 0.0400062054, -0.051905293, 0.0659605414, -0.0288124196, 0.0762375593, 0.0162468236, -0.0318818651, 0.0486760885, 0.0560950376, 0.060043361, -0.00582634518, -0.0502135791, 0.0287888143, 0.0156510845, -0.0538546219, 0.0691455081, -0.0179531537, 0.0264671575, 0.0702218562, 0.0467336103, -0.0404560938, 0.0759133994, 4.3574709E-4, 0.00821217149, -0.0361014456, -0.0224749725, 0.00158819545, 0.0734990314, 0.0739572644, 0.0417302065, -0.0625247955, 0.0583873242, 3.113900e-03, -0.0234998409, 0.0519884825, 0.0216293857, 0.0803768038, -0.00994867924, 0.0270153135, 0.0206179861, -0.0370627083, 0.0374855511, -0.0589820929], [-0.0411766171, 0.0723037794, 0.00221449579, 0.0201334264, -0.00561876455, 0.0671960264, -0.0414923355, 0.0291054752, -0.0148671968, 0.0355527475, 0.0461482666, 0.0224622674, -0.0493053347, 0.0416060872, 0.074386254, 0.0885109081, -0.0234359484, -0.0689911321, -0.0162510742, 0.0459132567, -0.0432479531, -0.0730463639, -0.0138845425, -0.0607176274, -0.0421696939, -8.338080e-02, 0.0704561546, 0.019157555, 0.00315244216, 0.00312091643, -0.0678987578, -0.0669416711, -0.0269257724, 0.00235365378, 0.051267691, -0.0778313949, 0.031160837, -0.054911878, -0.00177618198, -0.0284004193, 0.0233780481, -0.0421829335, -0.058990702, -0.0565360561, -0.0106222276, 0.0795407146, 0.0363804772, -0.040102493, -0.0486824624, 0.00711629725, 0.0135889482, 0.0408698134, -0.0381214321, 0.0297167022, -0.00337021379, -0.0315076448, -0.0475130342, 5.585950e-02, -0.00757747469, 0.0578855611, -0.00994243659, -0.0662088245, 0.0433920175, -0.0183029212, -0.062907733, -0.0503023081, 0.0830211639, 0.0798744559, 0.0565601811, -7.730150e-03, 0.0657289401, 0.0785741284, -0.0510997809, 0.0548309945, 0.00615095627, 0.0605404526, -0.0211510547, -0.0316896401, 0.0495528094, 0.0527848862, -0.0395218842, -0.08322189, 0.0529284775, 0.0408681072, 0.0743741915, -0.0245839693, 0.0188906193, 0.0249402095, 0.0768619105, -0.0379651375, -0.0538199097, 0.040340323, -0.039596267, -0.0286762416, 0.013430913, -0.0795052797, -0.076690726, 0.0014008322, 0.0193189178, -0.0134319635, -0.0757271945, -0.0762666613, -0.0663199499, 0.0555370152, 0.010414605, 0.0113182366, 0.0128523186, 0.00774451345, 0.0731821582, 0.0586378425, 0.0458080918, -0.0591290407, 0.00280288677, -0.0413458347, -0.0209685117, -0.0398727655, 0.0325413421, 0.0490293317, -0.0633390471, -0.0287779495, 0.0501135588, 0.00575296814, 0.0261853449, -0.00151309418, 0.0528228283, 0.0528290197, 0.0413905531, -0.0839710385], [0.037359044, -0.0364844352, -0.0415186547, 0.00156860054, 0.0673119947, -0.0454619899, -0.041605711, 0.0450774841, 0.0110777281, -0.0109803611, -0.0717178732, -0.0801314711, 0.0139129655, 0.0405365191, 0.0144436369, 0.0744871646, -0.0269784275, -0.0427828543, 0.0183239896, -0.0781039521, -0.0572203174, -0.00900296401, 0.0387119129, -0.0433220491, 0.0376519263, 0.0607843101, 0.0253165718, -0.0677769333, -0.0524893701, -0.0220929347, -0.0424341485, 0.0462394767, -0.00915308576, -0.00440908177, 0.0591929629, 0.00135392393, 0.0251647644, 0.031540107, -0.0735175684, 0.0580559447, 0.0373425595, -0.0453435928, 0.011278078, -0.0167728141, -0.0146858497, -0.0734962522, 0.0448583066, -0.0525369458, -0.0728298127, 0.0656579733, -0.0467359908, -0.0207377877, 0.017260937, 0.00781576521, -0.0114345495, 0.0776093304, -0.00675970828, -0.0303337201, 0.0565914735, -0.0227262061, 0.0165442806, -4.538410e-02, -0.0481159277, -0.0754243433, 0.0279190391, 0.0750319287, -0.0364392698, -0.0334267579, 0.0440511107, 0.0432548895, -0.0542154126, -0.0418645628, -0.0747676715, 0.0523141548, 0.0605760701, -0.07660418, 0.0805328115, -0.0347024202, -0.00461399509, -4.584960e-02, -0.0617328882, 0.0732274204, -0.00270878407, -0.0448728241, -0.0208416209, 0.0348764434, 0.0148940813, -0.0391821377, 0.0872166976, -0.04377985, 0.0214751139, 0.0191764329, 0.0630333051, -0.079688251, -0.0322963968, -0.066009514, 0.00481424108, 0.0896517857, -0.0457430817, -0.076523982, -0.024623638, -0.0334366746, 0.0152412439, 0.0539898761, 0.0427370928, 0.0552810095, 0.0560670495, 0.00241389032, -0.0582145117, 0.0371554233, -0.0393545628, -0.0796917826, 0.0496820249, 0.0598704964, -0.0716482401, -0.0432908349, 0.0485107936, -5.970590e-02, 0.037824858, -0.0274878461, 0.01518449, 0.0814319178, 0.0215584822, -0.0364405662, -0.08353661, 0.0222257655, -0.0917453765, 0.0699309408], [0.0481221229, -0.060467694, 0.0637915581, -0.0289105605, -0.0240839422, -0.0384176895, 0.0119966613, -0.0163523071, -0.0184721798, 0.0286875684, -0.047074154, -0.0170856435, -0.0114903376, 0.0452816375, 0.0461343601, -0.0683487579, -0.0560090505, 0.0439199507, -0.0860522836, 0.0453021191, 0.0547229797, 0.0557592511, 0.043093577, -0.00458097551, -0.0888042449, -0.018532373, -0.0486680344, -0.0242743455, -0.0201809183, 0.0713254362, 0.0213093795, -0.0557128452, 0.0216019675, 0.00992802344, 0.0704001933, -0.0618957393, -0.0579777472, -0.0427341238, -0.0358380154, 0.0250280406, 0.0204540864, -0.0331285261, 0.0271221492, -0.0215179063, -0.0329300873, -0.0361285508, -0.020507684, -0.0687096491, 0.0642623082, 0.0230520293, 0.0684425458, 0.0305568334, 0.00859866198, 0.0645634308, -0.00849460345, 0.0730194673, 0.0557074323, 0.0626857653, -0.0696753189, -0.0224321112, -0.0706155524, 5.449040e-02, 6.450380e-02, 0.0475591309, -0.0133548211, 0.0736076533, 0.0138953393, -0.0713662952, -0.0466799103, 0.0816363096, 0.0256465189, -0.0190003179, 0.0681461245, -0.0613778494, -0.0207045563, -0.00510186795, 0.0337447748, 0.0570210814, 0.0432172604, -0.0332683735, 0.00674777291, -0.0703627095, 0.0208086055, 0.0740299522, 0.0515670069, 0.00514046615, 0.054144755, -0.0747579038, -0.0242701322, -0.0163112804, -0.0408877544, -0.0109736696, 0.0705493167, -0.0251510199, 0.0254554935, 0.0727066398, 0.0231256634, -0.0193294603, -0.00777777517, -0.0179801751, 0.0553560331, 0.0592995323, 0.0776084735, -0.0155841466, 5.587050e-02, 0.0784024149, -0.021676315, -0.0250157379, -0.0298838448, 0.0775274112, -0.00749640819, 0.00429875683, 0.0560927577, -0.0304111503, -0.0549562424, -0.0166311767, -0.0151331108, 0.00283453497, -0.0691903755, -0.041926641, -0.0172093865, -0.0603989214, -4.313860e-02, 0.0296361316, 0.0123627614, 0.0117258448, 0.0564712733, 0.0319169238], [-0.00756630395, 0.00980574172, 0.034695942, -0.0622354895, -0.0741177797, -0.0122827571, -0.0712678581, 0.0593206286, -0.0183446072, 0.0345423855, 0.0266475808, -0.0398858041, 0.0357943438, 0.0329499915, 0.0109618884, -0.0507650375, 0.0100690071, -0.0669584051, 0.0298594572, 0.00524391886, 0.00740481075, -0.0483994111, 0.0201992895, 0.0611447804, -0.055938825, -0.0139462175, 0.0738736391, -0.0281327385, 0.0311609879, -0.0687735751, 0.0171929952, 0.00492928131, -0.0217356011, -0.089413844, 0.0404903367, -0.0274454895, 0.043058116, -0.0237059984, -0.0272943731, 0.0456463657, 0.0202286355, 7.304150e-02, -0.0821385085, -0.0626357943, -0.0619058907, -0.0761944875, -0.00934098847, 0.0545374714, -0.0271140281, -0.0214339532, 0.0692810342, 0.0465483293, -0.0643776208, -0.0617547892, -0.0550367199, 5.278580e-02, 0.0453845225, -0.00500112353, 0.0179799851, -0.0255164094, -0.0237265527, -0.00388635299, -0.0657728091, -0.0409413949, -0.0235489234, 0.0285926685, 0.0184715185, 0.0395753793, -0.00740716094, 0.0114862472, -0.0360435732, 0.0117708975, 8.988030e-03, 0.0172265768, 0.050691776, -0.0313403159, -0.0128115397, 0.0362694822, 0.020185275, -0.0125057586, -0.079247184, -0.0200662073, 0.0108163254, 0.00204185816, -9.352610e-03, -0.0453662872, -0.0280389152, 0.0726897642, 0.0771383047, 0.0442004092, -0.0211803839, -0.0703576952, -0.0448189899, -0.0286568925, -0.043952696, 0.0556744188, 0.0322713591, -0.0327286273, 0.0716628432, -0.0240614396, 0.0762872919, 0.0130485864, 7.077070e-02, -0.0818742514, -0.0501314104, -3.530440e-02, -0.0648792535, 0.0482741147, 0.0259758178, -0.0286532268, 0.00143566239, -7.599620e-02, -0.066733174, -0.0793463439, -0.068963252, 0.0804462284, 0.0367778279, 0.0129984096, -0.0266033132, -4.221070e-02, 3.88439425E-4, -0.0261208527, -0.0337025933, 0.00741082663, -0.0189223941, 0.00460693799, -0.0305804238, -0.0857939794], [0.0503871106, 0.0617883243, -0.0188253801, 0.0270489063, 0.030369468, 7.432880e-02, -2.02087642E-4, 0.012110915, 0.0623620413, -0.0275108945, -0.0443620868, -0.0620625056, 0.0579187497, 0.0732558444, -0.0420814827, 0.0622783489, 0.00666026445, 0.0255321525, -0.0726967901, -0.0497683287, 0.0800908804, 0.029034229, 0.0786616951, 0.0432470478, -0.0673553049, -0.0402326435, -0.0160815343, 0.016948482, -0.0614573583, 0.0072118924, 0.0413894504, -0.0713001787, -5.14185114E-4, 0.0150010036, -0.0304551646, 0.0794335455, -0.0821421444, 0.00720239943, 0.0114314063, 0.0843947381, -0.0263608638, 0.0114835557, 0.0692588612, -0.0405800939, -0.0117806075, -0.0745297149, 0.0529494025, -0.0462187529, -0.0317770876, 0.0274866596, 0.0322403684, 0.0756847709, 0.0455253646, 0.0713154227, -0.0329526737, 0.0506430939, -0.0686387643, -0.0421500318, -0.0173028372, 0.0444264263, 0.0599415526, -0.0754884258, -0.052371759, 0.0665173382, -0.050573349, -0.00908843986, -0.0314535685, -4.284880e-02, 0.0692688078, 0.0335156135, -0.0401912183, 0.056827262, 0.0436422937, -0.00329185487, -6.896340e-02, 0.061753273, -0.0200834237, 0.0448381752, 0.0310695879, 6.423480e-02, -0.0444193706, -0.00198051124, -0.0614686497, -0.0372603424, 0.0828461721, 0.0577860773, -0.00232209708, 0.0361044146, -0.00226284214, -0.0150917219, -4.858300e-02, -0.0631300583, -0.0453670099, -0.0808656141, -0.0376044773, -0.0287462026, 0.0695446804, 0.0811686665, 0.0158799365, -0.00851415656, 0.046775721, -0.00641086744, -0.00936331507, -0.0357296951, 0.0706938058, -0.0686307922, -0.0458275266, -0.0337256193, 0.0230567437, 0.0102110608, 0.0073551028, 0.080017969, -2.790570e-02, -0.0528518744, 0.0474993847, 0.0373202898, -0.0489577763, -0.0263176765, 0.0801543667, 0.0490677021, -0.0737151876, 0.0826304256, 0.0112468246, -0.0306017492, 0.044541508, -0.03069585, -0.0399862714, 0.0579131246], [0.0190691203, 0.0392772853, -0.0517104678, -0.0282688234, -0.0230598263, -0.0567824766, -0.0144292107, 0.0172699969, -0.0596526153, -0.0741908178, 0.0276364964, 0.0618049465, -0.0714919642, -0.00543750403, 0.0443401895, 0.00118755514, 0.0201289784, 0.0245573763, -0.0230990052, 0.0614830777, 0.0191960577, 0.0674650148, -0.0373794027, -5.241180e-02, -3.688370e-02, -0.0303542223, -0.0517318547, 0.0183362477, -0.0621454752, 0.0480660722, -0.0649005472, -0.0511435047, -0.00115361298, -0.0618763976, 0.0490348674, -0.0551366545, 0.0623538941, -0.0109984064, 0.0412331074, 0.076291725, 7.105280e-02, 0.0734120384, -0.0476351529, -0.00769144855, -0.0215737801, 0.0560502559, -0.0642579049, -0.02503841, 0.0778390914, 0.0330146588, 0.015877014, -0.0169435423, -0.0586396717, 0.0331627727, -0.030972546, -0.0287345015, -0.0203419272, 0.0346185192, -0.0566117801, -0.0314660035, -0.0641085803, -0.0763275474, -0.0467430167, 0.0251302551, -0.0129717039, -0.0133815296, 0.0274753682, 0.0140124373, 0.0103987185, 0.0261010751, 0.0211685318, 0.0437459275, -0.0762596056, 0.0264286641, -0.0441363305, -0.0263216794, -0.00667565409, 0.0530690774, 0.07600566, -0.044557739, 2.876660e-03, -0.013841887, 0.0208730213, 0.0256349575, -0.00865953881, -0.0808498784, -0.0761951059, 0.0555193238, 0.0748377815, 0.06591589, 0.052700635, 0.0244837161, -0.00254074647, -0.059398368, 0.0591774099, -0.0012061632, 0.00501266541, -0.0234619807, -0.0523589589, -0.0571319945, 0.024924187, 0.0193277206, 0.00255984254, -0.0369148403, 0.0216064695, 0.0338670909, 0.0653150752, 0.0236718934, -0.0579247698, 9.45637701E-4, 0.0618166886, -0.0402785502, 0.00929658767, -0.0345281065, 0.0364793316, -0.0205321908, 0.0374710634, -0.0284311697, 0.0479878038, -0.0298230723, -0.0394531302, -0.0335461795, 0.00393998437, 0.050388068, 0.0590297766, -0.02004572, 0.0290095136, -0.0709308087], [0.0138355624, -0.0709128827, 0.0630675554, -0.0381799117, -0.053778179, 5.403830e-02, -0.0209342614, 0.0443753414, 0.0829193443, -0.00175846403, -0.0648443326, -0.00139003317, 0.0631518364, 6.857320e-02, 0.00182265858, -0.0052779871, -0.0221867114, 0.035720408, 0.010970437, 0.0708755851, -0.0787194073, -0.0257517453, -0.0393948257, 0.0262940098, 0.0452871807, -0.0588889532, -0.00918609835, 0.0638973787, -0.0532135628, 0.051864069, -0.0679079518, -0.0320819877, 0.0665710792, 0.0635225847, -0.0604796037, -0.0697040334, 6.326810e-02, 0.0307573788, 0.0730503499, -0.0667260811, 0.00753515772, -0.00586780347, -0.0442212895, 0.0667109638, 0.00125822844, 0.0633521304, 0.0221688189, -0.0127217444, 0.0107024666, -0.053146366, 0.0439835265, 0.0606671385, -0.00946057308, 0.0880088955, -4.668450e-02, -0.00328684878, -0.0650194138, 0.0366588756, -0.0500823446, 0.0401177257, -0.0325208753, -0.0166265834, -0.0722104833, 0.0602452457, 0.0436331294, 0.0707211047, 0.024437638, 0.0500379242, -0.0485516712, 0.0156514123, 0.0266452301, -0.0729221851, -0.0439338945, 0.0384140275, 0.00298086856, -0.00852103624, 5.221330e-02, 0.0144590661, -0.0205876082, 0.0739413201, -0.0217561163, 0.039999716, -0.0455945916, 0.0574539937, 0.0730171129, -0.072115153, 0.0284436774, 0.0488588586, 0.0335825831, -0.0265432578, 0.0494364835, -0.035922002, -0.0453228243, -0.0597797483, -0.0662093908, 0.0480172895, 0.0634763315, 0.0453228764, 0.0198491365, 0.0443998128, -0.023717057, -0.0320623741, -0.00807207078, 0.0118030058, -0.0743478686, 0.0798778385, -0.0518457405, -0.0129878744, -0.0675843135, 0.0727577209, 0.0589486361, -0.0491344742, 0.0488209911, -0.00504454738, 0.0347500481, -0.0203818921, -0.0206773691, 0.0245280899, -0.00114036526, 0.0555838682, -0.0586518422, 0.0642871186, 0.0729963109, -0.0716157779, 0.0618090555, -0.0476233587, 0.0277493559, 0.021031443], [-0.0453490838, -0.0809882059, -0.0475461856, -0.0690161213, 0.0639374256, 0.00512820296, 0.0287130699, -0.0497675687, -0.0138602229, 0.0614764541, 0.0282613747, 0.00232585752, -0.0331671573, -0.0585698895, 0.0670064985, -0.0068131974, 0.00741165737, -0.0261253826, -0.0378181636, 0.0786404535, 0.0124370717, -0.00823429413, -0.0250371601, -0.0282393768, 0.0628573298, -0.0562992655, 0.0530575886, -0.0262117404, 0.0732566491, 0.0158479735, 0.00176976447, -0.011364569, -0.0365767293, -0.0185101442, 0.0245341305, -0.0379792266, -0.07745447, -0.0334629752, 0.0791521519, -0.0477728508, 0.0479835495, 0.050997965, -0.0361397378, -0.0237403885, -0.0202537756, 0.0241730232, -0.072834909, -0.00706723658, 0.0305269584, 0.0455101281, -0.0627904832, 0.0234256927, -0.0129049402, 0.0554751866, -0.065114513, 0.0211865641, -0.0577636398, -0.0282749012, 0.051471334, 0.0764047131, -0.00674785673, -0.059103936, -0.023015013, -1.289390e-02, 0.026872972, -0.0714594498, -0.0212002844, -0.0640047118, 0.069052808, -0.0708899274, 0.0613073595, -0.0748117715, 0.0596879199, 0.0711589605, -0.0101588713, 0.0641540959, 0.0390465744, 0.0274824835, 0.0579611957, 0.0741217583, -0.0201845933, -0.011866116, -0.0429035388, 0.00144181459, 0.00826984365, 0.00511137303, -0.0236993246, 0.0571875125, -0.00333552924, 0.0805197805, 0.0579425022, 0.0576642528, -0.0557703823, 0.022483021, 0.0708616599, 0.047307428, -0.0363934673, 2.797880e-03, 0.0363074765, 0.00625563087, 0.0291258339, -0.0299896263, -0.0171585381, 0.0241698865, 0.0287328307, -0.0586467087, 0.0666396915, 0.0785838663, 0.055059772, -0.0288628098, -0.0629924163, -0.0787203311, 0.0612182021, 0.0626351088, -0.0791428089, -0.0358915031, -0.0807712525, -0.0323075019, 0.0546598099, 6.095000e-02, -0.060831774, 0.0613303482, 0.0461657159, -0.0335887969, 0.0712631494, -0.0301726311, -0.0124068549, -0.0503529906], [-0.063564375, 0.0522655435, -0.0600037351, 0.00489705754, -0.0745554417, -0.0318483412, -0.0736252293, -0.0736751929, -0.0024717038, -0.0333799198, 0.0250303354, 0.00804712064, -0.0314048305, 0.0579779595, 0.011318177, 0.0765484646, 0.0426852629, -0.0348741263, -0.00650935713, -0.0274573788, 0.0163268279, -0.0560646206, 0.0701338276, 0.0786672905, -0.059213765, 0.0306630544, 0.0499416701, 0.0384437405, 0.069186464, -0.0630616173, -0.0619744658, -0.0253634229, -0.00997777562, 0.0605305247, -0.0468518138, -0.0499880239, 0.078219533, -0.0369949266, -0.0146013601, 0.0313622281, 0.0842901766, 0.0212803427, 0.0505574569, -0.0549559034, -0.0664954111, -0.0438459069, 0.00560938241, -0.0556093529, 0.0506301895, -0.00822388567, 0.0742425174, 0.0520074219, -0.0362773277, 0.0411595777, 0.0138971796, 0.0458126925, 0.0253672209, -0.0348121487, 0.024489576, -0.0260415617, -0.0174261276, 0.0667622834, -0.0174753573, 0.0747833103, -0.00639152713, 0.0494507961, 0.0119557884, 0.0294525102, -0.0560313426, 0.0534215644, 0.0232823826, 0.0835575386, -0.0238407776, -0.045725625, 0.0691915229, 0.0624836049, -0.0103600221, -0.040510416, 0.0409961343, 0.0689746141, 0.0567021482, -0.0585458241, -0.0111701079, 0.00233501475, 0.0558151528, 0.06295228, -5.401650e-02, 0.0525289699, -0.0337224714, 0.0303748902, 0.067464985, 0.0130214216, 0.0147539647, 0.0119460961, -0.0239247438, -0.00397080369, -0.00583022274, -0.0428219736, -0.0115101533, -0.0794260278, 0.0829900056, -0.0160497632, -0.0229358599, 0.0796357095, -0.00808194187, 0.0259810779, -4.083230e-02, -0.0657680928, 0.0273602176, -0.0272073764, 0.0119510191, 0.0571546331, 0.0333275944, 0.00748885423, -0.0441461094, -0.0143824304, -0.0558353253, 0.0203763694, -0.0355054475, 0.011032097, 0.036104355, 0.0321285538, 0.0126058077, -0.0128834657, -0.0235965382, 0.0524677038, -0.00196998171, 0.0479595661], [-0.0287728552, -0.0532444343, -0.0524346158, 0.0573066249, -0.0275181383, 0.0593891554, -0.0117954789, 0.0554313362, -0.0365713798, -0.0220379774, -0.00742492313, 0.052986715, -0.0466018133, -0.00443382841, -0.0740050822, -0.00484955544, -0.0764309242, 0.014776648, 0.06378299, 0.0366369262, 8.12402926E-4, 0.0508718714, -0.0655853376, 0.074992761, -0.0122327041, -0.0617273599, 0.0303243026, -0.0323555507, 0.0210707746, -0.0306405034, 0.0274663623, -0.0269884728, 0.0632907599, -0.0811269581, -0.0162605587, 0.0091290567, 0.0586103573, -0.0571304448, -0.0108072907, 0.0505887195, 0.0447318703, 0.0329085365, 0.0531642847, -0.0402959213, 0.0411667749, -0.00234800554, 0.0127403708, 0.056906309, 0.0329823382, -0.0768856928, 0.00923213456, 0.00514660589, -0.00539833447, -0.0596572459, 0.00721552781, -0.040253222, -0.0734520555, -0.0029622158, -0.0172632635, 0.0124539556, 0.0147766648, -0.0658939853, 0.0538791157, -0.0559662208, 0.0142271863, -0.0668252185, 0.0112398146, -0.0571658872, -0.0144670438, 0.0463722758, 0.0711485595, -0.0684783608, -0.0385839753, -0.0101890909, -0.0811076387, -0.0429614075, -0.0215352941, -5.687610e-02, -0.00370509201, 0.0770039558, 0.00487079564, -0.0596659631, 4.68905288E-4, 0.0246130563, -0.0650460571, 0.0667887181, -0.00913483276, 0.0392366052, -0.0666720718, -0.0575450845, 0.0117852641, -0.0768740103, -0.0298997276, 0.0452177487, -0.0340705775, 0.0155490208, -0.0281613376, -0.001943351, 0.0403490812, -1.14196875E-4, -0.0685599595, -0.0567654185, -0.0293112099, -0.0203704461, 0.0466800705, -0.0104689253, 0.0811034292, 0.0807919427, -0.00723820971, 0.0274780709, 0.0468251705, -0.052015353, -0.0765232816, -0.0349828377, -0.0704456642, -7.624020e-02, 0.025351258, -0.0391184539, 0.0560086854, -0.00222420506, 0.0768395737, 0.00336710899, -0.0753130242, 0.058881253, -0.0726916566, 0.0238159522, -0.0570237227, 0.0710678398], [-0.0557682626, -0.0609946847, 0.0120092556, 0.0224506725, -0.0468754582, -0.0335523821, 0.0551939495, -0.0749712214, -0.0732420087, -0.0222614668, 0.0621534437, 0.0210435241, 0.0553530939, -0.0216629095, -0.00175531907, 0.0304734074, -0.0587160885, -0.0407654308, -0.00128663704, 0.0381579772, 0.0445998944, 0.0040574912, -0.0117524331, 0.0797578766, -0.0754175857, -0.00880111567, -0.0517106317, -0.0319762677, -0.0756790563, -7.275440e-02, 0.0729237199, 0.0454441756, 0.0431306921, 0.0582520664, 0.00445706956, 0.0350564942, -0.0809436738, 0.0547058247, -0.0751315206, 0.00484425621, -0.0486423038, -0.0735050216, -0.0704355463, -5.431780e-02, 0.034207847, -0.0633142591, -0.0474174544, 0.00982375629, 0.0190946124, 0.0312582776, -0.0471799113, -0.0305385608, -0.0268564746, 0.0720680729, -0.0697296709, -0.0217094701, 0.079228945, -0.0646734387, 0.0415925793, 0.0438624918, 0.00636761636, -0.0049629123, 0.0624442101, 0.056659691, 0.0770635902, -7.130350e-02, -0.0343111902, -0.0713420957, -0.00680671865, -7.380900e-03, 0.0199441146, 0.0517988689, 0.00506755849, -0.0212747287, 0.0113137811, -0.0424888059, 0.0513581187, 0.016377734, -0.0269041825, -0.031191688, 0.0692098737, 0.0193010494, 0.0295950472, 0.0362246893, 0.0249578357, -0.0209250189, -7.802980e-02, -0.00792093109, 0.0684025511, -7.110300e-02, -0.0676313936, -0.0505272262, 0.0358632654, 0.029291613, -0.0324889198, 0.0602112524, 0.00909447483, -0.0224511642, 0.0556356572, 6.800650e-02, -0.00967673119, 0.0675949231, 0.0360378772, 0.0380815119, 0.0740512758, 0.0153236426, -0.0106663592, 0.0717152581, -0.0607796758, 0.0720583797, 0.0544182509, 0.0272949804, -0.00245232717, 4.93074593E-4, -0.0154872648, 0.0525929853, 0.0656767711, 0.00533091184, -0.0226475522, 0.0679510533, -0.0141723882, -0.0304143298, 0.0277129132, 0.0680532604, -0.0810574665, -0.059720654, 0.0116968881, 0.0396077447], [0.021851467, -0.0207324903, -0.0700490251, -0.00663978513, 3.05420661E-4, -0.0536356866, 0.036968492, -0.0299712177, 0.0619195997, -0.0291307177, 0.0220447779, -0.0124710947, -0.0776645839, -0.0159462057, 0.0472465418, -0.0272435509, 0.0572747961, 0.00157122849, -0.0557897724, -0.0563717224, -0.0632211044, 0.059234228, 0.00208778167, -0.0125924535, -0.0809832587, -0.060854651, 0.0494790822, -0.0374728814, -0.0538659208, 0.0044297236, 0.0162770767, -0.0221834555, 0.0530997589, -0.0124213696, 0.0765314251, -0.0122364191, -0.0104367584, 0.00810585078, 0.0050588469, 0.0624826252, -0.0669472665, -0.0675013214, -0.0387572236, 0.0451531932, 0.0756992399, 0.0639830306, 0.0419497117, -3.884290e-02, 0.0151596759, 0.0787937417, 7.149810e-02, -0.0638283864, 0.0169925131, -0.0570605248, -0.0688806325, -0.00885786674, -0.0466964953, -0.0189817026, 0.0610684268, -0.0802400186, -0.00155657658, 0.0137381777, -0.0319449641, -0.0560236163, -0.0804707109, 0.0681745335, -0.0155993346, 0.06513495, -0.0608648472, -0.0311605018, -0.0720431283, -7.198770e-02, 0.0116169117, -0.0309516117, 0.066923894, 0.047933612, 0.00896419957, 0.0202787761, 0.0559076071, 0.0215156935, -0.0354203843, -0.0703160539, 0.0610122085, -0.0616312511, -0.0678499341, 0.0621395707, -0.0457859896, -0.0321888328, 0.0175051801, 5.833790e-02, 0.0032326011, 0.0762867629, -0.0801107287, -5.89535193E-5, -0.0697462633, -0.0155339735, 0.0680277347, 0.0478608422, -0.0683133155, 0.0794803723, 0.0381251052, -0.0735238716, 7.458190e-02, -0.0541495457, 0.0429998785, -0.0181118157, -0.0324657932, 0.0222003646, -0.0104373442, 7.375170e-02, -0.0297421683, -5.921580e-03, -0.0612301454, 0.0205251556, 0.0176275969, -0.00672175875, 0.0475475863, 0.0797700062, 0.0573261604, -0.0586080849, -0.0265066791, 0.00941018201, 0.0460491665, 0.0115991412, -0.071673736, 0.0284283422, 0.0199360792, 0.033295095], [0.0739552528, -0.0017229222, -0.0219035354, -0.017991323, 0.0701633989, 0.0117381653, 0.0497348793, 0.034847077, -0.00933150481, 0.024350062, -0.0350349285, 0.0771284774, 0.00545954751, 0.0689685419, 0.0312187225, -0.00146402512, 0.0264645964, 0.0340714492, -0.034277074, 0.0258407108, 0.0352377966, -0.028501207, 0.0332954824, -0.0220161974, 0.047988981, 0.0489737391, 0.00740460679, 0.0295734517, -0.00681957603, -0.0783828646, -0.0024382507, 0.0493029654, -0.0717985705, -0.016044531, 0.0236844029, 0.0159804802, -0.0488329455, -0.0601699352, -6.001400e-02, 0.0170871429, -0.00635405863, -0.0760935098, 0.0499317944, 0.0145800421, -0.0324119404, 0.0631654337, 0.0244997088, 0.0396361575, 0.0762894601, -0.06479121, -0.0230839644, -0.062269792, 0.076832816, -0.0137966909, -0.0585920811, 0.013109467, 0.0727087185, 0.0723078548, -0.0681550726, 0.0106439907, 0.0368581675, -0.0196542479, -0.015117879, -0.0224397331, -0.062263105, 0.0807837322, 0.0461416505, -6.531350e-02, -0.0669718608, -0.0445238203, 0.00489371084, -0.0218266919, 0.0675426051, 0.0786430761, -0.0662531853, 0.00445595616, -0.0257063545, -0.0801004245, -0.00522116432, -0.00395173719, -0.00818275287, 0.0744830295, 0.0546639711, -0.0566028915, 0.0600983612, -0.0319832414, 0.0809189751, 0.0457304493, -0.0701694638, 0.0178601854, 0.0270389635, 0.0205939952, -0.0206142347, 0.0112688066, 0.0243167318, 0.0353334621, 0.0547452644, -0.075096488, -0.0372130871, 0.030320311, 0.0298458487, -0.00709825195, 6.273390e-02, 0.0463788249, 0.0378355421, -0.0372328162, -0.052240029, -0.0494734123, 0.0133227995, 0.0712972134, -0.0289887767, 0.0286122337, -0.0633572041, 0.00855785235, 0.0619818679, 0.0136453789, -0.0347827934, 0.0176584199, -0.0139347715, -0.0592527203, 0.0239618253, 0.00279071741, 0.0514316782, -0.0645962581, -0.0516517907, -0.0303332582, 0.0282830112, -0.0352276228], [-0.00631266832, 0.0135532226, -0.0528016239, 0.0594367497, -0.0305767916, 0.0274598096, -0.0309804324, -0.0719504952, 0.0641257614, -0.0722254216, -0.0232496057, 0.0444976389, -0.0659061819, -0.037001092, -0.013861143, -0.00351244258, 0.0540244803, -0.0392841138, 0.0124536846, -0.0782920271, -7.615880e-02, 0.0602195971, -0.0598694123, -0.0183012579, -0.0684475228, 0.0270564668, -0.0705152676, 0.0672065839, -0.0131208077, -4.487670e-03, -0.056202244, -0.0575182177, -0.0667679607, 0.0247873832, -0.0124605391, -0.0192201585, 0.0744874626, -0.02005383, 0.0801020637, 0.0030534761, -0.0207221787, 0.0300232321, 0.00805525947, -0.0792912542, -0.00189146679, 7.982870e-02, 0.0690309927, 0.0181720909, 0.0145712113, 0.0766427442, 0.0216476526, 0.0245730262, -0.0137970839, 0.0279055275, 0.011120325, -0.0154300099, 0.0700277612, 0.068048574, 0.0226166826, -0.0511579514, -0.0287479553, -0.0717718154, 0.0458843075, 0.00256236829, 0.0797308757, 0.038276922, -0.0517223924, 0.0690491423, 0.0766433105, 0.0738086179, -0.0536358468, -0.0608716942, -0.0323451161, 0.0308828745, -0.0444871597, -5.880070e-02, 0.0352368318, -0.00389291183, -0.0312895961, 0.0137469172, 0.0561411828, -0.0162614305, -2.607090e-03, -0.0685753077, 0.0400692523, 0.00553360675, -0.0413667746, 0.023172617, -0.0210907273, 0.0133053036, 0.0620874501, 0.0344675519, 0.0595994927, 0.00864896644, -0.00752707384, 0.0767424926, 0.0563531406, -0.0317151472, -0.0399644785, 0.0235310383, 0.00275077764, 0.0320285037, -0.0683445334, -0.0410163589, -0.0467788652, -0.00834983773, 2.964150e-02, 0.0020816566, -0.0242428631, -0.0684164837, -0.0555898845, 0.00968334078, -0.0500529334, -0.0298354719, -0.0642548651, -0.0362517573, -0.0286343396, 0.0501952358, -0.0437777452, -0.0753381327, 0.0570279285, -0.0746805667, -0.0139566762, -0.0141404346, -0.0538825616, 4.984860e-02, -0.00207024184, -0.00817190576], [-0.0397101343, -0.0277247392, -0.0544252098, -0.0512722507, -0.0232673958, 0.0129379779, 0.00626266748, -0.029085733, -0.0582692549, 0.0213964731, -6.922800e-02, 1.64240599E-4, -0.0601552241, 0.0669973418, -0.04028574, -0.0392365791, 0.0137038752, -0.0460049659, -0.0410173275, 0.0660733059, -0.052752059, -0.021761544, -5.828490e-02, -0.0442342348, 0.078033559, 0.039915137, 0.0719438717, 0.0388720334, -0.0774399861, 0.00506040454, -0.0714493095, -5.844270e-02, -0.01837001, -0.067021735, 0.0750035569, -0.064280279, -0.0125575364, -0.0449689329, 7.004500e-02, 0.0754748657, 0.0148685202, -0.0696397871, 0.0410008729, -0.071013838, -0.0145449564, -0.0769948438, -0.068727605, 0.0676368847, -0.022400286, -0.0340926088, -0.0064914003, 0.0548740104, 0.0291273445, -0.0152302235, 0.075587742, 0.0445524082, -0.0210875086, -0.0460818522, 0.00249090791, -0.0270669684, 0.0694201514, 0.0223149508, 0.0395687371, 0.0746440515, 0.0557678118, 0.0311296135, 0.0209894031, 0.0342619717, 0.00986340641, -0.0791236236, 0.0474058166, 0.0368612185, -0.0762997419, 0.0295211747, -0.0434581488, 0.0514004901, -0.0599729791, 0.0541973785, 0.0597860888, 0.06368763, -0.0536015108, -0.0207306184, 0.0144792795, 0.0708778277, -0.0161458403, -0.046760086, -0.0104970634, -0.0444666818, 0.0650047138, -0.0763324648, -0.0282018147, 0.0639560744, 4.680460e-02, 5.930210e-02, 0.0421185866, -0.0245989747, -0.00908420235, -0.0685600936, 0.0317582861, 0.0135294199, 0.0175357759, 0.0525639132, 0.0253501534, 0.0142412111, -0.00717209279, 0.0291584656, 0.0423443615, 7.659110e-02, -0.0667804256, 0.0367909595, -6.751500e-03, -0.0749675333, -0.074521184, -0.0374243297, 0.02305305, -0.0222596601, 0.0451389924, -0.050969068, 0.0467223451, -0.019911468, -0.0649739951, 0.0365403369, 0.0131617263, 0.022043705, -7.755220e-03, -0.0172985569, -0.00479429215, 0.0715790465], [-0.0491721928, 0.0488066152, 4.074920e-02, 0.00688230991, 0.0372915119, -0.0445583239, 0.051200591, 0.0701425448, -0.017856732, 0.0744534209, -0.0131262168, 0.0243684426, -0.0768934861, -0.0792576968, 0.0701517388, -0.03312378, 0.0493797734, -0.00327809155, 0.0548479483, 0.0274416059, 0.0282255858, 0.0289773792, 0.0108853132, 0.0267327428, 0.0349954963, 0.0143114626, 0.0178918689, 0.0404037833, -0.0118421912, -0.0533433072, 0.0464505032, -0.0242523551, -0.0523105077, -0.021745842, 0.0310429633, 0.00300733745, -0.0780904367, -0.0681773499, 0.0436930135, -0.044831343, 0.00684117526, 0.0388853922, -0.0236154683, -0.0768004953, 0.0515305474, 0.01178509, 0.0539475307, 0.0198753402, -0.0468662716, -0.0701419115, 0.00189814717, -0.0789075717, 0.0438839197, -0.036787983, 0.0174670666, -0.0101759657, 0.0128440335, 0.00993856787, -6.269430e-02, 0.0165932477, 0.0634790584, 0.00406351686, -0.0157604441, -0.0412358679, 0.0761905387, 5.896280e-02, -4.830160e-02, -0.0727230683, 0.0374037549, 7.146970e-02, -0.0536690205, 0.042785503, 0.0346530378, -0.0192061625, -0.0148390904, -0.0251061395, 0.023580797, 0.00112361461, 3.473050e-02, 0.0555764362, 0.0698947385, -0.0141090304, 0.0201775804, 0.00363854319, -0.0781280845, -0.0190626122, 0.0798153355, -0.0686081499, -0.0464723147, -0.0599284023, -0.0556690842, 0.0615290627, -0.0428559147, 0.0617494211, 0.0458002314, -0.0345801301, 0.0238779485, -7.476970e-02, 0.077088438, 0.0773737356, -0.017576158, -0.0576916598, 0.0766635314, 0.0238460749, 0.0286872238, -0.0230220929, -0.0255831219, -0.0201334134, 0.0690659806, -0.0274920054, 0.0103874728, 0.0547148958, -0.0782073736, -0.0638402104, 0.0329631567, 0.0449759588, -0.0805289894, 0.0322792605, 0.0477455631, 0.0401838869, -0.0178652853, 0.0638406798, -0.0347632617, 0.0505321696, 0.0458765402, 0.0606278554, 0.0586460605, 0.0387028977], [-0.0290646516, 8.90821218E-4, -0.0523358211, 0.0664326623, -2.463860e-02, 0.00296117365, 0.0382690802, 0.0505743101, 0.0585414097, -0.0703614354, -0.0617099255, -0.0761349276, -0.0718757212, 0.0359976664, 0.079460375, -0.017782554, -0.0211950466, 0.0395014361, 0.0290559679, 0.0629247352, 0.0789631381, -0.0372833125, -0.0588882565, -0.031929139, 0.00715078413, -0.00863079726, 0.0181795657, -0.0576058775, -0.0489578471, 0.0181759298, -0.0419646129, -0.0195580423, 0.0218307525, -0.0127512887, -0.0504700616, -0.0265643857, -0.0674585477, 0.0315578058, -0.0174212754, -0.0644190907, 0.0700297877, -0.0331151932, 0.0103516355, 0.0561772063, 0.00242268294, 0.0422866717, 0.0800978317, 0.0647862777, 0.0567152426, 0.00337453187, -0.0259999372, 0.029449679, 0.0652300939, -0.0795182958, -0.0545233712, 0.0222203434, -0.012950398, -5.857300e-02, 0.0046780929, 0.0175713226, 0.0629028156, 0.0418563783, -0.0140842572, 0.0287731662, -0.0277059637, 4.347250e-03, 0.0346373916, -0.0214460976, 0.0632039085, -0.0514221452, -0.0459215976, 0.0751473233, 0.0793308243, -0.025308013, -0.0018176809, 0.025216639, -0.0110265836, -0.0391826257, -0.0657099932, -0.0596980676, -0.0197482519, 0.0218680799, 0.0722154751, -0.0157122314, 0.0515247658, -0.0215126611, 0.0583857074, 0.0600026324, -0.0327531807, -0.0130918548, 0.0387592539, 0.0722928718, -0.0783309638, -0.0348858871, -0.022415489, -0.0341899209, 0.0704946741, -0.0170792937, -0.0377937853, -0.0781136974, 0.0415438712, -0.0166379958, -0.0296503305, 5.229830e-02, -0.00924372673, 0.0345156193, -0.0803878605, 0.0492014065, 0.0350014716, 0.0182518139, 0.0751433894, 0.0206354558, 0.00653997809, 0.0778456702, -0.0113563165, 0.0584024265, -0.0779786184, 0.0474359915, 0.0416596904, 0.0571212247, -0.0801376998, 0.00311878324, -0.0604293421, 0.0204785988, -0.0366556682, 1.734630e-02, -0.0645280927, -0.0434678383], [0.00882686674, -0.0644517466, -7.138550e-04, -0.0258720741, 0.0537257269, 0.0242486447, -0.0353745855, -0.0651271418, 0.0485479757, -0.0739547461, 0.0784789994, -0.0185034424, -0.0756617188, -0.0439401381, 0.0146627799, 5.785840e-02, -0.0269242898, 2.83308327E-4, -0.0250233151, -0.0450793765, -0.0774184837, 0.0400232449, -0.0668861717, -0.0217230208, 0.0203674436, -0.00970361381, 0.0316759199, -0.0171176642, -0.0251207612, -0.019367367, -0.0761495084, 0.0354531407, -0.0552322343, 0.0187647045, -0.0130280033, 0.0430754051, -0.0191678703, 0.0276810974, 0.00724969804, -0.0131520182, 0.0770938173, -0.0150961131, -0.0654681921, 0.0140771046, 0.0473962054, -0.055257123, -0.0519273765, 0.0202929899, 0.045879744, -0.0606390312, -0.00300230831, -0.0304265134, 0.0427787751, 0.0407106429, 0.0307088718, -0.024291981, -0.0142730549, 0.0485892817, -0.0649273545, 0.0506283417, 0.0373732746, -0.0396747068, -0.0703171715, -0.0184846893, -0.0444339216, -0.0555217676, -0.0483516268, 7.068930e-02, 0.0098132044, -0.00406475365, -0.0308171846, 0.0586718842, 0.0131701529, -0.0270346515, 0.0577912107, -0.0490847826, -0.0578049831, 0.0807702764, 0.0328254476, 0.0730779842, -0.0662701875, -0.0150425062, 0.0706453249, -0.079199098, -0.0450278781, -0.0246318504, 0.00653839856, 0.0801943764, -7.457980e-02, 0.0272669047, -0.0686567053, 0.0549660847, 0.0616811439, 0.0223806426, -0.020090811, 0.00265000761, 0.0736220106, 0.0262662247, -0.0456330106, -0.0217760094, -0.0190325044, -0.00418202579, 0.0196733177, 0.0602654591, 0.0602311119, 0.0578046963, -0.0358631089, 0.047769703, -0.015261963, 0.0251505971, -0.00197780877, 0.0628209934, 0.0799364075, -0.0333913453, 0.0694521144, -0.053990446, 0.0412794948, -0.0456584208, -0.0810759217, 0.0127981231, 0.0297119617, 0.0480681136, -0.017657958, 0.037494272, -0.0119268373, -0.0179363117, 0.0579442903, 0.0232029259], [-0.0530665778, 0.0546464808, -0.0299820267, -0.0290756505, -1.984780e-03, 0.0777484402, -0.00751360599, 0.048897367, 0.0618849657, -0.0756804943, 0.014919484, 0.0443458296, -0.0324036814, -0.00505957194, 0.0149241183, 0.0280450303, -0.0800544321, 0.0398659036, -6.00417901E-4, 0.051326327, -0.0622681528, -0.0581767857, -0.0679404735, -0.0531055108, -0.0679221228, 0.0183398649, -0.0152405044, -0.0433681309, -0.0204545278, -0.0361291915, -0.0556640439, 0.0795733258, 0.0608268157, -0.0284799095, -0.0075291195, 0.0505172759, 0.0706768929, 0.0519745871, 0.0473797619, -0.076560013, 0.00519089215, 0.0357767902, 0.00289433519, -0.0437281542, 0.0531740449, 0.00413962826, -0.0575356595, -0.0637991205, -0.070176512, 0.0339821056, 0.0208223108, 0.0754100159, -0.0674799606, -0.0660840794, 0.0420050509, -0.0761305615, -0.0762840882, -7.798440e-02, -0.0707926601, 0.0403701961, 0.0101207038, -0.032505244, 0.0805187821, -0.0804327949, 0.0393000841, -0.0761264339, 7.666900e-02, -0.0321476795, 3.671790e-02, 0.0335090645, 0.056272842, 0.0122935297, -0.0205290671, -0.0317215919, -0.0172143131, -0.0335715227, 0.0636439919, -0.0682192668, -0.0459234826, 0.0531735979, -0.0671089366, -0.0225484557, 0.011463712, -0.0789060518, 0.00872851815, 0.0606359765, 0.0271618553, 0.0584622025, 0.00538192224, 3.998740e-02, -0.0690320507, -0.0650890246, 0.0133588361, -0.0018952298, 0.0275821295, -0.0122625735, 0.0203056931, -0.0138640478, -0.0327925533, -0.0576799326, -0.032905452, 0.076935865, -0.0726833194, 0.0302713308, -0.0467176475, 0.0669365823, -0.0643987134, 0.030515898, -0.0761747584, -0.00967866462, -0.0487988517, -0.0188988186, -0.00161062158, 0.0278607253, 0.0697750822, 0.0369148366, 0.0293625612, 0.0152396299, 0.066835843, 0.024140602, -0.00285856705, -0.0367762856, 0.021157667, 0.0736710206, 0.0609030612, 0.0792566165, 6.841450e-02, -0.044499293], [0.0657575205, 0.0642471164, 0.0101121087, 0.0336875655, -0.057312157, -0.0196820889, -0.0739145204, 0.0654078349, 0.0761933699, -0.0552783348, 0.0356788822, -0.00169187295, -0.0102817984, 0.0629036874, -0.00212822622, 0.00635582441, -0.042921491, -0.0714173093, -0.00746490294, 0.0456718318, -0.0796338915, -0.0754076242, 0.0421375893, 0.0158982612, 0.0304923858, -0.0191700067, 6.350550e-02, -6.367820e-02, 7.67383783E-4, -0.0286514722, 0.0634467751, -0.0526182763, 0.0225647036, -0.0386623181, -0.0382076278, 0.0310630891, 0.0500786044, -0.0732706562, 0.00482816249, -0.0769392177, 0.0700921491, 0.0681103617, -1.992820e-02, 0.032502532, 0.0417774767, 0.0218685586, -0.0068135215, -0.0713743344, -0.0482368208, 0.068798244, 0.0546071455, -0.0682763383, -2.555160e-02, -0.00822564773, 0.0107877366, -0.073595427, 0.0737969652, 0.0698978677, 0.0603752509, -0.00974332354, 0.0481751487, -0.04139724, -0.0783472284, 0.0709304959, -0.0119885216, 0.044614803, -0.0367402658, 0.0394362062, 0.0457880199, 0.0725044385, 0.00521167181, -0.0645317361, -0.0687854737, -7.805100e-02, -5.997990e-02, 7.921010e-02, -0.0518204123, 0.0296120774, 0.00978554878, -0.00901034568, -0.0414071791, -0.0655828714, -0.0603540689, -0.0634073541, 0.0504550934, 0.0109698158, -0.0621079877, -0.0591099374, -0.0677943155, 0.0526882857, 0.0143001787, -0.0318812951, 2.929070e-02, -0.0577624328, -0.00669959373, 0.0301155914, -0.0611519217, -0.0338468775, -0.0286712404, 0.0515287779, -0.0139767705, 7.483520e-02, -0.0713948309, -0.0394913703, 0.0406645611, 0.0180140622, 0.0302228238, -0.0282072574, 0.0320944972, -0.0783845707, -0.0592029914, 0.00371082872, 0.00604104204, -0.0673187524, -0.0672889053, 0.0702188835, 0.0656202585, 0.0285143536, 0.0147643881, -0.0284497179, -0.045984637, 0.063332811, -0.0529059619, 0.0596177764, 0.0399126671, -0.0608664341, -0.0489906594, 0.0231561456], [0.00614744425, -0.0398467854, -0.0789223313, -0.0323579423, 0.0464036465, -0.00893117487, 0.0796222686, 0.00312454929, 0.0255026501, -0.0493854433, 0.0764663219, -0.0731166303, 0.0482615344, 0.0576451384, -0.0211234391, 0.0158664547, 0.0627674237, -0.0398253798, -0.0352454111, -0.0151596814, -7.242290e-03, 0.0278318319, 0.0198622178, -0.0278185159, 0.032326337, 0.0029891259, 0.0738965347, -0.021566689, -0.0161896609, 0.0568072423, -0.0628205761, 0.0408790559, 0.0548540726, 0.028982386, 0.0809789076, 0.0621891245, -0.0414531641, 0.0105671734, 0.0138516938, 0.0598004945, -0.0552056655, -0.0499527082, -0.0175863151, -0.0208740234, 0.058126539, 0.00513971737, 0.0295131151, 0.0661695078, -9.731780e-03, 0.056132365, 0.0181722585, 0.0171004869, -6.421970e-02, 0.0420218967, 0.0681710094, 0.0220872927, 0.0489440598, -0.0462500267, -0.0395513438, -0.0759460405, -9.363990e-03, 0.0233335849, 0.0204847865, -0.0710176229, 0.0605017953, 0.0522967912, -0.0280740298, 0.0485794544, -0.0308319647, -0.0390461236, 0.0125084277, -0.0743016675, 0.0468191504, 0.0641697869, -0.0221487544, -0.00842965394, -0.0684713051, 0.0516708307, -0.0716764331, -0.026591273, -0.040022213, 0.0401345491, 0.0512402877, -0.0790714919, -0.0516086109, -0.0618108138, -0.0718578771, 5.371300e-02, 0.0230770353, 0.0529602356, 0.0671654791, -0.02650268, 0.0494576208, -0.0718820095, 0.0134645225, 0.0585109964, -0.0800607055, -0.037978448, -0.074866265, -0.0549780838, -0.0173443612, 0.024324622, 0.0136640454, -0.0169528499, -0.0354079045, 0.0187565405, -0.0306757521, -0.0115270652, 0.00719454325, -0.00217964058, 0.0428362787, 0.0698257982, -0.0124596469, -0.0309444107, 0.0213309433, -0.0252436213, 0.0340240113, -0.0568421446, -0.00511886599, 0.0263124183, -0.0616768077, -0.0352701619, 0.0754110813, -0.00882688072, 0.00568565167, 0.0185560063, -0.0402493067, -0.0657703429], [0.0183841884, 0.0728884339, -0.0255000144, 7.321070e-02, -0.0058810194, 7.199650e-02, 0.0136426752, -0.0218895469, 0.0255397409, -0.0204634368, 0.0536601916, 0.00635155663, -0.0683242679, -0.0389266573, -0.060234338, 0.0238564238, 3.198280e-02, 0.0220003724, 0.0792520716, 0.0797957331, 0.00602833414, -0.0468480773, -0.0228439737, -0.0406780317, -0.0319322459, -0.0113670472, 0.0590268448, 0.00943177379, -0.0244644042, -0.0501287542, -0.0112151224, 0.0108140884, 0.0298224445, -0.0568685941, -0.0734648705, -0.0741991401, -0.0100104501, -0.00860191322, -0.0332070962, -0.0805948674, -1.781050e-02, 0.0499526225, 0.067606844, -0.0102992263, -0.0419687517, 0.0304939058, 0.0389864333, 0.0257833563, -0.0329968892, 0.0368714146, 0.0545390248, -0.00298199826, 0.0298114289, -0.0605573654, -0.0257987101, 0.00860586948, -0.0565458238, -0.0331397094, 0.00582510931, 0.0503865145, -0.0388980024, -0.0267871432, -6.391200e-02, 5.66689923E-5, 0.00965663604, 0.0487765372, 0.0599706247, 0.0304594673, 0.0297560878, 0.0725798458, 0.0786264911, -0.0385024212, 0.0396589972, 0.061947234, -0.0298321955, -0.0233435612, 0.064327158, 0.0402504615, -0.0144194216, -0.0305850618, -0.0190199986, -0.0333526023, -0.0324966349, 0.0637598931, -0.00999536085, -0.0727271438, -0.0778739303, 0.0145823322, 0.0356413424, 0.0358153395, -0.0388353653, 0.0424368642, 8.141690e-03, 0.0651654303, -0.059206076, 0.0328456648, 0.0742140189, -0.0517275408, 0.0621346645, 0.0448865741, -0.0479480885, 0.0313304104, 0.00930054485, 0.0451582782, 0.0528061874, -0.0271479022, -0.0524569415, -0.0802396536, -0.0190848708, -0.0135109751, 0.0593336299, 0.0329130217, 0.0791529864, 5.225730e-02, 0.00990527961, 0.0369098298, -0.0235448368, 0.0111529138, -0.0410757475, 0.00639985362, 0.0436565541, -0.054867819, 0.0810324475, -0.0718398467, -0.0578538887, 0.00985561124, -0.00133221585, -0.0396735929], [0.0727339387, 0.00527129695, -0.0448499136, -0.0744563937, 0.0485980101, 0.059434697, 0.018426558, -0.0318401344, 0.0414230712, 0.0729880109, -0.0491466224, -6.18542545E-4, -0.00419036578, 0.0359903686, -0.0748658404, 0.00433205068, -0.014851816, -0.00559082627, 0.0803573951, 0.0215295013, -0.0173101947, 0.00954715628, -0.0809624344, 0.0709397867, 0.0430707596, 0.0279989764, -0.0703736841, 0.0780706182, 0.0297213234, -0.018268371, 0.0811816975, -6.188570e-03, -0.0427411385, 0.0312069803, 0.0171450824, 0.0663527623, -0.0722803771, 3.546710e-02, -0.078188695, -0.0438172407, -0.0578550622, -0.0547569506, 0.0519082174, 0.0595559888, -0.0404059403, 0.0788225457, -0.0222823247, -0.0386832394, -0.0163302738, -0.0321852788, 0.00288548344, -0.0403167941, -0.00577816321, -0.010697891, -0.0189030617, -0.0206489768, -0.0150864897, 0.00272824033, 0.0479977056, -0.0161497835, 0.0665831566, 0.0533537865, 0.0320587456, -0.0805001184, -4.884820e-02, -0.0393646695, 0.0781703963, 0.0511898212, -0.0778479054, 0.0121846488, -0.0493299551, -0.0202916469, -0.0632976591, -0.0288126357, 0.0751131922, 0.053402517, 0.0556081198, 0.0215236228, 0.0363234803, -0.0181544032, 0.0301722921, -0.0706814528, 0.00441383244, 0.0125738271, -0.00428890809, 0.041010078, -0.0263552144, -0.00149489543, 0.014422914, -0.0570758879, 0.0156685952, -0.0281490833, -0.0471175052, 0.0518473163, 0.0769771412, -0.0322867706, 0.0712578371, -0.0146041466, -0.0275830068, -0.0516074635, -0.00420774613, -0.0110375034, 0.0565858632, -0.0151493382, -0.0630050302, -0.0203218069, 0.0202070978, -7.418860e-02, -0.0135692554, 0.0452136435, 0.00767319137, 0.0320068821, -0.0382378399, -0.0245386269, -0.0713239759, 0.0304420926, -6.908570e-02, 7.573520e-02, -0.0707374141, -0.0618459024, -0.0262664985, -0.00805357192, -0.0467015542, -0.0681215376, -0.0435968265, -8.114880e-02, -6.671510e-03, 0.00961188599], [0.0189937558, 0.0050635729, 0.0410489663, -0.0714214146, -0.0147023043, -0.036849536, -0.0669144318, -0.00196528132, 6.704130e-03, -0.0101682423, -0.0136385942, -0.00648407498, -0.0680366904, 5.44188835E-4, 0.003498198, 0.00102153898, -0.0486543179, 0.0216492936, 0.0274074394, -0.0728912428, 0.0473495498, 0.0460224785, -0.0486428365, -0.00333869923, 3.95589159E-4, -0.0456407107, -0.0290594175, -0.0356679708, 0.0596015379, -0.0108402697, -0.0158910044, 0.0481948853, 0.00504316529, -0.0674314424, -0.00813015177, -0.0551890768, 0.0187535733, 0.0309929717, 0.0225914512, 0.0111775091, 0.0802234783, 0.0492101274, -0.0578424819, -0.0692342371, 0.0173376538, 0.0333509222, 0.0349583924, 0.033958409, 0.00549741322, -0.0540230907, -0.061621774, 0.0820149257, -0.0255249962, -0.0435874611, -0.0504694507, -0.0661664382, -0.0540437773, -0.0806780382, 0.00482725957, 0.0468095578, 9.464520e-04, 5.076610e-02, -0.0559673123, -0.004955051, 0.0110793673, -0.0081677055, -0.0234574098, -0.0403454043, 0.0301021263, 0.0386751071, 0.0383312777, 0.0635841265, -0.00927772745, -0.0647713467, -0.0485761911, -0.0198324565, 0.0591896549, -0.0759803354, 0.0633935109, 0.0109063629, 0.056991592, -0.00524033047, 0.0572262369, -0.0705353543, -0.00971093122, -0.00283660833, 0.0198836979, -0.0301644634, 0.0613356158, 0.0141573092, 0.0128012793, -0.0774654597, 0.037522696, -0.015318363, -0.00833977758, 0.0597993098, 0.0446309485, -0.0711184666, -0.0122616366, 0.0151692424, -0.0203227513, -0.0785654336, -0.0594823137, 0.0426730774, -0.035366945, -0.0431547798, 0.0729002654, -2.849760e-02, 0.00381656294, 0.0162967704, -0.0258926265, 0.0272885039, 0.00909945276, -0.0173065457, 0.0380632058, -0.021841919, 0.0210774075, -0.071180515, -0.0647186338, -0.0452652648, 0.0653153583, -0.0291855279, -0.00799493491, -0.0284496974, -0.0511825904, 0.0198992863, -0.0142596448, 0.043335136], [-0.0321624279, -0.0700710639, 0.0479991063, 0.0243291128, -0.0239670724, 0.070850566, -9.243800e-04, 0.0179792158, -6.754360e-02, 0.0508603193, -0.00970794446, 0.0109371366, -0.0213307068, -0.00524758082, -0.0820323676, 0.0470511056, 0.0235235095, -0.0664793774, 0.0672205836, -0.0612300187, 0.0651850104, -0.0173757672, -3.537410e-02, 0.0190998279, 0.056558203, -0.080397293, 0.0173495989, 0.065303415, 0.0650086104, -0.0375740863, 6.894170e-02, 0.031133255, 0.00118457992, -0.0403338037, 0.030294273, 0.0740336403, -0.0787167772, -0.0150939599, 0.0309198648, -0.024941681, 0.0713358894, -0.0718077868, -0.0361070819, -0.0230305828, -0.0283377618, -0.0316267423, -0.0654518157, -0.0416445695, 0.00300792372, 0.0408813246, -0.0137346396, 0.0101629347, -0.0646014586, 0.0258641113, 0.0096695153, 0.0282613114, -0.055327516, 0.0369827524, -0.0675708949, 0.00456753606, 0.0175087396, 3.057290e-02, -0.0647430196, -0.0279066972, -0.033428032, -0.0490922779, -0.0518026464, 0.0653481856, -0.0659927055, -0.0218430199, 0.0500726849, 0.0785772725, 0.051613085, -0.0500995144, -0.0503164902, -0.0616639778, -0.0466096438, 0.0465914905, 0.0183044355, 0.0730108544, 0.0721594915, 0.0521262176, 7.005470e-02, -0.0361818969, -0.0302893631, -0.0388626792, 4.029290e-02, -0.0739887431, 0.0531719774, 0.00247540255, -0.0318026431, 0.0194012243, 0.0377642065, -6.715320e-02, 3.46370536E-4, 0.0359872431, -0.0778808072, 0.0641054511, -0.0186697226, -0.0665253922, 0.0450812168, 0.00905022583, 0.0157796908, 0.0507047065, 0.035650745, -0.0225360319, 0.0468022563, -0.052121032, 0.0206073709, -0.0311697479, -0.0133160865, 0.0173334498, -0.0214021802, 0.00401873747, 0.0456923433, 0.0374396928, 0.0515450425, 0.0471120253, 0.0368742533, -0.0756215602, -0.0124161625, 0.0381253958, 0.0223396011, -0.0435733795, 0.0774021074, 0.0470715612, 0.00586266909, -0.0333952196], [-0.069127053, -0.0740590543, -0.0367487594, 0.0613358803, 0.0777260214, -0.0502868146, -0.00129061588, -0.00783207733, -0.0700913817, -0.033188168, 5.742910e-02, -0.0585239194, 0.0685989261, -0.0015502125, -0.0221879613, 0.0819611251, -0.0376719832, -0.0184024144, -0.00614133431, -0.041482687, -0.0419957526, -0.0223011877, -0.0601582266, -0.0348448865, 0.042699907, 0.0636821315, 0.0752029716, -7.329060e-02, -0.0601438172, 6.20682957E-4, -0.0117925396, -0.0682693347, 0.0562448576, -0.0141457161, -0.082057856, 0.059358459, 0.0134938145, -0.0439996757, 0.0516180508, 0.0958471745, 0.00262898626, 0.0672083125, -0.0334831178, 0.0353467762, 0.020894235, 0.0122193443, 0.0468619056, -0.0819266587, 0.0801475569, 0.0469886102, 0.0115369176, -0.0546646677, -0.047680106, 0.0515939966, 0.0753481835, 0.0779441893, 0.0210651457, -0.00147591194, 0.0219967403, -0.0675245523, 0.0184725262, 0.0297164656, 0.00860645901, 0.0119353244, -0.0131543484, -4.691510e-02, 0.084787786, -0.0309725218, 0.0688875914, 4.468590e-02, 0.0694768876, 0.0914625599, 0.0798138305, 0.0540853068, 0.038647119, 0.00893381517, -0.0354820937, 0.0824450477, 0.033465106, 0.0587808266, -0.047705736, -0.0067195748, -0.00703532761, 0.00583060877, -0.065973796, 0.0604704767, 0.0540538356, 0.045042038, -0.0432363898, -0.0553209484, -0.0166353714, 0.0166719537, 0.0158006139, 0.0568013191, -0.00203671423, -0.00103092822, -0.0540710166, -0.0459698625, -0.00476336898, 0.00595468748, -0.0169395451, 0.0140043069, 0.0370032452, 0.00491858879, 0.0674591362, -0.0530436411, -0.0176460817, 0.0506961681, -0.0344979838, -0.0607175231, -0.00950885843, -0.0441239327, 0.0302854702, -0.00827976223, 0.0302885324, 0.085080631, -0.0466139615, -0.0552655309, -0.0524026826, -0.0120461127, -0.055709634, 0.0366836786, -0.0660860389, -0.0556158163, 0.00999189075, -0.0554319918, -0.0240730681, 0.00581601867], [-0.0208872277, 0.0116453189, -0.0390226357, 0.0235996973, -0.0713075399, -0.0689391717, -2.166350e-02, -0.0696805492, -0.044001475, 0.0212193914, -0.0641920716, 0.0291517451, -0.0535750538, 0.0725675747, -0.0491015203, -0.00163265411, -0.0447837487, 0.0679374337, 0.0242519975, 0.0548959263, -0.0667018145, -0.0568187796, 3.856350e-02, 0.0632652193, 0.00787865836, 0.0185423624, 0.0402914919, -0.0425828844, 0.0488720126, -0.0849806219, -0.0348363705, 0.0679821074, 0.005849218, -0.0876130163, -0.0661025867, 0.0309739318, 0.042124521, -0.00481324131, -0.0411660932, 0.0533336848, 1.28202097E-4, -0.0117164282, 0.0515576079, -0.0531964824, 0.0663304701, 0.0634874701, -6.785680e-02, -0.088233374, 0.0480069891, -0.00696098758, -0.0362053625, -0.0317604393, -0.0478413105, -0.0474337116, 0.0183793753, -0.0596259832, -0.0170423798, 0.0377612971, 0.0493573099, 0.0184886288, -0.0421990231, 0.00896741822, -0.046256531, 0.0533740558, 0.0576883517, -0.00379602844, 0.0418712683, 0.0750387534, -0.0768647491, 0.0515371263, 0.0440063775, 0.0925871655, -0.0139143011, 0.0346766822, -8.0470019E-4, 0.0264248718, 0.0360556208, -0.00137788826, 0.0209731497, -0.0542727895, 0.0241653509, -0.0762397721, 0.0459247082, -7.327170e-02, 0.0261441339, -0.0526106767, -0.0218693018, 0.0624875203, -0.0216212161, -0.0680627674, 0.0578096136, -0.0218075067, -0.0215546042, -0.0271496028, -0.0531804971, 0.0651543364, -0.0604260154, -0.0217093024, 0.00655880338, -0.0687901154, -0.00448398571, 0.00298654451, 0.0121288374, 6.454580e-02, -0.0218298472, 0.0237175114, 0.0377376191, 0.0362995416, -0.0664387494, 0.0209433418, 0.0404312126, -0.00890382751, -0.046675168, -0.0342873596, 0.0157626383, -0.00452961866, -0.0214153752, 4.540200e-03, 0.0206315611, 0.0660907552, 0.0547989644, -0.0719270632, -0.0149024203, -0.0209135655, -0.0846409053, 0.057824377, -0.0681635514, -0.056459561], [-0.0114176944, -5.609110e-02, 0.0112492954, -0.0502111241, 0.0592078902, -0.0264192596, 0.0442252345, 0.0291971155, 0.0400967933, -0.00890996773, -0.0312518775, 0.0148245161, 4.1583812E-4, 0.0894296243, 0.0608496405, 0.0918938369, -0.0554929599, 0.0312316231, 0.075725548, -0.0622776076, 0.0134578971, -0.00281790528, 0.0791170075, 0.0362175182, 0.00245630089, -0.00510179158, -6.538860e-02, -2.251410e-03, -0.057087522, 0.0216727387, -0.0714885443, -0.0582297742, 3.644960e-02, 0.0507826433, 0.0368637145, -0.00485466281, 0.0510190204, -0.0754003226, -0.00626225537, 0.0812853426, -0.0468748845, -0.0176173188, -0.0344586857, 0.0374637097, 0.0340845697, -0.0364080854, -0.00201888033, 0.00232646288, -0.0419729054, -0.00790495052, -0.0787626579, 0.0825386941, 0.0289129168, 0.0378655829, 0.0351723395, 0.0123105608, -0.0166184418, 0.0208440293, -0.0774348229, 0.0396959633, -0.0300827753, 0.027007807, -0.0897062272, -0.0646530687, 0.00966140162, -0.00314477179, -0.00953093637, 0.0175221488, -0.0175780151, 0.0029194972, 0.035563536, 0.0291151255, 0.0553368628, 0.0686920956, 0.0742552206, -0.00479056919, 0.0483034104, 0.0763501376, -0.0644694045, 0.0546661429, -0.030621035, 0.0220602658, 0.0593097322, 0.0436523296, 0.0763434693, -0.0851531401, -0.0331248194, -0.0167933349, -0.0300511178, -0.0665388256, 0.00897950772, 0.0406647101, 0.0515791327, -0.0456790142, 0.0477364101, -0.0228276681, -0.0858009532, -0.0156293623, 0.0597761422, -0.0620089061, -0.0794904679, -0.0843320414, 0.0351117365, -0.0556944907, 0.00885844696, 0.0034252773, 0.0513277352, -0.0222563278, -0.0751413181, -0.0385245904, 0.0694422275, 0.065389283, 0.00745732477, 0.0105272084, 0.0430301912, -0.00196125288, 0.0601289421, -0.0629883632, 0.0534895398, -0.0785255581, -0.0585751459, 0.0555628315, 0.0122641027, -0.0267289877, -0.0601094738, 0.0627995282, 0.0505944118, -0.0161608625], [0.0126999971, 0.0496332198, -0.0334751457, 0.0582064316, -0.063851513, 0.0382120125, 0.0432892889, -0.00804225355, 0.0772175044, 0.015710244, 0.0316434838, 0.0499493219, -0.0668017492, 0.0696135312, -0.0755903944, 0.0186502803, 0.0113619305, 0.0337835103, -0.0213123206, 0.0569008887, 0.0335558951, 0.0755770728, 0.0246846713, -0.0680648312, 0.0350035392, -0.063396886, -0.0238100514, 0.0245814864, -9.25651766E-5, -8.727580e-02, 0.00246627047, -0.0866072923, 0.0819512755, -0.0115237422, 0.0602487586, 0.0649417564, 0.0711795464, 0.0180659685, 0.0658391267, 0.0551157221, 0.0308408178, 0.0162621718, -0.0575228631, -6.08918141E-4, -0.0859192461, -0.035977874, 0.0334089063, -0.0531318448, 3.707490e-02, 0.0571798794, -0.0280075129, -0.0479749516, -0.074543327, 0.0304031223, -0.0599481389, 0.0568109378, 0.0583844408, -0.0531451032, 0.0211373828, -0.0577465035, -0.0356162861, -0.0170949753, 0.0159280114, 0.0311607961, -0.093101792, -0.0462082177, 0.0312249884, 0.0348084457, -0.054429695, -0.06502942, 3.884920e-02, 0.0679543465, 0.0286942814, 0.0951131135, -0.0453528613, -0.0831447318, -0.00420694193, -0.0334523879, -0.0482363217, -0.0430892557, -0.0327912979, 0.0453340411, 0.0555643775, -0.0801224336, -0.0103829121, -0.058346048, 0.00706249336, 0.0427871868, 0.0375702716, 0.0478843786, 0.0376519822, 0.0222358033, -0.0372485034, 0.0123849288, 7.799440e-03, -0.0349476337, -0.0983124598, 0.0632595569, -0.0280595049, -0.0587641448, -0.0521673448, 0.0102278888, 0.0288518127, -6.515630e-02, 0.010977529, 0.03602295, 0.0274256468, -0.0315018743, -0.0433068126, 0.0189645793, 0.054429926, 0.0720620528, 0.00204067701, -0.00798648316, 0.00938521325, -0.0229448061, 2.600650e-02, -0.0474556573, -0.0556289814, -0.0678932592, 0.051950302, -0.0442230627, 0.0252518188, -0.0708018243, 0.0465767719, -0.0718560666, 0.0666726157, -0.0869854838], [-0.0514883697, -0.0275776871, -0.00145023619, -0.0194169357, 0.0381204523, 0.019509254, 0.00915171858, 0.0445090532, 0.00596163096, -0.00726195518, -0.0642973334, 0.0461762883, -0.0359190516, -0.0367355049, 5.780700e-02, 0.0389108174, 0.0579288751, 0.010242329, 0.0550401919, 3.55712546E-4, -0.0427398719, 0.0565101393, 4.228510e-02, 0.0366865471, 0.0085680848, 7.74646353E-4, 0.0337255746, 0.0148200858, 0.0535753705, -0.0520419963, 0.0326192752, 0.00863282568, 0.104554333, -0.0695986375, -0.0107026566, 0.0751713216, 0.00149462698, -0.0323364064, -0.0668453202, 0.0378757156, 0.0557571799, 0.0154460957, 0.0272448268, -0.080017522, 0.0230844282, -0.00195926963, 0.00499987556, 0.00862835347, -0.0148228453, 0.0913819745, 0.0503608398, -0.0335254222, 0.0117646633, 0.0578511469, 0.0610546954, -0.00680288067, 0.0146847749, -0.00376602844, -0.0251127835, 0.00918240566, 0.0674395859, 0.0304393955, 0.0387700498, -0.0335839428, -0.0478738882, -0.0294074733, 0.0708109066, 0.0306118615, 0.0458771028, -0.043822702, -0.0286094323, -0.03648711, 0.00484904647, -0.0302184299, 0.0771637112, 0.00703885872, 0.0019739964, 0.041533675, -0.0818859786, 0.0928927958, 0.044596564, 0.0185082518, -0.0348517261, -0.0130423475, -0.0294616967, -0.062557973, -0.00892861839, -0.00319043524, 0.0370570421, 0.0433425233, -0.0721622556, 0.0211269502, -0.0466369614, -0.0446370356, -0.0393257104, 0.0656706542, -0.0418149345, 0.024106048, -0.0692777783, 0.0758372173, 0.00659092143, -0.0812826157, -0.00644180364, -0.010188208, -0.052322261, 0.052258268, -0.0394027531, 0.0466378555, -0.0556515865, -0.0712584928, 0.00462574372, 0.0799364149, 0.0607303306, -0.0172461253, -0.0850928351, 0.0439567566, -0.0700946674, -0.0721014887, 0.0558608361, -0.0220165253, 0.025889067, -0.0166703053, 0.00324503239, -0.0578872897, -0.00833144691, -0.0255717635, -0.0250759926, 0.0199329499], [0.0449141301, 0.0407053456, 0.00682934048, 0.0696977078, 0.0490083583, -0.00426223129, -0.0520344675, 0.0610525236, -0.0292125903, 0.0737087652, -0.00411406904, -0.0571142808, 0.0154695576, -0.0371447057, 0.0104463389, -0.00275499327, 0.0307644438, 0.0822735354, -0.00411362853, -0.0875809118, -0.0148721458, -0.0575778745, -0.0412920527, -0.0779594928, 0.0125719979, -0.00136945886, 0.0559394844, 0.0875979959, -6.678670e-02, 0.0218182709, -0.00475184852, -0.0773973986, -0.0351777337, -0.0300900023, -0.0384106301, -0.0233794525, 0.0653240532, -0.0569223687, 0.0440939739, 0.0415868536, -0.0456536748, -0.00928305183, 0.0474081747, -0.0184651557, -0.0590809137, 0.0613105223, 0.0616319068, -0.00982938427, -0.0111880125, -0.0503410809, -0.0595732369, -0.0400390029, -0.00962862558, 0.0362999216, 0.0526655167, 0.0287014283, -0.0578659736, -0.021145219, -0.0117845796, 0.0394732654, -0.0564047024, 0.0536994487, -0.0622404143, 0.054874815, 0.00207189703, 0.0659819692, 0.0320202708, 0.0211902652, -0.0533702523, 0.0192060918, -0.0255639199, 0.0250227451, 0.0504069962, 0.00826667062, 0.0394893549, -0.0694647655, -0.0310681779, -0.0353670307, 0.00780364173, -0.0186334383, -0.0225950126, -0.0613535084, 0.0663791746, 5.890490e-02, 0.0220410805, -0.0601628721, 0.00370081747, -0.0644081607, -0.032232713, 0.0427858084, -0.00845840759, 0.078084357, -0.0032758154, -0.03291291, 0.00495484285, -0.0395705886, -0.052138295, 0.0590789914, -0.00209579105, 0.0566818342, -0.0966452285, 0.0685482323, 0.0680423155, 0.0336656533, -4.73399268E-5, -0.0732336491, 0.0431622379, 0.0639443249, 0.00382193807, 0.00857884623, -0.00260473951, 0.0696086809, 0.0781165286, 0.0163346399, -0.0895779952, -5.596260e-02, -0.0388199575, 0.0311077368, 0.0742261931, -0.017011838, -0.0292946585, -0.0293457676, -0.0919659585, 0.0288118497, -0.0131738633, -0.0716840178, -0.0562859178, -0.0709226355], [-0.0406550765, 0.0741221681, -0.0593058616, -0.0482795797, 0.0551103577, 0.0280774981, 0.0381460749, 0.0204673633, 0.0702898279, -0.0584573597, 0.0398148075, -0.00813034735, -0.0343479924, 0.0183643512, -0.0835980474, -0.0408263206, -0.0643609613, 0.0173948612, -0.0506129973, -0.0149336765, -0.057787735, 0.0559631847, -0.0260456782, -0.034485165, -0.0907451286, 0.00594594283, 0.036241021, -0.0600617155, -0.0738701895, -0.0819148644, -0.0227551889, 0.0319448598, -0.0508128963, -0.00772686722, 0.0586296581, 0.0209884718, 0.0559819043, 0.0334086604, -0.0178356227, 0.0577754937, 0.0699158832, 0.0646214485, 0.0677109212, 0.0680850744, 0.0473866053, -0.0335790217, -0.0238133967, -0.00153151085, -0.00110943266, -0.0682539567, -0.0393675677, -0.0504322872, 0.0514002815, -0.0644032285, -0.0369821973, -0.0250255186, 0.0213562045, 0.0574977025, 0.034571819, -0.00293510477, -0.0123314625, -0.0177326296, 0.0592978559, 0.00554082263, 0.073741965, -0.0540329106, 0.0410191603, -0.0687294528, -0.0545491949, 0.0377354585, 0.0143542644, -0.0376297049, -0.0521782637, -0.014030138, 0.0326454453, -0.00546385767, 0.0357454643, -0.0205179323, 0.0338293314, 0.0339655802, 0.0254566241, 0.0125100398, 0.0576298237, -0.0531160235, 0.0788009762, -0.0703671649, 7.487550e-02, 0.0432604179, 0.00463801902, 0.0317740552, -0.0547492318, -0.047416009, -0.0213267654, -0.0350355245, -0.0010444232, 0.0126039898, 0.0400340557, 0.0234087948, 0.0213948358, 0.0701794624, 0.0465981178, 0.0155655257, 0.0353077501, 0.0448524654, -0.0236047339, -0.0482835695, 0.0731393992, -0.0573091134, -0.0224026088, -0.0401982255, 0.0824067518, 0.0647813379, -0.0147730215, -0.062268503, 0.0498139858, 0.00548526458, 0.0500582345, 0.0279729068, -0.0388197675, 0.0577651821, -0.093493998, 0.00552299805, -0.0468997173, 0.0696708411, 0.0535739623, 0.0773267895, -0.0273392797, -0.07429827], [-0.0567808114, 0.0684105083, 0.0688557625, 0.0347636119, 0.00327084376, 0.0585419387, -0.0163024385, -0.0655942485, 0.0221739169, -0.0440260433, 0.00284342957, 0.0421235971, 0.0776413232, 0.052197136, 0.0491436161, 0.0411603414, 0.030468937, -0.021668043, -0.00768462522, 0.0157982837, -0.0402414277, 0.0164567102, -0.0117132328, 0.0665904358, 0.0083107464, 0.0506880134, 0.0790542587, -0.0350530744, -0.00185993942, -0.0690208599, -0.00634680782, 0.0453401953, -0.0359913856, -0.00192740199, -0.0417168811, 0.0241531562, 0.0406507626, 7.198740e-03, -0.0612915307, 0.0662454441, 0.0064614634, 0.0616598539, -2.8630087E-4, 0.0544523038, -0.0327620022, -0.0359384045, 9.379030e-03, 0.0558461808, -0.0200031865, -0.0739701316, 0.0227283109, 0.0526065528, 0.00423263106, 0.0213790443, 0.00816591363, -0.0555344336, 0.0739636123, -0.00932947081, -0.0468548387, 0.0433746651, -0.0655887648, -0.0749220103, -0.0548894852, 0.0424863957, -0.0102197509, -0.0446776152, 0.0048972955, -0.00788602326, 0.00188867073, 7.816480e-02, 0.0363086835, 0.081831865, -0.0297114775, -0.00776589382, -0.028206341, -0.00617847918, 0.0371529609, 0.0119624995, 0.00184755935, 0.0762641281, 0.0334442332, -0.0490331464, 0.0616475567, -0.0401568152, 0.0204093512, -0.0170715638, 0.0435736477, 0.0198500194, 0.105452672, 0.0522567257, -0.0558373965, 0.0205818284, 0.0243097711, 0.0227616671, -0.0143194683, -0.00710396282, 0.0502545759, -0.0611782111, 0.0162993018, 0.0298432261, -0.0463520326, 0.039947249, 0.0591589101, -0.0031569812, -0.0689601377, -0.0382952243, -6.81183708E-4, -0.0414065905, -0.0232824478, -0.0683061108, -0.0332951844, -0.030499272, -0.0567665473, 0.0234522093, -0.0912164599, 0.00202773535, -0.0273169521, -0.0255245287, -0.0295404829, 0.032265678, 0.0426082574, 0.0462682843, -0.00721941469, -0.00243614288, -0.0694636479, -0.0403555557, 0.0690920129, -0.0645347908], [0.0291464888, 0.0186582729, 0.00310524879, -0.0703388676, -0.045603741, 0.0268728454, -0.0549542755, 0.0799721255, 0.0346891247, 0.0161066335, -0.070798859, 0.0130201457, 3.593500e-02, -0.0300900433, -0.0559017435, 0.0580448247, 0.0683023483, 0.0595994703, -0.0244095325, 0.0135441609, 0.0784831494, -0.0703031346, 0.0557745323, 0.0476175174, -0.0497483909, -0.0116128847, 0.077593863, 0.0829785838, 0.0386482291, 0.0294537488, -0.0777378752, -0.0250601303, -0.0519383848, -0.0684681832, 0.0155813666, 0.0421423055, -3.821370e-02, 0.0188356899, -0.0673502609, -0.0199223533, 0.0795182585, 0.022729056, 0.0434843749, 0.0340771973, 0.0608661734, -0.0486469753, 0.051527787, 0.0141112227, 0.058031816, 0.035395585, 0.00649070134, 0.00740740029, 0.0387384705, 0.0340747498, 7.559320e-02, 0.0582509711, -0.0460720509, 0.0405943543, 0.0209453553, -0.00829811953, -0.0362954326, 0.0157503616, -0.0467811786, -0.0464174449, -3.735240e-03, -0.076657623, 3.744780e-02, -0.0369699448, -0.0367985182, 0.0778542235, -0.0488580242, -0.032303609, 0.0692549348, 0.019349834, 0.0246576536, 0.0068468852, 0.0915784761, -0.0321023501, -0.0590178333, -0.061300084, -0.00441463804, 0.0364965089, -0.0576473363, 0.025318576, 0.0608119816, -0.0566888936, 7.060050e-02, -0.0342048854, 0.0807106346, -0.0320875272, -0.0527852029, 0.0884404256, -0.0486695953, -0.0556008518, 0.00986206625, -0.0683486685, -0.071007207, -0.0207722113, -0.0664620921, -0.0581117384, -0.0633933693, 0.00113562983, 0.0261472538, 0.0215885974, 0.0450602546, -0.0251768343, 0.0789260789, -0.00335059129, 0.0246140547, -0.0145845553, -0.00461129146, 0.0345484465, -0.0632629246, -0.0464551188, -0.00697591202, 0.029597098, -0.034586776, 0.0205693562, 0.0734451562, -0.0230091065, -0.0862729623, -0.0839096978, -0.0714520663, -0.00747172954, 0.0242172126, -0.0570862405, 0.0673964545, -0.0116286781], [0.0852941126, 0.0486560911, 0.0308554769, -0.0863776058, 0.054647848, 0.00531310681, -0.0530845672, -0.042903319, -0.0593063235, -0.0116962548, 0.0578045137, 0.0448558517, -0.046168372, -3.680120e-02, -1.141090e-02, 0.00367893581, -0.0438684635, -0.026302062, -0.0666770563, -0.05431398, -0.0611016378, 0.0225117803, 0.0265262797, -0.0352467112, -0.00814946554, -0.0622722022, -0.0120211039, -0.0462834761, -0.0449657738, -0.0942069217, -0.0426751077, 0.0232739318, 0.0412341841, 0.0514952205, -0.0566094816, 0.0707428604, 0.0697345883, 9.280770e-02, 0.00440446567, -0.0670400485, 0.0600381792, -0.0181835573, -0.0430704355, 0.00851163268, 0.0551231764, 0.0465473533, 0.0238581132, 0.0444991849, -0.0107644424, 0.0527375825, 0.0578374043, 0.0294527635, 0.049787797, -0.0242395867, -0.0652215778, -0.0259214044, 0.0763406381, 0.013019789, 0.00903544668, -0.0294180978, 0.00894486252, 0.00933548249, 0.0401201099, 0.0516801365, 0.0817639679, -0.0457748137, 0.0310128611, 0.0151024228, -3.81361635E-4, -0.0331856832, 0.0346442871, -0.0303506423, 0.0202028435, -0.0390630849, -0.0482261591, -0.0101815155, -0.032954827, -0.0148649709, -7.068020e-02, 0.0169959608, 0.0771651566, 0.0669550597, -0.0736534297, -0.0207528472, 0.0937665477, 0.03430859, -0.0455569439, 0.0700645521, 0.0396708436, -0.00385447056, -0.0790110379, 0.0729330778, 0.0392156579, 5.886720e-02, 0.0501311645, 0.047018934, 0.0640653521, 0.0170416236, -0.0626002401, -0.0667866841, 0.0141368592, -0.0506616309, 0.070066534, -4.06587555E-4, -0.0378857255, 0.034316048, 0.0312315784, -0.0700802207, 0.0592455268, -0.0613537356, -0.033630155, 0.0683296919, 0.0796137973, -0.0595834441, -0.0309016071, 0.0592927039, -0.0180770401, 7.879390e-03, 0.101765133, -0.0037676862, 0.0314812884, -0.0295463204, -0.0505330451, -0.0180561952, 0.00216722395, 0.0548852794, -7.838550e-03, 0.026729824], [0.0108108697, -0.0442140847, -0.0328437276, -0.0718514472, -0.0043254178, -0.0241196398, 0.0702785254, -0.0533188842, 0.0689796656, 0.0506974719, 0.0232473779, 0.0751893371, 0.0698841512, 0.0298868474, -0.0383042358, 0.0378773697, 7.350860e-02, -0.0528559871, 0.01864901, -0.0337879844, 0.0056350613, -0.0621548332, -0.0667001903, -0.0544611067, -0.0483486056, -0.0396695919, 0.0361466408, -0.0497636274, 0.0457291864, -0.0471372157, 0.0150982402, 0.0470223352, -2.37925909E-4, -0.0263078697, -0.0456779152, -0.0457416289, -0.0758640543, 0.030692013, 0.0623578206, -0.0749631524, -0.0210845359, -0.02089555, -0.0116654197, -0.0755409598, 0.0250556804, 0.0464174151, 0.0957881659, 0.0305086337, 0.0656026602, -0.00773733528, 0.0542510301, 0.0679744929, -0.0298054796, -0.0370387658, -0.0168874227, -6.132900e-02, 0.0744918734, -0.0730741471, 0.00617566658, 0.0287441295, -0.063732706, -0.0312722437, -0.0419327281, 0.060460642, -0.0624802187, 0.0569352582, -0.0753163993, 0.0704062805, -0.021160353, -0.0113880839, -0.0513997898, -0.0326144546, 0.0105050066, 0.0301963426, 0.0687674135, 0.0427604355, 0.0998592674, 0.0182385221, 0.0191877466, 0.00865366776, -0.00749842077, 0.0586443767, -0.0179803614, -0.0894251242, 0.0615436025, -0.0704591125, -0.0136266947, 0.0712101907, 0.0225835387, -0.0721950904, -0.0368119404, 0.0426664911, 0.0393498801, -0.00577900046, -0.0232031737, 0.0449844338, 0.0251017176, 0.0318467766, 0.0472481102, -0.00546927936, 0.038294144, -0.0789899304, 0.0747712255, 0.0350270979, -0.00403640606, -0.0732830241, -0.0131120859, -0.0275809877, 0.0644292161, 0.0339225978, 0.0692844763, 0.0680148601, -0.0441176109, 8.989100e-03, 0.0311192684, 0.0143645555, -0.00868997071, -0.0807075724, 0.071650967, -0.0581054278, -0.0103868917, -0.0195489135, -0.0443481952, 0.0676380768, -0.0186224729, -0.0567647181, -0.0728935301, -0.0510145836], [0.0789769366, -0.0105063515, -0.0260781944, -0.0394577943, -0.0505029298, -0.0740235895, -0.0372715779, 0.031942647, -0.0547032841, 0.0700625554, -0.0186795536, -0.00286881439, -0.0613489747, -0.0185038932, 0.0654126257, -0.011483212, 0.0203319918, -8.6981873E-4, -0.00333237206, 0.0140899532, 0.00118825084, -0.0753323808, -0.075707145, 0.0737247095, 0.0224200152, 0.049823314, -0.0336547904, -0.02491997, 0.0594487078, -0.0422839336, -0.00310471491, 0.0096017411, 0.0592694953, 0.0100022443, -0.0571372621, 0.0118177114, 0.0741758943, -0.0293144621, -0.00573780807, 0.0392055735, 0.0626828968, -0.0178818535, 0.0158329718, 0.0547561496, -0.0175168179, 0.0399310626, -0.0216559507, 0.0554084629, 0.0288827848, -0.0200730413, -0.0692911893, 0.0100546032, 0.0818851962, -0.0274582077, -0.0341243297, -0.0107093584, -0.0712231845, -0.0326645076, -0.0752172396, -0.0705808103, -3.303980e-03, -0.0520089492, -0.0169330053, 0.0460100472, 0.0258696489, -0.0115966611, -0.0635263621, -0.0629875436, 0.00161004462, -0.0598109886, 0.0732830167, -0.0632446185, -0.0743360221, 0.0236461554, -0.0339037888, -0.0260626283, -0.0195643436, 0.0100639584, -0.0546084568, 0.020553669, 0.0627318472, -0.0643275753, -0.0676042065, 0.0258647706, -0.0405973345, -0.0364009105, 0.0283282939, -3.21933301E-4, 0.0622488111, 0.0291405395, -0.0132185407, -0.0376221761, -0.0400123149, -0.0384335592, -0.00599753531, 0.0052187494, -0.0152268428, 0.0379299633, -0.0615255311, -0.0442271829, -0.0505411364, -0.0369817056, -0.0214220621, -0.0343271755, -0.0260078087, -0.0735941604, 0.067307815, 0.00838007499, 0.0715956464, -0.017757833, -0.0241425987, -0.0547693484, 0.0525738671, -0.0479878746, -0.036512658, -0.00962445885, -9.746633E-4, 0.0417061374, 0.00399171561, 0.00618740451, 0.0516562164, 0.0193334389, -0.0442423709, -0.0161489211, -0.0408385023, 0.0580131486, -0.0378442295, -0.0247922819], [0.0271037407, -0.0524680838, -0.0551504791, -0.0808242633, -0.0710259527, 0.0612749755, 0.0414082929, 0.00615626853, 0.059871424, -0.0220330376, -0.0267428048, -0.0461832099, 0.07499291, -0.0205207076, 0.0312312786, -0.0344307311, -0.0628057793, -0.00558495382, 0.0357019044, -0.00826126616, -0.0565086156, -0.024254648, -0.0768125206, 0.0354247205, -0.0799872428, -0.0332626887, -0.038374193, -0.0555064715, -0.00544632925, -0.0812015533, 0.067299284, 0.0359905101, 0.0771641135, 0.0101934988, -0.0184712838, 0.0507642888, -0.0116193481, 0.0126703596, 0.0147931604, 0.0498789065, 0.067459613, -0.0436599553, 0.046089232, 0.0746685639, -0.0251877047, -0.0106796855, -0.0530571155, -0.03254072, 0.0145757962, -0.0589574873, 0.0491653793, 0.0306219086, 0.00288933422, -0.0547477454, 0.0608456098, 0.0444682278, 0.0423237607, 0.0260483138, -0.0628477186, -0.0711885467, 0.0660061091, -0.0545145273, -0.0531765334, -0.0420116968, -0.0238098018, 0.0649028942, 0.0333330557, -0.0667764395, -0.00207936857, -0.0209909678, 0.0451095402, 0.0271639246, 0.0101352548, -0.0101859402, -0.0370815359, 6.825370e-02, 0.0628062636, -0.0278263595, 0.00483109569, 0.0199707411, -0.0518998578, -0.077425167, -0.0600029491, 0.0258267801, -0.053898003, 0.0017689931, 0.0270526279, 0.00296128658, -0.017870456, -0.0571636036, -0.0231165551, 0.0496308059, -0.0364209712, 0.0753206089, -0.0516505875, -0.0776140839, 0.0246756133, 4.094290e-02, 0.021290008, -0.0333372056, -0.0105789891, 9.76182229E-4, 0.031248128, 3.8311613E-4, -0.00905775744, -6.768700e-02, -0.0177130122, -0.0634468794, -0.0169693585, -0.0102184676, -0.0445671566, -0.0379485227, 0.0273473077, -6.400550e-02, -0.0673991069, 0.0540111102, -0.0228133257, 0.00458753621, 0.015902482, 0.0452024788, -0.0140579464, 0.0654762387, 0.0629109293, -0.0525238737, 0.00945009104, -0.070321463, 0.056000106, -0.0605568513], [0.0101262899, -0.0223912075, 0.0337074175, -0.0304773543, 0.00358109665, 0.0674796104, 0.0456607118, 0.0157682449, 0.00763882278, 0.00897119846, -0.0614924803, -0.0232999492, -0.0695946291, -0.0778548941, 0.0685969293, -0.0289849602, 0.0707257465, -6.097440e-03, 0.0626317188, -0.0407153182, 0.0760120302, -0.075001359, -0.0333319865, -0.03391755, -0.0579021908, 0.0178164337, -0.00491986703, -0.0368991382, -0.0540874191, 0.0210891273, 0.0724528729, 0.0112345293, 0.0527760349, 0.0383548811, -0.0691797361, 0.0741126388, 0.00808013975, -0.0495129861, -0.0279228762, -0.0356571078, 0.0286023598, 0.041529566, 0.00468079047, -0.0251523796, 0.0225481652, -0.0241254829, -0.0516498834, 0.0512857251, -0.0603410117, 0.0750644058, 0.0328784212, -0.0846397504, 0.0405328348, 0.0192669053, 0.0547795258, -0.0323804803, -0.0573371202, 0.0681703761, 0.0507697575, 0.0675953924, 0.0362933204, -0.0590499379, -0.0457716659, -0.0548892878, 0.0391147174, 0.0221531857, -0.00466047693, 0.00655389717, -0.0322044417, -0.0355989598, 0.0446687639, 0.020929534, -0.0393209569, 0.0164342336, 0.0675412193, -7.394140e-02, 0.0235311259, 0.0666238666, 0.0588424169, 0.0220932458, -0.00145702669, 0.0319336392, 0.0162320621, 0.055379387, -0.0255239978, -0.0362182111, -0.0777518525, -0.0733235925, -0.0188367888, -0.0123852184, 0.079318583, -0.016470613, -0.0570673309, 0.0338094793, 0.0783584043, -0.0544814579, 0.022214463, -0.0175675638, -0.0361600406, 0.0101868482, -0.037543375, -0.0315386876, 0.0200972594, -0.0222289171, 0.0592720508, 0.00756387087, 0.0782481581, -0.0662014484, 0.0408241898, 0.0106388507, -0.0306117255, -0.0559210628, -0.0178462099, 0.0176745579, 0.0210988969, 0.026530873, -0.0525708497, 0.0407117829, 0.0439596027, -0.0155545967, 0.015854273, 0.0436233096, -0.042528972, 0.0487029031, 0.010985028, 0.0303495787, 0.00854277238, -0.033591643], [-0.0181214195, 0.00712377531, -0.0362026654, 0.00234859367, -0.0354544967, 9.776980e-04, -0.00286412076, -0.03704831, -0.0300917849, 9.24477935E-4, 0.0342828557, 0.0641572103, -0.0249096937, -0.0742204711, 0.060734991, -0.071772851, -0.0351790749, 0.0606295466, -0.0542602465, 0.0778398886, -0.0328160301, 0.0305867828, -0.0759577453, 0.0243914723, -0.0652788281, -0.0396234654, -0.0386574864, -0.066514872, -0.0672309324, 0.0591747202, 0.0819669067, 0.0727989748, 0.041431848, -7.867490e-02, 0.0677919611, -0.0508209765, 0.055873923, 0.0163133889, 0.0779663994, 0.0299236607, -0.0637716874, -0.050308343, 0.0521656722, -0.067977935, 0.0786497444, 0.0426017381, -0.0629457235, 0.054419484, 0.0622875206, -0.0655408576, 0.0242624804, 0.0211140551, 0.0258179475, -0.0124398917, -0.03893774, 0.0139276516, -0.0448209234, -0.0185786914, 0.0267226081, 0.0612520799, -0.0283356812, -7.833970e-03, -0.0280737653, 0.0194461513, -0.015135875, -0.0779321789, -0.0598929375, -0.0617850386, -0.0441482551, -0.00911478419, 0.00189001427, 0.0170847457, 0.0327392928, 0.0723815188, 0.0217272416, -0.0390732624, 0.0670394376, -0.0118428124, -0.0060032527, -0.00405511772, -0.0747489929, 0.0702545717, -0.0457413793, -0.00184354815, -0.0379875489, -0.0447694063, 0.0719013512, 0.00324128452, 0.0752256662, -0.0192616973, 0.00162995653, 0.0547819883, -0.0317376032, 0.0132414019, 0.0289847367, 0.0344347619, -0.0757873282, -0.0260914657, 0.0598595925, 0.0265530739, -0.0467620343, 0.0550380684, 0.0587825254, 0.0392570123, -0.0448535755, 0.0713832676, 0.0472255759, 0.0594730228, -0.0408526585, 0.00190272799, 0.0191490892, 0.0684733614, -0.00521141058, 0.00245220796, -0.00320615526, -0.0655828416, -0.0509526394, -0.0219253469, 0.0266648587, 0.0385717452, -0.0457483716, 0.0457443558, -0.0763213113, 0.0306544099, 2.19986512E-4, 0.0142206484, -0.0470778495, 4.833710e-02], [0.0260837842, -0.0516390279, -8.36279941E-4, -0.00732559618, 0.0248829797, -0.0563863404, 0.0209357869, 0.0416194387, -0.0785390139, -0.0110972533, -0.0446894728, -4.74130677E-4, 0.0131259132, -0.0026685854, -0.0311913788, 0.0242682137, -0.0133886561, 0.0426683202, -0.0275001358, 0.039290078, -0.0441427454, -0.00786975771, 0.0173346978, 0.0501818806, 0.0245796684, 0.025410302, 0.0312621444, -0.0404794365, -0.0569004491, 0.075449273, -0.0198516846, 0.00373064098, -0.0187992249, -0.0649598538, 0.0107623152, -0.0143703436, -0.0453294143, 0.0470003448, 0.0489275865, 1.820890e-02, 0.0534283295, -0.0380751118, -0.0431230105, -2.208690e-02, 0.0360693745, 0.00153658842, -0.0779756605, -0.0206347536, -0.0399826318, 0.065103732, -3.492980e-02, 0.0048695039, 0.00840980932, 0.0697593838, -0.0363160633, -6.973110e-02, 0.0102232015, 0.0640319139, 0.0613862872, -0.0164691731, 0.0254766457, -0.0219820626, 0.00222056406, -0.0198656656, -0.00160922646, 0.0211062059, 0.0376845188, -0.0616777353, -0.00572845805, 0.0226479694, -0.0290180277, 0.0454982817, -0.0743787512, 0.0119517334, -0.0547277518, 2.137950e-02, -0.076247394, -0.0358399078, -0.0407293178, 0.0538209528, -0.0670136735, 0.0074492991, 0.0798611492, 0.0666183755, -0.0260574296, -0.0350987092, 0.0517796353, 0.0170119852, -0.0337674282, -0.00982694141, -0.00383113394, -0.0222655758, 0.0376995392, -0.0664966255, 0.0568314865, -0.0711124688, 2.595800e-02, 0.0154116331, 0.0280611124, 0.0434476733, 0.0226196852, -0.0352816917, 0.0345227271, 0.0357572623, -0.0268536601, 0.0149069382, 0.0601459965, 0.0159424488, -0.0735080466, -0.00270070671, 0.026969986, 0.0795171186, -0.020254463, -0.0331075527, 0.0326169394, 0.0206585862, 0.0182028413, -0.0771647766, -0.0154519891, -0.0795353055, 0.0177231282, 0.00957062654, -0.0683712959, -0.0753713474, -0.0570700094, -0.0364555791, -0.0602740422, -0.0810497701], [-0.053778287, -0.0808165073, 0.0314110406, -0.0102023073, 0.053290613, 0.080989249, -0.0350996703, 0.0468647443, -0.0175149757, -0.0207564142, 0.0474725328, 0.0213610213, -0.0636130124, -0.0244470481, -0.0295186024, -0.0587831698, 0.0715036467, -0.0500343442, -0.0363609232, 0.068198964, 0.00932467636, -0.0684972182, -0.0411406793, 0.0231999028, -4.838860e-02, 0.0668626577, 0.00815229117, 0.0485940315, -0.0730063468, 0.0665613934, -0.0804652571, 0.0711993352, 0.0364273712, -0.0503524542, 0.0132493991, 0.0304793678, -0.0563820601, -0.052383434, -0.0440627672, 0.0189807303, -0.00182166463, 0.043578174, 0.00175229367, 0.016689267, 0.0712286681, -0.00943323225, 0.0143872872, -0.00994891114, 0.0461662039, 0.012756872, -0.0303332042, 0.0126715731, -0.0411059558, 0.0249820966, -0.0660068765, -0.0287452471, -0.0624430179, 0.0508456044, 0.0338067077, 0.0268868916, 0.0768254101, -0.012080431, 0.0488585345, -0.0195216648, -0.0756289362, -0.0492781438, 0.00818219967, 6.744260e-02, -0.0599194616, -0.0120677445, -0.0638307184, 0.0495122261, 0.0677190199, 0.0426339507, 0.0379325598, 0.0130906785, 0.0649256706, 0.0722792298, 0.0792457759, -0.0558494441, -0.0531654097, -0.0059657176, 0.0450785942, 0.00883300416, 0.0574807562, 0.0332337394, -0.0352248177, -0.0437889323, -0.0436223447, 0.0613856353, -0.0144819347, 0.0253545735, 0.0602546744, 0.0358822234, -0.0300068203, -0.00821686722, 0.0339695103, 0.00141045346, -0.0651006177, 0.0506021306, -0.0289509706, -0.0317315273, 0.0385517739, -0.061515633, -0.00141796866, -0.0199405886, 0.0688744634, -0.0304414984, -0.0295749586, 0.0687240362, -0.0403112918, 0.0161262825, 0.00266853697, -0.0319643915, -0.0291807558, -0.0244653188, 1.35711845E-4, 0.00688301167, -0.0347004645, -0.0200628676, -0.0383148752, 0.0704856887, -0.0221007485, 0.00832881405, 0.0347628556, -0.0740345418, 6.485800e-02, -0.078305684], [-0.0132478857, 0.0482246578, -6.360520e-02, -0.0588286258, 0.0237305351, -0.0601168089, -0.0335394964, -0.0590890795, 0.0215256643, 0.0730590448, 0.055191543, 0.0535723753, 0.0171418786, 0.0649398565, -0.00268008583, -0.0795892328, 0.0697319433, 6.148150e-02, 0.0677743405, 0.0617811047, 0.0568063185, 0.0479389802, -0.0286929216, -0.0754070431, -0.0319792032, -0.0381899849, 0.0467037112, -0.0638453811, 0.0365818553, -0.020680869, -0.0680182651, 5.097230e-02, -7.247350e-02, -0.0135904495, 0.0485625267, -0.0220336765, -0.0265167356, 0.0116178524, 0.0746372119, -0.0471157469, 0.0092502078, -0.0501387827, -0.0550003909, 0.0159160886, -0.0351104625, 0.0106451632, 0.00784893147, 0.0449960493, -0.00401833886, -0.0178976245, 0.0369795449, -0.010193971, -0.0314212404, -0.0779810547, 0.0012915507, 0.0779524073, 0.0198110882, -0.0337219909, -0.0368939228, 0.0288031772, 0.0498383939, 0.0700115487, 0.0338766314, -0.0751873478, 0.0407799296, 0.0726017803, -0.0646128878, 0.0129693672, -0.0629945919, 0.0488152802, 0.0463007204, -0.0431210883, 0.0729412138, -0.0719299167, -0.0682644844, 0.0400007553, 0.0338866189, 0.0130520435, -0.067289345, 0.0129942978, -0.0715526789, -0.0233087633, 0.0748007521, -0.0427668095, -0.0574304946, 0.0332685933, -0.00976795144, 3.242120e-02, -0.012415343, 0.0365612954, -0.0159379039, 0.00269156601, 0.0571155734, -0.053736981, 0.0767377615, 0.00879549142, 0.0294987578, 0.0583791472, -0.0146244308, 0.0493263826, -0.0322041437, 0.0472393036, -0.0603127033, -0.0218782052, -0.0756333396, -0.0804421455, -0.0467641726, 2.8527528E-4, -0.066730991, 0.0752536505, -0.0189533308, -0.0191449933, -0.057021223, -0.0745881349, 0.0307157449, -0.0135220857, -0.0274770577, -0.00779803097, 0.0584165975, 0.0498775281, -0.0419469513, -0.00129437691, -0.0426269919, -0.0640286878, 0.074083954, 0.0733438879, 0.0316208266, 0.048212707], [-0.0590336621, -0.00285349786, -0.0386109091, -0.0723817497, 0.0515160635, -0.0758036599, 0.0475318953, -0.0810768306, 0.0467150882, 0.0546169803, -0.0757733956, 0.0713630393, 0.042555511, 0.0799120888, 0.0563274845, 0.0719546452, 0.00882662088, -0.0305086039, -0.0460371226, 0.0730028376, 0.0378884077, -0.0399061292, 0.0404747128, 0.0706325546, 0.0596268699, 0.0315822512, -0.0389699414, 4.134030e-02, -0.0335858874, -0.0781448706, 0.0705990717, -0.0482242666, -0.0171035081, -0.0417350084, 0.0698082075, -0.0137130171, 0.00201557577, -0.0134428442, -0.044607617, 0.0555446818, 0.0116208643, -0.0281651877, 0.069875963, -5.304890e-02, -0.0624704658, 0.00751908123, 0.060532786, 0.0474022105, 0.0186932534, 0.0245467201, -0.0236831121, -0.0407201536, -0.0421132073, 0.0454088375, -0.0509440228, -0.053924907, 0.0690858439, 0.00413157046, -0.0405571535, 0.0625373498, -0.0179109424, -0.0422580726, 0.0639056489, -0.0280051827, 0.0552808121, 0.0735454038, 2.286680e-03, 0.0620545223, 0.0804155543, 0.011351563, 0.00976464152, -0.076036185, 0.0475416854, 0.0408113748, -0.0423366055, 0.0274178982, 0.00536943227, -0.00652849674, 0.0600203648, 0.066838257, 0.0388899371, -0.0707938224, -0.0439494587, -0.00405713916, 0.0353898406, 0.0305701569, 5.106450e-02, -0.04199747, -0.0747678801, -0.0308028571, -0.0460336804, -0.0696957558, 0.0302564949, -0.057255134, -0.0607222616, 0.021729812, 0.0377304181, -0.0155745447, -0.0118860379, 0.00964900106, -0.0675657764, 0.00105552375, -0.0132192373, 0.077050589, 0.0477626249, -0.0788795128, -0.0310440622, -0.00834428519, 0.0339918956, -0.0116942376, -0.0342246518, -0.0137330741, 0.0632131323, 0.0765148327, 0.0446831509, 0.0366062596, 0.00483217835, -0.074029468, 0.0251486301, -0.0103799701, 0.0497840121, 0.0160579085, -0.0195811316, -0.0561808161, -0.0211799257, 0.061779581, 0.00869231671, -0.0640724078], [-0.0203780048, -0.0242620036, 0.0339474976, -0.0423921235, 0.0399383083, 2.78763473E-4, -0.0013641417, -0.0654139668, -0.073530674, 0.0352084339, 0.0784799084, -0.00776537508, -0.0110045373, 0.022927098, -0.0377859361, -5.424840e-02, -5.872150e-02, -0.0247157402, 0.0779515132, 0.0208278522, 0.0409536809, 0.0133208558, 0.0546517298, -0.0673128888, 0.0610238686, 0.0178051367, -0.0291217603, 0.0507308766, 0.0805747136, -0.0463707112, -0.0423571244, -0.0253154188, 0.0118329152, 0.0734382644, -0.0159237161, 0.075059019, 0.0426047295, -0.0423847362, -0.0396553129, 0.0307706743, -0.0510995835, -0.00751797854, 0.0621997342, 0.0696659312, 0.0629236922, 0.0503863469, -0.00316333771, 0.0148980916, -0.0600981191, 0.0703053549, 0.0584959611, 0.00348482281, -0.0378560387, 0.00312856585, -0.0230230205, -0.0566310138, 0.0608913228, -0.0728723779, -0.0582579635, 0.0288629532, 0.0616114363, -0.0664479434, -0.0378882736, 0.0808518454, 0.00349389017, 0.0718506202, -0.0618244261, -0.0129844472, 0.0125732794, -0.070611693, 0.0185891166, -0.0222534128, -0.0169849619, -0.0188178495, 0.0323683918, -0.0658527687, 0.0317649171, -0.0332714692, -0.0478795432, -0.0482755713, 0.0681142434, 0.0665808842, 0.0427887738, -0.0233307891, 0.0386387557, 0.0179454386, 0.0427579507, -0.0307641812, -0.0564227402, -0.0234178491, 1.914420e-02, -0.0324871652, 0.0441955701, 0.0128791705, -0.0472161621, 0.00213065743, -0.0626234114, 7.01025128E-5, -0.0193812512, -0.00547190756, 0.0355560556, -0.0206787921, 0.0725943372, -0.00918424129, -0.011435546, 0.0070156455, -0.0111527666, 0.0243486613, 0.0673106685, -0.0761564299, -0.0308835953, -0.0150497779, -0.0571130365, 0.0314882845, -0.00334906578, 0.0681402758, -0.0491341352, 0.0805849805, -0.00588352233, -5.713850e-04, 0.0134797618, 0.00473291427, 0.0660621896, -0.0264714621, 0.0687417164, -5.182550e-02, 0.0255508646, -6.722490e-02], [0.0304746255, -0.0579336844, -0.0766851753, -0.0545487776, -0.0721704811, 0.0734087378, 0.0350128189, -0.0206747223, 0.0502869412, -0.0259901192, 0.0307683386, -0.0725836679, 0.071935378, 0.015376173, 0.0439593717, 0.00488328375, 0.0570398048, -0.0804656595, 0.0480237268, 0.0173316449, -0.0497854091, -0.040778216, -0.0795049145, 0.0261165164, -0.0366511606, -0.0742140189, -0.0373579822, 0.0368027166, -0.00627864152, 0.00139005727, -0.0670927092, 0.0779828578, 0.0708509088, -0.0609883033, 0.0648136288, -0.028783381, 0.00291019608, 0.0712097585, 0.0299974624, 0.014337182, 0.080697462, 0.00706487894, -3.61627142E-4, -0.0631627962, -0.0808737948, -0.0703815743, -0.0373642892, 0.0251683388, -0.0760870054, 0.0371653549, 0.0457610674, -0.0254149716, -0.0387818255, -0.0741021782, -0.0466130562, 0.0642748922, 6.2026846E-4, 0.0758886859, 0.0549952276, 0.0209396239, -2.560860e-02, -0.0550349653, 7.794590e-02, -0.0129510388, -0.0783844143, -0.01320442, -0.0355059505, -0.0348128639, 0.00371122756, -0.0516180769, 5.544580e-03, 0.0656693205, 0.0146818375, 0.0633258894, 0.0253827348, -3.328490e-02, -0.0788290352, 0.0412692092, 2.384370e-02, 0.018188389, 0.0452212207, 0.0553180315, -0.0159994289, 0.0503786206, -0.0375919305, -0.0518995188, -0.0268781073, 0.0321503803, 0.0694938749, 0.0204008222, -0.0390888788, -0.0608186871, -0.0580909625, 0.0729459673, 0.0234671552, 0.0142692802, 0.0803643837, 0.0608388297, 0.0442751572, 0.0337582342, 0.055205781, -0.0419377461, -0.0679162368, -0.0609262287, -0.0749487876, -4.381920e-02, -0.0136650708, -0.0435639247, 0.0593444481, -0.0253038071, -0.0439489521, -0.066066578, 0.0232379697, -7.964130e-02, -0.0369883217, -0.0427273661, -0.00840571057, -5.571460e-02, -0.0623372904, 0.0727136359, 0.0449469462, 0.0594392829, 0.00643306226, 0.0163747165, 0.0447085835, 0.041579105, 0.0443505868, 0.0453860685], [0.0013890533, 0.0620125681, 0.0239425879, 0.0237324201, 0.0443039127, 0.0503550582, -0.055012092, 3.240780e-02, -0.0432266705, 0.0455162786, -0.0442389511, -0.0757555813, -0.0534431711, -0.033360213, -0.0448071361, 0.0127285477, -0.072777912, -0.0767406523, -0.044887431, -0.022335276, 0.0620217025, -0.049723357, 0.0216178764, -0.0602858588, 0.0581373684, -0.0213473737, 0.0711026192, 0.0705125629, -0.0145159466, -0.0536995307, -6.033510e-02, -0.0051169931, 0.00774710206, -0.0541070662, 0.0313763842, 0.0771370456, -0.0748040602, 0.049913466, 0.0515263677, 0.00156701065, -0.0675689355, 0.0732707456, 0.0133299911, 0.0342495367, -0.0441788584, -0.0673517659, -0.00339957746, 0.00742785679, -0.0494717918, -0.0295446105, 0.0355323628, 0.0121767102, -0.0689337179, -0.0258674584, 0.080992654, -0.0548316911, 0.0563227572, -0.0347655453, 0.0593957752, 0.0137913441, -0.0390024669, -0.0227816403, -0.0366870761, 0.0172937848, 0.0233709272, 4.366400e-02, -8.73465905E-4, 0.046968367, -0.0515799709, 0.0603961386, 7.234550e-02, -0.0703185946, -0.0414089225, 0.0592275783, -0.0169384554, -7.18900643E-4, 0.0344100744, 0.0632603616, 0.0765831172, -0.0797191485, 0.0789318755, 0.00346953864, 0.0758919269, 0.00686134771, 0.00186189264, -2.680150e-02, 0.0769884884, 0.05690106, -0.00957465079, 0.0337576829, -0.0402130559, -0.0384532847, -0.0301409904, 0.0685534403, 0.0656734258, 0.0501314923, 0.0623238272, -0.0147283785, 0.0464207679, -0.0663891509, -0.0484927855, 0.037368279, -0.0321985781, 0.0690469816, 0.0275751781, -0.0351263322, -0.0289474949, 0.00282060704, 0.0461257324, -0.065834932, -0.0603004023, 0.0264837593, -0.0189607479, 0.0791792423, -0.0447158851, 0.0576713793, 0.0578866974, 0.0709080547, 0.0453998744, -0.0697821751, 0.0505219325, -0.0242476594, 0.00902184657, -0.00426901085, -0.0345882624, 0.0325867683, -0.00315087917, 0.0599092469], [0.030279614, -0.0450001918, -0.042910628, -0.0368666574, -0.0180541072, -0.0608687252, 0.0785760432, -0.0392779559, 0.045989953, -0.0323161222, 0.0655093044, -0.0299370252, -0.0350904167, 0.0254927464, -0.0747428313, -0.0030595297, 0.0553583428, 0.0607425384, 0.0454064421, 0.0206850599, 0.0323996879, 0.0479075909, 0.0365857296, -0.071131967, 0.0606809556, -0.050695125, 0.0348763168, 0.0609520748, 7.010860e-02, 0.0295959096, 0.0745929256, -0.0576893762, -0.042758774, -0.0300569385, 5.915810e-02, 0.066663228, 0.0368117467, 0.0278886557, -0.0214969609, -0.0320255421, 0.0753673464, -0.046482645, 0.0234995242, 0.0557895973, -0.0168762673, -7.430730e-02, 0.00692151394, 2.98457344E-5, 0.0238920916, -0.00837552919, -0.0150798457, 0.0267931521, -0.03212028, -0.0107128136, -0.0806465446, 0.0490255058, -0.0140939374, 0.0257807933, -0.0481040366, -0.0355656743, 0.0020115152, -0.0247616712, 0.0416498892, -0.00498850411, 0.0709078535, -0.0049165017, 0.00275161583, -0.0360661931, 4.488880e-02, -0.0197464265, -0.0375873782, -0.0691883788, 0.0491350442, -0.0740326941, -0.0363907963, -0.0281361956, 0.0803714842, -0.0708391741, -0.0468712077, -0.0788865238, -0.0421106331, 0.0109549547, -0.0206370335, -0.0733704716, -0.0669184849, 0.0312912613, -0.0548052788, -0.0369584933, -0.011174297, -0.00809877086, -0.00122981193, -0.0786178186, 0.0765813068, -0.0445943475, -0.0766976848, -0.00248192949, -0.0510263257, 0.0133684697, -0.0392654277, 0.0274899807, -0.0758704916, -0.0591134205, 0.0788016468, 0.0388483033, -0.0628997535, -0.0155231385, -0.0300044548, -0.0169813577, -0.0682919174, 0.06779401, -0.0112604974, -0.0253444277, 0.0488590412, 0.0219493844, -0.0590703897, -0.0647080466, 0.0145521183, -6.613860e-02, 0.0497724675, 0.0387654789, -0.0261974391, -0.00970191508, 0.0207613744, 0.0425090864, 0.0413601734, 0.0637639388, -0.0741285756, 0.0413300283], [-0.0275360905, 0.0113936132, -0.0333298072, -0.048190359, 0.00154114328, -0.0420946479, -0.0580402315, 0.0327832177, -0.0516785644, -0.0194744244, 0.0363102779, -0.0484179035, -0.0605727136, 0.0756277069, -0.0141885672, 6.497080e-02, -0.0195061956, 0.0251819938, -0.00567681668, 0.0438173749, -0.0264171939, 0.0333371684, 0.0423771255, 0.00181475189, 0.0261987206, 0.0309980679, 4.360450e-03, -0.0199673865, -0.0178748276, 0.0174193103, -0.0540585704, 0.0753376931, -0.0270205289, -0.0369387195, 0.0226076078, 0.0583839603, 0.00376640353, -0.0672221929, -0.0344481468, -0.0762587413, -0.0750000402, 0.080987133, 0.0680749044, -0.0167411808, 0.0486250892, -0.0273199473, -0.0476490371, 0.00740110921, 0.053749036, 0.0519294366, -0.0369085334, -0.0389922895, 2.887230e-02, 0.0366699547, -0.0230982453, 7.44758931E-4, 0.0423966534, 0.0510986261, 0.070421882, -0.0223422665, -0.0343285725, -0.0514346175, -0.00723794382, -0.0161556825, 0.0486955531, -0.06388174, 0.0487409346, 0.0446577147, 0.0427052379, 0.0455241762, -0.0675563291, -0.0673243552, 0.0347706154, 0.0481523648, -0.00845666415, 0.0211464725, 0.0565354601, 0.0548904091, 0.0268294103, -0.0422445387, 0.015454174, 0.0185997915, 0.0412612036, -0.0175610334, 0.00773768685, 0.0429733656, 0.0476328582, -0.0262204148, -0.0247635115, 0.047486838, -0.0807266458, -0.0706733763, 0.0725669712, -0.0270109177, 0.0435447693, 0.030397106, -0.0628994554, 0.0290948339, 0.0110268407, -0.0176346935, -0.0129367327, 0.0636820346, -0.0581427701, -0.00672859419, 0.0101534119, 0.00456122123, 0.0448690057, -0.0215259176, 5.392080e-02, 0.0529511757, 0.00271529565, -0.0513550565, -0.066808559, 0.0149044255, 0.0645784437, 0.0229043383, -0.0552941337, -0.0278392155, 0.0407606438, -0.00494141877, -0.0454588346, -0.0652668178, 0.069611825, -0.0316520147, 0.0208684169, -0.0716564432, -0.0415056124, 0.00900496169], [-0.0723534077, -0.0199808385, 0.0727873146, -0.0276972689, 0.0163395721, 0.0214907788, -0.055861149, 0.0681133941, 0.0198256951, 0.0712517797, -0.0140392212, 0.0226418395, 0.0742349326, 0.049453415, -0.0326493084, 0.067923151, 0.0658605695, 0.0684006661, 0.0309634767, -0.0618796162, -7.421520e-02, 0.0575535782, -0.0428954475, 6.175280e-03, -0.0613939576, -0.069819048, -0.0667289272, 0.0138550857, 0.0797902197, -0.0472283699, -0.0116030322, 0.0215014089, 0.0173037257, 0.0206107665, -0.00953910872, -0.0397177078, -0.0526030548, 0.00918118376, 0.0748064219, -0.0480181538, 0.0698057786, 0.0425422229, 0.00440861285, -0.0569205135, 0.0222350452, -0.0452061184, 0.0306076407, -0.0240333565, 0.077661857, -0.060837891, -0.00376394019, -0.0306709222, 0.0104749668, 0.0545882136, 0.0762791708, -0.00496512558, 0.034995392, -3.59753438E-4, -0.074545756, -0.077402018, -0.05834933, -0.0430393033, 0.0486426912, 0.0579642355, -0.0540143102, -0.0172951035, -0.0656977147, 0.0318297409, -0.0756899342, -3.684060e-02, -0.0788989365, -0.070181407, 0.0277149063, -0.0227868576, 0.0117973154, -0.0682153925, -0.00326810195, 0.067630671, 0.00325345527, -0.0491761379, 0.0313106515, 0.0638702139, -0.0784228518, -0.075620763, 0.0695581287, 0.0540793315, -7.352950e-02, 0.00502024638, -0.0162261222, 0.068588376, -1.188620e-02, 0.0103230625, -0.0296666473, 0.0656233504, 0.061949674, 0.0774193555, -0.0740789324, -0.0130197145, -0.0388267487, 0.0119455904, -0.0753682553, 0.0654629469, 5.813080e-02, 0.068145372, 0.00307209906, 0.00583926169, -0.075828515, 0.0296083204, -0.0640499368, 0.0145754684, -0.0160689913, -0.00646470767, 0.0572664328, 0.0784416869, -0.0471708737, -0.00265195034, 0.0207015835, -0.0614068247, 0.0431682207, 0.0718185603, -0.071872659, 0.0381799303, -0.0659583211, 0.0714787542, -0.071221821, -0.0111321723, -0.0148487315, 0.0598625243], [-0.0744266211, 8.090320e-02, 0.0645209774, 0.0239044391, 0.0151837133, -0.0599086396, 0.0121038044, -0.0297059715, 0.0729985684, 0.0280945413, 0.0107641499, -0.0452548414, -0.0199848246, -0.0726557896, 0.0393943228, -0.0183249172, 6.336960e-02, 0.0233234633, 0.0325204805, -0.00549961254, -5.998060e-02, 0.0358835757, -0.0426182486, -0.00396757666, -0.0413273647, -0.0610758178, -0.075474374, 0.0279535726, -6.554040e-02, -0.0521633774, 0.0694183931, -0.0808236598, -0.0398025922, 0.0369548835, -0.0558959171, 0.00819317531, -0.0331994519, -0.0565630123, 0.0100835674, 0.0162737183, -0.0179233495, -0.00415776158, 0.00332771288, 0.0643649772, 0.0621205382, 0.0791149735, -0.0658746585, 0.0359637551, 0.0753311068, -0.0767164677, -0.0493244827, -0.0638900697, -0.0232059807, 0.0531476215, -0.0256118271, -0.0621260069, 0.0727477819, 0.0590018518, -0.0229814909, -0.00610887352, 0.00349639496, -0.0194936525, -0.00515435264, 0.0718613639, -0.0623856596, -0.0339070596, -0.0345270522, -0.0661286637, 0.0527851172, 0.0422084555, -0.0463724583, 0.0406884402, -0.0640527532, 0.0205704421, 0.0437866598, -0.0525116026, -0.0522275381, 0.0626549944, 0.0742232651, -7.275640e-02, 0.0374791175, -0.0745160505, -0.034893088, -0.0329936557, -0.0704260319, -0.0845864266, 0.0290902779, -0.0015403619, 0.0301541034, -0.0599999875, 0.055434797, 0.059668269, -0.0547101311, -0.0516535193, -0.0190655161, 0.0326629616, -0.0596361645, 0.0186223071, -0.0547444373, 0.0755098164, 1.407610e-03, 0.0626722947, 0.0611834824, 0.0149170104, 0.0277042594, 0.0723907724, -0.00797559693, -0.0691377818, 0.0167765915, 0.00844294391, -0.00829664897, 0.0705572664, -0.0070653772, 0.00768806925, 0.0605192706, 0.0365761407, 0.0258758031, -0.0376062952, 0.0762915239, 0.0691577345, 0.0440797284, -0.015751427, -0.0744377822, 8.15360167E-4, 0.0501805805, -0.0238654055, -0.0153513113, 0.0259113219], [-0.0151245408, 0.0885216445, 0.0427753739, 0.0580181032, 0.00486007333, 6.993810e-02, 0.0240608882, -0.0065633473, 0.0411479436, 0.0296984129, 0.00398921454, 5.31848869E-4, -0.0112477764, -0.0065068258, -0.0320569351, 0.00422121864, -0.0408004783, -0.0596773252, 0.036792174, 0.044112552, -0.0554094352, -0.021098299, -0.00712613808, 0.0590139516, 0.0572572351, 0.00773326959, 0.0125172175, -0.00356858969, -0.0249845609, -0.0162687358, 0.0775391086, -0.0520953685, 0.0855990946, -0.0652765185, 0.073618345, 7.267050e-02, 0.019115271, -0.00635576714, 0.0275915023, -0.050874617, 0.0214906707, -0.0197341237, 0.0734562054, 0.0069498294, 0.0314111561, -0.0527639538, -0.0602191389, -0.0792011618, 0.0574747622, -0.0546096489, 0.0118263988, -0.0605607368, 0.0226228628, -0.074812226, 0.00190634106, -0.0158327743, -0.0562301576, -0.0244023856, 0.0024240464, -0.0201763678, 0.0363343842, -0.0450725555, -0.00396425789, -0.0523373187, -0.0418257862, 0.0379755683, 0.0156129505, 0.0058647939, -0.0395854749, 0.0268238336, 0.0693004057, -0.0511626191, -0.0614608563, 0.0329839624, -0.073361747, -0.037525475, 0.0752595514, -0.0333275385, 0.00501855789, 0.0116532091, -0.0251890868, 0.0571913496, -0.0288815722, -0.0301040299, 0.00194049487, 0.0510756597, -0.0499457903, -0.0390310735, -0.0191922374, 9.75181697E-4, 0.0358423926, -0.0259979181, 0.0677512512, -0.063210085, -0.0284441691, 0.0321178026, 0.0536137968, -0.00336203561, -0.0454921052, 0.0718819499, -0.0708898455, -0.0677682459, -0.0125336973, -0.0357751846, -0.0629908219, 0.0541224293, -3.759960e-02, -0.0304810368, -0.0791288539, -0.0656673908, 0.0304523446, 6.142650e-02, -0.0087779425, -0.0584082194, 0.00451588258, -0.00567971729, 0.0318127349, 0.0573323816, 0.0300284941, 0.00750462851, -0.00857129786, 0.0675192401, 0.00471154787, -7.605990e-02, -0.00759406434, 0.0712909996, -0.0395783409, 0.0754204616], [-0.00813565496, -0.0273881014, 0.0134714525, 0.0861812755, -0.00199270109, 0.0149594937, -0.0684590191, 0.0803300738, 0.0228597596, -0.0517316386, -0.00834909547, 0.0724730045, 0.0278555956, 0.0390548445, 0.0781040713, 0.0844684615, -0.0639157593, 0.0178571362, -0.0481719561, 7.193970e-02, 0.0524046309, -0.0749746263, -0.0192301292, 0.00908216647, 0.0412610024, 5.269250e-02, -0.0645595863, 0.0463343039, -0.0701582953, 0.0563140213, 0.0407147221, 0.0564058349, 0.0207433738, 0.0582812391, -0.0873450264, -0.0616839118, 0.047092896, -0.0311657116, 0.0677014664, 0.087702848, -0.0272976905, 0.0643613636, -0.0786181315, 0.00310625206, -0.0806110352, -0.0078558959, -0.0201163217, -0.0721141174, 0.0541510209, 0.0461726151, 0.0447122976, -0.0469149202, -0.0715693384, -0.0668271929, -0.0436662734, -0.0051779137, 0.0211861525, 0.0130920457, 0.0337160453, -0.0670373514, 0.0189245511, -0.0618890338, 0.00171168894, 5.21611248E-4, -0.0875787586, -0.0381762572, 0.02147834, 0.0384056196, -0.059695784, -0.0282673761, -0.0569688044, -0.0140494732, -0.033849895, 0.0892139747, -0.068467252, -7.171010e-02, 7.628350e-02, -0.0596421286, 0.060459096, -0.0311609618, 0.0690215081, -0.0345885418, -2.865740e-02, -0.0228963289, -0.0627789646, -0.0873348712, -0.0804083868, -0.0296535678, -0.0320347026, 7.799800e-02, -0.105993614, 0.00896466058, 0.080579564, -0.0628637299, -0.00329288677, -0.0175099112, 0.0344519205, -0.0465763696, 0.0570497252, 0.0378285944, 0.017893808, -0.0444866829, -0.0782110542, 0.00219767122, -0.0799772888, -0.0512696393, 0.00907210354, -0.00745804515, 0.0620305091, -0.0683824345, -0.0261318684, 0.0726225302, 0.0616027229, 0.0363189094, 0.0839297696, 0.0125375409, 0.019158408, 0.0451752655, -0.0567004718, -0.00341308513, -0.060185492, 0.0523744747, -0.0124931689, -0.0294264592, -0.00607486069, 0.055814188, -0.0335317664, 0.0262028705], [-0.0715465099, -0.00769684324, 0.0325738117, 0.0177969597, 0.0701104924, 0.0361451358, 0.0253963657, 5.17807435E-4, -7.888390e-02, 0.0509804823, -0.0326681174, 0.0772377402, 0.0803705305, 0.0341730043, -0.0241552554, -0.0160465986, 0.021250207, -0.0665498301, 0.0172368102, -0.0447577648, -0.0584681407, 0.0509170219, -3.844780e-02, -0.0765398294, -0.0568000861, -2.47995427E-4, -0.0299744476, 0.0919303149, -0.00913301576, -0.0370983668, -0.0208712891, -0.0887204185, 0.0700780898, 0.0222995561, 0.00518984068, 0.0636699721, 0.0332806222, -0.0697078406, 0.051339142, 0.0937117263, -0.0377035625, 0.00767144701, 0.0530212335, -0.0387378857, 0.046741493, -0.0462548696, 0.0588886365, -0.0763751194, -0.00157684193, -0.0537358932, 0.0549238138, -0.0122751314, 0.0321490169, 0.0617529787, -0.0159533974, -0.00524027413, 0.0292466022, 1.911480e-02, -0.0565725192, 0.0581889115, 0.0158122052, 0.0225799922, 0.0692450106, -0.0591105558, 0.0564421713, -0.0768933445, 0.0178248547, -0.0730880052, 0.0502252392, -0.0187577251, 0.00282957032, 0.0204976052, 7.293230e-04, 0.0867781862, -0.0133367451, -0.0257805735, 0.0454642922, 0.0265433155, -0.0180770736, 0.00213412894, -0.0595448241, -0.0495224111, 0.0324362442, -0.00453505525, -0.0656479672, 0.00759456586, -0.074852623, -0.0218838062, -0.022226369, 0.0371203236, 0.038784951, -0.0172117166, -0.0187438875, 0.0597336441, -0.0755356327, -0.0515030622, 0.0226787832, 0.0121361064, 0.0224562082, 0.0471030176, -0.0534231737, -0.063021928, 0.0485986397, 0.0462259501, 0.0386954471, 0.0233319383, 0.0278129019, 0.0586011671, 0.0669931546, -0.00379270129, -0.0106503218, 0.0575740524, 0.00584152667, -0.0406483673, -0.025372766, -0.0127510848, -0.00974913593, -0.0834707617, -0.00285892677, -0.0750330612, -0.0154364072, 0.0669390485, -0.098264493, 0.00224864041, 0.0627891794, -0.0621291659, 0.00273285457, -0.0629084781], [0.0348091125, 0.0262577347, 0.0783328935, 0.0331967622, 0.0376566648, 0.0524952151, 0.0110438326, 2.27974873E-4, -0.0189488493, -0.00259702932, 0.025815919, -0.0328809544, 7.556810e-02, -0.0322989784, -0.0526331104, -0.0088955406, -0.0687610582, -0.0232750177, -0.0168969408, -0.084512718, 0.0344852619, -0.0140984217, -0.0720691085, -0.0783700421, 0.0577394105, -0.0453043245, -0.0395585261, 0.0593204647, 0.0241833776, -0.0523731224, 0.0153035205, -0.0530331768, -0.0439392775, -0.0542304069, -0.0190595966, -0.0518290624, 0.0243831854, 0.0482961908, -0.0932647213, 0.0112385945, 0.0531841964, -0.0729377717, -0.0790549218, -0.0545644052, 0.0412941314, -0.00280040363, 0.0601786375, -0.0173092689, 0.0201717261, 0.0331728496, -0.0520362481, -0.0239980854, -0.039283365, 0.00812325161, -0.0461765751, -0.0598551854, -0.012848936, 0.0129633769, -0.0618032254, -0.047102686, -0.0868700221, -0.0176834669, -7.623370e-02, -0.0175634213, -0.0291982871, 0.0107339257, -0.0219625533, -0.0685154497, -0.069779858, 1.556270e-02, -6.788570e-03, 0.0491808765, -0.0263997298, 0.101459876, 0.00306915049, -0.0839246511, 0.0659282207, -4.93582396E-4, -0.0842480063, 0.0837429091, 0.0640740096, 0.0799142569, 0.050574109, 0.0317909308, 0.0267343502, 0.016684629, -0.0674039498, -0.00431765709, 0.0429565459, 0.0488586761, -0.101214379, -0.0539477691, -0.0547710508, -0.0602619462, -0.0195327979, -0.0674366802, 0.025301069, -0.0838087797, -0.0437180363, 0.0425955169, 0.018335361, 0.0010157628, -0.0139018875, -0.0474590734, 0.0439534448, -0.0172596369, 0.0685694888, -0.0446644835, 0.0144939255, -0.0495602563, -0.0223636571, 0.0556250587, 0.0783608407, 0.054801207, 0.0100800321, -0.0478957966, -0.0284057949, -0.0206734538, -1.630890e-02, 0.0264760684, -0.0405768715, 0.0119480509, 0.00609252555, 0.0161277503, -0.0517478399, -0.00536493631, 0.0236704852, -0.0786075517], [-0.0239875447, 0.0976939201, 0.0390737467, 0.0379181616, -0.0129922768, -0.066136837, -0.0559117794, 0.0676414147, -0.0479547642, -0.00421827333, -0.0167983901, 0.0765491425, -0.055848293, -0.0209527705, -0.0532876141, 0.084821023, -5.485860e-02, -7.666410e-02, 0.0987844318, 0.0392493717, 0.0334442332, 0.0518390425, -0.020614434, -0.0291798431, -0.0106123229, -0.075307995, 0.0645442083, 0.0203868244, -0.0278571174, 0.0286075957, -0.0613237284, 0.0449530557, -0.0295625348, 0.00642335368, -0.0236190557, -0.0582269244, -0.0655602738, -0.0336245187, 0.054312937, 0.0703375787, -0.0050377748, 0.00310208928, -0.0305041689, -1.12425732E-5, -0.0291186012, 0.0364103168, -0.049488239, -0.054623533, -0.0203218237, 0.0605765022, 0.0282229912, 0.116173737, -0.00483535789, -0.0434415936, 0.0635115504, 0.0247482304, 0.0245541371, 0.0667378604, 0.0733385459, 0.0511387028, -0.0450763553, -0.0750648603, 0.0022682813, -0.0348093063, 0.013767343, -0.0718617961, -0.0614065528, -0.0250124186, 0.0735782757, -0.0962168201, 0.0661049336, -0.0186314657, 0.054424569, 0.0208424944, 0.0739284381, -5.509900e-02, 0.0289352685, 0.0164128114, -0.0251219273, 0.0511272624, -0.00540642533, -0.0621426366, -0.089149706, -0.0246598888, 0.00677789329, -0.0117109567, -0.07776957, 0.0305683278, 0.0966694578, 0.0148997316, -0.10243573, -0.0100975372, -0.0527061261, -0.0184169766, -0.0594047792, 0.0475441143, -0.0260540359, -0.0584817119, -0.0977634564, 0.00523183262, -0.0785738677, 0.0239403117, 0.067254439, 0.0656742528, -0.0238678064, -0.0221073329, 0.0500421263, 0.0597667359, 0.0304278955, -0.067174688, 0.105529487, 0.0469316132, -0.0645045638, 0.043803595, 0.0177988727, 0.0662942678, 0.024286272, -0.0250394084, -0.0152936494, -0.0618705079, -0.0353366137, -0.0256521031, -0.00882842857, -4.525320e-02, 0.0449788831, -0.00161558494, 0.036484316, -0.011377099], [0.0377405919, 0.0814402103, -0.00457138941, 0.084268406, -1.10657857E-4, 0.0113958856, 0.0371496677, -0.0433739349, 0.0167971607, -0.067823194, -5.212000e-02, 0.0214270242, -0.0616673119, 0.0846101269, 0.0381797962, 0.0145851308, 0.0578368194, -0.0725728497, 0.0612663738, -0.0454287566, -0.0146920281, -0.0167094786, -0.0169433039, 0.0281367712, -0.0631892607, -0.0458324514, 0.0797527134, -0.0388763733, 0.0600644276, -0.0367603377, -0.0525962263, 0.0162668843, -0.0283413511, -0.0124503272, 0.0679551661, 0.0927828103, 0.0642286241, -0.0670605153, -0.0491181724, 0.0972420796, -0.0671673566, 0.0707223788, 0.0314246826, 0.00433556037, -0.0102508711, 0.0646567866, -0.0743363574, -0.00416580448, -0.0162910279, 0.0561024509, 0.0563927591, 0.0511732623, -0.0345288068, 0.0517272875, -0.0467649065, 0.0271398444, 0.00111837487, -0.0541424826, 0.0483467169, 0.0602220669, 0.025463162, 0.0311045218, -0.0256779864, -0.0124898441, -0.0975052565, -0.063126646, 0.0847784281, 0.0299651902, 0.052230265, -0.0685968697, -0.0689154118, 0.061208237, 0.0355384797, -0.0444782451, 0.0705195516, 0.0156071475, 0.0397962481, 0.0451511145, 0.00110734021, 3.59496713E-4, -0.0512277372, 0.0841080918, 0.0264295209, -0.0609953143, 0.021994954, -0.0246478561, -0.072285451, 0.0541969724, 0.0571893975, -0.0743961632, -0.054254923, 9.903140e-02, -0.0419423319, -0.0573119558, 0.043021556, -0.0239056069, -0.0719613358, -0.00642920471, 0.0317065343, 0.0266839564, 0.0505601652, 0.0264108256, 0.0651735589, -0.0853332728, 0.0794379338, 3.57642188E-4, -3.448880e-02, -0.0382127464, -0.0330011733, -0.082365945, 0.0818236097, 0.0623065121, 0.0637825951, -0.0280089118, 0.023747012, 0.0660136193, 0.0439564958, -0.00273218355, 0.0377797484, -0.0323444568, -0.0234070886, -0.00598178105, -0.018954087, 0.0988641977, 0.076580435, -0.0816377848, 0.0552867539, 0.0340536647], [-0.0208782069, 0.00724499113, -0.0237644799, 0.115083821, 0.100481279, 0.0432193168, -0.00344885816, -0.0576840788, 0.0318898372, 0.0316208564, 0.0100124553, -0.0833705365, 0.0144593921, 0.077208586, -0.0337152146, 0.0699386075, -0.0607170239, 0.0398291238, -0.0107657574, -0.0321413353, -0.0278671775, -0.061013259, -0.0548879504, 0.0741239786, 0.0345297344, 0.0526437499, 0.0723037273, -0.0488080904, -0.0298504345, -7.056380e-02, 0.0680434182, -0.00223268359, 0.0268998072, -0.0322101749, 0.0278773326, 0.0927196517, 0.00287701446, -0.00773494644, 0.0146133257, 0.0692625642, -0.0050139185, -0.0675062537, 0.00649045594, -0.016623931, 0.0468102098, -0.0645177364, -0.039495144, 0.0192641448, 0.095799759, 0.00630505523, -4.22449739E-5, 0.0259079449, -0.0902390703, 0.0156353582, -0.0197901912, 2.091210e-02, 0.0315871052, 0.00906824227, -0.0623730794, 0.100894123, -0.0514532141, -0.0728413388, -0.0767465904, -0.0884224697, -0.0689867735, 0.0476706363, -0.0364953242, -0.0562463067, -0.023138281, -0.0606700629, -0.0342341736, 0.0737854168, 0.0384627022, 0.0122114988, -0.00101352821, 0.0217745267, -0.0116173457, 0.0798328295, -0.0529592782, 0.0663425103, 0.0201707613, 0.00671108952, -0.0777077451, -0.0554046445, 0.0233286954, -0.0173874293, -0.0709773153, -0.0534977578, 0.0881169214, -0.02881076, -0.0442266092, 0.0772123038, -0.034364108, 0.0609171316, -0.00714709098, -0.0207424387, 0.0193329621, -0.0159898661, 0.0267255828, -0.0869255661, -0.0269657876, -0.0638133734, 0.00559758255, -0.00150048954, 0.060651198, 0.0353292078, 0.0453306288, 0.0206261203, -0.072535336, -0.0016862927, 0.018268669, -0.0372306556, -0.0461621359, 0.0986502766, -0.00173256069, -0.0204188768, -0.0699429661, -0.0251195114, -0.0740241259, -0.0736883357, -0.0750817284, -0.0596234165, 0.00658589881, 0.104526803, -0.0353268497, -0.0231870022, 0.0297763478, -0.0479138047], [0.0642675236, 0.0851815194, -2.940010e-03, 0.0901915729, -0.0269739032, 0.0518654138, -0.0270429719, -0.0517004505, 0.0314206779, -0.0725227818, 0.00113741634, -0.068632707, 0.081622377, -0.035012465, 0.0385802239, 0.0524226502, -5.319010e-02, -0.0387004837, -0.0465099961, -0.0542584807, -0.0383942872, -0.00365313701, -0.0401248224, -0.0314098708, 0.0444172099, -0.0743908212, 0.0465664938, -0.0262343865, -0.00177189556, -0.0665801391, 0.00341463974, 0.0890365764, 0.0782620683, -0.095973663, -0.0644635782, 0.0637190043, 5.070290e-02, -0.0321427397, -0.00746382772, 0.0374354459, 0.0557557344, 0.0416972525, -0.0245321337, 0.050905209, -0.0151544111, 1.10594789E-4, 0.0470622554, 3.85945488E-4, -0.0365080796, -0.0560967736, 2.076540e-02, 0.0793372318, -0.0201415252, 0.0123482924, 0.069082506, -0.0105346432, 0.0266446695, -0.0183411147, 0.0437434502, 0.0227139089, 0.0535576157, -0.0540381074, -0.0220898893, -0.0213947855, -0.0718196779, 0.011279894, 0.0928084552, 0.0416939519, 0.0849072635, 0.00480987644, -0.0234609973, 0.0490800329, -0.0377546437, -0.0157549866, 0.0233842731, 0.0844852402, -0.0152624659, -0.0690343902, 0.0284801293, 0.0414082967, -0.0578140058, 0.0692860112, -0.0702932104, -0.00718139485, 0.0694786534, -0.0693945065, 0.0260122269, -0.0265935026, -0.0291928537, -0.0589459911, -0.0337767601, -0.0243005175, -0.0206268467, -0.0444264337, -0.0704015493, 0.00444015348, -0.0109399529, 0.00455843331, 0.0153812505, -0.0569061637, 0.0326519199, -0.0155414874, -0.0788253843, -0.0822913348, 0.00209411979, 0.0605428852, -0.0733175352, 0.0425615124, 0.0293198936, 0.0595851801, -0.0551283322, 0.0329252109, 0.0706687421, 0.134991735, 0.0430781655, -0.0334473774, -0.0296381563, -0.0265872423, 0.0228446163, -0.0602220036, -0.00909198355, 0.043414358, -0.0102548115, 0.0297581255, -0.0336374082, 0.097380042, 0.0483690128, 5.888810e-02], [0.0287250336, 0.0347607322, -0.058884453, 0.0699699447, -0.0183133502, 0.0718722865, -0.0138104986, -0.0114927208, -0.0486465655, -0.0260416046, 0.0618199483, -0.0480286106, 0.0708291828, 0.0943627581, 0.0723674148, 0.0633270144, 0.0227775313, 0.00366745191, 0.0768128931, -0.0266124103, -0.0607503578, 0.0039150482, -0.0342951603, -0.0365365371, 6.02921064E-4, 0.0375626683, 0.0192353539, -0.00855989381, 0.0170938354, -0.0260496922, 0.0384943374, 0.0857097432, 0.069590047, 0.0415593088, 0.0104810493, -0.0247843601, -0.096528314, 0.062233597, 0.0617697127, 0.053576041, 0.0780295431, 0.00566755515, 0.0373962484, 3.57176876E-4, -0.0734801516, -0.0114421621, 0.0708135813, -0.0948665589, 0.0346333645, -0.0531408116, -0.0609154701, 0.045940727, 0.0592314117, -0.087708719, -0.0728336498, -0.0212278608, -0.0158449449, 0.0440583862, -0.0229610279, -0.0237193909, 0.0314243473, 0.0293623172, 0.029086737, 0.0410829224, 0.064977929, -0.0526660606, 0.0419156961, -0.0160210989, 0.0192205068, -7.473250e-02, -0.043597769, 0.00457868678, 0.0612143129, 0.0541577823, 0.0166926831, -0.0648401901, 0.0305079408, 0.0186451953, -0.0310516078, 0.0855607241, 0.0500953458, 0.0412256196, 0.0139360782, 0.0480405279, -0.00506355241, -0.0291594826, 0.0345122926, -7.34316127E-4, -0.054586973, -0.0565208904, -0.0531517118, 0.01710945, 0.0410962217, -0.0664584711, -0.0366655812, -0.0648157596, -0.0381830968, 0.0561244749, 0.0258864183, 0.00399687095, 0.0258837529, 0.0488519818, -0.0118808597, 0.0674589202, -0.0277227256, -0.00147507316, -0.022189647, -0.0726288781, -0.0360677242, -0.0455978438, 0.100523308, 0.0322792195, -0.0226546675, 0.0410195328, -0.079575792, 0.0996301993, 0.0626994595, -0.0508957878, 0.00274090795, 0.075684756, 0.00329506909, 0.00617444189, 0.0180242825, -0.0194147639, 0.0148859471, 0.0951632261, -0.0601097718, 0.0552765466], [0.00793428346, 0.0446556248, -0.0451363884, 0.00777499843, 0.0839412733, -0.0441362821, -0.0251238346, 0.0224016886, 0.0158395711, -0.0560890958, 0.00538659748, -0.0376543812, 0.0133063179, 0.104498141, -0.0358273946, -0.0344939157, 0.0477811582, 0.080158338, 0.0583189763, 0.0590522774, 0.0240409691, -0.055298917, -0.0649710149, -0.0581728593, -0.0319326781, 0.00491541391, -0.0328962579, 0.0325958394, 0.0286621973, 0.00315995258, -0.0561563484, -0.0193863269, 0.0942229703, 0.0128665147, -3.758740e-02, 0.0458346456, 0.00877805426, 0.0727467686, 0.0189391915, 0.0180872697, -0.0679397881, -0.0762185082, 0.01008025, -0.00652394677, 0.0324299596, 0.022165563, 0.0548835956, -0.0803708434, 0.0919647366, 8.738800e-02, 0.0537179634, 0.0185745377, -0.0144178076, 0.0305450056, -0.00228831102, 0.0128895696, -0.0349035375, -0.076940082, 0.0202532373, 0.108336776, 0.0137969498, -0.0808348134, -0.0255610645, -0.0729768276, 0.0439863093, -7.089290e-02, 0.0538938604, 0.0762252509, 0.0370066948, -0.0066514574, -0.0577408224, -0.0700652897, -0.0118353097, 0.0741816312, 0.0202764273, 0.0448817387, 0.0621589086, 0.0169700589, 0.00692903483, 0.0364376567, 0.0395943075, -0.00864455476, -0.0529136769, 0.068427287, 0.0649429038, -0.0288107768, 0.02124561, 0.0155109661, 0.00153806282, -0.0143174902, 0.0374738835, 0.029469721, 0.0142219979, 0.0114161586, 6.818210e-02, -0.0401402377, -0.0442585945, -0.0340118967, 0.0329221077, 0.0481362976, 0.0265398119, 0.0672811866, -0.005021472, 0.0561442859, 0.0296559725, 0.0828376412, 0.0471609905, 0.0391883664, 0.0632332563, 0.0486629531, 0.0269956347, 0.0453463718, -0.0444738567, 0.0381059721, -0.0286001805, 0.0522981696, -0.0366466604, -0.0254887752, -0.00691396091, -0.0704322159, 0.0230333582, 0.00604914734, -0.0274885222, 0.0797957331, 0.01295783, -0.033684358, -0.0245611686, -0.0790228695], [-0.00768914167, -0.0200078655, -0.060325671, -0.0666334555, 0.0246904492, -0.037292704, -0.0425266027, 0.0852907821, -0.0270145088, -0.0401724391, 0.0508976169, -0.0663058609, -0.0398241282, 0.0969127863, 0.0315111689, 0.0495161414, -0.0275820084, 0.0300445538, 0.0363796242, 0.0831732675, -0.0534953736, -0.0762636885, -0.0777731612, 0.00468937354, -0.0858359038, -0.0109238504, 0.0644397735, 0.0131087517, 2.273380e-02, -0.0200707242, 0.0612592548, -0.056936305, 0.00813574344, -0.0707063526, -0.0596789718, -0.032756649, 0.0376649201, 0.0716332123, 0.0191943534, 0.0850878209, 0.0687930956, -4.198940e-02, -0.0641403273, -0.0617740154, 0.0207270961, -0.0471265502, 0.0478185154, -0.0550981499, -0.026569007, -0.0146291535, -0.0166708734, 0.00764710736, -3.509050e-02, -0.0136027131, 0.0340854339, 0.0364677943, 0.0780060365, 0.00329995272, 0.00815224274, 0.0787726492, -0.00447730301, 0.0435246527, -0.0335613936, -0.0853086039, -0.0318289958, 0.0728170797, 0.0742196068, -0.00428474275, 0.00165913394, -0.059837658, 0.0217886604, -0.0532968789, 0.0935520157, -0.0434763432, -0.0164232645, -0.022852147, 0.0115783438, 0.067046009, -0.0105311852, 0.0284368396, 0.0460310392, 0.0896336659, 0.0166116636, -0.0754692778, 0.0390367843, 0.0117810145, 0.0785722434, 0.0506961681, 0.0774981529, -0.0789835229, 0.0194917414, -6.732530e-02, -0.054988455, -2.267460e-02, -0.046178706, -0.0569744185, -0.0675753951, -0.0665920898, 0.066444084, 0.0272922516, -0.0242377743, -0.0772417187, 0.0590009578, -0.0138830598, -0.0440421402, 0.0781953632, 0.0101418914, -0.0403667577, -0.0227475632, -0.0324091911, 0.0535659045, -0.0103483424, 0.0587289706, 0.0234681685, -0.100251459, 0.00267926301, 0.00143063185, 0.0239201616, 0.0762863085, 0.0473978855, 0.00152077654, 0.0528879315, -0.0572945885, 0.0160880052, -0.0195194669, 0.0579736419, -0.0838049724, -6.789100e-02], [0.0715481266, 0.0456798784, 0.0616742522, -0.00987797882, -0.00948014203, 0.0767799839, 0.0183187388, 0.00701163942, -0.0322550349, -0.0497889705, -0.0115540652, 0.0128002744, -0.0500658974, 0.00367061654, -0.066315271, 0.0798897147, -0.0345006846, -0.0130202686, -0.0661784634, 7.500950e-02, 0.0633248687, -0.00353712775, -0.0200201459, -0.0373804234, 0.050866954, 0.0624009184, -0.0361129493, -0.00267385272, 0.0552420355, -0.0301337577, 0.0227640849, 0.0608198382, 0.112546384, -0.0317163281, 0.0583069511, 0.0207349937, 0.00595858926, -0.00825974717, 4.154970e-02, 0.0348341167, 0.0467159487, 0.0178081803, 0.0115655456, -0.0558929592, 0.0482337736, 0.0176121201, 0.0810505747, 0.0561884902, 0.0247453582, -0.0288230032, -0.0216816459, 0.0570615493, 0.0284663532, 0.0551468581, 0.0149525478, -0.00983876548, 0.0588430725, -0.0044429875, -0.0366205052, 0.0604130812, -0.0709598586, 0.0760189369, 0.0182246156, 0.043019712, 0.0493279174, -0.04523734, -0.0612015501, -0.0108746402, 0.0692293718, -0.0502294563, 0.0872963592, -0.0410308689, 0.0393846743, -0.00292696664, -0.0394599475, 0.066991955, -0.0144270882, -8.085270e-03, -0.0397447161, -0.0126206232, -0.0021990384, -0.0262980927, -0.0304222535, 0.0309402626, -0.0219589453, -0.0697385073, 0.0111313686, -0.0687772185, -0.0112754321, 0.0312822796, -0.0352677479, -0.0102380197, -0.0164444838, 0.0714444071, 0.00284782588, -0.00394509174, 0.0468495712, 0.0463162027, -0.069353342, -1.627850e-02, 0.0675706714, -0.0711120144, -0.0689593703, -8.809620e-03, 0.0278087426, 0.00711261202, -0.0134366313, 0.00663953228, 0.0444009341, -0.00553146703, 0.0213271398, -0.0383979566, 0.0524548218, 0.0117372079, -0.103330217, 0.099659428, -0.0469881296, -0.05014617, 0.0940464064, -0.053001713, -0.0634944886, -0.00666041113, 8.530830e-03, -0.0314198509, -0.077906154, 0.071921967, 0.0467580669, 0.0123391785], [0.0269811768, 0.054630693, 0.0492308699, -0.0233643483, 0.0744854509, -0.0270029977, -0.0315773636, -0.0495527312, -0.00777566712, 0.00922844466, -0.0163287465, -0.0669093803, 0.0352763832, 0.00561339874, -0.0812350661, 0.0316185467, -0.0258418545, -0.0078224577, 0.0602469258, 0.026796829, -0.0212195329, -0.0151216928, 0.0285620485, 4.51687054E-4, -0.00122454704, 0.041000817, -0.0560056046, -0.0604339242, 0.00480301213, -0.0318721123, -0.0215323083, -0.0149157261, 0.0267508272, 0.0476072207, 0.0576463826, -0.0262120515, -0.0540793873, 0.0366949476, 0.0157696158, -0.00747686392, 0.0760523528, -0.0105556985, 0.0384120569, 0.105336785, -0.0574351177, -0.0425886363, 9.673040e-02, -0.0155431814, -0.0176679213, -0.0345748216, 0.0107767945, -0.068764396, 0.00144347525, 0.0702623054, 0.0649406463, -0.00738460756, 0.0171880089, -0.00842075516, 0.0167995635, 0.0372494869, -0.0411106646, 0.0666633621, 0.060484346, -0.0379073024, -0.0294224732, -0.0346702151, 0.00693376223, 0.0451251678, -0.021558838, -0.0264739245, -0.0154116685, 0.0166636109, -0.018690275, -0.06918177, 0.0730574429, -0.0428953692, -0.0513043068, 0.0235979315, -0.0253442302, -0.0459132455, -0.0379795209, 0.00449381908, 0.0854270383, 0.029929718, 0.101398207, 0.0519897044, 0.0116418079, 0.0299874172, 0.0898400619, -0.0582705326, 0.0122146988, 0.0390660912, -0.0187356677, -0.0293511841, 0.0101235704, -0.0484899431, -0.0734146312, 0.0163634364, 0.0455841087, 0.0581509955, -1.591900e-02, -0.0160225406, -0.0815801545, -0.0105720246, 0.00571916159, 0.0296755657, -0.0480828546, 0.0716928691, 0.0183436777, 0.0723625794, -0.00813929457, 0.0521044545, 0.057560727, -0.0366885252, 0.00131194817, -0.0221151486, 0.078535743, -0.0722552165, 0.0457648337, -0.0317150094, 0.0274754856, 0.0365080461, -0.0324009433, 0.0599968247, 0.053329505, 0.027020527, -0.0346051715, -0.0792582333], [0.0388026163, 0.0274967924, -0.0484291799, -0.0608653501, 7.519740e-02, 0.0365928747, 0.0138486009, 0.0657805279, -0.0407696813, -0.0534915403, 0.0355009213, -0.0110876765, -0.0183346029, 0.0295291934, -0.0825557485, -0.0717024952, 0.00342292804, 0.0340303816, -0.0476010442, -0.00821489468, 0.00371371838, -0.0376960263, -0.066079475, 0.0302481987, -0.0688902065, 0.024147084, 0.0822391659, 3.327090e-02, -0.0483087748, -7.471140e-02, 0.0544202626, -0.0286495406, 4.896920e-02, -0.00495519722, 0.00718062464, -0.0528259464, -5.652570e-02, 0.00608428521, -0.0464999974, -0.0172498897, 0.0695737079, -0.0220342521, 0.0244362168, 0.00611931458, 0.0789210498, 0.0731581524, 0.0998399555, -0.0226146113, 0.00736927288, -0.0198426116, 0.0112611651, -0.0247119237, 0.0771959648, -0.0298277419, 0.0434750058, -0.0117164841, 0.0520984046, -0.041065827, 0.00387917599, -0.00384175661, 0.00898316316, -0.0254147574, 0.0305610429, 0.0455341563, -0.0251496267, -0.0474223197, 0.0249077808, -0.0175372586, -0.0518337823, -0.0483978428, 0.0749145523, -0.00328932423, 0.0284750499, 0.063371636, -0.0383206494, -0.0383700132, 0.0955912545, -0.00694444915, 0.0259725843, -0.0640724301, 0.0365462676, -0.079971686, -0.00873540528, 0.0366294719, -1.851770e-02, 0.0239913221, 0.049162291, -0.0856143087, 0.0631696135, -0.0693191662, 0.0243899412, 0.00155110541, 0.0753325671, 0.053187158, 0.0166238248, 0.020976793, -0.0337469503, -0.0142883835, -0.0103453062, 0.0375792384, 0.0318227783, -0.0454318672, -0.0208448563, -0.0476474538, -0.0152969798, 0.0400449224, 0.020681696, 3.347530e-02, 0.046521578, -0.0319125392, -0.022144882, 0.0451172292, 0.0217903163, 1.66314087E-4, -0.0620315484, 0.0939436182, 6.655360e-02, 0.0742076263, -0.021030575, -0.0350732692, -0.0732275769, -0.0519297235, 0.0807910487, -0.0806673467, -0.0484434776, -0.0574402921, 0.00623662956, 0.0343253054], [-6.566380e-02, -0.0391003154, -7.03023106E-4, -0.0935654491, -0.0288968664, 0.0271849334, -0.0550998785, -0.0468702875, 0.0474120155, -0.00543752732, -0.0830740705, 0.0167615227, -0.0244704783, -0.0662585869, 0.0162533652, -0.0789735242, -0.04882624, -0.0457039848, -0.0218667556, -0.0286051854, 0.070517309, -0.00535385683, -0.00999399647, 0.0490414463, -0.0636268109, -0.0471157953, 6.627200e-03, 0.0346513093, 0.00721312547, 0.0194153339, 0.0739169717, 0.0108296452, -0.0311055519, 0.00331271579, 0.0124353236, 5.23518247E-4, 0.0175074805, 0.0337736979, 0.0766291618, -0.00192511664, 0.0392514132, 0.0823803097, 0.0539924875, -0.0428573266, 0.00396618946, 0.0796324536, -0.00355224637, -0.0533716977, 0.0522771813, 0.0577395447, -0.0341738313, -0.0537372492, -0.0563553199, 0.0398826338, -0.0674683228, -0.0794030949, -0.0181808397, -0.0559760332, -0.0224555954, 0.0352814049, -0.0202312768, 0.0722655728, -0.0624986849, 0.0655272603, -0.0480628498, -0.0521438941, -0.0583150201, 0.0803487822, -0.0514003038, -0.0314448811, 4.172440e-02, 0.028792968, -3.845750e-02, 0.0592563227, 0.0679249614, -4.65508376E-4, -0.0143459802, 0.0619144365, -0.0119031696, -0.0809158608, 0.049542781, 0.0412646569, -0.015713444, -0.0302175432, 0.101900898, -0.0412440225, 0.00637765741, -0.075143978, 0.0590878427, 2.637460e-02, 0.0798991397, -0.071643114, -3.573460e-02, -0.0126058543, -0.0352015942, 0.0610381737, 0.0387792699, -0.0659524724, -0.0749890059, 7.94960215E-5, 0.0576147512, -0.0742083564, 0.0498437174, -0.00877137295, -0.0129054338, -0.0298034456, 0.0755190551, 0.050001014, -0.0500115119, 0.0791469067, -0.0549679883, -0.0526668876, -0.0482216142, -0.00606014905, -8.632660e-02, 0.084554851, -0.0359482095, -0.0534953922, 0.0568887852, 0.0570832565, -0.0581371039, -0.0562007055, 0.0975218713, 2.414440e-02, 0.0652918145, 0.0228863899, -0.0807670131, 0.0191676971], [-0.0263208337, 0.0527755506, -0.00446043722, 0.0467676334, 0.0631953478, -0.0642803088, 0.0542740971, -0.0369029269, -0.0579412431, 0.0192511138, 0.0576388538, 0.00486378698, 0.0708321109, 0.0481749326, -0.0308899153, -0.0762956291, 0.0500550605, -0.0115357805, -0.0858046635, -0.0700406805, -0.0411178209, -0.0275441576, 0.0185679905, -0.0171793196, -0.0603513084, -0.0324457549, -0.0579601526, 0.0151323434, 0.0617266037, 0.0407472663, 0.0242826398, 0.0638127923, 0.0794945955, 1.830640e-02, 0.055982098, -0.0468406193, -0.0569008552, -0.0706802383, 0.0799753144, 0.0448007621, -0.0638283491, 0.0201776549, -0.0184353329, 0.0765842274, 0.0104665356, -0.027680669, 0.068902187, 0.0823371931, -0.0437909327, -0.0339566842, 0.047538396, -1.906520e-02, 0.0197053067, 0.0618617311, -0.0474373214, -0.0542646721, 0.0188677479, -0.0621344373, 0.0544617251, 0.0701055452, -0.0815492123, -0.042180568, -0.0444441177, 0.0758934468, -0.00877469685, 8.11078789E-5, -0.0569939315, -0.0768243223, 0.0719602928, 0.00965985096, -0.0390363075, 0.0275518615, 7.250240e-02, -0.088554427, -0.0148776611, -0.0520625487, 0.0817703604, 0.03743992, -0.0410334021, 0.0329384506, 0.00375475478, 0.00634961436, -4.91623883E-4, 0.0708972067, 0.0923889949, -0.0726497248, 0.0415365919, 0.00738171767, 0.0104308538, -0.0188443288, 8.895600e-02, 0.00854334793, 0.0360948816, 0.0713306963, 0.0784093961, 0.0479213484, 0.0604616627, 0.0801922232, -7.57416419E-4, 0.0274175052, 0.0254003871, 0.0597063266, -0.0448258482, 0.0233538505, -7.499460e-02, -0.0106864711, 0.00393899903, 0.0539648682, 0.0818682164, 0.0253789444, -0.0842323154, -0.0656101108, -0.0523046628, -0.0711462647, -0.0184133351, 0.0223017745, -0.0156395081, -0.0828851312, 0.0529516563, -0.0294876304, -0.0483301803, 0.00670563197, 0.0653968081, 0.0660425797, 0.0630737916, 0.0513543561, 0.0536262877, -0.0275869668], [0.0243695509, -0.0787915289, -0.074937731, -0.0442117639, -0.0446664765, 0.0617075413, -0.0565508306, -0.0141739724, -0.0413642228, -0.0580884144, -0.0541434549, -0.0727051198, -6.02852495E-4, 0.0592606626, -0.0608750507, 0.0535916649, 0.0577938892, 6.67271451E-5, -0.0356915407, 0.0262768771, -0.0768681541, 0.0368143059, -0.0125804823, 0.0715872869, -0.0676565841, 0.00208758865, 0.0553909428, 0.0537636578, -0.0338520147, 0.0446214192, -4.393570e-02, 0.0196356345, -0.0626798719, 0.0502389111, 0.066941008, -0.0138970362, 0.0375694223, -0.0605234914, 0.0226690937, -0.00965350587, 0.0225995611, -0.0712726712, 0.0296276938, -0.02553319, 0.00243825791, 0.00901953503, 0.0641635209, 0.0443281196, -0.0471905805, -0.00685948646, 0.038328521, -0.0699547306, 0.0152570661, 0.00397450104, 0.0477998964, -0.0447039083, 0.0122816991, 0.0715767667, -0.00639064144, -0.00149170961, -0.00716563314, -0.05670752, -0.0678451136, 5.526920e-02, 0.0601519309, 0.0130566154, 0.00921598542, -0.0482810736, -0.0657537133, 8.2561915E-4, -0.014475056, 0.0124222264, 0.0788927972, -0.0235484298, 0.0418140851, 0.0522787757, -0.0275910795, 0.00936564151, 0.0459255315, 6.914000e-02, -0.00420141453, -0.0534306504, -0.0697571486, -0.0346464701, -0.0599364787, 0.0116451075, 0.0273101237, 0.067501761, 0.0425519757, 0.0025682454, -3.509390e-02, 0.0188610982, 0.0348356627, -0.00140651444, -0.00376714976, 0.0690183192, -0.0434827916, 0.0692174211, -0.0658858269, -0.0578268394, -0.046554774, -0.0740426108, 0.00110536476, 0.0552376062, 0.0415494293, -0.0309467521, 0.00355396117, -0.0241211876, -0.0420787111, -0.0167793222, 0.0701984391, -0.0535025373, -0.0252096262, -8.04541924E-4, -0.0284206364, -0.0508570224, 0.0718427673, 0.0242577046, 0.00107502146, 0.0531114452, -0.0717731938, -0.00813499745, -0.0511503965, -0.0622712895, -0.0564573593, 0.0425398238, -0.0575446337, -0.00481415307], [0.0761448592, 0.04908593, 0.0402943157, -0.0103165526, 0.0146568175, -0.0233434327, 0.00911011081, -0.0488849096, 9.365140e-03, -0.0448746458, -0.0216113944, -1.430380e-03, -0.0732368231, -0.0243496392, -0.0609428398, -0.00317582069, 0.00605599768, -0.0159653127, -0.0258778576, 0.028817324, -0.0160174444, -0.0748124272, 0.0618406087, 0.0198864099, -0.0502320826, -3.734510e-02, -0.0577614121, 0.0294661485, 7.9040497E-4, -0.0765362531, 0.027698081, 0.0511759855, 0.0799482912, -0.048576571, 0.0340953134, 0.0529778302, -0.0363718271, -0.0221148431, 0.0541008376, 0.035776224, 0.0707101226, -0.00417211093, -0.0243601184, 0.0780874491, 0.0760586113, 0.03400838, -0.0578348078, -0.00405282201, 0.0228007119, -0.0639752224, 0.0579016469, -0.0679041296, -0.0491439812, -0.00792323239, -0.0559960715, -0.0714349151, 0.0695408657, -0.0113702491, -0.0272530429, -0.0440847762, 0.0275478326, 0.0363321677, 0.0778586044, -0.0336497203, 0.0566713959, 0.0646775141, -0.021645233, -0.048315268, 0.0318562649, 0.0584128536, 0.00903736427, 0.0687824488, -0.0574832819, 0.0493954979, 0.0171626136, -0.056220036, -0.00841191224, -0.00867714453, 0.0444630831, 0.0588877387, 0.00637415191, -0.0817296281, -0.0478279702, 0.0348533764, 0.0821037441, -0.0747746304, 0.0314827412, 0.0255475324, 0.056023445, -0.0607968383, -0.0146795223, -0.0309358835, -0.0461203307, -3.430920e-02, 0.0499018207, 5.935390e-02, -0.0134416381, 0.0135843866, 0.0212106202, 0.0675540194, 0.0641176328, -0.00176620565, -0.0760497674, -0.0364002325, -0.0509948693, -4.578350e-02, 0.0204868577, 5.541660e-02, -0.0123685868, -8.01871589E-4, 0.066332534, 0.034853071, 0.0666589141, 0.0264336858, 0.0738334953, -5.305370e-02, -0.0515569486, -0.079858981, 0.052470576, -0.0205210838, -0.0806503966, 0.0110494485, 0.0727629289, -0.0703027099, -0.05904245, 0.00476486143, -0.0493050702, 0.0593321696], [0.0174170639, -4.634380e-02, 0.0542531759, -0.0818325877, 0.0441248119, -0.0697550475, -0.0730047598, -0.0749784708, 0.0307690278, -0.0195919108, -0.023823807, 0.0744766146, -0.0199999344, 0.0262677874, 0.0611542352, 0.00147333345, 0.00789901149, 0.0431903824, -0.0381092131, 0.0395315886, -0.028327411, -0.0479005352, -0.0530039333, 0.0534521192, 0.0623204447, -0.00971597154, -0.0429887511, -0.0672859699, 0.0216196664, -0.00912688393, -0.00170629204, -0.0181956384, 0.0513406843, 0.0738227889, 0.0309026074, 3.683520e-02, -0.022652965, -0.0609073862, 0.0645846799, -0.00297880732, -0.0541266501, -0.0291318502, -0.0714073256, 0.0451414287, 0.0718110725, -0.032340806, 0.0142999394, -0.0499467403, 0.0134914126, -0.0303022861, -0.00413683103, 0.00991040281, 0.0248384643, -0.020987032, -0.0304033179, 8.6924748E-4, 0.0122738136, 0.0295893122, -0.00408609072, -0.0685924292, -0.0793940052, 0.0619614459, -5.05385164E-4, -0.00965419877, -7.99966801E-4, 0.0309464969, -0.0622049421, 0.0532587133, -0.0558656752, 0.046143502, -0.0273527317, -0.041380696, 0.0480753146, -0.0569345616, 0.0327917039, -0.0732912496, 0.00728484383, -0.0234292503, -0.0606791973, 0.0726382509, 0.0488673821, 0.0209623892, 0.0639035553, -0.0649697706, 0.063089177, 0.0198692773, 0.0455581546, -9.137440e-03, 0.0494361669, 0.0254181325, -0.00680347718, 0.0746210665, 0.0271502342, 0.0806863159, -0.0339636132, 0.069939293, 0.0348991156, -0.0469080396, -0.00345294503, 0.0230475385, -0.00813054852, -0.0177119356, -0.0101283398, 0.0454897545, -0.0393847451, -0.0372119211, -0.0516935401, -0.0461203232, -0.0483595356, -0.0758378953, -0.0475793704, -0.0677148923, -0.0252878219, -0.0611208417, 0.0473466776, 0.0481048152, -0.048300039, -0.0753606483, -0.0392646566, 0.00809054915, 0.0165343601, -0.0396555215, -0.019035846, -0.0196900461, 0.0422308072, 0.0667829439, 0.0679930896, 0.070409961], [0.0023033109, 0.0650223643, 0.0352913477, -0.0374477208, 0.0587905161, -0.0352494679, 0.0547977835, -0.0562526658, -0.0273881853, -0.0141082527, -0.00110905862, -0.0765030831, -0.0359303616, -0.0347100236, -0.0780960172, 0.0404443219, 3.172800e-02, 0.0572557338, 0.0246947184, 0.0162314307, -0.00526792509, -0.0580704585, 0.037309017, -0.0702429935, -0.012697489, -0.0289212149, 0.0301005971, -0.0749090239, -0.045913361, -0.0412766598, 1.16166135E-4, 0.0417349041, -0.0461377613, 6.951250e-02, 0.00401686924, 0.0414303467, 0.0395382941, 0.00597835472, 0.0211444758, 0.0528535321, 0.0529611558, 0.0649571642, -0.00559425866, 0.00975218881, 0.0641363561, -0.024997633, 0.00346657494, 0.00432561385, -0.0652652234, 0.0267287232, -0.0697425827, -0.0349069051, -0.0493444465, 0.04425928, 0.0315820873, -0.0476725288, 0.00546309026, 0.0268593803, -0.0349757336, 0.0343678147, 0.0121330786, -0.0248277467, -7.024970e-02, 0.0228960682, -0.0322037525, 0.00733869756, 0.072057344, -0.0642538816, -0.0626434385, -6.578510e-02, 0.0363702141, -0.0474947728, -0.0735081211, 6.656380e-02, 0.0738014355, 0.0109220343, 0.0370858349, -0.0129580926, -6.898340e-02, 0.0366427824, 0.00646761945, 0.0130339377, 0.0346099436, 0.018597357, -0.0416522659, -0.0750223771, -0.071312584, -0.0302920658, -0.00934811122, 0.00102858956, 0.0486177318, 0.0301623121, 0.0140806986, -0.0557055622, -0.0169046465, 0.0384204686, -0.0670946911, -0.0494292304, -0.0247763395, -0.0390711166, -0.0537517443, -0.0316403396, -0.0767774507, 0.0071542114, -0.0675417408, 0.075373292, -0.0321370773, 0.0256426539, -0.080913119, 0.0484058373, 0.0587949157, -0.0679040626, 0.0643179044, 0.0610079691, 0.00122983684, 0.0783643872, -0.0106703397, -0.0702369735, 0.0656384602, -0.0469144769, 0.0347113684, -0.0263024531, 0.0127751343, -0.0730120763, 0.0502073541, 0.0155698899, -0.0680835098, -0.0346433707], [0.0415146202, 0.0163557269, -0.0677049905, -0.0801415368, -0.0791741982, 0.0596426278, 0.0641527325, -0.0274928715, -0.0508816168, 0.0279060379, -0.0717662573, 0.0036858886, 0.0606812611, -0.00558137661, 0.0442182571, 0.0117995506, 0.057493262, 0.0407242179, 0.0489571728, 0.0133409305, -0.0245109666, -0.0481658578, 0.00566488132, -0.0195651371, 0.0516310446, 0.0768568367, -0.01624896, 2.245730e-02, -0.0400589965, -0.00617664168, 0.0689887181, -0.0700326859, -0.0798782706, 0.0521762483, 0.0631679446, -0.0760130957, -0.0345304869, 0.0309519898, -0.0191290472, 0.0255648326, 0.0536281392, 0.0349353254, 0.0010362335, -8.2514016E-4, 0.0198686309, 0.0616718828, -0.0300831888, -0.0255146082, 0.00798886176, -0.0603532642, -0.0432366766, -0.074043192, 0.0762679055, 7.532890e-02, -0.0484887399, -0.0340878852, -0.0801423937, -0.0607623309, -0.0650981069, -0.0439515337, -0.0295685604, -0.0568765886, -0.00457322318, 5.655320e-02, -0.00827895663, -0.0479244664, 0.0474129468, 0.079059787, 0.036333371, -0.0782676264, -0.0154752238, -0.0102135418, -0.0339403488, 0.0683276132, -0.0739591121, -0.0483314767, 0.0114992214, -0.0568157658, -0.0548786819, 0.01092456, 0.0232634339, 0.0480170697, -0.0611035191, -0.0249395985, 0.0626242458, -0.0339080915, 0.0346576273, -0.068497844, -0.0575786605, 0.0154019892, -0.0115271714, -0.072525233, -0.0276837647, 0.039227739, -0.00314747565, -0.0106572639, 0.0296200812, -0.0104642455, -0.0403230228, 0.076495707, 0.0799293443, -0.0424326137, -0.0209730752, -0.0190993641, -6.952640e-02, 0.0576593652, -0.0477040112, 0.0654939488, 0.0222283415, 0.0781616345, 0.0733469054, -0.0772474781, -0.0606351234, -0.0579563156, 0.015494572, -0.0484322496, -0.0527692512, -0.0627995655, 0.0583946258, -0.0273481105, -0.0368148237, -0.0741653591, 0.0645798742, 0.0100829974, 0.0407810509, -0.0561489873, 0.0623484887, 6.485850e-02], [0.0665352717, 0.0298786238, -0.015923135, -7.171160e-02, -0.0282710269, -0.0059202686, -0.0711607859, 0.0131599829, 2.081240e-02, 0.0256417543, -0.00462003797, -0.036893107, -0.015681468, -0.0164794028, 0.0799140408, 0.0692883357, 0.0500424877, 0.0274281502, 0.0678156689, -0.0547909513, 0.0340298563, -0.0251109749, 0.0350791961, -0.0489986315, -0.0454202518, -0.00464829057, 0.0586657152, 0.038841553, -0.0654921681, -0.0360921919, 0.0239919871, 0.00120601058, -0.0731217265, 7.671980e-02, -0.0371816903, 0.0118645728, -0.0712593719, 0.00636345893, 0.0381617919, 0.029344365, 0.0720520839, 7.020980e-03, -0.00701653212, -0.0621479973, -0.0283890702, 0.0639359131, -0.0353625752, -0.0713346377, -0.00151418895, -0.0645088404, -0.0705955476, 0.00288413465, -0.062998414, 5.764950e-02, -0.0138769299, 0.0686154589, 0.0719891414, -0.0796035602, 0.0359596461, 0.075740464, -0.0694499835, 0.0268714949, -0.0169343352, -0.0592623353, 6.46747648E-4, -0.0578131825, -0.033845447, 0.0318744704, -0.0242809579, -0.0348787308, -0.0764937102, -0.0795391798, 0.0554273352, -0.0323591046, -0.0684870556, -0.0759343058, -0.0629799664, -0.0465231165, 0.0493214056, -0.0524235368, -7.492400e-02, 0.0548373833, 0.037194103, 0.0560060367, -0.0391459614, -0.0365918539, 0.0520517603, 0.0240201801, -0.00874879956, -0.0611543953, -0.0680078864, 0.0558281615, -0.0683713108, 0.0369552374, -0.0357034914, -0.0714349225, -0.0498958118, -3.341130e-02, 0.0665909722, -0.0417532474, 0.0788603052, -0.0374243297, -0.0275533833, 0.0754752234, -0.0324867181, 0.0556706861, 0.0321631506, -0.0159469619, 0.00360481441, 0.0295147672, 0.0389083922, -0.046918314, -0.0523372144, 0.00526785105, -0.078287378, -6.263770e-03, 8.64960253E-4, 0.00398641825, 0.0747561678, 0.0240463838, 0.0600590035, 0.0516798422, -0.0536177754, 0.01760526, -0.0398326442, 0.0169883296, 0.0613737628, 0.0659414306], [-0.0363552496, 0.0732969716, 0.0201891884, -0.0592052117, -5.928860e-02, -0.010180898, 0.0733344629, -0.0250308961, -0.028516449, -0.0738547817, 0.0420017615, 0.0210786313, 0.0380454957, -0.00944053382, 0.0301238671, 0.0608125255, -0.0196559317, -0.0599263385, 0.0300442502, -0.0336677656, 0.0709797218, -0.0607885346, -0.0662063733, 0.0606500283, -0.0457310565, 0.00205931813, 0.0196301937, 0.00825281441, -0.0778686404, 0.0613571778, 7.290720e-02, -0.0205999129, 0.00693716854, -0.0782539173, -0.0371587947, -0.0142297372, 0.00626036525, -0.0808431804, 0.0751805231, 0.0131893754, -0.0223415382, -0.0443093032, 0.0495741591, -0.0785913169, -0.0107185245, -0.0282567963, -0.0751268938, 0.0503880903, 0.0665001795, -0.0201513581, -0.00227764249, 0.0229728743, 0.03885144, 0.0439268127, 0.0568019822, 0.076444082, -0.0526159182, 0.00287361443, -0.0469813161, 0.0751753524, -0.0188537799, -0.0487094447, 0.0635977164, -0.0662802458, -0.0168514922, 0.0479838774, 0.054175131, -0.0191464461, -0.0274139345, 0.0561312661, 7.83607363E-4, 0.0268105045, 0.0605685487, 0.0725719407, -0.0737318322, -0.0283308625, -0.0423353873, -0.0532976091, 0.0217122287, -0.00761033594, -0.0637151748, -0.0208753459, 0.0245800242, -0.00564803928, 0.0418702438, 0.0106165558, 0.0598691329, 0.0567527786, 0.0778542086, -0.00164856762, 0.073293604, 0.0562501177, 0.0537581071, 0.0699244514, 0.0460839048, -0.00544475764, 0.029852435, 0.0618900433, -0.0651582554, 0.0664466247, 0.0410277471, -0.0263665728, -0.0478207916, -0.0746980533, -0.0422522128, -0.0318977125, -0.0732912496, 0.0124547556, -0.00465648621, -0.0656999424, -0.0516504906, 0.0277973786, 0.0304759443, 0.0318181366, 0.0756267384, 0.0104274601, -0.0722035244, 0.0789634212, 4.129170e-02, 0.043624267, -0.0632277876, 0.0227090791, 0.0274637863, -0.0527790189, -0.0312219374, -0.0543162376, -0.0180451646, 0.0797279402], [0.0285053141, 0.00673020817, 0.0422803424, 0.0748756751, -0.0292764399, -0.00562784914, 0.0466131717, 0.0320668854, -0.0270847157, -0.061183814, -0.0439970493, -0.0338082388, 0.0683165938, 0.0422440916, -0.0555553772, -0.0589403324, 0.0447754711, -0.0206456706, -0.0498560145, 0.0339743569, 0.0438298136, -0.0493043289, 0.0268953145, -0.0352352373, 0.0280526802, -0.0512667522, 0.0787922591, 0.0172460936, -0.0609758087, 0.0806310326, -0.0176261477, 0.0417700261, -7.865150e-02, 0.0123066008, 0.0376880914, 0.0628046691, -0.059737727, 0.0596397817, -0.0289384425, 0.0644931272, 0.025871817, -0.00337551185, 0.0280672871, 0.035895478, 0.0765618979, 0.0468020812, 0.0216132049, 0.07042218, -0.0504015349, -0.0105934776, 0.0806755275, -0.0581389666, 0.0694124996, -0.0543988645, -0.011868787, 0.00524214096, 0.0225640833, 0.040770717, 0.065452151, -0.0375064127, 0.0193978827, 0.050753586, 0.0206633583, -0.0446037129, 0.0505528972, 0.0149006015, -0.0520462319, -0.0621768198, 0.0180667788, -0.0457400121, 0.0471138135, -0.0801323875, -0.0194171164, -0.0456361547, -0.0430816524, 0.0122655025, 0.0125220465, 0.0444592759, 0.0728515536, 0.00554117514, -0.00708853919, 0.0383778438, 0.0118727395, -0.0525361076, -0.028330164, -5.323820e-02, -0.0434678569, -0.0661644712, 0.00136696803, 0.0293052699, -0.0775165185, -0.00754970592, -0.00736827822, -0.0786228254, -0.0311032012, 0.00992069486, 0.0806540846, 0.00656833081, -0.0487219691, 0.0805681869, 0.0430144817, 0.0643005446, -0.0572147258, 0.0109578129, 1.918370e-02, -0.00626164814, 0.0670840368, -0.0703635663, -0.0323093496, 0.0415999182, -0.0163941178, -0.034369614, 0.0689163953, -0.0387971103, 0.0134832412, 0.0188926514, 0.0379683599, -0.0330820046, 0.0502104536, 1.49602085E-4, -0.0737288147, 0.0746523067, -0.0747555047, 0.0385667421, -0.0164642353, 0.0212862473, 0.0389794335, -0.0570379943], [-0.0403843857, 0.0567035452, -0.0628072619, 0.030498581, 0.00196320331, -0.00397638557, -0.0120257121, 0.0240121577, 0.0594570599, -0.0733168423, -0.0550468937, -0.0729302391, 0.0488941334, -0.0335477181, 0.0798118039, 0.00477319583, 0.0732821226, 0.0124591133, -1.074730e-02, 0.0789789482, 0.0512448773, 0.0184038896, 0.0222673472, -0.0577636547, -0.0445892066, 0.0792173668, -0.074264355, 0.0553472415, 4.219100e-02, -0.0261099227, 0.0683656632, -0.0334213078, 0.0305760968, 0.0436169617, -0.02707614, 0.0371593088, 0.0547497831, -0.0346583948, 0.0736515671, -0.00116177893, -0.00174180698, 0.0210423041, -0.0480858125, 0.00119614089, -0.0520976894, 0.0626212358, -0.00656618038, 6.298520e-02, -0.0139597571, 0.00906343851, 0.0343362838, -0.00558930589, -0.0796744748, 0.00525941094, 0.0702177286, 0.0377970114, -0.0142508643, 0.051487647, 0.0102713993, -0.0638776123, 0.0732758865, -6.919290e-02, 0.0534087643, -0.0208384525, 0.0288747866, 0.00320393406, -0.0770575106, 0.0395572521, -0.0101002324, 0.0121757127, -0.0765401348, -0.0153540717, -0.04907123, -0.025749132, -0.0351901092, -0.0424494408, 0.0274141133, 0.0262608714, -0.0317187496, 0.051371593, -0.0687983334, -0.0606677085, 0.0143872453, -0.0301080886, 7.574990e-02, -0.0257279929, 0.0339990743, 0.0223136526, 0.0306429733, -0.0126719391, -0.0674741343, 0.0642035678, 0.0579532124, 0.0103148799, -0.0678265765, -0.0394337289, 3.151760e-02, -0.0208011325, -0.0117298439, -0.00746426499, 0.0118603986, -0.0331543572, -0.0331172124, -0.0787940323, 0.0328569598, -0.0143852215, -0.0244090222, -0.0679745525, -0.0301409923, -0.0134482747, 0.0684107766, 0.0210919417, -0.0342693254, -0.0365479551, 0.0393148921, 0.0515121706, 0.00269538793, 0.00123397424, 0.0156335179, 0.0105410237, -0.0693560913, -0.0388676561, 0.0187348258, -4.147320e-02, 0.0678972676, -0.00234807492, 0.0374296233, -0.0309144258], [0.0226819366, 0.0456305817, 5.692380e-02, 8.259450e-02, -0.0769994929, -0.0426934436, -0.0303400755, 0.0788224264, -0.00449159369, -0.0496808477, 0.0226616301, 0.0748130977, -0.0777963623, -0.0730368569, 0.0445906632, 0.0247051511, -0.0175888073, 0.0521877632, 0.0540200248, 0.0167706721, 0.0230059866, -0.0388089307, 0.0376169197, -0.0226801876, -0.0318319127, -0.0308202412, 0.074389182, -0.0674317256, -0.00860175397, -0.0539444946, 5.28537261E-4, -0.0426002033, -0.0108844051, 0.00682828296, 0.0149191823, 0.0339796767, 0.0432144701, 0.00139651098, -0.00904755946, 0.0456421822, 0.0652688443, 0.078061685, 0.0584012605, -0.0563229173, -0.0368522778, 0.0398668051, -0.0619794763, -0.0331855379, 0.0726611093, -0.072058782, -0.0417912044, 0.0756328255, 0.0622648448, 0.0541252904, -0.0360353664, 0.0127497585, -0.00155844307, 0.0358143188, 0.047307767, -0.0379327685, 0.0239511412, 0.0143725937, 0.0122891869, -0.0459507667, 0.0393650122, -0.0388478749, 0.0682862102, 0.0505283065, -0.051293727, 0.0110400487, 0.0529282466, -0.0147040477, -0.0781014561, -0.0790935382, 0.019081125, -0.0268388931, 0.0252457187, 0.0299387556, 0.0196437929, 0.03499097, -0.0239358712, 0.0726438537, 0.0254807435, 0.0563946627, 0.0644807145, -0.0611986034, 0.0645197257, -0.0519945547, 0.0685888826, -0.0326782912, -0.0457110628, -0.0607858486, 0.0619579703, 0.0303723738, 0.0576295517, 0.0426971503, 0.0365205444, 0.0429015085, 0.0745845065, 0.02060817, 0.0588464774, 6.96747797E-4, 0.0556038097, 0.078171283, -0.0326065198, 0.0745366961, 0.0408935919, 0.0722766146, -0.0685247257, -0.0216440577, 0.0379243419, 0.0682592615, -3.836880e-02, 0.0535390824, 7.410370e-02, 0.0367235169, 0.0204786025, 0.0321203284, -0.0569983311, 0.0798655375, 0.0155919371, -0.0703625455, -0.0177701339, 0.0679269955, 0.0806197747, 0.0152185233, -0.0255219545, 0.0477297865], [-0.03049054, 0.0878673345, 0.0555123426, 0.0298229512, 0.0082355421, 0.0616207309, 0.0763572156, 0.0815344825, 0.0157416947, 2.86679424E-5, -0.0542908087, -0.0700520203, 0.0571033247, 2.51019897E-4, 0.0730024278, -0.0301182475, 0.0386830121, 0.00877930503, 4.344690e-02, 0.0325293429, -0.0286906529, 5.362180e-02, -2.42659764E-4, 0.0471135713, 3.932120e-03, 0.0165259745, 0.0561575592, 0.078896813, -0.0126073696, -0.068199046, -0.0748215243, 0.0294973925, -0.0521993563, 5.739710e-02, -0.029170474, 0.0342523716, -0.0375974476, -0.0283254124, -0.0615517087, 0.0697258487, 0.00827137381, -2.703110e-02, -0.0213533826, 0.0346484631, 0.0616124272, -0.0738107339, 0.0418488123, -0.0329546854, -0.0720989779, -0.0623632297, 0.0751885772, 0.00858942885, 0.00426585786, 0.0381986834, 0.0314120837, 0.0429977216, -0.0682904571, 0.0147393225, 0.0494079851, 0.0492748506, -0.00269447663, 0.0591818132, -0.0732518658, -0.0349376947, -0.0168868471, -1.980110e-02, -0.0726646632, -0.00426378148, 0.00771336723, -0.0388301648, 0.0512710363, -0.0552446507, -0.0504119545, -0.0289437827, 0.0508497171, 0.0112317298, -0.0361361504, -0.0694998503, -0.0656164512, 0.0336959809, -0.0107844183, 0.0408264436, -0.0282509886, -0.0380358212, 0.0603067093, -0.0388898365, 0.0094651347, 0.0190849081, 0.0463091768, -0.0560035333, 0.0230544508, 1.799570e-02, 0.0281234141, -0.01995961, -0.0366437696, -0.0104382914, -0.0807371735, -7.38241127E-4, 0.0434914082, -0.00392071204, 0.0675522238, -0.0179592036, -0.0380959027, -0.045801755, 0.0177873019, -0.0452692732, 0.00472773053, -0.028823575, -0.0105896275, 0.0781527236, 0.0122188497, 0.0243239403, -0.0214085504, 0.0884167924, -0.0371545292, 0.00208115671, 0.0422214903, -0.0130627044, -0.0572425872, 0.0539640188, 0.0512458421, -0.0278118029, -0.0466740876, 0.0123611279, -0.0430480391, -0.067579776, 0.0143955238, -0.0473926514], [-0.0102138501, -0.0625809729, 0.0226656944, 0.0865989178, -0.0317134783, -0.041994784, 0.0784297436, 0.0740485415, -0.0437568203, -2.308650e-02, 0.0611472167, -0.0255630594, -0.0529061556, -0.0412154421, -0.0438005589, -0.0770650059, -0.0229078885, 0.0694613382, -0.00579256099, -0.0567116961, -0.0757423788, 0.0228007343, 0.0719660893, 0.0758321807, -0.00804078206, 0.00459816307, 0.0733335763, 0.0169833228, 0.0150752086, 0.0162314065, 0.0510377437, 0.0722899213, 0.00425897585, 0.0663070679, -0.0383354612, 0.0570164099, -0.0197739638, -0.026814092, -0.0368303582, 0.0891691669, 0.0186437592, -0.0364792123, -0.00802999176, 0.0253999047, 0.00106385769, -0.0427183509, -0.0685337409, 0.0364614539, -0.0336796753, -0.0520746522, -0.0288924519, -0.0169329494, -0.0159811769, -0.0119210435, 0.0565916598, -0.0514801852, -0.0140647274, -0.0756695866, -0.0381107852, 5.87912691E-5, 0.0220370591, -0.0172975287, 0.03089373, 0.0377886817, -0.0514472425, -0.069353044, 0.0185553934, -0.0758286789, 2.744860e-02, 0.0573583953, 0.0619040913, -0.0642257929, 0.0595581084, -0.0643824711, -0.00478942692, 0.0434204042, -0.0519201979, -0.0410004929, -0.00155487843, 0.0392853469, 0.00534423161, 0.00630677119, 0.01666978, -3.767900e-02, 0.00936731323, -0.0270818621, -0.0887319073, 0.0790124759, 0.0546248481, -0.00846491102, 0.00709243026, 0.073390916, -0.0412714407, 5.477420e-02, 0.0772125646, -3.142210e-02, -0.0581160858, -0.0116898986, 0.0468133688, 0.017297281, -0.085281387, 0.0453342795, -0.0296472199, 0.0578563958, 0.0125378612, -0.00261118053, 0.018582752, 0.0118756695, -0.0777719468, 0.0662767291, -0.0233994722, -0.0639032796, -0.0358233862, 0.0530702211, 0.0823813751, -0.0326608755, 0.0603802763, 0.0247128829, -0.0240242444, -0.0666421577, 0.0692779347, -0.0199669525, -0.0532587618, -0.0815045163, 0.0802388042, -0.0839674622, -0.0523401871, -0.0215760041], [0.0436644331, 0.075453423, 0.0702721477, -0.0149792247, 1.61802818E-4, 0.0514277965, 0.0486468561, 0.0725210905, -0.0165518895, 0.00432487903, -0.0649037883, -0.0159446094, 0.0430554822, -0.0603538342, -0.0594870262, 0.0108135715, 0.0730716214, 0.0148409223, 0.00366569497, -3.759530e-02, 0.0111033805, -0.0326682925, -0.0682244524, 0.00963338558, 0.00844505429, 0.0603133664, -0.0767452195, -0.0458586961, -0.00452121394, -0.0878628641, -0.0771504492, 6.902050e-02, -0.0162152015, -0.0368491858, 0.0613531172, 0.0508886203, 0.0243765283, 0.0660708323, -0.044550579, -0.0121021774, -0.0712155625, 0.0666004345, 0.0226735864, -0.0358039364, -0.00265808287, -0.0665517151, 0.0440778509, -0.0304632504, -0.0231616218, -0.0661935955, -0.0682971701, -0.0422221348, -0.0631744861, 0.0493145362, 0.0123086376, 0.0759947597, 0.0608542711, -0.00413594814, -4.34144953E-4, 0.0203871708, 0.0434580855, -0.0104172546, 0.0374689288, -0.0691851228, -0.0142022334, 0.0294120386, 0.0108280014, -0.0735092089, -0.0254289731, 0.0447295904, -0.0152578317, 0.0277959127, 0.0835227668, 0.079026632, -0.0242773406, 0.0277619418, 0.0396896563, 0.0323144756, -0.0213209353, -0.0395455323, -0.0102982102, 0.0671485364, -0.00291774631, -0.0222388245, 0.0725083649, -0.0366910137, -0.0481821671, 0.0779353901, 0.0640794188, 0.0425384752, -0.0892557949, -0.0157071501, -0.0348551348, -0.0692393258, -0.0435898416, -0.0225537233, -0.0190683175, -0.0544642061, -0.085963957, -0.0476756208, 0.0489332937, 0.0126872612, -3.552350e-02, -0.0412899218, -0.0628921464, -0.012575767, 0.039050255, -0.0317545794, 0.0512968078, -0.0554498471, -0.0271616355, -0.033838734, 3.258620e-02, -0.0485522747, 0.0107677924, 0.0463921838, 0.0776313096, -0.0285404529, -0.0626185313, -0.00737107499, 0.0117464261, -0.0018642745, -0.0349735022, -0.0267783385, 0.0314260796, 0.0152931567, -0.0325339697, -0.0605095662], [6.698930e-02, 0.0368120186, 0.0416742712, 0.00611716881, -0.00374985486, 0.00663199648, 0.0647227392, -0.0326938108, 0.0251123216, 0.0510016121, 0.00506136147, -0.060810376, 0.0265065748, 0.100537926, 0.00519897509, 0.0242687967, 0.026818987, -0.0774727836, 0.0162000693, -0.0799594521, -0.0493593849, 0.001460245, 0.00162508548, 1.365160e-03, 0.0147405844, -0.0120527213, -0.0773495212, 0.0214338135, 0.0555592105, 0.00443129288, -0.0224509202, 0.053594064, 0.0447179973, 0.0414355583, 0.0416737907, -0.0308611821, -0.0165186152, 0.0647556186, -0.0157316942, -0.0262549222, -0.0706948339, -0.0664535165, 0.0335997269, -0.0644670278, 0.0512292795, 0.050850071, -0.0724088922, 0.0394498482, -0.0441931225, -0.0276618656, 0.00443235645, -0.0168758333, -0.0399496071, -0.0681948662, -0.0366663076, 0.0715531111, -0.0774198771, -0.0629316717, 0.00420359662, -9.546890e-03, -0.0204902142, 0.0137761058, -0.0651444494, -0.0607434027, 0.00342559046, -0.093849726, -0.0105171772, 5.649610e-02, -0.0379271954, -0.0338632762, -0.0412824564, 0.0150934849, -0.0094886357, 0.0653843507, 0.050555326, -0.081118986, 0.0271938294, -0.0332465507, -0.0354210548, -0.065409787, 0.00285513676, -0.0160028972, 0.0152386557, -0.0706533268, -0.00878984667, -0.0238478687, -0.0292214919, -0.0396513194, -0.044689402, 0.0656886175, -0.0596451536, -0.0218668692, 0.080315724, 0.036107488, 0.0568450876, -0.0185490772, -0.0906349346, -4.310810e-02, -0.0593995675, -0.0132233696, 0.0155665092, -0.0258868448, 0.0305093024, 0.00523142656, -0.0832465514, -0.0500845313, 0.046750892, -0.0616615675, 0.0307989903, 6.54296542E-4, 0.0799270495, 0.0177906193, 0.0014569913, 0.0631614551, 0.0819118693, 7.791910e-03, 0.0611229092, -0.0794622451, -0.0533735305, 0.0661887303, -0.0508743972, 0.0287286136, 0.00809366814, 0.0352149233, -0.0171800088, 0.0125706736, 0.0527060479, -0.0584860854], [0.0585332662, 0.0739962459, 0.0587784387, 0.0197137594, -0.0360828489, 0.0827990174, -0.0368955135, 0.00610001665, -0.0234681442, -0.017205473, -0.0174716543, 0.04649451, -3.892940e-02, 0.110197403, 0.0563091151, -0.0652125254, 0.0440674685, 0.0708860755, 0.0524538644, 0.045681648, 0.0288795289, -0.0296289325, 0.0519998297, 0.0276727881, 0.0650486499, 5.457290e-03, -0.0492157303, 0.0826218128, -0.0330495499, -0.0540754162, -0.0220466927, -0.0452336743, -0.043692261, 0.0310864169, 0.0722913444, -0.00686862739, -0.00817956961, -0.0275016949, -0.0482234098, 0.0571660101, 0.0447167605, -0.012752681, -9.320070e-02, -0.0522311926, -0.0565826185, -0.0628439635, -0.0743072926, -0.0442433022, -0.038677752, -0.0580952689, 0.0123936106, 0.0437297188, 0.0654057115, 0.046022024, -0.0541541576, 0.00390690332, 0.0422778949, 0.0132523244, -0.0150559731, 0.0535212941, -0.0332549773, -0.0174831096, -0.0584914647, -0.0157474913, -0.0126920212, -0.0497454926, 0.0754172653, -0.0485664494, -0.00136654975, -0.0182083771, -0.0731859431, 0.0949097499, 0.0641512126, 0.0253363922, 0.0522187129, 0.0479793064, -0.053764727, -0.0315855853, 0.0134927491, -0.0498253964, -0.0236030649, -0.0473045297, -0.0923644825, 0.0483918637, -0.0485415943, -0.00929181836, -0.0412755646, -0.0355025269, 0.074048318, 0.0471922196, 0.0493972376, 1.549930e-02, 7.079020e-02, 0.00760436896, -0.0835387632, 0.0400383584, 0.0416010581, 0.00361203332, -0.0618776567, 0.00856892671, -0.0256354455, -0.0708494484, -0.0717327371, 0.0227648746, -0.040993467, -0.0334585495, -0.0323589779, 0.0314696617, -0.00697394973, -0.0322482549, 0.013154665, 0.0754072592, -0.0202622265, 0.0463980362, -0.0491266362, 0.0534801185, 0.0783613696, -0.043770045, 0.0575120524, 1.728580e-02, -0.0324120373, -0.0234080218, -0.0566026568, 0.0594429113, 0.0731768608, -0.0588497333, 0.0753365308, -0.0708643496], [0.0218810104, -0.0256755911, -0.0739260316, -0.0447751693, 0.0162171591, 0.0736702308, 0.0796998068, 0.0357709303, -0.0842459723, -0.02340075, -0.00836564694, 0.0838476344, 0.0143588521, 0.127088338, 0.0192969274, 0.034011811, -0.00246292329, -0.078664489, -0.0130584538, -0.054377906, 0.0411118902, -0.0735979825, -0.0252744984, -0.0718596131, -0.0182831958, 0.046444077, -0.0429409817, 0.0445065759, -0.0709949881, -0.0380777158, 0.070554778, -0.0629732906, 0.0602873117, 0.00205407594, -0.0462342612, 0.010680859, -0.034914311, -0.0234590136, -0.0732423887, 0.0742905289, 0.0125156771, -0.0060748267, -0.0352853164, 0.0537745506, -0.0103222672, -0.0706872493, -0.0585978776, -0.0327221826, 0.00487770746, -0.0415529907, 0.0622689873, 0.117373563, 0.0201645773, -0.0206180681, -0.0138274468, 0.0108920103, 0.00885500572, -0.0758869201, -0.0199017674, 0.0297128689, -8.890140e-02, -0.0819751545, 0.055792205, 0.043620117, -0.0698928833, -0.0451080166, 0.0228827316, 0.0780438259, 0.04523715, -0.0743915215, -0.0336179957, 0.0717981458, 0.00132822373, 0.0980209931, -0.0380259044, 0.0222972948, -0.071609497, -7.301330e-02, -0.0497506484, -0.0308498461, -0.0235107541, -0.0202081557, 0.0315338075, -0.0551818796, -0.0630966946, 0.0499516875, 0.0278299768, -0.0216209628, 0.0688852593, 0.0302211344, -9.127270e-02, 0.0115400823, 0.0428213552, 0.0143393353, -0.0860116929, -0.068694815, 8.749510e-03, -0.0827836394, -0.0853767618, -0.0613999255, 0.0543235578, 0.0357159227, -0.0735640451, 2.080290e-02, 0.0311732106, 0.00700356997, 0.0279725809, 0.0798649862, -0.0921493322, -0.0609466471, -0.0256646033, -0.0632957369, -0.0186245125, 0.105239861, -0.0598247349, 0.0605187491, 0.0475692451, -0.0409202501, -0.0691057444, -0.00558705581, 0.00394648826, 0.0521561131, -0.0538735576, -3.433140e-02, -0.0228350367, 0.0149513371, -0.0595248155, -0.0497185029], [-0.0519336797, 0.0701015443, 0.0704960525, 0.00283354404, -0.046988856, 0.00826065614, -7.223100e-02, 0.0203430299, -8.73639597E-4, 0.0785187557, 0.0731894746, 0.00392588973, 0.0176789183, 0.0177438669, -0.0488800481, 0.0604696721, -0.0261372086, -0.0324943289, 0.087047629, -0.00722048851, -0.0353778116, -0.0775353685, 0.0737203285, -0.0135539221, -0.00376423798, -0.0746625662, -0.0188009385, 0.0334071331, -0.045109082, -0.0871560499, -0.0080142077, 0.0309214033, 0.0208466407, 0.0325508229, 0.0200715158, 0.0422394201, -0.0022944482, 0.0491542257, 0.0657580271, 0.0446725711, 0.0253154412, 0.0196322128, 0.0746952444, 0.0390876718, -0.0703329741, 0.0176862553, -0.0588904023, 5.623080e-02, 0.0661831647, 0.0109067336, -0.0804581195, 0.119966038, -0.0484475158, 0.067340374, 0.0242465194, 0.0443216451, 0.00227421871, 0.01526995, 0.0387774222, -0.0348517448, 0.0225591641, 0.0604782477, -0.0909359231, 0.00359859085, 0.0305983685, -0.060068585, 0.0847652704, 0.0260683019, -0.045156762, 0.0216075089, -0.0139627922, 0.0845084115, -0.00354607264, -0.016112389, 0.0340580717, 0.00735029625, 0.0389486402, -0.0628607273, -0.0470010415, 0.0478281081, 0.0771978274, 0.0426851436, 0.0274627265, 0.0175485425, -0.0761211812, -0.0677981228, 0.0541723631, 0.0819365531, -0.0383255109, 0.0752808153, -0.0551021583, -0.0273281708, 0.0594854616, -0.0284269638, -0.0861644148, 0.0718889311, -0.00160215364, -5.715140e-03, 0.0627360493, -0.0604174063, 0.0465413928, -0.0167090315, 0.00270905439, 0.0514494963, -0.00106713618, -0.0410148725, 0.00155752769, -0.0393062644, -0.0185304116, -0.0582228824, -0.0481984094, 0.00908023398, -0.0605030357, -0.00768535631, -0.0536652543, 0.0338576697, 0.0250012577, 0.0349866375, -0.0772571191, -0.0817876383, -0.0971614718, 0.00721184118, -0.100766465, 0.0352598317, -0.0591142923, 0.0468351096, 0.0203683469, 0.05356282], [-5.890860e-02, -0.0515875109, 0.0707002059, 0.0230711158, 0.0106213959, 0.0683484375, -0.0297431089, 0.0581592694, -0.075846009, -0.0593603253, 0.0950116738, 0.0644095391, 0.0521786362, 0.0180142969, 0.0459143482, 0.0421453193, 0.0780587047, -0.0192308351, 0.0626993477, -0.0643886551, -0.0156582575, 0.0520028025, -0.0377936959, -0.0657095388, 0.013672214, 0.0570080541, 0.0259655062, -0.0406431518, -4.804410e-02, 0.0179695394, -0.00684720976, 0.0835124105, -0.0395395756, -0.0571684353, 0.0132864201, -0.00941628497, 0.0516034365, -0.0426285602, 0.0732776299, 0.0461147167, -0.0370787419, -0.0763546824, 6.984600e-02, -0.0640591457, 0.0460402481, -3.138030e-02, -0.025404118, 0.0138156097, 0.0466453508, 0.0252900384, -5.217390e-02, 0.0390356593, 0.0314958803, 0.0363448523, 0.0354348198, 0.0781301707, 0.0446768142, -0.0662181601, -0.0909465327, -6.272030e-03, 0.00332996738, 0.0348302536, -0.00228595245, -3.819150e-02, 0.0351569876, -0.0124678276, -0.0209316518, -6.508100e-02, 0.0674403086, -9.375020e-02, -0.00899555255, 0.0234810114, 0.0215067193, 4.876170e-02, 0.0546414033, 0.0290613621, -0.0703933164, 0.0062343739, -0.0516920239, 0.0414719135, -0.0331041887, 0.01627638, -0.0426916592, 0.0186088067, -0.0391461551, 0.0463955589, 0.049791038, -0.0300937425, 0.0269232821, 0.0134882461, -0.0340673141, 0.101128154, -0.0878921523, 0.0137716932, -0.0727859885, 0.0478551053, -0.0964713544, -0.00760338641, 0.0411172844, -0.0534589924, -0.080693841, -3.661250e-02, -0.060046345, 0.0569832101, -0.0168340579, 0.048528295, -0.051631026, 0.0287857223, -0.0258106403, 0.0444785058, 0.0729664043, -0.0312979966, 8.27529511E-5, -0.00915349275, 0.0789531767, 0.00740854815, -0.0204202179, -0.0549193956, 0.00136902696, 0.0276511759, 0.0181037914, 0.0159069691, -0.00856579281, 0.0468215793, -0.0335958637, 4.289310e-02, -0.0617160723, -0.0709207579], [-0.01404119, 0.0636551306, 0.0415173918, 0.0645776764, -0.0301605761, -0.0143580837, -0.0536277033, -0.00112983666, -0.0124873128, 0.0615692399, -0.0646126643, 0.0170035902, -0.0359622091, 0.0599305816, 0.0497247837, -0.0453983285, -0.0542466268, -0.0167373251, 0.0624621361, 0.021024473, 0.0844187587, -0.0281159412, -0.0641535595, 0.00282554724, -0.0152512314, 0.00809199921, -0.00751947938, 0.0130404877, -0.0852405726, 0.0169231258, 0.0272916481, 0.075257428, -0.00129578717, -0.0468630269, -0.0234366134, 0.0712290853, -0.0448437668, -0.00163217215, -0.0286131185, 0.0132533014, -0.0068851877, -0.00807811319, -0.0177940857, 0.0393948108, -0.101062849, -0.0230641961, 0.0652505606, 0.00715797069, -0.0219441187, 0.0888109281, 0.0628852174, 0.0797813311, -0.0583450422, -0.0470313393, -0.0362235866, -0.0627508238, 0.0123125836, -0.070807822, -0.0768311321, 0.0865820571, 0.0157568268, -0.0175477155, -0.0722061693, -0.0718778521, -0.039245557, 0.0676603466, 0.107687235, -0.0148570966, 7.231160e-02, 0.0103220874, -0.107698426, 0.0737585574, -0.047727745, -6.25245448E-4, 5.24613541E-4, 0.0613630451, -0.0731127933, -0.0460081436, -0.104251057, 0.0967019796, 0.00639535486, -0.00117299543, 3.886540e-03, 0.0578713492, -0.0928791463, 0.0788154974, -0.0407325141, -0.0203870256, -0.0146491183, 0.0268119052, -0.0622969493, 0.0408758298, 0.0429753438, -0.00379965641, -0.0863551571, 0.0028854541, -0.0737914592, -0.0420421548, -0.0756825358, 0.0466120392, -7.641250e-03, 0.0255589597, 0.0486201569, -0.0696081668, 0.0259345323, 6.145980e-02, 0.0821952075, -0.064547874, -0.0149193136, -0.0304470882, -0.034803234, 0.00479149492, 0.0460935198, 0.0570191965, 0.0401009321, 0.105087288, 0.0353977531, 0.0196180772, -0.080837436, -0.00328126969, 0.0188732035, -0.0382504687, -0.0853974595, 0.0752214938, -0.0352113098, 0.0495779552, 0.0311919544, 0.0680134371], [-0.038173791, 0.056305524, -0.0150173297, -0.00361932185, 0.076014556, -0.050494276, 3.368650e-03, 0.0543784834, 0.0882979109, 0.0616886057, -0.0261951145, -0.0454190336, 0.0402477495, 0.01157548, 9.021090e-02, 0.0352261476, -0.0673401132, -0.034366373, -0.0153451236, -0.012806532, 0.0791894569, -0.0885997266, -0.0546771102, -0.0461315699, 0.0881634206, 0.044658646, 0.118474938, 0.0483797193, 0.0522103347, 5.477500e-02, -0.0559265316, 0.101242147, -0.029550625, 0.06550432, 0.0155149698, 0.100669689, -0.0126459626, 0.0165630225, -0.0794723183, 0.0321188793, 0.0200536512, 0.0561580211, 0.0376127213, -0.0475729629, -0.0951984674, 0.0107429558, -0.0337563045, 0.0543056317, 0.0264848266, -0.0428857505, 0.0887267143, 0.0891418159, -0.0444969945, -0.0486203544, -0.060249757, -0.00695147831, -0.0638626739, -0.0679414719, 0.0434246585, 0.0191111173, 0.0317238085, -0.0162825771, -0.0883062929, 0.00130599365, 0.0519453697, -0.0417520963, 0.0268130749, -0.0544045642, -0.0399221852, -0.0676261708, -0.0638950393, 0.0326760933, 0.0155783026, 0.0707298145, -0.00671507325, 0.0797708184, -0.0277559087, 0.00181769533, 0.0472741537, -0.0267062373, 7.84824893E-4, 0.0831324458, -0.075478144, -0.0463847332, -0.0175738111, -0.0403569378, 0.0251013171, 0.0311138052, -0.00352915097, 0.0290164798, 0.0786913409, -0.039806556, 0.0307739619, -0.0527874269, 0.00294084917, 0.0297564194, -0.0708168447, 0.0067542824, -0.0651418716, -0.0263075437, 0.0297767483, -0.00256903493, -0.0947431102, 0.0268220101, -0.0270999372, -0.0443610176, -0.00224894518, 0.0234504975, -6.552390e-02, -0.0817508473, 0.0257872455, -7.298030e-02, -0.0184915736, 0.0747373253, 0.0059017064, 0.00556101464, 0.071271047, 0.0409077667, 0.0688680783, 0.0196357276, -0.1017249, -0.0566637032, 0.0269709658, 0.050551381, 0.0193943419, 0.0285130404, -0.0424012765, 4.868850e-02], [0.078924857, 0.00651708338, 0.0491922796, 0.0958894416, 0.0840651765, -4.520090e-02, 0.0237475466, 0.0395257175, 0.0563659221, -0.0487117171, 0.00246659177, -0.0393246077, 0.0384384431, 0.0258898567, 0.0199709535, -0.0718938634, 0.0226099901, 0.022033181, 0.0723779798, 0.0234270524, -0.051611539, -0.0252372362, -0.0587143227, -0.0476046503, 0.0609280728, -0.0354497731, 0.0259963796, -0.0560811535, 0.06299375, 0.051829163, 0.0564624481, 0.0329585336, 0.0689411685, 0.0261162464, 0.0204969905, 0.00763633382, -0.00581749249, -0.0604577474, -0.031953793, 0.0894896537, 0.0724205076, 0.0152805513, 0.0280700065, -0.0571410358, -0.0395042598, -7.008280e-02, 0.023830235, -0.0310809575, 0.128475219, 0.0904908254, -0.0524098761, 0.0952175855, 0.076932922, -0.0135699436, 0.024629863, 0.0352842547, 0.0541891977, -0.0217025168, -0.105177276, 0.0361678898, -0.0442855395, -0.00382005284, 0.0258159265, 0.020828018, 0.0521886498, -0.0402445942, -0.0414740033, 0.0385127962, -0.0020363857, 0.043369066, -0.0479355566, 0.0249243677, 0.00578081701, 0.0764568448, 0.0271551907, -0.0100245038, 0.0483397618, 0.003666637, -0.0602315702, -0.00301434728, 0.0244771726, -0.0294290744, 0.0337443724, 0.01086869, -0.0400324389, 0.0579444468, -0.0103227114, 0.048786331, -0.0721065849, -0.0840841085, 0.0561655201, 0.0229078159, -0.039793212, 0.0236627664, -0.0654492155, -0.0704253763, -0.0115445117, -0.00949777103, 0.0989258885, -0.0033655169, -0.0451541729, -0.0895824357, 0.0166768525, 0.04150315, 0.061644204, 0.0204708967, -0.00934343133, 0.0306750089, -0.0583041944, 0.0434351824, 0.0279170107, 0.0754375234, 0.0827495977, 0.0389742702, 0.0296589863, 0.113939404, 0.0490896255, -0.0434789285, -0.0359415039, -0.110361978, -0.0410399698, -0.00726238918, 0.0231443159, 0.0562822819, 0.0234807599, 0.0196193848, -0.0585725904, 3.193540e-03], [-0.0402242839, 0.0652820542, -0.0498247854, 0.0857495219, 0.0273776595, 0.00314173754, -0.0525792353, -0.0574264862, 0.0118018091, -0.076206319, 0.0271532144, 0.0115342159, -0.0473830588, 9.489580e-02, -0.0467782691, 0.0358064137, 0.0072204764, 0.0257560164, -0.0453672409, -0.0101242708, -0.0546598844, -0.0540527776, 0.0700578839, -0.0120468782, -0.055904448, -0.0993919149, 0.00668940088, -0.0749975144, 0.0567592643, -0.0502774194, -0.0106046675, -0.0201462321, -0.0401873812, 0.0359683484, 0.0483225621, 0.117735244, -0.0768763945, -0.0602956377, 0.0352496952, 0.0429584421, 0.0664416253, 0.0526209362, -0.0421883203, -0.0589773692, -0.0702101812, -0.0679812878, -0.0328680277, 0.015229078, 0.109838337, -0.0396523215, -5.420990e-02, 0.077948518, -0.0418652184, -0.0781035125, -0.067186892, -0.0268014912, -0.0212908164, 0.0794843807, 0.0174353868, 0.0194510575, -0.0898667201, 0.00754364813, 0.0473295115, -0.0728530884, -0.0215156022, 0.0126827555, -0.0150466422, -0.053117238, 0.0315382332, -0.0846912488, 0.0173369125, 0.0191571638, 0.0572242886, 0.00105588569, 0.0294387192, -0.0314924344, 0.0372002237, -0.0718333572, 2.431090e-02, -0.00713230483, -0.0463792421, 0.0305106267, -0.0792225673, -0.0253763665, 0.0702326521, -0.0146809425, -0.0152871758, 0.0200911798, -0.0857371464, 0.0259859953, -0.0238229278, 3.072240e-02, -0.054772865, 0.0296019018, -0.0592008159, 0.00970973726, 0.0267559625, -0.0149377957, -0.00467421673, 0.0368305035, 0.0432397537, 0.0131659573, -5.466870e-02, 0.0111485198, 0.0462418683, 0.0397797264, -0.0452372618, 0.0156551562, -0.0740257576, -0.0699383169, 0.0262336172, -0.00527551491, 0.0377694964, 0.0203896351, -0.043005161, 2.86687224E-4, 0.0138877975, -0.00164589111, 0.0121775875, -0.0191352032, -0.117745548, -0.0289170537, -0.0196884926, 0.0680641159, -0.0176555533, 0.0674112886, 0.00225396222, 0.0869811251], [-0.0335247405, -0.0441687293, -0.0779468119, 0.0849706084, 0.120339826, 0.0157142151, 0.00612422498, 0.069259122, 0.0299057011, 7.570210e-02, 0.0472890846, 0.0603014305, 0.031320177, 0.0384626389, 0.0518902466, 0.092814222, -0.0412467085, -0.0423880331, 0.0638911501, 0.0189382546, -0.0600057654, -0.078767322, 0.0739576593, -0.0734856278, -0.0347555205, -0.0203086771, 0.0951227918, 0.0353737026, -0.0435075387, -0.0230930634, -0.0307194497, -2.724560e-02, 0.102923892, 0.0118174832, -5.371420e-02, 0.0353036858, -0.0946540087, 0.00886405073, 0.0673083737, 0.00844426546, 0.0283825062, -0.0414205119, 0.0613314509, 0.0145026632, 0.0137557117, 0.0688719079, -0.0149589963, 0.0218081195, 0.0476285033, 0.0897886082, -0.0640492439, 0.0988100767, 0.0508239791, -0.0601931214, -0.0472400747, -0.054445006, 0.0680080727, -0.00739364466, 0.0303817261, 0.00775923952, 0.00419726269, 0.0589470416, 0.0238403939, -0.0317925438, -0.00216231355, -0.0557162724, -0.0462364927, -0.0669424757, 0.0351462476, 0.0217967369, -0.0756746754, 0.0215891916, 0.0203393828, 0.0236314945, 0.0734501109, -0.0430726446, -0.0190627072, 0.0285145417, -0.0957798063, -0.0314198807, 0.0340651087, -0.00661560706, 0.012496558, 0.0144535229, 0.0982251614, -0.0112909097, 0.127922162, 0.0612382703, -0.027281506, -0.0132223358, 0.0825201124, 0.0320500582, 0.00149702642, 0.0538693666, -0.00510466797, -7.374930e-02, -0.04439722, 0.00603609951, 0.0813554078, -0.0360914022, 0.0524888039, -0.0726541877, 4.113370e-02, 0.0570453703, 0.0310484339, 0.0119585814, 0.0874801576, 0.0465867259, 0.0342630073, -0.00718269218, 0.118960865, 0.073981829, -0.021150345, 0.0900057181, -0.00373440259, 0.0546902902, -0.0550763458, 0.069182083, -2.3857132E-5, 4.499280e-02, -0.0541323125, -1.43604935E-4, 0.0159728732, 6.409160e-02, 0.0813467652, -0.0545879602, -0.00194907142, -0.00644492171], [-0.0524581634, -0.0443446599, -0.0302243549, 0.0167901926, 0.093431808, 0.0254667401, 0.0387277529, -0.0437391736, 0.0122613339, 0.0607658699, 0.0650843307, -0.0125488536, -0.0263098758, 0.0104227476, 0.0709052533, -0.0504330955, -0.0684508532, -0.0476759374, 0.0692709386, -0.0216107778, -0.0095943436, -0.0551944226, 0.0808225274, 0.00903669745, -0.0141286198, -8.684840e-02, 0.0144824432, -0.035669256, -0.0505012311, -0.0822391286, 0.0436265208, 0.0626769289, 0.0121739358, 0.0525681302, 0.0766078681, 0.00849986356, -0.0171374585, -0.0549760424, -0.0252038352, -0.0476577058, -0.0489574596, 0.0819997638, -0.0577016734, 0.037185289, 7.542320e-02, 0.041286137, 0.0631934851, -0.0518097207, -0.0203842819, -0.0173982717, 0.0650017559, 0.0296471361, 0.0379510373, -0.0202571675, -0.0947752446, -0.0822536572, -0.0657470599, 0.0231947117, -0.074710317, -0.058811754, -0.0962542518, -0.0670763403, 0.0467369743, 0.0684748589, 0.0163433943, -0.0594608635, -0.0634490177, -0.0406187251, 3.905570e-02, -0.00773193641, 0.00243845698, 0.0187339857, -0.0146090779, 0.0240134373, -0.00768070621, 0.0311181694, 0.0368223898, 0.013403629, 0.0324414708, -0.0322153158, -0.0120838443, -0.00713307969, 0.0741919354, 0.0635833442, -0.0426780432, -0.0391205661, 0.029055262, 0.012108543, 9.14827222E-4, -0.0803013891, 0.0890375823, 0.0824022367, -0.0254673474, -0.0475022048, -0.0186330378, -0.0747583583, -0.0101871137, 0.0140806902, -0.0774020329, -0.0588781759, -0.0104579907, -0.097470574, -0.0101108951, 0.0177952051, -0.0293803066, -0.0123688439, 0.109626956, 0.0800586342, -0.0900350511, -0.0412436053, 0.0778340622, -0.0419288725, -0.00296855695, 0.0836832225, -0.0570118539, 0.0826942772, 0.0603725612, 0.049863372, 0.00277176802, -0.0784306749, 0.00516745774, 0.0396577567, 0.0267154742, -0.0358028635, -0.0655121431, -0.0648819506, 0.0510115363, 0.0571941808], [0.0557172671, -0.0658014566, -0.061490383, 0.0256573688, 0.0619518831, -0.0090450868, 0.00787128415, -0.0512151979, 0.0842764675, 0.056038294, -0.00247799116, -0.0371109731, -0.0631148368, 0.050921943, -0.0132607305, 0.0105536431, -0.0199561976, 0.0558388233, 0.074142538, 0.071369186, 0.00303482194, -0.0403128155, 0.00351511268, 0.0550984628, -0.079550676, -0.0533233285, -0.0195941459, -0.0380433947, -0.0767686516, 0.0401641428, 0.0479537621, 0.0689725131, 0.0362336747, 0.0575617291, -0.0591662638, 9.862610e-03, -0.0510671698, -0.0499272197, -0.0187737886, -0.0176217705, -0.0367530175, -0.058256276, 0.019034747, 0.120067261, -0.0105722202, 0.0315336399, -0.018945232, 0.0729839131, 0.0791262686, -0.0478357896, -0.0498571359, -0.00312150386, 0.069673121, -0.0123966467, 0.0502152406, -0.0649315342, -0.0279185697, -0.0696333944, 0.0295476466, 0.0601486042, -0.0854734182, -3.82269151E-4, -0.0437646583, -0.0148664983, 0.047489468, -0.047120735, -0.0397911891, -0.0597561747, -0.0581464544, 0.0547137856, -0.00240657153, 0.00686564436, 0.00131506799, -0.0708359777, -0.0324791111, 0.0619110129, -0.0521084219, -0.021817727, 0.0334515087, 0.0248645861, -0.0533032492, 0.0644891933, -0.00276653655, -0.0775770694, 0.0120800212, 0.00299625075, 0.0781216248, 0.0414023958, -0.0437456481, 0.0152482213, 0.0665295795, 0.0101563958, -0.0503522381, 0.069580853, 0.0104821501, -0.0359244347, -0.0292855203, -0.0347282588, -0.0382088162, -0.0417299941, -0.0315105394, -0.0458424762, -0.0203166883, -0.0102313142, -0.0565249473, -0.0546719842, 0.112562031, -0.0308345146, -0.0813843905, 0.0210935846, -0.0712069273, -0.0239723716, -0.05926916, -0.0044620824, -0.108141281, 0.0294849314, 1.770370e-02, 0.0410142913, 0.00379491574, -0.0184785593, -0.0892029256, 0.0379200615, -0.00440214388, -0.03321594, 0.0448327921, -0.0167383011, -0.0175109841, 0.0150865838], [0.00306448783, -0.0382914096, -0.0449377336, 0.0336366259, 0.0849446803, 0.00424268842, -0.0440487526, 0.079799518, -0.0746895596, -0.0291507393, 0.00827923789, 0.0638119653, 0.0738192722, 0.00270714122, -0.0847107321, -0.0554459281, 0.0588796251, -0.0776921362, 0.0220455658, 0.0365545377, 0.0254261494, -0.0621603131, 0.0449684635, 0.0264459178, -0.0819407626, -0.069458507, 0.0553322546, 0.0106660519, -1.575210e-02, -0.0327101313, 0.0392998345, -0.0612568036, 0.0663465038, -0.0115778949, 0.0646561235, 0.0606071129, 0.0176662151, -0.0440199673, 0.0163559653, 5.643200e-02, -0.031267453, -2.817740e-02, -0.0764980912, -4.97574394E-4, -0.0408873484, -0.0609533079, -0.0143087152, -0.0130521329, 0.0486191437, -0.0273723956, 0.0725096464, 0.0118418019, -0.0459040888, 0.0322028138, -6.852190e-02, -0.0078469906, -0.0504829809, -0.0779558718, 0.0383904576, -0.0464999527, 6.726010e-02, 0.0409396328, -0.0242372938, 0.0697762072, 0.048914291, -0.0292180609, 0.0290281158, 0.0223534945, -0.06709335, -0.06763836, 0.0272876583, 0.045043245, -0.0132886227, -0.0704363137, -0.0614415854, 0.10336221, 0.0437566116, -0.00605780492, -0.0339061506, 0.00927482359, -0.0310544837, 0.0651484877, 0.00329827145, -0.0323427655, 0.0702432469, -0.0293033123, 0.0793874934, -3.83414794E-4, -0.0425711162, -0.0200123526, 0.0352892466, 0.0353826582, 0.0447522178, -0.0589379109, 0.0903864354, -0.0076270937, -0.0493215807, -0.0190636795, -0.0527546667, -0.0180167276, -0.0497958548, -0.0164405424, -0.0692569092, 0.0316278376, 0.0276553538, 0.0574012585, -0.0330516472, -0.0657202825, 0.0376159512, 0.0135972891, 0.0383232757, 0.00171006063, -0.014183687, -0.0573138744, -0.041037675, -0.0384298824, 0.0129692703, -0.05934323, -0.0743722692, -0.0850195884, -0.0821626409, -0.0246253107, -0.0347787961, 0.00912223756, -0.0822713524, 0.0386144966, 0.00220342353, -0.0866320729], [-0.0271668825, -0.00357908546, -0.0515136644, -0.0170858447, -0.011760748, -0.0148242423, -0.0609139949, 6.89468361E-5, 0.00656181667, 0.0599859357, -0.0162437074, -0.0222438555, -0.0648010895, -0.035389971, -0.0593659654, 0.0460336655, -0.0497308746, -0.0428163745, -0.0174221471, -0.0406596959, -0.012318383, 0.0217857435, 0.00833632331, 0.0420250259, 0.0163725968, 0.0618734173, -0.0373478979, -4.559310e-02, -0.0485053957, 1.193420e-02, 0.0863672867, 5.53830352E-4, 0.0797436833, 0.0124294013, -0.0264591873, 0.018773878, -0.0210804082, -0.0108380094, 0.0815947801, -0.0814776048, -0.0152243115, -0.0156178987, 0.0202392768, -0.00995067413, 0.00425427733, 0.0273907818, -0.0134489583, 0.0780544132, 0.00326992478, -7.179700e-02, -0.0183133539, 0.0335747339, 0.0399850793, -0.0203061383, -0.0573555268, -0.0978972539, 0.0410135575, -0.0378080979, 0.0159335639, -0.0330466479, -0.042438779, -7.928910e-02, -0.0681191906, -0.0742386132, -0.0148115223, -0.005905631, -0.063634567, 0.0207757149, -0.0227591284, 0.07835114, -0.0431623571, -0.0470980778, -0.00225185603, -0.0202917047, 0.0129294461, 0.0879261717, 0.0356465578, 0.0120593524, -0.0154227857, -0.0682034269, -0.0401458479, 0.0224689785, -0.00363085512, 0.017682625, 0.0897532552, -0.0655700713, -0.0743289291, -0.0793514773, 0.0254855528, -0.00465400703, 0.0919604226, 0.0674182847, 0.0158370826, -0.0461053327, 0.0901002511, 0.00174983172, -0.0398979783, -0.026952574, -6.326070e-02, 0.0309220534, 0.089521408, -0.0140960878, -0.0430816747, 0.0852459445, 0.00719909556, 0.0579243563, 0.0388723128, -0.0403367169, -0.0665775463, -0.0496026166, 0.0112852165, -1.25181527E-4, 0.0638022944, 0.0431429446, 0.0279930476, 0.0203858018, 0.0187938549, 0.0177649688, 0.0566248819, -6.992880e-02, 0.0498255715, -0.00599753112, 0.0440618843, 0.0501552485, -0.0673244223, 0.0250883028, -0.00159012363, -0.0163832773], [0.0219834261, 0.0325507931, -5.993570e-02, -0.00426252838, -0.0256077331, 0.0474307053, 6.857330e-02, 0.0223544985, -0.0805164725, -0.0585148782, -7.95236788E-4, -0.0281060822, 0.0447711721, 0.0533995405, -0.0683342069, -0.0684188679, -6.263460e-02, -0.064231053, 0.0634024367, -0.0365834385, 0.00669105817, 0.0560727529, 0.0264860038, 0.0521026067, -0.0677369237, -0.0655446276, 0.0594495609, -0.0825400575, 0.0290065631, -0.0776932761, 8.399490e-02, 0.0366073027, -0.00984643585, -0.0145410923, 0.0681643412, -0.0575040765, 0.00532371039, -0.0288077686, 0.0852043852, 0.03007962, -0.0223920047, -0.0675284415, 0.0420560054, -0.0139619978, 0.0547878668, 0.0263108425, -0.024893675, 0.0776973516, -0.0454396047, -0.0603569038, -0.0347924866, 0.053913489, 0.0153185595, -0.0736149922, 0.0328779668, -0.0229668636, 0.0942370593, 0.0270358194, 0.0659289733, 0.0701499507, 0.0458291024, -0.0290836487, -0.0105349813, 0.0787945241, 0.0269907825, -0.0406055562, -0.00268820766, 0.0121728256, -0.0578078665, 0.0798323452, 5.965390e-02, 0.0458942875, 0.0562707111, 0.0254036374, -0.0218146015, 0.0941876098, 0.0845915749, -0.0317389295, 5.807170e-02, -0.00120661012, 0.0194132514, -0.0258506872, -0.0385623835, 0.0480351448, -0.00158537994, 0.0691144392, 0.00803515501, 0.0517495349, -0.00882653054, -7.476650e-02, 0.089669086, 0.0587917343, -0.0828177184, -0.00774612511, 0.0833483934, -0.0212596338, 0.0144753614, -0.0289334394, 0.0613171719, 0.00766524719, 0.0682864115, -0.0686190277, 0.00192219787, -0.00666112034, -0.0465092883, 0.0993326082, -0.00489307847, -0.0153205013, -0.0709534287, -0.0685704052, 0.0654179826, 0.0121374251, 6.907460e-02, -0.0636780113, -0.107751317, -0.0508099757, -0.0340077654, 0.0404498391, 0.00367765105, 0.0212373734, -0.0164571386, -0.0388924927, 0.0353416763, -0.0378903709, -0.00751442323, -0.0497381315, -0.0265576243, 0.0174718313], [0.0446430482, 0.0199748315, -0.0637547597, 0.0186955146, 0.0617494062, 0.0751397609, 0.0670838729, -0.0674802586, -0.0459958091, 0.0850899443, 0.0409755968, -0.0164253078, 0.0625718907, 0.0321069285, 0.0131606804, -0.00330813252, -6.945270e-02, -0.0643796474, 0.0382349044, -0.0676829368, 0.0219258927, -0.0216103196, 0.0683663115, -0.0274182204, 0.0448335819, -5.593020e-02, 0.0327574499, -0.03739224, -0.00628669141, -0.0511409566, -0.00625002664, 0.0740969777, -0.00819245353, 0.076280564, -0.0232277438, 6.436140e-02, -6.326140e-02, -0.00631464506, -0.00713877193, 0.0295398273, 0.00305178319, -0.0273809768, -4.733190e-04, 0.0829326435, 0.0252860785, -0.0564133972, 0.0327257216, -0.0504074544, -0.0896005406, -0.0311047751, -0.0259615071, -0.052684918, -0.0221765507, 6.05722773E-4, -0.0878234207, 0.0345412977, 0.0321195722, -0.0339274891, -0.00764724566, 0.0172306094, 0.066461511, -0.0231638458, -0.0705998763, -0.0562776811, 0.0174976774, 0.0202667732, 0.00376209244, 0.0474716313, 0.0480079427, 0.0394267775, 0.106417805, 0.0355638117, 4.968360e-02, 0.0394813418, -0.0747120678, -1.43876576E-4, 0.0657120422, 0.0441861078, -0.0712555348, 0.00191444554, -0.025346579, -0.011064454, 0.0645871833, 0.0102897137, 0.00962305348, -3.679570e-02, -0.00877884496, -1.98091162E-4, -0.0107248388, 0.0397727117, 0.0866478607, 0.045638334, -0.0761621147, -0.0366681851, -3.63288127E-4, -6.375130e-02, -0.0701326653, -0.0783466994, 0.0440217741, 0.0414307453, 0.00372201274, -0.038983915, -0.0235864334, 0.0244262926, 0.031075621, 0.080724664, 0.103195444, -0.0247117877, 0.00101923093, 0.0557058603, -0.00255057612, 0.0109163392, -0.0691508129, -0.0777166485, -0.0360642225, -0.0260769725, -0.0075421636, -0.0295400657, -0.0743540227, -0.0283774305, -0.0148513373, -0.0197676029, -0.049515456, 4.85154334E-4, -0.0709951371, 0.0494405031, -0.00290177669, -0.021641966], [0.0259518977, 0.00991564337, -0.0202599186, 0.0177469775, 0.0778415873, -0.00142437406, 0.0580612384, -0.0466212742, -4.898010e-02, -0.00386066874, 0.0598007366, 0.0716548041, -0.053033039, 0.00331658823, -0.0460487492, 0.00763157522, -0.00479762163, -0.0694467127, -0.0391583517, -0.0816329494, -0.0613682121, 0.0146819837, 0.059043318, -0.0055693551, 0.0249676611, -0.0113484943, -0.00193139445, -0.0124268401, -0.0786395222, 0.0379589535, 0.0409274474, 0.0379263535, 0.0860766172, -0.0526900068, 0.0852042437, 0.034698464, 0.0132743772, 0.0517206714, -6.13051583E-4, -0.0729456916, -0.0283848494, 0.0714877918, -0.0315207131, 0.0425623097, -0.0399857685, -0.0538134091, 0.0815870687, 0.00390594034, -0.0333533473, -0.0222361181, -0.0502375029, -0.0163715202, -0.0364071913, -0.0417994298, 0.0370661654, 0.0690229535, 0.0874140188, 0.0620336197, 0.0320016928, 0.0791662707, -0.0565233119, -0.0472747162, 0.0585333481, 0.0560291447, -0.0628088042, 1.48082647E-4, -0.0487531386, -0.0246404894, -0.0774153918, 0.0380664803, 0.0891688764, 0.0481369905, 0.0519816428, -0.0571747459, -0.0146185141, 0.0112452991, -0.00588507438, -0.0391352475, -0.0372533612, 0.0501308814, 0.0139102414, -0.0642365292, 0.029315237, 0.0391859487, -0.0219280403, -0.0161941964, 0.029015895, 0.0553426817, -0.0480665751, -0.0408768579, -0.0223162416, 7.941850e-02, -0.00825139507, 0.0129943239, -0.0228847284, -0.0195578299, -0.034727823, 0.0451313592, 0.0109418612, -0.0247185938, 0.00193458411, -0.0839894711, 0.0135486713, 0.0216179937, 0.0439023897, 0.0115506435, 0.0653784275, 0.00130974362, -0.00453836564, 0.0612586513, -0.0605848133, 0.0505802147, -0.035623014, 0.0123405615, -0.0352897495, -0.0786054879, 0.0743868202, -0.0605309903, 0.0494118519, -0.058355812, 0.0112905167, 0.0101722702, 0.0646978393, -6.52749207E-4, -0.0265469011, 6.365190e-02, -0.0311952177, 0.0419168323], [-0.0421752743, -0.0130366692, -0.0703385249, -0.0845520868, 0.0106166499, -0.0609260462, -1.380050e-02, -0.0384596922, -0.0283623338, -0.0591428652, -0.0718506276, 0.00387767516, 0.0744034945, 0.0752030313, -0.0486238524, 0.078235276, -0.029918462, -0.0416594148, -0.062463317, -0.0687718764, -0.0469903424, -0.0710398629, 0.067456916, 0.0368464552, -0.0070631397, 0.0050655934, 0.0115526496, -4.62175201E-4, 0.0360485092, -0.0760464296, -0.0061544464, 0.00666420347, 0.0781916454, -0.0466713756, -0.023079196, -0.0439645536, -0.00533406762, 0.062802054, 0.0227866322, -0.0249487031, 2.232900e-03, 0.0424030162, 0.0401986502, 0.011087467, 0.0509908386, 0.0123055857, -0.076319389, -0.0560194626, -0.0155309699, 0.0477335341, -0.032893993, 0.0471804217, 0.0474631749, -0.012723471, -0.0556776188, -0.00346271298, -2.899350e-02, 0.0169132072, -0.0660871938, -0.0263807047, 0.0115505299, -0.0158654358, -0.051418744, 7.000130e-02, -0.00338599272, -0.0258316416, -0.020477226, -0.0474669412, 0.0508095473, -0.0525206402, 0.0278935712, 0.0522375107, -0.0322518386, 0.0362848416, -0.0314447805, 0.0109141301, 0.0332559794, -0.00854654051, -0.0481210314, 0.0156757589, -0.0351016559, -0.0611136742, -0.0553357676, -0.0167713705, -0.0413564742, 0.0597977266, -0.0510038361, -0.0737875178, -0.070006974, 0.0125322128, 0.0352851599, 0.0793567449, 0.055881761, -0.050163094, -0.0093939118, -0.00443220092, 0.0813532546, -0.0152973579, -0.00824694335, 0.0149236135, -0.0503410771, 4.93316154E-4, -0.068125926, 0.0336288773, 0.0794819742, -0.0122346655, -0.0689165741, -0.0489572473, -0.0775892436, 0.0649355575, -0.0268058274, -0.0420286171, 0.0159981027, -0.0653199255, -0.0388197973, -0.0346207358, -0.0250846557, -0.0462064408, 0.0620718524, 0.0379005931, 0.00308967731, -0.0433699824, -0.0402848683, 0.0241784845, 0.0228616111, 0.0749831796, -0.0461165085, -0.0303136874], [-0.0804024115, 0.0637819916, 0.0484001711, -0.05491855, 0.00578942942, 0.0185095202, -0.00901670753, 0.0157467369, -0.0467315614, -0.0801386907, -0.0300534572, 0.0517658964, -2.219150e-04, -0.0633538142, -0.0620440766, -0.052277945, 0.0216921959, 0.0171281379, 0.030614268, 0.070517838, 0.00702900998, 0.0584362894, -0.0455012582, -0.0174733773, -0.0595264249, 0.0152213098, 0.0273696966, 0.00790345296, 0.0801545754, -0.0452610254, -0.0416584536, 0.0630585477, 0.00434754416, -0.0201822612, 0.053812962, 0.0398231968, 0.0407468379, 0.0808474421, -0.0431860797, -0.0743442103, -0.0471315607, -0.0133948615, 0.0347983651, 0.0141270421, -0.00313942111, -0.0807079374, -0.0356237441, 0.0474730954, -0.0667669103, -0.0260840766, -0.0450440235, -0.0110177314, -0.0548164397, -0.00576723088, -0.0782627835, 0.0782179161, 0.018392317, -0.0613829754, 0.0553357266, -0.0175117869, -0.0380934887, 0.0176622253, 0.0193418488, -0.0456897095, 0.0412206054, 0.0296343137, -0.0407599173, 0.0695950314, 0.0421941951, 0.0130522856, 0.0276832394, 0.00376908784, -0.0129988473, -0.080090098, 0.011095345, 0.0598028637, -0.00697643636, -0.0291275866, 0.0254557561, -0.030942915, 6.077240e-02, 0.0155293569, 0.0539790019, 0.0674977303, 0.0143873431, -0.0275371298, 0.00370824779, 0.0400946289, -0.0699345618, 0.0727134048, -0.0723149404, -0.0699302331, 0.0495504066, -0.0667922571, -4.861530e-02, -0.0374512933, -0.051719144, 0.0701620504, 0.047514189, -0.0134236142, 4.280650e-02, 0.0326937735, 0.0267422237, -0.0752615258, 0.00527834333, -0.00945441611, -0.0270933863, 0.014766261, -0.025228221, 0.0222391933, 0.0474444106, 0.00204824121, -0.0183034334, 0.0697599277, 3.727370e-02, 0.0743803754, -0.00692705391, 0.00236436678, 0.0686906502, -0.00979860872, -0.0493837111, -0.0324743912, 0.056120161, -0.0553843826, 0.0736075863, 0.05313861, -0.0654570535, -0.0138320802], [-0.0591383576, 0.0624876618, 0.0467652194, 0.0264245868, 0.0138744656, 6.988850e-02, -0.00163904869, -0.00210959231, -0.0280983113, -0.0270577595, 0.0366780125, -0.0762633383, 0.0368366838, 0.0512634404, -0.00367642078, 0.0801606848, 0.0391255021, 0.0800604298, -0.0329187289, 0.067777507, 0.0719562247, 0.0231335256, 0.012161158, -0.00580678927, -0.0708024427, 0.0356899686, 6.008700e-02, -0.0526899323, 0.075831458, -0.0385243706, 0.0737685412, -0.0406877622, -0.0407042131, 0.0320033319, 0.0246079396, -0.064919807, -0.0622913427, 0.0620514117, 0.0558582433, -7.950090e-03, -0.00468374044, 0.00920183491, -0.0314739905, -0.0410486907, -0.0374776721, -0.0331297591, -0.0642597601, 0.0197530501, -0.036231786, -2.242120e-02, 0.0448137037, -0.028487945, -0.0579006672, 0.0767486393, 0.0287321378, 0.0257179402, -0.0254649445, -0.0184856802, -0.0594441965, -0.0310346447, -0.0485588275, -0.0308741964, -5.060340e-02, -0.0676209926, 0.0273391679, -0.00630988879, 0.048782222, -0.0739449635, -0.0510627441, 6.631900e-02, -2.3661289E-4, 0.0737573281, 0.066177763, -0.0514596105, 0.0605704114, 0.0232941248, -0.0657960176, -6.181280e-02, 0.0221337825, -0.0352556743, 0.0232710894, -0.0254292637, 0.0641901046, -0.0689867809, 0.0310981795, -0.0256387088, -0.0273930226, -0.0644995868, 0.0585236922, -0.0406170785, 0.0590886064, 0.0555011891, 0.00178045128, 0.079361625, 0.0587968975, 0.0773037075, 5.54951082E-4, 0.0127375163, 0.00893569365, -9.12531977E-5, 0.0555265434, 0.0421705544, -0.0455164164, 0.0137511343, -0.0748308598, 0.0360048749, 0.0553103909, 0.0696386396, 0.0322363786, 0.0279404726, -0.0535138622, 0.0777439102, -0.0129407737, 0.0430782549, 0.0037079535, 0.0767996684, -0.0736199766, -0.0208865516, -0.0727093294, 0.0554702953, 0.0568641834, 0.0784083754, 0.0689891278, -0.0103535149, 0.0716711655, -0.0444587171, 0.0417631119, 5.726370e-02], [0.0778036043, -0.0229201987, 0.0561243072, -0.0680991635, -0.00858139247, 0.0386064798, 0.0317475721, -0.00173460692, -0.0712467432, 0.0509820506, 0.0563675091, 0.00192841887, -0.0732200593, 0.0142293125, 0.0491517708, -0.0126254335, 0.0172402263, 0.077515833, -0.0683260038, 0.033286646, -0.0623410121, 6.751420e-02, 7.677280e-02, -0.0226993337, 0.0574294701, 0.0271079615, 4.661510e-02, -0.0423320793, 0.0582487211, -0.0512091517, 0.0683745071, 0.0736599937, -0.0723495483, 0.0191124454, -0.0576774254, -0.0682431757, -0.0521893352, -0.0779870674, 0.0382403284, -0.00947251915, -0.0499562845, 0.0247665048, -0.0291112773, 0.00870453566, 0.071933113, 0.0166143849, 0.0294994712, 0.0809451863, -0.00418554246, -0.0433999971, -0.0123272017, 0.0304693878, 0.0055508092, -0.0361567028, 0.0303449258, 0.0148517191, -0.0696609244, -0.0669827834, 0.0713268444, 0.0710864291, -0.0248653404, -0.0215221345, 0.0113287419, 0.0243346766, -0.0594462827, 0.0116155893, -0.0785519778, -0.02891884, -0.0796042755, -0.0526826754, -0.0570963696, 0.0511188433, -0.0205041468, -0.0370749049, 0.0057054013, 0.0464361683, -0.0527202487, -0.0541669838, 0.0313628539, -0.0330090486, 0.0583526418, -0.00256948173, -0.0757857114, -0.0659712926, -0.00952117145, -0.0142654553, 0.00739654899, 0.0372577906, 0.0114598349, -0.043519102, -0.0378987566, -0.0627410263, -0.0780287832, 0.0617151931, -0.0487383753, 0.0597186759, 0.0286680833, 0.0728358701, -6.847800e-02, 0.0140017048, 0.0121762231, 0.0333539471, -0.0670913905, -0.0128950328, 0.0317017585, -0.072896421, 0.0214225426, 0.0532787815, -0.0327357203, 0.0455703959, -0.061373692, -0.07688535, 0.0219240412, 0.0590990111, -0.0649978444, -0.056785062, 0.0320502892, -0.05960913, -0.0145857036, -0.0300532654, 0.0348065794, 0.00992359966, -0.00690553337, 0.0650303438, 0.0672845617, 0.0477473214, 0.0192813128, 0.0790307745], [-0.0287075527, 0.0420474336, 0.0307114609, -0.0718993842, 0.0306929741, -0.0556304939, -0.0277705267, 0.034603592, -0.0306720342, -0.057616137, 0.0299793147, -0.00238251523, -0.0183485299, 0.0314738788, -0.0789803118, -0.0323194712, 0.0587895736, -6.273940e-02, 0.0754815638, -0.0165617969, 0.0245831981, -0.0744018927, 0.0276670661, 0.0398627333, 0.00119389722, -4.272880e-02, -0.0454267226, 0.00187851896, -0.0644597188, 0.03587199, -0.0486602262, 0.0396687277, 0.00309661916, 0.00549140573, -0.063269034, 0.00171845371, -0.0773463398, 0.0162988659, -0.00534116756, -0.0488361754, -0.0734267831, -0.0708614439, 0.0428896509, -0.00984936301, -0.054053396, 0.0176741183, 0.0228989925, -7.355680e-02, -0.0108453427, -0.036332462, 0.0385677665, -0.0181190558, 0.0759971514, -7.428740e-02, -0.0208044723, -0.00595573056, -7.260740e-02, 0.0463794544, 0.0536474213, 0.0304279737, -0.0665170252, 0.0694221854, 0.0137923881, -0.0692258254, -0.0174890384, 0.0138992909, 5.348750e-02, 0.068933107, 0.0156041821, -0.0179215856, -0.0391509943, 0.0276895259, 0.0240362454, 0.0439032912, 6.008590e-02, 2.90638738E-4, -0.0750720724, 0.0513591841, -0.0471907519, 4.805690e-03, -0.0140459929, -0.0578709394, 0.0768634155, -0.00233834935, 9.75081522E-4, -0.0654089078, -0.0408931784, 0.0258712322, 0.0158824809, 0.0222235173, -0.0393395685, -0.00219843537, -0.00542423362, -0.00201841933, -0.0236479659, 0.0539355129, 0.0600050539, 0.0277195312, -0.0106208725, -0.0415275842, 0.00300356653, 0.0343677513, 0.068375498, 0.0792630612, -0.00235464028, 0.0547751486, -0.0594411567, 0.0502864793, 0.0537617356, -0.0560293831, 0.0638727247, -0.0175781846, 0.0515855476, 3.652450e-04, -0.0796318501, -0.0575254224, 0.0275009479, -0.00369581906, -0.0536266118, 0.0499706529, -0.0582048073, -0.0716881453, -0.064061448, 0.0408779047, 0.0722743943, 0.0610822514, 0.0093679959, -0.0400880612], [-0.0278698299, -0.0380155295, 0.0439332761, 0.0428628437, 0.0659491643, -0.0349539891, 0.0484812334, -0.0553046204, 0.0203310121, -0.0230744425, -0.00513250288, -0.0103157908, 0.0306006223, -0.0109497299, -0.0278757121, -0.0439566374, -0.0112883877, 0.0135034239, -0.0305009652, -0.0215614717, 0.0193765182, 0.00493799429, -0.00287494482, -0.0691228285, -0.0539956205, -0.0402633399, -0.0488887951, -0.0588673279, -0.0515562557, 0.0104936995, -0.0357137397, 0.0648101866, -0.0175501499, -0.0733250901, -0.0449337959, -0.0227677096, 0.0625157058, -0.0384700336, 0.0148698837, 0.0696329549, 0.0475983769, 0.0557878874, -2.494740e-02, -0.0447555967, 0.0710712075, -0.0500694141, -0.0378016196, -0.00229848735, 0.0431668274, -0.0180439502, 0.0470203049, 0.0796512365, 0.0487204045, 0.0346842483, -0.074154973, -0.0573190972, -0.0408520475, 0.0205895714, -0.063064076, 0.0245951023, 0.0196387433, 0.00736960443, 0.030758854, -0.0773979425, -0.0171122719, 0.0669302195, -0.0644432306, 0.0523501821, -0.0753600374, 0.0687281489, 0.034808863, 0.041700311, -0.0511029474, 0.0294769146, -0.0377612598, -6.613260e-02, 0.0646428838, 0.0523680076, 0.0593619384, -0.0750939101, 0.0165270083, -0.0603112504, -0.0798555538, 0.0389789604, 0.0246040486, 0.0332704112, 0.0774446427, 0.0576588772, 0.0153616909, 0.0518017486, -0.0221211873, 0.0541277789, -0.0624632276, -0.0269816276, -0.00349927018, 0.0460766181, 0.0312320702, -0.0769154802, -0.0134428088, -0.0527853779, 0.0248100497, 0.0189006515, -0.0487336293, -0.0466726311, -0.054795865, -0.0433536582, -0.0134694511, 0.0709842816, -0.0759896412, -0.0508896634, -0.0565127544, -0.0285615642, -0.0479155071, 0.0197301675, 0.015989963, -0.0697612911, 0.0215469208, 0.0505246967, 0.0384654775, -0.0706024691, -0.0754096657, -0.0593604781, -0.0257425774, -0.0587369539, 0.0762416124, 0.0505827107, -0.0392118879, 0.0706480816], [0.0612050928, -0.0735954121, -0.0506650843, -0.0233338233, -0.0372425355, 0.0143989231, 0.0347259492, 0.0618081763, -0.0613997057, 0.00620669266, -0.0445269383, 0.064788565, 0.0368067101, -1.37320429E-4, 0.0619762502, 0.0802474692, -0.0288624279, -0.0715283081, -0.0316248126, 0.0777053684, 0.0778746754, -0.0491310619, 0.0363428183, 0.0463291295, 0.016054051, 0.0208754893, -0.0114006232, 0.00611388031, -0.0543729104, 0.069320783, -2.306730e-02, -0.0535184257, 4.28657586E-5, -0.0730936378, -0.010665114, -0.0793185308, 3.176770e-02, 0.027568955, -0.00431186939, 0.0305073857, 0.0275706239, 0.00622988958, 0.0595025644, 0.075861223, -0.0461308919, 0.0574707463, -7.511170e-02, -0.0148547301, 0.0514778979, 0.0138022648, 0.0445123613, -0.057371892, 0.067163676, -0.0524416491, 0.0723469406, -0.0177302733, 0.00643430511, 0.0458536521, -0.077825047, 0.0192569178, 0.0384527445, -0.0596834384, 0.0405443907, -0.0626435056, 0.0195249543, -0.0745609626, -0.0207567886, -0.0759713203, -0.00752542214, -0.0571056567, -0.0388502441, 0.0339287072, 8.028500e-02, 0.00185733335, -0.0497606955, 0.00481528323, -0.0212008767, 0.0077762804, -0.0663720593, 0.0494362563, 0.0561761968, -0.00230023428, -0.0274228938, 0.0216910876, 0.027444547, -0.0453285761, -0.0361947417, 0.0382879041, -0.0601484664, -0.0682983696, 0.0303621124, 0.0324708745, 0.0264494307, -0.0113598295, -0.011829677, -0.00534627168, -0.0573110953, -0.0635563508, 0.0649430826, -0.0064416863, -0.0285815634, -0.0414339826, 0.0635071173, 0.0486200489, -0.0438973084, 0.0630866662, 0.051686462, -0.0317203179, -0.0458674468, -0.0229444951, 0.0662063435, -0.0782271847, -0.0612119101, 0.0382508375, 0.0621104389, -0.0629794672, 0.0605263449, 0.0603292361, 0.0446615033, -0.0623757541, -0.0408042818, 0.0443924703, -0.0747396647, 0.0294718891, -0.0268925391, 0.036835134, -0.0250459928, 0.0386026949], [-0.00449208636, -7.340740e-02, -0.0341430381, -0.0600239895, 0.00963514298, 0.037867181, 0.0393776931, 0.0605105832, -0.0585480146, -0.0733522177, -0.0640121549, -0.0708413869, -0.00859948248, -0.00739180949, 0.0250358135, 0.0125424098, 0.00895418785, -0.0401672907, 0.0688479096, 0.0230953284, -0.0675388947, -0.0816132128, -0.0676596537, -0.0340901278, 0.010750493, -0.0125047956, 0.0328021199, -0.0550667457, -0.0223792233, -0.0769688115, -0.0658607855, 0.059782818, 0.0399248265, -0.0197127201, 0.0555368774, 0.0673771054, 0.0145678585, 5.528850e-03, -0.0284753889, -0.0203079302, -0.0473031066, 0.0267662928, -0.0577354804, 0.0370349362, 0.0521772541, 0.0822054222, 0.00493259262, -0.0156573895, 0.0524147414, 0.0783045515, 0.0449391045, -0.014104452, -0.0172253363, 0.0723658949, 0.0180455409, 0.0582047924, -0.0617688037, 0.0267166942, -0.0154723777, -0.0485406443, -0.0642745122, -0.0542601719, -0.0122894114, 8.728060e-03, -0.0192886051, -0.039357543, -0.00763615779, -0.00424098223, -0.0196603816, -6.77309406E-4, -0.0438843369, 0.0152217867, 0.0480107628, 0.00130453415, -0.00457520038, 0.0486350954, -0.0513271205, -0.0593978092, 0.0214043148, -0.0820799321, -0.0544293672, -0.0229066294, 0.0453241244, 0.0534195937, 0.0369967483, 0.00977116544, -0.0595501848, -0.0177722257, -0.0327983201, 0.00827097613, -0.0716385096, -0.0731019079, 0.00125135109, 0.0636945367, 0.0499118976, -0.0359391943, 0.0202397332, 0.00442211144, -0.0138515607, 0.0577954203, 0.0521768034, 0.0532142818, 0.0707326159, -0.0611273721, 0.0103359474, -0.0819290578, -0.0761141479, 0.0281574726, 0.0710232109, -0.00635912269, 0.0608833022, 0.0673513934, -0.0566225052, 0.0791720449, 0.0725277886, 0.01142867, 0.00324503216, 0.00115315453, 0.0458880067, 0.0195602234, -0.0780586749, 0.00697797677, -0.0435181856, -0.0536406562, 0.0526710227, -0.0145251751, 0.0447976813, 0.0563832074], [-0.00757369818, 0.0391125828, 6.619670e-02, 0.0305239186, 0.0396242775, -0.0539251231, -0.0320973545, 0.0346932523, 0.0408102833, -0.00986982136, -0.00584036391, -0.0215320308, -0.0075106658, 0.00572453951, 0.0398263261, 0.0237735268, 0.0174880233, -0.0745601878, -0.0410243608, 0.0369119085, -0.0468373895, -0.00194746675, -0.012312267, -0.00901291333, 0.0720389932, 1.387000e-02, 0.0421733633, -0.0393924192, 0.0350103937, -0.0186497495, -0.0536153503, -0.00695870584, 0.061220929, 0.0790494531, 0.0303611513, -0.0361282676, 0.0104302187, -0.0434421599, -0.071781978, -0.0357157849, -0.0688987672, 0.0280675609, 0.0640062615, 0.0654046088, -0.0442419834, 0.0851773247, -0.0444896147, -0.0317760743, -0.0220638942, -0.0436536931, -0.0142853353, 0.00586214941, 0.0561485738, -0.0355748348, 0.0469828248, -0.0217926167, 0.0304921418, -0.040811263, -0.00287743588, -0.00560465408, 0.0326416902, 0.0103466827, 0.0312967487, 0.0509974547, -0.0554457344, 0.0285451934, -0.0853566676, 0.00589930266, 0.0640907511, -0.0530352965, 0.0805820376, -0.0164516214, -0.0462714843, 0.0572117046, -0.0495118089, 0.0079115713, -0.0392192379, -0.0630191937, 0.0760300606, 0.0599330068, -0.0575519912, 0.0470564365, -0.0748012587, -0.0258048642, 0.0026493012, -0.0603013113, -0.0564918295, 0.071327418, 0.0273601525, 0.0323013961, -0.0903106779, -0.0167849492, 0.0434297062, -0.0678375363, -0.022541251, -0.0406870842, -0.00476976112, -0.0617559366, 0.00855737459, 0.0849906653, 0.0173932239, 0.0226438958, -0.0802380592, -0.0252602883, 0.0440245643, -0.0233869143, -0.0260789376, 0.0730672851, 0.0638672039, 0.0624483228, 0.0707797259, -0.0602236651, -0.0298966132, 0.0659593865, 0.0682510883, 0.037731491, 0.014403712, 0.00221318309, 0.0743013173, -0.0442894958, 0.0739798695, 0.0226349887, -0.0726887807, 0.0770399272, -0.00263950671, 0.0409194306, 0.0365803577, -0.0565093495], [-0.0403637588, 0.0672257096, -0.0285884719, 0.101082534, 0.0240421761, -0.0281876586, -0.073332645, -0.0483833961, 1.42775665E-4, -0.0448672809, -0.00231446489, 0.0616566464, -0.0644403771, 0.0849282145, 0.0143128457, 0.0219813306, 0.0603265166, -0.044050172, -0.027526265, -0.0540581346, 0.0713473335, -0.0374069661, 0.023058353, -0.0605328456, 0.0104288431, -0.0324407741, 0.0131839495, 0.0974883437, -0.0797372981, -0.0880092307, 0.0618288554, 0.0657489896, 0.0726812556, 0.0216054078, -0.0643424689, -0.0838508829, -0.0677323639, 0.0637659728, 0.0180512834, 0.00197059987, 0.0170037113, -0.0803941041, -0.0796695575, 0.0902396813, 0.0262434203, -0.0264716148, 0.00934978668, -0.041915644, -0.0250306316, 0.0754437521, 0.0487840585, 0.0926393344, -0.0324471369, -0.0733455941, -0.0574829318, -0.00442544185, -0.0183783732, -0.0144252842, -0.035257332, -0.0263015293, -0.0787968933, 0.0173820984, 0.0604419969, -0.0710911676, -5.983060e-02, -0.0731775835, -0.0770755559, 0.0497578941, -0.0112812612, -0.0172848385, 0.00709586684, 0.0438915752, 0.0501795523, -0.0294287931, 0.0123132076, 0.00276950328, -0.00200543529, 0.0420811139, -0.0700931773, 0.0737383515, 0.0105695184, 0.0695792288, 0.028101638, -0.0574400909, 0.063673988, -0.0854395404, -0.0270992238, 0.024774123, -0.0458290242, 0.0360487401, 0.0160394702, 0.0245293472, 0.0409044325, -0.0109000178, -0.0743579268, 0.0417020619, 0.0272377729, 0.0526506379, 0.0307772569, -0.0218268614, -0.06457223, -0.0665558428, -0.0369082689, -0.0614141523, 0.0877248272, 0.0566457137, 0.052278392, -0.00123870792, 0.0649296045, -0.0621378682, 0.0436302349, -0.0425687097, 0.0782918408, 0.0233451352, -0.054491736, 0.0172304939, 0.023136016, -0.0277206711, 0.0434392244, -6.567930e-02, -0.0641638786, -0.0502935313, -0.028716797, 0.0134139471, 0.0132203875, -0.0769457519, -0.00139032688, 3.9395396E-5], [-0.0455951095, 0.102598749, 0.025376901, -0.029662624, 0.0376783721, -4.912870e-02, 0.0490967445, -0.0609704703, 0.00239146897, 0.00902080256, -0.0032505847, -0.0358857475, -0.0112224529, 0.0316948295, -0.0263387095, -0.0167339854, 0.0792955979, -0.0504994802, 0.0577962175, -9.53409413E-4, 0.00530663831, -0.0731110274, 0.0589194074, -0.0442952402, -0.0100965016, -0.0451981276, -0.0393323638, 0.0506198779, 0.0499402955, -0.0711893216, 0.031811323, 0.00816647802, 0.071747236, -0.0722788498, 0.0620802045, -0.0562526882, 0.0111491643, -9.986160e-03, -0.0721059218, -0.0250156336, -0.0386242382, -0.0470915474, 0.00299174315, -5.936380e-03, -0.102435827, -0.0533613823, -0.0137129566, 0.0161627028, 0.078443259, -0.0653707832, -0.055101525, -0.026431663, -0.00362373446, 0.0663425326, -0.0339634195, -0.0275078658, 6.755950e-02, -0.0281513426, -0.0507265255, 6.867590e-02, -0.0892354771, -0.0653120875, 0.0192122888, 0.029480949, -0.0330642909, 0.0455649607, -0.0205656402, -0.0518541932, -0.043655619, -0.0839729085, 0.0239072405, 3.35357239E-4, 0.00316268345, 0.0229136515, -0.0476507172, 0.0442031398, 0.0111368606, -0.0210251659, 0.071206808, 0.0760667548, 0.0232649371, 0.055707071, 0.0191099122, -0.067046456, -0.0502454415, -0.0566398576, -0.0944148078, 0.0419483744, 0.0746244043, -0.0574776307, -0.108605988, 0.00816104933, 0.00411998248, -0.0380343795, 0.0825731381, -0.0130052334, -0.0427193604, 0.00938895903, -0.0191658586, 4.861700e-02, 0.0371055342, -0.00268914434, 0.0660690963, -0.0414466336, 0.0804540142, -0.0170428772, 0.0214875918, 0.0291536674, 0.0147104412, -0.0270685051, 0.0375757962, 0.0389616117, 0.00434564333, 0.0437449887, 0.021649193, 0.059257932, 0.0268314574, 0.0412679054, 0.0276924334, 0.0296296924, 0.0153342113, -0.050510373, -0.0519551747, 0.0327495411, -0.0640879199, -0.0360232964, 0.0113847833, 0.0672188699], [0.025707081, -0.014855586, 0.0535462797, -0.0280414354, 0.0421810932, -0.0215993878, -0.0362316519, 0.00733225746, -0.0286632609, 0.0303915609, -7.189680e-02, -0.0171556175, 0.017207006, 0.0566105545, 4.150460e-02, -0.0296858251, -0.0195418801, -0.0103045823, -0.0236675106, -0.0206586476, 0.0184589233, 0.0507053547, -0.0460645594, -0.0591354668, 0.00825266167, -0.0271686427, 0.0292866509, 0.0417501777, 0.0664664134, 0.0477312878, 0.078977026, -0.0195261706, -0.0397677794, -0.00673681078, -0.0109631047, -0.0178503916, 0.0567686744, -0.0890222043, -0.0902639254, 0.0701377317, 0.0741007701, -0.00496451417, 0.0248474237, 0.0686906129, 0.0277353171, -0.0474311821, 0.011693717, 0.0195098296, -0.0696743056, -0.0626682043, -0.00780763151, 0.0424190573, -0.00561774243, 0.0509877652, -0.030962253, 0.069628328, 0.0785798206, -7.053820e-02, 0.0058128438, -0.054061994, -0.0446640812, -0.0259218533, -0.0592592135, -0.0701326132, -6.419920e-02, -0.00262772525, -0.0705837458, -0.0801865234, 0.00186798209, -0.00972549058, -0.0602739304, -0.0585007444, -0.0781490281, 0.104570843, -0.0226697791, -0.0722928792, -0.0542575195, 0.0363025665, 0.0578240417, 0.0444026031, 2.145240e-02, -0.027958408, 0.0457805246, 0.0407087244, -0.0517129824, -0.0576356724, 0.0145011386, -0.0575982668, 0.0551355034, -0.0402997956, 0.0132010123, 0.0496212766, 0.0181746371, -0.0144361099, 0.0831182226, -0.0382725894, 0.0305943247, 0.0238054376, -0.083143711, 0.0751727074, -0.0378313735, -0.0179793462, 0.0843488574, -5.418190e-02, 0.0639417469, -0.0362814851, 0.0163255092, 0.00428837072, -6.642670e-02, -0.06117725, 0.0624158382, -0.047263708, -0.0527211875, 0.0853231474, -0.0506761484, -0.044319015, 0.077852793, -0.0319467895, -0.0220111683, 0.0531459711, -0.0278082769, -0.0529971682, -0.0732757524, -0.0756639466, -0.0556030832, -0.0570452102, -0.0389252417, 0.057931222], [0.0328742787, 0.0930905193, -0.0553739928, 0.0106035378, -0.015395646, 3.81772086E-4, -0.0678636283, -0.0426083729, -0.0477235541, -0.0591042154, -0.0466259941, 0.0875289514, 0.0768640563, 0.0476847515, -0.0272813383, 0.00142742041, 0.0134758009, -0.0134832179, -0.0486353822, 0.0116159096, 0.0141731696, -0.0139342211, -0.0555995852, -0.0195108205, -0.0279626027, 0.00465129875, -7.301390e-02, 0.0151403062, 0.0205310136, -0.0148397069, -0.0566099063, -0.092762269, 0.0989468991, 0.0690065846, -0.0017792586, 0.0119440844, -2.364960e-02, -0.00662775105, -0.0889824256, -3.236430e-02, -0.0730272755, -0.0386016257, -0.0298174098, -0.051200293, 0.0112814195, 0.059120357, -0.0809039473, -0.0297129583, 0.0408418849, -0.0529653281, -0.0362662077, 0.0967903956, -0.0666555315, -8.88240698E-4, -0.0676827803, -0.0608206019, 0.0645881295, -0.0434258692, 0.0306881275, 0.060267657, 0.0109549686, 0.0419462584, -0.0722832903, -0.0558624454, -0.0671321899, -0.0850075408, -0.0515384264, -0.00342769828, -0.071128808, -0.06414821, 0.09636534, -0.00235897372, -0.0782120675, 0.0801760107, 0.0283526611, -0.0185084902, 0.0494990498, -0.0714738294, -0.0467058755, -0.0352678262, 0.0126438625, -6.528730e-02, -0.0291262176, -0.0887077078, 0.0634418875, 0.0615648627, -0.0568761863, 0.0383390412, -0.06433516, -0.0504267439, -0.00384686328, 0.0417056754, -0.0271719974, -0.0714479089, 0.0403829031, -0.0483401716, 0.0215838589, -0.0718476474, -0.0564647913, -0.0662008821, -0.00451715756, -0.0682140886, -0.0669651106, -0.00188492774, -0.00530864159, 0.0777160823, 0.0207818281, -0.0577966906, -0.100192823, 0.0773866251, -0.0413904674, -0.0731786638, 0.0499272048, 0.0560232736, 0.0933681204, -0.0268355403, 0.00839388184, -0.0480868183, -0.02386906, -0.00203767163, -0.0706185549, -0.021406604, 0.00173853501, -0.0608977117, -0.0628187284, -0.0118265133, 0.0676336437, -0.045438081], [-0.0373073742, 0.0544065759, 0.0238524377, 0.0128362933, 0.0202300623, -0.0140555194, 0.0423400365, 0.0628124699, 0.00163199566, 0.0319430977, -0.0237343349, -0.0197917838, 0.0571153164, 0.0516331829, 0.0496454127, 0.0546338446, 0.0274026226, -0.0198221337, 0.0362247713, -0.00770959212, -0.0741307288, -0.0335959755, -0.0417594835, -0.0369695462, 0.0432897769, 0.012292536, -0.0541142188, 0.0173339397, 0.0694284216, -0.0178928822, 0.0781796426, 0.0589621626, 0.0351519957, 0.0664279386, 0.0564882383, 0.0582673252, -0.0724286214, 2.526420e-02, 0.0579923838, 0.0498690568, -4.659150e-03, 0.0461209975, 0.0439259261, -0.0426230244, 0.0268269777, 0.057973098, 0.00256821653, 0.00603183964, -0.0538328737, 0.0884129256, 0.021817768, 0.0723391101, 0.07071767, 0.0686132237, -0.058870703, 0.0322987027, -0.0426546559, 0.0552011542, -0.0571933724, -0.00169495807, -0.0590943024, -0.0481972881, -0.0564191379, -0.010764394, -0.00957075133, 0.0329427756, -0.0452696308, 0.00514711952, -0.0737712234, 0.0310160965, 0.0426806137, -0.0522090718, -0.0256877802, -0.00724101067, 6.945510e-02, -0.0610104613, 0.0285917018, -7.505850e-02, 0.026257325, 0.0444109328, 0.0479329936, 0.0474716499, -0.0889535918, 0.0694388673, -0.0604389571, -0.0783155858, -0.00327040814, -0.032996837, -0.0477933735, -3.02869186E-4, -0.0349709168, -0.0344722942, 0.00929587428, 0.0178286154, 0.0635416731, 0.0739536881, -0.0460331105, -0.0952549204, -0.050000675, -0.0715651065, -0.0823820233, 0.00850401912, 0.0830003395, 0.0758945643, -0.0625289902, -0.0552722253, -0.0198787879, 0.0395887792, -0.104119986, -0.0489294827, 0.016672805, -0.0725895464, -0.0134889297, 0.0311217122, 0.0851019621, 0.00395197142, -0.0116637209, -0.0547580868, 0.0593059062, -0.0511444844, 0.0287053976, -0.0121453758, -0.061007861, -0.0141615719, 0.0124794915, -0.0304183327, 0.00415186631, 0.0175141264], [-7.229940e-03, 0.0280674603, -0.017132353, 0.0325645432, -0.0333197266, -0.0504048429, 0.0542673133, 0.0467207544, 3.534920e-02, 0.079428032, 6.829910e-02, -0.059765283, 0.065252155, 0.0338570289, 0.0355431065, 0.0111896778, 0.0162972137, 0.0728798136, 0.0507601164, -0.0317528434, -0.043781627, -0.0796938092, 0.0158197414, 0.0375116915, -0.0176520012, -0.0788733884, 0.074383825, -0.0127405524, 0.0521797128, -0.0767238215, -0.0822344496, -0.0635890141, -0.0532571971, 0.0150754955, 0.0516811237, 0.0934014543, -0.0400929227, -0.0195439644, 0.00659245206, 0.0590297654, -0.0617253892, 0.0433125533, 0.0520099886, 0.0245784242, -0.0452779904, 0.062106289, 0.0269680768, 0.0391466282, -0.0591873042, -0.0658945218, -0.0834148302, 0.0905688256, 0.0397968106, 0.0552667938, 0.0544598289, 0.0430090949, 0.0718353763, 0.0325369276, -0.00837848521, -0.0568194762, -0.086193867, 0.0630639568, 0.0117052458, -0.0254433583, -0.049529694, 9.523480e-02, -0.0512378104, -0.0267731808, -0.00113321177, 0.00476799486, 8.897630e-02, 0.0859287083, 0.0739741847, 0.0892050638, -0.0671157688, 0.0628221259, 0.00668945629, 0.0606244579, -0.0180380084, 0.0257083867, 0.0888960286, 0.00300409575, -0.0206364356, -0.0545465685, -0.0653256699, 0.0771594122, -0.0243937727, 0.0497535355, -0.0232720599, -0.0169870518, -0.00397945195, 0.0455870219, -0.0192414336, -0.0767203346, -4.916710e-02, -0.0353495181, 0.0525787771, -0.0311972909, 0.00888004526, -0.0261643622, 0.0617475659, 0.037339475, 0.0477020442, 0.0579228587, -0.00938636716, -0.00277550891, -0.0536338165, 0.0658891201, 0.0227449555, -0.0316528678, -0.0480946116, -0.0254636649, 0.0812501162, -0.00483744126, 0.0729816556, -0.0366368964, -0.0215406697, -0.0400958695, -0.026060272, 0.0254358966, -0.0770792887, -0.0751528814, -0.045743145, -0.0395026729, 0.0481980965, 0.021891769, 0.0719295442, -0.0300500244], [0.0475437492, 0.0148654589, 0.0669215545, -0.0035922206, -0.0347088352, -0.0378077067, 0.0636982619, 0.0697790757, 0.0837725177, -0.011720133, 0.0226617176, -0.0602032542, -0.0303325448, 0.0447415896, -0.04356369, -0.00469243899, -6.53682044E-4, 0.0182079934, 0.0767716467, -0.0774094909, 0.0254251715, -0.0272408761, -0.0315548666, -0.0529043972, 0.0446964279, -0.0984056591, 0.00417581294, 0.0417422317, -0.0840202271, 0.0842346847, 0.0704905391, -0.0261815563, 0.0117114736, -0.0587988682, 0.04834361, -0.00923896767, 0.0608658567, 0.0821354389, -0.0440043509, 0.077424556, -6.56060118E-4, 0.00282845949, 0.0181782804, 4.832600e-02, -0.0297028851, -0.0108440388, 0.0301522687, 0.0514008962, -0.0613617674, 0.0151274428, -0.0511585735, 0.0530154109, 0.0124429585, -0.0642295405, -0.0664673224, 0.0903989524, -0.00775510258, -0.0785832628, 0.0479202159, -5.77980478E-4, -6.418510e-03, -0.0237131082, -0.00610715104, 0.035420645, -0.0192220099, 0.0523388535, 0.0492824838, 6.565030e-02, -0.0179542899, -0.0314745568, 0.0295781288, 0.0323340856, 0.0350632183, -0.0421885476, 0.0104524633, -0.0234601162, -3.142130e-04, 0.0654345825, 0.0435470082, 0.0373142324, 0.0778622403, -0.00685926853, -0.0530819148, -0.0365830362, -0.0745501593, 0.0175147075, -0.025949236, -0.0479495265, -0.0833245143, 0.0775095671, 0.0389936827, -0.0197543968, 0.0508367121, 0.0437770262, 0.0555073395, 0.00420487253, -0.0764821842, -0.0562890358, -0.0275823735, -0.0208301302, -0.0809457227, 0.0229821652, -0.0572542064, -0.0203320533, -0.00802063755, -0.00847760681, 0.0581980571, 0.0906168892, -0.0342003182, 0.0576176718, 0.0135495905, -0.0659353882, 0.0195999853, 0.0971647128, -0.00458737649, 0.0421735868, 0.0201162435, 0.0304219406, 0.0580224879, -0.0616661161, -0.100937419, -0.0293537229, 0.0554406717, 0.0475283861, -0.012783654, 0.0364378765, 0.0173206646, 0.0326511301], [-0.0704863071, -0.0155738732, 0.0570231788, 0.00367373484, -0.0293683857, 0.0076039047, 0.09189336, 0.0397093259, 0.0281482972, 0.00732129114, -0.0498558097, -0.050884217, -0.00436959462, 0.090414688, -0.0252209119, -0.0650805831, -0.032933481, -0.0600585416, 0.0570586286, -0.0482662916, -0.0551046431, -0.00261097378, -0.0217114426, 0.0480231158, -0.0257824268, 0.0123386215, 0.0139544504, -0.0764558688, 0.0620913542, 0.0544273481, 0.0271130614, 0.0444551893, 0.0493033938, 0.0299956184, 0.0863161161, -0.00788970198, -0.0230359342, 0.0923331752, 0.0201295242, 0.0633450821, -5.326970e-02, 0.0154948616, 0.0316283554, -0.00503852544, 0.0412035622, 0.0403639674, -0.0832233876, 0.0607300922, -0.0295527838, 0.00161678146, -0.0114821335, 0.12366689, -0.0937742292, 0.00115314743, -0.00282383291, 0.0116173355, 0.0176588036, -0.055813387, -0.069628574, -0.0147065185, 0.037887238, -0.012943984, 0.048862204, 0.0357225649, 0.0752977207, 0.00260569807, -0.00398004893, -0.0637374595, -0.0529150516, -0.0443036854, 0.0301193092, 0.11094597, -0.0264259763, -0.0039572604, 0.0449304506, -0.00996842887, 0.0533083528, 0.0430761948, -0.0766528919, -0.0468465351, -0.0199088417, -0.0192001946, -0.0635071844, 0.0016400395, -0.0305414721, 0.0832475275, 0.0419143327, -0.0382817872, -0.0605143234, 0.0916502848, 0.0249997396, -0.0507889651, 0.0673186556, 0.00287794322, -0.038943775, 0.0220233332, 0.0517840385, -0.00202256138, 0.0471083932, 0.0437456146, -0.0638829321, 0.0666344762, -0.0223539434, -0.0721893087, 0.0577459969, 0.0168399271, -0.039513126, 0.0369344279, -8.602290e-02, 0.0273952838, 0.0838048979, -0.0423524827, 0.0454972535, 0.0893149525, 0.0993593931, -0.0266882237, 0.00738983322, 0.0500727743, -0.0343288593, 0.0613791458, -0.0257042553, -0.0197093524, 0.0241366792, 0.0109920809, 0.0111789601, 0.0501729064, 0.07813555, -0.0370298102], [-0.0294189714, -0.0303565599, 0.0136676915, 0.072243847, 0.0663532764, 0.0664125234, 0.025736073, 0.0121348323, 0.0416916199, 0.0944246575, -0.0400624797, -4.354850e-02, -0.00734627899, 0.110308081, 0.113916866, 0.0840638056, -0.055354204, -0.0481663682, 0.0755878538, 0.0124085825, 0.0608271137, 0.0655758306, 0.0105139082, -0.0761330723, 0.0759805292, -0.0374217108, 0.0536040142, 0.0539109819, -0.0330192372, -0.071565941, -0.0600906461, 0.00929860211, -0.0194664747, -8.648350e-02, 0.0617777668, 0.071652472, 0.010952035, -0.00699352286, 0.0710833222, 0.0918945074, 0.0513298884, 0.0161106791, 0.0164659973, -0.133436233, 0.0616391338, 0.0609999821, -0.0224399753, 0.0459269956, 0.0661592111, -0.040521238, -0.00616834685, -0.0220486894, -0.00134771306, -0.0567738563, 6.815110e-02, 0.00557338353, -0.00363367959, -0.0620257444, -0.0788660049, 0.100528888, -0.0724744871, -0.0877538099, -0.0187402535, 0.0411813408, -0.0351112857, 0.0439737812, 0.0645659119, -0.0698408037, 0.0596653074, 0.011253166, -0.0873177275, 0.0151480101, -0.0231582988, 0.0173595343, -0.0243125055, -0.064078033, 0.0390421264, -0.0503692962, -0.0990479961, -0.021732036, 0.0631670579, 0.0506375208, -0.103075601, 0.0322713181, -0.112783253, -0.0213969629, 0.0505770929, -0.0360639766, -0.028601734, 0.0769491196, 0.0864040851, -0.0442666933, 0.066074349, -0.00431867503, -0.0316687487, 0.0353118815, -0.00290651782, -0.0271726418, 2.88449955E-4, -0.0071339854, -0.0861533209, -0.0395064317, -0.0253762975, -0.0315433964, 0.0462489612, 0.080862306, -0.0453091227, 0.101837806, -0.0953649654, -0.0729819685, 0.0430759564, 0.0610163324, -0.00563834282, 0.0858653262, 0.0388692953, 0.0478961803, -0.0101644062, -0.0101510854, 0.0191737954, 0.00697916653, -0.0760836899, 0.0407945402, -0.0662915185, 0.0278990064, 0.010438568, 0.0676713883, -0.00720644696, -0.00106683734], [0.0210965555, -0.103325017, -0.0561988652, 0.0490993783, 0.0146226613, -0.0587732792, 0.00362552842, -0.00693468424, -0.0201659687, 0.0647175759, -0.00138964062, -0.0244395118, 0.0151669821, 0.0615525804, 0.0861009582, -0.0208329689, -0.0660990328, 0.0386625081, 0.0170135275, -0.0131654143, -0.0283102803, -0.040950615, 0.00138654874, 0.0710033476, -0.0298732091, -0.0764097199, 0.021659404, -0.0499546938, -0.09639965, 0.076511167, -0.0427397117, 0.0220684353, 0.0700863674, -0.069437243, 0.0247599166, 0.0516226366, 0.010622927, 0.0622371137, -0.0135983787, 0.00206713378, -0.0349968411, -0.0397930294, 0.0207834803, -0.0458086021, 0.0625282227, 0.0513537526, -0.0530023165, 0.065615885, 0.0490356535, 0.102772348, 0.0405492187, 0.0963717401, 0.0493887551, 0.0123832934, -0.00745434361, 0.0981731787, 0.0384912193, -0.0486244373, 0.0206115171, -0.0413108319, -0.0837831497, -0.042028863, 0.0594789237, -0.0313574076, 0.062753424, 0.114284918, 0.0175978616, 7.131820e-02, 0.0657292232, -0.0100052645, 0.0170319844, -0.00553990761, 0.013790274, 0.0483325943, 0.0721193776, 0.0139871221, -7.013680e-02, 0.0714991986, -0.01200525, 0.106289677, -0.0293833036, -0.0718623251, -0.00431496184, -0.0489543863, 0.0409839042, 0.00340208644, 0.13012743, -0.0263148062, -0.0669052302, -0.00752900494, 0.051786948, 0.0372694395, 0.00155903446, -0.100941673, 0.0455239713, 0.0571871586, 0.0222840905, -3.06061236E-4, -0.0427689515, -0.0159364957, 5.678250e-02, 0.0801184847, -0.102053687, -0.0710406303, 0.0273489505, -0.0450632237, -0.0360070504, -0.0180228334, -0.0351326205, 0.0491383933, -0.0139201088, -0.0647188649, 0.0494281054, 0.0802741348, 0.090825051, 0.0222835485, -0.0145630771, 0.0131327994, 0.00573971868, -0.0149720749, -0.0012508356, -0.0324930772, -0.0178387836, -0.0454230681, -0.0105499551, 0.0049734218, 0.0353993252, -0.0209127013], [0.00779227353, 0.0381941721, 0.0110160485, 0.0108551225, 0.0364878774, -0.0164793991, 0.0237701796, 0.0403270051, -0.0126022063, 0.0921295583, 0.0588050708, -0.00351080741, 0.0515130498, 0.105495036, 0.0858490914, 0.0319023654, -0.0359855741, 0.0358220898, 0.0583487079, 0.0647480189, -0.0978317931, -0.0575472787, 0.0282618534, -0.0631592273, 0.0798438116, -0.109446391, 0.020206375, 0.0426223502, -0.0338410065, 0.0317766108, 0.0716180578, -0.0275814775, 0.0745837912, -0.058065705, -0.078823626, 0.0263891462, -0.089003846, 0.0365970135, -0.0396056138, 0.13925536, 0.0935532152, -0.053116288, 0.0834718421, -0.0997854173, -0.0399741679, -0.0124928094, -0.0824039056, 0.0436163098, 4.37755341E-4, 0.0149103235, -0.0547781065, 0.106262088, -0.0751934648, 0.0383530185, 0.0186315216, 0.0654176771, -0.0166384857, -0.033470504, 0.0195539091, 0.00387149025, -0.082737863, 0.0108112199, -0.0794155374, 0.00741656683, 0.0177382678, 0.00516399695, 0.0431871936, 0.013569871, 0.0617477261, -0.032426931, -0.0395676531, 0.0158926398, 0.0846197456, -0.0703949183, 0.0444996022, -0.0116123566, -0.00749737443, 0.0335064642, 0.0325489566, 0.0227655489, 0.0355268084, -0.0530998744, 0.0584448092, 0.0207189806, -0.0794843882, -0.031672962, 0.0507414714, 0.0587219931, -0.0727741718, 0.0399117768, 0.0286411494, 0.019057177, 0.0659344345, -0.0773213505, -0.059404131, -0.00496639544, -0.0463776626, -0.0757626817, -0.0155327776, 0.0274583027, 0.00565219065, 0.100456066, 0.0495845154, -0.0205683932, 0.0143482825, 0.00398039306, -0.0611197241, -0.0663518906, 0.0232382827, 0.0430328734, 0.0827611536, -0.015285681, -0.0115448125, -0.0141382245, 0.0661083757, 0.0719583109, -0.056918405, -0.0145091657, -0.0607396178, -0.0796713531, -0.109758325, 0.0796490535, -0.0537605509, -6.994390e-02, -0.04056352, -0.0119081959, -0.0263201036, 0.0111546619], [0.0648983493, -0.067191869, -0.0388106853, 0.0330997482, 0.0745048076, -0.0718400702, -0.0123997442, -1.1383016E-4, 0.0354986265, 0.0230406653, -0.0548925772, 5.158690e-02, -0.100140721, 0.00154909561, 0.0860458165, -0.0250131227, 0.0757326856, 0.0727371946, -0.0110105835, 0.0939841046, -0.0860506147, 0.0420834534, -0.0102749588, 0.0060666725, 0.0918639153, -6.503160e-02, -4.563160e-02, -0.0604660846, -0.0434148721, -0.0373596251, 0.0204160567, 0.041979719, -0.0383468643, -0.0198352113, -0.0670170262, 0.0418447442, -0.0737997293, 0.0663980097, 0.0868577286, -0.00487374468, 0.0694242716, -0.0416072123, 0.0429585911, -0.0571461692, 0.0876314491, -0.0322831795, -0.037812341, 0.0888560265, 0.0474247336, -0.0442004763, 0.0564926714, 0.106713921, -0.0798861831, -0.0151892882, -0.0119602559, -0.0490840115, -0.0265665092, -0.0813834592, -0.0937615931, 0.0233793054, 0.0157314353, -0.0106531382, -0.0293755569, 0.0114648687, -0.0655314177, 0.0088304691, 0.0255824234, -0.0343875624, 0.0748919919, -0.0442331508, -0.0815960839, -0.0343471617, -0.0205468982, -0.0510129742, 0.00650099386, -0.0663808286, 0.0440515801, 0.0120572718, -0.0185958091, 0.0963047146, 0.0700689703, 0.0271999799, 0.00119149324, -0.012770351, -0.0379933454, 0.0727047175, 0.0892380177, 0.0409392305, 0.00235621375, -0.0444024093, 0.0397653654, 0.0115322536, 0.0658723339, 0.0450796597, -0.042336069, 0.0984559804, 0.0342456885, 0.0638152882, -0.0330860168, -0.0383667871, -0.0203132629, -0.0415218212, -6.503420e-02, 0.079068996, 0.0818872079, -0.0819526836, 0.0643782839, 0.0245909784, 0.0144372825, -0.0443550535, -0.0174298696, -0.00757658714, 0.0581854172, 0.0613234416, -0.00393749261, -0.0640381276, -0.0245193224, 5.846910e-02, -0.0134764081, -0.0610159412, -0.0812986567, -0.00291312695, 0.0494348817, 6.732810e-02, -6.436420e-02, 0.0377620831, 0.062181659, -0.0713539869], [0.0186390206, 0.0648453906, -0.0565520264, 0.0783468485, 0.0482880175, -0.0163257644, 0.0245478787, 0.0597832873, -0.0484274253, 0.011906364, 0.0329188295, 0.0356545895, 0.00620762445, 0.020548379, -0.0276189595, 0.0361787565, 0.0852649807, 0.0738911405, 0.00452137878, 0.0816001519, -0.0611673184, 0.030001482, -0.0501814298, 0.0218969695, 0.0936345681, -0.0545997433, -0.0172012523, -0.00808024686, -0.0435352288, -0.0338531546, 0.0186982211, 0.0157609191, 8.083750e-02, -0.0408931859, -0.0134219574, 0.087914057, 0.0289687123, -0.040240705, 0.0380132236, 0.0265945476, 0.00819871202, -0.0052101817, 0.106167652, -0.0293105226, 0.0819955915, -0.0346474499, -0.0467254817, 0.0539386757, 0.0807604417, 0.0784497261, -0.0558206029, 0.108586125, -0.0991509333, -0.0443975478, -0.0199399404, -0.0319834352, 0.0197298918, 0.0799786299, 0.020550862, 0.0373394638, 0.0360125117, 0.00666038925, -0.0839836075, 0.0114711178, 0.025398789, 0.0195156261, -0.0195586123, 0.0225170832, 0.0337470323, 0.0621893629, 0.00230098283, 0.00792182703, 0.0551211834, 0.02955552, 0.0060598175, 0.0430445634, 0.0352279656, -0.00351725565, 0.0295178853, 0.0817179679, 0.00956585258, 0.0610884242, 0.051074136, -0.0820017084, 0.065130949, 0.00549871707, -5.632310e-03, 0.00121416443, 0.0395806842, -5.892170e-02, 0.00398241077, -0.0589438565, -0.078540273, -0.0263785589, -0.0561166555, 0.0721947252, -0.00632028561, 0.0165696442, -3.050550e-02, 0.0564583205, 0.0364102907, 0.0600399412, -0.00557118701, 0.0268059038, 0.0224739574, 0.0340354964, -0.0346501693, -0.0655604154, -0.00423939852, 0.0195985325, 0.106124215, 0.0298884679, 0.0485944673, -4.7806371E-4, -0.0857084468, -0.0212221313, -0.0625169873, 0.0646826178, 0.0555833913, -0.00321640517, -0.0827622115, -0.0196850337, 0.00925313402, -0.0311230496, 0.028221231, -0.00337693631, 0.0066414061, 0.0680640861], [-0.00214339281, -2.092220e-03, -0.0218130462, 0.0302738268, -0.00639452785, 0.0703133717, 0.0070084855, -0.0523443781, 0.0183544885, 0.0431941152, -0.0563624837, -0.0644665658, -0.0175184719, -0.0110858437, -0.00135332823, -0.0209240951, 0.033060737, -0.0595127754, -4.693440e-02, -0.0151083376, 0.0485201478, -0.0493758917, 0.0527894534, 0.0178207662, 0.0341503844, -0.0530999713, 0.0770375952, 0.0410097316, -0.0343591608, 0.0031726989, -0.00223998958, 0.0345779844, -0.0546626523, 0.001527227, -0.0330840871, 0.0915042162, -0.00348111382, 0.030060349, 0.0564829707, -0.0166681744, 0.056154687, 0.0493631586, 0.0640721172, 0.086209096, 0.0870675072, -0.0297749508, 0.0593969487, -0.0481609255, 0.0757657066, -0.040615283, 0.00486460561, 0.0521356426, 0.0419397168, -0.107885532, 0.0607976131, 0.0321323052, -0.0270376336, 0.0711559281, 0.0726085231, -0.0190398917, -0.0498478599, -0.0209356975, 0.00694911694, 0.0825483798, -0.0661266819, 0.011021262, 0.0637630522, 0.0752112567, -0.00321653113, -0.0861517786, 0.0194191486, -0.0681444183, -0.0180051625, -0.0685760379, 0.0172429625, -0.0536733605, 0.073574461, 0.0760453641, 0.0456923172, -0.00322480407, -0.0146973673, 0.0237257853, -0.0449381657, 0.0309792217, -0.002871037, 0.0125425793, 0.0677412376, -3.768900e-02, -0.0801186561, 0.0157048572, -3.21197556E-4, -0.0201992411, 0.0190029368, 0.0566826165, 0.0409418568, 8.606370e-02, 0.0973068252, 0.0411955304, -0.0611729361, -0.0696706623, 0.0652859733, -0.0165991075, -0.0124203982, -0.00708427373, 0.0356141925, 0.00989941787, 0.00740740495, 0.0475741699, -0.0783427507, 0.0275250077, 0.0482160076, 0.0788485929, -0.0749376416, -0.0822099819, 0.0185739379, 0.0142951375, -0.0614876747, -0.0460019223, -0.0742684454, -0.0217968468, 0.00665989611, -0.0294047892, -0.00708621601, -0.0354593806, -0.0558459796, -0.0845711231, -0.0710812286, -0.0348866954], [0.00496650906, 0.00832862873, 0.0450459234, -0.0446697809, 0.0121853147, 0.0505535044, 0.0284477137, -0.0371970274, 0.0231794231, -0.0148236779, 7.309090e-02, 0.00389109668, 0.0328393131, 0.056397859, 0.0166204143, -0.0692035853, -0.0675036088, 0.0171680283, 0.0684250221, 0.00242564804, 0.0261793919, 0.0020523807, -0.0172710344, 0.0619270913, 0.0411005802, -0.0953109636, -0.00259968825, 0.0178751554, 0.0634928793, -0.0214696638, 0.0728021637, 0.0198074859, 0.0682039037, 0.0625836104, -0.0041655316, 0.0234404095, -0.0926440134, 0.0935970917, -0.0369548351, -0.0388911776, -0.0496046953, -0.056081634, 0.0475864448, 0.106922187, 0.0785395875, 0.0627019256, 0.013311157, -0.0298403427, -0.0605933852, 0.0738632903, 0.0460148379, -0.0170881692, -0.0629804879, -0.011914569, -0.0196529925, -0.0627723783, -1.794190e-03, -0.0112895649, 0.0216296092, -7.026230e-02, -0.073539406, -0.0141407605, 0.0340793766, 0.0490894243, 0.0302936547, 4.307920e-02, 0.0685836449, 0.0429869704, -0.00435137795, -0.0293956716, 0.0644101277, -0.0115382699, 0.0578141101, 0.0407678224, 0.0200533792, 0.0754362941, -0.0407141782, -0.00288666016, -0.0735497624, -0.0212286785, 0.027684968, 0.0667513832, -0.0563964397, -0.0362766348, -0.0760950446, -0.0122348191, 0.0557083413, -0.0479410365, -0.026055254, 0.00731154531, 0.0565103367, 0.0753420293, -0.0949038937, -0.0607862324, 0.0937262475, 8.124950e-03, -0.0427520759, -0.00788687542, -0.0101222377, -0.0870669335, 0.0289625432, 0.0219607372, -0.0512055941, -0.0609656386, 0.092540197, 3.09286814E-4, 0.0141383195, -0.0133561818, 0.0247982573, -0.0749548897, 0.0399590917, -0.021634182, -4.45674057E-4, 0.0652002767, -0.0950795263, -0.0629798471, -0.0186069049, 0.00790795125, 9.08449073E-5, 0.0070810765, -0.10699439, 0.0541235059, 0.0771290883, -0.0179117434, -0.0294082891, -0.082972832, 0.00746282563, -0.05623696], [-0.0661289468, 0.0276857838, -0.00870013703, -0.0239181537, 0.0637198836, 0.0795721337, -0.0158008039, 0.0757462978, 0.0515226163, -0.0665749386, 0.0721951202, -0.037763644, 0.00779960444, -0.0662147924, -0.0203951932, -0.0802207887, 0.0589459389, 0.0464006662, 0.00988572463, -0.0421116687, -0.060173288, 0.00227790396, 0.0368821956, 0.0445232429, -7.83278781E-4, 0.0326324217, -0.0317207426, 0.0399932265, 0.0468823276, 0.0144682694, -0.0460760221, -0.0117872078, -0.0136028407, 0.055637069, -0.00912577752, 0.0320844501, -0.0674475282, -0.063375853, 0.0397335924, 0.00669955555, 8.23127048E-5, 0.0460135378, -0.0360476933, 0.116864428, 0.0309992842, 0.0247356724, -0.0653379038, -0.00521054678, 0.082728751, 0.0668703094, 0.0732341185, 0.0585735403, -0.0670359731, -0.104335994, 0.0426931642, -0.0293708798, -0.0299213734, -0.0640553236, 0.00669316063, 0.0453625731, 0.070293583, -3.85277584E-4, 0.0203934982, -0.0446511209, -0.05304984, -0.083519347, -0.0229287557, -0.0160895865, -0.0649242699, -0.0394715853, -0.00896067452, 0.0618905127, 0.0571133271, 0.0533393547, -0.0351006947, 0.0247534122, -0.00853576604, -0.0345661789, -0.0217737053, -0.0740814582, 0.058085721, 0.0658270344, -0.0107297888, -0.0317014121, -0.0672694072, -0.052806031, -0.0361708254, -0.0100598978, -0.118051052, 0.0273451637, -0.00236734981, -0.0217239279, 3.234610e-02, -0.0264287684, 0.00696920883, 3.27537855E-5, 0.01255445, 0.006583896, 0.0221109092, -0.0418756418, -0.0161717627, -0.0702235698, -0.0338055603, 0.0438050963, 0.0850098505, 0.0457764119, 0.00518945092, 0.0291907638, -0.0686191618, -0.0426945426, 0.0168500636, 0.0526398271, 0.070303008, -0.0524069406, -0.115162872, 0.0540513285, 0.0656005442, -0.0056298743, -0.0791879594, -0.0891095474, 0.016576184, -0.0343439095, 0.0531577319, 0.0470624082, -0.0113879321, 0.00227440428, -0.0577314422, -4.690889E-6], [0.0685795173, -0.057493154, 0.0401454717, -0.0367707163, 0.0565323383, 0.0273405407, -0.0319530405, 0.0270157624, 0.0320813321, 0.0222493373, -0.013915129, -0.052721221, -0.0372019894, -0.0758965909, 0.0226864144, -0.00429242291, -0.0513871461, 0.0285097286, 0.0482189842, 0.0265439656, -0.0715032071, 0.0215995051, -0.0739587247, 6.570670e-02, 0.0106840553, 0.0382332206, 0.0622005462, 0.0603715181, -0.0545823798, 0.0287803896, 0.00469553377, -0.0534256957, 0.0595719554, -0.00761465356, 0.107609272, -4.561400e-02, -0.019455364, 0.00698195817, -0.0250099059, -0.0661838129, -0.00982620846, 0.0151177291, -0.0109881358, 0.080454193, 0.0234922115, -0.0581405349, 0.0700658932, 0.0589922257, -0.0808556973, -8.176140e-03, -0.064156048, -0.0280476902, -0.0452434421, 0.0417922363, 0.00189178297, -0.100588195, -0.00713722827, 0.00200057239, -5.628160e-02, 0.0216312651, 0.0578268394, -0.0622393824, -0.0750387162, -6.53098163E-4, 0.072021462, -0.0350244269, -7.516170e-02, -0.0504799448, 0.041651465, -0.0654916763, -0.0368060693, -0.00493683899, -0.0426741466, -0.0353260823, -0.0479160659, -0.0386345834, -0.0154039683, 0.028581487, 0.0114328554, -0.0424115211, 0.0129724639, 0.0158927515, -0.005486893, 0.031299483, 0.0664130524, 0.0288980398, -0.0510385297, 0.0412933789, -0.116091199, -0.0492528304, 0.0402370542, -0.0573419631, 0.0424358808, 0.0279870816, 0.0228687022, -0.0488813482, 0.0817765966, -0.0171538517, -0.0346830599, -0.057981804, 0.0552276075, -0.0603922941, 0.0370336361, 0.0821085572, -0.0622833446, -0.0124713862, 0.0220835209, -0.0516204871, -0.0684520528, 0.0333011895, -0.0861918255, 0.0100082261, 0.0144913103, -0.0723313242, 0.0288701486, 0.0102581987, -8.490200e-02, -0.0478076935, 0.0698825046, 0.0544928275, 0.0328989811, 0.00720603624, -0.0623717234, 0.0058666789, 0.0649957284, 0.0188218933, -0.0134484312, -0.0514398217], [-0.0732435361, 0.068780072, -0.0702349544, -0.054627087, -0.0153873907, 0.0298821107, 0.00901360158, -0.0229129326, -0.0547248721, -0.0596537739, -0.0588556267, 0.0383352861, 0.0591948852, 0.0593241304, 0.00716049037, -0.00413832208, -8.28297415E-5, 0.00912310928, 0.0733867958, 0.0785574391, -0.0736973062, 0.0442280658, 0.0496847816, -0.0261695404, 0.0578939691, 0.0891572088, 0.00704904227, -0.0668623745, 0.00386535074, 0.0768157616, 0.00388515461, 0.0714533851, -0.0567603372, -0.0111653749, 0.082473427, 0.019071376, 0.0422180705, -0.0575701259, 0.0157598928, -0.116938859, -0.0767563283, 0.0391304716, -0.0478599705, -4.661380e-02, 0.024672417, -0.0500488915, 0.0400022194, 0.0837095752, 0.0438172966, 0.011965544, 0.0251409449, -0.0574282035, 0.0314780325, 0.034702383, -0.0464211144, -0.0767886862, 0.0690172911, 0.0227347203, 0.0842921733, 0.0440097935, 0.0190348029, -0.0692582503, -0.0465026274, 0.0541666336, 0.0865222365, 0.043964155, -0.0817275866, 0.0472961031, 0.0195596442, -0.0601404496, 0.00913975574, 0.0459804684, 0.0627753362, 0.0265887678, -0.00921289809, -0.0517267659, 0.0936991348, 0.00743877934, 0.0373671167, -0.0713024884, -0.0866363719, 0.0130317425, -0.00439554593, -0.0459849276, -0.0345466919, 0.0277482662, 0.0685064942, -0.04605202, -0.0754613876, -1.19997152E-4, 0.0645609349, 0.0206583235, -0.0429830477, 0.0711543038, -0.0623529851, -0.0386366211, -0.0478378497, 0.0626459196, 0.0483309403, 0.0488475934, 0.0163648892, -0.0435955077, 0.0356732272, -0.0244676583, 0.0819277539, 0.111475974, -0.022164708, -0.0279714353, -0.0331747122, 0.0615727529, -0.07357914, 0.0301004779, -0.0588707216, -0.072164543, -0.0569262356, 0.0193287078, -0.0547285937, 0.0587548278, 0.0219420586, -0.0706194043, -0.0331559107, 0.0139652658, 0.070997715, 0.0888709873, 0.04494223, -0.0610838383, -0.0288843866, 0.0259593111], [0.0412512086, -0.0763821453, 0.0587118268, -0.0559170321, 0.0293169282, 0.0160801709, -0.00978687778, -2.31902777E-6, -0.00490224175, 0.058766719, -0.0194348861, 0.085825257, -0.0584195442, -0.0612905808, 0.0376281179, -0.0856935754, -0.0059402897, 0.0784151703, 4.30524553E-4, -0.00894060638, 0.0782768279, -0.0374667645, 0.0112567078, 0.0941220149, -0.0335284583, -0.0103662377, 0.0561781116, -0.0698624253, 9.181790e-03, 0.013301352, -0.00720326416, 0.076464057, -0.0262885056, -0.0222135279, 0.0350771919, 0.0402298532, 0.0334007852, -0.0362777971, 0.00675689103, -0.0461085029, -0.0142840538, 0.0326843634, -0.0565632358, -0.0784796774, 0.0617925934, -0.00849644933, -0.0463649221, -0.0040654717, -0.0198244713, -0.0475740023, 0.038615346, 7.414700e-02, -0.0207381956, 0.0232680514, 0.0514917225, -0.0626283735, 0.0923953726, -0.0770417079, 0.00204243045, -0.0538817123, 0.0266049318, 0.0742749348, 0.0617496483, -0.033795286, 0.0988166183, -0.0160181448, -0.0406788588, -0.00582941482, 0.0259897094, -0.0274953749, 0.00773405097, -0.0652723387, -0.0683212131, -0.0701275765, 0.0148433959, -0.0455030687, 0.0237872582, -0.0380452238, -0.0406222194, 0.0165244974, -0.0771670714, 0.0249888077, 0.0604718179, -0.0452595763, -0.0655600726, -0.0209028609, -0.0124108959, -0.0020026234, -0.0260783788, 0.0249764677, -0.0183435529, 0.0387028418, 0.0684401393, 0.00398258818, -0.0590951331, 0.0384977236, 0.067953907, -4.186810e-02, -0.0562703349, 0.0428271666, -0.0275746938, -0.0869060456, 0.0590164624, -0.0440736748, -0.0729049668, -0.0496009067, 0.101323806, -0.0130295185, -0.0794132575, -0.0353438891, -0.0737382695, -0.0461636819, 0.0288939457, -0.0388077311, -0.0442556627, -0.07798969, 0.0694142357, -0.00712201605, 0.0497770272, -0.0286990255, -0.061017245, 0.0430979505, 0.0170614123, 0.0835131407, -0.0794718265, -0.0716141239, 0.0518615209, 0.0408444889], [0.0347632058, 0.0217140242, -0.050412789, -0.0604696386, 0.0490640551, -3.184010e-02, 0.0321392715, -0.0125859724, -0.0415207669, 0.0179050267, -0.0253805891, 0.0734220445, 0.0632099584, 0.0711906925, 0.0696239769, 0.031185694, 0.00124750054, 0.0309433844, -0.0214177296, -0.0745212436, 0.0134366816, 0.0781683847, 0.0748309717, -0.0402531512, -0.0580835156, -0.0220618248, 0.0512233488, 0.0190812983, -0.0382240899, 0.0308937822, 4.98810667E-4, -0.0513865761, -0.0127862291, 0.0244892966, -0.0225826949, -0.0472428612, 0.040847268, -0.0521472394, -0.0229509026, -0.0626288727, -0.0646006166, 0.0619017407, -0.039056316, 0.0652606338, 0.0773422942, 0.033917509, -0.0539993308, -0.0161236338, 0.0288996398, -0.0532938354, -6.482270e-02, 0.0368385464, 0.0694053769, 0.0525791757, 0.0635535046, 0.0273928437, 0.066870749, 0.0400434285, -0.0319768228, 0.0631635115, -0.0217829943, -0.0350248963, -0.0133444155, -0.00834968593, 0.0154081443, 0.065087758, 0.0477586091, 0.0146220215, 0.0435506627, -0.0240894631, -0.0268567447, 0.0763937458, 0.0255502854, -6.65439409E-4, -0.0387124121, 0.0756060928, -0.0289002061, 0.0761355534, 0.0671342537, 0.0720366687, -0.0519188941, -0.018472597, -0.00148549827, -0.0134450905, -6.021520e-02, 0.00916465185, 0.0476758592, 0.0450638644, -0.088158816, 0.0701933429, 0.0435789786, 0.0605371185, -0.0413153842, 0.0639010742, 0.0509498268, 0.00689054466, -0.0661710501, -0.0744384527, -0.0436634757, -0.00475013955, -0.0467088223, -0.0225309543, -0.0273720287, 0.00950026792, -0.0457555763, 0.0705818831, 0.0818713903, 0.0259232298, -0.0637255535, 0.0764950961, 0.07604862, 0.0383802317, 7.347740e-02, -0.0471724495, -0.073541969, 0.0279006232, 0.0709836856, -0.0557800271, -0.0736545101, 0.0658799484, -0.0485432111, -0.0539044403, -0.0223109759, -0.0635909513, -0.0534030423, 0.0306147672, -0.0806663259, -0.0331885032], [-0.072604239, -0.0322363041, 0.0718835369, 0.055553928, 0.0360585637, -0.0325405076, -0.0247640535, 0.0564188249, 0.0746651441, 0.0184334405, 0.0200968813, -0.0292881168, -7.627330e-02, 0.0579355545, 0.0093261851, -0.0290489104, -0.0140426857, 0.00381706213, 0.0799820348, -0.0806313231, 0.00734304544, 0.0737313926, 0.0479288325, -0.0483285226, -0.0794542953, 0.0192772709, 0.00647555571, -0.0814222767, 0.0536022857, -2.154090e-02, 0.07530386, 0.0794770867, 0.0248655435, 0.0213839505, 0.0228440892, -0.00995074305, 0.0219417103, 0.0816779807, -0.0733656734, -0.0788996071, 0.076651372, 0.0770004541, -0.00429252302, -0.029536156, 7.337670e-02, -5.463350e-02, -0.0760797784, 0.0209787488, -0.0771514177, -0.0598733053, 0.0249633919, 0.0148065286, 0.0612281077, 0.0221229754, -0.0589322187, -0.0243169013, -0.0472831205, -0.00591097539, 0.0181559585, -0.0075400793, 9.879300e-03, -0.0801931843, -0.0599666946, -0.0657558814, 0.0138967503, -0.0186673589, 0.0356070064, 0.0171279814, 0.0594723932, -0.0379972383, 0.0625992641, 0.0774578899, 4.029910e-03, 0.0129477363, -0.075053744, -0.0321599059, -0.0338542797, 0.0284467135, 0.0717414096, -1.844129E-4, -0.034676604, 0.0616926923, -0.0689722896, -0.00864913594, -0.0548348129, 0.0603618324, 0.0772774443, 0.028404057, -0.0783475562, -0.0681734607, 0.0717317238, 0.0171999112, 0.0764225423, 0.00831522886, -0.0390502587, -0.0605106279, -0.0503924936, 0.0421670265, -0.0682118833, 0.0478858463, 0.0549258776, 0.0681354776, -0.0768637732, -0.0579025634, -0.0445194021, -5.216350e-02, -0.0131690921, 0.0124959135, -0.0724264383, -0.0211432818, 0.0261679981, 0.0700330213, 0.0761850178, 0.0115644531, 0.00310145412, 0.0469336472, 0.0706202611, -0.0225805845, 0.0661250576, 0.0233168453, 0.0569234081, -0.0223831944, -0.0710448846, 0.0168676414, -0.0464826562, -0.0303556882, -0.0412115343, -0.0101274354], [0.0696030632, 0.0800634176, -0.0699723959, -0.0163457282, 0.0136084361, -0.0305523053, -0.0343963467, 0.0487188436, -0.0255554803, -0.00636930298, -0.0685474053, -0.0316691548, -0.0635071322, 0.00224284362, 0.0427866802, -0.0426082499, 0.0441522934, -7.015730e-02, 0.0160644706, -0.0483291261, -0.0321552791, -0.0231029224, 0.0277938023, 0.0119011896, 1.319120e-02, 0.0707615315, -0.0197995808, 0.0115692923, -4.529860e-02, -0.0355062634, -0.0166229699, -0.00362492306, -0.0297268834, -0.0533937849, -1.98222246E-4, 0.0193577781, -0.0786199421, -0.0172874127, -0.0374486744, -0.0087412307, -0.0243757069, -0.0731039643, 0.0343826786, 0.00537217595, 0.0310387723, -0.0753182173, -0.0439508893, 0.0373239629, -0.0111387586, 0.0615575872, -0.0297781434, -0.0540054552, 0.012194314, -0.0583148636, 0.0508304313, -4.471680e-03, 0.0719683543, -0.0557945967, 0.0540212132, -0.027646428, 0.0314240567, 0.0585604459, 0.045156505, -0.0553177893, 0.0517383888, -0.0536577553, -0.0619346313, 0.0803440436, -0.0173872747, 0.0373591781, 0.0628753304, -0.0525462478, 0.0492469594, -0.0777234882, 0.00453529507, 0.038101159, -0.0111399936, 0.0141644403, 0.0368768163, -0.0205808841, 0.0185903721, 0.00659520272, -0.021745922, -0.0534163378, -0.0388428085, -0.0594004169, -0.0353520624, 0.0465418212, 0.0540825464, -0.0793364718, 0.0032915005, -0.0441606678, 0.00268155523, -0.0701758489, 0.0720359236, 0.0125406338, 0.00412627915, -0.0206441302, -4.35525348E-4, -0.0561518222, -0.0515897274, -0.0152150365, 0.0349076353, -2.647780e-02, 0.010672342, -0.028541822, -0.0158642866, -0.0466017649, 0.0083681792, -0.0479937382, -0.0627463311, -0.0438064113, 0.0680971369, 0.017838791, -0.0143084228, 0.0172525253, -0.0358325206, 0.0758281872, -0.0112419184, -0.0630527064, -0.0198319331, -0.00636461889, -0.0190033447, 0.0318328664, -0.0465678237, -0.0625850633, -0.0206230916, -0.0579280294], [0.0795435979, 0.0474143885, -0.00275694556, -5.848940e-02, 0.00568974623, -0.0636616498, -0.0612465963, 0.0230296571, 0.00839517451, 0.0042122337, -0.0562696606, 0.0587551408, -0.0333512202, -0.0658247918, 0.0652883574, -0.0685889422, 0.0796222761, 0.00200520456, 0.0619569607, 0.00763104856, -0.00166046619, -0.0263022929, -0.0801924392, 0.0714001805, 0.00121507607, 0.0593710542, -0.00605024165, -0.0743918493, -0.0226240903, -0.0730589554, 0.0617636405, -0.0299040563, -0.0302936211, 0.0651933625, 9.80488955E-4, -0.00623224769, -0.069477275, -0.0455742292, -0.0665918812, 0.0445947759, -0.00915534049, 4.919430e-02, -0.0275684223, -0.0638838932, -0.0462268256, 0.0132248979, -0.0583104864, 0.00985442101, 0.061359819, 0.0600512624, 0.0714381114, -0.0434297659, -0.00388344657, -0.0726441219, -0.0686398149, 0.00454417616, 0.0329994448, -0.0527357236, -0.0367731042, -0.0181512553, 0.0372217596, 0.0382068381, -0.0393775888, -0.0211455747, 0.00189993542, 0.058702819, 0.0777409151, 0.0794212296, 0.0318429209, -0.0448747054, -0.0119960224, 0.0382277369, -0.0179913715, 0.019123992, -0.0801623985, -0.0707820654, 0.025696829, 0.0789251551, -0.0443496853, -0.0785630345, 0.0232563484, -0.0512112193, 0.0564178899, -0.0321532227, 0.0522788353, 0.0354700573, 0.048561655, 0.0065314942, 0.0169622786, 0.0124102551, 0.0524255857, -0.0343924351, -0.0520119704, -0.00234761322, -0.0309406035, -0.0621397197, 0.0540979803, 0.0389598086, 0.0471643284, -0.0239313468, 0.0141203422, -0.0083287619, -0.0351493023, -0.0503887236, 0.0543165132, -0.0069831442, -0.0398104042, 0.0300210044, 0.0293078348, 0.0458405279, -0.0369163118, -0.0248187426, 0.0809802934, -0.0782538428, -0.0561771281, 0.0385940075, 0.0337563455, -0.0126982052, 0.0402238294, -0.0452014022, -0.0714986771, 0.0566057377, 0.0414172113, 0.0425331108, -0.00742648263, 0.0506892428, 1.296150e-02, -0.0749728829], [-0.0261681415, 0.0653344467, 0.0807445943, -0.078345634, -0.0173917394, 0.0220959224, 0.0723200664, 0.0205411743, -0.071464017, 6.773590e-02, 0.0508683883, -0.079992786, -0.0346199758, 0.00107371213, -0.0459650978, -0.0360450707, -0.0791648551, -0.0633052885, -0.0766486153, 7.726310e-02, -0.0625728518, 0.0453608595, -0.00460923975, -0.0307503697, 0.0783488079, -0.041385714, -0.0330718756, 8.010610e-02, -0.0383469611, 0.0486272313, -0.075175643, -0.0101885898, 0.0554224327, -0.0277881455, 0.0498825051, 0.0022709521, -0.0447936207, -0.048381269, 0.0712406412, -0.0372164212, 0.0478767641, 0.0247494262, -0.00372752128, -7.809440e-02, 0.0655299053, -0.0555630922, 0.037910182, -0.0149501646, 0.0645219237, -0.0380919874, -0.0759165436, -0.0555992387, -0.0375062525, -0.0745311603, -0.0794797465, 0.0567499436, -0.00355049851, -0.0689315125, -0.0346524045, 0.00912375655, 0.0199942701, -0.00743808597, 5.302270e-02, -0.0415172838, 0.0382278226, 0.0154985804, -0.0179337449, -0.00835233647, 0.0313811712, 0.0675901472, -0.0676191375, -0.0156314317, 0.0109363068, 0.0403699614, -0.0618027411, 0.0605106689, 0.0310842823, -0.0802130401, -0.0328672566, -0.0554194748, -0.0161172934, -0.0527893044, -0.023562653, -0.00325179636, 7.665400e-03, -0.00490467204, 0.0386103913, 0.00411093282, 0.0716410056, -0.0403872319, -0.0721187145, 2.724010e-02, 0.0114045776, -0.0633747727, -0.0674771443, -0.0428105146, 0.0214544125, -0.0318167135, -0.0129230935, 0.0523205884, -0.00124458259, -0.0771524385, -0.0708236918, -0.0247053802, -0.0661323964, 0.0418820083, -7.592790e-02, -0.0136687625, -0.0623721182, 0.0267741829, 0.0607312769, 0.0687397644, -0.0473747924, -0.0284394175, -0.0544828475, 0.0637802779, -0.0644093677, -0.0317101479, -0.0689833909, 0.0201847665, 0.0509769134, 0.0347466841, 0.01067991, 0.0271547735, -0.0556848384, 0.0616407841, 0.0240901969, -0.0466086343], [0.0797658116, -0.0180172138, -0.0500290841, 2.5869254E-4, -0.0455004536, 0.0696617141, -0.0113482643, -0.033679992, -0.0496824868, 0.00482070632, 0.0216975249, -0.0641360953, 0.0424714163, -0.0792700871, -0.0713653788, 0.0575278625, 0.0541327186, -6.60788035E-4, -0.0569991246, 0.0623617917, 0.0554973446, -0.0110883648, -0.0247526467, -0.0683926269, -0.0421682782, -0.0329840817, -0.0207229163, -0.0732423961, 0.0800293535, -0.0175564401, 0.0260535311, 0.0622528903, 0.0185156688, -0.00859034433, 0.0149050439, -0.0306072105, 0.0189048033, -0.0620573498, 0.043856062, -0.0115694255, 0.00996387656, 0.0487686805, -0.02298069, 0.019451309, -0.00543719856, 0.0666443482, -0.035775423, -0.0381425768, -0.066303812, -0.01796804, -0.0660157427, -0.0294783041, 0.0336989462, -0.0197201408, -0.00189085607, 0.0761571601, 0.0184257571, -0.0538261198, 0.078224048, 0.0724158064, 0.0646150932, -0.0444179215, 0.0525427312, 1.85420271E-4, 0.0676211193, -0.0423328169, 0.00145955081, -0.0256108232, -0.0732498839, -0.0204167347, -0.0779629051, 0.0795556456, -0.0547460578, 0.0386779346, -0.00907076802, 0.021297032, 0.0234192871, 0.0187951028, -0.0646895617, 0.0078875348, 0.0518754572, 6.563840e-02, -0.0222074259, -0.0673024878, -0.0353692397, 0.00559766963, -0.0449804626, -0.0218006615, -0.0544836037, -0.010806255, 0.0271062646, -0.0233342275, 0.0736976415, 0.0803344771, 0.061109215, -0.0291928072, 0.00459517958, -0.0591334626, 0.050169751, 0.00580580859, 0.00979948416, 0.0368632115, 0.00372480974, -0.0749944672, -0.0576729923, 0.0690190047, 0.0503343977, 0.0759662241, -0.0292574242, -0.0598130189, 4.406150e-02, 0.0299805179, -0.016833894, -0.0107743265, -0.0792886168, -0.00334640662, -0.0562746637, 0.00330055109, 0.0448767245, 6.861720e-02, 0.0268250015, 0.0166842081, -0.0760269761, -0.0781856328, 0.0628466755, -0.00860482081, -0.0192784425, -6.013560e-02], [0.0745620653, -0.0175492186, -0.0610329285, 0.0789121165, 0.0664307177, 0.0584584028, -0.06060398, -0.0725590214, 0.0518880486, 0.0683385283, -0.0282070953, 0.0732757673, 0.0629215762, 0.0403775498, 0.0775590092, 0.0163656473, 0.0330944173, 0.0529638715, 0.0391555838, -7.730060e-02, 0.012963064, -0.0750807375, 0.0784492418, 0.0675310567, 0.0630437285, -0.0452185683, 0.030748859, 0.0681335106, -0.0437160395, 0.00539643271, 0.0788585543, -0.0477470085, -0.0695045739, 0.0539252535, 0.0403550565, -0.0680740401, -0.02933871, -0.0368406586, -0.0431729071, 0.0351184756, 0.0595747456, 0.00822463166, 0.0404335745, -0.0349774361, -0.0145257348, -0.0692126452, 0.0156275965, 0.0619385577, -0.0334694162, -0.0712952911, 0.0236089081, 0.0112450803, 4.600840e-02, 0.0585061535, 5.855640e-02, -0.0728085861, 0.0599218681, -0.0335665941, 0.0313819759, -0.00195399323, -0.0549836755, 0.0193016659, -0.0347959809, -3.279940e-02, 0.00980379898, 0.00515790144, -0.0631888509, 0.00312223029, 0.0285681821, 0.00223887432, -0.0704110488, 0.061972361, -0.0766249374, 0.0507820956, 0.0211296547, 0.0667686537, -0.00430060597, 0.0201699045, 0.0797958448, 7.836130e-02, -0.0536803156, 0.0459205322, -0.0247578025, -0.0484600104, -0.0803695097, -6.935960e-02, 0.0322928354, -0.0230785031, 0.0709127784, -0.038406387, -0.0178891141, -0.0582636036, -0.0109790759, -0.0631886721, 0.0154978372, 0.00414792448, 0.0673303902, 0.0548017211, -0.0431789756, -0.0581961311, -0.0578220412, -6.288880e-02, -0.0734539852, 0.0570281595, 0.0457883216, 0.0693634599, -0.0649175048, -0.0807086825, 0.0314162448, 0.0299868584, 0.051458735, -0.0351499729, 0.0432417914, 0.00184463023, -0.0205807146, 0.0570016615, -0.0443032421, 0.00396681111, 0.0084118098, 0.0519737862, -0.0736459717, 0.0765145048, -0.0613882504, 0.0443433858, -0.00656381249, -0.0763025954, 0.00595743582, -0.0735356509], [0.0556685738, 0.0798591971, 0.0634290352, 0.045230858, 0.0574835427, -0.0469185375, 0.0194409601, 0.033885479, -0.0789510384, -0.0159300081, -0.0019942529, -0.0600471497, 0.0113039333, 0.0455926359, -0.00132444559, -0.0566644184, 0.0251756813, 0.0742249042, 0.0718199238, 2.47366726E-4, 0.0587038063, 0.0471863374, 0.00619505579, 0.0685662925, -0.0591094792, 0.0567537174, -0.0785282179, 0.00247285538, -0.0362231098, 0.046676252, 0.00864994247, -0.0606432743, 0.0546059273, -0.0401218683, -0.0560411848, 0.0540374629, -0.0523586459, -0.00429400709, -0.00487780105, -0.0279738158, -0.0466044806, 0.0404183604, 0.045820903, -0.0586472936, 0.0217188243, -0.0436533466, 0.0253333468, -0.0477739349, 0.0698405728, 0.0588414185, 0.0502240174, -0.012656725, -0.0437255725, -0.00585203385, -0.0476858728, -0.037492726, -0.0774970427, -0.0115038445, -0.0165945813, -0.00612634141, -0.0244759936, -0.0711211488, -0.0183027107, -0.0182951204, 0.0627325624, 4.479780e-02, 0.0270119663, 0.0576588437, 2.59377528E-4, 9.694170e-03, -0.0398538262, 6.496950e-03, -0.0571719334, 0.00337518333, -0.0316135474, -2.96020735E-4, -0.0630780756, 0.0521815903, -0.0425442159, 6.39065809E-4, 0.0847527682, 0.0181338862, 0.0479223579, -0.0139284236, -0.0571108684, 0.0110594397, 0.00141638936, 0.0166499019, -0.0562417433, -0.0550313853, -0.0124783684, -0.0168202296, -0.0380855277, 0.010318785, 1.707620e-02, -0.0727919638, 0.0728648901, -0.0373817459, 0.0601060838, 0.0622229315, -0.0786530449, -0.0697349161, -4.101050e-02, -6.924010e-02, 0.0252983794, -0.0712561384, -0.0702350139, -0.0519233681, -0.0357205197, 0.0705058798, -0.0694321319, 0.076619193, -0.0448637865, -0.0237173103, -0.0621686652, 0.00602890505, -0.0293058734, 0.0540397279, 0.0237064864, -0.0308554675, -0.0126788989, 0.0243449826, -0.0701208264, -6.790320e-02, -0.00680277869, -0.0546636023, 0.0388804413, 0.040780887], [0.0711520314, -0.0243909676, -0.0590536669, 0.0530340262, -0.0358825214, -0.0700736939, 0.00310390047, 0.0644328743, -0.0285567865, 0.0577000231, -0.0392719395, 8.438680e-02, 0.0472634472, 2.04747324E-4, 0.04610461, -0.0573175699, 0.00569570297, 0.0359673053, -0.0642969459, -0.0682608709, -0.0207450483, -0.0118840579, 0.0806209668, 0.0540130362, -0.046294108, -0.081385076, 0.0142142307, -0.00212486414, 0.0565679893, 0.0411892943, 0.0398053415, 0.0265436843, 0.0562471449, -0.0421139523, -0.0292889737, -0.00266967458, 0.00818053354, 0.0743319765, -0.0801860839, 0.0412328206, 0.00382163166, 0.0349141769, -6.804250e-02, -0.0135941459, -0.0128710363, -0.00615711277, 0.0583831444, -0.0151449498, -0.0177433975, -0.00105013431, -0.0376884043, 0.0286434554, 0.00500319619, 0.0388842188, 0.040690586, -0.0648446083, -0.0149347801, -0.015073237, 0.0366857126, 0.0441964231, -0.0173233971, -0.0194199532, 0.0188483074, 0.00518718595, -0.00355967903, 0.00833670143, 0.0157669373, 0.0287206899, 0.0316035226, 0.0548393428, -7.464920e-02, 0.0173253398, -0.0430671275, -0.0258989781, 0.00845876057, -0.0396569744, 0.0113290818, 0.0724554359, 0.0162743628, -0.0528782234, 0.0499116965, 0.0444462486, 0.0735649914, -0.0742550865, 0.075252004, 0.0340572521, -8.433780e-02, 0.0253983103, 0.0823808759, -0.0457842052, -0.0399756022, 0.00890011899, -0.00544649549, -0.0397061929, 3.80866491E-4, 0.0714795887, -0.0426469296, 0.0681014508, 0.0390857458, -0.0106247514, 0.0105231712, -0.0609944239, 0.0566857345, 0.0706649795, 0.0236483086, -0.0231212117, -0.0356043056, -0.0477725491, -0.0248137694, 0.0540010855, 0.0122389682, 0.0366096236, 0.0485573784, -0.0236878786, -0.0507347025, 0.0256750118, -0.00874172896, 0.0866537243, -0.027523173, 0.0521832816, 0.051863309, -0.0229793023, 0.0389696769, -0.0411843881, 4.763160e-02, -0.0032403043, -0.0602132678, -0.0131657459], [-0.0490893088, -0.0187354628, -0.0188386757, 0.0537906848, 0.043389637, 0.0145914508, -0.0714802443, -0.0420255214, -0.0760444924, -0.0805368795, 0.0118814362, -0.050240051, -0.0581294708, 0.0389392376, -0.0823601409, -0.017408656, -0.0642247722, 0.00917661469, 0.00170112646, -6.545190e-02, 0.0794139206, -0.0217357036, 0.0633063093, -0.0541592352, -0.0403500348, 0.065637499, 0.0478336662, 0.0890032947, -0.047488939, 0.0411793776, 0.0107491538, 0.0123560559, -0.0130352695, 0.0429623686, 0.0175851956, 0.0678500608, 0.054443609, -0.0940095931, -0.0313571692, -0.0363191888, 0.032282345, 0.0683041885, 0.0547111705, -0.00699766772, 0.051466018, -0.06030754, -0.032660123, -0.0495583527, -0.0782308429, 0.0300015062, 0.0549787693, -0.047556933, -0.0655752122, 4.842890e-02, 0.0352585278, 0.0600992851, 0.0704856216, 0.0335457958, -0.0665903464, -0.0073916344, -0.0303580575, -5.716190e-02, -0.0833479687, 0.0585709587, -0.0850649401, 0.0568320751, -0.00618016953, -0.0315753296, -0.0217414089, -0.0784685686, 0.0373383388, -0.025460396, -0.00559516298, 0.0647874698, 0.0150462445, 0.0222763699, 0.0607465878, 0.0386393741, 0.026289776, -0.0126256058, -0.00451200781, -0.0275567118, -0.0714934468, -0.0085894633, -0.05463304, 0.0267722197, -0.0590160899, 0.0788075179, -0.0435869433, 0.032962352, -0.056167338, -0.00760335336, -0.0244500227, 0.0374395028, 0.0623992569, -0.00954684149, 0.0309059955, 0.0663391501, 0.0378269441, 0.0735848397, -0.0880047157, 0.0781535655, 0.0519341379, -0.0300655793, 0.0335382968, 0.0317550711, -0.0897955819, 0.0327416472, 0.0453934558, 0.0499460772, -0.0441681221, -0.0273611285, -0.0633719638, 0.0796274096, -0.0439866409, -0.0303862076, 0.00743348058, 3.754700e-02, 0.0138186794, 0.0514621101, 0.00100951339, -0.0622348673, 0.0663953647, 0.0106112855, -0.0375604592, -0.045153074, 0.0714708418, 0.0170935318], [-0.0596033894, 0.0911329463, -0.015206405, 0.0834265127, -0.0449913591, 0.0497636683, -0.0909980311, -0.0381832793, 0.0216307119, 0.0179400239, -0.0399816483, -0.0298441686, -0.0537304506, -0.0391315334, 0.0232108161, 0.0248985607, -0.0807363838, 0.0734490156, 0.08408349, 0.00897606742, -0.0369447283, -6.985340e-02, -0.00132004661, -0.066826649, -0.0330793299, -0.0165710859, -0.0410129726, 0.0586109124, -0.044915922, 0.0552988313, 0.0625417605, -8.693950e-02, -7.74297921E-4, -0.0618481264, 0.0407664776, -0.0244824886, -0.0752916783, 0.0250493698, -0.0459826626, 0.0588841289, 6.313470e-02, -0.0158887412, 0.0247145332, -0.0303487517, -0.0884248465, -0.0113476943, -0.0815478339, 0.0422836244, 0.066146031, 0.0230239499, -0.0313867442, 0.0539006703, 0.00829239469, 0.0986267253, -0.0794786438, 0.0432192683, -0.0341095738, -0.0346556269, 0.0675962269, 0.0503660552, -0.0382268317, 0.0132146133, -0.0962645038, -0.00500291819, 0.00881222076, 0.0610420405, -0.0814616233, 0.0281654242, 0.0127596743, -0.0201931708, 0.0281029399, -0.0488358773, 0.0231680144, 0.00442480436, -0.065495804, -3.51093884E-4, -0.0789214968, 0.0432251245, -0.0218881052, -0.0217042062, 0.00987931807, 0.0746547505, -0.0158368088, 0.0239713937, 0.0197677463, 0.0158279873, 0.0129988184, -0.0432649814, 0.00426594494, -0.0673603415, 0.0220633671, 0.027566351, -0.00817879196, -0.0233256705, 0.0554499403, -0.0205042493, 0.0764278695, -0.0813656374, -0.0059529976, 0.0210436974, 0.0446399748, 0.0230395906, -0.0668383762, -0.0040892032, 0.0370910838, 0.042859368, 0.0190249234, -0.0374271125, -0.0562131442, 0.0828822329, -0.0239737984, -0.0676129684, 0.0595538132, 0.118577279, -0.0073359916, -0.0380821787, -0.0405142494, 0.0391789749, 0.054314021, 0.0184799246, 0.0633150414, 0.0270569287, -0.0480990447, 0.0218502823, 0.00147413835, 0.0623909831, 0.0427829064, -0.0503090397], [0.0103743644, 0.0935935154, 0.063314572, 0.0296539068, 0.0498628914, 0.0311796311, 0.0356997289, 0.0749411732, 0.00144874479, -0.0456499904, 0.00132105569, 0.033564847, -0.0626468062, -0.00639243796, 0.0245111845, 0.091914989, -0.0648960248, 0.0435590148, -0.0206102636, -0.0565567352, -0.0692604929, -0.0320288651, 0.00934200268, 0.080223009, 0.0578591563, 0.0330682956, 6.15333032E-4, -0.0431244485, -0.0689712241, -0.064210169, -0.0498011708, -8.1370183E-4, 0.0783297121, 0.0589176044, -9.92392771E-4, -0.0109134791, -0.038238626, 0.035197787, 7.711740e-02, 0.0605169832, -0.0857283622, 0.00297549157, -0.0427432694, -0.0703707337, -0.0642728582, 0.0464420803, 0.0143866623, -0.0263367593, 0.0163670648, 0.0162308458, 0.0561547466, -0.0309444573, -0.0466234609, 0.0658245534, 0.0398150198, 6.985890e-04, -0.010296396, -0.049816642, 0.0514435135, -0.0579735972, -0.0982914343, -0.0411447175, 0.0344365612, 0.00225116382, 0.00872209109, -0.0517173111, -0.00480760261, -0.0236537643, -0.0311164018, -0.0558048673, 0.0434851907, 0.0531518273, -0.0628629401, 0.0251580682, -5.67356241E-4, -0.0259837359, 0.0220917668, -0.0418337137, 0.0796151086, 0.0636370555, -0.0537538119, -0.0476120673, -0.075124912, -0.0352040343, -5.53970051E-4, 0.00348702702, 0.0528009161, 0.0892717242, 0.043048583, -0.064097315, 0.0537388735, 0.0204526875, 0.0365348272, 0.0332096592, -0.0361620337, -0.0999595597, -0.044358395, -0.00518618431, -0.0381793939, 7.154190e-02, -0.0426911712, 0.0317873396, 0.0528607741, 0.0403175652, -0.0365928784, -0.0635893196, -0.00272271736, -0.0222422369, -0.0720952079, -0.0473234542, 0.0328351818, -0.0611003153, -0.013882474, 0.0432857461, 0.00995544344, -0.0996849834, -0.0177404899, -0.037573386, 0.0356126651, 0.00494977878, -0.0745347142, 0.0727269724, -0.0618878864, 0.0176784135, -0.0173438638, 0.00887852628, -0.0227845144, -0.00809621438], [0.0241415557, 0.0845997855, 0.0587771833, 0.0719831287, 0.0486625172, 0.0138476398, 0.0745073258, -0.0239365287, -9.509410e-02, 0.0150268264, -0.065706417, 8.498120e-02, -0.0130196661, -0.0188282914, 0.0449491665, 0.0622215569, 0.0542577542, 0.0403121859, 0.01059586, -0.061515253, 0.0288828257, 0.0329304859, 0.0544445515, -0.00549218804, 0.0705474168, -0.0117408428, -0.0663777515, -0.00343933189, 0.0644719228, 0.0210187752, -0.0180703159, -0.0722424239, 0.0585926101, -0.0616889149, -0.0213978235, -0.0278412513, 0.0390504599, -0.00672689592, -0.0214056224, -0.0268676691, -0.00182313181, 0.0205967948, -0.0460903831, -0.0111565292, -0.0468886867, 0.0350496955, 0.00195036083, 3.68555717E-4, -0.0357599966, -0.00938410778, 0.062480051, 0.0854182317, 0.0168738645, -0.0509219393, -0.0622797608, 0.0227579623, -0.00486209197, -0.00600761641, -0.0576963313, 5.459220e-02, -0.0342371427, 0.0376990475, -0.0205332041, 0.0775023699, 0.018634906, -0.0768885091, 0.0254468042, 0.0216230061, 0.065888755, 0.0543022081, 0.023014782, 0.0521485433, 0.0297922231, -0.0358077176, 0.0285820141, -0.0224757455, 0.0514227375, 0.0519126616, -0.013332855, 0.0256078318, 0.0555050448, 0.042065274, 0.0503571741, -0.0136206346, -0.0690001175, -0.049940072, -0.068632938, 0.0493314378, -0.0537809245, -0.0481924713, 0.0646285042, 0.019426357, -0.0468960702, 0.0473472439, 0.0497240908, -0.00652756169, 0.0867082328, 0.058051236, 0.0434918106, -0.00760013703, -0.0348507054, -0.0376096219, 0.0789581313, 0.0363127813, 0.0380970314, -0.0496069044, 0.0574868321, -3.75135947E-4, -0.0496155024, -0.0410228632, -0.0410687402, -0.0422231108, -0.0513277799, 0.00420755101, 0.0870744139, -0.067774862, -0.0362431034, 0.0655911639, 0.0366628692, 0.0294090118, 0.044429183, -0.0503970645, 0.0384363458, -0.0749638677, -0.0616507754, 0.0219962616, -0.044454176, 0.0688332543], [-0.0295422859, -0.00120301277, 0.0649735332, -0.0581729636, 0.0541716553, 0.107870527, -0.0233885944, -0.0678150355, -1.575100e-02, 0.109471604, 0.0339651778, 0.00409983145, -0.0650431961, 0.0352285095, 8.819310e-02, 0.0136608472, 0.0690142885, -0.0479982942, 0.0901080295, -0.0991263985, 0.0483133495, -0.0486835465, 0.0472368374, -0.00744753424, -0.00435530953, -0.0170185473, -0.00759094534, 0.0746309236, -0.0597186871, 0.105162226, 0.00406832201, -0.0257273596, 0.0566352122, 0.0573460795, -0.0360324048, 0.0637035593, 0.0618922114, 0.0870753378, 0.0299657471, 0.0465537459, -0.00991929974, -0.0627315938, 0.0837807431, 0.0212589093, 0.0605899207, 0.0935071408, 0.0279922206, -0.0832427442, 0.0220755823, 0.0288667772, 0.0473833531, 0.033803802, -0.0558765903, 0.0953544899, -0.0665504187, 7.329460e-02, -0.00579554075, 0.0429705121, -0.0805713385, 0.0909090638, -0.0147999078, 0.0173252355, 0.0196221881, 0.0726150423, -0.0765983313, 0.0737268776, 0.0443536229, -3.939760e-02, 0.0232489388, 0.0765986815, 0.0608945899, -0.0494018756, -4.088070e-02, 0.0873174518, -0.0312022604, -0.0416120738, -0.026510518, -0.00554061588, 0.0224938337, -0.00890278816, 9.731230e-02, -0.0716866478, 3.772370e-02, 0.054057043, 0.0559158437, 0.0768062919, 0.030240966, 0.0182951223, -0.0126044741, 0.0314119384, 0.00312669366, -0.0316876546, 0.00150520063, -0.0958726853, 0.105230555, 4.443340e-02, -0.0437336341, -0.0805313736, 0.0127195306, -0.0368998274, -0.00557793397, -0.00704248203, -0.0213442519, -0.00954686571, 0.0875895544, 0.0438481569, 0.047480531, 0.0422299877, -0.0374857336, 0.0739572644, 0.00374379568, -0.062713936, -0.0507233739, -0.0440700091, 0.0149539271, -0.103378646, 0.00458969036, -0.0733402669, -0.060301289, 0.0903299302, -0.0346671939, -0.0861755385, -0.0485737659, -0.0223152116, 0.015269774, -0.0509733446, -0.0584801361, 0.0192046147], [-0.0085174432, -0.0334572345, 0.0636553615, -0.00145133969, -0.0348822176, 0.00761132454, 0.00805334374, 0.00529899821, 0.0701090395, 0.0392200612, -0.0706257746, -0.0343555771, -0.0549528785, 0.137013406, 0.0599719025, 0.0973504185, -0.0373175666, 0.0361772776, 0.0909574106, -0.0307374354, -0.0122681214, -0.0208041053, -0.0570799522, -0.0220649559, 0.0173953865, 0.0429565273, 0.0189342033, 0.00373047194, -0.0191209931, -0.0237629749, -0.0480509959, -0.0465135463, -0.0328914486, -0.00835763476, -0.00919429585, 0.0442593172, 0.05017896, -0.0395372473, 0.0419832766, 0.0429328308, -0.0222998466, 0.0122970184, 0.019485265, 0.0148138134, 0.0820429846, 0.0324541032, -0.120764032, 0.00353820459, 0.0433443375, -0.0476344042, -0.0936594456, -0.0061361175, 0.0144332377, 0.0766594186, -0.0369956531, 0.0600888506, 0.00723027159, -0.0448600836, 0.0358115099, 0.00499074161, -0.0997538343, -0.0653372481, -0.070775114, -0.0522727221, 0.00580435852, 0.00226042187, 0.0826084986, 0.072538279, -0.0119344844, 0.0557319298, 0.0932375267, 0.0189394187, -0.0583032966, -0.0598386191, 0.045865342, -0.0337281376, 0.0358391441, -0.026211787, 0.0324777924, -0.0416352563, 0.0620604381, -0.0333961956, -0.0153452363, -0.0672804117, -0.080468662, -0.00498302421, 0.0828081667, -0.0174649041, 0.0433684625, -0.0170677304, 0.0650422946, -0.0165212937, 0.0193830989, -0.0851554274, -0.0104267746, 0.0294858832, 0.0313967094, -0.0658533573, 0.0223662537, -0.0177071653, 4.698460e-02, -0.0367853343, -0.0299600139, -0.00280258199, -0.0595102385, 0.0432194024, 0.0168545339, -0.0320769809, -0.0622298903, -0.0441397913, -0.00609661918, -3.46310298E-5, 0.0630919188, 0.0713028461, 0.0549538545, -0.0522793122, 0.0253088698, 0.0619234032, -0.0629811063, -0.0391941257, -6.628460e-02, 0.0501322672, -0.0606480166, 0.0112020718, 0.0595536679, -0.060558252, -0.0458365232, 0.0656762496], [-0.0277137533, 7.95249798E-5, 0.0207302216, 6.305510e-02, -0.0873028934, -0.0632434338, -0.0262397416, 0.0676458254, -1.441610e-02, 0.0356850885, 0.0286775958, -0.0463866144, 0.0500769764, -0.0033683395, 0.0834905058, 0.0322505087, 0.0398218818, 0.0616553538, 0.0882168859, -0.0460911207, 0.0833402201, 0.0224706642, 0.00868108496, 0.0102429623, 0.0153696835, -3.557770e-02, -0.020071933, 0.0410152338, 0.00290070288, 5.597540e-02, -0.0236926097, -0.0438793562, 0.0358571485, 0.0145142619, 0.105814144, 0.0697622597, -0.0399820209, -0.012760723, -0.00429056911, 0.0763272122, 0.0578972511, -0.00847097393, -0.0124277631, -0.0331600644, 0.054011222, 0.0537190773, -0.111030072, 0.0500702523, -0.00871050637, -0.0184447523, 0.00382906874, -0.00119420735, -0.0748657435, -0.0460007638, 0.0599825382, 0.0974527075, 0.0422617868, -0.0402242802, 0.0503626429, 0.069374904, -0.027811395, -0.0369944721, -7.722800e-02, -0.0683269202, 0.0106139006, -0.00139933103, -0.00104742672, 0.0235201679, -0.00958714541, 0.0353507921, 6.478280e-02, 0.0254115704, 0.0668968931, 0.0417380407, 0.0780483484, -0.00375023554, -0.102243625, -0.0642383992, 0.021117676, 0.00445262156, -0.0211345777, 0.0237004664, 0.00331456168, 0.0679354891, -0.0275271218, 0.0952422916, 0.0857171341, 0.00365749467, 0.0677189827, 0.0581659488, -0.0742340907, -0.0604985692, 0.0742788911, -0.064155221, -0.0476053357, -0.0315676294, 0.0366807431, -0.0609954223, 0.0182002261, -0.0794067606, 0.0423829593, 0.0193486027, 0.0119712343, -0.0408591479, -0.07363341, 0.041502919, 0.00456018141, 0.0755672827, -0.00589962536, -0.10282544, -0.00899187382, -0.00344126415, 0.0177094769, 0.0502539799, 2.31035039E-4, 0.00853919331, 0.0256946385, -0.0601204261, 0.045280274, 0.0384395495, 0.00956256687, -0.092085734, 0.0646875724, 0.0591236949, -0.0229086895, 0.0752880201, 0.0204892214, -0.00454731612], [0.0204028152, -0.0685942695, -0.0249244086, 0.139026076, -0.0116941873, 0.00696686841, 0.00145710981, 0.0740923062, 0.00548008224, 0.0438985415, 3.809920e-03, -0.00503230374, -0.0395547859, 0.0126291756, -0.00499512721, 0.0347892717, -7.435930e-02, 0.00203932659, 0.0710916072, -0.0566316359, -0.0245185811, -0.0241238028, 0.0818483904, -0.0272954721, -0.055019822, -0.104310736, 0.0379580259, -0.024565978, -0.0810581073, -0.0195511635, -0.0931147113, 0.0474107265, 0.013710795, -0.0832697674, 0.0892266631, 0.00510596251, 0.0483179055, 0.0885002538, -0.0291099884, 0.112546273, -0.0239811633, 0.0503235832, 0.105326943, -0.120352827, -0.0424468592, -0.0211436674, 0.0289482921, 3.609870e-02, -0.0674454942, 0.101283118, 0.0184318889, -0.0140086543, -0.0569182262, 0.0332908556, -0.0668680965, 0.0190975871, 0.055731345, 0.0304146949, 0.0610403716, 0.105880432, 0.00105381128, -0.0253162049, 0.0536446236, -0.05566708, 0.0393272713, 0.0283944476, 0.0557775311, -0.030690698, 0.0373267308, 0.0831157714, -0.00492932973, 7.517474E-4, -0.0684896559, 0.0367628634, -0.0410494022, -0.0686124116, -0.0518314764, -0.00149591686, 0.0336689614, 0.0359928496, 0.0655433685, 0.00903670303, -0.0311859716, -0.0460707694, -0.0641825199, 0.0227552075, 0.0751637071, -0.040227823, -0.0710502937, 0.0562642105, -0.0392084718, 0.0852957815, 0.0360833257, 0.00449495669, -0.0728251189, -0.031849429, 0.0431351252, 0.0335481837, 0.00655119959, -0.0549667589, -0.0666833147, 0.0072969757, 0.075357683, 0.0585536323, 0.0538279712, -0.0571345165, -0.0176481288, 0.0117129618, -0.0398052782, -0.0502712056, -0.0327997617, -0.0124066295, -0.0151527617, -0.0226583686, 0.13238956, -0.0382859819, -0.00575769739, 0.0312406719, -0.0954774916, 0.0297707543, -0.040958643, 0.0421375558, -0.0486420542, -0.0316540301, -0.0788963362, 0.0560592152, 0.0816031694, 0.0529360138], [0.00195427379, 0.0334250256, 0.0952522233, 0.123537965, -0.094573982, -0.0960072502, 0.0538161173, -0.014363979, 0.0295695756, 0.066882953, 0.0773681551, 0.0703218579, -0.0523660183, 0.106915802, 0.0106348917, 0.0308303107, 0.00434972951, 0.0107237864, 0.0204552971, 0.0271245409, 0.00686561782, -0.0105497232, 0.0234235115, 8.627470e-03, 0.00416561775, -0.0246568304, -0.0413788892, -0.0345761292, 0.0645397082, 0.0418074802, -0.0864629819, -0.016298933, 0.0134348189, -0.0309608225, 0.0279876348, 0.00168597873, 0.0968781113, 0.161444932, -0.0325952321, 0.112385415, 0.0817765519, -0.0351538435, 0.107249334, -6.560610e-02, -0.00409153523, 0.0676379353, -0.0507832393, 0.0307459161, -0.0604306459, 0.101084799, -0.017051151, 0.0112049738, 0.0154043734, -0.107410789, 0.0231571961, 0.130769342, 0.029320458, -0.0800882429, -0.0127189523, 0.0171226375, -0.0527787693, -0.105426461, -0.0479297414, 0.00137427903, -0.0544525646, 0.0633459911, 0.0884202719, 0.0293713268, 0.0203609634, 0.0362767689, -0.113361441, 0.0104791773, 0.0240087416, 0.0136640333, -0.0342928916, -0.0272960477, -0.0305173639, -0.0572832525, 0.034265507, 0.0316877477, 0.0352207683, -0.0121585308, -0.108901747, 0.0223618858, -0.0497789979, -0.0133865839, 0.0764274448, 5.367980e-02, 0.0427692793, 0.0741223916, -0.0668636486, 0.046348162, -1.371540e-03, -0.0361057781, 0.0598800443, 0.100542113, 0.102784604, -0.02261623, -0.0180894546, 0.0440117754, -0.0374187157, 0.128669351, -0.0250277687, 0.038062539, -0.0346411802, -0.0398972258, -0.00611168053, 0.101129837, 7.659420e-02, -0.117723502, 0.00352053344, -0.0262044258, -0.0197133683, -0.110684261, -0.00296094338, 0.0535089932, 0.0654752329, -0.0152039574, -0.049914062, 0.0835518688, 0.0285470411, 0.0709739551, 0.0532317795, 0.0154873366, -0.0285925884, -0.0167246703, 0.0176279154, -0.0187413152], [-0.0395399556, -0.00303417817, 0.0871662572, 0.113135524, -0.0756840631, 0.0125590842, 0.0130635826, 0.0464925729, 0.107718669, 5.98705141E-4, -0.0538342558, -0.0454923399, 0.00304228463, -0.0157046076, 0.0590656698, -0.0142219067, -0.0321799926, 0.0397136807, 0.0569010265, 0.00985182449, -0.0072778468, 0.0857405588, 0.0257948115, 0.038611386, -0.00574842235, -0.109328762, 0.0989309772, 0.0132933268, -0.0433883071, 0.0413869731, 0.0409568772, 0.0192807559, 0.081112422, -0.0412978269, -0.0533592664, 0.109795377, 0.00610297499, 0.0963174328, -0.0451313779, 0.0501161069, -0.0178860016, 0.0734491944, 0.0333401561, -0.0789675116, 0.0958276093, 0.018343864, 0.0566945672, 0.08652848, -0.0403432176, -0.0465113558, -0.0115994131, 0.0189904831, -0.0763812661, -0.0249327868, 0.0850807353, 0.00118098664, -0.0697874278, 0.0181615539, -0.0370591059, -0.0511760972, 0.0411323085, 0.0312524959, -0.0278369635, -0.055799298, -0.0626418218, 0.032264512, 0.0620318986, 0.0277452171, -0.00653835107, 0.0162943192, -0.0490123816, 8.44247465E-4, 0.0415227488, 0.0372870602, -0.0645313262, -0.0492338054, -0.00717118382, -0.0414535441, 0.00537188072, 0.118702434, -0.0201102924, -0.0229869187, -0.0821321979, -0.0298050251, -0.097842209, 0.0433391258, 0.033611767, 0.0667648464, 0.0117725292, 0.0998621582, -1.76838919E-4, 0.0445806459, 0.0618436523, -0.015493663, 0.00160715205, 0.0554445274, 0.0840188562, 0.0478319228, -0.0139192715, 0.0456879586, 0.0526468083, 0.102869377, 0.0327911638, -0.0295695774, -0.0738885328, -0.0947107896, -0.0864064171, 0.00292353122, -0.0314349607, 0.0402472839, 0.0509148277, 0.0332261547, -0.0451427624, 0.0306841545, 6.35005126E-4, 0.00943310279, 0.0877952575, -0.0220591091, 0.0123453103, -0.00473767705, 0.0401805751, -0.0592447296, 0.0866527333, 0.0709751174, -0.0447171517, -0.0424131081, 0.0705395117, 0.0117243417], [0.00971187558, -0.0622086264, 0.0392387509, 0.103520744, -0.0700018778, -0.0681390613, 0.0663423613, -0.0275446679, 0.115982912, 0.0854572951, -0.0634072945, -0.045103386, 0.0303850155, 0.132201508, 0.101040602, 0.00697713904, 1.8406281E-4, 0.0146761416, -0.0307655036, 0.0502158329, 0.0250336584, 0.0998433157, -0.050677523, 0.011685906, -0.0400337055, -0.111008443, 0.0132979583, 8.751860e-02, 0.0100147901, 0.115399331, -0.0300195105, 0.00845893379, 0.0245715883, -4.542570e-02, 0.0213599149, 0.104015775, 0.0455661267, 0.0807128176, 0.0895886048, -0.00150631019, -0.0407919735, 0.0706678554, 0.0984601527, 0.00141395465, 0.0901022702, 0.0253751203, -0.025015777, -0.010733759, 0.0218069144, 0.00462449621, -0.0489743687, 0.0328693129, -0.0933745801, -0.0935284346, 0.12069764, -0.0120600378, 0.0484278947, -0.00837561488, -0.0761188566, 0.0753390342, 0.0362307914, 0.049360849, 0.00162443658, -0.013942875, 0.0236603357, 0.0228645802, 0.118568771, -0.0203524698, 0.0235649478, 0.00392024266, 0.0303305387, -0.0241772272, 0.0176964123, 0.018803386, -0.0223777331, 0.0106838318, -0.070925191, 0.0248955227, 0.100670837, 0.0858143121, 0.0638694316, -0.0961346402, 0.0112765515, 0.0419472307, -0.119273968, 0.0961764156, 0.0920321047, -0.026622111, 9.35858523E-4, 0.0153471585, -0.00812594592, -0.0157212224, 0.0891846865, -0.0978758856, -0.021437766, 0.129430473, 0.0489374176, -5.737210e-02, -0.0463887118, 0.0764969811, -0.0563597083, 0.0582085848, -0.0619740188, -0.0367887244, 0.0591002554, 0.0101103634, -0.0463674776, 0.00737132598, -0.0314304978, -0.00282019656, -0.00781654473, -0.0675286948, -0.0724502951, 0.0286681764, 0.0401632637, -0.0778820812, -0.0223865658, 0.081265375, 0.00539341196, 4.38843737E-4, -3.310380e-02, 0.0494975261, -0.0324342661, 0.0126908943, -0.0587875545, -0.0124579826, -0.0159006063, 0.0758958384], [-0.0406903774, 0.0217825398, 0.0255346242, 0.0939841344, -0.0250927303, 0.0073364824, 0.00112220331, 0.0532573536, 0.0582837798, 0.0647993833, 0.0258129742, -0.0683903918, -0.0226582605, -0.00835914258, -0.0208035409, 0.0159577262, -0.0704818591, -0.0654405951, 0.0619210787, 0.0799127146, -0.109464109, 0.0389827825, 5.016540e-02, 0.0749058351, -0.00860584248, -0.121474884, -0.072808288, 0.0156449117, 0.0489086807, 0.104802079, 0.078122057, -0.0374461263, -0.0424375534, 5.246610e-02, 0.0472364426, 0.0841805264, -0.0310008787, 0.0876462534, 0.0785332173, 0.0433455892, 0.0440754667, -0.075677149, 0.0350078754, -0.026054943, 0.0201937966, -0.0364291407, -0.00728399679, 7.537310e-02, -0.0156167271, 0.0738193988, -0.0626477301, 0.0498118922, -0.0619235486, -0.0208375156, 0.120524235, 0.0561525747, 0.0559318773, 0.0306985062, 0.0256795287, 0.00895656272, 0.0439828336, 0.00640119566, -0.0651714578, 0.0877880305, 0.0569480024, 0.0278339591, -0.00962353405, -0.0528396405, -0.00435677636, 0.0841958299, -0.0647308752, -0.057385277, 0.0544138104, -0.0829016268, -0.0117992898, -0.026637366, 0.0102184229, 0.00531682093, 7.703480e-02, -0.0137467459, -0.0252713431, 0.0149666928, 0.0157509074, -4.796570e-02, 0.0356602892, 0.0531546213, -0.0180048738, 0.0269536767, -0.0232079644, 0.0146424901, 0.0673756376, -0.0293084048, -2.15330918E-4, -0.0355772115, -0.0204243232, 4.115610e-03, 0.129950196, -0.0336400867, 3.544690e-02, 0.0453508943, 0.0408453457, 0.0513148345, -0.0520724095, -0.0104969293, 0.0958582237, -0.0830732658, 0.0308089983, -0.074930124, -0.00149196223, 0.030140087, 0.0305378549, -0.0314509273, 0.0596994162, -9.03902866E-4, 0.00562632782, -0.00499424851, -0.0251892675, 0.0162602197, -0.0299397055, -0.0162288379, -0.0576007329, -0.0234042015, 0.0603762567, -0.0379673839, 0.0793991684, 0.00220841635, -0.0368727036, 0.0772804469], [0.00252832775, 7.302790e-02, 0.0331275314, -0.0224061273, 0.0325430594, -0.0418815427, 0.090053685, 0.0536028109, 0.0806623846, 0.0166327432, -0.0606550723, 0.0365250073, 0.0660150349, 0.125327557, 0.0886712372, -0.0403928831, 0.0133632794, -0.0493490137, 0.118012734, 0.0965170487, -0.0584082343, 0.0919558703, 0.0254804511, 0.0832661539, 0.00236158306, 0.0371357389, -0.0401675068, 0.0735036582, 0.0860878154, 0.0808754712, 0.0743647665, 0.0642634407, 0.0424771868, 0.0431808345, -0.00191464787, 0.043413423, 1.85887853E-4, 0.00514293788, -0.0132065369, -0.0263380017, 0.00867517199, 0.0526395887, 0.0854162052, 0.0826558396, -0.0359525308, -0.0595423393, -0.0130585544, 0.00654534204, 0.0568805672, 0.0923478975, -0.0150619447, 0.113905407, 0.0407022648, 0.042821411, 0.0117629627, 0.0238297097, 0.073916778, -0.0314190537, 0.0780914798, 0.0234384909, 0.0733301416, -0.0640559271, -0.0847688987, 0.00668195775, 0.0413065031, 0.0704228729, -0.0370093659, -0.0144235883, 0.00229007564, 0.0541631132, -0.049135454, 0.00331805577, 0.0657080486, -0.0511075854, 0.0243920982, 0.0286653489, -0.0893821939, -0.0667131618, 0.0965103879, 5.569340e-02, 0.00314518088, 9.573490e-03, 0.0512176789, -0.0185051318, -0.0646732897, 0.0042867451, 0.014655388, -0.0195308328, -2.991540e-02, -0.0863429457, 0.0479483195, 0.0518941917, -0.0398754254, -0.0589557961, 0.0523914061, 0.0972035899, 0.11139112, 0.0811853558, 0.0819146931, 0.0490748733, -0.0317401774, 0.0677484274, 0.00825840328, -0.0031533672, -0.0150971804, 0.0275214892, 0.0232922919, -0.0557741039, -0.034607362, -0.020431133, 0.0165374447, 0.0220346376, -0.00823371484, -0.0104557723, 5.307330e-02, 0.00653256848, -0.0188399199, 6.851360e-02, 0.00356647233, -0.0143624349, 0.0304095149, -0.0807884261, 0.0262771584, -0.0687836781, -0.0650135204, -0.0143862339, 0.0396418199, -0.038178172], [0.0540943518, -0.0198065564, 0.0228516478, 0.00366940699, -0.0102769211, 0.0486695059, -0.0656879619, -0.0151274661, 0.0145176202, 0.0557144433, 0.0611719936, 0.0612038225, 0.0314410105, -0.0255210679, 0.0957879051, -8.01601563E-4, -0.00329003064, -0.0399599262, 0.0767250732, -0.0540356785, -0.0503318384, 0.0496606268, -0.0535683557, 0.0828174427, 0.00773449196, -0.0709837601, 0.0285213199, 0.0590284914, 0.0552796796, 0.0487853549, -0.0168386344, -0.0135392062, 0.0672024861, -0.0601398088, 0.0449164286, -0.00179324555, 0.0464426726, 0.10387633, -0.0499663502, -0.0233309679, -0.0369539075, 0.0580700189, 0.0714207292, -0.013116939, -0.00126003427, 0.0754300877, -0.0564619973, -0.0399318226, 0.0656793565, 0.0590781905, 0.0340199508, 2.69196054E-4, -0.0913397669, -0.102381431, 0.0754564628, 0.00943672284, -0.0242761262, 0.0337784439, -0.058610566, -0.0583865754, -0.040840812, 0.0146488473, -0.0273444913, -0.00776450196, -0.0551800728, 0.0164268836, -0.0264905468, 0.0419637561, 0.0189770851, 0.0353226475, -0.0373436809, -0.019920107, -0.0225393735, -0.0558034517, 0.0177531466, 0.0736605674, -0.0121854795, -0.0512902141, 0.0636206716, 0.0132771786, 0.00263341586, 0.0195125546, 0.00785708521, 0.0332421772, 0.0203992147, 0.0893138125, 0.00337357935, 0.0594754629, 0.0117374454, -0.0823604986, -0.0300205275, -0.0528319925, -0.0849352702, -0.0121800518, 0.0347883962, 0.059672527, 0.0944999232, 0.0667182803, 0.0232943315, -0.0194504932, -0.0830100328, 0.116100699, -0.0241461638, 0.00298960484, -0.0381671973, -0.0386070386, 0.0504231602, -6.351430e-02, -0.0651047677, -0.0482819825, -0.0086524738, 0.0495506451, 0.0322114974, 0.0482093841, 0.0202355217, 0.0165827144, -0.0556372553, 0.0539040677, 0.0116031282, -0.00325595471, -0.0263198018, -0.0501530208, 0.0918886438, 0.0518082157, 0.071938388, -0.0518998764, 0.0787296444, 0.0168613289], [-0.0550581962, 0.084158875, -0.023403598, 9.006520e-02, -0.0557635725, -0.0711797848, 0.00173202844, -0.0374845788, -0.00958318356, -0.0184705947, -0.0268071275, -0.0731244087, 0.0578924268, 0.0624340474, -0.026232047, 0.0355091393, -0.0422021709, 0.00510382745, 0.0409385934, -0.0187206939, -0.0160141531, 0.0451400019, -0.0754831284, 0.11531201, -0.0132714203, 0.060725864, -0.00365527882, -0.0434207208, -0.0122072194, 0.0269723311, 0.0238789618, -0.0717949793, -0.0125172995, 0.0348912068, 0.0990307405, -0.0478677042, 2.948230e-03, -0.0103498977, -0.0247512739, -0.0189603325, -0.0284004658, -7.69417733E-4, 0.0230326671, 0.00472824555, -0.0320823155, 8.906370e-03, -0.0902746841, -0.0123145506, 0.0229147822, -0.0155604482, -0.00199528481, 0.0880136341, -0.0547510944, -0.0752954185, -0.0415930636, -0.0703904852, 0.0548410453, -0.0819704384, 0.012152846, 0.0408850908, 0.095596008, -0.0296049267, 0.00400118763, 0.00486995326, -0.0543323681, 0.065188624, 0.0496302657, 4.583680e-02, -0.0615836196, -0.0823960527, -0.0574432313, -0.0803607329, -0.00679671532, 0.0184384547, -0.0576444827, -0.0249533318, -0.0545194522, 4.303230e-02, -0.0543536283, 0.0234052259, 0.0876879692, 0.0909713283, -0.0496135615, -0.0339040682, -0.0735668838, -0.00530280871, 0.0277249794, 0.0394202471, -0.106456824, -0.0775883197, 0.105547719, -0.0429554321, 0.0423339866, 0.00191509374, -8.456520e-03, -0.0142495148, -0.0458144099, -0.0251622684, -0.0068011377, -0.0417801812, 0.0667782649, 0.0755604207, 0.0887388214, 0.0821450129, 0.0924079269, -0.0161103401, 0.00993721187, 0.0205596425, -0.0254930817, 0.07008636, 0.0544989556, 0.0034671349, 0.00787825789, 0.0606904253, -0.0191379525, -0.0217519235, 0.00671111094, 0.0818289294, 0.0243535805, 0.0589032471, -0.0339534171, -0.0656173378, 0.00503293052, -0.0469755307, 0.0316630304, -0.073057279, -0.0148024838, 0.0449201912], [-0.0415265076, -0.0619542748, 0.0404532552, 0.0578748696, 0.0948906466, 0.0128393359, -0.0741963908, 0.0215923805, 6.481530e-03, 0.0420361236, 0.0143103227, -0.0651189685, 0.0375987142, 0.0381341875, 0.0241264477, 0.0148856938, 0.0702652782, 0.052415356, -4.239390e-02, -0.00820667576, -0.0465322658, -0.0225619543, -0.0602196828, 3.983280e-02, -0.0434197299, -0.0437607132, -0.0456062108, 0.0549968481, -0.0373496935, 0.0883446708, -0.0152080599, 0.0803869739, 0.0774782151, 0.0203505624, 0.050495483, -0.060361743, 0.0226025097, -0.0399308279, 0.0607970618, 4.331750e-03, -0.0781188532, 0.0375317149, -0.0404603072, -0.0288775954, -0.0612475425, 0.0742188245, 0.008523033, 4.566620e-02, -0.0427310914, 0.078015834, -0.0211512037, -0.0351791345, 0.0198952984, -0.0230140612, -0.0889741852, 0.0525273569, -0.0135402093, -0.0100570358, 0.024871001, -0.0720405132, 0.0857782438, 0.0105278064, 0.0703553855, 0.0589102209, 0.00857494957, -0.033009585, -0.0656242594, 0.0118105477, 0.0178244356, 0.0191871487, 0.0198891722, -0.0392520763, -1.32731759E-4, 0.0538561381, 0.0623949394, 6.149350e-02, 0.00574774574, -0.0749539509, -0.0283993836, -0.0778902918, -0.0431569405, -0.0557055809, 0.0473295823, -6.34287483E-4, -0.0379281342, -0.0297960192, 0.0439969823, -0.0461027548, -0.0708093569, 0.0179006327, 0.0769114792, -0.0212663729, -0.0289845318, -0.0589026809, 0.0627882778, 6.240160e-02, 0.019373307, 0.0369696245, 0.0878751873, -0.0347195119, 0.0569571778, 0.0649621486, 0.0438003242, 0.0698620528, -0.061513096, 0.0289305747, -0.0656600818, -0.0613489151, -8.325610e-02, 0.022112757, -0.0454580747, 0.0392519534, -0.0867554545, 0.00200960529, -0.0676371157, -0.0605715439, 0.0768852383, 0.021851344, 0.001948199, 0.0351526253, -0.0554935411, 0.0144007439, -0.020473903, 0.00671235658, 0.0772013664, -0.054917898, -1.910580e-02, 0.00125698512], [0.00961141474, -0.0520126484, 0.0769525766, -0.0614490584, 0.0647151545, 0.0198683087, -0.0229394622, 5.040760e-02, 0.0236388203, -0.0536425896, -0.0070688976, -0.0183190685, 9.138940e-04, 0.0181458462, 0.0292612556, 0.0520474836, -0.0625323728, 8.272810e-02, 0.00134489709, 0.0174829308, -0.0800826549, -0.0496713929, -0.0554037318, 0.0518729165, -0.0630646497, 0.103027165, -0.0271914918, 0.0593499839, 0.0804689228, 0.0253655352, -0.0428874306, 0.0269684512, 0.0166435651, 0.0996925607, -0.0449798815, -0.0374025032, -0.0277487095, 0.0440606661, 0.0444576889, 0.00821865163, 0.0299590174, 0.00526033062, 0.00170627376, -0.00186916196, -0.00769252563, 0.00762437424, 0.0589681603, -0.0144241201, 0.0771010295, -0.0297942683, 0.0232845191, -0.0105053047, -0.0544599071, -0.0808938965, -0.0510134734, 0.0266909078, 0.0898497179, 0.00258951983, 0.037257988, 0.0407770164, -0.0326012895, -0.00482869102, 0.0153888585, 0.0339274891, 0.0289766621, -0.105987422, -0.0114072859, 0.0440066643, 0.0620527416, 0.00801037345, -0.0460946336, -0.0775992721, -0.0107482625, 0.0616693534, 0.0612546503, -0.0176886972, 0.00522122113, -0.0655908585, -0.0228054728, 0.00197659456, 0.058158461, 0.104126908, 0.0872340947, -6.244070e-02, -0.0556020401, 0.0266841073, 0.094900377, 0.0742541477, -0.140440181, 0.0150687061, 0.0725910142, 0.0434243158, 0.0500501767, -0.0165078212, 0.0193524119, 0.0773497596, 0.0573388338, 0.0572343767, 0.0595620722, -7.222170e-02, 0.0485233366, -0.00290858326, 0.0133203585, 0.0118814716, -0.0676986501, -8.834620e-03, 0.0138611319, 0.0448671319, -0.0761831104, 0.0145407924, -0.0669967607, -0.026423838, 0.0231113937, 0.0561387576, -0.0684727803, -0.022304235, 0.0445094593, -0.0268511213, 0.0333293974, 0.0572303757, -0.0228688736, -0.0395217501, 0.0366147384, 0.0655805096, 0.0139996633, -0.0786502659, -0.0813799798, 0.0291857291], [0.00465647597, 0.0369333737, -0.0417760499, -0.103093453, -0.0362685248, 0.0297370423, -0.0229045935, 0.0640670583, -0.0699042231, 0.0466373451, 0.0209560152, 0.015955355, -0.0113066332, 0.0244567636, 0.0433649905, -1.020010e-01, 0.0593098439, 0.00673942873, 0.0896623507, -0.0419846587, 9.8102726E-4, -0.0239735413, 0.0253426377, 0.0735378712, 0.0128799174, 0.0511141345, 0.0297372565, 0.0178719237, 0.0864746645, 0.0182237085, 0.065758355, -0.0191730969, 0.00974770262, -0.0329947881, 0.0303416345, -0.0534533933, -0.0304406937, -0.0336926393, 0.10152892, -0.00574619323, 0.06233019, 6.344500e-02, 0.028237151, 0.0603280663, -0.00902986806, -0.0545732155, -0.0465204529, -0.0596083291, 0.00337983877, -3.682450e-02, 0.0373797677, 0.0318237804, -0.067419365, -0.00742512382, -0.0853967518, 0.0178315546, -0.0521202497, 0.0740310847, 0.0768149495, -0.0606429726, -0.0345088691, 0.0649544373, -4.132940e-02, 0.0662832931, 0.022369016, 0.0150186224, 0.0471156389, 0.0341954976, -0.0840905383, -0.00601799134, -0.0225157849, -0.0244075544, -0.0515815951, -0.00599598605, -0.0640354827, 0.0363402218, 0.041706387, 0.0289273113, 0.0685685724, -0.0459456705, 0.00412860233, -0.0294538289, -0.030782083, -0.0778704137, -0.0100157689, -2.188870e-02, -0.0095167309, 0.00219527516, -0.046144098, -0.0202989765, 0.084853135, 0.009959599, -0.0963888242, 2.511840e-02, -0.0492783263, 0.0132822227, 0.0103442008, -0.021652475, -0.0258781854, -0.0333423913, 0.0359670296, -0.0942641571, 0.0752922744, 0.0974344685, -0.0166662391, -0.0302242395, 0.113025054, -0.0468284227, -0.05191366, 0.0767859295, 0.0599669181, -0.0214098059, 0.028450558, -0.0346588418, -0.104216635, -0.0359475426, 0.0377339497, -0.0273335557, -0.0114981923, -0.0588095188, 0.00462795468, 0.0504383892, 0.0691992342, -0.0122519741, -0.0588010177, 0.0598394796, -0.0592600107, 0.0376116298], [-0.0441235751, -0.0630403385, -0.00963668618, 0.0137402611, -0.0584785193, -0.0104571944, 0.0694404542, 0.0512944199, 2.082570e-02, -0.0569994375, 0.0409926176, 0.0329548232, -0.00139887189, -0.0014146762, -0.0674594268, -0.038376715, -0.0753875226, -0.00554705178, 0.0544913933, -0.0331814364, -0.0184598956, -0.0681472421, 0.00336210243, 6.780540e-02, -0.0429479964, 0.108139314, 0.0773283467, -0.0503895432, 0.0514248349, 0.0850312411, -0.0226960983, -0.0610869937, -0.0525252521, -0.0319079794, 0.0148813128, -0.0615681261, -0.0836171805, 0.0408746041, -0.0116824005, -0.0134323407, 0.0403621979, -0.00253666379, 0.0265295748, -0.0902370139, 0.0429680496, 0.0288486872, -0.0776192322, 0.0772629678, -0.0640064254, -0.0683501288, -0.013724043, -0.0364580974, 0.0380529761, 0.0413745269, -0.0205868576, -0.0743181482, 0.0176844224, 2.652050e-02, 0.00878028292, 0.0591483936, -3.161150e-02, 0.0101851067, -0.011449107, -0.0190076698, 0.0736678839, 0.0202166438, -0.0655149892, 0.0802325159, 2.13813764E-4, -0.0302010234, -0.0281990115, 0.00530713657, 0.0501356311, -0.0544404499, -0.0752510503, -0.0322685912, 0.0865332484, 0.0437226258, 0.010538388, 0.00447669066, 0.019221874, 0.0275101606, -6.623840e-02, -0.0809220821, -0.065313965, -0.0376330875, 0.114681721, 0.0138647333, -0.0984054803, -0.0256910138, 0.0486466102, 0.0636342913, -0.0728654638, 0.0501520149, 0.0626905188, 0.013566195, 0.0757876113, -0.0886052698, -0.0527251549, -0.00987409427, 0.0215949509, 0.0448027551, -0.0233601946, 0.0971478968, 6.639910e-02, 0.026836358, -0.0193479806, 0.0308421794, -0.0715228096, 0.0679871141, 0.0179715827, -0.00149539113, -0.00302041881, -0.00949728582, 0.0370125324, -0.0572083741, 0.0135118691, -0.0380711108, 0.0697770416, 0.0770827606, 0.00639287615, 0.044618044, 0.0887247324, 0.089677155, -0.0175857842, -0.0658869594, 0.0433916263, 0.0636911467], [-4.484630e-02, 0.0130631207, 0.012690587, 0.0520459414, -0.0421454832, -0.0470522158, 0.0370294042, 0.0752211958, 0.0132371737, 0.0531496704, 9.041640e-02, 3.849130e-02, -0.047156658, -0.0358362757, 0.0321136378, 0.032206893, -0.0531270839, -0.0729630664, -0.00948078651, -0.019113753, -0.0457725339, 0.0799323841, -0.03278834, 0.0819077193, -0.0420220532, 0.0677470639, 0.013211471, 0.0428901799, 0.0794655382, -0.0161148198, 0.0719190091, -0.01039318, -0.0540312864, 0.00616043108, -0.0451625176, 0.00686708419, 0.069687821, 0.0135618402, -0.00375918299, -0.0454170518, 0.0760549604, 0.045612324, 0.0297017358, -0.0215636268, 0.0578414239, 0.0197647903, -0.0165519398, 0.0696980655, -0.0307176504, -0.006066937, 0.00633908762, 0.0708698332, 0.0104985014, 0.070334807, -0.0172161907, -0.0683671534, 8.409760e-02, 0.065735139, 0.0600004457, -0.0458277613, -0.0484473631, 0.0754156038, 0.017629575, 0.0545623563, 0.00304295681, -0.0693565831, 9.13375348E-4, 0.0114732273, -0.0895151421, -0.0323515348, 0.072690241, 0.00604097405, 0.0152576622, 0.0804376751, -0.0311251692, 7.291780e-02, 0.0480075665, -0.0191533584, 0.0656941161, 0.0644796192, 0.00967218168, 0.0342366919, 0.0608258545, 0.0466630198, 0.0235199332, -0.0514165051, 0.0214682911, 0.00590082863, -0.0851936265, 0.0221672766, 0.0965566188, 0.0753631666, -0.045303762, -0.0213238522, 0.0415863059, 0.0013673869, 0.0452743769, -0.0333885066, 0.0633196086, 5.726110e-02, -0.0379763693, -0.0442966521, 0.0226656664, -0.0648594946, -0.0494883507, -0.0473266169, 0.0213497635, 0.0256826822, -0.0282017607, -0.0308606755, -0.0615470372, -0.0745801777, 0.0610333644, -0.0597410463, 5.725090e-02, -0.0134514328, 0.0832832977, 0.014949997, -0.0188430864, 0.054037787, 0.0596304089, -5.033190e-02, 5.24813251E-4, 0.0948666334, -0.0761330202, -0.00164685317, -0.021010302, -0.0674709975], [-0.0564792119, -0.0630687252, -0.0652136504, -3.787650e-02, 0.0287271906, 0.00301907118, 0.00410398887, -0.00647644093, -0.0599972904, -0.0273556579, -0.057005398, -0.0644893274, -0.0188582595, -0.0310523808, 0.00984401442, -0.0542926714, 0.0658222138, 0.0585825667, 0.0278787892, -0.0673579946, 7.981590e-02, -0.0469307341, 0.0613840595, -0.0285219308, -0.0766493976, -0.0236234665, -0.0363469943, -0.0441177301, -7.049900e-02, 0.0023718474, 0.069517985, 0.0599421486, -0.0313921608, -0.00310403598, 0.0511678234, 0.0371084288, -0.076970689, 0.0683940649, 6.842790e-02, -6.610590e-02, -0.00623453688, -0.0769604892, 0.0696840286, -0.0665623695, -0.0356130414, 0.025655644, 0.0512976758, 0.0289048012, -0.0313152745, -0.071308367, 0.0407427438, -0.00429950841, 0.017845111, -0.00539173605, 0.0701713115, -0.0357280634, 0.0147323972, 0.0737991109, -0.0526831038, 0.0429463275, 0.0323860571, -0.0600832216, -0.0747677609, 0.0618464388, -0.048572544, -0.0765198171, 0.0131608527, 0.0603640713, -0.0246229861, -0.0379345715, -0.0623004548, -0.0679427087, 0.0398149453, 0.0176091176, -1.99747723E-4, 0.0177119859, 0.00275881984, -0.0563178733, -0.0266559664, -0.0687125549, 0.0140438685, 0.0769242197, 0.0761726424, 0.0429656841, -0.0119000245, -0.0800398066, -0.0248722266, -0.0439049415, 0.0471658446, -0.079365842, 0.0122202924, -0.0261524599, -0.00507867383, 0.0146548022, -0.025575811, -0.0413413532, -0.0458717234, 0.0211144891, -0.0346832387, -0.0407263041, -7.061870e-02, 0.0567672402, -0.034826111, 0.0646161959, 0.022716932, 0.0117229382, 0.0626106709, -0.0316547751, 0.0155892745, 0.0282347016, -0.0511675403, 6.086320e-02, -0.0497749895, 0.0789508745, -0.080340758, 0.0296065249, 0.0653207451, 0.0204372648, -0.0364443064, -0.0198601801, 0.0157927684, 0.0293203779, -0.0134891663, 0.0800515487, -0.0542397872, -0.0665926486, -0.0679443926, -0.0231809858], [-0.0194691271, -0.0686940327, 0.0283956155, -0.0012170939, 0.0201005526, 0.0240291022, 0.0558899045, 0.0434267335, -0.0181050319, 0.0536650494, 0.019977523, 0.0806058943, -7.351060e-02, 0.0163703896, 0.038084209, 0.028597692, -0.0569677874, -0.00924736727, -0.0316754133, 0.0601147301, -0.0743475407, -0.0550480038, -0.0172680914, 0.0648587048, -0.0645609125, 0.015110339, -0.00417207461, 0.0290761869, -0.0432004221, 0.0444784947, -0.0210933611, 0.0723250583, 0.032201767, 0.0559704639, -0.0611782297, -0.0297214426, -0.0328909718, 0.0680391714, 0.038413465, -0.0225951299, 0.0464657955, 0.0230218824, -0.0290044956, -0.0083670048, 0.043095734, 0.0391366966, -0.0704769418, 0.0504723974, -0.0306956209, -0.0476790145, -0.059651237, 0.0402517505, 0.0428092703, -0.0300718974, -0.0100884326, 9.376520e-03, 0.0698109567, -0.0494597964, -0.0138720982, -0.00824468583, -0.0330445766, -0.0132663064, -0.031645909, 0.0160595756, 0.0535719767, -0.00286468375, -0.0397500321, -0.0724806637, -0.0478344932, -0.0431883112, -0.0621895082, -0.00298447255, 0.03754659, -0.0687618777, 0.0404504463, -0.0299711823, 0.00521183526, 0.0406936444, -6.748990e-02, 0.0544833541, 0.0303724594, 0.070119977, -0.0426489636, 0.0532922521, 0.0570180491, -0.0776468291, -0.0680473298, -0.0261218194, 0.0792615041, 0.0189361144, 0.0107261743, -8.725860e-03, -0.0755149871, -0.0362306945, 0.0465154648, 0.0673095435, 0.0277604014, 0.0104400227, 0.0473059975, 0.00495664682, 0.0246469714, -0.0210446436, -0.00439944677, 0.0611168742, 0.0388413295, 0.0534742624, -0.0251201671, 0.0102953017, 0.0723824352, 0.00935915671, 0.0418059379, 0.0219662786, -0.0749056488, 0.0553003289, -0.0331270769, 0.0671209097, -0.0384589285, 0.0287626162, -0.077810958, 0.00934861414, -0.0616152436, -0.0425090492, -0.0693442523, 0.05299991, 0.0285040513, -7.511090e-02, -0.0306354854, -2.79547785E-5], [0.0122998357, 0.0757832304, -0.0237494037, 0.04513219, 0.0503200889, 0.00886386446, -0.0105954492, 0.0369597711, 0.00422146451, -0.0805732086, 3.31203948E-4, -0.0429600738, 0.052892454, 0.015997408, -0.014945033, 0.080408208, -0.0149166584, -0.013649568, -0.015588969, -0.060701184, 0.0464519933, -0.0448702686, -0.0515371524, 0.0413463376, -0.0678495392, 7.271930e-02, -0.0518059582, -0.0175128914, 0.0691724047, 0.0498452969, -0.00733317621, 5.396020e-02, -0.0559668876, 0.0733801499, 0.032948263, -0.0315047614, 0.0387405269, 0.0129400454, -0.0246601105, -0.0251952093, 0.0624270514, 0.0804561078, -0.00773411151, 0.0725615099, -0.0636850744, -0.0750345364, 0.0536100157, 0.0246237684, -0.00302163512, 0.0483535379, -0.071968995, 0.0747956857, 0.020043131, -0.0485744625, 0.0649643093, -0.0518693663, 0.0414771773, -0.0110360552, -0.0260817483, -0.0271431934, 0.0122397589, 0.0095577985, 0.0444442779, 0.0111286994, 0.00447948929, 0.035665594, -0.063358441, -0.0578714311, -0.00396186952, -0.0767233967, -0.0625379831, -0.0608411171, 0.0626195446, -0.0626835302, -0.0276922137, -8.716800e-03, -0.0497648455, -0.0409816466, 0.0405205525, 0.0479317531, -0.00222743489, -0.0180950407, 0.0208730977, -0.0578898564, -0.0100368746, 0.0503839217, 0.0627330243, 0.00225222786, 0.0398046896, 0.0470334962, 0.00240832334, -0.00764663657, 0.0733359456, 0.0293706395, 0.00366583583, -0.0666624159, -0.00169637916, -0.0363177918, 0.0479820147, 0.0711907819, 0.0572198816, 0.00961224828, 0.0291627701, -0.0699911118, -0.0210534036, -0.0510936975, 0.0246644039, 0.0659952238, -0.0369686969, -0.0145626208, 0.0551709607, -0.0021044868, -0.0792690068, -0.045823358, 0.0187337827, 0.0426616147, -0.0268781818, 0.0418128632, -0.0113048013, -0.0418530554, -0.0518351533, 0.0580282733, -0.0508365929, -0.0810273737, 0.0726829246, 0.0209370181, 0.0415470637, -0.0179279353], [0.0326000489, 0.0234116185, 0.00545778964, -0.030640563, 0.0738688931, -0.0542661399, 0.0681223497, -0.0380642451, -0.0352217369, -0.0746443048, -0.041472733, -0.055450473, -0.0267047025, 0.0124165667, -0.0228625257, 0.0296921786, 0.0200420115, -0.0788066685, -0.0140345022, -0.0765859782, -0.0674758255, -0.0257173367, -6.840230e-02, -0.00571719278, -0.00114436052, 4.405030e-02, -0.0719152614, 0.0488248095, -0.0599114709, 0.0422557369, 0.0565021262, -7.652750e-03, -0.0256734844, -0.0400579534, -0.0632389635, -0.0427154191, -0.0714065284, -0.0535345264, -0.0314279757, 0.0204908196, 0.0212000832, -0.048872374, -0.067873776, 0.0471470468, 0.00832138117, -0.0344308652, 0.0681343079, 0.0171178579, -0.0452629365, -0.0726918206, -0.0115067149, 0.0512487441, -0.0100067109, -0.0634418502, 7.103840e-02, -0.0251129847, -0.0435874462, 0.0569464639, 0.026730543, -0.0605487265, -0.0095210541, -0.0217299052, 0.0610660464, -0.0383675359, 0.0153944446, 0.00926090498, 7.150610e-02, -0.0688595697, -0.0176398531, 0.0624976344, -0.0199653525, -0.0103584956, -0.0442458838, -0.0581669286, -0.0226355102, -0.00756872818, 0.00332317478, 0.0625690892, 0.044985272, 0.0363316685, -0.0450438783, 0.0554723181, 0.0156131368, 0.0343389623, -0.0241980664, 0.0163198374, -0.0296730306, -0.00279910024, 0.0278101116, -0.076603435, -0.0782934278, -0.0355311632, 0.00919973291, 0.0681777149, -0.0575520433, -0.0464054309, 0.00277461787, 0.073404476, -0.0160726327, 0.0535631366, 0.0302281678, 0.0542761832, 0.00690442324, 0.079781577, -0.0268695336, 0.014768891, 0.04266458, 0.0745963529, -0.0418644436, -0.0571102463, -0.00282914122, 0.0127694383, -0.00246372679, 0.00737938098, -0.0308335256, 0.0126212668, 0.0574696064, -0.058547888, -0.0175098944, 0.00427301694, 0.0485522225, -0.0400400609, 0.055959098, -0.00685175927, -0.0176407434, 0.0695829093, -0.0241221674, 0.0399364866], [-0.0424422659, 0.0622340515, 0.0441095121, 0.0501009114, 5.780120e-02, -0.00142219733, 0.0049904855, 0.0176327284, 0.0276180133, 0.0446482077, 0.0216985308, -0.00698399777, 0.00264787255, -0.0254566241, 0.0268398523, 0.0608996488, 0.023731146, -0.0233506821, -0.0161108226, 0.0273922179, -0.0768041611, 0.0186797902, -0.00487658707, 0.0192233417, -0.0666948184, -0.0279194899, -0.0170717742, -0.0498384759, 0.0364328846, 0.0197760351, 0.0359081961, 0.0288845804, -0.0446322821, 0.0545245782, -8.49147676E-4, 0.0763596445, -0.0697845593, 0.0350335315, 0.0619367472, 0.0738565847, 0.0496454351, 0.0138737494, 0.071866259, -0.0675972551, -0.0521276295, 0.046179004, 0.0136760287, 0.059310969, -0.0026105335, -0.0685884058, 0.0723495707, -0.0653715879, -0.0767479464, -0.0390316024, -0.0687648207, 0.0281916633, -0.0299151521, 0.0441808738, -0.0465946421, 7.921120e-03, -0.0367305726, -0.0480144732, -0.030684242, -0.0775965377, 0.033005733, -0.0741933584, 0.0508926548, -7.247140e-02, 0.0708063468, 0.0701458231, -0.0173840653, -0.0518808439, 6.820170e-02, 0.0071050385, -0.00260203425, 0.0610906743, -0.00799915288, -0.00495325308, -0.0142571246, 0.0426354147, -0.0334678032, 0.0154780857, -0.0280416626, 0.0107586458, -5.104760e-02, -0.0444391631, -0.0228186566, 0.0365878679, 0.0117968256, -0.0467654057, 0.0263380539, 0.0786615982, -0.00975457113, -0.0306874793, 0.00691171223, -0.0261797514, -0.00730256783, 0.0328012183, -0.0116937393, -0.0198136549, -0.070071958, -0.0277974736, -0.0444494039, 0.062355563, -0.0226192977, 0.0283972882, 0.00333993929, -0.0198781043, 0.0483782403, -0.0511943772, -0.0356704555, -0.0278432295, 0.0682216287, 0.0048041665, 0.0294739362, 0.0361809321, -0.0265092887, -0.0580516234, -0.0363723636, 0.0278726444, 0.0480467454, -0.0669612437, -0.0558976121, -0.0554014891, -0.0130626047, -0.024087593, -0.0526864976, 0.0196836386], [-0.0118185524, 0.00365210837, 0.0556656979, 0.0622062534, -0.0591705479, -0.0202099327, -0.0635687262, -0.0538580753, -0.0318699628, 0.062982209, 0.0713472366, 0.0372247025, 0.0633956566, 0.0516580716, -0.0452492945, -0.0703784674, -0.0731086284, 0.0311272759, -0.0545877665, -0.0337744504, 0.00470691361, 0.0440323837, -0.031964574, 0.053325627, 0.0220221132, -0.0214708876, -0.012237817, -0.0151671832, 0.0297863092, -0.0239856355, 0.0238518268, -0.0750227123, -0.0206339452, 0.00526526477, 0.0564554594, 0.0671508387, 0.0312959664, 0.0788164958, -0.0529880784, 0.014798522, 0.0398195684, 0.00503073633, 0.048974283, 0.0373794511, -0.00712574041, 0.0527368709, 0.0239960738, 0.0574682951, 0.0670474246, -0.0248856544, 0.0471900254, -0.0355837196, 4.053710e-02, -0.0411077961, -0.0124884341, 0.0430703759, 0.0313016772, 0.00573692285, -0.0450650491, 0.0780630857, -0.0159933046, 0.0150872329, 0.06539803, 0.0367119573, 0.0532559603, 0.0306246672, 0.0531822741, 0.018204581, -0.060234122, 0.0312982388, 0.0081661595, 0.0546292402, 0.0404782817, 0.0221807603, -0.0468087792, -0.0465578958, -0.0396113582, 0.00853302423, 0.0390235335, -0.00198280788, -0.0356544033, 0.0843360647, -0.0697827563, 0.0486627072, 0.0454759784, -0.01408887, 0.0727600828, 0.00391536485, 0.0393428728, 0.0783063471, -0.0864718779, 0.0533761159, -2.82550434E-4, -0.0143062416, 0.0714561716, -7.32013898E-4, -0.0157881156, -0.0684834346, 0.0250149053, 0.0684702769, 0.051886376, 0.0803715661, -0.0209116191, -0.0529783778, 0.0713747516, 0.00407821499, -0.0589035191, -0.00652464526, -0.0722976103, 0.0665850714, 0.078147836, -0.0464951061, -0.0191197135, -0.00641159574, -0.0555001125, -0.0747126042, -0.0698762909, 0.028065512, 0.047603935, -0.031762816, -0.0418064371, 0.0426406451, -0.054261934, 5.637420e-02, -0.0159069113, 0.0444994643, 0.00888159405, 0.0813020691], [-0.0329513513, 0.0170680452, -3.490010e-02, -0.0323306955, 0.0478740819, 0.0575875863, 0.0445339158, -0.0237984583, 0.00108766847, -0.0545137376, -0.0202329513, 0.0110649867, -0.028091656, 0.0535428375, 0.0316958576, -0.0529762246, -0.0169565547, 0.0455934815, -0.0205616355, -0.0455048271, 0.0779830813, -0.0661003143, -0.00220246823, 0.0671221465, 0.0456783213, 0.0517854616, -0.00642062863, 0.0133819208, -0.0377542377, 0.0226538386, 0.00872595329, -0.0552375428, -0.00810929574, 0.0290660989, -0.0895333588, -0.0614264794, -0.0742908642, 4.054780e-02, 0.0119695114, -3.490460e-02, 0.0373890437, 6.882240e-02, -0.0197250657, 0.0169809982, -0.0342795663, 0.0807307139, 0.0263519716, 0.0127634779, -0.0178958308, -0.0468453169, 0.0478743427, 0.0592424199, 0.0657618418, -0.00444419449, 0.0222322252, -0.0619292185, 0.0575316884, 0.0653048754, -0.0372582451, -0.064567849, -0.0616008081, 0.0790284574, 0.0492639691, -0.0469760336, 0.0398231111, -0.0285748243, 0.0513695329, 0.0571991391, 0.0740055367, -0.0377710201, -0.0725980327, -0.0526797026, -0.0346623324, -0.0537144132, 0.0544944182, 0.0287531838, -0.0378533117, 0.0766448751, 0.0576209389, 0.0456652455, -0.0216350574, 0.00952688325, 0.0748229324, 3.403850e-02, 0.00379601587, -0.00311677391, -0.0607373342, -0.0696445107, -0.0105994586, -0.0814802796, -0.103348605, 0.0235701781, 0.0130775888, -0.007515979, 0.0280551203, -0.0757046118, 0.0636925474, 0.0510142036, -0.0674180239, -0.0104474062, -5.254680e-02, -0.0165257547, -3.143220e-02, 0.0121491291, 0.0281047728, -0.057429228, 0.0308106169, 0.06890513, 0.0731631294, -0.0454254039, 0.08555682, 0.0607716814, -3.58147314E-4, 0.0410096906, -0.00331367855, -0.0479298271, 0.0266141314, -0.0387157649, -0.0529737025, -0.0291931778, -0.0423161089, -0.0523902811, -0.0793272778, -0.0308791213, -0.0455468409, -0.0434169397, 0.0228230376, 0.00299654296], [6.876280e-02, 0.0682521686, 0.0650201514, 0.0382997766, -0.0170408171, -0.0510034375, 0.0168662332, 0.0204861723, 5.09440375E-4, -0.00506131398, -0.00236700359, -0.0716579258, 0.0351178311, -0.0302272718, -0.024695944, -0.0495551676, 0.0336536318, -0.0460067429, 0.0682954937, 0.00967675727, -0.0617653951, -0.0423995629, -7.244050e-02, -0.0227976199, -0.0108548338, 0.00586915482, 0.0435688384, 0.0279635359, -0.0265504476, -0.0136028193, 0.0295012817, -0.0122767128, 0.0609458499, 0.0489204377, -0.0168271232, 0.0280378573, -0.0255042166, -0.0428571962, 0.0540176816, -0.0702460483, 0.00173349329, -0.0132536022, -0.0822807401, -0.0242180694, 0.0463997461, 0.00734832278, -0.0292603392, -3.191940e-02, -0.0335373133, 0.00892481487, -0.0159189329, 0.0244340077, 0.0122144455, 0.0672064125, -0.004042204, -0.0733664408, -0.0576395653, 0.0783993676, 0.0645112395, 8.217970e-02, 0.0594075397, -0.0471333228, -0.0889168381, -0.0314900875, 0.0303435251, -0.049630072, -0.0814939513, 0.057177566, 0.0673012212, -0.0475463793, 0.0598908141, 0.0589608215, -0.0718101189, -0.00754570216, 2.72218986E-5, -0.049668543, 0.0582964644, -0.0484402217, 0.0404046737, 0.0330532119, -0.0238363575, 0.0119634951, -0.070533812, 0.0362098515, 0.0595189221, -0.0843551829, -7.27591745E-4, 0.0877289325, -0.0705175101, 0.0263379309, -0.0639351904, 0.0799003615, 0.0312629454, -0.00153676211, 0.0224882159, -0.0525894687, 6.195470e-02, 0.027983021, 0.0126968874, 0.0279746279, -0.0424972661, 0.0767985955, -0.0684989467, 0.0250775237, -0.0691422373, 0.0614605807, -0.0141988937, -0.0827442631, 0.0590687692, 0.0383781195, 0.00524630677, 0.0326297916, -0.0244963598, 0.0950379967, -0.00529326731, -0.0214516297, -0.0513246357, 0.048778031, 0.0722401962, -0.00932018552, 0.00627237838, -0.025923077, -0.0513565913, 0.0239266679, -0.0642305538, -0.0487873964, -0.0213150177, 0.0626998395], [0.00554210925, 0.0997549295, -0.0403928123, 0.0782475173, 0.0681356266, 0.0640995651, -0.00161900348, 0.0280197673, -0.0596214943, -0.0896479785, -0.0822218358, 0.0872967764, -3.523000e-02, 0.0274608657, -0.00578162167, -0.0381113812, -0.0464854203, 0.033675056, 6.271560e-02, -0.0776872188, 0.0157339592, -0.00595501671, -0.0511232093, 0.00638306467, 0.0178279057, 0.0128335878, 0.0670233816, -6.30995783E-4, 0.00914571061, 0.0304405838, 0.0414480269, 0.00192328822, 0.0142616583, 0.00999203417, -5.201380e-03, -0.0480688959, 0.0804523602, -0.0754268691, -0.0596108697, -0.0690425709, -0.0168621931, -0.0493049212, 0.0355772339, -0.0477856658, 3.975030e-02, 0.0638015419, -0.0789576322, -0.0869637877, 0.021135265, -0.0555451922, 0.0424294174, 0.0075342753, -7.57558795E-4, -3.113320e-02, -0.0101522245, 0.0469016433, -0.0221395623, 0.0646728203, -0.00170060305, 0.0468391664, -0.0946185365, -0.0295311678, -7.517440e-02, 0.00559270475, -0.068161048, -0.0246673636, -0.0481801555, -0.00180500804, 0.0187356509, -0.0138124647, 0.0557225533, 0.00764559395, 0.0433117226, 0.0817245096, 0.0037598582, 0.0289493911, -5.842270e-02, -0.0461511053, -0.0497885905, 0.00158903876, -0.046827931, 0.0927007496, 0.0608907118, -0.082149066, 0.0652120262, 0.0340548791, 0.0344428979, 0.0416992456, -0.0435193814, -0.0555256195, -0.0994211062, 0.0671659559, 0.0539862774, 0.0694717094, -0.0813564136, -0.0603175387, 0.0611158796, -0.0675866753, -0.0793855115, -0.0155110434, 0.0444666408, 0.0554584973, -0.0093629295, -0.0646654144, -0.06847509, 0.0234159585, -0.00363285979, 0.00298301061, 0.038851928, -4.007990e-02, 0.0903255119, -5.056250e-02, -0.0573138893, 0.0451508202, 0.0636124611, 0.0304127242, 0.0384764783, 0.00192072871, -0.0463709161, 0.0687606484, -0.0777272135, -0.0560574159, -0.0889909043, -0.0179907344, 0.0285580568, -0.00794424675, 0.0691115111, 0.00279161218], [0.0661734566, 0.077761665, 0.0193962827, -0.0383331105, -0.0706175863, -0.0258836634, 0.0229407586, -0.0295321159, 0.0464009196, 7.776960e-03, 0.0703439414, -0.00174301653, 0.0395924821, -0.0226058457, -0.0759294778, -0.035815455, 0.0422684662, -0.0377887711, 0.0790305137, -0.0663911551, -0.0606614277, -0.0562282763, 0.0376747176, 0.00564397685, 0.0642365739, -0.0808184742, 0.0687193274, 0.0660604388, 0.0351685174, -0.0765858442, -0.0223499779, -0.0335790217, -0.0120862098, 9.267700e-02, 0.0116144195, 3.503390e-02, -0.0263265707, -0.0584593751, -0.0237966478, -0.07084281, 0.0432497673, 0.0669804737, -3.189120e-02, -0.00504322909, -0.0571353585, 0.0592005551, -0.0917200222, 0.0445591472, -0.0480846502, 0.0703163818, 0.00187333755, 0.0559998229, 0.0931944102, 0.103731371, -0.073335126, 0.0773458257, 0.0724839196, 0.0518774949, 0.054814782, 0.00175114127, -0.00179666863, 0.0334485434, -0.0646005273, 0.044208996, -0.0361927897, -0.0334262811, 0.012265279, 0.0314637162, 0.0635973141, 0.0454986356, -0.0175535288, -0.00499780336, -0.00165365229, -0.044911053, -0.0402278714, 0.0664529875, 0.0535387322, -0.00715514272, -0.0575322546, 9.056950e-02, 0.0232725479, -0.012144818, -0.00305678765, 0.048458498, 0.0518266261, 0.0129869506, 0.0204224233, 0.0277444385, 0.0577321313, -0.00480855396, 0.048565723, 0.0558847897, 0.100147784, -0.0457075797, 0.0565800741, -9.432510e-02, 0.0383496955, 0.0486918427, 0.0622325465, 0.0607244335, 0.0679837242, -0.057371702, 0.0578894131, -9.8696351E-4, 0.0388956629, -0.033627402, -0.0810893699, 0.0479824282, 0.00893475767, 0.0232694913, 0.0843131542, 0.0188633222, -0.07411585, 0.0593910776, -0.0125913611, 0.00958959106, 0.0528407544, 0.0332413465, 0.0447266735, 0.0781988576, -0.0039667231, -0.0712429583, 0.0113431569, 0.0466992073, 0.0159144513, -0.0896983519, 0.0871674045, 0.0609780885], [-0.0418403372, -0.0154733583, -0.0194408968, -3.633670e-02, 0.0652451813, 0.10411761, 0.0196291693, -0.0215839688, -0.0182243735, 0.0196723063, -0.0249111429, -0.00329542486, 0.0391430557, 0.0153217334, 0.0238249302, 0.0593643039, 0.0761821121, -0.0319370292, -0.0528305583, 0.05120093, -0.0672864765, -0.0889374465, -0.0466911606, -0.0623316504, 9.69901811E-5, 0.0617947951, -0.0670482367, 0.0514466353, 0.0254998244, 0.0329208896, -0.0650093928, -0.00795385148, -0.0331646204, 4.874390e-02, -0.0282283425, 0.0411672331, -0.0659493804, 0.0287151318, -0.0637333915, -0.0694289133, -0.0311744362, 0.0682510212, 0.0312576368, -0.0662408546, 0.037176948, 0.104232877, -0.033992175, -0.05503425, 0.0382449329, -0.00610721111, -0.0484509319, -0.044860851, 0.0583087541, 0.107313789, -0.0412715748, 0.0875998884, 0.0492490716, -0.0137893781, 0.0372848548, 0.0287373848, -0.112261266, 0.00108215271, -0.0560796894, -0.0335107706, -0.0368789211, 0.0491764061, -0.0727023184, 0.0461649522, 0.0243976042, -0.0491830036, 3.428860e-02, -0.0556699857, -0.0276093464, -0.0616884641, 0.0097257588, -0.0156848393, 0.056331832, -0.0543598495, 0.003442741, -0.038532313, 0.0366604365, -0.0300670452, -0.0351266302, -0.0508186817, 0.0162135437, -0.016167555, -0.00500343973, -0.0223223586, 0.0452706441, -0.0120067885, 0.0131185735, 0.00475474121, -0.0534145348, -0.0244345795, -0.028541727, 0.0623764581, 0.0469746813, -0.0885743648, 0.0428231955, 0.00235810061, -0.0103209475, 0.0292821825, 0.0925332531, 0.0440765582, -0.0619749538, -0.00896508805, -0.0656715855, 0.0482763723, -0.0812394246, -0.0172612686, 0.0457438454, 0.0636290461, -0.0149535947, 0.0847794339, 0.0542792454, 0.0172487702, 0.00878950488, 0.0134800961, 0.0472730733, 0.0572119616, 0.0277240556, 0.00853924825, -0.107025132, 0.0174240768, -0.016449172, -0.00344352843, -0.0392587185, 0.0845409781], [-0.0741455927, 0.0662892758, -0.0466935076, 0.0133717116, -0.0381259955, -0.0239099991, 0.0606332347, 0.0406444594, 1.330860e-02, 0.039991457, -0.0181278083, 0.0152554438, -0.0429837219, -0.0241913386, 0.056450069, 0.0907548591, -0.0359293297, -0.0241062474, -0.00546307676, 0.0526503474, 0.0620793737, -0.028876001, 3.520110e-02, -0.0299208704, -0.0135792755, 0.0062277508, 0.0181066263, -0.0146486806, -0.0578788109, 0.0703801364, 0.0224558022, -0.0475669913, 0.0233125463, -0.0414839871, -0.0266885497, -0.0536041893, -0.0337440856, 0.0177110173, -0.00880705378, -0.00182198151, -0.0682972446, 0.0138228815, -0.0531543382, 0.00235762075, 0.0654378906, 0.0924140512, -0.0842730998, 0.0801340789, 0.0347072668, 0.0631172358, -0.0361280777, 0.00514480658, 0.0471957624, 0.10190133, -0.0690920129, 0.0755055844, 0.00554168131, 0.0792840048, 0.0311025511, 2.645880e-02, -0.114229485, -0.0285020769, -0.103189833, 0.0809198319, -8.700400e-03, 0.0712678134, -0.0783352255, 0.027558215, -0.0130619258, -0.0386661254, -0.0388479456, 6.488910e-02, -0.0415605269, -0.00982141215, -0.0218026843, 0.0369810238, -0.0486504845, 0.0715087578, -5.78684267E-4, 0.0239333361, 0.0663743392, -0.0133972093, -0.0962464213, -0.0124253482, 0.0180672333, 0.0511643924, 0.0180685669, 0.0304281749, -0.0218538176, -0.0267543439, -0.0660488605, -0.0235176831, -0.00569194648, 0.0678258613, 0.0181335323, 0.00650173845, -1.297970e-02, 0.0469470322, 0.00971110631, -0.0739345774, -0.035309121, -0.021784259, 0.0969468876, 0.0140966941, 5.272120e-02, 0.00225228514, 0.0347541869, -0.00906149391, -0.123086236, 0.064212881, 0.0635699779, 0.0666279793, -0.00210573757, -0.0456771255, 0.0133012459, -0.0833722874, 6.668010e-02, 0.0453243852, 0.0388528407, 0.0505899563, -0.0621645115, 0.047866039, -0.00260498328, 0.0189327039, -0.0530880876, -0.0109916823, 0.0580713041, 7.088040e-02], [-0.0697714984, 0.0766917318, 0.0803274288, 0.0552634448, -7.127430e-02, 0.0171926524, -0.00364157883, -0.0150720868, -0.0388809703, -0.0578712188, -4.589620e-02, 0.0685534552, 0.0298233368, -0.00388996326, 0.0390717238, -0.00591222057, -0.0361890569, -0.0581368543, 0.0244712159, -0.0640112609, -0.0766068994, -0.0257327445, 0.0559277721, 0.0534300283, -0.0488741659, 0.0584743433, -0.0285549853, 0.0262233894, -4.25613223E-4, 0.0114699667, 0.0308976565, 0.00108995964, -0.0162021816, -0.0638885722, -0.00390888052, -0.00297800102, 0.00208971766, 0.0192497838, 0.049903892, 0.0659041106, -0.0382106341, 0.0192063581, 0.023004137, -0.00195549964, -0.0170234609, -0.00487746764, -0.104880616, 0.0517034531, 0.0371428356, -0.0262869634, -0.0776286199, 0.0283929911, 0.0758306086, 0.0729213506, -0.0473975316, -6.402310e-02, 0.040111877, 0.0812806785, 0.0545384288, -4.183910e-02, -0.10386005, -0.0149215432, 0.0384826176, 0.0798311308, -0.0665930807, -0.0151037332, 0.00100892794, 0.0383185223, -1.115170e-02, -0.058196757, -0.0209562778, 0.0165417921, 0.0217632148, -0.0032869135, -0.0555244237, -0.0584430657, -0.0605045073, -0.0556469373, -0.0543674305, 0.0378823355, 0.0260248333, 0.0258916169, -0.0878136977, 5.00264054E-4, -0.052120395, 0.091777794, 0.0688362867, -0.0431739725, -0.097992435, -0.0275878049, 0.093807131, 0.0728445202, 0.0283641815, -6.366310e-02, 0.0799335166, -0.0573477075, 0.118590429, -0.0680084154, 0.0562440716, -0.0698872581, 0.0510762185, 0.082913421, 0.0694057941, -0.0228241719, 0.0694862753, 0.083372645, 0.027718937, 0.0835152491, -0.0867687165, -0.0630459115, 0.0951278433, -0.0784286931, 0.0890924186, 0.0422320589, 0.0210642945, -0.0387246236, -0.0392662659, -0.0609560385, 0.058796078, -0.0180426855, 0.0255404543, -0.041240681, 0.0146013284, 0.0275084879, -0.00124575943, -0.0065583135, 0.0504895374, 0.0702838302], [0.0454846472, 0.0313075073, 0.0111328233, 8.91626521E-4, -0.0696907192, 0.0458458923, 0.0594637208, 0.0675004274, 0.0765890405, 0.0887794792, 0.0407325961, -0.0590136684, -0.05896613, 0.0147090061, 0.00353508862, 0.102792121, 0.0610392205, -0.0342392065, -0.00979624316, -7.281120e-02, -0.00976379401, 0.0514828153, -0.0944260359, 0.110390313, -0.0508915558, 0.0586028472, 0.0626299232, 0.0841902494, -0.0534148812, 0.0863930434, -0.0429142602, 0.0224208292, 0.0386350527, -0.0107239494, 0.0992485359, 0.0369647667, -0.0183264371, 0.0764541551, -0.0259639919, 0.0828561261, 4.375880e-02, -0.0664518848, 0.0868037343, 0.0481869802, -0.0229297057, 0.0823705866, 0.0218973495, -0.0164420176, 0.0343990661, 0.0325328261, -0.0215229187, -0.0362004265, 5.313880e-03, 0.105216466, 0.0554264598, -0.0489007309, 0.0593869127, -0.0288160276, 0.00875847227, 0.0640716552, -0.0209262948, -0.037588004, -0.0658687204, -0.0254509319, 0.0257401615, -0.0517993383, 0.0393406823, -0.0578243807, 0.047992155, -0.069681704, -0.0330707803, 0.0377281457, -0.0349456891, 0.0673368201, -0.0784216523, 0.077867195, -0.0535475053, -0.0106454352, -0.0416765213, 0.0370544791, -0.0414854549, 0.00865428057, -0.0878075435, -0.00537736714, 0.00993562582, -0.0417699367, -0.0268187355, 0.105585203, -4.66242869E-4, -1.28625121E-4, 0.0897771269, 0.0543165728, -0.00246814289, -0.0394922532, -0.0124727022, 0.0788338854, 0.0400377735, -0.00172314025, -0.0635460392, 0.0618738942, -0.031411875, 0.00551240565, 0.0302795283, -0.0583855547, -0.0300159063, -0.0250331126, 0.024702508, -0.0434519686, -0.0353583843, 0.0776772722, 0.0683127939, -0.0527993813, 0.0389285274, 0.0716925859, 0.0343331657, -0.0340361111, -0.0144912675, -0.00325314119, 0.0122467801, 0.0878917649, 0.00135917263, 0.0518558398, 0.0669038221, -0.0498404466, -0.0480591357, -0.0502972603, -0.0286796279, 0.0485712625], [-0.0558901243, -0.00973183569, -0.0207756776, 0.0157520082, -0.0357762575, 0.107537903, -0.00963133201, -0.0162634719, 0.0872410089, 0.0936824157, 0.0137946839, -0.075975351, 0.0332486704, -0.00634005573, -0.0345686935, 0.0324518308, -7.148510e-02, 0.0440987088, 0.0277305711, -0.035843242, -0.0533368848, -0.0928968116, -0.0598022714, 0.00963545498, -0.0499521866, -0.0204479899, 4.138520e-03, 0.0738640577, -0.0473509058, -0.0307590105, -0.0652592182, -0.014747276, -0.0533965714, -0.034731444, 0.00421972759, 0.0102646993, 0.0615380406, 0.0569577962, 0.0819126665, 0.0276156366, -0.042866271, 0.00528218551, 0.0318010524, -7.488460e-02, -0.06166761, 0.030718958, -0.038301412, 0.0619284622, -0.0155863492, -0.019675551, 0.0279607251, 0.0957354158, -0.0281016212, 0.0617224723, 0.078487888, -0.0191707443, -0.0322351754, 0.0126398979, 0.0536111072, -0.0440867469, 0.0901130363, 0.0115578212, 0.0520970561, 0.0779492184, 0.0426838733, 0.0302087776, 0.0138478456, 0.0104164025, -0.0118144695, 0.0195838269, -0.0330348313, -3.921490e-02, -0.018927874, -0.0469032526, 0.024524847, 0.0342154354, -0.0598049946, -0.0188606195, 0.0944853723, 0.104458265, -0.043297343, 0.0881049558, -0.0321710743, -0.0442886502, -0.0805930048, 0.100366838, -0.0558813922, 0.0248525608, 0.0434038043, 0.0826289281, 0.0764742643, 0.0721387342, -0.0103282193, -0.00836091674, 0.0194702726, -0.0238251705, 0.0481725521, -0.0628958121, 0.0210421141, 0.0601703897, 0.0595797673, -0.0295619555, -0.0344613791, -0.060117323, 0.0270884205, -0.0213943385, 0.0542456061, 0.048593469, 0.0256595016, 0.0671160519, 7.718600e-02, -0.0559403636, 0.0321065709, -0.0702584311, 0.13192448, -4.24438447E-4, -0.0192504246, 0.0218509194, 0.0536291413, 0.109342262, -0.0564970188, 0.0488427356, 0.0317152627, -0.0100520896, -0.020875534, 0.052110672, 0.0404584073, -0.0013289185], [0.0313227698, -0.0871585905, -0.0498059839, 0.101808764, 0.00854088366, -0.0589027777, -0.0470370091, -0.0178352352, -0.0558486618, 0.0787969306, -8.619330e-02, 0.0104792165, 0.0154999336, 0.0226647761, -0.0455027111, 0.113076784, -0.0134274103, 0.0421756022, 0.0476819538, 0.0259751827, 0.0347010754, 6.29454909E-4, 9.8013808E-4, 0.0602753684, 0.0217111204, -0.00289915712, -0.0592654832, -0.0294048823, 0.0111547885, 0.0704022422, -7.270090e-02, 0.0542763136, -6.2287887E-5, -0.0481393635, -0.0587824471, 0.0781884044, -0.053900227, 0.052903831, -0.0134442691, 0.00430921745, -0.081744112, 0.0513038039, 0.0319361947, -0.0605753809, -0.0209353715, 0.0549540035, -0.0712969154, 0.00828749593, -0.0525470786, -0.0466065072, -0.0672324896, 0.0271983091, -0.0810825676, 4.000830e-02, 0.0937463864, 0.0426356308, 0.047672227, 0.0316643529, -0.0601619184, 0.0697296411, -0.0334096588, 0.034928374, 0.0320281796, -0.0463614687, 0.07927154, 0.104587972, -0.0420184433, 0.0756725445, 0.0619691759, -0.0523983799, -0.0713342503, 0.0258004274, 0.00211982173, -0.057894323, 0.0143757658, 0.0240961034, 0.0194275323, -4.813530e-02, 0.0457393043, 0.115996644, -0.0604892112, 0.0277754422, -0.0718386471, 0.051142592, -0.0966604724, 0.0509330593, -0.00523723103, -0.0417151079, 0.0476486422, 0.0420787856, 0.0259977076, -0.0429333672, -5.361550e-03, -0.0224938914, -0.0504291244, -0.00872089807, -0.0313385315, 0.00373498048, -0.051538974, 0.0161041841, -0.0115739433, 0.106851719, -0.0642628521, -0.0582826585, -0.0421504602, -0.0458327867, 0.0410999842, 0.0456650928, 0.0304993112, 0.0178425424, 0.044128865, -0.0608028546, 0.0280090049, 0.0741400123, -0.0192210339, -0.0533593595, 0.120530702, 0.0426599085, 0.0501108877, 0.0938991978, 0.0791315957, -0.0354431421, -0.0103177512, 0.0322695449, -7.392200e-02, 0.00502403826, 0.00108948047, -0.047300715], [0.0421712473, 0.0360740535, -0.0601458102, 0.0824247375, -0.081222333, 0.0408719219, 0.0174580589, 0.0640835389, -0.00647654152, -0.0178244077, -0.081284441, -0.0653049126, 0.012146974, 0.0710738152, 0.0648686066, 0.0126929078, 8.066210e-03, 0.0714902431, 0.0851777941, -6.924230e-02, -0.0780243203, 0.0672648624, 0.0684868395, 0.118596114, 0.0122652305, -0.00318377954, 0.0483382754, 0.0452219062, -0.0309564341, 0.0190314502, 0.0454353802, -0.0426361747, 0.0546124354, 0.0142163718, 0.044815734, 0.0968064442, 0.0875864476, 0.0130078522, -0.0361511447, -9.157970e-03, 9.25421773E-6, -0.0399855673, 0.093369551, -0.0683860928, -0.0362978093, 0.058263842, 0.00815247837, -0.0218402352, 0.0400851332, 0.0488170385, -0.037646573, 0.0496019311, -0.0266856551, 0.0808974206, 0.0891923829, 0.0373628363, -0.070873253, -0.0659658089, -0.00793824717, 0.0618809313, 0.00442674244, 0.0498998873, 0.0249077845, -0.0144712524, 0.0531445369, 0.0160911661, 0.0153850922, 0.0357267037, -0.00914064143, -0.0464552194, -0.0739457086, -0.0718135759, -0.00625288952, 0.0502713472, 0.0733495578, -0.029419532, 0.0450695306, 0.00764857139, 0.0579355396, -0.0282547064, 0.0954223871, -0.00580253033, 0.0195691157, 0.0797474384, -0.0903456509, 0.00985406619, 0.077612698, 0.0623678416, 0.0888958275, 0.0629725158, 0.0400866345, -0.0505227745, -0.00693059852, -0.106744774, -0.0201414656, 0.0559253171, 0.0907647163, -0.0183630977, -0.00325634913, 0.00989872683, -0.0741997361, 0.0730122179, 0.0479645655, 0.00562256901, -0.0205316637, -0.0930795148, 0.012340907, -0.0549007915, 0.0543738455, 0.00810946151, -0.00485685561, 0.00835454277, -0.021513598, 0.0323216245, 0.108161867, 0.0408365056, 1.166580e-01, 0.0576374978, -0.0740225241, 0.100789428, 0.0583255328, 0.0410115197, 0.00893005263, 0.0510436706, 0.0535037294, 0.0632851645, 0.00286607398, -1.643660e-02], [0.0689914674, -0.0976411774, 0.0235421117, 0.101415262, -0.102428608, -2.37487446E-4, 0.087661229, -0.0679488331, -0.017720852, 0.0382350869, -0.0532633699, -0.0159196313, -0.0652496516, 0.00354345795, -0.0337500088, 0.0211603176, -0.0414087661, -0.0847892835, 0.012298326, -3.012670e-02, -0.0980356708, -0.0586719364, 0.0818181783, 0.0746347755, -0.0454537421, -0.0375929065, 0.0344130844, 0.0882021561, -0.0421950743, 0.0872237086, 0.0507788733, -0.0682369322, -6.66907115E-4, -0.0412079655, -0.0792477503, 0.0210955888, -0.053301435, 0.106459364, 0.02256044, 0.0437840521, -4.46717255E-4, 0.0144087803, 0.0250780378, -0.132771552, 0.057550557, -0.0473356731, -0.0422552079, -0.0500134453, 0.0483026281, -0.0340429693, -0.00905936677, 0.0266099479, -0.0105895065, -0.0240286756, 0.0729863345, 0.112395942, 0.0535441265, -0.0774590745, 0.0536781102, 0.0856367647, 0.0369853824, 0.0784833505, -0.0564364791, -0.0414437242, 0.0527935065, 0.0044887457, 0.0933589041, -4.813130e-02, -0.00725327479, -0.0265601017, -0.0507042371, -0.0402211621, 0.0161241144, 0.0138908131, 0.0636901706, 0.0073343222, 0.0313654803, -0.0467283167, 0.0230808314, -0.0258489437, 0.0878542587, -0.0683442354, -0.00203907257, -0.0105933473, -0.0224208813, -0.00223298813, -0.0563243926, 0.0317599289, 0.0335126594, -0.0364958197, 0.0418542847, -0.0107886475, 0.0463810228, 0.0289129447, -0.0363058671, 0.0975805371, 0.0479055941, 3.49291484E-4, 0.0689017251, 0.0265001766, -0.0141575718, 0.0927000791, 0.0504159741, 0.00398555072, -0.0533459187, -0.0650544614, -0.0421510041, -0.0378624536, 0.062797673, -0.0357583463, 0.0987571254, -0.0627310723, -0.0542328544, 0.0504479855, 0.0592694692, 0.0416852124, 0.0629332736, 0.0593213812, 0.0157675035, 0.0548788607, 0.0477166325, -0.0971351265, 0.0288093146, 0.0650517046, -0.0177078806, 0.0707657114, -0.0204544198, -0.0497671627], [0.0598203912, -0.0360524952, 0.0924355983, 0.146244556, 0.054861933, 0.0563749783, -0.0498352647, -0.0320994146, 0.0927455723, -3.443830e-02, 0.0239955783, 0.0250620097, -0.00630934117, 0.0401610062, -0.0228288379, -0.0433419645, -0.0100836856, -0.0532132164, 0.002946008, 0.055243928, -0.0936752781, -0.0125373015, 0.0645866841, 0.0378497578, 0.0685394928, -0.0844138488, 0.0357529521, -0.0032306998, 0.020869337, 0.00472124247, 0.0176490638, -0.0361143127, -0.0637024418, -0.0447166152, -0.0405301973, -0.024192499, 0.0418426096, 0.0374700315, -0.0170628764, 0.0169576779, 0.0516015068, -0.0818354859, 0.0214064922, -0.0512022972, 0.0555898957, -0.036099676, -0.0765275955, 0.0532567836, 0.0053411508, -0.0250435658, 0.0140785892, -0.0172373895, 0.0107562756, -0.0619336999, 0.122738667, 0.121795468, -0.0234443564, -0.0119136674, 0.0869248285, 0.0322386585, -0.00204363186, 0.0563034751, 0.0126516186, -0.0742212161, 0.0164568145, -0.0224952735, -0.0438016877, 0.017319981, 0.028259119, -4.661390e-02, -0.0275478605, 0.0278233364, 0.0840201527, 0.068686828, -0.0695748702, -6.005800e-02, -0.0160470475, -0.0522958785, 0.00190800638, -0.0315516517, 0.129008725, -0.0790538787, -0.0243737251, -0.0204242561, -0.0293388739, 0.0300574079, 0.0783751308, 0.0856460407, 0.0786738768, -0.0489330776, 0.055439081, -0.0781144574, 0.0674192384, -0.0416206792, 0.0585177243, 0.0243310425, 0.121048965, 0.070345521, -0.0493154936, 6.505040e-02, -0.0551386103, 0.0209572464, -0.0103899995, -0.0272043049, 2.733050e-02, -0.10846132, 0.00104283157, 0.0524269044, -0.0169557463, -3.972180e-02, 0.0162126496, -0.0457602069, -8.093540e-02, -0.060147386, 0.0769840851, 0.0684687942, 0.0303954761, 0.0165500343, 0.0321728475, 0.0117515605, 0.0424534455, -0.068678528, 0.0235856548, 0.0288119912, 0.0747921839, -0.0101251686, 0.0628868341, -0.0551258363], [0.00293270103, -0.00128512573, 0.0168775469, 0.0415106155, 0.020719314, -0.0327634551, 0.0790469869, 0.040839009, -0.046810925, 0.0952859893, -0.0651023685, -0.0226721223, -0.0883716642, 0.0174518358, 0.100990295, 1.8419468E-4, -0.0147587229, -0.0715167671, 0.00431303261, -0.0228052847, 0.00920401048, 0.0278810207, 0.0677006244, 0.049956046, 0.0834902301, -3.164630e-02, 0.0671191439, -0.0433099754, -0.0449999645, 0.0727765188, 0.108865768, 0.0710876286, -0.0286517609, 0.0201505926, -0.0438623242, 0.0699302405, 0.0641138181, 0.0280657467, -0.0783888995, -0.00708512124, 0.00156092865, -0.00230237492, 0.0215622038, 0.0447888896, 0.00778848352, -0.00603662757, -0.0555735528, -0.022886062, -0.0129304696, 0.0619891583, 0.0151094142, 0.0757918507, -0.0570742078, -0.029721057, 0.00269315299, 0.0783293098, 0.0230397191, 0.0619730577, 0.110093884, -0.0130795278, 0.0317975543, 0.036886014, -0.0442714393, 0.0433718115, -0.0952159762, 4.820430e-02, 0.0551442914, -0.0556761585, -0.0306261405, -0.0302202906, -0.0624212474, -0.0168988872, 0.0837030708, 0.020000618, -0.0476832651, -0.0163047574, -0.0279265251, -0.0321491621, 0.0481985584, -0.0577224307, 0.00382917444, 0.0611831471, 0.108668305, 0.0493502095, -0.0211144388, 0.0284154061, 0.0614807084, -0.0125629567, 0.0192305204, -0.0790041238, -0.0478177592, -0.00721454341, -0.0452114344, 0.0339138508, 0.01299801, -0.0399125665, 0.021324968, 0.0873417481, 0.0457866974, 0.0432540812, -0.107685186, 0.0746816918, 0.0605904236, 0.0394596905, -0.0237514637, 6.35650416E-4, -0.116130851, 0.0208479594, 4.51208791E-4, 0.04986782, -0.0490044802, 0.056233298, 0.0262664501, 0.0424197242, 0.0691586658, 0.00556924753, 0.067897588, 0.102959231, -0.0340401344, 0.0253918488, -0.0295951776, 0.027166253, 0.0133924074, -0.017585583, 0.00359549955, -0.016328495, 0.0942129418, 0.0690490231], [-0.0230080504, -7.985790e-03, 0.0839524269, -0.00316011347, 4.022790e-03, 0.0479108207, 0.059960261, -0.0532989465, -6.09285198E-4, 0.0841624588, -0.0399643332, -0.0793377459, -0.047039263, -0.0140844267, -0.0391799733, -0.0819476321, 0.0238844864, 0.0828810855, 0.0199244153, 4.746570e-02, -1.928000e-02, 0.0350237414, 0.039425265, 0.081585519, 0.0199689865, 0.0738351122, 0.038925305, 0.0512179323, 0.0126716811, 0.129406109, -0.0150925564, -0.00365025015, -0.0108143622, 0.108946398, -0.0164748318, 0.107096173, -0.0378302671, 0.0662952959, 0.0623555183, 0.0444851145, -0.0888513401, -0.0489949659, 0.0139755178, 0.00571357226, 0.0804216489, -0.0131888296, -0.0238579679, -0.0706988275, 0.0148627432, -0.0369135216, -0.0472701751, -0.024705749, -0.00567350723, 0.0391686074, -0.0305856206, 0.0861309618, 0.0565372035, 0.0205122568, 0.105450869, 0.0474226922, 0.046506878, -0.0321832709, -0.0495029576, 7.13226211E-4, -0.0746529996, 0.0252579544, 0.0691419616, -0.0774237811, 0.0480422527, 0.0449487753, -0.0417116433, 0.0122251417, 0.0266303178, -0.0537320375, -0.0362957865, -0.0290749464, -0.106694818, -0.0167314764, -0.0216888208, 0.0267442148, 0.0380889326, 0.054480996, 0.0434120521, -8.95901059E-4, -0.0803043246, -0.016125761, 0.00567849586, 7.470640e-02, 0.0262705814, 0.00459014485, 0.0593459159, -0.0748340264, 0.0438913889, -0.0255341474, -0.0466146693, 0.0526503175, 0.0315350033, -0.0146408062, -0.0590761602, -0.0291619487, -0.0885098428, 0.057790719, 0.0473970398, 0.0966486707, 0.0705483928, -0.0304492805, 0.0417762548, -0.0554038435, -0.0505523421, -0.0286200847, -0.0481383391, -0.014450076, -0.0886406078, 0.0972328782, 0.0486656576, -0.0842002183, 0.00553972274, -0.0111398455, -0.00152368203, 0.103358969, 0.102739155, 0.00134536671, -0.062989831, 0.0178441051, 0.0457651503, -0.0194719695, 0.0708882362, -0.0100534884], [-0.0670683458, -0.060229145, -0.0551774204, 0.123913899, 0.0669179931, 0.0806112885, -0.0292999595, 0.0896978825, -0.0119797103, 1.643380e-02, -0.0194865502, -0.065272592, -0.0647058263, 0.107874401, 0.107503079, -0.0596378148, 0.0733998119, -0.0527566411, 0.0544350706, 0.0642286167, -0.00801911112, 0.00508486619, -0.0818334519, 0.0227987673, 0.0466035828, 0.058512561, -0.079188995, -0.0527154505, 0.0624858364, -0.0274256319, 0.0546291843, -0.0108753378, -0.0703922287, 0.0536526814, -0.0112012289, 0.0498538464, 0.0330044404, -0.0620341487, -0.0581983179, -0.0408298261, 0.0253920499, -0.00148904894, -0.0336050205, 0.0748319253, -0.0584118292, 0.0424476601, 0.00978393946, 0.0212371927, -4.973230e-02, -0.0668619126, 0.0868994743, 0.035560254, -8.376640e-02, 0.00398577238, 0.0765819326, -0.0130290622, -0.0116156414, -0.0409424379, 0.0939842164, 0.0105842315, 0.0530577376, -0.0343199968, 0.0225233529, 0.0766944215, -0.0838217362, 0.0775644183, -0.0390995368, 0.00556128845, 0.0155942533, -0.010166876, 0.0379251502, -0.00958458055, 0.0354748331, -0.0419059247, -0.0742894635, 0.0242424197, 0.0140599627, -0.014086403, 0.0643787533, -0.0128361667, 0.0886103436, 0.00669930549, 0.0644602627, -0.052821815, -0.101060338, -0.0195850898, 0.0212281458, 0.0187004413, -0.00895884074, 0.0452628322, -1.728830e-02, 0.0460407324, -0.0208656676, -0.0570988879, 0.0490976572, -2.88181182E-4, 0.0940738841, -0.0698591098, -0.00432886742, -0.0298292227, 0.0323735513, 0.0357658789, 0.00375751778, -0.0422180817, 0.0239398144, -0.0709452629, -0.0386275165, 0.0246551111, -6.089660e-02, 7.285930e-02, 0.0472642221, -0.0208805837, -0.0504056625, 0.0711080208, -0.00871984288, -0.0034383412, -0.0254603941, -0.0292428788, -0.0316921026, -0.00137326936, -0.0296039674, 0.0591151081, -0.0273020584, -0.0132601466, 0.0119793797, 0.042688664, 0.0145528596, 0.051991526], [0.0671904981, 0.0794735923, 0.0901716724, 9.55629046E-4, 0.0382791273, -0.0402381718, 0.0741371959, 0.0198290274, 0.00779303629, 0.0264898688, 0.0425465815, 0.0328344889, -0.0489913039, 0.0976754426, 0.0426939353, -0.0299965888, -0.00744560919, 0.00444085244, 0.00725514768, 7.736900e-02, 0.0143325143, -0.019867586, -0.0239450578, 0.0424663238, 0.0995965451, 0.0303633399, -0.0790362805, 0.0603170805, -0.0413229428, -0.0460756309, -0.0608372651, -0.0277564432, 6.608040e-02, 0.107310511, 0.0642029941, 0.061474625, 0.0233885925, -0.0200133361, -0.0670537204, -0.00374509743, -0.0574323535, 0.0587668829, 0.0906176567, 0.0377289616, 0.0204805508, 0.0879309251, -0.03944299, 0.0666974932, 5.853000e-02, 0.0235847794, -3.84523883E-4, -0.0042682793, -0.0821187943, 0.021328114, -0.0292776283, 0.00378280343, 0.0619678385, 0.06323082, 0.0823370218, 0.0175919682, -0.0422085263, -0.0351863094, -0.0712972283, 0.0677517578, -0.0423270091, 0.0701941401, -0.0136418501, -7.907100e-02, 0.020749703, -0.0688274204, -0.0188095346, 0.0435926877, 0.0200541802, 0.0686892569, -0.0544642769, -0.0264434218, -0.0741311535, -0.0554498062, -0.0406249315, 0.0498458631, -0.0349675715, -0.0310451202, -0.0671462193, -0.0197602306, -0.0493024625, -0.0163521022, 0.0529653691, -0.0702469796, -0.0989760383, -0.0531043932, 8.413710e-02, -0.052389957, -0.0694821774, -0.0611622743, 0.0470363535, 8.571860e-02, 0.075241357, -8.804920e-03, -0.0487752855, -0.0203687903, 0.0322628021, 0.087922424, -0.0330053382, 0.00935690291, -0.0190232024, -0.0191103294, 0.0309626143, -0.0541125759, 0.0163722448, 9.674900e-02, 0.0459930152, -0.0759326592, 6.547640e-02, -0.0327152312, 0.0543769039, -0.0408396311, -0.0590622798, 0.0596324205, 0.0400843918, 0.0612383969, 0.0825371965, -0.00959584862, -0.0341218971, 0.0023681093, 9.070310e-03, -0.0702341422, 0.101711221, 0.0144624198], [-0.017385615, 0.0517474934, -0.00365272583, 0.0874442383, -0.0298283324, -0.0179137979, -5.497170e-02, 0.0543116704, -0.0793217868, -0.0606044792, -0.00259888405, -0.0275522117, -3.331890e-02, 0.0890835821, -0.00278214808, -2.206130e-02, 0.053443674, -0.0360812098, 0.0616071373, -0.0562129803, 0.0409881398, 0.0600077547, 0.00784264877, -0.0276670866, -0.00260568783, 0.0128409006, 0.0573558286, 0.0405680463, -0.0433635898, 0.0746789724, 0.0843637287, -0.0406893454, 0.0241100844, 0.0139927007, 0.00239367201, 0.101882435, 0.0252626818, 0.0653267204, 0.0514198877, 0.0220740382, -0.0518271141, -0.0711362436, 0.0238002632, -0.0453108214, 0.0173421446, -0.0338725597, -0.052527342, -0.0128110172, 0.00963017903, 0.0626481548, -0.0119778803, 0.0459780805, 0.0111011025, 0.0436689556, 0.00658981735, -0.015721526, 0.0654942095, -0.0584082417, 0.100456968, -0.0719673708, 0.0915978104, 0.0122132208, -0.0337241702, 0.0504042357, -0.060787227, 5.523610e-02, -0.0680129454, 0.0276914369, 0.0559006594, 0.048319675, -0.0167920869, -0.0575534105, 0.0758618638, 0.0604883842, -0.0627159476, 0.076881893, 0.0490264222, -0.0275590103, 0.0767665952, 0.00804000814, 0.0966598615, -0.0143468678, -0.0325266942, 0.0470781587, -0.0803067907, 0.0263363831, -0.0100146439, 0.00992359593, 0.00586302858, -0.0800714194, 0.0379560664, 0.0663529411, -0.0109806154, 0.0911584049, 0.0492969267, 0.0516108237, 0.0353993326, 0.0018550714, 0.00740529923, -0.00912841875, 0.0138791399, 0.0908952355, 0.095462516, -0.0533577353, -0.0544834808, -0.0160870645, 0.0573132634, -9.27645888E-4, -0.0488307476, 0.0142673124, -0.00402516918, 0.0750173405, -0.0286330283, -0.0781226083, -0.0418178812, 0.0454163328, 0.0261764266, -0.0322048739, 0.0543259829, 0.0743636116, 0.0523804277, -0.0476314798, -0.0137161715, 0.0214534849, 0.0599340871, 0.00144531613, 0.0209308341, 0.0424782038], [-0.0772073716, -0.0482172742, 0.00195654947, 0.078159228, 0.0834698975, -0.0463652834, -0.0334398448, -0.0230813604, 0.0599247664, 0.0208957586, 0.0813670232, 0.0669712573, 0.051399257, 0.088481307, -0.00719898334, 0.0397446156, -0.0608969107, 0.0625970587, 4.442470e-02, 0.0186943877, -5.617200e-03, -0.0261388961, -0.0539549887, 7.388520e-02, 0.0306693055, 0.108182371, -0.071698606, -0.0820722729, -0.0452554189, 0.0450049937, -0.0106204888, -2.896560e-02, -0.0219827369, -0.0273131393, -0.0575570464, -0.0225293189, 0.00947570521, -0.102145508, 0.0379664712, -0.00608444912, -0.00202364568, 0.00536462758, -0.0104702711, -0.0250592642, 0.0407705382, 0.0567927882, -0.0408779494, 0.046056781, 0.0924842357, -0.0582203642, -0.0359888338, -0.0254721325, -0.0719112083, -0.0489347875, -0.0427869558, 0.0177568216, -0.0592639484, -0.0093249632, 0.0242888816, -0.0016512376, -0.0306718722, -0.00801155343, 0.0441319197, -0.0100320987, 0.0674320459, 2.357820e-02, -0.0718113184, -0.00756290555, 0.0771602169, -0.0938118398, -0.0196474306, -0.00597494515, -0.0870217904, -1.957360e-02, -0.0149106337, -0.0295380447, 0.0408849567, 0.0194669422, -0.071457684, -0.048795905, -0.0230901707, 0.0321201645, -0.05852345, -0.0230317358, 0.0329668485, -0.0461460911, 6.794990e-02, 3.701570e-02, -0.14415285, -0.0535774305, 0.0248647891, -0.0579178222, -0.0691077113, -0.010560696, 0.038779676, 0.0457781516, -0.0495770834, 0.0173141845, 0.0757038817, -0.00401367433, 0.0326100588, -0.00755993742, -0.028536167, 0.0762077346, -0.00339360931, -0.032363534, 0.0277566295, -0.0523677655, -0.0604490712, 0.0491857715, 0.00783289782, 0.0267826188, 0.0377460234, 0.00776396831, -0.110981025, -0.0329162665, -0.0217704754, 0.0350118317, 0.00708472356, 0.00629498111, 0.0848090425, -0.0138837993, 0.0265643187, -0.0175407454, -0.0187182575, -7.9321739E-4, -0.0129204802, 0.0267464742], [0.0197119042, 0.00972367544, 0.0229855478, -0.098064132, 0.0571616478, -0.0202907063, -0.0686729923, -0.0544331893, -0.0628349632, -0.0712890401, 0.0947963669, -0.0118463663, 0.0532133244, 0.0939664542, -0.0139586451, -0.0793235972, 0.0268219747, 0.0189760961, 0.0650004894, 0.0476363748, -0.0507319458, 0.0090494547, -0.0395561755, -0.0449665152, 0.0687024519, -9.94574147E-5, -0.0214926805, -0.0206869598, -8.875120e-03, -0.0249368269, -0.024672227, -0.033402361, -3.146850e-02, -0.0220211558, -0.0301539563, 0.006194524, -0.106378317, -0.0266985707, -0.0291922409, -0.0512636751, 0.0224457271, -0.0270185489, 0.0520036146, -0.0931260511, -0.034983892, -0.0802631825, -0.012361669, -0.0500250682, -0.0368769318, -0.037308082, -6.834990e-02, 0.00555867096, 0.0688999146, 6.91458699E-4, -0.0369125269, -0.0124265747, -0.0232360419, -0.00538180023, 0.0823948979, -0.00197263411, 0.0983653217, -0.0315836631, 0.0477186926, -0.0194036737, 0.00720984582, 0.0269269608, -0.00992196425, 0.0373594947, 0.023761522, -0.075148955, 0.0679604411, 0.0113466578, 0.014104397, -0.0797987431, 0.0364213623, 0.0896157398, -0.0264848806, 0.0245131627, 0.0410591774, -0.0483490862, -0.0107461726, -0.0578414872, -0.077650167, -0.0779832229, -0.0826306045, 0.00670624059, 5.660120e-04, 0.0251489095, -0.0409492813, -0.0826154202, 0.133528471, 0.0459946431, -0.0677635148, 0.0186449829, -0.0317185894, 0.0342350826, 0.0743709356, -0.0471066423, -0.0521218143, -0.105679125, -0.0457327813, -0.0786165744, 0.0848573818, 0.064620763, -0.032799609, -0.00104804174, 0.0452010818, 0.0494192615, -0.02735929, 0.103169888, 0.0189587809, 0.0318044499, -0.0616720616, -0.119506508, -0.091269657, -0.0281600896, -0.00264839153, -0.070154883, -0.0796360149, 0.0685512573, -0.0044616214, 6.889460e-02, 0.0363820046, 0.0915749073, 0.0141598312, 0.00375919184, -0.0269427933, -0.0513439402], [0.0331881754, 2.651020e-02, -0.025851164, 0.0175769571, -0.0160758458, -0.0305072777, -0.0904545485, 0.0650190711, 0.0496306531, -0.0374675393, -0.0199287329, -9.29782516E-4, -0.00498855673, 0.0118770106, 0.0851516649, 0.0339605249, -0.0678484439, 4.286220e-02, 0.0166810118, 3.95457231E-4, 0.0790666714, 6.041840e-02, -0.0510980412, 0.0857724472, 0.0697048753, 0.107972726, 0.0699329376, -0.0290076919, -0.0441910364, 0.0211128276, 0.0107317269, 0.0118772602, -0.0544784479, 0.0250784904, 0.0787770301, 0.0802991986, -0.0934282466, -0.00398185803, -0.0128502594, -0.0434057526, 0.0131243411, 0.055591207, 0.097569324, -0.0880550071, 0.026694566, -0.0722468421, 0.0017136205, -0.0219862647, -0.0687684938, -2.524980e-02, 0.036350973, 0.0805879235, -0.0167834628, -0.0647179037, 6.57693949E-4, -0.0711751133, -0.0454312861, -0.0360986367, -0.0447460823, 0.046900522, 0.00396263413, -0.0474167205, 0.00165369327, -0.0491550379, 0.0951190442, 0.0051425565, -0.0302812681, 0.0284566171, 0.0185881536, 0.0331314765, 4.895460e-03, -0.0653282255, -5.959820e-02, 0.0314440802, -0.0420323834, 0.00509945117, -0.0264007729, -0.0264714789, 0.0652280897, -0.0545891598, -0.0348724052, -8.236050e-03, -0.051633697, -0.0515869595, -4.573050e-02, 0.020150641, 0.00615126826, 0.0169996526, -0.0986723452, 0.0304338373, 0.139095038, -0.011302487, 0.0256431494, -0.0553523265, -0.0542682372, -0.00960461888, 0.0518668927, -0.0475903042, 0.0642734915, -0.00357338856, -0.0294592697, 0.055202309, 0.00640758174, -0.0122698182, 0.0110452231, 0.11022453, -5.27246098E-4, -0.0294821225, 0.0567028411, 0.0345729776, -0.0141228838, 0.02940014, 0.0478971042, -0.0859416648, -0.0885470062, 0.0487542562, 7.386550e-02, -0.0820335447, -3.975900e-02, 0.0707353651, 0.0947641357, 0.0124910716, 0.0654552057, -0.0272899643, -7.589650e-02, 0.01042022, -0.0999530926, -0.0545929931], [-0.0367123149, -0.0214916505, 0.0185209271, -0.0294472668, 0.0286052227, -0.0172178596, 2.371050e-02, -0.0271780062, 0.080752097, 0.0611451939, 0.0663202778, -0.0535248742, -0.00366665097, -0.0486275293, 0.0407535434, 0.0461487919, 0.0331317931, 0.074080728, -0.0730997771, -0.0182516929, 0.0339531451, -0.0242419243, -0.0741776377, -0.00913789868, -0.0260613691, 1.012670e-01, -0.0394716449, -0.0462158322, -3.622850e-02, 0.00104117254, -0.0347153097, 0.0555723086, 0.00365091884, 0.0158300754, -0.0023847518, -0.0339219831, 0.040872436, -0.0160056017, 0.0772033706, -0.0100469766, 0.0772884041, -0.0455273241, 0.0582281351, -0.0952374711, -0.0108960522, 0.0412646085, -0.0950042083, -0.0529610962, -0.0372965373, 0.0712389647, -0.021153247, -0.0485318452, -0.0190219674, -0.073449865, -0.0191014167, -0.0553278364, 0.0323073603, -0.0198928472, 0.0572095215, 0.0866091847, -0.0554228462, 0.0513704866, -0.0596240833, 0.0290138088, 0.033679992, 0.00397809595, -0.0726971775, -0.0707383677, 0.042134434, 0.0604525283, 0.0105163287, 0.0386005268, 1.66724378E-4, 0.0225323159, 0.0354867242, 0.00595977344, 0.0528861247, 0.0265529323, 0.020511236, 0.0107962536, -0.0164475683, -0.0331893563, -0.0277112238, -0.0153692989, -0.0466979481, 0.0547476485, -0.00285580033, -0.0617009662, -0.00899279303, -0.0381798856, 0.0798183307, 0.0349292606, 0.0562724844, 0.0332977101, -0.00993845984, 0.0287420396, 0.0861630216, -0.0811962932, 0.0733712167, -0.0630382225, 0.0143582299, -0.0326483734, -0.0322615653, -0.0428111292, -0.0424620435, 6.646890e-02, 0.11048425, -0.0745855421, 0.0335247181, 0.0289136134, -0.0439649336, 0.0649002567, 0.0466452092, 0.0122874985, -0.0195192881, -0.0994874387, 0.00835572183, -0.0353646763, -0.0819267928, 0.0473973639, 0.0294793248, 0.0530679449, 0.0583802387, 0.0410630256, 0.0118758604, 0.039783366, -0.0172074288, 0.0363440104], [0.0479549021, 0.00908384752, -4.930720e-02, -0.00789893884, 0.0691932216, 0.0776666775, 0.0154358512, -0.0255545415, -0.0700848773, 0.0353914537, -0.0761101693, 0.0665692315, -0.0257131029, 0.0823785886, -0.0482436828, 0.0336365253, 0.0207219049, -0.0543041676, 0.0180847179, -0.018118903, -0.0636728853, -0.0152073912, -0.055214949, 0.0408040024, -0.0359703787, -0.019378135, -0.0196952038, 0.0179038625, 0.0495649204, -6.641700e-02, 0.028391704, 2.76343577E-4, 0.0310449749, 0.0252232347, -6.181450e-03, -0.0418282896, -0.0595114455, 0.00890602357, 0.0143840238, -0.052682735, -0.0759141743, 0.0567236356, 0.0747371763, 0.0651409104, 0.0576477535, 0.0374207497, -0.0305253714, -0.0598489419, -0.0651340708, 0.0528173074, 0.0549812093, 0.0251510553, 0.0366291367, 0.0545480587, -0.0266598891, -0.0031906548, -0.0206487775, -0.054814674, -0.00796389579, 0.00782584678, 0.0453644656, -0.0775621756, 0.0688623711, 0.0737240538, -0.00657472294, -0.0311288573, -0.00158300973, 0.00439560506, -0.0379967317, -0.0141416881, -0.0479120947, 0.0254125092, -0.0206012409, 0.0165002607, 0.0290741771, 8.12057754E-4, 0.0462025031, -0.0266851839, 0.0620421469, 0.0793720484, -0.0208746605, 0.0388044417, -0.0663383901, 0.0246712696, -0.0229795743, -0.0289953239, -0.0213955082, -0.00468814885, -0.014367721, 0.0189380758, 0.00569227664, -0.035207279, -0.00759078236, 5.126170e-02, 0.0464803427, -0.0312178228, 0.036473237, 0.0304057822, 4.014870e-02, 0.0484486632, 0.0147122797, 0.00924499519, -0.0182944834, 0.0211334247, 0.00752399675, 0.0398481302, 0.0344488025, -0.0187444948, 0.0576119088, -0.0274953116, -0.0352303684, 5.285420e-02, -0.0721892342, 0.0544455796, 0.0629049316, -0.0088469265, 0.00738892797, 0.0362020768, -0.0501357019, -0.0085966885, 0.0342779681, 0.0719955638, 0.0182822868, -0.0247224234, -0.0816184058, -0.00710410671, -0.0279569123, -0.0621390343], [0.0482220314, -0.0275709741, -0.0575254075, -0.0453220755, -0.0374431312, -0.0700783059, -0.020738529, -0.0126525406, 0.0103860153, -0.0667310581, 0.0556828789, -0.0776049494, -0.0195856225, 0.0798722878, 0.0336904787, -0.0597362891, 0.0694572851, -0.0171561874, 0.0376248471, -0.0570879206, -0.0785153508, -0.0618099198, 0.0450956523, -0.0408485457, -0.0641896948, 0.0283548199, -0.0380747952, 0.0437257923, -0.0606716722, 0.0588417873, -0.0304171685, 0.0235541184, -0.0524369627, 4.996480e-02, 0.0734120756, -0.0648948774, 2.40100839E-4, -0.0706453323, 0.0168248396, 0.022608282, -0.0511874184, -0.0347108282, 0.0643137395, -0.0249881707, 0.0509880148, 0.0413025133, -0.0400063619, 0.0760707185, 0.0219823811, -0.0536827184, -0.0078839818, -0.0166158397, -0.0485287569, 0.0304493904, -0.00999861117, 0.0356888175, 0.00386334141, 0.0278244708, 0.0155325299, 0.014790263, 0.0622244514, -0.0535496213, 0.05938733, -0.0756539628, -0.0266662836, -0.0491693765, 0.0390402488, 0.0329344161, -0.076400809, 0.00191572949, 0.00532664126, -0.0648029447, 0.014067539, 0.0752380118, -0.0706452876, 0.0458212271, -0.0413386635, 0.0581562631, -0.0240587257, -0.0421780832, -0.0292385183, 0.0131555554, 0.0160469431, 0.0787678808, -0.0478011593, -0.0768531188, 0.0258875545, 0.0494880788, -0.0177438594, 7.157340e-02, -0.0770593211, 0.0677568912, -0.0276254863, 0.0796306431, -0.0415596776, -0.0178671516, 0.0505095683, 0.00348364981, -0.0799202248, 0.0282302089, -0.0277465452, -0.0656775087, 0.0419381149, -0.0317522027, -0.0327424482, 0.022924548, 0.0318222195, -0.0622903556, 0.0215285402, 0.0466440618, -0.067701079, -0.0297650471, -0.0715472773, -0.0228027813, 0.051256042, -0.0208211727, 0.0575313643, 0.038472943, -0.0744724348, -0.00859352946, 0.0548508316, -0.0454553254, -0.023882905, 0.0296054482, 0.0570714325, -0.0469041169, 0.077604182, -0.0609114207], [-0.0528047755, 0.0467303917, -0.0116599528, 0.0281862505, 0.0226841439, -0.00219345605, -0.0419144146, -0.0446713604, 0.0126695726, -0.0791815742, 0.0749367625, -0.00569316279, -0.035124559, -5.09888865E-4, 0.0568231046, 0.0163218863, -0.00938087702, 0.0484357029, 0.0228231568, 0.0227402672, 0.0695628449, -0.0547577702, 0.0432286374, 0.0763162076, 0.0451607406, -0.0762580708, -0.060093008, 0.0249174871, 0.0193501189, 0.0769706219, 0.0238101333, -0.0203093421, 0.0482973307, -0.0601179339, 0.0358605869, -0.0732703805, -0.0379546322, 0.0604971237, -0.0715803429, 0.00460626138, 0.0648687705, 0.0548472628, 0.062346924, -0.0114866532, -0.0579702407, 0.034605056, 0.0568837188, -0.0099850865, -0.0219115056, 0.0788254887, 0.00207743444, -0.060999196, 0.00763338339, 0.0131518953, 0.00178014336, 0.0653714835, 0.0253737979, 5.54285536E-4, 0.0216013268, 0.0159544125, -0.0682446733, -0.027020853, 0.0724722892, -0.0628090501, -0.0392063484, -0.0461672507, 0.0442212299, 0.00266503659, -0.0409794636, 0.0306737944, 0.0236774683, 0.012222128, 0.0782656446, 0.00984688289, 0.0636462197, 0.0431728624, 0.0388262048, 0.0357062593, -0.0107961111, -0.00266434555, -0.0452565178, -0.0383178629, 0.0805585756, 0.00103468471, -6.919600e-02, 0.0196474288, 0.0155539559, 0.0392224975, -0.0245703552, 0.0200101417, 0.0270204209, -0.00941288937, 0.024192404, 0.0618860721, -0.0689965188, 0.0105308397, -0.0263325442, -0.00142170745, 0.0154961292, 0.0583404228, 0.0571451038, 0.0720427856, 0.0059988196, 0.0161313023, 0.0305820685, -0.0339301936, 0.0210410859, -0.0692505836, 0.0291170236, -0.00900683738, -0.058520779, -0.0395226441, 0.00576514564, -0.0219711363, -0.00177559664, 0.0770036429, 0.0251987167, 0.0662594289, 0.00107466825, -0.0133747207, -0.0273232777, 0.0247313958, 0.00958027598, 0.025713088, -0.0535470061, -0.0105458377, -0.00138928602, -0.0153997634], [0.00845329463, 0.0181613471, -0.0725070089, 0.0125736091, -0.0489335097, -0.00213244488, -0.0459518433, -0.0490765423, 0.0607499443, 0.0676489174, 0.0423893668, -0.0610614568, 0.0558543764, -0.033275649, -0.0409253277, -0.013893581, 0.0191305075, 0.0710810124, -0.0200758278, -0.0602182038, 0.0180415269, -0.0240308028, 0.0542591512, 0.0780675485, 0.0443231426, -0.0731007233, -0.0185204279, -0.0322616845, -0.0462511517, -0.0363696255, 0.0472738035, 0.0557380468, -0.0317915604, 0.0792187526, -0.0222354848, -0.0772410333, 0.0342092253, 0.0253308602, 0.0629897192, 0.0669417828, 0.0303314961, -0.00249757455, 0.0559107177, 0.0274624247, -0.0503212027, 0.0705445632, 0.0564586036, -0.0781237334, 0.0152070448, -0.0359484702, 0.0349223949, -0.0237050429, -0.0200723037, -0.0190816652, 0.0276754368, -0.00110105379, -0.0486346222, 0.0196939819, 8.96914804E-4, 0.0406712629, 0.0597889125, 0.0461228453, -0.0458640382, -0.0706629232, 0.0771271586, 0.0253675655, 0.080561541, -0.0257621333, -0.0758453682, -0.0118502928, -6.216140e-02, -0.033819709, 0.00394546753, -0.0232810881, 0.00245270017, 0.0310265049, 0.064383693, -0.0586252809, -0.0477064922, 0.0218539424, -0.0650750696, 0.076966837, 0.0366220884, 0.00736453942, -0.032804843, 0.0535230488, 0.0335054956, -0.0439880304, 7.2429626E-4, -0.0543059893, -0.0120068435, 0.00678461324, 0.0424390733, 0.0642019659, -0.0444417298, -0.0393768884, 0.00362118473, 0.0779576376, -0.0619399473, -0.0200275928, 0.0338293128, 0.0358373672, 0.0545455217, 0.0231858399, -0.0370924659, 0.0614633597, 0.052448269, 0.0162523258, 0.0335491262, 0.0707981065, -0.0775146111, -0.0622398108, -0.0191573016, -0.0399047956, -0.0567813441, -0.0803148671, -0.0305994656, -0.0659261346, 1.407910e-02, 0.0455198586, 0.0366043113, -0.053960707, 0.0125540951, 0.0102169234, 0.010709024, -0.0356145948, 0.0453835204, 0.0439946391], [-0.0381050631, 0.0820319578, 0.0546204671, 0.059465047, 0.0250406843, -0.0295938831, 0.0110558206, -0.0395385213, -0.00642968481, 0.0489911288, -0.0827757939, 0.0583355911, 0.0474292301, 6.27501344E-4, -0.0341359116, -0.0677875727, -0.0207169913, 0.00513833575, -0.0678420141, 0.016123699, -0.0257621258, 0.0442685746, -0.0628194958, 0.0656262413, -0.0575498082, -4.210190e-02, 0.0689367503, 0.0677933246, -0.00607843045, -0.00698295236, 0.0139005771, 0.0273067076, -0.0156352315, -0.00263372483, -0.0108742155, -0.0687258691, -0.0778738781, 0.0149091445, 0.0428354032, -0.0569490567, 7.058380e-02, -0.00772936502, 0.0482190959, -0.052445624, -0.058201734, -0.048601035, 0.046784386, 0.0211878922, 0.0572897084, -0.064287208, -0.00596737582, -0.0530820377, 0.0198778417, -0.0394611061, -0.00482386583, -0.0758135914, 0.046512682, 0.0376415886, 0.0367655642, 0.0263156444, -0.0476844572, 0.0461463816, 0.027240932, -0.0791488439, 0.0493783578, -0.0536479913, -0.0698144883, -0.00806096662, 0.00271471567, -0.0246971846, 0.049280297, -0.0291524101, -0.0152047044, 0.0328811035, -0.0265907198, -0.0659586638, 0.0336783268, 0.0128719416, -0.0169151127, 0.038462732, -0.0378202498, 0.012130497, 0.00659661321, 3.155410e-02, 0.0518519431, -0.0262897536, -0.0712119266, 0.0259324461, -0.0685177743, 0.0117515707, -0.0658599585, -0.0264898892, -0.0459548086, 0.0631707385, -0.0290400535, -0.037977092, -0.0646569878, -0.0577573702, 0.0779464915, 0.0081249522, 0.0541243888, -0.0556700677, 0.0735893175, 0.0229209326, 3.31882562E-4, -7.228660e-02, -0.0650960729, 0.00257112342, -0.0148512945, -0.0268174428, 0.0534129702, -0.0471093468, 0.0684953704, -0.0424434692, -0.0137549844, -0.0431067906, 0.0497682393, 0.0195170697, -0.00889051146, 0.00380755938, 0.0114540579, -0.0540839285, 0.00975271408, -0.0745467693, 0.0271396581, -0.0301582534, 0.00960672833, 0.0156516396], [0.0476299673, -0.0352552831, -0.0764792114, 0.0161787011, -0.00249689748, -0.0186044499, 0.0183372013, -0.0551189184, 0.053947445, 0.0148990527, -0.0750177726, -0.0114030344, -0.0371230431, 0.0674914345, 0.0385145321, -0.0334677212, -0.0489139445, -0.00700412876, 0.0802428275, 0.0135629661, 0.00639268709, -0.0206897072, -0.0276903231, -0.01154048, -0.0441936813, 0.074996151, 0.0256161392, -0.0671691224, 0.0189513657, -0.0478190072, 0.0284168273, 0.0613156594, -0.00241593318, 0.0574760027, -0.0425562449, 0.0628722385, 0.0456480123, -0.00847470201, -0.00415528333, 0.0490389876, -0.0729067847, -0.0398477316, -0.0237912294, 0.0823751464, 0.00290728663, -0.0198066477, -0.0330597796, 0.014046629, 0.0764786154, 0.0318984836, 0.0310407672, 0.0397099592, -0.0192825496, -0.02224553, -0.00884827226, -0.0581189841, -0.0398983099, 0.0740153715, -0.00804445427, -0.070416607, 0.0356184766, -0.0400054343, -0.00831496715, -0.0138182268, -0.0520317033, -0.0150734112, -0.0300358906, 0.0555078872, -0.0530925766, -0.0141741857, -0.073471725, 0.061197415, 0.0492995866, -0.0280165523, 4.778790e-02, -0.0801614597, -0.0744826645, 0.0538935103, 0.0478111319, -0.0436517037, -0.0114226472, 0.0704066232, 0.0490755662, 0.0197098348, -0.0678142905, 0.023141522, -0.0736633167, -0.0691202953, 0.0493494943, -0.00895279367, -0.0597634874, -0.012723471, 0.0301064625, 0.0844637156, -0.0275973491, 0.0617196597, -0.0589740351, -0.0038138763, -0.0705945641, -0.00814041309, -0.0112482496, 0.0135008246, 0.0668027698, 0.077992849, 0.0603514351, 2.81263317E-4, 0.0419473834, 0.0523338765, -0.0360234454, 0.0302206036, 0.0737082213, -0.012340311, -0.0585227646, -0.0236241706, -0.00297226198, 0.046014864, -0.0460015535, -0.0721988231, -0.0601461828, 0.0759244263, 0.0412361957, 0.0675024241, -0.041040346, -0.0277014207, -0.0485310927, 0.041099444, 0.0396218561, -4.524630e-02], [0.031121593, 0.0689304546, 0.011046472, 0.0362000354, 0.0493498854, -0.0363684744, 0.0201413371, 0.0092167547, 0.00319778058, 0.042712789, 0.060106378, 0.0294246301, 0.0201867148, -0.0166421738, -0.0712216347, 0.0707755163, 0.00378220575, -0.0424504168, 0.0694715976, 5.265630e-02, -0.0306617543, -0.0779635161, 0.0274305176, -0.0706679076, -0.0324171595, 0.0763119832, 0.0646292567, 0.0219065584, 0.0224776417, -0.0134644499, -0.02580145, 0.0106316246, -0.0102435602, 0.0417476594, 0.0605510063, -0.0090704225, -0.0299843885, 0.054625269, 0.0280193556, -0.0670433342, 0.0629688576, -0.0187528152, 0.00995059218, -0.0140778236, -0.0292383581, 0.00384444953, -0.0677415505, -0.00124405301, 0.0723127201, -0.0267571304, 0.0134390956, -0.0518157445, 0.0685832053, 0.0340336785, -0.0754109621, -0.0689135715, 7.219220e-02, 0.061662592, -0.0751862898, 0.0494612865, -0.0676507577, -0.0581385493, -0.0891561433, -0.0379203819, 0.0257613771, -0.0640092939, 0.06209274, 0.0373208225, 0.0729244724, -0.0432747602, 0.0154406242, -0.0787223055, 0.0594456047, -0.0597814694, -0.080655016, -0.0187442079, 0.0614407696, 0.07491301, 0.018758582, 0.00628499127, 6.440660e-02, 0.00956576411, -0.0569218695, -0.017235918, -0.00523390528, -0.0673439875, 0.0221693683, -0.0679902658, 0.0470803045, -0.0769588873, -0.01658516, 0.0330304615, -0.0736161917, 0.0655402467, 0.0375926569, 0.0600064471, 0.0118679339, 0.0172702819, 0.017462967, -0.0726307928, 0.0442171209, 0.0570765845, 0.0220998116, 0.0151110813, -0.0485135093, 0.0564889759, -0.0294391699, 0.0650827884, -0.00125564029, -0.0534231141, 0.0858427211, 0.0222809725, 0.0695438907, 0.0397659466, 0.061728783, 0.039049428, -0.047786355, -0.0288503692, -0.0468928255, 0.0393559523, -0.0587876849, -0.0465211533, -0.0695342794, -0.0399759449, -0.0345606953, 0.0619875676, 0.0355405062, 0.0560073815], [0.0168211013, 0.00275134412, 0.0505715087, 0.0257725641, 0.0097779138, 0.0381847769, -0.0612172186, 0.0248020403, 0.0526676811, -0.0153049445, -0.010622805, 0.0158474445, 0.00460381061, -0.0294055603, -0.0177612044, 0.092424415, -0.0653436407, 0.0655853674, -0.0318288207, 0.0550116338, -0.0767541453, -0.0836062356, 0.0481971726, -0.0303086098, 0.0449114777, 0.0138732251, -0.00266930414, 0.00729215844, 0.0108636282, -0.0207758173, 0.0460619926, 0.0182561409, -0.0489824452, 0.0234663635, 0.0117356973, -0.0438212119, -0.0290126093, -0.00972590316, -0.0182953496, -0.0191690065, -0.0340484492, -0.0146095008, -7.328250e-02, 0.0267759245, 0.0506167077, -4.671070e-02, -0.0660877377, -0.0522518605, 0.0558163449, 0.0501905158, 0.0767803416, -0.0133663965, 0.0270540901, 0.017639501, 0.0195786301, 0.0637932569, 0.0187590308, 0.0299491491, 0.065825738, -0.00315303751, -0.060580533, -0.0307744481, 0.00506261503, 0.0513551533, 0.018229017, -0.0718659386, 0.00577778742, 0.00734418724, -0.0265018903, 0.0644963085, 0.0136243431, 0.00514846668, 0.0623078421, 0.0627776086, -0.00652870024, 0.0252730399, -0.0170375146, -0.0450485051, 0.0850331783, 0.00975945591, -0.0379290804, 0.021907974, -0.0666382685, 0.0366113149, -0.0178268403, -0.0452192314, -0.00284608663, 0.0195041932, 0.0318090282, -0.0457858853, 0.0285070743, -0.0379315056, -0.0597849116, 0.00662883883, -0.0167801362, -0.00518430956, 0.0797964707, 0.058317475, 0.0688266754, 2.551920e-03, -0.0589880235, -0.0668427646, 0.0851147174, -0.0382322073, 0.0685627609, 0.0371479802, -0.0655534938, -0.0588348918, -0.00898774899, 0.0566513687, -0.0389216729, 0.056279134, -6.3267682E-4, 0.0283660833, 0.0473278873, -0.080095157, -0.0325329117, -0.041039452, 0.0689813867, 6.815850e-02, -0.0380056351, 0.0369399674, -0.0107719358, 0.0678651482, -0.0616321713, 0.00112924702, 0.0157158766, 0.0675060749], [-0.0656459481, -0.0475072749, 0.0481799468, -0.017830994, -0.00238113105, 0.0217234716, -0.0413599946, -0.0175658818, -0.0501749627, 0.0360196345, -0.0813918784, -0.0367525816, 0.0406860895, 0.0132640516, -0.00779543258, 0.0539420247, -0.0786877647, -0.0065458538, -0.0101609733, -0.00860823411, -0.0339654461, -0.00517708203, -0.0576777756, 0.0456175022, 0.0701682493, 0.0263470616, -0.0354841277, 0.0721334219, 0.0154972589, 0.00867122319, -0.0536527522, -0.0668691844, -0.0535649508, -0.0550585277, -0.0515161976, 0.0715958923, -0.0763495043, -0.0273108874, 0.0639933199, 0.00863038189, 0.00750477845, -0.0594600216, -0.0599215627, -0.0564526953, -0.0273976345, -0.02991754, -0.0129810758, -0.0773986801, -0.0333150476, -0.0691436231, 0.0297375154, -9.0050674E-4, 0.03160052, -0.041557055, 0.0496672466, -0.0398300178, 0.0408433676, -0.00541310338, 0.0237893034, -6.001150e-02, -0.0316988342, -0.0690305084, -0.07901486, 5.199990e-02, -0.0866091102, -0.0253129192, 0.0675491691, 0.0803572162, 0.0730374753, 0.0459684767, 0.0506021902, -0.0334103815, -0.0370483883, 0.0117901936, -0.0181443263, -0.0781477093, -0.0881893486, -0.0637482181, 0.0101998355, -0.0568749495, -0.0253345221, 0.00308929035, 0.0218786467, 0.052172672, -0.0553445593, -0.0731917694, -0.00921752676, 0.029693298, 1.21919489E-4, 0.0114602093, -0.0509589463, 0.0537993461, -0.0422926322, -0.0188172124, -0.032186728, 0.0259260032, 0.0768241882, -0.0864413232, -0.00238191849, -0.0753038526, 0.0230977163, 0.0324383043, 0.0804896727, -0.0630078316, 0.0147030987, 0.03679711, -4.877710e-02, -0.0381411128, -0.0819524228, 0.0426396579, -0.0575502105, 0.0156305339, 0.0383274369, -0.00107298035, 0.0971357151, 0.00961136352, -0.0384313278, -6.042680e-02, 0.0351536348, 5.259290e-02, -7.812730e-03, -0.0303652287, 0.0403496325, 0.0474586636, -0.0641349778, -0.0568757914, 0.0498600341, 0.00476823421], [-0.0405218117, -0.00786189548, 0.0812463983, -0.0721657202, -0.010234898, 0.0869327709, 0.0479789302, 0.0135662835, 0.0275188591, -0.0271660555, -0.0501743555, -0.0285116415, 0.0400872193, 0.0594775267, 0.00435726158, 0.0876906514, 0.0159800425, 0.056140814, -0.00715359906, -0.0670479164, 0.0335514396, -0.102369018, 0.0234853756, 0.00975709781, -0.0456463844, 0.00585240312, -0.0449222326, 0.0062051434, 0.00673102867, 5.128290e-02, 0.0289390851, -0.0596451573, -0.0673645064, 0.0598614551, 0.00189650292, 0.0667217523, -0.0455633253, -0.0303253364, 0.0255045705, 0.030763926, -0.0444383733, -0.00961775518, 0.0371019579, -0.00102694344, -0.064159289, 0.0452066064, -0.0521119647, 0.0403494351, 0.0139868418, 0.0442233756, -6.883160e-02, -0.0220223181, 0.027010316, 7.3122856E-4, -0.080579482, 0.00103397295, -0.0701238886, 0.00801839679, 7.842600e-02, 0.0502995737, -0.0499814712, 0.0535394512, 0.0117620677, 0.0192429479, -0.00374575239, 0.0197071135, -0.0158990864, 0.0233263876, -0.00374892633, -0.023651164, 0.0217207558, -0.0349407345, 0.0786504521, 0.0432030149, -0.0172157362, 0.0536515787, -0.0957819297, -0.0706022829, -0.063071765, -0.0465450063, 0.0413437821, 0.0245042611, 0.0249125604, 0.0327110589, 0.0363186263, 0.0614866726, -0.0487638935, 0.0447124392, -0.0981928706, -0.0324720144, 0.0782089456, 0.058889959, 0.00984244607, 0.024860926, -0.0563857146, -0.0810470134, 0.0696224123, 0.0109585207, 0.00241453201, -0.0387882777, -0.0534976684, 0.0281414855, 0.0506793074, -0.0371070765, -0.0550914668, 0.0493451208, -0.0605696626, -0.0559462272, -0.0957788154, -0.0625383928, 0.0423595421, -0.0854039713, -0.0684921816, -0.0441812538, 0.0236102976, -0.0508296676, -0.0430727825, -0.0382069722, 0.0172758531, 0.0262971818, 0.0380390324, 0.0743700563, 0.0268688239, -0.0314523503, 0.0634629503, 0.0233814213, 0.0384251438, 0.0552301556], [0.0502337255, -0.0771808252, -0.0135780452, 0.00268344348, -0.00516484026, -9.647270e-03, -0.04860783, 0.0292989127, 0.0816176757, -0.00182950264, 0.0208764654, -0.015536936, -0.0329528637, -0.045574788, -0.0472520217, 0.0840690881, 0.0257905964, -0.0679772794, -0.00891113094, -0.0920772328, -0.0120639643, -0.00798280258, -0.067411229, 0.0650814548, -0.00950969104, -0.0252194293, -0.0342468806, -0.0575727448, -0.0470686518, 0.108020164, -0.0202460643, -6.02437649E-4, 0.00886666961, 0.0188650433, -0.0544401184, -4.9237418E-4, 0.0160198119, -0.0119111957, -0.0155194588, 0.0103537608, -0.0406024493, -0.0237412043, 8.190110e-02, -0.106606297, 0.0714786052, -0.0436057672, 0.0152673796, 0.0888224691, -0.0767396465, -0.0202617478, 0.0341663882, -0.00150378142, -8.52156663E-4, -0.021538157, -0.0141733317, -0.0297379028, 0.0475767143, -0.0256336071, -0.0505213477, -0.0378758088, 0.0125767281, -0.0164324809, -0.0969027951, -0.00873487722, -0.0279677026, 0.0660499483, -0.0335043706, 0.0467689931, -0.0615079217, -0.0481031165, 0.0681542084, 0.0331284292, 0.014131465, -0.0832458809, -0.0428260118, 0.0309312865, -0.0334123112, -0.0816250964, 0.010286863, 0.0462345295, -0.0261343531, 0.079050824, 0.00436914479, 0.0547663458, 0.00743383821, 0.0480416045, -0.0391312949, -0.06168212, -0.0800412073, -0.0237597115, 0.0870073587, -0.00202789623, 0.0758646727, -0.0607331283, 0.00689716404, -0.00584608177, -0.020713523, 0.040514078, -0.00900786836, -0.0526836514, -4.529060e-02, 0.0525414906, 0.0784587636, 0.0956181437, -0.00592077849, 0.0776418372, 8.405710e-02, 0.035276141, -0.0175070856, 0.10740155, -0.0199277103, -0.0661122352, -0.0215540882, -0.031245634, 0.0463961065, -0.0595259219, 0.016566474, -0.0641233474, 0.0152383158, -0.0210137852, -0.006877854, -0.0677479207, -0.0657966509, 0.0169679821, -0.0108837634, 0.0198882781, -0.0650542304, 0.0865310654], [-0.0518056527, 0.0398570225, 0.0992952585, -0.0580100603, 0.00987161323, 0.0491219833, 0.0592517443, -0.052760914, -0.0370277204, -0.0112126777, 0.0754370242, 6.900000e-02, -0.0710904077, 0.0158537254, 0.00696134567, 0.0757552907, 0.0141487522, 2.182410e-02, -0.0203331914, 2.37989298E-6, -0.0609611906, -0.057420779, 0.0666636154, -0.0106751621, -0.0358542316, -0.0580169223, -0.0579973236, -0.015652677, -0.0538356341, 0.0879250094, -0.0327977836, 0.0808374881, -0.0013364047, 0.0646757111, -0.0483534522, -0.0352474786, -0.0074057756, -0.00235240022, -0.0354191139, -0.00880594458, -3.347510e-02, -0.0273219775, 0.0889878869, 0.010530727, 0.0846039057, 0.0661760494, -0.0077641718, -0.0344918706, 0.0770969465, -0.0329235345, -0.0332630239, -0.0679367855, 0.0254557822, 0.061898183, -5.566380e-02, 0.0589438826, 0.0390037335, -0.0672697201, -0.0717565417, -0.0314119533, 0.0123782819, -0.036033418, -0.0449352376, 0.0795446559, 0.0503554754, 0.0242835879, -0.0259653404, 0.0535130389, -8.57686391E-4, 0.0311867036, 0.102831535, 0.00282282429, -0.0539100319, -0.0694763437, 0.0532562174, 0.0438945219, 0.0414560884, 0.0264770593, -0.00532054901, -0.0108924741, -0.0243118033, 0.0180283953, 0.0644089505, -0.0216692667, -0.0475493371, 0.0914606377, 0.0104702553, 4.755710e-02, -0.0434406139, 0.044731345, -0.022564251, -0.0124111176, 0.068646878, -6.543050e-02, -0.0547627807, 0.050503958, 0.0822378248, 0.0447117053, 0.0653020665, 0.0613218285, -0.0117171034, 0.0712101907, 0.0841986835, 0.0697099492, -9.15083102E-4, 0.0706492141, 0.0178583115, -0.0567147881, -0.102222145, 0.0182502158, -0.0070993891, -0.0177005362, 0.00186069298, 0.0345445424, -0.0359574333, -0.0575664639, -0.00533297937, 0.0671887472, 0.0364396237, 0.0951929986, -0.0794668719, -0.0416109525, 0.025776403, 0.0594550855, -0.0247000922, 0.0329096243, -0.0724495202, -0.0147772785], [-0.0217472091, -0.0526265092, 0.0819173455, 0.0658973157, -0.00461296504, 8.144720e-02, -0.0811055154, -0.0636191219, 0.0203115754, 0.0781842693, -0.0586633906, 0.0190728754, 0.008460504, 0.0682311505, 0.0917313471, 0.0774198771, -0.0648437142, 0.0396650136, -0.0217087492, -0.0579475351, 0.0812482163, 0.020912515, 0.023483092, -0.0330591612, 0.0100017851, -0.0470739044, -0.0418374203, 0.0507138595, -0.0603091531, 0.103943184, 0.0205470491, -0.0124397306, 0.00820523686, 0.0124903889, 0.0111452872, 0.0996521413, 0.042216219, -0.0511758402, 0.00581990229, 0.0538709462, 0.031535808, 0.0340224952, 0.0661841332, 0.0482496694, -0.0190467145, -0.0608985461, -0.080243133, 0.0483856052, 0.0384527259, 0.0171822812, -0.048588343, -0.00182742917, -0.0295782909, 1.004630e-01, 0.0694722459, 0.0286387615, 0.0365513079, -0.0164971147, -0.0537504405, -0.0402116179, 0.0601669028, 0.0333781913, 0.0113542518, 0.0581674427, -0.0291870609, 0.0708750859, 0.052419208, 0.0114423698, 0.0167681202, 0.0406213515, 0.0083555663, 0.0766401961, -0.0329129323, 0.0655713528, 0.0275990665, 0.0199638978, 0.0381672569, -0.034633223, 9.283760e-02, -0.00983145367, 0.0356911533, 0.0298796669, 6.744760e-02, -0.023550855, -0.074433215, 0.04277879, 0.0843453183, 0.00304917945, -0.081880249, 0.0432304963, 0.0287892018, -0.0114755072, 0.0328806154, 0.0484107658, -0.0355683081, -0.0618201718, 0.123664789, -0.00594179798, -0.0598632507, -0.0298237707, -0.0458128601, 0.0638742819, 0.0553402975, 9.478790e-02, -0.0547413193, -0.0218383856, 0.0304699373, 0.0299646873, -0.103749253, -0.0256595761, 0.0263744015, 0.0251585245, 0.0471201204, 0.0387594365, -0.0437158048, -0.0527170673, 0.0436592139, 0.0398237519, 0.0499929041, 0.0192690976, -0.0270677805, 0.0357192345, 5.694590e-03, 0.0230930392, 0.035148669, 0.0129137775, -0.0728567168, 0.0237520337], [-0.0829200148, 0.0519861728, -0.0125030512, 2.735190e-02, -0.0373370163, 0.0700494722, 0.0573158227, 0.0446059443, 0.0492519811, 0.0554151684, -0.0538691841, 0.0883141607, -0.0143263992, 0.0752450228, -4.922760e-02, 0.11019621, 0.00389965787, -0.0143686868, 0.0208377149, 0.0489433073, 0.0110917771, 0.0497511737, 0.0670267269, -4.98355366E-4, 0.0117162922, 0.0666775107, 0.0687709674, -0.030745076, -0.0688337162, 0.0100150732, 0.00640614564, 0.0824331343, 0.0529477932, 6.107920e-02, 0.078163743, -0.0463196188, -0.0528640486, 0.0105843637, 7.592600e-02, -0.0921764821, -0.00457025971, 7.661210e-02, 0.0154456384, 0.0322698429, -0.0327855609, 0.0819605738, 0.0647272244, 0.0480009206, -0.0413721614, 0.0671098456, -0.0330933593, 0.0175279509, 0.0720915943, 0.0332955979, 0.00913997553, 0.0323329531, 0.0466575511, -0.00265591429, -0.0168222282, -0.0457003377, -0.0272347275, 0.0242927596, 0.0580883175, 0.0750488639, 0.0208029188, 0.0766994581, -0.00728344219, -0.00999072659, -0.0860540941, 0.0779779925, 0.00941245537, 0.0803052857, -0.0792721807, 0.0492503271, 0.0193144828, 0.0785824283, 0.0477606319, 0.0177192446, 0.0887518301, -0.0367322862, -0.00122006319, -0.0575498454, -0.0118199876, 0.0706789195, -0.00331889256, 0.097316578, 0.0380641744, 0.0777266547, -0.0946363136, 0.0815555229, 0.0355115458, 0.0327606313, 0.0809703692, 0.0758869797, -0.0611607358, 0.0543976761, 0.0502363034, 0.0117774839, -0.0317932032, -0.0651293694, 0.0887183845, 0.0678732246, 0.0246082414, 0.0986345112, -0.0167377237, 0.0800682678, 0.00725066196, 0.0717043653, 0.0403082296, -0.0748939067, 0.00723591028, 0.00189295993, 0.0169420876, -0.0201977734, -0.01323641, -0.0975375249, 0.120967358, 0.0946217998, -0.0682268888, 0.0181306954, 0.067212306, -0.0226684604, 7.168880e-02, 0.0254594237, 0.0524340086, 0.00611440744, -0.0148864016, -0.0107473163], [0.0761700794, -0.06849619, 5.905110e-02, -0.0309010595, -0.0334652774, 0.0404771455, -0.0626583472, 0.0614879578, -0.00807708129, -0.0460658558, -0.0214253385, 1.188040e-02, 0.0666015372, 0.0123642823, 0.0182763617, 0.0161296297, 0.0249258187, -0.00377165945, -4.240110e-02, 0.0794346109, 0.0553265028, -0.0336530469, 0.00784260686, 0.0655372142, 1.773550e-02, 0.015252864, 0.0765553564, 0.0645104051, -0.0290521719, 0.0416653305, -0.0802035853, 0.0233196728, -0.0048920908, -0.0062631066, 0.0205916055, -0.0282971691, -0.052784659, 0.0602924898, 0.0100186691, -0.0267883632, 0.00449909084, 0.0536818318, 0.0760009884, 0.0348572433, 0.0145489303, 0.0432242267, -0.0702756718, -0.0208030231, 7.946550e-03, 0.0948032886, 0.0143955424, -0.0617089309, 0.00693139434, 0.0275066253, 0.0269022379, -0.0160564687, -0.015931949, 0.0311711114, 0.0577024221, -0.0217865761, 0.0164695475, 0.0828600451, -0.0560837574, 0.0469212458, 0.0141726416, 0.0689723864, -0.0308409669, -0.045111645, -0.0472939909, -0.0404327549, -0.015474176, -0.0582415648, -0.0813519656, -0.0497584119, 0.0680945739, -0.013675659, -0.033044681, -0.0106562683, 0.101100318, 0.0337725244, 0.0673825815, -0.0492343232, -0.085569635, 0.0906284078, -0.0526736341, 0.10401921, 0.0780490934, 0.0943660289, 0.0206728522, 0.0820171386, 4.72249289E-4, -0.055317618, 0.0794643909, 0.015922863, -0.0415142514, 0.0393264554, 0.0785496532, -0.0684095174, -0.0218800083, -0.0145708984, -0.0408014432, 0.0355303548, 0.0929826125, 0.0335704423, -0.0615962334, -0.0215376038, 8.122930e-03, 0.0416846387, 0.00940373261, -0.0358594172, 0.06685099, 0.0646067783, -0.0562751591, -0.0858741552, -0.0162241049, 2.726780e-02, 0.0803847983, -0.00920861121, -0.0903741121, 0.0999709442, 0.0901130661, 0.0217672288, -0.0326213762, 0.0322015062, -0.0327374116, 0.0555018932, -0.0582842119, -0.0373067819], [0.0645981953, 0.016650239, -0.0290483702, 0.012372735, 0.00674543064, -0.00507746171, 9.28657827E-4, -0.0255157948, 0.0837319269, 0.0164369252, 0.0350468792, 0.0710658804, -0.0313870944, -0.041096136, 0.0816730186, -0.0484157838, -0.0343495347, -0.0133354496, 0.0625403672, 0.0754277483, -0.0635530502, 0.0408950821, 4.146840e-02, 9.605370e-03, -0.0407891162, 2.531910e-02, 0.037925221, 0.00893962663, 0.0568711534, 0.073098421, -0.0130443657, -0.00402785325, -0.0504019111, -0.051839482, 0.063651979, 0.0328941904, 0.0374184512, 0.0805842354, 4.460060e-02, 0.00733726518, -0.0354604535, 0.0298253223, 0.0047226334, -0.100770041, 0.0387281552, -0.0430013053, -0.0160630643, -0.0404468924, 0.0761412606, 0.0912834331, 0.0714969113, -0.0587909408, -5.813960e-02, -0.0205257013, -0.0232690033, 0.00962146372, -0.0766080618, 0.0378873348, 0.00624869298, 0.0391685627, 0.119962968, 0.0415404066, -0.0534602739, 0.0014766996, 0.0442499369, 0.0374512523, 0.00509371655, 0.0426796414, 0.0221811831, -0.0140976198, -0.0439157858, -0.0133101381, 0.0532036833, 0.0822430253, 0.0022936312, 0.020086376, -0.0458677858, -2.627580e-02, 0.038186226, 0.0269087236, 0.0696457624, -0.0626665503, -0.0106027573, 0.0734038427, -0.0234960951, 0.0955257863, 0.046444267, 0.035791114, 0.0741994678, 0.0625877753, 0.0311192684, -0.0902386233, -0.0539503135, 0.0840285718, -0.0164726079, -0.0636486709, -1.166860e-02, 0.0727869123, -0.00814955309, 0.0109550925, -0.0177030601, -0.0232338365, 0.0809075683, 0.0170340985, -2.164640e-02, 0.0684185103, 0.0468361564, -0.0733145177, 0.0365309455, -0.0034388192, -0.0175086092, 4.423540e-02, 0.0529297329, 0.0233896468, 0.0277854055, -0.0554742627, -0.0184375476, 0.0593304038, 0.0308416858, 0.0365551636, 0.0558800027, -0.0518271029, 0.104938492, 0.143661678, 0.0614214763, 0.0790298283, -0.0175183564, 0.0462360047], [-0.0298722126, -0.00852189679, 1.098560e-02, 0.0743470788, -0.0422027744, 0.0588103198, -0.0139621189, 0.0697643533, 0.0347057171, 0.024920674, 0.0313822664, 0.00943168532, -0.0650867298, 0.00148646417, 0.0804172158, -0.019501023, -0.0696329474, 0.055937428, 0.0503342338, 0.0184029527, 0.0133446092, 0.0224395934, -0.0457295403, 0.0875915065, -0.0289623681, 0.0584105179, -0.0569108948, 0.0735515803, -0.0745876804, 0.0689982176, 0.0467095226, 0.0683550835, 0.00225079292, -0.0655354559, -0.0818356946, 0.0167576615, 0.0640111863, 0.027123617, -0.041881673, 0.0148028517, -0.0389454551, 0.0476284809, -0.0265342742, -0.081889987, 0.0669326782, 0.00841262843, 0.0150267882, 0.0150959361, 0.0692469701, 0.0952551141, -0.0617525875, -0.0388465561, -0.0128389709, 0.110924788, 0.0125874756, 0.0152033558, -0.0593223646, 0.035011109, -0.0655203387, 0.0133299055, 0.086261563, -0.0467296541, 0.0495833755, -0.060273394, 7.733250e-02, 0.103601322, 0.110915951, 0.0267694183, 0.0863095373, -0.00875790138, -0.0606100522, -0.0622353032, 0.0652420074, -0.0613900349, -0.0472411215, 0.0119368713, -0.0102270199, 0.00842462387, 0.0265488699, -0.0064140209, -0.0324543305, -0.0597804114, 0.00506518595, 0.0206133183, -0.0264293756, -0.0203744695, 0.0146803334, -0.0600162745, 0.118599959, -0.0397523604, -0.0712613463, 0.0051389928, 0.00860363058, 0.0435881428, -0.0468943566, 0.0340754688, 0.030505266, 0.0270811133, 0.0833658427, 0.0474330932, -0.103263795, 0.130369827, 0.0171857402, -0.0475291423, 4.661580e-02, -0.0987744182, -4.40090254E-4, 0.0308279227, 0.0251367502, -0.0278674457, -0.0164950769, -0.0597789362, 0.0681989565, 0.0864814892, 0.0988963916, 6.8113982E-4, 0.0118742203, 0.0162894223, 0.0332858302, -0.019197166, 0.0742955878, -0.0301273651, -0.0564155877, 0.0848935171, -0.0562591292, 0.0140264211, -0.0295543969, -0.0748923272], [0.0146279903, -0.00233062636, 0.0683380216, 0.0330227315, 0.00163405354, -0.0544203669, 0.0844610854, 0.0837785378, -0.082049638, 0.0384606235, -0.104747079, -0.0536681451, -0.0861313641, 0.0102382964, 0.0873454436, -0.00818266254, 0.0651938915, -0.0256621316, 0.0799000188, 0.0219063498, 0.0607685744, -0.0202068724, 0.0474731214, 0.0294055082, -0.0693639889, -0.0358359627, 0.0226916093, -0.0177207012, 0.0103171663, 0.0506609939, -0.0207120348, 0.0835802927, 0.0692696199, 0.0113196038, -0.0120293163, -0.0617096424, 0.0450809672, 0.00867352262, -0.0839720591, -0.00656189024, -2.683000e-02, 0.0165696181, 0.0826205387, -0.120590962, 0.0933224484, 0.0041480409, -0.0716290921, -0.0186789185, 0.0986644253, 0.0237330869, -0.0185431242, 0.0807238072, -0.0480556153, 0.0838195681, -0.0546243228, 0.0542951934, -0.0954677612, -0.0389035419, -0.0324214026, 0.0774301812, 0.0761291236, -0.017205691, 0.0908762962, -0.0819496512, 0.0458419882, 0.064635925, 9.431510e-02, -4.543400e-02, 0.0863609686, 0.0908929407, 0.0415472351, -6.833320e-02, 0.0185266417, 0.0115840938, -0.0308399089, -0.0701612532, -0.0783767774, 0.0157016013, 0.101023808, 0.0579106249, 0.0775098577, 0.0647790805, -0.040515624, 0.025297923, -0.0156456865, 0.0738582388, -0.00242140424, 7.816290e-02, 0.162772804, 0.0226185396, -4.593150e-02, -0.0319869183, -0.00553280907, 0.0917920842, -0.0779076144, -0.0410559736, -1.202660e-02, 0.0496669598, -0.0237005837, 0.0411932692, -8.446490e-02, 0.038370233, -0.0219471697, -0.0687337667, 0.0335848033, 0.0204615593, -0.111779116, 0.0616324916, -0.0682147294, -0.0579575785, -0.0667606369, 0.072825633, -0.038577728, 0.0589254349, 0.0897482857, 0.00976276863, 0.0228726063, -0.0254025813, -0.0543316342, 0.101422027, 0.108840138, -0.066262953, 0.0370601974, 0.0445030369, 0.0117686484, 0.0124606034, 0.0139140543, 0.00906094071], [-0.0331656858, -0.0217655413, 0.0573137291, 0.060151007, 0.060300298, -0.0264450442, 0.0301705357, -0.0456579179, -0.116989449, 0.0577170514, -0.0132199079, -0.0454500094, 0.063580133, 0.0475646928, 0.0922963693, 0.048006542, 0.0124186799, 0.0149485255, -0.0320664644, 0.0336542167, -0.0220637098, 0.0350849815, 0.0220066477, 0.105746381, -0.0484318435, 7.929160e-02, 0.0504971668, 0.0946964994, 0.00412141765, -0.0183162261, 0.0765415356, 0.0422444195, 0.0239989329, 0.0675087273, -0.0581254885, -0.0175940432, -0.0596080348, 0.0598184541, 0.0128699448, 0.123840325, 0.0428455807, -0.00541730365, -5.59113862E-4, 0.0830977484, 0.0480942279, 0.0244908612, -0.0907183513, -0.0453117304, 0.0152723929, -0.0299400352, 0.0359157585, 0.0221729148, -0.0378952734, -0.0634853393, -0.0370948538, 0.037199989, 0.0187971424, 0.0149717648, 0.0775207951, 0.0504855253, -0.0280195195, -0.0236800332, 0.0857738405, -0.0704348683, -0.0339997225, 0.0583513118, -0.0364373401, 0.0269475915, 0.0573919937, -0.0553491227, 0.0490088053, 0.0120099206, 0.0696427748, -0.0580429733, 0.0811496302, 0.00839084759, -0.0573067516, -4.202180e-02, 0.0789199919, -0.00343923201, 0.118299648, 0.00840880815, -0.0338307247, 0.0524960868, -0.0775198712, 0.0282282867, -0.00112861115, 0.0462239534, 0.0982457473, -0.0107129412, -0.0220806375, -0.0447605252, 0.0206179339, -0.00723984558, -0.0301610064, -0.0384925865, 5.126320e-02, 0.0425151773, 0.0338633209, 0.0685081705, -0.0016537332, 0.103024945, -0.0137642846, 0.0302299988, 0.133277982, -0.0120224757, -1.300810e-01, -0.0328347199, 0.0392852537, 0.0213161334, -0.0625054464, -0.0257837269, -0.0129560921, 0.0977863892, -0.027102178, 0.0442663468, 0.0145551069, -0.0459108055, -0.0467431284, 0.0993102192, 0.111102261, -0.0321697146, 0.018476231, -0.0329284631, 0.0337254964, -0.0450039469, -0.0451721884, 0.0574379526], [0.0365638211, 0.0456590727, -0.00280793128, 0.135910615, -0.0143133616, -0.0278617479, -0.0555028357, -0.0254531521, -0.0254785903, 0.0247962046, -0.0373940282, 0.0620827786, 0.0319456905, 0.0445999876, 0.0136287669, 0.0437258445, -0.0624339357, 0.0537810661, 4.381900e-02, -0.0332924798, -0.020983208, 0.0394922979, 0.0241546296, 0.0314943753, 0.0256478563, 0.0707760453, 0.047081098, -0.0347843692, 0.0509972721, 0.100644857, 0.104580015, -0.0592007935, -0.0477915704, -0.0160286557, -0.0402298085, -0.0735179483, 0.0701534152, -0.0898650959, -0.0814168751, 0.0821632742, -0.0523381457, 0.0548040383, 0.0299057327, 0.116821595, -0.0457816087, -0.0516001582, -0.0145489061, 1.50459411E-4, -0.00294604502, 0.052852504, -0.0529650189, 0.00870058126, -0.0555758364, -0.0639305338, 0.0219390932, 4.636960e-02, -0.0537725836, 0.0550288558, 0.0277136415, -0.0689116865, 0.0432283245, -0.0122596938, -0.0603205785, 0.0211039502, -0.0602206439, -5.097170e-02, -7.390220e-02, -0.0254634209, 0.105529174, -0.0417753384, 0.0126947658, -0.00723999552, -0.0644521043, 0.0449093655, -0.0150837339, 0.0931135639, 0.00525326235, 0.0687373504, 0.115773447, -0.00930766388, 0.0519569963, 0.0222965367, 0.0269583184, 0.0850727111, -0.131522924, 0.0425992757, -0.0711912066, 0.0801143348, -0.0279428661, -0.0727490484, 0.0582180247, 0.0150525635, 0.0517549925, 0.0569697469, 0.00247271801, 0.0626364723, 0.031231083, -0.0590275228, -0.0197152626, -0.0378778204, -0.0704013109, 0.101141676, -0.00493817544, -0.0473946668, 0.0599550195, 0.0160517897, -0.124403231, -7.187530e-02, -0.0226346552, 0.0277882181, -9.598940e-03, -0.00592164882, -0.0202452969, 0.032215137, 0.119735524, -0.0704075396, 0.0451410972, 0.0974848121, 0.0245421361, 0.0344568901, 0.129796416, -0.0508759543, 0.0558207259, 0.0460619107, -0.0207256693, -0.098352693, 0.10480471, 6.81575853E-4], [-0.053082142, 0.0575436167, -0.033745449, 0.0699397177, 0.0694985687, 0.0307704676, 0.0584660284, -0.052506879, -0.0619999021, -0.00842535775, 0.00903664901, -0.0103456397, 0.0708103925, 0.0126488386, 0.0307777859, -0.079082407, 0.00590077322, -0.026468765, 0.0183877219, -0.075850822, -0.0315182395, -0.0319181979, -0.0282272082, 0.0676160306, 0.110690698, 0.0414240211, 0.0358736739, -0.0360550471, -0.00116335414, 0.0865966305, -0.043698322, -0.0205981266, -0.0183657799, 0.0419068523, -0.0202908535, 0.0404993109, 0.0285461675, -0.0149017312, -0.0534217209, 0.0689036697, -0.0139431013, -0.0229708403, 0.0202291794, 0.104836039, -0.00120406866, 0.00322817685, -0.0954912379, -0.0201832205, 0.00531796506, 0.0205324199, 0.00936545152, 0.0131053431, 0.0109535009, 0.00759102078, 0.0380453169, 0.034945026, 0.0475349687, -0.0115720853, -0.0271255262, -0.0951571316, 0.0757760257, 0.0716428161, 0.0675049797, 0.0132222418, -0.0262828469, 0.0637158528, 0.0247562453, 0.078989841, 0.0539047457, 0.0787346139, -0.0375490189, -0.00515487185, -0.0759797394, 9.384370e-03, 0.0206046663, 0.00544454949, 0.0324068852, 0.0156948194, 0.056271892, 0.035335362, 0.0762295872, -0.014903374, -0.0246281382, -0.0586065389, -0.0363092907, 0.0916665568, -0.0330045931, 0.0173768699, -0.0561173335, 0.0442993417, -0.013969929, -0.0304537173, 0.00733471848, 0.0076441057, 0.0719546601, 0.0526287891, -0.0257704984, -0.0328165106, 0.0747460723, 0.0321739726, -9.065460e-02, 0.0589938313, 0.0227037817, 0.0112319514, 0.0855535641, -0.0881574228, -0.0761773661, -0.0558213666, -0.0262704398, 0.0161124021, 0.0464721434, -0.0460598022, -0.0206127428, 0.0182230826, 0.0403865799, -0.00818865187, 0.079508692, 0.0910207927, -0.00429450069, 0.0854081884, 0.036332678, -0.0589191504, -0.0281083547, 0.0650022626, 0.0228541549, -0.0637713894, 0.0315193459, 0.0421359204], [-0.0959343165, -0.0233869478, 0.0247479789, 0.070909217, 0.0633101091, -0.0339820422, -0.0342842564, -0.0452310368, -0.0108060194, -0.0737026631, 0.0500348657, -0.0581916086, 0.00382752833, 0.0138871679, 0.0989653542, 0.0231080521, -0.0546942167, -0.00488563953, -0.00600137655, 0.0429924093, -0.0472496338, -0.0870315134, -0.0496120863, 0.112812355, -0.0426505096, 0.12567468, -0.0535437539, -0.0236341916, 0.0558731146, 5.730170e-02, 0.0852892846, 0.03768529, -0.085931316, 0.075143218, 0.0274335593, 0.0815289468, -0.0333404578, 0.0122080827, 0.04538868, 0.0227220505, 0.015578012, -0.0641271472, -0.0216391291, 0.0316736549, 0.032581795, -0.0490692146, -0.033057645, -0.0711793154, -0.0118704066, -0.013353291, 0.0252403319, 0.0770491809, -0.0224839672, -0.0576522611, 0.0664617941, 5.486280e-02, -0.00954471435, 0.0425891355, 0.0807780846, -0.023362577, -0.057382673, 0.0622566715, 8.66794725E-5, 0.0341120325, -0.00380257261, -0.0366633795, -0.0633012205, 0.0658389255, 0.0288153514, -0.0879745483, 0.0805966556, 0.0528440848, 0.0532828271, -0.0518337227, 0.0798499509, -0.0302863829, -0.0586109757, -0.0699262917, -0.00405644439, -0.0568713583, 0.0970091819, 0.0835485234, 0.0554321893, -0.045021154, -0.122009426, -4.999220e-02, 0.0743773877, -4.10782901E-4, 0.0194400866, -0.0201606192, -0.0423773825, -0.0592890233, 0.0790925398, -0.0469271876, 0.0909573212, 0.0265125502, 0.0542677715, -0.0202550068, 0.0751067698, -0.03020069, -0.0799486711, -0.00430814363, 0.0417960547, -0.0804130062, -0.00432351977, 0.00132992747, -0.105137378, -0.0674503669, 0.0474837422, 0.0364761166, -0.0550210141, 0.020548638, -0.0796444416, -0.0412688479, 0.0696806535, -0.0532490201, -0.0335982144, 0.00332771195, 0.0184197519, 0.0654950142, -0.00180685753, -0.0306004416, 0.0201278292, 0.0273427367, 0.101975419, -0.025211243, 0.00172911584, 0.0615944155], [-0.0105311545, -0.00526999682, -0.0512639806, 0.126808628, 0.0250997152, -0.00169402559, -0.0299541764, 0.0784238502, 0.0312295854, -0.0439316258, 0.0439948253, -0.0602473617, 0.0117910365, 2.226070e-02, 0.0764123201, -0.0353422724, -0.0375971645, -0.0224139877, 0.0260472447, -0.0519751571, 0.0440995954, 0.0184602011, -0.0553605594, 0.00178558752, 0.0507315136, 2.157720e-02, -0.00805746857, -0.00474256324, 0.00329563161, 0.0817736238, -0.0445827246, 0.0530401096, -0.0672274455, -0.0144834239, -0.00997233483, 0.0171880666, -0.0028368826, -0.0132748783, -0.0658641234, -0.0433600843, 0.0141475322, -0.0327798761, 0.0398236923, -0.0605386421, -0.0318201482, 0.0400749929, -0.0282589011, 0.0248228069, 0.00800358504, -0.0348354019, -0.0681397468, 0.0858426317, -0.0290714111, -0.0800009742, 0.018093029, 0.0983806326, -0.0302833524, -0.0706838742, 5.014930e-02, -6.139380e-02, 0.0608448461, -1.70440078E-4, 0.0276738368, 0.0516595617, -0.00608667219, -0.084387362, -0.04737515, -0.0191797242, 0.0171974953, -0.0228313524, -0.0091364216, -0.0759517252, -0.0483449437, -0.038061887, -0.0115636475, -0.011265425, -0.00549038313, 0.0312509798, -0.0284299366, 0.00680866558, 0.119457766, 0.0675635189, -0.0647893175, -0.0336479582, -0.118838146, -0.0523870811, -0.0333494432, -0.0226300601, -0.0302478168, 0.00159683009, 0.0826579928, 0.0448832065, -0.0611969084, 0.0618318319, 0.0970052406, -0.0539313108, -0.0332218893, -0.0672797635, 0.0369085483, 4.07869491E-4, 0.010009747, 0.0792972594, 0.043013297, -0.0750379041, 0.028272517, 0.0612119101, -0.0408612043, 0.00630379515, 0.0338503942, 0.0427952707, 0.0204293244, -0.0641711503, -0.0895950496, -0.0564238727, 0.0153771853, -0.0944209322, 0.0403520241, 0.0399375521, -0.0590036027, -0.00832385104, 0.0895516201, 0.0465669334, -0.0375292636, -0.0261117611, -0.0386938676, -0.0599596426, 0.0602444299, 0.0758750438], [0.00210370193, 0.0579245463, 0.0465244576, -0.01477951, -0.0526264757, 0.00934842136, 0.0729081184, -0.0156632904, -0.0602647178, -0.0956065506, -0.0144529715, 0.0217804406, 0.0439390689, -0.0204681978, 0.0234588068, 0.0415982679, 0.0334049687, -0.0433477424, -0.0446905829, 0.0341702774, -0.0132316258, -0.0340754688, 0.0315469801, 8.040790e-02, 0.0637547821, 0.09431573, -0.0412016064, 0.00845950283, -0.0101362187, 0.0806906819, -0.00795043073, -0.0176105984, -0.0563923717, -0.0282542072, -0.0428800248, 0.0571919791, -0.0566069968, 0.0255735498, 0.0389744081, 0.00830852612, -0.0428484604, -0.00508131459, 0.0767603666, -0.0437035747, -0.0297372919, 0.0126599148, -0.0290101897, 0.0775219425, 0.081693992, -0.049842108, -0.0169317946, 0.0780081525, -0.0342301242, -0.063180007, -0.0370809846, 0.0453472845, 0.00920078065, -0.0739727244, 0.0719586685, -0.0562445931, 0.0447212756, -0.0521214195, -0.106924899, 0.0143785542, -0.0346005782, 0.027544979, 0.0323632397, -0.0254462827, -0.0361405797, -0.0924730896, -0.0387303606, -0.0413395688, 0.0280158222, 0.00396604603, 0.057970006, -0.0138469981, 0.00333689665, 0.0686446726, 0.0829843059, -0.0296551809, -0.0153344674, 1.701980e-02, -0.0540052168, -0.0305137206, -0.0380933136, -0.043781586, -0.0332517587, -0.0441173874, -0.0679779798, -0.0331308432, -0.0171753597, -0.0483678766, 0.0437857211, 0.0219580699, 0.0151861841, 0.0884611979, -3.367280e-02, -0.0273491498, -0.0460204221, -0.085771121, -9.446520e-03, 0.001888411, 0.0781787634, 0.0676189065, 0.0257891919, -0.0145147815, 0.028421944, -0.0854859128, -0.0116053186, 0.0317327753, -0.0360283144, -0.07547158, -0.0342262238, -0.0348575227, -0.0117097534, 0.0110893743, -0.0461482033, 0.0231859945, 0.0120235654, -0.00521789677, 0.0884884372, -0.0593513362, -0.0371667743, 0.00265251705, 0.0739365816, -0.0613793954, 0.0440307669, 0.0521536022], [-0.041055508, -0.0102049131, 0.0214571133, 0.013933138, -0.0195117705, 0.0849044397, -0.0579028092, 0.038271945, -0.0228737835, 0.0131426174, 0.0679285079, -0.051940456, 0.0765351802, 0.0584390424, -0.0392250344, -0.0544696487, -0.065817304, 0.086365208, -0.0368852131, 0.00432490557, -0.0692599341, 0.0418692976, -0.0205393247, 0.0856060311, 0.0126046473, 0.118925318, 0.0804605782, 0.0468303598, -0.0669492483, 0.0783017874, -0.0113918129, 1.802670e-02, 0.0231419988, 0.0613458753, -0.0222045034, 0.0524897538, -0.0103709409, 0.0617216229, 0.0118507473, 0.0450645313, 0.00971152447, -0.0685679912, 0.0383824147, -0.0570622645, -0.0752350837, 0.0508840904, 0.00211021258, 0.0402517058, -0.0330819227, 0.0718954727, -0.0459209271, 0.0201349221, -0.0691419765, -0.0255461149, 0.0659980475, -0.0196157806, -0.0532662719, 0.0595686808, 0.0927155986, -0.0126820663, -0.010335858, 0.0873452723, -0.0785138681, 0.0264254585, -0.0040647327, 0.00263174111, -0.0192833915, -0.0575323962, 0.0166002307, -0.0107707698, 0.0332079418, -0.0755106285, -0.0861243531, -0.0269816853, -0.07540977, -0.0488547571, -0.0191026293, 0.0479804166, 0.0309752468, -0.0163444653, -0.0308387149, 0.0258442294, 0.00501313806, -0.0852839798, -0.0493556447, 0.0198339652, 0.0145970723, -0.0763249993, -0.0664027706, 0.0428920761, 0.112698242, -0.0588252805, 0.0123563102, -0.041354537, 0.00525211031, -0.0481379218, -0.0559130721, 6.446550e-02, -0.0609480888, -0.0577637069, 0.0708779693, 8.125460e-02, 0.0514607579, -0.02632243, -0.0789060593, -0.0219785087, -0.0323109552, -0.0595330186, -0.0825284644, 0.0699994415, 0.0899382383, 0.0411328748, 4.243870e-02, 0.014892146, -0.0316001028, -0.075382188, 0.0081843324, 0.0479964241, -0.056777738, 0.0687098056, 0.0265829526, 0.0267724544, 6.773170e-02, -0.0302015115, 0.0897970572, -0.044246316, -0.02368683, -0.0433605611], [0.0432820283, -0.048235368, -0.0578727908, -0.0562083945, 0.037284404, 0.0480943806, -0.0529271364, -0.042822063, 0.0871438458, -0.0204680413, -0.0279813558, -0.0119872224, 6.303710e-02, -0.0342591554, -0.015773885, 0.0617755279, 0.0724636614, 0.00159376964, 0.0904464945, 0.0131594473, 0.0528432801, 0.0025334449, -0.0219368748, 0.0440646708, -0.0284040514, 0.0653689355, -0.0540265106, 0.0298260879, -0.0256548468, 0.0866264328, -0.0771483257, -0.0448183306, -0.0416712239, 0.0458534844, -0.0113237584, 0.107695796, -0.046353925, -0.0191794168, 0.028303083, -0.0835467577, -0.019204719, -0.0580648705, 0.0172497835, 0.00715440465, -0.0213349219, -0.0210609473, -0.125209779, -0.0363389291, 0.0259692892, 0.0150939776, -0.0643372461, 9.25169661E-5, 0.0378169343, 0.025168987, -0.0190500636, -0.0226090923, 0.0262009744, 0.0416296795, 1.7319934E-4, 0.00909302849, -0.0215817578, -0.0638851523, -0.0684097484, 0.091800943, 0.0706468076, 0.0162032042, -0.0732309148, -0.0500931852, 0.0314870439, -0.0866282656, 3.649310e-02, -0.0956767201, -0.0341426134, 0.0417751707, 0.0736892223, -0.03559177, -0.084112145, -8.799070e-02, 0.00888878573, 0.0528576896, -0.0159111638, -0.0566277765, -0.0502196029, -0.0188259147, -0.057242807, 0.0793127567, 0.0875953733, 0.0399301425, -0.126579508, 0.040772263, 0.0267218109, 0.00584202213, 0.0559072047, 0.0580648221, -0.016092686, 0.0574424565, 0.0680957586, 0.0237676874, -0.0732931495, -0.0940406545, -0.0636872053, -0.00360284746, 0.0620055682, -0.0417183079, 0.00759842433, 0.0981051176, -0.035124708, -6.295090e-02, -0.0121599045, 0.0426353626, 0.0797233209, -0.0131596057, 0.0476268232, -0.0187130366, -0.0903075411, 0.0624099262, -0.0579798967, -0.0313706808, 0.00278188498, 0.0332856849, 0.0520196222, -6.584980e-02, -5.495550e-02, 0.0499305055, 0.017861288, -0.0770876482, -0.0570864603, -0.0456419364], [0.0470219702, 0.0654747486, -0.0458788574, -0.0882793962, 0.0480794944, 0.0707179904, 0.0101267276, 0.0495196357, 0.0479207411, 0.0294134989, 0.00133586477, -0.0832905247, -0.0143340984, 0.0786468461, 0.0522024818, 0.0374060422, 0.0354404859, -0.0194391273, 0.0767743215, -0.00274898857, 0.00732794777, 0.047580868, -0.0708621368, 0.010961066, 7.888100e-03, 0.0759373605, -0.00172614784, -0.0433347709, -0.0207756478, 0.0319710039, 0.00346119958, -0.061817348, 0.100704551, 0.0347073488, -0.0404260382, -0.0359217972, 0.0393933579, 0.0677295625, 0.111054249, -0.0595495477, -0.0342305154, 0.0422750413, 0.0989012569, -0.114978552, 0.0254099593, -4.091990e-02, -0.0840889811, -0.0607109554, -0.0420694761, -0.0572670251, -0.0119597008, 0.0727350563, 0.0681720077, 0.0100072203, -0.0352529436, 0.0744679198, 0.0120063024, 0.0478518158, -0.0404277183, 0.0351528153, 0.0821106657, 0.0790903046, 2.33222876E-4, -0.016676316, 0.045874767, -0.0332609303, -0.020393433, 0.0483502336, -0.0942245349, 0.0458258279, 0.0520513579, 0.0555266328, 5.796750e-02, -0.0218750183, 0.0589130521, -0.0649398938, 0.0390813574, -7.948560e-02, -0.0052242307, 0.0638555735, 0.0405308418, -0.0732924789, -0.0822820439, -0.077515915, 0.00755431643, -0.0277407337, 0.0545899421, 0.0511147156, -0.0861937851, -0.0550209023, 0.0759527758, -0.00549655454, -0.0268358402, 0.0194157027, 0.0594059415, 0.0649742335, 0.0608776622, 0.00526322192, -0.091889739, 0.0045717638, 0.031409625, -0.0282283295, 0.113505721, 0.00998780317, 0.0673563853, 0.0180782769, 0.0866876915, 0.0158817172, 0.0419110768, 5.340480e-02, 0.038892027, 0.074308522, 0.0587537922, -0.0736447274, -0.0986359789, -0.0401141308, 0.0868978649, -0.0350334123, -0.0229840148, 0.0289402232, 0.0885570049, 0.0323991477, -0.00132026325, 0.0567219108, 0.0277812853, -0.021497041, -6.903390e-03, 0.0036225908], [0.02412978, -0.0368779525, -0.0139748137, -0.0089343572, -0.0113384239, 0.0391441025, -0.0434569865, -0.0424758755, 0.0566204935, -0.0350994356, 0.0119100809, -0.0111869583, -0.054728549, 5.5361932E-5, 0.00763771962, -0.0404440127, 0.0503703468, -0.0439753719, 0.00721731409, -0.0614923574, 0.0830472111, 0.0323753506, 0.0492940545, 0.0677958354, 0.0286728684, 0.0585644133, 0.0772045478, -0.0591266714, -0.0768652633, -0.0386984609, 0.0692165717, -0.00682013156, 0.0128921289, -0.0788050368, 0.0336595513, -0.0497592688, -0.0713232234, 0.0456933379, -0.0302798804, -0.00612740265, 7.682790e-02, -0.00950897951, -0.0615328327, -0.0342110917, -0.0102662984, 0.0603151545, -9.572280e-02, -0.0878656358, -0.082609877, -0.080328837, 0.0118245315, 0.0389895178, 0.0119504333, 0.0546845272, 0.0238166526, -0.0657025799, 0.00568899931, -0.0287804697, 0.0491663702, -0.0538887717, 0.0450319573, -0.0327585302, 0.0649573728, -0.0190474428, -0.0606613643, -0.0847459957, -0.0795775726, -9.481360e-03, 0.0475730933, 0.00851398241, 0.0037023609, 0.0426902287, -0.0721890554, -0.0420985594, -0.0482159965, 0.071024023, 0.0587541275, -0.0220053717, -0.0534976907, 0.0582055449, 0.0185071342, 0.029483892, 0.0235997345, 0.0330523178, 0.0739406124, 0.0169568677, -0.00853496883, -0.0235625915, -0.0741150305, 0.0121413385, 0.0785390436, -0.0542904139, -0.014147426, -0.0387908071, 4.201050e-02, 0.0481669717, -0.0126820775, 0.038432207, 0.071401462, 0.0401888378, 0.0519683957, -0.0310160853, 0.0421593636, 0.0554473922, 0.0472597927, -0.039396029, 0.0726590306, -0.00532728946, -0.0570625328, 0.0584669039, 0.0691901892, -0.0472689942, 0.0203117952, 0.0561979935, 0.00430577062, 0.0130734537, 0.0757854804, -0.0949142128, -0.069589369, 0.0483477935, -0.0344728902, 0.0281459838, 0.0681504831, -0.0181834437, -0.0199008659, 0.0735037476, 0.0156471021, 0.0451932698], [-0.0313205905, 0.0420907401, 0.0607947819, 0.0113513842, 0.00888972915, -0.01733399, -6.21279542E-5, -0.0393697731, -0.0530872755, 0.0273113772, 0.0212382488, -0.0184054449, 0.0473411195, 0.0521119945, 0.0474056862, -0.0244739987, 0.0161907114, 0.0440364517, -0.0651558339, 0.00368596218, 0.0583044142, -0.0156025318, -0.00289654452, 0.0446090847, -0.0370829664, 0.0663857907, 0.00664912816, -0.0440426096, 0.0409759618, -0.0501171276, 2.48244469E-5, 0.0091889808, 0.0103831934, 0.0628162622, 0.0212521926, -0.074570857, 0.0585561097, 0.0818434879, -0.0513526052, 0.0319776312, 0.038381204, -0.0146939885, 3.660130e-02, 0.0736987218, 4.494420e-02, -4.07616135E-5, 0.0114191864, 0.0454267859, -2.40807625E-4, -0.0698803589, -0.00664507783, -0.0143127823, -0.0151361832, 0.0735530183, -0.0366988182, -0.075511314, -0.034468554, 0.0271975752, 0.0591454282, 0.0529964305, 0.0262835864, 0.0188898835, -0.071359016, 0.0609413609, 7.639390e-02, -0.0534945838, 0.0122588128, 0.0439968444, -0.0680183545, 0.0130984448, -0.0366805047, 0.0407074094, 0.0629008785, 0.0685237199, -0.0656781494, 0.0587263182, 0.00242654653, -0.00215948652, 0.0250582304, -0.0489262082, -0.0203188807, -0.0372996442, -0.0595052615, 0.0412197188, 0.0166034233, -0.0225153454, -0.060824804, -0.0196286943, -0.0326322392, -0.0108650764, -0.0639685541, 0.00303264917, -0.02684878, 0.0162523575, 0.0784273222, -0.0592085458, -0.0433154292, -0.0423721895, -0.0408479087, -0.00475933682, 0.0454989597, -0.0478104576, -0.00239853887, 0.0725784525, -0.05296259, -0.0459496304, -0.0720862821, 0.0687237456, -0.0769793987, 0.0759952962, 0.0488826297, -0.016300939, -0.0506488234, 0.0443148948, -0.00297704176, 0.0617052018, 0.0576781705, 0.03855744, -0.0556494258, 0.0684411898, -0.076003477, -0.0602145717, 0.00725815399, 0.00886816903, 0.078232944, -0.0376759879, 0.0127067342, -0.0765467435], [0.00238346914, 0.0136847366, -0.0462072492, -0.0261410773, 0.0157639049, -0.0710341483, -0.0393643975, 0.0311136767, -0.0719555393, -0.0460162349, 0.0684917867, 0.0543655753, 0.0736005455, 0.0311874952, 0.0517863818, 0.011373614, 0.0146659613, 0.0581172071, -0.0422235392, -0.0311980825, 0.0390662476, -0.05123391, -0.0255240761, -0.0474953093, -0.0391206741, 0.0545734428, 0.00828306656, 0.00729210302, -1.197190e-03, -0.0446774848, -0.0589379817, 0.0021483053, 0.0212921202, -0.0153089575, -0.0640780851, 0.0317333899, 0.0502224974, -0.0706683919, 0.0644219294, 0.0604341328, -0.0475634299, -0.0709341392, 0.0175076164, 0.0431913957, 0.0688494369, -0.0253798086, -0.0403602906, -0.0149765508, -0.0392674841, -0.0646356121, 0.0110738985, 0.0528378747, 0.0741680562, 0.0803219825, -0.00374998269, 0.0141434018, -0.0741745159, -0.0674148127, 0.0476801172, 0.0201451369, -0.025009755, -0.0441477075, 0.0750341639, 0.0628108755, 0.0475638621, -0.0262687951, 0.0378329791, 0.00613671681, -0.0167431384, -0.0678435862, 0.0248757228, -3.239930e-03, 0.0520584285, 0.00137634634, 0.0547999814, 0.0637210905, -0.0456264801, -0.00345165771, -0.0031885223, -0.0442970507, -0.0107395453, -0.0466889441, 0.0610973649, 0.0305353366, -0.0268742163, 0.0441056341, 0.00650874479, -0.0805573686, -0.0775287374, -0.0284222923, -0.0105310092, -0.0341581665, 0.00449386612, -0.0602624714, 0.0211023409, -0.0599722117, -0.00689969445, -0.0591524728, -0.0219552889, 0.00737011712, -0.0286992528, -0.0702293813, 0.0610482879, -0.013781854, 0.0369412675, -0.0651806071, -0.0326018892, 4.309920e-02, 0.0602545664, -0.0444377549, 0.00685635116, 0.0452573076, -0.0646880195, 0.0173275732, 0.0402867086, -0.0527846254, 0.0277893506, 0.0138401361, 0.0750117898, -0.0684358254, -0.0466892235, -0.0250596311, -0.0129342712, -0.0335692652, -0.0796470344, 0.0575997084, 0.0797669291, 0.0106093837], [0.0766989514, 0.0686537847, 0.0330328457, -0.0615294166, 0.0649604648, -0.0180513263, 0.0408674888, 0.0303744879, 0.0695960223, 0.0740236416, 0.0318153836, -0.0624040887, 0.0584889613, -0.0418721884, -0.0780925154, -0.00970049482, 0.00892131776, -0.0585726313, 0.0557209253, -0.0519321859, -0.0670765862, 0.0178748928, 0.0377557203, 0.0287910216, 0.0575907081, 0.0461394265, 0.0734344795, -0.0451366492, 0.0505691022, -0.0266053658, -0.0492235385, 0.0440036803, -0.044415541, -0.0378196388, 0.0469916798, 0.0233998913, 0.0258825589, -0.0453865677, 0.0572184846, 0.0448153019, -0.0122468248, -0.02848313, -0.0351096578, 0.015146967, 0.040944431, 0.0575919971, -0.0632751212, 0.0714364722, -0.0151898814, -0.0566115156, 0.0227685533, 0.00391025841, -0.0689414441, -0.00619376125, 0.0184446275, -0.0150313359, 3.929530e-02, -0.014157868, 0.0109521346, -0.0217971969, 3.887630e-02, 0.014450008, -0.0480024889, -0.0613635369, -0.01870423, 0.0694657788, 0.0343496539, -3.240300e-02, 0.0456337519, -0.0643877089, 0.0736443326, 0.0489237085, 0.0122771449, 0.0272092335, -0.0587356575, 3.727780e-02, 0.0064433259, 0.057155706, -0.0564290769, 0.0404156744, 0.065199323, 0.0781445875, -0.0595804118, 0.0157967731, -0.0465661287, -0.0019090483, -0.0588782169, -0.00424122717, 0.0703216493, 0.0681772828, -0.00657795463, -0.0641232133, -0.0728215277, 0.0269956272, -0.0534856543, -0.053162463, 0.0183613598, 0.0433930419, -0.0241158344, 0.0202962887, 0.0117792608, 0.0345767327, 0.0135797039, 0.0238160603, 0.0630136281, 0.0392679274, -0.062127944, 0.0666239411, -0.066505976, 0.0331162736, 0.0652488247, -0.0596469268, 0.0484294705, -0.0207798854, -0.0472083241, -0.0267203897, 0.0552126467, 0.00450464198, -7.764070e-02, -0.0355861075, 0.0586643368, 0.0199997891, -0.0762867555, -0.0069911466, 0.0747021138, 0.0242286082, 0.0808467418, 0.0603358671], [0.0390876196, -0.0108499527, 0.0143223433, 0.0231686942, -0.0379315875, -0.0345175602, 0.0507826209, -0.00993240345, 0.0343470685, 0.077639617, 0.00933449901, -0.041182369, 0.00683440547, -0.0745364279, 0.0717572421, 0.0411082171, 0.0230461322, 0.00396159058, 0.0143994484, -0.0715402886, -0.0573385619, -0.0676267818, -0.0715662837, 0.0518760718, -0.0794751048, 0.0513222478, 0.0499276631, 0.0451253764, -0.0277306326, 0.0172014218, -0.0788837745, 0.0139779467, -0.0109154517, -4.618320e-02, -0.0778984054, -0.0028321431, -0.0378175192, -0.0595728345, -0.0573486462, -0.0488762856, -0.0331572779, -0.0670710355, -0.0173562039, 0.0381457172, -8.637830e-03, -0.0710420087, 0.073427029, -0.0810918063, 0.0415125303, -0.0472593904, 0.0261671916, -0.0296436101, -0.0292386431, 0.0452492088, -0.0492903925, 0.0443706289, -0.0796691626, 0.0736612827, 0.042648036, 0.0575810485, 0.0732618496, 0.0575013831, -0.0734341442, -0.0378795825, -0.0517587736, -0.0110916942, -0.0361897908, 0.0488076881, -0.0103143873, -0.0253175814, -0.0385023355, -0.0577598624, 0.0257825349, 0.00456471788, -5.350920e-02, -0.0103666419, -6.126690e-02, -0.0242886413, -0.0330552645, 0.0127792545, -0.0635106713, -0.0441504605, -0.0429006815, -0.0398491174, 0.00264547952, 7.015880e-02, 0.0201146528, 0.0558903702, 0.0288081914, -0.0609986223, 0.0701737627, -0.073280558, 0.0527980104, -0.0520877466, 0.0184415374, -0.0299283359, -0.0610087215, 0.0524659716, -0.0484851524, 0.0415005386, 0.0551168248, 0.0160674937, 0.067893818, 6.89750653E-4, 0.0807618424, 0.0513534807, 0.0686639473, 0.0165457949, -0.035739325, 0.0205543693, 0.00795974303, -0.0807927176, 0.0611012429, 0.0627482086, -0.0619554817, 0.0808618366, -0.0308293588, -0.0424873605, 0.017349001, 0.00505455676, -0.00845956057, 0.0679714233, 0.0136459768, -0.0210518483, -0.0670213923, -0.0267056692, 0.021555759, -0.0797565206], [-0.0119339125, -0.00697571132, 0.0811643972, -0.0175262187, -0.0162147656, -7.87525088E-4, -0.0204180945, -0.0215694439, 0.0390510485, 0.0331533477, 0.0230834708, 0.00386192719, 0.0654024482, -0.0697859079, -0.0818157047, 0.00675150193, 0.0499156862, 0.0211410522, -0.0752662346, 0.073061347, 0.0255554877, 0.0770402327, 0.00490230601, 0.038965892, -0.0147406077, 0.0499776937, -8.256270e-02, 0.0777536482, 0.063768819, -0.0479498394, 0.0112695396, -0.0187104121, -0.0163210761, -0.0697788149, -0.0552717187, -0.0690020695, 0.0622414983, 0.044761546, -0.00306267175, -0.0324050635, -4.91442799E-4, 0.0486177951, -0.00585428346, 0.0782280638, 0.0593280718, -0.0124561526, -0.0649919211, 0.00209100265, 0.00457887072, -6.777470e-02, 0.0648658425, 0.0520468876, -0.00435504643, 0.0199033897, 0.0628156438, -0.0502719022, -0.0224627592, 0.0411273837, 0.0398699269, -0.048507791, -0.0196248014, 8.371200e-02, -0.0415574796, 0.0684820414, 0.032807488, 0.0643477291, 0.00741593214, -0.0686283782, 0.0421214215, -0.0715919882, -0.0625283942, 0.0765404701, -0.00543182809, -0.00412973529, 0.0120491805, 0.00899030175, -0.0417636037, 0.0485787243, -0.00637109904, 0.0159316268, 0.00156980485, -0.0464701653, -0.0601973869, 0.0627760738, 0.0296880249, -0.0593287535, 0.0510917604, 0.0743803829, 0.031961225, 0.00855013541, 0.0521518737, -0.0804109722, 0.0216969568, -0.0695585087, -0.0653384625, -0.0431616753, -0.0498325862, 0.0484428629, 0.0239072088, -0.0130842989, 0.00487300055, -0.0185769461, 0.0690538511, -0.0588405132, 0.0391003937, -0.0299986303, 0.0104619283, -0.00979189668, -0.0662729666, 0.0666886345, 0.0301643144, -6.956160e-02, 0.00953182391, 0.040221408, -0.0430423245, -0.0431582592, 0.0107715046, 0.0790609866, -0.0678625628, -0.0594913103, 0.0410792157, 0.0772633552, 0.050435394, -0.0124758342, 0.0273947082, -0.00701987324, 0.069283314, 0.0261544455], [8.09320656E-4, -0.0368699059, -0.0671592802, 4.365260e-03, 0.0537009649, -0.0123749478, -0.0475597978, -0.0769290552, 0.0461685359, 0.0112866703, 0.00850611366, -0.0555780232, 0.011118181, -0.0244328789, 0.0269265901, -0.0213061813, -0.00133831217, 0.0590626746, 0.0221618656, 0.0447206795, 0.0678807944, 0.0660601631, -0.0145452637, 0.0227347538, -0.0669094324, 0.0295511298, -0.0799501389, 0.0467598774, -0.0147506185, -0.0460815802, -0.0786116644, 0.03613263, -0.0693489835, 0.036112044, 0.00385985826, 0.0435843766, -0.060502436, 0.0246212631, -0.00895088445, -0.0124138482, -0.0172377899, -0.0765506327, -0.00492725568, -0.00726220291, 0.00103820954, 0.0515810363, 0.0608679391, 0.00184270425, 0.0593554787, -0.0696395114, 0.0788642764, 0.0420839526, 0.0108354222, 0.0490017943, 0.0725556761, 0.0625607893, 0.0765765235, 0.0386452377, 0.00580282789, 0.0124752969, 0.0644277185, -0.0495839678, 0.0459576137, -2.40826368E-4, 0.0576410517, -0.0321880691, 0.00886817928, -0.00552194659, -0.058786843, 0.0375886299, 0.0668517053, -0.00141624419, 0.0140399542, 0.0597923137, 0.0547706522, -0.0739664063, 0.0147073912, 0.046653606, -0.012260193, -0.0793458372, -4.164320e-02, 0.0054944735, 0.0040083304, 0.0355067775, 0.0470554642, -0.0362502597, 0.06990242, -0.0641464368, -0.016355224, 0.0214310698, 0.00686724344, 0.0430197418, -0.00692193396, 0.0666814819, 0.0500990301, -0.0180070959, -0.0313404463, 0.0186245814, -0.075556621, -0.0231812559, 0.0757438317, -0.04796369, 0.0658539831, 0.0150804343, -0.0115180695, 0.0540054366, -0.0455246419, 0.0130089065, 0.0409009121, -0.0676825046, 0.0546855964, -0.0316899978, 0.0174232218, 0.0589760803, 0.081564933, -0.0538081378, -5.125500e-02, 0.0410173312, 4.885570e-02, 0.0457514711, 0.0715550929, -0.0746677815, 0.0106876986, -0.0328264497, 0.0763977692, 0.0606905632, -0.0401487537, 0.00323917088], [-0.0508545935, 0.0689548552, 0.0663484856, -0.020017257, 0.0544697382, -0.0373624489, 0.0389011875, 0.00610918691, 0.026226325, 0.00599938165, -0.044799909, 0.00644891383, -0.0637125894, 0.068627134, 0.0187943075, 0.0129215065, 0.0554083772, -0.0312402546, 0.062141139, -0.043159306, 0.0593954548, -0.0707671195, -0.0488448106, 0.0491157845, -0.0574409775, -0.0709969774, 0.0266318098, 0.0515799485, -0.0205582455, -0.0399173684, 0.0368081182, -0.0264653154, 0.0236994401, -0.0176667701, -0.0443812162, -0.0179795958, 0.0610544793, 0.0433706865, -0.0238040704, -0.0483058058, -0.0738205388, 5.552200e-02, 0.0110496338, -0.0817779154, 4.911100e-02, -0.0334136486, 0.00260890415, -6.300420e-02, -0.0779859051, 0.0416772775, 0.0292971283, -0.0304102823, -0.0111541981, -0.0490087122, -0.0688979477, -0.0140976477, -0.0619197599, -0.0222822968, 0.0647280812, 0.0105102444, -0.049168922, 0.072517395, 0.0488494337, 0.0459009707, -0.0401254147, 0.0816015154, -0.00875909906, 0.0337469168, -0.0499397926, 0.0406500511, -0.0207940098, -0.0512412302, 0.0307764225, 0.0162277259, 0.0134457191, -0.0804209411, 0.0494892262, -0.00460812869, 0.0885896459, -0.0475439057, 0.0471667424, 2.394943E-4, -0.0800809189, -0.064654626, -0.0703744665, -0.016361082, -0.0413722433, 0.0490157902, 0.0314781293, -0.00128732296, -0.0281854272, -0.0391725227, -0.059563037, -0.054383751, 0.0221057776, 0.0590516515, 0.0261923503, -0.0389626361, -0.0578997433, -0.00237287208, 0.0788485705, -0.0229479764, -0.0446422473, 0.0404927693, -7.942800e-02, -0.0552858673, -0.0616496168, 0.0178067666, -0.0372995213, -3.314620e-02, 0.00820132345, -0.0108527485, 0.0459137522, 0.0574271493, 6.356160e-02, -0.0170191713, -0.0215352606, 0.0721374899, -0.0597336702, 0.00656585908, 0.0189568307, -0.0718588084, 0.0237889457, 2.42521201E-5, -0.0110835014, -0.0497553349, 0.00961685274, 0.00248954236], [0.0437887609, -0.0351058729, 0.0839904919, -0.0601058304, -0.0754265338, 0.00198802887, -0.0277259983, -0.0151659874, -7.247280e-02, -0.0775835514, -0.00267809979, 0.0031324483, -0.0042555877, -0.00366385723, -0.0114534227, -0.0161422882, -0.0583115108, -0.0050653317, -0.0776257962, 0.0638865456, 8.679810e-03, 0.0206230171, 0.00174055633, -0.0697536319, 0.0210746601, 0.0742966607, -0.0730971321, -0.0286116656, -0.0501110069, -0.0157904979, -0.0134233478, 0.0493438728, 0.0616387539, 0.0777092054, -0.0557460971, -0.0316091627, -0.00420460291, 0.0525719188, -0.0464789346, 0.0746909752, -0.00988953746, -0.0639079735, -0.0350605212, -0.070330672, 0.0570219457, 0.0536594763, -0.0199288372, 0.0284384433, 0.0141680809, 0.0514178723, 0.0257684812, -0.080767177, 0.0277336091, 0.0827617272, 0.0393845662, -0.0584465042, -0.0322815441, 0.0108075887, -0.0824487879, -0.0600514449, 0.0716954246, -0.0607845113, -0.0717981383, -0.00381667423, 0.0382925235, 0.0695336461, 0.0838419273, 0.0267718267, -0.0312359426, 0.0640570223, 0.0600965917, -0.0386643372, 0.0268732179, 0.0404301398, -0.0519886911, 0.0442331582, 0.0310154315, -0.080324091, -0.0200364329, -0.0282759834, -0.0697002336, -0.0452331901, -0.074562721, -0.0403216854, 0.0626699924, 0.0319516249, 0.0139538795, -0.0713715479, -0.026375033, -0.0683491156, 0.0254861489, 0.042326387, 0.00698717869, -0.00817846134, 0.0471224189, 0.00351259508, 0.0471716225, 0.00681234291, 0.00655241496, -0.0552639253, -0.0239342581, -0.0108322827, -0.0425077155, 0.0873742253, -0.0163818076, -0.0110807614, -0.0656239614, 0.05180097, -0.037149664, 0.068994239, -0.00914463773, 0.0147635024, -0.0397440717, -0.0480082929, 0.0872814431, 0.0259600468, -0.0667583793, 0.0247465018, -0.013076338, 0.0403893404, 0.057736069, 0.0180521328, -0.0929030329, -0.0570275448, 0.0378984958, 0.0120331701, -0.01786636, 0.0435853489], [0.0246690456, 0.0263384935, 0.0841644108, -0.0709701926, 0.0287052561, -0.0229223501, -0.00481706532, -0.00123395526, -0.0157612469, 0.0557868928, 0.0262642018, 0.0649682134, -0.0524460636, 0.02696741, 0.041602429, 0.0536786877, 6.663810e-02, 0.0797929391, -0.0554072447, -0.0733925626, -0.029154893, -0.0247497484, 0.0321669616, 0.0795789212, 0.0186477043, -0.0303046759, 0.0558105446, 4.9131416E-4, 0.044929333, 0.0637768358, -0.0328490883, -0.00374622643, -0.0703752264, -0.074448891, 0.0598371811, 0.0176157877, 0.0810833573, -0.014589238, -0.00192956196, -0.00954780821, -0.0121325962, 0.0437128767, -0.0240031704, 0.00920005328, -0.067216523, 0.0618199855, -7.5343193E-4, -0.0445338637, -0.0571761765, -0.028055843, -0.00581746409, -0.0542398654, 0.0506127737, -0.0113504985, 0.0383568667, 0.0236636531, 0.02717489, 0.0232466497, -0.0578602143, 0.0790796875, -0.0773071125, -0.0109824575, -0.0805372745, -0.060154289, -0.0806683525, 0.064805612, -0.0434409268, -5.834050e-02, -0.031641718, 0.0236546211, 0.0123194745, 0.0622978732, 0.0359206423, 0.019613415, -0.0283948388, 0.0245418157, -0.0708752349, 0.0328957215, 0.011569663, -0.0484116748, -0.0428688154, 0.00325662619, 0.0439063273, -0.0304555129, 0.0441054106, 0.0937307775, -0.00413178047, 0.0292264577, -0.00356528792, 0.0886822566, -0.0499018915, -0.0545351543, -0.0184683092, 0.00674246065, -0.0578307323, -0.0181973614, 0.0569943786, 0.0724857077, -0.064018093, 0.0577532761, 0.0449570045, -0.0581197105, 0.011406159, -0.0148339113, 0.046759665, 0.0337230675, -0.041067373, 0.0524470806, -0.0643319041, 0.00673547387, 0.0720746741, -0.00605388358, -0.0456363223, 0.0202893242, 0.0815911889, 0.0102703162, -0.0121545866, 0.0240552481, -8.518250e-03, 0.0752870589, 0.0653395653, 0.077440545, -0.041300483, 6.028910e-02, -0.0679294616, 0.00300703826, -0.0171675179, 0.0592744499], [0.00566751277, -0.0946181416, 1.26420113E-4, -2.571840e-02, -0.016272638, -0.0110472301, -0.0330797583, 0.00966523681, -0.0614180192, 0.0430164337, -0.0821547359, -9.695240e-03, 0.0382311903, 0.00238273758, -0.00703816814, 0.0762561038, 0.0475406423, 0.0462624244, -0.0114448136, -0.0544217937, 0.0286924597, -0.00608330825, -0.0747856498, -0.0648662224, -0.00979699288, 0.0716515258, 0.0525422655, -0.0193795674, 0.0216109864, -0.0401532575, 0.0172717366, 0.0526117086, 0.0339658484, -0.0335933827, -0.0705477595, -0.0343910046, -0.018105194, 0.0222960152, 0.0757896826, 0.0347164311, 0.0529587753, -0.029952528, 0.0487094745, -0.00181576039, -0.0661189333, 0.043974258, -0.0210881643, 0.0415508337, -0.0845746845, -0.0356975421, -0.0346828587, -0.00958979595, -0.00779683795, -0.00887454301, -0.031978365, -0.0611347295, 0.0704044476, -0.0157517046, 0.063368231, 0.0483533703, 0.0617732517, 0.0222387239, -0.00137237576, 0.0705730841, 0.0420047417, -0.00295986515, 0.0643482953, -0.0106860418, -0.0540179275, 0.0445876643, -0.0498021655, -0.032322105, -0.0211397689, -0.0408615023, 0.0719833896, -0.0503150932, 0.00395684037, 0.00846392382, 0.0528213866, -0.0490305386, 0.00388341676, -0.081235595, -0.0718710348, -0.0253600441, -0.0211141054, 0.0216925628, -0.029098561, 0.0796132311, -0.0162668303, 0.0539494939, 0.0573485233, 0.0477574691, 0.0968036875, 0.0494811796, -0.0241412017, 0.0559506901, -0.0400326885, -0.0300438628, -0.0160784516, -0.0791630521, 0.0158042964, 0.0677790791, 0.0152720194, 0.0445627235, -0.0515069477, -0.017589638, 0.0554326624, 0.0391720645, -0.0927644819, 0.05811207, -0.0683281347, 0.0293553695, 0.0344709493, 0.027034577, 0.00427417923, -0.0626342222, 6.81580103E-4, 0.0609444194, 0.0307902731, -0.061816372, -0.0358228534, 0.0640305132, -7.143060e-02, 0.0513459034, 0.0585191771, -0.0803807452, -0.00358257513, 0.0718296543], [-0.0877327322, 0.0450851358, 0.0461130328, -0.100545354, -0.0191032272, 0.0398325399, 0.0638010874, -0.0351097845, 0.100400418, -0.0285933092, -0.0249816664, 0.00296259392, -0.0444116294, 0.0504990034, 0.028566869, 0.00612534257, -0.0702707246, -0.00542152114, -0.0511070192, -0.0162102487, 0.02517456, -0.0792593211, -0.0794132277, 0.0800001993, 0.0137871169, -0.0435402542, 0.0644140169, 0.00536211487, -0.0481808521, -0.00647169584, -7.611220e-02, 0.0867042541, 0.0568350405, -0.0317617841, 0.0956633761, 0.0625200793, -0.0582914464, 0.0098488247, 0.0145446043, -0.0304402132, -0.0778984427, 0.0268963017, -0.00753638754, 0.0250242781, 0.0848766565, -0.0504262336, -0.0392323434, 0.0763001069, -0.0326115452, -0.00641085487, -0.0149204042, -0.0817947611, 0.0239365585, 0.0266769938, -0.002241462, -0.0347377621, 0.0112218177, 0.0113276187, -0.0405463129, 0.0276419055, 0.0207273588, 0.0568039306, -0.0663373098, -0.0432166643, -0.0668391138, 0.00436068745, 0.0566104576, -0.0297540929, -0.0603192411, 2.944930e-02, -0.0411624536, 0.0643539354, -0.0829878598, -0.0940703749, -0.0632270202, 0.0840567797, -0.00541977631, -0.0533558279, -6.491000e-02, 0.0451166444, -0.0181781128, -0.0641622618, -0.0139739178, -0.00419524312, 0.041440282, -0.00397295272, 0.0845058262, 0.0397611558, -0.0108832866, 0.0125073735, 0.0900638997, -0.105002888, 0.0746246129, 3.387280e-02, -0.0790620222, -0.0391638316, 0.0745501667, 0.0801852568, -0.041526109, 0.0411799029, 0.102052152, -0.0449444801, -0.0405049101, 0.0439410508, -0.0516349152, 0.0985663905, 0.125236124, 0.0423981212, -0.0761772916, 0.0938324332, -0.0850574076, 6.303760e-02, -0.00453290762, 0.0111159077, 0.0359435603, -0.0678813308, -0.0100255692, -0.0113626784, 0.0694164708, 0.0213226937, -6.77825417E-4, -0.0118060214, -0.0491661355, -0.0219026934, -0.0283579864, 0.0395734645, -0.03066241, 0.0040724948], [-0.080988422, 0.011865424, 0.0613728463, -0.0324796513, 7.099550e-02, -0.0278583113, 0.029133644, 0.0302522965, 0.114336565, 0.0078573348, 0.00449904473, 0.042034857, 0.0703269765, 0.0525806546, 0.0445683189, -0.00728344778, -0.0340565555, 0.079355523, 0.00199995283, -0.021554973, -0.0235205907, -0.0811498239, -0.0116570489, 0.0031364304, 0.0656998157, 0.0813446417, 0.0479583591, 0.0308147706, -0.0483826585, -0.0297115408, -0.0916895642, 0.0297844093, -0.0701575056, 0.00181778811, -0.0442547388, 0.0680296347, 4.06743238E-5, 0.0454959758, 0.0916034653, -0.0285292231, 0.0369420834, -0.0604869202, 0.00246033445, 0.0370902754, -0.0132898949, 0.0600637682, 0.00695254933, 0.0464417674, -0.0030629416, -0.07676813, -0.0353586748, -0.0340613127, -0.0109237246, 7.809810e-02, 0.0273337215, 0.0798396095, -0.0438077189, -0.0118435742, -0.0417589135, -0.0299135912, -0.0151721062, 0.0570919029, 0.0761050283, -0.0185770225, 0.0276892073, 5.589830e-02, 0.0754324421, -2.86349881E-4, -0.0443241373, 0.10124024, -0.0224246364, -0.0304213744, -0.00181902875, 0.0150531624, 0.017634023, 0.0355494246, 0.0273931306, 0.00816872902, -0.0311821345, -0.0538909137, 0.0187982079, -0.0630675405, 0.0153872203, -0.0116815036, 0.0303574558, -0.00805144664, 0.107464403, -4.570650e-02, -0.0806292742, 0.0386964157, 0.00917760562, 0.0201224536, -0.0156062217, 0.047177393, -0.0549512915, -4.925620e-02, -0.00688659679, -0.0123873977, 0.0266905054, -0.0741612092, 0.117356673, 0.0803024322, 0.0252936259, 0.00823558401, -0.0662313401, 0.00132655306, 0.0414912105, -0.0113572516, -0.0792365074, 0.0327618308, -0.0315942541, 0.0576872677, 0.0151608959, -0.0992357805, 0.0178586897, -0.0606615581, 0.0891132131, 0.0672724769, -0.0593152791, 0.0776410252, -0.0528223515, 0.0609914288, -0.0404192135, -0.038934052, -0.07412301, -0.0337913446, -0.0702988803, 0.0040781307], [-5.198670e-02, -0.0129960207, -0.0498910286, -0.0455708802, -0.0375145227, 0.0467212498, 0.0155459046, -0.0294895731, 0.0894337967, 0.0732486323, -4.64518642E-4, 0.0353526883, 0.0396356955, -0.0290449429, 0.0732489675, -0.00330540817, -0.0307788979, -0.0254910365, 0.0652754903, -0.0370867774, -0.0403648205, 0.0664109439, -0.0148976222, 0.0807241499, 0.0466566123, 0.0644658282, -0.006371947, -0.0280449595, -7.262420e-02, 0.0727206841, -0.0484235547, 0.0613435768, 0.0596484877, 0.085591562, 0.0786739066, 0.0175098181, -0.0524351187, 0.00840769615, 0.0218622088, -0.0274102297, -0.0691785142, 0.025133891, 0.0441036522, 0.0575901493, 0.00122754148, -0.0262841899, 0.0409106351, 0.0916418582, -0.0404725112, -0.00812087766, -0.00798522402, 0.0320324674, 0.109077334, 0.0114338854, -0.0516575389, -0.0247383676, -0.0392485186, -0.0128922369, -0.0849623084, -0.00439028163, -0.0142211532, -0.00127970858, -0.0402407385, 0.0669501498, -0.0683089569, 0.0077126394, 0.0255276691, -0.0565021262, -0.0176862571, -0.0053151343, 0.0341286585, -0.0626676455, -0.0145475799, -0.0465840362, -0.0775287747, 0.0358670168, -0.0521620102, -0.0583245382, 0.0236195307, 0.0423626378, -0.0847927555, -0.092342399, -0.0558256842, 0.0551245846, 0.00475145737, 0.0299011748, 0.0220357683, 0.0722943544, -0.00263024168, -0.0146810617, 0.0873225405, -0.0961981937, 0.0243415628, -0.0340497904, -0.011569282, -0.0589175895, 0.108968846, -0.0418338627, 0.00907241366, -0.0651069805, 0.0292125363, 0.0713774859, 0.0248006396, 0.0542561933, -0.125029683, 0.0909524411, 0.0566498153, 0.0443048775, 0.0462020785, 0.0870344042, 0.0116694504, 0.0118674375, 0.0436957926, -0.0671856776, 0.0231215395, 0.00184971921, 0.09540876, -0.0219891835, 0.0399854444, 0.0992067828, 0.096677035, -0.0441051498, 0.0593031831, 0.0425429121, 0.0605829656, -0.0084231384, -0.0644136593, 0.0380044542], [-0.0150994956, -0.0391881689, -0.0546768233, -0.020335421, 0.0269521903, 0.00435732212, -0.0320522338, -0.0735766664, 0.0570882037, -0.049065277, -4.860220e-02, 0.0490751676, 0.00271073217, -0.0592837222, 0.0239415076, -0.0135218604, -0.0491219312, -0.0419549979, 0.0238260701, -0.0509939194, 0.0680716857, 0.0589709617, -0.0470597446, -0.062394645, -0.0337330848, 0.0564597771, 0.0583998486, 0.0287468154, -0.0507889129, 0.0326418765, 0.00799325388, 0.0826101228, -0.0285708029, -0.0290081427, 0.0863802209, 0.00893354323, 0.0681816339, 0.0106631424, 0.095801428, -0.0681791455, -0.0831575691, -0.0407858565, -0.0488748886, -0.0631383732, 0.0378231406, 0.04003454, 0.0794541835, 0.0440818705, 0.0610685609, -0.0302281342, 0.0553052612, -0.0292311609, 0.0689188391, 0.0187533088, 0.0328408666, 0.0104960306, -0.0197646506, -0.0307100452, 0.0214558206, 0.011328483, 0.0329804718, 0.077599749, 0.0092461016, 0.0790428519, 0.114886515, 0.0634706467, 0.0687807053, 0.0722069293, -0.0844108909, 0.101885214, 0.0524861664, -0.00574631151, 0.0120493015, 0.0229833629, -0.0186683889, 0.0868931412, -0.0352549404, -0.0743315667, 0.0883193388, 7.117270e-02, -0.10818287, -0.0472570434, -0.0240009949, 0.00207408564, -0.0605695806, -0.0342038237, 0.0926213786, -0.0584280454, -0.0112840692, -0.0411814302, 0.135154456, -0.0795604661, 0.0126461321, 0.0832977816, -0.0406969674, 0.0187404826, 0.113789931, -0.00481803669, 0.116377905, -0.0565010682, 0.0696261898, 0.0585676357, -0.00887116417, 0.0494897626, -0.0108939465, 0.053772714, 0.085823737, 4.537420e-02, 0.091144219, -0.0106283026, -0.0872006788, 0.0239798035, 0.00820929184, -0.127198651, -0.0276560038, 0.0222949237, -0.0136500243, -0.0333540402, 0.0242962278, 0.00769614754, -0.0413116664, 0.0867015719, 0.101073757, 0.10376592, 0.00651782053, -0.0619726814, 0.00130393379, 0.00354249659], [-0.0256893728, -0.00308283349, 0.0477052629, -0.110434212, 0.0554307848, -0.0221648905, -0.0580011457, 0.0257342774, 0.0942932218, -0.0726434812, -0.0122309318, -0.00646941271, -0.0063439603, 0.0456510074, 0.0280559249, 0.0708334073, -0.0449122041, -0.0437703691, 0.0178624615, 0.0266640205, -0.0356534608, 0.0895558968, 0.0714740306, 0.0441631898, 0.0804654881, 0.0461478084, -0.0431556106, 7.045860e-02, 0.0678213313, 0.0625272319, -0.125157744, -0.0380340256, 0.0410389826, -0.0224632192, 0.0270710103, 0.00706170499, 0.0311168637, -0.034672074, 0.0378498808, -0.16431129, -0.0537167899, -8.68940376E-4, -0.0142488666, 0.0339026079, 0.0253084265, -0.0441780947, 0.0187987722, 0.0814088434, 5.706130e-02, 0.0104044219, -0.00813472457, 0.0288102739, 0.00477513392, 0.0401716977, -0.0479212813, -0.0540105328, 0.053526219, -1.375390e-03, -0.0174954049, 0.0182979275, 0.103400633, 0.0341667458, 0.0559266061, 0.0121177239, -0.0238421652, -0.0788974836, 0.0048594512, 0.00814237818, 0.0103843119, 4.200280e-02, 0.0476433337, -0.0732742324, 0.0268474389, 0.029086927, -0.0455323979, 0.0436059795, 7.55470595E-4, 0.0377704427, 0.0424074493, -0.048968479, 0.0199658815, -0.0405037031, -0.0434559733, 0.044004295, -0.0637560561, -0.0399359874, 0.0903962776, 0.0144039579, -0.0132030286, -0.0212336034, 0.0488792248, -0.0357525237, 0.0024544911, 0.0658895969, -0.00203724764, -0.052031938, -0.0209577605, -0.0638232306, -0.0356905684, 0.0189796481, 0.109700687, -0.0121580577, 0.0572951175, -0.0861433222, -0.129498333, 0.0216428954, 0.018785188, -0.0106190993, -0.0273988806, 0.0837594568, -0.0396785513, -0.0295159183, -0.0288159363, -0.0523989946, -0.0447668917, -0.0396880619, -0.0109834187, 0.0432420485, -0.0123433247, -0.00215174211, 0.0792882367, 0.0647878796, 0.0435477979, 0.0920323655, -0.0376521088, 0.0634533539, -0.0069049974, 0.0332498588], [-0.040748179, 0.0476730093, -0.0255438872, -0.0971742495, 0.0338447057, 0.0268255956, -0.0328970253, 5.878990e-02, 0.104868144, -0.0390890427, -0.0363942832, -0.0197984539, 0.065544948, -0.107670374, 0.0100605097, 0.0259296484, 0.0439300537, -6.481520e-02, 0.02046418, -0.0351729318, 0.0437397845, -0.0699840188, 0.0431972183, 0.054933738, -0.0287350118, 0.0850601941, -0.00479237828, 0.00336483051, 0.0505566634, 0.0556882285, -0.0468843244, 0.04723287, -0.0120525472, 0.0659463778, -0.034099102, 6.611750e-02, 0.0389677137, -0.0318489075, 0.0432178266, -0.0943870693, 0.0317353979, -0.0536696389, 0.00347001455, -0.0258013159, 0.0307805371, -0.00772077264, -0.0794675797, -0.0968998372, -0.0525775217, 0.0850027353, -0.0210539307, -0.0672455356, -0.0800890177, -0.00242096744, -0.0435957909, -0.0273102485, -7.366770e-03, -0.0153865675, 0.0329976454, 0.0587642901, 0.0665605888, 0.0495398603, -0.0574643202, 0.0258838609, 0.00680031395, -0.0616449974, -0.0141853085, 0.0591895916, -0.0284281112, 0.0439038686, 0.0287200976, -0.101379633, 0.0236769281, -0.0167014301, -0.0683313087, -0.0634674951, 0.0266483631, 0.0339512676, -0.0381956138, -0.0747482926, 0.103336081, 0.020078579, -0.0526926853, -0.0108969687, -0.0172499325, 4.81007446E-4, 0.0876851529, -0.0288801901, 0.0446749032, 0.0424688645, 0.0155787924, -0.0165331252, 0.0377330221, 0.0532678589, -0.10787186, -0.0556465425, 0.0767198876, 0.0974186658, 0.038314946, 0.0107007781, 3.486100e-02, 0.0370136462, 0.0702616796, 0.0203270093, 0.0173001494, -0.0151743237, 0.0846305489, -0.0653174296, 0.00760478387, 0.0428148955, -0.0381116271, 0.0490941666, -0.0358376615, -0.0299622957, 0.0411031544, 0.00913725141, 0.0816522688, -0.0874648094, -0.0781153291, -0.0114179691, 0.115677364, 0.0479254648, 0.00204589684, 0.139742419, -0.0317825712, -0.0236657169, -0.0515127629, -0.022177089], [0.0342355408, 0.0346649587, 0.0202940796, -0.00964008457, -0.0129295839, 0.0494811125, 0.0413610339, -0.0394225754, 0.0977231487, -0.0127701666, 0.0084493747, -0.0533498563, 0.0151016451, -0.0922450199, 0.0381418355, -0.0239992086, -0.00378040434, -0.0628699437, -0.0353177525, -0.111831367, -0.0686458945, -3.25129513E-4, 0.0515072271, -0.049978178, -0.0921682119, 0.0804208144, 6.950550e-02, 0.0416546538, 0.0200361032, 6.861210e-02, -0.0497811586, -0.0220366549, 0.063903667, 0.0222236328, -0.0131270615, -0.0467622615, -0.0196419917, 0.0187358297, -0.0167215858, -0.00533450488, 0.00623153104, -0.0103523433, 0.046694912, -0.11863973, 0.0145076625, -0.00260024262, -0.094018124, -0.0042598974, 0.0643030182, -0.0188909136, 0.0620730258, -0.0238524247, -0.11422985, 0.0720329285, -0.0750774294, 0.0893459469, 0.00295673427, 0.0721264779, 0.011063098, 0.147553459, 0.0270578414, 0.030546261, -0.0411502384, -0.0359897912, -0.0299258493, 0.128018349, 0.0581273101, 0.0507637113, -0.0607381314, -0.0103655746, -0.050787434, -0.0599651299, 0.106255271, 0.0531272255, -0.0570749156, 0.00440317718, 0.00762547832, -0.034541633, -0.0256780609, -0.0564758927, -0.0216057207, 0.0611540489, -0.018660007, 0.022201933, -0.0776501595, -0.0460145213, 0.0826363191, -0.0343244411, 0.0864039212, -0.0274534989, -0.0310246907, -0.0373500809, -0.0795127525, 0.105249636, -0.0904067606, 0.0820084586, 0.0387968123, 1.032410e-01, 0.0259999167, -0.0130848214, 0.044885058, 0.111909129, -0.0386959352, -0.115472689, -0.00819291733, -0.0753167197, -0.0718374625, -0.029772209, -0.0127637442, -0.022885764, -0.064802058, 0.0133401444, 0.0137074767, 0.0230376199, -0.0131289382, 0.00585855311, 0.116669141, -0.0438076928, -0.0712369233, 0.120293468, 0.0673363656, -0.0717298612, 0.00960898306, 0.0455866158, -0.0170183796, 0.0490470491, -0.0705340952, 0.0147899576], [-0.022824835, -0.0104341814, -0.0275699664, 0.130667657, 0.0514797568, -0.0478034057, 0.0478927791, 0.0428847559, -0.0317761563, -0.00969784706, 0.00789714232, 0.0192628838, -0.0835212767, 0.0159159917, 0.00603716634, -0.0150243239, 0.060901124, 0.0175603088, -0.00528023858, 0.00894024316, -0.051001329, -0.0167514309, -0.0899925231, -0.0473687537, 0.0274737887, 0.0517167114, -0.0230475124, 0.0157920253, -0.0278952606, -0.0183532462, 0.0341271088, 6.911990e-02, 0.0308215544, -0.0576661229, -0.114889666, 0.0149334986, 0.0138502317, 0.134915292, -0.0994886606, 0.0426098481, -0.0751347393, 4.788020e-03, -0.0171190668, -0.101987265, 0.001591963, 0.0483108386, -0.0981337875, -0.108983912, -0.0231219959, -0.0433043651, -0.023790732, 0.00237530028, -0.0264605787, 0.0751053392, -0.0344070345, -0.0354498774, -0.0223642234, -0.00172553945, -0.0420951732, 0.119104259, 0.0439578518, -0.0236611683, 0.0129657844, -0.0851218327, 0.0607455969, 0.132681698, -0.0657696053, 0.00156983454, 0.0951834246, -0.0115059167, -0.0366108343, 7.570920e-03, 0.0784462317, 7.660230e-02, 7.298100e-02, -0.0221737437, -0.0221230295, -0.0562552549, 0.0397977903, 0.0429623052, 0.0816993787, 0.0485870242, 0.123332568, 0.1024516, -0.120971918, 0.0893593878, 0.0251760148, -0.0593532771, 0.0821130275, -0.0487294272, -0.104824886, -0.00427056896, -0.0102519877, 0.0206506159, -0.0440196618, 0.0101218969, -0.00869500171, -0.0336850546, 5.239270e-03, 0.0436422117, -0.0512535684, 0.052663587, 0.0262368098, -6.938290e-02, 0.0582810603, 0.0417179838, -0.0100034522, 0.00250360556, 0.0536909588, -0.00669875648, -0.0423679687, -0.039543286, 0.0633362829, 0.0139204413, 0.0272988193, 0.0522346236, 0.0963722914, 3.309510e-02, 0.0170305036, 4.200890e-02, 0.128758952, -0.0144211864, -0.0266146529, 0.0313935205, -0.0150876641, 0.0508979857, -0.0428412482, 0.0708966181], [-0.0237996969, 0.102056354, 0.0307436548, -0.0228102226, 0.0100346813, 0.0958922132, -0.0170397237, 0.0765848085, -0.0507515408, 0.0230194163, -0.0326584317, -0.0570256151, -0.0831777751, 4.315020e-02, 0.00426799385, 0.0408081226, -0.0230982453, -0.0807834715, 0.022068901, -0.0192832705, 0.0634365529, -6.643780e-02, 0.0150808897, 0.0278840736, -0.00248905458, -0.0363601632, -0.0641264692, 0.0560120866, 0.0373941511, 0.0506661125, 0.106548429, -0.0657613054, -0.0998979061, 0.0582995564, 0.0208016019, 0.0136659965, 0.0689048693, 0.0489082821, -0.103709564, -0.061533045, -0.0316122025, -0.0336796865, -0.02221651, 0.0881000459, 0.0635074526, -0.0286548547, -0.0689155608, -0.109972022, 0.0143889654, 0.0139670325, -0.0764019787, 0.0753536597, -0.0191263873, 0.0553196706, 0.0394608676, -0.0584782921, -0.026925521, 0.0360863507, 0.0251557026, 0.0629712492, -0.0427076146, 0.00481439196, 7.707630e-02, 0.00390179036, 0.0466281883, 0.109151654, -0.0516239032, 0.0381482095, 0.021085104, 0.0389033407, 0.00109249237, -0.0886924639, -0.0282138363, -4.803160e-02, -0.0361970961, -0.0401675589, -0.0543202646, -0.0449820906, 0.0356891118, -0.0455268398, -0.0197940208, -9.841570e-03, 0.0156239569, 3.501960e-02, -0.0612362958, 0.0676720813, -0.009508471, 0.0200200658, 0.0121042859, 0.0470689833, -0.117875539, 0.0312141459, -0.00826114882, 0.0276982766, -0.0479182936, -0.0346228071, 0.05651398, 0.0528535508, 0.0386810936, -0.0410310142, -0.0515981652, 4.656030e-02, 0.0365877226, -0.0797360464, 0.111298494, 0.046910055, -0.0665487275, -0.0103293611, 0.0177069139, 0.012732286, 0.0435119942, -0.0207313523, -0.0589227937, 0.0745032653, 0.0120832091, -0.0423013158, -0.0591962188, -0.0728586316, -0.0286707319, 0.0251488145, 0.133569285, -0.0310588088, -0.0702834651, -0.0269160792, 0.0969259291, -0.0270332228, 0.0272596031, -0.0120738065], [-0.00674855197, -0.0297034495, 0.0146278543, 0.0663642734, 0.0716240779, 0.0109609179, 0.0279380288, 6.473140e-02, -0.111972392, 0.0121977683, 0.082224138, -0.0537608303, 0.0652897954, -0.0351800732, 0.0694421604, 0.0200156756, 0.0393138677, -0.0110681755, 0.0166413151, -0.0482041575, -0.0553198047, -0.0345054083, 0.0588831864, -0.0460438691, 0.0881081521, 0.0732385442, -0.0677668378, 0.0778395087, -0.0760738775, -0.0105520124, -0.00679724849, -0.0120934378, -0.0893878936, 0.104089789, 0.0129262703, -0.0408897288, 0.0160909481, -0.0208528023, -0.0028787544, 0.0959219112, -0.0527825058, -0.07492318, 0.0461131297, 0.0196309406, 0.071739316, 0.0220024455, -0.112894177, 0.0291590523, -0.0541105941, 0.03081882, 0.0184767898, 0.0212569349, -0.0462691262, 0.0555001274, -0.0391433127, 0.031545423, -2.361530e-03, -0.0323515423, -0.0173508283, 0.0161931068, 0.0695590078, -0.0297107641, -0.0134882685, 0.0125417551, 0.017833909, 0.00584326731, -0.0320854262, 0.0384543315, 0.0236672889, 0.0314103253, -0.0017340366, 0.0173142143, -1.884600e-03, 0.059413895, -0.0085407747, 0.0411384217, -0.0260642245, 0.0619335807, 0.0656203106, 0.0509207435, 0.0417075753, -0.0530823395, 0.0779652297, 0.075081557, -0.0467411131, -0.0435640663, -0.0686083883, 0.0887173488, 0.00263706292, -0.0441303849, -0.0541758426, -0.0732459277, 0.0531805083, -0.0257506706, 0.0934732854, -0.0309732258, -0.0199730266, 0.0680666193, -0.0159710739, -0.0667445511, -0.0633564591, 0.124729685, -0.0484248847, 0.0184484925, 0.123834714, 0.00312727084, -0.042593535, 0.0487807579, 0.00344472192, -0.0271828249, 0.0245825276, 0.0550826527, 0.00559169659, 0.0888409466, 0.116521306, 0.0446689911, 0.0275798086, -0.0105334567, 0.0150496345, 0.0116991503, 0.02026725, 0.0248217154, -0.0145830885, 0.0345042199, 0.0515328273, -0.0125607327, 0.0994057804, 0.0612028316], [-0.0502692834, -0.0125370864, -0.0480285399, 0.119867742, 0.0689396337, -0.0400530174, -5.834970e-02, -3.809120e-02, 0.0375439115, -0.105024062, 0.0307343155, -0.0687009618, 8.328220e-02, -0.0250111036, -0.016959168, 0.0572583862, -0.00753903436, 0.0186745524, 0.0311347507, -0.00464858254, 0.0722076818, -0.0890962481, -0.0545188859, 0.0611524283, -0.0303061213, 0.155270308, 0.010359413, -0.0230134223, -8.397080e-03, 0.0383927748, 0.058329232, -0.0610125661, -0.0145754479, 0.0370529667, 0.0583085492, -8.985110e-03, 0.0979762077, 0.0759111196, -0.0258797295, 0.0104182363, 0.0372808725, -0.00393444486, 0.0614333637, 0.0809030086, -0.0344669037, -0.067842409, 0.00647085113, 0.0717153549, 0.0682072788, 0.00962818507, -0.0469799824, 0.101360522, 1.890030e-02, -0.0141694229, 0.0674379244, -0.0200897064, -0.07093212, -0.0388732851, 0.0580203533, 0.00797105674, 0.0442807972, -0.0069010919, -0.0409563705, 0.0203269813, -0.0711805522, 0.0559990145, -0.0781372413, -0.00441769138, 0.0613118149, 0.0379739925, 0.0166658871, 0.0496624596, -0.073338829, 0.0752552152, -0.0254319962, 0.0524954684, -0.0302998163, 0.0670412257, 0.0623300783, 0.0240945648, 0.0957393124, 0.0386410728, 0.0367013775, -0.0406156704, -0.0545337833, 0.025472939, 0.069883652, 0.0579617657, -0.00897307135, 0.020571338, 4.92786116E-4, -0.028553715, -0.0576577857, 0.0588823147, -0.0293172318, 0.0166380554, -0.0192858558, -0.051085189, 0.0876148566, 0.0786048174, 0.0400414281, 0.111919962, 0.0756540373, 0.0144412061, 0.0410669968, -0.0141573194, -0.113964692, -0.0149878766, -0.0555230565, 0.0595641248, -0.0538962781, -0.0358581841, -0.0712621063, 0.0452574193, 0.127416566, 3.62405321E-4, 0.00886271521, 0.086314559, 0.00901225861, 0.0587286949, 0.0819922313, 0.016897168, -0.00518384716, -0.0516622253, 0.0365025885, -0.0489513353, 0.118638143, 0.0820190608], [-0.00453449925, 0.0632799565, -0.0323420949, 0.0462078527, -0.0273033921, -0.0206017029, 0.0613695905, 0.0160182398, 0.0387471244, -0.094703786, -0.0535488054, -0.0654557347, -0.0664334372, -0.0595014803, 0.0126949754, -0.0445501283, 0.0264448393, -0.0180439856, 0.0648503676, 0.00521840667, -0.0532210581, -0.0278011803, 0.0866887941, 0.0179206226, -0.0160453394, 0.0471106954, 0.0413353331, -0.0543472245, 0.0627676249, 0.0680397078, 0.046759475, 0.0361053757, 0.0430260375, 0.0896853134, 0.0410011858, 0.0231476761, 0.0664017424, -0.00113963254, -0.0839860141, 0.0663958713, -0.0518627092, -0.0533238351, 0.0843623653, 0.0634466484, -0.0838790163, 0.0024534252, -0.0431403182, 0.010318527, 0.0548992902, 0.058072038, 0.0179449879, 0.0967083573, -0.0619783289, -0.0859654173, -6.943050e-02, 0.0956164523, -0.0640416294, -0.00886851456, 0.0712192655, 0.0552902594, 0.0110555701, 0.0885810181, 8.25469498E-4, 0.0734780207, 0.0319825597, 0.0354535654, 0.0703031718, 0.0763548836, -0.0606202036, -0.0198475402, -0.00314773479, -0.0717240273, -0.0238031745, 0.0686260462, -0.0402300917, -0.013956598, -0.0594325624, -0.0904438197, 0.0668519363, -0.0425454043, 0.0742967203, 0.0301570743, -0.0474913679, 0.0428437889, -0.0336354971, 0.0780129432, -0.0682355613, 0.0824212804, -0.100526474, -0.037417803, -0.0802183672, -0.0659470484, 0.0506493635, -0.00490014534, -0.0161386393, 0.0642819926, -0.0570627563, 0.0714457929, -0.0157321524, 0.0334355794, 0.0366410948, -0.0163107738, 0.0394554846, 0.00466926955, -0.0540502928, -0.0407848507, -0.0759191215, 0.0181033462, 0.0120324157, 0.0929741114, 0.0446806923, -0.0270159449, -0.00613715174, -0.0429497883, -0.0224532094, -8.011520e-02, -0.0857398509, 0.123667784, -0.0766596198, 0.0476852469, 0.104071647, -0.0443708487, -0.0968720242, -0.0445207916, 0.106674239, 0.0434085764, 0.138981476, 0.0510966927], [-0.0791158154, -0.0288015082, 0.0113697005, 0.138249502, -0.0284703933, -0.0194288548, 0.064089626, -0.0726606473, -0.0374587886, -0.110456236, -0.0125706596, -0.0404237658, 0.0724056959, 0.00611496205, 0.0254610479, -0.0353940576, -0.0725167319, 0.056321267, 0.0885218382, 0.0615858771, -0.026720129, 0.0319205783, -0.00478702085, -0.0376150906, -0.0410151258, 0.0795297176, 0.0557563417, 0.0606603138, -0.052097477, 0.0808166787, -0.0148558011, -0.0763957053, -0.0618092865, 0.0508373603, -0.0741025433, 0.0131271193, 0.00497802347, 0.0619053281, 0.0503918529, 0.0247933064, -0.081849873, 0.0128973955, 0.0770830959, 0.0483301654, -0.0238250848, -5.869250e-03, -0.0528840385, 0.0835417434, 1.88563878E-4, 0.0678286552, 0.00576578546, 0.0408702828, -0.0589364544, 0.00242969766, 0.0721653551, 0.0604907833, -0.0821733102, 0.048716791, 0.0389259607, -0.0327424258, -0.0260764565, 6.57679193E-5, -0.0900696739, -0.0666796416, 0.0579170883, -0.0333807915, 0.0125191985, -0.0332487561, -0.00169111812, -0.00503611285, -0.0617933348, -0.0381689109, 0.0574433021, -0.0716301426, -0.0456162542, 0.0160526987, 0.0435575023, -0.0435448699, 0.011800129, 0.0298896432, 7.535350e-02, 0.0270269495, 5.248500e-02, -0.0444730483, -0.0859111696, 0.0652290955, -0.0123481043, -0.0270951688, -0.00541865267, -0.055642847, -0.0528968275, 0.00557017419, 0.0335938148, 0.00511995843, -0.023447575, 0.0505416282, 0.0369095244, -0.0645128414, 0.0426158309, -0.0547332764, 6.882060e-02, -0.00524309371, -0.0350668095, -0.0765280425, -7.734740e-02, -0.0188646745, -0.0901027992, 0.00612109434, 0.0288644768, 0.0859956294, 0.0496676825, 0.0652011558, -7.008790e-02, -0.0801076367, -0.0360854864, 0.0270109549, -0.0102926344, 0.102310069, -0.0819077491, 0.0604095161, 0.0777740926, -0.0631541461, -0.0393588357, 0.0609655231, -0.0474564433, -0.0411788374, 7.808840e-02, 0.025056785], [-0.0448518619, 0.032568533, 6.482770e-02, 0.0241107438, 0.0472593345, -0.0804564208, 0.0486475863, -0.0229898933, -0.0589610822, -0.0812216326, -0.0408169813, 0.0595820956, 0.0430920608, 0.048314672, -0.0304285493, 0.089077562, 0.0442570895, -0.012726224, -0.0517978035, -0.0308396015, 4.290480e-02, -0.0418510064, -0.0377726331, -7.752550e-03, -0.0225768499, -0.00827031117, -0.0622660405, -0.0190071445, -0.0656559467, -0.0239592642, 0.0704948679, 0.0129833696, -0.102152266, 0.00471054902, -0.0215650201, 0.0114403116, 0.0402926803, -0.0474752299, 0.0146798873, 0.0276029296, 0.0143251298, 4.99871152E-4, 0.00327312574, 7.34900706E-4, -0.0718376786, -0.0648407787, -6.647270e-02, 0.0417698063, 0.0723655223, 0.0935541838, 0.0294494107, 0.105780154, 0.00938939583, -0.0818735286, -0.0801251903, 0.0563856214, -0.015782062, 0.00824470445, 4.400780e-02, 0.071568653, 0.0325990506, 0.0976013317, -0.0474458486, -0.062143214, -0.0468132645, 0.0448560081, -0.0022021248, -4.703820e-02, 0.0781747773, 0.0629968643, 6.165590e-02, -5.867090e-02, -0.00812932848, 0.00470361533, 0.0360800624, -0.025446346, -0.00953549799, -0.0310873482, -0.0255924072, -0.0492524579, -0.0234477073, 0.0336385742, -0.0285352096, -0.0961401537, -0.113442615, 0.0298194364, 0.0206837207, 0.00549505139, -0.0322149657, -0.0135392733, 0.0339579135, 0.013765391, -0.0613488741, 0.089897871, -0.0458032861, 0.0645318478, 0.0224320181, -0.0316840038, -0.0176352169, -0.00562969502, -0.0601762049, -0.0220691171, 0.0242558233, -0.0993943363, 0.0179785546, -0.0330168828, -0.0242999792, -0.0202762857, -0.0533674136, 0.0695535242, 0.0312839337, -0.0811922848, -0.0474304669, -0.0943522602, 0.0076058046, -0.0120924879, 0.00796214771, 0.086353667, -0.0136139365, -0.0440522768, 0.0608426332, 0.00982129946, -0.0162162501, 0.0479299314, 0.0930325388, -0.0632442757, 0.0157917421, 0.0364084952], [-0.0920535177, 0.0352191813, 0.0359510407, 0.0593129881, 0.011435193, 0.0423591286, -0.0157461315, 0.0290119667, 0.0938166677, -0.094588153, 0.0554444678, 0.0790996178, -0.0688902363, 0.0300141908, 0.0839121341, 0.0787537247, 0.0620133168, -0.0262831394, 0.0667541921, 0.0932163745, 0.0780987069, -0.0250885244, -0.0415706336, -0.066447407, -0.00607034424, 0.0277135093, -0.0124334591, -0.0703550056, -0.00767631968, -0.0126423528, 0.0254809577, -0.0458574146, 0.0336500481, -0.0304274503, -0.0859913229, -0.00316183944, -0.0509385169, 0.0339259766, -0.0400595069, -0.0052691265, -0.0774873718, -0.0423119478, 0.0157089345, 0.017237667, -3.529790e-02, -0.0388797633, -0.0242048558, 0.0707651749, -0.00284203095, -0.0449205339, 0.0367557555, 0.100325443, 0.0108114313, -0.0723841339, -0.0203508083, 0.0392501056, -0.0673213229, -0.0597299077, -0.0363508835, -0.0415990688, 0.0490246154, -0.0407715701, -0.0473404676, -0.0201804824, -0.0497093163, -0.0559152551, 0.0186574087, 0.00443003187, 0.00531111378, 0.080989927, -0.0289846286, 0.0623173229, 0.0305847507, -0.0694005936, -0.0117404498, 0.0474558659, -0.0383715704, 0.0192602649, -0.00668577245, 0.0354206488, 0.0774091258, -0.00810345728, -8.315480e-02, -0.0198054146, -0.00459134905, 0.0561829358, 0.0662703961, 0.0498554036, -0.0623429119, -0.0821425244, 0.0130208042, 0.0154090691, -0.023707822, -0.0614081658, -0.0802039802, -0.063365221, -0.0155779058, 0.0173380468, 0.0621553957, -0.0513440371, 0.0440415703, 0.0523112938, -0.0738017857, -0.085031569, -0.0831632167, 0.0135580786, -0.0390207432, -0.079575099, -0.00334517239, -0.00294366688, -0.0589841269, -0.0453923941, 0.0268053804, -0.102047212, 0.0630851164, -0.0268067531, 0.00258485693, 0.062131796, 0.00365594355, 0.0332696885, 0.0264482684, 0.086798869, -0.0206330642, -0.03034449, -0.0135144889, 0.0446901619, 0.0600539409, 0.0603746288], [-0.0901754051, 0.019281948, 0.00794409215, -0.0632079691, 0.0830404981, -0.011341135, -4.518480e-02, 0.0661602616, 9.785020e-04, -0.0635003224, 0.0418557525, -0.0100063449, -0.0568655394, -0.00374340545, -0.0372650661, 0.0561569482, 0.0124456044, 0.0860614478, -0.00699221483, 0.0015412448, -0.00319193443, 0.0361219496, -0.0528039522, 0.0344254263, 0.0117756361, 0.0499221049, 0.0279786307, 0.0531094931, -0.0177904274, -0.0240269694, 0.0590088852, 0.0490478128, -0.0477807485, -0.0671899915, -0.0460470282, 0.079834856, -0.0697148219, -0.029896453, 0.104622245, -3.315100e-02, -0.00951414835, -0.0593900792, -0.0117136613, -0.0718256235, -0.0480286479, -0.0476987846, -0.0537425429, 0.0778240263, -0.00191642717, -0.00919136312, 0.0466646366, 0.0418149158, -0.0533600971, 0.0203651804, 0.0662299171, 0.0065372847, -0.0199347809, -0.0245848149, -0.00937987491, 0.0571172312, 0.0905755758, 0.0670225769, -0.0923588052, -0.0394616537, 0.0298318956, 0.066347912, 0.0311162807, 0.0740823522, 0.0654724911, -0.0663453862, -0.0810419246, -0.0330503359, 0.0603013262, -0.0280262977, 0.0139890201, 0.0412092954, -0.0931124091, 0.0697115361, 0.0431440957, 0.0865418314, -4.651590e-02, 0.0396728963, -0.0621878467, -0.0426677801, -0.0140776634, -0.0274135601, 0.0926314219, -0.062896207, -0.0120843975, 0.0266744085, -0.0278865676, -0.0506323539, 0.00695892144, 0.0115305306, 0.0175666884, -0.00926870201, -0.00226708804, -0.0135218529, -0.0830522999, -0.0105145117, -0.0724612847, 0.0326424651, -0.00589674944, -0.0332234353, 0.0260845181, 0.0297840517, 7.143210e-03, -0.0297702979, -0.0705559179, 0.00134310196, 0.0433207639, -0.048138503, -0.0566268042, 0.0329032317, -7.058380e-02, 3.153470e-02, 0.0198839325, -0.0253219772, 0.018506119, -0.0359583832, -0.0293082446, -0.0344933271, -0.0775784626, 0.0251953248, -0.011570151, 0.0375627503, -0.048201479, -0.065547809], [0.0174073931, 0.00132912886, -0.0297835283, 0.00870805141, -0.0604071766, -0.0195046663, 0.0415156819, -0.0458557382, -0.0117276879, -0.0387645699, 0.0804314166, -0.0335533023, -7.957300e-03, 0.0265306421, -0.0634811297, -0.053741578, 0.0449524336, 0.0170300156, -0.0569156744, -0.080253467, 0.073753491, -0.0428836308, -0.006547397, -0.0697682425, -0.0288072824, -0.0350790098, -0.0466030575, -0.0918527916, 0.022887012, 0.085022211, 0.0490933582, 0.0444100238, 0.0308938939, -0.0521364734, 0.0381503962, -0.017162852, -0.081590861, -0.0710950717, 0.0939836427, -0.0829747989, 0.0493524447, -0.0712023824, 0.104143538, -0.0998369082, -0.061193727, 0.0209635422, -0.0671657696, 0.019505037, -0.0777531042, 0.0495243296, -0.0276882444, 6.861220e-02, -0.0900430381, 0.0136669753, -0.0736686587, 0.0634247288, 0.0697291642, 0.0412203707, 0.071618624, 7.036210e-02, 0.0330961049, -0.0250786524, -0.0754451528, 0.0434781685, -0.0522769727, -0.0641968623, -0.0449524298, -0.059721671, 0.0115017025, -8.358170e-02, 0.0350975171, 0.0133874444, 0.00201404025, 0.0244549382, -0.0569445267, -0.0392035581, 0.011218789, 0.0584500134, -3.722810e-02, 0.056074474, -9.94521542E-4, 0.00194445485, -8.259890e-03, -0.0589903444, -0.0219344161, -0.00579969492, 0.120607398, -0.0829911157, -0.0512267798, 0.0667674839, 0.124590583, -0.0450638942, 0.0365304425, -0.0222786386, 0.0361436978, -0.0511915907, 0.0299497303, 0.0684689358, 0.0259719025, 0.0452288538, 0.0327211209, -0.0754549876, -0.0186227597, -0.0691094846, 0.0545381904, -0.022943899, 0.0934034511, 0.0088286139, -0.0705415681, 0.0125768967, -0.0276180655, -0.0305697657, -0.0142526254, 0.00349112577, -0.0263495035, -0.0777532905, 0.0392736606, -0.0256309491, -0.0658710896, 0.00959689915, 0.10329397, -0.0535769612, 0.0422880612, -0.0410490222, 0.0714523718, -0.057791993, -0.0393879637, -0.0761330649], [-0.0428297371, -0.0450599268, -0.0574521646, 0.00956415385, -0.0507732108, -0.0244940948, -0.0604269691, 0.0539514832, -0.0162089262, 0.0139467735, -0.0187326856, -0.0104901893, 0.0575722232, 6.81307632E-4, -0.0140101872, 0.0498639308, -0.0745207667, -0.0412143841, -0.0669346228, 0.0454937816, 0.0346468799, 0.00619656174, -0.0573102348, 0.0840349122, -8.508540e-03, 0.0639074146, -0.0448689163, 0.0612729489, -0.0105000501, -0.0388002023, 0.0706081539, 0.0116890352, 0.00395205524, 0.036636278, -0.0515099727, -0.0617424399, -0.00821535289, 0.0770385638, 0.0319650099, -0.0610376224, 0.0552782081, -0.0358632319, -0.0455086343, -0.0460864864, 0.0470317714, 0.0814857632, 0.00117022113, 0.0226408038, 0.0114563741, -0.0350870229, 0.0127306376, 0.0753737241, 0.0597065277, 0.0229036603, -0.0192016978, 0.044589825, -0.0683120042, 0.0647021234, 0.0519732535, 3.38307873E-4, 0.0492222607, -0.00159811205, -0.0209245551, 0.0471019782, -0.0223038085, -0.0910527706, -0.0696496442, -0.0337565653, 0.0647556409, -0.0010850504, 0.0033485936, -0.0284554418, -0.0269899145, -0.025751764, 0.0565445423, -0.0323453322, 0.00224287761, -0.0801775529, -0.0618132874, 0.00897238217, 0.00258147018, -0.00151845778, 0.0283532795, 0.0259648617, -5.271930e-02, -0.00693111448, 0.0337563306, -0.0039417427, -0.0888892188, -0.0638114959, -0.0104310559, 0.0268505197, -0.0931330174, -0.0572978929, 0.0140786702, 0.0482111461, 0.0868146196, 0.0690360293, -0.0619924329, -0.0122803617, 0.0442486145, 0.0656662956, 0.029374918, 0.0276372172, -0.0187095888, 0.00841627549, 0.0861273631, 0.0710897669, 0.0577215292, -0.0084075043, 0.0411982797, -0.0594931096, -0.0119692655, 0.0513731465, 0.0556960776, 0.036056634, 0.066304773, 7.61292409E-4, 0.00703498255, 0.0104027437, 0.0438220948, -0.0444858186, 0.0553518198, 0.0399881676, -0.0325281434, 0.0225410983, 0.0440557748, -0.0421594866], [0.044791352, -0.0256682876, -0.0212501399, 0.00125354086, -0.0755258799, -0.0708984584, -0.0361134447, -0.0728947595, 0.0268163942, 0.0754630938, -0.0322763324, -0.00152109936, -0.0500287972, 0.0680886656, -0.0586442426, -0.0568504445, -0.0348383188, -0.0167851485, 0.0658680126, 0.0118456399, -0.0671996251, 4.006940e-03, -5.54402941E-4, 0.036278218, 0.0151470695, 0.0150731206, 0.0536130369, -0.0636395514, 0.0233534612, 0.00874992646, -0.0314784981, -0.0763323456, 6.014860e-02, 0.0507343672, -0.00815170817, -0.0731669739, -0.0710156932, 3.333570e-02, 0.0358592048, -3.913920e-02, 0.0750827491, 0.0346040614, 0.010982696, -0.00273887441, 0.0535728894, -0.00753684482, 0.0721052215, 0.0145178745, 0.0106956838, -0.0542439744, -0.00951999519, 0.0305379052, -0.0492198467, 0.0708910152, 0.0347858891, 0.00679402612, 0.0632134154, -0.0119423764, -0.0134297218, -0.0142706139, 0.0855953246, -0.0260165948, 0.0399962775, -0.0490835235, 0.0423951708, 0.0070523615, -0.0522738472, 0.0238145348, 0.00808278378, -0.0201371182, 0.0359500349, -0.0609516352, 0.0430448279, -0.0288860369, 0.0701646283, -0.042011749, 0.0715518519, 0.0383397825, 0.0623655468, -7.483790e-02, -0.0205364618, 0.0488199107, -0.0536362976, -0.0665688962, -0.0390214175, -0.00805200171, -0.0239491034, 0.0287090577, -0.0395125858, -0.00425623218, -3.892450e-02, -0.0754849836, 0.0688550919, 0.0259029139, 0.071881853, 0.0395774879, 0.0758969113, 0.0104431156, -0.0408583768, -0.0338626765, -0.076925464, -0.0582508557, 0.0454388708, 0.049518168, -5.016770e-02, 0.0232916642, 0.0124522652, 0.0108418353, 0.0634499118, -0.0457203723, 0.00802121311, 0.0135104731, -0.0393924788, 0.0263544079, 0.0101250056, 0.0718222708, 0.0563490614, 4.51462867E-4, 0.0540644489, -0.0714737251, -0.028402064, -0.0700729936, 0.0214923471, 0.0618723147, 0.0479757823, 0.0389968343, -0.0427983329, 0.0261795502], [0.0756765753, 0.0196423139, 0.0792708546, -0.0134125669, -0.0542045087, -0.064567782, -0.0187043846, -0.0317113027, -0.024638297, -0.0680709556, 0.0443437584, -0.00627975632, -0.0226130597, -0.0583385862, 0.0665931553, 0.0806505233, 0.0252143666, -0.068486914, -0.0539246313, -0.0374766402, -0.0685634166, -0.0771510601, -0.00957720912, -0.0690207854, 0.0643374249, 0.0551011898, 0.0589585267, 0.0523975454, 0.0621124207, -0.0212350767, 0.0732777864, 0.0145024145, 0.0435191467, -0.0582905188, 0.0202358272, 0.0657340214, 0.0120490408, -0.00568648102, 0.0760297552, -1.17527932E-4, -0.0702327639, -0.039631512, -0.0749440715, -0.0452709757, 0.0772906393, -0.0357176475, -0.0646641254, 0.0749961734, 0.00917471293, 0.0156409089, 0.00678234035, -0.0173053034, -0.0102476589, -0.0562026501, -0.0622719042, -0.0371742323, -0.0564799607, 0.00598864583, 0.0412224159, -0.0804922655, -0.0327338316, 0.0358865224, 0.0758932605, 0.0771121159, 0.0325967297, -0.0583492629, -0.0230848361, -0.0737631842, 0.0552033558, 0.00191747118, -0.0180828255, 0.0223963112, -0.0697396323, 0.0467330217, 0.0735167488, 0.01179743, -0.0137626296, -0.0540812574, 0.0547025576, 0.0564216971, 0.0449179523, 0.0358359404, -0.0111746313, -0.0774605348, -0.0328634456, 0.0663019046, -0.017851647, 0.0450400561, -0.0466619506, 0.0353802294, 0.0638063326, 0.0537701249, 0.0451083668, 0.0628545135, 0.058807075, 0.0271652117, -0.0753108189, 0.0754358694, -0.0235640444, -0.0639257953, -0.0277795978, 0.00218705484, -0.0371434838, -0.0109900096, 0.0633618906, 0.0508348607, 0.0438861512, 0.0285389293, 0.0703542084, -0.00574639114, 0.0533167794, 0.0545270219, 0.0792749077, 0.0807632431, -0.0687823743, 0.00326295383, 0.053620711, 0.0374572389, -0.059231285, 0.00576528581, 0.0661535189, -0.0190890748, -0.0731913671, 0.058486186, -0.0284830257, -6.042850e-02, 0.00553710572, -0.027088983], [0.00290451944, 0.0211971551, -0.0616632737, 3.121140e-02, -0.00535284076, 0.030525513, 0.0113777043, -0.013837738, -0.0406400412, 0.024716299, -0.0736440495, 0.00559986057, -0.0327190608, 0.0390105806, -0.0224141013, -0.0235695615, -0.0305731557, -0.0733539909, 0.0535582453, 0.00286994129, 0.0771346614, 3.076410e-02, -0.0332528502, 0.0683723912, 0.0507090352, -1.79888855E-4, -0.0389565974, -0.0290215891, -0.0475560725, 0.0711144656, 0.0638692603, 0.0140038915, -0.00507661281, -0.0387263373, -0.0129677495, 0.0710306242, -0.0584906936, -0.0661780834, -0.0403672978, 0.0561059453, -0.0431433544, 7.388840e-02, -0.00741679734, 0.0798700824, -0.00936522893, -0.00409084512, 0.0625985265, 0.0480920747, -0.0501362905, 0.0753568336, 0.053643316, -0.0683903322, -0.0364949293, -0.0119774854, -0.0215731561, 0.0584578291, -0.0472435392, 0.0744301378, -0.052293133, 0.0372857116, -0.0635576695, -0.0150609398, 0.00266566267, 0.0238761567, -0.0673985407, 0.0744172632, -0.0683502256, 0.0511657596, 0.0631572083, -0.0195332505, -0.029718535, 0.00666108122, 0.0605095662, 0.0748040602, -0.0413339734, 0.0447862372, 0.0454311706, -0.00826230645, 0.0677390471, 0.0251729488, -0.0197409466, 7.17865245E-4, -0.0692206547, -0.0195252262, 0.0768098161, -0.054575175, -0.0297718011, -0.080150105, -0.0650054365, 0.0283693708, -0.0481467322, -0.0802174658, 0.0370798446, 6.089340e-02, 0.00769644976, -0.0282891262, 0.0575310737, -0.0361095071, -0.0764037743, -0.057672821, 0.061970409, -0.0471889079, 0.00978944544, -0.0800924301, -7.085500e-02, -0.00803701207, -0.0464210808, -0.0202612206, -0.0257231276, 0.0489534363, -0.0554511473, -0.01124766, 0.057374455, -0.0790690556, -7.682190e-02, -0.0311673693, -0.0605857037, -0.0639160797, 0.0726340637, -0.0165241491, -0.0569665916, -0.0352415331, -0.065306276, -0.0764048919, 0.0452710912, -0.0720206052, -0.0610180497, -0.0440906659], [0.0583097376, 0.0288543813, -0.0565711744, -0.0529655404, 0.00430195313, -0.00938338786, 5.94667683E-4, 0.056061551, 3.482770e-02, -0.00761803892, 0.0377810374, -0.0133852875, 0.0377629437, 0.0406295396, 0.0792677402, -0.0140890768, -0.0294127762, 0.0650079325, 0.0198112428, -0.00409354409, -0.055210948, 0.0299321953, -0.0396563485, -0.0156705398, -0.039700143, 0.0419183522, -0.0116513446, 0.0336804129, 0.0496444292, -0.0529789366, -0.00194872729, -0.0234706402, 0.0513212532, 0.0332917087, 0.0548811965, -0.0751849636, -0.0087617049, -0.067564629, -0.081184186, 0.0595601462, 0.0798731148, 0.0648817942, -0.0258288439, -0.0149865951, 0.0277878381, 0.0714527443, -0.0157245547, 0.0290655605, -0.078608565, -5.853930e-02, -0.0410388634, -0.0689105615, 0.0126769328, -2.204420e-02, -0.0640094504, 0.0406892225, 4.882610e-02, -0.0691482946, 0.0714543387, -0.0488692187, 0.048791822, 0.0390090309, 0.0356878527, 0.0719848946, 0.059423998, 0.00486648921, 0.070328258, 0.0423071906, -0.0222526137, 0.0434639528, 0.0161659215, 0.0336414352, -0.0379473493, -0.0235480219, -0.00944177154, -0.0653839633, 0.0231436081, 0.00664957333, 0.0656182617, -0.0564665943, 0.0176781919, -0.0159053039, -0.0388756916, -0.00421151239, -0.0787533149, 0.0239926893, 0.0127988346, 0.0763144121, 0.0654624104, -0.0478079803, 5.08534955E-4, -0.0151434559, -0.0245070364, -0.0491219535, 0.0702957287, -0.0518676415, -0.0779303535, -0.00499115745, -0.0192590784, -0.0415499546, 0.0165648367, -0.00724111358, -0.0760342255, -0.0361795761, -0.024363406, 0.0412060097, -0.0574770421, -0.0718687549, -0.00968546886, 0.0645190924, 0.0753141195, 0.0290515553, 0.00773074757, -0.0186256133, 0.0347372033, 0.0464040898, 0.0246323701, 0.0133040287, -0.0122121908, 0.0537945814, 0.0011008794, 0.00349769392, -0.0592936315, -0.022190107, 0.0418947861, 0.0154770501, -0.0361300185, 0.0705947727], [0.00158051506, 0.0793629214, 0.0231849346, 0.0741851553, -0.0833992063, -0.00598269748, -0.0278169066, 0.0314832628, -0.0189873185, -0.00730540324, -0.0842776075, 0.0167311076, -0.0170253031, 0.0626211166, 0.0497603156, 0.0276093539, 0.0458639711, 0.0810963138, -0.0545300655, -0.0380918197, -0.0207048841, 0.0582330748, 0.0413707718, 7.169680e-02, 0.0118678799, 0.0411098711, -0.0530934781, 0.071509473, -0.024328826, 0.0478336923, 0.0772828832, -0.0157522596, 0.0503815636, 0.0283523593, 0.0256620422, -0.00231194636, -0.0423320048, 0.0136693772, -0.0409918353, 0.0255639218, -0.0489111617, -0.062083099, 0.00588980457, 4.043840e-02, -0.0113156391, 0.070233725, -0.0795172304, 0.0174182281, -0.0428489596, -0.0339188464, 0.00797329191, 0.0228769612, 0.0337704681, 0.0830161347, -0.0536647961, -0.0151161253, -0.0331152156, -0.0788233503, 0.039735429, -0.0692048594, -0.0388939828, -0.0119916154, -0.0539894477, -0.0196693167, -0.0526609495, -0.00520521728, -0.0293136556, -2.954900e-02, 0.0182850119, -7.509570e-02, -0.0656091645, 6.546070e-02, 0.0121193118, -0.057368733, -0.0430256873, 0.0387775339, 0.0170928817, -0.01848072, -0.040844921, 0.0220537428, -0.0232835319, 0.0742264614, -0.00880375318, 0.0152876861, 0.0378482081, 0.0398398228, 0.0716473609, -0.0363928527, -0.032205984, 0.0339167267, -0.080061987, -0.0281238705, -0.0114312582, -0.0166524891, -0.0392289981, -0.00907156896, 0.0128758876, -0.0420414321, -0.0591191761, 0.0710020363, 0.0551033616, -0.0643200725, -0.018723812, 0.0193967633, -0.0458953194, -0.013283981, 0.0738066435, 0.0391247198, -0.0443863273, 0.0349179879, 0.0588472672, -0.00343802827, -0.00798246078, 0.0786018967, -0.0596140847, -0.0518735386, -0.0184723362, 0.0823638588, 0.0546151139, 0.0678967386, -0.0506953895, -0.0119676515, 0.00970063452, 0.0632983893, 0.078221172, 0.0596221425, -1.52123641E-4, 0.0356574841], [0.0762658194, -0.0224869028, 0.0456487164, -0.0658104867, -0.0733802318, 0.0239010435, -0.0118288789, -0.0449186787, -5.10673097E-4, 0.0582371876, 0.058606606, -0.0797801465, -0.0176867303, 0.063046284, 0.0772741138, 0.0308105927, 0.0169569366, 6.731410e-02, 0.0568944961, 0.0359263942, -0.0105466163, 0.029204933, 0.0193189085, 0.0411385931, -0.0466452949, 0.0591482595, -0.0780007913, -0.0456849635, 0.0733766183, -6.163280e-02, -1.852290e-02, 0.0413454138, 0.0421723649, 0.0182271786, -0.0218151696, -0.0260945857, 0.0874709486, -0.0629462451, -0.0572530404, 0.0516106226, 0.0177199971, 0.0105811125, 0.0559458137, 0.0502401963, -0.0743777081, -0.0372369848, -0.00188942137, -0.0774681791, -0.0387360677, -0.0760693103, -0.0262298435, 0.0504596867, 0.0391750559, 0.0442424156, 0.00184364722, -0.0121853715, 0.0238866527, -0.06975995, -0.0789465084, 0.0478478335, -0.0281900223, -0.0622105933, -0.0642108172, 0.0621491968, 0.0272463281, -0.0712576732, 0.00726031046, -0.0366548337, 2.13460706E-4, -0.0764780566, -0.0341820493, -0.0390287302, -0.00856538862, -0.00548313325, -0.00633209571, 8.73178651E-4, 0.0477929823, 0.0302448105, 0.00872805063, -0.0502637215, 0.0124653447, 0.0252888575, -0.011541524, 0.0704511777, 0.0414316729, -0.0693867803, -0.0856097638, 0.0663639382, 0.0637402162, -0.0780002176, 0.0213269237, 0.036569193, -0.0775130763, 0.026704317, 0.0547097623, 0.0264131017, 0.062990196, 0.070774734, 0.00867193472, -0.0186734777, 0.0532148071, -0.0621024221, -0.0408981293, -0.0716423914, -0.00349262636, -8.601870e-02, -0.00576576078, -0.0785934403, -0.0487249903, 0.00250596926, -0.0630589277, -0.0660596415, 6.748790e-02, -0.00158568821, 0.039307382, -0.0691564754, -0.02705984, -0.045131769, 0.0016223907, 0.0334572941, 0.0235680435, 0.0618389249, 0.0633351654, 0.0646801218, -0.0504799522, -0.00408077473, 0.00641474873, 0.0237426348], [0.0599608757, 0.0542686172, 0.00607357034, -0.0349897109, 0.0439102352, -0.0101300636, -0.0355716161, 0.00442049792, -0.080074124, -0.0102027319, 0.059144076, 8.775380e-03, 0.00119401887, 0.0721055195, 0.0709652081, -0.0658919737, -0.0252344795, 0.0653634444, 0.0265075061, 0.0463781767, -0.0366711244, 0.00893107522, 0.019788513, 0.05207197, -0.0479998402, -0.0387324654, 0.0563664064, 0.0193631649, 0.0668496937, -0.00108940469, 0.036772009, -0.0143672787, 0.0571637899, 6.375200e-02, 0.057019826, 0.0442610569, -0.00636996422, 0.0590417758, 4.455040e-02, 0.00634491397, 0.00926810875, -6.976120e-02, -0.0680274069, 0.0570615865, 0.0237613451, -0.0076096775, -0.0769141614, -0.0421466306, -7.664720e-02, -0.0278598666, 0.0178781152, -0.0396564454, -0.0563429594, 0.00794332568, 0.0293522235, -0.0364453979, -0.0557896644, -0.0613405854, 0.0719397739, -0.035855148, 0.0759245679, -0.0362454467, -0.0549708977, 0.0321741924, -0.0375959314, 2.198140e-02, -0.0565728433, 0.00535813952, 6.108050e-04, -0.0306313187, -0.0363557935, 4.547805E-4, 0.0373030677, -0.076630123, -0.0727044418, 0.0459614657, -0.03022133, 0.0282818712, -0.0619706921, -0.0646715462, 0.0703502595, 0.0170173422, -0.0614262894, -0.00255719805, -0.0377817787, 0.0327775516, 0.0604083948, 0.0354750454, 0.0205897763, -0.0107752373, -0.0654646382, 0.0269593745, -0.0600531735, 0.0572525859, -0.00628548767, -0.0620333925, -0.0646349639, 0.0742720366, -0.0618561804, -0.0247439295, 1.860150e-02, -0.00608319789, 0.0740380958, -0.0176753886, 0.0552750751, 0.01545431, 0.0469734631, -0.04312158, 0.0437405147, -0.0385850966, 0.06626755, 0.0523330234, -0.0253664963, 0.0717245266, 4.417210e-02, -0.0594260581, -0.0403397791, 0.0227517113, -0.0208785944, -0.0194608942, 0.0252802186, -0.00698697614, -0.0810766816, 0.039764978, 0.0252738409, 0.0453050621, -0.052036982, -0.0333207212], [0.005972513, -0.0930637717, 0.0422818623, 0.0464929529, -0.00801666453, 0.0699332058, 0.0102554169, 0.0234206263, 0.0826556161, -0.00630633067, -0.0543168858, 0.00191875594, 0.00745517667, -0.0133034522, 0.035521619, -0.0481741205, -0.0559615381, 0.0724753439, 0.0287602637, -0.0186188146, -0.0355860926, -0.0244702902, 0.0322752781, 0.0362461731, 0.038054686, -0.0745310187, 0.0412966646, 0.0276731178, 0.0702173039, 0.0653166547, -0.0478677601, 0.0489479378, -0.0524516217, -0.00967742596, 0.0794444754, -0.00380445388, 0.0752318427, -0.05974764, 0.0133168614, -0.0405531339, -0.0773088485, 0.0701978356, -0.0411318839, -0.0135007631, -0.0750646442, -0.0347023718, -0.0723950118, -0.0511613786, 0.0506441705, 0.0454098359, -0.0361775123, 0.0188804176, -0.0162905958, 0.0396024436, -0.0675375238, -0.0504877977, -5.300090e-02, 0.011842032, 0.0576152727, -0.00141553045, -0.0137933455, 0.0482830182, -0.0455786064, 0.060161788, -3.5754565E-4, 0.0840234681, 0.0772948265, -5.78821113E-4, -0.0350535959, 0.0239775684, -0.0315322541, 0.00432190346, -0.0830895304, 0.029068891, -0.0745579153, 0.0168752782, -0.0879669785, -0.0131641924, 0.0566589311, -0.0180540644, 0.0269883666, -0.0350212827, -0.00749596674, 0.036182005, 0.0160494819, 0.0867287218, 0.0664110184, 0.00413582614, 0.064198859, -0.0133292414, 0.0703537911, -0.0313711055, 0.00555987749, 0.0422290489, -0.0809913501, -0.00887875538, -0.0531250462, -0.0345720463, -0.0255691912, -0.0262047481, -0.00360253383, 0.0887055918, 0.00274833757, 0.0770850629, -0.0380527712, 0.0516017526, -7.812090e-02, 0.0148989111, -0.0830628797, -0.0147597687, -0.032007508, -0.0500985682, -6.721170e-03, -0.0211920477, 0.0520468391, 0.00557253882, -0.0164350681, 0.0630658716, 0.0256652571, 8.033420e-02, 0.0410057977, 0.0839499607, 0.0639068857, 0.00820475165, 0.0479087681, 0.00365798594, -0.0676984488, -0.00620685331], [0.0571218431, 0.0360616483, 0.0229576249, -0.0841223821, -0.0831501781, -0.0356404297, 0.00638855342, -0.0204527285, -0.0465433635, 0.0605162568, 0.011647528, 0.0720888153, -0.0291771553, -0.0589227304, 0.0517680794, -0.0348699354, -0.0093237618, 0.0798370838, -0.00846583117, -0.0566192642, -0.0846102685, 0.00821676664, -0.0678724572, -0.0734767541, 0.0552071258, -0.028240962, -0.058296714, 0.0228045266, -0.0223023463, 0.0671704486, -0.0424985029, 2.534660e-02, -0.0341966972, 0.0462319776, -0.0188380107, 0.0310919043, 0.0523732491, -0.0388020314, -0.0474382304, -7.181580e-02, 0.049359858, -9.697750e-03, 3.892190e-02, -0.0178836975, -0.00931219291, 0.0842512249, 0.0304933786, 0.028286932, 0.0128438836, -0.0329630859, -0.0445872396, -0.0244371165, -0.0338091478, -0.00261363783, 0.012219823, 0.0723400115, 0.0801022202, 0.0575215258, -0.072465606, -0.0338178389, 0.0551206768, 0.0804086477, 0.00439115195, 0.0160377771, 0.00299818418, 0.0539874695, -0.0171117336, -0.0329761915, 0.0570808239, -0.0441539027, 9.120550e-03, -0.0508556552, 0.0691411495, -0.0277174953, -0.0277838949, 0.0363547839, 0.038672775, -0.0438732207, -0.0518973805, -0.0508592911, -8.772500e-03, -0.0195336118, -0.0517053418, -0.0102365501, 0.0575864762, 0.0525023341, 0.00970883108, 0.0129457498, 0.0484056659, -0.00211383239, -0.0382453762, -0.0411395617, -0.0128169023, 0.0429361351, 0.0586252138, -0.0699286162, 0.00967733841, 0.0289492328, 0.0838197321, 0.0359765105, 0.0911040976, 0.0106959548, -0.0449413024, -0.00322101405, -0.0592546314, -0.0433755852, 0.0255541224, 0.0139851896, 0.04132507, 0.0411212705, -0.101846643, 0.021816846, -0.0204040818, -0.123840027, 0.0201860126, 0.0146406861, -0.0345272459, -0.0597161502, 0.0695829466, -0.023288168, -0.0392368026, -0.00227003219, -0.0887932926, -0.0126142241, 0.0679003373, -0.0654864088, 0.079632461, -0.00819398183], [0.00309197465, -0.0658564269, -0.011137764, -0.10998977, 0.054338716, -0.0525208712, 0.0286949202, -0.0671297684, 0.103640027, -0.055219423, 0.0017833832, -0.0203342102, 0.0880511179, 0.0289189667, -0.0259578284, 0.0555512123, -0.0141870445, 0.0616981387, -0.0868271365, -1.62396333E-4, 0.0194511544, -0.0837005228, 0.00500558456, -0.0524753071, 0.089785926, -0.0688456893, -0.0252227038, -0.0800981223, 0.089466989, 0.0866655781, 0.0411273688, -0.0482882261, -0.0454739071, 0.00340178749, 0.0934874191, -0.0120440014, 0.02526691, -0.038819015, 6.527630e-02, -0.0975313559, 0.0577633604, -0.0393849909, 0.0260898862, -1.014940e-01, 0.0556118898, 0.012155992, 0.0235455781, 0.0557008833, 0.0678762048, 0.00708251866, -0.0596292689, -0.0318500623, 0.0608190633, -0.00636578164, -0.0645963326, 0.0380156673, -0.0204161797, -0.0498973206, 0.0295942891, 0.0717771202, -0.04505606, 0.0912979766, -0.0492903367, -0.00146753166, -0.0708381906, 0.0854211524, 0.0610413626, -0.0342612267, -0.0599498302, -0.0025723814, 0.0960593149, 0.0448263101, 0.0606719702, 0.0275739692, 0.00271510053, -0.0583782829, 0.0718966573, -6.282290e-02, 0.070822008, 0.0677248165, -0.0542074032, -0.0621702746, 0.0264908243, 0.0132833906, 0.0516852699, 0.092754811, 0.0794922411, 7.079260e-03, 0.0125672165, 0.0108972983, 0.0656466112, 0.0164670683, -0.0409274213, -0.047666885, -0.0688727647, 5.32973732E-4, 0.0885382444, 0.0321533382, -0.0388021246, -0.0867700353, 0.104317203, 0.06849996, 0.00784072652, 0.0279958155, -0.0262250695, 0.0505729839, -0.0254661608, 0.0622321889, -0.0373452567, 0.0119119007, -0.0358168818, 9.25265834E-4, 0.0791628882, -0.13888596, -0.00940673891, 0.0519995019, 0.0379715152, -0.0532712117, -0.0214618724, 4.408080e-02, 0.0303544905, -0.0169073958, -0.0111908121, 0.00609269179, -0.023592148, 0.0167078208, 0.0229143072, 0.0160366185], [-0.034071546, 0.0164595209, 0.078208223, -0.0136656119, -0.0703245848, -0.046595268, -0.0275233258, -0.00106108165, -0.00566865783, 8.173210e-02, 0.053948123, -0.0305014383, 0.081911847, -0.0225062184, -0.039721854, 0.0208525751, 0.0332105756, 0.047687009, -0.023647543, -0.0466519929, -0.0155608393, -0.0782505125, 0.0511223488, 0.0207240433, 7.499500e-02, 4.105980e-03, -0.0692998692, 0.0218755286, 0.0741009265, 0.116386667, -0.0847250521, 0.026732279, -0.117891096, 0.0304588694, -9.677530e-03, -0.0551278554, -0.0117584113, 6.703430e-02, 0.104808591, -0.0665576905, -0.0133973043, 0.0195273813, 7.517430e-02, -0.0622775889, 0.0928968265, -0.0238934103, 0.0498216301, 0.0931999832, -0.0281475671, 0.0560389236, 0.046140071, -0.11868728, 0.101209521, 6.676800e-03, -0.0265230443, -0.0672056824, 0.0189318825, 7.142650e-02, 0.0441448241, -0.0343901776, -0.0728326663, 0.0280233771, 0.0620308667, 0.0975332558, 0.0692636147, 0.0185663234, 0.00844271667, -0.00241070287, 0.0686698928, 0.0779660642, 0.0791618525, 0.0824457854, 0.0619768649, -1.011650e-01, 0.0689130276, 0.0461519025, 0.0254971813, -0.00523896934, 0.0272371881, -0.04624236, -0.0298474636, -0.0179731734, -0.0498498604, -0.0290304311, 0.0360817648, 0.00604628306, -0.0403841063, -0.0813706144, -0.119903758, 0.00294619682, 0.146482304, -0.0829040557, 0.0549140535, -0.0672788545, -0.0796648934, 0.0549941063, 0.114401169, 0.0143587515, -4.22194309E-4, 0.0485202819, 0.0295190327, 0.0187811218, -0.0121476259, 0.0184272472, -0.0188225172, 0.109453022, 0.0424334705, -0.0592950359, -0.0726726651, 0.03312153, 0.0442174412, -0.0592736304, -0.055208087, -0.0985867753, 0.053676948, -0.0345945768, 0.0822626277, 0.0476353206, 0.0663878917, -0.00670430297, -0.0233035348, -0.0121184131, -6.129960e-04, -0.0144583294, -0.0408555679, -0.0613338687, -0.0236950163, 0.0020159916], [-0.0139525039, -0.020103503, -0.0160481911, -0.127331689, -0.0630215257, -0.0253549833, -0.00575459283, -0.017899638, 0.11050988, 0.0874936655, 0.0209208261, 0.00475910353, 0.062524125, -0.0840527191, 0.0346413776, 0.0557972416, -0.0508447066, -0.0282546338, 0.0551094376, -0.00888770819, 0.0265218075, 0.019385647, 0.0821414739, -0.0817295908, -0.0217063334, -0.0598276891, 0.0495303571, -0.00917524658, -0.0460121147, 0.0398146883, -0.010144568, 0.00256349053, -0.0136078941, 0.107953534, 0.1097835, -0.0627362505, -0.0305186957, -0.0617186166, 0.0161365606, -0.115763709, 0.0213351976, 0.0243261885, -0.026016511, -0.10062556, -0.0186159853, 0.0355644375, 0.0836892798, 0.0416973978, -0.068150796, 0.0239936654, -0.0504033752, -0.00682759564, 0.106328458, 0.065986149, -0.0535714477, 0.0675103441, 0.0455583259, 0.0143722827, -7.854800e-02, 0.0245316271, 0.0785921737, -0.0334652141, 2.092160e-03, 0.00817478914, -0.0727354363, -0.0213768594, 0.0623467676, 1.828920e-02, 0.06421379, -0.005990976, 0.0884803683, 0.0106596351, 0.0316694602, -0.0598351322, 0.0770998523, 0.0287211202, -0.0191123635, -0.025712125, 0.0222073048, -0.0629437267, -0.0253269039, -0.0893596932, 0.0314809456, -0.0508600213, -0.0155161722, -0.0133069465, 0.0918256118, -0.00320098503, -0.00697597163, 0.0359906256, 0.0141275022, 0.0251657702, -0.013469669, 3.166440e-02, 0.0497519076, -0.0661754906, 0.0146338269, -0.0175911356, 0.0746733621, -0.0259346962, 0.0249769669, 0.048448205, -0.0185852237, 0.0947056264, -0.061428085, 0.0204773955, 0.0989937856, -0.0191688798, -0.00762749603, 0.0237048399, -0.0998482927, -0.0331833772, 0.0801256448, -0.132325366, -0.0827848389, -0.0677451715, 0.0546724685, 8.014660e-03, 0.003509206, -0.0603940301, 0.0332194902, 0.0108641423, -0.0267170109, -0.0018053758, 0.0675641596, 0.06214023, -0.0145450123, 0.0338363685], [0.0914792641, -0.0270164516, 0.00469401525, -0.00191493542, 0.00480080163, 0.0389209911, -0.0199496467, -0.0103075206, 0.0555057712, 0.0606562644, -0.0109137306, 0.042369809, 0.0398963839, -0.0318617634, -0.0583500415, 0.0131157702, -0.00792666152, -0.027710313, -0.00677958969, 0.0615509935, -0.0588676557, -0.0535239875, 0.0529603176, -0.0704736561, 0.105651475, 0.00430288212, 0.0246516783, 0.0307903271, -7.656680e-02, 0.0717448816, -0.0414656103, 0.0189110152, 0.0245602131, 0.0299523398, 0.0143717686, -0.0679852962, -0.0195123684, -0.0189988501, 0.0340521522, -0.132619947, 0.0634598881, 0.0464406498, 0.0458306521, 0.0115915891, 3.666980e-02, 0.0677703246, 0.044930324, 0.0451315641, 0.00404173182, -0.0764880478, -0.00588862598, -0.0152577478, 0.0523022451, -0.0814004242, -0.0449745581, 0.0717947483, 0.00899444241, -0.0231641103, 0.0138557535, -0.0306601506, -0.0562339425, 0.00760046067, -0.0254424233, 0.00771525921, -0.0156592708, 0.017100703, -0.0732252895, -7.401620e-02, -0.0659857467, 0.09679313, 0.0715065673, 0.0248369705, -0.0558883213, -9.098900e-02, 0.0695764124, 0.0492791608, 9.22403589E-4, -0.0242039803, -0.0294729061, 0.00970075373, -0.0895944163, -0.092501983, -0.00172225735, 0.0420687273, -0.0516571291, 0.0990357547, 0.0873920544, 0.0636060535, 0.00863675773, 0.0105608534, 0.0743454695, -0.0293567106, 0.0132504497, -0.0359883308, 0.0202353392, -0.0689779595, 0.103347696, 0.0910691097, 0.021278372, -0.0258254856, 0.106947519, 0.0469267927, 0.00606030319, 0.0100391051, 0.00139264984, 0.0198972821, -0.0222902782, -0.0332258642, 0.0940143391, 0.0467679128, -0.0981705188, -0.0506419167, 0.0634539574, -0.0797496512, -0.0140414014, -0.0711519197, -0.0664465055, -0.0832305402, -0.0244672783, 0.0430693105, -0.0257319845, -0.0329903811, 0.135060012, 0.0698280185, 0.0903231278, -0.0912549868, 0.0627505109, -0.0292702131], [0.00840529985, 0.0497338474, 0.0485780388, -5.510870e-02, 0.0380245112, -0.0703699216, 0.0259340443, -0.0324077383, -0.025464043, -0.00873726327, 0.101155102, 0.0545848571, -0.0467752889, -6.292600e-02, -0.0755594596, 0.0362202078, 0.0613672026, 0.0815367922, 6.380310e-02, 2.765460e-02, -0.0382298268, 0.0941355824, -0.0243874323, -0.0736374259, 0.0107609378, 0.00910373777, -4.142910e-02, -0.0597249344, -0.0139658572, -0.0469713621, -0.0702980682, -0.0397638604, 0.0325757861, 9.733950e-02, -8.61694919E-4, -0.0448848829, -0.0141787883, -0.041666843, -0.0160425715, -0.146313369, -0.00984655506, -0.0544555634, 0.0390654765, -0.0467155091, 0.094183892, 0.0373221561, 0.0215191711, 0.0989436581, 0.067954421, -0.0690546855, 0.0323455296, -0.0977213457, -0.0121789668, -0.00138843711, -0.058544945, -0.0179424509, 0.0224959534, 0.0451059416, -0.0449465439, -0.0366493165, -0.0412814133, 0.0348252133, 0.0448819883, 0.0308906194, 0.0711358264, 0.00613837782, -0.0435584188, -0.0524828583, 0.0558994375, 0.113775492, 0.0701384693, -0.0510535426, -0.0295663197, -3.971160e-02, -0.0395666435, 0.0176692121, -0.049494762, -0.0463218838, 0.0788751319, 0.00237023947, -0.0801877602, -0.0687162429, -0.0491964333, -6.139410e-02, -0.0329677202, 0.0833278372, 0.0724870041, -0.0679043457, 0.0359956399, -0.106941044, 0.117328949, 0.0425996259, 0.0854211077, 0.101718076, -0.107257351, 0.0972943454, 0.0109277163, 0.0823864042, 0.0672020316, 0.0372243784, -0.00260093505, 0.0189154614, -0.00661475397, 0.0756390988, -0.132824376, 0.0677038804, -0.0276964456, -0.0809562578, 0.0645503327, 0.0344741493, -0.0284417141, 0.0568488091, -0.00159729866, -0.0775837078, 0.0679990724, 0.042035263, -0.0362139791, -0.0269733369, 0.0160194132, -0.0588155426, 0.0290990286, -0.0183370821, 0.0513436832, 0.118958183, 0.100103974, -0.0492927358, 0.036154937, 0.08721634], [0.0410358943, -0.032112591, -0.0268762093, -0.0603314973, 0.068861559, -0.0888519957, 0.0331730023, -0.0821421295, 0.0893773213, 0.0559724569, 0.0168590564, -0.0539679043, -0.00616481109, 0.0324719138, 0.020014938, 0.0380693562, -0.0411689766, 0.00356701366, 0.0345895588, 0.0404706262, 0.0602616481, 0.0331648365, 0.0296256356, -0.121678725, 0.0505688414, 0.103820197, 0.0694377571, -0.0619836263, -0.0319320373, -0.0132876122, -0.0168678481, 0.0184301529, -0.0733832195, 0.0273795929, -0.0103828907, -0.0483648106, -7.197030e-02, 0.0421499424, 0.0906105041, -0.113509506, 0.0579552129, 0.0337701179, 0.0226353165, -0.0236804336, 0.069692336, -0.0412261374, -5.353120e-02, 0.0649442226, -0.0669774264, -0.0698435456, -0.0476941243, -0.00964155886, 0.0965255573, 0.0373457484, -0.0690266415, 0.0250665452, 0.0333822593, 0.029985806, -0.0620934591, -0.0050976323, 0.025284769, 0.0212203469, -0.0334049352, 0.00643418217, 0.136201382, -0.0163864773, -0.0351016521, 0.0581955761, 0.024079239, 0.00259342114, 0.0601841174, -0.115683034, 0.0152803892, -0.0101025729, -0.0224274099, -0.0330539607, 0.105056211, 0.0551394634, -0.0576372333, -8.899140e-02, -0.0425072201, -0.0360829681, -0.0724568665, 0.0209489577, 0.02748961, 0.0816292166, -0.0146722104, -0.0886264368, 0.0119479289, 0.0241152402, -0.00623671338, 0.0458369628, -0.034369614, 0.110421978, 0.0356929339, 0.0792773813, -0.0444228612, -0.0111456979, -0.00635568983, 0.0261020605, 0.0541800559, 0.0563606545, 0.0430854335, 0.00544489361, 0.0214708038, 0.0902167708, 0.100898154, 0.0323455967, 0.0436532125, 0.0463495851, -0.0253461208, -0.0656073093, 0.00722873583, -0.0112077259, 0.0419950522, -0.047808744, -0.0641836897, -0.113995008, -0.0100999242, -0.00371171557, 0.0657993108, -0.00598535361, 0.122501813, 0.10994973, 0.0869827345, -0.0029637306, 0.073288925, -0.0721801147], [0.0449567959, 0.0249414239, 0.0460324734, -0.0089430008, -0.0585580841, 0.0420071781, -0.0533161499, 0.0681370794, 0.0472365245, -0.0245838538, 0.055290807, 0.0122035714, -0.00967255979, 0.0210012384, 0.0116757154, 0.0277687833, 0.0412717238, 0.0283459164, -0.0543313883, 0.00791326537, -0.0632531642, 0.0670193285, -0.0303569492, -0.0612749122, -0.0257788301, 0.0237444695, 0.0422902852, -0.0795325562, 0.0634761676, 0.0173965506, 0.00964999571, 0.136584014, -0.019934902, -0.0126448646, -0.0645091683, -0.0138717508, -0.06587217, 0.0396219306, -0.0197981969, -0.151993334, -0.0226928014, 0.0713300481, 0.0531414524, 7.37733033E-4, 0.0502266474, -0.016329892, -0.108469196, -0.0743863434, -0.0256648324, -0.0517904274, 0.0246583018, 0.0232124981, -0.0154140508, 0.0150395511, 0.0600389279, 0.0297324322, 0.0133523149, -0.0358432829, 0.0330928154, 0.132970527, 0.025047807, 0.0364050604, -0.0306389369, -0.072567068, 0.00594231253, -0.0189200807, 0.0327315442, -0.0143924206, -0.0545124263, 0.0277423728, 0.116474949, -0.0211373456, -0.0212416574, 0.0155606018, -0.0563909672, 0.070899494, 0.042872481, 0.0519054309, 0.05553611, -0.0164065752, 0.0904044658, 0.0514553376, -0.0533253476, 0.0458432622, -0.014804502, 0.0319471024, 0.0409313403, -0.0500604585, 0.0628333613, 0.0162223913, -0.0514965653, -0.030675659, -0.0305378065, 0.0581508316, 0.0295499507, -0.0156557858, -0.0334930941, 0.0913252755, -0.0644410923, -0.0475062579, 0.100795962, 0.0871458203, 0.0247642305, 0.0590460636, 0.0101497294, 0.0703108534, 0.0182552449, -0.039333988, -0.00206030253, 0.0782949551, -0.0893250704, 0.0431217514, -0.0698114931, 0.0567262396, -0.0718672573, 0.0524267219, 0.0628696904, -0.114017151, -9.9328719E-4, 0.0789534226, 0.0819120183, -0.0346583314, -0.00360832969, 0.124960966, 0.0567065068, -0.0243965611, 0.0358187035, -0.0697269589], [0.0542263575, -0.0766580254, 0.0596783347, 0.0193627737, 0.0512554236, -0.0615193695, 0.0135524487, -0.0323323645, 0.0527601056, 0.00888257846, 0.0500754118, -8.156980e-03, 0.02910484, -0.112602755, 0.0149877258, -0.0216304921, 0.0244142991, -0.0444451161, -0.0282824319, -0.0567550287, -0.0074449447, 0.0847452282, -0.0577697828, 0.00850726664, 0.0733140707, -0.00504976092, 0.0868997574, 0.0408998318, -0.0869322866, -0.0174368639, -0.0319877602, 0.0361028723, 0.0164257418, 0.0151061472, -0.115943417, 0.0284663048, -0.0257892162, 0.0753443539, -0.0565074198, -0.145883441, -0.0485763177, -0.0655692741, 0.0374787115, -0.0670538246, 0.00128194178, 9.405530e-02, 3.972940e-02, -0.0949310585, 0.105050482, -6.716730e-02, 0.0824692174, 0.016079193, -0.109923728, -0.0827671214, 0.0715379268, 0.00327949063, -0.0391991884, 0.0087078195, -0.0453853831, 0.173248425, 0.0213781036, -0.0269106887, -0.0450756438, -0.0287324581, 0.0116542475, 0.00259831245, 0.069677487, 0.0431557335, -0.023868829, -0.0298812613, -0.0270900317, -0.0495822355, 0.135704368, 0.0422747694, -0.0319713764, 0.0190408193, 0.0164266638, -0.0698630884, 0.00619182921, -0.0139168687, 0.0736755207, 0.00688286079, 0.0879966468, -4.83920681E-4, -0.0113246199, 5.043180e-02, -0.0173495635, -0.0985072255, 0.0139287869, -0.0025964973, 0.0200444255, -0.0190640297, -0.108581871, -0.0554615371, -0.0159876887, 0.0342511833, -0.0318223797, 0.0703225583, 0.00113106542, -0.0408801027, 0.0710225478, 0.0705571175, 0.0100826919, -0.0608377158, 0.0870484188, -7.210250e-02, 0.0349849947, -0.0309408959, 0.0865119696, -0.0393577255, -0.0604148246, 0.0615354218, 0.0689675882, 0.04550061, 0.0051032207, -0.0331518687, -9.92890214E-4, -0.0597191527, -0.0890023931, 0.0722983703, 0.0819903686, -0.0512918942, 0.102098607, 0.00449301489, 0.061261639, 0.0920790061, -0.0441068187, 0.0311635286], [-0.0721203089, 0.0144768246, 0.0517331548, 0.0492995977, -0.0375664122, -0.0703129396, 0.0820141136, 0.07446035, -0.0355404057, 0.0411508232, 0.0449761823, -0.0515550375, -0.0167275835, -0.0992421433, -0.0771959201, -0.0428292677, -0.058448229, -0.0788888931, -0.00210151915, -0.075282976, -0.0132638924, 0.0793277919, -0.0431583412, -0.0331996866, 0.0635147095, -0.0840869397, -0.0351479873, -0.0426189639, -0.0780709683, -0.00408316264, 0.0903226882, 0.102462538, -0.0226169545, 0.0688622221, -0.133014992, 6.99238269E-4, 0.0199168697, 0.157098278, -0.0996196791, -0.101995267, -0.0567651093, 0.0222604629, -0.0181112196, -0.0656657144, 0.090222232, 0.0226072297, 0.039677795, 0.0221455358, -7.351220e-02, 0.00779520906, 0.0413502976, -0.0292171128, 0.00542983692, 0.0830044671, 0.0371840671, -0.0129948491, -0.00556569267, -0.0506035648, 0.0590460598, 0.0302021075, 0.0846131518, -0.00875220168, 0.00724902097, 0.0539788827, 0.0874410197, 0.113841482, -0.0427313633, 0.0243391022, 0.0767254084, 0.0716039389, 0.033229284, 0.00239782385, 0.112510629, 0.0420323499, 0.0368265435, 0.0372919627, 0.0389221311, 0.0505244732, 0.0631673485, -0.0713359267, 0.0514458418, 0.058595024, 0.0389180481, 0.0962592065, 0.026520865, -0.0364417471, -0.014503614, 0.0361456499, 0.0250624381, 0.00883275364, -0.0145521527, 0.0686714277, 0.0419267267, -8.260490e-02, 0.0748617649, 0.0867946073, 0.00153139699, 0.0359519199, 0.0983241572, 0.0259171277, 0.0927403792, 0.0304861553, -0.0585240833, 0.0688660517, 0.041367311, 0.0513979904, -0.00895776972, -0.0324680768, -0.0498649329, 0.0678424165, -0.0363558754, -0.0325039625, -0.0616805777, 0.0613078699, -0.0306438729, 0.0623305589, -0.041453898, -0.112200744, 0.0315229408, 2.577340e-02, 0.0624442584, -0.0998809039, 0.0304279644, -0.037781667, 0.0909307971, 0.0160253458, 0.0742197633, 0.0647256523], [-0.0669473931, 0.00476387329, -0.0190988742, 0.0954974219, -0.0466583595, -0.0380610265, 0.0174005888, 0.0242191572, 0.0164780319, 0.0101772817, -0.0476760156, 0.0279321671, -0.0131038977, -0.105660826, 0.0621452332, -0.0662050098, 0.0850236043, 0.0446457155, 0.0196276866, -0.121182539, -0.0308605153, 0.054435581, 0.0231068023, -0.0579814576, 0.0826668143, -0.0153506398, -0.029415004, -0.0354211666, -0.0540791303, -0.0082535455, 0.096827507, 0.0842843875, -0.047553692, 0.140782952, -0.0211843643, -0.0900309756, 0.113786004, 0.0783909484, -0.109327845, -0.0782193839, -0.0458330438, -0.0550542027, -0.0600214265, 0.0133354571, -0.0479200147, 0.0824187919, -0.0619066618, 0.0537489168, -0.0032483933, -0.0423348173, -0.0450890288, -0.102126211, 0.0259773284, 0.0487144925, -0.0291065425, 0.0399061963, -0.0602423772, -0.0267735496, 0.0720513389, 0.0687257424, -0.0246615019, 0.0338167697, 0.030589072, 0.00664798496, -0.00352497585, 0.115444489, -0.0863949805, -0.00936631485, -0.0368466228, 0.066178225, 0.130994812, 0.0343396328, -9.612550e-03, -0.057771612, -0.0488947704, 7.133420e-02, -0.0697405487, 0.0109561374, -6.85726118E-4, -0.0200936235, 0.062814042, 0.0122954324, 0.121488877, 0.132838383, 0.0283129811, 0.0688653067, -0.119607054, -0.0194189027, -0.00982519891, -0.0307703279, 0.0252660401, 0.102238335, 0.0749091431, 0.0771037266, 0.0705452412, 0.0382292978, 0.0557678714, -0.00213777367, 0.0369849131, -6.46406144E-4, 0.0581897125, 0.0872449204, 0.0449260175, 0.0587116666, 0.0760871843, -0.0730747133, -0.0244697966, -0.0370756947, -0.0560591742, 0.0300281048, -0.0257227607, -3.589510e-02, 0.0599778295, 0.0987152531, 0.00139298046, -0.00562484935, -0.0249970965, 0.0110363672, -0.0737045854, -0.00990943331, 0.103892207, 0.0612263121, -0.0344991684, 0.00981591269, 0.0220537689, 0.0362070091, -0.0163523387, -0.00398528436], [-0.0906212255, 0.0694241449, -0.0551321134, 0.0399872214, 0.0262165572, 0.00594218448, 0.0223546419, 0.0648136362, -0.112904601, -0.0052300063, 0.0811249539, 0.0016287918, -0.0308432337, -0.103770278, 0.0167091154, 0.0543176904, -0.0413848087, 0.0590616912, -0.0112339966, -0.0545707569, 0.0617858246, 0.081086278, 0.0425360836, 0.0114671895, 0.022012813, 0.0924592763, -3.414850e-02, 0.0882380456, 0.00478121405, -0.0360840484, 0.0907015353, 4.651990e-02, -0.0934828817, 0.0207747351, -0.0785400271, 0.0236762874, 0.0638847724, -2.689590e-02, -0.0731028691, 0.0469115749, 0.0859468653, -0.0652605519, 0.0379248336, 0.0109338984, -0.0634988174, 0.0644057766, 0.0616122559, 0.0489144549, -0.0781461969, -0.00639738888, -0.0681200102, 0.0384795032, -0.0519394763, -0.0257214066, 0.0613286719, 0.0621398277, 0.00993693992, -0.0295651555, 0.0378491096, 0.0284612793, -0.0619176626, 0.0597109348, 0.04175983, -0.0353909172, -0.0154934023, 8.833260e-02, 0.0303601604, -0.0133098783, 7.650420e-02, 0.085533224, 0.085507825, 8.347840e-03, 6.916400e-02, 0.010948319, 0.0519797578, 0.0212756898, 0.0215739161, 0.0595718846, 0.0872835368, 0.044428546, 0.0936133041, -0.0486838073, -0.00663137157, -0.0259522088, -0.0270244908, 0.0732847303, 0.0336309299, -0.00367253437, -0.0352349281, 0.0824725702, -0.0783720985, 0.0347252786, -0.0305389632, -0.0342387557, 0.0681555942, -0.00910698436, 0.0181742683, 0.0451213457, 0.0760027617, -0.00898382067, 0.020881407, 0.0865529552, -0.0114022056, -0.0715017468, -0.0487240627, -0.0964646413, 0.00834049378, -0.0122469179, 0.0270304605, -0.0165210478, -4.109470e-03, 0.0705567896, 0.0462408327, 0.0497966669, 0.117480837, -0.088992536, 0.0453180373, 0.0755174458, 0.0266612154, 0.0204495955, 0.0595710538, 0.0674213916, -0.0442135409, -0.0369079486, 0.00369903119, 0.00526527874, 0.0257939119, 0.0587053597], [0.0202868674, 0.0567438565, -0.0571612529, -8.15897656E-4, -0.0125811649, -0.0626338646, 0.0829600319, -0.0655690208, -0.100835435, 0.0407397337, -0.0231318083, -0.00885546301, 0.0381660387, -0.0782530605, -0.0448405743, -0.0216568094, 4.934980e-02, 0.0517895557, -0.0495383553, 0.0577432513, 0.067414239, -0.0239344165, -0.0531709157, -0.0934530124, -0.0133816395, 0.0201758053, 0.00483467802, -0.0224876925, 0.0787591487, 0.0740248561, -0.0405202173, 0.0254707038, -0.0846189484, 0.0242396574, -5.667590e-02, 0.0364086442, -0.0145925535, -0.0418853387, -0.0187168904, -7.401930e-03, -0.0519458465, -0.0172480736, -0.0438904576, -9.41402337E-4, 0.0426127538, -0.0536751375, -3.470550e-03, 0.0139751052, 0.0026693407, -0.00339296646, 0.0595978126, 0.0830505639, 6.256720e-02, -0.0501310341, 0.0875525549, 0.103969716, 0.0634212717, 0.0799189061, 0.0197893623, -0.009466229, 0.0571293794, 0.0699568167, 0.01356804, -0.0662940815, -0.0391588248, 0.0694418475, -1.588520e-02, -0.0448344871, -0.0535655841, -0.0092643965, -0.0458438024, 0.0408456847, -0.01208359, 0.0236365292, -0.00135027897, 0.0386136137, -0.0466991737, -0.0212850366, 0.0565636866, -0.0376575179, -0.00177939876, 0.0292974934, 0.0589373112, 0.0177295115, -0.0273666158, 0.0485214889, -0.0522600263, 0.00197251374, -0.00559087796, 0.0283946451, -7.116390e-02, 8.137210e-02, 0.12261577, -0.0279643685, -4.671580e-02, 0.0639379621, 0.0031588634, 0.0294841062, 0.025714634, 0.0134072062, -0.00838463102, 0.0548417792, -0.098655641, -0.0906131267, -0.0226655249, -0.0585848913, -6.753640e-02, 0.0586892068, -0.00502187712, 0.0499448702, -0.016004378, -0.0187537074, 0.0504888296, 0.0117429756, 0.0432197265, 0.0496495515, 0.0383563712, 0.100955717, -0.00860143732, -0.0173882358, 0.0337362215, 5.092590e-02, -0.0901407599, 0.0185564961, -0.0234510265, -0.104967512, 0.131436437, -0.0148925763], [-0.0234044548, 0.061217051, -0.0662694797, 0.0488279462, 0.0173191596, -0.0416045263, 1.051250e-01, 0.0402574763, 0.0077385297, -0.0582115836, 0.0749518201, -4.36786388E-4, -0.0117105758, -0.0874422044, -0.0608929693, -0.0105007552, 0.0556582548, 0.0207414515, -0.0674315616, -0.00208129175, -0.00897793844, 0.0597130395, 0.0591470376, 0.0180477556, -0.0257649012, -0.0443976372, -0.0362842828, -0.00456385827, -0.0731820166, 0.0418879688, 0.0245278329, 0.0222160127, -0.0217315312, -0.0404805206, -0.00897417497, 0.0583943203, -0.0133693824, 0.0694777369, -0.00616779597, 0.0181374419, -3.691530e-02, -0.0550022796, -0.0375338718, -0.0207849499, -0.0421918631, -0.0727848262, -0.098568648, 0.0633254424, -0.0255440529, 0.0345081948, 0.0610975772, 0.0375359692, -0.0681392252, 0.0237480365, 0.0165298395, -0.00586699275, -0.030753959, -0.0238147713, -0.0544199944, 0.0585009418, -0.0455068536, 0.0117070302, 0.0393920168, -0.0521360487, -0.022329418, -0.0318035334, 0.0518345796, 0.0144893946, 0.0685156956, 0.0934277326, -0.105982475, 9.71792906E-4, -6.320510e-02, -0.0467294678, 0.00380890933, -0.0640167221, 0.0294331834, -0.0144806085, 0.0817233994, -0.0509915277, 0.0865686759, 0.0130740451, 0.0200033654, -0.0432330631, -0.0977605953, 0.0671796873, 0.00871352386, -0.00283158943, -0.0467939042, 0.0419776961, -0.0487530269, -0.00983568188, -2.307170e-02, -0.0428154469, -0.0521755032, -0.0876771659, 0.0010815775, -0.0388779864, 0.0432801284, 0.0233973321, 0.0583128408, 0.0444124117, -0.0418344028, -0.0563034043, -0.0736313611, -0.111554064, 0.0228522774, 0.00574609498, -0.0121335695, 0.0870420784, 0.0553053468, -0.0752586871, -5.125860e-02, -0.0356774367, 0.0842837467, 0.0207332652, -0.0443756729, 1.496330e-01, -0.0367041156, 0.0790191665, 0.0403499752, -0.0355366766, -0.0297007728, -0.103156373, 3.589870e-03, -0.0472011305, 0.037843477, -0.0166788939], [0.0335175954, -0.0112891281, -0.0225770045, 0.00401387969, 0.0100127086, -0.0775657743, -0.0431049317, -0.0866022557, -0.0292161796, 0.0154343769, -0.0289200041, 0.0352532193, -0.0520539507, -0.116358712, 0.00237160525, 0.0789119601, 0.0603411421, -0.0595569834, 0.0643303394, 0.0166490451, 0.00864400808, -0.0142149841, 0.0884989649, -0.0355397314, -0.0202557929, -0.00310522038, 0.0165163297, -0.0514835082, 0.0697632805, 0.0208088066, 0.0235244837, 0.0149223432, -0.134738669, -0.0405059792, -0.0117183272, 0.00822942145, 0.00282473443, -0.00979078468, -0.0897517204, 0.0792400911, 0.0563777536, 0.0623055286, -0.0454536825, 0.0218541324, 0.0348026119, -0.0240431912, -0.0444351956, 0.0586415567, 0.0662267581, 0.097166799, 0.0150551014, 0.0578509271, -0.0134333437, -8.043090e-02, 0.035750661, 0.0835716277, -0.0297951512, -0.0308835097, -0.070393294, -0.0162265487, 0.0446268432, 0.023311872, -0.00629023369, -0.0753864124, -0.0705463067, -0.010354789, 0.0881689936, -0.0102610588, 0.00450169388, -4.692330e-02, 0.015609392, 0.0562065206, 0.00240054238, 0.0490929261, 0.00291200122, -0.060855113, -0.0403667316, -0.0612678342, -0.0390814021, 0.0159925707, 0.103304967, -0.0151811168, 0.0271779913, -0.0588945933, 0.0333670788, 0.0908703804, 0.0419556797, -0.00476982398, -0.0254188962, 0.0167254899, -0.0811033919, -0.00747429114, 0.122511394, 0.00565536413, 0.0801847428, 0.0248546321, 0.0406106673, 0.0411517583, -0.00336971413, 0.0793156102, -0.0437706821, 0.0379191823, -0.0799294337, -0.064898178, -0.0181588102, -0.112483338, -0.0510386042, 0.0470311046, -0.023426054, -2.483470e-02, -0.00849515758, -0.00886327773, -0.029106997, -0.00808602757, -0.017861329, 0.00305109075, 0.00364802731, 0.126219258, -0.0368972942, 8.752910e-03, -0.00582981622, -0.0020178121, -0.0391093716, 0.00416042423, 0.0533139743, -0.010225852, 0.0876742526, 0.0429091528], [0.0427914038, -0.00427783467, 0.0615046918, 0.0301434938, -0.0872587338, 0.0415251516, 0.0260718912, -0.102089189, 0.0683440343, -0.0370405912, -0.00370594277, -0.0709825903, -0.0686680228, -0.0210069735, 0.0657066926, 0.0835852324, 0.0583435856, -0.00497250166, -0.0635119304, -0.00328771141, 6.944650e-02, 0.0115616042, -0.0261442754, -0.0580559596, 0.0868307873, 0.0282173306, -0.0118145877, 0.0339333639, 0.0108674895, 0.0766286253, -0.0292879529, 0.0356327556, -0.138427556, 0.0360124074, 0.0401579253, -0.00798866898, 0.00559289521, 0.041206792, -0.0563116409, 0.0702631921, 3.986330e-02, 0.0205003936, -0.036813587, -0.0106914891, 0.0539360829, -0.0337044485, -0.001352928, 0.0403751656, 0.0767373442, 0.0057132449, 0.0754034594, 0.0419466458, 0.0579687506, -0.0734468326, 0.025246121, -0.0048559811, 0.031577222, 0.0146889687, -0.0834719911, -0.0617189333, 0.0326960236, 0.0518075675, -0.0603917837, 3.89025779E-4, -0.0566488802, -0.0278973728, -6.25220768E-4, 0.015366341, -0.0594699271, -0.00457882136, 0.0233211629, -0.031986881, -0.0922965183, -0.0249297023, -0.0561020076, -0.0395335816, -0.0839940831, -0.0451699644, -0.0288424678, -0.0618445203, 0.0646358281, 0.0116749248, -0.0317504033, -0.069011189, -0.0757197812, 0.0388824791, -0.0636729375, -0.066655904, -0.0519954078, 0.0663077533, -0.0896804407, -0.0906154066, 0.0119487764, -0.0465714969, 0.00588699151, -0.0198850315, 0.0284962673, 0.0137833124, 0.0369320773, -0.0657047778, -0.0214318912, 0.00962777063, 0.00544979796, -0.105772629, 0.0286337901, -0.0920133814, -0.0645643249, -0.0729495659, -0.0414580442, 0.0766416565, 0.0211288519, 0.0374605209, -0.0499219336, -0.0860998705, 0.0115588959, 0.0134887453, 0.0429290049, 0.0253968798, 0.0666763782, 0.0723687112, 0.035939198, 0.0304498933, 0.018886419, -0.0556492731, 0.0735560283, -0.0397951901, -0.0223875437, 0.0706742778], [0.0589875132, -0.0173328053, -0.0705359429, -0.0311103575, 0.0486780852, -0.0051936619, -0.0282667652, -0.0782356634, 0.0357594751, -0.0264442321, 0.0359536186, -0.0251332279, 6.278650e-02, -0.102753542, 0.0670719817, 0.0096312305, -0.0682503507, 0.0194150396, -0.00286361552, 0.00527788233, -0.00143784226, -0.0732198805, 0.0559309386, -0.089128539, 0.00443025446, 0.0486911125, 0.0442496464, -0.0835425257, 0.0326908678, 0.057416033, 0.0247536339, -0.0446888879, -0.105070807, 0.084682554, -0.0727961659, 0.0545683764, 0.0777710452, 0.0525840931, 0.0716261864, -0.0381419063, -0.022769358, -0.0759419426, -0.015156839, 0.0149600357, -0.0613744258, -0.0250442512, -0.0433591455, -0.0553211682, 0.0830513239, 0.00433315942, -0.0619678087, -0.00794335175, 0.0182310157, -0.0605689175, -0.02769446, -0.00553369522, -8.404070e-02, 0.0446525402, -0.0750796944, -0.0295280106, -0.0385884978, 0.0659390613, -0.0647025332, -0.0135505237, -0.0078862505, 0.00521877501, 0.112269707, 0.00828510523, 2.92248704E-4, 0.0161586385, -3.324130e-02, 0.0729020312, -0.0231105424, 0.0156753398, -0.0247481111, 0.0240000952, -0.0581101552, -0.0645528137, 0.0416079797, 0.0156552754, -0.00399039173, -0.0542003214, -0.0195733979, -0.0520623662, -0.0039305673, 0.0201649088, -0.00440614345, -0.0588262677, 0.0630334616, -0.0810977295, 0.0741798877, -0.0926866158, 0.071444422, 0.0599632375, 0.0272084065, -0.0427264981, 0.00688695488, 0.0419799685, -0.0164010059, 0.0445403717, 0.0724510476, -0.0452668145, 0.00544063188, 0.0415039137, -0.0593463108, -0.0270329583, -7.563370e-02, 0.0790205076, 0.0503830537, 0.0295115709, 0.0237956587, -0.00171821797, 0.0490011238, 0.0443536118, -0.0467738546, 0.0843105167, 0.0714373216, 0.016055122, 0.0510681383, -0.0171050876, 0.0492320769, 0.0219210144, -0.0815386325, -0.0774281695, -0.0244001821, 0.0301375724, -0.0513793267, 5.212780e-02], [-0.0681321322, 0.0452225842, -0.0193397161, 0.0293611195, 0.0257019624, -0.0811224207, -0.0456616059, -0.065503627, 0.0688262731, 0.0419528298, -0.0402089916, -0.0548794083, -4.3843739E-4, -0.0527489223, 0.0714601129, -0.0190947335, -0.0354548581, -0.0207062755, -0.0452796668, 0.0590796247, -0.0511502884, -0.0351655707, 0.042670738, 0.032210242, 0.0745958462, -0.0139279636, -0.0592554696, 0.036859598, -0.0774553641, 0.0876904428, -0.0763284191, 0.00255408208, -0.0681896582, -0.0112868221, 0.0449032597, 0.108172916, -0.0429284237, 0.0208743289, -0.0108435769, -0.0601159409, 0.0399832055, 0.060390342, 0.0918680205, 0.0416132845, 0.00841907412, 0.0142981661, 0.00702108908, -0.0739914253, 0.0561068691, 0.100735784, 7.423000e-02, 0.0691274256, -0.0439287424, 0.034734156, 3.050560e-02, -0.0134469327, 0.0428201705, -0.0496009029, -0.0283927135, 0.0557917729, -0.0713052079, -0.0385759324, -0.087006256, 0.00175147259, 0.00235068961, 0.0637442544, -0.0213625487, 0.0757331625, 0.0839192718, 0.0469409786, -0.104662783, -0.0136377476, -0.0769089386, -0.0743656754, 0.00377084431, 0.0577339195, -0.0196689721, -0.0279981457, 0.00729953917, 0.0725731328, 0.0600675978, 0.0469820127, -0.0434348807, -0.0205166023, 0.0528859459, 0.00243689748, 0.0524398349, 0.00198354921, 2.463390e-02, 0.0124330213, 0.0899310037, -0.0927833542, 0.00356206507, 0.0747277141, 0.049926158, 0.0637170747, -0.0204580929, 0.103020683, -0.0518638417, -0.0112216407, 0.0862891376, 0.00856525451, 0.00425116951, -0.086105369, -0.0850364342, -0.0365328565, -0.0201134495, 0.0442051739, -0.0879957675, 0.0685182661, -0.0198919252, 0.0184125379, 0.058161106, 0.0229721777, -0.0592081621, 0.0530365929, -0.0409719907, 0.0246316046, -0.0169676449, 6.773870e-02, -0.0121247368, -0.0518043078, -2.420870e-02, -0.0723187327, -0.0262985472, -0.0628265589, -0.0483160242, -0.0296867862], [-0.0175097324, -0.0600630306, -0.0228845514, -0.029632261, -0.0219057389, 0.0251404587, 0.0299601033, -0.0436482243, -0.0625395328, -0.0578867942, 0.0429220274, -0.0138333319, 0.0237124674, -0.0219440535, 0.00271730893, -6.645510e-02, 0.0662288517, -0.0140879313, 0.0761700794, 0.00801128428, -0.0298416689, -0.0403274782, 0.053209506, -0.0154802222, 0.0159308836, 0.0287950709, -0.0657036081, 8.920160e-03, -0.0784710049, -0.0103036128, -0.0673924759, 0.0687712878, -0.0492173545, 0.0456711352, -0.082866244, -0.0181263387, -0.0506259836, 0.0180349406, -0.00808550138, -0.0689240396, 0.00296043651, -0.0153825078, -0.0254436191, 0.0288512427, -0.00301783392, -0.0840310528, -0.07422553, -0.07552623, -6.063460e-02, -0.0340349153, 0.0726355314, -0.0416691601, 0.0230320543, -0.0334907807, -0.0819075555, -0.0568558201, 0.0248539895, -0.00563037302, 0.0243458133, -0.0310505759, 0.0409559421, -0.0366117246, -0.0525464043, 0.00572834909, 6.346910e-02, 0.0609272383, 0.0133610312, 0.00768798869, -0.0242342185, 0.0172049627, 0.00211999752, 0.0459588096, -0.0488886312, 0.0431968495, -0.0122963181, 0.0889685153, -0.0484198816, 0.0623424313, -0.00665449863, 0.0506067127, 0.0045241029, -7.487900e-03, -0.0282210354, 0.0263562091, 0.00606759498, 0.00793191884, 0.0156343617, -0.077102825, -0.0273235198, -0.0630184785, -0.0315859355, -0.0989517271, -0.0458935536, -0.0769527704, 0.00480031781, -0.00544844568, 0.0409512259, -0.026451081, -0.0800864696, -0.054481525, 0.0756628364, -0.0563726351, -0.0119916517, 0.0190220177, -0.062983714, 0.0290884972, -0.0350369066, 0.00288456865, 0.0346872807, 0.0146198468, 0.0461271629, 0.0465310179, -0.0627465546, -0.0266609881, -0.0780066773, 0.0925886184, -0.0254586432, -0.0286496319, -0.0716896802, 0.0672276914, -0.0383199677, 0.0368379764, 0.0730941892, 0.0804066807, -0.0371582657, -7.24623678E-4, 0.0427235179, -0.0187328104], [-0.0355569385, 0.0377120338, -0.0723003522, -0.0680093467, -0.0188429151, 0.0400071144, 0.0156046068, 0.0475760028, 0.0532853082, -0.028916629, -0.00504954718, -0.0786956698, 6.01977226E-4, 0.0788631811, 0.00700987689, 0.0643398687, 0.0427764244, -0.0729679465, 0.0761906504, 0.0403881371, 0.0493008196, -3.552620e-02, -8.484870e-02, -0.0409797654, 0.0183480978, 0.0032340039, 0.0579843484, 0.0608012974, 0.0141830808, 0.0230240617, 0.0445861407, -7.71598075E-4, 0.0412872881, 0.00191268127, 0.0632139444, -0.0381247029, -0.00597589649, -0.0360096805, -0.0153413173, -0.0506886505, 0.00267745717, 0.0207121037, 0.0169716179, -0.0174661875, 0.00508751394, -0.0721026137, 0.023944404, 7.690340e-02, 0.0333764553, -0.0201539453, -0.0600284301, -5.262710e-02, -0.0604930036, -0.048016049, -0.0211314261, -0.0464599542, 0.0384567529, -0.0694181845, -0.0556438901, -0.0295323674, 0.0297659803, -0.0496184938, 0.0241656415, 0.0388479531, -0.00109419436, 0.0502183959, -0.00439430447, 0.0154205197, 0.0496491827, -0.072480984, 0.0284192655, 0.039275527, 0.0470901281, 0.0807621404, -0.016667936, -0.0372728892, 0.0193891246, 0.0458846502, -0.0381961241, 0.0802254826, 0.0437165834, 0.0103853308, -0.0212604795, -0.0674120485, 0.0186564382, -0.0791184902, 0.0449220538, -0.0514726155, -0.0281059667, 0.0749801397, 0.026656013, -0.0476678088, -0.0373084582, -0.00289301551, 0.061514385, -0.0536857434, 0.0187499877, -0.0625480041, 7.660210e-02, 0.0204652715, -0.0807407423, -0.0551035218, -0.0422229469, -0.0497682765, -0.0517020486, 0.066802606, -0.0671369284, -0.0683139414, -6.02881541E-4, 0.0571364872, 0.0254535787, 3.335500e-04, 0.0132966088, 0.0506672636, -0.0787575394, -0.00355304196, -0.0338317715, -0.0423515588, 0.0318520851, 0.0743687749, -0.0520309545, -0.0738571435, 0.0740507543, 0.0234859269, -0.0624466352, -0.08327955, 0.00353139476, -0.0276737623], [4.68951584E-5, 0.0113881044, 0.0494500473, 0.022763215, -0.02465312, 6.57974044E-4, 0.0220231265, -0.00502729369, 0.0431568101, 0.0810803994, -0.0395826623, -0.0348877311, -0.00783863477, -0.054712493, 0.0489724837, -0.0224491749, 0.0378468223, -0.00707038213, -0.0643400847, 0.00265556178, -0.0470063649, -5.191950e-02, 0.0416897871, -0.019966932, 0.00473499717, 0.0331846476, 0.0175304972, 0.0224027671, 0.00335970684, 0.0460836254, -0.0158215147, 0.0428590216, 0.0578269772, -0.03893115, 0.0657022223, -0.0635612383, -0.0228111073, -0.0233924314, 0.0675475448, -0.0148709482, -0.0457856357, 0.0241848733, -0.0642318055, -6.83433376E-4, -0.0566750839, 0.013881485, 0.0612867549, 0.00413538143, 0.0633379518, 0.0562586859, 0.0299448743, -0.00427095778, 0.00540745771, -0.0467112288, 0.0501332171, -9.081520e-03, 0.0667094812, 0.0397797115, -0.0269045923, -6.676900e-02, -0.0197519902, 0.0153681226, 0.0434281155, 0.0359314866, 0.0157755073, 0.00971983094, -0.0715421885, 0.0787340626, 0.0125472425, 0.0641967058, 0.0117623135, 0.00196078862, -0.0256242882, -0.0777320638, -0.0715896264, -0.0246074889, 0.019602159, -0.0197910909, -4.440450e-02, 3.981140e-02, -8.99508188E-4, 0.0346526019, 0.0786934122, -0.0099657271, -0.0364256538, 0.0595694296, 0.0560121872, -0.019805437, -0.00898103415, -0.00451350817, -0.0290705208, 0.0246422123, 0.019882407, -0.0212644283, 0.078854993, -0.014111165, 0.0159602407, 0.0103419703, -0.0655183941, 0.0190154314, -0.0456629544, -0.0678755641, -0.00568070542, 0.0170193408, 0.0778390616, -0.0551279858, -0.00849550589, -0.0719904825, 0.031981349, -0.0379428379, 0.0419772789, 0.0355168954, 0.0221027359, -0.0145989843, 0.012402989, -5.584820e-02, -0.0220303666, 0.017447263, -0.0394780524, 0.046108488, -0.0666904151, -0.0671072528, -0.0105105527, 0.0803577974, -0.0691353604, 0.0169076119, 0.0688980445, 0.0360653438], [-0.00283386279, 0.00276442012, 0.0344627723, -0.0616204962, -0.0212265253, -0.051213935, -0.0256697349, -0.0728360042, -0.0086484868, -0.00611458439, 0.0605097786, 0.0755834952, 0.0288685262, 8.906560e-03, 0.0593117103, 0.0314833373, 0.0605914369, 0.0548737571, 0.02692491, -0.0347607136, 0.0364332087, 0.0578028336, -0.0175474361, 0.0107404664, -0.0713762417, 0.0701939464, -1.035200e-02, -0.0541344881, 0.0546232574, -0.0398523584, -0.0474554449, -0.00967100728, -0.0718788206, -0.0792297795, 0.0145802442, -0.0612188764, 1.269520e-02, -0.0296094529, 0.0332017876, 0.00562824961, 0.0255305227, -0.0480476916, 0.0151510155, 0.0605584122, 0.0554594547, -0.0681533217, -0.0812310725, -0.0642942935, -0.0516974777, -0.0342099704, 0.0596396662, -0.0118903778, 0.00584431272, -0.0307038799, 0.0702081099, 0.0330019556, -0.0794522166, -0.0731393322, -0.0479795486, 0.0685793683, -0.0689188913, 0.0522538722, -3.347660e-02, 0.0493202247, 0.0211949479, 0.0326490365, 0.0570934564, 0.0198281184, -0.0458458588, -0.0294176806, -7.261090e-02, 0.0674174204, -0.00494948821, 0.00832847692, 0.00803536922, -0.0166603327, -0.0665988401, 0.00973572675, -0.0422225185, -0.0345337614, 0.0620909519, -0.0567333102, 0.0144364983, -0.0327994078, 0.0683807954, -0.0333564952, -0.00634155516, 0.0354106352, 0.0694813132, 0.0394588746, 0.0346530862, 0.0716894716, 0.0218992773, -0.0234559029, -0.00412706705, 0.0632785931, -0.0590978526, -0.0274625141, 0.0181837212, -0.0697990357, -0.066365324, 0.0718962848, -0.0637057573, -0.00293451012, -0.00534667075, -0.0657985061, 0.0650617406, -4.127830e-02, 0.0564436242, 0.0541315638, 0.0694256722, -0.0542661324, -0.0747249722, -0.014583176, 0.0525734611, 0.0197344087, 0.0259825159, 7.419520e-02, 0.0397340395, -0.0674480497, -0.0561427623, -0.014284634, 0.0540809631, 0.0515278392, 0.0222555902, -0.0428839773, -0.0646567419, 0.063376218], [0.0588773713, 0.062143784, 0.0715297237, -0.0673729107, -0.06328132, 0.05649326, -0.0776549354, -0.0131485518, -0.0640167295, 0.026832087, 0.0225700866, -0.0502158515, -0.03015773, 0.0117225908, -0.0217113979, -0.044956658, -0.0608080253, -0.0326515511, 0.0651171431, -0.0184736103, 0.0217167139, 0.0560569242, -0.042995479, -0.0561110564, 0.00819606055, 0.0254495218, -0.0287330952, -0.0372934565, -0.00282617658, -6.865680e-02, -0.0715846419, -0.0453043506, 0.0110754259, 0.00114733225, -0.0748800933, -0.0451489203, -0.0105724428, -0.0585523918, -0.0122309905, 5.142930e-02, 0.0419818312, 0.06484548, 0.00818512496, 0.0739764646, -4.024150e-02, 0.0434169807, -0.00248897821, 0.0377440117, -0.0605986491, -6.022860e-02, -0.00180757046, 0.0513882376, -0.05327712, 0.00139644858, -0.06633766, 0.0349883735, -0.0773745551, -0.0491959378, 0.0356425047, -0.0571062975, -0.00895664282, 0.0241115429, -0.0516492836, 0.00919273868, 0.0301875081, -0.0757707432, 0.0579815656, 0.0253560916, 0.0151879899, -0.021439746, -0.0545255132, 0.0520600937, -0.0629027858, 0.0413789488, 0.0736139193, 0.0142414877, -0.0223866012, -0.0766569077, 0.0578232966, 0.0199362841, -0.00748632709, -0.0578303076, 0.00255236775, 0.0753761306, 0.0165577848, 0.0561081953, 0.0734305158, 0.0412224457, 0.0545415804, -0.0466005616, 0.0453240611, -0.0491521098, 0.0174748264, 0.00723889377, -0.0197395124, 0.0687031373, -0.0531461202, 0.0215768665, -0.0210553687, -0.0327762477, 0.0050198934, 0.0308154747, -0.0139981797, 0.0205437858, 0.0194288194, 0.0378195494, -0.0197741855, 0.00303375232, -0.0386421978, 0.0106892288, -0.0293262098, -0.028103482, -0.00727379322, -0.00796839688, 0.0535939075, -0.0710320696, -0.0669701397, 0.026960358, -0.0718208104, 0.0138376057, 0.0443033241, -0.06017223, 7.240440e-03, -0.0469716378, -0.0130491517, -0.0616082624, -0.0556424595, -1.12716909E-4], [-0.0615401529, -0.0230348352, 0.0731423572, 0.0604619272, -0.0513526462, -0.0756615177, -0.0268440377, -0.048638951, -0.0175386257, 0.0793772563, -0.0384305231, -0.0616274811, -0.0683559105, 0.049286522, -0.0510007478, -0.0729003698, -5.298600e-02, 0.0163687244, 0.0149353789, 0.0606378764, -0.0675737709, -6.20154373E-4, -0.0757081583, 0.0428475104, 0.0174037442, -0.0576328225, 0.0771949291, 0.00145534007, 0.0596759915, 0.00610945094, 4.302700e-02, 0.00826112926, 0.0415536202, -0.0774767846, -0.0782751142, -0.0425817929, -0.0721998736, -0.032928139, 0.0427131169, -0.0777997524, 0.00390143367, 0.0748461782, -0.0491278619, 6.500490e-02, -0.0485035554, 0.0728470981, -0.0370570198, -0.0312633328, 0.00302343839, 0.0558494925, -0.0688307062, -0.0778961629, 0.0626788661, -0.0468441658, 0.040399503, 0.0326043628, 0.0313119739, -3.129580e-02, 0.0539996885, -0.00471462682, 0.0697504058, 0.0447321348, -0.0319458693, 0.0495663583, 6.410790e-02, -0.00978836697, -0.0645354465, -0.0148439882, 0.0613559335, -6.576070e-02, 0.0526105277, -0.0285821054, -0.0648477077, 0.0811409354, -0.0242459029, 0.06070498, 0.0559818558, -0.0354823582, 0.0127044478, -0.00841725897, -0.02255583, 0.065759249, -0.0074570626, 0.00184587785, 0.0427037776, 0.0287407618, 0.0389303938, 0.0242456067, 0.0265722666, 0.00376178324, 0.072693564, 0.0362177566, -0.031874232, -0.0124671189, -0.0382136591, 0.0134852035, 0.0788013786, 0.0133292507, -0.0437011868, -0.02674683, 0.0365082324, -0.00318637514, 0.0788265988, -0.0286321528, 0.0621278323, -0.021236157, -0.0391830839, 0.0253119264, 0.0625656098, -0.00607721088, -6.558420e-02, -0.0101106018, -0.0283778328, 0.00807262957, 0.0396831483, 0.0806374177, -0.0207936577, -0.0174618009, -0.0536465608, 0.00658928743, 0.0463037826, 0.0124543076, 0.0145822465, -0.0337719433, -0.00893347896, 0.00835058838, 0.0467333309, 0.0160780493], [-0.0770079643, -0.0352454036, -0.0195600223, -0.0399654359, -0.0350399651, 0.0270004533, -0.00273652095, -0.0637819394, 0.0635644272, 0.0219622608, 0.00541050639, -0.0105640572, -0.0564882159, -0.0435043871, -0.0069416957, -0.0439309515, 0.0392807461, -0.0468515344, 0.040979933, 0.040940538, 0.0132931285, 0.00217421027, 5.863190e-02, -0.0686962157, 0.0568911545, -0.0729812309, -0.0223842487, -0.0447628573, 0.0616285168, 0.0211004112, 0.0347702056, 0.0572979823, 0.0308704469, 0.0172118247, -0.043848943, 0.0351769738, -0.0365904532, -0.0397152491, -0.0639916286, -0.0463721156, -0.0116306599, -0.0787512585, 0.0159832183, 0.0619545057, -0.0102506895, -0.0487367585, 0.0389073342, 0.024898855, 0.0770584121, -0.0337923616, 0.0183030684, 0.0636331737, 0.0118324384, -0.0565939322, 0.0778585597, 0.0216285847, 0.0186506547, -0.0652880594, -0.0268112496, 0.0376241282, 0.00207374222, -5.696180e-02, 0.044642251, -0.0653544441, 0.00268627144, 2.632110e-02, 0.00596082304, 0.0666586161, -0.0197021794, 0.0480804406, -0.0257466566, -0.0331452116, 0.0431885533, -0.0662938654, 0.00437168591, 0.0698732287, -0.030895723, -0.0155472523, 0.0614338331, 0.0172203742, -0.0629470646, -0.0228836108, -0.0718731582, -0.00829033181, 0.039795056, -0.0295834839, 0.0608196184, -0.00229533226, 0.0118101668, 0.0203732122, 0.0302298442, 0.0692071616, 0.0425632074, -0.0112273367, -0.0317335911, -0.0210192613, -0.0170393698, 0.0316450596, -0.0354568064, -0.0780507251, -0.0654879659, -0.0436728112, -0.0249276403, 0.00660835439, 0.0360085778, 0.0372192152, 0.0422921032, -0.0776492655, -0.0133959958, -0.065285772, 0.0215752926, -0.049555324, 0.0367419757, 0.0385081396, 0.0608997531, 8.553700e-03, 0.0465525724, -0.0362405442, -0.0818337276, -0.0679429099, 0.0475996323, 3.394990e-02, 0.0761487111, 0.0121451318, -0.0156761557, 0.0720201731, -0.045226939, 0.0278951321], [0.0155078517, -0.0404868424, 0.0516127609, -0.0622564405, 0.00894905161, -0.0287301093, 0.0129956789, 0.0304170679, 0.0722245425, -0.0374624431, 0.0472606272, 0.0163068175, -0.0705099553, 0.0570028201, 0.0565921068, -0.00325986394, -0.077082023, 0.0235984717, 0.0636854768, -0.0698017254, 0.0219492204, 0.0312934555, 0.00941086095, -0.0382900313, 0.00503348187, -2.623310e-02, 0.0697437822, -0.0445394963, 0.0125792874, -0.0740648732, 0.0520221666, 0.0271923449, 0.0609836094, -0.0167471897, 0.0579249561, 0.0282909647, 0.0440293029, -0.0495015457, 0.0610196106, 0.039684657, 0.027404435, -0.0797059834, 0.0797050893, -0.0463466495, 0.0147691704, -0.0635659546, 0.0133365057, 0.00434209779, -0.0281028859, 0.023908047, 0.0356146842, -0.00141785352, 0.0745996386, 0.0112484312, -0.0755087808, 0.0653371364, -0.0626391768, 0.0474921241, 0.0494036414, 0.0736697689, -0.051034756, -0.00395094324, -0.038874153, -0.0190412421, 0.00149361033, 0.0669004098, -3.619720e-02, -0.036329709, -0.0608076155, 0.0270450097, -0.0548884459, -0.0792668983, 0.0222675819, -0.0615412481, 0.0748178288, 0.0274192989, 0.0413907021, -0.0661078393, -0.0170956049, -0.00799439196, 0.0758824423, 0.010000132, 0.0265646111, 0.0380940922, -0.0441411622, -0.059866868, 0.00662080152, 0.0446288139, -0.0127784982, -0.083004646, 0.0107280919, -8.167950e-02, 0.0288888663, 0.0805171057, 0.0727864504, 0.0140526071, 0.0326017588, -0.0142072579, 0.0086621195, 0.0311202202, -0.0170489736, 0.0781897976, 0.0420572236, 0.0774168148, -0.0368712619, 0.0377549343, 0.0443375483, -0.0249158312, 0.0116604222, 0.0135415271, 6.479500e-02, -0.0301781036, -0.0301248562, -0.0414741486, -0.0156760514, -0.0462239534, 0.00967722386, 0.0536514707, -0.0304658227, -0.0146678276, 0.0193238594, 0.0462389141, -0.0442379266, 0.0793393403, -0.0568338446, -0.0208508465, 7.408300e-02, 0.039987009], [0.00330237043, -0.0658938065, 0.0254210774, 0.0681077242, 0.00939184893, 6.468850e-02, -0.0641033202, -0.0518455245, -0.0376970321, 5.29929122E-4, -0.0387888588, 0.0165460743, -0.0425650366, -0.0352407061, -0.046783898, -0.00672611361, -0.00573678548, -0.0459505618, 0.0233105291, -0.0200916026, -0.00945955608, 0.0228495598, -0.0438452102, 0.0294369664, -0.0339722261, 0.0764162242, 0.00488272542, -0.0811354815, 0.0772051513, -0.00203452446, -0.0775113255, 0.049663987, -0.0327345207, 0.0305669587, 0.0201694984, 0.0248686336, -0.0071654832, 0.0662875101, -0.0779328197, -0.0533167534, 0.0351270661, -0.0718740597, -0.022150198, -0.0193850901, -0.0155179063, 0.0633613467, -0.0833692774, 0.0375699513, -0.0658841282, 0.0114395469, 0.0594542064, -0.0899009854, -0.0219766572, 0.020831937, -0.0599835329, 0.0177500471, 0.0454648882, -0.0687057077, 0.0420704186, 0.084767796, -0.056777332, 0.0136783691, 0.0637159795, -0.0320994072, 0.0570953786, 0.0937572047, 0.0175325796, -0.0605809167, 0.0707961321, 0.0766456723, 0.00317780091, -0.00864314474, -0.0548201278, 0.0321747959, 0.0437224209, -0.0228034575, -0.0263808519, -0.0646919608, -0.0689689443, 0.014521095, 8.84644862E-4, -0.0794464871, -0.00222965889, 0.0547147654, -0.00218570163, 8.45291142E-5, 0.00135916786, 0.0735483542, -0.0252683554, 0.0539128594, 0.0212183446, -0.0089034792, 0.0683140308, -0.0592230149, -0.0774725303, -0.0356219336, -0.021195624, 0.0344901457, -7.883920e-03, -0.0476028845, 0.0275578164, 0.0485613719, -0.0762344748, -0.0344720408, -0.00804785452, -0.024294002, -0.046401307, 0.0559359826, 0.010000702, 0.0162045732, -0.00304587069, -0.0560656488, -0.0729275793, 0.0241029765, 0.075684309, 0.0273127053, -0.0186781883, 0.0762176811, -0.0319875404, 0.0333640315, 0.0197907649, 0.0456009358, 0.0165245421, -0.0105409063, -0.0327574871, -0.0535716563, -0.0748783275, -0.031362433], [-0.0606566742, -0.0695415065, 0.030982485, -0.0242930036, 0.0262399036, -7.136100e-02, 0.0897409319, -0.0194192659, 0.085365124, 0.00916833431, -0.0398037806, -0.0421907417, 0.0672856271, -0.00616106065, 0.0686920285, 0.00105563598, 0.0575869642, 0.0320640802, -0.00746067334, 0.0178779662, -0.0480600633, -0.0282471664, -0.0323126093, 5.003590e-03, -0.0314353481, -0.0123186465, 0.0658545941, -0.0832584947, 0.0531526953, 0.0880473181, 0.0554854199, 0.0320631415, -0.0326179266, 0.0806775167, -0.0511208773, -0.082790181, -0.00712493761, -0.0348400511, 0.0361833572, -0.00431957282, 0.0498612411, -0.0412491672, 0.00732160034, -0.0654596612, -0.0179492533, 0.0445167907, 0.0857171937, -0.0126201902, -0.0523024686, 0.00868019461, 0.0591706149, 0.0422057807, 0.02703893, -0.0370495655, -6.672490e-02, 0.0640144795, 0.0762404576, -0.00512241526, 6.978460e-02, 4.93461208E-4, -0.0645933226, -2.761480e-02, -0.00801926665, -0.0518748835, -0.0306592602, 0.0497537628, -0.0772654712, -6.886850e-02, -0.0730131939, 0.00854073651, -0.0368136205, -0.00164428633, -0.0576642677, 1.822830e-02, -0.050251592, -5.74036792E-4, -0.0549481213, -0.0691694394, -0.0071981959, 0.0415792651, -0.0355249122, -0.107509106, -0.0211512875, 0.00294168643, -0.00141936599, 5.356900e-02, 0.0623974837, -0.0628925487, 0.0674341619, -0.0214863587, 0.0102765793, 0.00283666817, -0.0411202759, -0.0394017249, 0.0122745493, -0.0553247035, -0.0396808274, -0.0146836834, 0.0528501756, 0.0349239558, 0.102359824, -4.041490e-02, 0.0301255882, 0.0667587742, -0.0999970212, -0.0163444821, -0.036482431, 0.0732626616, -0.0626446083, -0.0191906337, -0.0986229255, -0.0645806119, -0.018431399, 0.0206571855, -0.0744740292, 0.041174911, -0.0358068161, -0.065022096, -0.0617316365, 0.0751510859, -0.0536883064, 0.0367474444, 0.0740804225, 0.0870431513, 0.045444902, 0.0187921617, -0.0671823322, 0.0751586407], [0.0318499096, 0.00336115668, -0.0551904365, -0.0178964604, -0.0104424618, -0.080476135, 0.0489517376, 0.0559298433, 0.111129992, -0.0136520648, -0.0257644095, -0.0128006712, -0.0400319844, -0.11568515, 0.0065066074, 0.005686027, -0.0335106924, 0.0924698412, -0.0634847954, -0.0128987506, 0.0457068533, -0.0653010383, -0.0057173376, -0.0685716942, 0.0180038065, 0.0357159711, -0.0380061604, -0.0410432741, -0.00206801551, 0.0616142749, 6.135070e-02, 0.0667003393, -0.0572347902, 0.0609083138, 0.00550809503, -0.0645978525, -0.0664657801, -0.039486248, 0.105510183, 0.021975629, 0.0691474602, 0.0139715644, 8.341360e-02, -0.0745088235, 0.050897792, 0.0187847968, -0.0363400765, -0.0373135284, 0.0111748334, -0.00137375924, 0.0529561825, -0.0415276289, -0.0282890759, 0.03361037, -0.0190780386, 0.00555354869, -0.0664127767, -0.0228468329, 0.0234381519, 0.00285075558, 0.0190570857, 0.0260824598, 0.049487941, 0.0857271403, 0.0219523478, 0.0748754591, 5.852950e-02, 0.0234303493, 0.0556058139, 0.074867852, -0.00285052857, 0.0120738531, -0.0621118843, 0.00244389544, -0.00542954588, 0.0323372781, 0.0719670877, 0.0595416911, -0.0167304818, -0.04075877, 4.798880e-02, -0.00654848106, -0.027647119, 0.029849533, 0.0414259061, -6.33496966E-4, 0.0420982502, 0.0104330825, -0.0598619208, 7.017800e-02, 0.0901671424, -0.0407385863, -0.0654346942, -0.0283333398, -0.0662552267, -0.0236236081, 0.0942316725, -0.024908198, -0.0343847871, -0.0429074317, 0.0908838137, -0.0261972379, 0.0499066524, 0.024444405, -0.0466904789, 0.0257848427, -0.00467464188, 0.0790710374, -0.0461103059, -0.00706015248, -0.0343220793, 0.0683281049, -0.00618316233, -0.115400754, 0.0492167324, 0.0634210184, 0.0182144046, -0.0842353776, 0.0688460842, -0.0121567752, 0.047272753, 0.0642282143, -0.0397271365, -0.012939644, -0.0252355933, 0.0113666588, 0.0541232377, -0.0618406236], [6.883030e-02, -0.0843307599, 0.0668754279, -0.0845465585, -0.10569182, 0.00912246387, -0.0681365952, -0.0630967841, -0.0278091021, -0.0366053842, -0.0719847381, 0.0159139354, 0.0327822901, 0.0108644404, 7.066870e-03, -0.0728413612, -0.0831398963, -0.0312065352, 0.0593093298, 0.0878011584, -0.0621915049, -5.598300e-03, -0.0370485522, -0.0612174533, -0.0605215356, 0.0360067412, 0.065187037, -0.02075148, -0.0139418961, -0.0315575264, -0.0699883848, -0.0622294024, -0.0166759714, 4.38268675E-4, -0.037664365, 0.0402016118, -0.0490406081, 0.00866410416, 0.0435450487, -0.0517418496, -0.0413754061, 0.0200725235, -0.00206583855, 0.0429425165, 0.0255093873, 0.0031630334, 0.0244141109, 0.0378438197, 0.0384555049, 0.0360039547, 0.041184403, -1.020040e-01, -0.0189459864, 0.0364927426, -0.0258973408, 0.0593170486, -0.0304862168, 0.0519160256, 0.0700523928, -0.048291713, 0.00500977831, 0.0185319744, -0.0259945989, -0.0171441603, -0.0604142249, 0.0445160605, -0.0577838644, -0.0369417779, -0.00386259146, 0.0240530092, 0.114722207, 0.043969404, 0.00905316323, -0.00226393039, 0.0415536426, -0.0274344385, 0.0158809703, 0.0374571234, -0.0416942351, 0.0191829987, 0.0138977608, -0.0752700344, 0.056218043, 0.0276231952, -0.0695338175, 0.00301012513, 0.0694397315, -3.090500e-02, -0.0249528121, 0.0489734374, 0.097050324, -0.00821745209, -0.0438228808, 0.0538618341, -0.0221956391, -0.0394292958, 0.0336450152, 0.0390744731, -0.0562858582, -0.0921989902, 0.0962026044, 0.0903046876, 0.0310193226, 0.0662626773, -0.0566062853, 0.0739479064, 0.0621379167, -0.0478739925, 0.0322169587, 0.085043393, -6.436850e-02, -0.0297443569, -0.0435499884, -0.0186838321, -1.93955217E-4, 0.0546825975, -0.00724435877, 0.0269562826, 0.0658530593, 0.0296022799, 0.0513608828, 0.0611302108, -0.0115874195, 0.0678824931, -3.576540e-03, -0.0737874061, -0.0556203872, -0.037480697], [-7.001680e-02, -8.320820e-02, -0.0195601899, -0.0251084045, -0.00571701489, 0.0475571752, 0.0703257322, -0.0183818974, -0.022796182, 0.024681367, 0.0535537265, 5.88720031E-5, -0.0322157778, -0.0412868671, 0.0287140552, -0.085232757, 0.0202331059, 0.0982334837, 0.0457976237, -0.0390807912, -0.0570987761, -0.0129463952, -0.0206158236, -0.0651462376, -0.0323345885, 0.0124993278, 0.0521273091, -0.0715876594, 0.0660332069, -0.0495753549, -8.504990e-03, -0.0186417121, 0.0114582973, 0.069933921, 0.0366563499, 0.0117677609, -0.0533189327, -0.0674471557, 2.10717772E-4, 0.010577484, -0.0160826389, 0.0395525806, -0.037213169, -0.00821027905, 0.0674797818, -0.0371416137, 0.0120904902, 0.0641231536, 0.0633004382, 0.0432888307, 0.0845130458, -0.108360372, 0.0484181754, -0.0783562883, -0.0193647891, 0.076205343, -0.0139167113, -0.0191519577, -0.0407288447, -0.0500492528, -0.0198906437, 0.0242058132, 5.867390e-02, -0.0302919652, -0.0100981146, 0.046867732, -0.00714096334, -0.0653998851, 0.0160164498, 0.0443933457, 0.114304759, -0.0146550573, -0.0737396404, -0.135909855, 0.0578104965, 0.0330778584, -0.0439254455, 0.0529904664, 0.0700107514, 0.0543196313, 0.0218480267, -0.111588962, 0.00829530507, 0.0672958866, -0.0590972453, 0.107244872, 0.0508852191, 3.30667128E-4, 0.0189944394, 0.0379211083, 0.0223151352, 0.0559556745, -0.0234150309, 0.0276975539, -0.018065026, -0.0701775327, -0.00658449251, 0.0945793241, 0.076704137, 0.0416328907, 0.0345860198, 0.0566122085, -0.0500071198, -0.0130590536, -0.00341970194, 0.0719932243, -0.0112367347, 0.0657890737, -0.0521390066, 0.0589826778, -0.103925094, -0.0627891645, -0.0531706661, -0.104624465, -0.0260433145, 0.0644453764, 0.0521828346, -6.810510e-03, -0.0321671627, 0.0715007781, 0.0265972912, 0.122298427, -0.018888006, 0.093328379, 0.0170437619, -6.14427321E-4, -0.0302056391, -0.0590257756], [0.0395692438, 0.0131478701, -6.636810e-02, -0.114493757, 0.0595822074, -0.00881211087, 0.00703793718, -0.0856603682, -0.0347329639, 0.0175656434, 0.026670631, 0.0545376465, 0.0963383466, -0.1104583, 0.0290222727, 0.0160788968, -0.0341603048, 0.0567986518, -0.0812196881, 0.0593428575, 0.00989471189, 0.00937848165, -0.0285091102, 0.0103039602, 0.0533154681, 0.0946652963, 1.969350e-02, 2.856920e-03, 0.019485658, 0.0460491255, -0.0539835542, -0.0584203303, -0.0829346925, -0.011926136, 0.101161078, 0.0777100474, -0.0433430709, -0.0546854548, -0.0281402674, -0.121778823, -0.055042658, 0.058956977, 0.0601919591, -0.0463591553, 0.0968229472, 0.0678544938, 0.0906021446, 0.044068709, -0.0401247479, 0.0507275164, -0.0134763252, 0.0133813452, 0.0882166698, -0.0794503838, -0.049475763, 0.0139017794, -0.0429294966, 0.0728280917, 0.05083514, 0.0889558568, -0.00148725882, 0.0360879563, 0.00296527636, -0.0347663835, -0.0801136047, -0.0597947948, 0.0675208494, 0.0315659232, -0.0202058759, -0.00601848122, 0.0165109988, 0.0517488122, -0.0650855377, -0.0196119472, -0.00816310197, -0.00630326476, -0.0577387698, -0.0257920064, -0.00519320369, -0.0664554164, -0.0898362398, -0.0589558557, -0.0663009286, 0.0436505787, 0.0160434041, 0.0247201584, -0.0204822924, 0.0596809648, 0.0310933013, -0.0167185012, -0.0146390293, 0.0268208124, -0.0223551746, 0.0176471807, 0.0473936833, 0.0328779519, -0.0256160758, -0.0416986831, -0.0246061143, 0.0408438891, 0.137878433, 0.0460105576, -0.00999730173, -0.0242400225, -0.0723361373, 0.0686173066, 0.0544098727, -0.0510994419, 0.0742258355, 0.0849984884, -0.100673839, -0.0728140175, 0.0703728721, -0.122712851, -0.00451618712, -0.0110208923, -0.0836318805, -0.0643786117, -0.0447904766, 0.0221955441, 0.0654432773, 0.0438681953, 0.0645107105, 0.113382071, -0.0487904549, -0.0497530065, 0.0638868585, 0.0739283413], [0.0601063818, 0.0552235544, 0.0681778416, -0.0543660037, -0.0867888927, -7.810040e-02, 0.0332337655, 0.0106157493, 2.331760e-02, 0.0970628485, 0.00167751696, 0.024999436, -0.0188465565, 0.00602345215, 0.0158564225, -0.102478579, -0.0489347354, 0.0761172473, -0.0500508808, 0.0541131049, 0.00337188412, 0.0633980781, 0.100530878, 0.038162332, 0.0966684594, 0.110214457, 0.026678523, -0.0401453152, 0.0223045908, 0.0271611735, 0.0746717751, 0.0364878811, 0.0414725021, -0.0368294083, -0.012535654, 0.0442101061, 0.105824232, -2.059290e-02, 0.0063126022, -0.0467494242, 0.0362392962, 0.0354095921, -0.0279061869, 0.0159190688, 0.0529943965, -0.0699022487, 0.0587036088, 0.143414825, -0.0239385627, -0.0113386344, 0.0600248687, -0.00280601461, 0.0546176471, -0.139267102, -0.0559563227, -0.0603227578, 0.0366223752, 0.0300182067, 0.00216567866, 0.0321273617, 0.044602748, 0.0774454846, -0.0236793067, 0.0706312209, 0.0685472116, -0.0145467687, -0.010037425, 0.0663117841, 1.98326117E-4, 0.0445969962, 0.0461761206, -0.0872749537, 0.0137031712, 0.045971442, -0.0295326561, 0.0408905856, 0.0223633274, -0.0459999554, 0.072089456, 0.0339445174, 0.0112374751, 0.0341401547, -0.0162564516, 0.0813203379, 0.103836156, 0.0773630589, 0.0668132305, -0.0945645645, 0.016212374, -0.0218596868, 0.0791458413, -0.0245188083, 0.0284593664, 0.0809283927, 0.0235856771, -0.0347137488, 0.0602294654, -0.0295800231, -0.00382058229, 0.0647772178, 0.0159503464, -0.0428828932, -0.093595013, 0.0052198139, -0.0101723764, 0.0339063928, -0.0116274888, 0.0216048509, 0.0333049111, 0.032489147, -0.092287518, -0.0266724154, 0.031635046, -0.0864626318, 0.0232403073, 0.0523235053, 0.00449973205, 0.0447736159, 0.092054896, 0.0409429669, 0.0829132944, 0.0454835519, -0.0217897426, 0.126514405, -0.0593843348, -0.0558874831, 0.0644219145, -0.0200942699], [0.103386529, 0.0656421482, 0.0265502334, -0.0884611606, -0.0655646697, -0.0323803164, -0.0272709094, -1.502560e-02, -0.037128251, 0.0342727192, 0.107468188, -0.0327667817, 0.0660547614, -0.0806861221, 0.0582404174, -0.0530278943, -0.0620761327, -0.0847167149, 2.024790e-02, -0.05700697, 0.0353144929, -0.00179573591, 0.00554520031, -0.125817522, 0.070991069, -0.00722824875, -0.0370314904, 0.00266838539, 0.0235642623, 7.80526258E-4, -0.01252189, 0.0453471206, 0.0703965723, -0.0330736302, 0.124776453, 0.0269774124, 0.0363381729, -0.0542507358, 0.0125754075, -0.0628610402, 0.0130796973, 0.00781030441, 0.0641265363, -0.0405872278, 0.00465673907, 0.0534740686, -0.067708075, 0.110430248, 0.0127534866, 0.0441578701, 0.0335432477, -0.0269656964, 0.0916605964, -0.0537746549, 0.0513117649, -0.0291175898, -0.0575548373, -0.0125258313, -0.0475093536, 0.0233816504, -0.00173602416, 0.0456541851, 0.0312063023, 0.0939396098, 0.00538648386, -0.0128761334, -0.0631123632, 0.0351338163, -0.0413069241, 0.026711382, 0.122904778, -0.0868255198, -0.0289845318, -0.00718389126, -0.0559084602, 0.0855554193, 0.12773557, 0.064171642, 5.194460e-02, -0.0501067378, -0.0014949867, -0.0873761102, 0.0231596529, -0.0586450286, 0.0932423099, 0.0637428835, 0.0887023956, 0.0503429025, -0.0707899481, 0.0399026498, 0.0616684631, -0.00585528277, -0.0537815355, 0.115708612, -0.0681617334, -0.0282803103, 0.0387048498, -0.0470708497, 0.0154609084, -0.0861038565, 0.106120154, 0.0372667275, 0.00978111103, 0.0532176457, -0.0831354856, 0.0179636683, 0.0503903888, -0.00417482387, 0.0862973257, 0.0164056793, -1.083470e-01, 0.0316754058, -0.0657239556, -0.111962013, -0.0997087284, 7.232990e-02, -0.106469966, -0.00254466198, 0.0312397741, 0.023903532, -0.0320850462, 0.0475166515, 0.103267573, 0.0422202647, -0.0128302258, -0.066337809, -0.0370296724, 0.0667696595], [-0.00159538747, -0.0207944736, -0.0563458465, -0.182693973, 0.0430034734, -0.0646238476, 0.0267216787, -5.39209112E-4, 0.00478272233, 0.0414593741, 0.098530434, -0.0257148389, -0.0355865434, -0.0464979522, -0.0853697955, -0.0427485146, -0.0168054365, 0.0661421716, -0.038964618, -0.0137685295, 0.0129398061, 0.0763585046, 0.0464161411, -0.0970729067, 0.0704546347, 0.0230693668, 0.0122318445, 0.02033245, -0.0116136046, -0.0213130414, -0.0150206052, 0.0698114261, -0.0139336511, 0.0220748708, -0.00657117925, 0.084611766, -0.0632734522, -0.0695775077, 0.0753137246, -0.0985924825, 0.0304155387, 0.0278315563, 0.00161982595, -0.0819428339, 0.0330783837, -0.0781353414, -0.0156030012, -2.903110e-02, 5.256700e-02, 0.066402249, -0.00605518138, -0.0354972593, -0.0242532026, -0.0737457052, -0.0641803071, 0.0316159762, -0.0321010724, -0.0662653223, -0.0371169485, -0.0157803111, 0.0216051769, -0.0775039792, 0.0178154856, 0.0325654671, -0.0324903727, -0.121620059, -0.0541484319, -0.075020276, -0.0215200204, 0.00658437936, 0.058164373, -0.0373913608, -0.0458924472, -2.494710e-02, -0.0207070317, -0.00102354609, 0.00109228981, -0.0898415818, 0.0383237079, 0.045429226, 0.046755217, -0.0511796512, 0.0222339481, -0.0705402717, 0.0317552127, 0.0507372431, 0.0996943861, -0.0467142202, -0.0597399324, -0.092086412, 0.0536606237, -0.052057676, 0.0556601137, 0.0511674099, 0.0354556553, 0.0954077095, 0.0414034538, 0.0469059795, -8.33269267E-4, -0.00101196242, 0.0183872916, -0.0482047126, -1.48484498E-4, -0.0306689553, -0.105656289, -0.0378864259, 0.0625277609, -0.0542096384, 0.0885062739, 0.117056184, -0.101912171, -0.00717647559, 0.0219991449, -0.09169472, -0.0896932632, 0.050530497, -0.0779579281, -0.127960578, 0.125724763, 0.0429155156, -0.0443116874, 0.0304166432, 0.089088425, 0.110556535, -0.0250664465, 0.0322352089, 0.00697411736, -0.0940409377], [0.0413678177, -0.00106799859, -0.0124210687, -0.107938945, 0.0239770319, -0.124027565, 0.0624647699, -8.71039344E-4, -0.0258578323, 0.0161802452, -0.00377400103, -0.0549048707, -0.0606441684, -0.0232145656, -0.0770102814, -0.0260445271, 0.0492179543, -0.0233202688, -0.0431008711, -0.027294118, 0.0438194834, 0.077319473, -0.0126926368, 0.0384279229, 0.0471892916, -0.0501022972, 0.148133069, -0.101314716, 0.0682661906, 0.0538413897, -0.0418410338, 0.123816565, -0.0501480214, 0.0688003227, -8.562690e-02, 8.425810e-03, 0.00376066566, 0.0995107069, 0.116595052, -0.0875548124, 0.0577032864, 0.0820789933, 0.0871691778, -0.102016024, -0.0021641159, -0.072913155, -0.0828008205, 0.0457217172, 0.0121198697, -0.0321105346, -0.01850578, -0.0370441638, -0.0629003942, -0.129220679, 0.00262970338, 0.0316956863, 0.0113838464, 0.0502511263, -6.389620e-02, 0.11741101, 0.0742196515, -0.0268879645, 0.0311174635, 0.0198700279, -0.0544390269, -0.0436065942, -0.0303376578, -0.0770327821, 0.0185952503, 0.0224634968, 0.0105367443, -0.0762964934, -0.0350391753, -0.0628541409, -0.0294431597, 0.104778066, 0.110401362, 0.0646754354, 0.0383319221, 0.0581485182, -0.0213402212, 0.0655323938, -8.36965045E-5, -4.4498546E-4, 0.0803475156, 0.0508452579, 0.122168429, -0.0869448408, 0.00313522434, -0.0805589556, 0.0444265194, 0.0436881483, 0.0190724209, -0.0559015125, -0.0530406907, 0.0167950038, -0.0726822615, 0.0554049611, 0.0109543344, -0.0332693644, 0.07085713, -0.0197939109, 0.0210278854, 0.0502361283, 0.0397000462, 0.0867357403, 0.068426691, 0.00924178399, -0.0153731573, -0.0257548522, 0.0367379189, 0.0305116288, -0.0192742255, 0.0649406537, -0.0457418598, 0.0499264412, -0.0872386395, -0.155325234, -0.00959366094, 0.0986651703, 0.0673582926, 0.0667302161, 0.0387484431, 0.00645977398, 0.0791856646, -0.0378532819, 0.0304064881, -0.00537935272], [0.0726593956, -0.109636299, -0.0171054509, 0.100893036, 0.0517158844, -1.679380e-02, 0.0959655195, -0.0312820151, 0.094186455, 0.0431394316, 0.0957580656, 0.050512027, 0.0323804505, -0.0915264114, 0.0174490754, -0.0275826342, 0.0133191748, 0.0129112229, -0.0571741611, -0.0262447409, 0.0134314774, 0.0405846946, -0.0946906656, -0.0567256957, -0.0545835756, -0.0173643362, 0.0130217355, -0.0671513602, -0.0850489289, 0.0507134609, 0.0860699638, 0.0509441197, 0.0350071117, 0.00686328718, -0.0122577511, -0.0262410957, -0.0659489333, 0.150391251, 0.0469852276, -0.109944031, 0.0147082182, -0.0799905732, 0.122220077, -0.0809040963, 7.81528535E-4, 0.0177344661, 0.0806423873, -0.109507427, 0.103761032, -0.104130417, 0.131364882, -0.125180379, 0.0240230989, -0.0467110276, -0.0497380272, 0.0596187152, 0.0269008446, 0.00540336594, -0.0449752808, 0.122136787, 0.0332008377, -0.0220913831, 0.10238409, -9.122260e-02, -0.0632487386, 0.0344535559, -0.0284460187, -0.0164871868, 0.0500357933, -0.0390310027, 0.093694508, -0.0644888803, 0.0344535038, -0.02155024, 0.0712349414, -0.0242912304, -0.0292287525, -0.0283049438, 0.0270920657, 0.07145641, -8.326040e-02, 0.0597435124, 0.120645732, 0.0971314609, -0.00512188207, -0.0571747683, -0.0218769647, -0.00778229628, 0.056784872, 0.0279328655, -0.0953751131, 0.0197657458, -0.0698343664, -0.0386835076, 0.0114003802, 0.0297355987, -0.0604464449, 0.00354043441, 0.0415728725, -1.182950e-01, 0.0329783075, -0.025848873, -0.0485701784, 0.00874593388, -4.686920e-02, -0.0378263555, -0.0845283717, 0.0941171199, 0.0855405778, -0.0861824229, 0.0422401614, -0.0805772617, -0.0736359059, 0.132397398, -0.0408308357, 0.0252269059, 0.0212558452, -0.094698295, -0.081195578, 0.0738269836, 0.0295008048, 0.0170011632, -0.0111524565, -0.0351704024, 0.0131755257, -0.00370720681, 0.0729349181, -0.0663619861], [0.0528291538, -0.0621302388, 0.00490943342, -0.0353306048, 0.048725754, 0.0449313484, 0.0754858628, 0.0794184059, -0.0292142909, 0.0331212655, 0.12650083, 0.0453475453, -0.0959737152, -0.0451631732, -0.0776118114, 0.00647341227, 0.0892526581, -0.0184987187, -0.103832878, -0.0653347074, -0.0621561221, 0.0259370077, -0.0557207838, 0.0530936271, -0.0279064979, -0.126381293, 0.0107962312, -0.0591896772, 0.0617250577, 0.042637378, -0.0455737561, 0.0153219458, -0.00355068408, 0.0382331312, 0.0105277793, -0.100428566, 0.03985928, 0.0510916486, -0.0653823167, -0.0438012779, -0.0274889469, -0.0379295163, 0.0173717961, -0.115120046, 0.0201087836, 0.0541042648, 0.0220524054, -0.00322354049, -0.0448590219, -4.721610e-02, -0.00741655892, -0.0386125185, -0.0801200047, -0.0384775028, 4.06594845E-5, 0.0414728038, 0.0314212069, -0.00618424313, -0.0472594611, 0.0528326258, 0.0577871613, -0.018218495, -0.00455614971, 0.0568224713, 0.0362921134, 0.0594609678, -0.0410446934, -0.0799038633, 0.0523039028, -0.0558001883, 0.0766099244, -0.113258466, 0.0542300642, -0.0459170528, -0.0226911046, 0.0647555515, 0.0455851816, 0.040726915, -0.0442872085, -3.74307943E-4, -0.0343361907, 0.0415010564, 0.0344555825, 0.0474076159, 0.0697310716, 0.0824200734, -0.0653427094, -0.0730542466, 0.0472382158, -0.0355650522, -0.0468364842, 0.045336552, -0.00254660845, 0.0038221362, -0.0362731293, 0.0483396687, -0.0725800246, 0.0936472415, -0.0053933817, -0.0228049215, 0.00923193526, 0.0306652021, -0.109007023, 0.0230882708, 0.00934829284, 9.84772341E-4, -0.102744408, 0.0608119406, -0.0136528052, 0.0473858453, -0.106519632, -0.020187147, 0.0170795228, 0.040681608, 0.0109285349, -0.0467834063, 0.0787448957, -0.0889639556, -0.0900531113, -0.0283997618, 0.0621445365, -0.0142332567, 0.0619890615, -0.0716743767, 0.0343630239, 0.0197837856, 0.121950306, -0.0875614359], [0.0580146536, -0.00590000767, 0.0185996313, -0.0132879922, 0.0347123705, -0.0153813604, 0.0507178754, -0.0575577542, -0.050430119, 6.459340e-02, 0.050744772, -0.0396937728, -0.0112975193, -0.0478631295, -0.0168083459, -0.0430378206, 0.0468383916, 0.0800007433, -0.0712827444, -9.560330e-02, 0.0469598696, 0.0768855661, -0.0432603024, -0.0490635969, 0.0642256364, -0.0611926764, -0.0127678048, 0.0792653858, 0.0400444083, -0.0267013945, 0.0603269972, 0.0457310528, -0.0693433284, 0.109421007, 0.0427802503, 0.00185747084, 0.0749032497, 0.0122566754, 0.036766652, -0.0297776889, 0.0672710165, -0.0667825639, 0.0611506887, -0.0527236164, 0.0252125636, -0.0318560265, 0.0501693301, 0.038012825, -0.0718893185, -0.0627787858, -0.0689582154, 0.00657312572, 0.00543634268, 0.0340779647, -0.0560605451, 0.0311792027, 0.0037154716, 0.002942811, -2.48226661E-5, -0.0217478294, 0.0485796928, 0.0171486959, 0.113153957, 0.0167153589, -0.0301615298, 0.0825883969, -2.17365377E-5, 0.0591070503, 0.0551925413, 0.0884894728, 0.0249009561, -0.0734193549, 0.0811320096, -5.038560e-03, -0.0393329598, -0.0420833714, -0.0146055799, -0.0361671299, -0.0053188582, -0.0251984783, 0.0418762341, 0.0113309929, -7.20117241E-4, 0.085474357, 0.0639755651, -0.0179237258, -0.0293983184, 0.0627599731, -0.0212417115, 0.0809199139, -0.0635955185, -0.0311255101, 0.0701849833, 0.0294922441, 0.0940327644, 0.0504427776, 6.754170e-02, 0.076335147, 0.0491565093, 0.042807579, 0.0255045332, 0.0460851751, -0.0807569697, -0.00476056524, 0.100024834, -0.0598278642, -0.0867116376, 0.0927661433, -0.00296293013, -0.00282826857, -0.0310810432, -3.369380e-02, -0.030938074, 0.103786558, 0.0581109188, -0.0743000358, -0.0642076358, 0.0634192601, -0.00521074515, -0.0197472535, 0.0563652478, -0.0546586253, 0.044771079, -0.0918999537, 4.911300e-02, -0.0420751646, 0.0657438487, 0.0749151856], [-0.0705946088, 0.00911072921, 0.0629237145, 0.072066158, -0.0119734658, 0.0418844149, 0.0658187717, -0.0409245528, -0.0686642751, 0.0136363879, 0.0857882201, 0.0389808081, -0.00504631083, 0.0374473594, 0.0259321555, -0.0152868964, 0.0460462086, -0.0150655815, -0.0295887403, -0.0132794296, 0.0622637607, 0.0307737086, -0.0410955586, 1.34989066E-4, 0.00313203456, -0.0227282979, 0.00264105387, -4.164030e-02, 0.0569824502, -0.0569683835, -0.069287844, 4.291890e-02, -0.057430096, 0.0638193712, 0.0250261016, -0.108201049, 0.113507509, 0.0165049098, 0.0478918776, 0.00643375097, 0.0534112044, 0.00705494499, -0.052116096, 0.116492465, -0.0199472066, 0.0918952599, 0.0116317188, 0.0989835709, 0.0263814237, 0.031479191, 0.0489896908, -0.101671807, 0.046253372, 0.0132944947, 0.0733616725, 0.080582492, -0.072624974, 0.073190175, 0.0507003628, 0.0752427279, -0.0351122804, -0.0109369084, 0.0658781826, -0.0390484855, -0.0646180734, 0.105009168, -0.00570479548, -0.0536969528, 0.0772042796, 5.507770e-02, 0.00428153295, -0.0246956088, -0.0308523756, -0.00969725102, 0.0259635188, -0.0159644131, -0.0602734088, 0.0263046622, 0.0757883787, -0.0110259345, 0.0467506386, -0.0311567187, 0.0376159325, -0.00317404536, 4.367150e-02, -0.0455694571, -6.851870e-02, -0.0214030501, -0.0168406814, 0.0686305463, 0.0222968198, 0.0989348888, 0.124253228, 0.0375863463, 0.0817499682, 0.0579698086, -0.058745753, -0.0136964228, -0.0187111683, -0.0529981591, 0.0266373176, 0.0921661928, -0.071084775, 0.00951978099, 0.058638528, -0.0033548621, -0.0215971768, -0.0138034578, -0.0351475365, -2.376620e-02, 0.00478263712, -0.0590051748, 0.0243003275, -0.0266074017, -0.0180362798, -0.091835089, 0.0268700607, 0.0693318918, 0.0640044436, 0.00191756978, 0.0398371592, 0.0379900485, -0.0356064737, -0.0396249853, -0.0160015412, -0.0884755775, 0.0589615777, -0.0638743639], [-0.0179905277, -0.0556276701, -0.0620662123, 0.0812390372, -0.0641834661, 0.0523002595, 0.0813947394, -0.0681132898, -0.0339610241, 0.0563197285, -0.0219975747, 0.0211276971, -0.063586235, -0.070529595, -0.0634025857, 0.0620451644, 0.0696660206, 0.00130963023, 0.0233735647, -0.0672425106, 0.0568633229, 0.0656362846, 0.0425954573, 0.0366131179, 0.0621407777, -0.0312650353, 0.0293485206, 0.0410729088, 0.003445426, -0.028417645, 0.0646098629, -0.0390127115, -0.0723708645, -0.0315480419, -0.0579443797, 0.0227321312, 0.0836483538, -0.0517440885, -1.025390e-02, 0.0535018966, 0.0100701563, -0.0733730644, 0.0779750272, -0.0278811082, -0.0490307063, 0.0300165974, -0.063927345, -0.0400326848, 0.0216985457, 0.045041889, -0.0501246378, -0.0593550354, -0.0581761673, -0.030954048, 0.105035864, -0.0322775096, -0.0125984773, -0.0506424867, -1.327970e-02, -0.0398252867, -0.0605501197, 0.0398114547, 0.024164116, 0.0806541964, -0.0349018686, 0.0936509966, -0.0269376412, 0.0378184132, -0.0492458344, -0.0274435319, 0.0400712341, -0.00119548629, 0.0503328145, 0.0332569405, -3.645210e-02, -0.0367481411, -0.0829051956, -0.0862335786, 0.0371168442, 0.0136729479, 0.0217591804, 0.00306910323, 0.0657242238, 0.0391786061, 0.0146526573, 0.0266645458, 0.0283208899, 0.0224403441, 0.069181338, -0.0142711224, 0.0449293479, 0.096463181, 0.0547019318, -0.0471979305, 0.0697151944, -0.0142988525, 0.0601926968, -0.0173506644, -0.0340025909, 0.0254931916, 0.00907093287, 0.0711060166, -0.0796542317, 0.0502987765, -0.0812444761, -0.0141209774, 0.0349733867, 0.0926860645, -0.0385963283, -0.0312617458, 0.00986803509, -0.0157660935, -0.0694665909, -0.0125073716, 0.00113248162, 0.0498555973, -0.0097701326, 0.0264565013, 0.0057779653, 0.0483768582, 0.0712159649, -0.0163257197, -0.0623787827, 0.050613407, 0.0261875689, -0.0173167884, 0.118650027, 0.0741025805], [-0.0547565892, 0.0426943861, 0.0243173577, 0.0564482622, 0.00202704663, -0.100007758, -0.0111208893, -0.0500234105, -0.0445720591, -0.0260474142, 0.0829658806, 0.0753189102, -0.0260566846, 0.0445198305, 0.0183026027, -0.0248292834, -0.0431853384, 0.0426756367, 0.0189391207, -0.0118246963, -0.0149580836, 0.0644006059, 0.0418589599, -0.0352211297, 0.0298082363, -0.0574172586, 0.0688783228, -0.0140275648, -0.0737404153, 0.0298322719, -0.0257150196, -0.0334839597, -0.103147238, 0.0905340313, -0.0453409702, -0.0221160185, 0.0109420707, 0.0775819793, 0.0277274698, -0.0342461057, -0.0156465527, -0.00525613874, 0.0330014229, -0.0706589519, -0.042355869, 0.0159737132, 0.0895646587, -0.019865213, -0.0213508978, -0.0179154556, -0.0388953239, -0.0656956583, -0.0630206689, 0.0432193354, -0.0453802198, 0.0935373231, -0.0635981336, 0.0293822344, -0.0733549222, 0.0641619265, -0.0384148173, 0.0598316714, 0.0321740471, -0.0656159744, 0.00873971079, -0.0294659883, 0.0374926068, 0.0161052402, -0.0408646315, -0.00679968158, -0.0843386203, -0.0285247322, -0.0758852214, 0.0488007888, -0.0625964552, 0.0388682336, -3.417010e-02, 0.0261888597, 0.0926033481, 0.0145863062, -0.00996436178, 0.0412421897, 6.316660e-02, 0.0391972475, 0.0242375154, 0.099839963, -0.010996392, 0.021794416, 3.73655173E-4, -0.00498437556, 0.0343063325, 0.0168952327, 0.0203715023, 0.0551783517, 0.083603233, -0.0475374088, -0.0115492595, 0.0638083667, -0.0340639837, 0.0998149439, 0.0859246105, 0.102564834, 0.0451717973, 0.028835861, -0.0760115534, 0.00630946643, -0.0413454846, 0.0844345763, -5.647560e-03, -0.0203627851, -0.0112780295, -0.0335119925, -3.248600e-02, -0.0485765673, 0.10895405, -0.0160869285, 0.0374014713, 0.10038124, -0.0366812646, 0.0191006381, 0.00513708638, 0.0775778666, -0.0936144143, 0.0232353769, -0.00355488085, -0.0306236167, 0.0459044278, 0.00328912656], [-0.0920852348, -0.00505957799, -0.0173177905, 0.101621427, -0.0554346368, 0.00692678056, 0.106070198, 6.18607854E-4, 0.0906152129, 0.0648702085, -0.0494996496, 6.516730e-02, 0.0536487065, -0.0101307007, 0.0263641179, -0.0502096601, 0.0302848332, 0.0500296392, 0.0165588614, 0.0790142416, 0.00541379722, 0.0366778262, 0.0391760357, 0.0232681893, 0.0971666425, -0.00465568807, 0.0386813357, -0.019524429, 0.0057457434, -0.0354042575, -0.0420495421, 0.042845685, -0.0307979286, -0.0464176089, -6.943760e-02, -0.0281514507, 0.114052758, -0.0260536708, -0.0053365482, 0.0104633635, 0.0560078099, -0.00937141199, 0.0823922827, -0.00848927162, 0.0210025571, 0.0486722253, 0.00923265796, 0.0655684397, -0.00584589364, -0.0174135752, 0.00349490158, 0.0591177195, 0.0278483238, 0.056301821, 0.0708580911, -7.918240e-03, -0.0932198688, -0.053909339, -0.0270045586, -0.0604044348, 0.00258845021, 0.0694492757, -0.0204909761, -0.0217291452, -4.317620e-02, 0.106242992, -0.00147824746, -0.0259237047, -0.0372320302, 0.0264276043, 0.0255635269, -0.0423591211, -0.0710966289, -0.0916588157, 0.0150900353, -0.0437604971, -0.0982500165, -0.0117858835, 0.0161039755, -0.0341005176, 0.0204637125, -0.0426365547, 0.00213722093, 0.00572103122, 0.0274487771, -0.00165989809, 0.0553967282, -0.0686191246, 0.0611164569, -0.0626416057, 0.0319535881, 0.0126312226, 0.120805614, -0.0562978797, -0.0464641713, 0.00596352667, -0.0302280094, 0.0505443402, 0.0640001819, 0.0670263097, 0.00304577174, 0.00935027841, -0.0678053424, 0.012334167, 0.0012996908, -0.0204793327, -0.0264617316, 0.0695782825, -0.057383351, -0.0414311104, -0.0515577458, -0.00224952819, -0.00743298093, -0.0835020542, 0.0831834301, -0.00133081619, -0.0791722908, -0.0046657729, 0.064428255, 0.0709320158, 0.0590747036, 0.0867937654, -7.440230e-02, 0.0423133671, 0.0287619699, 0.0470734537, 0.0465647765, -0.0224081576], [0.0305894576, -0.0903786122, 0.0084464578, -0.0430438109, -0.0174284391, 0.0480015837, 0.0859967842, -0.0890045911, -0.0445817895, -0.0251360163, -0.0931139886, -0.0230186787, -0.0550952032, -0.094998464, 0.00954110455, 0.00471791811, -0.031772837, -0.0620099864, -0.0811770334, 0.0846370906, 0.0283051245, -0.0740819499, 0.0834784805, 0.0253080912, 0.0288367663, 0.0390091427, 0.0823402181, -0.00887502823, -0.0218907222, -0.0413616709, 0.0357360542, 0.0790555849, -0.0109574953, 0.0649966225, -0.0138601428, 0.0594511554, -0.00994955841, 0.0323923565, 0.0722472965, 0.0794304981, 0.0202930141, -0.061253462, 0.0961271896, -0.0251493827, 0.082758531, -0.0635134503, 0.0800891891, 0.00145783764, -0.0305945724, 0.0389943942, -0.0639107078, -0.0755643248, 0.0881419629, 0.06342756, -0.0762729421, -0.00233601034, 0.0500275306, 0.00654817652, 0.00777094624, 5.806700e-02, -0.0734407455, 0.0180662069, 0.0408236682, -0.0785281956, -0.0685633122, 0.108126506, 0.0419205762, -5.096210e-02, -0.0551503375, -0.00750313885, -0.0301549174, 0.0705291182, -0.0762916282, 0.0250686407, -0.0503889509, 0.0116492612, 0.035893444, -0.0354880802, -0.0654857456, 0.0521947853, 0.011038431, -0.0724265277, 0.0103826104, 0.00710319262, 0.0363541171, -0.00938857533, -0.0253794584, 0.0134711275, -0.021690309, -0.00375914807, -2.89124509E-5, -0.0319548957, 0.0940068587, -0.073046878, -0.0376669951, -0.0699078664, -0.0382635817, 0.032075122, 0.0772702247, 0.0234823246, 0.0395211205, -0.0498369224, -0.0197858326, -0.0625883341, -0.0734345093, -0.0295345243, 0.0319100134, 0.0243958887, -0.104111321, -0.0384243689, -0.0550413243, 0.0575214773, -0.00598161202, -0.0989609211, -0.00136305625, -0.0391749516, -0.0688822791, 0.104872227, -0.0125726564, 0.0701192319, -0.0225292686, 0.083135955, -0.0137360897, -0.0632152632, 0.0786931068, -0.0102431308, 0.0466804765, 0.103191473], [-0.0564115606, 0.0595045909, 0.0381559134, -0.0603334606, 0.00285815145, -0.0435004421, -0.0138019854, -0.10533531, 0.100434825, -0.04601641, -0.0768093914, -0.0462434813, 0.051077038, 0.00345818559, 0.0669890195, -0.0337758474, -0.0502861887, 0.070896253, -0.0631744191, 0.0213943291, -0.0535485446, 0.0501425527, -0.0451763421, -0.0326203406, 0.0275728256, -0.0358301811, 0.0660500154, -0.0289354101, -0.0261569489, 0.0425189324, 0.00650096219, 0.0618515052, -0.0130126113, -0.0565196276, 0.0390102826, 0.0893672183, 0.0034423396, -0.0767043829, -0.0580974221, 0.0719446167, -0.0344436243, 0.0117032398, 0.0324496031, -0.0371435806, 0.0122358315, -0.0664953738, 0.0836238265, -0.0158912744, 0.0355154127, 0.0249724481, -0.00465053786, -0.08816576, -0.0185393151, 0.0174223576, -0.0154300295, -0.0235878695, -0.0280704442, -0.0158690587, -0.0392470844, 0.00675046816, 0.0193715915, -0.0470001101, -0.0900284945, -0.0902560576, -0.023080796, 0.0162779875, 8.724830e-02, -0.0231132768, 0.0410322621, 0.00568488659, 0.0174758211, 0.0884310454, 0.0128779858, 0.00713725062, 0.0788387358, 0.0445856154, -0.0093362676, 0.0800412595, -0.0657193437, 0.00523787783, 0.114607878, -0.0269976258, 0.0018337993, 0.0294157062, -0.0859724283, 0.0123507362, -0.0347649194, 0.0472331233, -0.0105657754, 0.0229443032, -0.0768112838, -0.139135569, 0.0853436887, -0.049383264, 0.0344778635, -0.0395253077, -0.00250488683, 0.120791286, 0.0394936278, -0.0884790644, 0.0508579575, 6.908810e-02, -0.0141858654, 0.036516577, -0.100869782, 0.0114269694, -0.0676628724, 0.0308145881, -0.0615448765, 0.0095664151, 0.0241020098, 0.00608672201, -0.0900903195, 0.0245011784, -0.0467917584, 0.0110060582, -0.0655733421, 0.0607696772, 0.0443786606, 0.0191152059, -2.12470361E-4, 0.031003857, 0.00235764799, 0.0707575753, -0.0612313785, -0.0581117794, 0.0846911669, 5.193000e-02], [0.0566665903, -0.00780039839, -0.042977944, -0.0771646797, -0.0176063254, -0.0704866871, 0.0301188808, -0.118452854, -0.00383295491, 0.0409418456, 0.0463184826, 0.0454763137, 0.0711259246, -0.00297933328, -0.0552821308, -0.0486850925, -0.0590534434, 0.0706002787, 0.0558836237, -0.0355696306, 9.305260e-03, -0.0783460959, -0.0488189682, -0.0760558695, 0.0398756452, -9.202990e-03, -0.0160361119, -0.0302052926, -4.392320e-02, 0.0825826525, -7.524150e-02, -0.0418618359, -0.0559740029, -0.0672244281, 0.0334703252, 0.0420979597, -0.0458557829, 0.0467012972, 0.0736205279, 0.0856612399, 0.0594354421, -0.0383173563, 0.0878846049, 0.00652077468, 0.00856618676, 6.015050e-04, 0.0283556599, 0.0641686916, 0.0989190861, 0.0619440786, 0.0797065645, -0.0167537965, -0.0190999862, 0.0500498787, -0.0512336902, -0.0300176349, -0.0302236695, -0.00446628546, -0.01799196, 0.0157596767, -0.0153486896, 0.0739928931, 0.0609212555, -0.0256988797, -0.0874429047, 0.055347193, -0.00273437379, -0.0309106801, 0.0782976746, 0.0953797549, -0.112494864, -0.0434531458, -0.0772535204, 0.0054260809, -0.0253762342, 0.0859701335, 0.0474562198, 0.0387948379, -0.046501372, 0.00642030174, -0.0167522915, -0.0476851799, 0.0172968805, 0.0294684805, -0.0328020379, -0.0203386471, -0.054637868, 0.0755467936, -0.0307054389, 0.0276107546, -0.0664257705, 0.0178609658, 0.0436469689, -0.0285307076, -0.00583771057, 0.0174653437, 0.00875341799, 0.00930203869, -0.0291128214, -0.0106747681, -0.0380928852, -0.0452134572, -0.0655148178, 0.0435267948, -0.0352962203, -0.0378277339, -0.0439496338, -0.0784894153, 0.0563799217, -0.00848808419, -7.306970e-02, 0.0409447588, -0.0176672544, -0.0918324515, -0.0641430914, 0.126910433, -0.0758153498, 0.045866698, 0.0167182591, -0.0825744792, 0.0532303713, 0.111841798, 0.0385534465, 5.90801821E-4, -6.915350e-02, 0.0231199581, 0.0358780548, -0.00230763806], [0.0461180508, -5.83786168E-4, 0.0454234034, 0.0518282913, 0.0494241379, -0.0276714787, 0.00175970758, 3.386060e-02, -0.0176178031, -0.00552454544, 0.0522501059, -0.0665111541, 0.0393869467, 0.0240719058, 0.0303146876, -0.0438044854, 0.0646786094, 5.36327192E-4, 6.622680e-02, -0.0375777557, 0.0706777424, 0.0215188954, 7.44389312E-4, -0.0688125491, -0.0157045517, 0.00118271552, -0.0280094668, 0.0438045114, 8.412280e-03, 0.0273443907, 0.0787199736, -0.0169765819, -0.0235516783, -0.0689353421, 0.0755436495, -0.0450841635, -0.0458512381, -0.0426025465, 0.0677269548, 0.00183862448, 0.0294339955, -0.0191190541, -0.0161834117, 0.062497843, -0.00343655539, -0.0336192437, -0.0683672875, -0.0668953434, 0.0521705784, -0.00726429652, 0.0329383165, -5.976880e-02, 0.0433274806, -0.0286796205, -0.0630799681, 0.011862888, -0.0766979083, 0.0438847393, 0.0744005218, -0.0582580976, -0.0463337749, 0.0574724451, 0.0445971563, 0.0590101592, -0.0565719046, 0.0300887879, -0.0108128386, 0.0360216871, 0.0430802926, 0.0521655306, 0.055251874, 0.0483576022, 0.0057908874, 0.0636810213, -0.047667332, 0.084229067, -0.0484995693, 0.0137396753, 0.0740412846, 0.0110309012, 0.067787379, -0.0292072464, 0.0543204471, -0.0696958899, -0.0323026888, 0.00888835918, -0.0513801165, 0.0407926217, -0.0389953479, -0.0665001348, -0.0259373076, -0.0406722873, -0.00675884401, -0.0598653667, -0.0594234839, -0.0289045349, 0.0254431721, 0.0767826661, 0.0393534191, 0.0412998609, 0.0284348764, -0.00697658769, -0.0798544213, -0.0666929707, -0.040747948, -0.0141881565, -0.030018948, 0.0330563746, -0.0789232552, 0.0178106111, 0.0696328804, -0.0484639704, -0.0444481894, -0.00944758113, -0.0615413301, 0.0113638304, 6.999050e-02, -0.0530919917, -0.0136830118, 0.00365191256, 0.0116453888, 0.0963912084, -0.0340083428, 0.0612848699, -0.0600473545, 0.0728257075, 0.0337315425, 0.0119259469], [0.0487809144, -0.0240041427, -0.0297255125, -0.0620332174, 0.0547013804, 0.0378393568, 0.0748624802, -0.0111912936, 0.0708074942, 0.0429996401, 0.00143774843, 0.00691025611, -0.0580494776, -0.066426225, 0.0251029227, 0.0130972089, -0.00788842794, -0.0308803748, -0.0642518699, 0.0612525716, 0.0584403761, 0.0480550043, -0.0398030952, 0.0123179648, -0.0118512437, -0.0629910231, 0.00844482053, 0.0115678851, -0.0193698984, 0.0683358609, -0.0727113187, 0.0446118414, -0.0508956686, 0.0556251369, 0.0720811114, -0.0716340765, 0.0627536773, 0.0570937879, -0.0729787647, 0.0268816631, -0.0757487863, 0.0245777853, 0.0575227253, 0.00703438092, 1.278090e-02, -0.0361163281, 0.0383483469, -0.00726790074, 0.0778149887, -0.00492575532, 0.00251559564, 0.0562884547, -0.0550753102, 0.0591388121, -0.0668371767, 0.0259783883, 0.001024383, -0.0497541614, 0.060208749, -0.0482919551, 0.0294381604, 0.0592250228, 0.0135930115, -0.0395237394, -0.055748675, -0.00586092286, 0.00363291497, -0.0488290899, 0.0179442633, 0.0144011583, 0.00381920021, -0.0293029789, -0.0328729935, 0.0465171076, 0.0605445057, 0.0743549764, 0.0246652681, 0.0481499247, 0.0768258795, 0.0273419172, 0.0715475678, 4.768800e-02, 0.020962216, 0.00454461202, 0.0782792717, 0.0181580074, -0.0483055264, -0.0107953139, 0.0101942103, -0.018896414, -0.0286447424, -0.0762559101, 0.0492177568, -0.0212846044, -0.0172930546, -0.0812773257, 0.0719883665, 0.0575633459, -0.0207699779, -0.0770133883, -0.00705338595, -0.0309671368, -0.0360985175, -0.0471818373, -0.0193770509, 0.0118814213, 0.0424590968, -0.023471389, 0.0107086617, 0.0561267547, -0.0595210493, -0.0114795398, 0.058125522, -0.0145145468, 0.0281931143, 0.0128417294, 0.0699309111, -0.0792718679, -0.0449409671, -0.077707298, -0.0071626883, -0.0384172648, -0.0574798547, 0.0427187309, 0.0232701804, 0.0725227594, 0.0277282577, 0.074725993], [-0.0108149676, 0.0453147404, -0.0521359816, 0.073466368, -0.00794525538, -0.0247233324, -0.0480394028, -0.00353992195, -0.00874750409, -0.0708512813, -0.0570869818, 0.0265474468, -0.059487883, 0.0574061647, 0.075791195, -0.0276333764, 0.0298768915, 0.0549825132, -0.0702062249, -0.0122361425, -0.0679343864, -0.0719843954, 0.0416606888, -0.0231842957, -0.0334097929, -1.33699737E-4, -0.0286331493, 0.0320851244, 0.0789980143, 0.0293266308, 0.0160065815, 0.079622291, 0.0657689497, -0.0403794833, -0.0688459501, 0.0797886177, 0.00837501418, 0.040382769, -6.69095665E-4, 0.0114506744, 0.0137577141, -0.0551026426, 0.0797736644, 0.0122301346, -0.036614418, -0.0419312194, -5.660910e-02, 0.0741606578, 1.5941789E-4, 8.51778662E-4, -1.192950e-02, 0.0818247422, -0.0573427156, 0.0491802283, 0.0547268353, 0.0458782092, 0.025968723, 0.0720185563, 0.012257712, 0.0283122323, -0.00626103114, -0.0611625351, -0.0552195683, -0.0321546346, 0.0382500477, -0.00399955176, -0.0502150841, 0.0236125626, 0.0784884616, -0.0149168987, -0.0551133938, 0.0423839279, -2.30387741E-4, 0.0538399294, -0.00426099589, 0.0670286118, -0.0228403918, 0.0735999271, -0.0348343924, 0.0790346562, 0.0152404066, -0.0615369119, 0.0603516772, -0.0499371663, 0.0652790442, 0.0688554719, 0.0333609618, 0.0146630974, 0.0207416173, -0.0532656088, -0.0771927759, -0.0567950718, 0.0585227348, 0.0626469702, 0.0367890522, -0.0171052571, 0.0058201449, -0.0807013363, -0.00897748116, -0.0608690605, -0.00665525394, -0.017943386, -0.0669626072, 0.0628672689, -0.0228028744, 0.0126589173, 0.0701405107, -0.00350378314, -0.0633569657, -0.0272051804, -0.052160576, 0.0794056877, 0.0205360837, 0.0423374176, 0.0338830166, -0.0347006358, 0.0312976427, -0.00757420389, -0.059185531, -0.0376029164, -0.00120039785, 0.0510236919, 0.0179412384, 0.0086820852, -0.0149646774, -0.00120663934, -0.019000886, -0.0409488194], [-0.0783119574, 0.0617789775, -0.0799260511, -0.0546086878, -0.00527667347, -0.042489551, -0.00572584616, -0.0541987978, 0.0698108748, -0.0142512433, 0.0636167601, 0.0651088357, -0.0806601271, -0.0269422643, -0.0644527897, -0.00548497774, 0.0228345096, 0.0749994591, 0.00394716579, 0.0661033243, 0.0649119392, -0.0748860836, 0.0751190335, -0.0231429022, -0.00555884186, 0.00729660597, -0.0217111502, 0.0635587722, -0.0309806671, -0.0098000206, 6.148800e-02, -0.0289014746, -0.0673002228, -6.921060e-02, 0.0646202862, 0.0637754499, 0.0579933338, 0.0700555518, -0.045223292, 0.0789926499, 0.0312057193, 0.0345239565, 0.0368068963, 0.0484073795, -0.0274349749, 0.0455473177, -0.0656067505, 0.0311609209, 0.0260717981, -0.0614437908, -0.0143051576, -0.0412164442, 0.0138105173, 0.0788072199, -0.00971366931, -0.0568227805, -0.0540556312, -0.0664744377, 5.623340e-02, -0.0105705298, 0.0501739942, -0.0347640067, -0.0417896211, 0.030467106, -7.909590e-03, -0.0329537876, -0.0382944085, -0.0238797795, 0.0329948179, -0.0573302917, 0.0314550474, -0.0193367135, 0.0621331781, 0.0130296694, -0.0739309192, -0.0241289381, 0.03665163, -0.0491649956, -0.066963546, 0.0770840123, -0.0102466792, 0.0423353538, 0.0772872344, 0.0512482822, -0.03261742, -0.0306638759, -0.0498576798, -0.0227286741, 0.0254143588, 0.0464788675, -0.0530864298, 0.0716601834, -0.0648695528, -0.0407344624, -0.0363822877, 0.0517659821, -0.0112175606, -0.0603830665, -0.0137460213, 0.00243111653, -0.033523839, -0.0793025493, 0.0432503037, 0.00376340724, -0.0403116867, -0.00892502255, -0.0134297404, 0.00453406619, 0.0181615725, 0.0389246307, 0.00801867712, 0.0736603364, -0.0595041551, -0.0270896573, 0.0155702876, 0.021441536, -0.00327730831, 0.0687330514, 0.0799285396, -0.0319750942, 0.0461966135, 0.0283379368, -0.0112136006, -0.00872354395, -0.0665727928, -0.0553038791, 0.0188245457, -0.0586716607], [0.0741421655, -0.0467608795, -0.0556068085, 0.0746210963, 0.0298468918, -0.00743688224, -0.0310390815, -0.0106021995, 0.020952493, 0.0191093758, -0.0245037246, 0.0577332415, -0.0111209527, -0.0296535958, 0.00255419477, 0.0592296273, -0.00948394834, 0.0243828669, 0.078243114, 0.0542971455, 0.00118498248, -0.0259501822, -0.0198329482, -0.0777574703, -0.0475352518, -0.00208381563, 0.0543209277, 0.0526538044, -0.0207107402, 0.00717852917, 0.052009888, 0.0648675561, -0.0119532691, 0.0547845922, -9.12055373E-4, -0.0706860721, 0.0223828387, 0.0610998534, 0.014359735, -0.0480333976, 0.0690007657, -0.0689140782, -0.0433589667, -0.0699746162, -0.00124694081, -0.0430291072, -0.0704742894, 0.0497617275, 0.0605275332, 0.00139320642, 0.0214933548, -0.0207747966, -7.602020e-02, 0.0294549316, -0.0578284152, -0.0750941485, -0.0580674633, -0.0367966071, 0.0499326065, -0.00705584418, -0.0389446728, 0.0385883749, 0.0502853766, -0.0357577614, -8.864180e-04, 0.0801235437, 0.0210945942, -0.0661304668, -0.00371973962, -0.0401749723, -0.043116603, -0.0478980877, -0.0642313287, 0.02055992, 0.014570944, -0.0678180605, -4.12697555E-4, -0.039317891, 0.0713346228, -0.0661176294, 0.0390614606, -0.0631524175, 0.044552207, 0.00125017029, -0.0178314112, 0.0807151645, -0.0466569923, 0.0694537908, -0.0224622153, 0.0774028673, -0.0738927349, -0.0524598323, -0.0396612026, 0.0391840786, 0.0123083228, 0.0593262427, -0.069143325, 9.21596511E-5, 0.0131547041, -0.0586078092, -0.00148740388, -0.0418047085, 0.0193795767, 0.0502323881, 0.0728516504, -0.00243035774, -0.0227449927, 0.0394506902, 0.0599530712, -0.0317599289, -0.0170762576, 0.043168962, -0.0275470018, -0.0112545192, 0.0103952438, 0.0580182821, 0.0705299452, 0.0099835284, -0.031290818, 0.0570810623, 0.0125014866, 0.0546851344, -0.0702498481, 0.0102575794, -0.03124086, -0.0337125324, 3.859750e-02, 0.059214931], [0.0324795842, -0.0022162532, 0.0606471039, -0.0311340988, 6.033580e-02, -0.0550936945, -0.0238130912, -0.0515970141, 0.0429979451, 0.0786769613, 0.0253573321, -0.0529958904, -0.0244781598, 0.0755707249, 0.0101380395, 0.0173030477, 0.0583003573, -0.0105031785, 0.00867583975, 0.0189683661, 0.0294535831, 0.0548171774, 0.0790944248, -0.0637060105, -0.0314106904, -0.0287410505, 0.0440000445, 0.0144531885, -0.0404930785, 3.59899859E-4, -0.0287264399, 0.0630602762, -0.00161388807, 0.0704937354, -0.049847357, -0.0375905409, 0.0260283817, 0.0277513377, 0.0477297194, 0.049389232, 0.00961390137, 0.0601162687, -0.0211380236, -0.0278126188, 0.0217597131, -0.00254721753, 0.0701357052, -0.0708311424, -0.0572343431, 6.599200e-02, -0.0129352082, 0.0800731331, -0.00258474518, 0.0761692673, 0.0195716843, 0.076308921, -0.00749191688, -0.00469639665, 0.00627459493, 0.0278876685, 0.0689548627, -0.0317056663, -0.0228323154, -0.0588465482, -0.00587640144, -0.022801552, 0.00656763138, -0.0360766128, -0.0183176678, -0.00442489842, 5.415220e-02, -0.0435926765, -0.0687617213, 0.0511005037, -0.00174839026, -0.0769696087, -0.0363147072, -0.0133521138, -0.0205721054, -0.0677984357, 2.31281228E-4, 5.746380e-02, -0.0723312721, 0.0804013982, 0.047168009, -0.0739476457, -0.0483633056, 0.0574744791, -0.0325143933, 0.0737037882, -0.0587938875, -0.0584849231, -0.0476825312, 0.030473398, 1.49947155E-4, 0.024184173, -0.0258334056, -0.0098420009, -0.00552156474, 0.00272412575, -0.078618735, -0.00553685846, 0.0228863284, -0.0221392326, -0.00386916404, 0.0736639574, 0.0691662878, -0.0773881674, -0.0279616527, 0.00775957899, -0.0122921448, -0.0662163869, 0.0702737048, -0.0588151477, 0.0286583174, 0.0395313464, 0.0127452584, 0.00988359656, -0.0134092877, 0.0375410505, -0.0472094677, 0.0207565855, 0.0573191531, 0.0243392251, -0.0630067959, -0.0122775957, 0.0649736151, 0.0656233877], [0.03904723, 0.0277702399, -0.0536376201, 0.0344755948, -0.068094328, -0.0435492061, -0.00111578021, -0.015600781, -0.0663249567, 0.0582443066, -0.00846611708, 0.0596659482, 0.074386023, -0.0237804577, 0.0670921802, -0.0444472395, 0.0363274701, -0.0291978773, -0.0167855769, 0.073387295, -0.0194281153, -0.0621762201, -0.0154079357, -0.0054319473, 0.065948531, -7.252760e-02, 0.0348603427, -0.0231158081, -0.0420757383, 0.0627438352, -0.058098726, 0.0491006523, -0.0341998301, 0.0223671477, -0.0282201916, -0.0827342942, 0.0755468607, 0.0112347314, -0.0495179221, -0.0712668597, 6.911100e-02, -0.049985677, 0.0387401134, -0.0474643596, 0.0278364178, 0.00157285994, -0.0340886489, 0.0216121059, 0.0748060047, -0.0700201616, -0.0177756585, 0.0600231253, 0.0317873396, -0.0550047718, 0.0047957357, -0.0276074801, 0.0434493572, -0.00406235643, -0.0163356327, 0.0716217905, 0.0157986023, -0.0304351039, 0.0140559021, 0.0119677978, -0.0173117258, 2.110880e-02, -0.00477189198, 0.0771792904, -0.0249972399, 8.115010e-02, -0.0759432911, 0.0340785161, 0.0489608794, 0.00844735373, 0.0420503095, 0.0456984639, -0.0280333105, -7.12315144E-4, 0.0198524352, -0.00841073226, 7.619140e-02, -0.00732562644, -0.0619209893, 0.0297806151, -0.0672310814, -0.0536591783, 0.0423467718, 0.0231897216, -0.0214454345, 0.0507480279, -0.0787285342, 0.0382176414, 0.0424696803, -0.0266997498, -0.0659692213, 0.0684195161, -0.00763097871, 0.0277155787, -0.0365655124, -0.0601756088, -0.00373703428, 0.038575504, 0.0754896849, 6.37440069E-4, -0.0290226657, 0.072604157, -0.0414359607, -0.0768062249, 0.0683068633, 0.0721733198, -0.00243662787, 0.0479323603, 0.0728324801, 0.00946977734, -0.0733694583, -0.0672684163, 0.0486291535, -0.0259872526, 0.0086869644, -0.0377128571, -0.00709396275, -0.0727511719, -0.0309170894, 0.0408915915, -0.0454925187, -0.00378612569, -0.0404750369, -0.0665500313], [0.0423210524, -0.0101573179, -0.0180762857, 0.045677539, 0.0233333558, 0.0639204234, -0.0539875068, -0.0735017285, -0.0734187812, -0.0770178661, -0.032913141, -0.079516977, -0.040423695, 0.0617370903, -0.0138305696, -0.0705229566, 0.00920352898, 0.0049898061, 0.0577050559, 0.0441230126, -0.0501197092, 0.0659188777, -0.0402416624, 7.521570e-02, 4.66418802E-4, -0.0288206544, 0.0248206332, 0.0569857098, 0.00851373281, -0.0307798162, 0.0610572286, -0.0140543832, 0.0211728141, -0.0296849571, -0.0650069565, -0.043489784, -0.01423572, 0.0826281979, 0.0400058143, -5.396420e-02, -0.0516962707, -0.0111406883, 6.559340e-03, -0.0117192939, -0.0173293743, 0.0770304576, -0.0250548832, -0.0294086598, 0.0739765093, -0.0582965948, -0.0417743362, 0.0217009243, -0.0117897866, -0.025117062, 0.0644271746, 0.0694535896, 0.00475028297, -0.00546908891, 0.0543068647, -0.0229595136, -0.0604153872, 0.0225224402, 0.0209144335, -0.0649655685, 0.0623123757, 0.0076224925, 0.0219602194, -2.223000e-02, 0.00759328901, -0.0628861486, -0.0636557415, -0.0484369881, 0.0697343871, 0.0254215784, 0.00438150903, -0.00105045212, -0.0596433878, 0.0069147204, -0.0733515471, -0.0199762732, -8.058840e-03, -0.0803009942, -0.067056261, -0.0419950485, -0.0771191046, -0.00357604446, 0.0195537359, 0.0313604251, -0.026260227, -0.00782478228, -0.0281346589, 0.0135547798, 0.0533592068, 0.00933138094, 0.0616698451, -0.0467156209, 0.0696356893, -0.0765310227, 0.0269061718, 0.0391469263, 0.0129954312, 0.0312542282, -8.248570e-03, -0.0632864162, 0.0591412447, -0.00277764094, -0.0473845787, -0.0389578491, -0.01949526, -0.00563620823, 0.0414108075, 0.018838644, -0.0624135695, 0.0230217576, -0.0634242222, -0.0453403555, -0.0209089331, 0.0609882735, -0.0329245105, 0.0128214322, -0.0700343102, -0.0466412269, 0.00348559674, -0.0525378734, 0.0072127129, 0.0306889974, 0.0347906575, -0.0506797284], [0.0397622362, -0.035982348, -0.048168391, 0.0540341064, -0.00185336336, 0.0292772818, 3.289430e-02, -0.0815115347, -0.0582856089, -0.0663459674, 0.0641241148, -0.00683336146, 0.046799548, -0.00233946461, -9.855810e-03, -6.350430e-02, 0.0630192906, 0.0423409976, -0.0380063057, 0.0435628034, 0.0259925649, 0.0316886045, -0.0563545674, -0.042207852, -0.0486437082, -7.624640e-02, 0.0201606508, 4.242090e-02, -0.0704475492, -0.0628780127, -0.0677030086, -0.0305338521, 0.0548633747, -0.0712673739, 4.827010e-03, -0.0210904051, -6.611790e-02, 0.0865605995, 0.060273502, 0.0495425761, 0.0105104093, -0.0492126606, 0.0110028153, 0.0382649973, 0.0553976595, -0.0610619411, -0.0602732189, -0.00433424814, -0.0824668332, -0.0420926847, 0.0755357817, -0.0785225331, 0.0182164591, -0.00321646291, -0.0194887761, -0.00209993613, 0.0745174661, -0.0379076302, -0.0402594469, -0.0421651639, 0.00292568863, 0.0381397679, 0.0544497594, 0.036005605, -0.0181997307, -6.532480e-02, -6.580460e-02, 0.0682464391, 0.0715956389, 0.0282656159, 0.0252903309, -0.0374834165, -0.0445845649, 0.0423493981, -0.0615618937, 0.00121695024, 0.025837874, -0.00997344125, 0.032358516, 0.0390478224, 0.0161990095, -0.0805294513, 0.0201471839, -0.00912359543, 0.0132314377, -0.0235971101, -0.0595435537, -0.0495177619, -0.015165667, 0.00930353906, -6.276710e-02, -0.0374512337, -0.0508569703, -6.58529694E-4, -0.0369340368, -0.0236234218, -0.0266874153, -0.0537709668, 0.0188743584, -0.0541712716, 7.656930e-02, 0.093701072, 0.0141150979, 0.0251069088, -0.037635047, -0.0705139786, -0.0149502242, 0.0510343127, -0.0569785051, -0.0136089623, -0.0276939087, 0.00808458496, -0.0802055969, 0.0454653725, -0.0453526862, -0.0700186789, -0.0282601286, 0.0622065551, -0.0709137618, 0.0781905502, -0.0387734026, -0.02551396, -0.0571607575, 0.0855763852, -0.0373514257, 0.00170688948, 0.0253219418, -0.0451905206], [-0.0316014104, 0.0570943728, 0.00997871998, -0.0240428373, -0.0167974308, 0.0129479207, -0.0161484815, -0.0821549445, 0.0714476779, 0.0853528082, 0.0437518358, -0.0407326333, -0.018606374, -0.0534542203, 0.0338334627, 0.0526463725, -0.0340650491, 0.0700489059, -0.0381183401, 0.00170758029, 0.0152948648, 0.0374400839, -0.066976212, 0.0229235124, 0.0880530476, 0.0733457655, -0.0572132133, -0.0837752521, 0.0146517195, 0.0174868666, 0.0210152417, -0.0808348134, -0.00499412976, 0.0110411709, -0.0653070882, 8.19434644E-4, 0.0507453158, 0.0144547811, -0.00296915742, -0.0279000867, -0.0476614349, 0.0550414696, -0.00872588065, -0.0364924371, -0.0354881398, -0.00512579177, -0.0325610936, -0.0470944382, -0.0564471483, -0.0877711772, -0.00226178207, -0.107638255, 0.0753667727, 0.0209999215, 0.0705382451, 1.565310e-02, -0.0133267576, -0.0533182137, -0.0859377682, -0.0106873149, -0.0574419573, -0.0766641051, 0.0421451703, 3.346130e-03, -0.0512230247, 0.0488315336, -0.0578191429, -0.0373217352, 0.0664412975, 0.0433416888, -0.0525304303, -0.015821049, -0.052652441, -0.0288092922, 0.0408706218, 0.0677362978, 0.0130665908, 0.00754272286, 0.0429989807, -0.00960929132, 0.02586261, -0.0179409012, -0.0388018824, -0.023885183, -4.560410e-02, -0.0383188799, 4.785640e-02, -0.0185988285, 0.00541676441, 0.0265769232, 0.0420821384, 0.0221045651, 0.0548727028, 0.0117915198, 0.0126079572, 0.0397548079, 0.0131578585, -0.0494438931, -0.0668710843, 0.00953609589, 0.0678328499, 0.0825080648, 0.0329660587, 0.0208929665, 0.00273672864, -0.0530243739, -0.063923113, -0.00777875073, 0.0424744226, 0.0421811454, -0.0852634981, 0.0652549788, -0.0383835919, -0.0789415761, -0.0479645729, 0.0335906222, 0.027486477, 0.0723473802, -3.194480e-02, 0.0503373705, 0.0884069949, 0.0968406423, 0.0277088918, 0.0124805914, -0.0396259949, -0.0561832748, 0.0882618278, 0.0482203327], [-0.035441611, -0.0598917231, -0.015464697, -0.137236223, -0.090886183, -5.888010e-02, -0.0461790338, -0.0260506608, -0.0410343371, -0.0198836625, 0.0721233338, -0.0295108017, 0.0503659844, 0.0171433464, -0.0315251946, 8.271270e-03, -0.0426465906, 0.0820152834, -0.0124068102, 0.0536327362, 0.0547595955, -7.596650e-02, 0.0323124751, 0.0261233766, 0.0255117919, -0.0617041289, -0.0631247386, -0.053100612, -0.0712595283, 0.0252806656, 0.0739614442, -0.00771864643, -0.0759893507, 0.0850829631, 0.0723300427, 0.0534480549, -0.0249549523, 0.0664552525, 0.0568846464, 0.00636553532, 0.0359610133, -0.0169962626, 0.0183215085, -0.00570611376, -0.0313598141, -0.00363859348, 0.0449700542, 0.11374639, -0.0791088715, -0.0918517932, 0.0761660412, -0.124119349, 0.0224909019, -0.101954028, -0.0479386225, 0.0455586612, 0.0302629601, 0.014109957, -0.0839744433, 0.0110987313, 0.0342474841, 0.073495388, 0.0530739687, -0.0375989377, 0.0628380552, 0.0195747912, 0.0369466282, -0.00294854771, 0.0543621033, 0.0143503742, 0.0906956568, 0.0852249116, 0.0509264916, 0.00623270171, -0.0358910747, 0.0395999551, -0.0419815294, 0.0729236156, 0.0103536295, 0.0477992147, 0.00689004222, -3.142780e-02, -0.069495216, -0.0317915156, 0.0468174666, 0.00130096613, -0.0746047794, -0.0558688082, -0.0136783328, -0.00878226943, 0.0497121923, -0.0169908106, -0.0561637096, -0.0579158776, 0.0527359284, 0.0206670538, -0.0242178198, 0.0159595795, -0.0532804392, -0.0684077889, -0.0155977411, 0.0160984211, 0.00446219649, -0.0290232729, -0.0317578912, -0.0471996404, 0.0926133096, 0.037376821, -0.0360696204, 0.0755515098, -0.107688434, 0.0592397526, 0.0772645771, -0.103283525, -0.0740012079, -0.0595296547, -0.0675616711, -0.0471441634, 0.0221659336, -0.0626168102, 0.0106425155, 0.0558825955, 0.0214925166, -0.00554907136, 0.0250859484, -0.0555550717, -0.0698351413, 0.0389805958], [0.0591461696, -0.0744535848, 0.0163885672, -0.0119922422, -0.0523444526, 0.00490679033, 0.0149512319, 0.0158606265, 0.0803092196, 0.0113793192, 0.0142750451, 0.060587626, -0.049606327, -0.062686868, -0.00264581875, -0.094612576, 0.0107660918, -0.0172740091, 0.0506491661, 0.048561357, 0.0595181398, 0.0234355982, 4.022650e-03, -0.0555443838, 6.623410e-02, 0.0136541734, -0.00933575351, -0.0265271757, -0.0742294267, 0.00478312932, 0.0496038496, 0.00103665027, -0.0888775661, 0.0483581126, 0.0170600228, -0.0607255995, 0.0856989473, -0.0686808378, 0.101935357, -0.00650489703, -0.0460954085, -0.0319925211, 0.0218615588, -0.0175483227, 0.0925184264, 0.00793447625, 0.103365637, -0.0238040276, -0.0859000832, 0.0687593445, -0.0539581329, 0.0095230611, 0.0538723394, -0.0837145745, -0.0332054272, 0.037306048, -0.00580750033, 0.0771339461, -0.0679211393, -0.0888134688, 0.0219056159, -0.00781571586, 0.0790082663, 0.0351005234, 0.0360886753, -0.0842460989, -0.0667010397, -0.0564899258, 0.0732884258, 0.111938447, 0.0532911047, -0.0494739115, -0.0703392922, -0.102981701, -0.0710343197, 0.0418837517, 0.0908461809, 0.0404478386, 0.0825937613, -0.0539071523, -0.0513158552, -0.106866531, 0.0473403372, 0.0103083923, -0.0489600338, 0.0474719629, 0.0769476444, -0.0602517165, -0.0388738364, -0.0137905069, 0.111047894, 0.00336776371, 0.00836607441, 0.021497665, -0.0173093714, -0.0503671318, -0.0215958171, 0.0116589721, 0.0443682633, 0.047936976, 0.0514088906, 0.0132485796, 0.0143888565, 0.0793610513, -0.0297617875, 0.0595372021, -0.0220279135, 4.090350e-02, -0.0427371822, 0.12324068, -0.00120474061, 0.020222621, -0.0629625097, -0.079552792, -0.0674140453, -0.0345612392, -0.0265700556, 5.890300e-02, 0.0393820032, 0.0470547266, -5.5887358E-5, 0.11632482, -0.02390543, 0.0215847772, -0.0718733743, 0.0675513744, 4.510760e-02, 0.00338071794], [0.0153430216, -0.0502655916, 0.0132199759, -0.00307674194, 0.0118267173, -0.0708992332, 7.722000e-02, 0.0182289649, 0.0803416892, 0.04124102, 0.0270515233, -5.77288854E-4, -0.00604836596, -0.0508660935, 0.0271719452, -0.0804009139, 0.0530569702, -0.025763765, 3.74305673E-4, -0.00383689324, 0.00196168246, -0.0678812191, 0.0760814697, -0.0316965245, 0.0369159169, -0.00818136055, 0.0417170897, 0.0417313613, -0.022417549, -0.0519082248, -0.0561903082, 0.0556522571, -0.0606557428, 0.0605007447, 5.860730e-02, 0.0360678807, 0.0200652666, -0.0321960896, -0.0138850594, -0.0449870601, 0.0847766175, 0.00136642088, -5.39480534E-4, 0.0408076607, 0.0654251054, 0.038639918, -0.0422192886, 0.0340024233, -0.0302173421, -0.00760236662, 0.0140489908, -0.00871903915, 0.00630600191, -0.0925997123, 0.0235457532, -0.00862306915, -0.0455883034, 0.0814704522, -0.0614497177, 0.0303106736, -0.0211980958, -0.00241012708, 0.0824791193, 0.0473516695, 0.0361968875, -0.0848849415, 0.0158242378, 0.0662820041, -0.0117036635, 0.10506022, 0.0197172761, -0.0324151441, -0.0509735309, -0.0597186573, -0.0723516047, 0.0573258959, 0.0534113646, -0.0224318597, -0.0359928422, 0.0405599326, -0.064106904, -0.00145668525, -0.0662381202, -0.0593431555, 0.0621866658, 0.0864898636, -0.0271514226, -0.0365855917, 0.061442744, 0.00968130399, 0.0431642383, 0.0337262228, 0.0070227636, 0.101897955, 0.0779572129, 0.0477818176, 0.0862185135, 0.0267436821, 0.0599392429, -4.273740e-02, -0.0124810096, 0.00619445369, -0.0577141941, 0.0612101741, -0.0610350817, 0.0784728601, 0.0443128236, -0.0446530841, 0.0344087109, 0.0675962865, -6.594770e-02, 0.0257323086, 0.0495991148, -0.128396302, 0.00574886799, 0.0953030958, -0.0105425278, 0.0566077791, 0.0309347976, 0.0686544105, 0.077944085, 0.0744639784, -0.0728171095, 0.0293978415, -0.066834867, -0.00265526492, 0.0698865578, -0.00166224584], [0.029874621, 0.0472063534, -0.00553143723, -0.0992334485, -0.0751410276, -0.0833625644, -0.0670209154, -0.0497798957, 0.027741665, -0.00805281661, 0.0533252098, 0.0552972183, -0.0368230082, -0.083936721, 0.0289363023, -0.0628359317, 0.038490437, -2.36604901E-4, 0.0196806211, 0.0741830319, 0.0404042974, 0.00686633121, 0.0527809113, -0.0601901039, 0.0337168388, 0.0173280183, -0.0619368888, -0.020691717, -0.0560189672, 0.0423754863, -0.0205197558, -0.00225311494, -0.0240615606, 0.0472973958, 0.0232338794, 0.104461528, 0.0705567375, 0.0122903027, 0.0479300469, -0.0901440456, 0.0785525143, -0.0503924973, 0.0331826694, 0.0481846817, -0.02846702, -0.0698942468, 0.0365438797, 0.112846375, -0.0531205758, -0.0388984792, 0.0496923774, -0.0781610906, -0.0092715444, -0.0978408455, 0.071366474, -0.0901581645, 0.0556929931, 0.024354415, -0.0386310071, -0.0521412231, 0.03564509, -3.262850e-02, 0.0551041029, 0.0741542578, -0.059706375, 0.0134567302, -0.0248076916, -0.0762627795, -0.0396070443, 0.0797914564, 0.0566303469, -0.0243602209, -0.0710388123, -0.0259217843, -0.0716719702, 0.0415925458, 0.070521079, 0.044788532, -0.0251185726, 3.838430e-02, 0.0764268338, -0.00241903216, -0.0550866164, 0.0401570834, 0.00140896952, -0.0128110899, 0.0383281671, -0.0229046866, 0.0486228429, 0.00231937203, 0.136872798, 0.0295434296, 0.033695627, 0.0891821309, -0.0448574722, -0.0233200826, -0.0177162569, -0.0128732966, 0.0899223089, -0.0832730233, 0.0448987782, -0.0102838455, 2.877230e-02, 0.0257366803, -0.0157221686, -3.520440e-02, 0.0839169919, -0.00814229343, 0.1052536, 0.11918816, 0.04394361, -0.00583635503, -0.024923224, -0.0330748186, 0.00915020425, -0.00311789382, -0.0534167364, 0.0447040945, 0.0752978921, -0.0573985316, 0.0269347057, 0.0368577428, -0.0526678897, 0.046242971, -0.0468195602, 0.00107341784, 0.0247307532, 0.0227054209], [0.0368841104, 0.0372982286, 0.02009142, -3.833040e-02, 0.0464626364, -0.0768779218, -0.0477316342, 0.0664806962, 0.0598916896, -0.0395904668, 0.065557979, 0.0695756748, 0.0908040404, 0.0256714858, 6.505930e-02, -0.0791554078, 0.0510085635, -0.0440204889, -0.0167519394, 0.0343371294, 3.924080e-02, -0.0142042199, -0.0301665608, -0.0959622636, 0.0627808496, -0.0203941818, 0.0584239215, -0.0307291411, 0.0382865742, -0.0467241555, -0.0638067573, -0.030573098, -0.0237068627, -0.0251093525, 0.0880519375, 0.0885664299, -0.0579122491, -0.0982803925, 0.0277782138, -0.0011175673, 0.0687186717, -0.0572854206, 0.0608497374, -0.0384853743, 0.0730418563, -0.0429807678, 0.0526257232, 0.156739697, 0.0707175359, 0.0541914478, -0.0869920551, 0.0154715823, 0.0278981552, -8.63548776E-4, 0.0368266106, -0.114846095, 0.0960691794, -0.0597464554, 0.0183447264, 0.0178715028, -0.0417938754, 0.0673548579, 0.0959970429, 0.0452580042, 0.0911600664, -0.0272161011, -0.0825677663, -0.0605742373, -0.0382778049, 0.0158453174, 0.0619524903, -0.0511021763, 0.0234506559, -0.069192715, 0.00603956264, 0.00468199793, 0.0249077026, 0.0194228031, 0.0912018492, -0.0118038729, -0.0466881059, -0.0340440422, -0.0365552902, 0.0599392541, 0.0259969328, 0.0462284386, 0.0627818257, -0.00722095231, -0.0251686517, -0.0149213756, 0.081078805, -0.0326526798, 0.0606795251, -0.00655566622, -0.0134713138, 0.0547763184, 0.0353856832, 0.0512914807, -0.0248746611, 0.0687088594, 0.0886902213, -0.0180776734, 0.0451695472, 0.0226303767, -0.0485582873, 0.0033703621, 0.119335078, 0.060075894, 0.0638544708, 0.0280863326, -0.028880775, 0.00386371021, 0.00709285215, -0.0895205736, -0.0222509839, -0.0441112928, -0.0179249663, -0.0528793223, 2.317370e-02, 0.0125490548, 0.00487179402, 0.0324829444, -0.0523180738, 0.105617948, -0.0425838158, -0.0197006352, 0.0864230245, -0.0684834197], [-0.00822983216, 0.00670248828, -0.0646730959, -0.0955436677, 0.0395320132, -0.058868926, -0.0122242179, 0.0397380441, -0.0749547333, -0.00793464109, 0.126553118, 0.0329890773, -0.055577293, -0.0212043524, 0.068201147, -0.0426509194, 0.0730808675, 0.0493769869, -0.0103094559, 0.0521582738, -0.0399923772, 0.0328554176, -0.0355745628, -0.0382690914, 0.0625720099, 0.0851115509, -0.0728652626, 0.0118376501, 0.0795761347, 0.0647363364, -0.0823199227, -0.0612265654, 0.0294962693, 0.0938157215, -0.0386740752, -0.0139842834, -0.0212500207, -0.11023286, 0.11329212, -0.0558430664, 0.0729018599, -0.0108778924, -0.0272716209, 0.0845390483, -0.0916306227, -5.058040e-02, 0.028410593, 0.0453136489, 0.0484112613, 4.217050e-02, 0.0719647557, 0.0782674104, -0.0230023842, -0.0201432798, -0.0309213027, -0.0932779833, -0.0650194883, 0.0382954516, -4.475970e-02, -0.0803691819, -0.10542208, 0.0271316301, -0.0292266253, 0.108464383, 0.102584951, 0.00410172949, -0.0844931453, 0.00757431611, 0.0706810579, 0.070970282, 0.0730109438, -0.0200191159, -0.0796930119, -0.0618187599, -0.0240843985, -0.0457582511, 0.122614808, -0.0293979459, -0.0770362392, -0.0628739074, 0.0193807185, 0.0557848066, -0.0335792638, -0.0444392227, 0.053306479, -0.0921012535, 0.080392979, 0.0225601271, -0.0545074157, 4.953740e-02, 0.0353911631, 0.0798743069, 0.0121365581, 0.0657266527, -0.0665947273, 0.062054038, -0.0991384238, 0.0179055464, -0.025955366, 0.00893502309, 0.0491152033, -0.0155978678, -0.0617004707, -0.0592465885, -0.0252774581, 0.0486144386, 0.103368111, 0.0234546345, 0.159963459, 0.12207827, 0.0564184859, -0.0681094825, 0.0700913369, -0.00202688109, 0.0112422854, -0.0449307375, -0.146392927, 0.0281874239, 0.133971542, -9.26778418E-4, 0.0309279878, -0.00861935411, -0.0197359081, -0.0135372328, -0.00609704526, 0.046442423, -4.15594841E-4, -0.058508683], [0.0735369548, -0.0338050425, -0.0100841345, -0.0492216907, 0.106609307, -0.116975844, 0.0574808344, 0.112638153, 0.0145925786, -0.0767453238, 0.109404661, 0.0199595541, -0.0847671181, -0.00344342832, 0.0693528503, -0.0359896682, 0.0066212248, 0.0108392173, 0.0309912041, -0.012315711, -0.0588666722, 0.0676638708, 0.0242922399, 0.0285993982, 0.110963747, 0.055735074, 0.0815989226, -0.0578922927, -0.00631835964, 0.0484814458, 0.02554553, -0.0132214464, 0.00208952953, 0.0649492368, -0.0178997777, 0.101273857, 0.0203374494, 0.0101303551, 0.0344031975, -0.0139059275, -0.0227223262, 0.0770553425, 0.00818225555, 0.0558301099, -0.106552713, -0.0599447452, 4.719930e-02, 0.0332392044, -0.00567048881, 0.0589554496, 0.0253602974, 0.00820143893, -0.0438336171, -0.121151805, 0.0451640226, -0.10132087, 0.0629766136, -0.033564128, 0.0386296548, 0.0405651107, -0.0554716066, -0.0430598333, -0.0501138344, 0.0656993091, -0.0121781211, -0.0690543279, 0.0750511065, -0.0455006585, 0.0627892688, -0.0030639749, 0.0340348817, -0.0876331627, 0.0755570382, 0.0245115887, -0.0203112941, 0.0579128228, 0.132193699, 0.0534293354, -0.0707343295, 0.0595256649, -0.071260482, 0.00587033061, 0.0442932583, 0.0452759378, 0.00413060281, -0.0588126257, 0.14841029, 0.0377934463, 0.0438354649, -0.0446802601, 0.00881033577, 0.0592179857, -0.0667643696, 0.0790657848, 0.027154116, 0.0673228353, -0.0275396183, -0.011114873, -0.0967409089, -0.0169508085, 0.0268557481, -0.0966255441, 0.0105337109, -0.0615446791, -0.0772028416, 0.0717162117, 0.0480080396, -0.0118826618, 0.0802412182, 0.0327702723, -0.0712593272, 0.0283164289, -0.03926019, -0.0135942604, 0.0261949133, -0.0608853698, -0.0345986262, -0.0463799238, 0.0779415741, 0.0387826487, -0.0225605052, 0.0852186679, 0.0551704764, -0.0215292703, 0.0810013487, 0.0652170107, 0.0930058062, 1.164290e-02], [1.281350e-02, 0.039383661, -0.0263418313, -0.100913517, 0.12042056, -0.127033561, -0.0431455597, 4.424590e-02, 7.288230e-02, -0.0163341388, 0.071882464, 0.0146535477, -0.0206293389, 0.0177623574, 0.0884390175, 0.0331101604, -0.0349245854, -0.02950464, -0.0894523039, -0.0244830269, -0.0549775921, 0.0367591977, -0.0323112048, -0.0126329344, -0.0495029539, 0.0609598272, 0.102137268, -0.0235594139, 0.00607734593, -0.0415459834, -0.0731234401, 0.135693014, -0.0696410313, -0.0291968025, -0.00689717336, 0.0985684096, -0.00847652461, 0.0963418931, 0.0434000827, -0.0592412576, 0.0983351767, -0.00937164854, 0.00857361499, 0.00524535123, -0.102208093, 0.0199970715, -0.0109837092, 0.0150106251, 0.106671855, 0.0422649123, 0.0128986081, 0.0850796326, 0.0208820906, -0.0991154089, -0.00743344239, 0.04787682, -0.0516690128, -2.91473581E-4, -0.0690223276, 0.141152486, 0.0832918435, -0.110158689, 0.00876726396, -0.0116370022, -0.0568055585, -0.0200022236, 0.075016059, 7.058080e-02, -0.0485063791, 0.0285148025, -4.34846384E-4, -0.0434734039, 0.0642093867, -0.0403642952, -0.0396613255, 0.0907571241, 0.0234282091, 0.0447522812, -0.116463639, 0.0940201058, -0.112386785, 0.0852700472, -0.0197383463, 0.00586451637, 3.551980e-02, 0.0146346027, 0.0445639491, -0.00541999238, 0.0586146452, 0.0813112706, -0.0111409649, 0.115984462, -0.0587637797, -0.0767264143, 0.0616926365, 0.0855590403, -0.0919165387, -0.0672587827, -0.0528466925, -0.0665455535, 0.0730181411, -0.00871343724, -0.0551570468, -0.0479654968, -0.0824013724, -0.0192359127, 0.0347487852, -0.0409550704, 0.0708006099, 0.0622068495, 0.0579020455, -0.0400952175, -0.0389771685, -0.012081936, -0.0935586318, 0.0453088246, -0.0676880702, -0.0808265656, 0.0551131144, 0.0728470832, 4.327190e-02, -0.0725056529, -0.039646741, 0.0118691083, 0.0993125811, 0.0641924888, 0.0960123538, -0.0802047402], [0.0149884718, 0.0158544499, -0.00463018799, -0.0066271848, 0.0522635207, -0.0610300452, 0.0431115255, 0.0381477624, 0.0577380173, -0.0612284392, 0.00433044229, -5.291040e-03, -0.101273008, 0.0628775656, -0.0547722764, -0.0201136339, -0.0551869124, -0.0689748526, -0.0886783301, -0.0546231419, -0.0235216562, 0.0291347876, 0.0244602505, -0.0362039395, 0.0445673503, -0.10780102, 0.11059916, 0.0674249679, -0.0442048423, -0.085856527, 0.0146494955, 0.105784498, 0.0153382225, 0.0656435937, -0.0267538261, -0.0608981289, 8.988190e-03, 0.148090541, -1.845800e-02, -0.0197452884, 0.0151109165, 0.0014724012, -0.00873289257, -0.133460715, 0.0832570791, 0.0428019315, 0.134511277, 0.0385587215, 0.0091677038, -0.114914052, 0.107851356, 0.00391081162, -0.0529733375, 0.00714996411, -0.0611451194, 0.0550331622, -0.0605834834, -0.0654290244, -0.0399258398, 0.0906831845, 0.0216979831, -0.0753441304, 0.126951814, -0.0835365131, -0.0285136048, 0.133373916, -0.00574888336, -0.0613769405, -0.00206798338, -0.0850610286, -0.038217254, -0.0649684146, 0.134287789, -0.0350440405, -0.0691518188, -0.00225137547, -0.03684875, 0.0304139927, -0.110598519, -0.0567132533, -0.0573310629, 0.0114222793, 0.0828103572, 0.146859899, 0.0326771215, -0.058394365, 0.0291373134, 0.0401939228, -0.0367570408, 0.00907228328, 0.0214942563, 0.0866902918, 0.0494051762, -0.112070486, 0.0233602058, 0.0843547136, -0.0326768681, -0.0351614468, 0.133080348, -0.117149338, 0.100840516, 0.00950978696, -0.014534153, -0.0578772202, 0.0494457558, 0.102341026, 0.0429788679, 0.0406282283, -0.0464525297, -0.0520211682, -0.0466146804, -0.0438495912, -0.0643577948, 0.126942664, 0.0143865738, 0.0643815622, -0.00348165259, 0.008018855, -0.0461425483, -0.0437351875, 0.0620227269, -0.00871497858, 0.0660033152, -0.050175868, 0.0422850065, -0.0314932726, 0.111546762, -0.0382976159], [-0.0316234715, 0.0205458328, -0.036834918, 0.0854299515, -0.0112465285, 0.0248173345, 0.0275014676, 0.0233443603, 0.00955701619, -0.0051195221, -0.0115294242, 0.0235811714, 0.0507273376, 0.00341355102, -0.0910129323, -0.100287646, 0.020685697, -0.0516937636, -3.664450e-02, -0.0233376566, -0.0738099888, 0.0549878031, 3.231580e-02, -0.00577374594, 0.0049290373, -0.0469778888, -0.00746307056, 0.0557600334, 0.0247320943, -0.0858101174, 0.073267728, 0.0767176077, 0.018993631, 0.00815844722, -0.103236705, -0.00295375311, 0.0206306875, 0.107891805, 0.0625449717, -0.0570692755, -0.00365907769, -0.0770187079, -0.00198437739, -0.00119830552, 0.0595533252, 0.0985011607, 0.11411915, 0.0128970845, 0.0756934434, -0.00467821071, 0.0330224782, -0.0796747133, 0.0599682853, -0.00597780151, 0.0608953647, 0.0390945971, -0.0859080925, 0.0718378052, -0.0660599172, 0.0885860175, -0.011984216, -0.007171412, -0.0177220982, 0.0674514547, 0.0767568797, 0.153254122, -0.0459407307, 0.0398326069, -0.0799515545, 0.0497266725, -0.0500089824, 0.048276294, 0.00640407344, 0.0481803529, -0.0453221686, 0.0148627805, 0.1029853, 0.0396802351, -0.00482254848, 0.0128363743, 0.00375269284, -4.3613973E-4, 0.0559640825, 0.1001269, 0.0313536339, 0.0650100932, -0.029679589, 0.034887813, 0.0826916248, 0.0644842386, 0.028503485, 0.04382376, 9.071990e-03, -0.139288127, 0.0124729509, 0.0592138246, -0.060296312, -0.0576686114, 0.0765720531, 0.0467970073, 0.126334801, -0.0398556441, -0.030051915, -0.0437511392, 0.0499419831, -0.0389835387, -0.0689050928, -0.0376934782, 0.00354195875, 0.0169167481, 0.00690510822, 0.069946453, -0.0622381121, 0.0746563897, -0.0456069112, 0.0782573819, -0.0346009843, -0.0163072124, 0.0164852552, -0.0409058891, -0.00846579857, 0.0581678711, -6.28812413E-4, -0.125768691, 0.0173584018, -0.0181705561, 0.0831517353, -0.00165820762], [4.0989212E-4, -0.00356417545, 0.0145660127, 0.0489246249, -0.0429755673, -0.080991663, -0.036644008, 0.0205038097, -0.044879809, -0.0554241762, 0.114013754, 0.0150447087, 0.078590557, 0.0344842374, 0.00113057706, -0.00637367135, 0.0435446128, -0.0590697713, -0.0621562711, -0.0200831611, 0.0180870946, 0.0624874458, 0.0551378578, -0.0408149622, 0.0454535782, -0.0514999665, -0.0645608157, 0.0296973139, -0.00629089074, -0.0700763687, 3.689820e-02, 0.0369783677, -0.109359622, 0.125706941, -0.099628739, 0.00408657128, 0.0951689631, 0.115249678, 0.0230223108, -0.106053695, 0.0800397321, 0.00663280487, 1.921380e-02, 0.0419220664, -0.00436526304, -0.0167771541, 0.00691508502, 0.0929549634, 0.022588199, 0.0223491248, 0.0707634985, -0.0393889621, 0.0309421699, 0.0227762088, 0.0302048773, 0.0452560708, -0.0246560834, -0.0604945831, 0.00607202109, 9.928300e-02, 0.00583918579, -0.00180297333, 0.0832652673, 0.0804315879, -0.0253990628, 0.077276729, -0.0390640125, -0.0474047959, 0.0128534622, -0.034298759, 0.0697561279, -7.746340e-02, -0.0403327495, -0.0447901152, -0.0449730754, -0.0715344921, 0.0590178408, 0.0414706208, -0.00285791978, -0.0724285096, -0.00572026148, -0.0754250363, 0.0584489368, 0.0858828723, 0.0362252966, 0.00975849945, -0.0443340205, 0.0361338109, -0.0753416866, -0.0152699752, -0.069419682, 0.107943103, 0.137409061, -0.0154134436, -0.0355352163, 0.0497914851, 0.0346248709, -0.0143364314, 0.117662974, 0.0622049532, 0.128545508, 0.0869062542, 0.0242676362, -0.0527765788, -0.0466889553, -0.0273267496, -0.0437850654, 0.0100465342, 0.00293297297, -0.0400916114, 0.00631139474, -0.0188132785, -0.0691736564, 0.00719504477, 0.0618747137, 0.039273113, -5.918680e-03, -0.0501570776, 0.0409028381, 0.0127316853, -0.012402473, 0.082714878, 0.0950429887, 8.415220e-03, -0.00670764456, -0.0881392211, 0.110441707, -0.0395613685], [-0.00419188896, 0.0460341945, 0.0879917964, -0.0506825671, 0.0191456638, -0.0314383432, 0.00786730554, 0.0287301876, 0.0322942473, 0.0269841589, 0.0678595379, -0.0108821495, 0.00852863863, -0.0243770443, -0.0111948829, -0.00861736294, 0.0364543647, -0.0586025193, 0.0275342856, -0.0454570353, -0.0730985478, 0.00385110034, 0.0685857758, 0.0381093621, 0.116118565, -0.0725329071, 0.0359568521, 0.0767299234, -0.0180168357, -0.0956342816, -0.0270704236, -8.65977781E-4, -0.100777335, -0.0138046918, 0.0456660576, 0.0337755457, 0.11657735, 0.00393922674, -0.0605676696, -0.100233287, 0.0895316452, 0.00542735308, -0.0420087315, 0.0805984586, 0.0796521082, 0.0894126892, 0.0789988636, 0.100607775, 7.164330e-02, 0.0237728022, -0.0754477754, 0.0202921182, -0.00242140819, 0.0199562274, 0.0367992222, -0.0688543096, 0.0342090167, 0.0461439975, 0.0364847928, -0.0632971674, -0.054969497, 0.0174139608, 0.0770578906, -0.00319797313, -0.0192925911, 0.0170931555, 0.0578104071, -0.0458387658, -0.0578301847, 0.0931961536, -0.021024799, 0.0122993598, 0.045948837, 0.0429896712, -0.0472644344, 0.0328364708, 0.0567159913, -0.0764930174, -0.0277846903, -0.0783390328, -0.0317356102, -0.0563230962, 0.0454923734, 0.027104089, 0.0515145659, 0.0263266694, 0.00693256641, 0.0888626873, 0.0259832051, 0.0804822444, -0.0980361476, 0.0652195662, 0.108184375, -0.00211152155, 0.043057017, 0.0215043537, 0.00824889354, -0.0261044279, 0.093578644, 0.0879538282, 0.045204483, 0.0392606258, 5.323880e-02, 0.078541696, -0.0112347817, -0.0554659665, -0.0679581389, -0.0544066876, -0.0510128513, -4.72158135E-4, 0.0399629362, -0.0364729203, -0.047306966, 0.0426671356, 0.0963493064, -0.0919427797, -0.0505862571, 0.0771083831, -0.0283649024, 0.0198960286, -0.00990609545, -0.0316379927, 0.0363286585, -0.0955519527, 0.0175433252, -0.0583816431, 0.108074054, 0.0259967539], [0.0095188748, 0.032710135, -0.0125103239, 0.0354626849, -0.0877939314, -0.0671060607, 0.0890134871, -0.0596593656, -0.0869177952, -4.01584344E-4, 0.106342725, 0.063346982, 1.22999118E-4, 0.0130101498, 0.0393748432, 0.00844729226, 0.0621308423, 0.0573401861, -0.0129270563, 0.0410878062, -0.0638562813, -0.00424568262, -0.0293360129, -0.00678408704, 0.0624280385, -0.0695737824, 0.0466136858, 0.0908624157, -0.0638813898, 0.0250072982, 0.0126869855, -0.0440326035, -0.0438072234, 0.0815127789, 0.0444092862, 0.0559550673, 0.0577864945, 0.0904814973, 0.079061076, -0.0504903533, 0.0801986083, -5.434870e-03, -0.0198573787, -0.0493944548, 0.0248004217, 0.095795013, 0.00618368154, 0.103286579, -0.0559602901, 0.0712533593, -0.0785412564, -0.0271070208, -0.078284353, 0.0402962118, -0.0258444045, -0.0214116871, 0.0732101202, -0.0462036245, -9.966120e-02, 0.0221640989, -0.0136245182, 0.0662228689, 0.0191099644, 8.968090e-02, -0.0477727279, 0.0072500864, -0.0457909703, -0.0143794874, 0.0575189702, 0.087088786, -0.023259785, 0.0401907079, -0.0396253057, 0.00775057357, -4.004560e-02, -0.0670426413, -0.00734835304, -0.0760993063, -0.0308855902, 0.00372323766, 0.0717236921, -0.0299486853, -0.0792479366, 0.0468352512, -0.0314928144, 0.0386517718, -0.043861229, -0.0704022347, 5.165120e-02, -0.0482203178, -0.00581025751, -0.0108088832, 0.0522793792, -0.00693777436, 0.0165597536, 0.015817441, 2.995270e-02, 0.0223026033, 0.0528609604, 0.101334281, -0.0079140095, 0.0082168607, -0.00756318588, -0.00722818216, 0.0313565172, 0.0161221027, -0.0256845076, 0.0219843611, 0.00714792497, 0.091679953, 0.0398289301, 0.0638229548, 0.00887812115, -0.0251394939, 0.126910716, -0.0940714552, 0.027653845, -0.00289617875, 0.0200172383, -0.0136201605, 0.0019287616, 0.0909112319, -1.126720e-02, -0.00453919964, 0.0104029318, 0.0140141323, 9.33370203E-4, 0.0736201331], [-0.055645559, 0.0217335336, -0.0403899215, -0.0154796932, -0.0935490354, -0.0453271866, -0.0364282541, -0.019270286, -0.0350348875, -0.0058416184, -0.016184086, 0.0986023172, 0.0285594203, -0.0269837808, -0.00274942117, -0.0234373361, -0.0020057254, -0.0710824728, -0.0502814911, -0.00399523089, -0.0279557612, -0.0873370245, 0.037033014, -0.0299409702, 0.0238470063, -0.0701197833, -0.0529550202, -0.0337370075, -0.0279761273, -0.0548669547, 0.0707332715, 0.0473781452, -0.0192198325, 0.0377822183, 0.0685461089, -0.00230875704, 0.0074331006, -0.0367322788, 0.0632192194, 7.142730e-02, 0.040815711, -0.0651457831, 0.0663294494, 0.0427396484, 0.0453464091, 0.0738509521, 0.0592568368, -0.0307394471, 0.0491453968, -0.0681406483, -0.0430575199, -0.0142293321, -0.0158796869, 0.00410665432, 0.0183566455, 0.0268081501, 0.0617591701, 0.0195180923, 0.0352544598, 0.0422083959, -0.0305790324, 7.40596442E-4, 0.0815058202, -0.0070138867, 0.00569366524, 0.0467436388, 0.103560455, -0.0214639939, -0.0372995213, 0.0627936646, 6.695640e-02, -0.0404630601, -0.0876545384, -0.00343226525, -0.0211919807, 0.0353023447, 0.00355987786, -0.0169530548, -0.0287411977, 0.0521773733, -0.0646530539, -0.00905376766, -0.00771530205, 0.00536059029, -0.0393295102, 8.76153179E-4, 0.0270951409, 0.0264418144, -0.0669901222, 0.0558232218, -0.0808712691, 0.0187932719, 0.00720745232, 0.0189890508, -0.0392106175, -0.0332462527, 0.0754491687, -0.0586125627, 0.0611331164, 0.0130347516, -0.0225479174, 0.101314433, -0.0879849046, 0.0388356932, -0.0642205104, 0.00699317316, -0.0753755793, -0.0483550169, -0.0532117486, 0.0527941287, -7.278230e-02, 0.00333178043, -0.0106139444, -0.00870020408, -0.0114665236, -0.0412751921, 0.00251951488, 0.129449949, -0.0370871425, -0.029998675, -0.0315435976, 0.0547377616, -0.0352241285, -0.0407552831, 0.0544179566, -0.0297731552, 0.0983512774, 0.0560311414], [-0.0564823635, 0.0198347606, 0.0556242242, 0.0461747497, -0.055889193, -0.0684600472, -0.00175577193, -0.0467493869, 0.0544508658, -0.0590964295, -0.0817618221, 0.0460282974, 6.81673758E-4, -0.0789276435, -0.0379682928, 0.0294701103, 0.00685057556, -0.0580282696, 0.00764441863, 0.112824328, 6.580360e-02, 0.0155734383, 0.0157461725, -0.099132277, 0.00938025583, -0.0955946743, -0.0490656756, 0.0280238055, 0.0407483429, -0.0346944816, 0.064970322, 0.0400250927, -0.0435701832, 0.0127320047, 0.0551942624, -0.0513671786, -0.0354972146, -0.0351306349, 0.00747526716, 0.0656321794, 0.0186750814, 0.0421165712, 0.035684593, 0.0150966868, 0.0335651524, 0.0884208455, 0.059133973, 0.0293041039, 0.0676094666, 0.0750726387, -0.0808080584, -0.00950440485, 0.0650214702, -0.0218031835, -0.0408431441, -0.0532958508, 0.0425948165, -0.0696247295, 0.0507873707, 0.0300318804, 0.00633110525, 4.004930e-03, -6.58594829E-4, 0.0391325839, -0.0409875214, -0.0291846618, 0.0783506929, -0.0700300783, -0.0281529762, 0.112038009, -0.0153814992, 0.0614976324, 0.055213403, -0.0367150493, 0.0811972171, 0.00977549236, 0.0280067883, -0.0671868175, 0.0104494905, -0.0338143185, 0.0249022413, -0.0592334792, -0.0243538544, -0.0352946334, -0.0683903545, -0.0459000766, 1.75006804E-4, -5.323510e-02, -0.0498222932, 0.0511656255, -6.343770e-02, -0.0728544667, 0.0389995575, 0.0604105853, -0.0711919814, -0.0960463881, -0.00146009971, -0.0470005684, 0.0751323253, 0.102295853, 0.00797087792, 0.0834247991, 0.00363390101, 0.058441367, -0.0879419595, -0.0580420755, 0.00232521817, -5.221950e-02, 4.81660245E-4, 0.0276824869, -0.0100997277, -0.00174536929, 0.0220886823, -0.0197085757, 0.0342323296, -0.0868409499, -0.0344068073, 0.0263184607, -0.0636259764, 0.0639881864, -0.0203530174, 0.0465444103, 0.0643172935, -0.0437008627, -0.0276105646, 0.00152849674, 0.0309442393, -0.0431828685], [0.0342927948, 0.037200287, 0.0173323248, 0.0673355907, 0.0155533329, -0.0316520743, 0.0590002127, -0.113335326, -0.030554045, 0.0443981662, -0.102065422, -0.0176887382, -1.732280e-03, -0.0361114256, -0.0399203151, -0.0162635501, -0.0770515054, 0.0284896903, -0.0396840163, -0.0205460154, 0.0436310917, 0.0531978235, 0.0491017736, -0.0603088848, 0.0835227221, -0.0458668098, -0.0193128437, -0.0483375713, -0.00241588452, 0.0917341783, -0.0705661773, -0.0547831543, -0.0554532111, 0.0748691931, 0.0330041461, -0.0285458174, 0.0312757455, -0.036580421, 0.0153805036, 0.0338963121, -0.0352555551, -0.0426970311, -0.00311313849, 0.0479354672, 0.0286449846, 0.0375605077, -0.0363623351, 0.0544836409, -0.0488187112, 0.0278740488, -0.0870075673, 0.0416806154, -0.053080406, 0.0361250266, 0.0323570557, -0.0422137491, -0.00439166138, 0.0806137323, -0.0353829414, -0.0612520166, -0.0122988233, 0.0751804039, -0.0690884441, 0.0516985469, -0.0340920128, -0.0208933447, 0.090849705, 0.041376397, 0.0718917772, 0.0505893044, -0.0773542523, -0.0465014912, 0.0227034688, -0.0796094313, -0.0186821558, -0.00775345787, -0.0391722918, -0.0173499063, 0.0610750019, 0.0262235701, -0.0371982157, -0.0512651131, -0.0196488053, -0.0139863165, 0.0541144088, 0.0624604262, -0.0454602093, -0.0689951926, 0.0367550291, -0.0123902457, 0.0727168322, -0.101146601, 0.0456138514, -6.930740e-02, -0.014146788, 0.0410097539, 0.0592631511, 9.209620e-02, -0.0793581456, -0.00358726131, -0.0288688745, 0.0809066072, -0.0367034078, -0.0682910308, -0.0337821506, -0.0520872585, 0.0591488369, 0.0425578505, -0.0580754168, 0.0416496731, -0.0114224181, 0.0233611278, -0.0465437137, -0.0403300114, -0.0287386775, -0.0400552042, 0.0452142097, 0.0613524392, -0.0579976551, 0.0573572367, -0.0927956774, 0.0925789326, -0.0768306479, -0.0438111611, -0.0290148593, -0.00553910434, 0.0460108668, -0.0241010115], [0.0283059794, 0.0397340208, 7.601380e-02, 0.0317074098, 0.00898813736, 0.0244667418, 0.0339346193, -0.0480089821, 0.0876743197, 0.0311982986, -0.0901155323, -0.0134409927, 0.0308834389, -0.0752164945, -0.0675561875, -0.0156224417, -0.00634328881, 5.79223386E-4, 0.0546012148, 0.104968987, 0.0564144552, -0.02584574, -0.0119550312, 0.0346722193, -0.0180995818, -0.0886467844, -0.0797690451, -0.02035865, -0.0654966757, -0.0144724073, 0.0309943836, -0.0015506543, -0.0939139575, 0.0365966372, 4.006410e-02, -0.0422207825, 0.0290368963, -0.0876832306, -0.0146378223, 0.0414464064, -0.0052335714, 0.00197819434, -0.0362734608, -0.0569614731, 0.0828612596, -0.0128056724, 0.0767156705, -0.00788539182, -0.0169067197, 0.0812036618, 0.0692920908, -0.0769915357, 0.0954498798, 0.0332949683, -0.0428086333, 0.0222072192, -0.03531285, 0.0480000414, 0.0162397977, 2.125270e-02, -0.0778297707, 0.0527442843, -0.0313017927, 0.0235465728, -0.0688840896, 0.0548752621, 0.0687198788, 0.0628350303, -0.00476057921, -0.0308193788, -0.10482081, 0.0153325237, -0.0444399081, -0.0753472596, -0.0776622593, -0.0423670784, -0.0497907884, -0.0358693451, 0.00401721755, -0.0652652159, -0.0291401464, 0.0132853193, -0.0457479618, -0.0599095337, 0.0641402304, -0.0458875634, -0.0174535923, -0.0623454116, 3.839980e-02, -0.0228426848, -0.0298886355, 0.0144107984, -0.012695591, 0.0788001567, 0.0572667383, 0.038812533, -0.00410743756, -0.0127252964, -0.0460680872, 0.0350575894, 0.0710952058, 0.079152815, -0.0099663129, 0.0234609302, -0.0120121641, -0.059779048, 0.0141782286, -0.066493459, 0.025320746, 0.0460461713, -0.0917685106, -0.00558663439, -0.0844132974, -0.00586798182, 0.0336409844, 0.00305192405, -0.00285460614, 0.0267029051, 0.0076516238, 0.0516499318, 0.0294111427, 0.126472205, -0.0508570746, 0.0791102648, 0.0289964657, 0.0301630702, 0.0270996895, 7.326090e-02], [-0.0217453148, -4.552140e-02, -0.0689565316, 0.0396603569, -0.040106643, -0.0912036225, -0.035751164, -0.033893235, 0.104133271, 0.0475955047, -0.0221469123, 0.0725111365, 0.0774008483, -0.0432122312, -0.0912008881, 0.0389133729, 1.296170e-02, 0.0396572687, 0.0594655499, -0.0182957016, 0.00163350464, 0.005179286, -0.00512750447, -0.0369892493, 0.0624854788, 0.00404458586, 0.00233653956, -0.0116831595, -0.0346011557, 0.0165856201, 0.0718769953, 0.0112413196, 0.0470529795, -0.0785256102, 0.0751926898, -0.0394223705, -0.0814548284, -0.0681239143, 0.0634595901, 0.0126171065, 0.0379636958, 0.0104055088, -0.066984348, -0.0172430854, 0.0698219835, 0.0175102726, -0.0128207747, 0.0191914048, 0.0773135275, 0.0914080292, 0.0172310639, -0.0326082855, 0.0609929189, 0.0535068624, 0.051230073, -0.06899786, -0.0802585632, 0.0499512628, 0.0195167251, -0.00574735692, 0.00133389491, -0.0398748629, -0.076186128, -0.0873939246, -0.0136969276, -0.062280871, 0.089324437, -0.0277826674, -0.00393309584, -0.00576693052, -0.0470543168, 0.0549903959, -0.0851729884, -0.060430415, -6.9129921E-4, 0.0606685765, -0.0565286539, 0.0747829899, -0.0518261567, -6.873350e-02, 0.061843805, 0.0419141203, 0.0436606258, -0.0748917237, 0.0027373808, -0.00776363118, -0.0604706891, -0.0493503213, 0.0219695754, -0.0105586927, 0.0748419911, 0.00905675254, 0.0676701739, -0.0111398902, -0.0252743773, 0.0307103861, -0.0928677991, 0.0486722589, 0.0458800085, 0.0121546537, -0.0632014945, 0.0275705289, -0.0173670202, -0.0675243661, 0.036007911, -0.0598186553, -0.0314131789, 0.0170930531, -0.0494031198, 0.0382417329, -0.0818530842, -0.0441278256, -0.0496446863, -0.0941833332, 0.0293022823, -0.00829390995, -0.0491605401, 0.0035955878, -0.0395435728, -0.00768104801, 0.0144291194, 0.105384685, -0.0254054368, -0.0178542454, -0.0347141847, 0.0150688142, -0.0645742938, -0.0477183349], [0.0317144729, -0.0168005917, -0.0488156229, -8.934170e-02, 0.00478927791, -7.201170e-02, 0.0151739921, -0.0173174702, -0.0216227677, 0.025297828, 0.0491917357, 0.0222801287, -0.0296949819, 0.0564369448, -0.0670451671, 0.0400745235, -0.0776560828, -0.0240473356, 0.00428922242, 0.0619007871, 0.013713113, -0.0492740832, 0.00228159945, -0.0838711261, -6.178210e-02, 0.051350113, -6.076840e-02, -0.0194393303, -0.020616632, 0.0520247631, -0.0694412291, -0.0783427283, -0.0261422805, 0.0508307517, -0.0223963745, 0.0895207077, -0.0479517318, -0.0755169615, 0.0735956877, 0.0932614058, 0.0202636067, -0.0393996648, -0.0242467746, 0.0176950954, 0.0304766726, 0.0692058429, 0.0284881946, 0.0610608347, 0.046832256, -0.0555929616, 0.0364679322, 0.0195410009, -0.0427689515, 0.011349733, 0.0101521732, -0.0337244645, 0.032160867, 0.0242217034, 0.0679270849, -0.00741302595, -0.0698530897, -0.024442587, 9.93355875E-4, -0.074834995, -0.00315106707, -0.0543753654, 0.07202214, -0.0454692952, 0.0330575183, -0.0343229733, -0.0577199049, 0.0182325412, 0.0731183514, -0.0111588649, -0.0101245772, -0.012606469, -0.0210867804, 0.00747365924, 0.0830778777, -0.076950416, 0.0199501142, 0.0758649483, 0.00753056211, 0.0653925762, 0.0250404775, 0.0257124975, 0.0563507751, 0.0631385297, 0.058059819, -0.0168271363, 0.0255074427, 0.0579993725, 0.0677578896, 0.0259352047, 0.0653968751, 0.0511714779, -0.0837637186, 0.0424962714, -0.00438857451, -6.353260e-02, 0.0128530869, 0.0362517834, -0.00387994503, -0.0631783828, -0.046527423, -0.059876319, -0.0730693042, 0.0457448512, -0.0840296968, -0.0673019364, 0.0780314132, 0.0424524583, 0.0550934039, -0.0370004028, -0.0715513751, 0.0893101096, -0.0212990306, 0.0724918917, 0.0505969711, 0.0561069734, 0.0163314473, -0.0333100259, -0.018900482, 0.0426728502, 0.0189784952, -0.019764347, -2.82098976E-4, -8.86048481E-4], [-0.0439754277, 0.0655735731, -0.068458274, 0.00715362607, -0.0462118611, -0.0775257721, -0.0225504078, 0.0512246639, 0.0170800705, -0.0827921778, 0.0607018545, -0.017018877, 0.0425118692, 0.0633815228, 0.00413365616, 0.0222134925, -0.0584512278, -0.0732561573, -0.0061527784, -0.0430842042, -0.0171864014, 0.0241517071, -0.00408689398, 0.0590349659, 0.0741663426, -0.0196186695, -0.0396598242, 0.0519790053, 0.0113544762, 0.0234409831, 0.0585399196, -6.040080e-02, -0.0334466137, 0.0185348261, -0.0298348069, 0.054264605, 0.00615015952, -2.362570e-03, 0.0575047657, 0.0709173158, 0.072924152, -0.0811933726, -0.0366758592, -0.0499791093, 0.00662487373, -0.0569784269, -0.0485381074, 0.0613334253, -0.0573700219, -0.0624754242, -0.0220919233, -0.0573875904, -0.0265347343, -0.0300588254, 0.0308725554, 0.0767737627, -0.0751627833, 0.0272502583, -0.0232193638, 0.0369849093, -0.0330778509, -0.0699650943, -0.0772136599, -0.0814732313, -0.0593627281, 0.0183696337, 0.0176227912, -0.0259828642, 0.0763059705, -0.0285117105, -0.0142660132, -0.00306502357, 0.0770274996, -0.046816349, 0.0205401033, -0.0119645819, 0.0312181786, -0.027567707, 0.022199709, 0.0209908094, 7.757430e-02, 0.051737912, 0.0331758931, 0.0428731889, 0.0323502794, -0.0524976179, 0.00528897392, 0.0284117311, 0.00698581757, 0.0205771811, 0.0686130896, -0.0293545797, 0.0640302449, 0.0623756722, 0.00266874209, 0.0705056414, 0.0627261847, 0.0406417921, 0.0230071936, 0.0161672421, -0.0468347222, -0.0276171621, -0.00106243207, 0.0732855573, -0.0730269179, 0.0142251262, -0.0365689099, -0.0291208513, 1.997950e-02, -0.0319448225, -7.054440e-02, 0.073048614, -0.0692311078, 0.0328980274, -0.0739174709, 0.0491957515, -0.0596216545, -0.00480460608, 0.0523057394, -0.0683787763, -0.075529322, -0.0346862264, 4.563640e-02, 0.0643027276, 0.0425610617, -0.077254653, -0.0605277903, -0.0300484207], [-0.0531129651, -0.0660330057, -0.0418349691, 0.0349418484, 0.0775661766, 0.00864099618, 0.0706877932, -0.0238224715, 0.0304217767, -0.0307500269, 0.077068597, 1.64228652E-4, -0.0185950045, -0.0371829122, 0.0229702983, -0.0668876171, 0.0810811147, 0.0522127971, 0.00896141491, -0.0773315355, 0.0326499455, 0.00966413691, 0.0755837783, 0.0432240702, -0.0794810578, 0.0175289605, 0.0494231842, 0.0620064772, -0.0438659452, 0.0411631055, 0.057057362, 0.0763907805, -0.0658214167, -0.0302194655, 0.0196335074, -0.0433222651, 0.0183415301, 0.0356749222, 0.0313399881, -0.0633492171, -0.0378945284, 0.0377767123, 0.0105952155, 0.070449315, 0.0102587473, -0.0380205512, 0.0466770381, 0.04274581, 0.0466908962, -0.0402408727, -0.0461555868, -0.0778145641, -2.70136341E-4, 0.0476212129, -0.00246832706, -0.0514559112, -0.0678729191, 0.0200043451, 2.829640e-02, -0.0294980779, 0.0592108108, -0.0165505037, 7.658950e-02, 0.00147012959, 0.0539672226, 0.0261735227, 0.0363284126, 0.0454688743, 0.0671592504, -0.040521957, -0.0586078539, 3.584070e-02, -0.0711098388, -0.0403349437, 0.0501774922, 0.0592322238, -0.0777959227, -8.908630e-04, -0.0683353394, -0.0236008745, -0.0596875697, -0.0280861948, -0.0494222231, 0.0538180768, -0.0400031842, 0.0534158275, -0.0229791161, 0.00677712681, -0.0805317909, -0.0710485652, 0.069875285, 0.0419883691, -0.0117716817, -0.0344621353, 0.0138727278, 0.012066179, 0.00868666172, -0.0205368269, 0.0363645777, 0.0516355596, 0.0764713734, -0.0442141518, -0.0310288016, -0.00230433047, 0.0556133911, -0.0695609078, -0.0568083115, -0.00643972307, -0.0640973076, -0.0163043495, -0.0501415469, 0.0111758346, -0.0318857431, 0.0779858306, -7.333950e-03, 0.0129746851, -0.0127140423, 0.0825792551, 0.0132519314, -0.0622551999, -0.0234007575, 0.0344664343, -0.0125323124, -0.018413268, 0.0576787479, -0.0268820282, -0.00839287508, 4.982170e-02], [-2.971160e-02, 0.0347876512, 0.0736564398, -0.00220818724, 0.0245314874, -0.0160910059, 0.0698747635, 0.0322288498, 0.0702398419, -0.0254766643, -0.0146504771, -0.0450287908, 0.0048473659, 0.0395381302, 0.0774734094, 0.0526662879, -0.0256327242, -0.0367633253, -0.0632209852, 0.0526797585, -0.0600574911, -0.0617556125, 0.022902241, -0.0622584633, 0.0475288741, -0.0682731568, 0.0432584472, 0.0556482486, 0.0350199416, -0.00975134223, 0.0603561215, -0.0276067574, 0.0339337923, 0.0520322919, 0.0490040593, -0.0332015753, 0.0728293359, -0.0393529199, -0.0143485125, -0.0311980583, 0.00188336929, -0.0193194263, 0.0805007517, 8.050200e-02, -0.0380591787, -0.0328714103, 0.00999226607, -0.0737487599, -0.06319011, 0.0331962369, 0.0667571351, -0.00869422964, 0.0170051679, -0.0290900227, -0.0739229918, -0.0251884013, -0.0109519074, -0.071198307, 0.0690898746, 0.0361301824, 0.0580004901, 0.031567283, 0.0121233864, 4.301250e-03, 0.0563640222, -5.443340e-02, -0.0156244356, -0.0537484325, -0.0110216727, -0.070254758, -0.0429811254, -0.043706011, -0.00931584183, -0.0459587574, -0.0649611949, 0.00307900971, 0.0334172063, 6.978710e-03, 0.0251644086, -0.0492650084, -0.0569018312, -0.0031649149, -0.0143041993, 0.0732901469, -0.0275258571, 0.0141071761, 0.076380521, 0.0795313492, -0.0268726069, 0.0716366246, -0.0725378469, -0.0276993234, 2.23385694E-4, -0.00324161025, -0.0532006808, -0.0254488792, 0.0177813731, -0.0106803849, 0.00675518904, 0.0379698202, -0.0364151262, -0.0154938828, 0.0250108223, -0.0203937907, 0.0783315375, 0.0374363065, 0.00636940217, -0.0503412448, 0.0764184818, 0.0422515646, 0.0140790027, -0.0196139105, 4.228420e-02, 0.0461154766, -0.0427096821, 0.039766565, -0.0346734673, -0.0486912355, 6.931610e-02, 0.00605471199, -0.00318995025, -0.065803729, -0.0255236477, -0.0405612513, -0.037190821, -0.012922301, -0.0632437169, -0.00956962443], [-0.0786404311, 0.0273745432, -0.0439583659, -6.756670e-02, -0.0662162676, 0.0338278115, -0.047001034, -0.0164492596, 0.0750875473, -0.00391175924, 0.0604845397, 0.065091148, 0.0721733943, 0.00236303755, -0.0721239075, 0.0687193498, 0.066027306, -0.0578387864, 0.0697108805, 0.0441241451, -0.0236044452, 0.0108122574, 0.0714367256, 0.0136396969, 0.0131374039, 0.0174542759, -0.0670291334, 0.0665731058, -0.0458689779, -0.0610544793, 0.0771579668, -0.0598772429, -0.0167327374, 0.0594914705, -0.0627863109, -0.0598445684, 0.0594100431, -0.00270121149, -0.00566692185, -0.079617843, 0.0139512438, 0.0386353917, -0.0624099746, 6.036400e-02, -0.0376265198, 0.0380501598, 0.00874557439, -0.0170974825, -0.0776573345, -0.0480380543, -0.0548560098, -0.0495596528, 0.0111961756, -0.0221448485, 0.0710918903, 0.00188583694, -0.00221667741, -0.0428623557, -0.0118539929, 2.523660e-02, -0.0409077518, 0.041526176, -0.0314406268, -0.0176415443, 0.00517923897, 0.0564946309, -0.0595996045, 0.00244931132, 0.0104020722, 0.0708489269, 0.00426398776, 0.00843573827, 0.054987289, -0.042336788, 0.0374460444, 0.0507045239, -6.541740e-02, -0.0506252348, 0.0729949474, 0.0411210209, 0.0195745695, -0.0641859919, -0.0615288243, 0.00493598729, 0.0379439592, -0.0777217447, 0.0250216927, 0.029278161, -0.018608721, 0.0740638748, -0.0649798736, -0.0212915149, -0.0127518326, -0.032491032, 0.0645261183, -0.025989268, -0.0687571391, -0.0310132615, 0.0432234742, 0.0289828107, -0.0345571376, -0.0281160772, 0.0226658396, 0.0638069659, -0.0335429758, 0.0138984742, -0.0777505114, -0.0707077384, -0.0670868829, -0.0473673381, 0.0731673613, -0.0551988184, 0.0362259783, -0.0267873146, -0.048082713, -2.62973772E-4, 0.0429478213, -0.0805714875, 0.030181231, -0.00385016459, 0.0718464851, -0.0573582947, 0.0540758818, -0.00553308381, -0.043354407, 0.07827463, 0.0494954288, -0.0408414081], [0.0317886546, 0.0276606306, -0.0379427746, 7.053280e-02, 0.0226127207, 0.00506376429, 0.0191800259, 0.0149882445, 0.0226336457, -0.0202590916, 0.0457329676, 0.0249980651, 0.0242292527, 0.0775947943, -0.0264832843, -0.0272346716, 0.0294909142, -0.0299026333, 0.0522483177, -0.0303352419, -0.0224371664, -5.125340e-02, -0.00221116911, 0.00660375739, 0.0616097115, -3.658200e-03, -0.0361261591, 0.0618900806, 0.0167541672, 0.0767532811, -0.0437025279, 0.0482427031, 0.0179056618, 0.0451455116, -0.0365960412, 0.0197362602, -0.0234533884, -0.0503796674, -0.0357489027, 0.0647811368, -0.0164912716, 0.042652864, -4.951570e-02, -0.0300464146, 0.0048716194, -0.00897369906, -0.0338711515, -0.0577484965, -0.0238531269, -0.0293259807, -0.0560841337, 0.076548174, 0.0269403178, -0.0626252517, 0.0240274966, 0.0641759038, 0.0787705108, 0.0343144909, 0.00429347577, -0.0649678186, 0.0366723724, -0.0600860082, 0.0475172587, 0.0451348573, 0.0193943866, -0.0742749869, 0.021676898, -0.0659061745, 0.071879752, -0.0682132691, -0.0394435488, -1.209440e-02, 0.0383663438, -0.0380997658, 0.0464489087, -0.031886749, 0.0316686817, -4.66137339E-4, 0.0201740265, -0.020223774, 0.0761634782, 0.0396205597, 0.0734390393, 4.468930e-02, 0.00299635087, -0.0229300912, 0.0278080758, 0.0113149006, -4.172870e-02, -0.0765894279, -0.00268497178, -0.0234286096, -0.0465288572, 0.015822025, -0.0476473272, 0.0131273502, 0.0166186597, -0.0358245112, 0.073431246, 0.0369734131, 0.00635624724, 0.0655673072, -0.0347468182, 0.0437986068, 0.0125835268, 0.0389788821, 7.389730e-02, 0.0559757315, -0.0227272082, -0.0391328335, 0.0674630627, -0.0523963124, 0.0565400682, -0.0722306594, -0.0385894477, 0.0784186199, 0.0315948799, -0.0678582564, 0.0666050836, 0.0197586846, -0.0598863587, 0.072132498, 0.0597229525, 0.0686084479, 0.0746220872, -0.0190008283, -0.0131408218, -0.0393604152], [-0.0490906276, 0.0801102444, -0.0520089865, -0.0164137054, 0.051271826, 0.0437237173, 0.0469863638, 0.0442800522, 0.0309584737, -0.0647561699, -0.074827604, 0.00940985698, 0.0419715419, -0.0338768773, 0.0295130629, 0.0651652068, 0.0437664129, -0.0232821982, -0.041267246, 0.0665962324, -0.0148364259, -0.0361599922, 0.0585136637, 0.0251846239, 0.0264034178, -0.0317915268, -0.00860413722, -0.0157013349, -0.0370759405, -0.0433207899, -0.0386057273, -3.30139155E-4, 0.0597605333, -0.0137533033, 0.0162560958, 0.0291597433, 0.0518854037, 0.0399525948, 0.0179909803, -0.00407283287, 0.0244477615, -0.0136594074, -0.00437241094, -0.0197018906, -0.0374769717, -0.0694968775, -0.0432510339, 0.0476088189, 0.0360612161, 0.0302627273, -0.0569904596, 0.00737305824, -0.0236118976, -0.0686136782, 0.0142789148, 7.954940e-02, 0.0527178273, 0.0562234409, -0.0597893372, 0.018385563, -0.0464717187, 0.0575339235, 0.0348888896, -0.0334309824, 0.0705602691, -0.00546210306, 0.0741798654, -0.0409859568, 0.0486791432, 0.039221637, 4.206720e-02, 0.0186395682, -0.00267980108, 0.067803286, -0.0601521619, -0.079224728, -0.00807529781, 0.0277106296, -0.0273878425, 0.0427168496, -0.0395148098, 0.0223752894, -0.0484791622, -0.0473256074, 0.0411964022, 0.0138877304, -0.0678585172, -0.0540044606, 0.0344547629, 0.0129923923, -0.0804792046, -0.00584892603, -0.0609543249, -0.00973720662, -0.0650199577, -0.0733402073, -0.0433495082, 0.0663800612, -0.00886129867, -0.0589961484, 0.0369722247, -0.0215634536, 0.0797204151, -0.0277846977, -0.00272456533, 0.0606503971, -0.0662781224, 0.0111918645, 0.0480701849, 0.0722226053, -0.0440260917, -0.0760635659, 0.00646601245, 0.00820270926, 0.051910881, -0.0688460842, -0.0778788179, 0.0717015639, -0.0297058932, 0.03483418, 0.0785149931, 0.0661664531, -0.00749203563, -0.0339023918, -7.490680e-02, -0.0169404615, 0.0206368696, 0.0538051613], [-0.0193308741, -0.063438572, 0.00454494776, 0.00247645308, -0.0771728531, -0.0749829262, 0.0291936472, -0.0218214933, -0.029669499, -0.0491768755, -0.0542005189, -0.00260382239, 0.0745439529, 7.69744045E-4, -0.0249348693, 0.0496997423, -3.364420e-02, 0.0254118871, -7.57402275E-4, -0.0180327874, 0.0288845226, -7.82879535E-4, -0.0391187482, -0.00117716682, 0.051735457, 0.0449167565, 0.071635887, -0.0426398627, 0.0799553692, -0.0625013635, -0.0803558602, -0.0288357865, -0.0578104369, 0.0507808402, -0.0144976042, 0.0701584741, 0.00379509921, -0.0199058969, 0.0274587609, -0.0704809204, -0.0696633086, 0.0365580507, 0.020467801, 0.057411477, 0.0565626957, -0.0615123585, -0.0386717506, -0.0598739535, -0.0126439454, -0.0378122143, -0.0590410084, 6.521210e-02, 0.0497469865, 0.0280670654, 0.0427540392, -0.0680729076, -6.332190e-02, -0.0142406737, 0.0419263504, -0.0529132299, -0.0352460332, 0.0824470818, 0.0106040388, -0.0410611182, -0.0053487448, 0.0211480148, 0.0217186734, -0.00959742814, 0.0187013224, -0.00428198557, -0.0344025679, -0.0532209948, -6.309930e-02, 0.0341844633, -0.0379134603, -0.0467729755, -4.459070e-02, -0.00751138944, -0.0105591258, 0.0674138144, 0.0232872739, 0.0454144701, -0.0139435465, -0.00320797716, -0.0214830358, 0.0610915124, 0.0698669702, -0.0758484899, 0.080773212, 0.0332716331, 0.0468927175, 0.0118369497, -0.072274439, -0.0128512038, 0.0289153345, -0.0496760942, -0.0384935848, -0.0436696969, 0.0406139195, 0.0257379878, -0.0778797492, 0.0816891789, -0.0226605833, 0.0530443564, 0.028919939, -0.0387203284, -0.0314817242, 0.00285683316, 0.00696293171, -0.00723652541, 0.0187227782, -0.0205992982, 0.0677718669, 0.0488576144, -0.0558087192, -0.0300513934, -0.020476507, -0.0281171277, -0.0779979154, 0.0353548788, 0.0544513278, -0.0290573649, 0.0275170263, 0.0585057065, 0.0410138667, 0.00300408527, -0.0531869158, -0.0770810172], [-0.0411750562, 0.0640518814, 0.0757471845, 0.00657723704, 0.0641182736, 0.0505198464, -0.00401634676, 0.0664102435, -0.0423293486, -0.0671621189, 0.0594699271, -0.0183880888, -0.00143720454, -0.0626026392, -0.0632805899, -0.0124293407, 0.00249526207, 0.0471820571, 0.00359928422, -0.00310800807, 0.0589512251, -0.0542889796, 0.0120214056, -0.072765328, 0.0208501518, 0.0428576134, -0.0365314297, -0.0413026251, -0.0600180924, 0.05397911, -0.05629934, -0.0529227853, 0.0584953241, 1.870680e-02, -0.056966681, -0.0819608494, 0.0497943349, 0.00296973111, -0.0588741861, -0.0333880186, 0.0782329887, 0.0142827416, 3.756950e-02, 0.0402863584, -0.00475063827, -0.0327542946, -0.0620726421, -0.0179084297, -0.061866153, -0.0507900231, 0.0354117155, -0.0783064737, -7.851450e-02, 0.0493720919, 0.0434938706, 0.032228142, 0.0360753462, -0.05503764, 0.028064223, 0.0610573813, 2.741760e-02, 0.0234382711, -0.0160151534, -0.0554925241, -0.00537757762, 8.855350e-02, -0.0847953185, -0.0229147803, -0.00526173972, 0.00241902564, -0.0118366946, 0.010274481, 0.0194293186, -0.00140519754, -0.0463300087, 2.639100e-02, 0.0732838586, -0.0162627269, -3.952240e-02, -0.0587788336, -0.0801937133, -0.0152911274, -0.0232135709, 0.0346418843, 0.0274198502, -0.00646665832, 0.0433819033, -0.0737982243, -0.0316813812, -6.12727774E-4, 0.0690142736, 0.00907274056, 0.0354410224, 0.0622870288, 0.056655746, 0.00692679966, -0.0176146608, 0.0241019912, -0.0296744853, -0.0362769738, 0.0766890272, -0.0254050829, -0.0393876359, 0.0551378764, 0.0238903798, 0.063309513, -0.0477721766, 0.0737083256, 0.0508829504, -0.0428257696, 0.037774317, -7.724650e-02, 0.0381435975, 0.0548577309, -0.0347026251, 0.00924278889, 0.0692625195, -0.0147797531, 0.0106370673, 0.00747635076, 0.00109753374, -0.0395227037, -0.0643274486, -0.0382754952, 0.03987021, 0.0049622627, 6.658620e-02, 0.0463863239], [0.0659546778, 0.0317584164, 0.0647040829, -0.0514994711, 0.0486946329, -0.0364251882, 0.0106615461, -0.0515006222, 0.051213026, 0.0665485933, -0.00800834317, 0.0331739374, -0.0577494204, -0.0179383811, 0.0104923537, -3.239730e-02, -0.0457336716, 0.0144812958, -0.0050627524, 0.0645111576, -0.0572575182, 7.450000e-03, 0.00971360598, -0.0184137393, 0.0389253348, 0.0387116969, -0.0192634314, -0.0113861961, 3.851190e-02, 0.0835465044, -0.0515438356, -0.03062208, -0.0402303115, 0.0462560207, -0.0484712757, 0.0169733539, 0.0582405254, -0.0569307767, 0.0746443421, 0.0352763347, 0.0138236852, 0.0310974829, 0.0318498053, 0.0401251912, 0.0481577516, 0.0179829802, 0.00542436587, 0.0773510858, 0.0392490365, 0.0522123799, -0.0358531363, -0.0421746187, 0.0248031933, -0.0323793106, 0.0603333227, -0.0016924527, -0.0730708092, 0.0530678332, -0.0520841144, -0.0283633266, 0.0289219711, 0.00601976039, 0.0302331336, 0.0610771701, -0.0262709893, 6.319480e-02, 0.034444306, 0.0346303508, 0.0453232937, 0.0390701592, -0.0472908281, -0.0304517299, -0.0739781856, -0.0875435695, -0.00673567178, -5.943040e-03, 0.0708568692, 0.0239910912, -0.035105139, -0.00890015811, 0.0438424721, -0.054427959, -0.034435913, 0.0272516776, -0.0395848826, 0.0569059439, 0.00978929176, 0.0662263333, -0.00394303538, 0.0194137841, 0.00958614331, -0.0723343119, 0.0292874612, 0.0143774617, 0.0616079234, 0.00157506473, -0.0010366817, 0.0823378116, -0.0278229825, -0.0581053235, -0.0489239357, 0.0267708171, -0.0350173935, 0.0826725363, 0.0315916426, 0.0494574532, -0.0325378925, 0.0804898813, -0.0371250398, 0.0513976254, -0.0897877439, 0.0475431755, -0.0380569883, -0.101894379, 0.0526015833, 0.0549428388, 0.0441827402, -0.0469237119, 0.0180360954, -0.0867711455, -0.0138778007, -0.0112251798, -0.0609353855, 0.0583004095, 0.0634501502, 5.247420e-02, -0.0131193716, 0.0618441589], [-0.0373816118, -0.031318862, -0.0112993335, -0.0307260808, 0.0458448604, 0.0543134846, -0.029618999, 0.035636045, 0.0908063054, 0.00601895154, -0.0892687663, 0.0547481477, 0.0242016502, -0.0874403119, -0.0453459397, -0.0208609458, -0.0124781542, -0.0188164134, -0.0437163711, -0.0266581457, 0.0274388846, -0.0652995631, 0.0805324614, -0.0178313106, 0.0210263748, 0.046976734, -0.0535804071, -0.00821534079, 0.00807241071, -0.0196713023, -0.0334349573, 0.0211053919, 0.0270023663, 0.0158653054, -0.0468911752, -0.0451480895, 0.0450436175, 0.00724444259, 6.796180e-03, 0.0325081423, 0.0824798569, -0.0373319089, 0.0172392949, -0.0249803942, -0.00232157926, -0.103443414, 0.119896941, 0.0938038677, -0.0166928153, -0.0174426492, 0.0475101508, -0.0307275057, -0.0427603684, -0.0281747915, -0.0304054059, 0.00914295949, -0.00590262888, -0.00514617236, -0.0353417061, -0.0741793886, -8.4716489E-4, 0.0391784348, -0.0366749354, 0.0409377441, 0.0624738447, 0.0219329316, -0.0376536809, -0.0229252577, 0.0161045473, 0.0441868864, -6.881140e-02, -0.00680868467, -0.029550476, 0.0247731525, 0.0773213431, -0.074211888, -0.0540675335, 0.0381147638, 0.00758298906, 0.0397156775, -0.0118956715, -0.096742399, -0.0042139031, -0.00530584343, 0.0353194326, -0.0161783565, -0.0219206437, 0.019008629, 0.0104768313, 0.00444334652, 0.0267885141, -0.0680359825, 0.0386309624, 0.0728092939, 0.00114020938, 0.0400511809, 0.08430624, 0.053721264, -0.0123618348, -0.00640350627, 0.0182875153, -0.023446437, -0.110124886, -0.0454803556, 0.0137599306, 0.0320646763, 0.036993742, -0.00968794152, -0.0655759647, 0.0204632431, 0.0195239466, 0.0632927641, -0.0776574463, -0.10951443, 5.080300e-03, -0.00108812854, -0.112555467, 0.0176727027, -0.0486952886, -0.034967389, 0.0650463998, 0.111005135, -0.0717807263, 0.104878291, 0.0494114272, -0.0598752498, 0.0103853839, -0.0562029034], [-0.0672106296, -0.0589769036, -0.048520416, -0.0474323332, -0.0343488604, -9.553530e-02, -0.0571656898, -0.00209520082, 0.0678734332, 0.08282502, -0.0432715863, 0.0303064641, 0.0145827299, -0.0805314406, 0.0365959778, -0.0604261532, 0.0521858633, 0.00711228047, -0.0342521444, 0.0154964505, -0.0209867824, -0.0345907398, 0.0723414198, -0.0316173472, -0.0213668477, 0.0999255776, 0.00369363953, -0.0534543581, -0.00480009895, -0.0765915811, -0.0596054792, 0.0170566291, -0.0695500075, 0.0663835183, 0.0844272747, 0.0105527071, -0.0302327871, -0.0340104625, -0.00376878562, -0.0301444735, -0.0154728647, -0.0177434701, 0.0832347199, -0.0182767902, 0.103994779, -0.00634227879, 0.0716241226, 0.09258347, -0.0654532686, 0.0785401687, -0.0563026331, -0.0999600812, -6.267790e-02, -0.0882321149, -0.0297722947, -0.0260717012, 0.0609544851, 0.0731010512, -0.0364699215, 0.00625048764, -0.0579050668, -0.0843664705, 0.0506825931, -0.00174021325, 6.998120e-02, 0.0198070984, -0.0247852709, -0.0610488541, -0.0150055243, 0.0724179149, 0.0688400865, 5.911440e-02, -0.0626199916, -0.0839589834, -0.0619881824, 0.0454147197, 0.0927077159, 0.0748920738, -0.0461496823, -0.0503526293, 9.41502163E-4, -0.0428176336, 0.0469275266, 0.0556891374, -0.0148249287, -0.00542366458, -0.0283529572, -0.0581412129, -0.00541032385, -0.0927951037, -0.00382550946, 0.00477904547, 0.0321527384, -0.0585534871, -0.0270034559, -0.014795783, 0.108584657, 0.0101876445, -0.0410147831, 0.0133008603, -0.0442479625, 0.0208761785, -0.0604642481, 0.0663988143, -0.0161921848, -0.0125340577, -0.048402179, -0.00754627213, 0.057645604, 3.39262886E-4, -0.0839387997, -0.0661446825, -0.00147711788, -0.0305559095, -0.0449310616, 0.0943971648, -0.0551286563, 0.0270052887, -0.047665257, -0.00536923343, 0.0802251175, 0.0239426456, -0.0673225373, 0.0477950908, -0.0604522862, -0.0751155615, -0.00767319743, 0.0263102092], [0.00581199443, -0.0375653505, 0.0601584353, -0.0674062818, -0.0147330202, -0.0820613801, 0.0276777595, -0.0482463613, 0.0568287335, 0.0380024053, 0.00855380855, 0.00614229543, -0.0274089761, -0.0651044771, 0.00564401457, -0.0287478231, -0.0672300159, -0.0527518094, -0.0516843684, -0.0329053812, 0.0345627107, 3.32449039E-4, 0.0572011359, -0.0112842172, 0.0868092328, 0.0655692443, -0.0376443937, -0.00931155495, 0.0462389737, 0.0474395715, -0.0158490501, -0.0460993387, 0.0906601771, 0.0599701628, 0.0552990101, -0.0472113341, 0.0117022367, 5.934950e-02, -0.0224976595, -0.0411045402, -0.0596626662, -0.0373676531, -0.0290921889, 0.0552919321, 0.0945628508, -0.0628798455, 0.0192458797, 0.042503003, 2.24348696E-4, -0.0570635237, 0.0692511648, -0.00270999572, 0.0631126314, -0.0107448585, -0.0864945799, -0.072070308, 0.0280476715, -0.04256225, -0.0296885334, 0.0397571288, -0.0275750346, 0.0111373784, 0.0724423304, 0.0152248163, -0.0810254439, -0.00638421299, 0.0588344559, 0.068509452, -0.0169621594, 0.00517027685, 0.0745188743, 0.115400828, 0.0250305962, -0.071397841, -0.048328653, 0.00857998338, 0.0143124619, -0.0714461431, 0.101694591, 0.0538196415, -0.0457124822, -0.0113489246, -0.0285612904, -0.0796594471, 0.0704048499, 0.0121286195, 0.0498140976, -7.368410e-02, 0.0423806086, -0.0395464189, 0.00890278537, 0.0606213547, 0.0116127608, 0.0497328602, 0.0504018925, 0.0390022956, -0.00577579392, 0.00162280537, 0.0393566787, 0.00447005173, 0.082316704, -0.0468695499, -0.0709751248, -0.0496915802, -0.0173601955, 0.0812859311, 0.059063755, -0.0557975955, 0.0894534066, 0.0511592776, -0.00731464708, 0.0239917487, 0.0407101586, -0.0329338536, -0.131105185, 0.0937555357, -0.0609769784, 0.0499178283, 0.0127634872, -0.0299900454, -0.0190586336, 0.0112365773, -0.0265427865, 0.0410779268, -0.0438451841, -0.0620680526, 0.00593530759, 0.0066615073], [0.0711430833, 0.0613434352, -0.0399763361, -0.103175044, 0.0622229949, -0.0948970988, 0.0601203516, 0.0586809106, 0.0363455638, 0.0877037867, -0.0311485678, -0.0481329858, 0.0114278384, 0.00116072444, 0.0823049619, 0.041348163, -0.0159406774, -0.0793026313, -9.591060e-02, 0.00537758181, -0.0642748475, 0.0822957605, 0.0750102922, -0.0267086718, -0.0473216102, 0.0864688456, -0.0220297892, 0.0217187945, 0.0199714191, 0.0778052956, 4.540940e-02, -0.0400650278, 5.150160e-02, 4.517780e-02, 0.0951457173, -0.00488671707, 0.0332241654, -0.0552845336, -0.0612447746, 0.0376163535, 0.0789698809, 0.0200759955, 0.016303204, 0.0465446264, 0.0568305701, 0.0473312326, 0.00607706327, 0.0721264854, -0.0416432396, 0.0924452543, 0.044759348, 0.0680745468, 0.0390754342, -0.0483373217, -0.0661824197, 0.0484424494, -0.025584098, -0.035953436, 4.878670e-02, -0.0981728285, -0.0311962273, -0.0476517193, 0.00709011778, 0.0930803716, 0.0708456039, -0.00725829741, -0.00173765398, -3.858720e-02, 0.0639001429, 0.0520167612, 0.0146259973, 0.0399234667, 0.0361136496, -0.0187721234, -0.072563991, -0.0590774417, -0.0364139602, -0.0510358326, 0.020533476, 0.0187091604, 0.0404496342, 0.0557931624, -0.0620256997, -0.0595900752, 0.0520051345, -0.0759253949, 0.0419920832, -0.039762985, 0.0113529274, -0.00454933196, 0.085073635, 0.106082961, 0.0060153571, 0.0609102063, 0.025855653, -0.0415572934, -0.0142800743, -0.0320323892, -0.0715971589, 0.024507761, -0.0495964028, -0.0674454942, -0.094918713, -0.0814478397, -0.0671380609, 0.00861379317, 0.0493916459, 0.00893785431, 0.0903932824, 0.0567355901, -0.0572963692, -0.0398159809, -0.036324624, -0.0575259142, -0.0451777354, -0.025817981, -0.100079633, -0.0666996837, 0.104314879, 0.0532128178, 0.107255444, 0.0898425579, -0.0754745826, 0.0909608826, -0.0350286476, -0.0149342529, -0.0453755297, -0.0909382477], [0.0460755117, 0.0101709962, -0.00563266873, -0.04120785, 0.108652629, -0.0148390606, 0.0634072274, 0.0182702057, -0.0446446948, 0.0594833046, -0.0380127802, -0.0154064503, 0.0329823568, -0.0557148457, -0.00326226838, 0.0080671776, 0.011585705, -0.00152963982, 0.0207701661, -0.0176638979, -0.0294502918, 0.107318267, 0.00713134976, -0.00915605388, 0.0187407788, 0.00956343394, -0.0674925521, -0.0257950574, 0.0288058668, -0.0515893176, -0.0390678123, -1.651810e-02, 0.0454253517, 0.0341274701, -0.0261163395, 0.0245285407, -0.0198365971, -0.0115909697, -0.0389794558, 0.0314424448, 0.00168284017, 0.0158987641, 0.0195575505, 0.095745787, 2.473360e-02, 0.00530861178, 0.051481612, 0.0114883576, -0.0578593649, 4.498490e-02, 0.0524530821, 0.0428922847, -0.0382568501, -0.102332644, -0.0465832092, -0.0956078395, 0.0652759448, 0.0589394793, -0.0556980744, -0.0319338664, 0.0215395819, -8.182130e-02, -0.0276186597, 0.0853775591, -0.0221775826, -0.108714566, -0.0509482138, 0.0317888334, 0.004580779, -0.0508719832, 0.0763710141, -0.0311997309, -0.0116725788, 0.0408894978, 0.0179401264, 0.009066049, 0.0815786495, -0.0626424327, -0.0507957526, 0.0245787501, 0.00822124351, 0.00683556544, -0.00847891066, -0.016388936, 0.0486359335, 0.0347166657, 0.00771512231, -0.0451934673, 0.0264621209, -0.0151742762, 0.0054117809, 0.0334363468, 0.0234765988, 0.0603764392, 0.0187565591, 0.0284846779, 0.0363745391, -0.0248364341, -0.0269830674, 0.0641658902, 0.0521054603, 0.0376028456, -0.0554138087, -0.00853051245, 0.0258373804, 0.0236430746, 0.0513576269, -0.00103106978, 0.116681337, 0.00758470735, 0.05092654, -0.0493115075, 0.0469560288, -0.0663197637, -0.0747650787, 0.0447947271, -0.0421650335, 0.0087581966, -0.0175684728, -0.040541105, 0.00739544863, 0.111905657, 0.0482305773, -0.00544552132, 0.0372854732, -0.0106951157, -0.0680098161, -0.0316288248], [0.00928743742, 0.0864381268, -0.055741515, -0.0395772457, -0.0193034261, 4.87906276E-4, -0.0269761495, 0.038331639, -0.0638635382, 0.0717562959, 0.0180482939, 0.111393824, -0.0308883879, 0.0414192416, 0.0123954518, 0.0499805883, -0.0759150535, -0.0221405588, -0.0836858823, 0.0131451162, 0.063333042, 0.11691013, 0.129684553, -0.0501122698, 0.0989391207, 0.0469504185, 0.0678407326, -0.0799718797, -0.0761509389, 0.0462817214, 0.0285348538, -0.0742426068, -0.0249496326, 0.0134318015, -0.0242512766, 0.096542716, -0.0504570194, -0.0164073128, 0.112242423, 0.0803766176, 0.0995969474, -0.0106827496, 0.00151066098, 0.0509930365, 0.0284356717, -0.08495529, -0.0105214445, 0.011486697, 8.786290e-02, 0.0216971412, -0.0615434907, -0.0418967307, 0.0281082951, -0.075243853, -0.0328449309, -0.0903985649, -0.00865460745, 0.020236332, -0.0339579396, 0.007271023, -0.118612587, -0.0527659692, -0.0375187695, -0.01033876, -0.0354411677, -0.0964478552, 0.045391649, 0.0111627728, 0.0147820916, -0.00824077334, 0.0976302474, -0.0508100875, -0.0526766814, -0.0564616509, 0.0561298057, -0.0114934258, 0.0352263711, 0.0300233942, -0.084047094, 0.0136328461, -0.0564665385, 0.0450106487, 0.0376608446, -0.00549360877, 0.0739544481, -0.0687324702, 8.226750e-02, -0.0393566154, -0.0261473749, 0.0508287176, 0.0830372273, 0.0799150243, 0.0486836433, 0.0347279683, -0.0236713868, 0.0487178825, 0.0212346129, -7.151030e-02, 0.0203624777, 0.00931460317, -0.0389404669, -0.0892678052, 0.0259228759, 0.0229035877, -0.0600500517, 0.101531409, 0.15167214, -0.0335395187, 0.00173019886, 0.0708840638, 3.222080e-02, -0.0105020022, 0.0575060658, -0.054600589, 0.0241259821, 0.0975966304, -0.14865467, -0.0807889774, 0.0869766548, -0.0603070259, 0.0315219834, 0.060034737, -0.00505926972, -9.94007918E-4, 0.0190622341, 0.010404177, 0.0236501303, -0.0422478765], [0.0405637883, 0.0303276125, -0.0459024757, -0.11356435, 0.0936037078, 0.0166911222, -0.0486291125, 0.00376798445, -0.00662953919, -0.0546382554, 0.0791893675, 0.0930408909, 0.0267672669, 0.0044755009, 0.0389116593, -0.00549650053, -0.0695390701, -0.0322671384, 0.0232949089, -0.017123621, 0.0669663101, 0.0592123233, 0.0322532058, -0.0509875938, 0.0736868083, -0.0556510091, 0.0586370192, -0.0562621281, 0.0720168874, 0.0250335243, -0.012028263, 0.00274412869, -0.0186142568, -0.0115361018, -0.0765840635, 0.0209389236, -0.0315889865, 0.0149773639, 0.0728947595, -0.04266119, 0.00466660969, -0.0278960653, 0.0747553706, 0.0557189286, -0.111432984, -0.0348995253, 0.0933217629, 0.10995961, 0.0116694951, 0.0213425569, -0.0222675018, 0.00559896091, -5.737900e-02, -0.107596934, -0.0468277298, 0.0518046133, 0.0685384795, 0.0477457084, -0.026993081, -0.00638324907, -0.0313069187, 0.00771877961, 0.050334271, 0.0411645509, -0.0191521849, -0.0680142418, 7.45715166E-4, -0.0168791041, -0.0336807929, 0.032227803, 0.0499856919, -0.0309070572, -0.0191245321, 0.0750355795, -0.0546982922, 0.0683552399, -0.0257874466, -0.00360887614, -0.101559348, -0.0434213579, 0.0105873868, 0.0723884627, 0.0388877355, -0.0289585777, 0.0371819809, 0.0111660287, 0.0532660112, -0.103056274, -0.0623732469, -0.072767958, 0.060110759, 0.116275758, 5.810690e-02, -0.109477088, -0.0291662365, 0.105770662, -0.0532949455, -0.0539925508, -0.0079298038, -6.883750e-02, -0.0168847572, -0.0637183636, -0.0232288092, -0.0249348637, 0.0018297846, 0.079157725, 0.099491097, 0.00473471778, 0.11151211, 0.019250514, 0.00306491228, -0.0481520891, 0.0418435484, 0.0427864306, 0.0145675195, 0.00371725927, -0.0721725449, -0.103766985, -1.99021975E-4, -0.0951042249, -0.0218397304, 0.0323493555, 0.0284510311, -0.0548440814, 5.442000e-02, -0.0844846889, 0.10566441, -0.0191186443], [-0.0335451178, 0.0535242744, -0.00219235173, 0.0558837019, -0.0047704149, 0.00196730811, 0.00313207251, 0.0131680081, 0.04285625, -0.0152199706, 0.0184623133, 0.0538634546, -0.10022337, 8.376010e-02, -0.0436727069, -0.0184085462, -0.0156802461, -0.0729147866, -0.0647053942, -0.0744921044, 0.0129963495, 0.0997568815, 0.0567405298, 0.0807199478, -0.0468324572, -0.0556545742, 0.143731311, -0.0728251263, 0.0798884257, 0.0119445669, 0.0556678027, 0.0484844446, -0.00265036337, -0.00735918479, 0.032062456, 0.0221300721, -0.0959960371, 0.0245044623, -0.0370968357, -0.021090826, 0.0708143041, 0.0537838042, 0.00643636379, -0.0377661772, -0.0337040797, -0.0332693569, 0.0564836673, -0.057806313, 0.0220102686, -0.0575022101, 4.036960e-02, -0.0507139564, -0.0563223846, 0.00734898448, -0.01721422, -0.0450978652, 0.0289757438, -0.0286892951, 0.0128619196, 0.147286519, 0.101744115, -0.0974086597, 0.108602427, -0.0238988176, -0.0778058543, 0.114117347, 0.0312068649, 0.0507962182, -0.0182649214, -0.131531417, 0.00642245496, 0.0693140402, 0.0383591838, 0.0320767388, -0.0417010933, 0.103777073, 0.0380006805, -0.00814501754, -0.0447626635, -0.0034836859, -0.0793971195, 0.00103620894, 0.0110441446, 0.082319729, -0.0300934725, -0.100373149, 0.0652898848, -0.0395063423, -0.0322953574, 0.0640801489, 0.0449192673, 0.0736273751, 0.0710285604, 2.656200e-02, -2.841630e-02, 1.275470e-01, -0.0917099416, -0.0687053278, 0.0399994962, 0.0412564874, 0.0533950776, -0.0904506445, -0.0804907903, 0.0596192852, 0.0440515764, 0.0720597357, 0.0901944637, -0.0655003339, 0.0926450788, -0.077240482, 3.123710e-02, -0.0728937909, -0.0553177521, 0.122572966, 0.0175703578, 0.0133946473, -0.00131913763, -0.0716728866, 0.049108021, -0.026797317, -0.0769313871, -0.0442543961, -0.0207390245, -0.123761505, -0.0569707416, -0.0535747334, 0.0961937978, 0.0371981487], [0.0315228067, 0.030722484, -0.0534861311, 0.0507732108, -0.00990749057, -0.0165163875, 0.086840175, 0.103312157, 0.0365604758, -0.0066934172, -0.0128516732, -0.0327232294, 0.0250745825, -0.00377819291, -0.0711036474, -0.0751768052, 0.0419458859, 0.0645090565, -0.109637529, 0.0514352247, 0.0387829915, 3.269770e-02, -0.00494236033, 0.0621454343, -0.0390325934, -0.030085532, 0.135334939, -0.0470069721, -0.0671161115, 0.0319220871, 0.0520601198, 0.126184806, 0.0485076532, 0.085219562, -0.0746367425, -0.0578248501, -0.0822851211, 0.14666912, 0.0129117966, -0.0582853965, 0.0396689363, -0.0749213472, -0.0629236102, -0.0424045287, -0.0270817727, 0.0750787556, 0.0657461807, 0.0360189267, -0.0137979817, 0.00164395792, -5.68679476E-4, -0.00505885947, 0.0537504293, -0.0221679527, 0.0357372239, 0.0354790501, -0.0124329776, -0.0749385804, 0.0798895657, 0.0835210234, 0.083151482, -9.017480e-02, 0.103162922, 0.0414235555, -0.0268972293, 0.0667605102, 0.00966202281, -0.0545491837, -0.0841646417, -0.0536955483, -0.00710722432, 2.72043602E-4, 0.0166113935, 0.0375797711, -0.0639638602, -0.0497269668, 0.0262289979, 0.0308873914, -0.140139475, 0.0377416797, -0.031250339, 0.0993674993, -0.00217296369, 0.107271835, -0.024800064, 0.00286273821, 0.00422110176, -0.091559425, -0.0468911827, -0.00202268502, 0.0273615289, 0.0445940308, 0.00745164929, -0.104450464, -5.216490e-02, 0.0548469462, -0.108566388, 1.89155908E-5, 8.805780e-02, 0.018666843, 0.0711433217, 0.0226423983, 0.0269047879, -0.0432079807, -0.0169290043, 0.0321481489, 0.0447838306, 0.0411991253, -0.00828588381, -0.0111391395, 0.0912957563, -0.0440542512, 0.0619869903, 0.0948325545, 0.0188470334, 0.0320654102, 0.0860679597, 0.0341404304, 0.0435580462, -0.079725705, -0.0893862695, -0.0780201777, 0.0533783473, -0.023005683, -0.0114986673, 0.0353159197, -0.0142229628, 0.0191534255], [-0.0815020501, -0.022114113, -0.0455654114, -0.0386422276, 0.023843443, 0.0381884612, -0.0243615769, -3.112450e-02, 0.0264491104, 0.0507503413, 0.016063787, 0.112078756, -0.0137757212, -0.0269128755, -0.0450339168, 0.00937812775, -0.0478763506, -2.300630e-02, -0.0850280672, 0.0379791483, 0.0446430929, -3.372320e-02, 0.0220462512, 0.0436226726, -0.0228637774, -0.00485594058, 0.0278333947, 0.0244566202, -0.00849846099, -0.0712679476, 0.0190493204, 0.00450936379, -0.0528079756, -0.0123910457, -0.0560617074, -0.050238274, 0.0858952328, 0.147820815, 0.0264270529, -0.103774257, 0.0469159745, -0.0176548939, -0.081875056, -8.352410e-02, 0.00802001823, 6.459080e-02, 0.0428544804, -0.028901618, -0.0293893591, -0.0643383562, 0.0101878857, -0.0587450378, -0.0225088671, 0.0911371931, 0.0626336932, 0.0123427091, 0.0019779182, 0.0593292713, 0.0621182173, 0.0917046368, 0.104733951, -0.0886537284, 0.026980117, -0.0262097511, -0.00610517943, 0.0511670932, 0.00492058694, -0.0567756295, -0.0966548473, -0.0353515856, -0.00810173433, 0.0108516589, 0.114412062, -0.0122909732, -0.0631247237, 0.0153395142, 0.0446887352, 0.0552160628, -0.0617838576, 0.0208558049, -0.0346177742, 0.076853767, -0.013369862, 0.0025614528, 0.0116298711, -0.0436342433, -0.00436843792, -0.048334755, -0.0316203944, -0.0414219573, -0.00649264734, 0.00366838905, -0.0264793523, -0.112688288, -0.0159714632, 0.0202525817, 0.0199886113, 0.0369303115, -0.00434929086, -0.0168027319, 0.126067206, -0.0320541859, -0.0663710088, 0.0623131357, 0.077685438, 0.0656652748, 0.00603396352, -0.0132098068, 0.0892026275, 0.0153044434, -0.00305400742, 4.59195522E-4, 9.07097273E-5, 0.122556821, 0.0450741798, 0.0270125642, -0.015162779, 0.0591776669, -0.0733513907, -0.0856253132, 0.0278140027, 0.00322232558, 0.0422393121, -0.14835532, -0.0444617458, 0.0108042201, 0.140395343, -0.0673913583], [-0.0640930235, 0.00879331399, -0.0165574923, -0.00771793257, -0.0843323767, -0.0671170726, 0.0355559476, 0.0595263243, -0.0234020967, 0.0853903666, -0.0128890835, 0.00583502371, -0.030255381, -0.0609139763, -0.0164644979, 0.0306345932, 0.0494633168, 0.0196446385, 0.0151373614, -0.0150615256, -0.0465975702, 0.00282070041, -0.0228033848, -0.0622113086, 0.0331436656, 0.0267866608, -0.0264016949, 0.0628669038, -0.0552086197, -0.0303804372, -0.0536329299, -0.0136771677, 0.028677918, 3.458460e-02, -0.051717151, -0.0432351045, 0.103378303, -0.0195696391, -0.0253176931, -0.0714627355, 0.109932132, 0.0748797357, 0.0131432265, -0.0182602331, 0.0771290361, 0.0321558155, 0.124757446, 0.119235322, 0.0463257544, -0.0282796286, -0.065263316, -0.107888661, 0.0873837396, -0.0346790627, 0.0272230394, 0.0136412932, -0.0616091378, -0.076249443, 0.0142326588, -0.00954269245, 0.0187866893, 0.0366540067, 0.116893508, 0.0140155181, 0.0156731457, 0.0819275975, 0.0604807623, 0.0246957708, -0.0384302251, -0.00961383432, -0.0357631184, 0.0232487731, -0.0610115938, 0.0490947142, -0.0348318964, -0.0163247157, 0.119188733, 0.0699790567, -0.0946126356, -5.66817936E-4, -9.57819516E-4, 0.0787145942, 0.0695159063, -0.0165122021, 0.0957795456, 0.00753464596, -0.0635320768, 8.997750e-02, 0.00415522698, 0.0414536819, -0.0705940351, 0.0519572236, 0.0114386538, 0.0311090983, 0.0669109896, 0.0112140011, 0.0498066507, -0.0520010181, 0.0257672593, -0.0160737876, 0.129562736, 0.0750007853, 0.0365072936, -0.0370992906, -0.0328843221, 0.0640557632, 0.0301101226, -0.0116960667, -4.7539809E-4, 0.0404134579, -0.0313650407, 1.31766365E-5, 0.0113574117, 0.101910748, 0.0218708944, -0.0199013948, 0.0339086317, 0.0580502637, 0.033801157, -7.413340e-02, -0.0265398193, -0.0136756524, 0.0122574484, -0.114758775, 0.0485926531, 0.0581341907, 0.125425696, -8.492540e-03], [-0.0664189085, 0.0492116436, 0.0344041809, -0.0656822249, -0.0665752739, 0.0140879797, -0.0171800219, 0.0122137833, -0.0333216675, 7.351750e-02, 0.0174083877, 0.119153239, -0.0582985431, 0.0635624453, -5.821220e-02, 0.0028067769, -0.00289082597, 0.0411850214, 0.0218370818, 0.0409607254, 0.00672804797, -0.0643143132, -0.00824193842, 0.00962229073, 0.0363076888, 0.039653331, -0.0276037231, 0.00104364066, 0.0251823589, -0.0652390793, -0.0227338839, 0.0330661163, -0.028616298, 0.061829593, -0.0780737847, -0.0911370516, 0.0942005962, -0.0351634324, -0.0183150824, -0.0297649894, 0.0351432636, 0.00315057742, -0.0734252855, -0.0553740785, -0.0259864703, 0.0867371335, -0.0341970399, 0.136607721, -0.0619497708, 0.0333855338, -0.0499750711, -0.0275899041, 0.0172852185, 0.0128769185, -0.0335801244, 0.0156751145, -0.045916494, -0.0616598763, -0.0657201186, 0.0396996588, -0.0479665026, 6.402640e-02, 0.140216991, 0.119035378, 0.0160357375, -0.0186606217, 3.046160e-02, 0.0276715495, -0.0573852248, -0.0411037058, 0.0704561919, -0.0139560038, -0.0833841711, 0.0546852089, 0.0046116095, -0.00578858657, 0.0635023266, 0.0323741585, 0.0284262076, -0.0338086449, -0.04730165, 0.00191886525, -0.0800570175, 0.0372542702, 0.0794123709, 0.00216348912, -0.0488064289, 0.0328425914, 0.017322531, -0.0186907984, -0.0692416951, 0.0173264612, 0.0467513837, -0.0367498919, -0.0512288548, 0.0686994195, 0.0411825478, -0.0240641683, 0.0285564121, 0.0417035446, -0.00993193779, 0.0515309796, -0.0506300554, 0.0818205997, -0.0541087613, 0.0388388224, -0.047803428, -0.0191487689, 0.00160338916, -0.0265544057, 0.0487034284, 0.0584330745, -0.0358118825, -0.0649301633, 0.140879527, -0.0543084033, -0.0517065041, 0.0306502394, 0.0927540064, 0.0292617511, 0.11836677, 0.112187855, 0.0456497557, -0.117297202, 0.0363576934, -0.0454703271, 0.133097589, 0.0347326733], [0.043291159, 0.0707632899, 0.0218213107, 0.0360490978, -0.1009942, -0.0507361442, 0.0282799657, -0.0643288642, 0.0332675613, 0.124886021, -0.0446216874, -0.00256765867, -0.0295969918, -0.0324323326, -0.0679225921, -0.0684956163, -0.00436520856, -8.431880e-02, -0.012034704, -0.0449127518, 0.0603636205, -0.0617158599, 4.993180e-02, -0.0564453937, -0.0192366987, -0.0355267674, -0.0588692576, -0.00284015806, -0.0351519138, -0.0490779579, -0.0430258512, 0.0273111891, 0.0561783798, 0.124271616, -0.00340043614, 0.0470247567, 0.118997358, 1.46550228E-4, -0.00253643515, -0.0687084571, 0.0218723528, -0.0420807898, 0.0455142334, 0.0468518697, 0.06843438, 0.102375425, -0.0508922376, 0.0479859337, 0.0252172984, 0.0324597768, -8.303930e-02, 0.0344019718, 0.0699970797, -0.00706476811, 0.00609650742, 0.00526374066, 0.0604574382, 0.00666020904, -0.0634514168, 0.00578825921, 0.00836663786, -0.0777392312, 0.0299222525, -0.0393076874, 0.0530181117, 0.0661297441, -8.099660e-03, 0.0428122804, 0.0349254906, 0.103744365, 1.235350e-01, -0.00166440941, -0.0251454692, -0.00452949805, 0.0153129352, -0.0553542674, -0.0797736048, 0.00517583406, 0.0542052276, -0.0127540808, 0.0428457074, 0.059198454, 0.0424133651, -0.0493637137, 0.0305257887, 0.00619894871, 0.026815569, 0.00191175321, 0.0364879258, 0.014919701, -0.0511612184, 0.0160227381, -0.0273367856, -0.0153102139, -0.0457185283, -0.0451803356, 0.00131215039, -0.0443706959, 0.0167765301, 0.0856390818, -0.0529886484, 0.0847080126, 0.0496470854, 0.0193309486, -0.0312662125, -0.029326437, -0.0563314259, 0.0790109708, -0.011489396, 0.0706950054, -0.0351269692, 0.0437551215, 0.0437048227, -0.0807492807, -0.00677183596, -0.0130825443, -0.0662321672, 0.0431224257, 0.0430111066, -0.0429867432, -0.0236997399, -8.504610e-03, 0.0132749546, 0.0338949487, 0.0146063985, -0.0356361233, 0.0644406602, 0.0688290149], [0.0223432127, -0.0391794816, -0.0287897084, 0.0367805734, -0.00178146723, 0.0383376554, 0.0677767172, 0.0486105755, 0.047636088, 0.0801385715, -0.018471729, -0.0192180667, 0.0845460445, -0.0224475637, -1.38810356E-4, 0.0643876195, -0.0815316886, -0.0140431952, -0.0387284756, 0.0627998039, 0.0268975347, -9.213390e-02, -0.0211047456, -0.0457662158, 0.0814606845, -0.0151156876, -0.0477600768, 0.00959179922, 0.0192699302, 0.0324337631, -7.874490e-02, -0.0562250391, -0.0836311206, 0.112866215, 0.0365651362, 0.048440855, 0.0492338948, 0.0594273284, -0.009496741, 0.0128501179, 0.0324992388, 6.378100e-02, -0.0374060795, 0.0570210926, 0.0703194141, 0.109745048, 0.058487583, -0.0474374406, -0.0448833033, 0.0440504178, -0.0170187596, 0.0528674461, 0.0302248113, 0.00417952472, -0.0576444529, -0.0386739112, -0.00468013063, 0.0749256536, 0.0533629134, 0.049231831, 0.00487302942, -0.0461427271, 0.0505233817, 0.0431680866, 0.0276480448, -0.059010528, -8.985580e-03, -0.0672673583, 4.478680e-02, 0.0765255541, 0.0842651278, 0.0880633593, -0.0407520086, -0.00665659085, 0.0455403142, 0.0185104907, -0.00975148473, 0.0214538686, 0.068311438, 0.0263773296, 0.0364896506, -0.0743065849, 0.0130932303, -0.0195210744, 0.0615894273, 0.0394022502, 0.0188752897, 0.068438217, 0.0901518687, -0.00164194836, 0.0615963489, -0.0489881262, 6.472060e-02, 0.059621118, -0.0203393511, 0.0106297964, 0.0872308761, 0.0211177226, 0.0586221442, 0.03609851, 0.0831582695, -0.0014562686, -0.00629432732, 0.01575079, -0.0555041917, -0.0269791894, -0.00842281989, 0.0606191419, -0.035822086, -0.0049442593, 0.0566265695, -0.0683411061, 0.0318680145, -0.0755790323, -0.0203140415, -0.0349720046, -0.0507674553, 0.132311165, 0.0533049107, 0.0668927729, -0.0550971031, -0.0491216555, -0.0292458478, -0.0250892583, 0.00890938845, 0.00595292961, 0.0477627926, 0.0292347763], [-0.0344534889, 6.702370e-02, -0.058614593, 0.0316108242, 0.0197989438, 0.0596544556, -0.014616265, -0.0564357787, 0.00182445883, 0.0193689913, -0.0754388868, 0.083333224, -0.0220080633, -0.0560865849, 0.021821849, 0.0748931095, -0.0113420738, -0.0392326191, -0.0583116263, 0.0461915843, -0.0200098753, -0.0333657786, -0.0176998712, -0.00764934765, 0.0313777328, -0.0223444719, -0.0212968774, 0.00296766264, 0.00650360808, -0.0383215919, 0.0294651296, -4.618420e-03, 0.0371415392, 0.0856671705, 0.0778812617, 0.0489255972, -0.0206688698, 0.0533604734, -0.0609014928, 0.0367908217, 0.0716059357, -0.00244948873, -0.0472839661, 0.00187983399, -0.0274170674, 0.0289207362, -0.0690165386, 0.00171646359, 0.037036404, -0.0569506921, -0.00180381734, -0.00100367155, 0.0574264228, -0.0399339795, -0.0606566519, 0.0401142277, 0.0557835475, 0.0732141957, -0.0750091299, -0.0596403703, -0.08967565, -0.0802937969, -0.0511973761, 0.0564152859, -0.046979472, 0.0361423492, -0.082890138, -0.0327469781, 0.0847325623, 0.0573910773, 0.00375941303, -0.0145247728, -0.0653229952, -0.0641492531, -0.0211507883, -0.09689527, 0.0687145442, 0.052383244, -0.00113920332, -0.0853990688, 0.0165175255, 0.0778537616, 0.0695547834, -0.0627989918, 0.0885612368, -0.047133144, -0.0808213949, -0.018839512, 0.0559413433, -0.0123761473, -0.00198083767, -0.0279411543, -7.36168702E-4, 0.0256336574, -0.00481586624, 0.00941629149, 0.0969701334, 0.0411302708, -0.0191767961, 0.0899476856, 0.0276372302, -0.0376637094, 0.0439133234, 0.0444955565, 0.0600262061, -0.0790811926, 0.04686746, 0.0445020199, -0.0534061939, 0.0690387711, 0.0362389572, 0.0551499277, 0.0173273701, -0.0750341564, -0.0544693619, 0.0346265249, 0.047092922, -0.0126658771, 0.0338296369, -0.0715659186, -0.039901197, -0.0178146735, 0.0350336097, 0.0153447045, -0.00999328307, -8.547600e-02, 0.0680562705, 0.0854133889], [0.076177828, 0.0886337906, -0.0198477786, 0.0231736396, 0.00848076493, -0.0347194187, 0.0402849205, 0.0112365251, -0.0623041168, 0.0667463914, -0.00207816833, -0.0180359371, 0.0165784117, -0.0954057425, -3.634230e-02, -0.017348472, 0.0761875957, -0.0347724594, -0.0530887581, 0.103429727, 0.0458966717, -0.0836516395, -0.0406188034, 0.0121222772, 0.00521358289, -0.0957147106, -0.0598767698, 6.166090e-02, 0.0216223411, -0.0348143317, -0.0528520718, -0.0952034146, -0.0304025821, 0.0206612665, -0.0335762836, -0.0163264368, 0.0637688264, -0.0340333544, 0.0406128541, 0.0683364868, 0.0830792636, 0.0028210883, -0.0478615612, 0.0924133732, -0.0249413699, 0.00583020365, -0.027903745, 0.0707040802, -5.503810e-02, -0.0157569498, -0.0145555865, -0.0235426966, -0.0621994994, 0.0518698841, -0.0466959067, -0.0296613649, 0.0413435325, -0.064031221, -0.0597851798, 0.00972177833, -0.0193162076, 0.0484929383, 0.0356703736, 0.018244423, -0.0118204588, -0.0303346626, 0.0387166701, -0.0702060461, 0.0376629718, 0.010088359, 0.0444291197, 0.0475553609, 0.070171766, 0.0222881045, 0.0430465862, -0.0861897543, 0.0707275644, -0.00148243713, 0.0274215266, -0.00102321967, 0.102724478, -0.0741218776, 0.0164781883, -0.0946778506, 0.0849214866, 0.0820065513, 0.0277598556, -0.0318392515, 0.0420872085, -0.0764851794, -0.0036374419, 0.0642861575, 0.0436662957, -0.0497264452, 0.0216659904, -0.0594222732, 0.00392841268, 0.0357315578, -0.0343024209, 0.0172810238, -0.00755848223, 0.00373520842, 0.0343432315, -0.0692792609, -0.0269612465, 0.00109819311, -0.0939636752, -0.0566527694, -0.0763834566, 0.114904664, 0.0171960853, 0.0774446875, -0.0318916105, -6.402540e-02, -0.0945463702, -0.00687806075, 0.0547445677, 0.0190533344, 0.0970802679, 7.264640e-04, 0.0107945492, 0.00191168697, 9.234620e-03, 7.996760e-02, 0.0527890474, 0.0640468895, -0.0811172425, -0.00844939611], [0.0410203449, -0.0631360337, -0.0815006271, -0.00148747931, 0.0475355387, -0.0581765585, 0.0240418147, 0.0160183534, 0.0453618914, 0.00951060932, -0.0855833888, -1.502600e-03, 0.0395889208, -0.0471800156, 0.0240246598, 0.0450636446, 0.0737334117, -0.0512911081, 0.0173402149, -0.0239565745, -0.076083824, -0.0223116055, -0.0580731705, 0.0151336221, -0.0389176458, -0.0698434487, -0.0527739748, -0.0335829966, -0.0304093212, -7.73955252E-4, 0.0078611616, 0.00345790084, 0.0283831153, 0.0724256188, 0.0605484135, 0.043968115, 0.0763795301, -0.10494858, 0.0348714292, 0.0409270637, -0.00935930386, 0.0396633931, -0.03683744, 0.0994836762, -0.0231901277, 0.056055367, 0.0225182585, -0.0380138867, -0.0157570839, 0.0860072821, -0.0423764251, 0.0538963303, -0.0433160216, 0.0289152488, -0.0366016068, 0.0170474108, 0.0710919499, 0.0628342703, 0.0370327272, -0.0531796403, -0.02400296, 0.0770658404, 0.0340333395, -0.0413418151, 0.00877805054, -0.0753918812, 0.018556824, -0.0652960539, -0.0539132543, 0.0168603826, 0.0782699734, 0.0641795098, -0.00460234284, -0.0134835811, -0.00898139458, 0.065886259, -0.0356402472, -0.0600780621, 0.0305583756, 0.0421866179, 0.115599498, -0.0383103378, -0.0551206246, 0.0625835285, -0.00479653431, -0.0299485344, -0.06423527, -0.0409061648, 0.086030595, -0.0768683776, -0.0785268694, -0.0674702227, 0.00485867308, 0.051571928, -0.0585351847, -0.0322961509, -0.0288387053, 0.0675211549, 0.0209925678, -0.0829700083, 0.00993320066, -0.0461953618, 0.00687220878, 0.0226052385, 0.00851905904, -0.0715567767, -0.0436025076, 0.00951585453, 0.0253202617, 0.0939932317, -0.0506763309, 0.0723977089, -0.0322347023, -0.0679097101, 0.0238389857, 0.0379046649, -0.00598749751, 0.0707421079, 0.00904940162, -0.0617141202, -9.045850e-02, 0.0598145574, 0.0265774261, 0.0517132618, 0.065652445, 0.0377803966, -0.0342677645, -0.0357752144], [-0.0533992164, -0.0692631677, 0.0155644836, -0.0418483019, -0.0529514663, -0.0711016431, 0.0106058912, 0.016954923, -0.051758673, 7.099770e-02, 9.411420e-03, -0.0725154206, -0.00366351823, -0.0751630291, 0.0144320931, 0.038664449, -0.0779537484, 0.0830616429, -7.844900e-02, 0.0725340396, -0.00216079457, -0.0176241808, 0.064741008, 0.0117491968, 0.0409717523, 0.0581200868, -0.0422772914, -0.05394236, 0.0109976586, 0.050673224, -0.0521134548, -7.276230e-02, 0.00259638694, -0.0113662314, -0.0493310019, 0.0399254523, 0.0112083014, -0.110840455, -0.0406863503, -0.0553952232, -0.0261040833, -0.0737963691, 0.053486947, -0.0403697565, 0.0764130055, 0.0450989269, 0.0451117344, 0.00444589881, 0.0996494367, 0.0912544354, 0.0702566057, 3.89572408E-4, -0.047657989, 0.0577167347, -0.0663118437, -0.0739967898, -0.00669461349, 0.0327631347, -0.00690449029, 0.0346009396, -0.0118495356, -0.0595653467, 0.0554404929, -0.0446037613, 0.0540001579, -0.066520363, 0.00703451922, 0.0195107404, -0.0272930339, -0.0335261151, -0.0257948525, 0.092385456, -0.00512519432, 0.0491418876, -0.0803849846, 0.0801554471, -0.0474129468, -0.0223376621, 0.0611691438, 0.0249884631, 0.0438493378, 0.0465952232, -0.0230475962, 0.0547289066, 0.00148554705, 0.0285031144, -0.0318686292, -0.0432013161, 0.0734998062, -0.0361801572, -0.0399863124, -0.00454718433, -0.0553761721, -0.007762691, 0.0420936979, -0.0263176616, 0.0565266237, 0.0120596718, -0.0982172638, -0.030590672, -0.0310999341, -0.018930858, 0.0204348154, -0.0088134557, 0.0628797263, 0.0421720631, 0.0841695144, -0.0366835296, -0.0320178345, 0.00685588922, 5.231040e-02, -0.0289815422, -0.0220781621, 0.0557751879, 0.0512207486, 0.0469780155, -0.07793013, -0.0118031884, 0.0233652256, -2.598960e-02, 0.0503217019, 0.10096772, -0.0365702137, -0.0311560612, 0.00514295883, -0.0638576895, -0.00109357201, -0.0384521149], [-0.00609776657, -0.0782423764, 6.522900e-02, -0.0359816216, 0.0644916147, -0.0635374263, 0.0445742123, -0.0636641085, -0.0127518624, 0.0491420254, -0.0683990791, 0.0669589936, -0.0125190169, -0.0248502661, -0.0150252655, 0.09267506, -0.0178299397, -0.0580829158, -0.0363188647, 0.0495967083, 0.0257786196, 0.036841657, -0.0307864472, -0.0555166751, 0.00815873314, -0.0746003389, 0.0400035642, -8.79239523E-4, 0.00975680444, 0.0356317684, -0.0755314678, 0.0372744948, 0.0814374536, -0.020651741, 0.0512271598, 0.0774294659, -0.0720287561, -0.0269327741, -0.0104084266, 0.0675507113, 0.0418245494, -0.079602018, -0.00455341628, 0.0677998737, -0.00516370824, 0.0367927141, -0.0568989143, 0.0428632572, -0.0595805719, -0.0504158325, -0.00821477175, -0.00152136083, 0.0447263904, 0.0830743238, 0.0623393804, 0.0698067322, -0.0244519338, -0.0130387824, 0.0104172202, -0.00878717191, 6.412520e-02, 0.0237856973, -0.0217985399, 0.0414888859, -0.0491898656, -0.0474594422, -0.0290054381, 0.0242527686, 7.195480e-02, 0.0104093766, -0.0397208892, -0.00131452421, 0.0484216101, 0.0407622345, -0.0777368098, 0.0438883305, -0.0263650808, 0.0582947209, -0.0103343818, -0.080357857, -0.0338463373, 0.0492444374, -0.0499613471, -0.0397780575, 0.0490031131, 0.0188101809, -0.0493756756, -0.051247362, 0.0482993089, 0.0641771108, 0.011197079, 0.0132738017, 0.0156236375, -0.0714279637, -0.0752301142, 0.0226713866, 0.0183298886, 0.0557374395, -0.0624826885, 0.0769494473, -0.0184584185, 0.0599130429, -0.0140398443, -0.00404027058, -0.00481691817, 0.0102064898, 0.0780392512, -0.0811886861, -0.0779708996, 0.0397465229, 0.0583371781, 7.98955269E-4, -0.0756335258, -0.0443012305, -6.507970e-02, -0.0143987555, 0.0137988627, 0.00472908933, -0.065698713, -0.0786428526, 0.00169527635, 0.0896930248, -0.0941152051, 0.0393325463, 0.0424547121, -0.027561795, -0.0640625805, 0.0047757444], [-0.0306507535, 0.0583102293, 0.0465862602, -0.0596570596, 0.0671689883, 0.0283891633, -0.0493646935, 0.0177215189, 0.0686227083, -0.0505514815, -0.0501578115, 0.0168145765, -0.0566025488, 0.049719885, -0.0740988478, -0.0707953647, -0.0731389448, -0.0147066573, -0.0320286788, -0.0654922649, 0.0543342978, -0.0175583456, -0.0154158082, -0.0543616675, 0.068558149, -0.038019333, -0.0117537389, -0.0329597928, -0.0774897188, -0.0792804732, -0.0129914992, -0.0310558528, -0.0770022795, 0.0454609916, -0.0272825211, -0.0738742799, -0.0249930844, -0.0509891696, 0.0116592608, -0.0509530678, -0.0010460281, 0.0193963107, 0.00758652947, 7.016810e-02, 0.011909523, 0.0415082276, 0.0286674146, -0.0632798076, -0.012760709, -0.0422886163, -0.00244693621, -0.0454975776, -0.0698960349, 0.0128519731, -0.0777636468, -0.0695809647, -0.0780691653, 0.0241016261, -0.0487699248, 0.0650028214, -0.0798963159, -0.0327594168, 0.0622025988, -0.0270410627, -0.0732024387, 0.052365087, 0.079448536, -0.0185261127, -0.0511363223, 0.0250751227, -0.082536213, 0.0309768077, 0.0323586613, 0.0140308766, 0.0576027595, 0.05633213, 0.0324359909, 0.0576508082, -0.0303486269, 0.0184563622, 0.0553380065, -0.0286428779, -0.057362508, 8.360580e-02, 0.0749706626, -0.0186528619, -5.396170e-02, 0.0810926333, 0.0406045206, 0.0293978099, 0.0406021625, 0.0299628042, -0.0189159457, -0.0598922782, -0.0101334751, 0.044391986, -0.0379532799, 0.0725448802, 0.067192927, -0.0162251871, -0.00866456423, 0.0547495708, 0.0262754634, -0.0181465149, 0.0367594063, 0.0206841044, -0.0737299845, -0.061874073, 0.0361557119, 0.0138174202, 0.0487588122, -0.0430977382, 0.00866219308, -0.0170727763, 0.0229739845, -0.0375915319, 0.0785812512, -0.0221382976, -0.0582140423, -6.861430e-02, -0.00974524393, 0.0035566038, -0.0788620188, 0.0194633566, 0.0436427593, -0.00592691358, 0.0258952342, 0.0222409833], [0.0431185924, 0.0579292178, -0.0540396981, -0.00276791258, -0.0683795959, -0.0138981789, 0.0119617749, -0.00265817065, -0.0254369881, -0.0641472861, 0.076249294, -7.657950e-02, -0.0691215917, -6.79729389E-4, -4.752140e-02, 0.0547273308, -0.050968878, -4.379270e-02, 0.0653711706, 0.0586719923, -0.0564160347, -0.00693666795, 0.0524301305, -0.0578386635, -0.0104072513, 0.0729455575, -0.0177993178, -0.0661511049, -0.0798850432, 0.0125228213, 0.0185550041, 0.0357805975, -0.0260292273, 0.0641236156, -0.0339183286, 0.0729878768, 1.79850569E-4, -0.0295849759, -0.0618827417, -0.0101420302, 0.0191023964, 0.0143463947, 0.0496401675, 0.0246130135, 0.017530622, 0.0522192121, -0.0342001319, -0.0108541651, 0.0641485304, 0.0281896871, -0.0422693864, 0.0056009274, 0.0290386733, 0.0638301298, -0.0233880393, -0.0301154647, 0.0718433037, 7.428490e-02, -0.0188103113, -0.0161713623, 0.00685922289, 0.0512228087, -0.0776155814, -0.0790195539, -0.0449666679, 0.0245256424, 0.0120751299, -0.015590623, 0.00138512242, 0.0458496921, -0.0420587957, -4.715380e-02, -0.0726804212, -5.42541849E-4, -0.0160142817, -0.0467561074, 0.0451570824, 0.0757331178, -0.0624426305, 0.0376363359, -2.99921783E-4, 0.0166216251, 0.0502297841, -0.0554730892, -0.0512200519, 0.0389943495, 0.0325267352, -0.0358683355, -0.0118671954, -0.0643242225, 7.664210e-02, 0.017588472, 0.0639027283, 0.0814268887, -0.0484388247, 0.0507609695, 0.0406770557, 0.0490465313, -0.05420224, 0.0590291619, 0.0667997077, -0.0180761088, -0.0153811853, -0.0175125729, -0.043919865, -0.0132411206, -0.0536643453, -0.0772530064, -0.0198095609, 0.0173474085, -0.0419649705, 0.0317623466, -0.0389221981, 0.0471743271, 0.0692338943, 0.0335689522, -0.070284918, 0.00430413289, -0.0652670339, -0.0286913458, -0.0377652086, 0.0229577087, 0.0268516243, -0.0731627494, -5.969890e-02, 0.00846476666, 0.0416350476, -0.0797127485], [0.00609327387, 0.0772428885, 0.0797267779, -0.0359296128, 0.0346219055, 0.0227096714, 0.0777428746, -0.0747867674, 0.0645726621, 0.0634724795, -0.0261341725, -0.0172977187, -0.0361533575, -0.0177051201, -0.0662906542, -0.0230113361, 0.0126914158, 0.0470567569, -0.0190327689, 0.0357306972, 0.0682962313, -0.0263529345, -3.571700e-02, 0.0805259123, -0.0193252619, -0.0628394186, 0.0674536377, 0.0793969482, 0.0408565849, -0.044161059, -0.0142710423, -0.00112594105, -0.0185055658, -0.0252749324, 0.0652757213, 0.0453898348, 7.190790e-02, 0.0438471511, -0.0426894352, 0.0480376706, 0.0312664025, -0.0479582138, -0.0676281229, 0.0031635561, -0.0373613276, 0.0257827193, 0.0258681402, -0.0221893787, 0.0735446662, 0.0264210515, -0.0408798754, 0.0439080335, 0.0283459872, 0.0773243234, 0.0365320221, 0.0190882832, -0.0307476092, 0.0514681488, -0.0435109623, -0.00462912815, -0.0221722983, 0.02081687, -0.0360369459, 0.00335788773, 0.0144248065, -0.0411501378, -0.0674703941, 0.0189476255, -0.0289963242, 0.0494256355, 0.0604253598, 0.0552787222, 0.0637977123, 0.0382817313, -2.78644264E-4, -0.0353616774, 0.00854118168, -0.0221916251, 0.0321666859, -0.0350860208, -0.051003933, -0.0575149804, -0.0018183589, 0.0222840495, -0.0323721394, 0.0181481317, -0.0090773683, 0.0145135308, -0.0751052946, -0.0534535423, 0.0705178678, 0.0612842962, -0.0390276127, -0.0149599388, 0.0542657636, 0.0461485423, -0.0405279249, 0.0779426321, -0.0549391881, -0.0633255914, 0.0741420537, 0.00305171753, 0.0766793787, -0.0422786623, 0.0787489488, 0.0365133397, -0.019080257, 0.058852382, -0.0795392245, 0.0278679151, 0.0200576577, -0.04722552, 0.0699545741, 0.0448153093, -0.0114682727, 0.0541347936, 0.0524855256, 0.0587221421, -0.016107345, -0.0294887312, 0.0251397789, 0.0201400872, -0.0325181484, -0.0767861381, -0.00745852757, -0.0606046282, 0.0399411581, 0.0529989414], [-0.0533450693, 0.0389843397, -2.463690e-02, -0.0224361606, -0.0392263271, 0.0790863782, -0.0653317571, -0.0642927364, 0.0334020704, 0.026299851, -0.0229591653, -0.0583456978, 0.0025529291, -0.0799337775, 0.0477304608, 0.0477731526, 0.0197277144, -0.0414581634, 0.0737877861, 0.0510611981, -0.0383387394, 0.00431221444, -0.0281830132, -0.0302164629, -0.054567568, -0.0471706614, 0.0264536589, 0.03212253, 9.14119184E-4, -0.0710605904, 0.043975912, -0.0497696325, -0.0425855815, -0.0244396571, 0.0236630887, -0.0590982698, 0.0400929935, -0.0673756376, 0.00557302684, 0.0443930291, -0.0388715081, -0.0185876396, 0.00435379334, 0.0396023132, -0.0790617763, -0.00151423027, -0.0335425772, 0.0696038678, -0.0452549607, -0.0296387058, -0.0660540313, 0.0123316487, 0.0610786937, -0.00577393919, -0.00608696882, -0.0724382326, 0.04234327, -0.0698804855, -0.0478215478, -0.0540058352, 0.0469224937, 0.0512578748, -0.0245003104, 0.0308733769, -0.0701002628, 0.0335549265, -0.0244193841, 0.0560460165, -0.0328532532, 0.0618578866, -0.0745182708, -0.00201392779, 0.063569136, -5.207690e-02, -0.063235566, -0.0237731636, 0.0692729055, 0.0078515783, -0.0530211031, -0.010131551, -0.0267842859, -0.0737304464, 0.0450258031, -5.49663731E-4, 0.0053587025, 0.0323131345, -0.0196771286, -0.00152142288, -0.0745564178, 0.00991636049, -0.0265531093, 0.0426251404, -0.0487280078, -0.0564149022, -0.0283284038, 0.00578834582, 0.047300864, -0.0111168539, 0.0343868695, -0.0401941314, -0.0790974349, -0.011344471, -0.00804827642, 0.0577479973, 0.0573874936, -0.0414873324, -0.0714283437, 1.95482688E-4, 0.00464546913, -0.0598714687, 0.0257484168, -0.0130718574, -0.0792685598, -0.0270293746, 0.031192245, 0.0237105377, -0.00827107113, -0.0519551188, 0.0643326491, 0.0101430472, 0.0407143608, 0.0517505705, -0.0763823091, -0.0442953296, -0.0271450076, -0.0780724138, -0.0225804783, -0.00898294803], [0.0236558244, 0.0385115296, -0.0103489533, -0.0537035093, -0.0088550467, -0.0192599632, 0.0525943525, 0.0595187135, -0.0152791748, -0.035034921, -0.0775288343, 0.00236709742, 0.0401565209, -0.0225446578, 0.0261509493, 0.0164925661, -0.0232241377, -0.0513916574, 0.0357427709, 0.0550351851, 0.0317478031, 0.0633546337, 0.0486391783, 0.0739953294, -0.0809241831, 0.0725285112, 0.0212962124, -0.00220869877, -0.00548778474, 0.0374938212, -0.0244798511, -0.0447930135, -0.0597489253, -0.0224508774, -0.0431765728, -0.0580312386, -0.0267540757, 0.0588376522, 0.0295443088, 0.0416141897, -0.0112623144, 0.0102862259, 0.00529509736, 0.0284759104, -0.0741930902, 0.0485910848, 0.0526263043, -0.0320444815, 0.0414897725, -0.0331234038, -0.0209923256, -0.0408113673, -0.0764990523, -0.00854747556, -0.0160600599, 0.0457072034, 0.0386845022, -7.739990e-02, 3.990140e-02, 0.0705694482, -0.0652028844, 0.0453057662, -0.06169964, 0.0223806668, 0.0511375368, 0.078100726, 0.0413420722, 0.0300976615, -0.0115925679, 0.0136819817, 0.0112741645, 0.0555846468, 0.02997756, 0.0501499772, 0.0538196638, -0.0235571861, -0.024547264, 0.0216616392, 0.0699621812, -0.0657855421, -0.0671929494, -0.0155782169, -0.027669549, 0.0265717879, 0.0477368571, -0.0056433985, 0.0472558551, 0.0538083576, -0.0140102971, 0.049796544, -0.0771622359, 0.0630935952, -0.0694698319, 0.0196553525, 0.0115572633, -0.0148473904, -0.00217829971, -0.0501935594, -0.0481347181, -0.0495828427, -0.0386614688, 0.0485507809, 0.0311566982, -0.0560884252, -2.545710e-02, -0.00642379932, 0.0555994101, -0.01105667, 0.0657835528, -0.00115447748, 0.0273454413, 0.0582018271, 0.0122237289, -0.0396170542, -0.0115297865, 0.0476682633, -0.0220234506, 0.0621284172, 0.0586445928, 0.0154250665, 0.0703970119, 0.0283837765, -0.0258665681, -0.0472893864, -6.066500e-03, -0.0504254699, -0.0628857091, -0.0687055662], [1.172620e-03, -0.0530922078, 0.0103554707, 0.00159761496, -0.0466273576, -0.0153756673, -0.0240867324, -0.00318360142, -0.0585440248, -0.0584502667, -0.0137783373, 0.0216677263, 0.0481103212, 0.017879175, -0.0382068306, -0.0724952146, -0.0142897265, -0.0226879474, -0.0500257425, -0.044657141, -0.0138386963, 0.0702918321, 0.0725618824, -0.00698663341, 0.0206303336, -0.0113111073, -0.0204723142, 0.0651140064, 0.00813299231, 0.0358449556, 0.0228577815, 0.0475700274, -0.00376050267, -0.0676898658, 0.0694327503, -0.0757666081, 0.00897797383, 0.017306082, -0.00693160389, -0.0232953578, 0.0624321289, 0.0800544992, -0.0511302501, 0.0322468393, -0.0633920282, 0.0548771769, -0.00705715921, -0.071892783, 0.025513757, -0.0479638316, -0.0543585755, 0.0046065161, 0.0806643888, -0.0413699485, -0.0708836764, 0.0416450314, 0.0414512567, 0.0262372419, -0.00781249674, -0.0737216622, -0.0579697043, -0.0578050204, -0.0559323505, -0.00836495868, -0.0595884658, 0.00732539407, 0.00349927437, 0.0684181079, 0.0598819889, -6.149410e-02, 0.0218704212, -0.0322291218, -0.0314222164, -0.0404715613, 0.0116579067, 0.0341927186, -0.00123727834, 0.039112255, -0.0697331354, 0.040883854, 0.0624176077, 0.00710911723, 0.0798233151, 0.0349370241, -0.0426735617, 3.47412832E-4, 0.0519814603, 0.0548266359, 0.0192652028, 0.0770859271, 0.0692109391, 0.0681228787, -0.0390663743, -0.0410122089, -0.0457334518, 0.0018635497, -0.044163119, 0.0205859281, -0.0763871297, -0.0581803769, -0.0762208179, 6.967590e-02, -0.0785955638, -0.00287143327, -0.0723564178, 0.0457278527, -0.0237085484, 0.0405155644, -0.0797197446, -0.0743861124, -0.00672993669, -0.0242713057, -0.0805763155, -0.0260927472, 0.0018664957, -0.00891581363, 0.0804197564, -0.0653813183, 0.0499300882, 7.356770e-02, 0.00550057832, -0.0711443424, 0.0345699824, 0.0269101858, -6.57112047E-4, 0.0291915536, 0.0504619889, -0.0615404136], [0.0305642635, 0.0666959285, -0.0385200381, 0.0645112693, -0.0496910959, 0.0482957587, 0.0287174545, 0.0142749473, -7.496530e-02, 3.507090e-02, 0.0387047492, 0.0687346533, 0.00961897335, 0.00516360067, -0.0327908099, -1.718400e-02, 0.0338321924, -0.0695643126, -0.0236638337, -0.0628041923, -0.03035127, -0.0252812449, -0.0226163026, 0.0213933494, 0.0815451666, 0.0812092721, -0.00890696421, -0.00402756641, 0.0573023073, -0.0275755432, 0.0498164184, -0.0592603274, 0.0385871418, -0.0581348874, 0.0141920652, 0.0721063911, -1.47960774E-4, -0.0401589721, 0.046377115, 0.0106317261, -0.0479542054, -0.0596154816, -0.0640457124, 0.0714368224, 0.0326219387, 0.07783892, 0.0349966809, -0.00526321307, 0.0501281284, -0.0514562689, 0.0108280154, 0.0531379506, -0.0526268296, -0.00120287575, -0.0140449889, 0.0363556631, 0.026679704, -0.0734740197, -0.00217355066, -0.0117964353, -0.0622587651, 0.00571816368, -0.0284721963, 0.03505769, -0.0217194259, 0.0166971516, -0.0101907328, -0.0333397835, -0.0414352603, -0.0597586259, -0.00694389129, -0.0739202872, 0.0320783891, 0.0466122217, -0.0449843071, -0.00128627231, 0.0517704263, -0.0639341623, 0.0212919209, 0.0409639366, -0.0659239665, -0.0414686464, 0.0415970646, -0.0753300115, 0.0488635227, 0.00403819792, -0.0475004166, -0.015387265, -0.0496033467, -0.0107769426, 0.0484054312, -0.033131104, 0.00624821382, 0.0666447431, -9.056340e-03, 0.00951082725, 0.0729924515, 0.0740329623, 0.00575401122, -0.00780810183, 0.0725690871, 0.0478610769, 0.0628348216, 0.0153595805, -0.0204436649, -0.0670655146, 0.0717116222, 0.0744492635, 0.0105085094, -0.0604854971, -0.0406712554, 0.0332211852, 0.0632094368, -0.00295353704, -0.0696106181, -0.0391390398, -0.077782467, 0.0223464798, 0.068906568, 0.0289613605, -0.0526398569, 0.0218308773, -0.0591599718, 0.0075291954, -0.01050823, 0.00806402881, 0.00670465268, 0.00623127166], [-0.0259323958, 0.0536984578, 0.00381401717, 0.00704566762, 7.248090e-02, 0.0068783015, 0.0837704092, 0.0363523737, -0.0556298979, -0.0373582877, 0.00114309217, -0.011535014, 0.041511409, 0.0168700088, -0.0129428152, -0.0134239066, -0.0130246989, -0.00695026154, -0.0105951168, 0.00515863067, 0.0152949197, -0.0561539941, 0.0763423294, -0.0169757307, -0.0430451483, -0.0139220962, -0.0189850554, 0.0290733092, -0.0621417351, -0.0385570452, -0.015323774, 0.0679399371, -0.0218070894, -0.0576224551, -0.036494609, -0.0654692203, -0.0431130901, -0.0241662301, 8.562110e-03, 0.0787358135, 0.0607034303, -0.010833622, 0.00223639188, -0.0472357273, 0.0759061426, -0.0269708522, -0.0300775841, -0.066884838, -0.0520959347, -0.0390984379, -0.0116539337, -0.0638735369, 6.571370e-02, 0.0122727267, -0.0498915575, -0.0660095587, 0.0265960954, 0.0204184186, -0.00695660105, -0.0287311282, -0.0660250559, -0.0653115212, 0.0696334466, 0.0605173223, 0.0255936049, 0.0841171294, -0.0281125866, 0.0536802374, 0.0648762062, 0.0115255322, -0.0765918121, 0.0446810573, 0.0738451257, -0.00648357673, -0.0508050062, -0.0349112861, -0.0716975406, -0.018340027, 0.0841956586, 0.0059796446, -0.0420872308, -0.0291129034, 5.90991811E-4, 0.00724662887, 0.0279622301, -0.0682016462, 0.00509671122, -0.083510235, -0.0768673941, -0.0431236513, -0.0762614757, -0.0256478619, -0.0332002752, 0.0832269341, 0.080684632, -0.0341635644, 0.0727839097, 0.0507641509, 0.0530904643, 0.0387285687, -0.0447083712, -0.0234124251, 3.478080e-02, 0.0660850406, 0.00242074183, 0.0302430764, -0.00763367396, 0.0645936131, 0.0292812064, 0.0557781458, -0.00554982666, 0.0292962082, -0.0802506953, -0.062521182, 0.0649037436, 0.0601874478, 0.00948794279, -0.0328981392, 0.0253076777, 0.0196896382, 0.0118425172, -0.0713500828, -0.0313961916, -0.0675760135, 0.0200230181, 0.0335756429, -0.0168734044, 0.0510022789], [0.0712065622, -0.0498374812, -0.0188063253, -0.0783225893, -0.0419368856, -0.0626349375, -0.0304217711, -0.0581849441, -0.0616779849, -0.0295178834, -0.0147735262, 0.0177362673, 9.50676447E-4, 0.00104521983, -0.0476178713, -0.0257065762, -0.0744347051, -0.00904599112, 0.0759304836, 0.103081524, -0.00167885516, -0.0535246544, -0.058786951, -0.0833157748, 0.0152550964, 0.0793086737, 0.0512368195, -0.0336812399, -0.0761332586, -0.0246399194, -0.0372260734, -0.00500316452, -0.00944267213, -0.0362675339, 0.0483180322, 0.0866558179, -0.0519440919, -0.00899029803, -0.025663387, -0.0163295269, 0.0876708552, 0.0397051871, 0.0745254233, 0.0277008526, 0.0407041535, -0.0334141552, 0.0343717076, -0.0506609082, 6.86014886E-4, -0.0371031277, -0.0779686793, -0.00692902599, -7.394930e-02, -0.0919792428, -0.0238438044, -0.0152842514, -0.0256917533, -0.00962366163, 0.0365927778, -0.0724163949, -0.0854004099, 0.0449178033, 0.0142806228, 0.0420217887, 0.0561691299, 0.0250952281, -0.0300941356, 3.92494767E-5, 0.087383911, 0.081669867, -0.0753142685, 0.0622214153, -0.0290810075, 0.0197499972, 0.0403896868, 0.0446005911, -0.00157571933, 0.0400152355, 0.0428169556, -0.0685722306, -0.0264816396, -0.0242238548, -0.0815614387, 0.0128851384, -5.02301438E-4, -0.0589422472, -0.0617317855, -0.0231113117, 0.0386208892, 0.054571908, -0.00949160568, -0.064623192, -0.0633513704, -0.0310067721, -0.00976671837, -0.053518232, 0.083879739, 0.0552109219, 0.019567756, 0.0572690293, 0.0384894349, 0.0601714253, 0.0385878459, -0.0224217679, -0.0119844852, -0.0730917081, 0.0676900595, -0.0646265447, -0.0882542133, -0.0108271651, -0.0509791225, -0.026020376, -0.0364729874, 0.00216447935, -3.193130e-03, -0.00729355728, 3.98055345E-5, 0.0225225072, -0.0213567819, 0.0121986056, 0.0384651572, -0.045902703, 0.029264193, 0.0192571282, 4.80483344E-4, -0.019751329, 0.0380190387, -0.0469852351], [-0.0716961101, 0.038245149, -0.0267701373, -0.0714827776, 0.0538789257, -0.00608582515, -0.0137804803, 0.078438051, -0.0111055244, -0.0366476402, -0.100592993, -0.0461272113, 0.0486995392, -0.0854801163, -0.0577247366, -0.0657172352, -0.034195967, 0.0498808101, -0.0469337218, 0.0819667801, 0.0541124642, 0.0107442923, -0.0128483856, 0.0497863479, -0.0249842275, 0.0383267403, -0.0161754694, -0.00326383091, 0.030972708, -0.0710142851, -0.00667091087, -0.0998863577, -0.0838119835, 0.0569135472, 0.0831232443, -0.05448379, -0.0690663158, 0.0367457494, 0.0536129288, 0.0808381363, 0.00194649585, -0.00776312593, -0.058494743, 0.0140456259, -0.0110551314, -0.102974996, 0.00239094417, -0.0394298956, -0.0218323339, -0.0324691162, 0.0141259497, -0.082906641, 1.08812033E-4, -0.0688811913, 0.0336255692, 0.00112375058, 0.0425254777, 0.00920464657, -0.0711535066, -0.100413263, 0.0135585899, 0.0136588393, -0.058622919, -0.0493450128, 0.0449125171, -0.0363432467, -0.011250563, -0.034553539, 0.0856458172, 0.0512764491, 0.0392477922, -0.0293805692, 0.0228869505, -0.0766075551, -0.0312590264, 0.047421027, 0.0422741808, 0.0804018303, 0.0110507356, 6.901430e-02, 0.00634529627, 0.0180582777, 0.0397126563, -0.0814089775, 0.0561016053, 0.0787173733, -9.05508873E-4, 0.0418984033, -0.0461013243, 0.0549619757, 0.0293666031, 0.0204228032, -0.041091539, -6.79040095E-4, 0.0183216315, -0.0333834365, 0.0839573145, -0.0620842651, -0.0431735218, 0.0726808086, -0.0231822636, -0.0661337823, -6.340390e-02, -0.0422684476, -0.0717267469, -0.0356030799, 0.0610348061, -0.0608812794, 0.0521407947, -0.0655917898, -4.009180e-02, -0.0805693268, -0.0286394469, -0.0312048085, -0.102530643, 0.0821109414, -0.115189627, 0.0740052462, 0.0644326732, -0.0534091331, -0.00716192741, 0.102653958, 0.0167282149, 0.0608972199, 0.0364757106, 0.0619440675, -0.0695321857, -0.0210402925], [-0.0371289626, -0.0413444452, 0.009842542, -0.0129777482, 0.0228230264, -0.0324825682, -0.00261941087, 0.00479594897, -0.0258102044, 0.0365131907, -0.0773979276, -0.036954198, 0.0224372707, -0.0334665887, 0.0621448419, 0.0478871129, 0.055144161, -0.0497804135, 0.0475826636, 0.0504565649, 0.0152808772, 0.0434621237, -0.0299182422, -0.0603801645, 0.049998004, 0.0841374769, 0.0102515668, -0.0169400629, -0.06704171, 0.032228861, 0.0617907755, -5.278020e-02, 0.0308277048, 0.00964266061, 0.0912399068, 0.0216472708, 0.0542086177, 0.0113077471, -0.06512627, 7.315190e-02, 0.040672686, -0.013389295, 0.0638946891, 0.0949250311, -0.0430467203, -0.0541703403, 0.0935800225, 0.0785699338, 0.00641333964, 0.033284504, -0.0920556709, -0.0228355043, 0.0660583749, -0.0611628927, 0.046526596, -0.0299918856, -0.0341034755, 0.0399298556, 0.0367316343, -0.101125211, 0.0466807038, 6.164810e-02, 0.0527227148, 0.00842049717, -0.00190639345, -0.0823909565, -0.0608835518, 0.0562517829, 0.0644148663, -0.0013274441, -0.0468190797, 0.0510279946, -0.0202133823, 0.0372230858, 0.0846530348, 0.0341778696, 0.0317213349, -0.0313539505, 0.0762817934, -0.0770497769, 0.0764782056, 0.0162592959, 0.0110752787, -0.0179023575, 0.0514859594, 0.0386399589, -0.0537899658, -0.089726977, -0.0423881784, -0.00124077348, -0.0357789509, -0.0527638793, 0.0470874906, 0.0949167832, -0.0437907316, -0.00644594524, -0.0350716747, -0.0332086757, 0.0750250742, 0.0235150494, 0.00712879375, -0.0255833343, -0.0215374846, -0.0212299917, -0.0534026846, -0.0312408581, -0.0270373728, 0.0242122374, 0.0371165276, 0.0889934971, -0.0435239188, -0.00388358952, -2.87991803E-4, 0.0195888225, -0.0567147769, 0.08386112, 0.00471769972, 0.0755376145, -0.0424243622, -0.0354073383, -0.00431375811, 0.0565165281, -0.0205612499, 0.058678221, 0.0251510944, -0.017993506, 0.0435255021, 0.0474437587], [-0.0514177121, -0.0245079491, 0.0528560467, -0.0992303043, 0.0829672589, 0.0147654545, 0.0634010658, 0.0643470064, -0.0434173718, -0.0134810172, 0.0195249841, 0.0421383567, 0.0016272841, 0.0075750621, 0.0843830258, -0.0382871665, 0.0511337928, 0.00472033443, -0.0382995643, 0.00433838041, -0.0103668552, -0.0224476457, 0.0270532537, 0.0416448861, 0.019983748, 0.0879455358, 0.0517708473, -0.0708542168, 0.069993481, 0.0486454666, 0.0431104451, -0.107258059, -0.0260756742, -0.0722258985, -0.0336376689, 0.0362847075, -0.0490768179, -0.0927903279, 0.0514901802, 0.0125223752, 0.106488235, -0.00985981337, 0.00706898095, 0.0634344444, -0.00216835691, -0.10149496, 0.0912152156, 0.0566551425, -0.0534385331, 0.0315772407, -0.0872612223, -0.0248762891, -0.0385877565, -9.393360e-02, -0.0126438467, -0.0601799414, -0.0817307308, 0.0244713407, -0.0861108675, -0.0789399594, 0.0624497458, 0.0363351442, 0.0394863337, 0.00332346652, 0.0055594286, 0.0320038684, 0.00947972853, -0.0637229308, 0.0606700256, -0.0245011263, 0.00775518454, -0.00268675759, 0.0161244143, 0.0482545234, -0.0624694488, -0.0391505845, -0.0420869216, -0.0535021834, 0.0909077674, 0.0235738382, 0.0328928567, -0.00619239173, 0.0619284436, -0.0180378258, -0.0559992678, 0.0251070131, -0.0570329316, 0.0634778664, 0.0884049832, -0.0459676981, 0.0333123542, 0.0341631249, -0.0721650347, 0.0662551597, 0.0066859629, -0.111119099, -0.0245383866, 0.0451377295, 0.0390350707, 0.0673207268, 0.0722510889, 0.0559645928, 0.0203491449, 0.0356273279, 0.0848851949, 0.0547164716, 0.0923883318, 0.0396522023, 0.0388828963, -0.0212018378, 0.0149128539, -0.0145247551, 0.032278765, -0.0196121242, -0.0445630215, 0.044119291, -0.0610766374, 0.0835011526, 0.00138587318, -0.0733456537, -0.0475589447, 0.0410955064, 0.0444962867, 0.0491866097, 0.017126834, 0.0398955569, -0.0671529919, -0.0147189032], [-0.0345550291, -0.065615572, -0.00850234833, -0.0940121337, 0.0508255363, 0.0277943853, -0.04734632, 0.0431042835, -0.0175604578, -0.0179567244, -0.0670538694, 0.0595312789, 0.00979659706, 0.0191914532, 0.0229322482, 0.0754050538, -0.0302665047, 0.0144409323, -0.0408111103, 0.00626991224, 0.0695942938, -0.0442165285, 0.0953543782, -0.0633019581, 0.0244335029, 0.0934288353, -0.0605400763, -0.0239855461, -0.0454370119, -0.0657135621, -0.021285614, 0.0430263057, -0.0365997702, 0.0517856702, 0.0621272698, 0.0483492278, -0.0748645067, 0.0402720459, -0.0124374069, 0.0363927037, 0.00102612411, -0.050970491, 0.0347297899, 0.0581720471, 0.0149658024, -0.0919158756, 0.103366531, 0.0690929889, -0.0152674206, 0.00753981154, -0.0118434131, -0.0582221448, 0.0545992106, -0.0876408144, -0.0834045186, -0.061245393, 0.0186065156, 0.0653520599, -0.093211323, -0.127042845, 0.00465597911, -0.0432485268, 0.090314418, -0.0135245565, -0.086581774, -6.822270e-02, 0.0702452957, 0.0212284587, 0.0814880133, 0.00839563366, 0.0341851376, 0.0954120084, -0.0495058633, 0.0314107873, 0.0755433515, -0.0439526178, -0.0503871627, -0.0120119052, 0.0525036119, -0.041665189, -0.069100894, 0.0827143788, -0.0481358357, 0.0200943146, 0.0715561435, -0.0534438677, -0.0491663665, -0.0482402965, 0.0720493942, -0.035999909, 0.0545462482, -0.062898919, -0.0412890762, 0.0256812256, -0.0383971818, -0.08292941, -0.0488151088, 0.00666209403, -0.0461772531, 0.0327884331, 0.0173050221, -0.025022937, -0.0531796515, 0.0568254292, 0.0481028371, -0.0291199982, -0.0326568745, 0.0341750532, 0.117215842, -0.00412450777, -5.289600e-02, 0.0486085638, -0.00309339771, -0.0797476693, -0.0612322204, 0.1132254, 0.0231221057, -0.0172390454, 0.0437070057, -0.0629191175, 0.0762142465, 0.0592663437, -0.00182250096, -0.084300138, -0.0433076248, -0.0781555101, -0.0612269156, -0.00714194402], [0.0928292722, -4.423300e-02, -8.22908652E-4, -0.0830794274, 0.0463546179, -0.0206467733, 0.0287682414, 0.0106054684, 0.0632147267, -0.0043666265, -0.00929437112, -0.0329782255, -0.0143689299, -0.00897481944, 0.00136777747, 0.0364123583, -0.0596338026, -0.0854891315, 0.0133328382, 0.111723945, 0.0191404372, 0.0172555316, -0.0144641912, 0.0458205566, 0.0542064048, 0.0413201563, -0.0254123099, -0.0559796244, -0.00980086159, 0.0553594977, 0.0309155248, -0.0893305838, 0.0274297092, 0.0509358756, -0.0462753102, 0.0852268114, -0.0584785603, -0.00687302323, 0.100693926, 0.0390654802, -0.00970055815, -0.0423967391, -0.0220128559, 0.054969456, 0.0791019275, -0.0169049185, -1.07337226E-4, 0.107093468, -4.094610e-02, 0.0401817299, -0.0639022812, 1.703340e-02, 0.00285629975, -0.074951075, 0.0764711797, 0.0262217186, -0.0848716273, -0.0364106074, 0.0364133045, -0.0674701482, -0.0711646751, -0.00637377333, 0.0859335139, -0.0538265519, -0.0368762352, -0.0667893067, -0.00720390817, 0.0503363386, -0.0262018181, 0.0757076517, 0.0115243839, 0.095122382, 0.0022541096, 0.0256797597, -0.0728434399, 5.429560e-03, 0.0202819426, 0.061401926, -0.0381823033, 0.0559714139, 0.0531597957, 0.0615884177, -0.0393077061, -0.0239284765, -0.0067279404, -0.0125298919, 0.0591141395, -0.0368883312, -0.0285603926, 0.0239017531, 0.118267156, 0.0481362343, -0.0678632259, -0.0323473364, 0.0268848222, -0.0471223816, 0.0161360092, 0.0817015767, 0.00996602512, -0.00222627097, 0.0700421408, 0.0241450276, -0.0696945414, -0.0354749337, 0.0174763333, 0.0874925777, 0.104800291, -0.0223442838, 0.0648781136, 4.386170e-02, -0.057593856, 0.0466337167, 0.00952197052, 0.00264144898, 0.00596720958, 0.0281751845, -8.428660e-02, 0.00891863462, 0.00146393396, -0.0682577714, 5.30491932E-4, 0.0341694057, -0.0795863047, -0.0656210929, 0.0522534028, 0.0486484878, -0.100006856, -0.0706681758], [-0.0106136771, 0.00185443729, 0.00587662729, -0.0558247641, 0.0477171242, 0.0285685975, -0.0270110015, -0.0544913262, 0.046238903, 0.0817884877, 8.231160e-02, -0.0506208129, -0.0864456593, 0.0519956201, -0.0242976919, -0.00605738116, -0.0479869656, -0.0293870345, -0.0279381722, 0.0773067549, -0.00542694191, 0.0398930088, 0.0740443692, 0.0298427232, 0.0160473175, -0.0178629551, -0.00628240546, -0.0448550768, 0.00540530775, -0.0785756558, -0.0345374942, -0.00889264513, 0.0674823895, 0.0493343733, -0.070714131, 0.111308351, -0.0792314782, -0.052285336, 0.0957430899, 0.0800917074, 0.00850244705, -0.0601458363, -0.0329755209, -0.043797534, -0.0103898225, -0.0285702571, 0.101863101, -0.0120405341, 0.0194483735, -0.0276680384, 9.973670e-03, 0.0913066193, 0.054204572, -0.109240249, 0.0364715941, 0.0167458877, 0.0574588962, -0.0503649451, -0.0316962115, 0.0488007404, 0.0302288216, -0.104030788, -0.0290187243, -0.0372798555, 0.0677812919, -0.00624418072, 0.0758523046, 0.0764699653, 0.0538698696, 0.0516266525, 0.0573358089, -0.0364215821, -0.0138788121, -4.68072656E-4, 0.0305648725, 0.0943951308, 0.0537531115, 0.00580465188, -0.0451089181, 0.0820503682, -0.0887385904, -0.0316227563, 0.0189136174, 0.0224686302, 0.0369665325, 0.00953694712, 0.0357232317, -0.0404477268, 0.0757558643, -0.00577131938, 0.0634102747, 0.0928644687, 0.0427768975, -0.10142301, 0.0391029082, 0.0250319131, -0.0556581803, 0.0640064478, -0.0834936351, -0.0364830978, 0.0791388079, -0.0727474242, 0.0211731493, 0.0846814811, 0.0645780116, 0.00638473919, 0.0169468615, 0.0395601057, 0.0754831508, -0.0183293391, 3.966750e-02, -0.0604243875, -0.0572278351, 0.0310591552, -0.0654818937, 0.008659699, -0.0437082611, 0.00820958149, 0.0486481376, -0.0843235105, -0.0335873701, -0.00132157502, -0.00489907293, -0.134842083, -0.0492801182, -0.0612962097, -0.0446296781, 0.064633958], [-0.067255415, 0.0117961559, -0.0330271684, -0.0604079291, -0.0122982142, -0.0321053155, 0.0779124423, 5.274810e-02, 0.0453361198, -0.0565130971, -0.00260270736, 0.0607822612, 0.0106642377, -0.0322087817, 0.0491371751, 0.052405905, 8.617260e-03, 0.0524037369, 0.0379091427, 0.0427826867, 0.0787854716, -0.00361260772, -0.0492653139, -0.029396221, -0.00999939348, 0.0239585415, -0.00700942939, -0.0235641208, -0.0709062666, -0.0187100563, -0.0275400486, -0.0224831812, -0.034832295, -0.00529624894, 0.0318775289, 0.0765311867, -0.0998237356, -0.0605633929, 0.0695974156, 0.00165665138, 0.0454649813, -0.0656107292, 0.0557569265, 0.0752793476, -0.0824461802, -0.0304653924, 0.0345293321, 0.0509838723, 0.0540652163, -0.0217262693, 0.0282779802, 0.0717580467, 0.0748419091, 0.050539352, 0.0555069596, -0.010038943, 0.0496101379, -0.0766607299, 0.0340266116, 0.110624522, 0.069740504, -0.0722159445, 0.0745864734, -0.0712041482, 0.0273125153, -0.0506238863, 0.0183674339, 0.0748995468, -0.0539166443, 0.0261745881, -0.0498763956, -0.023446044, -0.0320059955, 0.0922324955, 0.0179590192, 0.0265460145, -0.0348651893, 0.00237098173, 0.0219965633, 0.00825050194, -0.00127831497, 0.0695207343, -0.0486731157, 0.00439803768, 0.048042275, -0.0491825677, 0.0602233484, -0.0609389208, 0.0296214428, -0.0167320613, -3.441720e-03, 0.0291821212, 0.0166162308, -0.0319872797, 0.0421219245, 0.00182912359, 0.0215033498, -0.0536271743, -0.00317909266, 0.0778029263, 0.0156422332, -0.0809913277, -0.0841012671, 0.0816836878, 0.0932883545, 0.115709417, 0.052622959, 4.682000e-02, 0.0372997113, 0.0273049641, -0.00708696526, 0.0348439738, 0.0489885397, 0.0781697109, -0.0402737483, -0.0221276246, 0.0496910028, -0.0899162217, 0.0688987225, -0.104444578, -0.0363375209, 0.0910411626, 0.0272107739, -0.0409886241, -0.0371984206, -0.0707961693, 0.0498090349, -0.0161374956], [0.044906944, 0.0196534414, -1.428180e-02, 0.048844032, 0.113455765, -0.0857910365, -0.0397688597, -0.00175789022, 0.00114721875, 0.0228299201, -0.0274450649, 0.042483829, 0.00528251659, -0.0580126084, -0.0623873956, 0.0152724963, 0.032079149, -0.0869696885, -0.0243112966, 0.0566443056, -0.0664655119, -0.00584689621, -0.022401955, -0.0239895489, 0.0418082029, 0.0247943476, 0.06715855, 0.0211271569, -0.0487958156, -0.118315697, -0.0461639091, 0.0525047593, -0.0376108028, 9.84004233E-4, -0.0753740445, 0.0557550453, 0.0301925242, 0.135828137, -0.0687446594, 0.0369637907, 0.0220570508, 0.0440106876, -0.0318039544, -0.0441185795, -0.0243239384, -0.011946165, 0.0362096317, 0.0575803295, 0.0646713525, -0.0845478624, -0.0215132274, -0.0322185457, 0.0266974885, 0.0273707304, 0.0309423897, 0.092995949, 0.0340755917, 0.00790497288, 0.0108323861, 0.0978829339, 0.099771656, -0.0587110743, 0.0520148166, 0.0558690354, 0.0407640785, 0.103747748, 0.0195561517, -0.0570052564, -0.0756839215, -0.0937621816, -0.0817629322, -0.0395496674, 0.0839639157, 0.0131101012, -0.0579407401, -0.0162201785, 0.0173868649, 0.0148209529, 5.52846759E-4, 0.0623881109, -0.116115734, 0.137605786, 0.083179906, 0.113993593, 0.0593490489, -0.0642408878, -0.0408060066, -0.0481158346, 0.04793014, 0.0734828264, 0.0135501036, -0.00868982635, 0.0162765943, -0.130568787, 0.0218991656, 0.0631956309, -0.0351179354, -0.0483931154, 0.0777864083, -3.604470e-02, -0.00107411016, -0.0846442952, -0.0312353056, 0.110129938, 0.0849646702, 0.111730263, 0.0303928964, 0.0339914598, 0.00616962695, -0.0880549252, -0.0169097316, -0.0724247694, -0.033551909, 0.0693471283, 0.0467350185, -0.0452296846, 0.0191377439, -0.0904423892, 0.0680827349, -0.0452230237, -4.896800e-02, -0.0403417349, 0.0251836423, -0.0433073789, 0.072954528, 0.0249587148, 0.108118884, -0.0655192882], [-0.0196858142, 0.026016172, -0.00197531586, 0.0897422358, 0.129506856, 0.0669253841, -0.0504005104, 0.0157835484, 0.0508419313, -0.0859674066, 0.0907169803, 5.339420e-02, -0.0777138621, -0.0909379572, -0.0527322479, -0.0302517582, -0.045588918, -5.23061084E-4, 0.0147127425, -0.093417935, 0.0360081308, -0.0260337517, -0.0254021659, 0.00701236911, -0.0348323733, -0.0227515288, 0.0630726293, 0.0707104728, 0.0422004871, -1.45446713E-4, -0.0302857794, 0.113377906, -0.0352140702, 0.0899156704, -0.109348297, 0.0410083123, -0.00970794819, 0.0299957432, -0.0841634795, -0.0662687942, 0.0560107343, -3.893880e-02, -0.0487715229, -0.0115613462, -0.0817328616, -0.0168465599, 0.194097623, -0.00843190495, -0.0302370396, -0.0697634891, 0.0798501819, 0.00714306254, -0.0364114791, -0.0185375456, 0.00198417529, -0.0266852286, 0.0174318813, -0.0890194997, -0.0298578404, 0.102506824, 0.109257251, -0.0889555141, 0.00259511149, 0.0269240737, 0.0530087426, 0.0810950323, 0.0272814147, -0.0660649091, -0.0572941378, 0.00621459819, -0.0317926668, 0.0590231232, 0.151854187, -0.0536918454, -0.0325396247, -0.0310494266, 8.327830e-02, 0.098821409, -0.0925653353, -0.0220311843, -0.0255730897, 0.0315595865, 0.0636085569, 0.0583686121, -0.0401315056, 0.0462410487, -0.0666210651, 0.042143032, 0.0757899433, 0.0488958098, -0.0577435382, 0.0566487834, 8.922410e-02, 0.0195409805, 0.024198845, -0.0184904244, -5.183060e-02, 0.0227124672, 0.0672581717, -0.0788407623, 0.115494311, -0.0943037867, -9.96210146E-4, 0.0117396088, 0.124857388, 0.101373933, 0.0329128429, -0.0508574583, -0.0492988117, -0.0111187017, -6.43396343E-4, -0.0360230431, 0.0552582368, 0.117949955, -0.0392682441, 0.0211000983, -0.00280784536, -0.0391902328, -0.0628947317, 0.027097784, -0.0428889506, 7.129780e-02, 1.0813163E-4, -0.0282515958, -0.00668092445, 0.103962868, 0.103385724, -0.00192645844], [0.00677155657, 0.0840362086, 0.0615586229, 0.0586841069, -0.0151059236, 0.0228630919, -0.0454204306, -0.0747849047, 0.0362173468, -0.0399127863, 0.105344646, 0.122448117, 0.0541691221, -0.0896874964, 0.00797600392, 0.086252965, -0.013591392, 7.744870e-02, -0.0736683384, -0.0755913555, 0.0319824964, -0.0232008304, 0.0441887341, -0.0578585304, 0.00644814409, -0.0978505164, 0.0245617069, -0.0236445051, 0.0430271067, -0.0781356915, 0.0524953827, 0.0259861965, 0.00258893147, 1.536040e-02, -0.0169065576, -0.124307901, 0.0418288596, 0.105031714, -0.00936966948, 0.00790161266, 0.0185075402, -0.0243470948, -0.0218611713, 0.0409257039, -0.0423932523, 0.0592780374, 0.0452698879, 0.104145497, 0.0172248743, 0.036397405, 0.019935878, 0.0252101589, 0.0703984946, 0.0334962942, 0.0312994681, -0.0462680161, 0.010215601, 6.713160e-02, -0.0336583741, 0.0742477626, 0.0420101807, 0.0306751169, 0.0305333976, -0.0570347384, 0.0922093987, 0.067590557, 0.087777771, -0.0142898485, -0.0760540515, -0.0601156615, 0.0403833054, 0.0350838192, 0.0266805775, 0.0838796272, -0.0796878859, 0.0810331404, 0.0844891518, 0.0863067657, 0.0336550698, 0.0869541466, -0.0562217981, -0.0082331188, -0.0452548787, -0.0120527511, 0.0349901728, -0.0246177334, -0.00240804069, 0.0771583244, 0.0786239579, -0.0216877982, -0.0233406015, 0.014944084, -0.0226613022, 0.0096237259, 0.0283619445, -0.0477430783, 0.022872258, -0.065973416, 0.0430023037, -0.0349613726, 0.00619472796, 0.0658896267, 0.00217197416, -0.0373223536, 0.106911033, 0.116177142, 0.0537535921, -0.0755668953, 0.0355028063, 0.0884718075, 0.0388550498, -0.0575543568, -0.00697261421, 0.0276302844, 0.063458696, -0.00860001333, 0.0133154029, 0.0457404666, 0.0270660631, -0.01540023, 8.876460e-02, -0.00299572502, 0.0813584327, -0.0906546115, 0.0690028742, -0.0567373112, 0.0111409528, 0.0569760948], [0.057769049, -0.0306215733, 0.0633504242, 0.0511448048, -0.0127982963, 0.0759490579, 0.080685772, 0.0112202577, -0.00990817602, 0.032977093, 0.108495675, -0.00951231643, -0.0362207331, -0.00113327208, -0.0473084524, -0.0265767258, -0.019402571, -0.07408344, 0.00570587628, 0.042079851, 0.0126028061, 0.0121555571, 0.0329877622, 0.012658895, 0.056520097, -0.0431307852, -0.102767527, -0.00667795492, -0.0143257873, -0.0423962213, -0.0740301237, -0.0382266864, -0.0712266117, 0.0584483519, -0.0193637386, -0.0532699786, 0.128911957, -0.0684977323, -0.0229227487, -0.0835401937, 0.0651937127, -0.0559571907, 0.0456506461, 0.0811076238, -0.0418939479, 0.00408179732, 0.102843791, 0.136677235, 0.0331278183, -0.0335788243, 0.0713340789, -0.0321846865, -0.0416489802, 0.0248623695, -0.0170028191, -0.00918545574, -0.0529503711, -0.048134841, -0.0514040403, 0.0236432552, -0.0312504247, 0.0683485866, 0.029215347, -0.0124302171, -0.0501567647, -0.0273064319, 0.0357629247, 0.0426742248, 0.00301230629, 0.0836411566, 0.0741676167, -0.0224667806, -0.024334738, 0.0105006322, -0.001760802, 0.0575323254, 0.0037198225, -0.0491365567, 3.97417461E-4, 0.0677958503, 0.0465898365, 0.0575325489, -0.00954234507, 0.0404359102, -0.0443417765, -0.0416725054, -0.122537501, -0.0646346882, -8.60289496E-4, 0.034201093, -0.0950175077, 0.155741751, 0.00252144295, -0.050739713, 0.0536235496, 0.00464539789, -6.612750e-02, -0.0272874311, 0.0298191793, 0.0958407074, 0.0105747748, 0.0412125103, 0.0773181543, -0.0302443095, -0.0712979957, -0.0224865153, -0.0923532545, 0.0174676869, 7.157640e-02, 0.00646444084, -0.0628620088, -0.0581605323, -3.720920e-02, -0.0577583201, -0.00846270472, -0.0025375539, -0.0384457782, 0.00488221878, -0.02255724, -0.0149513241, 0.0140770916, 0.0777056738, 0.116146736, -0.0767781213, 0.0340793394, 0.00492456695, 0.14741677, 0.0121559296], [-0.0661674961, 0.0408111811, 0.054643672, 0.0565498658, 0.0234873034, 0.0014393311, 0.0752778798, 0.0447793417, 0.0214810539, -0.0167642701, 0.0537800491, 0.102184564, -0.0470284447, 0.0393696129, -0.0724103078, 0.0339901745, -0.0710006431, -0.0545752831, -0.101695076, -0.0686569661, -0.0302972104, -0.0316550024, 0.00582525274, -0.011032219, -0.0191279091, 0.0756494626, 0.0145085622, 0.089830242, -0.0177834034, 0.0358081423, -0.0814512297, -0.107139193, 0.0151741672, 0.0972204506, -0.0638033822, -0.0801654756, 0.0724697262, 0.0710712671, -0.00958041287, -0.00291196979, 0.0769288763, -0.00584416511, -0.051720608, -0.0266256779, -0.0723166689, -0.0260741841, 0.104770884, 0.0683549717, -0.0635959059, -0.0745948926, -0.046795059, -0.0335926749, 0.072709024, 5.547760e-02, 0.0940309837, 0.0362556539, -0.0651912466, 7.247470e-03, 0.0205572136, 0.02852186, -0.047765553, -0.00341174915, -0.0156272855, -2.196840e-02, 0.0929982215, -3.41953382E-5, 0.0489269607, 0.011892641, 0.00753624644, 0.0468887165, -0.0157220792, -7.944930e-02, 0.0504987091, -0.00454041548, -0.0336365215, -0.00962052401, 0.0110761393, 0.0568114258, 0.110755101, -0.0848942771, 0.0172403175, 0.0570045933, -0.0769586936, 0.0277446751, 0.0836086049, 0.057374645, -0.0108274184, 0.0149623016, -0.074257724, -0.0177896954, -0.0461195484, 0.0112296864, 0.111235864, -0.013977848, 0.0918364152, 0.0485115424, 0.122029588, -0.0355590582, 0.0398612842, 0.14461197, -0.033575926, 0.057602413, -0.0541749038, 0.0278947745, -0.0694009662, -0.0535875931, -8.134840e-03, 0.0163448285, 0.106261119, 0.0663502961, 0.0534931943, -0.00574497459, 5.492210e-02, -0.112593599, 0.0964917615, 0.0104343528, 0.0285648108, 0.109960116, 0.0191836506, -0.0403861105, 0.0539925881, 0.0176261421, -0.0173621606, 0.016387593, -0.00772764161, 0.0566603951, 0.085977517, 0.0706250965], [0.0211376678, 0.0408013947, 1.014310e-01, -0.0634136572, 0.053952232, 7.680380e-02, 0.0502361953, 0.0154167088, -0.0920646414, 0.0427231565, -0.0429791249, 0.0593081638, 0.0142324036, 0.067805849, 0.0366792344, -0.0663817748, -0.0363765247, -0.0528825745, -0.0829733833, -0.0555092767, -0.0500624925, -0.0461415872, 0.00659135403, 0.0323090591, -0.0337829888, 0.0573878847, 0.024821585, 0.0451607406, -0.0695222169, -0.0739764944, -0.0267723259, 0.0343559496, -0.0577463023, 0.0828307122, -0.0422066562, 0.0532644801, 0.0663894564, 0.0626204461, -0.078002587, 0.0447989628, -0.0341197625, -0.0296922922, 0.0138157504, 0.0718221664, -0.0198995937, -0.0126986187, 0.0305688083, 0.0740876123, -0.10044089, 0.0272976048, -0.0697557703, -0.0109233372, -0.00662080618, -0.0508005619, -0.0452584289, 0.0265374593, 0.0822892114, 0.087281607, 0.0591638722, -0.0487241074, -0.00507992553, 0.017757168, 0.074642837, 0.0929716229, -0.0134018539, 5.43356058E-4, 0.0507948287, -0.0743142664, 0.0666257814, 0.0208538976, 0.00762552349, 0.00845569838, 0.0606956743, 0.0393717661, -0.0122430651, -0.0620560572, -0.0801186115, 0.03919328, 0.0563608631, -0.0129833408, 0.0374830924, -0.0300337672, 0.0159804691, -0.0777316465, -6.14715158E-4, 0.0351365544, -0.0266214237, -0.0151193095, -0.0414762609, -0.0343963504, -0.012615826, -0.0364578776, -0.0238861479, -0.00551754376, -0.00523457816, 0.0382161848, 0.0466234237, -0.0081579443, 0.044195082, 0.05157784, -0.0246762764, 0.019630257, -0.0318624601, 0.0150572751, 0.0760524272, -0.0631353855, -6.697850e-02, -0.0295728482, -0.0174042247, 0.0258136243, -0.0217681304, -0.0331973396, 0.0683454946, -0.0813999697, -0.0133809196, -5.380880e-02, -0.0480319411, 0.11911764, 0.0399125591, -0.0313498601, -0.0244000852, -0.023338493, -0.0316853561, -0.0105906185, 0.0504761524, 0.0582671575, 0.100448117, -0.012031598], [-0.0549308769, 0.0701689497, 0.0312237684, -0.047312174, -3.058360e-02, 0.0398847796, 0.0731721595, 0.030113738, 0.0395255126, 0.00539672514, 0.00392791023, 0.040731512, -0.0482584611, -0.0528478771, -0.0511503853, 0.0125957048, 0.0375285372, 0.0162092634, -0.0819791629, 0.0838801414, -0.0318301618, -0.0571635067, 0.0309733544, 0.0779269114, -0.0395775922, 0.0280925669, -0.0371016115, 0.00531360414, 0.00206582155, -0.0633549318, -0.0267355051, 0.0325087123, 0.0639403313, -0.0325343534, 0.0361984819, -0.0940648093, 0.100445777, 0.0218386203, -0.030521635, -0.0306014922, 0.0549506173, 0.0113345915, -0.0749456659, 0.0680912957, 0.0358073041, -0.0302075706, -0.0760587528, 0.0623723901, -0.0874387696, -0.0454318114, -0.0533234812, -0.0761045218, 0.0415534265, -0.0383731388, 0.00879956595, -0.0716220364, -0.0306333657, 0.055949118, 0.0444634706, 0.0572837777, -0.00516211428, 0.0477921851, -0.0139945783, -0.0393014774, -0.0145303365, 0.0749691725, 0.0376046225, -0.0491942614, 0.0242215451, 0.089138545, 0.064067468, 0.0417067148, -0.0249117445, 0.0792644619, 0.0493899696, -0.0704373121, 0.042661231, 0.0518794283, 0.0222957246, -0.066248849, -0.0345104188, 0.0578076541, -0.0528273284, -0.0255955402, 0.0735087916, 0.02450688, -0.0315256603, 0.0266093817, 0.087294057, -0.00808284245, 0.0243846048, 0.0837828368, -0.00588536868, 0.0154278409, 0.0953119322, 0.0561960675, 0.116031483, -0.0707793161, -0.0563178845, -0.0208748542, -0.0760706142, 0.0297325384, 0.00162160338, 0.0731096789, 0.0872951522, 0.0120068081, -0.0140463514, 0.0488224551, -0.0622909889, 0.0228309054, -0.0291668754, -0.0396338366, -0.0170234181, -0.0600580275, -0.0276175216, -0.0272622816, -0.0960430726, 0.0527670756, 0.0514099188, -0.00926851667, 0.0489448309, 0.07826709, 0.0305336546, -0.0728087649, -0.0105889067, 0.0652312264, 0.0541078262, 0.0186249092], [0.0511655584, 0.0160392094, 0.0549847446, -0.0129176695, 0.040123567, -0.0157731771, 0.0286472477, 0.0592230633, -0.0640011206, 0.0344392247, -0.0591391921, -0.0528353378, 0.0485973507, -0.0434837863, -0.014970799, 0.0615045875, -0.0367224105, -0.0855846703, 0.0102673098, 0.0413993336, -0.00670865085, -0.0278017763, -0.00629957533, -0.0286829323, 0.0102167334, -0.036135029, -0.0172330234, 8.996270e-03, -0.0547370128, -0.0643793642, -0.0203829072, -0.050625585, 0.0253589582, 0.0672960356, 0.10049399, -0.047004886, 0.0141211739, 0.0477127135, -0.0785564631, -0.0360932872, 0.059415441, 0.0665331184, -0.0220816601, 0.0493266769, 0.0351138562, 0.113088861, 0.00379727338, 0.0625518188, 0.0208610427, 0.0632828698, -0.00667868275, 0.0502971783, -0.0721741095, 0.0350331217, 0.0624113976, 0.00999547634, -0.0565195531, 0.00446280092, 0.0235815551, 0.0352240689, -0.0179589055, -0.0296297185, 0.00454518525, 0.0726368949, 0.00298544299, -6.921240e-02, 0.0323361456, 0.0286769308, -0.0327890143, 0.0715956613, -0.00188260409, -0.0589420162, 0.0517178476, 0.0900357291, 0.0800723731, -0.021418225, -0.0372193567, -0.068827942, 0.00901203603, -0.0580192581, 0.120746925, 0.0510867126, -0.0571590699, -0.069642283, 0.00328000542, 0.00569628598, -5.675740e-02, -0.0472201034, 0.0242642853, 0.0384344049, -0.0507599488, 0.0902105793, -0.00543598691, -0.0261090137, 0.00900233909, -0.0324366502, 0.074512206, -0.0462232232, -0.066644527, -0.0289749634, -0.0677693561, -0.00753740966, 0.0210650023, -0.0752119869, 0.0748892054, -0.0393764861, 0.0404399969, 0.0126975439, -0.0492548458, 0.0749835819, -0.0138564305, 0.0446225666, 0.0800847634, 0.0518526733, 0.00969609804, 0.0772252902, -0.0034201704, 0.0274877362, 0.0468785129, 0.0733628348, -1.806060e-02, 0.0828012451, -0.0481353886, 0.0814468041, 0.00659723813, -0.0415470973, 0.0549203865, 0.00385554507], [0.0622142032, 0.0740979165, -0.0506645776, 0.00407935027, 0.00300386967, -0.0269306805, -0.0162216704, 0.0455071926, 0.028837448, 0.0115217781, -0.0517018698, -0.0680387095, -0.0484584235, -0.0286715087, 0.0218699053, 1.08684108E-4, -2.563490e-02, 0.056377355, -0.0590793341, -0.0672998875, 0.0126038222, 0.0465788431, -0.0392906331, -0.0475511365, 0.0106435791, -0.0796665325, 0.0465069562, 0.018766351, 0.0448931269, 0.0534886941, 0.0832743123, 0.0243088957, 0.0740234405, -0.0205793716, -0.0507150404, -0.0649171248, -0.045147609, 0.0485082678, -0.00677949237, 0.0788968652, 0.0258958489, -0.0702299773, -0.0519063734, 0.110836476, 0.0754291639, 0.066385746, -0.0132619916, 0.0354039036, 0.0589270741, 3.50943446E-4, 0.031190699, -0.0338522568, 0.0579936951, 0.050614737, -0.0164765362, -0.0122054797, -0.0624110735, 0.0624912716, 0.00135479309, -0.0830093398, 0.00234891032, -0.0209877286, -0.0857643261, -9.43398685E-4, 0.0124210864, -0.0823382884, -0.00766127231, -0.0353176817, 0.0292207953, 0.0312838778, -0.0316377953, 0.0479574166, -0.0597731583, -0.0156623032, 0.00922656245, 0.0388594605, -0.0343975388, -0.0232946817, 0.0856125876, 0.0277370848, 0.00185155671, -0.058819294, 0.0210472029, 0.00174230349, 0.0745033919, 0.0146839498, -0.0299663171, 0.0588653386, 0.0116285319, -0.0506025627, 0.0528937019, -7.082890e-02, 0.0393480361, -0.0368526652, -0.00717842299, 0.0129842767, -0.00424308283, 0.00731039559, -0.0686394051, -0.0114474967, -0.00283117406, 0.0521479584, -0.0805746838, 0.0322272964, -0.00542951841, -0.0161723308, 0.0316508263, 0.00481606601, 0.0471252799, -0.0376293287, -0.0912486166, -0.0094439229, 0.0270312019, 0.0685414076, 0.00305628846, 0.0474564172, 0.0421288088, -0.0464591235, 8.038630e-02, 0.00719539123, 0.069858931, 0.0479293913, 0.0166712943, -0.0385274962, -0.0600775778, -0.0641694888, 0.00578905735, 0.0215272177], [-0.071502015, -0.00374785741, 0.00514291553, -0.10197603, -0.0445211865, -0.060876593, 0.0676624551, -0.0345960483, -0.0142909316, -0.00950053614, -0.0442247763, -0.0244092196, 6.649930e-02, 0.053564664, 0.016953785, -0.0220388584, 0.052273076, -0.0587491058, 0.0191383064, -0.0333900675, 0.0764036253, -0.00519217597, -0.064256914, 0.0677930936, -0.05000728, 3.293970e-02, -0.0750982538, 0.0196872875, -0.0464409515, -0.0184485614, 0.0158977583, 0.0368542187, 0.0335773937, -0.0550589114, -0.0371357538, 0.0376730412, -0.0442184247, -0.0834282562, -0.0660972223, -0.0338365063, 0.0256682597, 0.0104650529, 0.012960487, 0.063436836, 0.0170508604, 0.0041107079, -0.0288176984, 1.946990e-02, 0.00282156025, -0.037900418, -0.0148728415, -0.0578525215, -0.0159879811, 0.0723157078, 0.0455793254, -0.0802067816, 0.00537245115, 0.0826371833, 0.0179893058, 0.0318794772, 0.00519755622, -0.0572667196, -0.0757517814, -0.0137418071, -0.0812345445, 0.00495595671, 0.0648435131, -0.0404362343, 0.0206135083, 0.0723513588, 0.00189831376, 0.030793244, 6.882630e-02, -0.0549079105, 0.0377189741, 0.0403419957, -0.0384487808, -2.68677482E-4, -0.0247937124, -0.0710414723, 0.130807728, -0.00358926365, -0.0396577269, -0.0871290639, 0.0339092314, -0.0648138374, 0.0495743081, 0.0255988836, 0.0960668399, -0.068998456, 0.0253717732, -0.0768817589, 0.0154008074, -0.00264411722, 0.0415736809, 0.0279059913, 0.00362959225, -0.0095899906, 0.0554710291, -0.0411733128, 0.011490304, -0.0287268963, -0.0265641958, -0.0659656524, 0.0834915489, 0.0124004297, 0.0296456516, -8.882350e-03, -0.0154394014, 0.106692173, 0.00901907123, 0.0379091948, -0.0324628912, 0.0293422714, -0.0116702579, 0.044212006, -0.0133292023, 0.0159648228, 0.00834463723, -0.0649983138, 0.0549816452, 0.0740838423, -0.0273409504, 0.0929231271, 0.0560667552, 0.0177038107, -0.085948661, -0.0379663035], [0.0266356152, -0.05747151, 0.0354687348, -0.0398524851, 0.0685173422, 0.0521030128, -0.00133162865, 3.068660e-02, 0.0388959162, 0.0486194491, -0.078982152, -0.0240139198, -0.01620185, -0.0187323429, 0.0391055085, 0.0301099624, 0.0723044723, 0.0267017223, 0.0820155963, 0.0486297309, 0.0227778014, -0.0489994623, -0.0653236359, -0.0274401624, -0.0390175506, 0.00642565451, 0.039874468, -0.0451058857, 0.0785287544, -0.050859645, -0.0418286435, 0.0349219888, -0.0568644069, 0.0155937085, -0.034051571, -0.0500695333, -0.0210951362, -0.0762018487, -0.0636056289, -0.0331791155, -0.0728651509, -6.785370e-02, -0.0431781337, 0.0685308203, -0.026084004, -0.0227997378, -0.0628827065, -0.0210140832, -0.0538466759, -0.0502309129, -0.0226896852, 0.034273874, 0.0540416203, -0.0228788145, -0.0714368746, -0.0271091871, 0.0599648356, 0.0153378062, -0.0769131929, -0.0325206816, 0.0497053377, -2.582350e-02, -0.0837290361, -0.0155821247, -0.0104435673, 0.0184643473, 0.0149391424, 0.00810760073, 0.00689516263, 0.0926736667, 0.0240309816, 0.0431626216, -3.37393176E-5, 0.0254654158, -0.018924132, -0.0370263718, 0.0412186645, -0.0319221318, 0.0150828892, -0.00806072913, 0.049749285, -6.531860e-02, -0.0643197373, 0.00371778663, 0.0478631556, 0.0192814711, 0.0803418905, 0.0386854373, -0.0360317156, 0.0237998366, 0.0766452625, -0.0172560345, 0.0206142161, 0.0665624141, 0.00483855652, 0.0453767888, -0.010075679, -0.0125684645, 0.0427024737, 0.0113363275, -0.00404397491, -0.0609632693, -0.0974757596, -0.0664095283, -0.046767693, -0.0478546023, -0.061048083, -0.0623280331, 0.0287236497, 0.0919675454, -0.00443382328, 0.0333482064, 0.0360821225, 0.0544074215, 0.0092733521, 0.0327730216, -0.0872172713, -0.0847184211, 6.453640e-02, 0.0814430267, -0.0662928894, -0.0476105101, 0.023559168, 0.0300033893, 0.0438229144, 0.0507766418, -0.0443434045, 0.00770816533], [6.718210e-02, -0.0478014834, 0.0672593489, -0.0581894778, 0.0758157447, -0.0580194332, -0.0845181718, 0.00923532341, 0.0610028468, 0.0417235643, 0.0304833874, 0.00828072336, 0.0797658562, 0.0416296721, -0.00257077417, -0.0157171767, 0.0569525622, 0.0847452208, -0.0105644977, -0.0267921966, 0.0320079029, 0.0525634438, -0.0565891378, 5.80894586E-4, 0.0389545038, -0.0787698328, -0.0387152657, 0.00184309517, 0.00229149195, -0.0738616511, -0.0514038764, -0.0524309613, 0.0366236567, 0.0246482845, -0.062743865, -0.0355713479, 0.019137593, -0.0327640697, -0.0214762669, 0.0848363339, 0.0305951275, 0.0122436201, 0.0578447282, -0.0524350815, 0.0176793877, 0.0711975768, 0.0712861866, 0.00649969466, 0.0260321759, 0.0628597662, -0.0722445622, 0.0693924353, -0.0234818552, -0.0424674451, 0.0488115326, -0.0693062841, 0.0320296362, 0.0707884878, 0.0146662127, 0.0503205732, 0.0239743702, 0.0288173556, 0.024404373, -0.0635988489, 0.0169264898, -0.00623868499, 0.0790141224, 0.0743309259, -0.0604156218, 0.0686184913, -0.0717842504, -0.0270922184, 5.725100e-02, 0.0143979257, -0.0816299095, 0.00704285688, 0.00570247928, 0.0825472399, 0.038142506, 0.0353773646, -0.0628639161, 0.0186323132, 0.0399696194, 0.0139828473, 0.0907492339, 0.0666053593, -0.0415477604, -0.0545842461, -5.648230e-02, -0.0170640126, -0.0256203525, -0.0178871322, -4.875670e-02, -0.0470189713, -0.00826343894, -0.0261883102, -0.0756431594, -0.0032490606, -0.00194277463, 5.248060e-03, 0.0370563418, 0.0526802838, -0.00635639951, -6.883560e-02, 0.0447245128, -0.070830524, 0.0618913397, 0.0569914803, 0.0608295165, 0.00857486669, -0.034590669, -6.518300e-02, 0.0264445376, 0.0683485419, 0.0512607209, 0.0888020619, 0.0213143136, -0.0460120589, -0.0541214868, 0.0704880804, 0.0648619682, 0.0399171636, 0.0396605246, 0.0863168761, 0.0704447776, 0.0364731029, -0.0606853254, 0.0100939628], [0.0572807342, 0.00782230682, 0.0541379564, -0.00865187775, -0.0724903792, 0.0333091319, -0.076180473, -0.0126076406, -6.190020e-02, -0.0268133078, 0.00780387968, 0.0708277673, -0.0113928691, -0.0111702122, 0.00857672654, -0.045520518, 0.0519399941, 0.0183035843, -0.0253562964, -0.062111672, -0.0425574817, 0.047040876, 0.0547760651, -0.045064453, 0.0620471052, 0.0223963093, 0.0602335073, -0.03694392, -0.0283796713, 0.0625603944, 0.0710158721, 0.00674001779, 0.0665905327, 0.0702314451, 0.0697147921, 0.0743856281, 0.0768432617, -0.0286963731, 0.00211161771, 0.0486199819, 0.0298843645, -0.0117497062, 0.0251599234, -0.0150021734, 0.0166404322, 0.0220878907, 0.0117735537, -0.0558634512, 1.546320e-02, -0.0397412926, -0.0357045569, -0.0696524679, 0.0205442961, 0.0300363507, 0.0451215543, -0.0520115867, 0.0677876696, -0.0414929055, 0.0588502474, 3.50460352E-4, -0.0420807824, -0.0577273704, -0.0806629061, 0.0677367746, -0.0285856947, -0.0364404395, 0.0276324712, 0.0332152471, 0.0122651579, -0.0388425887, -0.0211402923, -0.0512595922, -0.0316599347, 0.0215504169, 0.0685073659, 0.00227707368, 0.053972736, -0.0368477553, 0.0289690793, -0.0338756926, 0.0743487477, 0.0202247407, -0.0606843047, 0.0782329663, 0.0399756283, 0.0425852537, -0.0587611198, 0.00313480478, -0.0206545405, 0.0712328926, -0.0355531611, 0.0615798272, -0.0211153198, 0.0326961242, 0.0709302351, -0.00748492265, 0.0766120553, 0.0483103767, -0.0388396494, 0.0354830511, 0.0281448495, 0.0722001269, 0.0602922253, -0.0702591538, -0.00138883863, 0.0623956472, -0.0761340111, 0.0678129271, 0.0533766672, -1.34566377E-4, -0.0177371167, -0.0533051118, -0.0091658961, -0.0113169206, 0.0685782209, 0.0454946347, 0.0528613254, 0.0821721255, -0.00311222696, -0.00484174071, -0.037809059, -0.0605804585, -0.0374657474, -0.00108358217, -0.00980810355, -0.00601922302, 0.010051785, 0.0378454067], [-0.0325148813, -0.00743184751, 0.0604297146, 0.0594694242, -0.054841347, -0.0645916759, -0.0677277893, -0.0650412142, -0.047704462, 0.0462250821, -0.0787914097, 0.0490280092, -0.0506846271, 0.0731009692, -0.0558432676, -0.0112073859, 0.0702359676, 0.0483952798, -0.0223005861, 0.0209999494, 0.0594576746, 0.0483904891, 0.0285202824, -3.167790e-03, -0.0280568507, 0.0465047099, -0.0337861255, 0.0589633882, -0.0295954011, -0.0113416379, -0.0509023145, -0.0496374182, 0.0826921388, 4.208220e-02, -0.0527950488, 0.0357766747, -0.0352644548, 0.0569184087, -0.027063176, 0.0453298427, -0.0076854513, -0.0109013533, -0.0461692847, -0.0157904476, -4.147390e-02, -0.0281782448, -0.0571003594, 0.0617556795, 0.0281314515, 0.0548385717, 0.0092664007, -0.0257337987, -0.0648162141, 0.0243072417, -0.0197841916, -0.0514738671, 0.0511602946, 0.00914279185, 0.0615776032, 0.0499278083, 0.0390559137, 0.033231508, 0.0122929737, 0.0274427254, -0.0560475551, -0.0306472182, 0.0417289101, -0.0309007447, -0.0805537328, -0.0233501289, 0.0617299973, 0.0580678023, 0.00587657187, 0.0161564387, -0.0471677855, 0.063634865, -0.00921861641, -0.0668429285, 0.00189050485, 0.0363897569, 0.0313178264, -0.0372405499, 0.0222265087, 0.0307268184, -0.0248517636, 0.0564697459, 0.016861286, -0.0374078713, -0.0142475925, 0.0808056965, 0.0246974751, 0.00213992712, 9.057040e-03, 0.0359845385, -0.0101868194, 0.00580740487, -0.0170100071, 0.0578632317, 0.0174059197, 0.0779294744, -0.031877391, -0.0719019175, -0.0403279923, 0.0385066122, -0.00912816636, 0.0588704534, 0.0400143303, 0.0161176212, -0.0155105721, -0.0169495102, -0.00573512586, 0.0384766683, 0.0564637072, 0.00680151442, -0.0622516349, 0.0375281386, 0.0199288502, -0.0287022721, 0.0377983935, 0.0333602205, 0.0321128257, 0.0375535637, -0.00675788103, -0.0269715656, -0.0380300321, -0.0117434673, -0.0775015726, -0.0493023507], [0.0188797526, 0.0804885253, 0.0273955185, -0.00482011307, -0.0685239211, 0.0789573863, -0.0297896173, 0.0141853485, -0.0327802524, 6.756730e-02, -0.0516750179, -0.00299913203, 0.0441700965, -0.0505444221, -0.0460968539, -0.0368991084, 0.0567178354, 0.0328885913, 0.0177287683, -0.0545495115, 0.00792708806, -0.0140873082, -0.0077301641, -0.0376471318, -0.0562066063, -0.0503099114, -0.0496381968, 0.00336898514, -0.00264717662, -0.0274141282, -0.0301701352, -0.0714569389, 0.0451277122, -0.0127498703, -0.0720039904, -0.0573576503, -0.00972120091, -0.0590353683, 0.0422139801, -0.0679135844, 0.0731280893, 0.0135194175, 0.0531543717, 0.0220223926, 0.0354746729, 0.00460457336, 0.0115903569, -0.0388686918, 0.0646691322, 0.0606316738, -0.0183729958, 0.0538915209, 7.760330e-02, -0.00903049856, -0.0662947819, -0.0410806201, 0.0363231115, 0.0110946102, -0.0630921498, -0.080853343, 0.0779462904, 0.0449488834, -0.0604558252, -0.0523788035, 0.0583798438, -0.00297644734, -0.0324487649, 0.0547344759, 0.0595818125, 0.0631019399, -0.0348102897, -0.010002465, -0.0406840965, 0.0121331764, -6.562990e-02, 0.057537023, 0.0736615434, -5.113730e-02, 0.0722294077, 0.0196733903, 0.00517418794, 0.0241750181, 0.0191956479, 0.0426050164, -0.0662553683, -0.0452361032, -0.0126495594, 0.021304179, -0.0252322275, -0.0420398191, -0.0797942951, 0.0777729899, -0.00944389961, -0.0310526639, -0.0345811695, 0.00242809276, 0.0288550071, 0.0513517708, -0.0684079602, 0.0376230963, 0.0172852427, 0.0116878906, -0.044122234, 0.039179828, -0.0160048511, 0.0239048656, -0.00248226244, 0.0606737733, -0.0224889144, -0.0502087176, 0.0395122617, -0.0763810276, 0.0035778305, 0.0806487873, 0.0478901528, -0.0484717712, -0.0727233067, 0.0415751226, -0.0112101613, 0.0655286089, 0.013745374, -0.00542216143, -0.0760478824, 0.0283044465, 0.0714802593, 0.0496119671, 0.0364979431, 0.0347721763], [0.0376464501, -7.400070e-02, 0.00576482434, 0.0285201874, 0.0146254394, 9.71555768E-4, -0.0790609195, 0.00322979316, 0.0809932649, 0.0376366451, 0.0690873712, 0.0499711819, -0.00844297744, 3.84701154E-4, -0.0365785584, -0.0537559502, -0.0547269247, 0.00798816978, 0.0772887096, 0.0495308265, 0.0025569126, -0.0670988634, 0.0297670849, -0.077518098, -0.0472911671, 0.0694394112, 0.0146558974, 0.0459511876, -0.0682666153, -0.047217954, -0.0710117221, 0.0629088357, 0.00434322329, -0.0132500716, -0.0267653521, 0.0226998571, -0.00955322571, -5.71733632E-4, -0.0725858286, 3.24748456E-4, 0.00437327754, 0.0223630443, 0.0537870489, 0.0534743555, 0.0270260423, -0.0221485589, 0.0451079234, 0.0794711187, -0.0408662371, -0.0199362244, -0.0537035279, -0.0303548723, -0.0654153228, -0.0746743977, 0.0207496509, 0.0147039872, 0.0671430752, 0.0434304252, 0.0503257141, -0.0446463451, -0.0115354732, -0.0469962507, 0.049769666, 0.0227779858, -0.0220042448, 0.0796997696, -0.0233393349, 0.0567793176, 0.067104362, -0.0330200456, -0.033906702, -0.0766977295, -7.052850e-02, -0.0220745113, 0.0630346313, 0.0203725938, -0.0211534686, 0.0602961108, 0.0804786235, 0.065939866, -0.0434286743, 0.0402383842, -0.00545635819, -0.0651542768, -0.0574532449, 0.0361688659, -0.0567463152, 7.929570e-03, -0.0275377594, 0.0614047758, -0.0517954305, 0.0417379849, -0.0333346613, -5.752760e-02, 0.0510964468, -0.0654548556, -0.0607361645, -0.0810479298, -0.0402617045, 0.0155158369, -0.0690862909, 0.055475343, -0.0610621683, 0.0699920505, 5.084250e-02, 0.047119081, -0.0232052971, -0.0389360599, 0.0195925757, 0.0110827303, 0.00353689771, -0.0734957308, 0.0465535671, -0.034759067, -0.0547613576, -0.0265343338, -0.0418839566, -6.934260e-02, 0.0596642718, 7.325370e-02, -0.0567340739, -0.0607841834, -0.0127615985, -0.0617415681, -0.0588030182, 0.0319401622, -0.0172426179, 0.045547124], [0.0242567714, 0.0443420745, 0.00849778671, -0.0677641928, 0.0300876852, 0.0709474757, 0.0169269945, -3.195040e-02, -0.0710342303, 0.0152427778, -0.0113293482, 0.0470393114, -0.00582493097, -0.0406513922, -0.0565559976, -0.0562621467, -0.0676944628, 0.0397636667, 0.0104300715, -0.00122927723, 0.0598416887, 3.03635665E-4, -0.0233769454, 0.00970955659, 0.0452567935, 0.0186289959, 8.67062714E-4, 0.0633145794, 0.0520936325, -0.0233601891, 0.0260647777, 0.042501796, 0.0227775704, -0.0794062837, 0.0809561908, 0.0689005852, 0.0493532196, 0.0721900314, 7.580780e-02, -0.0691681877, 0.00131956011, 0.0554942228, -0.00675800862, 0.00705641787, -0.0558428653, -0.0306652579, -0.0660676286, 0.00698621711, -0.0443826243, -0.0165388566, -0.0350534618, -0.0319472402, -0.0696713924, 0.0582285635, 0.0714340806, 0.00167524081, 0.056461852, 0.0347750559, 0.0511304438, 0.00758671388, 0.0543008298, 0.0339503363, -0.0359731093, 0.0236128271, -0.0127337975, -0.048761759, -0.0198749509, 0.0678976849, -0.0677134618, -0.0398754813, -0.075753428, -0.0660532638, 0.0209409893, 0.056590315, -0.0676127523, 0.0476077497, 0.0093304636, -0.00349089643, 0.0410139225, -6.516090e-02, -0.0491543896, -0.0621227584, 0.0418048091, -0.0479276329, -0.00860863551, -0.018901471, 0.0165658332, 0.0186290741, 0.0289148875, 0.0381396078, 0.016672153, 0.0616391152, -0.0752400234, 0.0491920337, -0.0370160379, -0.0323598385, 0.0126144495, 0.0678961426, 0.067849949, -0.0530294627, 1.732410e-02, -0.0225781556, -0.031110784, -0.0100528216, -0.0537390038, 7.961360e-02, 0.0360971875, -0.0561341271, -0.040375486, 0.0748670474, 0.0373730399, 0.0768812075, 0.0216811989, -0.0622154176, 0.0026269278, -0.0219363868, 0.0370947868, -0.0282576922, 0.0333009474, 0.0205303319, 0.00112898694, 0.0162115116, -0.0162841268, -0.00921490229, -0.00928807165, 0.0577788278, 0.0453204624, 0.00964896474], [0.0341058336, -0.0785874798, 0.0695298687, 0.0120695457, 0.027361989, 6.698150e-02, -8.617130e-03, -0.0653050616, 0.0635164306, 0.0103591699, 0.0658656955, 0.0494484492, -0.0742762536, -0.018327212, 0.0316790305, 0.0617647544, 0.0391611978, 0.0296692345, -0.0587011203, 0.0600507371, 0.0155083174, -0.0186087843, -0.0448026322, 0.0708969757, 0.06488505, -0.0380819663, 0.00280757411, -0.0796079487, 0.0226166882, -0.0476865247, -0.0344332904, 0.00425647805, -0.058731474, 0.0403033234, -0.0158308614, 0.0629537776, 0.00122063723, 0.0819392427, 0.076204963, -0.0521140508, -0.0777870193, 0.0502172783, -0.0326758325, -0.0666359067, 0.0559549481, -0.031290736, -0.0824203342, 0.0650863051, -0.0504603758, -8.24696267E-4, -0.0133480951, -0.0369566679, 0.053249836, -0.0233416241, 0.00964960828, 0.0745265185, 0.062321335, -0.0422919244, -0.0237089507, -0.0193339363, 0.0767101869, -0.0584963411, -0.045162119, -0.0642083287, 0.0310453922, 0.0749280452, -0.0333507247, -0.0331851467, 0.0745443999, 0.0512409694, -0.0379600562, 0.048230838, -0.0276507027, -0.0312079545, -0.0604452491, -0.0552283376, -0.0395640954, 0.0515976474, -0.0724106729, -0.0699331686, -0.0311208647, -0.0169287343, -0.0217653196, 0.0379598401, 0.03094423, 0.0714439451, -0.0232299082, 0.0657036081, 0.0515894964, -0.0692979842, 0.0549974367, 0.070753254, -0.0746561363, 0.0471733958, 0.0233463719, 0.0690398142, -0.059293218, -0.0685845837, -0.0304934774, -0.00329883583, -0.0223546736, 0.00551673723, 0.0472516678, -0.014848141, 0.0193791706, 0.0594771691, 0.0112881539, -0.0337842032, -0.0472068638, 0.011128976, 0.0208996963, -0.0199798681, 0.0047784471, 0.0635010451, -0.0441182517, -0.0221048407, -0.0348198302, -0.0563405156, -0.00720709655, -0.0183247291, 0.0576248541, -0.0234354809, 0.0246109478, 0.0499556884, -0.0686076432, 0.0636849627, -0.00249087624, 0.0366710536], [-0.0262082163, -0.00341560016, 0.0796293765, 0.0429116823, 0.0578774326, -0.0577755161, -0.0117747616, -0.0255401246, 0.0594094805, 0.0428757481, -0.00122436334, -0.0588872544, 0.0038924017, -0.0528229699, 0.0440200567, -0.0402766466, 0.0151454927, -2.708600e-02, -9.020110e-03, 0.0659846514, -0.0283038616, 0.0243140198, 0.0715058818, 7.959140e-02, -0.0316382907, 0.00576083874, 0.0732545257, 0.0331212841, 0.00284279138, -0.00365514914, -0.0198188461, 0.0627335086, 0.0744864419, 7.953720e-02, -0.0801124349, -0.0575074926, -0.0608178042, -0.0251277238, -0.00657738466, -0.0118181324, 0.015350705, 0.0184371639, 0.0297858082, 0.0719988495, 0.0563245267, -0.00154036784, 0.0275167376, -0.0657231212, 1.20227422E-4, 0.0723139569, -0.0518159419, 0.0188562106, 0.0269347988, -0.0329298116, 0.0331135653, 0.0431532264, 0.0762311742, -0.04161226, -1.94884851E-4, 0.0383834578, 0.0462098084, -0.0708418489, 0.0642971918, 0.0743752643, 0.0083760405, -0.0421778224, -0.0793892145, -0.0138991643, -0.0221174713, 0.0354341827, -0.0109315738, -0.0502302647, -0.035007216, 0.0604642853, -8.414020e-03, 0.0383974425, -0.041728273, -0.0630045757, 8.273350e-03, -0.0238590594, 0.00885326508, -0.0338000469, 0.0755978897, 0.0766447559, -0.043961212, -0.0754089728, 3.274420e-02, -0.0576877668, 0.0577344224, 0.0702103302, 0.0265651979, 2.774340e-02, -0.0613211207, 0.0386290774, -0.0770022869, -0.0249400605, 0.0501521453, 0.0492893755, -0.0125019765, -0.00423023617, -0.0174144227, -0.074481815, -0.0530410334, -0.0551243387, -4.23325197E-4, 0.0375726037, 0.0536044128, -0.0703564286, -0.00850340445, -0.0535123758, 0.0341266803, -0.0164484959, -0.0720812231, -0.0471864343, 0.0352899693, 0.0215937141, -0.00780086219, 0.00840804726, 0.0178405475, 0.0542122945, 0.0751921386, -0.00645428663, 0.0762180313, 0.0460938737, -0.0668323264, -0.0725901872, 0.0227825139, 0.072720781], [7.847560e-02, 0.0241915267, -0.0370827504, 0.019681666, 0.00189022254, 0.0181771535, -0.0603210367, 0.026060883, 0.0289861169, 0.0822169632, -0.0765937567, -0.0748208761, -0.06234622, -0.0470642895, 0.0631786957, -0.0206999052, -0.0483616665, 0.0256368257, -0.0499346443, 0.0502840467, 0.0364073366, 0.0398293212, -0.0622540675, 0.0685618073, 0.0012951747, -0.0798002257, 0.0150472391, -0.0537711866, 0.0186450668, 0.0333657824, -0.0370067284, 0.0288014468, -0.0591785163, 0.0422548279, -0.0619465634, -0.0360897705, -0.0445500165, 0.0611826852, 0.0444400385, 0.0194597878, -0.0274363961, -0.0210956801, 0.0282904487, 0.0808097869, 0.0471259467, -0.0122607881, -0.0396802314, 0.0456904843, 0.0183443893, 0.075761281, -0.0718346685, -0.0402854308, 0.0530565418, -0.0292060152, 0.020475477, -0.0339242108, 0.0187347569, -0.00155723607, -2.19382651E-4, 0.0700187981, 0.0305477772, 0.0361901186, 0.0569875352, 0.0480388068, -5.145230e-02, -0.0360221379, 0.0468351915, 0.0221742634, -0.0103137018, 0.0193807241, 0.0230850093, 0.0126838908, -0.00633029547, 0.0765456781, -0.0796849429, -0.0807541683, 0.0488432609, -0.025463121, 0.0619369857, -0.0595082603, 0.0192584936, 0.0347784199, -0.0817921459, 0.0570184812, -0.0552733615, -0.0132570481, -0.043738097, 0.0377948694, -0.0733992904, 0.0529836826, 0.0683260858, -0.0620573461, -3.441800e-02, -0.0721574426, -0.0169317015, -0.0680815056, 0.0343711227, 0.0330563411, 0.0458262675, -0.0394632556, 0.0586774126, -0.04472249, 0.062126398, -0.0122340079, -0.0244360752, 0.0693137347, -0.00725884084, -0.00831239205, -0.0239044074, -0.0430597663, 0.0757484287, 0.01590297, 0.0489349701, -0.0148561932, -0.0518719219, 0.0463513955, -0.0721678957, -0.0249823928, -0.0135683073, -0.0208522882, -0.00488752127, 0.0241054147, -0.0341499597, 0.0705314576, 0.0267331544, 0.0681337789, 0.00576089509, 0.0637055039], [0.0532249548, 0.0826428905, -0.0033989544, -0.0357836671, -0.0173538625, 0.0169770252, 0.0628209934, -0.0252898391, -0.0706203877, 0.06341438, -2.561420e-02, -0.0501554534, -0.0464829691, -8.71453492E-4, 0.0258495547, 0.0702604726, 0.0280950144, 0.0578605346, -0.0710022673, 0.0153576303, -0.00882676616, 0.0238981117, 0.0290353894, 0.0285456572, -0.0507911742, 0.0486595705, 0.0296019614, -0.0130859101, 0.0284697339, -0.048918508, -0.025348952, -0.00826882571, -0.0521701761, 0.00428388081, 0.0235794969, -0.00563987624, -0.00264915498, -0.0453430936, -0.045629438, 0.0325753316, 0.0483434908, 0.0722669289, -0.00592113473, -0.0125398207, -0.0716899633, 0.0367157385, 0.00842054468, -0.0298306569, -0.0244860463, 0.0211677644, 0.00488968799, -0.00889849103, 0.0230758619, -0.0469035767, -3.605390e-02, 0.00872893817, -0.0284200739, -0.0836884602, -0.0508311316, -0.0484751351, -0.058860179, -0.0339957066, -0.040086586, 0.016596688, 0.0323947705, -0.049865406, -0.0701410174, -0.0264344942, 0.0283993799, 0.0183461346, -0.0155289685, 0.0464467518, -0.02302837, -0.0638145581, 0.0563283488, 0.0172882285, 0.00105052115, -0.0534255467, -0.0044355127, 0.00495195854, 0.0965309888, 0.00691421097, 0.00448006904, 0.0267627873, -8.476240e-02, -0.0286131464, -0.0799522325, -0.0678577647, -0.0717832372, 0.0253975783, 0.0595446378, 0.0148212453, 1.215650e-02, -0.00110316172, 0.00720252143, 0.0617819913, 0.0304837096, 0.0754889473, -0.0550945811, 0.0436797366, -0.00554089248, 0.021357676, 0.0284065306, 0.0299441312, 0.0593852364, -0.0702831224, -6.972170e-02, -0.0152406069, -0.0049645463, -0.0353033282, 0.0821202471, 0.0377404541, -0.0481687412, -0.0620530173, -0.0800708383, -0.026017366, 5.385800e-02, 0.0614302903, 6.517440e-02, -0.0246204957, -0.0261021331, 7.79569556E-4, -0.00291237631, 0.0198729858, 0.0284938868, 0.0112280995, -0.0571232922, -0.0761334896], [-0.0254181866, -0.00797854922, -0.0559499525, 0.00366325863, 0.086814396, 0.00547557138, 0.0299612321, -0.0332590975, -0.0713306218, 0.0258788615, -0.0801018625, -0.0351376049, -0.042533908, 0.0319795124, -0.0143266171, -0.0513467789, 0.0613416173, 0.0204953663, -0.0622777417, 0.051441621, 0.012424021, 0.026625257, -0.0223764908, -0.0715931803, 0.0138922986, -0.0670193359, -0.0809904858, -0.0278819129, 0.0839929581, -9.163930e-03, -0.0560508668, -0.0896567553, -0.0149509888, 0.0363391675, 0.0414912961, 0.0454479381, 0.0453462675, 0.0322555415, 0.0619399733, 0.0875241905, -0.0471112095, -0.0564050451, -6.973320e-02, 0.123391837, 0.0777720585, 0.00165759365, 0.032723207, -0.0101405764, -0.0025295876, -0.00925383158, -0.0649749934, -0.0129683921, 0.013657704, -0.0213609505, -0.0056151934, -5.640240e-02, 0.00810426846, 0.0288396385, -0.0557151772, -0.0167576112, 0.00969385821, -0.0883962139, 0.0705538243, -0.0730256885, 0.0173037313, 0.0106216082, 0.0119448202, -0.00195018994, 0.0464213341, -0.00540548516, -0.0856298282, -0.0247272514, -0.0316727981, 0.048659008, -0.055122979, 0.0332256705, -0.0432455651, -0.0467169471, 0.045115184, -0.0415373035, -0.0561517663, 0.076021865, 0.00472225482, -0.0656431392, 0.0471857265, -0.0395034403, 0.00934527255, -0.0476540476, -0.014106066, -0.0431472436, -0.0367101952, 0.0331781693, -0.0712731406, 0.0883311778, -0.0566195548, -0.0178468395, -0.0167602859, -0.0025569615, 0.0708364099, 0.00166160008, -0.0527172051, -0.00691548781, -0.0650028363, 0.0382994637, -0.0229303557, 0.0626160353, -0.0363589339, 0.0222675055, 0.0620356611, -0.0498508513, 0.083344005, -0.0228342842, -0.0866036415, 0.020827217, -0.0397901535, 0.00420634169, -0.0713171139, 0.102149345, -6.56495162E-4, 0.00658884132, -6.888800e-03, 0.064719744, -0.0492210239, 0.0153342476, 0.0485394113, -0.0544054247, -0.059380848, 0.0179893598], [-0.0259940773, -0.0436112247, -0.0764571577, -0.0778256356, 0.0155349057, 0.0604019053, 0.00724931108, -0.0134280045, -0.0360226929, 2.1073813E-4, 0.0370434783, 0.0490997769, 0.0562153943, -0.0863816812, -0.0205674898, -0.0238124151, 0.0779138281, 0.0303788986, 0.0146847321, 0.0560424142, -0.0705025718, -0.0638848096, 0.0721374899, -0.0429882854, 0.0256054271, -0.00823542383, -0.0281644166, 0.00456203241, -0.0438218825, 0.00802553817, 0.00546217477, -0.0425111912, -0.062902756, -0.0189275704, 0.0595868342, 8.087440e-02, -0.0699910074, -9.482800e-02, 0.0496258028, -0.00198430079, -0.0440195203, -0.0365196019, -0.0519166328, -0.00920198857, 0.086205691, -0.0351976044, 0.0172303747, -0.00622071931, -0.0660590902, 0.0124393273, 0.0449427329, 0.050549835, 0.0124621168, -0.0900971964, -0.0356753133, 0.0417513326, -0.0401737727, 0.0509619527, 0.0613586493, -0.100241788, -0.0505308434, -0.015241879, 0.0596417338, 5.458890e-02, 0.0566713586, -0.0895488932, 0.00721822632, -0.00749497907, 0.0888744518, -8.97991703E-4, -0.100070752, -0.0302141458, -0.0773437321, 0.0258130562, 0.0184508152, 0.00333243352, -0.0116551938, -0.0113717923, 0.086944595, -0.0439145602, 0.0515557677, -0.0500840358, 0.00318346382, -5.930480e-02, 0.0139857605, -0.0737214535, -0.0605251677, 0.010342502, -0.0379228294, -0.0591790676, 0.0502380952, -8.186250e-02, 0.0126371402, 0.020353524, -0.0735380351, -0.0739798545, -0.0697143525, 0.0432894081, -0.0628192201, 0.0451262854, -0.00981252734, -0.0524515584, -0.109421447, -0.0272162277, 0.0887147635, -0.0585103594, 0.0778662487, 4.484190e-03, -0.0734740347, 0.0808577091, -0.0563047677, -0.0154831717, -0.019154055, 0.0100819869, 0.0422430485, 0.0511084981, -0.0482040457, -0.0452758186, 0.103461027, -0.10144975, 0.0439443178, 0.0320194699, 0.0163014252, -0.0255962647, -0.0322916359, -0.0338374339, -0.0094746612, 0.0182269439], [0.0264330227, -0.0664319471, 0.0388854668, -0.0951806456, 0.0139953652, -0.0464297347, 0.00782430544, -0.0521956645, -0.0127083156, -2.883020e-02, 0.00602290826, 0.0566292964, 0.0278695934, 0.0408961885, -7.080950e-02, -0.050487753, -0.0398174152, 0.0311147291, -0.0707871169, 0.10523247, 0.0630723536, 0.015978083, 0.0178884622, -0.0393428802, 0.106225178, -0.0264554136, 0.0286748465, -0.00160988024, 0.00939016602, -0.041641891, -0.0216453169, 0.00769744581, 0.045725476, 0.0639662743, 0.00101693766, 0.0595609769, 0.0863485708, -0.0549143404, 0.026962107, -0.0151694901, -0.0252522547, -0.0319749117, 0.00895955599, 0.0139452294, -0.0304035749, -0.0990026891, 0.0299245901, 0.0614477284, 0.0305553246, -0.0162537284, -0.00481178751, -0.00650581624, -0.0615512915, 0.0225666817, -0.0527251884, -0.069283925, -0.0918256416, 0.00343503058, -0.047319714, -0.081895329, -0.00618031761, -0.0115704034, 0.0645776466, 0.0220638867, 0.0136265485, -0.0600309372, 0.103173554, 0.0244220756, 0.00954534579, -0.0443031862, -0.107487716, 0.0204035304, -0.00229062629, 0.0285165049, 0.0823230519, 0.068637453, -0.011883867, 0.06978558, 0.0897553339, 0.0122405738, -0.0475983918, 0.085682109, -0.0953761935, 0.00959746725, 0.0647861436, 0.0260631815, -0.0678808242, -0.0388881899, 0.00389534235, -0.0351815782, 0.0675155669, 0.0220527407, -0.0448943041, -0.068719089, -0.0194384642, -0.0884416252, 0.0259214826, -0.0110381674, -0.00517100561, 0.0987857803, 0.0637352764, -0.00699479179, -0.111779667, -0.0210868046, -0.01335862, 5.677390e-03, 0.060230229, -0.0271024406, 0.0342584141, 0.0952604115, -0.0126679335, 0.0495441519, 0.0315624364, 0.0120225279, -0.0611484572, 0.046296645, -5.28806762E-4, 0.025804596, 0.067419149, -3.708900e-02, -0.0319976583, 0.0279223323, 0.0497339107, -0.0870543196, 0.0189858284, -0.0824316591, 0.0128153693, 0.0724721625], [0.0815544724, 0.0323799774, -0.00419297628, 6.53988915E-4, -0.00736986753, -0.0150995124, 0.0270416476, -0.0110843685, 0.0421486311, -0.00172727741, 0.0466578044, 0.0851338207, -0.0233125128, 0.0126415445, -0.0028007275, -0.0252884217, 0.0713792592, -6.517690e-02, -0.030246947, 0.131547242, 0.0599001348, 0.0325092562, 0.0524144769, 0.00471161073, 0.012114685, -0.00409982912, -0.0525659397, -0.0365895815, 0.0648898259, 0.0157369561, 0.0504091494, 0.0102667166, -0.0599923879, 0.0483100787, 0.0305320621, 0.0423928089, 0.0834414586, -0.0626582727, -0.0228532217, 0.0973948091, -0.00620354433, 0.071217604, -0.0487737916, 0.0124276429, -0.0237196125, -0.0963804647, 0.0725800096, 0.0698657706, -0.0265642591, 0.0506038107, -0.0629940107, -3.347470e-02, 0.00821175612, -0.0797882229, 0.0150794815, -0.0744591728, -0.0159965362, -0.0649563596, -0.0077769123, -0.102237701, 0.00750815682, -2.377640e-03, 0.0124053182, -0.0249701571, -0.0185674652, -0.0205840357, 0.0850036889, -8.10131954E-4, -0.0676433593, -0.0019775047, 0.0264861435, -0.00189787056, -0.0250075106, 0.0451258123, 7.239300e-02, -0.00606644247, -0.085627906, 0.0727890655, -0.0475883335, 0.00531204045, 0.0528361127, -0.0226769056, 0.0165277589, -0.0540848076, -0.066356346, -0.0485801026, 0.0158358868, 0.0370778926, -0.00841073505, -0.0154987695, 0.032950934, -0.101635039, 0.0944754853, -0.100645043, 0.00554373069, -0.0378387161, 3.149510e-02, 0.0770472139, -0.031066671, 0.0542976409, 0.0342072435, -0.015800694, -0.0846767277, 0.0369656198, 0.0706859529, -0.0722635835, 0.0585672371, -0.0425833538, -0.0540200099, 0.00114815717, -0.0432033874, -0.0470438041, 3.286290e-02, 9.60328761E-5, -0.0251210053, 0.0587804392, -0.0974876135, 0.0825851411, 0.00301430025, 0.0241092406, 0.0760637373, 0.0118622361, -0.0387955643, -0.0598902665, 0.0245259814, -0.0640468597, -0.00765919592, -0.0372225903], [-0.0428906344, -0.0719360635, -0.0717478395, -0.0754322782, -0.0243300926, -0.0039017098, 0.0189874265, 0.0208870526, 0.0369037725, 0.0375091843, -0.0823613181, 0.0500363186, 4.660180e-02, 0.0388576575, 0.0141675714, 0.0445873924, -0.0481090471, -0.0338626541, -0.016952157, 0.0605217144, -0.042538695, -0.0407856554, 0.0114929369, 0.0281433705, 0.0565566234, -0.0735807344, 0.0351080596, 0.0528063029, 0.0484106317, 0.0166559163, 0.0264276844, -0.045246359, -0.00413247244, -0.0313951224, 0.0117892427, 0.123476654, -0.0196134243, -0.00186944765, 0.0879157185, 0.0466445647, -0.0355082974, -0.0510108471, 0.0552816428, 0.0995845943, 0.0497866161, -0.0576893538, 0.132619545, 0.0680253282, 0.09300749, 0.0828153565, -0.0875439271, 0.0405097343, 0.0185176581, 0.0140805617, 0.0483644456, -0.0629582629, -0.017062718, -0.0263745692, -4.201310e-02, -0.0381521247, 0.0322967358, -0.0347711928, 6.320160e-02, 0.0314681865, 0.0620047636, -0.0453038774, 0.0203449503, 0.0119923782, 0.0635628328, 0.00782206282, -0.0858729779, 0.113302678, 0.0276127346, 0.0375165269, 0.0201684106, 0.0144796781, 0.0451761819, -0.0226428863, 0.0372515731, -0.0211501457, -0.0896458625, 4.261490e-02, -0.044365216, -0.0476750657, 0.026003832, -0.0206771418, 3.111060e-02, 0.0237297826, -0.0211806223, 0.0851560756, -0.0428830683, -0.0370511115, 0.0454837494, -0.00132020668, -0.012455754, 8.441640e-04, 0.00405380037, 0.0185898636, -0.0883152782, 0.0680119768, 0.0120165078, -0.0569894202, -0.010319937, 0.0152629204, 0.013374134, 0.0840413495, 0.119840503, 0.0259960331, 0.0654845312, 0.0267873034, -0.00911056157, -0.0394297466, 1.070440e-02, -0.0102180773, -0.0145211313, 0.0743266419, 0.0055908612, -0.0385306552, 0.111327752, -0.0574406274, -0.0401729159, 0.0501988903, -0.00637232698, -0.14538376, -0.0247717351, 0.0617558584, -0.0402097255, -0.00969607755], [-0.0347327627, 0.0323457085, 0.030769458, -0.0326790027, 0.0365934931, -0.0897486955, -0.0722861141, -0.0857204496, -0.039103061, -0.0179138388, 0.0364080146, 0.00712152431, 0.0119043235, -0.00654612854, -0.0287511237, -0.0346985906, 0.040882159, 0.0430841632, -0.094854787, 0.0961782336, -0.0610014535, -0.038204506, -0.0236783531, 0.0620965063, 0.027307434, -0.0256955419, 0.0152374664, 0.0271498896, -0.0603436455, 0.039344538, -0.0480119288, -0.0148371952, 0.0898243114, -0.036656443, -0.0547238067, 0.02423683, 0.0341460072, -0.0765959173, -0.0237456076, 0.0230195895, 0.00394488731, -0.0600963198, -0.0471342504, -0.0282957964, 0.0586076379, -0.0252156351, 0.0475468487, 0.0318595879, -0.0307741407, 0.0289749354, 0.0569174327, 0.0497638099, 0.065614529, -0.00497210305, 0.00766006298, 0.075875625, 0.00320578553, 0.0258522425, 0.0161138624, -0.0702465549, 0.0296535622, -0.0135628451, -0.0285105873, -0.0272406545, -0.0218745135, -0.0373412296, 0.131544456, -0.0653371512, 0.0313484967, 0.0135191465, 0.0370067582, 0.0571525767, 0.0417058617, -0.0591746494, -0.027976973, -0.0509366803, 0.00686870608, -0.0231804643, -0.0706585422, 0.101777218, -0.0259976368, 0.00487575633, 0.00969612878, 0.02575198, -0.0107838018, -0.041221071, -0.03800961, -0.00707052182, -6.917770e-02, 0.0109027596, 0.0207952578, 0.0407575108, -0.0205594786, -0.0997875928, -0.0566195659, -0.0498466417, -0.0523480661, -0.0663081333, 0.0209258534, 6.315720e-02, -0.0253690518, -0.01857825, 0.0360476114, -0.0684577301, 0.0561662279, 0.00286961044, 0.0709010363, 0.00809302274, 0.00467406632, -0.00198836508, 0.0873819962, 0.0241970662, -0.0412855558, -0.084846206, 0.0761170685, 0.0760330557, -4.636020e-03, -0.00118283858, -0.0210262425, 0.0410211533, -0.026469063, 0.132372648, -3.334680e-02, -0.0948490872, 0.0385726281, -0.0185216349, -0.00696001109, 0.0662511885], [0.06741561, -0.0556392893, 0.0367351137, -0.0450532362, -0.00699903537, -0.0674092919, -0.0294464827, 0.0526850149, -0.0728729293, 0.035044957, -0.011655028, 0.103975862, -0.092560552, -0.0704420506, 0.0499797165, 0.0628294572, 0.0450634621, 0.0667968392, 0.0215012692, 0.0422673784, 0.0292560235, 0.0204974711, 0.089986585, -0.0245955139, 0.0707147643, -0.0733658299, 0.0418505557, 0.0154084973, 0.0497599058, -0.00988323241, 0.0461294539, 0.0682130903, -0.0521086715, 0.0792432874, -0.0636527166, -0.0643675551, -0.0521935113, 0.00359374355, 0.0579080544, 0.0169024114, -0.0463135131, 0.0730533525, 0.023570478, 0.0385268405, -0.0297217183, -0.0272033177, 0.143894017, 0.105324142, 0.00694571901, 0.0389309488, 0.0866266042, 0.0586190149, 0.029274784, -0.0321051627, 0.0600789264, -0.00795042701, -0.0727635473, 0.00192014466, 0.0434824228, 0.0453706533, 0.0287544318, -0.0878815948, -0.0150818322, -0.00335785584, -0.00378252077, 0.0379821695, 0.0760285184, -0.025866447, -0.0792366936, -0.0549113341, 0.0668808371, 0.0112114316, -0.0316645391, 0.0542756543, -0.0308218785, 0.0336285383, -0.00599448103, 0.0744156912, 0.00523827411, 0.0627352521, 0.0349277779, 0.0113123739, 0.0262425262, 0.0486163162, 0.0300862268, -0.0415797494, 0.0536089875, 0.00280864653, -0.0867904425, 0.0795678496, -0.0240706075, 0.0629484579, 0.0853407308, 0.00628323946, 0.022614209, 0.00913000293, -0.0925136954, -0.0631341934, 0.0386145338, 0.0421469212, 0.0647572502, -0.0484845862, -0.0405880623, 0.0219418705, 0.0610113293, 0.0347104892, 0.101407982, -0.00284509663, -0.0428153574, 0.0140088964, -0.0527857319, -0.052748017, 0.0738328993, 0.110666439, -0.0345732234, 0.0426643789, 0.10599865, -0.0412643477, 0.0266492683, 0.0185314287, -7.67409336E-4, 0.014545504, -0.00751223228, -0.0270995814, 0.0224170834, -8.769740e-02, 0.0787816122, -0.00694445288], [-0.026377229, -0.023760125, -0.0104457932, 0.0540255234, 0.142203093, 0.00171357917, 0.0212020129, 0.00282620639, -0.0376079232, -0.116623528, -0.01095731, -0.0236458033, -0.0403477475, -0.0711146444, -0.0568215363, 0.0663365349, -0.0437450483, -0.0747809485, 0.00236207712, -0.00499755563, 0.0721517429, 0.0461865515, 0.0277061425, -0.025603611, 0.0495841727, -0.0609584637, -0.0269451607, -0.0168474186, -0.00998431537, -0.0244438909, -0.0294481069, 0.0889120176, 0.0560481958, 0.0578230359, -0.0578956939, -0.00794858857, -0.0883316099, 0.100812525, -0.00254303054, 0.0331395604, -0.015605134, 0.0825161412, -0.080782622, 0.00818898436, -0.0514707603, 1.154240e-02, 0.107983038, 0.0676886737, -9.21541126E-4, -0.087559171, 0.0825566574, -0.0259640776, -0.0503784455, 0.0948381796, 0.032044448, -0.0230291989, -0.0657979547, -0.0362995379, 0.055795908, 0.0745515823, 0.104018085, -0.0687987804, 0.0317003764, -0.0576021522, -0.0744580924, 0.077896662, 0.0948184207, 0.0192464069, 0.0165426861, -0.0347431153, -0.00988747365, -0.00177888991, 0.0689181611, 0.0424945205, -0.0176156275, 0.0790436938, 3.239380e-02, 0.0678085536, -0.129289269, 0.0666011944, -0.0280356538, 0.0956013575, 0.0214878749, 0.115462914, 0.0693266689, -0.0213815048, -0.00568222208, -0.0377520099, 2.804210e-02, -0.0544515923, 0.0242232196, 0.053558372, -0.0574973784, 0.00701596914, -0.0402849838, -0.046497412, -0.100932069, 0.0486722402, 0.0930173844, 0.00524555333, -0.0289961342, -0.0566168092, 0.0228453223, 0.0239630863, 0.0236268528, 0.108059667, -0.0010443714, 0.00221146946, 0.0590904914, -4.3502578E-4, 0.0887857303, -0.0594828128, 0.0367989615, 0.0749524683, 0.0788684338, -7.650780e-02, 0.11077226, -0.0299655646, 0.0425201654, 0.0531608425, -5.967570e-02, 0.0729345679, 0.0126888966, -0.108605757, -0.0561875626, 0.059334904, 0.05143217, -6.275060e-02], [0.00782774202, -0.0671308637, -0.0764017999, 0.0069861454, 0.0916361287, 0.0963934361, -0.0536825433, 0.00266167591, 0.0576395802, -0.106402107, 0.0830742344, -0.025449818, -0.00142417441, -0.00326975482, -0.0818323865, 0.0510486253, -0.00244650291, 0.013517282, -6.48990681E-4, -0.00706091523, -0.0200829096, 0.0293508396, 0.0130277397, -0.014432569, 0.0271738525, -0.035858348, 0.106361158, 0.0806930735, -0.046657566, -0.0591996647, 0.037627656, 0.0185360033, 0.0715300366, 0.0255039278, -0.056005694, -0.0933667048, 0.0536288545, -0.00371776847, -0.00686943624, 0.0171867386, -0.0529359132, -0.0458560213, 0.0196595881, 0.0558717586, -0.0350310355, 0.0256590452, 0.19352147, -4.52440931E-4, 0.0238029733, -0.0507767424, 0.0768448487, -0.0205228925, 0.0735893101, 0.0257995576, -0.0619692318, -0.0415682122, 0.0335829966, -0.0901820809, 0.0226026662, 0.102571152, 0.0151268765, -0.0903957784, 0.0832322091, 0.0164853018, 0.041410923, -0.0337740108, -0.0455251187, -4.414720e-02, -0.0987196713, -0.0424389876, 0.0815106481, -3.749680e-03, 0.056564182, 0.00654981471, -0.0371342562, 0.0146452393, -0.0184450839, 0.0746337548, -0.00334290857, -0.0452098064, 0.0247427803, 0.121987939, 0.0209351014, -0.01091603, 0.0875271484, -0.0308475513, -0.0651740506, -0.0517687909, 0.0738879889, -0.0777120217, -0.0276936404, 0.0660724341, -0.0113459667, -0.0337945521, -0.022900166, -0.0183904041, -0.0739627406, -0.0711251274, 0.142882079, -0.0373893566, 0.0777676701, 5.004830e-03, -0.0346054845, -0.0481464826, 0.0859912112, 0.00648830644, -1.695430e-02, -0.0171181429, -0.077438049, -0.0174412988, -0.0353412814, -0.0471827835, -0.0324384198, 0.148545846, 0.0244627502, 0.050133463, 0.113090336, 0.0337796584, -0.0262333509, 0.0356466211, 0.0106200967, 0.0730300843, 0.0303128306, -0.120126411, 0.0568815395, -0.0186460242, 5.523630e-02, 0.00804374646], [-0.0745670199, 0.06100481, 0.0707256943, -8.496110e-02, 0.0566031076, -0.00227007596, -0.0571836419, 0.0232479554, -0.0587691218, -0.0316953808, -0.0330582336, -0.0188772492, 0.0439624675, -0.0112614436, -0.0919539257, -0.0305521712, -0.0425084792, -0.00686450116, -0.118779816, 0.0100502344, 0.0695955753, -0.0377129912, 0.0880895555, 0.00841615721, 0.0746125728, 0.00360103836, -0.00948892161, 0.0864246785, -0.0720441043, -0.113249287, -0.0715470836, 0.0783350915, -0.0899292082, 0.0989968553, 0.0435391329, -0.00953125581, 0.0706987903, 0.0662805587, -0.0179117136, -3.34486649E-5, 0.0189849846, -0.0439514332, -0.0911993533, 0.0353148766, 0.0412684344, 0.0424891748, 0.107939616, 0.0328148231, 0.0468813293, -0.0574644096, -0.0678247064, -0.0755684301, 0.0513259731, -0.00301669142, -7.469450e-02, -0.057366211, 0.0111812837, 0.00213774666, -0.0195837338, 0.0674340054, 0.083227463, -0.0850253999, 0.055078432, 0.0253830068, -0.00460499525, 0.091077134, -0.0783805474, 0.0512764826, -0.0557686836, -0.0276107881, 0.0166716464, -0.0673319176, -0.0473960377, -0.00257841684, 7.433400e-02, 0.00864432379, 0.0993119925, 0.0464645438, -0.00752502447, -0.0288846213, -0.0521655753, 0.0982508733, -0.0374662317, 7.080480e-02, 0.0541274026, 0.0660696104, -0.0704949945, -0.00297417445, 0.0333571732, 0.0650142729, 0.0103392797, 0.0536357053, 0.0655430183, -0.0846797377, 0.00939824059, -0.0107976468, 0.0671960637, -0.059236709, 0.12958692, 0.0954491198, -0.00799653213, -0.0335342139, 0.0694512054, 8.34865495E-4, 0.0683661923, 0.131307691, 0.015082404, -0.038238734, 0.0652376637, -0.0388287529, 0.0321568809, 0.0756900087, -0.00233185687, 0.126290634, 0.0509623662, -0.0650145709, -0.0661162362, -0.020751467, 0.0724381655, 0.00782589614, 0.0853722393, 0.0301952343, 0.0924387276, -0.0489649586, -0.069188945, -0.00131187553, 0.0203752648, 0.0386869423], [-0.0598746836, 0.0181625038, 0.0309386142, 0.0218852218, 0.0640866384, -5.59816952E-4, 0.0688125491, -0.00119533564, -0.029127337, -0.0305782109, 0.0582833029, 0.11383792, 0.0233006552, 0.01242192, -0.0249050632, 0.0231798068, 0.0402606949, 0.0442956947, -0.0434868261, 0.0177418403, 0.0452562049, 0.016589947, -0.0251446944, -0.0167636666, -0.0316340402, 0.0154849561, 0.0194743164, -0.0183285791, -0.0772993564, 0.0199157577, -0.0747413784, -0.0107530542, -0.0789299458, 0.126932874, 0.00138074614, -0.0062134848, 0.00112126209, -0.0259075239, 0.030322412, 0.0260222144, 0.00770703284, 0.0603020899, 0.0335404165, 0.0752117112, 0.0187670458, 0.0865882113, 0.0567389466, 0.080560863, -0.0618014633, -0.00415902818, 0.0171332974, -0.0250534937, 7.449990e-02, 0.0379011407, 0.0825184509, 0.0144711854, -0.0134697761, -0.0502542853, 0.0118785053, 0.0162278917, 0.0659796596, 0.00401691906, 0.0811922922, 0.0898615271, 0.0628286898, -0.0712538958, 0.0540251248, -0.0231703948, -0.00858731288, 0.0142171923, -0.00266392063, -0.0514051057, 0.0345672034, -0.0360523239, -0.073052153, -0.0120332846, 0.042518273, -0.022511825, -0.0211145319, -0.005568814, 0.00569117069, 0.013458115, -0.0462464765, 0.00252012303, 0.0225150753, -0.0296027791, 0.0143534215, -0.0179844424, 6.422910e-03, 0.0428863801, 0.0197439361, 0.0628110394, 8.107720e-02, 0.0507668518, -2.06906858E-4, 0.0382853076, 0.0789562687, -0.070960246, -0.0216295142, 0.042132061, -0.00921463314, -0.0018824552, 0.0805669948, 0.0379443243, 0.0682178437, 0.0137616694, 0.0235336982, 0.0179813262, 0.0897864475, -0.012007881, -0.0111597301, -0.00581367081, 0.0140105234, 0.0730338171, 0.121077806, -0.0767217502, -0.0611189305, 0.110295616, 0.0804541483, 0.0407544039, 0.07719931, 0.0897329449, -0.00764755858, -0.0944139212, -0.0435133539, 0.0625768378, -0.0140355704, -0.016253544], [0.0470463187, 0.0654324442, 0.0232765879, -0.0347987525, 0.0243083872, 0.0791019499, 0.0269650426, 0.00211689202, 0.0263168253, 0.0660022423, 0.0387743823, 0.110164195, -0.0518952869, 0.0194921028, 7.885500e-02, 0.0839249641, 0.0335816965, 0.0321565308, 0.0234145988, 0.0372270271, -0.0207127966, -0.0380113088, 0.015377989, -0.0388485342, 0.0196456015, 0.0857278854, 0.00321541145, -0.0130430209, -0.0377779305, 0.0364809223, 3.990940e-03, 6.94947492E-4, -0.0577620827, 0.121921279, -0.0267757438, 0.0463520885, 0.0290543921, 0.0602005497, 0.00954507477, -0.0110076452, 0.012139326, 0.0253120605, -0.0206761416, 0.0233463142, -0.00322111556, -0.0425896645, 0.00709390175, 0.0170128271, 0.0105677117, -0.0507263318, 0.0374940261, 0.040376436, 0.00574396644, 0.0248044077, -0.0427521467, -0.0569038093, 0.0769896209, 0.0669332743, 0.0223400053, 0.00612117816, -0.0523816459, -0.0497377142, 0.107995383, -4.213760e-02, 0.0741925612, 0.0249377899, 0.0608094856, -0.0556013882, -0.0408702902, 0.105284162, 0.0585332364, -0.0391642861, -0.0218375511, 0.0833618194, -0.0267442912, -0.0844909548, -0.0439901389, -5.979700e-02, 0.0377115123, -0.0840677395, -0.0389125384, -0.0299845375, -0.0375098065, -0.0229052827, -0.0572628379, 0.0722689703, 0.0154476902, 0.0639090165, -0.0643605887, 0.0185906552, 0.0539505705, 0.0759458393, -0.00523834629, 0.019386448, -0.0597001649, -0.0782430544, 0.0391569696, 0.033297509, 0.0802816525, 0.0611473657, -0.0347654894, -0.00427099364, 0.0763722137, -0.0513589866, -0.0634928122, 0.0682902858, -0.019820217, 0.0129449992, -0.029987555, -0.022330001, -0.0503100865, -0.0157248154, -0.0333735868, -0.0149307456, 0.00353598827, -0.0463533774, -0.078986004, 0.0531092249, 0.0934175253, 0.0291664302, -0.0386039019, 0.112889558, -0.0651894435, 0.0093593765, 0.052969832, -0.0396524258, 0.0320259631, 0.0500356928], [-0.0624896176, 0.0604958385, -0.0159754511, -0.0376683846, -9.307680e-02, 0.019114757, 0.0497299545, 0.0171575956, -0.0828175842, 0.0254431553, -0.0734248757, 0.0152670965, -0.0361044407, 4.303475E-5, -0.0581944659, -0.0543591082, 0.0637137443, -0.0361391716, 0.0410247929, -0.0627960339, -0.0269899257, -0.028080862, 0.0387040786, 7.521920e-02, -0.0140278833, -0.0573217683, -0.0252701193, -0.0169540811, 0.0297747329, -0.0578224175, -0.079604797, -0.0458308458, -0.0207973383, 0.0374249592, -0.019644931, -0.0446354374, 3.148230e-02, -0.0152602848, 0.0144625027, -0.021377299, -0.0293416772, -0.0383943059, 0.00160646311, -0.015857555, 0.0293309446, -0.00758642424, -0.066169396, -0.00490384083, 0.0414593518, 0.0610341541, 0.0475787707, 0.0072205984, 0.086249642, -0.0138964774, -0.00198629871, -0.0875451639, -0.0602566637, -0.0740660653, -0.0028168431, -0.0169432275, -0.0815192237, 0.0458712392, 0.0371845961, -0.0581212379, -0.0562604889, -0.0555351675, -0.0398556069, 7.182920e-02, 0.113557667, -0.00265358575, 6.023590e-02, 0.0196182337, -0.0645633637, -0.0118264034, -5.711670e-04, -0.0406107828, 0.0367100686, 0.0382342562, 0.0390875898, -7.452420e-02, 0.078615725, 0.0713485703, 0.0235815477, -0.0372576043, 0.0554515719, -0.0155710969, -0.0770336539, 0.00261410885, 0.0889821798, 0.0625736192, 0.0363675579, 0.104672171, -0.0133303255, 0.0723636374, -0.0410811938, -0.0217937026, -0.0133940196, -0.0530565158, -0.0683473572, 0.143447146, -0.0232976414, 0.0349424593, -0.0032165146, -0.0116900904, -0.0291544478, -1.23967882E-4, -0.0718480721, -0.045008935, -0.010830747, -0.00915262382, 0.0062327371, -0.0141417533, -0.066961199, -0.075574778, -0.0666494742, 0.00704398425, -0.0278992895, 0.0523840673, 0.00115753117, 0.0245951936, 0.067145966, -0.0430949517, 0.0492472276, -0.0499924272, 0.0597031862, -0.0813403651, -0.0537463166, -0.0273713451], [0.00440827059, 0.106548697, 0.0369911976, 0.00115875213, 0.0453892536, -0.0796706527, 0.00615996215, 0.0234917086, 1.00737088E-4, 0.0184103567, -0.0626524091, -0.0306104273, 0.0108953593, -0.0674039572, -0.0013052763, 0.0805043652, -0.017261494, -0.015911784, -0.0405704044, 0.0176534615, 5.080200e-02, 0.0281004626, 0.00489950506, 0.0320752971, 0.0477900133, 0.0551722869, -0.0689547061, -0.00997173413, -0.0260267667, -0.0303660072, 6.526460e-02, -0.0854338408, 0.0432632416, 0.00486314669, 0.117714949, -0.0428063087, 0.0906574055, -0.0134376036, 0.00732061546, -0.0321740173, -0.0211159959, 0.0375385433, -0.0693368167, 0.00528893759, -0.00312781357, 0.0177377909, -0.00816873368, 0.0506175384, -0.0671145617, -0.0128153199, 0.0138409538, 0.0855684056, 0.0351186655, 6.773960e-02, 0.0176717471, -0.0203206409, -0.0178651158, 0.0148782115, -0.0741763934, 0.0249610282, -0.083580546, -0.0565717742, -0.0636099428, -0.0213249773, -0.0378017277, 0.0224702861, -0.0991479828, -0.0577212758, 0.058405906, 0.028606955, -0.0280969553, -0.0586375706, 0.0427291431, 0.0763062164, -0.0482791215, 0.0128476899, -0.00478946744, -0.0367151499, 0.04816157, -0.0279849302, 0.00553211942, 0.0453382954, -0.078763239, -0.0282286275, 0.062453892, 0.063876085, 0.0124219917, -0.0373699106, 0.0378296599, -0.0903565883, -0.0582229979, -0.0145842517, -0.0183476862, 0.0864495486, -0.0480362289, 0.0463937297, 0.0683779716, 0.0111772036, -0.0204636231, 0.0834578946, 0.0221775882, -0.0207902342, 0.0374244377, -0.0572250821, -0.0337907597, -0.0283730347, -4.221240e-02, 0.017692171, 0.0139894709, 3.06735456E-4, -0.0346444324, 0.0758435354, 6.946820e-02, 0.042974446, 0.0395561866, -0.0443460606, -0.0775393471, 0.0906767845, -0.0239846501, 0.0316274092, 0.0181191489, -0.0573410951, -0.0572043434, -0.054841768, 0.066850394, -0.047100313, 0.0574668199, 0.025572991], [0.0367105417, -0.00548435049, 0.0190998986, 0.0414595306, -0.0637503043, 0.0382872075, -0.00413813628, 0.0709777698, -0.0546456948, 0.105713807, 0.00347125717, 0.0265972968, 0.0384945795, -0.00530342758, 0.00957512483, -0.0166134033, -0.0380888097, 4.306180e-02, 0.0588761903, 0.0578921698, 0.0622709543, -0.0824147164, 0.00944037921, -0.0102726901, -0.0300626978, 0.0274184719, -0.0308143385, 0.00793137494, 0.0805554762, 0.00257547502, -0.00154446182, -0.00312599214, -0.00801853091, 0.0311626755, 0.0274009779, -0.0637337118, -0.0113078877, 0.0732252449, 0.00554682314, 0.0536600314, -0.0203828812, 0.0483703651, 0.0638244599, -0.0212138239, 0.027489746, -0.008481035, -0.0446925722, 0.0525671281, 0.0596149936, -0.0496384501, 0.0642068833, 0.0417592898, 0.0405443609, -0.0743691847, -0.0473242886, 0.058137469, 0.0736908764, -0.0223541651, 0.0246130321, 0.0502539091, -0.0884955599, 0.0202344693, -0.0632209479, -0.0496990979, -0.00829903781, 0.00597772514, 0.0110755935, 0.0316946357, -0.00856255461, -0.0133369034, 0.00614387961, -0.0602872632, -0.0334717892, 0.0460179895, -0.0712135136, -6.649000e-02, -0.00814949907, 0.0511491336, 0.0438527167, -8.9893653E-4, 0.00433038594, 0.0607172139, -0.0233641397, -0.0861272663, 0.0434060656, 0.0258783735, 0.0481028967, 0.0447285101, 0.0827829837, 0.0418619886, -0.0197257083, 0.104709834, 0.0198725034, -0.0332470872, 0.0352903828, 0.0647812411, 0.101785041, 0.079217337, -0.0543582849, -0.0141273569, -0.0371268429, -1.87452053E-4, -0.0215419494, 0.0309780352, 0.0785412937, 0.0319812074, -0.0923126339, 0.0700706616, 0.067192331, 0.0692642182, 0.0481231697, -0.0569913238, -0.00191973499, -0.0150010269, -7.847490e-02, 0.0225575268, -0.0212876759, -0.0185478926, -0.0280101411, 0.00473370543, -0.051339604, 0.0564427599, -0.0119601944, -0.0301282667, -0.0624755434, 0.0103795417, 0.0372177251, 0.0744516328], [-0.0251976028, -0.0379242301, -0.0155485356, 0.0541833639, 0.0530617759, -0.00843473058, 0.0339820199, 1.636660e-02, -0.0600582846, -0.00473263767, -0.092488274, -0.0236850176, -0.063182421, 0.0548582412, -0.0126657039, 0.0153149711, -0.0426368415, 0.0523830689, -0.0189846046, 0.0432988182, -0.042867057, -0.0288646594, -0.0478237495, -0.0212439448, 0.00517655071, -0.0379529819, -0.0407784432, -0.0253133792, 0.0233497806, 0.0115629174, -0.0628200099, -0.0180569459, -0.0316093937, -0.0268838685, 0.0451104417, 0.0403079055, -0.0248776358, 0.0630376711, 9.54860414E-4, -4.039060e-02, -0.0411680527, -0.0811359062, 0.0651290491, -0.0269885398, -0.00621180609, -0.04186932, -0.0047984058, 0.0368928909, -0.0130326897, -0.0167830698, -0.0496356376, 0.0748015344, 0.072207436, 3.064750e-02, 0.0528217107, -0.0820312649, 4.619810e-02, -0.0387053415, -0.00136182085, 0.0499439351, -0.103082158, 6.699220e-02, -0.0989951267, 0.00933391694, 0.00286850892, 0.0374489836, -7.223960e-02, -0.075572513, 0.037115287, -7.265980e-02, -0.0272191484, -0.0603203289, -0.024890108, -0.0495209396, 0.0783662572, -0.075566642, 0.0387134291, 0.057144139, -0.00983240827, -0.0593520701, 0.0828456878, 0.0860784873, -0.0839383676, -0.0199789088, 0.0433994979, 0.0455587432, -0.0318621062, -0.0246607587, -0.00783979055, -0.0429803878, -0.0947981476, 0.0626047254, -0.0307183731, 0.0667362735, -0.0293517876, -0.0260045025, 0.0203701984, -0.0500294566, -0.0253850427, -0.0562526695, 0.0332010537, 0.0359487422, -0.0388601944, -0.0215952788, -0.0424486883, -0.00612003496, 0.0329915397, 0.0288923364, -0.0665461794, 0.0554692373, -0.0145710483, -3.660570e-02, -0.00201203069, -0.0446601883, -0.0961593687, 0.0888199731, -0.0934049636, -0.00161025079, 0.0215309747, -0.0601891726, -0.0779769048, -0.00426475424, -0.0422569662, 0.0702905133, 0.0323630944, 0.045484174, -7.903580e-02, -0.0435600504], [0.0187165197, -0.00620719464, -0.036187727, -0.0521372966, 0.00638382835, -0.0185967982, 0.0192046836, -0.00588090857, 0.00815252867, -0.0393387936, 0.00168732519, 0.0665407628, 9.658720e-02, -0.00644403929, 0.0402448215, -0.00299763121, -0.00813374575, -0.053070493, 0.055443842, 0.0584777668, -0.0659159198, 0.0354863815, 0.0812212676, -0.0732367188, -0.0283721741, 0.0318091251, 0.00814176909, 0.0220851563, 0.0042052418, -0.0537721589, 0.048566144, -0.0571884327, -0.0289403461, -0.0613321736, 0.055093132, 0.00856169499, -0.0122352792, 0.00576209184, -0.0160535872, 0.0534600876, -0.0280865654, 0.0699423179, 0.00482988497, 0.0839692875, -0.033177644, -0.0122263944, -0.0911457315, -0.0317660607, -4.45457874E-4, 0.0913669764, -0.0763793662, 0.0238927472, 0.0226090588, -0.0363208316, 0.00696410611, 0.0387622043, -0.0808582753, 0.0192303378, -0.0217561498, -0.0199099574, 0.048995629, -0.0760114864, -0.0461488925, 0.0514240116, 0.0290974379, 0.0177045316, 0.0299813505, -0.00232463656, 0.053816393, -0.0649631768, 0.0707266554, -0.020331094, -0.0580756627, 0.0466188714, 0.0466194898, -0.0862795338, -0.0587610565, 0.0125462739, 0.0611616299, 0.00703201443, 0.00143800606, 0.0340659618, 0.00535486359, 0.0231825151, 0.111575507, -0.0323973969, -0.0815744325, 0.0613483302, 0.102455981, -0.0025150103, -0.0935644581, -0.0483579561, -0.0350316875, -0.0401273817, -0.00763229607, 0.0448804423, -0.0341206789, -0.0256099738, -0.0974525436, 0.0847702547, -0.081095174, 0.0731409788, -0.0499756038, 0.0504606031, 0.0618269555, -0.0532099903, 0.0420231484, 0.0687203631, 0.0416676737, 0.0321146883, -0.0737767145, 0.0310340207, -0.0032359194, 7.915370e-02, -0.0900401398, -0.00667839404, -0.0179378297, -0.00481763948, 0.0478118584, -0.0531554744, 0.0211842321, -0.0511446781, -6.88945816E-4, -0.050096646, 0.0478390381, -1.31292269E-4, -0.0218795575, -4.838220e-02], [-0.0607570857, 0.0655346662, -0.0780299231, -0.00248013181, -0.0351591222, 3.260190e-02, -0.0650804117, 0.0522388928, -0.0487283431, -0.0480463654, 0.0259401929, 0.0284954011, 0.0470550656, 0.0618128441, -0.0614842065, 7.030710e-02, 0.0372244529, -0.0454160087, -0.0585947745, 0.0637313649, -0.0825771391, -0.00498775207, 0.00564998854, 0.0048255818, -0.0434447899, -0.050204616, -0.0761457458, -0.04781501, -0.0731652752, 0.0611749478, -0.0743118226, 0.0231476687, -0.0469878428, -0.0637283549, 0.0487548225, 0.0905501842, -0.0377682485, -0.0510639623, -0.0676582903, -0.0454140157, -0.0219280403, 0.0230271388, 0.00167332951, 0.0396181159, -0.0045137438, -0.0305842329, 0.0421406887, -0.048814211, -0.0575128607, 0.0670958459, -0.0163835082, 0.0354458354, 0.084113352, 0.0253071468, -0.00192639965, 0.0307616498, 0.0730773434, -0.0655615777, 0.00141824433, 0.0383482687, -7.026050e-02, 0.0310537834, 0.0545614213, -0.0204553083, 0.0322505087, 0.0190437417, 0.0734886155, -0.0113592828, 0.0547158532, -0.0847286358, -0.0333374068, -0.0621028394, 0.073871091, 0.00683357147, 0.0200189762, -0.0322548784, -0.0420173891, 0.0295098703, 0.0331400447, 0.0825676471, -0.00725720031, 0.0302627329, -0.0459316522, 0.0178419594, -0.0354444198, 0.0649233758, 0.043215666, 0.0440279134, -0.0438516326, -0.0792463571, -0.00654895789, 0.0324870721, -0.0385786742, 0.0217577517, -0.0672746152, -0.0635843948, -0.0283722039, -0.0592341758, -0.0255934484, -0.0224792045, 0.00307893939, -0.0107550593, -7.370440e-02, 0.00956704747, -0.0142630395, 0.0464797318, 0.0566231683, 0.0304641649, -0.0521460511, 0.0200380031, 0.0255722515, -0.019209411, 0.0684928074, -0.00927252322, -0.0521622412, 0.0695227682, 0.0449354202, 0.021005271, 0.075652048, -0.0670907423, 0.0692333132, -0.0724048391, 0.00359582715, 0.065369539, 0.0391547643, 0.0437421165, -0.041492749, 0.0461190715], [0.0442931391, -0.0174564049, 0.0211521219, 0.0396315753, 0.0202855989, -0.053892795, 0.0622239783, 0.084322445, -0.0242947359, 0.00645091245, 0.0041701314, -0.070623219, -0.0050559924, 0.0133634442, 0.0657468811, -0.00772768678, -0.0254564174, 7.146310e-02, 0.0809616744, -0.0547504351, 0.0498776622, 0.0641217157, 0.0438165404, -0.0443374589, -6.461790e-02, 0.0217354372, -0.0714022368, 0.0495582633, -0.0265690312, 0.00981133711, 0.0293901786, 0.039565593, -0.0364527293, 0.0467870645, -0.0384976305, 0.0489403941, 0.0604950711, -0.0611317083, -0.0321215875, 0.0884578526, -0.0087293256, 0.0723100826, -0.0408180952, 0.0265322328, -0.0717232078, 7.597800e-03, -0.0479129702, -0.0306508504, 0.0252164248, 9.393190e-02, -0.0209718291, 0.0741249174, -0.0765288249, -0.0205652062, 0.0291039422, 0.053004358, 0.0459539741, 0.00954594184, -0.0344909802, -0.0732256323, -0.0390065797, 0.0602918118, -0.0542038381, -0.0803658291, -0.0411118716, 0.0124039883, 0.0526019149, 0.0388586484, 0.0554993376, -0.0277898703, -0.0801644101, -0.0247058738, -0.00959322322, 0.0700044855, 0.0264459252, -0.038567096, 0.0508520417, 0.00203955569, 0.0609627664, 0.0192304403, 0.0331734978, 1.44213482E-4, -0.0226351116, -0.0632928386, -0.0430663228, -0.0714357793, 0.043568939, -0.0391274504, -0.0575667322, 5.695630e-02, 0.0510658473, 0.0606396496, 0.033351589, -0.0652425662, 0.00886709336, -0.0370460264, 0.0107066222, -0.0564092509, -0.00125261105, 0.0428610891, -0.073880881, -4.599820e-03, 0.00586493406, -0.0134089449, 2.307320e-02, 0.0253723264, -0.0308542419, -0.0210984573, -0.0138372229, 0.0329734348, 0.0343179405, 0.0467201099, -0.05046295, 0.0820963084, 0.0202021245, 0.0792182385, -0.0569265075, -0.0111487266, -0.0610894188, 0.0144319944, -0.0196206141, -0.0868432819, 0.0526385494, 0.0158311855, -0.0179606695, 0.0651576146, 0.0367492214, -0.0170974787], [9.72441048E-4, 0.00720486417, 0.033908803, 0.024774598, -0.0691638663, 0.0634782761, 0.0427147821, 0.0877003521, 0.0536193661, -0.0822009816, -7.637570e-02, 0.0247898493, 0.0716975331, 0.0826689228, -3.617800e-02, -0.0099070724, -0.0326802768, 0.0336349234, 0.00316536287, 0.061714109, -0.06438227, -0.0346568078, -0.058571212, -0.0777626261, 0.029565895, 0.0329633616, -0.0296153054, 0.0346898809, 0.0602262616, -0.031003641, -0.076040633, -0.0265486613, 0.0421682894, -0.0155459028, 0.0367280617, -0.0706198439, -0.0437401943, 0.0613311082, -0.0858519971, -0.0294276848, -0.0676243305, -0.0718698799, -0.0624740943, -0.0430959165, 0.0566829704, 0.0564902648, 0.0285333041, -0.0260278862, -0.050352592, -5.068240e-02, -0.0773673653, 8.42323061E-4, 0.0132328849, 6.406040e-02, -0.0821094512, -0.0694056675, -0.0693297461, 0.0524382293, -0.012806383, 0.0336082838, -0.0500207767, -0.0130828973, -0.0570575558, -0.0366322286, -0.0154048493, 0.00286032842, 0.0884059668, -0.0355941765, 0.0101131834, -0.0804685503, -2.437210e-02, 0.0596508831, 0.0343311615, -0.0281311106, 0.0293102674, 0.0794574618, -0.065128833, -0.0576635823, 0.0149142416, -0.00931220222, -0.00157523679, 7.822110e-02, -0.060141027, 0.0371283926, 0.019052688, 0.0688397661, -4.84425196E-4, -0.0361145623, -0.062730588, 0.00771879824, 0.00196706131, -0.0140091563, 0.016899189, -0.057952188, 0.0552379303, 0.0526589677, -0.0151556097, -0.0493065752, -0.0200854931, 0.0956090688, -0.0314967334, -0.0634254366, 0.0153616741, -0.0710320473, 0.0509983897, -0.00757809961, 0.0297949258, 0.00783080421, -0.0620834306, -2.103570e-02, -0.059559606, -0.0216829553, -0.0169819314, 0.0184683613, 0.0077005038, -0.0723574832, 0.0653148517, 0.0808088406, -0.0156258754, 0.0736760497, -0.0366968736, 6.99946948E-4, 0.00189675298, -0.0500551909, -0.00395966275, -0.0513174273, -0.0238365456, -0.0696395561], [-0.0430757366, -0.00548715284, 0.0656595379, -5.580740e-02, -0.0474795252, 0.0637914538, -0.0281121526, 0.0571485907, 0.0715104863, 5.995190e-02, -0.0215387736, -0.0607947074, 6.259250e-03, -0.0425067469, -0.00612008758, 5.152520e-02, -0.0753865391, 0.0528204292, 0.0805150419, 0.0807127878, 3.02855828E-4, 0.00854630861, 0.00424438156, -0.00143174862, 0.0131922495, 0.049794089, 0.0272933133, -0.0592616722, -0.00718293944, 0.0662022829, -0.0440740511, -0.0822803601, -0.00427086931, 0.00465971231, 0.0665400922, 0.0172786247, 0.0685011148, 0.048621539, 0.0285630412, 0.0643475353, -0.0735655576, 0.0333871394, 0.0123380041, -0.0137570864, 0.032301791, -0.043596793, -0.0562686175, -0.0177931041, -0.076097928, -0.026060421, -0.0206697285, 0.0754324719, -0.0223546419, -0.0400113389, -0.0753879845, -0.0210206937, -6.83475926E-4, -0.00807083305, -0.00938477647, -0.0741554648, 0.0732757151, 0.0558916815, -0.0519443788, 0.0580199771, -4.286890e-02, 0.0765724853, 0.0187476836, 0.0806994512, -0.0148399798, -0.0722033903, -0.075546205, 0.0732294768, -0.0767359883, -0.0365219973, 0.0698400065, 0.0295354128, -0.0701846853, 0.00994070991, -0.0245556198, -0.00565164536, -0.0471901335, -0.0312779807, 0.0303678662, 0.0147262951, -0.0789729357, 1.31405861E-4, 0.0593591221, -0.00945659075, 0.0678831413, -0.0543248765, -0.0311356802, 0.0152631905, -0.0535551831, -0.0265639797, 0.0163719244, 0.0637645945, -0.0620137155, 0.0194668509, -0.0202457737, 0.0169472881, 0.0661689416, -0.015586854, 0.0448565744, -0.0357458703, -0.0682264343, -0.0364301056, -0.013923849, -0.0046426882, -0.00998594611, 0.0413099863, 0.02923318, 0.0232309327, 0.0253329799, -0.0339414589, 0.0633800179, 0.0261582434, 0.0415558442, 0.0635170564, -0.0421746783, -7.58396753E-4, 0.0313498937, -0.0522575714, 0.00227242964, 0.0425649658, -0.0367876813, -0.0749733299, 0.0633524209, -0.0270862319], [5.969900e-03, -0.0232340842, 0.0277486127, 0.0468778946, -0.0528014936, 0.0754313245, -0.0188310184, -0.0404493809, 0.0418883525, 0.0316149034, -0.033195585, -0.00810521748, 0.00985921081, -3.946470e-02, -0.072269313, -0.0803986117, 0.0734875798, -0.0117799472, -0.0501268804, -0.01783829, -0.0567343719, -0.0353173539, 0.0279594027, -0.0329875946, 0.0670076683, 0.0105000902, 0.0333814286, -0.0267343726, -0.0675804615, -0.0593443662, -0.0504127592, -0.0381746516, -0.0181792397, 0.0021889871, 0.0282405913, 0.0418043286, 0.0118579445, -0.0563487746, 0.0214384608, 0.0298848189, -0.00624014763, 0.021171432, 0.0653632283, 0.0761172846, -0.042039711, -0.0141235357, -0.0784199908, 0.032966163, -0.00531751383, -0.0709296912, 0.0788176655, -0.0045465366, -0.036062561, -0.0399530753, 0.00957917608, -0.0567368679, -0.0632573441, 0.0720817596, -7.167220e-03, 0.0788182914, 0.0306645986, -0.0148119573, 0.0753729641, -0.0772308484, -3.392500e-02, -0.0590331554, -0.0198228434, -0.0557486713, 0.0516015738, 0.0297562722, 0.0122227622, 0.0702185631, 0.05303891, 6.581730e-02, 0.0721621662, -0.0721100196, -0.0386709869, -0.0803854689, -7.475690e-02, -4.04524588E-4, 0.061090041, -0.0264116507, -0.0097560212, -0.03262363, 0.0555231683, -0.0216212235, -0.0567183308, -0.0580298565, -1.30747736E-4, -0.0181271415, -0.0725246072, -0.0695384368, -0.0350286625, 0.0275381412, 0.0389871299, 0.0344709978, -0.0266575329, 0.0540406629, 0.0196851958, -0.0423345864, -0.0342669748, -0.0378547907, -0.062695682, 0.00200639316, 0.0382269211, -0.00302335713, 0.0312574208, -0.0535657182, -0.0435434915, -0.0787174478, 0.0281637814, 0.0794651284, -0.0746648163, -0.024757674, 0.0567449927, -0.0108529665, 0.0695195347, -0.071148023, 0.0795024335, -0.0799922272, -0.00194893987, 0.077793479, 0.0166461878, -0.0252745021, 0.0719712898, 0.074494794, 0.0275852494, -0.00913358387], [0.0784212574, -7.727080e-02, -0.0428833365, 0.0230012238, -0.0710727572, 0.00310269743, -0.0651246458, 0.0219726563, 0.0277005062, 0.0351923183, -0.0788391157, -0.069597207, 0.0332742557, -0.0228670947, -0.0670001283, 0.0582219735, -7.067680e-02, -0.0417136587, -0.0371112414, 0.076309137, 0.0215434432, 0.0132158324, 0.00728496909, -0.0142198578, 0.0408671871, 0.0514394715, -0.0178380758, 0.0112659484, 0.0772789344, -0.0472713336, -0.00905206799, 0.0279182941, -0.0411876775, 0.0683970675, 0.0780753567, -0.0733927339, -0.00147375464, 0.0378601551, 0.00442257524, 0.03421624, 0.0791596248, -0.079313159, 0.0806020572, -0.0462312028, -0.0538160317, -0.0578971654, 0.0799127444, -0.0195703804, 0.0796452686, -0.0778604075, -0.0207517371, 0.0164735988, -0.054796271, 0.0660095438, -0.0254110098, 0.0360539407, 0.0109514892, 0.050561823, 0.0047942549, 0.0646201149, 0.0314354524, -0.0604848042, -0.0694960877, 0.0428397283, 0.0712300763, -0.0748068467, -0.0509075709, -0.0798462331, -0.0731318444, 0.00505955517, 0.00802160054, -0.0685628578, 0.0103506297, -0.0308388248, 0.0474898592, -0.0456278287, -0.0427683331, -0.0562201515, -0.0410541669, -0.0757034868, -0.0803172588, 0.0318179056, -0.00956591963, -0.0485116132, 0.00688628852, 0.0595201477, 0.0151670277, 0.0696637109, -0.0437339917, -0.0325676873, -0.0205855034, -0.0342006534, 0.05016049, -0.0477836616, 0.060987331, 0.0363529101, -0.00807917118, 0.0319203362, -0.0395845324, 0.0597028658, -0.0404633246, 0.0554972515, 0.0162086636, -0.0254433826, -0.017792277, 0.054661341, 0.0676459894, 0.00231844932, -0.0513697378, 0.0585702434, -0.0669233948, 0.0498242155, 0.0427966267, -0.0726648569, -0.0520007648, -0.00162261724, 0.0804721638, 0.0616216287, 0.0740883574, 0.0107665434, -0.0257173292, 0.0412478223, -0.0493654385, -0.0785173103, -0.0744008422, -0.0538967103, 0.00382076204, -6.761770e-02], [-0.0227206461, -0.00917745195, 0.0810256749, -0.0456483513, -0.052524019, 0.0660247281, -0.0258411411, 0.048265703, -0.00716320844, 0.0800237581, -0.0649883747, 0.0482358932, -0.0245642625, -0.0798883512, -0.0118665006, 0.0280346107, -0.068454586, 0.0280724932, 0.0424964391, 0.0432684347, 0.0646034554, -0.0371691026, -0.0766745955, 0.0666344762, 0.0422441252, 0.0457755588, -2.44148512E-4, -0.0330329686, -8.102040e-02, 0.0474595539, 0.0596040487, 0.0175719783, -0.0552662797, 0.0433026291, -0.0362078324, -0.0521051027, -0.0479500443, 0.0484506115, 0.00171243551, -0.0151881967, 0.0726639777, 0.0048669274, 0.0259508248, 0.0780665054, -0.0643799454, 0.0232713185, 0.0652034059, -0.0411298797, 0.051343713, 0.0601664409, -0.0565825179, 0.066305764, 0.0556043349, 0.0692666098, -0.0632193238, -0.0256653074, -5.162820e-02, -0.0496972613, -0.0756970047, 0.0731134787, -0.0738447681, -0.0370992608, 0.0485170595, 0.0494542755, -0.058673881, 0.0657540783, -0.0437339321, -0.059900973, -0.0239988901, -0.00563189248, 0.0510141626, -0.0793011859, 0.0331270844, 0.0219548903, -0.0697326884, -0.0637830123, 0.0172299296, 0.0340828709, -0.00776148075, 0.0376438051, -0.0220793243, 2.698530e-02, 0.0295738839, 0.00177415868, 0.00226940797, 0.0484537221, -0.063557595, -0.0139290607, 0.0164386928, -0.0206044912, -0.00182132702, 0.0169951972, 0.0589780025, -0.0680024549, 0.0373843834, 0.0804782882, 5.883720e-03, 0.0124345031, -0.0389012434, 0.0175301842, -0.0175411031, 0.0247630123, 0.035708271, -0.00841538887, -0.0127228992, 0.00206524204, 0.0699072853, 0.0333857499, -0.0728400499, -0.022838043, 0.0589449331, -0.0628036633, 4.499820e-02, -0.0738358497, 0.0569766462, 0.0634889454, -0.0630258619, 0.0584999546, -0.0569506101, -0.00157850073, 0.00227636751, 0.0617564805, -0.0770394579, -0.0591391809, 0.00168528862, 0.04136502, 0.0385031663, -0.0197825022], [0.0132296141, 3.004390e-02, -0.0592786931, -0.0589706302, -0.0200306438, 0.0147642186, -0.00861087813, -0.0447955504, 0.0456439145, 0.0106084552, -0.0320315212, 0.048849117, 0.0608647913, 0.0645901933, -0.0255558286, 0.0482694507, 0.0341283083, -0.0285497699, 0.0792356655, -0.0107404906, -0.0453393944, -0.0195165314, 0.0728122889, 0.00126544852, -0.0430496223, 0.018697327, -0.0633179247, 0.0503977686, -0.00709778769, 0.0231279954, -0.0313411169, 0.025704693, 0.0659647584, 0.0522206947, -0.00731827086, 0.013548376, 0.0313882902, -0.0750546679, 6.657620e-02, -0.0612126738, 0.00814847834, 0.039666269, -0.0171161275, 0.00347000198, -0.0327858292, -0.074151732, -8.47412666E-5, -0.0215763394, 0.0797727853, 7.461580e-02, -0.0308716819, -0.0511186868, 0.0108275814, -0.00104731007, 0.0286694374, 0.0793126822, 0.0170221645, -0.0330026969, 0.0225699041, 0.0826187357, 0.00508383103, -0.0578989275, -0.0506126136, -0.0610882081, 0.00616817316, -0.022050079, 1.992980e-02, 0.00347420573, 0.0524156876, 0.016332671, 0.0577613041, -0.0384015627, 0.0345750637, -0.0604842417, -0.0758228227, 7.563860e-02, 0.0670135915, 0.00606189109, 0.0166204441, 0.0211719535, -0.0615183413, 0.0611951649, 0.0601086915, 0.0592128783, 0.0148250153, 0.048584722, -0.0797415301, -0.0634006858, 0.0386161841, 0.0799687356, -0.0571682714, 0.031577345, -0.00226364587, 0.00213848194, -0.0133090792, -0.0381501801, 0.0356593393, -0.0154596725, -0.0741393194, -0.026554022, -0.00533207925, 0.0581177883, 0.0354035832, -0.0276145358, 0.0227771644, 0.0558064021, -0.0520035028, -0.0659095645, -0.046145685, 0.0187852215, -0.0163966231, 0.0668499842, -0.0477509461, -0.00140016037, 0.0710725114, -0.0528625846, -0.00537963258, -0.0668984279, -0.0325237513, -0.0375572778, -0.00846167188, -0.0518511087, -0.079132013, 0.00826239585, -0.0185532812, -0.0137781063, 0.033364784, 0.00738865882], [0.0211174227, -0.0207755771, 0.0407902449, 0.090547949, -0.051372759, 0.0605877303, 0.00885949284, -0.0689064637, 0.0321054384, -0.00240848842, -0.0676335469, -0.0106410431, -0.016307503, -0.0436726622, 0.046888236, 0.0412942432, -0.0485907681, 0.0229849145, 0.0144641288, -0.0139550157, 0.0354441144, 0.0470223427, 0.00569224637, -6.919480e-03, 0.0151481349, -0.048463881, 0.0834043696, 0.0568725914, 0.0272487309, -0.00161038281, 0.067070663, 0.0766589865, 7.703790e-02, 0.0448659547, 0.0764372274, -0.0074493275, 2.243760e-02, -6.171330e-02, -0.0429862887, 8.526860e-02, -0.0146747436, 0.0725589916, 0.0400324948, -0.0216810443, -0.00481311604, -1.780130e-02, -8.438300e-02, 0.0476032756, 4.444260e-02, 0.0140755055, 0.0735256821, 0.0359039195, -0.00702301832, -0.080053173, -0.050530456, -0.0143431006, 0.0298705883, 0.0237244107, 0.00519800792, 0.0165569875, -0.0212387089, -0.0138210766, 0.0544891134, 0.0524993502, -0.0616520829, 0.0527292192, -0.0740022287, 0.0105285672, 0.0612336695, 0.0674806759, -0.0230830852, -0.0421595722, -0.0621924586, 0.00730795739, -0.0313774161, 0.0655520111, 0.0434107035, -0.0110889301, 0.0621700473, -0.00977739319, -0.0739099458, -0.0376228131, -0.0726286322, -0.0202653464, 0.029129114, -0.00798109173, -0.0248303059, -0.0493446141, 0.0194137637, 0.0624321774, 0.0493278019, 0.0147162713, -0.0303451978, 0.049570661, -0.00767453061, -0.0222945642, 0.0308811627, -6.910180e-02, 0.0364370868, -0.0674020275, 0.0449495651, -0.0575626381, 0.0381388329, 0.0616566576, -0.00481830724, -0.00745901885, 0.046474658, 0.0303922258, 0.0169771668, -0.057787966, -0.0672325193, -0.0394707806, -0.0657493845, 0.0807535946, 0.0143217854, -1.141800e-02, 0.0393248089, 0.0325165354, -0.050472334, 0.0390990041, -0.0143037848, 0.0153919347, -0.00993559416, -0.0578031875, -2.790620e-02, 0.0447147675, -8.14856728E-4, 0.0276117958], [-0.00936528295, -0.0252121389, -0.0658189878, 0.0124851651, 0.0702988729, -0.0260824542, 0.0607467182, -0.0254082028, -0.0747266188, 0.0235792901, -0.0793134048, -0.0161217507, -0.0150591386, 0.036050614, -0.036123421, 0.0244911015, 0.0282181129, -0.065885976, 0.00832102727, 0.0172739513, 0.0676582157, 0.0280857682, 0.0742845833, -0.0236939974, -0.0207479857, -0.0717501491, 0.0707387179, -0.027404204, -0.0285198558, 0.0470722355, 0.0764060467, 0.0574666411, -0.0233891308, 0.00260295742, 0.0513297431, -0.0208010804, 0.0631004125, 0.0850902199, -0.0115816044, 0.0884814262, 0.0186093394, 0.0309822112, -0.0366751067, -0.0339712873, 0.0273803659, 0.0135477139, 0.0253678598, 0.0292979646, -0.0191961359, 0.0842774286, 0.057806395, -0.0155814262, -0.0154987397, 0.0169943478, 0.0323250294, 0.0244548246, 8.8673475E-4, 0.0212597121, 0.0670878142, 0.00464268588, -0.07270246, 0.0474232212, 0.055602584, -0.0827437862, -0.0283198897, -0.0324522369, -0.0735908672, 0.0406155735, 0.0299341027, 0.0454467572, 4.625880e-02, 0.0710131526, 0.0400005914, 0.0573071688, 0.0217826087, 0.0656087548, -0.00659441343, -0.0659463703, 0.0500400774, 0.0213302299, 0.00398566574, -0.0550922379, 0.0133890975, -0.0075988262, -0.0195311029, -0.00869254395, 0.0474455543, 0.0314008482, -0.0196484756, 0.0205077212, -0.0567950159, 0.0879157335, 0.0655540675, 0.0714035481, 0.0289254878, -0.0610640123, 0.0420206636, 0.0480339639, 0.064007245, 0.0509605184, -0.101279519, -0.00751965959, -0.0616993941, 0.0568489842, -0.0648681596, 0.0651038215, 0.0563947856, 0.0684223697, -7.14488735E-4, 0.00851122662, 0.0869122147, 0.0461991467, -0.0784939528, 0.0800832361, -0.0322133116, -0.0612295792, -0.043185275, -0.0253661573, -0.0562775619, 0.0809921473, 0.0191831384, -6.623420e-02, 0.0199412983, 0.0375047177, 0.022393398, 0.0302478075, -0.0144913252, -0.0256104134], [-0.0830949768, 7.26307801E-4, -2.17791909E-4, -0.0460046194, -0.0443053767, 0.0618680604, 0.0728856102, 0.0228808131, -0.0135309193, -0.0223161131, 0.0478573665, 0.0356397666, -0.0478428379, -0.0076996535, -0.0121416943, -0.0647090748, 0.0453021266, -0.0251937322, 0.049762547, -0.0576121844, 9.495110e-03, 0.0413232557, 0.0290059242, 0.0752709061, 4.507030e-02, 0.0848000049, -0.0713949054, 0.0460228212, -0.0242550243, 0.0704448223, 0.0330775119, 0.0494826064, -0.0264529064, -0.0882706195, -0.00449959096, 0.0796193256, 0.0642622262, 0.01204402, -0.0309379194, 0.0585663579, -0.0768146366, 0.0558671691, -0.0583289713, -0.0144953039, -0.0962513163, 0.0326583609, -2.0706642E-4, -0.0109091867, -1.394520e-03, -0.033916004, -0.0644891262, -0.0335746966, -0.0747244135, 5.744900e-03, 0.07011123, -0.00150536222, -7.125280e-02, 0.00656189816, 0.0729990453, -0.0705212429, -0.0592850707, -0.0358210243, -0.0358665586, -0.0607493259, -0.0647631585, -0.0392802469, 0.0197743196, -0.00937470887, 0.00993121881, 0.0516248569, -0.0998603701, -0.0114922319, 0.0576787926, 0.0733126402, 0.0237803571, -0.0327109322, 0.0587544329, -0.033831127, 0.0341233276, 0.0257419087, -0.043949414, -0.0545619801, -0.0807492882, -0.0423726588, 0.0602812432, 0.0149269337, 0.0289515238, -0.0662553906, -0.0549351871, -0.0482691936, 0.0863979906, 0.0846751406, -0.00231769937, -0.0499996692, 0.0135753304, -0.0331123918, 0.0256715771, -0.0436318181, 0.0150995478, -0.0187109094, -0.0697639436, -0.0101702353, -0.00674339058, -0.039576333, 0.0887013077, 0.0361500941, -0.027963629, 0.0643947199, 0.0541476794, -0.00822035595, -0.0618599392, -0.0340259261, -0.0508539528, -0.0231849216, 0.00702937878, -0.0123604471, -0.0719833821, 0.0353907309, 0.0226493143, 0.034350805, 0.0740158707, -0.0845540613, -3.533050e-02, 0.0722010583, 0.0567040034, 0.0241897535, -0.0319496393, -0.0238272175], [-0.065564625, -0.0220131241, 0.05354793, -0.00544983149, 0.0407511592, -0.0588311069, -0.0583569519, 0.0773393065, 0.0635329634, -0.0441486612, 0.0253557265, -0.0020226019, 0.0979119241, -0.0319860689, 0.0516182519, 0.0402232222, -0.0481610745, -0.0413749106, -0.0480338037, 0.0524815656, 0.0745823458, -0.0999381393, 0.078673236, 0.0519150421, 0.0451725684, 0.059353631, -0.0650253221, 0.0369794667, -0.0045238249, -0.0249189921, 0.0300471336, 0.0180606395, -0.0647740737, -0.0666749552, -0.0645902454, 0.0165360644, -0.0185827184, -0.0902643501, 0.020023549, 0.1105524, 0.0481734835, -0.0316557288, -0.0349206515, 0.0881282389, 0.0217840169, -0.0574350469, -0.0274842922, 0.0374493971, -0.0536085963, -0.002251219, -0.0438247062, 0.0210324638, -0.0603445955, 0.0370387174, -0.0598931275, 0.0602762289, -0.0672684833, 0.0170936882, -0.0387738943, -0.0375202447, 0.0549044944, -0.0246913768, 0.0467482954, -0.0736761466, -0.0607564077, 0.043291904, -0.019236587, 0.0245557819, -0.0564358048, -0.0112244422, 0.0531275906, 0.0789481326, 0.00719801383, -0.0113488371, -0.0127056642, -0.0184132792, -0.0824351162, -0.0180766247, -0.0393455215, 0.0341182202, 0.0180911645, 0.0621292702, 0.0507481545, 0.0189680606, 0.00888613332, -9.906860e-03, 0.0552137755, -0.00209588977, 0.00166163919, 0.0511169657, 0.0957026183, 0.00252302294, -0.0580380186, -0.0645729452, 0.0109117785, 0.00183844299, -0.0412131622, -0.00389809022, -0.0858138501, 0.0755094215, -0.0921298861, -0.0564052612, 0.0601265319, -0.0764837489, 0.062577568, 0.0556623414, 0.0139302369, -0.0335723571, -0.0685765818, -0.0153344665, -0.0259231366, -0.0784239992, -0.0597246364, 0.026970176, -0.0692682117, -0.0332342274, -0.100006655, 0.0630155503, -0.0189282801, -0.0825121402, -0.0661237537, 0.0798298493, -0.112642691, 0.0413571633, -0.0411925614, 0.0491934828, -9.267900e-02, -0.0402050391], [0.0236826669, -0.0179240722, -0.0528377667, -0.0932305902, 0.0110949399, -0.0487354174, 0.0334971137, 0.0370270312, 0.0492748171, -0.0945884436, -0.0527668633, 0.0892280712, 0.0314359553, 0.0162983537, 0.0769615471, -0.00824752543, 0.00849979743, -0.0102932919, -0.031956397, 0.0884996802, 0.0422458239, -9.429240e-02, 0.0813574492, -0.0816175639, -0.022172913, -0.0541090183, -0.0603318028, -0.0588646792, 0.0608984455, 0.0341947041, -0.00309155369, -0.0220829807, -0.0611459613, -0.0831346809, 0.0281847455, 0.0208381414, -0.0423232689, -0.0315474123, 0.055341281, 0.0400410444, 0.0124594262, -0.0556529611, 0.04862855, 0.105656095, 0.0365673415, 0.0563904978, 6.400230e-02, 0.0848929882, 0.0257639717, -0.0366092548, -0.0117301065, -0.0429001674, -0.0648816898, 0.0130679226, -0.099780716, 0.0367016792, -0.0502422228, -0.0641325191, -0.0566525906, 0.0300669055, -0.0327834375, 0.0544328578, 0.0665429756, -0.019980913, -0.0308257211, -0.0181928016, -0.045592092, -0.0669283122, 0.0180779416, -0.00695893774, 0.0479539707, -0.00659917248, 0.0516813286, -0.041875992, 0.0107519422, 0.0571520217, 0.0242127292, -0.00857064686, -0.0371885896, -0.0419049561, 0.0737173706, 0.076051347, 0.0434007905, -0.095879428, -0.0434746258, 0.0661128536, -0.0151174041, 0.0375090875, 0.0758227482, 0.0431401394, 0.0222732481, -0.0612084493, -0.0366414189, -0.00901214685, -0.0234546755, 0.00420179497, 0.0334288292, -0.0681639835, 0.0599996969, -0.0253561176, -0.033545021, 0.0656344667, 0.0489117652, -0.0195176583, 0.0127569232, -0.0652166232, 0.0150633939, -0.0740080252, 0.00137934403, 0.0036386759, 0.0720437393, 0.0333776213, 0.042064786, 0.0336652659, 0.00425870204, 0.018364938, -0.0697920322, 0.0192620531, -0.017270539, -0.0218358338, 0.0347883664, 0.0219492633, 0.0229020119, -0.00676365196, -0.0811961814, -0.0315767229, 0.0165004581, -0.0238972753], [0.0559150837, -0.0457232669, -0.0816533118, -0.0478080921, 0.0143813845, -0.0406366661, 0.0646026507, -0.0915099605, -0.0463549718, -0.105086721, 0.0475153774, -0.0178343579, -0.0258630943, 0.00562903564, 0.0169161987, -0.0359066911, 0.0132934833, -7.071550e-02, -0.00464896671, 0.146161556, 0.00491592661, -0.109531648, -0.00272948178, -0.0371104181, 0.00186078809, -5.5079849E-4, -0.0349558406, 0.00977003574, -0.0364487022, -0.00301981205, 0.0737730935, 0.0104324836, 0.0136964349, -0.058957696, 0.0518940128, 0.100731827, 0.00388432411, -0.108095072, -0.0483310521, 0.0424402244, -3.87781547E-5, 0.00679345242, -0.0625503212, 0.0974509194, -0.0517869554, -0.0629717186, 0.0131935058, 0.0845335423, 0.0384068862, 0.0844865515, -0.0107597206, -0.0289462674, 0.00104409433, -0.0263107494, -0.0816777646, -0.00835318677, -0.101592131, 0.0583590493, 8.998190e-04, -0.0342603326, 0.0584061556, -6.027910e-02, 0.0328878313, 0.0669875816, -0.0303430036, -0.00116935326, 0.0499132313, -0.0610228702, 0.0783293321, 0.0594595298, -0.0872630849, 0.0148896528, -0.00263268757, 0.00804062746, -0.0550095253, 0.0770085901, -0.0104964413, 0.0819614679, -0.0160990935, 0.041575335, -0.0603472963, 0.0195942502, -0.0326290019, -0.0607979521, 0.0039558867, 0.0904013664, 0.0904599502, -0.0497014299, 0.035398256, 0.0745921955, 0.101500019, -0.0758180916, 0.0985379815, -0.0258192047, -0.0747933835, -0.0454227328, -0.0841973796, -0.0118353739, -0.0802696272, 0.114207074, 0.0832519903, -0.0484450422, -0.0981414541, -0.0810780376, -0.0164885558, -0.0564479157, -0.0627493039, -0.0301925521, 0.0219839811, 0.021091098, 0.0346807167, -0.0516394265, 0.0421068557, -0.0636140481, -0.0475255437, 0.0373932123, 0.0226892214, 0.0632440373, -0.00471643126, -0.10338372, 0.00605824869, 0.132779732, -0.0134661766, -0.0899470598, -0.00360787846, 0.0457286276, -0.037575908, 0.0025046065], [0.011169021, 0.0310477559, -0.0730496272, -9.115850e-02, 0.0851952731, -0.0611053668, 0.0624286458, -0.0486972593, 0.0228335019, -0.0568458475, -0.0776855871, -0.0113170613, 0.0880586281, -0.0905537232, -0.0209919102, 0.11638733, -0.0223686956, 0.031846948, 0.0397801809, 0.0155511526, -0.00178823969, 0.010335057, 0.0476358458, 0.0448574796, -0.0529653691, -0.076245971, 7.848270e-02, 0.0591401085, 0.0051017669, 0.0244207773, 0.0726988167, 3.917730e-02, 0.00711168488, -0.00954100862, 0.0114282677, -0.0111458125, -0.0473014936, -0.0309329499, 0.0856953039, -0.033773832, 0.0101542659, 0.0351257622, 0.0497465432, 0.00467703026, 0.0090125436, -0.0275071058, 0.12839286, 0.131305948, 0.113939494, 0.0145667167, 0.030442277, 0.0157415383, 0.0218274146, -0.0533104278, 0.0116326446, -0.0307084527, -0.022156205, 0.046998024, 4.514060e-02, -0.0245014019, -0.00626086211, -0.0327965468, -0.0136478264, -0.0484924093, -0.047603514, 0.0533539243, 0.124058232, -0.0728973523, -0.0146666542, 0.0151933078, -0.0381172076, 0.0722043514, 0.0570858456, 0.00830399897, -0.0397827514, 0.0882834568, -0.106895052, 0.0701696798, -0.0374481454, 0.0821182429, 0.0265781451, 0.073816441, 0.0267220847, 0.0309290811, 0.0177875701, -0.00482073659, 0.0544501022, 0.0402225219, -0.0527565181, -0.013616357, 0.0244225394, -0.0718065426, 0.0971337705, 0.0063602291, 0.0682905093, -0.0933218449, 0.00926305167, 0.0696770698, -0.0348461755, 0.098738633, -0.00256235013, 0.0420090705, -0.0510480367, -0.0202097818, -4.989670e-03, -0.0232356843, 0.0682465583, -0.0499789119, -0.0523864925, 0.00605821703, 0.0174019616, -0.0348114558, -0.015649423, 0.0217123106, 0.0636861622, -0.0176034048, 0.0606001131, -0.0368643813, -0.00858059805, 0.0236623529, 0.0931664258, 1.466950e-01, 0.0715071037, -0.00769825699, 0.0486592911, 0.0025753493, 0.0289514735, -0.0187791288], [-0.00368305226, 0.0322329514, 0.00981125887, 0.00154145434, 0.0704686493, -0.0779479891, -0.061683245, -0.0433859639, 0.0203010105, -0.0699390694, 0.0257615857, 0.0148651721, -0.0202809814, -0.0791011602, 0.0430413783, 0.128677905, 0.00986703112, -4.691860e-02, 0.0348047912, 0.101772882, -0.00703937886, -0.0533727519, 0.0787092819, -0.082768619, -0.0534101427, 0.042450022, 0.0137787843, -0.0723595172, -0.0243295301, -0.0762387514, -0.0587664694, -0.0825910493, -0.0253244136, -7.476810e-02, -0.00982616096, 9.706210e-02, 0.0195050277, -0.065234512, 0.00973328296, -0.00712199183, 0.0325260349, -0.0759472921, 0.00408659456, 0.0790297091, -0.0205049682, 0.0251268186, 0.0517780036, 0.0192663018, 0.0296404772, 0.101778843, 0.023465775, -0.0628847107, 0.0469167382, 0.0150471842, -0.0373682976, -0.0126005299, -0.0454242565, -0.0650892705, 0.0188712161, -0.0623735934, -0.0385411568, -0.0499947444, 0.0852528586, 0.0281120073, -0.0848075523, -0.0336359739, 0.100612909, 6.799030e-02, -0.065407455, 0.013527032, -0.0293914601, 0.034013994, -0.046817638, 0.0207575019, 0.0199802071, 0.0523477532, -0.068762742, -0.00847898237, -0.0223355889, 0.00902506243, 4.409570e-02, -0.0650562495, 0.0471129417, -0.0345567279, 0.0846104398, -2.182010e-02, -0.00334339822, -0.00469339499, 0.00264495052, 0.00206393376, 0.0425265208, -0.0943729132, -0.0127579989, 0.00606524525, 0.0674381331, -0.0655672178, -0.0758054331, 0.00841473974, -0.0746973306, 0.0719178468, 0.011683899, -0.0556462742, 0.0237711165, -0.100598074, 0.00722856913, 0.0539770275, 0.0127729885, -0.0531448163, 0.0782927647, 0.0247047022, -0.0786346122, -0.0649715737, 0.0191062745, 0.0131197674, -0.0422401428, 0.0395171233, -0.0769733786, 0.0401807167, -0.00826299097, -0.00766324345, -0.0458425358, 0.0172521491, 0.0391303301, -0.0680238679, -9.029240e-02, 0.0251303911, 5.65356575E-4, 0.00480433088], [0.0626607463, -0.0955611839, 0.0356848128, -0.0717736483, -0.0198344365, -5.648160e-02, 0.0173462089, 0.0327303074, 0.100190446, -0.0289902519, -0.00806158874, -0.0148051716, 0.0732035711, 0.0339232571, 0.0093922792, 0.00501977326, 1.799070e-02, -0.0146144265, -0.0992027819, 0.0599675812, -0.00883067492, -0.0705413148, 0.0142534589, -0.0748864189, -0.0656453297, -0.0575776026, 0.113936782, 0.0133209713, -0.0555994287, 0.0432423241, -0.0584877431, 5.33254861E-5, 0.0271207727, -0.0228511412, -0.0773388818, 0.0608205311, -0.0312194116, -0.0736975893, 0.0082347691, 0.0588867106, 0.0236617792, 0.0517260581, 0.0251903273, -0.0368474722, -0.0556310154, -0.0763254687, 0.0772605836, -0.00253867428, -0.0388131738, -0.0465707406, -0.0149865197, -0.0632518381, 0.0107542872, 0.104702108, 0.0313062668, 0.071648091, -0.0971937105, 0.0117833605, 0.0141584352, -0.0222866163, -0.043966189, -3.343570e-02, -0.0710850731, -0.013152549, -0.0718382522, 0.0577347577, 0.0352626406, 0.0257248394, 0.0126931192, 0.0840929374, -0.0722080916, 0.0478601269, 0.0195011217, 0.0367753506, 0.0347540975, 0.0262698643, 0.0431492403, 0.0210923236, 0.0145685496, 0.0184088182, 0.0348301642, -0.00759865809, 0.0411245637, 0.0762185156, -0.0594257116, 0.058807835, -0.0468114428, -0.0571055859, -0.0606220961, 0.110474028, 0.078169778, -0.0251089018, 0.0536964536, -0.0282421149, -0.0670892298, -0.14587988, -0.00245830417, -0.0220612511, 0.0624418668, 0.0440533869, -0.05269669, -0.0563983023, 0.0432246104, 0.0695963427, 0.0409501903, 0.0191670414, 0.036150571, -0.0419986434, 0.00364899472, 0.0677553639, -0.0392917097, 0.0445184745, -0.0643225163, -0.043127086, 0.0852670073, 0.0969143733, 0.0278455969, 0.018533159, 0.0323250629, 0.00638647563, 0.061848972, 0.0756918713, 0.0423218757, -0.137719706, -0.0064306613, 0.0565875918, -0.0589951202, -0.0249417387], [0.0571893491, 0.0318088904, 0.0927395746, 0.0141773317, 0.0728320926, 0.0548146218, -0.0462304056, 0.0293956511, -0.0515669622, -2.311940e-02, -0.0266203452, 0.020176515, -0.024277119, -0.0641640499, 0.0345765315, -0.0316582881, 0.0284481738, 0.0211591292, 0.0159253553, 0.0653126389, -0.0470050909, -0.0193895195, -0.052858647, 0.0556101054, -0.00352640916, -0.061313577, 0.0516197495, -0.0470195524, -0.0451117679, -0.0491616055, 0.0669938326, 0.0295131039, -0.0292509459, 0.0616911426, -0.0143365376, -0.0848296657, 0.0180286504, -0.0510026701, 0.102758087, -0.0277553163, 0.014573236, 0.0319648162, -0.0582191385, -0.00615614187, -0.0981683731, -0.0627931133, 0.154800147, 0.0936438813, 0.0640328899, -0.0608269162, 0.0829723849, -0.0444499925, 0.129560858, 0.0605705902, 5.529230e-02, 0.0306280889, 0.0283798669, 0.00587750087, 0.0569930486, 0.10444998, 0.0327322781, 0.0359075442, 0.0763287842, -0.0127729801, -0.0398625955, -0.025975449, 0.0568584353, -0.0596982166, 0.0130505543, -0.00708530843, -0.00271356851, 0.0608310848, 0.0262524411, 0.0437235348, 0.0175897069, -0.0253189597, -9.576000e-03, 0.0149330376, 0.0243667457, 3.943310e-02, -0.0899267346, 0.00549221644, -0.0114547592, 0.0725943819, 0.0524919182, -0.0464074351, 0.0459215753, 0.074286133, 0.0171837434, -0.015379088, 0.00266214274, -0.0788730532, 0.0414987281, -0.126270205, -0.0586989075, -0.0353480391, -0.00932491664, 0.0461238585, 0.0493844599, -0.0467814505, -0.00684677111, -0.0791064128, 0.0939721763, -0.0554809421, 0.0602357239, 0.0764596313, 0.0711012632, 0.020113213, -0.0107593872, 0.046312619, -0.00669180229, 0.0590867288, -0.0348792225, 0.0620291531, -0.0276142526, 0.0348135047, 0.0800609738, 0.0241191778, 0.0364773199, -0.00586743327, 7.661430e-02, 0.0355399586, 0.0652437508, -0.0748545825, -0.0907399505, 0.0182579327, 7.676950e-02, -0.00399558246], [-0.0304363836, 0.00775440736, -0.052926641, -0.00287234876, 0.0222734679, 0.0671214163, -0.0127822896, 7.002920e-02, -0.0369147509, -0.144672289, -0.0410452932, 8.78730788E-4, -0.0204108264, -0.0855450928, -0.0567374155, 0.0183359962, 0.022923369, 0.035631042, -0.0633830801, 0.0047540646, -0.0503652729, 0.0112699931, -0.00157302874, 0.0841789767, 0.00514826272, 0.0550958589, -2.966950e-02, 0.0539632216, -0.0342309959, 0.00252622575, 0.0398064815, 0.0502362475, 0.021808723, 0.0796538889, -0.0171482693, -0.0963721647, -0.00639736047, 6.97501818E-5, 0.0287879333, -0.0697255358, -0.0555181243, 0.0594319142, -0.0258743241, 0.00474430621, 0.0112223355, 0.0613487884, 0.155990556, -0.0268833376, 0.0230841283, 0.0223937295, 0.0656113252, -0.121463284, 0.0383007638, 0.0406071879, 0.0210209247, 0.0149631044, -0.0964192226, 0.0493866764, 0.036055997, 0.0714457706, 0.0448778979, 0.0126627563, 0.0354098044, -0.0198725015, -0.0625115334, -0.0226650331, -0.0380995385, -0.0155503498, -0.0683404952, -0.0405441858, -0.0902443379, 0.0399808511, 0.0447219461, -0.0474740975, -0.00820052344, -0.0340941176, 0.0380726978, 0.0802618414, 0.017879175, -0.0324621052, -0.123061091, 0.0346705467, 0.0917709842, -0.0104476279, -0.0859376713, -0.0476970524, -0.11514467, -0.00905997399, -0.0355862491, 0.0659443289, -0.0185426977, 7.59658171E-4, 0.0231333449, -8.025780e-02, -0.0396540463, -0.0100901974, -0.0387763605, -0.00857685692, 0.0923183113, 0.0217092764, 0.077684924, 0.00639884872, 0.0539730974, 0.0295145977, 0.0325349756, 0.122571364, 0.0390625447, -3.696770e-02, -0.0217689667, -0.0777260214, 0.0484969914, -0.0406130552, -0.0270912424, 0.0885833502, 0.0499571711, 0.0496280603, 0.0616191365, 0.0995373725, -0.0587246642, -0.101987958, -0.0517210364, -0.00818245206, 0.0927900076, -0.0840996205, 0.0152573232, 0.0840438827, -0.0200172476, 0.0429934673], [0.0664851293, 0.00990448612, 0.0110151106, 0.0176899284, -0.0253903512, 0.0599514842, 0.0496434905, 0.0483215265, -0.0727561191, -0.115377747, -0.0506945327, 0.10985139, -0.0335049815, 1.51191416E-4, 0.0255213138, -0.0587625131, -0.0566623956, -0.0450977236, 3.11250711E-4, -0.0440035164, -0.0509168692, -0.0540739931, 0.0252919123, 0.0805247873, -0.0519659817, 0.0390544385, -0.0134496326, -0.0117082307, 5.179730e-02, -0.0950509682, -0.0140159884, 0.0778860151, 0.0446646065, 0.125919208, -0.0389510095, -0.00360725773, 0.0566014685, -0.0394148529, 0.0743916705, -0.00292813312, 0.0241601318, -0.0143824592, -0.0994162708, -0.0529424585, 0.0268219151, 0.0644201711, 0.108979478, -0.00780213717, -0.0475471504, 0.0270404518, 0.0105826212, -0.0148733808, -0.00496205315, -0.00574573129, -0.102808006, 0.0191877782, 0.0387246646, -0.0562671684, -0.0595549457, 5.382250e-03, -0.0195185337, 0.00694976235, 0.0750823468, -0.0165428091, 0.0072433548, 0.0151602523, 0.0179964937, -0.0358311273, -0.0685501173, -0.0137245618, 0.0348847024, -0.0584918261, -0.00875896867, 0.0873384848, -0.0531345047, -0.0101931831, -0.0314562358, 0.0490284488, 0.0429793224, -0.0353665836, 0.0175992977, 0.112007931, -4.000460e-02, 0.0467266254, 0.024040617, 0.0399103798, -0.0981829389, 0.0688347369, 7.579520e-02, -0.0258921534, 0.0653191879, -0.0114915436, -0.0358393751, -0.0735473186, 0.0117025748, -4.207630e-02, -2.030580e-02, 0.0332624801, 0.0887324735, 0.0896651819, 0.0769267976, -0.104724161, -0.043152459, -0.0571578555, 0.133685157, 0.0240291692, 0.0285649449, 0.0122761009, 0.0749968663, -0.0130020007, -0.0435156673, 0.0093666166, 0.0669883639, 0.104776911, 0.0183622278, -0.00394661492, 0.0554610789, 0.0875510349, -0.00706326961, 0.0108151138, 0.0798745378, 0.0392351225, 0.0139360074, -0.129477531, -0.0208801478, 0.0630563125, 0.0153882699, 0.101401374], [-0.0483741052, -0.0289721042, -0.00103723246, 0.00219045416, 0.034146484, 0.100545943, 0.0733237416, -0.00247064838, 0.0517915599, -0.0932966992, -0.0424391031, 0.111552484, -8.068790e-02, -0.0161739867, 0.0559270605, -0.0781932324, -0.0351725519, -0.0117961857, -0.0084041059, -0.0539326593, 0.101040728, -0.00695703551, 0.0593063794, 0.0442235433, -0.0611569472, 0.0101209693, -0.0171194412, 0.0741794184, 0.0351611748, 0.0122490963, 0.0643118173, 0.00298824138, -0.0041055074, 0.057973329, -0.0749390498, -0.0707173422, 0.119134806, 0.0140386876, 0.0582141019, -0.109134942, -0.0374376513, -0.0501965359, -0.0285631679, 0.0526744537, 0.0307514928, -0.0592969246, 0.133956775, 0.00332530309, -0.00929816439, 0.0446323119, -0.0408407375, -0.0460337624, 0.0704601929, -0.0493322127, 0.0436233208, 0.0101877665, -0.0180409048, -0.0346845835, 0.0511608459, -0.0113904877, 0.078162752, -0.0312196836, 0.100120097, 0.047699593, 0.0269247349, 8.179660e-02, -0.0420247428, 0.0530911498, -0.0751177445, 0.058667846, 0.0640821531, 0.0251949988, -3.95847514E-4, 1.430450e-02, 0.0804325565, 0.0454266965, 0.0507767685, -0.073311761, -3.317020e-02, -0.0576291457, 5.196730e-03, -0.0405383222, 0.0550608188, 0.0636686161, -0.0770652145, 0.00378195266, -0.0465135686, -0.0198754035, 0.00433614943, 0.038568262, -0.0413447693, -0.0229816921, 0.091712825, 0.0308688879, 0.0487955548, 0.0228529833, 0.0618757345, -5.861670e-02, 0.112455033, 0.0884857327, 0.113359615, -0.0495571569, 0.0245788135, 0.0470164306, -0.0478788614, 0.0994637385, 0.0646808594, 8.898000e-03, 0.0714193657, 0.0410987511, -0.0278323982, 0.0528450049, -0.0403785482, 0.0761734322, 0.011311953, -8.960550e-02, 0.0751479715, 0.0729393959, 4.168510e-02, -0.075490512, 0.11282745, -0.026755657, 0.0405101962, -0.0369227603, -0.0447725058, 0.0578256175, 0.0725296065, 0.0174698643], [0.0173027217, 0.0273722634, -7.597230e-02, -0.063203685, -0.0211273246, 0.0606784858, -2.886700e-02, -0.0452948585, -0.0177218318, 0.0135316569, 0.0166326761, 2.428600e-02, 0.0487608798, -0.0579873249, -1.966440e-02, -0.0464795232, -0.0296029709, -0.0556438304, -0.0711672679, -0.0257280394, 0.0179713797, -0.0457453243, 0.0210289806, -0.0218979884, 0.101457894, 0.00196535233, -0.067577213, -0.0109132389, -0.0718914047, 0.0626024604, -0.0715520083, 0.0539647266, 0.078771986, -0.0229955334, -0.0365259834, 0.0709578618, 0.0317957029, 0.0158533342, -0.0761383101, 0.0287850369, -0.0394533835, 0.0836631134, -3.143170e-02, 6.784240e-02, 0.0484912805, -0.0626124814, -0.0500912145, 0.0837190672, -0.0558335781, 0.06331788, 0.0146549735, 0.0321040861, 0.041144643, 6.751340e-02, 0.0141421538, 6.787720e-02, -0.0727224051, 0.0132898837, 0.0390574187, -0.0660373866, -0.0664454103, 0.0248555616, 0.00310209696, -0.0111262426, 0.0540545844, 9.17405937E-4, -0.049600523, -0.0521876104, 0.0510704294, 0.0246448219, 0.0318822749, -0.0755691901, -0.047562141, 0.0894917101, 0.0575261861, -0.0436167531, -0.0552739538, 0.0505686067, -0.0224297792, 6.931450e-02, -3.708560e-02, 0.0734008625, 0.0103243664, -0.071482189, -0.0903966426, -0.01417764, -0.0985409244, -0.0632052869, -0.0606636442, -0.0730664507, 0.0149511704, 0.11674156, 0.0927436203, 0.0789160281, -0.0216993745, -0.0462918691, -0.00600069528, -0.0714726374, -0.0211046766, 0.126921102, 0.0361505188, -0.0617880039, -0.0581649207, -0.0279690698, 0.0759506077, 0.0888187959, -0.0670023784, 0.00480468292, 0.0605482236, 0.0589609109, 0.0421519056, 0.0307300631, 0.0764785558, 0.0670257136, -0.0446144603, 0.0280631632, -0.0760986432, 0.11935398, -0.0113635594, -0.0721771047, -0.0171545818, 0.00397773413, 6.517960e-03, -0.0785179287, -0.0239887293, 0.0317542069, 0.0926018506, 0.0182127412], [-0.0704019367, 0.0411439389, 0.0172322616, -0.00207617506, -0.0585092716, 0.0685474798, -0.0354712196, 0.051287625, 0.0446357392, 0.0910018235, 0.0986107513, 0.0464624763, -0.0585251525, 0.0311556794, 0.00655906042, -0.00309904129, -0.0484995432, 0.0320903249, -0.084857814, 0.00870609283, 0.0635229424, 0.065015927, 0.0235420242, -0.0362216197, -6.10333052E-4, -0.0331206545, -0.0821325406, -0.019169338, -0.0759894401, 0.0241843797, 0.0161436442, -0.110900849, 0.0121806562, -0.00488102715, 0.017021101, -0.010996609, 0.0504945442, -0.0464427657, -0.0278699473, 0.110677578, 0.0747934356, 0.08159855, 0.0338573791, 0.0715029836, -0.0505694337, -0.058895845, -0.0357435048, -0.0169570185, -0.0718372762, 0.0743911862, 0.0775733143, -0.0576638877, -0.0473418236, 0.0518958233, -0.0435929857, 0.0388445258, -0.0266268589, 0.065321371, -0.0349251144, -0.0630429313, 0.0228045303, 5.521990e-03, 0.00581225473, 0.034225326, 0.0651707575, -0.0501294062, 0.0149149233, -0.0413255244, 0.0322518907, 0.00786103401, 0.0491635241, 0.0768176243, -0.0193179734, 0.0922980606, -0.0140577862, 0.0225664489, 0.0299203321, -0.0694495216, 0.0293734241, 0.0112311961, 0.116865508, -0.079670988, -0.00313979667, -0.103796512, 0.00794999115, -0.0410825424, 0.0405455269, -0.0734505057, 0.00541071454, -0.0856678411, -0.0641062259, 0.0907037258, -0.0188796781, 0.0785393491, -0.0306458715, -0.0138293421, 0.00726580713, 0.00324558304, -0.0643490254, 0.0982726141, -0.0320402868, -0.0109089334, -0.0660916939, 0.0826189145, -0.0281728171, -0.026585862, -0.0442656465, -0.0586841442, 0.0323035531, 0.0155501124, 0.020771319, 0.0404476374, -0.0399285927, 0.0140123339, 0.0517898835, -0.0685950518, -0.0746524483, 0.0697418079, 0.00777211785, 0.00854037143, -0.00729880295, 0.0187083986, -0.0481098816, -0.0621879585, -0.0109830648, -0.0792187452, 0.0412748568, -0.0316956416], [-0.0658988878, 0.0304978248, 0.0772287771, -0.00199467689, 0.0167596452, -0.0197507385, -0.0501650833, 0.0346690193, 0.0271777231, -0.00425544614, -0.0200186707, -0.0347173065, 0.058159817, -0.056957256, 0.0485271886, -0.0221207086, -0.0533727705, -0.0642366186, 0.0566832721, -0.0793497785, -0.00993586052, -0.0874171778, 0.0948045775, -0.0279334486, 0.0369786471, 0.0718351379, -0.0518591851, 0.0547921024, 0.0655196607, 0.0520299114, -0.0332436934, -0.0748104751, 0.0600375198, -0.0674293041, 0.0611510202, 0.0150010278, -0.0258178469, 0.025237659, -0.0566989072, 0.108093508, 0.0331169218, 0.0051161875, 0.0196424946, 0.0646363198, -5.435060e-02, 0.0150500191, 0.00569122052, 0.0883718058, 0.0447825603, -0.042290438, 0.011036627, -0.0180618614, 0.0106239654, -0.0638379082, -0.0718075559, 9.516730e-03, 0.0654533803, -0.0456754491, 0.00238535274, 0.00248708879, -0.105061464, 0.0728488117, -0.0804129615, 0.0874519869, 0.0652997643, 0.0396582559, -0.0141444299, -0.044848334, -2.75971921E-4, -0.0401901715, -0.0355010889, -0.0490600243, 0.00634298148, -0.0238298029, -0.0249347258, 0.0390686058, 0.0163688306, 0.0305611342, 0.00613508513, -0.0161671303, 0.117407858, -0.0634442493, -0.0597232915, 5.85729256E-4, 0.0392963812, 0.00426908722, 0.0501084588, 0.077138342, 0.0400884598, 0.0171761923, 0.0293912888, 0.104217112, -0.0223135538, 0.0335573629, 0.0458169542, 0.0787661523, -0.0222878363, 0.0241825692, 0.0302008446, 0.0900815725, -0.0580687411, 0.0129752206, 0.0324884765, -0.0320967324, 0.0698454827, 0.0219440553, 0.0124051115, -0.0234950334, 0.00353738177, 0.0903079659, -0.0559331514, 0.00792717095, 0.0575597435, -0.0292894989, 0.0572942831, 0.0728573427, -0.0600457937, -0.0433494858, 0.0342143103, 0.0294340961, -0.00923540629, -0.0245940275, 0.037098363, 0.0570147708, -0.0568561181, -0.00748180458, -0.00679677166, 3.698950e-02], [-0.0562278926, 0.00525559718, 0.0695442632, -0.00289621949, -0.0704045072, 0.0661311746, 0.0699720904, -0.0469551496, -0.0879473909, 0.0583058074, 0.0703381076, 0.0116288168, 0.0590006672, 0.00345243467, -0.0738636777, 0.0348823816, -0.048314698, 0.0325953029, 0.00967360474, -0.0257730298, -0.0552232154, 0.00444533257, -0.0432493351, -0.0177823827, -0.0144675924, 0.00328466552, 0.0357434563, -0.0434095226, -0.0800214186, -0.0492201559, 0.0803794116, 0.0423692949, -0.0294359643, -0.0706524476, -0.002688193, 0.0754295439, 0.0575597882, 0.060704384, -0.0614282712, 0.00197305903, 0.0116989305, -0.0345885605, -0.0654743611, 0.0575453341, -0.0261662323, 0.0381980762, 0.0555524677, 0.0669917911, -0.0075162421, -0.0602700636, -4.345040e-02, 0.0368668586, 0.0644213781, -0.0514286049, -0.0956548675, 0.0126981316, 0.0177329555, 0.0314682648, 0.0135144526, -0.00666469056, -0.111091807, -0.0672166571, -0.0154439984, -0.0724299625, 0.032205347, -0.0794619322, -0.0736370608, 0.00892520789, -0.0388033837, 0.039518252, 0.102728643, 0.0193748176, -0.0433148779, 0.0306659732, -0.0372331478, 0.0345540941, -0.0129459258, -0.00481307972, 0.0334916264, 0.0258880872, -0.0141041819, -0.0181752779, 0.0349202119, -5.166270e-02, 1.099040e-01, 0.0615331568, 0.0291987415, 0.0371176228, 0.0774670243, -0.0049203909, -0.0983901098, 0.026995752, -0.00806670357, 0.0343193077, -0.0429879501, -6.324860e-02, -4.275960e-02, 0.0311166681, -0.0670936927, 0.0891658515, -0.00949452817, 0.0497608557, -0.0157836452, -0.0493962839, -0.00793733727, 0.0169434194, 0.074670434, -0.00473615946, -0.0780910849, 0.0477571189, 0.0795964822, -0.0327725522, -0.0581486709, -0.0332388394, -0.030666152, -0.037360549, -0.0815881267, 0.00812515896, 0.0121879745, 0.0137995183, -0.0658364593, 0.0555670224, -0.111077622, -0.0427237935, 0.00980150234, -0.0120479129, -0.00414835196, -0.0668195784], [-0.0341664329, 0.0116510196, 0.0293381363, 0.0267576482, -0.0456523895, -0.07676927, 0.0338947587, -0.00815477967, 0.00971870217, -0.0301123708, -0.0846422091, 0.0349761732, 0.0796421766, 0.0836689993, 0.0395639651, -0.00795788411, 0.0536982045, 0.0214422699, 0.0602439381, 0.00537694199, 0.0762553811, -0.00883201417, -0.05666136, -0.0654517934, 0.069386065, -0.0432836562, -0.056884747, 0.00602854276, -0.0443772301, 0.0267194435, -0.0568551049, -0.00395127526, 0.0762640685, -0.039173048, -0.0176868644, -0.0653268396, 0.0693641901, 0.0430734195, 0.0249561779, 0.00257109525, 0.00426856754, -0.0584219024, 0.0403046273, 0.0548327416, -0.0598850064, -0.0269030202, 0.0472214296, -0.0694085583, 0.0111758914, 0.0873694419, 0.0721604377, 3.959320e-02, -0.0774185433, 0.015648352, -0.0596090294, 0.0318497047, -0.0161472932, 0.078557074, -0.0116417799, 0.0810927227, -0.0286169928, -0.0368695892, -0.0801543593, -0.00172377948, -0.0431150645, -0.0855568349, 0.0338449925, 7.63054937E-4, -0.0429923348, 0.0683212057, 0.0513618216, -0.0369427651, 0.0219250843, 0.0821683183, 0.0434384979, 0.01383828, -0.0170230642, 0.0647436902, 0.0977392643, -0.0767800733, -0.0381165259, -0.0408220775, -0.0152350767, 0.0404015929, 0.0112012876, 0.0112183224, 0.00413154438, 0.0780146867, 0.052460283, -4.549390e-02, 0.0589961447, -0.0282898471, -0.0630439669, -0.00995221361, 0.0471052974, 0.0552708767, -0.0635997951, 0.064704828, -0.0613697693, 0.0422632806, -0.0201960821, 0.0712276623, 0.0476021022, 0.0753476843, -0.039127823, -0.0301911272, 0.0125683043, -0.0605036244, -5.770670e-02, 1.53586952E-4, -0.0144458162, -0.046352528, 0.0530767776, 0.0588696115, -0.0463253148, 0.0410032347, -0.0625635237, 0.056772057, 0.0974246412, -0.0637753084, -0.0808637887, -3.633010e-02, 0.0431346968, 0.037909504, -0.0433309115, -0.0502292737, 0.0673866719, -0.0241719019], [-0.0286869369, 0.0785459205, -0.0670020655, -0.054116521, -0.0197646338, 0.0244262032, -0.0497934893, -0.0225730576, -0.0901033207, 0.0468726233, -0.00202850206, 0.0426048897, 0.00610861601, -0.0506246723, 7.84701667E-4, -0.00217960612, 0.00892131776, 0.06499812, -0.0531020574, 0.0268979762, 0.0320004746, -0.00612023147, 0.0852414444, 0.0163205322, -0.0111421086, 0.0082565993, -0.0593086705, 0.0367845818, -0.0027557537, 0.00816628895, -0.0763748065, -0.00120939047, -0.0418343358, -0.0312703736, 0.0131081231, -0.0193366557, 0.0105956029, -0.0288224537, -0.0339553691, 0.0252838805, 0.025882151, 0.0574871823, 0.0345477797, 0.0356476195, 0.0315451324, 0.0523043834, 0.0468318574, -0.0968840867, 0.077003777, 0.00345856952, 0.0232603457, -0.00490508927, 0.0628047064, -0.0667093396, 0.00295556244, 0.0605270862, 0.0465237089, -0.0773448721, -0.0166921914, 0.0119002629, 0.0215125568, 0.0496187843, -0.0285367537, -0.00703690806, -0.0341369696, -0.00872750766, 0.00273551932, 3.550370e-02, 0.0778377205, 0.0169053711, -0.036433477, -0.0504684597, -0.00894789211, -0.0452034548, 0.00305645843, -6.24085544E-4, 0.0336866975, 0.0646166503, -0.0706268549, 0.0123385806, -0.00785662141, -0.0506548434, -0.0665548369, 0.0460108407, -0.00525101088, -0.0781717151, 0.0303249136, -0.0248929802, 0.0742955804, 0.0329088271, -0.042049706, -0.0183050539, 3.443200e-02, 0.0660320073, 0.0727272927, -0.0201513041, -0.0478909388, -0.036976032, -0.0925593078, 0.0732274353, -0.0472741388, 0.024451131, 0.0566558801, -0.0813695266, 0.0681043342, -0.0901695117, -0.0290543791, 0.00847975723, 0.0284498315, 0.0353116356, 0.0226151366, 0.0599962398, -0.0704073682, 0.0501346327, -0.0835339203, -0.0336255915, -0.0592435226, -0.0643677786, 0.0286331512, -0.0567253269, -0.031524606, -0.031182019, -0.088825956, -0.0396338776, -0.0446158051, 0.0466152616, -0.016253829, -0.0092763463], [-2.37174405E-4, -0.0326594561, -0.0518161096, 0.0297956578, 0.0113604702, 0.0257194471, -0.0713970587, 7.086920e-02, -0.0735652819, -4.38245712E-4, 0.0574915595, -0.0626461207, 0.0633258894, 0.0420437083, 0.0012991929, 0.0142563339, 0.0718098804, 0.0225364082, -0.0465192534, 0.0252637286, 0.0285764355, -0.00697185379, -0.00213974342, 0.0647741258, 0.0294288918, -0.0555453598, 0.086164236, 0.0736418068, -0.0347989723, 0.0346688665, 0.0361717381, -0.0851357206, 0.00543162599, -0.0767583922, 0.0768974572, -0.00184399309, 0.0108898534, -0.0764915124, -0.0457772166, -0.0544468425, -0.0279185269, -0.0115082571, 1.632990e-02, -0.0617802776, -0.0732768402, -0.0533575453, -0.00396795664, 0.0505279303, 0.0616114139, -0.00742759043, 0.0675984472, 0.0292977188, -0.0209241603, 0.063436687, -0.0651263892, -0.0788798406, -0.016870264, -0.0673978552, 0.0699933469, -0.0561084673, 0.00832598097, -0.0390511118, -0.0596071929, 0.00648355297, -0.101925708, -0.0834077894, 5.079300e-02, -0.0285932478, -0.0509327687, 0.0172873307, 0.00379162165, 0.0634379312, -0.0359401144, 0.0210305918, 0.0403490737, 0.0125808232, -0.0206675231, 0.0473467968, -0.0356420465, 0.0688048303, 0.0520996153, -0.0433993414, -0.0687825679, -0.0486998111, 0.0726323202, -0.0605710559, -7.493870e-02, -0.048481077, 0.0469579175, 0.00939394347, 7.722130e-03, 0.0902827531, -0.0496308468, -0.0169346128, -0.001307945, -0.0739360601, -6.50253438E-4, 0.0736930445, -0.033827655, 0.0628647283, 0.00910238549, -0.0762675404, 0.0242038891, -0.0330077745, 0.0279800892, 0.0432785936, -0.0376020931, -0.0499733947, -0.0328677669, 0.0513616614, 0.0541170239, 0.0515425801, -3.908560e-02, -0.0307571944, -0.0542234071, -0.0186240394, -0.0676324293, 0.00853047147, -0.00647591753, -0.0622143298, 0.0651538372, -6.901900e-02, -0.0553382672, 0.0621904656, 0.0285513196, 0.00587351434, -0.045045089, -0.0326070078], [-0.00433421088, 0.0754479542, -0.0176221393, 0.0131370826, 0.0224186387, 9.08613321E-4, 0.0689128637, 0.0184266977, 0.0495916046, 0.00194711669, -0.0719912201, -0.0107003683, 0.0800961256, 0.0586943477, 0.0835730582, -0.019878285, -0.0141217196, 0.0719209164, 0.0734217464, -7.143860e-02, 0.0216478426, 0.00323227234, 0.023205854, -6.073750e-02, -0.012935576, 0.0177210942, -0.0694659501, -0.0461970977, 0.0598032922, 0.0129680606, -0.0803914815, 0.0610554367, -0.0510233603, 0.0428521745, -0.0463130958, 0.00313877384, -0.0458622463, -0.0194839947, -0.00699235871, -0.00424931943, -0.0593934506, -0.00169686216, -0.0067837229, 0.0491224192, 0.0142089482, -0.0671283528, -0.0108829653, -0.0930949897, 0.0189366825, -0.0406511091, 0.0132761532, -0.0545895547, -0.0390258133, 0.051767841, -0.0234351512, -0.0493055321, 0.0424847938, 0.0512331761, -0.0569301434, -0.0352230631, -0.0746003911, 0.0614871942, 0.0278102886, -0.0108155841, 0.012347756, -0.048029732, -0.0282346327, 0.0232321154, 0.0062327343, -0.0189962238, 9.62542137E-4, 0.0834867805, 0.0402439572, 0.0805154294, -0.0148371467, 0.0179849472, 0.0578917041, -0.0479064584, 0.0143976705, 0.0740866587, 0.027026372, 0.0677872896, -0.018419968, -0.0332192071, -0.0101232128, -4.673080e-02, 0.0697175488, -0.0189970322, -0.0194047894, -0.0122981314, -5.878150e-02, 0.0707497075, -0.0467807353, 0.0149644818, -2.46402487E-4, 0.0479231328, -0.0379784107, 0.0639818534, 0.064990744, -0.00552951777, -0.104124017, -0.090163283, -0.0014690035, -0.0221502855, 0.0551033206, -0.0346769728, 0.0159846842, 0.0568450093, -0.0114399511, -0.0327433869, -0.047608465, 0.00431246031, 0.0405520163, -0.0454956889, -0.0711997375, 0.0371252522, -0.0611526258, 0.0454449914, 0.0353211164, 0.0594759695, 0.021644704, -0.0576747246, -0.100122243, 0.0371020921, -0.050141491, 0.024092827, -0.0101092532, 0.0450958088], [0.030408429, 0.0372418612, 0.0334464274, 0.056974303, 0.0222162642, -0.0677001849, 0.0156563185, -0.0291713104, -6.474490e-02, -0.0404086523, -0.0558358096, 0.0695021227, 0.0572267622, 1.711380e-02, -0.0558890887, 0.0974478349, 0.0486147664, 0.0663960055, 0.0744529217, -0.0323518626, -0.0631597042, 0.0587895699, -0.0449218899, -0.066295892, -5.71843178E-4, 0.00566285662, -0.0346066728, -0.0280198269, 0.0410537608, -0.0660751909, 0.0696951821, -0.0695347339, 0.0263456628, 0.00217297184, -0.0249123834, -0.0732514039, -0.040051613, 0.0370579064, 0.0200171378, 0.0559769943, 0.0522621833, 0.0102397213, 0.0494328663, -0.0585053451, -0.0447975174, 0.0548398234, 0.0294057615, 0.0373713188, -0.048491776, 0.0498241819, -0.0797501653, 0.0195252877, -1.264270e-02, 4.710910e-03, 0.0442561135, -0.0443758406, 0.0688393489, -0.0341722928, -0.0784817859, 0.0059074373, -0.0506359339, 0.0376216099, -4.434730e-02, 0.0783739909, 0.0652790367, 0.0362625532, -0.0132520897, -0.0381197184, -0.0172765832, -0.0213766918, -0.0669673085, 0.0164708849, -0.0539371148, 0.0602194853, 0.0768525153, 0.0484964177, 0.0520630255, 0.0146366702, 0.0377210602, -0.0335743129, -0.0808967947, 0.0627108216, -7.187100e-02, -0.0421913713, -0.0177681297, -0.00857922714, 0.0529706404, 0.0296659824, 0.0755588934, 0.0765514373, 4.650550e-02, 0.0823438615, 0.0207184143, 0.0285282321, 0.0704220459, -0.0386604555, -5.885430e-02, 0.00348551269, 0.0281083491, 0.0424715206, -0.0810738429, -0.0641428605, 0.00669855671, 0.049241025, 0.0117536588, 0.00569183566, -0.017024802, -0.0671262369, -0.077661775, 0.0280610844, 0.0268049911, -0.0202278495, -0.0476807915, 0.0537743084, 0.0334762111, 5.161060e-02, -0.0666296333, -0.0461303256, 0.00937901996, -0.0406198874, -0.053094741, 0.0260975193, -0.0105182491, -0.0676372498, 0.00683754636, -0.0474955849, -0.100072131, -0.0492355488], [-0.00937527231, 0.065784879, 0.0796005651, -0.0340408832, 0.0222758222, 0.0361373462, 0.0523682907, -0.0367231667, -0.0313124582, 0.0417222865, 0.0135730114, 0.0505484194, -0.0578922704, -0.0322792791, 0.0355608389, 0.0829096884, -0.0705653131, -0.0670532882, 0.0431876779, -0.0373778865, -0.075571306, 0.0110931415, -0.0336987823, 0.0518669486, -0.056838043, 0.0241005346, 0.0272246804, -0.0552613847, -0.0335091129, 0.0302273128, -0.0703389123, -0.0393924415, -0.0413828902, 3.930330e-02, 0.0393670909, 0.0317711793, 0.0504100248, -0.0261089262, -0.0622677356, -0.00167252729, 0.0462111831, -0.0414925255, -0.0112730144, -0.0578198135, -0.0141595928, 0.0134417331, 0.0278912168, -0.00466216635, -0.0170954801, 0.0155605469, 0.015347871, 0.00552752195, -0.00756472162, 0.0588015579, 0.0227614436, 0.0172559191, -0.0389153734, -0.0587298274, -0.0763852447, -0.0503752306, 0.00573443389, 0.0650325492, -0.0648047626, -0.0811486393, -0.0132911727, 0.0400901549, -0.0622951761, 0.00667581148, -0.0698248445, 0.0334264264, -0.0346828252, -1.776280e-02, -0.0733605474, -0.0624462552, 0.0714385509, -0.0409543179, -0.00137172057, -0.076152429, 0.0442670025, -0.00246808259, -0.0820024907, -0.0292105321, -0.0267989598, 0.0143139465, 0.0102902735, 0.0294390805, -0.057854943, -0.0577705167, 6.766970e-02, 0.0860617831, 0.0621221475, -0.05477909, 0.0290012099, 0.0243978593, -0.0248412564, -0.0179613419, 0.0529725961, -8.307040e-02, -0.0811949744, 0.0680374503, 0.056263335, 0.00468208129, 9.199610e-02, -0.0369122252, 3.84180836E-4, 0.0718339979, 0.0124246487, -0.0197689217, -0.0415622629, 0.051971402, 0.0541440248, -0.0702406317, -0.0341663845, -0.065281041, 8.75335463E-5, 3.236220e-02, -0.0598215573, -0.00125991134, 0.0354633033, 0.06402798, 0.0693504587, -0.0316210613, -0.0721111819, -0.0203717314, 0.0458696857, 0.0670346618, 0.0518306457, 0.0136902817], [0.0588955507, -0.0149087077, 0.0498872772, -0.0178407487, -0.0310364012, -0.0749754384, -0.0654156357, -0.0316978395, -0.0193591453, -0.0830318853, -0.0279697645, -0.0546506867, 0.0151572255, -0.0723059401, -0.00685471482, -0.0298012085, -0.0515304171, 0.0356046706, -0.0124716964, 0.042129606, -0.0072523579, -0.0657998323, -0.0621222443, 0.0749215558, 0.0675904602, 0.0163869429, -0.013977807, 0.0111478427, 0.0167909376, -8.322300e-03, 0.054561507, 0.0522882901, 0.059230309, 0.00246451981, -0.0708615258, 0.0285633914, -0.0255607255, -0.0240307823, -0.0666321814, 0.0844591781, 0.0168792773, -0.0620827376, 0.0321123973, -0.023881169, -0.0392452702, -0.0691297427, -0.0692406148, 0.0475085862, 0.0308549665, -0.0468628183, 0.0162365176, 0.0635899082, 0.0498612076, 0.0753625482, -0.00587161072, 0.0513375588, -0.065192692, -0.0590993762, 7.222620e-03, -0.0221031979, -0.0192717165, -0.00571152661, -0.0708674565, 0.028895868, -0.0120615112, 0.0526807345, 0.0156155908, 0.0725332722, 0.0656806231, 0.0433419198, -0.0277285855, 0.0668174326, 7.474240e-02, 0.0761190578, 0.0753790364, 0.0598691627, -0.0607857704, 0.0532830879, -0.0431578197, -0.049102705, 0.0569077209, -0.0570250899, 0.021423718, -0.0512191281, 0.0714102238, 0.0650227517, -0.0176062454, 0.026687583, -0.0711714327, 0.00361525198, 0.0350063741, 0.00797231775, 0.0726381391, 0.0330405161, 0.0657132044, 0.0677984208, 0.0499989577, -0.025685519, -0.0516311973, -0.0556214042, -0.0352342948, 0.00205466151, 0.0437282734, -0.0528190061, -0.0551180728, -0.0205059182, 0.0337101221, 0.062996596, 0.0568779483, -0.0753988251, 0.0501425117, -0.0413291343, -0.0173824895, 0.013349616, 5.376500e-02, -0.0821729376, 0.0719565079, -0.00397723448, -5.466630e-02, 0.0791182592, 0.0184095353, 0.0232667197, -0.00477722287, -0.00988489575, 0.037102893, -0.00348441512, -0.0350736268, -0.0541341826], [-0.0778917446, -0.0014344115, -0.0633535758, 0.0689555779, 0.0105419755, 0.00737282261, 0.069082275, -0.018617766, 0.0216032378, -0.0542136952, -0.0711191297, 0.0764021128, 0.0248422343, -0.010918011, -0.029379677, 0.0720374733, -0.056426432, -0.00847300049, 0.0216102153, -0.0377588347, -0.050806161, -4.092140e-02, -0.0750065073, -0.0466062315, -0.0635053515, -0.0333888568, 0.0477307215, 0.0333644077, -0.0775851532, 0.0652825534, 0.039011512, 0.00256384723, -0.041030664, -0.0508870892, 0.0261076391, 0.061316736, 0.0742479712, 0.0091820946, -0.0715667233, -0.0229886211, 0.0490065813, -0.0332830288, 0.0245664027, -0.0192065258, -0.0542747714, 0.0109631866, -0.054526519, 0.00540109491, -0.032343749, -0.0771861598, -0.00104580284, 0.0494474284, -0.0451815911, -7.811740e-02, 0.0188916605, 0.0197318811, 0.0386109278, -0.0364128761, 0.00567115098, 0.047455851, 0.0638476536, -0.0790027529, 0.0672708228, 0.0715030208, -0.0101871751, 0.0686782449, 0.0418843403, -0.0062895473, 0.0214726217, -0.0600476526, -3.715220e-02, -0.0275256671, -0.0742634535, -0.0539571308, -0.035614755, 0.0531156175, -0.0481271259, -0.00747610908, -0.069178164, 0.0280838907, -0.0302975215, 0.00799064151, -0.0214678347, 0.0231786612, -0.0521020405, 0.0117150331, 0.0756001174, -0.0537169389, -0.0745314583, -0.0791770294, -0.0202313215, -4.32497676E-4, 5.33498649E-4, 0.00246641575, 0.0700252056, 0.0020198517, 0.043581672, -0.0439174511, 0.0702745318, 0.0676263645, 0.0545578338, -0.0550758056, 0.0622737072, 0.0236173924, -0.048188135, -0.0344321877, 0.071235247, -0.018416293, -0.0139540406, 0.0770205781, -0.00431104051, 0.0283149462, -0.0665603504, -0.0329809338, 0.00670649484, 0.0445120521, -0.0327898413, 0.0378396176, -0.0271571036, 0.0147022186, 0.00453793071, -0.0263645221, -0.0644283294, -0.0537455715, -0.0725196078, -0.0596420616, 0.0189819317, -9.419830e-03], [4.152630e-02, 0.0333358981, -0.014785286, -0.0432047062, -0.00306366384, -0.0577840842, -0.0664268807, 0.047662463, -0.00984260347, 0.0211330298, -0.0466438495, 0.0338366367, -0.0548538603, 0.0460235402, 0.00626785866, -0.0743497163, -0.0797159225, 0.071902059, -0.00884566456, 0.074848555, 0.00398515211, -7.15020346E-4, 0.015419744, -0.0078760162, -0.0405125357, 0.0769153461, 0.038473323, 0.0293221436, -0.0123961139, 0.0548013821, 0.0191948842, -0.0588195287, -0.010518671, -0.0314905681, -0.0178829078, 0.0350192189, 0.00429106271, 0.0111883404, 0.0188629534, 0.0335394703, -0.068466723, -0.0650711283, 0.0642407313, -0.0613482147, 0.0666025578, 0.0544260964, -0.0367499143, -0.00191841484, -0.0181270037, 0.0158235095, -0.0500302501, 0.0581893697, -0.00640437054, -0.0600978658, -0.0239700694, 0.0459396802, 0.0658614337, -0.0564384609, -0.045977287, -0.058985319, -0.0382324979, 0.0112106502, 0.00740259932, -0.0547467209, -0.0228558499, -0.00949312746, 0.0342565142, 0.0680699274, -0.0333404727, 0.0591326356, 0.00366594619, -0.0106899785, -4.718040e-02, 0.0290769804, 0.00917579233, 0.0218650978, 0.0129913166, -0.0736548081, 0.0368564092, 0.0249979757, 0.0211160462, -0.024028739, 0.0281026345, 0.0611704551, 0.0520792231, -0.0141297625, -0.0466975607, 0.0550701506, 0.0197245404, -0.0798647627, -0.028107455, -0.0392121784, -0.034132082, 5.198570e-02, -0.0117475037, -0.0557151958, 0.0599748455, -0.0175172351, -0.079991594, -0.0551780164, -0.059198197, -0.00591025176, 0.0602976233, 0.0234064758, -0.0236543138, -0.027055908, 0.039309144, -0.0693074762, -0.0385653339, 0.0243970212, 0.0350854732, 0.0637559816, -0.0283787027, -0.0304436665, -0.0740398765, 0.0558999777, 0.0737039744, -0.0287788343, -0.00343574327, 0.0528970324, -0.0492451489, -0.0458996221, -0.0571821295, 0.0495917834, 0.00461049285, -0.0249634944, 0.037383426, -0.0804401263], [0.0153236575, 0.0707764551, 0.0332166106, 0.0301796738, 0.0227853879, -0.00439648191, 0.0255908016, 0.0793099775, -0.0778386146, -0.0310781915, -0.0149630988, -0.0690512508, -0.0761901289, -0.0692585408, 0.038117718, 0.00237852172, 0.00951560121, 0.0219349861, -0.0158199444, 3.551940e-02, -0.0576473027, -0.0446339771, 0.0178926364, -0.0304292906, -0.0721175149, -0.0578621067, 6.514150e-02, -0.0665279403, -3.305070e-03, 0.0304375123, 0.0660982429, 0.0326991156, 0.0214041118, -0.0112810601, -0.0170176551, 0.0304720066, -0.0755731836, 0.0657148957, -0.0338423066, -0.00798192806, 0.0146084521, 0.0446270965, 0.0362025537, -0.0674553439, 0.0477886759, 0.0737060308, 0.00883098877, 0.0206507761, 0.010366166, 0.0445979871, -0.0306755397, -0.0748140365, -0.0432847105, -0.0216561407, 0.0391118489, 0.0767454654, -0.00300601218, -0.0258480683, -0.0209773723, -0.0320584625, 0.012874308, -0.0329859965, 0.0063082031, -0.0452547818, -0.0135316085, -0.0761915892, -0.0406845175, 0.0665829778, 0.0734475479, 0.0237176549, 0.0312134568, 0.0419693328, 0.0225340333, -0.0500089489, -0.0179906711, 0.0786606818, 0.0313974246, -0.0636092499, -0.0538004152, -0.0583587587, 0.0186541807, -0.0603477918, 0.0652609468, 0.0570888259, 0.0608045124, -0.0750621781, 0.0773464888, 0.0388183333, -0.0771139264, -0.0509517044, 0.00577260181, 0.0799327269, 0.058094088, 0.0389130116, -0.0142691489, -0.0165936667, -0.0554242544, 0.0546451397, 0.0523119718, 0.0429072529, -0.0116985133, -0.0713709816, 0.0805085599, -0.0306471027, -0.0742201731, 0.0293219052, -0.00882936362, -0.0105775651, 0.023801323, 0.0255100969, -0.0754884481, 0.0564848855, -0.0641692951, -0.00844925269, -0.0579411313, -0.0249332972, -0.0661749318, -0.0750345141, 0.0380171947, 0.0225269515, 0.0181588326, 0.0366643481, -0.0526924878, 0.00522365095, -0.0477036834, -0.0478570908, -0.0767388641, 0.0786209404], [0.0370763056, 0.0343606807, 0.0301555432, -0.0792000591, -0.0198773611, -0.0733454898, 0.038578175, 0.0340834148, -0.0409992114, -0.0122760432, -0.0736905932, -0.0151225729, 0.0398568176, -0.0365564935, -0.0729875639, 0.0200157817, -0.0215437263, 0.0663879141, -0.0213697664, 0.0328755975, -0.0193987619, 0.014820057, 0.0371741652, -0.0636163354, -0.0778779685, 0.00766482623, 0.00543630868, -0.0749046877, -0.0383258946, 0.0762041435, 0.0773844644, 0.00798396393, 0.0806042477, -0.0702356398, -0.0476501398, 0.0123102246, 0.0761402175, -0.0334941037, -0.0778979063, -0.0196140055, -0.0368021876, -0.0300008282, -0.00308081065, 0.0461836308, -0.0295748916, -0.0151718007, 0.0491675474, 0.0634081513, -0.0614532456, 5.551270e-02, -0.0511184596, 0.0617158748, -0.013427753, 0.0342559144, 0.0315487795, 0.0694255158, 0.0343882479, 0.0595574901, -0.0466938913, -0.0754252076, -6.519240e-02, -0.0718524605, -0.0574294291, 0.057240054, -0.067298308, 0.0201630723, 0.0423037708, -0.0272622854, -0.0199530311, -0.0547967553, 0.0624467656, 0.0701540709, -0.0134248529, -0.0746072531, 0.026707653, -7.449310e-03, -0.0678308085, 0.0444393232, 6.755160e-02, 0.0374663211, -4.180870e-02, -0.0543397814, -0.00421090657, -0.0136036361, -0.0207627341, -0.0536323339, -0.0757114962, -0.0323143154, 0.0591146052, 0.0380403288, -0.0419315062, -0.0790164322, 0.0761343836, 0.0573921874, 0.0166625939, 0.0385745242, 0.042959027, 5.864920e-02, -0.0168504883, 0.0644888207, -0.0237992983, 0.0356396697, 0.0417894162, -0.0741673484, 0.0282415282, 0.00907638482, 0.0623286478, -0.0735433399, -0.0801441892, -0.0188539885, -0.0739623308, 0.049938567, 0.012882093, -0.00996572896, -0.0029251948, 0.0405956581, 0.0184645168, -0.0212149136, -0.0377311409, 0.0530909449, 0.0373641513, 0.0332671851, 0.0309247989, 0.0667576343, 0.052075278, 8.107050e-02, -0.0278914794, -0.0298619308], [0.0569047593, 0.00682158954, 0.0239451434, 0.0828656107, -0.0101584913, -0.0698634088, 0.0446814783, 0.0046800524, -0.0253198687, 0.0252481066, -0.0683970302, -0.0794274136, 0.00345352921, 0.0198269542, -0.0455108024, -0.0064665419, 0.0529903024, 7.200860e-02, -0.0190781821, -0.0705263168, 0.0590184107, 0.0335551314, 0.0672027096, 0.0747239441, -0.0727203935, -0.0809413046, 0.0775496289, 0.0882904306, -0.0798165053, 0.062819913, -0.0381497219, -0.0381086767, 0.0737907886, 0.00684530288, -0.043629393, 0.0441856124, -0.0399210975, 0.00902639888, -0.0359773375, -0.00943190139, -0.0295130406, -0.0118219173, 0.0541160218, -0.01161227, 0.0601179339, 0.0548996814, -0.00238060975, -0.0826931149, -0.0539331771, -0.0579999723, -0.0405640639, -0.0550161526, -0.0160376579, 0.0181814693, -0.0490198918, -0.0454490185, 0.0141883073, 0.0422151349, 0.0163248349, 0.0230460651, -0.00541537674, -0.0131290015, -0.0537258424, 0.0406581573, -0.0400501862, 0.0103971185, -0.060592752, -0.0714824274, 0.0513655432, -8.032370e-03, -0.0208245721, -0.070626229, -0.0168180875, 0.010958083, 2.308330e-02, -0.0527862534, -0.0178651288, -0.0471713506, -0.0483636297, -0.0451242849, -0.043157924, 0.00301198615, 0.016331831, 0.021881735, -0.00565604866, -0.0849114581, 0.0509583354, -0.0304984581, 4.826330e-04, -0.0691314414, 0.00919882674, 0.0824184119, -0.0174662694, 0.0745777935, -0.0576584339, -0.0240236484, -0.0553193092, -0.0387050286, 0.0596771948, 0.0156880766, -0.0812064185, -0.0128489546, -0.0152375819, 0.0241954159, 0.0115921563, 0.028365355, 2.881220e-02, -0.0140137793, -0.0716271624, -0.0260254703, 0.0883429944, 0.0388922319, 0.069076173, -0.0244089682, -0.0230844337, 0.0630454496, -0.00908091664, -0.0192954056, 0.0186118651, 0.0295527652, -0.0365336761, -0.0658096373, -0.0575282164, -0.0151421046, -0.04914134, 0.00123264885, 0.0754132122, 0.0285424832], [0.0421166569, 0.053017702, 0.0135056498, 0.0465955697, 0.0552326292, 0.00570791541, -0.0248227715, 0.00275057368, 0.00391462911, 0.00891832448, 0.0872676596, -3.789380e-02, -0.0726758391, 0.0728118643, 0.0235944614, 0.0135627463, 0.0327257179, -0.0292377025, 0.00616972055, 0.0702860653, -0.0311083198, -0.0348471664, 0.0014595273, -0.0667526796, -0.0679735839, 0.0497467145, -0.0364987887, 0.0776739418, -0.0157891568, 0.0779267624, 0.05181811, -0.0329768285, -0.012790814, -0.0611553751, 0.0141052343, -0.0067485855, -0.0658346936, 0.0517045707, -0.00690120505, 0.0894622952, -8.074480e-03, -2.859510e-03, -0.032389544, 0.088814944, -0.00833718944, 0.0239896793, -0.0420081057, -0.0274968743, -0.056602329, 0.0338492244, 0.00855410937, 0.0184706114, -0.0335448645, 0.0261624381, 0.0452497862, 0.0123473387, -0.00318824966, 0.0492540486, -0.0514087975, 0.0573290437, -7.785820e-02, 0.0198777076, -0.0685723647, 0.028110018, 0.0344774723, 0.00842217542, 0.0322279222, 0.01806481, -0.0216809083, -0.0261100028, -0.0210859794, 0.0735332221, 0.0335876308, 0.0818436369, -0.0330695584, 0.0728529841, 0.00314409053, 0.065014787, 0.0489407182, 0.0164145604, -0.00335586164, 0.0902235582, 0.0437920168, 7.747150e-03, -0.0748735517, 0.0188468751, 0.0667960122, 0.0670222789, -0.0297383126, -0.0639666915, 0.0617715642, 0.0773259625, -0.0884434506, 9.00410464E-4, -0.051308386, -0.0635254085, -0.0015828358, -0.0453554243, 0.00111773319, -0.0573358424, 0.0372850709, 0.0253243502, -0.0503085516, -0.00432284689, 0.0476868711, -0.0849761814, 0.0103297913, -0.0048137852, -0.034467455, -0.0794795752, 0.0821402445, -0.0765586048, 0.0234898757, -0.04699146, 0.00740067381, 0.0449608751, 0.074516654, -0.0180512592, -7.850410e-02, 0.0247151516, -0.0710428432, -0.0474769026, 0.0418818444, -0.0129418252, 0.0155076273, 0.0711923539, -0.0591407046, -0.0484927669], [0.0255849976, 0.00337519147, -0.0634963736, 0.0911817923, 0.00344465883, 0.0716868117, -0.0665480047, 1.1415562E-4, 0.0340036787, -0.0225890763, -0.0451741256, -0.0181449242, -0.0224237684, 0.0553865768, 0.0776541307, 0.0715662912, -0.0540995412, -0.018519178, -0.0573996343, -0.0135849658, 0.0689274594, 0.0240666959, -0.0616241097, -0.0517599247, 0.0445460714, -0.0180910118, -0.0307206903, 0.0723855793, -0.0705905557, -6.60938967E-4, -0.00521408813, 0.0524444878, -0.0567098558, -0.0590783246, -0.00196760497, 0.0382215343, -0.0314661413, -0.0415668264, -0.0819773375, -0.0158956777, 0.0368178487, -0.0702241659, 0.0452229232, 0.0810632855, 0.0171772782, -0.0157463737, 0.0311040878, -0.0472543575, -0.0176500529, -0.0360759199, -0.08411026, -0.00679350737, -0.0142796468, -0.00339461933, -0.018584013, -0.0023426991, -0.021611983, -0.0489765108, 0.0433447957, -0.0602163225, 0.018035531, 0.0631684661, 0.0634796098, -0.031939894, -0.0508173853, -0.0834767073, 0.0729700774, 0.0531815663, 0.00951895583, -0.00791622512, 6.263040e-03, 0.0639971942, -0.0648216456, 0.083803907, 0.0249340553, -0.0678414628, -0.0557337403, 0.0334822685, 0.0536297895, 0.0109030372, 0.0628567785, -0.00348580279, -0.0131456684, 0.0435846485, 0.0487897657, -0.0251886789, 0.0109399687, -0.027086867, -0.082320325, 0.0644599497, -0.0522284769, 0.0128595261, -0.0146900117, -6.217480e-02, -0.0379631855, 0.0518333167, 0.0335107297, -0.0834148824, 0.0436348021, 0.0239482801, -0.06443578, 0.0446308069, 0.0364016816, -0.0668227374, 0.0441893451, -0.0580918565, 5.755500e-02, -0.0337790586, -0.0270760618, 0.0398584045, 0.0858363807, 0.0330326483, 0.0542929247, 0.0748237744, 0.0130591532, -0.0107640121, 0.0492511466, 0.0530619696, 0.0756613165, 0.0383422039, 0.0408858024, -0.0573053472, 0.0307634175, 0.0321744233, -0.0765866935, -0.0731691718, -0.0696365535, -0.0371970981], [-0.066758506, -0.0652433559, 0.0327945165, 0.0308872703, 0.088462077, 0.0570966117, 0.0249315389, 0.060604725, -0.00708698155, -0.0869613439, -0.0501733124, 0.0908905565, 0.0867824777, -0.00344090164, -8.23073787E-4, -0.0119520491, -0.0559074767, 0.00295949471, -0.0459540635, 0.0672068223, -0.0141097633, -0.0251158345, -0.0725723803, -0.0340989977, -0.0157021377, 0.0618802905, -0.0542549491, -0.0333681703, -0.0634900182, -0.0355034322, -8.041640e-02, 0.0513239726, 0.0395352952, 2.82319816E-4, 0.0743701756, 7.726110e-02, -0.0701192617, 0.035706114, 0.0193428043, 0.100655653, -0.0130544538, -0.0161312297, -0.0322800353, -0.00489132944, -0.036817003, 0.0490802713, -0.006053044, -0.0181177426, 0.021461647, -0.044999361, -0.0939787104, 0.0536167733, 0.0376078151, 0.0320884921, -0.0286447909, -0.00199660286, -0.00938647426, 0.0237045418, -0.0477733053, 0.0299028102, 0.0824803635, 0.0149328643, -0.0368870944, -0.0635575876, 0.0294878911, -0.112627156, 0.0210280251, -0.0347724333, 0.0584390201, -0.0161029901, 0.0568754822, 0.0405618325, 0.0322489068, 0.0716527626, 0.0555113666, 0.041221898, 0.0474653952, -0.0495539419, 0.0445145704, 0.089804247, -0.00169367879, 0.0152488882, -0.00625025807, -0.0299789347, 0.0736522228, -0.0656737536, -0.0237019323, 0.0592230707, 0.00123817287, -0.0294170454, 0.0457556657, 0.0525980219, 0.0369703844, 0.0659376457, -0.0599248968, 0.0235394966, 0.0683615059, 0.00474723289, 0.0326614864, -0.0202416871, 0.0370688066, -0.0238118675, 0.0273508821, -0.0327143669, 0.0793603733, -0.0537799522, -4.599600e-02, 0.0476947315, -0.0654341355, -0.0277782548, 0.0617638864, 0.0772860274, 0.0126148881, -0.0528681874, -0.0578930713, -0.0598246343, 0.0593650043, -0.0388389416, -0.0764860958, -0.0330718979, -0.0168366414, -0.0726063475, -0.0287496019, 7.650650e-03, 0.0501880385, -0.0607324131, 0.0485533848, -0.040175572], [-0.00259340066, -0.0286881886, -0.0785835683, -0.0184342321, 0.102815248, -0.0411497019, -0.0571950339, 0.0743973255, 0.0732896179, -2.92344368E-4, 0.03886142, 0.00298947445, 0.00803483929, 0.055730205, -0.0285240691, 0.0422107354, -0.0223975219, 0.0471576303, -0.0732959509, -0.00524559384, 0.0500891656, -0.00523327198, -0.0678535849, 0.00973800104, 0.0233617853, 0.0740251169, 0.0184115693, 0.0514296293, -0.0149180489, 0.0395410843, -0.0709987134, 0.0409146398, 0.0885748043, -1.176780e-03, -0.0294053555, 0.0705835223, -0.0510158464, 0.0129055912, 0.0229633655, 0.111867733, 0.0167170707, 0.0814686194, -0.0546369925, 0.0188452564, -0.0685034394, -0.0455910601, -0.0329337195, 0.0586936437, 0.0505882278, 0.0621563158, 0.060858611, 0.0322800763, -0.0521863773, -0.0611374192, -0.051084809, -7.796140e-02, -0.00322046457, -0.0739656687, 0.0578158535, -5.811810e-02, -0.0197692644, -0.00250647939, -0.0313638449, -0.0737963319, 0.00112013111, -3.957980e-02, 6.910080e-02, -0.0593954846, 0.0478903949, -0.0275106747, -0.0498877168, 6.964220e-02, -0.0700573847, -0.0313006751, 0.0540057793, 0.0912161991, -0.0802355185, -0.070143953, 0.056434609, 0.0859380513, 0.0325261615, -3.545850e-02, -0.0803575069, 0.039838247, 0.0491465069, 0.0560622513, 6.102510e-02, -0.0564354919, 0.0152225876, 0.056208387, 0.0695620105, -0.015506994, 0.0549660362, -0.0352114141, 0.0282321721, 0.0683688372, -0.00845497567, 0.0163322836, -0.0550231449, 0.0385437496, -0.0846540555, -0.0829573571, -0.0249078721, -0.0529764928, 0.066623196, 0.092091076, 0.0366873816, -0.00435663667, -0.0574546792, 0.0624351911, 0.0671482235, -0.0524103791, -0.0735910162, -0.0101075638, 0.00140232651, -0.0413978659, -8.692040e-02, -0.0449019372, -0.0581922829, -0.00236907299, 0.0457567535, -0.016971115, 0.00856516417, 0.0252335332, -0.0653924122, 0.0190226249, -0.0910806209, 0.0354978107], [0.0438680649, -0.0675268173, 0.0711227953, -0.0801223665, 0.0270930026, -0.0791532918, -8.18174565E-4, 0.0621830038, -0.0135572013, 0.0202588495, -0.0228698757, 0.057157971, -0.0318068825, 0.0537965372, 0.0576863624, 0.042108424, -0.0668153688, 0.0541833192, -0.0933279097, 0.0666823089, -0.0176339429, -0.0845534875, 0.0148694767, -0.0355409794, 0.0185750313, 0.020972902, 0.0098437136, 6.356950e-02, -0.0472244583, -0.0258270465, -0.00205063378, -0.0800161585, -0.0199234933, 0.00913231168, -0.0221080594, 0.0077987141, -0.0267637502, -0.0324440524, 5.37340937E-4, 0.104246497, -0.0011293086, -5.584690e-02, 0.0368349962, 0.0312460866, -0.0738889128, -0.0595899299, 0.0291673187, -0.0311873164, 0.108261168, 0.0732837841, 0.00702321157, 0.021822758, 0.0457935892, -0.0193614587, 0.0531610884, -0.062010847, 0.0314739943, -0.0346763059, -0.0142786857, -0.0651223287, 0.0503033772, -0.0406665094, -0.0626210272, 0.00835853442, -0.0106290635, -0.0903071537, -0.023070164, -0.00224335515, -0.0447004437, -0.0673472285, -0.0584284216, 0.103655666, -0.0116172219, -0.0459624492, -0.00495799212, -0.054328274, 0.0419913977, -0.0133461133, -0.0496413149, 0.0920848697, 0.0179883894, 0.0314420797, -0.0335115381, -0.07448183, 0.0252948962, 0.0723278373, 0.111120641, 0.0216580201, -0.0856790543, -0.0369054265, 0.0625740513, -0.0745433569, 0.0212573409, 0.0313700661, -0.0572936609, 0.00569608575, -0.0649433061, 0.024828909, 0.0427391678, 0.062586166, 7.493260e-02, -0.0457758717, 0.00623956323, -0.0272665247, -1.925210e-02, 4.012020e-02, 0.0509954467, 8.76024598E-4, 0.0161785819, -0.00829515792, 0.00663812644, 0.0789933577, -0.0376507714, 0.00834537949, -0.0606204905, 0.0840631499, -0.0694434047, 0.0219271649, 0.0410337113, 0.0100746872, -0.0537286922, -0.0198383722, -6.466300e-02, 0.0276045967, -0.0666683763, -0.0187527128, -0.0510413423, -0.0356380343], [0.0580787547, -0.0766916201, 0.0530457906, 0.019045217, 0.013091757, 0.0607422926, -0.0652435571, 0.0063074883, -0.0241745971, -0.0548117273, -0.0239511337, -0.00528844306, 0.00431083608, 0.00278223446, 0.015139685, -0.0371319205, -0.0637157336, -0.0444515869, -0.0790037512, 0.10491015, 1.961280e-02, -0.107221678, 0.0770907328, -0.0519785061, -1.24833787E-5, -0.0148491701, 0.0456609502, -0.00717462506, -0.00646317704, -0.054861106, 6.516720e-02, 0.0409276634, 0.038921278, -0.0349962637, -0.0661206543, -0.0131113539, -0.0202647429, -0.0568820238, -0.031857118, 0.0233498588, -0.0458195545, 0.00535919517, 0.0499173515, 0.0423230156, 0.00423789397, -0.00368540618, 0.00959413778, 0.095496416, 0.0669532865, -0.00307685207, 0.0558928326, 0.0261835437, 0.0921912714, 0.0347576179, 0.0192762669, 0.0204788763, -0.105778061, 0.0543389022, 0.0420084298, 0.0356126539, -0.00668409606, -0.0156575199, -7.515090e-02, 0.0103107598, -0.0900072529, 0.0392475799, 0.0119979223, 0.0512333252, 0.0361280516, -7.57934409E-4, -0.0740258917, 0.0781094655, 0.0508490726, 0.00881300866, 0.0532117076, 0.100664072, -0.113573961, 0.0585669354, -0.0872635543, 0.0777240917, 0.0131954877, -0.0450011715, -0.0953404679, -0.0449113026, 0.0263925027, 0.0919596627, 0.0411232859, 0.0154429274, -0.00223272596, -0.0171751771, -0.0217437875, -0.0948371365, -0.0284592099, 0.0284672175, -0.0627050101, -0.0430396572, -0.0410812348, -3.110270e-04, -0.0782398059, 0.0499337688, -0.0372082256, 0.0266258847, 0.0531109944, -0.056089256, -0.0849435701, -0.0238560569, 0.0536467955, 0.0189029947, 0.0247597154, 0.0913776084, -0.0771054849, -0.0442976691, 0.00771403312, 0.0409571975, -4.003530e-02, 0.0409835204, 0.0669016391, -0.0457390249, 3.753720e-02, -0.0678795204, -0.0641654059, 0.0705703794, 0.0629018843, -0.0859208405, 0.0294151623, -0.0678677484, -0.0347354226, -0.0241897963], [0.0536128953, -0.0796544477, -0.0509510599, 0.0240717232, -0.026260931, 0.0215568431, 0.0185131598, -0.072383143, 0.0839735344, -0.0711665601, 0.0119205713, 0.0619197897, 0.0467186458, -0.121630698, 0.0910735726, 0.0401838124, -0.0042869295, -0.0470254049, -0.05504274, 0.101390392, 7.098880e-02, -0.116308428, -0.031015778, 3.861010e-02, -0.0768953711, 0.0353216492, 0.0567926466, 0.0536763184, 0.0270932224, -0.00515724812, 0.0478263125, -0.0288924612, 0.00895342696, -0.110672139, -0.0426140912, 0.0880305692, -0.0798969641, -0.116756812, 0.0357939266, 0.0379858837, -0.0173753332, 0.046765957, 0.0292721763, 0.0574681424, -0.0183582585, -0.0931608528, 0.0533349849, 0.11065308, 0.0714214146, 0.1076371, 0.0528760105, 0.0042871288, 0.072506994, 0.0265084822, -0.0242446177, 0.024239473, -0.00278413715, 0.053910654, -0.0585950539, -0.0598639771, 0.0383808315, -0.0893842279, -0.0440966412, 0.018935496, -8.526510e-02, -0.00294636958, 0.140450522, -0.0547235347, 0.00715627568, 0.089462608, -0.101079099, 0.0761165768, 0.0730361938, 0.022087954, -1.796190e-02, 0.0560464635, -0.1283077, -0.0477930866, -0.0359680243, 0.0332250223, -0.030283371, 0.0542685911, -0.0201589596, -0.0138235344, -0.024594672, 0.00599865802, -0.020467829, 0.0066836006, 0.0146330455, -0.0398976058, 0.0870434567, -0.0241414662, -0.0108954394, 7.950520e-03, -0.00325430301, 0.0072287186, -0.0616559237, 0.0737918168, -0.0171931144, 2.290640e-03, 0.0905983224, 0.0299520679, -0.0697413608, -0.0306169409, 0.0453943871, 0.0927017629, 0.14334625, -0.083744809, 0.0699487105, -0.0341732688, 0.0024059047, -0.012105979, 0.0182672124, -0.0243477412, 0.0845176503, 0.00409534387, 0.0930280387, -0.0305508804, 0.00855974201, 0.0501791239, 0.014256998, -0.00742126069, 0.0614457354, -0.113428488, 0.0555265509, -0.0995420217, 0.0099513689, 0.0574361347], [-0.0173200648, -0.0465973802, -0.0413741879, -0.0634908304, 0.0505690724, -0.0414835475, 0.0159524437, -0.0804462954, -0.0280981921, -0.160150975, 0.0134577909, 0.0541542657, -0.0549712256, 0.0248182677, 0.0222262591, 0.082327649, 0.0470534265, 6.089480e-02, -0.0751227885, 9.43977269E-4, -3.707650e-02, 0.0321494564, -0.0484392643, 0.0526750945, 5.714790e-02, -0.0669004247, 0.0794606134, -0.0827414244, -0.0400391072, 0.0244811308, -0.0449466184, 0.0131128216, -0.0872896611, 0.0552485436, 0.0670477152, 0.0975754931, -7.161000e-02, -0.0767778158, 0.02865256, -0.0232100207, 0.0614274368, 0.0319907106, 0.0364730656, 0.0663218349, 0.00191376533, 4.936050e-02, 0.111786887, 0.00201406819, -0.00966903474, 0.0533771031, 0.0850764811, 0.0136821549, 0.0920218601, -0.0288672782, -0.0655803457, -0.040898867, -0.100404486, -0.017419111, 0.00132666354, -3.053590e-02, 0.0366203226, -0.0538759418, -0.0507206433, -6.5732852E-4, -0.0633803457, 0.0284539144, 0.040877305, 0.08243002, 0.0483574383, 0.0285155736, -0.0150582669, 0.139013678, 0.056454543, 0.0493963473, 0.0806967988, -0.046483811, -0.064002648, -0.0471534394, -0.0241533648, 0.127595082, -0.0693140923, -0.0476666614, 0.0605757162, 0.0950599462, 0.0689166263, -0.0277668796, -0.0168656465, 0.0505706072, -0.0460584424, -0.0476179756, -0.0636037141, -0.0716904402, 0.0719229653, 0.027621951, 0.0831582248, -1.543800e-01, 3.54416436E-4, -0.00675285793, -0.0761000589, -0.0524459668, -0.0178726409, 0.0100961812, -0.050095953, -0.0763114616, -0.0837416052, -0.037766356, 0.139764071, 0.00972184446, -9.034540e-03, -0.0455694459, -0.0395316817, 0.0165177081, -0.0309424233, -0.00180419208, 0.00358653022, 0.115808621, -0.0501794182, 0.0197248533, 0.0376579277, 0.0461784825, 0.038531106, -2.188780e-03, 0.0668910146, -0.155542299, 0.0373668075, -0.0898622274, -0.0851534754, 0.0449662544], [0.0686828047, 0.0338147171, -0.0405767225, -0.00216646912, 0.114601508, -0.0273603182, -0.060528934, 0.0135881938, -0.0700539648, -0.0534086451, -0.0349498577, -0.0385229401, 0.032754302, -0.0460765846, 0.0886934847, 0.119135946, 0.0725671723, -0.0543507077, -0.0258296672, 0.0154045904, 8.602000e-02, -0.0673895553, 0.04443492, 0.00156324054, 0.0359306559, -4.563660e-02, -0.0299161412, 0.0521600656, 0.0618968606, 0.0318942107, 0.0505400598, 0.00935249403, -0.0217015035, -0.0140024591, -0.0309723429, 0.0383851528, 4.5596066E-4, -0.0420794711, 0.0634993166, 0.0245375987, -0.0533718765, 0.0874663069, -0.0954629555, 0.073585324, 0.00257437699, -0.0167624149, 0.11859145, -0.00937801226, -0.0541092381, 0.0264719967, -0.0267610345, -0.0142841553, 0.123941578, 0.104326084, -0.0420663618, 0.0266418606, -0.0820657238, 0.0180054102, 0.0038357866, 0.0817776694, -0.0545247868, -0.0770685076, -0.00634457124, -0.0555474274, -0.00343861873, 0.0836716815, 0.00555003714, 6.897290e-02, -0.0530757159, 0.00312626129, -0.101988561, 0.115334183, -0.00961822364, 0.0732604191, 0.0569129698, 0.0146313068, 7.2382146E-4, 0.0967441946, -0.0357774235, 0.0256015938, -0.0936801285, -0.00834160578, 0.0654545426, 0.0658193082, -0.00270724343, 0.0294181164, -0.0624551252, 0.0172629915, -0.0721480176, 0.0439793803, -0.0759926885, -0.0448709503, 0.0503740907, -6.375080e-02, 0.0873910859, -0.0295596961, -5.224010e-02, 0.0616279095, 0.0142991589, 0.0499793068, 0.100185856, -0.0426699631, 0.0949417725, -0.0621195212, -0.0461557172, 0.0985120534, 0.120490119, -0.0260407757, -0.0716597587, 0.0469059944, 0.064120926, -0.00519927638, 0.0196751263, -0.0324443616, 0.094398953, -0.0341430195, 0.109849304, 0.0526102856, 0.0701131597, 2.788980e-02, 0.0426359624, 0.0756201074, 0.0264040828, -0.0518191271, -0.0536246933, -0.0246189516, 0.0361563824, 0.00317846192], [-0.0840769335, -0.0123670837, 0.0341448151, -0.0176397599, 0.105885796, 0.0226706285, 0.0391192622, 0.029154839, -0.0392868742, -0.0782614201, 0.00528236106, -3.08172428E-4, -0.00953749194, -0.120572157, -0.0394992828, -0.0226174183, -0.0449318886, 0.0119319241, -0.0743715614, -0.07793013, 0.0493933372, 0.0298899841, 0.0253052674, -0.0211864058, -0.0214843228, -0.05147589, -0.0730592608, -0.0604988895, -0.0629926696, -0.0953763425, 0.0618656091, -0.0244666599, -0.0404233634, 0.00193671929, -0.0400091261, -0.0331216902, -0.00592670636, 4.011690e-02, -0.0571609549, 0.0608979352, -0.0211405214, 0.0867763832, -0.101720601, 0.013250377, -0.0413300171, -0.0548840053, 0.139723241, -0.00611608196, -0.0417086259, -0.060514003, 0.0339511745, -0.104584947, -0.0208354313, 0.0794078782, 0.0287897587, -0.0405166261, 0.00801710225, -0.0116634099, 0.00495740771, -0.0453189388, 0.0377605557, -0.088856712, -0.025686495, 0.0459819809, -0.00943187437, -0.0393703394, -0.0200395398, -0.0101179546, 0.0393958203, -0.0141469063, 0.0483722538, 0.0432119258, -0.0470943451, 0.0908426791, 0.041424945, 0.0745632127, -0.0650023073, 0.026592195, 6.835560e-02, 0.0579849631, 0.0184887126, 0.0848323479, 0.0189057644, -0.033389274, 0.064473927, -0.0179572329, -0.114040658, 0.0557902828, -0.0045474628, 0.011063369, -0.0420076326, -0.0926405042, 0.0024272853, -0.0132072158, -0.0251894109, -0.147180244, -0.0940173715, -0.0484387949, -0.0331392661, 0.0182123184, 0.0727856457, 0.0110444436, 0.0927149578, 0.00932865776, 0.0904121175, 1.365420e-01, 0.0621192567, -0.0240809415, 0.0479277596, 0.0202179644, 0.0584494919, 6.764780e-02, -0.00778878527, 0.0105794948, 0.0863198489, 0.0442280322, -0.0303257443, 0.0749786049, -0.0442781225, 0.00726709515, 0.0675049499, -0.0410852693, 0.0487885177, -0.0207331683, 0.00113441842, 0.103565805, -0.0689972267, -0.0318091922], [-0.0175992548, 0.00258484366, 0.0723944232, 0.0602365434, 0.0443218499, 0.106670097, 0.0454038158, 0.0783539861, 0.00545871025, -0.0874165669, 0.0259476937, -0.00451859785, 0.0134842284, 0.0245096274, -0.0215881169, -0.00952700898, -0.0302648507, 0.0215333626, -0.00614628522, 0.0225334987, 0.0431095436, 0.0268756133, -0.0132104149, -0.035023611, 0.0635313689, -0.0393480249, 0.0110573256, 6.348560e-02, 0.0685620084, -0.0201154482, -0.00933003798, -0.0024288334, 3.645390e-02, 0.04151145, -0.0216083396, -0.0169312954, 0.0296239294, -0.00601590658, -0.00969772413, -3.701570e-02, -0.0822529792, 0.0428566113, 0.0580774955, -0.0321510136, -0.00338412588, 0.0372659415, 0.0307169799, 0.0253584497, -0.0821668729, 0.00432781596, 5.178770e-02, -0.0634130388, 0.101888664, 0.131350011, -0.00886328332, -0.0233715959, -0.0356795564, 0.0375197604, 0.0551108457, 0.0924739689, -0.0144486958, 0.017120637, 0.0585412681, 0.0879408866, 0.0177547392, -0.0585049093, -0.0294855777, 0.00495487638, -0.0924869776, 0.0582231097, -0.0310000014, -0.0673629269, 0.0592452474, -0.0660918579, 0.0111044217, 4.691590e-02, -0.0241583418, 0.0122771226, 0.0428447314, 0.0165678244, -0.02772245, -0.0189487915, 0.0702261254, -0.0103685828, 0.0732962787, 0.0290071163, -0.0475522727, -0.056136325, -0.0186705012, 3.74323135E-4, 0.0521369539, -0.016156584, 0.0151830027, 0.0372539051, -0.00224207016, 0.00820076745, -0.0336448886, 0.044367481, 0.00682647247, 1.070740e-01, 0.0281077661, -0.0306416396, 0.015505204, -0.0489440411, 0.100376114, 0.102792449, -0.0183944292, -0.0211516172, 0.0533516891, -0.0205214024, -0.0851728841, 0.0333045162, -0.0606955551, -0.0456322134, -0.0113830185, -0.0412521698, -0.01641481, -0.0345613882, 0.044826135, -0.0731418654, -0.0214266665, 0.0634322166, -0.0497985668, -0.0322063752, 0.0152882123, 0.0569880679, -0.00310429372, -0.0577023849], [-0.0154852727, -0.00264781062, -0.0202912092, 0.0120545505, -0.0520940162, -0.0432116948, 0.0280076396, 0.0653180629, -0.0678134337, -0.0395220593, 0.0708517432, 0.0889011099, -0.0706013814, -0.0640210584, -0.0203760117, 0.00588085363, -0.0283807516, -0.0722694248, -0.0724947825, -0.0485420749, -0.0463008881, 0.0534005724, 0.0539039746, -0.0192233697, -5.380540e-02, 0.104877718, -0.0301165152, -0.00595959881, -0.0160403568, 0.0595247038, -0.0782096907, -0.0658794791, 0.0621548183, -0.00653330609, -0.0249097534, -0.0578899793, 0.0708145946, 0.0805784389, 0.0637257397, -0.0284990929, 0.0668001398, -0.00355814653, -0.0678674876, 0.0106063113, -0.0356385484, -7.1965449E-4, 8.07421165E-4, 5.215360e-02, 0.0109145828, 0.0305401869, -0.0301502831, -0.0135101005, -0.0524028763, 0.0707523897, 0.0232281163, -0.0415861495, -0.032739792, -0.0425319113, 0.0214086585, 0.046122998, 0.0646708682, -0.00603222288, 0.0521898568, 0.0586335622, -0.0432056263, 0.02650268, -0.00433820626, 0.0455614924, 0.0147204883, 0.0542970337, -0.0390653536, 0.0678938627, -0.0270808674, 0.0358524434, -0.019888144, -0.0386831462, -0.0480588786, -0.0593967959, -0.0204292983, 0.0216250941, -0.0588924699, 0.0347178318, -0.0758221745, -0.0739419684, 0.0714559332, -0.00907784794, -0.0928588211, 0.0754370391, 0.0705589727, 0.0597056709, 0.0225384384, 0.00263334136, 0.0873896479, -0.0670382455, -3.251140e-02, -0.0814053639, 0.00960553437, 0.0307246745, 0.0862385482, 0.137268454, -0.0370660238, 0.0113872616, 0.0988564193, -0.0643090382, 0.0641542226, 5.851540e-02, 0.0821485444, -0.0239471719, 0.0429232344, -0.0176017247, -0.0470848531, -0.0417189412, 0.00421756832, 0.0431263484, -0.0446382388, 0.00406584702, -0.0549974106, -0.0228518136, 0.0388484672, -0.0621681511, -0.0517036431, -0.0602663122, 0.0360222086, -0.038369637, 0.0443161577, 0.0673050284, 5.059710e-02, 0.068397738], [0.0310408715, -0.0563107803, -0.0123083098, -0.0728105828, 0.0313355848, -0.0697496682, -0.0464375056, 0.0401787087, -0.0331717394, 0.0462492108, -0.0506512858, 0.0313384198, 0.00193117105, -0.0236965921, -0.0085377628, 0.062995486, -0.0117132608, -0.00711321738, -0.00716087874, -0.0469784141, -0.0221565254, 0.0356195793, 0.024131408, 0.00247355201, 0.0218443889, 0.0237216335, 0.0352191962, 0.0771871284, 0.00935500301, 0.023750741, -0.0680256709, -0.0814845561, 0.0290900618, 0.00868289638, -0.0720521584, -0.0183137432, 0.0764574856, -0.0418404378, -0.0555190668, -0.00219002855, 0.0965113193, 2.427190e-02, -0.0604808144, -0.0408432335, -0.0384985656, -0.0241757203, -0.0323045067, -0.0709424838, 0.0289144702, 0.0388301201, 0.0238289069, 0.0625896155, 0.0562698245, -0.0600299649, 0.0962722375, -0.0433668531, -0.028872963, -0.00426701689, -0.042770952, 0.00841843709, -0.0465520397, 0.0288453531, -3.536590e-02, 0.0556473695, -0.0397769287, 0.0170908682, 0.0337633491, -0.0335569233, 0.0322163589, 0.0508480109, -0.0288991425, 0.0697391406, -0.0430811271, 8.843100e-02, 0.00381448143, 0.0213083345, -0.00270980364, 0.0303328633, 0.0459225439, 0.0336339399, 0.0685714409, -0.0389322564, -0.1082763, -0.0603408404, -0.00557404896, -0.0263310447, 0.00587968528, 0.0130859697, 0.0853741914, 0.00587783335, 0.0636217743, 0.0303540807, 0.0455230586, 0.0723129883, -0.0577366389, -0.0736808255, -0.0314067528, 0.05929894, 0.00281971972, 0.0553876981, 5.558700e-02, -0.0130478572, 0.036925707, -0.00241838326, -0.0487832837, -0.0160234272, 0.0172530264, 0.0455220826, -0.0313070826, -0.0221113451, 0.0642910525, 0.0638488531, -0.051697284, 0.028485734, -0.0286968481, -0.0286533628, 0.0675361902, -0.04918807, 0.0129299611, -0.0371702053, 0.00563519867, -0.0291757304, -0.0779652819, 0.055295229, 0.0481175743, -0.0818740501, 2.015930e-02, -0.0492907837], [-0.0637800172, -0.0208621193, 0.0481139868, 0.0200515836, -0.0180613846, -0.0817243233, 0.0402338654, 0.0796819478, -0.0499698557, 0.0951077416, 0.00867676269, 0.00152012473, 0.0334060416, -0.0468457378, -0.0275853947, 0.0281192046, 0.0730502754, 0.0337745175, 0.0127146607, -0.0528327413, 0.0250090957, -0.064894177, 0.0764577984, 0.0641722679, 0.0457080938, 9.722370e-03, 0.00188975874, -0.0406621769, -0.0393632501, 0.0474667922, -0.054086484, 0.0422808416, 0.0985513702, -0.00799386855, -0.0246477015, 0.0585247129, -0.0259582102, -0.00103185338, 0.0355656669, 5.715410e-02, -0.00855834782, -0.0620648674, 0.0415310077, -0.0606348701, 0.0170374047, -3.079750e-02, -0.0531197414, -0.0433130413, 0.0758713484, 0.0615435243, -0.0396080203, -0.00674799923, -0.035705775, -0.0135060484, 0.0859874114, -6.744610e-02, -0.00627392577, 0.00304333889, -0.0659820735, 0.0521459058, -3.24743683E-4, 0.00216053147, -0.0666074454, -6.657120e-02, 0.0899721086, -0.0913139135, -0.0432158746, 0.0346470177, 0.013701532, -0.0571655892, -0.0222509243, -0.0470649227, -0.059985131, 0.0122726923, -0.0679845586, -0.033094652, 0.0261188634, -0.016820671, -0.00273658824, 0.0788462981, -0.033879552, 6.696550e-02, -0.0859122723, -0.09125413, 0.0256236587, 0.0379695408, -0.00590909878, 0.0537082441, 0.00174428127, 0.0380534343, -0.025138218, 0.109697163, 0.0867182239, 0.0721813291, 0.0743204504, 0.00990076176, 0.0138110965, -0.0461642556, -0.0675496906, 0.0191169493, -0.0588935837, 0.100770496, -0.0440245345, 0.0224792827, -0.0491347946, 0.0311257932, 0.0736262277, -0.00731120072, 0.0443802699, -0.00558867957, 0.0122587681, -0.0664438307, -0.0325533673, 0.0342047848, -0.0887164473, -0.0265314505, -6.135930e-02, -0.0311758332, 0.108663976, -0.0282577481, -0.0627354458, 0.0483526066, 0.00922060199, 0.00420233281, -3.39651044E-4, -0.0201998521, -0.0460651182, 0.0406571664], [-0.0618688725, 0.0231492724, -0.056460619, 0.0569349378, -0.0103494423, -0.0172748249, 0.0134635344, 0.0325209089, 0.00767211197, 0.0673069879, 0.0329624489, -0.0528193675, -0.00847393926, 0.0176341552, -0.0241072662, 0.0820028111, -0.0775200874, -0.00911451876, 0.0401221439, 0.0256799869, 7.72581086E-4, 0.0378937759, -0.0458595231, 0.00553329475, 0.00420156028, -9.77358577E-5, -0.0230253339, -0.0436084159, -0.0643697903, 0.0302997623, -0.0180971902, -0.0882381424, -0.00314398715, -0.0596808344, 0.0892476887, -0.0428531282, -0.09160126, -0.0072247819, 0.0307126101, -0.0293255523, 0.0792827457, -0.0513711795, -2.992210e-02, 0.0276145898, 0.0274312962, -0.0112000713, 0.0596400537, 0.0173018277, 0.0769289359, -0.00152693968, 0.0767866671, 0.0576819889, 0.0245741289, -0.0157288518, 0.042150937, 0.0455069914, 0.0505883284, -0.0311242305, 0.0564613305, 0.0741701871, -0.00103530206, 0.0328595787, -0.0318116024, 0.0179687981, 0.0288633537, -0.0196364541, 0.0487956926, 0.0358607844, -0.0442024618, 0.0762023553, 0.0580782779, 0.0928777977, 0.0257369224, 0.0945305526, -0.0459880345, 0.0659014732, 0.020799065, 0.0227431618, 0.0554515272, -0.0150369713, 0.0500245839, -0.018339986, -0.00412535481, 0.0399257876, 0.0680104569, 0.0403705053, 0.0308365598, -0.0822469145, 0.0924697145, 5.893140e-02, -0.0112739103, -0.0245517306, 0.0250156224, 0.0139267938, 0.00241030287, 0.0835346281, -0.00219400506, 0.0843392536, -0.00953044742, 0.0166648105, -0.0200817715, -0.0408883207, 0.0477412231, -0.0109319286, -0.0589270107, -0.0642625987, 0.0727670565, 0.0706930458, -0.0505608805, -0.0127432216, 0.0596898459, 0.0216132011, 0.0859061256, -0.0136035876, -7.433360e-02, 0.0297187325, 0.0466471054, -0.042690374, 0.0804099068, 0.0584543049, -0.023292983, -0.0204628017, -0.011176398, 0.0333348103, -0.0338402875, 0.0569523871, -0.0319406725, -0.0419561453], [-0.06367594, 0.0695380047, -0.0521631502, 0.012254646, -0.00525498157, -0.0549269281, -0.00200705393, -0.01371413, -0.0712515637, -0.0129346652, 0.0229033288, -0.00939349178, -0.0624741278, 0.015307866, 0.0650221258, 0.0613934695, -0.0117648151, 0.0616978444, 0.0299731568, 0.0349218771, -0.0440238938, -0.0839321613, 8.33887723E-4, -0.00300525152, 0.00196868414, -0.0523730591, 0.00387493358, 0.0054857689, -0.0611134283, -0.0196820796, -0.0365435928, 0.0065406519, -0.00999691057, -0.0827885642, 0.0265607275, 0.0710341185, -0.0656893849, 0.111387007, 0.0356912725, 0.0468845703, -0.0447983295, -0.0484818481, -0.00404455094, 0.0802698954, 0.0724886134, 0.0265625902, -0.0777939856, -0.0033727244, 0.0436297916, 0.0136190047, 0.0206393339, 0.039052017, -0.085869357, -0.058694195, -0.07128153, 0.0274082683, -0.0604267195, 0.0477789156, 3.883960e-02, -0.0537139475, -0.0902365744, -0.00232334109, -0.0198074225, -0.0403284542, -0.0239860769, -0.0664033517, -0.0109992297, -0.018668009, -0.0338780247, 0.00722168385, 0.0787091627, 0.0945815816, 0.0359939784, -0.0110264523, -0.0215309449, 0.0224870071, 0.0416315608, 0.0422067605, 0.00678400462, 0.0542363152, -0.0488484055, 0.0426715538, 0.0236516781, -8.223970e-02, 0.0553794615, -0.0440041274, -0.0301168635, 0.0602887087, 0.0720742494, -0.0626159683, -0.0115681151, 0.0304583609, 0.00496934587, -0.0647378117, 0.0396515056, 0.0618453398, -0.0602294058, 0.0640009716, -0.0573879667, 0.062877804, 0.0650753676, -0.0305530317, -0.0847267583, 0.0583322085, -0.0658436343, -0.0153103312, -0.0555073805, -0.036352478, -0.0500193536, 0.0190856364, -0.0161453374, 0.0073090638, 0.0383069143, 0.0212982949, -0.0740063936, -0.0142367138, 0.0106022609, 0.0410469435, 0.0486977138, -0.0623586475, 0.016269343, -6.265120e-02, -0.0891597941, 0.0239643697, 0.0752594918, -0.0791578665, -0.0544093251, -0.0357143842], [0.0712035745, -0.016855564, -0.0194864143, -0.0527047776, 0.0597840324, -0.0357847214, -0.0223565698, 0.0599078126, 0.0203312114, 0.0175986551, -0.0213483814, -0.0434065461, 0.0864755064, 0.0427553467, 0.00527733518, 0.071694523, 0.0684299245, -0.00361694326, 0.0207335372, 0.00882673543, -0.0746988729, 0.0498266108, -0.0154418554, 0.0239436645, -4.660040e-03, -0.00528293056, -0.0275030509, 0.00155636226, -0.0135263717, 0.0226634666, -0.0198976584, -0.0415219367, 0.0985486209, -0.0977120399, -0.0451457314, 0.0816321298, -0.0916391164, 0.0390591435, 0.0270506609, -0.0319036171, 0.017781537, -0.0199376717, -0.0368811227, -0.0319638364, -0.0113157406, -0.0178400036, 0.0532420576, 0.0574439503, -0.066099979, -0.0277294181, -0.0191665236, -0.0311389212, 0.0637856647, 0.0444411896, 0.019411806, 0.0214246418, -0.00590272062, -0.0117303617, 4.96580207E-4, -0.0127395028, 0.0315560214, 0.038834285, -0.0538569205, -0.00967290532, 0.00259731291, -0.0323129855, -0.0100628128, 0.0368402898, 0.0429288335, 0.0541747622, 0.054302942, 0.0520593412, -0.041022215, 0.00842904206, -0.0310633238, 8.221000e-03, -0.0292028952, 0.0494234711, 0.0240772609, -0.0174439456, -0.0558815151, -0.0407785252, -0.0842221751, -0.0457808264, -0.060290236, -0.0880378112, -0.045830261, -0.0824092924, -0.0644494444, -0.0219221245, 0.0296425093, 0.0990888774, -0.0797064825, 0.027262276, -0.0185371153, -0.0358530395, 0.041720625, -0.0447521619, -0.0063980422, -0.00215781434, -0.0139195686, 0.00982887484, -0.0658890679, 0.0233984962, 6.92178378E-4, -0.0550574586, -0.0479376689, 0.0798426344, 0.0565017164, -0.081856586, 0.0164357554, -0.053774368, 0.0237645954, 0.025525745, -0.0190575905, 0.047476463, 0.0608786494, -7.71266816E-4, 0.0884079709, 0.0290788822, -0.0912234336, 0.0271246638, -0.0557661317, 0.0596639775, 0.0571697019, -0.0846214964, 0.0132069457, -0.0656803399], [-0.053339161, -0.0548358373, 0.0257878676, -0.0277932845, 7.10661057E-4, -0.0322661474, 0.0294804648, 0.0212395713, -0.0438708514, -0.00707568321, 0.0639604703, -0.0566289909, -0.0595403314, 0.0165696684, 0.0393383205, -0.0545243211, 0.041055005, 0.0303209685, 0.0777431949, -0.0668519884, 0.0511454865, -0.0456812233, 0.00213748822, 0.0673873574, -0.0337917395, -0.0909688472, -0.0488685742, 0.0941363722, 0.0706342906, -0.00637782598, -0.00180932146, -0.0827673971, 0.0831701755, 0.0156245874, -0.0148875313, 0.00511490414, 0.00448300736, -8.03837785E-4, 0.0174382459, 0.0266088042, 0.0529452451, -0.0151205957, 0.0496255569, 0.0517660417, -6.127590e-02, -0.0284607895, 0.0113311224, 0.00467530638, 0.093714185, -0.00434496067, 0.078903392, 0.0339381211, -0.0676350743, 0.0316790342, -0.0390500128, -3.724400e-02, 0.0790868551, -0.0721055791, -2.333600e-02, 0.0315462723, 0.00732446369, 0.0799774155, -0.0701792389, -0.0227072146, 0.0144611439, -0.0375101231, 0.0853299648, 0.05868835, -0.00109996181, -0.0184273385, 0.0600269735, -0.00373456604, 0.0367761478, 0.0436469205, -0.0234061219, -0.0654435232, -0.0714591071, 0.0623162649, -0.0714770853, 0.0283121578, -0.00755642354, 0.0141307777, -0.0172931943, -0.0303998049, 4.484890e-02, -0.0508961789, -0.0422081351, 0.0191463847, -0.0454619937, 0.0491214506, -0.0566274785, 0.0335352309, -0.0402917229, 0.0882868468, 0.00436215289, 0.0596709661, 0.0370492376, -0.0164818726, 0.0162353646, -0.0579554327, -0.013576935, -0.0135874059, 0.0553165115, -0.0800016596, 0.0547943935, -0.0695035159, 0.0737624839, 0.0105162458, 6.720530e-02, -0.084647417, -0.0650667474, -0.0402993858, -0.0764602497, 0.0733929202, -6.745690e-02, -0.0124306316, 0.0145455198, 0.0211304072, 0.0397666171, 0.0583562292, -0.0553305522, -0.08856038, -0.0787073448, 0.0843579173, 0.059236493, 0.0166003164, 0.00163110776, 0.0390775725], [0.0537289418, 0.00302124908, -0.0213145372, 0.0983514562, 0.0350408219, -0.00275305239, -0.0504431725, -0.0206557605, 0.0359058306, -0.0642738938, -0.00452461746, 0.0665422156, 0.0135322316, 0.0687266141, 0.0352867022, 0.0769364312, 0.0274582282, 0.00500275474, 0.0573259778, 0.0243552197, 0.0256621242, -0.00256376481, 0.0297242031, 0.0601126179, -0.0918086096, 0.0266197324, -0.0513725579, 0.0897334888, 0.0416205637, -0.0165450294, -0.0309060849, 0.0204049703, 0.0945801511, 0.00153329957, -0.0291975737, -0.0292868577, 0.0578069203, 0.0192094315, -0.0462869443, 0.0945882648, -0.0573682375, 0.0580686182, 3.778380e-03, 0.0546103343, -0.0167575404, -0.0335634388, -0.0353777371, -0.0940479785, 0.0466495454, 0.0648834929, -0.0266441461, -0.0238102116, -0.0642111748, -0.0112896385, -0.0285140444, -0.0752914324, 0.021120863, -0.0346712284, 0.0372262336, 0.0105084395, -0.0639294907, -0.0646591559, 0.0153999105, 0.00157939875, 0.0328815877, 0.0371476524, -0.0340012051, -0.0234775804, -0.0409366712, 0.00459173834, 0.00791196618, 0.0181195661, 0.0853330492, 0.104343645, -4.43474157E-4, 0.01864025, 0.00261255843, 7.101780e-02, 0.0465236902, 0.0817520245, -0.0639834702, 0.0338319764, -0.0621945969, -0.0258510448, -0.0477616116, 0.015995156, 0.0428414308, -0.0671404079, 0.00444806041, -0.0737546533, 0.0438533947, -0.0114872251, 0.024882067, 0.0187057331, -0.0549537092, 0.0268902779, -0.0349821337, -0.0289454944, 0.0432052948, 0.0888694301, 5.693210e-03, -3.673780e-02, 0.00916535221, -0.0655376464, 0.0774742588, -0.0496136658, 0.0120913489, -0.0501045622, 0.0329735652, 0.0381609611, -0.0402839705, -0.0473462567, 0.00903901457, -0.0276082084, -0.0150855212, 0.0870288088, 0.0135355722, -0.0442635752, 0.0590100214, -0.00357848336, -0.0635228455, 0.0532011874, -0.0314363912, -0.00743209152, 0.0529279076, -0.0163424592, 0.0598660558, 0.0133801317], [-0.0678632781, -0.0372676253, 0.0173105896, 0.0653227269, 0.0626564323, -0.0766358897, 0.0367376395, 0.0119590377, -0.0765132308, 0.0272818264, 0.0582128949, -0.0311880875, 0.0199377444, 0.0531603172, 0.0685098767, 0.00697491225, -0.0132156713, -0.0719552413, 0.0445587784, 0.019285379, -0.0442028642, -0.0467394069, 0.00195622304, 0.00902704615, -0.00292025739, -0.00590252411, -0.01880816, 0.0459751114, 0.0563163571, -0.0417147651, 0.0589195564, 0.0340361744, 0.0484156124, -0.0374960192, -0.0691996813, 0.0124209439, -0.0221583024, -0.0328250118, 0.0478493161, 0.0537959673, -0.0240576938, 0.0572060309, 0.0371677093, 0.0298907012, -0.0405883081, -0.0428794697, 0.00920402352, -0.0412234962, -0.0357492045, -0.0383767523, -0.0466947146, 0.101578325, -0.0517882966, 0.06795118, -0.0787687227, -0.0375628136, -0.0770281404, 0.0129228877, 3.974650e-02, 0.0840640515, 0.0325298458, 0.0735551864, -0.030094523, 0.0765035376, -0.0105265509, -0.0772315487, -0.0614827499, -0.0138087235, 0.0564586781, 0.0367130674, -0.00583977159, 0.0356704779, 0.0393474586, -3.829200e-02, 0.047174789, 0.021803217, -0.0848198384, -0.0421893336, 0.0120937321, 0.0288616344, 0.072577238, 0.0887359232, -0.0728977397, -0.0416154973, 0.0340451859, 0.0304557811, -0.0255474579, 0.0124843782, 0.0111438967, 0.0839677602, -0.0576399304, 0.0915882885, -0.0288422275, -9.922170e-03, -0.0223046374, -0.0661980957, 0.00890841148, -0.0406225696, -0.0171133392, 0.100371487, -0.0427495092, -0.0962093397, 4.686530e-02, -0.0303351227, -0.053952381, -0.0126161333, 0.0866275951, 0.0141688529, -0.0533497781, -0.093855746, 0.0104066133, 0.0122215059, -0.0238189772, -0.0375334807, 0.051370427, 0.0447878204, -0.0456803143, -0.0154145509, 0.0254472494, -0.0155336447, -0.0633207113, -0.0595693253, 0.0213491842, 0.0334846862, 0.0519208051, 0.0494820252, -0.0686688945, -0.0729257241], [-0.0244285557, 9.05375928E-5, 0.0565047972, -0.0402495377, -0.0443617515, -0.0434244946, -0.0126574766, 0.0517312698, -0.0281925965, -0.0584166497, 0.056383729, 4.638650e-02, 0.0674046278, -0.0226816311, -0.00883102789, 0.0928917601, 0.0692763701, 0.00424026418, 0.0449461378, 8.589220e-03, -0.0432567112, -0.0562243648, -0.0542997867, -0.0336072072, 0.0495964475, -0.0626114383, 0.0208795443, -0.0467996262, 0.069204472, 0.0169888102, 0.034385968, -0.0221680831, 0.080543138, -0.0272891819, 0.0316525884, -0.0250739716, -0.00479612919, -0.0920658707, 0.0431779474, -0.00619140733, 0.0140437419, 0.00982183404, -0.0493017845, 0.0320358947, 0.0458949842, 0.0471505672, 0.0708869621, 0.0338669904, -0.0452994071, 0.0271275267, 0.0175298974, 0.0554958098, -0.00489559071, 0.0915391892, -0.0503241271, 0.00460360292, -0.0371765532, 0.0268461369, 0.0426473767, 0.0604385659, 0.0188776422, 0.0418791734, 0.0727110952, -0.0585457534, -0.0743243396, -2.526350e-02, 0.0422977954, -0.0684440433, -0.0651107281, -0.110687599, -0.0546876714, 0.00653507793, -0.0608277693, -0.0547524318, -0.0452297293, -0.0531877652, -0.0377338976, 0.065254949, 0.0410151035, 0.034484636, 0.0232253186, 0.0752645507, 0.0234584548, -0.00861001946, 0.057281822, 0.0704041123, 0.06771373, 0.00483720284, 0.0561759062, -0.00905226916, 0.052677229, 0.01263915, -0.0170306768, -0.00442280807, -0.0738906562, 0.0645615608, 0.0660044923, -0.0969759449, 2.301150e-02, 0.0998327732, 0.0478177033, -0.0450838394, 0.0796905681, 0.036100138, -0.0236352794, 0.0758395195, 0.0671988055, 0.0172332637, 8.05193267E-4, 0.0266104713, -0.0275822915, 0.0129664177, 0.0270874631, -0.0224177428, -0.0349179767, -0.0292688236, 0.087586455, 0.0832742676, 0.00283741881, -0.0462323725, 7.27985462E-4, -0.0264102444, 0.0186065771, 4.330800e-02, 0.0447091609, 0.0343896225, -0.0782598406, 2.117360e-02], [-0.0090658823, 0.00165501109, -0.0347152166, 0.0014400119, 0.0232128352, 0.0315457061, -0.062573269, 0.0206358228, 0.0648766458, 6.757250e-02, 0.0405995771, -0.00595163926, 0.0185282733, 0.0741875395, -0.065465346, -0.00213273498, 0.0473509505, -0.0155721642, -0.0561119542, 0.00514060957, -0.0207021926, -0.0857480093, -0.00542165432, 0.0788309499, 0.0654021949, 0.0393966362, -0.0599351525, -0.0577261858, -0.07704553, -0.00935286377, -0.0446039438, -0.0413635634, 0.0894897729, 0.0239167698, 0.0309572052, -0.0618694201, 0.0459402762, -0.0384460203, -0.03534282, -0.0260878112, 0.0217438806, 0.00596579676, -0.0520866737, 0.0162307713, -0.0503453799, -0.0599751025, 0.018014418, 0.0538993701, 0.0446723141, -0.0139485244, -0.0536731407, -4.707500e-02, -0.0110707702, 0.0666675419, -0.00341819786, -0.0164688732, 0.0802142918, -0.0646806136, -0.0575083047, -0.0373058729, -0.0543346405, -0.0116112716, 0.0232760478, -0.0299228802, -0.0276502091, -0.040491119, 0.026792977, 0.0164693482, -0.0577942096, 0.0419879332, 0.0231141299, 0.0577502809, -0.0781940073, -0.0317123532, -0.075630933, 0.0370603502, 0.00947459508, -0.0806015283, -0.0384938158, 5.719240e-02, 0.0116362125, 0.0355655849, 0.0107913259, 0.0105385454, -0.0698979273, -0.0581766255, 0.0634001419, -0.0528030731, 0.0614256971, -0.0110462811, 0.0138608953, -0.0642836839, -6.454260e-02, 0.0218464434, -0.0807603895, 7.006420e-02, 0.0156651195, -0.0363626033, -0.0312404111, -0.0368836224, -0.0822587385, 0.0252780821, 0.0278833341, -0.0296178497, 0.0680095777, -0.0552631654, 0.0196504164, 0.0523822755, -0.0299228057, 0.0586550049, 7.965560e-02, -0.00715369591, 0.0621701776, -0.0320070423, -0.0479340851, -0.0380442291, 0.0102019664, -0.048184257, 0.0113409655, -0.0789639801, 7.941830e-03, 0.0139178522, 0.0561713837, -0.0714009926, 0.0168651603, -0.0460672826, -0.00211451785, -0.0519795157], [-0.00256177085, -0.0707407817, -0.065396674, 0.0123171685, -0.00183005782, 0.0378773212, 0.00379015063, 0.059625145, 2.207030e-02, 0.0517411754, -0.0245633069, 0.0263115522, -0.0528115556, -0.047935266, -0.0750197843, 0.00219946704, -0.0807620957, 0.0152299078, -0.0165651422, -0.0759946629, -0.0585145801, -0.0222162418, 0.0340139307, 0.0673456341, -0.0630577728, 0.00248353183, -0.036352139, -0.0758538172, 0.0373821631, -0.0455512814, 0.0569789857, 0.0156630445, 0.0311498586, -0.0780213475, -0.08009395, 0.00314989127, 0.0447820462, 0.0272007901, -0.0276761577, 0.0489876904, 0.0462230928, 0.0672943667, 0.0596507527, -0.00938992854, 0.0156895071, -0.0390085466, -0.0373747274, 0.0768403783, 0.00299566984, -0.0449037664, 0.0114885373, -0.0315123834, -0.0513352267, 0.0225750767, -0.0177610219, 0.0276075732, -0.0405277684, 0.0163036305, 0.0367885418, 0.0704710707, -0.072966814, -0.0730016529, 0.029117398, 0.0240194947, -0.0276088063, 0.0378535874, -0.00279789511, -0.0578684583, -0.00951688922, -0.0456373096, -0.0642018691, -0.0636169314, 0.0779158845, 4.313320e-02, 6.518190e-02, -0.0746261925, -0.0436835587, 0.0602973029, 0.057060618, 6.535810e-02, -0.0583526641, -0.0481255054, -0.0092653716, -0.0337135307, 0.00341573171, -0.0502090491, 0.0274842661, -0.0792971551, -0.0351808444, 0.0715639442, 0.0460463203, 0.0577385128, -0.0225940775, 0.0291874632, -9.241500e-03, -0.0264840815, 0.075236693, 0.0399956927, 0.0676963329, 0.0833287909, -0.0429229662, -0.0268265381, -0.0443668291, 0.0328638665, -0.0591567457, 0.0408262834, -0.0292702205, 0.00326038268, 0.0366917923, 0.0471265465, 0.0790683552, 3.739480e-02, 0.0754047707, 0.0283631422, -0.00501409872, -0.074710913, 0.0327433236, -0.0601052716, -0.0212175362, -0.0185154341, -0.050974492, 0.0705418736, -0.048729144, 0.0657138154, 0.0247248895, 0.00614018878, 0.0770845637, 0.0431692898], [-0.0384170562, -0.0623327083, 0.0371814854, 0.00913695712, -0.0343300179, 8.71428114E-4, 0.0190855954, -0.0345696397, 0.0599832721, -0.074267529, 0.0438892394, -0.0343661197, -0.0397558138, -0.0408659466, -0.0341923758, 0.0257172231, -0.0150784627, -0.0669632256, 0.00797764584, 0.0385413915, -0.0570591465, -0.0194353927, -0.0451739952, 0.0146907801, 0.0234540757, -0.0128548034, -0.0384001099, -0.0121509265, 0.0737575442, 0.0304907579, 0.0729526654, 0.0232379101, -0.0675794482, 0.00178586075, -0.0189093519, -0.0720544681, -0.0253178589, 0.0163424835, 0.00877277646, 7.447730e-02, -0.0208327565, -0.0571186915, 0.0487115644, 0.0657948553, 0.0184037462, 0.0185280852, 0.079029724, 0.0698569641, -0.035079997, -0.0719077662, 0.033779189, -0.0526065715, -0.0169154089, 0.0218021385, -0.00276032533, -0.0372594781, 0.0224128012, 0.00756289484, -0.00224136561, -0.0577758923, -0.0404176041, 0.0711416379, -0.0452656709, -0.0397006348, 0.0200961661, 0.0446205251, -0.00238617253, 0.0118487086, 0.0135207558, 0.0504980795, 0.0761808454, 0.0798095614, 0.00544888666, -0.0598109737, -0.0171655342, -0.0722740367, 0.00948047731, 0.0720660388, -0.0549051277, 0.0759286806, 0.0581316426, 0.0135696558, -0.0618236102, -0.0511278734, 0.0661906153, -0.0595734417, 0.00635659555, -0.00326730777, -2.188010e-02, -0.0560707487, -0.0507146269, 0.0298383404, 0.0487123802, 0.0376568213, 0.0221393332, 0.0801030173, -0.0388528258, 0.0246661976, -0.0213537887, -0.0401831567, -0.073543489, 0.0735499784, 0.0811436176, 0.0389765315, 0.0577515028, 0.0246966071, -0.0747492164, 0.0595819131, -0.0290614385, 0.017848324, 0.0211851336, 0.0309709143, 0.0797181949, -0.0640038252, 0.031339217, 0.0394342802, -0.030819919, -4.346440e-02, 0.00985434186, 8.168320e-03, -0.0189149287, 0.0105296643, 0.0591482557, -0.079650864, 0.0351864547, -0.017506158, -0.0244778357, 0.0157035962], [-0.0756570101, -0.0764564871, -0.0578441583, -0.00479017384, -0.0504163876, -0.00816466939, -0.00901293661, -0.0697929189, -0.0598420724, -0.0418788083, -0.0624498203, -0.0193611681, -0.0550635085, -0.0403183438, -0.0280371141, -0.0449378565, 0.0299892668, -0.056949921, -0.0393805541, -0.0335469209, 0.0777389332, -0.044271823, 0.0493938066, -0.0232438445, -0.0528193898, -0.0183396116, -0.068878293, 0.0211450923, 0.0374588296, 0.00610547327, 0.0528709255, -0.0129344668, 6.872130e-02, 0.0211039912, 0.0773681477, 0.0104464209, -0.0348714329, 0.0201780275, 0.0392005071, 0.0346477963, -0.0480899028, 0.0745830089, 0.0669455603, 0.0675275773, -0.0237755515, 0.0461483747, -0.0766377151, -0.0482859053, 0.0282372218, -0.0613660216, 0.0345557295, 0.0802161097, 0.0507202037, -0.0568063483, -0.015795758, -0.00574885216, -0.0342467576, 0.0105777606, -0.0529373847, 0.0666500852, -0.0181065872, -0.0262691677, -0.0323200449, 0.0429718122, -0.0289494172, -0.0767156258, -0.0601311326, -0.0281416737, 0.0210853443, -6.437310e-02, 0.0437178612, 0.0669968948, 0.0113177057, -0.0196340587, 0.0693274364, 0.0723217577, 0.0612762161, -0.0532029681, 0.0548515245, -0.0558755808, -0.0721410736, -0.0442370549, -0.0418866687, 0.0760708526, 0.0549778454, -0.0213583186, -0.0357369035, 0.0533030368, 0.0274632107, -0.00938789546, -0.0288061313, 0.0315227173, -0.0676712468, 0.0499557033, 0.0720944777, 0.078672707, -0.00795571506, 0.060698729, -0.0455680564, -0.0519967414, -0.0254669674, -0.0691546351, -0.0629941374, 4.685130e-02, 0.0160323698, -0.0570644513, 0.0474170446, 0.0511532798, -0.0174260475, 0.0593638644, 0.0411690176, 0.0615479127, 0.0569708571, -0.0693020895, 0.00974822138, -0.00249906885, -0.0197219849, -0.023515895, -0.0169867575, 0.0479880497, -0.0469649322, 0.0455557778, 0.0155554498, 0.0694211945, -0.0524483919, -0.0120452866, -0.00284456741, -0.0275214706], [-0.0320011526, 0.0113158934, 0.0263415221, -0.0780805051, 0.0679449886, -0.0064079985, 0.0400597975, 0.0807293355, 0.0139110303, -0.0569042824, -0.0708479509, -0.0312004499, -0.0261206254, -0.0560903437, -0.0451589786, 0.0691588596, 0.047738824, 0.0480582044, 0.0619293786, 0.00702885212, -5.061040e-02, -0.0543577857, 0.00289761764, -0.0621245987, -0.0150781395, -0.0453255139, -0.0346753933, 0.066413112, 0.073175855, -0.0118125705, -0.0726625249, 0.0160393119, -7.781980e-02, -0.00662872754, 0.0294296145, -0.00111212314, -0.00658602966, -0.0165919606, 0.00710554281, 0.0203402247, -0.0521568581, -0.0501545407, -0.0713538527, 0.0226741433, -5.372730e-02, 0.0625791847, 0.0626111776, 0.026481159, 0.0524411164, 0.0672617406, -5.09391946E-4, -0.0766377598, -0.0558636412, -0.0120538818, -0.00978249591, -0.0405682102, -0.0367988162, -0.00362645113, -0.0438408479, 0.0423188247, -0.0178636909, 2.432580e-02, 0.0210548472, 0.0401844904, -0.0212507844, -0.0274807066, 0.0562457517, 0.0393660665, -0.0171747394, 0.0037405116, 0.0752583891, -0.0806213542, 6.780420e-02, -0.0587374531, -0.033158958, -0.0259302203, -0.0450840071, 0.0688647479, 0.0102651408, 0.0779187381, 3.302630e-02, -0.0322541669, 0.0152603444, 0.0583158769, -0.0496684164, 0.0567066483, 0.0228683166, -0.0557285734, 0.0770256147, 0.0035094237, -0.0104794977, 0.0450204685, 0.0689159781, 0.0165778734, 0.00982439611, 0.0700144395, -0.0683602616, 0.0785852149, -0.0555882081, 0.0633649081, -0.0616446249, 0.0261501167, 0.00190731115, 0.0756166205, -0.0719528049, 0.040799357, 0.0176330395, -0.00350180198, -0.0510341488, 0.0571325161, -0.0570817851, 0.0517871156, -0.0557958074, 0.00195177132, -0.0088971192, -0.0320391171, -0.0336185656, 0.00543689961, 0.0493562631, 0.0756451189, -7.29982567E-4, -0.0458574742, -7.279150e-02, -0.0447602347, 0.0671925172, 7.103440e-02, 0.0497775823, -0.0676985309], [-0.0505705848, 0.0142981317, 0.0322355889, 0.0595614426, -0.0746433288, 0.0376881063, -0.0743731111, -0.0116376653, -0.0178018734, 0.0469448604, 0.02365968, -0.047300145, 0.0182700101, -0.033823587, -0.0389874279, 0.0283736568, -0.0442715175, -0.0446711481, -0.0259741098, 0.0353250168, 7.131720e-02, 0.0688885599, 0.0284496248, -0.0514263175, -0.00848457869, 0.061460305, 0.0177937709, -0.063528955, -0.0417775922, 0.0735128819, 0.0633930787, -0.00912480894, 0.0142931491, -0.00278521306, -0.00229939027, -0.021963004, -0.0200176369, 0.00948405079, -0.0413215309, -0.0639488101, 0.0323632285, 0.0650278926, -0.0164345447, 0.038926851, 0.0445064195, 0.0675444454, 0.068661429, -0.0288533885, 0.0408596322, -0.0773253142, 0.00868003163, 0.0771203116, 0.0511071309, -0.0725743622, -0.0481451452, -0.0055770711, 0.0789838507, 0.0241100453, 0.0555427745, -6.784160e-02, 0.0524538793, 0.0395501964, -0.0651263371, 0.040208336, -0.0143200178, -0.0755813792, -0.0702953786, -0.050088156, 0.00383252674, -0.024482986, -0.0179993175, -0.0443406515, -0.0150564518, 0.0290354155, 0.0578567497, 0.0640785545, 0.0536940545, -0.0489541478, 0.0323635414, -0.0804247111, -0.0711383447, -0.0578526855, -0.0700409636, -0.0652946904, -0.055301398, 0.045508191, -0.0132601457, -4.70365543E-4, 0.0108633619, -0.0600182973, 0.0715888143, 0.0824382156, -0.0455074348, -0.0100764362, 0.0362074673, -0.0649894177, 0.0112592354, -0.0694499686, -0.0314297192, -0.0121036656, 0.0458595827, 0.00383909047, 2.343390e-03, 0.0424145386, -0.0714968368, -0.0284514949, 0.0756255835, -0.0473799892, 0.0461348481, 0.0441101827, -0.0702612177, -0.032776814, -0.0722029284, 0.00539995963, -0.0328244567, -0.0335481167, 0.0436918698, -0.0207259282, -0.0132911848, 0.059602391, 3.218880e-02, -0.0143094659, 0.0144202048, -0.00215379824, -0.0555151366, 0.0396170653, -0.0247339252, -0.0723399221], [-0.0518680215, -0.00554729067, 0.012277076, 0.0869444385, -0.0752931163, -0.0803382471, 0.0477548689, 0.0463397875, -0.039009884, 0.0735539868, -0.00558149349, 0.0493517406, 0.0451347865, -0.0506388359, 0.0313135609, 0.0568354391, 0.0181696191, -0.00448468421, -0.0481048375, -0.0758009552, 0.0236598011, -0.0649193376, 0.053103447, 0.0334438123, -0.0335370935, 0.0441877954, -0.0550652333, 0.0751330182, -0.0778125151, 0.00253490987, 0.00713293673, 0.069017157, -0.0136009445, -7.909580e-02, 0.0406532735, -0.0201666728, -0.0631902665, 0.00720232679, 0.050816264, 0.0266793035, 0.0553974584, -0.0308225285, -0.0236564465, -0.0179841202, -0.0461978614, -0.0637742132, -0.0825821757, -0.00686009135, 0.0739967078, 0.0399362482, -0.0674890652, 0.045988325, -0.029333951, 0.0507732034, 0.0599517934, 0.00639984942, 0.0529143848, 0.0800470114, -0.036881648, -0.00720706955, -0.075937815, -0.0262951981, -0.0146730645, -0.0569726601, -0.0660911053, 0.0215483811, 0.00160116586, -0.0534423031, 0.0577729456, -0.0283762403, -0.0451310724, -7.239240e-02, -0.0543154925, 7.515300e-03, 0.0301153623, 0.0383794755, -0.0753287449, 0.0698242784, -0.0495673046, -0.0689398721, 0.089015305, 0.0116113359, -0.0433220156, -0.0357120335, 0.046931047, -0.0856014937, 0.0335856266, 0.00703032967, 0.0407504886, -0.072509177, -0.00949838943, 0.0216404907, 0.0616214387, 0.0128511209, -0.0766538903, 0.0529184863, 0.0461581647, -0.0632803217, -0.0421839021, 0.0265474841, 0.0583846085, 0.00296472968, 0.0430434793, 0.0443775393, 0.0150420861, -0.0545185506, 0.0674002841, -0.0624257848, -0.0185298752, 0.0248352606, 0.0860408321, -0.0372803882, 0.0459824763, 0.0227070786, 0.0577491038, 0.0582762249, 0.0410680883, -0.0244519543, 0.0356378518, 0.00972921867, 0.074756354, -0.0745906755, -0.0321433544, 0.055960644, -0.0413615778, -0.0779148489, 0.062586166, 0.0451181047], [-0.0524049476, 0.0430744924, -0.081523329, 0.0491366498, 0.0566963032, 0.0165676586, 0.0186627675, 0.0269875471, 0.0206443332, -0.0431439057, 0.0662984923, -0.0159435458, -0.028599482, 0.0321812183, 0.0648858547, -0.024105113, -0.0503873527, -0.0496439971, -0.0564819127, -0.0688659846, 0.0342181101, -0.0572998747, 0.0157082397, -0.0569056347, 0.0396080762, -0.0300114956, 0.00486288546, 0.029272588, -0.059221793, 0.00521202944, -0.00389257935, -0.0068808035, 0.055054754, -0.0288688168, 0.0469577573, 0.0796857699, -0.0528779738, 0.0135751311, -0.0719276368, -0.0233258419, -0.00140934438, -0.0143939778, -0.0550700687, -0.0571133047, -0.0794131607, 0.0107425358, 0.0138422418, 8.718020e-03, 0.0674695298, -0.0107580349, -0.0163115207, 0.100287132, 0.01322175, -0.0344847105, -0.0658339486, -2.864090e-03, 0.0650777668, 0.0682116374, -0.043438293, 0.0584733225, 0.0287101585, 7.581650e-02, 0.0260918718, 0.0511919521, 0.0610863194, 0.0424488485, -0.0528875627, 0.0660760254, -0.0698032454, 0.0477578714, 0.0666903108, -0.0586871319, -0.0574539825, 0.0100183897, -0.0256468058, 0.0631270558, 1.12276859E-4, 0.0127924252, 0.0278753024, 7.543750e-02, -3.348140e-02, -0.0559933707, 0.0452955924, -0.0449988209, -0.0147309462, -0.0606670976, -0.0469529293, 0.0026705449, -0.0449570641, -0.068439886, -0.0803943052, 0.0339660458, 0.0533427112, 0.00196579611, -0.00680591119, -0.00505159236, -0.0293292236, 0.0184599701, -0.0278203841, -0.0516084172, -0.0993217751, 0.0418884754, 0.0259556845, 0.0667214766, -0.064228259, -0.0191476811, 0.0471979193, -0.00974557828, 0.0205281544, 0.0215903837, 0.0904865562, -0.0245437436, 0.0587470904, 0.0268369187, -0.050589174, 0.0302784629, 0.0141477175, -0.0487012081, 0.0557626449, -0.0083138477, -0.0380076729, -0.0894878358, -0.0733614265, -3.715500e-02, 0.0768531262, -0.0570729822, 0.050711982, 0.00929741188], [0.0162872821, -0.0212767683, -0.0464195684, 0.0275726095, 0.0702397749, -0.0290880352, -0.0648422763, -0.0273377635, 0.0620873868, -0.0245960969, -0.00324564963, 0.047130771, 0.0244020764, 0.0137758264, 0.0175348762, -0.0216781721, -0.0661032051, 0.0540289171, -0.0491361395, 0.0391354077, 8.30669829E-4, 0.0486144423, -0.00490699895, 0.035455849, -0.0483410917, 0.042030625, -0.0288075246, 0.0742744729, 0.0405517891, 0.0295131132, -0.0149748912, -2.647620e-02, -0.0372339524, 0.00353029254, 0.0386083238, 0.0303904694, -0.0409832299, -0.0593755059, 0.00961741805, 0.0508019477, -0.0317172818, -0.0650796369, 0.0248616431, -0.00215191743, 7.971010e-03, 0.0467377976, -0.109839924, -0.0129141528, 0.0699567348, 0.0919272303, -0.0136467246, 0.134968802, -0.0897499546, -0.0575589761, 0.0636619851, 0.011492529, 0.0492738746, -0.018175358, -0.0545668453, -0.0278537348, -0.0137921665, 0.002643927, 0.0332062431, -0.0367435962, -3.617100e-02, -0.0613931604, 0.0132706249, 0.0492771752, -0.0129323239, -0.0981397107, 0.0689649731, -0.0106166797, 0.0469979495, -0.0323237218, 0.055252783, 0.0405739099, -0.00356421713, 0.0134761538, 0.0562950373, 0.0863577648, 0.0239190888, 0.0576917604, -0.0311807115, -0.0349663422, -0.00594095932, 2.16225468E-4, 0.0217828192, -0.0375707932, 0.0235074982, -0.0591569096, 0.0438511409, 0.074827455, 0.0140200583, 0.0123361684, -0.0536753722, -0.0266691018, 0.0435129367, -0.0604123697, 0.0309946071, -0.0702930689, -0.121367328, -0.0501693375, -0.0583776198, -0.0283580199, -0.0345205888, -0.0538489036, -0.026849959, 0.0358578824, -0.015199434, -0.00309396139, 0.00730716344, 0.0443485677, -0.0485394709, 7.694780e-02, 0.0459258556, 0.0162771419, 0.00822472852, -0.0394389927, 0.040003892, -9.607120e-03, -0.00396740204, -0.0549297333, -0.0131595982, 0.0457024686, -0.0344238617, -0.0780334472, 0.0848105549, -0.00576122385], [-0.0723566934, -3.30122275E-4, -0.00348738884, 0.081315279, -0.0207988098, 0.0100015048, -0.029173499, 0.115104884, -0.0422469601, 0.0288182944, 0.0722247586, 0.0366021171, -0.0690316111, 0.0554103702, 9.454590e-02, 0.00272225216, 0.0705425814, -0.0351813622, -0.0355808027, 0.0298246723, 0.0312057417, -0.0225377772, -0.0321784727, -0.0675677583, 0.0431947708, 0.0200990941, 0.0541009605, -0.0193758477, 0.0356459841, 0.0732426494, 0.0664200857, -0.0173490699, -0.0215412714, -0.019940475, 0.0452732742, 0.0901348665, -0.0469920151, 0.00363962608, 0.0698692426, 0.0469775312, 0.068914175, 0.0656992421, -0.0225510392, -0.0490103029, 0.0309250522, 0.00187636644, 0.00879300479, -0.0609006099, -0.0487105772, 0.0643971339, 0.0380567349, -0.00854930747, -0.0874631404, -0.0630893409, -0.0620041043, 0.00423753541, 0.0349843912, 0.010477962, -0.0599226542, -0.00519232359, 0.00929882657, -0.0615526699, -0.0997495353, -0.0658854097, 0.0469711162, -0.0740444288, -0.0151926419, -0.0766201392, -0.0130991964, 0.00588313676, -0.0217590965, 0.0645821542, -0.06804654, 0.0671810209, -5.278560e-02, 0.0014521695, 0.0153692542, 0.0545615032, -0.00542433746, -0.046277266, -0.0461673662, -0.0488307849, 0.0617017746, -0.113308489, 0.0120814638, -0.0837806761, -0.00725486688, -0.00516452733, -0.0819490775, 0.0198462158, -0.0227651875, 0.0690322518, 0.0121278437, 0.0444141291, 0.0491059497, 0.0481251404, 0.0552108251, 0.0235981122, -0.0455617532, -0.026535973, -0.0298750252, 0.0445386469, 0.0232181139, -0.0102707082, 0.0847486928, 0.0287355278, 8.560230e-02, 0.0269995071, -0.0683670491, -0.0174606685, 0.0860795602, -0.0114924777, -0.0396169648, -0.0858189091, -0.0179152228, 0.0447356179, -0.0310764201, 0.0109486515, 0.0387723148, 0.0872201696, -0.0596021973, 0.0456211083, -0.0152865686, -0.0394197479, 0.0239107907, 0.0558072105, -0.0254865196, -0.0731427222], [0.0535662137, -0.056184575, -0.0442834534, -0.0104895588, -0.00234472076, -0.00659229141, -0.0587299466, 0.0831370279, 0.00227933959, -0.0999155342, -0.00714704208, -9.220750e-03, 0.00200279802, 0.0841989591, -0.0536765978, 0.0392738357, -6.640660e-03, 0.0924787297, -0.035136763, 0.0336553194, -0.00624587433, 0.0512113944, -0.0206396393, -0.0543040484, 6.656870e-02, 0.0353290811, -0.0250262935, -0.0595573112, -0.0626466423, 0.0584689938, 0.0141440229, 0.0562951788, 0.079177767, -0.00787789654, -0.0300821979, -0.0161531437, -0.0162901133, -0.0708366632, 0.0576839522, 0.0884664282, -0.0124578504, -0.0617147423, 0.00918767881, 0.0145958047, -0.0241435897, 0.0109953051, -0.00612693094, 0.0584070198, -0.051270064, -0.00452443399, -0.0798109695, 0.0934498459, 0.0476203375, 0.0123546561, -0.0164260231, 0.0723945647, -0.0593438596, -0.0426162891, 0.051463861, 0.0605240092, -0.0674260929, -9.088240e-03, -0.0775774568, 0.0355259813, -0.0240241271, -0.0542822629, -7.95266766E-4, 0.0439232476, -0.0551373102, -0.0482960045, -0.0115453973, 0.0454289578, 0.0286020078, -0.0275775809, 0.0040215659, 7.257980e-02, -0.0389806703, -0.0206413437, 0.00768414279, 0.034875229, 0.0195846055, 0.0492908806, -0.0289999321, -0.0466002598, -0.0531815961, -0.0252142064, 0.0220356584, 0.0147141889, -0.0476557203, -0.0381510258, 0.0839952379, 0.069469735, -0.0265074857, -0.00621215766, -0.01006414, 0.0940167233, -0.0359308943, -0.0412890166, 0.0502570942, 0.0254408866, -0.0228331797, 0.0443927497, 0.0117759295, 0.0477893539, 0.0371837467, 0.00863871164, 0.0805069357, 0.0518330298, 0.0675534159, 0.0525294617, 0.0440960974, -0.0663830861, 0.0421216711, -0.0246259365, 0.00206222222, 0.0300038215, -0.0232241116, 0.0233803559, -0.00844977982, 9.221500e-02, -0.0166422799, 6.365560e-02, 0.0262309369, -0.0091238413, 0.00147153565, -0.0510468557, -0.0528194048, 0.0102925235], [-0.0636949241, 0.0705045834, 0.0404417068, 0.0110437097, -0.0184722487, -0.0119934231, -0.0357344151, 0.101259708, -0.0325755365, 0.00350667397, -0.0530454852, 0.0442099497, -0.0156936795, -0.0112295942, 0.0300908517, 0.0895057395, -0.0147115942, 0.00743409526, -0.0599225052, 0.065230526, -0.00927819218, -0.0115096727, 0.0601079501, 0.0393154621, 0.0303011592, -0.0572875775, 0.108091101, -0.0546776541, 0.0424583331, 0.0708580762, -0.0289654415, -0.0683756545, 0.0805586725, -0.00972339232, -0.0573736541, 0.0945591852, -0.00856115948, 3.425030e-02, 0.0554948561, 0.0605523884, -0.0720402673, 0.00296737324, -0.0343909115, -0.0542801395, -0.0238104034, -0.0301132612, -0.0697689131, 0.0144679993, 0.0918058604, 0.0925708562, 0.0741897672, -0.0160751138, 0.0437110625, -0.00339975464, -0.0109343659, 0.0474908203, 0.00993135664, 0.0415850691, 0.00145970122, 9.228890e-02, 0.0322973058, -0.0832725986, -0.0999936386, 0.0225023907, -1.941910e-02, -0.0312394202, 0.034831658, -0.0454995781, -0.0684170723, 0.0325784758, -0.0115332929, 0.0153732793, -0.0711059272, -0.00540718203, -0.0723850578, 0.0743967369, 0.0415274613, -0.0283785257, -8.918440e-02, -0.043161612, -0.0697706118, -0.0118364803, 0.0575967655, 0.0248284601, 0.0542273968, 0.0598919503, 0.092460066, 0.0122120129, -0.0360154882, -0.0487185121, -0.029347213, -0.0507090539, 0.0272416882, 0.0268370789, -0.032131929, 0.0351168774, -0.0598449036, 0.0794935375, 0.00273952866, 0.0176309105, 0.0180099253, -0.032745339, 0.026491994, -0.0720622539, -0.0103201317, 6.479290e-02, 0.0849463716, 0.0599233806, 0.0666086525, 0.0321751088, 0.0216963328, 0.0080685718, 0.0139518566, -0.0397081971, -0.0266338661, -0.0148489438, -0.0521588698, -0.0143619804, -0.0361421295, 0.0704026818, -0.0218092389, 5.484530e-02, -0.0374890491, -0.0653770119, 0.0337571576, 0.040180333, -0.0243008826, 0.0676422194], [0.0640597865, 0.0350910053, -0.0175559334, 0.0646570325, 0.0823307335, -0.0701810345, -0.105549522, 0.025500644, -0.0228468571, -0.0330865718, -0.0475588068, -0.0385949276, -0.00633549131, -0.0567655154, 0.0469809733, -0.00174605218, 0.0662281811, -0.0259554088, 4.95430781E-4, 0.0251290668, 0.0200875159, 3.687570e-02, 0.0694998503, -0.0233796518, 0.00447602803, -0.0310741384, -0.0170885436, 0.0619751886, 0.0588688143, -0.00576209696, 0.0349612795, 0.0755778253, 0.0328051969, -0.0674080178, 0.0530172586, 0.0352687687, -0.0815902948, 0.0309344139, 0.0190914962, -9.35723423E-4, 0.0304696336, 1.140290e-02, 0.0469694659, 0.105835006, 0.059632767, -0.0182706118, -0.0370492227, 0.0126164882, 0.0605424568, 0.0830428302, -0.0128068263, 0.0753979757, 0.0172666628, 0.0406535603, -0.0197760444, -0.00399693847, -0.0458713733, -0.0717241094, -0.0575003587, 0.0280487798, 0.00245695352, -0.0106604034, 0.00404447643, -0.0687426626, 0.0176446009, -0.0548117533, 0.0166308433, -5.368730e-02, 0.0231806226, -0.00370596559, -0.0385472625, -0.0246506166, 0.00394495949, -0.0160850231, -0.0493966863, 0.0920007303, -0.0374125317, 0.0403782614, 0.0610513538, 0.072517857, -0.00714796456, -0.0282185543, 0.0282962602, 0.00418776181, 0.0769128427, 0.0571912192, 0.102101088, -0.0584943071, -0.0941478833, -0.0433208235, 0.00675768218, 0.00761590386, -0.0314213336, -0.0734303519, -0.0143735223, 0.00484976824, -0.0603390224, 0.00293257297, -0.0524198823, -0.00430020969, -0.024364749, -0.00896142423, 0.0384315662, -0.0955304279, 0.0379713252, 0.00472824834, 0.0370437875, -0.0766127482, 0.0605852567, -0.0626063644, -0.0141692609, -0.0668208897, -0.0145739941, 0.0667033643, 0.0350944139, 0.0639408231, 0.0828970447, -0.0836865827, -0.06642133, 0.0226276051, -0.0707996041, 0.043245092, -0.077584371, 0.0254822746, -0.0538765937, -0.0119507862, -0.118271731, 0.0232901517], [0.0358703472, 0.0259207711, 0.0513812304, 0.00472596101, 0.0260472987, 0.00453362288, 0.0238472298, -0.0803589448, 0.0653785318, -0.132408679, -0.1152042, 0.0610880926, 0.0330773517, -0.0721782446, 0.0824341699, 0.10100919, -0.00181673095, -0.068790704, 0.0380650051, 0.0584051721, 0.00450199703, -0.0780326351, 0.0219443142, -0.00585346483, -0.0882936865, -0.0257060025, 0.0523108207, 0.0100383833, -0.0108557744, -0.0430501699, 0.0888982489, -0.0618948862, 0.0105116917, -0.0432951152, 0.0382553115, 0.117683209, -0.0628551394, -0.109204464, -0.0142366625, -0.0182088427, 0.0816539451, 0.0306481943, 0.0278274566, -0.0507459939, 0.00450813025, -0.0784559845, 0.0452640019, -0.0560097545, 0.0860229507, -0.0384244546, -0.0584755056, 0.0513081066, 0.1108597, 0.058383137, -0.0875916853, 0.0217638575, 0.0203017481, -0.00332902279, -0.00613717083, 0.00956125092, -0.0305376109, -8.708750e-02, -0.0863191038, -0.0983739644, 0.00984952598, -0.0151432818, -0.0210339203, 0.0337448269, 0.0658517331, -0.0482877269, -0.0595508628, 0.103774831, 0.0787600055, 0.0801308453, 0.0187335461, 0.0892444178, -0.0610798523, 0.0802551731, -0.0727719366, 0.00248223636, -0.0561017692, 0.0580015257, 0.0465031266, -0.0194558911, 0.0438637659, -0.0645671412, 0.141481042, 0.0779880732, 0.0786960646, 0.0460221805, -0.0159231015, -0.118932322, -0.0695102214, -0.0342627279, -0.0668501555, 0.0115238838, -0.109530538, -0.0150529612, -0.0252385437, -0.0361362547, 0.0700604245, -0.0172541011, -0.0288438015, -0.00772576733, 0.0195446331, -0.0500198491, 0.0188652761, 0.0366885439, 0.0362840705, -0.0541316569, -0.0475719683, 0.0177473072, -0.0588864759, -0.0401255228, -4.81345953E-4, 0.0458244868, 0.0868549272, -0.00114693516, 0.033130113, 0.0029609031, -0.0662725568, -0.0440696366, -0.0387017243, -0.0167954043, 0.0501632057, -0.0607378073, -0.115919806, 0.00778747257], [0.0392061025, -0.00679734162, 0.0733912215, -0.0468927845, -0.00376004144, -0.0511621647, -0.0541940704, 0.0239020232, 0.0888254493, -0.121668056, -0.0841722413, -0.0362547934, -0.0261861738, 0.0236074887, 0.0206168573, 0.0870715827, -0.0751698315, 0.030924568, -0.0679902211, -0.00319751655, 7.366140e-02, -0.0909282788, -0.0711203888, 0.0128148664, -0.0967528969, 0.074120529, 0.0463884659, -0.0336275622, 0.0628694072, -0.00546943163, 0.0774248242, -0.0528299212, 0.10922727, -0.103018589, -0.0369444452, 0.0557632633, -0.102856107, -0.0824807435, -0.0340849571, 0.0418949053, -0.026087543, 0.0604761578, 0.0677812323, 0.0101854205, -0.00720515242, 0.0306853075, 0.0817048549, 0.0358904451, 0.0239682365, 0.03797758, 0.0213920772, -0.0373053737, 0.0350437798, 0.107688263, -0.0360684283, 0.01956385, 0.0184925981, 0.0488780402, -0.0225825179, 0.0438758917, -0.071121566, -0.0280840192, 0.0224081222, -0.103658877, -0.113763101, -0.0731076896, 0.0911161378, 0.0443339981, 0.0180508383, 0.0529719777, 0.010228999, 0.0637571141, -0.0434862189, 0.0322919451, -0.0395621732, -0.0349693634, -0.0859611928, 0.0792379677, -0.0869839191, 0.118169241, -0.08174517, 0.0938351601, -0.0455626845, 4.148750e-02, 0.0623668916, 0.00210082601, 0.00403611967, 0.0245391801, 0.0948632881, 0.0619590319, 0.0221394077, -0.0908865928, -0.0637710914, -0.0809817165, 0.0746461675, 0.00191726605, -0.113215551, 0.0821262225, 0.0046866159, 0.0557531714, 0.0602113791, 0.0345578566, 0.0429677032, -0.0561582707, 0.054363288, 0.0512380674, 0.0584378056, 0.0222648568, 0.11031013, -0.0398144685, 0.050962165, 0.0076503451, -0.0434276462, -0.0614572093, -0.0421229526, 0.0840852633, -0.036113672, -0.0612887256, 0.0863796845, 0.0586673319, 0.0286743343, -0.00835611764, 0.0296741929, 0.0238093212, -0.098635219, -0.052601289, -0.116163619, -0.0757672936], [-0.017217651, -0.0117367068, -0.00485177943, -0.0153007898, 0.0858802571, -0.0329714492, -0.0219266769, -0.0766641125, -0.0106742932, -0.134292617, -0.0954028591, -0.0238271076, 0.0868417472, 0.0284221712, -0.0231219232, 0.0249041598, -0.0211024564, -0.0607428215, -0.0331116654, -0.124087706, -0.0110156834, -0.0604727454, -0.0370609686, -0.00482798042, 0.00332171307, 0.0697996542, 0.0835987478, -0.0597517043, 0.060423512, 0.0627440065, 0.031068027, 0.0372597948, -0.0311056897, -0.0838339105, -0.0326787308, 0.00646666111, 0.0198273417, 0.0617253557, 0.00405532494, 0.0805488601, -0.0364307798, 0.027962666, 0.0204453748, -0.00589300739, 0.0854990556, -0.02391617, -0.00208400027, 0.00750115141, -0.0621644519, 8.20793502E-4, -0.0062472159, 0.00783816352, 0.0181754082, 0.100071922, -0.0146495933, 0.041189488, -0.100749359, -0.0495057516, 0.0308796912, -0.0235979185, 0.0113330046, 0.0440827869, 0.0163423456, -0.00641739927, -0.0519039966, -0.0347459167, -0.0116280233, -0.0179056209, -0.0780058577, 0.00620914111, -0.0686943755, 0.0446545519, -3.210270e-02, -3.83681792E-4, -0.017651964, 0.0548840947, 0.0192765594, 0.0190380141, 6.2273466E-4, 0.0789624527, -0.0307779647, 0.103356384, 0.011444401, -7.00072618E-4, 0.112583749, -0.00331669068, 0.0197529253, -0.0393586755, -0.0339229405, -0.00972577091, 0.0766277388, -6.764110e-03, -0.00164097582, 0.022699425, 0.0665493235, -0.0591188483, -0.0125475759, 0.0367399342, -0.080686897, -0.0627073273, 0.0606038347, -0.0168955196, 0.0320100598, -0.0341647267, -0.0253324602, 0.0891743078, 0.0797874331, 0.0445431471, 0.0586750098, 0.0607896447, -0.0407043099, 0.0768715739, -0.0340972506, 0.0420794114, -0.0426804684, 0.011666012, 0.0689256415, -0.0257377271, 0.0459445938, -1.922720e-02, -0.0413712822, 0.0725060999, 0.022437891, -0.0777021422, -0.0292749275, 0.0283228159, -0.143462673, 0.0252925623], [-0.0756462663, 0.0107251685, 0.0578391552, 0.0627352223, 0.0750358924, 0.130815789, -0.103056692, 0.0650169328, 0.0726718158, -0.0270540249, -0.093579337, -0.0473134369, 0.0297974013, 0.0593285747, 0.0895681604, -0.0376206152, -7.121060e-02, 3.028120e-03, 0.0403166637, -0.01576964, -6.133390e-02, -0.0305401701, -0.00557595491, -0.00722305756, -0.0358704701, 0.0517382286, 0.0609398559, 0.0149571337, 0.0248279795, 0.0650544539, -0.0452509932, 0.0366957337, -0.0335888229, -7.982610e-02, 0.0363207236, -0.0996548831, 0.0233998988, 0.0650000945, 0.0481111705, -0.044837296, 8.13235936E-4, 0.0757763907, 0.0232182276, -0.0747806727, -0.0309175961, 0.00489270315, -0.0131097594, -0.0430054143, 0.0138997035, 0.0760298595, -0.025494555, -0.0186249912, -0.0326025039, 3.989450e-02, -0.0445730388, 0.0227416623, -0.0808192044, -0.0704274774, 5.781410e-02, -0.0357359163, -0.0335510857, -0.075161092, -0.0817079842, 0.0685931295, 3.178600e-02, 0.0531890616, 0.105539821, -0.0531192049, 0.0144994026, 0.0655493811, -0.00458282279, 0.0189412273, 0.0839469582, 0.0356850252, -0.0253358744, 0.0520476811, -0.0550689027, -0.0486680567, -0.0376127623, 1.49638174E-4, -0.0722318962, 0.0716672837, 0.0111211706, -0.0517850034, 0.100522615, -0.0128258038, -0.0493322425, 0.0150696179, 0.0949476957, -0.0668484866, 0.0322240852, 0.0617798679, 0.0462278798, 0.0758842379, 0.0414883271, -0.104157053, -0.0432984307, 0.00711138034, -0.00556437066, 0.0907948315, 0.0510175191, -0.0488301069, -0.0392214805, 0.0229295809, 0.0738546848, -0.0187902451, -0.0216048416, -9.231450e-02, 0.0968051627, -0.0188413579, -0.0589839108, 0.016234735, -0.0136974547, -0.0313235633, 0.0562858619, 0.0393002257, 0.0863718464, 0.0203291252, 0.00369753083, -0.0256121699, 0.0302786827, -0.0604674853, -0.0113609871, -0.0983970388, 0.0348180681, 0.0247557163, 0.0120340139, 0.0121031255], [-0.0470673107, 0.0368886515, -0.0704247951, 0.0366023257, 0.04125911, 0.0304907691, -0.0543601066, 0.0813162774, 0.0274442881, -0.0890123397, 0.00125140883, -0.00490507809, 0.0182030629, -0.0605730452, -0.0220485646, 0.0320971832, -0.0717822313, 0.0333862863, -0.0755381137, -0.0677796602, 0.0365922973, 0.0246380772, 0.0182066038, -0.0469471142, 0.0614975988, 0.0254365113, 0.0171916522, 0.00488319201, -0.03498834, -0.0552350171, 0.0555255935, 0.061023172, 0.0910711288, -0.0377193615, 0.0247193892, -0.0672695413, -0.0275604855, -0.0430268086, 0.0782821327, -3.68722802E-4, 0.0348249786, 0.0742791519, -0.067154169, -0.024516603, -1.41392942E-4, 0.00514850626, -0.0382202268, -0.00260905479, 0.0152020566, 0.00808284245, 0.0583249964, 0.0333198532, -0.0288556982, 0.123154342, 0.0107309129, -0.0437126085, 0.0217124671, -0.0171679314, 0.00547760352, 0.0224683136, 0.0963093787, 0.0785661563, 0.0434784479, 0.0568069071, 0.109333254, -0.0579512753, -0.0635569095, -0.0771869794, -0.0663535744, -0.00915324315, 0.0570706166, 0.0596649386, -0.0562232956, 0.0433016829, -0.0278095882, -0.0319008082, -0.0045429389, 0.0469521061, -0.0338001028, 0.0261104237, -0.0252987519, -0.0275633316, -0.0264230929, -0.0359069183, 0.0889803543, 0.0551310815, 0.0411649272, -0.0530308448, 0.083727777, -0.0405931957, -0.0657957494, -0.00308470149, -0.0415714122, -0.0377681255, 0.0422395244, -0.0505414233, 0.0405287296, -0.0066966475, 0.00118625304, -0.0280211102, 0.0358631536, -0.00197537243, 0.0515099354, -5.129990e-02, 0.064698711, 0.0200544782, 0.0862065554, -0.0651186332, -0.0328088216, 0.0257553086, -0.0445413366, -0.0778223201, -0.0174787827, 0.0128768608, -0.02556023, 0.0746523067, 0.0532919616, -0.0376781859, 0.0913079306, -0.019420702, 0.0609959401, -0.0336978547, -0.0360329822, 0.0375577025, -0.065322049, 0.113857783, -0.0640376583, -0.00475045247], [-0.027165506, 0.0676255897, -0.0529580452, -0.04989421, 0.0176432282, 0.0215930436, 0.0309249572, 0.0838143751, -0.0197076984, 0.00120870513, 0.00445766374, 0.0247865934, -0.0303165838, 0.055334013, -0.011638226, -0.0515615866, 0.0355399586, -0.0393023118, 2.124840e-02, -0.0735912472, 0.026193751, -0.0201611165, -0.067978777, 3.96241318E-4, 0.0676735118, 0.0439599156, -0.0277008936, 0.037785504, 0.0672005415, 0.0350448377, -0.053661406, 5.190530e-02, 0.0932732447, -0.0944602042, 0.046078451, 0.00920139066, 0.0361319855, -0.0371920131, 0.00907535385, 0.0553392842, 0.00653979741, 0.0242879484, -0.0745413527, 0.0745010152, -0.0155811524, 6.899840e-02, 0.0988260284, -0.0103787221, 0.06467399, -6.94566697E-4, -0.0404335447, 0.0207665302, 0.0951551795, 0.0892873853, 0.0883515477, 0.00984970201, 0.0360205919, 0.067883119, -0.0607765168, -0.00210892386, -0.0477450714, -0.0140688075, -0.0884736701, 0.0722891092, 0.022399988, -0.0260760803, 4.35851922E-4, -0.0600544512, 0.0260964707, -3.151460e-02, 0.0490076132, -0.019378569, 0.0252411328, -0.00497180875, 0.0536195673, -0.00356944115, 0.072476469, -0.00876276381, 0.0583599322, -0.0295970682, 0.00353629095, -5.374410e-02, -0.0845920667, -0.0176518317, 0.0562124662, -0.060236834, -0.0683040618, -0.00551532907, -0.00225351332, -0.00382849528, -0.0797050818, -0.00453147152, -0.0270976443, -6.505700e-02, -0.0639252588, -4.541900e-02, 0.0229796395, 0.0917832702, 0.0692610815, 0.0477434136, -0.0599121787, -0.030242525, -0.046736598, 0.0568419322, -4.585590e-03, -0.0366290435, -0.0162095185, -0.0287274458, 0.0407262556, 0.0508402921, 4.84400545E-4, -0.074028559, -0.0107847936, -9.60106321E-4, -0.0311661623, -0.0355206504, -0.0193437636, 0.0427431837, 0.092642419, 0.0234198086, -0.0848467573, 0.0697998852, 0.00389439333, 0.0315898471, -0.0191436522, 0.103985079, 7.551950e-04, 0.0531158894], [0.0511629507, 2.742190e-03, 0.00797477737, 0.0392808802, 0.0983561724, -0.00778680574, 0.0109133162, 0.0824625641, -0.0672887861, 0.0607851595, 0.0773029178, 0.0372631364, 0.00110214343, -0.0382479094, -0.0603981502, 0.066407755, -0.071750693, -0.0402319282, -0.0342097245, -0.0700682476, -0.00561367674, 0.0706115067, 0.0655564666, -0.0367963426, -0.0784313678, -0.050702773, -0.0212741215, -0.0593492761, 0.0122832898, -0.0575074963, -0.0461369976, -0.0258998796, 0.00281475019, 0.0394662172, 0.0134941163, 0.02861467, 0.0315858647, 0.0146474801, -0.0032263021, 0.0899004116, 9.206450e-02, -0.0558974445, -0.051510036, 0.0243092105, 0.0130707361, -0.00354070193, 0.0773844868, -0.0232732035, -0.026943069, -0.0231353883, 0.00769223086, -0.00526806246, 0.0785455629, -0.025454184, -0.0242498871, -0.0692091435, -0.0636342168, 0.0658389106, 0.050414715, 0.0516844764, 0.0171413701, 0.0218015928, -0.0303834714, 0.0356544852, 0.0917324423, -0.0372048095, 0.0654550492, -0.0648046508, 0.0696397349, 0.00816051289, 0.0588863045, 0.089432232, -0.0196086559, 0.0105688386, -0.0475320294, 0.0324030891, -0.00145594915, -0.0424375795, 0.0174203794, 5.709540e-02, -0.0369903333, -0.0581370965, -0.0850469693, 0.0151171703, 0.0520085581, 0.0445719212, -0.025022842, 0.00152676052, 0.0517792925, 0.0738643482, 0.0383381583, 0.0350691117, -0.011159732, 2.63249123E-4, 0.0171766356, 0.0411424264, 0.0332884677, 0.0465359315, 0.0290556494, 0.0551454835, 0.00186910946, 0.0161955468, 0.0531358682, -0.0874761566, -0.0332956724, -0.00921323709, 0.0449098349, -0.00978285354, -0.0239784084, -0.0244503766, -0.0163659286, -0.0851309448, -0.0417703055, 0.0342465416, -0.0867887884, 0.0267130435, 0.0505117364, 0.0509411246, 0.109574951, 0.0192001145, -0.0348922536, -0.0653229281, -0.0886379182, 0.113401838, 0.0343141891, -0.0339808092, 0.0379289277, 0.0280906279], [-5.913190e-02, 0.109133534, -0.0347967297, 0.0175477602, 0.0373908766, 0.0229876433, -0.0265207663, 0.0556719974, 0.0467009731, -0.0037625609, -0.0463293307, -0.0446615219, 0.0141330687, -0.0402920581, -0.00160577241, 0.0686024502, -0.075276427, -0.0595047399, 0.0273619052, -1.056520e-02, 0.067449905, 0.0810951292, 0.00853903685, 0.0553034954, 0.0599020831, -0.0845282152, 0.11510402, 0.0687623248, 0.0488661751, 0.0347895622, 0.00374320126, -0.0204398949, 0.0120650632, 0.0289166607, 0.0452289246, -0.0306144264, -0.0489640646, -0.0129450168, 0.0225039981, 0.0997676625, 0.00216958439, -0.0116324089, -0.0282937139, -0.0220252015, -0.0476517528, 0.00514917634, 0.0492931157, -0.00790512655, 0.0694959313, -0.0548323654, -0.0548754856, 0.0128909498, 0.0133491382, -0.0348263979, -0.0722024142, -0.0674143881, -0.0552602299, 0.0210471358, -0.0755792558, 0.0614457689, 0.0486944877, 0.0215210933, -0.0257280469, -5.756510e-02, -0.0385460854, -0.0524612479, -0.01048603, 0.0675479323, 0.08050742, -0.00944822375, 4.1053765E-5, 0.0447952189, 0.00628046924, 0.0215027947, 4.830540e-02, 0.0686162859, 0.0122552346, -0.047871653, -0.0563958623, -0.0163508989, -0.0346979573, 0.0517891832, -0.00494338898, -0.0880565866, 0.109300628, -0.0640110299, -0.0434684195, -0.0746532306, -0.0225413553, 0.0199312642, 0.0463371612, 0.111332841, 0.0334020033, -0.0567908548, -0.0380008556, -0.0152818635, 0.0204114839, 0.0485351682, 0.00377343129, 0.0635406449, 0.039996136, 0.035035681, 0.046948202, -0.00535603659, 0.0309082419, 0.0252314452, 0.0163739715, 0.0604134053, -5.411470e-02, 0.092628695, -0.00620455714, 0.0524740405, -0.016718097, 0.0627731681, -0.0826278552, -0.0252492726, 0.0723744556, -0.0191910975, 0.0729720369, -0.0794444382, -0.0106140319, 0.0372324511, -0.0958773568, 0.00578661356, -7.814420e-03, -0.0639603734, 0.0567723103, 0.012426164], [-0.0132296523, -0.00532169035, -0.00242558564, 0.0309180897, 0.0142113604, -0.0079557905, 0.0589300506, 0.0338876173, -0.0179305952, -0.0335417539, 0.0887420848, 0.0248507373, -0.0490725748, 0.0631095394, -0.0169729665, 0.00217818795, -0.0334283449, 0.0761689618, 0.0290508252, -0.0381119139, -0.0459542945, 0.0497935377, 0.0393114425, 0.0062302649, -0.0679115802, 0.0161522143, 0.089796327, 0.0708107203, -0.0796931087, -0.00524619501, -0.0570910126, 0.0273405258, 0.124148227, 0.0385439694, 0.0419911593, -0.0164710637, 0.0369284451, 0.0162269231, 0.0372145325, 0.0554304384, 0.0932021141, -0.0450689234, -0.0399698205, 0.0672452077, 0.0516839065, -0.0365156755, 0.0832458809, 0.0249838177, 0.0988855361, -0.0470827334, 0.0563309565, -0.0138786025, 0.0141242603, 0.024573572, -0.0480413474, -0.0398724824, 0.0524932742, -0.00312387361, -0.0369068794, 0.0199709926, 0.0469472744, -0.0447460786, -0.109621286, -0.0282418318, -0.0430410355, -0.0640996248, -0.00206378917, 0.00462160492, 0.0229540095, 0.00787628069, -0.0188940894, 0.0910805165, 0.063854523, -0.0186882373, 0.0490915515, -0.0372619666, 0.0448509865, -0.0705229566, 0.0215404388, 0.0885187685, 0.0399920344, 0.0241819099, -0.0146292904, 0.0522317812, 0.10664048, -0.0122688068, 0.0693597495, 0.0225419607, 0.0346746445, 0.0439807214, -0.0228644833, -0.035764806, 0.077220805, 0.0883287936, 0.0764301047, -0.0395907052, -0.0252207462, 0.0129314438, -9.312400e-02, -0.00335984351, -0.0875149369, 0.0734444782, -0.056454055, 0.0593773611, 0.0674014762, -0.0115032243, 0.00321904244, 0.0558759719, -6.705470e-02, 0.0176259391, 0.0851844251, 4.01864643E-4, 0.0704082921, -0.0520352423, 0.00231092749, -0.0185111277, -0.0308493245, -0.0509768911, -0.0195612404, -0.0267173685, 0.0132243931, -0.0203189608, -0.0250133649, 0.0696224868, 0.0632910579, 0.00448372075, -8.595990e-03, -0.0400341041], [-0.0364471078, 0.0314058959, 0.0208657514, -0.00923583657, 0.0793084502, -0.0620722286, 0.0748707876, -0.0462432317, -0.0522292294, 0.0206524581, -0.0488271266, 0.0148434751, 0.0595981888, 0.0573261641, 0.0377330333, -0.0291256495, -0.0325231142, -0.049741488, -0.0287191309, -0.0390654653, -0.0362151377, -0.0733483508, -0.0623025522, 0.0759794191, 0.0790880098, 0.0409274623, 0.101646267, 0.0561440773, -0.0588812865, -0.0469838753, -0.058935862, 3.482220e-02, 0.0820337831, 0.0240416527, 0.0288638696, 5.476930e-02, -0.0218507592, 0.0539104901, -0.0335850753, 0.0606628917, 2.566720e-02, -0.0244704913, 0.0685028285, 0.0198990367, -0.0560600907, 0.00127908692, -0.0282168426, 0.031609308, 0.0108194808, 0.0398575328, 0.0160816815, 0.0123592345, -0.070442073, 0.019177705, -0.086245954, -0.0363561027, -0.0741972551, -0.0402793773, 6.433660e-02, 9.466130e-02, -0.0924896821, 0.00293068471, 0.0137510505, -0.0697806179, -0.0559006929, -0.0589574948, -0.0011355204, -0.0517555922, 0.0735285208, 0.071424216, -0.00540264277, 0.0529587269, 7.789950e-02, 0.00118204951, 0.0798220857, 0.00939603149, 0.0363950394, -0.0747147799, -0.0263971388, -0.050396312, -0.0378047153, 0.00238567428, -0.0187620185, -0.0259418618, -0.0592958704, -0.0349021554, -0.0465182625, -0.0959157497, -0.0476682857, -0.0683558583, -0.0291632246, 0.0525994338, 0.0450159088, 0.0333798267, 0.0211179052, 0.00697283912, -0.0284808036, -5.348890e-02, -0.0503591262, -0.00176844443, 0.0309522469, 5.813670e-02, -0.0517772958, 0.0639947354, 0.00981730595, -0.0243119411, -0.0080280574, -0.0553164333, 0.0554517508, -0.0517705157, -0.00307905255, -0.0427238606, 0.0859967544, -0.00977692287, -0.064751707, 0.04735443, 0.0637081563, -0.0443163402, 0.0850821956, 0.025816435, -0.0237248689, 0.0725765675, -4.549760e-02, 0.0751598924, 0.0375461429, -0.0688020959, 0.0896552056, 0.0360370837], [0.0497979745, -0.0676559955, 5.940510e-02, 0.0832022726, 0.0129108857, 0.0198408198, 5.134810e-02, -0.00766955549, -0.0695131272, 0.0855167806, -0.0350378044, 0.0141087594, 0.0320809856, 0.083279714, -0.00625617802, 0.0619966164, -0.0321317539, -0.0385799594, -0.0605288073, 0.00146472012, 3.933240e-02, 0.0091595212, 0.0799475237, -0.0752064139, 5.48452139E-4, -0.0138781052, 0.0858367756, -0.00791416597, 0.0121722985, 0.0588975586, 0.0710765049, 0.00634496472, 0.0931405723, 0.0350667872, 0.0538274124, 0.0906827301, 0.0397055149, 0.0563408248, 0.0448453054, 0.02753474, 0.0366333351, 0.0253735408, -0.0342172273, -0.0451056771, 0.0575338416, -0.0391465202, -2.240410e-02, 0.00883249473, 0.0254930072, -0.019907929, 0.044010032, -0.00786449481, 0.0438652895, 0.0455975831, -0.0281859953, 0.0744304656, 0.0280789062, 0.0090462314, 0.0264239702, 0.0573997535, -0.0624678507, 0.00129911874, -0.0183580164, 0.00183248706, 0.0118902689, -0.0199147779, -0.0308121964, -0.0229812283, 0.0725824907, 0.0129449675, 0.0403545052, 0.0530917495, -0.0119075598, 8.772210e-02, -0.0412870981, 0.031981945, 0.00220062467, -0.0182257481, -6.014050e-02, -0.0558891632, 0.0678802356, 0.0455700904, 0.0308310557, -0.0493470356, 0.020861974, -0.0450376943, 0.0709568337, 0.0423464216, 0.00416984782, 0.0486949794, -0.0735133141, 0.0200218409, -0.0552688912, 0.0176779479, 0.0157155488, 0.0821175351, 4.840030e-02, 0.0668453202, -0.0392301828, -0.00126702758, -8.611720e-03, -8.780190e-02, -0.0661314651, 0.0838327259, -0.0589414425, -0.0493932329, 0.0922059342, 0.0604202524, -0.087822698, -0.0252792127, -0.00279584946, -0.0371348076, 0.0231974386, 0.0558459088, -0.0656995922, 0.0640946627, -0.0462463461, -0.078257285, 0.0892944783, -4.940980e-02, -0.0993736609, -0.00130231876, -0.0644144267, -0.00996731407, -0.0285166688, 4.346960e-02, 0.0804253593, -0.0695519447], [-0.0572845861, 0.047001075, 0.0124404225, 0.0200670622, 0.026032893, -0.0246560536, -0.0151224639, 0.0715400428, -0.0115895355, 0.064536646, 0.0104465354, -0.0483852327, 0.0484421551, 0.0506549105, -0.00697580306, -0.0386326686, -0.0664482414, 0.0690987185, -0.0246389974, -0.0614822507, 5.899230e-02, 0.0202165972, -0.0767342672, -0.00433761301, 0.00465168525, 0.00103441428, 0.0153938252, 0.0191855021, 0.0807412863, 0.0520367622, -0.0806398392, -0.00351429358, 0.119386643, -0.0460169688, 0.0621848367, 0.103365965, 6.64163497E-4, -0.0209047198, 0.0158840902, 0.085341759, -0.0162434429, -0.00215600943, 0.0297558829, -0.0747906491, -0.0557937324, -0.0806387141, 8.840660e-02, 0.0230420176, 0.0325886086, -0.00516324164, -0.0682652667, 0.111798011, 0.0199258085, -0.0129863685, -0.0256067216, 0.0324622802, 5.209480e-04, 0.0536770858, 0.0163585953, -0.0487567596, 0.0514241084, -0.0499095842, 0.0280007422, 0.00677337311, -0.0381875299, -0.0495558344, 0.0727657154, -0.0237695184, -0.0394014604, -0.0113312788, -0.0467485674, 0.00270423363, 0.0913999826, -0.023155367, 0.0104353447, -0.0142859593, 0.0766194239, -0.0251010507, -0.0499131493, 0.0989156886, -0.0154223246, 0.0696488916, -0.0750205218, 8.36519524E-4, 0.0626568571, 0.0153475711, -0.00471883826, 0.0199509859, 0.0717629045, 0.00908127893, -0.00470820675, -0.037431851, 0.0140060848, 0.0251635239, 0.0284633432, 0.0619613379, -0.102775887, -0.0946967751, -0.00106754014, -0.0225679241, -0.0408187732, -0.0211169738, 0.0229622237, 0.0142647186, -0.0893609375, -0.0424587168, -0.0576224849, -0.0612214319, -0.0825659186, 0.012033551, 0.101369254, -0.0706993416, -0.0707900748, 3.387090e-03, -0.0146391112, 0.0489937663, 0.0799340903, -0.067791827, 0.00845113117, -0.0419682674, 0.0333167836, -0.0586993061, -0.0110616051, -0.0384696461, -0.0699712709, 0.0159919746, 0.0144705931, -0.0327316187], [0.0768968314, 0.0385247283, 0.0566374399, 0.0966625884, 0.0191981904, -0.00587901659, 0.0525693074, -0.00700243469, 0.0600001141, -3.98853037E-4, 0.07267382, 0.0359751359, 0.0528032929, 0.041179087, -0.0504566766, 0.0212311745, 0.08012034, 0.0477616861, 0.044267565, 0.0278852489, 0.0741803348, -0.101925448, -0.00377128785, -0.0657479167, 0.017423097, -0.0247187894, 0.0878841132, 0.0311838537, -0.0360379629, -0.00880314875, 0.0471262261, 0.0117156813, 0.105704218, 0.00666521396, -0.0624488816, 0.083140321, -0.0355921797, 0.0285131671, -0.0466057882, 0.0644391328, -0.0222458784, 0.0722221807, 0.0339608192, 0.0117786964, 0.0400863402, 0.0271804314, 6.770720e-02, 0.0168867651, 0.0914244353, 0.0190347135, -0.0367715694, 9.74149152E-4, -0.0737090558, -0.0403644107, 0.05170957, -0.0618667714, -0.0057423166, 0.038866505, -0.0368207283, -0.0628674924, -0.0632831901, 0.0744810402, -0.0540380627, -0.0268874019, -0.0961617231, -0.0644967631, -0.0453289114, -0.0526667386, -0.0367720909, -0.10113278, -0.0946303978, -0.0211984962, -0.0391545892, 0.104971819, -0.00447213138, -0.0596456155, -0.0576865114, 0.0746653304, 0.024788538, -0.0182859451, -8.912300e-02, -0.061019741, -0.0385652035, 0.00521736685, 0.0721960887, 0.0202600397, 7.95293308E-4, -0.0710863695, -0.0622200183, 3.51962954E-4, -0.0518869869, 0.0719850883, -0.0330472477, 0.0317979641, -0.015959017, -0.0337952264, -0.00244324026, -2.421310e-03, -0.074309796, -0.0521042943, -0.0391019769, 0.0330916941, -0.0502343439, -0.0313961916, -0.0537094772, 0.0767866895, 0.0596522838, 0.0748179704, 0.0447796062, 0.0071962676, 0.0867452695, 0.0305896979, 0.0279845782, 0.0513299629, -0.026862964, -0.00159460376, 0.0555052683, -0.0318200663, 0.0320197903, -0.0721428767, -0.0287019797, -0.0773682594, -0.11106506, -0.00758449174, 0.0671115815, 0.0396034531, 0.0344798639, -0.038766332], [0.011996191, -0.0583675615, 0.0679484903, 0.0560730472, -0.0516699888, 0.0587781705, -0.0401087105, 0.0548602305, 0.0500456579, 0.0415841788, -0.0455001593, -0.0376628526, -7.358860e-02, 0.114537805, 0.0494792908, 0.0130686201, -0.0649807751, -0.0666226894, 0.090096727, -0.0626801401, 0.0337894596, 0.00681406492, -0.0126463259, 0.0143979611, 0.0136498483, 0.0749092251, -0.0431517437, -0.0247195773, 0.0455285609, -0.0294138584, -0.0678468868, -0.00961147156, 0.0154983029, -0.0248189364, -0.0313021615, -0.0306032039, 0.0112311933, -0.0677805915, 0.0509892926, -0.0312173627, 0.052721478, -0.0825735926, -0.048007071, -0.0708794594, 0.0281780828, -0.0244330931, 0.0339302607, 0.0180109739, 0.0628507584, 0.0577144809, 0.0396206565, -0.00347732729, 0.0296968333, 0.0723961219, 2.913990e-02, 0.067964457, 0.0339513086, 0.0645842105, 0.0516627803, -0.0608256496, -0.0554536954, -0.0618278645, 0.0325089619, 0.0445728563, -0.0323499963, 0.0039799707, 0.083755061, -0.0517120026, -0.0884129256, -0.0391088054, 4.492940e-02, -3.357950e-02, 0.0603651628, 0.0163078848, -0.0255726781, 0.0502968803, 0.0456305519, -0.0312971584, -0.0131003205, 0.0282372907, 0.0282290634, 0.0265117139, 0.0129459314, 0.00162543333, 0.00845636986, 0.0564742871, -0.0531586409, 0.0111687342, -0.031925682, -0.0330584832, 0.0315253139, -0.0470685586, -0.0463833921, -0.0379829444, -0.0244306717, -0.0307639744, 0.027576508, -0.0797608196, -0.0770166889, 0.072430253, -0.0811020806, 0.0298024341, -0.0131492484, -0.0595334917, -0.0115363803, -0.0547287799, 0.0493794642, 0.0445355698, 0.0134027349, -0.0148730194, 0.0444736034, 0.0750769824, 0.0396648645, -0.0333719701, 8.222850e-02, -0.0140106892, 0.0382784531, 0.0556523688, -0.0370886028, 0.0761283115, -0.0346081518, -0.0335058905, -0.0212799944, -0.0682813153, 0.0606969669, 0.0295772161, 0.0574190244, 0.0687419698], [-0.0354261473, -0.00138615514, 0.00303645432, 0.0163895581, -0.0528406836, 0.0501350574, 0.0600020178, 0.0744280145, -0.0209481232, 0.0337635353, -0.0247813985, 0.0141040366, 0.0670792535, 0.070386216, 0.056390781, 0.0733839124, -0.0419156551, 0.00152924494, 0.010797929, 0.0227813907, -0.0703531578, 0.0202003885, 0.00855622534, -7.480950e-02, -0.0213082973, -0.0584815107, -0.0661620498, -0.0435738638, -0.0618311204, 0.0489074439, -0.0608128086, -0.0418516174, 0.082982324, 0.0301446021, 0.0301713683, 0.0217151195, -0.0450101122, 0.0169179477, 0.0159846134, -0.0252581127, -0.0744129047, -0.00868379604, -0.0340645425, 0.0164085925, -0.0135494871, -0.0719611123, -0.0504508503, 0.0368262902, -0.00756845763, 0.0799014791, -0.0747884959, -0.0399825498, -0.066476792, 0.0155080864, -0.0145592513, -0.0432949886, -0.0585373528, -0.0199995153, -0.0740426853, -0.0292645693, 0.0846192464, -0.0337810777, 0.0118792141, 0.0511334613, -0.0367525183, -0.0687597767, -3.260100e-02, -0.033339154, -0.0727738962, 0.0487693362, -0.0286444891, 0.00535120023, -0.0552111194, 0.0948865637, -0.0268751103, 0.0578085147, 0.0114395367, 0.0499108918, -0.0188990533, 0.0766471699, 0.0385054946, 0.0419402756, 0.0546318777, 0.0307836309, -0.0391413756, -0.0784247517, 0.02806307, 0.0424456447, 0.0354302712, -0.0503282063, -0.0467457771, 0.0515924655, -0.0427460819, -0.0569926575, 0.0566730089, 0.0410151184, 0.0722717419, 0.0207864717, 0.0730676875, 0.00495449407, -0.00336470595, -0.0889822319, 0.0157470666, -0.00681071169, -0.00494355243, 0.0555443838, 0.0682853386, 0.058870621, -0.0186934546, -0.0291918516, 0.0328644104, 0.0236552302, 0.0282195956, 0.0636412203, 0.0347861797, -0.0383779332, 0.0127231292, 0.0623296312, 0.0274961237, 0.0244371407, -0.0225485861, 7.66327314E-4, 0.0555545315, -0.0378661565, -7.381430e-02, 0.0572638176, -0.0207598805, 0.0562422425], [0.0556535162, 0.0684320331, 0.0261650942, -6.528060e-02, 0.0376547091, 0.001358315, 0.0366878062, 0.0266624987, -0.0862399265, -9.39494174E-4, -0.062176045, -0.0442166254, -0.031895414, -0.0328743793, 0.0156908873, 0.0598178841, 0.00882709212, -0.0780888423, -0.038775105, 0.0288407896, 0.0186543688, 8.756060e-03, 0.0497902595, -0.0407714285, -0.079624027, 0.00364172109, 0.0267555322, -0.0518336631, 0.0181062687, -0.0160514954, -0.0306144711, 0.0598899499, 0.0326153897, -0.0803553909, 0.0253536217, -0.0430549458, -0.00869427342, -0.0432016551, -0.0328746065, 0.0174252689, 0.0179755576, 0.0370586626, -0.0455434285, 0.0176832806, -0.0474831909, 0.0570100397, 0.0154040949, -0.0505415685, -0.0644048229, -0.00504238624, 0.00866403244, 0.0301697385, 0.0644592717, -0.0311536714, 0.0408798754, 0.0428595915, -0.0746143609, 0.0496507771, 0.0379083529, -0.0512818694, -0.011797349, 0.0157673657, -0.056288749, 0.0499842502, 0.00290503795, -0.0225494597, 0.0250213239, -0.0375498943, -7.586630e-02, 0.0243191142, -0.0842199773, 0.0504011214, -0.0131736081, 0.00390835898, -0.0206832942, -0.0647568404, 0.0412432291, -0.0308847986, -5.845140e-02, 0.00534862792, 6.04483066E-4, 0.03203585, -0.0453459024, -0.0136545384, 0.0393494405, 0.0356402807, 0.0751041248, 0.0143015012, -0.0779860839, -0.0420156196, 0.0371989422, 5.673790e-02, 0.0711633265, 0.0338396765, 0.0375175625, -0.059319295, 0.0325599201, 0.0252752695, -0.0648186132, 0.0547128431, 0.00298373308, -0.0767978131, 0.0514568165, -0.0112977838, 0.0767534077, -0.0494584441, 0.0738390535, 0.0696433634, -0.0774624795, 0.0409990065, -0.010658497, 0.0679331496, -0.0315245911, -0.0606020577, 0.0815260931, -0.00228986563, 0.015575829, -0.00965959299, -0.0513819642, 0.0295429863, 0.0188617818, 0.0159031581, -0.0739844814, 0.0719448254, -0.0206595417, -0.0143792573, 0.0593185574, -0.0271171182], [0.00110967492, 0.0645477623, 0.0364591219, 0.0238062795, -0.0375264473, -0.0372124277, -0.00570396567, -0.0571509413, 0.0615368038, -0.0661231354, -0.0634727404, 0.0175971873, -0.0154540371, 0.0448738374, -0.0437339209, 0.0405798592, -0.00301815057, -0.0556914173, 0.0806328728, 0.00808238704, -0.0579915866, -0.0335117318, 0.0299172103, -0.0153173544, -0.0796887576, -0.0361315683, -0.00756314723, 0.0718582422, -0.0770730674, 0.070955433, -0.0646136403, 0.0254632793, 0.0550760813, 0.02360286, 0.0148514519, -0.0417887866, -0.0628226101, 0.0225166306, -0.0755174384, -0.0174808726, 0.0437303968, -0.0269975048, -0.0322951712, -0.00360357948, 0.0327954665, 0.0738045573, 0.0483027808, -0.0171565544, 0.0390739441, -0.0192957427, 0.00989401806, -0.00564150605, 0.0644512549, -0.0142001808, -0.0255338959, -0.0527133532, -0.0352980606, 0.0180216711, -0.0790560841, 0.0208887346, 0.00716950046, 0.0200230852, 0.0169350468, 0.0270175356, -0.0501247868, 0.0802033097, -0.0555010326, 0.0664280653, -0.0256578289, -0.0571564473, 0.0251374207, -0.0230453778, -0.00492156204, 2.008380e-03, -0.0678857937, 0.0491672866, 0.0301542245, 0.00380720873, -0.0143875936, -0.00209045294, 0.061594855, 0.050089322, 0.0809022858, 3.762140e-02, 0.00382611016, -0.0601824485, -0.00103583711, 0.0429092795, 0.0736321508, 0.069978632, 0.0283655655, 0.0147367055, 0.0365047157, 0.0102770589, -0.0481217876, 0.0087360572, 0.0534274429, 0.01270069, -0.00444752444, 0.0363267735, 0.053948611, -0.0272735562, 0.0037494204, 0.00909311883, -0.0110219074, -0.0356416069, -0.0639640912, 0.0391305387, -0.0296275746, -3.763540e-02, 0.0647988543, 0.00927669368, -0.051254835, 3.969340e-02, -0.0197665058, 0.0445996858, -0.0365587026, -0.0688720047, 0.0380609706, 0.0628171861, 0.0265186504, 0.0635046735, -0.0348698199, -0.0514420718, -4.729730e-02, -0.0576719306, -6.101680e-02, -0.0582706966], [0.0566044711, -0.0779167488, -0.0691529289, 0.0733776986, -6.369080e-02, -0.0022207303, -0.028217921, 0.0649259835, -0.069786936, -0.0504852049, -0.0490401573, 0.0525110587, 0.00203157705, 0.0232782308, -0.0150102042, -0.0574093312, -0.0475056022, 0.0419186428, -6.50070375E-4, -0.0336297266, 9.57556068E-4, -5.615600e-02, -0.0654869973, 0.00238608196, 0.0624511949, 0.0798105373, 0.022150429, -0.0338658132, -0.067945458, 0.0392740034, 0.052231323, -0.0347138643, -0.0233284943, -0.0463235229, 0.0714798346, 0.0238134339, 0.0561613329, -0.02199471, 0.0268724188, -0.0580054931, -0.0578452349, -0.00330560259, -0.0323150381, 0.0650561675, -0.0185307264, -0.0700220317, -0.00158052926, 0.0533173457, -0.0332690179, 0.0452296101, 0.0334380157, 0.014211636, -0.0065137865, -0.0744021609, -0.0721585527, 0.0207465738, -0.0163243134, -0.0696922392, -0.0563044734, -0.0748525559, 0.044153925, -0.054523468, -0.0574230254, -4.599250e-02, 0.00161031983, -0.0644213334, 2.987560e-02, 0.0253026299, 0.0106663899, -0.0512867048, -0.0267530382, 0.013332733, 0.0720166042, -0.0480170734, 0.020405598, -0.038024772, 0.0508060232, -0.0106354319, 0.0572731681, -4.233120e-03, 0.0748398602, -0.0439303704, 0.0803004652, 0.0757879168, 0.0398136191, 0.0502112471, -0.0104569886, -0.0702757239, 0.0270296205, -0.0364479646, 0.0357987657, 0.0281656515, -0.0286207739, -0.0433514342, -0.0546869971, -0.0619761049, 0.0409088843, 0.0214236882, -0.0133174844, 0.00689390628, -0.0310602114, -0.0525534488, -0.0472908616, 0.0335567482, -0.019445885, 0.0485026352, 0.0763261467, 0.026795784, -0.0183340479, -0.0776834637, 0.0189955626, -0.0323621854, -0.0720094293, -0.0525183789, 0.0421251804, -0.0721520409, -0.0505536385, -0.00484124944, 0.0163399894, 0.0367157646, -0.0558101498, -0.0648074746, -0.0631471053, 0.0130669605, -0.0102400109, 0.0592309944, 0.0133309783, 0.0505291782], [0.038387455, 0.0681297556, -0.0727810636, -0.0361939482, -0.0606332272, 0.026283592, 0.0460293368, -0.0609364323, -0.0612342246, 0.0466499105, -0.0525884777, -0.0398770235, 0.0637891665, -0.073852539, 0.0611233339, -0.0390043072, 0.0219512507, 0.0394132957, 0.0391101614, 0.0303785577, -0.0160359591, 0.0492745265, 0.0511930957, 0.0194137022, 0.00148092955, 0.0494801477, 0.0749659613, -0.0106566027, -0.0096296966, -1.047720e-02, 0.0705760792, -0.00122632086, 0.0324358419, -0.0183025822, -0.0555745214, 0.032076396, -5.235640e-02, -0.0376289673, 0.0347397104, 0.00583214313, 0.00967670977, -0.0167829394, -0.0690418482, -0.0409372263, -0.0641608387, 0.0660997406, 0.0272255391, 0.0126997679, -0.0745416209, 0.0690311566, 0.00173272938, -0.00932842493, -0.0664044768, -0.0506898798, 0.04076121, 0.0186557919, -0.0368978418, 0.0186962262, -0.00703111291, 0.0687448308, 0.0332106128, 0.00751681625, 7.032650e-02, -0.0689957439, -0.0638904124, 8.382030e-03, 0.00639169663, 0.0456863418, -0.00819976627, -0.0377846584, -0.0652713254, -0.00418589264, -4.47973609E-4, 0.0469509289, 0.033766143, 0.00251618028, 0.0659578368, -0.0311510041, -0.0559845902, -5.718720e-02, 0.0399288908, 0.0456173047, -0.0475350507, -0.0364112146, 0.0336974338, 0.00301045179, -0.00467918813, 0.0649556145, 0.023095496, 0.00328120589, -0.011361964, -0.0578348041, 0.00254743546, -0.0307656117, 0.068803139, 0.0274663046, -0.0756916105, 0.00588292629, -0.0479670689, 0.0675466284, -0.0304940417, 0.0789598599, -0.00255300105, -0.0725402236, 0.00736876577, 0.0561671481, -0.0226954669, 0.021165885, -0.0452171415, -0.0560321435, 0.0796585902, 0.0177353099, -0.0556877106, 6.791400e-02, 0.0694648698, -0.0620161071, 0.00259349495, 0.0193538442, 0.0536062345, 2.80752778E-4, 0.0419399962, 0.0667176768, -0.0232250094, 0.0182327479, -0.0279673189, -0.0316103064, -0.0527444594, 0.0514981672], [7.964100e-02, 0.0475033857, 0.0175626874, 5.013150e-02, 0.0138115846, 5.689620e-02, -0.0355717577, 0.0249698386, -0.062333785, 0.0193925183, -0.0303546954, 0.0626144707, -0.0295498017, -0.0383017845, 0.0575277247, 0.0228232294, -0.0669521913, -7.059150e-02, 0.0596544817, -0.0100004589, -0.0305369943, 0.0410132222, 0.0525534451, 0.00325076561, 0.0759793594, 0.0425461195, 0.0720970928, -0.0578684472, -0.0327242501, -0.0530251339, 0.0630229637, -0.0688706413, 0.0384291708, 0.0767887607, -0.00652825693, 5.346360e-02, -0.0780821219, 0.0706500411, -0.026777463, 0.0254965369, 0.0200384967, 0.0103437603, 2.421660e-02, -0.0133110816, -0.0756630376, 0.0374006294, 0.0224820971, 0.0475828499, -0.0726022422, -0.00629906869, 0.0565279089, -0.0609404072, -0.0430548415, -4.583730e-03, -0.0332311243, -0.0557412468, 0.0561747439, -0.059501715, -4.718140e-02, 0.0226857215, 0.0433907919, 0.0694618151, -0.0623620376, -0.0329017267, -0.0548603721, 0.0667765811, -0.038078215, -0.0192695893, 7.969280e-02, -0.0540779196, -0.0787729397, -0.0410072394, -0.0697198585, -0.035420347, 0.0756564438, -0.047923509, 0.0125026936, 6.313900e-02, 0.0329482704, 0.0753991678, 0.0780110955, 0.0725012943, 0.00466148462, -0.0227807965, 0.0255825408, -0.0675456077, -0.0303559136, -0.0629710257, -0.0279328022, -0.0228859652, 8.033280e-02, 0.0299076885, -0.0148514435, -0.0183232017, 0.0512835234, 0.0408735424, -0.0679614469, 0.0395628922, -0.0195066407, -0.00127828901, 0.0285557639, -0.0675468668, 0.0595621504, 0.0627515614, -0.0729092807, 0.0729196817, -3.176600e-02, -0.0695195794, -0.0505556129, 0.0166370906, -0.0141950408, 0.0484892651, -0.0524799116, 0.00512079848, 0.00763094565, -0.0519457124, 0.0374775678, -0.0202913191, -0.0464353561, -0.0473771282, 0.0608274192, -0.00492669549, 0.0638837814, -0.0800378099, -0.0191934146, 0.0409462526, 0.0587239526, -0.0726751685], [0.0191277061, -0.0109450612, -0.0788464099, -0.0673538297, 0.0471809916, -0.0142252073, -0.0731141344, -0.0142028648, 0.0106344735, -0.00223570294, -0.0491948426, -0.0133095207, 0.00366067467, -0.00788233056, -0.0595644265, -3.17780738E-4, 0.0539160855, 0.0683728158, 0.0160467438, -0.0753554851, 0.0299411193, 6.575870e-03, -0.0267984774, -0.052526556, -4.213870e-02, 0.0490857325, 0.0769823864, -0.0134442886, -0.00322918594, -0.00338256522, 0.0468669347, -0.067714408, -0.0497104675, 0.00675343629, 0.0760876462, 0.040577881, -0.0749002397, -0.0750523731, -0.044799529, -0.0380033851, 6.455220e-02, 0.0743911788, -0.0603892542, -0.0250709727, 0.0534165129, -0.0515638739, 0.00611812295, -0.0483908281, 0.019423224, 0.066994831, -0.0171695109, -0.0712213069, -0.00781245204, 0.00314931083, 0.0552017316, 0.04690082, -0.0535266958, 0.0474202596, 0.0288472082, -0.0432765856, -0.0314230807, 0.0604666732, -0.0662454441, -0.064024739, 0.0384124927, 0.0353233404, -0.0712665096, -0.0798378736, 0.0209360626, -0.0277185515, -0.0323755369, -0.0205953345, -0.0310067032, -0.0732338428, -0.064474821, 0.0537416041, -0.0613073744, -0.025726812, -6.806810e-02, -0.0249033831, -0.0223272983, -0.0716983825, -0.0237959828, 0.0383348614, -0.0481127873, -0.0106707681, 0.00240644859, -0.00779275317, 0.030803185, 6.457180e-02, 0.0606820397, 0.082026191, 0.0548790433, 0.0401970558, 0.0370478891, -0.053691674, -8.13713705E-4, -0.00166437076, -0.0390607044, 0.00737436302, 0.0247239806, -0.0457571819, -0.00282679428, 0.0449304357, -0.0209552012, -0.0051362738, 0.00997314881, -0.0535577461, -0.0209324267, -0.0619258955, -0.00634279894, -0.0698903576, 0.00860639382, -0.0733263642, 0.0589873791, -0.00439673243, 0.0174175091, -7.657670e-02, -0.0227165185, 0.0314781144, 0.0369722508, 0.0757080838, 0.0792521834, -0.0455326363, 0.0475755967, 0.0133752869, -0.0598694198, -0.0388086066], [0.00842616707, -0.0081426492, 0.0415117778, -0.0229879357, -0.0429533347, 4.12253838E-4, 0.017008571, -0.0653748512, 0.0506205373, -0.00605527405, -0.0682619438, 0.0197906885, -0.0245076418, -0.0196904819, 0.0365365297, 0.0305215567, -0.0132275857, 0.0411907434, 0.0265103839, -0.0781107098, 0.00309389783, 0.0142046669, 0.0233132727, 0.0586355776, -0.033074975, -1.6781356E-4, 0.0140583469, 0.0128454175, 0.0680552423, 0.0366446376, 5.148870e-02, -0.0448919088, -0.0508607216, 0.073796913, -0.0205063466, -1.764780e-02, 0.0260847211, 0.0320544206, -0.00648342166, 0.048146192, 0.0858809426, 0.0399978943, 0.074259907, 0.0684948638, 0.0545372367, 6.634890e-02, 0.0189198274, -0.0840803161, 0.0208217502, 0.0763045922, 0.0748512149, -0.0638594106, 0.0400438085, -0.00551884249, 0.0623736791, 0.0462545604, 0.0371207185, 0.0658747851, 0.0109907202, -0.00960382912, -0.0820114091, 0.0799567848, -0.042424418, 0.0720139369, 0.0445027053, -0.0147060575, -0.0537164956, 0.0202052537, -0.0702782944, 0.0481794439, 2.099360e-02, -0.0353047512, 0.0298120733, 0.0631155521, 0.00548389368, -0.0655228868, -0.00709579047, -0.00208876817, -0.00649748137, 0.0301069617, 0.0309063848, -0.0680186525, 0.0696191788, 0.0641226098, -0.020443758, -0.0234100968, -0.0692654699, 0.054229416, -4.459850e-02, 0.0172360055, 0.0378444046, -0.0296788663, 0.0558969714, 0.0793682262, -0.0681350604, 0.040816009, 0.0449100323, 0.0517970622, -0.0515053086, 0.0705731362, -0.0353770778, -0.031437058, -0.0099093914, -0.0323901959, 0.0168696959, 0.0191404354, 0.0686583593, -0.0384973958, -0.0206599236, -3.136440e-02, 0.0653665885, 0.0440667942, 0.0686991587, -0.0293048453, 0.0155719612, -0.0628253892, -0.00573456241, 0.0377198793, 0.0769094378, -0.0691087693, 0.0672721863, -0.0497346036, -0.0533429608, -0.0357840396, -0.0591423474, -0.00782428681, 0.0452650525, -0.0490794331], [0.0626051053, 0.020617282, 0.0320362188, 0.0844229236, -0.056844268, 0.055161003, 0.0894624069, -0.0370235853, -0.0786757543, 0.0258651134, 0.057121966, 0.0256023761, 0.0348752514, 0.00862887222, 0.060770303, 0.0701866746, 0.0102275144, -0.0501862057, -0.0314052701, 0.0464502834, 0.0232201442, -0.0550956614, -0.00667791348, -0.0666068196, 0.0670030937, -0.0153940553, 0.0756812841, 0.0168919507, 0.0613476522, -0.0258001406, -0.0411522426, 5.166520e-02, 0.0339110419, -0.0537156947, 0.0187581386, 0.0548286438, 0.0500129089, 0.0425482392, -0.0778320729, 0.0427378714, 0.00606673816, 0.0759914368, 0.0479412079, 0.0428468771, -0.0434907414, 0.00298990915, 0.0579576045, 0.0270276349, -0.0470986106, -0.070372574, 0.0477820076, 0.0489045531, 0.041354537, -0.0917722284, -0.00367048942, 0.0543044433, -0.0630408227, 0.00136609189, 0.0663133934, 0.0472561717, 0.043910861, 0.0168318618, 0.00816868431, -0.016256256, 0.0176444445, 0.0279056467, 0.0487964191, -0.0181809384, 0.0125844637, -0.0727945194, -0.00683114305, 8.040310e-03, 0.014461128, -2.45507428E-4, 0.0562046804, -0.0690463185, -0.0613454059, 0.0496749096, -0.0766406953, -0.0453518033, -0.0561292395, 0.0735201687, 0.00925343669, -0.0270681269, 0.0565233342, -0.00584239094, 0.0519308336, 0.0529890358, -0.014203825, -0.0627641305, 0.0544270054, 0.0168379135, 0.0173729304, -0.0629706457, -3.286050e-02, 0.0137293376, -0.00324902963, 0.0648504496, 0.0146914516, 0.0533917099, 0.0324863233, 0.0333956592, -7.046290e-02, 0.0799878091, 0.0701203421, 0.0604590811, 0.0401844718, 0.0493775085, -0.0113087567, 0.0131937927, -0.00242283242, -0.071052663, -0.0353990942, 0.0634185299, -0.0214012247, 0.0761266052, 0.00823793187, -0.0305352714, -0.00238434458, -0.058526706, 0.0300609469, 0.0333791524, 0.0174063183, -0.019081112, -0.0168688502, 0.0612728558, 0.0179491099, 0.0409156717], [-0.0765766799, -0.0219936837, -0.0206142217, 0.0782669112, 0.06860964, -0.0393136591, -0.00827910472, 0.104318433, -0.0741052925, 5.558280e-02, 0.0377519093, -0.0655457526, -0.0796962827, 0.109350197, -0.0246991254, 0.0657216608, -0.038471207, 0.0597504526, 0.0666312352, 0.0421244726, -0.0786504372, -0.0611374304, -0.067969948, -0.02637312, -0.0135717168, -0.0152246421, 0.0278804898, -0.0338916071, -0.0292955637, 0.0698921829, 0.0756673366, -0.0748547688, -0.0530855209, -0.0951021686, 0.028234886, -0.00866319239, 0.046264261, -0.0304829683, 0.0130887423, 0.111511961, -0.0606967583, 0.0523622185, -3.922340e-02, -0.0242470186, -0.0787710919, 0.0786212608, -8.920840e-02, -0.00867958087, -0.0329807922, 0.0284139533, 0.055529207, 0.107116908, -0.100196868, 0.0285469908, -0.0374788977, -0.0030199585, 0.0264997669, 0.0292577036, -0.0791893824, 0.0166158043, -0.00593242375, 0.0138973249, -0.0671161041, -0.0176192224, 0.0203381162, 8.40694759E-4, 0.0471307561, -0.0390897542, -6.692400e-02, -0.0478886776, 0.0436548553, -0.0157472044, -0.0559957176, -0.0374656133, 0.0072029368, 0.0472914279, -0.0271986593, -0.0757086873, -0.0433591977, 0.0183338095, -6.330140e-03, 0.0209035724, -0.02647187, 0.0211253129, -0.0413452163, 0.0504430197, -0.0655584037, 0.0784839392, -0.0511777438, 0.0158555452, -0.0241912194, 0.025317343, -0.0232836492, 0.0246913265, 0.0475110896, 7.653140e-02, -0.0727894306, 0.0386240743, -0.026369825, -0.0761285871, -0.029344026, 0.0519905128, -0.0278706681, 0.0860987976, -0.0206140298, -0.0548228398, 0.0401032232, -0.0349905491, 0.0420223661, -0.0079081431, 0.0448731333, -0.0610692464, -0.00969582144, -0.0653078482, -0.0320521854, 5.642830e-02, -0.029379867, -0.0266461186, -0.0410699733, -0.0393944904, 2.03275529E-4, -0.0658358857, 0.0375364609, -0.0211201422, -0.0641496479, -0.0269234404, -0.0048734867, -0.062944375], [0.0676341429, -0.0622866303, 0.0348188058, 0.0336084329, 0.0430821776, -0.0573461428, -0.0731638967, 0.0869750678, -0.0351911746, 0.0154469637, 9.575320e-02, -0.00522777578, 0.0255324394, 0.0573271737, 0.0621050671, -0.0792519971, -6.597870e-02, 0.0106852269, 0.0524522252, 0.0865697786, -0.0411812104, 0.0723608658, -0.0694244057, 0.0262141861, 0.0456645861, -0.0456848294, 0.0207408015, -0.0198507886, -0.0523555689, 0.00486696418, 0.0161373075, 0.0463620462, 0.0482253134, -0.104711041, -0.00505749229, 0.0438322723, -0.0634776801, 0.0280132927, 0.00456272205, 0.112607591, 0.0685407147, 0.0486055128, 0.0327214897, 0.0201001037, -0.1121279, 0.0553047843, 0.0152512491, -0.0736361295, 0.0959655046, 0.0573815331, -0.0180749986, 0.063131839, -0.0494282134, 4.240520e-02, -0.0407141298, 0.0521957092, 0.00644411146, 0.0247013588, -3.587870e-02, -0.0223431513, 0.0606209673, 0.0317292623, -0.0840870812, -0.0282377861, 9.8310539E-4, 0.0501307659, 0.0686627552, 0.0246880408, -0.0324546807, -0.122491889, 0.00158441789, -0.0199313331, -0.0408400521, 0.0754282251, -0.0795181468, 0.0771845207, 0.00522108143, -0.0696113407, -0.0292799529, 0.0771800205, 0.0525259785, -0.00135695632, 0.00130761589, -0.056519907, 0.0569819957, 0.0702548921, 0.00564425346, -0.0667801648, -0.0926919579, 0.0285909474, 0.0632435605, 0.0112598445, -0.0389826186, -0.0179278683, -0.0421465449, -0.00672344025, -0.0266675558, -0.0206454918, -0.0267114062, -0.0885118544, 0.0208736639, -0.0513872951, -0.0657120273, 0.0381411575, -0.0106926654, 0.0588549338, -0.0140912514, -0.00851650536, -0.0947570204, 0.00672921026, 0.0934867412, -0.0580338538, 0.0665191337, 0.0633494705, 0.0422577783, -0.00369856972, 0.0215703845, -0.0575894639, -0.0108126858, 0.0265451167, -0.0392663404, -8.852580e-02, -0.00783000235, 0.026716413, 0.00620353408, -0.0843705386, -0.0565418266, -0.0298482422], [-0.0161085576, -0.029651463, -0.0411058739, 0.0925587192, 0.00746504311, -0.0789319947, -0.0560457855, 0.107504442, 0.0284037441, -0.0758918598, -0.044808615, -0.0325210467, -0.00986471865, -0.00241661677, -0.05741686, 0.00820308086, 4.718630e-02, 0.0849020257, -4.910480e-02, 0.0654737279, -0.00562491687, -0.0032055371, -0.0298399534, 0.0447386764, 0.0430724099, -3.747660e-02, 0.094065994, 0.0401317291, 0.0697692409, -0.0320398174, -0.0810606629, -0.0476979613, 0.118875831, -0.0176489297, 0.0552901253, 0.106972173, -0.0479485616, 0.0302717071, -0.00497875409, 0.0332519338, -0.030768808, 0.0176460035, 0.0144331679, 0.0817906708, -0.117114693, -0.0581649356, -0.0522838794, -0.0448464379, 0.062787123, -0.0111126285, 0.00646292185, 0.07396999, -0.0778916999, 0.0115572028, -0.0221130569, 0.0516589954, -0.0535968877, 0.0398741178, 0.0287714787, 0.0436816551, -0.019947825, -0.0657172352, -0.0198249202, -3.278650e-02, 0.027013449, 0.0184021518, 0.061802268, -0.0203155652, -0.0425306521, -0.13398242, 0.0142996218, 0.0159958303, 0.0579571575, 0.0322572738, 0.053672228, 7.866150e-02, 0.0217029881, -0.0816098079, 2.60183879E-4, 0.0985630601, -0.0660342574, 0.0231390782, -0.0377612412, 0.0160059836, -0.0117471069, -0.0834448487, 0.00689594494, -0.0627164766, 0.0100920219, -0.0659433678, 0.0281628463, 0.0283952616, 0.00300655095, 0.0471604392, -0.0803913474, 0.0287194848, 0.0333230607, -0.049156528, -0.00528541068, -0.108696967, -0.0562553927, -0.0294568855, -0.0370624885, -0.0622964613, 0.0241385866, 0.0161966793, -0.0304874294, 0.0671157092, 0.0513461269, -0.0185007453, 0.0679611861, -0.0611444563, -0.0149972038, 0.0857022479, -0.0965767428, -0.0685626417, -0.0170915723, 0.0593477152, -0.0654745251, 0.0555360913, -0.00346974027, -0.0318133496, 0.0267063621, 0.0468868054, 0.0109700272, -0.0331317894, -0.0705496073, -0.0100842305], [0.074040778, -0.0420391299, 0.00908285379, 0.00295529794, 0.0662620589, -0.0604023971, -0.0806628242, -0.0209814943, -6.74675452E-4, -0.00264544389, 0.00982432161, 0.0380487442, -0.0640953779, 0.104769923, 0.0169235021, 0.0860566273, -0.0790200382, -0.0164603703, 0.0826443508, 0.0490671918, 0.0506177545, -0.0369703807, 0.0253914911, 0.00914693996, 3.243450e-02, 0.026454214, -0.0110796979, 0.0131775523, -0.0746004581, -0.0285461787, 0.0229393654, -0.0538335443, 0.00319144153, -0.10403803, -0.00331379077, 0.0605565757, -0.130037293, 0.0703724623, 0.0764039755, 6.605820e-02, 0.0626555756, 0.00546649238, 0.0357356779, 0.0149824051, 0.0363150947, -0.0194018949, -0.0376087539, -0.0605278835, 0.0773285478, -0.015069467, -0.0682513267, 0.130420685, 0.0151330577, -0.0209919624, 0.0221822523, -3.50185175E-4, -0.0776561573, -0.0573142916, 0.014963883, 0.0621521361, -0.0296634156, -0.0587217845, -0.0896132588, 0.0578794144, 0.0182893202, 0.0467699505, 0.0130801732, -0.0341074057, -0.0572989285, -0.0369774401, -0.0364189856, 0.105157152, 0.0220632553, 7.293940e-02, -0.0608472452, -0.027720863, 0.027129367, 0.0439240187, 0.0356405228, 0.0782225802, 0.0451036319, 0.0331797116, 0.0417744815, 0.044480823, -0.0187347289, -5.816750e-02, 0.139073804, -0.0653681159, -0.0773460566, -0.0519907624, -0.00612323312, 0.0234239455, 4.274640e-02, -0.0573279634, 0.016051583, -0.0471640974, -0.0938829779, 0.0310609695, 0.022320725, -1.81955169E-4, 0.0366800167, 0.00665640971, 0.0627845153, -0.00938263721, -0.00366823259, 0.057248123, 0.125312507, -9.10141331E-4, -0.0248954482, -0.0713868067, -0.017343333, -0.0650700703, 0.0371046476, -0.054791972, -0.058082208, -0.0107156485, 0.0323783495, -0.0464666821, 0.0138032129, -6.40156678E-4, -0.0125425216, 0.0136180222, 0.026519794, 0.106987812, 0.040098127, 0.0605046339, 0.0290418193, 0.0175004881], [0.0707431212, -5.533780e-02, 0.00372784748, 0.0882362872, 0.082843259, 0.0151300617, 0.0440005437, 0.083673112, -0.055808451, 0.0243372899, 0.0122673679, -0.00304121897, -0.0588809513, 0.0370837338, -0.0516471714, 0.0803542584, 0.0226986483, -0.0424479842, 6.11241208E-4, 0.033477433, 3.54725693E-4, 0.065122731, -6.735140e-02, -0.0106999744, -0.0976295098, 0.0093287481, 0.12010064, 0.0471950695, -0.0382112339, 0.089839749, 0.0769008249, 0.0459852591, 0.0412007757, -0.015240211, 0.00723400479, 0.0590210445, 0.0103513477, 0.0676524565, 0.0488282405, 0.0633295625, -0.00493768975, -0.0693932473, 0.0435687378, 9.083790e-02, 0.0448885225, -0.0103494404, -0.0560201779, -0.0443244241, -0.0307367537, -0.0218922589, 0.0261697937, 0.0926860794, 0.0601337701, -0.00398288853, -7.15799164E-4, -0.0478478782, 0.00626409892, 0.0312254075, 0.0607594736, -0.0191446021, 0.0712805837, -0.0717939734, -0.0868649632, -0.0276456028, 0.0435995497, -0.0179536324, 0.062494494, -5.572370e-04, 0.00165252644, -0.0605532192, 0.014428867, 0.00352463918, -0.0348907225, 0.00681832619, -0.0655323938, 0.0168143455, -0.0279652327, -0.00366462325, -0.062061984, 0.0125935627, 0.0375309065, 0.0571767613, 0.06151345, -0.0464882068, 0.033490058, -0.0542120412, 0.109807074, 6.525290e-02, -0.0549833179, 0.0705998465, 0.0521112569, -0.044127211, -0.059319865, -0.0491656773, -0.069508329, -0.0548438057, -0.110011525, 8.117290e-02, 0.0104143731, -0.0135673266, -0.0304583982, -0.00822343863, -0.00755019579, -0.0678095445, 0.03837597, 0.0710840449, 0.0604897626, 0.0530185774, -0.0141409757, 0.0611043945, -0.0105286352, 0.0454864092, 0.0581900254, -0.0617417768, 0.0318912677, 0.122258231, 0.00738999248, 0.0500227734, -0.0290656518, 0.0283778701, -0.076275304, 0.0488737263, -0.072743982, 0.066907987, -0.0608736612, -0.0353151932, -0.0846939161, -0.0490320437], [0.0659619867, -0.0115466779, 0.00431942241, -0.0659081637, 0.0940220281, -0.0611765198, 0.0254908577, 0.0101507604, -0.0327445157, -0.0146616325, 0.01939971, 0.0111632943, -0.00380667974, 0.0338890366, 0.092350997, 0.0190391857, 3.011460e-02, -0.0343990289, -0.0109430654, 0.0782241523, 0.05687619, -0.072927691, -6.94660062E-4, 0.0111606326, -2.431570e-02, 0.0153724542, 0.0421017706, 0.0246813167, 0.0763589293, 0.0543775037, 0.0461631343, 0.0618499741, 0.0119546959, -0.0409213826, 0.0100831809, 0.0333284922, -0.0142678916, -9.82801866E-5, 0.0944928899, 0.0789716095, 0.0251985081, 0.0179002918, -0.0398872569, 0.0060575339, 0.0475411601, 0.0362054706, -0.0712841377, -0.0920824856, -0.0330468379, 0.0876034945, -0.0254909135, 0.019678494, 0.0547655262, 0.020017948, 0.0430070758, -0.0308259744, -0.0352885313, -0.0657908618, 0.066242963, -0.0402959846, 0.00235007959, -0.0443753228, -0.0489734337, -0.0544767193, -0.0351162963, -0.0241596177, 3.037620e-02, 0.0331741869, 0.0494055264, -0.0824009478, -0.00868441258, 0.0951346233, -0.0286116917, 0.0771840885, -0.0362892598, 0.0224187393, 0.0418360755, 0.0396880433, 0.0292921923, 0.110507138, 0.0428433567, -0.0552357472, 7.497210e-02, 0.0516914427, 0.0371897221, 0.027514955, 0.134020254, -0.0291113816, 0.0243543852, -0.0351895541, 0.0809527933, -0.0946754813, -0.0619819127, -0.00427507842, -0.0113985818, -0.0600462444, -0.0296959747, 0.0316552892, -0.0765796751, -0.113697514, -0.0598808601, -0.0762019306, 0.0714060441, -0.0782892778, -6.955390e-03, 5.151550e-02, 0.045513656, 0.0252736453, 0.0408206433, 0.0121265631, -0.0422602631, -0.0686183795, -5.427230e-02, -0.0414174572, -0.0284725055, 0.13489373, 0.0451307371, 0.0431530476, 0.0469137914, -0.0234362669, -0.0322189257, -0.00870343298, -0.0574180484, -0.0131227365, -0.0423299223, -0.0313306302, 0.00250254571, 0.0100074299], [-0.0144698163, 0.0875544548, -6.27748959E-4, 0.047203932, 0.011310719, 0.0348692425, 7.817460e-03, 0.0553834736, 0.0847532674, -0.117498077, -0.0807247311, -0.0475733839, -0.0450846255, 0.0492475443, 0.0149078704, 0.0251518618, 0.00838037952, 0.0393307209, 0.00509212073, 0.013635911, 0.00438341545, -0.0445384048, 0.00980580132, -0.0323808827, -0.0804281309, 0.0108538261, 0.0593363792, 0.0608591139, -0.072311677, 0.0581641756, 0.0743407384, 0.0565223061, 0.138022289, -0.0635153726, 0.0640334263, 0.104053877, -0.0176344216, -0.00151588663, 0.016837135, 0.106084011, -0.0221784711, 0.0306918696, 1.871320e-02, 0.0534956306, 0.0208107904, -0.0355115607, 0.0615493953, -0.111327529, 0.0111957137, 0.0830172598, 0.0879074558, -3.50805291E-4, 0.0131405834, 0.0781273245, 0.0389501229, -0.0541825593, 0.0370481089, -0.0603172965, 0.0404906571, -0.0136595462, -0.00528977299, 0.0783607289, -0.0791356489, -7.652610e-02, 0.00129439088, -0.0825973525, 0.0512744077, -0.0422784388, -0.0475834087, 0.0366138034, 0.0441707596, 0.033781413, -0.0247668941, -0.0333649777, -0.0129877813, -0.0336628631, 9.094040e-02, 0.0707906186, -0.0109637538, 0.019799253, 0.0749188289, 0.0494942144, 0.040867731, -0.0790523291, 0.0191636384, 0.0616022721, 0.0991772189, -0.0245295614, -0.0483232439, -0.0471181609, 0.0113311894, 0.0155256586, -0.0724277422, 0.00259985612, 0.0698838606, -0.0875340998, -0.0727025792, 0.0807015225, -0.00807655323, 0.019005321, -0.0456221849, 0.00703298161, -0.0518098101, -0.00648514088, -0.0761534199, 0.0486065112, 0.0460958891, -0.0806374475, 0.10442128, 0.0603669658, 0.0643109605, 0.0122633865, -0.0576824211, -0.0803422629, 0.055597879, 4.219270e-02, 0.0742731765, -0.0104229236, 0.0541624464, 0.0352938548, -0.0150083946, 0.0469762869, 0.0109340856, -0.0423365869, -0.0403308608, 0.0142862443, -0.123765141, 0.0530938804], [-0.0332472175, -0.0281093568, 0.0504345484, 0.0135301882, 0.108705446, 0.0785852298, -0.109008498, -0.0214261711, -0.0526247434, -0.115476295, -0.0134880133, 0.0222670361, 0.0488304831, 0.00136754883, -0.00566927483, 0.00973750092, -0.0670223311, -0.00709227193, 0.0797322243, -0.0661426857, 0.0116477115, 3.62810097E-4, -0.0713449344, -0.00484155165, -0.0136210052, 0.00960557442, -0.0149130728, -0.060927663, 0.0348632559, -0.0463230461, -0.00417153118, 0.0128134359, -0.00808391254, -0.0946762487, -0.0263888277, -0.0345242284, 0.0487122759, -0.0106845144, -0.0235657264, -0.0137255732, -4.185610e-02, 0.0661875159, -0.0165190697, 0.0743291527, 0.0535073876, 0.061793752, 0.0774645954, -0.0119995158, -0.0486299433, 0.0910877957, -0.06486471, -0.035705924, -0.00898778532, 0.00958597473, 0.0040204674, 0.00972436275, -0.0731373355, -0.0284871459, -0.05466488, 0.0254544206, 0.0421560481, 0.00259349775, -0.0731720105, -0.00997562427, 0.0604690649, -0.00730702654, -0.0329504348, 0.077771306, -0.0803139135, -0.00539313769, -0.0474379063, 0.0935618877, -0.018400766, -0.0374645106, 0.00922509841, 0.0494053923, -0.00252181897, -0.0226591136, 0.0485155508, 3.663460e-02, 0.0855609179, -4.310980e-02, -0.0545955934, 0.0575632267, 8.570300e-02, -0.0326319635, 0.0565129593, -0.0489520729, 0.11770428, -0.0750347673, -0.0185246412, -0.00992491189, 0.0495701805, 0.0430628359, 0.0634363591, -0.0329645723, 0.024183521, -0.0253083054, -0.0775866284, -0.0233248938, -0.0119371265, -0.0633521229, -0.00112920837, -0.0762819201, -0.0468115099, 0.0178485755, 0.0262742676, -0.0268303975, 0.0223516449, 0.0746940225, -0.0433654897, -7.109220e-02, 0.0343644768, -0.0478523336, 0.052242849, 0.00521351211, 0.0729765072, -0.0972651243, -0.0232400429, 0.0156885274, -0.00458420068, -0.0438582078, 0.0849964768, 0.0611929633, -0.0932470113, -0.034843009, -0.124657035, 0.0394370295], [-0.0682787746, 0.0471952744, -0.0247129686, 0.0637431368, 0.0595612936, 0.0704820305, -0.0818088502, -0.0853003711, 0.101584218, -0.117803968, -0.0516427569, -0.0458911173, 0.0506756231, 0.0322929397, 0.00553248031, 0.0880432724, 0.0456917509, 0.0480350442, -0.0190954059, 0.0150915962, 0.04008127, 0.0287055019, -0.0779866948, 0.0529278591, -0.0961320176, -0.0111867599, -0.0403038077, -0.0418852046, 0.0261694919, 0.0642831251, -0.0721483156, -0.036917109, 0.0782159716, -0.0765771493, 0.0676565245, 0.0384552591, 0.0625268072, 0.0502083451, 5.731380e-02, 0.0618673749, 5.3031696E-4, -0.0384763293, -0.0375366025, 4.743230e-02, -0.00352653419, 0.019792363, 0.0565258041, -0.0259520058, -0.0389037132, -1.962370e-02, -0.0501010045, 0.0634852052, 0.0583164543, 0.12535283, -0.0404586941, 0.00736979442, -0.035319455, 0.0111804651, -0.0486168712, 0.00165866758, -0.0449430235, -0.0575487129, -0.0265512485, 0.0621865243, 0.00156594254, -0.0735150277, 0.0790961757, -7.450520e-03, 0.0181394741, -0.00144647632, -0.00848010834, 0.0909058749, -0.0190509073, 0.0326542817, -0.0764136389, -0.0641501843, 0.00376033713, 0.0156717934, -0.0652611255, -0.0268307123, 5.109780e-03, 0.0138236331, 0.0635651574, -0.0436133221, 0.00894260127, -0.0095992498, -0.0596720614, -0.0304282773, 0.0949364602, -0.0312348846, -0.0877464488, -0.0391786471, -0.0114533547, -0.0494841263, 0.0655050427, -0.0188439284, -0.0423997417, 0.0443192311, 0.0191060547, 0.0380532146, 0.0514631048, 0.0688866078, 0.0677920729, 0.0741591677, -0.0893018171, -0.0280647334, 0.0801770091, -0.0696963668, 0.0673122853, -0.00166111556, 0.0402078517, 0.0564035289, 0.0533355772, -0.0924849957, -0.0858271941, 0.10303627, 0.00965292938, -0.0784493461, 0.0143161081, -0.0538862497, -0.0846320167, 6.727840e-02, 0.0422110893, 0.0502585694, -0.0241642222, 0.0716106668, -0.018876059, 0.0392688438], [-0.0394191146, 0.0408039503, 0.0394736156, 0.0106138494, -0.0524769537, 0.0615723431, 0.0449225977, -0.0270174015, 0.017029427, -0.0650094375, -0.0678535327, -0.0181392245, 0.0724217072, -0.0139634097, -0.00681606261, 0.0478776842, 0.0293663405, -0.0192321464, 0.0287337694, -7.099190e-02, -0.0151013369, 6.567460e-02, 0.0250171106, 0.0564051457, 0.0205475334, 0.068972975, 0.0589373261, 0.0572913773, -0.0100591499, -0.0759084522, -0.0629292428, 0.0710954666, 0.0180909298, 0.00396681763, -0.00116486836, -0.00326427328, 0.0829038396, 0.0408539064, 0.049617663, 0.00216336548, 0.0284002218, 0.0205972325, 0.00847667176, 0.0441411734, 0.0763767213, 0.0759666413, 0.0136395777, 0.00341475778, -0.00631125271, 0.0447268412, 0.0348389298, 0.0160302259, -0.00357997301, 0.077065885, 0.0211400799, -8.513470e-02, 0.0281944666, -0.0510139614, -0.0218073688, 0.0524957702, 0.044209443, 0.00218270416, 0.0471721143, -0.059433151, -0.0389568619, -0.00895294081, 7.081430e-02, -3.30521696E-4, -0.0187210739, 0.045589231, -0.0274355728, -0.0285218768, -0.0150938313, 0.0124629848, 0.0692547113, 0.0780880376, -1.657790e-02, 0.0342425443, 0.0149891777, -0.0329559445, -0.0440733135, -0.0661522895, 0.0218197852, 0.0146936271, -0.00376489758, 5.084050e-02, -0.0414015949, -0.0281006973, 0.0120007759, -0.0676420256, -0.0270125177, -0.035466034, -0.00713364361, 0.0344886594, 0.0621309802, 3.892510e-02, -7.303480e-02, -0.021595316, 0.0611569732, 0.0304069072, -0.023751162, -0.0321068317, -0.0884098634, -0.028995527, -0.0408430248, -0.0480396934, 0.00102960994, -0.0356221423, 0.0642653555, -0.0115983691, 0.0343517028, -0.0478421561, -0.00439957809, 0.0372176021, -0.0512015745, 0.0633209199, 0.071590133, -0.0974391326, 0.0308382679, -0.0384176448, 0.0235017147, -0.0145904562, 0.0795120224, 0.0940170139, -0.076603435, 0.0722539126, 0.0393504463, 0.0446701348], [-0.0691592991, 0.0641851425, -0.0279854033, 0.0419045575, 0.0155956848, 0.0515883602, -0.00667814212, 0.0163495895, 0.04118485, 0.0186071321, 0.0570419058, -0.0101445531, 0.0642167255, 6.803990e-02, -0.0648422465, 0.0872382373, -0.0177089144, -0.0739618391, 0.00546049187, -0.0188483782, -0.0176508352, 0.0474269316, 0.0705048069, -0.0327260531, 0.0293628965, 0.0523394756, 0.063846454, -0.0212632678, -0.0316081084, -0.0234079752, -0.106908955, 0.0317563638, -0.0212414023, -0.0654455721, 0.0303097852, 0.0691488609, 0.00282546529, 0.0689585581, -0.00783203169, 0.0348261185, -0.0434167758, -0.0698382109, -0.0822870358, -0.0239256304, -0.0610652156, 0.0306383036, 0.0593775548, -0.0353613608, 0.0634747669, 0.0251412839, -0.0136402631, -0.00260376348, 0.0753554702, 0.0494704582, 0.00933084358, 0.0668608621, 6.253640e-02, -0.0302529577, 0.068609327, -0.0393786915, -0.0456444174, -0.0456071571, -0.103198186, 0.00349032786, 0.0380076505, 0.0606534667, 0.0345309898, -0.0260745604, 0.0900145992, 0.093761012, 0.0655089914, 0.100184649, -0.0177451298, -0.0464039817, 0.0274992939, -0.0618523099, 0.121692657, 0.0119340746, -0.0257927477, -0.0684672296, -0.0010192499, -0.0522596091, -0.118528292, -0.071212329, 0.00111777161, -0.0582597516, -0.0142272385, 0.0506455898, 0.0418195911, -0.0492824055, 0.0400756262, -0.0128836306, -0.0444813073, 0.0676882714, 0.0455485433, -0.0236643422, 0.0561386198, 0.025643995, -0.0644059479, 0.0542733148, -4.068020e-02, -0.0694588572, -0.0745987594, 0.0460269786, -0.0549855158, 0.0443102233, 0.0529537797, 0.0541499779, 0.0877764672, 0.104346596, 0.0011175765, 0.0509479605, 0.0242355615, -0.0642612204, 0.0522551835, 0.107124314, 0.0664615706, -0.0738036409, 2.45208299E-4, -0.017977098, -0.0731230676, 0.0778600051, -5.16929897E-4, 0.0313357338, 7.141740e-02, 0.0508910753, 0.00534166582, -0.0488328338], [-0.030490065, 0.100880355, -0.0678417757, -0.0237545092, 0.0148358541, -4.530150e-02, 0.0603226572, 0.00953150354, 0.0756067559, 0.0981484279, 0.0343254022, 0.0186023582, 0.0578752197, 0.0975544229, 0.047881607, -0.0418786444, -0.0164177623, -0.059896294, 0.0910440683, -0.0484473966, -0.0446771346, -0.052570913, -0.0420084968, 6.829760e-02, 0.0591787286, 3.651170e-02, 0.0288645159, -0.0670348182, -0.0197626837, 0.0344619788, -0.0701939091, 0.0814679339, 0.0369982496, -0.0994409695, 0.0324973948, -0.0482173078, -0.0168819372, -0.0309303403, 7.94349296E-4, 0.096886307, 0.0891675502, -0.0317199267, -5.927710e-02, -0.0091812415, 0.0748818144, -0.02003273, -0.0366607271, 9.107470e-03, 0.0349864885, 0.0783523991, 5.446310e-02, -0.0314690731, 0.00397478696, -4.79437556E-4, -0.0437007099, -0.0170636736, -2.933310e-02, 0.0639327615, -0.0738477856, 0.00211406313, -0.0540212467, -0.00539828883, -0.00565046305, 0.0173132122, 0.018407153, 0.0164409392, 0.00478503713, -7.700890e-03, -0.0078140255, -0.0348530971, 0.00586896623, 0.085923031, 0.0288335588, 0.0571403652, -0.0711517781, -0.043921683, 0.0704761371, -0.0420172326, -0.013851786, -0.0195409693, -0.0116382902, 0.0323187746, -0.0032993576, -0.0448032357, 0.0973818823, -0.0474580117, -0.0166601948, -0.0487512127, -0.00987295527, -0.0730351731, -0.0885567516, 0.0843530222, 6.571710e-02, 0.0104207685, -0.0673726201, 0.0628199279, -0.0756923854, -0.00519877207, 0.0623090789, -0.0410998799, 0.0706481933, 0.00231177709, 0.0342485569, -0.0226845071, 0.0106690852, -0.042613294, 0.0884645804, -0.0259488504, -0.0196414832, 0.00957771111, -0.0191701297, -0.0263049919, 0.0183126386, -0.0488378927, 0.00217524869, 0.0254378505, 0.0228762571, 0.0079426663, -0.0293086674, 0.039134834, -0.069801107, -0.0509218313, -0.0440170206, 0.0668982938, 3.711860e-02, -0.0469338894, 0.0117061837, -0.0421709493], [-0.0705458447, -0.0252710246, 0.0167901758, -0.0140657024, 0.0668061525, 0.0346186981, 0.0934605672, 0.0151226353, -0.0612445548, 0.0572381243, 0.0220275167, -0.013710537, -0.0588698462, 0.0840994269, -0.0296799801, 0.0103446711, 0.0431936979, 0.0128556294, 0.0553583615, 0.00204166584, -0.014122678, 0.0475987978, 0.0332052484, -0.0381789841, -0.038304586, -0.0613211431, 0.053584557, -0.0672789812, -0.00581856305, -0.0429364517, -0.0455615334, 0.0758046284, 0.0853585824, 0.0395432934, 0.0109351119, 0.00759963552, 0.0382252038, 0.0735396594, 4.16705676E-4, 0.0369317383, 0.041015096, 0.072605744, 0.0182855576, 0.00806100573, -0.0712974071, -0.0118137049, 0.0068900194, -0.0525396168, 0.0174340419, 0.0247033034, 0.0648785308, -0.00882612262, 0.0639261827, -0.0193341579, -0.0908295586, -0.0779812335, -0.0788545086, -6.132680e-02, 0.0484413207, 0.0168692395, -0.0672716796, 0.0109731806, -0.0623006076, -0.0681892633, 0.0187190659, 0.0165674146, -0.0291672125, 0.036204692, 0.0744555146, 0.0414387286, 0.0079798121, 0.0694142506, 0.0408697464, -0.0421872884, -0.0652891472, -0.0632020533, -0.0465506054, 0.0379636884, -0.0157384891, 0.0296058618, 0.0166400056, 0.0323923752, -0.112974897, -0.00473579764, 0.0920136049, -0.0884488374, -0.019486839, -0.0277509037, 0.0995521545, -0.0426550247, -0.0585678592, 0.111052573, 0.0323705971, -0.0148733817, 6.264860e-02, -0.0684132203, 5.136610e-02, -0.0117196403, -0.0825163573, -3.99644981E-4, -0.0377139039, 0.0406565331, -0.00359428511, 0.0611131862, -0.0429287925, -0.0478062406, 0.0205987636, 0.0858064517, 0.0797671527, 0.00355906459, -0.0056516761, -0.0749909579, 0.0830873772, 0.0555675812, -0.0595752373, 0.0941122695, 0.0367042385, -0.104089782, 0.00130081351, 0.0387689285, -0.0852899253, 0.0612905622, 0.0344212241, 0.0211794022, -0.00374001171, 0.0131933466, 0.0286017209, 0.0166438408], [0.0445323139, -0.0576887652, 0.070624724, -0.0796920135, 0.0754500479, -0.0109087965, 0.0191568863, 0.0468546227, 0.0212691743, -0.0322575122, 0.0518031605, 0.0493367091, 0.0346136019, -0.0198053326, -0.0733737051, -0.0356950313, -0.00169362384, -0.0217590369, 0.00854437705, 0.0553507768, -0.0776913538, -0.0360838845, 0.0312354751, -0.052963946, 0.0178691652, 0.0165234413, -0.0205322802, 0.0667652339, -0.0470937118, -0.0695242658, 0.0506774969, -0.0202744771, -0.0131035093, -0.0992995351, -0.0540400334, 0.0602575317, -0.0582931489, 0.0242123511, -0.0766627863, 0.0973772555, 0.0954436436, 0.0499073565, 0.0316659249, 0.0672295987, 0.0438194387, -0.0453414358, 0.0311551839, -0.0454735309, 0.0208602566, -0.014386693, 0.0882203653, 0.0962022542, 0.0177893583, 0.00952364876, -0.049429629, -0.0611729957, 0.0554880649, 0.026995264, 0.0515030026, -5.978760e-04, 0.00680960668, 0.0329147056, -0.0465199575, -0.0747595131, 0.0649633482, 0.0344548188, -0.0474456809, -0.0474501923, 0.0125820264, -0.00637427252, 0.0393446125, -0.0296621211, 0.0636332929, 0.0567995533, 0.0746905803, 0.0897418112, 0.0748905391, -0.0405942351, -0.062239062, 0.0250080675, -0.0462093055, -0.0531243235, -0.0200633164, 0.0273810457, -0.0273397341, -0.0215406585, 0.0119342878, 0.0610476881, 0.0812069327, -0.041001603, -0.0714822412, 0.111036427, 0.0122509021, -0.0256722122, -0.0708583668, 0.0212402977, 0.0361541361, 0.0415411517, -0.0430750437, 0.0272619128, -0.0550411902, 0.0369772501, -0.0104522444, 3.504200e-02, -0.0494145229, 0.0692164451, -1.905360e-03, 0.0690056831, -6.600960e-02, -0.00261866604, -0.00845868234, 2.924360e-02, 2.36009088E-4, -0.050680507, 0.0318533033, 8.698520e-03, 0.0275005735, -0.0107913362, -0.0395505577, 0.0608345121, -0.0935411453, 0.0808046758, -0.0493689254, 0.0119364904, 0.0669538155, -0.0555367246, -0.0658293292, -0.0248632878], [-0.0349882431, 0.0519939028, 0.0114103844, 0.0074483389, -0.0380511023, 0.018488666, 0.031664785, -0.0228356924, 0.0365618393, -0.0600368567, -5.924090e-02, 0.0857708677, 0.00501380209, 0.101308532, 0.0392753445, -0.0196670908, -0.0512204468, 0.00713231368, 5.810520e-02, -0.0274621192, -0.0553010851, -0.0659114495, -0.052208811, 0.0183251053, -0.0366576687, -0.0788810849, 0.116378188, 0.0803463384, 0.0320515633, 0.00200602249, 0.0470756441, 0.0475556962, 0.06870386, -0.00606165873, 0.0684244558, 0.0942139402, 2.819000e-02, 7.630160e-02, -0.023137752, 0.103507146, 0.0525471643, 0.0677233562, 0.0403891131, 0.0347452238, -0.0589284115, 0.0264609382, 0.0119508393, 0.04799968, 0.00230384874, -0.00832594186, 0.0812388285, -0.0371194668, -0.0556363873, -0.0451763906, -0.0199410226, -0.0597317033, 0.0112591833, -5.125680e-02, -0.0339529775, 0.0696116983, 4.037780e-02, -0.0305217393, -0.0040554218, 0.0277962796, 0.0106313676, 0.0088383425, 0.00497549027, -0.0536631271, -0.0527783819, -0.065931119, 0.0352928229, 0.0563547425, 0.0206063986, 0.0356903784, 0.0814177691, -6.90885354E-4, 0.0109862285, 0.0330280885, -0.0989959239, 0.0360787213, -0.0327899307, -0.0681308135, -0.0869409516, 0.0122138532, -0.0701123849, -0.032856293, -0.0176741946, -0.0736127868, 0.0734088197, 0.0571772419, 0.0582048222, 0.00234445627, -0.0589102767, 0.00321253436, -0.0516406521, -0.0181020144, 0.0355022028, -0.00305634458, -0.0807843878, 0.0098309461, -0.0381762348, 0.00967035815, 0.0353140607, 0.0812817215, -0.0441630557, 0.077368781, -0.00510707218, -0.0250267535, -0.0575542077, -0.05776098, 0.0824871063, -0.0382727124, -0.0646460578, 0.0563649833, 0.0628730059, 0.064454563, -0.0528848656, 0.00332338875, 0.0891370251, -0.0600784272, -0.0954610854, 0.0496412776, -0.0652652383, 0.00860684272, -0.0337522961, -0.0576386638, -0.0180594921, -0.0549752638], [0.0926558226, 0.0617152452, 0.00276206667, 0.0261453595, -0.0241006501, 0.0288605541, 0.0656719729, -0.0331555307, 0.054801397, -0.00644560298, -0.0637092292, -0.0473778769, 0.0125127845, -0.0244229976, -0.0128747644, 0.0260198601, 0.0792555138, -0.0813372135, -0.0166216735, -0.0464113913, -0.0794119909, -0.0561991073, -0.0440374129, 0.0104215695, 0.00912806485, -0.0159413051, 0.0461505055, 0.0467257276, -0.0744420961, 0.0394861177, 0.0241155867, 0.020802021, 0.019504061, -0.0937882959, 0.0329280756, 0.090928547, -0.0789416059, 0.0171885844, 0.0617299341, -0.031515941, 9.500110e-02, 0.0532159209, 0.0723722056, -0.0509955287, 0.0395081639, -0.00219925563, -4.053570e-02, 0.0581208616, -0.0167155378, 0.0176621508, -0.0495774634, -0.050623484, -0.0911830961, -0.0702393054, -0.0678445771, -0.0488742925, 0.0578432381, 0.0654540956, -0.0904115512, 0.0548339412, 0.0665785894, 0.0399130173, 0.0163992718, -0.0136745591, -0.0815125257, -0.0788360759, 0.0108552966, 0.0705955178, -0.0845927372, 0.0182028338, -0.0355651863, 0.120074883, -0.00926687382, 0.0970658734, 0.0157316718, 0.0418503694, -0.0593935102, -0.0634042472, -0.0501634665, 0.0460908152, -0.0815454125, -0.072013557, 0.0621306449, 0.0277151149, 0.0196721163, -0.0270161759, 0.0278791208, -0.00607005507, 0.036594931, -5.418740e-02, 0.0231301691, 0.113177091, 0.0465423539, -0.00993701722, -0.0768190846, -0.0658191815, -0.0378750898, 0.0193519946, -0.0519426279, -0.074491784, -0.0562920086, -0.0174760353, -0.0114796599, -1.946190e-03, 0.0456208959, 0.005791164, -0.0679515302, 0.0756247193, -0.061983861, 0.0241293833, 0.0268092249, 0.00662016124, -0.0736028552, -0.0416901708, 0.0589185581, -0.00320771406, -0.0633556768, -0.0629028379, -0.0325586759, -0.0782155618, 0.0223986227, 0.0631468818, 0.0175652355, -0.0248841718, -0.0219086446, -0.0659259111, 0.0582538173, 0.0654659495], [-0.0479480214, 0.0750482678, 0.00376536488, 0.0741903633, 0.0735316724, 0.0804961547, 0.0344135687, 0.0499243923, -0.0448061936, 0.00606061891, 0.0243913718, 0.0394386612, 0.0670508221, 0.104902774, -0.0255318861, -0.0433518179, -0.0670834258, 0.0519118831, 0.00753821712, 0.00413633836, 0.0576207489, 0.0274041779, 0.0048004454, 0.0675299167, 0.0553149544, -0.0329354554, 0.0277535692, -0.0239807051, 0.0124831703, -0.00717420271, 0.0568933561, 0.0144278212, 0.0714888349, 0.00212852261, 3.77885357E-4, -0.0323333219, 0.037230514, -0.00896400493, -0.0430948064, -7.59872899E-4, 0.0490110964, -0.0262987744, -0.0420608297, 0.0529022589, 0.0171275977, 0.0224384535, -0.0150494995, 0.019907346, 0.0311605986, -0.0212230869, 0.0419251062, 0.0693894625, -0.0772387534, -0.0546672717, 0.0163004268, 0.00733835808, 0.0234467257, 0.0271669608, -0.0487762094, 0.0326853134, 0.00522491243, -0.024323849, 0.0586266741, -0.00852306094, -0.0364269055, -0.00531640742, -0.00330084981, -0.0454387292, 0.0423263051, -0.0165850613, 0.0370476767, -3.330870e-02, 0.0633319095, 0.0474983864, -0.0495629087, 0.0645689592, -0.0394998193, -0.0133230845, -0.0505844466, 3.128200e-02, -0.00943539477, -0.049398981, 0.0077210227, 0.018617278, -0.0129337898, -0.0474153087, -0.0553143099, -0.0517809801, -0.0663860217, 0.0167375449, 0.0674665421, 0.0700059384, 0.0598012917, -0.0759022757, -0.0608075038, 0.0137420595, -0.0494605601, -0.102786086, -0.0419602431, -0.00510245794, -0.0348933712, -0.0623979829, -0.0394093804, -0.037273638, 0.039981585, 0.0279048868, -0.0272520371, 0.0670328736, -0.0677234679, -0.0825289189, 0.00223393645, 0.0430144668, -0.0494879335, -0.0199088901, 0.04163941, 0.0604280457, -0.0564202704, -0.0226652157, 0.0626328289, -0.0173321664, -0.078721337, 0.044066444, -0.103574015, -0.058066126, 0.0504871719, 6.268730e-03, -0.0500599816, 0.0509190597], [-7.389510e-02, 0.0386531129, -0.0456527621, 0.0919529423, 0.00619757734, 0.0065844832, -0.0470782407, -0.0157672688, 0.0374802947, -0.0494824164, -0.00440139463, 0.0346195176, -0.0152917858, 0.0070581981, -0.00959337223, 0.0637155324, 0.0579926185, 0.0431524888, 0.00669347215, 0.0551805831, -0.0583347343, 0.0255063418, -0.0403649844, -0.0826036706, -0.0340082198, 0.0190598443, 0.041040279, 0.0383848883, 0.0612073242, -0.0402141064, 0.0346557386, 0.0756767765, -0.0159421377, -0.0188345909, -4.488540e-02, 0.0632455646, 0.00588535331, -0.0756104812, 0.0312331934, 0.0124478117, -0.0558492653, -0.00880853366, 0.0684268624, -0.0401133634, -0.0884720161, 0.0714653805, 0.0835475996, 0.0540656038, -0.00942252856, 0.0540413521, -0.00888227578, 0.0541012883, -0.012891992, 4.465880e-02, -2.03576128E-4, 0.0385958031, 0.0265361816, -0.037969131, -0.0482704565, 0.07772585, 0.0351551212, -0.0540315956, -0.0244605113, -4.822770e-02, 0.0407653973, -0.00261313282, -0.0610349402, 0.0787239522, 0.0529057495, -0.107649125, 6.08544273E-4, 0.113418467, -0.0238682162, 0.094669938, -0.0268305615, -0.0588118285, -0.0435706675, -0.0302928574, 0.0572434887, 0.039810311, -0.0261456445, 0.064079985, -0.0696946532, 0.0710910931, -0.0206324514, 0.0242030844, 0.0477779694, -0.0506315455, 0.0430559143, 0.0541555882, 0.0771413147, 0.0955972597, 0.0129955579, -0.0103951264, 0.0747847632, -0.0483791865, -0.055784788, -0.0985572412, -0.047059238, -0.00164709147, -0.0645446181, -0.0389186479, 0.0218447112, 0.0894716903, -0.0682559311, -0.0200277418, 0.00961721315, -0.0667527616, -0.032553535, -0.0853312835, 0.0602304339, 0.0401616283, -0.0496891811, 0.0134915356, -0.0657135919, -0.0962076634, -0.017809052, 0.0157825202, -0.0423963405, 0.0535020083, 0.00900349114, -0.0300663803, 0.0189569313, 0.0567009598, -0.0379972607, -0.0584972203, -0.0475707762, -0.0035956006], [-0.00358440448, -0.0203542709, -0.0294711739, -0.0535706431, 0.0787560343, 0.0157108884, -0.0650608763, -0.0144837806, -0.0504512899, 0.0578371324, 0.0163425468, 0.0654949843, 0.0183810964, 0.0299185086, 0.0845894068, -0.0211887266, 0.0283989124, -0.0657554939, -0.0628139526, -0.0146907018, 0.00638038898, 0.0194527917, 0.0284908488, 0.0448780507, 0.0406014211, -0.0330860205, 0.0723724663, -0.00238841586, -0.0374444537, -0.0128509151, 0.0590372384, -0.0176680852, 0.105344281, -0.0457821451, 0.00297788973, 0.0468995869, -0.0598532036, -0.0619754046, -0.0223876517, -0.00712832157, 0.0298562348, 0.0526402891, 0.0660991222, 0.0624219924, -0.100694507, -0.0603166595, -0.00482350495, 0.0666829795, 0.0378650613, -2.361980e-02, 0.0253918432, 0.0522116795, 0.0506815687, -0.0478843525, -0.00361082563, -0.0505701303, -5.411670e-02, 0.018015461, -0.0678977296, -0.0308169592, 0.0119675649, 0.0370733812, -8.445580e-02, 0.0524566062, -0.0447530858, -0.0442956872, 0.0409487486, -0.0148807475, 0.0598120131, -0.101207629, 0.0649775564, 0.00901710614, 0.037000522, 0.0170972478, -0.0487870649, 0.0744690523, 0.0503860973, -4.042810e-03, 0.0495259836, 0.0594504923, -0.0072835912, -0.0494922474, 0.0222141873, -0.0487010591, 0.0070452271, 0.058014892, 0.0690331459, 0.0636086538, -0.0587661564, 0.0132302353, 7.550750e-02, 0.0849112272, 0.0144112185, -0.0231207274, -0.0746961534, 0.0682450682, -0.00232796115, 0.0328661539, -0.0349946059, -0.0297243129, -0.0880102217, 6.49575435E-4, 0.0425801352, 0.0129797133, 0.00332979346, 0.00154112955, 0.00614117645, -0.00153805246, -0.0489259697, 0.01775782, 0.0938866063, 0.0197581463, -0.00956590753, 0.0383043848, -0.0323698148, -0.0260833744, 0.0118300673, -0.0697655156, -0.0121094836, -0.045076225, -0.0204520077, -0.0606944188, -0.0594587587, -0.0149235521, 0.0265680291, -0.0576275326, -0.0579542033, 0.0337563716], [0.0636547059, -0.0252790451, -0.00576793542, 0.0799321755, 0.022678161, 0.0413277335, 0.0329889059, 0.0192707349, 3.225500e-02, -0.0140221976, -0.0585021712, -0.0319721326, 0.0628657937, 0.0687744319, 0.0349629261, 0.0528107882, -0.0509768911, -0.0451256931, 0.00204573153, 0.0684752837, -0.0236379839, 0.0215981696, -0.0419477448, -0.0485477932, 0.0569866113, 0.0069134715, 0.0167968106, 0.00500203948, -0.0198210534, -0.0016474023, 0.0713217109, 0.0377142616, 8.207500e-02, 0.00724686822, 0.00235084957, 0.061478626, -0.0186279248, 0.0338367634, 0.0359847434, 0.0932080745, 0.00565684261, 0.0600501597, -0.0123511655, -0.0684454963, 0.0545057058, 0.0313307531, -0.0550994948, -0.0188089646, -0.0366753675, -0.0230235811, 0.0233435724, 0.0250818785, -0.0666463748, 0.0774675459, 0.0372543745, 0.0745689422, 0.0090360418, 0.0531951115, 0.0227281228, -0.0223006755, -0.0155213391, -0.0715832189, -0.0409772918, 0.0441228412, 0.0327887721, 0.0574738085, 0.0458397716, -0.0135214319, -0.0161472764, -0.0172146522, -0.0775449946, 0.0806874782, -0.00973079074, -0.0300685856, -0.0541449636, 0.0511694923, -0.0896412879, -0.0243934374, 0.0375338495, 0.0754624531, -0.0488328896, -0.0100640254, -0.050361231, -0.0371591486, -0.0438982174, -3.83468752E-4, 1.982520e-02, -0.0157113988, -0.0475688539, 0.00962316058, 0.0487638824, -0.0709776208, 0.0680097267, 0.0496629365, -0.00659810192, -0.0124890432, 0.0808008238, -0.00203475705, -0.0105386954, 0.0972085595, -0.0476620309, 0.0571240187, 0.0519864261, 0.0674261748, 0.0120360916, 0.0398614928, 0.0675341859, 0.0347534977, 0.0358563922, -0.0506672598, -0.0526057817, 0.0683519468, -0.0324267633, 0.0244348329, 0.0325834081, 0.0606218241, 0.0472449213, 0.0686456561, -0.0669483393, 0.0368792675, -0.00901777483, -0.0243532546, 0.0682893619, 0.0687199682, -0.0289829485, -0.0031817744, -0.0438498072, -0.0223832782], [-0.044636324, 0.0800155848, 0.0643852875, 0.00450373255, 0.0219621938, -0.0311554819, -0.0177711435, 0.0150944982, -0.0792127326, -0.0191750769, -8.001510e-02, 0.0132748317, 0.0428796969, 0.0510640591, -0.0597618558, 0.081436865, -0.0279882755, -0.0139364507, -0.00706711551, -0.0713743195, 0.035678342, 0.0611152946, 0.0156900585, -0.0492577553, 0.00644357456, 0.0513810925, 0.0612049326, -0.0314213559, -0.0485948138, 0.036012169, -0.00140106713, -0.00804203097, 0.055621095, 0.0604693107, 0.049889449, -0.0307397749, 0.0182990804, 0.0484158285, -0.0623248965, 0.0178034864, -0.0432654023, 0.0290355943, -0.0504651703, 0.0363410525, -0.0505825393, 0.080134131, -0.00583016407, 0.00853096601, 0.0164722223, 0.0245973635, 0.0159898773, 0.0193470027, -0.0131651061, -0.0707607716, -0.0704154745, 0.0753779933, -0.0480384603, -0.0306737684, -0.00863708741, -0.0641674325, -0.0746715366, -0.0333226137, -0.00678739883, -0.0640313178, 0.0283737574, 0.0291078109, -0.019684026, 0.0334034301, 3.751740e-02, -0.0654499903, 0.00359645323, -0.0540386476, 0.0284617878, 0.061372824, -0.0452263355, 0.0593849793, 0.038251698, -0.00619986886, 0.0797038376, 0.0380005203, -0.0355300605, -0.054172121, -0.0794105529, -0.0101142442, -0.0371348262, -0.02056887, -0.0244579464, 0.0509023853, 0.0327246822, -0.0551525019, 0.0617551692, 0.0158387665, 0.0798761919, 0.053215906, 0.00478814635, -0.047280103, 0.0275618955, -0.0678049698, 0.0595252141, -0.0646075532, -3.942990e-02, 0.060106948, -0.0618820935, 0.0492931046, -0.0722865313, 0.0261625797, -0.00533631677, 0.0558121726, -0.00268305023, 0.072753951, -0.0282430183, -0.0219697226, -0.0453390554, 0.0256012455, 0.0418042503, -0.00445397804, 0.0426027179, 0.0704828575, 7.411090e-02, -0.0498059466, -0.0784670264, -0.0142564178, 0.0728889704, 0.0539223477, -0.0327632912, 0.0698890612, 0.0413110331, 0.0185580011], [-0.0507298447, 0.0352030583, 0.0214485247, -0.00672400463, -0.00752931135, -0.0762062966, 0.0145005845, 0.0185898338, -0.0525329039, -0.080492191, 0.0648777336, 0.0300627016, -0.0145087363, -0.063451089, 0.0686216429, 0.0662599131, 0.0145910922, 7.811810e-02, -0.0190038644, 0.0445858352, -0.0401462503, -0.0648558661, 0.0195302237, -0.0567379929, 0.0450209156, 0.0626459047, -0.0169048887, -0.0791102573, -0.059705995, -0.0287426766, -0.0579876229, 0.0148072699, -0.0373290703, -7.603930e-02, -0.065579243, 0.0789737552, -0.0300144833, -0.0169164147, -0.0263568815, -0.0120692644, -0.0275491569, -0.0678113996, 0.0560936071, -0.0499441437, 0.038373258, -0.0799154639, 0.0634247884, 0.0710413903, -0.0149539663, 0.0052813408, -0.0180793069, -0.0493551828, 0.0392228179, -0.0136010554, 0.0255055595, -0.0418008566, 0.0446188189, -0.0677266046, 0.00119379617, 0.0340534933, -0.0552762635, -0.0727720186, 0.0313519128, 4.11465822E-4, -0.0642390326, 0.0474114642, 0.066678226, -0.0584085733, -0.0570268072, 0.00947997719, 0.0760340914, 0.00505042169, -0.0752431154, 0.013291127, -0.0394563526, -0.0570353754, -0.0273307357, -0.0653431863, -0.0796063169, -0.0719988793, -0.0653289184, 9.070800e-03, 0.0622898415, 0.0504389219, 0.0463318937, 0.0179781672, -0.0227557831, -0.0461795628, 0.0575021207, -0.0337586626, 0.00583778135, -0.0341352634, 0.018894935, 0.0732047185, -0.0364341363, -0.0606662892, 0.0759994387, 0.0112761576, 0.00452461513, 0.0333494954, -0.0112091973, 0.0192451142, -0.0440424643, -0.0307531264, -0.0134909311, -0.017366888, -0.010335587, -0.0490767397, -0.0399626233, 0.0449981615, -0.0330061279, -0.045773603, -0.00764492201, -0.0121349748, -0.0788973644, -0.0490245968, 0.0102342488, 0.0796659216, -0.0435706489, 0.00924908183, -0.0304865353, 0.0224642772, -0.0174311958, 0.00400852738, -0.0390751399, 0.00329151517, -0.0548738725, -0.0602894351], [-0.0602312721, -0.0690214857, 0.0275906473, 0.00169442792, 0.0289716311, -0.0479495563, 0.0604275428, -0.0253851134, -0.0410285555, 1.605500e-02, 0.0168934353, 0.0545421615, 0.0589449331, -0.00319270883, 0.0368861556, -0.0111703677, -0.0135690318, 0.0184859652, -0.0177594591, -0.038998764, 0.0599136129, 0.0421329513, -0.0191142894, -0.0236779787, -0.0554188043, -0.0373045132, -0.0181605592, 0.00158786098, 0.0212983713, -0.07232178, -0.0793517083, -0.0268890355, -5.417460e-02, 0.0110385343, -0.0309509113, 0.0133738192, 0.0485895164, 0.052670598, 0.0321761481, 0.00146929256, -0.0650404766, -0.0757969171, -0.00849100574, 0.0219942201, 0.0452481844, -0.0292412303, -0.00382251735, 0.0382421054, 0.0409280509, 0.0553533509, -0.0678386986, 0.0166792553, 0.00685569085, 0.0751665607, -0.074742265, 0.042891074, 0.0110502727, 0.0660637692, -0.0665926188, 0.0166075062, -0.0513408184, -0.0742428452, 0.0411437564, 0.0322148651, -0.0373138078, -0.0549274571, -0.0278740544, -0.0258552879, -0.00826892164, -5.252470e-02, 0.0732881128, -0.0429827832, 0.0190163609, -0.0495286323, 0.0108635202, 0.0388471261, -0.0623395927, 7.610470e-02, 0.0056454516, -0.0243927799, 0.0753546581, -0.0754675865, 0.0290372316, 0.041822236, -0.0692953169, 0.0271998774, 0.0638912842, -0.013222361, 0.00573145691, -0.0604169667, -0.0673984215, -0.0672242343, 0.00352281146, -0.0756482109, -0.0757278501, 0.0666917413, 0.0743639767, 5.360400e-02, -0.0130010573, -0.0621273443, -0.053691905, 0.0461512655, 0.0381495133, 0.0478043519, 0.0273156986, 0.0391036756, 0.0475751795, 0.0530670211, -0.0115573946, 0.0229781885, -0.0523191802, -0.0119449794, -0.0170222614, -0.0661535561, -0.0532628261, -0.0665131509, -0.0119646518, 0.0798734948, -0.0139881875, -0.0741252676, -0.0251255054, 0.0211725645, -5.245950e-02, 0.00385989225, 0.0248975214, 0.0290434957, -0.0185918342, -0.0198182128], [-0.0187701806, -0.0550921075, -0.00252023665, -0.00114727765, 0.0520455912, -0.0121233119, -0.0281837117, 0.00858090445, 0.0767716095, 0.065526627, -0.0675370246, 0.0195508208, 0.0243809745, 0.023431275, -0.0324915051, -0.0486255325, -0.0224884525, 0.0236326754, 0.0467480794, 0.0150585873, -0.0340443812, -0.0683989525, -0.0638979301, 0.00314743072, -0.0539424382, -0.0692308843, -0.0301622581, -0.0200757179, 1.530470e-02, -0.0291738715, 0.063540481, -0.0216674544, -0.0186996292, -0.03469963, 0.0307904035, -0.0716828405, 0.00531629426, -0.0131667946, 0.060369473, 0.0278135817, -0.02027498, 0.0190698709, 0.0169768427, -0.00652439147, -0.0139206266, 0.0519163273, 0.0607319437, 0.0193018913, 0.0739918277, -0.039384611, -0.0748110041, -0.015601228, -0.046336107, 0.0151425553, -0.0414242595, -0.0131749464, -0.0466049351, -0.0609127656, -0.0202481393, -0.0372124538, -0.0294396859, -0.0365660191, -0.0160361752, 0.0636486188, -0.0485230722, -7.037910e-02, 0.0336158201, -0.0322361141, 0.00571764261, 0.069437243, -0.00572664849, -0.0614485592, 0.00922374799, 0.0145516265, 3.199000e-02, -0.029759448, 0.0453127101, -0.0636461154, -0.027832048, -0.0341963433, 0.0496041328, 0.0683979541, 0.0407204702, 0.0579666942, 0.0142054688, 0.0756945685, 0.00562317343, -0.0787912234, 0.0101073012, 0.0164730027, 0.0322662778, -0.0312859826, -0.0470993854, 0.0340022407, -0.0737005472, -0.0605856329, 0.0601726063, 0.00450603571, -0.00103734422, -0.0333891399, -0.027252879, -0.0192281958, -7.320510e-02, -0.0332258083, -0.028162092, -0.0396155417, 0.0249029174, -0.076706253, 0.0430101939, 0.0170533936, -0.0650490075, -0.00214774907, 0.0719180927, 0.0724957362, -0.0721430108, 0.048466403, -0.012201828, 0.0712623075, -0.0364911705, 0.00380447251, 0.0744810849, 0.0776278451, -0.011357246, 0.0297225043, 0.0737461448, 0.0667297542, -0.0404410623, -0.0209892597], [-0.0284079127, -0.0229206923, 0.0468440801, 0.00456013577, 0.0619821846, 0.0794243664, -0.0156603139, 0.0562197119, 0.0141344164, 0.0091902623, -0.0555818155, -0.0697044134, -0.0807138308, 0.0108958837, 0.0807809457, 0.0432566218, 0.00744178146, -0.0239830893, 0.0209862012, -0.00272788759, 0.0134841502, 0.030078575, 0.048119992, 0.0522082895, 0.0486179814, 0.00361698889, -0.0672469884, 0.0687433258, -0.0285624377, -0.0550962351, -0.0399666056, 0.0185408369, -0.0132126445, -0.0413397029, 0.0145950727, -0.0318190046, 0.0234168787, -0.0439769961, 0.0484273769, 0.0713773146, -0.00991204287, 0.0576532297, -0.0575084463, -0.0133532388, 0.0293068606, -0.0264980663, 0.0324037559, 0.049569767, -0.0426241569, -0.0462296419, 0.0165174175, -0.0632778555, 0.0575340055, 0.0714241117, 0.03325589, 0.0220954493, -0.0569182187, -0.075654611, -0.0314096473, -0.0473769158, 0.0146003561, -0.0569810495, -0.0567833632, -0.0555703603, -0.0121369017, 0.0702599213, -0.0209641326, 0.0613009408, -0.0396233313, -0.00765756937, -0.00969727244, -6.68658584E-4, -0.0301912297, -0.0177942179, 0.00275446451, -0.0535521582, -0.0670581907, -0.035721302, 0.0495967492, 0.00318963197, 0.0467466116, 0.0770334825, 0.0623759404, 1.485140e-02, -0.079250887, -0.0693156272, 0.03786771, 0.0798928141, 0.0693698525, -0.00886093732, -0.0113663683, -0.00228386908, -0.0406534486, -0.0325988047, -0.00581606338, -0.0444271229, 0.0555033945, -0.0797959044, 0.0140652247, 0.0365949124, 0.0512275361, -0.0352835767, -0.0350983515, 0.0100032631, -0.0246455092, 0.0641395673, 0.0799868628, 0.0441119149, 0.00563642476, 0.0790111944, 0.0611597597, 0.0744860694, 0.0075707892, -0.0124934148, -0.0296117533, 0.00907030795, -0.022013478, 0.0751560628, -0.0574623086, 0.0649119913, 0.00814364199, 0.0260469485, 0.0613327883, 0.0362266861, -0.0292804632, -0.00505627505, 0.0482930019, 0.0756079256], [-0.0503198169, 0.061916694, 0.0130276615, -0.0359959677, 0.0107457032, -0.0618838965, 0.0547468737, -0.00554945832, -0.0122947097, 0.00726587791, 0.046914611, -0.0513743684, -0.0501452573, 0.0382732265, 0.0747434348, 0.0307944641, 0.0417732745, 0.077386573, -0.0598667301, -0.0748207495, -7.120800e-02, -0.0355893262, -0.0134026529, 0.0800148398, -0.0108894939, -0.0459275395, 0.0490347929, -0.0277867168, -0.0315197259, 0.0547918156, -0.0620272309, 0.0517621525, 0.0269515086, -0.0453912243, -0.0774798169, 5.841790e-03, -0.0287350938, 0.0236768648, 0.0142610464, -0.0373887904, 0.0576349348, -0.0686000586, 0.0135774454, -0.0194216184, -0.0486080386, 0.0615879223, -0.0341310687, -0.023555778, -0.0012641733, -0.0730278715, -0.0571404174, -0.0611165166, 0.0232320856, 0.0459718294, 4.31442255E-4, -0.079183206, -0.0674525276, 0.0278496109, -0.0581256673, -0.017099265, 0.00210527284, -0.0107084801, -0.0433556437, 0.0613443665, -0.0070638936, 0.00106483861, 0.00961899105, 0.07313364, -0.0297389291, -0.0814758688, 0.0655590892, -0.074536778, -0.0248062089, 0.0513917841, 0.0424732789, -0.0220213681, -0.0338454433, 6.089280e-02, 0.00586781884, -0.0629329309, 0.0320668183, -0.00579573307, -0.0695649907, -0.0749675929, 0.0695469379, -0.0399284773, -0.012193921, 0.0555020608, -9.50060261E-4, 0.0477063917, -0.0182925072, -0.0154286074, -0.0120721841, 0.0187408738, -0.0482409559, 0.0263492651, -0.0728623569, -0.079442434, 0.0350444391, -0.0711840093, 0.030572392, 0.0189127196, 4.997050e-02, 0.0473693945, 0.0684395358, 0.0776225179, -0.0480678454, -0.0260392651, -0.0516693182, -0.0102696419, -0.00173380738, 9.47713851E-6, 0.00134141441, -0.0455769598, -0.0700177327, -0.0796763152, 0.0522824749, -0.0446514636, 0.0345325954, 0.0465021692, 0.0735989809, 0.0311798323, -0.0732694417, 0.0132675748, 0.0546970963, -0.0403413735, -0.0280663799, -0.0744860098], [-0.0461833179, 0.0802448615, -0.0720610619, -0.0203277636, 7.009910e-02, -8.298680e-03, 0.0444972627, -0.0459338427, 0.00434444565, -0.0684205443, 0.0561809465, 0.0470799357, -0.0499921851, 0.0694217979, -0.00742373429, 0.0210740417, 0.0374812968, 0.00176432519, 0.0545109734, 0.0478939712, -5.917010e-02, -0.0251322389, 0.00320456689, -0.037942186, 0.0682124123, -0.0676291659, 0.04828953, -0.0164811332, -0.0149769634, -0.0660149232, -0.0126567613, -0.062097989, -0.0735362694, -0.00297768437, -0.0742278621, -0.0102746496, 0.00529630855, 0.00420380943, -0.0760035887, 0.0320847929, 0.048805166, -0.00195958582, -0.0134811699, 6.22042454E-4, 0.0101308897, 0.0647545158, -0.0525050797, -0.036109753, 0.0288141817, 0.0116599817, 0.0454103425, 0.0254231244, 0.0111776218, 0.0549107194, -0.0100361574, 0.045025941, -0.0712885558, -5.476950e-02, 0.0142405108, -0.0271010511, 0.00554285571, -0.00400598068, 0.0658405647, 0.0555646531, -0.0613603293, 0.0803778693, -0.0522400849, -0.0289890636, 0.0439555943, -0.066925764, 0.0181137919, -0.0165369473, 0.0732453093, 0.0443855748, 0.0397819951, -0.0297408495, 0.00172605459, -0.0780838281, -0.0509848185, -0.0237285104, -0.025886951, -0.0775938406, -0.00427521951, -0.0505540781, -0.0176230576, 0.00195004279, -0.051097881, -0.0601337329, -0.0286751948, 0.0243306421, -0.0265390407, 0.0210511498, 0.0331416391, -0.00523077045, 0.0232176892, -0.0540275276, -0.0517059043, -0.0217434913, -0.0294317342, -0.0282820426, -0.00856931414, -0.0322723202, 0.055233445, 0.0743613765, -0.0239320192, -0.0788538903, -0.0231407396, 0.0527124591, 0.0467689447, 0.0516401827, 0.0501713865, 0.048203662, -0.0444493517, 0.0167240053, -0.00810750667, -0.0539149754, 0.0790571942, 0.0224736799, 0.00228177453, 3.723600e-02, 0.0416914448, 0.0526209772, -0.0811889842, -0.00567068486, -0.0455471724, -0.0717317089, 0.0661257282, -0.0149778845], [0.0602537878, 0.0751360506, -0.0813433602, 0.0709301308, 0.0267827883, -0.043792773, 0.0138518419, 5.561710e-02, -0.0650096833, 0.0041436716, 0.0834600552, -4.722460e-02, 0.00962973758, 0.0820412337, -0.00223177648, 0.0195947494, -0.0648884177, 0.0658282787, -0.00905031711, -0.00403910456, 0.00261131511, -0.0565407127, -0.0755546167, 0.0827842652, -0.040926978, -3.270060e-02, -0.0653669089, 0.0370388664, -0.0718252808, -0.0403185338, -0.0488164574, 0.0367306024, 0.0374673344, 0.0708743781, -0.0561213382, -6.46373606E-4, 0.0281962603, 0.0958660468, -0.00410622824, 0.0608980767, 0.0690597892, 0.0350562744, 0.0707385241, 0.00675622327, 0.0592689402, -0.0633857548, 0.0465193689, 0.04085337, 0.0270974915, 0.0453886837, 0.0306036547, 0.097100526, 0.0272694901, 8.041690e-03, -0.0557739623, -0.0178357195, 0.00145634043, 0.00121595885, 0.0804504305, 0.0046319589, -0.0363582186, -0.036202617, -0.0647549108, 0.0100241974, 1.008780e-02, 0.0751797929, -6.565180e-02, -0.0266099703, 0.0182888638, 0.0158022474, -0.037274804, 0.0785320848, 0.0812374949, 0.0519562028, 0.0621719099, -7.032620e-02, 0.0459459461, -7.293490e-02, 0.0128081627, 0.0529307425, -0.0445917249, 3.328600e-02, -0.0461430326, -0.0567789637, -0.0246886425, 0.0495219268, 0.00426425133, -0.00907181948, 0.0237838849, 0.0496738628, -0.0216742642, 0.0636881366, -0.0868612379, 0.0621605143, -0.012533864, -0.050176993, -0.0466710143, -0.06371703, 0.0409194864, -0.0360160097, -0.0927474573, -0.0256322529, -0.074514769, -0.0697118565, -0.0113603696, -0.0449154712, -0.0330832079, 0.0286648441, -0.0290801972, 0.034476053, -0.0346489623, 0.056173265, -0.0436445773, -0.0502805822, -3.755140e-02, 0.050417114, -0.00149848207, -0.0759310573, 0.0730979442, 0.0205723699, -0.00665602135, -0.0042705331, 0.0363358818, 0.00590351038, 0.0361249223, -0.0608591177, -0.0179130435, 0.0255225841], [-0.00950012728, 0.00944067072, -0.0379389897, 0.0186949354, 0.0949283242, 0.00216139504, -0.018978443, 0.0659538284, -0.0134359617, 0.0665574371, 0.00404400751, 0.0572684184, 0.0489865765, 0.10909275, -0.070467554, 0.0356937163, 0.0615119487, 0.0317286327, -0.0427853763, -0.0585276708, -0.0719776153, -0.0391867906, 0.0357580371, -0.00853700843, -0.0135423681, 0.0423073322, 0.0519577339, -0.00360418647, -0.0650369525, 0.0589861199, -0.0159303844, 0.0274136867, 0.0228283517, 0.0465132892, -0.0332040936, -0.033046551, -0.0675367564, 3.681720e-02, -0.0963491052, 0.0608925372, 6.363930e-02, 0.077640757, -0.0560582951, 0.0454400778, -0.0289607067, 0.0214333106, 0.00934658851, -0.0949177369, 0.0182898398, -0.0631999448, 0.0344817266, 0.088989973, 0.0461939909, 0.047396034, -0.0379756093, -0.0227921959, 0.0212975238, 0.0736748651, -0.0148356827, 0.0313970037, 0.0313986614, 0.0188332275, -0.05713946, 0.0513910651, -0.0127644232, 0.0350236036, -0.00840606447, 0.0699184238, -0.0275895391, 0.0548475161, -0.068093814, 0.0443007909, -0.0512255505, 0.0892959907, 0.0664962232, 0.0573132709, -0.03036098, -0.0430731401, 0.0210875161, -0.0136784762, 0.0230596103, 0.0813229903, -0.0263030622, -0.0797644705, 0.0114762355, -0.0632188246, -0.0290332511, -0.0749548375, 0.02262678, 0.00716290111, -0.00780020608, 0.0849740058, -0.0593094267, 0.0383031368, 0.05704391, 0.0790786817, -0.0109602064, -0.0182586815, -4.67182545E-4, 0.00880829245, -0.0641823038, -0.0617194958, 0.0586449578, -7.132960e-02, 0.0201068893, 0.0105758123, -0.0439475402, 0.0577497631, 0.0422240831, -0.0640983879, -0.0522706285, -0.0481744111, 0.0496359579, 5.01239847E-4, -0.0259693451, 0.0313107595, 0.00869088154, -0.0661662518, 0.0146042909, -0.0338165723, -0.0459531583, -0.0257343166, 0.0556519218, 4.371220e-02, 0.0696303546, 0.0595384836, 0.025459893, -0.024620004], [-0.0599286072, 0.0415886901, -0.0167101417, 0.0152440192, 0.103357494, 0.0666974932, -0.0153057417, 0.0340444595, -3.544800e-02, -0.0366735347, -0.0170856901, -0.021648014, 0.00967491884, 0.042182669, 0.0796696171, -0.0256335232, -0.0696021542, 0.0682029799, 0.001294669, 0.0262469575, -0.0800710171, -0.0251333714, 0.0167426672, 0.0270307977, -0.0207455587, 0.0494658612, 0.0134147713, -7.681270e-02, -0.0335680433, 0.0694603324, -0.0589814372, 0.0951375886, -0.0559132956, -0.0642441735, -0.0870853662, 0.0658213645, -0.0946571826, -0.00870829634, -8.567540e-02, -0.0367821194, 0.0399625972, 0.0244894773, 0.0547036082, 0.0126392767, 0.0123596452, 0.0309084374, -0.010380079, -0.0179595593, -0.0476262346, 0.0112442346, -0.0081238728, 0.0893611237, 0.0116156088, -0.0873328819, -0.0421271957, 0.0152663281, 0.0123107294, -0.0577429421, -0.0371142626, 0.00927982945, -0.0772491172, -0.0806362852, 0.0575668588, -0.0523239821, -0.00349388039, -0.0158735309, 0.0620355681, 0.0781501159, -0.0265271403, 0.0213071462, -0.00454252446, -0.071089685, 0.0943840742, 0.00619862322, 0.0217357967, -0.00848762877, -0.0586687848, -0.0328453295, 0.0622963123, 0.011352961, -0.00362405949, 0.0338271521, 0.0578880906, -0.0293385889, -0.0930368527, -0.0244875886, 0.0955044478, -0.054891333, -0.010386765, -0.0221356694, 0.00644042855, -0.00458999258, -0.0163735393, -0.0476268493, -0.00282604457, 0.0664309934, -0.0951450914, 0.00335486862, -0.0432050638, 0.0414026789, -0.00385467242, -0.0740489364, 0.0607384071, 0.0848337412, -0.0497984774, 0.0419803448, -0.0506497659, -0.0159615222, -0.00250513665, -0.00617443444, -0.0264583472, -0.00390040386, -0.0408776365, -0.0534584075, -0.0312770084, -0.0120313801, -0.0527713187, 0.0728369504, -0.0104428856, 3.45832668E-4, 0.00746510876, -0.0164050553, 0.0222822409, 0.0813849046, -0.0390247479, -0.0475210212, 0.0292204209, -0.0704963952], [0.0663496852, 0.0618914515, 0.0073742289, -0.00588398287, -0.00685244706, -0.0808274969, -0.0275035016, -0.0162002016, -0.0189415347, -0.013405337, 0.0868345871, 0.00301632402, 0.0470506325, 0.0709454641, -0.0103949253, 0.00392736914, -0.00999569427, 0.0192836467, -0.0227428135, 0.0434157848, 3.395180e-02, -0.0575069599, -0.0104987975, 0.0431781225, -0.0371188596, 0.0359910205, 0.10506627, -6.19183877E-4, 0.0118980864, 0.0470736139, 0.0584071055, -0.0489311367, 0.0155380378, 0.019165026, -0.0333079323, 0.0610504262, -0.111005254, -0.0593017563, 0.00487411534, -0.0192134548, -0.0576923527, -0.046509888, 4.540800e-02, -0.0190082323, -0.0197945423, -0.0630282909, -0.00938252453, -2.085670e-02, -0.0491570495, 0.0928952619, 0.0430673733, 0.0889086499, -0.0485364497, -0.0365791842, 0.04355748, 0.0197486505, -0.00291142543, 0.0107022636, -0.0542042181, -0.00229021721, -0.06129409, -0.0533681847, 0.0308236703, -0.0334044509, 0.0118599329, -0.0202195179, 0.0454518832, -0.0115893427, 0.0602512881, -0.0185329989, -0.0118792793, -0.0664184839, 0.021270927, 0.0164901074, 0.034716215, 0.0476320609, -0.00979622919, -1.23055841E-4, -0.0214511715, 0.0892785043, 0.0539642908, -0.0285115819, -0.0359908901, -0.0306100976, 0.00813581235, -0.0372761153, 0.00155728369, -0.0304272939, 0.0116398586, 0.0473020375, -0.0342818759, -0.0696079284, -0.0631432533, 0.0169506874, -6.832960e-02, -0.0171130635, -0.0239565764, 0.0563139282, 0.0231627058, 0.0228476487, -0.0357838571, 0.00224913354, -0.0145999733, 0.0360532403, 0.0694043934, -0.0168628059, 0.0687242672, 0.0179923102, -0.0179300532, 0.00624504173, 0.0329024345, 0.0407092534, 0.0421071909, 0.0438812189, 0.0178763643, 4.44918755E-4, -0.0309334546, 0.0579366237, -0.0329313055, -0.029920103, -0.100564942, -0.0371904932, -0.0597039759, -0.0377204157, -0.0148938987, -0.0796584561, -0.0183285922, -0.0599977039], [-0.083980225, 0.0705389902, -0.0374691524, -0.0465015024, 0.0598071814, -0.0147323189, -0.0661845133, 0.0323518664, 0.028045468, 0.0308457762, 0.0100025628, 0.0804743916, 0.057490319, 0.0204105843, 0.0596088022, -0.0253332295, 0.0768732578, 0.0801626145, 0.102881394, -0.0225118324, -0.070154041, -0.0381785259, -0.056133952, -0.0709987804, 0.0545029603, 0.00588468462, 0.0880826861, -0.075808242, -0.0288744941, 0.0486722365, -0.00991662964, 0.0644415841, 0.0377517156, 0.0128663424, -0.015114638, 0.0679311156, -3.814100e-02, 0.0539455265, -0.00126495748, 0.060839925, -0.0712112412, 0.0132579664, 0.0129737007, -0.0170152131, -0.0485831909, 0.0681511834, 0.0126130329, -0.0853072181, 0.0837094858, -0.028884491, -0.00727160228, 0.108313859, 0.0229449403, 0.0614516139, -0.0621347204, 0.0703857541, 0.041529227, 0.0593501925, -0.0296451636, -0.0350013115, -0.0233538896, 0.0124168471, 0.00790430605, -0.00823709182, -0.00859704986, -0.067247428, 0.074613288, -0.0747858807, -0.029916307, -0.0162876975, 0.054986205, -0.0121432878, -5.479470e-02, -0.0452328287, -6.597220e-02, 0.0670384392, -0.0608234443, -0.0239399429, 7.733710e-03, 6.712230e-02, 0.0311443843, 0.0706379413, 0.0304082446, -0.0286800452, 0.0677363574, 0.062303856, 0.0167204682, -0.0741019398, -0.0616409034, 0.0116802007, -0.0304443352, 0.0216657463, -0.0375915281, 0.0511485972, 0.0160493013, 0.0867030546, 0.0532019883, -0.0115693714, -0.0402536429, 0.0261194911, -0.0254446715, 0.0679943189, 0.0383882225, -0.0118472986, -0.0582458228, 7.140330e-02, -0.00222119829, -0.00816740281, -0.0865409821, 0.0203154553, -0.0514392108, -7.83367781E-4, -0.0131194899, 0.0628709868, -0.0325401723, 0.0750654787, -0.00846250541, 0.0162437446, -0.0764220431, 5.73157682E-4, 6.18726772E-4, 0.00795549899, 0.0302838478, 0.0437808931, -0.0232070833, 0.0438490063, -0.0308039561, 0.0240478422], [0.0140614277, -0.0829305798, -0.0724007413, 0.0886028483, 0.0467880331, 0.0810642763, -0.0881598815, 0.0902455449, -0.0488605797, 0.0126160551, 0.0108647738, -7.642120e-02, -0.081267111, 0.0171569586, -0.0208325181, 0.0845989138, 0.0291191544, 0.0261674933, 0.03432706, 0.0665700361, 0.00664699823, 5.293740e-02, 0.0328107253, -0.039546337, -0.0538025126, 0.0355864875, -0.0210122727, 0.0570357256, 0.0121809635, 0.0245470833, 0.0264792591, 0.0522792712, 0.0699288845, -0.0182592459, -0.0751361623, -0.034175355, -0.103000239, 0.0238440763, 0.0827618911, 0.00741355214, 0.0440120585, -0.0577119179, 0.0670217276, -0.0159075353, -0.0837461724, -0.0122184884, -0.0619480907, -0.0662169904, 0.0506980345, 0.0753384307, 0.00468590669, 0.0426564664, -0.00988075323, -0.00589671638, 0.0252955016, -0.0223772433, 0.0486448966, -0.0711474791, -0.0786560103, -0.0173134878, 0.0547886752, 0.0562646389, 0.01613166, -0.0339200087, 0.0463706367, 0.0226944126, -0.00784514379, -0.043351192, 0.0455207415, 0.0152704101, 0.01336715, -0.0372376852, 0.0972883179, -1.53848378E-4, 0.0393631794, -0.0478160754, 0.0140533308, -0.031565886, 0.00174819923, 0.0372836702, -0.0686010346, 0.0340236723, 4.352490e-02, 0.0331985317, 0.0475696959, -0.0702604279, -0.0284199249, -0.0770598426, -0.0807136371, -0.0839314982, 9.368040e-02, 0.0739497095, -0.0668870508, -0.016244594, -0.0724785551, 0.00489288662, 0.00273875543, 8.280690e-02, -0.0607501231, -0.0931255444, -0.0704676956, 0.00796413049, -0.0221473724, 0.0463431105, -0.0713719577, -0.0639466718, -0.0159029197, -0.051397752, -0.0747716426, -1.82686068E-4, 0.0365751311, 0.0742579624, 3.314900e-02, 0.0183190778, -0.0644769594, -0.031033311, 0.0477889143, 0.0129268458, 0.0367683098, 0.0174929425, -0.0201216433, -0.0306133777, 8.651450e-03, 0.0159863122, 0.0661107078, -0.0139891468, -0.0894687175, -0.0350472219], [-0.0675614551, -0.019131748, -0.0309594274, 0.0884574353, 0.0135120628, 0.0591558665, -0.0553443618, -0.00694497582, -0.0199613012, -0.074368991, 0.0472080484, 0.0178627539, -6.385570e-02, 0.0341445878, -0.047532957, 0.0836839527, 0.00232529128, 0.0503134206, 0.0346575975, 0.0946895107, -0.0781889111, 0.0540965684, -0.0656217784, -0.0832783356, -0.0856353566, -0.0445653386, 0.018405471, -0.018325191, -0.0306638349, -0.0482199118, 0.0623760447, 0.0161292683, 0.069398053, -0.0603487641, 0.0163771156, -0.0444189683, -0.0358167812, -0.0382035747, 0.0298390742, 0.0287631974, 0.0326781981, 0.0316685252, -0.0434281304, 0.106475346, 0.0596052036, -0.0311297551, 0.0391117409, -0.0985227227, 0.0401930213, 0.0663908049, 3.153310e-02, -0.00298802136, 0.0800540521, 0.0909629836, 0.00512234308, 0.0499628671, 0.0646148473, -0.0413714722, 0.0598658696, -0.0217386112, -0.0393677577, -0.0686438977, -0.0724378526, 0.0439182371, -0.0996318832, 0.0119376015, -0.0399410091, -0.00113739888, 0.047736045, -0.0102691185, -0.0533002429, -0.00201046467, 0.0149529204, -0.0548096821, 0.0130614368, -0.00414201245, 0.0638882592, -0.0514214076, 0.0200674385, 0.0663159266, -0.0166946352, 0.00193167641, 0.0127175404, 0.0562939607, 0.0737446248, -0.0329846814, 0.0937035754, 0.0546971858, -0.0278257355, -0.044908572, 0.0950282588, 0.0151932845, -0.0283645354, 0.0786979944, -6.294750e-03, 0.0769231766, -0.0829540491, 0.0105149793, -0.0577826127, -0.0423157252, 6.647210e-02, 0.0640379712, 5.630610e-03, 0.0245281421, -0.0589732379, 9.585750e-03, 0.045368515, 0.0549083538, 0.00821116566, -0.0461903848, 0.0502565503, -0.0383456163, -0.0686543881, -0.0816276445, 0.00234816805, 0.0682347118, 0.0766420886, 0.0415295623, -0.0677543208, -0.071187444, -0.0870970413, 0.00193655735, 0.0434942059, 0.0929860472, -0.0237999726, 0.0211492665, -0.0757134854, -0.0355197117], [0.0742001086, -0.0147544956, -0.055726245, -0.0221584942, 0.0546390563, 4.820880e-02, -0.0344169736, 0.0610903613, 0.00281845941, -0.00819512829, -0.00120386202, 0.053047739, -0.0557642058, 0.0113903098, -0.0627351329, 0.0624002293, 0.0342147686, -0.0720277429, 0.00416943664, 0.0417388529, 0.0700289235, -0.0397352427, 0.0654762089, -0.0276024695, 0.00858088117, -0.0246287603, 0.0604041964, -0.0691065043, 0.057178203, -0.051252339, 0.101420917, -0.0914862081, 0.121088065, -0.0707922279, 0.00789487175, 0.0151352482, -0.0185149666, 3.78417433E-4, 0.0718118846, -0.0440519974, -0.0273086317, -3.613100e-02, -0.0672660171, 0.00845031812, 0.0046410705, 0.0284421165, -0.0227667838, 0.0226059407, 0.0127192093, 0.0565243028, 0.0475795195, 0.0266497787, 0.0177810732, 5.884390e-02, 0.069857873, 0.0201699305, 0.0573300533, -0.00660746079, 0.0202007666, 3.913450e-03, -0.0383266136, -0.0674049333, 0.00672014384, -0.025489239, 0.0580191463, 0.0456398167, 0.0118184285, 0.0830984116, -0.0746545941, 0.037670169, -0.0278350599, -0.0236493573, 0.0607700981, -0.00890422798, 0.0229132678, 1.27014631E-4, 0.0293319989, 0.0489678606, 1.619140e-02, -0.0405715108, 0.0372027345, 0.0843140632, -0.0609738752, -0.0173902139, 0.0688191652, 0.0526732765, 0.120591618, -0.0464331619, 0.0682797059, -0.0383972414, -0.00282958592, -0.0592395477, -0.0316620842, 0.0737959072, 0.0312975869, -0.0293473639, 0.0135350684, 7.554400e-02, -4.02315287E-4, -0.0466846228, -0.0576440617, -0.0591557361, 0.0556173474, -0.0358771645, -0.0186332595, -0.0606759861, 0.0104524344, 0.0457301028, -0.026098866, 0.0524405912, 0.0562695675, -0.0596014708, -0.045375593, -0.0936116576, 0.0781071633, 0.013339038, 0.0277892929, 0.0264176466, -0.0158038512, 0.0108299237, 0.0269505922, 0.0435466692, -0.058577612, 0.0970799252, 0.0590240844, -0.0207299609, -0.0892282724, -0.0280598551], [-0.0173476227, 0.0793592408, -0.0472208858, 0.0482910909, -0.00281214598, -0.0395576544, -0.0299637541, 0.03669535, 0.00224862271, -0.0501591302, 0.00530520733, 0.0501321554, 0.0715229809, -0.0219337158, -0.00361390552, -0.0107394382, -3.362710e-02, 0.00342939189, 0.0233675297, -0.0606625341, -0.0349283852, -0.0318880677, -0.0167788304, 0.00834957231, 0.0420434661, -0.0297803804, 0.029692715, -0.0317339785, 0.0116732204, 0.0571098067, -0.0269779097, 0.0560094491, 0.0501668677, 0.0256280508, 0.00256733969, 0.0448074043, -0.0943970307, -0.013793177, 0.00373995933, 0.0742795765, -0.0547810085, -0.0217847917, 0.0531603098, 0.0287990142, 0.106646173, 0.0408336334, 0.0260365773, -0.0151230711, 8.75164405E-4, 0.0418483242, -0.054838676, -0.00276742387, 3.403770e-02, -0.0185789652, 0.0129903173, -0.017564822, 0.0521067791, 0.0753052234, -0.0492447689, 0.00755580608, -0.103268377, -0.00830976572, -0.0652415529, 0.0597157329, 0.0703747943, -0.0414146148, 0.0566557907, -0.0345607437, -0.0895457342, -0.013284483, 0.0374250114, -0.0249966774, 0.0521660931, -0.0367649756, 7.746110e-02, 0.0308438148, 0.0961618274, 0.0490115024, 0.0132583873, 0.0897997319, 0.0486845374, -0.0260518566, 0.0449898019, 7.32170709E-4, 0.0509811193, -0.0581230335, 8.894640e-02, -0.0594961606, -0.0387753472, -0.0108383475, -0.0206159111, -0.00204212056, 0.0323198438, 0.0898202658, 0.0389060155, 1.355660e-02, -0.0551539846, 0.0824002325, -0.0588895977, -0.0842743069, -0.0373850688, -0.0419026725, 0.0569753908, 0.0107600251, -0.101709098, 0.0223997962, 0.102828898, 0.0115823364, 0.0693171173, 0.0541313961, 0.0688836351, 0.0351125821, -0.0645800233, -0.0929613783, 0.0370507874, 0.141203105, 0.0831027552, -0.0463888906, 0.0394429341, -0.029107403, -0.0757548809, 0.0409867577, 0.0164401121, 0.0909112468, -0.0263215359, -0.0280931331, -0.0730495452, -0.0698120221], [0.0884028598, -0.0271398425, -0.0418282114, -0.0668559745, -0.01129728, -0.0443798192, -0.0778615624, -0.0427568778, 0.0382531025, 0.0467314869, -0.0438476615, -0.0590843856, 0.0288219359, 0.113187715, -0.00721779699, 0.118514791, -0.0299646631, -0.040273115, -0.0413858891, 0.0573472343, -0.042535197, 0.00217931205, 0.0594845302, -0.0650221407, -0.0521288216, -0.0269859172, -0.00895113777, -0.0686416775, 0.0255273432, 0.063936837, -0.0502817035, -0.074087739, -0.00349130086, -0.0724488198, 0.072917141, 0.00751772476, -0.0716846511, 0.0643232688, -0.0299057085, 0.0396236703, 0.0811775774, -0.00823463406, 0.0579933599, -0.0635739862, 0.0617879368, 0.0372069702, 0.0432646498, 0.0646762922, 0.00141260913, 0.056572292, -0.010890685, -0.00916766282, 0.00311257807, -0.0179732945, -0.036140129, 0.0189746507, 0.043863371, -0.0677370429, -0.0713069066, 0.065612331, 4.08852611E-5, 0.00992746185, -0.0938215777, 2.889730e-02, 0.0045935954, -0.0660554468, 0.0713624805, 0.0667531937, 0.0514746793, 0.0174186826, 0.0807074531, -0.0326832235, 0.0256448593, 0.0386608355, -0.0569010712, -0.0535762273, -0.0355942473, -0.0632602721, -0.0444959551, 0.00435188599, -9.55956581E-4, -0.0575490594, 0.0107793137, -0.0218266249, 0.0555647537, -0.0370522216, 0.0554821081, 0.0639895126, 0.0674748942, 0.00240710494, 0.0552815758, -0.084159866, 0.0563752428, -0.0285947323, -0.0804965794, 0.00598809076, -0.0406734608, 0.102888294, 0.0446762666, -0.0351476856, 0.039857436, -0.0883722603, 0.0536826625, 0.0150777726, -0.0501273833, -0.0125057781, -0.00989579968, -0.036625497, 0.075518921, 0.125644013, 0.0147939231, 0.0325958878, 0.0198166259, 0.0530676879, -0.0163225122, 0.0741818473, -0.0316699333, 0.0200797543, 0.13170816, -0.0191912986, 0.0166842788, 0.0398086943, 0.0801102444, 0.0207872447, 0.0727674067, 0.0164671037, -0.0258921273, 0.00844676606], [-0.0617695525, 0.00740868272, -0.0342529193, 0.0525246486, 0.022539692, 4.108880e-02, 0.0666124746, -0.00732115516, 0.0946847051, -0.0562087297, 0.0364963971, 0.00488644559, -0.0440315753, 0.0104872901, -0.0358435176, 0.102836385, -0.0788441821, 0.0812154039, 0.0869745537, 0.0686079711, 0.0260469932, -0.0182250943, -7.272130e-02, -0.0166107509, -0.0207070746, 0.0292185526, 0.0788762047, 5.712280e-02, 0.0609080344, -0.0870001241, -0.0502442196, 0.0262915771, 0.104056261, 0.049674686, -0.00795425288, -0.040971484, 0.00963229127, 0.0639732257, 0.030892143, 0.070140712, 0.0812051072, -0.0631219446, 0.0373969153, 0.0246957876, 0.0729856566, 0.0868637561, 8.312450e-02, 0.0323801637, 8.841130e-03, -0.00407889858, -0.0576178655, -0.041611664, 0.10391672, 0.101763785, -0.0514736325, -0.06236298, 0.038146928, -0.0349723771, 0.0283423755, 0.090942271, -0.00885135587, -0.0679080039, -0.0421396308, 0.0217823554, 0.112277679, -0.0270948168, 0.00153070409, 0.0158688668, 0.0521936156, -0.00188417581, -0.00613754243, 0.0356688648, -0.035308443, -0.00968542136, -0.0787104293, 0.019783698, 0.04802173, 8.915320e-02, 0.00842845253, -0.0312781818, 0.0691322908, -0.0211574342, -0.0159385242, -0.0536020324, 0.0135418428, -0.0531714521, 7.102210e-02, -0.050699953, 0.075633578, -0.0200594719, -8.576010e-02, -0.0860033184, 0.0163525231, 0.00910941232, -0.0159031097, 0.0417030863, 0.0471701249, 0.0526854843, 0.00178821583, 0.01886818, -0.010941566, 0.0166794173, -0.0638893321, 0.0734582171, -0.0047351378, 0.0469438881, 0.0896254554, 0.076008983, 0.00889072474, 0.0344699547, 0.06027909, 0.0181881245, 0.011505343, 0.0544767343, -0.00744936848, 0.0688396245, -0.0156154437, 0.00308883679, 4.543650e-02, -0.0196758341, -0.13049373, 0.0902248844, 0.0349685848, 0.0996240675, 9.91238513E-4, 0.0705599561, -0.0719232857, 0.066072233], [0.0550854579, 0.0649920403, 0.00898077432, 0.053902965, 3.581870e-02, 0.0710797235, 0.0484312288, -0.02459497, 0.0378973372, 0.0618114583, -0.0556538217, 0.0194793288, 0.0087316744, -0.0340004601, -0.0810599178, -0.0319013782, -0.0511835031, 0.0727268532, 0.0343659744, 0.0153889526, 0.0210919455, 0.0782930329, -0.0460380651, 0.0397600532, -0.0287452377, -0.0135672204, 0.0264445972, -0.0836971402, 0.0581771657, -0.0393819846, -0.00917518232, 0.0798743143, 0.0278818123, -0.0435416624, -0.0334951021, -0.0401669554, 0.0151644703, 0.0794815942, 0.0240124147, 0.0786636099, -0.0463998504, 0.0802172124, -0.0610104539, -0.0460619889, 0.0559468754, -0.00498236716, -0.0392208397, -0.0526094362, 0.0373249575, 0.0767611191, 0.0806234255, 0.0049925372, 0.0587788709, -0.0178519506, 0.0200909525, -0.0322945863, 0.00813344214, -0.0420026779, 0.00319542922, 0.046630539, -0.01042563, 0.0258238744, 0.0486149043, 0.0197617561, 0.0690650716, 0.0558378771, 0.0468972288, -0.072399579, 0.00817172322, -0.0477079749, 0.0133849336, 0.0153302765, -0.00608698511, 0.00392626738, -0.0678669065, -0.0248927064, 0.103147954, 0.0832648873, -0.0287150797, 0.0442393534, 0.0665777251, -0.0581039302, 0.00388668198, -0.0442529358, 0.00935410708, 0.0653653666, -0.0136420839, -0.0602783337, 0.0321312249, 0.0434137098, 0.00970440637, 0.052298896, 0.0216696672, 0.0475393124, -0.0525988452, -4.232000e-02, 0.0168187395, 0.112471767, 0.0716312453, -0.0262574516, -0.0408244506, -0.0151251368, -0.0245605335, 0.00764016341, 0.00786331295, -0.0460913777, 0.110943161, 0.0579470322, -0.0153032178, 0.0908349528, 0.0395928808, -0.052397158, -0.0252945386, -3.719400e-02, -0.00327352807, 0.0774326771, 0.0125294439, -0.106566951, -0.0318251178, -0.0178679563, 0.0201277528, 0.0245795064, -0.0834171399, 6.372210e-02, -0.0441031232, 0.0307549238, -0.0214901045, 0.0159471724], [-0.0657079294, -0.0362955816, 0.0237437487, -0.0634540096, 0.0631906167, 0.00715214107, 0.00299125351, 0.0822463185, 0.0713293701, 0.0200894531, -0.0342004448, 0.00923427846, -0.025484791, -0.0430177152, 0.00182008208, 0.0193390492, 7.88274247E-6, 0.0297161974, -0.0545273907, 0.0616048649, -0.0107589811, -0.0309645794, 0.0385670178, -0.074739337, -0.00145821401, -0.0654406622, 0.0829695463, -0.0709548593, -0.0118827084, 0.028713705, -0.0354731157, -0.0278445072, 0.107774243, -0.0148217548, -0.0325608328, 0.030160198, -0.0067894985, 0.0402958654, -0.0417515822, 0.0489097685, 0.0872438251, -0.0607029721, 0.00101367116, 0.0501714423, 0.0566320159, -0.0180710945, 0.0566051528, -0.0543114208, 0.0121357888, -9.571740e-03, -0.0458263196, 0.0218710545, 0.0478210822, 0.0220112111, -0.069630757, 0.0350936875, -0.0281141289, 0.00729208346, 0.0483876616, 0.0447864607, 0.0364401899, -0.0156242736, -0.0889049619, 0.0497971065, -3.71780363E-4, 0.0632828102, 0.0234036148, -0.0373580195, 0.0440610684, 0.0839367806, 0.0754797458, 0.0980909988, 0.0261557978, 0.0826715901, -0.0771662294, -0.0328050293, 7.817050e-02, -0.0122691365, -0.0650457144, 0.0619702414, -0.0110092778, 0.0013693095, -0.0311866775, -3.994250e-02, 0.087177217, -0.0370629355, 0.0218596887, -0.0425736308, 0.0866872295, -0.0116357785, 0.0414059646, 0.0886096432, 0.03774257, 0.0624906234, -0.0589953251, -0.00422100164, 0.0322011933, 0.0589180849, 6.183210e-02, -0.0716559663, 0.0477718785, -0.0841128155, -0.101555534, -0.0842968747, -0.0194179155, -0.0504183732, -4.135480e-02, -0.0488378741, -0.0532042831, 0.0185510293, 0.0207930878, 0.0206004269, 0.0847637355, -0.0268124137, -0.042294763, 0.119237944, 0.00370139419, 0.048929438, 0.00351794553, 0.0247343145, -0.0120619573, -0.00364356954, 0.0304891169, 0.0140754469, -0.00903025921, 0.0261839479, 6.616410e-02, -0.0407345928], [0.0663599446, -0.00534448307, 0.0454123728, 0.0111861667, -0.0115218405, 0.0375240631, 0.0314956531, 0.0737141371, 0.0521704443, 0.0135399131, 0.0710084736, 0.00368858059, 0.0781761482, 0.105085887, 0.0353119224, -0.0491715707, -0.016599074, -0.0458522141, 0.0396642685, -0.0693704337, -0.0377734751, -0.0254587736, 0.0628906488, -0.0386248566, -0.0751286373, 0.0460554659, 0.0165881589, 0.0676581487, 0.014568001, -0.0638757274, 0.0551185198, -0.0379488356, 0.119565882, -0.0757844448, 0.0411178134, 0.0291365851, 0.0383276902, 0.0273250304, -0.0123454854, -0.0500378385, 0.0370776057, -0.0635775775, -0.0606747121, 0.0250647552, 0.0115814935, -0.0114677381, -0.00117143849, 0.00303904386, 0.0258948151, -0.0435234904, 0.0540821776, 0.0126519557, 0.0233114101, 3.192070e-02, -0.0738844201, 0.0711915791, -0.0236883741, 0.0895767137, -0.0222545527, -0.0180927608, -0.0595696419, -0.053433992, -0.0560250133, -0.0197794512, 0.0943733603, 0.0417365953, -0.0624491423, 0.00551569415, -0.0386291146, -0.0743699893, -0.0241359454, 0.10513474, 0.086700268, 0.00428804709, -0.0253287647, -0.0381369703, -0.0163808838, -0.0525463633, -0.0693410784, -0.0579159334, 0.0124247177, -0.0519953892, 0.0358393192, 0.0128385173, -0.027651852, -0.0477311388, -0.00437232759, 4.107110e-02, -0.0178845618, 0.0374302268, 0.00443327241, -0.00383456564, 0.0270329714, 0.0526291132, 0.0124781737, -0.0152533362, -0.0741921887, -0.0252611917, 0.0310017616, 0.0104607008, 0.0754536688, 0.015493976, -0.0492130108, -0.00388177577, -3.413520e-02, 0.0285125729, 0.0511417799, 0.0558351539, 0.0470688641, -0.010855454, -0.0156490486, 0.0386355631, 0.0599173456, 0.0591947809, -0.0745586082, 0.0609423108, -0.00961211137, 0.00546876481, -0.0138604157, 0.0326441862, 0.00133507478, 0.0240747835, -0.0380902141, -0.00421685912, 0.0608581156, -0.0376324877, -6.572250e-02, 0.0226022676], [0.0967129766, 0.00319774286, -0.065922007, 0.021653248, -0.01557298, 0.0130112087, 0.026528541, 0.0147567447, 0.0280771516, 0.0288904663, 0.04679377, -0.0169738755, 0.0635065809, 0.0520746633, -0.0494686216, 0.0486142151, -0.0740446895, 0.0359619595, 0.0793802663, -0.0673812255, 0.0140200304, -0.0563710034, 0.0432322696, 0.00402336335, -0.0234884173, -0.0527591147, -0.00783365778, -0.0498669632, 0.0546654649, -0.0131918704, 0.0322550386, -0.00226237276, 0.0804157555, 0.0189582519, 0.0154094473, -0.0584628582, -0.0203806553, 5.83938323E-4, 0.0110203428, -0.0365080275, -0.0179122202, 0.0758924559, 0.014379818, 0.0138388555, -0.098135814, 6.260730e-02, 0.0812729969, -0.0560704246, 0.0281882584, -0.00357266446, -0.0095061399, -0.0409087688, 0.071228005, -0.0549775735, -0.0275288895, -0.0746696591, -0.0292340368, 0.0366105326, -0.0194564648, 9.949780e-02, -0.0957456231, -0.0618817843, 0.0288539454, 0.068537809, 0.0264349226, -0.0214039795, -0.0225284733, 0.0672112182, 0.0404852591, -0.0288936123, 0.0343737975, 0.0232697446, 0.0308027267, -0.0395091884, -0.0249200929, 0.0258488413, 0.0441312455, 0.00591256935, -0.0447967835, 0.0346999466, -0.00992159452, 0.00295392284, -0.0743129775, -0.0186408795, 0.00825398322, 0.0385494493, 0.061925482, -0.047768414, 0.0673444867, 0.025478689, 0.0600112341, 0.0861509516, 0.0587870814, -0.0150540974, 0.0234459918, -0.0404331796, -0.106180765, 0.0214911141, 0.0200397074, 0.00282637635, 0.0436131805, 0.00160727405, 0.0248327442, 0.0831620991, -3.372184E-4, -0.0468356945, -0.00297601195, -0.0357660614, 0.00274217152, 0.00674888445, -0.00121317012, -0.00791979861, 0.0687922761, 0.0588006712, 0.0625248477, 0.0072313468, 0.0393600911, -0.0233577322, 0.0169492681, -0.0441906527, -0.0775407106, -0.0760565624, -0.104301214, -0.00669208216, 0.0556200668, -0.01485964, 0.0385476165, -0.037015669], [0.00421575783, 0.0689769834, -0.0594658032, 4.716950e-02, 0.0405752733, 7.605700e-02, -0.0555734336, -0.0435153805, -0.0406944901, 0.00239091786, 0.0785061642, 0.0813673884, 0.0234695729, 0.106505044, 0.0753489584, 0.0400601737, -0.0142696965, -0.0386427827, 0.0115531832, -0.021905018, 0.0440520048, -0.0607995205, -0.0179584064, -0.0634415671, 0.0403116755, -0.0283229575, 0.0794089586, 0.0500798598, -0.0527655631, -0.087437734, -0.00306714186, 0.0646725222, 0.0901475548, -0.0440133624, 0.0572113022, -0.0306691043, -0.0469741933, 0.0477439649, -0.0618637279, -0.0183321089, 0.0102916854, 0.0354065225, 0.029333774, -0.054184556, -0.0508778095, -0.0321339406, -0.0179114882, 0.0632335395, -0.0531065203, -0.0562020503, 0.0826530531, -0.0325980335, -0.0224456191, 0.0285474844, -0.0604524575, -0.0248583741, 0.0484696962, -0.00298527675, -0.0708361715, 0.0941514298, 0.0602530316, 0.0585107394, -0.0676400736, -0.0162267014, -0.0582708754, 0.0355833583, 0.0462677293, 0.0396133885, 0.0735959485, -0.0593349412, -0.0581930168, -0.0209962744, 0.0174736176, -0.0396554433, 0.0838842168, 0.00682677608, 0.0421779193, 0.0395740718, 0.0502994023, -0.0524621271, -0.0172294155, 0.0693324208, -0.0934804454, -0.064916715, 0.0566947162, 0.0383333936, 0.0078125745, -0.0669184774, 0.0629818067, 0.00194852334, -0.0584320612, 0.075461477, 0.0337608382, 0.0611588284, -6.726350e-02, 0.0794126987, -0.100438356, -0.111508302, -0.037783362, -0.0938699617, -0.0897750183, -0.0167783033, 0.0255191959, 0.0344956331, 0.0469418466, 0.0554526486, -0.00883501209, 0.0262457654, 0.00146427273, 0.00783669389, 0.036592681, 0.0626904368, -5.464050e-02, -0.0173613969, 0.024167059, -7.131580e-02, -0.042778302, 0.0186376721, -0.0222044867, -0.0246539693, -0.0185686331, -0.0414617024, -0.0407704189, -0.00758660119, 0.0526139922, -0.0159108713, -0.0291023217, -0.0713356584], [-0.0135559877, 0.0292453915, -8.013620e-02, 0.0786218866, -0.0272527635, -0.0317674726, -0.0731301755, 0.0570617095, -0.0491063893, -0.0490604751, 0.041515477, 0.0844306871, -0.0163801219, 0.0140651381, -0.0690761506, -0.0574943796, -0.0658409223, 0.034436319, 0.0144765321, 0.0113599133, 3.632990e-02, 0.0146677783, -0.0319691077, 0.04297328, 0.0177344307, -0.045133274, -0.0481621064, 0.00352784269, -0.0397242121, -0.0752482712, 0.037212763, 6.175300e-02, 0.0703977123, -0.0818198174, -0.0629026368, 0.0873970985, -0.0052974564, 0.0286646672, 0.0681422204, -0.0286037847, 0.0624866485, -0.0360528827, -0.0318413451, -0.0662449896, -0.0291158482, -0.0204496533, -0.00632843841, -0.00201445352, 0.0219432935, -0.00499236956, -0.0251666903, -0.0337407552, -0.0484208539, -0.032841906, 0.0100975009, 0.0386122167, -0.0739922896, 0.0173714571, -0.0163652506, -0.0554246195, -8.985720e-03, -0.0572801679, -0.00283878064, -0.00253862957, -0.0908943191, -0.0748097896, 0.06878227, -0.0655914173, 0.0504160449, 0.0153505523, -2.872600e-02, 0.031210104, 0.0585570857, 0.0845231413, -0.0524828546, 0.0808222517, -0.0200039279, -0.0381324776, -0.0826669633, 0.0152226966, 0.0284906719, 0.0451157168, -0.0163354203, -0.0566627048, 0.0201004371, -0.0156963728, -0.0261233617, -0.0348121934, -0.00268681906, 0.00192538404, 0.0537359156, 0.04972855, -0.0392643623, 0.00192560907, -0.0441348441, -0.052874092, 0.0317924879, -0.0481061414, 0.0536190979, 0.0198800527, 0.0135726249, -0.0643474161, -0.0745904595, 0.0802510232, -0.0945047885, 0.102258094, -0.0126791615, -3.177290e-02, -0.0613561459, -0.0398537964, 0.0304697119, 0.0310924891, -0.0433389433, 0.023568349, -0.0184495132, 0.00740411318, 0.0335944705, 0.0474494398, 0.0170202088, 0.0651927218, -0.0414191037, -0.068975009, -0.0634397566, -0.0112290867, -0.0790397226, 0.029079238, -0.0641892776, 0.0310771372], [-0.0387435742, 0.0462901667, 0.058494579, 0.0408078507, 0.068001911, -0.0117150657, 0.0348975807, 0.0335972868, 0.0434198268, -0.043702703, 0.0647054762, -0.0584273487, 0.069088541, 9.259750e-02, 0.0295595676, 0.0618843287, 0.0149951931, -0.0845616311, 0.0323752947, 0.0743285641, -0.0114384191, -0.0824545323, -0.0663150102, -0.0687253177, 0.0656970441, -0.0443711355, 0.0292404126, 0.0559182167, 0.0126862302, 3.560840e-02, -0.0365532525, -0.0155535582, -0.0310907457, -0.0216117706, 0.0683558434, 0.0912091955, -0.0821340158, 0.00793438498, -0.0393262245, -0.0264216773, 0.021966869, -0.0734391734, -0.0233563539, -0.0389096811, -0.00887156557, -0.0450585298, 0.031652987, -0.00305660069, 0.0631935373, -0.0433373824, 0.061318174, -0.036808528, 0.0273894537, -0.0197583977, 0.0191692617, -0.0487607308, -0.0491278395, 0.0293278638, 0.0382193439, -0.0460784435, 0.0249636471, -0.0221651308, -0.00814296398, -0.058941897, -0.0463643298, -5.390140e-03, -0.0117690973, -9.149050e-03, -0.0323657356, 0.0317018405, 0.0491311215, 0.0484088846, -0.013812311, -0.0639445111, 0.0399054065, 0.0254090335, -0.0941942781, 0.067375645, -0.0758679286, 0.0233425535, -0.0717077777, 0.0575482287, 0.0287396852, -0.00928647164, 0.061670579, -0.0500081293, 0.0266167615, -0.0621103681, -0.0849797279, 0.0414263941, -0.00595110701, 0.0299684945, -0.0865403265, -0.0390039161, 0.0216838568, -0.048351597, 0.0607944652, -0.00268230634, -0.00480738562, 0.0579107776, 0.0130226687, 0.0534648821, 0.0726508871, 0.0940964221, -0.0532567054, 0.0597023629, 0.0642081201, 0.00899404287, 0.0352434255, -0.098418869, 0.0940536558, 0.00548653025, -0.00388150034, -0.0243924763, -0.076556094, 0.0440341942, 0.0286174584, -0.0492064841, -0.0708538443, 0.027931923, -0.031151142, -0.0752199739, -0.0113351177, -0.00553021953, 0.0165048484, 0.00687897485, -0.0649644434, 0.0222478658], [0.0087375911, 0.0389445052, -0.0562961213, -0.0428125449, 0.00712261582, -0.0232270416, 0.0490142666, -0.00702236592, -6.739080e-02, 0.0406568795, -0.0811829194, 0.0232262965, -7.15879141E-4, 0.0183009412, 0.0647399053, -0.0107479487, 4.286020e-02, -0.0310715493, 0.030376412, -0.00238779397, 4.557400e-02, 0.020718405, -0.0306338109, 0.00351868826, -0.0341020487, -0.0681079924, -0.0549259968, 0.0118417358, 0.0821535736, 0.0618442781, -0.0758084804, 0.0197373796, 5.414220e-02, -0.028170241, -0.0733884722, 0.00773215806, -0.0168422833, -0.0704521388, -0.0274047088, -0.0409260243, 0.0221110322, 0.0603352785, 5.08474332E-5, 0.0605860278, -0.0638860315, -0.0649475232, 0.0601462871, 0.0136885596, 0.0342215598, 0.0295586921, 0.0338360853, -0.00252845604, 0.00569647178, 0.0532308333, -0.0735914186, -0.0367703773, 0.0553687066, -0.00430063438, 0.0489805043, 0.00106883864, 0.0666755139, -7.884470e-03, 0.0248797499, -0.0166304596, 0.0339912213, 0.0586836822, 0.0533000119, -0.0423718356, 0.0631144196, -0.00158159225, 0.0604439229, -0.046682559, -0.0104744844, 0.0601139441, -0.0717995316, -0.0478371121, -0.00322636776, 0.0534914397, 0.029773416, 0.0394368656, 0.0336966775, -0.0602176301, 4.70827363E-5, 0.0431352295, 0.0257040057, -0.0819407328, -0.0352900177, -0.0271761417, 0.0505086705, -0.0208580177, -0.0367471538, 0.0787925049, -0.0389178731, -0.0442046747, 0.0131703494, 0.0584024191, -0.007155756, -0.0959302783, 0.00383189437, -0.044581145, 0.0191614479, -0.0815104767, -0.00267627439, 0.0153035754, -0.0451594852, -0.0375798941, 0.0773174241, 0.0466090366, -0.0668842569, -0.0526542403, 0.0512581505, 0.0309982821, 0.0598468594, 0.0106233424, 0.0503149256, -0.0400313921, -0.0555423386, -0.0448935665, 0.0330046453, 0.0556545071, 0.0584182739, 0.0313227363, -0.0390839726, -0.0170765221, 0.00172321801, 0.0153451404, 0.00943758246, 0.0460993387], [0.0546116568, 0.0337529257, 0.0672860295, 0.0300631076, -0.0509618744, 0.0803750082, 0.0266496167, 0.00836464762, -0.0235917084, -0.0822790488, -8.284470e-03, -0.058835879, 0.0292353146, -0.0116011314, -0.0306514446, -0.0190244969, 0.0185335744, -0.0786783173, -0.0155120175, 0.0175832175, 4.413010e-02, 0.0528954566, -0.0403323099, -0.069607608, 0.0365300141, -0.0251069907, 0.0692441389, -0.0274931043, 0.0249226633, -0.0409947596, 0.0785389096, -0.0764736608, 0.0916094854, 0.00707275886, 0.0518121384, 0.0560696609, -0.0693489835, 0.0354873575, -0.0553804412, -0.03983761, 0.0714781359, 0.0078154495, -0.00815121084, -0.0757300109, 0.0408701412, 0.0543590188, 0.0833801776, 0.0201027971, -0.0639356524, -0.0135903666, 0.0139172338, -0.0034221774, -0.034079928, -0.0411342792, -0.0512818545, 0.00113908073, -0.0612951182, -0.0249495208, -0.00729493704, 0.0456481315, 0.0804968476, -0.0159455985, -0.014125512, 0.0435555167, -0.0145405522, -0.047752995, 0.0889836624, 0.0227461085, -0.00279119634, -0.0147235831, 0.0120630581, 5.554610e-02, -0.0593590252, 0.065363124, 0.0688355714, 0.0445361137, 0.0142361624, -0.00211911323, -0.00278314273, 0.00459773745, 0.0554612763, 0.0748355612, 0.0599159673, -0.0336036496, 0.0772456303, -0.0501107424, 0.0515446886, -0.030364329, -0.0211837813, 0.0462706722, 0.0820341259, 0.0223374721, 0.00193508342, -0.0284271799, -0.0163741875, -0.0360782556, 0.076603353, 0.0537212826, -0.0138213886, 0.0938045755, -0.0233721454, 0.00468977122, -0.0558106564, 0.0303795151, 0.0331227072, 0.00384415267, -0.0572963245, 0.062440183, -0.0523477234, -0.0103887673, 0.0453210734, 6.792210e-02, -0.047738716, 0.0617307425, 0.0631022081, 0.0227861591, -6.337180e-02, -0.0318681523, 0.067864269, -0.0809689685, -0.0240392406, -0.0507623628, -0.0233648699, -0.0789928212, -0.0765123293, -0.0707760379, -0.0380723961, 0.0736071467], [0.0605628155, -0.0248727612, 0.078899078, -0.0144066699, -0.0615205094, -0.0606713742, 0.0638988167, -0.00467493199, 2.779720e-03, 0.015921535, -0.0469192751, 0.0329150409, -0.0639901236, -0.0430433527, -0.0331386365, -0.03857309, 0.00361105031, 0.0142569784, -0.0566262305, -0.0294501111, -0.0778721272, 0.0234401301, -0.0725739598, -0.0268226713, -0.0365013294, 7.545190e-02, 0.0687603652, 0.0168208648, 0.0689672157, 0.0484891646, -5.196360e-02, -0.00328070112, -0.0225790609, -0.0688057243, -0.0621688924, -5.480490e-02, -0.0648934245, 0.0605876073, -5.585690e-02, 0.0751636401, 0.0234927479, 0.0233271159, 0.067218855, 0.014077439, 0.00601991313, -0.0781976059, -0.0067777657, 5.311370e-02, 0.067700848, -0.0210929029, 0.0388980471, -0.0224816352, 0.065043807, -0.0290588774, 0.0185706448, -0.0104219047, 0.0154934805, 0.0632790476, 0.0206805449, -0.0840217918, 0.0551979207, 0.0364401601, 0.0612040758, -0.0725246221, -0.0615661964, -0.0444206446, 0.0471387878, -0.0154851116, 0.0561655834, -0.0608682483, -0.0631852447, 0.0338415541, -0.0785847157, 0.0549454503, 0.0514520518, 0.00784985442, -0.0478411317, 0.070383504, 0.0636551082, 0.0249539707, -0.0571221448, 0.0328567363, -0.0247197673, 0.0717452168, -0.0460100472, -0.0164334662, 0.0394256972, -0.0488811918, -0.061558526, -0.0410574824, -0.045898553, 0.0480824411, -0.0139832785, 0.0338686407, 0.0627807304, -0.0522969738, 0.0162737984, 0.0683835074, 0.0112207374, 0.0202770736, 0.00601413939, 0.0256215502, -0.0535109378, -0.00459449878, -0.0314980075, -0.0259461962, 0.0780996158, 0.0622146837, -0.0656146705, -0.0682628452, 0.0148933297, -0.0681728125, 0.00569326337, 0.0740390121, 0.0152788917, -0.0142763304, 0.0582005419, -0.0233306233, 0.0159639362, -0.00189964962, -0.0569677912, 0.00201423909, -0.0606318898, -0.0835297257, 0.0725075528, 9.37645615E-4, -0.0257717799, -0.0780659541], [0.0391256474, -0.00585474353, 0.0401425585, -0.0378263704, -0.0124751469, -0.0597232357, 0.0142663093, 0.0576780923, 0.0302195903, -0.0778385848, 5.494100e-02, -0.0522970557, 0.0403575711, -0.0381807834, 0.0718684494, -0.0460506678, -0.0591716953, -0.0463129431, 0.0732560158, 0.0422578417, 6.701860e-04, 0.0207212269, -0.074272275, 0.0680480599, -0.0346525609, -0.0597698838, -0.00210451405, 0.00314834621, 0.0489131734, 0.0462233946, -0.0803128108, -5.276960e-02, -0.0758257881, -0.0334957391, -4.22456767E-4, 0.0702840537, -0.0110956477, -0.0662517324, -0.00537811453, 0.0828326941, -0.00943322759, -0.01307749, 5.154480e-02, 0.0502964407, 0.0375566408, 0.00970889535, 0.0553691946, -0.0285351574, -0.0695652142, 0.0550627559, -0.0754826962, 0.0422614478, 0.0420181639, -0.0490642413, -0.0352285877, -0.0261056609, -7.396090e-03, -0.0418821871, -0.0453401059, -0.00930169877, -0.0590966865, 8.47020419E-4, 0.0346579403, -0.0462684855, -0.0456306636, 0.0453179218, -0.0787008479, 0.0401861034, 0.0181602947, 0.0258318745, -0.0315786935, 0.0397231206, 0.0110448506, 0.0132660707, -0.00202438817, 0.0349237956, 0.0243246779, -0.00556463376, -0.0543613657, -0.0473385416, 0.0666139423, 0.0658240914, -0.00478454912, 0.0258023124, 0.0537906252, -0.0266426131, -0.0559117682, -6.155480e-02, 0.0767619759, 2.113330e-02, 0.0128418608, -0.0436650775, -0.0309007578, -0.0228540692, -0.00869145524, -0.0235867705, 0.0294476841, -0.0751332566, -0.0508025922, -0.0268996339, -0.0348147377, 0.0582697652, -0.0240499452, -0.00600225525, -0.0157390498, 0.0350562967, 0.0218970701, -0.00575079769, 0.0110056698, 0.0344546102, 0.0339363664, 0.019541759, -0.0518110245, -0.0132549703, 0.0361811519, 0.0561941676, -0.00463879853, -0.0193835199, -7.442510e-02, 0.0538401976, 0.0101463515, -5.949050e-03, -0.0513739176, -0.0686802045, -0.0274099112, 0.0420390368, -0.0207920801, 0.0561096296], [-0.0556402765, 0.0540439785, 0.0135475565, 0.0394291691, 7.23631703E-4, 0.057145752, -0.0544433482, -0.04388275, -0.0173110813, -0.0690737218, -0.0618407838, 0.00152494304, 0.0567797571, -0.0780017227, -0.00860543828, 0.00187925401, 0.0597967133, -5.383610e-02, 0.022514632, -0.0290305074, -0.0571264401, 0.0657012612, -0.0310707018, 0.0151914442, 0.0767216459, 0.0219640583, 0.0305412784, 0.0735263675, -0.0369974971, 0.048329968, 0.00547612971, 0.0590324774, -0.0549977496, 0.0266785361, -0.0474140942, -0.0134431226, 0.00474998169, 0.0503830127, 0.043426577, 0.0669024065, 0.0197938383, 0.0653563216, 0.00249116076, -0.0158274528, -0.0207573324, 0.0108527075, 0.0645828396, 0.0456817336, -0.0246080142, -0.0144605385, -0.013248519, -0.0279433336, -0.0372544788, 0.0054789125, -0.0702636167, 0.0110084005, -0.0788787901, -0.0350382291, -0.00208267965, -0.0662291049, 7.569180e-02, -0.0620174296, -0.0668702498, 0.0116601363, 0.0212203376, 0.00482640322, 0.00379355252, 7.881870e-03, 0.0148430187, -0.0246652532, 0.0782957747, -0.0399100743, -0.0221351031, -0.0320471525, 0.031828776, 0.0337732546, -0.0207471456, 0.0559393242, 0.0317131355, -0.0265055709, -0.0141135855, 0.0193794798, 0.0175942164, 0.0656783804, 0.0609929636, 0.0289370418, 0.042502977, -0.0179704167, 0.0413069241, 0.0314664543, 0.0597274899, -0.0739900619, -0.0352065526, -0.0214027185, 0.0441283286, -0.0747647658, 0.038019456, -0.0744601861, -0.0655781776, 0.00190481893, 0.0180907678, -0.0725697055, 0.0394034497, -0.0408104956, 0.0392863154, -0.0606602319, 0.036409393, 0.0570943095, 0.0684687421, 0.0649097263, -0.0741924047, -0.0186751783, 0.0310083032, -0.00841639657, 0.0426489338, 0.0765670389, -0.0699156224, 0.039158605, -0.0523632728, -0.0545566566, 0.0478870571, 0.0486219451, 0.0625327304, 0.0327477343, 0.0444257446, 0.0672823563, 0.0562273674, -0.064304851], [0.0109461751, -0.0574412569, -0.0307261236, -0.0222999454, 0.00538396556, -0.0599597916, 0.0352428928, 0.0690242797, 0.0277999081, 0.0289141145, 0.0699541568, 7.322120e-02, -0.0273494255, 0.0137998471, 0.0145335943, -0.0795796811, -1.954750e-02, 0.0285847336, -0.0492470302, 0.00464854483, -0.0136546595, 0.0486287326, -0.0133727631, 0.033453092, 0.0695883557, 0.0614561588, 0.00798016693, -0.00750899082, -0.0547421239, -0.0588975362, 0.0309691317, 0.0487785563, 0.0527470969, 0.0178285651, 0.00993808545, -0.0267768987, -4.387180e-02, 0.0591906644, 0.038900774, 0.0553288981, 0.00602383353, -0.0790481269, 0.0116904238, 0.00962745864, -0.0394926183, -0.0241914056, -0.0174148213, -0.0457418486, 0.0249931272, 0.0707261562, -0.0166322123, 7.410790e-02, -0.0227085091, -0.00444038445, -0.0193237178, -0.0683104545, 0.0427800827, -0.0274612568, 0.0681073144, 0.0727488994, 0.0609515868, 0.017554149, 0.0288969502, -0.0265645888, 0.0260349214, -0.0307581984, -0.0125920009, -0.0729591325, 0.0188977104, 0.00902450084, 0.0193570424, 5.444530e-02, 0.0289589781, -0.0353501551, -0.0321952924, -0.0803489387, 0.0189326536, -0.0458616838, 0.0197867379, 0.0509524196, -0.0599591434, -0.0502512231, 0.0328657702, 0.00481322408, -0.00212796195, -0.00794374198, 0.0437290855, -0.06040911, 0.01814452, -0.0544618741, -0.0632251129, 0.0592997186, 0.0775385499, 0.0160135422, -0.021427203, 0.0720322654, 0.0640565827, 0.0449606664, 0.0761496872, -7.546320e-02, 0.00434603821, 0.076258637, -0.0536739267, -0.0363613777, 0.0217355564, -0.00393701391, 0.0789084285, 0.0386493355, 0.013021227, -0.03927866, 6.067340e-02, 0.0205420107, -0.0582830496, 0.077187106, 0.0262190402, -0.0218806546, 5.890790e-02, 0.03674604, -0.042964492, 0.0584429502, 0.0762497112, 0.0573556237, -0.0167846512, -0.00652261637, -0.0518156663, -0.063100636, 0.0703220591, -0.076891996], [0.00798051059, -0.0661984831, -0.0568424203, -0.0665813163, -3.609620e-02, -0.0126309963, -0.0427333228, -0.0767065287, 0.0716713145, 0.0690242276, 7.321450e-02, -0.033542525, 0.0690443888, -0.0370796323, -0.0525317527, 0.0700688735, 5.736070e-02, 0.00212030858, -0.0212134011, 0.0623224191, 0.06988848, -0.0786140635, -0.0189499147, 0.0249077231, -0.0543565191, 0.0178370327, -0.046979934, 0.0356347039, 0.0777189806, -0.048338674, 0.00574499415, -0.0323986746, 0.0454315506, 0.0736496896, -0.0190966502, 0.00249488605, 0.00983274635, 0.0460598953, -0.0506327264, 0.0686807409, -0.0181336403, -0.00236263685, 0.0595707111, 0.00420946628, 0.029727431, 0.0729438588, 3.277100e-02, -0.0496125035, 0.00129106548, -0.0207476169, 0.0382013768, -0.0143151274, -0.0179285035, -0.0664165542, -0.074971661, 0.0038078595, -0.0603308119, -0.00528283417, -0.0582219064, -0.0184894707, -7.610270e-03, 0.033523947, 0.0512684062, -0.0252975337, -0.00730627449, -0.051869005, 0.0382928662, 0.00819009542, 0.0795092061, 0.0798774436, -0.0192064233, 0.0802332908, -0.0409041792, 0.00168836815, -7.086360e-02, 0.0038557474, 0.0237676669, -0.0718811601, -0.0413161777, 0.0556225367, 0.0656796172, -0.0562466644, -0.0369736217, -0.0141209988, 0.0184278153, 0.0708227381, 0.0376801603, 0.0429826677, -0.0103088673, 0.0736195371, 0.0563632585, -0.0351566114, -0.0257010199, -0.0491655506, -0.00474262377, -0.00830739085, -0.00220475625, 0.0538057126, -0.0374327637, 0.046160575, 0.0104305316, -0.0667793527, -0.0220953785, -0.0265940316, -0.0535428114, 0.0724062547, -0.0378341079, 0.0487768576, 0.0362651683, 0.0799912884, -0.0303907301, 0.0404088274, -0.0183990374, 6.590260e-02, 0.0296617057, -0.0115056075, -0.0754526407, -0.00722668367, 0.0212519709, -0.0486205928, -0.0245217551, -0.041567713, 0.0621394701, 0.00712614506, 0.0728686452, 0.0481600538, -0.0031552026, 0.0321760923], [-0.0663414299, 0.0426300354, 0.0392529331, 0.0611105226, 0.0573889464, -0.0590767153, -0.0561398827, 0.0238894094, -0.0566790551, -0.0335385762, -0.0559744202, 0.0186106116, 0.0119777396, 0.0395276584, 0.0387611128, -0.00128051499, -0.0373972543, -0.0577906892, -0.0182825718, -0.0499629788, -0.0308159664, -0.0470888615, 0.0622589327, -0.0623621121, 0.0396273918, 0.0578912497, -5.110950e-02, 0.0308944359, -0.0219755806, 0.0639634803, 0.0570002757, 0.0231316034, 0.00627252785, 0.0290817823, 0.0326242484, 0.0259922016, -0.0613687299, 0.0171401221, 4.59064526E-4, -0.0511835217, -0.00417151256, 0.0476814769, -0.00533484621, 3.344470e-02, -0.0342706628, 0.0223388597, 0.0633552372, 0.0564655028, 0.080354847, 0.0764386207, -0.0529776253, 0.0363639556, -0.0387989283, -0.0738141611, 0.0318501666, -0.00410670601, 0.0751962438, 0.00710201543, -0.0567157641, -0.0174378715, -0.0251768306, 0.0622683428, -0.02213756, 0.0399087816, -0.0420284942, 0.0358434357, 0.0603999607, -0.0507845581, -0.0269602966, -0.0399740525, 0.0212676656, -3.239620e-02, 0.045934692, -0.036061611, -0.0704785809, 0.0695105195, -0.0782374591, 0.0176997855, 0.00984176714, -0.0161457621, 0.0187970325, -0.0386791863, -0.0185269397, -0.0104862433, 0.0375368185, 0.025326347, -0.00624514371, 0.0255232807, -0.0422569476, 0.0439435169, -0.0292671155, 0.0351407938, -0.0123177087, 0.0600680597, 0.00150038803, -0.0528158471, 0.0493467636, 0.0485049412, 0.0791520476, 0.0575387403, -0.019515017, -0.0321618468, -0.0108314166, 0.0771240592, 0.00207841699, -0.0472438186, -0.0738942251, -0.0663815886, 0.00384735572, 0.0763699561, 5.471970e-02, 0.0230642855, 0.0497170091, -0.0407000445, -0.00997102074, 0.0492263176, 0.0339818895, -0.0725697428, 0.0793505832, 0.00777261751, -0.0442768931, 0.0596242025, -0.0119486889, -0.0532545149, -0.00830631796, 0.0601019561, 0.0746248737, 0.0733791366], [-0.0486217141, -0.0146728037, -0.079799205, 0.0743714272, 0.0464412868, -6.474670e-02, 0.0664585307, 0.061475765, 0.0452268086, -0.0390301868, -0.046561759, 0.0551337674, -0.0271581765, -0.00552322343, 0.0537556969, 0.0557196029, -0.0115918294, -0.0629414842, 0.0731756315, 0.0168386586, -0.0634528399, 0.0768120065, 0.0811319872, 0.0397798643, 0.0553797558, -7.461610e-02, 0.0775227323, -0.0185907967, -0.0748567134, -0.0219507907, -6.984650e-04, -0.0162735451, -0.0739787668, -0.0493590124, 0.073097989, 0.0280885696, -0.0387752354, 0.00192115514, 0.0798240453, 0.061501462, 0.00479116477, -0.0297697932, 0.0775909125, -0.0312815234, -0.0550530888, 0.0467722826, 0.0471627712, 0.055735752, 0.0328898281, -0.026456112, -0.0718301907, 0.00193598121, -0.0445897803, -0.0320654809, 0.0487738363, -6.841080e-02, -0.0482220091, 0.0799339786, 0.0277275182, -0.0408963971, -0.0472789332, 0.0209430177, 0.0663665459, -0.0194402132, 0.0221723784, -0.0674988404, 0.0326812714, 0.0637393519, -0.0124477409, -0.0454397798, 0.00227077515, 0.0160468537, -0.0566070937, 0.068409048, 0.0378433056, 0.0586102307, -0.0490396097, 0.0136979483, 0.059442088, -0.0804930254, -0.0619038641, 0.0606136173, 0.00991172157, -6.482120e-02, 0.0092181079, 0.0764659196, 0.0172662716, 0.0148256868, -0.0232630633, -0.0606739856, -0.0475577451, -0.0152511494, 0.0649844334, 0.0578783788, 0.0589183122, 0.0752493143, 0.0107416511, 0.0285146143, 0.0140851866, -0.0564229153, 0.00267119147, 0.0424824283, -0.0212919544, 0.0646387115, 0.0789888203, 0.0716974512, 0.0604809672, -2.90420343E-4, -0.0304146502, 0.015259875, -0.0497435294, -0.0393154807, 0.0135381771, 0.0105765844, 0.065152511, -0.0776007771, 0.00638851291, -0.0804690644, 0.0719811469, -4.500660e-02, 0.0125345103, -0.00232141139, 0.0462715887, -0.00185353414, 0.0217545554, -0.0580672473, -0.0106159858, -0.0507289656], [0.0714268088, 0.0112134181, -0.0304813944, 0.0683481246, -0.0133202411, -0.0585959107, -0.0729096681, -0.0304196496, -0.0308561344, 0.0103826988, 0.0489888564, -0.0312732533, 0.0183005892, 0.035875719, -0.0194203164, -0.0365101025, -0.00857852306, 0.0349496491, 0.0616781637, 0.0705441534, 0.0414293334, -0.0493687317, 0.0532591045, 0.0247784555, -0.00329281786, 0.0113496445, 0.0745041072, 0.0206780415, -0.00697550363, 0.0451697558, -0.0681819245, 0.0227148123, -0.00672126561, -0.0502851903, 0.0493182875, -0.0531232432, 0.0208575707, -0.0555366687, 0.0162045266, -0.0157995895, 0.0817713811, -0.021509273, -0.0103988862, 0.0443130471, 0.0143957213, 0.0629125759, 0.0290539488, -0.023775978, 0.0721048042, -0.00297991559, -0.0512995124, -0.0507286228, 0.0207890812, -0.0225404687, 0.0263026021, -0.021583328, 0.00415911479, 0.00941103883, 0.0206777807, -0.0447807126, -0.0726041272, -3.13610828E-4, 0.0369798504, -0.0102590937, -0.046580147, 0.0282290708, -0.0631151646, -0.0320167057, 6.81426376E-4, 0.0260981061, -0.0520595312, 0.0680316836, 0.00644419109, 0.0696212575, 0.0525219031, -0.0387533791, -0.0649810508, 0.0250346251, 0.0648608878, -0.0773968548, 0.0556959696, -0.0245965552, 0.0115540484, -0.0819002464, 0.0583262257, 0.016657846, -0.0256109238, 0.0736513287, 5.239660e-03, -0.0180534404, 0.013970199, 0.0507377535, 0.0223268401, -0.0199789908, -0.0510273091, -0.0587335043, -0.0489804111, 0.0592907332, 0.0421697795, -0.0757159888, 0.0571935438, -0.0303517245, -0.0784696713, -0.0475359634, 0.0549759753, -0.0421555825, 0.0326109678, -0.0797454714, 0.0267315898, 0.0665745884, 0.0425229743, -0.0114320274, -0.0676178858, -0.0174970441, -0.00783542357, -0.0679945051, -0.0158580132, 0.0747282505, -0.0384153128, -0.0462595895, 0.0622931123, -0.0546475723, 0.00848481431, -0.0425358489, 0.00479358342, -0.0350108482, 0.0226746332, 0.0450907797], [-0.0422744602, -0.0249016061, -0.0625398159, 0.0471370183, -0.072490938, 0.0628238842, -7.303280e-02, -0.0347947739, 6.295980e-02, -0.0677463785, 0.0846853256, -0.0560773946, 0.00900010113, 0.0591649376, 0.0781412646, 0.00324116624, -0.0312752686, -0.00371290068, -0.0585099496, -0.0420459025, 0.00682722777, 0.0692950785, -8.65828421E-4, 0.00558589399, 0.0461560637, -0.0714456961, -0.0441331267, -0.0035459653, 0.060133189, -0.04347739, 0.0182567816, 0.0127587654, -0.0172074661, -0.0385967642, 0.047279276, 0.0215765256, -0.00240555173, -0.0118850162, 0.0545654446, -0.00256581441, 0.0483225621, 0.0446453132, 0.0571008734, 0.0118485596, -0.0827324986, 0.0753601715, 0.00185801962, 0.0396092907, 0.0184463356, 0.0709939674, -0.0081389742, 0.00206340686, -4.469300e-02, -0.0439375974, 0.0575432703, 0.0670683235, -0.0338168927, -0.00105886697, -0.027889533, -0.0340468176, 0.0780090094, -0.028032884, -0.0334121361, 0.0603060275, -0.0808971151, -0.0069125616, -0.0178481154, 0.0805351808, 0.0235898159, 0.0233920924, 0.0626179129, -0.0474897325, 0.0421739146, -0.0150645124, -0.00899823103, -0.024418883, -0.0533701181, 0.012760371, -0.069218412, 0.0657951906, 0.0187276136, 0.0568836108, 8.64076428E-4, 0.0173703562, -7.011300e-02, -0.0403874144, 0.0220776703, 0.0445551239, 0.0483072773, -0.0211730115, -0.00492343493, 0.070700258, -0.0692334324, 0.0315008536, 0.0172720645, 0.0170868132, -0.056036111, -0.0642867684, 0.0565737672, 0.0104221348, -0.0230798237, 0.0401904061, -0.0792617723, -0.0505990162, 0.0510038882, -0.0700919703, -6.683330e-02, -0.00459803315, 0.0105583807, -0.0541088767, 0.0523984432, -0.0152758202, 0.0664917156, 0.0463931039, 0.0665975511, 0.0197648332, -0.0168348867, -0.0368068442, -0.00847891625, -0.020082023, -0.0338723175, -0.0829035714, 0.0312289651, -0.0525095314, 0.0692117885, -0.0463359728, 0.00176613627, -0.048553668], [0.063907817, -0.0131564531, 0.0736204758, 0.0743543133, 0.0690898746, 0.0581090637, -0.0445553362, 0.0244816635, 0.032386627, -0.06277477, -0.0602955259, -0.0569996163, -0.0407385901, 0.0769400746, 0.00513196038, -0.0247029103, -0.0128415311, 0.0119571537, -0.0553139411, -0.0642875433, -0.0693937242, -0.0752053112, -0.013411548, 0.0848491266, -0.0449573658, -0.0887713059, -0.0573000386, -8.15980136E-4, -0.0558324121, -0.0361593589, 0.0134279644, 0.0711456388, -0.039098572, 0.0357508361, -0.0566071048, -0.0316163339, -0.00997823197, -0.00356775732, -0.0505043864, -0.0246739369, -0.0212147143, -0.0318488255, 0.0141264135, -0.0107912458, -0.0694154724, -5.0685805E-4, -0.0843981504, -0.0607628673, -0.0549386144, -0.0555038899, 0.0327112377, -0.0287841633, 0.0224474967, -0.0765149817, -0.0590764545, -0.0647782609, -0.0433782227, 0.0489753745, 0.0163349882, -0.0493855104, -0.0659269988, 0.0776515975, 0.00443822518, -0.082282111, -0.0538426265, -0.00852420553, 0.0190573316, -0.00734505663, 0.0274666231, -0.0767363459, -0.0311338697, -0.00445567165, 0.0166097954, -0.0234314259, -0.0757991225, 0.0740281343, -0.0757617354, 0.0620178953, -0.0137499264, 0.0688715354, 0.00328152347, 0.0651612133, 0.0524242111, 0.0115192039, -0.0517939664, -0.0412048139, -0.044230029, 0.0416025519, 0.0153851984, 0.029941557, 5.772830e-02, -0.0562309586, -0.0531770363, 0.0500757359, 0.0729532614, -0.0337075442, 0.00710453885, 0.0353513882, -0.0218103547, 0.0226088185, -0.0522301905, -0.0741119682, -0.0392488576, 0.0474328436, 0.00848560407, 0.0303402338, 0.0594960861, 0.00337756937, 0.00641849684, -0.0895335525, 3.781180e-02, 0.0118568726, 0.0497931615, 0.0777392163, 0.0361854918, 0.0317280367, 0.0442076959, 0.0342789106, -0.0486716591, -0.0419913158, -0.065473482, -0.0614704787, -0.0308058076, -0.0446392633, -0.0290314984, -0.0367047302, 0.0236300044, -0.0860759467], [-0.0250521395, -0.0493684523, -0.0340731181, 0.00863918475, -0.0147432853, -0.0306789335, -8.29353812E-5, -0.0491145216, 0.0529567115, -0.00506058102, 0.0145294033, -0.0561887287, 0.0263788924, -0.0272664353, 0.0438418202, 0.0325997397, -0.0248676185, 9.82760539E-5, 0.063945204, 0.0140154846, 0.0642023757, 0.0567308329, -4.857150e-03, 0.0402581953, 0.0693203583, 0.0525840409, 0.0305592194, 0.0779651552, -0.0349164568, -0.0375358537, 0.0648702383, -0.0369331539, 0.0423527919, -0.0100773675, -0.025228573, 0.0284088831, -0.0193219595, -0.0325312465, -0.0355308056, 0.0554434508, -0.0760447755, 0.0118691828, 0.0143496823, 0.0399739221, 0.0150745101, 0.0719799623, -0.0431385599, -0.0159779601, -4.343030e-02, -0.0733595714, 0.0685985312, -0.05496433, -0.0850772932, -0.0548730679, -0.0466240868, -0.0531612709, 0.0482868738, 0.0383717492, 0.0528714694, 0.00573452329, 0.0763110071, -0.0528134406, 0.0237881262, 0.0210698564, -0.0812815279, 0.0923655256, 0.0383222364, -0.00707762828, 0.0855745896, 3.08918592E-4, -0.038501814, 0.0350977629, 0.0792996808, 0.0205739513, -0.0315261707, 0.0721745715, -0.0552735254, -0.05652887, 0.0624184236, -0.0704075322, 0.0613637455, 0.00506618712, 1.150750e-02, -0.0141455419, -0.0502128676, -0.0087494906, 0.0648617074, 0.0775311812, 0.0182657987, -0.0773239508, -0.0184720233, -0.0328558385, -0.0871119946, 0.0695953667, -0.0257880185, 0.027236104, 0.0406814665, 0.0582605526, -3.747960e-02, -0.0848741605, 0.0334581919, -0.0173592363, 0.0292386319, 3.80354672E-4, -0.00581658818, 0.00992244761, -0.0519340709, 0.0780895426, 0.0389423519, -0.0904909595, 0.0337253548, 0.0134366807, 0.0424818844, -0.026074592, -0.0416875705, 0.0795093924, 0.062019825, 0.00491041457, 0.0370146818, -0.0242456105, -0.0954775661, 0.0503762737, -0.0688226596, 0.0352055728, -0.0212508645, 0.0588935278, 0.0549156815, -0.0835164114], [0.0574425235, 0.0214736052, -0.0309171043, 0.0383824967, -0.0344571881, -0.07454817, 0.0254958216, -0.0428260565, -0.080906868, 0.0167214777, 0.0536312386, 0.0195060167, 0.00643361034, 0.0620487816, 0.0685874596, 0.0282530338, -0.0400688201, -0.05729739, 0.102810934, -0.0338037163, 0.0220796522, 0.0121202944, 0.0025879126, 0.0403261967, 0.0323549472, -0.0398921855, -0.0489798263, -0.0725559071, 0.0209349934, 0.0560412966, 0.0450772569, -0.0535575598, 0.0559883714, -0.06490767, -0.0917415693, 3.664460e-02, -0.0919698178, -0.00197956408, -0.0276481956, 0.0613993444, -0.0432123095, -0.0224514846, 0.0654874742, -0.0334666371, -0.075110197, 0.00805534143, -0.0592717342, -0.0374491401, -0.00255554682, -0.0386472046, -0.0354833268, -0.0616656169, -0.091830492, -0.0271956958, 0.0155270761, -0.0544718355, 0.00934184249, -0.0269625094, 8.238020e-03, 0.0961596295, 7.260780e-02, 0.027107751, -0.0104808016, 0.0396373272, -0.0834479704, -0.0495361052, 0.00893564243, -0.0515513755, 0.0054388484, -0.00637838803, -0.0530327447, 0.06907361, 0.0718600675, -0.0196999572, -0.0367436409, -0.0442022681, -0.0111161591, -0.0517091192, -0.031093115, -0.0572211705, 0.0579076596, 0.0976455137, -0.0215503275, 0.0390963852, -0.0258294884, -0.015658373, 0.0504704863, -0.0335891843, -0.0913539528, 0.0268109832, 0.02067874, -0.0244331304, -0.0946936532, -0.019870162, -2.731020e-02, -0.0369400866, 0.0261856299, -0.0216286387, -0.0615513622, -0.092334196, -0.0203340072, -0.0667831749, -0.00871338788, 0.0672840774, 0.101811185, -0.0108058015, -0.018866919, 0.0637814998, 0.00477998611, 0.0250103455, 0.0951400101, -0.0150321377, 0.0307860542, 0.0302883424, -0.0153392563, 0.0706667453, 0.0140786637, 0.0470731817, -0.0165377595, 0.0100403763, -0.00453006569, -0.0905995816, -0.0492639281, -0.0376341715, 0.0632686316, -0.00150884851, 0.0649277493, -0.0476601534], [0.0234144628, 0.00452617649, -0.067600064, 0.00273092068, 0.0735261142, 0.0148820095, 0.0374228358, 0.0525659509, 0.0275418144, 0.0299004968, -0.0124305701, -0.0416707471, -0.0814829468, 0.0705272555, -0.0227089841, -0.0431977585, 0.00335622812, 0.0219442677, 0.0459293425, -0.0124870893, -0.0547720343, 0.0357278883, 0.060596317, 0.053037148, -0.0280202758, 0.0379843451, -0.0313359946, 0.0366149805, -0.0126740979, 0.00422499096, -0.00624793069, 0.0309818722, 0.00315030268, -0.0190620702, 0.0516854748, 0.0817462429, -0.0562670976, -0.0175973214, -0.025115665, 0.0172931254, -0.0935595408, 6.04631263E-4, -0.0293754414, -0.00504078344, -0.0571220741, -0.0114546344, -0.0692881346, -0.0436600745, 0.0250870381, 0.0663261115, 0.0358998142, 0.0221869145, -0.0978872776, 0.0398926884, -6.531060e-02, 0.0509136096, -0.0234751031, 0.0721791684, 0.028067667, -0.0316815525, 0.0248269569, -0.0210111458, -0.0235216152, -0.0516087674, -0.0701153055, -0.0551176593, 0.0322771259, -2.882030e-02, 0.0261112433, -0.0759935752, 0.0480146408, 0.063895084, 0.0952717587, -0.0228148755, 0.0376003161, 0.088401705, 5.608240e-02, 0.00392679684, 0.00195987592, 0.014552366, 0.0498283729, 0.0384716466, 0.0334434472, 0.00989164878, 7.838180e-03, -0.0373643264, 0.0261951052, -0.0418703184, 0.0264237802, -0.0612577647, 0.0827350467, -0.040676523, 0.0291740149, 0.0254465509, 0.0814749374, -0.0202777497, -8.1003108E-4, 0.0520911217, 0.0625012591, -0.0474561974, -0.0428945236, -0.0312429145, 0.0493192933, 0.00588930305, 0.0124385674, -0.0506039634, -0.0659177676, 0.0207447335, -0.00223716092, 5.15216263E-4, -0.0466809161, -0.078922756, 0.0678913519, -0.0168921016, -0.0352497697, 0.0295602493, -0.0126842959, 0.0762967542, -0.00937727279, -0.0637629703, -0.0278547667, 0.0614869967, 0.0233711451, -0.046806965, -0.0436038449, -0.0291981902, 7.543370e-02, -0.0092033064], [-0.0861246585, -0.012361601, 0.0266292486, 0.0388118848, 0.00573828723, 0.0362790264, -0.0728596076, -0.043079529, -0.0202453267, 0.0217022765, 0.0108002173, 0.0581834875, -2.427190e-02, 0.0978155359, 0.0688474178, -0.0512216464, 0.0548145734, 0.0626626387, 0.107625127, -0.0333481692, 0.0364992544, 0.0652735233, -0.0413742065, -0.0141054671, -0.0408760309, -0.016493978, 3.883690e-02, -0.0200438816, 0.00277416175, 0.0875750258, 0.0976366698, 0.0525529943, 0.00424488122, -0.0733915269, 0.0268038511, -0.0378065854, -0.0604462065, -0.0451671183, 0.0652150139, 0.0147097902, 0.0507435836, -0.0540167578, -0.00677435333, 0.0527846515, 0.076500468, 0.0388461277, 0.0423865914, 0.0186349358, -0.0394841731, 0.0269490816, -0.072192885, 0.0858815163, 0.0320031606, -0.0294355247, -0.0320599526, 0.0461034626, 0.0538521744, 0.0175070819, 0.0639488176, -0.0227484237, -0.0522532538, 0.0794181377, 0.0355536826, 0.0340993293, -0.0858239383, 0.00158239226, 0.00751269422, -0.00357570825, -0.0373982862, -0.0225769747, 0.0182334948, 0.00208560843, -0.00806042179, 0.0577608459, 0.0440705866, -0.0284262691, -0.0818658769, 0.0164970811, -0.0267740134, 0.0226261877, 9.670790e-03, -0.0282400362, 0.012399273, -0.0475378819, -0.00874623749, -0.00194183958, -0.00300106429, -0.0752399564, 0.0620596371, -0.0259919502, -0.0104632182, 5.072140e-02, -0.0965770855, -0.0767083094, 0.0429821536, 0.0706640407, -0.061666742, 0.0577722676, 0.0523353331, -0.0493798442, 0.00233308086, -0.0277501754, 0.0443401933, 0.0530692413, -0.0559527054, 0.0762547702, 0.0736006796, -0.0294465907, -0.0166592877, 0.0551203266, -0.0123193813, 0.0162831862, 0.0451480784, 0.0548620969, 0.0689232349, 0.0112643428, -0.00158737472, -0.0447601825, -0.0400838479, 0.0884858965, 0.0359014533, -0.0377949849, 0.0785550177, 0.00535605848, 0.0131374067, -0.0772413313, -0.0786099508, 0.0585779846], [0.0364617519, -0.00430716947, -0.0475215428, -0.00551758148, 0.0802368298, 0.0695774332, 0.0773623287, 0.0183161758, 0.0519183911, -0.0717069432, 0.0625478327, 5.687500e-02, -0.048816707, 0.111407846, -0.0386708714, -0.00987507496, -0.00783227197, -0.0199813098, 0.044151213, 0.119779177, 0.0171201862, 0.0797860771, -0.0369558595, -0.0725812613, -0.021207219, 0.0464158878, -0.0220043771, -0.046231892, -0.0461681113, 3.52307281E-4, 0.0127465622, -0.0332011096, -0.0268150363, 0.0125996796, 0.0109435786, 0.0707424283, 0.0568591468, 0.011291909, -0.0217177384, -0.0296717435, 0.0445949957, 0.0660821572, 0.0951673761, -0.0292377826, -0.0568191595, 0.0632235631, 0.00185558386, -0.0652971119, 0.0738404617, 0.0987651944, 0.0177900717, -0.00957775582, -0.0426142327, -0.0499986112, -0.0445965752, -0.0436837226, 0.0727763921, -0.0681507736, 0.0333446078, -0.029189676, 0.0342172161, -0.0588622913, -0.0983915179, -0.0714923441, 0.0386948325, 0.0600713678, 0.0685799792, -0.0412568226, -0.0379808955, -0.0619885735, -0.0570191815, 0.06971284, -0.0152396392, -0.0144447368, -0.0432103537, 0.0130348084, -0.0645384118, 0.0384855196, -0.045416709, -0.0191878732, 0.0543090701, 0.01310567, -0.0631052553, -0.0949687511, -0.079719223, 0.0412624851, 0.114423238, 0.0158246513, 0.0311507359, -0.0779451579, -0.0222699549, -0.0145426337, 0.0552015305, 0.0578723662, 0.082372345, -0.0071537178, 0.0542968698, -0.0345175229, -0.00452399952, -0.108941704, 0.0189852063, 0.00673890207, 0.0516154543, -0.0178470798, -0.00878532603, -0.0332511887, 0.060858488, -0.0688946322, -0.0560662635, 0.0630160421, 0.0637605786, -0.0620574392, -0.0210742038, 6.543030e-02, -0.0251576584, -0.0245487709, -0.0121150212, 0.0683644786, -0.0539560802, 0.00401367806, -0.108456463, -0.0447415113, -0.0177805498, -0.0198699161, -0.0818176121, -0.0712623075, -0.0552458055, 0.0300313979], [0.0425129645, 0.0269562118, 0.0316283964, 0.0736315101, 0.0596923754, 0.0721422583, -0.0403856523, 0.00972415506, -0.00155863934, 0.0762793421, -0.0547685064, -0.055042956, 0.0535016656, -0.00348787918, -0.0290145129, 0.0281829089, -0.021509517, 0.0144503582, 0.102900907, 0.00554953888, 0.0737559572, -0.00265400042, -0.0161229707, 0.0596615858, 0.0352391489, 0.00321447663, 0.0301658548, 0.0686744601, 0.0595465116, -0.0378046855, -0.0489789248, -0.00397287216, 0.0294670779, 0.0127835246, 0.107906826, 0.0926322638, 0.0382410549, 0.0197088737, 0.0949268937, 0.100419216, 0.0265826378, 0.0820887238, 0.0486097187, 0.033081267, -0.0541892909, 3.685520e-02, -0.0516835637, 0.0895283669, 0.0376315154, 0.0573786162, 0.0527865775, 0.00781866256, -0.0591903627, 0.0156173566, -0.0468290932, 0.0526809357, 0.0617087223, -0.0200331938, -0.0607888177, 0.0353038907, 0.054389596, -0.0186389238, 0.00642992789, 0.0415780656, 0.032941591, -0.0545115583, -0.038392894, -0.0571833886, -0.0665673911, -4.109630e-02, -0.0657444596, -0.00465117441, -7.46727048E-4, -0.0577585921, -0.0653178319, 0.00486441562, 0.0530498587, 0.0498739965, -0.0269986913, -0.0626884922, -0.0360890515, -7.76949397E-4, -0.0325445533, 0.0404099375, -0.0513815805, 0.102560148, -7.24156213E-4, -0.0433036797, 0.0229252744, -0.101168193, -0.0192461796, 0.0613805316, 0.0507010594, -0.023970779, 0.06970682, 0.0107056489, 0.0567658283, -0.0641064644, 0.0443527251, -0.0751128718, -0.00573056424, 0.0624480769, -0.00423386926, 0.0931143462, -0.0766790956, 0.0596867949, -0.0227519292, -0.0465592183, -0.00801356136, 0.0478196479, -0.00739313429, -0.0414738283, -0.0367326327, -0.0607153811, 0.0468926504, 0.105178915, 0.0059942049, -0.0635250807, 7.391180e-02, -0.0548894182, -0.0720602125, 0.0973404273, -0.0660787225, 0.00629013404, 0.0753066837, 0.0395625494, 0.0293874256, 0.0884247943], [0.0801483169, 0.0514836386, -0.065484643, 0.00428553065, 0.0835094153, 0.0868383571, -0.0205152109, 0.034838371, -0.0489731804, -0.00425328361, 0.0884099304, 0.0825964212, -0.0236516818, 0.0978109911, 0.0844372362, 0.100010544, -0.0483348109, -0.0768551826, 0.0487834439, -0.026271807, 0.0217800774, 0.047314506, 0.0457570963, -0.0110712769, 0.00262077269, -0.0224123634, 0.0516533144, -0.0275075622, -0.0497749932, -0.028493261, -0.0189424884, -0.0573929884, 0.0534101687, 0.0774414613, 0.0774281099, 0.0791448578, 0.00230580638, 0.0410569906, -0.0211026426, 0.0404727273, 0.0681453198, 0.0805672779, 0.0804275646, 0.0338303745, 0.0589332059, 0.00843271799, -0.00182396884, 0.0209349245, 0.106001623, 0.126157403, 0.0107448222, 0.0193587877, 0.10133262, -0.0250557326, 0.055638995, -0.0051803547, 0.0352769047, 0.0670760646, -0.0888141468, 0.0303387046, -0.0373311564, -0.015675772, -0.0468666591, 0.0357480794, -4.213580e-03, 0.0397294387, 0.00525249867, -0.0320563726, -0.0735259652, 4.60960437E-4, 0.0788778364, -0.0465622954, 0.0433159359, -0.0101879062, 0.00872353371, 0.0445893221, 0.0259987265, -0.0428625196, -0.0458892658, 0.0477216095, -0.0285893735, -0.0607931613, -0.0533828065, 0.0135951629, 0.00398117909, -0.043649707, -0.00386540382, 0.0852926373, 0.0291437041, -0.0800098776, 0.0789904445, -0.00585186714, -0.0156703014, -0.0561830774, 0.0188228469, -0.00420990959, 0.0549527965, 0.0883998424, -0.0781696513, -0.0210017059, 0.043102324, 0.0238225721, 0.0687703043, 0.00554024521, -0.032430727, 0.0407173783, 2.33207305E-4, 0.00457919342, -0.02616309, 0.0665168315, -3.992110e-03, 0.0207162425, -0.0222237539, -0.0642239079, -0.0322203673, -0.0234148726, -0.00929685775, 0.054104533, -0.0191127695, -0.0072595668, -0.0767564177, 0.00537980068, 0.0198052265, -0.0269675162, -0.0618532896, 0.0325606242, 0.0872152373, 0.0673850626], [0.0267568566, 0.0289426725, 0.0164902024, 0.0297443978, 0.0948733165, 0.0883034542, -0.0452606082, 0.0581341945, 0.0993203744, 0.0556107685, 0.0467752665, 0.00713326595, -0.077133365, 0.00522130029, -0.0176059213, 0.0530755632, -0.0708302408, 0.00608530547, 0.113348506, 0.101395421, -0.00892095454, -0.0314874463, -0.00568507705, -0.0436318442, 0.0909500643, -0.0554080568, -0.0462674685, -0.0288489871, 0.0518872552, 0.106279902, -7.110920e-02, -0.0651606098, -0.0602293722, -0.0172183216, 0.0216595028, 0.0199323893, -0.0441480838, -0.00690754363, 0.0490073599, -0.0588883534, 0.026026845, 0.0577535369, -0.0615180954, 0.0064211227, -0.0435263887, 0.0308502149, -0.0226969533, -5.37306245E-4, 0.0711636245, 0.0558707826, -0.0153277442, 0.0276935976, -0.025448449, 0.011576416, 0.0497699529, 0.0205995291, 0.0236111898, -0.0572533906, 0.0153234927, 0.0580205768, 0.0178248808, -0.00614102883, -0.107501127, -3.774070e-02, 0.0163556319, -0.015756147, 0.0933253914, 0.0434619896, -0.0627518296, 0.0555082411, 0.0531581938, 0.0163887087, 0.0205416568, 0.0699215904, 0.0298116673, 0.082177043, 0.0873155743, -0.0807478725, 0.0392529927, 0.0241213292, 0.0322472453, -0.0249639116, 0.0485260077, -0.0703464374, 0.0300505254, 8.007690e-02, 0.141742885, 0.0870295912, 0.028296534, 0.0140807955, 0.118504569, -0.0369159393, -0.0315718949, -0.0448912904, -0.0425725095, 0.00757561624, 0.0836879164, 0.0708571374, 0.00385276414, -0.0474840552, -0.0479157567, -0.0161431264, 0.0471980274, 0.0154262669, -0.0761131123, 0.00841927155, -0.00592218479, -0.0353715494, -0.0666910782, 0.0278160665, 0.0658900514, -0.0657336637, 0.00558616826, -0.00715375831, 0.0505729169, 0.0555525348, 0.0536657721, 0.0166545827, -0.00675629638, 0.0240341164, 0.0182897188, 0.00501820631, 0.0866351723, 0.0346744023, 0.0163584482, 0.0185822379, 0.0601112954, -0.0234030951], [0.0835665091, -0.00423310045, -0.0283922181, 0.0122449156, -0.0508843474, -0.0539890677, 0.0564402938, 0.0229231305, 0.0762971118, 6.775950e-02, 0.0423577949, 0.0476493761, -0.0134417005, 0.0615261458, 5.343700e-02, -0.00172191719, -0.0323937908, 0.0668742284, 0.0600404888, -0.0275403485, 0.0403917469, 0.0420102067, 0.0414517224, 0.0479520783, 0.0114099663, -0.053275276, 0.083251819, 0.0306068901, 4.784260e-02, -0.032173764, -0.036745403, 0.0451488122, 6.463230e-02, 0.0757886842, 0.0795718505, 0.0772263184, -0.0648036897, 0.0286190044, 2.101160e-02, 0.0332781114, 0.0362699218, -0.0137760406, 0.0217451863, -0.0835221782, -0.0447248891, -0.0287895575, 0.0338796079, -0.0514771715, 0.0149657726, 0.0247736908, 0.0809274167, 0.0575070754, 0.0635557696, -0.0275134798, -0.082363531, 0.092076987, 0.0316777192, -0.0599770434, -0.050673265, -0.0416627154, -0.00186655123, 0.0549008474, 0.0362357572, 0.0329210162, -0.0114487596, -0.0361698605, 0.0687693059, 0.0192341059, -0.00850349292, 0.0109496154, -0.0239517111, -0.0888047218, 0.0150440987, 0.025762653, -0.0170604121, 0.0405913293, 0.0744694695, 0.00542329717, 9.60364588E-4, 0.0214169547, 0.0753287748, 0.0671178699, -0.0658270418, 0.0475322418, -0.0194007028, -0.0243852343, 0.0281467047, 0.0895005614, -0.0840126648, 0.0305892304, 0.0785796344, -0.0981882214, -0.0237927753, 0.0382989906, 0.0282132477, -0.0745958387, 0.0249836147, -0.0122334408, -2.919890e-02, -0.133508235, -0.0322931744, 4.123270e-02, 0.0536317267, -0.00283115241, 0.0189162213, 7.876750e-03, 0.0793054327, -0.0252654366, -3.279720e-02, 0.0170634314, 0.0474670343, 0.0173332114, -0.0644746274, -0.0538422465, -3.089030e-02, 0.101974532, -0.05693046, -0.0505883433, 0.0623324141, -0.00915185641, -0.053419739, 0.00847317558, 0.0612115338, 0.109892286, 0.0775291249, -0.00707634678, 0.0862172916, 0.0983521938], [0.066796653, -0.0274440963, -0.0210558772, -0.0442534462, 0.0625420138, 0.0284798108, -0.0145676332, 0.0782773942, 0.0146278329, -0.0344152674, 0.0546632297, 0.00619027903, 0.06310381, 0.0101187965, 0.0356165655, -0.00592082692, -0.00718252687, 0.0132005736, 0.0151105328, -0.0205611419, 0.071645923, 0.0147204418, -0.0853946358, 0.0753273963, -0.013844844, -8.132760e-02, -0.0397340134, -0.069402121, -0.0290658101, -0.0458739586, -0.0255638622, 0.0936295762, 0.066253148, -0.0375122577, 0.0564083606, 0.0491912663, -0.026678877, -0.0225369167, 0.0496238098, 0.0109086465, -0.0742043853, -0.0117987907, 0.0456859507, -0.0253739394, 0.00671957945, -0.00801345705, 0.0297704581, 0.0616854914, -0.0420354381, -0.0427576974, -0.0624733791, -0.0216622967, -0.00409287075, -0.0736634284, -0.00808180682, -0.0248728395, -0.0184770506, -0.0372072905, 0.0587062947, 0.0270876624, -0.0438431576, 0.00935463142, -0.0738575384, -0.0097043924, -0.0339601636, 0.072203815, 0.0740754455, -0.0250136461, 7.517260e-02, -0.0172839295, 0.0422308445, -4.700970e-02, 0.00974709726, -0.0605954379, 0.00169386086, 0.0743770823, 0.0706516206, -0.0282292496, 0.0214776844, 0.0206157174, 0.0709903166, 0.0136373378, -0.0185947362, -0.059495423, -0.00973706878, -0.018024236, -0.0378146134, -0.0234886725, 0.0369681045, 0.0377140082, 0.0588877089, 0.048732955, 0.035626635, -0.0416454785, 0.064493753, 0.0613136925, -0.00554117747, 0.0410855711, -0.0425343029, -0.0465403311, 0.0487511121, -7.277720e-02, 0.0564723909, 3.354830e-02, -0.0299419966, 0.00117639871, -0.0714503824, 0.0450618044, -0.0340318084, -0.0381919406, 0.0701060817, -0.0137441047, 0.044977475, 0.032451123, 0.052284956, 0.00757392915, -0.00601913407, -0.00168698665, -0.00223307568, -0.0487794951, -0.0945284441, 0.0124406936, 0.0777820572, 0.130464926, -0.0169893522, -0.00729232095, -0.0209828541, 0.0507260449], [0.025236357, -0.0413151085, -0.0625434443, 0.0781685561, 0.0936193317, -0.0239128694, -0.0393009149, -0.0353122242, 0.00612831162, 0.102139346, 0.0853357687, 1.392570e-02, 0.0817387923, 6.749290e-02, -0.0593398102, 5.537490e-03, -0.0747418329, -0.0252938122, 0.0620669685, -0.0377901047, -0.030174613, -0.0225471929, 0.0159932058, -0.0530872419, -0.00197658478, -0.0466121696, -0.0388278365, -0.0521137789, -0.0813548043, 0.0452358276, -0.0103943245, -0.0463688225, 0.0418158211, 0.00163563469, 0.0914750173, 0.0185326934, -0.00903408881, 0.0348972715, -0.0189354736, -0.0643989444, 0.0231222454, 0.0467790551, 0.026431188, 0.0584842339, 0.028677756, 0.0662872717, -0.0140139433, -0.0599538051, 0.0847503915, -6.965890e-02, 0.0880230516, -0.0626391619, 0.0617653243, -0.0700876117, -0.00860246271, -0.0192389823, 0.0502223969, -0.0282852855, -0.0227446742, -0.0464114137, -0.0143396109, -0.0501123443, -0.0543211438, 0.0573773794, 0.0269018337, 0.0468203798, -0.063938275, -0.0260470174, 0.061541371, 0.0229251534, 0.0641818196, -0.0422924533, 0.0564560592, 0.0398020856, -0.00882508513, -5.875320e-02, 0.0544133931, -0.00167230249, -0.0572505444, 0.0261834655, -0.0342226326, 0.0709632337, -0.0584511347, 0.077254042, 0.0206868947, 0.0594323762, 0.023297159, 0.0366290323, 0.0620989129, -0.0295922942, -0.0594171658, -0.00868005399, -0.0347488746, 0.0207350757, -0.0687693208, -0.0325112604, 0.0145123107, -0.0479474515, 0.066875048, -0.113192014, 0.00387448841, 0.0245556347, 0.0574756116, 0.0263539962, -0.0242560711, 0.0589910038, 0.0450137183, 9.334110e-03, -0.0674083456, 0.0681542754, 0.0960907936, -0.0295516122, -0.0336602181, 0.0432872698, -0.0497876741, 0.109108284, 0.094851464, 0.0526518561, 0.0504652523, 0.0177171379, -0.0678464323, 0.00793147553, 0.049297072, 0.0163790025, 0.0881622657, 0.0617071204, 0.0708617941, 0.0226961784], [0.0395734794, 6.416850e-02, -0.0706859156, 0.0346285477, 0.103862233, 0.0152990194, 0.00438539265, 0.0307764355, 0.0468718372, 0.00663783075, 0.085589096, -0.0394170135, 0.0365459807, -0.00217309152, -0.0711829439, -0.0601294711, 0.0735126138, 0.0145765096, 0.0264187492, -0.00743090222, 0.0213861708, 0.00157073361, -0.0536608696, 0.0503858626, -0.00597294373, -5.071260e-02, 0.00402341178, -0.0248578023, -0.00118558819, 0.0557629205, -0.0745430663, 0.0163470022, 0.057233464, -0.0102900229, 0.0353923403, -0.0511539057, -0.0244878139, -0.0567817762, -0.0387956947, 0.0696792379, -0.0757062361, -0.0544795021, 0.0400605202, -0.064342685, -0.0255734324, -0.0734694377, 0.020050155, 0.0584115274, 0.0325989053, -0.0525165908, 0.0261649154, 0.0801534056, -0.0035692777, 0.00453729415, -0.0689382553, 0.057219483, -0.00576861482, -0.050279852, -0.0611474775, 0.0470093116, 0.0503201038, 0.0396553203, -0.0311417039, 0.0296864659, -0.0282187779, -0.0395258069, -0.0193067361, 0.00411546743, 0.013484939, 0.00792462379, -0.0162936579, 0.0908699929, 0.0666544735, -0.0784511938, 0.0354919769, 0.0732809231, 0.035270147, 0.0697878674, -0.00210835529, -0.0741553232, 0.0155465528, -0.0469013453, 0.0444481596, 0.09164346, 0.0646680221, -0.074310191, 0.081036225, -0.0382047929, -0.0636556298, 0.00968397222, 0.0581126399, -1.577230e-02, -0.10296315, 0.0573296845, 6.875830e-02, 0.0104721319, -0.0787224323, -0.04363003, 0.098288156, -0.0556607768, -0.0115072569, -0.0205801036, -0.0600988939, -0.0124833742, 0.0315198824, -0.0300667509, -0.0515951887, 0.0105596334, -0.00624413602, -0.0399625152, -0.0341104083, -0.0395452343, -0.0507722646, 0.0050237854, -0.00715036551, -0.00968000665, -0.0212293956, 0.0486992858, -0.0340247676, -0.0362791196, -0.0480441824, 0.0663050264, -0.0211603716, 0.117731124, -0.0181887411, 0.00521363737, -0.0282160789, 7.45963538E-4], [0.0351147391, 0.0178599153, -0.0277819242, 0.0305451844, 0.0552629158, 0.0262156967, -0.0659629628, 7.01702199E-4, -0.037664935, 0.072860457, -0.0409013145, -0.00916527119, -0.0326987877, -0.023473626, -0.0641630589, 0.0114218798, -0.0528477095, 0.00255627767, -0.0487506837, 0.0681733191, -0.0456817634, -0.0925359204, 0.0719165653, 0.0258985087, 0.0466166437, -0.012668523, -0.0136987306, -6.523530e-02, 0.0303014684, 0.0250749197, -0.033163894, -0.0565524474, 0.0746462718, 0.0677408874, -0.002913679, 0.0758247599, -0.0541168824, 0.0107012708, 7.068410e-02, 0.0336741582, -0.034468133, -0.0335174166, 0.0237928871, -0.0687052384, 0.0284385234, -0.0259989221, -0.0337896943, 0.0579169542, 0.0187577959, -0.0552339219, 0.0821511521, 0.085797213, -4.886890e-02, -0.00538038788, 0.0303981435, 0.0145487189, 0.0295979362, -0.0421747044, 0.0371388122, 0.013013917, -0.082596764, 0.0163227022, -0.00422014855, -0.0504022762, -0.0721721873, 0.00121424929, 0.0590284355, -0.0356384106, 0.0438709855, -0.0234150365, -0.0479919948, 0.0066219354, -0.0544815175, 0.0292069018, 0.022324739, -0.0287214741, -0.0504279435, 0.0435310826, -0.092531681, 0.07578329, -0.0607580468, 0.0340804979, -0.0438379161, -0.0161534082, 0.0496209748, 8.202310e-03, -0.0142891649, -0.0655294954, -0.0170235485, 0.00330356904, -0.00399124436, 0.0645144954, 0.0356451944, 0.0826633572, 0.0351827666, -0.0773037821, -0.0747895166, -0.00397448754, 0.094223164, 0.0286929794, -0.0210069269, 0.00142758386, -0.0695931762, 0.0953643321, -0.0715144128, 0.0628524795, -0.0502675585, -0.0449387282, 0.0431965403, 0.0276112109, 0.0138058057, 0.0290806107, 0.0161375683, 0.0126981661, 0.0627512783, 0.0518501364, 0.0114393812, -0.0591943413, -0.010181156, 0.0594944768, 0.0235959068, 0.0342139564, -0.00161484792, -0.0317971446, -0.0267951395, -0.030312527, -0.051211521, -0.0138874715], [-0.0103880223, 0.0124593833, 0.0331318714, 0.0589666627, 0.049881041, 0.0789314433, -0.0677420646, 0.0339764096, -0.0409182832, -0.0232350361, 0.0796347708, -0.00599560933, -0.00351068401, 0.0921732559, 0.0613622218, -0.0590044223, 0.00603760034, -0.0665556639, 0.0367023237, -4.190880e-02, -0.0208191816, 0.0449193083, 0.0331476741, -0.0583044179, -0.0532681383, 0.0436467603, 0.0103567336, -0.0329175517, 0.0576813146, 0.0411262587, -0.0512653403, 0.0577687584, 0.0413344167, -0.00486752298, -0.0495162457, 0.0319985859, -0.0944944843, 0.0528688319, 0.0824918747, 0.0214254446, -0.0259880405, 6.464430e-02, -0.0664079338, 0.0777177364, -0.0825237929, -0.022596553, 0.0993585884, 0.0564771295, 0.00968745723, 0.00545343664, 0.00610281387, 0.03698368, 0.0287410636, 0.0272647087, -0.0494515225, -0.0776823386, -0.0495355949, -0.0677248836, -0.0594958887, 0.0477160178, 0.0369645618, -0.0216260012, -0.00303194509, 0.0788792073, -0.0713758916, -0.0809480399, -0.0562806465, 0.0661600679, -0.0828215479, 2.39965462E-4, 0.0560778379, 0.0997413322, -0.027255971, 0.0710045844, -5.022570e-02, 0.0336903781, -0.0337586254, -0.0675498247, 0.0331377871, -0.0657220408, -0.0143069429, 0.0105419206, -0.0738569647, -0.0199799836, 0.0749743878, -0.0368214622, 0.00534103392, 0.0520816818, -0.0287415497, 0.0260574333, -0.0022438257, -0.0301609468, -0.0222148933, 0.0738126933, 0.0629655868, 0.0686748698, -0.0173812788, -0.0415745527, 0.0143875163, -0.0165206306, -0.0452212319, -0.0206016935, -0.0486801676, -0.0566064976, 0.0678652897, 0.0137596698, -0.00554868532, 0.0816838964, -6.368630e-02, -0.0859044119, 0.019053584, 0.0774717256, -0.0310396887, -0.0642774329, -0.00145931414, 0.0573278926, -6.37355609E-4, 0.0502849855, -5.388340e-02, -0.0650297627, 0.00308390963, -0.0665283054, -0.0433944687, 0.0493717492, -0.0608102083, -0.0606133491, 0.0413826667, 0.0457098745], [0.0074395705, -0.0175178498, 0.00806995108, -0.0151924007, -0.0387616977, -0.0626345202, -0.0394886769, 0.0482354611, -0.0308839083, 0.0663652867, 0.0729936138, 0.0678695515, 0.0428595357, 0.0533630885, 0.00293580675, -0.00930647086, -5.274490e-02, -0.00347875152, -5.987320e-02, -0.0308626033, 0.0180198662, 0.018578548, 0.0686687604, -0.0495456755, 0.0376377031, -0.020003004, -0.0271533597, -0.0167107526, 0.0508049726, 0.0340763703, -0.0647005662, -0.0543983914, 0.068275705, 5.709990e-02, 0.0475491099, 0.0145165632, -0.00460377801, 0.0403030552, 0.0495756231, -0.0116379522, 0.032393381, -0.0690436885, 0.04472569, -0.0443199091, -0.0461194701, -0.0479163565, -0.016425956, 0.0210831203, 0.0462094098, 0.0402700044, 0.0288383439, 0.0466718972, -2.65112321E-4, -0.0984369292, -0.0325367451, -0.0880318656, 0.0280998517, 0.0123763802, -0.083242394, 0.0744467825, 0.0421290807, -0.00965640135, 3.050390e-04, 0.0765829831, -0.0721153766, 0.0392191261, -0.0573187545, 0.0132942367, 0.0243576337, -0.057492733, 0.0316648819, -0.0344122835, -0.0553378947, 0.0863120779, 0.00419916492, 0.0108173359, 0.0119833173, 0.0160449892, -0.0808681101, -0.0591759868, 0.0393233374, 0.022316467, -0.0197879858, 0.0567647107, 0.0277313646, 0.0572627448, 0.035162542, 0.0649430603, -0.103735946, -0.0439722873, 0.0530137084, 0.103259228, 0.0810669214, -0.0174665097, -6.918880e-02, -0.0607498847, -0.0268503707, 0.0102547156, 0.0434690863, 0.0473910905, -0.0172313657, -0.0410701036, -0.0243117642, -0.0154057713, 0.0203179494, 0.113521099, -0.0472133048, 6.558280e-02, -0.0601005219, 0.0178846437, 0.0535662174, 0.0567943901, -0.00685608386, -0.00256162626, -0.00264685485, -7.573630e-02, 0.0460651964, -0.0147883184, -0.0722569302, -0.0668677166, -0.0264187455, 0.0137106273, -0.0706734359, 0.00348611129, -0.00543215591, 0.0580063462, 0.0460066423, -0.0171988215], [-0.0707240924, 0.0549645238, 0.0739427209, -7.580860e-02, 0.0618242398, -0.0107097542, -0.0790661647, -0.0496655069, -0.0165380873, -0.0450473726, 0.00138528587, 0.0564695597, 0.0307917241, -0.0198032744, -0.00797931571, 0.01534406, -0.023214465, 0.0560502447, 0.0319544449, 0.0413084701, -4.232350e-02, 0.0143347606, -0.0131713692, 0.0368864164, -0.0204365589, -0.0123491101, 0.0814951285, -0.049684234, -0.0748884305, -0.0789797157, -0.0786701441, -0.036317043, -0.00172664225, -0.019006189, 0.0731951073, -0.0453953631, -0.0345871672, 0.026783986, 0.0535511896, 0.0947026833, -0.0476401374, 0.072407484, 0.0231331158, -0.0113900686, -0.0950153097, 0.0297669265, 0.0953065827, 0.028466735, 0.0522965305, -0.0569711179, 0.0132953525, -0.0304451305, -0.00771052297, 3.532540e-02, 5.975640e-03, 0.00168354681, -0.00443576137, 0.00807472877, -3.625300e-02, -0.0602232218, 0.0827755853, 0.0244301409, 0.0148942536, 0.0410674848, 0.0196222868, 0.00727599068, 0.0195906777, 0.00847003516, -0.0542545766, 0.0532507859, -0.0921075344, 0.0644662902, 0.0551591553, -0.00580020528, 0.0519804806, 0.0430443138, 0.0265139416, 9.35750548E-4, 0.0566426814, -0.0230007786, 0.0133103728, -0.0180092435, 0.0540560223, 0.0718496739, 0.00489737792, 0.0498582311, -0.0482206158, 0.0581199527, 0.0161107909, 0.0254340749, -0.0347402133, 0.0539511256, -0.0741733685, -0.0363781936, 0.0633391812, 0.0626844391, 0.0485151708, 0.00527637033, -0.0554551519, 0.0399210714, 0.0490256771, -0.0969598218, 0.0630569458, 0.0646624342, 0.0524874702, 0.0907129198, 0.0391978733, 0.0685352907, 0.0601966456, -3.65857384E-4, -0.0589769036, 0.0670980439, -5.1437458E-4, -0.022928806, 0.0145913456, -0.0512158833, 0.0291459188, -0.0691024885, -0.0841882526, -7.076820e-02, 0.0541598722, 0.0475773737, 0.0504258834, 0.014836262, 0.0377273113, 0.00710072881, 0.0565415211, 0.0557061508], [0.00123952853, 0.0326217189, 0.0723488405, -0.0340605602, -0.0106745409, 0.0472846851, 0.026798889, 0.0208891947, -0.0306091774, -0.00528796064, 0.0336148776, 0.0658213422, 0.00581406197, 0.0292000063, -0.00756846182, -6.71383052E-4, -0.0228711367, -0.080264464, 0.0123677868, 0.00498397695, -0.0719398484, 0.0521408468, -0.0481569022, -0.0454418547, -0.0174960755, 0.0299316328, -0.0552671514, 0.0499611348, 0.016893521, -0.0593904406, -0.0183718987, 0.0592028573, -0.025464505, -0.00875521544, -0.0192276109, 0.0112974858, -0.0718449801, -0.0402695164, 0.0674688518, -0.0372388959, -0.0689360946, 0.035633862, 0.0103399549, -0.0585126244, -0.00417377939, -0.0770724863, 0.0215327535, 6.351660e-04, 0.0483052693, -0.00711687794, -0.0443833768, 0.0295699313, -0.0452590659, -0.0411812812, 0.0689214766, 0.0145178717, 0.0132181114, -0.0265972372, -0.0526984818, -0.0562912077, 0.056052044, -0.0157558024, 0.0352106243, 0.0573021322, 0.00431923429, -0.0323902704, -0.0128083257, -0.00779297622, 0.0225247256, 0.0332244262, -0.0234134402, 0.0994922146, 0.0536319315, 0.0697895363, -0.0369391479, -0.0620287284, -0.00618923316, -0.0630953684, -0.0530420393, 0.0337948948, -8.241630e-02, 0.0458438769, 0.0151243312, 0.0149580548, -0.0455900468, -0.0655078366, -0.0538797416, -0.0225132387, 0.029075101, -0.0058240979, 4.93917905E-4, 0.081397444, -0.0621480308, -0.064534381, -0.00192073383, 0.0580774695, 0.0135881174, -0.0118883476, -0.0532104783, -0.0250499416, -0.0786865577, -0.0601722747, -0.0135330651, 0.0876295641, -0.0522867516, -0.0102272453, -0.0450433977, 0.00900179613, -0.0793870612, -0.0824242457, -0.0436686836, -0.0481835492, 0.0478792377, 0.0697594881, 0.0529692583, -0.0157406051, -3.751180e-02, 0.0874371528, 0.0523010977, -0.0491193682, 0.00268490659, -0.0145168193, -0.0819626078, -0.0476201028, 0.0663645491, 0.0234642513, 0.00427111471, -0.00942207686], [-0.0628240779, -0.0136454515, 0.0184592027, -0.0220987499, 0.0560907573, 0.043407958, -0.0625519082, -0.00916703884, 0.0181260668, 0.0218839832, -0.0707873926, -0.00562907103, 0.00237681507, 0.0850272178, 0.0245309751, 0.0792504847, -0.0314993262, -0.0174727459, 0.026112847, 0.0214084927, -0.0393574797, -0.0719215423, -0.0706788823, -0.0532267392, 0.00789005495, 0.0100519722, -0.0391716026, 0.0299910791, -0.0641202628, 0.0435371362, -0.0616628229, -3.665760e-02, -0.00942535792, 0.0622269026, -0.040191181, -0.0738080516, 0.0648405179, -0.00231415196, 0.0140622761, -0.0417665131, -0.0402453654, 0.058519315, 0.0635190159, -0.0164969955, -0.0076451716, 0.0042411848, 0.0623742118, -0.0628307313, -0.0824392735, -0.0733223259, 0.0183237456, -0.0637391433, -0.014389866, -0.019582076, -0.0720722079, 0.0278108362, 0.00152062846, 4.526740e-02, 0.00784527137, 0.0065754964, 0.0784251764, -0.00766075775, -0.0635867715, 5.463900e-02, -0.0225588419, -0.0562980585, -0.0433656685, -0.0605677478, 0.00160002382, -0.0163970236, 0.0082699256, -0.00763139501, -0.06009138, 0.0786892995, 0.0128709842, -0.0173406303, -0.0474989451, 0.0418609381, 0.0292180311, 1.71531632E-4, 0.0547900796, -0.0398012213, -0.0636914819, 0.0328272209, -0.0619521886, 0.0633459166, -0.00775526697, -0.0450049378, -0.0184331518, -0.0450177342, -0.019057868, 0.0541408472, -0.00553409569, 0.0348039269, 0.0453123674, 0.0563469455, -0.0203318261, 0.0253713429, -0.00187600544, -0.0145575488, 0.0445353352, 0.0696881041, 0.00839308463, 0.0116718477, 0.0409697965, -0.00587842753, 0.00763902626, 0.0323035568, 0.033675205, -0.0305168517, 0.0619508661, 0.0179979764, -0.0144886468, -0.0369497314, -0.0703643337, 0.0162337311, -0.060658738, -0.0471542701, 0.0249621104, -0.0347217098, -0.0280026551, -0.0775900632, 0.0683081895, 0.00621984154, 0.0166591704, -0.0444655567, 0.0649295226, 7.860980e-02], [0.0429744087, 0.0133281462, 0.02759468, 0.022724187, -0.00784333516, 0.064133197, -0.0537529476, -0.0128600951, -0.0677171126, -0.0779451877, -0.0667837709, 0.0203483347, -0.0821174308, -0.0489770323, -0.00612121634, 0.0114551513, 0.0645024478, -0.038325239, -0.00163730129, -0.0475328602, 0.0300815105, 0.0108305486, -0.0407219492, 0.0116115371, 0.0631050617, -0.043912217, 0.0282047708, 0.0498104766, 0.0729049444, -0.0449992716, 0.00530684227, 0.0366107263, 0.0512253158, 0.0375898443, -0.0397416465, -0.0612284876, 0.0317049138, 0.0160985775, 0.0105403597, 0.0838197917, 0.0324452966, 0.0509578548, 0.0742427111, -0.0482018851, 8.57683131E-4, 0.0319725648, -0.0297709461, -0.0639467388, -0.0724542737, 0.0355228409, -0.0310202651, 0.0323063694, -0.0227085967, 0.0178362634, -0.054796081, -0.046179045, 0.0724223852, -0.0472442582, -0.0746521875, 0.0227334946, 0.00421106163, -0.0134079801, -0.0509816855, 0.0179976393, -1.348760e-02, -0.0035805814, -0.0165251829, -0.0772964656, -0.0778665692, 0.0599473715, 0.0201020073, 0.0838502496, 0.043749176, 2.83799891E-5, -0.0778831318, -0.046097029, 0.0742046759, 0.0162035134, 0.0151718324, -0.0335656367, -0.0153175825, -0.0083892392, -0.0783609673, 0.0703180581, 0.034781158, 0.0777833759, -0.037654493, -0.0380081758, 7.399170e-02, -0.0238936413, -0.073072426, -0.0752154365, 0.0312498938, -7.535030e-02, 0.0663096085, -0.00504398765, -0.0479154512, -0.028262835, -0.0745145455, 0.0405566283, -0.0236136168, -0.00938938465, 0.00651220605, -0.0281143561, -0.0684559867, 0.0813515335, -0.0110677173, -0.0730191544, 0.0703899413, -0.0138413273, 0.0618116557, -0.00386584806, 0.0128295561, 0.0557931177, -0.0513519458, 7.81358918E-4, 0.0321712457, -0.0205542836, 0.0197011698, 0.0440701544, -0.0081073055, -0.0814641788, 0.0753783733, -0.0278308056, 0.0320483036, 0.0337610319, -0.00736217294, 0.0232272949], [7.018070e-02, 0.0339931585, 0.0699138194, -0.0469315089, 0.0600557178, -0.0590229705, 0.0689958483, 0.0669408142, -0.0145386066, 0.0666850954, -0.0759994835, -0.0765772313, -0.0686084405, -0.0278844237, 0.0124183744, -0.0244728178, -0.0632552654, 6.613570e-02, -0.0353586078, 6.705590e-02, -0.0600673184, 0.0781419798, 0.0485838614, -0.0104640536, 0.0173721034, -0.00629385794, -0.0729512423, 0.0526419245, -0.0800624862, 0.0198417604, -0.06931822, -0.0134225218, 4.437780e-02, -0.00930448622, 0.0471307375, 0.0677601397, -1.401850e-02, -0.0803712531, 0.00873725582, 0.0357119553, 0.0330300927, -0.022171801, -0.0284919199, 0.0277259443, -0.00387550471, -0.0310502369, -0.0660108179, 0.0634201616, -0.0557307936, -0.0318662561, 0.0675025433, -0.0168766696, 0.0104516707, -0.0178847965, -0.0725520328, -0.0236236714, -0.0650315285, -0.0694000199, -0.0798966512, 0.0196344517, -0.0359380282, 0.0587255843, -0.0138755105, 0.0768536255, -0.0720469057, -0.0508185141, -0.0286640711, -0.00225958088, -0.0384309255, 0.0536055639, -0.0677827075, -0.0469085723, -0.045734074, -0.0451903343, 0.0791864991, 4.567610e-02, -0.0521211103, -0.0787092745, -0.0610069856, -0.0678403377, -0.0733353793, -0.0505856872, 9.342410e-03, -0.0451581031, -0.0496552251, 0.0658758357, 0.0539789759, -0.00859428383, -0.00306254555, 0.0213818401, 0.0351010375, 0.0296061765, 0.0355691165, 0.0208223108, 0.00751766609, 0.0281939022, 0.0593884289, -0.0389039405, -0.0315775201, 0.0246494506, -0.0291622058, 0.0688329041, -0.0213239752, 0.0590512156, 0.0236645974, 0.0488771498, 0.0769745782, 0.0351584777, 0.0664903298, -0.0443345495, -0.0125631578, 3.872190e-02, 0.046094276, 0.0441591814, -0.0462532975, 0.00901391078, -0.0181472488, -0.0373797454, -0.0656939149, -2.65364924E-5, -0.00750177214, -0.0737906918, -0.070581235, -0.0759442672, 0.0412567891, -0.0231362116, 0.00942675862, 0.011148518], [-0.0784595608, -0.0469540097, -0.0469180308, 0.0226187464, 0.0321716927, -0.0711577535, -0.0528659523, 0.0235562492, -0.0447966605, 0.0752669051, 0.0153758768, 0.0297957156, 1.93729953E-4, -0.0422311388, -0.0409840941, 0.0449433699, -0.0692991465, -0.0223737806, 0.0520788208, -0.00900093373, 0.073938027, -0.0168213733, -0.0802067741, 0.00682594953, -0.0354700759, -7.013960e-02, -0.0345938094, -0.0737830326, 0.0810623839, -0.0695850328, 0.0299654622, 0.031120047, 0.00493393419, -0.0683275908, -0.0633985177, 0.0546887219, 0.0353839099, 0.0790704638, -0.0292439554, -0.0514322557, 0.0201875586, -0.0606235117, -0.0372882113, 0.0692301616, 0.0549389198, -0.0366289206, -0.014100966, -0.0597167946, -0.00247186632, -0.0435670875, -0.04282609, -0.014908514, -0.0745338574, -0.0260739475, -0.0508734956, 0.0125817414, -0.024421189, -0.0238531921, -0.00959852058, -0.0327173099, -0.0323382393, 8.974130e-03, 0.00102143502, -0.0621969476, -0.017322138, 0.00470649218, 0.0575437099, -0.0473798029, 0.035321217, -0.0398928858, 0.0318363532, 0.0262983236, 0.0245056208, -0.0115878033, 0.0276031792, 0.0413680933, 0.0441229194, 0.0530672334, -0.0244434532, 0.0719674453, -0.0411296636, 0.0388520248, -0.00719822431, -0.0242805034, -0.0207869094, 0.0379077196, 0.0561988242, 0.0306110457, -0.0180244371, 0.0533447266, -0.00182288012, -0.00829703454, -0.0746623874, -0.0164881628, -0.00973092764, -0.0231131278, 0.0609873869, 0.0257745869, 0.0670201182, 0.0800820067, -0.0669580773, -0.037436191, 0.0496279895, -0.0566934347, -0.0694650412, 0.0403544456, 0.0020993948, -0.0325276218, -0.0296475068, -0.00546470098, -0.0635257587, -0.0148916394, -0.0494835824, -0.0689681768, 0.0771437734, -0.0245854035, -0.0302599929, 0.0531537533, -0.0584235564, -0.00728327315, -0.0276326779, 0.0679956972, -0.0282787513, 0.0697297454, -0.0645547807, -0.0576199405, 0.009252836, -9.81318531E-4], [-0.0533890165, 4.023120e-02, -0.0532160811, 0.0191056523, 0.047677692, -0.0509608313, -0.0355214924, 0.0543829761, 0.00115963526, -0.0130117266, -0.0417939723, -0.0190401878, 0.0168560445, 0.0450775363, -0.0131156622, -0.0183585938, 0.0293942541, -0.0251730308, 0.0697236806, 0.0118913911, -0.00413520215, 0.0685450211, -0.0359569117, -0.0647142455, -0.0702507495, 0.0478398874, 0.0761728808, 7.580500e-04, -0.0672343373, -0.0658896863, -0.00204728078, 0.0146927359, -0.0287711136, -0.00172667019, -0.072504662, -0.0365273058, -0.00346804969, 0.0554780513, 0.0731493831, -0.0216717515, -0.0322916694, -0.0345960818, -0.0692076907, -0.0158484839, -0.0503206328, 0.00720707094, 0.0727258548, -0.00718231359, -0.0620175153, 0.0475871637, 0.03297019, -0.0809109732, -0.051591713, -0.0695680752, -0.0792375058, -0.0310972966, -0.0346282981, 0.0279574767, 0.0675393865, -0.015091504, 0.0775947496, -0.076993972, 0.0622166023, -0.0614841171, 0.0647603646, 0.00323630706, -0.0510128066, 0.0261930898, -0.0381251611, 0.0588554144, -0.073431842, -0.0454376861, -0.0643683225, -0.0621705986, 0.0206357613, 8.33959144E-4, 6.514680e-02, -3.6516742E-4, 5.567840e-02, 0.055504404, -0.0228429418, -0.0373917706, 0.0325563177, -0.0552500859, -0.0523200259, -0.0657908395, -0.0203462709, -0.0522411466, 0.0571968742, 0.0186803117, -0.0568425842, 0.0317500196, -0.0212439112, 0.0179719161, 4.328940e-02, 0.0172260031, 0.0286876466, -0.0734299794, 0.0570174828, -0.0303302985, 0.0622035898, -0.0757854878, -0.0416693315, 0.0710792914, 0.0387364551, -0.0740114152, 0.0405662172, -0.0598439872, 0.0734027177, -0.0693773329, -0.0784604102, -0.0421772189, 0.0479377955, -0.0569710881, 0.0428972617, 0.0778648704, 0.0269676503, 0.0540383905, -0.0415349305, 0.0638868808, -0.0546739958, -0.0345080569, 0.00131569826, 0.0164350476, -0.0440271124, -0.0164572597, 0.0288367439, -0.0327761509], [0.0192789286, 0.0137437135, -0.0696068406, 0.0128045082, 6.472040e-02, -0.0171365738, 0.00729481131, 0.0500424132, 0.0348997712, 0.0273895338, 0.0577260181, -0.0675219595, -0.0386761166, -0.0704982132, 0.0116009489, 0.0275401771, 0.0296505615, -0.0211034045, -0.0617268458, 0.0540677533, -0.0425138809, 0.0457494631, 0.027188994, -0.0241247229, -0.0772679746, 0.00189495832, -0.0652459561, 0.030208379, -0.00123542547, -0.0555124469, -0.0164238065, -0.071520634, -0.0254118592, 0.0671770796, 0.0455369577, 0.0743189231, -0.00595248491, -0.0166795552, 0.039062202, 0.0308139399, -0.0546773225, 0.0284428671, 0.050493978, -0.0380747132, 0.0286621228, -0.0368172787, -0.0552029759, -0.0100711733, 0.0411838517, -0.0132200271, 0.0573092625, -0.0188995898, 0.0358749405, 0.0322116092, -0.017218858, -0.0240295976, -0.00916346907, -0.0362592526, 0.0223228931, 0.00990420579, 0.0359965786, 0.0699250475, -0.0140833482, -0.0790752544, -0.0423397385, -0.0454482548, 0.0562052205, 0.0412828624, -0.049701944, -0.0625767857, 0.0634728298, 0.0209872797, -0.0211569704, -0.00893690437, -0.0373948179, -4.130140e-03, -0.0314807035, 0.0636166856, 0.069947429, 0.0300625488, 0.00551240146, -0.0774412825, -0.0125441328, -0.0343408957, 0.0537846461, 0.0240708664, -0.0697005689, -0.00934016704, 4.107900e-03, -0.0354395434, -0.0168078467, -0.00458721817, -0.0744496509, 0.00712248683, 0.0145308375, 0.0559193566, 0.0100575611, 0.0589486286, -0.0188078322, 0.0131919682, -0.0802625119, 0.00214049965, -0.021391023, 0.00571721047, -0.0808274373, -0.0429631472, 0.0781450197, -0.0428848639, -0.0597389676, 0.0179911181, -0.00759664177, 0.06938418, 0.0119971558, 9.46223735E-4, -0.0434479378, 0.0166175142, 0.0558460876, 0.0577671155, 0.001964055, -0.0749572963, -0.0154112354, 0.0160210431, -0.0178938285, 5.506620e-03, -0.0725941584, 0.00167328864, -0.0546632633, 0.0545917079], [-0.077448979, 0.0471085981, 0.0227692798, 0.0279084519, -0.0784652903, -0.0283830725, 0.0775673165, -3.703180e-02, 0.0031510219, -0.00668169558, 0.0629660115, -0.0205125026, 0.0190777183, -5.548340e-02, -0.0777603685, -0.064003408, -0.05775778, 0.0332045779, -7.511870e-02, -0.0123828128, 0.0280639529, 0.0202552825, 0.0194957927, -0.064432025, 0.0567161962, 0.0691280141, -0.0295944437, 0.065338321, 0.0436032414, -0.0168781206, 0.00557366759, 0.0783952996, 0.0244687125, 0.043466717, 0.0145348385, 0.0416200235, -0.0747453496, 0.0129723251, -0.0351213701, -0.0140695423, 0.0110871494, 0.0071478039, -0.0186132491, 0.0504200086, 0.0334912315, -0.0400600806, -0.0630518869, -0.078266628, 0.0618881062, -0.0512918048, -0.0398993976, 0.0328269601, -0.0415724143, -0.0231270418, 0.0417102724, -0.0475807637, -0.0348988622, -0.0471562296, -0.0669082925, 0.0737528875, 0.0581501946, 0.0724295154, -0.0308084823, -0.0314263813, -3.482180e-02, -0.0286589526, 0.0381565914, 0.00142513961, -0.0320490748, 0.0257297233, -0.0280217193, 5.77047467E-5, 0.0722521767, 0.0606396422, 0.00149382651, 0.0231239274, -0.0784699246, 0.0718505606, 0.0676399246, 0.00517245382, -0.0594935454, -0.0501561835, -0.0185917467, 0.0276406556, 0.00784173607, 0.020325169, 0.0649482384, 0.00519664586, -0.0601862222, -0.010784775, 0.0799573287, 0.0258548409, -0.0193481818, 0.00602927804, 0.0353229716, -0.00679766387, 0.0241914168, -0.0307843499, -0.0620949901, -0.0266105458, -0.054264836, -0.00511500239, 0.0726578608, 0.0791819766, 0.0292543024, 0.0342690498, -0.0188332982, 0.0385246575, 0.0250701904, 0.0553615466, 0.0632185116, -0.0786724761, -0.0571002364, 0.0559234694, -0.042445384, 0.0348962098, 0.0275235847, 0.0136916116, -0.0347793922, -2.394300e-02, -0.0333875343, -0.0458946377, 0.0425998941, 0.0668967739, -0.0769880936, 0.0213648155, -0.0358808823, -0.0622191615], [-0.0613503903, -0.0142183015, 6.947410e-02, -0.00755935954, 0.0402076319, -0.050302688, -1.731010e-03, 0.0797340571, 0.0768091828, -0.0214224588, -0.0351166576, -0.0280647222, 0.0599025115, 0.0602075718, 0.0773790255, 0.0782270655, 0.0506265275, -0.0502487384, 0.0523352064, -0.0301330686, -0.0445502773, 0.0649572834, 0.0193497185, -0.00225218572, -0.0271172412, 0.0177611653, 0.0491890162, -0.056193281, -5.43625851E-4, 0.0768545642, 0.0690693855, -0.0798094645, 0.030446643, -0.016494533, 0.0393402763, -0.00709196925, -0.0518600866, 0.063747935, 0.0065048011, -0.0795290097, -0.0455530956, 0.0220083762, -0.061974436, 1.21040008E-4, 0.0772037581, -0.00355090434, 0.0232846327, 0.00287496252, -0.0646537393, -0.0676078722, -0.0349769257, -0.0417709202, -0.00794087444, -0.0537529476, -0.0269985925, -0.0330077857, -0.0220087748, -0.0307041164, 7.531690e-02, 0.01719152, 0.00116342434, 0.0404675044, -0.0590624772, -0.0801103115, -9.539920e-03, -0.0539967641, 0.00529717701, -0.058745876, -0.012489845, -0.0452756844, 0.0346363522, -0.0625081435, 0.0586787499, 0.0102507723, 0.0335239954, -0.0470319204, -0.00970830209, -0.0363976471, -0.00744169205, 0.0282030292, -0.0613161363, -0.0623678192, 0.0152692227, 0.0654841438, 5.292650e-03, -0.0486136787, 0.0425868779, -0.0692320615, -0.00317017222, -0.00914147682, -0.0203944184, 0.0308235828, 0.0574796684, -0.0061009964, -0.0210493673, -0.0595819652, 5.460880e-02, 0.00217745476, 0.0201080181, 0.00766829494, -0.0165305268, 0.0368909277, 0.0100929681, 0.0377467461, -0.0124024674, -0.054035224, 0.00650145299, 0.0810428411, 0.0189983044, -0.00351166795, -0.0191896856, -0.0658882409, -0.0384187028, 0.0234007221, -0.0445968695, -0.0598918125, 0.0537577532, 0.0452068634, -0.0417486131, -3.830230e-02, -0.027578542, -0.0715524703, -0.0013187658, 0.00220985222, -0.0725865066, -0.0440836847, -0.0754540488, 0.0443226174], [-0.0404579192, -0.0187803265, 0.039063897, -0.0495482944, 0.0609126762, 0.0383456685, 6.490420e-02, 0.0818671882, 0.045760382, 0.0664629787, -0.049155131, 0.0557330325, 0.0203707907, 0.0522448607, 0.0613259114, -0.0703312606, 0.0642510653, 0.0364713073, 0.0621437616, 0.0498943068, 0.0702481866, 0.0740283281, -0.059656173, 0.0641645938, 0.0680682212, -0.0283257458, 0.0617216602, -0.013323565, -0.0432593077, -0.0481065363, -0.0466454513, 0.0115867052, -0.0756980106, -0.0147760352, -0.00604341226, 0.0704104826, -0.0801836699, 0.0677528232, -0.0481639579, 0.0763473511, -0.0382929519, -0.0482470542, -0.0811684355, -0.0220892821, -0.0133268274, -0.071384795, -7.702100e-02, -0.0830410793, 0.0693869144, -0.0478283577, -0.0389718078, -0.076491043, 0.0352050774, -0.0435065851, 0.0471370369, -0.0127123138, -0.00173653767, 0.0704675317, 0.0615432076, 0.0182256754, 0.00232699676, -0.0806914121, -0.0447733626, 0.0214161351, 1.1386953E-4, -0.0272009727, -0.0517864376, 0.0144120548, -0.00698762061, -0.0272427388, 0.00573206088, -0.0321385711, 0.0481552146, 0.0335505679, 0.0261879507, 0.0416879877, 0.0388936587, -0.00343877799, -0.0213744417, 0.071335107, -0.054320246, -0.0701327473, 0.0275883079, -0.0791662335, 0.0568550937, 0.0292309616, -0.00957161281, 0.0608990267, 0.0361410417, 0.0769529194, -0.0108021265, 0.0584015511, -0.0611811243, 0.0474513471, -0.0136793694, 0.050948333, -0.0247735102, 7.469780e-02, 0.0280323699, 0.0718395934, 0.0603893213, 0.0192386918, 0.0124662323, 0.0710719749, -0.0618521422, -0.0803265423, 0.0402319171, -0.0471581593, 0.00329853664, 0.0125744054, -0.0465302952, -0.00178297632, -0.00548499404, 0.081805855, -0.0644520894, 0.0340865031, -6.313100e-02, -0.0797304138, 0.0315097272, -0.0462562777, 0.0713702217, -0.0236620642, -0.0790476352, -0.0589157268, -0.0440187529, -0.051659096, 0.043171607, 0.039181605], [-0.0473112687, 0.0590068847, 0.00968877598, -0.0691654161, -6.109480e-02, -0.0589133352, 0.0196390413, 0.0265772045, -0.0800207778, -0.0118077928, 0.074372977, -4.02347941E-4, -0.0261416882, 0.085701026, -0.0835710689, -0.0797791704, 0.0456729196, -0.0602623448, -0.0456730202, 0.0449889451, -0.00511169247, -0.0479547307, 0.0585721768, 0.0648480951, -0.080150552, 0.00375750475, 0.00603855168, 0.0728520751, 0.0802361071, 0.0033609923, 0.0745718181, -0.0823574364, 0.0180115774, 0.0512427539, -0.0312440712, 0.0224782061, 0.018512249, 0.0375457555, 6.949360e-02, -0.0463350154, 0.0196125619, -0.0515367463, 0.0394057669, -0.0566416532, -0.0219535176, -0.0672344565, 0.0294487067, -0.0278851818, 0.00820612721, 0.0545714498, 8.28459305E-5, 0.0594348423, -0.0710194781, -0.0531588495, -0.0476335399, 0.0243263207, -0.0156920031, -0.0446618535, -0.0442843251, 0.0795539095, 0.0321651325, 0.0696385354, 5.120530e-02, 0.00854443572, 0.0204341207, 0.0533526205, 0.0240357351, 0.0656304359, -0.0107707717, 0.0634813756, -0.00657753926, -0.0738556534, -0.0318600051, -0.0285191536, -5.140610e-02, -0.0326566324, 0.0154738873, 0.0636956319, -0.0731681883, 3.214860e-02, -0.0240076557, -0.0720849559, 0.0714889541, -0.0388286076, 0.0575972646, 0.00182185683, -0.0307397228, -0.0562821589, -0.035991732, 0.0741045922, 0.0482035205, 0.055261191, -0.0557925664, -0.0687354207, -0.0366663672, 0.0115172928, 0.0468579903, 0.0161897931, 0.0772588328, 0.0680676699, -0.0162243284, -6.854390e-02, -0.0561757684, 0.0596308447, -0.00458209682, 0.0156605858, -0.0665814578, 0.0494378135, 0.00769489352, -0.0485049672, -0.0761254802, 0.00685592042, -0.0247941762, 0.0852373838, 0.036631152, 0.0632773861, 4.175740e-02, 0.0127381794, -0.0252378695, -0.00212166528, -7.497470e-02, -0.0513192341, 0.0327264369, -0.0087093683, 0.025551267, -7.902570e-02, 0.0695930123, -0.0310060754], [-0.0356641263, 0.0313126221, 0.0518184304, 0.0186769627, -0.00330816582, -0.0469296612, 0.0315676816, 2.53422069E-4, 0.0641376451, 0.0708093494, -0.0511915088, -0.0103653781, -0.00367899588, 0.080254741, -0.0531263873, -0.017926814, -0.00450887484, 0.022972418, -0.030263735, 0.0514820181, 0.0495987684, -0.00648799445, -0.0601880848, -0.00976402685, 0.0524191968, -0.00138152763, -0.0141911274, -0.0425915755, -0.0538309962, 0.0723541528, 0.00512524648, 0.0407199748, 0.00901215803, -0.00354239414, -0.0559909381, 0.0503208227, -0.0436378568, 0.0445868783, 0.0177923702, 0.047203932, -0.0727107599, 0.0417557545, -0.0546561107, -0.0318305641, -0.0586397387, -5.172540e-02, 0.0206013266, 0.00574137084, -0.082281217, 0.0137653789, 0.0150121925, 0.0762834623, -0.0873143747, 0.0750665739, 0.0137209073, 0.0276603866, 0.068614088, 0.0594094172, -0.0461708121, -0.0471555442, -0.0474577658, 5.485870e-02, -0.00900918804, -0.0525552742, 0.0366121344, -0.00251417211, 0.0612724572, -4.794190e-02, -0.0571994185, -0.0833029225, 8.071050e-02, -0.0446677878, 6.437270e-02, 0.0134111298, 0.0657129064, 0.0366168022, -0.0243283957, -0.0245602801, -0.0340758599, 3.41332256E-4, 0.0738776773, 0.0541187227, -0.0572220795, -0.0694024562, -0.0760915875, -0.0105895065, -0.00562470593, -0.0195228532, 0.0684068874, -2.210060e-02, -0.0298227705, 0.0430931263, 0.00212224456, 0.0782282203, 0.0384710357, 0.0105562853, 0.0774852111, 3.478970e-02, -0.0237279739, 0.0731234625, -0.0164911523, 0.0248691346, 0.0431783907, -4.644850e-02, -0.0593924783, 0.0538066439, -0.0339883864, 0.0368334725, -0.0662368834, 0.0241314285, 0.0539945066, 0.0169513356, 0.0101358201, 0.0376787521, 0.0380522124, -0.0687920451, 0.0321854167, 0.0260030609, 0.0172820799, 0.0392589271, -0.0700070113, -0.0550187863, 0.0141433338, 5.287520e-02, 0.00284823589, -0.0301803164, -6.255030e-02, -0.0411812738], [-0.050583981, 0.0800332501, 0.0449548252, -0.0206771698, 6.820900e-02, -0.0234863218, 0.0476580299, -0.0398123302, -0.0746109337, 0.0460443534, 0.00348729221, 0.0395416468, -0.00346846553, 0.0398308896, -0.0262631364, -0.0692240074, 0.0442174561, 0.0200317837, -0.0131197898, -0.00135292939, -0.00834874622, -0.0683807358, -0.0590019152, 0.00833054353, -0.020790806, -0.0669239908, -0.0574195124, 0.0553393215, -0.0161227323, 0.061766915, 0.0459019169, -0.00776125211, 0.0887553989, -0.0303177983, -0.020095123, -0.0457911976, 0.0722510368, -0.022900084, 0.0314071402, 0.0799303204, -0.0381719396, -0.0272694137, 0.0597017407, -0.0100297108, 0.0120514762, 7.772880e-02, -3.729490e-02, -0.0844895691, 0.046288006, -0.0442039967, 0.00534824654, 0.010006262, -0.0498950593, 0.0516100898, 0.0612977147, 0.00899191759, -0.0426722616, 0.0584157631, -0.0339716412, 0.0180409197, 2.826830e-02, 0.0601526387, 0.0105138337, -0.0833132863, -0.0233615376, -0.00543370144, -0.0722492635, 0.0100181093, 0.0119623393, 0.0154804355, 0.0184533764, -0.0842053368, 0.0604588352, 0.0818748772, -0.0250837971, 0.0625565872, 0.0396980271, -0.0771612152, -0.0620248019, 0.0298417602, 0.0194651671, -0.00454095751, -0.044819843, -0.0779630467, 0.00162631238, -0.00757661415, -0.086377792, 0.0534460954, 0.0404044949, -0.0751524121, 0.0367036499, 0.0621447972, 0.030770706, -0.00733796554, 0.0787824317, 6.858970e-02, -0.04755437, 0.0808034837, 0.0552435517, 0.00789883267, 4.853270e-02, -0.0130072888, 0.00792605337, 0.0104435431, 0.0731254667, 0.0500885323, -6.402660e-02, -4.103680e-02, 0.0657329932, 4.096940e-02, 2.15124688E-4, 0.0114536127, 0.0484306589, -1.377080e-02, -0.0346857868, 0.020415483, -0.0048068203, 0.0118741253, -0.0497660823, 0.0385706797, -0.0412751324, -0.00369146187, 0.0292851124, -0.0407694839, -0.00683541084, -0.0274404883, -0.0343372896, -0.0298503023], [-0.0426849648, 0.0258555524, 0.0037485389, -0.00326093473, -0.0530342199, -0.0438775793, 0.0169587899, 0.00513637625, 0.0147759793, -0.0190029815, 0.0665470659, -0.0282995161, -0.0489066131, 0.0771511942, -0.0222709123, -0.0481461659, 0.0255292784, -0.0558044165, -0.0654272214, 0.0162056722, -0.00937089603, 0.0736783221, -0.0713255331, -0.0255688652, -0.00105789606, 0.046034459, 0.0358091965, -8.11802689E-4, 0.0146671506, 0.0702264606, 0.0483120047, -0.00847372878, -0.0451626144, -0.0222228784, 0.00345241954, -0.0687822923, 5.003660e-02, 0.0193083379, 0.0323021188, -6.857230e-02, 0.0344917625, -0.0117106335, -0.0418906808, 0.0606949218, 0.0642737225, 0.0701201111, -0.105547339, -1.904090e-02, 0.0378428474, 0.0673313886, -0.0366100259, -0.0541962162, -0.0963857173, 0.0716311187, -0.0233184397, 0.0819711312, 4.139800e-03, 4.575440e-02, 0.0304216072, -0.00561572844, -0.0270220041, 0.0458577834, 0.00768697308, 0.0102700442, 0.0673232302, -0.00347013865, -0.0352570973, 0.0413580798, 0.0286652446, -0.0421123803, -0.0339785255, -0.0502948835, -0.0452792607, 0.0886268988, 0.0558249168, 0.0241404679, 0.00331626809, 0.0752364472, 0.0240683481, -0.0807876884, -0.0456760786, 0.0787078067, 0.0859123989, 0.066001974, 0.0496468917, 5.015860e-02, 0.0359342247, 0.0398759097, 0.0723393857, -0.0712463111, -0.0497360043, 0.064053461, -0.0834448487, 0.0131596187, -0.0186109357, 0.0153119117, 0.0128870662, 0.018865712, 0.0234026238, -0.0583014525, -0.0622164495, 0.0172414631, 0.059708856, 0.029891381, 0.0483999401, -0.0183676612, -0.0162074864, -0.054368984, 0.0136854909, 0.00108152069, -3.668310e-02, -7.611890e-02, 0.0458635949, -0.0166837592, -0.0316245034, 0.0618967526, 0.00214112923, -0.0158976316, -0.0485432521, -0.0535743386, 0.0584530123, -0.0501242131, -0.074616611, -0.0112633863, -0.0498343445, 0.0245423373, 0.0727458224, 0.0451307297], [-0.0251720324, 0.0138311377, -0.00301105273, -0.0492863432, -0.0190802142, 0.0268330611, 0.0559697822, -0.0741025954, 0.0210834201, 0.0812353343, 0.067359671, 0.040673431, -0.0698239878, 0.0875725671, -0.0708278269, 0.0161716975, 0.0504263975, -0.0650747269, 0.0678110942, 0.0189303644, 0.0454447605, -0.0338797122, -0.0405107029, -0.0346636698, 0.0278637297, 0.0206954572, 0.00937088765, 0.0554765873, 0.0671969578, -0.0034704851, 0.0428281054, 0.0869848728, -0.0406591967, -0.0561079755, 0.0617623962, 0.0164106674, -3.140100e-02, -0.0563328899, 4.54300316E-4, 0.0563538522, 0.0369655192, -0.0669474825, 0.00711201271, 0.0773886293, 0.0100744339, 0.0497315228, -0.0862471386, -0.0369276404, 0.0666195229, -0.0133287357, 0.0326705873, 0.0197626725, 0.0450131111, -0.0223205946, 0.0813171938, 0.0155546414, -6.460000e-02, -0.048755724, -0.0626782328, -0.0233823489, 0.00282994774, 0.0618495867, -0.00413337909, -0.0403183512, -4.678790e-02, 0.0662935898, 0.00612370344, 0.0147987073, 0.0236265287, -0.0764484331, -0.0339068733, 5.730740e-02, 0.0884050876, -0.0816624835, 0.0196270179, -0.023193568, 0.0114152739, -0.0464840271, 0.0173703097, 0.00681344746, -3.182280e-02, 0.0288550165, 0.0176110081, -0.0339916497, -0.011197201, -0.0414323546, -0.0251447093, -0.0199281555, -0.0754465908, 0.0575553626, -0.00395714259, -0.0466469117, -0.0354684405, 0.0385820568, -0.0631742105, 0.0415844508, 0.0274031404, 0.00476372801, -0.0405073352, -0.0213006977, 0.0586832948, 0.00548840407, -6.950570e-02, -0.0500187911, 0.0772592872, -0.0421242267, 0.0594339222, -0.0578958839, -0.0720699579, 0.0709777474, -0.0228598583, -0.0391727798, 0.0499417596, 0.00115956913, -0.0744878128, 0.0607798547, 0.0266745426, -0.036795523, -5.13212522E-4, -0.0631575137, 0.059012115, 0.0242179967, -0.0297235809, -0.0316458344, 0.0347538032, -0.0468494371, 0.0291818678, 0.0643321573], [-0.018738592, 0.0357962474, -0.0410189144, 0.0518350862, 0.0387149155, 0.044832848, -0.0471035056, -0.0636766106, -0.070311971, -0.0118128527, 0.0165062957, -0.0839371979, -0.0675991923, 0.0314193889, 0.0727272779, 0.0482397564, 0.0420207307, 0.0410979316, 0.0351385213, 0.0863862112, 0.0508403145, 0.0620102547, 0.0365312435, -0.0395946838, 0.0632953122, 0.0636146143, -0.0382920615, -0.0800443515, 0.00250206771, -0.0402071103, 0.0821355134, -0.0219087433, 0.0426870547, -0.0282898415, -0.02337889, 0.0744899958, 0.0537566282, 0.0221589468, 0.00906394701, 0.0235052183, 0.0451051109, 0.018936405, -0.0499433279, 0.0385348499, 0.0441694781, 0.0711514651, -0.044703763, 0.0186380949, 0.0646389052, 0.0186510142, -0.0404880606, -0.0323329382, 0.0374149606, -0.0427352786, 0.0322183818, 0.0712695419, 0.0685102493, -0.0809832811, 0.0140465349, -0.053309653, 7.204540e-02, 0.0158397481, 0.0391988084, -0.0632879585, -0.0766566917, -0.0283804908, -0.0818740427, -0.0784026235, 0.0235206671, 0.0587505102, -0.0391890481, 0.0177982207, -0.0348450318, -0.0253577456, 0.0559819825, 0.0376594849, -0.0095427772, 4.347650e-02, -0.0182055477, 0.00552234938, -0.0279998984, 0.00187463185, -0.0480767936, -0.0822719187, -0.00665370887, -0.0584376231, -9.20871389E-4, 0.0081911562, 0.0548911281, -0.0888053774, 0.0555477515, -0.0705000535, 0.0419788435, 0.052219335, -0.0550876111, 0.00734952558, -0.0491867289, 0.0118516991, 0.061939396, -0.0662160739, -0.0398468375, -0.0140336687, 0.00308278343, 0.0791953876, -0.0105050672, -0.0328964628, -0.0327373259, -0.0267437045, 0.0225981399, -0.0158779472, 7.188750e-02, -0.00678587518, -0.0554921106, 0.0538019799, 0.0604127795, -0.07945849, 0.0455722921, 0.0501052327, 0.0358460918, 0.0289074387, -0.0310784914, 6.23992353E-4, 0.0625028536, -0.0669675544, 5.242450e-03, -0.0823994875, -0.043255955, -0.0756092891], [0.0511189252, 0.0845862403, 0.016619103, 0.09721037, -0.0602062866, 0.0710187927, 0.0471089631, 0.011952783, -0.0602899306, -0.0202012528, 0.111663498, -0.0232059136, -0.0582265072, -0.00174111687, -0.0558048971, -0.00850627571, 0.0227641575, -0.054699257, 0.0811921283, 0.0362077616, -0.0710470676, 0.0555891171, -7.057730e-02, 0.0729341208, -0.0172339678, -0.0244705658, 0.0580488145, -0.0784213468, 0.00907247699, 4.838580e-02, 0.0091250008, 0.0443255156, 0.0420615189, -0.0331319459, 0.0340168439, 0.0434095114, 0.0754508525, -0.070778586, 0.0447295345, 0.00711046206, -0.0559216291, -0.00724186748, 0.0788698941, 0.00377349695, 0.0472548641, 0.077907905, -0.0868985206, -0.0549004935, -0.0049422849, 0.0780305415, -0.07484667, -0.0276419315, -0.0922964289, -0.0274298694, 0.0214949865, 0.0545257814, -3.433940e-02, 0.0246578213, 0.0719009265, -0.00532506779, 0.0395996235, 0.0787525102, 0.0401696302, 0.0876275375, -0.0562936589, -0.0710563957, 0.00863164663, -0.0541066676, 4.861480e-02, -0.0541662872, 0.0855286717, -0.0952185094, 0.0334673077, 0.0196990073, -0.031870421, -0.00246001058, -0.0732814744, 0.0112844855, -0.0141781289, 0.0431675836, 0.0226204768, 6.34992961E-4, -0.052693937, 0.0152941328, -0.0488275439, 0.0740337297, 0.0123563092, -0.00408594869, -8.145490e-02, -0.0609818324, -3.55013908E-4, 0.0540867634, 0.011948497, 0.00550371595, 0.0716757402, 0.05860506, 0.021981949, -0.0266662724, 0.0501059555, -0.096631363, 0.0454296619, -0.0261207763, -0.0381227471, -0.0286660399, 0.0517525449, 0.010456156, 3.75039846E-4, 0.063697122, -0.0217246134, -0.0516087152, 0.0837248265, 0.0342311971, 0.010382263, -0.0355541408, -0.0572695136, -0.0536198057, -0.0370871276, 0.0573398024, 0.0422492474, -0.0173204727, 0.0394271798, -0.0372676924, 0.063734889, -0.0419181809, -0.0459286086, -6.49505761E-4, 0.0187507439, 0.0480164289], [-0.0479218252, 8.299150e-02, -0.0290053319, -0.0464894921, -0.0600533299, -0.0686306506, -0.0237566847, 0.0344329886, 0.0731068477, 0.00254013319, -0.0170193221, 0.00225578924, -0.00336132059, -0.00189887302, 0.0775486305, -0.00390027021, 5.072810e-02, 8.482930e-02, 0.108671613, 0.11825975, 0.0172659978, 0.0367497616, -0.0839761496, -0.0153223304, 0.0650163814, 8.56380793E-4, 0.029578805, -0.0695985779, 0.0342008844, 0.0166226905, -0.0364063494, -0.0401655287, -0.0270315688, 0.0949511826, 0.0603046715, 0.128372818, -0.0600113273, 0.038590692, 0.022184683, -0.0187273659, 0.0124151362, -9.36569588E-4, 0.102798119, 0.0187889263, 0.0383512042, -0.0533099733, -0.00572981127, -0.0407936908, 0.031354256, -0.0279612299, 0.0726378411, 0.0826767832, 0.0439616814, 0.0386993773, 0.0179114453, 0.0742232054, 0.0890308246, 3.010420e-02, 0.0650248825, -0.00964723248, -0.038415283, 0.0764509365, 0.0439592861, -0.0558088347, 0.0287383292, 0.0366297662, 0.0410123393, -0.0237134714, 0.069079645, -0.032827273, 0.0122215804, 0.0168951042, -0.0612054765, 0.0718160123, 0.0374988392, 0.081013523, -0.0336553268, -0.00217786082, 0.0377091765, 0.0667556301, 0.0188491382, 0.0281744879, 0.0103250872, 0.0681477413, 0.023069296, -0.065133594, -0.00803265348, -5.434180e-02, -0.0158056542, 0.0305430815, 0.0745472535, -0.0656829402, 0.00416569738, 0.0857003852, 0.0109301973, 0.0510331914, 0.0299083553, 0.0762357413, 0.0599722303, -0.0958859473, -0.00690905796, 0.0504137762, -0.0157277416, 0.0657745823, -0.0430067964, -0.0541760325, -0.00353990495, -0.0424735323, -0.100765534, 0.087001577, 0.0658364445, -0.0676487163, 0.0326735042, -0.032402724, 0.00107340887, 0.00857780501, -0.0030907297, -0.0349668749, -0.0310501251, 0.0320208855, -0.0877438039, 0.0260315351, -0.0703350157, 0.0104560284, -0.0128653301, -0.0267375894, -0.0157988071, -0.0282663461], [-0.0877625495, -0.0275655352, -0.0348966345, 0.0150498059, 0.0893825888, 0.0442051589, -0.072655268, -0.00327612367, -0.00549533265, -0.00514454767, -0.0334555767, -0.00859062187, 6.735590e-02, -0.00550980167, -0.0163660739, 0.0228458159, -2.55131628E-4, 0.0643032044, -0.0161670577, 0.0558769442, 0.0866155177, 0.0318278894, 0.0503311083, 0.00207638089, -0.0143630849, -0.0663795248, -0.0308803543, -4.449760e-02, 0.00568055641, 0.117333174, 5.16637738E-5, 0.00970944855, 0.0180555601, 0.0801828131, -0.0369634889, 0.0832358226, -0.0233907811, -3.659590e-02, 0.0568849817, -0.0146656102, 0.0146980574, -0.0506953374, 0.0857264623, -0.0498410687, -0.0585849434, -0.0796523392, -0.0509726517, 0.0278246552, -0.015711464, -0.0303314924, 0.0518801957, 0.113524474, 0.0609984361, -0.00584154343, -0.0337743275, 0.0981450453, 0.0717426464, 0.0673709735, -0.061432641, 0.0290334132, 0.00565592106, -0.0258022379, 0.0292961858, 0.0761050283, -0.0173996631, -0.056979388, 0.010225011, -0.00233268226, 0.0107871592, 6.003600e-02, 0.068828389, 0.0402019434, -0.0695321038, -0.0739770233, 0.0362426266, -0.0421770848, -0.0122816777, -0.028565852, 0.0656265616, -0.0126957325, 0.0844745114, -0.0843398198, 0.0496487543, -0.042501092, -0.0240955465, 0.0629410073, 0.0490044914, -0.0542540178, 0.00371136935, -0.0878537371, 0.0219074022, -0.0769427418, -0.0361900441, 0.0279489737, -0.0465377644, 5.379170e-02, 0.0548884869, 0.0556873791, -1.29582564E-4, -0.0250857212, -0.0329821743, -0.0627924874, 0.0741502494, 0.0603888743, 4.246000e-03, -0.0663329512, 0.0306016207, -0.0172948968, -0.0622554198, 0.0218851846, 0.0427360088, -0.00408717943, 0.0713998154, -0.00868333783, 0.0635083392, -0.0509465039, 0.0628530383, -0.0732209608, 0.0651085451, 0.0521238558, -0.043359112, 0.095434077, -0.0416561663, 0.0819588601, 0.0949289202, 0.0518728197, 0.0529549196, -0.0494604893], [0.0613757484, -0.0594857931, 0.00863255467, 0.0282297134, 0.0244480707, -0.0692213625, 0.0783717632, 0.0322521105, 0.0188820641, 0.0801495835, 0.00550221745, 0.0665508881, -0.0775571167, -0.036074508, 8.378650e-02, 0.0672580301, -0.0541078076, -0.00944056362, 0.0897890478, 0.0456553847, 0.0817351117, -0.0301690809, -0.0897298529, -0.0719609112, 0.0121116927, 0.0246258341, 0.0913229286, -0.0120653082, 0.0218857527, -0.0129292766, -0.0535494909, -0.00757283205, 0.0576020032, 0.0148151033, -0.0433386974, 0.12516354, -0.0535344183, -0.056478478, 0.11078155, -0.0472818539, -7.731890e-02, 0.0281534493, 0.0569203608, -0.0486173928, -0.059482649, -0.0141930254, -0.0238179527, 0.00826409739, 0.119747549, -0.0369465537, -0.0168518983, 0.00935527589, 0.0263142958, 0.0160682704, -0.0177908279, 0.0908383354, -0.00491623301, 0.017120719, -0.0347329155, -0.0543599762, -0.0499680862, -0.0130167967, 0.0448105931, -0.0109866802, 0.0535977334, -0.012039084, -0.0229918305, 0.00279849023, -0.0587746836, -0.0655661076, 0.0649606958, 0.0420189947, 0.061794091, 0.0509652644, -0.0672635511, 0.0152131775, -0.0269279741, 0.0243866257, 0.0256068353, 0.0128573226, 0.0344881751, -0.0399726406, -0.0250139702, 0.0617067218, 0.00791795365, 0.0372652337, 0.0527165867, 0.0452552363, -0.0896039679, 0.0474504344, 0.0736939833, 0.0143400393, -0.0557711907, 0.00550152874, 0.00506148161, 0.035363242, 0.0424685441, 0.0710623711, 0.0584818721, -0.0819315537, 0.0679231808, 0.0375604369, -0.00235516857, 0.0665863454, -0.0516067706, 0.0102324337, -0.0436010249, -0.0577886291, -0.108432859, 0.0304213837, 0.0964010357, -0.0284789111, 0.0140215885, 0.0577692464, -0.00777730485, 0.072431013, -0.0666682199, 0.0226868819, 0.0277202427, 0.0730040222, 0.00273705693, 0.07362286, 0.0854715183, 0.0501485094, 0.01042182, 0.0710201785, -0.0283456035, -0.0213867445], [0.0440699793, -0.010669833, -0.067225188, 0.0574400723, 0.0801848992, 0.0584169701, 0.0431651808, 0.0602195375, -0.0428188704, 0.0933572351, -0.0091899531, 0.0116479043, 0.0232313164, -0.0156270433, -0.0121067055, -0.0603311732, 0.0164540745, -0.00702862768, 0.0839618369, 0.0172734559, -0.010661616, 0.0455524959, -0.0235265754, 0.0110625438, 0.100176468, 0.0763282403, 0.0612640306, 0.067881979, -0.0368319973, 0.0451151654, -0.00250253337, 7.295720e-02, -0.00474892324, 0.0845097675, -0.0363088325, 7.766660e-02, 0.0596857853, -0.0286372416, -2.98866144E-5, 0.00576744415, 0.0455770418, -0.0594987273, 0.0138030183, -0.0294784158, 0.0396771878, 0.00467310194, -0.0783035978, -0.0172649249, 0.0125854714, 0.00248685596, 0.0854200572, 0.0716986954, 0.102054104, -0.0917246639, 0.018075509, 0.0499827303, -0.0576916449, 0.0515773296, 0.0180636458, -0.0208884422, 0.0135545749, 0.00423833588, -0.0357083976, 0.0444927886, 0.0597204976, -0.0316290595, 0.0337138213, 0.0312416237, 0.0305362493, 0.0371922441, -0.0275939554, -0.0567753538, -0.070126839, 0.043402642, 0.0121517153, 4.801170e-02, 0.00430026138, 0.0241684783, -0.0534463301, 0.0182828791, 0.0455397815, -0.0698651075, 0.0411659926, 0.0362899937, -0.0268112347, 0.083017543, 0.0429224595, 0.0938866436, -0.0330839232, -0.076847516, 0.0729256794, -0.0561796129, -0.0248175375, 0.0522865504, 0.00573411677, -0.0603533424, -0.0147460345, 0.0130383242, 0.0896186307, -6.254740e-02, -0.0502380654, -0.0532924719, 0.0644766092, -0.0364162214, 0.00485715596, -0.0165695325, -0.0674848109, -0.0649762079, -0.105909862, 0.0427401811, 0.0270007141, 0.0433063656, -0.0604651459, 0.0622795708, -0.0147541445, -0.0544337817, 0.0730993747, -0.0448280759, -0.0544251613, 0.0255924221, -0.00162951578, 0.0437168069, 0.0907755866, -0.0225517284, 0.0911983401, 0.0288111549, 0.0410089381, 0.0714877397], [-0.0677682236, -0.0312400237, 0.0230402704, 0.0837056264, -0.0427434072, 0.0314414725, -0.0575443581, -5.148520e-02, -0.0113166682, 0.0576807037, 0.076278083, 0.039498549, -0.0114260325, -0.00142418162, 0.0292579141, 0.0447194204, 0.0599214509, -0.0534477085, 0.108334623, -0.00554782432, -0.0443757847, -0.0371020474, -0.0780485272, -0.0315738432, 0.0509549752, 0.0916940942, 0.0167047698, -0.0364457034, 0.0524804145, 0.0229282118, -0.0394629836, 0.00379248639, 0.0624491907, 0.0853577479, 0.0693272203, 0.0365935899, 0.0538988039, -0.0734421611, 0.00219852687, 0.0266847555, 0.0106080426, -0.0160355438, 0.0477531217, -0.0212709885, -0.0376139097, 3.11192067E-4, -0.0210097656, -0.030542966, -0.0400146879, -0.0265750419, -0.0063735079, -0.0393862426, 0.0853225514, 0.0333153568, -0.0897514298, 0.0817245394, 0.0025263757, -0.0602087118, -0.0170460232, 0.0892073884, -0.0732766166, -0.0376307592, -0.0697706639, 0.0307964943, 0.0538735949, 0.0900915712, 0.0123898452, -7.032300e-02, -6.62656676E-4, -0.0315375254, -0.015655078, -0.0796184241, -9.5141679E-4, -0.0269317664, -3.12230077E-5, 0.0577609614, 0.0391267203, -0.038803719, -0.0118633136, 0.0455126539, -0.0689290389, 0.00739055779, 0.0569502972, -0.0582995415, -0.0314766876, -0.0411612093, 0.101337135, -0.0459938534, -0.0272431765, -0.0509833917, 0.0275241397, -0.0389277451, -1.28756423E-4, 0.0763969868, 0.0178610552, 0.0345672294, 2.35540938E-4, -0.0177311096, 0.041292984, -0.0882817208, 0.0421069972, 0.0639250278, 0.0153637473, -0.0458128527, 0.0246371124, 0.0419635028, 0.0056402781, 0.00485090027, -0.022455167, 0.0347326808, 0.046675656, 0.0583288074, -0.0215179045, 0.0284642857, 0.0692726299, 0.00782199762, 9.146120e-03, -0.0423358642, 0.0522997491, -0.0308298804, 0.0178615861, 0.0165468864, -1.39451367E-4, 0.072263956, 0.0509293042, -0.0514652692, 0.0135486368, 1.40018048E-4], [0.0486137085, -0.0762359351, -0.0172274597, 0.0439954847, 0.050325036, -0.0256796554, -0.0200587921, -0.0682357475, 0.0328679681, -0.0167060345, 0.0931498631, 0.0767235309, 7.96083536E-4, 0.00252695684, 0.0610721372, 0.0211526323, 0.0172774438, 0.0765294954, -0.00714695547, -0.0619557462, -0.0540045649, -9.40643717E-4, -0.0317356661, 0.0360408388, -0.0170357805, 0.0894484296, -0.0426988229, -0.0880698636, 0.0365069509, 0.0780968815, -0.0568836778, 0.10348963, 0.00592759904, -0.031542629, 0.042389337, 0.0419026241, 0.0128401155, 0.0494878702, 0.0519963168, -0.0712443292, 0.017497858, -0.0745199248, 0.0214059688, -0.0958960503, -9.04369866E-4, -0.0747490153, 0.0417460799, -0.0560251512, 0.002044352, -0.0679254159, 0.0667579696, 0.0733734816, -0.0241134819, 0.0322341323, 0.02550379, 0.0325005688, 0.00921148062, 0.0820710137, 0.0497501716, 0.0277423356, -0.0708597526, 0.0349623375, 0.00421758415, 0.0638527498, -0.0525032878, -0.0429591089, -0.0187841039, 0.00184166059, 0.0392259099, 0.0541312434, 0.098137468, -0.0514268093, -0.036718864, 0.0226114746, -0.0271214526, 0.0520515181, 0.00828506983, 0.00193036825, -0.0436017513, -0.0542460531, -0.0205514841, 0.040387772, -0.0823513269, -0.0407739542, -0.0611197241, 0.0516734049, 0.0186008792, 0.0608069748, -0.0623086728, -0.0167682525, 0.0836092234, -0.0287267324, 0.0264755823, 0.0335003361, -0.0120870685, -0.0481911749, -0.0166853573, -0.0286270045, -0.0194163155, -0.0832197293, -0.00794499553, 0.0669992045, -4.643900e-02, 0.0780807211, 0.0591668896, -0.0253660195, 0.00897642877, 0.0875803828, -0.10110724, 0.0265719183, 0.00347746653, -0.0698288679, -0.0534022301, 0.0399190933, 0.0304646119, -0.0152658867, 0.00549283903, -0.0521841832, -0.057547573, -0.0248580426, -0.0551592112, -0.0491827317, -0.0124826673, 0.00795610528, 0.0450469367, 0.0147654405, -4.767310e-02, 0.0777703375], [0.00708222855, 0.0774776116, 0.0739586726, -0.0525118522, -0.0214767493, 0.0110221365, 0.0582473949, -0.0200890377, 0.0737407207, -0.0489997342, -0.0423611104, 0.0768950954, -0.0322637595, 0.0183009785, -0.0236016698, -0.0677896068, 0.0347390473, -0.00162658468, 0.0592095479, -0.00943683181, 0.0491099171, -0.0714861155, -0.0400310047, 0.0584418699, 0.0397407338, -2.39499903E-4, -0.0200441387, 0.071364589, 0.0638459474, -0.0353883319, 0.0166522935, 5.17010689E-4, 0.0818333402, 0.0807801559, -0.0521432646, -0.0259159636, -0.0354977176, 0.0669649318, -0.0333905034, -0.0139344586, 0.0515916944, -0.0373909175, -0.0637380555, -0.0318795852, -0.0325545333, -0.0450636782, 0.0214614216, -0.0666227788, -0.0449777357, 0.0581224151, -0.0468948111, -0.0293639861, -0.0464735106, 0.0530828163, -0.0749661326, 0.0357393101, 0.075284563, 4.451450e-02, 0.0206050295, 0.0915342271, -0.0625058636, -0.0299379583, -0.00363695598, 0.0913937837, 0.0266129021, -0.0607732087, 0.048854664, 0.0382691249, 0.088443622, 0.0124419322, -0.0363195427, 0.0199986529, 0.0540199503, 0.0261434261, 0.0598295555, -0.020008266, 0.024479216, 0.00904091261, 0.0707838833, -0.0236919876, 4.076590e-02, -0.0675138235, -0.0621712096, -0.0286644883, 0.0168937203, 0.0227632988, -0.0228124447, -0.0106539438, -0.040327996, -0.0414668061, -0.0128358761, -0.0205009822, 0.0629499331, -0.0456745587, -0.0514362454, -0.0545288883, 0.0772082657, 0.00609302381, 3.524960e-02, -0.0875640586, 0.0624728724, -0.0250248238, 0.0275819115, 0.0241567437, -0.0107580861, 0.0520748049, 0.0638733804, -0.0220067147, -0.0906290859, 0.0581587888, 0.0599709786, 0.0267954394, 0.0669850931, 0.0703258589, -0.0164349154, -0.0666425526, 0.071504578, 0.0108402865, -0.0295692347, 0.0513820872, -0.0869626179, -0.068079792, 0.0664410815, -0.00688195322, -0.0537189618, -4.821990e-02, 0.0245617889, -0.0455747731], [0.01037136, 0.0674951449, -0.0471584536, 0.041896604, 0.10781832, -0.0486167222, 0.0507515408, 0.0476081409, -0.0770763531, -0.0308293719, 0.029886812, -0.0350089446, -0.0316741318, -0.0155001516, -0.00656893523, -0.0257867295, -1.51379514E-4, 0.0431927443, 0.0754345804, 0.0318455212, -0.0777801722, -0.0657733976, -0.0125527661, 0.0161198229, 0.00904861465, 0.0386274979, 0.0672795922, -0.0755863115, 0.0621721894, -0.0526198484, 0.0415780097, 0.029417254, -0.0412358046, 0.0334866568, -0.0547494255, 0.0668110251, 0.0193405747, 0.0663572475, -0.0510421023, -0.0548328236, 0.076603286, 0.0778743401, -0.0169303715, 0.0204929486, -0.0245100297, 0.0765775666, -0.0457555465, -0.0181929469, -0.016841054, 0.0372640118, 0.0736820772, 0.0579269193, -0.066612877, -0.0622287504, 0.0289000813, -0.0556748696, -0.0241776817, 0.019725509, 0.0735152513, 0.00995070114, -0.0581843629, -5.662720e-02, 0.0875539407, -0.0333451703, 0.0454296693, 0.0332351401, 0.0241898485, 0.0549591146, 0.0567805767, -0.0351958685, 0.01419154, 0.0265334472, -0.0186659228, 0.0282027982, -0.0375625119, 0.0615071282, 0.0430763923, -0.053933844, 0.0107520055, -0.0242374055, -0.019758204, 0.0776949078, -0.0735922083, 0.0715513229, -5.816790e-02, 0.00808840431, 0.0480779223, -0.011480025, -0.0222226772, 0.0402918942, 0.0583313107, 0.0993979051, -0.0243374836, -7.398490e-02, 0.0200610943, -5.343020e-02, -0.0340802111, -0.106579944, 0.0741384551, -0.0558470674, 0.0904604569, -0.0193420574, -0.0702745095, 0.0922054871, -0.0315162949, 0.0117894728, 0.0155567685, -0.073175922, 0.012460147, -0.0618456192, -0.0200756248, -0.0596979447, -0.0687119514, -0.00679508782, -0.0370112285, -0.0898611099, -0.0299316607, 0.0273499116, -0.0254968796, 0.00680955267, -7.091980e-02, 8.457430e-02, -5.947070e-02, -0.0166082922, 0.020246448, -0.069832392, 0.0921421051, -0.0382375419], [0.0404662527, -0.0216735229, 0.0242568403, 0.056846384, 0.0805283412, 6.391850e-03, 2.09366714E-4, 0.0476840287, -0.0157942809, -0.0367692895, -0.0153747387, 0.0194025096, -0.0771959648, -0.0524076484, 0.0330924876, -0.0818627328, -0.00205233297, -0.0538851209, -0.0476255752, -0.050829988, -0.0212630574, 0.052142445, 0.0603861585, 0.0592564903, -0.019018041, -0.0825716927, 0.0214972682, 3.970330e-02, -0.0349324867, 5.948720e-03, 0.00818622671, -0.0541388579, 0.0799277722, 0.0366674177, -0.0523878932, -0.0623205639, -0.0088091772, -0.038380228, 0.0848830938, -0.0275133234, -0.0627266765, 0.0146424342, -0.0786429495, 0.0751399174, -0.0394856855, 0.0173744336, -0.012415166, -0.0263351854, -0.0786366686, -0.0603939965, -0.068358317, 0.0193285178, -0.0540571846, -0.0233559106, -0.049976144, 0.060116943, 0.0194694027, -6.295440e-02, 0.021118585, 0.0718057752, -0.0462889224, 0.0612020642, -0.0262503568, -0.0169826578, 0.0487165041, 0.0104204305, 0.0687288939, 0.0537784025, -0.0505779646, 0.0182508174, 0.0421583094, 4.19718126E-4, 0.0585025959, -0.0448468179, 0.072861515, 7.018230e-02, 6.68667838E-4, -0.0284352079, -0.039634645, -0.0469853431, -0.0196422078, -0.0290628988, 0.00184094906, -0.00606454397, -0.0603144579, 0.0298209731, 0.0306367986, -0.0355491042, -0.102943435, -0.0347177424, 0.021291839, 0.0252038278, 0.0595528632, 0.0327080898, -0.0242740829, -0.00155352708, -0.0383267291, -0.0633307099, 4.470110e-03, -0.088831745, -0.0645323247, -0.0563464388, 0.0269523486, -0.0260981489, -0.049064707, 0.00685154041, 0.055714123, -0.0369673185, -0.0272820089, 0.0564248115, -0.0169283729, -0.0715771243, -0.0717473626, -0.0273448564, -0.0120447129, 0.0147889471, 0.0474259779, -0.0211710054, -5.856350e-02, -0.100026108, 0.0429897197, 0.0643434301, -0.040842358, 0.0138566503, 0.00501375087, 0.0761208832, 0.0204677023, 0.00992747396], [0.0751765519, 0.0340373144, -0.0540414862, 0.0485440828, 0.0127759874, -0.0396802835, 0.0173576549, -0.0608891249, 0.0264868122, 0.0369994417, -0.0708890706, -0.0563253053, -0.0729628354, -0.06332279, 0.0854502469, -0.0146716787, 0.0688946545, -0.0542244464, 0.0696801767, -0.0142668551, 0.0022324312, 0.00750282733, 0.0446751229, 0.0109524596, 0.0140429297, -0.00376588106, -0.0587257333, 0.029155191, -0.0128406938, 0.0291549098, -0.0659916773, -0.022099629, -0.0321060866, -0.0679013208, -0.0510562696, -0.0755987242, -0.0605463274, -0.055294212, -1.86731209E-4, 0.00802222825, 0.0187624041, -0.0396679975, -0.0158659741, 0.0333625227, 0.0121536106, -0.0454063341, 0.0241971016, 0.0756648555, 0.00619318802, -0.0818389728, -0.0681971461, -0.0629151836, 0.0422420613, -0.0380452611, 0.0246592853, -0.00884406548, 0.0618254952, -0.00444530509, 0.00270405505, 0.0426107645, -0.0213472191, 0.0321638323, -0.0243395306, 0.0740811378, -0.0429537669, 0.0470975116, -0.00268044486, -0.0166534334, 0.00978614297, 0.0227761827, -0.0811897292, -0.0429781303, 0.0395193845, 0.0670952946, -0.00959942769, 0.0427518077, -0.078225404, 4.51106083E-4, -0.0557172894, 4.899080e-03, -0.0455002077, 0.00318439957, -0.0278988685, -0.00148772018, -0.0560403951, -0.0310748573, 0.0451564379, 0.014848371, -0.0513721518, 0.0214226563, -0.00579441758, 0.103702977, 0.0219245013, 0.0196933094, 0.00376383797, 0.0362884887, 0.012430544, 0.0262595676, 0.0709888563, -0.00827815477, 5.128350e-03, -0.0395869911, -0.00314968103, -0.0484029576, 0.00392472744, 0.00178576016, 0.0288927313, 0.0494253337, -0.0288524218, -0.0546669476, 7.807980e-03, 0.044855956, 0.0577080809, 0.0528791361, 0.00536395889, -0.0485810228, 0.0174615495, 0.00246592914, -0.0721409395, -0.00309403636, 0.0702551678, -0.0508788489, 0.0181819685, 0.0258437023, 0.0279303808, 0.0729487315, 0.0286547821, -0.0020276173], [-0.0562733859, -0.0788441076, -0.00514573604, 0.0365318134, 0.0713090524, -0.0382299609, 0.0724256411, 0.0260942895, 0.00285455072, 0.0190194696, 0.0149025675, 0.0121300481, -0.0612053946, 0.0326920189, 0.0478565879, -0.0515148081, 0.0254191179, -7.02453195E-4, 0.036354769, -0.03415123, 0.0430774577, 0.0607349463, 0.00878739915, 2.67296535E-4, -6.761760e-02, -0.0740899145, -0.0529715456, -0.0909013599, 0.00157672388, 0.0572949313, -0.0110829528, 0.0454351641, -0.0424055867, -0.0128684239, -0.017113857, 6.461710e-03, 0.0138971386, -0.0416675694, -0.0660891756, 0.0174819678, -0.0524686165, -0.0197206754, 0.0714812204, 0.0238798801, 0.0342810266, -0.0373107754, 0.03168834, -0.0298029296, -0.0122519601, -0.0487385839, 0.0669693202, 0.0713631436, -0.076573193, 0.0216895603, -0.0605359115, 0.031704355, -0.0642341301, 0.0250845868, -0.018769037, -0.0803892165, -0.0380726345, -0.0161879454, 0.0134499092, -0.00224346668, 0.00134758686, -0.0691556185, 0.0203913599, -0.0706832185, -0.0373848788, 0.072361365, -0.0655975938, 0.0796636268, -0.0417173877, 0.0412300266, -0.0400295109, -0.049392283, -0.0646543503, -0.0499989167, -0.0322016589, 0.0298214667, 0.0355321877, 0.0318454206, 0.0565417707, 0.0802791714, 0.032341361, 0.0714271441, -0.0424658246, 0.00917843636, 0.00597426808, -0.0552455224, 0.0420412868, 0.0815408677, -0.00728141377, -0.0516811386, 0.0242854264, 0.0473523438, -0.0568118356, 0.0306010637, 0.0333941281, -0.00279074488, -0.0632727072, 0.0362863205, 0.042485524, -0.0584064871, -0.0728023052, 0.015319692, -0.00714249024, 0.0181678534, -0.00291017583, 0.0481452495, -0.015087639, -0.0389577076, 0.035016682, -0.0751346797, 0.0339322351, -0.0892011597, 0.0747640803, -0.0421068333, 0.00401685294, 0.00765322288, 0.0411263518, 0.0666914135, -0.0309523642, 0.0507609844, 0.0660307854, 0.0401689932, 0.0490771122, 0.0664908885], [-0.0780773684, 0.0351240188, -0.00170173624, -0.00743807526, 0.0132636065, -0.0105823688, -0.0167989936, 0.00811894145, 0.0440100618, 0.0663264095, -0.0721071288, 0.0210914258, 0.0541075543, 0.0162508525, 0.00436988939, 0.0630485043, 0.0518507734, 0.0527026504, -0.0603764951, -0.0324427783, 0.00104224344, 0.0110636204, -0.0326655395, -0.00582135655, 0.0345673412, -0.0576717742, -0.0426020361, 0.0720432102, 0.0369070619, -0.0787906646, -0.00208595325, 0.0727410689, 0.0287346244, 0.00834553223, -0.0439980552, 0.0390740186, 0.0407268927, 0.078218922, 0.0011898313, -0.00559540559, -0.0447231084, 0.00196108967, 0.00685301051, 0.0556986779, 0.0329115801, 4.496820e-02, -0.0105280224, -8.50131444E-4, -0.0616011657, 0.0132784359, -0.0676148757, -0.0742686763, -0.0111018931, 0.0742081925, 0.035781797, 0.0405759811, 0.0287657231, 0.0612146445, -0.054767631, 0.0265216697, -0.031484507, -0.0567744486, 7.706160e-02, 0.0214532856, 0.0336959064, -0.0551752858, 0.00308127166, 0.0600948744, -0.0383156128, -0.015756432, 0.0470693856, 0.0390982553, 0.041110713, -0.0318106785, 0.0313787721, -0.068448022, -0.0156287532, 0.0784832313, 0.0229980964, 0.036915306, -0.0538050458, 0.0710017607, 0.0761836916, -0.0249114968, 0.00958626717, -0.0521851182, 0.0579689927, -0.0544728711, -0.0773079917, 0.0344347134, -0.0153065352, 0.0856093093, -0.013004588, -0.0712100863, 0.00556652853, -0.069168441, -0.0466499217, -0.0782560929, -0.0343690328, -0.0535213947, -0.0753440708, 0.0439888611, 0.0299362279, -6.211970e-02, -0.0435637906, -0.00975935906, -0.0568015575, 0.0236784685, 0.010206135, -0.0584593713, -0.0554560274, 0.0275319144, -0.00320207886, -0.0340121351, 0.0105970157, -0.0412604548, -0.0244739316, 0.0326282158, -0.0362160094, 0.0663011894, 0.0699761584, 0.0439764746, -7.596890e-03, -0.0299613234, -0.0278277602, 6.474920e-02, -0.0213624574, 0.0263619814], [-0.0393454358, -0.0771421865, -0.0522012748, 0.00920701958, -0.0236287899, 0.0300035514, 0.00965470541, 0.0643460527, 0.0481642634, 0.0727438852, -0.0520865545, 0.0746394545, -0.0241019558, 0.0852979943, 0.0301045626, -0.017352609, -0.0237492975, -0.0744753331, 0.0191611629, -0.00526277814, 0.0397092625, -0.0398185067, -0.00157267647, 0.0178424604, -2.754740e-02, 0.0647058487, -0.00641194544, 0.0669776052, 0.00369872828, -0.00480092317, -0.0798774063, 0.0587483235, 0.0067024664, -0.0665378943, 0.033133015, 0.0189556051, 0.0368515067, 0.030094007, -0.0462375022, 0.0151267527, 0.0430209786, -0.0579832457, -0.0754838287, -0.0580120981, -0.030884048, 1.67389182E-4, 0.0731259659, -0.0163546242, 0.0784949958, -0.0211902503, -0.0529009588, -0.0318994261, -0.0459154844, 0.0661721453, -0.0116478959, 0.0196678713, 0.0478344113, 0.0683111623, -0.0398146324, -0.0808815062, 0.0342238918, -0.0315038413, -0.00421422301, 0.00212443364, 4.37193114E-4, 0.0467507206, 0.0564111806, 0.030502947, 0.068304494, -0.0819055959, -0.0732205957, 0.0514372922, 0.0411293507, -0.0100781908, -0.065613985, -0.0757482275, -0.0524568707, -0.0466420166, -0.0573287942, -0.0619994551, 0.0567392297, -0.0310513154, 0.075521037, 0.0822670981, -0.0777139365, 0.0475409739, 0.0477934703, 0.0702291876, -5.150700e-02, -0.049859371, 0.0542588606, 0.0362011977, 0.0780856534, 0.0335630774, -0.00400416739, -0.0338073932, 0.0492312647, -0.0144936163, 0.0307293274, -4.744450e-02, 0.0480927527, 0.0187046491, -0.0719886199, -0.0254844408, -4.46373044E-4, -0.0424627215, -0.067876704, -0.0482527427, 0.0639665723, 0.0154143032, -0.0742087364, 0.070701018, 0.00580350216, 0.00412676251, 0.00540692406, 0.0127146775, -0.0550148189, 0.0389082506, -0.0337518901, 0.00383638614, 0.0733001232, -0.00125963171, 0.046530623, 0.0222823434, 0.0653647408, -0.0403603315, -0.0103231417, 0.00531289773], [0.0314586759, -0.0033275953, 0.0299201757, -0.03316883, 0.0467505269, 0.0740010962, -0.0670227259, 0.0121033173, -0.01276985, -0.0710224584, -0.0568942316, -0.0565516911, 0.0270208325, -0.0624265633, 0.0156383757, -0.0103742871, -0.0627697706, -0.0568448827, -0.0799074396, -3.499450e-02, -0.0584202595, 0.026215449, -0.00853373669, 0.04508093, 0.0369636044, 0.0436396785, -0.071059674, 0.0335307196, 0.0608039163, 0.0699862465, -0.0056326692, 0.00951598678, -0.0524228923, 0.0219208449, -0.0199930854, 0.0241231322, 9.99573618E-4, 4.932040e-02, -0.0213088728, 9.751300e-03, 0.0342287198, 0.0647816584, -0.0264144186, -0.0194217544, -0.0184784271, -0.048370935, -0.0442030765, 0.0557124838, 0.0411550961, -0.0736615285, -0.0269397199, -0.0292305462, 0.0402106866, -0.0797219201, -0.039932631, -0.0524443798, 0.00146516506, -0.0452503301, 0.0420826301, -0.0700709373, 0.0332711823, 0.0353676081, 0.0170296263, -0.043864388, -0.0365578756, -0.0588992201, -0.00821414497, 0.0111288689, -0.0287230629, -0.0671396703, 0.0467651524, 0.0628853962, 0.00983162969, -0.0732891411, -0.0664972886, -0.00896317605, 0.0724959597, 0.0770599619, 0.0797229409, 0.0219879691, 0.0284301415, -0.027776707, 0.0579296425, 0.00470346818, -0.0161241554, 0.0773101449, -0.00340459589, 8.43650603E-4, -0.0487601198, -0.0577602573, 0.00458983518, 0.0155090448, 0.0530128591, -0.0369502082, 0.0341674797, -0.0461544916, -0.0631179288, 0.033615537, 0.0661648065, -0.0494552292, -0.0316772833, -0.0601601154, -0.0258041359, 0.0768355802, -0.0398576967, 0.00307505275, -0.0101044662, -0.0210603941, 0.0250283219, 0.0608245395, 0.0453321151, -0.0519632772, 0.0431615673, 0.0451740772, -0.068308346, 0.00289581367, 5.166250e-02, 0.0264544375, -0.00722635537, -0.0347201303, 0.0137938503, -0.0255711284, 0.0278712828, -0.0747507885, 0.0122521231, -0.00469639897, 0.0238610655, 0.0118899895], [-0.00676369341, -0.038743794, -0.0782637968, 5.032520e-02, -0.032273531, -0.0298230983, 0.0332005136, 0.0719438419, 0.0496029519, 0.0508123226, -0.0350589156, 0.073131375, -0.0505095422, 0.0175490379, -0.0416789949, -0.029842332, -0.00652662804, 0.0187755227, 0.0569060184, 0.0147638079, 0.0183773842, -0.0189939607, 0.0625062659, 0.0705592334, -0.0650681183, -0.0435707495, 0.0128399162, -0.0767669081, 0.0722738951, -0.024051385, -0.0360668413, 0.049520392, 0.0405527949, -2.109610e-02, -0.0392067321, -0.0178691484, -0.0527889878, 0.0177316871, -0.0622946061, 0.00826408621, 0.00122945115, 0.0694415495, -0.0403093919, 0.0213396046, 0.038624838, 0.0707011148, 0.0029815631, -0.0517431423, -0.0599896461, 0.0387332924, 0.0473713875, 0.0806109979, -0.0570887066, -0.0129259964, 0.015292461, 0.00166425877, -0.0547293611, 0.0804305896, -0.0491392463, -0.0636658892, 0.0334686562, 0.0724159107, 0.0552937277, -0.0347403064, 0.024545474, 0.0477403849, 0.0479612909, 0.0104070846, -0.0310930125, 0.0199340601, -0.0195686612, -0.0708437263, -0.0361588411, 0.0191291235, -0.0432511047, -0.0140530327, 0.03591929, 0.053623531, 0.034500204, -0.0137932608, -0.0552408285, -0.0416229181, -0.0268746465, -0.0802784562, -0.0236005783, -0.0272102579, 0.0487011783, -0.0550859943, -0.0522701256, 0.00676172879, -0.0777754709, 0.0567014478, -0.0254962593, -0.0744095817, -0.0267706439, -0.0679467469, 0.0159679577, -0.0792606622, -0.0779479742, -0.058884427, 6.116310e-02, 0.0275881458, -0.0407134444, -0.0457235314, -0.0570415035, 0.0601230562, 0.0794143304, -0.0475904495, 0.0331202522, 0.0311817247, 0.0626511871, -0.0129595026, -0.0155744441, 0.0311194062, 0.00136552949, -0.0738192946, 0.0345133543, 0.0533533469, 0.0560371727, 0.0122315483, 0.0677238479, 0.0491227731, 0.0605064668, 8.97306949E-4, 0.0384586416, 0.079978995, 0.052676104, -0.0565027706], [0.0568307452, 0.0473230816, -0.0115425829, 0.0344494358, -0.0499982126, 0.067187205, 0.0631758124, -0.0393528789, 0.0126440907, 0.04530783, -0.0638534799, -0.0715196356, -0.0478447527, 0.0360104851, -0.0562659837, -0.0376720428, -5.094910e-02, 0.0311533809, -0.00306652114, 0.0547373146, -0.0316770449, 0.0784538909, 0.0241483077, 0.0126390783, -0.0015313871, 0.0586814806, 0.0330919065, -0.0481532663, 0.0206494518, -0.0610216334, 0.00947296991, 0.0119491508, -0.0184457134, -0.0511443764, 0.0185277518, 0.0103016319, 0.0661184192, 0.0186783485, 0.00686334493, -0.0387851894, -0.0191030093, 0.0404950157, -0.0680979639, 0.0260210428, -0.0692314133, -0.00626470381, -0.018254282, 0.0472399704, -0.0467426516, -0.0607724115, -0.0293644052, -0.0498085842, 0.0782221257, -0.0621920601, -0.00669939769, 0.0398820899, 0.0449308679, -0.0523238406, 0.06443014, 0.0355387889, -0.0522474945, 0.0693247169, -0.00542503316, 0.0441150069, -0.00752119441, 0.0168256424, 0.0548449233, -0.0065574795, 6.03186374E-4, 0.0245468728, 0.0702534541, -0.0641878694, -1.613930e-02, 0.0132557619, -0.0649440661, 0.0557176024, 0.018746851, 0.00580861606, -0.0342968032, 0.0782824904, -0.0770195723, -0.0297841951, 0.0382705145, -0.00687286863, 0.0259382948, -0.0452608503, -0.0176264215, 0.0409940332, -0.0152429314, 0.00977752823, 0.0412116572, 0.0739009529, 0.0118012233, 6.429910e-02, -0.0403901972, -0.00228803395, 0.0611562021, 2.157370e-02, 0.029068334, -0.0322678052, 0.0134985652, 0.00522833504, 0.0460541658, -0.0790348574, -0.0648402795, -0.0801009237, 0.0239459723, -0.0439110659, 0.00805370323, -0.032021571, -0.0455983467, 0.0626727268, 2.512990e-02, -0.041560635, -0.0212321747, -0.00265998347, -0.0315515026, -0.0540365577, 0.0619462579, -0.0195400734, 0.00489972904, 6.474890e-02, -0.0535284318, -0.0636484846, 0.0680076703, -0.0590996034, 0.0123919696, 0.0408080146], [-0.061810948, 0.00154035538, -0.0240624547, 0.0136379898, -0.00629343837, 0.060556151, 0.0041879639, -0.0119196251, 0.0370118394, -0.0748165175, 0.0341412872, 0.0225917548, -0.0412454978, -0.0289898925, -0.0314406529, -0.0362342298, -0.0246771201, 0.0366893932, 0.00130655617, -0.037494354, 0.0582516715, 0.0408288389, 0.01056014, 0.0565390959, 0.0328222588, -0.0332614705, 0.0210353509, -0.0153138041, -0.0608292222, 0.0100447163, -0.0498275086, -0.0696134344, 0.0051503703, 0.074741073, -0.00973785668, -0.0457312316, 0.0253760815, -0.0236155652, 0.0406065062, -0.0314659476, 0.00653097034, -0.0125966966, 0.0545340702, 0.0387761518, 0.0262705758, -0.00873973221, -0.0702049285, 0.0440169647, 0.0579145178, 0.0331891626, 0.0269462988, 0.00194371492, 0.0525424257, 0.0660712793, 0.0513162836, -0.00533477962, -0.0267655812, 0.0474768355, 0.0628399476, 0.0384704545, 0.00340942293, -0.0775990113, -0.00831384212, 0.0328524262, 0.0220592394, 0.042431809, -0.0776730925, -0.0575811602, -0.0802099854, -0.0801374316, 0.077378489, -0.00286771357, 0.0206077397, 0.0103603974, -0.011818178, 0.0237587616, -0.0131424442, 6.44698739E-4, 0.00643000752, -0.0372364968, -0.0121884272, -0.0357396528, 0.00770952553, 0.0461111739, -0.0289155357, -0.0014314428, 0.0460207388, -0.0111839175, -0.0130507201, -0.0189322717, 0.0493333563, -0.041221153, 0.039481461, -0.0754906684, -0.0114572272, 0.0765631124, -0.0284646414, -0.0537131913, 6.559060e-02, -0.00884009897, -0.0686063692, -0.0532143787, -0.0759685933, -0.00354296714, -0.0517052561, -2.40162015E-4, 0.0334200263, -5.547170e-02, -0.0505289473, -0.043911729, -0.00993651896, -7.324010e-03, 0.070487313, 0.0221201554, -0.0479873903, 0.0359435976, 0.0711837634, 0.0366187692, 0.0456789657, 0.0772793666, 0.0592385754, 0.0391345099, 0.0291883573, -0.00303350389, -0.00138454139, -0.00412897766, 0.00497052073, 0.0492232814], [-0.0522087663, 0.0799164697, 0.0233009681, 0.0633551255, 0.0157895312, 0.0556952134, 0.0190291777, -0.0311538838, 0.0151685923, 0.00100176036, -0.0338103697, -0.04437875, -0.0183538646, 0.0639200285, -0.046782285, 0.0207977816, -0.0349673219, -0.0409066528, -9.81688499E-4, -0.0270333178, -0.0279496461, -0.0126305968, 0.0356866866, 0.0218737051, 0.0383388177, 0.0185288563, -0.0631389096, 0.0265155137, -0.0739594251, -0.0592651814, -0.0602998734, 0.0730547532, -0.0720379129, 0.0731592253, 0.0293244645, 0.0791603699, 0.0250196233, 0.00406881422, 0.0123292506, 0.0146093518, -0.0314017422, 0.00541383773, -0.040331766, 0.0484044179, 0.064483054, 0.02227474, 0.0297293887, 6.530640e-02, 0.0599213913, 0.00883284211, 0.0272189677, 0.0747022256, 0.0186340958, 0.0593919083, -0.0750508606, -0.0364005603, -8.81031155E-4, 0.0126253739, 0.0559323654, 0.0608535036, 0.0632200316, 0.0548826978, 0.0649783239, 0.068304874, -0.0431100801, 0.0660696849, -7.236510e-02, 0.00217074156, 3.84345651E-4, 7.143470e-02, 0.0406653732, 0.0723694339, -0.0264260769, 0.026046522, -0.0300934315, 0.0515226051, 0.0372091159, -0.0699778423, -0.0366628654, -0.0807709321, -0.0512182787, -0.0733794123, -0.0237337984, 0.0308507159, 0.0799554958, -0.0264892951, -0.0121894702, -0.0434755348, -0.0520527288, 0.0324131921, -0.0453431867, -0.0353702717, 0.0163585544, -0.0644467771, 0.0352395251, 0.0630727932, -0.0762429684, -0.0177108496, -0.0734617114, 0.0473702624, -3.535900e-02, 0.076542221, -0.0162229389, 0.0279671624, 0.0506022349, -0.0482371636, -0.055927407, 0.0281710103, -0.0299947485, 0.0363317952, -0.0379580483, -0.0797645673, -0.0763635412, 0.0595506355, -0.0544847511, 0.0253899097, 0.0306614563, 0.0779007598, 0.0530949906, -0.0337626413, 0.0160635933, -0.00572545081, 0.0553149208, -0.0651570708, 0.00967802852, 0.0342263728, 0.0358302146, -0.0528696738], [-0.0727309436, 0.0171692595, 0.0454426035, -0.0467574187, -0.0776902437, 0.0323684663, 0.0590926483, -0.062985383, 0.077518247, 0.0397295281, -0.0377941541, 0.0527952835, 0.0622082427, -0.0264312029, -0.0439072028, -0.0797436237, 0.0386479571, -0.0296169557, -0.0747189521, -0.0478673205, -0.0699432865, 0.0657627359, 0.0106092244, 0.062369965, -0.0575075783, -0.0590750277, 0.0214808881, -0.0793730468, -0.0240646563, -0.0308348425, 0.00166814029, -0.0174552873, 0.00481214374, 5.973900e-02, 0.00530210137, 0.0804175063, -0.0679231286, -0.0489457585, -0.0548598543, 0.0751394853, -0.00523392856, -0.0792355686, 0.0758149698, -0.0639068335, 0.0611488447, -0.0739073306, -4.929530e-02, -0.0590155423, 0.00245399028, 0.0275283381, -0.0790282041, 0.0313933343, 0.0351706818, 0.0182868019, -0.0111806914, -0.0458017588, -5.098220e-02, 0.010731943, 0.0285075754, 0.0396752134, 0.0603070632, -0.0195753686, 0.0558037385, 0.0202585906, 0.0123896599, 0.0399668291, 3.764090e-02, 0.0612067208, -0.0261416286, -0.0612217485, -0.0146198869, -0.074680686, -0.00690209121, -0.0420979112, -0.0789945945, 4.072530e-02, 0.0183899477, -0.0728155822, 0.028688468, 0.0700916424, -0.0275703818, -0.0201475881, -0.0300487988, -0.050456062, 5.522100e-02, -0.0501824431, 0.00561189651, 6.645400e-02, 4.55744565E-4, -0.0455025733, 0.00541924685, -0.0582530722, -0.0189437792, -0.0350270756, 0.0103009939, 0.0425139591, -0.0646971166, 0.0185905844, -0.0476293415, 4.270100e-02, -0.0332779065, -0.0199624449, -0.0472870544, -0.0346208178, 0.0394251868, 0.0615480617, -0.0788791627, -0.0647150204, 0.0618982092, 0.0576254055, 0.0734606311, 0.00338924676, -0.0598792471, 0.00934527069, 0.0299514458, 0.00783893465, -0.0152682438, -0.017687507, 0.0523807481, 0.0574045107, 6.847330e-02, -0.0755609274, 0.0516154096, 0.00242368877, -0.0706128553, -0.004715994, 0.0602110103, 0.00691083074], [-0.0773319303, 8.19831796E-4, -0.0378462523, 0.0491798185, 6.273910e-02, 0.0517697111, -0.00931815803, 2.436030e-02, 0.0403373763, 0.0229302011, 0.0765005648, -0.0233625323, -0.0321552455, 0.0613908395, 0.0714167431, -0.00171601679, -0.0625300109, 0.0334838629, -0.0665468797, 0.074070692, 0.0144383134, -0.0654688925, 0.0451334976, -0.00731688458, -0.0768230259, 0.0458617248, -0.0735058784, 0.0760492682, 0.0165252686, -4.32989822E-4, -0.0527623519, -0.0300225914, -0.0401028767, -0.0321656242, -0.0781552717, -0.0182449836, 0.0182092395, -3.27428745E-4, 0.0418390408, 0.0458440408, -0.0731936693, -0.0214950815, -0.013568785, -0.00406748708, 0.00462402543, 0.00665587559, 0.0761480257, 0.0541283116, -0.0398776904, 0.0797348394, -0.0808607637, -0.0581330471, -0.010012418, 0.0189617332, -0.0334115587, -0.00186294538, 0.0466921329, -0.0509164073, -0.0537861548, 0.0755068361, -0.0714207515, 0.0505483784, 4.63950913E-4, 0.0361301303, -0.0113291861, 6.668860e-02, 0.0374298245, 0.0244911555, 0.00384134427, 0.0242734347, 0.0579735972, -0.0219891816, -0.0328618661, 0.0496967174, 0.0132221747, 0.0574069023, -0.0724616721, -0.0559935905, 0.0567064062, -0.0649599209, 0.0264660046, -0.0342502147, 0.00860251672, 0.0723677427, 0.0342983827, 0.072663635, 0.0195384957, -0.0459081307, -0.0107330419, 0.0377886556, -0.075053215, -0.0421013087, -0.0668910071, -0.065182507, -0.053868562, 0.0233790409, -0.0133634396, 0.0647983253, -0.052683875, 0.0273315907, 0.0410484187, -0.0531671569, 0.0176029149, -0.0351078808, 0.0114052892, 0.0665142089, 0.00442770682, -0.0330666527, 0.00197715964, 0.0418485925, 0.0339695513, -0.0799312443, 0.0344119482, -0.0687992572, -0.0167326555, 0.0523659177, -0.0378631502, -0.0500641093, -0.0659315586, -0.0515057109, 0.0247513764, -0.0268997308, 0.0600771904, 0.0157287344, 0.0244597811, 0.0377132073, -0.0114666624, 0.0656640753], [0.0363838486, -0.0736450329, 0.0317771584, -0.0334735066, 0.0363105163, -0.0712134913, -0.00329609122, 0.0587430671, -0.038158562, -0.0782124921, 0.0816170573, 0.0272303112, 0.0745582655, 0.057794217, -0.0451972075, -0.0401138365, 0.0172810331, 0.0357911512, 0.0393953063, 0.00560178608, -0.0181204733, 0.0105225379, -0.0416284949, -0.0662185847, -0.0333406776, 0.0305029359, -0.0779168307, 0.00489585847, 5.335020e-02, -7.364210e-02, 0.00731280725, -0.0634075105, 0.0136698093, -0.0714132935, -0.0639268532, 0.0149897737, 2.891610e-02, 0.0481751896, -0.0694109499, 0.0565789826, 0.0460158847, 0.00478423806, -0.0177607238, 0.0626357123, -0.041851569, -0.0755729303, 0.0512811355, -0.0557379499, 0.041215159, -0.0562850386, 0.049996376, 0.0705061704, -0.0547832884, 0.0292825475, 0.0294327661, -0.0017980726, -0.0175067671, 0.0574179888, -0.00940181128, -0.0339451879, -0.0547249056, -0.036335215, -0.016240444, -0.0678080916, 0.00708715338, 0.0475525483, 0.0705953315, 0.029837586, 0.0327516682, 0.0190336518, -0.00514119351, 0.0398579277, -0.0555366166, 0.0493684858, 0.067266129, 0.0765526071, 0.0488580763, -0.0295259319, -0.0762779638, -0.0274716131, -0.0779038668, -0.0517916568, -0.0163522102, -0.0366549976, -0.0369275659, 0.0728542209, 0.0769003331, 0.0320619754, -0.0442398936, -0.0289811324, -0.0502301529, -0.061538782, 0.0451319329, -0.0634421706, 0.0300590098, -0.0760137066, 0.010476687, -0.0531797297, -0.00510179438, 0.00275982916, 0.0523106977, -0.0408088341, -0.0274790898, -0.0126200281, -0.00429125316, 0.0321877934, -0.0332468562, 0.0625220165, -0.0336535163, -0.0419273227, -0.0277417041, -0.0409367047, 0.053422872, -0.00376074272, 0.00255987188, -0.0720023587, 0.0453262329, -0.00841649622, -0.0200612172, -0.0132637126, 0.0106993178, -0.0201299321, -0.0249451324, -0.0391688533, 0.0166320894, 0.0567412525, -0.0738593489, -0.0166668706], [0.00881999358, -0.0461390764, 0.0394624844, -0.0404162146, -0.00359053561, -0.0501709506, -0.0613043718, 0.0505339839, 0.0710868314, 0.0276876707, 0.0689686686, -0.0454694442, -0.0244313627, 0.081069149, -0.0681201816, 1.483170e-02, -0.00409386121, 0.0436856821, 0.0464356467, -0.0791744217, -0.0818748474, 0.0609135404, 0.00963945594, 0.0787372365, -0.0335824639, 0.0617429055, -0.0719430148, 0.0379504114, -0.0525993295, 0.0387762338, 0.0765631944, 0.0241782386, 3.703250e-02, -0.0429682769, -0.0184970777, -0.00447017606, 0.036219079, 0.0362424441, -0.0487084053, 0.0398417041, -0.0283919554, 0.0315216109, 0.0485490896, 0.00286391098, 0.0284385383, 0.0653072447, -0.0641962513, -0.0373795889, -0.0601752736, -0.0557678565, -0.0190005451, 0.0325292721, 0.00321502751, 0.0701086521, -0.0763755515, 0.0495630912, -0.00659619644, 0.0200607069, 0.0714845657, -0.017717354, -0.0304230489, 6.526250e-02, -0.0193061233, 0.0466797464, -0.0811969861, 0.0695186928, -0.0108639067, 0.0756112933, 0.0802030861, 0.0346426778, -0.0270381123, 3.14931938E-4, 0.0518331826, -0.0492640622, 0.00561090093, 0.0466608144, -0.050002072, 0.0477916263, 0.0104017602, -0.0771639273, 0.0459258184, 0.0762917697, -0.0208894331, -0.0816211923, -0.0696293563, -0.0648055151, -0.0340470225, 0.00880860258, -0.0622042678, -0.0300866943, -0.00619901624, 0.0658067614, 0.0539498217, -0.0164879858, 0.0216996092, -0.0650277287, -0.0706752166, -0.0220997036, -0.044650469, -0.0248094555, 0.0555914529, -0.0185303371, 0.0257904474, -0.0280368403, -0.0373562463, 0.0735041127, -1.97823727E-4, -0.0152943917, -0.0554061979, -0.068612285, -0.0230369624, -0.0371398181, 0.0612441339, 0.026026791, 0.0606609583, 1.35796785E-4, -0.0773715078, -0.0243244749, 0.0637162253, 0.0621113107, 0.0114764478, -0.0752832666, -0.0726813971, 0.0641844198, -0.0404007174, -0.033067666, -0.010883891, -0.0543704443], [-0.0492262952, 0.0610052869, -0.0496799313, -0.0140941031, -0.0230573397, -0.0367598459, 0.0277371556, 0.0391673781, -0.0152944727, 0.0169886425, 0.0650008097, -0.0246957168, -0.0348654166, -0.0557759963, -0.0282485727, -0.0794596746, 0.0369685851, -0.0420988053, 0.0520409532, -0.0847988948, 0.00561105926, 0.0497449785, 0.0372349359, -0.0267199818, 0.0273439288, -0.0121000335, 0.0680137426, -0.0592150502, 0.0174566489, -0.0667888448, -0.0197608545, -6.588830e-02, 0.0428734384, -5.243980e-02, -0.0140359951, 2.22258037E-4, -0.011747147, 0.0551943108, -0.00976604223, -0.0450481437, 0.0588170588, 0.0569472872, -0.0110507309, 0.0177220497, 0.0850509628, -0.0698927045, -0.0760451332, -0.0520863123, 0.0121601894, 0.067551367, -0.0667101145, -0.0730401874, -0.0237046909, 0.0546587594, -0.0723400787, 0.0581694357, 0.0802292153, 0.0574147925, -0.0406849347, 0.0676284805, -0.00415993249, -0.0301496107, 0.0155597217, -0.0398570895, -0.0368392691, -0.0140087679, 0.00813911668, -0.00743082119, 0.0559335463, -0.0185184423, 0.0463210493, -0.0480688848, 0.0439190455, 0.0675719529, 0.00600386504, -0.00898266863, 0.00934602413, -0.0494282171, 0.0534765199, 4.901460e-02, 0.0612379313, 0.0345855951, 0.0705928802, 0.0458090864, -0.0124118542, -0.067494221, -0.0303363577, -7.336440e-02, 7.45564175E-4, -0.076846242, 0.052699849, 0.0139964866, -0.0399169698, 0.0438860655, 0.0337630697, -0.00222638668, -0.0410510115, 0.0554352775, 0.0261583626, 0.0732858703, 0.0595834553, -0.03767341, -3.707930e-02, 0.0336372443, -0.0488644876, -0.0843675062, -0.0948529243, 0.0563940071, 0.0445414893, -0.0547949113, 0.0388551094, 0.029964909, 0.00580598367, -0.0299884733, -0.0539366417, -0.0127736684, 0.00318523427, 0.0828799605, -0.0376420505, 0.0430601351, -0.00625358708, -0.0147674037, 0.0726453959, 0.0169040468, 0.0112537984, -0.0250705127, 0.0298097208, 0.0301827062], [-0.00308881258, 0.0695325211, -0.0397851057, 0.084089756, -0.0201287903, 0.0128583796, 0.0436523594, -0.0222703312, -4.1741872E-4, 0.020849511, 0.0121214883, -0.022417441, 0.0187564306, -0.07960397, -0.00248766504, -0.0400554314, -0.0550728366, -0.0474210158, -0.0699990392, -0.027931856, 0.0458006114, -0.0123343049, -0.00773617811, 0.0794203057, 0.0161043238, -8.266670e-02, 0.054098431, 0.00688028103, -0.0505489036, 0.0460325591, -0.0104271043, -0.00357656111, -0.0548566356, -0.0413817652, -0.00293670408, 0.0603513718, -0.0242958739, 0.0265912693, -8.873350e-03, 0.0611693189, -0.0552034341, -0.016430445, -7.18652445E-5, 1.47467421E-4, 0.0604711697, 0.00588125875, 1.062220e-02, -0.0851095393, -0.00522757508, 0.0237045437, 0.05874772, -5.09075529E-4, -0.0876538828, -0.0500644259, -0.0274808593, -0.0488881506, -0.0294399653, 0.0576478764, 0.0707362294, 0.0446815863, -0.0396248251, 0.0504094027, -0.0729581118, 0.0504901856, 0.0243704133, -0.0563715138, -0.0020279591, -0.0407909863, 0.0179044902, 0.0330174267, -0.00955654121, -7.152640e-02, 0.0828177928, -0.0636481717, -0.0532494895, -0.0672832429, 0.0384165272, -0.0228574947, -0.0168129746, -0.0835550203, -0.0152449105, -0.0533902869, -0.0252386071, 0.0395402946, -0.0650667101, 0.0220388528, 0.0587795414, 0.0246873684, -0.0153314248, 0.0353658907, -0.0727988333, 0.0800319686, -0.0313036069, -5.735090e-02, -0.0580672473, 0.0564337634, 0.051191926, 0.06768208, -0.065366365, 9.28024587E-4, 0.0443486534, -0.0114215771, -0.0208389163, -0.0135380859, -0.0427278914, -0.059900593, -0.0652125403, -0.0518238917, -0.0759968459, -0.0373261422, -0.0713487267, 0.0332053639, 0.0621880516, 0.0889748558, 0.0183944535, -0.00729729048, 0.0311001893, 0.0224315636, -0.0229185056, -0.0366233774, 0.0554766543, -0.0598700456, 0.0239029322, 0.00385239976, -0.0495967641, -0.0133616272, 0.0504622348, -0.0842862203], [0.0290133413, 0.0248706527, -0.0197580028, -0.00952982251, -0.0749824122, -0.0628159642, 0.0587593727, 0.0667960271, -0.0575963892, -0.0228997674, 0.0357446223, -0.0312275346, -0.0264079664, -0.0714212731, -0.039564196, -0.0264145564, -0.0231647883, -0.0172974784, 0.0236332603, -0.0151665267, 0.0369355567, 0.0596563518, -0.0451316312, -0.0216466822, -0.050735727, 0.0432953686, -0.0759195983, 0.0529277734, 0.00700608594, 0.0279695801, -0.0489334054, 0.0544985235, -0.0613381192, 0.0806540698, 0.0413359068, -0.0564828217, 0.0401734039, 0.0595490858, -0.0450908057, 0.0346243195, 0.0607274063, -0.0101397671, 0.0517652705, 0.037116304, 0.0717494562, -0.0551731065, -0.00204574619, -0.060550306, -0.0118237389, 0.0124451062, -0.00580228306, -0.0567413978, -0.0520511754, -5.094410e-02, 0.061517369, 0.0279342122, -0.0508298166, -0.0602797419, -0.0679602697, 0.00514389062, 0.0414459072, 0.0304566808, -0.0677695349, -0.0270526689, -0.0176177099, 0.0892837643, -0.0221451726, -0.0600723922, -0.0616406128, -0.0252534915, -0.032512337, 0.00796665344, -0.0520213731, 0.0360548496, 0.042213317, 0.0270674396, 0.0115874792, 0.0180364959, 0.0477984883, -0.0907526984, 0.112217061, -0.0105487145, -0.0211780649, -0.025816774, -0.0599132255, 0.0376935713, 0.00616656058, -0.0355546549, 1.4291011E-4, -0.0335904211, 0.00470633525, -0.0523551702, 0.0765868425, 0.0197320301, 0.0231941491, -0.0276548117, 0.0822390467, 0.0115062455, 0.0166798886, -0.0755636469, 0.0122847138, 0.0930202528, -0.00211449014, -0.0531858392, -0.0526483543, -0.0609756708, -0.114791527, 0.0170810055, -0.0390890166, 0.0852998048, 0.0584182516, -0.059740413, -0.0087138908, 0.0415472053, 0.0637324303, -0.0563257448, -0.0185259543, 0.0177358296, -0.0201063305, 0.0111779273, 0.0095510371, 0.0232907552, -0.0543132797, 0.0556420423, -0.0157396682, -0.0400492102, 0.0970711484, -0.0753621459], [-0.0783534049, -0.0395043865, -0.0580418184, 0.0303248353, -0.022092415, -0.0755727887, -8.795120e-03, -4.52551758E-4, -0.0590014756, -0.00237926771, 0.0304762311, 0.0458941534, 0.00492571155, -0.0447699577, -0.00113852322, -0.0686310679, -0.0559470542, 0.0171658043, 0.0194079839, 0.0249072257, 0.0142574338, 0.0427361317, -0.0658442602, -0.0172085147, -0.032139454, -0.0375838242, -0.00201850571, 0.00525766471, 0.0561747961, 0.0238122344, 0.0241538547, 0.00697465148, 0.0191775262, -0.0614409894, 0.0551174358, 0.0187950823, -0.0467327498, -0.0340968892, -0.0138883078, 0.00593267754, 0.0762307271, -0.0513899103, 0.0484303236, 0.0345898829, -0.0590344258, -0.00383632467, 0.00877282489, 0.044328317, -0.00183795567, -0.0412906781, -0.0090897074, 0.0563935898, -0.0657803565, -0.0521461852, -0.035212636, -0.0307698585, 0.0812589079, -0.015619359, -1.971910e-02, -0.0176647771, 3.279390e-02, -0.00362554844, -0.0310336817, 7.50740815E-4, -0.0455572046, 0.093900986, -0.0261103567, 5.657040e-02, -0.0262123477, 0.0388152711, 0.0645383373, -0.0133564901, -0.00772687094, 0.0679412484, -0.0526341572, 4.997770e-02, 0.0339908749, -0.0347233489, 0.00544114225, 0.0432196558, 0.0964855477, 7.170050e-02, -0.0676284134, -0.041278895, -0.0642149076, 0.0237619225, 0.0410005711, 0.0813333392, 0.0908831954, 0.0117269568, -0.0229452513, -0.0571008027, 0.0808774084, -0.020389298, -0.00239670067, 9.71733185E-4, 0.0536362678, -0.0697735325, -0.0483795181, 0.0435828194, 0.0453895777, 0.0240770951, 0.0353520662, 0.0105928322, -0.0522756651, 0.0352892429, -0.00144652138, -0.0578127541, -0.0149459345, -0.0155028207, 0.0391707979, 0.00836787745, 5.063000e-02, 0.0578431375, 0.0564852543, -0.00869141332, 0.0625182316, 0.0174079407, 0.0114365034, -0.0427110642, -0.027649438, 0.0527648628, -0.0717440248, 0.0771168768, 0.0980275645, 0.040031679, -0.044236619, -7.818960e-02], [-0.0363733694, 0.0653201193, 0.0194947012, 0.0798139423, -0.0848847479, -0.0436701365, 0.0741791278, 0.0500419624, 0.0102449683, -0.0268694572, -0.0306303911, 0.0465982296, 0.0524706133, 0.0634910688, 0.0408102088, 0.0459917225, -0.0670217723, -0.00466180686, -0.0533601083, -0.095626995, -0.00988135859, -0.0282403138, 0.0317559093, 0.0730425566, 0.006380192, -0.0830263197, -0.055996418, 0.0734885558, -0.0578253791, 0.0260689612, -0.0423545726, 0.0393785611, 5.176830e-02, -0.044712726, 0.0427604467, -0.0635746345, 0.0647926927, 0.0266136695, 0.0655991286, -0.0823479667, 0.0281856675, -0.0251410548, -0.034519773, -0.0748459101, 0.0540626608, 0.0263120495, 0.0149667906, 0.027589798, -0.0466460772, 0.0458516367, 0.0204092264, 0.0654793233, 0.0566795208, 0.0408704244, 0.0446083024, -0.0163282529, 0.00107001932, 0.0131754233, -0.0495132916, 0.0118396757, 0.0158666149, 0.0104036285, 0.00307453447, -0.0101227667, 0.0125658382, -0.0550643951, -0.0217083748, -3.634000e-02, -0.0765419379, -0.00557932258, 0.0137927607, -0.0994917526, -0.0785282254, -0.0384217761, 0.0477852114, -0.0513979681, -8.16884392E-4, 0.0670595914, -0.00916565768, -0.0313203707, -0.0323393419, 0.0161440168, 0.0508710183, -0.0911134406, -0.0242545605, 0.0033179468, 0.0765759944, -0.0356132425, 0.0797047466, -0.0201935414, -0.00485952757, 0.0744813085, 0.0184198283, -0.0380971022, 0.024736261, -0.0220956504, 0.0782063677, 0.0151404915, -0.0818508118, -0.030681774, 0.031504415, 6.767770e-02, -0.011799179, 0.0661176369, -0.00675711734, 0.0118864495, 0.00155111344, 0.0400529243, 0.0410503186, -0.0041696243, 0.0396048315, 0.0202118885, -4.25015896E-4, 0.0375738516, -0.0024911915, -0.0957670211, 0.0267527774, -0.0104581611, -0.0745489746, -0.0567779616, 0.0277140327, -0.0895127728, -0.0651484206, 0.0698260069, 0.0838086083, -0.093522638, 0.102992728, 0.0705642775], [-0.00138278503, -0.0404644795, 0.0468829162, -0.0486908518, -0.0206404608, 0.0283956602, 0.0512842089, 0.0411951318, 0.0321347192, -0.0392103605, -0.0166117102, 0.0618745089, 3.517620e-02, 0.0418362468, -0.0144763058, -0.021908721, 0.0199777372, -0.0555493869, -0.0377779938, 0.016313538, 0.0192809366, -0.0275995433, -0.022023676, -0.0125459265, -0.0528586023, 0.0332475677, -0.0270899832, -0.0597850457, 0.0810217782, -0.0205415804, -0.0855231881, 0.0212715883, -0.0145773096, -0.0356061086, 0.0647909567, 0.0294908285, -0.0570163839, -0.00418213289, 0.0177678298, -0.0130092055, 7.112270e-02, -0.0376748666, -0.0206973478, 0.050083831, 0.00290286215, 0.0367805287, 7.47543701E-4, -0.00972866919, 0.0129430089, -0.0800685212, -0.0255010277, 0.0305920057, -0.00849269424, -0.0468730256, -0.0393519178, 0.0705648512, 0.02774054, -0.0706718192, 0.0287132058, -0.0445558652, 0.0506680682, -0.00744453445, -0.0412800089, -0.0403138138, -0.0611891225, 0.0204557385, 0.0324200578, 0.0384683311, -0.0310728978, -0.0345199108, 0.0351911671, -0.0531616621, -0.0286448598, -0.0537976436, 0.0615100451, -0.0925029218, 0.0740132555, 0.0435790792, -0.0174664054, -0.0630153119, 0.0470272116, -0.0734356046, 0.0343708843, 0.0622557513, 0.0605317689, -0.0563282818, -0.0597569309, 0.0591849908, -0.0217122864, -0.0781562775, -0.0414524078, -0.0137328301, -0.0509268083, 0.0483120121, -0.0527787879, 0.0554541759, 0.0295895394, -0.0778438225, -0.05930176, -0.0130077265, 0.0429670289, 5.102190e-02, 0.0599224642, 0.0318354219, 0.0238811374, -0.0138544645, 0.0286442861, -0.0436440483, -0.0813478156, 0.080222465, -0.0171384029, -0.0304024853, 0.0268996246, -0.0714431107, 0.0160803832, -0.0194240604, -0.00761836767, 0.0379621461, 0.0373241752, 0.0927663445, 0.0400922298, 0.0110172024, -0.0500570796, -0.0396217704, 6.029100e-02, 0.0580127649, 0.0426317528, 0.0441387445], [-0.0760664045, 0.00331258704, -0.0275066551, 0.00210279902, -0.0498387888, 0.0174253192, -0.00952018145, 0.0532295331, 3.187000e-02, 0.0329252444, 0.0985842719, -0.0392114408, -0.0276952665, 0.0540406331, 0.06386365, -0.00388927059, 0.0794827565, -0.0353319645, 0.0580967069, 0.0633811131, -0.0424022637, 0.0508161336, -1.761480e-02, 0.0894477143, 0.0851997584, -0.0265431367, 0.00425074669, 0.0696960762, -0.0660492554, -0.00524508161, -0.037925221, 0.0304240771, 0.00809714384, 0.076642625, -3.763790e-02, -0.0249965675, -0.0564511195, -0.0272019096, -6.367140e-02, 0.0632807463, -0.0159129631, -0.0172179546, 0.0551507287, 4.270270e-02, 0.0261055157, 0.0438065827, -0.0213184059, 5.470740e-03, -0.0410231277, 0.00359713216, -0.0384603292, 0.0574680679, -0.00605892809, -0.0290128943, 0.0588170178, -0.0197606422, 0.0547013357, -0.0287436172, 0.029693123, 0.0633654222, -0.0666187853, -0.0395379178, 0.0421481468, 0.0458956435, 0.0161687285, -0.0693434253, -0.0682173371, -0.0477046371, -0.0303546395, 0.0351794884, 0.0826706737, -0.0117901098, 0.0696723536, -0.0069954521, 0.0737512559, -0.0213995446, -0.00148415065, -7.119520e-02, 0.0740576163, 0.0749735609, 0.108975358, -0.00512148859, 0.00727195339, -0.0714608878, -0.056141343, -0.0415785611, 0.0786092802, -0.0399974734, -0.0484917946, -0.0409225635, -0.0435080528, -0.00770854345, 0.0295258239, -0.0510430597, -0.00894733611, 0.07583718, 0.00844134576, -0.0367574468, -0.0549340174, -0.056422893, -0.0125253452, 0.110261925, 0.0616802759, -0.0645502135, -0.0237455051, -0.0322305821, -0.042746976, -0.0758589208, -0.0835280194, 0.0557664819, -0.0401782021, 0.0533366352, 0.0681934804, -0.0278660636, 0.0341462754, -0.0213769525, 0.0531395338, -0.00736071821, -0.0113395127, -0.0141853299, -0.0310471971, -0.0173824988, -0.04539828, 0.0557003915, -0.0125186834, 0.0124024758, 0.0507601351, -0.0715795308], [-0.0669936165, 0.0429338105, -0.0238551721, -0.0535558909, -0.0142678646, 8.71388241E-4, 0.0227639712, 0.0119625321, 0.00232015876, 0.0551874079, 0.0299630091, 0.056419529, -0.00486610178, 0.0076471013, -0.00946661364, -0.0324510597, 0.017058244, -0.00570948888, -0.00890221633, 0.00189109135, 0.0656050444, 0.0336805098, 0.0334403403, 0.0321260169, 0.0622729436, -0.0482910201, 0.0471213609, -0.0196904987, -0.0510528535, 0.0280369036, -0.0555293635, 0.0858851671, -0.0371098965, 0.00290355482, 0.0907240882, 0.0790473893, -0.0162228011, 0.0783179774, 0.071363382, -0.0663298592, -0.00627375441, -0.0572629087, 0.0786589905, -0.0525136515, -0.0740946531, 0.0576488264, -0.120563641, 0.00816670246, 0.079858318, -0.00953198876, 0.0349029861, 0.0367293656, -0.0116783911, 0.0525147095, 0.0700470656, -0.0530156121, 0.0954910889, -0.0581728406, 0.0450238734, 0.0623417571, -0.0818242058, 0.0342500024, -0.0816946253, -0.0438194163, -0.0113372141, -0.0491459444, -0.0144129582, -0.0233637672, -0.020708099, 0.0229558088, 0.0701384544, -0.0335682333, -0.0119114267, -0.0454249047, -0.0583619103, 0.0286309402, -0.0502650477, -1.28108644E-4, 0.0600827634, -0.0613300055, 0.0649545789, 0.0742946044, -0.0657959953, -0.0680905506, -0.0837778523, 0.00507837208, 0.112125643, 0.0172725432, 0.0223847609, 0.0494316332, -6.732800e-02, 0.0604833588, -0.0590239093, -0.0504008345, 0.0633945242, -0.0321572348, 0.0377332568, 0.0261579957, 0.0628654361, 0.0350239612, 0.0394702703, -0.00944182556, -0.0607926845, -0.047994785, -0.0419387184, -0.0658461303, 0.0798583776, 0.0835394337, -0.0371038504, 0.0290275868, 0.0210127048, 0.0108943591, 0.0788280815, -0.00121456489, -0.00783544686, -0.0700094476, 0.0296645183, -0.0237473901, 0.0486173518, 0.0954544544, 0.00321806525, 0.0622532293, 0.0214230549, -0.015091883, 0.0913662984, -0.0766831338, 0.0926611274, 0.0516165867], [-0.0508203432, 0.0461149737, 0.0539226905, -0.00705710193, 0.00916560925, 0.0382312238, -0.0147081967, -0.0678577721, -0.025886232, 0.0676861256, 0.0474293083, 0.0558133945, 6.228620e-02, 0.0604603104, 0.00188347208, 0.0609420389, 0.0822070688, -0.0316321664, 0.0483545735, 0.0416240692, 0.0775256604, 6.415720e-02, 0.0576414019, -0.0381968692, 0.00633944711, -0.0242059212, 0.0408400781, 0.0840317234, 0.00642142817, 0.040536806, 0.0314860381, -0.0212054942, -0.0203288943, 0.021122925, -0.0153395636, 0.0987985357, -0.0220427886, 0.0493358076, 2.985780e-02, 0.0230113324, -0.0418241285, -0.00920243841, -0.0250330176, -0.0707143098, -0.0829819292, -0.0376829803, -0.0910936817, -0.0712000355, -0.00894816872, 0.0422524437, -0.0642409548, 0.015409478, 0.0295109488, -0.00510904612, 0.0172281824, -0.0326561145, -0.018290529, -0.0230596289, -0.0701172724, 0.0262012258, 0.017042907, -0.0319371335, -0.0855261832, -0.00233796029, 0.0672688708, -0.0220935978, 0.0432714298, -0.0453726947, -0.0256819706, 0.0668371767, 0.0851967409, -0.0483847223, 0.0770747885, 0.041660022, 0.0618373789, -0.0536615737, 0.0409436412, -9.65916318E-4, -0.0647903755, -0.0568512157, 0.0852219834, -0.0462568328, -0.00630183239, -0.0477140285, -0.0318702273, 0.0351557359, 0.0770313516, 0.0865634829, -0.00606885832, -0.0229669791, -0.0387618579, -0.0774711966, 0.0447477177, 0.0265780017, -2.099130e-02, 0.0394792333, 0.0887148529, 0.00619011186, 0.00155175326, 0.0167550296, -0.0424948782, 9.285420e-02, 0.0881852284, 0.0695322752, -0.0402268283, -0.0255022775, 0.0706938282, -0.00818211492, -0.0841842219, 0.0822464675, -0.0141424797, 0.0218471792, 0.0296813678, 3.198100e-02, -0.0112475157, -0.102944225, -0.0392232426, 0.00163428753, -0.0476010218, 0.039128758, 0.00736507867, 0.0278400183, 0.0347411297, -0.0172122288, 0.0368516408, 0.0209643282, -0.039242886, -0.014471883], [0.0694921166, 0.093311496, -0.00600168342, -0.0443140715, 0.0727813243, 0.101731651, -0.0729176551, -0.0665912405, -0.030169731, -0.0108456127, 0.0510109551, 0.0281462576, -0.0516387448, -0.0332593396, 0.0850594937, -0.0642983764, -0.00256487913, 0.00199641474, -0.0233585667, 0.0547343753, -8.42634123E-4, -0.0328101777, -0.0678032637, 0.051405333, 0.00970531162, 0.0363874696, 0.0345321484, 0.0446046032, -0.00927510113, 0.00743524684, 0.0562935211, 0.0180743914, 0.0689897388, 0.033868257, 0.0100188125, -0.021886874, -0.0437979437, 0.0460410342, 0.00554325199, -0.0551751591, -0.0226814765, 0.0554133095, 0.0763455331, -0.0185923837, -0.00115797401, 0.0465735383, -0.062234465, 0.0488415584, -0.0483467095, 0.0675658435, 0.0322900228, 0.0659814551, 0.0151702669, -0.0579373389, 0.0182386953, 0.0457738787, 0.0510115214, 0.0470940582, 0.0449769348, 0.0903181285, -0.0390228182, -0.0116681512, 0.0513992496, 0.0172506962, -0.0395820029, 0.0165637545, -0.0179733448, -0.0756377577, 0.0151484795, -0.00821495522, 0.0115270456, 0.0505812913, -0.0184996687, 0.0413758308, -0.0014691432, -0.0579917915, 0.041737318, -0.0781333968, 0.0141130369, 0.0465493947, 0.0258316118, 0.0697795674, 0.0726520717, -0.0821020156, 0.0321926475, 6.335320e-02, 0.102942556, -0.0137190176, -0.0174521245, 0.0206484962, 0.0506562367, 7.847480e-03, 0.0714971796, 7.653760e-02, 0.0788288414, 0.0624404103, 0.0216588471, -0.0540642478, -0.035124246, -0.0825107247, 0.0223689824, -0.00319516985, 0.0302753597, 0.0164254848, 0.0743900761, -0.0243414324, -0.0566904359, 0.0387384854, -0.0158254821, -0.0491835773, 0.0948778092, 0.0468981937, -0.0573022775, 0.0262409672, 0.0524848402, 0.0259754937, -0.0298008695, -0.0615794063, -0.0164954308, -0.00412132451, 0.00172815239, -0.0599449798, -0.0270264633, 0.0320802629, -0.0624079853, -0.0216285978, 0.0101948483, 0.04948375], [-0.0559980422, 0.0389664434, 0.0694794655, 0.0564437062, 0.0412810519, 0.0281816367, 0.0422464348, -0.0468589962, -0.0367973745, -0.0639134049, 0.0849277973, 0.0788482428, 0.00980598479, 0.0793023258, -0.00619642297, 0.0431786701, 0.0157571472, -5.149750e-02, 0.00571129145, 6.786300e-02, -0.0727146342, -0.0817075148, -0.0561721474, 0.0941373631, 0.0743231773, -0.0188599862, -0.0381131843, 0.0373376794, -0.0350836664, 0.0320798568, 0.0163116548, -0.0594784431, -0.0689672455, 0.0797096788, -0.0199512485, -0.0111047802, 0.0159531292, 0.00683051813, 0.0881878733, 0.0354903713, 0.0301662441, 0.00860303547, 0.0891043618, 0.0305853281, -0.0577334352, 0.0440350138, 0.0464309901, -0.0659069791, -0.00494983094, -0.056260556, -6.977260e-02, 0.0551773347, -0.0436077826, 2.63120528E-5, 0.0400128886, 0.0505463332, 0.0118104406, 0.079933457, 0.00452999491, -0.0351902246, -0.00926857721, -0.0380127728, -0.0787862762, 0.0015470857, -0.04618074, 0.0197938103, -0.059042342, 0.0383063182, -0.0459830798, -1.193020e-02, 0.0208560601, -0.016380446, -0.0214927271, -0.0505675822, -0.0654213801, 0.0287878215, -0.0294387396, 0.0415304303, 0.0171005353, 0.0419135913, 0.0175440721, 0.00854674167, -0.0446703658, 0.0324346125, 0.027913902, -0.0491136052, 0.0989679321, 0.00818318222, -0.0666492283, -0.0755134225, 0.0920165702, -0.0234935917, 0.00447936682, -0.0637987629, -0.0204666946, 0.0591700338, -0.0336336605, 0.05518068, 0.0723691359, 0.0543028153, -0.0606729053, 0.0522506759, 0.0631989241, 0.0394759104, -0.0763571709, -0.0644824356, -0.0232822411, 0.0314642936, 0.00602212222, 0.0821429491, -1.18429642E-4, -0.00549710728, 5.069890e-02, 0.0215555485, -0.0283501446, -0.0511825159, -0.00357100507, -0.0823956131, 0.0131542385, 0.0194246061, -0.00897268578, -0.0694614649, -0.0322035477, -0.0187270977, 6.494600e-02, -0.0122312866, 0.0713642165, -0.0271642581], [0.0113524916, 0.0750559866, -0.0463027582, -0.0212698895, -0.0383696258, -0.0491634794, 0.076040931, -0.0214257687, -0.0414669365, -0.0465279371, -0.0550642125, 0.00113316532, -0.0584363639, 0.027533412, 0.0483432151, 0.0823387653, -0.0481560044, 0.0831809789, -0.0663836077, -0.0786000564, -0.0591469668, -0.00246299757, -0.082756333, -0.022724187, 0.0774825438, -0.0467105433, 0.040483322, -0.0145264016, 0.0431806594, -0.00578491623, 0.0629955307, -0.0446474589, -0.0437776148, 0.0874727591, 0.0351227671, 0.023629969, -0.0104111061, -0.0308427829, -0.0294743627, 0.0207925607, 0.0015827918, -0.00806336571, 0.00789890997, -0.0370181464, 0.019511072, 0.0424194485, -0.00373959844, -0.0425414182, -0.0704114586, -6.924670e-02, 0.0529058091, 0.0751776546, 0.0445921905, 0.0634041652, -0.0270283949, -0.0294666104, 0.0608018786, -0.041485928, -0.0181930717, 0.0210170671, 0.0164340083, -0.0327717364, 0.00744176563, 0.0716644451, 0.00945584662, -9.238270e-03, -0.0305266399, 0.0562445521, 0.00936621986, -0.0247302316, 0.0135835018, -0.0477510244, 0.0438448377, -0.0829616934, 0.0744075105, -0.0704064369, -0.00426412886, -0.010550336, 0.0421154723, -0.0220329203, -0.0616311766, 0.0521152318, 0.0365629233, -0.048169408, 0.0469270684, -0.0267056059, -0.0319456086, -0.022001259, -0.027486939, 0.0342968181, -0.054621432, 0.00497409701, 0.0831242353, 0.00867273938, 0.00293828174, 0.0783949419, -0.0626917258, -0.0218108762, -0.0167394429, 0.0405940227, -0.01446277, -0.0349167958, 0.0927640572, 0.0774302482, -0.0841006264, 0.00433827471, 0.071111992, 0.0620366074, -1.200540e-02, 0.0793297365, 0.0434995778, -8.033950e-04, -0.0349117704, 3.803340e-02, -0.0284278039, -0.067346774, 0.0407316945, -0.0739159361, 0.01784187, 0.0835070237, -0.0655660555, 3.340990e-02, 0.0323442109, -0.00890064239, 0.0849209875, 0.0175835136, 0.031028213, -0.0129413605], [-0.0622143224, -0.0205905046, 4.164870e-02, 0.00684817927, -0.0485960245, 0.0437987968, 0.0195770022, -0.0238134973, 0.0120293265, -6.322800e-02, 0.061823342, -0.0316452384, 0.0503228717, -0.00977766327, 0.0831709951, -0.0341644846, -0.0654182434, -0.0743983313, -0.0593330637, -0.0367543101, -0.0378257707, 0.0528663769, 0.0424298756, -0.0511521772, -0.0718374252, 0.0196291339, -0.0299562179, 0.0447471961, -0.0254361182, 0.00379406195, 0.019634556, 0.0718582049, -0.0635842085, -0.00114136317, 0.0603573062, 0.0204126649, 0.0277868081, 0.0678705722, 0.0912364721, -0.0287059601, -0.0368270352, -0.0723655223, 0.0704767257, -0.0587244146, 0.0694090724, -0.0218433943, -0.0454422422, 0.0367815532, -0.0552274734, -0.0337034203, -0.0613770373, -0.0488245115, 0.0784153044, -0.0289103128, -0.03725655, -0.0687321275, 0.0205618683, 0.0664108768, 0.0225674398, 0.0310463458, -0.00501507288, -0.0314424075, 0.05874639, 0.0159698483, 0.0741068348, 0.0538008586, -0.0269721374, -7.708160e-02, 0.0422303155, -0.0072143334, -0.0052764318, -0.0168435127, -0.0118170483, 0.0445646271, -0.0471041836, 0.0693933219, -0.0109750321, -0.0734016746, 0.00888045412, 0.00567899132, -0.0558389276, 4.25898412E-4, 0.0114154676, -0.0732220486, -0.0956401899, 0.0428134762, -3.241550e-02, -0.0677220747, 0.0149656562, -0.00709382864, 6.380920e-02, -0.0206145346, 0.06001845, 0.0128845805, 0.045139797, 0.0549999811, -0.0154482778, -0.05127462, -0.0379617587, 0.0206603464, 0.070211947, 0.0643990412, -0.0721352175, 0.0563630499, -0.0270242095, 0.00276029692, -0.0599081144, 0.0297876671, 4.752130e-02, -0.0526767336, 0.0379192084, -0.0821844413, 0.0785012766, -0.0310375355, 0.0180022437, 0.0653337687, 0.0676527247, -0.028277643, -0.0720212832, 0.00273960782, 0.0801877081, -0.0234695394, 0.0736293271, -0.0191430822, -0.0505512953, 0.0312758572, 0.0168418121, -0.0656657368], [0.0550952293, -0.0915363132, 0.042123802, 0.0631063878, 0.0577106066, 0.0447796583, 0.0309474468, 0.0362086184, -0.0662637278, -0.0638795719, 0.0703000352, 0.0573470481, 0.0463915877, 0.0400567204, -0.0704348907, 0.0459752716, 4.568390e-02, -0.0533123873, -0.0786428675, 0.0299066231, 0.00625699107, -0.0154515672, 0.0235485286, 0.0667077452, -0.0115082283, 1.855670e-02, -0.00700930599, -0.0734497309, 0.0146932192, 0.0742844045, -0.0713533685, -0.0143564017, 0.0637081936, -0.0334787741, -0.0114408974, 0.0321919546, -0.0330283307, 0.00943395589, 0.0386211835, -0.040740367, 0.0164254233, -0.0443491563, 0.051498659, -0.0990353301, -0.0454619266, 0.0358431526, 0.0616184548, 0.0516052283, -0.0344309472, -0.0866906046, 0.0352786668, -0.0190473162, 0.0415079333, 0.00618965505, -0.0485731959, 0.0688055679, -0.0588891767, -0.0529601313, 0.0679477826, -0.024628602, -0.05347877, 0.0122022033, 0.0872599706, 0.00110529992, -0.0775125846, 0.0520370454, -0.0260246433, -0.0233442225, -0.045075994, -0.0693239719, 0.0694861338, 0.0393683761, 0.0148165971, -0.0708943084, 0.0212732311, -0.0631783679, -0.0303566698, -0.0310841501, 0.0077553778, -0.07016965, 0.0345038474, -0.0623920746, -0.0583276302, 0.00150394498, 0.0158050656, -0.0053853807, 0.0124819959, 0.0664709359, -0.0401057936, -0.0399561152, 0.0154508343, 0.0228903443, 0.0748745501, 0.0392449573, 0.0591423288, 0.0202409588, -0.0399172045, 0.00243236567, -0.0316990726, 0.0611084811, 0.0288193803, 0.0566934571, -0.0612745322, 0.0886191651, -0.0797087922, 0.0454457626, -0.0432453416, -0.00332523067, -0.0503672771, -0.0758158266, 0.0796085447, 0.0501572303, -0.0493881926, 0.0460514277, 0.0135286655, 0.0180859696, 0.00807600562, -0.0389681347, -0.041070722, 0.0050804317, -0.0693508238, 0.0456945561, -0.0471152812, 0.0727484301, -0.0608813427, -0.0770126209, 0.0912201106, 0.0389873274], [0.00107256637, -0.0181188025, 0.00447209366, 0.0270366706, -0.0188352074, 0.0303018484, 0.0182971507, 0.034133248, -0.0138213011, 9.92453773E-4, 0.0118500432, 0.0767094642, -0.021169113, 0.0271653887, -0.0206686836, -0.0697424859, 0.0492070206, 0.00170048827, -0.0768374279, -0.0216451082, 0.028090572, -0.085584335, -0.0151109342, 0.0243145563, -0.0168721732, -0.0740132704, -0.00147151982, 0.0174751263, -0.0398071334, -0.0141099878, -0.039839182, 0.00271090399, -0.0276050381, -0.0708603784, 0.0124819549, -0.00745621696, 0.0326611176, 0.0240124222, 0.0695374757, -0.0503177345, -0.0515353978, 0.00276821759, 0.0759038106, -0.0333932564, -0.0297923777, 0.0464409553, -0.0490471497, -0.0276532955, -0.0479172803, 0.0122546433, 0.00440619094, -0.0411110893, -0.0552743152, 0.0246781204, 0.00842746347, 0.0618012845, 0.0305978227, 0.0150764873, 0.0607186928, -0.0605452098, 0.0477844849, 0.0594298914, -0.0219026357, -0.0668659881, -0.0337111428, -0.0729608089, -0.050089173, 0.048034057, 0.0616030619, 0.00613053842, -0.044701729, -0.0623202361, 0.0551185384, -7.972860e-02, 8.371660e-02, -0.00200022641, 0.0366131552, 0.0601107739, -0.0510845818, 0.0336770825, 0.0465004556, -0.0219340753, 0.0536680296, 0.0370705165, -9.395850e-02, -0.0330430195, 0.0601150878, -0.086763896, -0.0752484873, 0.0284890402, 0.0345042087, -0.0257759877, -0.046498131, 0.00188995188, -0.0670822933, -0.00340172392, 0.0358939506, 0.0492180251, 0.0278054252, 0.0615036785, 0.0398039967, 0.0335502364, -0.00922257733, 0.0731196627, -0.0385970622, 0.036533948, 0.0850088298, 5.806620e-02, -0.080676712, 0.0385228358, -0.0614601038, 0.0458959304, 0.0170084368, -0.0169606581, -0.00722338539, -0.0837942362, 0.0675071179, -0.0509129353, 0.0149714136, -0.0262789112, -0.0420923233, 0.0034561255, 0.00149668951, 4.888680e-02, 0.00712371105, 0.0500205085, 0.0680223256, 0.0523185581], [0.0481391586, -0.0233470909, -0.0026918815, 0.0637467057, -0.0128324367, -0.0426566564, 0.0228177831, 0.0803334042, -0.0384640321, 0.034546528, -0.0461942442, -0.00210801815, 0.0452725403, 0.00233549718, -0.0197398085, -0.0147504425, 0.00129262474, 0.0275332201, -3.52068746E-4, -0.0591973588, 0.0644609481, -0.0379384272, 0.0609089099, -0.0543951169, 0.0509301871, -0.0550469048, -0.0614738502, -0.0431049727, -0.0608893894, 0.0143178264, -0.0633372217, -0.0568727404, 0.0231953915, -0.0798627361, 0.0724322349, -0.0349041186, 0.0166187901, -0.0345395431, 0.0762254595, 0.0660323948, 0.00323117292, -0.0547453277, 0.0141789066, 0.0504540317, 0.0664237365, -0.0290928446, 0.0410203561, -0.0198451541, -0.00703326566, -0.0392581858, 0.0564701669, -0.0330047384, 0.0721851662, -0.0699345842, -0.053661976, 0.0746445805, -0.00650575059, -0.0423822217, 0.0302150398, 0.0441547818, 0.0268485285, -0.0113761881, 0.0733356848, 0.019874895, 0.0439400449, 0.0154183786, 0.0176684093, 0.0339077786, -0.0102866394, -0.0655522644, 0.0469175577, -0.0660560131, -0.0473372787, -0.0926216244, 0.0292067714, 0.0719905197, -5.155750e-02, 0.0237618107, -0.00361853349, -0.0191315543, -0.0125595238, -0.050261274, 0.0609310195, -0.0139220338, -0.0494561717, 6.512910e-02, -0.0251673851, -0.00505932653, 0.052368816, -0.06854105, -0.05446757, -0.0679563284, 0.0702701807, -0.0867702737, -0.00826180074, 0.0461404324, -0.0802600979, -0.0649270564, 0.0902371332, 0.0697820038, -0.0365886204, 0.0132563477, 0.0261935201, 0.0639777258, -0.0922764614, 0.00971491448, -0.0294882134, 0.0799198448, -0.00554214558, -0.0258172676, 0.0108401738, 0.0364811905, -0.0316214785, 0.04337826, -0.00352670648, 0.0491087586, -0.00255010417, 0.0585913956, -0.011765386, 0.0672145783, 0.0076874746, 0.0689694062, -0.0405633748, 0.0182758328, 0.0677064285, 0.0112099769, 0.0318710096, 0.0277788769], [-0.0539220981, 0.0291867442, 0.0420308746, 0.0657050684, -0.0155400382, -0.00637524063, 0.0330949537, -0.0359853469, 0.0734083652, 0.0645482242, 0.0824203342, 0.00693659578, 0.0405296497, -0.0761277675, 0.0298267938, -0.00979365967, 0.0538346693, -0.0552012622, 0.03184117, -0.0187203754, -0.0664762333, -0.0212322418, 0.0672338828, -0.00821916666, -0.0385286212, 0.0521859787, 0.0440581478, -0.0579601824, 0.0397921577, 0.0473603494, -0.0151640531, -0.0144686513, -0.0725549906, 0.0640549064, -0.0499205403, -0.0414708667, -0.0642474219, 0.0917680636, -0.0591730103, -0.0354167372, -0.0136905303, -0.0652079806, 4.569400e-02, -0.00270492164, -0.0437429287, -0.0452239551, -0.0360239223, 0.00317146396, 0.0055724089, -0.0681119785, 0.0214221105, 0.0166799445, -0.0112149213, -0.0300876815, -0.0719398782, 0.0764181837, 0.0740039125, 0.0438323915, -0.00946462713, -0.0632848293, 0.069038175, -0.0570835248, -0.0571708381, -0.035466671, -0.0434763879, 0.0304919351, -0.0722186565, -0.0382601172, -0.0024216664, 0.0150508769, -0.0470134132, -0.00650829542, 0.0345070139, 0.0274993945, 0.0218461026, -0.0319270417, 0.0574350283, 0.0441950709, -0.0586702153, 0.0080490252, 0.0122393519, 0.0274244025, -0.0330739804, -0.0120518226, -0.0323851779, 0.0197481848, 0.0425233059, 0.0364348032, -0.0420620926, 0.0533945784, -0.0075918734, -0.0171697456, 0.012776209, 0.036657244, 0.0116418181, 0.0275602173, 0.00930617563, -0.047243271, 0.0818435103, 0.0366506912, -0.0253878348, -0.0513404571, -0.0104900561, -0.00582836149, -0.0228560828, -4.277800e-02, -0.00216708332, -0.0158537887, 0.0355980471, -0.0816690549, 0.0327270031, 0.0250357147, -0.0410964862, 0.036053136, -0.0479320288, 0.0653963983, 0.0396567211, -0.0670698509, 9.822320e-03, 0.0074950261, -0.0469527319, 0.0600109361, 0.0340725034, 0.0132597098, 0.0422200747, -0.00826830789, 0.00671384856, 0.0697979704], [-0.0376746841, -0.0311283711, 0.0588992871, -0.0262696762, 0.0207561366, -0.0654473677, -0.0290360358, -0.0487497523, -0.0759749785, 0.0634316579, -0.00144774595, 0.0236325804, -0.0716601312, 0.030850213, 0.0441704541, 0.0334377177, -0.017054379, -6.873270e-02, -0.0200607963, 0.0228336211, 0.0291652512, 0.00108027353, 0.0429738536, -4.62149706E-4, 0.0561354831, -0.00642356789, -0.0645869598, 0.0106962593, 0.0257300138, -0.0393010192, 0.00226372341, -0.0307314582, -0.0200357046, 0.0682987124, -0.0489178337, -0.0442983881, -0.0760229453, 0.0798774734, 0.0207366534, -0.0115918666, 0.0477330796, 0.0682688206, -0.0633473471, 0.0284050368, 0.0335043333, -0.071059458, -0.00588012114, -0.0414035544, -0.0820870847, -0.0388476923, -0.0289074183, 0.0199123062, 0.0293683633, 0.0109058311, -0.0578111894, 0.0793384984, -0.00744517427, -0.0525423288, -0.0340226479, 0.0559359938, 0.0635421574, -0.00249749469, 0.0709782913, -0.0542992614, -0.0626335815, -0.0634333789, -0.0521152243, -0.0134618441, 0.0430193208, 0.0193700343, -0.055472672, -0.0647503361, -0.0091952635, 0.0124453427, -0.00344052445, -0.0520779304, -0.0170851927, 0.074915655, -0.0596741214, -0.0231152177, 0.0204692911, -7.552340e-02, 0.0111500658, 0.0194223169, -2.150080e-02, 0.00642363355, -0.0439919122, -0.0758523196, -0.0522854142, 0.0828952714, 0.0724472404, -0.0407516472, -0.021941524, -0.0129059386, -0.00661067339, 0.0564161651, 0.00499557424, 0.0751955658, -0.0390266292, 0.0104806283, -0.0375691392, 0.041420389, 0.0712654367, -0.0784915164, -0.0580188632, -0.00760741392, 0.0750261694, -0.0125988368, 0.0198126696, 0.0539025255, -0.020343326, 0.0526115373, -0.0622706823, -0.0682416931, 0.0538425557, 0.0613555312, -6.219890e-02, -0.00148368266, 0.0572583601, 0.0678826496, 0.00379153783, -0.0673702732, 0.0661038384, -0.0568480752, 0.0473573916, -0.0229763817, -0.0459407642, -0.069368355], [-0.0175257213, 0.0374632105, 0.0438785702, 0.0462195389, 0.0426176414, 0.023810124, -0.0677766204, -0.0620326064, -0.0437828265, 6.281080e-02, 0.00460743392, -0.00226534251, -0.0732267275, 0.0758258104, 6.84699335E-4, 6.874410e-02, 0.0625750199, -7.398050e-02, -0.0798915401, -0.0434844121, -0.0325212516, -0.0276529379, -0.00684692151, -0.0100992993, -0.0353882052, -0.0223670714, 0.0325328372, 0.0288952962, 0.059787333, -0.05097799, -0.0368887447, -3.429520e-02, -0.0102172792, -0.0575660765, 0.0373249426, 0.0396339707, 5.055950e-02, -0.0528187454, 0.032185372, -0.0590925589, 0.0063538081, -0.0397084244, 0.0606127083, -0.0243607815, -0.0107880123, -0.00189768069, 0.0732840151, 0.0154651431, -0.0223579202, 0.0306008477, 0.0185031295, 0.0661051497, -0.0516071469, -0.0290536378, 0.0767459199, 0.00988164451, 0.00578465033, 0.0502852872, 0.00133244018, 0.00755586475, 0.0189070664, -0.0233247951, 0.0796517729, -0.0499860868, -0.0671289414, 0.0213564467, 0.0261008404, 0.0272489227, -0.035466779, -0.03257671, -0.03982649, -0.0373381414, -0.00506158033, 0.0711713061, -0.0567920804, 0.0118179182, 0.0241228044, -0.069524467, 0.0356990881, -0.0329109654, 0.0143180666, 0.0377663597, 0.00749994814, -0.021188736, 0.0564614497, 0.0620666519, -0.0670944154, -0.0267152525, -0.0565728769, 0.0403322317, 0.0163403861, 0.0794685557, -0.0744051933, -7.955030e-02, -0.00636634696, -0.0188971572, -0.0507342815, -0.0226269681, -0.0665561929, 0.0235943347, 0.0399619192, -0.0733011216, 0.0572171025, 0.0455536693, 0.0327515565, 0.0262634754, 4.648500e-02, -0.0502187088, 0.0720190331, -0.0765624642, 0.0246337876, 2.24744334E-4, 0.0430457629, -0.0340771489, 0.0537165105, -0.0308808908, -0.0398862697, 0.00764288101, 0.0782796218, 0.0260225367, 0.0561749786, -0.065017648, -0.0243080165, 0.0134784458, 0.0586100742, 0.0414637476, 0.0482722893, -0.0828545913], [-0.0745359957, -0.0488269664, 0.0339544378, 0.0616054236, 0.0597354621, 0.00648239581, 0.031662792, 0.0633752868, 0.0573887266, -0.0124627221, -0.0426742807, 0.0751256794, 0.041564364, 0.0378137305, 0.0112625202, -0.0202821027, 0.074181065, -0.0571531653, -0.00914472714, -0.073719196, 0.0222606957, -0.0209109802, -0.0803267359, -5.44381852E-4, 0.019264685, 0.0265840702, -0.028480513, 5.375800e-02, 0.0546720698, -0.0194141679, 0.0699121431, -0.069226034, 0.00258959015, -0.008988888, 0.0098431576, -0.00353615521, -0.0717939138, -0.0488486141, 0.0316190384, 0.0434749164, -0.02979194, 0.00729980366, 0.0403725281, -0.0495326221, -4.66894926E-5, 5.324630e-02, -0.0271161608, -0.0174495503, 0.0516628735, 0.0447115675, -1.25843027E-4, -0.0689065307, -0.0237634853, -0.0577818528, -0.0147308176, 0.0271338113, -0.0563433245, -0.00665051303, 0.07544972, 0.0310150366, -0.0615920089, -6.360730e-02, -0.0522757433, -0.0174535159, -0.0337799527, 0.0138513967, -0.00293502142, -0.00132393104, -0.0220651384, 0.0161315911, -0.054828234, -0.0241448581, -0.0626589507, -0.0584219098, -0.0460683554, 0.0775043517, 0.0482971109, -0.0722400099, 0.0623655468, 0.0625468269, 0.0199479647, 0.00636064867, -0.0166715402, -0.00218802807, 0.016852811, 0.0529994331, -0.060697604, 0.0711658224, -0.0298673268, -0.0416248962, -0.0456414595, -0.04923198, -0.064733617, 0.0619133673, 0.0784555599, 0.0183681604, -0.0249417946, -0.0285884943, 0.00245301495, -0.0156463888, -0.0380657725, 0.0139964251, -0.044142317, 0.00988885295, 0.0470661595, -0.0220339429, -8.41084169E-4, 0.0739747882, 0.0687211528, 0.0445231795, -0.0398001261, -0.0667707548, -0.0015013701, -0.0360351503, -0.0684746727, -0.0749085173, -0.0625234321, -0.0328643806, 0.00652038864, 0.0594404154, 0.0556061491, 0.0455884226, -0.0384683572, 0.0149141792, -0.0780672878, -0.0210214406, -0.020365553, 0.0791186541], [-0.00156234193, -0.0256422199, -0.0735050216, 0.00745427888, -0.0696224496, 0.0653095692, -0.0528025217, -0.0702405572, 0.0503249951, 0.0596768446, 0.0636443197, -0.0726805404, -0.0796120464, -0.0618820749, -0.0687776655, 0.0236038864, 0.0609705672, 0.0424145758, -0.0567271151, 0.0344569236, 0.0113179563, -0.0177680235, -0.0359918624, -0.0551190823, 0.00482403953, -0.0222170874, 0.0469498225, -9.80522134E-4, 0.00846833736, -3.609640e-02, 0.0130823646, -0.0695906729, 0.0386707261, 0.0586977527, 0.0431582369, 0.0745559484, -0.0604789779, -0.0600716285, -0.0317322388, 0.00583526492, -0.00610553054, 0.0768248588, -3.726570e-02, 7.9054333E-4, -0.0371144265, 0.0291292183, -0.0041540321, 0.0530433059, 0.0663823113, 0.0718825832, 0.0366031341, -0.0207338817, 0.0485026538, -0.00557361031, -0.0666242391, -0.015532135, -0.0531217679, 0.0114033539, 0.0420055352, 0.0588799752, 0.0174601823, -0.0660039261, -0.0274889972, 0.0588862523, 0.0573766269, 0.0650484487, -0.0476878509, -0.0159265473, -2.55600055E-4, 0.0206472538, 0.00369855226, -0.0201352891, -0.0476605818, 0.0470908433, 0.0382790267, 0.0340289585, -0.0363061912, -0.0298625473, -0.0443165191, 0.0232430585, 0.0395475514, -0.0140826572, 0.0309283379, 0.0638803318, 0.0575945154, -0.00580685074, -0.0522001088, -0.0706741363, -0.0348849632, -0.0145985764, -0.0365063027, -0.0659504682, -0.0663823411, 0.0454754531, -0.00150404486, 0.0430555865, -0.0257435627, 0.0124909272, 0.0724302083, 0.0111450078, 0.0159701705, 0.0727002844, -0.0619291476, -0.0458791032, -0.0237128511, -4.359910e-03, 0.0151844611, -0.0433336236, -0.0629859567, -0.0360454358, 0.0543224439, -0.0435334705, -0.0596357211, 0.00522985402, 0.0048185396, 0.0670753494, -0.0798547044, 5.213520e-02, -0.00623455318, -0.0614488348, -0.0690855309, 0.0640345513, 0.0601986498, 0.00286990125, -0.0379905701, -0.0793049633, 0.00344354962, 0.0538680516], [0.0432057753, -0.0413145237, -0.00772569887, 0.05442084, 0.0141748851, -0.0491907261, 0.0498118326, 0.0484059714, 0.0674329177, -0.0644524172, 0.00696738111, 0.0760741159, -0.0396667197, 0.0756852776, 0.04003888, -0.0428176261, 0.0437832624, -0.00523717701, 0.00372296572, 0.0494970307, 0.0688438565, 0.0702762901, -0.0644720569, -0.0039397548, 0.0195740648, 0.00260347174, -0.0394354649, 0.054492671, 0.0598927662, 0.0666699558, 0.0243204683, -0.0375479348, -0.0598976165, 0.00764787477, -0.0342529528, 0.00216552918, -0.0687248558, -6.471360e-02, 0.0637951046, 0.0139082782, -0.0151233524, 0.00567734521, 0.0696482137, -0.0486989915, -1.942130e-02, -0.0662607402, 0.051049754, 0.0737653449, -0.0588745177, 0.0540182963, -0.0275218822, -0.0772964954, -0.0449814238, 0.0593168698, 0.0798578038, 0.0789214447, 0.0131570147, -0.0614416823, 0.0695922151, 0.0308817625, 0.025634272, 5.332110e-02, -0.011029778, -0.0787049159, 0.0489271171, 0.068805404, -0.0634124503, -0.0620905049, 0.0521306023, 0.0753329396, 0.0533282459, 0.0576798804, -0.0646300763, 0.059574347, 0.0595513657, -5.242410e-02, -0.0126276268, 0.0355107896, 0.0232743975, 4.63768694E-4, 0.0341442227, 0.0710971579, 0.00313185854, -0.0336738192, -0.0653892457, 0.0367296971, 0.0450181738, -0.0257186927, -0.0181937143, -0.0768267736, -0.0439696349, -0.039034225, -0.0399271548, -0.0298924558, -5.72577119E-5, -0.0362330042, -0.0301004909, 0.00927550718, 0.0317634232, 0.013107135, 0.0342352539, -0.03656644, -0.0492969789, 0.0305085108, -0.00928801764, -0.0771825909, -0.0781019404, 0.0555491932, -0.0730153099, -0.020175036, -2.15261214E-4, 0.00784181803, -0.0508458465, -0.0746204406, -0.080603078, -0.0504031368, -0.0492235869, -0.0719712078, -0.0590308271, 5.862590e-03, 0.0663447082, -0.0445180312, 0.0158152208, 0.0724841654, 0.0316207036, 0.0698157623, 0.0548016764, -0.0723452345], [-0.0742076486, 0.0477702245, -0.0394377932, 5.25556505E-4, -5.056480e-02, 0.0351221636, 0.0774937942, 0.05075147, -0.0310545042, 0.0730139539, -0.0490629673, -0.0494809859, 0.00392930955, 0.0296580642, 0.0119748563, 0.0384214669, -0.016223073, -0.0687203854, 0.0470070913, 0.0192594603, -0.0398436263, -0.0613388047, 0.0239243358, -0.066041939, 0.0241792351, -0.0646202862, -8.14333558E-4, -0.0681763291, 0.0254272372, 0.0276628956, -0.0492166504, 0.0243189186, -0.0441721752, 0.0267134234, -0.0665250644, -0.0582399592, -0.0706911385, 0.0760324076, -0.0465678461, -0.0546204858, 0.0332694948, -0.0467426442, -0.047205545, -0.0373287797, 0.0190706402, 0.0629907623, 0.0727893785, 0.00895477086, 0.0531702712, 0.0692107901, 0.0625596568, -0.0606325492, 0.00169852376, 0.0527186021, 0.036488004, 0.0407758653, -0.00481382757, -0.0677716509, -0.0301737227, -0.0290872604, -0.0611721091, -0.069879666, -0.00286348164, 0.0742516294, 0.0722732618, 0.0717593506, 0.0342524201, 0.0742950216, 0.0197478831, 0.0287540033, -0.0598272681, -0.0369878821, 0.0550071374, -0.0274541974, -0.055507049, 0.0676354095, 0.0652298853, 0.0298400819, 0.00850312411, 0.0578754917, -0.0364901908, 0.0356522277, -0.0283196829, 1.837980e-02, 0.0343253836, 0.0634674355, -0.0594954975, 0.0646673515, 0.0490926132, -0.077932477, 0.0793489292, -0.0286482014, -3.51533294E-4, -9.63725149E-4, -0.0132840797, 0.0290469378, -0.0560364202, -0.0389739834, 0.0424413979, 0.0449583307, 0.0758878365, -7.257370e-02, -0.0481398366, 0.046299912, -0.0114493743, -0.0486819446, 0.0128244087, -0.0205001049, 0.0708503798, -0.0699036792, 0.0387505516, 0.015911825, -0.017436415, 0.00233951211, 0.00726801157, -6.436320e-02, 0.0262904763, 0.0409284309, 0.0153748393, 0.0489014909, -0.0559491068, 7.019500e-02, -9.564720e-03, 0.0195064694, 0.013500765, 0.0217333883, 0.0414265096, -0.0337143913], [-0.0244701616, -0.0555093326, 0.0454501137, -0.0577359833, 0.0129435509, -0.0359978601, -0.035867285, 0.00610786676, 0.0130216405, -0.0374497771, 0.0480714366, 0.0630477145, -0.00133380294, 0.0525073931, 0.0451453254, 0.0496083423, 0.0499304906, -0.0440776497, -0.0587732308, -0.00182265043, -0.0772175416, 0.0654166862, -0.0434391573, -0.055714976, 0.00386096537, -0.0660704225, -0.0613906309, 0.0166269317, -0.0327725597, -0.0122500807, -0.0470851436, 0.0752110258, 0.0086934939, -0.0205329023, 0.0510641113, -0.0574866347, -0.0639601499, -0.0342789739, 0.017024897, -0.0701064244, -0.0684183091, 0.0358111486, 0.00582254678, -0.0478773192, 0.0366381705, 0.0638453588, 0.0641449317, -0.0677666441, -0.0488034487, 0.0648023859, 0.0105309039, -0.0632596761, -0.0148753524, -0.0574699454, 0.0221795216, -0.0319537371, -0.0528916791, -0.0267301723, 7.57619738E-4, 7.39112496E-4, 0.044088982, 0.0631059632, -0.0631520823, -0.0494539887, -0.0166143626, 0.0746220872, -0.0800415351, 0.0234340355, -0.0544426143, -0.0280399173, 0.00705182552, -0.00179111212, -0.0600063577, 0.00167801976, -0.0648154616, 0.0226471797, 0.048329778, 0.028338328, -0.0305513404, -0.0390932038, -0.00442371517, 0.0274245739, 0.0581887439, -0.0645596981, 0.0320284218, 0.0527296886, -0.0192283429, -0.00791988522, 0.00822351872, -0.0763933435, -0.0239108391, -0.0393881314, -0.00924896449, -0.0557552576, 0.0384232327, -0.0274201445, 0.069589071, -0.0242637247, -0.054610312, 0.028672643, -0.0652077049, -0.050708428, -0.0514863096, -0.0429362841, 0.00873526186, -0.0683500245, 0.0688780919, -0.053575445, -0.0794593543, 0.0460984334, -0.00578710437, 0.0184625462, 0.0118312091, -0.0305037126, 0.0801699534, 0.0190629065, 0.0413760319, -0.0309347808, 0.0633975044, -0.0595081076, -7.338610e-02, -0.0559220687, -0.070876278, 0.0511218235, -0.0369236805, 0.079041548, 0.0783588811, 0.0806486532], [-6.233550e-02, -0.0147994682, 0.030363664, -0.0616292283, 0.0314525068, 0.069979839, 0.0527930781, 0.0656972304, 0.0562254861, -0.062133532, 0.00148607045, -0.0756983458, -0.0629211068, -0.0647819117, -0.0340765417, 0.0530432537, -0.0298230238, 0.0468569919, -0.0479625426, 0.0227075368, 0.0406731665, -0.00422974676, -6.172530e-02, -0.0360555053, 0.0559946522, 0.0800395384, -0.0128412694, 0.0788078084, 0.0633565113, -0.0153485984, -0.0794496089, -0.0416996405, -0.0631484091, 0.0560045615, -0.0560323186, -0.00486342609, -0.0276269466, -0.0206433833, 0.0405858681, -0.0776686295, 0.0429212973, -0.00684861838, 0.074098058, 0.0732253864, 0.0319404155, -6.956290e-02, 0.0145057365, 8.94844532E-4, 0.0220795423, 0.0597513691, -0.0269098245, 0.0789393857, 0.0327812582, 0.077915959, -0.0329001136, 0.0430025384, 0.0290193632, 0.0471571162, 0.0104220659, -0.047738526, 0.0623970404, -7.154420e-03, -0.0196614228, 0.0113268644, 0.0444848165, -0.0315660052, 0.0533294603, -0.0250518769, -0.0195196345, 0.00997726619, 0.0524967685, 9.865880e-03, -0.074966386, -0.0377524234, -0.0538625792, 0.0395764336, 0.0127473846, -0.0521593019, -0.0194918066, 0.077227138, -0.0656691343, 0.0148075894, -0.0683978274, -0.00656444579, -7.480200e-02, -0.0783587545, 0.0712306872, 0.0586712584, -0.06418778, -0.0207283758, -0.0307254642, -0.0626831055, -0.0411531962, 0.0624777749, -0.065706186, 0.0481770262, 0.051254563, -0.0567924306, -0.0599847585, -0.0743682757, 0.023315683, 0.0210299343, 0.0626498386, 0.036197722, -0.0506788939, -0.0582069084, 0.0603935048, 0.0149315074, 0.0177104026, -0.0643479228, -0.0603249557, 0.00365062803, -0.0741453618, 0.0468525365, 0.0749509558, -0.0444570296, -0.064042896, -0.0112759322, 0.0151057243, -0.0332126804, -0.0507035144, 0.0193275884, -0.0726000592, 0.00554020703, -0.0123819858, -0.00869869441, -0.00246149302, 6.85058534E-4], [0.0178855136, 0.008268876, 0.0456484742, -0.0506376848, -0.0738391727, 0.0355637111, -0.0198470894, -0.0356599726, 0.0266645402, -0.025299754, -0.0598256774, 0.032608185, -0.03177635, 0.0510842428, -0.0406785682, -0.0749389753, 0.0445737317, 0.00136825745, 0.0261844601, -0.00295328884, -0.00958746671, -0.0254931692, -0.0225393921, -2.079460e-02, 6.321730e-03, 0.0356699266, -0.0688665956, -0.0066724522, -1.34572387E-4, -0.0807785987, 0.0355989225, -0.0301341284, 0.0690048933, 0.00977767817, 4.078150e-02, -0.0420071632, -0.042137675, -0.0576970428, 3.4225767E-4, 0.0177576989, 0.0388980098, 0.0638776794, 0.0752635673, -0.0294821169, 0.0715845525, 0.00567118637, 0.0520907864, -0.0126341153, -0.0301445629, 0.0314123407, 0.0677518546, -0.00937319174, -4.05348721E-4, -0.0755666345, 0.0624812767, -0.0289734714, 0.0314469635, 0.0262992661, -0.0186596867, -0.0291960947, -0.0760427862, 0.0255134162, -0.0742972642, 0.0512610525, 0.0340312198, 3.403840e-02, -0.00307888933, -0.0369095802, 0.0175749399, -0.0251243096, 0.0684777275, 0.0642921776, -0.0571219251, -0.0429630838, -0.0562966913, 0.0608871505, 0.0606888197, 0.053849645, -0.0424463302, -0.0679689422, 4.691740e-02, -0.0284275506, -0.0760339573, 0.0507583693, -0.0108741969, 0.0237073582, 0.026040921, -0.0409256965, 0.0352943279, -0.00765725086, 0.0729128048, -0.0109276082, -0.00139447104, -0.035896156, -0.0524060577, 0.07170856, 0.0190524813, 0.00322697405, -3.27290094E-4, 0.00793799292, 0.00462389225, 0.00718709081, 0.0274343546, 0.0750956908, 0.0400943235, 0.0320971236, -0.0435470417, 0.0526309609, 0.0777945592, -0.00682502752, 0.0351688303, -0.0340469927, -0.0683238953, -0.039965719, 0.0615752861, 0.0773504898, 0.0177113935, -0.0294841416, -0.0634419173, -0.00752571272, -0.0260677673, 0.0796500071, -0.00933638494, 0.0767207816, 0.0320010111, -0.00783326477, 0.0660440847, -0.0777606442], [-0.0313651524, -0.0254307389, -0.0473963171, -0.0735764727, -0.0813680589, 0.0245977212, -0.057638552, 0.0625386611, 0.0769823343, 0.00388684287, -0.0486955158, -0.00542361941, -0.0479114465, -0.0163173154, 0.0239784513, 0.0789642333, 0.0311898086, -0.00280950661, 0.03859777, -0.0630479231, -0.0395775326, 0.0316301808, 0.0027968036, -0.0379295573, 0.0253414791, -0.0679520518, 0.0426150896, 8.041950e-02, -0.0137048792, -0.073015064, -0.0674333945, -0.046244815, -0.0126230903, -0.049556978, 0.0380662829, 0.0609928481, 0.0466788672, -0.066583477, -7.551890e-02, 0.00429478521, -0.0358969346, -0.0381041169, 0.0737105832, 0.0170243662, 0.0538466685, 0.0430797487, 0.00356204808, 0.0790655091, 0.0759028196, 0.00599075342, 0.0340156518, 0.00386823365, 0.029550029, -0.0740310624, -0.0690915584, 0.041803997, -0.0621330515, -0.0434523597, -0.0548566729, 0.00594174815, -0.0255326871, -7.837320e-02, 0.0168738849, 0.0659563243, 0.043092642, -0.00981997326, 0.0442339219, -0.0528359823, -0.0075128153, 0.0415672734, 0.0212677121, 3.324650e-02, -1.68898609E-4, -0.0348641761, 0.00382730365, -0.00455978792, -4.786690e-02, 0.0370472707, 0.0241960175, 0.0228064731, -0.00650565652, -0.00277564838, 0.0661395192, 0.0735254139, 3.520000e-02, -0.0691235811, 0.0797338262, 0.0758900121, 0.0105242562, -0.0286466256, -0.0732611119, -0.0809130222, 0.0457575731, -0.0633655787, 0.0771883279, -7.444110e-02, 0.00767597649, 0.014073479, 0.0547250584, -0.0715973601, 0.0423921458, 6.10496616E-4, 0.00904322509, -0.0468563214, -0.0384028852, 0.00477551855, 0.0737311617, 0.01286963, 0.0477710478, 0.0476105958, -0.0779262706, -0.0548246801, -0.0284286365, -0.0344365202, -0.00174492714, -0.0336955637, -0.0551698953, 0.0392940529, 0.0401851796, -0.0248070341, 0.0326563418, -0.0266377795, 0.0745031535, -0.0507477969, -0.0422661789, -0.0651249886, 0.0125875631, 0.0449308977], [0.0575653054, -0.0447319299, -0.0200415459, -0.00194094097, -0.0156998616, 0.0487489253, 0.0440061763, 0.0591104068, -0.0262719784, -0.00435288763, -0.0402467325, -0.0751922428, -0.0763818175, -0.0135537861, -0.026161585, 0.0277840253, -0.00362373423, 0.0526694432, -6.787060e-02, 0.0528899394, -0.0408972092, -0.0484023839, -0.0247342736, -0.0778035596, -0.0526108742, -0.0234890319, 0.0510423221, -0.0564443395, 0.0630284324, -0.0248388853, -2.88512208E-4, 0.00381564302, 0.0574467406, 0.0625982582, -0.0128743481, -0.0050971196, 0.0351096392, -0.029623583, -0.040401496, -0.0738896206, 0.0498775057, 0.0726392046, 0.020781111, -0.0570601486, -0.0604918972, 0.0292718634, -0.0383968949, -0.0091562625, 0.0169464815, -0.0751580297, 0.0192380939, 0.0452318117, 0.00564426882, 0.0734318271, -0.00755605474, -0.0263459533, 0.0243948773, 0.0740730539, -0.0807620138, 0.0151822362, 0.00863355119, -0.0221762955, 0.0228078049, -0.0736334845, 0.044200357, 0.0278081223, -0.0553353131, -0.06152023, -0.0333197154, 0.0214745644, 0.0242986586, 0.0451997705, -0.00687442673, 0.0692515448, 0.00358504802, 0.0407073423, 0.0646995902, 0.0153675303, 0.0372735411, 0.0057905321, -0.0289833024, 0.0443366729, -0.0184211414, 0.00459795631, -0.0228151754, -0.0273181126, 0.0491830446, 0.0323846154, -0.042689018, -0.0103930347, -0.0375822708, -0.0457531065, 0.0376119427, -0.0541240051, 0.069247365, -0.0208096039, -0.0170479268, 0.0478506535, -0.0366104171, 0.064376995, 0.0379454717, -0.0496938638, 0.0668484047, -0.0650422126, -0.00430010818, -0.0801920444, -0.0577455387, -0.0725716725, -5.567140e-02, 0.05937903, 0.0791350156, 0.00759322196, 0.032378044, 0.0198272392, 0.0710505694, -0.0198825281, 0.0714308545, -0.0398811363, 0.0127509367, 0.0182517096, 0.0801494941, 0.05014975, -0.0741338059, -0.0712972134, -0.0264179073, 0.0416314751, -0.0130346641, 0.0378734171], [0.0481833667, 0.0380132422, -0.0740017071, 0.0283124913, -0.0785147473, -0.0661063269, -0.0197377596, 0.0407967679, 0.0525911227, 0.0449697971, 0.0020932483, 0.00124085438, 0.0147762448, 0.0725038797, 0.0767953693, -0.0172279198, -0.0554739349, 0.0188568383, 0.0108208498, -0.0117943771, -0.0417003222, 0.0152460989, -0.0137925679, -0.0674195886, 0.0756036788, -0.0411595255, 0.0459163524, 0.0292887632, 0.0724830404, 0.0196315423, -0.0624499917, -0.0627486333, 0.0454460382, -0.0566858165, 0.0142146843, 0.0455261096, 0.00279224594, 0.0716793388, -0.0774367824, -0.0231307615, 0.0574726947, 0.0671109259, 0.00549829425, -0.0471197292, 0.0564701781, -0.0517728925, 0.0375107452, -0.0565852225, -0.0130554074, -0.0202224124, -0.0757855177, 0.0245967694, -0.01826947, -0.0156756528, 0.0106279608, -0.0209658295, 0.0762409642, 0.0173413493, 0.00305678509, -0.00712223258, -0.0433447547, 0.0625446662, 0.0583340302, 0.0374714434, 0.0798484981, 0.0414684191, 0.0421516597, 0.020767739, 0.0578025952, 0.0232406817, 0.0310113057, -0.0612183101, -0.00816223677, 0.0364453271, -0.0789488777, 0.0530083217, -0.0475517027, 0.0633519143, 0.0417546257, 0.0196284484, -0.0729194507, -0.02090385, 0.0304800048, -0.0506775565, 0.070912376, 0.0663476735, 0.0443897694, -0.0477089658, 0.044478178, 0.078220807, 0.0617165342, -0.067597568, -0.0728450269, -0.00522876438, 0.0110494178, 0.0801887214, -0.0326731615, 0.0292940605, 0.0299020074, -0.00572727947, -0.0106901554, -0.0476423278, 0.0812475383, 0.0443237796, 9.417580e-03, 0.00655894401, -0.021433061, 0.0161195602, 0.0191094819, 0.0302814618, 0.0558502488, -0.08038906, -0.0115237636, 0.070542708, 0.0347599387, 0.0442697071, 6.406720e-02, 0.038029559, 0.0106128994, 0.0317838117, -0.0377061702, -0.0603652038, 3.888990e-02, -0.0311861113, -0.0228419583, 0.0240343679, -0.00812313426, -0.0584690496], [0.0114734517, 0.07138668, 0.0471385643, 0.0485119596, -0.0211265385, 0.00776376482, -0.0120987026, 0.00327369198, 0.0667231604, 8.047210e-03, -8.05102347E-4, -0.0186694469, -0.0538124964, 0.0379383564, -3.35012883E-4, 0.0172286108, -0.0464397967, -0.0111858062, -0.00668625347, -0.0780189558, 0.0602624938, -0.00580890058, 0.0108418725, 0.0670715272, -0.062549457, 0.0508246981, -0.0355969071, 0.0619679577, 0.0565790758, 0.0155077856, 0.0736386403, -0.0655348748, -0.0768638402, 0.068899475, 0.0492423773, -0.0132208643, -0.0162826981, -0.0659434795, 0.0138484659, -0.0538825244, -0.00116559525, -0.0100537054, 0.0475255698, -0.0584863089, 0.0155695928, -0.00794652198, -0.0206968468, 0.0449659638, -0.0283586364, -0.0473005809, -0.0150292525, -0.062254481, -0.0517057888, 0.0667571649, 0.0584651977, -0.065074712, 3.84022511E-4, -0.0597287454, 0.0802575275, -0.00302178203, 0.016233664, -0.00695290649, -0.0663688704, -6.674570e-02, 0.0338934213, -0.0352546945, 0.028091833, -0.0621702708, -0.0174852293, -0.0632061437, -0.0429057777, 0.021917114, 0.0387521312, -0.0470979363, -0.0574395359, -0.0516422354, 0.05326562, -0.0274672024, 0.0354252718, 0.0245164931, 0.0496435128, 0.0255224071, -0.0806293711, 0.0492991917, -0.0317019597, 0.0782073959, 0.0333285704, -0.0563159473, 6.161670e-02, 0.00473482115, 0.0410414077, -0.0639118105, -0.0447725244, -0.0543380976, -0.0460998341, 0.00727613829, 0.023951143, 0.0497177318, -0.00556170847, -0.0736141801, -0.0479869843, 0.0947841405, -0.0665410087, -0.0248879474, 0.00149911235, 0.0357691199, -0.0577842556, 0.041258052, 0.0508092195, 0.0569283254, 0.0276303347, -0.059150666, -0.0309162084, -0.0300307535, 0.00645627128, -0.0819371939, -0.0525317267, -0.015840916, -0.0486046821, 0.0720829069, -0.0301382467, -0.00511462707, -0.0204817839, -0.042695418, 0.0809028521, -0.031921234, 0.0396733023, -0.0572928414], [-0.0637308061, -0.0503176637, 0.0752320439, 0.0296691675, 0.044046849, -0.00144122471, 0.0348491371, 0.0130402921, -0.0576260202, 0.0095827356, 0.00363133755, 0.0409593806, -0.0554434285, -0.0246663988, -0.0743598267, 0.036609944, 0.0292634387, -9.454980e-05, -0.0478527099, 0.0204324108, 0.0141536584, 0.0612347312, -0.0145516936, 0.0307371467, -0.00855138339, 0.0157087073, 0.0179376844, -0.0276516024, -0.0277520157, 6.327160e-02, 7.301010e-02, -0.0189791042, 0.0117254974, 0.0402463116, -0.0759846419, 0.0582909286, 0.0876125842, -0.0303157941, -4.637500e-02, 0.0735849738, -0.0314766392, 0.0659513771, 0.0494136624, 0.0402426422, 0.00149678602, -0.0279464778, -0.062839292, 0.0536478758, 0.0697853714, -0.0638900101, -0.0631964654, -0.0321563557, 0.0484084077, 0.0356613435, 0.0794110298, -0.0166054424, -0.013315049, 0.010105026, 0.0619095489, -0.0288475193, 0.0672269836, 0.0415836684, 0.0394877605, -0.0814029276, 0.0484729484, 0.00142332714, 0.0377020128, -0.0696443319, 0.0643949956, -8.03044357E-4, 0.040982563, 0.0478224643, 0.0234404989, -0.00665661506, -0.0756238252, -0.0302547812, 2.367880e-02, 0.0513702631, -0.0170970485, -0.0194824226, 0.0160077792, -0.0428991579, -0.0477768965, -0.0230826195, -0.0131473541, 0.0210440196, 0.0356689356, 0.0694945231, -5.918440e-02, -0.0516456217, 0.0523349829, -0.0589534082, 0.0420725048, 0.0503570884, 0.00124670891, -0.0534798913, 0.0863519608, 0.0154762622, -0.0127843861, -1.737450e-02, -0.0408351794, -0.0048781205, -0.0319655873, -0.0495369472, -0.0540864505, 0.0535167232, -0.0810705497, -0.0713495538, 0.00440422725, -0.0409848318, -0.0807884932, 0.0456234477, 0.0549435541, -0.00905952416, 6.860830e-02, 0.0362200737, -0.0421755277, -0.0463307686, 0.0269743875, 0.0293835532, 0.00878522917, -0.0154644409, 0.0744330734, 0.0222235825, -0.0427744053, -0.0402325243, 0.0416463315, 0.0424755663], [0.0260460079, -0.0548074618, -0.0738147795, 0.0148106646, -0.107459441, 0.0392013751, 0.0602419786, 0.0675301924, -0.0400159843, 0.0342120677, -0.0241999961, -0.0551576242, 0.0126690622, -0.0701243505, 0.0540885031, -0.0357531495, -0.0263400674, 0.0683230087, -0.0767370239, 0.0125067458, 0.00708607538, -0.0154767567, 0.0209427699, -0.003824502, -0.013506203, 0.00547570223, 0.0384355634, -0.0137456926, 0.0508113056, 0.0173824634, -0.00696676876, 0.0585305281, -0.0197483469, 0.0542667843, 0.0706869215, -0.0343940556, -0.00684463652, -0.0486979112, -0.0310069826, -0.0389242955, 0.0724984407, -0.0756140053, 0.0299888495, 0.0588326789, -0.00273539755, 6.350500e-02, -0.0141007723, 0.0478663556, -0.089547649, -0.0286417231, 0.00727977371, -0.026274886, 0.0506711788, -0.0369400531, 0.0498444885, -0.00947023741, 0.048890233, 0.0484792702, 0.0351200551, -0.0599572845, -4.434000e-02, -0.0517107397, -0.0644366145, 0.0750677958, -8.63964728E-4, 0.0398714468, -0.0724256113, -0.0459670797, 0.00520100072, 0.0607796125, 0.0316633508, -0.0290412232, -0.0631669387, 7.318030e-02, -0.0236316808, -0.0447133593, -0.0338603258, -0.0111538311, 0.00867885817, -0.0549625568, 0.108741224, -0.0629672557, -0.0257137604, 0.0670652166, -0.0106001534, -0.0309202727, -0.0610140227, -0.018750703, 0.00807350687, -0.0452485904, -0.0782491341, -0.07000795, -0.0341355577, -0.0246160645, 2.164650e-02, -0.0253946539, -5.323590e-02, -0.036959812, -0.0422072634, 0.00222270074, 0.0674713105, 0.0730416924, 0.0756307468, -0.0680663287, 0.044272922, -0.00149119576, 0.0317829624, 0.0316615887, 0.0677655637, -0.0497497953, -0.0213621482, 0.0553585887, 0.0582911968, 0.0758414119, -0.0218043048, -0.0292747151, -0.00856686197, 0.0464283526, -0.0200578347, -0.0216103904, -0.0290549155, 0.00608731341, -0.0356301889, 0.052026324, 0.0142974658, 0.00394729851, -0.0191226024, -0.0812747926], [0.0493990891, -0.0362568535, -0.0648872256, 0.0821589902, -0.103180684, -0.0139602972, -0.0178016629, 0.0233516693, -0.0138993254, 0.0943850576, -0.00283537083, -0.0376859307, 0.043605417, -0.0631822645, 0.0287701115, 0.0399611741, -0.0332298838, 0.0627577677, -0.0655113086, -0.0180928092, -0.0736967474, -0.0543703884, 0.032239072, -0.0231771898, -0.0100372629, -0.0219405405, -0.0690227151, 0.0617110096, -0.0623044446, 0.0745737702, 0.0163387097, -0.0226792898, 0.0304275155, 0.0628100708, -0.0088799782, -0.0543350168, 0.0536747761, -0.00961476564, 0.00325729605, -0.0695297122, 0.0715331733, -0.00855558831, 0.0673315078, 0.0205396116, 0.063339524, -0.0521872342, -0.103065483, -0.0318717547, 0.0493969209, -0.0742838084, -0.0554155894, -0.0548998155, 0.0326561183, 0.00102741667, 0.0835247635, 0.0345042907, 0.0408558957, -0.0720516667, -0.0219056923, -0.0321307741, -0.0554786697, 0.00477792602, -0.0820292159, 0.0711307525, -0.0656809583, 0.0407542065, 0.0143370517, 0.0378446244, -0.0608845241, 0.0502085835, -0.0251564961, 0.0439470038, 0.00577419531, -0.0192263909, -0.0610987693, -0.0287988149, 0.026173288, 0.0466209464, 0.0580165163, 9.87283536E-4, 0.106867328, 0.0218513906, -0.0551575385, -0.0502013899, -0.0861419513, 0.00715020578, 0.0566282645, -0.0603278428, 0.0607059747, -0.0661127791, -0.083727695, 0.0672965795, 0.0442736708, 0.0584649146, 0.0279007889, 0.0484603234, 0.0284288712, -0.0750189871, -0.0517524667, 0.0775253549, 0.045897264, 0.0125077656, -0.00817963294, -0.0149904275, 0.033212658, -0.0480634421, 0.0260509681, 0.0593376346, 0.0448358618, -0.0168304592, -0.0259740837, -0.00732371537, -0.0372598208, -0.0191411283, 0.0310225058, 0.00988008454, -0.0548099279, -0.0686297417, -0.0588258877, 0.027934758, -5.925140e-02, -0.0381553918, -0.0598613881, 0.0629820079, 0.0472138338, -0.0464632027, -0.046728313, -0.0308890641], [0.0271680858, 0.00501340395, -0.0275218915, -0.038846612, -0.0843339264, -0.0608110204, 0.0893313512, 0.0715850443, 0.0597282834, 0.0957924649, 0.0622335151, 0.0760659873, -0.0540639907, 0.0836764797, 0.0363183133, 0.0409905873, -0.0052637076, 0.044352226, -0.0105318036, 0.024692867, -0.0265658386, 0.0175677482, 0.0141182728, 0.0263317637, 0.0764232054, 0.0734830722, -4.689230e-02, -2.960520e-04, 0.017137723, 0.0671114251, -0.0248927083, -0.0731081814, 0.0567461327, -0.0069727255, -0.00505551277, -0.00303114695, -0.0588947684, 0.0983068123, -0.0619139075, -2.290630e-02, 0.0537258573, -0.053602308, 0.0383632481, -0.0183941517, 0.00520443544, 0.0926944091, 0.0240500197, -0.0711576864, -0.00803335756, 0.0123967612, 0.0600772686, 0.076214239, -0.0686448812, 0.0613630414, 0.0680552721, -0.0344849937, 0.0169004612, 0.0547436364, 0.0797889828, -0.0618528314, -0.0599190667, -0.051868014, 0.0556807406, -0.0307761971, -0.0159435291, 0.024091525, -0.0396636277, -0.0312036108, 0.00152154069, -0.00638112612, 0.0525274165, -0.0280309413, 0.0719465688, -0.0274603292, -0.0283768252, 0.0317166671, 0.052598577, -0.0624936707, 0.06460426, 0.065013364, 2.538160e-02, 0.0141660674, 0.0497109033, -0.0515646227, -0.0573463701, 0.05458498, 0.0362580977, 0.0156943984, 0.0113305533, -0.0427758917, 0.0391053036, 0.038483046, 9.847050e-02, -0.0257001612, -0.0626604259, 0.0169720203, 0.079540342, 0.0824561864, 0.0146353822, 6.812400e-02, 0.042159982, 0.110131435, 0.036356166, 0.0280689877, 0.0322029442, -0.0887160971, -0.0692083612, -3.503050e-02, -0.0230802968, 0.0289266091, -0.0129358023, -0.0764619634, -0.0208435375, 0.0608832836, 0.0567292795, -0.11183583, 0.0474827774, -0.0674569085, -0.0528274179, 0.0305541921, 7.586010e-02, 0.0225044843, 0.0425291806, 0.00638190378, 0.0917213782, 0.0228285957, 0.0997006818, 4.534290e-02], [-0.0475507081, 0.0169565454, 0.0016620605, 0.0970227122, -0.118120529, -0.0101390621, 0.0925057455, 0.0600410551, 0.0165232103, 0.0318862945, 0.0286961179, -0.0400535651, 5.05244971E-5, 0.0392590798, -0.0515001677, -0.0160262436, -0.0769023076, 0.00256349589, -0.0247120839, -0.0747567788, -0.0319027901, 0.0414982811, -0.0827751905, 0.0979619398, 0.0153998276, -0.054323975, -0.0532659888, -0.00966220535, 0.0747439638, 0.0780063271, 0.0371857844, -0.0157821272, -0.0537048951, -0.00413700566, -0.0831352546, -0.0568808056, -0.0239134878, -0.0216273274, 0.064433977, -0.0799491405, 0.010125584, 0.0107511962, 0.045988854, -0.0976141169, 0.0741421506, -0.0341799743, -0.0755224228, -0.0165177137, -0.0463444181, -0.00358705688, 0.00870320573, 0.0920278877, -5.793900e-02, 0.0406475961, -0.027351005, 0.0459076762, 0.0358701311, 0.00190906331, -0.0436235629, 0.0825711786, -0.0160378516, 0.0719374716, -0.072938107, -0.0514530763, 0.026622219, -0.0590440854, -0.0388336405, 0.0609964877, 0.0269097462, -0.0295919962, 0.0243600924, -0.0755436942, 0.00582059333, -0.0351771191, -0.0525587946, -0.0680972114, 0.0441887528, 0.0280863196, -0.0273389462, -0.0237712078, 0.0140639832, -0.0255263671, -0.0654589683, -0.0728387162, -0.0714045838, -8.215530e-02, 0.0172939394, 0.00670122495, -0.049972605, -0.0604513809, -0.0582550205, -0.0405527353, 0.0976936519, 0.0769613832, -0.0401925929, -0.0332181491, 0.0426255353, -0.0658243448, -0.0499055684, 0.054608956, 0.0625512823, 0.0066736103, -0.0217648707, -0.0778956264, -0.00728003634, 0.0382916592, -0.057475429, -0.0751455128, 0.0520942919, -0.0237095393, 0.0598222092, 0.075080432, 0.0424411371, 0.0147491321, 0.0796607062, -6.494660e-02, 0.0120224804, 0.0479838476, 0.0169871189, -3.787520e-02, -0.0484854914, -0.0848639607, -0.0420180708, 0.0722726956, -0.0343424045, -0.0596043542, 0.0210741535, 0.062344823], [0.0685468614, 0.0152007416, 0.0169974715, 0.103782453, -0.00820580124, 0.0640831217, 0.0704517066, 0.0121641951, 0.0200242493, -0.0241309796, -0.0180663224, 0.0342517681, 0.0519513302, 0.041101221, -0.0794703513, -0.0146900304, 0.0336338431, -0.0616552755, -0.0658499599, -0.105315976, -0.0695868284, 0.0098665012, -0.0402383395, -0.00846108142, 0.0121770389, -0.0248625726, -0.0308524072, 0.0786878988, -0.0418495424, 0.00187358703, -0.0452135429, -0.0512134582, 0.058413893, 0.059150368, 0.0409992151, 0.0330605246, 0.0150576942, 0.0246820878, -0.0161920805, -0.0345490947, 0.0233612638, -0.0557954349, 0.01341365, 0.0177454837, 0.0395050123, 0.0838093534, -0.0386497714, 0.00134332827, -0.006871772, 0.0445019864, -0.0247935839, -0.0602755956, -0.0175991524, -0.0128025012, -0.0374675952, 0.0798471197, 0.0494740792, -0.0779045075, 0.078741625, 0.0568485968, 0.0204515029, 0.0766284167, -0.0251199398, -0.0202047359, 0.0507696532, 0.0333969295, -0.0160149205, 0.0537104085, -0.0803867802, 0.0453523025, 0.0834706649, 0.0126007674, -0.0655166208, 0.0662680343, -0.00665859878, 0.0052058138, 9.6904383E-5, -0.0253342669, -0.00377341965, -3.295200e-02, 0.0110231116, -0.0314138681, -0.0558175817, 0.0320825055, 0.00207750127, -5.31436177E-4, 0.026159944, 0.0346814729, -0.0212738831, -0.0194687862, -5.180980e-02, 0.0564525835, -0.0185064543, -0.0672338232, 0.058687117, -0.0143329566, -0.0396135077, 0.0707971081, -0.0722266957, -0.0400679186, 0.0430740714, 0.0648381859, -0.0383136384, 3.096340e-03, -0.0271627363, 0.0506944507, -0.0385476202, -0.0418573692, -0.0346653089, -0.0476103127, -0.0608057305, -0.0246836934, 0.0099267112, 0.0390864052, 0.0365740173, 0.0111300936, -0.0523074046, -0.0702946335, -0.0573142171, 0.0247561187, -0.00321625778, -0.0672730729, 0.0400630496, -0.0480454229, -0.0664790124, 0.0383783765, -0.0185520481, 0.044653669], [0.0366236046, -0.0504958406, 0.0922235324, 0.101349659, -0.100134403, -0.0211126432, -0.0223222263, 0.0260089189, -0.0368543342, 0.0828447863, 0.0512831658, -0.0812371894, 0.0464623831, -0.0199348312, -0.0337949544, -0.0303242225, 0.072081834, 0.0059422818, -0.0389175266, -0.100186601, -0.0842633768, -0.0587653667, -0.0496522449, 0.0602064468, -0.0673905909, 0.00335071119, 0.0482155643, 0.0972167477, -0.0186720788, 0.0262306258, 0.0247932635, -0.0421713293, 0.038649831, -0.0432894565, -0.0017688158, -0.0149959205, 6.457520e-02, 0.0963540673, 6.294030e-02, -0.0430674739, 0.078860104, 0.0532555655, 0.0595777556, 0.0483573787, -0.0214297324, -0.0224917307, -0.0175403208, -0.0522878878, -0.0790694505, -0.0769028142, -0.0150686242, -0.0507970713, 0.0551809333, 0.0502147935, 0.10318175, 1.347350e-03, 0.0777616352, -0.0694969296, -0.0433792174, 0.0513134301, -0.0707351416, -0.0400754139, -0.0324470252, 0.046040412, -0.00382763054, -0.0441965237, -0.0794579088, -0.0717658177, -0.0672943518, 0.100098148, -0.0463713333, 0.0195712708, -8.366680e-02, -0.0669580996, -0.0593848974, -0.090790078, -0.0218863077, 0.0247488823, 0.0615036711, -0.0616354905, 0.00694633508, -0.0380156785, 0.0217576176, 0.0161233973, -0.0123800701, -0.0569914207, 0.0770742372, 0.0477305576, 0.060791105, 0.0912611782, 0.0276052672, -0.0174701512, -0.0571823791, 0.0759795457, 0.0182013214, 0.018229153, -0.00497590657, 0.0186048392, 0.0386282094, 0.0241749547, 0.0500145108, -0.00929435715, 0.0598217696, 0.0444167592, -0.0509931035, -0.00374166272, -0.0426563211, -0.0745019168, 0.076037012, 0.0827201232, 0.022155229, 0.0464452207, 0.0167052615, 0.0469916351, -0.0107403481, -0.111712337, -0.0126640843, -0.0699570701, 0.022186242, 0.00500369444, 0.090486966, -0.0268125497, -6.720680e-03, -0.0508772135, -0.040443942, 0.027967697, 0.0151342535, -0.0781730115], [0.0559573099, -0.0141157443, -0.0487281978, 0.0448830537, -0.0948343798, -0.0646308362, 0.0152131282, -0.0415964872, 0.0394022949, -0.022875404, -0.00305565866, -0.0500457101, 0.0633518397, 0.0650695935, -0.0135615962, 0.00218819734, -0.0547499061, -0.0760476664, -0.0319688246, -0.0156630483, -0.0213570222, 0.0105985878, 0.0273398869, 0.0629971698, 0.0558992252, 0.0259879641, 0.0155303944, 0.0390054435, 0.0681610778, 0.00278931065, 0.00761978421, -0.0226767268, -0.0217974931, 0.014426427, -0.0445623435, 0.0711641386, 0.0123444963, 0.0833984166, 0.0642250106, 0.0119429836, 0.0507748239, 0.0249785185, 0.0144872637, -0.0301959589, 0.0436215624, 0.0573840812, -0.0900139064, 0.0335499272, -0.0551676825, -0.0694054738, 0.0343058668, 0.00810677838, 0.0743012428, 0.0716388673, 0.0959544629, 0.0953122302, 0.0724565163, 0.0300548393, 0.0233471338, 0.0714494735, -0.0302121565, -0.0130705899, 0.0482708439, -0.00412998255, 0.0357116349, -0.03873704, -0.0425787084, -0.020344872, -0.028939778, 0.0140996501, -0.0484813452, 0.0521145053, -7.129800e-02, 0.0663562343, -0.0564764962, -0.0686773285, -0.0709473416, -0.0151193524, 0.0694915056, 0.0288127419, 0.0311835427, 0.026599301, 0.0698334872, -0.00579267461, 0.00632166537, 0.0308392532, -0.038617745, 0.0107864449, 0.0658204928, -0.0386069901, -0.045253627, 0.0257421546, 0.045089215, -0.0227385517, -0.0480424911, 0.0359384902, 0.0182356108, 0.0252658017, 0.0605230257, 0.0550077111, -0.0667417124, -0.0117228944, 0.01132116, 0.012952731, 0.0472898521, 0.00589140551, -0.0367833972, 0.076632373, 0.0554380305, -2.60881585E-4, 0.0146700917, 0.0523449183, -0.0633326694, -1.716290e-02, 0.0747847557, -0.0891642943, 0.076997526, 0.00530935265, 0.0676748306, 0.00687615201, -0.0401852392, -0.0710845291, -0.0575511716, -0.049592156, -0.0589531586, 0.0659004747, 0.0284728948, -0.013308485], [-0.0121818511, -0.0618929267, 0.0978085622, -0.0114430757, -0.0681681484, -0.0251265075, -0.0697522461, -0.0615180656, 0.0552572682, -0.0570140779, 0.0287174769, -0.0326570123, -0.0823157802, 0.0808762609, 0.0381500497, 0.010364702, 0.0567823462, -0.0725942478, 0.083232671, -0.00964313652, -0.0707032755, 0.068835482, 0.0271641053, -0.031665042, -0.0392538607, -0.0168745592, -0.0225406792, -0.0589379147, 0.0348818488, 0.0577882119, 0.0630543679, 0.0273158774, -0.0918563902, -0.0430706963, -0.0174199678, 0.046192456, 0.0235337578, 0.00399098778, -0.011054663, -4.004800e-03, -0.0429495908, -0.0287255421, -0.033875078, -0.0795891657, -0.0335423611, -0.0218877755, -9.121470e-02, 0.0412141271, -1.297740e-02, -0.0443236418, 0.0569426939, -0.0325618722, -0.0107664317, -0.0439268686, -0.0225783195, 0.056331683, 0.0686617941, 0.0147676636, 0.0234680418, -0.0390960276, 0.06902612, -0.00743223773, -0.0335123539, 0.0708261505, -0.0554091744, 0.0114497477, -0.034416426, -0.00278904475, -0.0775604173, -0.00398793677, 0.0813365951, -0.064751111, -0.0709418431, -0.0379654355, -0.00915728603, -0.0396023616, -0.0219520163, 4.366860e-02, 0.0511026867, 0.07679943, -0.00346845901, -0.0635683536, -0.0655921549, -0.0296696071, -0.00733294105, 0.0638309717, 0.00730048725, -0.0356985815, 0.0468344428, -0.0143424012, -0.0652675182, -0.0719713718, -0.0142301032, 0.055019062, 0.0603111051, 0.0356722698, 0.0623695366, -0.0335041136, 0.0194128659, 0.0188624561, -0.0764374956, 7.20998331E-4, -0.0357855745, -0.0443315767, -0.0705696642, -0.0423549749, 0.0283290856, -0.0151551422, 0.0402078554, 0.0882777869, -0.0414475873, 6.36598794E-4, -0.00734706642, 0.0486505292, 0.0561170168, 0.0177007541, -0.00157667906, -0.040316578, -0.0583651327, -0.0313853659, -0.0052425554, -0.0162335709, 7.33346737E-4, 0.0431857593, 0.0831185206, -0.0516314358, -0.00798409804, 0.0920872092], [0.0677549168, -0.0715951249, -0.0114227748, 0.0672936737, -0.0408951864, 0.0759347975, 0.0143365292, 0.0194038134, 0.0496501885, 0.0772219896, -0.0102306353, -0.0319812596, -0.0321625024, 0.0363497138, -0.0590089187, 0.0227638297, 0.0773382261, -0.0259111356, -0.016072223, -0.0828136355, 0.0221544541, 0.0800596401, 0.0152457561, 0.0934978947, 0.0488511175, 0.0581669919, 0.0496425517, 0.0248475838, -0.0569419451, 0.0268602744, 0.0263858587, 0.05192063, -7.915230e-02, 0.034207955, -0.088077344, 0.0205763225, 0.0432278663, -0.0138233872, 0.0130989868, 1.68950311E-4, -0.0186137371, 0.0403313152, -0.0443946198, -0.0882832631, -0.0264441017, 0.0538918599, -0.0831503868, -0.0239952393, 0.0126190195, 0.0230702851, -0.0263282266, 0.0555136055, 0.00518293818, 0.0946517884, 0.00315373973, -0.00710335467, 0.0246160533, 0.0388901532, -0.0546673685, -0.0335923806, 0.0472547151, 0.0442477353, 0.0643091425, -0.0587255098, 0.0719841868, 0.085775949, -0.0145422732, 0.07199727, 0.0207334701, 0.0104236575, -0.00637914706, 0.012822995, -0.0465764701, 0.0637702346, -0.0809542909, 0.0144746741, 0.0532048829, 0.0762615725, 0.0211157147, 0.0716769174, -0.0562629439, -0.0846507474, -0.0772730708, -0.0243301578, 0.0191604644, 0.0356321894, -0.0173438396, -0.01916546, -0.0396699458, -0.0162636377, 0.0270245094, 0.0200837757, -0.042819865, -0.0677728354, -0.0708012655, 0.0438437797, 0.0903396606, -0.0126130031, 0.0149721336, -0.00509499526, 0.0754443929, 0.0700564831, 0.00645971159, -0.0454374067, -0.0705827176, 0.00792214554, 0.0268316101, 0.0440792143, 0.0132766487, -0.0224920306, -0.0487289317, 0.0494572632, 0.0790629238, 0.0378437042, 0.07763388, -0.0331812911, -0.0404579379, -0.0657056272, 0.0537568033, 0.0496474728, -0.0368224047, -0.0897059292, -0.00773539254, -0.0310677961, -0.0329806432, -0.0496740639, 0.0696149692, -0.0235367604], [-0.0679823905, 0.0041871476, 0.00872881617, 0.0576735847, -0.0153316213, 0.042403765, 0.0954757109, -0.0687880367, -0.0404344499, -0.00203309907, 0.0468574688, 0.053583473, -0.0458967425, 0.0630620494, -0.0466644689, 0.0231955536, -0.0778687149, 0.0663576052, -0.0376847088, -0.0467130281, -0.0485372245, 0.035111282, -0.0672635362, 0.085478574, -0.0838037654, 0.0500381961, -0.00520969974, -0.0634618253, 0.0581635311, 0.0525101461, 7.625040e-02, 0.0783502086, -0.0148598608, -0.0925326868, -0.0369332135, -0.0668654218, 0.0227385703, 0.0692371279, 0.00859095714, 0.0208793879, 0.0657146648, 0.0584515892, 0.0211880356, -0.0659870654, 0.0513263196, 0.0179131255, 0.0367577709, -0.0299350899, 0.0101176398, 0.0139562404, -1.397100e-02, 0.0690478235, -0.00267510884, -0.0109830489, -2.354720e-03, -0.0367197581, -0.0467533246, -0.0750295817, 0.081143029, -0.04697413, 0.0682261586, 0.0505877398, -0.0143261123, 0.0369015671, -0.0265877675, -0.0489167422, -0.00218664017, 0.00850115343, 0.0615635328, 0.0384007022, -0.0093458714, 0.0229851846, 0.0351926871, 0.0195654165, 0.0344980508, -0.096348226, 0.0500900894, -0.0575515702, 0.0054585957, 0.0908798649, 0.0465656705, -7.853530e-02, 0.0148615958, -0.0623533651, -0.088256374, 0.0642867163, -0.0529471412, -0.0556932911, -0.0678706095, 0.0730253085, -0.0295474343, -0.0472166911, 3.735720e-02, -0.0398602895, 0.0175403263, 0.0582531616, -0.0105508454, -0.0621385053, -0.078335613, -7.336950e-02, -0.0571937077, -0.0266988855, -0.0112241041, 0.0469697081, 0.00223419536, -0.0833455771, 0.0654700175, -0.0352714285, -0.0497025177, 0.0592746325, 0.0514670499, 0.0538053028, 0.0168047417, 0.0462167598, 0.00399886211, -0.0795493573, -5.369480e-02, 0.0241544414, 0.0553553291, 0.0310334321, 0.096891433, -0.0723345577, 0.0824476331, -0.00118202867, 0.0192739107, -0.029386811, -7.157860e-02, -0.00491741858], [-0.0743162333, -0.0459168255, -0.0390986688, -0.0061998032, -0.0783596709, -0.0097526852, 0.0277692489, 0.0395400748, 0.044906795, 0.062841706, -0.0243307855, 0.00241085351, -0.0529260039, 0.0416174605, 0.0139948241, 0.0685278177, -0.0545213968, -0.0402634516, -0.0350856893, -0.00589423114, -0.0550471097, 0.0716700554, 0.0416417718, 0.0433304235, -0.00678588636, -0.0591394342, -0.0360270515, -0.0106744962, 0.0682098791, 0.0444843173, 0.00811051391, -0.063370794, -0.0910380408, -0.013794912, 0.0307556968, 0.039700456, 0.0404493921, 0.00822317786, 0.0811415166, -0.0572372265, 0.0500337668, -0.00353384088, 0.0914953425, 0.0247649103, -0.031716872, 0.0478193872, -0.00833118614, -0.00268168957, -0.0599772222, -0.0478071235, 0.0331965946, -0.0697485656, -0.0495641902, 0.0743878335, -0.0706686527, -0.0109092128, 0.012995596, -0.0465956368, -0.0652567595, 1.57155038E-4, -2.495360e-03, 0.0649247617, 0.043135725, 0.0386059619, -0.00884123053, 0.0718078613, 0.0914123281, -0.0477977172, 0.0132227931, 0.0142849963, -0.0669960603, -0.0335033312, -3.787180e-02, -0.0907490253, -0.00126515736, -5.742090e-02, -0.0828329399, 0.017308617, -0.0206619836, 0.0800929367, 0.0723686442, -0.0960658044, -0.0684884787, 0.082010895, -0.0257462785, -0.0494938195, 0.0175893623, 0.0679028854, 0.00187641766, 0.0645445958, 0.0211859737, -0.0506891124, 7.75537745E-4, -0.00518526323, 0.0218232758, -0.0423017628, 0.0595129803, -0.0312260408, 7.30169587E-4, -0.0493164435, -0.0594367646, 0.0918901264, -0.0415963791, -0.0319980383, 0.0176472925, 0.0700411871, 0.0471466854, -0.0582597442, 0.0267206244, -0.0659214556, -0.0301131196, 0.0462653376, 0.05067458, -0.0486710779, -0.0124795362, -0.0769035444, 0.059429463, -0.0199294686, 0.0356562063, 0.0174857192, 0.0740058422, -0.065638341, -0.0231177174, -6.620930e-02, -0.0665579364, -0.00913472659, -0.0136319641, -0.0385348238], [-0.0100777578, -0.0618078336, 0.0885729491, 0.0795256943, 0.0157388244, 0.0648121387, 0.0924678668, -0.0527895577, -0.00645209569, -6.525530e-02, 6.882620e-02, -0.0498328581, -0.0600653589, 0.04428811, 0.0696519241, -0.0388788246, 3.919670e-02, -0.0165351685, -0.0193650015, -0.0417478755, -0.0290278383, -0.0272727031, 0.0171131436, 0.0314704776, 0.0822175219, 0.0490211882, -0.021732809, 0.0109546296, 0.0619942136, 0.0742765069, -0.0292549785, 0.063398473, -0.0714597553, -0.0726676732, -0.0222378038, 0.00549298106, -0.0622976683, -0.0160775091, -0.0413693376, 0.00911305472, 0.0437451452, -0.0706259906, -0.0634711385, -0.103412285, 0.0352573283, 0.0512164161, -0.00884698611, -0.0427354798, 0.0272835214, -0.0800044238, -0.01844492, -0.0125306798, -0.0206152815, 0.00895808637, 0.0458471887, -0.0475456491, 0.075799033, -0.0451661535, -0.0316164829, -0.0558720417, 0.0521158688, 0.0514940917, -0.013716978, -0.064489685, 0.00146384968, 0.0899918526, 0.0790256783, -0.022448875, 0.0070815403, -8.909990e-03, -0.0416437425, 0.0115823196, -0.07864023, -0.0889808535, 0.0525884479, 0.0608315915, -0.0553787351, 0.0275382306, 0.0201626513, -0.040785715, -0.061962612, -0.0961599797, 0.0104983589, -0.0486269519, 0.0342854783, -0.0578746088, -0.0439486876, -0.0181671679, -0.0112749981, 0.0877902061, 0.0718036145, -0.0613675825, -0.0262003206, 5.201540e-02, 0.0116637601, 0.0188657306, -0.0713977739, -0.00706218276, 0.0729474723, -0.049647186, -0.0744866803, -0.00843180623, -0.00129836053, -0.0764769465, 0.0166520718, -0.00246379594, 0.00467453152, -0.0621731356, -0.0547302924, 0.0345567651, 0.0107304677, -0.0616520047, 0.0758319199, -0.0652698353, -0.0470193401, -0.00684925821, 0.074562937, 0.0450172126, -0.0835381522, 0.0278115924, 0.00549150398, 0.0298234355, 0.0747372881, 0.0498334207, -7.225660e-02, 0.06793084, -0.00514156045, -0.0819578245], [-0.0453978516, -0.0786847993, -7.017100e-02, -0.0247854274, -0.0382648185, -0.0310173016, 7.887730e-02, 0.0173648298, 0.0272933636, -0.0567966811, -0.00204748684, -0.0770025477, -0.0577414073, -0.0584374778, -0.0411687717, 0.0125597361, -0.0756211876, -0.0272543896, 0.0372586772, 0.0535733737, -0.0296728276, -0.0148208458, 0.0810001045, 0.0270088688, -0.0501647629, 0.0197293554, 0.0265563764, 0.0267656185, 0.0759147331, -0.0481180809, 0.07726942, 0.0221339371, -0.0545784459, 0.00838019606, -0.0242852867, 0.0638143197, -0.0719020218, 0.0677703619, 0.0252103955, 0.0148514649, -0.0188490301, -0.0150888339, -0.0697684288, -0.0580488443, 0.0468746759, 0.0220243335, 0.0824421122, 0.00156802533, 0.0214420706, -0.0822186991, 0.0243867617, -0.0463560037, 0.0516845286, -1.202680e-02, -1.81699666E-4, 0.0265548583, -0.0784882232, 6.499130e-02, -0.0580706112, 0.0830270648, -0.0494585335, -0.0550227873, 0.0444416963, 0.0649789497, 0.0456621051, 0.0429809801, 0.0407997556, -0.0430686958, -0.00269673462, -0.0255716648, -0.0243199524, 0.0268587377, -7.201200e-02, -0.0537713617, 0.0390498228, -0.0201878827, -0.0132045578, -0.0139255514, -0.0772602186, -0.022104783, -0.049728971, -0.08114928, -0.0149555113, 0.0374593288, -0.0844596773, -0.0643891841, -0.0737234429, 0.0443672687, 0.0236903857, 0.072066009, -0.0582177229, 0.00871285423, 0.00964455865, 0.0565596707, 0.0556376055, 0.0950677171, 0.0273964982, 0.0773867518, 0.0831970125, 0.0697983876, -0.0771572887, -0.0258925091, 0.00898200459, -0.0718352348, -0.0299409274, 0.0648547262, -0.0102648269, -0.0135847908, 0.0656304881, -0.0823119655, 0.0353402048, 0.0173362419, -0.0645242855, -0.0904272422, -0.0661201477, -0.0371128209, 0.0690504462, -0.00177743658, -0.0345199853, 1.814250e-02, 0.0820106491, 0.0285286754, -0.0701598823, 0.0156877562, -0.0622707158, -0.0314060822, -0.032102108, -0.0532316715], [-0.0789338499, -0.0377206542, -0.0679823309, 0.0656776354, -5.491930e-02, -0.0342833437, 0.088348858, 0.0673417076, 0.0622027032, 0.0342699438, 0.0778468847, -0.0677035748, 0.0798864141, 0.0744472295, -0.0328836255, -0.0344901904, -0.0430011526, -0.0200358573, -0.0745861456, -0.0243591778, 0.0269700028, -0.0509022437, 0.0190973841, -0.0834789127, 0.0202696174, 0.0586752966, 0.047385551, 0.0171745215, -0.0736426339, 0.065158993, 0.0339231677, 0.0251151174, 0.0088108927, 0.0703737885, 0.0222027432, -0.0461368486, 0.080718927, -0.0597743541, 0.0398027897, -0.0197426546, 0.0502851419, 0.071066685, 6.041530e-02, 0.0189273916, -0.047744032, -0.0507510081, -0.0273925271, 0.0502921455, 0.0531412214, 0.0168002099, 0.0511607453, 0.0524243079, 0.0540133268, 0.046364747, -7.849260e-02, -0.00869514141, 0.0738810152, 0.0221183393, 0.0374025255, -0.0568333939, -0.0436045676, -0.0289342217, -1.95288172E-4, 0.0752050504, -0.0477959327, 0.065026477, -0.00322694704, -0.0808428898, -0.034773957, -0.0398007557, 0.0384339169, 0.0848299265, 0.0282844491, -0.0471853353, 0.0768390745, -2.622050e-02, -0.0250752568, 0.0291971639, 0.0712772608, 0.0319135599, 0.0174897369, -0.0269216094, -0.0413317159, 0.0142001873, -0.0719327182, -0.0628918111, 0.0316708498, 0.0117226318, -0.0688170269, 0.0649198815, -0.023888126, 0.00271848566, -0.0389584042, -0.0881794542, -0.0168624781, -0.015003942, -0.00585042639, -0.0721175075, 0.0337972231, -0.0486691482, 0.0472619906, 0.00509270933, 0.0670770258, -0.0777068883, 0.0548953302, 0.0287088826, -0.0530642793, -4.512600e-02, -0.0782883837, 0.0700238496, 0.0260477476, 0.0574485362, 0.0468161292, -0.0506413728, 0.0688796416, -0.020640038, 0.0261336416, 0.0694708377, 0.00481114443, -0.0607174747, -0.0467249192, 0.0532174408, -0.0231041852, -0.078732118, 0.0206321459, 0.0489814878, -0.0577683561, -0.0276155099], [-0.0774986222, -0.0373363644, 0.0664879754, -0.0282582473, 7.432260e-02, 0.00820970535, 0.00320204673, -0.0693052262, 0.0221913029, -0.0254871063, 0.010242492, -0.0306781605, -0.0641024709, 0.0191954561, -0.0108661624, 0.0765877143, -0.0171044767, -0.0308133513, 0.0571607202, -0.0514838733, 0.0174586345, 0.00982767437, -0.022866562, 0.0768737718, 0.0429883599, -0.0786280259, 0.0433426164, 8.07256088E-4, -0.0186890606, -0.0403303429, -0.0451915562, 0.0731886402, 0.0339945927, -0.00433447119, -0.0197761934, -0.0399837047, 0.033148583, 0.0569562763, -0.00777193857, 0.0810396373, -6.881560e-02, 0.0556797534, -2.377690e-02, -0.0709123835, -0.0724468306, -0.0748433322, -0.044899378, 0.00630222727, 0.0737977102, -0.0577528365, -0.0465867594, 0.0410004333, -0.0555169694, 0.0394431762, -0.065283753, 0.065650031, 0.0624415092, 2.58968445E-4, -0.0547129735, -0.0102924565, -0.0194331165, 0.0224287622, 0.00685989158, -0.025779115, 0.079959087, 0.049324058, -7.613110e-02, 0.0647973791, -0.0791541785, -0.0484874025, -0.0422594659, -0.0232817456, 0.0177616309, 6.378120e-02, 2.978650e-02, 0.048954282, 0.0448039956, 0.0368561223, 0.0370580219, -0.00978818815, 0.0392761119, 0.0325098075, 0.0568236522, 0.0520256795, -0.0533300638, -0.0462946407, -0.0412904061, -0.0477615818, 0.0446449444, 0.0377805158, -0.0216348227, 0.082372345, 0.0131202135, 0.0636173114, -0.0154723087, -0.0320182517, 7.571470e-02, -0.0408415608, 0.00886487402, 0.0637793168, -0.0431713276, 0.0292488132, 0.0127798123, 0.0454765633, 0.0175696425, -0.0292313509, -0.0362529978, 0.0521076284, -0.00731910625, -0.00531889126, 0.0399387926, -0.00206454704, 0.0715513155, -0.0530061871, 0.0723757669, -0.00726268766, 0.0773103908, -0.0456174575, -0.0611178912, 0.00765136862, -0.0630112514, -0.0501155332, 0.0277479962, -0.00803359411, -3.307940e-02, -0.0323125757, 0.0660415366, 0.0676112846], [-0.0545829795, 0.0189915281, 0.0532824174, 0.0264235511, -0.0653685927, 0.0700451881, -0.0737670884, -0.00363551429, 0.0715920478, 0.0799614042, 0.0520072617, 0.0191402882, -0.0607645921, -0.0223246403, 0.0431650802, -0.0409047268, -0.0278220549, -0.056575425, -0.0777204632, 0.0163999833, 0.0729656517, -0.00778514612, 0.0275837388, -0.00835288874, -0.0301785879, -0.048300378, -0.00466358429, -0.0256368965, -0.00485219061, 0.0289305858, -0.0670833513, -0.0157406069, -0.068488866, 0.0346537456, -0.078523308, 0.0393371768, 0.0324656256, -0.0559526682, 0.029839091, 0.0409759134, -0.0171026364, -0.0675575882, -0.00500453589, -0.0166866388, 0.0262584984, -4.585920e-02, -0.0469773263, -0.0484288186, 0.0651468262, -0.060792543, 0.0580449253, 0.0036716254, -0.0120260632, 0.047097832, 4.690370e-02, -0.0615672879, 0.0191896465, 0.0694476888, -0.0420998819, -6.38220634E-4, -0.0765947699, -0.0711385086, 0.0535165593, 0.0707579181, -0.0654330179, -0.0286741145, 0.048729904, 0.0456088111, -0.00759295607, 0.0249986406, 0.0760373399, -4.203470e-02, 0.0193802658, -0.0424343795, 0.0281023122, -0.0670200288, -0.0287804231, 0.0244758073, -0.00959628447, 0.0690108165, 0.00408921577, -0.0307157896, 0.0249923822, 0.0577715822, -0.0558936745, -0.0504342169, -0.0737475082, -0.0236023143, -0.0753777698, -0.0551181398, -2.60778645E-4, -0.027652245, 0.0811203569, -0.00932676066, -0.077180028, 0.0104924804, 0.0321928672, 0.00845803227, 6.296770e-02, 0.0188899543, 0.0419075154, 0.0656229481, 0.0482830629, 0.0709710121, -0.00716950838, -0.0143101551, -8.573260e-03, -0.0180549454, -0.079801321, 0.0126310876, 0.0773460492, 0.0736632571, 0.0134407775, -0.0266145766, 0.0105385436, -0.0573047921, 0.00941796414, -0.010932711, 0.00365077425, -6.028210e-02, 0.00513433851, 0.0571277924, 0.030837303, 0.0266711432, 0.00141196058, -0.0591387227, -0.0299685895, -0.0355005637], [0.059133444, -0.051165428, 0.0113334628, -0.0507289134, 0.0177323613, 0.0705338195, -0.0412180647, 0.0298207868, -0.0494777411, -0.0723843724, 2.293540e-02, 0.058645077, -0.0431654342, 0.0365944766, -0.0368426442, 0.0572528616, 0.0109245246, 0.0441084541, 0.0327465273, -0.0667697042, -0.00872405059, -0.0376774818, -0.0368292332, 0.00370766595, 6.317480e-02, -0.0503628105, -0.0417976119, 0.0138624888, 0.0765534788, 0.0231212396, 0.00219105929, 0.0522769243, -0.0132323373, 0.0686754733, 0.0181199238, -0.0342780799, 0.0379831716, 0.0329394937, -0.0142035866, -0.0727094561, 0.0378836691, -0.0755224228, -0.0238646567, -0.0139936982, -0.0266959611, -0.0497988388, 0.0737839267, 0.0306996536, -0.0358031169, 0.0226574801, -0.0165481232, -0.00238361023, -0.0541888922, -0.0598207973, 0.0563563071, 0.037304841, -0.0705281273, 0.051650919, 0.00814726483, -0.0653192996, 0.0517274365, -0.0273943488, -0.0290444344, -0.0563331023, 0.0634847283, -0.0177577641, -0.0490432568, 0.019120248, 0.0303080399, 0.0585110784, 0.00216406095, -0.064627938, -0.0309864692, -0.0309187956, -0.0729492679, 0.0777909309, -0.0438307561, -0.0591247343, -0.0207475275, -2.166920e-02, 0.0800410434, -0.0591516867, 0.0356770866, 0.0515333563, -0.0769257247, -0.0252670199, 0.0129863638, -0.0383655094, 0.066174224, -0.0668491796, -0.0731600896, -0.0132039916, -0.0145141445, 0.0368876867, 0.0433888808, -0.0270155296, 0.066963695, 0.0392187834, 0.011387459, -7.71840569E-4, -0.0104149198, 0.0729158521, 2.788630e-02, 0.0525630601, -0.012583917, -0.0743490234, -0.00205346313, -0.0773533508, 0.0060863453, 0.0553055629, 4.016010e-02, -0.0125474893, -0.010032068, -0.04128059, 0.0113333678, 0.0249130148, 0.00323382253, -0.052884087, 0.0258572977, -4.098340e-02, -0.0747467205, 0.0503353775, -0.0655560717, -0.0365380123, 0.0382050574, 0.0557886362, -0.010920058, 6.998180e-02], [-0.0468743369, 0.079554297, 0.0336141251, 0.0109817637, -0.042480953, -0.012232976, 0.0754472762, -0.0764717459, -0.0261635873, -0.04696282, 0.00225514593, 0.0204461664, -0.0682043061, 0.0130443554, 0.0801237598, -0.0549960583, -0.00293501467, -0.0352158956, -0.0360268578, -0.0360384323, -0.0521016233, 0.0293367356, 0.0474039912, 0.0333662145, -0.0340123884, 0.0755011961, 0.0450275168, 0.0262213163, -0.0302306153, 0.0264173523, -0.0229878612, 0.0229778979, -0.0347260237, 0.069378376, -0.00187713141, 0.00315982429, 0.076451838, -0.0167116616, 0.0447940938, -0.0706012174, -0.0529833585, 0.0696829781, 0.0336743668, 0.0224993136, -0.0102747381, -7.113450e-02, 0.0644969121, 0.0490097441, 0.0762202889, 0.081129074, -0.0534821823, -0.0491932482, -0.0493080728, -0.0253985543, 0.0504176542, 0.0750989169, -0.0150796482, -0.0445113517, -0.0794208795, 0.0616808534, -0.0690391436, -0.0212108977, 0.0348447934, 0.0413284041, 0.0759323761, 0.0733342468, -0.0653870553, -0.00304574519, 0.034038268, 0.0779165775, -0.0171688087, 0.0506679527, 0.0655070841, 0.0544550233, -0.0557867028, 0.0401474163, -0.0582830831, -0.00277611543, 0.0598381497, 0.0538043939, -0.0154103898, -0.00150998263, -0.0362972952, 0.00143705436, 0.0477877818, -0.0771521925, -0.0580522493, 0.0803429931, -0.0338078439, 0.00639373297, -0.0136666847, -0.0413674638, 0.0182183497, 0.0286080092, 0.0772770643, 0.0499368496, 0.0533848852, 0.0187205877, -0.0747646242, 0.0388022885, 0.0387554467, 0.047446575, 0.0808586403, 0.0423000716, -0.0452300981, 0.00254558888, 0.0252883695, -0.0339850262, 0.00148097507, 0.0480959192, 0.0213294569, 0.0340550914, -0.0417675935, 0.00678100623, -0.00509182271, 0.025503464, 0.0156655125, -0.0139390798, -0.0583605357, 0.0612232722, -0.030446941, -0.00344717875, 0.0125471847, -0.0480211377, -0.0131963724, 0.0176846012, -0.00736177294, -0.0610625073], [-0.0389366038, 0.0782138407, -0.0567994192, 0.0260665677, -0.0591716841, -0.00105030392, 0.00736378739, 0.00927871279, -0.0728153661, -0.00204918534, 9.69800225E-4, -0.00287966221, -0.0261270888, 0.0443700179, 0.0763522982, -0.0213550925, -0.00637514144, 0.00148804486, -0.0614740774, -0.067528829, -0.00920858141, -0.0137530966, -0.0240172073, -0.0510571897, 0.0399038531, -0.0500765666, 0.0438349918, -0.0313019156, -7.946630e-02, 0.0533375256, -0.0695179402, -0.0480792522, 0.0400472209, 0.0464239679, -0.0472034551, -0.0243097022, 0.00308413361, -0.00350594404, 0.0286219399, 0.0202890895, 0.0055192057, 0.0466792956, -0.00385758025, -0.0767908916, 0.0183717888, -0.0250576157, -1.569300e-02, -0.0497819968, 0.0350326933, 0.008630055, -0.0344361067, 0.0418424718, -0.0162229855, -0.0155395251, -0.0171218347, 0.0717085227, 0.0412050858, -0.0469926484, 0.0232600924, 1.916140e-02, 0.00470828032, 0.0431326106, 0.0340712965, -0.0306552313, 0.0683299825, 0.00655414257, 0.0429756232, 0.0404853076, 0.0349098295, -0.0616563037, -0.0579761639, 0.0699705556, 0.0552140586, -0.0152144954, -0.0314036384, 0.0231003333, -0.0648852363, -0.023289388, -0.0392726623, -0.0426560491, -0.0563395321, 0.0718562827, -0.0776857659, -0.0353992544, 0.0696230158, 0.0103588346, -0.061836727, 0.0471235327, -0.0725274906, 0.0212183576, 0.0539282076, -0.0489951558, 0.0773567632, -0.0807104408, 0.0674086586, 0.00834149122, 0.0109898057, -0.0442896225, -0.0530308262, -0.0748433843, 0.0112506254, -0.0360462368, 0.00489304028, 0.00837585143, -0.0176000968, -0.0598889329, 0.0660024956, -0.0424995162, -0.0431521796, -0.0572984703, -0.025030667, 0.00759654492, -0.0411047451, 0.0268152673, -0.0385582633, -0.0803328753, 0.0795494467, 0.00671867374, -4.328620e-02, -0.0169112179, -0.069764033, -0.0451666042, 0.00348834717, -0.0310044587, 0.0255211387, -0.067921564, -0.0720262602, -0.0439208038], [0.0491072163, 0.0770218447, -0.039790526, -0.0338163823, -0.0385906212, -0.0756822377, 0.0713992342, -0.041924756, 4.796820e-02, 0.0265052468, 0.0191888511, -0.0802949965, 0.0611753538, 0.0176346749, -0.00911127776, 0.0585829094, 0.00442086905, -0.0327958427, 0.0329853222, -0.0495886616, 0.0255789235, 0.0225690529, 0.0649369434, 0.00211302191, -0.0721414313, -0.0606989972, 0.0422436073, -0.00943047553, -0.00599089265, -0.0296699032, 0.0732922778, 0.0272168368, 0.0213254616, -0.00937008113, -0.00303750485, -0.0263118632, -0.0696464404, 0.0284377635, 0.0390903056, 0.0664873049, 4.355930e-03, 0.0116509199, -0.0354035348, -0.0624580458, -0.014297463, 0.0554487631, -0.0165112317, -0.04218008, 0.00320979208, -5.999860e-02, -0.0289883241, 0.066304557, 0.0620266721, 0.0302349105, -0.0792566091, -0.0424329862, 0.0714944676, 0.0115801245, -0.0614213608, -0.0288550258, -0.0352741815, 0.0658444092, 0.0116036162, -0.0639150366, 0.0508901849, 0.0195783675, -0.0195862167, 2.565100e-02, 0.0237099901, -0.00914819538, 0.0606042072, 0.00587771833, 0.0164713785, -7.858120e-03, -0.033594165, -0.046788089, -0.00317884982, -0.0666333586, -0.00786817073, 6.46904111E-4, -0.0450975522, -0.00192829967, 0.0278059393, 0.0431508422, -0.0103864297, 0.0675684884, 0.00585443527, 0.064551495, -0.0190540478, -0.0771693513, -0.036794927, -0.028963048, 0.00445672125, -0.0315862112, 0.0661707744, -0.00472554564, -0.0647264272, 0.0601123348, -0.0319474116, 0.0414023325, -0.0118658841, -0.0623250604, -0.0762460455, -0.0096944645, -0.0420619622, 7.880040e-02, 0.0040095821, -0.070416607, -0.0695422292, 0.0702765062, -0.0465983786, -0.00194599479, 0.0594251081, 0.0692077354, 0.065538384, 0.0422438979, 0.0404933766, -0.0613093339, 0.0210535526, 0.0211869031, 0.00855934619, -0.0144924894, -0.0391574837, -0.0169442594, 0.0179147869, 0.0552127287, -0.0301467068, -0.00275655836], [0.043846868, -0.0237600394, -7.244680e-02, -0.0106356964, 0.0537637845, 0.0231916308, -0.065109618, 0.029668875, -0.0645951629, 0.0360594913, -0.046771884, -0.0460489802, 0.0352397561, -0.0501682684, 6.171710e-02, 0.0337219313, 0.0192840174, 0.0114124417, 0.00407014787, -7.408100e-02, 0.0721796677, -0.0506622642, -0.0384719633, 4.072640e-02, 0.072301887, 0.0790777429, -0.0493513606, -0.0497499406, -0.0284365453, -0.0356634408, 0.0635669455, -0.00890828669, -0.0557157695, -0.0558107197, 0.0436508581, 0.0328066126, -0.0349888429, -0.0717594251, 0.0139526427, 0.0038490966, 0.0699675307, -0.0239716396, -0.0205615237, -0.0335554108, -0.0515606478, -0.0760689229, 0.00187046081, -0.0216850415, 0.0256546885, -0.0121236071, 0.004642196, 0.045742549, -0.0449500605, -0.00864199549, -0.0425419584, -0.041848179, 0.0203168541, 0.0216177404, 0.0126947016, -0.0304341316, 0.0228794515, -0.0149824619, 0.00600709766, 0.016346164, 0.0188403577, 0.0106194541, 0.0127714574, 0.0586765632, 0.0508933738, -0.0103567019, 0.0452158526, -0.0240871496, -0.010922946, 0.0317616314, -0.024851203, 0.0520227477, 0.0322698206, -0.0283421353, 0.0151743963, 0.0668062493, 0.0322919264, -0.0669820308, -0.0450251698, 0.00338681042, -3.992200e-03, -0.07708662, -0.0463647358, 0.0526210293, 0.0680258349, -0.00168284029, -0.0435616486, 0.023287721, 0.0618174151, -0.0395139493, -0.0667074844, 0.0329850838, -0.011165686, -5.34415245E-4, -0.0176706016, 0.0376689956, 3.612510e-02, 0.0792717561, -0.0778119042, 0.00405121595, 0.0530175939, -0.0211522318, -0.0691317767, 0.0438133776, -0.0151028484, 0.0721220151, -0.0717104748, -0.0533376224, -0.0671374499, -0.0232710727, 0.0577682629, 0.0177140385, 0.0641807392, 7.204250e-02, -0.0534530915, 0.0212010071, -0.0685640797, 0.0205059499, -0.0614127368, 0.0221934244, 0.00952047855, -0.0602266192, -0.0430275612, -0.0676382333], [-0.0030002594, -0.0656909496, -0.00202035159, -0.00447739661, -0.00213086605, -0.0550219305, -0.0433971174, 0.0285979435, -0.0273079015, -0.00726729631, 0.0253262147, -0.0551916808, -0.0221534148, -0.010587275, 0.0593375042, 0.0724925175, 0.0461035892, -0.00281903893, 0.058579959, -0.0596561581, -5.139130e-02, -0.0560058653, -0.0351381749, -0.0521591827, -0.0287472717, 0.00782317668, 0.0734722093, 0.0445310399, -0.0213401429, -0.028643176, -0.0642500892, 0.034998104, -0.0558871627, -0.0105416551, 0.0770109072, -0.0599254668, 0.033118479, 0.0319152325, -0.0654776096, 0.0244895965, 0.0151496083, -0.0421003476, -0.00750691444, -0.068126604, -0.0434279032, 0.0405473262, 0.0684126392, 0.0790423825, 0.06531737, -0.0656805784, 0.0760252401, -0.0341726728, -0.044463411, 0.0243066549, -0.0181666091, 0.0576809719, -0.0362514593, 0.0598035529, -0.020076599, 0.0553481504, -0.0576232225, -0.0517934188, 0.0758674219, 0.0151326284, -0.0518677756, 0.0363412127, -0.0631533936, -0.0184468031, -0.0242334045, -0.0587013699, -0.0329845846, 0.0671101883, 5.742550e-03, -0.00347480178, -0.0566712916, -0.0164898634, 0.0621868297, 0.0702289268, -0.0688332468, 0.0355394036, 0.0301536918, -0.0118987039, -0.0756316632, -0.0208316036, -0.0655721501, -0.00912023336, 0.00296773016, 0.0310965255, -0.0775084495, -0.0729384422, -0.0445328765, -0.02803858, 0.00141327828, -0.0503959358, -0.0416514874, -0.0411727503, -0.0441151857, -0.0799829736, -0.00480653346, 0.0514109507, -0.00840767473, 0.0392204896, -0.0367552638, -0.00226832181, 0.0785398408, 0.00576353073, 0.0285706967, 0.0808765664, -6.438580e-02, -6.880760e-03, 0.0758272931, -0.0348283164, -0.0578652211, 0.0685160384, 0.0658727661, -0.0154301077, 0.0613276735, 0.0307808891, 0.0393260568, -0.00965451449, -0.0373210236, -0.0210016854, -0.0736778378, 0.0392167345, 0.0285271034, 0.0358768553, -0.0678680167, 0.0733763352], [-0.0267200954, 0.0653254762, -0.0443099812, 0.00974921137, -3.358030e-02, -0.0155038983, -0.0205828361, -0.0653976798, 0.0759912208, 0.0409503579, -0.00333183259, 0.0691125765, -0.0386185832, 0.0339614376, -0.0789535418, -0.0185838193, -0.00421085954, 0.0164780915, 0.0377408415, 6.41584396E-4, -0.0570624322, -0.0605632439, 0.0334769785, -0.0600957386, -6.746160e-02, 0.0574210808, -0.0527678207, -0.0083976984, 0.0116257444, 0.0543708131, -0.022985328, -0.0735355467, 4.134950e-02, -0.00180841982, 0.0570688769, -0.00147957355, -0.0688210279, 0.0114465132, 0.0419943929, -0.0730975717, -0.0705455542, 0.0186673552, 0.0472210273, -0.0531655326, 6.079880e-02, 0.00919482111, -0.0782980918, 0.00196891278, 0.0672471747, 0.0541644767, -0.0139842033, 0.0336926207, 0.0683158711, 0.0585669354, 0.0152140558, 0.0442114696, -0.00956246256, 0.0782718881, -0.00338679552, 0.044262819, -0.00895531475, 0.078332208, -0.0728707388, 0.0424640626, 0.0643024221, 0.00902172178, 0.0455940887, 0.00132407248, 0.0522803739, -0.00132923573, 0.00716213137, -0.0416546203, -0.0302085318, 0.0458449796, 0.0188108832, -5.296760e-02, 0.0705585107, 0.0200665817, 0.0191156939, 0.0468314961, -0.0169562101, -0.0458763056, 0.0793499425, -0.0215213411, -0.0420140624, -0.0553847551, -0.00808754563, -0.0130445957, -0.00443065912, 0.0103836209, -0.054405909, 0.00460332632, 0.0743817762, 1.299420e-02, -0.0395511948, -3.598730e-02, 0.0642794594, 0.0289443135, -6.356650e-02, -0.0145392269, -7.621400e-03, -0.0653573647, -0.0751175731, 0.0397968888, -0.0735400915, 0.0317539722, -0.0368994102, -0.0350115672, 0.0207108334, 0.0087647736, -0.01842089, -0.0539912581, -0.00504720211, 0.0796428993, 0.0291340798, -0.0706698596, 0.02423089, 0.0546273366, 0.0412873253, 0.0477762148, 0.0486825332, 0.049863182, 1.43103302E-4, 0.0604779646, 0.0447862074, -0.0589758381, -0.0529764406, -0.0512317196], [0.00597820431, 0.0744603872, 4.211710e-02, -0.065181382, -0.00119272748, -0.0617477335, 0.0491940826, -0.0117784673, 0.0358713977, 0.0674477369, -6.952500e-02, 0.0798187107, -0.0278609581, -0.0136002051, 0.0156508796, -0.0742824227, 0.0668681785, -0.0648105294, -0.0335277542, -0.00458095176, -0.0677061751, 0.0478843525, -0.0448871069, -0.0163778514, 0.0377877615, -0.00805794448, 0.0748098194, -0.0111449305, 0.0148603227, -0.071147427, -0.0126282275, 0.0423650965, -0.0159732252, -0.0431049317, 0.0331842192, 0.0347966217, -0.0692616627, 0.0375963524, -0.0598421134, -0.0239848085, 0.0134081459, -0.026381338, 0.0791001915, 0.0376314446, 0.0308392961, 0.0170912985, 0.0491264611, 0.0361570604, -0.074347809, -0.013004479, 0.0747327506, -0.0648413152, 0.0716711357, -6.342580e-02, -0.0347568206, 0.0559446402, -0.0148510356, 0.00919095427, -0.0682641491, 0.00115375814, 0.0182630122, 0.0606753342, -4.313110e-02, -0.0585024618, -0.0594842061, -0.0597925112, 0.0237861555, -0.0629494563, 0.0617279485, -0.0492807515, -0.0102162091, 0.0237334147, -0.0424883254, 0.0720755159, -0.019639533, -0.0492922589, 0.0412996374, 0.0660520345, 0.0513180308, 0.0298303682, 0.0426161326, 0.0197402295, 0.0172556918, -5.143600e-03, -0.07321392, 0.0182893649, -0.0527937897, 0.0520015731, 0.0624441504, 0.0632308051, 0.00601477083, 0.012933122, -0.00402879389, 0.0453603193, -0.068389222, 0.00294324709, -0.0635316223, 0.0764731169, 0.0346129388, -0.00115060771, 0.00208330108, -0.0507697351, -0.0567290485, -0.0269615948, 0.0630595088, -0.0578750484, 0.068347916, -0.0234168433, -3.921950e-02, 0.050696183, -0.0142961284, 0.0502580628, -0.0180836916, 0.0180164557, 0.0353529602, 0.0734430328, -0.0764786378, 0.0426905453, 0.0455258414, -0.00737298513, -0.0268102568, -0.0634777769, -0.0158998277, -0.0464262962, 0.0717216805, 0.0178005546, 0.017142484, 0.0104670292], [-0.0488697216, -0.0020911966, -0.0521354675, 0.0623382479, 0.0244623832, -4.509530e-02, -0.0101494472, -0.0362177081, -0.0136650987, 0.0159701705, -0.0307928268, -0.045821283, -0.0535091311, -0.00646309229, 0.0542541742, -0.0803375467, -0.0414989628, -0.0421726517, 0.0135282213, 0.0184418019, -0.0570807308, 0.022057863, -0.0589980297, 0.0333256274, 0.0592173822, -0.0591370277, -0.0431808606, 0.0263137538, 0.0060698553, -0.00581365218, -0.0546245091, -0.0791003406, 0.00345347147, -0.00128397567, -0.0060011344, -0.0367899202, 0.0154056558, -0.00562162325, 0.00201270054, -0.0442283377, -0.0569499582, -0.0732912272, 0.0249770377, -0.0672284439, 0.0278931167, 0.0123865092, -0.00202098861, 0.0448244661, 0.0466152877, 0.00653627468, 0.00605899747, -0.0351871513, -0.0318586193, -0.0113871591, 0.0444336571, 0.0511263236, 7.536330e-02, 0.0621519871, -0.036721047, 0.0321033448, -0.0349776186, -0.0130580319, 0.0476618595, -0.0472987778, 0.0218346231, 0.00307505718, 0.0307611208, -0.00264747441, 0.0369802266, 0.0655575096, 0.0268457588, 0.0226240866, -4.510670e-02, -0.05113158, -0.0654748976, -0.0485141873, 0.061359182, 0.00436951872, -0.0448222384, 0.0699436814, 8.047310e-02, -0.080047287, -0.0395357497, 0.0625950247, 0.0800976678, -0.0191377271, 0.0726812407, 0.00162151724, 0.0588358454, -0.0544563681, -0.0266040154, 0.0531296954, -0.00653049163, -0.0660038069, -0.0626169071, -0.0605453923, 0.0328711644, -3.192130e-02, 0.0385042205, 0.0421981588, -0.0404911675, 0.0762099773, -0.0383824743, 0.05300612, -0.0298230927, 0.0400295332, 0.0483232252, 0.0458138511, -0.0785091593, -0.0663361177, 0.0737232715, 0.08061897, 0.0604891405, 0.0371470191, -0.0271695703, -3.20477411E-4, -0.0757534057, -0.0760446042, -0.0660923198, 0.0429254286, -0.015147958, 0.0734333097, -0.0511199906, -0.0747117698, -0.0380398817, 0.00199801475, 0.0137432432, -0.00457118731], [-0.0406892598, -0.0366746113, -0.0728205666, 0.047426071, -0.0734212324, -5.999680e-02, -0.0354200788, 0.076354973, 0.0250785891, -0.00644237734, 3.270090e-02, -0.036240641, -0.0412740521, -0.0652908757, 0.0634623542, 0.064161554, 0.0638439134, -0.0469717309, 0.00220974116, 0.0571930483, 0.026038995, 0.0470737778, 0.0269543957, -0.0358972028, 0.0285728127, -0.0236489829, -0.014724643, -0.0650011227, 0.0282786656, 0.0521254651, 0.0698338598, -0.0599196926, -0.019442888, -0.072272718, 0.0484687686, -0.00826710835, 0.0254833084, -0.0476871654, 0.0528466366, -0.0692519471, -0.0448490903, -0.0387476049, -0.00557773095, 0.00655273302, 0.0436410792, -0.0285386853, 0.0216382295, -0.0710986554, 0.0416576564, -0.0101346523, 0.0359763727, -0.0015901702, 0.0634641349, -0.0510947444, -0.0560407862, 0.0182282794, 0.0605375059, 0.0781999379, 0.0332496092, -0.080157034, 0.0422365889, -0.00141508086, 0.0124322018, 2.191000e-02, -0.0513369851, -0.0365037471, -0.0555184521, -0.0316143297, 7.56069319E-4, 0.0165606104, 7.687090e-02, 0.0128909964, -0.0760635585, -0.0113459565, -0.0276426487, -0.0288330764, -0.0411710441, 0.0760979578, 0.0149405953, -0.0671915263, 0.0446511284, 1.674510e-02, -4.508050e-02, 0.0658388883, -0.0304361656, 0.030253768, 7.772850e-02, 0.0360586867, -0.0544171855, 0.0620743632, 0.0143904202, -0.00819578766, 0.0373310633, -0.0673182532, 0.0638889447, -0.0350845419, -0.0244358256, 0.0123312678, 0.0806971713, 0.0715146362, -0.0351659134, 0.0225128364, 0.0542368814, -0.038302362, 0.0362324491, 0.0272915363, 0.0149974581, -0.0348225534, -0.0351985395, 0.0152902119, 0.0258812569, 0.00712945685, -0.0313293785, 0.0539262556, 0.0299303569, 0.0455206186, -0.0707918555, 0.0617320724, -0.0671219751, 0.0503979586, 0.00663002487, 0.0113583114, 0.0299050175, 0.0369099043, 5.55325765E-4, -0.0556207299, -0.0760733709, 0.0155282272], [-0.0057312632, -0.00947643165, 0.0507308133, -0.0656186119, -0.0768060461, 0.0154587431, 0.0148038436, 0.0386766791, -0.0461259708, -0.0592248663, -0.0760873556, 0.0342269577, -0.00561151607, 9.99577314E-5, -0.0409442261, -0.0512300692, -0.058885321, -0.0537179112, -0.0679378361, 0.0518599264, -0.0557535402, 0.00201030122, 0.00856525823, 0.00106573082, -0.06054838, -0.0729133487, -0.0111348955, -0.00644805189, -0.0103898682, 0.0220495649, -0.0325214081, 0.0468020961, -0.0513712689, 0.0227176063, 0.0505774766, 0.0470633842, 0.00545716844, -0.0241706073, -0.00911710411, 0.0228318404, 1.26614425E-4, -0.0196742024, -0.0762703046, 0.00706579816, 0.0497058854, 0.048627805, 0.0488648415, 0.0310652684, 0.0421245359, -0.0210027583, 0.056548208, -0.0754921734, 0.0409733132, -0.0633386448, 0.0145123405, 5.557070e-02, 0.0601947643, -3.185550e-02, -0.0247076061, 0.0728647336, 0.0595367663, 0.0562091693, -0.0499354266, 0.0714999139, -0.0337769873, -0.0548945591, -0.0474645756, 0.0411073938, 0.00491378549, -0.0150552029, -0.074218221, -0.0389340334, -0.0738223866, -0.077364184, 0.0803310871, 0.0348097458, 0.0428270102, 0.0351627842, 0.0559704676, -0.0116695091, 0.0248988364, 0.0115745319, 0.0537225865, -0.0757464692, -0.0514766574, -0.0759133398, 0.0369704738, 0.0750268921, 0.0673292056, 0.0362993628, -0.0499299914, 3.39155173E-4, 6.583220e-02, -5.189710e-02, 0.0324411951, 0.0223718379, -5.52842161E-4, 0.0177256633, 0.0373562947, 0.0172233805, -0.0446165912, 0.0526316836, 0.0379208252, 0.0275556426, -0.0180148259, 0.0591969192, 0.0667756945, -0.038040638, -0.0526852645, -0.051262375, -0.0653258637, -0.02065509, -0.0583899058, 0.0279608835, 4.9461989E-4, -0.0723690912, 0.0466838926, -0.0336887799, 0.06000375, 0.0710073709, 0.0705734119, 0.0628192946, 0.0414220281, -0.00275207474, -0.0194660611, 0.0326166712, -0.0375044905, 0.0295299236], [0.0450164154, 0.0089321956, -7.21066608E-4, 0.0108983712, 0.0720496699, 0.0120195746, 0.0288320649, -0.063011542, 0.0296388138, 0.02015635, 0.0390448309, -0.0763295218, -0.0704493448, 0.0526039638, 0.0616404191, 0.0457154214, 0.00151466054, 0.00204192731, -0.0412296541, -0.0351611525, 0.0668839663, -0.0432929844, -0.0668684244, 0.0488789864, -0.0535565466, 0.0133073656, -0.0556230769, -0.0654499382, -0.0604541339, 0.0343374833, 0.0272475891, -0.0827339589, -0.0216270518, 0.0581667051, 0.0343052819, -0.0760247931, -0.0432302319, 0.0665762573, -0.0557030402, -0.0809736401, -0.00865580141, 0.0487169884, -0.0663568154, -0.0700070559, -0.00603596726, -0.0649145916, 0.0670165494, 0.05450546, 0.0718822628, 0.0501758195, -0.0372853316, 0.00438612094, -0.0112216752, -0.0712691918, 6.531580e-02, 0.0499711707, -0.0171339195, -0.0534589961, 7.051190e-02, 0.00417847186, -0.0394954123, 0.0575605482, 0.055650562, -0.0143070286, 0.0770030171, -0.0237480737, -0.00678535644, -0.0534840785, 0.00400238438, 0.0558782406, -6.40351616E-4, 0.00162919273, -0.0299465898, -0.0386870168, 0.0733051598, -9.614570e-03, 0.0597239323, -0.00561298709, 0.0125723975, 0.0741568357, 0.0444827862, -0.0295198876, 0.038059745, -0.061558146, -0.0563963093, 0.06604743, 0.0108854342, -0.0310327504, -0.0285919514, -0.0224950723, 0.060834609, -0.0240407027, 0.0666977242, -0.00842980854, -0.00607866794, -0.0789971575, -0.0557665415, -0.0376161076, 0.0214182399, 0.0121578854, -0.00937976595, -0.00817575771, 0.0312168654, 0.0646334365, -0.00594295561, 0.0321920961, 8.27732671E-4, 0.0344149657, -0.047203172, -0.0384519696, 0.00958010181, 0.0455816761, -0.0476598032, 0.0684642419, 0.0789049342, -0.0305060316, -0.0332056582, -0.0583922043, 5.261250e-02, 0.00177541829, -0.0296573602, -0.0289537143, -0.0355318747, 0.0777852535, 0.0166312754, 0.0473536216, 0.0522349738, -0.0558774434], [-0.0215474721, -0.0819291099, -0.0649309903, -0.0675329044, -0.0540286116, 0.039133884, -0.0106316274, -0.0821319594, -0.0171298794, -0.0234753918, -0.0815985873, -0.0255714823, 0.00608189171, 0.0170332361, 0.0196603555, -0.0259451773, -0.0521541163, 0.0530733615, 0.0537779145, -0.0519252457, 0.0418569483, 0.0525798835, 0.0274286494, -0.0382163823, 0.0602074265, -0.0661200508, -0.0485754572, -0.047627978, 0.0611948967, -0.0633196458, -0.0174509622, 0.00690341461, 0.0230119489, 0.0355088077, 0.0245276075, -0.0473346822, 0.0707075372, 0.0303070173, -0.0614171103, 0.0180048738, 0.0246829223, 0.0635249912, 0.020257581, -0.0418680906, -0.0445888489, -0.0339115895, -0.0428500921, -0.0711104497, -0.0242574308, -0.0266599674, -0.0618107207, 0.0270968191, -0.0277437065, -0.0303871222, 0.0786208733, -0.041213464, -0.0564361401, -0.0738825351, -0.0491460599, -0.0607281849, 0.0797713324, 0.0116843078, 0.0342698805, -0.0629781038, -0.0363507271, 0.0274825543, -0.0264241304, -0.0184686184, -0.068757914, 0.012800172, 0.0771046728, -0.0478485227, -0.0324432366, -0.0133354571, 0.0400835872, -0.021704834, -0.0360094905, 0.023012301, 0.0209106952, 0.0780846924, 0.025491653, 0.0341432542, 0.0784565731, 9.52762959E-4, 0.0253812093, 0.0618096367, -0.0699485764, -0.068286635, 0.0556294546, 0.031708464, 0.0561353453, -0.0729380101, 0.0631283075, 0.0257813558, -0.0481159724, 0.0493164249, 0.0703274608, 0.0541102439, 0.0155057218, -0.0640039816, -0.00882610399, -0.0673306063, -0.00787652284, 0.0322318673, 0.055741474, -0.0264298245, 0.0275715161, 0.0660540685, -0.0659517124, 0.0784842446, -0.0662635192, -0.0487049893, -0.00394835277, -2.21623282E-4, 0.0671426281, -0.0672580301, -0.0500839911, -0.0706906616, 0.0300279614, 0.084385164, 7.684410e-02, 0.00110875966, 0.0813527256, -0.0128458971, -0.0170561019, -0.0722295418, 0.0680139362, -0.0530818366], [0.0443673767, -0.0535069071, 0.0700450614, 0.0485407114, 0.0302778576, 3.051540e-02, -1.05466192E-4, -0.0395215079, -0.0620420352, -0.0702755898, 9.148370e-04, -0.0594541393, -0.0271549039, -0.010975467, 4.821920e-02, 0.0157665126, -0.03920516, -0.0160721857, -0.0465967953, -0.022539366, -0.0107101947, -0.032970313, -0.0103399148, 0.0311765987, -0.0767244846, 0.0379498675, 0.0456260294, 0.016776178, -0.0442755558, 0.0244462322, -0.0242474675, -0.0515145138, 0.0746776834, 0.0235173088, -0.062833488, -0.00399439223, 0.0792091712, 0.0154086323, 0.0412404165, -0.0760124475, 0.0359116346, 0.00381637458, -0.0647074655, -0.0377817899, 2.01353032E-4, -0.0660449639, -0.0460419878, -0.0475402288, 0.0424671657, -0.0754134431, -0.0696245655, -0.0272872429, 0.0171372481, -0.0455565825, -0.0313558243, -0.0353230201, 0.0541351326, -0.0658270717, 0.038750384, -0.0577312596, -0.0399689749, 0.00814740359, -0.0420694947, 0.0594955198, -0.0482413024, -0.0493921861, -0.0287051629, -0.0388862342, -0.0140715875, 0.0102453632, -0.0594516061, 0.0451593548, -0.0748943612, 5.432820e-02, 0.0280103609, -0.0245631579, 0.0138333524, 0.0571226925, 0.0699373409, 0.0264465604, -0.0594891086, 0.0599234737, 0.0587625802, -0.029916713, -0.0378732421, -0.0451287851, -0.0595643744, 0.0292967577, 0.0597693548, -0.0543282926, -0.0662924871, -0.0530550405, -0.0176559817, 0.00606481917, -0.0184947252, 7.044810e-02, 0.0325662978, -0.0227425396, 6.113210e-02, 0.0521991849, -0.0399637073, 0.0307315532, 0.0568979718, -0.0539852306, 0.0555483475, 0.0412389711, 0.0146207213, 0.0324511901, -0.0405586474, 0.0603943653, -0.0372830182, 0.0370542482, -0.0709759817, -0.0321985409, 0.0466396362, 0.016258372, 0.0280130468, 0.0898320526, -0.00701820711, 0.0456227697, 0.016267674, -0.0102907317, 0.0523867384, 0.0445721857, -0.0339695066, -0.00652002869, -0.00586464908, 0.0379534028], [0.0350704491, -0.0171483159, -0.0712046847, 0.00908740982, -0.0473985262, -0.0190316122, -0.0497905426, 3.195870e-02, 0.0545228273, 0.0576923303, 0.0505935289, 0.0118286973, -0.0690073222, 0.0805693492, -0.00156825734, 0.0598146245, -0.0766594857, 0.0271063205, -0.0235120691, 0.0149338879, 0.0179505255, 0.0465035215, 0.0171654429, 0.0453312062, 0.0291306153, -0.0454445034, 0.0299708117, -0.0598226674, -0.0695225894, -4.057280e-02, -0.0195458084, -0.0387537517, -0.0399103574, -0.00169435691, 0.0599851422, -0.0813169851, -0.0549733602, -0.034507934, 0.00709924102, 0.0133814849, -7.225660e-02, 0.0355002806, -0.0413484871, -0.0538004152, -0.0652165487, -0.0292068608, -6.658110e-02, -0.0837757587, 0.0515075065, 0.00471815886, 0.0175654218, 0.0528232493, 0.025905421, 0.0226306655, 0.0287495144, 0.00671867607, -0.0754423663, 5.8859674E-4, -0.0564864688, 0.0325657353, -0.0355298892, 0.0107390331, -0.0671508387, -0.067634657, -0.0212457404, 0.0264504533, 0.0639104471, -0.0412869491, -0.0122556835, -0.0424057022, 0.0300243255, 0.0742625371, -0.0836574286, -0.0617710128, 0.0187809784, -0.0739114136, -0.0179492552, 0.0734099671, 0.066057831, -0.0187967774, 0.0731525123, -0.0595354922, -0.0518283695, 0.0655146241, 0.0200699754, -0.0462531112, 0.0156371891, 0.0121492101, 0.070929125, 0.0606525615, -0.0410524383, 0.0309662186, 0.0850669518, -0.0571575388, 0.00246784929, -0.0596070066, 0.0833788588, 0.0371442214, -0.00648840237, -0.0758934245, 0.0647514835, 0.095881693, 6.475970e-02, -0.0469946675, 0.0106612695, 0.0568200834, -0.0521654822, 0.0250382368, 0.0133377975, -0.00316885253, 0.0392156243, 0.0241978075, 0.0301854629, -0.0392902792, -0.0389401205, 0.0482214354, 0.0436084196, 0.0667082891, -0.0122287674, -0.0522324666, 0.0156691112, 0.0565818548, -0.0287754852, 0.0281249043, -0.0451953523, 0.0549922325, -0.0141218789, -0.0161929727], [0.0785131081, -0.0200721677, 0.00336057413, 0.00687391357, -0.0675399154, 0.0135139897, 0.00159556849, -0.0241289176, 0.026392838, -0.00741671957, -0.0137535604, 0.0587895364, 0.0362853445, -0.01617584, -0.00358170015, -0.00702847866, 0.0128911501, 0.0595663674, 0.0599798858, 0.0641212463, -0.0122571243, -0.0113421055, -0.0513857864, -0.0244695283, 0.0165154152, -0.0365562364, -0.0746126845, 0.026468778, -0.021280406, 0.02285151, -0.0721356645, 0.0376147255, 0.0539905652, 0.0487601385, 0.0535358675, 0.0155931087, 0.0764936656, 0.0119258799, -0.00802302081, 0.0431607254, -0.0798170194, -0.0379164554, -0.010897412, 6.843940e-02, 0.0823586657, 0.0896197408, 0.0555906631, -0.00618205918, -0.0480716489, 0.0753986761, 3.984660e-02, 0.0118604181, -0.0550782159, -0.0505954027, -0.054801777, -0.0573269203, -0.0116209779, -0.00289975922, -0.0547759831, -0.0653663725, 0.0640716404, 0.0654105172, -0.0197211411, -7.151740e-02, -0.0697074905, -0.0114483209, 0.064337261, 0.0595593788, -0.0692764222, -0.0129868956, -0.0468085185, 0.0705467463, 0.0576103106, -0.0655869097, 0.0608751848, 0.00794943235, -0.0812770351, 0.0704446882, 0.0139425192, -0.0374394879, -0.0371868908, -0.0433913507, 2.65782874E-4, -0.00527270045, -0.0559508875, 0.0665994138, -0.0113167986, -0.0186356511, 0.0693232343, -0.0381161049, 6.325330e-02, 0.00823793374, 0.0448181629, -0.0194815081, -0.0707115158, 0.0688448399, -0.00824399478, -0.0253277253, -0.0326421559, -0.0456817932, 0.0071017514, -0.0125815626, 0.0248985291, 0.00241233525, -0.020788746, -0.0506073721, -0.0489945039, -0.0115646552, 0.0302636493, -0.0182421356, 0.0531279556, -0.0512488335, 0.0534241907, 0.0204148591, 0.0352563262, -0.0678872392, -0.0492926128, 0.0617845878, -0.051637888, 0.0303753521, -0.0157212745, 0.00119177334, 0.0685403273, -0.0715521052, -0.0733940899, -0.00221394096, -0.0267774891, 0.00302512315], [-0.0480087362, -0.0613627546, -0.0478161201, 0.045044858, -0.00224792026, 0.0451478027, -0.00844184122, -3.634160e-02, 0.0323484428, -0.0242166184, -0.0670073703, -0.0456824712, -0.0798268988, 0.0568648316, 0.0448186845, 0.0665175914, -0.0186514966, -0.0487574451, -0.0192590952, 0.0492845327, -0.0618087687, 0.0572937913, -0.0520796664, -0.0726481602, 0.0192753039, 0.0614731647, -0.0364969857, -0.0244921278, -3.643080e-02, -0.0678097829, -0.0270409957, 0.0321426541, 0.0104618808, 0.0620514191, -0.0781820565, 0.0134959081, -0.0486642793, -0.0333301648, 0.00200760923, 0.0291127451, -0.0289217178, -0.0374202318, 0.0656208097, -0.0662150607, 0.0572076812, 0.025951039, 0.0151054431, 0.0351088792, -0.0822437405, 0.0275274795, -0.0768966376, -0.0347387046, -0.0416953154, -0.0541978143, -0.0415271744, -0.00956887193, -0.0140640242, 0.0360040143, 0.0754575208, -0.0657078475, 0.00134004222, -0.0480610393, 0.0169862732, -0.0361623876, -0.00762642454, 0.045965109, 0.0155688422, -0.0478618778, 0.0404863618, 0.0521952212, 0.0764051676, -0.035364639, 0.0624922476, 0.0636213645, -0.012511746, -0.0267468691, 0.0108778886, 0.0398588628, 0.0618397258, -0.0671714097, 0.00876911823, 0.0240894817, 0.00436718762, -0.030827377, 0.028007362, -0.00650598528, 0.0152190011, -0.0599555299, -0.0519342199, 0.0786584616, -0.0170836579, 0.0616744831, 0.00312615349, 0.0496296957, 0.0548567958, -0.0680233762, 0.0490894429, 0.00486194482, -0.018604992, 0.00611452525, 0.0164435096, 0.0839775875, 0.07085976, -0.0860002264, 0.0537051521, -0.0330470391, -0.0282300971, 0.0423179939, 0.0854149162, -0.0745118335, 0.0165621191, -0.080784507, -0.0402241871, 0.0218298826, 0.00415241951, 0.00138620252, 6.003430e-02, 0.0771631524, 0.0703488439, -0.0604286604, 0.0553887859, -0.0449789502, -0.0110294968, 0.0259365793, 0.0268729534, -0.0617539473, 0.0600920245, -0.0251200292], [-0.0110735567, -0.0131979985, -0.053906884, 0.0812809914, -0.0946643725, 0.0629744902, 0.0440273546, -0.0215642285, 1.099340e-02, 0.066535525, 0.0349641964, 0.0277882069, 2.433830e-02, 0.0310842674, 0.0239826292, -0.047796417, -9.77893359E-4, 0.0405392386, 0.0534867384, 0.00194539141, 0.0415591933, 0.00850919261, 0.0609996468, 0.0708475038, -0.0448743701, 0.0214122478, -0.0185912605, -0.0690847114, 0.0495467521, -0.0138425445, -0.0139557505, 0.0320306122, 0.0524875782, -0.0675981864, -0.0775670483, 0.0496612154, 0.0919304415, 0.0607980639, 0.070790492, 0.00999326817, 0.0515576415, -0.0350831486, 0.0225468166, -0.0305053294, 0.046233017, 0.0325360149, -0.0536250584, -0.0455652662, 0.067009978, -3.621920e-02, 0.0310742296, 0.0562969558, 0.0314685218, 0.0839194655, -0.0530287884, -0.0631347373, -0.019045528, 0.00639145728, -0.00137756974, -0.0528453849, 0.011551722, 0.0578375943, -0.0112047922, 0.0671812668, -0.0706196576, -0.0622586831, 0.0728738531, 0.0210134052, 0.0521790572, -0.0230015106, -0.0180570595, -0.00650230516, -0.0817037597, -0.0304734614, -0.0671324804, -0.00533382408, 0.0162769184, 0.0343521237, 0.0790837928, 0.0219668411, 0.0293107741, -0.0341567546, 0.0241359733, 0.0811191573, 0.0101496466, -0.0477818288, 0.0645711869, 0.0524958298, -0.016300004, 0.0139159458, 1.71876061E-4, -0.0693731159, -0.00110968959, 0.0258802399, 0.00399614265, -0.0375819243, 0.0594663247, -0.0596147031, 0.0758561343, -0.0044407337, -0.00457145693, 0.0925961211, 0.077298753, 0.021544151, 0.0260504484, 0.013514136, 0.0499048866, 0.0697917342, 0.0483752042, 0.0472512282, 0.014207148, 0.030204583, -0.0623656772, -0.0705377087, -0.0401050746, 0.0163286012, -0.0283429753, 0.0328654461, 0.0315080658, -0.0226571411, 0.0585543141, -0.0838046148, -0.0381830111, -0.062873438, 0.0796139314, -5.017540e-02, 0.00352366082, -0.0219180454], [-0.00836684648, -0.0677255094, 0.0139484713, -0.0222189743, 0.0420458131, 0.0625818669, -0.0351730064, -0.0746972262, -0.0656004325, 0.0711200163, -0.0763573125, -0.0767625123, 0.0113994116, -0.0123623339, 0.00170911825, 0.0570366383, -6.715260e-02, 0.0297868904, -0.0148826865, 0.0410217308, 0.065397732, 0.0103622936, -0.00793639384, -0.0316545293, -0.0718948692, 0.013710876, -6.865850e-02, -0.0400232151, 0.0289290678, 0.0856158807, 0.0132923238, -0.00648151524, -0.0695169494, -0.0493769981, -0.0745626912, -0.0297479723, 0.0224114805, -0.0443392806, -0.0685623363, -0.0684182867, 6.901230e-02, -0.0190555174, 0.0256676394, -0.0236862116, 0.0100510195, 0.0810643882, 0.0581112429, -0.00753604481, -0.0545318387, -0.0172902867, -7.055470e-02, -0.069417581, -0.0273321345, -0.0450403467, 0.0770015493, 0.012645415, -0.0314139947, -0.0199249201, -0.0760290399, -0.0607218519, 0.0627919137, 0.0494463146, 0.0709860697, -0.0233178269, -0.0807363242, 0.0472701751, 0.0478508547, 0.0432690978, -0.0654076039, 0.00445713103, -5.93588411E-5, -0.052225031, 0.0659129247, -0.0399923958, -0.0263105109, 0.0382634811, 0.0626153499, -0.0471685976, -0.0174803901, -0.0260493569, 0.0571358576, -0.0372542664, -0.00382850668, 0.0423447639, 0.0415415876, 0.0286744386, 0.00995393469, 0.0708515421, 0.040656615, 0.058673963, -0.00616073469, 0.0245613344, -0.0131740831, 0.0194027293, -0.0332324579, -0.0138764065, -0.052007772, -0.0279390831, -0.0414628424, 0.0489796549, 0.055216562, 0.0793519839, 0.0673408434, -0.00654149801, -0.0802426114, 0.0446078368, 0.0607780479, -0.0067799734, 0.00803828053, -0.043861527, 0.0388335325, -0.0155448942, -0.0279456936, -0.00428279722, 0.0202634633, -0.025493525, 0.0372156762, -0.00763500948, 0.0163827688, -0.0375432149, 0.0920451432, -0.052571021, 0.0313876048, 0.025690401, -0.0328067094, 5.441160e-02, -0.034348134, 6.457090e-02], [0.041210454, -0.0231644735, 0.0799103379, 0.0417977609, 0.0550302342, 0.0467672944, -0.017238671, -0.0849176719, 0.0551731959, 0.0379816554, -0.00728589529, 0.0237470679, -0.00730013615, 0.0548785925, 7.409430e-02, 0.091699548, 0.0739756301, -0.0110312086, -0.00717872754, 0.041187223, 0.0567589626, 0.0664695725, 0.0339160189, 0.0510454252, 0.0379222743, -0.0440246277, -0.0315004885, 0.0616001636, 0.0193140414, -0.0317015164, -0.051781483, -0.0533326119, 0.0267073456, 0.0376213752, 0.0445193723, -0.0886209979, -0.0575733855, 0.028593652, -0.00922745373, 0.0184199587, 0.047991775, -0.0384561382, -0.0460630916, 0.0575700141, 0.053584449, -0.0453499444, -0.0577639379, 0.0636765137, 0.0262523964, 0.0613908097, 0.0132656675, 0.0553632379, -0.0648574308, 0.0820115134, -0.0619410202, 0.0160142854, -0.0709518939, 0.0717322528, 0.0761931092, 0.0100985207, 0.0250509903, 0.0909438878, -0.0732058361, -0.0369505733, 0.0633892268, 0.0412692912, 0.0762391835, 0.0235697832, 0.0519807413, -0.021083666, 0.0256066751, 0.00887308456, 0.00802742783, 0.0168374777, 0.013389349, 4.518480e-02, 0.00620122487, -0.00606830604, 0.0525193736, 0.0414819233, 0.0874711871, -7.653200e-02, 0.0146379853, 0.061568547, -0.0799172744, -0.0569220297, 0.0126027595, -0.0555971228, 0.0116179595, 0.0633971095, -0.0608746223, 0.0271732341, 0.0448404476, 5.697300e-02, -0.0475576818, -0.0501110703, -0.019315131, 0.00959353894, 0.0718734562, 0.0169494636, -0.0437125079, 0.0771854445, 0.0569800027, 0.0459217727, 0.014937615, -0.0582045801, -0.0022654396, 0.0669034347, -0.0552625395, -0.00100505573, -0.0498549417, 0.0541427024, 0.0499869399, -0.0626923367, 0.0298161935, 0.0151178297, 0.0733733401, 0.0255228505, 0.0150664262, 0.0642080456, -0.0456012376, -0.0204395447, -0.0371298231, 0.0698405653, -0.0200330541, 0.0792051181, 0.0543192662, 0.0260085184], [-0.0092311073, 0.0279376674, 0.0461119749, 0.0835012868, -0.0991025716, -0.0161402021, 0.0763691738, 0.0702379867, 0.00164296292, 0.0542727783, 0.0209762659, -0.056067571, 0.0184065886, 0.0427352414, 0.0704287588, -0.0626958832, -0.0246452987, -0.077558048, 0.00236823922, 0.0282815434, 0.0335410833, 0.0356118977, -0.0760909095, 0.0771288425, -1.515410e-02, -0.0300700758, 0.0694402531, -0.024864044, 0.0128958551, -0.0457180142, -0.0241246372, 0.0459361225, -0.0347336121, -0.085426487, 0.0588472299, -0.048129648, 0.0238709059, -0.0503118373, 0.0596047193, 0.0557746887, 0.0554516278, -0.0385404266, -0.0408421829, -0.00714285159, -0.00695987045, 0.0800515413, 0.0450960398, 0.0590139441, -0.0699332133, -0.0159551129, -0.0350909457, -0.0617376305, 0.0681293904, 0.0768079832, 6.754980e-02, -0.0226378813, 0.0692939162, 0.0441713408, 0.0252869558, 0.0166865494, 0.0260193739, -0.041493725, -0.0110236481, -0.0785892903, -0.0370963328, -0.0214755107, 7.605700e-02, 0.0526807718, -0.0480705388, -0.0258098748, -0.053401649, -0.0400930196, 0.0211074408, 0.0724262744, -0.0172166917, -0.0267739091, 0.0139783407, -0.0816845819, -0.0407626852, -0.0593464412, -0.0201525297, -0.0551764332, -0.00233618519, -0.0356452651, -0.00928104482, -0.0132388724, 0.0128094144, -0.00715403724, 0.0199121032, 0.0246414337, -0.0339652821, 0.0436968058, 0.0670757517, -0.0535631776, 0.0630516707, -0.0066928044, 0.0497000553, -0.0565563776, 0.0201483909, -0.0340577178, -0.0605268925, 0.0449877903, -0.00108162372, -7.492250e-02, 0.0595968775, 0.0229588542, 0.0522864647, 0.0329243168, -0.0328811556, 3.73217918E-4, 0.0645800382, -0.0733359456, 0.0456385724, 0.0462261848, 0.0288851932, -0.071078211, -0.0110787041, 0.0585633069, -0.0456160381, 0.0510620363, 0.0164997894, -0.0602327548, -0.00408076216, 0.0637297183, -0.00225129281, -0.0131610129, -0.075945057, 6.283010e-02], [0.0795891582, 0.0658252165, -0.0541591682, 0.0680394247, -0.0123760039, -0.0452831425, -0.0067453878, -0.0596360825, 0.0181972142, -0.0554774962, 0.0435124636, 0.00614208914, -0.0257182475, 0.0484243296, -0.0464010946, -0.00331138493, -0.0644188672, 0.0147084398, -0.0356126688, -0.0749802962, -0.080679439, 0.0291633569, -0.02657203, 0.0616636313, -0.00722850254, -0.00601922255, -0.0586048402, -0.00767098973, 0.00823322218, -0.0214895774, 0.0539986528, -0.0420551859, -0.0351829045, 0.0151536902, 0.00715418159, 0.0108899157, 0.0876458287, 0.0632786304, -0.0619456768, 0.0356546938, -0.0649088472, 0.066946663, -0.0311471969, 0.0638823211, -0.0568357483, 0.0733537227, -0.0505864844, -0.0312143192, 0.0360242203, -0.00834897719, 0.0118555855, -0.0661278516, -0.064801909, -0.0209270976, 0.0765712932, -0.0106906416, 0.0389953479, -0.0493147783, -0.0139949219, -0.0589863583, -6.998070e-02, 0.0688026249, 0.0666280686, 0.0138019994, -0.0345165022, 0.0669708326, 0.020617526, 0.048616983, -0.0317293145, -0.0449066423, 0.0530902445, -0.00707399612, -0.0411043838, -0.0073703425, 0.0347905494, -0.0494506806, 0.0554781482, -0.0596140698, 0.00832313671, -0.0512565076, -0.0726268366, -0.0304647442, -4.867160e-02, -0.0157552138, 0.0634676442, -0.0549795628, 0.0703888088, 0.0169559903, -0.0120936939, -0.0216834433, 0.0513046384, -0.0778429806, 0.0227624923, 0.0312887356, -0.00913970638, -0.0209343061, -0.0414852463, 0.0113350032, 0.0757198706, 9.631650e-03, -0.0646941661, 0.0684226826, -0.0393282697, -0.0515948609, 0.0620775185, -0.0581309088, 0.0588496849, -0.0714092925, 0.0345902853, 0.0639084279, 0.0149946073, -0.0668621734, -0.0682831779, 0.0495602414, 0.0746020526, -0.0366587453, 0.0182029456, -0.0465406515, -0.027553061, 0.0395688079, 4.834340e-02, -0.012139366, 0.013912308, 0.00248155557, -0.0127827497, 6.86612737E-4, 0.0782671198, -0.0707533732], [-2.90817756E-4, 0.0211494416, 0.0599823296, -0.0264219195, -0.0911100804, -0.0711979493, 0.0145767294, 0.00273033464, 0.0296071954, -0.0700620785, 0.0515100844, -0.0618509911, -0.0763561427, 0.0413418487, 0.0210217796, 0.0733835548, 0.046662204, -0.0238937978, -0.0346257091, -0.00401054695, -0.0666336492, -0.0395802483, -0.0658289716, 0.0671706125, 0.0365230516, -0.00271277735, 0.0424269326, 0.0765438377, -0.0568259135, -0.0682481825, 0.08222799, -0.0809615924, -0.0472052842, 0.0450967103, 0.0435063131, -3.923780e-02, -0.0661186054, -0.061720591, -0.0223874226, 0.0438206047, -0.00815255661, -0.0309077166, -0.0533038303, 0.0060170535, 9.11958923E-4, 0.0288419593, 0.0682385265, -0.0783984288, 0.0363838822, -0.0449173152, -0.0107093053, -0.0302824527, 0.00523106707, -4.5817878E-4, -0.00248060469, -0.0274065044, 0.0234072339, 0.0709577799, -0.0711713657, 0.0508737676, 0.0703413635, 0.0563893355, -0.0367191099, -0.0302467532, -0.0279450584, -0.0215701666, 0.00936370622, -0.0290885083, -0.00510957418, 0.0725026578, -0.0608434714, -0.0611025505, 0.0581743121, 0.0510496311, 0.017273115, -0.0427410379, -0.0741670206, -0.0424588434, -0.0223450214, -0.0349524692, 0.0807211622, 0.0251470245, -0.00189756125, 0.0329514146, 0.0479629375, 0.0469552875, -0.0234027579, -0.0367620923, 0.00728878099, -0.0720627233, -0.0659832954, 0.013773609, 0.0082912594, 0.0562012643, 0.0484436937, -6.929870e-02, 0.0324774086, 0.071943894, -0.0204190444, 7.61221803E-4, -0.0690554827, -0.0357544236, 0.0320228226, -0.0425616652, 0.0259945523, 0.0132780224, 0.0437210649, 0.0590666831, 0.0765052139, 0.0656917468, 0.0279728826, 0.0236138813, 0.0319434702, -0.0420006141, 0.0445552729, -0.0453391373, -0.0339755267, 0.0544626229, 0.0340360962, 0.00463308487, 0.0588396117, 0.0171309672, 0.0143531812, 0.00419611577, 0.0564977489, -0.00688880915, -0.0260909293, 0.0543849356], [0.0338676758, -0.0270244014, -0.0755592882, -0.0533530749, -0.0760349333, -0.0128353126, 0.0342910551, -0.0256157424, 0.0379923247, -0.0235013664, -0.0762296543, -0.0458119959, 0.0498578176, -0.0772890896, 0.047545176, -0.00469497172, 0.0163201969, 0.0527799688, 0.0517898276, -0.0626717359, -5.668830e-02, 0.0562651902, 0.0441063754, -0.0106896954, 0.0670466572, 0.0331946462, 0.0193378907, -0.0707401559, -0.0551475324, 0.0421787649, 0.0798822343, -0.0780686438, 0.0179373398, 0.04098038, 0.04564441, -0.0402720682, -0.0290959664, 0.0364941023, 0.0529942885, 0.0685066953, 0.0457511246, -4.022010e-02, 0.074720338, 0.0309422556, -0.0300202854, -0.00864740461, -0.0223322529, 0.0277317427, 0.00373060117, 0.021718502, -0.035772942, -0.0658445358, -7.603180e-02, -0.0115914354, 0.0362151191, -0.036643751, -0.0250450708, -0.0198264178, 0.00904572662, 0.0492662601, -0.0451408625, -0.0783593133, -0.0707910582, 0.038801983, -0.00340647204, -0.0463266633, 1.855050e-02, 0.0373853929, -0.0234709531, 0.0302969329, 0.0498257205, -0.018024845, 0.0372053422, 0.0657264814, -0.0313543417, -0.012561122, -0.0703807175, 0.0496308766, 0.00756461127, 0.0653331429, 0.0223138649, -0.0662796274, -0.0687629431, 0.0806897953, -0.0371186361, 0.0344317406, 0.0636077747, 7.464140e-02, 6.787890e-02, 0.0747775435, 0.0386422127, 0.0361185744, -0.0069226143, -0.0586233884, -0.0157201849, 0.0236607064, -0.012851987, -0.0791777148, 0.0616422109, 0.0345474407, 0.0791864991, 0.0640469715, -0.0775738582, -0.0412063524, 0.0658541545, -0.0481101908, -0.0227387622, -0.00682830205, -0.0455374233, -0.0324175432, 5.819610e-02, 0.0236258581, -0.0673615411, -0.0301017743, 0.0731158927, -0.0379041471, -0.0539632626, 0.0240155142, 0.0689348429, 0.0452511795, -0.0465826802, -0.0567793027, -0.0493125245, 0.070143126, 0.0487990603, -0.0616067722, -0.00534360902, 0.0435018614], [-0.055882521, 0.0190276895, -0.0772019178, 0.0709088146, 0.00568401953, -0.0272335317, -0.0157435909, 0.0722248331, -0.0335950367, 0.0138764288, 0.0609192252, -0.00753504504, -0.0069453381, 0.076163657, -0.0458561145, 0.0400892124, 0.0255092718, 0.0615462773, -0.00689821504, -0.0638535246, -0.00139626628, 0.00478243921, 0.0657825469, 0.0457799248, 0.0604361035, 0.0742488578, -0.044625815, 0.0246460903, -0.072161682, -0.0434744693, 0.0138741527, 0.0384623967, 0.0776195228, -0.0447897613, -4.2116586E-5, -0.0402737781, 0.0376064442, 0.0882886722, 0.00754732685, -0.0748013556, 0.0725547671, 0.00558652077, -0.0230848361, -0.0713351294, 0.0303533338, -0.0293335151, 0.00347321155, 0.0288948752, 0.0732005388, 0.0547914878, -0.00662645791, -0.0575255156, -0.0308441725, -0.0208646301, 0.0497300103, 0.0744462907, -0.0274086017, 0.0315124467, 0.0561381727, -0.0473168939, 0.0275649708, 0.0384761803, 0.0258456822, -0.029359784, 0.0120485965, 0.0401402973, -0.0472228192, 0.0797603875, 0.00198641419, -1.645590e-02, 0.0430890135, -0.0219467916, 0.0390726104, -0.0154190613, 0.0043920991, -0.032485947, 0.029027123, -0.0562012307, -0.071778819, -0.061613474, 0.0248278771, 0.0471403413, 0.0388165787, 0.0421570502, 0.0600748472, 0.0222696941, 0.0574951358, -0.0231962055, -0.0439106524, 0.0183066875, 0.0670017451, -0.0263661053, -0.0494472273, -0.00975106842, 0.0222457536, 0.0349841081, -0.0250580106, -0.0221246667, -0.0779448301, 0.050022278, 0.020955367, 0.0018157193, -0.0160517246, -0.052631177, 0.0693451315, -0.0717828721, 0.0624039844, 0.0513672456, -0.0263074078, 0.053542152, -0.0178008191, -9.210180e-03, -0.0695660338, -0.0400750302, -0.0535436496, 0.00344173121, -0.00345702493, -0.00560943224, -0.0485972054, -0.0364975557, 0.0384548418, -0.00621727295, 4.79028269E-4, 0.0116926879, -0.0222775172, -0.0590395555, -0.0678471401, -0.0462134965], [-0.0196723342, 0.0480557531, -0.0169896502, -0.0349453352, -0.0595596619, -0.0378552675, -0.0379483104, 0.0129894679, -0.0723053589, -0.0154825794, -0.0690215081, 0.0288903117, -0.0762864202, -0.0522049703, -0.0797904879, 0.040681351, -0.0131961154, 0.0765796378, 0.0348424055, -0.00383310625, 0.0309214126, -0.0693810359, 0.0219401438, 0.0589414798, 0.0614541061, -0.0387463532, -0.0614962615, -0.059633825, -0.0554403551, -0.0659992248, 0.0131357051, -7.104560e-02, 0.0760746896, 0.0483862832, -0.00679219887, -0.017743703, 0.0599513911, -0.0015802359, 0.073292762, 0.0201221984, -3.969220e-02, 0.0613103881, 0.00830944162, 0.011699046, 0.0100711863, -0.0534062237, -0.0299114101, -0.0598864593, 0.0630642473, 6.26436085E-4, -0.0408522189, 0.0596940443, 0.0210100822, 0.0428892933, 0.0235111136, -0.0196233373, -0.068048425, -0.0489166565, 0.0515980758, 0.0659457445, 0.0683858692, 0.0310023129, -0.00631823298, 0.0184257459, -0.0764447748, 0.0516133122, 0.0637316778, -0.0777708664, -0.0697681531, -7.159100e-02, -0.0743916109, 0.0285117086, -0.00357808103, -0.00987923611, -0.0793598294, -0.0786967799, -0.0242352057, -0.0327209607, 0.0770757794, -0.0078473622, 0.0543525256, 0.0765892491, 0.0642848536, 0.00838058721, 0.075995028, 0.0419334881, -0.0538378134, 0.0132925762, -0.0334443152, 0.0195137709, 0.0756358802, 6.315930e-02, 0.0427778326, -0.0189517681, 0.0255072061, 0.079571262, 0.0743318498, -0.0271557588, 0.0105691934, -0.0227287356, -0.0214749072, -0.0433794409, 0.00154363411, -0.0691803247, 6.821940e-02, -0.0345716812, -0.0344986022, -0.00492091849, -0.0210005976, 0.00661718706, 0.0317715742, -0.0696093142, -0.0643610135, 0.00685537094, -0.0747470557, 0.0452398211, -0.056045305, -0.0458462723, 0.0473331809, -2.469290e-02, 0.0676860511, -0.0486604944, 0.0449813753, 0.0129142636, -0.0730368793, 0.00708772661, -0.0565227941, 0.0218725149], [-0.031232439, 0.0270939022, 0.00587763824, 0.0318066739, -0.0392608568, -0.0575050674, 0.076988332, 0.0350454524, -0.0576425865, 0.00379055133, 0.0331695713, 0.0627761111, 0.0330799855, 0.0764249936, -0.0422718041, -0.0172415879, -0.00268916902, -0.00786438677, -0.0110026384, 0.0705967471, 0.0293657202, 0.00260024448, -0.0559964925, -0.0244883467, -0.0179593693, 0.0703814923, -0.0623836331, -0.00605035946, 0.0751611739, -0.0735514163, 0.00574524468, 0.03729764, -0.0656217113, -0.0360228568, -0.0238731857, 0.0226912498, 0.0365245901, -0.0456451662, 0.018568743, -0.0464574136, -0.0672478377, 0.0433925763, 0.0577548705, -0.0381089039, -0.0619898475, -0.0288314838, -0.0615756623, -0.0217053648, 3.92101756E-5, -0.00262394873, -0.0161665156, 0.0222721715, -0.0125331907, -7.94607214E-4, 0.0208771378, 0.0276704226, -0.0718798116, 0.0291222967, 0.0624837726, -0.0634282753, 0.0139392298, -0.0373301432, 0.0203651246, 0.0450301804, -0.0799619331, -0.079577662, 0.0125138229, 0.0425543971, 0.0645811483, -0.0201103315, -0.0340563133, 0.00158430694, -0.04093986, 0.0782669559, -0.0754795521, -0.0747616962, 0.0583946817, 0.0470091328, 0.0222126357, -0.0469079167, 0.0245110318, -0.00250063394, 0.0183939748, 0.0434767939, 0.0740839615, -0.0358110443, 0.0452386066, -0.0680222139, -0.0239370558, -0.00122088951, 0.0219827034, 0.0710156038, 0.0623868033, 0.0150480689, 0.0607872419, -0.0297688209, 0.0233599059, 0.00180797675, -0.0499252751, -0.0623042583, 0.0228025708, -0.0318922661, -0.0483592935, -0.0152116707, -0.0798685178, 0.0321090557, 0.0545972474, 0.0534572341, -0.0615477934, -0.0323142335, -0.0412578657, -0.053349115, 0.0140318219, -0.0440083258, -0.0264959056, -0.0230436902, -0.0389076583, 0.0362304896, -0.0163377374, -0.0370074138, 0.0714931563, -0.0152759738, 0.0318608582, 0.0462217778, 0.0435799807, 0.0616865233, -0.0108795632, 0.00520115579], [-0.0141031519, -0.0306990892, 0.0312584937, -0.0743066892, 0.0180375874, 0.068695955, 3.29623901E-4, -0.00933815538, 0.015332439, -0.0181098152, 0.0212395545, 0.00491528725, -0.032430172, 0.0458616763, -0.0231797714, -0.0644200444, -0.0114366533, 0.0754700527, -0.0369009376, 6.395470e-02, 0.0693085119, 0.0153981606, 0.01473903, -0.0751470476, -0.0731605291, 0.0143665699, 0.0516128764, -0.0585776381, 0.0168123804, -0.0276676249, 0.0663079768, -0.0416004583, 0.0735278502, 0.0531860739, 0.00406847475, -0.07055825, -0.0757204592, -0.0705118328, 0.0446116328, 3.74829513E-4, 0.0205506701, -0.00213883584, 5.29007928E-4, 0.0351932533, -0.0532652177, -0.0401264206, -0.0184953026, 0.0431795307, 0.0117761046, 2.455430e-02, 0.0678394809, -0.0173284914, -0.0359454229, 0.00714848703, -0.011203018, 0.0114403088, 0.0126105761, 0.063514106, 0.00704271393, -0.0724959075, 0.0711099803, 0.0338120311, 0.0754671171, -0.044743374, -0.0172640681, 0.0298006535, 0.00113897247, 0.0628611222, -0.00862573553, -0.0784686431, -0.0678842515, 0.0434161611, 0.0368122198, 0.00377405109, -0.0477066189, 1.587970e-03, 0.0404683687, -0.00798137952, 0.0267050546, 0.0201279316, 0.00760631543, 0.0287020523, -0.06287352, -0.0106377928, -0.0726886764, -0.0375438482, 0.0380847417, 0.0466046892, -0.0663398579, 0.0580172837, -0.0249694735, -0.0503661633, -0.0511749238, -0.0269372016, 0.00201901654, -0.0358377323, -0.0243011117, -0.0228793137, 0.0278039742, 0.027979102, -0.0581919923, 0.0408906601, 0.0491258278, -0.00143890537, -0.07904239, 0.0347983763, -0.0249343887, -0.0375303701, -0.00126990373, 0.0735772252, 0.00800172425, -0.00892708544, -0.0600119047, -0.00417380407, 0.00534447469, 0.0242202561, -0.0633864328, -0.0146986907, 0.00524812192, 0.0233099796, 0.0281783137, -0.0636352226, 0.0778927803, 0.0100138821, 0.0248566903, 0.0340658389, 0.0284599084, -5.993470e-02], [-0.0756248385, 0.0732352063, 0.0253104605, 0.00194726977, -0.0602454767, 0.0100597851, -0.0789786726, -0.0509458408, 0.0407475531, -0.0486948825, -0.0688450187, 0.0479630157, 0.00633893348, 0.0412559472, 3.972200e-02, 0.00910045113, -0.00693458552, 0.0285781473, -0.061454311, 0.0559253879, 0.0799287558, 0.00506649073, -0.0740417689, 2.569070e-03, 0.0251965411, 0.0452955961, 0.0383002274, -0.0587495528, -0.0118231103, -5.690020e-03, 0.0574855134, 0.0136370417, 0.00267342059, -0.0584279411, 0.034115538, -0.0554272756, 0.0623624324, -0.0728029236, -0.0157364234, -0.0565254092, -0.0653158799, -0.0410099179, 0.0404557735, -7.94977241E-4, 0.00539701665, 0.0304436218, 0.0435322523, -0.0348249264, -0.0544254258, -0.0526298285, -0.037129283, -6.513290e-02, 0.0255459119, 0.0356496871, -0.0786791667, 0.00518776104, -0.0662895963, -0.0118468041, -0.00742317736, -0.00111521629, -0.0498587117, -0.0409902707, 0.0365653336, 0.0110840751, -0.0665137917, 0.0575568601, 0.00286384323, 0.0283707157, 0.0720038638, -0.0616839603, 0.0112976069, 0.00418824796, 0.0329622254, -0.0769745335, 0.0280048549, -0.0706788749, 0.0085955765, -0.0435931757, 0.0575294867, 0.0634875223, -0.0708652362, 5.59818363E-5, 0.0393022597, -0.0268087201, -0.00209788652, 0.0246361587, -0.0787140578, 0.0575347729, 0.0394623503, -0.0618470386, -0.0184800457, -0.0744774565, 0.0361811332, -0.0047531575, 0.0284612719, -0.0290590972, 0.0285951961, -0.0747639611, -0.0459714532, 0.0798928439, 0.0133938938, 0.0431920923, -0.0606138296, -0.0770601779, -0.0291177519, -0.0803016424, -0.0401270725, 0.0349803865, 0.0278829671, -0.0798496455, 0.0541544296, 0.0550985709, 0.00403105887, -0.042709671, 0.0426030159, -0.0188034382, 0.046237357, -0.0204637293, 0.0261743143, -0.0272865798, 0.0629603788, -0.02028949, 0.0783807337, -0.0560305789, 0.0721714944, 0.0513730869, 0.0725210458, -0.0605370142], [-0.0148020163, -0.0782393813, -0.0644906089, -0.0468950421, -0.0259702038, 0.0420070738, 0.0528495498, -0.0263735875, 0.0334964357, 0.0480817817, 0.0482556857, -0.0674735904, -0.0164528117, 4.13251197E-4, -0.05517032, -0.0367734097, 0.0180720463, 0.0212963596, 0.0384887159, -0.00573636359, -0.0540514104, -0.0453915633, 0.0461038463, 0.0130796731, -0.074171856, 0.0775550455, -0.0155336633, 0.00603076257, 0.0156782568, -0.0771713108, 0.0368849635, 0.0645111352, -0.00911367685, 0.0435052253, 0.0718613192, 0.0311709121, -0.0682052448, -0.0611314736, -0.0667419583, 0.00371877896, -0.0515987799, -0.0381254107, 0.0483407266, -0.0102499304, 5.680480e-02, 1.6244073E-4, -0.0405365676, -0.00932035408, -0.0161919519, -0.0157671403, 0.050166212, 0.0416625179, -0.0705830678, 0.0238922685, -0.0293982718, 0.034674786, -0.077606149, 0.0428584889, -6.762990e-02, 0.0160618797, -0.0435453355, 0.00525668589, 0.00992754101, 0.020212559, -0.0630906075, -0.0768075883, 0.0567527153, -0.0176463127, -0.0791662037, 0.0347349942, 0.0499128029, -0.0530093536, -0.0350775085, -0.0316292197, 0.0525940433, 0.00403523166, 0.0548201129, 0.0799747332, 0.0281714462, 0.0016128188, 0.00113157823, -0.0356061906, -0.0775100365, -0.0472770743, -0.0232184716, 0.0293243434, -0.0731546506, -0.0163986608, -0.0446776487, 0.0320461765, -0.0529027395, 0.0423348844, 4.891650e-02, 0.0435148031, 0.034511447, -0.00894294214, -6.336410e-02, -0.0477146916, 0.0477222502, 0.0256183334, -0.0249330606, 0.0262603369, -0.0795743763, -0.076719731, 0.0437018275, 0.00838942639, -0.0263401084, 0.01184614, 0.0577442423, 0.0581289344, -0.0778031423, -0.0654830634, 0.0791040286, -0.0668290928, 0.0516876876, -0.0306312852, 0.035947226, 0.0408847779, -0.0289990604, 0.05247676, -0.0230485611, 0.00282498472, 0.0452707522, 0.0733083114, -0.0378544331, 0.0416544303, -0.0522499196, -0.035177011], [-0.029464338, 0.0671793744, 0.0798030272, -0.00322408229, 0.0114608034, 0.0310979038, -0.0749469548, -0.048433546, 0.0336857513, -0.0709962398, 0.0306220204, -0.0616097376, -0.0592674389, -0.0240004174, 0.0304254517, -0.0594564751, -0.0306073651, -0.059107244, 0.016723007, -0.0803449675, 0.0189384595, 0.0718546882, 0.055735819, -0.0802354366, 0.0763495192, 0.0605138317, 0.0536406413, 0.035343647, -0.0280457959, -0.016352199, 0.0703585371, 0.00847874581, 0.0717796907, -0.0685141459, 0.0195108578, -2.717070e-02, -0.0204855427, 0.0725619271, -0.0639895797, -0.0794262066, -0.0279399194, -0.0622188523, -0.0482945628, 0.0556311682, 0.0025395602, 0.00643837451, 0.0572203919, -0.0391643308, -0.0462796651, -0.0633282363, 0.0259257928, -0.00185626745, 0.00864097476, -0.0656655356, 0.0166915655, 0.027848199, 0.0273604468, -0.00576745719, -0.0254163481, 0.0268031955, -0.0664218962, -5.981230e-02, -0.0635356158, -0.0151902735, 0.0602164343, 0.042963706, -0.065851666, 0.00205202401, 0.0136162341, 0.00496587902, -0.0357017107, -0.0543308184, 0.0496687666, 0.0315857306, -0.0264592245, -4.960500e-02, 0.0713726506, -0.0419801623, -0.0259079449, 0.0656521544, 0.07082472, 0.0573764071, 0.0655650571, -0.00936657935, -0.0735202879, -0.0603702441, 0.0755855665, -0.018228285, -0.0124260336, 0.00757187605, -0.00227395445, -0.0042360574, -0.0651033148, -0.0189487487, -0.0210643969, 0.0764529184, -0.0121576041, -0.0396296866, 0.0575569496, 0.0238126814, 0.0400924906, -9.169310e-03, 0.00576598197, -0.0413106121, 0.0458424613, -0.0768320337, -0.0614382029, -0.0396029055, -0.0415613726, -0.0417910516, -0.0206172392, -0.0173132122, 0.0246627107, 0.00317250937, -0.0176967904, -0.0721999108, 0.05402904, -0.0133391917, 0.0334308892, -0.0557993278, -0.0343284421, -0.0451921932, -0.0135655105, 0.0581292734, 0.00279535353, -0.0155416727, -0.0277430154, 0.0278827548], [-0.032833185, -0.0622672736, 0.0119580925, 0.0631419793, -0.0102669373, -0.00409186631, -0.060939312, -0.0534119755, 0.0575278029, 0.00770493597, -0.0251433291, 0.0295521319, -0.0123370588, 0.0070547089, -0.025173746, -0.0152304024, -0.0659135729, 0.00750619918, -0.0670735389, 0.0649766102, 0.0190689191, -0.0381032564, -0.0456763096, 0.0104996115, -0.0343158916, 0.0391679853, 0.0728420541, 0.0334882662, 0.0626534298, -0.0322173163, 0.0757007375, 0.00865553319, -0.0166596919, -0.0751491785, -0.0658913478, -0.0696998909, 0.00167386234, -0.0250946172, -0.0563314036, -0.0655336081, -0.00553260744, 0.0465719476, -0.0315667205, 0.0230643675, 0.0559048429, 0.0399952233, -0.0268955901, -0.0801604613, -0.0474921577, 0.0297807083, 0.0686981305, -0.0694517642, 7.481510e-02, -0.0379647575, 0.0766218081, -0.034130957, 0.0628969148, 0.0181002989, -0.0193477571, 0.0234675854, -0.0583426282, -0.0539092235, 0.0273893923, 0.00987516343, -0.0448026061, -0.0146758333, -0.0560474619, 0.0206525922, 0.0235305503, -0.0202614926, -0.0309095085, -0.0299687758, -0.0121295005, 0.0791022107, -0.0274683349, -0.039457269, -0.0455117039, -0.0286116339, -0.0562383458, 0.0197744593, -0.0262470059, 0.0254850164, 0.0518096611, 0.034392722, -0.00640728325, -0.0335963108, -0.040102914, 0.0727676228, -0.0759327411, -0.0280520022, -0.00559634715, 0.0338504165, -3.890460e-02, 0.0515297279, -0.0133691058, -0.0390521884, 3.441330e-02, 2.7783215E-4, 0.0412505865, -0.0748744756, 0.0337239802, -0.0116389468, -4.876210e-02, 0.0623913929, -0.0206165053, -0.0222708955, -0.0720558167, 0.00671587884, -0.00508591533, 0.0352139026, 0.0711905733, -0.0325895809, 0.0641544238, -0.0708210692, -0.0424921624, -0.0603906289, 0.0688606724, 0.0574879125, -0.01734332, -0.0144282058, -0.0066666454, 0.066788055, 0.0553107485, 0.0518606082, -0.0320756435, -0.00609555095, -0.0663032159, -0.0510974377], [0.0549705103, -0.0800917148, 0.0611854568, 0.00445795804, 0.0348719805, 0.0187617838, 0.0113534406, -0.0506156236, 0.0765970722, 0.00761455297, -0.0440560132, -0.078600794, 0.0776275917, 0.0781288966, 0.0127925724, -0.0325164609, -6.534400e-02, -0.0515039451, 0.0494507179, -0.0540028401, -0.015436098, -0.0367846563, -0.00603736192, -0.0361834876, 0.0620270297, -4.60408628E-4, 0.0690884516, 0.0171078593, 0.0650042221, -0.0234356597, -0.0293313861, -0.0668921471, 0.0122521445, -0.0431308672, 0.0664729849, 0.0715341493, 0.00981411337, 0.0412013084, -0.0246260501, 0.0308374316, -0.064962536, 0.0313468799, -0.021151267, 0.0254673436, -0.0326270945, 0.0271187946, 0.00738052279, 0.0249560773, 0.0648291931, -0.0777357072, -0.0643164366, 0.075886406, 0.0684593543, 0.0461151376, -0.0728286356, 0.0706632361, -0.0472069941, 0.0377183482, -0.0456510745, 0.0080512464, -0.0393598229, 0.0434539169, 7.137620e-02, 0.0476443842, -0.00131046027, -0.0246762112, 0.0409312546, -3.858380e-03, -0.057082288, 0.048680909, -0.00180451572, 0.0739499852, 0.0111639202, -3.51086259E-4, -0.070428133, 0.0714712366, 0.0548102185, -0.0722834915, 0.0300431475, 0.0387816876, -0.00207041949, 0.0808882191, -0.0407606103, 0.0757487789, 0.0216678903, 0.0135234892, 0.0309553221, 0.0409917235, 0.0707235858, 0.0506659076, -0.0810666158, -0.0338905044, -0.0705021247, -0.0403843261, -0.0527883545, -0.0320387483, -0.03850035, -0.0787477642, 0.0357365608, -0.0683446824, 0.0189270899, 0.0290056318, 5.27143478E-4, 0.0354179591, -0.0418702811, 3.372240e-02, 0.00713567435, -0.0604493767, 0.0322393402, 0.0699968114, -0.0160940886, 0.00983458757, -0.0408997312, -0.0714086816, 0.0548166409, 0.0707563385, -0.0471183099, -0.0748761743, -0.04391554, -0.0156322271, -0.0710888281, -0.0693063363, 0.0708671287, 7.021830e-02, 0.0725321099, 0.00292412192, -0.0298828371, -0.0624194294], [-0.0100320727, -0.0682521537, -0.0734418556, 0.0727861896, 0.0368951559, -0.0112641901, -0.0321769193, 0.0670640096, -0.0778150334, -0.074086517, -0.0553585142, 0.0319480896, 4.60274518E-4, 0.0220284909, -0.00213148445, 0.0256945491, -0.0783148333, -0.0108375922, -0.0353844091, 0.0735523477, 0.080580838, -4.345370e-02, -0.0634483248, -0.00685076415, -0.0391752198, -0.0694439337, 0.00497247279, 0.0057740882, -0.0601527095, -0.0773302838, 0.0144075528, -0.0204964131, -7.084660e-03, 0.0203721225, 0.0617621467, 0.0260854885, 0.0324986354, 0.0393985361, -0.0392259806, -0.0281753018, 0.0186391026, 0.0400555357, -0.0683260411, 0.0273151174, -0.055511944, -0.0309687406, 0.0352683961, 0.072111778, -0.0174744353, -0.0516595021, 0.0476352647, -0.0498015769, 0.0468439981, 0.0511975512, 0.0452227071, 0.0244061947, -0.0376889743, 0.0810716971, 0.0587763414, 0.0280892849, -0.0179741532, 0.0611503944, 0.0238573104, -0.0316082761, -0.0357154049, 8.2091242E-4, 0.0491333082, -0.0400854535, -0.0762805417, -0.0212709494, 0.0181193873, -0.0549898893, 0.00869825482, 0.0589161292, -0.0161307529, 3.741750e-02, 0.0132268593, 0.0751167461, 0.0144112483, -0.00347617269, -0.0314077772, 0.0476550087, -0.0296234898, -0.0267245434, 0.0502700135, -0.0451100469, -0.0733409821, 0.0247032866, 0.0251509473, -6.72951341E-4, -0.0371750966, -0.0282619968, 0.0680024698, 0.0227219835, -0.0218206979, 0.0284722447, -0.0360258408, -0.0611488819, 0.0557789728, 0.0272812173, 4.305450e-02, -0.0583186671, -0.0112761855, 0.0440204367, -0.0471784696, -0.0302225538, 0.04964111, 0.0450750813, 0.0596847609, 0.0543614253, 0.0524052456, 0.0268144682, 0.0463119075, -0.0674431697, 0.01767838, 0.00544543564, -0.0476576351, -0.0136429593, -0.0591800511, -0.035192959, 0.0517775789, -0.011481978, 0.0584335104, 0.0308299288, -0.0575732328, 0.0584074035, 0.0499119535, -0.0654951259], [0.0708152279, -0.0805557742, 2.907090e-03, -0.060318768, 0.0312642902, -0.00534532219, 0.0143255219, -0.0101738051, 0.027381137, 0.03084784, -0.0767109916, -0.0366415903, -0.0399960317, -0.0619547293, 0.0683129057, -0.0284568481, 0.0119618773, -0.00747481734, -0.0160741284, 0.0609901026, 0.013548471, -0.0120585114, -0.0749281198, 0.0306384191, -0.0511502698, 0.0182183236, -5.602520e-02, 0.0482980832, -0.0586899593, 0.026346907, 0.0563755557, -0.0225201286, -0.0164879709, -0.0627632067, 0.0673913136, -2.153420e-02, -0.0548410416, 0.0370944962, 0.01377666, 0.039473027, -0.0429188237, 0.0254687369, -0.0616653115, 0.0708810762, 7.013110e-02, -0.0773375556, 0.0280752108, -0.0242387801, -0.00459714234, 0.0314765051, 0.0543392822, -0.00174605846, -0.024343092, 0.0554821268, -0.0474976674, 0.0470797941, -0.0290567428, 0.0472812578, 0.034734562, -7.009010e-02, 3.529770e-02, -0.0603428446, -0.0395110659, 3.886550e-02, -0.0223837905, 0.015266411, -0.0760479197, -0.0163940787, 0.039084442, -0.00973636656, -0.0520896837, -0.0390350334, 0.0316584408, 0.0179976895, 0.0628828779, -0.00801414251, 0.0343478397, -0.0258218534, -0.0175173879, 0.066099219, 0.0412234515, 0.0802162364, -9.7476691E-4, 0.0663300231, -0.0494965129, -0.0531709455, 0.0199617445, -0.0422457941, -0.0263498053, 0.0557313785, -0.080915913, 0.00986585766, 0.0689361468, -0.0396479256, -0.0226468518, -0.0303464904, 0.0401071087, -0.00248990208, 0.0538506433, 0.0411948562, 0.0810678675, 0.0165807903, 0.00304862857, -0.0330566391, 0.080350928, -0.0504673719, 0.027477093, -0.0458180793, 0.045521237, -0.00561448932, 0.0150724053, -0.0368838049, 0.0762882903, -0.0695485696, -0.00762470812, 0.0128038079, -7.069910e-02, -0.0177959576, 0.0282804295, -0.0745345429, 0.0183157697, -0.0750370919, 0.0644972101, 0.0597045049, 0.0370596051, -0.0590165071, -0.0436519757, -0.027223628], [0.0589648709, -3.775090e-03, 0.0368006527, -0.056319803, 0.0244902521, 0.00485958159, 0.0759550557, 0.011092566, -0.0387686491, -3.978180e-02, -0.0515849926, -0.0347155146, 0.038133055, 0.0298134089, -0.0396699123, -0.0620371103, -0.0613300838, -0.0772778764, -0.0201445706, 0.0172466114, -0.0117883012, -0.0163253546, 0.0504417047, 0.0580820367, 0.0713914409, -0.0166838095, -0.0348894633, -0.0492064208, 0.065909259, -0.0529105142, -0.0605306774, -0.0800305456, -0.0132973045, 0.00797215849, 0.0684297159, -0.00330108404, -0.0463043228, -0.0672449767, -0.044801116, -0.00898902118, 0.0807154551, -0.0251761079, 0.00718320906, 0.0190950632, 0.020979777, 0.00204829127, 0.0557777658, 0.0616189465, 0.0561566576, 0.0735896602, -0.0463924073, -0.0337225348, 0.0328461602, -0.0328484625, 0.0395552516, -0.0592451468, 0.0282812938, -0.0145817921, 0.0119822025, 0.0237368122, 9.1779232E-4, 0.0626426861, -0.072022751, 0.0229773223, 0.0604962781, -0.00147071481, 0.0498951301, -0.0195447169, -3.50773335E-5, -0.0305728652, 0.0525276735, -0.0430620797, 0.0308076292, -0.0324704945, 0.0352653787, -0.0275322869, 0.0311117843, -0.0515893437, -0.0240263678, 0.0133724362, 0.0136826783, -0.0675351471, -0.0421239622, 0.0144503713, 0.0806540325, -0.0669247657, -0.074792631, 0.030734092, 0.0142715275, 0.0487685874, -0.064510554, -0.00722657144, -4.89972532E-4, 0.0451096222, -0.0786571428, 5.037620e-02, -0.045923356, -0.0253204666, 0.0330395401, 0.0334724113, 0.0708060637, -0.00332052261, 0.00991544127, -0.0255976059, 0.0101311207, 0.0631440505, 0.0167502537, 0.0586949065, 0.029829137, 0.0257383287, 0.0228416845, 0.0586177185, -0.0454091132, 0.00665338337, 0.0128325447, 0.0674648359, -0.0522509813, 0.0463643447, -0.00980925559, 0.0533794835, 0.077212356, 0.0753107592, 0.066111587, -0.00365323573, -0.0753555521, -0.0597328171, 0.0424320772, -0.0467331484], [0.0765076354, 0.0376486667, -0.038464766, -0.0285839718, -0.0557910167, -0.0132533275, 0.0540192649, 0.0225043744, -0.0779823214, 0.0775519832, 0.0142386593, 0.0158794709, 0.0652702227, 0.00901089049, 0.0740551948, 0.0246621892, 0.0234359968, 0.0577151552, 0.0313540064, 0.0393309891, -9.90649918E-4, -0.0497842319, -0.0673562214, 0.0299354121, -0.0809292271, 0.045366589, -0.0288429242, -0.0328003652, 0.0497884676, 0.0205617137, 0.0528051443, 0.0040739649, 3.756600e-02, -0.0360192768, -0.0524153039, -0.0199953914, -0.0441492163, -0.0707153975, -0.0440775417, -0.0350745842, -0.0436061323, -9.71384346E-4, -0.0431946404, -0.0155822849, 0.0145325679, 0.0303226635, -0.045719035, 0.0469629057, -0.0589447245, 0.0491681397, 0.05992838, -0.00728694908, -0.0708716661, 0.00815821997, 0.0741405189, -0.0484846458, 0.0459256694, -0.0322433487, -0.00382637628, 0.0506562069, -0.0770161599, -0.0542594828, -0.0230513401, -0.0664866268, 0.0362263024, 0.0405726656, -0.0179818552, 0.0717331097, 0.0163550116, 0.0690630525, -0.0586051531, -1.477570e-02, 0.014154206, -0.00586511567, -0.064249374, -0.0688173324, -0.032298483, -0.0315692872, -0.0598151088, -0.032519903, 6.312620e-02, 0.00663326774, -0.011796427, -0.0644692406, 5.10841608E-4, 0.0079102423, 0.00466277497, -0.0684852749, -0.0409738682, -0.0640174299, 0.0572100207, 0.0342298374, 0.0267574545, 0.0723029524, -0.0355352424, 0.00525885168, -0.00531652896, 0.0561961904, -0.0587233491, -0.0671349093, 0.0321908072, -0.0050436575, -0.0644819438, 0.0143437171, 0.0178747568, -0.0571570061, -0.00243885536, -0.00338029861, 0.0278242268, -0.0487342924, 0.0231142454, 0.0688062683, -0.0113243759, -0.0610526577, 0.0496237762, 0.0514186583, 5.461160e-02, 0.0190027785, -0.0412263647, -0.0712647587, -0.0627938658, -0.00634189323, 0.00346373417, -0.018207863, -0.0661657304, -0.00517017394, -0.0699444562, -0.0477663912], [0.0232623294, -0.00392077724, 0.00692052627, -0.0103590172, 0.0310844071, -0.0672743842, 0.0151826087, 0.0550751872, -0.0325462669, 0.0206552353, -0.00265037268, 9.186330e-03, 0.0502723083, -0.0785411596, -0.0669916645, 0.0353288352, 6.573270e-02, 0.0457059108, 0.0176163092, -0.0504639186, -0.0801080837, -0.0571369044, 0.0487616211, 0.0382226147, 0.06886594, -0.0108816726, -0.00160749315, -0.0106220264, 0.0555118844, 0.0378754437, 0.0199548695, 0.00476653222, -0.00745232403, 0.0353887752, -0.0279333927, -0.0744171739, -0.0805927664, 0.0537660718, -0.0271556452, -0.0572383888, 0.0603823811, -0.0133444676, 0.02111491, -0.0595545731, -0.00618583523, 5.906840e-02, 0.0679576098, 0.0438822135, -0.0497731268, 0.00932287611, -0.0191572253, 0.0486387387, 0.0227571223, 0.0287825707, -0.0603558719, 0.0101090474, 0.00514256814, 0.0175174028, 0.0657322481, -6.32441894E-4, -0.0175970178, 9.50843387E-4, 0.0571667515, -0.0190089885, -0.0397643335, 0.0221653562, 0.0330641121, 0.0438181534, 0.0720234513, 0.0101514086, 0.0234566703, -0.00913187861, 0.0755647197, 0.0652994961, 0.0657227859, -0.0231516864, -0.076219812, 0.0338573754, 0.0202743243, 0.0153540531, 0.0547632873, 0.0379526503, -0.063520804, -4.069740e-02, -0.0758768171, 0.0634204075, -0.0206808075, 0.0603629723, 0.0602100194, -0.0594314858, 0.0691043735, -0.0429032594, -0.0608169138, 0.00766888447, 0.0786322504, 0.0212139674, 0.013835378, -0.019936787, -0.034980014, 0.068223007, 0.0424329266, -0.0700942501, -7.061870e-02, 0.00319981296, 0.0513356216, 0.0619689301, -0.0391741134, -0.0795847252, -0.0640898048, 0.0529656895, 0.0218219049, 0.0200709887, -0.0512743816, -0.0805580541, -0.00396152819, 0.017741099, 0.0154745243, -0.0632930845, 0.0618199892, -0.0252413023, 0.00952892191, -0.00992868281, -0.0566087961, 7.987420e-02, 0.0519397035, -6.100630e-02, 0.078311026, -0.0646064281], [-0.0525665842, 0.0648538172, 0.0613657869, 0.0791074261, 0.00472278753, -0.0418128408, -0.0613358542, 2.54805374E-4, -0.0594431795, 0.0308483262, -0.0662584156, 0.013199958, 0.0137471762, 0.0377578214, 0.0712131932, 0.0088409828, 0.0584592596, 0.0135557586, -7.987640e-02, -0.0361213759, 0.0802665725, -0.0766704828, 0.0229065809, 0.0371320583, 0.0391248278, 0.0342203975, -6.209430e-03, -0.0743883774, 7.091380e-02, -0.0505040735, -0.0742717609, -0.00876503996, -0.0634107068, -0.00585845439, -0.056662973, 0.0795722976, 0.0477822311, -0.0120219951, -0.0456894301, 0.0274142213, 0.0595650934, -0.0386602394, 0.032014221, 0.057155408, 0.0172256213, 0.0247181039, -0.0808610841, -0.0455373935, 0.0302839391, -7.007550e-02, 0.0325860903, -0.0204190295, 0.0272394475, -0.0315030739, -0.064053461, -0.057703007, -0.0521596894, 0.00504104327, -0.0128846876, 7.702120e-03, 0.0376179591, 0.0271035153, -0.0125430496, 0.0801996067, -0.0787884444, 0.0463303439, 0.059506271, -0.0315120146, 0.0155862626, 0.0335113779, -0.0451740064, 0.0773285776, -0.0645813643, -0.0772373602, 1.923190e-02, -0.0450590365, 0.0146092409, 0.0428698137, -0.057778988, 0.0209623147, 0.0475821681, 0.0341781788, -0.0775494799, 0.0739103257, 0.029798802, 0.0525282323, -0.0268900543, -0.0440652445, -0.0384815075, 0.0471611843, -0.0424242429, 0.0193570014, -0.0457622819, -0.0702361315, -0.0200134292, -0.0515767857, 0.0566030592, -0.0282687265, -0.0217378046, 0.0499015525, -0.0579441264, -0.019545652, -0.0590741448, 0.0646256655, 0.0468064919, 0.0747770444, -0.0405081175, -0.0805532187, 0.0522509851, -0.0481425114, 0.0530133508, -5.141620e-02, -0.0524177738, -0.0265240576, 0.0394324847, -0.0727626532, 0.0777034611, -0.0651517436, -0.0194722582, -0.0566069782, 0.0672995299, -0.0694506243, 0.00648132665, -0.0475882217, 0.00244263373, -0.0685746967, 0.0755413547, 7.356270e-02], [-0.0283114053, -0.0465051308, 0.0162098669, -0.0722052156, 0.0012669391, 0.0810739845, -0.0307136122, -0.0302364808, 0.0748415217, -0.00533631211, 0.0624319389, 0.0703838468, 0.0595202707, 0.0144454353, 0.0662339404, 0.00576435262, -0.0575066954, -0.0440486111, 0.0799674243, 0.00384599646, 6.956600e-02, 0.0694129318, -0.0617661029, -0.0516860075, -0.0439783745, -0.0384765379, -0.0278590415, 0.0404202938, -0.0188134983, -0.0622938201, -0.00127899949, 0.0768794194, 0.0731567219, -0.0615095273, 0.0172884054, -0.00467566866, -0.0689196736, 0.049072329, 0.0761177689, -0.0585644171, 0.0774468854, -0.0217231885, -0.0501679853, -0.055247739, -0.0668144226, -0.0305651166, -0.0432942249, 0.053459879, 9.80674958E-5, -0.0796486288, 0.067308858, 0.0334886499, -0.022256244, -0.0628347844, 0.0190752093, 0.0471972674, 0.0162536465, 7.305100e-02, -0.0657528639, 0.0646417663, -0.0304266326, 0.0658969954, 0.0173190832, -0.0566481911, 0.050328657, 0.0308877882, 0.0583247393, 0.0524907745, -0.0318598598, 0.0079430174, 0.00738664577, 0.035794735, -0.0181894377, 0.0499727279, 0.060937047, 0.0145870242, -0.0391219221, 0.0739630162, -0.0281571019, 0.0393744484, 0.0393332131, 0.0332158916, 0.0166407824, -0.0741858706, -0.0394911245, -0.0111310268, 0.0319518261, 0.0661079511, 0.00636275066, -0.0131264329, 0.0507903695, -0.0325664878, -0.0306765065, 0.0125052528, -0.0487295762, -0.0114487298, -0.00577800116, 0.0374389365, 0.0167326201, -0.0669215843, 0.0124872746, -6.808660e-03, -0.0674458072, -0.065134123, -0.0388833955, 0.0301743578, 0.074110873, 0.023197474, -0.0444324873, -0.0742118731, 0.0753796473, 0.0639302135, -0.0273144655, -0.00850438233, -0.0227261167, -0.00249170628, -0.046959538, 0.00541141536, 0.00647751288, 6.973880e-02, -0.0315687731, -0.0468961336, -0.0249715038, 0.0573435165, 0.00695989886, -0.043323338, 0.00608714949, -0.0237046089], [0.0107619539, 0.0509734266, -0.0184756238, 0.0753989443, -0.0489339679, 0.079126656, 0.0113193905, 0.00979788508, 0.0144189289, -0.00425433693, -0.049120497, 0.077354148, -0.0437701866, -0.0255875662, -0.0462269895, -0.0644829646, 4.444740e-02, -0.0752062797, -0.00616069417, -7.245320e-02, 0.0308303125, 0.00595421717, 0.0480211377, -0.0386817455, -0.00625492073, 0.0581200756, 0.0258799512, -0.0730558261, -0.0583160147, -0.0175362881, -0.044593554, 0.0792480856, 0.0562174059, -0.00679393625, -0.0339642428, -0.0386394747, 0.0188386552, 0.0759151429, 0.0135568986, 0.0225923508, 0.047242865, 0.0339775719, -0.0540058054, -0.020397136, -0.0658192709, -0.01137337, -0.00749281887, -0.0310192872, -0.0738756954, 0.00515214773, -0.0297025703, -0.00453971932, -0.0625808612, 0.0395232327, -0.0705134943, 0.055185914, -0.0715367049, 0.0168058965, -0.00881022308, 0.0179385729, -0.014364114, 0.00408201711, 0.0602376834, 0.0669519827, 0.00740516931, 0.0788836255, -0.0761189088, 0.0771642104, -0.0228471365, -0.0662131831, -0.0222447794, 0.0141494125, -0.0535925701, 0.067426756, 0.0141913341, 0.00382185099, -0.0777692795, 0.0306335595, -0.0760752112, -0.0418556929, -0.0365234166, 0.0319573581, 0.0308643878, 0.059181504, -0.0620379522, -0.0229941085, 0.0286395382, -0.015603832, 0.0610939302, -0.0451489948, 0.0497380495, -0.0226413812, -0.0426038764, 0.0327425078, 0.0394180976, -0.0396879241, 0.0595778227, -0.0524605326, -0.0768475384, 0.0710143372, 0.00521891797, -0.00787313655, 0.0557939522, -0.0518997535, 0.019909963, -0.0405540206, -0.0572730452, -5.8212853E-4, 0.0570953824, 0.0761728659, -0.0208090432, 0.0605710559, -0.0499445647, -0.0487431251, 0.0711197257, -0.0229467079, 0.043040406, 0.00336868875, 0.0770613477, 0.00426531397, -0.0636544898, -0.0324977823, -0.0440138765, -0.0623733289, -0.077106446, 0.0488477238, 0.0282724444, -0.0293684248], [-0.0617265776, 0.0500802398, -0.029438315, -0.0297284443, -0.020000089, 0.0691597909, 6.162070e-03, 0.0549676493, -0.0750670657, 0.0413509831, -0.0187208466, 2.186530e-02, -0.00948640704, -0.0532328747, -0.0107178446, -0.0741011277, 6.775210e-02, 0.0792052597, -0.0468310826, 0.0729712545, -7.530410e-02, -0.0610143207, -0.0156580638, 0.0681400672, -1.788010e-02, -0.0690429732, 0.0621149428, -0.0209787637, -0.0725469962, 0.0497208461, 0.0201983824, 0.0431066044, -0.0228279624, -0.072743319, 0.00774737727, -0.0767052844, -0.0108073773, 0.0396343768, 0.0164566822, 0.0620458052, 0.0689463243, 0.045435898, 0.0769278631, -4.797570e-02, -0.0185001716, -0.0149375014, 0.0389627069, -0.0554757118, -0.0613945089, 0.0625501871, -0.05948871, -0.0561643168, 0.0726262182, -0.00531853968, -0.0500879809, 0.00430020643, -0.0355095491, 0.0313088596, -0.00865170359, 0.0727622285, -0.0316971764, -0.0334719494, -0.0416258425, -0.035017278, -0.013018108, -0.0473359674, 0.0156104313, 0.0125823095, -0.00570445554, 9.75319941E-4, 0.068802774, -0.0629252493, -4.351540e-02, 0.0429143608, 0.0383792818, 0.0739251077, 0.0710575357, 0.0070088408, 0.054535538, -0.0262537301, -0.0348865576, 0.0438878872, -0.061552994, -0.055405926, -0.0708265454, -0.0345894247, 0.0688024238, -0.017173944, 0.0193135925, -0.0311268084, 0.063630119, 0.00453696726, -0.0367668755, -6.01535372E-4, 0.00900097098, -0.0375060216, -0.00114000554, 0.0189366434, 0.0285446513, -0.0350079723, 0.0248433165, -0.0262324866, -0.030070981, -0.0321759842, -0.0418227166, 0.0105450088, -0.0375536866, -0.0276802033, 0.0695372447, 3.119580e-02, 0.0672234297, 0.0218114518, 0.035318248, 0.0808507427, -0.0696362108, -0.0471910909, 0.0727095753, 0.0273760222, 0.0713512152, -0.0420127213, 0.0488372706, 0.014215962, 0.0271651149, 0.0321449451, -0.0134838577, 0.00644088537, 0.00387297617, -0.0548730418], [-0.0209865607, 0.0547756553, -0.0173828639, 0.0348824635, -0.0334332585, -0.0391263254, -0.00250263372, 0.0228797924, 0.0543518215, -0.0661949068, 0.0108929193, -0.0704698041, -0.041332528, 0.00748126395, -6.495710e-02, -0.0433096476, 0.0486775562, 0.0718242675, 0.0444870479, -6.298160e-02, -0.0102771558, -0.0624064914, 0.0259457566, 0.0764849856, -0.0585548952, 0.0465968139, -0.00924486667, 0.0652826652, 0.0135963168, -0.0233020205, 0.0344492085, -0.0617583878, -0.0388652384, 0.0665314719, -0.0312318802, -0.0511548296, -0.0182208754, -3.219790e-02, 0.0237893816, 0.0173982307, -0.0242988113, 0.0350409821, -0.0475502424, 0.0566880964, 0.0367575325, 0.0360031463, -0.00750896288, -0.0471063256, -0.0285582133, 0.0581572913, -0.0069532711, -2.824960e-02, -0.0792207345, 0.0509223379, -0.0434429869, -0.0559225045, -0.0124285277, 0.0418319553, 0.0354671218, 0.0388710685, -0.00826996565, -0.00507592922, 0.0753274485, -0.00504794344, 0.0603214689, 0.0591751337, -0.0790664479, -0.0449930057, -1.62322336E-4, -0.00312280096, -0.022359008, -0.0339920409, 1.031010e-02, 0.038552057, 0.022970356, -0.0538491197, -0.0191402454, 0.00365261198, 0.0290598627, 0.0494357608, -0.00402935175, -0.0448596105, -0.0744866654, 0.0145414798, 0.0193027649, -0.0430561975, 0.00296723889, 0.0512452684, -0.0356785543, -0.0432620905, -9.22979204E-4, -0.0197073277, -0.0379627943, 0.0434563234, 0.00919104647, 0.0787893385, 0.0604631267, 0.0470526218, 0.0726361871, 0.0598168895, 0.0327198356, 0.0161638707, 0.0114397267, 0.0364323705, -0.0587174185, -0.079586491, -0.0767934322, 0.0761007816, 0.0043153055, 0.0740169659, 0.0106943604, 0.0475087911, -0.0338577777, -0.0606206581, 0.0541389585, 0.03996443, 0.0461827554, -0.0486521237, -0.0486281738, -0.0543750152, 0.0244632103, 0.0268577058, -0.0209050719, 0.0618420616, -0.0661912486, -5.824280e-02, 0.0144548574, 0.0521231331], [0.0556543805, 0.00188493519, 0.0749343708, -0.0442062765, -0.0810940117, 0.0258659888, -0.0104132574, -0.00845314189, 0.0325382613, 0.0774724707, -0.025255125, 0.044094637, 1.82250205E-5, 0.0153500736, -0.0474273749, -0.0316708349, 0.079361014, 0.0657710955, 0.0153263183, -0.0217251256, 0.0782770962, -0.0779255107, -0.0458716303, 0.0673797727, -0.0744175836, 0.0697749257, 0.0372888111, 0.0549436063, -0.00212008948, -0.055932764, 0.0278743077, 0.0796059072, -0.0196186863, 0.0468547046, -0.0209981408, 0.03886858, -0.0310474094, 0.0592474528, -0.0547708385, -0.0393605642, -0.0296076555, -0.06741409, 4.244270e-02, 0.00417517265, -0.0142581668, -0.0673720241, 0.0179039352, 0.0649271458, -0.0806073099, 0.0485198572, 0.0229186453, 0.0101116598, -0.0407515429, 3.428350e-02, -0.0181642976, -7.330880e-02, 0.0107464883, 0.00196915376, 0.0646512732, -0.0333841406, -0.046364326, 0.0635890439, -0.0137106031, -0.0582782589, -0.0594448745, -0.0486393906, -0.0223189685, 0.0340378918, -0.0334162116, 0.0141451797, -0.0129614072, -0.036439646, 6.703770e-02, -0.0766431838, -0.0422875062, -0.0783422365, 0.0341251418, -0.0614595152, 0.0505268201, 5.88557916E-4, -0.0421488211, -0.0511430092, -0.0342577547, 0.0197610334, -0.0364386663, 0.0697303861, 0.0468480662, 0.0245939922, -0.00325530046, -0.0720176697, 0.0393973887, 0.0372704677, -0.0499172434, -0.0463917553, -8.111850e-02, -0.0322517417, -0.00313784881, 0.0428227261, 4.829560e-02, -0.040924076, -0.0518184565, -0.0793560892, -0.0477857739, 0.0650524721, -0.0355532281, 0.0462225899, -0.0767251477, 0.0510267131, 0.0052849981, 0.0103372484, -0.01437655, 0.071485728, -0.0170488097, -0.00724620838, -0.0698976368, -0.018722875, 0.0392518789, 0.0260058809, 0.0671815425, -0.0340353064, 0.0196625199, -0.046834562, -0.0745213851, -0.0187114254, 0.0475327782, -0.049141556, 0.0577730685, 0.0238043498], [0.0414061956, 0.0732500777, -0.00718155643, -0.0497017615, -0.0067379605, 0.0490998216, 0.0532665178, -0.0226629116, 0.0700647532, -0.0212333854, -0.0810787827, -0.0402927659, 0.0173899364, -0.0705286637, -0.0442313179, 0.0459256656, -0.06568259, 0.0770344809, 0.068457514, -0.0529073365, -0.0624941662, -0.0464129411, -0.0497856848, -0.0723263696, -0.0106479572, -0.0653113946, 0.0247296561, 0.063768208, 0.0509905256, 0.0402912423, 6.811080e-03, 0.03402986, 0.0379692614, -0.00966607406, -0.0343458839, 0.0370281041, 0.0804858655, -0.0360138528, 0.0432118215, -0.0337852202, 0.0249787439, -0.0401321836, 0.0217632446, -6.808650e-02, -0.0788099393, 0.0327138454, -0.0569978394, 0.0404814258, 0.0431318581, -0.0395284705, -0.0784810259, -0.00533753447, -0.0617653616, -0.0160156358, -0.00598909706, -0.0114147197, 0.0216444731, -0.0802224725, 0.0715715438, -0.0763669237, 0.0660762786, -0.0509772599, -0.0237631164, 0.0143513819, 0.0302270018, 0.00150993874, -0.00790286064, 0.00733176479, -0.00899699423, 0.0234270636, -0.0553625934, -0.0192122981, 0.030078331, -0.079184413, 0.0414310694, -0.059109658, -0.0418081954, 0.0432695709, -0.0267176237, 0.0109670646, -0.0538574047, -0.0747367516, -0.0194428973, -0.0630162731, -0.0339305811, 0.0147177316, 0.0525834188, -0.056782186, 0.038872011, -0.0785824283, -0.0482594259, -0.0131469937, 0.0310462303, -0.00562151661, 0.00361734279, 0.0618883558, 0.0642318279, 0.0619720668, -0.0356571376, -0.0750410408, 0.0242933612, 0.00693566445, -0.0197975188, -0.0793868079, -0.072376132, -0.0740384609, -0.0678929984, -0.0399386249, 0.00665389932, -0.0150171639, -2.41087211E-4, -0.00689154351, -0.0764064788, 0.0298105404, 0.0470431261, -0.0224311743, -0.00823963713, 5.314270e-02, -0.0102312258, 0.00125599816, -0.0274198409, -0.076341629, 0.0576306693, 0.046233926, 0.0667901337, 0.0505340844, -0.0470966212, 0.0525352918], [-0.0263741277, -0.0121007524, -0.0398272872, 0.0505702123, -7.782570e-02, 0.0415240601, -0.0315453745, -0.0591055118, -0.0699052215, 0.0208119508, -0.0116829509, 0.0138018569, 0.0203216337, -0.0161954723, -0.0465331301, -0.0390695222, -0.0734098479, 7.261960e-02, 0.007903344, -0.0354456864, 0.00356051512, 0.0608972721, -0.0552477539, -0.0730241388, 0.05342336, 0.0434031226, -0.0510150604, 5.932280e-02, -0.00282032415, 0.0452497862, 0.0028167842, -0.0374069773, 0.0283459444, 0.0779105052, -0.0450156815, 0.00965371076, -0.0048892051, 0.0803551301, -0.00527393632, -0.044453077, 0.0689685717, -0.0506951027, -0.0592842959, 0.0310309604, 0.0689190924, -0.0157198031, -0.0651662499, -0.0300892834, -0.00752570293, 0.0734301656, 0.0269012339, -0.062061686, -0.0152749438, 0.0220228173, -0.0750508532, 0.0568043813, 0.00415083766, -0.00455218321, 0.0074451291, 0.0600057542, 0.0148552917, 0.0438923165, -0.0576310828, -0.00835880172, -0.0360885821, -0.00114013127, 0.0643839836, -0.0783804431, -0.073770538, 0.0169609021, 0.0256640818, 0.0114613464, -0.0334980786, 0.0337555781, 0.046850808, 0.00554046407, 0.0189200789, -0.0524348058, 0.0136418408, -0.0221576318, -0.0149177872, -0.0507420413, -0.00657412177, 0.010345134, 0.053502243, -0.00155159377, -0.0540903062, -0.0127241891, -0.0569872409, 0.0528202876, 9.507790e-03, -0.0218062624, 0.0156707335, 0.028854331, 0.0269285552, -0.0498111695, -0.0750124305, -0.0443467125, 0.0577909872, 0.0489969514, -0.00511404872, 5.10000136E-5, 0.00326775131, 0.0185019821, -0.0256110542, -0.0660660788, 0.0543290749, -0.00749642448, 0.0132916719, 5.0057983E-4, 0.019187497, 0.0262438022, -0.0639115869, -0.0461442843, 0.0791773721, -0.018799087, -0.0290878732, 0.00229232176, 0.0725961775, 0.0449084677, -0.0779684185, 0.0420503654, -0.0693393051, 0.0366204716, -0.0713230371, 0.0040096594, 0.050253991, -0.0137285283], [-0.0444547795, 0.0218910072, 0.00998595077, -0.00955716893, -2.453170e-02, 0.0391812846, 4.647100e-02, -0.0203274302, 0.0657609403, 0.0654702634, -0.0456222221, -0.0306808893, -0.057816539, -0.0687925369, -0.00377561687, -0.0648733675, -0.0130809853, 0.05801991, 7.62844633E-4, 0.0563985668, -0.0227617603, 0.0140343178, -0.0346856341, -0.0311190113, -0.0365855359, -0.0606091432, -0.0583224297, 0.00245716958, -0.0515421554, -0.0439832248, -0.016785115, 0.0694232881, -0.0451885462, 0.0357660502, 0.0785903782, -0.0591465831, 0.0692410693, -0.0355544128, 0.0243920162, -0.0555065311, 2.886660e-02, 0.0409698449, -0.00334276794, 0.0658994764, -0.0395137332, 0.079391554, -0.061011944, 0.024281038, -0.0263613462, 0.00251551275, -0.00634940807, -0.0569836758, -0.0412326157, 0.0392872021, -0.0615021177, -0.0418702327, 0.0235089976, -0.0676743463, 0.0432115383, -0.0325849839, 0.0111174965, -0.0274342913, 0.0138888387, 0.001890969, 3.477830e-02, 0.0235203877, 0.0458420739, -0.00845579989, 0.0344090946, 0.00904421601, 0.0164132789, 0.0228360854, 0.00707533396, -0.0415599979, -0.0521417409, 0.0053877458, -0.00285076699, 0.0717164725, 0.0653341264, -0.0582882799, -0.0254131574, 0.0487133749, -0.065455988, 0.0143345864, 0.0379545242, -0.0699883252, -0.0557227731, -0.0645385086, -0.059193898, 0.0577353388, -0.0764233246, 0.00165513356, -0.00176062388, -6.552000e-02, 0.00843208563, -0.0334206894, 0.0103355236, -0.0624427348, -0.0764888898, -0.0312771238, 0.0141393244, 0.0245865658, -0.00382015132, -0.0316000134, -0.048647292, 7.526490e-02, 0.0562903881, -0.0723789334, -0.0612081848, 0.0632125959, 0.0229661651, 0.0636819154, 0.0264436286, 0.0648372099, 0.0701705366, 0.0703835934, 0.0807660073, 8.186230e-02, -0.0128661077, 0.00231211609, -0.0738913268, -0.0178447049, 0.0462080762, -0.0140109416, -0.0365559533, -0.0532173663, 0.0308298357, 0.0171223879], [-0.051201418, 0.0173152778, -0.0606523268, 0.0317452885, 0.0284002591, 0.0821846575, 0.012351376, 0.00239844853, -0.0588886514, -0.0624763779, -0.0103122182, 0.0723561421, 0.071831584, 0.0607585274, 0.00707492232, -0.0177914202, -0.0300599057, 0.0660994127, 0.0137193706, -0.00123619288, 0.0156653449, -0.0384481512, -0.0467538424, -0.0588834323, 0.0112741627, -4.829370e-02, -0.0646161139, -0.0341328606, -0.0610676109, 0.00578793325, -0.0351370461, 0.0596173406, -0.0746287182, 0.0188531857, 0.00334890815, 0.0763300955, 0.0664408058, 0.0545338541, -0.0549748801, -0.00202719867, -0.0515946522, 0.0315727852, 0.0333103612, 0.0790844336, 0.0311655756, 0.0567831397, -0.0364854522, 0.00371732563, -0.0611408167, -0.0429851115, 0.0506497324, 0.0735455677, -0.00567035656, -4.108200e-02, 0.0352583677, 0.0139468014, -0.0702334195, -0.00687292079, 4.620790e-02, -0.0205892455, -0.0157125145, 0.0474277958, 0.0116275195, 0.0247623287, 0.0241451766, -0.0271308981, -0.0692175478, 0.044667419, -0.0120461015, 0.0310183223, 0.0436497629, -0.00560223265, -5.176500e-02, -0.037332084, 0.0705107599, -0.0169811714, -0.0258709453, -0.0668495893, -0.0164399538, 0.0233991779, -0.0570435226, -0.0598434731, 0.0491269976, -0.0682353154, 0.0732966959, -0.0545021333, -0.0595203228, -0.0656606928, 0.0539337546, -4.01454541E-4, -0.0594557747, -0.0813059285, -0.0666103736, 0.013367122, -0.0209451858, -0.0435247049, 0.0219869427, -0.0487065762, 0.0616792813, 0.0471685529, -0.0609254539, 0.0307174698, -0.077740319, -0.018389523, -0.0193941463, 0.0623470768, 0.0339093655, -0.0270793978, 0.0495732836, 0.058391273, 0.00276663899, 0.0771952495, 0.0188593231, -0.0761752426, 0.0189612061, -0.0455236621, 0.0663020387, 0.0297169983, 0.0657587498, 0.0473760515, -0.0441492088, 0.0390956588, -0.0662700534, 2.547800e-02, 0.0180316623, 0.0228309091, -0.0600861646, -0.0444277599], [0.0507366918, 0.0660850257, -0.0539693795, 0.0290627833, -0.0485311523, -0.00144665048, 0.0169903245, -0.0625584945, -0.0758595318, 0.0374381579, 0.0159378219, 0.0347056724, 0.0127015561, 0.034007825, 0.00686555635, 0.0328790061, -0.0433475524, 0.0402991138, -0.037510477, -0.0514362529, 0.0197316278, 0.0137461061, 0.0488743931, 0.0782967061, 0.00736273685, 0.0454067774, 0.0740381554, -0.00413082447, 0.0156670827, 0.0307583231, -0.0574257895, 0.0012772656, -2.877740e-02, 0.0511684194, 0.0315699428, -0.0485976897, -1.33816457E-5, 0.0699460283, -0.0588642433, -0.0109554678, -0.0299362764, 0.0222077221, 0.032892298, 0.0418536402, 0.0114665236, -0.0134364665, -0.0745928958, 0.0506222807, -0.0277956594, -0.0444545262, 0.0468199402, -0.0531702265, -0.0724000856, -0.0766852275, 0.0506600551, -0.0169106685, 2.727530e-02, -0.0171920508, -0.0290781204, -0.0684210435, -0.021184966, -0.0513765216, 0.0113912672, 0.0677279085, 0.0684744641, -0.0738864914, -0.0181825534, 0.0144683886, 0.00760289282, -0.0418567769, -0.0176876336, -0.00213264464, 0.0790584459, 0.0264176875, 0.00700787734, -0.0260075852, -0.0331849195, -0.0558580942, -0.0748030469, 0.00306076044, 0.0136525221, -0.0107682645, -0.0453193933, -0.0670737773, -0.0226330236, -7.819110e-02, 0.00466514286, 0.0517401546, 0.0588977151, -0.0125983069, 0.0760844573, 0.0421815179, 0.0393525623, 0.0446950085, 0.0209100954, 0.0120319454, 0.0470215119, 0.0422273949, -0.00105287321, -0.0586067326, -0.0120749371, -0.070899345, -0.00178277853, -0.022687858, -0.0392852314, 0.046460025, 0.0289963186, -0.0122738201, 0.0431136489, 0.0159125663, -0.0147979958, 0.0386926197, 0.0314909965, -0.033183869, 0.0670365244, 0.075784795, 0.0290799327, -0.0583468117, 3.155940e-02, 0.0187829137, 0.020934457, 0.0344630815, 0.0564573966, -0.0528952852, -0.0640479475, 0.0472624935, -0.0391252264, 0.00252529932], [0.0538403578, 2.178190e-02, -0.00248377095, -0.0522277169, 0.018420931, -0.0381668285, 0.00822619069, -0.00291010155, -0.0130762719, 0.00891662202, -0.0430302881, 0.0710093901, 0.0461318158, -0.0701181516, 0.00166668708, 0.0243059583, -0.0273926686, 0.00198679557, 0.0795735493, -0.0640694275, 0.0797220469, -0.0809403509, 0.00178792456, 0.0441108532, 0.0293087475, -0.063240081, -0.0527372472, 0.0634602159, -0.0727304071, -0.0203770269, -0.0472613573, 0.0080315629, 0.00492039928, 0.0409275405, -0.0179413278, 0.0101243239, 0.00416703383, 0.0669165924, 7.30595551E-4, -0.058353629, -0.0397190154, 0.00149228342, 0.0535052344, 0.0699050054, 0.0325055383, 0.0268961471, 0.0578442477, 0.0435242616, -0.0468736701, -0.0422739126, -0.0740949512, 0.0447438508, 0.0798505917, -0.0192985237, 0.0316906944, 0.0299317054, -0.0224895887, -0.0120901354, -0.00145290268, -0.054668583, 0.0664399937, -0.027561646, 0.0345471427, -0.0759893805, 0.0284594782, -0.0535626672, 0.0580682158, -0.0286485218, 0.00860424247, 0.0706265494, -0.026185194, 0.0751718506, 0.0653523579, -0.0539645106, 0.00522813387, 0.055757042, -0.0595729277, -0.0529410467, 0.0714270845, 0.0456954725, 0.0414996408, 0.0600573458, -0.0246831458, 0.0765833556, -0.0173807554, -0.0341681316, 0.0582481846, -0.0773906633, 0.0709789097, 0.0394277833, 0.0230275765, -0.0675931945, 0.0114323422, 0.0669406205, 0.0517143235, 0.0588567667, -0.0165783744, 0.014715096, 0.0396883376, -0.0501986407, -0.00863225199, 0.0285616852, -0.0466603339, 0.0102817463, 0.031877812, 0.046246592, 0.0336319581, -0.0807765126, -0.044495631, -0.0323487744, -0.0737188309, 0.0596555322, -0.0669323578, -0.0515883714, -2.383020e-03, 6.763600e-02, -0.0707832351, 0.0723476186, 0.078000836, 0.00683821178, -0.0178815965, 0.0726638883, -0.0328204483, 0.00478859898, 0.05514016, 0.0344468541, 0.0514969528, -0.0105725182], [-0.0590728223, -0.00611744029, 0.0529676266, 0.00759340404, -0.0809907168, 0.0176026709, -0.0466775633, 0.0107398611, 0.0733640939, -0.00468797702, 0.0399892703, 0.0373189449, -0.0681420862, 0.0383692794, 0.0409104563, -0.0457238704, -0.017466208, -0.0282002147, 0.0735550448, -0.0630196556, 0.00272634299, 0.0288534146, 0.0285230223, -0.00902065355, 0.0301130228, 0.0608554892, 0.0650422499, -0.0360086523, 0.0666133389, -0.0556897372, 0.0477206521, -0.0747365952, -0.0479829237, -0.0695225224, 0.0352367945, 0.0527075268, -0.0725824088, -0.00516806124, -4.857900e-02, -0.0518719107, 7.236950e-02, 0.0036645669, 0.0518520363, 0.0289793033, -0.0488429964, -4.074420e-02, -0.0445670746, -0.0606512241, -0.0780615806, -0.0621245466, -0.0188760962, 0.0131823299, 0.0603692531, 0.0163683649, -0.0628274232, 0.0260234941, -0.0457796082, 0.0356825553, 0.00431080535, -0.0273275077, 0.0334233753, -0.0594435371, 0.0386011712, 0.0029665255, -5.26286487E-4, -0.0540990606, 0.0421223268, -0.0218685623, -0.0733515397, 6.95907569E-4, 0.00947419088, 0.0551823825, 0.055343125, 9.89528372E-4, -0.0123037966, 0.0279534459, 0.0669165701, -0.0584750026, -0.0152327055, -0.0469139963, 0.0196171813, -0.0328917056, 0.0598014221, 4.31793778E-5, -4.443800e-02, -0.0148320263, 0.0140386075, -0.057866469, 0.0334943645, -0.0288658217, -0.0430278704, 0.0175933894, -0.00798432715, -0.0355515182, -0.0412003398, 0.021618532, 0.014180026, -0.0601734333, 0.05381722, -0.0595251769, 0.072611548, 0.014775347, 0.00430025347, -0.0802760347, 0.0785544514, 0.0360442363, -0.0501116253, -0.0733721256, 0.0697349607, 0.0302218329, 0.0497439615, -0.0597575642, 0.0224082936, 0.0354359336, 0.0336577706, 0.0326975212, -0.0694214925, -0.0173359979, 0.0443440825, 0.0638467818, -0.0425981134, -0.00522036524, -0.042448312, 8.046530e-02, 0.00418365933, -2.697700e-02, 0.0534022264, -0.0782383903], [-0.0611510873, -0.0801630169, 0.0687789172, -0.00769763998, -0.0270229913, 3.161520e-02, -0.0133179417, -0.056397941, -0.0594164245, -0.0231613256, -0.0757815167, 0.070729658, -0.0739140212, 0.011368094, 0.0390016362, 0.0717237219, 0.0106560709, -0.0425367467, -0.00720347092, -0.029114414, -0.0435741656, -2.029830e-02, 0.0181538127, 6.915160e-02, 0.0695123598, -0.0459736325, -0.0646943524, 0.00660332059, -0.0291694626, -0.0334622227, -0.0247019175, 0.0765244439, -0.00816357694, -0.0468036868, -0.032666605, 0.0299071167, 0.0077816192, 0.05189218, -0.0216710065, 0.0343498439, 0.0383202322, 0.0541585311, -0.0531550087, 0.0467053168, -0.052848503, 0.0155872684, -0.0224554893, -0.00838523544, 0.00296355574, -0.0702893957, -0.0676591247, -0.0806085169, -0.0123105608, 0.0588536039, 0.0740823671, 7.230740e-02, -0.028356567, 0.0456883274, 0.033571586, -0.0704184472, -0.0323306583, 5.37587213E-4, -0.02839645, -0.0643129498, 0.0441163741, -0.0801007226, -0.0767600238, -0.0155506404, -0.0720536187, -0.0523438081, -0.0660021529, 0.0022407861, 0.0121004758, 0.060537409, -0.0590044186, -0.0199695602, 0.0601317585, -0.0770124868, 0.00691551063, 0.0121179176, -0.0612375662, 0.0718724802, -0.00774238398, -0.0168353207, -6.037280e-02, 0.026223151, -0.0465577953, 0.0357984975, -0.00865243561, 5.837480e-02, -0.0684062615, 0.0726402551, -0.0271824151, 0.0741824508, -0.0556599908, 0.0556630045, -0.0274604559, 0.0202051904, 0.0079224715, -0.0600584149, -0.0321377218, 0.0100595718, 0.0323696621, -0.0324981324, -0.0015867186, -0.0608147569, 0.00326672453, -0.0320317373, -0.00876650866, 0.00723144412, 0.040425688, -0.0373233184, 0.0100059845, 0.0158011504, 0.0150449416, 0.0589808151, -0.071912311, -0.0663546845, 0.0541501753, -0.0551872887, -0.0596924499, -0.065036945, 0.0629849285, 0.0442285351, 0.00840084627, 0.0653021559, -0.0132719111, -0.0682764798], [0.0567407981, 0.0476622544, 0.0541117042, 0.0108415848, -0.0133851944, 0.0513266809, -0.0585565343, 0.0123918643, -0.0261129644, -0.0739210322, -0.00569256209, -0.0508318879, 0.012795982, 0.0152603723, 0.07068564, -0.0347078927, 0.0721392855, -0.0107239168, -0.0600944683, -0.0234478153, 0.0524345823, 0.0316547826, -0.0619888082, 0.0624299943, -0.041424647, 0.0436006151, 0.0793997943, -0.0252658594, 0.0211645737, 0.0563570708, 0.0524335168, -0.0799300224, -0.0798984989, -7.951980e-03, -3.798810e-02, -0.0637498572, -0.052373793, 0.00516666332, -0.0706766769, -0.0387669839, -0.0341990702, -0.0360535905, -0.0471439399, 9.571000e-03, 0.0396907479, -0.0159156583, -0.0549622029, 0.0715772361, -0.0256078262, 0.0436542705, -0.0227348451, 0.0724101439, -0.0433048904, -0.0500116423, -4.601690e-02, -0.0488683619, -0.0107741272, 0.00102430652, 0.0645601675, -0.0183816422, 0.0322604068, 0.07981994, -7.536630e-02, -0.0500973612, 7.910040e-02, 0.014076828, -0.0380168892, -0.0330169313, 0.0272743981, 0.0724126398, -0.0747098327, -0.0690397248, 0.0409100726, 0.0446931608, 0.00404429343, 0.0593970157, 0.00877153128, -0.0185261443, -0.0156507362, 0.0394683816, 0.07552623, -0.00638463534, -0.0334647186, -0.0440555178, -0.079616256, -0.0212952048, 0.00436519971, -0.00710850675, -0.0785318538, -0.0781584084, -0.00218251883, 0.0518796295, 0.0554669574, 0.0475409143, -0.0666513517, -0.0297019053, 0.0348048359, 0.0690984279, -0.0188275184, -0.0220820531, -0.0640111193, 0.0270623695, -0.0755745843, -0.057060115, 0.0275576748, -7.837530e-03, 0.0513044745, 0.0258777719, -0.0229476299, 0.0420322344, 0.0702140331, -0.0489184968, 0.00311741559, 0.0160862859, -0.0750280097, -0.0594519898, 0.0648283213, -0.0656614304, -0.0717060491, -0.0221472606, 0.0316978171, 0.0206541643, 0.0065963082, -0.013121997, 0.0504142269, 0.0388272926, 0.0274122618, 0.0393495336], [-0.0176628679, 0.0413929671, -0.0256869178, -0.0617098361, -0.0327232443, 0.0174216349, 0.0497377031, -0.0280384514, -0.017286079, 0.0526382551, -0.012534692, -0.0602009334, 0.0553010739, -0.0450631529, 0.070835866, 0.0654882044, -0.0661286712, 0.0362410806, 0.07257507, 0.0523800887, 0.0429122634, -0.00680172769, 0.0112513267, 0.00140170194, -0.0512841269, 0.0157397985, -0.02070258, -0.0748692676, 0.0493262075, -0.0773470327, 0.0787746608, -0.0389482044, 0.0617208182, 0.0720121339, -0.0347544886, -0.0628691465, -0.0512407571, 0.0772201716, 0.0717197433, -0.048464559, -0.00337780826, 0.0654661581, 0.0145157538, 0.0256941132, -0.0728340521, 0.0431241617, 0.0193027444, -0.0651721731, 0.066256091, -0.0779077858, 0.040247798, 0.0534086041, 0.0723745599, -0.020646356, 0.00917079113, 0.0372869037, -0.0589572638, -0.00193758751, 0.0764926299, -0.0690795332, -0.0293114707, -0.00817027129, 0.00543901743, -0.0660958439, 0.0109002553, 0.0567818657, 0.0569869094, -0.023864612, 0.0745187625, 0.039388746, -0.0439293981, -0.00724027259, 0.0408216342, -0.077407524, -0.00187891035, -0.00345857022, 0.0162245296, 0.0676579103, 0.0506961569, -0.0729655623, 0.0556482933, -0.0452237464, -0.0144600989, -0.080289185, -0.0438764952, 0.0707574636, -0.0615626574, -0.0132719036, -0.0152174374, 0.0360718369, 0.0750003755, -0.0295316949, 0.0355593525, -0.00512646744, -0.0175851118, -7.84575357E-4, -0.0491794944, 0.0777377858, 0.072926648, 0.0279127937, 0.0213843323, -0.0413118228, 0.0586810932, 0.0180290937, 0.0688395202, 0.0191958882, 0.0626132637, -0.0641553625, -0.058158081, -0.03688116, 4.41740616E-4, -0.0737523288, 0.0499888137, -0.0604877844, 0.0405532345, 0.0395278074, -0.00121103099, -0.0637956038, -0.0311977975, -0.0339122266, -6.45717781E-4, 0.019580571, 0.0361930281, 0.018013902, -0.0337933749, 0.0422979258, 0.0103604198, 0.028928129], [0.0601344779, -0.0634318963, -0.0480816476, 7.018030e-02, 0.0380026475, 0.0280132014, -0.0348563753, 0.013908227, 0.0758340731, 0.0323972031, -0.0483631939, -0.0751745402, 0.00274239411, -0.0130342655, -0.0308406688, -0.0142407017, 0.0595880039, 0.0598947182, -0.0557395257, 0.018261008, 0.0455918834, -0.0418881066, -0.0492246933, -0.0690501779, 0.00468474673, 0.0506040156, -0.00712832436, 0.0241887718, 0.0364579596, -0.0469585359, 0.0341044106, -0.065221943, 0.00396102481, 0.0129494695, 0.0698037893, -0.0444998182, 0.0399509594, -0.068085514, 0.0800236985, -0.0191016644, 0.0590494871, 0.0463549159, -0.0226700306, 0.0721268132, 0.0446071774, 0.00137043511, 3.823190e-02, -0.020590866, 0.00417677732, -0.0161164906, 0.0172337126, 0.0549091026, -0.0726065561, 0.0785713047, 0.0350237265, 0.0642540082, 0.0183990151, -0.0295324083, -0.0143982414, -0.0212864429, -0.078263916, -0.0683050156, 0.0452608913, -7.327280e-02, 0.0464673527, -0.0060307621, 0.0495857671, -0.0598638803, 0.0502134524, 0.0168364029, 0.00299114827, -0.0386516266, 6.256780e-02, -0.0537864082, 0.0771738812, -0.0157364346, 0.0793903097, 0.00275033969, 0.0771758854, -0.068932727, -0.0715808123, -0.047726389, -0.0423832685, -0.0703718886, 0.0185267366, 0.0533370376, -0.0202136841, -0.0274418797, 0.0778212472, 0.0715238228, -0.0564196855, -0.0160323195, 0.0813059806, -0.0452038273, -0.0100795813, -0.0411569439, -0.030143002, -0.00992798339, 0.0679368228, -0.0444288552, -0.0477858856, -0.019553205, -0.0017550342, 0.049338378, 0.0597615205, 0.00659250281, -0.0653048679, -0.0798749551, 0.0463089384, 0.0194338523, -0.0666083395, -0.0719929039, -0.0750206783, -0.0655912608, 0.074309796, 0.0690194592, -0.0254724137, 0.077789694, -0.00641492056, -0.0725448579, 0.0153152095, -0.0460007079, 0.0731983408, 0.0697260126, -0.0625987723, -0.0194487292, 0.00303600775, -0.0704924166], [0.0296428502, -0.0505207777, 0.0265910216, 0.0633292869, 0.0535964705, 0.0667803511, -0.0645058081, 0.00112715852, -0.0525735281, -0.0691490918, -0.0630257279, 0.0471681245, 0.0630579069, 0.0159421302, 0.0664525628, -0.0734198466, -0.0801326185, -0.0810075476, 0.0741929412, 0.0383510292, 0.0705162808, 0.0498419777, 0.0422041081, 0.0600713566, -0.0111026606, -0.00277040456, 0.0453880653, 0.019822292, -0.0432495922, 0.0752902328, 0.0584721491, -0.0190636627, 0.0661019757, 0.0464097299, 0.0802996606, 0.0549007095, 0.0309812874, -0.00741978828, -0.0563723892, -0.00502112089, 0.0608572029, 0.072724618, 0.0142697459, -0.0418016724, -0.0518672206, -0.056876123, -0.0228978246, -0.0332155898, 0.0359708592, -0.0712861046, 0.0758687109, 0.0150717432, 0.0586673245, -0.0348456763, -0.045367863, -0.0797440559, 0.0638123676, 0.0699474216, 0.0582202598, -0.0628056526, -0.0798160135, -0.0386155955, 0.0636697039, -0.00201050774, 0.0701831654, 0.0219408199, -0.0514023565, -0.00540494174, 5.557030e-02, 0.00102697674, -0.0633998737, -0.0592701323, 0.0148762316, 0.0405639298, -0.0251652971, 0.0429548584, 0.0306383874, -3.60057369E-4, -0.0434264019, 1.82069838E-4, -0.0379245654, 0.0423279926, -0.00239736587, -0.0549841672, -0.0484442301, 0.0495740585, 0.0169438943, -0.0691917762, 0.00525557809, 0.0662242472, 0.0369339213, 0.0181056149, 0.0523858592, 0.00215983344, -0.0524194203, -0.0220423434, 0.0615996495, 0.0528558381, -0.0706042871, 0.0348972753, 0.0391882956, -0.0346335061, -0.049787987, 0.0546707883, -0.0580401085, 0.0293041226, -0.064075157, -0.0256116726, -0.0519270077, -0.0386728644, 0.0464544818, -0.0483786613, 0.0667650327, 0.00146477669, -0.0204489678, -0.0771620646, -0.0355348811, -0.0730931088, -0.0728794634, 0.0613173321, -0.0456313081, -0.0445287824, 0.0558046587, 0.0747524872, -0.00816192571, -0.00429623574, 0.05898479, -0.00407513278], [0.0448157564, -0.0222490616, 0.0200798288, 0.00161352754, 0.0703214779, 0.049570851, -6.771000e-02, 0.0725758448, -0.0135620087, 0.00990089774, -0.00661168992, 0.0387806222, 0.0644810274, 0.012018986, -0.00602187216, 0.0566589311, -0.0312541723, 0.0719104782, -0.00953206419, 0.0180299506, -0.0335637666, -7.130230e-02, -0.00520910323, 0.0466913953, 0.0166201293, 0.0640125349, -0.0792167186, -0.0493147336, 0.0521430448, 0.0106579587, -0.0601774827, -0.0593460724, 0.0525600091, 0.0399787277, -0.0438677929, -0.0480419844, 0.044438608, -0.0251407176, -0.00663216412, -0.0260887183, -0.051808022, 0.0688135549, -0.0163582116, 0.0529210493, 0.0448439345, -0.0173527971, -0.0674481243, 0.0101169646, 0.0749161318, -0.010855265, -0.0191034153, 0.064992927, -0.0287500955, 0.0275730863, 0.0528025106, 0.00929027051, 0.00638484954, -0.0694558844, -3.668170e-02, -0.0635775402, -0.0413639657, 0.0451055244, -0.0521170273, 0.0417619646, -0.0743732229, -0.0483571775, -0.0681067109, -7.168580e-02, 0.0645601079, -0.0526896343, 0.0528236851, 0.0377499834, -0.0448414572, -0.0724961162, -0.0758111253, -0.0386113338, 0.0220919773, 0.00988861918, 0.0614641979, -0.0178062245, 0.00152739882, 5.592420e-02, -0.0294986032, -0.024633456, 0.00774438679, -0.0126237348, 0.0570134893, 0.013460502, 0.0129856691, -0.019669719, 0.0230680183, 0.0215810761, -6.313520e-02, 0.059336938, -7.168520e-02, -0.00664719194, -0.0309566744, -0.0175430104, -0.0293129757, 0.0425717384, 0.0556513146, 0.0710931644, 0.0475951657, -0.0482430421, 0.0739939436, -0.0260988921, 0.0603197739, -0.0373097882, 0.0419438258, -0.0313028283, 0.0560015365, -0.00669979304, 5.543620e-02, -0.0718000159, -0.0611494258, -0.0496111885, -7.578380e-02, -0.0118024722, -0.0675353929, 0.0177452713, -0.041801516, 0.0609416738, -0.0341497548, 0.0184747502, -0.0609043129, 0.0556606725, -0.00783156604, -0.0575129353], [7.621240e-02, 0.074480392, 9.223290e-03, -0.0281113498, -0.00636835396, -0.0518654548, 0.0171958506, 0.0119882375, 0.0684517846, -0.0316335894, 0.0341671035, 0.011911951, -0.00807617604, -0.0733866989, -0.068945758, -0.0591697618, -0.0289033316, 0.078707166, 0.00584838539, -0.024033349, 0.0217259228, -0.0537456051, 0.0606168285, -0.0502055511, -0.0646810606, 0.0139922872, -0.00183760375, 0.068610467, -0.0456037931, 0.0130651891, -0.0640208125, -0.0200199187, 0.0627210811, 0.0699302778, -0.0659824163, -0.0508989096, -0.0207561664, 0.0488185957, 0.0546107367, -0.0489089377, -0.0499910712, 0.0626198277, 0.0687492266, 0.0462695882, -0.0501900427, -0.017974101, -0.0361772403, -0.0225919485, -0.0660059452, 0.00907635688, 0.0502822623, -0.00156684965, -0.0628328621, 0.0399484634, 0.0494921878, -0.0771465674, -0.0266018249, -0.00988864153, -0.0570538417, 0.0631824955, 0.00811481475, -0.0234374776, -0.0685977489, -0.0780122131, 0.0431514829, -0.0713755339, -0.0543567128, -0.0314409621, -0.0551530272, -0.0511740893, -0.0707274154, -0.0310319178, -0.00213833153, -3.695640e-02, -0.0433405712, 0.0409213156, 0.0408882797, 0.0628317371, 0.0484759286, -0.067936182, 0.00436426699, 0.0379533246, 7.683590e-03, 0.0467904136, 0.00773005933, -0.0149371922, 0.0278469771, -0.0735929832, 0.0596264228, -0.0699174479, 0.0766030625, 0.042320244, -0.0418196134, 0.0187781677, -0.074823983, 0.0296217278, -0.0212013312, 0.0503523424, -0.0767235607, 0.0682022646, -0.0657359883, -0.0239733793, 0.0780971571, -0.0525218919, 0.00984025746, -0.0325028487, 0.0213182867, 0.0221591741, -0.059899047, -0.0698750764, 0.0584421381, 0.0292908326, 0.0288918465, -0.0205429606, -0.0287752934, -0.00162994862, 0.0555348918, 0.0385644734, 0.0109225586, 0.0550932661, -0.0748753249, 0.0669545457, -0.0583443269, -0.0322729163, -0.0355349779, 0.0609195903, -0.00761480629, -0.00849461555], [-0.0768734962, -0.0228731483, -0.0169980973, -0.0316767171, 0.00835517048, -0.00127805024, -0.0705443621, 0.0192895085, -0.0173191279, -0.0286500193, 0.0435297564, -0.0157391876, -0.0783767178, -0.0323675163, 0.0778076127, -0.0244425088, 0.049579151, 0.0513037518, 0.0532352254, 0.0422879681, 0.00884868204, 0.0566510037, 0.0401602164, 0.0672559962, 0.00995468348, 0.0255966932, -7.847890e-03, 0.0607344285, -0.0519435443, -0.0778120383, 0.00364007056, 0.0121023506, 0.0105861947, 9.801790e-03, -0.0157079026, -0.0134535953, 0.0220352411, 0.0763749108, -0.0445843153, 0.00920609385, -0.0292658284, 0.0383458957, 0.0531891808, 7.67946243E-4, 0.0439606979, 0.0249995515, -0.0744910538, -0.079111129, -0.0679220259, 0.031234853, -0.0452152267, 0.0735080317, 0.0646732822, -0.0269793272, 0.00726990401, 0.075831674, 0.0571486726, -0.0128523484, -0.0620960742, 0.0644023195, 0.0619158372, -0.029881753, 0.0449709222, -0.0379410684, 0.0280897692, 0.0702396855, -0.0134817958, -0.0141421929, 0.0136188641, -0.0373582877, -0.0566174388, 0.0587306246, -0.0322280489, 0.0642581359, -7.82251358E-4, -0.016838789, -0.08040113, -0.0715478212, 0.014099516, 0.00754199177, 0.0328031927, 0.0293558463, 0.00991014391, -0.0583368242, 0.0214605406, 0.0393504426, -0.0779305249, 0.0185770467, -0.0167307779, -0.0261583179, -0.0394413322, -0.0185698345, 0.0336006805, 0.0320069715, -0.0273451284, 0.0182834938, -0.0603808463, -0.0017286092, -0.0122030675, 0.0375837758, -0.00651018321, 0.0372323394, 0.0444193259, 7.475010e-02, 0.0320945755, -0.0763602182, -0.0265279897, -0.0153708905, 0.0223581865, -0.0484031066, -0.0352112912, -0.013298966, -0.0303629488, 0.0638252273, -0.0483947732, -0.038467709, 0.0388300121, -0.0688973516, 0.0327401087, 0.0565356687, -0.0328828841, -0.0488408878, 0.0250262916, -0.0621956288, -0.0264544636, -0.023867581, -0.0731407925, -0.0190520734], [-0.0384828299, -0.064039208, 0.0631723478, 0.0303257629, 0.0741117373, -0.0362666026, -0.0368685834, 0.0635522678, 0.00109037012, -0.0720141232, -0.0799147486, -0.080323115, -0.0254844949, -0.0143775642, -0.0806180239, -0.0347872823, -0.0269893259, 0.0406244695, 0.0719846413, -0.0181339309, 0.0388331264, 0.0406994596, -5.415380e-03, 0.0556571558, 0.0522339121, -0.0384512134, 0.0285544693, -0.0524641126, 0.019233793, 0.0432517678, 0.0599418804, 0.0110938624, 0.038349703, 0.0502398834, 0.0356972069, -0.0377372019, -0.00275100768, 0.00347031653, 0.0635362342, -0.0643264204, 0.0591426566, -0.0245325863, -0.0612871908, 0.0410635844, -0.0222225301, -0.0733666494, -0.0405417196, 0.0536415055, -0.0781137943, -0.0238366015, 0.0195744038, -0.0746594518, -0.0629815906, 0.040160507, -0.023567006, -0.0627794117, -0.0786693245, 9.86523926E-4, -0.0553699434, 0.0356555507, 0.036181882, 8.06488096E-4, 0.0723272339, 0.0223370865, -0.0251749456, 0.0297435224, -0.034174528, 0.0133875012, -0.0419172347, -2.92725861E-4, -0.0232044309, 0.0498777851, 0.0733823702, -0.0799678117, 0.0285432935, -0.0107759982, 0.0631709173, 0.027177237, 0.0698456839, 0.0808635577, 8.18513333E-4, -0.0113907605, -0.0372573249, -0.0255311765, -0.0772801414, 0.069240652, 0.0123182684, -0.035617765, -0.0297188684, -6.688000e-02, -0.0279571861, 0.0547015294, -0.0151561573, -0.0752832443, -0.0467480384, 0.0586661175, 0.035885796, 0.052400358, -0.0788172632, 0.0302420035, 0.013267912, -0.0152025372, 0.0298452079, -7.479840e-02, -4.200650e-02, -0.0204779431, 0.0615517572, -0.00442452729, 0.0247340351, 0.0502245948, -0.0520289242, -0.0331560187, -0.041064512, -0.0432068296, -0.0141573399, -0.0723352209, 0.0692276284, 0.0531086698, 3.103540e-02, 7.183800e-02, 0.00816766917, 0.0469877198, -0.0218576565, 0.0556124225, -0.0733844563, 0.0540607944, -0.0769495517, -0.0219218396]]> : tensor<784x128xf32> + %4 = "xla_hlo.dot"(%3, %cst_0) {name = "dot.5", precision_config = ["DEFAULT", "DEFAULT"]} : (tensor<1x784xf32>, tensor<784x128xf32>) -> tensor<1x128xf32> + %cst_1 = constant {name = "constant.6"} dense<[0.00180979178, -0.00128747581, -0.001201501, -0.0360582806, 0.0236181393, 0.0454008877, 0.0120176105, 0.0890586376, -0.00548917893, 0.0535316467, -0.0195648093, 0.00574736623, -0.00725247897, 0.0310232546, 2.836130e-02, -0.0188486483, -0.00258476194, 0.0227473844, 0.00291113905, -0.0495234169, 0.00637712515, 0.0514211543, 0.00229622098, 0.0491134487, -0.0558668263, 0.0909168571, -0.011906784, 0.0151743693, -0.0105079701, 0.00263013947, 0.0189482253, -0.0108587798, 0.0751609355, -0.0358677395, 0.0590642393, -0.0775720253, 0.0589618646, 0.10349872, -0.00274306419, -0.0312502868, 0.00333261886, 0.00910504907, 0.0216336083, 0.056877967, 0.0864347144, 0.0466054045, 0.0586982816, -0.0203834437, -5.626260e-02, -0.017801106, -0.00396474544, -0.017970033, -0.0256265551, 0.0489372499, -0.00651295763, -0.0350460112, 0.0291766617, 0.00479861069, 0.0237274338, 0.0348542407, 0.0677872151, 0.0245250557, 0.0508054681, 0.0422587655, 0.0730319619, 0.0025935811, -0.0277704448, 9.64390754E-4, 0.00164659088, 0.0798677206, 0.133678138, 0.0322072394, 0.0558781214, 0.0193853434, 0.00515032187, -0.00355286896, 0.0668371543, 0.0142582273, 0.0781719386, 0.00483097089, 0.00813726429, 0.0445716828, 0.0245931037, 0.0166326128, 0.0474203937, 0.0324643776, -0.0310089402, 0.0192547292, 0.0636251494, -0.00645098882, 0.051795654, 0.0795653983, -0.0386578813, 0.0565713793, 0.00196879101, 0.0201079044, 0.11367888, 0.0214288831, 0.0393215232, 0.0381225236, 1.174720e-02, 0.0623513386, 0.0427070446, 0.0237223972, 0.0676762611, 0.0697290376, 0.0389622599, 0.00733669475, 0.0565525182, -0.00873772241, -0.0214094054, -0.0121570313, 0.0171261076, -0.042719733, -0.0709377304, -0.00414954033, 0.0314523205, 0.00450869836, 0.00449752575, 0.041478429, 0.0945244356, -0.0497375578, 0.0145085724, 0.105547592, -0.0437812582, 0.01818325, -0.0811406821, -0.0240946412]> : tensor<128xf32> + %5 = "xla_hlo.broadcast_in_dim"(%cst_1) {broadcast_dimensions = dense<1> : tensor<1xi64>, name = "broadcast.7"} : (tensor<128xf32>) -> tensor<1x128xf32> + %6 = "xla_hlo.add"(%4, %5) {name = "add.8"} : (tensor<1x128xf32>, tensor<1x128xf32>) -> tensor<1x128xf32> + %7 = "xla_hlo.max"(%0, %6) {name = "maximum.11"} : (tensor<1x128xf32>, tensor<1x128xf32>) -> tensor<1x128xf32> + %cst_2 = constant {name = "constant.12"} dense<[[-0.17363435, -0.0707960129, 6.038730e-02, 0.121469341, -8.579290e-02, -0.143173471, 0.228746504, -0.145717144, -0.135060593, 0.140740767], [5.333820e-03, -0.135003075, 0.107362889, 0.311269701, 0.107835442, -0.00957506895, -0.0113238972, 9.436320e-02, -0.0351445265, -0.249150515], [-0.060859222, -0.157955885, 0.0543048941, -0.010158102, -0.0983931943, 0.0325914212, -0.0221404675, 0.140480801, 0.219866768, 0.148706794], [-0.0231706034, 0.189667493, 0.339141279, 0.297713399, -0.400857776, -0.257582575, -0.42581442, 0.225847244, 0.0542479306, -0.00848766416], [0.155154109, 0.262343377, 0.258146912, 0.0306026582, 0.141189575, 0.127351299, 0.0131782508, -0.245702147, 0.185197026, -0.398140848], [-0.213607386, 0.0672283098, 0.236905813, -0.225607336, 0.0225677192, 0.0694317594, -0.113246329, 0.245288506, -0.0194486771, -0.210125729], [-0.181720048, -0.113102444, -0.258585364, 0.187338203, -0.00800009817, -0.126790538, -0.114432767, -0.0659572855, -0.186335564, 0.270344526], [-0.0809494108, 0.0859531611, 0.19903186, 0.156178951, 0.00449119089, 0.264205396, -0.14980799, -0.179858178, -0.161061853, -0.0697564408], [0.111823864, 0.139123663, -0.215125829, -0.237549827, -2.723160e-01, 0.0774042978, 0.173556924, -0.102436289, 0.132224619, 0.204164654], [0.166308358, -0.0898675844, -0.309499949, 0.262152076, 0.166523397, 0.350022405, -0.134389818, 0.0800818502, -0.250730485, 0.278443933], [-0.182188168, -0.0662540421, -0.16912438, 0.253069848, 2.666290e-01, 0.147774175, -0.181770161, -0.239382803, 0.180332914, 0.14152202], [-0.213285133, -0.119788148, 0.146969259, 0.00588027807, 0.285986096, -0.0156045202, 0.0618853755, -0.0404547676, 0.199955285, 0.00802243501], [0.121508963, -0.0461791642, -0.153700531, 0.167498261, 0.172541648, -0.0350595154, 0.201652139, 0.0710540265, 0.0755672902, -0.0469412282], [0.0830276161, -0.191616312, 0.383806795, 0.183871031, -0.349610031, 0.333669364, -0.168258801, -0.120922968, -0.0744043514, -0.157934666], [0.217354283, -0.0641109496, 0.199787721, -0.0672617331, -0.0763153731, 0.130093575, -0.212879151, -0.0289937686, 0.0341918021, 0.131831452], [0.101335697, -0.217161059, 0.323923737, -1.124570e-02, -0.243533522, 0.0500382632, 0.267564029, 0.162072048, 0.169035599, 0.0175218079], [0.0720043182, 0.181623921, 0.104101956, 0.0909626707, -0.0998992919, 0.107782066, 0.13765879, -0.166188747, 0.087232843, 0.201865882], [0.185548037, 0.0476345345, 0.0827088207, -0.145057097, -0.11100968, -0.125786468, 0.203262866, 0.204047367, -0.196534082, -0.170384556], [0.30700314, -0.0496439189, 0.179890692, 0.0554856658, -0.142839238, 0.127258584, -0.230494335, -0.00550242327, -0.0320532843, -3.000920e-01], [0.31118235, 0.00334946788, 0.111910984, -0.302830279, 0.121877529, -0.284978181, 0.0194577854, -0.261625707, 0.121911936, 0.138116539], [-0.19934088, -0.194442376, -0.0388123132, -0.0742789208, 0.0712441728, 8.882910e-02, 0.0401109159, -0.148758903, 0.0713230371, -0.238374695], [-0.00572971953, 0.0051445188, -0.241541177, 4.522450e-02, 0.0202827249, 0.229814544, 0.129838377, -0.156346813, -0.177438587, 0.307789236], [0.00586719438, -0.23780936, 0.0713653266, -0.12190327, 0.182632983, -0.209033892, -0.00922673941, -0.151551381, -0.16923815, 0.112413712], [-0.134286478, 0.121913657, -0.0402135141, 0.233039021, -0.206312865, 0.319349617, 0.00143496622, 0.254222333, 0.0337958224, -0.0956300869], [0.0190694649, -0.28896293, -0.209214076, 0.0274076108, 0.200228855, -0.165110469, -0.222465485, -0.150394216, 0.158042848, 0.0871670097], [0.131613567, -0.0569929108, 0.0477676541, -0.154282197, 0.177263156, 0.379474372, -0.198784053, 0.235023528, -0.152511865, -0.345084906], [-0.00747684622, 0.240096062, 0.0852414817, 0.149832889, -0.208482146, 0.108278856, 0.158599466, -0.261659771, 0.28667134, 0.00529837143], [-0.185448363, -0.142471164, -0.00380585133, 0.205363795, -0.115163729, -0.176866487, -0.1503966, 0.0945485532, -0.249040067, -0.146563202], [0.0507423021, -0.162201419, -0.104880594, -0.119434029, -0.0137509312, 0.0857468545, -0.14507775, -0.0310742967, 0.142568573, 0.137628734], [0.292243481, -0.198636204, -0.141308501, -0.0398842096, -0.23759377, 0.158741564, -0.255989969, 0.0791561231, 0.135853305, 0.0416310765], [0.111437723, 0.293795198, -7.0257741E-4, 0.0395522788, -0.213674441, -0.0549810939, 0.0761651173, 0.108064711, -0.258550107, 0.129741773], [-0.227937847, 0.310785294, -0.156648576, -0.0240825266, -0.0934058725, -0.0344269089, -0.145666569, -0.169869885, 0.240113556, -0.0654262826], [0.0319960192, 0.0288540535, 0.256446928, 0.1365605, -0.157555774, 0.301516473, 0.174934179, -0.267932802, -0.20990786, -0.289210945], [-0.0986520349, -0.0171110164, -0.221782967, 0.00511969253, 0.441806704, -0.197293893, -0.291264772, 0.219537139, 0.205427408, -0.0171957519], [0.218552783, -0.155077934, -0.0703493133, -0.0902480334, 0.232104361, 0.23108305, 0.134619236, 0.0748627409, -0.275893092, -0.0390154459], [0.381019801, -0.339646637, 0.0624736026, 0.133897841, -0.19746834, 6.436870e-02, -7.945660e-03, -0.362655699, 0.233752429, 0.0368957892], [-0.0580835491, 0.0224719848, 0.00401857076, -0.0762882828, 0.151949599, -0.221072719, 0.025190426, 0.327190787, -0.274966955, 0.248151094], [-0.394082546, 0.247909665, -0.188450053, 0.281511843, -3.341570e-01, 0.125203356, -0.141429037, -0.0718699395, -0.143214151, 0.229724839], [-0.0137553262, -0.228617609, -0.140930399, -2.799820e-01, -0.0553898327, 0.178554818, 0.130356207, -0.258220017, 0.202061623, 0.00610400829], [0.261053771, -0.287141502, 0.509392798, 0.163613275, -0.207967952, -0.279594868, 0.0682455376, -0.140134558, -0.330549598, 0.221153483], [-0.072769165, -0.107394435, -0.0957670435, 0.174531385, 0.0684501082, -0.00831179414, 0.189872384, -0.237616166, -0.0591602065, 0.125886753], [-1.765560e-01, 0.102832273, 0.0100140488, -0.0257020816, 0.181918576, 6.714420e-02, 0.100126542, -0.0761945695, 0.122161649, 0.059734337], [0.14992249, -0.042751465, -0.242604434, -0.0491485447, -0.30338645, 0.0788762643, -0.299695104, -0.13762036, 0.0150603065, 0.0795048698], [0.172013864, 0.206088021, 0.22058697, -0.125324488, 0.310980856, -0.216724753, 0.0800841823, 0.0215022508, -0.381094307, -0.271564096], [0.187171385, 0.162661836, -0.226146564, -0.101458885, 0.0100699505, 0.123740695, 0.225523502, 0.25284794, -0.159023091, 0.253711969], [-0.301451921, 0.0313369371, 0.00644301111, 0.0920508429, -0.123227581, 0.0879808217, -0.0213221256, 0.2880193, 0.00138571649, -0.0611888096], [-0.31910035, 0.235809758, 0.169649586, -0.34670642, 0.302606314, -0.222152904, 0.393572927, -0.348440439, -0.0699117631, 0.0882392675], [-0.0748971105, -0.431775421, -0.0966364666, -0.226431355, 0.382968128, -0.0831228048, -0.00299148099, -0.19580473, 0.0980860963, 0.250937313], [0.178527981, 0.076650247, -0.00415729918, 0.0951654613, -2.375450e-01, -0.216295198, 0.0110081164, -0.284828633, 0.192532718, -0.157802403], [0.299306184, -0.271432847, 0.183518261, 0.026472874, -0.0726066753, -0.0390495062, 0.107637577, 0.00404472463, 0.0666918904, -0.0401334688], [-0.030999558, 0.259209841, -0.0465492047, 0.0529589504, 0.0611805804, -0.0262979791, 0.0688910112, -7.44911667E-4, 0.167459026, -0.26001966], [0.255778253, -0.24220413, 0.350433469, 0.350896806, -0.244471326, 0.178872496, -0.100021079, 0.0284363627, 0.096491307, -0.193851605], [0.0302060489, -0.179981887, 0.0229005795, -0.280266136, 0.180455044, -0.0580929406, 0.355026275, 0.125420481, 0.255031645, -0.131591201], [-0.275648683, -0.0157170743, 0.260199428, -0.224597305, -0.303664744, 0.0232193582, 0.229699343, 0.366713554, -0.0230590478, -0.232098207], [0.141025662, -0.127063408, -0.18927443, 0.0753805339, -0.223815992, -0.0688226222, 0.171983212, 0.208703697, -0.0607432127, 0.26896134], [0.00368685625, 0.0151929613, 0.0179557707, 0.0680684745, -0.308445036, -0.0548807904, -0.0591526218, 0.15423733, 0.235452116, 0.145921558], [0.0204494651, -0.0814779475, 0.00520399632, 0.0311975423, 0.0860458388, 0.343610466, -0.118007861, 0.0334062725, -0.0875458344, -0.0238407608], [-0.0935574844, -0.194300473, -0.166375577, 0.0796527639, 0.0147504658, -0.0735810548, -0.0172830094, 0.168246791, 0.0553085096, -0.0942590088], [-0.238010228, 0.111086741, -0.146693811, -0.0474509485, -1.254460e-01, 0.079645589, -0.0469389521, 0.0198243875, -0.0817721113, -0.272092968], [-0.396736801, 0.332016051, -0.0264706276, 0.283282727, -0.2508744, 0.104432434, 0.00444465084, -0.0408442765, 0.254120976, -0.0482885838], [-0.09970849, 0.331348449, 0.00756065454, -9.879130e-02, -0.0698768348, 0.295237154, -0.216183186, 0.0113916555, 3.391220e-02, 0.232303575], [0.0236333422, -0.0779071822, 0.0288420524, 0.119856723, -0.169540137, -0.028769955, -0.0591493063, 0.276199877, -0.136728957, -0.200735331], [-0.201988384, 0.137718067, 0.00461096223, -0.176960647, 0.295218349, -0.14878659, -0.195739403, -0.124464899, -0.247669652, 0.331599027], [-0.0809967219, -0.0614988767, 0.0474593081, -0.0996292084, 0.292685807, 0.224856764, -0.0335971639, 0.0651911348, 0.0482374281, 0.0931988508], [-0.084696129, 0.178495869, -0.242035404, -0.146512449, 0.220621809, 0.247746885, 0.0751499459, 0.177417561, -0.163688406, -0.0584881492], [-0.111735344, 0.296827227, -0.0607453175, 0.00291904272, -0.162802532, -0.237988725, -0.245114595, 0.14962557, 0.0833254978, 0.282257915], [0.129119486, 0.00415245071, 0.221430361, -0.128097311, -0.291034788, -0.201655984, 0.240159303, -0.111073785, 0.156223252, 0.18907699], [0.0207739267, 0.202450186, 0.184913442, 0.0671677887, -0.15945904, 0.100802258, 0.204368159, 0.0663899928, 0.11844565, 0.00254672393], [0.147628516, -0.108610041, -0.120382845, 0.0630499422, 0.148190275, -0.173440784, -0.21185419, 0.116352729, -0.141718119, -4.59181669E-4], [6.394150e-02, -0.214126274, -0.262781709, -0.195635393, 0.10030704, 0.0192587748, 0.2180603, 0.276796103, -0.19745253, 0.250166684], [-0.264964819, -0.172969192, -0.320235193, 8.573610e-02, 0.251054466, 0.349534839, -0.0533160381, 0.215639099, -0.112659849, -0.117045388], [0.0715036839, 0.032501433, 0.290438265, -0.247630075, -0.144847691, -0.0191427749, 2.999250e-01, -0.257535666, -0.189960018, 0.130332768], [-0.203149259, 0.272928387, -7.304530e-02, 0.0234719831, -0.170207545, -0.11776901, -0.0428751446, -0.120133057, -0.110699892, -0.260757744], [-0.1397838, -0.126450539, 0.30517146, 0.184304431, -0.0930630564, -7.3810824E-4, 0.0160566941, 0.0380945243, -0.0847687795, -2.037490e-01], [-0.0283999704, -0.162262782, -0.0509804338, -0.0250009838, 0.205954537, 0.0246407222, -0.15305832, 0.01729705, 0.00185124797, 0.145579904], [0.107952237, 0.17434299, 0.0141776204, -0.128442273, 0.120245196, 0.0513597503, -0.0178853422, -0.165091217, 0.228567258, -0.205565557], [-0.272574723, 0.110321201, -0.249659762, 0.102074012, 0.214858428, 0.104048625, 0.260062635, -0.246641651, -0.162898168, -0.085429348], [0.0347707979, 0.121450476, 0.0601244941, -0.036875464, 0.0601903833, 0.0391768366, 0.177384406, 0.127554491, -0.00152025023, -0.0243941341], [0.182021648, -0.141452819, 0.0396866649, -0.0725521743, 0.0052056904, 6.541660e-02, -0.158098981, 0.368424326, -0.246160388, 0.0400460847], [-0.0415195115, -0.0794856697, 0.256927609, -0.0828218385, -0.333121389, 0.0690186173, 0.00953093916, -0.125028566, 0.0978005602, -0.0218165014], [0.254078567, -0.273427039, -0.0916603953, 0.284437299, -6.798040e-02, -0.186892554, 0.0270918794, 0.323385864, -0.13483052, -0.0614349432], [-0.0134747522, 0.229401737, 0.290741056, 0.134622812, 0.150844321, 0.0588404089, -0.0920286179, 0.243959576, 0.0415463932, -0.364745617], [0.0721129328, 0.335333854, -0.181168884, -0.0742811784, 0.202567935, -0.0618376844, -0.144345105, -0.109949991, -0.224407122, 0.046522852], [0.0943903848, 0.308363646, 0.20886232, -0.267283678, 0.0870875865, -0.266921341, -0.0927265435, -0.02150652, 0.0339837074, 2.529730e-01], [-0.202570871, -0.115333825, -0.0360322334, 0.0286047366, -0.061448779, 0.194968984, 0.439014375, -0.0769344866, -0.203588605, -0.392715514], [0.134818882, 0.0152576296, -0.10848958, -0.263930678, -0.0335265473, -0.0154081546, -0.152287707, 0.25470081, 0.0869387611, 0.104704529], [0.261753291, -0.241538271, -0.151260406, -0.0338756777, -0.26997748, 0.305188417, 0.0146879423, -0.280027121, 0.284972101, -0.0740737915], [-0.0162396524, -0.224762663, 0.0944506824, -0.119866684, -0.099188067, -0.0926269814, -0.168556571, 0.151416615, -0.148968905, -0.139955819], [0.102667071, 0.326664209, 0.0275067855, 0.244041294, -0.171143144, -0.274285197, 0.327633411, 0.179012597, -0.370987713, 0.0510097407], [-0.00375827751, -0.00335882837, 0.117868364, 0.126431495, -0.154993251, -0.133509174, 0.102136597, -0.116545178, -0.0755984187, 0.301457614], [0.298711061, -0.0935267061, -5.403840e-02, -0.380565524, 0.199013397, 0.402721375, -0.119996756, -0.236993909, 0.159043133, 0.0585093945], [-0.399404407, -0.0206954628, 0.176582664, 0.357191801, 0.255237281, 0.178168833, -0.0550820492, -0.18484962, -0.212347254, 0.0684524626], [-0.153790683, -0.262719393, 0.0698453411, 0.0273735207, 0.103013918, -0.169419676, 0.144809633, 0.140440971, 0.163974196, 0.267165035], [0.135177806, -0.14423652, 0.0656731352, 0.181822658, 0.0578323714, 7.627260e-02, -0.0565378331, 0.32406491, -0.211514086, -0.303026915], [0.124107487, 0.0760767832, -0.0458727963, 0.126206547, 0.167545691, -0.0626680106, -0.0209572129, 0.180526376, -0.0798619166, -0.175614953], [-0.046498958, -0.0247157011, -0.262536258, 0.260433704, -0.168176442, 0.218313709, -0.296603799, -0.208868355, -0.0718513504, 0.173542261], [0.0661545843, -0.153439388, 0.0171819553, -0.215688378, 0.0260388013, 0.222951055, -0.330462664, 0.361601323, -0.274192423, 0.140660316], [0.223756596, 0.158161566, -0.255715221, 0.0161067527, -0.227913514, -0.0569394939, 0.270599872, 0.09462893, 0.0185249355, -0.0796072855], [-0.292885542, 0.19097206, -3.517100e-02, -0.175302714, 0.135310337, -0.1766157, -0.286993176, 0.0181622561, 3.830600e-02, 0.0484179817], [-0.141062513, -0.277427912, 0.349744588, -0.174416408, 0.162776902, -0.23186703, -0.0472403578, 0.0647463575, -0.366720527, 0.143775359], [-0.259507775, 0.119444966, -0.228743598, -0.21865353, 0.255828887, 0.031033745, 0.198043287, 0.112054668, 0.289750516, 0.101838917], [0.0101497006, -0.153242037, -0.274101019, 0.098690778, -0.160164535, -0.0547989979, -0.317405343, 0.305983931, -0.121285819, 0.31579864], [-0.145706415, -0.127123475, 0.204059973, -0.160934925, -0.0418121815, 0.267757356, -0.0834247842, 0.210071325, 0.0511884503, -0.120828666], [-0.175574556, -0.207353354, 0.0341571234, -0.0626117662, 9.453990e-02, 0.351277143, -0.188232571, -0.0355442725, 0.104938827, -0.12084689], [-0.129702508, 0.307411969, 0.0600315407, 0.167721972, 0.159753412, 0.127536565, -0.267242491, 0.261240542, -0.202292293, -0.292796612], [-0.295962691, 0.114182763, 0.209769711, -0.229083464, 0.195489138, 0.307449818, 0.0942512751, -0.134523198, 0.189630061, -0.0974499732], [-6.004620e-02, -0.137981221, 0.0733050704, -0.281068265, -0.0486596823, 0.306077898, 0.30778569, -0.392285049, 0.109688483, -0.226047099], [-0.0333333872, 0.0985616743, -0.00378937298, 6.560670e-02, -0.0436274335, 0.19267787, 0.0276476815, 0.038748417, 0.0321611241, 0.238686085], [0.0775437951, 0.00480031688, 0.0890703275, 0.0813173428, 0.156091303, 0.207606182, 0.335114539, -0.150621563, -0.394196957, 0.268333822], [0.153643906, -0.335132509, -0.312138796, 5.368770e-02, 0.235048816, 0.0310786515, 0.15642716, 0.208705202, 0.215524375, -0.134175345], [0.10188897, -0.0959656313, 0.367264211, 0.222671688, -0.266171932, 0.110148534, 0.0319485664, 0.0366752297, 0.123669371, -0.175061852], [-0.097238183, -0.0620822273, 0.0445226021, -0.088823609, -0.187241077, -4.567820e-02, -0.202858582, -0.139873311, 0.146043435, 0.124500103], [-0.0526548438, -0.198502213, -0.145166814, -0.0782446712, -0.146434411, 0.185514018, 0.184155926, 0.161838412, -0.0683488697, 0.159353316], [-0.271061063, 0.371537626, 0.267237723, 0.377635479, 0.0745741948, -0.265771866, -0.263954222, 0.269931138, 0.189870864, -0.260571092], [0.123850107, -1.806440e-01, 0.321234286, 0.0514725558, -0.0060727438, -0.395113736, -0.251065284, 0.272852391, 0.201696262, 0.234759703], [0.265518636, 0.211273089, -5.142330e-02, -7.88406352E-4, -0.175895303, -0.24738647, 0.330014527, -0.2292009, -0.0545187145, -0.35968858], [-0.157047808, 0.277920961, 0.273503274, -0.0130148269, -0.326813549, 0.143175527, -0.00963719096, 0.167440012, 0.210747063, -0.0891537964], [0.0310225263, -0.0776267573, 0.276278019, -0.139116824, 0.121790454, -0.395071626, -2.326950e-01, 0.257626921, -0.127569184, 0.264735043], [0.0237917807, -0.278158903, -0.0585371666, 0.0991684943, 0.145461574, -0.0225479398, 0.320570618, -0.0800962671, -4.982980e-02, -0.0771938786], [0.107598737, 0.0567134842, 0.0050173765, 0.135508671, -0.242831439, 0.206493735, -0.257223487, 0.25564608, 0.13105084, -0.0108935023], [-0.0815954431, -0.0600086078, 0.0493025444, -0.241900951, 0.0250020418, 0.35276854, -0.313097537, 0.270228475, -0.270950615, 0.253449827], [0.114727132, -0.222702235, -0.151071653, -0.208600566, 0.314997226, -0.174035355, 0.344284892, -0.143412217, 0.188850269, 0.124864407], [-0.190904632, 0.0299436115, -0.191528931, -0.381732821, 0.175428227, 0.0987710803, 0.192728072, 0.17655462, 0.216957316, 0.232865736], [0.303614467, 0.23164387, -0.26379016, 0.217460141, -0.259905845, 0.298513204, -0.184439868, 0.232353494, -0.381244898, -0.227710232], [0.0208108909, 0.00839824323, -0.126433596, 0.192780793, 0.0649723336, -0.0253005717, -0.0480954498, 0.0768189952, 0.209024802, 0.00325355236], [0.161162391, 0.30670771, -0.0654156655, -0.227617249, 0.066418536, -0.0750855356, 0.159480572, -0.00124128303, -0.0957149043, -0.116955064], [-0.211400643, -0.0276241545, -0.0924634262, 0.395741016, 0.252569556, -0.28579244, -0.236819223, 0.046667546, 0.264011323, 0.311949909], [0.0855956227, -0.110965215, 0.111237064, -0.131076068, 0.0891130492, -0.155705988, -0.130113155, 0.254974216, 0.153850585, -0.0406821221]]> : tensor<128x10xf32> + %8 = "xla_hlo.dot"(%7, %cst_2) {name = "dot.13", precision_config = ["DEFAULT", "DEFAULT"]} : (tensor<1x128xf32>, tensor<128x10xf32>) -> tensor<1x10xf32> + %cst_3 = constant {name = "constant.14"} dense<[-0.0990044474, 0.106343128, 0.00506669516, -0.0421980098, 0.021251943, 0.200870886, 9.79547156E-4, 0.0746576563, -0.234273568, -0.0336938724]> : tensor<10xf32> + %9 = "xla_hlo.broadcast_in_dim"(%cst_3) {broadcast_dimensions = dense<1> : tensor<1xi64>, name = "broadcast.15"} : (tensor<10xf32>) -> tensor<1x10xf32> + %10 = "xla_hlo.add"(%8, %9) {name = "add.16"} : (tensor<1x10xf32>, tensor<1x10xf32>) -> tensor<1x10xf32> + %cst_4 = constant {name = "constant.17"} dense<0xFF800000> : tensor + %11 = "xla_hlo.reduce"(%10, %cst_4) ( { + ^bb0(%arg1: tensor, %arg2: tensor): // no predecessors + %20 = "xla_hlo.max"(%arg1, %arg2) {name = "maximum.21"} : (tensor, tensor) -> tensor + "xla_hlo.return"(%20) : (tensor) -> () + }) {dimensions = dense<1> : tensor<1xi64>} : (tensor<1x10xf32>, tensor) -> tensor<1xf32> + %12 = "xla_hlo.broadcast_in_dim"(%11) {broadcast_dimensions = dense<0> : tensor<1xi64>, name = "broadcast.23"} : (tensor<1xf32>) -> tensor<1x10xf32> + %13 = "xla_hlo.sub"(%10, %12) {name = "subtract.24"} : (tensor<1x10xf32>, tensor<1x10xf32>) -> tensor<1x10xf32> + %14 = "xla_hlo.exp"(%13) {name = "exponential.25"} : (tensor<1x10xf32>) -> tensor<1x10xf32> + %cst_5 = constant {name = "constant.27"} dense<0.000000e+00> : tensor + %15 = "xla_hlo.reduce"(%14, %cst_5) ( { + ^bb0(%arg3: tensor, %arg4: tensor): // no predecessors + %21 = "xla_hlo.add"(%arg3, %arg4) {name = "add.31"} : (tensor, tensor) -> tensor + "xla_hlo.return"(%21) : (tensor) -> () + }) {dimensions = dense<1> : tensor<1xi64>} : (tensor<1x10xf32>, tensor) -> tensor<1xf32> + %16 = "xla_hlo.broadcast_in_dim"(%15) {broadcast_dimensions = dense<0> : tensor<1xi64>, name = "broadcast.34"} : (tensor<1xf32>) -> tensor<1x10xf32> + %17 = "xla_hlo.div"(%14, %16) {name = "divide.35"} : (tensor<1x10xf32>, tensor<1x10xf32>) -> tensor<1x10xf32> + %18 = "xla_hlo.reshape"(%17) {name = "reshape.36"} : (tensor<1x10xf32>) -> tensor<1x10xf32> + %19 = "xla_hlo.tuple"(%18) {name = "tuple.37"} : (tensor<1x10xf32>) -> tuple> + return %19 : tuple> + } +} diff --git a/tools/debugger/debug_app.cc b/tools/debugger/debug_app.cc new file mode 100644 index 000000000000..aff429ac3663 --- /dev/null +++ b/tools/debugger/debug_app.cc @@ -0,0 +1,1435 @@ +// Copyright 2019 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "third_party/mlir_edge/iree/tools/debugger/debug_app.h" + +#include + +#include +#include + +#include "third_party/absl/flags/flag.h" +#include "third_party/absl/memory/memory.h" +#include "third_party/absl/strings/str_join.h" +#include "third_party/absl/strings/str_split.h" +#include "third_party/absl/types/optional.h" +#include "third_party/absl/types/source_location.h" +#include "third_party/dear_imgui/imgui.h" +#include "third_party/dear_imgui/imgui_internal.h" +#include "third_party/mlir_edge/iree/base/status.h" +#include "third_party/mlir_edge/iree/schemas/debug_service_generated.h" +#include "third_party/mlir_edge/iree/vm/bytecode_printer.h" +#include "third_party/mlir_edge/iree/vm/bytecode_tables_sequencer.h" +#include "third_party/mlir_edge/iree/vm/debug/debug_client.h" +#include "third_party/mlir_edge/iree/vm/module.h" +#include "third_party/mlir_edge/iree/vm/source_map.h" + +namespace iree { +namespace vm { +namespace debug { +namespace { + +void PushButtonHue(float hue) { + ImGui::PushStyleColor(ImGuiCol_Button, + (ImVec4)ImColor::HSV(hue / 7.0f, 0.6f, 0.6f)); + ImGui::PushStyleColor(ImGuiCol_ButtonHovered, + (ImVec4)ImColor::HSV(hue / 7.0f, 0.7f, 0.7f)); + ImGui::PushStyleColor(ImGuiCol_ButtonActive, + (ImVec4)ImColor::HSV(hue / 7.0f, 0.8f, 0.8f)); +} + +void PushButtonColor(const ImVec4& color) { + ImGui::PushStyleColor(ImGuiCol_Button, color); + ImGui::PushStyleColor(ImGuiCol_ButtonHovered, color); + ImGui::PushStyleColor(ImGuiCol_ButtonActive, color); +} + +void PopButtonStyle() { ImGui::PopStyleColor(3); } + +bool AreBreakpointsEqual(const RemoteBreakpoint& breakpoint, + const DebugApp::UserBreakpoint& user_breakpoint) { + if (user_breakpoint.active_breakpoint == &breakpoint) { + return true; + } else if (user_breakpoint.type != breakpoint.type()) { + return false; + } + switch (breakpoint.type()) { + case RemoteBreakpoint::Type::kBytecodeFunction: + if (user_breakpoint.function_ordinal != -1 && + user_breakpoint.function_ordinal != breakpoint.function_ordinal()) { + return false; + } + return breakpoint.module_name() == user_breakpoint.module_name && + breakpoint.function_name() == user_breakpoint.function_name && + breakpoint.bytecode_offset() == user_breakpoint.bytecode_offset; + case RemoteBreakpoint::Type::kNativeFunction: + return breakpoint.function_name() == user_breakpoint.native_function; + default: + return false; + } +} + +} // namespace + +// static +void DebugApp::PumpMainLoopThunk(void* arg) { + auto status = reinterpret_cast(arg)->PumpMainLoop(); + if (IsCancelled(status)) { + return; + } else if (!status.ok()) { + CHECK_OK(status); + } +} + +DebugApp::DebugApp(SDL_Window* window, SDL_GLContext gl_context, + const char* glsl_version) + : window_(window), gl_context_(gl_context) { + VLOG(1) << "DebugApp initializing..."; + IMGUI_CHECKVERSION(); + ImGui::CreateContext(); + ImGuiIO& io = ImGui::GetIO(); + io.ConfigFlags |= ImGuiConfigFlags_NavEnableKeyboard; + io.ConfigFlags |= ImGuiConfigFlags_DockingEnable; + + // TODO(benvanik): ini file for settings. + io.IniFilename = nullptr; + // ImGui::LoadIniSettingsFromMemory() + // ImGui::SaveIniSettingsToMemory() + + // TODO(benvanik): theming. + ImGui::StyleColorsDark(); + + // Setup Platform/Renderer bindings + ImGui_ImplSDL2_InitForOpenGL(window_, gl_context_); + ImGui_ImplOpenGL3_Init(glsl_version); + SDL_GL_MakeCurrent(nullptr, nullptr); + VLOG(1) << "DebugApp initialized"; +} + +DebugApp::~DebugApp() { + VLOG(1) << "DebugApp shutting down..."; + ImGui_ImplOpenGL3_Shutdown(); + ImGui_ImplSDL2_Shutdown(); + ImGui::DestroyContext(); + + SDL_GL_DeleteContext(gl_context_); + SDL_GL_MakeCurrent(nullptr, nullptr); + SDL_DestroyWindow(window_); + SDL_Quit(); + VLOG(1) << "DebugApp shut down (SDL_Quit)"; +} + +Status DebugApp::Connect(absl::string_view service_address) { + VLOG(1) << "Connecting to debug service at " << service_address << "..."; + ASSIGN_OR_RETURN(debug_client_, DebugClient::Connect(service_address, this)); + + // TODO(benvanik): load breakpoints from file. + UserBreakpoint user_breakpoint; + user_breakpoint.module_name = "module"; + user_breakpoint.function_name = "main"; + user_breakpoint.bytecode_offset = 0; + user_breakpoint.wants_enabled = true; + user_breakpoint_list_.push_back(std::move(user_breakpoint)); + RETURN_IF_ERROR(RefreshActiveBreakpoints()); + + // Set paused so that we need to resume to continue execution. + is_paused_ = true; + return OkStatus(); +} + +Status DebugApp::Disconnect() { + VLOG(1) << "Disconnecting from debug service"; + debug_client_.reset(); + return OkStatus(); +} + +bool DebugApp::is_paused() const { + if (!debug_client_) { + return false; + } + if (!hit_breakpoints_.empty()) { + return true; // One or more breakpoints hit. + } + return is_paused_ || !is_stepping_; +} + +RemoteFiberState* DebugApp::GetSelectedFiberState() const { + if (!debug_client_ || !selected_fiber_state_id_.has_value()) { + return nullptr; + } + for (auto* fiber_state : debug_client_->fiber_states()) { + if (fiber_state->id() == selected_fiber_state_id_.value()) { + return fiber_state; + } + } + return nullptr; +} + +Status DebugApp::RefreshActiveBreakpoints() { + // Set all breakpoints to disabled. We'll re-enable them as we find them + // below. + for (auto& user_breakpoint : user_breakpoint_list_) { + user_breakpoint.active_breakpoint = nullptr; + } + + // If not connected then no breakpoints are active. + if (!debug_client_) { + return OkStatus(); + } + + // Reconcile the user breakpoint list with the breakpoints available on the + // server. + for (auto* breakpoint : debug_client_->breakpoints()) { + auto it = + std::find_if(user_breakpoint_list_.begin(), user_breakpoint_list_.end(), + [breakpoint](const UserBreakpoint& user_breakpoint) { + return AreBreakpointsEqual(*breakpoint, user_breakpoint); + }); + if (it == user_breakpoint_list_.end()) { + // Breakpoint not found - add to user list. + UserBreakpoint user_breakpoint; + user_breakpoint.type = breakpoint->type(); + user_breakpoint.active_breakpoint = breakpoint; + user_breakpoint.module_name = breakpoint->module_name(); + user_breakpoint.function_name = breakpoint->function_name(); + user_breakpoint.function_ordinal = breakpoint->function_ordinal(); + user_breakpoint.bytecode_offset = breakpoint->bytecode_offset(); + user_breakpoint_list_.push_back(std::move(user_breakpoint)); + } else { + // Breakpoint found - set the active pointer. + UserBreakpoint& user_breakpoint = *it; + user_breakpoint.active_breakpoint = breakpoint; + user_breakpoint.is_enabling = false; + user_breakpoint.module_name = breakpoint->module_name(); + user_breakpoint.function_name = breakpoint->function_name(); + user_breakpoint.function_ordinal = breakpoint->function_ordinal(); + user_breakpoint.bytecode_offset = breakpoint->bytecode_offset(); + } + } + + // Ensure any breakpoint the user wants enabled is active/otherwise. + for (auto& user_breakpoint : user_breakpoint_list_) { + if (user_breakpoint.wants_enabled && !user_breakpoint.is_enabling && + !user_breakpoint.active_breakpoint) { + // Add breakpoint on server. + switch (user_breakpoint.type) { + case RemoteBreakpoint::Type::kBytecodeFunction: + RETURN_IF_ERROR(debug_client_->AddFunctionBreakpoint( + user_breakpoint.module_name, user_breakpoint.function_name, + user_breakpoint.bytecode_offset, + [&user_breakpoint](const RemoteBreakpoint& breakpoint) { + user_breakpoint.function_ordinal = + breakpoint.function_ordinal(); + })); + break; + case RemoteBreakpoint::Type::kNativeFunction: + // TODO(benvanik): native breakpoint support. + return UnimplementedErrorBuilder(ABSL_LOC) + << "Native function breakpoints are TODO"; + default: + return UnimplementedErrorBuilder(ABSL_LOC) + << "Unimplemented breakpoint type"; + } + user_breakpoint.is_enabling = true; + } else if (!user_breakpoint.wants_enabled && + user_breakpoint.active_breakpoint) { + // Remove breakpoint from server. + RETURN_IF_ERROR( + debug_client_->RemoveBreakpoint(*user_breakpoint.active_breakpoint)); + + user_breakpoint.active_breakpoint = nullptr; + } + } + + return OkStatus(); +} + +bool DebugApp::IsStoppedAtBreakpoint( + const UserBreakpoint& user_breakpoint) const { + return std::find(hit_breakpoints_.begin(), hit_breakpoints_.end(), + user_breakpoint.active_breakpoint) != hit_breakpoints_.end(); +} + +int DebugApp::FindMatchingUserBreakpointIndex(absl::string_view module_name, + int function_ordinal, + int offset) { + for (int i = 0; i < user_breakpoint_list_.size(); ++i) { + auto& user_breakpoint = user_breakpoint_list_[i]; + if (user_breakpoint.module_name == module_name && + user_breakpoint.function_ordinal == function_ordinal && + user_breakpoint.bytecode_offset == offset) { + return i; + } + } + return -1; +} + +int DebugApp::FindMatchingUserBreakpointIndex(absl::string_view module_name, + absl::string_view function_name, + int offset) { + for (int i = 0; i < user_breakpoint_list_.size(); ++i) { + auto& user_breakpoint = user_breakpoint_list_[i]; + if (user_breakpoint.module_name == module_name && + user_breakpoint.function_name == function_name && + user_breakpoint.bytecode_offset == offset) { + return i; + } + } + return -1; +} + +Status DebugApp::ResumeFromBreakpoint(UserBreakpoint* user_breakpoint) { + if (!user_breakpoint->active_breakpoint) { + return FailedPreconditionErrorBuilder(ABSL_LOC) << "Breakpoint not active"; + } + VLOG(1) << "Resuming from breakpoint " + << user_breakpoint->active_breakpoint->id() << "..."; + auto it = std::find(hit_breakpoints_.begin(), hit_breakpoints_.end(), + user_breakpoint->active_breakpoint); + if (it == hit_breakpoints_.end()) { + return NotFoundErrorBuilder(ABSL_LOC) << "Breakpoint not found"; + } + hit_breakpoints_.erase(it); + return debug_client_->MakeReady(); +} + +Status DebugApp::OnContextRegistered(const RemoteContext& context) { + // Ack event. + return debug_client_->MakeReady(); +} + +Status DebugApp::OnContextUnregistered(const RemoteContext& context) { + // Close documents that may reference modules in the context. + std::vector closing_documents; + for (auto& document : documents_) { + auto* module = document->function->module(); + if (module->context_id() != context.id()) { + // Document is not from this context so it's fine. + continue; + } + + // See if any other live context still has the module loaded. We can change + // the document over to that. + RemoteModule* replacement_module = nullptr; + for (auto* context : debug_client_->contexts()) { + for (auto* other_module : context->modules()) { + if (other_module->name() == module->name()) { + replacement_module = other_module; + break; + } + } + if (replacement_module) break; + } + if (replacement_module && replacement_module->is_loaded()) { + // Replace document module reference. + int function_ordinal = document->function->ordinal(); + auto functions = replacement_module->functions(); + if (function_ordinal < functions.size()) { + document->function = functions[function_ordinal]; + } else { + document->function = nullptr; + } + } else { + document->function = nullptr; + } + + if (!document->function) { + // Close the document if we don't have a valid function for it. + VLOG(1) + << "Closing document " << document->title + << " because the last context using the module is being unregistered"; + closing_documents.push_back(document.get()); + } + } + for (auto* document : closing_documents) { + auto it = std::find_if( + documents_.begin(), documents_.end(), + [document](const std::unique_ptr& open_document) { + return document == open_document.get(); + }); + documents_.erase(it); + } + + // Ack event. + return debug_client_->MakeReady(); +} + +Status DebugApp::OnModuleLoaded(const RemoteContext& context, + const RemoteModule& module) { + // Ack event. + return debug_client_->MakeReady(); +} + +Status DebugApp::OnFiberRegistered(const RemoteFiberState& fiber_state) { + if (!selected_fiber_state_id_.has_value()) { + selected_fiber_state_id_ = fiber_state.id(); + selected_stack_frame_index_ = {}; + } + + // Ack event. + return debug_client_->MakeReady(); +} + +Status DebugApp::OnFiberUnregistered(const RemoteFiberState& fiber_state) { + if (selected_fiber_state_id_.has_value() && + selected_fiber_state_id_.value() == fiber_state.id()) { + selected_fiber_state_id_ = {}; + selected_stack_frame_index_ = {}; + } + + // Ack event. + return debug_client_->MakeReady(); +} + +Status DebugApp::OnBreakpointHit(const RemoteBreakpoint& breakpoint, + const RemoteFiberState& fiber_state) { + // Keep track of where we are stopped. + hit_breakpoints_.push_back(&breakpoint); + return NavigateToCodeView(fiber_state, -1, NavigationMode::kMatchDocument); +} + +Status DebugApp::PumpMainLoop() { + ImGuiIO& io = ImGui::GetIO(); + + if (debug_client_) { + RETURN_IF_ERROR(debug_client_->Poll()); + } + RETURN_IF_ERROR(RefreshActiveBreakpoints()); + + SDL_GL_MakeCurrent(window_, gl_context_); + + SDL_Event event; + while (SDL_PollEvent(&event)) { + ImGui_ImplSDL2_ProcessEvent(&event); + if (event.type == SDL_QUIT) { + return CancelledErrorBuilder(ABSL_LOC) << "Quit hotkey"; + } else if (event.type == SDL_WINDOWEVENT && + event.window.event == SDL_WINDOWEVENT_CLOSE && + event.window.windowID == SDL_GetWindowID(window_)) { + return CancelledErrorBuilder(ABSL_LOC) << "Window closed"; + } + } + ImGui_ImplOpenGL3_NewFrame(); + ImGui_ImplSDL2_NewFrame(window_); + ImGui::NewFrame(); + + auto draw_status = DrawUI(); + if (!draw_status.ok()) { + // TODO(benvanik): show on screen? Probably all messed up. + LOG(ERROR) << draw_status; + } + + // Blit the entire ImGui UI. + ImGui::Render(); + SDL_GL_MakeCurrent(window_, gl_context_); + glViewport(0, 0, (int)io.DisplaySize.x, (int)io.DisplaySize.y); + glClearColor(0.45f, 0.55f, 0.60f, 1.0f); + glClear(GL_COLOR_BUFFER_BIT); + IREE_DISABLE_LEAK_CHECKS(); + ImGui_ImplOpenGL3_RenderDrawData(ImGui::GetDrawData()); + IREE_ENABLE_LEAK_CHECKS(); + + // Render additional viewport windows (desktop only). + if (io.ConfigFlags & ImGuiConfigFlags_ViewportsEnable) { + SDL_Window* backup_current_window = SDL_GL_GetCurrentWindow(); + SDL_GLContext backup_current_context = SDL_GL_GetCurrentContext(); + ImGui::UpdatePlatformWindows(); + ImGui::RenderPlatformWindowsDefault(); + SDL_GL_MakeCurrent(backup_current_window, backup_current_context); + } + + SDL_GL_SwapWindow(window_); + return OkStatus(); +} + +Status DebugApp::LayoutInitialDockSpace() { + dockspace_id_ = ImGui::GetID("MainDockSpace"); + if (ImGui::DockBuilderGetNode(dockspace_id_)) { + // Already configured. + return OkStatus(); + } + ImGui::DockBuilderAddNode(dockspace_id_, ImGuiDockNodeFlags_DockSpace); + + dock_content_id_ = dockspace_id_; + dock_top_id_ = ImGui::DockBuilderSplitNode(dock_content_id_, ImGuiDir_Up, + 0.05f, nullptr, &dock_content_id_); + dock_left_id_ = ImGui::DockBuilderSplitNode( + dock_content_id_, ImGuiDir_Left, 0.20f, nullptr, &dock_content_id_); + dock_bottom_id_ = ImGui::DockBuilderSplitNode( + dock_content_id_, ImGuiDir_Down, 0.20f, nullptr, &dock_content_id_); + dock_right_id_ = ImGui::DockBuilderSplitNode( + dock_content_id_, ImGuiDir_Right, 0.20f, nullptr, &dock_content_id_); + dock_bottom_left_id_ = ImGui::DockBuilderSplitNode( + dock_bottom_id_, ImGuiDir_Left, 0.50f, nullptr, &dock_bottom_right_id_); + + ImGui::DockBuilderDockWindow("Toolbar", dock_top_id_); + auto* dock_top_node = ImGui::DockBuilderGetNode(dock_top_id_); + dock_top_node->LocalFlags = ImGuiDockNodeFlags_NoSplit | + ImGuiDockNodeFlags_NoResize | + ImGuiDockNodeFlags_AutoHideTabBar; + + ImGui::DockBuilderDockWindow("Modules", dock_left_id_); + ImGui::DockBuilderDockWindow("Locals", dock_bottom_left_id_); + ImGui::DockBuilderDockWindow("Fibers", dock_bottom_right_id_); + ImGui::DockBuilderDockWindow("Breakpoints", dock_bottom_right_id_); + + ImGui::DockBuilderFinish(dockspace_id_); + return OkStatus(); +} + +Status DebugApp::DrawUI() { + ImGuiWindowFlags window_flags = + ImGuiWindowFlags_MenuBar | ImGuiWindowFlags_NoDocking; + window_flags |= ImGuiWindowFlags_NoTitleBar | ImGuiWindowFlags_NoCollapse | + ImGuiWindowFlags_NoResize | ImGuiWindowFlags_NoMove | + ImGuiWindowFlags_NoNavFocus; + + ImGuiViewport* viewport = ImGui::GetMainViewport(); + ImGui::SetNextWindowPos(viewport->Pos); + ImGui::SetNextWindowSize(viewport->Size); + ImGui::SetNextWindowViewport(viewport->ID); + ImGui::PushStyleVar(ImGuiStyleVar_WindowRounding, 0.0f); + ImGui::PushStyleVar(ImGuiStyleVar_WindowBorderSize, 0.0f); + ImGui::PushStyleVar(ImGuiStyleVar_WindowPadding, ImVec2(0.0f, 0.0f)); + ImGui::Begin("IREEDebugRoot", nullptr, window_flags); + ImGui::PopStyleVar(3); + + RETURN_IF_ERROR(LayoutInitialDockSpace()); + ImGui::DockSpace(dockspace_id_, ImVec2(0.0f, 0.0f), ImGuiDockNodeFlags_None); + + RETURN_IF_ERROR(DrawMainMenu()); + RETURN_IF_ERROR(DrawToolbar()); + + ImGui::PushStyleVar(ImGuiStyleVar_WindowPadding, ImVec2(2, 2)); + RETURN_IF_ERROR(DrawBreakpointListPanel()); + RETURN_IF_ERROR(DrawModuleListPanel()); + RETURN_IF_ERROR(DrawLocalListPanel()); + RETURN_IF_ERROR(DrawFiberStateListPanel()); + ImGui::PopStyleVar(); + + RETURN_IF_ERROR(DrawCodeViewPanels()); + + ImGui::End(); + return OkStatus(); +} + +Status DebugApp::DrawMainMenu() { + if (!ImGui::BeginMenuBar()) return OkStatus(); + + // TODO(benvanik): main menu. + if (ImGui::BeginMenu("File")) { + ImGui::EndMenu(); + } + + ImGui::EndMenuBar(); + return OkStatus(); +} + +Status DebugApp::DrawToolbar() { + // TODO(benvanik): figure out how to make this not grow. + ImGui::Begin("Toolbar", nullptr, + ImGuiWindowFlags_NoResize | ImGuiWindowFlags_NoTitleBar | + ImGuiWindowFlags_NoMove | ImGuiWindowFlags_NoCollapse | + ImGuiWindowFlags_NoScrollbar); + ImGui::BeginGroup(); + +#if !defined(IMGUI_DISABLE_DEMO_WINDOWS) + static bool show_demo_window = false; + if (ImGui::Button("Demo")) { + show_demo_window = !show_demo_window; + } + if (show_demo_window) { + ImGui::SetNextWindowDockID(dock_content_id_); + ImGui::ShowDemoWindow(&show_demo_window); + } +#endif // !IMGUI_DISABLE_DEMO_WINDOWS + + ImGui::SameLine(); + if (!debug_client_) { + if (ImGui::Button("Connect")) { + // TODO(benvanik): connection dialog and/or autoconnect. + } + } else { + if (ImGui::Button("Disconnect")) { + debug_client_.reset(); + } + } + + ImGui::SameLine(); + if (debug_client_) { + ImGui::Text(""); + } else { + ImGui::TextDisabled("disconnected"); + } + + ImGui::SameLine(); + ImGui::Spacing(); + ImGui::SameLine(); + ImGui::Spacing(); + + ImGui::SameLine(); + ImGui::BeginGroup(); + ImGui::Text("Fiber: "); + ImGui::SameLine(); + ImGui::SetNextItemWidth(300); + auto* selected_fiber_state = GetSelectedFiberState(); + const std::string& active_fiber_name = + selected_fiber_state ? selected_fiber_state->name() : ""; + if (ImGui::BeginCombo("##active_fiber", active_fiber_name.c_str(), + ImGuiComboFlags_PopupAlignLeft)) { + if (debug_client_) { + for (auto* fiber_state : debug_client_->fiber_states()) { + ImGui::PushID(fiber_state->id()); + bool is_selected = fiber_state == selected_fiber_state; + if (ImGui::Selectable(fiber_state->name().c_str(), is_selected)) { + RETURN_IF_ERROR(NavigateToCodeView(*fiber_state, -1, + NavigationMode::kMatchDocument)); + } + if (is_selected) { + ImGui::SetItemDefaultFocus(); + } + ImGui::PopID(); + } + } + ImGui::EndCombo(); + } + ImGui::EndGroup(); + + ImGui::SameLine(); + ImGui::BeginGroup(); + static const float kPauseButtonHue = 0.0f; + static const float kResumeButtonHue = 2.0f; + static const float kStepButtonHue = 1.0f; + if (debug_client_ && !is_paused()) { + PushButtonHue(kPauseButtonHue); + if (ImGui::Button("Pause")) { + RETURN_IF_ERROR(debug_client_->SuspendAllFibers()); + } + PopButtonStyle(); + } else if (debug_client_ && is_paused()) { + ImGui::PushStyleColor(ImGuiCol_Button, 0xFF666666); + ImGui::PushStyleColor(ImGuiCol_Text, 0xFFAAAAAA); + ImGui::ButtonEx("Pause", {}, ImGuiButtonFlags_Disabled); + ImGui::PopStyleColor(2); + } + if (debug_client_ && is_paused()) { + ImGui::SameLine(); + PushButtonHue(kResumeButtonHue); + if (ImGui::Button("Resume")) { + if (is_paused_) { + is_paused_ = false; + RETURN_IF_ERROR(debug_client_->MakeReady()); + } + while (!hit_breakpoints_.empty()) { + hit_breakpoints_.pop_back(); + RETURN_IF_ERROR(debug_client_->MakeReady()); + } + } + PopButtonStyle(); + } else { + ImGui::PushStyleColor(ImGuiCol_Button, 0xFF666666); + ImGui::PushStyleColor(ImGuiCol_Text, 0xFFAAAAAA); + ImGui::SameLine(); + ImGui::ButtonEx("Resume", {}, ImGuiButtonFlags_Disabled); + ImGui::PopStyleColor(2); + } + + if (debug_client_ && is_paused() && selected_fiber_state) { + ImGui::SameLine(); + PushButtonHue(kStepButtonHue); + if (ImGui::Button("Step Into")) { + RETURN_IF_ERROR(debug_client_->StepFiber(*selected_fiber_state, [this]() { + is_paused_ = true; + is_stepping_ = false; + })); + is_stepping_ = true; + } + PopButtonStyle(); + ImGui::SameLine(); + if (ImGui::Button("Step Over")) { + RETURN_IF_ERROR( + debug_client_->StepFiberOver(*selected_fiber_state, [this]() { + is_paused_ = true; + is_stepping_ = false; + })); + is_stepping_ = true; + } + ImGui::SameLine(); + if (ImGui::Button("Step Out")) { + RETURN_IF_ERROR( + debug_client_->StepFiberOut(*selected_fiber_state, [this]() { + is_paused_ = true; + is_stepping_ = false; + })); + is_stepping_ = true; + } + if (ImGui::BeginPopup("Step to...")) { + // TODO(benvanik): step to Invoke exit, next FFI call, etc + ImGui::MenuItem("(stuff)"); + ImGui::EndPopup(); + } + ImGui::SameLine(); + if (ImGui::Button("Step to...")) { + ImGui::OpenPopup("Step to..."); + } + } else { + ImGui::PushStyleColor(ImGuiCol_Button, 0xFF666666); + ImGui::PushStyleColor(ImGuiCol_Text, 0xFFAAAAAA); + ImGui::SameLine(); + ImGui::ButtonEx("Step Into", {}, ImGuiButtonFlags_Disabled); + ImGui::SameLine(); + ImGui::ButtonEx("Step Over", {}, ImGuiButtonFlags_Disabled); + ImGui::SameLine(); + ImGui::ButtonEx("Step Out", {}, ImGuiButtonFlags_Disabled); + ImGui::SameLine(); + ImGui::ButtonEx("Step to...", {}, ImGuiButtonFlags_Disabled); + ImGui::PopStyleColor(2); + } + ImGui::EndGroup(); + + ImGui::EndGroup(); + ImGui::End(); + return OkStatus(); +} + +Status DebugApp::DrawBreakpointListPanel() { + static bool is_panel_visible = true; + if (!ImGui::Begin("Breakpoints", &is_panel_visible, ImGuiWindowFlags_None)) { + ImGui::End(); + return OkStatus(); + } + + ImGui::PushStyleVar(ImGuiStyleVar_WindowPadding, ImVec2(8, 8)); + absl::optional add_breakpoint_type; + if (ImGui::BeginPopup("+ Function")) { + if (ImGui::MenuItem("Bytecode Function")) { + add_breakpoint_type = RemoteBreakpoint::Type::kBytecodeFunction; + } + if (ImGui::MenuItem("Native Function")) { + add_breakpoint_type = RemoteBreakpoint::Type::kNativeFunction; + } + ImGui::EndPopup(); + } + ImGui::PopStyleVar(); + if (ImGui::Button("+ Function")) { + ImGui::OpenPopup("+ Function"); + } + RETURN_IF_ERROR(DrawAddBreakpointDialogs(add_breakpoint_type)); + + ImGui::SameLine(); + if (ImGui::Button("Remove All")) { + // FIXME(benvanik): removal all is broken - need removebreakpoints or a + // 'want_removal' flag so that RefreshActiveBreakpoints handles things. + // Right now if you have 2 breakpoints and hit remove all the second will + // come back during the next refresh (as the server hasn't removed it yet). + for (auto& user_breakpoint : user_breakpoint_list_) { + if (user_breakpoint.active_breakpoint) { + RETURN_IF_ERROR(debug_client_->RemoveBreakpoint( + *user_breakpoint.active_breakpoint)); + user_breakpoint.active_breakpoint = nullptr; + } + } + user_breakpoint_list_.clear(); + } + ImGui::Separator(); + + ImGui::BeginChild("BreakpointList", ImVec2(-1, -1), false, + ImGuiWindowFlags_AlwaysVerticalScrollbar); + std::vector dead_breakpoints; + for (auto& user_breakpoint : user_breakpoint_list_) { + ASSIGN_OR_RETURN(bool should_keep, DrawBreakpoint(&user_breakpoint)); + if (!should_keep) { + dead_breakpoints.push_back(&user_breakpoint); + } + } + for (auto* user_breakpoint : dead_breakpoints) { + for (auto it = user_breakpoint_list_.begin(); + it != user_breakpoint_list_.end(); ++it) { + if (&*it == user_breakpoint) { + if (user_breakpoint->active_breakpoint) { + RETURN_IF_ERROR(debug_client_->RemoveBreakpoint( + *user_breakpoint->active_breakpoint)); + } + user_breakpoint_list_.erase(it); + break; + } + } + } + ImGui::EndChild(); + + ImGui::End(); + return OkStatus(); +} + +StatusOr DebugApp::DrawBreakpoint(UserBreakpoint* user_breakpoint) { + std::string breakpoint_name; + switch (user_breakpoint->type) { + case RemoteBreakpoint::Type::kBytecodeFunction: + breakpoint_name = + absl::StrCat("[bytecode] ", user_breakpoint->module_name, ":", + user_breakpoint->function_name, ":", + user_breakpoint->bytecode_offset); + if (user_breakpoint->function_ordinal != -1) { + absl::StrAppend(&breakpoint_name, " @", + user_breakpoint->function_ordinal); + } + break; + case RemoteBreakpoint::Type::kNativeFunction: + breakpoint_name = + absl::StrCat("[native ] ", user_breakpoint->native_function); + break; + } + ImGui::BeginGroup(); + bool is_closing = true; + bool is_expanded = ImGui::CollapsingHeader( + ("##" + breakpoint_name).c_str(), &is_closing, + ImGuiTreeNodeFlags_Framed | ImGuiTreeNodeFlags_NoTreePushOnOpen | + ImGuiTreeNodeFlags_NoAutoOpenOnLog | ImGuiTreeNodeFlags_OpenOnArrow | + ImGuiTreeNodeFlags_OpenOnDoubleClick); + ImGui::SameLine(); + ImGui::Checkbox(breakpoint_name.c_str(), &user_breakpoint->wants_enabled); + ImGui::EndGroup(); + if (!is_expanded) { + return is_closing; + } + ImGui::PushID(breakpoint_name.c_str()); + + ImGui::Text("(breakpoint stats/etc)"); + + ImGui::PopID(); + return is_closing; +} + +Status DebugApp::DrawAddBreakpointDialogs( + absl::optional add_breakpoint_type) { + if (add_breakpoint_type.has_value()) { + switch (add_breakpoint_type.value()) { + case RemoteBreakpoint::Type::kBytecodeFunction: + ImGui::OpenPopup("Add Bytecode Function Breakpoint"); + break; + case RemoteBreakpoint::Type::kNativeFunction: + ImGui::OpenPopup("Add Native Function Breakpoint"); + break; + } + } + ImGui::PushStyleVar(ImGuiStyleVar_WindowPadding, ImVec2(8, 8)); + RETURN_IF_ERROR(DrawAddBytecodeFunctionBreakpointDialog()); + RETURN_IF_ERROR(DrawAddNativeFunctionBreakpointDialog()); + ImGui::PopStyleVar(); + return OkStatus(); +} + +Status DebugApp::DrawAddBytecodeFunctionBreakpointDialog() { + ImGui::SetNextWindowSize(ImVec2(400, 400), ImGuiCond_FirstUseEver); + bool close_popup = true; + if (!ImGui::BeginPopupModal("Add Bytecode Function Breakpoint", &close_popup, + ImGuiWindowFlags_None)) { + return OkStatus(); + } + ImGui::BeginGroup(); + ImGui::BeginChild("##data_entry", + ImVec2(0, -ImGui::GetFrameHeightWithSpacing())); + + ImGui::TextWrapped( + "Adds a breakpoint set on the entry of the function (offset=0)."); + ImGui::Separator(); + + // TODO(benvanik): fancy list, filtering, etc. + + static char module_name[256] = {0}; + ImGui::InputText("Module", module_name, sizeof(module_name)); + ImGui::SetItemDefaultFocus(); + + static char function_name[256] = {0}; + ImGui::InputText("Function", function_name, sizeof(function_name)); + + ImGui::EndChild(); + ImGui::Separator(); + + if (ImGui::Button("Add")) { + int offset = 0; + if (FindMatchingUserBreakpointIndex(module_name, function_name, offset) == + -1) { + UserBreakpoint user_breakpoint; + user_breakpoint.type = RemoteBreakpoint::Type::kBytecodeFunction; + user_breakpoint.module_name = module_name; + user_breakpoint.function_name = function_name; + user_breakpoint.bytecode_offset = offset; + user_breakpoint.wants_enabled = true; + user_breakpoint_list_.push_back(std::move(user_breakpoint)); + } + ImGui::CloseCurrentPopup(); + } + ImGui::SameLine(); + if (ImGui::Button("Cancel")) { + ImGui::CloseCurrentPopup(); + } + + ImGui::EndGroup(); + ImGui::EndPopup(); + return OkStatus(); +} + +Status DebugApp::DrawAddNativeFunctionBreakpointDialog() { + ImGui::SetNextWindowSize(ImVec2(400, 400), ImGuiCond_FirstUseEver); + bool close_popup = true; + if (!ImGui::BeginPopupModal("Add Native Function Breakpoint", &close_popup, + ImGuiWindowFlags_None)) { + return OkStatus(); + } + ImGui::BeginGroup(); + ImGui::BeginChild("##data_entry", + ImVec2(0, -ImGui::GetFrameHeightWithSpacing())); + + ImGui::TextWrapped( + "Adds a breakpoint set on any call to the given FFI imported " + "function."); + ImGui::Separator(); + + static char function_name[256] = {0}; + ImGui::InputText("Function", function_name, sizeof(function_name)); + ImGui::SetItemDefaultFocus(); + + ImGui::EndChild(); + ImGui::Separator(); + + if (ImGui::Button("Add")) { + UserBreakpoint user_breakpoint; + user_breakpoint.type = RemoteBreakpoint::Type::kNativeFunction; + user_breakpoint.native_function = function_name; + user_breakpoint.wants_enabled = true; + user_breakpoint_list_.push_back(std::move(user_breakpoint)); + ImGui::CloseCurrentPopup(); + } + ImGui::SameLine(); + if (ImGui::Button("Cancel")) { + ImGui::CloseCurrentPopup(); + } + + ImGui::EndGroup(); + ImGui::EndPopup(); + return OkStatus(); +} + +Status DebugApp::DrawModuleListPanel() { + static bool is_panel_visible = true; + if (!ImGui::Begin("Modules", &is_panel_visible, ImGuiWindowFlags_None)) { + ImGui::End(); + return OkStatus(); + } else if (!debug_client_) { + ImGui::TextDisabled("disconnected"); + ImGui::End(); + return OkStatus(); + } + ImGui::PushStyleVar(ImGuiStyleVar_WindowPadding, ImVec2(4, 4)); + + ImGui::BeginGroup(); + ImGui::SetNextItemWidth(ImGui::GetContentRegionAvailWidth()); + static char function_name_filter_text[256] = {0}; + ImGui::InputTextWithHint( + "##function_name_filter", "Filter functions", function_name_filter_text, + sizeof(function_name_filter_text), ImGuiInputTextFlags_AutoSelectAll); + ImGuiTextFilter function_name_filter(function_name_filter_text); + ImGui::EndGroup(); + + ImGui::Separator(); + + ImGui::BeginGroup(); + ImGui::BeginChild("##context_list", ImVec2(0, -ImGui::GetFrameHeight())); + for (auto* context : debug_client_->contexts()) { + RETURN_IF_ERROR(DrawContext(*context, function_name_filter)); + } + ImGui::EndChild(); + ImGui::EndGroup(); + + ImGui::PopStyleVar(); + ImGui::End(); + return OkStatus(); +} + +Status DebugApp::DrawContext(const RemoteContext& context, + const ImGuiTextFilter& filter) { + std::string context_name = absl::StrCat("Context ", context.id()); + if (!ImGui::CollapsingHeader(context_name.c_str(), nullptr, + ImGuiTreeNodeFlags_DefaultOpen | + ImGuiTreeNodeFlags_Framed | + ImGuiTreeNodeFlags_NoTreePushOnOpen | + ImGuiTreeNodeFlags_NoAutoOpenOnLog | + ImGuiTreeNodeFlags_OpenOnArrow | + ImGuiTreeNodeFlags_OpenOnDoubleClick)) { + return OkStatus(); + } + ImGui::PushID(context.id()); + for (auto* module : context.modules()) { + RETURN_IF_ERROR(DrawModule(module, filter)); + } + ImGui::PopID(); + return OkStatus(); +} + +Status DebugApp::DrawModule(RemoteModule* module, + const ImGuiTextFilter& filter) { + ImGui::PushID(module->name().c_str()); + if (ImGui::TreeNodeEx(module->name().c_str(), + ImGuiTreeNodeFlags_Framed | + ImGuiTreeNodeFlags_DefaultOpen | + ImGuiTreeNodeFlags_OpenOnDoubleClick | + ImGuiTreeNodeFlags_OpenOnArrow)) { + if (module->CheckLoadedOrRequest()) { + for (auto* function : module->functions()) { + char function_name[128]; + if (function->name().empty()) { + std::snprintf(function_name, sizeof(function_name), "@%d", + function->ordinal()); + } else { + std::snprintf(function_name, sizeof(function_name), "@%d %s", + function->ordinal(), function->name().c_str()); + } + if (filter.IsActive() && !filter.PassFilter(function_name)) { + continue; + } + ImGui::PushID(function->ordinal()); + bool is_selected = false; + if (ImGui::Selectable("##selectable", &is_selected, + ImGuiSelectableFlags_AllowDoubleClick | + ImGuiSelectableFlags_DrawFillAvailWidth)) { + if (is_selected) { + RETURN_IF_ERROR(NavigateToCodeView(module->name(), + function->ordinal(), 0, + NavigationMode::kMatchDocument)); + } + } + ImGui::SameLine(); + // TODO(benvanik): detect if breakpoint active at offset 0. + ImGui::BulletText("%s", function_name); + ImGui::PopID(); + } + } else { + ImGui::TextDisabled("Loading..."); + } + ImGui::TreePop(); + } + ImGui::PopID(); + return OkStatus(); +} + +Status DebugApp::DrawLocalListPanel() { + static bool is_panel_visible = true; + if (!ImGui::Begin("Locals", &is_panel_visible, ImGuiWindowFlags_None)) { + ImGui::End(); + return OkStatus(); + } else if (!debug_client_) { + ImGui::TextDisabled("disconnected"); + ImGui::End(); + return OkStatus(); + } + auto* fiber_state = GetSelectedFiberState(); + if (!fiber_state) { + ImGui::TextDisabled("select a fiber to view locals"); + ImGui::End(); + return OkStatus(); + } else if (fiber_state->def().frames.empty()) { + ImGui::TextDisabled("(fiber has no frames)"); + ImGui::End(); + return OkStatus(); + } + int stack_frame_index = selected_stack_frame_index_.value_or(-1); + if (stack_frame_index == -1) { + stack_frame_index = fiber_state->def().frames.size() - 1; + } + auto& stack_frame = fiber_state->def().frames[stack_frame_index]; + + // TODO(benvanik): toggle for IREE VM locals vs. source locals. + for (int i = 0; i < stack_frame->locals.size(); ++i) { + auto& local = stack_frame->locals[i]; + RETURN_IF_ERROR(DrawLocal(fiber_state, stack_frame_index, i, *local)); + } + + ImGui::End(); + return OkStatus(); +} + +Status DebugApp::DrawLocal(RemoteFiberState* fiber_state, int stack_frame_index, + int local_index, const rpc::BufferViewDefT& local) { + // TODO(benvanik): columns and such in fancy table. + ImGui::Text("l%d", local_index); + ImGui::SameLine(50); + if (local.is_valid) { + auto shape_str = + absl::StrCat(absl::StrJoin(local.shape, "x"), "x", local.element_size); + ImGui::Text("%s", shape_str.c_str()); + } else { + ImGui::TextDisabled("∅"); + } + // TODO(benvanik): editing options (change shape, change contents, upload). + // TODO(benvanik): save/download/log options. + return OkStatus(); +} + +Status DebugApp::DrawFiberStateListPanel() { + static bool is_panel_visible = true; + if (!ImGui::Begin("Fibers", &is_panel_visible, ImGuiWindowFlags_None)) { + ImGui::End(); + return OkStatus(); + } else if (!debug_client_) { + ImGui::TextDisabled("disconnected"); + ImGui::End(); + return OkStatus(); + } + for (auto* fiber_state : debug_client_->fiber_states()) { + RETURN_IF_ERROR(DrawFiberState(*fiber_state)); + } + ImGui::End(); + return OkStatus(); +} + +Status DebugApp::DrawFiberState(const RemoteFiberState& fiber_state) { + // TODO(benvanik): expand if any breakpoints are stopped in fiber. + if (selected_fiber_state_id_.has_value() && + selected_fiber_state_id_.value() == fiber_state.id()) { + ImGui::SetNextTreeNodeOpen(true); + } + if (!ImGui::CollapsingHeader(fiber_state.name().c_str())) { + return OkStatus(); + } + ImGui::PushID(fiber_state.id()); + + for (int i = 0; i < fiber_state.def().frames.size(); ++i) { + const auto& stack_frame = fiber_state.def().frames[i]; + ImGui::PushID(i); + // TODO(benvanik): highlight frames with breakpoints in them. + bool is_selected = selected_fiber_state_id_.has_value() && + selected_fiber_state_id_.value() == fiber_state.id() && + selected_stack_frame_index_.has_value() && + selected_stack_frame_index_.value() == i; + if (ImGui::Selectable("##selectable", &is_selected, + ImGuiSelectableFlags_AllowDoubleClick | + ImGuiSelectableFlags_DrawFillAvailWidth)) { + // TODO(benvanik): detect when clicking but already selected. + if (is_selected) { + RETURN_IF_ERROR( + NavigateToCodeView(fiber_state, i, NavigationMode::kMatchDocument)); + } + } + ImGui::SameLine(); + ImGui::Bullet(); + ImGui::SameLine(); + // TODO(benvanik): better naming/etc (resolve function). + ImGui::Text("%s:%d:%d", stack_frame->module_name.c_str(), + stack_frame->function_ordinal, stack_frame->offset); + + ImGui::PopID(); + } + + ImGui::PopID(); + return OkStatus(); +} + +DebugApp::CodeViewDocument* DebugApp::FindMatchingDocument( + absl::string_view module_name, int function_ordinal) { + for (auto& document : documents_) { + if (document->function->module()->name() == module_name && + document->function->ordinal() == function_ordinal) { + return document.get(); + } + } + return nullptr; +} + +Status DebugApp::NavigateToCodeView(absl::string_view module_name, + int function_ordinal, int offset, + NavigationMode navigation_mode) { + if (!debug_client_) { + return UnavailableErrorBuilder(ABSL_LOC) << "No connection established"; + } + VLOG(1) << "NavigateToCodeView(" << module_name << ", " << function_ordinal + << ", " << offset << ")"; + CodeViewDocument* existing_document = nullptr; + switch (navigation_mode) { + case NavigationMode::kNewDocument: + // Fall through and create below. + break; + case NavigationMode::kCurrentDocument: + // Not yet done - treat as a new document. + break; + case NavigationMode::kMatchDocument: + existing_document = FindMatchingDocument(module_name, function_ordinal); + break; + } + if (existing_document) { + ImGui::SetWindowFocus(existing_document->title.c_str()); + return OkStatus(); + } + + // TODO(benvanik): make this common code. + RETURN_IF_ERROR(debug_client_->GetFunction( + std::string(module_name), function_ordinal, + [this, offset](StatusOr function_or) { + if (!function_or.ok()) { + // TODO(benvanik): error dialog. + CHECK_OK(function_or.status()); + } + auto* function = function_or.ValueOrDie(); + auto document = absl::make_unique(); + document->title = + absl::StrCat(function->module()->name(), ":", function->name()); + document->function = function; + document->focus_offset = offset; + ImGui::SetWindowFocus(document->title.c_str()); + documents_.push_back(std::move(document)); + })); + return OkStatus(); +} + +Status DebugApp::NavigateToCodeView(absl::string_view module_name, + absl::string_view function_name, int offset, + NavigationMode navigation_mode) { + if (!debug_client_) { + return UnavailableErrorBuilder(ABSL_LOC) << "No connection established"; + } + return debug_client_->ResolveFunction( + std::string(module_name), std::string(function_name), + [this, navigation_mode, module_name, + offset](StatusOr function_ordinal) { + CHECK_OK(function_ordinal.status()); + CHECK_OK(NavigateToCodeView(module_name, function_ordinal.ValueOrDie(), + offset, navigation_mode)); + }); +} + +Status DebugApp::NavigateToCodeView(const RemoteFiberState& fiber_state, + int stack_frame_index, + NavigationMode navigation_mode) { + if (!debug_client_) { + return UnavailableErrorBuilder(ABSL_LOC) << "No connection established"; + } + const auto& stack_frame = stack_frame_index == -1 + ? *fiber_state.def().frames.back() + : *fiber_state.def().frames[stack_frame_index]; + selected_fiber_state_id_ = fiber_state.id(); + selected_stack_frame_index_ = stack_frame_index; + return NavigateToCodeView(stack_frame.module_name, + stack_frame.function_ordinal, stack_frame.offset, + NavigationMode::kMatchDocument); +} + +Status DebugApp::NavigateToCodeView(const UserBreakpoint& user_breakpoint, + NavigationMode navigation_mode) { + if (!debug_client_) { + return UnavailableErrorBuilder(ABSL_LOC) << "No connection established"; + } + switch (user_breakpoint.type) { + case RemoteBreakpoint::Type::kBytecodeFunction: + if (user_breakpoint.function_ordinal != -1) { + return NavigateToCodeView( + user_breakpoint.module_name, user_breakpoint.function_ordinal, + user_breakpoint.bytecode_offset, navigation_mode); + } else { + return NavigateToCodeView( + user_breakpoint.module_name, user_breakpoint.function_name, + user_breakpoint.bytecode_offset, navigation_mode); + } + case RemoteBreakpoint::Type::kNativeFunction: + return UnimplementedErrorBuilder(ABSL_LOC) + << "Navigation to non-bytecode functions unimplemented"; + } +} + +Status DebugApp::DrawCodeViewPanels() { + // If we've disconnected then we need to clear bodies. + // TODO(benvanik): allow documents to persist by caching all required info. + if (!debug_client_) { + documents_.clear(); + return OkStatus(); + } + + std::vector closing_documents; + for (auto& document : documents_) { + ASSIGN_OR_RETURN(bool is_open, DrawCodeViewDocument(document.get())); + if (!is_open) { + closing_documents.push_back(document.get()); + } + } + for (auto* closing_document : closing_documents) { + auto it = std::find_if( + documents_.begin(), documents_.end(), + [closing_document](const std::unique_ptr& document) { + return document.get() == closing_document; + }); + documents_.erase(it); + } + return OkStatus(); +} + +StatusOr DebugApp::DrawCodeViewDocument(CodeViewDocument* document) { + ImGui::SetNextWindowDockID(dockspace_id_, ImGuiCond_FirstUseEver); + ImGui::PushStyleVar(ImGuiStyleVar_WindowPadding, ImVec2(0, 0)); + bool is_open = true; + bool is_visible = + ImGui::Begin(document->title.c_str(), &is_open, ImGuiWindowFlags_None); + if (!is_open || !is_visible) { + ImGui::End(); + ImGui::PopStyleVar(); + return is_open; + } + ImGui::PopStyleVar(); + + auto* remote_module = document->function->module(); + auto* remote_function = document->function; + if (remote_module->CheckLoadedOrRequest() && + remote_function->CheckLoadedOrRequest()) { + // TODO(benvanik): draw function signature. + if (remote_function->bytecode()) { + RETURN_IF_ERROR(DrawBytecodeCodeView(document)); + } else { + // TODO(benvanik): display native registration info. + ImGui::TextDisabled("(native)"); + } + } else { + ImGui::TextDisabled("loading..."); + } + + ImGui::End(); + return true; +} + +Status DebugApp::PrepareBytecodeCodeView(CodeViewDocument* document) { + auto* remote_module = document->function->module(); + auto* remote_function = document->function; + + ASSIGN_OR_RETURN(auto module, Module::FromDef(remote_module->def())); + + // TODO(benvanik): source map support. + // Want line count including source lines, IR lines, and bytecode lines. + // May want lower level (JIT/etc) lines too. + + // TODO(benvanik): bytecode iterator for richer display. + auto source_map_resolver = SourceMapResolver::FromFunction( + module->def(), remote_function->ordinal()); + vm::BytecodePrinter printer(vm::sequencer_opcode_table(), + module->function_table(), + module->executable_table(), source_map_resolver); + ASSIGN_OR_RETURN(std::string full_string, + printer.Print(*remote_function->bytecode())); + document->bytecode_info.lines = absl::StrSplit(full_string, '\n'); + + return OkStatus(); +} + +Status DebugApp::DrawBytecodeCodeView(CodeViewDocument* document) { + // Ensure we have cached our line information. + RETURN_IF_ERROR(PrepareBytecodeCodeView(document)); + + auto* remote_module = document->function->module(); + auto* remote_function = document->function; + + ImGui::BeginGroup(); + ImGui::BeginChild("##bytecode_view", ImVec2(0, 0), false, + ImGuiWindowFlags_AlwaysVerticalScrollbar); + ImGui::PushStyleVar(ImGuiStyleVar_ItemSpacing, ImVec2(0, 0)); + + // TODO(benvanik): cache breakpoints for this function for faster lookup. + + auto& bytecode_info = document->bytecode_info; + ImGuiListClipper clipper(bytecode_info.lines.size(), + ImGui::GetTextLineHeightWithSpacing()); + while (clipper.Step()) { + for (int i = clipper.DisplayStart; i < clipper.DisplayEnd; ++i) { + ImGui::PushID(i); + + // TODO(benvanik): lookup line info. + int bytecode_offset = 0; + int breakpoint_index = FindMatchingUserBreakpointIndex( + remote_module->name(), remote_function->ordinal(), bytecode_offset); + bool has_breakpoint = breakpoint_index != -1; + bool active_on_any_fiber = false; + bool active_on_selected_fiber = false; + + ImGui::Dummy(ImVec2(4, 0)); + + // Gutter breakpoint button. + ImGui::SameLine(); + if (has_breakpoint) { + PushButtonHue(0.0f); // Red + if (ImGui::Button(" ##toggle_breakpoint")) { + CHECK_GE(breakpoint_index, 0); + auto& user_breakpoint = user_breakpoint_list_[breakpoint_index]; + if (user_breakpoint.active_breakpoint) { + RETURN_IF_ERROR(debug_client_->RemoveBreakpoint( + *user_breakpoint.active_breakpoint)); + } + user_breakpoint_list_.erase(user_breakpoint_list_.begin() + + breakpoint_index); + } + PopButtonStyle(); + if (ImGui::IsItemHovered()) { + ImGui::SetTooltip("Remove the breakpoint at this offset."); + } + } else { + PushButtonColor(ImGui::GetStyleColorVec4(ImGuiCol_ChildBg)); + if (ImGui::Button(" ##toggle_breakpoint")) { + UserBreakpoint user_breakpoint; + user_breakpoint.type = RemoteBreakpoint::Type::kBytecodeFunction; + user_breakpoint.module_name = remote_module->name(); + user_breakpoint.function_name = remote_function->name(); + user_breakpoint.bytecode_offset = bytecode_offset; + user_breakpoint.wants_enabled = true; + user_breakpoint_list_.push_back(std::move(user_breakpoint)); + } + PopButtonStyle(); + if (ImGui::IsItemHovered()) { + ImGui::SetTooltip("Add a breakpoint at this offset."); + } + } + + // Active execution chevron (shows when active or any fiber is executing + // this region). + ImGui::SameLine(); + if (active_on_selected_fiber) { + // The selected fiber is active here. + ImGui::TextColored(ImGui::GetStyleColorVec4(ImGuiCol_SeparatorActive), + " > "); + } else if (active_on_any_fiber) { + // At least one other fiber is active here. + ImGui::TextColored(ImGui::GetStyleColorVec4(ImGuiCol_Separator), " > "); + } else { + // Not active. + ImGui::Text(" "); + } + + // Line contents. + ImGui::SameLine(); + ImGui::Text("%s", bytecode_info.lines[i].c_str()); + + if (document->focus_offset.has_value() && + bytecode_offset == document->focus_offset.value()) { + document->bytecode_offset = document->focus_offset.value(); + document->focus_offset = {}; + ImGui::SetScrollHereY(); + } + + ImGui::PopID(); + } + } + + ImGui::PopStyleVar(); + ImGui::EndChild(); + ImGui::EndGroup(); + + return OkStatus(); +} + +} // namespace debug +} // namespace vm +} // namespace iree diff --git a/tools/debugger/debug_app.h b/tools/debugger/debug_app.h new file mode 100644 index 000000000000..d1674f2991f8 --- /dev/null +++ b/tools/debugger/debug_app.h @@ -0,0 +1,214 @@ +// Copyright 2019 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#ifndef THIRD_PARTY_MLIR_EDGE_IREE_TOOLS_DEBUGGER_DEBUG_APP_H_ +#define THIRD_PARTY_MLIR_EDGE_IREE_TOOLS_DEBUGGER_DEBUG_APP_H_ + +#include + +#include "third_party/absl/strings/string_view.h" +#include "third_party/absl/types/optional.h" +#include "third_party/mlir_edge/iree/base/status.h" +#include "third_party/mlir_edge/iree/vm/debug/debug_client.h" + +// NOTE: order matters here, imgui must come first: +#include "third_party/dear_imgui/imgui.h" +// NOTE: must follow imgui.h: +#include "third_party/dear_imgui/examples/imgui_impl_opengl3.h" +#include "third_party/dear_imgui/examples/imgui_impl_sdl.h" + +// Workaround for terrible bad SDL/graphics driver leaks. +// If you see these macros being used it means that the code between is not +// really under our control and not a leak we would be able to prevent. +#if defined(__has_feature) +#if __has_feature(address_sanitizer) +#include +#define IREE_DISABLE_LEAK_CHECKS() __lsan_disable() +#define IREE_ENABLE_LEAK_CHECKS() __lsan_enable() +#else +#define IREE_DISABLE_LEAK_CHECKS() +#define IREE_ENABLE_LEAK_CHECKS() +#endif // __has_feature(address_sanitizer) +#endif // __has_feature + +namespace iree { +namespace vm { +namespace debug { + +// Debug client app UI. +// Uses a DebugClient to communicate with a remote DebugServer and ImGui to +// display a nifty UI. +// +// See the ImGui site for more info: https://github.com/ocornut/imgui +// The most useful thing is the imgui_demo.cpp file that contains example usage +// of most features. +class DebugApp : private DebugClient::Listener { + public: + struct UserBreakpoint { + RemoteBreakpoint::Type type = RemoteBreakpoint::Type::kBytecodeFunction; + const RemoteBreakpoint* active_breakpoint = nullptr; + bool wants_enabled = true; + bool is_enabling = false; + // TODO(benvanik): reuse BreakpointDef here? + std::string module_name; + std::string function_name; + int function_ordinal = -1; + int bytecode_offset = 0; + std::string native_function; + }; + + static void PumpMainLoopThunk(void* arg); + + DebugApp(SDL_Window* window, SDL_GLContext gl_context, + const char* glsl_version); + ~DebugApp(); + + // Connects to the service at the specified address. + Status Connect(absl::string_view service_address); + // Disconnects from the currently connected service, if any. + Status Disconnect(); + + // Returns true if the remote service is paused at our request. + bool is_paused() const; + + // Pumps the main UI loop once. + // This polls the DebugClient, SDL input, and renders the UI. + // It should be called as frequently as possible to ensure snappy UI updates. + // Returns CancelledError if the app is being closed by the user. + Status PumpMainLoop(); + + // Defines how NavigationToCodeView methods behave. + enum class NavigationMode { + // The target will be opened in a new document tab. + kNewDocument, + // The target will be opened in the current document tab, replacing the + // current contents. + kCurrentDocument, + // The target will be opened in a document tab that mostly matches (like + // the same function in a module at a different offset), otherwise a new + // document will be opened. + kMatchDocument, + }; + + // Navigates to a particular function offset based on resolution of the given + // arguments. Navigation may happen asynchronously if targets need to be + // resolved or contents fetched. + Status NavigateToCodeView(absl::string_view module_name, int function_ordinal, + int offset, NavigationMode navigation_mode); + Status NavigateToCodeView(absl::string_view module_name, + absl::string_view function_name, int offset, + NavigationMode navigation_mode); + Status NavigateToCodeView(const RemoteFiberState& fiber_state, + int stack_frame_index, + NavigationMode navigation_mode); + Status NavigateToCodeView(const UserBreakpoint& user_breakpoint, + NavigationMode navigation_mode); + + private: + struct CodeViewDocument { + // Document display title (and ID). + std::string title; + // Function (and offset within the function) being displayed. + RemoteFunction* function = nullptr; + int bytecode_offset = 0; + // Set to a bytecode offset to have the document focus there. + absl::optional focus_offset; + // Cached info for bytecode display. + struct { + std::vector lines; + } bytecode_info; + }; + + CodeViewDocument* FindMatchingDocument(absl::string_view module_name, + int function_ordinal); + RemoteFiberState* GetSelectedFiberState() const; + + Status RefreshActiveBreakpoints(); + bool IsStoppedAtBreakpoint(const UserBreakpoint& user_breakpoint) const; + int FindMatchingUserBreakpointIndex(absl::string_view module_name, + int function_ordinal, int offset); + int FindMatchingUserBreakpointIndex(absl::string_view module_name, + absl::string_view function_name, + int offset); + Status ResumeFromBreakpoint(UserBreakpoint* user_breakpoint); + + Status OnContextRegistered(const RemoteContext& context) override; + Status OnContextUnregistered(const RemoteContext& context) override; + Status OnModuleLoaded(const RemoteContext& context, + const RemoteModule& module) override; + Status OnFiberRegistered(const RemoteFiberState& fiber_state) override; + Status OnFiberUnregistered(const RemoteFiberState& fiber_state) override; + Status OnBreakpointHit(const RemoteBreakpoint& breakpoint, + const RemoteFiberState& fiber_state) override; + + Status LayoutInitialDockSpace(); + + Status DrawUI(); + Status DrawMainMenu(); + Status DrawToolbar(); + + Status DrawBreakpointListPanel(); + StatusOr DrawBreakpoint(UserBreakpoint* user_breakpoint); + Status DrawAddBreakpointDialogs( + absl::optional add_breakpoint_type); + Status DrawAddBytecodeFunctionBreakpointDialog(); + Status DrawAddNativeFunctionBreakpointDialog(); + + Status DrawModuleListPanel(); + Status DrawContext(const RemoteContext& context, + const ImGuiTextFilter& filter); + Status DrawModule(RemoteModule* module, const ImGuiTextFilter& filter); + + Status DrawLocalListPanel(); + Status DrawLocal(RemoteFiberState* fiber_state, int stack_frame_index, + int local_index, const rpc::BufferViewDefT& local); + + Status DrawFiberStateListPanel(); + Status DrawFiberState(const RemoteFiberState& fiber_state); + + Status DrawCodeViewPanels(); + StatusOr DrawCodeViewDocument(CodeViewDocument* document); + Status PrepareBytecodeCodeView(CodeViewDocument* document); + Status DrawBytecodeCodeView(CodeViewDocument* document); + + SDL_Window* window_ = nullptr; + SDL_GLContext gl_context_ = nullptr; + + ImGuiID dockspace_id_; + ImGuiID dock_top_id_; + ImGuiID dock_left_id_; + ImGuiID dock_bottom_id_; + ImGuiID dock_bottom_left_id_; + ImGuiID dock_bottom_right_id_; + ImGuiID dock_right_id_; + ImGuiID dock_content_id_; + + std::unique_ptr debug_client_; + std::vector user_breakpoint_list_; + + bool is_paused_ = false; + std::vector hit_breakpoints_; + bool is_stepping_ = false; + + absl::optional selected_fiber_state_id_; + absl::optional selected_stack_frame_index_; + + std::vector> documents_; +}; + +} // namespace debug +} // namespace vm +} // namespace iree + +#endif // THIRD_PARTY_MLIR_EDGE_IREE_TOOLS_DEBUGGER_DEBUG_APP_H_ diff --git a/tools/debugger/debug_app.html b/tools/debugger/debug_app.html new file mode 100644 index 000000000000..b55264730415 --- /dev/null +++ b/tools/debugger/debug_app.html @@ -0,0 +1,75 @@ + + + + + + + + IREE Debugger + + + + + + + + diff --git a/tools/debugger/debug_app_embedded.cc b/tools/debugger/debug_app_embedded.cc new file mode 100644 index 000000000000..2ac69a97c9c2 --- /dev/null +++ b/tools/debugger/debug_app_embedded.cc @@ -0,0 +1,151 @@ +// Copyright 2019 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "third_party/mlir_edge/iree/tools/debugger/debug_app_embedded.h" + +#include + +#include // NOLINT + +#include "third_party/SDL2/include/SDL_thread.h" +#include "third_party/absl/base/thread_annotations.h" +#include "third_party/absl/memory/memory.h" +#include "third_party/absl/synchronization/mutex.h" +#include "third_party/mlir_edge/iree/base/status.h" +#include "third_party/mlir_edge/iree/tools/debugger/debug_app.h" + +namespace iree { +namespace vm { +namespace debug { + +class InProcessEmbeddedDebugger : public EmbeddedDebugger { + public: + explicit InProcessEmbeddedDebugger(std::unique_ptr app) + : app_(std::move(app)) { + thread_ = + SDL_CreateThread(&ThreadMainThunk, "InProcessEmbeddedDebugger", this); + } + + ~InProcessEmbeddedDebugger() override { + VLOG(1) << "Setting shutdown flag and waiting on thread..."; + shutdown_flag_ = true; + int status = 0; + SDL_WaitThread(thread_, &status); + VLOG(1) << "Thread shutdown, killing app..."; + app_.reset(); + } + + Status AwaitClose() override { + await_mutex_.LockWhen(absl::Condition( + +[](bool* is_shutdown) { return *is_shutdown; }, &is_shutdown_)); + auto status = std::move(shutdown_status_); + await_mutex_.Unlock(); + return status; + } + + private: + static int ThreadMainThunk(void* arg) { + return reinterpret_cast(arg)->ThreadMain(); + } + + int ThreadMain() { + VLOG(1) << "Thread entry"; + while (!shutdown_flag_) { + auto status = app_->PumpMainLoop(); + if (IsCancelled(status)) { + shutdown_flag_ = true; + break; + } else if (!shutdown_flag_ && !status.ok()) { + absl::MutexLock lock(&await_mutex_); + shutdown_status_ = std::move(status); + // TODO(benvanik): don't check unless no one is watching. + CHECK_OK(shutdown_status_); + } + } + app_.reset(); + { + absl::MutexLock lock(&await_mutex_); + is_shutdown_ = true; + } + VLOG(1) << "Thread exit"; + return 0; + } + + std::unique_ptr app_; + SDL_Thread* thread_; + std::atomic shutdown_flag_ = {false}; + absl::Mutex await_mutex_; + bool is_shutdown_ ABSL_GUARDED_BY(await_mutex_) = false; + Status shutdown_status_ ABSL_GUARDED_BY(await_mutex_); +}; + +StatusOr> LaunchDebugger() { + return AttachDebugger(""); +} + +StatusOr> AttachDebugger( + absl::string_view service_address) { + LOG(INFO) << "Launching embedded debugger; service=" << service_address; + IREE_DISABLE_LEAK_CHECKS(); + + if (SDL_Init(SDL_INIT_VIDEO | SDL_INIT_TIMER) != 0) { + return InternalErrorBuilder(ABSL_LOC) + << "Unable to init SDL: " << SDL_GetError(); + } + +#if __APPLE__ + // GL 3.2 Core + GLSL 150 + const char* glsl_version = "#version 150"; + SDL_GL_SetAttribute( + SDL_GL_CONTEXT_FLAGS, + SDL_GL_CONTEXT_FORWARD_COMPATIBLE_FLAG); // Always required on Mac + SDL_GL_SetAttribute(SDL_GL_CONTEXT_PROFILE_MASK, SDL_GL_CONTEXT_PROFILE_CORE); + SDL_GL_SetAttribute(SDL_GL_CONTEXT_MAJOR_VERSION, 3); + SDL_GL_SetAttribute(SDL_GL_CONTEXT_MINOR_VERSION, 2); +#else + // GL 3.0 + GLSL 130 + const char* glsl_version = "#version 130"; + SDL_GL_SetAttribute(SDL_GL_CONTEXT_FLAGS, 0); + SDL_GL_SetAttribute(SDL_GL_CONTEXT_PROFILE_MASK, SDL_GL_CONTEXT_PROFILE_CORE); + SDL_GL_SetAttribute(SDL_GL_CONTEXT_MAJOR_VERSION, 3); + SDL_GL_SetAttribute(SDL_GL_CONTEXT_MINOR_VERSION, 0); +#endif + + SDL_GL_SetAttribute(SDL_GL_DOUBLEBUFFER, 1); + SDL_GL_SetAttribute(SDL_GL_DEPTH_SIZE, 24); + SDL_GL_SetAttribute(SDL_GL_STENCIL_SIZE, 8); + SDL_DisplayMode current; + SDL_GetCurrentDisplayMode(0, ¤t); + SDL_WindowFlags window_flags = (SDL_WindowFlags)( + SDL_WINDOW_OPENGL | SDL_WINDOW_RESIZABLE | SDL_WINDOW_ALLOW_HIGHDPI); + SDL_Window* window = + SDL_CreateWindow("IREE Debugger (embedded)", SDL_WINDOWPOS_CENTERED, + SDL_WINDOWPOS_CENTERED, 1280, 720, window_flags); + SDL_GLContext gl_context = SDL_GL_CreateContext(window); + SDL_GL_MakeCurrent(nullptr, nullptr); + + IREE_ENABLE_LEAK_CHECKS(); + + auto app = absl::make_unique(window, gl_context, glsl_version); + if (!service_address.empty()) { + RETURN_IF_ERROR(app->Connect(service_address)); + } + + auto handle = absl::make_unique(std::move(app)); + return handle; +} + +} // namespace debug +} // namespace vm +} // namespace iree diff --git a/tools/debugger/debug_app_embedded.h b/tools/debugger/debug_app_embedded.h new file mode 100644 index 000000000000..d7e3f0b6dd9b --- /dev/null +++ b/tools/debugger/debug_app_embedded.h @@ -0,0 +1,52 @@ +// Copyright 2019 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#ifndef THIRD_PARTY_MLIR_EDGE_IREE_TOOLS_DEBUGGER_DEBUG_APP_EMBEDDED_H_ +#define THIRD_PARTY_MLIR_EDGE_IREE_TOOLS_DEBUGGER_DEBUG_APP_EMBEDDED_H_ + +#include + +#include "third_party/absl/strings/string_view.h" +#include "third_party/mlir_edge/iree/base/status.h" + +namespace iree { +namespace vm { +namespace debug { + +// RAII handle for keeping the debugger alive. +// When the instance is destroyed the debugger app will be closed. +class EmbeddedDebugger { + public: + virtual ~EmbeddedDebugger() = default; + + // Blocks the caller until the debugger is closed by the user. + virtual Status AwaitClose() = 0; +}; + +// Launches the debugger app. +// Returns a handle that can be used to wait for the debugger to close or +// force it to close. +StatusOr> LaunchDebugger(); + +// Launches the debugger app and attaches to the given server address. +// Returns a handle that can be used to wait for the debugger to close or +// force it to close. +StatusOr> AttachDebugger( + absl::string_view service_address); + +} // namespace debug +} // namespace vm +} // namespace iree + +#endif // THIRD_PARTY_MLIR_EDGE_IREE_TOOLS_DEBUGGER_DEBUG_APP_EMBEDDED_H_ diff --git a/tools/debugger/debug_app_main_emscripten.cc b/tools/debugger/debug_app_main_emscripten.cc new file mode 100644 index 000000000000..b789a687085e --- /dev/null +++ b/tools/debugger/debug_app_main_emscripten.cc @@ -0,0 +1,69 @@ +// Copyright 2019 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// Emscripten debug_app entry point. +// Though we are using SDL here we need to do some emscripten-specific magic to +// handle the different main looping mode (as we can't block in main() like on +// other platforms) as well as support some emscripten-specific features for +// file upload/download/etc. + +#include +#include + +#include "third_party/mlir_edge/iree/base/init.h" +#include "third_party/mlir_edge/iree/tools/debugger/debug_app.h" + +namespace iree { +namespace vm { +namespace debug { + +extern "C" int main(int argc, char** argv) { + InitializeEnvironment(&argc, &argv); + + if (SDL_Init(SDL_INIT_VIDEO) != 0) { + printf("Error: %s\n", SDL_GetError()); + return -1; + } + + const char* glsl_version = "#version 100"; + SDL_GL_SetAttribute(SDL_GL_CONTEXT_FLAGS, 0); + SDL_GL_SetAttribute(SDL_GL_CONTEXT_PROFILE_MASK, SDL_GL_CONTEXT_PROFILE_ES); + SDL_GL_SetAttribute(SDL_GL_CONTEXT_MAJOR_VERSION, 2); + SDL_GL_SetAttribute(SDL_GL_CONTEXT_MINOR_VERSION, 0); + + SDL_GL_SetAttribute(SDL_GL_DOUBLEBUFFER, 1); + SDL_GL_SetAttribute(SDL_GL_DEPTH_SIZE, 24); + SDL_GL_SetAttribute(SDL_GL_STENCIL_SIZE, 8); + SDL_DisplayMode current; + SDL_GetCurrentDisplayMode(0, ¤t); + SDL_WindowFlags window_flags = (SDL_WindowFlags)( + SDL_WINDOW_OPENGL | SDL_WINDOW_RESIZABLE | SDL_WINDOW_ALLOW_HIGHDPI); + SDL_Window* window = + SDL_CreateWindow("IREE Debugger", SDL_WINDOWPOS_CENTERED, + SDL_WINDOWPOS_CENTERED, 1280, 720, window_flags); + SDL_GLContext gl_context = SDL_GL_CreateContext(window); + if (!gl_context) { + printf("Failed to initialize WebGL context!\n"); + return 1; + } + + auto app = absl::make_unique(window, gl_context, glsl_version); + ::emscripten_set_main_loop_arg(DebugApp::PumpMainLoopThunk, app.release(), 0, + false); + return 0; +} + +} // namespace debug +} // namespace vm +} // namespace iree diff --git a/tools/debugger/debug_app_main_native.cc b/tools/debugger/debug_app_main_native.cc new file mode 100644 index 000000000000..869f2a5bf4d3 --- /dev/null +++ b/tools/debugger/debug_app_main_native.cc @@ -0,0 +1,45 @@ +// Copyright 2019 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// Native (linux/etc) debug_app entry point. +// This should work on any platform with pthreads and SDL support. + +#include "third_party/mlir_edge/iree/base/init.h" +#include "third_party/mlir_edge/iree/base/status.h" +#include "third_party/mlir_edge/iree/tools/debugger/debug_app_embedded.h" + +namespace iree { +namespace vm { +namespace debug { + +Status Run() { + ASSIGN_OR_RETURN(auto handle, LaunchDebugger()); + RETURN_IF_ERROR(handle->AwaitClose()); + handle.reset(); + return OkStatus(); +} + +extern "C" int main(int argc, char** argv) { + InitializeEnvironment(&argc, &argv); + auto status = Run(); + if (!status.ok()) { + LOG(ERROR) << "Debugger error: " << status; + return 1; + } + return 0; +} + +} // namespace debug +} // namespace vm +} // namespace iree diff --git a/tools/debugger/debug_cli_main.cc b/tools/debugger/debug_cli_main.cc new file mode 100644 index 000000000000..5d189f3cc2f7 --- /dev/null +++ b/tools/debugger/debug_cli_main.cc @@ -0,0 +1,40 @@ +// Copyright 2019 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "third_party/absl/flags/flag.h" +#include "third_party/mlir_edge/iree/base/init.h" +#include "third_party/mlir_edge/iree/base/status.h" +#include "third_party/mlir_edge/iree/tools/debugger/debug_prompt.h" + +ABSL_FLAG(std::string, debug_service_uri, "0.0.0.0:6000", + "IP/port of debug service to connect to."); + +namespace iree { +namespace vm { +namespace debug { + +Status Run() { + // TODO(benvanik): retry until connected? would allow auto-build reconnects. + return AttachDebugPrompt(absl::GetFlag(FLAGS_debug_service_uri)); +} + +extern "C" int main(int argc, char** argv) { + InitializeEnvironment(&argc, &argv); + CHECK_OK(Run()); + return 0; +} + +} // namespace debug +} // namespace vm +} // namespace iree diff --git a/tools/debugger/debug_prompt.cc b/tools/debugger/debug_prompt.cc new file mode 100644 index 000000000000..25a445e6a0b3 --- /dev/null +++ b/tools/debugger/debug_prompt.cc @@ -0,0 +1,90 @@ +// Copyright 2019 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "third_party/mlir_edge/iree/tools/debugger/debug_prompt.h" + +#include "third_party/mlir_edge/iree/base/status.h" +#include "third_party/mlir_edge/iree/vm/debug/debug_client.h" + +namespace iree { +namespace vm { +namespace debug { +namespace { + +class DebugPrompt : private DebugClient::Listener { + public: + Status Connect(absl::string_view debug_service_uri) { + // Connect to the debug service. + ASSIGN_OR_RETURN(debug_client_, + DebugClient::Connect(debug_service_uri, this)); + return OkStatus(); + } + + Status Run() { + // Query commands, transmit requests, and dispatch responses. + while (true) { + RETURN_IF_ERROR(debug_client_->Poll()); + + // TODO(benvanik): ask for a command. + // TODO(benvanik): display stuff. + } + } + + private: + Status OnContextRegistered(const RemoteContext& context) override { + // Ack. + return debug_client_->MakeReady(); + } + + Status OnContextUnregistered(const RemoteContext& context) override { + // Ack. + return debug_client_->MakeReady(); + } + + Status OnModuleLoaded(const RemoteContext& context, + const RemoteModule& module) override { + // Ack. + return debug_client_->MakeReady(); + } + + Status OnFiberRegistered(const RemoteFiberState& fiber_state) override { + // Ack. + return debug_client_->MakeReady(); + } + + Status OnFiberUnregistered(const RemoteFiberState& fiber_state) override { + // Ack. + return debug_client_->MakeReady(); + } + + Status OnBreakpointHit(const RemoteBreakpoint& breakpoint, + const RemoteFiberState& fiber_state) override { + // Ack. + return debug_client_->MakeReady(); + } + + std::unique_ptr debug_client_; +}; + +} // namespace + +Status AttachDebugPrompt(absl::string_view debug_service_uri) { + DebugPrompt debug_prompt; + RETURN_IF_ERROR(debug_prompt.Connect(debug_service_uri)); + return debug_prompt.Run(); +} + +} // namespace debug +} // namespace vm +} // namespace iree diff --git a/tools/debugger/debug_prompt.h b/tools/debugger/debug_prompt.h new file mode 100644 index 000000000000..bc95da093bf9 --- /dev/null +++ b/tools/debugger/debug_prompt.h @@ -0,0 +1,35 @@ +// Copyright 2019 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#ifndef THIRD_PARTY_MLIR_EDGE_IREE_TOOLS_DEBUGGER_DEBUG_PROMPT_H_ +#define THIRD_PARTY_MLIR_EDGE_IREE_TOOLS_DEBUGGER_DEBUG_PROMPT_H_ + +#include "third_party/absl/strings/string_view.h" +#include "third_party/mlir_edge/iree/base/status.h" + +namespace iree { +namespace vm { +namespace debug { + +// TODO(benvanik): take stdin/stdout as arguments. +// Attaches a debug prompt reading stdin for commands and printing results to +// stdout. The calling thread will block until the debugger is exited or the +// debug service closes. +Status AttachDebugPrompt(absl::string_view debug_service_uri); + +} // namespace debug +} // namespace vm +} // namespace iree + +#endif // THIRD_PARTY_MLIR_EDGE_IREE_TOOLS_DEBUGGER_DEBUG_PROMPT_H_ diff --git a/tools/run_mlir_main.cc b/tools/run_mlir_main.cc new file mode 100644 index 000000000000..f49abc1b8aec --- /dev/null +++ b/tools/run_mlir_main.cc @@ -0,0 +1,343 @@ +// Copyright 2019 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// IREE source.mlir -> execution output test runner. +// This is meant to be called from LIT for FileCheck tests, and tries to match +// the interface of mlir-opt (featuring -split-input-file, etc) so it's easier +// to work with there. If you want a more generalized runner for standalone +// precompiled IREE modules use //third_party/mlir_edge/iree/tools:run_module. +// +// By default all exported functions in the module will be run in order. +// All input values, provided via -input-values, will be passed to the +// functions (this means all input signatures must match). Results from the +// executed functions will be printed to stdout for checking. +// Use -output_types to set the function output data types, which like args will +// be used for all functions executed. +// +// Example input: +// // RUN: iree-run %s | FileCheck %s +// // CHECK-LABEL: @foo +// // CHECK: 1xf32: 2 +// func @foo() -> memref attributes {iree.module.export} { +// %0 = "iree.constant"() {value: dense, 2.0>} : () -> memref +// return %0 : memref +// } + +#include "third_party/absl/flags/flag.h" +#include "third_party/absl/strings/numbers.h" +#include "third_party/absl/strings/str_replace.h" +#include "third_party/absl/strings/str_split.h" +#include "third_party/absl/strings/string_view.h" +#include "third_party/absl/types/source_location.h" +#include "third_party/llvm/llvm/include/llvm/ADT/StringRef.h" +#include "third_party/llvm/llvm/include/llvm/Support/SourceMgr.h" +#include "third_party/llvm/llvm/projects/google_mlir/include/mlir/IR/Attributes.h" +#include "third_party/llvm/llvm/projects/google_mlir/include/mlir/IR/Function.h" +#include "third_party/llvm/llvm/projects/google_mlir/include/mlir/IR/MLIRContext.h" +#include "third_party/llvm/llvm/projects/google_mlir/include/mlir/IR/Module.h" +#include "third_party/llvm/llvm/projects/google_mlir/include/mlir/Parser.h" +#include "third_party/llvm/llvm/projects/google_mlir/include/mlir/Support/FileUtilities.h" +#include "third_party/mlir_edge/iree/base/init.h" +#include "third_party/mlir_edge/iree/base/status.h" +#include "third_party/mlir_edge/iree/compiler/Translation/SequencerModuleTranslation.h" +#include "third_party/mlir_edge/iree/hal/buffer_view_string_util.h" +#include "third_party/mlir_edge/iree/hal/driver_registry.h" +#include "third_party/mlir_edge/iree/schemas/module_def_generated.h" +#include "third_party/mlir_edge/iree/vm/bytecode_tables_sequencer.h" +#include "third_party/mlir_edge/iree/vm/debug/debug_server_flags.h" +#include "third_party/mlir_edge/iree/vm/fiber_state.h" +#include "third_party/mlir_edge/iree/vm/instance.h" +#include "third_party/mlir_edge/iree/vm/module.h" +#include "third_party/mlir_edge/iree/vm/module_printer.h" +#include "third_party/mlir_edge/iree/vm/sequencer_context.h" + +ABSL_FLAG(bool, split_input_file, true, + "Split the input file into multiple modules."); + +ABSL_FLAG(std::string, target_backends, "", + "Comma-separated list of target backends to translate executables " + "into. Omit to translate using all linked-in backend translators."); +ABSL_FLAG( + bool, export_all, true, + "Automatically add the iree.module.export attribute to all functions."); + +ABSL_FLAG(std::string, input_values, "", "Input shapes and optional values."); +ABSL_FLAG(std::string, output_types, "", + "Output data types (comma delimited list of b/i/u/f for " + "binary/signed int/unsigned int/float)."); + +// TODO(benvanik): is there a more canonical flag we can use? +ABSL_FLAG(bool, print_mlir, true, "Prints MLIR IR during translation."); + +ABSL_FLAG(bool, print_bytecode, false, + "Prints IREE bytecode after translation."); + +namespace iree { +namespace { + +using ::iree::hal::BufferView; +using ::iree::vm::Function; +using ::iree::vm::Module; +using ::iree::vm::ModuleFile; + +// Returns a driver name capable of handling input from the given backend. +std::string BackendToDriverName(std::string backend) { + size_t dash = backend.find('-'); + if (dash == std::string::npos) { + return backend; + } else { + return backend.substr(0, dash); + } +} + +// Prepares a module for evaluation by running MLIR import and IREE translation. +StatusOr> PrepareModule( + std::string target_backend, + std::unique_ptr file_buffer) { + mlir::MLIRContext context; + + // Parse input MLIR module. + llvm::SourceMgr source_mgr; + source_mgr.AddNewSourceBuffer(std::move(file_buffer), llvm::SMLoc()); + mlir::OwningModuleRef mlir_module = + mlir::parseSourceFile(source_mgr, &context); + + if (absl::GetFlag(FLAGS_export_all)) { + for (auto function : mlir_module->getOps()) { + function.setAttr("iree.module.export", mlir::UnitAttr::get(&context)); + } + } + + // Translate from MLIR to IREE bytecode. + mlir::iree_compiler::ModuleTranslationOptions options; + options.print_mlir = absl::GetFlag(FLAGS_print_mlir); + options.target_backends = {target_backend}; + auto iree_module_bytes = + mlir::iree_compiler::translateMlirToIreeSequencerModule(mlir_module.get(), + options); + if (iree_module_bytes.empty()) { + return iree::InternalErrorBuilder(ABSL_LOC) + << "Error translating MLIR to an IREE sequencer module"; + } + + if (absl::GetFlag(FLAGS_print_mlir)) { + mlir_module->dump(); + } + + // Wrap module in a file handle. + ASSIGN_OR_RETURN(auto iree_module_file, + ModuleFile::FromBuffer(ModuleDefIdentifier(), + std::move(iree_module_bytes))); + return Module::FromFile(std::move(iree_module_file)); +} + +// Parses a list of input shapes and values from a string of newline-separated +// inputs. Expects the contents to have one value per line with each value +// listed as +// [shape]xtype=[value] +// Example: +// 4x4xi8=0,1,2,3 +StatusOr> ParseInputsFromFlags( + hal::Allocator* allocator) { + std::string file_contents = + absl::StrReplaceAll(absl::GetFlag(FLAGS_input_values), {{"\\n", "\n"}}); + std::vector inputs; + for (const auto& line : + absl::StrSplit(file_contents, '\n', absl::SkipWhitespace())) { + ASSIGN_OR_RETURN(auto input, + hal::ParseBufferViewFromString(line, allocator)); + inputs.push_back(input); + } + return inputs; +} + +// Outputs all results from the function to stdout in IREE BufferView format. +Status OutputFunctionResults(const Function& function, + absl::Span results) { + std::vector output_types = + absl::StrSplit(absl::GetFlag(FLAGS_output_types), + absl::delimiter::AnyOf(", "), absl::SkipWhitespace()); + if (!output_types.empty() && output_types.size() != results.size()) { + return InvalidArgumentErrorBuilder(ABSL_LOC) + << "--output_types= specified but has " << output_types.size() + << " types when the function returns " << results.size(); + } + + for (int i = 0; i < results.size(); ++i) { + const auto& result = results[i]; + auto print_mode = hal::BufferViewPrintMode::kFloatingPoint; + if (!output_types.empty()) { + ASSIGN_OR_RETURN(print_mode, + hal::ParseBufferViewPrintMode(output_types[i])); + } + ASSIGN_OR_RETURN(auto result_str, + hal::PrintBufferViewToString(result, print_mode, 1024)); + LOG(INFO) << "result[" << i << "]: " << result.buffer->DebugString(); + std::cout << result_str << "\n"; + } + + return OkStatus(); +} + +// Evaluates a single function in its own fiber, printing the results to stdout. +Status EvaluateFunction(std::shared_ptr instance, + vm::SequencerContext* context, + hal::Allocator* allocator, const Function& function) { + // Setup our dummy fiber we will run with. + vm::FiberState fiber_state(instance); + + std::cout << "EXEC @" << function.name() << std::endl; + + // Marshal inputs. + ASSIGN_OR_RETURN(std::vector args, + ParseInputsFromFlags(allocator)); + std::vector results; + results.resize(function.result_count()); + + // Call into the main function. + RETURN_IF_ERROR(context->Invoke(&fiber_state, function, absl::MakeSpan(args), + absl::MakeSpan(results))); + + // Print outputs. + RETURN_IF_ERROR(OutputFunctionResults(function, absl::MakeSpan(results))); + + return OkStatus(); +} + +// Evaluates all exported functions within given module. +Status EvaluateFunctions(absl::string_view target_backend, + std::unique_ptr module) { + // Create the context we'll use for this (ensuring that we can't interfere + // with other running evaluations, such as when in a multithreaded test + // runner). + ASSIGN_OR_RETURN(auto debug_server, vm::debug::CreateDebugServerFromFlags()); + auto instance = std::make_shared(std::move(debug_server)); + ASSIGN_OR_RETURN(auto driver, hal::DriverRegistry::shared_registry()->Create( + target_backend)); + ASSIGN_OR_RETURN(auto device, driver->CreateDefaultDevice()); + RETURN_IF_ERROR(instance->device_manager()->RegisterDevice(device)); + vm::SequencerContext context(instance); + + if (absl::GetFlag(FLAGS_print_bytecode)) { + vm::PrintModuleFlagBitfield print_flags = vm::PrintModuleFlag::kNone; + RETURN_IF_ERROR(vm::PrintModuleToStream(vm::sequencer_opcode_table(), + *module, print_flags, &std::cout)); + } + + // Register module with the context. + RETURN_IF_ERROR(context.RegisterModule(std::move(module))); + + // Evaluate all exported functions. + for (auto& module : context.modules()) { + for (int function_ordinal : *module->def().function_table()->exports()) { + ASSIGN_OR_RETURN(auto function, module->function_table().LookupFunction( + function_ordinal)); + RETURN_IF_ERROR( + EvaluateFunction(instance, &context, device->allocator(), function)); + } + } + + RETURN_IF_ERROR(instance->device_manager()->UnregisterDevice(device.get())); + device.reset(); + driver.reset(); + + return OkStatus(); +} + +// Translates and runs a single LLVM file buffer. +Status EvaluateFile(std::unique_ptr file_buffer) { + std::vector target_backends; + if (absl::GetFlag(FLAGS_target_backends).empty()) { + target_backends = + hal::DriverRegistry::shared_registry()->EnumerateAvailableDrivers(); + } else { + // We need to map specific backends names to drivers (like 'vulkan-spirv' to + // the driver 'vulkan'). + target_backends = absl::StrSplit(absl::GetFlag(FLAGS_target_backends), ','); + } + + for (auto target_backend : target_backends) { + // Prepare the module for execution and evaluate it. + auto cloned_file_buffer = llvm::MemoryBuffer::getMemBufferCopy( + file_buffer->getBuffer(), file_buffer->getBufferIdentifier()); + ASSIGN_OR_RETURN(auto module, PrepareModule(target_backend + '*', + std::move(cloned_file_buffer))); + RETURN_IF_ERROR(EvaluateFunctions(BackendToDriverName(target_backend), + std::move(module))); + } + + return OkStatus(); +} + +// Runs the given .mlir file based on the current flags. +Status RunFile(std::string mlir_filename) { + // Load input file/from stdin. + std::string error_message; + auto file = mlir::openInputFile(mlir_filename, &error_message); + if (!file) { + return NotFoundErrorBuilder(ABSL_LOC) + << "Unable to open input file " << mlir_filename << ": " + << error_message; + } + + if (!absl::GetFlag(FLAGS_split_input_file)) { + // Use entire buffer as a single module. + return EvaluateFile(std::move(file)); + } + + // Split the buffer into separate modules and evaluate independently. + // This matches the -split-input-file arg to mlir-opt. + const char kSplitMarker[] = "// -----\n"; + auto* full_buffer = file.get(); + llvm::SmallVector source_buffers; + full_buffer->getBuffer().split(source_buffers, kSplitMarker); + + // Add the original buffer to the source manager. + llvm::SourceMgr fileSourceMgr; + fileSourceMgr.AddNewSourceBuffer(std::move(file), llvm::SMLoc()); + + // Process each chunk in turn. Only return the first error (but log all). + Status any_failure; + for (auto& sub_source_buffer : source_buffers) { + auto split_loc = llvm::SMLoc::getFromPointer(sub_source_buffer.data()); + unsigned split_line = fileSourceMgr.getLineAndColumn(split_loc).first; + auto sub_buffer = llvm::MemoryBuffer::getMemBufferCopy( + sub_source_buffer, full_buffer->getBufferIdentifier() + + llvm::Twine(" split at line #") + + llvm::Twine(split_line)); + auto sub_failure = EvaluateFile(std::move(sub_buffer)); + if (!sub_failure.ok()) { + LOG(ERROR) << sub_failure; + if (any_failure.ok()) { + any_failure = std::move(sub_failure); + } + } + } + + return any_failure; +} + +} // namespace + +extern "C" int main(int argc, char** argv) { + InitializeEnvironment(&argc, &argv); + if (argc < 2) { + LOG(ERROR) << "Must supply an input .mlir file."; + return 1; + } + QCHECK_OK(RunFile(argv[1])); + return 0; +} + +} // namespace iree diff --git a/tools/run_module_main.cc b/tools/run_module_main.cc new file mode 100644 index 000000000000..d4b8a1257175 --- /dev/null +++ b/tools/run_module_main.cc @@ -0,0 +1,203 @@ +// Copyright 2019 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include + +#include "third_party/absl/flags/flag.h" +#include "third_party/absl/strings/numbers.h" +#include "third_party/absl/strings/str_replace.h" +#include "third_party/absl/strings/str_split.h" +#include "third_party/absl/strings/string_view.h" +#include "third_party/absl/types/source_location.h" +#include "third_party/mlir_edge/iree/base/file_io.h" +#include "third_party/mlir_edge/iree/base/init.h" +#include "third_party/mlir_edge/iree/base/status.h" +#include "third_party/mlir_edge/iree/hal/buffer_view_string_util.h" +#include "third_party/mlir_edge/iree/hal/driver_registry.h" +#include "third_party/mlir_edge/iree/schemas/module_def_generated.h" +#include "third_party/mlir_edge/iree/vm/bytecode_printer.h" +#include "third_party/mlir_edge/iree/vm/bytecode_tables_sequencer.h" +#include "third_party/mlir_edge/iree/vm/debug/debug_server_flags.h" +#include "third_party/mlir_edge/iree/vm/fiber_state.h" +#include "third_party/mlir_edge/iree/vm/function.h" +#include "third_party/mlir_edge/iree/vm/instance.h" +#include "third_party/mlir_edge/iree/vm/module.h" +#include "third_party/mlir_edge/iree/vm/module_printer.h" +#include "third_party/mlir_edge/iree/vm/sequencer_context.h" + +ABSL_FLAG(std::string, main_module, "", "Main module with entry point."); +ABSL_FLAG(std::string, main_function, "", + "Function within the main module to execute."); + +ABSL_FLAG(bool, print_source_info, false, + "Prints source map information in bytecode output."); + +ABSL_FLAG(std::string, input_values, "", "Input shapes and optional values."); +ABSL_FLAG(std::string, input_file, "", + "Input shapes and optional values serialized in a file."); + +ABSL_FLAG(std::string, output_types, "", + "Output data types (comma delimited list of b/i/u/f for " + "binary/signed int/unsigned int/float)."); + +namespace iree { +namespace vm { +namespace { + +using ::iree::hal::BufferView; + +// Parses a list of input shapes and values from a string of newline-separated +// inputs. Expects the contents to have one value per line with each value +// listed as +// [shape]xtype=[value] +// Example: +// 4x4xi8=0,1,2,3 +StatusOr> ParseInputsFromFlags( + hal::Allocator* allocator) { + std::string file_contents; + if (!absl::GetFlag(FLAGS_input_values).empty()) { + file_contents = + absl::StrReplaceAll(absl::GetFlag(FLAGS_input_values), {{"\\n", "\n"}}); + } else if (!absl::GetFlag(FLAGS_input_file).empty()) { + ASSIGN_OR_RETURN(file_contents, + file_io::GetFileContents(absl::GetFlag(FLAGS_input_file))); + } + std::vector inputs; + for (const auto& line : + absl::StrSplit(file_contents, '\n', absl::SkipWhitespace())) { + ASSIGN_OR_RETURN(auto input, + hal::ParseBufferViewFromString(line, allocator)); + inputs.push_back(input); + } + return inputs; +} + +} // namespace + +Status Run() { + ASSIGN_OR_RETURN(auto debug_server, debug::CreateDebugServerFromFlags()); + auto instance = std::make_shared(std::move(debug_server)); + ASSIGN_OR_RETURN(auto driver, hal::DriverRegistry::shared_registry()->Create( + "interpreter")); + ASSIGN_OR_RETURN(auto device, driver->CreateDefaultDevice()); + RETURN_IF_ERROR(instance->device_manager()->RegisterDevice(device)); + SequencerContext context(instance); + + // Load main module. + ASSIGN_OR_RETURN( + auto main_module_file, + ModuleFile::LoadFile(ModuleDefIdentifier(), + absl::GetFlag(FLAGS_main_module)), + _ << "while loading module file " << absl::GetFlag(FLAGS_main_module)); + ASSIGN_OR_RETURN(auto main_module, + Module::FromFile(std::move(main_module_file))); + + // Add native functions for use by the module. + RETURN_IF_ERROR(context.RegisterNativeFunction( + "fabsf", + [](Stack* stack, absl::Span args, + absl::Span results) -> Status { + // TODO(benvanik): example native functions. + LOG(INFO) << "fabsf"; + return OkStatus(); + })); + + // Register the main module with the context. + // We could add additional modules (specializations, shared libraries, etc). + // ModuleFioles are stateless so we could have the same module_file used by + // multiple contexts simultaneously. + auto* main_module_ptr = main_module.get(); + RETURN_IF_ERROR(context.RegisterModule(std::move(main_module))); + + // Dump the registered modules. + PrintModuleFlagBitfield print_flags = PrintModuleFlag::kNone; + if (absl::GetFlag(FLAGS_print_source_info)) { + print_flags |= PrintModuleFlag::kIncludeSourceMapping; + } + for (const auto& module : context.modules()) { + RETURN_IF_ERROR(PrintModuleToStream(sequencer_opcode_table(), *module, + print_flags, &std::cout)); + } + + // Setup a new fiber. + FiberState fiber_state(instance); + + // Setup arguments and storage for results. + Function main_function; + if (!absl::GetFlag(FLAGS_main_function).empty()) { + // User-specified main function. + ASSIGN_OR_RETURN(main_function, + context.LookupExport(absl::GetFlag(FLAGS_main_function))); + } else { + // No main function specified; to prevent non-deterministic behavior we + // require one unless there's exactly one exported function in the module. + auto* exports = main_module_ptr->function_table().def().exports(); + if (exports && exports->size() == 1) { + ASSIGN_OR_RETURN( + main_function, + main_module_ptr->function_table().LookupFunction(exports->Get(0))); + } else { + return InvalidArgumentErrorBuilder(ABSL_LOC) + << "--main_function= must be specified to disambiguate the " + "function to run"; + } + } + ASSIGN_OR_RETURN(std::vector args, + ParseInputsFromFlags(device->allocator())); + std::vector results; + results.resize(main_function.result_count()); + + // Call into the main function. + RETURN_IF_ERROR(context.Invoke(&fiber_state, main_function, + absl::MakeSpan(args), + absl::MakeSpan(results))); + + // Dump all results to stdout. + std::vector output_types = + absl::StrSplit(absl::GetFlag(FLAGS_output_types), + absl::delimiter::AnyOf(", "), absl::SkipWhitespace()); + if (!output_types.empty() && output_types.size() != results.size()) { + return InvalidArgumentErrorBuilder(ABSL_LOC) + << "--output_types= specified but has " << output_types.size() + << " types when the function returns " << results.size(); + } + for (int i = 0; i < results.size(); ++i) { + const auto& result = results[i]; + auto print_mode = hal::BufferViewPrintMode::kFloatingPoint; + if (!output_types.empty()) { + ASSIGN_OR_RETURN(print_mode, + hal::ParseBufferViewPrintMode(output_types[i])); + } + ASSIGN_OR_RETURN(auto result_str, + PrintBufferViewToString(result, print_mode, 1024)); + const auto& buffer = result.buffer; + if (!buffer) { + return InternalErrorBuilder(ABSL_LOC) + << "result[" << i << "] unexpectedly has no buffer"; + } + LOG(INFO) << "result[" << i << "]: " << buffer->DebugString(); + std::cout << result_str << "\n"; + } + + return OkStatus(); +} + +extern "C" int main(int argc, char** argv) { + InitializeEnvironment(&argc, &argv); + CHECK_OK(Run()); + return 0; +} + +} // namespace vm +} // namespace iree diff --git a/tools/sanitizer_suppressions.txt b/tools/sanitizer_suppressions.txt new file mode 100644 index 000000000000..e8c20b9300d3 --- /dev/null +++ b/tools/sanitizer_suppressions.txt @@ -0,0 +1 @@ +leak:libGLX_nvidia.so diff --git a/tools/web/run_module.html b/tools/web/run_module.html new file mode 100644 index 000000000000..f3f749710aef --- /dev/null +++ b/tools/web/run_module.html @@ -0,0 +1,319 @@ + + + + + + + IREE - MNIST Demo + + + + + + + +
+
+

IREE MNIST demo, CPU interpreter using WebAssembly

+

WebAssembly threads must be enabled. See chrome://flags/#enable-webassembly-threads

+
+ +
+
+
+ +
+

Input arguments:

+ +
+
+ +

Input image for MNIST (handwritten digit) classification:

+
    +
  • Left click drag to draw
    • +
  • Right click drag to erase
    • +
+
+ + +
+
+ + +
+
+

+ +
+ +
+
+
+

MNIST Prediction:  +

+

+

MNIST Prediction confidences:  +

+









+

+

+
+ + + + + + + + diff --git a/tools/web/run_module_emscripten.cc b/tools/web/run_module_emscripten.cc new file mode 100644 index 000000000000..46f5de1e12ef --- /dev/null +++ b/tools/web/run_module_emscripten.cc @@ -0,0 +1,147 @@ +// Copyright 2019 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include +#include + +#include + +#include "third_party/absl/strings/str_replace.h" +#include "third_party/absl/strings/str_split.h" +#include "third_party/absl/strings/string_view.h" +#include "third_party/mlir_edge/iree/base/flatbuffer_util.h" +#include "third_party/mlir_edge/iree/base/init.h" +#include "third_party/mlir_edge/iree/base/status.h" +#include "third_party/mlir_edge/iree/hal/buffer_view.h" +#include "third_party/mlir_edge/iree/hal/buffer_view_string_util.h" +#include "third_party/mlir_edge/iree/hal/driver_registry.h" +#include "third_party/mlir_edge/iree/schemas/module_def_generated.h" +#include "third_party/mlir_edge/iree/vm/bytecode_tables_sequencer.h" +#include "third_party/mlir_edge/iree/vm/fiber_state.h" +#include "third_party/mlir_edge/iree/vm/function.h" +#include "third_party/mlir_edge/iree/vm/instance.h" +#include "third_party/mlir_edge/iree/vm/module.h" +#include "third_party/mlir_edge/iree/vm/module_printer.h" +#include "third_party/mlir_edge/iree/vm/sequencer_context.h" + +namespace iree { + +// Parses a list of input shapes and values from a string of newline-separated +// inputs. Expects the contents to have one value per line with each value +// listed as +// [shape]xtype=[value] +// Example: +// 4x4xi8=0,1,2,3 +StatusOr> ParseInputs( + absl::string_view inputs_string, hal::Allocator* allocator) { + std::string input_lines = absl::StrReplaceAll(inputs_string, {{"\\n", "\n"}}); + std::vector input_buffer_views; + for (const auto& input_line : + absl::StrSplit(input_lines, '\n', absl::SkipWhitespace())) { + ASSIGN_OR_RETURN(auto input_buffer_view, + hal::ParseBufferViewFromString(input_line, allocator)); + input_buffer_views.push_back(input_buffer_view); + } + return input_buffer_views; +} + +// Runs an IREE module with the provided inputs and returns its outputs. +StatusOr RunIreeModule(std::string module_file_data, + absl::string_view inputs_string) { + auto instance = std::make_shared(/* debug_server= */ nullptr); + + // Create driver and device. + ASSIGN_OR_RETURN(auto driver, hal::DriverRegistry::shared_registry()->Create( + "interpreter")); + ASSIGN_OR_RETURN(auto device, driver->CreateDefaultDevice()); + RETURN_IF_ERROR(instance->device_manager()->RegisterDevice(device)); + vm::SequencerContext context(instance); + + // Load main module FlatBuffer. + ASSIGN_OR_RETURN(auto main_module_file, + FlatBufferFile::FromString(ModuleDefIdentifier(), + module_file_data)); + ASSIGN_OR_RETURN(auto main_module, + vm::Module::FromFile(std::move(main_module_file))); + + // Register the main module with the context. + RETURN_IF_ERROR(context.RegisterModule(std::move(main_module))); + + // Dump the registered modules. + vm::PrintModuleFlagBitfield print_flags = + vm::PrintModuleFlag::kIncludeSourceMapping; + for (const auto& module : context.modules()) { + RETURN_IF_ERROR(vm::PrintModuleToStream(vm::sequencer_opcode_table(), + *module, print_flags, &std::cout)); + } + + // Setup a new fiber. + vm::FiberState fiber_state(instance); + + // Setup arguments and storage for results. + // TODO(scotttodd): Receive main function name from JS + ASSIGN_OR_RETURN(vm::Function main_function, context.LookupExport("main")); + + ASSIGN_OR_RETURN(std::vector args, + ParseInputs(inputs_string, device->allocator())); + std::vector results; + results.resize(main_function.result_count()); + + // Call into the main function. + RETURN_IF_ERROR(context.Invoke(&fiber_state, main_function, + absl::MakeSpan(args), + absl::MakeSpan(results))); + + // Dump all results to stdout. + // TODO(scotttodd): Receive output types / print mode from JS + // TODO(scotttodd): Return list of outputs instead of just the first (proto?) + for (int i = 0; i < results.size(); ++i) { + const auto& result = results[i]; + auto print_mode = hal::BufferViewPrintMode::kFloatingPoint; + ASSIGN_OR_RETURN(auto result_str, + PrintBufferViewToString(result, print_mode, 1024)); + const auto& buffer = result.buffer; + if (!buffer) { + return InternalErrorBuilder(ABSL_LOC) + << "result[" << i << "] unexpectedly has no buffer"; + } + + return result_str; + } + + return InternalErrorBuilder(ABSL_LOC) << "Received no results"; +} + +std::string RunIreeModuleEntry(std::string module_file_data, + std::string inputs_string) { + // TODO(scotttodd): optimize, minimize copies + // https://groups.google.com/d/msg/emscripten-discuss/CMfYljLWMvY/Di52WB2QAgAJ + auto result_or = RunIreeModule(std::move(module_file_data), inputs_string); + if (!result_or.ok()) { + return "Error: " + result_or.status().ToString(); + } else { + return result_or.ValueOrDie(); + } +} + +EMSCRIPTEN_BINDINGS(iree) { + emscripten::function("runIreeModule", &RunIreeModuleEntry); +} + +extern "C" int main(int argc, char** argv) { + InitializeEnvironment(&argc, &argv); + return 0; +} + +} // namespace iree diff --git a/vm/bytecode_printer.cc b/vm/bytecode_printer.cc new file mode 100644 index 000000000000..72a8b0d66861 --- /dev/null +++ b/vm/bytecode_printer.cc @@ -0,0 +1,506 @@ +// Copyright 2019 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "third_party/mlir_edge/iree/vm/bytecode_printer.h" + +#include +#include + +#include "third_party/absl/base/macros.h" +#include "third_party/absl/container/inlined_vector.h" +#include "third_party/absl/strings/str_join.h" +#include "third_party/absl/strings/string_view.h" +#include "third_party/absl/types/span.h" +#include "third_party/mlir_edge/iree/base/status.h" +#include "third_party/mlir_edge/iree/schemas/bytecode/bytecode_v0.h" +#include "third_party/mlir_edge/iree/schemas/source_map_def_generated.h" +#include "third_party/mlir_edge/iree/vm/bytecode_util.h" +#include "third_party/mlir_edge/iree/vm/module.h" +#include "third_party/mlir_edge/iree/vm/type.h" + +namespace iree { +namespace vm { + +namespace { + +template +StatusOr ReadValue(absl::Span data, int* offset) { + if (*offset + sizeof(T) > data.size()) { + return OutOfRangeErrorBuilder(ABSL_LOC) << "Bytecode data underrun"; + } + auto value = *reinterpret_cast(&data[*offset]); + *offset = *offset + sizeof(T); + return value; +} + +StatusOr ReadType(absl::Span data, int* offset) { + ASSIGN_OR_RETURN(uint8_t type_index, ReadValue(data, offset)); + return Type::FromTypeIndex(type_index); +} + +StatusOr ReadCount(absl::Span data, int* offset) { + return ReadValue(data, offset); +} + +StatusOr ReadValueSlot(absl::Span data, int* offset) { + return ReadValue(data, offset); +} + +absl::string_view ConstantEncodingToString(ConstantEncoding encoding) { + switch (encoding) { +#define GET_NAME(ordinal, enum_name, str, ...) \ + case ConstantEncoding::enum_name: \ + return str; + IREE_CONSTANT_ENCODING_LIST(GET_NAME) +#undef GET_NAME + default: + return "unknown"; + } +} + +template +std::string TypedDataToString(absl::Span bytes) { + auto typed_data = absl::Span{ + reinterpret_cast(bytes.data()), bytes.size() / sizeof(T)}; + return absl::StrJoin(typed_data, ","); +} + +std::string ConstantToString(const Type& type, + absl::Span bytes) { + if (!type.is_builtin()) { + return absl::StrJoin(bytes, ","); + } + switch (type.builtin_type()) { + case BuiltinType::kI8: + return TypedDataToString(bytes); + case BuiltinType::kI16: + return TypedDataToString(bytes); + case BuiltinType::kI32: + return TypedDataToString(bytes); + case BuiltinType::kI64: + return TypedDataToString(bytes); + case BuiltinType::kF16: + return TypedDataToString(bytes); + case BuiltinType::kF32: + return TypedDataToString(bytes); + case BuiltinType::kF64: + return TypedDataToString(bytes); + default: + return ""; + } +} + +} // namespace + +// static +std::string BytecodePrinter::ToString( + OpcodeTable opcode_table, const FunctionTable& function_table, + const ExecutableTable& executable_table, + const SourceMapResolver& source_map_resolver, + const BytecodeDef& bytecode_def) { + BytecodePrinter printer(opcode_table, function_table, executable_table, + source_map_resolver); + auto result = printer.Print(bytecode_def); + if (!result.ok()) { + return result.status().ToString(); + } + return result.ValueOrDie(); +} + +StatusOr BytecodePrinter::Print( + const BytecodeDef& bytecode_def) const { + std::ostringstream stream; + RETURN_IF_ERROR(PrintToStream(bytecode_def, &stream)) << stream.str(); + return stream.str(); +} + +Status BytecodePrinter::PrintToStream(const BytecodeDef& bytecode_def, + std::ostream* stream) const { + if (!bytecode_def.contents()) { + return OkStatus(); + } + auto data = absl::MakeSpan( + reinterpret_cast(bytecode_def.contents()->data()), + bytecode_def.contents()->size()); + return PrintToStream(data, stream); +} + +Status BytecodePrinter::PrintToStream(absl::Span data, + std::ostream* stream) const { + // TODO(benvanik): scan and find all branch offsets to insert labels + + int offset = 0; + absl::optional previous_location; + while (offset < data.length()) { + auto source_location = source_map_resolver_.ResolveBytecodeOffset(offset); + if (source_location.has_value()) { + if (previous_location != source_location) { + *stream << std::setw(10) << "; " << source_location.value() << "\n"; + } + previous_location = source_location; + } + + *stream << std::setw(6) << offset << ": "; + + uint8_t opcode = data[offset++]; + const auto& opcode_info = GetOpcodeInfo(opcode_table_, opcode); + if (!opcode_info.mnemonic) { + return UnimplementedErrorBuilder(ABSL_LOC) + << "Unhandled opcode " << opcode << " at offset " << (offset - 1); + } + int payload_offset = offset; + + // Print out return values, if any. + int base_result_index = 0; + int printed_result_count = 0; + for (int i = base_result_index; i < ABSL_ARRAYSIZE(opcode_info.operands); + ++i) { + if (opcode_info.operands[i] == OperandEncoding::kNone) break; + if (printed_result_count > 0) { + *stream << ", "; + } + switch (opcode_info.operands[i]) { + default: + case OperandEncoding::kNone: + return UnimplementedErrorBuilder(ABSL_LOC) + << "Unhandled op encoding " + << static_cast(opcode_info.operands[i]) << " at offset " + << (offset - 1); + case OperandEncoding::kInputSlot: + case OperandEncoding::kOutputSlot: { + // Printing handled below. + offset += sizeof(uint16_t); + break; + } + case OperandEncoding::kVariadicInputSlots: + case OperandEncoding::kVariadicOutputSlots: { + // Printing handled below. + ASSIGN_OR_RETURN(int count, ReadCount(data, &offset)); + offset += count * sizeof(uint16_t); + break; + } + case OperandEncoding::kResultSlot: { + ++printed_result_count; + ASSIGN_OR_RETURN(uint16_t slot_ordinal, ReadValueSlot(data, &offset)); + *stream << "%" << slot_ordinal; + break; + } + case OperandEncoding::kVariadicResultSlots: { + ++printed_result_count; + *stream << "["; + ASSIGN_OR_RETURN(int count, ReadCount(data, &offset)); + for (int j = 0; j < count; ++j) { + ASSIGN_OR_RETURN(uint16_t slot_ordinal, + ReadValueSlot(data, &offset)); + if (j > 0) *stream << ", "; + *stream << "%" << slot_ordinal; + } + *stream << "]"; + break; + } + case OperandEncoding::kVariadicTransferSlots: { + // Printing handled below. + ASSIGN_OR_RETURN(int count, ReadCount(data, &offset)); + offset += count * 2 * sizeof(uint16_t); + break; + } + case OperandEncoding::kConstant: { + // Printing handled below. + ASSIGN_OR_RETURN(auto type, ReadType(data, &offset)); + ASSIGN_OR_RETURN(int rank, ReadCount(data, &offset)); + int element_count = 1; + for (int j = 0; j < rank; ++j) { + ASSIGN_OR_RETURN(int dim, ReadValue(data, &offset)); + element_count *= dim; + } + offset += sizeof(ConstantEncoding); + offset += element_count * type.element_size(); + break; + } + case OperandEncoding::kFunctionOrdinal: { + // Printing handled below. + offset += sizeof(uint32_t); + break; + } + case OperandEncoding::kDispatchOrdinal: { + // Printing handled below. + offset += sizeof(uint32_t) + sizeof(uint16_t); + break; + } + case OperandEncoding::kBlockOffset: { + // Printing handled below. + offset += sizeof(uint32_t); + break; + } + case OperandEncoding::kTypeIndex: { + // Printing handled below. + offset += sizeof(uint8_t); + break; + } + case OperandEncoding::kIndex: { + // Printing handled below. + offset += sizeof(int32_t); + break; + } + case OperandEncoding::kIndexList: { + // Printing handled below. + ASSIGN_OR_RETURN(int count, ReadCount(data, &offset)); + offset += count * sizeof(int32_t); + break; + } + case OperandEncoding::kCmpIPredicate: + case OperandEncoding::kCmpFPredicate: { + // Printing handled below. + offset += sizeof(uint8_t); + break; + } + } + } + if (printed_result_count > 0) { + *stream << " = "; + } + offset = payload_offset; + + *stream << opcode_info.mnemonic; + + // Print out operands. + int base_operand_index = 0; + int printed_operand_count = 0; + for (int i = base_operand_index; i < ABSL_ARRAYSIZE(opcode_info.operands); + ++i) { + if (opcode_info.operands[i] == OperandEncoding::kNone) break; + if (opcode_info.operands[i] != OperandEncoding::kResultSlot && + opcode_info.operands[i] != OperandEncoding::kVariadicResultSlots) { + if (i == base_operand_index) { + *stream << " "; + } else if (printed_operand_count > 0) { + *stream << ", "; + } + } + switch (opcode_info.operands[i]) { + default: + case OperandEncoding::kNone: + return UnimplementedErrorBuilder(ABSL_LOC) + << "Unhandled op encoding " + << static_cast(opcode_info.operands[i]) << " at offset " + << (offset - 1); + case OperandEncoding::kInputSlot: { + ++printed_operand_count; + ASSIGN_OR_RETURN(uint16_t slot_ordinal, ReadValueSlot(data, &offset)); + *stream << "%" << slot_ordinal; + break; + } + case OperandEncoding::kVariadicInputSlots: { + ++printed_operand_count; + *stream << "["; + ASSIGN_OR_RETURN(int count, ReadCount(data, &offset)); + for (int j = 0; j < count; ++j) { + ASSIGN_OR_RETURN(uint16_t slot_ordinal, + ReadValueSlot(data, &offset)); + if (j > 0) *stream << ", "; + *stream << "%" << slot_ordinal; + } + *stream << "]"; + break; + } + case OperandEncoding::kOutputSlot: { + ++printed_operand_count; + ASSIGN_OR_RETURN(uint16_t slot_ordinal, ReadValueSlot(data, &offset)); + *stream << "&" + << "%" << slot_ordinal; + break; + } + case OperandEncoding::kVariadicOutputSlots: { + ++printed_operand_count; + *stream << "["; + ASSIGN_OR_RETURN(int count, ReadCount(data, &offset)); + for (int j = 0; j < count; ++j) { + ASSIGN_OR_RETURN(uint16_t slot_ordinal, + ReadValueSlot(data, &offset)); + if (j > 0) *stream << ", "; + *stream << "&" + << "%" << slot_ordinal; + } + *stream << "]"; + break; + } + case OperandEncoding::kResultSlot: { + // Printing handled above. + offset += sizeof(uint16_t); + break; + } + case OperandEncoding::kVariadicResultSlots: { + // Printing handled above. + ASSIGN_OR_RETURN(int count, ReadCount(data, &offset)); + offset += count * sizeof(uint16_t); + break; + } + case OperandEncoding::kVariadicTransferSlots: { + ++printed_operand_count; + *stream << "["; + ASSIGN_OR_RETURN(int count, ReadCount(data, &offset)); + for (int j = 0; j < count; ++j) { + ASSIGN_OR_RETURN(uint16_t src_slot_ordinal, + ReadValueSlot(data, &offset)); + ASSIGN_OR_RETURN(uint16_t dst_slot_ordinal, + ReadValueSlot(data, &offset)); + if (j > 0) *stream << ", "; + *stream << "%" << src_slot_ordinal << "=>%" << dst_slot_ordinal; + } + *stream << "]"; + break; + } + case OperandEncoding::kConstant: { + ++printed_operand_count; + ASSIGN_OR_RETURN(auto type, ReadType(data, &offset)); + ASSIGN_OR_RETURN(int rank, ReadCount(data, &offset)); + absl::InlinedVector shape(rank); + int element_count = 1; + for (int j = 0; j < rank; ++j) { + ASSIGN_OR_RETURN(int dim, ReadValue(data, &offset)); + shape[j] = dim; + element_count *= dim; + } + ASSIGN_OR_RETURN(auto encoding, + ReadValue(data, &offset)); + *stream << ConstantEncodingToString(encoding); + int serialized_element_count = 1; + switch (encoding) { + case ConstantEncoding::kDense: + serialized_element_count = element_count; + break; + case ConstantEncoding::kSplat: + serialized_element_count = 1; + break; + default: + return UnimplementedErrorBuilder(ABSL_LOC) + << "Unimplemented constant encoding " + << static_cast(encoding); + } + *stream << " buffer_view<"; + if (!shape.empty()) { + *stream << absl::StrJoin(shape, "x") << "x"; + } + *stream << type << ">{"; + size_t element_size = type.element_size(); + auto bytes = data.subspan( + offset, std::min(serialized_element_count, 1024) * element_size); + *stream << ConstantToString(type, bytes); + if (serialized_element_count > 1024) *stream << "..."; + offset += serialized_element_count * element_size; + *stream << "}"; + break; + } + case OperandEncoding::kFunctionOrdinal: { + ++printed_operand_count; + ASSIGN_OR_RETURN(auto function_ordinal, + ReadValue(data, &offset)); + ASSIGN_OR_RETURN(auto function, + function_table_.LookupFunction(function_ordinal)); + *stream << "@" << function_ordinal << " " << function.name(); + break; + } + case OperandEncoding::kDispatchOrdinal: { + ++printed_operand_count; + ASSIGN_OR_RETURN(auto dispatch_ordinal, + ReadValue(data, &offset)); + ASSIGN_OR_RETURN(auto export_ordinal, + ReadValue(data, &offset)); + // TODO(benvanik): lookup in executable table. + *stream << "@" << dispatch_ordinal << ":" << export_ordinal; + break; + } + case OperandEncoding::kImportOrdinal: { + ++printed_operand_count; + ASSIGN_OR_RETURN(auto import_ordinal, + ReadValue(data, &offset)); + ASSIGN_OR_RETURN(auto* function, + function_table_.LookupImport(import_ordinal)); + *stream << "@i" << import_ordinal << " "; + switch (function->link_type()) { + default: + *stream << "??"; + break; + case ImportFunction::LinkType::kNativeFunction: + *stream << ""; + break; + case ImportFunction::LinkType::kModule: + *stream << function->linked_function().module().name() << ":" + << function->linked_function().name(); + break; + } + break; + } + case OperandEncoding::kBlockOffset: { + ++printed_operand_count; + ASSIGN_OR_RETURN(uint32_t block_offset, + ReadValue(data, &offset)); + *stream << ":" << block_offset; + break; + } + case OperandEncoding::kTypeIndex: { + ++printed_operand_count; + ASSIGN_OR_RETURN(auto type, ReadType(data, &offset)); + *stream << type; + break; + } + case OperandEncoding::kIndex: { + ++printed_operand_count; + ASSIGN_OR_RETURN(auto index, ReadValue(data, &offset)); + *stream << "#" << index; + break; + } + case OperandEncoding::kIndexList: { + ++printed_operand_count; + *stream << "{"; + ASSIGN_OR_RETURN(int count, ReadCount(data, &offset)); + for (int j = 0; j < count; ++j) { + ASSIGN_OR_RETURN(auto dim, ReadValue(data, &offset)); + if (j > 0) *stream << ","; + *stream << dim; + } + *stream << "}"; + break; + } + case OperandEncoding::kCmpIPredicate: { + ++printed_operand_count; + ASSIGN_OR_RETURN(auto predicate_value, + ReadValue(data, &offset)); + *stream << "<" + << PredicateToString( + static_cast(predicate_value)) + << ">"; + break; + } + case OperandEncoding::kCmpFPredicate: { + ++printed_operand_count; + ASSIGN_OR_RETURN(auto predicate_value, + ReadValue(data, &offset)); + *stream << "<" + << PredicateToString( + static_cast(predicate_value)) + << ">"; + break; + } + } + } + + *stream << "\n"; + } + + return OkStatus(); +} + +} // namespace vm +} // namespace iree diff --git a/vm/bytecode_printer.h b/vm/bytecode_printer.h new file mode 100644 index 000000000000..7f572175700c --- /dev/null +++ b/vm/bytecode_printer.h @@ -0,0 +1,68 @@ +// Copyright 2019 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#ifndef THIRD_PARTY_MLIR_EDGE_IREE_VM_BYTECODE_PRINTER_H_ +#define THIRD_PARTY_MLIR_EDGE_IREE_VM_BYTECODE_PRINTER_H_ + +#include + +#include "third_party/mlir_edge/iree/base/status.h" +#include "third_party/mlir_edge/iree/schemas/bytecode_def_generated.h" +#include "third_party/mlir_edge/iree/schemas/source_map_def_generated.h" +#include "third_party/mlir_edge/iree/vm/executable_table.h" +#include "third_party/mlir_edge/iree/vm/function_table.h" +#include "third_party/mlir_edge/iree/vm/opcode_info.h" +#include "third_party/mlir_edge/iree/vm/source_map.h" + +namespace iree { +namespace vm { + +// Prints bytecode in a text format to enable human-inspection. +// Optionally can interleave original source location information if a SourceMap +// is available. +class BytecodePrinter { + public: + static std::string ToString(OpcodeTable opcode_table, + const FunctionTable& function_table, + const ExecutableTable& executable_table, + const SourceMapResolver& source_map_resolver, + const BytecodeDef& bytecode_def); + + explicit BytecodePrinter(OpcodeTable opcode_table, + const FunctionTable& function_table, + const ExecutableTable& executable_table, + const SourceMapResolver& source_map_resolver) + : opcode_table_(opcode_table), + function_table_(function_table), + executable_table_(executable_table), + source_map_resolver_(source_map_resolver) {} + + StatusOr Print(const BytecodeDef& bytecode_def) const; + + Status PrintToStream(const BytecodeDef& bytecode_def, + std::ostream* stream) const; + Status PrintToStream(absl::Span data, + std::ostream* stream) const; + + private: + OpcodeTable opcode_table_; + const FunctionTable& function_table_; + const ExecutableTable& executable_table_; + const SourceMapResolver& source_map_resolver_; +}; + +} // namespace vm +} // namespace iree + +#endif // THIRD_PARTY_MLIR_EDGE_IREE_VM_BYTECODE_PRINTER_H_ diff --git a/vm/bytecode_reader.cc b/vm/bytecode_reader.cc new file mode 100644 index 000000000000..97b7b1a94cea --- /dev/null +++ b/vm/bytecode_reader.cc @@ -0,0 +1,296 @@ +// Copyright 2019 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "third_party/mlir_edge/iree/vm/bytecode_reader.h" + +#include "third_party/mlir_edge/iree/base/shape.h" +#include "third_party/mlir_edge/iree/base/status.h" +#include "third_party/mlir_edge/iree/hal/heap_buffer.h" + +namespace iree { +namespace vm { + +namespace { +using ::iree::hal::BufferView; +} // namespace + +StatusOr BytecodeReader::AdvanceOffset() { + *stack_frame_->mutable_offset() = offset(); + // TODO(benvanik): make a flag and/or remove. + DVLOG(1) << "dispatch(" << stack_frame_->function().name() << "@" << offset() + << "): " << int(*bytecode_pc_); + for (int i = 0; i < locals_.size(); ++i) { + DVLOG(1) << "local[" << i << "] " << locals_[i].DebugStringShort(); + } + + if (breakpoint_table_) { + auto it = breakpoint_table_->find(offset()); + if (it != breakpoint_table_->end()) { + // Breakpoint hit! + RETURN_IF_ERROR(it->second(*stack_)); + } + } + + return bytecode_pc_++; +} + +Status BytecodeReader::SkipLocals(int count) { + size_t stride = sizeof(uint16_t) * count; + if (bytecode_pc_ + stride >= bytecode_limit_) { + return OutOfRangeErrorBuilder(ABSL_LOC) << "Bytecode underflow"; + } + bytecode_pc_ += stride; + return OkStatus(); +} + +Status BytecodeReader::ReadShape(Shape* out_shape) { + ASSIGN_OR_RETURN(auto shape_dims, ReadIndexList()); + *out_shape = Shape(shape_dims); + return OkStatus(); +} + +StatusOr BytecodeReader::ReadShapePieces() { + // TODO(benvanik): rewrite to be faster (multiple offsets to walk both lists). + ASSIGN_OR_RETURN(auto shape_dims, ReadIndexList()); + if (shape_dims.size() >= kMaxRank) { + return UnimplementedErrorBuilder(ABSL_LOC) + << "Shapes limited to rank " << kMaxRank << " right now"; + } + int expected_dynamic_dims = 0; + for (int i = 0; i < shape_dims.size(); ++i) { + if (shape_dims[i] == -1) { + ++expected_dynamic_dims; + } + } + + Shape shape(shape_dims); + ASSIGN_OR_RETURN(int dynamic_dims, ReadCount()); + if (dynamic_dims != expected_dynamic_dims) { + return InvalidArgumentErrorBuilder(ABSL_LOC) + << "Expected " << expected_dynamic_dims << " dynamic dims but only " + << dynamic_dims << " provided"; + } else if (dynamic_dims) { + for (int i = 0; i < shape_dims.size(); ++i) { + if (shape_dims[i] != -1) { + continue; + } + // TODO(benvanik): kill this embarrassment. + ASSIGN_OR_RETURN(auto dims_piece, ReadSlotElements()); + if (dims_piece.size() != 1) { + return InvalidArgumentErrorBuilder(ABSL_LOC) + << "Dims piece has rank " << dims_piece.size() << "; must be 1"; + } + shape[i] = dims_piece[0]; + } + } + return shape; +} + +StatusOr BytecodeReader::ReadShapePieces(size_t* out_element_count) { + ASSIGN_OR_RETURN(auto shape, ReadShapePieces()); + *out_element_count = shape.element_count(); + return shape; +} + +StatusOr> BytecodeReader::ReadIndexList() { + ASSIGN_OR_RETURN(int count, ReadCount()); + int stride = count * sizeof(int32_t); + if (bytecode_pc_ + stride >= bytecode_limit_) { + return OutOfRangeErrorBuilder(ABSL_LOC) << "Bytecode underflow"; + } + auto list = absl::Span( + reinterpret_cast(bytecode_pc_), count); + bytecode_pc_ += stride; + return list; +} + +Status BytecodeReader::SwitchStackFrame(StackFrame* new_stack_frame) { + // Flush old state. + auto* old_stack_frame = stack_frame_; + if (old_stack_frame) { + *old_stack_frame->mutable_offset() = offset(); + } + + // Switch the frame. The FiberState holds the full stack, this is just the + // current one for easy access. + stack_frame_ = new_stack_frame; + + // Setup state pointers for faster dereferencing. + const auto& function = new_stack_frame->function(); + const auto& bytecode = *function.def().bytecode(); + bytecode_base_ = bytecode.contents()->Data(); + bytecode_limit_ = bytecode_base_ + bytecode.contents()->size(); + bytecode_pc_ = bytecode_base_ + new_stack_frame->offset(); + locals_ = new_stack_frame->mutable_locals(); + // TODO(benvanik): reimplement breakpoints as bytecode rewriting. + int function_ordinal = function.module() + .function_table() + .LookupFunctionOrdinal(function) + .ValueOrDie(); + breakpoint_table_ = + function.module().function_table().GetFunctionBreakpointTable( + function_ordinal); + return OkStatus(); +} + +Status BytecodeReader::CopyInputsAndSwitchStackFrame( + StackFrame* src_stack_frame, StackFrame* dst_stack_frame) { + ASSIGN_OR_RETURN(int32_t src_count, ReadCount()); + for (int i = 0; i < src_count; ++i) { + ASSIGN_OR_RETURN(auto* src_local, + ReadLocal(src_stack_frame->mutable_locals())); + *dst_stack_frame->mutable_local(i) = *src_local; + } + return SwitchStackFrame(dst_stack_frame); +} + +Status BytecodeReader::CopyResultsAndSwitchStackFrame( + StackFrame* src_stack_frame, StackFrame* dst_stack_frame) { + ASSIGN_OR_RETURN(int32_t src_count, ReadCount()); + // TODO(benvanik): avoid vector. + absl::InlinedVector src_locals(src_count); + for (int i = 0; i < src_count; ++i) { + ASSIGN_OR_RETURN(src_locals[i], + ReadLocal(src_stack_frame->mutable_locals())); + } + RETURN_IF_ERROR(SwitchStackFrame(dst_stack_frame)); + ASSIGN_OR_RETURN(int32_t dst_count, ReadCount()); + if (src_count != dst_count) { + return OutOfRangeErrorBuilder(ABSL_LOC) + << "Src and dst value counts differ: " << src_count << " vs " + << dst_count; + } + for (int i = 0; i < dst_count; ++i) { + ASSIGN_OR_RETURN(auto* dst_local, + ReadLocal(dst_stack_frame->mutable_locals())); + *dst_local = *src_locals[i]; + } + return OkStatus(); +} + +Status BytecodeReader::CopySlots() { + ASSIGN_OR_RETURN(int32_t count, ReadCount()); + for (int i = 0; i < count; ++i) { + ASSIGN_OR_RETURN(auto* src_local, + ReadLocal(stack_frame_->mutable_locals())); + ASSIGN_OR_RETURN(auto* dst_local, + ReadLocal(stack_frame_->mutable_locals())); + *dst_local = *src_local; + } + return OkStatus(); +} + +Status BytecodeReader::BranchToOffset(int32_t offset) { + const uint8_t* new_bytecode_pc = bytecode_base_ + offset; + if (new_bytecode_pc < bytecode_base_ || new_bytecode_pc > bytecode_limit_) { + return OutOfRangeErrorBuilder(ABSL_LOC) + << "Branch target " << offset + << " is out of bounds of the function bytecode (" + << static_cast(bytecode_limit_ - bytecode_base_) + << "b total)"; + } + bytecode_pc_ = new_bytecode_pc; + return OkStatus(); +} + +StatusOr BytecodeReader::ReadConstant() { + BufferView buffer_view; + + // Element type defines the buffer_view size (but we don't really care about + // the data format). + ASSIGN_OR_RETURN(auto element_type, ReadType()); + buffer_view.element_size = element_type.element_size(); + + // Parse shape - constants always define a full shape. + RETURN_IF_ERROR(ReadShape(&buffer_view.shape)); + + // Read encoding to determine how the constant data is stored in the file. + ASSIGN_OR_RETURN(auto encoding, ReadValue()); + + // Get buffer for the constant data. + switch (encoding) { + case ConstantEncoding::kDense: { + // Validate we have all constant data present. + device_size_t serialized_length = buffer_view.byte_length(); + if (bytecode_pc_ + serialized_length >= bytecode_limit_) { + return OutOfRangeErrorBuilder(ABSL_LOC) + << "Constant data out of bounds"; + } + + buffer_view.buffer = hal::HeapBuffer::Wrap( + hal::MemoryType::kHostLocal, hal::BufferUsage::kAll, bytecode_pc_, + serialized_length); + bytecode_pc_ += serialized_length; + break; + } + case ConstantEncoding::kSplat: { + // Validate we have at least one element worth of data in the buffer. + if (bytecode_pc_ + buffer_view.element_size >= bytecode_limit_) { + return OutOfRangeErrorBuilder(ABSL_LOC) + << "Constant data out of bounds"; + } + + // TODO(benvanik): replace with fancy constant pool and such. + // NOTE: this is not much different than if a alloc_heap+broadcast pair + // had been in the IR. + buffer_view.buffer = hal::HeapBuffer::Allocate( + hal::MemoryType::kHostLocal, hal::BufferUsage::kAll, + buffer_view.byte_length()); + switch (buffer_view.element_size) { + case 1: { + uint8_t value = *reinterpret_cast(bytecode_pc_); + RETURN_IF_ERROR(buffer_view.buffer->Fill8(value)); + break; + } + case 2: { + uint16_t value = *reinterpret_cast(bytecode_pc_); + RETURN_IF_ERROR(buffer_view.buffer->Fill16(value)); + break; + } + case 4: { + uint32_t value = *reinterpret_cast(bytecode_pc_); + RETURN_IF_ERROR(buffer_view.buffer->Fill32(value)); + break; + } + case 8: { + // TODO(benvanik): add Fill64. + uint64_t value = *reinterpret_cast(bytecode_pc_); + ASSIGN_OR_RETURN(auto mapping, + buffer_view.buffer->MapMemory( + hal::MemoryAccess::kDiscardWrite)); + auto mapped_data = mapping.mutable_contents(); + for (int i = 0; i < mapping.size(); ++i) { + mapped_data[i] = value; + } + break; + } + default: + return UnimplementedErrorBuilder(ABSL_LOC) + << "Unimplemented splat element stride " + << buffer_view.element_size; + } + bytecode_pc_ += buffer_view.element_size; + break; + } + default: + return UnimplementedErrorBuilder(ABSL_LOC) + << "Unimplemented constant encoding " + << static_cast(encoding); + } + + return buffer_view; +} + +} // namespace vm +} // namespace iree diff --git a/vm/bytecode_reader.h b/vm/bytecode_reader.h new file mode 100644 index 000000000000..6ac84da305b8 --- /dev/null +++ b/vm/bytecode_reader.h @@ -0,0 +1,169 @@ +// Copyright 2019 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#ifndef THIRD_PARTY_MLIR_EDGE_IREE_VM_BYTECODE_READER_H_ +#define THIRD_PARTY_MLIR_EDGE_IREE_VM_BYTECODE_READER_H_ + +#include "third_party/absl/base/attributes.h" +#include "third_party/absl/container/inlined_vector.h" +#include "third_party/mlir_edge/iree/base/status.h" +#include "third_party/mlir_edge/iree/hal/buffer_view.h" +#include "third_party/mlir_edge/iree/schemas/bytecode/bytecode_v0.h" +#include "third_party/mlir_edge/iree/vm/function.h" +#include "third_party/mlir_edge/iree/vm/stack.h" +#include "third_party/mlir_edge/iree/vm/stack_frame.h" +#include "third_party/mlir_edge/iree/vm/type.h" + +namespace iree { +namespace vm { + +class BytecodeReader { + public: + explicit BytecodeReader(Stack* stack) : stack_(stack) {} + + int offset() const { return static_cast(bytecode_pc_ - bytecode_base_); } + + StatusOr AdvanceOffset(); + + Status SwitchStackFrame(StackFrame* new_stack_frame); + Status BranchToOffset(int32_t offset); + + Status CopyInputsAndSwitchStackFrame(StackFrame* src_stack_frame, + StackFrame* dst_stack_frame); + Status CopyResultsAndSwitchStackFrame(StackFrame* src_stack_frame, + StackFrame* dst_stack_frame); + Status CopySlots(); + + StatusOr ReadConstant(); + + ABSL_ATTRIBUTE_ALWAYS_INLINE StatusOr ReadCount() { + return ReadValue(); + } + + ABSL_ATTRIBUTE_ALWAYS_INLINE StatusOr ReadType() { + ASSIGN_OR_RETURN(uint8_t type_index, ReadValue()); + return Type::FromTypeIndex(type_index); + } + + ABSL_ATTRIBUTE_ALWAYS_INLINE StatusOr ReadFunction() { + ASSIGN_OR_RETURN(auto value, ReadValue()); + const auto& module = stack_frame_->module(); + return module.function_table().LookupFunction(value); + } + + ABSL_ATTRIBUTE_ALWAYS_INLINE StatusOr + ReadImportFunction() { + ASSIGN_OR_RETURN(auto value, ReadValue()); + const auto& module = stack_frame_->module(); + return module.function_table().LookupImport(value); + } + + ABSL_ATTRIBUTE_ALWAYS_INLINE StatusOr ReadLocal( + absl::Span locals) { + ASSIGN_OR_RETURN(auto value, ReadValue()); + if (value > locals.size()) { + return OutOfRangeErrorBuilder(ABSL_LOC) + << "Out of bounds local access " << value << " of " + << locals.size(); + } + return &locals[value]; + } + + ABSL_ATTRIBUTE_ALWAYS_INLINE StatusOr ReadLocal() { + return ReadLocal(locals_); + } + + Status SkipLocals(int count); + + ABSL_ATTRIBUTE_ALWAYS_INLINE StatusOr ReadUint8_t() { + return ReadValue(); + } + + ABSL_ATTRIBUTE_ALWAYS_INLINE StatusOr ReadUint16_t() { + return ReadValue(); + } + + ABSL_ATTRIBUTE_ALWAYS_INLINE StatusOr ReadInt32() { + return ReadValue(); + } + + ABSL_ATTRIBUTE_ALWAYS_INLINE StatusOr ReadBlockOffset() { + return ReadValue(); + } + + template + ABSL_ATTRIBUTE_ALWAYS_INLINE StatusOr> + ReadSlotElements() { + ASSIGN_OR_RETURN(auto* local, ReadLocal(locals_)); + absl::InlinedVector result(local->shape.element_count()); + if (sizeof(T) == local->element_size) { + // Fast(ish) path: requested element size matches the actual element size. + RETURN_IF_ERROR( + local->buffer->ReadData(0, result.data(), result.size() * sizeof(T))); + } else { + // Slow path: need to convert the data. + switch (local->element_size) { + case 4: { + ASSIGN_OR_RETURN(auto mapping, local->buffer->MapMemory( + hal::MemoryAccess::kRead)); + for (size_t i = 0; i < result.size(); ++i) { + result[i] = static_cast(mapping[i]); + } + break; + } + case 8: { + ASSIGN_OR_RETURN(auto mapping, local->buffer->MapMemory( + hal::MemoryAccess::kRead)); + for (size_t i = 0; i < result.size(); ++i) { + result[i] = static_cast(mapping[i]); + } + break; + } + default: + return UnimplementedErrorBuilder(ABSL_LOC) + << "Unsupported local element size: " << local->element_size; + } + } + return result; + } + + Status ReadShape(Shape* out_shape); + + StatusOr ReadShapePieces(); + StatusOr ReadShapePieces(size_t* out_element_count); + + StatusOr> ReadIndexList(); + + private: + template + ABSL_ATTRIBUTE_ALWAYS_INLINE StatusOr ReadValue() { + // TODO(benvanik): validate bounds. + T value = *reinterpret_cast(bytecode_pc_); + bytecode_pc_ += sizeof(T); + return value; + } + + Stack* stack_ = nullptr; + StackFrame* stack_frame_ = nullptr; + const uint8_t* bytecode_base_ = nullptr; + const uint8_t* bytecode_limit_ = nullptr; + const uint8_t* bytecode_pc_ = nullptr; + absl::Span locals_; + FunctionTable::BreakpointTable* breakpoint_table_ = nullptr; +}; + +} // namespace vm +} // namespace iree + +#endif // THIRD_PARTY_MLIR_EDGE_IREE_VM_BYTECODE_READER_H_ diff --git a/vm/bytecode_tables_interpreter.cc b/vm/bytecode_tables_interpreter.cc new file mode 100644 index 000000000000..79f41f7493d7 --- /dev/null +++ b/vm/bytecode_tables_interpreter.cc @@ -0,0 +1,44 @@ +// Copyright 2019 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "third_party/mlir_edge/iree/vm/bytecode_tables_interpreter.h" + +#include "third_party/mlir_edge/iree/schemas/bytecode/interpreter_bytecode_v0.h" + +namespace iree { +namespace vm { + +namespace { + +// Info table mapping 1:1 with bytecode ops. +// +// Note that we ensure the table is 256 elements long exactly to make sure +// that unused opcodes are handled gracefully. +static const OpcodeInfo kInfoTable[256] = { +#define DECLARE_INFO(ordinal, enum_value, name, flags, operand_encodings, ...) \ + OpcodeInfo{ \ + name, \ + flags, \ + {operand_encodings}, \ + }, + IREE_INTERPRETER_OPCODE_LIST(DECLARE_INFO, DECLARE_INFO) +#undef DECLARE_INFO +}; + +} // namespace + +OpcodeTable interpreter_opcode_table() { return kInfoTable; } + +} // namespace vm +} // namespace iree diff --git a/vm/bytecode_tables_interpreter.h b/vm/bytecode_tables_interpreter.h new file mode 100644 index 000000000000..09148b8815d2 --- /dev/null +++ b/vm/bytecode_tables_interpreter.h @@ -0,0 +1,28 @@ +// Copyright 2019 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#ifndef THIRD_PARTY_MLIR_EDGE_IREE_VM_BYTECODE_TABLES_INTERPRETER_H_ +#define THIRD_PARTY_MLIR_EDGE_IREE_VM_BYTECODE_TABLES_INTERPRETER_H_ + +#include "third_party/mlir_edge/iree/vm/opcode_info.h" + +namespace iree { +namespace vm { + +OpcodeTable interpreter_opcode_table(); + +} // namespace vm +} // namespace iree + +#endif // THIRD_PARTY_MLIR_EDGE_IREE_VM_BYTECODE_TABLES_INTERPRETER_H_ diff --git a/vm/bytecode_tables_sequencer.cc b/vm/bytecode_tables_sequencer.cc new file mode 100644 index 000000000000..737f64eda410 --- /dev/null +++ b/vm/bytecode_tables_sequencer.cc @@ -0,0 +1,44 @@ +// Copyright 2019 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "third_party/mlir_edge/iree/vm/bytecode_tables_sequencer.h" + +#include "third_party/mlir_edge/iree/schemas/bytecode/sequencer_bytecode_v0.h" + +namespace iree { +namespace vm { + +namespace { + +// Info table mapping 1:1 with bytecode ops. +// +// Note that we ensure the table is 256 elements long exactly to make sure +// that unused opcodes are handled gracefully. +static const OpcodeInfo kInfoTable[256] = { +#define DECLARE_INFO(ordinal, enum_value, name, flags, operand_encodings, ...) \ + OpcodeInfo{ \ + name, \ + flags, \ + {operand_encodings}, \ + }, + IREE_SEQUENCER_OPCODE_LIST(DECLARE_INFO, DECLARE_INFO) +#undef DECLARE_INFO +}; + +} // namespace + +OpcodeTable sequencer_opcode_table() { return kInfoTable; } + +} // namespace vm +} // namespace iree diff --git a/vm/bytecode_tables_sequencer.h b/vm/bytecode_tables_sequencer.h new file mode 100644 index 000000000000..361d75c8376b --- /dev/null +++ b/vm/bytecode_tables_sequencer.h @@ -0,0 +1,28 @@ +// Copyright 2019 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#ifndef THIRD_PARTY_MLIR_EDGE_IREE_VM_BYTECODE_TABLES_SEQUENCER_H_ +#define THIRD_PARTY_MLIR_EDGE_IREE_VM_BYTECODE_TABLES_SEQUENCER_H_ + +#include "third_party/mlir_edge/iree/vm/opcode_info.h" + +namespace iree { +namespace vm { + +OpcodeTable sequencer_opcode_table(); + +} // namespace vm +} // namespace iree + +#endif // THIRD_PARTY_MLIR_EDGE_IREE_VM_BYTECODE_TABLES_SEQUENCER_H_ diff --git a/vm/bytecode_util.cc b/vm/bytecode_util.cc new file mode 100644 index 000000000000..1259516d666d --- /dev/null +++ b/vm/bytecode_util.cc @@ -0,0 +1,43 @@ +// Copyright 2019 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "third_party/mlir_edge/iree/vm/bytecode_util.h" + +namespace iree { +namespace vm { + +absl::string_view PredicateToString(CmpIPredicate p) { +#define PRED(index, name, str, ...) \ + case CmpIPredicate::name: \ + return str; + switch (p) { + IREE_CMPI_PREDICATE_LIST(PRED) +#undef PRED + } + return ""; +} + +absl::string_view PredicateToString(CmpFPredicate p) { +#define PRED(index, name, str, ...) \ + case CmpFPredicate::name: \ + return str; + switch (p) { + IREE_CMPF_PREDICATE_LIST(PRED) +#undef PRED + } + return ""; +} + +} // namespace vm +} // namespace iree diff --git a/vm/bytecode_util.h b/vm/bytecode_util.h new file mode 100644 index 000000000000..b7a4599b2381 --- /dev/null +++ b/vm/bytecode_util.h @@ -0,0 +1,31 @@ +// Copyright 2019 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#ifndef THIRD_PARTY_MLIR_EDGE_IREE_VM_BYTECODE_UTIL_H_ +#define THIRD_PARTY_MLIR_EDGE_IREE_VM_BYTECODE_UTIL_H_ + +#include "third_party/absl/strings/string_view.h" +#include "third_party/mlir_edge/iree/schemas/bytecode/bytecode_v0.h" + +namespace iree { +namespace vm { + +absl::string_view PredicateToString(CmpIPredicate predicate); + +absl::string_view PredicateToString(CmpFPredicate predicate); + +} // namespace vm +} // namespace iree + +#endif // THIRD_PARTY_MLIR_EDGE_IREE_VM_BYTECODE_UTIL_H_ diff --git a/vm/bytecode_validator.cc b/vm/bytecode_validator.cc new file mode 100644 index 000000000000..78a2875d045e --- /dev/null +++ b/vm/bytecode_validator.cc @@ -0,0 +1,28 @@ +// Copyright 2019 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "third_party/mlir_edge/iree/vm/bytecode_validator.h" + +namespace iree { +namespace vm { + +// static +Status BytecodeValidator::Validate(const Context& context, const Module& module, + const BytecodeDef& bytecode_def) { + // TODO(benvanik): validate bytecode. + return OkStatus(); +} + +} // namespace vm +} // namespace iree diff --git a/vm/bytecode_validator.h b/vm/bytecode_validator.h new file mode 100644 index 000000000000..7359d0133c70 --- /dev/null +++ b/vm/bytecode_validator.h @@ -0,0 +1,38 @@ +// Copyright 2019 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#ifndef THIRD_PARTY_MLIR_EDGE_IREE_VM_BYTECODE_VALIDATOR_H_ +#define THIRD_PARTY_MLIR_EDGE_IREE_VM_BYTECODE_VALIDATOR_H_ + +#include "third_party/mlir_edge/iree/base/status.h" +#include "third_party/mlir_edge/iree/schemas/bytecode_def_generated.h" +#include "third_party/mlir_edge/iree/vm/context.h" +#include "third_party/mlir_edge/iree/vm/module.h" + +namespace iree { +namespace vm { + +// Validates bytecode such that success indicates the bytecode does not +// reference undefined types, functions, or required imports and all imports can +// be resolved with matching signatures. +class BytecodeValidator { + public: + static Status Validate(const Context& context, const Module& module, + const BytecodeDef& bytecode_def); +}; + +} // namespace vm +} // namespace iree + +#endif // THIRD_PARTY_MLIR_EDGE_IREE_VM_BYTECODE_VALIDATOR_H_ diff --git a/vm/context.cc b/vm/context.cc new file mode 100644 index 000000000000..da318bfb7674 --- /dev/null +++ b/vm/context.cc @@ -0,0 +1,111 @@ +// Copyright 2019 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "third_party/mlir_edge/iree/vm/context.h" + +#include "third_party/mlir_edge/iree/base/flatbuffer_util.h" +#include "third_party/mlir_edge/iree/base/status.h" + +namespace iree { +namespace vm { + +namespace { + +int NextUniqueId() { + static int next_id = 0; + return ++next_id; +} + +} // namespace + +Context::Context() : id_(NextUniqueId()) {} + +Context::~Context() = default; + +Status Context::RegisterNativeFunction(std::string name, + NativeFunction native_function) { + native_functions_.emplace_back(std::move(name), std::move(native_function)); + return OkStatus(); +} + +Status Context::RegisterModule(std::unique_ptr module) { + // Attempt to link the module. + RETURN_IF_ERROR(module->mutable_function_table()->ResolveImports( + [&](const Module& importing_module, + const FunctionDef& import_function_def) -> StatusOr { + absl::string_view export_name = WrapString(import_function_def.name()); + + // Try to find a native function (we prefer these). + for (const auto& native_function : native_functions_) { + if (native_function.first == export_name) { + LOG(INFO) << "Resolved import '" << export_name + << "' to native function"; + return ImportFunction(importing_module, import_function_def, + native_function.second); + } + } + + // Try to find an export in an existing module. + // We prefer the more recently registered modules. + // NOTE: slow O(n*m) search through all modules * exports. + for (auto it = modules_.rbegin(); it != modules_.rend(); ++it) { + const auto& module = *it; + auto export_or = module->function_table().LookupExport(export_name); + if (export_or.ok()) { + LOG(INFO) << "Resolved import '" << export_name << "' to module " + << module->name(); + return ImportFunction(importing_module, import_function_def, + export_or.ValueOrDie()); + } + } + + return NotFoundErrorBuilder(ABSL_LOC) + << "Import '" << export_name << "' could not be resolved"; + })); + + modules_.push_back(std::move(module)); + return OkStatus(); +} + +StatusOr Context::LookupModule( + absl::string_view module_name) const { + return const_cast(this)->LookupModule(module_name); +} + +StatusOr Context::LookupModule(absl::string_view module_name) { + for (const auto& module : modules_) { + if (module->name() == module_name) { + return module.get(); + } + } + return NotFoundErrorBuilder(ABSL_LOC) + << "No module with the name '" << module_name + << "' has been registered"; +} + +StatusOr Context::LookupExport( + absl::string_view export_name) const { + for (const auto& module : modules_) { + auto export_or = module->function_table().LookupExport(export_name); + if (export_or.ok()) { + return export_or.ValueOrDie(); + } + } + return NotFoundErrorBuilder(ABSL_LOC) + << "No export with the name '" << export_name + << "' is present in the context"; +} + +} // namespace vm +} // namespace iree diff --git a/vm/context.h b/vm/context.h new file mode 100644 index 000000000000..2656eb57301b --- /dev/null +++ b/vm/context.h @@ -0,0 +1,91 @@ +// Copyright 2019 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#ifndef THIRD_PARTY_MLIR_EDGE_IREE_VM_CONTEXT_H_ +#define THIRD_PARTY_MLIR_EDGE_IREE_VM_CONTEXT_H_ + +#include +#include + +#include "third_party/absl/strings/string_view.h" +#include "third_party/absl/types/span.h" +#include "third_party/mlir_edge/iree/base/status.h" +#include "third_party/mlir_edge/iree/vm/function.h" +#include "third_party/mlir_edge/iree/vm/module.h" + +namespace iree { +namespace vm { + +// An isolated execution context. +// Effectively a sandbox where modules can be loaded and run with restricted +// visibility. Each context may have its own set of imports that modules can +// access and its own resource constraints. +// +// The function namespace is shared within a context, meaning that an import of +// function 'a' from a module will resolve to an export of function 'a' from +// another. Functions internal to a module are not resolved through the +// namespace and may share names (or have no names at all). +// +// Modules have imports resolved automatically when loaded by searching existing +// modules. This means that load order is important to ensure overrides are +// respected. For example, target-specific modules should be loaded prior to +// generic modules that may import functions defined there and if a function is +// not available in the target-specific modules the fallback provided by the +// generic module will be used. +// +// TODO(benvanik): evaluate if worth making thread-safe (epochs/generational). +// Contexts are thread-compatible; const methods may be called concurrently from +// any thread (including Invoke), however no threads must be using a shared +// Context while new native functions or modules are registered. +class Context { + public: + Context(); + Context(const Context&) = delete; + Context& operator=(const Context&) = delete; + Context(Context&&) = default; + Context& operator=(Context&&) = default; + virtual ~Context(); + + // A process-unique ID for the context. + int id() const { return id_; } + + // TODO(benvanik): make immutable by moving to a static Create fn. + virtual Status RegisterNativeFunction(std::string name, + NativeFunction native_function); + + virtual Status RegisterModule(std::unique_ptr module); + + const std::vector>& native_functions() + const { + return native_functions_; + } + + const std::vector>& modules() const { + return modules_; + } + + StatusOr LookupModule(absl::string_view module_name) const; + StatusOr LookupModule(absl::string_view module_name); + StatusOr LookupExport(absl::string_view export_name) const; + + private: + int id_; + std::vector> native_functions_; + std::vector> modules_; +}; + +} // namespace vm +} // namespace iree + +#endif // THIRD_PARTY_MLIR_EDGE_IREE_VM_CONTEXT_H_ diff --git a/vm/debug/debug_client.cc b/vm/debug/debug_client.cc new file mode 100644 index 000000000000..4f089c72dc46 --- /dev/null +++ b/vm/debug/debug_client.cc @@ -0,0 +1,64 @@ +// Copyright 2019 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "third_party/mlir_edge/iree/vm/debug/debug_client.h" + +#include "third_party/absl/types/source_location.h" +#include "third_party/mlir_edge/iree/base/status.h" + +namespace iree { +namespace vm { +namespace debug { + +Status DebugClient::GetFunction( + std::string module_name, std::string function_name, + std::function function)> callback) { + return ResolveFunction( + module_name, function_name, + [this, module_name, callback](StatusOr function_ordinal) { + if (!function_ordinal.ok()) { + callback(function_ordinal.status()); + return; + } + auto status = + GetFunction(module_name, function_ordinal.ValueOrDie(), callback); + if (!status.ok()) { + callback(std::move(status)); + } + }); +} + +Status DebugClient::StepFiberOver(const RemoteFiberState& fiber_state, + std::function callback) { + // TODO(benvanik): implement bytecode stepping search. + // int bytecode_offset = 0; + // return StepFiberToOffset(fiber_state, bytecode_offset, + // std::move(callback)); + return UnimplementedErrorBuilder(ABSL_LOC) + << "StepFiberOver not yet implemented"; +} + +Status DebugClient::StepFiberOut(const RemoteFiberState& fiber_state, + std::function callback) { + // TODO(benvanik): implement bytecode stepping search. + // int bytecode_offset = 0; + // return StepFiberToOffset(fiber_state, bytecode_offset, + // std::move(callback)); + return UnimplementedErrorBuilder(ABSL_LOC) + << "StepFiberOut not yet implemented"; +} + +} // namespace debug +} // namespace vm +} // namespace iree diff --git a/vm/debug/debug_client.h b/vm/debug/debug_client.h new file mode 100644 index 000000000000..60d2f36e06e0 --- /dev/null +++ b/vm/debug/debug_client.h @@ -0,0 +1,280 @@ +// Copyright 2019 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#ifndef THIRD_PARTY_MLIR_EDGE_IREE_VM_DEBUG_DEBUG_CLIENT_H_ +#define THIRD_PARTY_MLIR_EDGE_IREE_VM_DEBUG_DEBUG_CLIENT_H_ + +#include +#include + +#include "third_party/absl/container/flat_hash_map.h" +#include "third_party/absl/strings/string_view.h" +#include "third_party/absl/types/optional.h" +#include "third_party/absl/types/span.h" +#include "third_party/mlir_edge/iree/base/status.h" +#include "third_party/mlir_edge/iree/schemas/debug_service_generated.h" + +namespace iree { +namespace vm { +namespace debug { + +// Remote breakpoint currently active on the server. +class RemoteBreakpoint { + public: + enum class Type { + kBytecodeFunction = 0, + kNativeFunction = 1, + }; + + virtual ~RemoteBreakpoint() = default; + + int id() const { return id_; } + Type type() const { return type_; } + + virtual const std::string& module_name() const = 0; + virtual const std::string& function_name() const = 0; + virtual int function_ordinal() const = 0; + virtual int bytecode_offset() const = 0; + + protected: + explicit RemoteBreakpoint(int id, Type type) : id_(id), type_(type) {} + + private: + int id_; + Type type_; +}; + +class RemoteModule; + +class RemoteFunction { + public: + virtual ~RemoteFunction() = default; + + RemoteModule* module() const { return module_; } + int ordinal() const { return function_ordinal_; } + virtual const std::string& name() const = 0; + + virtual const FunctionDef& def() = 0; + + virtual bool is_loaded() const = 0; + virtual bool CheckLoadedOrRequest() = 0; + + using LoadCallback = std::function)>; + virtual void WhenLoaded(LoadCallback callback) = 0; + + virtual const BytecodeDef* bytecode() = 0; + + protected: + RemoteFunction(RemoteModule* module, int function_ordinal) + : module_(module), function_ordinal_(function_ordinal) {} + + RemoteModule* module_; + int function_ordinal_; +}; + +class RemoteModule { + public: + virtual ~RemoteModule() = default; + + int context_id() const { return context_id_; } + const std::string& name() const { return name_; } + + virtual const ModuleDef& def() = 0; + + virtual bool is_loaded() const = 0; + virtual bool CheckLoadedOrRequest() = 0; + + using LoadCallback = std::function)>; + virtual void WhenLoaded(LoadCallback callback) = 0; + + virtual absl::Span functions() = 0; + + protected: + RemoteModule(int context_id, std::string name) + : context_id_(context_id), name_(std::move(name)) {} + + private: + int context_id_; + std::string name_; +}; + +class RemoteContext { + public: + virtual ~RemoteContext() = default; + + int id() const { return id_; } + + virtual absl::Span modules() const = 0; + + protected: + explicit RemoteContext(int id) : id_(id) {} + + private: + int id_; +}; + +class RemoteFiberState { + public: + virtual ~RemoteFiberState() = default; + + int id() const { return id_; } + const std::string& name() const { return name_; } + + virtual const rpc::FiberStateDefT& def() const = 0; + + protected: + explicit RemoteFiberState(int id) + : id_(id), name_(absl::StrCat("Fiber ", id)) {} + + private: + int id_; + std::string name_; +}; + +// Debugger RPC server client. +// Statefully tracks a DebugServer to provide common client operations and +// memoized queries. +// +// Thread-compatible. Do not use the client from multiple threads concurrently. +// All remote updates of local state are performed by the Poll function. See +// Poll for more details. +class DebugClient { + public: + // Debug event listener interface. + // Event methods will be called from within Poll calls (so on that thread). + // + // When the server posts an event it will mark the client as unready and + // suspend execution of all fibers until MakeReady is used to indicate that + // the client is ready for the server to resume. Each event needs a matching + // MakeReady ack. + // + // Listeners can defer acking if they need to perform additional queries or + // state changes to the server or wait for user interaction. Multiple events + // may come in while unready if there was a series of events pending on the + // server. + class Listener { + public: + virtual ~Listener() = default; + + // Signals that a context has been registered on the server. + virtual Status OnContextRegistered(const RemoteContext& context) = 0; + virtual Status OnContextUnregistered(const RemoteContext& context) = 0; + + // Signals that a module has been loaded into a context on the server. + virtual Status OnModuleLoaded(const RemoteContext& context, + const RemoteModule& module) = 0; + + // Signals that a fiber has been registered on the server. + virtual Status OnFiberRegistered(const RemoteFiberState& fiber_state) = 0; + virtual Status OnFiberUnregistered(const RemoteFiberState& fiber_state) = 0; + + // Signals that a breakpoint has been hit by a fiber on the server. + virtual Status OnBreakpointHit(const RemoteBreakpoint& breakpoint, + const RemoteFiberState& fiber_state) = 0; + }; + + // Connects to a remote debug service at the provided IP:port. + // The specified |listener| will receive async event notifications. + static StatusOr> Connect( + absl::string_view service_address, Listener* listener); + + virtual ~DebugClient() = default; + + // Returns true if the client is connected to a service. + // virtual bool is_connected() const = 0; + + // A list of all contexts registered with the server. + virtual absl::Span contexts() const = 0; + + // A list of all fibers registered with the server. + virtual absl::Span fiber_states() const = 0; + + // A list of all breakpoints registered with the server. + virtual absl::Span breakpoints() const = 0; + + // Resolves a function to a module ordinal. + // This will occur asynchronously and the |callback| will be issued on the + // polling thread. + virtual Status ResolveFunction( + std::string module_name, std::string function_name, + std::function function_ordinal)> callback) = 0; + + // Gets a function body instance. + // The provided |callback| will be issued on the polling thread when the + // function is available. + virtual Status GetFunction( + std::string module_name, int function_ordinal, + std::function function)> callback) = 0; + Status GetFunction( + std::string module_name, std::string function_name, + std::function function)> callback); + + // Adds a breakpoint for the given module:function:offset. + // The breakpoint will apply to all contexts with the module loaded. + virtual Status AddFunctionBreakpoint( + std::string module_name, std::string function_name, int offset, + std::function callback = + nullptr) = 0; + + // Removes a breakpoint from the server. + virtual Status RemoveBreakpoint(const RemoteBreakpoint& breakpoint) = 0; + + // Notifies the server that the debug session is ready to continue. + // This must be called once on connection to and in acknowledgement to any + // events posted by the server (read: any call to the Listener::On* methods). + virtual Status MakeReady() = 0; + + // Suspends all fibers running on the server. + virtual Status SuspendAllFibers() = 0; + + // Resumes all fibers running on the server. + virtual Status ResumeAllFibers() = 0; + + // Suspends a list of fibers running on the server. Fibers not in the provided + // list will not be suspended, such as new fibers created while the request + // is pending. + virtual Status SuspendFibers(absl::Span fibers) = 0; + + // Resumes a list of fibers running on the server. + virtual Status ResumeFibers(absl::Span fibers) = 0; + + // Steps a fiber one bytecode operation. + virtual Status StepFiber(const RemoteFiberState& fiber_state, + std::function callback) = 0; + // Steps a fiber over one bytecode operation, not stopping until it completes. + Status StepFiberOver(const RemoteFiberState& fiber_state, + std::function callback); + // Steps a fiber out of the current block. + Status StepFiberOut(const RemoteFiberState& fiber_state, + std::function callback); + // Steps a fiber to a specific bytecode offset within the current function. + virtual Status StepFiberToOffset(const RemoteFiberState& fiber_state, + int bytecode_offset, + std::function callback) = 0; + + // TODO(benvanik): profiling modes. + + // Polls for the current state of the debug service and processes incoming + // responses. Must be called as frequently as the UI is desired to update. + // Returns CancelledError when the service is being shutdown/disconnected. + // + // Events on the Listener will be called from within this method. + virtual Status Poll() = 0; +}; + +} // namespace debug +} // namespace vm +} // namespace iree + +#endif // THIRD_PARTY_MLIR_EDGE_IREE_VM_DEBUG_DEBUG_CLIENT_H_ diff --git a/vm/debug/debug_client_tcp.cc b/vm/debug/debug_client_tcp.cc new file mode 100644 index 000000000000..851a96837989 --- /dev/null +++ b/vm/debug/debug_client_tcp.cc @@ -0,0 +1,1121 @@ +// Copyright 2019 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include +#include +#include +#include +#include + +#include +#include +#include + +#include "third_party/absl/container/flat_hash_map.h" +#include "third_party/absl/memory/memory.h" +#include "third_party/absl/strings/ascii.h" +#include "third_party/absl/strings/numbers.h" +#include "third_party/absl/strings/str_split.h" +#include "third_party/absl/types/span.h" +#include "third_party/flatbuffers/include/flatbuffers/base.h" +#include "third_party/flatbuffers/include/flatbuffers/flatbuffers.h" +#include "third_party/mlir_edge/iree/base/flatbuffer_util.h" +#include "third_party/mlir_edge/iree/base/status.h" +#include "third_party/mlir_edge/iree/schemas/debug_service_generated.h" +#include "third_party/mlir_edge/iree/schemas/module_def_generated.h" +#include "third_party/mlir_edge/iree/vm/debug/debug_client.h" +#include "third_party/mlir_edge/iree/vm/debug/debug_tcp_util.h" +#include "third_party/mlir_edge/iree/vm/module.h" + +namespace iree { +namespace vm { +namespace debug { +namespace { + +using ::flatbuffers::FlatBufferBuilder; +using ::iree::vm::ModuleFile; + +// Parses a host:port address, with support for the RFC 3986 IPv6 [host]:port +// format. Returns a pair of (hostname, port), with port being 0 if none was +// specified. +// +// Parses: +// foo (port 0) / foo:123 +// 1.2.3.4 (port 0) / 1.2.3.4:123 +// [foo] (port 0) / [foo]:123 +// [::1] (port 0) / [::1]:123 +StatusOr> ParseAddress(absl::string_view address) { + address = absl::StripAsciiWhitespace(address); + absl::string_view hostname; + absl::string_view port_str; + size_t bracket_loc = address.find_last_of(']'); + if (bracket_loc != std::string::npos) { + // Has at least a ]. Let's assume it's mostly right. + if (address.find('[') != 0) { + return InvalidArgumentErrorBuilder(ABSL_LOC) + << "Mismatched brackets in address: " << address; + } + hostname = address.substr(1, bracket_loc - 1); + port_str = address.substr(bracket_loc + 1); + if (port_str.find(':') == 0) { + port_str.remove_prefix(1); + } + } else { + size_t colon_loc = address.find_last_of(':'); + if (colon_loc != std::string::npos) { + hostname = address.substr(0, colon_loc); + port_str = address.substr(colon_loc + 1); + } else { + hostname = address; + port_str = ""; + } + } + int port = 0; + if (!port_str.empty() && !absl::SimpleAtoi(port_str, &port)) { + return InvalidArgumentErrorBuilder(ABSL_LOC) + << "Unable to parse port '" << port_str << "' from " << address; + } + return std::make_pair(std::string(hostname), port); +} + +class TcpDebugClient final : public DebugClient { + public: + class TcpRemoteBreakpoint : public RemoteBreakpoint { + public: + TcpRemoteBreakpoint(int id, Type type, TcpDebugClient* client) + : RemoteBreakpoint(id, type) {} + + const std::string& module_name() const override { return def_.module_name; } + const std::string& function_name() const override { + return def_.function_name; + } + int function_ordinal() const override { return def_.function_ordinal; } + int bytecode_offset() const override { return def_.bytecode_offset; } + + Status MergeFrom(const rpc::BreakpointDef& breakpoint_def) { + breakpoint_def.UnPackTo(&def_); + return OkStatus(); + } + + private: + rpc::BreakpointDefT def_; + }; + + class TcpRemoteFunction final : public RemoteFunction { + public: + TcpRemoteFunction(RemoteModule* module, int function_ordinal, + const FunctionDef* function_def, TcpDebugClient* client) + : RemoteFunction(module, function_ordinal), + def_(function_def), + client_(client) { + name_ = def_->name() ? std::string(WrapString(def_->name())) : ""; + } + + const std::string& name() const override { return name_; } + + const FunctionDef& def() override { return *def_; } + + bool is_loaded() const override { + return contents_.flatbuffers_buffer.size() > 0; + } + + bool CheckLoadedOrRequest() override { + if (!is_loaded()) { + DemandContents(); + } + return is_loaded(); + } + + void WhenLoaded(LoadCallback callback) override { + if (is_loaded()) { + callback(this); + return; + } + load_callbacks_.push_back(std::move(callback)); + } + + const BytecodeDef* bytecode() override { + CHECK(is_loaded()); + return contents_.bytecode_def; + } + + private: + void DemandContents() { + if (!has_requested_contents_) { + VLOG(2) << "Client " << client_->fd() << ": GetFunction(" + << module()->context_id() << ", " << module()->name() << ", " + << ordinal() << ")"; + FlatBufferBuilder fbb; + rpc::GetFunctionRequestT request; + request.session_id = client_->session_id(); + request.context_id = module()->context_id(); + request.module_name = module()->name(); + request.function_ordinal = ordinal(); + auto status = + client_->IssueRequest( + rpc::GetFunctionRequest::Pack(fbb, &request), std::move(fbb), + [this](Status status, + const rpc::Response& response_union) -> Status { + if (!status.ok()) return status; + const auto& response = + *response_union.message_as_GetFunctionResponse(); + VLOG(2) << "Client " << client_->fd() << ": GetFunction(" + << module()->context_id() << ", " << module()->name() + << ", " << ordinal() << ") = ..."; + RETURN_IF_ERROR(MergeFrom(response)); + for (auto& callback : load_callbacks_) { + callback(this); + } + load_callbacks_.clear(); + return OkStatus(); + }); + if (!status.ok()) { + LOG(ERROR) << "Failed to request module: " << status; + return; + } + has_requested_contents_ = true; + } + } + + Status MergeFrom(const rpc::GetFunctionResponse& response) { + // Clone and retain the contents. + // TODO(benvanik): find a way to steal to avoid the reserialization. + BytecodeDefT bytecode_def_storage; + response.bytecode()->UnPackTo(&bytecode_def_storage); + ::flatbuffers::FlatBufferBuilder fbb; + fbb.Finish(response.bytecode()->Pack(fbb, &bytecode_def_storage)); + contents_.flatbuffers_buffer = fbb.Release(); + contents_.bytecode_def = ::flatbuffers::GetRoot( + contents_.flatbuffers_buffer.data()); + return OkStatus(); + } + + const FunctionDef* def_; + TcpDebugClient* client_; + std::string name_; + bool has_requested_contents_ = false; + std::vector load_callbacks_; + struct { + ::flatbuffers::DetachedBuffer flatbuffers_buffer; + const BytecodeDef* bytecode_def = nullptr; + } contents_; + }; + + class TcpRemoteModule final : public RemoteModule { + public: + TcpRemoteModule(int context_id, std::string module_name, + TcpDebugClient* client) + : RemoteModule(context_id, std::move(module_name)), client_(client) {} + + const ModuleDef& def() override { + CHECK(is_loaded()); + return *module_file_->root(); + } + + bool is_loaded() const override { return module_file_ != nullptr; } + + bool CheckLoadedOrRequest() override { + if (!is_loaded()) { + DemandModuleDef(); + } + return is_loaded(); + } + + void WhenLoaded(LoadCallback callback) override { + if (is_loaded()) { + callback(this); + return; + } + load_callbacks_.push_back(std::move(callback)); + } + + absl::Span functions() override { + auto* module_def = DemandModuleDef(); + if (!module_def) return {}; + return {reinterpret_cast(functions_.data()), + functions_.size()}; + } + + private: + const ModuleDef* DemandModuleDef() { + if (module_file_) { + return module_file_->root(); + } + if (!has_requested_module_def_) { + VLOG(2) << "Client " << client_->fd() << ": GetModule(" << context_id() + << ", " << name() << ")"; + FlatBufferBuilder fbb; + rpc::GetModuleRequestT request; + request.session_id = client_->session_id(); + request.context_id = context_id(); + request.module_name = name(); + auto status = + client_->IssueRequest( + rpc::GetModuleRequest::Pack(fbb, &request), std::move(fbb), + [this](Status status, + const rpc::Response& response_union) -> Status { + if (!status.ok()) return status; + const auto& response = + *response_union.message_as_GetModuleResponse(); + VLOG(2) << "Client " << client_->fd() << ": GetModule(" + << context_id() << ", " << name() << ") = ..."; + RETURN_IF_ERROR(MergeFrom(response)); + for (auto& callback : load_callbacks_) { + callback(this); + } + load_callbacks_.clear(); + return OkStatus(); + }); + if (!status.ok()) { + LOG(ERROR) << "Failed to request module: " << status; + return nullptr; + } + has_requested_module_def_ = true; + } + return nullptr; + } + + Status MergeFrom(const rpc::GetModuleResponse& response) { + // Clone and retain the module. + // TODO(benvanik): find a way to steal to avoid the reserialization. + ModuleDefT module_def_storage; + response.module_()->UnPackTo(&module_def_storage); + FlatBufferBuilder fbb; + auto module_offs = response.module_()->Pack(fbb, &module_def_storage); + FinishModuleDefBuffer(fbb, module_offs); + ASSIGN_OR_RETURN(auto module_file, + ModuleFile::CreateWithBackingBuffer(fbb.Release())); + + const auto& module_def = module_file->root(); + const auto& function_table = *module_def->function_table(); + functions_.reserve(function_table.functions()->size()); + for (int i = 0; i < function_table.functions()->size(); ++i) { + const auto* function_def = function_table.functions()->Get(i); + functions_.push_back(absl::make_unique( + this, i, function_def, client_)); + } + + module_file_ = std::move(module_file); + return OkStatus(); + } + + TcpDebugClient* client_; + bool has_requested_module_def_ = false; + std::vector load_callbacks_; + std::unique_ptr module_file_; + std::vector> functions_; + }; + + class TcpRemoteContext final : public RemoteContext { + public: + TcpRemoteContext(int context_id, TcpDebugClient* client) + : RemoteContext(context_id), client_(client) {} + + absl::Span modules() const override { + return absl::MakeConstSpan(modules_); + } + + Status AddModule(std::unique_ptr module) { + modules_.push_back(module.get()); + module_map_.insert({module->name(), std::move(module)}); + return OkStatus(); + } + + Status MergeFrom(const rpc::ContextDef& context_def) { return OkStatus(); } + + private: + TcpDebugClient* client_; + std::vector modules_; + absl::flat_hash_map> + module_map_; + }; + + class TcpRemoteFiberState final : public RemoteFiberState { + public: + TcpRemoteFiberState(int fiber_id, TcpDebugClient* client) + : RemoteFiberState(fiber_id), client_(client) {} + + const rpc::FiberStateDefT& def() const override { return def_; } + + Status MergeFrom(const rpc::FiberStateDef& fiber_state_def) { + fiber_state_def.UnPackTo(&def_); + return OkStatus(); + } + + private: + TcpDebugClient* client_; + rpc::FiberStateDefT def_; + }; + + static StatusOr> Create(int fd, + Listener* listener) { + VLOG(2) << "Client " << fd << ": Setting up socket options..."; + // Disable Nagel's algorithm to ensure we have low latency. + RETURN_IF_ERROR(tcp::ToggleSocketNagelsAlgorithm(fd, false)); + // Enable keepalive assuming the client is local and this high freq is ok. + RETURN_IF_ERROR(tcp::ToggleSocketLocalKeepalive(fd, true)); + // Linger around for a bit to flush all data. + RETURN_IF_ERROR(tcp::ToggleSocketLinger(fd, true)); + // Disable blocking as we are poll based. + RETURN_IF_ERROR(tcp::ToggleSocketBlocking(fd, false)); + + auto client = absl::make_unique(fd, listener); + RETURN_IF_ERROR(client->Refresh()); + return client; + } + + TcpDebugClient(int fd, Listener* listener) : fd_(fd), listener_(listener) {} + + ~TcpDebugClient() override { + VLOG(2) << "Client " << fd_ << ": Shutting down session socket..."; + ::shutdown(fd_, SHUT_WR); + VLOG(2) << "Client " << fd_ << ": Closing session socket..."; + ::close(fd_); + VLOG(2) << "Client " << fd_ << ": Closed session socket!"; + fd_ = -1; + } + + int fd() const { return fd_; } + int session_id() const { return session_id_; } + + absl::Span contexts() const override { + return absl::MakeConstSpan(contexts_); + } + + absl::Span fiber_states() const override { + return absl::MakeConstSpan(fiber_states_); + } + + absl::Span breakpoints() const override { + return absl::MakeConstSpan(breakpoints_); + } + + // Writes the given typed request message to the given fd by wrapping it in + // a size-prefixed rpc::Request union. + // + // Example: + // FlatBufferBuilder fbb; + // rpc::SuspendFiberRequestBuilder request(fbb); + // RETURN_IF_ERROR(WriteRequest(fd_, request.Finish(), std::move(fbb))); + template + Status WriteRequest(int fd, ::flatbuffers::Offset request_offs, + FlatBufferBuilder fbb) { + rpc::RequestBuilder request_builder(fbb); + request_builder.add_message_type(rpc::RequestUnionTraits::enum_value); + request_builder.add_message(request_offs.Union()); + fbb.FinishSizePrefixed(request_builder.Finish()); + auto write_status = tcp::WriteBuffer(fd, fbb.Release()); + if (shutdown_pending_ && IsUnavailable(write_status)) { + return OkStatus(); + } + return write_status; + } + + Status ResolveFunction( + std::string module_name, std::string function_name, + std::function function_ordinal)> callback) override { + VLOG(2) << "Client " << fd_ << ": ResolveFunction(" << module_name << ", " + << function_name << ")"; + FlatBufferBuilder fbb; + rpc::ResolveFunctionRequestT request; + request.session_id = session_id_; + request.module_name = module_name; + request.function_name = function_name; + return IssueRequest( + rpc::ResolveFunctionRequest::Pack(fbb, &request), std::move(fbb), + [this, module_name, function_name, callback]( + Status status, const rpc::Response& response_union) -> Status { + if (status.ok()) { + const auto& response = + *response_union.message_as_ResolveFunctionResponse(); + VLOG(2) << "Client " << fd_ << ": ResolveFunction(" << module_name + << ", " << function_name + << ") = " << response.function_ordinal(); + callback(response.function_ordinal()); + } else { + callback(std::move(status)); + } + return OkStatus(); + }); + } + + Status GetFunction(std::string module_name, int function_ordinal, + std::function function)> + callback) override { + // See if we have the module already. If not, we'll fetch it first. + RemoteModule* target_module = nullptr; + for (auto* context : contexts_) { + for (auto* module : context->modules()) { + if (module->name() == module_name) { + target_module = module; + break; + } + } + if (target_module) break; + } + if (!target_module) { + // TODO(benvanik): fetch contexts first. + return UnimplementedErrorBuilder(ABSL_LOC) + << "Demand fetch contexts not yet implemented"; + } + // Found at least one module with the right name. + if (target_module->is_loaded()) { + callback(target_module->functions()[function_ordinal]); + return OkStatus(); + } else { + // Wait until the module completes loading. + target_module->WhenLoaded( + [callback, function_ordinal](StatusOr module_or) { + if (!module_or.ok()) { + callback(module_or.status()); + return; + } + callback(module_or.ValueOrDie()->functions()[function_ordinal]); + }); + return OkStatus(); + } + } + + Status AddFunctionBreakpoint( + std::string module_name, std::string function_name, int offset, + std::function callback) + override { + VLOG(2) << "Client " << fd_ << ": AddFunctionBreakpoint(" << module_name + << ", " << function_name << ", " << offset << ")"; + FlatBufferBuilder fbb; + + auto breakpoint = absl::make_unique(); + breakpoint->module_name = module_name; + breakpoint->function_name = function_name; + breakpoint->function_ordinal = -1; + breakpoint->bytecode_offset = offset; + rpc::AddBreakpointRequestT request; + request.session_id = session_id_; + request.breakpoint = std::move(breakpoint); + return IssueRequest( + rpc::AddBreakpointRequest::Pack(fbb, &request), std::move(fbb), + [this, callback](Status status, + const rpc::Response& response_union) -> Status { + if (!status.ok()) return status; + const auto& response = + *response_union.message_as_AddBreakpointResponse(); + RETURN_IF_ERROR(RegisterBreakpoint(*response.breakpoint())); + if (callback) { + ASSIGN_OR_RETURN( + auto breakpoint, + GetBreakpoint(response.breakpoint()->breakpoint_id())); + callback(*breakpoint); + } + return OkStatus(); + }); + } + + Status RemoveBreakpoint(const RemoteBreakpoint& breakpoint) override { + VLOG(2) << "Client " << fd_ << ": RemoveBreakpoint(" << breakpoint.id() + << ")"; + int breakpoint_id = breakpoint.id(); + ASSIGN_OR_RETURN(auto* breakpoint_ptr, GetBreakpoint(breakpoint_id)); + RETURN_IF_ERROR(UnregisterBreakpoint(breakpoint_ptr)); + FlatBufferBuilder fbb; + rpc::RemoveBreakpointRequestBuilder request(fbb); + request.add_session_id(session_id_); + request.add_breakpoint_id(breakpoint_id); + return IssueRequest( + request.Finish(), std::move(fbb), + [](Status status, const rpc::Response& response_union) -> Status { + if (!status.ok()) return status; + // No non-error status. + return OkStatus(); + }); + } + + Status MakeReady() override { + FlatBufferBuilder fbb; + rpc::MakeReadyRequestBuilder request(fbb); + request.add_session_id(session_id_); + return IssueRequest( + request.Finish(), std::move(fbb), + [](Status status, const rpc::Response& response_union) { + return status; + }); + } + + Status SuspendAllFibers() override { + VLOG(2) << "Client " << fd_ << ": SuspendAllFibers()"; + FlatBufferBuilder fbb; + rpc::SuspendFibersRequestBuilder request(fbb); + request.add_session_id(session_id_); + return IssueRequest( + request.Finish(), std::move(fbb), + [this](Status status, const rpc::Response& response_union) -> Status { + if (!status.ok()) return status; + return RefreshFiberStates(); + }); + } + + Status ResumeAllFibers() override { + VLOG(2) << "Client " << fd_ << ": ResumeAllFibers()"; + FlatBufferBuilder fbb; + rpc::ResumeFibersRequestBuilder request(fbb); + request.add_session_id(session_id_); + return IssueRequest( + request.Finish(), std::move(fbb), + [this](Status status, const rpc::Response& response_union) -> Status { + if (!status.ok()) return status; + return RefreshFiberStates(); + }); + } + + Status SuspendFibers(absl::Span fibers) override { + VLOG(2) << "Client " << fd_ << ": SuspendFibers(...)"; + FlatBufferBuilder fbb; + auto fiber_ids_offs = fbb.CreateVector( + fibers.size(), [&fibers](size_t i) { return fibers[i]->id(); }); + rpc::SuspendFibersRequestBuilder request(fbb); + request.add_session_id(session_id_); + request.add_fiber_ids(fiber_ids_offs); + return IssueRequest( + request.Finish(), std::move(fbb), + [this](Status status, const rpc::Response& response_union) -> Status { + if (!status.ok()) return status; + return RefreshFiberStates(); + }); + } + + Status ResumeFibers(absl::Span fibers) override { + VLOG(2) << "Client " << fd_ << ": ResumeFibers(...)"; + FlatBufferBuilder fbb; + auto fiber_ids_offs = fbb.CreateVector( + fibers.size(), [&fibers](size_t i) { return fibers[i]->id(); }); + rpc::ResumeFibersRequestBuilder request(fbb); + request.add_session_id(session_id_); + request.add_fiber_ids(fiber_ids_offs); + return IssueRequest( + request.Finish(), std::move(fbb), + [this](Status status, const rpc::Response& response_union) -> Status { + if (!status.ok()) return status; + return RefreshFiberStates(); + }); + } + + Status StepFiber(const RemoteFiberState& fiber_state, + std::function callback) override { + int step_id = next_step_id_++; + VLOG(2) << "Client " << fd_ << ": StepFiber(" << fiber_state.id() + << ") as step_id=" << step_id; + rpc::StepFiberRequestT step_request; + step_request.step_id = step_id; + step_request.fiber_id = fiber_state.id(); + step_request.step_mode = rpc::StepMode::STEP_ONCE; + return StepFiber(&step_request, std::move(callback)); + } + + Status StepFiberToOffset(const RemoteFiberState& fiber_state, + int bytecode_offset, + std::function callback) override { + int step_id = next_step_id_++; + VLOG(2) << "Client " << fd_ << ": StepFiberToOffset(" << fiber_state.id() + << ", " << bytecode_offset << ") as step_id=" << step_id; + rpc::StepFiberRequestT step_request; + step_request.step_id = step_id; + step_request.fiber_id = fiber_state.id(); + step_request.step_mode = rpc::StepMode::STEP_TO_OFFSET; + step_request.bytecode_offset = bytecode_offset; + return StepFiber(&step_request, std::move(callback)); + } + + Status Poll() override { + while (true) { + // If nothing awaiting then return immediately. + if (!tcp::CanReadBuffer(fd_)) { + break; + } + + // Read the pending response and dispatch. + auto packet_buffer_or = tcp::ReadBuffer(fd_); + if (!packet_buffer_or.ok()) { + if (shutdown_pending_ && IsUnavailable(packet_buffer_or.status())) { + // This is a graceful close. + return CancelledErrorBuilder(ABSL_LOC) << "Service shutdown"; + } + return packet_buffer_or.status(); + } + const auto& packet = packet_buffer_or.ValueOrDie().GetRoot(); + if (packet.response()) { + RETURN_IF_ERROR(DispatchResponse(*packet.response())); + } + if (packet.event()) { + RETURN_IF_ERROR(DispatchEvent(packet)); + } + } + return OkStatus(); + } + + using ResponseCallback = + std::function; + + template + Status IssueRequest(::flatbuffers::Offset request_offs, + FlatBufferBuilder fbb, ResponseCallback callback) { + RETURN_IF_ERROR(WriteRequest(fd_, request_offs, std::move(fbb))); + pending_responses_.push({response_type, std::move(callback)}); + return OkStatus(); + } + + private: + Status Refresh() { + RETURN_IF_ERROR(RefreshContexts()); + RETURN_IF_ERROR(RefreshFiberStates()); + RETURN_IF_ERROR(RefreshBreakpoints()); + return OkStatus(); + } + + Status RefreshContexts() { + VLOG(2) << "Request contexts refresh..."; + FlatBufferBuilder fbb; + rpc::ListContextsRequestBuilder request(fbb); + request.add_session_id(session_id_); + return IssueRequest( + request.Finish(), std::move(fbb), + [this](Status status, const rpc::Response& response_union) -> Status { + if (!status.ok()) return status; + VLOG(2) << "Refreshing contexts..."; + const auto& response = + *response_union.message_as_ListContextsResponse(); + for (auto* context_def : *response.contexts()) { + auto context_or = GetContext(context_def->context_id()); + if (!context_or.ok()) { + // Not found; add new. + RETURN_IF_ERROR(RegisterContext(context_def->context_id())); + context_or = GetContext(context_def->context_id()); + } + RETURN_IF_ERROR(context_or.status()); + RETURN_IF_ERROR(context_or.ValueOrDie()->MergeFrom(*context_def)); + } + VLOG(2) << "Refreshed contexts!"; + return OkStatus(); + }); + } + + Status RefreshFiberStates() { + VLOG(2) << "Request fiber states refresh..."; + FlatBufferBuilder fbb; + rpc::ListFibersRequestBuilder request(fbb); + request.add_session_id(session_id_); + return IssueRequest( + request.Finish(), std::move(fbb), + [this](Status status, const rpc::Response& response_union) -> Status { + if (!status.ok()) return status; + VLOG(2) << "Refreshing fiber states..."; + const auto& response = + *response_union.message_as_ListFibersResponse(); + for (auto* fiber_state_def : *response.fiber_states()) { + auto fiber_state_or = GetFiberState(fiber_state_def->fiber_id()); + if (!fiber_state_or.ok()) { + // Not found; add new. + RETURN_IF_ERROR(RegisterFiberState(fiber_state_def->fiber_id())); + fiber_state_or = GetFiberState(fiber_state_def->fiber_id()); + } + RETURN_IF_ERROR(fiber_state_or.status()); + RETURN_IF_ERROR( + fiber_state_or.ValueOrDie()->MergeFrom(*fiber_state_def)); + } + // TODO(benvanik): handle removals/deaths. + VLOG(2) << "Refreshed fiber states!"; + return OkStatus(); + }); + } + + Status RefreshBreakpoints() { + VLOG(2) << "Requesting breakpoint refresh..."; + FlatBufferBuilder fbb; + rpc::ListBreakpointsRequestBuilder request(fbb); + request.add_session_id(session_id_); + return IssueRequest( + request.Finish(), std::move(fbb), + [this](Status status, const rpc::Response& response_union) -> Status { + if (!status.ok()) return status; + VLOG(2) << "Refreshing breakpoints..."; + const auto& response = + *response_union.message_as_ListBreakpointsResponse(); + for (auto* breakpoint_def : *response.breakpoints()) { + auto breakpoint_or = GetBreakpoint(breakpoint_def->breakpoint_id()); + if (!breakpoint_or.ok()) { + // Not found; add new. + RETURN_IF_ERROR(RegisterBreakpoint(*breakpoint_def)); + breakpoint_or = GetBreakpoint(breakpoint_def->breakpoint_id()); + } + RETURN_IF_ERROR(breakpoint_or.status()); + RETURN_IF_ERROR( + breakpoint_or.ValueOrDie()->MergeFrom(*breakpoint_def)); + } + // TODO(benvanik): handle removals/deaths. + VLOG(2) << "Refreshed breakpoints!"; + return OkStatus(); + }); + } + + Status DispatchResponse(const rpc::Response& response) { + if (pending_responses_.empty()) { + return FailedPreconditionErrorBuilder(ABSL_LOC) + << "Response received but no request is pending"; + } + auto type_callback = std::move(pending_responses_.front()); + pending_responses_.pop(); + + if (response.status()) { + const auto& status = *response.status(); + Status client_status = + StatusBuilder(static_cast(status.code()), ABSL_LOC) + << "Server request failed: " << WrapString(status.message()); + return type_callback.second(std::move(client_status), response); + } + + if (!response.message()) { + return InvalidArgumentErrorBuilder(ABSL_LOC) + << "Response contains no message body"; + } + + if (response.message_type() != type_callback.first) { + return DataLossErrorBuilder(ABSL_LOC) + << "Out of order response (mismatch pending)"; + } + return type_callback.second(OkStatus(), response); + } + + Status DispatchEvent(const rpc::ServicePacket& packet) { + switch (packet.event_type()) { +#define DISPATCH_EVENT(event_name) \ + case rpc::EventUnion::event_name##Event: { \ + VLOG(2) << "EVENT: " << #event_name; \ + return On##event_name(*packet.event_as_##event_name##Event()); \ + } + DISPATCH_EVENT(ServiceShutdown); + DISPATCH_EVENT(ContextRegistered); + DISPATCH_EVENT(ContextUnregistered); + DISPATCH_EVENT(ModuleLoaded); + DISPATCH_EVENT(FiberRegistered); + DISPATCH_EVENT(FiberUnregistered); + DISPATCH_EVENT(BreakpointResolved); + DISPATCH_EVENT(BreakpointHit); + DISPATCH_EVENT(StepCompleted); + default: + return UnimplementedErrorBuilder(ABSL_LOC) + << "Unimplemented debug service event: " + << static_cast(packet.event_type()); + } + } + + StatusOr GetContext(int context_id) { + auto it = context_map_.find(context_id); + if (it == context_map_.end()) { + return NotFoundErrorBuilder(ABSL_LOC) << "Context was never registered"; + } + return it->second.get(); + } + + Status OnServiceShutdown(const rpc::ServiceShutdownEvent& event) { + LOG(INFO) << "Service is shutting down; setting pending shutdown flag"; + shutdown_pending_ = true; + return OkStatus(); + } + + Status RegisterContext(int context_id) { + auto context = absl::make_unique(context_id, this); + VLOG(2) << "RegisterContext(" << context_id << ")"; + auto context_ptr = context.get(); + context_map_.insert({context_id, std::move(context)}); + contexts_.push_back(context_ptr); + return listener_->OnContextRegistered(*context_ptr); + } + + Status OnContextRegistered(const rpc::ContextRegisteredEvent& event) { + VLOG(2) << "OnContextRegistered(" << event.context_id() << ")"; + auto it = context_map_.find(event.context_id()); + if (it != context_map_.end()) { + return FailedPreconditionErrorBuilder(ABSL_LOC) + << "Context already registered"; + } + return RegisterContext(event.context_id()); + } + + Status OnContextUnregistered(const rpc::ContextUnregisteredEvent& event) { + VLOG(2) << "OnContextUnregistered(" << event.context_id() << ")"; + auto it = context_map_.find(event.context_id()); + if (it == context_map_.end()) { + return FailedPreconditionErrorBuilder(ABSL_LOC) + << "Context was never registered"; + } + auto context = std::move(it->second); + context_map_.erase(it); + auto list_it = std::find(contexts_.begin(), contexts_.end(), context.get()); + contexts_.erase(list_it); + return listener_->OnContextUnregistered(*context); + } + + Status OnModuleLoaded(const rpc::ModuleLoadedEvent& event) { + VLOG(2) << "OnModuleLoaded(" << event.context_id() << ", " + << WrapString(event.module_name()) << ")"; + ASSIGN_OR_RETURN(auto* context, GetContext(event.context_id())); + auto module_name = WrapString(event.module_name()); + auto module = absl::make_unique( + event.context_id(), std::string(module_name), this); + auto* module_ptr = module.get(); + RETURN_IF_ERROR(context->AddModule(std::move(module))); + return listener_->OnModuleLoaded(*context, *module_ptr); + } + + StatusOr GetFiberState(int fiber_id) { + auto it = fiber_state_map_.find(fiber_id); + if (it == fiber_state_map_.end()) { + return NotFoundErrorBuilder(ABSL_LOC) << "Fiber was never registered"; + } + return it->second.get(); + } + + Status RegisterFiberState(int fiber_id) { + VLOG(2) << "RegisterFiberState(" << fiber_id << ")"; + auto fiber_state = absl::make_unique(fiber_id, this); + auto fiber_state_ptr = fiber_state.get(); + fiber_state_map_.insert({fiber_id, std::move(fiber_state)}); + fiber_states_.push_back(fiber_state_ptr); + RETURN_IF_ERROR(RefreshFiberStates()); + return listener_->OnFiberRegistered(*fiber_state_ptr); + } + + Status OnFiberRegistered(const rpc::FiberRegisteredEvent& event) { + VLOG(2) << "OnFiberRegistered(" << event.fiber_id() << ")"; + auto it = fiber_state_map_.find(event.fiber_id()); + if (it != fiber_state_map_.end()) { + return FailedPreconditionErrorBuilder(ABSL_LOC) + << "Fiber already registered"; + } + return RegisterFiberState(event.fiber_id()); + } + + Status OnFiberUnregistered(const rpc::FiberUnregisteredEvent& event) { + VLOG(2) << "OnFiberUnregistered(" << event.fiber_id() << ")"; + auto it = fiber_state_map_.find(event.fiber_id()); + if (it == fiber_state_map_.end()) { + return FailedPreconditionErrorBuilder(ABSL_LOC) + << "Fiber was never registered"; + } + auto fiber_state = std::move(it->second); + fiber_state_map_.erase(it); + auto list_it = std::find(fiber_states_.begin(), fiber_states_.end(), + fiber_state.get()); + fiber_states_.erase(list_it); + return listener_->OnFiberUnregistered(*fiber_state); + } + + StatusOr GetBreakpoint(int breakpoint_id) { + auto it = breakpoint_map_.find(breakpoint_id); + if (it == breakpoint_map_.end()) { + return NotFoundErrorBuilder(ABSL_LOC) + << "Breakpoint " << breakpoint_id << " was never registered"; + } + return it->second.get(); + } + + Status RegisterBreakpoint(const rpc::BreakpointDef& breakpoint_def) { + auto it = breakpoint_map_.find(breakpoint_def.breakpoint_id()); + if (it != breakpoint_map_.end()) { + VLOG(2) << "RegisterBreakpoint(" << breakpoint_def.breakpoint_id() + << ") (update)"; + return it->second->MergeFrom(breakpoint_def); + } + + VLOG(2) << "RegisterBreakpoint(" << breakpoint_def.breakpoint_id() << ")"; + auto breakpoint = absl::make_unique( + breakpoint_def.breakpoint_id(), + static_cast(breakpoint_def.breakpoint_type()), + this); + RETURN_IF_ERROR(breakpoint->MergeFrom(breakpoint_def)); + breakpoints_.push_back(breakpoint.get()); + breakpoint_map_.insert({breakpoint->id(), std::move(breakpoint)}); + return OkStatus(); + } + + Status UnregisterBreakpoint(RemoteBreakpoint* breakpoint) { + VLOG(2) << "UnregisterBreakpoint(" << breakpoint->id() << ")"; + auto it = breakpoint_map_.find(breakpoint->id()); + if (it == breakpoint_map_.end()) { + return NotFoundErrorBuilder(ABSL_LOC) + << "Breakpoint was never registered"; + } + breakpoint_map_.erase(it); + auto list_it = + std::find(breakpoints_.begin(), breakpoints_.end(), breakpoint); + breakpoints_.erase(list_it); + return OkStatus(); + } + + Status OnBreakpointResolved(const rpc::BreakpointResolvedEvent& event) { + VLOG(2) << "OnBreakpointResolved(" << event.breakpoint()->breakpoint_id() + << ")"; + auto it = breakpoint_map_.find(event.breakpoint()->breakpoint_id()); + if (it == breakpoint_map_.end()) { + RETURN_IF_ERROR(RegisterBreakpoint(*event.breakpoint())); + } else { + RETURN_IF_ERROR(it->second->MergeFrom(*event.breakpoint())); + } + return OkStatus(); + } + + Status OnBreakpointHit(const rpc::BreakpointHitEvent& event) { + VLOG(2) << "OnBreakpointHit(" << event.breakpoint_id() << ")"; + ASSIGN_OR_RETURN(auto* breakpoint, GetBreakpoint(event.breakpoint_id())); + auto* fiber_state_def = event.fiber_state(); + auto fiber_state_or = GetFiberState(fiber_state_def->fiber_id()); + if (!fiber_state_or.ok()) { + // Not found; add new. + RETURN_IF_ERROR(RegisterFiberState(fiber_state_def->fiber_id())); + fiber_state_or = GetFiberState(fiber_state_def->fiber_id()); + } + RETURN_IF_ERROR(fiber_state_or.status()); + RETURN_IF_ERROR(fiber_state_or.ValueOrDie()->MergeFrom(*fiber_state_def)); + return listener_->OnBreakpointHit(*breakpoint, + *fiber_state_or.ValueOrDie()); + } + + Status StepFiber(rpc::StepFiberRequestT* step_request, + std::function callback) { + FlatBufferBuilder fbb; + auto status = IssueRequest( + rpc::StepFiberRequest::Pack(fbb, step_request), std::move(fbb), + [](Status status, const rpc::Response& response_union) -> Status { + return status; + }); + RETURN_IF_ERROR(status); + pending_step_callbacks_[step_request->step_id] = std::move(callback); + return OkStatus(); + } + + Status OnStepCompleted(const rpc::StepCompletedEvent& event) { + VLOG(2) << "OnStepCompleted(" << event.step_id() << ")"; + + // Update all fiber states that are contained. + // This may only be a subset of relevant states. + for (auto* fiber_state_def : *event.fiber_states()) { + ASSIGN_OR_RETURN(auto fiber_state, + GetFiberState(fiber_state_def->fiber_id())); + RETURN_IF_ERROR(fiber_state->MergeFrom(*fiber_state_def)); + } + + // Dispatch step callback. Note that it may have been cancelled and that's + // ok. We'll just make ready to resume execution. + auto it = pending_step_callbacks_.find(event.step_id()); + if (it != pending_step_callbacks_.end()) { + it->second(); + pending_step_callbacks_.erase(it); + } else { + LOG(WARNING) << "Step " << event.step_id() + << " not found; was cancelled?"; + RETURN_IF_ERROR(MakeReady()); + } + return OkStatus(); + } + + int session_id_ = 123; + + int fd_ = -1; + Listener* listener_; + bool shutdown_pending_ = false; + std::queue> + pending_responses_; + + std::vector contexts_; + absl::flat_hash_map> context_map_; + std::vector fiber_states_; + absl::flat_hash_map> + fiber_state_map_; + std::vector breakpoints_; + absl::flat_hash_map> + breakpoint_map_; + + int next_step_id_ = 1; + absl::flat_hash_map> pending_step_callbacks_; +}; + +} // namespace + +// static +StatusOr> DebugClient::Connect( + absl::string_view service_address, Listener* listener) { + // Parse address into hostname and port. + ASSIGN_OR_RETURN(auto hostname_port, ParseAddress(service_address)); + if (hostname_port.second == 0) { + return InvalidArgumentErrorBuilder(ABSL_LOC) + << "No port specified in service address; port must match the " + "server: " + << service_address; + } + + // Attempt to resolve the address. + // Note that if we only wanted local debugging we could remove the dep on + // getaddrinfo/having a valid DNS setup. + addrinfo hints = {0}; + hints.ai_family = AF_UNSPEC; + hints.ai_socktype = SOCK_STREAM; + addrinfo* resolved_address = nullptr; + auto port_str = std::to_string(hostname_port.second); + int getaddrinfo_ret = ::getaddrinfo( + hostname_port.first.c_str(), port_str.c_str(), &hints, &resolved_address); + if (getaddrinfo_ret != 0) { + return UnavailableErrorBuilder(ABSL_LOC) + << "Unable to resolve debug service address for " << service_address + << ": (" << getaddrinfo_ret << ") " + << ::gai_strerror(getaddrinfo_ret); + } + + // Attempt to connect with each address returned from the query. + int fd = -1; + for (addrinfo* rp = resolved_address; rp != nullptr; rp = rp->ai_next) { + fd = ::socket(rp->ai_family, rp->ai_socktype, rp->ai_protocol); + if (fd == -1) continue; + if (::connect(fd, rp->ai_addr, rp->ai_addrlen) == 0) { + break; // Success! + } + ::close(fd); + fd = -1; + } + ::freeaddrinfo(resolved_address); + if (fd == -1) { + return UnavailableErrorBuilder(ABSL_LOC) + << "Unable to connect to " << service_address << " on any address: (" + << errno << ") " << ::strerror(errno); + } + + LOG(INFO) << "Connected to debug service at " << service_address; + + return TcpDebugClient::Create(fd, listener); +} + +} // namespace debug +} // namespace vm +} // namespace iree diff --git a/vm/debug/debug_server.h b/vm/debug/debug_server.h new file mode 100644 index 000000000000..a1a963634237 --- /dev/null +++ b/vm/debug/debug_server.h @@ -0,0 +1,86 @@ +// Copyright 2019 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#ifndef THIRD_PARTY_MLIR_EDGE_IREE_VM_DEBUG_DEBUG_SERVER_H_ +#define THIRD_PARTY_MLIR_EDGE_IREE_VM_DEBUG_DEBUG_SERVER_H_ + +#include "third_party/mlir_edge/iree/base/status.h" + +namespace iree { +namespace vm { +class FiberState; +class SequencerContext; +class Module; +} // namespace vm +} // namespace iree + +namespace iree { +namespace vm { +namespace debug { + +// Runtime debugging server. +// Enabled only when compiled in (by defining IREE_DEBUG=1), this provides an +// RPC server that allows debuggers to attach, query, and manipulate contexts. +// This interface is used by various parts of the runtime such as dispatch to +// query the current debug state and signal events. +// +// Thread-safe. Contexts may be registered and unregistered from any thread. +class DebugServer { + public: + // Creates a new debug service listening on the provided |port|. + // Even when disabled the device can still be created however it will not + // perform any actual operations and act as if the debugger is not attached. + static StatusOr> Create(int listen_port); + + // TODO(benvanik): ensure this gets optimized out when disabled. + // Seems to be the case: https://gcc.godbolt.org/z/0zf-L4 + virtual ~DebugServer() = default; + + // Attaches a callback that will be made when the debug server is shutting + // down. This can be used to keep resources alive that require the debugger. + // The callback will be made from a random thread. + virtual void AtExit(std::function callback) = 0; + + // Blocks the caller until a client session connects and resumes all fibers. + // Returns AbortedError if a session connects/is connected but disconnects + // during the wait. + virtual Status WaitUntilSessionReady() = 0; + + protected: + friend class ::iree::vm::SequencerContext; + + // Registers a context with the debug service. + // Ownership remains with the caller and UnregisterContext must be called + // prior to the context being destroyed. + virtual Status RegisterContext(SequencerContext* context) = 0; + virtual Status UnregisterContext(SequencerContext* context) = 0; + + // Registers a new module linked into an existing Context. + virtual Status RegisterContextModule(SequencerContext* context, + vm::Module* module) = 0; + + friend class ::iree::vm::FiberState; + + // Registers a fiber state with the debug service. + // Ownership remains with the caller and UnregisterFiberState must be called + // prior to the fiber state being destroyed. + virtual Status RegisterFiberState(FiberState* fiber_state) = 0; + virtual Status UnregisterFiberState(FiberState* fiber_state) = 0; +}; + +} // namespace debug +} // namespace vm +} // namespace iree + +#endif // THIRD_PARTY_MLIR_EDGE_IREE_VM_DEBUG_DEBUG_SERVER_H_ diff --git a/vm/debug/debug_server_disabled.cc b/vm/debug/debug_server_disabled.cc new file mode 100644 index 000000000000..0c8020282dc6 --- /dev/null +++ b/vm/debug/debug_server_disabled.cc @@ -0,0 +1,28 @@ +// Copyright 2019 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "third_party/mlir_edge/iree/vm/debug/debug_server.h" + +namespace iree { +namespace vm { +namespace debug { + +// static +StatusOr> DebugServer::Create(int listen_port) { + return {nullptr}; +} + +} // namespace debug +} // namespace vm +} // namespace iree diff --git a/vm/debug/debug_server_flags.cc b/vm/debug/debug_server_flags.cc new file mode 100644 index 000000000000..31f7333e5244 --- /dev/null +++ b/vm/debug/debug_server_flags.cc @@ -0,0 +1,80 @@ +// Copyright 2019 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "third_party/mlir_edge/iree/vm/debug/debug_server_flags.h" + +#include "third_party/absl/flags/flag.h" +#include "third_party/absl/strings/str_cat.h" +#include "third_party/mlir_edge/iree/base/memory.h" +#include "third_party/mlir_edge/iree/base/status.h" + +#if defined(IREE_DEBUG_EMBEDDED_APP_PRESENT) +#include "third_party/mlir_edge/iree/tools/debugger/debug_app_embedded.h" +#endif // IREE_DEBUG_EMBEDDED_APP_PRESENT + +ABSL_FLAG(int32_t, iree_debug_service_port, 6000, + "TCP port to listen for debug service connections."); +ABSL_FLAG(bool, iree_wait_for_debugger, false, + "Waits until a debugger connects to continue startup."); +ABSL_FLAG(bool, iree_attach_debugger, false, "Attaches a debugger at startup."); + +namespace iree { +namespace vm { +namespace debug { + +StatusOr> CreateDebugServerFromFlags() { + // Create the server based on whatever version is compiled in. + // Note that this will return nullptr if no server is available. + ASSIGN_OR_RETURN( + auto debug_server, + DebugServer::Create(absl::GetFlag(FLAGS_iree_debug_service_port))); + if (!debug_server) { + return nullptr; + } + +#if defined(IREE_DEBUG_EMBEDDED_APP_PRESENT) + // If the embedded debug UI is present then we can launch that now. + std::unique_ptr debugger; + if (absl::GetFlag(FLAGS_iree_attach_debugger)) { + LOG(INFO) << "Attaching debugger at startup..."; + ASSIGN_OR_RETURN( + debugger, + AttachDebugger(absl::StrCat( + "localhost:", absl::GetFlag(FLAGS_iree_debug_service_port)))); + RETURN_IF_ERROR(debug_server->WaitUntilSessionReady()); + LOG(INFO) << "Debugger attached"; + // TODO(benvanik): C++14 to avoid this. + auto debugger_baton = MoveToLambda(debugger); + debug_server->AtExit([debugger_baton]() { debugger_baton.value.reset(); }); + } +#else + if (absl::GetFlag(FLAGS_iree_attach_debugger)) { + LOG(WARNING) << "--iree_attach_debugger specified but no embedded debugger " + "is present. Build with --define=IREE_DEBUG=1."; + } +#endif // IREE_DEBUG_EMBEDDED_APP_PRESENT + + // Wait for a debugger to connect. + if (absl::GetFlag(FLAGS_iree_wait_for_debugger)) { + LOG(INFO) << "Waiting for a debugger to connect..."; + RETURN_IF_ERROR(debug_server->WaitUntilSessionReady()); + LOG(INFO) << "Debugger ready, resuming..."; + } + + return std::move(debug_server); +} + +} // namespace debug +} // namespace vm +} // namespace iree diff --git a/vm/debug/debug_server_flags.h b/vm/debug/debug_server_flags.h new file mode 100644 index 000000000000..707828cdf04b --- /dev/null +++ b/vm/debug/debug_server_flags.h @@ -0,0 +1,33 @@ +// Copyright 2019 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#ifndef THIRD_PARTY_MLIR_EDGE_IREE_VM_DEBUG_DEBUG_SERVER_FLAGS_H_ +#define THIRD_PARTY_MLIR_EDGE_IREE_VM_DEBUG_DEBUG_SERVER_FLAGS_H_ + +#include "third_party/mlir_edge/iree/base/status.h" +#include "third_party/mlir_edge/iree/vm/debug/debug_server.h" + +namespace iree { +namespace vm { +namespace debug { + +// Creates a debug server based on the current --iree_* debug flags. +// Returns nullptr if no server is compiled in or the flags are not set. +StatusOr> CreateDebugServerFromFlags(); + +} // namespace debug +} // namespace vm +} // namespace iree + +#endif // THIRD_PARTY_MLIR_EDGE_IREE_VM_DEBUG_DEBUG_SERVER_FLAGS_H_ diff --git a/vm/debug/debug_server_tcp.cc b/vm/debug/debug_server_tcp.cc new file mode 100644 index 000000000000..41905afbb689 --- /dev/null +++ b/vm/debug/debug_server_tcp.cc @@ -0,0 +1,462 @@ +// Copyright 2019 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include +#include +#include +#include +#include + +#include +#include +#include +#include // NOLINT + +#include "third_party/absl/base/thread_annotations.h" +#include "third_party/absl/memory/memory.h" +#include "third_party/absl/synchronization/mutex.h" +#include "third_party/flatbuffers/include/flatbuffers/flatbuffers.h" +#include "third_party/mlir_edge/iree/base/status.h" +#include "third_party/mlir_edge/iree/schemas/debug_service_generated.h" +#include "third_party/mlir_edge/iree/vm/debug/debug_server.h" +#include "third_party/mlir_edge/iree/vm/debug/debug_service.h" +#include "third_party/mlir_edge/iree/vm/debug/debug_tcp_util.h" + +namespace iree { +namespace vm { +namespace debug { +namespace { + +using ::iree::vm::Module; + +// Writes the given typed response message to the given fd by wrapping it in +// a size-prefixed rpc::Request union. +// +// Example: +// ::flatbuffers::FlatBufferBuilder fbb; +// rpc::SuspendFiberResponseBuilder response(fbb); +// RETURN_IF_ERROR(WriteResponse(fd_, response.Finish(), std::move(fbb))); +template +Status WriteResponse(int fd, ::flatbuffers::Offset message_offs, + ::flatbuffers::FlatBufferBuilder fbb) { + rpc::ResponseBuilder response_builder(fbb); + response_builder.add_message_type(rpc::ResponseUnionTraits::enum_value); + response_builder.add_message(message_offs.Union()); + auto response_offs = response_builder.Finish(); + rpc::ServicePacketBuilder packet_builder(fbb); + packet_builder.add_response(response_offs); + fbb.FinishSizePrefixed(packet_builder.Finish()); + return tcp::WriteBuffer(fd, fbb.Release()); +} + +class TcpDebugSession : public DebugSession { + public: + using ClosedCallback = + std::function; + + static StatusOr> Accept( + DebugService* debug_service, int client_fd, + ClosedCallback closed_callback) { + VLOG(2) << "Client " << client_fd << ": Setting up socket options..."; + // Disable Nagel's algorithm to ensure we have low latency. + RETURN_IF_ERROR(tcp::ToggleSocketNagelsAlgorithm(client_fd, false)); + // Enable keepalive assuming the client is local and this high freq is ok. + RETURN_IF_ERROR(tcp::ToggleSocketLocalKeepalive(client_fd, true)); + // Linger around for a bit to flush all data. + RETURN_IF_ERROR(tcp::ToggleSocketLinger(client_fd, true)); + + return absl::make_unique(debug_service, client_fd, + std::move(closed_callback)); + } + + TcpDebugSession(DebugService* debug_service, int client_fd, + ClosedCallback closed_callback) + : debug_service_(debug_service), + client_fd_(client_fd), + closed_callback_(std::move(closed_callback)) { + CHECK_OK(debug_service_->RegisterDebugSession(this)); + session_thread_ = std::thread([this]() { SessionThread(); }); + } + + ~TcpDebugSession() override { + CHECK_OK(debug_service_->UnregisterDebugSession(this)); + VLOG(2) << "Client " << client_fd_ << ": Shutting down session socket..."; + ::shutdown(client_fd_, SHUT_RD); + if (session_thread_.joinable() && + session_thread_.get_id() != std::this_thread::get_id()) { + VLOG(2) << "Client " << client_fd_ << ": Joining socket thread..."; + session_thread_.join(); + VLOG(2) << "Client " << client_fd_ << ": Joined socket thread!"; + } else { + VLOG(2) << "Client " << client_fd_ << ": Detaching socket thread..."; + session_thread_.detach(); + } + VLOG(2) << "Client " << client_fd_ << ": Closing session socket..."; + ::close(client_fd_); + VLOG(2) << "Client " << client_fd_ << ": Closed session socket!"; + client_fd_ = -1; + } + + Status OnServiceShutdown() { + VLOG(2) << "Client " << client_fd_ << ": Post OnServiceShutdown()"; + ::flatbuffers::FlatBufferBuilder fbb; + rpc::ServiceShutdownEventBuilder event(fbb); + return PostEvent(event.Finish(), std::move(fbb)); + } + + Status OnContextRegistered(SequencerContext* context) override { + VLOG(2) << "Client " << client_fd_ << ": Post OnContextRegistered(" + << context->id() << ")"; + ::flatbuffers::FlatBufferBuilder fbb; + rpc::ContextRegisteredEventBuilder event(fbb); + event.add_context_id(context->id()); + return PostEvent(event.Finish(), std::move(fbb)); + } + Status OnContextUnregistered(SequencerContext* context) override { + VLOG(2) << "Client " << client_fd_ << ": Post OnContextUnregistered(" + << context->id() << ")"; + ::flatbuffers::FlatBufferBuilder fbb; + rpc::ContextUnregisteredEventBuilder event(fbb); + event.add_context_id(context->id()); + return PostEvent(event.Finish(), std::move(fbb)); + } + + Status OnModuleLoaded(SequencerContext* context, Module* module) override { + VLOG(2) << "Client " << client_fd_ << ": Post OnModuleLoaded(" + << context->id() << ", " << module->name() << ")"; + ::flatbuffers::FlatBufferBuilder fbb; + auto module_name_offs = + fbb.CreateString(module->name().data(), module->name().size()); + rpc::ModuleLoadedEventBuilder event(fbb); + event.add_context_id(context->id()); + event.add_module_name(module_name_offs); + return PostEvent(event.Finish(), std::move(fbb)); + } + + Status OnFiberRegistered(FiberState* fiber_state) override { + VLOG(2) << "Client " << client_fd_ << ": Post OnFiberRegistered(" + << fiber_state->id() << ")"; + ::flatbuffers::FlatBufferBuilder fbb; + rpc::FiberRegisteredEventBuilder event(fbb); + event.add_fiber_id(fiber_state->id()); + return PostEvent(event.Finish(), std::move(fbb)); + } + Status OnFiberUnregistered(FiberState* fiber_state) override { + VLOG(2) << "Client " << client_fd_ << ": Post OnFiberUnregistered(" + << fiber_state->id() << ")"; + ::flatbuffers::FlatBufferBuilder fbb; + rpc::FiberUnregisteredEventBuilder event(fbb); + event.add_fiber_id(fiber_state->id()); + return PostEvent(event.Finish(), std::move(fbb)); + } + + Status OnBreakpointResolved(const rpc::BreakpointDefT& breakpoint, + SequencerContext* context) override { + VLOG(2) << "Client " << client_fd_ << ": Post OnBreakpointResolved(" + << breakpoint.breakpoint_id << ", " << context->id() << ", " + << breakpoint.function_ordinal << ")"; + rpc::BreakpointResolvedEventT event; + event.breakpoint = absl::make_unique(); + *event.breakpoint = breakpoint; + event.context_id = context->id(); + ::flatbuffers::FlatBufferBuilder fbb; + return PostEvent(rpc::BreakpointResolvedEvent::Pack(fbb, &event), + std::move(fbb)); + } + + Status OnBreakpointHit(int breakpoint_id, + const FiberState& fiber_state) override { + VLOG(2) << "Client " << client_fd_ << ": Post OnBreakpointHit(" + << breakpoint_id << ", " << fiber_state.id() << ")"; + ::flatbuffers::FlatBufferBuilder fbb; + ASSIGN_OR_RETURN(auto fiber_state_offs, + debug_service_->SerializeFiberState(fiber_state, &fbb)); + rpc::BreakpointHitEventBuilder event(fbb); + event.add_breakpoint_id(breakpoint_id); + event.add_fiber_state(fiber_state_offs); + return PostEvent(event.Finish(), std::move(fbb)); + } + + private: + void SessionThread() { + VLOG(2) << "Client " << client_fd_ << ": Thread entry"; + Status session_status = OkStatus(); + while (session_status.ok()) { + auto buffer_or = tcp::ReadBuffer(client_fd_); + if (!buffer_or.ok()) { + if (IsCancelled(buffer_or.status())) { + // Graceful shutdown. + VLOG(2) << "Client " << client_fd_ << ": Graceful shutdown requested"; + break; + } + // Error reading. + session_status = std::move(buffer_or).status(); + LOG(ERROR) << "Client " << client_fd_ + << ": Error reading request buffer: " << session_status; + break; + } + auto request_buffer = std::move(buffer_or).ValueOrDie(); + session_status = DispatchRequest(request_buffer.GetRoot()); + if (!session_status.ok()) { + LOG(ERROR) << "Client " << client_fd_ + << ": Error dispatching request: " << session_status; + break; + } + } + VLOG(2) << "Client " << client_fd_ << ": Thread exit"; + AbortSession(session_status); + } + + void AbortSession(Status status) { + if (status.ok()) { + VLOG(2) << "Debug client disconnected"; + } else { + LOG(ERROR) << "Debug session aborted; " << status; + ::flatbuffers::FlatBufferBuilder fbb; + auto message_offs = + fbb.CreateString(status.message().data(), status.message().size()); + rpc::StatusBuilder status_builder(fbb); + status_builder.add_code(static_cast(status.code())); + status_builder.add_message(message_offs); + auto status_offs = status_builder.Finish(); + rpc::ResponseBuilder response(fbb); + response.add_status(status_offs); + fbb.FinishSizePrefixed(response.Finish()); + tcp::WriteBuffer(client_fd_, fbb.Release()).IgnoreError(); + } + closed_callback_(this, std::move(status)); + } + + template + Status PostEvent(::flatbuffers::Offset event_offs, + ::flatbuffers::FlatBufferBuilder fbb) { + rpc::ServicePacketBuilder packet_builder(fbb); + packet_builder.add_event_type(rpc::EventUnionTraits::enum_value); + packet_builder.add_event(event_offs.Union()); + fbb.FinishSizePrefixed(packet_builder.Finish()); + return tcp::WriteBuffer(client_fd_, fbb.Release()); + } + + Status DispatchRequest(const rpc::Request& request) { + ::flatbuffers::FlatBufferBuilder fbb; + switch (request.message_type()) { +#define DISPATCH_REQUEST(method_name) \ + case rpc::RequestUnion::method_name##Request: { \ + VLOG(2) << "Client " << client_fd_ \ + << ": DispatchRequest(" #method_name ")..."; \ + ASSIGN_OR_RETURN(auto response_offs, \ + debug_service_->method_name( \ + *request.message_as_##method_name##Request(), &fbb)); \ + return WriteResponse(client_fd_, response_offs, std::move(fbb)); \ + } + DISPATCH_REQUEST(MakeReady); + DISPATCH_REQUEST(GetStatus); + DISPATCH_REQUEST(ListContexts); + DISPATCH_REQUEST(GetModule); + DISPATCH_REQUEST(GetFunction); + DISPATCH_REQUEST(ListFibers); + DISPATCH_REQUEST(SuspendFibers); + DISPATCH_REQUEST(ResumeFibers); + DISPATCH_REQUEST(StepFiber); + DISPATCH_REQUEST(GetFiberLocal); + DISPATCH_REQUEST(SetFiberLocal); + DISPATCH_REQUEST(ListBreakpoints); + DISPATCH_REQUEST(AddBreakpoint); + DISPATCH_REQUEST(RemoveBreakpoint); + DISPATCH_REQUEST(StartProfiling); + DISPATCH_REQUEST(StopProfiling); + default: + return UnimplementedErrorBuilder(ABSL_LOC) + << "Unimplemented debug service request: " + << static_cast(request.message_type()); + } + } + + DebugService* debug_service_; + int client_fd_; + ClosedCallback closed_callback_; + std::thread session_thread_; +}; + +class TcpDebugServer final : public DebugServer { + public: + static StatusOr> Listen(int port) { + // We support both IPv4 and IPv6 by using the IN6ADDR_ANY. This requires + // that we setup the socket as INET6 and enable reuse (so the same port can + // be bound for both IPv4 and IPv6). + int listen_fd = ::socket(AF_INET6, SOCK_STREAM, 0); + RETURN_IF_ERROR(tcp::ToggleSocketAddressReuse(listen_fd, true)); + + struct sockaddr_in6 socket_addr = {0}; + socket_addr.sin6_family = AF_INET6; + socket_addr.sin6_port = htons(port); + socket_addr.sin6_addr = in6addr_any; + if (::bind(listen_fd, reinterpret_cast(&socket_addr), + sizeof(socket_addr)) < 0) { + return AlreadyExistsErrorBuilder(ABSL_LOC) + << "Unable to bind socket to port " << port << ": (" << errno + << ") " << ::strerror(errno); + } + if (::listen(listen_fd, 1)) { + ::close(listen_fd); + return AlreadyExistsErrorBuilder(ABSL_LOC) + << "Unable to listen on port " << port << ": (" << errno << ") " + << ::strerror(errno); + } + return absl::make_unique(listen_fd); + } + + TcpDebugServer(int listen_fd) : listen_fd_(listen_fd) { + server_thread_ = std::thread([this]() { ListenThread(); }); + } + + ~TcpDebugServer() ABSL_LOCKS_EXCLUDED(mutex_) override { + absl::ReleasableMutexLock lock(&mutex_); + LOG(INFO) << "Shutting down debug server..."; + + // Notify all sessions. + for (auto& session : sessions_) { + session->OnServiceShutdown().IgnoreError(); + } + + // Shut down listen socket first so that we can't accept new connections. + VLOG(2) << "Shutting down listen socket..."; + ::shutdown(listen_fd_, SHUT_RDWR); + if (server_thread_.joinable()) { + VLOG(2) << "Joining listen thread..."; + server_thread_.join(); + VLOG(2) << "Joined listen thread!"; + } + VLOG(2) << "Closing listen socket..."; + ::close(listen_fd_); + listen_fd_ = -1; + VLOG(2) << "Closed listen socket!"; + + // Kill all active sessions. Note that we must do this outside of our lock. + std::vector> sessions = + std::move(sessions_); + std::vector> at_exit_callbacks = + std::move(at_exit_callbacks_); + lock.Release(); + VLOG(2) << "Clearing live sessions..."; + sessions.clear(); + VLOG(2) << "Calling AtExit callbacks..."; + for (auto& callback : at_exit_callbacks) { + callback(); + } + LOG(INFO) << "Debug server shutdown!"; + } + + DebugService* debug_service() { return &debug_service_; } + + Status AcceptNewSession(int client_fd) { + LOG(INFO) << "Accepting new client session as " << client_fd; + ASSIGN_OR_RETURN(auto session, + TcpDebugSession::Accept( + &debug_service_, client_fd, + [this](TcpDebugSession* session, Status status) { + absl::MutexLock lock(&mutex_); + for (auto it = sessions_.begin(); + it != sessions_.end(); ++it) { + if (it->get() == session) { + sessions_.erase(it); + break; + } + } + return OkStatus(); + })); + + absl::MutexLock lock(&mutex_); + sessions_.push_back(std::move(session)); + return OkStatus(); + } + + void AtExit(std::function callback) override { + absl::MutexLock lock(&mutex_); + at_exit_callbacks_.push_back(std::move(callback)); + } + + Status WaitUntilSessionReady() override { + return debug_service_.WaitUntilAllSessionsReady(); + } + + protected: + Status RegisterContext(SequencerContext* context) override { + return debug_service_.RegisterContext(context); + } + Status UnregisterContext(SequencerContext* context) override { + return debug_service_.UnregisterContext(context); + } + Status RegisterContextModule(SequencerContext* context, + Module* module) override { + return debug_service_.RegisterContextModule(context, module); + } + Status RegisterFiberState(FiberState* fiber_state) override { + return debug_service_.RegisterFiberState(fiber_state); + } + Status UnregisterFiberState(FiberState* fiber_state) override { + return debug_service_.UnregisterFiberState(fiber_state); + } + + private: + void ListenThread() { + VLOG(2) << "Listen thread entry"; + while (true) { + struct sockaddr_in accept_socket_addr; + socklen_t accept_socket_addr_length = sizeof(accept_socket_addr); + int accepted_fd = ::accept( + listen_fd_, reinterpret_cast(&accept_socket_addr), + &accept_socket_addr_length); + if (accepted_fd < 0) { + if (errno == EINVAL) { + // Shutting down gracefully. + break; + } + // We may be able to recover from some of these cases, but... shrug. + LOG(FATAL) << "Failed to accept client socket: (" << errno << ") " + << ::strerror(errno); + break; + } + auto accept_status = AcceptNewSession(accepted_fd); + if (!accept_status.ok()) { + LOG(ERROR) << "Failed to accept incoming debug client: " + << accept_status; + } + } + VLOG(2) << "Listen thread exit"; + } + + int listen_fd_; + std::thread server_thread_; + + absl::Mutex mutex_; + std::vector> sessions_ + ABSL_GUARDED_BY(mutex_); + std::vector> at_exit_callbacks_ ABSL_GUARDED_BY(mutex_); + + DebugService debug_service_; +}; + +} // namespace + +// static +StatusOr> DebugServer::Create(int listen_port) { + ASSIGN_OR_RETURN(auto debug_server, TcpDebugServer::Listen(listen_port)); + LOG(INFO) << "Debug server listening on localhost:" << listen_port; + return debug_server; +} + +} // namespace debug +} // namespace vm +} // namespace iree diff --git a/vm/debug/debug_service.cc b/vm/debug/debug_service.cc new file mode 100644 index 000000000000..1013b365bc4b --- /dev/null +++ b/vm/debug/debug_service.cc @@ -0,0 +1,854 @@ +// Copyright 2019 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "third_party/mlir_edge/iree/vm/debug/debug_service.h" + +#include +#include + +#include "third_party/absl/strings/str_join.h" +#include "third_party/absl/synchronization/mutex.h" +#include "third_party/absl/types/source_location.h" +#include "third_party/flatbuffers/include/flatbuffers/flatbuffers.h" +#include "third_party/flatbuffers/include/flatbuffers/reflection.h" +#include "third_party/mlir_edge/iree/base/flatbuffer_util.h" +#include "third_party/mlir_edge/iree/base/status.h" +#include "third_party/mlir_edge/iree/schemas/debug_service_generated.h" +#include "third_party/mlir_edge/iree/schemas/reflection_data.h" +#include "third_party/mlir_edge/iree/vm/instance.h" + +namespace iree { +namespace vm { +namespace debug { +namespace { + +using ::flatbuffers::FlatBufferBuilder; +using ::flatbuffers::Offset; +using ::iree::hal::BufferView; +using ::iree::vm::Module; +using ::iree::vm::StackFrame; + +// Gets an embedded flatbuffers reflection schema. +const ::reflection::Schema& GetSchema(const char* schema_name) { + for (const auto* file_toc = schemas::reflection_data_create(); + file_toc != nullptr; ++file_toc) { + if (std::strcmp(file_toc->name, schema_name) == 0) { + return *::reflection::GetSchema(file_toc->data); + } + } + LOG(FATAL) << "FlatBuffer schema '" << schema_name + << "' not found in binary; ensure it is in :reflection_data"; +} + +// Recursively copies a flatbuffer table, returning the root offset in |fbb|. +template +StatusOr> DeepCopyTable(const char* schema_name, const T& table_def, + FlatBufferBuilder* fbb) { + const auto* root_table = + reinterpret_cast(std::addressof(table_def)); + const auto& schema = GetSchema(schema_name); + return {::flatbuffers::CopyTable(*fbb, schema, *schema.root_table(), + *root_table, + /*use_string_pooling=*/false) + .o}; +} + +// Serializes a buffer_view value, optionally including the entire buffer +// contents. +StatusOr> SerializeBufferView( + const BufferView& buffer_view, bool include_buffer_contents, + FlatBufferBuilder* fbb) { + auto shape_offs = fbb->CreateVector(buffer_view.shape.subspan().data(), + buffer_view.shape.subspan().size()); + rpc::BufferViewDefBuilder value(*fbb); + value.add_is_valid(buffer_view.buffer != nullptr); + value.add_shape(shape_offs); + value.add_element_size(buffer_view.element_size); + if (include_buffer_contents) { + // TODO(benvanik): add buffer data. + } + return value.Finish(); +} + +// Serializes a stack frame. +StatusOr> SerializeStackFrame( + const StackFrame& stack_frame, FlatBufferBuilder* fbb) { + ASSIGN_OR_RETURN(int function_ordinal, + stack_frame.module().function_table().LookupFunctionOrdinal( + stack_frame.function())); + auto module_name_offs = fbb->CreateString(stack_frame.module().name().data(), + stack_frame.module().name().size()); + std::vector> local_offs_list; + for (const auto& local : stack_frame.locals()) { + ASSIGN_OR_RETURN( + auto local_offs, + SerializeBufferView(local, /*include_buffer_contents=*/false, fbb)); + local_offs_list.push_back(local_offs); + } + auto locals_offs = fbb->CreateVector(local_offs_list); + rpc::StackFrameDefBuilder sfb(*fbb); + sfb.add_module_name(module_name_offs); + sfb.add_function_ordinal(function_ordinal); + sfb.add_offset(stack_frame.offset()); + sfb.add_locals(locals_offs); + return sfb.Finish(); +} + +// Resolves a local from a fiber:frame:local_index to a BufferView. +StatusOr ResolveFiberLocal(FiberState* fiber_state, + int frame_index, int local_index) { + auto frames = fiber_state->mutable_stack()->mutable_frames(); + if (frame_index < 0 || frame_index > frames.size()) { + return InvalidArgumentErrorBuilder(ABSL_LOC) + << "Frame index " << frame_index << " out of bounds (" + << frames.size() << ")"; + } + auto locals = frames[frame_index].mutable_locals(); + if (local_index < 0 || local_index > locals.size()) { + return InvalidArgumentErrorBuilder(ABSL_LOC) + << "Local index " << local_index << " out of bounds (" + << locals.size() << ")"; + } + return &locals[local_index]; +} + +// Suspends a set of fibers and blocks until all have been suspended (or one or +// more fails to suspend). +// This works only when the caller is *not* one of the threads executing a +// fiber in |fiber_states| (this normally shouldn't happen, but may if we +// support eval()-like semantics). +Status SuspendFibersAndWait(absl::Span fiber_states) { + absl::Mutex suspend_mutex; + Status one_suspend_status = OkStatus(); + std::list pending_suspend_ids; + for (auto* fiber_state : fiber_states) { + pending_suspend_ids.push_back(fiber_state->id()); + } + for (auto* fiber_state : fiber_states) { + auto suspend_callback = [&, fiber_state](Status suspend_status) { + absl::MutexLock lock(&suspend_mutex); + auto it = std::find(pending_suspend_ids.begin(), + pending_suspend_ids.end(), fiber_state->id()); + CHECK(it != pending_suspend_ids.end()); + pending_suspend_ids.erase(it); + if (!suspend_status.ok()) { + one_suspend_status = std::move(suspend_status); + } + }; + RETURN_IF_ERROR(fiber_state->Suspend(suspend_callback)); + } + suspend_mutex.LockWhen(absl::Condition( + +[](std::list* pending_suspend_ids) { + return pending_suspend_ids->empty(); + }, + &pending_suspend_ids)); + suspend_mutex.Unlock(); + return one_suspend_status; +} + +} // namespace + +Status DebugService::SuspendAllFibers() { + VLOG(2) << "SuspendAllFibers"; + for (auto* fiber_state : fiber_states_) { + RETURN_IF_ERROR(fiber_state->Suspend()); + } + return OkStatus(); +} + +Status DebugService::ResumeAllFibers() { + VLOG(2) << "ResumeAllFibers"; + for (auto* fiber_state : fiber_states_) { + RETURN_IF_ERROR(fiber_state->Resume()); + } + return OkStatus(); +} + +Status DebugService::RegisterContext(SequencerContext* context) { + absl::MutexLock lock(&mutex_); + VLOG(2) << "RegisterContext(" << context->id() << ")"; + RETURN_IF_ERROR(SuspendAllFibers()); + RETURN_IF_ERROR(UnreadyAllSessions()); + contexts_.push_back(context); + for (auto* session : sessions_) { + RETURN_IF_ERROR(session->OnContextRegistered(context)); + } + RETURN_IF_ERROR(ResumeAllFibers()); + return OkStatus(); +} + +Status DebugService::UnregisterContext(SequencerContext* context) { + absl::MutexLock lock(&mutex_); + VLOG(2) << "UnregisterContext(" << context->id() << ")"; + auto it = std::find(contexts_.begin(), contexts_.end(), context); + if (it == contexts_.end()) { + return NotFoundErrorBuilder(ABSL_LOC) << "Context not registered"; + } + RETURN_IF_ERROR(SuspendAllFibers()); + RETURN_IF_ERROR(UnreadyAllSessions()); + for (auto* session : sessions_) { + RETURN_IF_ERROR(session->OnContextUnregistered(context)); + } + contexts_.erase(it); + RETURN_IF_ERROR(ResumeAllFibers()); + return OkStatus(); +} + +StatusOr DebugService::GetContext(int context_id) const { + for (auto* context : contexts_) { + if (context->id() == context_id) { + return context; + } + } + return NotFoundErrorBuilder(ABSL_LOC) + << "Context with ID " << context_id + << " not registered with the debug service"; +} + +Status DebugService::RegisterContextModule(SequencerContext* context, + Module* module) { + absl::MutexLock lock(&mutex_); + VLOG(2) << "RegisterContextModule(" << context->id() << ", " << module->name() + << ")"; + RETURN_IF_ERROR(SuspendAllFibers()); + RETURN_IF_ERROR(UnreadyAllSessions()); + RETURN_IF_ERROR(RegisterModuleBreakpoints(context, module)); + for (auto* session : sessions_) { + RETURN_IF_ERROR(session->OnModuleLoaded(context, module)); + } + RETURN_IF_ERROR(ResumeAllFibers()); + return OkStatus(); +} + +StatusOr DebugService::GetModule(int context_id, + absl::string_view module_name) const { + ASSIGN_OR_RETURN(auto* context, GetContext(context_id)); + for (const auto& module : context->modules()) { + if (module->name() == module_name) { + return module.get(); + } + } + return NotFoundErrorBuilder(ABSL_LOC) + << "Module '" << module_name << "' not found on context " + << context_id; +} + +Status DebugService::RegisterFiberState(FiberState* fiber_state) { + absl::MutexLock lock(&mutex_); + VLOG(2) << "RegisterFiberState(" << fiber_state->id() << ")"; + RETURN_IF_ERROR(SuspendAllFibers()); + RETURN_IF_ERROR(UnreadyAllSessions()); + fiber_states_.push_back(fiber_state); + if (sessions_unready_) { + // Suspend immediately as a debugger is not yet read. + RETURN_IF_ERROR(fiber_state->Suspend()); + } + for (auto* session : sessions_) { + RETURN_IF_ERROR(session->OnFiberRegistered(fiber_state)); + } + RETURN_IF_ERROR(ResumeAllFibers()); + return OkStatus(); +} + +Status DebugService::UnregisterFiberState(FiberState* fiber_state) { + absl::MutexLock lock(&mutex_); + VLOG(2) << "UnregisterFiberState(" << fiber_state->id() << ")"; + auto it = std::find(fiber_states_.begin(), fiber_states_.end(), fiber_state); + if (it == fiber_states_.end()) { + return NotFoundErrorBuilder(ABSL_LOC) << "Fiber state not registered"; + } + RETURN_IF_ERROR(SuspendAllFibers()); + RETURN_IF_ERROR(UnreadyAllSessions()); + for (auto* session : sessions_) { + RETURN_IF_ERROR(session->OnFiberUnregistered(fiber_state)); + } + fiber_states_.erase(it); + RETURN_IF_ERROR(ResumeAllFibers()); + return OkStatus(); +} + +StatusOr DebugService::GetFiberState(int fiber_id) const { + for (auto* fiber_state : fiber_states_) { + if (fiber_state->id() == fiber_id) { + return fiber_state; + } + } + return NotFoundErrorBuilder(ABSL_LOC) + << "Fiber state with ID " << fiber_id + << " not registered with the debug service"; +} + +StatusOr> DebugService::SerializeFiberState( + const FiberState& fiber_state, FlatBufferBuilder* fbb) { + std::vector> frame_offs_list; + for (const auto& frame : fiber_state.stack().frames()) { + ASSIGN_OR_RETURN(auto frame_offs, SerializeStackFrame(frame, fbb)); + frame_offs_list.push_back(frame_offs); + } + auto frames_offs = fbb->CreateVector(frame_offs_list); + rpc::FiberStateDefBuilder fsb(*fbb); + fsb.add_fiber_id(fiber_state.id()); + fsb.add_frames(frames_offs); + return fsb.Finish(); +} + +Status DebugService::RegisterDebugSession(DebugSession* session) { + absl::MutexLock lock(&mutex_); + VLOG(2) << "RegisterDebugSession(" << session->id() << ")"; + sessions_.push_back(session); + if (session->is_ready()) { + ++sessions_ready_; + } else { + // Immediately suspend all fibers until the session readies up (or + // disconnects). + ++sessions_unready_; + RETURN_IF_ERROR(SuspendAllFibers()); + } + return OkStatus(); +} + +Status DebugService::UnregisterDebugSession(DebugSession* session) { + absl::MutexLock lock(&mutex_); + VLOG(2) << "UnregisterDebugSession(" << session->id() << ")"; + auto it = std::find(sessions_.begin(), sessions_.end(), session); + if (it == sessions_.end()) { + return NotFoundErrorBuilder(ABSL_LOC) << "Session not registered"; + } + sessions_.erase(it); + if (session->is_ready()) { + --sessions_ready_; + } else { + // If the session never readied up then we still have all fibers suspended + // waiting for it. We should resume so that we don't block forever. + --sessions_unready_; + RETURN_IF_ERROR(ResumeAllFibers()); + } + return OkStatus(); +} + +Status DebugService::WaitUntilAllSessionsReady() { + VLOG(1) << "Waiting until all sessions are ready..."; + struct CondState { + DebugService* service; + bool had_sessions; + bool consider_aborted; + } cond_state; + { + absl::MutexLock lock(&mutex_); + cond_state.service = this; + cond_state.had_sessions = !sessions_.empty(); + cond_state.consider_aborted = false; + } + mutex_.LockWhen(absl::Condition( + +[](CondState* cond_state) { + cond_state->service->mutex_.AssertHeld(); + if (cond_state->service->sessions_ready_ > 0) { + // One or more sessions are ready. + return true; + } + if (cond_state->service->sessions_unready_ > 0) { + // One or more sessions are connected but not yet ready. + cond_state->had_sessions = true; + return false; + } + if (cond_state->had_sessions && + cond_state->service->sessions_.empty()) { + // We had sessions but now we don't, consider this an error and bail. + // This can happen when a session connects but never readies up. + cond_state->consider_aborted = true; + return true; + } + return false; + }, + &cond_state)); + mutex_.Unlock(); + if (cond_state.consider_aborted) { + return AbortedErrorBuilder(ABSL_LOC) + << "At least one session connected but never readied up"; + } + VLOG(1) << "Sessions ready, resuming"; + return OkStatus(); +} + +StatusOr> DebugService::MakeReady( + const rpc::MakeReadyRequest& request, FlatBufferBuilder* fbb) { + absl::MutexLock lock(&mutex_); + VLOG(1) << "RPC: MakeReady()"; + // TODO(benvanik): support more than one session. + CHECK_LE(sessions_.size(), 1) << "Only one session is currently supported"; + if (!sessions_.empty()) { + RETURN_IF_ERROR(sessions_[0]->OnReady()); + } + sessions_ready_ = 0; + sessions_unready_ = 0; + for (auto* session : sessions_) { + sessions_ready_ += session->is_ready() ? 1 : 0; + sessions_unready_ += session->is_ready() ? 0 : 1; + } + rpc::MakeReadyResponseBuilder response(*fbb); + return response.Finish(); +} + +Status DebugService::UnreadyAllSessions() { + for (auto* session : sessions_) { + RETURN_IF_ERROR(session->OnUnready()); + } + sessions_ready_ = 0; + sessions_unready_ = sessions_.size(); + return OkStatus(); +} + +StatusOr> DebugService::GetStatus( + const rpc::GetStatusRequest& request, FlatBufferBuilder* fbb) { + absl::MutexLock lock(&mutex_); + VLOG(1) << "RPC: GetStatus()"; + rpc::GetStatusResponseBuilder response(*fbb); + response.add_protocol(0); + return response.Finish(); +} + +StatusOr> DebugService::ListContexts( + const rpc::ListContextsRequest& request, FlatBufferBuilder* fbb) { + absl::MutexLock lock(&mutex_); + VLOG(1) << "RPC: ListContexts()"; + std::vector> context_offs; + for (auto* context : contexts_) { + std::vector> native_function_offs_list; + for (const auto& pair : context->native_functions()) { + auto name_offs = fbb->CreateString(pair.first); + rpc::NativeFunctionDefBuilder native_function(*fbb); + native_function.add_name(name_offs); + native_function_offs_list.push_back(native_function.Finish()); + } + auto native_functions_offs = fbb->CreateVector(native_function_offs_list); + + std::vector module_names; + for (const auto& module : context->modules()) { + module_names.push_back(std::string(module->name())); + } + auto module_names_offs = fbb->CreateVectorOfStrings(module_names); + + rpc::ContextDefBuilder context_def(*fbb); + context_def.add_context_id(context->id()); + context_def.add_native_functions(native_functions_offs); + context_def.add_module_names(module_names_offs); + context_offs.push_back(context_def.Finish()); + } + + auto contexts_offs = fbb->CreateVector(context_offs); + rpc::ListContextsResponseBuilder response(*fbb); + response.add_contexts(contexts_offs); + return response.Finish(); +} + +StatusOr> DebugService::GetModule( + const rpc::GetModuleRequest& request, FlatBufferBuilder* fbb) { + absl::MutexLock lock(&mutex_); + VLOG(1) << "RPC: GetModule(" << request.context_id() << ", " + << WrapString(request.module_name()) << ")"; + ASSIGN_OR_RETURN(auto* module, GetModule(request.context_id(), + WrapString(request.module_name()))); + // TODO(benvanik): find a way to do this without possibly duping all memory. + // I suspect that when we make constants poolable then there's only one + // place to kill and there may be magic we could use to do that during a + // reflection pass. + ModuleDefT module_t; + module->def().UnPackTo(&module_t); + for (auto& function : module_t.function_table->functions) { + function->bytecode->contents.clear(); + } + auto trimmed_module_offs = ModuleDef::Pack(*fbb, &module_t); + rpc::GetModuleResponseBuilder response(*fbb); + response.add_module_(trimmed_module_offs); + return response.Finish(); +} + +StatusOr> DebugService::GetFunction( + const rpc::GetFunctionRequest& request, FlatBufferBuilder* fbb) { + absl::MutexLock lock(&mutex_); + VLOG(1) << "RPC: GetFunction(" << WrapString(request.module_name()) << ", " + << request.function_ordinal() << ")"; + ASSIGN_OR_RETURN(auto* module, GetModule(request.context_id(), + WrapString(request.module_name()))); + ASSIGN_OR_RETURN(auto& function, module->function_table().LookupFunction( + request.function_ordinal())); + Offset bytecode_offs; + if (function.def().bytecode()) { + ASSIGN_OR_RETURN( + bytecode_offs, + DeepCopyTable("bytecode_def.bfbs", *function.def().bytecode(), fbb)); + } + rpc::GetFunctionResponseBuilder response(*fbb); + response.add_bytecode(bytecode_offs); + return response.Finish(); +} + +StatusOr> DebugService::ResolveFunction( + const rpc::ResolveFunctionRequest& request, FlatBufferBuilder* fbb) { + absl::MutexLock lock(&mutex_); + VLOG(1) << "RPC: ResolveFunction(" << WrapString(request.module_name()) + << ", " << WrapString(request.function_name()) << ")"; + std::vector context_ids; + auto context_ids_offs = fbb->CreateVector(context_ids); + int function_ordinal = -1; + for (auto* context : contexts_) { + for (const auto& module : context->modules()) { + if (module->name() == WrapString(request.module_name())) { + ASSIGN_OR_RETURN(function_ordinal, + module->function_table().LookupFunctionOrdinalByName( + WrapString(request.function_name()))); + context_ids.push_back(context->id()); + break; + } + } + } + rpc::ResolveFunctionResponseBuilder response(*fbb); + response.add_context_ids(context_ids_offs); + response.add_function_ordinal(function_ordinal); + return response.Finish(); +} + +StatusOr> DebugService::ListFibers( + const rpc::ListFibersRequest& request, FlatBufferBuilder* fbb) { + absl::MutexLock lock(&mutex_); + VLOG(1) << "RPC: ListFibers()"; + std::vector> fiber_state_offsets; + for (auto* fiber_state : fiber_states_) { + ASSIGN_OR_RETURN(auto fiber_state_offs, + SerializeFiberState(*fiber_state, fbb)); + fiber_state_offsets.push_back(fiber_state_offs); + } + auto fiber_states_offs = fbb->CreateVector(fiber_state_offsets); + rpc::ListFibersResponseBuilder response(*fbb); + response.add_fiber_states(fiber_states_offs); + return response.Finish(); +} + +StatusOr> DebugService::SuspendFibers( + const rpc::SuspendFibersRequest& request, FlatBufferBuilder* fbb) { + absl::MutexLock lock(&mutex_); + VLOG(1) << "RPC: SuspendFibers(fiber_ids=[" + << (request.fiber_ids() ? absl::StrJoin(*request.fiber_ids(), ", ") + : "") + << "])"; + std::vector> fiber_state_offsets; + if (request.fiber_ids() && request.fiber_ids()->size() > 0) { + // Suspending a list of fibers. + std::vector fibers_to_suspend; + for (int fiber_id : *request.fiber_ids()) { + ASSIGN_OR_RETURN(auto* fiber_state, GetFiberState(fiber_id)); + fibers_to_suspend.push_back(fiber_state); + } + RETURN_IF_ERROR(SuspendFibersAndWait(absl::MakeSpan(fibers_to_suspend))); + for (auto* fiber_state : fibers_to_suspend) { + ASSIGN_OR_RETURN(auto fiber_state_offs, + SerializeFiberState(*fiber_state, fbb)); + fiber_state_offsets.push_back(fiber_state_offs); + } + } else { + // Suspending all fibers. + RETURN_IF_ERROR(SuspendAllFibers()); + for (auto* fiber_state : fiber_states_) { + ASSIGN_OR_RETURN(auto fiber_state_offs, + SerializeFiberState(*fiber_state, fbb)); + fiber_state_offsets.push_back(fiber_state_offs); + } + } + auto fiber_states_offs = fbb->CreateVector(fiber_state_offsets); + rpc::SuspendFibersResponseBuilder response(*fbb); + response.add_fiber_states(fiber_states_offs); + return response.Finish(); +} + +StatusOr> DebugService::ResumeFibers( + const rpc::ResumeFibersRequest& request, FlatBufferBuilder* fbb) { + VLOG(1) << "RPC: ResumeFibers(fiber_ids=[" + << (request.fiber_ids() ? absl::StrJoin(*request.fiber_ids(), ", ") + : "") + << "])"; + absl::MutexLock lock(&mutex_); + if (request.fiber_ids() && request.fiber_ids()->size() > 0) { + // Resuming a list of fibers. + for (int fiber_id : *request.fiber_ids()) { + ASSIGN_OR_RETURN(auto* fiber_state, GetFiberState(fiber_id)); + RETURN_IF_ERROR(fiber_state->Resume()); + } + } else { + // Resuming all fibers. + RETURN_IF_ERROR(ResumeAllFibers()); + } + rpc::ResumeFibersResponseBuilder response(*fbb); + return response.Finish(); +} + +StatusOr> DebugService::StepFiber( + const rpc::StepFiberRequest& request, FlatBufferBuilder* fbb) { + absl::MutexLock lock(&mutex_); + VLOG(1) << "RPC: StepFiber(" << request.fiber_id() << ")"; + ASSIGN_OR_RETURN(auto* fiber_state, GetFiberState(request.fiber_id())); + FiberState::StepTarget step_target; + // TODO(benvanik): step settings. + RETURN_IF_ERROR(fiber_state->Step(step_target)); + rpc::StepFiberResponseBuilder response(*fbb); + return response.Finish(); +} + +StatusOr> DebugService::GetFiberLocal( + const rpc::GetFiberLocalRequest& request, FlatBufferBuilder* fbb) { + absl::MutexLock lock(&mutex_); + VLOG(1) << "RPC: GetFiberLocal(" << request.fiber_id() << ", " + << request.frame_index() << ", " << request.local_index() << ")"; + ASSIGN_OR_RETURN(auto* fiber_state, GetFiberState(request.fiber_id())); + ASSIGN_OR_RETURN(auto* local, + ResolveFiberLocal(fiber_state, request.frame_index(), + request.local_index())); + + ASSIGN_OR_RETURN( + auto value_offs, + SerializeBufferView(*local, /*include_buffer_contents=*/true, fbb)); + rpc::GetFiberLocalResponseBuilder response(*fbb); + response.add_value(value_offs); + return response.Finish(); +} + +StatusOr> DebugService::SetFiberLocal( + const rpc::SetFiberLocalRequest& request, FlatBufferBuilder* fbb) { + absl::MutexLock lock(&mutex_); + VLOG(1) << "RPC: SetFiberLocal(" << request.fiber_id() << ", " + << request.frame_index() << ", " << request.local_index() << ")"; + ASSIGN_OR_RETURN(auto* fiber_state, GetFiberState(request.fiber_id())); + ASSIGN_OR_RETURN(auto* local, + ResolveFiberLocal(fiber_state, request.frame_index(), + request.local_index())); + + if (!request.value()) { + local->shape.clear(); + local->element_size = 0; + local->buffer.reset(); + } else { + const auto& value = *request.value(); + local->shape.clear(); + if (value.shape()) { + for (int dim : *value.shape()) { + local->shape.push_back(dim); + } + } + local->element_size = value.element_size(); + // TODO(benvanik): copy buffer data. + } + + ASSIGN_OR_RETURN( + auto value_offs, + SerializeBufferView(*local, /*include_buffer_contents=*/true, fbb)); + rpc::SetFiberLocalResponseBuilder response(*fbb); + response.add_value(value_offs); + return response.Finish(); +} + +StatusOr> DebugService::ListBreakpoints( + const rpc::ListBreakpointsRequest& request, FlatBufferBuilder* fbb) { + absl::MutexLock lock(&mutex_); + VLOG(1) << "RPC: ListBreakpoints()"; + std::vector> breakpoint_offs; + for (const auto& breakpoint : breakpoints_) { + breakpoint_offs.push_back(rpc::BreakpointDef::Pack(*fbb, &breakpoint)); + } + auto breakpoints_offs = fbb->CreateVector(breakpoint_offs); + rpc::ListBreakpointsResponseBuilder response(*fbb); + response.add_breakpoints(breakpoints_offs); + return response.Finish(); +} + +StatusOr> DebugService::AddBreakpoint( + const rpc::AddBreakpointRequest& request, FlatBufferBuilder* fbb) { + if (!request.breakpoint()) { + return InvalidArgumentErrorBuilder(ABSL_LOC) << "No breakpoint specified"; + } + absl::MutexLock lock(&mutex_); + int breakpoint_id = Instance::NextUniqueId(); + VLOG(1) << "RPC: AddBreakpoint(" << breakpoint_id << ")"; + + RETURN_IF_ERROR(SuspendAllFibers()); + + rpc::BreakpointDefT breakpoint; + request.breakpoint()->UnPackTo(&breakpoint); + breakpoint.breakpoint_id = breakpoint_id; + switch (breakpoint.breakpoint_type) { + case rpc::BreakpointType::BYTECODE_FUNCTION: + case rpc::BreakpointType::NATIVE_FUNCTION: + for (auto* context : contexts_) { + auto module_or = context->LookupModule(breakpoint.module_name); + if (!module_or.ok()) continue; + auto* module = module_or.ValueOrDie(); + RETURN_IF_ERROR( + RegisterFunctionBreakpoint(context, module, &breakpoint)); + } + break; + default: + return UnimplementedErrorBuilder(ABSL_LOC) << "Unhandled breakpoint type"; + } + breakpoints_.push_back(std::move(breakpoint)); + + RETURN_IF_ERROR(ResumeAllFibers()); + + auto breakpoint_offs = rpc::BreakpointDef::Pack(*fbb, &breakpoints_.back()); + rpc::AddBreakpointResponseBuilder response(*fbb); + response.add_breakpoint(breakpoint_offs); + return response.Finish(); +} + +StatusOr> DebugService::RemoveBreakpoint( + const rpc::RemoveBreakpointRequest& request, FlatBufferBuilder* fbb) { + absl::MutexLock lock(&mutex_); + VLOG(1) << "RPC: RemoveBreakpoint(" << request.breakpoint_id() << ")"; + RETURN_IF_ERROR(SuspendAllFibers()); + + bool found = false; + for (auto it = breakpoints_.begin(); it != breakpoints_.end(); ++it) { + if (it->breakpoint_id == request.breakpoint_id()) { + auto& breakpoint = *it; + found = true; + switch (breakpoint.breakpoint_type) { + case rpc::BreakpointType::BYTECODE_FUNCTION: + case rpc::BreakpointType::NATIVE_FUNCTION: + RETURN_IF_ERROR(UnregisterFunctionBreakpoint(breakpoint)); + break; + default: + return UnimplementedErrorBuilder(ABSL_LOC) + << "Unhandled breakpoint type"; + } + breakpoints_.erase(it); + break; + } + } + + RETURN_IF_ERROR(ResumeAllFibers()); + if (!found) { + return InvalidArgumentErrorBuilder(ABSL_LOC) + << "Breakpoint ID " << request.breakpoint_id() << " not found"; + } + + rpc::RemoveBreakpointResponseBuilder response(*fbb); + return response.Finish(); +} + +Status DebugService::RegisterModuleBreakpoints(SequencerContext* context, + Module* module) { + for (auto& breakpoint : breakpoints_) { + switch (breakpoint.breakpoint_type) { + case rpc::BreakpointType::BYTECODE_FUNCTION: + if (breakpoint.module_name == module->name()) { + RETURN_IF_ERROR( + RegisterFunctionBreakpoint(context, module, &breakpoint)); + } + break; + default: + // Not relevant to modules. + break; + } + } + return OkStatus(); +} + +Status DebugService::RegisterFunctionBreakpoint( + SequencerContext* context, Module* module, + rpc::BreakpointDefT* breakpoint) { + if (!breakpoint->function_name.empty()) { + ASSIGN_OR_RETURN(breakpoint->function_ordinal, + module->function_table().LookupFunctionOrdinalByName( + breakpoint->function_name)); + } + RETURN_IF_ERROR(module->mutable_function_table()->RegisterBreakpoint( + breakpoint->function_ordinal, breakpoint->bytecode_offset, + std::bind(&DebugService::OnFunctionBreakpointHit, this, + breakpoint->breakpoint_id, std::placeholders::_1))); + for (auto* session : sessions_) { + RETURN_IF_ERROR(session->OnBreakpointResolved(*breakpoint, context)); + } + return OkStatus(); +} + +Status DebugService::UnregisterFunctionBreakpoint( + const rpc::BreakpointDefT& breakpoint) { + for (auto* context : contexts_) { + auto module_or = context->LookupModule(breakpoint.module_name); + if (!module_or.ok()) continue; + auto* module = module_or.ValueOrDie(); + RETURN_IF_ERROR(module->mutable_function_table()->UnregisterBreakpoint( + breakpoint.function_ordinal, breakpoint.bytecode_offset)); + } + return OkStatus(); +} + +Status DebugService::OnFunctionBreakpointHit(int breakpoint_id, + const vm::Stack& stack) { + absl::ReleasableMutexLock lock(&mutex_); + LOG(INFO) << "Breakpoint hit: " << breakpoint_id; + FiberState* source_fiber_state = nullptr; + for (auto* fiber_state : fiber_states_) { + if (fiber_state->mutable_stack() == std::addressof(stack)) { + source_fiber_state = fiber_state; + break; + } + } + if (!source_fiber_state) { + return InternalErrorBuilder(ABSL_LOC) + << "Fiber state not found for stack - race?"; + } + RETURN_IF_ERROR(UnreadyAllSessions()); + for (auto* session : sessions_) { + RETURN_IF_ERROR( + session->OnBreakpointHit(breakpoint_id, *source_fiber_state)); + } + lock.Release(); + + // TODO(benvanik): on-demand attach if desired? + + // Wait until all clients are ready. + auto wait_status = WaitUntilAllSessionsReady(); + if (IsAborted(wait_status)) { + // This means we lost all sessions. Just continue. + VLOG(1) << "No sessions active; ignoring breakpoint and continuing"; + return OkStatus(); + } + return wait_status; +} + +StatusOr> DebugService::StartProfiling( + const rpc::StartProfilingRequest& request, FlatBufferBuilder* fbb) { + absl::MutexLock lock(&mutex_); + VLOG(1) << "RPC: StartProfiling()"; + // TODO(benvanik): implement profiling. + // ASSIGN_OR_RETURN(auto* context, GetContext(request.context_id())); + // rpc::StartProfilingResponseBuilder response(*fbb); + // return response.Finish(); + return UnimplementedErrorBuilder(ABSL_LOC) + << "StartProfiling not yet implemented"; +} + +StatusOr> DebugService::StopProfiling( + const rpc::StopProfilingRequest& request, FlatBufferBuilder* fbb) { + absl::MutexLock lock(&mutex_); + VLOG(1) << "RPC: StopProfiling()"; + // TODO(benvanik): implement profiling. + // ASSIGN_OR_RETURN(auto* context, GetContext(request.context_id())); + // rpc::StopProfilingResponseBuilder response(*fbb); + // return response.Finish(); + return UnimplementedErrorBuilder(ABSL_LOC) + << "StopProfiling not yet implemented"; +} + +} // namespace debug +} // namespace vm +} // namespace iree diff --git a/vm/debug/debug_service.h b/vm/debug/debug_service.h new file mode 100644 index 000000000000..c8f74a2058bc --- /dev/null +++ b/vm/debug/debug_service.h @@ -0,0 +1,178 @@ +// Copyright 2019 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#ifndef THIRD_PARTY_MLIR_EDGE_IREE_VM_DEBUG_DEBUG_SERVICE_H_ +#define THIRD_PARTY_MLIR_EDGE_IREE_VM_DEBUG_DEBUG_SERVICE_H_ + +#include + +#include "third_party/absl/base/thread_annotations.h" +#include "third_party/absl/strings/string_view.h" +#include "third_party/absl/synchronization/mutex.h" +#include "third_party/flatbuffers/include/flatbuffers/flatbuffers.h" +#include "third_party/mlir_edge/iree/base/status.h" +#include "third_party/mlir_edge/iree/schemas/debug_service_generated.h" +#include "third_party/mlir_edge/iree/vm/debug/debug_session.h" +#include "third_party/mlir_edge/iree/vm/fiber_state.h" +#include "third_party/mlir_edge/iree/vm/sequencer_context.h" + +namespace iree { +namespace vm { +namespace debug { + +// Debugging service used to implement the DebugService RPC methods in a +// transport-independent way. Specific DebugServer implementations can compose +// with a DebugService to avoid needing to maintain state themselves. Multiple +// DebugServer instances could share the same DebugService instance to ensure +// all clients - regardless of transport - share the same state. +// +// Thread-safe. +class DebugService { + public: + // Registers a context with the debug service. + // Ownership remains with the caller and UnregisterContext must be called + // prior to the context being destroyed. + Status RegisterContext(SequencerContext* context); + Status UnregisterContext(SequencerContext* context); + + // Registers a new module linked into an existing Context. + Status RegisterContextModule(SequencerContext* context, vm::Module* module); + + // Registers a fiber state with the debug service. + // Ownership remains with the caller and UnregisterFiberState must be called + // prior to the fiber state being destroyed. + Status RegisterFiberState(FiberState* fiber_state); + Status UnregisterFiberState(FiberState* fiber_state); + + // Registers a debug session with the service. + Status RegisterDebugSession(DebugSession* session); + Status UnregisterDebugSession(DebugSession* session); + + // Blocks the caller until all sessions are ready. + // Returns AbortedError if a session connects/is already connected but + // disconnects during the wait. + Status WaitUntilAllSessionsReady(); + + StatusOr<::flatbuffers::Offset> MakeReady( + const rpc::MakeReadyRequest& request, + ::flatbuffers::FlatBufferBuilder* fbb); + + StatusOr<::flatbuffers::Offset> GetStatus( + const rpc::GetStatusRequest& request, + ::flatbuffers::FlatBufferBuilder* fbb); + + StatusOr<::flatbuffers::Offset> ListContexts( + const rpc::ListContextsRequest& request, + ::flatbuffers::FlatBufferBuilder* fbb); + + StatusOr<::flatbuffers::Offset> GetModule( + const rpc::GetModuleRequest& request, + ::flatbuffers::FlatBufferBuilder* fbb); + StatusOr<::flatbuffers::Offset> GetFunction( + const rpc::GetFunctionRequest& request, + ::flatbuffers::FlatBufferBuilder* fbb); + StatusOr<::flatbuffers::Offset> ResolveFunction( + const rpc::ResolveFunctionRequest& request, + ::flatbuffers::FlatBufferBuilder* fbb); + + StatusOr<::flatbuffers::Offset> ListFibers( + const rpc::ListFibersRequest& request, + ::flatbuffers::FlatBufferBuilder* fbb); + StatusOr<::flatbuffers::Offset> SuspendFibers( + const rpc::SuspendFibersRequest& request, + ::flatbuffers::FlatBufferBuilder* fbb); + StatusOr<::flatbuffers::Offset> ResumeFibers( + const rpc::ResumeFibersRequest& request, + ::flatbuffers::FlatBufferBuilder* fbb); + StatusOr<::flatbuffers::Offset> StepFiber( + const rpc::StepFiberRequest& request, + ::flatbuffers::FlatBufferBuilder* fbb); + StatusOr<::flatbuffers::Offset> GetFiberLocal( + const rpc::GetFiberLocalRequest& request, + ::flatbuffers::FlatBufferBuilder* fbb); + StatusOr<::flatbuffers::Offset> SetFiberLocal( + const rpc::SetFiberLocalRequest& request, + ::flatbuffers::FlatBufferBuilder* fbb); + + StatusOr<::flatbuffers::Offset> ListBreakpoints( + const rpc::ListBreakpointsRequest& request, + ::flatbuffers::FlatBufferBuilder* fbb); + StatusOr<::flatbuffers::Offset> AddBreakpoint( + const rpc::AddBreakpointRequest& request, + ::flatbuffers::FlatBufferBuilder* fbb); + StatusOr<::flatbuffers::Offset> + RemoveBreakpoint(const rpc::RemoveBreakpointRequest& request, + ::flatbuffers::FlatBufferBuilder* fbb); + + StatusOr<::flatbuffers::Offset> StartProfiling( + const rpc::StartProfilingRequest& request, + ::flatbuffers::FlatBufferBuilder* fbb); + StatusOr<::flatbuffers::Offset> StopProfiling( + const rpc::StopProfilingRequest& request, + ::flatbuffers::FlatBufferBuilder* fbb); + + // Serializes a fiber state and its stack frames. + StatusOr<::flatbuffers::Offset> SerializeFiberState( + const FiberState& fiber_state, ::flatbuffers::FlatBufferBuilder* fbb); + + private: + StatusOr GetContext(int context_id) const + ABSL_EXCLUSIVE_LOCKS_REQUIRED(mutex_); + StatusOr GetModule(int context_id, + absl::string_view module_name) const + ABSL_EXCLUSIVE_LOCKS_REQUIRED(mutex_); + StatusOr GetFiberState(int fiber_id) const + ABSL_EXCLUSIVE_LOCKS_REQUIRED(mutex_); + + // Suspends all fibers on all contexts. Returns only once all fibers have been + // suspended successfully. Fails if any fiber fails to suspend. + Status SuspendAllFibers() ABSL_EXCLUSIVE_LOCKS_REQUIRED(mutex_); + + // Resumes all fibers on all contexts (the inverse of SuspendAllFibers). + // Returns immediately. + Status ResumeAllFibers() ABSL_EXCLUSIVE_LOCKS_REQUIRED(mutex_); + + // Marks all sessions as unready. + Status UnreadyAllSessions() ABSL_EXCLUSIVE_LOCKS_REQUIRED(mutex_); + + // Attempts to re-register all breakpoints for a module. + Status RegisterModuleBreakpoints(SequencerContext* context, + vm::Module* module) + ABSL_EXCLUSIVE_LOCKS_REQUIRED(mutex_); + Status RegisterFunctionBreakpoint(SequencerContext* context, + vm::Module* module, + rpc::BreakpointDefT* breakpoint) + ABSL_EXCLUSIVE_LOCKS_REQUIRED(mutex_); + Status UnregisterFunctionBreakpoint(const rpc::BreakpointDefT& breakpoint) + ABSL_EXCLUSIVE_LOCKS_REQUIRED(mutex_); + // Signals that the given breakpoint was hit by the specified fiber. + // Called without the debug lock held. + Status OnFunctionBreakpointHit(int breakpoint_id, + const vm::Stack& fiber_state); + + absl::Mutex mutex_; + std::vector contexts_ ABSL_GUARDED_BY(mutex_); + std::vector fiber_states_ ABSL_GUARDED_BY(mutex_); + std::vector sessions_ ABSL_GUARDED_BY(mutex_); + int sessions_unready_ ABSL_GUARDED_BY(mutex_) = 0; + int sessions_ready_ ABSL_GUARDED_BY(mutex_) = 0; + + std::vector breakpoints_ ABSL_GUARDED_BY(mutex_); +}; + +} // namespace debug +} // namespace vm +} // namespace iree + +#endif // THIRD_PARTY_MLIR_EDGE_IREE_VM_DEBUG_DEBUG_SERVICE_H_ diff --git a/vm/debug/debug_session.cc b/vm/debug/debug_session.cc new file mode 100644 index 000000000000..2444d3382874 --- /dev/null +++ b/vm/debug/debug_session.cc @@ -0,0 +1,49 @@ +// Copyright 2019 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "third_party/mlir_edge/iree/vm/debug/debug_session.h" + +#include "third_party/absl/types/source_location.h" +#include "third_party/mlir_edge/iree/base/status.h" + +namespace iree { +namespace vm { +namespace debug { + +bool DebugSession::is_ready() const { + absl::MutexLock lock(&mutex_); + return ready_ == 0; +} + +Status DebugSession::OnReady() { + absl::MutexLock lock(&mutex_); + if (ready_ > 0) { + return FailedPreconditionErrorBuilder(ABSL_LOC) + << "Session has already readied up"; + } + ++ready_; + VLOG(2) << "Session " << id() << ": ++ready = " << ready_; + return OkStatus(); +} + +Status DebugSession::OnUnready() { + absl::MutexLock lock(&mutex_); + --ready_; + VLOG(2) << "Session " << id() << ": --ready = " << ready_; + return OkStatus(); +} + +} // namespace debug +} // namespace vm +} // namespace iree diff --git a/vm/debug/debug_session.h b/vm/debug/debug_session.h new file mode 100644 index 000000000000..e52b363adb6b --- /dev/null +++ b/vm/debug/debug_session.h @@ -0,0 +1,93 @@ +// Copyright 2019 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#ifndef THIRD_PARTY_MLIR_EDGE_IREE_VM_DEBUG_DEBUG_SESSION_H_ +#define THIRD_PARTY_MLIR_EDGE_IREE_VM_DEBUG_DEBUG_SESSION_H_ + +#include "third_party/absl/base/thread_annotations.h" +#include "third_party/absl/synchronization/mutex.h" +#include "third_party/mlir_edge/iree/base/status.h" +#include "third_party/mlir_edge/iree/schemas/debug_service_generated.h" +#include "third_party/mlir_edge/iree/vm/fiber_state.h" +#include "third_party/mlir_edge/iree/vm/sequencer_context.h" + +namespace iree { +namespace vm { +namespace debug { + +// An active debugging session maintained by the DebugService. +// Each connected client gets a session and transport-specific implementations +// use the event methods to receive signals from the service. +// +// All methods are called only while the debug lock is held and may be called +// from any thread. +class DebugSession { + public: + virtual ~DebugSession() = default; + + // Session ID used in all RPCs related to this session. + // This can be used for attributing RPCs to the originating session when + // multiple sessions may be active at a time/over the same transport. + int id() const { return session_id_; } + + // Returns true if the session has issued a MakeReady request and is ok if + // execution resumes. + bool is_ready() const; + + // Signals that the session has readied up and is now active. + // Called with the global debug lock held. + virtual Status OnReady(); + + // Signals that the session has gone unready (from an event/etc) and the + // service is now awaiting it to ready up. + // Called with the global debug lock held. + virtual Status OnUnready(); + + // Signals that a context has been registered. + // Called with the global debug lock held. + virtual Status OnContextRegistered(SequencerContext* context) = 0; + virtual Status OnContextUnregistered(SequencerContext* context) = 0; + + // Signals that a module has been loaded in a context. + // Called with the global debug lock held. + virtual Status OnModuleLoaded(SequencerContext* context, + vm::Module* module) = 0; + + // Signals that a fiber has been registered. + // Called with the global debug lock held. + virtual Status OnFiberRegistered(FiberState* fiber_state) = 0; + virtual Status OnFiberUnregistered(FiberState* fiber_state) = 0; + + // Signals that a breakpoint has been resolved to a particular function in a + // context. + // Called with the global debug lock held. + virtual Status OnBreakpointResolved(const rpc::BreakpointDefT& breakpoint, + SequencerContext* context) = 0; + + // Signals that the given breakpoint has been hit during execution. + // Called with the global debug lock held. + virtual Status OnBreakpointHit(int breakpoint_id, + const FiberState& fiber_state) = 0; + + private: + mutable absl::Mutex mutex_; + int session_id_ = 0; + int ready_ ABSL_GUARDED_BY(mutex_) = -1; +}; + +} // namespace debug +} // namespace vm +} // namespace iree + +#endif // THIRD_PARTY_MLIR_EDGE_IREE_VM_DEBUG_DEBUG_SESSION_H_ diff --git a/vm/debug/debug_tcp_util.h b/vm/debug/debug_tcp_util.h new file mode 100644 index 000000000000..1a0dfe8bc63f --- /dev/null +++ b/vm/debug/debug_tcp_util.h @@ -0,0 +1,217 @@ +// Copyright 2019 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// Utilities for working with TCP sockets. +// These are (mostly) portable to systems implementing BSD sockets. + +#ifndef THIRD_PARTY_MLIR_EDGE_IREE_VM_DEBUG_DEBUG_TCP_UTIL_H_ +#define THIRD_PARTY_MLIR_EDGE_IREE_VM_DEBUG_DEBUG_TCP_UTIL_H_ + +#include +#include +#include +#include +#include + +#include + +#include "third_party/flatbuffers/include/flatbuffers/base.h" +#include "third_party/flatbuffers/include/flatbuffers/flatbuffers.h" +#include "third_party/mlir_edge/iree/base/status.h" +#include "third_party/mlir_edge/iree/schemas/debug_service_generated.h" + +namespace iree { +namespace vm { +namespace debug { +namespace tcp { + +// Toggles address reuse on a socket. Call prior to binding. +// This is useful if a socket is sitting in close_wait from a previous process +// while a new one is trying to bind to it. +inline Status ToggleSocketAddressReuse(int fd, bool is_enabled) { + int toggle = is_enabled ? 1 : 0; + ::setsockopt(fd, SOL_SOCKET, SO_REUSEADDR, &toggle, sizeof(toggle)); + return OkStatus(); +} + +// Toggles the linger option on a socket. +// Enabling linger will ensure all data on the socket is sent (if it can be +// sent within N sec) prior to closing. Disabling linger will cause the socket +// to close gracefully. +inline Status ToggleSocketLinger(int fd, bool is_enabled) { + struct linger linger; + linger.l_onoff = is_enabled ? 1 : 0; + linger.l_linger = 1; + ::setsockopt(fd, SOL_SOCKET, SO_LINGER, &linger, sizeof(linger)); + return OkStatus(); +} + +// Toggles Nagel's algorithm on a socket. +// Enabled by default, sockets have ~250ms delay for small packets. Disabling +// the algorithm will make socket flushes actually send data. +inline Status ToggleSocketNagelsAlgorithm(int fd, bool is_enabled) { + int toggle = is_enabled ? 1 : 0; + ::setsockopt(fd, SOL_TCP, TCP_NODELAY, &toggle, sizeof(toggle)); + return OkStatus(); +} + +// Toggles TCP keepalive on a socket. +// Assumes that the remote side is on the local machine/network and that we can +// spam it with packets. +// +// NOTE: we may want to adjust this when real debuggers are attached (to prevent +// dropping our own connections). Need to figure out how to reliably detect +// debug suspends vs. actual death. +inline Status ToggleSocketLocalKeepalive(int fd, bool is_enabled) { + // Toggle keepalive. + int keepalive_enable = is_enabled ? 1 : 0; + ::setsockopt(fd, SOL_SOCKET, SO_KEEPALIVE, &keepalive_enable, + sizeof(keepalive_enable)); + // Begin sending keepalive probes after N sec. + int keepalive_idle_delay = 3; + ::setsockopt(fd, SOL_TCP, TCP_KEEPIDLE, &keepalive_idle_delay, + sizeof(keepalive_idle_delay)); + // Try one probe and bail (faster detection). + int keepalive_retry_count = 1; + ::setsockopt(fd, SOL_TCP, TCP_KEEPINTVL, &keepalive_retry_count, + sizeof(keepalive_retry_count)); + // Send keepalives every N sec. + int keepalive_interval = 1; + ::setsockopt(fd, SOL_TCP, TCP_KEEPINTVL, &keepalive_interval, + sizeof(keepalive_interval)); + return OkStatus(); +} + +// Toggles the blocking state of a socket. +// If a socket has been set to non-blocking methods like read and write will +// return EWOULDBLOCK if they would have blocked on the specific operation. +inline Status ToggleSocketBlocking(int fd, bool is_blocking) { + if (is_blocking) { + ::fcntl(fd, F_SETFL, ::fcntl(fd, F_GETFL) & ~O_NONBLOCK); + } else { + ::fcntl(fd, F_SETFL, ::fcntl(fd, F_GETFL) | O_NONBLOCK); + } + return OkStatus(); +} + +// RAII wrapper for messages containing flatbuffer roots of type T. +template +struct MessageBuffer { + public: + explicit MessageBuffer(std::vector buffer) + : buffer_(std::move(buffer)) {} + MessageBuffer(const MessageBuffer&) = delete; + MessageBuffer& operator=(const MessageBuffer&) = delete; + MessageBuffer(MessageBuffer&&) = default; + MessageBuffer& operator=(MessageBuffer&&) = default; + + const T& GetRoot() const { + return *::flatbuffers::GetRoot(buffer_.data()); + } + + private: + std::vector buffer_; +}; + +// Reads a size prefix value from the given fd. +// If |poll_only| is true then the size prefix is not consumed from the stream +// and the call will return 0 if there is no size prefix available. +// Returns CancelledError if a (probably) graceful close is detected. +inline StatusOr ReadSizePrefix(int fd, bool poll_only) { + ::flatbuffers::uoffset_t size_prefix = 0; + int read_bytes = ::recv(fd, &size_prefix, sizeof(size_prefix), + poll_only ? (MSG_PEEK | MSG_DONTWAIT) : 0); + if (read_bytes == 0) { + // Remote side disconnected. + return CancelledErrorBuilder(ABSL_LOC) << "Graceful remote close"; + } else if (read_bytes < 0) { + if (errno == ECONNRESET) { + return CancelledErrorBuilder(ABSL_LOC) << "Ungraceful remote close"; + } + return DataLossErrorBuilder(ABSL_LOC) + << "Failed to read size prefix from socket: (" << errno << ") " + << ::strerror(errno); + } else if (read_bytes != sizeof(size_prefix)) { + if (poll_only) { + // No data available. + return 0; + } else { + return DataLossErrorBuilder(ABSL_LOC) + << "Failed to read full size prefix (got " << read_bytes << "b of " + << sizeof(size_prefix) << "b expected)"; + } + } + return size_prefix; +} + +// Returns true if ReadBuffer will (likely) not block when called. +// Returns CancelledError if a (probably) graceful close is detected. +inline StatusOr CanReadBuffer(int fd) { + ASSIGN_OR_RETURN(size_t size_prefix, ReadSizePrefix(fd, /*poll_only=*/true)); + return size_prefix != 0; +} + +// Reads a size-prefixed message from the given fd. +// This will block until the entire message contents are available. +// Returns a buffer reference that will deallocate the buffer automatically or +// CancelledError if a (probably) graceful close is detected. +template +StatusOr> ReadBuffer(int fd) { + // Read the size prefix (written as a uoffset_t by the Write* methods). + ASSIGN_OR_RETURN(size_t size_prefix, ReadSizePrefix(fd, /*poll_only=*/false)); + + // Allocate the buffer for the entire message. + // We'll use the BufferRef to free() it when it's no longer required. + std::vector buffer(size_prefix); + + // Read the entire message contents. + int full_read_bytes = ::recv(fd, buffer.data(), buffer.size(), 0); + if (full_read_bytes < 0) { + return DataLossErrorBuilder(ABSL_LOC) + << "Failed to read full message contents from socket: (" << errno + << ") " << ::strerror(errno); + } else if (full_read_bytes != buffer.size()) { + return DataLossErrorBuilder(ABSL_LOC) + << "Failed to read full message contents (got " << full_read_bytes + << "b of " << buffer.size() << "b expected)"; + } + + // Verify the contents. Not strictly required (as we won't ever ship this to + // prod), but useful in ensuring our socket code isn't corrupting things. + ::flatbuffers::Verifier verifier(buffer.data(), buffer.size()); + if (!verifier.VerifyBuffer()) { + return DataLossErrorBuilder(ABSL_LOC) + << "Verification of input buffer of type " << typeid(T).name() + << " (" << buffer.size() << "b) failed"; + } + + // Wrap the buffer to get some RAII goodness. + return MessageBuffer(std::move(buffer)); +} + +// Writes a buffer to the given fd. +inline Status WriteBuffer(int fd, ::flatbuffers::DetachedBuffer buffer) { + if (::send(fd, buffer.data(), buffer.size(), 0) < 0) { + return UnavailableErrorBuilder(ABSL_LOC) + << "Write failed: (" << errno << ") " << ::strerror(errno); + } + return OkStatus(); +} + +} // namespace tcp +} // namespace debug +} // namespace vm +} // namespace iree + +#endif // THIRD_PARTY_MLIR_EDGE_IREE_VM_DEBUG_DEBUG_TCP_UTIL_H_ diff --git a/vm/executable_table.cc b/vm/executable_table.cc new file mode 100644 index 000000000000..870fa03d6840 --- /dev/null +++ b/vm/executable_table.cc @@ -0,0 +1,65 @@ +// Copyright 2019 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "third_party/mlir_edge/iree/vm/executable_table.h" + +#include "third_party/mlir_edge/iree/base/flatbuffer_util.h" +#include "third_party/mlir_edge/iree/base/status.h" + +namespace iree { +namespace vm { + +// static +Status ExecutableTable::ValidateStructure( + const ExecutableTableDef& executable_table_def) { + if (!executable_table_def.multi_arch_executables()) { + // May have sequencer only fns. Fine to not have dispatchable executables. + return OkStatus(); + } + + // All fat executables need at least one device-specific executable. + const auto& multi_arch_executables = + *executable_table_def.multi_arch_executables(); + for (int i = 0; i < multi_arch_executables.size(); ++i) { + const auto* multi_arch_executable = multi_arch_executables[i]; + if (!multi_arch_executable || !multi_arch_executable->executables() || + multi_arch_executable->executables()->size() == 0) { + return InvalidArgumentErrorBuilder(ABSL_LOC) + << "Multi-arch executable ordinal " << i + << " is missing its contents"; + } + } + + return OkStatus(); +} + +ExecutableTable::ExecutableTable(const ExecutableTableDef& executable_table_def) + : executable_table_def_(executable_table_def) {} + +ExecutableTable::~ExecutableTable() = default; + +StatusOr +ExecutableTable::LookupMultiArchExecutable(int executable_ordinal) const { + if (executable_ordinal < 0 || + executable_ordinal >= + executable_table_def_.multi_arch_executables()->size()) { + return InvalidArgumentErrorBuilder(ABSL_LOC) + << "Invalid multi-arch executable ordinal " << executable_ordinal; + } + return executable_table_def_.multi_arch_executables()->Get( + executable_ordinal); +} + +} // namespace vm +} // namespace iree diff --git a/vm/executable_table.h b/vm/executable_table.h new file mode 100644 index 000000000000..33196ee1fbe0 --- /dev/null +++ b/vm/executable_table.h @@ -0,0 +1,54 @@ +// Copyright 2019 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#ifndef THIRD_PARTY_MLIR_EDGE_IREE_VM_EXECUTABLE_TABLE_H_ +#define THIRD_PARTY_MLIR_EDGE_IREE_VM_EXECUTABLE_TABLE_H_ + +#include "third_party/mlir_edge/iree/base/status.h" +#include "third_party/mlir_edge/iree/schemas/executable_table_def_generated.h" + +namespace iree { +namespace vm { + +// A table of executables present within a module. +// Manages lookup and selection of executables based on target devices. +// +// Thread-safe. +class ExecutableTable { + public: + static Status ValidateStructure( + const ExecutableTableDef& executable_table_def); + + explicit ExecutableTable(const ExecutableTableDef& executable_table_def); + ExecutableTable(const ExecutableTable&) = delete; + ExecutableTable& operator=(const ExecutableTable&) = delete; + ~ExecutableTable(); + + const ExecutableTableDef& def() const { return executable_table_def_; } + + StatusOr LookupMultiArchExecutable( + int executable_ordinal) const; + + // TODO(benvanik): resolve executable by ID+format+features (ExecutableDef). + + // TODO(benvanik): insert/get HAL executables (thread-safe!). + + private: + const ExecutableTableDef& executable_table_def_; +}; + +} // namespace vm +} // namespace iree + +#endif // THIRD_PARTY_MLIR_EDGE_IREE_VM_EXECUTABLE_TABLE_H_ diff --git a/vm/fiber_state.cc b/vm/fiber_state.cc new file mode 100644 index 000000000000..48aa75153f57 --- /dev/null +++ b/vm/fiber_state.cc @@ -0,0 +1,75 @@ +// Copyright 2019 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "third_party/mlir_edge/iree/vm/fiber_state.h" + +#include + +#include "third_party/absl/strings/str_join.h" +#include "third_party/mlir_edge/iree/base/status.h" + +namespace iree { +namespace vm { + +FiberState::FiberState(std::shared_ptr instance) + : instance_(std::move(instance)), id_(Instance::NextUniqueId()) { + if (instance_->debug_server()) { + CHECK_OK(instance_->debug_server()->RegisterFiberState(this)); + } +} + +FiberState::~FiberState() { + if (instance_->debug_server()) { + CHECK_OK(instance_->debug_server()->UnregisterFiberState(this)); + } +} + +bool FiberState::is_suspended() { + // TODO(benvanik): implement. + return false; +} + +Status FiberState::Suspend(SuspendCallback suspend_callback) { + DVLOG(1) << "Suspending fiber " << id(); + return OkStatus(); +} + +Status FiberState::Resume() { + DVLOG(1) << "Resuming fiber " << id(); + return OkStatus(); +} + +Status FiberState::Step(StepTarget step_target, + SuspendCallback suspend_callback) { + return UnimplementedErrorBuilder(ABSL_LOC) << "Step not yet implemented"; +} + +namespace { +struct StackFrameFormatter { + void operator()(std::string* out, const StackFrame& stack_frame) const { + out->append(absl::StrCat(stack_frame.module().name(), ":", + stack_frame.function().name(), "@", + stack_frame.offset())); + } +}; +} // namespace + +std::string FiberState::DebugString() const { + auto frames = stack_.frames(); + return absl::StrJoin(frames.begin(), frames.end(), "\n", + StackFrameFormatter()); +} + +} // namespace vm +} // namespace iree diff --git a/vm/fiber_state.h b/vm/fiber_state.h new file mode 100644 index 000000000000..66e4a72fab05 --- /dev/null +++ b/vm/fiber_state.h @@ -0,0 +1,113 @@ +// Copyright 2019 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#ifndef THIRD_PARTY_MLIR_EDGE_IREE_VM_FIBER_STATE_H_ +#define THIRD_PARTY_MLIR_EDGE_IREE_VM_FIBER_STATE_H_ + +#include + +#include "third_party/absl/types/span.h" +#include "third_party/mlir_edge/iree/base/status.h" +#include "third_party/mlir_edge/iree/vm/instance.h" +#include "third_party/mlir_edge/iree/vm/stack.h" + +namespace iree { +namespace vm { + +// Fiber call stack and state machine model. +// Fibers may not line up with host application threads and execution may move +// across threads. +// +// Fibers are thread-compatible. Certain methods, such as Suspend and Resume +// (and others that explicitly call them) may be called from other threads, +// however members and other methods should be assumed safe to use only from +// the owning thread or when is_suspended returns true. +class FiberState { + public: + // Called when a fiber completes suspending (in response to a Suspend or Step + // request). The |suspend_status| will indicate if the suspension was + // successful. + using SuspendCallback = std::function; + + struct StepTarget { + // TODO(benvanik): step target info (matching RPC message). + // module / function / offset + // relative to current: once, out, return, etc + }; + + explicit FiberState(std::shared_ptr instance); + FiberState(const FiberState&) = delete; + FiberState& operator=(const FiberState&) = delete; + ~FiberState(); + + // A process-unique ID for the fiber. + int id() const { return id_; } + + const std::shared_ptr& instance() const { return instance_; } + + // VM call stack. + // NOTE: only valid while suspended. + const Stack& stack() const { return stack_; } + Stack* mutable_stack() { return &stack_; } + + // Returns true if the fiber is suspended. + // This only returns true if the fiber has been requested to suspend with + // Suspend and the runtime has acked the suspend. Once suspended (and until + // resumed) fiber state will not change and may be observed from any thread. + // + // Safe to call from any thread. + bool is_suspended(); + + // Suspends the fiber at the next possible chance. + // + // Fibers have a suspension depth and each call to Suspend must be matched + // with a call to Resume. Fibers will only resume excution when all prior + // Suspend calls have their matching Resume called. + // + // Optionally callers may provide a |suspend_callback| that will be called + // from a random thread when the fiber is suspended (or fails to suspend). + // + // Safe to call from any thread. + Status Suspend(SuspendCallback suspend_callback = nullptr); + + // Resumes the fiber if it is suspended (or cancels a pending suspend). + // This may wake threads if they are currently waiting on the fiber to + // execute. + // + // Safe to call from any thread. + Status Resume(); + + // Steps fiber execution. + // This will attempt to resume the fiber and will complete asynchronously. + // Upon returning the fiber should be assumed resumed and callers must query + // is_suspended to wait until the fiber suspends again. Optionally callers may + // provide a |suspend_callback| that will be called from a random thread when + // the fiber is suspended (or fails to suspend). + // + // Safe to call from any thread while the fiber is suspended. + Status Step(StepTarget step_target, + SuspendCallback suspend_callback = nullptr); + + std::string DebugString() const; + + private: + std::shared_ptr instance_; + int id_; + Stack stack_; +}; + +} // namespace vm +} // namespace iree + +#endif // THIRD_PARTY_MLIR_EDGE_IREE_VM_FIBER_STATE_H_ diff --git a/vm/function.cc b/vm/function.cc new file mode 100644 index 000000000000..f2ca4f83e567 --- /dev/null +++ b/vm/function.cc @@ -0,0 +1,121 @@ +// Copyright 2019 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "third_party/mlir_edge/iree/vm/function.h" + +#include "third_party/absl/strings/str_cat.h" +#include "third_party/absl/strings/str_join.h" +#include "third_party/mlir_edge/iree/base/flatbuffer_util.h" +#include "third_party/mlir_edge/iree/schemas/type_def_generated.h" + +namespace iree { +namespace vm { + +namespace { + +struct TypeFormatter { + void operator()(std::string* out, const TypeDef* type_def) const { + switch (type_def->type_union_type()) { + case TypeDefUnion::MemRefTypeDef: + (*this)(out, type_def->type_union_as_MemRefTypeDef()); + return; + case TypeDefUnion::DeviceTypeDef: + out->append("device"); + return; + case TypeDefUnion::CommandBufferTypeDef: + out->append("command_buffer"); + return; + case TypeDefUnion::EventTypeDef: + out->append("event"); + return; + case TypeDefUnion::SemaphoreTypeDef: + out->append("semaphore"); + return; + case TypeDefUnion::FenceTypeDef: + out->append("fence"); + return; + default: + out->append(""); + return; + } + } + + void operator()(std::string* out, + const MemRefTypeDef* mem_ref_type_def) const { + out->append("memref<"); + if (mem_ref_type_def->shape()) { + for (int dim : *mem_ref_type_def->shape()) { + out->append(std::to_string(dim)); + out->append("x"); + } + } else { + out->append("?x"); + } + (*this)(out, mem_ref_type_def->element_type()); + out->append(">"); + } + + void operator()(std::string* out, const ElementTypeDef* type_def) const { + switch (type_def->type_union_type()) { + case ElementTypeDefUnion::FloatTypeDef: { + const auto* float_type_def = type_def->type_union_as_FloatTypeDef(); + out->append("f"); + out->append(std::to_string(float_type_def->width())); + break; + } + case ElementTypeDefUnion::IntegerTypeDef: { + const auto* int_type_def = type_def->type_union_as_IntegerTypeDef(); + out->append("i"); + out->append(std::to_string(int_type_def->width())); + break; + } + case ElementTypeDefUnion::UnknownTypeDef: { + const auto* unknown_type_def = type_def->type_union_as_UnknownTypeDef(); + out->append("unknown<"); + auto dialect_str = WrapString(unknown_type_def->dialect()); + out->append(dialect_str.data(), dialect_str.size()); + auto type_data_str = WrapString(unknown_type_def->type_data()); + out->append(type_data_str.data(), type_data_str.size()); + out->append(">"); + break; + } + default: + out->append(""); + return; + } + } +}; + +} // namespace + +std::string Function::DebugStringShort() const { + return absl::StrCat( + name(), "(", + type_def().inputs() + ? absl::StrJoin(*type_def().inputs(), ", ", TypeFormatter()) + : "", + ") -> (", + type_def().results() + ? absl::StrJoin(*type_def().results(), ", ", TypeFormatter()) + : "", + ")"); +} + +std::string ImportFunction::DebugStringShort() const { + // TODO(benvanik): import function strings. + return "(IMPORT)"; +} + +} // namespace vm +} // namespace iree diff --git a/vm/function.h b/vm/function.h new file mode 100644 index 000000000000..8039c95a0669 --- /dev/null +++ b/vm/function.h @@ -0,0 +1,124 @@ +// Copyright 2019 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#ifndef THIRD_PARTY_MLIR_EDGE_IREE_VM_FUNCTION_H_ +#define THIRD_PARTY_MLIR_EDGE_IREE_VM_FUNCTION_H_ + +#include + +#include "third_party/absl/strings/string_view.h" +#include "third_party/absl/types/span.h" +#include "third_party/mlir_edge/iree/base/flatbuffer_util.h" +#include "third_party/mlir_edge/iree/base/status.h" +#include "third_party/mlir_edge/iree/hal/buffer_view.h" +#include "third_party/mlir_edge/iree/schemas/function_def_generated.h" +#include "third_party/mlir_edge/iree/schemas/type_def_generated.h" + +namespace iree { +namespace vm { + +class Stack; +class Module; + +// TODO(benvanik): reorganize this; I don't like it. maybe ImportFunction +// shouldn't derive from Function at all? + +// A function defined within a Module. +// Imported functions may be of the ImportFunction type and contain additional +// runtime linkage information. +class Function { + public: + Function() = default; + Function(const Module& module, const FunctionDef& function_def) + : module_(&module), function_def_(&function_def) {} + + absl::string_view name() const { return WrapString(function_def_->name()); } + + const Module& module() const { return *module_; } + const FunctionDef& def() const { return *function_def_; } + const FunctionTypeDef& type_def() const { return *def().type(); } + + int input_count() const { + return type_def().inputs() ? type_def().inputs()->size() : 0; + } + int result_count() const { + return type_def().results() ? type_def().results()->size() : 0; + } + + std::string DebugStringShort() const; + + private: + const Module* module_ = nullptr; + const FunctionDef* function_def_ = nullptr; +}; + +inline std::ostream& operator<<(std::ostream& stream, + const Function& function) { + stream << function.DebugStringShort(); + return stream; +} + +// TODO(benvanik): make an interface as well. +// TODO(benvanik): pass through additional attributes. +using NativeFunction = + std::function args, + absl::Span results)>; + +// A function imported into a Module from either a native function or other +// module. +class ImportFunction : public Function { + public: + enum class LinkType { + kNativeFunction, + kModule, + }; + + ImportFunction() = default; + ImportFunction(const Module& module, const FunctionDef& function_def, + NativeFunction native_function) + : Function(module, function_def), + link_type_(LinkType::kNativeFunction), + native_function_(std::move(native_function)) {} + ImportFunction(const Module& module, const FunctionDef& function_def, + Function linked_function) + : Function(module, function_def), + link_type_(LinkType::kModule), + linked_function_(std::move(linked_function)) {} + ImportFunction(const ImportFunction&) = delete; + ImportFunction& operator=(const ImportFunction&) = delete; + ImportFunction(ImportFunction&&) = default; + ImportFunction& operator=(ImportFunction&&) = default; + + LinkType link_type() const { return link_type_; } + const NativeFunction& native_function() const { return native_function_; } + const Function& linked_function() const { return linked_function_; } + + std::string DebugStringShort() const; + + private: + LinkType link_type_; + NativeFunction native_function_; + Function linked_function_; +}; + +inline std::ostream& operator<<(std::ostream& stream, + const ImportFunction& function) { + stream << function.DebugStringShort(); + return stream; +} + +} // namespace vm +} // namespace iree + +#endif // THIRD_PARTY_MLIR_EDGE_IREE_VM_FUNCTION_H_ diff --git a/vm/function_table.cc b/vm/function_table.cc new file mode 100644 index 000000000000..bfb73a48d0d8 --- /dev/null +++ b/vm/function_table.cc @@ -0,0 +1,261 @@ +// Copyright 2019 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "third_party/mlir_edge/iree/vm/function_table.h" + +#include "third_party/absl/container/flat_hash_map.h" +#include "third_party/mlir_edge/iree/base/flatbuffer_util.h" +#include "third_party/mlir_edge/iree/base/status.h" + +namespace iree { +namespace vm { + +namespace { + +Status ValidateType(const FunctionTypeDef& type_def) { + // Ensure all fields are populated. + return OkStatus(); +} + +} // namespace + +// static +Status FunctionTable::ValidateStructure( + const FunctionTableDef& function_table_def) { + if (!function_table_def.functions()) { + return InvalidArgumentErrorBuilder(ABSL_LOC) + << "Function table is missing the function listing"; + } + + // All functions must contain a valid type. + const auto& functions = *function_table_def.functions(); + for (int i = 0; i < functions.size(); ++i) { + const auto* function = functions[i]; + if (!function) { + return InvalidArgumentErrorBuilder(ABSL_LOC) + << "Function ordinal " << i << " is missing its contents"; + } + if (!function->type()) { + return InvalidArgumentErrorBuilder(ABSL_LOC) + << "Function ordinal " << i << " is missing its type"; + } + RETURN_IF_ERROR(ValidateType(*function->type())); + } + + // Imports must also have a name (that we can use to resolve it). + if (function_table_def.imports()) { + const auto& imports = *function_table_def.imports(); + for (int i = 0; i < imports.size(); ++i) { + int function_index = imports[i]; + if (!functions[function_index]->name()) { + return InvalidArgumentErrorBuilder(ABSL_LOC) + << "Import ordinal " << i << " is missing its contents"; + } + } + } + + // Exports must also have a name (that others will use to look it up). + if (function_table_def.exports()) { + const auto& exports = *function_table_def.exports(); + for (int i = 0; i < exports.size(); ++i) { + int function_index = exports[i]; + if (!functions[function_index]->name()) { + return InvalidArgumentErrorBuilder(ABSL_LOC) + << "Export ordinal " << i << " is missing its contents"; + } + } + } + + return OkStatus(); +} + +FunctionTable::FunctionTable(const Module& module, + const FunctionTableDef& function_table_def) + : module_(module), function_table_def_(function_table_def) {} + +FunctionTable::~FunctionTable() = default; + +Status FunctionTable::ResolveImports(ImportResolver import_resolver) { + if (!function_table_def_.imports()) { + // No imports to resolve. + return OkStatus(); + } + + const auto& imports = *function_table_def_.imports(); + const auto& functions = *function_table_def_.functions(); + for (int i = 0; i < imports.size(); ++i) { + const auto* function_def = functions[imports[i]]; + ASSIGN_OR_RETURN(auto import_function, + import_resolver(module_, *function_def)); + import_functions_.push_back(std::move(import_function)); + } + + return OkStatus(); +} + +StatusOr FunctionTable::LookupImportOrdinal( + absl::string_view import_name) const { + if (function_table_def_.imports()) { + const auto& imports = *function_table_def_.imports(); + const auto& functions = *function_table_def_.functions(); + for (int i = 0; i < imports.size(); ++i) { + if (WrapString(functions[imports[i]]->name()) == import_name) { + return i; + } + } + } + return NotFoundErrorBuilder(ABSL_LOC) + << "Import with the name '" << import_name << "' not found in module"; +} + +StatusOr FunctionTable::LookupImport( + absl::string_view import_name) const { + ASSIGN_OR_RETURN(int import_ordinal, LookupImportOrdinal(import_name)); + return LookupImport(import_ordinal); +} + +StatusOr FunctionTable::LookupImport( + int import_ordinal) const { + if (import_ordinal < 0 || import_ordinal >= import_functions_.size()) { + return InvalidArgumentErrorBuilder(ABSL_LOC) + << "Import ordinal " << import_ordinal + << " is outside the valid range [0, " << import_functions_.size() + << ")"; + } + return {&import_functions_[import_ordinal]}; +} + +StatusOr FunctionTable::LookupExportFunctionOrdinal( + absl::string_view export_name) const { + // NOTE: this is a linear scan of the export table, but since export count + // is usually small and the only time this lookup should happen is on module + // load it's (probably) fine. + if (function_table_def_.exports()) { + const auto& exports = *function_table_def_.exports(); + for (int i = 0; i < exports.size(); ++i) { + int export_ordinal = exports.Get(i); + const auto& function_def = + *function_table_def_.functions()->Get(export_ordinal); + if (WrapString(function_def.name()) == export_name) { + return export_ordinal; + } + } + } + return NotFoundErrorBuilder(ABSL_LOC) + << "Export with the name '" << export_name << "' not found in module"; +} + +StatusOr FunctionTable::LookupExport( + absl::string_view export_name) const { + ASSIGN_OR_RETURN(int export_ordinal, + LookupExportFunctionOrdinal(export_name)); + return LookupFunction(export_ordinal); +} + +StatusOr FunctionTable::LookupExport(int export_ordinal) const { + if (!function_table_def_.exports() || export_ordinal < 0 || + export_ordinal >= function_table_def_.exports()->size()) { + return InvalidArgumentErrorBuilder(ABSL_LOC) + << "Export ordinal " << export_ordinal + << " is outside the valid range [0, " + << function_table_def_.exports()->size() << ")"; + } + const auto& exports = *function_table_def_.exports(); + int function_ordinal = exports.Get(export_ordinal); + return LookupFunction(function_ordinal); +} + +StatusOr FunctionTable::LookupFunction(int ordinal) const { + if (ordinal < 0 || ordinal >= function_table_def_.functions()->size()) { + return OutOfRangeErrorBuilder(ABSL_LOC) + << "Function ordinal " << ordinal + << " is outside the valid range [0, " + << function_table_def_.functions()->size() << ")"; + } + const auto* function_def = function_table_def_.functions()->Get(ordinal); + return Function(module_, *function_def); +} + +StatusOr FunctionTable::LookupFunctionOrdinal( + const Function& function) const { + const auto& functions = *function_table_def_.functions(); + for (int i = 0; i < functions.size(); ++i) { + if (&function.def() == functions.Get(i)) { + return i; + } + } + return NotFoundErrorBuilder(ABSL_LOC) << "Function not a member of module"; +} + +StatusOr FunctionTable::LookupFunctionOrdinalByName( + absl::string_view name) const { + for (int i = 0; i < function_table_def_.functions()->size(); ++i) { + const auto* function_def = function_table_def_.functions()->Get(i); + if (WrapString(function_def->name()) == name) { + return i; + } + } + return NotFoundErrorBuilder(ABSL_LOC) + << "Function '" << name + << "' not found in function table (or names have been stripped)"; +} + +Status FunctionTable::RegisterBreakpoint(int function_ordinal, int offset, + BreakpointCallback callback) { + if (breakpoint_tables_.empty()) { + breakpoint_tables_.resize(function_table_def_.functions()->size()); + } + if (function_ordinal < 0 || function_ordinal > breakpoint_tables_.size()) { + return InvalidArgumentErrorBuilder(ABSL_LOC) + << "Function ordinal " << function_ordinal << " out of bounds"; + } + if (!breakpoint_tables_[function_ordinal]) { + breakpoint_tables_[function_ordinal] = + absl::make_unique>(); + } + auto& function_table = *breakpoint_tables_[function_ordinal]; + function_table[offset] = std::move(callback); + return OkStatus(); +} + +Status FunctionTable::UnregisterBreakpoint(int function_ordinal, int offset) { + if (function_ordinal < 0 || function_ordinal > breakpoint_tables_.size()) { + return InvalidArgumentErrorBuilder(ABSL_LOC) + << "Function ordinal " << function_ordinal << " out of bounds"; + } + auto* function_table = breakpoint_tables_[function_ordinal].get(); + if (function_table) { + auto it = function_table->find(offset); + if (it != function_table->end()) { + function_table->erase(it); + } + } + return OkStatus(); +} + +Status FunctionTable::UnregisterAllBreakpoints() { + breakpoint_tables_.clear(); + return OkStatus(); +} + +FunctionTable::BreakpointTable* FunctionTable::GetFunctionBreakpointTable( + int function_ordinal) const { + if (function_ordinal < 0 || function_ordinal >= breakpoint_tables_.size()) { + return nullptr; + } + return breakpoint_tables_[function_ordinal].get(); +} + +} // namespace vm +} // namespace iree diff --git a/vm/function_table.h b/vm/function_table.h new file mode 100644 index 000000000000..f252501474df --- /dev/null +++ b/vm/function_table.h @@ -0,0 +1,125 @@ +// Copyright 2019 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#ifndef THIRD_PARTY_MLIR_EDGE_IREE_VM_FUNCTION_TABLE_H_ +#define THIRD_PARTY_MLIR_EDGE_IREE_VM_FUNCTION_TABLE_H_ + +#include +#include + +#include "third_party/absl/container/flat_hash_map.h" +#include "third_party/absl/strings/string_view.h" +#include "third_party/absl/types/span.h" +#include "third_party/mlir_edge/iree/base/status.h" +#include "third_party/mlir_edge/iree/schemas/function_table_def_generated.h" +#include "third_party/mlir_edge/iree/vm/function.h" + +namespace iree { +namespace vm { + +class Stack; +class Module; + +// A table of functions present within a module. +// Manages the import table, local function resolution, and breakpoints. +// +// Function tables are normally thread-compatible. Debugging-specific methods +// like RegisterBreakpoint must only be called when the debugger has suspended +// all fibers that could be executing functions from the table. +class FunctionTable { + public: + static Status ValidateStructure(const FunctionTableDef& function_table_def); + + FunctionTable(const Module& module, + const FunctionTableDef& function_table_def); + FunctionTable(const FunctionTable&) = delete; + FunctionTable& operator=(const FunctionTable&) = delete; + ~FunctionTable(); + + const FunctionTableDef& def() const { return function_table_def_; } + + using ImportResolver = std::function( + const Module& importing_module, const FunctionDef& import_function_def)>; + Status ResolveImports(ImportResolver import_resolver); + + StatusOr LookupImport( + absl::string_view import_name) const; + StatusOr LookupImport(int import_ordinal) const; + + StatusOr LookupExport(absl::string_view export_name) const; + StatusOr LookupExport(int export_ordinal) const; + + StatusOr LookupFunction(int ordinal) const; + + StatusOr LookupFunctionOrdinal(const Function& function) const; + StatusOr LookupFunctionOrdinalByName(absl::string_view name) const; + + // Handles breakpoints that are encountered during execution. + // The current function and offset within the function will be provided. + // The fiber is set as suspended prior to issuing the callback and resumed + // if the callback returns ok. + // + // Implementations can use the return status to indicate intended program + // flow: + // - return ok to resume the fiber and continue execution + // - return abort to terminate the fiber + // - return an error to propagate via normal error handling logic + using BreakpointCallback = std::function; + + // Registers a breakpoint for an operation offset within a function. + // The provided callback will be issued when the breakpoint is hit. If a + // breakpoint already exists for the given offset it will be replaced. + // + // The global debug lock must be held and all fibers must be suspended. + Status RegisterBreakpoint(int function_ordinal, int offset, + BreakpointCallback callback); + + // Unregisters a breakpoint, if one has been registered. + // + // The global debug lock must be held and all fibers must be suspended. + Status UnregisterBreakpoint(int function_ordinal, int offset); + + // Unregisters all breakpoints in the function table. + // + // The global debug lock must be held and all fibers must be suspended. + Status UnregisterAllBreakpoints(); + + using BreakpointTable = absl::flat_hash_map; + + // Returns the breakpoint table mapping offset to breakpoint callback. + // Returns nullptr if the given function does not have a breakpoint table. + // + // This table is not synchronized and while the debug lock is held it must not + // be accessed by any other threads. Reading is otherwise safe. + BreakpointTable* GetFunctionBreakpointTable(int function_ordinal) const; + + private: + StatusOr LookupImportOrdinal(absl::string_view import_name) const; + StatusOr LookupExportFunctionOrdinal( + absl::string_view export_name) const; + + const Module& module_; + const FunctionTableDef& function_table_def_; + std::vector import_functions_; + + // One slot per function in the function table. The hash map contains the + // breakpoints for that particular function mapped by offset within the + // function. + std::vector> breakpoint_tables_; +}; + +} // namespace vm +} // namespace iree + +#endif // THIRD_PARTY_MLIR_EDGE_IREE_VM_FUNCTION_TABLE_H_ diff --git a/vm/instance.cc b/vm/instance.cc new file mode 100644 index 000000000000..5339b35a52da --- /dev/null +++ b/vm/instance.cc @@ -0,0 +1,37 @@ +// Copyright 2019 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "third_party/mlir_edge/iree/vm/instance.h" + +#include "third_party/absl/memory/memory.h" +#include "third_party/absl/types/source_location.h" +#include "third_party/mlir_edge/iree/base/status.h" + +namespace iree { +namespace vm { + +// static +int Instance::NextUniqueId() { + static int next_id = 0; + return ++next_id; +} + +Instance::Instance(std::unique_ptr debug_server) + : debug_server_(std::move(debug_server)), + device_manager_(absl::make_unique()) {} + +Instance::~Instance() = default; + +} // namespace vm +} // namespace iree diff --git a/vm/instance.h b/vm/instance.h new file mode 100644 index 000000000000..b04950a8b55f --- /dev/null +++ b/vm/instance.h @@ -0,0 +1,55 @@ +// Copyright 2019 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#ifndef THIRD_PARTY_MLIR_EDGE_IREE_VM_INSTANCE_H_ +#define THIRD_PARTY_MLIR_EDGE_IREE_VM_INSTANCE_H_ + +#include + +#include "third_party/mlir_edge/iree/hal/device_manager.h" +#include "third_party/mlir_edge/iree/vm/debug/debug_server.h" + +namespace iree { +namespace vm { + +// Shared runtime instance responsible for routing Context events, enumerating +// and creating hardware device interfaces, and managing thread pools. +// +// A single runtime instance can service multiple contexts and hosting +// applications should try to reuse a runtime as much as possible. This ensures +// that resource allocation across contexts is handled and extraneous device +// interaction is avoided. +class Instance { + public: + // Allocates a global unique ID. + static int NextUniqueId(); + + explicit Instance(std::unique_ptr debug_server = nullptr); + ~Instance(); + Instance(const Instance&) = delete; + Instance& operator=(const Instance&) = delete; + + debug::DebugServer* debug_server() const { return debug_server_.get(); } + + hal::DeviceManager* device_manager() const { return device_manager_.get(); } + + private: + std::unique_ptr debug_server_; + std::unique_ptr device_manager_; +}; + +} // namespace vm +} // namespace iree + +#endif // THIRD_PARTY_MLIR_EDGE_IREE_VM_INSTANCE_H_ diff --git a/vm/module.cc b/vm/module.cc new file mode 100644 index 000000000000..f948787f6f8c --- /dev/null +++ b/vm/module.cc @@ -0,0 +1,77 @@ +// Copyright 2019 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "third_party/mlir_edge/iree/vm/module.h" + +#include "third_party/absl/memory/memory.h" +#include "third_party/mlir_edge/iree/base/status.h" + +namespace iree { +namespace vm { + +// static +Status Module::ValidateStructure(const ModuleDef& module_def) { + // Must have a function table. + if (module_def.function_table()) { + RETURN_IF_ERROR( + FunctionTable::ValidateStructure(*module_def.function_table())); + } else { + return InvalidArgumentErrorBuilder(ABSL_LOC) + << "ModuleDef is missing a function table"; + } + + // May optionally have an executable table. + if (module_def.executable_table()) { + RETURN_IF_ERROR( + ExecutableTable::ValidateStructure(*module_def.executable_table())); + } + + return OkStatus(); +} + +// static +StatusOr> Module::FromDef(const ModuleDef& module_def) { + ASSIGN_OR_RETURN(auto module_file, ModuleFile::Create(&module_def, []() {})); + return FromFile(std::move(module_file)); +} + +// static +StatusOr> Module::FromFile( + std::unique_ptr module_file) { + if (module_file->root() == nullptr) { + return InvalidArgumentErrorBuilder(ABSL_LOC) << "No root ModuleDef present"; + } + const auto& module_def = *module_file->root(); + + // Validates the structure of the module (but not bytecode). + // This ensures we don't have flatbuffer vectors will null entries, etc. + RETURN_IF_ERROR(Module::ValidateStructure(module_def)); + + auto module = absl::WrapUnique(new Module(std::move(module_file))); + + // TODO(benvanik): validate internals here? or make explicit? + + return {std::move(module)}; +} + +Module::Module(std::unique_ptr module_file) + : module_file_(std::move(module_file)), + module_def_(*module_file_->root()), + function_table_(*this, *module_def_.function_table()), + executable_table_(*module_def_.executable_table()) {} + +Module::~Module() = default; + +} // namespace vm +} // namespace iree diff --git a/vm/module.h b/vm/module.h new file mode 100644 index 000000000000..d49521a28dc8 --- /dev/null +++ b/vm/module.h @@ -0,0 +1,63 @@ +// Copyright 2019 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#ifndef THIRD_PARTY_MLIR_EDGE_IREE_VM_MODULE_H_ +#define THIRD_PARTY_MLIR_EDGE_IREE_VM_MODULE_H_ + +#include + +#include "third_party/mlir_edge/iree/base/flatbuffer_util.h" +#include "third_party/mlir_edge/iree/schemas/module_def_generated.h" +#include "third_party/mlir_edge/iree/vm/executable_table.h" +#include "third_party/mlir_edge/iree/vm/function_table.h" + +namespace iree { +namespace vm { + +using ModuleFile = FlatBufferFile; + +// A loaded bytecode module. +class Module { + public: + static Status ValidateStructure(const ModuleDef& module_def); + + static StatusOr> FromDef(const ModuleDef& module_def); + static StatusOr> FromFile( + std::unique_ptr module_file); + + Module(const Module&) = delete; + Module& operator=(const Module&) = delete; + ~Module(); + + absl::string_view name() const { return WrapString(module_def_.name()); } + + const ModuleDef& def() const { return module_def_; } + const FunctionTable& function_table() const { return function_table_; } + FunctionTable* mutable_function_table() { return &function_table_; } + const ExecutableTable& executable_table() const { return executable_table_; } + ExecutableTable* mutable_executable_table() { return &executable_table_; } + + private: + explicit Module(std::unique_ptr module_file); + + std::unique_ptr module_file_; + const ModuleDef& module_def_; + FunctionTable function_table_; + ExecutableTable executable_table_; +}; + +} // namespace vm +} // namespace iree + +#endif // THIRD_PARTY_MLIR_EDGE_IREE_VM_MODULE_H_ diff --git a/vm/module_printer.cc b/vm/module_printer.cc new file mode 100644 index 000000000000..3fd2653e012f --- /dev/null +++ b/vm/module_printer.cc @@ -0,0 +1,55 @@ +// Copyright 2019 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "third_party/mlir_edge/iree/vm/module_printer.h" + +#include "third_party/mlir_edge/iree/vm/bytecode_printer.h" +#include "third_party/mlir_edge/iree/vm/source_map.h" + +namespace iree { +namespace vm { + +Status PrintModuleToStream(OpcodeTable opcode_table, const Module& module, + PrintModuleFlagBitfield flags, + std::ostream* stream) { + // TODO(benvanik): custom FunctionTable Function iterator. + for (int i = 0; i < module.function_table().def().functions()->size(); ++i) { + ASSIGN_OR_RETURN(const auto& function, + module.function_table().LookupFunction(i)); + if (function.def().bytecode()) { + auto source_map_resolver = + AllBitsSet(flags, PrintModuleFlag::kIncludeSourceMapping) + ? SourceMapResolver::FromFunction(module.def(), i) + : SourceMapResolver(); + BytecodePrinter printer(opcode_table, module.function_table(), + module.executable_table(), source_map_resolver); + *stream << "Function " << i << ": " << function << "\n"; + RETURN_IF_ERROR( + printer.PrintToStream(*function.def().bytecode(), stream)); + *stream << "\n"; + } else { + *stream << "Function " << i << ": " << function.name() << " (import)\n"; + } + } + return OkStatus(); +} + +Status PrintModuleToStream(OpcodeTable opcode_table, const Module& module, + std::ostream* stream) { + return PrintModuleToStream(opcode_table, module, PrintModuleFlag::kNone, + stream); +} + +} // namespace vm +} // namespace iree diff --git a/vm/module_printer.h b/vm/module_printer.h new file mode 100644 index 000000000000..f5946d963f00 --- /dev/null +++ b/vm/module_printer.h @@ -0,0 +1,44 @@ +// Copyright 2019 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#ifndef THIRD_PARTY_MLIR_EDGE_IREE_VM_MODULE_PRINTER_H_ +#define THIRD_PARTY_MLIR_EDGE_IREE_VM_MODULE_PRINTER_H_ + +#include + +#include "third_party/mlir_edge/iree/base/bitfield.h" +#include "third_party/mlir_edge/iree/base/status.h" +#include "third_party/mlir_edge/iree/vm/module.h" +#include "third_party/mlir_edge/iree/vm/opcode_info.h" + +namespace iree { +namespace vm { + +enum class PrintModuleFlag { + kNone = 0, + kIncludeSourceMapping = 1, +}; +IREE_BITFIELD(PrintModuleFlag); +using PrintModuleFlagBitfield = PrintModuleFlag; + +// Prints all functions within the module to the given |stream|. +Status PrintModuleToStream(OpcodeTable opcode_table, const Module& module, + std::ostream* stream); +Status PrintModuleToStream(OpcodeTable opcode_table, const Module& module, + PrintModuleFlagBitfield flags, std::ostream* stream); + +} // namespace vm +} // namespace iree + +#endif // THIRD_PARTY_MLIR_EDGE_IREE_VM_MODULE_PRINTER_H_ diff --git a/vm/opcode_info.h b/vm/opcode_info.h new file mode 100644 index 000000000000..cac6d4b3fd7e --- /dev/null +++ b/vm/opcode_info.h @@ -0,0 +1,45 @@ +// Copyright 2019 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#ifndef THIRD_PARTY_MLIR_EDGE_IREE_VM_OPCODE_INFO_H_ +#define THIRD_PARTY_MLIR_EDGE_IREE_VM_OPCODE_INFO_H_ + +#include "third_party/absl/strings/string_view.h" +#include "third_party/absl/types/optional.h" +#include "third_party/absl/types/span.h" +#include "third_party/mlir_edge/iree/schemas/bytecode/bytecode_v0.h" + +namespace iree { +namespace vm { + +struct OpcodeInfo { + const char* mnemonic; + OpcodeFlagBitfield flag; + union { + const char operands_value[8]; + const OperandEncoding operands[8]; + }; +}; + +using OpcodeTable = absl::Span; + +template +inline const OpcodeInfo& GetOpcodeInfo(OpcodeTable opcode_table, T opcode) { + return opcode_table[static_cast(opcode)]; +} + +} // namespace vm +} // namespace iree + +#endif // THIRD_PARTY_MLIR_EDGE_IREE_VM_OPCODE_INFO_H_ diff --git a/vm/sequencer_context.cc b/vm/sequencer_context.cc new file mode 100644 index 000000000000..4e688606429c --- /dev/null +++ b/vm/sequencer_context.cc @@ -0,0 +1,167 @@ +// Copyright 2019 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "third_party/mlir_edge/iree/vm/sequencer_context.h" + +#include "third_party/mlir_edge/iree/base/flatbuffer_util.h" +#include "third_party/mlir_edge/iree/base/status.h" +#include "third_party/mlir_edge/iree/hal/buffer_view.h" +#include "third_party/mlir_edge/iree/vm/fiber_state.h" +#include "third_party/mlir_edge/iree/vm/sequencer_dispatch.h" + +namespace iree { +namespace vm { + +namespace { + +using ::iree::hal::BufferView; + +Status ValidateElementSize(int element_bit_width, + const ElementTypeDef& expected_element_type) { + switch (expected_element_type.type_union_type()) { + case ElementTypeDefUnion::FloatTypeDef: { + auto expected_bit_width = + expected_element_type.type_union_as_FloatTypeDef()->width(); + if (element_bit_width != expected_bit_width) { + return InvalidArgumentErrorBuilder(ABSL_LOC) + << "Has element bit width " << element_bit_width + << " but expected " << expected_bit_width; + } + return OkStatus(); + } + case ElementTypeDefUnion::IntegerTypeDef: { + auto expected_bit_width = + expected_element_type.type_union_as_IntegerTypeDef()->width(); + if (element_bit_width != expected_bit_width) { + return InvalidArgumentErrorBuilder(ABSL_LOC) + << "Has element bit width " << element_bit_width + << " but expected " << expected_bit_width; + } + return OkStatus(); + } + case ElementTypeDefUnion::UnknownTypeDef: + case ElementTypeDefUnion::NONE: { + } + } + return InvalidArgumentErrorBuilder(ABSL_LOC) + << "Defined type has unsupported element type " + << EnumNameElementTypeDefUnion( + expected_element_type.type_union_type()); +} + +Status ValidateArgType(const BufferView& arg, + const MemRefTypeDef& expected_type) { + RETURN_IF_ERROR( + ValidateElementSize(arg.element_size * 8, *expected_type.element_type())); + + auto expected_shape = expected_type.shape(); + if (arg.shape.size() != expected_shape->size()) { + return InvalidArgumentErrorBuilder(ABSL_LOC) + << "Argument should have rank " << expected_shape->size() + << " but has rank " << arg.shape.size(); + } + for (int i = 0; i < expected_shape->size(); ++i) { + auto dim_size = arg.shape[i]; + auto expected_dim_size = expected_shape->Get(i); + if (dim_size != expected_dim_size) { + return InvalidArgumentErrorBuilder(ABSL_LOC) + << "Argument dimension " << i << " should have size " + << expected_dim_size << " but has size " << dim_size; + } + } + return OkStatus(); +} + +} // namespace + +SequencerContext::SequencerContext(std::shared_ptr instance) + : instance_(std::move(instance)) { + if (instance_->debug_server()) { + CHECK_OK(instance_->debug_server()->RegisterContext(this)); + } +} + +SequencerContext::~SequencerContext() { + if (instance_->debug_server()) { + CHECK_OK(instance_->debug_server()->UnregisterContext(this)); + } +} + +Status SequencerContext::RegisterNativeFunction( + std::string name, NativeFunction native_function) { + // TODO(benvanik): provide to debugger. + return Context::RegisterNativeFunction(std::move(name), + std::move(native_function)); +} + +Status SequencerContext::RegisterModule(std::unique_ptr module) { + auto* module_ptr = module.get(); + RETURN_IF_ERROR(Context::RegisterModule(std::move(module))); + if (instance_->debug_server()) { + RETURN_IF_ERROR( + instance_->debug_server()->RegisterContextModule(this, module_ptr)); + } + return OkStatus(); +} + +Status SequencerContext::Invoke(FiberState* fiber_state, Function function, + absl::Span args, + absl::Span results) const { + // Verify arg/result counts. + if (args.size() != function.input_count()) { + return InvalidArgumentErrorBuilder(ABSL_LOC) + << "Function " << function.name() << " requires " + << function.input_count() << " inputs but " << args.size() + << " provided"; + } + if (results.size() != function.result_count()) { + return InvalidArgumentErrorBuilder(ABSL_LOC) + << "Function " << function.name() << " requires " + << function.result_count() << " outputs but " << results.size() + << " provided"; + } + + // Push stack frame for the function we are calling. + auto* stack = fiber_state->mutable_stack(); + ASSIGN_OR_RETURN(auto* callee_stack_frame, stack->PushFrame(function)); + + // Marshal input arguments. + for (int i = 0; i < args.size(); ++i) { + auto arg = args[i]; + auto expected_arg_type = function.type_def().inputs()->Get(i); + RETURN_IF_ERROR( + ValidateArgType(arg, *expected_arg_type->type_union_as_MemRefTypeDef())) + << "Function " << function.name() << " argument " << i; + *callee_stack_frame->mutable_local(i) = std::move(arg); + } + + // TODO(benvanik): change to: + // get command queue (any command queue) + // make command buffer + // record dispatch + // submit + // wait on fence + ASSIGN_OR_RETURN(auto placement, + instance_->device_manager()->ResolvePlacement({})); + RETURN_IF_ERROR( + DispatchSequence(placement, stack, callee_stack_frame, results)); + + // Pop the callee frame to balance out the stack. + RETURN_IF_ERROR(stack->PopFrame()); + + return OkStatus(); +} + +} // namespace vm +} // namespace iree diff --git a/vm/sequencer_context.h b/vm/sequencer_context.h new file mode 100644 index 000000000000..5033a546a688 --- /dev/null +++ b/vm/sequencer_context.h @@ -0,0 +1,55 @@ +// Copyright 2019 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#ifndef THIRD_PARTY_MLIR_EDGE_IREE_VM_SEQUENCER_CONTEXT_H_ +#define THIRD_PARTY_MLIR_EDGE_IREE_VM_SEQUENCER_CONTEXT_H_ + +#include +#include + +#include "third_party/absl/strings/string_view.h" +#include "third_party/absl/types/span.h" +#include "third_party/mlir_edge/iree/base/status.h" +#include "third_party/mlir_edge/iree/hal/buffer_view.h" +#include "third_party/mlir_edge/iree/vm/context.h" +#include "third_party/mlir_edge/iree/vm/function.h" +#include "third_party/mlir_edge/iree/vm/instance.h" +#include "third_party/mlir_edge/iree/vm/module.h" + +namespace iree { +namespace vm { + +class SequencerContext final : public Context { + public: + explicit SequencerContext(std::shared_ptr instance); + ~SequencerContext() override; + + Status RegisterNativeFunction(std::string name, + NativeFunction native_function) override; + + Status RegisterModule(std::unique_ptr module) override; + + // TODO(benvanik): helpers to make passing args easier + Status Invoke(FiberState* fiber_state, vm::Function function, + absl::Span args, + absl::Span results) const; + + private: + std::shared_ptr instance_; +}; + +} // namespace vm +} // namespace iree + +#endif // THIRD_PARTY_MLIR_EDGE_IREE_VM_CONTEXT_H_ diff --git a/vm/sequencer_dispatch.cc b/vm/sequencer_dispatch.cc new file mode 100644 index 000000000000..ac7350a1fd36 --- /dev/null +++ b/vm/sequencer_dispatch.cc @@ -0,0 +1,561 @@ +// Copyright 2019 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// Implements a full bytecode dispatch system for sequencer ops. +// TODO(benvanik): rework to be async against CommandBuffers. + +#include "third_party/mlir_edge/iree/vm/sequencer_dispatch.h" + +#include + +#include "third_party/absl/base/attributes.h" +#include "third_party/absl/container/inlined_vector.h" +#include "third_party/absl/strings/str_join.h" +#include "third_party/absl/time/time.h" +#include "third_party/absl/types/span.h" +#include "third_party/mlir_edge/iree/base/logging.h" +#include "third_party/mlir_edge/iree/base/memory.h" +#include "third_party/mlir_edge/iree/base/status.h" +#include "third_party/mlir_edge/iree/hal/buffer_view.h" +#include "third_party/mlir_edge/iree/hal/command_queue.h" +#include "third_party/mlir_edge/iree/hal/device.h" +#include "third_party/mlir_edge/iree/hal/heap_buffer.h" +#include "third_party/mlir_edge/iree/schemas/bytecode/sequencer_bytecode_v0.h" +#include "third_party/mlir_edge/iree/vm/bytecode_reader.h" +#include "third_party/mlir_edge/iree/vm/bytecode_tables_sequencer.h" +#include "third_party/mlir_edge/iree/vm/bytecode_util.h" +#include "third_party/mlir_edge/iree/vm/function.h" +#include "third_party/mlir_edge/iree/vm/opcode_info.h" + +namespace iree { +namespace vm { + +namespace { + +using ::iree::hal::Buffer; +using ::iree::hal::BufferView; + +// TODO(benvanik): remove (this should happen via predication). +bool BufferViewIsTrue(const BufferView& buffer_view) { + if (buffer_view.element_size == 0 || !buffer_view.buffer || + buffer_view.byte_length() == 0) { + return false; + } + // TODO(benvanik): map more efficiently (based on element size?). + auto mapping = + buffer_view.buffer->MapMemory(hal::MemoryAccess::kRead); + if (!mapping.ok()) { + return false; + } + for (uint8_t value : mapping.ValueOrDie().contents()) { + if (value) return true; + } + return false; +} + +// TODO(benvanik): insert fence callbacks and wait on fence. +Status CallNativeFunction(Stack* stack, const ImportFunction& function) { + auto* stack_frame = stack->current_frame(); + + // Marshal inputs and outputs. + auto args = stack_frame->mutable_locals().subspan(0, function.input_count()); + auto results = stack_frame->mutable_locals().subspan(args.size()); + + const auto& fn = function.native_function(); + return fn(stack, args, results); +} + +// Pretty prints an array, e.g. [1, 2, 3, 4] +inline std::string PrettyPrint(absl::Span arr) { + return "[" + absl::StrJoin(arr, ",") + "]"; +} + +// Calculates the byte offset into a buffer corresponding to the indices in the +// given shape. +StatusOr CalculateOffset(absl::Span indices, + Shape shape, uint8_t element_size) { + if (shape.empty() || indices.size() > shape.size()) { + return InvalidArgumentErrorBuilder(ABSL_LOC) + << "Indices " << PrettyPrint(indices) << " out of bounds of shape " + << PrettyPrint(shape.subspan()); + } + device_size_t offset = 0; + for (int i = 0; i < indices.size(); ++i) { + if (indices[i] >= shape[i]) { + return InvalidArgumentErrorBuilder(ABSL_LOC) + << "Indices[" << i << "]=" << indices[i] + << " out of bounds of shape " << PrettyPrint(shape.subspan()); + } + device_size_t axis_offset = indices[i]; + for (int j = i + 1; j < shape.size(); ++j) { + axis_offset *= shape[j]; + } + offset += axis_offset; + } + offset *= element_size; + return offset; +} + +} // namespace + +Status DispatchSequence(const hal::DevicePlacement& placement, Stack* stack, + StackFrame* entry_stack_frame, + absl::Span entry_results) { + // Dispatch table mapping 1:1 with bytecode ops. + // Each entry is a label within this function that can be used for computed + // goto. You can find more information on computed goto here: + // https://eli.thegreenplace.net/2012/07/12/computed-goto-for-efficient-dispatch-tables + // + // Note that we ensure the table is 256 elements long exactly to make sure + // that unused opcodes are handled gracefully. + static const void* kDispatchTable[256] = { +#define DECLARE_DISPATCH(ordinal, name, ...) &&_dispatch_##name, +#define DECLARE_DISPATCH_RESERVED(ordinal, name, ...) &&_dispatch_unhandled, + IREE_SEQUENCER_OPCODE_LIST(DECLARE_DISPATCH, DECLARE_DISPATCH_RESERVED) +#undef DECLARE_DISPATCH +#undef DECLARE_DISPATCH_RESERVED + }; + + // Primary dispatch state. This is our 'native stack frame' and really just + // enough to make dereferencing common addresses (like the current offset) + // faster. You can think of this like CPU state (like PC). + // + // We hope that LLVM decides to keep these in registers (as they are touched + // for every instruction executed). The stack_frame will change as we call + // into different functions. + BytecodeReader reader(stack); + RETURN_IF_ERROR(reader.SwitchStackFrame(entry_stack_frame)); + +#define DISPATCH_NEXT() \ + { \ + uint8_t opcode = *reader.AdvanceOffset().ValueOrDie(); \ + DVLOG(1) << "Sequencer dispatching op code: " \ + << GetOpcodeInfo(sequencer_opcode_table(), opcode).mnemonic; \ + goto* kDispatchTable[opcode]; \ + } + +#define DISPATCH_CORE_OPCODE(opcode, body) \ + _dispatch_##opcode : {body} DISPATCH_NEXT() + + DISPATCH_NEXT(); + + DISPATCH_CORE_OPCODE(kConstant, { + ASSIGN_OR_RETURN(auto value, reader.ReadConstant()); + ASSIGN_OR_RETURN(auto* dst_local, reader.ReadLocal()); + // TODO(b/139121143): until we have full command buffers we need to do this. + ASSIGN_OR_RETURN(value.buffer, + placement.device->allocator()->AllocateConstant( + hal::BufferUsage::kConstant | hal::BufferUsage::kAll, + std::move(value.buffer))); + *dst_local = std::move(value); + }); + + DISPATCH_CORE_OPCODE(kCall, { + auto* old_stack_frame = stack->current_frame(); + ASSIGN_OR_RETURN(const auto& target_function, reader.ReadFunction()); + ASSIGN_OR_RETURN(auto* new_stack_frame, stack->PushFrame(target_function)); + RETURN_IF_ERROR( + reader.CopyInputsAndSwitchStackFrame(old_stack_frame, new_stack_frame)); + DVLOG(1) << "Call; stack now: " << stack->DebugString(); + }); + + DISPATCH_CORE_OPCODE(kCallImport, { + auto* old_stack_frame = stack->current_frame(); + ASSIGN_OR_RETURN(const auto* target_function, reader.ReadImportFunction()); + switch (target_function->link_type()) { + case ImportFunction::LinkType::kModule: { + ASSIGN_OR_RETURN(auto* new_stack_frame, + stack->PushFrame(target_function->linked_function())); + RETURN_IF_ERROR(reader.CopyInputsAndSwitchStackFrame(old_stack_frame, + new_stack_frame)); + DVLOG(1) << "Call module import; stack now: " << stack->DebugString(); + break; + } + case ImportFunction::LinkType::kNativeFunction: { + ASSIGN_OR_RETURN(auto* new_stack_frame, + stack->PushFrame(*target_function)); + RETURN_IF_ERROR(reader.CopyInputsAndSwitchStackFrame(old_stack_frame, + new_stack_frame)); + DVLOG(1) << "Call native import; stack now: " << stack->DebugString(); + RETURN_IF_ERROR(CallNativeFunction(stack, *target_function)); + RETURN_IF_ERROR(reader.CopyResultsAndSwitchStackFrame(old_stack_frame, + new_stack_frame)); + RETURN_IF_ERROR(stack->PopFrame()); + DVLOG(1) << "Return from native; stack now: " << stack->DebugString(); + break; + } + } + }); + + DISPATCH_CORE_OPCODE(kCallIndirect, { + return UnimplementedErrorBuilder(ABSL_LOC) << "Unimplemented call_indirect"; + }); + + DISPATCH_CORE_OPCODE(kReturn, { + auto* old_stack_frame = stack->current_frame(); + auto* new_stack_frame = stack->caller_frame(); + if (old_stack_frame == entry_stack_frame) { + // Returning from entry function. Marshal results from the return stmt. + ASSIGN_OR_RETURN(int32_t src_count, reader.ReadCount()); + for (int i = 0; i < src_count; ++i) { + ASSIGN_OR_RETURN(auto* src_local, + reader.ReadLocal(old_stack_frame->mutable_locals())); + entry_results[i] = std::move(*src_local); + } + DVLOG(1) << "Returning to entry"; + return OkStatus(); + } else if (!new_stack_frame) { + return FailedPreconditionErrorBuilder(ABSL_LOC) << "Stack underflow"; + } + RETURN_IF_ERROR(reader.CopyResultsAndSwitchStackFrame(old_stack_frame, + new_stack_frame)); + RETURN_IF_ERROR(stack->PopFrame()); + DVLOG(1) << "Return; stack now: " << stack->DebugString(); + }); + + DISPATCH_CORE_OPCODE(kBranch, { + ASSIGN_OR_RETURN(int32_t offset, reader.ReadBlockOffset()); + RETURN_IF_ERROR(reader.CopySlots()); + RETURN_IF_ERROR(reader.BranchToOffset(offset)); + }); + + DISPATCH_CORE_OPCODE(kCondBranch, { + // Evaluate condition first so we can do the copies as we read them for + // which side of the branch we take. + ASSIGN_OR_RETURN(auto* cond_local, reader.ReadLocal()); + bool cond_value = BufferViewIsTrue(*cond_local); + ASSIGN_OR_RETURN(int32_t true_offset, reader.ReadBlockOffset()); + + if (cond_value) { + RETURN_IF_ERROR(reader.CopySlots()); + RETURN_IF_ERROR(reader.BranchToOffset(true_offset)); + } else { + ASSIGN_OR_RETURN(int32_t true_op_count, reader.ReadCount()); + RETURN_IF_ERROR(reader.SkipLocals(2 * true_op_count)); + ASSIGN_OR_RETURN(int32_t false_offset, reader.ReadBlockOffset()); + + RETURN_IF_ERROR(reader.CopySlots()); + RETURN_IF_ERROR(reader.BranchToOffset(false_offset)); + } + }); + + DISPATCH_CORE_OPCODE(kDynamicDispatch, { + return UnimplementedErrorBuilder(ABSL_LOC) + << "Unimplemented dynamic_dispatch"; + }); + + DISPATCH_CORE_OPCODE(kStaticDispatch, { + // TODO(benvanik): the real sequencer :) + ASSIGN_OR_RETURN(auto dispatch_ordinal, reader.ReadInt32()); + ASSIGN_OR_RETURN(auto export_ordinal, reader.ReadUint16_t()); + auto& executable_table = + stack->current_frame()->module().executable_table(); + ASSIGN_OR_RETURN( + auto* multi_arch_executable_def, + executable_table.LookupMultiArchExecutable(dispatch_ordinal)); + if (export_ordinal >= multi_arch_executable_def->entry_point_count()) { + return InvalidArgumentErrorBuilder(ABSL_LOC) + << "Invalid executable export ordinal " << export_ordinal; + } + auto* executable_def = multi_arch_executable_def->executables()->Get(0); + hal::ExecutableSpec executable_spec; + executable_spec.format = executable_def->format(); + executable_spec.executable_data = absl::Span( + executable_def->contents()->data(), executable_def->contents()->size()); + auto executable_cache = placement.device->CreateExecutableCache(); + ref_ptr executable; + for (auto* executable_def : *multi_arch_executable_def->executables()) { + if (!executable_cache->CanPrepareFormat(executable_def->format())) { + continue; + } + hal::ExecutableSpec executable_spec; + executable_spec.format = executable_def->format(); + executable_spec.executable_data = + absl::Span(executable_def->contents()->data(), + executable_def->contents()->size()); + ASSIGN_OR_RETURN(executable, + executable_cache->PrepareExecutable( + hal::ExecutableCachingMode::kDefault | + hal::ExecutableCachingMode::kAliasProvidedData, + executable_spec), + _.LogError()); + break; + } + if (!executable) { + return InvalidArgumentErrorBuilder(ABSL_LOC) + << "No executable found for the current driver"; + } + + ASSIGN_OR_RETURN(int workload_x, reader.ReadInt32()); + ASSIGN_OR_RETURN(int workload_y, reader.ReadInt32()); + ASSIGN_OR_RETURN(int workload_z, reader.ReadInt32()); + + std::vector bindings; + ASSIGN_OR_RETURN(int input_count, reader.ReadCount()); + for (int i = 0; i < input_count; ++i) { + ASSIGN_OR_RETURN(auto* input_local, reader.ReadLocal()); + bindings.push_back(hal::BufferBinding( + input_local->buffer->allowed_access() & hal::MemoryAccess::kAll, + *input_local)); + } + ASSIGN_OR_RETURN(int output_count, reader.ReadCount()); + for (int i = 0; i < output_count; ++i) { + ASSIGN_OR_RETURN(auto* output_local, reader.ReadLocal()); + bindings.push_back( + hal::BufferBinding(hal::MemoryAccess::kWrite, *output_local)); + } + ASSIGN_OR_RETURN(int result_count, reader.ReadCount()); + CHECK_EQ(0, result_count) << "Results not yet implemented"; + + ASSIGN_OR_RETURN( + auto cmd, + placement.device->CreateCommandBuffer( + hal::CommandBufferMode::kOneShot, + hal::CommandCategory::kTransfer | hal::CommandCategory::kDispatch), + _.LogError()); + RETURN_IF_ERROR(cmd->Begin()); + hal::DispatchRequest dispatch_request; + dispatch_request.executable = executable.get(); + dispatch_request.entry_point = export_ordinal; + dispatch_request.workload[0] = workload_x; + dispatch_request.workload[1] = workload_y; + dispatch_request.workload[2] = workload_z; + dispatch_request.bindings = bindings; + RETURN_IF_ERROR(cmd->Dispatch(dispatch_request)); + RETURN_IF_ERROR(cmd->End()); + auto* cmd_ptr = cmd.get(); + + auto* queue = placement.device->dispatch_queues().front(); + hal::SubmissionBatch batch; + batch.command_buffers = absl::MakeConstSpan(&cmd_ptr, 1); + ASSIGN_OR_RETURN(auto fence, placement.device->CreateFence(0u)); + RETURN_IF_ERROR(queue->Submit(batch, {fence.get(), 1u})); + RETURN_IF_ERROR(placement.device->WaitAllFences({{fence.get(), 1u}}, + absl::InfiniteFuture())); + }); + + DISPATCH_CORE_OPCODE(kAllocStatic, { + return UnimplementedErrorBuilder(ABSL_LOC) << "Unimplemented alloc_static"; + }); + + DISPATCH_CORE_OPCODE(kAllocStack, { + return UnimplementedErrorBuilder(ABSL_LOC) << "Unimplemented alloc_stack"; + }); + + DISPATCH_CORE_OPCODE(kAllocStackInit, { + return UnimplementedErrorBuilder(ABSL_LOC) + << "Unimplemented alloc_stack_init"; + }); + + DISPATCH_CORE_OPCODE(kAllocHeap, { + ASSIGN_OR_RETURN(auto heap_type, reader.ReadInt32()); + ASSIGN_OR_RETURN(auto type, reader.ReadType()); + size_t element_size = type.element_size(); + + // TODO(benvanik): more efficient reading and storage. + size_t element_count = 0; + ASSIGN_OR_RETURN(auto shape, reader.ReadShapePieces(&element_count)); + size_t allocation_size = element_size * element_count; + + ASSIGN_OR_RETURN(auto* dst_local, reader.ReadLocal()); + dst_local->element_size = element_size; + dst_local->shape = shape; + + // TODO(benvanik): pick an allocator and use that instead. + CHECK_EQ(heap_type, 0); + auto* allocator = placement.device->allocator(); + ASSIGN_OR_RETURN( + dst_local->buffer, + allocator->Allocate( + hal::MemoryType::kHostLocal | hal::MemoryType::kDeviceVisible, + hal::BufferUsage::kAll, allocation_size)); + }); + + DISPATCH_CORE_OPCODE(kDiscard, { + // NOTE: if we were an encoder we would actually discard the buffer. + ASSIGN_OR_RETURN(auto* local, reader.ReadLocal()); + *local = {}; + }); + + DISPATCH_CORE_OPCODE(kComputeRange, { + ASSIGN_OR_RETURN(auto shape_data, reader.ReadSlotElements()); + ASSIGN_OR_RETURN(auto element_size, reader.ReadUint8_t()); + ASSIGN_OR_RETURN(auto indices, reader.ReadSlotElements()); + ASSIGN_OR_RETURN(auto lengths, reader.ReadSlotElements()); + ASSIGN_OR_RETURN(auto* dst_offset_local, reader.ReadLocal()); + ASSIGN_OR_RETURN(auto* dst_length_local, reader.ReadLocal()); + + Shape shape(shape_data); + ASSIGN_OR_RETURN(device_size_t dst_offset, + CalculateOffset(indices, shape, element_size)); + RETURN_IF_ERROR( + dst_offset_local->buffer->WriteData(0, &dst_offset, sizeof(int32_t))); + + // A buffer range can only be computed for contiguous memory. To ensure that + // this only requests such, we validate that the offset in the buffer + // between the start and end indices is the same as the requested size. + device_size_t dst_length = element_size; + for (int i = 0; i < lengths.size(); ++i) { + dst_length *= lengths[i]; + indices[i] += lengths[i] - 1; + } + ASSIGN_OR_RETURN(auto end_offset, + CalculateOffset(indices, shape, element_size)); + auto offset_based_length = end_offset - dst_offset + element_size; + if (dst_length != offset_based_length) { + return InvalidArgumentErrorBuilder(ABSL_LOC) + << "Cannot compute range for non-contiguous region of memory;" + << " shape: " << PrettyPrint(shape.subspan()) + << " indices: " << PrettyPrint(indices) + << " lengths: " << PrettyPrint(lengths); + } + RETURN_IF_ERROR( + dst_length_local->buffer->WriteData(0, &dst_length, sizeof(int32_t))); + }); + + DISPATCH_CORE_OPCODE(kShape, { + ASSIGN_OR_RETURN(auto* src_local, reader.ReadLocal()); + ASSIGN_OR_RETURN(auto* dst_local, reader.ReadLocal()); + RETURN_IF_ERROR(dst_local->buffer->WriteData( + 0, src_local->shape.subspan().data(), + src_local->shape.subspan().size() * sizeof(int32_t))); + }); + + DISPATCH_CORE_OPCODE(kLength, { + ASSIGN_OR_RETURN(auto* src_local, reader.ReadLocal()); + ASSIGN_OR_RETURN(auto* dst_local, reader.ReadLocal()); + int32_t length = src_local->shape.element_count(); + RETURN_IF_ERROR(dst_local->buffer->WriteData(0, &length, sizeof(int32_t))); + }); + + DISPATCH_CORE_OPCODE(kDynamicSlice, { + // TODO(b/139299169): implement indirect copies to avoid CPU readback. + return UnimplementedErrorBuilder(ABSL_LOC) << "Unimplemented dynamic_slice"; + }); + + DISPATCH_CORE_OPCODE(kStaticSlice, { + ASSIGN_OR_RETURN(auto* src_local, reader.ReadLocal()); + ASSIGN_OR_RETURN(auto offset, reader.ReadInt32()); + ASSIGN_OR_RETURN(auto length, reader.ReadInt32()); + ASSIGN_OR_RETURN(auto type, reader.ReadType()); + ASSIGN_OR_RETURN(auto shape_data, reader.ReadSlotElements()); + ASSIGN_OR_RETURN(auto* dst_local, reader.ReadLocal()); + Shape new_shape = Shape{shape_data}; + if (new_shape.element_count() * type.element_size() != length) { + return InvalidArgumentErrorBuilder(ABSL_LOC) + << "New element count " << new_shape.element_count() + << " != length slice " << length; + } + ASSIGN_OR_RETURN(dst_local->buffer, + Buffer::Subspan(src_local->buffer, offset, length)); + dst_local->shape = new_shape; + dst_local->element_size = type.element_size(); + }); + + DISPATCH_CORE_OPCODE(kDynamicCopy, { + // TODO(b/139299169): implement indirect copies to avoid CPU readback. + ASSIGN_OR_RETURN(auto* src_local, reader.ReadLocal()); + ASSIGN_OR_RETURN(auto src_offset_span, reader.ReadSlotElements()); + ASSIGN_OR_RETURN(auto* dst_local, reader.ReadLocal()); + ASSIGN_OR_RETURN(auto dst_offset_span, reader.ReadSlotElements()); + ASSIGN_OR_RETURN(auto length_span, reader.ReadSlotElements()); + RETURN_IF_ERROR(dst_local->buffer->CopyData( + dst_offset_span.front(), src_local->buffer.get(), + src_offset_span.front(), length_span.front())); + }); + + DISPATCH_CORE_OPCODE(kStaticCopy, { + ASSIGN_OR_RETURN(auto* src_local, reader.ReadLocal()); + ASSIGN_OR_RETURN(auto src_offset, reader.ReadInt32()); + ASSIGN_OR_RETURN(auto* dst_local, reader.ReadLocal()); + ASSIGN_OR_RETURN(auto dst_offset, reader.ReadInt32()); + ASSIGN_OR_RETURN(auto length, reader.ReadInt32()); + RETURN_IF_ERROR(dst_local->buffer->CopyData( + dst_offset, src_local->buffer.get(), src_offset, length)); + }); + + DISPATCH_CORE_OPCODE(kDynamicFill, { + // TODO(b/139299169): implement indirect fills to avoid CPU readback. + return UnimplementedErrorBuilder(ABSL_LOC) << "Unimplemented dynamic_fill"; + }); + + DISPATCH_CORE_OPCODE(kStaticFill, { + ASSIGN_OR_RETURN(auto value, reader.ReadInt32()); + ASSIGN_OR_RETURN(auto* dst_local, reader.ReadLocal()); + ASSIGN_OR_RETURN(auto dst_offset, reader.ReadInt32()); + ASSIGN_OR_RETURN(auto length, reader.ReadInt32()); + RETURN_IF_ERROR(dst_local->buffer->Fill32(dst_offset, length, value)); + }); + + DISPATCH_CORE_OPCODE(kClone, { + ASSIGN_OR_RETURN(auto* src_local, reader.ReadLocal()); + ASSIGN_OR_RETURN(auto* dst_local, reader.ReadLocal()); + dst_local->element_size = src_local->element_size; + dst_local->shape = src_local->shape; + ASSIGN_OR_RETURN(dst_local->buffer, placement.device->allocator()->Allocate( + src_local->buffer->memory_type(), + src_local->buffer->usage(), + src_local->buffer->byte_length())); + RETURN_IF_ERROR(dst_local->buffer->CopyData(0, src_local->buffer.get())); + }); + + DISPATCH_CORE_OPCODE(kAssign, { + ASSIGN_OR_RETURN(auto* src_local, reader.ReadLocal()); + ASSIGN_OR_RETURN(auto* dst_local, reader.ReadLocal()); + *dst_local = *src_local; + }); + + DISPATCH_CORE_OPCODE(kCondAssign, { + ASSIGN_OR_RETURN(auto* cond_local, reader.ReadLocal()); + ASSIGN_OR_RETURN(auto* lhs_local, reader.ReadLocal()); + ASSIGN_OR_RETURN(auto* rhs_local, reader.ReadLocal()); + ASSIGN_OR_RETURN(auto* dst_local, reader.ReadLocal()); + *dst_local = BufferViewIsTrue(*cond_local) ? *lhs_local : *rhs_local; + }); + + DISPATCH_CORE_OPCODE(kReshape, { + // TODO(benvanik): more logic required if strides differ. + ASSIGN_OR_RETURN(auto* src_local, reader.ReadLocal()); + ASSIGN_OR_RETURN(auto shape_data, reader.ReadSlotElements()); + ASSIGN_OR_RETURN(auto* dst_local, reader.ReadLocal()); + Shape new_shape = Shape{shape_data}; + if (src_local->shape.element_count() != new_shape.element_count()) { + return InvalidArgumentErrorBuilder(ABSL_LOC) + << "New element count " << new_shape.element_count() + << " != source element count " << src_local->shape.element_count(); + } + dst_local->shape = new_shape; + dst_local->buffer = add_ref(src_local->buffer); + dst_local->element_size = src_local->element_size; + }); + + DISPATCH_CORE_OPCODE(kTrace, { + return UnimplementedErrorBuilder(ABSL_LOC) << "Unimplemented trace"; + }); + + DISPATCH_CORE_OPCODE(kBreak, { + return UnimplementedErrorBuilder(ABSL_LOC) << "Unimplemented break"; + }); + + DISPATCH_CORE_OPCODE(kCondBreak, { + return UnimplementedErrorBuilder(ABSL_LOC) << "Unimplemented cond_break"; + }); + +_dispatch_unhandled: + // TODO(benvanik): better tracing. + return UnimplementedErrorBuilder(ABSL_LOC) << "Unknown dispatch opcode"; +} + +} // namespace vm +} // namespace iree diff --git a/vm/sequencer_dispatch.h b/vm/sequencer_dispatch.h new file mode 100644 index 000000000000..116849a4ecdc --- /dev/null +++ b/vm/sequencer_dispatch.h @@ -0,0 +1,35 @@ +// Copyright 2019 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#ifndef THIRD_PARTY_MLIR_EDGE_IREE_VM_SEQUENCER_DISPATCH_H_ +#define THIRD_PARTY_MLIR_EDGE_IREE_VM_SEQUENCER_DISPATCH_H_ + +#include "third_party/mlir_edge/iree/base/status.h" +#include "third_party/mlir_edge/iree/hal/buffer_view.h" +#include "third_party/mlir_edge/iree/hal/device_placement.h" +#include "third_party/mlir_edge/iree/vm/stack.h" +#include "third_party/mlir_edge/iree/vm/stack_frame.h" + +namespace iree { +namespace vm { + +// TODO(benvanik): API that supports yielding. +Status DispatchSequence(const hal::DevicePlacement& placement, Stack* stack, + StackFrame* entry_stack_frame, + absl::Span entry_results); + +} // namespace vm +} // namespace iree + +#endif // THIRD_PARTY_MLIR_EDGE_IREE_VM_SEQUENCER_DISPATCH_H_ diff --git a/vm/source_map.cc b/vm/source_map.cc new file mode 100644 index 000000000000..747e3abc8aa1 --- /dev/null +++ b/vm/source_map.cc @@ -0,0 +1,206 @@ +// Copyright 2019 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "third_party/mlir_edge/iree/vm/source_map.h" + +#include + +#include "third_party/mlir_edge/iree/base/flatbuffer_util.h" +#include "third_party/mlir_edge/iree/base/status.h" + +namespace iree { +namespace vm { + +namespace { + +Status PrintLocation(const SourceMap& source_map, + const FunctionSourceMapDef& function_source_map, + const LocationDef& location, std::ostream* stream); + +Status PrintFileLocation(const SourceMap& source_map, + const FunctionSourceMapDef& function_source_map, + const FileLocationDef& location, + std::ostream* stream) { + ASSIGN_OR_RETURN(auto filename, + source_map.GetUniqueString(location.filename())); + *stream << filename << ":" << location.line() << ":" << location.column(); + return OkStatus(); +} + +Status PrintNameLocation(const SourceMap& source_map, + const FunctionSourceMapDef& function_source_map, + const NameLocationDef& location, + std::ostream* stream) { + ASSIGN_OR_RETURN(auto name, source_map.GetUniqueString(location.name())); + *stream << "\"" << name << "\""; + return OkStatus(); +} + +Status PrintCallSiteLocation(const SourceMap& source_map, + const FunctionSourceMapDef& function_source_map, + const CallSiteLocationDef& location, + std::ostream* stream) { + *stream << "(callsites todo)"; + return OkStatus(); +} + +Status PrintFusedLocation(const SourceMap& source_map, + const FunctionSourceMapDef& function_source_map, + const FusedLocationDef& location, + std::ostream* stream) { + *stream << "fused["; + if (location.locations()) { + for (int i = 0; i < location.locations()->size(); ++i) { + if (i > 0) *stream << ", "; + int location_ordinal = location.locations()->Get(i); + const auto& child_location = + *function_source_map.location_table()->Get(location_ordinal); + RETURN_IF_ERROR(PrintLocation(source_map, function_source_map, + child_location, stream)); + } + } + *stream << "]"; + return OkStatus(); +} + +Status PrintLocation(const SourceMap& source_map, + const FunctionSourceMapDef& function_source_map, + const LocationDef& location, std::ostream* stream) { + switch (location.location_union_type()) { + case LocationDefUnion::FileLocationDef: + return PrintFileLocation(source_map, function_source_map, + *location.location_union_as_FileLocationDef(), + stream); + case LocationDefUnion::NameLocationDef: + return PrintNameLocation(source_map, function_source_map, + *location.location_union_as_NameLocationDef(), + stream); + case LocationDefUnion::CallSiteLocationDef: + return PrintCallSiteLocation( + source_map, function_source_map, + *location.location_union_as_CallSiteLocationDef(), stream); + case LocationDefUnion::FusedLocationDef: + return PrintFusedLocation(source_map, function_source_map, + *location.location_union_as_FusedLocationDef(), + stream); + default: + return UnimplementedErrorBuilder(ABSL_LOC) + << "Unhandled location type " + << static_cast(location.location_union_type()); + } +} + +} // namespace + +// static +bool SourceLocation::Equal(const SourceLocation& a, const SourceLocation& b) { + return a.source_map_def_ == b.source_map_def_ && + a.function_source_map_ == b.function_source_map_ && + a.location_ordinal_ == b.location_ordinal_; +} + +std::string SourceLocation::DebugStringShort() const { + if (empty()) { + return ""; + } + std::ostringstream stream; + const auto& location = + *function_source_map_->location_table()->Get(location_ordinal_); + auto status = PrintLocation(SourceMap(*source_map_def_), + *function_source_map_, location, &stream); + if (!status.ok()) { + stream << status; + } + return stream.str(); +} + +// static +SourceMap SourceMap::FromModule(const ModuleDef& module_def) { + if (module_def.source_map()) { + return SourceMap{*module_def.source_map()}; + } + return {}; +} + +StatusOr SourceMap::GetUniqueString(int string_index) const { + if (empty()) { + return NotFoundErrorBuilder(ABSL_LOC) << "No source map present"; + } + const auto* string_table = source_map_def_->string_table(); + if (string_table && string_table->size() > string_index) { + return WrapString(string_table->Get(string_index)); + } + return NotFoundErrorBuilder(ABSL_LOC) + << "String index " << string_index << " not present in string table"; +} + +StatusOr SourceMap::GetFunctionSourceMap( + int function_ordinal) const { + if (empty()) { + return NotFoundErrorBuilder(ABSL_LOC) << "No source map present"; + } + const auto* function_table = source_map_def_->function_table(); + if (function_table && function_table->size() > function_ordinal) { + const auto* function_source_map = function_table->Get(function_ordinal); + if (function_source_map && function_source_map->location_table() && + function_source_map->bytecode_map()) { + return function_source_map; + } + } + return NotFoundErrorBuilder(ABSL_LOC) + << "Function ordinal " << function_ordinal + << " source map not present in function table"; +} + +// static +SourceMapResolver SourceMapResolver::FromFunction(const ModuleDef& module_def, + int function_ordinal) { + auto source_map = SourceMap::FromModule(module_def); + if (source_map.empty()) { + return {}; + } + auto function_source_map_or = + source_map.GetFunctionSourceMap(function_ordinal); + if (!function_source_map_or.ok()) { + return {}; + } + return SourceMapResolver(source_map, *function_source_map_or.ValueOrDie()); +} + +absl::optional SourceMapResolver::ResolveBytecodeOffset( + int offset) const { + if (!function_source_map_) { + return {}; + } + + const auto* bytecode_map = function_source_map_->bytecode_map(); + + // TODO(benvanik): allow fuzzy offset matching/table sparsity. + int location_ordinal = -1; + for (const auto* map_loc : *bytecode_map) { + if (map_loc->offset() == offset) { + location_ordinal = map_loc->location(); + break; + } + } + if (location_ordinal == -1) { + return {}; + } + + return SourceLocation(*source_map_.def(), *function_source_map_, + location_ordinal); +} + +} // namespace vm +} // namespace iree diff --git a/vm/source_map.h b/vm/source_map.h new file mode 100644 index 000000000000..6a6aa14ada7a --- /dev/null +++ b/vm/source_map.h @@ -0,0 +1,104 @@ +// Copyright 2019 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#ifndef THIRD_PARTY_MLIR_EDGE_IREE_VM_SOURCE_MAP_H_ +#define THIRD_PARTY_MLIR_EDGE_IREE_VM_SOURCE_MAP_H_ + +#include "third_party/absl/strings/string_view.h" +#include "third_party/absl/types/optional.h" +#include "third_party/mlir_edge/iree/base/status.h" +#include "third_party/mlir_edge/iree/schemas/module_def_generated.h" +#include "third_party/mlir_edge/iree/schemas/source_map_def_generated.h" + +namespace iree { +namespace vm { + +class SourceLocation { + public: + static bool Equal(const SourceLocation& a, const SourceLocation& b); + + SourceLocation() = default; + SourceLocation(const SourceMapDef& source_map_def, + const FunctionSourceMapDef& function_source_map, + int location_ordinal) + : source_map_def_(&source_map_def), + function_source_map_(&function_source_map), + location_ordinal_(location_ordinal) {} + + std::string DebugStringShort() const; + + bool empty() const { return source_map_def_ == nullptr; } + + private: + const SourceMapDef* source_map_def_ = nullptr; + const FunctionSourceMapDef* function_source_map_ = nullptr; + int location_ordinal_ = 0; +}; + +inline bool operator==(const SourceLocation& a, const SourceLocation& b) { + return SourceLocation::Equal(a, b); +} + +inline bool operator!=(const SourceLocation& a, const SourceLocation& b) { + return !(a == b); +} + +class SourceMap { + public: + static SourceMap FromModule(const ModuleDef& module_def); + + SourceMap() = default; + explicit SourceMap(const SourceMapDef& source_map_def) + : source_map_def_(&source_map_def) {} + + bool empty() const { return source_map_def_ == nullptr; } + const SourceMapDef* def() const { return source_map_def_; } + + StatusOr GetUniqueString(int string_index) const; + + StatusOr GetFunctionSourceMap( + int function_ordinal) const; + + private: + const SourceMapDef* source_map_def_ = nullptr; +}; +inline std::ostream& operator<<(std::ostream& stream, + const SourceLocation& location) { + stream << location.DebugStringShort(); + return stream; +} + +class SourceMapResolver { + public: + static SourceMapResolver FromFunction(const ModuleDef& module_def, + int function_ordinal); + + SourceMapResolver() = default; + + absl::optional ResolveBytecodeOffset(int offset) const; + + private: + SourceMapResolver(SourceMap source_map, + const FunctionSourceMapDef& function_source_map) + : source_map_(std::move(source_map)), + function_source_map_(&function_source_map) {} + + SourceMap source_map_; + const FunctionSourceMapDef* function_source_map_ = nullptr; +}; + +} // namespace vm +} // namespace iree + +#endif // THIRD_PARTY_MLIR_EDGE_IREE_VM_SOURCE_MAP_H_ diff --git a/vm/stack.cc b/vm/stack.cc new file mode 100644 index 000000000000..ee6840d3f6a2 --- /dev/null +++ b/vm/stack.cc @@ -0,0 +1,82 @@ +// Copyright 2019 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "third_party/mlir_edge/iree/vm/stack.h" + +#include + +#include "third_party/absl/strings/str_join.h" +#include "third_party/mlir_edge/iree/base/status.h" + +namespace iree { +namespace vm { + +constexpr int Stack::kMaxStackDepth; + +Stack::Stack() = default; + +Stack::~Stack() = default; + +StatusOr Stack::PushFrame(Function function) { + if (stack_depth_ + 1 > kMaxStackDepth) { + return InternalErrorBuilder(ABSL_LOC) + << "Max stack depth of " << kMaxStackDepth << " exceeded"; + } + stack_[stack_depth_++] = StackFrame(function); + + // TODO(benvanik): WTF scope enter. + + return current_frame(); +} + +StatusOr Stack::PushFrame(const ImportFunction& function) { + if (stack_depth_ + 1 > kMaxStackDepth) { + return InternalErrorBuilder(ABSL_LOC) + << "Max stack depth of " << kMaxStackDepth << " exceeded"; + } + stack_[stack_depth_++] = StackFrame(function); + + // TODO(benvanik): WTF scope enter. + + return current_frame(); +} + +Status Stack::PopFrame() { + if (stack_depth_ == 0) { + return InternalErrorBuilder(ABSL_LOC) << "Unbalanced stack pop"; + } + + // TODO(benvanik): WTF scope leave. + + --stack_depth_; + return OkStatus(); +} + +namespace { +struct StackFrameFormatter { + void operator()(std::string* out, const StackFrame& stack_frame) const { + out->append(absl::StrCat(stack_frame.module().name(), ":", + stack_frame.function().name(), "@", + stack_frame.offset())); + } +}; +} // namespace + +std::string Stack::DebugString() const { + return absl::StrJoin(std::begin(stack_), std::begin(stack_) + stack_depth_, + "\n", StackFrameFormatter()); +} + +} // namespace vm +} // namespace iree diff --git a/vm/stack.h b/vm/stack.h new file mode 100644 index 000000000000..7ac75d660b90 --- /dev/null +++ b/vm/stack.h @@ -0,0 +1,73 @@ +// Copyright 2019 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#ifndef THIRD_PARTY_MLIR_EDGE_IREE_VM_STACK_H_ +#define THIRD_PARTY_MLIR_EDGE_IREE_VM_STACK_H_ + +#include + +#include "third_party/absl/types/span.h" +#include "third_party/mlir_edge/iree/base/status.h" +#include "third_party/mlir_edge/iree/vm/stack_frame.h" + +namespace iree { +namespace vm { + +// VM call stack. +// +// Stacks are thread-compatible. +class Stack { + public: + static constexpr int kMaxStackDepth = 32; + + Stack(); + Stack(const Stack&) = delete; + Stack& operator=(const Stack&) = delete; + ~Stack(); + + absl::Span frames() const { + return absl::MakeConstSpan(stack_, stack_depth_); + } + absl::Span mutable_frames() { + return absl::MakeSpan(stack_, stack_depth_); + } + + StackFrame* current_frame() { + return stack_depth_ > 0 ? &stack_[stack_depth_ - 1] : nullptr; + } + const StackFrame* current_frame() const { + return stack_depth_ > 0 ? &stack_[stack_depth_ - 1] : nullptr; + } + StackFrame* caller_frame() { + return stack_depth_ > 1 ? &stack_[stack_depth_ - 2] : nullptr; + } + const StackFrame* caller_frame() const { + return stack_depth_ > 1 ? &stack_[stack_depth_ - 2] : nullptr; + } + + StatusOr PushFrame(Function function); + StatusOr PushFrame(const ImportFunction& function); + Status PopFrame(); + + std::string DebugString() const; + + private: + StackFrame stack_[kMaxStackDepth]; + int stack_depth_ = 0; +}; + +} // namespace vm +} // namespace iree + +#endif // THIRD_PARTY_MLIR_EDGE_IREE_VM_STACK_H_ diff --git a/vm/stack_frame.cc b/vm/stack_frame.cc new file mode 100644 index 000000000000..e7a0a9e4f8ca --- /dev/null +++ b/vm/stack_frame.cc @@ -0,0 +1,46 @@ +// Copyright 2019 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "third_party/mlir_edge/iree/vm/stack_frame.h" + +#include "third_party/mlir_edge/iree/base/status.h" + +namespace iree { +namespace vm { + +StackFrame::StackFrame(Function function) : function_(function) { + const auto* bytecode_def = function_.def().bytecode(); + if (bytecode_def) { + offset_limit_ = bytecode_def->contents()->Length(); + locals_.resize(bytecode_def->local_count()); + } else { + locals_.resize(function_.input_count() + function_.result_count()); + } +} + +StackFrame::StackFrame(const ImportFunction& function) + : function_(function), import_function_(&function) {} + +Status StackFrame::set_offset(int offset) { + if (offset < 0 || offset > offset_limit_) { + return OutOfRangeErrorBuilder(ABSL_LOC) + << "Offset " << offset + << " is outside of the bytecode body limit of " << offset_limit_; + } + offset_ = offset; + return OkStatus(); +} + +} // namespace vm +} // namespace iree diff --git a/vm/stack_frame.h b/vm/stack_frame.h new file mode 100644 index 000000000000..19c89c3f226a --- /dev/null +++ b/vm/stack_frame.h @@ -0,0 +1,77 @@ +// Copyright 2019 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#ifndef THIRD_PARTY_MLIR_EDGE_IREE_VM_STACK_FRAME_H_ +#define THIRD_PARTY_MLIR_EDGE_IREE_VM_STACK_FRAME_H_ + +#include + +#include "third_party/absl/types/span.h" +#include "third_party/mlir_edge/iree/hal/buffer_view.h" +#include "third_party/mlir_edge/iree/vm/function.h" +#include "third_party/mlir_edge/iree/vm/module.h" + +namespace iree { +namespace vm { + +// A single frame on the call stack containing current execution state and +// local values. +// +// StackFrames are designed to be serialized so that suspend and resume is +// possible. This means that most state is stored either entirely within the +// frame or references to non-pointer values (such as other function indices). +// BufferViews require special care to allow rendezvous and liveness tracking. +class StackFrame { + public: + StackFrame() = default; + explicit StackFrame(Function function); + explicit StackFrame(const ImportFunction& function); + StackFrame(const StackFrame&) = delete; + StackFrame& operator=(const StackFrame&) = delete; + StackFrame(StackFrame&&) = default; + StackFrame& operator=(StackFrame&&) = default; + + const Module& module() const { return function_.module(); } + const Function& function() const { return function_; } + + inline int offset() const { return offset_; } + Status set_offset(int offset); + inline int* mutable_offset() { return &offset_; } + + inline const hal::BufferView& local(int ordinal) { return locals_[ordinal]; } + inline hal::BufferView* mutable_local(int ordinal) { + return &locals_[ordinal]; + } + + inline absl::Span locals() const { + return absl::MakeConstSpan(locals_); + } + inline absl::Span mutable_locals() { + return absl::MakeSpan(locals_); + } + + private: + Function function_; + const ImportFunction* import_function_; + int offset_ = 0; + int offset_limit_ = 0; + + // TODO(benvanik): replace with a placed allocation. + std::vector locals_; +}; + +} // namespace vm +} // namespace iree + +#endif // THIRD_PARTY_MLIR_EDGE_IREE_VM_STACK_FRAME_H_ diff --git a/vm/type.cc b/vm/type.cc new file mode 100644 index 000000000000..18ca7830fb9b --- /dev/null +++ b/vm/type.cc @@ -0,0 +1,64 @@ +// Copyright 2019 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "third_party/mlir_edge/iree/vm/type.h" + +#include "third_party/mlir_edge/iree/base/status.h" + +namespace iree { +namespace vm { + +// static +StatusOr Type::FromTypeIndex(uint8_t type_index) { + // Currently we only support the builtin types. + if (type_index == static_cast(BuiltinType::kOpaque)) { + return Type(type_index); + } else if (type_index < kBuiltinTypeCount) { + return Type(type_index); + } + return InvalidArgumentErrorBuilder(ABSL_LOC) + << "Type index " << static_cast(type_index) << " not supported"; +} + +// static +const Type Type::FromBuiltin(BuiltinType type) { + return Type(static_cast(type)); +} + +std::string Type::DebugString() const { + switch (type_index_) { +#define TYPE_NAME(index, name, str, size) \ + case index: \ + return str; + IREE_TYPE_LIST(TYPE_NAME) +#undef TYPE_NAME + default: + return ""; + } +} + +size_t Type::element_size() const { + switch (type_index_) { +#define TYPE_SIZE(index, name, str, size) \ + case index: \ + return size; + IREE_TYPE_LIST(TYPE_SIZE) +#undef TYPE_SIZE + default: + return 0; + } +} + +} // namespace vm +} // namespace iree diff --git a/vm/type.h b/vm/type.h new file mode 100644 index 000000000000..c4cd923d2f1d --- /dev/null +++ b/vm/type.h @@ -0,0 +1,65 @@ +// Copyright 2019 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#ifndef THIRD_PARTY_MLIR_EDGE_IREE_VM_TYPE_H_ +#define THIRD_PARTY_MLIR_EDGE_IREE_VM_TYPE_H_ + +#include "third_party/mlir_edge/iree/base/status.h" +#include "third_party/mlir_edge/iree/schemas/bytecode/bytecode_v0.h" +#include "third_party/mlir_edge/iree/schemas/type_def_generated.h" + +namespace iree { +namespace vm { + +class Type { + public: + static StatusOr FromTypeIndex(uint8_t type_index); + static const Type FromBuiltin(BuiltinType type); + + std::string DebugString() const; + + uint8_t type_index() const { return type_index_; } + + bool is_opaque() const { + return type_index_ == static_cast(BuiltinType::kOpaque); + } + bool is_builtin() const { return !is_opaque(); } + BuiltinType builtin_type() const { + DCHECK(is_builtin()); + return static_cast(type_index_); + } + + size_t element_size() const; + + private: + explicit Type(uint8_t type_index) : type_index_(type_index) {} + + uint8_t type_index_; +}; + +inline bool operator==(const Type& a, const Type& b) { + return a.type_index() == b.type_index(); +} + +inline bool operator!=(const Type& a, const Type& b) { return !(a == b); } + +inline std::ostream& operator<<(std::ostream& stream, const Type& type) { + stream << type.DebugString(); + return stream; +} + +} // namespace vm +} // namespace iree + +#endif // THIRD_PARTY_MLIR_EDGE_IREE_VM_TYPE_H_