From 1e7b32490dfdccddd04f46d4b0416b48d749d51b Mon Sep 17 00:00:00 2001 From: kim <89579420+NyaaaWhatsUpDoc@users.noreply.github.com> Date: Mon, 27 May 2024 15:46:15 +0000 Subject: [PATCH] [experiment] add alternative wasm sqlite3 implementation available via build-tag (#2863) This allows for building GoToSocial with [SQLite transpiled to WASM](https://github.com/ncruces/go-sqlite3) and accessed through [Wazero](https://wazero.io/). --- go.mod | 5 +- go.sum | 6 + internal/api/client/media/mediacreate_test.go | 28 +- internal/api/client/media/mediaupdate_test.go | 27 +- internal/api/fileserver/fileserver_test.go | 6 +- internal/config/global.go | 6 + internal/config/state.go | 48 +- internal/db/bundb/admin_test.go | 1 + internal/db/bundb/bundb.go | 25 +- internal/db/bundb/drivers.go | 346 +- internal/db/bundb/errors.go | 105 - internal/db/bundb/tag_test.go | 13 +- internal/db/error.go | 4 - internal/db/postgres/driver.go | 209 + internal/db/postgres/errors.go | 46 + internal/db/sqlite/driver.go | 197 + internal/db/sqlite/driver_wasmsqlite3.go | 211 + internal/db/sqlite/errors.go | 62 + internal/db/sqlite/errors_wasmsqlite3.go | 60 + internal/db/util.go | 35 + internal/gtserror/multi_test.go | 4 - internal/util/paging_test.go | 5 + test/run-postgres.sh | 24 +- testrig/config.go | 263 +- testrig/db.go | 22 +- .../github.com/ncruces/go-sqlite3/.gitignore | 16 + vendor/github.com/ncruces/go-sqlite3/LICENSE | 21 + .../github.com/ncruces/go-sqlite3/README.md | 113 + .../github.com/ncruces/go-sqlite3/backup.go | 134 + vendor/github.com/ncruces/go-sqlite3/blob.go | 250 + .../github.com/ncruces/go-sqlite3/config.go | 164 + vendor/github.com/ncruces/go-sqlite3/conn.go | 426 ++ vendor/github.com/ncruces/go-sqlite3/const.go | 360 ++ .../github.com/ncruces/go-sqlite3/context.go | 229 + .../ncruces/go-sqlite3/driver/driver.go | 579 +++ .../ncruces/go-sqlite3/driver/savepoint.go | 27 + .../ncruces/go-sqlite3/driver/time.go | 31 + .../ncruces/go-sqlite3/driver/util.go | 14 + .../ncruces/go-sqlite3/embed/README.md | 27 + .../ncruces/go-sqlite3/embed/build.sh | 32 + .../ncruces/go-sqlite3/embed/exports.txt | 130 + .../ncruces/go-sqlite3/embed/init.go | 20 + .../ncruces/go-sqlite3/embed/sqlite3.wasm | Bin 0 -> 1365178 bytes vendor/github.com/ncruces/go-sqlite3/error.go | 162 + vendor/github.com/ncruces/go-sqlite3/func.go | 214 + vendor/github.com/ncruces/go-sqlite3/go.work | 6 + .../github.com/ncruces/go-sqlite3/go.work.sum | 9 + .../go-sqlite3/internal/util/alloc_other.go | 9 + .../go-sqlite3/internal/util/alloc_slice.go | 25 + .../go-sqlite3/internal/util/alloc_unix.go | 67 + .../go-sqlite3/internal/util/alloc_windows.go | 76 + .../ncruces/go-sqlite3/internal/util/bool.go | 22 + .../ncruces/go-sqlite3/internal/util/const.go | 117 + .../ncruces/go-sqlite3/internal/util/error.go | 106 + .../ncruces/go-sqlite3/internal/util/func.go | 193 + .../go-sqlite3/internal/util/handle.go | 65 + .../ncruces/go-sqlite3/internal/util/json.go | 35 + .../ncruces/go-sqlite3/internal/util/mem.go | 134 + .../ncruces/go-sqlite3/internal/util/mmap.go | 97 + .../go-sqlite3/internal/util/mmap_other.go | 21 + .../go-sqlite3/internal/util/module.go | 21 + .../go-sqlite3/internal/util/pointer.go | 11 + .../go-sqlite3/internal/util/reflect.go | 10 + vendor/github.com/ncruces/go-sqlite3/json.go | 11 + .../github.com/ncruces/go-sqlite3/pointer.go | 12 + vendor/github.com/ncruces/go-sqlite3/quote.go | 112 + .../github.com/ncruces/go-sqlite3/sqlite.go | 341 ++ vendor/github.com/ncruces/go-sqlite3/stmt.go | 639 +++ vendor/github.com/ncruces/go-sqlite3/time.go | 354 ++ vendor/github.com/ncruces/go-sqlite3/txn.go | 294 ++ .../ncruces/go-sqlite3/util/osutil/open.go | 16 + .../go-sqlite3/util/osutil/open_windows.go | 112 + .../ncruces/go-sqlite3/util/osutil/osfs.go | 33 + .../ncruces/go-sqlite3/util/osutil/osutil.go | 2 + vendor/github.com/ncruces/go-sqlite3/value.go | 236 + .../ncruces/go-sqlite3/vfs/README.md | 86 + .../github.com/ncruces/go-sqlite3/vfs/api.go | 175 + .../ncruces/go-sqlite3/vfs/const.go | 234 + .../github.com/ncruces/go-sqlite3/vfs/file.go | 217 + .../ncruces/go-sqlite3/vfs/filename.go | 174 + .../github.com/ncruces/go-sqlite3/vfs/lock.go | 144 + .../ncruces/go-sqlite3/vfs/lock_other.go | 23 + .../ncruces/go-sqlite3/vfs/memdb/README.md | 9 + .../ncruces/go-sqlite3/vfs/memdb/api.go | 68 + .../ncruces/go-sqlite3/vfs/memdb/memdb.go | 311 ++ .../ncruces/go-sqlite3/vfs/os_bsd.go | 33 + .../ncruces/go-sqlite3/vfs/os_darwin.go | 95 + .../ncruces/go-sqlite3/vfs/os_f2fs_linux.go | 34 + .../ncruces/go-sqlite3/vfs/os_linux.go | 71 + .../ncruces/go-sqlite3/vfs/os_std_access.go | 36 + .../ncruces/go-sqlite3/vfs/os_std_alloc.go | 19 + .../ncruces/go-sqlite3/vfs/os_std_atomic.go | 9 + .../ncruces/go-sqlite3/vfs/os_std_mode.go | 14 + .../ncruces/go-sqlite3/vfs/os_std_sleep.go | 9 + .../ncruces/go-sqlite3/vfs/os_std_sync.go | 9 + .../ncruces/go-sqlite3/vfs/os_unix.go | 33 + .../ncruces/go-sqlite3/vfs/os_unix_lock.go | 106 + .../ncruces/go-sqlite3/vfs/os_windows.go | 186 + .../ncruces/go-sqlite3/vfs/registry.go | 48 + .../github.com/ncruces/go-sqlite3/vfs/shm.go | 173 + .../ncruces/go-sqlite3/vfs/shm_other.go | 21 + .../github.com/ncruces/go-sqlite3/vfs/vfs.go | 459 ++ vendor/github.com/ncruces/go-sqlite3/vtab.go | 663 +++ .../github.com/ncruces/julianday/.gitignore | 15 + vendor/github.com/ncruces/julianday/LICENSE | 21 + vendor/github.com/ncruces/julianday/README.md | 9 + .../github.com/ncruces/julianday/julianday.go | 124 + .../tetratelabs/wazero/.editorconfig | 7 + .../tetratelabs/wazero/.gitattributes | 2 + .../github.com/tetratelabs/wazero/.gitignore | 45 + .../github.com/tetratelabs/wazero/.gitmodules | 3 + .../tetratelabs/wazero/CONTRIBUTING.md | 75 + vendor/github.com/tetratelabs/wazero/LICENSE | 201 + vendor/github.com/tetratelabs/wazero/Makefile | 381 ++ vendor/github.com/tetratelabs/wazero/NOTICE | 2 + .../tetratelabs/wazero/RATIONALE.md | 1587 ++++++ .../github.com/tetratelabs/wazero/README.md | 132 + .../tetratelabs/wazero/api/features.go | 214 + .../github.com/tetratelabs/wazero/api/wasm.go | 762 +++ .../github.com/tetratelabs/wazero/builder.go | 352 ++ vendor/github.com/tetratelabs/wazero/cache.go | 116 + .../github.com/tetratelabs/wazero/codecov.yml | 9 + .../github.com/tetratelabs/wazero/config.go | 876 ++++ .../tetratelabs/wazero/config_supported.go | 14 + .../tetratelabs/wazero/config_unsupported.go | 8 + .../wazero/experimental/checkpoint.go | 48 + .../tetratelabs/wazero/experimental/close.go | 63 + .../wazero/experimental/experimental.go | 41 + .../wazero/experimental/features.go | 15 + .../wazero/experimental/listener.go | 330 ++ .../tetratelabs/wazero/experimental/memory.go | 50 + .../wazero/experimental/sys/dir.go | 92 + .../wazero/experimental/sys/errno.go | 98 + .../wazero/experimental/sys/error.go | 45 + .../wazero/experimental/sys/file.go | 316 ++ .../tetratelabs/wazero/experimental/sys/fs.go | 292 ++ .../wazero/experimental/sys/oflag.go | 70 + .../wazero/experimental/sys/syscall_errno.go | 106 + .../sys/syscall_errno_notwindows.go | 13 + .../sys/syscall_errno_unsupported.go | 7 + .../experimental/sys/syscall_errno_windows.go | 62 + .../wazero/experimental/sys/time.go | 10 + .../wazero/experimental/sys/unimplemented.go | 160 + .../github.com/tetratelabs/wazero/fsconfig.go | 213 + .../wazero/internal/descriptor/table.go | 164 + .../internal/engine/interpreter/compiler.go | 3634 +++++++++++++ .../internal/engine/interpreter/format.go | 22 + .../engine/interpreter/interpreter.go | 4583 +++++++++++++++++ .../internal/engine/interpreter/operations.go | 2812 ++++++++++ .../internal/engine/interpreter/signature.go | 767 +++ .../internal/engine/wazevo/backend/abi.go | 170 + .../internal/engine/wazevo/backend/backend.go | 3 + .../engine/wazevo/backend/compiler.go | 417 ++ .../engine/wazevo/backend/compiler_lower.go | 226 + .../wazevo/backend/executable_context.go | 219 + .../internal/engine/wazevo/backend/go_call.go | 33 + .../engine/wazevo/backend/isa/amd64/abi.go | 186 + .../backend/isa/amd64/abi_entry_amd64.go | 9 + .../backend/isa/amd64/abi_entry_amd64.s | 29 + .../backend/isa/amd64/abi_entry_preamble.go | 248 + .../wazevo/backend/isa/amd64/abi_go_call.go | 443 ++ .../engine/wazevo/backend/isa/amd64/cond.go | 168 + .../engine/wazevo/backend/isa/amd64/ext.go | 35 + .../engine/wazevo/backend/isa/amd64/instr.go | 2472 +++++++++ .../backend/isa/amd64/instr_encoding.go | 1683 ++++++ .../backend/isa/amd64/lower_constant.go | 71 + .../wazevo/backend/isa/amd64/lower_mem.go | 187 + .../wazevo/backend/isa/amd64/machine.go | 3611 +++++++++++++ .../isa/amd64/machine_pro_epi_logue.go | 304 ++ .../backend/isa/amd64/machine_regalloc.go | 153 + .../wazevo/backend/isa/amd64/machine_vec.go | 992 ++++ .../wazevo/backend/isa/amd64/operands.go | 346 ++ .../wazevo/backend/isa/amd64/reflect.go | 11 + .../backend/isa/amd64/reflect_tinygo.go | 11 + .../engine/wazevo/backend/isa/amd64/reg.go | 181 + .../engine/wazevo/backend/isa/amd64/stack.go | 128 + .../engine/wazevo/backend/isa/arm64/abi.go | 332 ++ .../backend/isa/arm64/abi_entry_arm64.go | 9 + .../backend/isa/arm64/abi_entry_arm64.s | 29 + .../backend/isa/arm64/abi_entry_preamble.go | 230 + .../wazevo/backend/isa/arm64/abi_go_call.go | 428 ++ .../engine/wazevo/backend/isa/arm64/cond.go | 215 + .../engine/wazevo/backend/isa/arm64/instr.go | 2545 +++++++++ .../backend/isa/arm64/instr_encoding.go | 2351 +++++++++ .../backend/isa/arm64/lower_constant.go | 301 ++ .../wazevo/backend/isa/arm64/lower_instr.go | 2221 ++++++++ .../backend/isa/arm64/lower_instr_operands.go | 350 ++ .../wazevo/backend/isa/arm64/lower_mem.go | 440 ++ .../wazevo/backend/isa/arm64/machine.go | 515 ++ .../isa/arm64/machine_pro_epi_logue.go | 469 ++ .../backend/isa/arm64/machine_regalloc.go | 152 + .../backend/isa/arm64/machine_relocation.go | 117 + .../engine/wazevo/backend/isa/arm64/reg.go | 397 ++ .../wazevo/backend/isa/arm64/unwind_stack.go | 90 + .../internal/engine/wazevo/backend/machine.go | 100 + .../engine/wazevo/backend/regalloc.go | 319 ++ .../engine/wazevo/backend/regalloc/api.go | 136 + .../engine/wazevo/backend/regalloc/reg.go | 123 + .../wazevo/backend/regalloc/regalloc.go | 1212 +++++ .../engine/wazevo/backend/regalloc/regset.go | 108 + .../internal/engine/wazevo/backend/vdef.go | 43 + .../internal/engine/wazevo/call_engine.go | 722 +++ .../wazero/internal/engine/wazevo/engine.go | 843 +++ .../internal/engine/wazevo/engine_cache.go | 296 ++ .../engine/wazevo/entrypoint_amd64.go | 15 + .../engine/wazevo/entrypoint_arm64.go | 15 + .../engine/wazevo/entrypoint_others.go | 15 + .../engine/wazevo/frontend/frontend.go | 594 +++ .../internal/engine/wazevo/frontend/lower.go | 4268 +++++++++++++++ .../internal/engine/wazevo/frontend/misc.go | 10 + .../engine/wazevo/frontend/sort_id.go | 15 + .../engine/wazevo/frontend/sort_id_old.go | 17 + .../internal/engine/wazevo/hostmodule.go | 82 + .../internal/engine/wazevo/isa_amd64.go | 30 + .../internal/engine/wazevo/isa_arm64.go | 32 + .../internal/engine/wazevo/isa_other.go | 29 + .../wazero/internal/engine/wazevo/memmove.go | 11 + .../internal/engine/wazevo/module_engine.go | 344 ++ .../wazero/internal/engine/wazevo/reflect.go | 11 + .../internal/engine/wazevo/reflect_tinygo.go | 11 + .../internal/engine/wazevo/ssa/basic_block.go | 407 ++ .../engine/wazevo/ssa/basic_block_sort.go | 34 + .../engine/wazevo/ssa/basic_block_sort_old.go | 24 + .../internal/engine/wazevo/ssa/builder.go | 731 +++ .../wazero/internal/engine/wazevo/ssa/cmp.go | 107 + .../internal/engine/wazevo/ssa/funcref.go | 12 + .../engine/wazevo/ssa/instructions.go | 2967 +++++++++++ .../wazero/internal/engine/wazevo/ssa/pass.go | 417 ++ .../engine/wazevo/ssa/pass_blk_layouts.go | 335 ++ .../internal/engine/wazevo/ssa/pass_cfg.go | 312 ++ .../internal/engine/wazevo/ssa/signature.go | 49 + .../wazero/internal/engine/wazevo/ssa/ssa.go | 14 + .../wazero/internal/engine/wazevo/ssa/type.go | 112 + .../wazero/internal/engine/wazevo/ssa/vs.go | 87 + .../engine/wazevo/wazevoapi/debug_options.go | 196 + .../engine/wazevo/wazevoapi/exitcode.go | 109 + .../engine/wazevo/wazevoapi/offsetdata.go | 216 + .../engine/wazevo/wazevoapi/perfmap.go | 96 + .../wazevo/wazevoapi/perfmap_disabled.go | 5 + .../wazevo/wazevoapi/perfmap_enabled.go | 5 + .../internal/engine/wazevo/wazevoapi/pool.go | 215 + .../internal/engine/wazevo/wazevoapi/ptr.go | 15 + .../internal/engine/wazevo/wazevoapi/queue.go | 26 + .../engine/wazevo/wazevoapi/resetmap.go | 13 + .../wazero/internal/expctxkeys/checkpoint.go | 10 + .../wazero/internal/expctxkeys/close.go | 5 + .../wazero/internal/expctxkeys/expctxkeys.go | 2 + .../wazero/internal/expctxkeys/listener.go | 7 + .../wazero/internal/expctxkeys/memory.go | 4 + .../internal/filecache/compilationcache.go | 42 + .../wazero/internal/filecache/file_cache.go | 76 + .../tetratelabs/wazero/internal/fsapi/file.go | 69 + .../tetratelabs/wazero/internal/fsapi/poll.go | 20 + .../wazero/internal/fsapi/unimplemented.go | 27 + .../wazero/internal/ieee754/ieee754.go | 29 + .../wazero/internal/internalapi/internal.go | 9 + .../wazero/internal/leb128/leb128.go | 285 + .../wazero/internal/moremath/moremath.go | 271 + .../wazero/internal/platform/cpuid.go | 25 + .../wazero/internal/platform/cpuid_amd64.go | 59 + .../wazero/internal/platform/cpuid_amd64.s | 14 + .../internal/platform/cpuid_unsupported.go | 14 + .../wazero/internal/platform/crypto.go | 17 + .../wazero/internal/platform/mmap_linux.go | 76 + .../wazero/internal/platform/mmap_other.go | 18 + .../wazero/internal/platform/mmap_unix.go | 49 + .../internal/platform/mmap_unsupported.go | 28 + .../wazero/internal/platform/mmap_windows.go | 97 + .../wazero/internal/platform/mremap_other.go | 23 + .../wazero/internal/platform/mremap_unix.go | 21 + .../wazero/internal/platform/path.go | 6 + .../wazero/internal/platform/path_windows.go | 17 + .../wazero/internal/platform/platform.go | 81 + .../internal/platform/platform_amd64.go | 7 + .../internal/platform/platform_arm64.go | 7 + .../wazero/internal/platform/time.go | 76 + .../wazero/internal/platform/time_cgo.go | 11 + .../wazero/internal/platform/time_notcgo.go | 7 + .../wazero/internal/platform/time_windows.go | 40 + .../tetratelabs/wazero/internal/sock/sock.go | 89 + .../wazero/internal/sock/sock_supported.go | 11 + .../wazero/internal/sock/sock_unsupported.go | 10 + .../tetratelabs/wazero/internal/sys/fs.go | 457 ++ .../tetratelabs/wazero/internal/sys/lazy.go | 151 + .../tetratelabs/wazero/internal/sys/stdio.go | 128 + .../tetratelabs/wazero/internal/sys/sys.go | 228 + .../wazero/internal/sysfs/adapter.go | 105 + .../wazero/internal/sysfs/datasync_linux.go | 14 + .../wazero/internal/sysfs/datasync_tinygo.go | 13 + .../internal/sysfs/datasync_unsupported.go | 14 + .../tetratelabs/wazero/internal/sysfs/dir.go | 24 + .../wazero/internal/sysfs/dirfs.go | 99 + .../wazero/internal/sysfs/dirfs_supported.go | 42 + .../internal/sysfs/dirfs_unsupported.go | 34 + .../tetratelabs/wazero/internal/sysfs/file.go | 520 ++ .../wazero/internal/sysfs/file_unix.go | 39 + .../wazero/internal/sysfs/file_unsupported.go | 28 + .../wazero/internal/sysfs/file_windows.go | 175 + .../wazero/internal/sysfs/futimens.go | 37 + .../wazero/internal/sysfs/futimens_darwin.go | 51 + .../wazero/internal/sysfs/futimens_darwin.s | 8 + .../wazero/internal/sysfs/futimens_linux.go | 49 + .../internal/sysfs/futimens_unsupported.go | 18 + .../wazero/internal/sysfs/futimens_windows.go | 42 + .../tetratelabs/wazero/internal/sysfs/ino.go | 22 + .../wazero/internal/sysfs/ino_plan9.go | 15 + .../wazero/internal/sysfs/ino_tinygo.go | 14 + .../wazero/internal/sysfs/ino_windows.go | 28 + .../wazero/internal/sysfs/nonblock_unix.go | 17 + .../internal/sysfs/nonblock_unsupported.go | 13 + .../wazero/internal/sysfs/nonblock_windows.go | 23 + .../wazero/internal/sysfs/oflag.go | 38 + .../wazero/internal/sysfs/open_file_darwin.go | 26 + .../internal/sysfs/open_file_freebsd.go | 24 + .../wazero/internal/sysfs/open_file_linux.go | 30 + .../internal/sysfs/open_file_notwindows.go | 20 + .../wazero/internal/sysfs/open_file_sun.go | 31 + .../wazero/internal/sysfs/open_file_tinygo.go | 25 + .../internal/sysfs/open_file_unsupported.go | 18 + .../internal/sysfs/open_file_windows.go | 161 + .../wazero/internal/sysfs/osfile.go | 295 ++ .../tetratelabs/wazero/internal/sysfs/poll.go | 18 + .../wazero/internal/sysfs/poll_darwin.go | 55 + .../wazero/internal/sysfs/poll_darwin.s | 8 + .../wazero/internal/sysfs/poll_linux.go | 59 + .../wazero/internal/sysfs/poll_unsupported.go | 13 + .../wazero/internal/sysfs/poll_windows.go | 224 + .../wazero/internal/sysfs/readfs.go | 117 + .../wazero/internal/sysfs/rename.go | 16 + .../wazero/internal/sysfs/rename_plan9.go | 14 + .../wazero/internal/sysfs/rename_windows.go | 55 + .../tetratelabs/wazero/internal/sysfs/sock.go | 187 + .../wazero/internal/sysfs/sock_supported.go | 77 + .../wazero/internal/sysfs/sock_unix.go | 49 + .../wazero/internal/sysfs/sock_unsupported.go | 81 + .../wazero/internal/sysfs/sock_windows.go | 80 + .../tetratelabs/wazero/internal/sysfs/stat.go | 16 + .../wazero/internal/sysfs/stat_bsd.go | 37 + .../wazero/internal/sysfs/stat_linux.go | 40 + .../wazero/internal/sysfs/stat_unsupported.go | 40 + .../wazero/internal/sysfs/stat_windows.go | 120 + .../tetratelabs/wazero/internal/sysfs/sync.go | 13 + .../wazero/internal/sysfs/sync_windows.go | 20 + .../wazero/internal/sysfs/syscall6_darwin.go | 13 + .../wazero/internal/sysfs/sysfs.go | 6 + .../wazero/internal/sysfs/unlink.go | 17 + .../wazero/internal/sysfs/unlink_plan9.go | 12 + .../wazero/internal/sysfs/unlink_windows.go | 25 + .../tetratelabs/wazero/internal/u32/u32.go | 11 + .../tetratelabs/wazero/internal/u64/u64.go | 15 + .../wazero/internal/version/version.go | 52 + .../wazero/internal/wasm/binary/code.go | 100 + .../wazero/internal/wasm/binary/const_expr.go | 105 + .../wazero/internal/wasm/binary/custom.go | 22 + .../wazero/internal/wasm/binary/data.go | 79 + .../wazero/internal/wasm/binary/decoder.go | 193 + .../wazero/internal/wasm/binary/element.go | 269 + .../wazero/internal/wasm/binary/errors.go | 11 + .../wazero/internal/wasm/binary/export.go | 32 + .../wazero/internal/wasm/binary/function.go | 56 + .../wazero/internal/wasm/binary/global.go | 50 + .../wazero/internal/wasm/binary/header.go | 9 + .../wazero/internal/wasm/binary/import.go | 52 + .../wazero/internal/wasm/binary/limits.go | 47 + .../wazero/internal/wasm/binary/memory.go | 42 + .../wazero/internal/wasm/binary/names.go | 151 + .../wazero/internal/wasm/binary/section.go | 226 + .../wazero/internal/wasm/binary/table.go | 43 + .../wazero/internal/wasm/binary/value.go | 60 + .../wazero/internal/wasm/counts.go | 51 + .../wazero/internal/wasm/engine.go | 72 + .../wazero/internal/wasm/func_validation.go | 2340 +++++++++ .../internal/wasm/function_definition.go | 188 + .../wazero/internal/wasm/global.go | 55 + .../wazero/internal/wasm/gofunc.go | 279 + .../tetratelabs/wazero/internal/wasm/host.go | 179 + .../wazero/internal/wasm/instruction.go | 1866 +++++++ .../wazero/internal/wasm/memory.go | 461 ++ .../wazero/internal/wasm/memory_definition.go | 128 + .../wazero/internal/wasm/module.go | 1083 ++++ .../wazero/internal/wasm/module_instance.go | 251 + .../internal/wasm/module_instance_lookup.go | 73 + .../tetratelabs/wazero/internal/wasm/store.go | 668 +++ .../wazero/internal/wasm/store_module_list.go | 97 + .../tetratelabs/wazero/internal/wasm/table.go | 339 ++ .../wazero/internal/wasmdebug/debug.go | 170 + .../wazero/internal/wasmdebug/dwarf.go | 226 + .../wazero/internal/wasmruntime/errors.go | 50 + .../tetratelabs/wazero/netlify.toml | 15 + .../github.com/tetratelabs/wazero/runtime.go | 374 ++ .../tetratelabs/wazero/sys/clock.go | 26 + .../tetratelabs/wazero/sys/error.go | 83 + .../github.com/tetratelabs/wazero/sys/stat.go | 107 + .../tetratelabs/wazero/sys/stat_bsd.go | 29 + .../tetratelabs/wazero/sys/stat_linux.go | 32 + .../wazero/sys/stat_unsupported.go | 17 + .../tetratelabs/wazero/sys/stat_windows.go | 26 + vendor/modules.txt | 49 +- 398 files changed, 86174 insertions(+), 684 deletions(-) delete mode 100644 internal/db/bundb/errors.go create mode 100644 internal/db/postgres/driver.go create mode 100644 internal/db/postgres/errors.go create mode 100644 internal/db/sqlite/driver.go create mode 100644 internal/db/sqlite/driver_wasmsqlite3.go create mode 100644 internal/db/sqlite/errors.go create mode 100644 internal/db/sqlite/errors_wasmsqlite3.go create mode 100644 internal/db/util.go create mode 100644 vendor/github.com/ncruces/go-sqlite3/.gitignore create mode 100644 vendor/github.com/ncruces/go-sqlite3/LICENSE create mode 100644 vendor/github.com/ncruces/go-sqlite3/README.md create mode 100644 vendor/github.com/ncruces/go-sqlite3/backup.go create mode 100644 vendor/github.com/ncruces/go-sqlite3/blob.go create mode 100644 vendor/github.com/ncruces/go-sqlite3/config.go create mode 100644 vendor/github.com/ncruces/go-sqlite3/conn.go create mode 100644 vendor/github.com/ncruces/go-sqlite3/const.go create mode 100644 vendor/github.com/ncruces/go-sqlite3/context.go create mode 100644 vendor/github.com/ncruces/go-sqlite3/driver/driver.go create mode 100644 vendor/github.com/ncruces/go-sqlite3/driver/savepoint.go create mode 100644 vendor/github.com/ncruces/go-sqlite3/driver/time.go create mode 100644 vendor/github.com/ncruces/go-sqlite3/driver/util.go create mode 100644 vendor/github.com/ncruces/go-sqlite3/embed/README.md create mode 100644 vendor/github.com/ncruces/go-sqlite3/embed/build.sh create mode 100644 vendor/github.com/ncruces/go-sqlite3/embed/exports.txt create mode 100644 vendor/github.com/ncruces/go-sqlite3/embed/init.go create mode 100644 vendor/github.com/ncruces/go-sqlite3/embed/sqlite3.wasm create mode 100644 vendor/github.com/ncruces/go-sqlite3/error.go create mode 100644 vendor/github.com/ncruces/go-sqlite3/func.go create mode 100644 vendor/github.com/ncruces/go-sqlite3/go.work create mode 100644 vendor/github.com/ncruces/go-sqlite3/go.work.sum create mode 100644 vendor/github.com/ncruces/go-sqlite3/internal/util/alloc_other.go create mode 100644 vendor/github.com/ncruces/go-sqlite3/internal/util/alloc_slice.go create mode 100644 vendor/github.com/ncruces/go-sqlite3/internal/util/alloc_unix.go create mode 100644 vendor/github.com/ncruces/go-sqlite3/internal/util/alloc_windows.go create mode 100644 vendor/github.com/ncruces/go-sqlite3/internal/util/bool.go create mode 100644 vendor/github.com/ncruces/go-sqlite3/internal/util/const.go create mode 100644 vendor/github.com/ncruces/go-sqlite3/internal/util/error.go create mode 100644 vendor/github.com/ncruces/go-sqlite3/internal/util/func.go create mode 100644 vendor/github.com/ncruces/go-sqlite3/internal/util/handle.go create mode 100644 vendor/github.com/ncruces/go-sqlite3/internal/util/json.go create mode 100644 vendor/github.com/ncruces/go-sqlite3/internal/util/mem.go create mode 100644 vendor/github.com/ncruces/go-sqlite3/internal/util/mmap.go create mode 100644 vendor/github.com/ncruces/go-sqlite3/internal/util/mmap_other.go create mode 100644 vendor/github.com/ncruces/go-sqlite3/internal/util/module.go create mode 100644 vendor/github.com/ncruces/go-sqlite3/internal/util/pointer.go create mode 100644 vendor/github.com/ncruces/go-sqlite3/internal/util/reflect.go create mode 100644 vendor/github.com/ncruces/go-sqlite3/json.go create mode 100644 vendor/github.com/ncruces/go-sqlite3/pointer.go create mode 100644 vendor/github.com/ncruces/go-sqlite3/quote.go create mode 100644 vendor/github.com/ncruces/go-sqlite3/sqlite.go create mode 100644 vendor/github.com/ncruces/go-sqlite3/stmt.go create mode 100644 vendor/github.com/ncruces/go-sqlite3/time.go create mode 100644 vendor/github.com/ncruces/go-sqlite3/txn.go create mode 100644 vendor/github.com/ncruces/go-sqlite3/util/osutil/open.go create mode 100644 vendor/github.com/ncruces/go-sqlite3/util/osutil/open_windows.go create mode 100644 vendor/github.com/ncruces/go-sqlite3/util/osutil/osfs.go create mode 100644 vendor/github.com/ncruces/go-sqlite3/util/osutil/osutil.go create mode 100644 vendor/github.com/ncruces/go-sqlite3/value.go create mode 100644 vendor/github.com/ncruces/go-sqlite3/vfs/README.md create mode 100644 vendor/github.com/ncruces/go-sqlite3/vfs/api.go create mode 100644 vendor/github.com/ncruces/go-sqlite3/vfs/const.go create mode 100644 vendor/github.com/ncruces/go-sqlite3/vfs/file.go create mode 100644 vendor/github.com/ncruces/go-sqlite3/vfs/filename.go create mode 100644 vendor/github.com/ncruces/go-sqlite3/vfs/lock.go create mode 100644 vendor/github.com/ncruces/go-sqlite3/vfs/lock_other.go create mode 100644 vendor/github.com/ncruces/go-sqlite3/vfs/memdb/README.md create mode 100644 vendor/github.com/ncruces/go-sqlite3/vfs/memdb/api.go create mode 100644 vendor/github.com/ncruces/go-sqlite3/vfs/memdb/memdb.go create mode 100644 vendor/github.com/ncruces/go-sqlite3/vfs/os_bsd.go create mode 100644 vendor/github.com/ncruces/go-sqlite3/vfs/os_darwin.go create mode 100644 vendor/github.com/ncruces/go-sqlite3/vfs/os_f2fs_linux.go create mode 100644 vendor/github.com/ncruces/go-sqlite3/vfs/os_linux.go create mode 100644 vendor/github.com/ncruces/go-sqlite3/vfs/os_std_access.go create mode 100644 vendor/github.com/ncruces/go-sqlite3/vfs/os_std_alloc.go create mode 100644 vendor/github.com/ncruces/go-sqlite3/vfs/os_std_atomic.go create mode 100644 vendor/github.com/ncruces/go-sqlite3/vfs/os_std_mode.go create mode 100644 vendor/github.com/ncruces/go-sqlite3/vfs/os_std_sleep.go create mode 100644 vendor/github.com/ncruces/go-sqlite3/vfs/os_std_sync.go create mode 100644 vendor/github.com/ncruces/go-sqlite3/vfs/os_unix.go create mode 100644 vendor/github.com/ncruces/go-sqlite3/vfs/os_unix_lock.go create mode 100644 vendor/github.com/ncruces/go-sqlite3/vfs/os_windows.go create mode 100644 vendor/github.com/ncruces/go-sqlite3/vfs/registry.go create mode 100644 vendor/github.com/ncruces/go-sqlite3/vfs/shm.go create mode 100644 vendor/github.com/ncruces/go-sqlite3/vfs/shm_other.go create mode 100644 vendor/github.com/ncruces/go-sqlite3/vfs/vfs.go create mode 100644 vendor/github.com/ncruces/go-sqlite3/vtab.go create mode 100644 vendor/github.com/ncruces/julianday/.gitignore create mode 100644 vendor/github.com/ncruces/julianday/LICENSE create mode 100644 vendor/github.com/ncruces/julianday/README.md create mode 100644 vendor/github.com/ncruces/julianday/julianday.go create mode 100644 vendor/github.com/tetratelabs/wazero/.editorconfig create mode 100644 vendor/github.com/tetratelabs/wazero/.gitattributes create mode 100644 vendor/github.com/tetratelabs/wazero/.gitignore create mode 100644 vendor/github.com/tetratelabs/wazero/.gitmodules create mode 100644 vendor/github.com/tetratelabs/wazero/CONTRIBUTING.md create mode 100644 vendor/github.com/tetratelabs/wazero/LICENSE create mode 100644 vendor/github.com/tetratelabs/wazero/Makefile create mode 100644 vendor/github.com/tetratelabs/wazero/NOTICE create mode 100644 vendor/github.com/tetratelabs/wazero/RATIONALE.md create mode 100644 vendor/github.com/tetratelabs/wazero/README.md create mode 100644 vendor/github.com/tetratelabs/wazero/api/features.go create mode 100644 vendor/github.com/tetratelabs/wazero/api/wasm.go create mode 100644 vendor/github.com/tetratelabs/wazero/builder.go create mode 100644 vendor/github.com/tetratelabs/wazero/cache.go create mode 100644 vendor/github.com/tetratelabs/wazero/codecov.yml create mode 100644 vendor/github.com/tetratelabs/wazero/config.go create mode 100644 vendor/github.com/tetratelabs/wazero/config_supported.go create mode 100644 vendor/github.com/tetratelabs/wazero/config_unsupported.go create mode 100644 vendor/github.com/tetratelabs/wazero/experimental/checkpoint.go create mode 100644 vendor/github.com/tetratelabs/wazero/experimental/close.go create mode 100644 vendor/github.com/tetratelabs/wazero/experimental/experimental.go create mode 100644 vendor/github.com/tetratelabs/wazero/experimental/features.go create mode 100644 vendor/github.com/tetratelabs/wazero/experimental/listener.go create mode 100644 vendor/github.com/tetratelabs/wazero/experimental/memory.go create mode 100644 vendor/github.com/tetratelabs/wazero/experimental/sys/dir.go create mode 100644 vendor/github.com/tetratelabs/wazero/experimental/sys/errno.go create mode 100644 vendor/github.com/tetratelabs/wazero/experimental/sys/error.go create mode 100644 vendor/github.com/tetratelabs/wazero/experimental/sys/file.go create mode 100644 vendor/github.com/tetratelabs/wazero/experimental/sys/fs.go create mode 100644 vendor/github.com/tetratelabs/wazero/experimental/sys/oflag.go create mode 100644 vendor/github.com/tetratelabs/wazero/experimental/sys/syscall_errno.go create mode 100644 vendor/github.com/tetratelabs/wazero/experimental/sys/syscall_errno_notwindows.go create mode 100644 vendor/github.com/tetratelabs/wazero/experimental/sys/syscall_errno_unsupported.go create mode 100644 vendor/github.com/tetratelabs/wazero/experimental/sys/syscall_errno_windows.go create mode 100644 vendor/github.com/tetratelabs/wazero/experimental/sys/time.go create mode 100644 vendor/github.com/tetratelabs/wazero/experimental/sys/unimplemented.go create mode 100644 vendor/github.com/tetratelabs/wazero/fsconfig.go create mode 100644 vendor/github.com/tetratelabs/wazero/internal/descriptor/table.go create mode 100644 vendor/github.com/tetratelabs/wazero/internal/engine/interpreter/compiler.go create mode 100644 vendor/github.com/tetratelabs/wazero/internal/engine/interpreter/format.go create mode 100644 vendor/github.com/tetratelabs/wazero/internal/engine/interpreter/interpreter.go create mode 100644 vendor/github.com/tetratelabs/wazero/internal/engine/interpreter/operations.go create mode 100644 vendor/github.com/tetratelabs/wazero/internal/engine/interpreter/signature.go create mode 100644 vendor/github.com/tetratelabs/wazero/internal/engine/wazevo/backend/abi.go create mode 100644 vendor/github.com/tetratelabs/wazero/internal/engine/wazevo/backend/backend.go create mode 100644 vendor/github.com/tetratelabs/wazero/internal/engine/wazevo/backend/compiler.go create mode 100644 vendor/github.com/tetratelabs/wazero/internal/engine/wazevo/backend/compiler_lower.go create mode 100644 vendor/github.com/tetratelabs/wazero/internal/engine/wazevo/backend/executable_context.go create mode 100644 vendor/github.com/tetratelabs/wazero/internal/engine/wazevo/backend/go_call.go create mode 100644 vendor/github.com/tetratelabs/wazero/internal/engine/wazevo/backend/isa/amd64/abi.go create mode 100644 vendor/github.com/tetratelabs/wazero/internal/engine/wazevo/backend/isa/amd64/abi_entry_amd64.go create mode 100644 vendor/github.com/tetratelabs/wazero/internal/engine/wazevo/backend/isa/amd64/abi_entry_amd64.s create mode 100644 vendor/github.com/tetratelabs/wazero/internal/engine/wazevo/backend/isa/amd64/abi_entry_preamble.go create mode 100644 vendor/github.com/tetratelabs/wazero/internal/engine/wazevo/backend/isa/amd64/abi_go_call.go create mode 100644 vendor/github.com/tetratelabs/wazero/internal/engine/wazevo/backend/isa/amd64/cond.go create mode 100644 vendor/github.com/tetratelabs/wazero/internal/engine/wazevo/backend/isa/amd64/ext.go create mode 100644 vendor/github.com/tetratelabs/wazero/internal/engine/wazevo/backend/isa/amd64/instr.go create mode 100644 vendor/github.com/tetratelabs/wazero/internal/engine/wazevo/backend/isa/amd64/instr_encoding.go create mode 100644 vendor/github.com/tetratelabs/wazero/internal/engine/wazevo/backend/isa/amd64/lower_constant.go create mode 100644 vendor/github.com/tetratelabs/wazero/internal/engine/wazevo/backend/isa/amd64/lower_mem.go create mode 100644 vendor/github.com/tetratelabs/wazero/internal/engine/wazevo/backend/isa/amd64/machine.go create mode 100644 vendor/github.com/tetratelabs/wazero/internal/engine/wazevo/backend/isa/amd64/machine_pro_epi_logue.go create mode 100644 vendor/github.com/tetratelabs/wazero/internal/engine/wazevo/backend/isa/amd64/machine_regalloc.go create mode 100644 vendor/github.com/tetratelabs/wazero/internal/engine/wazevo/backend/isa/amd64/machine_vec.go create mode 100644 vendor/github.com/tetratelabs/wazero/internal/engine/wazevo/backend/isa/amd64/operands.go create mode 100644 vendor/github.com/tetratelabs/wazero/internal/engine/wazevo/backend/isa/amd64/reflect.go create mode 100644 vendor/github.com/tetratelabs/wazero/internal/engine/wazevo/backend/isa/amd64/reflect_tinygo.go create mode 100644 vendor/github.com/tetratelabs/wazero/internal/engine/wazevo/backend/isa/amd64/reg.go create mode 100644 vendor/github.com/tetratelabs/wazero/internal/engine/wazevo/backend/isa/amd64/stack.go create mode 100644 vendor/github.com/tetratelabs/wazero/internal/engine/wazevo/backend/isa/arm64/abi.go create mode 100644 vendor/github.com/tetratelabs/wazero/internal/engine/wazevo/backend/isa/arm64/abi_entry_arm64.go create mode 100644 vendor/github.com/tetratelabs/wazero/internal/engine/wazevo/backend/isa/arm64/abi_entry_arm64.s create mode 100644 vendor/github.com/tetratelabs/wazero/internal/engine/wazevo/backend/isa/arm64/abi_entry_preamble.go create mode 100644 vendor/github.com/tetratelabs/wazero/internal/engine/wazevo/backend/isa/arm64/abi_go_call.go create mode 100644 vendor/github.com/tetratelabs/wazero/internal/engine/wazevo/backend/isa/arm64/cond.go create mode 100644 vendor/github.com/tetratelabs/wazero/internal/engine/wazevo/backend/isa/arm64/instr.go create mode 100644 vendor/github.com/tetratelabs/wazero/internal/engine/wazevo/backend/isa/arm64/instr_encoding.go create mode 100644 vendor/github.com/tetratelabs/wazero/internal/engine/wazevo/backend/isa/arm64/lower_constant.go create mode 100644 vendor/github.com/tetratelabs/wazero/internal/engine/wazevo/backend/isa/arm64/lower_instr.go create mode 100644 vendor/github.com/tetratelabs/wazero/internal/engine/wazevo/backend/isa/arm64/lower_instr_operands.go create mode 100644 vendor/github.com/tetratelabs/wazero/internal/engine/wazevo/backend/isa/arm64/lower_mem.go create mode 100644 vendor/github.com/tetratelabs/wazero/internal/engine/wazevo/backend/isa/arm64/machine.go create mode 100644 vendor/github.com/tetratelabs/wazero/internal/engine/wazevo/backend/isa/arm64/machine_pro_epi_logue.go create mode 100644 vendor/github.com/tetratelabs/wazero/internal/engine/wazevo/backend/isa/arm64/machine_regalloc.go create mode 100644 vendor/github.com/tetratelabs/wazero/internal/engine/wazevo/backend/isa/arm64/machine_relocation.go create mode 100644 vendor/github.com/tetratelabs/wazero/internal/engine/wazevo/backend/isa/arm64/reg.go create mode 100644 vendor/github.com/tetratelabs/wazero/internal/engine/wazevo/backend/isa/arm64/unwind_stack.go create mode 100644 vendor/github.com/tetratelabs/wazero/internal/engine/wazevo/backend/machine.go create mode 100644 vendor/github.com/tetratelabs/wazero/internal/engine/wazevo/backend/regalloc.go create mode 100644 vendor/github.com/tetratelabs/wazero/internal/engine/wazevo/backend/regalloc/api.go create mode 100644 vendor/github.com/tetratelabs/wazero/internal/engine/wazevo/backend/regalloc/reg.go create mode 100644 vendor/github.com/tetratelabs/wazero/internal/engine/wazevo/backend/regalloc/regalloc.go create mode 100644 vendor/github.com/tetratelabs/wazero/internal/engine/wazevo/backend/regalloc/regset.go create mode 100644 vendor/github.com/tetratelabs/wazero/internal/engine/wazevo/backend/vdef.go create mode 100644 vendor/github.com/tetratelabs/wazero/internal/engine/wazevo/call_engine.go create mode 100644 vendor/github.com/tetratelabs/wazero/internal/engine/wazevo/engine.go create mode 100644 vendor/github.com/tetratelabs/wazero/internal/engine/wazevo/engine_cache.go create mode 100644 vendor/github.com/tetratelabs/wazero/internal/engine/wazevo/entrypoint_amd64.go create mode 100644 vendor/github.com/tetratelabs/wazero/internal/engine/wazevo/entrypoint_arm64.go create mode 100644 vendor/github.com/tetratelabs/wazero/internal/engine/wazevo/entrypoint_others.go create mode 100644 vendor/github.com/tetratelabs/wazero/internal/engine/wazevo/frontend/frontend.go create mode 100644 vendor/github.com/tetratelabs/wazero/internal/engine/wazevo/frontend/lower.go create mode 100644 vendor/github.com/tetratelabs/wazero/internal/engine/wazevo/frontend/misc.go create mode 100644 vendor/github.com/tetratelabs/wazero/internal/engine/wazevo/frontend/sort_id.go create mode 100644 vendor/github.com/tetratelabs/wazero/internal/engine/wazevo/frontend/sort_id_old.go create mode 100644 vendor/github.com/tetratelabs/wazero/internal/engine/wazevo/hostmodule.go create mode 100644 vendor/github.com/tetratelabs/wazero/internal/engine/wazevo/isa_amd64.go create mode 100644 vendor/github.com/tetratelabs/wazero/internal/engine/wazevo/isa_arm64.go create mode 100644 vendor/github.com/tetratelabs/wazero/internal/engine/wazevo/isa_other.go create mode 100644 vendor/github.com/tetratelabs/wazero/internal/engine/wazevo/memmove.go create mode 100644 vendor/github.com/tetratelabs/wazero/internal/engine/wazevo/module_engine.go create mode 100644 vendor/github.com/tetratelabs/wazero/internal/engine/wazevo/reflect.go create mode 100644 vendor/github.com/tetratelabs/wazero/internal/engine/wazevo/reflect_tinygo.go create mode 100644 vendor/github.com/tetratelabs/wazero/internal/engine/wazevo/ssa/basic_block.go create mode 100644 vendor/github.com/tetratelabs/wazero/internal/engine/wazevo/ssa/basic_block_sort.go create mode 100644 vendor/github.com/tetratelabs/wazero/internal/engine/wazevo/ssa/basic_block_sort_old.go create mode 100644 vendor/github.com/tetratelabs/wazero/internal/engine/wazevo/ssa/builder.go create mode 100644 vendor/github.com/tetratelabs/wazero/internal/engine/wazevo/ssa/cmp.go create mode 100644 vendor/github.com/tetratelabs/wazero/internal/engine/wazevo/ssa/funcref.go create mode 100644 vendor/github.com/tetratelabs/wazero/internal/engine/wazevo/ssa/instructions.go create mode 100644 vendor/github.com/tetratelabs/wazero/internal/engine/wazevo/ssa/pass.go create mode 100644 vendor/github.com/tetratelabs/wazero/internal/engine/wazevo/ssa/pass_blk_layouts.go create mode 100644 vendor/github.com/tetratelabs/wazero/internal/engine/wazevo/ssa/pass_cfg.go create mode 100644 vendor/github.com/tetratelabs/wazero/internal/engine/wazevo/ssa/signature.go create mode 100644 vendor/github.com/tetratelabs/wazero/internal/engine/wazevo/ssa/ssa.go create mode 100644 vendor/github.com/tetratelabs/wazero/internal/engine/wazevo/ssa/type.go create mode 100644 vendor/github.com/tetratelabs/wazero/internal/engine/wazevo/ssa/vs.go create mode 100644 vendor/github.com/tetratelabs/wazero/internal/engine/wazevo/wazevoapi/debug_options.go create mode 100644 vendor/github.com/tetratelabs/wazero/internal/engine/wazevo/wazevoapi/exitcode.go create mode 100644 vendor/github.com/tetratelabs/wazero/internal/engine/wazevo/wazevoapi/offsetdata.go create mode 100644 vendor/github.com/tetratelabs/wazero/internal/engine/wazevo/wazevoapi/perfmap.go create mode 100644 vendor/github.com/tetratelabs/wazero/internal/engine/wazevo/wazevoapi/perfmap_disabled.go create mode 100644 vendor/github.com/tetratelabs/wazero/internal/engine/wazevo/wazevoapi/perfmap_enabled.go create mode 100644 vendor/github.com/tetratelabs/wazero/internal/engine/wazevo/wazevoapi/pool.go create mode 100644 vendor/github.com/tetratelabs/wazero/internal/engine/wazevo/wazevoapi/ptr.go create mode 100644 vendor/github.com/tetratelabs/wazero/internal/engine/wazevo/wazevoapi/queue.go create mode 100644 vendor/github.com/tetratelabs/wazero/internal/engine/wazevo/wazevoapi/resetmap.go create mode 100644 vendor/github.com/tetratelabs/wazero/internal/expctxkeys/checkpoint.go create mode 100644 vendor/github.com/tetratelabs/wazero/internal/expctxkeys/close.go create mode 100644 vendor/github.com/tetratelabs/wazero/internal/expctxkeys/expctxkeys.go create mode 100644 vendor/github.com/tetratelabs/wazero/internal/expctxkeys/listener.go create mode 100644 vendor/github.com/tetratelabs/wazero/internal/expctxkeys/memory.go create mode 100644 vendor/github.com/tetratelabs/wazero/internal/filecache/compilationcache.go create mode 100644 vendor/github.com/tetratelabs/wazero/internal/filecache/file_cache.go create mode 100644 vendor/github.com/tetratelabs/wazero/internal/fsapi/file.go create mode 100644 vendor/github.com/tetratelabs/wazero/internal/fsapi/poll.go create mode 100644 vendor/github.com/tetratelabs/wazero/internal/fsapi/unimplemented.go create mode 100644 vendor/github.com/tetratelabs/wazero/internal/ieee754/ieee754.go create mode 100644 vendor/github.com/tetratelabs/wazero/internal/internalapi/internal.go create mode 100644 vendor/github.com/tetratelabs/wazero/internal/leb128/leb128.go create mode 100644 vendor/github.com/tetratelabs/wazero/internal/moremath/moremath.go create mode 100644 vendor/github.com/tetratelabs/wazero/internal/platform/cpuid.go create mode 100644 vendor/github.com/tetratelabs/wazero/internal/platform/cpuid_amd64.go create mode 100644 vendor/github.com/tetratelabs/wazero/internal/platform/cpuid_amd64.s create mode 100644 vendor/github.com/tetratelabs/wazero/internal/platform/cpuid_unsupported.go create mode 100644 vendor/github.com/tetratelabs/wazero/internal/platform/crypto.go create mode 100644 vendor/github.com/tetratelabs/wazero/internal/platform/mmap_linux.go create mode 100644 vendor/github.com/tetratelabs/wazero/internal/platform/mmap_other.go create mode 100644 vendor/github.com/tetratelabs/wazero/internal/platform/mmap_unix.go create mode 100644 vendor/github.com/tetratelabs/wazero/internal/platform/mmap_unsupported.go create mode 100644 vendor/github.com/tetratelabs/wazero/internal/platform/mmap_windows.go create mode 100644 vendor/github.com/tetratelabs/wazero/internal/platform/mremap_other.go create mode 100644 vendor/github.com/tetratelabs/wazero/internal/platform/mremap_unix.go create mode 100644 vendor/github.com/tetratelabs/wazero/internal/platform/path.go create mode 100644 vendor/github.com/tetratelabs/wazero/internal/platform/path_windows.go create mode 100644 vendor/github.com/tetratelabs/wazero/internal/platform/platform.go create mode 100644 vendor/github.com/tetratelabs/wazero/internal/platform/platform_amd64.go create mode 100644 vendor/github.com/tetratelabs/wazero/internal/platform/platform_arm64.go create mode 100644 vendor/github.com/tetratelabs/wazero/internal/platform/time.go create mode 100644 vendor/github.com/tetratelabs/wazero/internal/platform/time_cgo.go create mode 100644 vendor/github.com/tetratelabs/wazero/internal/platform/time_notcgo.go create mode 100644 vendor/github.com/tetratelabs/wazero/internal/platform/time_windows.go create mode 100644 vendor/github.com/tetratelabs/wazero/internal/sock/sock.go create mode 100644 vendor/github.com/tetratelabs/wazero/internal/sock/sock_supported.go create mode 100644 vendor/github.com/tetratelabs/wazero/internal/sock/sock_unsupported.go create mode 100644 vendor/github.com/tetratelabs/wazero/internal/sys/fs.go create mode 100644 vendor/github.com/tetratelabs/wazero/internal/sys/lazy.go create mode 100644 vendor/github.com/tetratelabs/wazero/internal/sys/stdio.go create mode 100644 vendor/github.com/tetratelabs/wazero/internal/sys/sys.go create mode 100644 vendor/github.com/tetratelabs/wazero/internal/sysfs/adapter.go create mode 100644 vendor/github.com/tetratelabs/wazero/internal/sysfs/datasync_linux.go create mode 100644 vendor/github.com/tetratelabs/wazero/internal/sysfs/datasync_tinygo.go create mode 100644 vendor/github.com/tetratelabs/wazero/internal/sysfs/datasync_unsupported.go create mode 100644 vendor/github.com/tetratelabs/wazero/internal/sysfs/dir.go create mode 100644 vendor/github.com/tetratelabs/wazero/internal/sysfs/dirfs.go create mode 100644 vendor/github.com/tetratelabs/wazero/internal/sysfs/dirfs_supported.go create mode 100644 vendor/github.com/tetratelabs/wazero/internal/sysfs/dirfs_unsupported.go create mode 100644 vendor/github.com/tetratelabs/wazero/internal/sysfs/file.go create mode 100644 vendor/github.com/tetratelabs/wazero/internal/sysfs/file_unix.go create mode 100644 vendor/github.com/tetratelabs/wazero/internal/sysfs/file_unsupported.go create mode 100644 vendor/github.com/tetratelabs/wazero/internal/sysfs/file_windows.go create mode 100644 vendor/github.com/tetratelabs/wazero/internal/sysfs/futimens.go create mode 100644 vendor/github.com/tetratelabs/wazero/internal/sysfs/futimens_darwin.go create mode 100644 vendor/github.com/tetratelabs/wazero/internal/sysfs/futimens_darwin.s create mode 100644 vendor/github.com/tetratelabs/wazero/internal/sysfs/futimens_linux.go create mode 100644 vendor/github.com/tetratelabs/wazero/internal/sysfs/futimens_unsupported.go create mode 100644 vendor/github.com/tetratelabs/wazero/internal/sysfs/futimens_windows.go create mode 100644 vendor/github.com/tetratelabs/wazero/internal/sysfs/ino.go create mode 100644 vendor/github.com/tetratelabs/wazero/internal/sysfs/ino_plan9.go create mode 100644 vendor/github.com/tetratelabs/wazero/internal/sysfs/ino_tinygo.go create mode 100644 vendor/github.com/tetratelabs/wazero/internal/sysfs/ino_windows.go create mode 100644 vendor/github.com/tetratelabs/wazero/internal/sysfs/nonblock_unix.go create mode 100644 vendor/github.com/tetratelabs/wazero/internal/sysfs/nonblock_unsupported.go create mode 100644 vendor/github.com/tetratelabs/wazero/internal/sysfs/nonblock_windows.go create mode 100644 vendor/github.com/tetratelabs/wazero/internal/sysfs/oflag.go create mode 100644 vendor/github.com/tetratelabs/wazero/internal/sysfs/open_file_darwin.go create mode 100644 vendor/github.com/tetratelabs/wazero/internal/sysfs/open_file_freebsd.go create mode 100644 vendor/github.com/tetratelabs/wazero/internal/sysfs/open_file_linux.go create mode 100644 vendor/github.com/tetratelabs/wazero/internal/sysfs/open_file_notwindows.go create mode 100644 vendor/github.com/tetratelabs/wazero/internal/sysfs/open_file_sun.go create mode 100644 vendor/github.com/tetratelabs/wazero/internal/sysfs/open_file_tinygo.go create mode 100644 vendor/github.com/tetratelabs/wazero/internal/sysfs/open_file_unsupported.go create mode 100644 vendor/github.com/tetratelabs/wazero/internal/sysfs/open_file_windows.go create mode 100644 vendor/github.com/tetratelabs/wazero/internal/sysfs/osfile.go create mode 100644 vendor/github.com/tetratelabs/wazero/internal/sysfs/poll.go create mode 100644 vendor/github.com/tetratelabs/wazero/internal/sysfs/poll_darwin.go create mode 100644 vendor/github.com/tetratelabs/wazero/internal/sysfs/poll_darwin.s create mode 100644 vendor/github.com/tetratelabs/wazero/internal/sysfs/poll_linux.go create mode 100644 vendor/github.com/tetratelabs/wazero/internal/sysfs/poll_unsupported.go create mode 100644 vendor/github.com/tetratelabs/wazero/internal/sysfs/poll_windows.go create mode 100644 vendor/github.com/tetratelabs/wazero/internal/sysfs/readfs.go create mode 100644 vendor/github.com/tetratelabs/wazero/internal/sysfs/rename.go create mode 100644 vendor/github.com/tetratelabs/wazero/internal/sysfs/rename_plan9.go create mode 100644 vendor/github.com/tetratelabs/wazero/internal/sysfs/rename_windows.go create mode 100644 vendor/github.com/tetratelabs/wazero/internal/sysfs/sock.go create mode 100644 vendor/github.com/tetratelabs/wazero/internal/sysfs/sock_supported.go create mode 100644 vendor/github.com/tetratelabs/wazero/internal/sysfs/sock_unix.go create mode 100644 vendor/github.com/tetratelabs/wazero/internal/sysfs/sock_unsupported.go create mode 100644 vendor/github.com/tetratelabs/wazero/internal/sysfs/sock_windows.go create mode 100644 vendor/github.com/tetratelabs/wazero/internal/sysfs/stat.go create mode 100644 vendor/github.com/tetratelabs/wazero/internal/sysfs/stat_bsd.go create mode 100644 vendor/github.com/tetratelabs/wazero/internal/sysfs/stat_linux.go create mode 100644 vendor/github.com/tetratelabs/wazero/internal/sysfs/stat_unsupported.go create mode 100644 vendor/github.com/tetratelabs/wazero/internal/sysfs/stat_windows.go create mode 100644 vendor/github.com/tetratelabs/wazero/internal/sysfs/sync.go create mode 100644 vendor/github.com/tetratelabs/wazero/internal/sysfs/sync_windows.go create mode 100644 vendor/github.com/tetratelabs/wazero/internal/sysfs/syscall6_darwin.go create mode 100644 vendor/github.com/tetratelabs/wazero/internal/sysfs/sysfs.go create mode 100644 vendor/github.com/tetratelabs/wazero/internal/sysfs/unlink.go create mode 100644 vendor/github.com/tetratelabs/wazero/internal/sysfs/unlink_plan9.go create mode 100644 vendor/github.com/tetratelabs/wazero/internal/sysfs/unlink_windows.go create mode 100644 vendor/github.com/tetratelabs/wazero/internal/u32/u32.go create mode 100644 vendor/github.com/tetratelabs/wazero/internal/u64/u64.go create mode 100644 vendor/github.com/tetratelabs/wazero/internal/version/version.go create mode 100644 vendor/github.com/tetratelabs/wazero/internal/wasm/binary/code.go create mode 100644 vendor/github.com/tetratelabs/wazero/internal/wasm/binary/const_expr.go create mode 100644 vendor/github.com/tetratelabs/wazero/internal/wasm/binary/custom.go create mode 100644 vendor/github.com/tetratelabs/wazero/internal/wasm/binary/data.go create mode 100644 vendor/github.com/tetratelabs/wazero/internal/wasm/binary/decoder.go create mode 100644 vendor/github.com/tetratelabs/wazero/internal/wasm/binary/element.go create mode 100644 vendor/github.com/tetratelabs/wazero/internal/wasm/binary/errors.go create mode 100644 vendor/github.com/tetratelabs/wazero/internal/wasm/binary/export.go create mode 100644 vendor/github.com/tetratelabs/wazero/internal/wasm/binary/function.go create mode 100644 vendor/github.com/tetratelabs/wazero/internal/wasm/binary/global.go create mode 100644 vendor/github.com/tetratelabs/wazero/internal/wasm/binary/header.go create mode 100644 vendor/github.com/tetratelabs/wazero/internal/wasm/binary/import.go create mode 100644 vendor/github.com/tetratelabs/wazero/internal/wasm/binary/limits.go create mode 100644 vendor/github.com/tetratelabs/wazero/internal/wasm/binary/memory.go create mode 100644 vendor/github.com/tetratelabs/wazero/internal/wasm/binary/names.go create mode 100644 vendor/github.com/tetratelabs/wazero/internal/wasm/binary/section.go create mode 100644 vendor/github.com/tetratelabs/wazero/internal/wasm/binary/table.go create mode 100644 vendor/github.com/tetratelabs/wazero/internal/wasm/binary/value.go create mode 100644 vendor/github.com/tetratelabs/wazero/internal/wasm/counts.go create mode 100644 vendor/github.com/tetratelabs/wazero/internal/wasm/engine.go create mode 100644 vendor/github.com/tetratelabs/wazero/internal/wasm/func_validation.go create mode 100644 vendor/github.com/tetratelabs/wazero/internal/wasm/function_definition.go create mode 100644 vendor/github.com/tetratelabs/wazero/internal/wasm/global.go create mode 100644 vendor/github.com/tetratelabs/wazero/internal/wasm/gofunc.go create mode 100644 vendor/github.com/tetratelabs/wazero/internal/wasm/host.go create mode 100644 vendor/github.com/tetratelabs/wazero/internal/wasm/instruction.go create mode 100644 vendor/github.com/tetratelabs/wazero/internal/wasm/memory.go create mode 100644 vendor/github.com/tetratelabs/wazero/internal/wasm/memory_definition.go create mode 100644 vendor/github.com/tetratelabs/wazero/internal/wasm/module.go create mode 100644 vendor/github.com/tetratelabs/wazero/internal/wasm/module_instance.go create mode 100644 vendor/github.com/tetratelabs/wazero/internal/wasm/module_instance_lookup.go create mode 100644 vendor/github.com/tetratelabs/wazero/internal/wasm/store.go create mode 100644 vendor/github.com/tetratelabs/wazero/internal/wasm/store_module_list.go create mode 100644 vendor/github.com/tetratelabs/wazero/internal/wasm/table.go create mode 100644 vendor/github.com/tetratelabs/wazero/internal/wasmdebug/debug.go create mode 100644 vendor/github.com/tetratelabs/wazero/internal/wasmdebug/dwarf.go create mode 100644 vendor/github.com/tetratelabs/wazero/internal/wasmruntime/errors.go create mode 100644 vendor/github.com/tetratelabs/wazero/netlify.toml create mode 100644 vendor/github.com/tetratelabs/wazero/runtime.go create mode 100644 vendor/github.com/tetratelabs/wazero/sys/clock.go create mode 100644 vendor/github.com/tetratelabs/wazero/sys/error.go create mode 100644 vendor/github.com/tetratelabs/wazero/sys/stat.go create mode 100644 vendor/github.com/tetratelabs/wazero/sys/stat_bsd.go create mode 100644 vendor/github.com/tetratelabs/wazero/sys/stat_linux.go create mode 100644 vendor/github.com/tetratelabs/wazero/sys/stat_unsupported.go create mode 100644 vendor/github.com/tetratelabs/wazero/sys/stat_windows.go diff --git a/go.mod b/go.mod index f7114bfd6..b0165a1d9 100644 --- a/go.mod +++ b/go.mod @@ -44,6 +44,7 @@ require ( github.com/miekg/dns v1.1.59 github.com/minio/minio-go/v7 v7.0.70 github.com/mitchellh/mapstructure v1.5.0 + github.com/ncruces/go-sqlite3 v0.16.0 github.com/oklog/ulid v1.3.1 github.com/prometheus/client_golang v1.19.1 github.com/spf13/cobra v1.8.0 @@ -78,7 +79,7 @@ require ( golang.org/x/text v0.15.0 gopkg.in/mcuadros/go-syslog.v2 v2.3.0 gopkg.in/yaml.v3 v3.0.1 - modernc.org/sqlite v1.29.8 + modernc.org/sqlite v0.0.0-00010101000000-000000000000 mvdan.cc/xurls/v2 v2.5.0 ) @@ -173,6 +174,7 @@ require ( github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect github.com/modern-go/reflect2 v1.0.2 // indirect github.com/ncruces/go-strftime v0.1.9 // indirect + github.com/ncruces/julianday v1.0.0 // indirect github.com/opencontainers/runtime-spec v1.0.2 // indirect github.com/pbnjay/memory v0.0.0-20210728143218-7b4eea64cf58 // indirect github.com/pelletier/go-toml/v2 v2.2.2 // indirect @@ -197,6 +199,7 @@ require ( github.com/superseriousbusiness/go-jpeg-image-structure/v2 v2.0.0-20220321154430-d89a106fdabe // indirect github.com/superseriousbusiness/go-png-image-structure/v2 v2.0.1-SSB // indirect github.com/tdewolff/parse/v2 v2.7.14 // indirect + github.com/tetratelabs/wazero v1.7.2 // indirect github.com/tmthrgd/go-hex v0.0.0-20190904060850-447a3041c3bc // indirect github.com/toqueteos/webbrowser v1.2.0 // indirect github.com/twitchyliquid64/golang-asm v0.15.1 // indirect diff --git a/go.sum b/go.sum index 7c2471ed3..07de139d2 100644 --- a/go.sum +++ b/go.sum @@ -445,8 +445,12 @@ github.com/modern-go/reflect2 v1.0.2 h1:xBagoLtFs94CBntxluKeaWgTMpvLxC4ur3nMaC9G github.com/modern-go/reflect2 v1.0.2/go.mod h1:yWuevngMOJpCy52FWWMvUC8ws7m/LJsjYzDa0/r8luk= github.com/moul/http2curl v1.0.0 h1:dRMWoAtb+ePxMlLkrCbAqh4TlPHXvoGUSQ323/9Zahs= github.com/moul/http2curl v1.0.0/go.mod h1:8UbvGypXm98wA/IqH45anm5Y2Z6ep6O31QGOAZ3H0fQ= +github.com/ncruces/go-sqlite3 v0.16.0 h1:O7eULuEjvSBnS1QCN+dDL/ixLQZoUGWr466A02Gx1xc= +github.com/ncruces/go-sqlite3 v0.16.0/go.mod h1:2TmAeD93ImsKXJRsUIKohfMvt17dZSbS6pzJ3k6YYFg= github.com/ncruces/go-strftime v0.1.9 h1:bY0MQC28UADQmHmaF5dgpLmImcShSi2kHU9XLdhx/f4= github.com/ncruces/go-strftime v0.1.9/go.mod h1:Fwc5htZGVVkseilnfgOVb9mKy6w1naJmn9CehxcKcls= +github.com/ncruces/julianday v1.0.0 h1:fH0OKwa7NWvniGQtxdJRxAgkBMolni2BjDHaWTxqt7M= +github.com/ncruces/julianday v1.0.0/go.mod h1:Dusn2KvZrrovOMJuOt0TNXL6tB7U2E8kvza5fFc9G7g= github.com/nxadm/tail v1.4.4/go.mod h1:kenIhsEOeOJmVchQTgglprH7qJGnHDVpk1VPCcaMI8A= github.com/oklog/ulid v1.3.1 h1:EGfNDEx6MqHz8B3uNV6QAib1UR2Lm97sHi3ocA6ESJ4= github.com/oklog/ulid v1.3.1/go.mod h1:CirwcVhetQ6Lv90oh/F+FBtV6XMibvdAFo93nm5qn4U= @@ -558,6 +562,8 @@ github.com/tdewolff/test v1.0.11-0.20240106005702-7de5f7df4739 h1:IkjBCtQOOjIn03 github.com/tdewolff/test v1.0.11-0.20240106005702-7de5f7df4739/go.mod h1:XPuWBzvdUzhCuxWO1ojpXsyzsA5bFoS3tO/Q3kFuTG8= github.com/technologize/otel-go-contrib v1.1.1 h1:wZH9aSPNWZWIkEh3vfaKfMb15AJ80jJ1aVj/4GZdqIw= github.com/technologize/otel-go-contrib v1.1.1/go.mod h1:dCN/wj2WyUO8aFZFdIN+6tfJHImjTML/8r2YVYAy3So= +github.com/tetratelabs/wazero v1.7.2 h1:1+z5nXJNwMLPAWaTePFi49SSTL0IMx/i3Fg8Yc25GDc= +github.com/tetratelabs/wazero v1.7.2/go.mod h1:ytl6Zuh20R/eROuyDaGPkp82O9C/DJfXAwJfQ3X6/7Y= github.com/tidwall/btree v0.0.0-20191029221954-400434d76274 h1:G6Z6HvJuPjG6XfNGi/feOATzeJrfgTNJY+rGrHbA04E= github.com/tidwall/btree v0.0.0-20191029221954-400434d76274/go.mod h1:huei1BkDWJ3/sLXmO+bsCNELL+Bp2Kks9OLyQFkzvA8= github.com/tidwall/buntdb v1.1.2 h1:noCrqQXL9EKMtcdwJcmuVKSEjqu1ua99RHHgbLTEHRo= diff --git a/internal/api/client/media/mediacreate_test.go b/internal/api/client/media/mediacreate_test.go index 00f385032..c2871aff0 100644 --- a/internal/api/client/media/mediacreate_test.go +++ b/internal/api/client/media/mediacreate_test.go @@ -38,7 +38,6 @@ import ( "github.com/superseriousbusiness/gotosocial/internal/federation" "github.com/superseriousbusiness/gotosocial/internal/filter/visibility" "github.com/superseriousbusiness/gotosocial/internal/gtsmodel" - "github.com/superseriousbusiness/gotosocial/internal/log" "github.com/superseriousbusiness/gotosocial/internal/media" "github.com/superseriousbusiness/gotosocial/internal/oauth" "github.com/superseriousbusiness/gotosocial/internal/processing" @@ -77,19 +76,22 @@ type MediaCreateTestSuite struct { TEST INFRASTRUCTURE */ -func (suite *MediaCreateTestSuite) SetupSuite() { - suite.state.Caches.Init() +func (suite *MediaCreateTestSuite) SetupTest() { testrig.StartNoopWorkers(&suite.state) // setup standard items testrig.InitTestConfig() testrig.InitTestLog() - suite.db = testrig.NewTestDB(&suite.state) - suite.state.DB = suite.db + suite.state.Caches.Init() + suite.storage = testrig.NewInMemoryStorage() suite.state.Storage = suite.storage + suite.db = testrig.NewTestDB(&suite.state) + testrig.StandardDBSetup(suite.db, nil) + testrig.StandardStorageSetup(suite.storage, "../../../../testrig/media") + suite.tc = typeutils.NewConverter(&suite.state) testrig.StartTimelines( @@ -106,21 +108,8 @@ func (suite *MediaCreateTestSuite) SetupSuite() { // setup module being tested suite.mediaModule = mediamodule.New(suite.processor) -} - -func (suite *MediaCreateTestSuite) TearDownSuite() { - if err := suite.db.Close(); err != nil { - log.Panicf(nil, "error closing db connection: %s", err) - } - testrig.StopWorkers(&suite.state) -} - -func (suite *MediaCreateTestSuite) SetupTest() { - suite.state.Caches.Init() - - testrig.StandardDBSetup(suite.db, nil) - testrig.StandardStorageSetup(suite.storage, "../../../../testrig/media") + // setup test data suite.testTokens = testrig.NewTestTokens() suite.testClients = testrig.NewTestClients() suite.testApplications = testrig.NewTestApplications() @@ -132,6 +121,7 @@ func (suite *MediaCreateTestSuite) SetupTest() { func (suite *MediaCreateTestSuite) TearDownTest() { testrig.StandardDBTeardown(suite.db) testrig.StandardStorageTeardown(suite.storage) + testrig.StopWorkers(&suite.state) } /* diff --git a/internal/api/client/media/mediaupdate_test.go b/internal/api/client/media/mediaupdate_test.go index bb4e0f4ad..bb260ae4d 100644 --- a/internal/api/client/media/mediaupdate_test.go +++ b/internal/api/client/media/mediaupdate_test.go @@ -36,7 +36,6 @@ import ( "github.com/superseriousbusiness/gotosocial/internal/federation" "github.com/superseriousbusiness/gotosocial/internal/filter/visibility" "github.com/superseriousbusiness/gotosocial/internal/gtsmodel" - "github.com/superseriousbusiness/gotosocial/internal/log" "github.com/superseriousbusiness/gotosocial/internal/media" "github.com/superseriousbusiness/gotosocial/internal/oauth" "github.com/superseriousbusiness/gotosocial/internal/processing" @@ -75,18 +74,22 @@ type MediaUpdateTestSuite struct { TEST INFRASTRUCTURE */ -func (suite *MediaUpdateTestSuite) SetupSuite() { +func (suite *MediaUpdateTestSuite) SetupTest() { testrig.StartNoopWorkers(&suite.state) // setup standard items testrig.InitTestConfig() testrig.InitTestLog() - suite.db = testrig.NewTestDB(&suite.state) - suite.state.DB = suite.db + suite.state.Caches.Init() + suite.storage = testrig.NewInMemoryStorage() suite.state.Storage = suite.storage + suite.db = testrig.NewTestDB(&suite.state) + testrig.StandardDBSetup(suite.db, nil) + testrig.StandardStorageSetup(suite.storage, "../../../../testrig/media") + suite.tc = typeutils.NewConverter(&suite.state) testrig.StartTimelines( @@ -103,21 +106,8 @@ func (suite *MediaUpdateTestSuite) SetupSuite() { // setup module being tested suite.mediaModule = mediamodule.New(suite.processor) -} - -func (suite *MediaUpdateTestSuite) TearDownSuite() { - if err := suite.db.Close(); err != nil { - log.Panicf(nil, "error closing db connection: %s", err) - } - testrig.StopWorkers(&suite.state) -} - -func (suite *MediaUpdateTestSuite) SetupTest() { - suite.state.Caches.Init() - - testrig.StandardDBSetup(suite.db, nil) - testrig.StandardStorageSetup(suite.storage, "../../../../testrig/media") + // setup test data suite.testTokens = testrig.NewTestTokens() suite.testClients = testrig.NewTestClients() suite.testApplications = testrig.NewTestApplications() @@ -129,6 +119,7 @@ func (suite *MediaUpdateTestSuite) SetupTest() { func (suite *MediaUpdateTestSuite) TearDownTest() { testrig.StandardDBTeardown(suite.db) testrig.StandardStorageTeardown(suite.storage) + testrig.StopWorkers(&suite.state) } /* diff --git a/internal/api/fileserver/fileserver_test.go b/internal/api/fileserver/fileserver_test.go index b58433b9f..e5f684d0c 100644 --- a/internal/api/fileserver/fileserver_test.go +++ b/internal/api/fileserver/fileserver_test.go @@ -70,8 +70,6 @@ func (suite *FileserverTestSuite) SetupSuite() { testrig.InitTestConfig() testrig.InitTestLog() - suite.db = testrig.NewTestDB(&suite.state) - suite.state.DB = suite.db suite.storage = testrig.NewInMemoryStorage() suite.state.Storage = suite.storage @@ -98,8 +96,12 @@ func (suite *FileserverTestSuite) SetupTest() { suite.state.Caches.Init() testrig.StartNoopWorkers(&suite.state) + suite.db = testrig.NewTestDB(&suite.state) + suite.state.DB = suite.db + testrig.StandardDBSetup(suite.db, nil) testrig.StandardStorageSetup(suite.storage, "../../../testrig/media") + suite.testTokens = testrig.NewTestTokens() suite.testClients = testrig.NewTestClients() suite.testApplications = testrig.NewTestApplications() diff --git a/internal/config/global.go b/internal/config/global.go index 4bc5ac3d2..57af89d05 100644 --- a/internal/config/global.go +++ b/internal/config/global.go @@ -52,3 +52,9 @@ func LoadEarlyFlags(cmd *cobra.Command) error { func BindFlags(cmd *cobra.Command) error { return global.BindFlags(cmd) } + +// Reset will totally clear global +// ConfigState{}, loading defaults. +func Reset() { + global.Reset() +} diff --git a/internal/config/state.go b/internal/config/state.go index c55f7b2ec..d01e853a5 100644 --- a/internal/config/state.go +++ b/internal/config/state.go @@ -37,25 +37,9 @@ type ConfigState struct { //nolint // NewState returns a new initialized ConfigState instance. func NewState() *ConfigState { - viper := viper.New() - - // Flag 'some-flag-name' becomes env var 'GTS_SOME_FLAG_NAME' - viper.SetEnvKeyReplacer(strings.NewReplacer("-", "_")) - viper.SetEnvPrefix("gts") - - // Load appropriate named vals from env - viper.AutomaticEnv() - - // Create new ConfigState with defaults - state := &ConfigState{ - viper: viper, - config: Defaults, - } - - // Perform initial load into viper - state.reloadToViper() - - return state + st := new(ConfigState) + st.Reset() + return st } // Config provides safe access to the ConfigState's contained Configuration, @@ -116,6 +100,32 @@ func (st *ConfigState) Reload() (err error) { return } +// Reset will totally clear +// ConfigState{}, loading defaults. +func (st *ConfigState) Reset() { + // Do within lock. + st.mutex.Lock() + defer st.mutex.Unlock() + + // Create new viper. + viper := viper.New() + + // Flag 'some-flag-name' becomes env var 'GTS_SOME_FLAG_NAME' + viper.SetEnvKeyReplacer(strings.NewReplacer("-", "_")) + viper.SetEnvPrefix("gts") + + // Load appropriate + // named vals from env. + viper.AutomaticEnv() + + // Reset variables. + st.viper = viper + st.config = Defaults + + // Load into viper. + st.reloadToViper() +} + // reloadToViper will reload Configuration{} values into viper. func (st *ConfigState) reloadToViper() { raw, err := st.config.MarshalMap() diff --git a/internal/db/bundb/admin_test.go b/internal/db/bundb/admin_test.go index da3370c4e..8018ef3fa 100644 --- a/internal/db/bundb/admin_test.go +++ b/internal/db/bundb/admin_test.go @@ -74,6 +74,7 @@ func (suite *AdminTestSuite) TestCreateInstanceAccount() { // we need to take an empty db for this... testrig.StandardDBTeardown(suite.db) // ...with tables created but no data + suite.db = testrig.NewTestDB(&suite.state) testrig.CreateTestTables(suite.db) // make sure there's no instance account in the db yet diff --git a/internal/db/bundb/bundb.go b/internal/db/bundb/bundb.go index b0ce575e6..e7256c276 100644 --- a/internal/db/bundb/bundb.go +++ b/internal/db/bundb/bundb.go @@ -48,8 +48,6 @@ import ( "github.com/uptrace/bun/dialect/pgdialect" "github.com/uptrace/bun/dialect/sqlitedialect" "github.com/uptrace/bun/migrate" - - "modernc.org/sqlite" ) // DBService satisfies the DB interface @@ -133,12 +131,12 @@ func NewBunDBService(ctx context.Context, state *state.State) (db.DB, error) { switch t { case "postgres": - db, err = pgConn(ctx, state) + db, err = pgConn(ctx) if err != nil { return nil, err } case "sqlite": - db, err = sqliteConn(ctx, state) + db, err = sqliteConn(ctx) if err != nil { return nil, err } @@ -295,7 +293,7 @@ func NewBunDBService(ctx context.Context, state *state.State) (db.DB, error) { return ps, nil } -func pgConn(ctx context.Context, state *state.State) (*bun.DB, error) { +func pgConn(ctx context.Context) (*bun.DB, error) { opts, err := deriveBunDBPGOptions() //nolint:contextcheck if err != nil { return nil, fmt.Errorf("could not create bundb postgres options: %w", err) @@ -326,7 +324,7 @@ func pgConn(ctx context.Context, state *state.State) (*bun.DB, error) { return db, nil } -func sqliteConn(ctx context.Context, state *state.State) (*bun.DB, error) { +func sqliteConn(ctx context.Context) (*bun.DB, error) { // validate db address has actually been set address := config.GetDbAddress() if address == "" { @@ -339,9 +337,6 @@ func sqliteConn(ctx context.Context, state *state.State) (*bun.DB, error) { // Open new DB instance sqldb, err := sql.Open("sqlite-gts", address) if err != nil { - if errWithCode, ok := err.(*sqlite.Error); ok { - err = errors.New(sqlite.ErrorCodeString[errWithCode.Code()]) - } return nil, fmt.Errorf("could not open sqlite db with address %s: %w", address, err) } @@ -356,11 +351,9 @@ func sqliteConn(ctx context.Context, state *state.State) (*bun.DB, error) { // ping to check the db is there and listening if err := db.PingContext(ctx); err != nil { - if errWithCode, ok := err.(*sqlite.Error); ok { - err = errors.New(sqlite.ErrorCodeString[errWithCode.Code()]) - } return nil, fmt.Errorf("sqlite ping: %w", err) } + log.Infof(ctx, "connected to SQLITE database with address %s", address) return db, nil @@ -528,12 +521,8 @@ func buildSQLiteAddress(addr string) string { // Use random name for in-memory instead of ':memory:', so // multiple in-mem databases can be created without conflict. - addr = uuid.NewString() - - // in-mem-specific preferences - // (shared cache so that tests don't fail) - prefs.Add("mode", "memory") - prefs.Add("cache", "shared") + addr = "/" + uuid.NewString() + prefs.Add("vfs", "memdb") } if dur := config.GetDbSqliteBusyTimeout(); dur > 0 { diff --git a/internal/db/bundb/drivers.go b/internal/db/bundb/drivers.go index 1811ad533..f39189c9d 100644 --- a/internal/db/bundb/drivers.go +++ b/internal/db/bundb/drivers.go @@ -18,350 +18,14 @@ package bundb import ( - "context" "database/sql" - "database/sql/driver" - "time" - _ "unsafe" // linkname shenanigans - pgx "github.com/jackc/pgx/v5/stdlib" - "github.com/superseriousbusiness/gotosocial/internal/db" - "github.com/superseriousbusiness/gotosocial/internal/gtserror" - "modernc.org/sqlite" + "github.com/superseriousbusiness/gotosocial/internal/db/postgres" + "github.com/superseriousbusiness/gotosocial/internal/db/sqlite" ) -var ( - // global SQL driver instances. - postgresDriver = pgx.GetDefaultDriver() - sqliteDriver = getSQLiteDriver() - - // check the postgres connection - // conforms to our conn{} interface. - // (note SQLite doesn't export their - // conn type, and gets checked in - // tests very regularly anywho). - _ conn = (*pgx.Conn)(nil) -) - -//go:linkname getSQLiteDriver modernc.org/sqlite.newDriver -func getSQLiteDriver() *sqlite.Driver - func init() { - sql.Register("pgx-gts", &PostgreSQLDriver{}) - sql.Register("sqlite-gts", &SQLiteDriver{}) -} - -// PostgreSQLDriver is our own wrapper around the -// pgx/stdlib.Driver{} type in order to wrap further -// SQL driver types with our own err processing. -type PostgreSQLDriver struct{} - -func (d *PostgreSQLDriver) Open(name string) (driver.Conn, error) { - c, err := postgresDriver.Open(name) - if err != nil { - return nil, err - } - return &PostgreSQLConn{conn: c.(conn)}, nil -} - -type PostgreSQLConn struct{ conn } - -func (c *PostgreSQLConn) Begin() (driver.Tx, error) { - return c.BeginTx(context.Background(), driver.TxOptions{}) -} - -func (c *PostgreSQLConn) BeginTx(ctx context.Context, opts driver.TxOptions) (driver.Tx, error) { - tx, err := c.conn.BeginTx(ctx, opts) - err = processPostgresError(err) - if err != nil { - return nil, err - } - return &PostgreSQLTx{tx}, nil -} - -func (c *PostgreSQLConn) Prepare(query string) (driver.Stmt, error) { - return c.PrepareContext(context.Background(), query) -} - -func (c *PostgreSQLConn) PrepareContext(ctx context.Context, query string) (driver.Stmt, error) { - st, err := c.conn.PrepareContext(ctx, query) - err = processPostgresError(err) - if err != nil { - return nil, err - } - return &PostgreSQLStmt{stmt: st.(stmt)}, nil -} - -func (c *PostgreSQLConn) Exec(query string, args []driver.Value) (driver.Result, error) { - return c.ExecContext(context.Background(), query, toNamedValues(args)) -} - -func (c *PostgreSQLConn) ExecContext(ctx context.Context, query string, args []driver.NamedValue) (driver.Result, error) { - result, err := c.conn.ExecContext(ctx, query, args) - err = processPostgresError(err) - return result, err -} - -func (c *PostgreSQLConn) Query(query string, args []driver.Value) (driver.Rows, error) { - return c.QueryContext(context.Background(), query, toNamedValues(args)) -} - -func (c *PostgreSQLConn) QueryContext(ctx context.Context, query string, args []driver.NamedValue) (driver.Rows, error) { - rows, err := c.conn.QueryContext(ctx, query, args) - err = processPostgresError(err) - return rows, err -} - -func (c *PostgreSQLConn) Close() error { - return c.conn.Close() -} - -type PostgreSQLTx struct{ driver.Tx } - -func (tx *PostgreSQLTx) Commit() error { - err := tx.Tx.Commit() - return processPostgresError(err) -} - -func (tx *PostgreSQLTx) Rollback() error { - err := tx.Tx.Rollback() - return processPostgresError(err) -} - -type PostgreSQLStmt struct{ stmt } - -func (stmt *PostgreSQLStmt) Exec(args []driver.Value) (driver.Result, error) { - return stmt.ExecContext(context.Background(), toNamedValues(args)) -} - -func (stmt *PostgreSQLStmt) ExecContext(ctx context.Context, args []driver.NamedValue) (driver.Result, error) { - res, err := stmt.stmt.ExecContext(ctx, args) - err = processPostgresError(err) - return res, err -} - -func (stmt *PostgreSQLStmt) Query(args []driver.Value) (driver.Rows, error) { - return stmt.QueryContext(context.Background(), toNamedValues(args)) -} - -func (stmt *PostgreSQLStmt) QueryContext(ctx context.Context, args []driver.NamedValue) (driver.Rows, error) { - rows, err := stmt.stmt.QueryContext(ctx, args) - err = processPostgresError(err) - return rows, err -} - -// SQLiteDriver is our own wrapper around the -// sqlite.Driver{} type in order to wrap further -// SQL driver types with our own functionality, -// e.g. hooks, retries and err processing. -type SQLiteDriver struct{} - -func (d *SQLiteDriver) Open(name string) (driver.Conn, error) { - c, err := sqliteDriver.Open(name) - if err != nil { - return nil, err - } - return &SQLiteConn{conn: c.(conn)}, nil -} - -type SQLiteConn struct{ conn } - -func (c *SQLiteConn) Begin() (driver.Tx, error) { - return c.BeginTx(context.Background(), driver.TxOptions{}) -} - -func (c *SQLiteConn) BeginTx(ctx context.Context, opts driver.TxOptions) (tx driver.Tx, err error) { - err = retryOnBusy(ctx, func() error { - tx, err = c.conn.BeginTx(ctx, opts) - err = processSQLiteError(err) - return err - }) - if err != nil { - return nil, err - } - return &SQLiteTx{Context: ctx, Tx: tx}, nil -} - -func (c *SQLiteConn) Prepare(query string) (driver.Stmt, error) { - return c.PrepareContext(context.Background(), query) -} - -func (c *SQLiteConn) PrepareContext(ctx context.Context, query string) (st driver.Stmt, err error) { - err = retryOnBusy(ctx, func() error { - st, err = c.conn.PrepareContext(ctx, query) - err = processSQLiteError(err) - return err - }) - if err != nil { - return nil, err - } - return &SQLiteStmt{st.(stmt)}, nil -} - -func (c *SQLiteConn) Exec(query string, args []driver.Value) (driver.Result, error) { - return c.ExecContext(context.Background(), query, toNamedValues(args)) -} - -func (c *SQLiteConn) ExecContext(ctx context.Context, query string, args []driver.NamedValue) (result driver.Result, err error) { - err = retryOnBusy(ctx, func() error { - result, err = c.conn.ExecContext(ctx, query, args) - err = processSQLiteError(err) - return err - }) - return -} - -func (c *SQLiteConn) Query(query string, args []driver.Value) (driver.Rows, error) { - return c.QueryContext(context.Background(), query, toNamedValues(args)) -} - -func (c *SQLiteConn) QueryContext(ctx context.Context, query string, args []driver.NamedValue) (rows driver.Rows, err error) { - err = retryOnBusy(ctx, func() error { - rows, err = c.conn.QueryContext(ctx, query, args) - err = processSQLiteError(err) - return err - }) - return -} - -func (c *SQLiteConn) Close() error { - // see: https://www.sqlite.org/pragma.html#pragma_optimize - const onClose = "PRAGMA analysis_limit=1000; PRAGMA optimize;" - _, _ = c.conn.ExecContext(context.Background(), onClose, nil) - return c.conn.Close() -} - -type SQLiteTx struct { - context.Context - driver.Tx -} - -func (tx *SQLiteTx) Commit() (err error) { - err = retryOnBusy(tx.Context, func() error { - err = tx.Tx.Commit() - err = processSQLiteError(err) - return err - }) - return -} - -func (tx *SQLiteTx) Rollback() (err error) { - err = retryOnBusy(tx.Context, func() error { - err = tx.Tx.Rollback() - err = processSQLiteError(err) - return err - }) - return -} - -type SQLiteStmt struct{ stmt } - -func (stmt *SQLiteStmt) Exec(args []driver.Value) (driver.Result, error) { - return stmt.ExecContext(context.Background(), toNamedValues(args)) -} - -func (stmt *SQLiteStmt) ExecContext(ctx context.Context, args []driver.NamedValue) (res driver.Result, err error) { - err = retryOnBusy(ctx, func() error { - res, err = stmt.stmt.ExecContext(ctx, args) - err = processSQLiteError(err) - return err - }) - return -} - -func (stmt *SQLiteStmt) Query(args []driver.Value) (driver.Rows, error) { - return stmt.QueryContext(context.Background(), toNamedValues(args)) -} - -func (stmt *SQLiteStmt) QueryContext(ctx context.Context, args []driver.NamedValue) (rows driver.Rows, err error) { - err = retryOnBusy(ctx, func() error { - rows, err = stmt.stmt.QueryContext(ctx, args) - err = processSQLiteError(err) - return err - }) - return -} - -type conn interface { - driver.Conn - driver.ConnPrepareContext - driver.ExecerContext - driver.QueryerContext - driver.ConnBeginTx -} - -type stmt interface { - driver.Stmt - driver.StmtExecContext - driver.StmtQueryContext -} - -// retryOnBusy will retry given function on returned 'errBusy'. -func retryOnBusy(ctx context.Context, fn func() error) error { - if err := fn(); err != errBusy { - return err - } - return retryOnBusySlow(ctx, fn) -} - -// retryOnBusySlow is the outlined form of retryOnBusy, to allow the fast path (i.e. only -// 1 attempt) to be inlined, leaving the slow retry loop to be a separate function call. -func retryOnBusySlow(ctx context.Context, fn func() error) error { - var backoff time.Duration - - for i := 0; ; i++ { - // backoff according to a multiplier of 2ms * 2^2n, - // up to a maximum possible backoff time of 5 minutes. - // - // this works out as the following: - // 4ms - // 16ms - // 64ms - // 256ms - // 1.024s - // 4.096s - // 16.384s - // 1m5.536s - // 4m22.144s - backoff = 2 * time.Millisecond * (1 << (2*i + 1)) - if backoff >= 5*time.Minute { - break - } - - select { - // Context cancelled. - case <-ctx.Done(): - return ctx.Err() - - // Backoff for some time. - case <-time.After(backoff): - } - - // Perform func. - err := fn() - - if err != errBusy { - // May be nil, or may be - // some other error, either - // way return here. - return err - } - } - - return gtserror.Newf("%w (waited > %s)", db.ErrBusyTimeout, backoff) -} - -// toNamedValues converts older driver.Value types to driver.NamedValue types. -func toNamedValues(args []driver.Value) []driver.NamedValue { - if args == nil { - return nil - } - args2 := make([]driver.NamedValue, len(args)) - for i := range args { - args2[i] = driver.NamedValue{ - Ordinal: i + 1, - Value: args[i], - } - } - return args2 + // register our SQL driver implementations. + sql.Register("pgx-gts", &postgres.Driver{}) + sql.Register("sqlite-gts", &sqlite.Driver{}) } diff --git a/internal/db/bundb/errors.go b/internal/db/bundb/errors.go deleted file mode 100644 index f2633786a..000000000 --- a/internal/db/bundb/errors.go +++ /dev/null @@ -1,105 +0,0 @@ -// GoToSocial -// Copyright (C) GoToSocial Authors admin@gotosocial.org -// SPDX-License-Identifier: AGPL-3.0-or-later -// -// This program is free software: you can redistribute it and/or modify -// it under the terms of the GNU Affero General Public License as published by -// the Free Software Foundation, either version 3 of the License, or -// (at your option) any later version. -// -// This program is distributed in the hope that it will be useful, -// but WITHOUT ANY WARRANTY; without even the implied warranty of -// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -// GNU Affero General Public License for more details. -// -// You should have received a copy of the GNU Affero General Public License -// along with this program. If not, see . - -package bundb - -import ( - "database/sql/driver" - "errors" - - "github.com/jackc/pgx/v5/pgconn" - "github.com/superseriousbusiness/gotosocial/internal/db" - "modernc.org/sqlite" - sqlite3 "modernc.org/sqlite/lib" -) - -// errBusy is a sentinel error indicating -// busy database (e.g. retry needed). -var errBusy = errors.New("busy") - -// processPostgresError processes an error, replacing any postgres specific errors with our own error type -func processPostgresError(err error) error { - // Catch nil errs. - if err == nil { - return nil - } - - // Attempt to cast as postgres - pgErr, ok := err.(*pgconn.PgError) - if !ok { - return err - } - - // Handle supplied error code: - // (https://www.postgresql.org/docs/10/errcodes-appendix.html) - switch pgErr.Code { //nolint - case "23505" /* unique_violation */ : - return db.ErrAlreadyExists - } - - return err -} - -// processSQLiteError processes an error, replacing any sqlite specific errors with our own error type -func processSQLiteError(err error) error { - // Catch nil errs. - if err == nil { - return nil - } - - // Attempt to cast as sqlite - sqliteErr, ok := err.(*sqlite.Error) - if !ok { - return err - } - - // Handle supplied error code: - switch sqliteErr.Code() { - case sqlite3.SQLITE_CONSTRAINT_UNIQUE, - sqlite3.SQLITE_CONSTRAINT_PRIMARYKEY: - return db.ErrAlreadyExists - case sqlite3.SQLITE_BUSY, - sqlite3.SQLITE_BUSY_SNAPSHOT, - sqlite3.SQLITE_BUSY_RECOVERY: - return errBusy - case sqlite3.SQLITE_BUSY_TIMEOUT: - return db.ErrBusyTimeout - - // WORKAROUND: - // text copied from matrix dev chat: - // - // okay i've found a workaround for now. so between - // v1.29.0 and v1.29.2 (modernc.org/sqlite) is that - // slightly tweaked interruptOnDone() behaviour, which - // causes interrupt to (imo, correctly) get called when - // a context is cancelled to cancel the running query. the - // issue is that every single query after that point seems - // to still then return interrupted. so as you thought, - // maybe that query count isn't being decremented. i don't - // think it's our code, but i haven't ruled it out yet. - // - // the workaround for now is adding to our sqlite error - // processor to replace an SQLITE_INTERRUPTED code with - // driver.ErrBadConn, which hints to the golang sql package - // that the conn needs to be closed and a new one opened - // - case sqlite3.SQLITE_INTERRUPT: - return driver.ErrBadConn - } - - return err -} diff --git a/internal/db/bundb/tag_test.go b/internal/db/bundb/tag_test.go index 324398d27..3647c92de 100644 --- a/internal/db/bundb/tag_test.go +++ b/internal/db/bundb/tag_test.go @@ -19,6 +19,7 @@ package bundb_test import ( "context" + "errors" "testing" "github.com/stretchr/testify/suite" @@ -82,10 +83,20 @@ func (suite *TagTestSuite) TestPutTag() { // Subsequent inserts should fail // since all these tags are equivalent. - suite.ErrorIs(err, db.ErrAlreadyExists) + if !suite.ErrorIs(err, db.ErrAlreadyExists) { + suite.T().Logf("%T(%v) %v", err, err, unwrap(err)) + } } } func TestTagTestSuite(t *testing.T) { suite.Run(t, new(TagTestSuite)) } + +func unwrap(err error) (errs []error) { + for err != nil { + errs = append(errs, err) + err = errors.Unwrap(err) + } + return +} diff --git a/internal/db/error.go b/internal/db/error.go index b8e488297..43dd34df7 100644 --- a/internal/db/error.go +++ b/internal/db/error.go @@ -29,8 +29,4 @@ var ( // ErrAlreadyExists is returned when a conflict was encountered in the db when doing an insert. ErrAlreadyExists = errors.New("already exists") - - // ErrBusyTimeout is returned if the database connection indicates the connection is too busy - // to complete the supplied query. This is generally intended to be handled internally by the DB. - ErrBusyTimeout = errors.New("busy timeout") ) diff --git a/internal/db/postgres/driver.go b/internal/db/postgres/driver.go new file mode 100644 index 000000000..994c9ffba --- /dev/null +++ b/internal/db/postgres/driver.go @@ -0,0 +1,209 @@ +// GoToSocial +// Copyright (C) GoToSocial Authors admin@gotosocial.org +// SPDX-License-Identifier: AGPL-3.0-or-later +// +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// This program is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. +// +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . + +package postgres + +import ( + "context" + "database/sql/driver" + + pgx "github.com/jackc/pgx/v5/stdlib" + "github.com/superseriousbusiness/gotosocial/internal/db" +) + +var ( + // global PostgreSQL driver instances. + postgresDriver = pgx.GetDefaultDriver().(*pgx.Driver) + + // check the postgres driver types + // conforms to our interface types. + // (note SQLite doesn't export their + // driver types, and gets checked in + // tests very regularly anywho). + _ connIface = (*pgx.Conn)(nil) + _ stmtIface = (*pgx.Stmt)(nil) + _ rowsIface = (*pgx.Rows)(nil) +) + +// Driver is our own wrapper around the +// pgx/stdlib.Driver{} type in order to wrap further +// SQL driver types with our own err processing. +type Driver struct{} + +func (d *Driver) Open(name string) (driver.Conn, error) { + conn, err := postgresDriver.Open(name) + if err != nil { + err = processPostgresError(err) + return nil, err + } + return &postgresConn{conn.(connIface)}, nil +} + +func (d *Driver) OpenConnector(name string) (driver.Connector, error) { + cc, err := postgresDriver.OpenConnector(name) + if err != nil { + err = processPostgresError(err) + return nil, err + } + return &postgresConnector{driver: d, Connector: cc}, nil +} + +type postgresConnector struct { + driver *Driver + driver.Connector +} + +func (c *postgresConnector) Driver() driver.Driver { return c.driver } + +func (c *postgresConnector) Connect(ctx context.Context) (driver.Conn, error) { + conn, err := c.Connector.Connect(ctx) + if err != nil { + err = processPostgresError(err) + return nil, err + } + return &postgresConn{conn.(connIface)}, nil +} + +type postgresConn struct{ connIface } + +func (c *postgresConn) Begin() (driver.Tx, error) { + return c.BeginTx(context.Background(), driver.TxOptions{}) +} + +func (c *postgresConn) BeginTx(ctx context.Context, opts driver.TxOptions) (driver.Tx, error) { + tx, err := c.connIface.BeginTx(ctx, opts) + err = processPostgresError(err) + if err != nil { + return nil, err + } + return &postgresTx{tx}, nil +} + +func (c *postgresConn) Prepare(query string) (driver.Stmt, error) { + return c.PrepareContext(context.Background(), query) +} + +func (c *postgresConn) PrepareContext(ctx context.Context, query string) (driver.Stmt, error) { + st, err := c.connIface.PrepareContext(ctx, query) + err = processPostgresError(err) + if err != nil { + return nil, err + } + return &postgresStmt{st.(stmtIface)}, nil +} + +func (c *postgresConn) Exec(query string, args []driver.Value) (driver.Result, error) { + return c.ExecContext(context.Background(), query, db.ToNamedValues(args)) +} + +func (c *postgresConn) ExecContext(ctx context.Context, query string, args []driver.NamedValue) (driver.Result, error) { + result, err := c.connIface.ExecContext(ctx, query, args) + err = processPostgresError(err) + return result, err +} + +func (c *postgresConn) Query(query string, args []driver.Value) (driver.Rows, error) { + return c.QueryContext(context.Background(), query, db.ToNamedValues(args)) +} + +func (c *postgresConn) QueryContext(ctx context.Context, query string, args []driver.NamedValue) (driver.Rows, error) { + rows, err := c.connIface.QueryContext(ctx, query, args) + err = processPostgresError(err) + if err != nil { + return nil, err + } + return &postgresRows{rows.(rowsIface)}, nil +} + +func (c *postgresConn) Close() error { + err := c.connIface.Close() + return processPostgresError(err) +} + +type postgresTx struct{ driver.Tx } + +func (tx *postgresTx) Commit() error { + err := tx.Tx.Commit() + return processPostgresError(err) +} + +func (tx *postgresTx) Rollback() error { + err := tx.Tx.Rollback() + return processPostgresError(err) +} + +type postgresStmt struct{ stmtIface } + +func (stmt *postgresStmt) Exec(args []driver.Value) (driver.Result, error) { + return stmt.ExecContext(context.Background(), db.ToNamedValues(args)) +} + +func (stmt *postgresStmt) ExecContext(ctx context.Context, args []driver.NamedValue) (driver.Result, error) { + res, err := stmt.stmtIface.ExecContext(ctx, args) + err = processPostgresError(err) + return res, err +} + +func (stmt *postgresStmt) Query(args []driver.Value) (driver.Rows, error) { + return stmt.QueryContext(context.Background(), db.ToNamedValues(args)) +} + +func (stmt *postgresStmt) QueryContext(ctx context.Context, args []driver.NamedValue) (driver.Rows, error) { + rows, err := stmt.stmtIface.QueryContext(ctx, args) + err = processPostgresError(err) + if err != nil { + return nil, err + } + return &postgresRows{rows.(rowsIface)}, nil +} + +type postgresRows struct{ rowsIface } + +func (r *postgresRows) Next(dest []driver.Value) error { + err := r.rowsIface.Next(dest) + err = processPostgresError(err) + return err +} + +func (r *postgresRows) Close() error { + err := r.rowsIface.Close() + err = processPostgresError(err) + return err +} + +type connIface interface { + driver.Conn + driver.ConnPrepareContext + driver.ExecerContext + driver.QueryerContext + driver.ConnBeginTx +} + +type stmtIface interface { + driver.Stmt + driver.StmtExecContext + driver.StmtQueryContext +} + +type rowsIface interface { + driver.Rows + driver.RowsColumnTypeDatabaseTypeName + driver.RowsColumnTypeLength + driver.RowsColumnTypePrecisionScale + driver.RowsColumnTypeScanType + driver.RowsColumnTypeScanType +} diff --git a/internal/db/postgres/errors.go b/internal/db/postgres/errors.go new file mode 100644 index 000000000..cb8989a73 --- /dev/null +++ b/internal/db/postgres/errors.go @@ -0,0 +1,46 @@ +// GoToSocial +// Copyright (C) GoToSocial Authors admin@gotosocial.org +// SPDX-License-Identifier: AGPL-3.0-or-later +// +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// This program is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. +// +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . + +package postgres + +import ( + "fmt" + + "github.com/jackc/pgx/v5/pgconn" + "github.com/superseriousbusiness/gotosocial/internal/db" +) + +// processPostgresError processes an error, replacing any +// postgres specific errors with our own error type +func processPostgresError(err error) error { + // Attempt to cast as postgres + pgErr, ok := err.(*pgconn.PgError) + if !ok { + return err + } + + // Handle supplied error code: + // (https://www.postgresql.org/docs/10/errcodes-appendix.html) + switch pgErr.Code { //nolint + case "23505" /* unique_violation */ : + return db.ErrAlreadyExists + } + + // Wrap the returned error with the code and + // extended code for easier debugging later. + return fmt.Errorf("%w (code=%s)", err, pgErr.Code) +} diff --git a/internal/db/sqlite/driver.go b/internal/db/sqlite/driver.go new file mode 100644 index 000000000..11cb6b27d --- /dev/null +++ b/internal/db/sqlite/driver.go @@ -0,0 +1,197 @@ +// GoToSocial +// Copyright (C) GoToSocial Authors admin@gotosocial.org +// SPDX-License-Identifier: AGPL-3.0-or-later +// +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// This program is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. +// +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . + +//go:build !wasmsqlite3 + +package sqlite + +import ( + "context" + "database/sql/driver" + + "modernc.org/sqlite" + + "github.com/superseriousbusiness/gotosocial/internal/db" +) + +// Driver is our own wrapper around the +// sqlite.Driver{} type in order to wrap +// further SQL types with our own +// functionality, e.g. err processing. +type Driver struct{ sqlite.Driver } + +func (d *Driver) Open(name string) (driver.Conn, error) { + conn, err := d.Driver.Open(name) + if err != nil { + err = processSQLiteError(err) + return nil, err + } + return &sqliteConn{conn.(connIface)}, nil +} + +type sqliteConn struct{ connIface } + +func (c *sqliteConn) Begin() (driver.Tx, error) { + return c.BeginTx(context.Background(), driver.TxOptions{}) +} + +func (c *sqliteConn) BeginTx(ctx context.Context, opts driver.TxOptions) (tx driver.Tx, err error) { + tx, err = c.connIface.BeginTx(ctx, opts) + err = processSQLiteError(err) + if err != nil { + return nil, err + } + return &sqliteTx{tx}, nil +} + +func (c *sqliteConn) Prepare(query string) (driver.Stmt, error) { + return c.PrepareContext(context.Background(), query) +} + +func (c *sqliteConn) PrepareContext(ctx context.Context, query string) (stmt driver.Stmt, err error) { + stmt, err = c.connIface.PrepareContext(ctx, query) + err = processSQLiteError(err) + if err != nil { + return nil, err + } + return &sqliteStmt{stmtIface: stmt.(stmtIface)}, nil +} + +func (c *sqliteConn) Exec(query string, args []driver.Value) (driver.Result, error) { + return c.ExecContext(context.Background(), query, db.ToNamedValues(args)) +} + +func (c *sqliteConn) ExecContext(ctx context.Context, query string, args []driver.NamedValue) (res driver.Result, err error) { + res, err = c.connIface.ExecContext(ctx, query, args) + err = processSQLiteError(err) + return +} + +func (c *sqliteConn) Query(query string, args []driver.Value) (driver.Rows, error) { + return c.QueryContext(context.Background(), query, db.ToNamedValues(args)) +} + +func (c *sqliteConn) QueryContext(ctx context.Context, query string, args []driver.NamedValue) (rows driver.Rows, err error) { + rows, err = c.connIface.QueryContext(ctx, query, args) + err = processSQLiteError(err) + if err != nil { + return nil, err + } + return &sqliteRows{rows.(rowsIface)}, nil +} + +func (c *sqliteConn) Close() (err error) { + // see: https://www.sqlite.org/pragma.html#pragma_optimize + const onClose = "PRAGMA analysis_limit=1000; PRAGMA optimize;" + _, _ = c.connIface.ExecContext(context.Background(), onClose, nil) + + // Finally, close the conn. + err = c.connIface.Close() + return +} + +type sqliteTx struct{ driver.Tx } + +func (tx *sqliteTx) Commit() (err error) { + err = tx.Tx.Commit() + err = processSQLiteError(err) + return +} + +func (tx *sqliteTx) Rollback() (err error) { + err = tx.Tx.Rollback() + err = processSQLiteError(err) + return +} + +type sqliteStmt struct{ stmtIface } + +func (stmt *sqliteStmt) Exec(args []driver.Value) (driver.Result, error) { + return stmt.ExecContext(context.Background(), db.ToNamedValues(args)) +} + +func (stmt *sqliteStmt) ExecContext(ctx context.Context, args []driver.NamedValue) (res driver.Result, err error) { + res, err = stmt.stmtIface.ExecContext(ctx, args) + err = processSQLiteError(err) + return +} + +func (stmt *sqliteStmt) Query(args []driver.Value) (driver.Rows, error) { + return stmt.QueryContext(context.Background(), db.ToNamedValues(args)) +} + +func (stmt *sqliteStmt) QueryContext(ctx context.Context, args []driver.NamedValue) (rows driver.Rows, err error) { + rows, err = stmt.stmtIface.QueryContext(ctx, args) + err = processSQLiteError(err) + if err != nil { + return nil, err + } + return &sqliteRows{rows.(rowsIface)}, nil +} + +func (stmt *sqliteStmt) Close() (err error) { + err = stmt.stmtIface.Close() + err = processSQLiteError(err) + return +} + +type sqliteRows struct{ rowsIface } + +func (r *sqliteRows) Next(dest []driver.Value) (err error) { + err = r.rowsIface.Next(dest) + err = processSQLiteError(err) + return +} + +func (r *sqliteRows) Close() (err error) { + err = r.rowsIface.Close() + err = processSQLiteError(err) + return +} + +// connIface is the driver.Conn interface +// types (and the like) that modernc.org/sqlite.conn +// conforms to. Useful so you don't need +// to repeatedly perform checks yourself. +type connIface interface { + driver.Conn + driver.ConnBeginTx + driver.ConnPrepareContext + driver.ExecerContext + driver.QueryerContext +} + +// StmtIface is the driver.Stmt interface +// types (and the like) that modernc.org/sqlite.stmt +// conforms to. Useful so you don't need +// to repeatedly perform checks yourself. +type stmtIface interface { + driver.Stmt + driver.StmtExecContext + driver.StmtQueryContext +} + +// RowsIface is the driver.Rows interface +// types (and the like) that modernc.org/sqlite.rows +// conforms to. Useful so you don't need +// to repeatedly perform checks yourself. +type rowsIface interface { + driver.Rows + driver.RowsColumnTypeDatabaseTypeName + driver.RowsColumnTypeLength + driver.RowsColumnTypeScanType +} diff --git a/internal/db/sqlite/driver_wasmsqlite3.go b/internal/db/sqlite/driver_wasmsqlite3.go new file mode 100644 index 000000000..afe499a98 --- /dev/null +++ b/internal/db/sqlite/driver_wasmsqlite3.go @@ -0,0 +1,211 @@ +// GoToSocial +// Copyright (C) GoToSocial Authors admin@gotosocial.org +// SPDX-License-Identifier: AGPL-3.0-or-later +// +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// This program is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. +// +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . + +//go:build wasmsqlite3 + +package sqlite + +import ( + "context" + "database/sql/driver" + + "github.com/superseriousbusiness/gotosocial/internal/db" + + "github.com/ncruces/go-sqlite3" + sqlite3driver "github.com/ncruces/go-sqlite3/driver" + _ "github.com/ncruces/go-sqlite3/embed" // embed wasm binary + _ "github.com/ncruces/go-sqlite3/vfs/memdb" // include memdb vfs +) + +// Driver is our own wrapper around the +// driver.SQLite{} type in order to wrap +// further SQL types with our own +// functionality, e.g. err processing. +type Driver struct{ sqlite3driver.SQLite } + +func (d *Driver) Open(name string) (driver.Conn, error) { + conn, err := d.SQLite.Open(name) + if err != nil { + err = processSQLiteError(err) + return nil, err + } + return &sqliteConn{conn.(connIface)}, nil +} + +func (d *Driver) OpenConnector(name string) (driver.Connector, error) { + cc, err := d.SQLite.OpenConnector(name) + if err != nil { + return nil, err + } + return &sqliteConnector{driver: d, Connector: cc}, nil +} + +type sqliteConnector struct { + driver *Driver + driver.Connector +} + +func (c *sqliteConnector) Driver() driver.Driver { return c.driver } + +func (c *sqliteConnector) Connect(ctx context.Context) (driver.Conn, error) { + conn, err := c.Connector.Connect(ctx) + err = processSQLiteError(err) + if err != nil { + return nil, err + } + return &sqliteConn{conn.(connIface)}, nil +} + +type sqliteConn struct{ connIface } + +func (c *sqliteConn) Begin() (driver.Tx, error) { + return c.BeginTx(context.Background(), driver.TxOptions{}) +} + +func (c *sqliteConn) BeginTx(ctx context.Context, opts driver.TxOptions) (tx driver.Tx, err error) { + tx, err = c.connIface.BeginTx(ctx, opts) + err = processSQLiteError(err) + if err != nil { + return nil, err + } + return &sqliteTx{tx}, nil +} + +func (c *sqliteConn) Prepare(query string) (driver.Stmt, error) { + return c.PrepareContext(context.Background(), query) +} + +func (c *sqliteConn) PrepareContext(ctx context.Context, query string) (stmt driver.Stmt, err error) { + stmt, err = c.connIface.PrepareContext(ctx, query) + err = processSQLiteError(err) + if err != nil { + return nil, err + } + return &sqliteStmt{stmtIface: stmt.(stmtIface)}, nil +} + +func (c *sqliteConn) Exec(query string, args []driver.Value) (driver.Result, error) { + return c.ExecContext(context.Background(), query, db.ToNamedValues(args)) +} + +func (c *sqliteConn) ExecContext(ctx context.Context, query string, args []driver.NamedValue) (res driver.Result, err error) { + res, err = c.connIface.ExecContext(ctx, query, args) + err = processSQLiteError(err) + return +} + +func (c *sqliteConn) Close() (err error) { + // Get acces the underlying raw sqlite3 conn. + raw := c.connIface.(sqlite3.DriverConn).Raw() + + // see: https://www.sqlite.org/pragma.html#pragma_optimize + const onClose = "PRAGMA analysis_limit=1000; PRAGMA optimize;" + _ = raw.Exec(onClose) + + // Finally, close. + err = raw.Close() + return +} + +type sqliteTx struct{ driver.Tx } + +func (tx *sqliteTx) Commit() (err error) { + err = tx.Tx.Commit() + err = processSQLiteError(err) + return +} + +func (tx *sqliteTx) Rollback() (err error) { + err = tx.Tx.Rollback() + err = processSQLiteError(err) + return +} + +type sqliteStmt struct{ stmtIface } + +func (stmt *sqliteStmt) Exec(args []driver.Value) (driver.Result, error) { + return stmt.ExecContext(context.Background(), db.ToNamedValues(args)) +} + +func (stmt *sqliteStmt) ExecContext(ctx context.Context, args []driver.NamedValue) (res driver.Result, err error) { + res, err = stmt.stmtIface.ExecContext(ctx, args) + err = processSQLiteError(err) + return +} + +func (stmt *sqliteStmt) Query(args []driver.Value) (driver.Rows, error) { + return stmt.QueryContext(context.Background(), db.ToNamedValues(args)) +} + +func (stmt *sqliteStmt) QueryContext(ctx context.Context, args []driver.NamedValue) (rows driver.Rows, err error) { + rows, err = stmt.stmtIface.QueryContext(ctx, args) + err = processSQLiteError(err) + if err != nil { + return nil, err + } + return &sqliteRows{rows.(rowsIface)}, nil +} + +func (stmt *sqliteStmt) Close() (err error) { + err = stmt.stmtIface.Close() + err = processSQLiteError(err) + return +} + +type sqliteRows struct{ rowsIface } + +func (r *sqliteRows) Next(dest []driver.Value) (err error) { + err = r.rowsIface.Next(dest) + err = processSQLiteError(err) + return +} + +func (r *sqliteRows) Close() (err error) { + err = r.rowsIface.Close() + err = processSQLiteError(err) + return +} + +// connIface is the driver.Conn interface +// types (and the like) that go-sqlite3/driver.conn +// conforms to. Useful so you don't need +// to repeatedly perform checks yourself. +type connIface interface { + driver.Conn + driver.ConnBeginTx + driver.ConnPrepareContext + driver.ExecerContext +} + +// StmtIface is the driver.Stmt interface +// types (and the like) that go-sqlite3/driver.stmt +// conforms to. Useful so you don't need +// to repeatedly perform checks yourself. +type stmtIface interface { + driver.Stmt + driver.StmtExecContext + driver.StmtQueryContext +} + +// RowsIface is the driver.Rows interface +// types (and the like) that go-sqlite3/driver.rows +// conforms to. Useful so you don't need +// to repeatedly perform checks yourself. +type rowsIface interface { + driver.Rows + driver.RowsColumnTypeDatabaseTypeName +} diff --git a/internal/db/sqlite/errors.go b/internal/db/sqlite/errors.go new file mode 100644 index 000000000..b07b026de --- /dev/null +++ b/internal/db/sqlite/errors.go @@ -0,0 +1,62 @@ +// GoToSocial +// Copyright (C) GoToSocial Authors admin@gotosocial.org +// SPDX-License-Identifier: AGPL-3.0-or-later +// +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// This program is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. +// +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . + +//go:build !wasmsqlite3 + +package sqlite + +import ( + "database/sql/driver" + "fmt" + + "modernc.org/sqlite" + sqlite3 "modernc.org/sqlite/lib" + + "github.com/superseriousbusiness/gotosocial/internal/db" +) + +// processSQLiteError processes an sqlite3.Error to +// handle conversion to any of our common db types. +func processSQLiteError(err error) error { + // Attempt to cast as sqlite error. + sqliteErr, ok := err.(*sqlite.Error) + if !ok { + return err + } + + // Handle supplied error code: + switch sqliteErr.Code() { + case sqlite3.SQLITE_CONSTRAINT_UNIQUE, + sqlite3.SQLITE_CONSTRAINT_PRIMARYKEY: + return db.ErrAlreadyExists + + // Busy should be very rare, but + // on busy tell the database to close + // the connection, re-open and re-attempt + // which should give a necessary timeout. + case sqlite3.SQLITE_BUSY, + sqlite3.SQLITE_BUSY_RECOVERY, + sqlite3.SQLITE_BUSY_SNAPSHOT: + return driver.ErrBadConn + } + + // Wrap the returned error with the code and + // extended code for easier debugging later. + return fmt.Errorf("%w (code=%d)", err, + sqliteErr.Code(), + ) +} diff --git a/internal/db/sqlite/errors_wasmsqlite3.go b/internal/db/sqlite/errors_wasmsqlite3.go new file mode 100644 index 000000000..26668a898 --- /dev/null +++ b/internal/db/sqlite/errors_wasmsqlite3.go @@ -0,0 +1,60 @@ +// GoToSocial +// Copyright (C) GoToSocial Authors admin@gotosocial.org +// SPDX-License-Identifier: AGPL-3.0-or-later +// +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// This program is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. +// +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . + +//go:build wasmsqlite3 + +package sqlite + +import ( + "database/sql/driver" + "fmt" + + "github.com/ncruces/go-sqlite3" + "github.com/superseriousbusiness/gotosocial/internal/db" +) + +// processSQLiteError processes an sqlite3.Error to +// handle conversion to any of our common db types. +func processSQLiteError(err error) error { + // Attempt to cast as sqlite error. + sqliteErr, ok := err.(*sqlite3.Error) + if !ok { + return err + } + + // Handle supplied error code: + switch sqliteErr.ExtendedCode() { + case sqlite3.CONSTRAINT_UNIQUE, + sqlite3.CONSTRAINT_PRIMARYKEY: + return db.ErrAlreadyExists + + // Busy should be very rare, but on + // busy tell the database to close the + // connection, re-open and re-attempt + // which should give necessary timeout. + case sqlite3.BUSY_RECOVERY, + sqlite3.BUSY_SNAPSHOT: + return driver.ErrBadConn + } + + // Wrap the returned error with the code and + // extended code for easier debugging later. + return fmt.Errorf("%w (code=%d extended=%d)", err, + sqliteErr.Code(), + sqliteErr.ExtendedCode(), + ) +} diff --git a/internal/db/util.go b/internal/db/util.go new file mode 100644 index 000000000..9cd29f2fc --- /dev/null +++ b/internal/db/util.go @@ -0,0 +1,35 @@ +// GoToSocial +// Copyright (C) GoToSocial Authors admin@gotosocial.org +// SPDX-License-Identifier: AGPL-3.0-or-later +// +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// This program is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. +// +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . + +package db + +import "database/sql/driver" + +// ToNamedValues converts older driver.Value types to driver.NamedValue types. +func ToNamedValues(args []driver.Value) []driver.NamedValue { + if args == nil { + return nil + } + args2 := make([]driver.NamedValue, len(args)) + for i := range args { + args2[i] = driver.NamedValue{ + Ordinal: i + 1, + Value: args[i], + } + } + return args2 +} diff --git a/internal/gtserror/multi_test.go b/internal/gtserror/multi_test.go index 10c342415..b58c1b881 100644 --- a/internal/gtserror/multi_test.go +++ b/internal/gtserror/multi_test.go @@ -43,10 +43,6 @@ func TestMultiError(t *testing.T) { t.Error("should be db.ErrAlreadyExists") } - if errors.Is(err, db.ErrBusyTimeout) { - t.Error("should not be db.ErrBusyTimeout") - } - errString := err.Error() expected := `sql: no rows in result set oopsie woopsie we did a fucky wucky etc diff --git a/internal/util/paging_test.go b/internal/util/paging_test.go index 66c5b2c56..bab7dcb7b 100644 --- a/internal/util/paging_test.go +++ b/internal/util/paging_test.go @@ -30,6 +30,7 @@ type PagingSuite struct { } func (suite *PagingSuite) TestPagingStandard() { + config.SetProtocol("https") config.SetHost("example.org") params := util.PageableResponseParams{ @@ -52,6 +53,7 @@ func (suite *PagingSuite) TestPagingStandard() { } func (suite *PagingSuite) TestPagingNoLimit() { + config.SetProtocol("https") config.SetHost("example.org") params := util.PageableResponseParams{ @@ -73,6 +75,7 @@ func (suite *PagingSuite) TestPagingNoLimit() { } func (suite *PagingSuite) TestPagingNoNextID() { + config.SetProtocol("https") config.SetHost("example.org") params := util.PageableResponseParams{ @@ -94,6 +97,7 @@ func (suite *PagingSuite) TestPagingNoNextID() { } func (suite *PagingSuite) TestPagingNoPrevID() { + config.SetProtocol("https") config.SetHost("example.org") params := util.PageableResponseParams{ @@ -115,6 +119,7 @@ func (suite *PagingSuite) TestPagingNoPrevID() { } func (suite *PagingSuite) TestPagingNoItems() { + config.SetProtocol("https") config.SetHost("example.org") params := util.PageableResponseParams{ diff --git a/test/run-postgres.sh b/test/run-postgres.sh index 029a72793..8da1a3276 100755 --- a/test/run-postgres.sh +++ b/test/run-postgres.sh @@ -2,6 +2,11 @@ set -e +# Determine available docker binary +_docker=$(command -v 'podman') || \ +_docker=$(command -v 'docker') || \ +{ echo 'docker not found'; exit 1; } + # Ensure test args are set. ARGS=${@}; [ -z "$ARGS" ] && \ ARGS='./...' @@ -10,33 +15,32 @@ ARGS='./...' DB_NAME='postgres' DB_USER='postgres' DB_PASS='postgres' +DB_IP='127.0.0.1' DB_PORT=5432 # Start postgres container -CID=$(docker run --detach \ +CID=$($_docker run --detach \ + --publish "${DB_IP}:${DB_PORT}:${DB_PORT}" \ --env "POSTGRES_DB=${DB_NAME}" \ --env "POSTGRES_USER=${DB_USER}" \ --env "POSTGRES_PASSWORD=${DB_PASS}" \ --env "POSTGRES_HOST_AUTH_METHOD=trust" \ --env "PGHOST=0.0.0.0" \ --env "PGPORT=${DB_PORT}" \ - 'postgres:latest') + 'docker.io/postgres:latest') # On exit kill the container -trap "docker kill ${CID}" exit +trap "$_docker kill ${CID}" exit sleep 5 #docker exec "$CID" psql --user "$DB_USER" --password "$DB_PASS" -c "CREATE DATABASE \"${DB_NAME}\" WITH LOCALE \"C.UTF-8\" TEMPLATE \"template0\";" -docker exec "$CID" psql --user "$DB_USER" --password "$DB_PASS" -c "GRANT ALL PRIVILEGES ON DATABASE \"${DB_NAME}\" TO \"${DB_USER}\";" - -# Get running container IP -IP=$(docker container inspect "${CID}" \ - --format '{{ .NetworkSettings.IPAddress }}') +$_docker exec "$CID" psql --user "$DB_USER" --password "$DB_PASS" -c "GRANT ALL PRIVILEGES ON DATABASE \"${DB_NAME}\" TO \"${DB_USER}\";" +env \ GTS_DB_TYPE=postgres \ -GTS_DB_ADDRESS=${IP} \ +GTS_DB_ADDRESS=${DB_IP} \ GTS_DB_PORT=${DB_PORT} \ GTS_DB_USER=${DB_USER} \ GTS_DB_PASSWORD=${DB_PASS} \ GTS_DB_DATABASE=${DB_NAME} \ -go test ./... -p 1 ${ARGS} \ No newline at end of file +go test -p 1 ${ARGS} \ No newline at end of file diff --git a/testrig/config.go b/testrig/config.go index 93f3c5523..30beaa910 100644 --- a/testrig/config.go +++ b/testrig/config.go @@ -18,8 +18,8 @@ package testrig import ( - "cmp" "os" + "strconv" "time" "codeberg.org/gruf/go-bytesize" @@ -28,128 +28,149 @@ import ( "github.com/superseriousbusiness/gotosocial/internal/language" ) -// InitTestConfig initializes viper configuration with test defaults. +// InitTestConfig initializes viper +// configuration with test defaults. func InitTestConfig() { - config.Config(func(cfg *config.Configuration) { - *cfg = testDefaults + config.Defaults = testDefaults() + config.Reset() +} + +func testDefaults() config.Configuration { + return config.Configuration{ + LogLevel: envStr("GTS_LOG_LEVEL", "error"), + LogTimestampFormat: "02/01/2006 15:04:05.000", + LogDbQueries: true, + ApplicationName: "gotosocial", + LandingPageUser: "", + ConfigPath: "", + Host: "localhost:8080", + AccountDomain: "localhost:8080", + Protocol: "http", + BindAddress: "127.0.0.1", + Port: 8080, + TrustedProxies: []string{"127.0.0.1/32", "::1"}, + DbType: envStr("GTS_DB_TYPE", "sqlite"), + DbAddress: envStr("GTS_DB_ADDRESS", ":memory:"), + DbPort: envInt("GTS_DB_PORT", 0), + DbUser: envStr("GTS_DB_USER", ""), + DbPassword: envStr("GTS_DB_PASSWORD", ""), + DbDatabase: envStr("GTS_DB_DATABASE", ""), + DbTLSMode: envStr("GTS_DB_TLS_MODE", ""), + DbTLSCACert: envStr("GTS_DB_TLS_CA_CERT", ""), + DbMaxOpenConnsMultiplier: 8, + DbSqliteJournalMode: "WAL", + DbSqliteSynchronous: "NORMAL", + DbSqliteCacheSize: 8 * bytesize.MiB, + DbSqliteBusyTimeout: time.Minute * 5, + + WebTemplateBaseDir: "./web/template/", + WebAssetBaseDir: "./web/assets/", + + InstanceFederationMode: config.InstanceFederationModeDefault, + InstanceFederationSpamFilter: true, + InstanceExposePeers: true, + InstanceExposeSuspended: true, + InstanceExposeSuspendedWeb: true, + InstanceDeliverToSharedInboxes: true, + InstanceLanguages: language.Languages{ + { + TagStr: "nl", + }, + { + TagStr: "en-gb", + }, + }, + + AccountsRegistrationOpen: true, + AccountsReasonRequired: true, + AccountsAllowCustomCSS: true, + AccountsCustomCSSLength: 10000, + + MediaImageMaxSize: 10485760, // 10MiB + MediaVideoMaxSize: 41943040, // 40MiB + MediaDescriptionMinChars: 0, + MediaDescriptionMaxChars: 500, + MediaRemoteCacheDays: 7, + MediaEmojiLocalMaxSize: 51200, // 50KiB + MediaEmojiRemoteMaxSize: 102400, // 100KiB + MediaCleanupFrom: "00:00", // midnight. + MediaCleanupEvery: 24 * time.Hour, // 1/day. + + // the testrig only uses in-memory storage, so we can + // safely set this value to 'test' to avoid running storage + // migrations, and other silly things like that + StorageBackend: "test", + StorageLocalBasePath: "", + + StatusesMaxChars: 5000, + StatusesPollMaxOptions: 6, + StatusesPollOptionMaxChars: 50, + StatusesMediaMaxFiles: 6, + + LetsEncryptEnabled: false, + LetsEncryptPort: 0, + LetsEncryptCertDir: "", + LetsEncryptEmailAddress: "", + + OIDCEnabled: false, + OIDCIdpName: "", + OIDCSkipVerification: false, + OIDCIssuer: "", + OIDCClientID: "", + OIDCClientSecret: "", + OIDCScopes: []string{oidc.ScopeOpenID, "profile", "email", "groups"}, + OIDCLinkExisting: false, + OIDCAdminGroups: []string{"adminRole"}, + OIDCAllowedGroups: []string{"allowedRole"}, + + SMTPHost: "", + SMTPPort: 0, + SMTPUsername: "", + SMTPPassword: "", + SMTPFrom: "GoToSocial", + SMTPDiscloseRecipients: false, + + TracingEnabled: false, + TracingEndpoint: "localhost:4317", + TracingTransport: "grpc", + TracingInsecureTransport: true, + + MetricsEnabled: false, + MetricsAuthEnabled: false, + + SyslogEnabled: false, + SyslogProtocol: "udp", + SyslogAddress: "localhost:514", + + AdvancedCookiesSamesite: "lax", + AdvancedRateLimitRequests: 0, // disabled + AdvancedThrottlingMultiplier: 0, // disabled + AdvancedSenderMultiplier: 0, // 1 sender only, regardless of CPU + + SoftwareVersion: "0.0.0-testrig", + + // simply use cache defaults. + Cache: config.Defaults.Cache, + } +} + +func envInt(key string, _default int) int { + return env(key, _default, func(value string) int { + i, _ := strconv.Atoi(value) + return i }) } -var testDefaults = config.Configuration{ - LogLevel: cmp.Or(os.Getenv("GTS_LOG_LEVEL"), "error"), - LogTimestampFormat: "02/01/2006 15:04:05.000", - LogDbQueries: true, - ApplicationName: "gotosocial", - LandingPageUser: "", - ConfigPath: "", - Host: "localhost:8080", - AccountDomain: "localhost:8080", - Protocol: "http", - BindAddress: "127.0.0.1", - Port: 8080, - TrustedProxies: []string{"127.0.0.1/32", "::1"}, - - DbType: "sqlite", - DbAddress: ":memory:", - DbPort: 5432, - DbUser: "postgres", - DbPassword: "postgres", - DbDatabase: "postgres", - DbTLSMode: "disable", - DbTLSCACert: "", - DbMaxOpenConnsMultiplier: 8, - DbSqliteJournalMode: "WAL", - DbSqliteSynchronous: "NORMAL", - DbSqliteCacheSize: 8 * bytesize.MiB, - DbSqliteBusyTimeout: time.Minute * 5, - - WebTemplateBaseDir: "./web/template/", - WebAssetBaseDir: "./web/assets/", - - InstanceFederationMode: config.InstanceFederationModeDefault, - InstanceFederationSpamFilter: true, - InstanceExposePeers: true, - InstanceExposeSuspended: true, - InstanceExposeSuspendedWeb: true, - InstanceDeliverToSharedInboxes: true, - InstanceLanguages: language.Languages{ - { - TagStr: "nl", - }, - { - TagStr: "en-gb", - }, - }, - - AccountsRegistrationOpen: true, - AccountsReasonRequired: true, - AccountsAllowCustomCSS: true, - AccountsCustomCSSLength: 10000, - - MediaImageMaxSize: 10485760, // 10MiB - MediaVideoMaxSize: 41943040, // 40MiB - MediaDescriptionMinChars: 0, - MediaDescriptionMaxChars: 500, - MediaRemoteCacheDays: 7, - MediaEmojiLocalMaxSize: 51200, // 50KiB - MediaEmojiRemoteMaxSize: 102400, // 100KiB - MediaCleanupFrom: "00:00", // midnight. - MediaCleanupEvery: 24 * time.Hour, // 1/day. - - // the testrig only uses in-memory storage, so we can - // safely set this value to 'test' to avoid running storage - // migrations, and other silly things like that - StorageBackend: "test", - StorageLocalBasePath: "", - - StatusesMaxChars: 5000, - StatusesPollMaxOptions: 6, - StatusesPollOptionMaxChars: 50, - StatusesMediaMaxFiles: 6, - - LetsEncryptEnabled: false, - LetsEncryptPort: 0, - LetsEncryptCertDir: "", - LetsEncryptEmailAddress: "", - - OIDCEnabled: false, - OIDCIdpName: "", - OIDCSkipVerification: false, - OIDCIssuer: "", - OIDCClientID: "", - OIDCClientSecret: "", - OIDCScopes: []string{oidc.ScopeOpenID, "profile", "email", "groups"}, - OIDCLinkExisting: false, - OIDCAdminGroups: []string{"adminRole"}, - OIDCAllowedGroups: []string{"allowedRole"}, - - SMTPHost: "", - SMTPPort: 0, - SMTPUsername: "", - SMTPPassword: "", - SMTPFrom: "GoToSocial", - SMTPDiscloseRecipients: false, - - TracingEnabled: false, - TracingEndpoint: "localhost:4317", - TracingTransport: "grpc", - TracingInsecureTransport: true, - - MetricsEnabled: false, - MetricsAuthEnabled: false, - - SyslogEnabled: false, - SyslogProtocol: "udp", - SyslogAddress: "localhost:514", - - AdvancedCookiesSamesite: "lax", - AdvancedRateLimitRequests: 0, // disabled - AdvancedThrottlingMultiplier: 0, // disabled - AdvancedSenderMultiplier: 0, // 1 sender only, regardless of CPU - AdvancedHeaderFilterMode: config.RequestHeaderFilterModeBlock, - - SoftwareVersion: "0.0.0-testrig", - - // simply use cache defaults. - Cache: config.Defaults.Cache, +func envStr(key string, _default string) string { + return env(key, _default, func(value string) string { + return value + }) +} + +func env[T any](key string, _default T, parse func(string) T) T { + value, ok := os.LookupEnv(key) + if ok { + return parse(value) + } + return _default } diff --git a/testrig/db.go b/testrig/db.go index 83bc46ec8..faa7a910d 100644 --- a/testrig/db.go +++ b/testrig/db.go @@ -19,10 +19,7 @@ package testrig import ( "context" - "os" - "strconv" - "github.com/superseriousbusiness/gotosocial/internal/config" "github.com/superseriousbusiness/gotosocial/internal/db" "github.com/superseriousbusiness/gotosocial/internal/db/bundb" "github.com/superseriousbusiness/gotosocial/internal/gtsmodel" @@ -84,22 +81,6 @@ var testModels = []interface{}{ // If the environment variable GTS_DB_PORT is set, it will take that // value as the port instead. func NewTestDB(state *state.State) db.DB { - if alternateAddress := os.Getenv("GTS_DB_ADDRESS"); alternateAddress != "" { - config.SetDbAddress(alternateAddress) - } - - if alternateDBType := os.Getenv("GTS_DB_TYPE"); alternateDBType != "" { - config.SetDbType(alternateDBType) - } - - if alternateDBPort := os.Getenv("GTS_DB_PORT"); alternateDBPort != "" { - port, err := strconv.ParseUint(alternateDBPort, 10, 16) - if err != nil { - panic(err) - } - config.SetDbPort(int(port)) - } - state.Caches.Init() testDB, err := bundb.NewBunDBService(context.Background(), state) @@ -374,9 +355,10 @@ func StandardDBTeardown(db db.DB) { if db == nil { return } + defer db.Close() for _, m := range testModels { if err := db.DropTable(ctx, m); err != nil { - log.Panic(nil, err) + log.Error(ctx, err) } } } diff --git a/vendor/github.com/ncruces/go-sqlite3/.gitignore b/vendor/github.com/ncruces/go-sqlite3/.gitignore new file mode 100644 index 000000000..c8b2376cd --- /dev/null +++ b/vendor/github.com/ncruces/go-sqlite3/.gitignore @@ -0,0 +1,16 @@ +# Binaries for programs and plugins +*.exe +*.exe~ +*.dll +*.so +*.dylib + +# Test binary, built with `go test -c` +*.test + +# Output of the go coverage tool, specifically when used with LiteIDE +*.out + +# Dependency directories (remove the comment below to include it) +# vendor/ +tools \ No newline at end of file diff --git a/vendor/github.com/ncruces/go-sqlite3/LICENSE b/vendor/github.com/ncruces/go-sqlite3/LICENSE new file mode 100644 index 000000000..9bdc1df48 --- /dev/null +++ b/vendor/github.com/ncruces/go-sqlite3/LICENSE @@ -0,0 +1,21 @@ +MIT License + +Copyright (c) 2023 Nuno Cruces + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. diff --git a/vendor/github.com/ncruces/go-sqlite3/README.md b/vendor/github.com/ncruces/go-sqlite3/README.md new file mode 100644 index 000000000..c31414724 --- /dev/null +++ b/vendor/github.com/ncruces/go-sqlite3/README.md @@ -0,0 +1,113 @@ +# Go bindings to SQLite using Wazero + +[![Go Reference](https://pkg.go.dev/badge/image)](https://pkg.go.dev/github.com/ncruces/go-sqlite3) +[![Go Report](https://goreportcard.com/badge/github.com/ncruces/go-sqlite3)](https://goreportcard.com/report/github.com/ncruces/go-sqlite3) +[![Go Coverage](https://github.com/ncruces/go-sqlite3/wiki/coverage.svg)](https://github.com/ncruces/go-sqlite3/wiki/Test-coverage-report) + +Go module `github.com/ncruces/go-sqlite3` is a `cgo`-free [SQLite](https://sqlite.org/) wrapper.\ +It provides a [`database/sql`](https://pkg.go.dev/database/sql) compatible driver, +as well as direct access to most of the [C SQLite API](https://sqlite.org/cintro.html). + +It wraps a [Wasm](https://webassembly.org/) [build](embed/) of SQLite, +and uses [wazero](https://wazero.io/) as the runtime.\ +Go, wazero and [`x/sys`](https://pkg.go.dev/golang.org/x/sys) are the _only_ runtime dependencies [^1]. + +### Packages + +- [`github.com/ncruces/go-sqlite3`](https://pkg.go.dev/github.com/ncruces/go-sqlite3) + wraps the [C SQLite API](https://sqlite.org/cintro.html) + ([example usage](https://pkg.go.dev/github.com/ncruces/go-sqlite3#example-package)). +- [`github.com/ncruces/go-sqlite3/driver`](https://pkg.go.dev/github.com/ncruces/go-sqlite3/driver) + provides a [`database/sql`](https://pkg.go.dev/database/sql) driver + ([example usage](https://pkg.go.dev/github.com/ncruces/go-sqlite3/driver#example-package)). +- [`github.com/ncruces/go-sqlite3/embed`](https://pkg.go.dev/github.com/ncruces/go-sqlite3/embed) + embeds a build of SQLite into your application. +- [`github.com/ncruces/go-sqlite3/vfs`](https://pkg.go.dev/github.com/ncruces/go-sqlite3/vfs) + wraps the [C SQLite VFS API](https://sqlite.org/vfs.html) and provides a pure Go implementation. +- [`github.com/ncruces/go-sqlite3/gormlite`](https://pkg.go.dev/github.com/ncruces/go-sqlite3/gormlite) + provides a [GORM](https://gorm.io) driver. + +### Extensions + +- [`github.com/ncruces/go-sqlite3/ext/array`](https://pkg.go.dev/github.com/ncruces/go-sqlite3/ext/array) + provides the [`array`](https://sqlite.org/carray.html) table-valued function. +- [`github.com/ncruces/go-sqlite3/ext/blobio`](https://pkg.go.dev/github.com/ncruces/go-sqlite3/ext/blobio) + simplifies [incremental BLOB I/O](https://sqlite.org/c3ref/blob_open.html). +- [`github.com/ncruces/go-sqlite3/ext/csv`](https://pkg.go.dev/github.com/ncruces/go-sqlite3/ext/csv) + reads [comma-separated values](https://sqlite.org/csv.html). +- [`github.com/ncruces/go-sqlite3/ext/fileio`](https://pkg.go.dev/github.com/ncruces/go-sqlite3/ext/fileio) + reads, writes and lists files. +- [`github.com/ncruces/go-sqlite3/ext/hash`](https://pkg.go.dev/github.com/ncruces/go-sqlite3/ext/hash) + provides cryptographic hash functions. +- [`github.com/ncruces/go-sqlite3/ext/lines`](https://pkg.go.dev/github.com/ncruces/go-sqlite3/ext/lines) + reads data [line-by-line](https://github.com/asg017/sqlite-lines). +- [`github.com/ncruces/go-sqlite3/ext/pivot`](https://pkg.go.dev/github.com/ncruces/go-sqlite3/ext/pivot) + creates [pivot tables](https://github.com/jakethaw/pivot_vtab). +- [`github.com/ncruces/go-sqlite3/ext/statement`](https://pkg.go.dev/github.com/ncruces/go-sqlite3/ext/statement) + creates [parameterized views](https://github.com/0x09/sqlite-statement-vtab). +- [`github.com/ncruces/go-sqlite3/ext/stats`](https://pkg.go.dev/github.com/ncruces/go-sqlite3/ext/stats) + provides [statistics](https://www.oreilly.com/library/view/sql-in-a/9780596155322/ch04s02.html) functions. +- [`github.com/ncruces/go-sqlite3/ext/unicode`](https://pkg.go.dev/github.com/ncruces/go-sqlite3/ext/unicode) + provides [Unicode aware](https://sqlite.org/src/dir/ext/icu) functions. +- [`github.com/ncruces/go-sqlite3/ext/zorder`](https://pkg.go.dev/github.com/ncruces/go-sqlite3/ext/zorder) + maps multidimensional data to one dimension. +- [`github.com/ncruces/go-sqlite3/vfs/memdb`](https://pkg.go.dev/github.com/ncruces/go-sqlite3/vfs/memdb) + implements an in-memory VFS. +- [`github.com/ncruces/go-sqlite3/vfs/readervfs`](https://pkg.go.dev/github.com/ncruces/go-sqlite3/vfs/readervfs) + implements a VFS for immutable databases. +- [`github.com/ncruces/go-sqlite3/vfs/adiantum`](https://pkg.go.dev/github.com/ncruces/go-sqlite3/vfs/adiantum) + wraps a VFS to offer encryption at rest. + +### Advanced features + +- [incremental BLOB I/O](https://sqlite.org/c3ref/blob_open.html) +- [nested transactions](https://sqlite.org/lang_savepoint.html) +- [custom functions](https://sqlite.org/c3ref/create_function.html) +- [virtual tables](https://sqlite.org/vtab.html) +- [custom VFSes](https://sqlite.org/vfs.html) +- [online backup](https://sqlite.org/backup.html) +- [JSON support](https://sqlite.org/json1.html) +- [math functions](https://sqlite.org/lang_mathfunc.html) +- [full-text search](https://sqlite.org/fts5.html) +- [geospatial search](https://sqlite.org/geopoly.html) +- [encryption at rest](vfs/adiantum/README.md) +- [and more…](embed/README.md) + +### Caveats + +This module replaces the SQLite [OS Interface](https://sqlite.org/vfs.html) +(aka VFS) with a [pure Go](vfs/) implementation, +which has advantages and disadvantages. + +Read more about the Go VFS design [here](vfs/README.md). + +### Testing + +This project aims for [high test coverage](https://github.com/ncruces/go-sqlite3/wiki/Test-coverage-report). +It also benefits greatly from [SQLite's](https://sqlite.org/testing.html) and +[wazero's](https://tetrate.io/blog/introducing-wazero-from-tetrate/#:~:text=Rock%2Dsolid%20test%20approach) thorough testing. + +Every commit is [tested](.github/workflows/test.yml) on +Linux (amd64/arm64/386/riscv64/s390x), macOS (amd64/arm64), +Windows (amd64), FreeBSD (amd64), illumos (amd64), and Solaris (amd64). + +The Go VFS is tested by running SQLite's +[mptest](https://github.com/sqlite/sqlite/blob/master/mptest/mptest.c). + +### Performance + +Perfomance of the [`database/sql`](https://pkg.go.dev/database/sql) driver is +[competitive](https://github.com/cvilsmeier/go-sqlite-bench) with alternatives. + +The Wasm and VFS layers are also tested by running SQLite's +[speedtest1](https://github.com/sqlite/sqlite/blob/master/test/speedtest1.c). + +### Alternatives + +- [`modernc.org/sqlite`](https://pkg.go.dev/modernc.org/sqlite) +- [`crawshaw.io/sqlite`](https://pkg.go.dev/crawshaw.io/sqlite) +- [`github.com/mattn/go-sqlite3`](https://pkg.go.dev/github.com/mattn/go-sqlite3) +- [`github.com/zombiezen/go-sqlite`](https://pkg.go.dev/github.com/zombiezen/go-sqlite) + +[^1]: anything else you find in `go.mod` is either a test dependency, + or needed by one of the extensions. diff --git a/vendor/github.com/ncruces/go-sqlite3/backup.go b/vendor/github.com/ncruces/go-sqlite3/backup.go new file mode 100644 index 000000000..b16c7511e --- /dev/null +++ b/vendor/github.com/ncruces/go-sqlite3/backup.go @@ -0,0 +1,134 @@ +package sqlite3 + +// Backup is an handle to an ongoing online backup operation. +// +// https://sqlite.org/c3ref/backup.html +type Backup struct { + c *Conn + handle uint32 + otherc uint32 +} + +// Backup backs up srcDB on the src connection to the "main" database in dstURI. +// +// Backup opens the SQLite database file dstURI, +// and blocks until the entire backup is complete. +// Use [Conn.BackupInit] for incremental backup. +// +// https://sqlite.org/backup.html +func (src *Conn) Backup(srcDB, dstURI string) error { + b, err := src.BackupInit(srcDB, dstURI) + if err != nil { + return err + } + defer b.Close() + _, err = b.Step(-1) + return err +} + +// Restore restores dstDB on the dst connection from the "main" database in srcURI. +// +// Restore opens the SQLite database file srcURI, +// and blocks until the entire restore is complete. +// +// https://sqlite.org/backup.html +func (dst *Conn) Restore(dstDB, srcURI string) error { + src, err := dst.openDB(srcURI, OPEN_READONLY|OPEN_URI) + if err != nil { + return err + } + b, err := dst.backupInit(dst.handle, dstDB, src, "main") + if err != nil { + return err + } + defer b.Close() + _, err = b.Step(-1) + return err +} + +// BackupInit initializes a backup operation to copy the content of one database into another. +// +// BackupInit opens the SQLite database file dstURI, +// then initializes a backup that copies the contents of srcDB on the src connection +// to the "main" database in dstURI. +// +// https://sqlite.org/c3ref/backup_finish.html#sqlite3backupinit +func (src *Conn) BackupInit(srcDB, dstURI string) (*Backup, error) { + dst, err := src.openDB(dstURI, OPEN_READWRITE|OPEN_CREATE|OPEN_URI) + if err != nil { + return nil, err + } + return src.backupInit(dst, "main", src.handle, srcDB) +} + +func (c *Conn) backupInit(dst uint32, dstName string, src uint32, srcName string) (*Backup, error) { + defer c.arena.mark()() + dstPtr := c.arena.string(dstName) + srcPtr := c.arena.string(srcName) + + other := dst + if c.handle == dst { + other = src + } + + r := c.call("sqlite3_backup_init", + uint64(dst), uint64(dstPtr), + uint64(src), uint64(srcPtr)) + if r == 0 { + defer c.closeDB(other) + r = c.call("sqlite3_errcode", uint64(dst)) + return nil, c.sqlite.error(r, dst) + } + + return &Backup{ + c: c, + otherc: other, + handle: uint32(r), + }, nil +} + +// Close finishes a backup operation. +// +// It is safe to close a nil, zero or closed Backup. +// +// https://sqlite.org/c3ref/backup_finish.html#sqlite3backupfinish +func (b *Backup) Close() error { + if b == nil || b.handle == 0 { + return nil + } + + r := b.c.call("sqlite3_backup_finish", uint64(b.handle)) + b.c.closeDB(b.otherc) + b.handle = 0 + return b.c.error(r) +} + +// Step copies up to nPage pages between the source and destination databases. +// If nPage is negative, all remaining source pages are copied. +// +// https://sqlite.org/c3ref/backup_finish.html#sqlite3backupstep +func (b *Backup) Step(nPage int) (done bool, err error) { + r := b.c.call("sqlite3_backup_step", uint64(b.handle), uint64(nPage)) + if r == _DONE { + return true, nil + } + return false, b.c.error(r) +} + +// Remaining returns the number of pages still to be backed up +// at the conclusion of the most recent [Backup.Step]. +// +// https://sqlite.org/c3ref/backup_finish.html#sqlite3backupremaining +func (b *Backup) Remaining() int { + r := b.c.call("sqlite3_backup_remaining", uint64(b.handle)) + return int(int32(r)) +} + +// PageCount returns the total number of pages in the source database +// at the conclusion of the most recent [Backup.Step]. +// +// https://sqlite.org/c3ref/backup_finish.html#sqlite3backuppagecount +func (b *Backup) PageCount() int { + r := b.c.call("sqlite3_backup_pagecount", uint64(b.handle)) + return int(int32(r)) +} diff --git a/vendor/github.com/ncruces/go-sqlite3/blob.go b/vendor/github.com/ncruces/go-sqlite3/blob.go new file mode 100644 index 000000000..bb10c5fa2 --- /dev/null +++ b/vendor/github.com/ncruces/go-sqlite3/blob.go @@ -0,0 +1,250 @@ +package sqlite3 + +import ( + "io" + + "github.com/ncruces/go-sqlite3/internal/util" +) + +// ZeroBlob represents a zero-filled, length n BLOB +// that can be used as an argument to +// [database/sql.DB.Exec] and similar methods. +type ZeroBlob int64 + +// Blob is an handle to an open BLOB. +// +// It implements [io.ReadWriteSeeker] for incremental BLOB I/O. +// +// https://sqlite.org/c3ref/blob.html +type Blob struct { + c *Conn + bytes int64 + offset int64 + handle uint32 +} + +var _ io.ReadWriteSeeker = &Blob{} + +// OpenBlob opens a BLOB for incremental I/O. +// +// https://sqlite.org/c3ref/blob_open.html +func (c *Conn) OpenBlob(db, table, column string, row int64, write bool) (*Blob, error) { + c.checkInterrupt() + defer c.arena.mark()() + blobPtr := c.arena.new(ptrlen) + dbPtr := c.arena.string(db) + tablePtr := c.arena.string(table) + columnPtr := c.arena.string(column) + + var flags uint64 + if write { + flags = 1 + } + + r := c.call("sqlite3_blob_open", uint64(c.handle), + uint64(dbPtr), uint64(tablePtr), uint64(columnPtr), + uint64(row), flags, uint64(blobPtr)) + + if err := c.error(r); err != nil { + return nil, err + } + + blob := Blob{c: c} + blob.handle = util.ReadUint32(c.mod, blobPtr) + blob.bytes = int64(c.call("sqlite3_blob_bytes", uint64(blob.handle))) + return &blob, nil +} + +// Close closes a BLOB handle. +// +// It is safe to close a nil, zero or closed Blob. +// +// https://sqlite.org/c3ref/blob_close.html +func (b *Blob) Close() error { + if b == nil || b.handle == 0 { + return nil + } + + r := b.c.call("sqlite3_blob_close", uint64(b.handle)) + + b.handle = 0 + return b.c.error(r) +} + +// Size returns the size of the BLOB in bytes. +// +// https://sqlite.org/c3ref/blob_bytes.html +func (b *Blob) Size() int64 { + return b.bytes +} + +// Read implements the [io.Reader] interface. +// +// https://sqlite.org/c3ref/blob_read.html +func (b *Blob) Read(p []byte) (n int, err error) { + if b.offset >= b.bytes { + return 0, io.EOF + } + + avail := b.bytes - b.offset + want := int64(len(p)) + if want > avail { + want = avail + } + + defer b.c.arena.mark()() + ptr := b.c.arena.new(uint64(want)) + + r := b.c.call("sqlite3_blob_read", uint64(b.handle), + uint64(ptr), uint64(want), uint64(b.offset)) + err = b.c.error(r) + if err != nil { + return 0, err + } + b.offset += want + if b.offset >= b.bytes { + err = io.EOF + } + + copy(p, util.View(b.c.mod, ptr, uint64(want))) + return int(want), err +} + +// WriteTo implements the [io.WriterTo] interface. +// +// https://sqlite.org/c3ref/blob_read.html +func (b *Blob) WriteTo(w io.Writer) (n int64, err error) { + if b.offset >= b.bytes { + return 0, nil + } + + want := int64(1024 * 1024) + avail := b.bytes - b.offset + if want > avail { + want = avail + } + + defer b.c.arena.mark()() + ptr := b.c.arena.new(uint64(want)) + + for want > 0 { + r := b.c.call("sqlite3_blob_read", uint64(b.handle), + uint64(ptr), uint64(want), uint64(b.offset)) + err = b.c.error(r) + if err != nil { + return n, err + } + + mem := util.View(b.c.mod, ptr, uint64(want)) + m, err := w.Write(mem[:want]) + b.offset += int64(m) + n += int64(m) + if err != nil { + return n, err + } + if int64(m) != want { + return n, io.ErrShortWrite + } + + avail = b.bytes - b.offset + if want > avail { + want = avail + } + } + return n, nil +} + +// Write implements the [io.Writer] interface. +// +// https://sqlite.org/c3ref/blob_write.html +func (b *Blob) Write(p []byte) (n int, err error) { + defer b.c.arena.mark()() + ptr := b.c.arena.bytes(p) + + r := b.c.call("sqlite3_blob_write", uint64(b.handle), + uint64(ptr), uint64(len(p)), uint64(b.offset)) + err = b.c.error(r) + if err != nil { + return 0, err + } + b.offset += int64(len(p)) + return len(p), nil +} + +// ReadFrom implements the [io.ReaderFrom] interface. +// +// https://sqlite.org/c3ref/blob_write.html +func (b *Blob) ReadFrom(r io.Reader) (n int64, err error) { + want := int64(1024 * 1024) + avail := b.bytes - b.offset + if l, ok := r.(*io.LimitedReader); ok && want > l.N { + want = l.N + } + if want > avail { + want = avail + } + if want < 1 { + want = 1 + } + + defer b.c.arena.mark()() + ptr := b.c.arena.new(uint64(want)) + + for { + mem := util.View(b.c.mod, ptr, uint64(want)) + m, err := r.Read(mem[:want]) + if m > 0 { + r := b.c.call("sqlite3_blob_write", uint64(b.handle), + uint64(ptr), uint64(m), uint64(b.offset)) + err := b.c.error(r) + if err != nil { + return n, err + } + b.offset += int64(m) + n += int64(m) + } + if err == io.EOF { + return n, nil + } + if err != nil { + return n, err + } + + avail = b.bytes - b.offset + if want > avail { + want = avail + } + if want < 1 { + want = 1 + } + } +} + +// Seek implements the [io.Seeker] interface. +func (b *Blob) Seek(offset int64, whence int) (int64, error) { + switch whence { + default: + return 0, util.WhenceErr + case io.SeekStart: + break + case io.SeekCurrent: + offset += b.offset + case io.SeekEnd: + offset += b.bytes + } + if offset < 0 { + return 0, util.OffsetErr + } + b.offset = offset + return offset, nil +} + +// Reopen moves a BLOB handle to a new row of the same database table. +// +// https://sqlite.org/c3ref/blob_reopen.html +func (b *Blob) Reopen(row int64) error { + err := b.c.error(b.c.call("sqlite3_blob_reopen", uint64(b.handle), uint64(row))) + b.bytes = int64(b.c.call("sqlite3_blob_bytes", uint64(b.handle))) + b.offset = 0 + return err +} diff --git a/vendor/github.com/ncruces/go-sqlite3/config.go b/vendor/github.com/ncruces/go-sqlite3/config.go new file mode 100644 index 000000000..0342be7fb --- /dev/null +++ b/vendor/github.com/ncruces/go-sqlite3/config.go @@ -0,0 +1,164 @@ +package sqlite3 + +import ( + "context" + + "github.com/ncruces/go-sqlite3/internal/util" + "github.com/tetratelabs/wazero/api" +) + +// Config makes configuration changes to a database connection. +// Only boolean configuration options are supported. +// Called with no arg reads the current configuration value, +// called with one arg sets and returns the new value. +// +// https://sqlite.org/c3ref/db_config.html +func (c *Conn) Config(op DBConfig, arg ...bool) (bool, error) { + defer c.arena.mark()() + argsPtr := c.arena.new(2 * ptrlen) + + var flag int + switch { + case len(arg) == 0: + flag = -1 + case arg[0]: + flag = 1 + } + + util.WriteUint32(c.mod, argsPtr+0*ptrlen, uint32(flag)) + util.WriteUint32(c.mod, argsPtr+1*ptrlen, argsPtr) + + r := c.call("sqlite3_db_config", uint64(c.handle), + uint64(op), uint64(argsPtr)) + return util.ReadUint32(c.mod, argsPtr) != 0, c.error(r) +} + +// ConfigLog sets up the error logging callback for the connection. +// +// https://sqlite.org/errlog.html +func (c *Conn) ConfigLog(cb func(code ExtendedErrorCode, msg string)) error { + var enable uint64 + if cb != nil { + enable = 1 + } + r := c.call("sqlite3_config_log_go", enable) + if err := c.error(r); err != nil { + return err + } + c.log = cb + return nil +} + +func logCallback(ctx context.Context, mod api.Module, _, iCode, zMsg uint32) { + if c, ok := ctx.Value(connKey{}).(*Conn); ok && c.log != nil { + msg := util.ReadString(mod, zMsg, _MAX_LENGTH) + c.log(xErrorCode(iCode), msg) + } +} + +// Limit allows the size of various constructs to be +// limited on a connection by connection basis. +// +// https://sqlite.org/c3ref/limit.html +func (c *Conn) Limit(id LimitCategory, value int) int { + r := c.call("sqlite3_limit", uint64(c.handle), uint64(id), uint64(value)) + return int(int32(r)) +} + +// SetAuthorizer registers an authorizer callback with the database connection. +// +// https://sqlite.org/c3ref/set_authorizer.html +func (c *Conn) SetAuthorizer(cb func(action AuthorizerActionCode, name3rd, name4th, schema, nameInner string) AuthorizerReturnCode) error { + var enable uint64 + if cb != nil { + enable = 1 + } + r := c.call("sqlite3_set_authorizer_go", uint64(c.handle), enable) + if err := c.error(r); err != nil { + return err + } + c.authorizer = cb + return nil + +} + +func authorizerCallback(ctx context.Context, mod api.Module, pDB uint32, action AuthorizerActionCode, zName3rd, zName4th, zSchema, zNameInner uint32) (rc AuthorizerReturnCode) { + if c, ok := ctx.Value(connKey{}).(*Conn); ok && c.handle == pDB && c.authorizer != nil { + var name3rd, name4th, schema, nameInner string + if zName3rd != 0 { + name3rd = util.ReadString(mod, zName3rd, _MAX_NAME) + } + if zName4th != 0 { + name4th = util.ReadString(mod, zName4th, _MAX_NAME) + } + if zSchema != 0 { + schema = util.ReadString(mod, zSchema, _MAX_NAME) + } + if zNameInner != 0 { + nameInner = util.ReadString(mod, zNameInner, _MAX_NAME) + } + rc = c.authorizer(action, name3rd, name4th, schema, nameInner) + } + return rc +} + +// WalCheckpoint checkpoints a WAL database. +// +// https://sqlite.org/c3ref/wal_checkpoint_v2.html +func (c *Conn) WalCheckpoint(schema string, mode CheckpointMode) (nLog, nCkpt int, err error) { + defer c.arena.mark()() + nLogPtr := c.arena.new(ptrlen) + nCkptPtr := c.arena.new(ptrlen) + schemaPtr := c.arena.string(schema) + r := c.call("sqlite3_wal_checkpoint_v2", + uint64(c.handle), uint64(schemaPtr), uint64(mode), + uint64(nLogPtr), uint64(nCkptPtr)) + nLog = int(int32(util.ReadUint32(c.mod, nLogPtr))) + nCkpt = int(int32(util.ReadUint32(c.mod, nCkptPtr))) + return nLog, nCkpt, c.error(r) +} + +// WalAutoCheckpoint configures WAL auto-checkpoints. +// +// https://sqlite.org/c3ref/wal_autocheckpoint.html +func (c *Conn) WalAutoCheckpoint(pages int) error { + r := c.call("sqlite3_wal_autocheckpoint", uint64(c.handle), uint64(pages)) + return c.error(r) +} + +// WalHook registers a callback function to be invoked +// each time data is committed to a database in WAL mode. +// +// https://sqlite.org/c3ref/wal_hook.html +func (c *Conn) WalHook(cb func(db *Conn, schema string, pages int) error) { + var enable uint64 + if cb != nil { + enable = 1 + } + c.call("sqlite3_wal_hook_go", uint64(c.handle), enable) + c.wal = cb +} + +func walCallback(ctx context.Context, mod api.Module, _, pDB, zSchema uint32, pages int32) (rc uint32) { + if c, ok := ctx.Value(connKey{}).(*Conn); ok && c.handle == pDB && c.wal != nil { + schema := util.ReadString(mod, zSchema, _MAX_NAME) + err := c.wal(c, schema, int(pages)) + _, rc = errorCode(err, ERROR) + } + return rc +} + +// AutoVacuumPages registers a autovacuum compaction amount callback. +// +// https://sqlite.org/c3ref/autovacuum_pages.html +func (c *Conn) AutoVacuumPages(cb func(schema string, dbPages, freePages, bytesPerPage uint) uint) error { + funcPtr := util.AddHandle(c.ctx, cb) + r := c.call("sqlite3_autovacuum_pages_go", uint64(c.handle), uint64(funcPtr)) + return c.error(r) +} + +func autoVacuumCallback(ctx context.Context, mod api.Module, pApp, zSchema, nDbPage, nFreePage, nBytePerPage uint32) uint32 { + fn := util.GetHandle(ctx, pApp).(func(schema string, dbPages, freePages, bytesPerPage uint) uint) + schema := util.ReadString(mod, zSchema, _MAX_NAME) + return uint32(fn(schema, uint(nDbPage), uint(nFreePage), uint(nBytePerPage))) +} diff --git a/vendor/github.com/ncruces/go-sqlite3/conn.go b/vendor/github.com/ncruces/go-sqlite3/conn.go new file mode 100644 index 000000000..f170ccf57 --- /dev/null +++ b/vendor/github.com/ncruces/go-sqlite3/conn.go @@ -0,0 +1,426 @@ +package sqlite3 + +import ( + "context" + "fmt" + "math" + "net/url" + "strings" + "time" + + "github.com/ncruces/go-sqlite3/internal/util" + "github.com/ncruces/go-sqlite3/vfs" + "github.com/tetratelabs/wazero/api" +) + +// Conn is a database connection handle. +// A Conn is not safe for concurrent use by multiple goroutines. +// +// https://sqlite.org/c3ref/sqlite3.html +type Conn struct { + *sqlite + + interrupt context.Context + pending *Stmt + busy func(int) bool + log func(xErrorCode, string) + collation func(*Conn, string) + authorizer func(AuthorizerActionCode, string, string, string, string) AuthorizerReturnCode + update func(AuthorizerActionCode, string, string, int64) + commit func() bool + rollback func() + wal func(*Conn, string, int) error + arena arena + + handle uint32 +} + +// Open calls [OpenFlags] with [OPEN_READWRITE], [OPEN_CREATE], [OPEN_URI] and [OPEN_NOFOLLOW]. +func Open(filename string) (*Conn, error) { + return newConn(filename, OPEN_READWRITE|OPEN_CREATE|OPEN_URI|OPEN_NOFOLLOW) +} + +// OpenFlags opens an SQLite database file as specified by the filename argument. +// +// If none of the required flags is used, a combination of [OPEN_READWRITE] and [OPEN_CREATE] is used. +// If a URI filename is used, PRAGMA statements to execute can be specified using "_pragma": +// +// sqlite3.Open("file:demo.db?_pragma=busy_timeout(10000)") +// +// https://sqlite.org/c3ref/open.html +func OpenFlags(filename string, flags OpenFlag) (*Conn, error) { + if flags&(OPEN_READONLY|OPEN_READWRITE|OPEN_CREATE) == 0 { + flags |= OPEN_READWRITE | OPEN_CREATE + } + return newConn(filename, flags) +} + +type connKey struct{} + +func newConn(filename string, flags OpenFlag) (conn *Conn, err error) { + sqlite, err := instantiateSQLite() + if err != nil { + return nil, err + } + defer func() { + if conn == nil { + sqlite.close() + } + }() + + c := &Conn{sqlite: sqlite} + c.arena = c.newArena(1024) + c.ctx = context.WithValue(c.ctx, connKey{}, c) + c.handle, err = c.openDB(filename, flags) + if err != nil { + return nil, err + } + return c, nil +} + +func (c *Conn) openDB(filename string, flags OpenFlag) (uint32, error) { + defer c.arena.mark()() + connPtr := c.arena.new(ptrlen) + namePtr := c.arena.string(filename) + + flags |= OPEN_EXRESCODE + r := c.call("sqlite3_open_v2", uint64(namePtr), uint64(connPtr), uint64(flags), 0) + + handle := util.ReadUint32(c.mod, connPtr) + if err := c.sqlite.error(r, handle); err != nil { + c.closeDB(handle) + return 0, err + } + + if flags|OPEN_URI != 0 && strings.HasPrefix(filename, "file:") { + var pragmas strings.Builder + if _, after, ok := strings.Cut(filename, "?"); ok { + query, _ := url.ParseQuery(after) + for _, p := range query["_pragma"] { + pragmas.WriteString(`PRAGMA `) + pragmas.WriteString(p) + pragmas.WriteString(`;`) + } + } + if pragmas.Len() != 0 { + pragmaPtr := c.arena.string(pragmas.String()) + r := c.call("sqlite3_exec", uint64(handle), uint64(pragmaPtr), 0, 0, 0) + if err := c.sqlite.error(r, handle, pragmas.String()); err != nil { + err = fmt.Errorf("sqlite3: invalid _pragma: %w", err) + c.closeDB(handle) + return 0, err + } + } + } + c.call("sqlite3_progress_handler_go", uint64(handle), 100) + return handle, nil +} + +func (c *Conn) closeDB(handle uint32) { + r := c.call("sqlite3_close_v2", uint64(handle)) + if err := c.sqlite.error(r, handle); err != nil { + panic(err) + } +} + +// Close closes the database connection. +// +// If the database connection is associated with unfinalized prepared statements, +// open blob handles, and/or unfinished backup objects, +// Close will leave the database connection open and return [BUSY]. +// +// It is safe to close a nil, zero or closed Conn. +// +// https://sqlite.org/c3ref/close.html +func (c *Conn) Close() error { + if c == nil || c.handle == 0 { + return nil + } + + c.pending.Close() + c.pending = nil + + r := c.call("sqlite3_close", uint64(c.handle)) + if err := c.error(r); err != nil { + return err + } + + c.handle = 0 + return c.close() +} + +// Exec is a convenience function that allows an application to run +// multiple statements of SQL without having to use a lot of code. +// +// https://sqlite.org/c3ref/exec.html +func (c *Conn) Exec(sql string) error { + c.checkInterrupt() + defer c.arena.mark()() + sqlPtr := c.arena.string(sql) + + r := c.call("sqlite3_exec", uint64(c.handle), uint64(sqlPtr), 0, 0, 0) + return c.error(r, sql) +} + +// Prepare calls [Conn.PrepareFlags] with no flags. +func (c *Conn) Prepare(sql string) (stmt *Stmt, tail string, err error) { + return c.PrepareFlags(sql, 0) +} + +// PrepareFlags compiles the first SQL statement in sql; +// tail is left pointing to what remains uncompiled. +// If the input text contains no SQL (if the input is an empty string or a comment), +// both stmt and err will be nil. +// +// https://sqlite.org/c3ref/prepare.html +func (c *Conn) PrepareFlags(sql string, flags PrepareFlag) (stmt *Stmt, tail string, err error) { + if len(sql) > _MAX_SQL_LENGTH { + return nil, "", TOOBIG + } + + defer c.arena.mark()() + stmtPtr := c.arena.new(ptrlen) + tailPtr := c.arena.new(ptrlen) + sqlPtr := c.arena.string(sql) + + r := c.call("sqlite3_prepare_v3", uint64(c.handle), + uint64(sqlPtr), uint64(len(sql)+1), uint64(flags), + uint64(stmtPtr), uint64(tailPtr)) + + stmt = &Stmt{c: c} + stmt.handle = util.ReadUint32(c.mod, stmtPtr) + if sql := sql[util.ReadUint32(c.mod, tailPtr)-sqlPtr:]; sql != "" { + tail = sql + } + + if err := c.error(r, sql); err != nil { + return nil, "", err + } + if stmt.handle == 0 { + return nil, "", nil + } + return stmt, tail, nil +} + +// DBName returns the schema name for n-th database on the database connection. +// +// https://sqlite.org/c3ref/db_name.html +func (c *Conn) DBName(n int) string { + r := c.call("sqlite3_db_name", uint64(c.handle), uint64(n)) + + ptr := uint32(r) + if ptr == 0 { + return "" + } + return util.ReadString(c.mod, ptr, _MAX_NAME) +} + +// Filename returns the filename for a database. +// +// https://sqlite.org/c3ref/db_filename.html +func (c *Conn) Filename(schema string) *vfs.Filename { + var ptr uint32 + if schema != "" { + defer c.arena.mark()() + ptr = c.arena.string(schema) + } + + r := c.call("sqlite3_db_filename", uint64(c.handle), uint64(ptr)) + return vfs.OpenFilename(c.ctx, c.mod, uint32(r), vfs.OPEN_MAIN_DB) +} + +// ReadOnly determines if a database is read-only. +// +// https://sqlite.org/c3ref/db_readonly.html +func (c *Conn) ReadOnly(schema string) (ro bool, ok bool) { + var ptr uint32 + if schema != "" { + defer c.arena.mark()() + ptr = c.arena.string(schema) + } + r := c.call("sqlite3_db_readonly", uint64(c.handle), uint64(ptr)) + return int32(r) > 0, int32(r) < 0 +} + +// GetAutocommit tests the connection for auto-commit mode. +// +// https://sqlite.org/c3ref/get_autocommit.html +func (c *Conn) GetAutocommit() bool { + r := c.call("sqlite3_get_autocommit", uint64(c.handle)) + return r != 0 +} + +// LastInsertRowID returns the rowid of the most recent successful INSERT +// on the database connection. +// +// https://sqlite.org/c3ref/last_insert_rowid.html +func (c *Conn) LastInsertRowID() int64 { + r := c.call("sqlite3_last_insert_rowid", uint64(c.handle)) + return int64(r) +} + +// SetLastInsertRowID allows the application to set the value returned by +// [Conn.LastInsertRowID]. +// +// https://sqlite.org/c3ref/set_last_insert_rowid.html +func (c *Conn) SetLastInsertRowID(id int64) { + c.call("sqlite3_set_last_insert_rowid", uint64(c.handle), uint64(id)) +} + +// Changes returns the number of rows modified, inserted or deleted +// by the most recently completed INSERT, UPDATE or DELETE statement +// on the database connection. +// +// https://sqlite.org/c3ref/changes.html +func (c *Conn) Changes() int64 { + r := c.call("sqlite3_changes64", uint64(c.handle)) + return int64(r) +} + +// TotalChanges returns the number of rows modified, inserted or deleted +// by all INSERT, UPDATE or DELETE statements completed +// since the database connection was opened. +// +// https://sqlite.org/c3ref/total_changes.html +func (c *Conn) TotalChanges() int64 { + r := c.call("sqlite3_total_changes64", uint64(c.handle)) + return int64(r) +} + +// ReleaseMemory frees memory used by a database connection. +// +// https://sqlite.org/c3ref/db_release_memory.html +func (c *Conn) ReleaseMemory() error { + r := c.call("sqlite3_db_release_memory", uint64(c.handle)) + return c.error(r) +} + +// GetInterrupt gets the context set with [Conn.SetInterrupt], +// or nil if none was set. +func (c *Conn) GetInterrupt() context.Context { + return c.interrupt +} + +// SetInterrupt interrupts a long-running query when a context is done. +// +// Subsequent uses of the connection will return [INTERRUPT] +// until the context is reset by another call to SetInterrupt. +// +// To associate a timeout with a connection: +// +// ctx, cancel := context.WithTimeout(context.TODO(), 100*time.Millisecond) +// conn.SetInterrupt(ctx) +// defer cancel() +// +// SetInterrupt returns the old context assigned to the connection. +// +// https://sqlite.org/c3ref/interrupt.html +func (c *Conn) SetInterrupt(ctx context.Context) (old context.Context) { + // Is it the same context? + if ctx == c.interrupt { + return ctx + } + + // A busy SQL statement prevents SQLite from ignoring an interrupt + // that comes before any other statements are started. + if c.pending == nil { + c.pending, _, _ = c.Prepare(`WITH RECURSIVE c(x) AS (VALUES(0) UNION ALL SELECT x FROM c) SELECT x FROM c`) + } + + old = c.interrupt + c.interrupt = ctx + + if old != nil && old.Done() != nil && (ctx == nil || ctx.Err() == nil) { + c.pending.Reset() + } + if ctx != nil && ctx.Done() != nil { + c.pending.Step() + } + return old +} + +func (c *Conn) checkInterrupt() { + if c.interrupt != nil && c.interrupt.Err() != nil { + c.call("sqlite3_interrupt", uint64(c.handle)) + } +} + +func progressCallback(ctx context.Context, mod api.Module, pDB uint32) (interrupt uint32) { + if c, ok := ctx.Value(connKey{}).(*Conn); ok && c.handle == pDB && + c.interrupt != nil && c.interrupt.Err() != nil { + interrupt = 1 + } + return interrupt +} + +// BusyTimeout sets a busy timeout. +// +// https://sqlite.org/c3ref/busy_timeout.html +func (c *Conn) BusyTimeout(timeout time.Duration) error { + ms := min((timeout+time.Millisecond-1)/time.Millisecond, math.MaxInt32) + r := c.call("sqlite3_busy_timeout", uint64(c.handle), uint64(ms)) + return c.error(r) +} + +func timeoutCallback(ctx context.Context, mod api.Module, pDB uint32, count, tmout int32) (retry uint32) { + if c, ok := ctx.Value(connKey{}).(*Conn); ok && + (c.interrupt == nil || c.interrupt.Err() == nil) { + const delays = "\x01\x02\x05\x0a\x0f\x14\x19\x19\x19\x32\x32\x64" + const totals = "\x00\x01\x03\x08\x12\x21\x35\x4e\x67\x80\xb2\xe4" + const ndelay = int32(len(delays) - 1) + + var delay, prior int32 + if count <= ndelay { + delay = int32(delays[count]) + prior = int32(totals[count]) + } else { + delay = int32(delays[ndelay]) + prior = int32(totals[ndelay]) + delay*(count-ndelay) + } + + if delay = min(delay, tmout-prior); delay > 0 { + time.Sleep(time.Duration(delay) * time.Millisecond) + retry = 1 + } + } + return retry +} + +// BusyHandler registers a callback to handle [BUSY] errors. +// +// https://sqlite.org/c3ref/busy_handler.html +func (c *Conn) BusyHandler(cb func(count int) (retry bool)) error { + var enable uint64 + if cb != nil { + enable = 1 + } + r := c.call("sqlite3_busy_handler_go", uint64(c.handle), enable) + if err := c.error(r); err != nil { + return err + } + c.busy = cb + return nil +} + +func busyCallback(ctx context.Context, mod api.Module, pDB uint32, count int32) (retry uint32) { + if c, ok := ctx.Value(connKey{}).(*Conn); ok && c.handle == pDB && c.busy != nil && + (c.interrupt == nil || c.interrupt.Err() == nil) { + if c.busy(int(count)) { + retry = 1 + } + } + return retry +} + +func (c *Conn) error(rc uint64, sql ...string) error { + return c.sqlite.error(rc, c.handle, sql...) +} + +// DriverConn is implemented by the SQLite [database/sql] driver connection. +// +// It can be used to access SQLite features like [online backup]. +// +// [online backup]: https://sqlite.org/backup.html +type DriverConn interface { + Raw() *Conn +} diff --git a/vendor/github.com/ncruces/go-sqlite3/const.go b/vendor/github.com/ncruces/go-sqlite3/const.go new file mode 100644 index 000000000..2bb53656f --- /dev/null +++ b/vendor/github.com/ncruces/go-sqlite3/const.go @@ -0,0 +1,360 @@ +package sqlite3 + +import "strconv" + +const ( + _OK = 0 /* Successful result */ + _ROW = 100 /* sqlite3_step() has another row ready */ + _DONE = 101 /* sqlite3_step() has finished executing */ + + _UTF8 = 1 + + _MAX_NAME = 1e6 // Self-imposed limit for most NUL terminated strings. + _MAX_LENGTH = 1e9 + _MAX_SQL_LENGTH = 1e9 + _MAX_ALLOCATION_SIZE = 0x7ffffeff + _MAX_FUNCTION_ARG = 100 + + ptrlen = 4 +) + +// ErrorCode is a result code that [Error.Code] might return. +// +// https://sqlite.org/rescode.html +type ErrorCode uint8 + +const ( + ERROR ErrorCode = 1 /* Generic error */ + INTERNAL ErrorCode = 2 /* Internal logic error in SQLite */ + PERM ErrorCode = 3 /* Access permission denied */ + ABORT ErrorCode = 4 /* Callback routine requested an abort */ + BUSY ErrorCode = 5 /* The database file is locked */ + LOCKED ErrorCode = 6 /* A table in the database is locked */ + NOMEM ErrorCode = 7 /* A malloc() failed */ + READONLY ErrorCode = 8 /* Attempt to write a readonly database */ + INTERRUPT ErrorCode = 9 /* Operation terminated by sqlite3_interrupt() */ + IOERR ErrorCode = 10 /* Some kind of disk I/O error occurred */ + CORRUPT ErrorCode = 11 /* The database disk image is malformed */ + NOTFOUND ErrorCode = 12 /* Unknown opcode in sqlite3_file_control() */ + FULL ErrorCode = 13 /* Insertion failed because database is full */ + CANTOPEN ErrorCode = 14 /* Unable to open the database file */ + PROTOCOL ErrorCode = 15 /* Database lock protocol error */ + EMPTY ErrorCode = 16 /* Internal use only */ + SCHEMA ErrorCode = 17 /* The database schema changed */ + TOOBIG ErrorCode = 18 /* String or BLOB exceeds size limit */ + CONSTRAINT ErrorCode = 19 /* Abort due to constraint violation */ + MISMATCH ErrorCode = 20 /* Data type mismatch */ + MISUSE ErrorCode = 21 /* Library used incorrectly */ + NOLFS ErrorCode = 22 /* Uses OS features not supported on host */ + AUTH ErrorCode = 23 /* Authorization denied */ + FORMAT ErrorCode = 24 /* Not used */ + RANGE ErrorCode = 25 /* 2nd parameter to sqlite3_bind out of range */ + NOTADB ErrorCode = 26 /* File opened that is not a database file */ + NOTICE ErrorCode = 27 /* Notifications from sqlite3_log() */ + WARNING ErrorCode = 28 /* Warnings from sqlite3_log() */ +) + +// ExtendedErrorCode is a result code that [Error.ExtendedCode] might return. +// +// https://sqlite.org/rescode.html +type ( + ExtendedErrorCode uint16 + xErrorCode = ExtendedErrorCode +) + +const ( + ERROR_MISSING_COLLSEQ ExtendedErrorCode = xErrorCode(ERROR) | (1 << 8) + ERROR_RETRY ExtendedErrorCode = xErrorCode(ERROR) | (2 << 8) + ERROR_SNAPSHOT ExtendedErrorCode = xErrorCode(ERROR) | (3 << 8) + IOERR_READ ExtendedErrorCode = xErrorCode(IOERR) | (1 << 8) + IOERR_SHORT_READ ExtendedErrorCode = xErrorCode(IOERR) | (2 << 8) + IOERR_WRITE ExtendedErrorCode = xErrorCode(IOERR) | (3 << 8) + IOERR_FSYNC ExtendedErrorCode = xErrorCode(IOERR) | (4 << 8) + IOERR_DIR_FSYNC ExtendedErrorCode = xErrorCode(IOERR) | (5 << 8) + IOERR_TRUNCATE ExtendedErrorCode = xErrorCode(IOERR) | (6 << 8) + IOERR_FSTAT ExtendedErrorCode = xErrorCode(IOERR) | (7 << 8) + IOERR_UNLOCK ExtendedErrorCode = xErrorCode(IOERR) | (8 << 8) + IOERR_RDLOCK ExtendedErrorCode = xErrorCode(IOERR) | (9 << 8) + IOERR_DELETE ExtendedErrorCode = xErrorCode(IOERR) | (10 << 8) + IOERR_BLOCKED ExtendedErrorCode = xErrorCode(IOERR) | (11 << 8) + IOERR_NOMEM ExtendedErrorCode = xErrorCode(IOERR) | (12 << 8) + IOERR_ACCESS ExtendedErrorCode = xErrorCode(IOERR) | (13 << 8) + IOERR_CHECKRESERVEDLOCK ExtendedErrorCode = xErrorCode(IOERR) | (14 << 8) + IOERR_LOCK ExtendedErrorCode = xErrorCode(IOERR) | (15 << 8) + IOERR_CLOSE ExtendedErrorCode = xErrorCode(IOERR) | (16 << 8) + IOERR_DIR_CLOSE ExtendedErrorCode = xErrorCode(IOERR) | (17 << 8) + IOERR_SHMOPEN ExtendedErrorCode = xErrorCode(IOERR) | (18 << 8) + IOERR_SHMSIZE ExtendedErrorCode = xErrorCode(IOERR) | (19 << 8) + IOERR_SHMLOCK ExtendedErrorCode = xErrorCode(IOERR) | (20 << 8) + IOERR_SHMMAP ExtendedErrorCode = xErrorCode(IOERR) | (21 << 8) + IOERR_SEEK ExtendedErrorCode = xErrorCode(IOERR) | (22 << 8) + IOERR_DELETE_NOENT ExtendedErrorCode = xErrorCode(IOERR) | (23 << 8) + IOERR_MMAP ExtendedErrorCode = xErrorCode(IOERR) | (24 << 8) + IOERR_GETTEMPPATH ExtendedErrorCode = xErrorCode(IOERR) | (25 << 8) + IOERR_CONVPATH ExtendedErrorCode = xErrorCode(IOERR) | (26 << 8) + IOERR_VNODE ExtendedErrorCode = xErrorCode(IOERR) | (27 << 8) + IOERR_AUTH ExtendedErrorCode = xErrorCode(IOERR) | (28 << 8) + IOERR_BEGIN_ATOMIC ExtendedErrorCode = xErrorCode(IOERR) | (29 << 8) + IOERR_COMMIT_ATOMIC ExtendedErrorCode = xErrorCode(IOERR) | (30 << 8) + IOERR_ROLLBACK_ATOMIC ExtendedErrorCode = xErrorCode(IOERR) | (31 << 8) + IOERR_DATA ExtendedErrorCode = xErrorCode(IOERR) | (32 << 8) + IOERR_CORRUPTFS ExtendedErrorCode = xErrorCode(IOERR) | (33 << 8) + IOERR_IN_PAGE ExtendedErrorCode = xErrorCode(IOERR) | (34 << 8) + LOCKED_SHAREDCACHE ExtendedErrorCode = xErrorCode(LOCKED) | (1 << 8) + LOCKED_VTAB ExtendedErrorCode = xErrorCode(LOCKED) | (2 << 8) + BUSY_RECOVERY ExtendedErrorCode = xErrorCode(BUSY) | (1 << 8) + BUSY_SNAPSHOT ExtendedErrorCode = xErrorCode(BUSY) | (2 << 8) + BUSY_TIMEOUT ExtendedErrorCode = xErrorCode(BUSY) | (3 << 8) + CANTOPEN_NOTEMPDIR ExtendedErrorCode = xErrorCode(CANTOPEN) | (1 << 8) + CANTOPEN_ISDIR ExtendedErrorCode = xErrorCode(CANTOPEN) | (2 << 8) + CANTOPEN_FULLPATH ExtendedErrorCode = xErrorCode(CANTOPEN) | (3 << 8) + CANTOPEN_CONVPATH ExtendedErrorCode = xErrorCode(CANTOPEN) | (4 << 8) + CANTOPEN_DIRTYWAL ExtendedErrorCode = xErrorCode(CANTOPEN) | (5 << 8) /* Not Used */ + CANTOPEN_SYMLINK ExtendedErrorCode = xErrorCode(CANTOPEN) | (6 << 8) + CORRUPT_VTAB ExtendedErrorCode = xErrorCode(CORRUPT) | (1 << 8) + CORRUPT_SEQUENCE ExtendedErrorCode = xErrorCode(CORRUPT) | (2 << 8) + CORRUPT_INDEX ExtendedErrorCode = xErrorCode(CORRUPT) | (3 << 8) + READONLY_RECOVERY ExtendedErrorCode = xErrorCode(READONLY) | (1 << 8) + READONLY_CANTLOCK ExtendedErrorCode = xErrorCode(READONLY) | (2 << 8) + READONLY_ROLLBACK ExtendedErrorCode = xErrorCode(READONLY) | (3 << 8) + READONLY_DBMOVED ExtendedErrorCode = xErrorCode(READONLY) | (4 << 8) + READONLY_CANTINIT ExtendedErrorCode = xErrorCode(READONLY) | (5 << 8) + READONLY_DIRECTORY ExtendedErrorCode = xErrorCode(READONLY) | (6 << 8) + ABORT_ROLLBACK ExtendedErrorCode = xErrorCode(ABORT) | (2 << 8) + CONSTRAINT_CHECK ExtendedErrorCode = xErrorCode(CONSTRAINT) | (1 << 8) + CONSTRAINT_COMMITHOOK ExtendedErrorCode = xErrorCode(CONSTRAINT) | (2 << 8) + CONSTRAINT_FOREIGNKEY ExtendedErrorCode = xErrorCode(CONSTRAINT) | (3 << 8) + CONSTRAINT_FUNCTION ExtendedErrorCode = xErrorCode(CONSTRAINT) | (4 << 8) + CONSTRAINT_NOTNULL ExtendedErrorCode = xErrorCode(CONSTRAINT) | (5 << 8) + CONSTRAINT_PRIMARYKEY ExtendedErrorCode = xErrorCode(CONSTRAINT) | (6 << 8) + CONSTRAINT_TRIGGER ExtendedErrorCode = xErrorCode(CONSTRAINT) | (7 << 8) + CONSTRAINT_UNIQUE ExtendedErrorCode = xErrorCode(CONSTRAINT) | (8 << 8) + CONSTRAINT_VTAB ExtendedErrorCode = xErrorCode(CONSTRAINT) | (9 << 8) + CONSTRAINT_ROWID ExtendedErrorCode = xErrorCode(CONSTRAINT) | (10 << 8) + CONSTRAINT_PINNED ExtendedErrorCode = xErrorCode(CONSTRAINT) | (11 << 8) + CONSTRAINT_DATATYPE ExtendedErrorCode = xErrorCode(CONSTRAINT) | (12 << 8) + NOTICE_RECOVER_WAL ExtendedErrorCode = xErrorCode(NOTICE) | (1 << 8) + NOTICE_RECOVER_ROLLBACK ExtendedErrorCode = xErrorCode(NOTICE) | (2 << 8) + NOTICE_RBU ExtendedErrorCode = xErrorCode(NOTICE) | (3 << 8) + WARNING_AUTOINDEX ExtendedErrorCode = xErrorCode(WARNING) | (1 << 8) + AUTH_USER ExtendedErrorCode = xErrorCode(AUTH) | (1 << 8) +) + +// OpenFlag is a flag for the [OpenFlags] function. +// +// https://sqlite.org/c3ref/c_open_autoproxy.html +type OpenFlag uint32 + +const ( + OPEN_READONLY OpenFlag = 0x00000001 /* Ok for sqlite3_open_v2() */ + OPEN_READWRITE OpenFlag = 0x00000002 /* Ok for sqlite3_open_v2() */ + OPEN_CREATE OpenFlag = 0x00000004 /* Ok for sqlite3_open_v2() */ + OPEN_URI OpenFlag = 0x00000040 /* Ok for sqlite3_open_v2() */ + OPEN_MEMORY OpenFlag = 0x00000080 /* Ok for sqlite3_open_v2() */ + OPEN_NOMUTEX OpenFlag = 0x00008000 /* Ok for sqlite3_open_v2() */ + OPEN_FULLMUTEX OpenFlag = 0x00010000 /* Ok for sqlite3_open_v2() */ + OPEN_SHAREDCACHE OpenFlag = 0x00020000 /* Ok for sqlite3_open_v2() */ + OPEN_PRIVATECACHE OpenFlag = 0x00040000 /* Ok for sqlite3_open_v2() */ + OPEN_NOFOLLOW OpenFlag = 0x01000000 /* Ok for sqlite3_open_v2() */ + OPEN_EXRESCODE OpenFlag = 0x02000000 /* Extended result codes */ +) + +// PrepareFlag is a flag that can be passed to [Conn.PrepareFlags]. +// +// https://sqlite.org/c3ref/c_prepare_normalize.html +type PrepareFlag uint32 + +const ( + PREPARE_PERSISTENT PrepareFlag = 0x01 + PREPARE_NORMALIZE PrepareFlag = 0x02 + PREPARE_NO_VTAB PrepareFlag = 0x04 +) + +// FunctionFlag is a flag that can be passed to +// [Conn.CreateFunction] and [Conn.CreateWindowFunction]. +// +// https://sqlite.org/c3ref/c_deterministic.html +type FunctionFlag uint32 + +const ( + DETERMINISTIC FunctionFlag = 0x000000800 + DIRECTONLY FunctionFlag = 0x000080000 + SUBTYPE FunctionFlag = 0x000100000 + INNOCUOUS FunctionFlag = 0x000200000 + RESULT_SUBTYPE FunctionFlag = 0x001000000 +) + +// StmtStatus name counter values associated with the [Stmt.Status] method. +// +// https://sqlite.org/c3ref/c_stmtstatus_counter.html +type StmtStatus uint32 + +const ( + STMTSTATUS_FULLSCAN_STEP StmtStatus = 1 + STMTSTATUS_SORT StmtStatus = 2 + STMTSTATUS_AUTOINDEX StmtStatus = 3 + STMTSTATUS_VM_STEP StmtStatus = 4 + STMTSTATUS_REPREPARE StmtStatus = 5 + STMTSTATUS_RUN StmtStatus = 6 + STMTSTATUS_FILTER_MISS StmtStatus = 7 + STMTSTATUS_FILTER_HIT StmtStatus = 8 + STMTSTATUS_MEMUSED StmtStatus = 99 +) + +// DBConfig are the available database connection configuration options. +// +// https://sqlite.org/c3ref/c_dbconfig_defensive.html +type DBConfig uint32 + +const ( + // DBCONFIG_MAINDBNAME DBConfig = 1000 + // DBCONFIG_LOOKASIDE DBConfig = 1001 + DBCONFIG_ENABLE_FKEY DBConfig = 1002 + DBCONFIG_ENABLE_TRIGGER DBConfig = 1003 + DBCONFIG_ENABLE_FTS3_TOKENIZER DBConfig = 1004 + DBCONFIG_ENABLE_LOAD_EXTENSION DBConfig = 1005 + DBCONFIG_NO_CKPT_ON_CLOSE DBConfig = 1006 + DBCONFIG_ENABLE_QPSG DBConfig = 1007 + DBCONFIG_TRIGGER_EQP DBConfig = 1008 + DBCONFIG_RESET_DATABASE DBConfig = 1009 + DBCONFIG_DEFENSIVE DBConfig = 1010 + DBCONFIG_WRITABLE_SCHEMA DBConfig = 1011 + DBCONFIG_LEGACY_ALTER_TABLE DBConfig = 1012 + DBCONFIG_DQS_DML DBConfig = 1013 + DBCONFIG_DQS_DDL DBConfig = 1014 + DBCONFIG_ENABLE_VIEW DBConfig = 1015 + DBCONFIG_LEGACY_FILE_FORMAT DBConfig = 1016 + DBCONFIG_TRUSTED_SCHEMA DBConfig = 1017 + DBCONFIG_STMT_SCANSTATUS DBConfig = 1018 + DBCONFIG_REVERSE_SCANORDER DBConfig = 1019 +) + +// LimitCategory are the available run-time limit categories. +// +// https://sqlite.org/c3ref/c_limit_attached.html +type LimitCategory uint32 + +const ( + LIMIT_LENGTH LimitCategory = 0 + LIMIT_SQL_LENGTH LimitCategory = 1 + LIMIT_COLUMN LimitCategory = 2 + LIMIT_EXPR_DEPTH LimitCategory = 3 + LIMIT_COMPOUND_SELECT LimitCategory = 4 + LIMIT_VDBE_OP LimitCategory = 5 + LIMIT_FUNCTION_ARG LimitCategory = 6 + LIMIT_ATTACHED LimitCategory = 7 + LIMIT_LIKE_PATTERN_LENGTH LimitCategory = 8 + LIMIT_VARIABLE_NUMBER LimitCategory = 9 + LIMIT_TRIGGER_DEPTH LimitCategory = 10 + LIMIT_WORKER_THREADS LimitCategory = 11 +) + +// AuthorizerActionCode are the integer action codes +// that the authorizer callback may be passed. +// +// https://sqlite.org/c3ref/c_alter_table.html +type AuthorizerActionCode uint32 + +const ( + /***************************************************** 3rd ************ 4th ***********/ + AUTH_CREATE_INDEX AuthorizerActionCode = 1 /* Index Name Table Name */ + AUTH_CREATE_TABLE AuthorizerActionCode = 2 /* Table Name NULL */ + AUTH_CREATE_TEMP_INDEX AuthorizerActionCode = 3 /* Index Name Table Name */ + AUTH_CREATE_TEMP_TABLE AuthorizerActionCode = 4 /* Table Name NULL */ + AUTH_CREATE_TEMP_TRIGGER AuthorizerActionCode = 5 /* Trigger Name Table Name */ + AUTH_CREATE_TEMP_VIEW AuthorizerActionCode = 6 /* View Name NULL */ + AUTH_CREATE_TRIGGER AuthorizerActionCode = 7 /* Trigger Name Table Name */ + AUTH_CREATE_VIEW AuthorizerActionCode = 8 /* View Name NULL */ + AUTH_DELETE AuthorizerActionCode = 9 /* Table Name NULL */ + AUTH_DROP_INDEX AuthorizerActionCode = 10 /* Index Name Table Name */ + AUTH_DROP_TABLE AuthorizerActionCode = 11 /* Table Name NULL */ + AUTH_DROP_TEMP_INDEX AuthorizerActionCode = 12 /* Index Name Table Name */ + AUTH_DROP_TEMP_TABLE AuthorizerActionCode = 13 /* Table Name NULL */ + AUTH_DROP_TEMP_TRIGGER AuthorizerActionCode = 14 /* Trigger Name Table Name */ + AUTH_DROP_TEMP_VIEW AuthorizerActionCode = 15 /* View Name NULL */ + AUTH_DROP_TRIGGER AuthorizerActionCode = 16 /* Trigger Name Table Name */ + AUTH_DROP_VIEW AuthorizerActionCode = 17 /* View Name NULL */ + AUTH_INSERT AuthorizerActionCode = 18 /* Table Name NULL */ + AUTH_PRAGMA AuthorizerActionCode = 19 /* Pragma Name 1st arg or NULL */ + AUTH_READ AuthorizerActionCode = 20 /* Table Name Column Name */ + AUTH_SELECT AuthorizerActionCode = 21 /* NULL NULL */ + AUTH_TRANSACTION AuthorizerActionCode = 22 /* Operation NULL */ + AUTH_UPDATE AuthorizerActionCode = 23 /* Table Name Column Name */ + AUTH_ATTACH AuthorizerActionCode = 24 /* Filename NULL */ + AUTH_DETACH AuthorizerActionCode = 25 /* Database Name NULL */ + AUTH_ALTER_TABLE AuthorizerActionCode = 26 /* Database Name Table Name */ + AUTH_REINDEX AuthorizerActionCode = 27 /* Index Name NULL */ + AUTH_ANALYZE AuthorizerActionCode = 28 /* Table Name NULL */ + AUTH_CREATE_VTABLE AuthorizerActionCode = 29 /* Table Name Module Name */ + AUTH_DROP_VTABLE AuthorizerActionCode = 30 /* Table Name Module Name */ + AUTH_FUNCTION AuthorizerActionCode = 31 /* NULL Function Name */ + AUTH_SAVEPOINT AuthorizerActionCode = 32 /* Operation Savepoint Name */ + AUTH_COPY AuthorizerActionCode = 0 /* No longer used */ + AUTH_RECURSIVE AuthorizerActionCode = 33 /* NULL NULL */ +) + +// AuthorizerReturnCode are the integer codes +// that the authorizer callback may return. +// +// https://sqlite.org/c3ref/c_deny.html +type AuthorizerReturnCode uint32 + +const ( + AUTH_OK AuthorizerReturnCode = 0 + AUTH_DENY AuthorizerReturnCode = 1 /* Abort the SQL statement with an error */ + AUTH_IGNORE AuthorizerReturnCode = 2 /* Don't allow access, but don't generate an error */ +) + +// CheckpointMode are all the checkpoint mode values. +// +// https://sqlite.org/c3ref/c_checkpoint_full.html +type CheckpointMode uint32 + +const ( + CHECKPOINT_PASSIVE CheckpointMode = 0 /* Do as much as possible w/o blocking */ + CHECKPOINT_FULL CheckpointMode = 1 /* Wait for writers, then checkpoint */ + CHECKPOINT_RESTART CheckpointMode = 2 /* Like FULL but wait for readers */ + CHECKPOINT_TRUNCATE CheckpointMode = 3 /* Like RESTART but also truncate WAL */ +) + +// TxnState are the allowed return values from [Conn.TxnState]. +// +// https://sqlite.org/c3ref/c_txn_none.html +type TxnState uint32 + +const ( + TXN_NONE TxnState = 0 + TXN_READ TxnState = 1 + TXN_WRITE TxnState = 2 +) + +// Datatype is a fundamental datatype of SQLite. +// +// https://sqlite.org/c3ref/c_blob.html +type Datatype uint32 + +const ( + INTEGER Datatype = 1 + FLOAT Datatype = 2 + TEXT Datatype = 3 + BLOB Datatype = 4 + NULL Datatype = 5 +) + +// String implements the [fmt.Stringer] interface. +func (t Datatype) String() string { + const name = "INTEGERFLOATEXTBLOBNULL" + switch t { + case INTEGER: + return name[0:7] + case FLOAT: + return name[7:12] + case TEXT: + return name[11:15] + case BLOB: + return name[15:19] + case NULL: + return name[19:23] + } + return strconv.FormatUint(uint64(t), 10) +} diff --git a/vendor/github.com/ncruces/go-sqlite3/context.go b/vendor/github.com/ncruces/go-sqlite3/context.go new file mode 100644 index 000000000..8d7604c66 --- /dev/null +++ b/vendor/github.com/ncruces/go-sqlite3/context.go @@ -0,0 +1,229 @@ +package sqlite3 + +import ( + "encoding/json" + "errors" + "math" + "time" + + "github.com/ncruces/go-sqlite3/internal/util" +) + +// Context is the context in which an SQL function executes. +// An SQLite [Context] is in no way related to a Go [context.Context]. +// +// https://sqlite.org/c3ref/context.html +type Context struct { + c *Conn + handle uint32 +} + +// Conn returns the database connection of the +// [Conn.CreateFunction] or [Conn.CreateWindowFunction] +// routines that originally registered the application defined function. +// +// https://sqlite.org/c3ref/context_db_handle.html +func (ctx Context) Conn() *Conn { + return ctx.c +} + +// SetAuxData saves metadata for argument n of the function. +// +// https://sqlite.org/c3ref/get_auxdata.html +func (ctx Context) SetAuxData(n int, data any) { + ptr := util.AddHandle(ctx.c.ctx, data) + ctx.c.call("sqlite3_set_auxdata_go", uint64(ctx.handle), uint64(n), uint64(ptr)) +} + +// GetAuxData returns metadata for argument n of the function. +// +// https://sqlite.org/c3ref/get_auxdata.html +func (ctx Context) GetAuxData(n int) any { + ptr := uint32(ctx.c.call("sqlite3_get_auxdata", uint64(ctx.handle), uint64(n))) + return util.GetHandle(ctx.c.ctx, ptr) +} + +// ResultBool sets the result of the function to a bool. +// SQLite does not have a separate boolean storage class. +// Instead, boolean values are stored as integers 0 (false) and 1 (true). +// +// https://sqlite.org/c3ref/result_blob.html +func (ctx Context) ResultBool(value bool) { + var i int64 + if value { + i = 1 + } + ctx.ResultInt64(i) +} + +// ResultInt sets the result of the function to an int. +// +// https://sqlite.org/c3ref/result_blob.html +func (ctx Context) ResultInt(value int) { + ctx.ResultInt64(int64(value)) +} + +// ResultInt64 sets the result of the function to an int64. +// +// https://sqlite.org/c3ref/result_blob.html +func (ctx Context) ResultInt64(value int64) { + ctx.c.call("sqlite3_result_int64", + uint64(ctx.handle), uint64(value)) +} + +// ResultFloat sets the result of the function to a float64. +// +// https://sqlite.org/c3ref/result_blob.html +func (ctx Context) ResultFloat(value float64) { + ctx.c.call("sqlite3_result_double", + uint64(ctx.handle), math.Float64bits(value)) +} + +// ResultText sets the result of the function to a string. +// +// https://sqlite.org/c3ref/result_blob.html +func (ctx Context) ResultText(value string) { + ptr := ctx.c.newString(value) + ctx.c.call("sqlite3_result_text64", + uint64(ctx.handle), uint64(ptr), uint64(len(value)), + uint64(ctx.c.freer), _UTF8) +} + +// ResultRawText sets the text result of the function to a []byte. +// +// https://sqlite.org/c3ref/result_blob.html +func (ctx Context) ResultRawText(value []byte) { + ptr := ctx.c.newBytes(value) + ctx.c.call("sqlite3_result_text64", + uint64(ctx.handle), uint64(ptr), uint64(len(value)), + uint64(ctx.c.freer), _UTF8) +} + +// ResultBlob sets the result of the function to a []byte. +// Returning a nil slice is the same as calling [Context.ResultNull]. +// +// https://sqlite.org/c3ref/result_blob.html +func (ctx Context) ResultBlob(value []byte) { + ptr := ctx.c.newBytes(value) + ctx.c.call("sqlite3_result_blob64", + uint64(ctx.handle), uint64(ptr), uint64(len(value)), + uint64(ctx.c.freer)) +} + +// ResultZeroBlob sets the result of the function to a zero-filled, length n BLOB. +// +// https://sqlite.org/c3ref/result_blob.html +func (ctx Context) ResultZeroBlob(n int64) { + ctx.c.call("sqlite3_result_zeroblob64", + uint64(ctx.handle), uint64(n)) +} + +// ResultNull sets the result of the function to NULL. +// +// https://sqlite.org/c3ref/result_blob.html +func (ctx Context) ResultNull() { + ctx.c.call("sqlite3_result_null", + uint64(ctx.handle)) +} + +// ResultTime sets the result of the function to a [time.Time]. +// +// https://sqlite.org/c3ref/result_blob.html +func (ctx Context) ResultTime(value time.Time, format TimeFormat) { + if format == TimeFormatDefault { + ctx.resultRFC3339Nano(value) + return + } + switch v := format.Encode(value).(type) { + case string: + ctx.ResultText(v) + case int64: + ctx.ResultInt64(v) + case float64: + ctx.ResultFloat(v) + default: + panic(util.AssertErr()) + } +} + +func (ctx Context) resultRFC3339Nano(value time.Time) { + const maxlen = uint64(len(time.RFC3339Nano)) + 5 + + ptr := ctx.c.new(maxlen) + buf := util.View(ctx.c.mod, ptr, maxlen) + buf = value.AppendFormat(buf[:0], time.RFC3339Nano) + + ctx.c.call("sqlite3_result_text64", + uint64(ctx.handle), uint64(ptr), uint64(len(buf)), + uint64(ctx.c.freer), _UTF8) +} + +// ResultPointer sets the result of the function to NULL, just like [Context.ResultNull], +// except that it also associates ptr with that NULL value such that it can be retrieved +// within an application-defined SQL function using [Value.Pointer]. +// +// https://sqlite.org/c3ref/result_blob.html +func (ctx Context) ResultPointer(ptr any) { + valPtr := util.AddHandle(ctx.c.ctx, ptr) + ctx.c.call("sqlite3_result_pointer_go", uint64(valPtr)) +} + +// ResultJSON sets the result of the function to the JSON encoding of value. +// +// https://sqlite.org/c3ref/result_blob.html +func (ctx Context) ResultJSON(value any) { + data, err := json.Marshal(value) + if err != nil { + ctx.ResultError(err) + return + } + ctx.ResultRawText(data) +} + +// ResultValue sets the result of the function to a copy of [Value]. +// +// https://sqlite.org/c3ref/result_blob.html +func (ctx Context) ResultValue(value Value) { + if value.c != ctx.c { + ctx.ResultError(MISUSE) + return + } + ctx.c.call("sqlite3_result_value", + uint64(ctx.handle), uint64(value.handle)) +} + +// ResultError sets the result of the function an error. +// +// https://sqlite.org/c3ref/result_blob.html +func (ctx Context) ResultError(err error) { + if errors.Is(err, NOMEM) { + ctx.c.call("sqlite3_result_error_nomem", uint64(ctx.handle)) + return + } + + if errors.Is(err, TOOBIG) { + ctx.c.call("sqlite3_result_error_toobig", uint64(ctx.handle)) + return + } + + msg, code := errorCode(err, _OK) + if msg != "" { + defer ctx.c.arena.mark()() + ptr := ctx.c.arena.string(msg) + ctx.c.call("sqlite3_result_error", + uint64(ctx.handle), uint64(ptr), uint64(len(msg))) + } + if code != _OK { + ctx.c.call("sqlite3_result_error_code", + uint64(ctx.handle), uint64(code)) + } +} + +// VTabNoChange may return true if a column is being fetched as part +// of an update during which the column value will not change. +// +// https://sqlite.org/c3ref/vtab_nochange.html +func (ctx Context) VTabNoChange() bool { + r := ctx.c.call("sqlite3_vtab_nochange", uint64(ctx.handle)) + return r != 0 +} diff --git a/vendor/github.com/ncruces/go-sqlite3/driver/driver.go b/vendor/github.com/ncruces/go-sqlite3/driver/driver.go new file mode 100644 index 000000000..b496f76ec --- /dev/null +++ b/vendor/github.com/ncruces/go-sqlite3/driver/driver.go @@ -0,0 +1,579 @@ +// Package driver provides a database/sql driver for SQLite. +// +// Importing package driver registers a [database/sql] driver named "sqlite3". +// You may also need to import package embed. +// +// import _ "github.com/ncruces/go-sqlite3/driver" +// import _ "github.com/ncruces/go-sqlite3/embed" +// +// The data source name for "sqlite3" databases can be a filename or a "file:" [URI]. +// +// The [TRANSACTION] mode can be specified using "_txlock": +// +// sql.Open("sqlite3", "file:demo.db?_txlock=immediate") +// +// Possible values are: "deferred", "immediate", "exclusive". +// A [read-only] transaction is always "deferred", regardless of "_txlock". +// +// The time encoding/decoding format can be specified using "_timefmt": +// +// sql.Open("sqlite3", "file:demo.db?_timefmt=sqlite") +// +// Possible values are: "auto" (the default), "sqlite", "rfc3339"; +// "auto" encodes as RFC 3339 and decodes any [format] supported by SQLite; +// "sqlite" encodes as SQLite and decodes any [format] supported by SQLite; +// "rfc3339" encodes and decodes RFC 3339 only. +// +// [PRAGMA] statements can be specified using "_pragma": +// +// sql.Open("sqlite3", "file:demo.db?_pragma=busy_timeout(10000)") +// +// If no PRAGMAs are specified, a busy timeout of 1 minute is set. +// +// Order matters: +// busy timeout and locking mode should be the first PRAGMAs set, in that order. +// +// [URI]: https://sqlite.org/uri.html +// [PRAGMA]: https://sqlite.org/pragma.html +// [format]: https://sqlite.org/lang_datefunc.html#time_values +// [TRANSACTION]: https://sqlite.org/lang_transaction.html#deferred_immediate_and_exclusive_transactions +// [read-only]: https://pkg.go.dev/database/sql#TxOptions +package driver + +import ( + "context" + "database/sql" + "database/sql/driver" + "errors" + "fmt" + "io" + "net/url" + "strings" + "time" + "unsafe" + + "github.com/ncruces/go-sqlite3" + "github.com/ncruces/go-sqlite3/internal/util" +) + +// This variable can be replaced with -ldflags: +// +// go build -ldflags="-X github.com/ncruces/go-sqlite3/driver.driverName=sqlite" +var driverName = "sqlite3" + +func init() { + if driverName != "" { + sql.Register(driverName, &SQLite{}) + } +} + +// Open opens the SQLite database specified by dataSourceName as a [database/sql.DB]. +// +// The init function is called by the driver on new connections. +// The [sqlite3.Conn] can be used to execute queries, register functions, etc. +// Any error returned closes the connection and is returned to [database/sql]. +func Open(dataSourceName string, init func(*sqlite3.Conn) error) (*sql.DB, error) { + c, err := (&SQLite{Init: init}).OpenConnector(dataSourceName) + if err != nil { + return nil, err + } + return sql.OpenDB(c), nil +} + +// SQLite implements [database/sql/driver.Driver]. +type SQLite struct { + // Init function is called by the driver on new connections. + // The [sqlite3.Conn] can be used to execute queries, register functions, etc. + // Any error returned closes the connection and is returned to [database/sql]. + Init func(*sqlite3.Conn) error +} + +// Open implements [database/sql/driver.Driver]. +func (d *SQLite) Open(name string) (driver.Conn, error) { + c, err := d.newConnector(name) + if err != nil { + return nil, err + } + return c.Connect(context.Background()) +} + +// OpenConnector implements [database/sql/driver.DriverContext]. +func (d *SQLite) OpenConnector(name string) (driver.Connector, error) { + return d.newConnector(name) +} + +func (d *SQLite) newConnector(name string) (*connector, error) { + c := connector{driver: d, name: name} + + var txlock, timefmt string + if strings.HasPrefix(name, "file:") { + if _, after, ok := strings.Cut(name, "?"); ok { + query, err := url.ParseQuery(after) + if err != nil { + return nil, err + } + txlock = query.Get("_txlock") + timefmt = query.Get("_timefmt") + c.pragmas = query.Has("_pragma") + } + } + + switch txlock { + case "": + c.txBegin = "BEGIN" + case "deferred", "immediate", "exclusive": + c.txBegin = "BEGIN " + txlock + default: + return nil, fmt.Errorf("sqlite3: invalid _txlock: %s", txlock) + } + + switch timefmt { + case "": + c.tmRead = sqlite3.TimeFormatAuto + c.tmWrite = sqlite3.TimeFormatDefault + case "sqlite": + c.tmRead = sqlite3.TimeFormatAuto + c.tmWrite = sqlite3.TimeFormat3 + case "rfc3339": + c.tmRead = sqlite3.TimeFormatDefault + c.tmWrite = sqlite3.TimeFormatDefault + default: + c.tmRead = sqlite3.TimeFormat(timefmt) + c.tmWrite = sqlite3.TimeFormat(timefmt) + } + return &c, nil +} + +type connector struct { + driver *SQLite + name string + txBegin string + tmRead sqlite3.TimeFormat + tmWrite sqlite3.TimeFormat + pragmas bool +} + +func (n *connector) Driver() driver.Driver { + return n.driver +} + +func (n *connector) Connect(ctx context.Context) (_ driver.Conn, err error) { + c := &conn{ + txBegin: n.txBegin, + tmRead: n.tmRead, + tmWrite: n.tmWrite, + } + + c.Conn, err = sqlite3.Open(n.name) + if err != nil { + return nil, err + } + defer func() { + if err != nil { + c.Close() + } + }() + + old := c.Conn.SetInterrupt(ctx) + defer c.Conn.SetInterrupt(old) + + if !n.pragmas { + err = c.Conn.BusyTimeout(60 * time.Second) + if err != nil { + return nil, err + } + } + if n.driver.Init != nil { + err = n.driver.Init(c.Conn) + if err != nil { + return nil, err + } + } + if n.pragmas || n.driver.Init != nil { + s, _, err := c.Conn.Prepare(`PRAGMA query_only`) + if err != nil { + return nil, err + } + if s.Step() && s.ColumnBool(0) { + c.readOnly = '1' + } else { + c.readOnly = '0' + } + err = s.Close() + if err != nil { + return nil, err + } + } + return c, nil +} + +type conn struct { + *sqlite3.Conn + txBegin string + txCommit string + txRollback string + tmRead sqlite3.TimeFormat + tmWrite sqlite3.TimeFormat + readOnly byte +} + +var ( + // Ensure these interfaces are implemented: + _ driver.ConnPrepareContext = &conn{} + _ driver.ExecerContext = &conn{} + _ driver.ConnBeginTx = &conn{} + _ sqlite3.DriverConn = &conn{} +) + +func (c *conn) Raw() *sqlite3.Conn { + return c.Conn +} + +func (c *conn) Begin() (driver.Tx, error) { + return c.BeginTx(context.Background(), driver.TxOptions{}) +} + +func (c *conn) BeginTx(ctx context.Context, opts driver.TxOptions) (driver.Tx, error) { + txBegin := c.txBegin + c.txCommit = `COMMIT` + c.txRollback = `ROLLBACK` + + if opts.ReadOnly { + txBegin = ` + BEGIN deferred; + PRAGMA query_only=on` + c.txRollback = ` + ROLLBACK; + PRAGMA query_only=` + string(c.readOnly) + c.txCommit = c.txRollback + } + + switch opts.Isolation { + default: + return nil, util.IsolationErr + case + driver.IsolationLevel(sql.LevelDefault), + driver.IsolationLevel(sql.LevelSerializable): + break + } + + old := c.Conn.SetInterrupt(ctx) + defer c.Conn.SetInterrupt(old) + + err := c.Conn.Exec(txBegin) + if err != nil { + return nil, err + } + return c, nil +} + +func (c *conn) Commit() error { + err := c.Conn.Exec(c.txCommit) + if err != nil && !c.Conn.GetAutocommit() { + c.Rollback() + } + return err +} + +func (c *conn) Rollback() error { + err := c.Conn.Exec(c.txRollback) + if errors.Is(err, sqlite3.INTERRUPT) { + old := c.Conn.SetInterrupt(context.Background()) + defer c.Conn.SetInterrupt(old) + err = c.Conn.Exec(c.txRollback) + } + return err +} + +func (c *conn) Prepare(query string) (driver.Stmt, error) { + return c.PrepareContext(context.Background(), query) +} + +func (c *conn) PrepareContext(ctx context.Context, query string) (driver.Stmt, error) { + old := c.Conn.SetInterrupt(ctx) + defer c.Conn.SetInterrupt(old) + + s, tail, err := c.Conn.Prepare(query) + if err != nil { + return nil, err + } + if tail != "" { + s.Close() + return nil, util.TailErr + } + return &stmt{Stmt: s, tmRead: c.tmRead, tmWrite: c.tmWrite}, nil +} + +func (c *conn) ExecContext(ctx context.Context, query string, args []driver.NamedValue) (driver.Result, error) { + if len(args) != 0 { + // Slow path. + return nil, driver.ErrSkip + } + + if savept, ok := ctx.(*saveptCtx); ok { + // Called from driver.Savepoint. + savept.Savepoint = c.Conn.Savepoint() + return resultRowsAffected(0), nil + } + + old := c.Conn.SetInterrupt(ctx) + defer c.Conn.SetInterrupt(old) + + err := c.Conn.Exec(query) + if err != nil { + return nil, err + } + + return newResult(c.Conn), nil +} + +func (c *conn) CheckNamedValue(arg *driver.NamedValue) error { + return nil +} + +type stmt struct { + *sqlite3.Stmt + tmWrite sqlite3.TimeFormat + tmRead sqlite3.TimeFormat +} + +var ( + // Ensure these interfaces are implemented: + _ driver.StmtExecContext = &stmt{} + _ driver.StmtQueryContext = &stmt{} + _ driver.NamedValueChecker = &stmt{} +) + +func (s *stmt) NumInput() int { + n := s.Stmt.BindCount() + for i := 1; i <= n; i++ { + if s.Stmt.BindName(i) != "" { + return -1 + } + } + return n +} + +// Deprecated: use ExecContext instead. +func (s *stmt) Exec(args []driver.Value) (driver.Result, error) { + return s.ExecContext(context.Background(), namedValues(args)) +} + +// Deprecated: use QueryContext instead. +func (s *stmt) Query(args []driver.Value) (driver.Rows, error) { + return s.QueryContext(context.Background(), namedValues(args)) +} + +func (s *stmt) ExecContext(ctx context.Context, args []driver.NamedValue) (driver.Result, error) { + err := s.setupBindings(args) + if err != nil { + return nil, err + } + + old := s.Stmt.Conn().SetInterrupt(ctx) + defer s.Stmt.Conn().SetInterrupt(old) + + err = s.Stmt.Exec() + if err != nil { + return nil, err + } + + return newResult(s.Stmt.Conn()), nil +} + +func (s *stmt) QueryContext(ctx context.Context, args []driver.NamedValue) (driver.Rows, error) { + err := s.setupBindings(args) + if err != nil { + return nil, err + } + return &rows{ctx: ctx, stmt: s}, nil +} + +func (s *stmt) setupBindings(args []driver.NamedValue) error { + err := s.Stmt.ClearBindings() + if err != nil { + return err + } + + var ids [3]int + for _, arg := range args { + ids := ids[:0] + if arg.Name == "" { + ids = append(ids, arg.Ordinal) + } else { + for _, prefix := range []string{":", "@", "$"} { + if id := s.Stmt.BindIndex(prefix + arg.Name); id != 0 { + ids = append(ids, id) + } + } + } + + for _, id := range ids { + switch a := arg.Value.(type) { + case bool: + err = s.Stmt.BindBool(id, a) + case int: + err = s.Stmt.BindInt(id, a) + case int64: + err = s.Stmt.BindInt64(id, a) + case float64: + err = s.Stmt.BindFloat(id, a) + case string: + err = s.Stmt.BindText(id, a) + case []byte: + err = s.Stmt.BindBlob(id, a) + case sqlite3.ZeroBlob: + err = s.Stmt.BindZeroBlob(id, int64(a)) + case time.Time: + err = s.Stmt.BindTime(id, a, s.tmWrite) + case util.JSON: + err = s.Stmt.BindJSON(id, a.Value) + case util.PointerUnwrap: + err = s.Stmt.BindPointer(id, util.UnwrapPointer(a)) + case nil: + err = s.Stmt.BindNull(id) + default: + panic(util.AssertErr()) + } + } + if err != nil { + return err + } + } + return nil +} + +func (s *stmt) CheckNamedValue(arg *driver.NamedValue) error { + switch arg.Value.(type) { + case bool, int, int64, float64, string, []byte, + time.Time, sqlite3.ZeroBlob, + util.JSON, util.PointerUnwrap, + nil: + return nil + default: + return driver.ErrSkip + } +} + +func newResult(c *sqlite3.Conn) driver.Result { + rows := c.Changes() + if rows != 0 { + id := c.LastInsertRowID() + if id != 0 { + return result{id, rows} + } + } + return resultRowsAffected(rows) +} + +type result struct{ lastInsertId, rowsAffected int64 } + +func (r result) LastInsertId() (int64, error) { + return r.lastInsertId, nil +} + +func (r result) RowsAffected() (int64, error) { + return r.rowsAffected, nil +} + +type resultRowsAffected int64 + +func (r resultRowsAffected) LastInsertId() (int64, error) { + return 0, nil +} + +func (r resultRowsAffected) RowsAffected() (int64, error) { + return int64(r), nil +} + +type rows struct { + ctx context.Context + *stmt + names []string + types []string +} + +func (r *rows) Close() error { + r.Stmt.ClearBindings() + return r.Stmt.Reset() +} + +func (r *rows) Columns() []string { + if r.names == nil { + count := r.Stmt.ColumnCount() + r.names = make([]string, count) + for i := range r.names { + r.names[i] = r.Stmt.ColumnName(i) + } + } + return r.names +} + +func (r *rows) declType(index int) string { + if r.types == nil { + count := r.Stmt.ColumnCount() + r.types = make([]string, count) + for i := range r.types { + r.types[i] = strings.ToUpper(r.Stmt.ColumnDeclType(i)) + } + } + return r.types[index] +} + +func (r *rows) ColumnTypeDatabaseTypeName(index int) string { + decltype := r.declType(index) + if len := len(decltype); len > 0 && decltype[len-1] == ')' { + if i := strings.LastIndexByte(decltype, '('); i >= 0 { + decltype = decltype[:i] + } + } + return strings.TrimSpace(decltype) +} + +func (r *rows) Next(dest []driver.Value) error { + old := r.Stmt.Conn().SetInterrupt(r.ctx) + defer r.Stmt.Conn().SetInterrupt(old) + + if !r.Stmt.Step() { + if err := r.Stmt.Err(); err != nil { + return err + } + return io.EOF + } + + data := unsafe.Slice((*any)(unsafe.SliceData(dest)), len(dest)) + err := r.Stmt.Columns(data) + for i := range dest { + if t, ok := r.decodeTime(i, dest[i]); ok { + dest[i] = t + continue + } + if s, ok := dest[i].(string); ok { + t, ok := maybeTime(s) + if ok { + dest[i] = t + } + } + } + return err +} + +func (r *rows) decodeTime(i int, v any) (_ time.Time, _ bool) { + if r.tmRead == sqlite3.TimeFormatDefault { + return + } + switch r.declType(i) { + case "DATE", "TIME", "DATETIME", "TIMESTAMP": + // maybe + default: + return + } + switch v.(type) { + case int64, float64, string: + // maybe + default: + return + } + t, err := r.tmRead.Decode(v) + return t, err == nil +} diff --git a/vendor/github.com/ncruces/go-sqlite3/driver/savepoint.go b/vendor/github.com/ncruces/go-sqlite3/driver/savepoint.go new file mode 100644 index 000000000..60aa6b991 --- /dev/null +++ b/vendor/github.com/ncruces/go-sqlite3/driver/savepoint.go @@ -0,0 +1,27 @@ +package driver + +import ( + "database/sql" + "time" + + "github.com/ncruces/go-sqlite3" +) + +// Savepoint establishes a new transaction savepoint. +// +// https://sqlite.org/lang_savepoint.html +func Savepoint(tx *sql.Tx) sqlite3.Savepoint { + var ctx saveptCtx + tx.ExecContext(&ctx, "") + return ctx.Savepoint +} + +type saveptCtx struct{ sqlite3.Savepoint } + +func (*saveptCtx) Deadline() (deadline time.Time, ok bool) { return } + +func (*saveptCtx) Done() <-chan struct{} { return nil } + +func (*saveptCtx) Err() error { return nil } + +func (*saveptCtx) Value(key any) any { return nil } diff --git a/vendor/github.com/ncruces/go-sqlite3/driver/time.go b/vendor/github.com/ncruces/go-sqlite3/driver/time.go new file mode 100644 index 000000000..630a5b10b --- /dev/null +++ b/vendor/github.com/ncruces/go-sqlite3/driver/time.go @@ -0,0 +1,31 @@ +package driver + +import ( + "time" +) + +// Convert a string in [time.RFC3339Nano] format into a [time.Time] +// if it roundtrips back to the same string. +// This way times can be persisted to, and recovered from, the database, +// but if a string is needed, [database/sql] will recover the same string. +func maybeTime(text string) (_ time.Time, _ bool) { + // Weed out (some) values that can't possibly be + // [time.RFC3339Nano] timestamps. + if len(text) < len("2006-01-02T15:04:05Z") { + return + } + if len(text) > len(time.RFC3339Nano) { + return + } + if text[4] != '-' || text[10] != 'T' || text[16] != ':' { + return + } + + // Slow path. + var buf [len(time.RFC3339Nano)]byte + date, err := time.Parse(time.RFC3339Nano, text) + if err == nil && text == string(date.AppendFormat(buf[:0], time.RFC3339Nano)) { + return date, true + } + return +} diff --git a/vendor/github.com/ncruces/go-sqlite3/driver/util.go b/vendor/github.com/ncruces/go-sqlite3/driver/util.go new file mode 100644 index 000000000..033841157 --- /dev/null +++ b/vendor/github.com/ncruces/go-sqlite3/driver/util.go @@ -0,0 +1,14 @@ +package driver + +import "database/sql/driver" + +func namedValues(args []driver.Value) []driver.NamedValue { + named := make([]driver.NamedValue, len(args)) + for i, v := range args { + named[i] = driver.NamedValue{ + Ordinal: i + 1, + Value: v, + } + } + return named +} diff --git a/vendor/github.com/ncruces/go-sqlite3/embed/README.md b/vendor/github.com/ncruces/go-sqlite3/embed/README.md new file mode 100644 index 000000000..400fe870a --- /dev/null +++ b/vendor/github.com/ncruces/go-sqlite3/embed/README.md @@ -0,0 +1,27 @@ +# Embeddable Wasm build of SQLite + +This folder includes an embeddable Wasm build of SQLite 3.46.0 for use with +[`github.com/ncruces/go-sqlite3`](https://pkg.go.dev/github.com/ncruces/go-sqlite3). + +The following optional features are compiled in: +- [math functions](https://sqlite.org/lang_mathfunc.html) +- [FTS5](https://sqlite.org/fts5.html) +- [JSON](https://sqlite.org/json1.html) +- [R*Tree](https://sqlite.org/rtree.html) +- [GeoPoly](https://sqlite.org/geopoly.html) +- [soundex](https://sqlite.org/lang_corefunc.html#soundex) +- [stat4](https://sqlite.org/compile.html#enable_stat4) +- [base64](https://github.com/sqlite/sqlite/blob/master/ext/misc/base64.c) +- [decimal](https://github.com/sqlite/sqlite/blob/master/ext/misc/decimal.c) +- [ieee754](https://github.com/sqlite/sqlite/blob/master/ext/misc/ieee754.c) +- [regexp](https://github.com/sqlite/sqlite/blob/master/ext/misc/regexp.c) +- [series](https://github.com/sqlite/sqlite/blob/master/ext/misc/series.c) +- [uint](https://github.com/sqlite/sqlite/blob/master/ext/misc/uint.c) +- [uuid](https://github.com/sqlite/sqlite/blob/master/ext/misc/uuid.c) +- [time](../sqlite3/time.c) + +See the [configuration options](../sqlite3/sqlite_cfg.h), +and [patches](../sqlite3) applied. + +Built using [`wasi-sdk`](https://github.com/WebAssembly/wasi-sdk), +and [`binaryen`](https://github.com/WebAssembly/binaryen). \ No newline at end of file diff --git a/vendor/github.com/ncruces/go-sqlite3/embed/build.sh b/vendor/github.com/ncruces/go-sqlite3/embed/build.sh new file mode 100644 index 000000000..abe5e60c4 --- /dev/null +++ b/vendor/github.com/ncruces/go-sqlite3/embed/build.sh @@ -0,0 +1,32 @@ +#!/usr/bin/env bash +set -euo pipefail + +cd -P -- "$(dirname -- "$0")" + +ROOT=../ +BINARYEN="$ROOT/tools/binaryen-version_117/bin" +WASI_SDK="$ROOT/tools/wasi-sdk-22.0/bin" + +"$WASI_SDK/clang" --target=wasm32-wasi -std=c17 -flto -g0 -O2 \ + -Wall -Wextra -Wno-unused-parameter -Wno-unused-function \ + -o sqlite3.wasm "$ROOT/sqlite3/main.c" \ + -I"$ROOT/sqlite3" \ + -mexec-model=reactor \ + -msimd128 -mmutable-globals \ + -mbulk-memory -mreference-types \ + -mnontrapping-fptoint -msign-ext \ + -fno-stack-protector -fno-stack-clash-protection \ + -Wl,--initial-memory=327680 \ + -Wl,--stack-first \ + -Wl,--import-undefined \ + -D_HAVE_SQLITE_CONFIG_H \ + -DSQLITE_CUSTOM_INCLUDE=sqlite_opt.h \ + $(awk '{print "-Wl,--export="$0}' exports.txt) + +trap 'rm -f sqlite3.tmp' EXIT +"$BINARYEN/wasm-ctor-eval" -g -c _initialize sqlite3.wasm -o sqlite3.tmp +"$BINARYEN/wasm-opt" -g --strip --strip-producers -c -O3 \ + sqlite3.tmp -o sqlite3.wasm \ + --enable-simd --enable-mutable-globals --enable-multivalue \ + --enable-bulk-memory --enable-reference-types \ + --enable-nontrapping-float-to-int --enable-sign-ext \ No newline at end of file diff --git a/vendor/github.com/ncruces/go-sqlite3/embed/exports.txt b/vendor/github.com/ncruces/go-sqlite3/embed/exports.txt new file mode 100644 index 000000000..b3cb1581c --- /dev/null +++ b/vendor/github.com/ncruces/go-sqlite3/embed/exports.txt @@ -0,0 +1,130 @@ +aligned_alloc +free +malloc +malloc_destructor +sqlite3_anycollseq_init +sqlite3_autovacuum_pages_go +sqlite3_backup_finish +sqlite3_backup_init +sqlite3_backup_pagecount +sqlite3_backup_remaining +sqlite3_backup_step +sqlite3_bind_blob64 +sqlite3_bind_double +sqlite3_bind_int64 +sqlite3_bind_null +sqlite3_bind_parameter_count +sqlite3_bind_parameter_index +sqlite3_bind_parameter_name +sqlite3_bind_pointer_go +sqlite3_bind_text64 +sqlite3_bind_value +sqlite3_bind_zeroblob64 +sqlite3_blob_bytes +sqlite3_blob_close +sqlite3_blob_open +sqlite3_blob_read +sqlite3_blob_reopen +sqlite3_blob_write +sqlite3_busy_handler_go +sqlite3_busy_timeout +sqlite3_changes64 +sqlite3_clear_bindings +sqlite3_close +sqlite3_close_v2 +sqlite3_collation_needed_go +sqlite3_column_blob +sqlite3_column_bytes +sqlite3_column_count +sqlite3_column_database_name +sqlite3_column_decltype +sqlite3_column_double +sqlite3_column_int64 +sqlite3_column_name +sqlite3_column_origin_name +sqlite3_column_table_name +sqlite3_column_text +sqlite3_column_type +sqlite3_column_value +sqlite3_columns_go +sqlite3_commit_hook_go +sqlite3_config_log_go +sqlite3_create_aggregate_function_go +sqlite3_create_collation_go +sqlite3_create_function_go +sqlite3_create_module_go +sqlite3_create_window_function_go +sqlite3_database_file_object +sqlite3_db_config +sqlite3_db_filename +sqlite3_db_name +sqlite3_db_readonly +sqlite3_db_release_memory +sqlite3_declare_vtab +sqlite3_errcode +sqlite3_errmsg +sqlite3_error_offset +sqlite3_errstr +sqlite3_exec +sqlite3_filename_database +sqlite3_filename_journal +sqlite3_filename_wal +sqlite3_finalize +sqlite3_get_autocommit +sqlite3_get_auxdata +sqlite3_interrupt +sqlite3_last_insert_rowid +sqlite3_limit +sqlite3_open_v2 +sqlite3_overload_function +sqlite3_prepare_v3 +sqlite3_progress_handler_go +sqlite3_reset +sqlite3_result_blob64 +sqlite3_result_double +sqlite3_result_error +sqlite3_result_error_code +sqlite3_result_error_nomem +sqlite3_result_error_toobig +sqlite3_result_int64 +sqlite3_result_null +sqlite3_result_pointer_go +sqlite3_result_text64 +sqlite3_result_value +sqlite3_result_zeroblob64 +sqlite3_rollback_hook_go +sqlite3_set_authorizer_go +sqlite3_set_auxdata_go +sqlite3_set_last_insert_rowid +sqlite3_step +sqlite3_stmt_busy +sqlite3_stmt_readonly +sqlite3_stmt_status +sqlite3_total_changes64 +sqlite3_txn_state +sqlite3_update_hook_go +sqlite3_uri_key +sqlite3_uri_parameter +sqlite3_value_blob +sqlite3_value_bytes +sqlite3_value_double +sqlite3_value_dup +sqlite3_value_free +sqlite3_value_int64 +sqlite3_value_nochange +sqlite3_value_numeric_type +sqlite3_value_pointer_go +sqlite3_value_text +sqlite3_value_type +sqlite3_vtab_collation +sqlite3_vtab_config_go +sqlite3_vtab_distinct +sqlite3_vtab_in +sqlite3_vtab_in_first +sqlite3_vtab_in_next +sqlite3_vtab_nochange +sqlite3_vtab_on_conflict +sqlite3_vtab_rhs_value +sqlite3_wal_autocheckpoint +sqlite3_wal_checkpoint_v2 +sqlite3_wal_hook_go \ No newline at end of file diff --git a/vendor/github.com/ncruces/go-sqlite3/embed/init.go b/vendor/github.com/ncruces/go-sqlite3/embed/init.go new file mode 100644 index 000000000..da527abd0 --- /dev/null +++ b/vendor/github.com/ncruces/go-sqlite3/embed/init.go @@ -0,0 +1,20 @@ +// Package embed embeds SQLite into your application. +// +// Importing package embed initializes the [sqlite3.Binary] variable +// with an appropriate build of SQLite: +// +// import _ "github.com/ncruces/go-sqlite3/embed" +package embed + +import ( + _ "embed" + + "github.com/ncruces/go-sqlite3" +) + +//go:embed sqlite3.wasm +var binary []byte + +func init() { + sqlite3.Binary = binary +} diff --git a/vendor/github.com/ncruces/go-sqlite3/embed/sqlite3.wasm b/vendor/github.com/ncruces/go-sqlite3/embed/sqlite3.wasm new file mode 100644 index 0000000000000000000000000000000000000000..2689f773a6c213fe98c56e4700584c25904c5a9e GIT binary patch literal 1365178 zcmd3P3w&Hvo&R~=$IP8crlmmJ6gYF~vURmum92tea$-uUfPf;d?teF=?NBbp3HXQvXIoR2Jh!34D=!<9bHWg!NaTuQ&8bfiad6=CohEPyehmk!l$<9tib)X4Z z9kvY)QFLX^b;@mcj#XjHC;?=W# zbRB;%Wk_(U77z-*b_$0GoCHEyts)aa(vAc92=4gTK_aU1=mC#nEZr#!eXG4?{oUuR z>RZCn@!Mdfu#yFM^YLwYUr#TS@$2fI-c<$W;x)ajzk7M#>Oy(2U<)6ztiO9ne{XM3 zWqE&JcVD5fw6N5ELDnVxD^~WD3(SklE-egJ%Khgvjs%olS?*s}E({KKpWD;7w6{=Z z3}0H8^>;5`HF$oKDv;V@Yh1Cs(tU1!f5~ReE$ixorD-7D*IDh1}C7~Xr9EGY~Q+N#m9b5`~CcCYNIoZHv4qR?ou+|#$T ze?=d%Qc=Ksu(wcHX~#Jg<6KoP7y2sQmE|i6-SdyNA2QUlB|W`JW!ZR0tIrw4lr6OY zLFv_%p2gjZ3xk#J<$X&FYiwzZb;+u7_c_aZD}}Nh1R#R6zQUS{El$(6s@z@ZKgTA- z#g_E_ zEp!hq|47swWa#Z*QnDFmHx7Ag-B`!q3i19OA6gf&h07pEU6UA%LgmVmkiqPnD@bRS9GuFS=m4hBv4m~ zSg77}S9CA#DVLWQ%9hYDASABpLrEGV2GqZ@&}YdhnS&+$eSL)`6Jo$X^ zhk!B8nJZW(!!r0YEi>(PI(o`XoAq~@jLu{nM+oLHkq=-0hWy+&3McE>Kk`jtmXw@J z&zGJjrSxQ4xm>uCWgUKmBeOEcj}}>}lp~lsl{4lBo+||7uIsuI|G2K}3p5}+#+)ev zA32wf%a|)AcR6#@E|)Gxs>@v=6n7nmIXugR!@MIHV^Ro*G09RXCImZFI<9gU_monM zJC0I<3&-P*uY?e;niouQgeR2C0+tq1;6I0R*X6GFW_eiPq?N;i+37&09mj2XizB&| zN(v=~be&e!>N`p(DV?+dh_5{a&I1pPYFQ)x%ebltykk?sr0-E`Vbi@;S<3dd11r7Og& zG-Iy!RwwNSGle^wOIOHN(IO8&w)L>X9qCBdk=%8sC`Y)$;cmKx35K!zBAuQoJmER+ zVQ$ds3K?J&XiK>17S!!Jfl_JaO%+Ng<~Yicf;0E;6{bN5&sC1}gb$#5j4{U*F7Qb@ zEXSASnK70rrB3nBc&FnTF@ zpjsuMmUbM;m|{H5n3PgFF2MLU2?Ul>G7jwG%#j?JZ2xfkfc$Tdl7i?r`opGh_M4xCH7Jnc%*SKvFYRMc$Lwj+c@1My$# z6gBAy!CWukAPy-71JD>#4*eVrmHW6*PT@T3GE2M19|?pILcHS`@y@B@7-uSwf;lG~ z%9QeyVp*RFP_4>M$>mU?0P3S&M|qy7JRSsruY9iP9}5HS10@*u7-x=9%J+T2I73R? z-?rfw$yp&3y5>mq)U-@8bzE7-!dp=`o z4nYKbNidcL$6>7~r7W8|e8vo!#tb@6`jDeQ%&to3lt`!3M@gPer)N0KmF)~D(t?h1 zjtN5LoH+_WbUK7k;D2JOf9RpkR0U9a5`bggq0DzY#)RV$LFJU=D?gRuj__1kNze1M zdDg|~Ju?Wy!E?inmtIo!Xv z2od*d|B(1Q^wL!;yO;MZudv@7kW6~_3(ldFa-XxjZ~5T4?2pd$Ngv9E6+O%QmiH}V zzx5}7gKoN{e^pJO6THOi}*6RlU7z{DArau=&TbZ+Z!AqWm;E zw$W*e2i4qeccs67@$zMCBsF8w=Z_T1{g|5h$Fj8te55{iFX>-eU>BxlPAX|X_VuqQ ztYBZ~nfNPV=wm--RNhOJ(|CZ4=7;g>0<}fPZ<2Plk`5S$4SvDU}9Uen5?oDAyFO~R2Dq$A!rHBl-q*k)2W1(D3y@+s!26OsJm%W4Hi7jj_|Gl|AL26@^LxmgTQ&~00#u^}w*vzUGh4S(x-86eY@fzUi?O$x|l6n&ykZ4VlYa5MH zDjA8NpOn?wieH$NYEk<7MhBw`$O`&mDxuKAn!*w`>a@hTRw$QO3@&5;+Tfon%LTYx zx>vuIZEp|)kdaa=J&W0WJP{p4ul=CFzH@q)FR8E{sYEK!x5T%@zLaW>TaBok-7qP0 z`P{(>C3ZF@hB7aOsoA%r!ftFJ`_jeTWowpqTdB%!RSB$@E~Y*$U&g-NDAC(P9*x05 zxe~b^ZcZH$cWSUuIY@@DH0H$0j=}lIvadD}w$fke>23OSOCx_}O&_61f!&%)WT&0! zucZ#>Fo`KgtrRrC-MMT z$HpCvqWIeQf=+C86J}<4eE&J;K-k~en7l%{Oj6;lR5KP0R?6(|#xQ%A!{u^MqZAMk z4ScgP<(8*c`Xm3$#hFArtST>WP&)gLlOPU0lnUpwk7W{B2KNPMJyE3CC7A?0laIyz zRb}`jFU=(5OTL}g)5|`dN!nPt*z%4~WD@DKba9fKZJdw{5+#OlStjA%J$>f`>jw)1 zR%^RFldw(8f(N^o^|LE83Fk^^B=Uda^X8_{#?I2AFpfR8@Si< zWpLW!!`8;ICVOu2d0W%4ATx=o3RLA68tp|ISkuSr8zZw~qUqZgGm4bWE_X#wZ*Tt+ z)|sGiG=VGnm#*q9p#AL)C`25VI=`%+eKj-17PlU(Rq)_Rp6%&fzO1jXwASip8qS~z(2}A#xLa`=bzvk`DJ{Y`j~f#_X+Q_e674t zUMIGS4dSEXZgG$Jrnpz!C%!1R%Uk5F@*DCtxl2AIuMppn-<6Nb@5>*^AIcxe7x;h3 zr{yo@^Kz~Fr23Tlyt+kwO?_AWRQ*itQ)`{?t3Rnft1HB#>M`{L^;`Fk?nCa2?#=G+ z^u@=>E|Ck^7{(-ut}&L;pwq&;7^!|M0I%ZALJE`xczL$D9^+;-)dNlR@)MKe1q<)zCQR>I3pQL`8`dMmU>Ko}@ z>D$wHrnjW8O@AW2G5rnsc>0O-lj*0@Pp5yGekT2F`d8`Sr1z(vOaCstKKOWWNpNX! zTd+5{H`o_E9y}5JCK%0Jlv$tod2mJMA=h!ePi8)q`E=&0%%+T=N(Y&2OKa}PBWAVD z+>!ZG=7!9SotYamU(Ot*Z_3=9nSFGa|J!5!<|~;oukLN}uJu0eUEy8nebW1scQ1c{ zKg7SqAL9SUzs>jZ@9^*Ou1|=K;xh3vzd~Fo);k-VhxkXGi=B@-mpC7HKH*&E)ZH=n zbM9x|PrEmJU-7=`-QwNqea-v2_YLnhZxy=o%}2B!@u^rf8+h1Q%^hNeIHz0_|TGb-h2A{-?QKQt@oVwJMS_7-~A{2 zC;eaeclvkvcl-DF-}LYG@AL2XAMo$fhpj&HuXp4gWTOm;Zb3kKRk(xOb8NQU7E9rT!=U%lymzEBq_{Px_zo zukttfoBhxDpY@;eyPo!c=|AH?>p$XO=WX@2d0+H)cwh2v@NV?J>|O8Om9 zHW&-G1fLJK20MbC!HvPq!B>JigS&!f{kwy22KNQ`2M+`f1`h?_3ceRS96S;{8hk%^ zEcik2!{EokPlBHYKMVdn_(kv^!BfF6gJ*(ggI@)|4(>|dmA*Uu5Pyi@lfEzgK>Azh zz3K0yzni`}{nhj>>EEaRZ|eEf3#nVvBk2p%UrUdszn;D%eQEmA^kwPG(^sUgOkbb= zVtRXeNBT?Y_1+EXo#`9XH>LNdo=g2M^Q|{JQm^~F*S}%z8{ahV&2M?@v421RZEw$rAIkI(ww?Tkz-5}{-W^;ymj~6OnaP#H z^lWCX-kzExSm-TK9p=Y3(6hqxQ#YhKt8Wv`?7Jb=IWjUjGBUD<;r+)qq;%T&MYD3i zJa+vf*E@runlG5ye?!VJJnV@c0(18bDYN&c6cVr8o;+^bp3+p}+U+S*#mD<@NSU1v zrRNJKJ2W?Ywx^1ko13?%ilLYCw?Rb-T50_ zFIXTt4)#|w^DIAAb(m2LWY^?0Glvw-emS6-2^O`5j*+WFwmRfS4yX=+;qw8_%7zby z!pKUnGqccWmRtbe+T9nF~YgQRJ7d=gSCH`Ws_;~}N z8}&=X{Svlc+DZCC74o9y+9}x?X4)x+N=w^0?W+#$n>7Oji99z~Q9tuSRJCik_KjR! z37r98?dbNDcC=e2JRx|@5q=Q5)U4+C@qteBMMewl&KG_RBh+fq@{;W-LX=Dv_4f$Gr0OrvakK^C(DNK@ueXr+Z# zi$c+sSAeM&ZF#17USxO>U{0pzQ<^3x6Uv3E1Ee!Yu$fxoqaSh%fu=c=K>;(Z7z&^T zwaaV|vQjgXE9ONYJG97qnuoHHG0U{1uUb;}EDN+p#G6S6rkZy&GrOxNXBlB@$jr)t zyc5I1Uyx;n(YPa=uj18=h(%d8(;TH8Gd@1_65AdiRGef5gUtKaxr_50c+bVg6?=hfbfVx_pmStMxrFKJwzU7q` zd7hD=hIk%a?P_4_5vz?FD4HBijySPRbM3Z)LJY6unHE_V5D}GakE9Vg1$1YcmCD*5 z2pPe)85Q!FU`#WZ*Mx~)gc`Nd%ucn$gG8QO$W&D`M{|oNL5MjBv|Ny7T1AAYoe~Bn zff1gTc*umT86)Upf)Q-K$YfH1ZklLdqMc&&+RnMf2v_s433X_yfiDpP0V75zHGEK$rh;&?P#wE-iUrM+cG2?2%LDfE$ziU&|X}RMKSG|ds#7*q8iCQ)nT3> zo)0X16%m+e5`k#$00Btxflw4fMI1DA2SRb0>M*~*A!X))FQx<0VQ#ZRmxHWsw$DEJ zb4vRsYpzqL4G>QA=35hL=L6*#J10W+lzEDjKGFX`q;Q1F+W9vW|Am%a+8Qr1z>L z?!3}&G4!-LHS21wy+v6!&~EM&%+kUv&v*d{qNLq{FsVjS21fn19X=5ZEYAbcpGaq3X~~yG0;8B^##QQtp(Q85Ix- zR2OEQj08T&jE5hVallop3$udY3|P+*%qR$zK#O?nT#MW(z}Va=#6PMhLTuU}ruk`j z)2tMt|8Fmzn&kl?6hR3vA{@PyD%X6G>cBH0Zlq1bs+nk-TMMI*n1~8h@x-iX%xXpi zk+29kprZL{ZD3s5)010M|7SsyZ zLBwJ#BC~|EXs&_W2UQL&B+)48m_mC|qdcAy%}#Ef3yYP~Qf;g}S_#6mBwUKAmH6gd zhOR=)#DorggGeTnAa#oq(3`}`knGH=rgpeUMlfLdx{a>Ye$ zp)^MyeCUM87`g{m4Jl(Dx*_#C0dqmq2fmdFgD0gz6sl^KWft)VDb)scy6tcr8PwIan)CW<51P>-1= zMH;$R5R%520oqu_5i-{5k3N{jQF+t?UT7OSMyrEqJJ1kF>71oSCwrU7X&!4oc^2zc z(s0cyp^NU=2I&)+9Q8_|@#$K`U4bGE9Y>iNFd9ft;#N=M=sl@QTEbc&RY?)AFh{J0 z3Ic5hVpB%iQ0RjFOA`B*Y?_ zN_khrY)vZ|Fu)oI=m7i53U*K($$V&`);i%qq;%Lq(A25WCEHW6u5>^lG9sw2ARjeH zD6+8(KM@kbtQp9s$l8iO)zHQ<-zrkufldvzUB0@KCasc2Iy1ER1zCok%n`G~1joocAkf)aWtwF< z&@Mtfj4&CVXL4qm_U4G3RwuSu2x>-F3k_f z3A$etJ0z>mfD##l>l64Z%Lz_v?TS(Dx? zpbkbDE)Rm?aEVO9aP3><4t?m`0%mT4-tyq+ZSAE4(HmwJ>Bz9r4nS`ZooR&>kXmV{ z9J0i}G0Duq3`p2Y5VclP$zT8u=5%T|6rI+?!L2S9&e`T=+ z(tky)cL?j9X2Au;`iRxg;bXz&##m2+%Xu}y#a}xLlZHZyljzVX%TL)su#H)4BXv(& zeyW2BtTaWqmI5nfz%`qdSgF)0(3|zvSmn*KQaN8tw@#e|*MN92E2Ym3Tw@m3c!Fyj ziV`^?V#!HzFq8?dH9T)J!L?U*ur=IkniP}&jDwBjHO5YMJv_e{ApDYnPMchDHq+cX zrxyelX1G@s9S}GmG9UgjW>krQVb%b!z8M!~xi!V?r_q=Ds%a1T;;Aa~@R6M=RTRYoq%3Wy~uLJ7LD z;so{&$PpH_lHnI<*CMk3GRs&2r8YHm9!O)9z3dQV5? zxlocjNCKnD5k1|)!9syXH!Y?HRDsXhfq@^5jSo;ytyN?v+3JdOAk|{Mkvv3AdLxVU z#$?=DL~7&0tV_BgnYCAKw{eQrM5&}2p;S_hO-ki~HNtaFKyD34iPWq}ve;^oj>RYk z!t$@hD92)yV=DnhC5~TiYn+5xP6J_pSx$mk4&ly!2D4&RY7Dah%t#t1SmYc8ivY;K z4vU6*$80nTQ5w9%L;+mog zqzD}tRbcal-wM?dKKR%+WQix7TTeJ789d>6FbU8~VjCu;2^VW@9V)}Mtb`s}hop#E zY3vLvxddLj-Xv{^tPpA0&p&$&P0* zNf7@sycE3}@&*aKfibh=8O?w6nHiB}^x!t49GF~{q^F{EIFaN)2$hex4P+3k;2}!m zR02F!+7gt>w?6>3U$UjR$wW`m%s|zYak~APW`7V$Bx{D7TvBXW@-#hpniW??rh=JX z4kf&2n(EcEn1_dtb+Wg~>GmgQe|-Cc#0kyzo@Ab|jm;$)EP3A>~i zh99#_a+4^lQZ)%;(!696Cu?GtO%j{tngp>GYfXY!MuJ!vIRHT{VLLSmVv>wxG7=8N zg4n7Mh|hu+;IL(Rk?>SfGZXT+l9|*@A|ynoCYkLAmCU4TlIv5DWTrT0q-%nw61Rb5 zhIMEGlkijwo^-YNvz5Vu zMR5ad{(>mBXz-fQhD7m!Xp@NINoX@c6d!;#2kW+O;D8my(d5PYIP`n#rxg=LG2CyJ zkSC%z_R}I_PD(fYw2@~Pvc-BR;Pio;ClbXx5yjjb16I3wStzV17EPjL_tiVZ#o#0?Fsrt%Hl!fFpQSGAUP}&Im{C|EE;lHOpwEfQfm)YgTzP< zQ+pV))jlGxBejlr8}$X^0zD%!Y=IL5Uy$6XVzB^x7d-dw)<9H62NtSOVjCg=hL=e1 zka-AGqp{jS3rNrkBE}RDa2m=_HEE88FDnN$r*$u~Xfnk}fX~#z_$3M` zLnv$FF%+)D{2EmX9WW@k0ZGLp!Hu9_Y#%}!;U&|0Y8xWTumZ6t!-YaXejH^Ny&x>A z1)>k6838VEtnq=ef!`3cM)4DPCnL{{1Iq-_zc^E*Jdmq5&V)$dLg*MI{;-nA8aZnQ zLQG+#OFDy-H@g9wy`yeq4mJ0b|NVW zYX;1aS)+Md5jJ1oFwL%^e?J?%8(v)1(aub+(n_5*--GyjxN zbsz@vp|;i_`2tiM+tM0{AuUni%UEU*W&dNlmS*k9$m4QRme1_+{?D|x9&_Dg??e&J z>Tq_%f4fr8zm)y=6GVHxYr~iMs(rs%)LyH1z2g(-er3*u*Y2*>yFSphq2&X2+>g`? z`k#?gAAOio>CZjwwR%#~g_pna#(QqPslDFyA%1CFuloz+yYJ!;UH#*To? z&-&Zr<(^tS8p8he+Lrp5W@AaGYpnr>`Tpi{utx6aX0-FzVR3BwcuVQvSlM!mu|4xj8^?psBiy^JdO~1H}cw0i=P(kXN zTbydp%UFg7QJ$yUn^93{Q3ERSnwANDfr$w%2uDS23*qP3Sj?CGaZ4@t&$Iw;Ghl+A29HZAheTw-?+V=m@V%yBo-@rBXL$pp>^i6( zgv-An31IL-h&!o`nraT2k#vdGqh@iZmwvQ9&vX*p~b=0x$ya|s3haLU$HwSLRhvDPL@7UO0 z?|LS?diKfx@&(ATGfvoBJ12W19{cnu_k3%?EqEMX$1b0}=uSLlNxD7KUaLnd6)b4c zOXwrD`dDj#^}GPID!v}ZzD?K4)NBgm+L*;ew}FW-3gG|`x#m5nttV2oI3yq66?&NU zP+0a>(%#n4fi|F7?hJ@VbL46x20|AM(d%sAu(h0*2<}b5742mq{@c0 z+oaf7EY(FVprVlJvt?86jKGE-uu8H;D#o#NU~{9-VDMI(8N01E29eQtjhajC0Wc+c zxX<>`a)FN|S{N|_F=W-TSfWYu5-mrO=4aj_tPit{iZ{uscFo7O)Sw0P7=tOv%E;V5 zM~VB|>j`Z`n`sL!=UfgoidbZCNOJf^|4wl_;DzYtg0bCd^9ujN?~R;g?yln?+RxV8j@BCHC31wAKqn29BJ(X=1+Ic zhP@@spTGPu`(bt``w)4a!2~+a{8Rpq&;)!gMiYegPwIu8vi%l9YBGdoG(l+pq+UcNErf<* z2)%gM6CYJ}Z5=SbYu$bKub9{UL?Rd7{_a?|zWuIRm`~8BUJJF($GCDrJv@3WWI!Te zrmaSRGtov%X39xtT08v0W1oqjV8P)2NlPvw@c<)I&BFhC1Ih zxKh6H;u%}sQP*0BmU$=bO=#*ZC#en{<~6YajiQV(TNHJn@;eskzuCO*UyiHmQ0obG z0nN2$WBTZ^DCJXHLikDeA!JlLV?FS* zWZ(gAJ(`)FsuX&&?W~zk`DC`2%=8s@gN$MCX$ucA=k~&*o)#TubS;H_v(!L1wTNu6 zvk%czOL?hNI$I1~?1>PnLmy&9MPo}u$y~Ko74ujGH9U}awP*H-qV=~>z&`X1d#@VO zjx}Zj}JhD>60<)7; z%<>s?tWRv?S_P-^Eoc-`{OP4Uxw3WEF#P|Xw!E)XcB%A0ehOePkZ;i`BP&=LyHgbN zF6~T#)K3|9N|QXbP9~Ob3@y*}RMnx~ywI(Ag^;=NNm&uEm#nT8rcT%-m=`r{yD+&@ z9xH6feLqDz7^s7R2D$}fY{iY)X7B!7!QVurP0Si&X^$4&{U zeF9uE%tP%k4>NDl=~Di1GY>!0v}b-Si{W&gHKJ0?Ptl&aRTT3xv}g9pVtyu;B$e`Q zI&Ef@^0RanAfZ+K5d5J)51~n5B2Eci+6#b5;3ul8mUyDI_yc+DYidKkJhM+0Eto`A zR1;lz<-6*ko=9SPB8kOhbq7ZOmnaL4M`cuz>W?6=H6%_%nE$;^VVbqZv|{41y~oZj z7MsS?N_$bDEz#lu3j}^ejSWzZ7>677Wutt|tM-p7#3H5mfY^SA%Q5MKj%U z%UG;BimDDU?MFa$hBF$(18j;Gyg;#4Tikp;!C-d-C zL6|e_lq~EQBl5WfHv8JoA@s`Gh^b3^vanvwusk?8?Z`vCF2T7YBV;3gFr3+T?-xWE z+!9%Aff@1hu6DAKktML7qlCpkETD;0#qcPUw5u@ljxCAJFjUlms3SMBJ_<`isVKKa zePH{M+6N90DNde#AJb1O#PR~>0jh86mV@nH7#ASr!q;nS&nx&xkH}K=Ucx~yI`pg- z!I=n94M*&yc{W@uiD6n`6K0@Wz#dw3&R|n%H1*^;%QFHu2A9Bme><>cj1>DQFJnEl z6IAN#DcJv|vsl(7u?3IjAn)od#gt*G3Nr-KpGp0R1O@fQ5Q8S-wC@@ERpoNI+@K)W zs8Zh5KAh}6F-J#r=;_)eUKLNPRl2eKN4U$PRfxqPSzt!%xJ9B{Z!b|NJK z@uEchQOk}IF`C-}#r@8wwFG5|4v?W1p<4=Dbyl~Y)D{L%mKq@s(qu%34(w*q*fM7J zgANQYLd)|L!nYeBLFQ17%u}DTJ{h6Y#c)cj;0UuXN}odNPonSu9)6QbM=c=215GX| zHcQ2$+RpHty?24{&t-7VfD;IRWjHFkB@cqV4Ib!*R|{#dgR+5 zKK=(!o>1?4=IYm9{EH_)e0!~qk6jnMeB%cnKjPkZ)w>?)xno9+{~dfh*S(ONGG)rz z_F4^^t7dh`A80h#x#iTm7TFNoZuoTI@gf&wL=*ek>)O$(K33O!OshC=d%ZRme2oiH z)BJS2kSWkzC)UA3K@+naiZ82 zx?7E^nEkB-p)wn6fy{`R5W7y`luq87$Ji2dzL)R1nxVK!;ez*wy*RkN{LEv|Y0?wX@z z9GS?mde9nZFKIsrzU@dwL8J(glSyrg=R4-by_a6th|@R|=VSkFa#-64=lYJNSc4x= z-@Ymk$1!*P__3EI{LhZL@t!-!C61GD$+v2F-sxKX&h!8NH^=?*1Z+d>Vr(BmrMqib z5m#+|-PQcH9S&=bvEoi*K)Y9rwOptbgF! zd*QCWVpHdCA8pY=C%;<&I(uF+s9cAW&K)&ON4hL|oUt2FP|3r;U2BcQ?F@O>@{@ zi;J*5#0LmA4fM4%>X>n_xTEvf^}ER~IL?OklGbA86GN>EpS`F9u>Of{@!-*s__>{` z9Hc5kIYv`b!@pW49opYs(~ddh)HW^ZHXibv6lxfDBbNE@3RNdxAIj6oL&_J!)st_G(GCiU4>z$PMk@e>mS?mfFcepi_r$PvtwQH&r!%uY#CjG5 zpABhCt`<~RC;JeIXvc8vmUgTMxjP5CKK;=kFT}8>1DIIe(+X9}xJnP2k*6h%DXtQW z!M&(Tb?_>ccD45&^oq8Ls|!%&`=QOCyYFkVKpDdXqtB%1ahzD-LEF<_-i5{|UbQqb zCEYj+Rw=0!%@Bc?cVuKlooS&X^Y)|_C;l)pQe77*nrc|fIN85IyTFD~a(Mxc8v-px zcLz5q=??L#pd}T4oc;Os3v55Gxs(4x(_AKtIwC}?nR>`;Brr6uoBOswoP??#IyJ;3 zt-2%XR^!pqT=fNHsB2+1Mo-Lauuo=$erq*Eu*?foQ(ZH&-#(eA=r{VjFZujf^m%XN zGiXdJifh%N^f2v$t=DL^mftyV!`9F<`^S07Tr)CEQR-bW(8tFaBu-35CVDiE^ zJ0r_A6&*nNh1xUYMr!hcJa&S{5dK~J1!1c5*ni!6 zODN37XgI0PxmZoZ28^nNp4s*Vl9ja(`f<&^XkX(PG5WJ+-37bxMrbz20@hnvWlG&%i?vcIw4XZ`!o!2X=WK z&1+ZJ``-1lslUCmz25ax`QWFPo_H_vedvnt%Gry)i^nU-9C)(5R>#E3hSb)gR-*|G zf{ke@hf_mN!t`ksOqio3T~o~Mekq)4#)nT%BI99;8E-9_3s>Se9m`HtCy%(r_;~+^ z8x!3Q$6$=BFSMVL6&G%{9PF_u_V*Ve`sF)kRI z-%0%;BA~vI0q1Pij+VKz4L`9B>Ig}LxOPZR2r|3o5ZS`kNL5J48xTMcrq%Eh&D2W1 z712N&i6aIAbOF$?_%cSwyrZX>+i4j=GAs)cfcoU-#IqJQ%2gAQ&Vb-8P2z@_OH->c zpuuQpgE#U1j&-}k6qGCay!ZOggYP6gShEh|43psx(GQmUVbWqYtR|8%gzE*qx8CNF)!GnwGb<+JuP=@XRID(YhX`KE%bC6Cr^#l zbXNP?#fl%dR@Z^qmeervfaQUj>5O)W2@(3c_0R!x$Ke+HY;N-y{Ssr_8ztRmlPGOZ zly+;JRtx^BU)mFfDYD_zI?PE1MX==_`P4jw9yOE8SSZIJ8$(qDMV~K_4)@o#? z(wZso_XA8c$bkb)G-p>R6B8|$m}n5Td8TuTi6(XHjv+GBgl@5BT7*4hrX^@aW*U~s zG#1H4W?GVEfBR4H3|ZPKImRpFH7X1o;=4z-u^ym@UMWiT%t4qj(o zI{dwD=H*so!Ac{ucFlX57{S@n)^+PjI))%B%a4{ERh*zT?ku^&WNVX~D*6|}79vhzfN7|tY6@pel(rSAQ z3f3sCF@|TbVlw8D<4Sw?H_&ex%M+d z7(me~wWr$wtj(l4g-+X8w2W=-JJHx??X->6@(o~PJsGJ1V4A8P%C03O?jLEdlRHpp zB+`ja@Fxzw|Lh+W=8J4ck(Xg&-_c&9Dx=!E9{$Mg%F4g{)4S?jPjB7-meyORBI~)iOJ_cR%sf0Ex^Cag z_xnxe$*po15CcKK4auBOo^XUkfFCKx71B=@_q+ZXvBzMv&L4wq?vZ z-iVL^1qkP49mW3Lns31-lE!Lv_(Gc&uY)6u0VF(;UhNPEs^(}O1Zz1L(ILKYe7*|G z$7{ixQGV%bLwRy~Vhb-HK#?Y-eSizBMMm2RnhNvG*7eag5VLkW7i^Dpp z_c%3iE@Rj(qzq0})6A^BaMbUmUBEa(NHd%~TpiScK9mN`$l41>*9_p$HQSD|?ZBgF zgTTby$N7NWNo*UyG67n>%F(q($ilE-0hd4O4?hZvaybe?pnyXBjq?H2O$Jx4q*)P`7bXmDeCWy|q6q2@ zOCy(Y$v4rpOe76}0#f>h+O!kM1Id4P{SwAk*Yvm>VkGYFo?0046U2(cuh z3BQRoU~_hqA#MxR(5VpwUTinVx=8{YAXEz8!tIK#T!}g-J8;Co0t8X$tv7-sMuve4E>p3z28T8x6Tph0o~CiV-=Wt=e6Zs8_ev7c~EF9lYy*PA}m zbexC=Fy!ELPA5f3vxg3?#nBZV#U1POqLW#PKu0PIEq8=!J`b;jZhoT{ z;P=Kv#edkkFpfmp63uV6Pg;y((OEJk;MY34KY(d*CSKo-amR){?__i$e1yLd-mXVgJ3_T#6h|FswE;dGHUWPhObe&l zioFo~SSfU7vm`>$zAlf0TGU0&sDb5K5lGtCh1pKMA>T^ckAlM)e&T_U+{Gocc4Qb1 zpCmbjuO)Ng$T0cT_~}Va7J-5oH^;}>nY#Jy3>-_G*vfh|pQdQ}=}A{@1+g)0D`3beF_D4)>KML7m$aFwj%a+@Xce z()e1}h1ucoP_1;!@K88aXHfAWT6Vs9!)TmAYE&{!=k$~_;g{CAPQ77tD9mZ0TRTTa z)M%I)4iD2=s-smO(y2Glr1_)h+VD^~1zO?oPQ)bR_Nev z31^3vLw40RWENuI5Omy@je8Pk6 z{is%>yMukodR25O6m}P(+4!74N8mKdPOjOG^|%SixBgNs%=5o<=nORT(UD-jn5KnU z_j6~y$eBa)(j00wUFy5tYq2)}O9zRyv@-5XU--W7q_?Kx72|}AA0Qig40JMge-`K78l^>U^~}i?MwOw!OH8tt z0@(gIynTNo0rx0O>c*a^8#qSAJ>csLzqOr2vwybFbl20!bCizm(BXU;3<C)Iv! zR6ZW0VCG>YnR&1rQ0v|Fs-rOrW^Sg07zl#V`J59$_i&@n<}v$>UQthQ9Q`cvs(sCT%(B_7lpPZ$@n$&w#Pm6bdI1;hhXRu-^dA7-lxy=o)U; zZme;`Ovf13vdPl9G;@i@t4NOJ*X0_Q5!OGK6RiNhx%&>xAy`L6`m>R~DI_J5wYUc7*N)@6R%GE4_za zC;K2aqG|U6%|D2nILI)@nrPcPj=Hwy{ws}EtCGDw%xs`8Y23X+z1^@gz8V59pyoPB z`3W8qFtf36hMAoYJ2Xopn5A?)M}}s}UbxhlC7!#GhdfKi4KnkYyW*1iwU@+3Ui0Ho z4~v>(Owqz*Z~!hXSI4}qrQQJR5~Q&m!o_WHBE)E;BD9glv3o^miMU^bVek2zS556Ak8XOx8hScC5M#p~4>Y!s z5%~XnnESM2Wg^IN)G5xBvSXyp_A+-N56JsTgG5k;&VjRg^guv?mcd78Q@Cd4%DWsR zA`qI~#M)nRLZfkh6=0TtbFtz-5*=c7#S5-dHQ%RY!JpbX1h z$Q{Fi;H?eT@bm`E1v67#4W2h6N=JBly z@uzEWc}4tJOVd7JYR|hxI(W2XH;35`wUtofRw;8ITY;-*5E8&nYi79B5uW&=(^)m# zRCAx>_8Owx*J7fXBN&>)bzMnQd@kz=h4~dLwWI}){@AOCzK+AeHHj$UG$gfcWfoPGcig&W%7?*m}OYP0cIK464nSEEE#G4lqj^Vvlv zwYuEM#jzubidM`!kxLY9F1I-swog#5_JKxGX}1mq*9e?Dn+N+)m7k+33H6=YVC$v#E&ek-A6DZXBPz)^y|K%2V)B;ZOF`RfXez5Wx2p_e>C$&QiMp!Qh zqE=XB{g>N;Rf>^NI|$X%Ugyc}2x=#jj7MQSE^TBw!&#=@Wdgw zfQVa(SL71`Ca~2hxciM;DId=S-nbTJBUT^JxP1#urcyTP)yFeBnIToor*H)TZNk$` zr*%rFONLp703UEHTy~8%3l|%Sg4h8`nO|S+HY5+kMn`Ej3ocPyMe&Zt**SDvwquy~ z;OzGnz*n>ojFkXu#)lC)_7~{Xf)3BK&qQ0~_CTqmO zzZ;oH5ZQ1bazrjs8Qh5Y6&}*j5G^?WQx50=tg__|i2qU4bB?$b_Z&JDqI1miS2~8} zt{{d?Cx1fOB-tn5#ING!&*O5y@bWwORUCnJpRU)!lsMWeM}{4|_<`YQ8v2j%qVS>S zX0{FJ=Dx~x8csZ&#uRcKUkdkJ3Ma;M?ZW}VaWT_01hjC@E7!t`4cu0Y%LA~tz0F=P zLRW*x0iDGm>LYv*7rK$vgHujuzaLa|t6#&rYA&9>5Rp1CTv&s1#NGC;P&QxGUS}Cr;v^Wc zJ!BVr2v+mX6euULJJsn}xuMwwT7DplnWY>M9#-WQk$Z)#-vwE!REvfU8rj;><&Kfz zNtZjq=%l_)u)w2?S6d5}!$K9U0p24=1K_i@(WrF-+{-DJMD}0q(_ToIH4_fb5xC0* z1bLN5)USDNlSemyWyZ8!?CCl zxgv1>gWMiE2Sp1HXyIw+0__4VR1;cg*_a)=*##A9*#Jv|668XPP(tCpPoM-6C!+*q zj8K9y0v#wL!hg}NVcaz1B2r0dh_yy=OAf@%kYpyo!aMofgbF&Ze}j6%*W_esp7usHP2MF-pfUE4Yia_e9N?=8f>GKZ zdr;hA2jY$;1z=MKuA^q2(wMCOL@(ZqO}Ym`-$`AF$a0ex1nUK?_Jb5T5aa*iTtOJy znIdN(Xaep)yI5_3z39SD;{VS1`-!Agcbey}_Hee3mU!TiHbz%K*~`|TnhFp<-F}Tr zsYF{|5{d%|ZFF`z?zoyGd~9DMHJ}ZrhASKgNvR$Vu5q=89q&-T=ir_@{DY{T(DFTP zxpQd|rYLEJ+UeLEiEcoP0t9gpZ$bf0Oh2!HwlV6eS73aiiSae6X~wkuKkzlY;>)e#{X*8u4V8uj|a_qr4feq%Nqa0Hmp@Vu`SHvLKHg3rb%VI;qu8U zqZ`?Wk;+KK1DOZ^+07Du!w_PR)*HEafQjJYY&W_?a=?uIfgCkObK~!R^4s%!p-jW9 z$MI^&i8F?RfK*8+GIAx(UI^XDXTZ#M35$c7ow69@eY5SR6i#@fLw`ct5GOk{T;ib* z*Ap(9_Fx1e2N_k=SlEIkkvNaZ&!-Xi2;yGpmiUSGrN&(%Gl#D4#R+v`in^GFko0hW zn5VO5T5r)zE0LV1+iL0XwqTCP(2CVWT}9U&^VLsA>MHbTP=;&wtZMfFZPi697A9F8 z%w;M}o9D57l1&0RVg(ptv>!Hwu2|q3W)IizP@2o?KEFtjMYP6r1!fbr*D##N`JP93OBA+9kAa9{v?h6})1QrI)=7`*hStu*GzYG^l z5#zj1XqLf+)dauZ(`&^E%r0lKB$`m*?fti9=Bj_|R(JK=EV z*d}$+jqC{X)>Cnp(9V=OcA~m?f({7XSuof&nhqVky>A%kAl+Z zH7kv7lhO!a*xVU6fE3ZHp^!vUr5Mtc?UDBA#@ZtiCu@&L({7|co|@&PJt7f+J+`^s zEQ^}ON;CveGtus^&`;3Un?dg{$Xb2f{~v?+kxdZyNtaMC+H=rArggr+a%(~t8ioS?zoLdghGwi` zypnRP*E8v7yn4o!B7BpW>izI>hDcx6Pm+FERX`%2Nj5I8FsW%ga z7y=Q|Rg(&yBls*-?&BU^PW8VX3|9YJkcAS7*avtLe&=9Na|G>{i^lBQF>5DF2|2iu z<-#l+F1}HD2TKER_7!f^l;q3i3$p@ohB;!Ic8JY$yDH0Y9MWfb+c>Q+Qra(?S%{Ll ztIZx>CT$;agpq1F&yAo%{W!%uaPyHoIg9K3d}uXPDzn<`v!z^fL{tq*A5asH8^j5V zfmXIbw1jYqJ%tpTJ6zQ3oAe@ZZ>wAXuSVrB?RN4BDmz%j;>Tx9B38yrbR!E5cP?XX z!F(~t8VnXg^F9~~Mfq_tH{C%s~;cLeVT=L$)}TJ86DX<&H`M33DkQ-D9l^1Tp+8nCRfDG*-vnL0;e#dLID#WBC8tL zMLDP_4xkJ2Ie2L2i>J7zjbjx$`uH6jT8Wr1p5;1g_VJ?e%dzKOcId3Z#n*GhbSzs5 z^c0+Z>)MQ=D2DdRR-J*8hZ8CTGp}UQi*#VR2h1I;WM(cZ>%jbsomj?F)aWa3{>tg= zVE%B>pAO9NC9`@_I31qfg#-B+I#{5mmh&@lLTS;cG7eFlBXape(5TA0Ixwk{aTev< zbYS$t{4AoY@o}~=e}vWcWB>JE_p1*9{TN<>i!@9F<3b$z(lV8abdbrnWM|Ei%5gnE z#b;TzuG2UL653O+FpGnOi3<5o?QPUaA&|D1 zK!P=EBnb%htA5|!x87=Ry=Q3aMD%hs$&C0>YPF)G2BkHA6j?`pD>oWdTSSWW^CLUuv3+XVuiOGEeC=6(-DW zuRpb0aLU8gl88kJ8Cj?3ehF{R9rz1N+W||<&=wq4<|mlvV1^oc9!Sa<&jSe3whlJm zI85}hBffwV4mRR?kL)5JkIHt#GkK|xALSYufhS>982dyCkX%{J<%Bn90VcUxfdKs^ zfOQECs90&{T?l47l37-2 z`ZL4c;3j}To4L_#Zqzkko!QpuxR#s1-oD5}4hdbtAx4k*dN?p)!@vN<_*Jop4hVzb zGZ`Ax81A_k%%jbGJB)4%X|jQyVRX^8!4m;*j{92JPp};z88j>sjBE^L@=8v76yLAt{WGs7T0-qvR7YF$iEm?fp(F#`@ zA4}EWWP1lu1*|s>H0Sn?j%i1H$OPywn;Zfc+B#GG56dHS`H=lSA1H6jv!UiSQ>@ch z7QX;91fo&%#|#A);{nxoepi2!yM0fc44AK#;N29Q7r&_nbL^Kzew zKX4E{1JA!vOKcOP2KF(0;7Zf;l+pUW+@5%Hr_8(ow?tv=S3eMWl>;P~XBAwzZ5&*$ z%XnJ{K)kV>V;TFkTgo67zhL0$LRA@FVwWuo}SbbvgQ!OK7P7mML);TLQ9I?WG46Xpq%3XChP zzBn{s8Z2~vI=pW)l-Xjye!(mLSsugXi=$gml3YUEQS%KxX~~AZHQWaLOf~uSW!g__5z@b#uIZzbcUf? z38efcG?kRU&Vhvx^VjJobQC0CddazAZrH&jY0UUuDY?-)D^hN$*2U*a`?z{VXb+5L zU12oatbkQJFN)sc$kYgt}z~U z@LM5X6rnM(k6t`DLlF5FGt?V3aS%J#D$!|KzwJJsfAyr3SRu8+CG!9?gb+iwPP%uRB|RXPE>D+`G8D6%xUFfIzlQ>+it; zqPO{_Iy_BMZs;1IQN>P~Z)!U1F)gRe{dfaL+D@7K$ffOibx8Y&K==7uxnuLD%;_;y zC$fItVEfz;z!0G_(xO=aTlmhaS0l1b`yc9A-(sj2cVW1d&n%M&btu7*lCfA}r`>4I zWdtgm81fW4V!`a*MDxl5-w8KZ_)Z{>@fz=(p32`Q4jOU5>}6ai@)ycQP3{1oSXNJ2mC^ss40b**M%8RTSMadP^c4;GOAFmceEIQPZINc1^uvSOAq-a3mZdIKc_+IJot9pcv zMGHNyaO?qa00EqEW<^k}@|hrbm9c+JueAFM6gFe_m22m3;~Z?~FV|TaL0pWya`u4? zGUG`-Z!vo#7x^=&rcOEfNEovhVVF32aKZ72Ve{-FWhGP$i75CG)CU_dguMTkL>>qj zEZ0Dhhi2_cmgej)1s{T8n3_BMP!J%rg=y-YlW=;Tu zHJF1C2%}t_!zgeV$nwl{a=ON0vd3yfVxCNQr#cI5@XacJ3(|*9(K&#?{Jg^};r_?} ztR0gOOTPH;5MJ|e|qwFAu4TOSW2#{g#}?EY`-Hf$`dCP6O5k z@~90uH34JHBskA{_0-MkEX3O|dvb%M#GdL5WB@&7CdP<5s31ooRS>_Nq)YNK1dMXF zlB6}J6?T@vw89al6@aW@X0QyS#E{x*S|Qh&?J%ttv_tq@HLWayC`JDp(l+a1F%~Yl z$}MM?2>hvszJUWDQZ_~Qu2ambm&ACYwji}Wl3ZL2%9J_N89+m{H`fypuSsa5zb7;CNX(C@i~fc}i_Rw9c(`J2(|xp4p{=wB)K9Uk^eWubj<)b z53bJy!Yt;opf73{g%%#lG`2owZuHyIn+IV47KX-bfzXhX1;h^9 z`!|(X>S&ya!Q>_gX4HkvS>o=pDZ>;jbOw_v!Wnpkg1wXOvdFI`&s=BK+-6A!gMG6Vose ziZ-Lxc~DsTGd9q&jF`w9Chx)t31k9U6)q~M+cK9*XKp2OmY7i$V=yYZ(*VW1rcwVW zrUId$-pJx>eyOw0_(#l)JbirNK3VkNh#KvmkClSgCF`G1ja8ktQ8`YGr7I$(RZ}G` zWmV=^79-C1&!`=piS|mcf|2$F*7!(!MdocW(jL!zRE)I8G9MQsUKgY7Jo8C0;>#jx zk7mA6jI>8GpB5wSPUf4%NV}c+Rx#4fGM^PA?N;X7#YnrE`A#tc-ZN5+fQ^h6Bj7!G zF#_H*R*b;OJzk7}Vk?Rf6lR$yMzCm%|F+uEv3?cbkF~$N|J;aA&MQ~@4}3?=z^g{a zw6!*BG~y&pTTPW}vkJ!1F|av_za8V#{SV%Lzjh-5H4bm97RfNLG0WyPMVEmJWHm={ zmuRbBU$ko&Aw{TT-SA{ANvR8R(WgS&tqZ61Q@Y#??XLmk=S%m|_zK-M_ZqzYw*HyS zt@a<3RqYSlm!LxLYJbgllHM;VHGNpWyf}eifUTP=S*XlXr#(OhRts7Fcm7{~L9(D+XIe7cK zWenuB$VcK7wJ)He?-lKm2=^#qZ*5dv5Ug65Xpp0cgxsDRL*G#Ow1~aM_7eITS2>x? z3l+33d|exfn+A2g53?SzKYaJ?RybHkxz+wJ0}qV+ouC(NnRnbrwkSpvbUkq~DIy8z z)eGsUBN$xNrgAwkKv-6V8(Q=o{a~vdi<|tfaKaG#(%5y?889?h=-#8HRQy8Dg_?LS z^NFAT_~%b?4cFk0Cv*`2eO6b{1j+kWG5?h{sl0NdGerHP7kkv}U+b2w0+3WV=plaE zy5l#R8)-P)tgh4j@|;ScR+WVv`C9GVTMGxzvh^p*afR}%eyh{=AGjf5z3va(kjS%$ zE0u3D*K6#K2n3qwv?vB*{p7&}Vf`aEtAlv;LF5{T1m#pBDkhi>S?8@CwKSQbvJ&hV zv0+gyZ28F8ut-wyk<)_|d|DO+GNDY+9dyGb{(vUZJG25EAZBj|AQ@b;S>1uizX;DZ zM_#;BW9s6fEzQuKX0!UL_7Me87;3AxarW3yC)SlvYgh)g4_oOU{(7nXN64UVGjJJ_qImV+3=N8LY$R3|QqaKjt;lA`^^3Ddghq6_FUL&RqGmy@pRCV#z4j$8Hr8MI-e{{Yj$Hbz5IgvuxE zyz5?GnwcKWug1l_avT?emfmnCXz84-%8kp1wtKiF3i>2q3A?W4CNG~UTVBJ-)NTZg z5db3$7f6R!RC2Tx1sB|dtfwOJaj{BRnx5_OD6A~Wg`nn_3^czY6+D0T2j<09KG`98 z1357Pm@>jWc5v7k{A@_pY7*M5R}r}F$MxB zb+%QEk!L`y9;=A!#{7+clK7iHmAKKa8|%^;g>rWgQm0ihiCbucPhI26;sl(+B?gdS z_R%?*RQ7JfegrlM^Drvn+O4h&h`=tGTO~sT4hN`cyg(Wtur6}Rc8G26glWCIJb}ee zSUO&qy<%!n*G=x4w!6y7>Az;#JT8rqA49?$#iS7lZ*)-!Z}g}V9`mr=kcc-TBHjq_ zGADs9iU3x6V-o+U%}Pz{ViL=%%Y>C_Rh)p22mWOjaEaaNABi0WD9^>i5Ya}r&E(}h zlCNxmKTQffBiDlQE^*z&ZrDMb8VsXw(({zdZ)m z<}K`ggZYKrb^yS|F^&@(rD`c z?uNwcaJ!N$6WU9ooElO@G%;XJE+KBG}}#Xkus6`#zTtukfJFjdumBO zokhtF>{Cajw~YJ{y(K%dqBgoD#O9O&>#I+R!HlCnO!Di@dIk6r>{DjSIKcfp9=QO9 zIm}fQR&YnlPE^fMv{B^G)sPL(&bTIq3?^pU!-f9DREPyrp!k`754`Cy86bm`B6{=X zCtp_5%NOM{F6xP&FvgmnVWy!NLBNqQ(V$Osl#}fkPK1ETBtB@_u_B1amHi(wC(PZ{ z;gEeJ(lT}gJWW5IqF&M9u>rnzL=!yGCL=sPfwNPh7Poa84p>WwSYW_OqlmD%4|ZOt zg*kE!I}iln;N>4`o@XP70O3VRuZWLyow?4TNmZ0kNf}wVYqo4i^iv5(3k(!-@6c~3 zb*Oz{!M{UKaOI&6Ae-v{4z*#zuKzpqaI`}kv@}X7tMEJ=)&Kv{6AoRai_#f~NQlgp z`0r3BZML#@2Pn0M70(Y{rOR*$$qChI{5^D)KGao32f;_%J1}$G*$lGoOeu>a41??R z=v6RhgJza!HPwryRxlX&|iv**8ToD6@X5oMSZ|_alE5^I{dEIj^e}S|9B2D!4uCwdxye=l8 zs6OWP&$;WFcRcRD(vV~j%s>5+?|kd<1D9U%qYutazwMR#F1z#c*M0J1D&I-55pl=< zUe#m6|M-X7?tI->uKeik=}#Yc*SD|w(fhvetNHnm^HmS8z$@=31bqIY5L42hu*~5YDp|saKMUs{WrdS;GWyQ z_*d_poBq=Gzx&PG|Ks1j^;oW7lw??EpM*e!;pNLr7rV}G<=VKpTD?mz&1!^ZI4+{@YJ&?fh^xl|J&(&wqj4I5w2xDy`8a zOr;Q#@VMtmWE|Q4(mTpqKk&5&9#bdO-b*jN?y*Y~N-iH`LcNu<|5?{pgp%Xy9(#-k zMcnP$UM+-rpiy>R@zw_#yIs*=_0NSiSv=5i-CdO5&Q$wuedRMweQ~ugrF>f7^1pT~ z8C=z-KUe$WRu}_BlZkBRC^Btu)}ID>WcsZi_}Zr~#sAMu^Unvqb_mQl_*N_`mRO0$ zf^ao>aPE_de8w)1q<4{l|+DIW3S`P8) z4+Ov~v(>feO@in>Q?{VLSr}Z6{~JyC*hinmES#Hl&u`eQkSW;0ghSvZ)spaFJ7o^w z#71*KhmK%Pw}N>(a;qzPsObu5_bU?%Fjyx{IM)bf*d`Y6J5} zIr#NVPdwjUeWLyFmPCyt_2Wrng2}gFN~zCwvaC* zLEn6|+3E;BOLZ5+Vb60htS6B_X|n?C18k?Ls>cv|06z!kGZuGPaneh4D8nq45PjPI z!X?;+BfkfAxJP9?jo$#5E2?wsai|}QrE^YKjEik})?<`h9YuXCbiiy4cbYleM>A8a zh#h>lenZi&KKtBg&b8v9FK_e9iMdV(dBu0}dis60BmJ3n4?c;jheV9eMV4F<`3*FK zfnZU1G))Q_7bu%XcyUy{n2AM2n|2!kd)K|{-LyrwkbgZ>w&;g5TitkY*o$!qHG#o6 zVB(Jbh8+lhm6ylp7@-=~XQ2VU+DW&$xuXo`ckw2Xl^g8x6OTQ*RZ zw9NemfJ4XFAX-dQ!y)&Jae%1K^xU|F_P1tyj%z4!S2j?SAx2tJv?OagHU9ViN5 zycL4Bwuqw}@1$XQsqY!|iAZ&HUYJ^+QpR5NU8vlxLt1T$WB^1L50U|Jgk%61wUG>9 z;1WTeR|r}jcvw@+M>I!Hz~arZuB+4lmx`Va5^V*Xfw}I1 zOcd*wLlGQ2-?+Q(;4yvIknSO3g}H9`GS}?_7I!7Fz3%O&AVRW94zCHDrs1Y3=SXaf zPL-Bj_aaLsS>%W>JJ?;Z2qc~jc@~k=*}Vai7m^e<;ww$>SG0P%td=W6yxR4kGR^;- zG0e3=8WVCyQGKuBdrx-%cl%XnSPMjJW&CTJ` zjYmH5#&GGtf_^bKHh%{MM8I-8<*!BZH7O}QcP6_U)uNCqp_mlObi*wgW=_RCXn&K! zI9VjdAbU$qnG2BQg+xf#BW|-#2WgO1(qlWMpktRVzW_9^2Ty?0Pln z8%$w5R%u}&R^e#5|Hw)K8<4cHdi)~3_Ih0cLjuzBz075Ka!8=!h_m%i2r8fLO^uzi zdz{n|8iH|9`J0&xk96lP02h~3u8A^%+xS#-K|DQw$-mxaGv<_=X-KnPOTw(zJEE=r z!l-Dt=CpbiG>IYx)7AOR{vzcSNW0CtOMTkxa+*2_T@TQoe>7K^|*7V$_ix~XeEjT{i17H$zphiiy^;1Y2#Clx;o$x%s- zAEKt`BFRw^VmJcIKuC^C7E6vo-I0na-tvz~CdRO|Ya{ zp@tkY4dW+AZV)0g0VsL!k_DMwRjyme^cqfTLwhN;u+gmMG#!c)T1_efWMeIYG@YM6 zpj_15iKu4m!kAD@_taz0l88@4d8@h6&g|}EhAK|K{Z2W-9pd=E)f(V z?SCYtsbc>ly`v+MhuR;;??Ax({j4Mby0jsJ<3%_~OU!#DC4rQDHO{hK;Dfc}DjQrO z(gYvCp_-3&BK6YQ>{T7pM1=vOt+yHZK*VBO<>Me{xe@C0EX-MMNX~K}F~J5gk`)0gF)OEL*4-P03ki=uDF+(GdbynpLL; z*~|5uWvS)IoMnj_Mh_)XIwce~cU+)TLXJRLJ=4XZ0=bpoheM7)u`g=p&}Rs;1O#lv z1&F#5^_WN*id{lWBl2ut?x`WxE(-#u4u2zSjTRaaA|TKr>)J|QybS#@tUn-+MN}2) zBRDH2ZdB9ge^0G`!}8=fJCNg8=Ow%(a>*zZeNOHOlumiZfQ|QG=kpp4Zoh5?a6Sn{ zjIiWd-HP@Q5OD{fT343sMS+M%P`6^lwRgMD(_ohaCd}1>rN|K2ekD(ayW{dw@Wa4U ztKf%i1wV|F3*m^X(|t-0!S>tbWbf88!@(w-!Va-1GzZ)jX$CJY zGc2hlb0#jBh2?~^IKXSC4#@Oag0L0|czIha3sTgwV6hYGRLDyMeEmb?O8`Wm^w@L_ zmkFW*nGf)wz$wr@T1y$&EnJP~1iDKuA*@XS+Ywh1#O_2GQV&)`!ODtUD`WEjC4VH}MT*U0SP|)>e3yJcy`X4h{7T_h=qxbdz^(u&PGUmE zLpKR=&W~l(2S2uI4Bm$HLhOf;)R`E75|dxze2r=Hfq~Ui!%gcv6i?}+8*>Z5DOL?a zkWh9-BFXsB5+b0Fu&;j4Msud{1;h)@Q*j_f@@f|a#epagDgFe-0WC#wh@d#c0Y8OA zL0OARaR8cX*f0TP2uekL#UTzzDYya9uiV3f>R?56ud%no3~ zXwydtD1i;L6w4N>4&kh*35J>hnlMI-h!Ru^D@2LVKvV|?lR+@Ef}E`j%bJP2FMVs2PiShqGQKOlMN#EG0_Nee~>g;`cS21kP(WIX?Ky~gQv7P0ytQ09_2vI{1^gL-P5z&fUd>FMUqiD;a`f>P$voIOK!C|0?IF0ftkigfg zaX112A#|l)11_e0D(WF>4%4A2ztA>i$hw$031mQdh-HkI-*HJLdhm|-yIvaVPS96H zZRO35m4;QmUn47qos};_UlnlGv}-f&9dFS)vxW9yJ=_VKTp)Wy@#-bY^! zls_<2{zny9QS+r%Xk835Fb%H#y>pSRB%Xemezu?o_U#SaJ7Tdn{EAl3>;Pd0ZkPOh zG}hqP1DIDKpdfZF3gyUUgRt^eaJBc7 z@dJ!1P!w)p2C;&*(ovGXTdaXKZWU6fgu5bT%IV8#@y*t19RTAjYGZ<3wlU|3S4SMX zwJN(LHjO#G$Lpd2Z$E29vt9!<9<$Y3XJ<;d@H9N)uNNOhX-pa%dN8u<1$!T6(=wMsVba%etyn1 z2yl>1E0^tdjc&HdQtCr2@LbV^+nOw@RW_a32^|rB(8yvinxxcG=!9}rLMiJKzv8@7 zLFrV^p54*FS%mYMTAT78zdk`gOkrJwKagW`xQ~R8U{}2YI*CPZU@)9dy= zi9;biU~E;hRjQ#a7zIwS1ca8-PnvdkZFWJxg#yYd#F;;FI|?tX-(GgQy)rq6^DopB zLi(D_JJxhyoKnI#C1>`3+~w1)J(zc8KD{@@e;xl+&JQGBhMW7hhViR+D*%im*b1qb zaAR^KJ{h|UV6>sIVm>Dh-+E+s^69r!FGjvF;mTV>eJgUjnDCp`lLV66vXh~doOe6@ z-2n#*q5_s*)d3rBgwaS9&8qUdkfyf!@%SPE9s^dt9D#TDZ-iP8YPN+W;c-CLKLCUt zLenn%)X=u0{l)*FT>AjPG6*o`8aVR=0RJrVU8t6fcMVMm_~qpVgDGJ$mm^P8XI1Rr zeDy=QPK6o*-TpkMfSS)xrP_EuUDz5)%n4D9i9y@AeEQ3O`s%A*_sY-zwRx%oEYnz# zhtR@&x^-bO#wzJVvKnKqs8=oqz;9lR1(v{Qf-!(;j7_&L#7MGIcA=2nD6mq3-kamK zz*yC0mmB+VIYL0UHDJ;G7WGh|_rbEEgdf|EN!xV$0K95(i3JafYFn zbXSQzQuzyJX^UiUCG@=BTL|9B3YAp4erKg&SfgvV{zSjn*%W<6(Rmc+klAOeDSp4< zxLt%W&xzWc*ylDo0pAAW0Aj-GD!hDy}XK^CrC$fEu(*r1q77@de`j`Z`5M&5C{z16yi za!!>}a|=p!k=|8G&CNW6yKk}Y#GhL&XxmSzIe_cYiEMYGJz)2pZm$ie@bvy|Xjiqm z)-vJf>d+2<(YabmO?0B&Yll;6s@8b=iS}x|2sFZhoZ4skO?!u)y4UU#I*}H&m4XHG zprlcoY80uGRFiy*N1iY)xs2~x<{>898#uP6eoWq1fodKDuM4ctb)KAnomSh?v z>Ij*J5_Nu3GR6LP&j!3{uT+IZbt=kk4^Eqqn=_zIm7B8!ZTh)6$C5UK+#I3JP;Smo zN*ky)Arx!GjqWR}E&bj(H{wSA3m_5eQwrt*?zH!qYNx$bcG`Qeeco%h(=L(k$Z?3) zAq6%<5DSGY3J$Ga*$6{}_<5H4Xv{!}WlbAle6T+U0rpyX=goM~E%MGK(s-hE_FA+$ zFnAr5DtZ-luvhe7vsYeo1s-tq(O=^)lD2!%c3*fI1B|(X%!j^K*!4xIXTg`1Q^>)WBDw-4xNsppcv;kM^L5nifjqDAn zF>{*1uB!`MnL7p9LW;@2{D-h2z=;FBL8IssvQrpG##jtQfdJD`fVC~uE_eY3!Nl6X zOh1vK))q!uq8~H+YH`H zD`V}g=)I>wL^tFC2xA_2Zx-HSLnt$|TP%wOvDgBiM}XXD;MS=}!Ys?J zQ;USbmy=eA`{pNKKoYQs0>=b~Xnf6*?!{%jvKFJkF^2bmZ%#eJvt<_0v4Bg}xG#j0 zk3jyYZ)1kr7)QfiR5~LYFWP2p~7L9$1w5B zOqM`yTJ>SuVQ@F-@60y9y(7qnI^I#jDT@Cs^g9 zVcas3rjA=y`_^ouSq<-(xC2s&S-|>)%Vf&5D~|v?O{Hd;1AZhuE+})rD=dL<`0b|V zQ9@l?mEAF7S}0y3)tTunVmSLaO*0?@o@@JFHM@vc4S0TZ2f~*w?m*vwAw(R4(HeC0 zAvxCUl8!Yz7_K#-E5Wx$reWz@qq7V#l6z*$L;^6FB$t*}pwBF#FcU;zFuetgD`7qq z?6lUwTmv=T!Hve1$eJl;56J>E6EebPF;Utsx^h~-YFfVn?c}%Fl@$_~3EnEx0Fg|| zf09Uf`S~+u{ro(jiva#e?2{)CCs+7NDHx?*xD- z#7N^(!UJU)93g(stpCwri|F4&hppsK!Mq3Sas9*#X8rut3!AYzIc?FAPhX70X2ax0 zp(*f#(ieL>>u&>bU>%Myy2!$6q4YUA*1~A*L<$`oZR2cyD{1KwydQ&n0!0U$UsNvd zyT=CP!wwbomLOlF0+tf;HI5B$6iT@W@=xpWSRtr}T`PDhYW0{f6B8R!GH5xZmttRa@NTTH%WrKx-r9mqpJaom;L7u-_-WD#~m@6^NrzP6YN5@J+dFGKiD== zb(F36(IIXhPk)bLM$f}h)37vav1M!7vKe2YftL>_rP{0>0Rp-NK=|=1UA()Dr&|{a z5W`wOf$06|*0UwX8`>a)dwM;boi7{xrpm?BUsX?cm2R3a69|m7e+x?&a=?IsWqW>= zvojD>{7PqMgfSSuw&z($6(x{R_6TBijz|Y2eHrhN6lTmWWPc%21YdF;HV=KGRZx*w zr^#ROrP!JL3WT^2#Lo($pvSV38%idvRoTrp1cp5Wax{`%DBuOzl-WE()q?OiRGpLX z3RViNCIOO)t1*xLyw31tl2K)uMWZV1D5MstDf3w)!IcYZc}b|TWLn*7s;r`gmAvY$ z7+Z;Op1m5c8_37?;L2r{@~Q<_;z^9wxPZ~-2qY9TRZ&7el_IfKAg5%$EiW}L$}U72 zp*u2njd4aH5X*rVRO-vmDBzEJ^ZUY=20k@-Ush_`JTQ|{8gkLXYY(L~ zB+Jq%r?ZbrplziL>Ao_gkFE^qzA}UoVwE~1IXhqq$kM{!#p5G|(I0-{Y{?>ZXt;Qo zA--FbAzBY~r3|66b>?*Tmee#~n#_?y`CCF(i>RwOWMNW&%QemQL|DizHP*i*cVY_&}qw-Q#e!5ro30$Xg)nqa(gmWTcLYRM{vj>7gQpE+Cg- z zDo3c;ce{v%Y}`S%17dh|Tf(So(`U>~8|K>&VZ>(_7B6OYE`xbGSS@1=*X(u<^OXkvER;DgF2if{5)Df4S)dd}qkVS}^2$k0v zO&Y^69Ca*ug*`(R4alaeF`+sUDs*AMs>I)(KkfzIobL}_)8v4rVlr}p$slY3-4OX zTbWt614a@AI43?n2L_|euj`f3{$d1zGYy#|0oP=NrtcUL-!TBS1;&bLjIn@jA*^(Y z@|r_l40xXuSSFg$dNoxxusr>`UcYFSYt)wvv{u9}Vv%r4I*xKY#T^XR_qXd3`S6_Z zPy?u$1qT!2tWj3I=IX)Gu&W0tm~pv))VUBy_88G|PYXK?Gh|2*>Sw{>R47KJz>t2Zwma1; zfXNV0pKx~l_8x#BkcY5g#MZP%; ztIFuCQ-c$#=cDt9TSf#tb`6-AIn2)<$r$_si?fv+^Zd_%HdEorw6nj}q`pud%2ap< z?RUc}75p-lbIJDgDG8lNuGunlDNNvYUze&>q@3Ai;#8HHmeO>iYc+g^hns38#ySi& z6R)y=3X6u~lOFOH>io_0&?y2{L81_+7M3dVr%=Wl)8(gGlm5rBuw)<3bv(z_4u?Nd zj*;^^!OxPL$YffG!~%n5Ymce>=??e)*m;)#CzY@*c z;e|LrHoI&zj3fwITzv~2C)6$Ec?wpBTWG*nQ!vhjG+;GlZXPtaSuznr&NGaLN6Ge# z>M6h|-J$#|u5RN0h1tUVQTNejNb5yhAHM!1B->SXgd1S$sZmb*Q+tpek!BOakg8~Y z!stzz)sW#JUefi55C-DhCR64T)VWLiCGh$PqlZ6!HD8M<+aD*g|C8ucWjYfX8<_{i zcs*q{I1!`av`5FnA5u944D_kY)U}Gr5vdG6E~y;C1Zq@{2uU`)ZHEwnfqAXwF)!je znd2qMJQN)Ohb5C)>97N_k+Z|kBGpOi1sh&9b!rgan?_3l@W?4fw6gXo%&{hfbRYv@>YUq$xDoec*CYL{Cl+K1zy%!xau{sBvd>FB# z7=C}OVOy_G#F+V0E2b&Vi38QUc~NcArPEmXth-*_K~@k3U$S*;YxAL#1*>#9WclgO z|@L!vySu#pk4h&j~#@6^CKfz1GyumKu4LEU9|7MWZPMHn;6L^}PGH3Kp z2p#Q|+1jras3hJoqA(-|Tq+|sLT_IU0SUG@)To#;C=SS+m}+kREM#oGl^vRuUSQ#8KtmWs##U_tMg+(eLanmXi{Ic@~=t6Z2segnF@X()g$^MPe=T zQ!BB!3RC&%D#6PV@$0G!@k1Cq$yyw&i{0;gB{5Wer^%(#Pc-xsG4IrZYXd;?_)(141b?j{B=jSUdEAX(X{FAcV0>Kb_QNc3HtT6O`cv8X{mN2of^`gu! zR7eIgR|n|k&&(8N>oyv@1pgo60tDj?WET8;82La|d@ih2)z?+o8^MKCv51c9aFJ`PSn11U;EmDaa^*1Y>6^9O=2=0Bg^p7{Cf_ImL$1+R|vTW2T+ z4C|dgi_ImeFL?xVKbZA9;HCSB{;B$xQ1}zA8WnV_4;-*+0?@6dz<@B~2rf?;=RX2j z+E2e`x@OG)}KtSDF}7W%s)#a|=tELbSu5{kHIsWeV0s zAJRE~u37It8Ey9fap{vuezO*AhZu5sz3xGKu~`1BiseUObFJ<{G8D^?5Ho#cnJgO1 zpCXn&b;)km_?gD?2S|BXOd{VN1oa2x_-&}722@5?B}>Jv<$(97^1sn*7+XR#c$N7^ zmVK$A6X1~^FieS()S{D|#>^*Dz)8LQv)WnAA_Lo7(dX1HYUqYxde&+k%iJ8UZKYn9VvYK9gCknRtmHyJ#A}cHJ<)`p%}ey>LOp7Hk~f6RZ1X$4HDI!R1A z^Jb&DMcK8HPw?btury-@_6@dsjTRY_$(9Ui){@S;S$z`y)0@>il700SjiS8H z_>+20=bw`}pQ@M(9FwH)Wd;}vQa&JfF-3WbxI`uTWv z=SDNhxP{8WNp~l98dUL`2b!w_EdVCW|2>snZdk-%2czZ~!%z{q_!PAXIkJXHx$!Cb zm%ZMBSRRTxEa_koCkUWQ=8y4}DJnlPfDXfoIkN#X(;*hfzxR5CPUcKEhfJe*;`Zg#!Sg?#T;yGR@Vs90+L_|?QhmVNRRRhDQ$za zuls21RsKfpxtb2JV&8X=jq`Bj9jnMOgiTFh?a@YKtHope`)}srvBQhU=5OHR2~s>M ze3n)a?e)w^q_nXpDva%MvQ$3mZ8VddL3oSWta7gqpNlz#r7`XmL=_$yIYg~VT46V< zF&|1N1V&)}&FIub)v1aRSYZTMdo@nrS7NVoNMC$KqOiciiL=vM6OyEeJvaQZ`2GUb~RJ=J;8}#mckX7lOj|n zX)`3Tu;4p9kvr=TD4Y>%6cU*QoyJQ!?XuZ*DXJiE?|SW;=NB!$RUVI6c9-oe2qV}K zEcD1Vy*a-prBkeH?JR7DnDP+g(GE@wE?pMzAOIs7KRWaFxmiSlrP~Ur2Y{Nz910** zoP{V&@Ca7EL13ibteB$@MU(}kS^Y^$6-&zotEDugWdjUP64yAu=`x!lIv(Hycy-JN z#_~X9;MX6RgMLSA%3XdAwLLKxQ>@E@qq3G{SEO}KJ6%H49$N3p-}x;pwZGGV1;GOG zD;#Fu+zS!jN;8g~^*|(WditKmCcA;9LQY^fs_A{`bUQ^BD)0($@-yrg2tr_-&Zj=d zGLO2>hACrxG3yI?;*BddS(YpoxMn;*WbjUSkXu&4C1L5KHjG&dD>3U}8Egk3fq1nu z(>DR=?!f#1e#<806T{U5*amQC%wmIDFsMtX^Gl_G20{o!;x~R*wP0YV1!FV)S}-=$ zg24@#m4hmc2&*5Ag?gwlWsEC!x{nN|Nj96$7gPG|aculP&iKiAcP*dJ3KC8YkG+Up z^F*u4>`cLGzx@->?bK1pi`exqJ??oO?Xm|F$#(&2bt~jUHK$C9+1SkA#pr6~;!QS{ z@&$zG(jm>t6iqT@h!eUWR+_c zp$hKm9*Pm-R4ka=uE`c4EBIcz6NY=%&O|D3x1h+Ry@R|8LMOJ-d)?v(Dy4Vz#=5;837a2&*IG z*R$lg@g|##D+2O~-8%gjp9}xZaYg@%&M%tQFMkphA-Kf?ngySeId!P5D-_r8?@$(c z$^pkzIZbdUxCoSYJ!VNe$Ha1xi%%95^WVWFs)o;_dHTB$Q&FbwR*!RGkiRiW@Ws| zIt)tU((uk_4Bm+%-!KXcHG7IAKWtW~amLW58E>*vu2CENJ=0hYKN41V_^^6Xym0c{U-kl4&zn#=DWq2Ed*@yO>Id)VMvnn8e5mM|KL7-q3RiGSF3a zsWy6#1{lO?u-BtQehQNZ3EKYja86IFr?9VjTJJ`8@d%W%#ViZ6Z8sl?Pyh(^r%AGk z0GB(>qTIN9*AjQ{8oryTFL*S2wJn>`6~G!54Fg5PLeaqM(w=aayhks#U(LK4bTPw+ zNJK}u0Pe!s{L@vj=1A~ttdIm1^M=w;P!$;O3C+O4vI%Ab390y1Am;nVb#o3|&Oj&n zaeqP312ERUchDs}^03IJEo1cL32L-EKDHuXx#sxQ$E}(e>9m_^BZ(W3G{1~`4JeN7 z3MPk0@{PY|WS9Ssn3Uslin;PKtUL=p0AYizgzZ zqKTV%2h1)=yJ9MkrY?g&P>`ss97Z{+e%Xba6t*l>-2{(dWeB#Kg25;vcabiz;08a2 z{&yJn;quqCIQezRq#Zn!)o4%w#jgY8CYk{rj;+yAApsy-jNi(!tfOyqCZvqjS--kufe ze=?i2O#Lk{<4uBx+C({z^ZmZVN$(V`@Qxp_?urb=%jAGSd3YeswmQIHyYL4I(zRjF zQc}{mDm*UjsJR08EM!r$+HQn71I^OU?+YF01mvcc`h$PUlbDN11|9-BKWaw@8EbE) zdTGS+2&EGy@Ah$k!do~Eh6WwkFdTJqDSY7O8UG^0y=nhi zJiEOz^KS#ZqV{jYG1m|IfI)#I+ITc_8t>;Av2)d|N+Ef8OJkjKVwD&y4=MX*{Ow9e z`XM|XlM<`2tuk)E_>G_{vlSMhVBf&vjoDEcHex)1e^1`gG5%8Qqltf;-qDG~rlaN2 z{zDq2NP%&Ov}JG{j2a0I$UehbK435s*)A1eV_I1x8w~AHE*piFWT{34u?0;$Ot>46@A~WyXSS0On z%%!jK=NNt-kCRB*!?b#UW2F7R@OqBx|6cuMgU4VDtBFh2e=6=5NUy-o{bNE7uhahY z@O16J&gbx!jX2rq!y(9%&zKppMrrcgP5ryV)1CJB@ct3}JPnuI`1NQgjK;HioA!^{ z-sSVZ#ilLn4(#+Vh818W4p0G_)3B=4F9Hh^ngB-ql43$S6;`&ne`8m;6?~IFQ*`?q{Mo=yuHFQ{j>A{Q__YeZfB;-A zW9`?^jEpO@;GZ^`FgIx@JsZge;<9G_>}vhMRI2^Gc(=Dcu&7@RH2YU}i9!)(SE!X{ zoRu}i71WONGiL2v1n9ksTQlWoRjhcWe?UQzEgt4pPMK>NqPG%ODCYdb zWc5vmr!1+y5~UN1(yfHyVcxh={5{Q$;_8`h)Q$QZRj-_ccL)dfGQ*h3*z8vKFsI$9zpn{5SJ|B@ALi9rl#W@Qk*4eb6M)$;I%V!_maF``^;sQ_ zF70Eyf=3DOBO0%Qs|+j9b$iTgrY^8vDM8!ldR44~Of+Je@W$z4RVb~;8G@J0j7%tP zk_G=_&Onsh0-k{z^%u41q{A8X@S;{}DlZ*`r|+s655!()@W|@tmas9N@Ati{0m*=8 z{FY*sGk(jCPK1r_=uaO!8#cr6Z2a7^oqvX(jTU|jg6y|J(z+YXr{Ly6h_%MFUZ-x><M0slg3&2+8h6SBkgbrkZ;E7B z(&qz7_Rl$nMZ!tWeI3jWEm0N`jf}tL{cOZCg5Z38nlOtQdnN2piN9M7-w?J~uofrx>1MsfF#$GL!i~@I(vU}lU?mbnE(QI3YWN^DaY0-H;RPu zhl5Ko23Q#V|NYIOCA)U-nS04g zU-Hs<{W5)_ez|^yex-hu{sVoH{zLt0{YUyW`nCFx_3QL$kGojEUcW)VQD36pq~EOH zqW9{(`cnN?{Wg7>{uBMD`t5q3eusXiewTi?evkgMq5m$|SLi?2f1%&2uhjeXRr-DU zYJIi7M!#QwK%f3W{Lf$N|EWKuuhrM-59^QU>-9@t@{*^0RR5KJ+70@E{+RwaetbgT zsN1K1Qs1QiTEFxqH|xLAf2;pae@Z{=S%0tpLI0z^MgNojwEndIjQ*_toc?G1d3~$? zg8riZlD}t`F)v^vCp_`m6dbeK%WuP5+C&M}J*^L;tJ(rv5kmE&cEMUj4uH zxAk}U-+lT%{ayV%{e6AE{tx{F{X_kLKBRx759@jJGIODMxp{?orHS8YUS;tAe_&o^ zE;4^;UTt1&{>Z$>yw<$Y{IPkRx!AnkyurK?$W2q`9^H8Z5GV9z^%>oML;vLKankVQ z>!n!EdNnSnZip%Ys#rW_)&dD1;AYS0=ruI*qpS{NPCMYwu7OaL^@`P06Cen*w@XJIS(ktQ|1M5s~Z0q<@sO!CX15WkS)FLZ8Ybh z&GV7Vavs9$KSRtayO?JQMFZe>R~u1>Ih``U!78kLQ&M}=TGzy@@;6zrUP}I5|4m7F z6A_Os;+j%tT?kiLOZ7ILGS8#K9N!}tI7-Y@Dugk$dPV3$8}b7_wpr`KSCnfz>$1Ve z!qw96LX%w}9%t^FE`zMsM953(h?EIm{8u0dizTYG>_`>ke8k&6t6U4^LzeY6k$wAb zYazy|=|3xPBS=_NXbX_MBS!I2E(LW5Z(4K^g|zY}k`@$B@h0mZ#m~ZnB>;?g$8-ML z4>V((eZGuOMrzQ&w{Rlew7iLQ({d-$(p`~GV9b3RqOsZPwX^4!DQ5d`Fz|`~l*_bB zcW%U=c*;D}SzzrIO``OE z`dB<=ZWXk&kjrZPUuapoW)6shQ|2NpDe-KEm@>$DYp4V^7~Be78b4N-mIzpxe^LGx zi?YnjgylOpp9L}JRv0tmDRX|A1!kg(94FEuwOfHH+h)+I!q2o!Ts~!Tm-qvN)U;TQ@ ztih4p5o7fVzE|)?;R_~f!xQ=#^ex}U(?3yrI>G3-Xvnh@7`-N{U^QVPO!+w9=A;>s z5F*r|#iVczzjelq%db=W=PuDu?@g~vrY-J^bS8 zrw|WW8H`$La2aDDrx0|+-D4`lASz@%_En=rwnaldSQIV*fwJ$~(m`_0rxL<{c3ZHqGVJ?HS z)#j~0i_dD7Qg5)Soig_rzskS!vIMId?|48vGnT3{^)N5^;9N~yuaq=nDx7N zBk>Q8UBvzF0|*>`*j@oyaymV)&66K14gH4v(ZHD*Ya9x29}rhcADe4x)d^D?=oeB zF(Hdh44FeFOpO2aCy9UC8xm)`wfWJ&{Ag73BUv~<8jH*iZ~)AY#*a5YWF^%FJJ5ef z%)xHbA5D?#s!A3jd>G+BX8L9G^Yb|tRr@uNlM$Qfqh9_Q)BYnK17ur{U(I8HTFLPr z;+U(Ar(EPu!nz}AB0quO`Dac0A~uUTCwrUKYAgc8K3<99A>fysLVc9c_8(LO39iw8 z6^|xn#2)-g4l5oI25TFJ_Z6Wd6X-}f!-z?*^3R#}%h~17r708TpEJCQb#|fQgJ7^i zj`M{>uYFl~Ex^I25bHi=)K2tg9v2v`ig$6{Sx*dNyqbM>V+b6ZaAP2G?qYxFJ+T`@ zEbTrm_xp5)HuP_QKlX9{&#-G!CJeVmZp;G|wz{>t&-!^{#U4gJ0%Po%OD(-H9ucS> zReh{~^Tn}G@_zIj7ri%j&tx-^{0XMDx7@S0M z+BN827Pm(X2ejPtW!{6O&pSTFJ9u*Fj%aYlSBCHSJn!Jip*!s0j@yUtkY4fR&>d!Q z$HCz{zQA|z1VwQe4>1A+IHZyp{F&;RzUcsjagKlG{ zsxU_xQbfXL&5HsVjZj^wpMwL*e@qp`+r&_o#y8J~(HjHQ@XZ;B#z5m{9e0N}OI3*5 ze;70B(9F4v-TZ5On9JCkBExkIP{XGr1uID$JFJUyyLFnJ2(jjUw&70hfEvZarf{Y zpB;Y3-PIknY4f$}4w=mI9WwEY3|B#2LbLx%)v`&d-iYZE@ZpOLSF@tYfg@(BL(z}H zl!;{co$*b29dt25GfRZfhbE`eU>XfiqN?rCWRe@<%)?x?XJZpKG-H>nXgUV2%=__uu?KC51GZB>qGP^oAB`jmM7(}`i0ns8WL7N16+L*-bs=+ z_S&@(Qm?mn3ur95SFCVtzfXEwh<(e#fhJeWHn*Mr0_8%*ReZGCu|Kf#{m$&)wQE}M zDjsR(T#H#2j|^b%a4}>hs|w2&K&~FUrt9S2G~giE0-R;F2vr~iq8ZALVV;!fU?|Rt zw)!t4NHFzZQro*l0~sbHIUw_!q!yC}8YPShGHSw?FjHODa~8nxfqDRyXC{bTW6l}h zSikh5>;Dos1W&IV?GSWQl2bssp5chW#)K+6OktvQy#kz`k`@&lbmp z?q_hSl<9iaaP*$hV6xgP`93&i!uL(M)UD>19uEi3L`S=ENbZm(9{8WJ8^>PX+SW;3 zt4bekp3^aKx=-}`dwlha9%>;6BTbsFRZIEC(J(#^UIedwt4fk?p3?!TTMO`U*e(W_ z)x9!v9SNQ5Tshs|Gu_54sVx3;4tuR@Rnvt*(g8xH0Th0e1s#-me&qDa^s_sx8r<(f z1MxtQbgfX|`I*>*TCU+9nZHkG+}ug~qLaB?RMv*;klZcUKqD9~K))6yN5h$pb}cuu zStYLJR`}?ABc@e`z3Zl|pEvFM>JOquIP^efJt0pAZNI~|vA>E4 zQ&8Y;g^u+gcYT{$4Lfc(xY&PJxiP1uGhpP_StJRkJGZD!L>8Q&0Xb{m3%XHlr`d~LuIQG3 z$+Z7nuGh3+`3d7&ew%53OU9_WyCOJ7@XMzCZ~dnS?HYSAFMh?ezln=f`pmY@ziQ;9 zoyflfh<2Q1`^{=~`(HWjP>fmBscL{0)(A1Z5A)Kb4NEd5D8}Bx+{}6J#BoNv6(goXdTHBr*E?pon z&|1eJ=74L?-5TwN_dT|QNwwPdQrB1r+1_}l`(9(P?`7?8)YpMm`R%6t^>C`RfU43sx#3!k2kMt6ia_!LJ6t0PJdxAO?D&GuLL6DIohm#a419B$*$gc!5h zUkcayefVP4m=T&^sE&B+t>hNf-rN3spr7k=jfydOvZjA#^C63&d`RF{!k|94U=+gB zgkgb4eKs_>QOSF}gCVdQwExZB+ko3~mG`|fYi7^>n7xli@^Lsw0@fZBvsHAkl89qn z*|*hFEr0OoQYdbkwA@>6FWE<-mg7rlBU^{yfJ69jfPyK8IDudi?9_ZTCAlWp!2u^g zVp^K(CQfl1L!96wCeYLr3-|f`-!*&BIXbdM?B4ru?MKqyduG)xXzcRnx#cAO1UiV*?anL^Aw>evn)3E`97UjK6FB zm>|65EyDwPuN(f3PT8|6f0|z%7ftWTgp`2sdALiRANPD7_H*I5e0mb@O;+pfE=3{| zh21;PO7DsVn>yi-eum`e8$=v{22`L=5!^JTK+p6uF$)4W2a-K{I)zGzjG|9d(T<&P?0h%un&Jf_mE9g}uQfRF zUE+*c_3N2fLlDcbUZ$6?P_Jbsvuwx2^26f+;vlVlJHx-A0w#MD`b?|OW@G0|04UG2 z1C<|GrJc@hcV}9kb)$5Uvkv)dZk!r%CZI`S`o&w{xF`5mP-EPf!0z-m&XXF@4d=nL zg8RGlnM^0T^GRONtjqGO_AL^6PopV!H-nbA=lO2fz2YviAw8eeR52G&=dcf#i41JE ze+g{z1VIP;ta>d0WCt^@5rGDGkwlrrhj=1MnbMjxZOW`?cMuqc-c&E9jb%`Ub|&uH z7{&vK$lH_0S1%Ibt>A`|L6ki&2IVNrm7v4TP&P1{ph4?~dz0y|Wcc~{%Eb|GTgJU^ z_&K!8a8tTzW;X76cJcS8erxC-NEG^GJ^QWUhICr}XSZEn*>XOsh~+HU@Y!ZV>DUF$ zE&q!uv;4M>`qj;k{uW0?iFbH(C(%Y%{gE5q8C$GjbcmkN<}B%!`|^W+E4%djn`<;( zI&u3|mleUh&&{@9rqP!*;A*|MT6Z<(7CqXBO@~&?OBXds9v=B;NAaawOsDxuOfm_t zPcN`MrUS|MU`?{I{+7xoIs?P90X1{AIpH(hcoxU|bnHfWy<`u3vw65D(Ll|2e{G*l zGzyZ2C>7QTgxQX(QL4VI?%bgU?~e`Ij_gmP<*0bmm6>|XxUruYXI~UYEAFCr6Lp4} z@hO`b0+_=L&XYF`;=@m=LpBraF0FU7hK*z$H}3Uy@7rr8G)RG6S}a`)d-mA>F5_-T z>WsLI3wN-)^12@#gSH+S7Q_2@Ao>-Q99oc3q!$O0GkBxA*Qo(bs(?kXIV`S5WYh3= zJ@g*bC~eS4M~$#NmOYC&6-1nx9c@R9sm~+0+NHWiKd)U%n<$fa;ES*WSXmKvf>J0D zAQ*7zx#^NzlUQ$lP`wsV&e`eGVH{Xo*+u!SkXPom@ZAPf0jK`N0lXGJk22>Yya;fN z%pJpDP_#yO)7M1IenU`gxl6z@{KF4yMUbayq@;gp9p(g)IO z>nq5CwOo=gc^>YqIA0CtTLasdtGqBhs|VBIrgW_cBb=gzst(&C8-#4CXt>Wbza<^JVKPMxt*l$U>A0&m z5Du>5K9QX&r3`-=>kpmE_AnIMu4;cw)G;klqx<>hCgb=zD=QU6ND${NhhWxa% zx-r{s=6X~F3c2e{D4`hSX5UdPBxkJh;jimsM4}BpG2f#E&O}UTORv7-AQNwFciWNSD}cr;}yVhA_Y z5?5^llF}vT2=ixk+-=3n3@YbtYFsNU^v3Nw_pHcwZj-<|wpFYJOw)%SA#%iAEr++F zQB7QO==*Tf$Q8#3)$+ITV2l(h04Ht=?~%5&IVX+|B?pqdniw+T=n-5>^`-!69sHOr zF7js1O`pn3L09Wc+yg0ISLq3aT*gv?L^&oroAV|wi*9o_Ol-@p`o`Kt7@C3ft z?>pg5R`Xs@ z5!tr1Edf9|aMm+J!>5TTCF62uc{Xh@yX{(Y&w5kN>ns;+FQG_f>a*&8#~zsARGSHt zwMx1jD53a>^Df5S#K!R>N*I6`pm~JW$Y~>*Y3JHZx}msYT4RIhrc~lZHRQcxQi;@V z_I)Scjh38!Kl=E5&&cwk-#YVZ6LHn^J@$dM+Y)U8-i2k^s-jei=hZN#d)wmUjidy~ zIevDs9KQ8PcJ$n2*=MvUuXjRNhs*|dl@33wciKc+Zei8NkzmqR zUv*;4iVPy#`k}Wf5+Wdt!=Ia-68&~J=!f-t)J$yK z=GWX%3%mLYH~iVy2%C)r0~7AS#=1M{^!Ms?LFwM3j~dbI^)XUo4&SYB5iU6VSIsvW zweM=q)FWRPkEu6|!8@BDI^pfDKpBT%)RFJg4mI{Jfthh!?)RKH2jw~0M^;JZuoufhNXKy6C2;ZD#}>x_tl?3V<*Lcb&NeM z_18R}5k82gX>0hE*LCo8b;8en2sGuv)#!7K^TSjOB)6FWS{Dv%j#ko18>=(l4L%z? zw$1~B;n30qzHV<@7;kMj8j6``ASBf{-0)hB?z*|f2z_Op2wGgN{@4xQqEqi!FTkRk zf@}V(Ue}v7V48FKl{sf)m7z+;t?EtjwsEI9sg*azVUGuFH8tlx{>lx1I-VVo zyp1SpKk8q(;Tv?%Mi>4meQUt4=FzJD(haZC?`!F5e1@5OWbUfj;F96l|C^ru^}1HB z9eGq{P8~gl$-4R3uhZFk>_)R!oy4|&Qb+H?t0TozVqx8#XQy6mk59{g>4vY>h0SCB zguczApv>vl#NXPANItC|(le}aa3%osV?I328eQMWRnqU+#Ij3t0h--i3920e?$OF8^nw_Qsa7w`O5 zwuCq?|H2f2@NPq>lPXYs)2Z~M@>jHnNVUh=ZUXY3Gx`+OX4jFzi?Q@{^=-}`q?c+4 z=mC&vjn_5mL%1Gi?=|WKf)#e3?$eV{t=2PORhR%5n$&0{!`0^k)#<1^HF}tV4UR?Dz#h6+^_~Af&4yt$q<^rE%8} zcW1qFe3tKr>jh6R)(ww}dY1V1{aN1K^im#a*hFj|cx~Z_N>tHqX;I!hWET0w_W&cn z2(lq4XbJlc2c(*_&=MuiEaTK{8Zu#!kGnGfCcZbdLC#s}7yO_)LJVtQ*dj6VH?8(iNadp= zDpNqn)f++Gc=B8DMR@gvJn^`NvDk5WOanlQHm7sq@1WjFnSH2Hfl!8renzbS$n57QX; zyy=j8U1L9MOgkvSYzra!s)`kNKEDFW0?-B)!1-s~h?@~1o|hzb;d&#qHJXC>oV^I& zyPeDFsf6i9Ls;kdBCqqInC);?*TTMol#z+cmBKc%sM%SLwso4!BmMwTkoO{@Rzwhv!z+kFkIlIBk|CU`R>hO$>9 zl)i>Gto;iWh_p7*{L5jKhqXIMKFPgPH2R;>=Kd(nc1Zz>TOS?Jq9(#DNhza8U=wlu z5nG7${#aa6AxOu@VM_nT$Klk*SETm>byvQ3(4v+q6H1hRs`Hghz`8%fnTx;f7c6jtk_1kCFZRzlp9BX;( zY(&1T*Z!mN7#6%Hn{w-LXwL9*^JVzNKfi??8&94*rdGU^tweLI;-A^GN>Z1;AmHtk zeaAnu!#>mMGDri@T_E3>&xLAZ&18BkF&$4l!XU_?#f;tU^7 z;psH~k2$@W8F)|}q+jaJ?7{zc&KZyt6h4(qW1=bJ9dWC(^^+<2Oa(5at=^nkn4=&+ zAkKhz*cXJ47;TGW{Nr+Sz7WOPPwA|txW_4Wp3Lr$MISnk&|9@V{eR<2+%DowW~wur z=CcSRF-ne?alYIn|419Ki4WVH=hw8{+F`FYH4UKB@_{gkrkQL@t$ca(XvnJG#f|y! z=pV-AqKFE(VrJeywBE}Z(F7>4j@Z)Rq+%}ZGH`<|o4Ok_Q$lti`C@tseS*>s6D#Pj zo5+{5@UmQ8Kw`GztL6IWzKlidE&Nny*MX_wn_udLciuOXEjy~UEc&N-8gR?Qe`tUn z205hU5dk13QIqKM2QgE23hc3!9^An9^xX7g&`B_AhW9AcBlDX0t!JknQ*7Agx&>f~ zsT+Hp0(qROGJ&mO?u46DF$5?I(e<71Y{l*rmtdp`s?K_;@YEaB;n|Y6O%X0J69WoVZToRq*fcI{%wAh&_U|SheaF6y?F1^N)x4O0p}W; zWhbnG9?9SoK*cccvV(w#4Qnu9AzNdE*cjklUt-I3^xNDq(LguG8R1tw9YRtt8fTGU z1)nY8>HCE7`^v_u6~9a_ig{)Q`Z^c}seVPc>DEs8`LBz)V4 zOxHKrtRTkXY$Th}_@OvA@s+gaw<+lg;uy^AYBXdi>xW1GK(3VyC1br2UhSTU+N8teS=Waj+^i2jxXHUfCD`yTesJBp z_`xUsUBDYNQ~We;r|~NOero{UgSRL;5!r4u2#gr7Ra?Jg8rw!UKd#t)*ED*zu#U$o z(w@+gaX*>aJ=DU+?J=4^Ow~)9ZSyzM;idCVsgb4wGXMehN;E?(QvOn|Fb!wDu(UaK z()oXn^G{OXt*w{HoKZ8Vcn`|c_+q=D=`m~Y@0x*w^HgrzI18hp^wpU;==1Q627Qgk zmsVv`86++w)je(=e0ve&@Qsx8ju*}2?GL!&i|B#+@^|>uxLz;eOSD88a9e|MU`@Sv zoeawt+7oEeS!^vr^LR)*gbAF63^8?LKlj^l5NDVzO-O3$)xz*>a2Pn z9YiM>&~36z3^4)lOqCAG+^9GceJ5D=3OysV;(O7Bi6tJn#@ zbfV*X#+Shvyf^uh`|x=&gaW>cZao9_UWDTfd>h=LbV~qLq8A(Zp-}GzXiCsL?{k(6 zMe$5AcqQW@jSt5Q<|Ot-lTrjCU3#eqjBscdVyWqe`#;eM-}(d+AiZEmfN#%;j0Ou=EOhh%*3J{S|;RhnB5n>LyC!v{fLJjjW48y>hh}UxZ8ORHl&YEwKRw@NX;t+`*vUp{7^j%sdrb@ z1@kHc#HM-WY+hx|D-^p7Lc8Mf6=?17aW01R@&n1DUYwD}30R@pF~CO=8a;duFv8sY z`{Zg=)9NGK*oDtLlIx`Ex6E_IvdWCWO|<<~64JwE05LNWwE@J;!hLca`l12uW>OQO z{>cDhLf9q%1ozF%^^p;1hMtR!R!s=M+o1=0ysK`^{UBUNZVaI^`X0m*ryZpF6wDlZ zann9OLM+%@U!M)1XQCqV@9@iW=f5bXACl^tZ1^1ebq)<8+|PlF?Z6%b;b?od0~c*_ zMamTyZgPbVTwuX2$$2aa%pV$5w1EzLw>p(|)ChsDSw!Q})01!uaM>p7rq94XM9s4f zyTC({j}QDI_9lCqV2I7)SFoWUvtC&Lq|q6ZfUSQ3$VM;d2bT4sy>bD!ys)sykH|N! zH&jsGT}gHjL9Dn5~x~4a-CiQahtm>XCwB+F1{@r_yJ7WQHhxrbk^UrO)&* z4)@x`+~a%U%b)D9iR$NlmUVama*hY`r7|+`+q1KqveXNoOQxC;sDff#ywsYzL_7?W z>56-q?}dG2O?D$4es|(})vsp~Bwed~W)jkz(paTNgE2q2WGr<;2@K2(gAGul*%O_) z?JP+>v_^)lMmmz6kOC4uV-I`|vXjJ_iL0#TUbruT%dja?thIVI!PG(Z>#go^iLZw| zi90BO>uL8)%t)E9T0QHYvS&kFL7NOnhi(s>tk2UKkni-i!_9BYO{;pSB+xEX(yGmv zDQ&Nk9fq`@^|?kblm+ZqJB|6-pCR>0^$AZGN~{uZXrHRNAGHp$EKD`$S_y9%?BhHTx= zSkCh$XDf~H%orY%f?cA=X0c=&0)iz{joo_pT(ISt4h+WX6@)SX!bxWAlSidc%@(6C-=Z?Z|n zNUh*Re@_3_=Y$#k8(-F9?tW~>&8mPm2gzC;G9b4S@0{I`6lgnCFejoa$4AZs-b{<; zjaoF4%O!>WeMV@V0j_={XM@|1x+eii*|a&LXw0xN-B#Q#8+5e7YyOl$7mFHj00m>K zds60B^?4%Jx1`9+!ZmM52f=B#u=m!0!Qyi9>y8n3wDvtBY}Lwd31#Yav=uX z#{6;D${)|##fr026f1VE{BgKzp&)g~S@As;5#x2RY4))Nvh}urRMQsY`5itijPGed?I~{$dZp0e$L zeF+4NDXtXaSNoE8c3A`p9rq))J(lvXDd<^-t3{00XfGvrj`!1pFTyQD0y7AzQT1yX zlt&WRGE+?w!fnf~JK@iNZkF%OWC%~0i)ML42J3|NAPVb&32Q42Yyw+(WC(2Puq5JI zGo6mRe0qKt?JR$$BvZ46zWm7!`bAQ`Hv>y!|Hlwa-}oWrcF@pGOdz@HH-J<4?GU4} zHL)!}@Kz>?JW#D-a zViPaJ>wdNf-+pJ|w}tTqJ9tB#@a}gMzQkDqjifH*ie7MBuM4sIzQ|dT@-i)sB3#t!d% z<N^f3eV_TK97seXJ#pumva{~RYv>JgC>66w z4tHu^kTY9TCga6_L29!6cZN0dXb;Rzls3sPJWnC1ekE!O9Y3soF5|MoPQvtGFk{-8 z(iDUqOM+-=HDseX>Bv>c%;;;NO zE91US^mW^!uX{S8Z{SDx1qObh`gAI9`XE9lWIMA}4SZHXEK)*_z6%fTK_nN4p5&h# zS*Q+|{z-l@emNf(b;IENx6NG}ZkJd35Z|h47$Zx94#gdMe#Jq47L`gq4X=q*Dv@=I z%l=kuLk<2-wJ}H8|F3+LC_D#v}%RIu0zPy7P{SX&Rq`GcuJeDBja z;myJi4kW3CAFI;$x`$?zPW6eV~T^^(3if95+9J>P6(=dK3) zHVxRe-zdA(;^>4&m{5CGF`;<&ZtK}o*g5P;CGf2Ozn#&4v4(gk1m$?)rS@=>>!GlG zUBtIYBlK64Fr{M6`VfM#OEA*B)$y7_UX^R9C=4;W(4wv*`F>^x z+Q80iT+en%{S1GWFK(*Oy7KSm2g7Z*^p#l?K5nP(zGRg8-TtgA{WB=Zo|ihqe4s_R zx49?o1(7HEVr~|21NiEc4i~ObpM3A){2-svJCZV7eM_G|0zBA>z89|756>TEeA0;+ zV0-7cE9(LcJz@ZiuqDWL+B7XYA%OITCt10R3G{*q(6D;;Y1^g{9!_GT(z0HK6Wez; zCJ`IC0du-3v1(K^&}W}k5;6M!JdHRyIeIgOwybe-umognC`e=lm0R~@*#>{W^8Qe6 zb-vtBnt8i?Avo;5Q-`KG_C&*eKO-+v3k~3#5Ol~7BP2|2v>$hP z^!@V`Q7-s?E>M!#_wflObGSDKDMz#Qvp9#NRP#V(Df*zOB9}9#no&J%7N{&Y1S-pU zpt2zLQ;N`S(?I3G%FR@MfZ(17Duqa&1u6?1X#)b4iWuS9SQn_&EoOnra@hiv@z)rr zjQ#*@J-YKG^BqO|mr@*{VMh(H z>>4QFD66T+Itxd(O;PhRi-fcg7H?p&&+wMJI@-s95{t>P?n>3P(57q5+Ijm5ZN}!P z5bci7yE?49v0~7Iq!%mDLW-!SvY`sIuo1gp|C;uv;lp45!>|4GBg9KhHHbF!K#OGo zQ9f~#%R+Vq4(?pug>Kk`^Sm4OPRnsOeDI@v1(`%$0tab5ELGxc)X1f9$D84e zi{Y+0F3FU^pUQ^M;E=-mMN+1RFT*d;<{^75!cCOAoM^l5}gHHNQ#U$jxxl54=-7Pm#iO-!;-Gi zvDi0jI3nO;#Eh6M>wD%U!&~slOrC$1IX3hx9;dPa(Ghp58&C>I5W5*mm>g!^UC+yQ|y zV&oWCw5_S(WVVztz#N5NsBR#ZVjtgUA2H2-(ZVRwf7eK+x&2q55Foj(p%WZbpT}bc zN9jF(-th(Qm_+--*s*4wNq$;np2N<0;NgyV0C_2$u$>q|NN^f*@e5-7s2^@iFP6Fy)eKQi!=%*wP;<)Bu!i#*%6vqSsN{%}t3m}_ za^IL&2BSzAzCIKB8s{TKnpWH^d>048%k3YMIe+F$g%Qz#2I$IRCy-zl4`7qIRjk0o z$3HOfkuGHoVEw%3MjAM{%JseQ^}p=sz3b($pHsS9mq3fSU{2FdzKcuta!Uh-8VO>o zcIKy?2Yx6s!>GU_)VD=z19#R(e~&fWSi*QPX{0cV0NNmI~ z4Dpg?<84M4*_32!NJrNpnkvbTo-j?so3;su!s9kI*}ylcCR?=;%H-xXw`M@0D%RX` zm|3IXR_jX75mGFTPWYqWnK{0B89^)euAJo5_Vc@Q=i3 zTQ7{+vcqkOMYN4$md#0GCG*1-_iC`=EA9%Pg?HZDS#g*7PI%i_KyH(C{gLP&99e0a zu*FJh3Xn8p57EUh_gVOf_Y?punu|#a|47DscFKiS`r{i@^Lm8aK_$L11#j~S_~EjN zdt)lRs;mBsNIKOw5KZNZd$G^L-EUMtqiLS3boI@<=IM0BHQ*%572zeqdw;jHVp7E( z2&wWBMPr!OX7cs;oYzGY3O9MnHl#)26W! zUv^=E^g>q<5>yF+iQA*Ho_@+G3f;y2Cdc%W!J~k+mj=@M=DT@7EOWk_-i4 z7H$7-aXS88LCVH{WD&&7jy^~k zzcchHzBBqGc9cQLs^5VbnFy1j#VV~8wy?)m?1}d#N7;b4g{NwZOl`Sp&E>Y2piKVO zy<{bzn`BXBK4WfbfPKiR8|lOhutQtxfWd(ImD!8l zHVB{iLgcdf{QcPS;<8~`h^<>nuiO|zKDUY;K(G($WLAlV%zh#tSOoGH6era}Dtuv; zSgC@ zJ>6n-#`ux1&|Rp3;6nfC;SrU4dO&yv;lpn!!q2`}Nb{pREN=YgU*)A(+*q8iv3T9Q zK9b7V*}#||Kw$68SJ(@c@0wOMK#(8>{?h|AD1-uU*=mJa7ntmC`bo?nv%8p|QtPD9 z&5T$Eey-3#rqPVtI)yH46gmtrSRPE3tA(mcd{5>Wo&@!w&d3!DN%gx%u98)Esn(HZ z>rOvFWm=+A5@93FU9|px_h}xcfi8E0Gu0i$#((*IC!}gk#d-DOZ0MJh0TGhKSX0sY zDSTXS)BZj29lwn-8sTdyRb7T%ep_{8Hhzkox2IlYT$fhd9%jgLvNNn%nnf92Pj#!M z>Vw&Kn>tUDsIlk_^(h685CL#u1u}@bs64c)sbohL+owZIK07N{eNfrBTNN8V7jXISM7P_b&T?~*1i6<1_*=aqdy07L-{bUTIC*5uyO*Ohb zB;y6m;00d+7g(Ui$1V-tw0>$ca+LZdPI7j-uO1Vbzzkm?C++C?dsz#4U7)TrHOMvo zUbqD%cDjBX4OyloR|t%8yW-h)1>7jUQe#|J@UVp+Ym(KP&AN=mpl770rt{m$Ht*j4 zoK*&BEHpmy=Figx^#^{;8oe4WDA`QVBZ0O{l^*3kRS3 zT~jiRJ80@DTrABnLWEN<_$S>iuNlxJYMiOR_(;?7D9#zptUXN^IHNA;2gdtQeapIhvet-3V^|&N-#?MQfD=+mCoM#Z=lqele+|}DwgHz%7GCad? z^G^xsR8z8q%GhZ~2PSzueEg&Rj$QZ^E)=v!wdBBu3LY1EWO|smJwI%jxO`i0;u3E{ zK^!(FZVwZe^VXZVI?oBfdyQZ+L*Ltoee2DGT&HK^_U0xo+(Ksv0Dyt8D;b`p6@I!E ze=Q_9h~m6jtrzCfyqHUKHl2x+H%T62QonHerSFO`DWP>t5x=d{>>^aSrIZQnqq;mwj^cP z0Xi5&GkRn)-`9OcS{|l8N#2_G&omh7toI?vYR6lApZF8{7I0wpryhAuDl6QlLSXdl z^91JHcVXXD`n~)lA*ntq2cHtcVp@eT#pf}!6Srr@?GdK<>S!WAGOzW@4a;!-DqJR zg0Ncea#C$t>)px+TdjBe*K|8H9;brjxtYhwlIL)xGE!zSe>O&JMr7Kpb%v5V`oB1xv5GyXaUo{vY2?`pr!F>-kD6z8{Ws&Bf|5(@XLc4L}4vG3KKt7 zI1QssQ9g(a2>y0%(C*S_UHaA~7! zmvc9HAvd{nZJ7qTHMRt%1^^n~6D>O%{DHz|a9w_2fn?wRK)Fd|Hc_e^u>(wM> zhPNXD8EO*etyhzD-ug&DwlNZreWz+tHdm9FwoN3UkD%O@40kEMTD>tFJ{_G&Vb?ci z!Wi%i0L}qr+&q(sAERFUsF=QhHdZ-szRkq2Wp}pC z;9NM}&?Uu{t%9QRzRdV$ad%~dZ>XS1?do+E6iskyZF5Wx1)*#i2ND&&0JkQP@P0_D zFGQzOlWb=~ziUFDnkmTocgwuCdeYIMT*TZe2X2jav+4_QvfhzSH7@*>PZ_Y=tO!7K z)@x|Yk9FQ$jibA$?+PSurgBm^GOBUW37}w2rp#``I?X{k^XzTjVx~yPQ?UCitB+r* z8K9en9!`B1F;p~x52wHf+pxqCTsbo_1pR<=xN(G*!={$Ynl?W*_n-(E^pUOOYG*;5 zrv8H=R8M0Q_q-@7cvYqEK+3-6hCV`kTx^a+0cl++K|MqYf=a75g?}|MG)jsRcdE8tCDZ8tCEDV?htI(}W&Uv-y^OgdS#d zEFj*$3@3eiyO}``@wGP4L)SnL<$&D)J=BQ?JhT&m9-61t(CuMqe$SF(X$+6v$tg_% zk*_DJ6^dyFbVCYvPvz89Q4Wzsy*VElN_%3i1a*uyO-7^^g3>W$0kA~)+RBWK8&l6# zuXOifmK~|U&tvF68#kt&ZA_v{LjReUb_I12%b1~btSjxxvDwBXg4Tgj{3R;NNXJPC zn+dvO_~aQ!B$M~eLh>!yJZ%v~H^-8p+(LXexU3Sp-V)1_@C6U0S;rXq-$P4Ej_(;u zSxy2;JA1G-oRAQr&CGI5<9--Nl(igvxY`*Nw3SW5r&!)~cxJrgtVo?u+6^QzB-OPN znTv7tK|nKm8bQ$``-+Pvp23N>CfV^g*TBEpqbydbE!LtA3M6Y@8F-mN*k)at-Af!Z zYI1g$t#+5zDe^&CUn>q)Q$e17AFA;GAinVGXVm%@j;>?OX;}Q!7aLPu;`8Msk5?^l z)%r>%v5Z0MswUuXgvYU|b}J)VIk`MSVrE(Hs|$}X|bF*cG+%o(Zp%m74}`3l=Ii7jg!|l(SD&Qk-e|qkaFH! z-HG7b`;zLdnMInz2hsxx{G4{q#OJh=bLwnHs4b($i$`<1A4M5BBKcGL@!DnF=^~0Byxz-QA+0f~{i#GMHuz30yWw=WE;lOHm)y2XwX+AXH zdEoxc)ae=iAfD3T!g~tkDWr*uk=hIo#A2}WIO|SIy7qmmFFin5*z?gBWlp?1HIr{5 zRQeF!WTulqt3$$_iq_^#<@4JA)19BRHQXkw4#sCUe7R4W-WI=Q;`Y(J zSo$1a&#SZNlM%<~gLSrA_rmZ;>wZ`|67K9zYullc=j^z7d=@tRY8d{g>3e=BZy#-y zNeF;~7F2}?#}w-!^Ji7BuEkBz+b6;K9%W%(Cb~51%-hCE6 zpOPrZP%sC8xc+{sQ;CQa?7qT^82Z@xovbA$<4iMLmnSC`EvjXwl_Ktr`!ko5m$L5Y zFe@MtUqR@WLSH9H#F%>%Q(qN~UtMr%j&+1yc;9;p(T81~Uoc9;^`GeI+au}GItx3W zJ4(aVYgZ3^_A1gg0Nc4M!VT}0HX!9LeDp6mJg(kra8iczNJri^AG3+T;uALbepI|`?O^$pVm;Nb5ohfPBOtl zN!be$ve-t6KeI`0PklPE^P&AuppL`cczBWO{zakx5r`eA-PC_rgY-2 zlYy5^+%Y*Llj=XEiqh$2?pkpNOC3aLO9b&I9Itg=>Hy6nZH*V-7%xH^2!zPakd;1d z#w%i&QjOP!^IS>4EJT$)I)>@ACe%85h6Fq77^9egzDp-#^TAEDU z6*lcspPZe(H4U93)tB%-*%9yJ2)uBV(9PEK^KJd}tE3Z&d(HEw%oomMzRaIr=V1-U z`Qm(P^F_n{1T88a;9k+T5Yk|_E~&sA)0V&T>RxjevgcX2z*;0xX*pJKZD~bUcp~SG|~8U=Xd!-*($k) zObLWHfAwl5n`KrywhsR4Z7m{TcXFB=hjB4I%EqzTmiPp&DV`^)dkN}$ zlyS`t|J}DqoX`8w7E2XlaR7bL)zSx+YG{l;=)8k_%cnlOAV2OM1&Gd0<@3Qd(rdmo zMYkybGS%BBApuXrIAf3!TAR3`>DGx)12##l+Luz_A&rW4`{X#t2)sHqOh>w$T#97D zclJ`I&mEB@ENw`_(m6?(6uvo0SQ<&f&_L$wVD@Wj6kw@Xzj&b;N{bl7)Jj7O7Ee&% zo@X?t-q$E|y7~V;4WCa}7k(@$=l`W?S(a}0-Py-1N%FFES?W?(b`qDkGRrtIE6TD= zU0IZ6m+z@d%Q7v?UfD0x9>-=)m1U>f?YgvE4h92au^PI=qk+biL8?BP+R9IO%zpg6 z`i=Ilz2?{3_?0)JhN@4k|N6G9)lx_23Ave01Mbwr-#`_khg z60?WF4wMr1-Tg4Um!bTA3=_UVNsg-ev=r(MFDmL}?H85C7sae&n~}kZ#GAdRs8QQ{ zI{CEuYeJ)LOUAZ=s_;r<%O|Iyb(a6`d)@GOdX<&~o;hM6vgUVbZrs&K%9On`mzeRc z`f|G6ERef-uw=#E#gM=rP0c-_DpJJj#J$WbD{MA)U(vS9uL>CHB5ZVa+c896DK+Y1 zHgN(uaY6x&FX^%`rg&^6w<&wH;=U+ulk>)aJ?~Cnfh&-IIh{C;{0NgH(=l8@Ak}Pf z8Q_lh_!V7!aGY?z4MTI+>)PH9$8Feoh4qS0aPQXelh6EH1Do1 zbd!|-62yP-_-#Qg2RAO+-Li0XUfoaC@jpp@aws_`n-Hl#ki3M3)S1I#gyd^T#>C)D zEUUMtA;V8=r|r%T9%9CG)ZAF^K=O8`^xKLRx63XH4}2RZ;-A{{&t)oFKtt)8%_v>7 z<>V#E^NdhDcf2lDirIzif@Xa0l5&&Jx^sC$G|LRD)7{xx?$h1bL)o!I$xE~x)vNHf zVw^W!Lm!Ua3aL3t;*CZrF_>(3=IfD4Nc-0wN?ruQ(2ovLuInGY_MFUza!fG%P$H)2 z2b0yhgV5qJ#_*pdEACp50bQSuHPa(jzwX*&zd#HUjl{xQP;kO>na{J#lAiBFdgDBh z{jaC9MTSJcDV%c;hCP0a@2MMO1=++c`rI}X<@gCHNY!_3MEIK1*T2?X$pY`?I?)MUjSu}5Vd=y?p!Ph=DJ_jWXY_Jq7T9lz9&7}T#wAgs*7C*MFF zUh+Ht0FyjF7`kHzLIJc4%-Fr^qeQnl6Zi7kh2E<~|7tlpEAIR-k|3)4#Ff8#Eo~iK zt^O({!C?Ns&jIuJ4&<6d=KR(?0e3*FuVDC9iD7RTg+Kh&j_(jWcxxJRfAYOO55R88 zZZq=Tcuzm&p1sGqPq;5>l4w?yHElOe!YI7y-*kMZdaFvS?_`Q(u`#!`lWd2yDGta< z!%@wgD08Bzm1WJ7G;WD97 z@4`FQQo`U9x3skj24Kd@ApWS$wbDV`%QiTLzSHub#{2u8a3Z{~BY?%A=dpAM*w8|5 zdyv+h?+Sn*sPHguH;K@q(g} zsXN#b?K0c{nnT7=oX(JSPa3jr>uDa>kf|DiRVGWZX%7)m zWfF(uYHLWfi!zE8Q4tsUgO zU19Q2YfJh#Yby=83~~t#1L~T5>6FM4-@ z8Pe%Ex9%bo9t4gmMa$LsLXkxsGIYjy=xm}V<>3uWgh%u6t@LPs>dEzZ_5#*u?uA{SyT}+YE4)J(SVr?#v&G%9q_DvWFRiY8tl2WYCYNL9`R5 z)k-~T--bzs|(Tp*W9HG=f6gW=Fa9L*`0{ ziCld%t&tFh2cB)G9W*-7FCq24>c4Xi4iFM(k)~u|Sycte`Ms1r`j$$WaOH5~_D_gwCs(a^i%a?&MVrEdh}Uv>HAbsx%LU4r69QY(NV=kpz7`F*t$H${zFJMrd^ z?Q`P?;IYvz%>pnhWnL|*mTLM?)R#g3>cnPvP@F9bJ1Td8obCPFvnl^?Xv9C4T1|)! zdNK9jc@L)Mqt;)AS`T%yA8bX-ibuT}0qJY6J|}~}yQ1njm;s3RQN*L-b!|)gXr`_4!toM=TOQ>i+dc8^5Xv~k3v$UpxdcWP9)hyCs=!>9#bo#M!`sx19irQ zJD8NVkj*SR)|juSBVOl_wI(#Wxvbc59>z`#aYXUjw&B~dLY$qAC&!&avsoWoV_!J- zXtuE|;OkPYB{;?11y~Nr?rbt1Y5T&EeuX}@?Q1iv8#CKXT9zXlo*$i)LF1C`&Mxz# zW9MX-mGMLlW*8{iCO)np^mS=1YdAmdQ>p^WNY-&|WgH7+szS1$W4GF|TLm!54kYhP zF|Prh1ZWD_aD*XtCYIA&nn?9lO$hU3+Hp%QC4 zw|XGCG%n5SoIoBv30uLUg=;lJN%aop2_!UIVR#;gl_5nm9rSs5B1?s=#W`v2X68Qb zFpbF~0`8jpu(8mv0yeaMOKg3~;`>Tm(P>^gyY_^89N+3i zCrRo&{0m8RkGCfW3tIhW`P#q@8_IDOc=(^?&(;R~#JaQpyhzjIb4z{wC%G>c$^zG7 zC-1r}?JA%xf6{W2o06ePz8+n%nJ0e(=!HI&GaDgWELlh zAd-!75?=O}b>O=v6h%_LIg7ztJ(+ZV|NoEpW_wb9_w4pR>5y?!1cxm4l!&{^bePh| zKfUPUU_+R=N-igk64>KIUfqgM12?=vPjTb_Zkm?&q-n>1pXhW;b*2!k#XDaXy)4L^ z)aQgeQ-ZQChF=XliN^)jV=};|b;oDM_h17EPoqBV!y&IvLue56X+MeH`c%Q+s84&e zPvPow7#E)4T*{}5_@v)1!D_(4GmkFXwUZThUkoWvcs+Z|U+cMy05y@E+Jy@10O09g zq(=|%7ANi!xr!73;4P!$%SqZ>tuu-$bPl;6lu6M!lwNsGM#{!q=W1s{pO#Crl?lzT z$$4pNhUS$raoRrGD9749T1ci^_*x5SMtavMfXz7@bqrZ#4FDiBoJO11wY`I%me$Y` zy)sE99E548zLIW07kU@snEzoJsJMVW_Y9Lg&=!;E$?xhYcR|;{vw`%C6IPaD4%A%Qs9DAOXjZWtPInOorl)s1pK~LfU{G_NaHp{7 z(g0Wm02?QgYJb?AbPs0bw*ki*Y_X%g^TZ$fsrkXd7=PfNPRV zY{fSw_Iv`km6%-8TJcXT7}MtvwMZqiwKQ@0eD`nb58o-PvYbV<|In;`R;8805^Tr9 zyirry{HCY%R$EF7_q(RSoczCZKNY19cO?QWsJemjBcjI4jKP_R@JdZ5G6QiU0};u7 z${{~3wX^l~amZ5^GIvX;=BI!5+dn;@hP^nSGubp0dRO+_OiDFYPBU#=_<;=`1qrGR zy6HNIsuZqf@tSkNI^mx%AH%FY+Goquv-k@P@vep%k&Tt|Rh6Ikh$bwOZ{T!mTzze-k2g#r_m zy0g=rb;ksDx|~dejGMT9Tmvn{9bW@?u{9Ii7(Q1JdE~j&Iclc-@ICLwWcSQ1nTb)| z=x7eC$EV;BcQ=7)P?)@qJz%uFDoIW(Qo08PL*ydO4|cy;chYLMWO!?Irm|7<*mGWP zdLY?NAVj{q4(a)cCo@UE^vd$h2<3TKmy@jv-7HO#B$1cDdY|)2^^1b-FDJOwj2q(@ z-1gS*osbi@O(yPq>UY}5Js<>FPkYb?wlPp)t#@v+l_*Q~OEdua(>4>@drGxe{&XI* zi$L+&y(PMocy-it}tz{j(N}QVb%MXLVT)Grt>hI z63L#pd(y3WBF~i!6sg0cv58rW6ZRVul2-9W_Ez^XhzXU6m7#>uqzDN72wKyCDI8!5 zHwRMyl{7GgGcbjKLml-gf+@J&MRZ{igebOWiY7g?JyY493MMad-NEXFis%roOfVRt zLypb-Kyn5X)`InMaVBI!SxLWyJ)(!+sO@x5bpGv;cU+e*3bjGqWoMSMdH-ojn#gHd z%9u&JlJeJ_Bn=P`OJrb=xK?>&4FcS?L=FO`S9iFX7+_3O5ot=t7cxzKZklFZ-NztE zvcQF2ciA%@0s(|B(e}$eim=9tlVFs5l&!I;%-(RG4fyEBn&oNBd=8gORKSi|TP&t0Xwx)}u2+^1;IJ{#u05g77BWW?h%E&I zwTv3O%$}M;Zd(6GtbPhob~)KI2a>I(!R+xaJS99;m5r(UP53PX`I6JxqRqTTd7l>9gSDw?ZO2Uwe}>}AaZ@u6 z<~ZF?+R}pJSB$Urw3&LPtLJ-Kx|}Qm#@YfBE93O&1(ardntNMT-rP-x$SS{D_ssTh z5P7yba}vU~(pN+efA#QzW#G&^&^>(%AH2ZdOLa%yg={(5CV7;BDJ+6}VSYSh0n!@> zmzgZWN$9*Xq|?i&s+vy2+S=NhG81;al8Pf!V3xk19#YXdR+n5Oj-_7sZ(r{~sniR9 z{29{HeJ_0IGadAka_mAG-u0%!_o}yADX0b^xa+ZnT7PUwqZH(PIISV&?^4NP02oo0 zLx`Ngf~`=5blgFfGbGaY$OFN%AnU@wbK-2zh4P-bZ%|%d#aKAK@O=tlf_I00H}U<8 zcOW-ZTBumHq!31H%(<-5BBU@lCBR+>tDszEqeMzA~IIgH(9$(v%?}eYZGbeBqUt86FZDbng&2i_Dj+u+OF8~^80y?1( zCB}oaCsji0`2GdTg&@xX@~)o)(#5c$8x%OLV%mzuI@2HXE_^w;5ZgikOq(VbRX-0A z8m$pAvZ)i2mBhEd?2^9<+^Nselk&Nql+W~}Tzb+bwjdG=$VAJ{9kpEsMQHhj(7)MSyu&}Q+eMD4?A zEvhG;M@s*H=LaS_ehf^}IpSHvz86dLTXM~FQG<9+nSAMd@};xMmzt7}=9Su(HbXpT z039nR7gB`1M+}``G%3C>H_t!n zJf#`tznq^%kI0ah*QD(3JyMeduro##FH*ph4UkCmwXahd>WmSZJ{!eu zclN>bP=4&H>f0bC89JJnDoD0JYP0u4L&wZ*4147L{C}e^6tj-5tzFo=YO1;mJ6EM4 zSC5bin}+UKG?E%}Pk}wXm?1^O>=xvi;W5~qT{0eqWZVhq7;@Ocl#FaS*(2e}y7pR9 zCz2#NQ6~F*1_?WA4CSLNuWome)WdTUY8!qoneqzx$$t2ve(=+HJ8x7 z84PXD7d>4zapx<1p(j`fX&6uIP8hz5V&W6G#AH^+D0g%!WnQZYt1(E1Yd~ao z!kWLhb~~(haJq)2Y4sY_kXXWON-Zn#e5HA>zXUA%+L|h<$EIF(G}No!?O0XXHVWc^HBKMA?P{phHv z@9fTS0Q;eLVX_JiV&q4c>==)-gMLKfK=M*z`C3G{6dR+g+a?t4CTSFB+wO=toz^mE1da{6_R+ ze0%JUQ1MYRd)*C*o2~03a@=PZ#|bH=lh``>a_CKBQC+It&GYTzoD5=7i}ryHAv;m1 z1*Yg_=$T3@As3S$R@@$`QfPF(9HTjDwN6~}Lh)PX;H+(tcET=xM<`|N2EG#}@dwlv zM4Pc2kHzQts=B{%S@fG_(ZhDNBx;-EPRk-|mPLR4vgoyz1^Ta+#q1Ka1}zIY8u;Co zMYdyAx$w3udb4FQU@ntuP1z42q~o&K?0}ZVW(VfWLQS1wS>#$46u&OLvt`kN!c7_x zNj1FMRqG-et9$|b(LOEt1m!gJrqo7lP218J2q30M>WAlKiIP;vtd#bYm$&9=SF(EI z_L{eM$Zo$MafRbbshH#@b5cgY!0-oT6zZj2Y;b{BAf2-6C#}24&F)Afz}HKo`e2Hx zn^mtf1+RL&E9>q^8BbJUy!Q3o6G@H?KucBzL+@3_Bb z$>73mwKuJyXn7UXMc|<( zzOcb38L94s5j4D=(C}#!*Fy2-w~n%$C+aER3;WImGE4{Nv?uKJG}5rs;3@N+>M3|1 z1sh{=Zo17pkYp)vS3G|*cK6kt<>br*310usBz@{ix}Z~XFh?C&R27F!SIuy9`2jRN^ICZ)k!L!_K`+{0b5B5BoGoODSSu0?#bsE-)ev~cO`Qy^&%iEpHkGreT)OTmq-$k)!Xf9!psCOh+>5xz;{0gW& zbI(yb`g}T}Q0b^G$GyU#(*EL|_Cdj~LW%O2Q(Y_d#Y->?kBtk{U&K@_Pf`;Pe;hqJ zsW*^C=dHvOpPrLd7C7_D(?EohX0kLzny^=uH#+BzXIC+Bv#|+ld8>p&+LUm3Bx`Co ztm$fZFx+xW9}%A&N?tk=UhUc7dQhHksykQyeXw4)^i}>Wd_10diG*3_N}s=E1PyYl zh(kIrs6esu9+|cGY>5 z85eZ*XR$JRzTzgzGt)?1$w+7rwJOtjv{)O<%ka0bWSKjao;Jb{I_1C0U3NUvqDR%Y z4bdYDkSFenn<4#{LtoQ9bhi**624J70II*wSA)BPmz)rUU$Q{5my>2Wd8T$uE4Q(3 zS**F%y2xki;t`xaDUCt*ifSjm*39PF0SQUQhBR@=`%lMnrd=R2p(7X)8iAR50}7Xgki5huQF*W ztZE~vA%Xk~t1EBh1BE42$rq0WnCU$1CpiKY^65$$BPv#Thl*F#l&%wY!R~KA8Z^#f z(q?wRFe0cf+Nura6P{+y+=)AeUAFPIohH)MS%2Gchi*OUt#uB!@5I|%2aM`jV-)KQ z>FqOCltz<(R9Xq%-DqE9jC;{Dq*13n-F`ksA5R`0CB3E7ItkteCfq`?0q@Fh4zm97 z^v6@YOx3EYI*PSf^_u{AhC36V?O?*x1K%g0Ez&DhvO;%(HKmJlc9IOo9g0xXzz2nZ zNe<=5RQ9TMo*giw&)DMg>CMNE)mzTVlJMnZ45QG}75Ew&et^!*Th=jenHgW)=KW%i z;v#+B`CN0(F|d@gfL>qLVB%ezp33BhyC2UCe_mOSj;xLg-|-_}Oc?4>36ij|#C7m< zPs4fBp$@~y7vV&5Sv_oCB0&H_jikqPaI<7E$k^VB5`H4Na7XL>reR-%>{Z-uSw+vg zM~y&V`X~J#_$`O>D}5hw(|0&b6WQOae%bFDwijPW)7Q;uq5sznqR2d^a|m18M=Z>=*d|#HiOFl*7@&mLzo=?;dER8y(ex^GHhf>7vawIx%zu`ELuF9Lsr&VK`4ui z_Nq5%5EgS~#(^ga^qTN(o!U=tcqH7p7W&TT45@!OP!#|wcE7cp36Gq||xFqvB zZ8kar?mCC3lJdvcSzZW!#kwFy)HV6g4c+NjuIHtCfa@+}D!;tWiDOWAS`pkHp&FLZq90sr5CPMQm^O;; zg1IpbO5iDsnGSzUlwvEjMJXy9!^!1Ql%^(1okXeBMFm}C%PD+8>$qFt@;bsHcm!wdPWZ6sR_$&+w3b~}S2s?5$~n~` zf6a}ooN#?XP^xoctBL4dk|b4{ni^_a*n5O0Ad!!ga9nGo>`z=+&XJIluS*<=% zW*k~N<)KY4>50SZVR$yfJ%VBrylw8d_mqthV;%yfZ~qs&P@VG7DK5c_79OU0y9>$l zc9i3Y_ucQa@ZQ@Jfnu=Ddp;xDFGq+eQW?&meX^9~YIyH$v4p+2K~+f(e+KsX#9f5I zdA8d>u6%)_Io@hI)Y4-DzE+Z<&wOdtXXUp7E~qQHjZHtTvyp_^MerSQuMeia!umuR zZ=HfuYdqkDS4f@X(|(~6_rqe>)R{Nd!ca*W0GFyuUhZW=%UL6E7qE~)^#3K9D<3Wl?N_Y=`k>2;z{1dy^UG0^ z<(VV>g#ZzHBF4oB)8jYRZuO{YEs9;U@ae_iN_>B{1A=Xp_e=HJ;<-D@x-?i)AIVlA z`epQBW!2{U6EKW=gYr}KIiLs}I>4#qfA0v(I`zGlDLoU@8{~iQwB amasoECF?y zpCHAZ>$?WY&!YiSr=y%u_^^7t8w1%N=v(c~_~;$oF&_BLQj+Ayg?XElB&`Seag~xJ z=*ZlU3q%+{uFN5Y8_oX>tZ454mLDUR{~Jaf(PcAVvCe=6oaFx|zo^SXR4u=*s$uu! z28U2X{L*@a$Ju)#yE0YM!@e|E#_%~u*}nmJ3=|teRc!2G0=)C5lJ~`)3I*Fpo9wA% zpEgfrq=<6y%cl=k#_=#_j;0iM*;yyTlkbG-YnB!Uz8}1WRAXgq$#1BoGJglCoO%ip z!9;tRuGXc3X$cOzxTFM)J+rCunU-%^$OgyLA9pcsu}H*oD*#z6x(;FFR5ofS0ysM0 zmKABlSEK^+9jb&plHxc$PsSJ?buF)sY@i9k#xAtPu1&XPqO=cp0nMh(rkl=0P0C@nYMOrAxOhkrVD~dp6&h7IK$w^7= zQ% z9uXp?<}>Ey%pD_Xbc`rqO*S8n5%Za7bqo!apP6fffa^THPk2-4u$+1ya%DZKB#Fh6K>K8oFZET4{y=zR!y;RgvaqF>ZbzE|G6 zJW2RoQ~6#A!Jf+Zn&EpT7Sl7|tJQz6`_cEB%J(|p?Pdo(3UemkYbM`o_L#m`V%fNK zNpWeE?vnpE_r1nLr|`Y8N;szwqkx2r&$!mrAVc0MBTxcD`qW?}vx-WA(7D-ZdwoKTEqDfoLR_kj-NW z*({cjMcDUHI7WpvhJLh8VN{FUZ?81Oy;O##Wle-*hk7_+IQ&1@d;1_eulv65Jm-12 zFVDTZcY!6jz(Ulyk6>#dVg;CkfEa+|;B5H!OSTzD4rzywr6`FNTec}%b<~dIFd4OV5;m15^i*k_l%BYb>k>le^ZlLY z-n)y%i=^bgN<-|u_jxlUOCA^WXocsDoD;1Rz-b;wf_33> z(3`wJ6qAPZgf%af5?W#TszbJAO7oKj*FTQ4J0(+D(iwxZ1$v<^P~BwU1dnBhOfsO% zcekJren=u0VNh)0C8hcS?Yk6R8Qkcj9)({@r9}E6SW!w=_#sGIf1ylCSFlkk&^Rt& ztVEf47mnv5b}q#2Ywmo=j=GFf-1#kVk)#~7CuK*SGI-p1oo~nej^nvqI(IzZalE+h zI7QS_BeJ8eyzY3p<9K!5@oLBM&UME-bH@RV?`(Z{Qa3qoTwd9@mVJ0!`AZrh1h_gT z=bi0fs5Uwu`zU*BbmtJeM_qp7L&c6m8{6B@KAUkd`GNYwp8(Ge4;X z&nTqIh6+e5{96Ec*>>HrY#|WP0#0`(0~L~oY?JllB2iW7W>(K+`^48sV(Sn2`j1pk zQZrZ8JI1-cq`7PO{ex@*^K;kR+!dI9GCfIiSBG>ywqC~EZEun>n7fbXJC2hq%k1HK zc73tqIE-mKkC!`+SJxe{b{y|qcf2!qyq&wVr!#l`QI{n;hUhgMUadar+82XtF8HIa zU1VOZI_iov!M$2>6ux~;c(11(b=CBHSJ!u@-}|x04ZG2uTb%s_HaC-=^M-qD zb?$sjG{HxF{N8t1Y%hI%14uOcGhPX73Qge)6RcN391Fq}jRzAD^7#8XQ7yBdqvNZ9 zKk7LAZCBj~v?fEzt_Loj+z;A}PU%99Q>5Bz>%}!${QX=wMqVRS^-qhg3DZ#CmA(B^ zBMk2wsqWax?1&rgK6l)#aiUQU>4DO8d}8jZOz}dTRVyejyt*t>Foh~_wZkS&BOyrl+pzO zTiwQ~>^~ai+c-!J@aEK%}QLCPtqY;|(j{@wq-}=&8ftb`GuO9}6EUj_WQ2hka-b)$(d|l{a=XSRq4%wAxctA*+b?r%ik(dhtVlUSO1tnm*dB&Vd7s3`Nc*t?$LV_ ziaAeF+#{@A3w#4KR#7#ZUDC)4P-VRU`uHMg9`8-op|gw-G-`7dN_18)J^MmavaR3- z2<0XzcA-4C+H{hLB>L_-0BfP89w3>_4TdG1g-!}_0Epo}y7P|v%-uorKiz@7j*Ef* zu&=~5!yyy=-*BN=6@sKP-V57FfKDWvw76pZ^!+Ti(GRB(wryG(A#;oy<~@}#Jc8K$ z6N*CTr0=l-)3GLY211h9Yh=M&55Qub%pK++eJ*J;l5Cgl1Fg3nSu`Cwl%jIQxWwc1 z@s)8!C?6=Qdz6I2D-Y;e) zVv`Fv8>m`JWFPfWWA%sygX$LwV&~6ff6nqOsR_*ZSanKOqg0ZWY7I2mRhPT*DD3UB z&A=1pi7#M7jF1;Y8Qu* zMS;S?eK{tll|KcGgd^_SH~)I~{9N@m${OQ0-r%-=T(-%g0*i#ju<-N?Aqxv3d+B0W z{JzzO&#dhfQt1!VLEvPpG2m@CO&qwL1Z3$QdnOL>h@I)q=roH0?aMD7To{^WQnA&$ zP!p``fhO-_QMfR`w#?)Zg^aN}u`iryn{izL!INp~EJFvt%pIg8B|qb@G&?)wphrICWwDdIZp$j2C-w%-!H~sYb>tc~SsYym z*;d?sqjMf)f%FP!JF9c6czn06;QrdY^32Amiy8?-TuOb$Nvr1;Amv}on$8Q!DWfaq z;Xvj+`*`uNVt_1NgUYEq#m1%k@(%_NZ5to$=4xk)ax%xdRnQJu{h#e9v`ui-(4%Pv z2|6mI966|NaO)@_v|;J)u6}fy=tC*FJW`Vv0@Dlew^hHEr;IF1T{16bqaP!C2T2BQ zo9unu`J(Inw_V?vjw9|r^btgR`+xM6MMKUXO5mtD$|TE@<$f;;O4wojDw#3ya!AL~ zE~Ew&mKjIpEyd!9lS+Ck|0<8$;^=y0O8BMeA$!uPuK9D7(fz4LSF&W=yk z1OS|=;*JI>vBJm%3{-RMU(<6aXSJdozjNr&U{38{c<@$XEN zOV~i-7xi`Avy9P|2D`Jkw6Xmo|N7N@!-eik8~u+xtvm2fn7nkU>4&}@|Nhk%$^iUt zsZwmWR^dSYol?tI!u?+Ey#0@H;mc$J*qL#$1)PP!6f%di?L$e2o z8>ecN^Iarv^Nd5~j$zPUrfS&E^ZxMXre&*cRGnuBg}MljzYaXDx`JB&A_;It2~)xN za?*c>gZIo-bzKD-KF#N7Ao1f2qzRRqF*R;!ao;TNJH>&hCPiY%FO8c0;{?t{Q z*Dtk|uTGduQQdVmP7KpES$qfw6JIO?g`91xShM`4f=NRRMv_B$1p286VA2(W=8U7E zln0-#;K#zqAw@f)RW9CLs{)4a{`Ho%`H+=Dl zP^e8wVWpBODYZ+2(79FetC8q^O4)9X2cdeYSqRw`P$UaaH^UdkS?InrUR3pLwr*5K ze>M!m;-wcF(v+Y+LNBz3e&8V?Bgu#mGG07?{T5UJ^cheAO;{$jFDF7V%T*>qOoNoG zY{iIuC{nJng3J|~FkS@ywOr*Q&;dXA6ENBgP>A4VlHtLjSD7VDzjfAVx`m^1-1}2TLQe=#%P9zyu7x z5O0@ByVEV$WUDy15Sz`qSprUaT$i=u3EPUKN+CV zB^-x6W5LHU#B1fD14=5jH)Id3S|x5COe-wUFBC03R}p{zugW!6%;>rDM9)<~@>JEK zFUXQQsT;Ek)J9az??P(LCHCnaIfUfeDq#U$L7sDN88drYZliNz?d|4HV(9AE41 z!Qe(eE@EJ0NVqRQB`+y9)zJxAviW`asgQ3hUBo^FHK0VrBtgWeJo<^YJ|JCjYO*(y zKprD1%rTpVAsLCGZ7THV$xd&prorm z3Z|-*qM@H~|#{khO( zO;4E!GUzLqC5h^!8+x0QSZ;>;1F5jD+-jQd&cexa^HT@eyT~o=y$gB7ms~l$wnoNU zzMAI)oktb{hF&i@^&u|+SlCX7LFk0~1GBJ#U}_aPKj%R4g!(B`#=7Rd0b*xFc!Vv+ z!%#e!L@0f9+=f3*nzc~HU;0O?$ebok*fA%A2<423aS(<;=m}52=@)9_Pg0;t*+eoe zN?ZeKWisE{Wt3z3XuzDP02_}!EhGHTYxx zFrY@Vt9XK0>IC~K|Gh~%U_a*k#pdtvcjdxX=tr;Lx)-*#7>Qo!oK%Pg=rB{&nuM>W zRYRDq!Y@3^(kdF7bIUnuquj%}2Am;fP-$s)k|Hy=G&}Q4v$JbycG{)c(bDYPWNG%; zU)rVFr>pp4OH&zTbT3=0aC3~%~leP<} zrEi$VA#Xe6A?M`6VpL`?G+Ld2=iGeSgm^j9cu9(Nv~@%qTd4G0yZ*QE*mdL}E^ZvP0iUN~+XTLATcY zam$ItVj};?IG~=#7wkf3-F1v!y@Nv^h7@Ikk<4wWh>&I$0WJ<}z5M7x=?o^pv(!;~ z#BSY4!D-druBM!B$vB;A%TeowEY%X#@SNdHPFOZIxt^xEpY*3f8DCS(BW4e+o_0O7 zT<01V19oSd9PFlm9w$JL;58M%Da=G%^Fj=?Va}H8`CQ|s8mj=-#?QbO1!J@bzLfUP zYT7Z4oTgE$m`d9Gj(d2u{_8vW#IxCOX7F&9mK!!3PVx54kJxG`$uUatX`nbQe|VcM zsZcZtnF7|9@zS3mepo(*+YUYT+j*tyk&My*%=$NH0%vAbJ=a z3Ms2FaPz|eL$c9JJU}*P7+n$CRY52=;m_*X#2Z7-Mwx3%?-^L)xbhPZr8{*)Pj@m1 zZ-8rBah`8;eTie~-_T|>K>WI9T_TrvnV6O2K9)yoV%98J?RPsCteKc43{}NDl=1`G z(~7aF=*YLTY^HOx9SialUS}-GX_`*N=~%P_p-wH{u_~m7H3TK4MA0rnKKeNk3XZP8 zIj99tV64wiGtem()bz>$k<3yYeQqviG2awWX2)DMK;A>EA)C!$VM2(0h^l`-;+}Up z&IVT0Po%K!vf*ZCDtX4fkJ|v1JY(O-10$xZk5bSBV(hl`K&IV4Yj=M)`j|?7pqJOb zW+g(96H84NtHb+;Vd=%;+AR|3M(xn&T$|LF)-0dJP>^c^Y zSn>!p%X31r)a{9lAn_l-iD_&}3>85$^WjMeIB2>1@-B{;Tk@S8irMIMUL;SJ+D}t< z)%vkAJov6MZNMU268F=t4cL-8B+`fpAP5XO!h2ZN9Fek!H83$fk-OGorSoy7K}jjmXsYBewemh%8Yg(-E{CbLxdk4q z`Lp9L6Pz*n-HrbTj&Un z!#hMBEZ0`cZ_0B7Xc_^wi3xu6fDlsY?zEw+5P!(y_W=Kfeu7Dq11rdfrnaoR{RR^t zImEUu!18IH$VEC;W%38b;Lw|TNjvo4Ou)#nXg#E@VS$I#2#;i1;4Ir^THqtKGw_n) zeuuovp3=fUaDwA$=p@H!XU%=ExfM`cIMiE!$(#J7TjAdDhPfZmr?~y7HN# zqN8C8%2QYvdgsM#L;agV>Hu}(E2><@BvD)ihboYUPKu zO;RLUT2uutxf^ei45XPWaf9?kb3||;K4XIp~Z$Gik&wn z_EHi3NvMGq{oAOP@>)qA9mb_-LDPa=MDT>+3r!{06ibEFs>%+)fpX}XF#O8X3~M`Z zBBGVNBv384d^d*qhsz&7CsE_rlD%l!L0Ne5?YhnVE!S?GPVi8jp^mj zWDjkQUSz5j1Qn5a+2{+c7?-bFq}eW(r`;0vAyB?HZ~C7O=CC+K2`?dImQ;v?$+hnG zMgNW#pZlcppW_sy_WqC)O(G8u^f1vK#Y<>cOOBtM)S4Bh*%ifECWUfy;K8@RYp!J= zA4fanO6oIs_WOzK%%TnRQTf}URoxYpC(?CFl%@XtXz%7fNOD^9P&xa|@QTfLI%(#3 zRKJrZ7(TyS^o%Cy^y> z1+UE#ILX@hS|3LNDq=~#8s!9&GE*8Gl3RC!=pIh~l_U9F6Uc?7RiDe6-s4-w+%(rJ zH8Q^bu}(Z>O=Ul+X{+8DZ7l8Ua1znhINHM6nQI^-uP0thdj-pKOD9Q2x3>d35qj@! zcPpI6b}NbJ*}G!atUCzo@-)NdM!>EXLE|Bt0*X6*NzLnpe5tfB#8)l7Zl*g^&P2$9&HaTT{JXGLX4tzZV!pa(s0;?I=!8 zVkb!k?qqpoqH<_2n8XelxI^#19w=dLmho?C{~Ab{F?Pr#x{29;cP&4!>;?M71g8S% zWFaG4z`c!rRAsV^GanNLZFyz6eLmyd!<2IK@`C3b*P zRDaxKANDDe`!v;l#P+>xO6`)UB8t@(umkEd#mq76>yLEIuOGzFfz`&-I;}&p6JPjO zs^-&~d4j6smFuL?sM`Em#(sYSTX)?ZiTU|C8U+INvD2Drun-)9W&q`?1W)rNgU7)0 zzUB`-n%oPbw=6V^Fh)u9kV$~KhJ3^=VOVl)NL?-{*sVx6))mH)Z)efT^3yZlh~`Nr zISOR4TBw}&a*e2!w24Kt;|->ZRahE(R3b+Ny9zmC+673dZHXbom6(Vjh|4BNKBRvv z16ej2Wd`G`SRE(%GcD($>G~Nl@TDPH37z;47?S07NN|)+ha@>l#R);l4BNU|L1%Co z7DC9BM5X&bPx*j2wDCCvwHE*$xh~}?*GVC<#M~dU(Z~HktpM)yByOHZ3bYTSI#Zn+ z4+MUi35-0j%X9^fo>a6aWdT>J*uy<5xP=iHf;sLI`kwc z>l#Id2KTHWTP4qOuvyz-(=${wO-{$dkdRLC&%XZWWvXT@?ufK|Gr8ucR+?BzvAto{ zClp(DD}aYP6ai*z;a*w1PN*F5g4X1mX+(zte4~pMen;Qv% z+R5fOH!J*&DO8Ce9P5eW?EFi5;!#!Gk)mhMT-%Atz7e;)RF{Ox=5G6~AP;YeoENV^ zyc;jMO;8mWw}*22a&UXIqtss*_yyxeJ)WxQF*~80NJaB957PfE@s{1tv$~lb5q?>2@3%fTgpt(pbX2cWL_YqGdx)wG@#Doq!-|d(6Io9V zoK#>LKKnY9#)4a^kCb&Y&IgVnT_U`i>2Q{M*46ry56~!7;fYFXD4k*JVLBhkJ?ryY zJEw%n!KEGhhfT4wl9lGA$>SZcd?&&qzc7?ml>Vo<(KOw)Z?>4+S~C|NMrvP?{-)p z9TjHba{@dlzX|8waDOfVR#QuXb`KDN&%*v9A?sj-y2-_ zf_WH(IxLL-7WLdfel=wC48mgTpc%+{f!iq$HQhd>?p1#@G2tNZ0=+;f=4;|(4Ol0j zZ_CY>zE6HFg>_gZGqc<n`~u2z}V5YFlJDPjEXJe}XM5EEEd8Qfx-$hC^=&*qZSrzl&_QaBfwp0*M*9uxPcf zKU1idLCbMQLHJn+!cRjG-VjqPo-!7D*9!3PRSiNm29H$5{Crby zHXS8Z4}!b;1I}Ij!R+b_+11ZypoqcIu{vAc{{jEsn|mr9x&il;4@N+s{AV&a`;$N| zxk>13>ghhXqs{?;z`W5zDPh7oR7WA-%HUTix=XG?${Q@|pFEK50|oSTx7qX$1GaQ0 z{STOAA2g$c+^OibYSwGjwAU*2TC<;7a=cjtm)bYW|6J(DzxPj7b+B)i|JhiN%VhcY zf9hzHht!}wd02!C9G7@M{uX?;K!M6(#a{5f=pWJsNBz<|j8NMvLYc(2 z0ZpqifR)3mF<7gF(5MqKvr3@pnpMV4Yhg3$7MoEe&~zoxbU7^bkW*p55Q(?wl9LV- z1u~FD45g+vm-DHnvZX@n49i;CD%v74nV5j+cTOAwdac%75a_j9cMXVFo%ogsayfLi zn%=v^{;>RR;igmHcSAq^Tl&6e`<@!()4r$UOLT=^=*L&pt-nOK79{Qb0G)WM=@94~ zU1$d2B0+97OIv)bSnzk~W2d=uYdkb%GpU?iqD9TVp$kJ}DLY4SW?5%3B;8p|d&rP< z(~u0)kaW|K3~fkiK72akU&FC?(~xx2kcf%)l;v0S$b_Tkh+(VvwOpPuMTPwFABui6 z41wUMep(6r_!s|~Fk@+s;tHAeQ*wp$>afrF`g&LtMf48^vP`I?1=yHK5nQtT5onfb zqEqO8XZFg1?xSYRL-#xL(EW1Q=d5|?ekZ+hC%tm74sz$!7$?Iq??cTx_)c1& ze21I0ps$CVnmt!tyr}QE=6cNn^R`@f-DaSfTN1R`$K2&(-7IZ2%7jS6a4PiU6^7wN zI}D^}%!c8(pnIbGFQ)BMx`0#1{yXRDs z`>q2xu%}e7`PXMmPlX|3;a25(PC7B{p2FWEwq=0=`~?8s=%c)d5al9x6{fo*Y#*GV zEvCB*Y@x%+JFtDibo)!QnbA~X-)Yx`DlD9KV=}W&J2JQ4qJ$16%IK$9_dPzRvvK1r zoOY@RgbIpCs>2c)ZgQ^K27y^2eHU!B_@T+~wrp-Ahwa2#RTM%m=9^$-Z`CuT1AA5Y z8TZ04Q9@($L}zBq5U&CIeorcAM=CoUY)ImpR7eW>W>an^MEIWtS&T`0ZgpISPF$Rf z7s4PcO~M_MaOVV4tfl|B^2FNo8o1C$|t!E5Sq0g;0sO)7J zJkkt!9#&n`3*9gPNxFc3un>A-$=IM4;qDOIR(H}2Zl}+h^4$bx-#IV)*O2`zro-)G zBClk&m@KcPLUm1&tb8YRUiP8K=7ESnTNE5MUum;RuJw>>3u$-l&E(nw>#LXK+CmcL zy(HHbY<+e4@ac?yZGH8UTMa7F>dT6{*gu7<_(!sj&}GHarv zwFv*SvQI5q;F?|-B7uAMz|LdD5pqQmx@g;AP<#CMfo7oWo-(My)qFK9VgZ;P&#P29 zGig?Iqm#H)#|t_QHZw|2!)7+Eo2*J%SoW&=RtAFQVPF#khfAuIH9OznaWNY&uyGIJ zA$ibbpz)AT7qj9^yt=|^c-H{XCWo3oizex8lc=Zgs2cZ+$r)a}7Cfr9G}Y_nOfAs= zGL2uh9Ul$MR;r4dNENfn#5f!Fry27tl_xHTGKOzlo)~onrE@Hb!$rXpPWt0J(7{Dm z#G~ZGB1B#st{*fIOG-Q`XNY%|>bpUv$wRB5GL&$M&rmWDBCoU?g?vtbdLL0N1S@VW zx?T-6{TM{5ngPSyEI-966^d+3ZRBJ?^S3PicG4X(0;S_h?iXl1Yg=Qq!|l8_^`wQoYSln44VR8IHUU1>0d4%K8z3t{KE{w{^l{|C_9?x3g&i62yD* z5|Mp@bJP6CbE!Kbo*EE7=`xj71VbU4b4ILp0=GX3w$NqD(aCq`=!g?rl_FDRqE9ck z*l5V;6r_>6ei%K$v0fW9?ZtogKS+D;T^}>;nc2jJ=Y$j0d-M}5At`?7+TW7mNrwGR z6;HCBLI=+6DtgH0(OWI=9+KUvrY>A4hwh-VfC#a=gEVKQ9+PZZ#eZ)Re18_5%_;y; z7IMawpP()xF3>5nLI8MjlsP+B2tZ*{6~Z0eXQcTh%ubNz%iCumP+Z1gJgP%QGs`gA z@ZWEF52j*aF(wXu49!ZSPwA8+T&25ItV*IUjnhi##V`JW0;y@7<^!o9u^1}P+tWvI z2Amh=FoRW`jra&E-t-Wz_g|JD3{;X{TgIzJAuBL#KslL=$g>g+(4ZU~Tx;-y+OGw^ zYCUwoNk>DMLR;`WiSME`0Jq_lFF|T_6a9Jzm%>qP-d2}RJBpT;YZi=reW5r@ZN-yTm#*YffvF(!^)l6^ z)6dkC6ZtxwAu`gA^+gywsE&P*+|?>9j(*u&CrXpnFdGpA@y>Rb1pvODh(W{L?v)hr z?8T#qh#99v0UvpP&4+gmJ%8^PY=BQ+hpMthkxV6EksuDOqSTJzKG0-OZ3))NgjTCyM>4IVB~E>YQupWo+w)XnS2b8fS0hSN>h6K7H+E7*hPWODgK&;$5{T$g zjTT;Q!;obt9txMpTP`wG{Dj%&tQwgjbOKSJmJmiKZ^zhHt-386id?=K9RS$`;du(a zP^1ja#jE7sQ4;?UA)VOWYN|6C!l08T4-Pd=9)3qPQov9%^{9E-z) zk)WH*Xn{BjhF+7WN|Q%Ft2E)!hBg2psOGG!892PPvTgtlx6D~3WE7SR0%w7zbcVs@ zveB;~fov5*ZE!mof(g30Rz)MQg-q zYhE=qO7;q+X{k`}T%xIf3#E3Uxir&!nr50|Au`)b%ZsBS(T)4bWteFX^H#Y-5b{$^ zpZD!xFm+}IQ)g;0b!0H%NXgpmE{lKJw!7^5?Ji3)v`-xK0(w)M>{?83S?HE0%@0A` zQsQv*W$jODvRsViv22BEw|If`hsO(Ot)}*$YHE%iq@&J;E)^b?dk*D3OgT$32qB}WRP{T`>C*k{4JgeHc44|xLIaTH8)Fzd*ikq_^>Rc z+Zh({481>wA3zfu5K8>12Ad&5+g7Q1Yzj31TU5}NMXW0zUe!b#;{$>(C( z&K5UPh#FYpf&Tnm{tkoiKxqQ+UI!JE8XF3gW#$K&C%>S8lL7!V({M^F!9UhitBpWQ zBg*cu9d_xjVh{yKOFv z%_UwcM_q$d8Y;-T*kBS`*Y>Y17?+)nG%H{jr3cKN&zxBYW(j|r=`MO_IccBRm zkV|F!GJyeBW=U5{^W&dUBM;{Cq^UwNVFu~iBi=O!U(jiX2uu}|rkXTjrz@d=2vs;p zu}?d~Z0m%BR>G?|?e4)sIPH#!VjB_gHhn@y9MLCuxYlGwaP;l^#L;zq;^<9=?y{&70B#%TkfzD0VhM|JwJ^|}FGG-PHT@`wGzGO-~8M95*zD7 zSDJ&6VS(SB4ruIdtmulS8~x^$aPXOp;}hlVfCpqcW28)F;zazm3ZYPs#<^hpf!dK6;I@RbPwUt?`|$Pm(TQ4!!mT z!o_C3@tW1ex)bzu-DTr0-#jFL>nV5`w~S}QW2#dhzl~E5qn6ycCv*?M7Sae#@6jZ#gm&DJMpU+E_(>bj2zro{Ue6&@ z=IJ)O^Wc~64ZtsRbeo|Pi*6Vq;!>mrr!Y0t8NhY-J?0TqQFXm*SLt?m>#O3K7I0F8 z3cv}3BHP;lP6_}gSIUj;P&Ws|0jN!z)vy{4KKpdDDw;ZbPB;)&!3K{Y6)eJtSVtWW zj1P)b^7uqTMY1{gaM0vIr|pS@;n2nAU|2QY!vkSeCD`|cVc0j7&&2$p0zsDJyJ!w< zIFKu4I3$3CIh?e1&-QE2*e?c81vuQmT3e!}{hgnXi3 z?dD&1>A?FqkmRJ)Hy_V#h6NFdJ9kyQgJI#SeLfz%bcBT%CQY7x6uH$ss!Yt2$Pl-! z{C;g+g-MuQN0Gywk19hy)d^)+mo+WM2LU@FV4wsUV<#P0+Ue-85!^@L(LCYw zao3=hoy2KX)jv`C!pJUykhvCZItsGSK9ZpcRm{5E*Z5exo2o$%pI72ai(OWD%0~-a zt@%TOKw9daC-u|C;@g_P z33rzp5hZlv0-!;)U0B7IVg{9bEG^Mm+Zju)(B$Q)AE)$7_FyQtcpp@>6^&jNEGUWJZZhoFsVO=(Gi-Dk7^6`^d{B4y)xO>auAqfwwEdvORr70h3 zJQy0*2KT^fRPbILYgll>UwMXkBVSdE4=$(&2O6E~Xmj}6J;4C3vaK)7RaE z%00XhFiZGglrMA$jY)hdBjVPUCQmVLNCd|{K0Ivc-8-zuEIOgUa*zp!N*mBH8z_`6;iqrhSgD_fO*Iv<2qvu2-<5X{J5OMoHXJo1At-iXigS z+L3|LO7PAwl>}5sN2+Vv$SWj z(>W@E&3+m23xzt;IkH8bus_?V3GSp!lS?Q4QxN5y^dGc87wUhy=sPAK&Q9cUGDI?3 z1@9q|SH7z_8yA(-|1X{3m~0{?)g@wA0rRKgTc6^~W2>y}` zX-MV!1>Y14Ro3hEdahT;OHga9NmF=m*j1Q78#la0Lv)bzy3{Dbu++UZPj)x>(JzuDLrO;KSKP@w3Le|R#b$8v`-hnUospX*W}hOt zkIqp3{#!-)&sE&1Loo#9rgb+~a*Q-us}l#yihQL{Oi9>rj5EKQthf|1%}S=RO|scE zX3DR-dulPPcqOZ8zNb^9=3)#iRIaM3%ELRZd4}LSF_Ak|6U~-_#;s^vaz)NBg$k8d z+%=@^0sW0(?oE(h_8uU^y9}=d`KE8wl#exj707=ISCZOVy#uiQJGYvy9U0k$^GAAi zWcm6d*(%kiq7U8Gn1|kKEMTsqR!~c|Yg+Bwqi2^d18jJ~uTs@_-Q7b?yK$E)2~e|U zEr?bZ;86)gSYaV9tQoKy8!30l9eKtjknc323<#*Q2JgzLNh|c65Ib!7CNi)u= zBEZv)<4xC;#CM}yfUP_&Q?EjKWb#Au|NY{wJ+Ce;Jp3IVGgJ5~% zQ-$hi%vAZZgDkJI{kX#_;KWYoqoAV-$@e=GLwP);UL zE~%nnsNRtnE`9vt=k-)H0X=%|rh8vQh11!|LW$4fibmTp?Y(wP z8}$6KL)^uVDKs;G?C|VZu#Xg?wtF6(9aD{r`Rh*1jve29-Tkv;r>4hp>!h=A^s1Ba zBa{HZV2<>zek${cH-5@%fj%tG3X`nThvhp`=3n++^BHlL);ZG+PA$ukmI3#a%(VFL z{!R&PClpKqY1ThqT2+zbsi;W8E@yEJA<5uF##6cSE?MGiKj3z)5)NhbEC(W2wHx^& z8iBdD!#kSJDzx?&B%gp>Rq@coD&P+uIp+25>)3U#D2z94ogI2AY`MVZk2!lr z<>um_ztMq@+~~l^IFL_om8V;YB9j0|2FfHjf#jgu9|qTUBO6#-7b#bDn&Mby3{02) z48;@V=0!1Z#0TVtvI0S~v*n|y%{O0!?wO1?70=>aA(>zPsDjILg=E^|L#dDqhBr{%n^??`Mkr z9;XVS)>`&C{~yQ2*OaOs#+bDq2R=d>>F&bVf$-BHc=y*fg~-!RNTk2v2IZFO?q z(lXW~Sh^@nWWDj5V}m;iyRlRMBhP+!OHgL_(h8`(DmU4l1>j}$eZ}or0bY)pWat<1 z8;E1=i6#~miYSicPi~CUE{$9)5V(~95_Yor-(?RK>+U~k%J|LfJRc*;e8svut5m*X z-EFAsO8oPG>5X5cjy*p;Vrk&Tx_d`D`y6LCJ$`!U)>eZfCELW3^YPhjScpD+5hQ{N z)gBHGloQj^KR$kP+fP&i zdP-nnB%dD&-L3Hw?>lMsq0RJch{Mxzu{8UyGzTsrUa>6$lxO0KN+KrYj0GYH>s5&?_N-x>0-{4lb8GDafL zVFA7qmWtqqsbRQ4S@+(8#`ErTAX?Bp2FKz7{LDvu<{jSslzGIb-f}sc9>5un^Z;y; zV#WlB2-=4&z{pZM3i1n*ur?=Bo1}`A{H*&R@=>Whez2wP$pW< zqr6bLZnxXb@F#G0BAS>)KsDot-x~Pt~rQ<3Y+O_G6CzowwwQo8Hp= zSi5dn6h)EADX5^q&RcTDO>b#_u3fi;3<#v9T6uvrv*X=zUA2pxlMswli8bkxawR>b zN|G1|qxdNw|K$}zi-PLP%|G?+i|S_g4b52+r5{Us02p#~S3}HoWA>}Do|(D>-y~oQ zs64sk-Uo*T>`QQO+$&vpR*WaU_?)!4B<9_`m>>PHTyyuTNKCwUl6+hpW2|#y5pJ8O z93NMP!PHdqKCXG8(3$sfb#CC}nxn3vQ&+}X+vekfbBINcbvtH~i_NDaUPblHoGVZn zAW|{TD%c~OXbP8nym)9;VvxN-9eWR9kk^&Z=j#s>GDJ~qgATryHoq&7+@*dJVLBh_X$pd|rad~DP|@^J<@IzXp)b#p z7WZM&(@;RzHvkEaZmA_m0a^2pbN2EroeP|81ig@gt~nh4H!lXp{+;1Hsq3H2v$AD< z@rTjL5VlG3B5zG7<{V*TK*G*`YDH*%eD}Oia?VWF3YEhRX4ev|%;VY3*)O`=@vC)C z>i}86%Lpoj_y+poy1SpnkJpWV&hgJ#{yD`z$MH+^;RG9x;7)wIKk#MlhX;25@aWtR ze$Bs+4)tt!XpEkG4k4Z>Ct|@a9)P(udXnat2;tFJU4Hy3`Z>L19d^~_F8ws(C*C*x z!mj$OPBlbT($HgyZC?K(la~%r3L&{H^GT$H3fpAHivZ?50wKpS zlvs4>f`R;)N=P>=Rh-!*^ZJxyDW=_&V>v4rpkIpspE{bP%55E`Iu(x1lw;{F$I{v@ z38F+0s1&?-eaW%pIjMN~4yIS7T{77GLuSfXjoG-5C6t}?D^yrL6#M6+g_TJSg}PAP zF>8C?=7+W*kTr{$DWamX8CQzJ^tian(_k!`7oHc1z|aub^D#_6PK2relUf z2yh(Nwr6@i6|7pN-$~Ov!n-Riu>P#iIx}Y;%`PnRe}c7d4g}Fo^5}s*+L6g6ipxw} zE@@n%%BHp|v1drn=^yk|`p0-=Dc3ixARIGd4_oK)UlPM?I-_5e<{=Jr-2b-&Od-Lz zGkRGgpFt@ghmp3&bRTO%%0=Ryx01@bCoR%#H8rQs$dJ{~6qgVoNpWjiM-u0(oTIaHj&l;^YutpileV&) z%x0v=+#S3C1HnyTB2au1=Nuz)tGdJ?S9Ta9{ZpEEPL+-wkujuqF(Ht~r<}b^32XSs z_(C|1FBxAHEEC%JXezKbCENf-1;Fn)NNi{IA8%S^)9TPMBl^sUu&J#Zb<`*=_;?C& z-Lo>4qhB*_(bV}qzWhaXzMmzzwH5i;mQ#|OG`4|7fe(Z7**wE~(#@L=x9>L~>#>i^ z(3za{_aiPa07)WaAy(Fi*qxlo_Hpo-%bSc`$}Dz2wAEy1GUh^eGgcp<+WfWrJjgfS zri1$Uo3ylOTMBpCw58K#3ij+UOqN+TMQl`xE8ujrvQ0z_z{8lp5FY`(CFgX!82=p2 z?5>g+Cj+;~iTj!oxH_VRpje;1pQVQ7Be)n;rqI%-OErjSkn(ln^V}DMC{^&x*)EEZ@0{KQDPm0pq7#$Fen}BBg$TfhzBBvfv*>_fZUtPjlwWm%otAmmNz@ zrb}+3#6t5Y*cp#yZ=1wIWKXwRV^K;nH69VB##vc^)0?boXSPPG$=BT-r)8;Sv9IVj zL*KQD+HVD+iW6pSal$tX0u+OE^Z2YkL#_FcuVr_i%oYsEGqzJ2JH<`UILH}t7VmM! z+!X!cpTG4Kb=J<}^jtgr=0f4j1a|Es-g4tc9JXeDmz|MKo2eI}@AtY#YUWDt@n>n~ za=Vg9Y@M#8)>=kQ*F5?}4u`so(*C5$kIEjqmYo;2tBj7AR}niG;`TLnKIBIoDVo>Z z`K>WCTgU}-U1{4s>bxD+H9OAn6RYu*A9eYS&dF!zV9)VHp1tEf#q6B&M(32XbE+Gi zQ_arl+~}N6d(OJ+?r05RNFwA?4F~+Wavz3MvY=1;F6(_E6mVnuP@KuAnYkT3;kuYmIEA0ZY*!-Z*55+rIDe+$G zCdpYIf9hcXkK_xU(yJ zyABd&oL<@Ab+A8wu-s(c^?vdJ5EY{Sy*#T_ zX|$) zT-b`&kb7pErd!We;g+6~vw#t5cA1~MjIB6bhG^(quqz6#SVzR!LlC7e;{&uOpaXGg z$m6~bUD*iM%F-TFHWf{^$BfJNzVMc}*W(nGz^3WDZ&TTAs&t!z){>pjUhH|}?Mvq& zok8eN?@`a1(y@DV(mhJugH~H-&E2hN?>2p-Vs@J@Rg#IrUx?e^e3rl?x>@jX@xs>D z6|w8Z#q;3rE^JM^p;v!~9g8WFyu;$!0o1_65%!#@)Vj+^e>*Rg2Y%|NT*w`K^x{BN zx?lV&cdGaf){7BX=5lIEj9`o61s{Dn)%wjxzl$W68IbD%@%OUxr=8UopZ~$IO88~+ z2Zt){MYe{UFL~$Ykc!#77gv<#dvKh)8B!4$Cp{!b%fJPtiV)U5Wy}CN0>7oi{rShv zXP1`qk1cQDJf6!gO%4HM;@o5DA^N3#E|}vw6!q_StkV2*MFJv57oRIs1!lwDC%1w? zL-ObzH@+cuDh?ve$q587m?AIyBv2FHYr{0DPOJb+lzWMNi;m(pC+g~ zmc5J6G1<{F&aj0g8QIPMKy(oA3T1q*m>hQXD@9gaiXP(u3BBf*qkohuqc?19JUlL= zw^}huVGHG?sbaAy-z@M|riOVM%@e)mU-dyIszRk-L-VtbuPEPHZp&AIn$YvYm9P2V z@rv`Xs(l-5{R}z|SxB+rpFY7_gZ$I1j}+(@SD;VM6r4 zD^!3!hRS7*Zs_3%z$w(ZObpHv1DC;Av<2(YXkGjY*=mru6i0co|&$>hiW0MJ~cVamaxW_fF}*n>4G2qDoQ`wPy55m_CvS*A!egrp{bm8X!Ps4JhTT# zzlq(?q0#8qRL=_R3N$0$=AZGd*fdN&6StP8p(1_lH1sF^D}FZrJRPSK+WF^C`meTd zxXV_ZE%?zVmCQ2jweLhE*XTrT-tMcO{9I`@%32k$8HP*n6pAkT@TR`Pt(9PMdmsv+;YOKoxU8KEWzxR}Fh=R5~=e(V;w^+F7yN3B)mL`@F@G z-;VMzZ*kQBurlz#$h3LY-9iB;n!&m=)p9g@+`JsSt*vuUUF5VYS}FjGzmMhbH!fKA zYnan0ZDugwL&+pe+3%9EJ`#EwsM4hp;EOmgx_ciAWhGG-`#k=70*!gfiTMU8jWE>l zj<6ey{;@x3%GWpWD}U=X;H;(8+;XFz0h?#i=mG7%(v9s571Jh35-Pq_8jZ)XW6~e7FVV5qB9`{lUcwmI&b+NjZB{yj#GtS2?D7qPi)Ht0d=5D>1USUVj2x zrS`}8i7!ZLW*3RirO%HIUHqgzk25~7qex=`l=8;U_4rR*!?Ha;{e>R?>4%9J4CC`( zAPMraemsYA$N22f*S9~{5}(ak4D$==wk(FZg_Hmd z^&b{fR`Lo~a#Ric60J*vGPqbuEj3J;l8aG}6$-frr3DZK@Lc6?@oDr=sO*suNu@ZX z!R|G_VA!*9haLnaU#0wY7Q{ggpn4A?k?klG8_a^3mE49KeNp&~QkyDk;-1DQTv$r2 z=SeDo{z!2N2f%!yW*#hYsyi?9yE{hlHb$pAv$}X#Fxj)BusPgCQeWYmlOIZ%nwe?P z>+T-NMzlJp9<9a}6rfjn)mlQcao6t676Ym7X?Rjy`Bd#xlE4K|SL!z5##Y~LsYvtt z+iXIAC)1ADx_~rg{mQ}oO_%a#vOo707oX2Io55jch5ru4E@|%T?(I|V22D3_)SN|A zM^cFUi5O7u*1o!v(A>J_uVrJY9t7Ww1L+=1R^$ejqG8N}++f@{QmcL}NuMu5o+yso z82V`b==uFrY8woxC3J*bhlVVJ0XbE_;K%(yqSSDFpbl_oP9aw{*$K5luEqxuvaNN# z!hlxcAePjijhPSPhkso0v%xH8-cO)~!8B$Lom3)dT+xhVnXRt*CmthEvl?G>lkp;E zld&Z<@F#=BBk@pIo((I_LRbv75|4&lzJk7969rq^qxex(3z+1*#t-|7Dd?+jue0xh6#KV*EVRQP@UHZQq!+@xkA7xB6 zGGVz*XQ=V%SuK?4ViT5EDQwVY=PifBq~`T0j(NBN_#65|A9kKvkM9bDup9>$VzU_? zx2R*b)qEGYs5)jBG^o_njLT0oOA`Sn_(SIl`hw($rcds^YYN5aV z@TS;#Y^X$_%z2l)!j-Oa9XEbkxO-fMH^o1^cI{f{eA6Efb9&=#SL=*oO(SrzZ@W6^ zx}E>O`a{|HZD9yu%MfouKfK9G@KPfu4C3)gbnzSRjT#i@D7x{!K|!WjAJ)3ZR>ytK z@|g%){8pv*;!JA8i}IGRK+&Z>lVYMWc~$7cY*!H-uELuVTOQMkNAKU!#+)5&U$~o6 zd>1C;JN#KbsS45^z>fhk8n!^w>OvZgXd0d5I zSUT6-iF{KA;zBsYO)n_@WMOAdmMuD)KH_Vu0g^_{c6-rx51UXR_~*ZZxn_nm8&rhR=!IK)k@ukRf1 zV}91tUeoLji(xq~KZWmed6IgXlHK2>J_kU2mwg&c!rgpIm6yI-bfG5fc5SbfIABy9 z5ops*w%4%8G)LxY14o394~5mRLZ%D%CLAW7WYq_gW>kiKNg2Ne>?=SwIFg5EjX%emBi}!C;Ye5u?vMEg z!dsuYxUs!mT>L)DgSBX>LHylMC<2zYXux9AEgGh`MDUSH=895+>i|aJGp;93=`SS-JoTL zmYHu;mW7zYRm=I=a7S3CCXH+EYPTenL@dMT-9sOSi7WXoD#VrJ5wBrFgN%cznCyqU zZ4T8jdjz?@7)Bf(K`Sn^@p%t-Q|!T*_uf@)ubx^fhfpvU!V12=QE2#+2DcLqh6Tv< z!kXBhZG@B_+HYxGN}csqCi3Q$XVNJmdEgZKa=gFRs+PoA#0Zg|&~J5adW(FGEPj{G zj_F9)G?;2qYkrh6Bhb{}%2YV_Ow!kt)ZZUwR=mz^oUkt)w8@Fo)pULWwaYSD>|I+* zrd~SVa(q}25Lz>6<-@D>xbM$~15EM$(4k53mRwzE$%?R zLs2#g&#ycLe`O=PIQmp(Y>X6`_u7S_*A?Et@CiP?{_m@aL0pQn@qfdv_wk=QnCN1b z;;14|6QfSW1Oc4dQWNoOD3qWkYLg0V02uf&B*@uBwE+p@m-f_T&SUmsx0yHU&)PC^ z>+W&$vdP@K3y;yK^F*Z~NO-a4U-MScu!x@*(B6C$U(1GB%#w*ytod*F7(`_if8(-v zO28{{k2kVVE_Fpx40qmL<0`)PhoxE>KmXNI(Xgk_TlyVP!_o8j-Kvx!KsE+b((vgi zNYjS8+Q7$Ls4FLp3o;X{Rb=rc2au(WbhQx>MY^s(O<`+#N51pLIm>+B)#T=#8SVz4 zin$nwphwA*6Y}`qjs9AGu&#e|(Gn((Z#F}U?aci@891)#re24@c+J0*;k?O0cVqv? zLg>cNX+)ZS%pPj|Dw9ucMHey{oBYd}D%!?x{8AbJ-(M2z^+T(1e_RNDRdaK8BCKL? z0Qy)=8b#*m`h&10kA6HCYKg6jJe7dW{a4uW!esP(rmNO`F=QSW6H3Ch4I6iitc;wL z&bY7rI{wh_*eGMRX5&89V=&BT1SX%|t~3qIHPG3y=Hwsl+JdXw1($?U7HVFKq><1I zby{>i@lsv4=sKZei*A5lOgleiNqbs!HH&W0z|KY;&!)q#T5>{w*DN_J@F`5D$}a(O z>ZK(YdZd%Pb@#Q5B9sI$JHd~BS20MRh6X8)N6@>{9 zyx161)nMFX4va#d@G_dqbZyI3^_sT|NLjm3MV;|Ah<7v62n-bd4IlSSCq$3iMhK~F zr4CrB0};-|yT-Xzsv?1`)GQ2UD>Z9ZYR*dabr{$}Et6!y$9B673Y15{gYa9IyAp5I z|J~6f+9~02U6Xn@X;N?6zl_At?drX}l`C@{MhLu^jD9k2dgxf|?mtHu2&}UbM_VWT z3Ka=DAs;=TD^W>&zuCIGXPkv0vESt|oQ`D{{X?q|i@Er}{%|Q3kp&}Cy#DB~=fRDB zDknH=xBMF_vOH`P^#7;NmSM>Gz7E}ummSNS0ztvWj{Lq1+F|)n4s%Wxm-pOd9fIfZ z4G-_vY0JC1DpcUf!KzOB_X~l-B0LEVC-uE15ZfFL zS^SqS0EG3UznLGjLHcyYO)qSnq`GVtM+Ze9rTE1!^n}}bk}%U#6IIWJk+KNomh{|zfPcOky_g@wP@=>B3jUQU zW~0BEpY)f_9evw7a0>OT`KQ1@ulWyx81?b@hy%zEIQpyhB#@`zbe`g@Cj-tX2CZby zdGX?-8h}&x8^8BQ`F*emLiU{k3@Q(q#cQ;_L+e8p|DiKLbA0u3H(p|+3R&DHD?AUK zY4?!2g*lb+SjS`IzRd{j5%chm978EGQn*4$Ihsw?qU3H=u%*!Z=|8`qa_SNOFh{<51iVRDY_&0{cL7CpGhiImNkO%BYKuHcS;A_GAJo z%k<&=x$?gM)W4fii3K`UiuHKl1|af!CdmQye+Io}G^)w)<`@H{+M-V$DQSz^qJV z6&K+)>hnGUO=XC8XiGW(?K~8>*%%H{FSQ{DpD^@zT4JRBve7K$aa-s*u4pJ1D?$E) zeRgY2Mc;`dChuo+VL zTJIj+=yjFY-IJ12)l&9GROry&I1C*+!zh<3YXEw=$3~V)zugULH}pa`RA`+fj3@O^ zO$z1cbzifX6CwarP^?&M{yAAxD18CXQI|pN$<>cPhRIlpVo8%ix$yF|bm$m1-L7ci z*WEuMPtppWq!4kJ<^4AM5?%D8SLIVUOyx=}KuXh7ujZJ$FXrbB3}22bmO=^GE|sFU zsiM=NzPs*pCTn{-O1vsC-`gyjvMc}lFlKtE9*Nvb)Z3i}K!#xn=Q*x2e-@&ssd`Us{x&tBw1^!ek)AEroE9$?sGxdUcaq4wZ!=U>XwtqL&V3~_S=T@t zMJ1@F7Zx?WNfBkr)Mv7L`CA=x_cdhczuH;!@|o;}KO08u z1C)U`{lL6PxvFzS4rCtEAXdw5e2uZDzN6dL# zfXi}V?YFmPFz?Xu<=J4~(GI5dBm%wU+OX$nF4<__#b_=KGX`*}9ky^+`{@<~h&Qn& z_(`;-jbnyIehovsg}pF3lbt~G*cWy3BK-*0cqvSMl#U=CD0ydcN-?D@=DMMN%4cyqPxP^g)? z5?&hn?XJzQ-Q5PGu;xQ)rIC6p`L>LewwICWU@;Tu(nY$xXR}8BO=eDhHuT2bpc*@J zop$l9cNL2Z<=ZniULp*^hB*vNybZVN+g31S7eECF=O0wiizYV1cAWPSSPY9ALuW1O zw^D`&t3h2gsOh!2-q$r?`uO(Y+J)vGEsy6Svmp#qABM0Sa?&-wN}0T#a^b2B-_vxWOPY%GHv$3Qi$=?O1IqK%m4BbW+ zsw*E0D<2uJAkTD<)UN=!>#(wM>7(VwrTV`73H4{%tgk(D9?dYx_xp`_aC`fi@sLhm zfVdgeC0oi z)rT3(4O|$+Porf}Nn)gZZoH2)eX85BNYzEAB}|%r>@$(MgVrNBg?{X^ zW-V}qs%s57(TIwVa8KydwYX5i;v{xKl@b*g8MbJjB_?(EGKxLukuOp=E&})f!!W?O z`XY{zOD6#g4lp)e(O)?rzsg5<(*KB{eR%YZmwCKM@#B;JlbJy~I&1!Z5D7HmIUfe` zyHEO0@b|4J{biU&-AVruzF*!h;^pmSrahj8m$?5R{s=bUr#b!+KMB?7!}&pG9rI>@ z+8>|W_UE?<@D4yZTJ^fYn*WH$f#4oxTTHz=?uUhq<7ij%Y@qJua+e z_w;rWN)RG?t68OpfAKZiH3cAUHV9_KRt z@%)ot{R`REpUJL%mWsxxhO7)|n2ExxEa8Ex!)~u%4~7eiF7rA6UHxp2y1sA89#>NPcZ+rN9LyvAynDN>BG7uR^)%Eq-f|EFPPqqsa?NkVyLBm3BR zCEhLHeOTETy~nh}*tZLGerD%WYk1{_*f}4w3-R^8++k5pVt=!y%8pdgMWKlo#p1De zEIZMl&>Uom4$v?5S)`GoXY37JpKq3s2gD#yHBE5txBS+9VZR36 zwvTGula!nwT5oVh#TkYAS>+O02Q?HrwK{bKMB!5Cf>pSn`^aj+W|^5Q>#iyzo7bca zeehoDeKU2a3449dLpC(@H}Cn>(^A{_>S>nBY-DcI@5kxU6ABJ3OR6?;UTjufh5LyB zcPQeQ-RQG9LAnAz?y;;vqi*=TMwvZ+T|AQ=CVs{5mLW`Bz0GJ;u@q%AE|Sr>XyBh* z8I6+aUt&WTo_1^udy2)E#bRr_$>_HMoLlqC&>#I=u2-aX#7;wc_Odljclg!@2=Omv zxH$7r#@}{)|4PQcU#0e_pEOorWycw^j z=Q88rlI7`tgv;}V{2&`M%hZ^8depYZ_aNo)QuM6X?EHAiWH6k;BlG4H5z5$kzX=h^ z{uYjbHeP9?S3Y+)Hd^ZwV5%cSYQMEx$wk8w9Fth@IP3;w*1KdhejGG)J# z*-}|xK|b&C^u6RKBF7I4obV#CyQ& z$cU(_Z|{nNQ$=QF$fQYiWlkGkE~<)qvwG=hkFFX2m_KHns5OhM&gpTMV1!aY$=EFv zAf+H+j{>8V7K7bww~VBMP>#21DG?Q=KyZoywYbPV$XUPV{r0&;o=me4X3Z32oOAZs zXYX%+m-l-w&-=Vz&n&_dj#|heWxDAU0IuN}M?)B*K7k!0!@ahs7a>2s?@ycl6@LnI z#?zlw{h6sidVKcAXdb&AK{|05kx1wHLm1w zwArRI%r4y_kT#Z2bN_Qqm#^y{6R(oW#&DAYooW91u6aCa_~e3rhlRDP!WEMrRt2sf zm%4(a9Nk-Rkq^J8>A$WUm4$2irLp!Xc2}Mpsp&q`Z>Ax_$| zD%cG(`X;rm<<_tkg1o`2=5oBr)@Zo1`lxQ+(? zH+}(k<|EO#`12SPHztSc*Wt{t3{2dY9oJL2S5osiZ`uZ&@*-jM|~(S|fjbGj2lRwmpWz zjkYVNfHzOXiuNSCt{GzXG#5(ZHfySlFzek7me^+=3u6WgaNwL=K$>+?Ecgw=H5q+f zz;%c=)9}5WvI7KwtqVGB7t2n}PR5%YL7>WC6#1X~iH6GvJd1uZ*_jyGF7m%JwYJ3V zwn)E8xxr$ykFOT~q;Q%4ck%5}!2s=))Qn1Yeyw|a>-|t zvuATK8?(TqE3v)I{4-e?+lcTrsBSQjyExkECJ{)H-P8RuZdfxlGeK zR??g_B&+rC8L}{)aUZlR1yJHCHVRy`)Va*>{pg;fML;~NJJBNg z@4add_b9{AF0-IV(+32Kkm6tYXhZJ>yQLiG-rn9`tU{yH9e|;*$dM%w$1+A{FZ23O zEVxWk3fVK+NT!#$tKU!&M}T5cE`)^MLvR-PUo^H;dV*fg^B-@;!@vUDnZg3Yeqlwr z2tze3FfQMg)?Va)Za0fgS0V;i_+5)-fcDm0jXBMLkib!N$AHI|2?j%;+%8 zbw}#ApiwNGKERvq2AH8CxY5%QKzMpm#EQ}|GQ&xBPehr2(RqI{y32T8C3g&J_-vup z@q@=9+N<#GbRY;VmrZUz84iW+=OWVBc2+``u4srm{=iRKi1k~*+7OSn=mMl^0IxPs zF>tlSzF-2}9{o`uyOIXRGF&3aAT5ZQ!Z15B^Pzzk*PTfT#bZ7B1``v36iCRD`O z3U*yed*8XvcglB1jsNtoV8<9@F14y1e;ej6s-Tc9NbI<{DCMROn~N1lWx*_Fju`}T zYW|^}T9zu@1TrB<;WiDLqvkUMmVPtYumlJWF zxr8fxvNOF(hVP-Eycu~ORpC)U2<8hsqL!_C!bPjO8wMC;IBGCRtzBiZtK7ztY1bKJ zi#LeWJS||S^}@*JPF~z`Wno%2p@82g^JG^$WXAIt%v?^y30Am3qs|SBybQTzyk=lz zTxkJ~e9>S(L!QhO<#O4@xl#y+LLVBe`#(|ZS%~FG+OQ9KC1=%f7NEIvysM zkMF!ax>xWF;zDkjnGI^T2V4;&0m1t9caKM|we|3~qzv z8L(``LZzjIlj1$f8R#e5JQhNk8E73zN1rv+6iE}OlZP0vS9}`y_(L{eedfy&Wd%0C zNTGR&M{ZzXmB9p`fK^*A39g1Gdl2FhW=JJ>^m{(iU~@|$Wfj|8ruk}6cNC>Hm5D^si1%Pwe!s&7b~z^QV7pa(Zf~e|`S+-yy(nJAHrt z^hf7U-=Cb`u+zUWfBJ9FpZ<->=}kNR(#+`*DQlBnx%3lF(gX(V<^wWkB`Jc^WvMOf zYW1SkyeBQh|MgcuQNKm=hSBH|%FvJ`LPYKik~k8h0JHR$z7isX0-bT>ef5z%cZY1r zfp$r@-Qf%!;c)RcNi{GFVbB|3zYJKsnghj~(tbZ^qVaS4i7+=MdqCaLp&Pikc#9g} zC!_XNH2Pov3XOi1P4)qKe4GW?J9ZtmWSzRlsXs52c)%~}_RABPcwbHo7PBT&Tn;P| znmon<3devPij@$t(lsww*S*LM{qMh&6$kS1&*SM6Lac!5KyMEi4@_(-iJL+ZKi5L6 z(}V%URuz*n0L-uI6c$K$H=j&eQO6`pJ8ND-p->%~jsyy_|0HgX+7 z8C*|4?%>eufs_*7v90$jdXHDK?K)$0yq=5C#CTX^A?}{;`u-#F*!A2Iwkqii-`nTM zeX1FW=!ePZ@-cdo!v&~p9qa&%3TqVI+?3LX#8}uIC!JN2d+I+oMkUD2dL* zPj@{n-YS-0ACE{eSXI4w5(E-OyM9sbR25%}-67675k0SrT~^Me=JZ^CIw2oeBi%b0 zKi!r;ih4E~Y2wf%`=rk;5AlUnrU|mAt)+k(J7>fwLZQPLF`&<6#KZwgxr z?LYHF0_S;Sa&4$Yqi5*E+;NJ~U2>;uBJ6D$8VwWxlk39Rb!gjcBh;G(3#^mvy3S3mJ`T;;43*GF_*7vaKlCc!98;(XXE?ss?d5hwYOR+UK?W_yA_h+6rm?l zrN|a8*Fh`$MhEtL2=pEhON52hY31_<#Y)-`?nmwrBGI5#Z8=-2ShHS; z)MVQ&0l)-j+PMQzNZAU-WQD!L;4cfEUmZny0M%G^ON`Y7FL_MLm+Pw6t&aLrNs@Q)# zf^@gz|0+5YKgIu*NaLuM;)DR$T-UPk$=E;eD$*U}3stP8peuJ=>$&7r5ExCy;--%M zQGP=8tCHjplCHwZhF9*Lh)#pS0djP~3?prmPvBch0D|V~=}Hx|dRR_d;fXa;Df{zz^ZkCRqJ6wtOKlC_tz`c(yiCPDzqg1;D-N-Q|x@Ai?Lf@ z0IfO`Z{_*9h<4c^C|9s|Zuc4K57NDwsG2~v)K7NE>BQnR?5X;=Wm2&*sb~2J%$ib_5gfu2>qN#Z3|PRXd>FQ49qZfk>B)1ZE#= zP{%FZIdyVr3L&~%VL@S?A$w28kvjxzF7W8ob5{-|CKffBEycV?9+hoD?4$pEx&0y6E3rq)XW}w)jOoN9oGzz+0l=CumhrSoUU?7gNod3Wq0Hl`LVso zJN7~s`7i8YbrA1=ORS%1vy$@`Q`>cPk=vuw&~iv^=@=%RsyRN9CSXj3wS$xSAB~bP z#3Zh@4G^5Wo{a7hNulEY3x*Kw4Zrtq{j6z2u5a`*&vJeLjt{jUS??PI+;(7p>-+5; z74W4BR?qeNUCc!l<4lz96`#kUFGJ$EgruNrme;D_Qmg)vd`qObq9^}439gp7`WjqL zt&q6d%%l!H90RcMzh}jO+hZRScr8g&IE=E;rLC&^16&$b$!Lq4-T9zYjtxmWy|#uCM!PPj$;~=s)#mP1g@#h}geXj2-~e z6k)i#RI2SP_Do5py-{-9;Af4~NgF3nf8rO;P{<~J;dtT~j@O08Hpg(^KVl|0Ofh)o z=F?JPO6ZM}Rp^=M9@jG*#jyA-FkUSHW5EGR>fMIw;##$Y{~u6vdE?0XTt0cT`InGH?Sx-FnSyw zb+CJ2R71m=VE4cc4I>7h+$a7Vn8@wXQv5*$$}dnTv}j z5(7yS$ghN@%t+xTBpMg*O~aRp0WmaU zM;bly_5VtG30g*SA|jdrw{B@vGuUktoWvM{8BF9I!&#_^>O7jF$=#4D6os-?LMW@P zdsrMradpiVBOJ|tvX?AboW$)^gBs=1t~DgX6+aOhKgG0P>+kE>WY_gj23o`tTE-cw zG9HjWh+L%Xv&b@kV*Frv1caep_hqxpAnL%zmxh1MGZ!*#g%b_E6&^$wLg?im>QeWE&XuOW;nyEqgV>qC|N)F}dAJbSE>CpD5s;R4oS_ zJ04dG1e*&4RIbAPAq~^g8Wba)aJG=d(ACbvx|Dymo+*Sjp-5zgaQBwY7@|++*Y8tE zyQSTrrw&OpN2HOFJ#olPeJ)H2qgqa4pIcGO`$7Wo49Y9*3wPVn6xbxE3(z66xmHJ{ z1cc}*3roYzY;CwcQg1%e8ceG0UHt~zNxxRwW;0ZUxth>81qD@!aR#Z?1AA8HCL0oY zQ-UhuOyPIf>#TA3qSzDF3yb_D^m3vn8;zMDS9`9`%yZRr3NsPp7GtOB({lNk^^0X} z>Zb{#gon2Dl&dUYLL&;T{PK6Id6U!t!tIf z&|NeS?xM2{ao4nq&efPQeFtVJRqc=t)27}lNrVUYPEa)3gkhRaM345y? zl@I=G5WUQOd@_v?E2ui47GZ$u_-yB7Jgtr$uy@)FnmdCei2!Y}+a`DPQ~yP6=WK>l z*d);QA>b1JQxU}B1mo=J@95~~!cppJa_%4L=oi9Ks%UcbQ62sJaFqIy9Q|V*{a83k z$w-d=rjC9l9HmYsM}H6RDFDL>z{t=du5mKnlpB51yHZ33dK4JA{O2Qq4XH_7HFcQ} z*3C;f8i%9SyuYJfT6Q8z!qGp}xt|M1({R)t@RQ+a7LNY5&iw>=2p1vW$WEvXAuWHO zT5g9aK{y+#6Wn*hC4zo*aH^zW#>v7=@2QeAF@hxxWEx4G?>R*l8A#VkPFtp7RtNP2 ztUa?c=!=dp{6|rA6gGnlCM@ZN)iyxpT^TcYrScp4)q{d$sZ0_CI-*^;-T=q}1;~Dy zKub@WO$$Tqb`@iwZnlfbU^G^z38wa~pK7|cRzIsp4$kgI2xF>uqHcv z_x%b(e!oH`!V!%UoaRC`LAybZ0cB7IWM*JAY11^Dy}c#8Xqln=$WRI@wiH;P3gN8b-?u61@3k4eMwXeN@~M?j9j|I#(8_*H@0HUmjT0|;3lZ$hGy zSSwn#lDkO0mL*@d>gD9R^*hN04TCy!LR6vrxHg#Gb#C?p)V_4-g{lSWu?^Pyp3}(> zu-fphw2SwX2s}zwK$4LoQ>Ct9IUr&l6CYjUGE3MJX#r7)hK>YrIcS*#sJ4qb+~Coa zFn0oO&t{?g(yK-aZ3A+ANU@~Wm zNy#v4$~18uW|zOdRB4MhS!THn470tEA~9e|l!81Fe}kUCP6t{u zbGo24c5wdmrkxJ7ram2LjU5Q5VzrUG*feP6%wCPbV}0DR>-^LMz(HlQa$42WnD6@g zU(Lln8lSIvrf!hr*`%*2cAIVc*Sxw>N@7}H8*<3hnlk6xFf<)I{k8L_|E7>blha>2 zIlXJAzi$5YN25FY^~04p-;z5veN6H_>c_mZpC-FJm-5W+BBgxV>euUWnS`1lbIoY+ z?R={rm_PlGW?S9j+i-el^-s^Ae&FgqJvlwMhy0oO(|>>d>OV6%y=SMtVg7UjUxhAg z@oi{K-%fwy{ONN8)Z*K4`oK>A+4<8CT>WP!rw{G)H_e}Zpl9ARIlZvc-#mZ%2ctXt zwGv#m_%<(_3U~mV=PsD{)b9Z&k6g#!D92V|pGL_~q{fu;K6pMmqvXp1 z8&g)8*bhjnQSvG1~r{7`z54M>_ts**x>E-t0F!_0{z zJV1?o!gi}m0YiTCF9zaH)9na`n&iJK?XsFfz@SaQK-nd(%T`o8tYj5}_K$zCA>mp4 z4nFA#foUEJrd< zuFGB4Gx!U>KjmAJ+h)wzxN5M324>q0QKJS7bJ@i19@I^8!}^`vPIY^xj+Gv$=N|0~ zLXo5a;ke~T!%8KdSjk~(cU>hwiKwx-qBJcy3Piak=FXP?4iCCL zDqd%_o#qsQwzCC;fk;KgwF+$~$nqYZB)qW6xp(gE?G;AciR0s^F%obvM8*42V?Ilj zxQOG5!c|%oYM?cOSiYt)pC_#Wzx(7`BgTQ`+;`LzqECj))YR5lT8G5HKk~0>`b1Q9 z#xg3p)6`((Vlx%n0JNf+5j(K%8-Dj+l4=bKA{iyOaV9m4spUa7Us~)5w3UoxbOM5* zALKr_K%>*c~uaks@ot!SH>lZJm z0jzkadzPI6yMx?oOz4J`T`7N!TJ4%JHB0YnNYOnUIQ>kyo!9#s(zrmANV(;1Y>*QL zH?P^5QSO7*mD}WP!nPTYlC4UyJAcBZ#qW#cH9DPaVbD?!(|_aArI(y1Jqo*CgvCs` zKTzVVnVo8?J(_-kaE1xgE)TC`d+Zv;A99->43Oyjm5`kz2d(mPltg(z$-d#=v1JF8 zEJ?L41^)TNnA)M~9l48KTs)EzhK_3?C69PKN=^}h78#%4+q)Pk6U_Wfc*;VSch#Mf zh%{mG7GI#oPMj+cjd8G#2JC7b0EFRomv`y;AW(U7b}As(EIk$+-Rd_489 zRUt$FUC+<9dGARt1ytrAdZ=@A8dWFHVcL_&`JG8r-Re%d^c~gKOa2Ade5iw~GxEO{ zPJLlNb|>A|OWdvBBhEaXq3gNSKUUwA$)=L4vmEx^V3b7TD&vCpS#`zi+1WPlmGhJP zdmrlXFBEI!M3?zD>cI2+3GXJ!L92nF8uflv&ej;#+1UK3np8DaHKn(s+1B;tCtsN8 zs6^hy$95r^Pfj~Ls}gVVY7GzjyaKPl#D5pF%e=LZO1SS|(Q zOb%z;SX$|4{bWl&QPg^?_+YBjz*olJjjJPzvYkPfxy*m*(tCDUV;3XVSlEY{sN(5k z=YLc?XHV~}N{?2q)iz@}Wp0_5_H+7?Th?GW%1P}Oe~g-LXA{p|4e{JKtny_-`wFy( zkS7C6z~E+M3Y^8GWCN&>e^iCw6uMVe{T^NN9jC=>=AV9ATr)VoFLE57GdRd1<1S;% zfXmFS1P-&NahMrP+3DoN>{xAtx}cqzN&Yst>Pb{a&nA@eMG%a|OR-@y>fwkLJ%esT z0TU&TWu$x~{uLMei3Dwu{t^)H$vp&u69TsXZ#+Z~R0wqjlU=76bCbt;-gKr!#iP;@ zEp1O<5wsm8ZGB}4m3m8Bq7ow8qvTn5hH5Lk4Zgy%>I-Bz<@c=9h8Bqa2M7Mz`;6jz z37`bZf)L4b`DEBl0~BgggPt?!Sv~0TF(%KlB+J=BS6;39rcJ8>AHl#%$CM3o#xk4q z4ROl)8$cMNFjpGcu1*qYMvk}JqaVbrb~8%}$ajLb^~F96`}e@}f%-IaaB<@14m z)pk|jU*9%dJ`?*#nYP$Je%7nR{(*2%AwaD7{0ge9Q(CQwr1x=a;%(3$qIhP!NY(55 z->thLXrA40-@7@dCXBC{BIr2M-E%b0?A-5yKM-yDX4SS*kZ8K{VG13S|~B|7i&Q+OY9;nVoVZd zMMm|-zziT`1TUaFqU9rcWZ_{`JG?5J?mgHnQdLuz?MF$p&#sWd%OR$N>5(mtY>j7* zIF7jK5fv$_7Wb$<2B48%pxupi-rxHa(Cnw%8H#sdm0owiEjAJZ(GV1CqeM zky1z)R=N|>_qxcRb{Xm;>MpA~MjUJ3xWc~IBR4VJYT+r+mq}`CS3#>2H68CVpHjlG z1S{>vRfSKLUFzrKaao$0RzAgD@Bb(omm_z*|Ig8QN*g*=TJ>&s_A_G`xtxLqV78kV zsx_k6Qc=Y_vLSyp363p)d~AH_nSa>KZL~nwf`S2q)n%CB#Yd#zD>Va30=&7{tow^Qh1iGa>F0Ub-oc1spJ0EH^9$|c{t zF!n1umNJg{+eDnhoFS({>fd7zllg7M2COkqG~dx5kf*U>N(=$tDh0%ekOq~E(0vv& z0f45qVNzON^9q=hws(rRz%!Bc4gv1HRW`4Y>j#tIM6`TIf0%Oz)7bh0n8s!GSbzq- z01b%;4O&Eo{bFWAf0UdRoK=;7%$^=$`e!mKCPtHhu7bFefDwoNQ`Hd2v9YViclHT- zd?r#mtMmQBLVneCqvVAZl}7u0_moi*_4|%FT{Yd>xLT&3?g}3pqvV++9unkGflWjD ztkYMn?DBFJ9w8~&XZwBbTJ)7smsLZL`>WsU$zr17ttO^irpNJGX_?4@3P{X&iW-}8 z^;1SyafitNNrYl3U_=DrRs)^jI6e*bqK4{KVFrb()2XWa05{2TU7n7O*K!}{ct{$+ zX{0J|7GH+2pfO60%fRQtE3Sb;Ubu$;1MNu-luR~<2zY7(S?09vWW>tSOkpz;e&DS3 zXuIYn9T|rVT@k}BIBy&+L^g>OI3qeiE`9jE8Pb#H9j1X#pa8)YeIf!W zAVyOyfbJ++v0{}@<#}mR;<{m7#c%^OO}a8l3B7nqui=c0Gg1`~5l_S`l;+mJ*3UH9 zn|(1>p5?{5zz&=e*N7>~8&W>1zH8$1F^>+a$o)(s%r21`MrUHXKcPKl;i;Mn=Swd+ z@7rR)(aIks8}gWR5sXUTohXADh4-ZFp$mz5#S|YgDt{a3jr=PEztP@X_7Xq|HPaC@ z6^MKdFGzh{ya!VWu(|)gnaPdf!%4$sFo!iOC>u3%P?z8l(ti6t;eC;$P1p zoneA%)4v`h!m@T`?Z-h;4)uZB{y0gl(oo8w-xZm!!39~)%?sn|sK0G|a~hqC{&^p! zf3i6v)3WKv5F3`znnWol?lM_K7NCUi3|#!tP?vR)+f1bgwegv&Oyt4ue7G@jmr-_* z2RIt$q$$)FVGeW(q1Q zNGxu4F81HWspM&==PM@Pcm1>V2Vd@Py|=tIHwe%*oJJLtlx&Eds@0EBA}O7HOz6cG zGG;XTx*4Qyb~84XnVU&}Fb_id;JHhe(wCMekzwZ`c+!8n=VG_@a<_HvyZ-BSck4@S zy2*36^){bfU@eP7j^zz&ma!sJqR4+mp}BsbEALCI@R#26-wFj(^JVSrNkpjEPiZ;j zZtGn9;%dvC{O80OwCQUHpG@=9z!q3&sFwBr+)+%G6$T$ZGBt5BwbB#OGK&NEQ9{Oz zwpz_G4Ac|Rac#6A$^C<-a%c**#Z`?CuPw3oC;>B>uAm9}89%<8Q!!@fvG{yngjSPJ zkt6KdwOK&FRK(Q`(+UbJ^^Zh5E=AR&RVra^^!9c0neb4p)y|%Q@AA3rGVu`~4Y3zN zD|%JNOdR;n19H>}xm#sY6=!0=?mUrd4tz8<$vR3_r4De_j8RN2sYI6u*XE0nZnnzX zm6LH2t~O2QQI5`06qJ@3O-J0ocZy#~E2Uq7>vb!R0je@zU`rB<1I;SO;J^=yUr23j zZ-t`tnHw%Nc5p`aZjZ3HP&nSm>#{0aMLzaF_Ik8k*dQJwc(Q8lND)RK4K<`!^#D&1 zatbn&nZip04E(^@JWvdzO@QpZqQ-^nlzMM`QDzjlGd5N(fLg2{s{%F;qVl>Myxblg zLcw8+&_v*xNO6-JQ<_{Fnp|jSLP{%zYTD=8x1im?g&nDZ z;tQ`*kyIbCArZ(cy-Kh4L$9XOUZnt7?C)2nLWJ=TcB&1s(iB;;BP9m?JPvp_Pz#D* zN!4{3&7AAHCaFrO1L|O^(n@m;J*a1F=eBn;w1<%dP(p0O$kfG;*$vQntyi z!Kiv%lH+)}i2xZ{v|^KJ1-{#x6#eR)Hw~}{U8=iDUSXLj?^FH=ZzX-i^8{}GdUunL zir-DiH5=7mn<3HKzLtIJg>iiAtuO?XK@1_8N7b0ilPKi7Lcaty8-2KKSc1{#ZAD<7JCcj5|V#jN%JqFRUZ7zZ{&3x6F`-+VcL?iWb zD&>D^ue`}MRa8o@DS-ADy;im1v5jWo6nwKdcnIE-$U`t~amqG=Q?|i+AVp^qfNw~(1`Z&W zeXHVm&?(!MQx**sJ?o1h$cIOR>Sc>>q^qhCf`$h;zj9sZ$ve}NC zbG?hh34sOFQx#~BGc5BXD7QMLYJyw+sv{_?!ewCuYqz@H6L)?Su5^_=r#9PV_xb>s zAfVdUaxbadTCucs>DaM(0$tEogTDTa0%mU~Nk$KDZ>{-?64o9();d67m*TluKsCtQN#Uge(g zc9KUZs``wJ9S(`^mNKO1!ia7<%XQ~#hje`k3pm}YFL*Mwi^m%!?nD&(Pk{r}T|v<3 zvCfuFrhqzI9yG;EY-xf@nRd2aHZ099+d7p+a87m4CC^COOas^p!YJ`vYdkLeV ztBbu^l%mv(ke?JKqS+UD7+97Gd-M?B)NqbHnoszq=u!Pl^!@mzChXBed{dlMta7t-~leiR$qXT@?WOMMZC3* zm{MQgT`5i(x?HuBYavCxTo~fe<R6AF``I>58+A=hq5rSFx8|CnA=#|5;2MxL3Ty0uH2l7~P@|OpADFJgdJ)a; zZ<1Jbcd3mQ?7W8kZB}B9k{?*Sty#70*+qH{*RLg_1vNwWrqo^2WSp1p-?EJpN!T_5 zgo{$PH%L!~X(cKCDy?n+DMTRSaPDrnHGX=P=_|rHvCGiNAag-q!C7;|iRh$x#)5*Q zZ&B_mfh61w$DNQ&_BlbthB@`F*AaWfS@8*MdW4*5nAK6A?m!}Un;p;&w~B;5irgED z__wa=b<9WQjXo;ZeN=8(=%Lt@*7wXRgO|p!e%l8?tXI~$T2f$L1s^&}+ma(yW0XX< z##OoE3V-}cVadXSB1F1|$2d%D5AhhqLd$)_6>b%hjDq}%!W9;wC~%*yxB?To#|sMD zb)ahscO7M}o^m?5%_n85&&jx2J`GecSTovxKQv2(ot=mZ|EGfZ$Cy|OFpiSd;%m(S zff*&=hh<_ea@+H&6yXBnW{R+}H&47Wy!ooVX%GtnpsNexvh*L;J~_E1y7u&z8^zaf z`sc1dRo50TL;{)9a1oDjj2w3bdC!}!kDJm`4VLsoX%t(6Tuv{@+)vA{WtiyZJoe-y z%T?lfDa_cy?UvRuqd;}pu5+Wrpc+lO`FXA!mpAgRZ2$wh>c(+-BNU6r$G9K>AIIZr zgJCZGDT?^k$JXuG+1X=Hwdue~Y6<(-vD~fHB}P+aS(m1%3T-QH^#HQ{ODeCXQx%bX zfA*Vz`Bg=1lFAh3G8IvAt+s2pCH9Pl|4Y-oS3-A~HY9GJBUXO%?n*9}aL12Du5>Hz zNUgiH^t;I&V6@i zld=h@c56tz24TN`?ACY)sdff8k!rVwYS*a4j%%oPYpyX9cI4B1VUV>Dc5KvP$2C+t z&O@~`2ou3sooDO<_o3P~+}c6aZf!=jYs{*4SE>uLHm}+Vn@)GusvZ37y0NBmt=gqF zL}_gtUN!&&x^s3Rn9&~=!hvZ?1jeP-{i$izsfQSbS*c5J@)D+e*Hyw{y)LyG+HwXm z4cFe3x?WFtVtUN=%I11kxT+O=Z4w~Qw9*LkYw8i3JQG82;oPZjaG>$h{BVlvM%N)e z%K9N@C+H>CZGWoHT=gC-UplBiw{9=Z%+L7;*o$51j=27{35C90{K(wJ_l$ie8ztYZ zR055{+RP}NlBbqilxxU|mRp+@b!!@hiKx>(^a)W%xfRy{Y-Ch|PsYV!Icas&=9J}N z9tVjni=^uF60D4aKsXY3OdQLC>8p)`b5M{>o!mDdTz_J*2Wz;W?Gggg-cv2P=CefC zsg0Nb;R-N_zL_F@Ges;;vn{bLlXJFZ(q@lLhyR*drV;9!{UcYCtg9p6maDJK`3O*% zt0M-Yx$If)k9m@7d=^`eeV&Dw)BQHh($Tl`S(=aDHZACF8%c$JUh)^CWI!Z=<11RiAWmzM+zBU z)sPN!S3mau#6QC|{D&w&pjvczo@Uev0l3KQeqA7uTBue`HnX-iGp4g>;^^q60Ueqy zHNQ(uTtG6>KJLoPf3+pY*>8Fr#$I)K4re0$51_; z{+J%NSQ&gIayL@p``UV5)Cj3NX4~P7cE_-v|3kMq<~E+>Z8q#S$DZVE)}Q2U)~|Y- z%O}wzkvry|`Xq1i)CD(LH1glhl`6d>)Ji@kS?)CWhl)#b{<3na6j@8fC*K}dedN23 zioKSy#+T&_pNeu_BN`>QVMi)FXsMnhw3z?IUFX{FDK0I3wB`C9kegkoZ}mI5=#_e4 z#+P+XvP!Afd?e&F6}TVv6fOgCYG&j@Qm z4{6wp8KTJmHReVp+;pa4L{leY1&X#Fq?uh;({yf-u42pEh9Z-`?HXZ2NoVp_uQ*6% z&MDn|nzw3f_2>f1mAXvEpP~PKe0A%#R=x#?Dlmf@>?dg6XC_-MP|E zZ#)*1yqoc|Sn!~>&+4tdl5nb(XHB2Q4<`FDw+DlS|8%m6Z86h;O;?7UOI#(3)v@~i zwQ>}kEv6iqGH41SF1RbLv~;?1!b)4PTBhfO)izyQdQ9~#<7b3EmkGTf4X5hrWPPPU zrp3ixt853MrVoM^hO0|j{F#2GYqxz&0FB?kh(o%!U4ZcqcNcxQond0r&X{Z$-I?xY zyO_&ducLFChx5@n8|^L@X4`dV6+2z=vRT~51k5c;OViYMYK!td1UWd+l$jXEHOXzo z@E7Ztc<}wjcXeoPG}>>K+xjz|Y4+?4sioS%M?iQ_ zSP8D;n5ox}X`!>dIhj*by8#ZR_3__Z8*4?mjhQp|^^6`(DUU7bYqPDpUz7>g*t~ja zHB5P$m~~#HnVLN+;JL&dH4eZ|@z+*t{gbJ|4p)n}fjD%X;*+UWv+5e+rp{Ne>>&oI>09S5 z?MrTCVRM=3sXrnJl>f2;t8l?*7_@}E3ciwNMp(mTey{@quj$*Aja5Bpkp4RVN9^T) zu$TYFUS41?|I}V?wHL5q(V2KadJy<`vP}Sl{YYIf1?H5xts*X&S3)Nu$+tT@;PIWQ=#603>fSN-DaHLWcw+^_PrqQEt18 zif=B19KuF{^CSK~roWHq@Ai)JB$XOzIh3s^+W+n-iusf9C*@DZp9X)L{Auy0&7Tf` zy8Oxc)8kK{KLh>@_j@%1W&Pj&3#yyx{p! z9F?SPxq)H>T5_!&|6+m~;58AKF5M}Y928L95L3&@z9bXY;&Un^8?M_@tpe|abhKNM zda1{p-n~T!&7FcO4SYLt>JNR*YZ- zp@=aX$h*NXf{;zN`3CO_IczJ_G!?@#*NI^yD_W|r4Y9e{+a&`iUd|9wPXi@60m;Al zL(5>x&}t!gf=-}O6Gb6f>ayQr*J8sttSBadUK$jVS|5yeJT&S)DO-Vg=OIzlL%b_@ zpkm*t|LJ&VFieu9IsGrjUGM*1b<`ScQbxJp$h#mn0nwYejN)nmWRa6Fq5P-Jk>_GX z+B0hMkp43~e@8##saw4i7-Da%C?|XfA&GCc3;y_$6+^c!pt@Gs@)Rpr+!G?tLT%7B zxr?|In(lfqfFQ+%k+#%5R;*FGTi&F8WxNbI+&M+aU-)!t!1oT5)<-4PjqAzKS+ft0 z!Eb{y>8x4br+4%zUzSdS5#gJa@C_knV*k($U2gzl=ouQh-|Ko{9oI`;|J~FzioZ^( z;d-y5xdP;Ok|DXF2&>IBrL$T_H1NU9u)wnums6O@nSo509#12wMi2z1r6pB@v`~qm zFtM}n2m(SAPvDHq-@CWhKEJLM8L34Iv@`(-O$b$~c8HQIt}&`C z%r)Y2D`gWnQt@+4?=Hsoq~xx`^L6T{Ys{A1ZIs>MnOKoxg<(eiJ#N+$^Q&)?I!Z3) zGb~xUacjIqQpc_Fh?z@gpgMqjcimg#trxJ(G|H|VRMR`a(&PsI3261;IFF0Zm>S!K zfe8+VZt*v;nS^sh*TFd~Z1OW2{)dXc#py(^9mf(K6Pl;!3zN;s>R7F^v7$Z$|8Wuc zxyIPkT>n*4WZsspS(T7BH7!m=ABBX#p9fq#c$G{+feg(yPF2b%s>e1LwN# zq?5^umbO(k{Kda*qWrbZx)ApWpR~*^nXhBgxOT%lZ z48&;-4~iz4x3mr}KKXMpmFY$pt_r&$ySfr(Kh^hAieyMez$|R{=t<}->FV`j6__pM z8?8AWS=7nvWfznY0&kJQa0AI&H;fX8#efH>Mn#b-m6gL@SF(raqA#b|kEPSe zaiv}i%*tBQ=vjww_GJ7GAs9w~6&RxpeB_Ga31WKzrT6X_CMVR?dwc{$4{B%!mZyLd z@?@12pOv?USw_#AXl4~FlPw)3z2b8z^;?<0GAz}ooLpfjS`3WG&$}n06XErE)et1F zJNAz^#zvW9+4j&WVAPbB3FO-T!_iLJJ`s(KX8;98za9JgnnJxbLVg4Dv8oFMD@IvF_quW291|5SM?rziF;EUo5 zd=>j$oN6v{H@F*$zt)6`@)8|Oa9&{sWYGouZkLf*B@5{~H}GH5z+|pC87Dq+*C}_T zJZiJ%wrUmgRLH)_Oq_ssUGv$vs>ao_GemiqQENck)`L(GieSw*#?_7bt*ge$N+ad4 za))%5O{K0IQgO|9F8Dnt;6m0(a_9Ozt`nw~0}{>zxZVvZo}fA3QJyGE3C3NiP#aZS zNW+scte|L7Ga;__WRb6kvnZp!OXmNn&{O+MOUT<(DKdoBPk$=TQFcwlR`mxk5spM?UMU*!R8fHE{U$!{3Qf}bK zhMI*D+F*p>*big#RLZ7OOTo4M*3PYQ^t|e@2#Z|XJNt~atbMMq&;9UOCkrN*y2E1% z?DWSjv##5wqmMCo$JB!@rrT5L_ziV@yT0k{MZ#}YC9_2!g|uv>ZkJV8{Cx^g#z(~$ zQ;MwuBVhSa_T@R6wrxMqOyj9tV_apo#*u5{YM?{9yLp;@XOM?_UMQ~h0$)^ zb}h?$EC4CscZg0zhbom6pY8yS)5&q$6Uan|N#~Gmb16O#{ilDU>gkb?H2@*m2V54` z?EeyTdx#L;p5h>_E0Y|mGT*{NrMc8Yw=0ILAt^zliI-eze;>vQYa72y7XMC-R9Z4J zS={ra)VgsFBr3|CLofyr*TQLSC3FN3{zpIhSExVPaF#?n@RdFJ4Ypk}h(;7)atC~N z^Tvo`UfHJUw`>!hE*3p-M)2CQDW)K z>S1>+CPGe8dl03P3Wx#~m}-yJ4aQX=xiA3JOD9t1ieECKVF4J4p?&i+K{+dN-iSWZ zu56FB%3j`fYoxdlShx<6GXSM}qf;`ewkuuxI(MA}y?ElLr-U7CjTt?#9hzxsl1x*Z z`^l27IZvDBl5U2;S=0X^xW*f&L49SY3M<^4q>WEh1ZF&#!y7HAA8?I-Xg*@LzP@XR z4f2dz50h$kOAJZSi?T+Q>==UdV29OTg)58C;x#tEd{~}g5N@)mdyJ!ufR@d`9o0Y& zW&LS4oo|`Dp?Ie>qPxi-wF}p~`g?<_?*^`I-s&^1R(*U@8w8WHOm~sW!j@B|GkVJb zS2$cic)_PX9=Jo?BB-4~$IuhnS(TjRsyJIR>A{AYnh8%r&@LU1^GTT%_D;oEKSkk7 zBg?$M87L6LCPjrnYz8HZWR^Bvukck&8^z4wH`uW{qw?ri>qgP**2USd1Mw}Y{zlny z36^qJZitqrT;`NjbgWU*R;6>YP!9X}u63sOs{fPETGi4_Kzj_C-}%X=TP+^bL#&bj z53zdWALlUC>W>LlwN8~6>Due$gCtbDVJ?f$U4f2{?>aa^C(#uOH``S1FqLB=JH>li z82u>)loefxDa}POWI=%!T^h~x=##wkQC3MDB4o6m(7)Qdh_V_Y>KV%FP`ACNtX`)( zCuJ1fvhVB+8gY~)g36rWdsN=25Hv+K$xM zrrtle^HzF}dhbo#vfq!-`yQ1RDbPa6rc!m?r~7ye_%fV#7ghYM5LRqOc#czR3CqCf zFVW~k_#P26-l<~SqvM0SK22{LY3QX`CNIu_GR8sm5B@YRo17Pya8#+TFp(*)uu&yc zUdKWqO>)B?!&SL3q|Ve4xgC;#Dir~Ju8RTah~k05|^QAK*qFz{e%%CFDY|KNbzy8OxQvrV)c}oDmanVKQQZ z3~@8v$_yR0r!u>|3qOQPxH~ICYY&2kb-wG)UAlDX($KOrGK=JY$Yw{INN(&7k4sX4 zq@pQvqaVs(G=2J9abUhL8kgP6C$;PEZ$#xSu36h#Y1vP=-7T(56m!RQv4y)=u!Vck zf&cbTU;>lThVh#NG>WBY=WC^nILe}g%277cX4L>SYQ^Q#MqNb@ zK#65yEB$RUu`S_l(Zw%CZsVKWOTqOb=O#HimkH6m*lnmFZ@`CKU5F;ua*;GoE*^4p z?9!}AEybYy)Wj%kgdL1vm40P&UvbUi_37H>O?o9wBFnQ`{8YN8A1>@pps0nLbf;+# z^%KAHYq+i1LkWqPL4pNOk+Ydyt~;^dZLVHnFYl_NmA~O%Rj|BsPpfQlS$BR6TB2-< zu3y}8o1GGEGYX+xi*d+<% zfJ!?mgUY1zlQ*eW&8_4HZX`n-6hiczG9%68quZmGE;t<}d zT1HpBMF7)H--FaPH~pqE@S&7uM9<$7ICE1SqT(bL-SKg?VP%RRi_TX!nqqJ1*1DPN zT6Ec}X)0pBiT^%albh#0dS2B>sCH+`mN9-{-GVKbvEUiRB^)nY|BcOA-ec(?3V>?$zPDK;0mHp?I)$DZ3UniavtWt^re zMRZDTe5hQ@KDZ1{i(xiF`x9Nas-QHS7Sxuzj>E&$`&ZLhk&-@<@wR zQqOs}+;Vc!u*LRs_u0zE53}3potm&VwJ6rarUl` z1?zw%!dAyBE1i8s<#pLj(sh5kYk=?42FDJ(b2?cCuVeOzSQw7_-4c_&?^?!y@7Bh) z0Lc$_mR9bnRR(5gz%tk8J}mfhi84s%%R@vGAg!|6%S1+oDC0+qFQ>4Rr2rsK%cI0oLVyM9HG*F#w))*x~6h)oQ7_lLHMzSVMk5cjz9R3VEPk$=$;0M``2^FR-c9 zh$|dlxx(?b1-KX<2_CN+3TxwPg+`??aDct{U?9HsW^;)uI?zM+8Mq&y%Ngp-lICE{dX26h&K66x|j@QF)8C zy&XkS5k=7|$DZaI<&ryWg?RjHTVum8xBY9I)3=T3+idzaoxV+`Z=ur!1>N>5;T_1e z>X3Wd0;}|e)$#rHNc_stnYib4RuVeKdnttg6S|9uOtdds)rs!!YRZAVcw`wuVL(_Gu{5tQYovtRPw+-yt8XZC;B za7W}MTdfmC+U~GiWdW(~@2v%9!+#+P60_kyF@5{!^zHkmZ%<85-JHCu1+IYlptO{; zQU$uEE_zb9w}_Dp)rG7MSO;@@No(ojE88+9W6K!&gcwmJmK)i1P&|sS;89WGiM|TD zsHDSjDNR&#Xym7p&0LxfHbKvAhC+OKsA;!;01OUUx{%(1MGQ4I)X(Uln{J#=9sooR zMjW1+ucmom!RM!BaD>`dP=kQK?c+x|YtM0(Qf@2gC5?33RK=9?GNS|>J>_1~E$o?q zD_B?sG6kYjtBMLoue#L#aUdD*qbd?Q2QMJhf7y@@RSngv*(f*WvJ_o`y6)Y{*a!1BfFPF#@e$BUeSAvTe-P9kOPo)va=uV>=MM zM~_se8*6?~`6P*Nhaz7oMDZlbBxVevGWT3^a}qrxeLR$?Jv@o_&lj9NyqKZ5t(fuS zP&)U}Ny)-OQ59_BGgiH}ZLooMaQwE7$$K#M<}=sL@!P%}Ho&w6HZU;QKyc^>=Hi|o zz$fkpMxOz~s_Of_P{D>Zr-B=NaklE-Q8s;C4qaRhQjyc2!c1?XD^nLSs|KUwnbviE z8Ie4Ybn+T(P1$fw*HhxW3?{rNSVHjoFoDDs5f>VDcG(F)3I;eTSHV4FsW1Ci#L#+B z0Zm0v@^qV$%+`U=AyrB~>xPe>r!bwOQe9U60Jyu(9-NHH0dNDn^aG~}JFbRsyqU={ z6!w1g+;bKXs<|mGrw2hxULJz z=#kQEY7Nauc5M7*LlCF{?(W@%Is*Nb0{(QO@*1y6;G~9qIrAL&oSc zC7<-XlFDEW-(zVW0Rl>Xtq9e9bNp-#8=yNSw7IV0OGClS18`5(8_NN8R#M8x4c(A{ z(vX1C@CpH?|1I9{5$c-#j{pxNF~p=9y0*dZQbI&>9Tng82JSI6y z{u5*6K;a4hbGe>*{nc@W4gaaRa|Gcj-2V zj?W$Hap>&ap~4!ly;BZw7y8+6Xvw$(<5kW)12yRg93z(n7 zUpN>GrdangD5H2>4aC~$qjC-5<(B<7;`3GCzZ^5380wmTISzzXiWXw3Iul(3gn)L0 z0eg5H4wP-gV7g8HUP^7LPOD184yrc$$3Xkr&{N2{{?!mh*38HMN(fo(S_x82VLxEp zQ{URDQs2g|P0g-V<%0CQCct2noO)9K*XSno=%y2Z-r}p2gIp-+AlS>n%-$`h3HPry z*iCL@@4*pd6}>`2kJT;~yY80%)d!lzH{^2La2un!1F>rsA7}L=z%7e`0-UN-Mv(({ zg8+phU!*Rs8sp4nn?qu)mtA=f%cVt=?-8Wh(vzHgOSU}*UZ=n=B)|FWyncifggKN` zz4I=Jqt`pH<&Ybl=!A$+J1Sb*YN&n@e1x{lsx~rk2pFeV7g1TbtElvK|MrLkz=bEM ze6qZ)>mkKaeR?z~pKPW6E72HW4i!9}pY2Ng__sfy!W9kw>NkS5cBvtdI*v6x@K zpBR}6#cWKg?<^Yn_eI?M8i&5lw%2V_|3336DrTgy+dwoCVd7ECC0;HA#OQf)YpM!z z(iu8+(iunV&R9Cw8LI4arOr4y+Zk+B(A%dy1WWY2iPa~H-N|Pp z?JIQD(I?$eYEs=%Dpa`>+t<3dZoVP!6v2Tsl+M0)OM2rri{n|uopA_uRn_;O-{olIiDTJC~3hIRwW zq+;JBt_-+t((FX*{NSa9mPibgi5`DzoK&gX;3c6q(u&9Vjxq6`)Fu9gF(`0EtQBo0 z@gmvrkGxV9D>%r|k=LjnDew_~By&#R^P_b3M?jcrR`+H9;}**<6KhARR*@SJE8Tkr zEH?Y}UB?c@(;Gmtv*_^;Dme4pfpc)z*-fVB)+e9<6Q8p^JCD_>_ZZ_Mt3{at_cmm+ z7iP%jkf_&iVMVJ0(@Z^mXRkTa5dsk})btKk(NRaJ6bqKkTt_G@AEp=`F*n5+)X5ZE zT%gCQrdvm3^RsvM(saej?CiaUOADoxh1pB6NG2!~)KCQVTp}?i2KP|+SD&G{?3wIG zokj8egZ;JOp=@-&m5=07QeL&!v1N!wygk$Msi0Cgt|P5tG8*D-x@^V=&PK_pbbg>` zClPY$o12*Ro>W6sB&KFBec3=?!<;uZ_JAE%hvs_qt%Qkq7V^j~e(x|wAW-c5} zk}YO045~)>6FpAsANV)5LhWP!vhZO7^)#d4J|3fVikeAihpgW;P4yJ`29ZK2tAY2u(@BpdgQ?f{OD-+mmX?EH8{x^7`%xn}stg&= zvXXCC73#G=$hWJ^0W=G#FJ!0AICpK=lL211ftfG^6R_0m)8Z78cBOW`;+4Hk?pCKf zwah!JXj4OSXeJ?vhZH|uXh6QD9^VHvzE#$5Oor9!_5{nd=Mx{BZ#MJy#dogP6;&tp zWoGte=9A0pOVW;F|Is&NUn=crYDd_l(oMDwjxyHk%ca0wHsEsHoUI1gwma+w#c$!s zsgZ0dVy1$-F^>7~74T4$vMC-DEpn!p5Dksue_Aq zqOI+p>V{VPR@J6;=&SgpqB>EHXHk{LJ#%W;$!+yAcr304 zGg}O}xS1^`Ws5OcH?wz{om7X6w9kCwe}!)1EqqmPSLL)sy>6*k7nJmDW8lF)k(h+e z{5w@XmeDy8JsK;*GgR~=8zqk?UWBhO3XQ1Voi4R0K#L^sN6VrA#FradFqD-qF%MsI zUR9GwgP}<-LiY9W5Tsx?L*-RllA;=G)$^dm6p*9~a_=mkdRF7NjZ;SJ4z#3v7c9r; z^`kYH`xE0b3!Z9mvR#aDE0Yr2yf8L&7A?qd#lspobyY*OvBZ70%9UD99Kn&DAt`q2 z9U9{U1xy0OH+{pctOsAJZ~3-cS*KR`be`sND3SxB<(FPn>|@|qj~`4GE(G-pqU;(m9tv2Yp#bj{*dNS zI7bVS3y6Y1b=$SK#&!Y&{PvXcU!knBy9Y-qRzG5C3xB+Tssk!l`!+)o`TzLUFjtuo z0e`9Z`+$&WfW$@&ogCn7R_}SkL=iz1vhC3g1c>Z|MBQ{pT#JRCxn+M}?3PF>4iCOe zqD*++c@nIZPB94qO`1Qk-{rcZ4x*F`(c5jMzKGkP9uf`^sp_-8lrR%N0JJ!=ut|IK**1X z`~VL{7_W?l$VMT1f&6*P%TVJ9aYJyn3KkaJmk za_xyvL>MyEeaih06)%%AgXpmGRrV|exY%8YmjM|xLalCv(wYRHV6L28Hq-@Arw-z8 zqVKfWKb!z^s2#)$`wq@q#tdK2)g8o7w(oM6xZcgl7vg6mJ@_>enVU<(Jj0^FHnrt? z(3<%-zh0o^JJ(MqrxY&30)PKovAwnk$)s!sgTUG(d(6i^)|wUl)DZSc83O|$a;A3< z+rMNC_NBBNB;8}SdDq+b)vZlVhxp?y2?(Gp|H&lIc8$0#+q6>gc+Hpy%J~=C#{#2E4YBEZ;l+Z!oNmM?`C^-x0!2jF(VfIPsH_4Ku zcum?%bN{bjrQ<#xvm0ivIUDmz%qGFTaWLj34kXrV8!ju}B27W;mBy;aZTx+r zw!yuDv6#3Ng1yLpL?Ib|0$mIBH1p5KaU*m{gAPG*DJLW@Q&^`AgH8?w8S7W*g;ZG^ z^nzdraK9#2Nompx%vbY&^Xxoy!;nvPpzo>=t!iNPLpQGe&`NSd4b2WdG_bnODF2ge zG6%Y(CII2-7&6D8Ed9t){M*0wjZFxMfE&gB6>kb3GL$o}pSZO6bfRP*a#6(IVpK$r z#y%l@PErbLRl#bmsMLysffcK3<_=C!v+9b^pmnjaM?wan>Oe7Rs%OVV51xx(3}Qve zQwiA@U++c)rBO}5)+Zp4BLBpfn&ab9q>M!W7m8Z@Kc{F^=HoJg_>;QAE&)?ei4`QP zQpXyP$CZ!jyZ9|h8Ln1EZ0Q_(zaRlEdpO9L1@u!o z^=9N+`MVlPwwwOQjgfyO+Nts|x6b`u#JG_ik&Rti16{tFiM zLeB@|P+SSj6*CdGTny$W_FLNJ;N!gFm3~#ouGn9?l%9#NHwhK11*8MNQs1hbO7rQ- zvCa^0xQ-e29eOcy9T4}4>$n~f!OpqeXC#!`Q=~N6)pg}GjL$txUtkVgd;+Z`+x8Df z{t3NbilQ?1uYHJ+w*Nr%{A`;K{$cr#l5PLcL!Fz`sOp~+wl1=`=XWMiHE?|wzoY8E zi~Wqa+$D z_UrFc>s<2eY@7EXxAWeII{cFi2q(UA=EOJFCyKcw$$8|-P4g-d9lGFhD*fsZ$CNA4Qn(qI24EXVdLXtZO*7m&mY(VN*oZ!^$|%kC+m%8#k?kcf)H zMgDlb|N2749}mjKR+R6Z>|&##Ju%4f8i6Bd=DE6ybz%D*CR04l;YR{;cfaV6UYU)!!9{9+5DJyyr|4ya0M3{-8LQSrD!GVUegitN#g$+)5e_tJ6I<8wN$ zpzGMn#uaYWi-5)W+!$BTYh7%Nt0g`+AySR~#U@E;vEOTrs}c2 zC~_-KZ(?`I-Xw0>-jEfjuQFHIn+66rziPUHy=jq(!I8G>*_)2b?G3O4eFeNsZ+fn6 zZ~Bm(@T&n-HM~*R(CaSo#SWW5UITd^XYgXWidzF#YhR2HiI;&6wJhk z%6|?c2UQ*kBZ;bjQZ08Is_Ty2B@!tioLQD@az|f)3EPtNR+2xP2xLn_*EcAQSrEbE z)d=f=yvi!MOyDvjCjvJK0ynu_;1XD^q@4zV8@V(HTv272rdqNKpgNQED1nP0W^HWm zNbY;7*7d=Op6OC)nfaN>Y(BQ4<^`5mWP4TQ5AD=CX~wz3+$rM94ycf7{+?#sFr}t! zne`}@)pVN*>U@4Ajahw8x-LmMFtPi12V1b^;^Mt_Uc-+TDrjls8s~m=Z*Q-FRC95j zG&GW{SNyfV?aE(mjm@hOoOLJI?-&4UJlOQ(9ed>mZRr3;J$OwQUk0AMfUfP?t~ z<3O1!SuhWt-%n&5$Y4!!UrcY8b3a^s=b~ps#9=fyNP`h~ON78xk{5(K2$RXGH(g+yPiH78+F zr&9jxE3uQQ2*?2;9%Ykn%VJ9bx4%78=>)zk%^Lt*t&-eMreS%@a0IXh%F13RXGxLwzcYfzS zQVw0;43PAc>(cr*^dRk~&#wOhy9LrNauj+Ref?@)VaYvos(3cbIB=4rcwU>dv*`N z&D3yAbx*Tv!{2n~d_5!ReurZvRgsn#)hC@l|G(+e^z=A&=)9atQ!}ZPeCj=+^qu&8 zQFr{y&Ur^BV8>VId}Xh8DC;6p9x9@UJsiYO~uDs*vxe#>KPnElukIi3LPqV+ihAD`MI(C z&B=onyM!mG za7po}E6Ks_xN(rR*Qqb2_^19HSJRbs7eZXYX38kQviPMD>8|q(9Ox3!9+>m0Q z@cp2qh2-fjd(CxRkAaXw&G!MIMdc?_C6NIIK`=^0$PVuiPoalhkblr|Bd=GSJiZ;* zEq@l15trABi_s}Fm&yMqMKKT?;R{|-P%Wf|ZumrYAr=)nx5_|vs1;`P=hQ#Q=!G}BC zlSYenotK}IdkCedEdMT;8P)}d0zJ4>xy~w(q{1-$Yb=*elZ{6-waac|ZFGeGkoDBr z5B;%VaJlCb-?bP3$=6zOtVF|sWZ*!$4Gv^MI1mgd;XqQ?o!~&IUn?BQ0yq%4Cr*yI z%Mi2x$1@;3{IElwrixY~4bZWqnrKQ074{MQ79+c$dNXQDwHka;MnqLD2wur|BImMB zzcF~G_stFG{xmGwh~`-$sY_4955~Sce@FbFXv;*DgbI+Y*2OUS>!cUD-i2CJ{Fu0% z^P9)JN4Cf@@6Aibc^S2j~1=%sJ(8sypQFp*rWa&Cp&c*)XTPa zAwjskueZCz&Oo-dQ9?U1>9i$a4!jZHY8#Ham)ZBX*>}#hskJ%UOkUyI1Q!yj*M%@3 zu>t2FXc!LKLyFmvBtl%+sN;;tZN{%Fe^W)hG;%h=Z{p8J^ge|G7&pPT=5gt!maIB& zV1}GrWSgqQb$!Qq?dDf)S#E=d4iW#B+3-5O z>>ASHP53m0rIA}qb-DTaqjWfJ+Eu9B8(6tJT-C8rX$L+IWy(FPMbSbc(CDkpcAZY8A?;;~4<`CiF4fkLt z9<&-$beNkgm(! ztdjFxF`^Pnzq~(lU_UEj<|v^kyIXZvN63oQ=nBc?!NL@DZ+#qu+J~<=)&)k10@?_s5FHVkJ+aDj5KbVQSc5fG8h^44npHV66Zno{Z zM)Kf#YD57I3_zuI zw*=mN&Xe&<{~V|Hb_#k(B^JxS zmlX~Q8M0J>o@uGWs;5nR)Ga9$mjTk;fUVqTCPMZOGoi_HbHLJQ;#}s&DUDxdombi28CPT`vnqwzD_|lxg~Wt&bS18dN3~&3eo(gfszK~ zx(?jT6by`az?a)a)2_;xWV3B*TKZ!QYjo8Lb5Kp z0t{@hE7#Qmoy8a0O>Dm0GDb>tadBAw3Fx25f94Ygw^T~U?a})JSqsISG>4BRVyyVV z_>}_HrNiV=nM0}nIO)4G$qh`WsRP2F;lOzyE`Ln*0R-ccD@KwF6Mi%KG<*CB7zG6P z2shw|{ueaf%H~PK)xU7%tBJ=0tW<@st-WX}oCal7j6I=-izI+uOVRXt`^< zuA!P zy{U4Rm>u|&*!2G1m=j38|9gp=#X@$OqcZY$T9OqZjn`66MW6ejOZ@ZE$baSMTdr09 zQCwkB&ySEnyedcjy)8tCZi6k1@h1OWg6NkEF!Aq=Z;y_<#IHl$bX-`C1ftD!VseGw ztc734emFw0E`Ky8G@{Tp%}O_-@05-f-yXe|-~U2v@X{0{#6toUK8CxQMoF4HYT0X# zim-Pl32s3D&aVd~dOFmnK1Vl;l=Fo<8Z4ZBriHey#k$VY{CKPgmQ`t6kBY~VINo;V zEwETf*dD|}EK$MYDL)@OE=c{SBgO&5fQ4l6M@lE(`hGKfgfGuk7B%z3w4 z2H_Ev=x$^XAzNLyTL$Sm1L)l=Q;$H;gS${F(~I@R387?K2&Hh2O4|oS{srT6fqIk` zlNP`JSJGRy)yf=u*0R9$A=`{}7h zZD!n?Q7`0LFvXl($ho55nP(+OHc_g6nC$he7_?t16_$v9D>7=vzid?gKvq-^H_^@2 z^>TF^Nie&|s4K2FdFGaMRsnmNWw>WGj1#rRqVIM)I~$_V4Ot7VS!sLFYn%AVr_m&@BbOL(Z1 z--;1{2g(Aswg*TNP#LV?G^9+1?Wsz2w6#sNEoSjJ%tE0vh}Zn#^#;W zI#urGtLu*c;%{C0#P&IL7rFI17OuY$|KyH4x4KDkmMFhAlQ+5<3ZuHVTq$klK12I^ zb={sgQ)X1y`+T55K z$MR_N>MUgUs;AQos)SB)BJL@xO#IUAP~ttVzD1Z+82+&XXWS9#&;A#Gx9w)gQ_Bli zu{SGsNMgiV&a5v9H`s0@q$?DJL(;wbT^3_6=|U&e78v`AUD-61ngNmBfrOj!@9%$T z_wNPw*!TAY-PM9!zkkoJ-wSoQ-_Oqu@@Ay^5tv6CHG3V#`u>O7)e%PAZ&#H@9SARC zBV0E_#tE6apZXZ0yPOu?7(K3H>kI75pfxAjb4jf|(!SiQdqATbu`-`e`~lV!b(lF~ zU7vk6aqV_+?W*#~Hg4UPYnwCI$Ccc>WHQR3ONkeOn7v$Up;dxlG{HfMZYux}AIFT_H) zTR4wqsqv&z)0`!C^=);D{9uo`*;mO#M}*H(P^ujHj=~tT?ugZU-B%o@2#0yw?*G$f z@;wmLD|&0Ye^$GH=0tph-MlJ^%cI-=N8cq(M_g%*WN0sn_z)JO(w}Yn zMFz}S5#rdtY;=12w2G(S?Bg@5Dn{#DXLbh{dT!nwRebW6@|{_AU31{o?d+60;9C$8 z-r}RP?m8d8{Csu9WxhCD9k6W{#P|`NV7Ou1(Cngt&G?HtAJXW>4M-S>d^`{u1F2Le z;#<)PQs1eN^*Brx8UKm1s^i)x;@fau_T!=v00`0!Zajn`KD2pvs5!z|vp%}ppzrNr zGIWIw`TU!2w{;c~zPIh@1SjHKnKAg>G*u->%pxzI!S58guK5FEM%ZEG^)+ydfYfo_==F@5hG&t$loCUE@>k+j)F*&T{0jmu5`|pqP{3s>m#m^M>@BB)7jOkaw~4Zm8;ba?g-s18%B(6cfC)Hs*pw= z@ezoD`j|TijzDG)S;&?2#!!2NYurJc9n~=zJn$B}E`gP(sE+A`Zfd{g>ya=QAj1yX zIlWTPjt}}3mjk+5b2&Oehj)5NM(_@SjGOI7>b95y&__jqGlKw%PW_d7#m!F6_?3|R z?^ecYfLAPKk#j#fguiw@{Ji#9igjIgW&=00mmEd%@#K+&qN_mcxM~lVF3{gDZ6;g-E8)P^c5>$moiPSmOkOrMN3*sA9^q{1J0T zn@B>(tr#bl5`vX2r&x%pDO&|frM0QZt?X^L!iYvJ?#x9EXM&n=yqOps$OEi9RP4E-VH2u4$O#35#4QA6*eqMjQj)QPXVYPdP{)%@}443uh{1Q7Cj}Ngew|xKeHr)<32FrTo708f&gUe84oNOLm zbp#h?v=-?d*xNp4XdJyiJiD^SRA1~tA^~dUNT`<>!l9JZ>pDgt1})~-fqx0Gp` zD0L`EzIV{XauNh?>mHbfdQ7Ay?iGBc2|fjZBmWL-;v90eg(P#iyUDc}9aIel<+3N@ z!iNWf3tr3V4m}+}wyo)Vlmxzh+cd<8b?n>UZCs)v_bz*OU+E7B-?=D5a=AZ$zpQp%MPx z=Ab$r32a)Hzm0!P2Yuv!R)lHRFtv{*z+!i8;&3Lum=p|S#eFvuY9xnSEBaWs>~2(j zNII@%YoO)7O3Ro{qVtt9soq$(!0TG9yjgnV=ejmA+|sSMsR1F@JYIUcpU1Xfcy;Es zn2OA`LY2?`ZvFx%`ttN>U*+cXiJLDo+ATZKTcODO8g{p%acWOm0NR9ZY?y+ z_|~D=tF!@WKMUm-ag*4?rZtmwAOY&ei%?8#QpOs|x`55ZJr4QpyX6GXUU8+dUu!#b z`4fb}=NPn>o165Eqj5YVT?qqaJeIDD&#!OEjU7vTB*D}Q@QS~G%zoi=*CPtU~g~LHWiyqxP_ckR3`T$AOhcoazE4FcI#7WM;gM zr*40lI1|bCc+^w^Q!TTVKh0k>Q3eZ}(P_cp!AqlA% zsE(D*J1E|bX-^m(?X~Jb$#sk#(}XzxlH_)ZeU26 zj#~iG(72yhwH$_SfW{0{cc=VdTGbGnSep;h>f?3pma=WPbRxbLWvD*!RDEbv7Zjm4 zVseZw612^+g^7ed&p`|tJNUYrhaGf>My@>yUO9@>=;EufWi70FQ8{PB@z_ChodHWTvITB5xx;s3^8P zd#Y;}7rt+!GFD$)+pVL%Y!IW3s{?MsfJoJ?ZsS<8Sie+wql50Hk5n(=yIbV^;;FQo z0WLzjlM0q+3{=-v6WvQsM}Jj$^auIDjqaeE+Y+S2#}}#t6ZGlSlF3dvX1LLo6%cZV zR8$3qvM%w4%(Vqw>VP_Gqbf9RR~$uKbs_uh0K;7qW~pZJ!wpP7Sr8uMfI)N`FjJe= zy{METO2)))wOOsimLpdS)IM-^`CRH1V6@k|H}#J=fO2=pxO>XFoo%~z%dNR3A&3qM zbus`!v?{*wLm-H-$PS?2pQIoJ?`m+baPb?O_JB{$x;{g)5kxwRTX{35=w9SrVhZJr z?j?+D$cj7Ie5m0LA962hbh?9n`}O|1zcm_zYI|U%;M2M>_@`!|aRDr)-5}f%%kzrY zi8uq=CbTmqfc>X3l|mm4c4J-!5Oj(DN#&F~g*zY%w(DNPBI(*%8Td8Vbt`T*3>8D- z4g~BR(w|v#Om!Gv6To}|GK5b1 zPt}WfAS1VoFH$^Nd~7aX>5BwPg1-~^p^A?5{Tt6!9oKKFwH24!q1A0JCkR0?$KC(A zw*MTVCMu}7w*Mvldpj?0;h&m+miebQy3~oIORtHen12%fN%<$^pBDeL`6u_SOYe#= zeY?Bu(*1D-O8D8x1Ww0lnz-BeZ#psEnsn*@xDH=gX8NS1Q(qs?e6L z4!e1!Om$otB2q-{oKh$xjB@@+A8lF{v^h57xCVhFQZ9XT;>biFmAkX!^3$KeRl@FM zX}bbUSe+A7!9MkeE|6=FQo$vCEUDZ?Fv2Bl!Q3sngYNJuQBbhd_!^raZg2y%ZPRKh zI-aC{f}}4CAo69`VG^Va*qC=T=02)!HNhb`@_5`cki3G2G{G|~ztXXhRTKrI*JAzF z7mTQ0`FE+TX+)J$|++oq&o2;&@X z#BAU96viqhvuI(BtBo>fZ;NzkaVhU=dL>%i>@ZP@zC|T6q7orSi|3^7yTebJ>t|{Z zI8?`@5RO~UJ~wn-x9)Cm3n7q{L%k56b8~KnEP;6+l|M{*b?CZXexCchxWNb#*DZMP z3%8U%qWa+hk)fu^?dR6r&~KMNnz>sw#y=Rlu0g~V*t(0*)ho5p2aq*(8$Q)=dmp(Q z!tK=^>kM3-8LGhUO9Y*#W>oXZ8}%=OCVY1%@$FY zy9M`RTQbXXxpZziDrDve_1Ee555yLbSkwNbCoxpQnWqX zms24=2UL{}QMX-b+DveY_Iby?W#M;RQvP0hj1BA5;DKHQl%;Bi*n49n43nK0zj8|V zm`1ul;U0nZRY)QWSqJFSJ)e|6*~a(HR^J9?)0ObU(G4pwAKMaj=UP54?y7Rrn3Sd= zirKb5sePrJcl?{AhRaV-Ou0W*+Q!dsxm^B%4pn|Sm=MhnTxqiu?zr~iXCrsKNzZUn zpn{O3@=zlrB~#JceY`c3B$1BJIbh=`_mYzKcHob?Rae`=IB+ZF=c(lsP~J*RcoR|R z-9Yc-?9qWc?teERy^rz1F`y-Yyw?~x+5G{~Jr9_2-mOi6DS5XWOgSIGlxq!`a-K7Z z>vn@FbzM>bmoPBpnw#GXrktN3e_tG!a()_2xi$f*uZ1E9n6_(yLwg3sn)Rr7upM{F zAwRjLL{sV$;1*L5#X<=U>7P+Xhyp|KxfV=J2#(CZlMJh5JJ}|`IzB@ovW+NdxlHIH zT|f+$#K@13uW^V{I1*W8JK{B$!Kx0yANDUj2X2DvoJ1(7vV8(XSN>3WbF;e;N{kFnHH*|y@79*4xF9B! zWKA(qCG@xWGbtt-hhn0!JU$vmOD!VM&nseGt#PcYHO@dqdot-RFIJ9heX}m{usAXGUFl^bQ4;SVAy&l#LB{splZr5!EJETC_n%1 z|B|k6w+(LmBq!UtN&svoA6M~!&EyGNRu#n1WMXHh+#zi`B-1Rz^pKcVKwSqyM^c=H z@xIOetW}4x7HO*}uT&8^s?c;V&*|orV=Vw;&KnRjaM8fDz}dbFOv|Ccz|x-Sv~F^L z8vwIPw{h1AfLRHUO6yh%cWwaYA^@2CV*qpEiV1+3?T4XiHma%hDE3!HGfVaFP+b;a zLBJZ)CEZJC=G@M387TP}&Ac4Y%z1NqPk?jM3z`u@yD~KMGCIgIN0f|qP0#m`?FK;8 z5Y4lgskWurM{ZY@!I z4m4ffBuW=blqPpuiPAlnPZFg`E}h`&ut1s0{xDgqt_!%f{=P}<1*^2!7iz76doXUW z)_wm1jhq#Sy1>>f0?;IF1IDnF1K_*W;r zJkz^TNiH!wl4fwq_i7o}9Ur^yg}ROJmuP`yTzB`&xNe`{oMc?*WL$^ko{}lSoI3Ri zuJdJ1LBmE4llu~*7CS}R@(tnb#6?ngd=3nbGqbn95#3bl)41?SRTRig$DWL=L2X09 z+wDkk5Sz084DHXI_6Hw!0>?VB1g=^WqQ~w+Iy}PnW6J;2w$G~-2^%uFd0r>kHYBimY-b^dybf?!JH^yr_6>$(F5vet;5h#ha-Sq7Z_zmuIKD7cfITV-!OC4&OY zw>#kbud~}yl=?*6!(^{>9)~-;UI-tZg?%O0BFu%kSvV1+vm3FLwF6cum6ZbW$s3_Q z+jk4~H4XSIDiSO^%^fuBbORRb30#yPGDK5A(OgxpT<*8a|B?;lLho!uEuzgWwXoxR zvhUk|Cp(vIBPT?r+{{Xip$N{i?K8{o&Nm|P`NVn*p5@~aik({sw<^P}7Or%w@>;hl z-`l3L66uI;zW{|6EeGIx)qu0DBsXD5E!>FBRyryQ?$c1qq2fXFoIF^K!=@APZ7os7{5 z;m1ly%pp}`iXsM!v4XFt#2#)3-VYa*?@2APQT|w{PBN?)tcZySbo=FxnIl_;a5}p~ zM&R*G?pyJ6D~;2U-(#xQ&JqhWhAO3P4!R`_J2U7O_TIDe;_lhOGCx2%w*OL8K8SfZ z9%aT}f|#O1OgihRB zuwd7e8zf72L1lw3(hSFa7wo?C<92{bZWFm!FnWcuX)C{P`|DJfSfO7k%oMI){)Dii zuftupJ9MPBELvMA=E5-Ku2VjYFnoPge} z2`)F>X1Ra%9k~)uj=0>_dUZYWzh|+@mtwJAg5MD9z;G_ zmV?B1eaDq!aOrB51ApbBf*b2VZ)V!H!yGdsns!?!-!H0IA2V8@g&+{!0P!}N_D~*NrJbvjcJ5yHK2nCM(Wm@aR#m~HM_9Qz(MA@nKTws8 z7Air6c|;UnR3;@hVrNt>y6X^C)s3iLYyQd3xx1=Wzb%c{_)kHj8ZQI2bEup# zF6EaLj}9cvGX(nzzk<1*`d<<8SmDz*(5g8PYjt|AZu|5?wQNRwCyJsnilUV$imIDj zN@F^%t$ks|Wn9mnvTPvNBLi58L9T~&ZZA9TCc>LhH2wce_tZB-#f^V?8{C?%N}?#5 z{{N+W>f@JRl}1rC{r^k%)Ek#yl|@lB{r^k%)VExIRV#|3>HlB4r#`Xgs+-{lpK9mv zJ?}U6yuW47`xATK-wcWQ?)&X}zp>~0TlTy^vFH8GkfiUv->&x?d%nMA&-)X5-rsbo z-;Sba`v3dP*4|Os+pE2;y6I9skD_S$|9|eg_8|1GdfeMVjy?5%NY`(|gJmch*LHJe z)r^DjE_KTcn?<9E1KDzI`#Z<4ZX3TkK7O@6ezi7!wK#s&AHO=aMUJLh#)4b6umtHx zJQL1dtT!dKQlLq89#JKNHD(rIVpS__@y3(tR-URCESEMiTutf@(t~;_q&?>xx0who z=fE$xIgG7&g;xrrI?Zs8`@0nago3DJP9UG%iAMo zM7L0$NIhqUEa%yrTew?f80vV_?Qv+zJXE1?oOVq%sXSFUP|*D++4kdhp3$-1WP{q+ z@cSm4wE4Z_8~gr#I-7Lz`zD*zes9^N`~3Xu0Mp3)VtcbmeZ*K-g}YIH=UMW={*s$T ziN)pm`yVC;;MCT{MW(Ualbw1H0e{fV*g7PI3;Mrg8@e0~+GUQ74FyfDBplVENTx=O zVlm-E6@MM}$V%e{8CvzpgSrlOvXjbWdFy+bRu$B8t5_Adu?}jv5Cf1)b9@4BRR`Qk zwP5y018O4=PR^x)M`4N#6G2|MRKVGN-OUf<4}L5j=|QY!4ctR~I=DUhKn&8#gV%f^ zCdvCzSV0pviy~3}b~0%5^z}iTT;50HY6X}@`BSMLO%1%=3cZjQ;)W~0-gV^RVQ>zj zk2u0G+|LYGb4*{As*p6q9CgdeV49QatobEc z?ja%;&kkZwaT>lYSz^2eMoNcirA4@pyPgo0D1lbTsPNj3qHmZRlUNpPB@AV`R)-DZ zHTd?V)?xmSS4X#TTS|DgO(|3koh4l{qYP48Y^1CyFszo~1mjvTesfA!&$ zOF=HV9<0-i>Wj(Hw6m#SOTICZ;s~+j#D37`2b5q%ur$Nc`~V6H26>ipYx{1X=n5O_ zjTC#_OfLASTJceJ&>hZBxx-rQ-pG}gjs1E~p_#8lBkGz!fR=lsT0rkzpf~Nqr2eo$?O?1XQKT(Z6F-MZ-JV~0^0QNwmVW_)W z;t@^k+f-Jx{eNg&2RC4T*HFXVN2L>$Oj$C|^POCh;?r{EECv*<# zoS)-coN2GhPF+5QF2A&f%AWiF>qx-4af>WdQZZFrWyyd2h4@@OBmdG(07E(V+XO?A z^7~;5``;by{_oPOE`RK$TP{5koA%ZAC((g=C}BN(pf~Z~G{6whV3JazCwz5T>6;J7 z)V_h;Z3&W=+|6uBIC6p!RUw;;p*pV^tW@fuP_+$Nxve z1m$KhX;8^nUuIA-kvjgCQMFGvQ^WeEbM>+!ePQ&r7(M=3-`aQd{?};tfBTOf0|b7O zp~+fN+|HwTAM@!|;~_?G>@viFkpApL+;zEI4nWQ2t*Ymi8Qabnui3S+{f`=308Ig{ z+0)p{lYNlT=2qPX{yggrl59MSWTMjhqHz>`^#!yQaTG~SkwaPr>_KAhQS;+A^@ep1 zQj7oihxwK0F~YCVfP-awXm<+8ZW6)EC~B4T7}7tM5zvKOZ*tMwj+_pBkv^j8ri-C}cYNR}L12zA zFIeRfbwa8EkDEFg!u3 zbSgsJsOmC-c+jklX-8!{#bPc%TyR}<{hZLeXbwci9GKk@)*_rmK>~WF)j+je|B-jM z{Xctmo1%U~EtE29+CDwA%6G24s^Rs&_fsv`F&28+DXJt&v_!;AvtolAN?jZ#PsG(R z*D^EB9n&PZA*;F#$@RLiGCOHqxU)3n*OC1|5k({pHR@15_K}j{T6*ZG2Hfb8`t%-9 z!|`VF0Dq&+iA&|lTm_d8)@uQFP_eKF8dYqac<5#_5a$ih+Z+rc4Tz8wp zuCs|w+-=TNSeASKK737|1`sjB-RJFWona|4Q)}QD_yP!M&p#fI+>%%jdPDBrY0g)U zUmf!BACJ$`o@>aJ!o%m?;(5^8y-(_8g*GEC4pbQ?N{Cp0)s*F(2D>Q;yNT{h`mqY==GX5`M>z0v+OT0X9> zW3f*<^$LhUs_R(usyKgxrmnhPhE|MX(hZ@9LFBF<2a~TyUsfRa5e0%fc=}#=A(c?> zc{lPLZ4iI;+)-8|i7dBA&h^|4uvGQ@Bk}FgJs?N9lZeEwXC{|0`rZ6Sw22>;2i6@o z4^szQoZ1{Z9i{FXEB24bGEUD;{5ZlP%GqC5-p!^F8oWP^78&?={Ac#AJ6 zAlmicctQ1e=3#7EoPrXqv7^6)q2hXO%{t;n)fyYQ=el>qubXs`&kNV7X511-Eqv6( zBiGb6AHN*rr7fEo&Y>BRcaBM0j29o~Ks3!3n7Oc^>1219thsrb(Iv>o@oDtFx$B-8 zz)J=(FG}(}Hp{$4Ylq2NNL;Fki4GHDQazHB9v8%@=W01{OJCa3@<06t=s@MD{G%Am zxP5BU=wJVU)o*Q<3qP%D-L=k@p8)RQI^gY30}OQ63G1quB+W@qxg%(W$_K(0nK1)Z z1iUa~Q8f?00`D-{svO&~3qG6t)s5)ESm^<+>HD?{ zmkK4vX}d@qU(}4#(dtDOH#9b`N3@zk25zy5R-47=^70>L?A1`8h@AlOy~e3$OUxEn zn}BpzV$iJx(42fHtdKu{6B3dr_0<11H*353t*Tcvi$SSgJ(uGiVdeq5Ew~ky`hb-C z9@Cwp{qQZxbSnE`?zEa=`o~$_b2*9}>x18PhY3i^o#foAZY|ZxqE`+0S{}e)3Q4a} zZd-1dx|4}T%K-{j{$VPNeqmORl(KDt^P->SO+tI8povO1VRgU6wb=sQU3aqm zaByEsTdCUxifTs*W&B*ZWlzI17h?#iZe;(Kc-_ZaayIpm+)a`R-1zU$8V%A7V5le` z7vZ)O1WRPyZ4Hdye*MU0E-ycjCEH=B*t+epz20r?bq2@mb+N{vL8#xzt1eX<=c=uv5B|)-C6%r zh&el-8d&QEp5W$!4%_AL6LDP%mQW~H_6;G`hf)V$iAFwht@4L6cW9CzqX1-IC@p}^ zmcj#{LFo?QseuA&m|V!VY2vdnu%)}ohqA&gH~9n0d-DfCBKpMj-GVy3<&&+j4GKBK zKA$%GoNR#(D^+7rW&D{&aA;I!EaED5f&oYajy)4{$p3-sjNNP3ya796kG;5ybbi_H zxa`05Rb!o>V~rS^*d4a?L3_s{D+6ac zACgypa`LzW-*%z;tW+6eIj91wRU}Zd2hestXKG+X|72J)Y{!2Ez1eKBKZHKQznI}B zZIjAC>@LAas_H;2gchDuN`&}`DvND@1LRn|g5&8dHHq5AziTD!?er}OlE|tE0z4-d zYFkapu3o-!T;={sS1E^N*6d)_zD`}{Kdb@`>}M`XC?ONO^h4!O#a2^P5sdJ8`AgP< zbpKWp*)|YKga4OkRLX~tni;RB3Td{GbMO*bAEZ7 z_K{vsUGi}m%}l()E!RGFT9Z65g)M`qPP_;B5zOBNSWVxVCU_z{!Ls&Aw}dmBtR!e zc8Sd-X@u%km_j}krkGP+JpD{@hFd_D;iFv@uDPMK`?u~mRD8(f>ocq8kuCdN&JrN~ zO9inBV^EXZiej_urz#6mqeS(+o1)m32f_k#Dv}(EVnY=bs{V}^AMtQG=d$9r@+8|% z-eQ)eo!L>H`VH0^)MQn3dbTlkfX~$`rrV$$HMihH@-v0?t=bhE}My>5A z7#+d9g8D|`Sjwufaue!;`;%wPE5PfEe{B3S{D`VY@r5US{s!MY2b^Q}9F;Ukyu0Rw zVtzcY!W&k5_M7=d7~N|%X6hNKHs&|;i%?AYn|Ow8{%__NM$-Kve*rnw-Ght*9-6kv zObjyF99jgLX{Bi;#Bb}&-@fJF@NHK4@r<&WSPJ7J>{K*idfDxXKe?r#Z|~d5&-l>k z*|&kH-iYFQ{8d-l*6H3`?E5#_1CrxUNlU;5N~ZJWI0i0S;>~tZ~Oy_{ey-KioR4P zq}M|~)K(2qj+vNfQwwTK=Av^Rp9LHUx>+dan)n@uzg9V%G%b2Xw(UvDtr8vn3K+uU zU6B_Glrle8cMy!$lxzeIE%f8|(oJ5Teh-&Muu$1W$c>H8$WYobp#= z2}TKDbdi6zj`XNWH*Q#iZLRlF`Z&*uzbZ|HQpEglXmy+Jb}`+`>e%n>+)L6MCG0ax zOl|rKZRIE4Zviz56XEMj%3q9&;`b(ckBznxkQ0?^4V8@SwM~H4<;e8J^Zsq}>T&Sz zh+4~ZD`}>Y{|hcIe-*Af{})0%sf^%LpP@Z{tN-XH+AcE#Me;9MxA^ zBdT+j)h*j5%wXz|OZj4j_NSV!s!c96S(2R}{{ZDqv#RpcVSYb}cR@CT)SD=W*82(reYJgf<<$_G>O9_F(we2*_9;5nVFylW&2ghOBlJ&>>=LG*YDDSR zdy!6~v@tYnle(y&vKm##7JX9uVk^$arAb|MB3`UhqoIo}r}HjV7H$+&si9Jw6aTJw zYvyPx-}S$A@-EA=Jdg7{Z{>N5GUNV^t(igGp8PKp{fJ~k;&*l^8@u=hm-_VO;^yJA z9pwn6zIArcmcGqamOz%F%2JTSG70I$ah(P#MP;K(__Cfj@d$||28U7NLCYmotoWhW zgd3m5+tLBcApXmSsEvJcHzgE8`SO^sPHcu9qZLRyXbAULt;!C9ZzFoOkIJ8k^>{Ue z(>tW)4;fYuzoTh=2>;}elJM_{7Ym;hvjxE%jZMy^3noCNb-v$ ziI0mviQ`nuSfurFo$&i?ju~M;_~hIyE6}W*w?dF+2Swke2ifs9Wr`bV; zRS2_r*iuq1Vyda*qeA5N0^ z`?Bq^4rl|Ok>)EcP$uSe`HAU*ndyRYqYJjG^v|OUCYSD!S|(kPC1dr{ z6aL#*7YyI*uM1+Bx1z?dl!IP-tA!+vbmm{x#sSk0AQD?;E$3ehy5#3{1x+=POfPCx^q=X$bVV0T7pTH;n!5DtXNuQMUGvO28&EI%BhEYC0{lT~H z?3CY1Pa>ZHnbAtiN#GbJth9XJm6pP`k=t79X{{Y{9TwZOc)3_@9sg_!y$YvI-mJKu zi;K!&G1Wk&ycA82%3s2E8ziAB8#F;Wey#8|o_6R1^W^ZxgT`p=UC~+CqI);@88@t8eTQ*;!yw8#< z_!`19Sz(M-2e1Fr;5$B`5qLTsmH$4rl9-cIN*Z>TB|O!o;53e>x|F#*bSb+=m-2C! za+d|Qq`--p*GPoRx|EGvf&H*wm$I?KO5GqWf^CQ+5t|KDST<2ewO;CYNbNYe`{Rkc zl)b0uer}qpsBRYAqH4q&y-f>xo4BF3$whC|YUphk?Kfkp5_4RKJZENjTeEg1mGqiqfCk5? zE@qA5s%bHW+=hD)Lvi@30gmey`Dzo!qH%K8S@c4?bf%py4wB^feOUH5_~jaIGnB~@C@JYlATv6O=7Y$I{ge(9Fm>Z!H9rx6P|X_ zR3tnm$QWY74v%8w-~M=){|lta2MFNjE<%bj7ZZ&4Pe#>@5ob3;vuaBWlkB!+o%c6^3Xe9EizzJ8{$fg$ z?+`UPhRMYgH^BE0Btcm6SRa*-Wj;SUxXE?PPp15Q`gszJeOb}1WSh}u3%R~V8RhvO zyu-wRAg^rxM5gI@>!-~t9amxNQ_#7u_57jCM`sHa?m(8qACIgHmU!lq3sq{ukc56Y z#er2N8wuPNeDgmxu zOC`v$a2k2kSVpmn_dXy3v_GC56~&r*J4~_@@rp?}-cVzlJG}m!`g^`^sg)EfN#5Yo zb01<>hlT4Wz3@TVVb9e;XM`VK|35c@r}4V$!WlJM`dI76ju`3R`1bcKPPJrZ=G7PBW)aV8m(8 zozYd|T4ausJICS9F>Gh7hWN~C`BPc{Msv5=-1{H=?Uw78TfwVYN9AV&U>cGY*Y`j7 zcUrFbFm68lqknH7!b#-+?Qc)6NUNMl>-&H9^=$mJtoiT@ziw9!V&5_nmqdid*DIV^ zN_XNVG;~RT#LKVdP z3cD4J+IA>MMX+%Dq7Bt)7Apjzz%`O6{kGLgG`7>*de;pULeT#DYs3?E`bFjA*v6#lAQ8+JOwCXB~G0c2vt&>hxac-XwQT$J&U=3EH`CLRiyk}O`~Elc)jI#U z(PP^+n@Hb(*XkoJhu?Ls9rYpJ(9XaQ;qVo|rD=|YrdbHj#Wk*C#`W&()c7$idkhh7 z{<-Wiiq7-R8i%uM-M#u)>vU9%M$>i-)SdntZ_{TVI^FvIw|{HuT(|t`jG+kf;kwD7d`t8lG}jf* zj*mXZAjH~&h@?IRLPi;d3UQmdsPXsf`^2UGQ@?AwUS=HWC`0#9%nWrmqVMoAWm(uA z?47ChT8}oO*P!HXL>+T3j9X-Z$uBBqKIuYC7mA{35wR2c)eGR)T^8ewNgA?}%Nm=ok!C~<7KDOlXo(CbyWpSyp?q{KjkL;PkCf7T*vzozOG$zkUU>l;;K%a69UT?3Vtj5G2P^=Z>D<{yT4D106lR zv*Z3}94=_rI~mt-t9u5dRW#}!Wv+|=HDC7oYA)d@rSP%$S74Ldum*(o;-arukcX~N7bzBpx@#|aU)g7n3LhR?Q$~O z60rx!B+QI>xpyp$?hz41JgVksXQJSpkE_fl7$4|d37Kxb{f0VQEvf~Xa54@C>?6)V z3;-AEI;av!Lh3`Nm}aaD#>lm9iIqUvvih><2B2**(byaETd7iZi~&M}Z9E~#WuS~o zCWZeNu5mu55{-ir{*ajGT@iBUimLD*{%p&C?z1iH;cB&nM~UjO+pyX8Jj}MPRc-+s zwp^urBg*P!Uw%79pt0ZXJm){~6&1T~`G54~c2PjGV6?P8I$IrZnJex(&mJp?-^%N? zfI2Z_V2RR{jt&KZ*QjC&g-8b{)S?QH&TSd?J%-hiWXq3bE`>si3fd$v=GuvMXgzt&CCNA62 zk+7L4BiR4J=W+JKuJ!LjxV>?YAg8wRfSBkeAfL%ScQGyhX$F}}cbNQU(!kog!{iy! z{mTS=I$@H!%+AvZOTIfyej_nlyDs?jKn$L%D@h`t_OI*qMH)nokxlYM@Lg+?{m;C$h0&%nE|=7TNFj*}fAQB_!fE-R*=hOT`~!e>hl7c@ z3I<4K9vv_-sn&7EV*HqMqoun7WHw*%pWfG27$n(tvAiO`9w8!4I9DycXHtZ8m2>&4 z`hk4kI8lCW_vK$5U!K_IU)z28uk613YvapPyZr0BFaMXjFaP@Z^5E&dwEOaBc3*yJ ze0j_6{*B$2|Kje;zcIeNZI{0=bvX=hlTmr$zBU0cA_t}e3uHA_Xvyc{P>ezx2Or?q ze$EcXIZi5kfDhl-HhXM*7P`6SE@*oD-Q@YbH+i>ivMY`-zKQ%f(?{#{>zvB5IIPH7 zuXA5gc8%RiDJqwPT@UZsy?0tz>}5#P&Ez)MaRV6K+CKLG`Ll!oP9js~dWG^&TEqrZY?^j)x_Kk;}MmjvZ!9}i!Dm*Wb)d^~)K zSHylyGncOMsK08z96!zt@hiIFYgXOSdscaMI(m&ePoEJyCGLlR!k<6`pvdX_XTeT~pcs#zab3;yZZ zqFSD1+tr}p9Q%QL$ic{+8uZT=LXD+G!hO67d;pCppH}~~VY23Tl5_5Wlt-bDwsl`G{ba?2ynp(#q_*jHOF_U@IdF80-^DRK*~`1or4P14r~c24(nPRG<< z7e~s+QVXj-&c6HS{~2}caYG)Y@L{9pZA1$?nrs&+su7PF3Jrc`!~n5;epB@P3QrI~ z#Y^Z?|9@Ekl?X)~A{4shJrG|mO)gPcp(s?E7BI2r5;rHV3)l7^jLRQS4`FD&7W)6! zpC2r^f&YAT_h7*-`RAh7xxxQl#bS~a#$_tn`@QhA3H=hGR4@4Fz;(s`x#*Rva*)V2 zmaBp&R@8*ABBx`~mdSf$V5;5~dL(wFpdjNf)<)wmhEj@m1*;Yye=(F0Dv^Ire0DGo z;s`$y3D_o-*s_Q|n-qr$S1zE(&cXq1>nb z**#BnmpyedNP};BkY=X_iAWH;Nf3hpTA`>MUQM?21MkxO;X)k9Xj{P{G8i;VKiS?D zSbyllmMWN-Ql0R7MA>BVjm_&bVhGam-xHnx@#ZEI+h-#p%r-`PbNuM+0txltyE=Gy z!+MR$5R~U!v`|<-eTV&p)J2?0ehKeEH9Q+efc_+pm4`&wks-uYB8| z|8unVi7Vgs;hze#evJv+Z)~5`ZTFe5AG!8T%~kvmaruAuF|3}43>~?!&m4aGU7Z5` zdYF{NeIi_5xHF^GI=Vf2B?@=>e`jpIDZwXrf(0Jib6sDJsvD$6S3Yvf zZuKlEqR1_)BoRsI2i!9HccwPUzjYvJ;E)nDyPxM^uy*R$mAXfahPVBCGgnv6H$%T=^^a3>>$NC19u&*kM12!mPDd=z2HXDq#2IoAhWq>kH)rgGI^;|RqB8I@dZVfXxBaa?A$H1| zmtRTI;LYR#f~l z61#WeBmemCQ^uY{o5-PtAOGrj;Kf&v+A=UBY+SEN4(?oW0<(adh(YXHRqkTdC&`=0 z0YXHY`M9{3!s*n}L2*_FSsro_@C+8bGso&W6Ri(>(Dtaz5Pt%g`=wC-xuX30;6G6G zv?KtiE39l_ZmqM@rPO)?jxTP9)B@_Me`zrBHRKt(B*9{hOY=mHYIx!oBcCz$<=;2j zBeJTe;Nz(v@y!!Y|LvWfos&Sw)6-wvA$v?X$ZJkm;2?)&CIl0-eiv(txnR#IDcB~B z$i;Zl2JFpV*t&#xjsXE~#4`aXmKrb#mTrPsT?!A*I=w z&sSup{I;!8)mK-dh0n~B0(eM4(G0@s)}=nci3Zxs|EgmyHNx*gEBhE35x&3WT2NNE zu0&al9XkErzLeD)?%0c`tUe}M6)D%5;l123{@W*m>BDgyzPc=h$ymY3^e%)YOI@*r z_rMh}PWqm$so-jG=k78Pcc14@l~{>gtfKX2w)!!3z7$7^gYLg;4aO(p+kCuLAXKeL zf+;3%Rd$U<)S|9azA@cDX=ADQp$g6p>)Hre+%Ad37*>h>g7C8}@E-U|I?Em69|B!LWR0}u|8$l>HBW~U6kgI4z z1o@f;h}}aK09=SDIaVG>73&E27leiZ1*2vvsy&;Ya>Y~BJ2^p+m@=wFiEVpOlupha z4TtNw)>Gxb$ck##Efo|zN7;B;eNe?PjJR_JyLgraOV{@Eq`Oc$g#{VTe7ceuhvy-u zxV2Rv`Q8bI;Vj0gSo#WE6hm-Xpc(p0L<$aGbB^MI{oMU(;UoWVevF!{=Hr?7_}^2l z!#`smo(p|=mOi{qeRx)VC~UP*`S;+K(6wUZiigR|Q10&>*;r?LZB zRA_WQ)~_eJR{xv5gY<}6vT|Jgwx_IXmhla2OQmfPzmLk#qy_{I={AvDl3I9Mv!y|8 z*V!h0!22cLIU-iWm`h{fIBj7n`p%YgGE+sVj!}Mvuga~lQ_LqXH}s7ZZ&(w`Cjl|# z@Oeh4MMMzojCy>}#{A`oMD-<@n~ z$we{yk1W&A0wPle)$^&F<;krR@om)Dy8JnZQCD2%7RvXA#OL;YPs+i+l%xuxW9BA( za(iIzfB37$TsV$L>WJ8Vh>G*b5GCy{7)-?HmeS5swQ!aY@1U+dR?%f3(WG)_}s=c zZ2S8G&dXcv+Ww_7nlsIQuupI_kIyB4Z!<}xi8OO!HF+BY&|hi>mqHVhv+7D_wMiVE zMXeKY(9)D9gjOKKtOZ+tj)R(#wJ2T%jo-a>beLh_&U4GKDLw~J4SMc)-ONv~8x?A$ zYy?Yn9(R7<&71N#Z%cLFf6n$s?6MKX;`Tn_HqUlgc~K8=EA{}2CcF95AG2i|5nRpC za0~^fVL%V1;vb#^J#%`^bP)+k!M=*w9{VoYxBGL0;`1+!XO)72+pm`bcg2IEm_8^} zu~vo{Fol(zn{!L$be!xRsAYYG+Q{f zxrtAA&C`I2r8{1>dV`MvLxYvq3gsz+26|{B&`-p-S|EO0cp0KL7msdI>Tiwd;o?UHO%gUb*tBy68xp{x9CXdv(;LWS^@xmK#rRha?6xYwS z7oCwAK4zT+Cs|&;w{b@8w?L4ZCHtSv+boc#XFqZI zLH75Xm3gHOQ)>LhY=UMGA=#Ft9`gC! z+_(Zk+*BZ778{^J02yFRr~*L@V~JeRYi_+zrPxAL?ezeWD>ttB=E;=@kbuuExN#el z_8<}<+gL)b7eSkmLzSP*0+B#~ZHH=?M!dkCnpeRF#~10%Kp!UGZC2`sW6JEhS_f;2 zin^jhYenADzgfNylk!%d;{Kk}&@U5Fa)wyw*gT=w=N4pW3jYP{8-_~mFNRzxWeUKs z|4+XG{fLYF=cAE-A@BpfY~+9Sneh3s$;TmgW=l_ELP@h!^84O)6(IEbeUgXo8uVP( zzyD&(CqDL1-rw=h{c(Go58$J-V7+4h8#?a7q4(DK(9~~4zm@pD&lI>)zMcQt3opEd zLPuG+5Zw9|?_x?bBwkE)__2%lIgD5m=*MeVy`bPeZo!6-%H4pl26qjVE&<#=_UWiv zHpzM*B_7<@fD+asQ2_7=s}WDLVdJBMtO0S?fuT1DSK?2yrZ@x!cMSxKC(P00Sl?+7 zr=6X*G@eu3gZNQ}J2Kt}%x>&3WvJ%PJSUL%gov6vCNUwd)FrpXHD|yiT0>%jqBSCA zGq{ndf^G&h(zKsH873#iJEUCG819E$s#}*m-+|aN6;y4E42VRGqRe(Ul(NAfLP}F|?3VzL^a9^}3JPE(_Muo@gjA zzq;Jg+DBj!k^6;o+{jW|W~owO7v+}!LOLpcK9kd4uBLeo`Z+AO93$)v;5O#P~qxift#?dg0=cYq??-mtE@eQN2{&kDHv$L^bEJQhTdgqMF6z3?Vo)wuR)D zstY@a^@sGaInI!Rnx%M|o5NqBJm5*@S!VvN8>(Jr2f@Wu)TROyW4^1dXV$IK4-n{kC;{i&jN0^R|BR9oriX36!@kj9GSgpX z%szfbHh_<)Q$$9?6&2-tT2Z#hD!mNqk4-eE%m1LPgnXJIB!4kk3E|*IK~)h{_R>)w zp>zu|sP0`sMP8Wb6pq7{is=2j1|13ynM6o#1Rg#TRFGtt+-bN>WSpXt$Unp! z?h|%T*!di56+%rcdnt7N^rmp_)0i|#u90|~y3j(;$Ly07EklZ>hX;o)SY zD%9vF=HYFD$3tZgX%!NhqC6%-^X2zbE|Gr=H2*_Bw}i|YpOO(cL%9CIivkK&1%YvN zpN;6?)vn8pi>jKD#H>n}`gi{*wNUU`^Jjnt9}!XEPshz}Cik%gO2PIz1Y~!3A8i8M z1Nr`*_u{JB(U!0%uqE`712;bTd&tioADH~?cM5;;|Bwf*<68dW`(=1RgPKqQuc6p-T$5A>CZpzdhyTVj`KPj`}l9vd!e#>03M-ZlOzA9TYG!Sjo2EdElY| zj4Qqc%)d@nE9Q{7dRnxL9sdksKcz@ia=}0Qk*=QE6Hj_CL`RWVI#RvjzeZ17o?FEW z_X}@rRT_pBVkv)oUt2w6SXN>9#^q3n*6Mh~=GS?Yie@-|y^TLUZ1P3p&EyyGFc+K2 zll+Z+dX6?de!mvLX7Vf6?f+MK2v4fpdO%-8$KF~Gduu>|W8e=ca@+;I69%$v@x6{n z0^AlP0ni9HBbBVmdr%VlR|L@DA0*K>{TD|`>UXC5pYiFg{zvLR1WELNK;Ciae~&Y1 zs{gWw=&V@yV$v;86w+sOXTkH{r!3ks=j@&3q`rYgBMs@tN$dC8V;?kn_2Yg7k90pYh%Zb+m=`4(It z*@$MdZU4asa_zEVBy<6ZY>CbE}QYKTfe&rt@rJ@FQYrU>aTv6mi!8;-GQdn__`uP?a&b zs&uJHQZ74D7}lv1MPs3m0g6%L2z%>9NqyV6s6wLiJ$16mNXPzGf168IDIH3jcbN36 z9wc(ot?v>%Q$bFPAu}HMfO7eO6gb+}5ep&p>lih#ZV*^%j4xQG=-<>%`cbJl7bD4E zdc&>OERBm(5LanPvEygN{%`#@>Z>Xv>U->t-wlO|e^|GAR=46@2YJl?JT7f3L|qG% z#Bnf3BVqb@gRL)rIx8rFEV{^UQ$%QyB0`IvmTJ z({+}dd9Uo}u3Oxje7QJ9gr)*nP4Z=Q1vO`JI{A`BFLbi-r|CtXUi5eMf_J-nL2~}C zUi9e&*X{0wuG`&<{@z~nzu8{&r+d-y4`h&l{%$wU;e|als&i;zr~hMuqq;ZYPgK=q zAfOcQedX=ZtHmo;^;wWBuCo!n3hHVVP=+2ND*t&}7Z9MyQs`I&PPLG_s>tB8t*S$j zT{wMo0#4n6l8{@IyJWx@%B_m>#PL?$4RN{Vfog((%el>Dr66qaH#p3~r{#Hls`3^f z85gg>wd?YIIe5VVaIRBfmbBrDgD?)_ZYlXtnP3dJY52W(zoAppMI%63`C=7cj2q~Q9{}2jo`IPCd&^C7v6CAey zPzMFP^Z$>%caO5{s_#3`-sf@ORi|1dC{zvXduVWBx}~1GeN2X2@H-L;}|eDiX#UP2wMX22yw;^G$3Qym_!7e)UwGRlNqzDzIE?D=j^i|zx{iBf8SqMpn9RZREzGn)uN9s zbhT7htyGA0buiaeQbJNm>@$?SUh8`A`PBAaY0%}Yn-l}II_KZ4ZdU51@)RDUn~&Z} zycKCIMf{yj2YB$Z_*G#E09~rIE;}L@xoDqv$!D8mMlNbeHaX;?WQOC6V(~rVLLYL` zyd@VMiCh$ne|+x)<&?v*BNy$9JtWOMlWn{roL>}EC539kzcMjG@5{?O9k4J&X_H5z zp%$680JVG{n!c!xgB+fGbCi~wEMZZu^jUdD=+{4E;bnHNea3wD`mBQX7In@s7W`2P zLv2B8j>DpU0Bd6HdSg+$xe+qRD=qKOFgqFft;~W zX}kkklM%a7rLDwgYIxce7=&I4rdsbp?I?F5;UYmYXDm7mywWJ^! z%C*F>?$jVlpO^DmwsjGArWhMk;G0%*PfrIbD={sC4^&)Nct~|zJ-KM>joTFZM#7U)|!I=~OCENyQxvxJ?X=0Bq`r<%shZi&Lai{)>w58SLmWETR>EnmzRk1^Q zsJ1&KVVi)Uz=Z@5wtHe zhlee>XZ{|^QH$xYSz(v(S8rS@pK;qT60MqR=N9xo(98SlxN z$rAf&H0{T`-#?H?OGXa+9z#pBfDT)PoYf%bwCJHn67Gh?FRynvT6Nwm=W-Roi!z);ez7k=8om8?Vm+>pj&rM)hL78zTnR z)K%{4MwnE%_(oK zw8DSUy?v->BAYzu8+PJX9hLSq)=zDP3?m-u7s{pJ?4T-=cS#Gz;#5xVG?aU#$ zuuAi9)<$CofWoTZo}1)-JnzF1&-?ht*mZl_J&s*>NlZ-(yUuAT;pDkkm16ZBv}8EJ zwPbwgwH~D`RPchf%8*Wic>ZgNrOzl0_nL{LFtJ*EZy%YRUvD*oMdi1cDZesIW#8xX z%5TY$zs7EDm0u-qncCXp&?*jAeyIdhu+OBv%-zD2-|j9dvRDhRvXAnso^{G^k(A%` zttr2%U_>RF#BE0WBCCI0ZB}D`qp?v6Mw|0B8dIa%ddBGk(}{Uh;U!LGYjQ(sL#_Ag zD!`gJ%Mo&f8QR_7w1%MK_g(!P>^7@OVtk}VAJ)p!L8?X~;$QKK;1Tr1lDIFaf*LtN zl{bj}_js_Xgww;k6|!diBP2T^n;%%(dOsm+44#5T1J6$>=#6JLYMA)e_}tN3y=wiT zp^%h=z9NN<-%M|%{0P-h@X2~@0gu<{(M4yzXiumRI5sgw#~)R6_Dyu=V)hUDqlwNu ziOy`pZ#2L!aMVx(Ktbdz{@R72{tc2tdhj&R3~CZJuRp=}RsEa7!7Fzz^k8$usbPWM zYmIG-b3vYuIYo_^!|R;tKk*;rRIMCUr>pp0qOH*mZEa6G zlNol16rtSf+brj_wM!~hQO>lr5#r$o+5TqBYWY2uvqYf+MOE=pxP+f@lrU=Xvsch* z3yK1uaPvNjcNpMlKlfqAB?&)kLK=oX!wRKxBA7ZVnEHv@FYkD&1E1QY8k4Puhn=h~ zttMf5GKl`n7{V8FyEMyYcuy)uMA4=oHmv$nTB{Xk7$gH3C&`+Hk*umerGOXNbsne7 zQ7eboo(Q_nhJR%;-eG#Z+=_q2`<;ky6+}?a&bci(u}N0S2wcLbR>U{I2~W{+6&;d? zpjLA6fWuInQEU}eei^pWdZoxBQ>^@O8>>&JAc<1N*+y?{J|(!6pe$;kx= z5Gg02*CKGNNPM?|Jf|!hHr*h2Z2|_hu&I+{w&SRdSoz&*(2kjEowBHYlbj!)9aet z>5LJ{tACI=`NsJ-=d~+}ZPo zVF);Me%+rvf0$VQq4Vqc+4G0DVjhFV4|TC8u5Jk)o?^xwBSVx&`W+4wpm6L`Alo)6lcEs9Nc4YjK7o!MjKPR>)t< z56`S+{#um3;6E1RZ-xA&-J<+eyG8k1?UldP4rX0i%- z-j4Gum-CE(z4SMdHF9x3%TYT9R&O{ITF|3P!5nj*Rn$$KdORFm!Pc4kXs*gm!ZS!0 z8=*vxlJiVqeZf2GPJN^!XvI41UV_(TLS414ObB7+=~7ru%q0SNLxYJU+1{fsKbnZ( zTk6)tk4}0&F0ZVqu(Ao6Ex|0?n+`+8Fo@gP^P=V-Xa~E=Er^xIo&;WMSn4!Ln$wUM z8(P&4c~%J0r1n#S<`k3(BiKK-O4xowxjrIFP=%#Uuo+qWJtd@bKaIzRf60n*yN~Jg z;kX;lh>%}%Wz$TOU6_PH{f}8I#j))BCysw!%s(8A-_I(zKZ(Ku34WI%bocm8$wISL z?&hH<{M}kJyB?Zj)Ps(~x6sKhqVF+h%&&;R*l@abHqfc0Ht>-Kcm+9<*b@&c%Uj`& zhrRbZ`K`=?uBjb=H&2noI?6B8lCCVR<%$_-db0+dVXUWAX()m$_HWn-`Prjo4 z8;PG4uJAng`hF3-wsYv2LyBw6=8(S@0V_~ZB4zV-I?0AsyG0kg+A*t%@|?%Ll~Svn zgqh#imxj9WGo3=ifOMpy2@(`}nz2!uOvIS6-h8H4ZD1JWgFm17?EJ_ZmHl7gm3fs* zd1ce(#PXhAc}}SQwO)Ay**+(}3K_Z4f6C2)7q*&0qD6l8q%acUl(vI^3(+#|w-h9z z@)PL-S$o0A<_4nLm{2m|b-yn|t4}oqoHWI$_{(3=SK|p>#{O7#*`yaA{6qTRhBNG5?t(K6wnGMT!-IeL1guQz zlqfLy1(zOQkLB)D!7S620d34nsEpN($eFEf{$h2{{J-q=h1H$0x@pJi-qWsK-AL~| zWP4TS`RbaNeH4U*W%Y}0A^*PpZdt4bGL2#ol0f~#+HBdc78ukf9Z5Qol2Uvm&EXO@_Tfh3&ufFxmMj&gPc)vmw|E|E@a8cKvciI>|<**Y4(PW4pS?}{wRx9H#JH4_d z{#a&=wg^*n>(>^#i;Pi{({AA;wHM}C&<$`$)M}={OvY~A^J5kNZX!hkqQZoX8Lkg{ zm)kO%mc-jS7RdZccLX#NC<_7baZ{JcO^pGjV>48wr$fWq`A*6?H#KK>Z3_wCiDtP& z)TL!r(A6}7%Z#V!COH>ySj;nZ<GD-D5~SQ>mnGI!a$E5FW4q6VK=dtq@IWyqgoE z#pzeOzinoiYsIGYcCeW5_M6|~vKMn2AsdSNJ#J(eKaLF>C!s1GBvBa1JR$A^$}nAu zFWeP-Zmk<%yCh zjASm$UnF&}jJxLoRb+*U+uel(k6%_}Uye7GulX`%=T$pYOtZ;D5+5YI3IuD{Z)4%& z3%bwYvyGpUVx)*sw&7pKr<)r`-3>NwAt>vcwY7MMIRR>1Nv*_2j^7!Q^-5WXiU%nmOnW>LM z5ZQ6~ymhn|fGUq%$hUO^)QLQdPz}cSG)tk18#mwE9J-}he&=Rb3WvV8>8-P7UYuGk z!S1)-P@0a0%eOSO8ZLitGhA=H)i&prOXObDaDYVhXYW1Td)4W$UAS=Jskb%tEzOFq zs=v1xtrI`BZY+hBFyb1@jyL1^OIB{t1uNg%Ox8)}|3TKGKNXfwXJNP=4xP?IZ~axF zKAnZp`m4gq=`2jvU)3y~&iuVSHPp|3Ak-fKXJEo#`0`F2wjC_Jcua-%F8{I+9I|{dtucvG?M2kLtG3y#Ulywb0 zmvs$JZM8SJw)^=x-LH?{&%2y;wAgBjc&qV!>J(iiR-@O}hkV-vbY%{kDg~+Z4z7C! z*_QSOMySLKIOPtbPHZKkxY7t{bfi_g*%;{jYVW`pq+4`jK)A7Mz!;;&S zer?BK|Nb!sbgZX+;xwP zbZ%*FVus6)sU&LI^cuZQWR=u3`7~s&VC~p>VFBORH)3ZgN^bP^!c{xKMp~b-JjL=q zB;fiy%F@oM7G(+M1BzBDo(geY1YGY>mPK?VdCJAiRWvVAYl}u;JO%PP01(A`L&<$p z_N>Ka%(v4%QwsGev^!fM7NoDh>l}&>7*mU8Rv|YvP-?GS<2mb|ODqtkFp*H71j4w1 zG7wuwQ|EHJ0hPu&le#AKG_Pd>*KQhr{K15A>Rz_uWvO?Z+0DCR8MAnsZf-AZm~7*S zWOe|zHUw^o7m6B~&cVH4GKYqXp zgM)h}M?`SF>n7TV{Hqz#e3t~*vZl|>l!_anxP4XLOET8%lZpbS{E4&-0n8&wj%9~{ z=w`NixR(HS6p4f+1_IkzQL|JQXzJS{k&dEfx`3i)f?8U0D-vn(%BgRQM7m>{x8@Wz zn`>@GBD?0PNTfTKQ3JTJ`ek#?tw?0oJQax)Nwk_%)NHP~6^ZPcry`LsG+T3un$0!0 zB9UG5R3y?Jn=fkCHiu1l)>K6zbIpwOb^*8BNf0ghFjZi_X(BiM9l8B`zq5b8<_!Q+ zZ1^|k*G(w#SpP6qrOY~n38qO_-$l_MYO3*-=Vur{z`7ME;lb7K;v#~%7MF{k`llB0 z;sEF&d^?=BCS5mZ#TjvQfw}9qR1?)D@2Zi{R@VVyBxDrxRZ<>|MT5>WKN#KTa0$tz z6L!|O*x3!grbxJ-prqb4w8Z4I@07>pyKzLyHCy&7dvP3dST?l)deIA&vmVcMU9nH?YnZc>v|T3dtA$=GPps1F$a%ersC?e zWrM|`*w14x8wG!}X2Ky#2A-d0ESqj@!laeqk3C9qjT967C)&MZA==ozf(ziOYE?Fs zX`VZ1v)Mr#{|hb{J318Xb<7KVHzR{ zAYj{hK?WiBF1T&ZrKpu<`gBs%ISqdPj=92dXrBnn?jAFB7Gat0(CNNh{}T(>Qx1~a z%=Jfc)9!N{RX$AeT=|JZ>Oouc`LDYJgiWLR?mM9Pw0kUz^_wYo`=rWTBxbaMSaFLR zNOBjCVhmzm6ADf`56V_MW^Jk`^U@ZHmm-L40voa*mpWZg zlRT&NSQTM5U%c=AAdeSc>0}p~Y!e)C{lA1t+QFrS8$q3vaHFcDS;&-dGrC7{@EOh- zA~N$|^_2o-pC6T;17C;S1WFm0&p9Evj?~V?`%J9c0uBo)Cq|1Acjf~QP*LKUqUD_~ zx-&wXkIv_An4&DJ&A`-rDL6xE$EqMl;zmFE?|OMtq3JLbV^!%oHHTr$=Ij8f3^zqG z2jnd>Gks=Kb?WK7aO7R4oO2`GIYyeuXmuJt365Udvf!?H3ny%i|L*=bVELrm$?AtB z;mBUyOmfwLbW&D^(#?pYUvL)_B-r~*ITWSR8~I#CiqeK(GuE3(#8u2k@&&F&EiF=3DB5Dj_+`e^rvzcPN}f9f8k zw4nBFi2ErcL62flbdR!tbsP9=>1hteXykLRqoDNc8TixdH`_CiU)?i`1vm&>elBi* z@S}>+yKx<(Sl~Os4Mr&6bab?I>=ztLY;AzAA@5-Skl0P!NWl4g9etFVkut0a3b8Hv z!L!oI>(xqD@47*Z_b80;iPZ zF+0~t@<8iQl85UUQlwsj{eWhW!4}IOslDA1m-G()k^7mW%^5?=>_qO;J?4(KR>ekw zmdF4rR4I-E6Lzds9?E}WPP(sYo7zM#F>D}$B4mL5fq0MpB6l8Yq&ITEBM7x4DXdn; z!<#JbER_50SU(p1EeN9cfPigf{OxR$V~M+*+A=>~8K0wkL8Xe8BnWJPQ6VBiRo~s+ z`V0egtvXIxyxcX{uQ&WwoFVB3@CszV1XoH3)?R6MI1A-O{FC21|KXi&fN&+-WY;p* zo}qh(?e!D>RWK!q-0BW~`$`6v^lkYSMw&1AezMukN^%bLx*|(aAGq=;A1jCKl3kz&~qK zd`agrk^BvyU9bOT4k15L_ap0;%IP-z)l+H6)4NRElAHhi!}%g5vNM4-Ckqd=YI z5&SrZCE5>*o;}#^#N!HQQ#@wrU*G|Ow68!Po$5bO;YHIhDt1I~q|7W;GJeEOsb2qN zwWf5y0OrU?oJeE5ZOu-bZW_3w!O#N(wPTS7llJ^9%X59v=GN z9rLqiPGVuPzh`Xl41wONC4@(K>%HwoWKY082f3H{480_P--)k=@0F#FNq)saQ)zf? zI{xZiM{cjpr>+(e5w&BvzAu(~)IEL^i7kw+(#9s7dc65W3xJt#vb4c8gcGVQpdWTy z%!e#(u|{N%MHruf>Zn^5P&U#502FNn1@>AmkY9%$RGguHI;T_x@c@%nZ=X>Kz{;Ms zd)Mx{Z2L@v+eO;xghCvOBu1KRA|s#zp^KBp`428(UYK zLJr8O|5uJr@6pC*&%pGFviyJLO^+lOr)G7m?66nw;(S;Ob0?%$Q0q5JXQY9F2-tIK z+Cjc@9W`TaFmlL>q{g>a^+V`g>Fnj7uneTmo5)y;NJ|5wQK20zAxz@0zK!c9xwXx$ za#PP#uL*k!^&H&>;aRhV2k{eMNOVf#m#;DKXJxdlo6oUsB7;H4La@5w@wq`@TEAW_ zK`U4fE3~;}@v{7KQRI$>4;01^+4ePsM+F=f_{au=v08FKC`{{s*$G7y3aTsxYw+WfrlmjdONWyAi5xf6&o|?um#5AAe|JJaIeI7Yl%kRWOK0Qtc5j$_D`^8 z1%(XOoX4@jnz2n)=v#{?55stPv%ZZ=*;FDIVm#i`H-MBh(tYx&47GY7-v_{c7sxN(0JE^96JaC!Sle!yg| z&$$D+Un=?3^ulTzt^{ps#R)%{E0(&1AYtz^pW%?Ubp6NFE1EmRlGK=Ql-GZ(+D_DP z37ImfLsuu?$)T5eM(23bWb6zLPLLpAiG&pfCpgQfhojp%=?*KxAik6ner;T_qsPLA z=qLpal#*~~U&v%0&wX(S-FE(q<-K1#Vec20@BN}2@!WbkVPTF%R6$DNZTS_V*@pO%wgwBx*E#B+4qJN=?E^u*HBz9m~9LszeD$#*7Rs3nUwY2QLuV?=UO%dep z*H7G7NMFU$W7wW7(ZLNC?h+;MaKxbCvYZPmQ0GH+rE6Y*nG3?Ft;TxYWN8$}h8>Dm zNZQqWnO>N^m9kI1<9f_Em$sEyx(>Kt8O~S9FY3 z7r&3kuP=bW=`bNZEb@h-=SEgw{;7_0z{IF%;Tv`nhiUJ>B%Fn?DqW@xQ#i zJmG(^>n3${_H;8ikjiYI+1SA_4l|6#&g5&_ASGQ@Dh{b{`0z}2hKlMRNsgRTVN$hd z%+X2e+pb5tjEw7O%^Izx#jz^)*o5=X;Ivs&9Lrt=6<-wjj5#~WGQ;DK+o_K4a5t<| zDq*r&m16$*KPI8_Z=^%|CU*U7DO3#ap{CV|D}3`s>W445zE=b{A=Gifs<3 zR$z*-3AebA4HNv-xv}v0BU$jmChzX(?n;15qA&npe>gB@`oEYA+vm zYw2WV{8n-?S-dvh`H_s&Rqf$&ZgZ4-Hbi$po88r}BYyC=Xf zDBDANLQ0$+obb;DE2mk24kJN%Lm98!=1{W>ZU`>18^mSN{@Od19WvJM3?AgQ zNWQ5@Kh16s^#(zyyHx0?)h!)v@K!w`rH@qI&lcz{Vtt?h;S*8||3cL)Qy>X{cYNT1 za=J{9*RGokDf9vsCO7zdIqAm_e!7}2QxLOf_nCj%^M#?HPAh%md6X$=_bQc!H~e@q z2)XVcbm|DEtqcb?gG797+sLU!Kmw?o|Bbt*KkHC=>ZeD`h{ z@BTbFLj5!A*a*G&GoP-cn=dT!m`#@{6^RljygC!xu3;A+coXke7(S!rQJ#4FUPpL7 zvOzE^Il;kkwNHE+8;-W_rRDxq)^RW1{TX)MaY@_xO{X5g`o(Zdh?^>yf)&J4(7;V;Gdzo` zeV9mJnP>8ueMfCD(fEWuP1Z!J!KyF-N&Uf`E+I$t`{9Frhvs$Vs%+(Cw!)`lS=}r_ zp`y>L*JP_?Cd}~8i1Dm+jwgw!f?P@1J=?|AvW= zJq=9g#V@@r`9*v{WM) zaQNcq+u!RjO3Xrvw2y;7ef;#d$47n%Nbz{H{!8EZbsQ+d9UkELz#VbmPY1e7I$NxD zlR&K{5WpCeAKl!2SDUmvLHn3!_cY-YagWOkgRGalbmeS5L!t^uzrR3x1M;SrtB zGD#p{W&r#3r?G2TFMR5YpWN_HvVg-R^y}X#BvYn7RMSD+-A%bF$>g~klL!B1>}@s^ zZE`#9Hv9&+)0x^@)IgU#)f$JYa18M&&H?~!`wd#2h0y`um+_;D+z(G@&(7cI!!Uj< zJK^8Nhx@Y={!VXx$45|xhVg5_o9^a27WfJOHBNg~-!#suIjJbnxD}$(*8?Ow^_Roo zAjt@Nj6my5GTPn8-Q8EaPo4DrW<6)0A=f)up9MIeJHrkCdNF@8R}4bq+eFbsTIT1t zJb0OXNXwM!ni!E1X_?ZW_mY+Y1xS>9{+rBM#;aWAe%@XM&#PS9VSFq*i~;C3quWI7 zk}qkj6U+*>u{)0O!?EnJwE!J5Na~2<;PH(_iC;d|)oVBx2aLoSIJgWpl7%Y>GHvq} zUEMQQ`gh;N>3(X9-{y~QR0vs(oTaE2=mj&wFN+)6 zOs(a2DF*6S9H!r|lq!hbnW#>Rg&lU6Z1aJ`L%%f#LoFjdp2RVpO|J;Tw}G4DGnEQg z_-j1E9P3Y)c<)(2wzuKm*iKr?v=;N9>KvxTbhR-bZPKxUn`t$Rv4FHVLE?0g_)aVZ zf#c4CCs*)j496Xgaaa(hPt;paSHp_I@>W>JV>Ik)TZbE8NcC8SG~5Ul_*@ILOF>~P z)id>B#UulYJL*Sa5d$TJS)ONVJD#rLgYQtoGi^YaItv&iRX{pG3T0RLks0=iL`7%- z(pdQpsZ5+vv5XmlkG={6lV)JLhNoj+K%ZC%;Fqu*n#FOVPBA70I>2gh(<_Aa+nhEK zpl6%Yr`R++IV@c_eJVl4FrHo}wJoJ}N|(G0M`f_Ft`#W?0X2l6B5^BTFA(G6?ViRr zc`Bgxq@)6l;xjgzDMtxBJ6^x85?nfh+)*6+{X_}Q`&@p*WYAJh*eAi~0Xy|KyX8t(*gFqDDJ7&dhRXZ6E=>gl(fAD(|yD_6f(A*(+I z^*+CAa&j zT^I;_7k*lW(R6|af{>j)oxk{Y6fP&SdU<%d0eF|Lo8$p2lcvz?qmiv34xa>)dJTcF zhSeVb;=@(FIZ-Lp5eKwfov=!^+O?ga7L#H1T$IYwWjx*hcf-Y-m`4J)OnZ@2*@TG> zNz1a+oe2OhoQQLG7Hm}~BI5l?ge?fagr)GLuvUK_f{@X?BBN8nv>8mB{8V@{+KTBh zni%G-^ARFx21%S*bp;tj4TBK8jQBRkXrQBpAQVeT&=kCk4Dge#bWO8tz`Q46FoAF= zEK-%uvHr2FrwxJ?H@8n0gyo^MOCL)LJb{Du@>U8Q%1?zSghQ7g1rA>%1t!&aJTI6J zx37D^PUlQ)4u#8GAOZN z#S^7Rb;bxm0EFJYHb}==WizHSDS5nL#3r4|@n>JPo9ErlTV;zLZ~)9s@a{3!PXtz* zUl32U^AFXpeUr<`eb_a<<1)h5h}F6G`jc0iDuuOU@1;kRuk^&Ii3S>G5R+-2u5>E% ztSizl=Q@I)(;h>t$jy!E_Kt31C)qLb^@Q*+-+T zlMszN;ZQilJ;U|u=3khN_hR|x_PChah2W^BB)CYPY8QrOu~3CPEZ<1B-Tb*}GmYW1 zpaox?FFrTXS_ZN>S}mCC7qnoiSg=#~nVp4&Joo$sEtr#ApxE}1>>hu^q}S>M7i+$G z+zI_uAINVrczafS9+t`LG%bh@K2e+BLb=Ip@k04v`$8~<{3O7oX&v8brNk!V0RuSE z>fP{1{?(iDc)!4dKuDf_Y6v)glJCL2c6J5y9hCDv?d)8U^ihrTn-{Iu+y+e^QeFM$# znG^nvZWKRh#_VL=V;|Sapq^k+SyUBUh~uqUjvL!E87cMPVkw&=>@SwGouC)hSbHMR zUEfOC_WC_icJBJQV|DKOR?6D-d!=l5`3;l)!BQ3zw-d6Ffb!EfN!uf*?7Ajf2M&e| zwV>8v)KN`BSfAE=1#(@SLN~o07y|kxBWQ8!B@o_m*P=?&738BBwbn^rClFqWy+Cg{ zj|}Rq6jy^Fv?rEPj-p`LtURWri0U|Zz?Pa@c;D>N<{-fjTZsN~*o0!K7El`o5Z>)z?R!!0nce269@^(NxQVI*l=y0KJqs5S;)M$r{@(Tp zuOnB`O7{5b0$i8x>x;j~8?}$0E`Jn1-GLEvVy^pzhJ9vMA%M*VFWh7K?&PNvBy7$n zcjLr-imgwsVo=S2If`HTWEns7#WMJHrF5(a0#u&xpC#tJbi)5O|E`?ypW)x^gny77 zeEuEm;0LvX(L0~YsiQUdK)USy)chRHAHJLB=i271JkFOZfNLg^sEa>9(C;gsEKm6R zxa`CKgr$97dzrQQ`E+B#lkvCS&yDX=3+*;;`>5f;YCrf}CHPy9T9hjbfz9|An23KB z52oX$tHH%T`qR>&5BIQ!87!AjE3Xy|?(wd^8MJVQztd^$fZ1m;6Vr=y^=TF+o+6e{ z#w@5PiSoHjk))J3oir2s^7PK>olt$dYno@h?)1)U!?R9*=l5Y$oOba$No9}U`NH*k zUpESqXXeLme|CPHSK+K|T`hPfl-ll8<6dC%8ZV?~hDqU6c}O!a+O{&j)7=ot_(@Ax zTuUlPK}iM9WHjcuBNt<98G>?x2XyN3(Tc5@>k<1}?^X&jRPi`Xu!hV?PE>V4dO{Tj zWFQk&eTp66r?$i~L7c3Scb<2XBSpRK>1eQAGzNJ9J~bn+A*9bJGF$_NIp*?(e4D`q z=pM5A_lxL-v9}<(AP zaaj73u21QlGQ`x1=y4T#=obzn>tc`TN0+7$dg>-@{d&(gYmU z>R4Vs>`(ZsNAE6tzLVdoFCmhLM!P!w%R9H7K9}A8>^vBEQvEM;(i@+h2Q;P)f5a$c z$S1g3zIIH67+ESTw#kT%xr80wRw{Ccfg?6PfRZnuEtXf;DW_8~qNF?<~4*Y4q zQ!9gr`da2$7hlah>yt1}st5#{;Q@EjJp+s;4qJ!K?_#l8|GR>Dgp{TJ18$^LyFldi z`#D{x0Hz9=`U9!C#)?)}rXtCCaP=>%Cfg4?*VXfyBiM@fw-Cw14u>q6V~jG6Ek!@- z;8~;aVp=&{!Lf{*A4bj$www&`*kz-iuUsYm*7y`TS1l5Col;ub9BI9Qh`cC{12a`z zr6xqZX;4Y}5x;J6Im@?^mPg}h8Gq^h1B6R=eIu%_xVklJT;lr{;HK5@_^S}_Pe(2m zKd%HoD@9Njd1i5W#5r@MWox5W)$AF+hZzEYMmN6uuzuwXUJb$sXi?72 z?^CvtNF?m#ivOiSpun?gaLo-GDw7Qwhk^~7F?88UYU|_MU31#Ks=kdB2{86}G&%Z< zQ?V3ak0F9Drk9c;%m)K|A{K%&VBA%%p>7zOzSv2|n^V{;lrCOt}ZCu4#fcj3> zvyq91V)JO0#gAsVDG6axA#N0kjxUTI=*1&O&0Gf0k%SOj{Zm#hNoS_&-(VY2FFUh( z+1aenR@T@CaL1DQ2O)+0J1PekUCyjl)STNnk;_)ci;QAN;To) z7k>A>KUq;*GK+uu(Vu%8IhygK%Gx0}8svbO!_@F?)eRwwUwQb=Z|~iJ8c<|T>Pkz#j1;tlBvX3 zNMKf^BQU65vlU-K@IxVXfp>f$$;Tpz#TUO?REnheTE z!^P+4M*pU?O4s#P(?61hKDAnkQ}}YuWz!+)dq{H3P%MVxl)iEGe};?MjpA`A>C=hW zPjnl}sHkSgmN0VEu71vD;!+`0k%i)(`fYARwRQ$UtTV?IR4|(_nZ2G&ORfya`pjcDv~zq3xaU|G7AhYw@_w{RaHM z+g=Aivs-MhHv>uYLA;6*)*upN7LSL)mdbRgC`%D9T=>fm?`$g}yW9@LGiRC#m4*7i zm{+|`{-*LPKDJQmj%Da=@^sX(N@)zCXEg*VT54^JrgtXp?%pn%I>7vltlxnFSjdJ< z8u4$~h7)C>>J4V`C)^F|^(XR?xf$BDHFZZy#D?Ffzf_#?8wrkG%Ib!tladwki&gP` za+!8CrEF>}JfR<8`2haQDK_S*xWmvQs6>iX0U}XlNnfOQd;b0ZSuG&hJRoR)hH4>dzE08;xGo*(>fNxjh}(- zt<>K>cM!qt3^^aO(JxHgxYY@vV^y*wM_QsDeQqe@pPj2tNXR_$OBJk+W&G@yN_Z!g zG8JG&6&))6&ec$wQ@unjIl>OK9OWmM>nkBQJ>V%t`kGaC59j*mObt z%>Kur(Qn3>pJ*iPt=E)I5g*F7mb}kJO>?_YC`3^n`bSlLTS-J2t}j9EtU#3Y0gJ~N zi^SCXG%Y*AJJh$*1QJaOS$zMWl-x%SqAdQp{Z)+Oo&TcLqAxh_nkpXto=FyO-@Sbp zDnc2!?I?tL*UuU(mjf2NlY0@8+VBwTz<^W4qnjHPPl{`25slx-w^1xYFZN%dzd|pr zz2eLnq-{J&KS_#-*XYwE6!E{)C#sb`N1v9l+Tw<8T5!{1cGIG}X`!2L=#< z!?sn}t$^Sq3)wUCaQc?po}C8@I`Is3>G%nMR7>0zID!gZ-=Pjsa~5Xt7yq1u(SJ{F zy?n*0-;cz<5ILvA~ldz<5BSzO*^1AxjlG(valx6z zu?#&MRF*J7l^4W5T31znLW_14DVFHG%-k|2Tf(r}HQ9UNRFLf)iqX&JuAgkI-jU4m z)B26pm9|aYU>HnuN@eQZRa2K$iyluLg!g{&Kgw>oInB=!(_whEImU9eEs2E zyM&}NUO*!+H%XxcquG7s3Nl-m1xuS&6pFGp)ar1{i!xDVdO3=XiMB=WLz5 zcxPV=9GYqbu-Z~e9J4VGnd$QG0Kv*su=1d}&>y;a2bP&VW-sy-@`SDFfcIP|+-m54 zu@#L?_{tF^FUt@OW!ZRm$#EbQH$0XdNeT*Cc32;wFw~9lj?6}|hd{UTx4rJS+UO9c zOH%8Xx@BA@Jy&Bmv85y!>SW?{c}c{w#b7RCPxl?e-hpFiAO}^4voZACd^U#NV=fxQ zCB|?`#cwGrFN)M@PqNF0J`_YOt}uM2KRP|c5sFng3WuJZXYt?NQ-%?I3YfdtTsGR( zhX1S&zTFwr4c~Dr2YQz7w4<&}K8ayy-@Iaf(^Lp*%DwKPVwle;) z`3A}En0F@n1DuKF+|kC8Vf!)DbF!oH3K|cgNJb@lI_OUeC1a&v20|cZFGgI7#KP4BpBq$mg(BC4iO1PpUt z0-GtF8i}NuPe&10Mf~S)Cpso}a-6gF_-yp9(v`|xrJSH76aaDDOcqJ)ufWEGw>$c6 zv8XuJMv(o>?wN%xa5RyJ)#A(at8S{257$jp1u=!-jUX0^l}-X2vHa6nYbaEBT%u6; zCO_`zL619bi{hL7xN8r3+>Lsih5N=$8NU_HU8kxbd-3BSAQ4DL!l)vHd}~@160Uv+ zo30zKU{B(xZ;5VlriJ<|US@^KNShj8e<)3j95GK%TZA9QA1dZ6xg`>5j*bdq^NTaB zc1E5AF_m5?Nhe1!@ z!bK_e5)U<3ZU3)e)B5a>A%-#(KyrIg85qoV%_9kI;?i zhLlj4_L6&=5?uXR1s*E7AXxgUz(YV&6?h6&;6b6$LK&C@ksApkoZ3dlMobHUn#w1f zV=XGC6=&cEJx@JIK4}^|sL{OgT zsz3Nib4wrJo|IAYq)SJEM;qM(7}6M;{9v zR6Hg+XlVf*v>cW~aZj_fFFL3O9aM*a-mniz8N-s*!^PL*-fgV;<*(h?3sUQ7G3n;`%VC`!n$6vuy#G5azvCEMFXqa(&mIGO#m12t5o7 zz}25h_W3fb=2P4YcF9OHqHSz+Qmljp)}f9^I@j$ce?yo0gdE zs;hFeanzv7;xGOR$XyWyq97(irk%osl9-=D!g@?0N`lOHfq;}U2gUMt%bE;&9SksA3tbvtQ zTIE6RW~#z<6Z!fx;fL%-EkZ@yI)orR3Wd2bM)5PdW&Fs0QexNf=Zw zaDcF~gv_p?Rbq(Ls<5B&kO*I0-=$=~627zJgyYK5Ye1kTbQ_MNhr9)4tC_-IRoPO5P(b^cKifoyb)cfqw4;wv@(9hIhh4236k_ujB#fu4G+ znc$b<&Ku1r`b}~jrAy*(QBpK$K~tx8cW<2vO{e^BJZpF)5<`wWYp`F*vsQ+lAteN^ z6RAD#@{l*({Ni_|XJ(i4#&Mks@xS%M!2FQv<0CHQynWSD%a(~+)+K7$P|e;v5piY6 zg|QYp3r7XzDMD#e@aVO20d=rsak|N-!Gl@_)`LV`~Eku1!gzSGjK&$?p)! zk;gwr4VQ3;uLNde%LE^V<*Lm!2}5hb%|~GX@IR~HngfUc4%FixRqo8=5zcHO1*O&z zdPq>r9OO<<#XBY8Ym2`DHy$Zg5RW9EKz1yo6vp+g zqrVpD^0<9sZU8#q{Ir?-;*RVPjrUV~n!k;86?ksM9fjC?3Ho z>E>)Y#sZM>F-Bg&NZ<`FKi{FBpepF#arS6nJreAqh(Dnb40-%AoWWHXw5w0OB=il) zOFbJ1aC>Mg__+RxSZ~Lbm!EVR{Q(Wql5AM#wj?DWa!G}5R&|q$(7~EcADc9r?pSt0 zyHZn7&E0(vvuH?1tEkE@H|(*xSZTp`*kdQ`v4I@Hp?G(++x!+g%(q}eFSTxp?OpzC z?IDHc3^Zmv^wS|LTX#rf`&#Xo(3|ZT?AdN?7m?SLWg)^N&u7=o*S0&xSp?TyfEp4x z^wN{wKw6MZqeK;8y16F_5(r3;0a3Ob*#9HT0rgM>_CcjT9f94LALqW!%vHUD9JAFXS(H9nicEbpK1Up=9(!>7}n!tq#u!ABA$4o1-eY%0{ zGm|n|{7czAt;pST*FDpE33t`Bws*Bj-E+@&$ET%o3HQ{7@9sH~3wHF=E+JmNS@BT$ zaUUJ4-609JTXX`p@lXSEi%N3z09{L7Xiy1fS?YN#&&K{4{D|gw+oN}9bEZ=sU+rFK z9?Pt?m857BTYR?%eD!?kBqXcfTffIOxX;lp&N&eP6C(d88YHk^!fV1xXHBO+P^3b| zty)U3Jt;y)lrs8vBe#@-rPB8l1vNL3k}XhkRmc7Va^s808`?fRY8KwXb=C{jzOJ*L z@R|4%IlYQt)!{wce#76LTlxB`D_?(4j_sE%S^<}$g3Ib0!lV<=6 z{#s<-#h^uhE!#Cz7}m0sz0Fu5*>qAV>`^9TtqjH9C{!_LZeLOqs+f;Lm7vOWMwh2I zu#l7TRc|)_0JlX#f6xD)+!;j-DaA?tgjEi;lTKl)(>prtf7V^0v_Tj_8XsR%Y{%CW zqyiRX6(}i?U^w&T!9%%FL|?zdY2uV2rI9ZI1gVhC8yS3ZNWK8Y13WfM>?&N2Ffi-jU-Lb<0Va&%@MD9b4IHimKLQ+ zZPKI%uCi7^kuiRRF1`(3?yDlAIzXB%fGC?m63 zlH6aQTn|&ZV5MAj0B~4SDerw1kqt`#rT1B=%%O<#(#o+MImT%; zlVjECSG&J$ERo&emm|2St@>`i`5i8MG0tnQtLpc-QL?t3y|>cId;)_DU^En5#tlcK z_xLVooU5LxJQiTA<0>&J6u)EHr(I$=O|ArTfMlHhY*Bxxv|<%;_vfs@07oruf>f;v z_^GN*n!hX-+?E^tko2>AirBk>FcsniuB8%wXcRrb@og-Q!XIP@sk-P} zX)1(y1Auti`3z!s7JUP;YvO)42fpPk`c`%-ghIYag-|4}M1{yhuiNyJ?|W5<@{&{t zL%Hm$LdaWNSxL++{>eWpLuGX$m3dHM)KE2C$>c&m^+RQDUhrEumCVtoVz9Ut@$izE zqFv8^pz@{pdC@%N7)TJ!E!y2=84@Pr7 zEAH|8d)sxkb;3VKo9#;1n2Sc)ADMZ$fBYM~zdYRHE3uRtcE!<{)xTlk;Pi`mOQ+Nx z&aTNmBW&jel-)R7aM1x0uYVh3wjpO#kpKj3w6#f1DB~}savw$wIvl@{QIi0QB_-dUbVt%G%KJr?Cr+x~3BP7| zKIyihsDBwCkHF()boGaoAhn@47l!d})ofChOYq5Nv5cj@2HS_?q`QaSZuo0OXW?ED zy`ZAxxVCF(moSid!1K``_T2<&6toRLMvGL|7s2 z%#BHM&Jju6=_S-03;t}=0|8Osg+6n_zXbhIievq83f2i``C|R_V*WUmtwwj|nrzje zMbEOveRBl%lV_kGe?b|OGrybho3ax6xbRxfrP1x|ulAy%)|TV10eg-xdTc+AuEU%$ z&9ceUMlNLV$#x=LI~^O!(@>LrUUw~w^;!l5DeKnc;PyIVI@`uw3Un| zVF|eW_pIY_{~p>1M`YG6xop}3pjjm@c6V>B-)YgOG{NJ|X%%;@rUYu2WPdgtO4LuQ z&6kd8wjP88e7d>GE#Wlf9=p~dM$E!|f=Uz47hFIhLA z#(D;8Rt?i<85Jkn9qD1`Im2`DjgVhps?Jk{-Ms)CKagSlRqs54<}Rlk_jactkp;l64EJGI$kdJKTN|g0DYboS?2Gn0?wDX?jraKr5u&NGsPS(ocTg zNVnLq)<^ekeYCy*@znY_76>_LO3b>NYg-x%4mEsWSS|KML{4tLS8)_v%X8sf%1`e+ z+plu*RnDJuL~N`0ci>60*E?1JmYOjd&$TB?WZbB9V8MIy0I^UZaGs<%rV^OKFBN3S zQb0gyUq|FXV>C!Guw*YAi5t;|zCVRh-G+azAQl(ItD*s{Pk1peb|JQ3lgGV?58#is zB@PvMf*Ti9CH)Wt0lF;?NYopov0!v^k?ED5W1^w{umGCTJE1C z+33@`AMZG5wzM{kCo0)OpR*Js^xL>go`IQ`d_flNf+pK~Q*~I*TwEiPs#6gZx{wj$ zB&N)iF)@kGz=7q&t`@Wr8h)_zgD~DywnBqLh;%ulVd5eDF}Em$BGO?S-)E`@6IGB% z(+Q3-$3;y};8bNn1gaP}xv+RTny`dt%bLE{A)hk8oNZ%#B?N>?3*cRBF&M^Va_@8* z7Dz4C1t;j1zqn`5IxEXsrllgr1CkO7zJO16U%2y=$nia8_yos z`a48fGNbWfgl;#{${xDi1rNOo2aykMrOIM9dWgV;DbO?jN9T$VFCgxi`0Lu!IiV&Y zVxAyxe&P`N4I>d^i#xVBzRv8VBgOXPsmUkyG@%aB`f6 z`dU6>BT02)-C z%p@g%+pOPJfT>c7Mi$1W@6T?VPQv(faocpPt*tCP7YAEB#8^LFEcweI>>PcTLylt2 zqLLkYHuN_`DcAfCu6Z-bCpY0E;EH!j?0L%Hl;*aF?JvO>O4$DFbNtgkm-H*)kn#&` ze{i{;&i)oB7JcKy_h|n*X6rDBzws{1+_NOtwtK2os_uDp?+SGf`{bhjTZ6g&D}Abk z6Cl{$ds`K9-5)7G2xW_>{GsCg(Whr+GZWf5K+{ zhlM5hrZLg-JWFilAG-KpcJpSa%`lUhgo@wj+J0@;_Q+$ky~}m?4Ug4!!UVQ^@6Hz_qdD4A0`-GG)ou z4EghjBW|$N%rGbd)SIZd^7ei(Hv+IM{`bMvpOkB7)*L;!P=wYiL!1d?dm@lTW690% zM$F#~#ksxNd~+$)h=;#n65i6H%rT)pr+4E9=}6e{eIU&D{A&QGcPSc%*EeRPA8^LI zi9C|_lQLMO?wB;x*Qh9#b+m8U=UGB-TlRVMuV%@&eL3K5s0$O1xBGn^$dabikp9HVY8A(OaA= z1WFAKZU(tPrH7kd&gL;M4TMh6Jr!bQNI5mCpa}Bw7?hQiQ>5DPv5fKL@S~o)shCQe z(!#C(s1Vu#kC?nLCWad;@0oV^=-V7a$JnfK2-ae&x6Q^@sikFNpLmp|I=az4usG=y zOS?W9L&h_);h~w&bzDJRrZuA1OI1HJ=x?=M)?}#mH7~}vb!uyBc3?{c##-~Cx?VNT z@Gr%8x~-*&bKd(b<@_u7(=rUPq&b!>-h;u~=1*b;pj|aE@%zz8!{$jqQFEQchCtv3 zRxL;I4x5MmR7v*vsfg+!5C^57>OZ1-*<5*IvPbRuBu4G56`+qR@Na%FPYMlt;%+oWZ-p;FEOwcS;jV2vxtW{=54G2PCe-`#OC#kPDwc^D` zua{3+R@dRqyaeEmDH4;ez=mV;f|=~U#QYVdwXJGo9sxCaZP#yho{ z2o3GLm%?>!SNWzK2{3~nMefH3q4P2S*42y+o)x*^iA%&Eaa&7;%Q7cE@18csb0?yr zxw?^$w}+&3CV??+gqcsI2z`j2zape^B98tXgvTv^tEp*R`Z);f3|jlTR;kme zXgVAsynZCd*Co$9)(R&K1lY1NNs>2*wRTli?1ah!A%G%vSX=J5+P8jkHX7$cA>xPR z?63@=eDq-7_srE{)S4z2`ev0}E5L^7cx@UAXlk-i*d0vQB_-?VcnC>J!hMS<6V2jn zMCIW{)GjE9ZfQpebzQ_?!|yr=P#5370=or)T}v;}KB;ZHz&*hm$C8*4cz#`xG=9N&x13#|s7n6WKgjK2%OeZo1gvgbLI{J66Tnbl zCW?G45dbgPJAb$!{^3kf$m?*1tSsF z6-DoP%S&n5>C`WP<)v(tt@P?~aspV+aI1PlVCXtpim?e=R z1Mi%$y_e)#dRVKfP*$H5u2#ex*7B2|j?)+I^dHW?J7jm43`bJb1Gi*t6q3j-9fV0H zxTc)*^=t7}QB{-Aqiu@bZir!z@`R3m5rM{dVq_LfNO>p5y-6 z9;}6cmaeWgjsI+Pr78UNeB$4H)ipkT?P@dcKM_rcBi5xmF8X&l(&c?)Ht+Ek&sBn? zEZy-Al9J!{;<=E+Y{27f7OVtV$?s6^@koqcwIS>_I!23f@3qgd8@sTs} zkHnCLmS{6)Kh|7%&{{^DuJm7jvx@oB|8G%fs8RZ_=hMm7?(XjH zvu`4RQo4CaT2p`T1Rm1Vrx)XM<{xah(*J8N1rciL|4>)uEB-UHmwyyZTnTc({_^{g zU3@p@;vrDG#E{nAk}C;aHjZH~v6u|AgoV<%#BWp|gi555Bp2M_`BcaFuH<=$A+4h6 zE9u11@`+qX;?GT*Bev?o2AWx9&2Q!cSaqiN9x3pP=6* zok?XMEztTQaq|>hEfBD$S*MyfrnEZ1BvSFT$WFn`%_D3=zvl%mo46%6#%x_er2%dH zv&VXFDO63H=N4_93u)Te6$dRdDchU-=EcoD`XadPg@OJ~y28!lt}|6NBr@w0+H-OF zm5HJbZvL4%aVOOIX`$2fNuy5=X!^8plcrD5jKA`AYWj5Ij+9nAk(+a4Mdd`;+5orS zWBn^^D!*~Pos$G)?oFlyBB`7H>83%B#6KlIU!`6y#wS$>VbL?{2BFUNk3YJbw63M> zL_H=3zcAm^ryFIL+~_8cFuM8Ejc)#QBmWA{*eI+Qd)aa>0@99b4~_7_*g< zjrcMBi%U_6IM{mnLXDa*diATysYw*cxG~#$ ztixjtB!!~pSm+JzLR_)*n-f>9CyKUGgy2I%$Kzla5>TY)qt=J%quCbSSud3%$n73%YoIOY}txH)i@FBXDei5yD0(Ti+d1RX~@Ts}=r_cde^D}oLpw3u{4i!@PH zXfd6I76tlGLTC|PrM!;%@!$!;@RTs7M5PIY@Iz}1${gIB5Cv!AHEXL7*JI3{5I!?q zXQ1W4?Nn?=cd?fA1Xm2KI2+$-QjGEvAA#g$c`rmOZs;6wYyF7a(7X`!j$Cs?Stsu9 zq;f<5*5qPz9I>dJJ&_X}+9p36OH4h~9FgpsoQW}5X`%;>UTTcKL$)<;jhYtbcH@E^ zrDzajI210m%r>^h!;E?8|K@snF~01EH-1)ct-H}|Rgk~K>O-Ou8@fc_tA43hl3lbD zq2_b1C;hS+jX1wp3@ObslhW~7EeWAc%c(aD2!&+t&2_xSOn{*|_NY`7f9TNv5S!y{ zyrwyKNAP^LOQ!n;(B2P;1hIazxabEC>V$N#k{sp^csRC3yQso6)+3 z?igA#HdRDxtkc$wcpU6lbu?Ha>rsGEIz#EX>}bx=H@hWH&1JWw1PW7B&!=JYYDEwO+wp@Xp}#Py_*YQbeHH=_*!eN*g^l=#bpX~&MpNjeHtwvMTw-b{%fUnMiH>5^NJZ$_16 zsgTE&nBl=kt@|I^Rkw{~Yv*oyfF$(PFv5JIe@%)}N;!+f-JL$3JTrYtJ#@RK7zN=o zC$}%gcVcodh;ihO`&tWM*@tA*1hW{!n--cxVAH4S(;<)6THor8&NI<#Pv*BwN(%v| z>&ZE`e(B0P(#u=1+j#8JyVJ{r5Y)E!wAB{71ta5d8HXKdtXVxW|Zs1NX9<7oY^H33So0h8f<0F&;8k@ zt=Q#HJyvgU&_k|*7%_QqmN49$A86LlS8C{z>Q{R820|&S=S#QYHXp4w1qf|2H*X%H z*?^X3Xie94tcH?7zx@Alyr1<2|iOD7SW4at>ea#hlLG4LRVaqpS7EmHv)N zJ)ZK*^LI`toN2hO9Mq<2k18im5iM8{%@y~YE;r^EqYX~Nf#R%ycDo}jnhm!pF)#0I z>N1z>$+tJ?x^q`>^I_nthb{{a<~qv3ZSXyg7+Z7!FNqVC67X@#7#qK-le0Mc5RH7I z*@y0LJzSz?Xed>b?Y%?hhsNDL^RY7GcA0&nbQ|g%{s(5rHP+5KE)yuW&Lf*nHZbs*Il16@I!X!svDDE{3GenN_SP;hn0EI3o z^Xa8FAO<*0@M0lF635J!rCVH2F4uFaK)FrNZ%vzqo6+ee@d;}`7bc3fE_i+VuzIa) zXQh`gA3`59{7wAt8JEDoKNY)?+2pElO#FAYyorfzF3plV`k)M?Db_@i0P{-GD-NL> zvY*dnI|R({N|ybfN4ft}R2ruuf5OA1`23Zx=9OZ-((??#@9w@;y%zD;MrKDgrzI)W z<5$I-r?=vN%s-FUr!_qTm-@F<|1MP$?n(=`9gss7NgFJBEm7HA5Mm}r=Dbr7-l|mf zEW4)IkFp`Mv(eo?y~+$L0;_>hhRgJ_v@<}U%kB?}#J1hdG80c~DlDR=W5xe@RFCmU zz8KYGuH@FZ(+CK_5liM^ED2o3Q4G9sW}loiVJsx*f{*IeF77Kk8=vP=kpg1Jd{Ljk z`5e{D3cWP9v8isC6(b~F=2qpY6#ij6U{0Dl5lGhQ{_K{~7zO?cF|;8TKDO}Lm_l*> zz8RgikDYJEyfIezk*^jM7>_5u*p}-(b}3Oaa9(SwSizf8Rv3R^%u@^MHCF5ls8_3b zo)kyMZ8wQ^-`;{4pYPV&?7tLCM~N0kQHH|S%KieVl}-Uv(H1}*YYU)O$bilYpib{8 zfZBAYUGZdd`oIFHGZa9bai`znHg6!xE@l-;%%>4$7qg_`v;d0Rw?x@1Em5|fX&+HG z@3|HK9&u`a<2d1!K8RJeGP_;L{Ze#Yw4PihNn~JKHx4jVu2bOnj$kWi#R?+z>nvK~ zj%-G!{AZ)9K6>CngxoIJJXJH!c3fa9cUko=Bjj37s?seWv@S#or8{Qs7)=heRsS+_ z$Ex3=25QwOq{U9KOHa53Y{LcZY4bZnX_EgE$0j2vDup=;qDUcrkywHGa7U{DxmUVH zd-e>T4;cRjhAJ2YfXKj&QRi;HO}HaREITgQu@@kMt4izlLK~PXvwK7sf zceCmpnMvft&+F7A{%yNoh=_@o1Yg9ORc=OSC0gvk5`u1eJri3#xfl!MyNC`TwuA37 z&-qk7<;{V}lElKdsej-6LyhGH5Io{wm=6gDaWII0)ui{-1NC_^JV*(YW#=ruCu3dp_wgUdQBD&c!k+Vgh z;0tyKEkeugpb9os$5M5bvIyDMZ;?gbeHt7-hR4c;4XxV>04HXljrO)_=eEB|b#Ds| zQoYe3E6=@I`2Y?2P2CRtit0jL1m{Eku=ZHexz(2iy6KM3qM@AGP zy%3!x7;z!mbaQ4en2}bxMVXz8ZF-==aDvZSJj3TLbdeYwp~f%7=u%N7$b(NrSOF9T zFci$j^D?t6#lJZ@8%0yIJ2#$;fgifWpW=HiF)N~azfp1y{CxG^w9aq#8Ld`_n^cRG zvqJ6j(%RH4Ap!7sO(8jaZ(C8l>7#nlnw+0#Nw^AX5Z?%J@=CJ`fkfZWBh0n$>Q(l5 z-_d%TgVo0}S`~pr;^z@v2d26sHrkXmg&bs)SXwfdW4FvNf|#pq+4*Kp69xjbIjX2P zCgx>W;l#X1Q#m|x9tKwxmk`gen!4%W-KN<(`FuHSY*^G$fc8diQMLL?CbxAMW0zA zZK4t{H}hU~Y3oRsn@HwTF88A=%@M~GtS8xIBg(6f3ziJMj-fLOcfybjN7z6d_7hT! z{ym9*{$o9NqWW)R0x5+;>-ulUVkj>6uYApFb9(A`Ya&J#B*?2z_Qoj@&hv0GJ`|3+ zg$!H{eUK0t&k&$4UDB)%@YbfYQfvTJKm{c?Ba+G72RYojDkR;J-+ zedp{ear@qRcXsREN19E-a_;5s71wTynse^_wcDb{uPyRx+`Nom52R_P=Yt8@qGv&TxsB@#?Gl_O)>oCuuPlmes=I(#r7@tJgjl*DrVHuZ^;BllJGe zPsa6mcjv)>xYM29%HATc2O;)y_i}gVwK3NnjpydM2minJ^Ko;hyOTdcFk@T1E*>)Wi6BY)-hy^nR=7D-`q#8(eB ziJ$vU>jJO9&ir>ai97NPZSRsvQ_xe#zxnC}FjlkjWPQ4|w}-R_bf*I^!fEj$45@RA zYLcswTLBU_bVK_lfUKcAaxp&74<8e)bwiiA6}5pW^t#Y%a<_Ez0f`T=g}Pm?B&&7k z-+NvCWtauYsg)*qO?IgrsaiAFCr$p`#;4e!3XQhnqj0yaCkKlU+^}Y4kNCJ9@lgly zaXaFpKzytO@lpRR;-i>}k7lKn9OipX#YgAJZN;v5>M>XB69q#Ywu}svE1-0Z+)DLp zy{5=^!OthbT15ILOGJa*t`j?64jD117Gz+MZqwZx8aYth+bh~i(1A8S~>0~1LW>F-Qq zX|R}Gx420pqae_BrqyrvY%Hx&4ggv>Xgy`#dCK34{JTE6$0)aET2q5X7Sgo6hCQcx za2GB&Pv8p!IRcEF;v;AX=~5T8S5wgccLDoCIF-VzH+hDM%U>s>>DOg5nbo9B+kumn z`(GC%_25QR5M@lwrdD-YoC$CFhLICBj$G9`K!*;SO!a71^DJWb+=EJ1_pmE_rj#m- zFqYgYr-nt*d(3mT-VDzy~!-;y1899Zy%OUDCD`R zC&edRG>DG%W?*Lt%v=Cw#@H(iBQ?URAv{P$7nlZ$^_CP>?RcBjw37<0jGM$dsTjPd z(@CX*>P}i~$>{+^=Y&1s{l%wWwWDC|kP#?xF7` zJRJd5tjx&hE-;Q&=%9mcz%b+tb-Su*pgl~EMcJOIb;WMAj)nDh$p{X)eA{Sem1ka} zkV4RzJ?PdiRW|gi+f`buX;%~Lt7Bs__^1}PjkN>dH*v(;VSZRM7JVrAxx`YEORi$u z=^$rq+YtFJv^-)fW5Zw%aE@(=rQ3Twxv|#VrF4r{%OAN#Ak&H^p3)l4Q>aCZq`#iL zwqD^{m+LjxHvm%#r|0E*(Ppb4U-L%P?v>^UuJo|$N*`g4Wt&Pf4LMCmx#=K&B9Y3Yff(^Z^3MO z%NBq(K#a}8s$1~cYgV_4?`clE(KF4+r`u0G=1%+6jkce9+~xk{>u^L*>vG>}FW>da z=$lt_y;HtVVomdy^F!vy(EnBhz-_hxhrxEaVv~8Wnpe~{=Y)YsrC3~?2)g(}=i;YK zo>xDb)(f^$M3S)~v{IsmpV|XTD z`Z~so!Hil}L}~GCywZ$({6JHqI}V!7G4??6x6URBIrVGrE5WY#8M|#8=Td9qnz0*A z?n<1kO&=Q}*E=9q;t8bCI@B*F`$o9V62mTcyEF7ZhXtv{)-BDR^AF>*iqvYJADEscHJ68QfS~{rZmfcYaygnM^ydDuW{VGuU z$}PKN)jKm`6YxJmhJ{6G@i&6~T8`wP^~O9M&y1ae*PErS80bL;YFO`N$7!j?Ai!qJ zu7))?E_)cqnj5`NJ7dIZ+*Wc2kg)FdZQ1Ta!Lcz{@|n4k1C*|IGmXb=%N*y(tQHND zO;t)fv_f*MT-Hody_8XRPm{G946>vMQs0bRevC{)K}Iw-kz$&X+tnj}NFi3OLY#@D zlED?0%;sHwl4z;zZDc0FTdqQ(XK}N){xb%V{AcBvx|uXQ*S#sNTgam+ zrIl^{dYjUkZ^09wc&@l|;@{shx{S=9qhcn|DDlI8HrlRnGU8WjbT!aSpeJN%o{c8; zONCi>=xT?`$;b1PhRiWxc3u}5k(Oy>}(nfhPA2I1T>8*?smYc8)o)jPqF ztJ**Tk03q_A6LKAV?gu%Hx#cuN?z9RJusF7fyr-tPMw#I3wG#pnsjfUGn|(e$=cV3 zX#&jW=z6kpa3)xE7mG7}GLSf#X1WiK!!24IE+m8l-NhnH5Bgf?q?>j9br`z$WYgQ2 zfwGJT-$U7Sb+F_(hkFcs5446`bVu!o`v2PRxTAh;0$3DG=&A_tLsj74#8*iqu2>HG zC==#NH|LI-__I9hy>8BbWXHc!K&n*^i|B@?+m=Q(+OTp0v4{#;dYPZ6%>L-v_@v9e zD?qR3Tv2^mRZj(FO#!xb!)_kiLkVa(x1L&!Fa*0T$i0KTc!Lx#QX_%)_#i-=`;gm_h5v!dc zQaB7lm(LE*AL|1q2wvG$X@pTRczG!pcwsPPMNV?>=iB zbznkRsqU|>_F2`%)6!>liOdU~VHyQttymilxn4U=(6S4)hbW%hrR7}-9?8nq%fPCL z^2I_V$lCOPgdPtJ>DEpE6kUj58YrM@JtzJEY>%6M`~xcBR*@eC+{Sz;`eW@T3s)p; zmzCfETI{BDE1~WgmE2_3qMN5ihNxoVQB_O`elTupm&ZPr0U%*413+M!NJj(#Jm({~ z=F;j@Jt<9D5Q$5HgYJ*1m`{a-Z=14O|6EDblph`xXIIhZQ~K0giGB<^_kv*+Ww?FhTfhT z5{Hb35p-^K%x-m3e#4yqNPHoBg&apZV*9rxSN+?P(5efB1~q1GY*r3J4RN*za#=0p z(6SPBZS@yoWznT>ZB-ojsatewtHLWT`yYSuGrt_)6{$)tv1A@0t4fl%N&N@n`)MJ! z86_MO;n!3Dc7-`p|3MXTlh$O%L7(f_REUc_!vbz&>7mJ73!!4O@eM(Rwwu&q=>f?; z>q?%MZkZJa{-wf4w6R!Sb(+iE zG4*Ac+J1-0L~(dc+c0a%l2#^WGnrU|ZWUio91;l@f53!p&&sTUF?$P?l=G)pp9AD} zisSHAs`e06-TRDb_Zg)fXU9J?<4Y9Q1-1{2cfZpMCbss)VOwGtw#4x8EfK%kz|N|h zzpE|D_ILZQb87&J{5}3Ix3*fEZccXGzt8^D&D@;pSD)_fGZ(<{-RtJunt$n=FcoI= zz|NG1JJp*+zK-pcQ-^F4v*xB~v#1nJMHA5`R-wAJ2w@(|DNfm{l%W?r@DVg ziB5KDY9^!|;kM>K`U8a_;$PtgZq5G-`xO;Zy+5rJ8oXSH?zX7WgkVC_`8qOVk7Se~ zi9ZooE{3C2(#4a&6x_T5qns#9J4Gzd#OX=4_`opHCj*Ezln^Zd2&Vd!vbruHII|KW zh}2nY&QYDH66o3vUXf$@!BG{`qF``QkQ`B4ME=QLHL9yVn;_1BWB=KY8Cy_tNrCnj-b%<{9!W%YkDw(3 z0OLxpW&(_>kX2;ll*qC#bMO*B)+18?rc{J5sgO{uV7};4>&4bVBU_IwArVQ}9dT>bpBmi>E6SE+ zhfxO(<79V-!Gr++GCRzQz9W~z?=Y_ZGObgQUyK1^ozior&=R|WTfkUES0yA_T)%p! z)Yhc#yHf^MwD7Is3TvIdA_IS$9x-)eJ)&Dv6%;%4^TeUX?nw1>=?OOgmvPb^2_HOR z^;=TpA+j*TbI%YKJ3On9^rD4V;tz|H32sv#yb6Z3>~6H!nAAr^I$_PdM8tf`d+K)G z_~_7`aJO~sPojg<(9+&YmNuAm8=<8XbjW+`TNNe7;ObTc|@+p%O)Yk(y8XD^vs-M(DbrN6yWj!w)8rPWEZKPH)GqW~>5f)^ zmI|Jj`{e3sjqeeyC!dSehFhQ#2MX`o&l?2!@nf*7!aRJw#LMexu`omqW( zd*#3yS3vjNy>dD=+qzfA#ewOBdb2rqJB2%TEN${v+)xQS*0EOnr+Z1w_=^lCBPV)* z`#h>Xn7VjLvUITT{TYJoU4hCK`^5-@R3XE1q8PnmU6V_ zEf99p{{zV$vjhkJ@z~v7y$@px;yn6F$#MHvJ)xuIHYs_u0Cr==_1)s6`gm5K0Iz<^ z$Ms1kdOBiq>(q$VCrdl5vv*FFe%)8DSaoySAvMWYSz#lK>dMX-5-Le>a4xwK1&CIf zn^yn3(DVa;r^{Szb|LGC5v!;9-5yBF{-i!)&C?0iCBaEfKOBmpe9Rm@ zQCO`_7}1OLK#`AMD>|N&riw?^r+fU74`-@sIN55@z8g+>CLh-?F^l|2`$^&d`%KA; zT;Gr+LG5^qs$*W5S5f(~Z?S`&j{i*DS(#^Zt z0z2}6xXp`JJ$a#%s}H)h2U%mYq>RAA_W*s7cJUO6DV8?yFGTnU#hEzrgNYx0NA*4| zu8|u*h~`U-@KEwKGZ2VQqsq__=drL&hlogvCl%^p6#sB*>LE#+Qv@D1EcCbpZP zu~R-0UZd)2w&Mo=8k?@~`m%EPBRy7HY6KAsJ^|goTEZme_)~H?btr_dQ|6<;M_V6+ zL3~F{?Y2AC#`F%wKN6tn;Ad~gUP=5r1XRKxg(~O?iqqAXhmu&h?befT9_y%&n$ewc zvz0uym3>dMx%I;J>(}QWQH6@U12Mhst56Z&4)SI5eUztUZgeJYaLtJ@kkm$gN%t(CGOHRb4#sfB#EE-yU#|VN6ds3L!WEs)*09)9qFzK&BVGIoJH|8Pg zJK=6{L;vQhjX0C2Y})%jm-y!*{L$ahzyGWLy}%o1_^08Yqx>_RTr1+}+J72HG5;j| zlk!i-KRN&O_^0plYww7!eUrQM+B@U=PM1JFS3^EhSSpDcq`d`7_DI*>xvxNJ_N1LZ zQOwl+-?ewf*Nzjl8oEjll%cD(>jjsBNH+>(U>XTd-_DzSPBzfxCW}Tq*8;L;K?seM z-l@3t#$6D((b@PE>9y@iowIdV7K(xpMPR_FEqn$WJh1i2Bpyp=A%<`!6D2rRX88No zCpUvPoE>Rxa%*tUc= zI}o?GF@h5)j82$%!GOO^>pm0@cWT=Y(@TCwM?^?E)br3T!4;IpCyMv)I0HM458C_c1ypv`1&mrnCQEn-vAUMf1r4Yo=b5;=( z`0*G|12tEPzqsa(xfe5^61LPi;OTGnj7VfsbRpa7k^O z~TbqahJMD09W z1$`~{v9VkBE0cN=vN&TbnBr`;;`s0cv{OtHbm-Z5*z|pm*v{M?>Fb+Km)()B2kNNc z6DAK{h{RVQs?AEZ2qL&509R3%ms??;b;G+VmvVI+2lQA4$mR&o1EpEOh>6zhgoU8s zGaG9k{ZPYkh@>tiO8Ap+&~hZd@{Ccz5L$@egRoD8|E zmCMe?P+42~HJnW5Y9brlF>G{Ik9=0(M6IC+MOJD&&*(z5?v~w4;XBZeO4F3A0Ba?1 zxvaX#4^($2&Mk{`^bVK$hY1cYyYyoG&E~%TQcs(GiOYcr zbjw7vbA`u~umOmAQ#^KU`i{%PYjC_U4-*NBG^Td#QqKQ$?Pf8|?OJA)_{dEDoKszA zKukK&l0q9=3HxYf%UOjVU4#QD4J4!8Sdg^TCo@nQ`C>jPP@1gz6sM3hhUaJPHs-WI=FMB~5`D z38R$7!R^BFMrluopQvZK>n_2y*|`+SBBQb%MQ{OykJXvDhWz!=1UWkN|dP``4)j z--80Wg`;Y7z4hz6yStU@a_82Qb4_MluPM*-x)PART%VeoS0)p>LRIjv-y&DF@x}}A|)`R^533{7wKGD z3Cm9$6`LW3>6Hu)L2EEU3w)u#C=|JoHQ!yWN64NL#FW;~Zqy8HISXU?DuTQ#{6#mQ zy<#hlifL}7m?i~g_-^&UN@?g!Ze(D&k&AK!W90u5A_@Jm!$Xb?G?gbiZX_kLZkQ&qjkhD<{hL_>Bogh9!_f+FqA;B~*?FCh=0K zMwiwDc-@T2JXIK&gae}=bVmOi>SZ`9FE7*$lhlxgKhQD{Y-I;jfsNe2EY3klKD53L zS#8+BMXki$1@5R!m%0U?ZqJob%zsG|#|h?9`-qhmiy5--x%t`41wS&1DK6+1{9Na^ zvQF^(AFL|{Oq!u)wn0)hi#Allkjq$Ts1ustM%mthS0q zn{^+2U1v41vpTg2byi#ZZvRh3hv3l32T4&R{a%*zlXSo+%>L6MmNllWK7~LYYS9h( z{tZJ>)|mbpA;>@o*mAN5jD-_NAxdeML%?#I8FbLa-yXZMETFOl%^rMti4KFhad~-? ziB+d1Q~PHK1j)uvPMRj52Urw*IT{gkM#QL$EG*g&@-V3@HBvq>?J}s5?&~rLjN5=) z+(?CTk^e!JeDqO+*xI-&S_%^7${p`p6q(H#dpR@1NARxJ_>tMjd6mpvYcisI3b%E= zm|-y4g>jVu4r&0OQb~>>5@Bva$4?t_#*Z-Mu(-Y1;#%G(NL86_Y;vs6j2#M&Sc@z2 zLd^B);x2ed&@Bq7dt7P65d$M%-nr*(@GWDz&z77t%v6-`t(>+Ul|x%eL85?&d26=Y zfVvldJKmU7(BW6N4VAM)DRk(AM4ytl+9D z;MjBVU8|_JOPlpS`z|R(xSC2gK=f+AryN3}Gu*13iWas%kV&A!F}!cw?;P9yVaK+w zCIf43?$zTSpKDzKJJh|hO~0!O6E&m-D`bfkI*$h(#o4+YdK72t)|DD}5?bjlTicrS z0HI;a7MYhQ&?lWt@6=qNqi_GNM#YE`jN=4bZ;(-PO0(`l9&$Td$cRG|IwEts;FKn2 z+ZMJr1}-IG#sn88l?SRX_|x>nO{cX#JqKT_zHvn>fz)AY*_EV9+i`al>fg1Z$U8fZ zmfSa06!`&)LVn~ptT`3whb=<8=3_0w)Q=>>GNMrc$TVt(r@J+~$u7>YD#uuraw{}D zdli_ni?{e#MjD|b#HOJ1lSjH?!+nMj5sgz&|3~D*ZORI@qzpDkCUMns5#=N24GB2ddyhXx#Qm&Nb02Vdm+09jH4#vaB_DR0g z+D2~JxqRSA&-lbzBN7E7Cs|kJ<|6D9iGnHg@J=96$V;whlv1Jh5Z%b__GP_9Fv|Es zA;VuVfQPbo@_4Q^bF4$q3%8*ZWR|=;t>@74g{+IIzMPZm(=&~xj^ZP0GhU#y@cepH zeJNIlsU^_$uWqyi2Hkbu=U>SZ7}z=w%*-QTh9KKFA_AJ)RNb2?>;>#}YEvF^Guu=H zvybJ|WkN4XYUt zWCK9r6c#}<)4h059aEa{LoaSL;qpef_?~#?K~FCEJg;QryDh?a@a54(oj|6ny6LmG zfy_B$q6Sh(=(K}>lup%#573dR-zFBd*5Wuwn$w~S=8h!<=oGzrOR>ky(5qQ%ZIsMM zRJsxAkC?o1tlZRohvMs9{I!|}p&$C!>po;`PDSPKWw8)1v9QF}p*0FW+AMxT7C&)O zCyUQ-WbsMu0RB+9zl43%X7LjYH9kicf8XbDQOM%Uu5-!$K3z#Cji18xANc-Vx((m| zqaPUXe`Sk!gA@6OUt(1kVq}H+>kjw|M)lV`8mKpJWi)7seWL*#q$d)9{ZH-TB8ro~ z%F*!n<{PxD5wjWu9A}sce%srN9YHqVd3>7j>SKFbv9}+aJ$+pLcs$>E__*`%YP--b zMRb#;h&xO1jyTSVV!G`n5gWI*G(A?^AV*Byjrw+@k8}152SRda*(LrJCY4?;spP*- zQVAJ_EL0yD^gLa<_2F4GgV^8t%EwBFfaHoQldPv!vKbzWYUyKR10K%4;@M70fq* z$wn9yg*rN@z`*YLEH@2!&iG=yY;glNc&x@8$=P_B-5gU(^j2E_h$?1R4QOI(!N@HH zB4q^8*14@a{R_X`b4m5rNt3Ty;Wj|6gHqssTe)*sm zDzeqkn>zI`yQt#tb4IvIB@m3kNXk<}HderoS%IcEO~(RhZ~FXN4_)8I(_d9<-g9y1 zks24=k&>@)u@!fze-*#?-5~s>2k=4fC9VwL71eAp z{jQom-wjr|mJx>ar2-d@j@Gl_Fq}e^oFn?5l|M5`20QV?wp)r*vdLB`EiR}VG_-;; z^8HyPjYh__hTfhV$4N`59Yy%CnLi;YIpiBD7^BM|T`b{FW4R_$(V*v;iYh&-{=||% z(=rbPQtVZ{NBhxKiFM2rc=|=@qbt$lT01WFN!{aCr^e|`JJP4{l^x6jz@HA)_%{Cc6>Vf0UNK6x&nb3OiRBPp_ z0lD4Tsa+U_Yf3IlHc?{*3zOfe{v_kWr|KR+lb(!%CdC@mR)9o+FY;juJb5@`c9;~2 z>p`j1BIVvt_o#oeT#2hAsVy$uq)#@bE4Sjegp^vjb?)J<6@V6)8WHYE;!2|pSLP#t z?IIKdUgu0Plv{6&-(V@QDY`B~74Tp`pG;mJN8e1FT@s~b^$T&?kmb)2!p`qVO)JpP zd|a6{(LuW*qI9aFL3qTeVhA$wPZ>j?~jf0F5<(S>Lk@uV*0hUyZ-Wl9C&vSArku1s7{VmQq7kh-{Ol~zDs9$`n=^IDK}eRf?UEamLzBp2h8fJb^nI3%Nb<*%i2F;lQEW)LeP z0w#&Jg2rb#qE}}-N(TYQa0715DWlU+!1MBJOa3C2?4!D8t;{2!GOX2Cqn2S@l3MOd zJL+)cQ?;B+Et|WQ28>v%V=NTc z|I4eaPcTfrBP5!yG(A2nCtZ4gMnx{K-koC0$*93YR}Q*oIcNkcC0+tzDmtPR1r}0d zrS*}MaE|=7_(BxBUips#ZhNx^@;p&^0*^9I_m8CE|M*Qpz_0~B`=rlePW zJhftzbUh)D4XrQ?c)U&0cMw`I)8)tYC)_hswZkG1Smdp^vFRHn9ix@#B;kHGx|`04 znE;x=H+HiJFGhE}$iJ0|L?EJTq*2gtk?Q=>Zq3vVKg27-%G1{fCP#jcCBDZkI_QG| z-E3|V-#yrqxFut}>lgW`d|Mo+0nyMlM3Mpf`I!AhV^WdpgMMx-O_Lk`8NnXAr21aq z3aW2mqrU}BZ2JliP~YINR7|nrZwbrM{YzOsc_ux!lRO-MH#dwda<>e@;xv@L(Wr)v z1~K(fCjCl%jE+R?|Lta=a`dJT`(CraB`E3b`*-ywU@Sns3$LWbN&#J@GtdRY74@aA zs%|^!jQWx{ArgtmZYib^%Tfnd28|poplVEqjF0M+CjMg_R2W}FP=^CziKhezg-zp> zA}>HFLZdV}P|-Thp>gBbpA-(DOu};v(NW3R zy|QN(Nal4IhLgi4!2)d>ec|G*?{AS{R0Oe?+1TbSv!ONDBNX;MXr>lJL=-I!#PbFF zI2MSA>Cleq`=3{Ne?4c<%>|;GL43dlG1PQQZ0}!FW<|NKTUydNTT;d7V`7#psU|Qs z+MXD5$~cy;G(!uZ>VOQ+VCGaIu@$`3j+Js>V8cBRmE- zuI$D~TTa=#EnyVED8qumctY^t`X?(S-}pxXQX2ZJyJP26fw$)KP?m5=Lg%9c-CGh=+( zg>6dxp!=+ZPZ644XcGX0ZnwcD|MV{o(5?~0omxW-64^-0S0+uXd;I!#DTAcc8EUZH zI9kfGX+pZsc@asj=QP3!$SFJ9)*iGqcPt+)N1t$EfS(xwkk0c_*Y{5v0}UKUvENjT z%*t{I_#tSJE5iStiJOUu@Eg`iO-r+{(f|>JAr4JTUEjYTuLYM6L$GV#JT8cLcXbqSHbSDAF^WdLIf0BBdE?Y80a}(WUmyT1Y~5Bv(N!jnZYh!Z$%- zXHI~Dx(-CK;!_BtN~8zo{74!jhSYX)W45&2L4GlE3mCq)rt#%)O4TDN7EI%R(Ja|T zmx|D6dfUI3DWyjW%Hd-wdF=ZiF`EY66U>_fZVQa7B_mYe>yU+PZ8X~_?RpC8CrdEg z@PNDAaGG!7E@nz}PqP;11}j2GtVGBb3aGr=4eB1z1^j?!2^|>87#L4^mBv&T%x9Oj zqI%^;k$*PYZmLsKpqB}Nv+<}8s{5{z+XZ8S%AcAA8kUl23m)yr$NtF=r85+cx*~McI1|X z9w&G^WfEKB^4~^)Hz%r}3;WQdZHlhiB5799Q@f(}Z`h^AtEl}0u07?qtLP9o?2^w7 z`CEO$RsQ@0V}r*SZMIb{u5u@?l_{oFMQ&&*FnjGfA3^O^!tlpKZ_p`x(!@Yr%k^Zf zUf}d(0@Vmw-|cLz1s?E2TD(u)-W&=N5lX0*Ja9}7W{O48X59;XF)=gHiH6PyQA!D5 zAXbg)9@N*WCsz|{Sl6g0r=ljh+oxC6DE$OyZ&?J^)RSjK?gm2aDq22(4$P=AV?}oR zAb2s4A+LcBtVg9BdDEJ{gdK9g0U_skCSlO>Rdf&L(=fLX1nJ>yD@b_Y(Gk=a#IIr? z!v7NB7no>9Qar!se0Jv+i0?wUjmpgqo>U+`mCYvA8VeZi13RaKDP)O-t7Cq2#p_Iv zn0V!3M)iOng%CaZMp){h(7RJFf`q5pLAuJ`h)1hPgshEuLTnoIOUvrdC^};QksDqT z_m#Hd=i+9N?eKQdkYnh$lRYds?V>%zTJ4VCz7TN`o>lo2YN1|2V;!Bz^}$pWo3-ac zTe@74xb}9$OLZ$nu?XM;v=c!&?-Q=hUOSH6AherH?W~X(%i2D31Y_wMri@smvZbaM z*x`Gj&s-k*%(3aL&s+|-CxfmBu1D8s`;DM#kJ(gH5|*;3 zLS-;ocRjhM7PI!ezo$bRw#dyyl{jW24OFh0uP23Ims>k`CkVy%@LHFc+|41*8T@!X zaY8e|NYPsNie$41lm=jB@=0CV^e-sYBci_r`K1KfdtPLUduKk;W5_q11DMz&P)I7y z%t=XgW-@cVR??`zsfW~v1L3V5`S=;Eeu9X-V7^n^?A7mZr|!8}8U}uf8a+GGMKvrJ z2d0vA(xlHAU}GLGnIV8G$sOX5Bn@c3#?z{XIaaEci;3i7DB0D(YgK~C>3J|14AJ%c z_3QCf1VOGP0OkO4)W1`ik=dKeD9**ViG=*ETcS)$>gaUt5C}luQ zYA}mQ4aob3nACunlmMvmFcd^sB#xKx)>eNCT>XX}Ym=FCU_ z(ckE~J_H7zOBJ09^v%g;bmTq=+&Bj$q6l1UM$4dQ#8(ixOaPf=GosVj-ZvSRooa+% zL(nCjR4=3oUs9&1l$R3C%e0Xia^9(XM?MHyzn(-4b%YM+4pqt0iX0eHGK@C(dj&Km z1u#_3)4!HK4i?IhvId?63zrslJ>tg;_M zE9W6QOTXo0Jy~NO6)hiVWx)mTmerkdbHTD{&~rd5;Ab&jI=##eYy_+(W^{^FKu{gs z5GP6%puTjNO$VTK=AeV2s%p-V;a9GzvBMoRvM75S|W%+f%fcl$=0BF~OY7=L&pe%9LmWkQ4 zPs_ySv7oYmOAbaW`<(gG%`HHJmB4dw8GFc{D<+t;gU#qDO1|JAifqH?x?ZveUk><( zo?9lS+jC2kdZ8VK_D~qbFU4*Fw$L_c(kG=hb4yEw+Dab3Vz5Zm(!#i<1x-r-OBNt1 zx(bX&s_+}BQ5<_g96A5K%U#%VPwNkgy~|zPu-v>eU2Y%MW1G@;HuE~utjj_*?LXw) zS$!fmn;r&8C`bkO+?>4lFH=Qs+r!B6$FkKhW6-7AEo!$M<~ab(r$vSRY1~n}Wp9Ag zN(ly+Zpy3f zaBSpvOsf^NF984^urIa5Y{|l^6$(&bRVY6ld z0mWp&`&f?0wO#gNhS8gg&% zkcV(MwKf~4x-(=u0}3-$R_(|w)DE_1TAsL-BuC~(lY{T^p;+82Q@ilcdzf9Qi+AcP zKV)_x(^Ct+1SrpDw^I>+XnmSh#J4enk(--{EZU|-6ry1lNC6|N1KLLf=rUtx9RAbY zi0UCQV7z3l0v1yk>=eS$Bc;HGVobpChXT%81stH0Sw{ggKol@;5DHig=A?kLRsjd( zWF`f??{lzk2L%i__K(y1<6zum)>Xh1;QlT3WC#b%0o$lmz#vnE?bKmu6|gp1C=Q)X z8wyz7m#7RlWSZ2F*Vv+HexP1TK%R2>hNq6tTw{pe3YeOnkKe>}-uOS) z^Gn-vKNZKt!T&{3S`@v0o@e^+;1B&h{+AX-VO0}x^~1d;r-OwS&ZX7!>1aT(U_-EF z+q3pQ71 zVwkAw4ivJIvHkpO$m2|^p})z~2jeE&}5(&~rQc?QZpTaToAr{0-ux-@J$ zE!I16-cvT1Rffaayr;s{0_}T-Ap}Y*S{df84?_qPZe@3gt&TNR#YGQbrmF29 zE8nm>_{U-+Y@Ef}Vmm&qGZ7I9QzZwt-)`de0rtjKDShBOYnF0RfpFKAh#V^2g)4FK z@YaU-Udpdz(}WK-r=ay(xU^(<&K8{1;L&9J5(d2%UuC$*Rv{n0T90Yle6^mVF79eQ zkBM^Ce=2%YGw{zw52)Q5_Hpb-0>jYJI}p{yQ1JET)6*u6I}G{ycCCaNT}lLI7|}8; zH-t_ePEPIyu0Q49AiXAp2jW$CD~`)(-v_*FvPw3a|e>}F^ggMXKNOI z6%(}ZYh?8b+-G8M$;Egj)Dd}RKATkU=o!NdCE}{eYDZC=jaP^#C@5L|Y}WK$?|zx7 z)!5wHlm~AmjK;u_g9#`WVtnUkB~?fd5fYHCm`y}946EpRQcRqB?WAYpM6m&KP@13r zK=T(y_>SHQoc4;n6QUPG3tiBK@>Eowi(_IBG6jqfAiLC76hexxdNrG|xcW$3mOqdu z*-k=%bhX{|+Uz8FP5MeojK79Nfd52HH`Jd01jd?+{BM4L&-JP&=*owkzV(UNB~QBS z$*uUY?`aBGY<(jBmer=%>OHnqJc^ZU@ej&^L#gWl90-H0x5nRU1iDkw2C*ItTr@3p z)q2NnQ2j!z^Q|XHRhXyR)7TZ?KsFxkr0jCGqyA8tOSi4j8f*oPZ2s*g$K3?93b$CU zSJ4!n)Ca;8b>T^*#T4J@e5T@^8K}v)uU&?A<$^Ar_^*h%3lqBY1{~fI?9ztM{JW^WLRxm-$QEoliNw*y%>fg9(5?`;5zGuNfcYy`6V<<89SH^OLkt z!W^Xv21tzMj#_PXf=7BWb>uHOsJ{0Qy=2tvmvfs=YUULB0l`{#9*|D3L_K&FLb}u2_27Cs{V>_ zNF`J)_f?c=S(V~;b|M^~@WBWRj{@|v{O4($w}(k+4DqJQz{YB^?C-SDmJYILRh*aV z`5uF9*)8m*d`I^)W|Q^5vn!qZJGc`1j`3~i(Kp&(PwCfF{Kd&zNA{~woVLKX(?QfeizGG|DW8?Hpk14ukdom&a7q2Jh zn^B8O;Xc=R1f#_A z`{T^=u52!fFhE(x+tszylDNVLsrwF^$I>^c8iBcCwe7E82LRKk30HVqD1*YkY@&Y2 z3bBR1BnQ>%i(}(pvWGywano=8xj+?(5o?^AG^IRcAUUu(P%&A&FpPA1rJ7wCV3>*D z{W$s=#!<>)=|F?u%=%VtG?BN1#_tn(@3-e|0M>ZYj10iuE3d^dU=a=*W<-943e_-d z*#_ywSoLrhW84V&w+Ufq9x2+4-f$txDC@PNFuyz@hX;s(c4|J`mH?XPgrV|*b8SX+ z$%?4<|0hiJ$Kw|59N(2Z0fmu&GMQksP@o*GC-1j-&HKe6vzxT_dvAXM!1)t03wM(y z@_i;KT2DS2%ebXwlf9>^(Mx&e168slSyZ*j$28B{&e7?t{>kbZ6#$}uT)y=9?ftzg?wIiLSNPHZ))o(SHP(&O8$ z)#{__3x>=AhhS!nKw@zD6P>rCk4ifgxnB9+#kgSikdkiZuxAExvB+FQv8C8J7(Lj^ zc}Rt*3Va!1n158xSD`(LNh{8<(VpbK#psnA%UmfR#*NB)-p~>TEV6>QQ9Vv}jAyX$ z{|EGJ5Lic3pi_^dVT_2BAGM!wJ;N}8cbn4fDYPFRc8+NEEQt!H$l`~3S`}L2OEAGh zOQH@9lB(ZmCN-z?sR`-vgQJk`;BSi$_uJh(LoWHUih?S05lHSTMWZi;r+GFEtPLp} zzF|6f`=adPcoyFv5peO&^;%esIMBM67FTB59!2DZZ2(^*-Fsm5=SXO z3Nv)^bf+AY?B-spGX(a}&F@r1xf!$o0+Qdk8yMNm1_&3+4MUu=!4&apvfcFT+r52W zD_?+1frJrji5S!z?Bn_a#gWezw2e<|683=ytuu;Ef~d9$^#%wAt;-bszTqC%BQ+$L z#=s9I{Ifkgje*;4Y-8X?SZw#esJP)CyIA)fu)(4CuyZo)a2CkNyS&9A;6Iy#DQ__r z4bHTd^w2yFHeZSS$vPhf*%h53Ou>OWw7)Oz!xBl zt|#c9wnNXh!{CtFe7nsia_4Z8M&a_cO5~10v!XDcElFc8^4clwB1Z8|XCPoTsU%@L zsS0ak-8Kp`z#K?a5a)c#TlQzPwq2od3v|LNx0~Kcbi%>C`g>;H^pTs}7UrIoWvs|O%ymm*0Fwr_ z_ZnucKx{OlcCfJ{Em%>04chi&Sh1bzwL|Hn+i6zy8U>URKclkzcNmAZf=@qznI3uZ z!0U6t(ACcQ(?m{{ZPSq-h%|_cy;1huUCA3MDq2sTPhLf{#`WY&iR&#ys`x+fntC*e zfJ@YzY0uuPbR%tc;=qtmMS-XW{Q0lMWqDPP;1sZm{mUlA8RXyk+b065_;~V+$e7}7 z|LofZzjl${f=TdY`cTX#_|?8cv^u;q`%XNmF+rXwA182OF*M`IWK0V1nD1CnkRjhe zrQC6yN}Q0N$^27F#bo~HV{%17Au1-35TAbp0zq;R664A@w%^y*QzD6cqS_t(G#-|5 zXtNGm#kXdc{5t5Kt=+^zYtZ0rC2ts~C-{Sc$ir=YB71|2w&KU?K)|OztrKgEwyxj) zk6pBtJ+8l%F50Tzpb^GYhwuUYJ#POVyXbwsf7Qni`7ixY9~XJzhuh^d^;$5C1bo7 zOF4&^WR<5rKdzfxeWG7-^`QaOSET9_m&*Bu<6p&(+JD?S^M4KPS6O~2Q%EXI8vCcM zMJH%6`=5)k4V&1%J>H(XBhC-`FUtGjuPN?*1ZhA#Vu`JtV_?8gV-z-=z3ZYg@!HHW zt)tL@f#fGmeX8c^>GMG{SvB&^a1j_5&|0=X-VXge`Uy7}OF7XdN z*hK#G&wcPGgN5;h=RWwuWDs2B|KeBwyPxbmFivD>@Q0u5-4DdlKmVm4{n`HgLf-mk zz%^;I{l}v1xs+DVJm#PN>W}_#{{ea`Jomw0>^*SRMg9wa^1u9S?*Yil{I7Mtr6$m{ zh>wU3^{@KpfB(JYJN?HQl2H1YRotaaRb2Xyt2gnPcnzISwN`v4Zd~iQi?sjB#3T-lVzxSTpCDjw^l_m1+Kgp+E-l)f?MO;(XR_oJ@VRukrd^2b4 zu1&#Yt|#}>wTcjG*p>xR4Ei8A=pKk2W)?RU)6qadNpPOch$4rmna5ZijcF4(_rO>? zWH?r5>(O+B&$q8A`Ib#s4+7e86qPtDn^EE4E&YRos)XN=fQ~Xh7Vwkv>s*d|slZRc zB55-z)vs&$+R;Rp!ioo05yEolUFkpb(>?!tKTQSOZhZf0 zy{Pa{;;%>d(+t7CHGZI8b~!lGcA7S$q8U4}rqQC7N2;1RUI{jI#C(4uw=`N1pZ~mt z{OFcWn=}(BKoUpkB{%0rK7D8s>EkWK>oE&rIxnqJ zVQ^2X$gFv465B@*@=b!&qSIA4F+K9NQ!)|6a4yoz&YkNpv^t__UulWk=aHV# z5*sMl#8xc;O#xN1opq2FR9J}AP2A233i()QVSeaXlRasPTYLUgn}SfAIRHG#L(HL% zFvJNP7YKYOy`eme=4rdhl{mF$FvCi#A!pXp$Ua*&m^A&yd|Z!4soh_-#eJI)%|zcC zZTH);&t_q}=_5*P0>X4ilyE)1rx!k{?K!eQYoT2MuMvW;8MM2iCmJQl!PEmy{elGy zVAVs|`K#L_VrjTxj|^;&475jj+9Mn=?Gb97QdP)D$SL+nv3HMHo6cr{2pDsi>OOm9 zJe{3E`|2ip1Pn$q)E@35(7 zX@|h{h+UV4= zRF&TsMN0h0R{ReU&zsTWI1cxx5w|vl$<=f{`N5PEPts}}*@yV;D@w6T-@R*^IO%zWhVu4n?Tx zi!n75a#ZQ8%PQ`lrLvBvlyZJVvh+aif0@hF&1RG8^T`Q?Pr#%pe9}URel`h&)`8R4 z#nu0vHVW2Ap~mKu#~}9E%G&J$)nTo~LSZRF5x0P!z-=FQP}|_}KyBN(!n`MpMdYO| z7!cDv9|N>~W1Q-^rRk6p&Rt-LV~W)UPorMd?aYr;D5*#5*L<#E~fa?(ED+de6~X8ey$h2w4&< z?Alj_R>n5AF4wv6M^cqr#UE79CdIDzQo1+9vw#bXv5l~?u`w7YPOu4zW3|5Q#i zb-Di?y?$7)XL#dzyfnNV<7GHKEaK?!@5E8eOTtUaOU6r&mzNn^yZ_fK4rg8tE-8P~if7`=shTMJh+;PaDSG= z|1LWp6<8_WqAIY01EMA%14VVY*P+EP-O#8ohIiA6BP;(jAt3@Z0OYlczahTs{`h;V zABchI(^DR&iVh0dfiG%JhpIY|p)M8KdUAzYLZrGVnG##!STp`Ztd@gwW2}L_WY5Xr zt$r}0wY!QUvZ;SO@>e34Jk}83zY@K4N54ECy?n<|(h1=+(3%2Ysw$cBKu@G|&~HMd zE*#ED12x7B>+?4ca>5o-LO{SLh2dI|Q!J2^0C+Ad-=PRURzOPRwAX?^;bJ2Xs#0-Fur&ZgutUlF5Q4_kQKvCW+K#7 zYuxhVCl53x;k={y7O`I+RjB^}lsFvqN_y6fMHI6_VDYkWx+|xm zfls0G;hX$9Y>VL^Zr~yr@9ZU)W7r!Nj-yZtQCxz`EzaR_mKZaomO}(LqbsVP&+9P6 zgS(S&*7Ya#7|C2>R!=V0N&&?TrEKmyJ&%+->@^QjuKjy}64Vc(41id$3UY{RV~#4&B8ycbNHF(fS|${oP#8f#_ELKz!w zvS{6m>j|q)*3F6=?@#LqrWuyTiaX{eSEf*M#;4Fzf2rk`vGHd9`_T)Qw;=4IWD<^d?*S!iLnzdh$lnI= z=TCh=X~HH)8fC`mowcDps0l5N_@E>d23%>uFepGxF-uNxFRc@)l5_dT7%={?6)u4RP@_Nhv?(lLP{FhP$I1ikW3P-6FfP!AHcdOg)Cp zPB>?04Ehqb*T5&|e0(X41dZ)$sj9~!(X=s_>V&6YQQC%hdkXO?)?S*6B_S>Q4Lfy= z%5bVmZ6|b=29UC#!2!FbUO~v0#IeTc_K*fBLmD6@#7prFX@H4Byx;~^bxWEmJqrtE z)QAnXbeYLoj^J_UsHLds%Ml&NZc?X&cxTUe9txK`-;T!~I+-5Bg=u`0i>Wz}6|p$j z(f)Hj!+WzesOkI@+o+?U*AlcJ@?gIE6L!B$c7^h^`;((~zdq3JC*Ict#JSr9*fA1L zynu{tyY@q;B7n|g3n!5bvcKx zUa#nq{Gz&Vv{dZAKUJ8%OK*PUG<@nAnPq;bKSO8)UI1?GYK#$i;qhX)saM!}m^+NJzYsQw4Y* z8PR4j_Afa4JO4J~OFHTk(b0!JF*}Uh`V@3@?4MTC!Dw|bgu7VET zOMH@h4ZPoS@t0ph@&EMxnEnv6t{4VA1fNp zFo2G@vHxQpWc4$#oXAZ1Qkus-CcE!?C6ho^h@c{oc^Imj!~a<|xmV45#xmaW?1(KHcGe>1bbP0&J<`P?E(3F|6DY10nPYVS9b`Wg@VV9| zSLbCiO-FuoWZUSTQFT#gU=?0H5iZ^r?=nKmwqSzwswWeRC}=Ad{KyD)=u?tKkB1Iy z6lk;UNInz@&Zh7iLA(`yL<3osyA;N5LPM5>da%$$lj#uWvS0BNC@G%nfqfKgH~wle zPcYc8)P>8R7dJk7slu{Uu7}BENPmm-MPTbI&DY?1^JPeEOV%)7gV}ryX7e@Z&X-D` zxOu9dSTsykX7kj!``Od;(fsKhu|zOCkNoJ!)suzQleMeIR)em_`dO^Kg|A@7eN672 zNSPIk5xi!Y|1z!niwmzmAG!{{dLsLsg=bKICQ>;wsPDqUBL-~?FJ~f+%_wjwhlNKl z$g0UW0bXXCUVP1~5&3_V$Usv}Df5>vUw$m2;+WJm#f2>RA(y}#qG=12lBCy$E`c%b zbbM#eA1Kg?Lo;j>YXlZ$g@a30JB*W2`Ok`Ec#w8;Mw;2H%;{60YFZ!(y_IDiD&T4M zIq(43A#m*nphlJyi@N-9H>w%IQ}X$nA#wb}5k;uof8M&(G`&lQtNd;*TGPLDxEt&5 zN;%5n(%~*ZB`Pj?UfHF?-PH9CS6%;bH+IG0ZsM}T-SlvDQxrwr{}1mw{5Cf_dCHbLDU+ z3CnWja3|G@9I*fw|2zEp!*|Dr-{!`L57py#5zN=%n1Eb2*7VHRV4*cwxRf<0YJ*Fi z1xV2>3-C&J1twRmzz*ug@=`7@-;&0o1Akyxxaax=LJ9mTH52|mta_L1$b}}$n_Q|V zWJ234a3SaoahHKpA$}WbB6(n~W0l0OaP?!k$@M7k){iVunV4r3;F7BWOUWwf!X;N} z2_Z0!;F4oQ`K{qdkH;~n@HtLuZDIWh1(>#Q)Uf`h&CrEv@gTYB%2>KW$TdwzybwSZ z<$?u}`wAdiLl~~ltU?Dv!c5ulHV~|k`OoKd-{h_W$o+8?F5>!bGBttcii;pE#eQJv zjehydWpd0i{%*fT$yf{5_Y-hPNZ?sWL=0DQFbZ)e97S^_%C1mJ`X8hHl_ZX=KCPl= zfmECy!g;`MH`~#2QvHx$?Q=acQ~oi$9|K?EO{WAfAvVjWR8Xj@197R^R@tb;r9-p_ zqqH7KaiHaDNP^%}9mehw$njn>IuK8Xs;$ zW!6la75`qqziblwdX((z5-KcS}&M_W6%kd&Ww&cPXs1bn&66__!wF8waFfxwUN zPId(|Os_~7fGb2282*wa&E~=C8l7)?)20MWlC~LS0{-g$7H}d3apA3i&w~m6bX=H< zNX(*1AU{we9qdDO4uiVk66`pIdXPbFPMCRyB$2AkrY@^~Al-ID#}aEl6b005FhUv- zez-yc8JGuD=OIy|6;3jZRx)V-e6(&EpbyMPtf`zt2{#-PB}<&L`h*)H`IaO~QYA`K zB}!5yN>UOfDT$JlL`h1bBz^XYk`p9KPPkNw5~AkCzM>}X1pjrF*Bl5EK3bM22_GGi zC=rje*4~KDO_nI3zg)VID4};sD`(iGE!qaFuD_Tl$qam)00W`5)D+Q56smOnD=wab zBx;}Smjxm6F^2>wsmUsAcy%i4tBgQ#^8%~bVjqGq6{^k{zozF zXLsy3H&VS5ighYPimHH$sVplc4^F^`GI1w5aGKJcknK|B*YfiO->WRw*sMYYI=Rq{In_kCLh?Vy-0JCg zi1=@|9%$AshPtU8R>Y3M-BgUTMO6-Y=TJsKY%MwK6O3M8p4)MSEBM(Q_dOQgF*}j( z)yKI)F-}UK)W@06HCJD-RP^LZvt<(YIkQwW41^ugx@HkCYI?|;O0-T8t!r-5iq?&5 zm3I9CS1~h7MC(|hbu7_3mS`O#TE~c1ijdqG(K>$iqV)xc))%<3MC;hCAzIfEH80@5 z<)Srwv`n;ykB$(nYi{B;Bw8!C(TY~(!4^d8+O%GCV@$X)Utl|LSPr^!TZ^J~t=`fW zN3>4d+WxfOD&4Nzx}q*aK^Y(S@%~CL!Lt$CYlYpj!4IVmSNu0vRHS-yT5oW=6FFV-=NOy-dMfHp#kgx2b-2 z6^6iSYY1?TZMSZ@O=m$2E{K&E6_M!tDKKDKpWtCd+Fxyp!aW{dVtr2>TnnzUtk$K1Btx%dFWbPOxjF~OU*Q8JNx4%v(+ zEZ-fuT^Cnx&#d|>s@DvTS=mB&L7^p5r>^G@UI|2+^>5y_fSOxAX@zgS?pE3B)GThu zipWIl%334$aHDs`IMc}8c&*4(Q`{Q?TJE_U{R?QMS7it8MoqPh<{MAPC&hn<%s+fO zz2N#gCaT$sYecRv4!~AzRqq$IiqL6mpwq^%Y;i(2+lnHEubk`dMqgErWwsxk?Z@P- zTlX0{<(j8Qx7qOYd{(cs3Rj-_6BNC%z89;jZB$o2aqDi~_s;D!>-90I;s(N5%k;?# zIMT!!vsFGE`8OqQodkqzl82jPE^^0AF8HM9j=`Kz`AD|M80o3#iCS=f*L=aI_>w+y zYx}~%${jmm3rJq1$POz`t*2LYxq8EGQIoiZ z3f%C+^G*3^{letHx$LZS?uFIcRXF`ZmA5p8pE>3l&J6?K$TiduH*N!4R{*zSTJN|~ zU8QHa-$;+8g6j?og)Nqxbvwdmpe9N;Ij3kRA_DctU?vm*l&yI?c3aa1>;{J;x+B-P zV_dH(O)05E8#TJ|v_4jHwW_JqH7iV9Gp453HB-ej2%nX1BJgu|c6PGL+SEyA%uj~Q zz+husVATkUe7)PK$^0c}ox3e{%?t0p{8&tT?F@!^{))n1s>{4L7~}IqFmnN1M1`V; z`aTuCjnXe8FS%?m%pcC0uNx=ZT=MH=YF-7>^wEPne^&cdpOj;zAdl%|4v>9)gTWuQ zLnVlVeEkS9gB!X67MEgX4lIX1-he(U@BavCK)gGI2qNUz-O0U(1DgBzsIJU3tlSA% z=Mpum99JT0iUzxsBr#sg*EN-^uutnTX9#PYGq9kT@35$tJ=RqC$?2nplty+I-T5v22zMT;sxX9Z;W8`kmhPS3B27N=Vz{0xlg3f3%_{&0V;*}-P+Z}A zD$TKd%R;|}&Ra;5$A_C?D%)n0XT<`gm>(3bniG6jR zb9*Jh9ws0*Mznn7IZr^?WNzZ0FGXrMVwzGqTwg_pKY|ixBD$BoiEc@AD9`mTD+J=~ zvLYOt_?x)wob5(ij}C+@c^t=Cl1BI~W0v9A#nsQ{UPjRkDt>-0i|%1y&aRY*-;l5R zb-)o(&OY+I9#n74?8drV<9FE@vkvj29!umpo5QdoC7T0WA%VBZ$D^#I=0Bv`$_K{j zwvEX!(a1E$gq#;n#DJ3ip!%822Rcn~M=;VpngAn?Gz+OT{Th|^ zb9uSTM{c#F%}TUcIoD>~eyh#OYqgm+EqFy&vHZ16_V{t_cH1*MV6VFX_q5$g+iv9% z+l{n|wi~NK!LiEvX;RcnWYh35!WAfg~nLp^h&PNYksPB~ks%i8QPwQ{A7Hel@Rn~W#DR~cH z^HKdAg%6&SW& z2@`{T&t(1YAiZp7tzK!iEd~@sRJS7GIFSI`h@C*tCT8Z+q;UNLcf6jsZU69vX0u*f z8vmIn{xebhXQKGeg!s>d_|Js+&xH8T){g-$98SJXRsdD zO!u#v?*9bqVHSL5do&Z(t+8)P^;^DXmwy)LnFSmg)m?X&bD{Yzce`A697t$eq$%0k zb$0^RtjIlYzm}abBGPd$MtV!jNcG30x}4!(g}8?ba^|j=K8Z`hOn1wD{A)U2l^x7Y z(B8GN*Rnp1qt9*ZB^$H5ai?WDc3q0dNO$&@IS^w^-|@8$1g6d0eHh1bQ0fDF*+ z0|aIgATX;yV32~~_*Vsi0coP}`^^S{A&Anf0)d&hjaK?1D5J)%)NEqiuZ?53T5n^W z&433aWji^P*1cp{RPmJVW_H-k8sGL_QpjQYYgtg4E2s5I_Q#!b&3Ai8^X<5k?l!hX z^_H|2+w+RUKbx8Sx63Y(wyW6KwkvLv?sF$?SG@f%L2s9Cx86h*xfAD_Zx~1JrVh2O zn()lMzs*^DRrT&38&C0A&oT71MptUK=Ss<%U!6Dy6F(AgYg?&k1z>7Vbm z-G;_Q7w2ZYQ*XQ5h|%w8;x)$`%<=O;rI;ym-z=S_BwSt3?U+tcSohc7gD z9FfQR+p=TkwUmDod91Npb$7^)S#@{Ny0~_C9Bs#_q%znsiYk*|D_u?4yUP|aUa(`R zr(DIq+;-oH5#x@#4fjn`ceC}S4-4Ev>Pt4BDX7NQfR?FQ>9W*MD^w6gQIf`rc%OFb z)jM;gg;c@0?oQ5K>#ktxxYlMxUq@`Ta%o^r%HSSZ&l-3=CKzE(|AeI*oc`yckDiOb zyjNPcd+B#m#nP$qTWUJVbg$gRl5~?cUh-}ehUDZe&avrkx|HnD`wZR5u zv{%TB&Xw$ODcNTn+jBdl>{m^-j3_R=7gMFRHzyL{Fs?B++79Ep<@H*>lEj=s1q8>quRkSTdNi#MW z7S^hNgZPCer{kpF;QqL?%+%$l+^OD~>)VutE=la_$tm<-kbS`kXdlZ_ZUC!l%wrKYLMK;5@vPf) zyXVwKCRk3Jv-Zkk+E*ro6lYBxl)Is+gBJTZL_R2TG{C}pzRIr>5+1$wO@lr87 zD@v^_wSkF?{XPjH?8&KUbH-bA%}PK7J^xv*gy+vi&`ba=n@UC8Y$|lS`alb3zj{q*A;Or>k=N*vUImfoatT^Rh{oicWd`vs1Bal zy~}I6Z{=PxiPLZ|I^#7VI7RbHSau55@N+xDraOdHcf;bwdIm$##>!0OWY+Ff_ojt+ z!cvb%gximGZO$bwcgurrHfQFJ>259Do^@@`WjpEUaQm^Z5N@AcR^j&9WfgAEEZnZW zNHCqp+!fa>$Yb4yxa77JQmZVo2E>xgNtIJ*{Ykin_4irD)?G3;irdx`l7EqOLOb@BZJHHDu@7Q3YJ1`+ z_&eFwLiaqEM03MXVrc)R`V_y!NaBXKSQ{F)Z825)D=8$C>Ry$ zq28{}{Q+P`nd9{;^l4BNRB&*KRhqLCP6R|!1Yrd0(9{(EbyV6qf4kyK8={ z8^!_^yAd@#5RhfM!39d(SjCzxC2m68oYsVMVqQ64YKpCypeL>ypiWf7L%@i?ej_a# zkVL>_pwStDBP*uWdBJ0Lg3JoDz4hR0%Z&sW>Wb;2wEU>B{vZ@)j`|!Eg0SRQGLvnM+I58+@#K3 z={8}rTiI`pL7=w;X_qU~`ZvI4w*}H}YsO~R2WdxPhTG!5lWW=RARB0b^Xe^Uyg?Hxp1S@?-}v7uuy&iSzdwc6WzCgWte+bX zy6N*n!e!v^bh+D@%SGc~N4X7%6D(4tnXk}y8_1SE!XHi)GubFMYYMF(13_y=m0&xz zD6pkhQvTv`><0cFvFk(lEO}<|uEH8liUTIZin3`fmB^ZG>uiM-t|-t0ht$V98r3VV zm{kM-aF&b_1GTX7^mQ!}RG(LG7*?lVGPjZ)AVH~h?zu_%yf|Vi`FOSZLn?j663sx^ zuH8Ce#?+Q;LX{_bU=NZEN5s0)z}PP?@dvJ~-j}1?DM$_bQ4M|jp)Lq}%j)Jh@vc(; zr=k>o(}lv{D69ZWAv|^=dAWYl}8>TLl!k_7lBTtl#~&H&sLd*Lb! zv<>GZ-+hsMQ*9qezH@5Ylo8gQZ`hryb;O2|d~dY(I@ix4G+MlCk$i{q&Pl#Qh2DCI zJX%0dR)+;V%Tp29^EK=;t1F-zfm-Qp7{1AY=;|ul>@2D_m8yNfD3`E~0zb66k1j(1 zr1YiSnl}BVA7T+5$@QEcO)8zwX`uT&xTv9flgrf?oOU%-nmgD@n3M2P{akxb%-b$s8dZ zzXnMrma~>T2~6?HsC)-xt}`lT*=WcGO4!;8v6yWWTWg?Ma2kT2dcb7U>_+fZQao#Y>h%WXH27j_iX z$B@37S=GNTcCZQzT3@6Ht1EP?U9=YnDvpc|7JEUR;QIb`Ca~?)CRNneQr~SM)#ytY z7OfdUpD>mRbPA#+BPz`AMQ65=tUh4t;DGfBnMfM3niVo}s??&49g=h;WrmvA2vxIz zZ1>f@cqC|TQ4jdI5CXIC(Sts_)U3nWeBq*xFRJCsNMcK-Z&;V(J&TIT#8Ly@6{`oD zSz1F{CA2d)n8BT7X)VVqu3%}EFvWt*GD~Y67Q)lfJG#`xM8m@`TDw$Uc61s3hlFhhf1axQFA<-Uh1SK#g# z=5W?%)6QXGZsDul$qr*o(L2qUN7-1X+1 z%rIhi#@!My89Ne7ab!&_AxRP^GbN)Tyr#AX5ho{JW50cuE*St*SF!g5qC6W^Xdb_eeLocH`-|;UtE;X<3#*cNH8)h<xc$bmtO2f&A5)R3V40R=r967o#bW2pn3p z3NEI26V%qI~g8*h?;+V)J1qyy1e`qS_Ix4QNNywv0nWGc__!#oM%VR&hN4{M7B$XXL4V>CH?_<%t`h7 zT$%Nhy4+YL2I=W|iyJ|y+vnC!4l>1f9B+G3!ZXz+?*y0=9SgD<#))rI(<9-VxmXCN zR1GT1H@z&~Jm^`q-Xrw>J{d%OP9eeF$!lsRxZncj5WL80kYg`ON4BxT zyIG)BfwY1SJTGb;h`Y>R>WS#XAHd>)2eyx`MJcr&oJ*@e$@Go5`a!;bCHhvVB2PuD z{t7*}uvPhsulo-wcboFlG)L+mHw#l`gK9WJ1R}+Q?D>?fD|S7w@#>GOotXL!+_-tP zigVjWOz|j<9KE&YEm~E;x{XP%mfVHkkjo|pKXE-3DvQ>IwU)w#K0_L;=w7Lfit0*> zfmyp!W;E=X(R^niS&VPSU3E`B2U#n-Lp^inF4}VoW4)BtD2hdEWEg9v@I^K2Ji0r1 zQCNaM$$rcDn~PJYyqGqaPnX{dAz~-@tV?aEZ=G)nj-F2o)zJzU+83ZY=?uP5z%XrP zq1w}_p)dPMN}3vR_N$z|3`HsWJAY6TgMHapjl@+=XKVQxydHTpvOa23zrMfnu6VL> z%i_O*{W%^FZ@K08vDMX5{}zt9L4TcCc(3zsJo|Uc*5sC3dYzAE|E{`fmM8t)*@y93 zln&QVu2jYL=FTlAZz=I=Zfw-A<|O3dTR1#)4t=6tA07Y;D7apim5*!vo^X_dVMe1 zV&#bdpR&vFdwe?HZTeSOU7Aeny=?S#c2f;%{ns-XY%=4wSLuEe8`FdFOQfM;h%7fP zUaPhr&`5~x1J4DiKAbj*4!zj)X&a(HR7nl;G`4;rkmB@Qzw}Q(WvxOO$Eh&ux49fx zBH{S=q}x|Lf|Q=F7Ps9?UMd<17QPx0YDVdMVA^}6x_x_4aP|}3k)VZ9sdp+;Dy~Op zTdZRQ?Xs}NGrtiS%if$CX#rt4+u+$eYB#hEE?zLKGllDhLAa`eBa7!qJ>F5qt1e+M0cXW&tG9zxu$yI2hG$odF15!)>aG?#B74-`KB>K_F@( z{~3B}xk!Cn*>mv)dIrX>cVCPLRcE7Hl=G)yarHy_=p-xmBeV~CO&)f~%{0Ei0njl& zoR1og`Ez3N3EEYkXi?Kr&;NFK3zo}SH(e6MeV_aT>m>zU2Z2T2X}jwh^>E)OFH%+k z09fBA^clVJTjZ&w{#6&IKAF1SOITUoL%$nB@-BJ_hB5YSGxSdyTY7dl+KQrRJBp&+ zD2i^3qNqM`II5y3T8pA+BZ{J9ytjQgDxxSFMo~mdsukYbv9LrEMNt+-5tx&D*P4x~ z0W;lf>@v$}`bLAR{#_g7aFIJf6hDf({~tcoY#r{TQy00d!<{7jRY?d{#%FrC^~#a6vezGyrjHjy!3dsZr2i;7uDuGN#SY$Cn_e!n_iVXlRV<{}0I@3gi60Gu;w|*G*EITXn{7 zl1!-fpG;luKZU{z((6~!UJ12MlvYpv-Q#sX_HYO^UwEp`+ISK{XOS)TI!OFaWOG0w64~q5$fZiUNwFmZ&QZx z=QOVlNs zUbTs!!|evPT;Hw2<*d~&1FOvoUpKbQ79h5qC>L!N-E804***={I&oDSZ@uo>h+xjo zW;OMF|7oioirpHULW7^QgFkRX|9nF8eK)b;e*d z502Md;n!j4ml9cHQIi3Djn+CQy=1f-_*dIIJnWnhi?*E>7e=?ZdS?;wd=_fXG4DgD zIe0NPUqgG|!}~vS`SRr|6!9yTUlGEv4&3LlOI8>`W^%WBUCt@LyG}KT8lOCNDUVH~ zn6J8ZS^mM7(kQIeWo)C+51Qv_)x}}et+9W?**=XQnYg}I-P%_;8!7Dh*$h4x+o~I_ z&Xo6crd3`AojbWZ-3}3)Hu%Fiay@A9982!TLO;4NVF-nOVr!feyOcd0tM%Qe3b=IU zbOgJ7QWZHY}d5m;;%Cc1a7XLZlm95=Nq%>dJAoF zzFYbY&ROdbc1@T02usPG5mc~-={spz4hz2-_AK2?PP*QUNp1`{9Q5XspCsh2*4Lwv z8trYhVL9S*kZAc91a5_edwa>f_K8w0#i{5OYR^N?0aV0*q{DZ(0ZE5fz%pp&S8`wc zonb9oj$v>*{!Wf(;~Z44r)#%Sc5T4^E09lJGM$Qdkl9OpRQ*akBOIl)l4y{wE?`g! z+k2ukkfK5H%_0M|%t%NitmILWBnj~ia6}=bLfC}+xtA8?`^`|sF$WMb|4G8~2_mEK zw@8Q5At+!mVnz7*q2&}yvkm8RT2U3rlo&a^eBln!%d>W;=rt(`f*`s&od31rRtB&O z(ZTLY(AgPG2=YLu<3BDOl!h2GZ8~h0X%m$P`8MdTC9aqP=Aq3MlOE}jE2gN1Tyale zT~0|w2;9a;^dm@-bj$k@@X@t#k3;*)PmBL2MchM~fn0l6CQ@^ZL3<#)!IgpGDoS;X zzb;>Xr$~Hl%%5)K?k+Uz>w_gtt@8wiJx%$H`m;ukE?u-Six!Zr3E`2FW43TaTF?`H{aAcKo19=68Tb*;{N3mpNMoVKNza(h=>uuBLBA6pf>IZXJ^adyJw^IAeu(X^{DkxinE!a9=2bO>QUyr zdTrh$5R0YypvkmQmGS5)9Fb`;HJQr@NhssrWQ-ZcAD6X(h%^Ku$QpG55lzM#9AKq= zGo8clCUfkc_?ey?__O=fJ7d1Eg%D9K!7i#!hPhNgPmyI*KsOI$as}-`viJ@8%qX|q zhKn%xb4uON^vZX*wEClTBt8`yZQvaKXpX_v&U$h|lt!{rLfq+>u@1>n$(;IWHRs)H zEZE=lNbM+JmC{iso}O2X+lC-5uN)^D>go&_9N4oA7}@-Jlm^*;DmvcwbPtxpLXr+M zVp=PsN}dkAJx|Ip`8ooK#Vj|+X>-4!;VL)qv#A;*M`xnr%%!E#l-TX@)^#i#0aAj6 zW3A{nT3TVnvP5c#HBuW_UOmz2o3-q~0r51hn7R+r?%Pr0z!FQ>uRbX9C0WXnac;lK z1f#%MIB@+&`PMcFhuu1hMT4>J*h2G_NI+AVPLxtw{YY*ZFXfX-9uXEJ z^rh`%$*@)p3}nzW~@(0eIq6 zhE~%9Vf)jV_x-Xg zV)aGE(p#gk^+P(Yj)EiOxox`C(`oC*OwDJ&OlHM-K61r=3(p?W2;*LYkgc(pQ8g5R zg&D?TA!Ct^Et593WCP6&o74?e6rhUDV785Dwsa{(IwcR~EOm16rNcF_oy%9r6LuoG za)-FwwMH=S;>D4Z*I|lP%n+0=cVlF=&Km6{-`k`Zai^j+-RCtg30v5de^3*9BNU@Ey#i_K?JdxUi13U6BRu}gPc@3AKBP@GD! z+WDb4O?g~&!TX6+jR@(0sbVGh0BtW=9QjBN#b7UaO>9o1HU4bFsf2heoEp!M|95QzJLmcnqN(q@#1^c5AY422<|9{$C{Re%5Mux6n`_P#mO+Q zYd#9Q;MS3!?6!S;s~Nsl;WpxzyG3Ff;eX33Jg=MweXkK-z|^vRiG3IO!l>LG z{cd%1yx+t=ehHGYd;~}`S3VW>8+91mseVqa?__$K#s*u=oMW!c8|t#_{NDLmA>!O8 zm;94&?@#L_q;kV|!)Y!#TEQ-SpRcQIfxFzBHuS13%kszPjfkWB3+~*8;y;R>5qjCb zQV@Yy4j~Pl=O4PnK9m+e->6<#doA*xqawAsFu2<{U#%ICVeb>N^~ z@83F({9E;V z-QyQFw&H5!+@AV{0sqgong6K|4ESGfOgVAn-^q9UYxL%e0(nf_S<5|t`TYa_=h8i` z<=L8FJnl=sFyR0Bao6thcwx=6UcXsS%bRbmG4O6K@jnB1bHNpKeO+@hpirrVl?4sTKy%s3w3Xl{GNq&FwLh_*D7|M`X76P zO6jfhkldow|FFI1pL~5mL1Ib4j`=`z{==bx=4{@yzR9tdqPD=V8G<$l?o=r68?Xxp zwhNIe-Q!8h(?u?KL#r@^YS=$~(f{L@>?2ETlApi#a!f{ zdNi;8p{F!9z0gQ9DgYWi2#oXR;ga;D2yjN93xt$m;~2R`@hYGYX4g{w6({s z+5hDWwR7VOwd8>Q_Zp`<+{fl;v|aA=fV)zk(Z1oRGJ90FM-ufIcn#&)3GX!QdEqVRbU@fzJyRR% zV|?Dxd0tO8HrSQCJUJI?DdrNN&QC9l1Idjbt>Fn$(LiW|rpsC)i ze@AAXFyeN7rreS^l>+=)P3pJN2u}X>2=m)gcmKUV@alKhnE_+aJLqBJJ0lf0lahlD7NjSq$kT4hf)KflJT-a$6 z-Zd~50gnq0pc^;Ad|x~hZAw}vmzvm*TpyX8`Rc+B;X;@1Ts#BUPRJi9_}FXp?Ee+V zeY0j7p>wP}P*NwyN~00qw4qU+*)Wp*=u6ZqRg+dX6*WBzwH8F8dRC{~YXyxf+TT=3 zjAVd|g&&Dk63a@2L@C+fVt<#wu>-r0HcAMIUnB;SF;6)49|j1%muv|4q;!buD-uvX zv1nfRur}zmmQr06T<*lBk|de`PtpEL8E5T3@*F)P{-cI~7DlOd@W%v`ikfUkr9SDj z;}qN7NLDU>Gj=I8>8ucIEK0cx`;1^s1OOU(3AtesJYB8dR=cQpZWcw!6-BW!{LFvE zurlHLl%Z#+Zalq3^MaAgT}(F2k@iLFVy0Ya=I?#dt`rRFcc*a?s>P_dQsP`Catddt z#mI+#Iyd@}w|%Jx5(NQsD*9iW!vB|Oe`OexJLO**(KGp9+v0!p%RL^a!d;c$^gsLS z1lp@<^+xp;VdX(Vp}mK==e}4PYD`#tqz5qZRP^njsU83|R*n<6Ib7ja1t#*}ml4|w zimvo0UNnyV_hmJ0pG*SYYxB3!j3gDvm-!JXT4uF!eT?;H4qRv0YIty0M z!2kC@>+w(q{@4ER0{>{>f8cX6q&DlJ-}G;LeeMR;8wv4QG;~ z!N^6$mgZ)by0+l#Za zK2v$Nd5wAobWtaxC0FwEbQANR`~qmI)mS@VgD6+R6S3lB$_wp@MCCRYQz#^4Q+!mt zJEwa}`GaT(hH@r)1>T4M<#c{I|10nR?LXJ^eoy+{qBv`tbVvTZDOZa8`_pMzr;Ba6 zsi{DnFyE9aiT6T+6Vd4ei)iRV{=^U2?-HRoQG{CUy7N|ecEzafPx6zEZ!%pWCP}J- zR8I(BVx)S~&g;lzjZwrx}Wu{8nx+)fKSYoTpP%bg7 zA+dawI}A8x;nQR)&$(*@-A63a$G~T_+1V2vA!SBTV&X7Sw_;Wy`ysji-?YAvjkhcN z6iDR(znSRe(azO}!Fao9Ss*^b+zW6@7niTa3zXt#yv7sT^RKe}iwV}yX+XH|J?dfG zV06Q}mV%#m9f)mTS^CChR_m&*Xr-=_$agVvKns@cQqlDrvhsb~>9Mx{_3X46o&@q6 zh335yx=L#F{iE^z${Bsif^^r?W;oqlJd*M%#H~pu7 zme{pUXOi*KUmtn2&#_Q8Mh_M8e@ns@WT+w*e>c>l#G-~6urIhEG>D^U&a z2W&ErM>EQczW>t?zWGi42c6FSf&9S>oZ(M??OpkU>iPSd-EXOCJFkI!s?5iK@zejD zG6bsh3bg!)Z6}tcXx`FPX-HRaW%N z`a&jWGji!;E~)-oHWKK>0@OUhzJUxQK!*Oee}e~oP}j3WvnX50IjGN;d2JtVvpqM&9{K&45Xb-ko}y0M)jMK|!@PSI#Z~R$kG_r-m2#p;T52l(udM?U5e3F((UrawxT+Zwvvh99eQUlZ#|CtxjlxOwW_mlG3O z8{w~?iq>4hXO!E$H?9Y;8@1-skfJ13EF4q2bxw|-^haKOkr@BGXykSX2;EhWv0&gy#9_|Q)C6ijdelcjcVJ1i|5@p}X|wmD&BVJX z?j#&Xyd2zQVV~f5B76KRaU(Xq*kArQ&1i{3z4LN%VVn!9N_{2Ht3`b2l(AUjaI8^gkro)6VM}n08!eXlVbEg z|EoPW1RB}Z(XGzBN)K~L(8G)#J=_dX^rz#hRu@j9hHDsqO!!EVz{^CoHUElE!(}UM z<-?E#GMGllzLr;L4Psf15gITH)|U?$23MdBJySiO>T|g18D+cFJ&JLaMO9XRVJ1iQ z7NamTfVQ2hmz|Ye$|b1zG@OeOHbtkR?@;C_J`;UA=IDo75CZPnJ7wCuHwnR98LjUz zcj-NH*e)lQR*L;=p8{_3PI-rylP2;i*@^a&UzOQ~FCE)^-FxSzH~=5m#>g~RpNmU3 z^k?@^Ms84kf130T;(uhxN?10aW)Em!_-@lKw7`>{Txxa{pGA|eYZq_`=fq|blLNpZtR%Sds*vm}&(w>}1HZpVG9v^f5DQCV9kYZ_L=a`hD*k&5k=VDCv&jThZ)7Id zVC%J_EqveVnE6i&5%k~!`sp|>mj8>Qv?%g^uV*gY@`L^c|D{DyBn7#YxO!9Gd4{&ow&fNDgZ^$gP}#~d<^>XkEOW2FG*W3Z(4KHGF3>4_d_XVID4`1 z@fXuk-)K~9?N%T$vL-d0Dbk6EX^SsaLB8QjDFGL{l!)ZW(50QPN^bppe6E4Ah#iRv zh>7%=*7MPWuWX}4VPIwi=&vuPqugi(dZHERK`UTY8ah*>y8L3HF2At4{KDYyt_TR^ zh_!2Y@EpZPIUnUZb1;yyl1{QS<3oNz~OUr)D ziWJ4{M@{)coqf2};{R{~<#y>iHE1 zXLZI_btS%_Q9}nn=BNeI8i4C^#^7Rrw26SU45Ecn7|GrDUA}xdzChL6WyL41J4{K& zXE#iQTu)a?>?(;!5}`+y8$VkKTI~yu&sUnlRF6*v$#pKRRNwsZg~UTeO4*8=TCsns z1$HX$umqTjY*%Gp&_&Pq8wNT1fAf%3E$R}Z$wb9T_`yTKPBJP-M)k<*@opPD#<;3` z7+fr4=FwL5IbEje6{aw@YbQ}Tr zkoaU`u0j?Tvvz>jMHVWXCBE3*CKJGr^7L*yz`_l#Y6qC<>-~s|$BMq!lkqaj0N# zdI1~A0+apj|LhOG@V|=$lFATlY(pg|i-z`M_g$Un6D}~Kppon+ z+f#(0wo|z}9)rP}uL4Hotfig-YKjR0vCVtT@_PbRwZmXEK*IF64AD~h_X`PE{`4@O z1S5Jx?W*5nLDnns*2P#ICYYWKV)&yAC6lccmbGN?nOkA&AR(Bqq0nB|qGkBNSoI>q z*)aRgoH6j3TDIB}Fvx|o^ z@o-NBSm@$auQK^G76RnZW7#C zOzDbDf|}1yiO7|>dMyrA3#nndks^25j&3@+)T}WsHR=ef7_eef*kbOp-wJ|K3Yb|- z9Y>BXy)7vl$cWgA~~&!$V7Dv5AYsE zznL2u+&1zb`+ssb0_o32WVb+kDBA+GHov#P^c-M1DVsSkoxcF4b3CgglD#4`z6GY| zz=t^wVER?ZS;fBr(+SAD72kRe(#<!|`hZk5RVR@q_%UBUZTgSa zBhxiegK{6bQ8?X5r<0WjtfxdjdgOd#bsLmIUgzL)n-g|%iz2M~S zF>ouodDwR?3N9@zi%+*3GWYXqqA|S-^n1kQ+23s&m{@r9Q_&}5uf{s!(t>#JPa%R* zZF~9JJQ*CsuM$nUv99}%%DbmBJ--AeeCnf@?!1>_4zh#5W!uQ}Q<=JAH7^!uX5fa? zMrtAEH)Qop%#NnXfp}caG{lD57GVGr`Di`z(QNH4E*9>7Tj_wlCMOT;86a!bo>UdO z?V+}ui|d^f8k23%g|5&^`sn{v>~&oEq@~0@t6Ko)!~z0%Y0hX?p&wVtHTF7|&Xfi6n%V2FiGk z?V+=~r1}^)!2$Xtyf5hvAu}WWP^8HUH~T!{4pd{B3d|)>U7KQyqqpV<3!TO>q1g-9 z*o9lQ=XW{ECDm`~tUU{A>S5nCO%7ZyTt-7Cv~Wl;9t4-3sEACgTH0pxUaoxa^tM#F zb^_g^;ftq@WnqMQa|3*n^65M-gV6|52stfOiZV+$ed;49Z)vcXR0{C=151e^P5MEG z1yr5f1!;ZY(EPI16k&Za0kSBuz@^I&E@0f2^?9Pt zD^nZVEh`Kh(Co}Og$AKTIex5<&*8LNlq|GYBtayxh50KJAK;J9H`q)5IPojKa>Wia zYQ!?;iaZs&-fCft8rfNYyw3gSztpqu|Kdk`a$x?kphJo!SOx`MJwgYk)}b&0Tx|m- zdi9H{T2Pw`6cZdmtUSr7K}6gS~Si;1e14ZZL=P>8HX*}X6<1lJ!BE}s<+0I zj$zh&wuV`6ZkSC-Q)OpRK?EQ+W*G?!$>~i6c@&APT;Z=kX-}sXLv;831(nK(95bX5 z#eQj+*0p_?`{${53$ysJ72t%L`0}5n@z5$7SORUs7Mt*g7itVLAG`QcleRSmnQ9E; zPK^PDe92z&9WK2H8=0i$UNi8ix_eWJbe1rHn9)AH045xN(PvgJ_95i}bN`e8iOomLVD+@Oq7ZVKYhGhd&leUO;!1phmM`)uv_S-7xsJ7UUHfat)k563bKM_gs78iDAvsjX@b*ICD3**`+|HRN7dwMl zZljOc3~FSA;OhGyvm)&w0S6)$#k3~gOk+_`rbU2*^L3#)s)sgbgV0S;GPURo=nGb> z6Z^F7Wkt-RGbHPS68TiLfsBQw&!N~vJ)t81g}=l$vJ2C-MxAj9e(K9?{^6-ed=o)4 zYyNBNnpH#d8}Bz`i{e>}GF zTj^Pc!B}q@#%Cxi2lIh)ON-5-GnzswoEB(G+Oa)IE6gWl4abo>>w_Yf0cTLh^|=#tEvAxy<*c^-UJ7;yJ1VB1<-* z@Gp18i+3oIWsAU^^JG-LmGT>Pb;x`^ZFUzFrJ@sx434Fjd(*mh8EubXCQkr5%9cD~ zhT@8aN^ej)wcc@=FQ;|mGEfW85d%kki(5s0)HnOMc0R68D$qzxT;j$LQnN}|H&GwW z8=%>|(&tU-Ma<;fjZepSHbuS86H`D*Z{1a|gP2nMYTZ&G97r&N)^wx;>~3#TJ2I>P zl#ndl@oA$3Pgsn(7UOs?xm`wipS_;*LwmjNpQMHOz-_hhcS^@>WA)h^M#zv`ygzqa zY^>bk@3}RJ%l(sPq(p99y*^`2{&BkeS|6bSc0I2cyY972uH>#puIQ4$cda(fvTlnv z@)4F}^(;?b*~!-tTJ6bgs=?J?Gg-QU?Pqz7?IkzBEIs5)G088z(&urR#S zkjssLdiGd87aohJ(fA3U`7iw_o9?672)eV4&N4VjKSSMEy}?@SyN&KL*idq)T?U37 znO*24VzoVKcWr${*M64MWsjsKmUYn0@6^udn!bgU7gV!rj8TH&-sV>1@+K+IRbZEv z9)PoQ*^sX41$M-=su$SO_OB5L`WXqjjYU9jl?Sj%!*3zgu-2L5Z*a%mW_LE$x>LNi zFvaca-?YVZ+!m;~SIJt2sI$cOfaL8f8nsztgG&dJ3=IWajj2_4|bU`$=#-v!aSE8KC|sb5w7gaRX4_XE@Y z8uWlc!?Sf+Y_r&?)Yi5Ub@U@?pT$`y7iWRy#Isq@31>T{eT+fQlwCP<#PVERBG(bd z?YV7<1b%p}NbSKc^O$Z`3(Rlqm*@moy~U*Z(w%yf)#QdUzpnFr5om0jpW62|vsl(x zhcj&wo?2!8H`3Agpb^<#_t9ZDWRBqu@PHXiUNv( z-z}f>yKzTBK&@IY(`qlcUuR?kJaaDw^s(<sPaPI3evo{kdx7w)C4V3MxHQb%_H_+*|)khqfWH(;(SA}CQoV_rfUxM@__wMkK>zUl!NDeyPsl3^9HN3dkf^fOosm8KHW zB9bZ%?xFPpJhmwe%u#4EcI`+UpQw`XKs_UuX!Ze2SZmpvwcw5J`m6L)%#$Ri*DNN7T*%g^=q#8*$9#4wx8Mc zwiotY5J~|>aj@_p0eOd~3Ev(-3TC05@G#j@#`J;74M>p#q`0M23XPCa>`u6mPyIU| zF$0A0f451g^9otXW|WJRJFSbpzZ3Co28kd*{h7ljVP+1WR&zz=7hqNyK-h6=uH1^( zuI#SnCmooQxsL3<>ZBdG*P`LMh3Vvu`?z{bs?^wWt6~UTokq*YnSIsMb{Z|eH8`g{ zSlyG_v!D-}ZUd_pJFw+osj_e?ix0WVrT!;ct~^tcw@3 zi(7X+B(3GqBcapWo2z8McB+1Dovv8yHTH zNxo~enk-X6>gbH*qk4lUZ?PT%vI=f!O@&4W9Rp@*5TRXD?RXaLQ7qaM?wC*fM`w$f z-M8xr%>2YZg?Hm8((QKfb;b14B8cohre`3gMSYhO$zWkWEt7uP$uGwg{>*!3A7R)E zwv}_%j=vC2ZauTL2-@(-{4rr76>xDbs3q-Nu(Er}E@-nJrAP3`A$_sF|QzLvXiM)h#b2E5*O zPrsH6id%vP-UQus+2f0ocjSF$U(015gTBAL;&VDz^t=8hSMEq-UTpzDZ0*)~c5Q_8 z`Vv6RT=Q1g3elWf*PPrHgqL%F7@wg{!@4r)5G_PNIvrYLXIlMg=7>=d&~1Z=SC0v~ zxZ~DcGgnO4k$A2#p|oxu?~aLp7TdY>p2P$uy&TZda0v9jP?#D_5vW{eca^ZN7uS<1wi!>^ zTa?WzzmRqrmgV&-J(Fo%2226(@;_de3qO$@xI`2t|6Yx(6Tw01l+Q7f0i^1t{9oz| z^!w1^pIAKnFK34*cKBZ{9{z6^5C5y#;Y>w#)<3m)`2V?h_*1jPGduin@$gSB9)37G zyl01hdGYYyTs-{Cv%_;c{OQi&(YY~+W$@~C$|ogg6ZKO7feEyxMbKCkDRBd8;2@!e zb(exTni{DJ5(vx9%ION`0Ep&w5KX3(4BO8yC06s0?j?_;S|!Sm!b|%@hdtM$!F)_X zsWH;IxcqdUWCsbQy;AnZ${L8KTL?e^f?$ZGEV>>)wCGYWATf-Ct(Kx!mzh&3tIis3 z&G@cmm!~+2#hmXtU_f;Jo_}*##XhpW!Jw>X-1$Au*>dN*1bba5Y+^OJZNuJP@>8)j znj#h>D8RMF>n=Uqjq1WapwTz8$`{&It9@GU&Gs3mhPLu2)ewFncgK0me zF`D?m3+g76l%fANK}c{nQzD(}V23~JBLAA7H(67<>`|*M1L$Z7*G4VEQr9==w|ZfH zTI7T-B4Q10cn+e^{P)E#H&WCYoqI&E8Wy?tB?L9U6`BUhA^X)Y#AQQD9|hSx@pWk9 zxfu;wrHexRwFs~U+W4!EYiZ-5Q#982E;zr-8vl&I`5o4H6sPzZZimWsss;s`l;lz< zoj35CFQ1m{7rG50pIG;!&Z#51;(gKz z`K$qk7S#QBfFw7@%+RE1=>=FG)%3`ZmUleVxBiFtV^Ruj?u&246me?iRz95?qU!9_ zT29@{< zdl!hjPTcs2@sEypKSgmyE~lNn%GUp#A*p*Vqlgh65m5kFzpls}${oCmU^dH{TOd4a zX`oXc)t+7Cq)b)_Bf-?EXQS%ez`zSa!-5BoPRFWo!U}q^wORlFsCyfDORu`Vcdhk2 z`|a$#&pb18CYj6`VeN;LxC1$4wDBa)(92mzG6@hNDhSrz78P5%PkJwBOey!8WQHV? zCA_WP`EmR1(pYQL#p1t>(%!KgbZB0JO z+56eg+j{%2|I6?H`-hZEtJ7@C+xpF%h9?YLS9i3GQ*4V!y8fi@Wxg^&3CnRfjs~bI zrpgB}#FTJzgl05t<)aOvqR3b!5*(~tp(1|UG2F_MD97QZ5D9INOQ$rSn^?jkIO!h` zN_J@0frfMM6=^W3!<{3PT+4mw=VX<>l2wdP14InTDoW#W)YM}gS>8iZa%3!R?+EM+cqsjdfihvb^=IqeitAMt(^ zR_HP3W6@$myqBaZ6#W;JQ+|w=2ykJFWnBC?955x(kAe|Hn;LtA-&`zRP?6;T>kos2 zEQ4MAe2-KXk*hG_NFKBo=to49k}K7SNeOr*6ttV8kf{t+QnsJMsNG}_=~fz*jo)!L zEPS}m_ig%6?C*EP^4QvN7ZzRP4N;y+7g`xt-5YEKVHN!yr!G@f>Bu8Cz6W7s8p zeO6@PH0u+d)6@Rs4mwIUb|=xL1JJY!Vs+Zn->pm|m&KGiExPKyK;EoqnJ!RVPm~KQ zu>Ry`>8LZa>(Ypeh9%fmTG3$%#4!lwAHwX2o&Qaa&yRtkpb z7ygg`RpLWR@a1z3;9_^aUt%9pl!J912r;#J|0^VS*NyRG|LD7Tw>a9LgSBlIJQH>a zb_O^vNd+Mnc4MX4XmZE!9walIa=+11iR`udbn0QI_&7S*ICR}eA;n;bx#rZqg2I6(NEh$s`Mx@f-zS(CM7-R@~XBk7MrRzBs^2Jva*r(m zDh#iOgmE#TfjG+{m z?+dOzH;!YtqaI)VV?@sGQcY5q-eade`tqTVkB;N366Vv$w~+y z1?F+`xxR=-*QFb+C<$07K1+V}w^bdn6JtoX>eL?`mVDBQda#>4uV1$^i`QDH25q^u zgzU_f2#~W(m3omGl1HmE4v?RA(%H(g8jy>tS-;3MwR*-GphOu)_VxvKi=F(ugQTPx zO*M&knc;sITNsJ#3u*79-owe^;vq2A9c$ruU|Lg-EKt=|lX54^u}OB{)qFZlvb(ve zW8bX6MVjFG)2$b^P1-J;Nzy?0*Rbh~igeVdssPXi6F0fuf${#CtX5s(fu&ceuIn-2 z?tlFeFx@(vpjJzm{iLb{he;t%AysIK#|W5*%_Dt69yXV*bFc7~9;Xf50aKJLB$jm-G_yz1}v zh=&(z;c=oAJQj`|a%XSLbVGNt;?Q6BTj_=VlKRY{OGih$igk$cVtp3B4uuhpF>P>%w6JM~nMx`VR8WZ@#Yq`UKUrm4!wLn||DrUi53^;@I*9 zdQGVRCdj2)(~HSgV@%gofH)4S-yn$X*222h0W;*Lz$}A3@i)}5&A}c5T6d(JH8k{O zfL&l*asj?yCQ>sZ=%%WXis`A$7gyV2st1s5Yq$Uh57xpx6ebO8;rIFL*22f>j?cc9 z?)bR6WBLwVGWbg|URCecTB)?W3V15XlB`uv39GE9mAe*>bF=rqo#}bpZe~;_a9-hx z#5K29j@@F_JqB%z`QldH(nzco$UMA~%A-`+w7V8ga8q0BC#G{gMP&TTyTU0kLR~;Hu_9?Cj}tD9a)V-OdZh3xEPwd-^Jbk{Ihh{UHR?~i+Cv(=lfcT zfif0rOG9F>-A6AT(vj97zl0wEM$$f%h5b}cs{dTZGno#1(yi}orQg1*ktrwK*0OXz z&9r@7rr8G?GUvEpIh{%^WZs|dcsg*}OHN}8E7g7P1U-dRvX&GoNfQhC42&$*a-7RM zM`N7);?*B#`)8c>Y{gB55e%^#m7kB|qcSAN$dY{RtLWNe9^!AABif=*T1#!Mrr4qf zTh@b`N>*fS0l3Jl3w z^)3rI_E+6Xy&IAcen{u!bv-OO>%1Y}@m-Jgb-lB(%;7!Vk-KIWu)zUzH?>=J3)8B2 zs;E5~4K3$f)nq!csI^yFCih52J^(JBOU9AySVO59#e5=jY{^rgc*wL@4FO0zf|om^ zc{$Zo_ghwKs;ti(K^Q2>)rl}LnZM>NE8>hs5znH*Na47kPx~0*F!b?EB3GpCcn; zu+0rkjd>UsFOjC)&nqgkX)MGbbJSsqlSc5Ecl%yIvl~_LZ?ig~hKpXwx0Dd~_BWv~Dumm)vM)fj+}r7{s084-)Z1c2!JghWZ1# z)Q3Mgnz@@R<1$+{@5r;$EIHNW(4^H5MI25ihl&H5+Qj7OQ1%$sp!CK7u>Q6%u^t_; zdsKf^iZW>&ZVRITxb}@u53JEyLN@{NrCQwx4wHHJ|c&J8Fu=C~4Edh>Qq z0kuP8IiryXY|HTy=B#aF`%waBvvy8hlPio#xoy^F=TMF-Nur^pfB5xCwDrx#N)gRx zIOGAy6~MSZ36ZwkJ=hMJ+BP37(?yewqZ>zK@x8vT$1mJC`g3u-@%X1NQ$0cnau-pY zyg0XT8$LarEaLR|r!3$3!s}J{pkrOG-Y7EVgX+;2raOSqRe9XB^CW#&mfxy7PhMOF zSBGgCjrPvO&9s=2rSzGeZ2_wI2$MD%x65NYz330vjX$oPs2_*SV-oV}Ah`IY z(1GE38%9@ss2B|)Dw_M9(rep)=O$UB|BR)>!ih=-m@XZqWBpH^<(MqraGJ}|`lg4a9f#@mUjV!Q)iBah$-`nA^r@6j zjq7QnEpVNj_NwtD?qlL9VJT!WCQIg!e?$z3F)U*AQw_v|Htt4-G3DdJX~fOKN#bVr zL&=)mad#tT7UIIZn9;5$xdLSvLd6pKcS<(RjA(=kiy{ZFh>e`4)!%B+>if)DZ!s&J zxTbZ>Rx41aEB?T*b>sEN!WQm~Qqj=zC^1nYaS*iIud#Kc6&FyA+ zJJD@w+_l_;zGIVm3iuY)*8}uXJJBe1N-02};1&ZU;D8<*$1|$`0XKTS2z@hO*GD~` zX(<(23da2U=4ems^QRvdx%Z+RR>^NlT?y;)yO=T*J(bz9hu7I}1^K=U4{DLzXipAE zSad+zU%FWpxI%h2r4lo#T7|@Zw?AzAUWGD7C&fKSMtj82RZ|ge?g>ljShY3UZBH7h zV<>5Qx4Uk1u2qz&laum^bu%Iv)oOn_e1hfZQOS)4vbuK~6<5LgD3o?H?}d+|(l7R& z_zTESqC6sw;apsl)8c4Z*=$kj%#K17fz@aUSv^p3inxTG)9*+8=^)DB_Z+5kCiNp| zx2E;bruFEIgv3BWWAOT;A`Yh3kzAK$Hj-F^rL@&pHrvl^OL_>Gq^Y21r2p~$HZS%# zMP2sE+^-2woyq~J2&#sqv$K8S?U0&m| zwHNTZVg#_1m)Al>ZwLwrH{V_Rqz1zJ-6ewjhFFjUoW=OwH{X^x{KI3VZ*9B zXzMxJxGyd}*4J?y#G<6EeJOK5#PU}tWKPY= znGMNxG890oi8ip-X4F+@Qb4tHcnD1RA@3t3%8PG%(4DNdYUAIZ^2HvuWrnL>@g&r8 z--qieKai?;5Bb1Y&-N7M5~9~eqS-~|Kt9cg zJfy8~+QAAoSh6hxgQZ~5)QT{UsJ;=?vWqs%sWWxG)mXf!xG1pmKdG)%EBVm^+}wFD zE%S+|^u~*X`n1#Tbl!Mfb+a5fqJTc+AEDye!ukqTS)${szC)wqgZ@;Y2>@w8#c$1- z0AgG#WtlXCCl&c@lL^BnquTfR!YHP8IXHLg? zwjlq1(3FIN+oma%@yywREHgXqY#_^KjEe`2acN^*#)amb7UzWicU-F2@HTRU4Ja<4 zyOkDv5s7&-worNXt&%{A_BG(o%zz76Zc3~uGh|Ncp^2b*m9TtBm9U(ugnqOGwJ`3p zYH$x*F=Ql+0t}oVu>|SdHs$=2l_z_K1&ErhJj2-__KGd@tuI)3sREaCQg{jY49hky z$QfQ}Gd5?|7{+^y&>pGgF;?jXZ$=M|O`HjbJGw(Lg})V;?qw{|B(B;@MG6oAJ=WKU zmygs7hv2RY>kr!eBzpIL6QVcAk{)@=&F_&9PxS7U=*7X!evGaR<;-PLKal>r9$Nb6 zS7MfAwsJZy**I0x z@jzRPRr)Pi4Jrq0R>RC!`?Uv(ExQxT1WqJ6AqOK{4>?-^S#>i}c7YLhnpu!h5cx@J zr4i-Vow!275~Sk+yTXB3X?yoGVDp##U27^cp;+LdKElZ64MFThbE0vDLDXM!JbM z240oh;8?O=BV8hAB8jIEUr3Bz8Cy3v5MbSo$wiL$QYH8f|X_= zRyid=yo2d8FK($X6k!VZcy1)GiL(s|n2sH*5NB+?ikdzvw$rQrjFtRJIjOmlpY96a zw9Ec4s#b|YqQa{PJV2cJ7}y>ma3dfM@}y+$(-*>;?w^cg{mR<&smw&5W_1GVHe{Xh zC93p*vZ6HDhdiLrmC0z4&ajj_mG!%GbS98K4E=a!4!@tN@Vof)GgUtNjhqOmKNPQG zuklY=u6NFEOtLR7kS?+R^2lf@cBxviCoaHoRICfzrm-1HcJ9Druk3Xt95WC}M6Elf zsBMhYu#P*&Sy8r@>bVhb7c2L%WAZnhRL-q#sh}IC<$OUmyH@ZXat*~quXypzpj6OT z1Y5&oYjG@xiedkzN_ZaU8XBotLK! zUHYppw?-Yp9-zQ%zuv+!WlS&%fZ9_)wnL+*Hh3BJn401v|Km^0Nzl^);WQycWl)Jh(kNp;(W5MxD4sWtMSRsRxRW^%x+*R9$98=f>IX z#&b2M+3zteC)5$R%j{!SQ_(yaK&UYedyULo8GF;D z1x=RS#$J8~qXIOpGaAqbGvwMC&D*G7vkPFZdv{^7@3I%lcoMOq%5oR8Jz;mhkd9HT zeL+=cW*Tyba#z`7Qzrj|>>rxm%psdx)G5l37UOAWpbb^uGwHuPVc629`+_`VI zY1_I^AnS6*t<3B`jed}uUi85qX{;3L(GJ3rWnDFC7OO}I z);t>tnfE-z*o|F1(5vm-!Wfbqd(iBm_T$`*Glwx+pwt#J2Eb#y4QG6LGX;I3?LqCYMDHJ zJ~f!uWhE&q!67MZkeQj$pI2Q3R;yS-dVs1{&pb}VP*zi}?A+|0E?nl_3R4A%c!d(je_j#p} zTZf=2z1eRr78HWd9`oEU%06J|=S??Ux>ST5hrKF+BM+woN*}Pqgm*G7g&d=zZ`-6R z$++A*lA=;+=#^0Djfz8MDBBzUYEJ%P^JWuS6WQ|lAis|s@=Gk0bG*OC-q3OHu;G4~ zeUy$Jhj>A7S6^6k`~45KycpDMbHd_B`&Aw@EGdNr*r{y&Cw?pQq!e1rqq5;iaqy(* z5_nR$M1VXlFs7901(_+ut-4?rz?8D>0^>@VUXZy`+^Q?=0=QDPU0`e}(+e_N3JK44 z0c?fD#jU!QT>xLowhN3gWqLtoOmVBOZ5P0pGP^*7JUv;QDTUPdzIK3_ z9Q6IvvFtyWI+OH;_qD3Y#WFk*+b$NY?i$S0OK!4YXo&&fsdIreY3?)wG)Vi`W@JD* zqGvoq`@sASzxqD2(gYEE@C~OJOYFQ;UH``XLf`26O9=Yq7ZRR$?(>6Q>fzsXJKQt5 z{+%t~hAZIXPDMr7aV>9A9uzBVgfL{^G6*j70)i(a(kOnKW_r17NgTIo2kW^j}IRaiSx z!RuZ=0@RrMO-~R=r8}t?nj^8upD1lIRs}V>GJ0yM2`55HxBR6O+xZj7>~3?oh%GxT zXoR_Ga*i&3nq2mb2ijJp z*xF}{P+VIAGT~kY{DzvkLlY}&s!FdElV^`NrYa?SYg-;U7*UQ=G1(k)EgI65YYRJC z?l{(IcE^E#6ELhfhtRIXRVBZlC6I8-t}N~aiKfahlXZLQD^!uD%xOSo z9tPe35>gV6kkvz^h_gJ|;n0ICnbw7!&wrJ1Lzzx1aN9B2IJlTRpbT1j)?GjPARgjvosgp-vavSmfrl_F2-4OC_bTwNe*tOXap&R%t<+Fwsgs4o^^ZnMlx~aHkHF7L~0ZTJ@h*!i-g6 zFO6Fa2-qP%U@hkY(1L_%GVvw?lM;iqfy5i4%XAG+$Wgb!D6ksjhc{CJXsVo;GBUr? z@r#*xsW%f519Rty&qh(oVm*es-_TVs$mDhQ(m01$dTH+|fh2{Cy;H#%3bMJWt&Ax5 zkM*^s^dtR8J85Mp(p~AO`(qoI;ioiq`(r!qjEfZOTab_%Rxd2;&OwKhFvPw@P$!*m zPr_+Uz}4L07g;>D6h=93&efln`Y{mXz*6`H*Gk5bpFJs`YrOuTt@Bz#G;Vy@3AWD8qS0^?y;g%eT z()4fLs0C8Sr4{L4?btf#@KCe$3`I@ms{cxgcYD>fMmx~%2nnTse9#-I;21S<9EEK4UJVupZeo>ybO}H7}8&T@L#>A zeaTQhB>nK3wwWMby6+Wl=p2?KlZNKxwDcjjxmb7yD_uyR``{~H*106=-tTH(Ld5M; zpMBpO+Ls_Z(r?aRCGtN;1ij#a&Q|);Cw_$@V(D&L6-=s)I+jgW$rHH4T%uf={3xXV zdTZOtF{yWUqz|~w#dZ-|&bQi_NaSgzQaVv;5grW`P`vF+njBD-8s2b*wi>nIhGxfu zra+;5Dq~Zl3Z03qY!Cy6*_?(o(XXE=`^JRRwkiJAuNGT}{DPs%gJI^RKKO}35c#yy zeCL!|`ioMKAvnBQ=GAXXJNJE7C*bUU3+>*RZ${FGj&?WgCsw(E4-TVwbpme0Is>^m2ix%n6*#4|EPPKOW z1%cq(3fOszB>Ttu{nBSL`AlgLL?hjZ3%CX3h}k<04KPEavh3Tq2B>XgKx3g8)B4 zo9oWU?#xP3k!NYiD*uNRBf1feq7z(-1@%}Twf5|5ixA{mSV?!>BL37CZNsS#JXX%F z4hpI#b5#lYooW+v!K24f5e}o>r|O9~RecK2OOM-1x7rf#6ERU=sj?h1N0EE#%b;*u z44?R_Dh0RGQROla@E*x$PH#a{Mfb;^A`ft712$m8qWTH;8JpY^FMHY^lIn`_peoqo zxxVS=MDx}-oJV&Sd>Zpf7%7$E${dHW1HqjbiZDf|rC$9H%BkocijB zd&pvV8dp*MRc<$ZzytSobIVv{>=S{zi*ax6?%ZX&{~d2-E}4Fh@fw1M(uvzk$<9uk z7L)bWEuL$^|ILCcel7y>u)Oeo7F>bN>tD5fkL4u0ilHC1$lR7Bl*5uw(bvlO@aZmp zxI7AoG_<1c<7KaXdXwUoKCK)cLW=h>_N?f07+=I3PlF3>9J2-XT$u$q&%X*ySC{WvS!In4hrP8PLQBKrR8{m35*K=(6Iw5>`|PTiR4+a5f3i4X_(a z>wN9N137&Bl#dM)EKr*h2!~meMeT>J=03FnT$)3zRfltE`dzKYJ-_WBO0pC-`eFWA zvLU1$`AzZ4=}slsHXvofWq_NJA6%ALx1XH|;{aUCHZ1a8nEk8nX{Ld@>h5BBw9t|0e5PtYZ0+2Pjl=>v|q&?LF)edRv3 ziorAp&?N{k4xjLZ4URpZBFY9u@tar1GClH1%SYD2C%j@~ZX7=26_?)!%4Zy&9m_Pn zVv@$23S-o$z6l7_K1HmmcX(9>C*j^UyH+;5k1O8Ys$Q9CpKB2OMqDFbR4bPEyTW`* zY>*H9urYe3FS^6! zQL4#XtOLcis6Jb!@I)dGtIm-(DKWsA>&=-wVlX>XPqMl37NzxZWmawiWK8ZUL!aIP z!5x8(SYYs&5TLBPKtXc0g3Cx>beoI4;O(DMxu#{h-ES_oocBag)Ijr}WY0`png^we zqZ{11OzEkR=r*4Y4*}P;2eJ%sAvP{J-b+EPE^w;7&m+gCC88gG8S}((^3{DMIVLS) z%yPpRV#eGs^t!T2)Ck3Vf(u69h{jM&%xnrgSYVoqhh9D)d`orLBwdqcMId|n+VitWLu@s+`-e>)HDg4mjKP3`F*d$Z2mDHKTGSyab5TK5CqxKN{`Ul5AmY|1 zcuixO_Gk2{l)=`j`*HS27;1j})V>{gr$yZ^PbrQvX}x87!3$!hmBL@h+yj?-p1HKf zcHYYLJm4+RrIql=7IJ5a&Q`*4@6>mNsKj$+tNWn)abeWPVTC;8R^$j^S$tQTCU(f+ zlNQg^juMhblQrc8Fan|7ys%McA^bp0E>FVL5crUP`Az^BKU zTKm#o4h-}uX%kpl?i0+#dtV_f6{@!I=G0D(9-UoV7LlvxZxM85qX_6-)#uCTC#YVr z$f?N{tk!69Z*7Iv(E_GgZmF2Dv7p}T%qE{Ijp^Qbs>h4<^|%nbSH~V)kF9yG$70642!_89*JCkLV5eM_huls(wsg=f z@o5Zu;YIf8^0jce{raqf?pbzh?eNRC1p_@ z4cCOLqCekPbqSekRkw~nYtEL`+mAVn9ibC`#(0 z=Xw-c6P83y>2KLs1Xj^~E#8bFv00;D2l9}4Z1V<$k{ex1&2!OTx?DYnH1y~f}TF zHD8-UQ#CLht^S|@XUHMj?`MbDuxv}^;;}BCrib-o*?)7V@JIbs-mom1#>`;};Otv8 z3tC>+mMeWI$l{_-i#ic}m?rqe!?rmS_NIu}j{RtL3r$)(MI^UR^_pCY$yLAVVQ?m_p7AY<2i-1z#kSvS*Y&t`>>9<{m1Xc`3m=Kbk+O*iT zG?@6AOGJxg1La-+9EE6&1yw$Q==lQ={KHBiE$y@Z}?!02N3n(u9n&=2pEDZpjgYoDlx<}S8@ZtBAi(luK|$z-#Jtks5t z&ML@j0cGv7y2IDSaMo!>y7q1g3X?3si5O(Y;Eh&^qa4_hS@+T}LplS`(40Gpzf~HpZ)`RVZPF^ zX--9Nv!`O9@6)^FO zxrra*^xF?Sn{ToBTDvaa`;)|u=N83P8(06RDlwE|E5ji$o8x-W2SFwx0&K`1auWLO zq5;htGj=K(@+4HfkcT503u^A!QJ7FRxB3jb~I|k8ecyo`$0%6gpK|77Egr|7sI(lSFpZt&D<{c;62Xmn_JJ@wm7$sJr0hf@(E39 z-%ku(>H-#T?pa>le0n}Hh4Ou|c<=ju*o1X)Vc}9Kan9C#;@Ml#AL*yhf7xUk*651s zZ_MLK8#!SPG$}q=P+MyyQ{;0%n7v6XyPtT@PZeN$ioO-aN2CH5V_!O z=95n5Hi~aPZqfhQtaK6HW^;@FCx49Gk6zjnI zdJ!IqY7)IUb1h8fr>>RXwa`kgL4i_+7 z>SPXC5BE8;E(>owsqUh`$6+sbEA5-X1*7Z#v*54sM=8Rx37Xs0ek1$?)5$jc)fW~- z+@>;aKHo>BDzeKWhKG?V^{1gg18L=0Lfevpas0Qzvq%ZDSyXqbxB8|IdHXR%*Psdb z+EEdQw5Ar4Q4eH~ zDAWC>`{8VsfLxI1+Z+8Q{Z%$Z#bWMIET+k3F)S$xJ&ST&{g)C4=*NBN9>s{uflz*f zTsd_!)f){dbkyK01@|rwI&eAoA*T^Rm2v7$nj|Iw1qO>oQP|gbnYwpE=QQ;YEo3j# zg1k()`}Yw`iCwvfTFUKg!;nBKPZk&gjsZyXWcA2RX<1{ojdC4ls?Q|_%L_H>gu=?A7(M{!|crYFx%t+P_L$a3Lhr% zv&M(nX?&O*JlTh-KwD>8uf}|s9K?sYfBtcqQHhP_8Eo>Fc`Bp-8O-P=^VBt&r!L(Q zCYETYg>k!axZEyN>sZJEf?4vZQ%b70L{qQiQ&zA7>q?X=Jlc0W%oE1c0n4hb_s=SYT`62|V%hfBG;w(!Q}Yb|zj;E^!n z|8i52S-4=hrE)zkSROgz=#PJ`6&FkvG2ZG*0Ld=>>aQcHrkpeR>fu}l^SrY9u(thq zkj*K~vo@;(RN1VUDc>;9OoF0}w^eb#ebQPw(`+`XSiUXa+xe6 z2ZuYNc*z8xDIQ7j@R0nNfz@^%fRtKf+=_MDkk>XUH=2s^Rd%BunZ)U%!wJ|~ZwqNU@?nfp&(V>MDfJ+lw zt4Hm0tP2Z7Cg5#<04Gyy)7S!ENy+vl3VMS3RccGxpmh$qyXfeG!msj;7pv}_*+C@NYYDpDfzlu_C6`x6 z$^QXVOZ7ofP8+9BJ4dHtQQgqI;%#5*u7!6iLvv95Kjl7DAGyqlOT~@=WYYs0L8=WD zAhuLin)HV^V9Dl@zUzUq8SB~b6x)XfXmN3x;h~2}H5Y0p+JzdPmJN@O#j?>b0S*(o z$c8EUCt4;mMuDiKii?LV3yc^A#_S?J7?C=lp=I;<9hZ-zCe zM{D?+sogkCqVNB^-B#he+N=lWkt&rul2g0gs_T+R!btdw*``SJGKYXmkCGQs=myY* zi_B*neS6c_<(b*r*Ejm7TuyRCLh0IN1)Q3(BCaWIs;h{^5qb`T2C)8VYO62T~ z!#b^G8=z);m-b(9@vz(HrzaS1iSZ}XF*_Z8Digx7SACS>fM>=XbUR}&k*vpzuYRjV zhhNHVRT8rLye9p<{@kL1Y6BSWUuthDW1q7-Y0o76^B04i+yn>p^2UL*_VSKto~9N+hUC^u@?a)Y7V1C zX?8wAZDx9wn~V{17CW`_THDT>(=<-E6}RR! z4MiTFVADJ<#$k0!U^sPveC$CXeYA+Jr=kY|@)(DbS|=}{jP`XK{)Z^?lD_=cEo{9( z`u86wSwn;Ls~@HpQh(G<{q*Xa+i_66R1^@s^vU9hP(3!?o(EyR_gOhal=of<;GkJ( z#dM(+T4>2#!kp9D>ksejT{#sWZ{>o~EA-N%HmNgIGI_5|$BNDLfd4Q27iO0?z&W8Jxf z4&usL+qEQpEUrwC5+p<&a5NT5a--!&1T}4W;EY`c;%~9jwJ9*V3nG|sDF}4^KHsTC z9l0w_J7sTjA0Io$N1%Q$NUrU2_TAE6PWYY5Qv!X}Nk3OYLe#0droiY0cE&d8^p_iR z;C_5`W2+IrVt*ojiLZL42^vtRYx`(v^p9)dJnXbUHpy~b9CVs?OR;#O?Y0NqQlD9= zrF?*^v{7F4aJDweZ4~2hY4v)+@~DfY_;BR3 z^iQ#zFO52~k4e2?tJBM~Bi4%6!Eg2hZ30LTQ6zvEk*5`yU z^3nMI>-vr$+!=tjba0QIz<^&1CyIe*`#P$RMPJN3Rw9=d9MiYsIv~x$OqzwMGz*eu ziW1=Qk;Fh8jKh6Jtz66Jz|`Abb%r*Mu6loq@gG=5K)3(85DJTA$&>UR3P+gx1mA^%h0>Pf#wGEYC<`SljUuPfUAv}e>xd(xYJqh;K< zFD3IgZ;RdZh1v4O?*$W08C<+}16N+Al=-Tsr6bWstl>%R6=dskGPejF-6Q_k;xI*Z6kKb=**!l!$V zclzLT9uHTfNFn>N6OQt6#W*J%HI|=o@sOOgeypl%A@W(-cr@9uWfVpHQ|==)ZN!!n|Jrgf1pUqd>r!IjI<=C0?k62h)0`4ZB5rHzxb&-)Edf$ z_^?}QzS(=iX|D7mqv8iLBP>s8kpua;@J*=<|h(8>X8 z+tP|VtLuDDpTfgqyO(4=x)wkK#{|!qDj2sqOfZ}e7SgeqE3$iWYrT1I*fHho@0c@$ z

jV*+IP{=f)OdoR>>M@vZ&o|J=e4k3yX5C*imFok zD%Ne|bVRA{;%>cAe6%jLh<$u7gOgfMT{g1NE_pyGiREMcIdzdfr4;ir{h2%D&q;r# zQVvC0*ff~W$}*7QZVHoh%kd7;lPhbRV!Cq~!;|sQ#V)iq2Kr5B8p@dOO}NEQexik? zR^1v~Oc!4+a`O;mKqApmuqT>N@0WQV7hP46?+48j^X+!hc}$gmT{fCfqXholnk+5jZa#0C z5#6qCZn4YAEOzJyc*}`kSDzDw7BGmoh^M$fO6iNbmiz=NnI8j4rexwSj$1tV#rf{S zRBYiamB$&6t?Q!<7vpf*u#l=jHrMoeH@gORU6H<58b^^Xm;iq)a7r@E{Q2Hd3-k$D zfvQ7XNPDkHOOsf_|8ht58ehxylaK$h*v2)kWsg=$RvHldB6g8A6vHXg(iL@Yf<(Dg z?_}YU{ppzEB~Z?}%Z60W#f3*bT>fm0s5#h^GL{ZpK?St*bg&_bwib2hDIOYgzU&*u?gl*nF{x9p)ys zauO`p2OWBTVjGDM6B}M9pf98BZ83F!IDNy4B|e_In&Le!W|iufmU+gwTtK zH^^6i(GrY%A-T)yzSz03aKf^&&d8|m5woLaL#tTdaHE|{R6BqNZY%W=#bE>pwOEfk zHx{mm#hn`u`0EIzjg&G-fmB7nOYuYJ3l~fQ3rX zGEj*~qFo^syODt+gK@D$2KLMw&`ae+;S5ETumM#`2|G}EM{O+D)KXOw(tS|!?>XM7 z-s_EZPuq5BMqxQK3iZ2w92E^m-iiwB5SN--goSimEJEa|7_1!nYsRiTPQIkaiJfOE z%~(y^@r!=!gM{x&5XbfRm2N)On;|Pnu(SPbBMW63h?U{j zA@VYkC9{Wx8))p|LiRBGI9}RJ!h5_MN`p?{7B-cR6w<8?(l!8%dpWA-NBmki&Ih*^ zZshOcTDXP3fA&Cmu5%ZXu9p77KKsK;bz29{sg#yghvUSry0%2u7+5t#eBsKvNcX*_ zV=(XwVn{cJNjmWvs9Fw8s@s9j^|-DBkp&UyFTJK?8fr-AS(LBcTlEpUDuNa8I=WNi5Rh zr0&cf`0iWVX>t0%&dCpa_uK>9lk{uNSM(>9)CiPEHD($bsCa|nw&osk<9|YjY+AMb zA?aKz32e=@eN9M;TbMh5!NNGa+}yUS?zKuyC1QVLm{h;yVKwJp3B2{q>LXtMvw1-L z-Z?;gs{zEH*#P4AZU@9K7$BZNV;E+;s=+D@V>JLfoA*bh!7Qk#Zz<3Ux42+lSuQdh6dy#u^5W8mY_QwH9Jr#mmtS;50ZslT# z*woa-71*?hy@PHK>hM9ggqsD?8X}$s^v0Hu!J+zX*}N(#(8i#GUPuAm7}9O3KiKa7 zSuoo1C~*LyUT~va`C;lqF3gAAFgJM$r55SeypRC_k28ef<~kqcI8-n7=`(-Y8kv_^$pK6g1^LXwN|)K8TMYHD9nZKo<}$M2R8Wc$Dw;6t#=ix+ zg^aA)TN3!XX1|(MqrEW78|Ams`I03Gp`BRimoLv8XAp|i2olZ7{ZzLPaZOJlVVZ$ADc}ls^CRwH( z%d2DSLCG>xslWSO9{>%~+iSh5TUnQikjAuzc* zgHEEJ4)mC4G3Wpr4WD7pU219bS1IiWyCr=m3$m=OWHAb_XVfWGPE0K07!U0Ra6 z@U*(sw|M8J*?Z3CEa^oftvcl_rf_9+6T!O`2NM8{?OD=OY&@Mg=?NK)Nl)Cp=u90&!!SH0 zJ+-rInjOfS3psYxZ+lBA45|ntLh(#BpqiGzj+%N#KwuB~%M1y7sr^}xp3UtdQbu5b z`kFJ<@CnhUizanQH~&>Tb*k43b+{xXlV{1q7}CFeKS>r+{mdm#>vQB6QFJMe&ROO| zf#K`NVI9SiyNe6)_4286_O{Le5B_d3daGyP@wjM`ZhJK^ zChf9sdlilbQ;qbftT0h|0HN!JVOV2}fdd~VU*@K;g&ON&YlO57-U zi9x{hcUly|!t%QBF80CX@UxF{=|r(v7ZhM%^M>+_i8mbj^r08ETIs$QwfZBM*sk}m z{85F9Mina3%^z){3KiQ`p<)_E|BRcA_C}X(^pnx9=+Ybgq}~JD`B`yDESm*wO2eys z9fWU{IfiZ$QbE55LtVL5cZ*Mjc*4?7D7eKgbe?;qhqSYduKHt~0@R}(hxdrxRvw7k zz~=+4{6`rf+ncLGGY%igxC(YarE`HblIC@3BPn*puroMuve5W@{6s5^V{un39#h<7 zyavyG^~=-izt&cYwCa4k2RQOtc#kJT`xY8_!oiafTK4-1`~5`zy^l9rrOSJmySnz* z1x5%5-K)?>C>^S>4-C;==IV`p!#y1*Uv$+=NhYcOqX$sWsdN#1Bi^oN*1}nE?bnM# z{^dS8AqJm&Vc2goT0CRex9YpDc}ehV`083nkO&`-g1i?tXtXH}wj?#&I6SlN;1}Mh z88lkp9ApN7g`_Oyg>#D-ccyOjClZ1ysdIQJ{jqx;@KdG3UNIV2XUT(F8I{?#YXtOQ z>=#h6+x&SZe=JnLFw4Icc$mts3$s|x0w>J!Zx@tl8!m=Id6s{hkD41I$n(G{nwZMJ zm4)26Ch`f`=MZ?&4g-Ws2UK4mM@zy04NdcwiJr zVX3vJZxQ>PZ09{i?kHT4qupUWo-uTBcMNg@E|G}xHPJ+{04Qy0s2)aoiEL#96tt$# zh<(CUS=Cq+ZExMimf?yv8Olh4SXp(A5(gBOHryrEuX=3s72_F%Ocfbei4hK7q z37>_|r(GZ3Hypf?S5dIR;aZ9;ty!t-mFUs~e?x_>&*CmCvHJ@&tIr{3-CFp}l(y%@ zJ3n^l!thDHy9>iTKE3YuTA7G>N{_F(F!%rF{UOfv>5G1Iak1!??QW;nzxL_X zuSa$I0{GFotymA7y6vt>sahM_3cbp}Q9DYA(C+Zr<-MyG)Skro)+OydIMxc}+)~fotHI zi}Xh>oyTN3yu5~9VO{d67jU~1y4^_=itwu3C7*%mW;U6gh`6z6ggKu{7wzhi9_$3sjC6n)I>n)o*#W>unM-{8?^WF>|^;NcbbYXua zp(a3LqbeoQzPBz2*|$(JNmtii zE7P$XB{w#XChAAOj>-eG#-8R)m^ItqV_TEovHksT^Cva=uKe~CyKY@Tqt&zjY5Qd# zIr*}C@?{_4vcKm;zU(cgp*6jAOE~Fiw+HL1TVz#q((i6R|3sKnugop-hy2GwKehW% z2u80yIYWKo;?A}3e2{QSv;G-%SXLm&vJPZB+->^*-9EbNWfUHp9^KSZFG2p|UAvbu zWneYe!a56a*SUo=o1oX&1ni~H7Xt6os*UhzU9#$)PW-Y*PGjFHeb^3;6OSZmr+B#t z9XP9s<)lXP$WoHIKMtrz+KHM2gV7Fihz!sZK*{&U;rdK+lk?tg3{mh(7k5N51V;nC z#Yp8KvxZ4=n20Kd)-!1Ay}{7y?U^%E+YbU;Y`=c&8B#~)k=YoT6SFa90J&f5b7&J0 zu#!Ob33m z69;_$$Qw3CxvXmyxVmJdk%B~>9lB1LX)U@?7Jxc)EqsKIY@x4EYlY8r?m_Ed-NUgB zWgYV)KJEd@)FN%#3hM?J&x);dw>KA@i+f`0eiz>8@8>5ao6k)1)?)Wen4|v_r;!z<0khb~ zd~P>UU*5~~YNd9r&rk02C!Oc}%w{nq48PM**C9egR-Dq#5P-v`&_Gv-6ea^>I}^r<>r-~n zRg_GOWp%A52Ytn&n1qv5Ely)}6}r-if)HET?`n>g7x(|r=OBn%b$jX}1~~5SIo?t3 zGYd#zC=*A5%!rWnxVP*eVwB@6CL&eU_|(CmGC@$8zH!(Gz=oo&r?}<&r|gwg-zbBz zZ-WXpi(=Z6T5z-Hyb40KmRG?nDXQZwi>%S~SSVO$fbE1`2l=s{Gim=L|lzw<@1YmC`Emjoz8{eP6Nt|cP-SoHf3Ne``se+NpZS?9|M?Z0KmF4`UDrST%xC=QkDPnnBlOQBf9R=Cx#ZCItgVIf zTkTG_Kfo_p?cB9{Y0ut$`@icQK0fL(@9>X(oqw&r)xXBS(!avL)Sq?MS!bQeKWDtm zzs#Tg5`Ww;9QVKIU+i!8H~Dw?U+_QgU*M1Vqt{%0)zALl|JU>X^cARpZ=3) z|I|%3pcn^rF&=RK)67e#kcvm8%507 zoF2UGr1Q80ZZU2`x(atDDdXbdoqq7vlTOYLHu1jg`Hkmn)E?Z?Tr}Wqc@F^hziO=! zy2du2k9!LqDoM#>%qUH^d(gdHaY=!y%C+!u0F&(-NLnYj z^Iv>wkuBj8F5kh+DJ`8oEp1_CdDI`gtN|E3)4B`t`esu!o7PvaP`H7ul^GzgYx{yg zJ%qtqzQxPO!F+UXaJwZNZF@W=yp*{-RiIHU^LvE+5+QZbfi;EATjvIE&Nr*Y7|^YI zxjofH`p>BwR^%JXwNur@@@O1Rolj&h8eSQzM%0_b>CQZS9cGChl6Fl*Qdf}ip=uQ6 zdK|`Gne!SVxVTGXJaJcbhabMx>r^;@*^9~EIOO4KqEuq@+Deve>slEk~N<`|H*eK4739W=~&xH%U); z9__k3Unw`bi<%}n05jC!S56uMuM1EEMnyOpPgdPtVFQICWy27CUr7HKKz@KcTb9a( z7NR@Jw9yh9#IQ-R*~^A+wA!$$#d^^s`ya>}Agk0SaY)r>_Opast02G%f}vGzntl+l zrpSn=Fp??^^apQjq*<1|!rrA5RJ8S~NArg10yw#APc?Vv)(~woy8zF4PYg-(6I8a8 zxX-brWH0)3ylWLxR4zz57DEsygqPftmeF#2^~x5kFK?0w){Ef|0XklXC6>oeV>Wob zjnuRCCV4uHH#Jnu;a1wERhRZ}BJjDC!c|#&Eheo97eHF#S5387H(k}uH)F`>TckBi z#Zr1fnqell^KoY5iU`rA_rr;Q!WPBgU**jsjjtfl94S5~hl#xl>l^CWIM^n))O&LV zckaO0;=I|@z|hF_&Q|yI&FW%mb%&|dU1;V^n`AbSuioj0Op_MV2U67WWVC=yzDbUL ze}J1YZ_KBBd%csylNxc-3Ne$UdR0pucD{Fp77cO!1c{`=@5Ihz3QyA&eS9Ez!xZO;gsck8(B3cw|dmG ze1)0HGV=DtCdSDP)SFbVY7Kr_n9+GNPOgqQCrKl};}()PPnEgy32yMZW)f;fWA3cX zKfAd>-r3>`{&TLGF3jlT3LD#Nm4n$w^H>K|c`Fc!fN~ODdRbfgO8VT_n^|Q&^VSvqVJKamDIaH)Di3@S@aJ`2@YqxsUZ5}NGe%CcY*11)1 zf+nnrduWVGQjn@9?lCzhhtU>_ep&?W+B}%kb{Z~1XI8PL%`siGrB?YTFV`w*(Vb-n zP(v#<(V$?PW7U89r zVw4XV>EX-S9ajO}9E}iuRIhH0LR=(Sh{K{nDmVz)Y}Dl!4`1eGX^q0*rCEHT(B8s) zvVHiZm9*hlNo)agD+wQf8{9H$%#bmdN&}m>Hc`n2r8#->3ZvOf)RP{?M4di}G2#gT zwsPcEiynod_n`WBD`2l&{ph*U8{6vNt;Fe1s&4VGDWVYXo}F^v&^Vu3#~BuVOu*0nIjoIUQWWxdFp| z$i-dNUzIcsT}%*JB*5@xJ}r$(tpMnSXHOa6TA#l$wl~F4U)~f^a&Gi!Brp%MTB|Ca zI(oGHm^NwsxrKXgbHu!tp1O>W)@7J0K0caqF=z~WxYcER`AHAY`Ul&ZRMNT}zAA62 z5-r`htNLPzgYsp5)+K|w(#US#ymsoCgj{c{-eWgqo!2X|#h&A&ZXqlQ~?XCaw%i2AKdya9NnMZkP7wKsf)1lMBvumimMAu{0N4uMYDxE>25 z0Tg`6Rew`9n`rd( zx;Zy0DDOQERrQ5(w3N3SdMPf!*yi`eB@xV6XZk+F+t1rF?K0!-0l$TrH~dAfoob1} zXDnZ=$<{LkRv_7SDzQ}n!t|t(59%S-<8j%U-O-{ppwSM!Q(b`HB5yau-e>_Nr=g}Q zp}4BpqjNEgH19ne?n_KlHpsX*u{ldj(p6t5dz_39+%^ewbk*QI=@lzHk*j8VP{M6lAhZ zt5?NfqGg|C%k211_T?my5X)BN-@-H0G9|PXS4`6J7kARDU)<6BOXnS4o_vpU^~y&0 zA&o;;cINgXGy8whhFyXwZ(Q>yS=G}QTQ8co1#BuFka-Dx%EqbtY+MGU|9uO+NDHf< zxkI*3jjhBeD6LgKjnm&9H`yO6UMJ|7-wQ=<4*wkNS&XngEh{Sx`IIyC}MGv5C z>q?@rRPuu~TY-_|l) zu}`mmaR+%0eo38NCBJzkLM$83ieVUGV7jfLr`x=;FU4SMc%2`-P+RzHQBPNh9orTJ zX&Tm@dPV0KWwxxCe>ZHi!(TYP&cw~OtFqF(DP~9Y8uZCu(B%OXCWRo z$})1#`u)FsZr$px24Q=xtXU&1sjBWh&;9uJx4-ZI{r}>^Xs?+*1bm9mfTtjur*wG} zSm_Hiy*gYN`~cg^MR4)P1Ph77B*EEKr%JSbv3^e}>JbfGiVlXkSmfkh^+Q?z41W4v z2ItU+MSOx|V}G3@XTI;Y+%&g__(5O}5Dd{S%xOZpqFNI@JKw5+>f>%+>LRF8TXP$Yd z)YDr-C;qYWO(3T~l>uID!&gS$Y2hoC13eAGaprraC))6q@-9V_#gBdgQ28s16Sf$C zIJ=>GN8*6Iy%D9eLTYawqU4iBf1`xUEm{wA|6U8<-?c7XDd?Y#K4@u;M8G1xw2wP2 z+K)XdWcRbRlv!ds$xZNa~3 zN!H*_hLSa`fRjWNwCm!-L6Z?VP6lA;`O;>LMHOaO*)Te8XIe1f98J-4ceh)GMQ4K> zZWqyw{>wF#GYN*VSZXf-J>2zCYQsA$$Ndk6)*&UB-r-6Ej^3c`)iZ;{)OAMABqOv6mtF z0!YHDTLxDD=smMzxDqJ6T5mFyewAVGGXoBs{fGLr z7d^8K#l-~LyTLO%!i&PnGw;uy8H}BCv;RHwE7>#e&%)kkDv%^py0s;3LX+-&W4eye3{dnl87NiQN{5`6Ln37C>m9iyS#}cn{h>gK7HgF3e^vikvFbNk6~_6vQ3wDb zp^$)9T%Ng{Hi}Cggsx5{i)0Xb=ku>?mx&7-1;ErI9S`kAGUv#m-wr> zutIjQusNu>X*U_;yHySKPN-5TZkTg7&r< zLL&`5JT^kaYJ69!)|_8&*rX7ZU|TyZA=MLwcF_)e#^>F-eyUI{iwf+=DzG2RUwgY16Pwc1#Vo@# z?TV|BH^U<{yiP1`sD3bjevuA!CS1{Rtvgq=*BksW5I|-120z5pcCN@nGa_Uh?zuJ( z$quV^2{&IvJEe})xRY`ATHy+k@@#V@{AH?*5J;TPnxgt4!lr2}$%<(s@s=H)cCh>1 zji`-10Xo{>b@#P-;At}&M=?Qq=E1+O-*-mVtoNapK!2R`=O2FH@o9$28=Hnm&tk*_$M;&bQ9PiBKV(n6t~ z!4IfZY{XMh<2CGtU9mGczXVy9ahH)j2{d(Y8RqGS&b9u5BFtm`PvP`2BvEc)Ag%g$ zi7>bx7MAF+&93TJHclx@sm+RZQ$430=9e{_-11l_!jUlwkwo#am_m&fSxS~;WifNk zdFNbCiePWZwQ#UN&965LX>r-R?LrrJJhNl_MKBo|n58 zEmBa%-aeUtlvAb00wyT8^5j_)G<8sAiDpbt7g1%Y$6?RLKJ1vcPONR@-=Q|{@)g6L zDuw;dS8pA>?fe-ppxVF*xAa~Hrm3-vo+2Y=02mStJk3lMf4EvVZia!jZ3AWd8^6u^E>}oP&7fVy zweo}z@rio;N&B!IZyPv?5Xl$_I3b=bc@ zPJtE&J=$EyAMQ;T>lp6?ga&BAMrFC`mqLHoH5a;Tm4<9k zcczj)z|?|UN?w##Tr$ju-vMC(+f?ztbkdWAZvCXiNu+oq+ITy4joGt=a3{}~OLjdq zo`A`ZPZwyJllHBNHhn*x{zx%tvh5bZ9FqStMItzx-}aDGEQe@m!Vfz$^!u&I0nO0| zzBb$|`WebzJd{#V;2&0_Mm5Au9~vDkKol;;0S`!LZL{8RySVF z6jfl$p9r0JucbM?MeLL7yskufADsO38V`PEDZQ=3)%){ zLE)^g__xB=*DStZ{zHwSq1n~DURs|maLnoV&}EY9Efyrukss9ig${o50uz!( z(&=OWv~RLlem`SBJAKH%^U%)Y!x1b!Sc;Q*v%oibTGm>!T*^)4l;Lyy~BZnna;2{u%PN;vSKuUx+(F1Q@sGibIrLK5Z2A>N6+BWAGE)^UWBL8q)H0OYb*%!x zTe?>4`bk8r(<<@IcB<62B^x#hofl{C_o+L5>P{|6cZRGg;D)d+EpaYNA{$MyoxziQ zz-E_vx)eLS55oGf@AN}Y_FD=aHfEn)cXIGkf)9QvpV-4WHLY4Hhca8iU-8@R)V&nn z4}Gn})>X_xqg}1PxY_03H+bv$UL+p65-eH#O_Z6l!R=r5pfm0$2d!!FXnA5G#Vc52 zj|=&R0z|BsWMk9*FkrmzHvpp6LayLu&Pe1E>?L*NV8Vo~F}DoUnb-=uydCXm{9bM4 zObTzrTsj73vlaHtrc!vBr0|9dwBR_FZy8dMniG=g#tc?xu>e&O@>D5Vj@~ltMY)}M zWbv7P+<;zzsGHap5$Niz;M23o4a>n#s1xX74MKMBnDak08RA!I{L`l+fX~87lD!l(B^ln%V*+kJHsZ-JOlx+C0cmuQoK6GEfEpN6N zZ=`mg&BJ_r^xqb%{z-n5X6k41*dG>2G<80HCZ{52@O}M&y_6g9y%xlf6p?|I9$FE6&O(y!VNWB`Y_}^^a)E|&bFS;hx39;!qpSY9zA|vb%Mg0m@4nMk zqow9K`gVywEgbcipZd$QZ@K%>I|^b_F~da>xd!eMdwv2`BH6%C>8&{Aw&BL9qq`Am zE_QgsUg$jI@;49na`Mh$ZSi7O?@Qya|0B5O4zAUI2GO(J)$15Prnd*QxYoTc!Fxkc zkQ%AvVUy1-MV_A@5#Fg;1hWQ>zMGIeW!F-9G80D*x!WqG{%R(f7T=}VIVxWoCbMIc zZ*(aHw(stH9wg&7T$JR4xq`6hz+`I2YQW!Wu6Swymdf#P8UJWBZ1Be}9qx!9{ycd_ zLY!lN4I!W-#fd{3_x;yT=Z74AP4As|^3{m{g~{m?&7QEwzJJ5=gXHu=v1887Jcqbl z+ZNYDXT1~H9+LaYt%eJ{PE1zgt$z3bmEAO^w-_o9Q+b7ght@_>KSS8-I;s&xT zG>wGNi&TH%}pjzSzOK=_Pcq0iW|mcgWP`Ciwc<+GL?(5 zv%cJveCqnc=fe=P;x37QiA}o1u6V2|;vhVKS+}d#$z(HqkM4JL^ri9V-ZNMKdD&!D z7OKO^5i+jFe8nLsmm^p&;W*wQyMbB&j9eD8W6{+=q8h~1&6E^WtrqZ8d1FExbi$i& zF$xmY1G!NBn4TwAjUxiZ*UvNnTS-OOO7x6i=E;`aiL=%?Rnj)Z3z2oJ} z4HPP;3mj#pSbJm#!jJS7HiEhic7WPI?{#nFH1g&OStnp*dYpmdXH-Uxi8gmF&wSMD%-94ALKe z`C$&JIYd()UCsEz>uFnSN3u1$H2&!y&IO;Y6z@`EL35i_1NHz&V-f$|Ng-*Pp5@u~ z-A>vNN=s#?fT#faZL4S3-$-HAj!7>qsc#@7moa^)K-5NL=|zQ^a|3 zlcAa2t-lPXy=R+w^%}(2S`jU?C5C;GLbIvEfxw1VFY#B0>KZ?~OF}}CkbOyTu!PNU zfFn>aBnc*M@2RSUI&2%=)>lgtFdV31S@3N;Z&%3boVc1E%fSedMN*OT_0uJQ$^Q=A zgkS^@ofPk%QC@(ciMgc;e%KqXI5?D~6?(h&HMjFtZ)z$D99BF^&C<2EskAvRF+b7O z{|wK-gi{JlTbfkAFm{=1W)V|EuDBh=2`N^W3}eb>HFkBJo)TWULGPZ@kvz2>XD!Y4ZAhzbmmGB_a4V6uVQ~QYN${F_Hc5ws1*tFt7~PpMyn6 zsucP`(zE`8G7V2S$sSAYFtbO$=f*PMEBL2 zIbtdp7+mz|neC)3udr(uLbt&G=Q3%5*21uee2qlPsZZ>W9f# zNe|**o@BSCZ&(DUMbWzAii7jH69$c$$}^6XhGZ`9@1rD%pD zNBltC#R#?&8~f?U%J}h5l^g~o;S;1*e*5DpnpbK?!Hu`D{k6(=iz5}A6a2FoyB^c$ z9N4U1aThgyVa{b)mN{0H6^94o)kFu`QYgrnbmjXj7h84Zb5-1OQ%N4Fnf??>X83VA zH|!{!JZIFxF)Dfe@e)Tlx%2i=a()$3)FUm#;#FZGmN&}L8{oc|@vtex@ zKcE<%n%C_}MaznArFF;%Q1ogU}7R@cgkXq1jNo9!ox zCdEfiiqipS68h>jS=@XbF~@W!ql8lTRJE7b0D6IH{>_Bl_p+ImjQm4|B{1R3K2^j_ zxj0;JRUOD#BM?jEPo}|g{>88s#d9GYY#)ZhUI?mjChA(-GUP)|-*iFzldXXPItvT_ zgfzl!BQLfb`BS^%9HUa3`JL=Wuph(9Msgu4K2w{)WZGP3$8!HWOSYuao884oPe)1_ zTNWcHPHx{kmT_{QgC5HG|7WD~SpP4oILSj57WwpyzPs?j{icZJ4HTbgi?dUkzaF;; z`EtK04!3H6K2(w|Gz3FA>;@@)nMWDXJz2mL&6gME-0ipDetUL%%6HJ(^J2~`V%9%R zFmooIbuRS7kmoJbHKD$#QtOBLYy1O+PS3V*a*|e^IZd>jLQ9Qx&O?RpRa;m&B`KhD z+Rd?;R^DXi-Kr1&eo6x;EW(hOxFH7(2VD<~uzMOM+r6D<;n_P+kz!jGo^A4E!gw-< ziF_*MFoiGdTyfvB-+h?xidALjh3Y8S>Sd|OIiXbui$1DG7tpb;e=6%ka|lzU2;(GT zuS$Y!W>39Ukz+_11p5GSblFyVfX?z@)t}kSyr0qXF!%?e6@W3HhOG!^H9T z_>Q?d_Kz2*eG>g}yATvG^=@`&3d#!Ss zo~fmKpD228v3fLQ!R7Y5=Y*aC*m}Y5ce}76fZKGdonbEi{C^TP#hl!AR-FU^Xe%Ro zV=rFV-V`3Et9SIl`GB~56A;(?nt-_c|2u%V-noFd{ObqAwM{w`_9yBH7On>tu17$0 zcN8m9u}mBSSX>V~o*l0K&I~M$S}|NYTp&VvcqoLEE-ch#1$dI`V)P(3LxNIM?~%=c z{fI$IxF;)JZ^b>wl=5OtL7T-7FEFtWWr-nJDR8r3hca0(8;OiAJ@O3v4ljfptXwy! zDs{wAB(I1gXj7sBnVS5@-`TWI@@}aGGZ2;ibK=$2_h8EoWp~2ibnNt(^|Bd zb8fA^n;PNQRo|J82Tu2$7XKa^R9egE*NNAo#(sP|Y!HHD5YRkyf=holM0J#8sm4c@ zq*Oc~ae)P6w&Uw=|&&?x$7mrMp??6xe?fpQFnH#5KAgdr`8J1W) z!;P)JmG&f_qOJAJvsL86ikIh=kN>smnMLGa+h1B|n|fwtX4GS_zid_-w^7=brl@6C zVzj5HW(>?ewF-@A3-8qlX@G2O&>QF6Wbobnna2O>rP=MP42(G`y-H@K`7ko^DydBG zoL8Z8EsWk!eP=4&ZigHDk}LA3b96?&lV3NATdQy5TVn)yP4yk^r*9m52iwC2)VtxR z6~o*p+M4f;f+HvZAC6cX4C1KEhtctC^QUvZ7trybJ~+tmId_nYstJE~wrB`+s%pep zBCRkjqLHm(N046kqkr-DI{fcyV?xu1!+Z*qn-a}%?`(!%2FuZs%dpJNP|JLqC7#?? z4wi4zgk|(=9R!2M>dIJ4(?Qz>%vqVx9&8W49h+m*LoWck8))4lGIp(fD+z1YGEsV) z0+G?q_wix_t>2U24;xykGhwGaMlL;m(oLF$q%>T8ml%^4zS74=LqFeCn`BYi3%ABE z-Bs*MRXiC$Q7DLZU8_E)ec1p(bh2>tnK(W8ZRe`rXP>7Vxge?rvHG3ERi%Q-*n5J1VK_dxECtLx+sBnD4=v9aw_BaEu;wN*9j}< z2g3l_d3h=Wd5XNu02Clk0j9;`KkGl#2wZgfBc+cji8=Ieg zI-4GgyhFO*qnn>9bYoBPuFX&H-8}Y5H@(7V-1G|9O+VZ>{cO88JL4zKAgG2%ZvzpB zin*@J4uV=9`|E~14IHRX9L|>5SD(m+rLpinK_FCmRi5cOCjJC^csMea(XkngBXrG^ zI*YaO$#mA3i)iExR{iVA603%uj?+wl#c(1PL-nn@B7gI>`K_V}GP0U%)xU}TW~h`M z&ny9)uz=r2Y~aD~0nGXdS*XZoK@Us3wMcLMEo~ zuK72YhdrV)373msf{<+o;H*-kohCPog@Og0kP#;7hz_hDa@PIy1$dibmR?@duXk0p zBrJmYW$gD(OW8YqS!%^OYE=*|o%M?Lf8^DrScZ<>5-oBo-Ty?8l*}=j-5KkT7w5>2t5nx?@2j@^SekyHR|Z-GvU5t}t9W+K(X&Ost#{f~c&!Hb*;_ zNJb*MKu?>_Yf{8!fwt0M@ithxX|N!gJ?u*ENrS~i!-Yg0^g|C&I{UxgRoO0q#CM^$ zyh!#9*rN3l7(?O1pn6XBb|fzdauQbU&ZLBmwqO6souyeZ0DV3UHxLJ6wlH(N|3;SOxXrhD{6ByqdKQld)Hqm`!Q3f^Qz3MAQS#xv|_IpD*$-= z*Y&Nkqh)2=w!YZHC$G0m%WTwz(^cV>sLs;XVbdeXFj;Rj)YOdEPh-7q2IF#7mXL-P z7L$&Z{G>#a#&~2*2&X7B397X)v>RFenB7QhS9WxVJyQ3IJPh}QT@j=LW?2}`PY}1x z_PgJwn&C;@UFlNgXG+`U-OtZ_Rn-of6e?-ZuUBL5@8f&5{;L0IsbF@ywT$^4lDv4h zz8lFIAZU=opW=>g)t^j1xdR0YP(muB8PSz54JLrm>IrOC|i~;vm0iQupImKG5YqYuY578sDV@ zL7%@X9VkEcw7rwWO8@jJlJ}IYV4=6W^2Q@w{v*j#POp42UD@%QJ%Yuuo+!6Kj0D-f z4?T0*=_Dm4l)8$d!3${`;ZK{l5>0R`YU$tl!rZEVh&H~roJ0lQ$!j)#>`Iu!@ z$Ik*mRsVeI!iP$<3<@hC0knx}-XX8J+secdXN^w5V^BfGwmN^qy+NR{f z-<5~?L)o?d9@;uv^&4$l|Ag9lLv7vHJL2!?nA2&#=vD47CNul{`d^g0SQzEJH~AC)-`~RlUXZs!*b|O zNQnp0IMc)UMQW7Kv>ow_U!i$EqmBrAuajj+_cPqVg>MM!z&SVUryBxqNkhtR_(Xfd zr{BS>ctSVS{qDoU=98N$K+jTY)Sw0q&eOKziRI-9p4F{RmlwUxbbyWh6J?5l#&>C9 z2vi`)lj-+YOjnqpx8;R(L%UwLWuedlv$^KTG`DL`me!mjEx2T6sqnLlg4b2fXdS^j z+;ZI0ALu4E`*hqrk97G@>l7!_{ZHBB_)frIxqb-_hiniz@ZX@f%*Nzcu%xZj8O zk5zDf>_4goCE*X!yp+4Cd9qa*$IsyH8O2ZF*eCZ|ez3L)zJ=e@D?0A+^N-8>IFI)V zl7+O%{WuVRGjDDAdZrx=*+b%KOyccoVM+iuP|yf}ogrJ_@3VtrJ?mI03?&1!aUKVh z4!|_qA66=aVTdZ(-?NR5HtZF9Wf=0Zq|2?+9J^B^!j%%h-!NytQ?}(VVX^&93s(C) zGC3-Q=@E|V>G@_5((ctM375PPc(J!lGd!a|LhslnVb z5T}FNw(pju*>1^cZ`z;d7;v9XBTh#4a|4kPNe4~N%uaA2>Fzrxauar@yXW=CHUY18 z`)^?go{#VuAf!Xw)W(IJh-v+q;lF7dPZS9bBGK#o#<>@Nt>rpZy;syfDuSZd@Wz5! zV@0?Hu59}Rnblqm>fZ%YA~fLlD^x{w)};#I)*s1LwXZMr66e*WYksmWP&|fPpbYYs zd(epNyT<2>(zxY_W>K~&#^Y_2Z0)%+tr_$~R z^p*8#w%>gaXUVBXPCrk>Hju^N!IwK#Ah&l<*~nI$E`ncf3arYb#Zhzazf2J6!`$%K z|H<5{f0W;Z>wm;gG5wEtF#W*%tqcCVFq9>-EkJFQa(<10!VuQGvHF(5YdN&D;@%Xq z$$s}Hdo*K-_)Y_dKM8V&K5EDcs(TC>&dWRrwsh=2n-3AQQhB>NxU=d%OM#SHOo$A< zWPJ}JkRA3uPYNc)VKhU6n(H~%_PaOPnsN%19}q`FDZM(mC#O(^`=yW4+NXS5+J36Q zw&|^2J9rHP`P^j1xj703#xwKHpr3U10b(A z1MTV$!cL$!>)V@g>Gr}#QnmSJedeBCKE0V<)jr`4ycvk@G!1#Xf?F7ph30b>A8JO5 zvmd`UA9Fa{scEW2Q8lH>(m*N>SFae{dTzq3B7VCds+8HmSS5_*LfSqos4kp&^-<-U z9%nUzbS3eEs3`B&^pHi9%?{T=v)DFxf`S7tznY4fR2+0arsJ7EFIN41QzLTHZz&Es zDMMl&k)jI>M^0HDv^S8s#^Ao2uX~ z=FO%{vj;nLgzPIREa`p&`D31tCd=cKK*`twFYk9<+FOuf7T+l(-*)ZP?u*iV=@_-j zZ2_8}7NB_*pxHW2T|RfKA1bZpStU}`MFq^Os>AcKKbesUQb>%l0iooz=5JU*_l8!s ztp0CASr~BHu7O6bsq9U!M_umB^@>%0G7H6pu2*n9Ys?}rKD0D(k+w0B?o*9FK)udP zV-{N)6U`qK4!1U@ZyVDpk<`4d{(z@?rGD6lqGHeP6n!jp|9i_x{eHez#DQWG+Ngfm zi_2QYGc;MD*3=iei0==3Y4=GP&08Aa_{!5zKAgcrm4#fDbentFEap~z2gZK7b7uC1 zLUnlBFH?VK1rcn>is0*C0Q!lG`Yd|Zt7mCMf}QkiZkbp=-nM6lJqpQ%lBq&ZnrKBO z0AA=Fu3nL>?$iXN**{S1T_I8BuX?StAVt2X1j<0-!5cs=aiEyrt* z_}sxO`k}J4hdm@Kq4`M%viGn;0|52$%A1z=yZ1^icMC_1yedPI0SHK;X3+Q=0!9}E zTctO)N{K6N+Om5Yq3?dz+^WB4YRfjZ+OiF^Wjq$+s`c(3WkGkwD4@uU-W}(Vb%M?? zLd+RZe(;G1vkNuZk!A4h7DM7%CpR*##xOwpDA}R<7qL#SV(>mx3>B}TrocP^0z?+a zbM-GVv{kUJw>wo>f}gR+)z*xDGFxAc?&Z~y!Oh$G8!Ik9T)kX~!Yna>zMKv^k5;v7 z)TOFP5pC%!`WR@HSGTJ<$FOzfQHDJPGM#Exg1SSTMYlqdupZm<)IaH1+{R;Z)HrTZkAqZnjD*Q89Hrx~w zK3hPQb!{SlsGijSrUcG-MP5tcmCgFt-)b3tr8s>o>W9^MbVLhhqCWS~J2o~pvbzsu zuU3p*J@Pab>G2<*TlHV!x1avE<*I)wi~neDSj2xc7ysEmCh+=HrmFB;l2|UqTm8Y2 z^DRh=wlf~`wqi)a8ol}_^`w2#CR--D13zun_%j;M#MYRBLbNJ1)K((eH1$UfgqrPj zaIWY}L%;gz1SmGAso?xtqiu_6Q&l~=b*{osq41<;0&s!YA=X&p)?FVd9KGIrULX5M zv~MyZ;&gyd2axXH@2;N);V6q7G+p60A{y}BxRF6F(mf4lE_Whu1#Z)QYZx7xzjrvY zI2Xk7fSH=?^l%S_a@Y+eaty1RD4MB+y2kH=jLQdq8s(bfueIDM+{|D^FAx6oZIug` zbEHZ=&nRB3D1BMKpKZ5L3j1`5R%~A6YGEu0HSR$OMqWYy-i9?>^&L6AY3YT_630lf zno^OkP@(cxj{SAbBFl1a;n3>za8FpYw;vtK-q|2|g42|T5f4~2m-7;@UFHiA2)8Xa zqx$XAeG<3IE4gom?)Hh0EwkwBk#utgJDI0yLNuvuO1A@3*KWlE>7ax z*Cz>LOzupbEdV2Io}P4fGNZ19S+nn5F=A^}$msLawyLLy~*L(QhbX4A`cyd|w>y(PSuHWu6z*Z9G; zxuRQm<|%?j$m&4$VJ*Or2!fi(;*V$REE!2Vakok%KfoXn;Akln)lHiFH=IBB*-wt@ zM{s%b3w+yT@oyycR0xgIM;B#cP10?%UGbaOF3L=r=XqFSb;w3D2*jMs#OEto>@jj z0Z)kBOF^tBya*UJNoI7w&@kq>GDRsVp;?bh(()0oFH2-Yp=u4Y`+uYWbn5I z1|Iv(EGX;Oq8 zB5BwW$CLU)42HRW*ipZq9=*XII(&C}-Gum+Lh z;+c7nJK9eULVl>e7BqIqAylz*({M4m;UzgSDYxqHE5cQ3 zBjzF%wZQ^=Q0{;h?f1p{`{TabC(W{WV^os*H_+c-e_s#s3-vRLmil=nPdx4CVuoF_ zXxTOS@LYCHJ_|*BPeLJie$#>EP`C6m*n~9lw6VZ#OFgaa*&t-r)4-5YPhS?Yu#7*E z)xSc2i3m$L3m@z6qa#Qh3g~Bqa+s-{cuqC1Z3GbU#dUhZLOpOwBpBo+jJ8YCV1n#iaf< zKAvte+RDW3cX7GhyHIY@l0DoqTLU9{f9pE>OT~=LH4j8ib@IgcNKjYQld@u4MI`~8UbYv+d3+yp&)L|At3dKtLtzr_PgZk z><&IY@S!ejq{7yfv2%34qs+*bP#tTiyXvLHg+}$SmxGt;W?PuzQ^nCaD4`VEZ}r@_ zTQlLTHs0C6OYwWDqb1mZr<7A1&2A84Iq@_Tj$@@e1>uEY$^ui2{nafKYogJ9nLJXt z%e@sRl!*mi__5zx5~pCq}J7fLb6!8CZj&9ffUxMI(K%Ix~T$(}yir zu1$p1P7U-DOaq2p5H={}FV5x;jqf`2xPyvQ%zA?cC`;MWaB7#;2f8f|GC^`ly99dn zK(>=Tb_~z}Bk1R?c~K_gE=GaeHgBKm4NuNb&C3&_911&XDlZ5q9Q_8dhm@ydPp#4k zcH>INT0QAhNQOeAmW;ALxW;Tzs@&V`V7%x0m&)qJG?mOlP@n80(M=5wrKE3e%W%-3 z>0FV&_4u{{)ea~Fm9`oWUgIlD8wBPFzDgyR2_I7m3 zj37Fs8b+r#Y6=C;LvckOuFdx;2RhQh^~0D~PZWTy-fCDPM4@PpLLM*PvL&^r!1;SI zmiEP%kKL$^RxjqOKg}Y%=8S9Lr~5*_<6<{>vAwYA#C6heN~NXkUrMp&#a-cBT#+)P{FXp`e&Goi^*gVrZs*U56=#p5<)j8h~G}XAx5;{kz(<2{*Jhj zp=c8=UKISp^+zdqnDnjS>aC_AI68FJyB9$`ti3Fy-sfDl7e=~j zZ-QN2OwsT%EG9eUC(RD{=dfhWWiv_HPvXOz{z9?pKf!N8c03^{#}wW1gvI7om1a8D zR99sSio`6!lavR40}BWtkAnzkY4iACWD8F~hqy*ZWk#`8&ZOTw8Ye<b3GCC&0XG!TsD)?iZV1+V%;3wdTbf{7xwW ztGG@IZ``YBVexEo&%;V>j*^`^oX*l#iMCG|&Qqdf4x&&;x@5XNo0m??6zQ$J`)RKy zsm<5^nxOSnm*C@MnncbcfO*$WkA;AR$@@E_Il7?W<-#`E@y!-(bEp?+QDw)u!+k}w zP@Ldm`F|GqrJi1Jj+8{Qvqcr4_w6T>Y)VOGv$Lh6fdVP(V<=%=x~%!81kC(=I=i+XZ1y zT6T8X3R6zA%h~4g(?>DU!kCiR6i{YU#)f>dSYAbu2GL9yn0^M9y_3**AQ)HuN7?*z zZTb_F6b529aN4%6Td~k|KDMg6xgnV+?Q_PWO*VXjS9r6sfaaEcTqvq*gt4`C>#}%} z-XIq1%a$S+@i%Y3<6_sJSr!N1xVhm>6ZE|NQn|$oE|cd^fnvh-v~E|s0E#fpfV&nQ zn^ry9hDj(UZJ647Vtq*q5dirJySi)5?ifGsh`6`Xv-2th!5uk;md_?(1Pa39%_^g3 zn@ zToI@+&r&c{+04UUkYJ(JYL?s0$V;>e+?O^;v4xf> zFiLqN)zMnJG<(z>A)`V=H0$Ybd1_Xh(MfjeLtsRed79qgo9;f89ZwyBT1glQX+{Tr zET#PKg)0i2Bb1{){wH@+N*Sp_Br{3S(3LgJZBv>nDgk^TB`g`AS*#!@E(R~gynLOx z&-2EnYKiK`hDM#5ba0go2}`<}scCFCqLRmvmF1jKjv-)yNGh4Q1pB>uf(FDh(v%2U z`W+@GwzKiKUAU2Vuh9}RrYKlmtW1Zp zCXhH5;NoOK8fiD11*5I%utzC|{lS%zJKD=TCnw}_T#tKxWNsC{u;hfC+3JLxF(>3j zYNz_VX1yw#yQR8v+s(kN>E@9nF@=ZU6r_;FXnOW+j(MD?`;}m3_oHQ)9of|U?IvAp zkffHHLl!ivf3mDrv=JCL#W|ZcSOQfZPqFJiH^;ZWJL|JVA=&|a-8euMNX$ZURW=v1 z3A;BQ155OPKw$&rpF&puhA?{j-KF+;%*XU-U_`~;Pu)$DW`zaaer+=>+3J=IOR5a% zjHelvOqJvy&8Z4ehDAjovl*6|n5l)}Q{*({7s#+|sjVq03J`@i5R4Y!7Wy;^|`Qq?%gWglbq zEt5BApImYN7~;y8TWsfY&S2GVrKo3;k67oqCI?TN1%Du$<5+F+2aUg_dO`c?3)(xx zCDz~S;$3|RmJwdzXQ`!`jv@i#cu6Kq3!%)Pai?kRoJx?|nR`R^LX7g2G zULR(*eGWLuqtki!$kuuHh|Rmb$uan+uNMclNVQrt@PZ_?Hj1{XQx#R!rZmg~} zpD-WJ4W8e2A`y@j65U)w4rFQkZqi;eiIQDUx|{?~Owz9D;xES+tw*ajVSJ?>3M=uT zQH=7UX0iUo3@=-{oG;ApEr(ZzGLc16`?pmfc&Y&W+kfY4YXghj@XG65`saHj;gTqu ziaXoi-u^9B7OwnQ7=7epuTffAj8PGZ5@8$gHA>G@0d43W$iSFYw&3y%#Pof< z^ycb?43_okh4MC(l2y2$@KR^J>M%4=;IUkZe8_JkyD^e0R)3j|oH`E3Wwt&MRF+l} z2LHzO;L+cVM4x8d8LEFjMocNU1(xlS*3D$Jgi|0*ZJ-2}uV06M6!pz461B->}(xVFx^`hw4!ROvVcw8IS75D4ft1$)dC*S1bWm;DZN95zujg`b+ z6h;BZz|^65g-m%lcoDu6VMDd&B{u>#WC+<~TnN^H)!i1FGr=I|sSljqqa)lj*634Sbz$N;?$A!SD=|73TL^I>~OV*P#Osq{m_>kB~ep~MCiKR+nN;>L2NpVqg$)z z=^16~`ClhyDFgHTw3-RPqX!tO3J8b7A8X2ZI-4|I@m;F?MGNpqvDQ7jy!^S)CG#v~ zvV`Q~{n>>0`R;bNszSwJ-cBgv&WWNq+7CXE1u64}{6O|zK@qe4?wd_J=75WaYXxEW z31>1(iI83E)G8Ft)J%PAloU%A-zAaw;BV*t9NU*A@`pCw9{lMBEh#bJj{V>oe`~Yn zkbCWN*mIX@I&5rg_%|#+2m?*%A6Z7U{qd(dX*>}!x=x9bgi|8f>*9Ry%T}h6U6q~6 z;y^XCV}Ck>ugFj+9V=vn{Z@%1T=OjEAlKTE!H{8~C~TVa(}EVXNXXx-wFUyt*`C61 zOq-*v4R}EWXtKAR;9rzoa|@G(rclSb)bZ0C8=+XGeW&Q@GE8M6-Z(bjQOAqFDQs3#G8R6)>%)fYsXZyuG#pn3nYYLL^&1+b^_ zN3GgRkZ|e#UCksQ4S2>}>`0x4r_TIpogArZm>z)P?Cp2@>Dg!gMYe8N?%WtX8tbL3 zrQWdqTsGLVnRz5WUvsIeiBMXtYX=9cATZvlu%Uin1)*x2&3Y=l@Pkx;QtpvY3#jB? zK#Xg;$X3;&YAS()1oBs92Z;n#Qx%)lR6+!8UKut>&V$CEu%10#sw&U54qqI+`i~UySPCDu?~S^R{uOwCW)4t%0r>yrqa$ zX3ZRs2v)u>+*7=#7#0H*BsU8TfsP`YL%))0yDIaRYU>CTK$OkL&pTZ@cGhEYbf{{M z!HT;D_kP8_J0qsX&hYCB9ixyPS_ElMYI$K+qbFhkJ-uBgvzbflk zGQx6T(?Nwf_!)&&pVtncLX&Jde7cqrDi}EEyzHV%34*U_Pitt|Xi{AC>Fh?$V0u>h zvzV{%cX?|?cQY_%I@C0fFhNPR1rx~s1}YRvKsZ|v*A~3vrIbYG5cNMB)3)N-ikc;^ zBqv^Sl#nHoM;19_d?ltsZ>goF$Z(lAN$?)zhd+|t&X7|g8^CjR2JT;gc7j; zsx5V8ewBBe2s;P+-7RW-gmBnt3#V-4cnqzi^d>g4S*166#^yjP6h2*kmIvQ(e#7aq zf(o^C;{Eqy`7zRuWK7Cd!VgI&|8v(<+t3p^EQKeG|; zzO{0(A3G`$IpglWb-6)M?78gMIT`98+!{!EDga|8&jovX0nWmT^> zPfun3UxF%;b@@wN*c1`%gR5DtI#!gqR{!f#@)lrUNPy0+`Kj2wEwox`-yr7=G$v6V zdpdunpC6GRMq5CU3Pm*+8RjT0*py4Ya(x3&+zFSCy{n&whoBSx@EvpYV~UE+nt`U} zFCa56;FV?f1$|<3&!kpZ)NX(WeO30#*g1M2du2QQjqrfDnc+q@k5E&L{XP>!+i56E z7Y9;9GBGwgXT`nJD1b8&DM*WtP(cke9hRP^AB3u#nX7+#mK94Gcd7D4ZViy0ixJyUN!#)_wfAvRYeD@!f!7s<3 z_;^W3e1K?M0gGvL^`B&eWm`oATf?ZT@SuG_E3|&!PbPj+oV@4q;!7*d z;wNU`zaiX|bd1kY4CYO=%cjOfL@6d7%&nIayQt^{Pi4WyKmVXe8!|{6B$^|>${yIq zv@CKOgJs2iheE6{rYdM;gJ?Qj)L!R~7io&!)()F3gQb*}8R67YEbI2W+qC*fZgnA^ zc$-v~#s93piXvRbDzc8hWf10G%-$yKauy46Khyax8q;`B5fsp6lBWcZwyb38^q6y6iw?_F6$ZT=p zZ6x^{wZLoNvxJ(c2GReB+6dvF+sM60u)bDp#B>6!F@f3_3SHQ6Y>BN}S`G%Cv{E~v zi%riNBb>qf)9DeSC)xe(GcNBu)6bvI;kWZQhJ2lkXh(gf>A!3aj$3g*mBswV>azHs z>GPQ3K(w2lHW~}GGjZPxte&|3aJiX_b^OYZ@h6gZkmF{j$&%H7P&VC#9hp(?P5*IF z{;I|+0`ATanwdJ~E58eKJod+$?uzTMVeDR&Eg`|Z;?O}Hh2r5uXV2z$AGh7XD0E+( zEge^dM7H9tH$1JPd}Lk!pk!hxo%XkIlL%86``z{CdS~&!CxzJH&&2Fl$R_dMH~}`R z1KAB66pAy1rQ_31YgM`Gk9o;>9$_z*7!hVrG-yS(D4Op1C>jZ!3P;02c>_GnV}GpQ z@RChI+3eYTU4nd+Bapkgp-+Jd<;AK;(vpu;*U;e0_cU88F-&FQ@Y{FZ(*g1wDmmOW#-m_O|4KOlxLKf zCH@U&!NuQ6+tGT`u%~N!Nz%6rz4$3yhhPzOl!sV2Ssi);fE(LD9K*gjpd=4%sqP^+ zy!?u&sy9)X+PVb7uL_;3$1e0=Ea*-T)+=kTaz# z_-4-`_Znpe7gXP)ETM8xEDT{W)}$%3;Q|kHdB{#5A70MW^yuUIm$K?PT(YBELR`T0 z3Db(L8{9L`A|ZQ`SJ(0E7xQ&+%2`;LpbK23+F5F2A&z6@l-tzKE<7Ce&~QDfK6jH6 znScuF*~s#thneSnLO1@Uy9I6Fm`zY+mNGL1#S*0N+I^B6bAF9(MtN?Y9xAtz)(buq zF{bj!q~GMHoAL36I^AYZ%eUHO*Z3s~yvIY`)$?0p9rG|n;zoE zW4}b>*NSNTlG5=tpX$Uq)PTQNXu9b7)n&|YCNZ5>TYL6kqrW_LCm^Jg9dr%qaQVx` zAW2Y6GU)NFvK?8-XP~qp3*-8+QutT}%$UGf3<7a@(oaxwJ=u}xpy#FGAElHJDI2b>K#W9xVbPoRNp%sUFdtDP1EnMb9opY>a>Sn z=cu@z`4h39nLi0lgVJU2g6IWIe9lY6g`1R;A6T%2yeVaD1bfBR6>=pM4Ek)BQsqVh zAQRn60AzkW2Ps^J%mB#3mio<`uvNO?Er_I!LBC5)qYFY{i)puHPmLmpYZ;BFw!2g| zFxhURf>`d{EI0c$GoY4C4ad_jXWX|G2V)q@uG3iFY*XwEd-3yU^S&sskvRzEdgz(i z--Iz}G*J6brRw0~6us-cC~K(VxTU%XgHSXJvB$G-u2C&>r2I{95BeEuG+-tR*dT&s zib`mSQ8xx3%eB4EP(=7SEF;Q>&N3JKIJd?cR@~8aqhDJnU7q_)eAe}PRYX7RvM^L2 z!tyvb>|*Lv#7Zcj&Hhfsq-I*`jaA0#H_T(;oGOmFUTN3MQoDUo2Xxp4R^*vw*10fV zWH8NSWbrE(95-^>0M9fAYds#*z*{n+0EqJEJu=PmZ*diyeqcW=zyT#@%) z9Qz}$s4^5c7AY>Mc|w#m@CP=!X@ppv<%d)4{v@3J}jZOsmK!@$5Vb`BXfRpxkMp$D1=pG$*qc;sw z(S)R9bidG@GJs6a5erT?=INA#j{UeFF5v&(TQ+-S%IY^(S=jR+L*3p&kB-~X;01>@kd@$Wd^?BkJnSe&y=_N_66K;ED__hf}Q1qg{JpOt6maW z6b0Efmf+)=V^-HD_&BJTD;697gmzbE$ri%`lV*|rUV|`0muEO{dwCGS3{@2OZ1BuZ zmqY|(%9<)f%Hp3ZJ)pD|&g4njp0 zUHxy$_AbL-7zDetiF%52M5Ck04JMF7+Af(FxpRs`y#3`>&c*|GEMpGfIGYrV<+KDb z3`D>?G*gv4jrLx+Zs(d5K)dVhh=#3ah+E1c1 zv=Be#>i2?|NHcn)XgwgxLPycH2e2yBlJPDWJK~KJ%7a zFpY2rKT|087pE*ch0_=|;)HgH@PcitW@ zo|v9l2NNldt$SI0e)Xwx)!)N!s^Z(&ic{S%oNBKx=^`rfCFNyR1~xC#@P{w`Pu$u< z;fl=XtRSO9PWoO*duCzJw8r49`%&Z~W&igaa&MrF|4?>ZEMWTQMA&oZYb(iYj70Qw zO0+8Gg9ytgqzyN?O4fEX=qn)Fa?9E#37NlZ>vQV;XnXetwa8{6 z$=#NLL3g!K80B>pDGuIabmx}l{L$ALK7861+18%ovF^MLQ@baO?(ExpvQ0f)`My7{ ziXwJIhTI6{3v+X@EF6Z3`N$yaPcgDBBx6@7>wl5;hgbUJVPWR{eCzr7h0o7*{tNv< ze|%=70)4lES|GR64=)T?zK>?UGQ9BTXwyx&?sr!O-~K#bZNuidT_Q| zJ~F(LW3LP^Jk@Wm3|D>+-QN>lce6a(Cbobi7AGrN_FEy?*%iNshoFFU>mi_BJPng> zo;7nxvrxI^JZGo9K~ZKuQW?T2UmT3xVKFEuFMUC{;#)~3jQyXAls^kE2%VSsFZtuc z7bwBtL$(8=4A(z&V)z1`$i0Bp&kbB(F|5vtd!6OPUe6CQg|Fk}$C5Sb5W_r0b^={K zHbn_N@(|Q>tNtP74mb1w42K{8N)cUsN{_{-1RVz$&yQvedur)&>a#%@jQvO#8SyIz zC@AgHMgBw#MSTik2ZEuQ^%~&9&Pz`{J$%s&;*bApg^iQ39EbV%i|?Hy!JABFi2dPU zTJId7BE;;uu|JNd8Gn4~hRvOP7{n)Cg3rVceXblr#T7rUU$TCD@@M70Q=Os`tf>x{I!M#SfH;lG_tQbG+4Mn16Y&5MTr>>_gtF;H5m58 zbFC9>kMmq+(JTxi7A7Cg1bo&=2(GZtX4y`X#g&;ry z6k?#KW5VE+wj@KEznVTafBp9J7gsnN4nQAthXpg4c5n#GHo_*jaK(cNVxp#0PDU>D z7s(P1G{x_$vpGPnZpD3fd{CYY;APSlV!$w;!y&+d92p7#_@p41^RchlHmXV^CE$$H z9t7rRv*7AKCcW5R2-bQoWVM^RLjAb}0t5W(sPN)oEvyk(NIE{Ly=*;2+O+A|{7B`Z zALend9+A6%+9~As^1O4@vv9XU5OlmwTBN;OdIe#B@&riHRoX~1Ze;zt1{b<7SE-tq zOYVdIEyD}X>L)GI2#DnOgd+?NjkVNV)cX_mCUfsZ1gVhtbEzI_&iNnK-+s33 zO$_XOs>dz;Ed)G59CY;~QuS3AJVIAIY7KNj39x6?)mv_-Yd@5_`OzFym&qW{bn{D0|aSh;x5bkN%EjI&;oU6XWo+}HrTLcy-)mO$^C{sJGcO%=MI35N!es@-<7 zc<1#;1r)b?bM`%(Gv*#%_1@{tH^=f{=OtMJ+^YXdnl<-+?b-76cGPsXIK-fKA``ve2V@@rt1v}h1I&N-@j+@&~#|`6r zj*gq#PREVK@!1*1uYaPrv41+V)nN?RbtdEB;vcjxicXTq>D=^T1>@iB^I7}#bw9x7 zW!y|G=UL&G+)ON||6Cq#cK71ypOjNk5WhHT87YS;qpnEVdSI`3m@df=SE6x*?lrUQ zJhM!~?j1WR>SSf;?J~Nc^7BG%PxJHr;QQZ}WheAV;#=iuKy*0b(K1WZf_@&nhPO;0 zb%oB;i&!|fz4Xue_23oE?Y}e2vJS}f3MGq6VG?CUTgA^ zHHqL^&fmoxA)#2b*a@{g?xNM@!u(xGPATgjQaOgTdA5rqzj}ic>K(MtoFEZt>%+UN zM=zqPHr=a(uLtZ}kzh!w83XkDMfERaF}p<(20M(EJ>Ol02IzgNl+m2q9d=zyj&RgJV_EfFozY@zU| zAM*Y`%zWzDY(yv*v1!j~tqySU%$hjX~tz&hsmduPbj$V&|8Xo&n+{G^*So#^7@S;&_s!+8W< zk2OYdAqNIdJr~Nhwlg3bXKceJHn7wb6t2#VeGvVFNF(I*+ecQnu(J(km#u{_U}1nE z{u2VS(#rgWQYWrEQ@Av$Sg}lhQb&b?szZ)Lsq>uUi14&Zon^Ho_OYCG zh{_8%z{QMx4D+Xl-7sfGB0C%E){D?h#1&l;R}@%|Ii5R6|1QkYT1$}`KNsd#{gRfU zH>sH?Mr%=QmmbJ@O@AP}MyUA<9s#2$V8%~iL3tbjYE+hI1+bwrX`JdL2tb*0yZzPr zq3Hi#Iq)2};>vbovf>sE^5EANa+f)Rf|4V`M(lSWm!XswjZC?2F>$xz(oXVvyyoN{ zli4bEILJk91j%+#)=A%E**JJN=yfu~j?fQfjXO%WOOJ#VX#eGd9f@hRejHXCK1g8hEX6-!sX}rrmdB1B z$d+tPOogv5(d1JC?CKxEU?J^DVj_pxtXy#yHQ1Q5+4>E}PlUF`#bFN7bBaW1ftib? zDq0u)w2GF7R27icjDnHi#{SJ9pPb5QGPch3ZmTQN9B*^fGwb-&BtI{atfBoqPzN?=C9U2=5;TS3K?CX9~a@uK&m%w zwgcHx@crHLQuG@1^CvRntZ^_a_WwA`a<%tPs{`BguNVY8p{*MABOskDxu{Up{9<1( zW#!H&V&iy8GOF>2ePH;`k(eQ#QQAbOPZlJ1&Sso zR1Dq4mxbS>)kT$@$B!br%y?}D0a!j)-0Prpn8QJMTraQ9w5qP0NO+_%^w{}dJ?8DJ zGv{0ug!jbr(PfX$j#+=(qH9(C`0B zFRO4TJ_F$09L1f(D&$sZ%ajMNqZ%Au<*-z;<)iVAuP6@2U;L4BIEVAV@IljE9wMdZ z`5lvH9zLNtIL7`%I*tWCKuv9`&_f+wH1kL{#ZO2jY(8;B76;>RaaQOUk#NB@prmqM zW0*kN{EC}zdiBVam_{2tFBq11nnr#a5W3kBd;RB9U1f^4G!)q-#CG((n^!=0YJ(pa-n*$CY8O z57VQ+Bll%t>m zTFdF8*v0SJ+Cbec4b**>4OFiBw+5Hm=P9?feY$P?^r!7}GMgBq3Y9q45=(G)9w{Ky?a4r@%q@g(oHX;{D3o#n!fKzD@j^SFx(Nmj*>4*$! z@IbYm9Y}cQfxCI`gKPBzkYM{YAho>j5aM&q-6_k)Y(dPtXl3UN3@TdLIi7nSGq8nl z)M#ZpG+t-3@jBDS>m+SbW5BlOdV*D+IRxvljS(O)q0;;%;Z?-H_;eY6{y$J)X|J!1 zIut;-??rI&DPBx7uz;L*uis_W$f6|6FV9N0@9Q`ZK>ae@NbZ7%D0; z9xn4hdxvt*Zh=_$S@cAqo)BfoW>0kKi7ww+PZVQ%qW^4q!sb5G@9{)JaZ*5_<&r`H zjM5xA7uhbX(p<7#q`U+0*}%!Na3 z21kdNhQ%n%d032PI^w7mHFp{{PDgcx+CW{A-x@A``1rN?t^Js-(dN+~TML&^cyz)x z=!;ZVJR9!rB3y(+!fX7VJuw!k3|aj-7JQ6xd?=d$S&=j0*q(?n6!1|UPxOZi;Zn%W zx<>8>Tz@bbK1UsXDc$PB+!gslIEyyjq&7gg{O}vXb3mRQqkg^&gUN6ZcE)mT_`I+a zQtitq7A3M}O-Xtuoh0ESDym;Efp%*C?P{&eI7A~%T zzSN9S0jtQ~zm{t-EH^J#w&O%J_(ZmBtf*P)e^EANSg;7gM}OMdE|g}L+SX7EEG5R9~^lD5JkgfEqr}?r3^~8SlWLT-+YpEb(V}MU{meA#jnUa4|p1 znBN?FL+LYXwyX^c`K=-bK5u-6$~Ldy1i(O~dB}3HIr_t;2Jq>GOBls)cORQHX*g2~6@)H9b%nyy)=A;@AKIOKlYJ~pbi116px z-e36P70FXj^VvSUPY+KeHUG2uEii_$y*pXNG=()g`$8hjvo&LXm)bUkQo#jDKiFz4 zRD+;+uEd5pi49!Lqu69Sd@{Is8J+XC)$5xotQ{bj`5>(qHcYD5;71WN%sK7UblDOW z!vSbGM(WW9)0PIa_)Dzc$KIWDk5{E0qkg6-L0I3>AVY@!4zvoP06>pULKoV#frnL` z<#4C1M0nc2dpC1Xo_6q7n&ODB?F{{R;%#wb%qo2?vM_PO42#HudG$@56U|7OP<+y|b%t zuE4bv4W!HrgI5(NG&;1|Dz%B4QMIRtHG<93$)mU2upCO$w+AtYhc)w^X}1=*GC7Fl zIG#v5lQC4E_L#n_Ls=mMCLzwfP0zS)N~0=&fu-Sty?jRuF@!L`l9_@w)S!(LKk$^^7r9q8mM;@=kV=G&ajW6|x*D=jHGya-N z*E*yH@Yf3S*C1s>J(jG@lP=6H?{{6|LLENDdC%huXo^4pgtuLBFQ5a~2*Jh-&owt} z_)^H@qkmQ~?sG$i*E5RP=qUpE>(mc>qHH93+TZ5`V;p%Xs6guva38+27u7TDo7C_3 zdN)PK_BC^3f297owqx+7=eROlqt@7St9~Cgrx0CX&eDEHA=s;xWSTo%DP?h_anrtF zJ4Q(LB$>{Pi0Vnjf|)czU~KjUK3M(VRNKVRCqV6BRnS7DDDk{z#yN4~`Pn$}yyC=U zbuCUjPn;OKnt8>Ep<3Q1PHe>%e`AjyuVIXTbs-tVM1(TA8rRI`$36zdH4_?lW`3xF z_mS?KTXEf?*m3s_MTd9|Yw%JWuw`*!KJ7BbVSeo^eZrh!p44C!MK!s;+;ovLV#(`7 z4kj$vQ4J7J(~0E-b25&dJM5b|`W#@g@ zd7kII+^0@^gN?ercwi48n zj>^l9H7!f=4ncr6F+l+>XqN<;ctn(V!q+p+S66>&;3W8uDbGcm8E)> zjIXq&)SoLMv)~c$Lb)5C3fx7ZDCO9n9{Uiz+*5b&x~-|YZyND0aG}-qLh~zu_CckU z*&KQn1Ull@3^0o`1H^-co(WYzHzJ--4A5y@lB`d5W}6gpk$7I9AiIGW8X39z1)eO& zDTc{CP!L=s7o!^cLzwkPadP>-tUsTQcF|Kn3eQ5*Q(C4erMIcJncatOygNy*9xNyT zms+nNn|cMAyn?wKLYT;F%j6HdHS+CpnZ3E9_@PYnHE4B+imVv>W2UuK)E_OU0r8Zq za#tp+$a^wIm8^1?C99k*LP||Or$z6oi%?3-9dEK=tTA0Y5cQ|m9Z`xuo$X-uIrBL9 zcD5_eQ8Lr=>HH?*WYR@B^P3-}`JFSrdCw1wwlsDl<$~SqbjX^$C!pq>`l|bw2j5ImVSo7x&+G$39ywvqmh5`c7#eG0TywQn6ooM|(0Gs_9xh ze2Fht{~1qUk>w4$gnW&q_8oy$l+!XS6Gtym$Zokp4#q%XuJSNK)1St)Sshi{Vp*38 zZL}pTa3KzbiZNcMFVkYIbPuU2ixmY!!x_SYIwLlK0|j4n&Q?n7y8iY-kV(~Rdxu*Mu zpN=oD@g=>vL>g!V$iQ8vnUbsInFizwj<^F&g$CD9PQt@{7OMEWurAeSPPzk@_s_=u z6zFRUcpt4VIfnhIkPoS^Rji(w{;uL2N#pv5Jykz_Y1D_WHEJ-1o(2de6A4(&q54wMwUfXclUpXZxt z$+Yke1DsLuN|}OGtHGCOC%CF@Qv%5#Z^sVZfAXfhf6hPd&%QN_U(PJyBxd5SnOMjI zLvjqP3qqj$BC7HaO5gfW=9>;4v&jho{w-QQko?Rjk76Ejhq9a)K40r-Y@?mHdnwya zhl>g%PpU2vm;mXGPy2X|;O?nr>4mVQdbIkFt&e0*U2@y#zd|N58iX#CI9(Nj8Rgfx zkhi1-c}siH4}+vN%deWjMTnsu6!Ln(C|Dd+C-yurzSu2C4+CENRK*$kC_T7-ofAvu zenK&QDm$E~($VeXj25o;K-5lVh#(L8O)rSh%E$dlVo9a|-gXxVb8cu}7_byfiwo~2 z_ZBJ5TDW(8R7NNx;yKJ1%!kG9g42H>4y(k8_O`6ttUny`w`IlV+J~Rwg-ah~e9+W< zl=F|@?0)#Er#6Ef)zPsPpEXN&<~U%8CUnD6TTb11ax?pm^-yo_?q1}L{(Q&!N%s&M zAYtk6HOpc7dxkxzx3kew{6D3ap^U@Q-)>gkuS{ah!g5&o-t|F=$5c{sznqGqzwnFS zJM7Kg{gFSR-m^pAsG2$~Ss}YVSqk-ca&$WN(CchHe9|8JSKLEWgAbqo9|kn6_0H!- z32f?UmI*#ZbT~g`(cudU$Q;f*E147h8Q;`9XU<%pG&#p(yWZ}6LHMe-Wu=-s^?+}? zq)^lA{L{<6NKddU=XYdTv$T2q!%T~>=1*>pKm3$t$Qzv}H`ny(?(XYLPue+_HoNDU zJ+Ci48J0HJc>PA_Nq#Usub+pd&Ew~vdg>_#D`kH#_M4^e9Cm5J_YRT!Dd9@U6B+gY zmj>Vb(0l0ead)Q4r$aA?GNKF6eVJjn@_vnsmG8Z0C~CoWSSbH^YODUGvOReDd(s%| z^|cmVWz?oq(VcSxb@sFAhkxWxs8Kak|ERTZJ6>TovVl76YWS<0qu-WBF>S6~536sf z)i)Hg#Y$RktgdCdOtJW?H*JN!>e6cm+tRCJLlwW6P3lkkTWy(o__B4EAuu38d5(bT zs2KD#K{d5=(atE|nRs+M8~&YyRuFof0JBH}1?3Ufg<7c%-y9EBeEH=PUnGTZjwatv zdCMPx@U@?UX3%+%s$ipEC8yCWDh}jNx`)WSN?_zWcqM(>_oOusH!L6Ve5bZTH!Ox> z{dz%nu@Yc}QPJJSW?6)@?eu#6a8nMu`b{S+0&}u15IrAJE9h)kU~Lb<1fBH;s+el< zxlO}%Bs5T&{w#aMfip{wWS-_)avP`7V6oS_K;TA zx&3U_r@gSkN;U~mb5eB|B z_(R#vBayGn1yO(Z=Vijqf~a;Lrl4mE?q?np_dum)k`s04%MQVd5HiKwF+wz%)&sea z-NPBVLbrX-4=_rJ$b@o>$dgb#iW~i}OMYOOr&qGArgJchU&uy1Om8N6am01wFiB+< zO^#Y{W%SQuUtrB{OuU&nu*K|v?~Yrh!`1UL|n%b?{h3D@%GPAAddsCE@dZqPMa)j)%umhM0y9<_;5Rqu__^KJ&xveZpGX-e<{umY3fr zy}C_QB~SOs^zzgD+;f*oU`$iBx-+f_k&roWr$Z7c7IRx(c;4Zhx;-;aIEct zqw_sLHrWO{Ss!8PnB%G4(^RMPkB@Su?h&^ZN(vV>;(E;LDerzU$Gcz5@$M7fFy4KQ zwv+`cuY@zPNJzj=@FpYOFIv)l&SIqdBUDealkV;9R8QIkT8N9d&!P?~f!B$D{vRvq zn#SLJ)U}hMt_rt+v6--JhZ0sM3@ePz*E3b@?pUL+h9R1u>))#Y5fkSOXzRn7C(%f7 zaG3V6HNl5@CceIh(k^BsDM!*uDa$HeMa#0H6w^Xn#N2l+h|3%^(_iBG_AoO|=a<=e zjFRkdb~nq7)7=wYnpbn!@H;C0IO*Qb)O{smkIwra8n$~hVWL!JZL)?56(C1iyP}GW zJcuV3KmvTGZ2X3QB={jvNY&nD@Y>mWtQsfV@NCtx?m{8#Z~l7&uUJ@(!P?gF3i|LP zG6L!9(E<&5s_8L3c5I;>b+t~8DlMZUEnCMqY}MvfQNLQU36K3L1S&{Oij1qfLnjVc zUIS=0R%hAL>Bd-Ze~ug5lfa(dnB5hG#H#D`+al+pr_Dx?elB-`x5*0-dlQ!14$*qI z6Z7w9pXG)qe^d^u7qa>}-*~l}HW5`BP*@0d>g|a>ENyrHR>&mFY|BK!m}u3Us792j zlJbD}VgP06+uGTQ@(m^am2*d3btZvy;|B_e%P2TbYK`U3ONaAbZq1_}l!%PNF#e_? zq=d3~MDdPx)6L^g(t>xl5CAsaX$#Ia-E{mo^hlyvr2ub}8Up67Z&`vQ)skYy;jd1-#v?rjoIfGdUY&g@hv( zM(Dr8Ec)a3;eN0!)t@ho$?Os8iumq(LLR@AP3nJIvO_iIUyPfAp`ZZpIIK3B=C z!;(`2I1NjY?pl=9)ec zJJwazT1d)3NfytfbT(&3LQvj4>LB2dvN2tj8R3gjhs4JywgS~lf;{}Qa>GA@Gj9*E zg%NvMTQC>IY=HFu(;uE7Qpvp2 zvhyp{ox|La)nB4qhT41iSx_aq;5_0d1&_7KHk0nC9Pc-!hrHz({`i1_8cy9-mplokB zF}u4;suVUiot{vI`mRjGE(A<6lya(2zkn1#wtXU77N)FK(5RyyDd$RPBGmQEC0QZu zbjCE$y)dc++V33=cO_Pd(CPS2@LLn4^8$e;0O69- zGU|uo!f09A;|qbAzZ90wimiR@2g^N7<)u)cHE}ppGRiH;T5RB>04c>x?kpFLv;oG9 zTRO>)RKyl)nvD@%wmKq}MOdvB%4@Vjc_B4}bhp!U6BGJ|A3UFba0G5{^uci?cs@h; zN6G0r`Ma#o?hu(rC0u|Z>}@fD-^ylq^IO~{W$`jRV$TijhZduP5&sSHsLc~&kI>;(%NU%cB2OQE>1cTizvPk*h2 zZuU0x$ivQVzc+XL*}R{aUBgf8&E39mezMR1$^pN4&a*Z?&kP~1hSvbrmn`zYCNRcb zCF8EexFe97pzR!aYSb0WqE*ACrGrqb(2CuA33yHs?;%r48Q8g*dW`>U!+*rWaDmb} z8Dj;Xg+jbVZJ5mR7^PW+?+ayo^2^1Be@~l3e<-pTSrD>R0iB^MiMkn~P#F}5N#~A# zlHekX)Yw0&Bst-UWbqBm#+yE?!r6D^-O-9Yb9K33`T)BwJ#?kISxmkaOuj3n)6LL! zZTu5Dd2CA##xy_TG7)FQL+qX`?SX#|Ozs?|BZYSKgA#@fa~iYaZ&uPGMv(-n=X=h*huVt-TK$9+%CBK zH%c^D;cz&Z$z)f}k8ECJIY64Mq@oh)rD?}9$bIzNAwNUgmdV(!Vz^N|^+lX+o&G|5Pii$<6_SP7<^Ks%Deb_*oO#CeY4ZAiHru{g z!BFqg4H}i_DPCH60ngwmUUJ?I{{b5^JD3Txc88DZsM@_yyBlnW2TJXp#_j1)TD#xf zj$1sL*|@dvAv6}DpvCcr^^CCc)W42OEk==jR{sX?lLbnlcrwE^oZ~USRj=#_C*=; zzAF3?n&}*Ii&TPjBuhYlbrQ_6gE zPRrAD4rdEshf^9i-g_<&kdy zqb^|7{$$is2R93$OYE#5x~f4zbhl?U3ycfYX6vMIlIx4m4LLw)o=0QqIj0Kk4%n57 zHVxGlW93+bI!G+G9>trh@bjACq$wt&WlM>mENn|M`mEhBJiQKZicljLEif>+3K@y? znTfiQmX;n@7BKAz%@$k4yE!b!E@a88Cm5QN9t=Yc7_!E(aXsro1yV&oP*r6K7C%>& z)x!lFh9NQ+<5+%wbdyK7P%CSzvXH-^Dr-1bmDNjCS@s?IJ1vOM(NL&w#b+Ih-g9DQ zQRPXo;UBhuS_-RiOYB;Zf-NyMo^wqRi949G9RvBT^jT32_gH{SG-a;z{7N?MD9wJ^`iI(0AttssP}n#9cpS?3OZ@m?`(uKTvmaX!67x9p!$R_bgfpQMkZqrR?Vl>}GPkdRLPW=l zS|O|#8hFy=N_4UFPt}-n1)^D|wNz<&kk$`0P-DN!64#*{2Fe0c`~`eFp^!D#}v%4G*E5u{964zkCR>cPnKSI&`EKs^2VyZ&kJTDjHRn^6f3UC8vKMZi-3U zhWxf2bjnFqX})Ylw?iWnLnj7Vq}2!oQl69sl~nW^KogCI0{E>m(ExF+=tfCz#$kI< zE~+w|46-|n;;}!L7T`ey0ipwCQ(&Nf5B-L``z!g-0dcD~NYE7GpoMJe6oUjrY^;g$ zW~E~(O$2yY2yFIIf~Sr26#!A6XI`a6g!ULnl*-KCQl77-LO)bZEjSdV`;o! zHY$_Gdp6a0w@xSLIiD45HQuilVxH}WZHQTvR(p3t(|ETaGn=4Ve5c>&fxL;kAI_4% zh!7W9{ZcuS?T;1$$tI6BkH#bIkv`PEq2$0UNx>DgPf2dFyQ6H*nQfKjJ8Ty>vt8WO zc5$*@&~PW3E}vPm>o`B}wUv)I^m~zqTz;qW^nO3?6iZlmezzpjM>WmT6^#XB_V3H} zgX{m7!j*}rS^NnKs4%q>37Um$lnT~7*QC4Z#Il;by%Xy+yNH?-%IM2x&#RyNnzM&r zGTq_RnJ|7_6re<(Nd zj#}VBI-($BE^hU)huf?1@%ST?`nOAySQMp{C^84YusD=@T1v5TG8HjRZ~j^3i@-mtP+*LgyAmfC6bsyFsDzf{ z;tjDb_=rR}K4|i-c>MdY;P~AA0ACR>5Y`w9dhjsZp&}9KR?XjRMlqe6(I$25Py_RfhxPK4z8%i>R?{T`O{R4az2?3Mdsy#Xs7M#e=&j;i zkHmV?bd+rty4%{CDhaDb$0g1}hfxP_C`v-`i4)R;MWmtzsvCNDrCz+q@a+3j@uiozyQV}sRk^WmLy(Nfz=fv# z;CGV+e5nVbAd8={Lpu*CGF^T!RFL8oeI23mLABjpl!Zm!1OSlsLy~V}pBZIZJg=Jy zkYe$gSXI{FBlncJ)r$C>4t>|q@k$5M*B7(B*JH=JDmWED*>U9Q#Tzlcs#1ZzTD#U) z2F3=K$!H-E*Jb{7Edt%sB1ZG(ruIi*Cwl1jawJKLEpd2xz3OW5R)Qp> z|CPY*3FJy%T^S6^L39C8un{Q*C{f3eNS^6YF>4=xWI|?2_RWG>9rO)6H&E%N^N=@^ zYKK@K!U9nnY#-uObWY%(Rd5)#&AoG-xQRsJx=GQaXHSDZwdj%xxYKhia#aBP2UHuA z7H}&kNoL4jcS)4Uq7q>B{o-=A7pk8-sl6~?_Smd4P_c@C@0>|^<@+qePAiqRRD_v6 z@5HK{VlTx0h_v9@-cy{~Jh#*j?pqNeP=6Qd;&O__H{U<}C;v-v6Xxo=N zB0`*d+XJENmlvvT`X0%-NPKf1buy<3@pZ+eED9*2qyH0yUzqZ1FNk{}JCWVY5>beg z2Cuj2TqKiaCF>Udqv}urObDZ_>QG6)*x3o4Goucs3J>Y2JWL1Z_|TbF{n!sw2il1# zf94^1r4)2j#B*VRl;q+IVR2f6-|2@=T2+eBfgZ$#?)`W_Oi;w*H^aUFTew zsdIt1Qv_D_sHc1Vz2*&()Vq`28A4^ z@o!dfFjkZx(#uXrpl0dY$uMY<(?$@MPzHzROP`-he%#WgI~yE;mE@rj0&R-;TNylb zSm%ISgk}U_6x_F)K2Ta*i^XMULLSrqB%;<u z)e(0qdpKB;q3BYo!`p>@b0OE zb*?Jx=RT_l-dt5!n^sd*7|4X^CS@SE64z9tt`>J_z^^sXKG;|X7}`BObnzSpnhQ(c zJ%l->eCJS+n=NI}AhB~lD6f^hLGkV?WiP#z#b#mY&OE50+n%^QapKt|L{b4>O21d( z9Bx87YwI7umqZi<5@QxU&*3*#E=B0T%B4K9fvY$7*C>?Y_o+~dF<+**TPPG;ORn7X z*=XKCET`!I)!Th-Q0rZ{)$|RZMkxaURD!pZy#3P`t9pXvSKTw}DiASL@phLD>${>? z@ro0w!;1^01aj0=vGdpn5JB}=D9-dpD|F`6OpjsCH!ND;@GY%x2-EhQYk2b>*KmKh zITRO~o3H8`u01qF_Pz>h;pPv9{a3h$#hkka$qvVY`h5MTv6Zj%5^>I3FR^O9L_WOS zOOzL1UFIe7A@ecJUlm#^$qQXw&@L-1R1665Kq3_^;%ago`}h`;KA5%TXXYL4LED(W z49$!@?ay51XF&+ivy7Mi`|Y?}oC#&< zhUGKsioWdn?bv;6JuPv)n8nXXRkf$3I`rdi*zjjD%&o}noc_l=SfQ|MU&!9H3E!dNEV|Gz0Xt7xi zXYlb1Y`3qsLOIc$>y;e>35GUeDswq`H!P%+G+2!+aL_j_hQU_!6W}-fHq+UUpZ*s) zA=`fZE5AcfD=rG7+6SY$II8AQ(cRgEyD?wAJ3~dYDl^!MKFmt1#ASH_s%#Klf3(QA z$P+vgFA6FUFJ|vthp_W?**i_mIvW;c8Ww#1f$d5B1wUzSQWru3h|9Wjs%-_BdAVsJ{R&JpP36Jo5Ehw|0`rMn&aZS^mKCFwWy$NmFN z2D;O2H`IWH3Is!R+eEX7WQldOVwzwB93|Mf&<_Kyx?7MA5L5*8kB(+G!4$ciWpOuv zLD&sksXZ0dfz0-}TSN%pv(R16vPNYea@ma3Y8T(JTCW~ME zY+3(@5Q40TJ5wrU!JUo#N5f8bTnN(x^%M6)$5U7SSVO%mn^>V z(>?w#MG#G}=RV%!|4AYQBOpHi@m|Q{d3|`=9Fq76-RGzdEA#|mJQ$Yo>rWTwHg`Yr zyRYV(Sr}f}T=?(@wbAfTm|Qs5+#GJU@4tKfg)$8Lpuz>hUU`Teh06X51%VM&rX z3|g289^XJ(umgG<3|klRQ2l|pdi6xMM7HBZwiv1H?u3PVPGl8WO&>ig{b&#ugbJD4 zLR{Q^smK3mH*11v{m|ayUjOM{GD*^LPrcOR|I@Z3RrH#r3|uz1V|C^(cSF(T&g?XB z9%Z8n)sM7WQGKFkhF(zI!JSZ@$Qb?uT;0y*xjXZhSO7nlopc|exjKilo8ngqa^2J} zF9NXM_~HU;ZML`!zV_l0y;iUVRx_cF!@e^DQ0YdM7yj*R!+(x@ye;0&Z5&$P@UJ@Z z;>@KE3ALkl=I2Q=)su#~^%~|E>mq9&sZe8QMzHO*Y*WiD2@E0AqvNXL+vJTWWij=+$(DO6G$UReaEjU5JCIQD-FZ6^x zK-!n<^$%wwOPBFnhI1brkrD1ol=wQRrI_7EsiBU3=(-ps9S!MeK+rd>qq+I6 zf?*$2_1T#GMZq?9EE#)QT5Df=G>L*9i@UpTYGS9k|9xrESgsW5uGFy8#uqLni?@d! zHQ8;ID|W5A4dWAmkyrfO#F}#!Zfs)B7s$<~`J4-PW2c#uEl*JZOxa!GDXacpHBCeo zyoX0c>?lr~js5$Bi$@=MvHm~;KA|Z42u#V?VV&$iV4L5WP!*7R3|Y3 zP@*;tas=3#YBT#z?a|-npA!Z@4;7nA9@k{09%n-N`Vtx+Ildt|QK7;tr6fV{Z0x`2 zEWH*3sDn?cYTb?$bM9y6J{WVvD{~(l(rkGY2Umrzp=#*vLR^2Ykti%*7yp726(u4{@e5m{>WB z80s(zS&(dV24mi0*!7b*QcVJ_98o|>ZVks`h{(Wpm#n`{6PmoaR z@2kPTQw0iyo|^@6bUpqgx(hv02PQk0SzvlWsUr8}Q+MW&j?}NoWh)Ri0XPKP_WjO@ zDLiZ39r>=R>t4^#ZX)QdCXjLc5%4ZQT2XV6#V>soS-*efqKiLTkre$g7qbV~`}c7s zavbmvWWlLejwD}O4VMKs4~oaWgCR7&f!lH0!jOyP^i z-MnLK?yX|8iv!~=?sU4{%=K8T1P}E1Q?5>i$C4JYlddL{aufg#*fM>O(yU;@hch>F zw6WPM>05fEyL0Bk^dQec9NW3du^oWdi55Bi0(oTYpLD+KiO94+o4cD%$ljKfqntkl zG3~cyIVtkDWgc)`d*(cd<+>7|aw1d45bf?5jdHr5w~;hTjM$t|hi8cqJ9}GV#I~#O ze#VIX6mLa_Z9HAEy018w2(j}Sd#DJpi^~YH%UJaHH23tj9rVj~Z{C(MuLON`j{y?n z4fNs$x$)!KY!Z7w9B_X-tKWfSnC$ewi!;j>`%gD|Bh`^I$R@VA8N@_%gsT8Gfve@np z-g_yzTe(Ka2nURP_D&;9X1QXFxyg(5UsnCF#C3vKcuPr6Wfz6bE$V+rVH;y(>|e(~ zn{4>kRSy$i^_(>n@6rjf_*uP&%b2SCSJ3BC1-URTILyM1e=hwB8bxt`UU?i8b7Oy= z_Rd04KWi{Gxs;;NpQaja3|KH{r(!AaFvxqouF{DtuEf9iPgppAl-avdf5;gTj~?Ig z`29eLN^z#wF&o8a5P!x&?rPzcRrA8sMC1rVenN{cP_B!X$QRn@RSgsMKtTw6Wi0slzf_*0yKjjbJN*@^5nzzV1#r|&=?CAy=O80{R+Zc|C? z;p}J7m_kxBu~@xxNV?Q&iK|%bSebY|5j56#YoTr>Ev*Ze(l`7Nh@q}G_F*CvUb@Vc z>TupWEC}_N+b_v&9()2w1_A%7$I83kw|Kdoc~GQ?z1O{VIWl$Itrtt*aog*iD}S9r&|Lf^?N(d_~ zh_m)nAP0BT?P{l|Z5fuJtx2grZL#ULO~(FJKV4|Y{<9*xP^rQzhSMfA@rqAR_&Fci z<#p^oH9O!_#xTaxYMX%8)_Ja-`53?r(Rj_Rt3)3p|K;m$(~? zO~g<=G+{Xjq!PVp(;U(UEP%wA15Y(L&11!Nly&4@aXrRo(3rK8f6=AI`wzOYZR)ON z&-3ZNPO*4XIA-ID#}RH8y;2I@2->abMWKop=e#nAv^o z&k(bDZGrT9JYO;44`+)*$o#av(ULwE?+!ke_d%ylBc`|^V^|7@?HIQ5*jSv%R``8C zxW*3^9-588)c>5PjGiH{gqr@OL0N0CF(|nm&`MP1c=K3m9#6yb5W6GazYMVTL$vB4 z{nA=hr3`x7Ln5x{SJUF$U;AyPtn+EgdgN2un&RBK)-+ZrEpYR#nQlw&v`kjBZuc) z6g5}+a)zcZKd&FQRXH2`M`X%~U-OSMR|vwf5Y=4ZkP)eJ+-oEVf80u~XH;3T4|SIe zVs!y!^SDvd;lMexa}~=a$GU!8v=wA%JtQSC zdWdFdqdgjc$FwrfCOWRh)kbFhY0BK!5A?HFsMg4kFDCa*pT{khqFVR z^KkYp%y8Asp{nU(PbCmze|)*fu$72FJj|aH*&+LLq#s!PIbL9q`E$TLmQRG7jV?;R zcK%$KF6AfPSPl6_7E4WTs2O>A2vRL6a{rufd^{(j9qm7d&V1b?Pt(krd%4{-ZeS=~ zYbztz#)!JgR+djk)HS!Vls9<3HO3BbKF#uw)qlgd2e1Iv%n*S0Qk8E<%|iJzl|qd+ zkvgr)y-g>xm~1mFkQHNHag0gV*XuotxsOHaW*H>f(GF+i{6PtTQqsz<-T5Uqk}SZN z$ksAi3Eo(ViG2Pq+Q0-}kswk|IdN)PZ7(<_N1KwZ*`6Xl5_7~e%D8bXAQ}h@2xFD? z@kF){%T<9`!XYj=yN-!_2&?JRsA3F}@^hr29|5qfZi-L%lkO-Bz=`ZzLMMJ0NZ3H+ z$|Y}793gGcnv(E;nENCL@elGCDzB6Wtb$T79*vK?0sw1r9A(Pw215-qTi(V)xDEeB zF@m(-FSDh|qLFYP&fczx4xhB-If`?nAn11H#Mpf6j+<_`44IjzY~iZ%DaiF|5RCmP>p^ON3@@Yntd?!iX=7#d@b8Za2odT&v|0ldFQ(balTX%u`j7cP`+*1mbM{`vbAc4;=(<87pCj4-I zIP0-4#9FTN;j9~+UNH{4!Mzw@j@&%{6>~R#W z&MWPT)PiAgTkPQ4tSc0#w5%&47r&A`Vh^#m#!r%dP%Rn87abAyh(JIr_KlHM9M0~R zHN+s9zn;8j_2CC7K;2;rGb=NN@K_gfqO1&mj3^R?Ucpr+D8YtlNaGu!<`71TbGkcWXr7?+7vCrM|diamECyGM&10bFQoILo8b_?UMkUi!7mnM$Ty z_{Q1E%FS{hnEvGz2gcl?dNqaEiJj7VsbY{pQbZ>%EAw8+PQT10S6kTx=+35>Oqukg zh0SSNj+@pR)OLCu;7R?f1QtA{6l$MXo6Vs*0^k3Nw~5~l^>&a;m}Sn(`pf^|@}{|* z0%^WIJOEwkvq0@)5-lRkXhE7uqJ=C#>Zv&4Opwc%_D!{=z%)rr`-K?0ICu{!AQN&rF6gAc!NbpYl2Fm9gJR$2QLEYl z|34Gx3X&O4saALWpP|)fKF`6%#a#jW-Z9J)N>IiKul1||0B4v$N^BJq1QVDiOyE0- zNV@o}pm2ampM9yP_+)<%yl}#-=D???Me$Fk17(UO9)4V_c`M`^XP!gt<SY#Nb;OaZZ%u7GuE&GL7aVxJX!4YU$k;ng(Q>yI5o#I23Q|s zp7J4C^wU>uj)aZDb_hz=cu<+ZlO~(O`l+3D6N%+faBM$d*{vt^=JYs@8j0e__?q^9 zBMb_#_LuVM&V0yrYvdCW$D&G>-!QTAq0T)!HWSlp}4Bm!CmeK)R-}6>a#+dq=JNufkVNb0OcwL zLJg#AW~AMY(YPrdl1sDg>hTb3DOeL8+;ts?wOQ#Kac`ek)z-(rs`97SDWf+b9 zG479n^+^M`wsZ>pZ&6h`8$(yLWE?L;UprASq=u{izQ48K{|=X{TZFy)M6NX{p&~Al z-eHfF^Gw+BWqrc3Luev50rrbj$(Qk5f5TrSekx{cg)z`6>pz`)cg{b~gVAk&cza@U zbhc)+|MDf<_hQc!k-MD|CEz_;$fJT1k87wQj|%qf8`DD`6>P2l^@_;Gu_;aD0!`#% zrio1AJ!vBIa;Aw~%xNMQbDGG-R1>){O=NkYo6gFp5rbAJR572X{G$x~L)qFAc%V{6 zh=;OUfL9!Gt47#uP44QCm4>=#)ihG2LugLq)K-O3>KgP7I}klamXWQp z|I^<2L|a!95V5G+Q8;50N;jbrLFs+|Z_7rRL+Q2RYC`Fyq)I~wp@jG=$EAGLkuibW z%1&sFC+Z$Swj@W%Fd;F9@g_9eX@jvHGwYtRKTus8=$FB3C9C@0^%GeSu-)JHijA}+ zb5~_a=B6@}gD|tivd+xz4ZmxCm&}ssT>_QBEXeTkDdCpP#7{xz<5CPMHJKUdk2cjD zs~6Qprd?y)s)^OBlUTh}U8#(WsFR_tnbCPwM;w`5O5dyCUm!43|9r`KCpVo^vK-YF zVcJ{|z478@J*1IF@0bJv*D2}?W#2LuWB;hOn9DM=Ihk49@>ynL|kO6cR+=NQT+Em@2Nn`DBpI*2a1+^T=QY*cHk zI%HcfIw7Bgk|cw?uXN9(loaN{AhLTd=I^<g-?OK$)a0z-;(dQ`u*ZSP2TZp2k&@iJM;#i7_ zV0Wd}NGyBo0BJQ+TN4tBl46?L#dEvCkC{x#Rdxf-QI$O+jG&PMNxL5-YKt41&sbZH z_Tj7fHW2v7&LqM64${(!G9OnII;Dg-U;XQ4UocL^He`KWworD(#<_rIOd6=Y_`Uoj_|N}$`JK)-<~l*_Apo)!lg$+30Ro}A2#C(yJfSnqfGW}=3?v^Rn{ayhP{n)# zLz5x)?25+;({aUP?G=wsuPEi&*k_?ELnTHT`_VYK_+@@{u>DahT z38=#z2zwji2i2ATrY4#q$w2NuTpR2(eAQ8$8Ah00;jT7ZL)JArT>Y0osU+`LNzI* zWS4o(;X;v0yg03q1aZK$ea($z zqKOJF6~~Y?t2fwRKIlw#E~>q(4b(%G&t3DYT@I+99{7BN0SAj=eM;;LK*mk zNiBm-8U&q1mDu51u?)HMS4Ndg+`<_&=dO_tP=9i5;zFmk*oFG9lJv0rUmMmq?AnM=LFaxpn*}e^W#DD(iU2BXR@kg2P#_U6A4Dfm8^RgZitb z6(X^{BpW6(dS3|0TdsK`29+e_sKOUm6xYa9J6tSP`^25c?qlQ(T&$)aO>+qLWQR$l z9L^RdTM+^7RqhT-1ktZ@OTo=a3ZPmSbL19pVNtbn7m5e3O;5ZU!P%}FWbt6u|M$78 zXx*bzedOaT^)vy@sU*%dC$b}MB|R-t#ip^-nQZ}`v;{OJs{|~flzW46AKSz3?iR!$ z=O^Va{@mr0vZc_OO-jQS(Hs-AHW2~SuH!^5MBhIX?R%Mi>#Ta5;PwntP# zkbE2>;uMxtFZv&PU;YQqxyRlA@YXCocRJmGmNWQgcJb z(hwL=iZbiaJ+iY97m296bI%Zw_w>e~YZ9mJ8h*x7IAk6P=}zF+A^+-| z)Zx?4{+^dU`LR*SwNJCytfoK^MzbtCa6BA6y(uJ8R{P;^LMOxfGWwM z{^3)fK2;x5s}7wyBkDi7?a?Bi_C_7<_xLF1&oLI&wK(9m@ENQ;Y#chcJ*p^V3@@Tt zMhd2yu6cOFw&qo=dF5=ut61=O&!RNcD*+Ok&eVhn!?0NYqQ5l^L+7Mh2}4~QdXXV3 z>VQA2xI;^W@pU=&2b*$YU6S)43q;bnfS3|*)kEn4F|w>{u4|&h0-}iR*^t53@d0%k zw2NH%?qKj$nEYUJ7kZ&A1i@@jt$%Jt8$DxxiU7PNldtPOIr0eTtE1nq-(3 z1HL~`%JK)9eKj0kW8JFIB-~gcj1c$BhT7hqWPh9|yXeG@UqP2zRDq zo{h!EJZrmtkT!W1Z@uRSVuN460R<)#zqIPUbM!(FtyZI9s z;7r7vnF)CUoC{DByW2P1q7LyqY)2Cz=Z>~c!U%8ECbwxLI1kSvBf<=0;THw~8t)2& z=r{cPX*jN?_PdLl8-xM1s?TAY~IZbd_%cOqSrI^$(2`8?@I$clPK zOm}#!dZd}hN_YhUG!=T{JxD$@4346Q)-(!`t^X{54K{)crWUIrodVaFL0gGv)>Sqa$@9yyxoH2i3@`roA)5>NbIe9mXoa=8q%PffwRG`ZQrCaYq;4m=dq>}l z%|o(bbX&|uw_|4ImL)0T>#Pyqu?J-F$-T?u3L*{kam+ zsHd^Ic$QEO{jDrCc>2hp*i1zI@74CmWF9Ys6$KUj4sxPJKP(dMUz-3hTnuCs7Q(*C zXhlYbfDuZi)sG5bLVH zFYAmg{Kgaoqj&I(&TLN_as|qQ5(DOXtkLyYcLK$ls9+zHuK_vnV`oOt)2!{l*q0*bQ1JG%>oSPEAZkjhgH#`VaTjNpzFiR z;cUQk5qpu`K0avEVmWb8Dp-i-bsBOsK>}1)S@5!=) zf4clLh}}0oMwq6;j3f+-Vzyz{M`4i7>pS%hyL3%y9c20DxEkdt&8c`tdh)iNV6#1xb z=xxQb7*xH`K}VrIT`#OK>vtw1$K-q#?S}y#oBSYCNsAZ5sblFXd*9`~6z%Q(fcHn+ z?eYOM-e=V#p7ylP|fM z`Wi~vUB7*rrW4+JfA|A5Npz1jxn@qow)DK4Jb=@kj)$z?%H-qY+Y?Ug@89$FTkp3Y zg*~5Ftb~*TF;SWW@$5G0&^xrY&BUH4%xxwr63GD5od?%WtScrVkxdI_D&UCy+a7zJ zN!X>syCA)pH)$^B3@{`ZlDdJTB8fN3s<5#3mjG!Jj#^Ar$n zjGiIgTF664+oqsbS%7Dq6Zo)*4X|QTejPboF@Ipl*yZGA4%(0CCc7$z&|+mXWb2x6HmLt!=Z>DXLopiyF2{xJ-fpfX1fEL(sS)Z?p*>LW!NW~si2*D5(ir& z${agsyV8i%dV+&AX6CT8Sc|H$7HLuvakitWravQfahE|dy2jbSB!`cIXJa#)#7{YX zfA2a>=QpvK*tJ9%Y(ig5O$-Ch6g*d_d1ND4cjf3wtq?L?-4Y!cEB1it!=h6G8X&7o z=0wSBvtwZ8V03WSH0=#XpkKEm<{}%?1ifz)Iu?QBzI)~Ek2<0TDk$4UNp0*ez^j~% zI^mWiKxV5|gCV->Su12q67V5hFj~ac_mj}04J^n?#2S!1pEp=;Bf#Ap-7M1Reln8P zKpJ2&B9L6%KRN)aDl7t&I6xaN*bqfm;bG zX|0ozxvs)eSmn-_r%|UPE^9m*4BYDTZW3JmMAcV=Zi!~`rc|Ta4a)@Ym%=hj&lNvg zPCq02y((yI=tCe#_o!#W3J9)#dL*sFUD|I^5_Z$}8!fYirq#`c173z+?6LDy&mJL=Yo(i-G7M-s})Gt(5~iJy6aAH zPC6a&Ws$GII~yENm3r)SDB|Cy$L?->?9`qfv*a!2o@AXCn+}Ay)Ir}A-Oko%nI(U# zSsg8=nEs-%eP{%w;nT1uSWUfSMr~=jHGa~J_;evGrN;G>=B;#oC)~6}WxslHHdE>N zTl?|x$YQyA-}>laSYo{HOa$g22&LGt-)j2m{=Gk? z3f#h{b4)2rH?B+q*hfgNZ@qV&L~sdU%A5+Ic_02)f;S|m;^>}1CWym8CeV5)l>Cr8 z9hGc;!-d<{EzW9JEr$y%f#)4y;T|mV#217iItF!0il>%g2{#11Pgt@%0v|Q|L!Y?F z{%JgQ|1_SuAB$fAAxoO>$6w$*KTwN|Szjj2(k!OwLSH!q7nX>kf~Z+i_*8lG0e{FL zsNNA~SJeP?gNRr-xQ8#^Qy`VMuFWXm(^ynBweq^ckS?tAyhR&@EaaIh+lAQJ%t9M3k$6aTgnE;GZ?UinEBS7 zy;L=zJ{|k)&7Lpb{4Sq4{_*^izu7Nr+tw7VlRds=dYxrhAT}ZQTH?|Rpp}s8Oq>4~ z{DdFs?m}+!U4@pEKrsiB88)@_+5Iu~?3B@s_Ve;Ss4Ot(2_NFz5V$Z#f404RglAbc ze#>f9F{;%NB9Aa-St7r(C}d5HeQ`f*5G2b5FbrLDroZ52-lnAq(-m}s%mU6}XQgAN zXT5(Nd&SyudJz#Awiki8K6!(UQ;Q+`6Uylo8w7>yge(N7oAQV|FzUf&iw$IKh$I2= zx|dEmr7a5U*jU zgkbZg41Go!4R5uVsUxz{eyT2)LX+CShkkp@;CQc#&DqGwr_S2V zCw%dabw*}u!m2jHrcY)@l)#?$<_n&<)PV5)nwDJpj)2N7p@>iA?Dsd_qW~+8uY!9a zmoF;>Bx-y+6FIA5L%t0&>J~M4(~7?DmHHSdLgktXCu;+{rNwV3LYg`(U`f!j90{v)YHf=JcEITy@-yw1 zq)D`XHRVv|?ek8A)3jDElp?zFr^1pzK=D;gJ@wD$>?H=oXYJ*(1+Hm^+d>gPW1rp{ zGsS2x`e_$q5|90-P#7_L`YrtZs8iSW!WzWA0UK3@bZb%`V3(%YAn8n7snH_Gz?Mea zmC{HYXW2$3q3oNKqvtnnt7(6jwz+cbA6gb!F+;(h1EzJ}HRD9t>B3l)o$Suzma{Vt z;};rd8f8=hxG8665#{u|Fov`{B%nkwAbi>{i>z z^$I8IHoeL*pY*E0P}i$6R-p>FoL*-Z<7K_128Vj=0am<8s2&hjsSdYm1yl;1w2``+ zn98UZ7P;5OcGJ2YtK#<1r@i|3w9!>FB_NjlmETpeHklG6YiIFEOV(D5mWAlT!dBBm zp!cSG2$iaP)!jeg7(Q~NZsPAnZ{7dAx}T6!=*74EVB>xOzzZACSKMUM91ws+vGpEA z|3ZWz6uQ?(O%qkcM-L8FfF+rqO!*5H`|c2 zwcHAu!ldbM5f9|cCM~IqeGt2+5SGJ0T4W3#AKlKo!DuOT6_{lo4t<`X%ZpHLnSWJ( zsA^pB?Y2zl}QQ_Zq@D?6+qE$Sm+a!Mkz(JoPb?&lg|UEXYmpV*UqH5jWx=18+Q?^s|>EIyJC3uw`L_VQvU%C zkN=S3{VW6^YCOApddep&Y5{?gyS^GmP|R**f)hSh0@a+kNpmZgSTatoJS?$uaPhDt z=L3griANS>PVZX0TtD0tZoR^hxmAu#=uU5}eANfp{N6CY08~#bCtt5jq2;GoljV@bM+pOGW%(m`fgYS))4(DCC9S^L#6Wlz` zi*?8xn)oa2*uWXeTT;ZESmg-a9C32oT)0IsuQYUTv4Dsa9Q!6E88e|800=Fh8eXqm2GhFe;j>av5<)+>w3<1+f_@!~>WMn(h>gL1gg zC+Lm!-wi9t+i#d)Km+>hb8;3jbx8d&xRAFuO1E!jy8?4EH8k+fl(Qr$1=9Ll{Zu7S z%7r@Zg_lI_^E6wb3XzUkJyKDBv@|#Eo9>Y<6SMoa!+zzwWCNhvBpZF*@`%_)`zIP0z>B7~N)5MA<-S;65gllTcmIuzU= zEn;=4Thpb8dH)vi4vdx-=RUhwo7)m5dfx;X5CIB3s!Gd{jh+aTrWA5&1)d*|o0V5! zBq8mt3Q6hAK~g$bfuyjSs@iPo(a>P#%49NG%`^LbyWliELiaqAQ?gOY`n*}SU@_JB8EK{b7 zy~oXo)Yx%XiqzaZD-r95wb9M}1otyU>J{uBHwGn|_M7OH?$+cnIs$)@DUz^q{bopR zRZM{xDL2-rhF?jIO0eg8jjCg=kS)5bQM36*Wkt)b-l#W3CA6ne2k{wcfw=yQN$R3* z8Mt9xXLnDv^Q;-~tN6#DJ6#2(cqgz;K$uL{?Bm zn|e}3L;lb=o9o5 zaQuXtlMWs1OoEm{>i-L(2JZ6sY?{KTC+ewx=v_s?n$lk)i(bh$#nugU?I;SM_FK#c zRqSlVKXv~QwxufWKBHo=25mqR)g9R;3!@&%VBikD!&$b~$Vme?Oca+tJ%x^&jx6np!$}STtjzj)bXdN5r3fR5tey3iga$&jV_*(r`LI4z3GJB zy2`VQQ}G;Ds-gAPErq?1uzPp_O;@B3qN6^5|6c)&63sz95?)=#R0rcVWcAOZL@zpW zMW`T)_*h9~AN7mS-J(p-&NcS+;VfJ4|Gj*U-mZs=EjqQ*qRmDN)0Vy1bP@&-!G_E9 z$UCjF1A63I?kJkSh8DRUY+B?xHo8gZnWDq3@m%O3=rDXC45xw)SOet!aV8cfx+(Q@ zl6J>l;-&`JzM~EheXSLXMqCCl$273Lz5*UO(v=#v~2k z5=0R!X0a2p^^qupRNO^ruWB(=d^dqYq)Qm#jy!Af55B--Qmm4P?84bledju3Uok!? zSGft_|E%-HRC}D_;_Qw*BiF-b!%8h;)=<4b67kbg=+Gg-y?FM{oUNdJ^l_74c;W@K z<#O^qve;p6CZutVAM@A{*vOR|OzJnwCa3rZ1vg4nm$uJfOAn^NTV{oV$0#_cp0571 z7aOw3-o;py_g`G_OXn(h1x!&rbVR($Gu?p#;~jbTyN$+`8$2qWzEH-alV(U1tWS`s z7%hhKj=T#nkoPur22JlmQ=9azXa{n_WEQ4if~m-#b(s;->cfcMlOfTsV>4jM5?ZiqbCH&3tUJvHg14 zNiz30Bg)5top|#k?5kfcS(`Wgw?F1(vKjz1|4aIK8Rr_u{6i*-dHi2tiZ-Y^nJ`euj~5aunz?1q*vx zSKpZ*=Vpw)SJG!hMr?7X1Z^uUhvLHCHX$mbadKyVoGzNZla}zDQr*0m@C~ul&xzx5 z-r*fNO5mXc+%b&Z39E}VNI|Rl^$|VBL}~VYy8=2+ett^?T#o&!ky*hSaZSFxV%`fW zWLRJ6h4^_dgq7v@_uY?4zs;%? zShxby^S7+NM07yLZ>Bo`(XstaKT4eBmJCRechKt`H+d2Nk@JuKw%Df^*7H;Yt%o%1}<> zy6m*iCmz%Hn!A8l&=#PPkMj8ib|mO+kplgjf{gSPVJrI|cg~-SpL?cb^bU#41#xD3 z5+2YVFWqHgw%`d}yoItH7jLQmf=lcbJe!+)Ha~qP_|D9LL5B(KzCBkzNVrlNtdJR_ znwbTJQx0eEy=N%hkdW0Ma{Uc|u;2f*Z^gzSh{#opo%wUv3$joSXYb|wq(v#wR{eVo zLX=v1cZVk)q_UrPlJLG$fF;tR@x1jc#Y9EC_;G^Er?%?fken9@GE8bqRu+htO64Wh z-X0Hm{PUmfLeP*~*r+9FpxV_K5{Qk`x8yKC_LEg`>qz$&@wYxnlrv0pC2VcNf?x{y zs~bg^Q55e*{5L<(ZPj+LtWUeI_C_e=`Y~=~z6jXr>U9_W4L=}=YD^11mAejQ*I&-0 zG&#LE?Whqzw^2Z=@trkY%qyOtS3eK6n&74^A)*0h)d%AwdkT%B!x@Plbfb8+Ksr)U zbxumbDZBfT?9P~PLvXD6(^Z8OiRM*t_dkfw1Dn8JiTlp%0U_$UK$q&jmUA+8%OmdX zSWMC}82g6+bA}@3J543=dGRTc!-CJ+Q^)!cog&(%hctNVUy#xG5cU9Buc2C|=liu`gmIhe()|Axz!Z|18=z?iKE!S8wuVKMf? zeq5ZA+ok``LCg6qdDfJ7U($cS`HSB>L?1ByM;-d~t3UF7fIV+i?WtB2ex^+{rt81v z@aF$(XM8ALfIMsQT={tW+jlk^h<7zrTzzCRa_C~35r216LXq!_Z%q2)WxVsLSFYwUF;#!MC#xyHAv;2|F1tLYtvrL&`@u)O(Nc6PL^ zB+8buf~!Q`jc6p9n9E`1yM>ah|CrLiOCQv2u7u?e?owg}4TYL8>5*Y%vovuMqJl|!pS0zC-w8bDHppribUR{fL2NYjf=8X*CCT88zXF0`?r~pX&KMr zZ$D9nqW%dX?mRoqk*2&Czi`}CA-@-s00zEJD72+;6MqheA+B}h zVXd$f`}nhNY0hH*h9u;l%zbsvjr z$=q3$4M}Bnv(Bc5IfPwwflbj?_h_vgx*D`mt}==S5ktf!;lXBC~BaW7`Z;M8PThbk46y*p{N_b|F`xzC$ln(Y83B{ zk=Dxd*k?c1`qsC;=l?Hm6fi?=;Bko6teQzX$%x8Ctd>zWY+IMY399hGcLoIVTjerV z0<2^cu4si2DntctgIa_cjmjs4Ln?!r^(hRb@e6@EjF~nK9zE5d#1Zuu_eeyS5c{rb zaguQw(P~p}P_5Q^%#j-AjUFV{Y5(KgKCv@yI}a09xzDKN5Hikdz{+Ewh1G~T#Xuwm zhQa5Tii2%FAf0(rMv!~}06cDYF4=G51GnT+W;n-XI$119J#)Wq!X)m)WdxXb&Z zDI_l6jvQ}i=R)#qybN*P*WXTUj)uUfx3n|_r@)#(Hk-dLj1MM>Fxec!y_12!B8uzE zbEL#fK&OZl-$XJUSHF0Mx|ts{x`l)c)X)gvrHO{q=F>_N53zBy1&W|Uv!;$0!DkvL zx6l|P_FBE;(sT}TWNJlR1DH$A+frc%!&{2t==z35P;3k<(guK&}bR+=JiGE!Y31} zwo95nQ5=MJyx0Fo8M5ZA08D@X&3dg&k0kUnb{Hj^B4GN%6g?HyH za406E+=^GZ`kI_GqP$7-KpuK$fX-$_ZQHZNk8rr)O_oWoyqmNMC>hEN9#5)Zlzi}Z zRnM#GSq!Pp!qCR8sUjMAzRnjddVMI>dO!DE_=5T#3icbHVX{+BN!4!Z^BC=9xFSt0fRPF15U?T3&u>~V z$EJADDbFkClT%N0=7pBABe0UU-Rx@_XjjYVMS=IDsGG)8=!aQ~Y5<`EW-^ zRT+LH$D(a%DL`$h)y2)F zOd^$$SQmdGi)|OwY!=h%f||7z!#`E<}8_^J(#l@Eh#qAw4~74gY+tJ#lZ*(O4&xPnX#0K^sa9sPurj&0ng>~!zE4!#k~Nm& zB0Y8G=OjbiJ?8B&T`_}4GHCuF;wrT-E~Ns#}sQkiZ1ayz-{YVR?#jrlUFg*Xd7e2D>LI@Shg6J z&zd*uG>OTt6CW@|UvE;-mtA>o*DiWb*Ic!WLFX93_1lgS&kB;Krr&wRdiuc$EdIMOKH#>W}GXjX#W?&z52B z5kl~VMPUrmjhY*av(S3CQGHu}I!wliFE)1Ex5d5C7aQ({qwEM@D3LnY)Dig$eX)_= zGbV;Hble#5(E94qlfT&)8~MHZuJ*-7^Fpn)Bn<2TeV=yUC%(8p^tM26dF*?7v?xr& zVAI0oebt8qe`ZQ_ADZ7$Y=`anv}-{y98#YXI{d}$t+q#Wki(5EN+E+h{L#Z8SU>u= zW%!+>`CO52hfnSPL|HT3#i8g^e1%c}ckW0~k@HZR+1bd#<64^In(ZOqyb>R zk-RxQfw`X=yf)(!{JG#28IE%b(+k7vf;SqVz@MtI!u!^4 zOd(HU-6d11H?TsvBp?vEWxR{oy^{f;^k;VWc2e@<-9sdL2vrF25Hd|lV!++=?s*Iz zh5yH?3Gvp?tn)g39)IE!uB)HN7rs8Tbt{S3+u-7{E-@w+q-+Yv+9X_{KtQC*+-kh+ zZ6ivIsR)YJ(wLdi1Q_jxet^pxs2ijW(v&Z+6<`6ipZ)rHf{xO)iczYgdsGDsr_JYS zO0kcCsz;2ahw$fo)_ktOkiBiB=3!||8)j}_yfyM}jIh@}n2j?xNu=-+VaoX}YR=Vr zHB^o<0a>u<5zXV_T))sr>!JLt}xj6p&wNXoW8J{DN^q93(dDnsrWLi#Vj>OgypMC z71P)sL>L9c;E`B*ML4=eWolo$ilc^H-1%6-62Y zzwk3|$XLmUwQSnWGdf6c&ZtT+T{fmxdSNkYqK(XYX-|lBBT!O4Q~?j?+KN>*R|U#9 zC+%z%NsZ-*Od$=R?-y@)L156Bof@0T4HGwC{}SzWUo5PDpNyB8^aNM())A7iulvQ<_w$wOVIcukBwE`Aya^I0I$(`CJab}&@XxegNq(d?u0@7r$ z99o=hSNOQPKDmEqdqu!8Dk-PcmLZ7zZlC-A^a&X4BK)2qlbL7(iU52bGCc^tmz)6} zp&NX5ySM}vMBECw=Tb6XD4n2mvd$PtD`K5UPtFBp-#PfA(oO<`CJjq17U4J-FK!*t zqWybtg;CK6MfQLW|J@3OA=JCnNkobIu%u4cJBzUL{1 z3E&xMw$C$yz+1wJEPQz`T#AGh zml%dvCs8iG1LdCdk=alV!J%+=+2V@&3aduGa$dia^rjULVR?gfpsWM(mrA*A`*b!5 zii&oY7q+`~<6$u?8$&)AyjoOkssS;OtXR9Tuy%G&2xwVz#g*Dnq_d>djXMbNO~Q zk8&$fM?kcz;l-a6j>ZGj>|$^D-u)o>$e_R(r+BxmUVVR?=i5G$SG{K3BFAv+pO#g= z_o}vkKaq-WTF0oE)e);;ms8YK8y5zZqqyTS4`n8fh^lVw>K10S3ZyoRRS8Fh7qqdH z`s{&?^d6su3(0fWfg^iBMqbtSaAqst^L>IYT<@swH56r86|Qn7gmI4Yq+bp7uEG)A zgRv=fw^7(2q`uf}a}^V4k_7NSX<>=WzjnRF7wPJeb)uuH#GDdL8pabsF{M+;Qq?0L zCHg`9f;A#R%IZJrrAx#Tc49u37H_i^`WEFKNaGPJ6+=PO@GvPJs45f`$8EA7&NzYE zCght7%rbC0{Kxm~;da==PeMxqF@S<**<4>mJH91p5#~5 zV?y6nJ2sqggPa?yh3v)|{!i*ldNxfGXxYS?^nGf?Q@=Rs)YSbB_j5o^9iwD|=m+cw zae$#G{#|~y+6{0=6BT(yYz_>1 zf<;Wq&>AxUh#@P|O2^(rfffy7Z9ba;0w74txfa}1Na@7Nltn=Mfk5m<(O6P>J4H<` z^1bK?<9%hSAaB&Q0YT~Bxi29!%5BPH)nyvr8Qq8di{Ot#+*{YWq@&D0QEi4FNfvh)2CHExw(qWpQGIr=C6+n(lV zsSGtnOZ60f8LeCS<%J(kd>($9A1=0V+YhaMJE-Jyg$LUdbRA7o&1PHrK4@^&c?=`r zIa@2@01sv0OEmTZ^2g6{jn%GmyIXE5ffA2p&I&4g6D5gQ2iTk7-sjUXd>(E(9tmMk zb3!N$)NWOqcH{+KEK5eRKwD(buo7l$N7?aYZM>V0p`6;n79T@3BV&LSNt7k*M<-ZT z&x(CqIbBXn)*RKtpb;#SZnHh*P7?3xUyFo!k0zV5Xos_Ia0!gc8E@WV~#pMvIpPU)HcSBo5Bk25iPc!8TCHl_+_%z~IOPfTQJkNZ=z^SqDca zQ&CO%%tW80ojkY*KSq7PILCjW3A4j>A?<4o0AWC+w`?8f&drb^GgE0Vw}QXQ?yvbm z0cf7i7tcPca-{N0a$q2qIzM>|U@UcWzUN0@3)&xZUkD>?Iv!X!qD}o!%~*QUbf3FP z7w#q!5nkzT%m*LH&7jiP59$wR=^*A~ddH}V@5(?yBWd)Ukq1%G1L^rO@*^dHzd>l( zuzFBMWYY!L6IGGD2gSiuk-Z1`%~X-SCje-w$leRaC#s_SYuD>`q@MLgx<&o7NJEV% zSSnZc_~3Y=LbZV?SX44ZP<|*vt0g|DghKLMDGZRX$4i!}jY?|S_sxV`p87ueq-VBE z<$K|Ee;`HFo1rvFpY)pVOY0y=oh5K1wbC=c1k%qwEzN~*$b=W#Sn0wSt04?dM;K~`9<)L=S#2*>k#+bvAF9PG_Q&j|!F|RA#opw+m)0o^;|}cn>E%)}6HDMs+iT1e`*uZ01!rW_vIne&~CqS~`UQ z7MWaQ)~cYiwcP@lJfbf-F5xK-Lc9q|=|8|tfEd|f%}9$mdn^qpJS~_RESUXnhe?K- zSp3fv6AD_R{;4djqS32DMyy+dV*q-6ip#9@8FS1sxAG=g2Ln*Mgy2sYlzRt}IKjop z=L;h5d{kIUt=ql`+4EJ=0CZS?ii@Wd?^k`@h=g!&N?u$KBM_9t7ob5%dUEyxK4_B# zPY@UCqRoBuQMU$bd~}BJ z|Ns47Zb!|HTD88fyRvQHjLsnBo7g__1UQXv+{?yjCz$`51%=(`)B3-|FN)wEli{b@ zA=?)O5Dbcu96*weib$P>v~l8I#%5K7Rd6(@9m!%&R~ep@?kvaL;ngNX_{K_`GJ@oK zM+=ppIr9MGY-FFMl`7WyrN@)6P(#1W1M39?hSKMv2d1_~2!dAFr`PO3poxZx9u#H0 zh#2H5^?6TLTPpWkP#mf;Zr(~eyL5HlypJ}T>ysk9Pup-29%V?WsDbG@q|FaxM`}U1 z7ZZ1u?v;t7Gcw-}?|oNaHHYzWYnAJ36&^#MaI02KQ@3^pW+H zg`b39qA5VS6sMN!kpgI@%}?d*KdM0t$!%;vx1Ce`_i0%}Oa=|#Uue`)2U73WQ5qh- zl+XORo>B<>t_&wIzqJ6+Wiu@w@|%7<8#HQWtAhCthKX zP=sH5xC}q~*QIysihza(-&Fddd52OMJPdH94Td=pxR5qKNqde=gyJi*wYPp%wkCMb zWTR?+C^bxoJp7;)T_+(4=b3cnL+HI;d$w4sy6=7WJq4zdJtz54!m7|oUapfMrVC@X zk@)_r7}_)IzVF?-ox1y#PwZFZnn=Xf- zeJZ()z;Nq%d64}Cd{{KBg5Wz|q=x8LaHKL3iWdh1tZCgXAR ztTSXbIm{s45U1H$=lv!C) zBq~&=m95s%#yUucbZc=#ZsS*yXVYwx%@ z^QvnnMhLHO>EMMH@OoeFcU?Czx|nG@IO&d&8AG9$%*F|Vn)+cFoJ#!VL;PWec;I`T z+O*-FJ@oFuWq)FD2R67LW^kF{GGzOn{FiiclPHe6;>k@X&rbeJHgnq55DpMUV~u9X zp||jy8z(lN8Q%3ULVgQXd97PVZ>deXx9?C94cZ^uKws^Th@h{2NRV0zZr_|{En&DKvkwU+k6iJOxXA`~Jq)3e>C7v7%`F6MbCUxzRQtCOnL_hev0 znf;e+oZwMZlX<|CV0pFsN+Mkp2R};|ZbXJN!Xr1Q*TEyHz6-1gbjiJU@3<`lS2>l8 zwKU|9u+GoTzGg;Mzatw&=7L`Kn)iXlsBWUK?w+&xwd{yqGYWKzs@k@j$=O_f{=IwW zk}K>>UWnmy@Kh4bJYd-JIF(8_%%?KH!c^wr8BC=@X;?@$l{{^nB`km|P16Cla>Yq{ zlLW3C{%*2t9_Eu^%~LCDxg0`hI>t_X`K5$*2q4Fq||RMy{?{5b}+EARM)Le zp}1+hUqyf+l}3}&Arv>$Xt~sQXMd}0$hvj0zhwyK;r$qbvCZ%(8|kU!Fk0^PCVj2- zf2Ew=D%ym3p$MBnFpvab5#`Y&5m0z%kbo&04HCt5%pifLK?X@W^W>A`2`rm-)I%af z&ceD@VK|EJ&8X@k2c`hHc+Jx0wb>Ct7l^yP&2ZQwyBS3eY4c-wn`5!2s+&~9t~EVz{?ce}F>ApTBhB*j70eSJv+m_-24UH#*ZY!Z$a^`Gc`dFbW_xGmLPm@PK@j8RH_+WZN&vo=iE z{7K$cs<4P33sJp0i9DV87f2@F42}j@(6HDU^qRB5Lgku!as-Y)Dk5MGh*c6KzYBYo z`>rr5&$i$$o1I3Mc=hZw9NJW#-GxU~BAwdb@P7T@BMip-lCN9$PJc8^s=#<8dG0zj z-?i_ZV}8Jjsee6>O}@|R;=2s0VD}ge#o^-}$ZDdNuCl087W#`W70ncyhd<3#F2{Z{ zRh?Oq-#1$7G9j5doE`NWrTm9uS5#YMJgVrlyT z+|t%2ZCP<`rezwDWLuWinh6PjTxZ5HYaZ+RpunVLvm}^qYSL%T`?K0m zL@{2TqceeYA<{Cv%EhC$iutG!9#ZzISJf2dDpXZwH>8{K;1wc(*$qh&{`OL~2tiRzwbJ?Vjg(2*{&Q*{EAs(`jexyhT}#mNwtZrw)=o7 z=f(Gy<7Kyv(}gLc0?QE!E8nJ0&lP^8ih_28bqki9ABCSuZyUKI#6jemiA49PG~mP~ zbz6nvECzo0;P{%ZlY5k(_9K5y^FxYlUemlbt=lE|z9NK-A`7B*uu#UtdISkd`xP|{ zQ+#w=ySna|grVMiUpYP$#RpA8&md^uNn6I-Z*=?RuG{Z!-Z$~yMRPy9ccOXn#^G#O z0Zr!LmFm*9p=2|Boocia-!!j_Uzwb?BM-08^~!XNM^F~rK1u>^-pDWzbS+;f1Kn$W zlq&y*{E{f+dUz_EZQuM@KRlH@5glzh)RfVZPE#mgbeqyQq8qAW1DzVFMpipJ8V`Ko z;WH6(s@<{_y~T;TjRe2KLIDO9-l_gQ8wvHeoQU5YZA+D2;ihg{(Nr>Vz8(0A22$IW zIDZSXsQE3H@zZ0ZwI7WOp2b&2B*w}c2y~XEakd@uCq{%m!kVw=-UmGJrfd`u8p;qD zBJ2$yrEIT5$bBn6WV_<7zyVy8{7SM=l<30&V5B#&YGf$&-+I>as%J*44+QD#0Xfb=T&nplp=keH#JvI ze>nV)?0FBNWx%_!%2=Pkq=?Q5)l`KVeg)Y+4Eu%^(qpl;R5!I{cre{Gws13yx)oI| zuBbBnV7j@qToi?&v&qju3RAG2_%AT~mMHRRNqg)V-KO(($I1ZG|c`O$!LqAy*g#1^6_xfDTJf!dq#J!&q;zi%NxftyOuIb(Pwvr<)4q=HV;Dvq14b zvyOM}5BoZr8G0I}yq$*`nl+P$XNDVRN4tHGPIYu>&cOnq=PaUW;>!yHp<^>@?KZj| zI%XH&jHRVLEsEMRAv0cFS+!gkwO`&B`+eXu~ebV=Amt=~xNrGlM z-)Mq+?#QTZ>UcaS;UyX}9?S6YbMa>Of|e(%Z9Q)u3aWxZPZ@yE(1d zEQ^Rrvo+g`YnEW);+j=~R?V6%ShEFdR*+md5gJv@Y0Xm1SS+kr2oBNeLQXkLnrNoe zpNY}EC?D*9vz6PEiW7`^V`%0>dMH@AJyYquD0q<$q`HlkM(`=(Dn>savMp~#Ol$i| z|8<1h{NU>J?7;M_2`Tr2OkX#b{=*AT*Pllq4>Ov%s4HfVcQ|VBzG?}-n0ai9;ZZh) zDnPbc$xX4xGM$r$qxLAiYW^&yO~SPdFh6SkEE^xz)@@*zD?(e4j}g9-*lqJp5k zxp&RasA+7-0mHEUa!b+-TZqdrJGwR4TUdiVL%&~KgWZ6O{3)uNV~TGoGP`POVRdoZ zS9k4foxa&U;>V)C1&BPN0kL0q%ayab@o;8lVBw7qg-RI9Jym1vw_6uiw_CI1Ffh%; z>{h;+m~rK&toWQ`Qd`JJY(F?UE&L(YFk3R~{)pS2E&D?{M?Ajneqbw}wO(U#0AwjW ztDP{8=*Ka^dcK&;Fy(bKNvk!YqWf(CCP9X4qJ&~yM~3V0akCZja3i(I`nd0@B_ba#?7Hr*7q=aDhc9j0vE8AY``izkKg9QhXB|;O zl6`Kwk4ZHnW)i&U5NhAlGVD^C13w}!IJmmUB{5>oip6K>B8h{=pANEV2~aNBqvoJo zyxR@R1^qPEOVL3#En#hN+-~4E9cQS$2H8~jrM;kBOA7=pR|U$ov;bZpvOjbA4HJ#m z-J>k?D9b)scog38D2TN6X~2!E=uP)1%RCCl@hE$a+l@Z8y!TO-7arxR=unmyA7#Z- zZ|^J9k_g{Bh=f3^LmAM13O7?n)D^zlNe}+reOLHy|9Gq}cjCY^?t6zX^ZS2(`hd?H ziX2(+jQe2~udwF{Eq`T9+2jz1Y0}5$pizuKlGnDp^CbZhQ6v%@z zbtsQqgc9))(Y3@i6JO$y>#OLIr_lk${=?twJo41{RA!kda$(%EzK=&vU<%-Ue$bD# zv+~GGD}w7{!+nblWAm#Wgf8K2VbCaTy(dcCRtLn1siz}uo`#1a*-NG9Is`QjZ;ika6FdO zZ{HmypGAq(w&&uc=CXdBO{#V>MTuM&CCZ}#47%0a0Ip+fql*JK9h4fnIs#Cc%vmb6 zP0v6)V-?$=17~bJGc4yM{NTL+KQ~h<((=B+3vVnjG?7I>naD+`g>QiL)+P1 z>33S=%2F&+Y6rK%<-vfylBJ@Y)eAyJw8DbRwNIM2WmJ^w>q)yT8w;CP{ezG%2sXEy zFtY}jGeTvI0r9EZM@hErFR%=#vrA$2ZEpJJygBE}`TxMg z?T5cnkF#*T4a~uBQ6xyfrGG20aNbJ8=bu zCnYgNu$|3O8HS@sj{J}{UbXe{jlgRXgVKrpoY>O_XVIyd`A_+0^sP{Y5R>=NPjFsy=+>41F z`{sPx)5FnkdhVoqF{fY*%I1UF5sNks%{e_ham2yM`(WP>h@MQ-ZR2Y~I$lvlSKnWgZ~N7PZ{t=`MODK-8&wUlV5yqkURARi zRZUMPn5mlh1F4!)swR9Op<0h@QAt9@caE1G+ z6lB@h_QMD789zy5J6@H~I`s7o>5VPCn!XHQja6O8^s4v_?SN+zmMQ%ZQ@uW(oCN-C zBN@XkwW{4z4saK}QkG|59iyob)t5G@1B|kH^r|1{g;bHviu6AlJxdCkPLtDHh+dVo zKfLP$&DSi08*RBi$(4ACDdeufp-b?VBzQ+=7gjn}!NXPf5qb)Z zIj#)lAjYPK=nA}@z8{_$r5ma5KX^}j#ILV;s1tWh{cCBl^O)obPJ9@QBysJZYh1!MIr{O?;{c+&9SEd zt-&#I7SeP0jg(G3W|4@**Ujs5E(Q6wHd)0Tx_p&*&{R1=fJ%yWfRow0sjPKhSE0PJ zDSm3ro}?ZF(m5fO`8l_{Y!e$&FJPY362%)w{Sw6+YkrAb@+w5QkW4I*k+tHDr5i*# z0TMONfy{seuH&>0+vDYEMF=Kw5(K=G+>pL0)wn(Z$q&iR>6;X^_RGziRi!T0Cn{pP z56rz`gC;RZU=1Gz){{O|FO2FreMo{4LkH_GMX;fBo{OX?sf33~O;&JG|U{6SBuQmD|!&OUaE z;NzdTaQ&HeQ4jH}+1K6W(E=rNTB4TmVKwVY7Tyd}p#LD$9P#X^Cxv0BcIGQ}0mD9g+!+qamI zUa@)g%Ut1rARP6@tn~Zt?}jr~rQh_+?yCKsWi=KXD5(D4m0AJYXciIPzwBG76}3^& zKVi{L)qT%`&Wl(0YhxN%$nIUmB}DWfFdHuM*D>g9rY6-e3v)LgMKd)ODr9D)?o>am z+H+OAGL#Fo=b9|lR!RhOt3B`OW^Hely5MzN%|>PaHKRCrHm^OeK)e1jOT#>jP-8-O zVTAgNgc0hfhS|~PY8VhGF(P0B>c&U>i6iqFrV$X~H)mGff<3$-DkCKi_9~J}iiEXF z-Qs8BmW^*zg|zv6W|$SL9Ukw@CnhQZtLri;=U7nRr^iJF^}Sg^{ip6NeIGYuS?8q2 zr5|pL)l_{H&-L_`Fctk!pGv?lR{;|&USbXD6BN1$d9f_RuVk^@jf8FNk;;5O9HXN3 zRzsIpI+Yk3XGxofBVMqIFZxVv@Ty-0Q9>c$A;UH<>vGN9yhe@(2cU*VQ5^kP0R%~9ER3-t5!F;h6;j3er<^GEtCmE#Lwk`iSpizh3YA z*sl}c$9_GAn*xJcMuW4CIX3mR6(Rfn4e3p!Dy2P=wr81wp7f<@**m6Tb+heVp9oX6 zX+8m>fJ~2G_(gVR6wbmgrQ7L(IrtYu#1%Zcbm`LnO8lE%iXLu(FR1*?!zT!Ir_@M= zr^E)vPQfjz{?~!yIpEOd`!hAJpt2S;T&MQM1SrYDR8m*WOvAnIb5^qn-Z2Pb;|!-` zlLE1hLw24c&AL({E@KmybrT+Qr1K2p6%W=POb7$t$7+GuA*Dp`&E@A4km62+E0-yL z1ZG5=bvZkjEomQG$l37@m+0AmvYLwoE_9AI*y#(p5usnLp;TO_>z>JTc89g z7gBcFqHa0}I=!6hCT+`@uLNpnrJ1<1A>CBvDIK3Q*waBGd! z=P5|pPZk7$K-P=4(wPZxfB|6U?Mwvjwc>YfY3wu^2N)|>st|3o9zr}}A0%Nj+jZYh zpp`vlb|dt`&2Gf}pfIF31IeZQ3>Jv(X!(&32nU;&nswS6zF$ zTq|m0ew^}gZAR2alr77Ir#nc;tb4Ty#X>~H1y@Qq=z^zo_3sr~MylXWYuikZCBlel ze%^QnVuaqEc6af4-ciubh#w_kqJ1=`$NmD-qw}+O|MI1y!uLOK_|BqsOgUdgQ5DpV z_5H0|a!KlF)ld}T=JXZ|M%-k7PWTjIg@h%WI3GTloDH8$rU1QR7+0YI0^-;@G=|U4 zC`y@3!|%PC3Xf*myfgkPmWToXly10<)Mro2nzn?BKu}_ipM#-zXIpL5cYX6ZMJIRq zYNsuIx#9jNF&667zsspPuVW*EiJfjy-J5C=*+8N6drK)Wh>3d{9iQ})QxEcksHfzl zdl`=l*&BuX)PiOcfq_8ktL6rMvnKRLBibxw?zKr+QDek?VvPq`CszyG&AY)dkSpkd+q40i}<`_328!YqEr#3q>zY0o~^Nc;4ey_(ucL*@E#R}lF;Im@nr_!;kapG z$Psp}rT+vammxi`iXcmVnQ?g>PZ|Zn((taoEX;~maO{)G=8|(sT8CfebJDAC`#*{? z5sst8knY~Od*Gdrw*eM#=fgYRUMa4scAhM313A#VJN0Sv<1n$M$CGP99vgW?1}yH7 zaHM&+h}z^EB=JYJepSCMcl~X5mx1{l#XNGi3r0JcxR+4CDG^a}i-2NwzIB4cyWT|j z*_rhtDL9Z!+>1cJ$}#YLtD`YY5OswU&U461bw;>hjG)K2(BEo348NH;yB~fdtJUY6 z%lRCK1tdP?pTGbuJDlh!F*fLV6|(a^0BjB|fm7dr;UqMEp4 zf|H@{xoqCrA?Axd#+H}-ma`F?;SQmT5}-o4X^9+d!G{Xz7UsUsNmp$eN`tRxJ%V!^ z(oKmxTbROyvRTTp11yI@&8BSu8Ef*50?)tWXgevoXZhCILU3t?Qqt+!E3uptrL;xa9h*&8C4> z-Z5*of>=t00c!F}k%47yc>p=Ac9Xu(0F>HnvoJZ^7E@A_d0Y7bkJ>$af7&*jN#M~K zYZWaXz8#5C)}Dz(g8ft(&R!=$hmfEZe+UT@LYjcmkf6#hNrEV-tpxXq1R3(1p=dE* zDc}v9T|#>Z36j9gezoEcbrRGdL1iaFs%oyygzfBxWh!|Xf>s^Sx#)=U*~*EV{ENJL zz=ZtyQjLtcy5LRIuOkSRd`zqrz(hV=%K=Y2@&W3ytW&r9Zb zx7%jT_u8yEHY>;Nwpn#tXR|UlP|oc!PV&cVwB~c8RfUyWGj^6fsEX34pNA4zxVj zBrJ1^$-w8L$*PcW#JH`e?a*X!AVY{hzAg8Y@Y45`zl78njaf$>IYh!uJ4MP^D+7(ld0UAo3fK(Vf&eQ* z$-XBJ+L-F%Pv^o#=HSKUfRwNtz==gZM2WlGg^Sd|wjA~xw|hCj9n4mK;W2^*%02y= zUalm`*e&{S?tu8xBK;NMDrRLSC*4|oPt0KKdQjFOi$DJ99yG4|MW2K?F%LQqkag>R z=99#AZ$8PM<964VUUA)DRUmC~-P@EtAZ#6Bw6MwI{;i2N$ssf%tkpRD#4xM2-CgOO zGCHFZ!Rx$DZ%CUqH^(7`&RiiXa~&2yb-<~GF@Ch*N9=7CI?6`zxyQEzf9a!>nk-&jW-WM!YSD2e{aY#Oo|8;*aQq>x3Cv$db z@wBJ`(E&wPfLTrXW_CWZz=<>-05-}k#U;;A*k`hddy!%cz2*m1quR4wdVXuY`K|T# z+?r1>CRwu24gPw1&^`8)MeOZ!kD8T@RKWLp?^O4Bw9S2wf)dmr&*6JK==@H37J}0R z11eedl^AZYA?&<`O~45cXNs8Yg+BeXEa9>UTRGi-tcZ8wtKpf|&(0cVG!7kFD0Qbd zKJt!7UQ8Dg}{UNZltfh->TkB_E4`TFp1pO-LjW|NwT+*j0qo@{N+*pa+5#l zxYK}{X&i(8BsTeYWs0Has6MJ#~$A4s*t#0A=rKyajFYV%&0YzP{#*Hive z_`Lb~w0T>}_V=sF6p46o`17~@;2ZL@;pZPsL{fw)lu((L8E+k+dCHfd2ed>(waLy~ zBVR9QhdaLBNbmH+oo|${{-I=Mr+k1jhAs$4v~{z2EAb?Nn7Q~-`eq8sg|IT{hBkK~ zISD}e#J$9j)gWHL1%X^1L!(auf8ZU#8TLyN$S9gn$ali&NacvB>LZD7+A)pjJpX@v5RiT5HuaRr~Y-#Z=lj(<`F^FF_usYtfp294!YT`hne8>(xt0Y| zg`O=#em;CQ!MpBR-)#oF>)KhI4rs^D3azQ|nYL*IcxZlsVaH0&Fp+b`7{drKPV%WrA#dbFP8j_qJ@9NU!F&CLTf!YM&_g_-rCFn+dek_tQJK=5w&% zs5{8cw^gLN0iALUsDg9rC*6zaMwQAs!GV=<(9mGpM9p&H=xy|Ov?IR2m*K$&{_X?i zS>yYqGSV>;**wB(BavM;Im;GsJ#ccaI1*W$AdA6LxAJ4KeHcNws*Lo6%F&(I(}zEN z-EZ9es`xx zUBc+Zo;NvbvoyPmE@A9T>~>DsEwBeWw-AvIN%#}OllA9%;^D9six;V#0{4zMHAo7L zQmcwLW)Or#Bn25$Ff?nI=%O2hVii68U8n6eCAvXqb{^f}-EMb|Y!jA@>oPB@N8>o6 z8*_I~#}OtcBZoc#*JzJBmp)#1E?tOjSa+2o2}g;?%T_R`AHu7OI1^jI)3(wqMu&(u zOaep_vuVqoD%VUSWII{({TQwPVMa)!uzQ4z!kCdrDB$bt-IQc?>UCRMq?D^I#r+p#BDBJPItnabc{))QtYGKS=X0sr(gq2~ba4F6kQ_8Fm~ z9EB@4u>rZCs9*Tgzbb=kepF~lDonbCdfY<&W4S?@V1>&toRs4!j^TZLVjV{irR@L$3 z+K{V~7tpA;+O$pFu*O4!zC7Cs?u>OWf(334_!nzCBZM11n1nYD!y6m#*4YzPnmuu^ zY*i$_j`wv*??a2rW!#ozO0 z==;sDI%us2U44Ncb+1$mPTr5aHGjyS*6vUYDtTvkCILM&x_g+F3DQC`TC7{4 zPBW$?6bnzOO7&uOe}#m(k? ze!{`NC!EQYX4@pmW;=Fh&iWD4@PTaVRr-?7AZ;w*Vf8KLu!zYyL|CGCfaW2tAOlKsZ3v{DY)D1i0#Z zanr~AKb8DzyxEh0zs@V9e6h5e7{t`#|5CPZgR?U5;H3HSk~@Mn+fn1y8$+rc3AKwr z|GGP#+^&7$c=CMa?}cZyn$(CAe&i47&i|eCq&pEJYOV?-bCxmSMG1}hL#F(A@~z{N z3Hal*t@G2ACw~RB5MT%S@>D_~b>bTGGv&wyIb*;={OqggtC(M+dje3?)ucd@|Aure z47UJh>-ipU{u)Tn^4!ho8fW$4rmc{j8~1`rb0x>zm+)(oQ%RrYeRHC|60-Tdt+ovL z8GQj9@7J~UsifjQLwe>^B4%{<7uZYy3q6$#f(#EAvY>K+t@XC#)1-#~)%Swmx|K^_ z6TXmcoja9Wqm#MwtqVgxTGt`X7WC$cuh-9=QUpK?Q_o4eD$cUHl#_L0yNHho1WZ~p zZE0*a(#+y~G98WHyyw>b$oDsLnYgITfJOs^;5VjciwLmHH zn&=-+Oq(AOoK6?;9O;KwSePk}Xv&6Fek7389hbqkSe z5ni`OS6A;g@N=!U>RU|MzAS3SMV0b6)QpwIp6K@0nS164N}#6aK?&aN_BC18N}F{4 znWN-_<93%^=s0T#S2-w!jC*1df4|vHqr&|iGxH}r&{&4CuN9OE7GYNcsA4&3D5?`|vf4I`3AyyxH~z!h6`4Hzdi$=Hl?Fd;WLvDgN-q zQFp3Yg8cH7`PX0k^>lm~y-qDlkF0r*kn3UoyEv-)gL*hxT24}zUiHiQ zU7ju3V8EYx=D|A)zgjo1jWk5Pac@{*%O@uj0PE<&YJ1^*50@cp&T4lJwD9((-De=H zgd%{p_;U}JVc48qr{{uEJO}oL<>DX(94`^kfX!|N&Ap6afKI3Pw%ho9so9SZf={6u zkdmHp-w(q-yKR()e2_wpgNQ0i}M#k?-ilH<%iR- zyamqV?2|*&ke@%->iQ#hL|&b44?I^8OrCOPRG1y^%kVX~jgs&+<8r7PgUTQ|%J9@P zBlgvQab~1#xC&0-9pR}Q{!|@q&(oV3;W*?gw%q{0SNOX213be-V7{|(%oGa>qLSXJ-vucE2Y-PheLOPLk-qlL);Th1VP;|Kc{^3s1dmD`M$lF4&4jlv_cUl(&u`FnP+E5#6`xCSWp8<&ESx&O0Z` zPX{-y#3h%R5oTf_gEdkeQ~q-6?CM{0MhN?oD$KE`;?!|?B&6eI?fWc0mKg>qE~QA5 z?LbML_)x3qgS!@euUqupiZ#5q)TkBVuIYqwDLlNrHEu8pRQA=Zk&p`i`Dgnwmv0>v zM|@BH^n&GSn(3`93X+mcDF@n$XTBSLEgITLsmF&y;<~-&R>OjlUK{3hc*jG!LTP2& zoN@2D2_F$(v0?|BW#Vs$WBYVeAUzxq{{7DueiUVf@e*2{@die9gep=rMZhVY846g3 znW4561yYi3>gJim!Q9{a`}D3L5jLRpc$rr|X*R?EolKj5*;3Fc*<`9dmQ34zo4!0` zDu+j##`eb~WCluMtu11(FKFV7aCRzHC^(1-0D^BUS6;!dWY_VI<(MeSd{xJ1c09G- zS9N?&6<5Na8IC`dutj+FYwoT#x5Cb=Uvu{>w;*>TRj8?)|3=`D?S4PFRUUHc?a|p+ zWUa6sKs77CKeWL~p<{|)M5OoBaQnias?)Iy|2+{}VfeFzTy`1$occWNlWx^O!5d$q zl1RbssMWdLc#VJ_pl~>lwF`NDp)YC0x!srHFI3v1w+P^0DqdC!-{dB|MkYTb;JM{$ z^z#ltl%)0}1E%QJe^a{bE|~G=|CyD8Z&E;RoQ|?6rSKDeGzpK*>NeBxCM~k?eF;nWvwH0KA(n1yQ`A*$vNR-&XM;M7fX)9y}Bz89VxH1hU;+mLw){V&5W@& zDG6`W2hSflUg<#46;IFB?e2QL4?A2^*qa9Z<}y9T&Wq$PY5YFajf zz27Tp6~FgI_*6n8yhtT)RN(7&G{?TtHO#^E6mgYs)4sT*sNOI+_^Y)8zGC;A-3g<3 zBN|rv4F5Q>Fq{g=z6I~23&SZUMi?%chvAfpo`>O*2>~~y3W|9c&OS}dKThNyQ)Ea= zI)28dT^KGNU`nR`rWs?Tp~Wh)@8ANSxLif`IJ9K_42JZpj9OD%hr{rz$=R^;m+)OA zJTv5vKB@07T{+`rDPttv8ml&5y>s`@&dvp^BDea?LiL&V(}R`oJ|O?fw4reG1?*SZ zu1>dXBcanxIeL|pk*c)HL++eJO^>iUunaVQ75mm8Vy6}J3d9yt5vxEvg1eh|k#H)* zWMTU}P=CnHyhX8fwcN~Onc-t!$mbrj_4B8}`ZB5MN!iFs-H_5u*+QUycbbViQ zv#4@DMKKVIFOqJV&4QW>N!VBnT)Ebqz@bl@&lR^mXsuS4mp%3oQjK1$d+e*enbg#= zr2g&F__h6JTf@($!Qa+m5_zD#<~Bo4io7;*`^wkhNA9b^BdNn{{~MM?xMp02YeWrQ zH{Zu4!*JX;Un(+>$pyno#~V~ya$p%rWfTT9>L})EuoyAHj^&X?7;h$~bH zsU&&ss2G-je0B}T!|s))-~5LN7bioWgt-J(4zw?4Ss|oAj&WOY{+`Nb`<%SjopOE| zlPzS`&I$s?<`{&8=`+~kZ_~5$DLUE?eNolK(Y<*lvZsfO#Yn{<=OLQRA zq_}U8un|OLjtgW5%|vD;3B^ej0Ts=gg`=5Z5i6C*fa%O{iSb9k{o@2}pLEZuITJe) zST=fMj(82pQ|{-zDVr+O$lh5SC0|+pHeBdj@rLx7gt>1>pGiz3xgougH#h3d@#G|W z_atvJ<7g!r8d5p}nJ2P%hVMmN;0-CdEl{y&>Mdznju+0mSv^YP- zJasj5e&p^&=cnwPA9Zx@aej25xjuFv&W{dM%z9&u%&St83POgp-FHBn#eT+pf-lfO zL&z&I!gd6%Cqj^&bna9gT^Pl(8^!f*6pwzfQ7o@+6n!x+hsEMZmJ81T9J7wdYYk}s z%QK+y68s5A=aX)!NwS~e zS8)_+Nwd=Z9uCt<_c6N7^KeuZ+&c~%24tZHA1a@?V?-hI@Bw##Uv~hR^Shzuawpwe zh#Ax;-JAH!wvNI67V&l+PpVVN&+@roPj5~i((B{NlTIbiQiH)!fto&KroCzoUGuiA z6)uWQ1mUvD@ly*2X^Xc%G|T&qouID-_ohukkU%q5mYAJe#B3v9MZR<1LfU2LW~?kR zJGU3Jt;>rR5f%w{(mk+9i}}v0>NH(Q&LNN0#N7e2q9G!+#&=j49#SPP_J0%ckl4E9 z(1gEJKi0s=5xGf4dWF($d*;kG^F16~qXS`wMHpCuvZGWj@L-@hoziVp#0k(_jo>Th@|TkwwGT0iN2SBHm(-1MaTGrs+( zgRcV4V<&Hf^M1&+!*Jh@+Y+lq4kb~z?;WqXknjQLp12QCj&H4TA5~S2z2mFd{Yu8E z+6qncX+iXxN!9aC3>R1*EzOs?;5qnnOeK1kQkpQbP~SEl`J9Y*3jp$XnI1Gh%C|wo zX9nt1$-8AZ9#cV~_{7O)zL#z8(xo2gCB7H_z4~rskwaxNvp-0(#K{Aik^KO7uKCR&cmCZ%0DHmQVe%}asg|{c0 z2COM<<{kXmYpr>2Up0jz+E^mXrvI}013~gvG9u0sceJ^f+a1|^-~4&meCCVY8NCjF zzdM6tZP%Sap103vHJ|!o_hPU^s?quqM+q;T>)W`DGrzRoz`DdQ%h{sp^;E(t3PFtG{h$~iAJpM8y9$uZV zi~u7DLy51$A1Bl1FDOS&00Ob#Hv9(@cUF}}eV;$gJ!ivz_p?P*norz2g-_VGjHyUJ zA5ikNYJSTc!vltS?;vw7-!)lli1!w*Ty>`gLp&=5xuFD~1|fuf;U=!CaAjAg zHA|-Xn5sq|b3|Lq`ny-sWvba9omI0Z$CDr6Ihl}q)3&C4PN4Dl5#uHPRO7xc!>5xe zURyMp$}pj3A_*Uo4JAl?3OzrZ5#b1@Hzh)ryi;V#mo$9i>S2zRIP~`sx=X>@6ezhM z-8sjvtA(Un8qG(D8BW}NseIZcpEnm$V=`zP(jpdDNwm?=9`D~y>MdM3c9_A|9EqHoj?4$v3ln!BsUh|k!5h?47%5C6lz8;mNc((5~@6Y1*GMtz?UNO9|u9}L5 za*W&_#uay?nB>Unh&itcVMyp%OqK!UYN~%3X~@!Pdr0GC*$O}Ohq!NFkOZv!a=hTs zPVNgR;i(@@lJNOU*%qxnJ|{Fxi)lKg`{C(n+jXa7q(l>ru<-ox^DpxF`Iqweps>{4 z8Tr4I`~&P;;ARyhxq%?PZCOve0A~*>C_jpm`l31vS=D>35o<4^pTnM45Xr&<>VYFfbE^5VY8gL3xu;U zoO9iP^3j;NGa2it#WVHUITAs&dSRC4VsF59*ZVSjBxQXZIL>$H!q1TweOMBc;7^!AlSsSL78i8D%_bU*3>@ zRO-JBPmi+!`1}Xb#mhdb!Wk(4ClK#XI;>R&LSUlXt+=gx(4oq-37;!*8)lI%8hc%A~F^ z&-$2WVgXk;!8N`x1lxkON~-d=8281|VUnr5jtvO39Lf9BRfUKculflxp4pti!Fu~< zSKXJv!4|}f9pV#qakZ_8Hu}ZVQo2;O@PL=p;W3Bk1s-kT0XuJHH?M_g1u}ciEL^fn z)f299`4x5uG|(j-E!-cQ0cPx-FIhLC1BD{g&S)h&(7QW`XK*IK)08vNGhUX0$ z9wd41$*=Q+03d4|`YWEBlg3hedY1+Yl1H-blJT-f@2K4olkP#?d82kGYHEIkQq839 zsJEPTVY1A}%b2Gnt|B#UD0xTn+OAJ$;y12*tKOHdLzH2+TTaSfTdxR`xxis<4k_FIBsMRS$~?7 z-Aw5=Zaw2@=$Xj&ItG;`cF?8R7}(8_x+CMIuGI{UeBFG1W_5fRx(gR84T+nuUgI-A z+HaD6`0orZy}+HU1EHli{9zHc)sd=}Rj?2?+a&gn%5t?rVuS9l_|Y*EaT?W}Kk-q23%hbbml8T$c+lNi&2*`Ps3dQq zE>)sl5NS{(cGeg&l=&eG8vp;p(r?2y!E#1JD9(oE9GverpLX>Jp5mJA^g9`v$InE0 zQ@h40KNRUMAf)U;B`9#IwXv~bwS9IrJ@_Cz!`J@MT*ZsXTBnB0F%#VP zhcIvY%zEzo;Th6@(5Av^Nw!m2=WDcuQk(0j@=e<25xy6|E^iNmrs!3>cbG-9>`wbe zxgZibhCRlgkc?L-I{t?I^yMzY_(^`5B68o*7qzt6u8iyKGk~L;GdFzdZD=Z`t;&A_$OU_Lppwv z?}tYd%Gv8sZHhq5KZ*YqEB5V{F1{gc<5v{$FW%?ES1w!pBH^oxPXaw{Ni8Ttzp^=A z^2>gOq+u9L#pBw)Ax*;e%R@n5g|DY$8aRjC)e=InB#pN0hemNYVDK??$4V_8z9BsZ z6XfP}JgogSe)!0gJR4%LTVS5~_ZFCEn!AAZ5&O()JH)I6Sa*CFs}DUIGno7AXv3@9 z(S}i_C2jRcNA@VeTb>IZDq48_)&7Eq^d2a9*|_gVzKViZ{F4y85rBv=(GSl;r2BXT zFYCB9=YppKrey@LlHiSMU;5D_6zr=CN3tjS(WtC`gKXt+QCoEep|sWh_Q()zC6Ec+ zpx=-|*a@~2Vm$eXGwa-jdpJ;yJsYb|Uhg75vMnIvV8OeZ!1?gwQVb@J!HDk ze>V0sXB?+)n-^l&I2|s$OA*f`y!Tyw{2d;eIus@*_)W}n3a2$u#_O&%xpvQrDhvO7TyI0A6bBkHK2FYo+c8XgJ^hGxHZ}JN83tuQg9+~l5f?ye7oX@ z)056~T(mDMeR93b!10RUC9cOIuyL*U6Q}GkCbd6zSODN!ke*6Hsz`E!xr6jnmOrj+ z&Z;@7W(KL|t}Q!s0mX%Fd-)O0H`}%$u#VgID%uk_eR+TFiCKG~_Dsipg}H3oZvIJV z&&oGw&(t@Md)uadZFqKid_jAv`f?TRxh(Bj(FL=@c7Khp9+}E}xNKd11ZnIqYS61# zn5sWWaUxjrA2F9Y?%yJ_-B0!D+T*4Zu%U0H*RL__@QE`ky`b^QEX38_8}$?BqV|N& zLR{Tn)Y2<$)L4ikzZyj?{U;%6BcGaum_BY%TRvc;egdMl+HKUMN960Ge$Ino)h^3i zTsBn!$kAo0Vg$kZWyXE#mk-twqjBFahgWX>4?kbhZepzQFB>gjy5u*uvAF}ttl77B#-GefA+9@7z0;ROzT>aW4d zN>LT)h#Th?N@G5uEEN3|3#Ad>RK%(=6|(CNE;Apla0mOF58+XWG=|ik0%exw+dfrw zWa6{tzP$EB0ubF6yJ)&x>ks=w{I=tW8|S-rJ3m2FARs&%1NVy+@F*MWzrd^&X5 z%$8+G9qbEesaEZFdMbHh?UUwhnL!f2a=cVSVC|P;B&HU`SgC*w^|?Qg9U+XW>QB0c z>Rr{J`OT?PyAhUMaUCAg+1X;?L7PIfWZn*1hS9K!e{G;ny_uBlPfOcF%Eayh7r@&U zTMUU`+mszLKzFT!F;Qt0erLw#WCcq)TL-Pfyjlpj_<=fA+?7R%}vKqrEUE1KU#Pp&J>2 zXWPYP`V6aShpkm;!h`)*iIB+}KeW3X+9p0+2-?{4%)_zi_!?h~hqOfj0OKhwBAn?* z6m}OVi~5n~p>n+BSLEUF-Oyj-i}g8%VS6aw_J?+-g!^4eSP&|g2XTZy1)Ji5Umchd zt}G=?8pT(ngjbcXAp?<7Iys59Y92alVQ=|wo9ta}k2`fl-sO7xWB4{IIq zO=_rp@V(eQsX6F?3xjKT;8zFUZf3Weh5M}<4jBU#9|;52)yh3GN<9Dso!y$o=>+5S z<&gIq5v)$Yl?_s~RC=uZ;vW+%HsqShSPvvdf(+;SsFg4goERk)gzdkbSHdGCC2r>rVwLd-yid0?^=WJ8aT@u-s5GJRV-UF zucr34tD``L#)1(x|_FwY{8K9WQlQm_~<%93Be1Q3wG0#38%SX`ao#hid7n{G# zYCYSEUuLvOq%eN66|mRRO>&L{AO+5{iF6^gZ6av}Z|^YET;o(R#KK(q*`BLnDk^E~ zsQV?d%Lg=;Xqg?TQH!hC03MREoh{wPg{$o;GY2!pfyr3WTjasjGg$cyg)<&l?~%e( zJogRhfM#c9FuVMDCZE%1;j@%>llISE5iUJ>D?FN}1!|sCI^fX#a!dytdO*&JF15Y^ z>HGesb$Xulm4g4ky63yHP9>@9SeMo#>jZig1#Oy|8`425xIw0ja6;zG&FQ#=8_@Yu zk$*~(itY!;lj2lDh_M%T#0-k)s+4C&{vy~GvuASoHu)HkCcZzOh(&7J_Qvo?QSPF4 zSgWN}aZxFtqB)2W1+b#b1l%BQP%6ks_L&I8Jvmt1aoi64)u&W6onTaeKnyS)CyQGY z_IIj!K=py*V1qo08co8^2h=2}MHoJi0CG~G7}jcE)Nk#WT&=!A{qFQ^(o~y=oeP%q z12B0EgRR^4xjHd0_dq^WN526o=iyUy5i@nu8c!3zV~c=0gPhE1WoM+4=4VSBC3)LJ z@0y=6jBfl?rNH??{a+#Yl8Knw)*Gtu~swv8G4~ zT4~K13mh2&W}3gbPvbTLl(_Y3`s&#kd7pCPPBXTu%HwWnU$Dxz44{~a8-^2+nUQx* zNYAqo^99w~)<0wI`3bn^plUkvqOrGDNm>=*1Wd%Hp}My6{4{OOVGd=Msh@eR%^y&3 zHkS?x6EP_4I0MTKe|3=(GxxkveuS^oZ#&qes(&MYp|3X5d&U*>kRjrcUZC2z^yF{m zfP3vbYL4ulLV9yrVc%67%?nlWl99FJZ2CoNzW{@#x5+}Z zH{sKI+m3Q(yS_zR0pAn8<@wM?>869@wEs_p9LfYF8ca0 z8+2oo(c~)OpujIb9~-r`3OADRb^H{iqD1Y&FcfhZ#@E?E3>ogvs*Dm63tgZGTD6;u1=@;P+O%C*uB%u4FleBC@8 z<;;e=_G?Bf+AXaNCu9(I{JEP`1-*Da0~lj9jq!17F~}~gs6?|rTT*?NR42G`MWOFm zOS8Maj-4ysb>ySassiCgxGvX8X4-2v(tE!-KkcvENMAJ`^P^Gt-0Q7Jf_HrHuY2Jv z{%VUR3D+wKdm+(=wEHJq;?6C4oc>?#-ag2#{H*Uh@Auq$?(II^YDulWtyafS1fBVP_UR z#D*nHWgRRT2b_e22_aR(E-zkUHYTAqlN1i#1);6a_xYXMw_DOMCiZT^*fZUI?|J#X zKELPX`+T2lwg%ghRXB0w<_=Z9N@&H0oFDc-H}GdDGd-k|dhF8c{*!-(Cu3ZTIAc#5 zaSXULY;OQnu>{mm$2BC+;D~3-T8D;-f}jdCeT|Hyn@;O)YKEs$=T6*sx+&byN+rN~ z48RMp7h{Q3pk$*;QlL@@X5crr+9OuyDKkR;jB=NPoPM&yN37rqCE1bc5G63BMWWZ! zX2IXR8~NSZ731b`xa&n_ut+UfVZ13RWaKC$*%a5Oeic#6BiXw2abl%c&}rIKkZ_Lj zCG;(-q1+bx=Mrap%8#-sSk=bZ;Z_hd$e$&w@l`;yCBMUBK>9UUfjKHR6TO*#cx$1b z^!xpO-0xGSGxHyhpBRweU!6AVZi$W_b4$Y3`=}vRgzWJAO_8s++Le<4 zN7&5j4kmKc9_0hMb~r;&xqcd*C}2+ErqRlyltPe zB(}dqHHz77QF|B#m^(+Y8@R(jkQej=Zn{4BvJ`xoSYbjNc_k@F(#hx77Sm5ACTq(y za8-^uB0WjeOk@8su^k-^L5$sadz5IPmFxid`$%LiMkzE&{ws7IE0IlhhsZgN!w5%a z{mHl`*`;%;l>11r${I8_p5}H@97BvN$t&P#RU(!Q5DE?6{d3#x;^kOLRG|bMxJYCw zr@W90ySwqV%W>_>`X?cwITzQ>Vzqg5_rtIQCT>wRRiBoeSe#>FDrW^NN|v3#cZ=48 z(h61#EBaiL{W3{@E%rR`I}rjt-n#Bmf8wd}wTao8BHp2&7w!FEk3;ZwLFy&hL>YQKevuxDoW%*?Hc z3d_fOQ0ilZM1TyZK%($!vy{+Fhft>*f$CkTwcIhG*@-sT5a&Lw9<%jzbS}ND%{qh3 za$O%_T9J(Lz^@)?kPZBoWL*sq5CaS{mC+k?1{vHk3vL+d1*w!>2DPkiD!Si7rL%!c z+y0c|dJCk*yM^iLC~}7%pSBr;*$%PeH14z7X@wGN((XaY6?bIs06#(!$Y&y0c#e3H z7yFds^~i+0B=&1Y!2;3#Q_Ks6P|Cvi>a{fa_wY^wdFQ_tL8SE?1M52iB}R7ApL*V9!KYacuD!%yY!E z1h+~9%yUFXLgE6yA4ptCLA^ke^EWf&q=UL5IJQ*dh@wYAh^=hNzc;i53cWG|6oy+A zB^yu}zjdH6gRRr+%B>i8-y)Q^2vF?6oO5Hf?Q z@~c_gDtM?1sr%-k7COI+TNTo2*d5CF0IQhI9J;Q!7A`=k^XaxNR>%?y5jmuM;Q|(p zhtN5f$(ENBTYoDeAtx&`Fs*;GXw&+qL=?6vH-T(#JRl??kjBIZJ=hb=%W>&rwi#Q5 zgojJf0&;en6T%3gF=1C*torj-hV5`r!j!5LfftUzB1P>0ZzANAxV&CWzfmMh1M0xY z&*(puh^oPeOAS)!2AG&xel*Q;46vqQg?mY*f5QT3Q=+r0sMOdcHD-w`y;HppzpG12#J=E*(j9o%3Lk=f zQ4DU?l~yl3enbGw!~S7BV&Y{8Dzu4z1>13sCR?m%lXRY{jYjGq|AZw%4%=m=4sPx= z51G~Qg19ef$*k=JBUdoMvdeMMEbuZYopZIxxp#A?Ie}eoI@q~*NRb55MFWO87k@nQ zf1Ra0x3Ky3Q&#KfQxtlT^+nqr8^u&18;GFAn8k6Gu$s}H2e7j2`~0)^Ip5r}2i_uo za`OfDM%$A)D#Jg=RXhaKl5ED!aX_ZH5!wX~j+J)haoK>!`SkbF_NWX%A5-s50(S*P zGZ=q8nXG%p-%+`N+a5Z;Vq@Fbh>g?9Sx#_2;{}v3SZRm$CKukcaPUp@ZVTSz=H9fh z?@h;@yV)FBixEKdkW?z233$VQT}7B(Y| zo~>4U%+So@9tWi&!g3`o2+JJ>)Ak$vo@qOlf^se%`dg}w7bbl{lWlYVRhx26gsa?w zKR<02q}aFp4uy8Yxf{(1a-tHfrDcK*%fciyxjV5Ly;q#Cr91Hm`|ti6FS+7QJWYrL zM30C(hE_3x)uq}A495=DpoV_8#EIU+4;QboYZ$XuMKt^1tmIBqD4z0I=bxEYB(kEk z_h-cdd;Xm|lftv;*U+USF&y>$=OUdSIfX(~9x0m^761G*TnK$UdgJh1JRqDixF$0! z;`d>jdpREXcGwo-kDQ}D_rHBN-4^@ruij0!$6}$x3gbl-jKH-nVn^sH?Tb2mQHL+| zF;N(E+M4g5lv61^`ofpqx!kXAk6HUZ-g5c&xXJxR)m-WMi(ACa7?7S%-0-@Sn-GEx zg-a+}rs5bsS=2vOh&5RLZ%sv4IN>2tC@;=DpQ+yw1;(pHCt>}zkVI$gzRdPR5 z=B~$%{4}%-@xb+&!IhTS$h_!s)WVT}%w*O|CrO6E0jd(BL1qmP$gCCV>1{wt9(PEP z_mX<9xuz^j{FJi18nV1N>f>e^fi~NQ3TJ4=*aBPe&ppu`RrfNIMp+_Sh<-o`)FCR4 z^$4%CkD6uIGcOC?k%Wgt&CnfX-yUI7hd2ks;u*fH__VG+n6<~qGMOa9`h%G}_QY64 zF81DXE4n5$sC9*gy9FJOT{&)1_Yaw`gkNGNs;hy+)N%Wpd%8lv5lEgm`^ z<_hXa%RGW|C74IEVe6erCwQWJF#UBVQjlpS>MvHvA(BQUlPhJsFR^>%hrvX%Fk0s~ z?p*r1S&7doa3s%>9G$Urv;w0HbG|KRg`IZ3-T!8MZJV{%lecq9A1vEwMzFWJnHPPH)Fob>o5#gQm$P@p4V z9AjKF*7>x}&&6e1T#ngunZBPH-}+QchzmZ!)<$9s!A8nZTM!Ea^B+uXi*qa#agroy zlvI8umLgB%Wgm+z) zp$a}O%&m>Inj|G~mPS+;VNwJ0W?LGKzp5^m#w4^y+&@=!S*$KN&P_X^I1?zR!Q)NN znT_*mZ8oqPx1o(2KPn(&|~tj&NnFhyYZ7>zX12RIpNAVd1oV_L-cQ>Yr$a zM=q~_Ckr$&Na(#Q1q#r&V9;kT7h8jcY?5tyWsfjt$w-eTnqJNzV2CciTYkIF^wg!N zlA)Lhhnnj)_Zw7VNL}tXwtVvKenWE8Wp3$IQa(a$REv9N?u2V@K9V}zq%&zC37)cI z_K5^)=^_$nx(ASiNPvx)A)iVr$lEdSBC!u4WR{r0zzAHv9B=uAf*(Yq0u@y0CsTX3 zOZ_KQvoAXnB@*}?OsW5C@z%m3!fEfHnEJQ=O8`5h4+Vga!y^xWB@#c@xeR$AfoaCf zVlkOZ&&Dl$9PC3SbY^~$m@S~oAN+!9m{k791cF>tscECpLTD(igk-N2aBP5}hSpm_ z6K%94Nsy)yCS9^)dXI)R4?QF}n7h1E4|D&azz#@cmZ4oy7Lu@F>wn`Km z>*EIo$wJ7ym}1yRO-A_PS7M)Nui&jQE?q0ztV9oazV}G{!c^1qF#2iedD_ijlk%Ru zo@*I?wN+K?Ip{C-Tp)W9Az`}EM>-3Ah*b^G++64*y%sL$R6i}m9FpcK(w>v`JRj z>rAi~k`kFoiLhNg_}m~RiXbJ>$L6I3K51}B)Y?65Hj|=y z(C7&fq5aXI5Qvq5C?znI{h0M?zm#nwe}R+406_l7mrR9(u?a@U(eqH+g*mgWM6h^I`^<1)P-_jU!mvZbxV$Kg378-a{^Nn}4oxv_uu17+Pysbl zDiqYs6#ht4^{kX;+CbBqR{pb6dNm+s$Y%ZmMX{5Z2?2+5SfeiI4rTE6^2*Au4eaqzBICp7Sxm_+OFuF*Ys*Y>!YWS%jE zu9`IS?f{YxU|c%^J}HK=&MC=%&MBO+i2#8 ziVR$5HRdIU0A205> z$geARjwnhqqDjZ_p|Ivke$-7r-4QP^z!dmX1t3|+pH5~yISSXO+zm;_0$z8DMi5eP zePB|GI4ZCc@YVuXR-D5uUbnz+i>8Yt@&SYpixo^$@m$Ke)2D#`R8^R zAphmy%g@D1KU#mB2ugqi=j2G{a@te0O#o2A)q0@U?1|bu-E@+`w0=2euMm8?6s_qu z&_nA}E8@!#S&c37)LI??D-4(D2X@7zjyFO(j&wnVl`y!Z?#99;zbW$Px7rj)D9JjU z1c{~-XJ==E#rHA6Is*)XEZJX5)tK3I3O}J|gIY)}VI6;-R<6&k14NaZc_l#q$sJZL z5h$QsG5fTJy13HHOk94e{wz+;HF-&?tMc~?)o@r75q{O+k~$AFe;hcJ)uHY_Dn^D| zvMW5I5z0~J(pZgXOr}Yw`$=#*pd%$1-5PKd-K0~Fu$#Z6J^lps<$|&Y=}{9)lTi7{ z?+$rH@($Hj#4_pBHWg!C<>l+_#pGPPj*z-4Bwb{T3aDdnq|`1`^}F+cF8HXm_^%Nh zk{J!(d<*nQdkd0`Gtt^NWjj8ix*5^`HoFuZ<8{UtP*p1|GWKx&!R%7B>Z1Cmi|Suw z$>L7-oQwPmAB)8kB6U*twbqd+_?faZ(P`~QmJi;xSpLfrS8t}6CnB{pT|;kL_h0Ny zd1ww0u-=Rt5@)gi!9bB}3kzq~eENH7TWURdN-b4Q46`olErc_Zmp|Cl+Qe4i+X{)a zFyw*1cl9G_ne5=iI}@F3WB*&x_O%SY+~u3il8b!?;s z@Wc}pnJ)UDm@8^i_A-)9YQY$;Y_-X|r1a6kBzcZ(AR?(81Yd|GLtC#krloaRGzuY;MK({@Wu1m|T|P9|+@K8>>mK7m~2ulB?^VM~11% zkz1;Np{P{yl6lR}L{<##R9Z4L%RJw1`dq>On0?`5|JyjITucqP4yuz@e%(L+DuJx; zhop{8fbo81t4%{2ELCgZ^7>~A7gyhv#HnC#7e%!9PBM`|y>X_P9E<#~37*y!V~MEW zG)W>n8;Oe^P>dxyP>dx45+_wwcrKI!(aBJuLxGK`E5>5g7-!`|=1Xa%z{x_5J(q3) zA)7BH&IbsFFIa(c+<381FURLG9H~N$30eOvyCWhgJVqNhUH$d7#91lAdaF&EkxNce zD>e56cJ66(+sDV$VED$dHi;4w*^6xztmmj%wgmfT)g^A(4JcztAx>96GbQ0}aq2&? zz10p+Mq;wfueZsx?LWXbl|}a_E6HGO=}(&#SYWRE_*#=F@}k2pcOpiCdZWo*5S_>akZ;`@{Qy6n5j+AZ8yhh({}TaJLc0Z|J=4ur_Bi;i#&VS zpvI~4>`@<20iXk)JHpLJLb}FLpImR&eAJBHQJuMs^HFoi^^MkMlDreoku1^tR2sP< zmG9rQx$`}OQ$*T7?Me5h?oWS8BH>m9%4K*quG`F~SI0WQ^Rw|X@mJJ|mFewPaG}J% zedy~#j$!jW5v9Hx+YoCjc02n$Q z9T7Ps!3F%(Qrd~z4mltv&WT8}oYw!KFzTBTLr!9DsGQM%Zl=1ax z+fZFL8L2i-JxSJat6ZM?hyEfNv~8DxvzJFp$t=sGO$404OOujCn;`93c8P1rqHW2d zZCNx4S+t3tO!XC6v@Kb*d{?=)%b88cndM%6A9vp;uDvH`cCb}B6;miJRchIlTmB2t zGj91Ww&{GUje&s1p$segH>v#3-X)jo|A%(}VzkAaG+E`r)-Zi&_q$4s%Y5wzt{kJ( zL}%l-aTt)2{*;8mx>#m+lWkvq$Cxkt;3*PDZu{!0QdwW#yn8pj(X^qnDt5b@JUP-K zeBZIxx@$9^Y&8YyT)SC+D;xJKnTdzDswQ!bOQ>Fy5I0V)xZ*Z(&)b%EUTEy!K6M#8 z7A?5jcMJsY1*0&x?~q4i%{x+2*yk$zy#Fq%7w59MO{Oxt+*;i>a5>~+h=w=0lQ)~z z9cofnX_2-rIh97=%x?BakUg&a_FJ%kNu_lxLc<8NK9ei%P@A~=Dw%#RyX|_npBklZ zLHmF~w%SPBD6Pe9w|HBR;RwvH>q+j=v^ga45@gwJNiMtBiV{oVMewEWu!*(9;iDB- z+F2^g3{^fYtCGIe}Qi&Vc{zs)>1r)aG*2Ht5zt=+X zpV}QFYo$_76+f1Rnq1)MG{@^P>J?w{vB0Gnqh}~J{c5;K)AbAK_wSJPT{1!%+l#;@ zn??F=)Zg7u0U2pZmvw^iVz_bfcDvw8xA0V3j+;fTi*no)T#QkXOq(>>aS6O2*!ewI z-iG^E#i_Q#J9v26^qdK2|0~_4i>%4a^{!G(d{LJu@mrfq*Qv1%`pj zqFHbcG5gJeCNlK2a6P@h*tZUdlQn{5B%xyoFE5y@r`m*QINiE?ZZjNx&x~l2BL(P)T>Gr4_==&ury1Ha`=Mu+u&!^i>!be*^ zeKOneaqjy5`(gifJ^$=BshK|C^3QIk#0HnQT<+h&e||aQzyB`Ua(&l78-vgNe6)2o zj-DE~1rQ28xx!4$f8Y*Wp|2Y%H1Fz_sn517X*Qo$P0xy_5+`B1k7@(^?X@nVX-R$~NWyneJdb&0p~ zVI|6O?r-*{WD(X-Q1ka6=2PkpP|G93uu~01zW9}b4776Yvji)T+P0rij|nuOcbgqH z2y7aW|8fqZ>ec9(X^Ta!N|6{qy^Z85+(o>+cPYAvqsaASOrGZgB4U^Z=i-UqO}G6n z=1$Ov&qU|sSj3Jcb`kTwt=7taYp{%x5oqDabOv1uE>#}go58~}h z(Rp)jl~zS9a(y(C_)>Hxw1Oi{Czx=Wq2e76VzWf9FJW{QRbNcqz~|4Ptf~OdYS+R* zPu#=ot^BSc{guC&Z2OxDM%GOCVfpUXp`^L%&G(~JDPbHHQILUGLtlJ$?NW5U(x^#{ z;hm6>^{ISog4y9^3Ku4D;b2?n3-v5I9i8EpeJ!CL=o=VecyKhr2isMKc2QWOz8t@U zFig3;gX+?4wS~M?Vj7Lwom7B`tKI}HEi*&a=hxuJSluObv-MBb}n93^a`U9_Y)%D5JL)l zmZWhfrg7pb1&o&%7$a}{{xi{&m;xE@Hh~{>ze=rRn6K71VodT;4DjKk|6GKeid`?; zFFaNhU9wFttFh}>l9>fFYWH;GrGi8&DkrfY9K0Xuu zv)J+x5&}nB);SSl;6%Z_8sq-D*jAmRNP97A%4EkAi9WyG4nR1OlmZowkHNsNj^Z^= zLy`iXK>3nO3Sh);aykA(g1ym3^5xiNn_mFTa_cAm@#RO7*HFF6jpU0eHnovl=lUNh z{6F~^*%dK0qCX(0=q?Vn-{mP^jHmt&{wmxkuSL`PB?zK&zdH~q4=FLY&I)-ch#JqZ zdZAbrt7TU93E;seKU@%41JtlRiv2Io{*3wam;QF{U;W#p9^iPAjpjf3SGdFIhm<>k zYb7eUr!>{d{2Bj^}pQx3A_n1 zq>aScgXErd^q0Xt+YAWdxTe(58}6-lCYt#3+aaN&Ym_nq4>aoIzgqZY)~M8`B{a&v z@Jd9DMTQm5iI;y=kW>vQ?n{|v{*UgaTMJ1N#g+eezn`1JL|8-H>2{!g#9GWieb!+x z{_r%h2Lc`zVAqqx4W1mkjGlyg zUO6jc&%N0wTVm1{-m&&{x6`bxDWm1*f3&DyQA)wYhJbA>Jk;z`zs?Bg!T+1CR_s_S zE*1)W|B8G5R|_B0K}?l{lY8n?gnQHWOrnD%7Y2=v{kz}Ksd=dvb9nw3g|I%aP3AzL z{^ww{L_IX{=Q+*7El<0||J_e;@hiI6zY~4@&-izW5NKR&;6B!{{n9V8Z?A0CFKI)V z1Q{tDxr*h|ZQzaMBDyGKjdtU2{f|8UJWGaHr0r{CT`6bd>VNQK)T?{R2s&5=zZ;wD zTYK)yaY)(6X)RA(b|$(Ld)SohHT4hKKIKND6+{227(3+|gxTk*-rf~$msp#gjH=(r zRwZGK6&>1C}@2{~N-4=by2V^eWH_CWh>)7?GKFDm~h} zGU^kh(-XX$q1`zz&@aTH)5fgf)Z`CZ4(gn_`V%eOFm6e=aHZYZ!c7>jR}$bui56}c zuB2PIy1;USlg_hG_05kWsrFXx{Aj%)r~ce%u_*$p#8=vEd@QfqZ2UL}nmw0P{~y$B z*4X(ePzIvC;>xN1HFAYeF?L7dcb`{~Qm1ety-Q)Mb=*WPfO`GyVxT0t z3RK>G7br;{umb1msuOHo&%y5f5k}WKV{E=bwBWrL+H$cL*N8yY;u{YWbven>e1)2h z8VCgTwqL^bBdWyqyZ_~>LM`(_BgOrj{VQxfZ1jez%+H05!t*M+repLg7b$Ax8;~ii zxVaFRHcD9Edow$~TfSo1VgP`ss(!Fa%AM>vg+zsq`6+DBO@lrdS!W5AA?<-lRE%p8 zyj_XqbAkU7g&`+|AYHgEbfOah2~LL1sA>Y-B$#Dl5DYeBhrJ8vt!BW(=H9*ukJVlS zkk@++Ks%a6oYR*-*a-1@pGGi*DEr17$jSNXq8X5XU)^lvR*gYff34f6w~?iD|0b%h z4{oN`0aKCSkdRZ;Qd=Xhsu-A&lVqj7RoHtB7PB(TpTR!2vxFhcXQ|}K{ynxwh)rk+ zw^L44NpEw&h>rBKn<+t}SSL11qz!}zi0d(Px!l}x|As(0jDqtsHP6i#Uag^+v0lz* zOx6*e$$=Z*R3KZruxgZ7uHr*xvjcp+6mKo$F}Nw13`CVew(b)|K@v(a+9wI2T<9cW2X5sY4P>ss4?bNGiXYv)X|l*w z)sGF6!A}01i?tP{j8fAT>jLY4uo*QAcI2+lO<=BdJL7Q0b~W;MD2>Vv0XS~x2W`Cx?f&t8<27 z;bwzN8$^`Lg`nDunnj>7omRBBQU6qQxV{OVYQDqP(er=WI=XnXfo5@TX0;h9&%K8h zw9>ryS(mmd1q-;}W`t7NmSmt+7HaXpPPCU3_~>>bkkr{uL?@`4Vb`kD{}Hu{0`WIp zebX)8bi6|L zdN7?{M>Omc3!*s@HH0)X0RdwH0sBf`=t*HVB3^)i5(@!!I}Zdz$C%AQcgPPN&4e5S zsYr$3%u@&;NWI)&fq%B`(O$81ACLn89RK8k?|?c}JwWiyt1|}$AANd*f=`qx4-kBs zbQMVEX$1sYAZCo{;-bLbZdCnl5|?wKm>n&GR7-$lM9Y0v@*IvdJUPb8L{x$i%+JN| zFp}ZYKvMW}g~fpm=8x_YD0HRIuD1#YPXY#5!Fll3Fb&(%5w&Qx2Q|0cSYv_ZLIfL= z27S)nRf@h3uzJKrrP>iNpiqDfK)R*nxTQ2%SdUNKP32HfgA__kf|C#7rqf82G3CAFU;lOvMGxMPCfl4v)pNSqP-8o&v zJpD}cRQ00OazO_SjZ_UB^G<|M#0!BB<9D*fpem7O9K;3!XWMwkO#ZfCp{if)%}4RT~@{y?JW zB#8?mb_A;{C1M4LK;Ik`nLj;kIQQ+u%BVZOYH5-9xeHg|>F)T~l$&UDW)AfZV`d80 zHwmC#s@ZUw-TgNKT`?moQ2~ub|Ef{=%Y{na>2_H-4ACt+o=LH$hq6r!CipNTLq;F6vQ0!-w=J?n zu_n&ikjkfDN%d2Xt4PK=;w95r#A#7LlM@+TOA*@M<@?OjwLq@Svtd|PMzeB=g8@RW z9onZfjp_+N3_8qmFt8cRgK_QV=4=NESH!SSW{OX)A*wQ8@K@E0TvfFLpI)D;mRI(& zq|}Z(#8iuN#U=F*E6$h*O`=TXL;gd5nE9xFA@)5(8IVAj(<VGYowo5t{-ID8RQWn9P1u4BhR)rsQ~vIkq8OU7x^_E4=#a_Nl4J#4vlnbHo>eJLG#c-u$d z)^k_@uG3BLu8!T|Ywh8UU#bc z^p)rlKi6H=bIY8Mg?}g0X&FK~GQ^a>99Rm5N~BvhmV)I#t+8wz1#TD;>4y6#3PR#j z*N6J**p04Xm9ZmzwzV*VkEQ!3Qma7j>2&kXfCa3|JqExdA$81fd7_tKIm=(W0QH0I zqfmR}kC6&egC!MA4GvOa(N(O_wii-&NyXKcC&ZIE7|$phj_@!9;}(z~rE68pZtd6c z%P8quVIc~zkTK(G4>`!h{#@3ocQsNzY8UE%!un?;xV-+gBCK?jg3TRxliWy%RS~>p zG_I_F9jFZ;HH)l<>%a*_`i@f%05r&yrpJT6r#JbE0pLUgY?cB5T?nmJkUe{}cm-5$Wr1+r6?!cBNE+8!xqh-yO+W>!pvBd;0S z7BUa?g4X=zP9tzKQ_o^*Qa*7-Gf1ysDIz7$KQkq9`Phw?N|dP|V&Kkwy5xV3K? zyTl!JYwsJApW{};->dH%3nMP7fPqhJ(qI@B09S5t9tzWWIrmgDlMXzka!Gr1a|bYb zPpEfd1j!Y5)Q!J+6uF~rogak5V|VnXCXW&>m=-o?Oo5Bvs}#9D2cxEFjK>}_0Ta1+ z4>61MoVgXS?@8J0UiOH~u5~ZI?;IKcD4!(jvT5;P5{2B0n9avR-w>c+E*4ttEne(K zM=EPt{m8Mz?ZhZRFT`6x1Ck!Da8=X-iUOg#hG{ZiG2)aceR?^1!j=BUZU*e3^!cUeaWsX? zF#z_8q|OpZ{afINk?Qabu2cmcXe9YmS z#`YWeMu%fKF~P4&WB#0wh1Dcc^GWal-l>%PL+En3tIo!6X$NiUtM|afcpzb97lh2~ z=jr?jvP0l$ybX1p;Haj6EhFvV`4K6B!X^LpRnm<8iK&11$!pDmf7qV}A-`mt&>-(k~41-O}Jl3b4YKlo7T(k%+}_KQBI_oJb4O8RPXtA_35smuSM42!jup0;a(U=>N0Az*qudg_%_rc1;8I98AaRRH za#6N0L}8@gCa9T5+;N(z+E^e&2P85S{^85<5h4?};YkA<`qcPxye#6N)IW@BaM+EN zK-A=J{fwQMOKyQE^qSZJ{KJ(4tIyMRcViBnf_#$|S2#w_#mY#DTv7iaj2UM&>RgwLW@$ z1R6U5IUx8tN{3V>0(D2|v7UjOG35F1B=q|0otF|N@W^}|oBfCmE*UaMm4P37D3qJ0 zGp6V|W7)u^4!g#{4d!oI!}U5;t8fJ_)&EM|nzV^Lui!0Aq{ZM%)cime6f0LG1|Lk~ zSXH^v?U0|}I~P6DA}zg|jvlL}?6o4twt7Y;K!@MqlkF{EPg&)MusrL}lctH(Fc+Mm zOiS?wGH@jfzyGB>EQP2IYw)Dt4Tx>RcAO7B0-Il+N~5NgPMvKeljMqPt!@DS?7SCU zz#6-&rQ=$veTW4F=fKd{G6nLzm6#cSMIsP72s9P*we6u3pm#35$Vv_?LYnQ_IPzcr z93h?bL{2EGEgsW!KmZt>^}Fy4f#2qx=8jlogk-R>FIfh9s4KI7z)5r#77sA$mA8jNzLv{COS$Z4EcS@=Fax~pYJ{pr(0i$ZnfZs?} zH|-;GESW^c<3XAhah#{J9si8qF%#pv4DeB?4RJe7a6>LLf%epLS-1+dOD(W=?Y;x=3NyHJ3xv)|3($@4`p znoN=v9slxN^&x25fI>Q?YO`Etr0MU7* zW&bM$Gxq1;v>~fO`1jEMl=hPWve$n8+TVUy2IbK2+MikqPJW>Mx{mTK9aI!9_X8bB zztRq*a~*JS!X|Y!6C$znnnl+2BB$X>S9RUL5KCxPUmhlJ-bt`eUdy)HJj8bbkyT3_ zvt|+2Z|IV8?z&%$AjRGE;n(?YH=efjgYHCTey5Sh$P^NZR!OE;-CC5oM>>_xgrLr_ zZJIH}-&m^6x+}43kCRX$rMjM~2JnaCX>)>Uh^GyeLh5PL@Mkn_S}p%YKMLxYQCkSk zR-$7sVjQNY(ic+r+wfzu@>HZ!20U6_%>HZeS%m5>N z?}E<>5t*Nhue673&{Pj~IlJO&*SbSKyG}$=bwLg_QqUS#yUM&bN9Vql7xSo{ge%J9 zEvIfJ__dd(%}H}Hy%|D;Y4a8wj=Ji~mOHGgU?xhgxHoeuKuBLJ!&=ohQoL74YY}ej z6VpAmV2}9!oS2ryoV7R*xrxYfm-8 zL*4e)F_oHJ>fSomJ>glZqj_bk$=n(_+-p$Z<#MPV(SlVNCD<5g+wWSV@uDrdzlPmn zMkiCb+8ty4xxO2YY5nTEN2Va4%eaNv3WnzZ&RqPZwulMhxt?73tYDJME};DA$i2y+2qh+UC^Q%2UM)%0M6H!mXPFoNiD@R9&ES8ZGxw8I^q8&&bm*$wc;z`>uR@9fT z7HRaNTMzgN|60b$GpK%iIXS*laq@gV+G_hU1C%3U`73C!B%T7vC8%)NfBn0pU>v5c zH&XxI_*ZV!+v8?!bH^QixqZtAH+R0Ld7FzwT=tNQ{A;iTxs>NLj-T;%(~!y&P32?R zjkg;@RlD_e)4J^C=8P=CQ=5^rVa|bvZ#K9Rf<1Q1oq3<-46nLVFNXB*X5daohd1y0g z-ppaO8M`-sus!Wg2Cq---uxm<4eud5p7)sXEQijFFCS#a_g>eG@1Ya*?rX$DGz+cX zI?Sysv{l!4V}eThp?bo23;U=frRx)lTIWhKKKuT=lFQLCR{|Zf;=|?T_^RvIKV1m2 zMKzedH4xx9!A-yFuD-RYan$;!QX zGk&&xvwO>X>FoedNG1zhIt%5rca)8|ElQN*c&vN`HHxB|P0tN1DVigrqzA2qAo$FZJ5zyhFRN#3!F zVP9=W--N{B_p=-!a+q9fd#>tu6f{vo+KPW;Nc3*pEV5sQA&aT!juP-A=+i8_5ok0g zl-5>TKHg^P<&q9~ER(1qyh(@0?*9(UW*12|c+Q|0oQpi+k5S z_e9k!qVztd+Hcz7?xcJ3^Dmlrz1!XGGW#2=KJ4$#N%ux}xAr!c^Sp}=encO+X3XTc z+V8wa`zmlG|q?Pzb)F%Lj9SiRJm2Aht_0!SFt54vU_h_oiQq}9YY?@RR<7A&s4&b4c=l#{mgDL4{)1@J9YmW^^HzXJd&d!rb%^(Y;1*#E+(x*y zEOTO(8uV5!SZzm~dtXi;H=6U`OWU4OC&UTLeeg1zTeA6*x^+))8|m2wpD2~1HyAgh#?AAN~I8RDm$1W8tIy0P@Xjdq#$1bY>DT#@{v53ya=Uwt1 zn4Ejy3w>|uzjK?!&+9MOKOC#9-I_~TOc!;rPvjA>gq4mi_HpU*UqKEqE1l&i4ut)A zvip*M4_tTObtNeJUetenalN5D*}0rw2q!Y5icL3?i$1EV-_L9neIgb+H%p>yN;cy~ z^Q+-BhuQIR{BZ&q(MIx8?9zGWn-`!7&E*znor=7xjeV)u1a{B)=|=J~ZvH5lGX95t zmF!G0z~Iz{yeA3Ph)AyDE`|SW>>xOj4}cjWq9)cw;Xb6BXJTQ&_=ku?k*kl!Q~wix z)95qgfISzVZDWFRtEhCbkFO)AMbBc8Z#BZT1#id%3hN!@Xrbx_OI)-ty%|SK&grUW zpOx=*lDrhR)H=gCGf7?$eUAFQkF7;DLK(&h<5Eo9Vvel^sQ)}$i^dlCBOQZtCWWIV zBczmx0X%e0CMzH!H3DjIBL4duF=JH0JqZ2AtP$~4j4Gyas$Cu5h@KaK_UR<~TjCo@ z&qOZ=z7aJn@Qvt3_(n?jM#}r}ji^Ey_~$Q_aR%02xGu#2P|I3Ll@hm1v~S2FW8YB- zKVuc`(c2&i<^~LrZ~r3z%i;-7SH=nJ+n)aN9J|c^r~8TF+7tv&U@?m;Ef;AAv!`oC zb^+VAIJjhl`Rj;g{OAC`M(%VBvB?qs&6k82bC3{Q=LJNB@;~FYq%eI0nQQOb1DA?(i9>*{O z%-BFS7;m)&dS!COE#q10yOi{JJpjjg+Ll}G$fwj6;I`yC_buI7YS!JzKfB%b^p3G$ zcBp%$A^PR-H{4iTQ!;Cc8}q+Aqq0A!hNFeW zC{9xTz4H@s8@|f&qSt*KzPdAxh!DCFx!kGiP31;&C7R`|-RTu@oaUZ&OOF#%yk{UF zg5OAntOMpbYm2}_LFM4U)NRtAk{6L%@+xKMR>&Xw{-QY?5G$Z9lz2l3(h$0mWl?(s z_l2kwn?pW_HrXAqGSi0$a2^%#Jh|SC1ey~nbHyFS%Lbm0P2NDpSat8iu@}G!$m32b3y$cnG~^$d)@gNzU&TQ)ugeb@RRLX2`tC zeN^IWtbMuF7XDF0T;bF2;5bQ4d zj-qIl&(>Yp^j)QGTlkOk%!SXd#on*^-mjpZ2?G`WBkA6+$=V{@*)Kw`+IC6yedOL#L~fD2nF)z4LT?{NAh5D2nF)z4LVY(1EMg?u_I4XCFWC z_d^H%Ub{0+4qU(Q?}rY2zjkMA95O0F+2i^9{(k7d-{Utl0^>VxWCX@{{!<)*^*h5X zilX^{@3RID&fCFxIXLd)ugO!_?+o)Oist|Q>a=zKiTwUmc_t&D|L<3)t@BUh_y6FhuHPA6iK1x!-#b4MH$a(pOWa%I`(H42edIIQkgjyA zDxXzLIxD$`Oq&eF32n^Z>;z`%e~-L3BxXsCrtVi|!`Y{w-TU<7?E90m-#URS*dh9y zTR`+uSg)#_T-f2=q~zG^!;__S4blIGy7=fMx!4|6Wxb|v3W;pKL+Cel-Y?2S_Wi&N zX4;NhQkLCDvRb)A#MO?tLzH8TE4L`TJJ2f?pKZ&1<#J0)L@Q@SwaY)ITrTObZt?1v z5<|ty&54e~nv=D3CqmZY31uCUck?>>87H$VNGb{Xo15!V<()t2vd3HoO6T!$I~q5o zI;jK(Df?onFlC?UtJFDkeS+vH)qA>rAzfcc_}Uc~N-ikM1kx+I9FufP${c@P0${$3 z)&`^QZcV5JBj5_!kSv@>nJ@DI;0SmW9U>sgP&kd}dYRupth>q3<^gTbEl%5lw3&o8 zg`QQKKa=2Y3;rE_kJi52p4hCP#Sv=xoF^!Lqmr{%Q12>~7jzJnX=d;XpQvG{iXysNJ)jl^%$RK5zxpJ%Ev6cg3MG%n`y7cz2tIN zOi4O(hZzt}#Bk2)`d(PJD_LDe+A>fbu$kF4)KrX6%4(+{L!jrVXd|1Ek^f$31x$kv zYe?EHIbKGU-zg8yzm+zp9!L-yXM)R*2?dzj0H@zuyBmWK_&D=43i|!B{?ThprPogp~7qQoeZqXgt zNEFvxWVFFH-Y|q`@DO`d5fhB`#xX7x6N8?vV6ecL?hdcSypv5UWcjz?0DxhXtj2 zpXF4w4#fru0mKibVm)aH9D##Z^HlYOC7~wNKa^J0N0KDni7Aj6lgOX$WgdaQ;Mt)X ztb(Y~sr2bKck!uoGQ%yUKBMeBm)7`kAiKcxE66Q$x6iclv?r?ayVXXQq6J86VthCW zFOBS_q54(~^q-PZ2j35w#i?}Sa=lvVD4AzGcFXOti>{$0>@`A+rh+{vtF~3JRXE4F z2B?S?qodB#4iZ+Y>Nk7wh|poZ)mA5oim3-NbpVU20c=kLR63>S|5jn34pw5%-zqp# z+WcI6)@sc~E(@d8qvQ&eVf@Vk7#$dL0o1JkU01^7B1aOj$mmRT$N)*wI!@D$-bQlT z=Fr@?m!dqP4@%Q+{hk6f?*r-eOGX{nRs1H@{cX?q|=3lFL}-4mm< z&xB3ElGLd*JGCVPA%|c|7C7oSydf#5gyv|BMUY@ocu8q5xuwi=xoUsFJc00B%)O_S zP*b8|ZBL$iPtV?S>;Dn#p8O4Bn)sZohZP>De_jZYT?zMNP~=hPK#`cWaD3klAm;B+ z-2n6v&Iyaqo{r#lfq6S`Mo2KLju#F&n*J33*=nGSYE&&r;lKrI51X@8m}a;lbz#dX z%~Qnz0}`8hJfOAWhNSdJm9-Z9=NTI)APSe)KbWp@Fd?ipJgzVWH4|Y85jqy+n>Qu9t_o&a~TZze5gKu0H1U@pTPjU$HFkGi}L}J zCBmws%m&LyoL?*m<>Yh=FW*~uxkWj0YAr)*p@o;DRlx&g3!<{KIdVgmr--ai6Yvfj zw_tPFAzT@`(C3CpwG%&wytvwqLMaheQ#46Vx49dEa9eQcv|XH%_sV1-)$O<*sVs-Q zqJ5-ZPk`Tk{o#yX=}E8NVz2gJZfYnf@M5|;XyvuE@~VT|D_Oti`lc>wh4kHkHRuLZ z$N~+bI?>v(B5{$jU6^OR@A6LDCTcvlpHl57MWkyBl7grfWk+hBb<^&Xu|&hc$BSWP zPPuyBr<-!6uq{}7>GCN>;iXNX%cE*B`8hvA2l_nUbX1EUtNUm<5mNw1pvZmuQ?klxgaeCB#5<*>w?V(5Zk3cu#z?&r z%SKV*ffc>6(d05d{ZETFb(BCR(+Eeq zcpFkx=EOBqvI9)70&=b|TF&VM1GO8spO zfa>9rG$qosPw#9Kb_eTn8x|@c1c0N2p)U7^T;*aVU^_=iPs2WQlxWzaBr5m*NkuIB zndm9HUuygdmp|rG7}vIS&^^X-q@h}E)G#vLnV_?k9FPDYXJl^kXb4srsV=ZMYIP@eoWkQU$PO7RhU#2y78nuZv>_ID9xD^B;;S&6D!Xnpl?jfM4c`zcgwofrbERz34jCV78{Xdcf zkYp$RssVIbR-QloxIC{a;HPB?B{||)UOfm`xQYT!-b!=G#1>g>2EZ|r85%L9V&pUY zq@E|uuX>(_VCU{(mmzznIjfYUhegoPVNsiq86n`6c+*AZgpIh~#KtZ`l`hG#Kw%+b zig4IkG6oBDS9Q0rw+)(`XeWYN?dlS^ZT1t2#fndIu)BXa!|vKJ#PGX`cOB?%{acuw z(s2XhdSIf!P**>(9E^V9Yra@8cky$46>A{k<4BX36F3$Wb<_IB=cpU5L zbzJOn(mm?3&0~+G=Uuk>6F>VSANg1R{FT4+E72qA#J+VdLz^aJ=T!n8^jn@(VVC>< zQD`n7Nv|kjNgN5QQ-KBi#c5kvTm#dwPZD*hQE;mK3eJgTsKi}TQUe}+q- z4fM>myKZYEc{C_L!-bV3)<0C%=>ADKO+jnK{CK~P_4rcsXL-+O!|9Sj9Q1;KryuX) z(?6azP-G zsaYrKV{_7d=)VwcEhJH#RQ{*_O|C>0{}1#_T=_5j_eE2>p}(0?q3~mWFV|mp+c#x{^st@fdgoOmPnmk|H6+LS!aoVx`bt9`#u$ zGtm;;hg3gsE`bswB=4=lTFF`{cXm$y$LTqP6c2W3qn`facpE^cjS{uGSiQe;Aba(;0dUWk`ZdEpz*QZ&X-A@2@mYTz=00 zEl-m7Hw(Du!#FWh4;ZOqT7`^KzfECDFI~0W_GhCcp}cavJ?u(%c)MNGC=q8{(`6C^VgK9)?{Hy+9i^9gZtU+DHOI# z#-ljkZX&0|&W;`tUu}O2)J`;fpQW zBHhSSJQI<);T9dRs^BRb`5*jT?iOKUjhhnHt=8e-#F%JhxCf+7MX*RXGP(eSoryU4 z7!JRPV=Z^XAQ%Q@Lq4SSeT~dA%6BjNe<%{S0Y}ujPp(~x))B@1t3oto6Be->LX0}} z?#VG$tP;dd+H*FNlZI`vcJxKHCwvp!YnnuDXx&`vw_2%w~`lI+sHJB8X zQRI#w-6U@VAX0uvffeUZE?%2W5|ZWQ%#Y?*);zajI`0>JPzujJZbeSGbd_{mLX0dl zY&Lwjjj}_h?J$&^4_=Hu8S%zKtSO!ybD|j%d^4Q5V0`z@GvQ6mGdU1psue4Q0}V>H z0Su=pCs+i-DVc@qKHk1|NxT=-sZz!bxwRSy4rEfn`NL&9{3uD#!YiZ&y$ovq;dIiD zzrnW3YhVB88UN&pvrqo~KmK+9=ar{9{=;4yZbWOML8&r&q5IlHQJr@kNtlU3J*}ZwHt8kSHy+?*OUz z-&Rd16gzmI(2C*MCwswAm#GT5a(+_nt1BHJpNZo5aQzpKH%c&nE8&+#z)$mZ;ue7*|Kj~URZL!RnXG=+zi#@JvUFe`Kd9J zqfqXnp#Uw0VcY8x9;rrOy%#K1>>dn!$E=j=Gkr4_RP2Zv)f`JEVo3q3CAS=d2|aNU1&yN@r2f7Ok~nNa0frA4}75O z71F)3rPpO!ZDb@URTC-90GibGKrx%mxc53^L9yHCIw`AL4alFeL~6Kppaj7aZN&7E zU!FqgK^(sL+ILFQ2~AJt()wqO_K|Ht6^(Q8d0L<0_pASjQXuXdA%T_V0o6qGRs8@8 zFfWcIZdKveJ5g$sSgDKme)l!sLYUQki}s~d(ezw=I;4ua*iRH_P^2Cq!+~p#u0Sas z7%Yh88>#S`-YVnQ6KZwoeC7wuLJ}(A9-WIG5vL}iO$!BBR`tKfmOdAUBwq&8q^1t~ z&KHHyVkr&l-&9Uns_G*5IHXi<%T%t&w*sE*#@k~OdA-|iR1$9$)Ph8lGL$ns_X4{VcV&V=1r@7 zg=EloXFEQUpe!XVZ<6h=vK&@SBT>{NJ(W({fg5%xj#FvbR&H?PX~ytOG%(IeEjHCh zB?Uu%=ZhGMg&io)fDV$usmsKQBc7t}LTmp)DaBSBtSt#PJ;8Pq<_Cdmu- z7De5J(hGJyGJ_iJiL!hw5q0SMn0mVG1B$Y4Bp;xj=p^~Q@90)rjv9oV;in($w519cXu;!IWS~L@co^QOU8jS zMrfaDAcPbiXA8AN?+BF6X$V3YX3-A#8bE;JdtEUmzv2geMebu&<{i7tKlkpjUvUZS z@TJZxrP;Fm2JlA-nUmn82yRg@BTd4Or-aYdJ=GRz^Jn8D1OHRWwu}9nwI`D8F}%;%fC3grm!b)C4biK}h*mAL#G(IRirxln zjbK_Pwj!>pU0;(#7{B!E=(5d8ScHx=4~Z?c+2F6!+3NUBBX*K)Zg? zPXw1~{BH7OcEw$6E(~L|@Bzlez+IFvaS>zULUWOWD7oS;Y}RmQU1&Iib?ty_FJM1h z*o>cVFE$r+W)0lMdn|~Dx0>TTL>20ffb(X z&bU>7VyijJUt7&NH}?NDnl|U%*#B}gZQgF-gq7wUlDmookNmFK$aBP)P7|vLK@GQk zGgsUu#>|K?gM3x&MbOYNdVc5eD%>p|$@YTFPo*vh&5^GOkLTKI#Fgn(m4^s zxpfi6xswo#m+mc>;V5#4V1HPrw8pvkJSkFRSGtE)kwfI=WIQ79E^xtV5+y-xgICql zIKrWSn1byRY=}A)8#mpB&7JSLAs`jZ~G$^`010D~Nn{-gp1+O49z9 zD2cJeftkP?k4mWb8R9{I4q4y)1D6vSLpd$RrO9 zy+;_ZK}MWk$~cg)FYv}fC+@$?1&Y1B9fts7LvgwSDL_XMF~mC{bK8QPDz>)c7Zy>S z(g#OAkcXxL~Q&{atguQ0^GBgqvSW!9k zw4Qggh>_)D#P7nv+5T|whvFxohH^!hppk`QL1NUm!1g4GDxG=pMR#Zm>QJocwoq(b zn_p$Wv}dF+EfawsKyG#&W-Ci<4T8SO+;X7f63kfDU)QJF2VEpn99^q!ks*Y{ua(Tt zT+bbvwoBop()i+JSaD*Bc8A;wM=YN&--aQM2)ul_ksF4$w?jUpSOd;*1AHvoPyciJ z|7{YnKNG!c>?-|WT_|zG03WxaYnA`3G3Bn9Ua0f3Ivclch3x7I$-5Fx$!4IqJIgZO zeu83kqP#OMe@)_o%j(~evT?-?Gn}y9m*`{mZ94!gOh)eSWEEuyprc0cE`^h~5!ODU z9uH{+AOe<+QEUZ~zN`fb*47oOo7VJLN6RyYv6b$(c|tz`BF#XAe#8MGH7?x9pR}dd zGvQqLc5nH)Tm*Z|GY}FE-c0fu(d^^tp42Y&$cy?y^gLEBvRyVIZ6^C^3yMvAZ7(5e zbU9v^$fpII5`|x$Ye8WlXi0s&69AXeS8!=ID=u%A@y%Y?47FE+RGJ<5;keJ-h5H=$ zSULmuw&~W?52hg;j=NXk@~1|Ldpk>51?v>egB0ynWSVsNAhX7DwwB~janS33`i(P0 z%exT<;5Y?L{)YFAOpR<$phEE8a(k3Hft)n!(j4nH-$=eM=Eik2 zC{)2hZ1%aUAbwAtb3ivYqvoO;o=V@(dLIhA+xM?U)B67`)?EKo`hoT$Kfd$Xin9YC zY~UWm_fa&t%bLQiG$Xgn1F|M}%gxBGR7zteg4s+v8{3uX7jOMfY{@8#fo-66VJe zD+_o#!F0dVj5vIEn)$wgE0QTmvejJQLXj%VfPX%2mAymRDvS?A1-%@FUYh70SuaPk zUXHq6K0tIg*{qkuENxPo(Y{{xx?W~@GOd>v+y(b_-!g74%zBA#r3Cozi?Krvj|vas z{DQmqE$Gnh;$Nz7!l|&brHT%V@Kp9Hs`$B{TY75TEDejwR!`k#gc^X+i3;z-=rC)n ztGY5$%c{xE6#70|9c?7f$3|43;%)D13ohOxRTQQ6Cu253o7EqQjlUAqCo6OT#D(kC zpG})ZMtgChQcw$I9fpr-*osZyv5tZCm|Gw@kg20aZ@jH_@_q5N{$jznlAa|V(<^Qn z-yQE$$o0S-Eafj!79p%+`Gk+y(lcZ}o4h%~*-wsK?IUpFtju=w*eF|rB()y@6)E>Tt0skHmL9)Kw+UY26MV|{q&@6|{-;K6? z@4CycDSX~tm~^Ivu6McbUH7|pqwR3R!u4xhpN8whD*FdXlldL3v-ZM^?t&_^)ZaR8 z4p;v9AIO{B9d>Vh7ZjlG@HdSTclhPzf~$giKXYr-_9zX{Z?%)I;YVrsQ5yaxrlZN- zo2=oJxrXOHnO&H);R0Ph*KoT*4bOcVuHS3;(XQbaUvw8|4bLlg*k$hMw9TH(uK2%9 zCpe5QVDZP_l0tsi$6U|al%ZdW0|dK9@6 z^(W)D)_GF%&^34AMR#Q|h zzwEtxlx5dl-+A^vk9%*Os_N6Nw%k^A%)Teeq8xQwZM&`aEcY&~*3-6TV!~t1T6w@9 zo_pI_E?GEU6G>IF8X=4wUI_>=i30%{EG3G?WC6jk`~Z^y91{Y=D_9QB5FQx>#>RlG zp7r_u_PO`mdZ}gE8P?3SmsM5wp0m$B`|;bq$M^UB)q#))iIr8kaifA7YH?jI2JPZqaw@YV4x=fYspE}SdVJc{48_b-+0>0p9umgy=okp- za57%zL-Rt+$NDe}CmX-ehWyatkO#tlZrXm?%`1Mxjb9lKGv0?l)UT?v>WN8sVRYfk zc8~X^Hr{z}yvZ~?#CRVT&aCK<_xP3SJ?lA53Wb}-%bZi}Fi2|JCR{4qV0aM+690ew zTACzehuz5PRb8Y(-4?4W+3NgXwpu5Nh3|M6Ucr%8T<zrTk-v z3#`{KacB4HD!z>{nBYGTD+^O0`8qDKPC7xXOv7+ky6fR^7{#h{1FqGQb_!M$Np9Qa zgNdaD4#;$zqBY8INCDH%SE;*rh)_s&oeJs>sb*+8TM@IHrWgV@QD9?<>KvSEOSi9e ztGL$sq*2`AsSX@RU^7uBQ}!{wiW)- zit<_m9z=C!E0UGkqeLK|_S-{lkThxKwW_W+gLG#Z52v0$Vln97$rcB=N4kUO zc?#SF!!T~1tWv28%(L)f@-~!HL#V*SPHEMJWc!u)QBT8bia#}!ufg4$M&FB?r262v zQi^4vN+>uM%M_4Wwb0QjdUO|%;%xo{ud)VK)jfrybdkbQD}_m%2+85xZ4x zeZIcZ9V5CYj@-;N5(TN(z3_dRrF*+8-Ak3u!7pv8k6V8mdJ&QPJoqVJooIb1rz(BO zf@E(F$R)Uo@TyXE62uKC>69A5=DCKsa6k%)7|T*IpX~M(rHP&n)kLMp$Ksk!WyN!{ zmi^d(9vMXj({bx&zbpHwt^>^B-f!j50a$oecp@-FmWUw?kpk`hWOkSq4qMiQPvQBl z=`w z@Ah^n4yWToEdQfzA(t}l_EhcKI%&tEJpNgHxz2I&8fvg!qfms=Eo*IKs(b~{-(i}# zAynr`<{;HStW2VRV_SskjezRe%ksa6`rNctP7tAk;jPdZ6*0dn)Zy?s2!IcTIvzdO zE>Ffs;L4eRMO}urJ!KN1D-2EBK)FY2xCF2`zX^dh9so1Dr{_lU{76%TjN0(+0iS?g4J3ajs<|$u!30Qk9!``UA17CBY0W6rqGLpfwRMfxZU70BS2SOewlb*xq?POn3`OO329R$ZwIRLeV zohlOkoTy42hXTS`8nGzvDWwUeJ?+eRxGbx8XuiTxN^~TfASF7Iv!O)CdcQ9GZ4iY=x0rc}UsdI&U7jAC<0y@Idu=1S>?KPWy3mo-o3P69&Mp z2H4{N73eC-s)D>3s;@|isw5Ev#mXvj+EH3n#C}-+aBhRzOc~TJ!J3|+2qBiNW<+v8 zK(Hg|&vFai7}YLsK3%j9-(NNsr?Q(D9tAYjliBN=Kc-j%gL#!digrw?2fm;>aa^8+ym>#~ex(?z+G#eXk|f!qEMn#(krsVx z?v${qb0z3}LRVn*|5dISi?7*SDV8e=y62)yQTX&I5VH`hI}Kn8jmzd(na&T)s5V7h(@hU9{f-7?`#S5{~B?<+Oa zylToV7MYx?vY@A-I#4)406NuB zD2#yR!0kdSi+Z~&2DN(llUS%P>;!4a%0-# zRzBR_f5%#TG!D)NJ`hqyG*6^u-8BLC`6A?u{4D1FKWrd3{eTg}R{LSL%HcSiG#@Us z>Ip>YG(Q@bZVID2!_hdniK>(?26Ct<^*vq-N8@M;S2RqE!?p-7n1r=ZtZ@<=`64{V zDkOi5x{(ZCW<}JP=mO7x(&(`Q6%|~2+HmW%heUvlV^hOF;prX!XKagOnGINU7wu_Rebp*e(p1gTHg>j zmWmWswTULanHVA?c1$7zLT_EE2Vn9d$AlkI=55-xW*ya@Pq8&2hVT& z&35T3U)0)UM(?=HkJ;B@>CUUXc{ofpM?wA&+(lAgInGP{Jq(8!@8`7`_I<5t$11SV z7Tk)_1{hU@OT(dVv=8ky+Fy!A@Ja@|oAt!5DA1YMom4VIq&VGR!~I#q+5-QEI3P;_ zLToMqpjr(!D?@`@Yj6e2(%&}tS$JV6n`iv^h0+lJ5^uu)fBXG++I}X7;nCVN3spIM z`%080lf$I^?IUxRYzHnO@0OArUWy;z?&R00=9q-cgh#E&eZ9z)J}!_kZ`DhdpLX*<1YO;^Obl1HAiM{8Ga98FI{@eZSgw zHU_pG$plorK06Qxc!~~GKYGf?mtu*S^3hu(LOcOH1@r}B8*2o8bV2zj6@O%Butw&( zNm?TWyt<_?WkW9Zf%u?MKn8G*dsc_iYpf)j2y~X>lNtJWaitq7gaMVw71Z{Or={~+ zcubApM+K2ec`E?|?aPkg`(FcIA=P?lx*KYGchY~cciNx9z%?62D`2)28Vm!yaM3)a zOgxhP+W|icf>rvi2}SeaylxvR{f-+<@d2ah@jvQsApWB;6bUF($BPvr;A5o>$YA6;NI-X7 zhs+?0k(U@AOx9fV(-@0=(ir<>%`A6}tYkQ4?IjfN=^IUcE4)zl+)8#Cjw0}z+9=!G z_gXod;ku`84)7?k_P|1dBa?-S_y)LcH2=emk144X$j{0?a%5$H5wpUiy(Yj2y6?>& z)x55-JXX@lzY_o8neruBc6A2b%QchuC(o3hrv^ivWnm@$bT*0iU9b!I+h3Qtu#$vR zl0lXWyR0>BD|e-N%Ev6c&`2^xl=)vgV~Oxsj_Z5&FwesB#(k7q!pFM2*GuFu-5lHZ z?s(1qU!9uZg1~0>!JMg=$eGG?0*iWdd_0oGVr2-B_4M_)2v6A|mjOu+1!hSY)K~e2 zWsPe_RB+n8wghMVsn|~@lsb;oP{5+KjN=z-Y;AIeDlG3YR5b*@bK)Ukif6<_JUkqV zzno24pB@DGTUDf}T&1E36nyYT-E?Xa3hGg`-nLi~FQ2q8z&Y!~C1z}eB3?5I0|;mk zGAe8ZkXkp85DlY>Ykp2S&w^GB@z*y0yFM^XWUA$0VpZguddDNphp1H!^mVUUg^Y0j zbv@dDL671k$f;2{!hnvtBXdM>BOOanZi>=M?}M2pEY7-ya*C7bJw6OWX30Fm_%9Ss zs;_sCcm*AXI+E%@^b+xcy$sB47@Fn@xH$AX?-6AMb|N5h9Rx^(dhWJ4^1{irZTuU%M_Fh8HC{ zf7o2FMg6CRuV&7#(x7xE`f&%QtRP$SgGc4pm-Y~bX~hq<;uSrmro})D-j5aVWHQrk zSJLZVlbOR0uMDT`QMk zL(x+XQm7VL4nI>+jhefme&j$F`~IiBF_>g0v;XL#1KhUb3l*@IREt^MZeoUTbD=aQ z;q-<1lX%Xj7(KO-TjN?5UcsDHB$XfC5t&;WY|%R+L3F{B&G_@`suHOqvgnS;oFkI! zI3jmk2hgQEBFV>pNY48wo+5;&B6WYDUk_*5#nm(DGcC#I2G4x`_0DYti>#<(qvi#Q zC9_P?#rcT_uInKfL=5cw-(ZG_+LviccxUi2e*T+Bv1rkKj;#v`; zC<|A#s34iP-kbmwl*Awj-}MBqhK;-XYF z+x#LL5e6ks)Jn!>39ks}1MxaY)`~i2^~>0Ud<@k?D$If7-s;q+4+2P{?vBq8+!MlQY}U5__^M-K7bF(=9LJHCeo3y82Dd5#`E#RTcVoaM*8dbDwM-k)EVo(bCqyrWu*TA8$>z!=do zj#_)jB4Mg1iN&yg1=+4_0;c0(1$KdrBvy`-_PvSeItY;x9M6cZ{(y6((y?bGr-cEa zNZu8YBTIwIYj~Fn^~vn zqe8?0f`QOx-G{i<{IOh7r&|Nq6>IhB~JXlvHqe!9BSXU0S=VKta= z5x;B(`Z*WKYmyR}q3d8Q1!BNncg7ilgtT3%b)}<-yAia6(Jc6sf_DzkLty*;o1q2v z&rHx+H2En+Z>j0*F0VXvYo5jpn8j~!Sf<+Gw!XoAh{OcT^D=G|}VdHZ2 zdE7ljxz{}*OYx#eFgr&|tS8gXf36-aX3g1O?D=J18n9XYe-v$)5e1!WQE%I^N+qk6V>V3@IX zN@JpP_4hdMQX!}Xd47x|Wu3P{#jLdAxBY z+t}LJ3hw2uZSw~h4Yd=+9mzUp;rk3LWX5iiUP?ZX2%64HTE zJ@|YmongW^y6LKiK0jd|S{H^Y*fPXlr@-yG@gPu_eT?r*5Q0G%PR1)_J4}SVF`Z|` z2~>oVOng(jo8K+Q%Z&M(#;g1|rxUdZHFk~7y8K$OP9Qc5$g z%{VKK#lmQv6(w;>L`{WBB>cHw^)p;=qXUc$sdv(@-a;iYGm0P=DeDZpGw0(ATULdOW}& zvvgy-63PjMFOe8*Im)brY6@gY1(|TC8hZ{^e8d$gSAJ3%hV^(=CpSBuUsNHq z#`~i*wrdPT8H%Wa3~SA2ISrQN7C$WdRFr4cs5sXuETeiggZK=UKD*i~LY*S}+2cY^ zEmmVRJX2wFr$8UsVb64PSK8T9LC+lyPY^pQgD}J=|0cxSIjfjf#D9;zi0+cAshS?j zV+*#R)l1X!h$5wEQN^4VtVYppZLgd3!tJc3jsv;)HRM zAI{f%`{6DsRMqptX+m|l^^VEHj!9I5Jiav@lWhO^Ih~N&YH1`H+D!dz%dk9ck1&2i z4dS$UI+yZUhGlF%Q{8Zzk)B}D!QsSs+mv-+ncgVUmlRv!672x25`_|&oltuAV6q}JGabxOjF$5|F@A|@CClW z+qy+Ml(VSYcwgud577umye%qOn*9Ub&#M$3T*O!h{H%zvj@m38@bOz5%BwpmRd8+W zDfUeQhlXFO$k3>Q1XrKWbMR=p%w4Ch4{*EACqZy_I&JnuDx@rhdS*|ko?{j~@-yq| zIo4@|9~W=6CWw$?Iz@|A*im~?6J6tf8`kx=dRL@eC7--+V%Uc*LOWG{GomJWJb||< zkFUHbW!9KmLWD;P#&v63_sg3#&F3n9b_qgWnMU;+skxn;Me<=b z8d*lb$5&2c`MTia$(xh{q}yZGJUF;4>KtJla2U~S^FTofKfMW$v+MM5Mz?7m9Mtvq z$)!BYu{1STCcaWvmDPybqxEsC`Mn~#+vNJg2SqwF2dz)^D!6Doxgm$-iCm8Vo(y08 znX(?y;o$X~xl!i%Dj1~lt6LIgaLwED`qx}em2DNaYNl;|d19*OAT$p>r{~G+_3^`e zvpk7e^Zr65YnH7jKsLr@Ch7o3nQgJkkx4sNsDNkF$Gnj`|2yc9;D^dRo9f|JP}D?!hG|CN^vS5j@hlm*gPx@3OZY;JbJMmmgHqLdJEIc&y^;-TYBNR?ewZSD6dR?QOMZ?o zlLxYv7S-lXfsnP7Z+sNQmqs_{ueF!Jy^x6NJ2{%q+zS>Ev+m_P-3wA=Kcg~rF^fO& z=_+n1<3>F$L3|AW!S=d-E#98sy5kVi;9k#71Dn-+bx4B}Pm;J2P#em4t&T1m6AVl- zLcYOX8q58cs3F17rb|3WvqJr8^j2^)JjcP)iJa3R#@b4N_uE%K`;*^db+UNp-UIOs zg|h$72h5he^UOp{>Ru``JV1Kq;I_KgOTG(D`^jbp3%V6B47?tz75F}HY5b6XgyxQ zkYM#$T7w*02l-V9Yt{9iEX_6^nEH#^c;q4j));5R*m(7I|J8_9vc)>ODV-Q6v%~S% zP2{foYot!Abhk>?YOG2tKi$)INv*VHtLoi*Z#cP}y7JRJ_=de3541d=hXdQ*p{ko2 zXZ^M(h~LBHMVY3WGRfk&n`y@fJ|bzyOnlGa_${;OX?m=Y>1*rrL5S;9Q;;Rw*8Le9 z@|y*>!PTYUM^;;b@zdiXc+ID}tZ-x+@gIU|2zG(wSHaxtjkx($gaT*`3;=Ro9?Htp zt}6bJR7B0VS%I=x&8*LfJ~qVFE)q(!!!rNNVZm$M8MK-ZLM6cLOXT++n2jbEbG>2T84@VWho>-TK8X#Yf`%K zEUcM9eK|tZad4OG{+d?&5ixD0Yp_17CS@;}u_QWe?xHS_ zE?4Qg6Uxm`ANdpv;cHkqd-uy~2VrX=H606u6@J&XRxVwx;zYe!LL%)#p~B^yo>LH( z$L-ZZyAq1YRXz}re%8*f?A7@iHH#dF;124zNEtPp(CY9^$_2T}c!fZ4fl_F_>-DWp zebcZoX1s^)_(6KOM8%;0@b0YD!}mUGr6SJGTDR^m z7lQ-u_|<#A;|qgMjP86n@D+I}VpLM09dSZpR4+_^8NGop|MTHn?#EEPawqVa=nW3Z zEx&X4Qv55morZ5jg-$$5yQb(W#Y=`4*L`^rO zb#qq_h}^EG9)OlPXpM9|ShY2xV6ao<*1)a2;!Khtdzr$lJrZ5#OFoit3aD(!m+ZQr z_Q1GnbWhKVOgjr+R0Y=>P#{p$qfdfv^0$6Ed$XWmUm zrZJDQFiDpuQP=D`iOBe>xfP{vMJQKm|_N%KIQxg98DCzSDy>%9Ee z4-(vP2M0G?KM#i@fLT2M*oglVt(-_E3HIhvt)kubjA{uE`D@;WK z=6rHjgA1$4WV6LRo7hFw)9kg%R<{14whB=Kt%;qImBP5Iec9I8Tn7kN+cX;^nO$-$ z(X1gK`&zh~(6X!P)ez^xNYu3W<(h9CP_A}LNkJ}MWIJs$ktTcF4@+{rC{U06)ZBk=JQq(sb7xm1fYY^ zR`IE~4{r)p{PVXDZ&G=$q>qUNc|fT_=3b&#{gQ*qlnlB2k_SktbaGJf`wpF|(jvJ= z)s?i`wS4+V`GOwNfeEx$_YQtYk0>^NuJcpzHoObuF>c-8f%c+W@2AE~yC3;maC!|x z#lfil`sV4%w(tSZ=42=TREKb3=Tl)e@4S)j5bVF;>!kttH@ijLA&UuNafOyfxbxE z#O^nEv4xQ4ipsN3gPoGISk;TQfsj^!UDo{_dk@ub;jSnBDtQbJ9x9>JW)q$uB*xuSY6zEYibU?y-=>5l7&)DG=t8pt&){le zl_QB*h581_?qL|%Hf7CdYkDZR^_Z32kI@7zhn4vR9q-LGM3;WxbR3wUJ-NMy zeYdx=W;INCcf!-I4NBe(2rXEY!+fE0&F;m>_vhFv9Z561)&cUU_xcPosiZx5bkO^2 zm~%UTW>1>Qqt)jpk6b5@2Dgzy0zZ2!2L>L;tj7dw2eu6;r3WhM^HOvLp)NRUHR zvkJ)3Fqw%Ij-O+l+ofVFEG+_!DkW#{Wg-qS5jh~q#cz#k7~#y)5_U-=QeviYILu~T z*-_e9a?~!4iOu*7Dxec6o11BJK$;qC1YOFj6HScJ^p=a9g-o^@q#NA?>jTlM=b8{&E!t|k+}IgzY#yvd)!Awq zZ#l1&@WbC};Bhcfl6n>^$dG1gcF|YG7dUbf8aOlC*f3ox7pqnmXZ7PGZscYQ9%A!! z*o(a;RS`=eF<5P$_A~R1bL-9pgEg3(;?LHy9a#dJ4MS$e;KN5;Z4TOY07~PUo*prC z)52)I!t5DR%u$H)3=R&9dJ$;nJhgp85N}Ks!Cw|Nr_g;F;L#r7ttJgusRqy~DC{o=^l9tuZokv25~EET zeTG#BtjZ;7#TTeM!^I%>ayr8mj9r%VX=iv$LF|e%1|LOIcbFYDuW+`29{p!51caf}>(`%?dVW%{|b)E}~xmZfIU0ik-gQMuy zVs|mQ+bPcxr;Rd*GEy6}c*5W%jN3X1(akfmlCyaE{EIO5+C?oKCwlchlH*r0^u}a0 zD8=poD945aKTz6Bmr3Ils0uz)DH(XEG^57PWT^tDQWq@J^gnk~ux~%Kn4W7t1jC^p zT1NCE$q2G>E!)|s|ND=98@YqMu>GD$md?pkj^CU8w~yehQ(XXh zhXtm=P2TUrmm5!QckUh71>2zV(e@Hy%=OlrTgape4yN>OoIQtRBNddT7#-#tX~!rZ3&E(;e& z*cNUt9+Au`L=dy9GW>V(=Sk~ir&^%G@{RK@^e9C%A5+EkOvqEBp0=~Dq*Q>;Xp1g# z3g}RnCvTCtx5KyQxQ*2*c=YiWbCl~su0WsK3#|*ef{+5UbKIm)PH2#blo}n$c`7Z^ zcOpSyxw%j&Rawcf4nj5cHbb8HgfKOEl6CS#kH9gRL&z_?`M9{&k?F;nzD2#*rcX&sGerld`E;xWu}p^eHSEgHAA{=Wa9fw~K;V#ZbXdKR zuq^A}T25J&85PP&N)_m2V3gJdcfG?Xtp|zHdH|($r3NWdJYG&Y@EW9HUmqA6Zx`fC zZrYA)%tT+`E_EY^KW7w{I(>iwx9li!<8h;(wnN>Bwy>m+7S}>8`-21=#f)w4uW46! zr2IzA@rXH=)!CdM#H-^q@!H_xRcF_Ni`VjB6-6^ps@rT=*Q7}M=mA1?d4e8eLi1L| zS5RG3^Cn2<-U**_*EL~=FduRB7x5JfXHOQdnywbk=b7<-lXNzJ4(p>sQh-FTKXwD~8I@p>-b}y4Om(mP^8jV5;)v_e?GBRvJsJmu8>{XE9TxT)Y-derM9F z5+X}I;v$k#^#o<3a#-q>oVlZ%v1&O(VaDIj@X@cX$zQzdnXvebp5rr>2Ry@3$OYy? zp<^l`KY0xC%KhgUM~lLAvpx2ud3_wT>wZYkUDckgmhq4zg48=CrB)xEUWmn7%y^c6 zo%Y$D$+(ty9kZ}96$Z)N9lOqfD){7=*D)Xh5r#r169kjJBM!0@PN5w~9HXzx6-b<) zi2GfxKw|xG3P0R(_o4(GWg#&1FrrW|Re8vyva(WOScO(Pk;|jspCBDg%*ixJd0!{YQBW)e(4R_D|8yOX%MEsG3dZT<`qY2g8 z7~)A)7sx_w#BHdyU2wm^VZ^CCXW6Vb?sKnSk@hk35EXccFdqJepS;3lUoA!)Z1v_} zx*F0X^Q;`D-UQHalCYp0&GuBU$OFhB05OUcX#~e`;fv3dL0OjvNW(*7k31?@y5sHW zdWSbLc~@3d7wTk|2~MQU5}`z9V+WFCDZxjuN?G9R9BejTe%0o}$B`B7Zyj)##| z%CLOHrQWb36th72l#qWgX0J779(SelE#PaUgi5@1*n!K#AU-&gfrU>JBHT*-#xeU2tf+Ao=HzoK&!8 zTlAY1MuEsN*#t}L{tnU_mYFFX*%@O-qS%a`1EyosvNJ$?<`Po1^d^kV`>Z0dLrhwP{!-_x zL`h{_s4p{n@0ZWu^J9S_lu;wEyWZQ}R9#Am8R7y{U@dogn6h7%QiPlpsi!DDkVf}c zU`g)09oqbzmqAZiOkC-_9SUP5Z-)l;%-)UvB2u)AAwFB)e%vcy)P!SXQ8EP)~!?0(kU*}@nY+5>)L?_L}{- zIUX~&VE1{5LZ-7dmS#UWd$gY|;sp3E-^;WzG&BC zVk2zAik$?zR;;9251Mk|$lFOU+ZvK;WWOYxPK_VpN7~efQ8yHM(3?Jw<#SVw|RAEis;7lp@x z<8pjBjL@N=hhG(Y;V{*R46Bp&Nb`e5+k_*itcQ`)BYtSG4E|qpK!z!o8DyN6dZ4)Xw%AqpfKLR{9lx-GSnV3RWLq|6K z2HR59j4a1hSS2Uf0KcnHdbxFf3T<#zi;3u8!^^P^$IEJ!VNjb@uTApl(Xgv|b5zHe zHK#a9QSWh|XfiJ1R^V_W-rKby%@%p zENA>WGZ(u-@K5cjUN#ogB(e;9;{V7LntZpJc&Czt@LM5C1J%5kOBvub>{RU6o|HjW zv+Ki(YNmpsIv!q7a~5TFIFw}0t`kU-&Nh)wpd}2EC014KWh4|B38L&&T}a`KplG&>xAH!5#qxfq+iNgdT zw!0>=n6k~YnkvQvxO%bDVGbojAALRV=t7YRP)M~yzCRknotvt#n5%gVe*}?I^hdm~ z89Aii%bqVWGdrHavU_7ea8kLSu^D6FS5Ex)IP7<^cm(x^+F|U8sxDiP)xYC`*d*S2gT$?|l;c z?}ez}F`(i=RhYAoQ$9dL6VyQTcRfu71Y>T+Xs5%it{8H~%cHw1`pNtvS@{%R&(mdv zHRO5?F$~?;S~sWD{r-A)+MabBdzc!_&7Ie0S3|A%Ob%LtI1R5t^dAq44sR7DABHNG z{_axGK9SuN^7z!BRw@BmZQi)4B3q>$uVqmEKWajcseOe5gp#rGx@r6dV_X;*ovROn zNpz~Jy8RhzEI${j*&qI{H;2KSBJlj6WIj6q-_PbNB*{llHZM_+V8K|qsD~d^M@nGU8C+8P$ zO%-u0tNv~V!BaK6AlC)4y@`Zji1G5-?S4XnA@gOVyA@BgMLD~MM<5@HRF=j_!s}yM z$eMQy+AM}U*+%h$*)(2u%Ub=B+zn%5BCcbt9_*7RRn(^AFA&CcTf(T<?NH5q0HfmP+-7`AeZf7SPuhX#y+Sc*Rbaa6J(=~QP0{fXv_u<@$~tw3?j5-+Zf`%8segIb>w=EnNe8z-l1=0G_L*DO2ywdlH{ql) zaeb=Qe4S{^*RV81AcjOKA8p=BZ5Ft5RgX%1HIzo@i^9Grukxo@Ja{G{3xG)g`BmPG z<}?BDJ-xAl?{As5u~89X7+)b~Hd)(DcWr?aU5&v?;+% z^^@kEgUdQD6k>Gm9CVd@Es_S7PY+PzM^iwTZd>|S|B*K%b?HYPeEo-3wz!)c9Cov^ z6|Se=`M!GR_fhZMT)mS*bM%YY-n?5`uUi+D=$7@mRevt``Br|ff&h?W>pZiNz`FmG zQ>8Z-vzxWX6hJS~w@L$9XdoNu8B{_sWy_!OTUaZZm5JLpYN!3I$MI-qql4tE;14>?o`<2vBhX3r5c5u(vFf)&h(B-oD<96xjjunTe z@SZUI$`!XfsC9*m_fW(qyJxadrJjdkK6E@_-Di^)d{lf;eG26*#TkBQPiOT)&Y)cw z%J?o_`kEEbZ@Lg@>(Tk+YMuw?WS<}glne3^aJ-rGSxdcP`)LxCT3B4=Kc%EE#WdFa zQKS$RrjDYr12M2-0tMy1v%~PWC;^77Bph&DrLcw%Ip5arU3E)awuz1%hG$A18P)X8 z;}-Kxc*FGW6T7RYvqrlAQBiuj`ZZNu*DlM8y7u*sf8~z9`~OBu^`QUv&VNJbk!{N= zSF?yJ*wOsMLA$iF9Dnq;D_E%eGDKeOPJ+DTCjL%M9iNEw2fk#3Wp$aM|(e-6i>taD}CmuH4IGWw~fplZ3 zE6S<99?rgg@2p)S!LxRk2g!c1PEz%qKI0srU*+un@8tgYyT3LYqNJV-UQ6i>sj>i& zJSEnBI{)){YAP?==r)~nrYvDQxVUr_Erw$B65 z#ODtf%^xtDJ)pD@<^FSK?iQU@dreyMhvOc9=istBKNHV?q9WhW{@33E@i(&Rs&_(2 zuK49)9e-oj0cBQEI-ah(X~o6w{Buoy^Bn`>Zj;G3KhZSM>4uZGzQiK56g)ket!=cJ zh2H$89e&oZeRHC_%U(pHzcv{s;g3U@#~K zbzD(hqqwSiM)A-%U(V~;jB})FEi_wPw(?fpE0AHCb}QC|h6@c%W8;F;mgxM}HdGRZ z@$=bDN`4rbvF*a>_LVGt{MRbg17cM&A!Vqh!Z(W!3WE$>pM!XD82w#D?K-VdL5ve^ z5r&i3*!Zh_=3QYpX$L(O(KugGlrX=0UkrBWj%Bb~6-`@64V;9b3{~z;I%;1-3`_Q{ zUfsz3*_?2qU}{lS4k7<9(;HUNShBSUN4(O=T%Y9Rh7rvDqcDF_FU*;ulxc8_u*AK8 zX+IfQ{@g;(RRv1?WK(-?!E*&!g~G`0&G)%i zNROTOvK>PiOKqgwO62-DM7B)8705=dP#(T^=$so4hePN1ZQHg@08}9x)Se?39cuv4eM=TR& z0Dx5l3NAJnuJ%E2aeJuJLcq45zfIRTGL%WG`*CN9f0^Pzad;MF@RhES#Vs1nFt{Sm z#uX}yQ;C$;8DHn%C3*G}-50QI7Gco*kQ-OaqQq29Hb?&lcD)RQ>C*w*F3hRNN_W&8|cRjz@{rqBqQq)5!YWMQ0sxC0p&*ld~Ms0li$Fi?k z$u^$)R}_25`T13A%Xz)=Z66bObM~xR+&aWB9;%e(~qiCwcct(S1^O zpH$~JhPrET{sUu@IP{nz6!w_fNC!8vzfqjo$U=FKt?{zW4JuY_{LbL+jgKArn@N#w zvv^qk&TlgxNAq1MU~zc8YeKe>U7f4PdB?xne62iq#fG~Z_T=v4@cNK%xCh6WEOpKA z_iQ|4^sIxg?(Vopcen>yeQn=f7`&c4{Jo*r_=Ud@1Ao5pvA-XRdmh}#Lb36!fB$RO z#`y*lwej;1)fXE-KlqxpjnfZwPk!jb+1>aE^I3!UU%= zn5k6lKgz*`>hH={vk?e6BOsw#eJ@|WI!ph)ZteYTeg}g*&kE>uqAj{E49_!M4}^T< z^fx`+dO=5(g7?1RtpLyN;6EGg{)gpEL5~6YuF(uW{;T1wE1rvdC^r24A!m7x{T)d| zps=XGcfF2Lizo~EJu1p78G$ms?g2Zk=9%-<&en?s(q$hXsy0FvKShR5Gl?Jn*(wa1 z&$_rpGJNx6DonN4(1pDD5B6m&v|hT=vQ{6l@W`$ax7#%4z1zUZZ@bCmnPL+6Iz(eL zZ5_o!nKqB~z#kajojmjyXn6dbznc&Tk`7{ES}x`+!auR>>e-})!bHSn!8bqT>i4(# z#x=~$_A~kY8|x38*NT36aR0_JeY(B8L-&V#;~2j`J-DCG*B{jF zA>X*>!SmiRoyDfir?|i6iA~^wgiE1va7bmjnCeksH>dBrWzB}S~l1S$!Y6eV>4if^=-2Z7cOz(00lsG&lp+*=2ghYMC)>^ZDKzjF$7R<1q{9Zx z=X2hCKcU6EH|yOeM<6wtZDjkDxz}&cp{I0dVYg)S-6Ep3=Eu9;;@B!#I;XSloX)y) zI(u0GYU{+5&xui zk2{&E2NMr6pIVD!Q_2%)CL|Mou#3ZYDMLZd=P>^g5xaaL_L|HJ=#Ng*A?I^yK zk0@kzYI0o$k60l{tr7IxtmwQ|S3Qi;R#_w>HvFOBrVmxjb<|>U*YQo81a_gS4F2<3 zmVJKt)Q*3j|9hXWGyGqwk(l@dOAaXlp_IxvYl`%p1d)nRwth-=O2QHumN~2xK8z-e zpL8y|ZVt7var}wJ64+5OjdzQnVO)kH-B?aqLioD79RW46`v)1`hVS;s$bA|ahnNhF%vQWm8%z{-2i>^+k5Lwsla4KwviXxU1hN7D; z8gYR8^vDF~c4U5x;DL!-Z=PsL3;sgPlup<#(+VeiD08hw)TSHdS}n%%{tHv$NgIHS zqyo)G3=vMEagnws!SxDb%@dU*BC@csEQk-H0I+#Zp&HiE?hE0E={~hUZdzJ7kP&FM zSPot7!yyJP!_y6mwxSeQ6EdZ}`a)mq>m+3~*jA;Jo@-v@81&KE;`v7w*#m;exz^&%NI~aBDx4}@oXi~7=4@?~N5FzqDMy3poXorlcfM1_Xf0CI>S3$s zaDgx+rpLORniIkX%9dg=C14-&o7Y;J5Xepk{yHQpO7qUzu2=?L~>WZC{Q72 z(#Z^uwQx>_@^BR2g`u;G_X2n*bV|@E@HbWoCFEctSeGZVY%*zm{4Bv_a6`AYUV=K^ zpf#>dKor7FpY66B;0xl&Cy831J8;gW4rr6kJCIoiGIap;a&#+?w}I#E)9@_=qG;Y= z(2X1vhv{)yC~wrhAD5hMDxk%&K?I?Y#AXntLI6-!mAA1JiI7`r3*uZ|!&!(d9B?i< z4G+ayTk>~LWZ5)$hH1%0DZ6<-M!J!>{)n+zvTNg)eZ%g{`IIAz*m#oBHatqgvb1mI zdMgf)x}cfT(=(pV5ZTk~`a?rsra&?RY#!}xdeL6rMSBl*(O#mQwLc)*>n}3eTlAy7 zeja`$9ZJ#qkna)g1+CXbdwmz}6@+g`w3p9*Ez?*C=pwwk_9M zFGqRG@;ZT10`#PaOXmyE6baY*xNzi zp}NcKf+9tt^Q|ix!PxXvf!SDp^QWrT6?>RRD{vzGZ5E1N7lf>`Muz%hNH}f=iRo>r zu^FmM`=k@z?$&lp2&Q%#K_xEmOA^Tp0&CFQCRaV`Bu2s#w zbnT$$ARFKytK=YaYjV}OC0 zI%tdL-2)XGFkr_Q4~8_Kx*qoB;?PV*4>p9wi)SsA_2Z7>4GC6IO{o{eQr2?yp*qV7 zJ!^H#>^w=!P^3&s`RTM9t_0QA)k&B@Fwm_{?(H-uh3ASOnS`cVsoWWxOOKwTi1}0G zCMm2w)E~pT{*%JxQoG_mdzYKW4}A2VTUN68uRc)4oLN9LlT$0K>q@2oCkJ@^qkmh) zfBqO8ciF9Lb}h^&mq{f;ScEKo**CQLn?pXeL{$U&`QVsQptC@Mvglujp=+OJ(YMc3YWcd3i{}-c{6?N+C^>D1?-w1%PH$>>er(O?Ykf^s*C-$ zQIzK0^z$S7;+pdFMTsQU%2;lFe+2t&UKBpA*Jf`7`x9|d=)c}XNYc6bH*&YsDf_6* z((%>llNt5&(5vkSe`)bzLr$VM#bI%`l*TqCWnR}Hwq;``BY+sFAgqhp%pr?GfsaVq;?g<1 z3rlZo^Y~eJE)2{wfnURi;k3qZ)GbSJUY^XdCnf|{-`EaFUCHd~FwniZ{z?c1h;MiS zd`Z>V(mUu~^B#7&#EJ$_C0~d01do%LY|Ggo&+xxbW2$et0fcS%K!e2g=1afR#e`I&|?WM|~?fqdWq?l5K23 zrz8Nj#^a{q46cvhbhL;;a*ZldAP|VXOf~jCxV4tlFpLs?Iz)yJ!f+}L4XFs*g`}zr z<*jSY8vqsuu)k|HbQw22Dun0Sc*J<@@-l}JF`0gFWrSlHp`eug0|eG{-6y^6_Ixx8 zGot0J+*g)9kFGs0Sz15oodb3`Un5#x^EJ_Y9~?}s`JRC}xPx=A*I<5L5yZ+;yBP>MEe1zpTlqit74l}EUJa4`SB_4y| zz%hRwOXvBl>Qs;LOVzdKwH)ei&3CM7A@F8nj>}E{wsgdaHx(Ba32S!h`E&*bkEWEa zvwiTFS1-a}dEkWnKeh=z;ypEV{h5uAzx#u4E9qv|p;h{LAE4XBxdop|;H58lWHSr- z!)J{~{! zzES+#AAy5tvl~&!pVsBuf6PZATjQgwUR`LW{v&c>xD_?w^tt$vY|<8T4N3SU6Fgl& z3S0bmHqrA!&c!92s@o#AlPjHQvkHP5Z7AZYiQ;C;VhBY%F|pJIgq(5Qq@tYWlg7b) z6M0HN9^E8{2_zSJ^CN?hQ%46;$a9b392VD2BwB;^h$12@A0D+P1w7X$qEgG({b_C` z?%Mq5fDq;X>BxuOmYH~(R4S#4`PZzq6u)S_0)5@vdo)=JLAEKo527wXlC_8xcz~t} ziXwg)hT=`YVBuyIgT_FlU0vu4=T8}oBZg+L`64bfOY}O;ioVKUQ`f&uDxlOTt97{U zPu++M*j~p)_ZCci{<^%)Z^JbqTt6OL+!W^jy~ZknD0V2iA5JV)CsR&{8apL-&naI2 zmw$Kn+T15r{ zR?!xu6+V4oz^cQUUf#B5MITiRKKQ9vyfIc>9G#0xe~82`%_5tRUA%CzHF~!grHafp2YqB3ecWSD}e2*@=ao-KCRtjCJ(OwAjBfl zwrgycY3-?z+1hI!8_*5w6;5hJsMh@{Ghffhz?lzvU#+_+#kHr_2|DO+sOy0(|N5tv ze6?lpR%|Z44?pZ>c!|Oylq^~Y@N}M`7W|5nO%7WdX;VsUqfn$M(@f zXM<-vv8Qm_lR{I%mWdJ9Q(XZYY~7y@g|bqc9~q*Pt=CS@;-#}u6UU0>6^wJH&qI`6`&0~HGzXR4ybQ!08(~g3F z^j;aXrzuRbkp=(A#?k}#)GZZB@<;D&M`09}zH*IVO>q_Y5BW|0pi3Y$6ygn*xjK~R zL|)IP9lWtEQ(WB>c$5iFuB}1?$8B3ScoFP0PRDSrla~POfB-zlK|S{TJi9_nP)&$? ziWx2@q0|lhaU6bW{kA+&GNd>jE%lb>ptPMpjb}lq?n$%$lsfUPKk4zFfFO<{Rrr@6}CAm_D3k(30a*=dWkoNryp;zL@)w*4P^T(w{5WqjA;BmU1c;Ys16cRx;K z?m>NcJDn`zLt6K@6=(QOJ+I@_|8m6tn9SYL1g>Vy=KwcluHsj2H4g1}6hS?#3M%OsIbWwJk6OB{(=4si zn-i;C-xK5d3&~pf;us;C6gz;rTBUDJx%AiA$#S|i{WqQ9QWbrF_r9)2jnpboSj#o!c3B@JEk37I=xmJduu7Afn<=cSQQ>@aAY~w9mP}b13Mf`zJSD{#IVGUw{0QnL1Ze7PE(}LqhrD1v6Wk$>p|JnPiScqrx$4v0x zIgVRzj-POMZeEuG&%cmu#>c)}WsI#t0O|{;j2?HRQX+l-s+1aT>0u1T1-D7at&GD- z$npFeY|zkQ<`|}D*fvq9EOuzRI-Vkxd5Sr$Gy^_X!GR95<KI`m0Gc0mwRzmP$OXo}m9rgS);GViBIg3&>sAoEP5dPs z@MJiBrkackuAn)bRaTJU?dcYfYr;GH;^GZ&pWWc?hHu$Zyx||tZm8^rZ=Kx`DL3(n?5u*H-5`DI zJ1fw(ETmeYjE{<60L~JD@|cM0vs5Y zOwj;f%$${|cxBf(tU3Ty6E#v>-UP~#_GjowLPsE}uu+6`>|H-5J)J`_ls9qoRY6;$ z+iLM=vdxujLjh1wjqM;_&w00z)fQRX9Wbez|6oYdl#D{mzpfv0&fOifaHCXGNf(n6 zGZ&oJm>h609a~00-peSU#LVpPulV|v{IZZMb4-_eWfI6795O`89(%aW;&rzfk5X@( zYvF1$w%K9A)}QXBS zjtB9_9vY5^(T!Cx12spuFzftINuAH$0E^tF#BGY_%H9A-S+^1~4Gi6$bN<4aA%xNs z?iRemtQc0CU{@049rIL~NQ}-(a%(?Sb}!mg_2@?aK&!fI8E5p-)Bder$(a<7pbGl< z3%_1&kWQ5~|HM)JZ3SyZbwz;6<9Y-3Iqw~aUw_v`weBg+;O}ZH(-JK;I7>Y?D7&W3 z^og#+i`@+a!N$>+R8@ph%1U`QFMyfk@)_1fn<+R;ekKqL7>>)Z1Sp3AVG%5wO*_J3 z&cbr@h8sa?LhK|nNNDmI%N!ec1ViS?fTm&DSpMj9%|c2q&FY!|&H+aaoxL7k2NW8N zzfey}wO2RlXYGbPU-Rd%)q1{;Vb5H=D8KV?j$}X~!6>p^xw`%t=gKX=doSj%a825U zawhrL7F!Y63g*YUAFdgP=`!r77|^1XRK`VG+tVgZ><2FA4Tc=42F*C{_s`Ka5L_s8H(G^coX6?C;eRttb&pqX{t!H4_F83ugnW$^;&-9I zS$qw4Gf5e&P?-xpxX)*K_W3%Oh8!>at<6>x{+R?Xs*%CT1Iihv&-S*vQ$XV`&!K5GEVzqWuInpUwk(G$UDHp)^moOddPvH^ z{6xChIi*}fPiGxcdZv3>c8QSfnxlDF)qj#nH5mP*(1&vGb+mOzP?JFEAYMoYT`e+_ z-Ygd^RXHLunX=*u#oFPZoOnfM38^gPf^|ofY5KFQwMWJ!v?`#1_O}tNaS-c>7oD9X zbTLI}K@x5Wd+vnDG31JZr>T%w(VazFHfIutAbJ$~k$^YkPf5r-aOyoN&*SwaKo2!! zk}l*^x^FQ0UD{9cUGpp(EycZNrra?zB^uH%6h2>=nVG)d=^S0cqVf@)^I00gE~|Ax z{qJYB8fdMTr^l)bE~yV)MxnRF)<=x-KP+CWSU{C8ux(Fvbue9>D!g0H6ynDaukl>qTvV!dYgMf~iN}-XOdu-{ z&VOHh@cxITuoj?TrmS8-D=BY;kLdEQq3uts=D(5 zio{UvkU=#5@WP#@tRha1b?wtV7T@B+M!)A&mrz&g=^@#HU%=b6F3N?}_b2 z%Nl!MJy+qNj7SC7#7y7F$&!|~^f$1osK+TxVEIre4p4}M!Oi#$z~Y^d>w;lOmDIQv+op7HS{uzmt&(pK*<3tk)33l&MdL{+iB zbp8F^HP(W5%n7&^ULzTs+B;@R^QTiFSWToEubM}Ak$f5!g)&#tb2A00Sz{RFA?UV@bC5~RMoOOT?5RWCuRSfIlsYtKE>R(sN67SLf9q#%^gVJ;*{xn9N+ z2zHR=eF)f$4wD$i^G}dkUwj;j#a?WfIYJ;zL=q#-yY>5AhB0D_xdJvBYPlAo0wpdC zq@$x7U-(46sghdaEjYsVKuTQrr+#y$oxA3JC{~&;#f5?G%l{Pe_}_jm-@Kk|P-hge z-wsdkUsYC;WvGx$YG^6DrWQ8gd>4 zkjRG#)T#?7vvDjqw0!fu68~H+DHAV~e>RV&U?F*j_s+1-I$`Ajm+HZ*Uxa?Ez1!)x zhSE-^Uwp#7j`WiFIyH*BYKa%vaokTyO*rCQ2a`Zix=cPDVL^~Yaor*^e zKxNY}MWS;Z_US-QhK-Jq#4}U9q@$g|%n^Zh87fGuhCOd!5oUjV#n}AQ>1t@p-!l7}T_!te;So$t5s$njKGof% z2X;N6&4M<&4``(j+e7jjS!rb1EO{4PP`bkOk$7oz&CjUjK=YHXejB-+!EfAqKgZ44 zwbm<>Z1}ajH3cS9x{io*I~N>B!Y(fyMBo15$X+XAads`_&5xnI*!huEjE}g@NH!ul zAjy_g*0x2CR8)D}%CHC~7B4+p5S5tW54d_6FxiAGB5uW$A+KoNUyipW{sxZu^a*}b zKwGcHjD!zEV}aT%yL-F9K~dN5&Assmu_)6%w(YxU6PKCy-uW((5@^L9-s?R)*IlCC zg$o(hP`8Y=aLtFE{MUx*o3o!48`;;0tcvCc))_G0qbvmDF$oLSAgQm@q; zVa!g%&1{}_(iv3?Q|Oq&uBkofjH)`mI8+a}NctDQc8#pS?{pH7GTO2V+h6b z82M}SV;ucy>`#68>o{DyzrNz@iDODNP9Ht_f$o=A%`VHbuUmWf)74K{RJ#cIwYf5= z(2f4L-z-DX{G3GvP3uN;4n;gR32qkfimv%lci9{g(A&<>8QvUrGKh?TDBcKoy0Y38ypJ1v=MW~G@`4OOU){XIvPZNMv;)B#&&(q=SFDM`y9{&X@X2*odVN1@OW%#s;5)9t$$N;uqLB) z7QA9|<(@EBnr?ExFZTrnpwXY@bkRmKZQ4%Z{doI)&lz;F-sd)d1lCO9wQVPz_HX@~ z1NMSCijpy^sH|e& zosb-W9#Y9dCnd8}=L!?<2lN~AUYPoK^j+vpyIY>hnTK+*=jP;0edqX_^<8z2tnVh@ zJCA7ZAi*OFcwDFt0R$NCLG*9zp!K-j*Zo`J>e^e{Cu$Q(j>NPApA;1oRa$(QViwI$ z!b(IXX~A<#c#D_#Hdhg4TSp6Mt2i_#aG)|0QT@W^*v+g`kKHK{me4O%QVt~H3&G2I;gLM=hasgqEo&3*@U1bY0UiW;Dk(?64XAv=&tx>-w;Qe!A~N^t>zz4 zW$!N8oT-#pbut5MU9pL)0MR`%7DlV#G-eXBhkBVG{lNT1<(BpX^McA?;>C<@)_iPG z|0m-tNp`*5jE-5+CO0T10=$bJ>~;^Q*n!-`BPeS{sePHbheydhJWRgv&OHoLM5u_R zP(5<@c!|lMwgX;XI|ti0%1VhZVPxjh8w%Ct;&U$2A{;U74RcC7rI) zsZ-AER5BV+$Gk?^2PwAkE$)urK*K#L}YV>zxi=EJ#FEPCohh5iuW_g(qoX)CNuu8yMC0 zQ4nRS*P|!~>$Bc;G82vYIHJo8C(YxIjL#t)-hYFI0ljn2K)?TFosR!Vs0_I6A)AtGLaqbD)5}k^;Kp6_TXplP z@Fvx;f7m5?uXK)w<9mM!TBGm4+7X=h)1#a~zh3&u%ow>HReoR3kE{G0tjDYUs|~}W zCFSYK^X!A;-{iYYd%2TqOHZxL^zLy$&e zcrkNlraGJsko25yJq?#Q-0TQUL4S*BcgWPFowtPyuWF;XyLlLL^^@7_am%jzSIcKF zhMp1@VUa0whzqK`X!NA=rhQXN2oeK4i(gd~#D+{P@w<6=eBzg1FSo_7ek|h`)EG*V z+#!dR9sLKLIPhU5H6$H60?_UZMC4s~wzd=8J1*k)rKrHe*o9rEV8B)d;?U)&0Q zt#ccLJ0sBN>iGB{mWDCcy8IMnzfX7J4@U&e6fX&=>>;C9V@^!KZ=RMa!u_s_{=Z$apLT`N2#oqYc1HbZr zv-d7gc3kC|=zdhysdMU_?vm7&+iEfFQx!Hyjs2~1Y*;s%o8 zR@(s?jj!)YOKLv?PC`HoBw%ntTb^8*V1aWX!RBT0yabqxhesBPO>n>oPIz1gg5wYy z-SvI{t~zy2pO$6pkeRiny{tZ6ryjfZ{`bHC{r}(p{iN~?>o->i*2kXecV#SLw>3fE zLui!&SYi;^BdQpGebtl|>EQYz1aqtBpU3%LwiW5#Pt$h_gPp zE^jMxalF>16gJ5&UMwa&YbGK*YbB`&8L$0G7x_c4zOBCw)u|V6>#xTdVIyV0eoNG2 zha<$eFYw|(i2z3DlG1hH#0x-rAYFiB=NcQM3L*4G_nGEum%H^#MZTa9!14{g1c>*r z^B8&zcye798G`fH<)3i5Teq~7UtMTmqT!T9L_I?BB7#0ZUhaCX#k~d~@`$1kjXYUq zB=1Eoi<=OlYMzZ31Gsvzkm6r# zy=c_1elg;uNBijM24)cg0GbN%e*-q`v_H&hI2sJL>uK)U6y_el>)=-0VHlBFf+&7~+e=unqv#$D zMrFv-5d|=WH|dI{71R{y;b3D`?m6V~gB~8aZbZlhcq;hn?$E)F$z&-MvWUd#3s@Ib zdwA+#(>nCz!K%Q#MegQtDWB8w3T4T~NeneiqVSPq8o1P%V?xhe05MCQy^|)>pcs^l zudF|{f*fo~*|^o_XDUI^dXPppSLvAX3%42*1WhxSexm3sDAzgQQh^v;cUfUU@aLf? z!Xo=jb9L5FUHcPsO$A>LzrS`YF6NE?rRg|3x$MV7PhO3;F;Po#X_5 zee3`p-*Fw69w-nW&v1>iVEs#R6bD|w1F7p?2KObsh;YcYFY8HWldaI?_!@(ObEtKY zT~i&P>k&8+$!y|9h~}eNMVm##mvA>aPA1z&a zw5p7R)rWOD!gG~NF9Tl+w^yK!7PqLd0yUan9~j?!?NT&9mFC042ol9;@iv%x`KMah z>)X~GL@g!>$?Gl1FeR)bZX{e(=`Ar5E`&{0$j_WaWedTA%4*J@5V zuA^IRPC|P)n9b6lTv~ddirv2oN*Dn|lv2~z6!yTK*nU}Tb8#Ghg$TD&Sk_6u>rl}i zQEl#(q3MXX6^&LG!3rABtlBllBJQCG*12{Xx_p~e+&yCs<@jfM1y4|BiAQeap&MV0 z)ghda-Ylna53vn**%prFD;aK>${l$8Tq~7n=>`Z8s^GQcyp98eO6osV-Lt%W2rd!0 zOzNHE$6wZKI=}kS(5BL0Q;8Nyfhf(fV*)B`^F<5l1++lmh15e{v|o48OM0@k#I$8m zwxxw@Y7~pJ2VTHx1b{Lv?X)>IVj%qp=<8|>#0SJ~&;P0=<#<=GHI~CP^j5mz(nIcB zwG45sQFt%hFTz7bUBP;U_BGfovudTq8LSMX=2icHGVu@`&AjTHCCsP1sLKmJY4fEy zS4PH(Rg|8@aWc^7fXZO)Igz`m!sxDl=d>GPa#H9x~Uty}+pFvrn zb{5$Yxg(@=ep zPp`}h1?VGyyiWO_+$Z7QQ1_2~Hd|Eo&maz^`i2{*YQ~7D07QrH^fiTsYMllaIjA~= zDVXD*fvUri8+nMTt~w#TSx!ScaX$W$V|ANZ3` zHB%7*jA;q|=sD%EjLfM%?S(aQfmk?*3f_j;$A_ySe)xpCuJQT#1iPA*io$PI{9^=R zWt{mX$%Lwap2L2fai=rvc!;saP9t9=++BT;nLHD*EHP~#7c!+D_w;HuUPE%n2#smt(Mwncbp zu+(Ec%!+0W*q?=>K~>cFK(#_PlwJN;t$_l0_56*ThNB(w zo|WD9wjZ0}Rql?Dvv=O|xKtE;E1?Lnyu&Xb zdpk6~sV~xy79R8kmgs2LLd68~C5dQ;9DEc;2^_wRQ*zIuvh1aj67twx6CFg)s2GGQ zk{iED7n+eg3xhn1f1So3U<$_vQqwx5e{GfLh%a67o`}H|HT3M3&~G4L$Rr}q+&OT( zhP^#=a{yidVFb3((Jrq+q3|qJCwZ317xWU{XbU<&stZMg9pT_53yP3i4WvilNh50) zD@2?P*C+C#QNXBTxgo9J=AbhpxaDYWMZrnL2y-)^a3rjYfHSCZ&{P(1k$%``$ep%5g} zJ9!K>GA{(oan@`mZY*&vALHTaqwD$|7xjt41HsVJz9Nw-Zm#oa1_o$wT$6tnzF_ zFYhp$nJH4e|FQgPc=oV>w&v9o4tLqH`F~+Alg1NHTAEsBzl%yAE!I5K(GKI_k7sW} zNT(@ugRqZkI0%PAKLRhO#3BEVPFaEpC~=Y&&{u+e^aDE_nUzZTwF{OtA1kN8B$ArCMbmcpT{VldPqp3!snY)Rb>6PIR@Ye^6xZq~14y z`(n)o_hTXXt(dknM2L(Tgn4s_^eg}ws;0L2f~D(tGzFygyN@*oDm5#tB?3~Z%+#R_ z21bb>N@Qzds+HK?I7SJFe6ft@sb?_M=q|v5>KVqUe^sKonqd_t7X&&;!3w0O$VUMo zFmcot%7u^;s#>Uu5(oGDbdhl+1E@I(nLujm2vx@0;?KO$0*D2LBl?_CeG+z)GdBVx zoEZna6|UfNjq^!4%r*>U?ulfjU|1C^f7R|7!T?31l5a!}_T_yW3I9pF{ zfJQDF%HWBZ1BPC{Df`=wO_$(!4if-9tQTnmRL$%mSU({GFf0! z4a8k6 zzU`Y=8rP^@E4P3X8bWoXh+T|lM=rYH9iO`T?)KY0e*TdQ-gEWqpYq9lpF4l$$OX|4 z-g4uOH~z)>D@Q_s;OK%2p8r!{UH=asJO9W9e`P=WyD#7K|Khxtys3EemWw`*k8gSr ze*WhCl_M)FM^K62W|!Q&LOBUdbLj;;w*AAp@BQ#4KgknTF8I>1w|(Z>Z~ov<9$8sg zS%Kf#B0z+FEcy`czz2Wlo&R+H0d~4q;%JwD;O`sQBJ z9%M_8tPqsmz8Nnm&f#of`wd6I>Uu&i`gbfJyd>IHiS3D10&5_!!8Q8u3H7rC*F1=b z)`KVXUW$#rgu!+QGz8_Ara*50=_B>7c*P#*XW8I(-#h<6(Oo#_hNA#bM*9oxm%ka6 z+pKb3RM33?N|99EjfA#`|6ab#T(@e_)z=-k3*UT~-`tFa@q6;|NabgPfX1QU!?2+H zo%JiB^WIwZCr0CNICw*K(E8(vuhm*UFv3AiPyokziVA-BrsKW1xscHOI5hoZzW3At??u8w;zDQ#pj)tzh;IfK=-&MMEmDcp zYo%~h;ungLOAbWEz^QllBwS!^{E} zFt5(SU$~(3Du1X|#QwmgF7^laQeTB^49St)-Gi-*ha70&PVUF^DVr-Oc%%zP$h*!Q-ceVo8M8|Mf4kLI)SQ6 zE+?L;c?(V%m;MY1v4|uY|I`RB6FF<2NPtg1FV8SC`)G=N=Ak z0j~?bT$IWbg47b9#K7(1EGK!uH&Zisv#&^JK-KH@zJ64wc7Ff{+11U*i zE(SOM!n5gSm!CsZJ91c_pi`XN};l8RK3q&FzgH@xx;*Z8)F!?AO*5x^%Im(7$^pWh zJCtEvD@=l&;ja>(I%wRh)R3S%cOWXj1~ zpw7EKMQJTWc|)j4TDJw@wtOY^Et(Bss=!y@)7>P?n(u)~K>jeHZ&fPD>h&Fj>KPa% zS&GInB@nkoh*gqv1+mShrt%Aco@7ovZC*XiCFm&wCcuYV&hT+qr;%Tmxi&N5oXtMl z(_LUrmNjX7`~d)avCjbdGWPedAVciWfzKiCg>p-C>MY?$Q*-J}A)FKD_gQmxh@lbK z$3!tW{rvp^kYJ;MqTBbgTKE0xl9?htmcL!gs1OHuCb3?a{3t8L0!OG?ge%2^IH%46 zb_RaN$UmYmjj$vwm5A|FZLfTMR%uJ8B$zVnOwnQ$YP*QVeWvuvtNa+AV|SSC5J%iK z#>1Y8f6mU2VT$!z_=2Zc>VvW>R2@4FYLBYFbdmT%C|yj+D&??l6Ep@Z1g7per~_w( z9?Kcm!R4|_!o@0u51ByaHH$H_06HI1bawY**O3QEjseY7e(X5w?`7)2VqxUF>C+JNO@!QI`nnLpTXA#1A(GkK^7!3BU-A01irE<(u{f2fb{N=RNZxvxllw&<5G z>Y;8R>;vh0s4!MbTPh}9;lE@3^9DhT{qz-U7Ei;?X%`1!)LrCc3Ef>6e<8%&_AI$2EByWaK>e2>3Y=} zZPzJXl0VdfV9uJ^9Rm*V1oTXTTO0H!Cc#;#^afb@bXl~< zEy#p{Jv%TohPEcCP+n6 zU4%ldeCzx@Y0>gaSbG$@2HAq9g!5_ETv&1~!8Zw}1CP0O$!Sr~Z9wwzP3mG!LP$ly zd7?@Tmds2^Fhc3CXx3c7J5^uIV!RP)x;;hzenOS`_3|e9^(kk(L}6v6^chOe9~27M zrkM6SgQ?c`^Nao}(2BSFF~IAnNO{~PzH@o$w{w4qxC0^kDe4U1=0%LF{uAQU;9$ko zvHyzNgGHk4&)kdhCQPl6r=p!dBmpk|wxHv;e7UlSFJ%qPknBSMZFPDZ>l?ekx^{_+ zVi6nA#iGh|5&Jo;5oS(VY0*^|^{ltmJ9Mje;QDj2cSu|pb|SheLgAP$3SIG)_5#@BC_F)S?wnuNwVE%7IbmH4(`9T%hmADqZxsk#y`EpiVylZDe#cLHr>SKN#&2ULa2d>cuspVj!Gn&DXR{A zZ=Xim>|tH{Yd)VilRre4#K2F`+~J2ehD+x&YF=%qDHn56E`rt60C)NTVzmxw*bEYC zp_>#(%?>k7j;QNXX_uaa@XAar78;8S9!Ngr%!gan#1+X-E}c zeOi?%4jSpV4$-GPv8eQ;hQ$|xh}bQVbdH=|xL*M!hqZZLJsq?SNYh4f#iy5Ix-y}6 zc0tSL6hru2tdqzQx(xQ}EX>=ZC8+Ngp~$|f_i>?D$k3X0F;O)>Fa6p14}=oO#k3$D z3T4l;WR=fL@%sHstMW?Ot=190q62~>=KcPI07j;e4Ar<1THwEe)IAJ7ANLiN|A3B} zTr3`l+1i3e^2FG!EuBVZWOt?}I%F_b zyGmi0M7!lb@$ZcCEyPCKNFlzz6?q;2+#}PS2`T4vzJ7vBEEb29VnpNIp6LVtuiv7n``~$>A zL;En2yGznMd78No1KoDBhnzlEw4{?T`sgB|a%C6ckSycuA=e6dB9x(%yb`&sBwau3 zFd@XiYzcMi8ud)7t{}((n@i@r%S(v@1~&m@yK(P3kh@mj8=a8L9&k==@SjoGRnz`% zh=sF*mj+SI$VUeJhCPGR-oDlw4CszIH7%z{*bO6C zQzQ!p$#U8y3j{G@2U62}JW*VSo>$XOja^o0{`yU*fD7Al*WwnHy(I<#5F$}{F)38T zcZ2h?TK}EOe_vMiIW>)e6tw`g5pwDf1K!00An@60LpC7vDLW0T1g3*rXpf4YfY;2a zGsMio?U1-u1L#TfEtHClNnIq@ zu$Dbeh4-n%mHMI{qJq&eq%o5vl2SMVoO0=CNsbf&T^EPg80n|pY)vB73c_4ZXbeO5 zw&0185~Du43ivE@$OEo?m%fCRY=Zd{`R*dTTZUz27z)KsFM-*Ur-b%d@oEoua`=xMa)TatnZN9(0YI^^bB+Ua6IBxMrvV zQIoiab+wvtO(=x^60=sHBdZjumQriDT>Epm)p3* zpz)OwU)}gXXvM}i39Z=ps!mZAwFtG+Sic<}F_mANG8skBscpz07C3?w;{nJ(3Pr=u z8hk;KU->%{0mB=F4i~6ueq9Rpmq>io5F7eH94oYVVOs;0UWPV zr8j&pP6xPQSUw(YfC~ywLKFIiZd4&(-Fs|_G#DE6!O%0wf?mp4dX3K-uWV30mL3pu=2Y(BraZq7Idpw8kJ3c;; zem$p%B*?ZK5a;JziJS}&h;30W&VI9}Tj;QSv1rkGWc}Z@kY+&+ZPFjMpe?#^T)0+P zDp=QQaE3u;U~rPs`4*Rm2h`t+_HO|2s5y1qsCSfPEE%fkoULSX6gi^l5zkcHZ(kOZJ+K%ehC(9_>Q^4DSsBcpt1C< z^2=}IY-}DXa01A_aguNoplo3q$f>8IB+S_Fq&-ggd)3}vie=%FUh407EsO%bYN32P z|Bn!U|9vt!#Xc^bxwMCZzR34MHJL(TZF}&aq;rrh#Cjr)n&o8wBpYOX!9I)K1a&6k$#Jc67sU*SqJU|tmbg8OVrrn zL}Zxi!@ z{a6y}AJkrZn#OUsN9;ezdm%UpXBgfp@WS+?Zci9(+C5NXE5}rfz+oD(5*uxk5Dut% zIqG)dG(-2Glu57*aWLG$@Dy(dluTSyux}~1ej6+@t8C^{1V<3o5?Xl7MX@ov9g@Qk z!3Hafa|qT7W{f;|rXOK=!xV}cr)Uf`uC2Rt4t04_5=T`RBnSfx20V5u05R#1Zp>Lj*Zh;?bNFpbHb?9e|Z!QsV~UZKKT5&ljFc&dt#J zN<=k8lu{$ly`&fMS=DJGibB*6DaKRS38G>^t}34dk^#M+x=!i!UUVdCctZo}<~^Z! zAhNZ_ky_uO*Z6>es)3`ap?pt4@$#8#C|(AxV_J4psNkN82HG@pEeXp<*$00@!t(iI zk!|5#zFTe1n#;%NXPG+9qhxt2Kr~deCmz&i*zl7iahz0t41N-OI{w&bt?jiZy75H) z(;A#K{zuzR^~F^9VEFqq`AgZX`u>Uh#3#Qvj+-aMmfiq_F6kv{6TsnKnvmAoI)kObU+~AMN*Du)cd*TtFCvZ&-`We-1utPjCj3H(1WyIx~+l zVg6oxi9daM)x6T4{!4xfJL+3Q-y+0le0~+LbjDBfir}*rCl?E6IZOwG%6xp@KA0-K z(6{T_t6Lw;9Z`$&R;k?Zb3cpc8Sr*-4Z zBS;9&!2CK?IL~|wrceTDE(;x0^pLX&d#k`SMA)Y zb=A%_)>T=3?VO5+^bu?qq_Co{Jb^^UW$+?{mxgeR7UyusnIa0)qE+AEQwj@6FwW!r zI#4fEx*N+#i|>MMWP&h{Y>MR#qENM^sT&dO6t~3aQQ6oI9rY;-ZpZ>W_y!HYm_dS3 z6QmF?Wf9lnE6aw9T-p?2l(?V>MhDn$ zArk53nRl}!o(M8;Kr%9(V0o$tFfO71^CBs}SIcr?9R*tEoCVO77$2r*&a0|Ff*qT`v=Sy>wgVPhsDjGA^@e$XlC~*^sR38y37im2{=3h=LozU z+SA({&hBo`d=e&-*ulo14U@?DM^QA-nW`1Q_tN0@58jyOzl!RZn6`XjUV|6LHRQ<( z(|HNaGydS--b|Rz!R@;l1T2t>^VNN`?ENK#3}0k6R8i52J=8d~#0_oWW+vr-+=P9EuhdERn=IvHAN%hdOY&R9NboBE zFKMNxx~FG2u`D`Xtj8&QgER06%LrT$>%&3{Cq*f_n~g!`+$I$oZ4)!g!I~Kfxjcv^(J+fSixs_RA?nnQvk^7NQp80FXc7T^+^Jou2Ek;T>c zd+EW%L#w5e)`*yn1@|9C%XbzhUYWMrwms%Y<5(KL(A9@i`QerPS3XsrwMyH7V*twz zp>66atoN%m7>=&qE8*zsy;7Qa5H+Wq+pRq_R8&-<2#910Y>8hWRLm)S@{$qM^;vCaD>5q-##`g&v+_Xx&Edk6yk6=PPeQ_dDI8&1_ zm$PJdkW&eih@&QK5hMQ$SkmCT@30FAFnv04>$qjG9QjdFo1iP`^`tTY!&$Z2{F;_6 zV()o9?X@%e{WYIfu;3w=kWjP_V*<}YrLSaERzD9<0nXfbwHFyI!E=pUtmwTxU3>nFWK!+Hu&3f|(X1=uSM+9Sd!He}W653)n5v+m+w;a(6*lYL*1#qd{b z0jCwFvZRUu92c-hW-Ox0G-_YzQz6&>2W2l1 z@lqh-FGWPJv#&ql+KBT_x;Dy1=Wy$)vcJw*|4mi)yIv!Ag)oGUH~ULuaEBC8pcGlS zqRDx%x^M+K4;X8y%22|*TO>q?lYH>67>Zlq(QSDN<`V5Cx!<{1A}3D3w;{}ssCf67 zor7*MyB6+V`RKJV(+>mx=Fw=^g6^*m-BI<oE>vy7mH@Ni+MmQ;Pz5n)aC*Az_9jqVDyGwgs8W&nuX0zI)s_i-7&e1v7d!c+iUKhW055gUyecsPBA0M& z8v0`F3!-_eUa4xA4ADK@3q{BsV>t&ze4&!Q5{}=$s@HH+O5UA}O?%G~;j2b6vOo8Ly^*VoYdVP`|17ZTOL2oeYT zQo;(h+Lspc1i*pi@(M9)d0RHLx(QMhGVHJa56-P{w*wq`NH0zd(hIYDPQ3yP6Aq?} z1%R=|dO@p3xHs5Shk(aXL$_v4-I{^3a14oI)YPp^J3g7lz@&f?*_t)crKBV=SoJ2s z4#LcmQ=C}jTwq@n5i^32V}y(#d1Iy_54|x6=?8C&nSh+xU4a8+*J8~9l0JT<=Hq56 zfz1n~CO8tRf{x9OkDDDIKWQB=)^dctjt03>AxF>cFb=s=*54g+r7S8fJ`@}!RobSNqGvI#NPfhmMA+dXBtX}Xg<=y}A`>^58t8VIeDtyTV?P#K@*)aZ zS`pU+FIierZ9`^NNHe0w8=q$MVz*w>jG!n7c}AGbm~~a25#Ey0M*Kff?ukDLyD;&G zV2!T{xP>LxLyZ2(ZW5M*aLeM1h5R|lx4Coj}I1u96c1(pt ztRWA=t~e<4F!^(m+JOO#z21M^x32Q-eZcrl?mOn}LiC#J8KT2pK9G605c>E3cq~C% zTrz{Y8Et2GN_rtWqmAUkw0_rQch7F}s*mgkc4X0#c1p52Uy@-G=F1}~feti!?v3Bh zF3=!bK%PJghVWt^b=8z>l|>3SIG~Xtnj9h6cgDpb8;W~dW<%A36T`Lx+yni64m=9V zA-5kfI*x%P<3p+`@PoVh?Q7oEZ>PnGWm=a7Eal84eYm;Dw9Xf6-v{QaJeg>|N)SWh z)Q~8SkwP{(WFkVJ*rQ{Fky&4SM(Qk66r@1}K32(N)#YpuJSb=hBPOeQ<+0udeT@xQ zHIrH{#l8p76^O!W;2T*&V(2Q+Cltyh7#@lth&3d{lKHa`PR=T?H94%&>49fkx#Xe| zHR04?*0!7)(GxhdYSv~Xpu&txMC8z9(5<6AKx7gO+*FUUDK!3`Zw)qtb!XoD3GN7k z+QE)c_^B(mZ9~nrmR`shWGjD13YQg%;xM3y$2aaMdJKbYuy~--j5#QcE0n)aLfl0L zOby_i*}PHBt=Z(bb<3u{sYaXwt;Mr8_!}&%*R$!p$ma zKMOhPm|sOBgqd}%;nJt9-yfLf=imeDu< z`VXTn_^t5px8VD4d4s}WDN1}M`??X-q_p^|lsu1kCb=K{jr!KvE< zif-V~UEN8<37Ha2VwWKBQU6&a!eEa#;29c+6kMRAV^3v;9$!87P`&6$XgUN&h6ag{ z%~f~%m&%7Q=63&rX%Xm7ViyNhUKYtk2m??6Kw@&XpGUj*&GfO%$Ij%h1)2JoH7#3n z)!?i~kMQ+lmq=P5ZX0`*?ZzP?L2ka2C}t=b>K58@qGe!1BEFA4k|g0UBf@g=C|?`F z?A5*aC|+B3Z>zfNo|AMJ7%r$+B}Jv#VZv&|Mj?)Tl0K?;mtz_xWESikNJi`uV*DF3 z@kj3v*%J|Dy-KCjycd8gasdybkiQN@UO1GN757>8u?dUvge;hPH^g|0c#M%GP6kTa z{E44p77=K05X^z&AR10WZCd?9I#u?8DS|Jk>Rwy?Hj0u~D@oH%22bi#ukE6t3Q!R$ zwh8j|Je|YfApc8zW z=4E8FbS=~cr$fV=w=fSa940)ECOlA{w=L);P`X7V;&N#qQUKOdDvHC! zI$h@Pj$q;dHY2rl;)+ftLZ%atOanR@{DpW#C(y}snCOXF5m)hu2x&_=4}3Bs=K;-M z5z4^%JQ9y+da622PgUnsJXJ_=SWqt6YNw0r0H_@O+#YBcW;~9j2WO%G>x{ij`k(Na zxxGb1jEIG)Jns66?RizwG7 zLoX*?zvWd(tJBz|F|97H5g~1-7iF@&Gv$6FkrFT`6kt0*<~j>|Qg)qKl&n87S$)UU zCTGFOzf80@G2F>j0Z9OCP1&!ml7t7`vH*B()n(nQ0zyDcV~hI?ZmbZj4Yy+U8#ao( z71_@QCjz9t{wn`qXpcN;=vK76EuJ^HN~&350RZ3GB7^1f`q-d`*B75ep%-Op^xt4uG3*lN-;J0Rhmr^x8LWB|3h#1`Q$KMe9FCY(d2Emtiy&23qLRLN? zcxV!sfi6W4VhT+bna3TZ5U>bfM&7z;CjU*4$qFIE4I52Y3a9TXi~~%4x0n5%i~V%jp8#}Xu|J8hFSg%9+~+=8tTXu&aG=)= z=wN%G%|1x9AIg$v_$^`WD2xmP!EGXC@(*ZwJZV(J)Yw9hMfIG1lY!@K~*L%WmstwAA(|p5okxaT?s~=q(GSg z$yH2f6zpwSsj)1`?}Br6=+%eVa)du64OPrwC`~hRKxu!x41X}mPKDwRDrSV^Z_O3^ zlprEQD|TEHcf@NGcVM2bCHiBJr;wm{mKONNM}^~zZrfu42qFw|HHA`orPzAyMDGf5Jg0ivFc6;0+NBABl14*_ysk`bC?0m zHLV_|NMtDyb()-Xn0O^7fD992ATk0`7=sE+%RvD95hp3OjwhLSq`w*Z7VnzW3Mm<6HCGBrBtMLaOI-qm zlmRBS4R|#eIN0qA84hB*Bqea|IR(-a2$9+4CMT?7L=V6rKyu`Uct(DVv$};H&sna; zV>`^*@+sY6o+*qRv%C~LF~J-$b3h&%`GeR=M_|(CGDqXD0}pb?Wf#)ZrCcljr!5qXrV7QM0`mkccUO z!H7F(3m8nfb^wDV4NOwg5%74;3XwOE@fPix?zerE+vIU{ziq+c4AK3z3Bz$N77r;q zfNq``B`lhSj^a6~7ON=1gY_)foiWuN)-zc6xSBM;sLRrF$&eK~bf=~j3Im!}DAy`@ zA%U=hBUAh6krV#{g`T5{5xAi-c4!M1_^z-!13$wA0N)%xw zxp4}0lw$g%PGkBE-4%o`aKn<_xazlvHx##b#coUox$NY^{C`;9~fjw>SVp8Z`U|`Ne`4Kcoue*^##c0 zf={u4t1^Qv!VNE21cPT-2B^Ldc;PJzcOdSOI3siuLNQ#UaJJeUopq94;RC8`Rlx96 z&PZNwiQz%}7Ma^`k)lv_Aw~yAob|=xM4~|@BFP<~ib8Vs$f#L{hB0cEqZXfR)FeP0 zNUcHO)*Ms1+R*wnzL$6?^zld&Bsz^IwAchP(Td<;g@d?+2jfq6335TrK}cqZNyDKC zd=edA5|hTCAtpVzu#QPL4plMf#vvAn9*#*jzK!AjTNRVW*q}q05yO#zOIkC}JiDq! z>uV69g4Ijn;)32cRgrM6M(7HazdoIalrdT|>O&zs;_uLhjkRiMw+r_Js?=%xn|&eG z(;$3vEPW#d1E=K|f;=FGuXQv?PJl?jn{~MXAWVI_EaLo!EhO;Gn(02~nt~&aocGMY zl=?K&i1&hnFc>1t!4wcobwl(4_EyuZX`@+2W12b*HEN)fL4YL4@dx!jp`&47u)3(2 zfGuQX8nQjD<~@rTUxUqyc;N{cE-jRZm?Q_Ri_2nymzDi#UeWs#P} zI>4TA+Bz(ZY}x7Iw7i&c>)h!jx<5fdnbh^*>*GG=j8Xzw=8!uL1#eD6!JERmDHu&% zmt#Mcx}NLiuhU1F0V5wq1&f=F|*@X5gr&V6#96 zn_PvSz$1Xm>iZ#W0}R}=23S}6>_=V=`-|{uDB?4Cwaj(gWbo+Enpr8{2xW!rXM9MJ zV9ZGih8E1f%@}*|ppZ;5%*t7_8I$)2XW}e6E%T^Rg67X44jQrUrL);@gc>9(;PON| zqTgI%1i%)qc7hdEbX&+w5}t32D`H-5jqIdZW+%;;JIofWujlD!^MET+b%gcW#84-Z z401@3j~oiQk=9c;cF{xBp4=9}LocXr`jjP;431^%|LMDMCVH}8s(dknaYoW&=BND#)fCr<((Q1vg+y% zty@Y`PCpe$-UZYl+*>(!r^0Qsb7DR_ZptFQphbXOFUvtfyE=C3QHjLs$9XRQR4j@jk#abt)K1(G|B%A&II9nalVt zyRo@_LLIYMXhu2sduEYh|D@YHc&k6kPux}s>-=B4EuIlZ?daxwy--l(@f}x zuSy8%dYi40WYO&^}M7&;tVH+}ipr*qn^5T5f&}4n@FMfgLcgh=kWxDow=x zz(vr6P>OYN2_f4y0N#PeK|G*_9#DuE)&}kc2iv(OI^anJ2G9qJf^1u4k3M+d@cvOO zSlS`NOES67oBe+I_sCy{vN2xe??a?47(%#X_T9*u3TY^{tgs8C#Ri&TS~mx2krXWP zVf_|t9#B??E8d_ERgqkP#O)I&zWVW(fyt)&>!)9izaC+Ni4D;?DQ>5o&Tr9K5ijTi z3zJA2ZNA($-_vlYa_|~&KKv!(4oCYHbetACTiXotM=;4yGz7JgcJ~yDrJ=~fzMft{ z9SXdOq@O>zHIt~Fh$aYSlW;)#^2x281T&_to`fPR*QC3TarYM9&k9dC8{kdl%xF|0 zr0YReOg*OxVcDRgThFPd^>l_l69)x_jglD@V-w4%Tnq>xogLA6I>!N)8VMzkZqrf% z=@9LLLZ72^uK->Fxkvss-W(E=!P^2+#I7O2-jd~rrk!R*ywqvG{3rT6b%xUbm*!j= zSZxW<)&zYom=|KE!f{2|vaZ3wx(4Y=(wZQHr5vEF6YvISS(`Ad{4HsAZDb;;kaM4G zpg23iTbRaMsPGboF9Cldi!jy(d_udIIZA^>N>lqb|t8rHf)3-pcQ0cL1k$+5IdM@=SpRg;P z`$kHGf;7CvnD~%62w>D|Ij$r?#Mn}dxtU5SkPEus_PS5DfI!sFEU1V?o7je+D7w%D zxdrcVZx*p^PH(ATnS*}dXgM^!8HJ)AhHHs`J$4xxAY7}`^akFA z;E>=hvBnk)=Oc>tfL)iYrwmI`8^u&anD@F4OX<3uP;+dmgr#VzMokrT(-Toa)gHO2 zd&G3dHhZ=1q3Moq=pXuDDo9fF{D~h#|AVNYo6_u?_D8J?H5Nvuu;&`C4XjWd7-3b! zNIv!gJ6+n{nWO&Gsf1IBNgR4#bo0BXiZ?2tgvC1sw0S|tAAFRU>K zQ44viO;MNjH7Jg1ompDwprX9}xf!XI#C?p^jr1BH&gey(oLOiu<=5##!^-AS52QNj z0TiQJ8s;2EdQ8I@3}_6w?J(k}W^9QCL6tK*ZR%B){aIw} zAq!I?xGuWEbunx12Xio_TOwYOfBj$oGBTcFVig7pwRlO>!t@wRrZIx~!)jj9!nCAn z)EiPi?H?o&&N`*~#=rUYHj1g9jsV!3Z*TkQ9R~S#{`TA3;L3;>)X(>bXMdJaMFKzD zZAE_Yhn8y##6 z0KfuPty^w2YA2ATtg!8;fJ4FSlk`QA1#7}AGO6$@4X*5dD=@yb$|CXK2J@NM`%xP@o?)750u1a-5g!w;ZO#HOha5Ly ziEE2YF&`k{utREG(ongvzeZn5x@DJCI+nR`u}l2BP5BIs0Ru%<!9#^02D+06Q0>5 z2<8b6W$RH0<}5d3Pv#{VRnIGO|5fg_S#6Np(OB`2st0} z2mKPqjg%~sJ(c+Asv`2eJu|4Wxo3t5sN?$5nSNvr5qi+Qs7Fx#vcZ($9s~BM@o4Ju za&%GAnaPm76h%}MQ313q<5Cez##2U6tP|1P#NQ7p!5|oRrT~3aa?y++e{gDEw5suE z5jd248>B|sF#ZNP$$^^@BZvS}J^p|qV*79`e~(7|iFGj}6v9-P#u$INOpzf49%aOn zVq}LQ(2bo?2{j8Hh_3*9Sj2A($e~vwr50b~UI~X|PN~=>8N$|pYJ>_p_!#X6*3_Z6 zFS6KJ0Y@Y7S>=@Bn@;r&02YSG%3Ez(`^T2CFU@~4LV&GNwJ-CoL3YgpU zdsN6-%8|4A&8aMeEMEaM&HwDvAUmOH#>)HhwbR0al+pOVSuKIUdtWqDM8`yzks+np zgv-i;9bc`XY$c(AOw=|cG#Y3qTS=&qY+YMRC;=(pl7WUY2#~^3IFlX;=q89LxJ6+& zoyqh-j$5U#L1zvYa+&UIeUmqw-~2OD)p%K$g#_>d+y=|=;TB23`}CI}Ljw0Fs8?Jd z>PJtCnLM?e7@SsRDFg8QRy{c|D4q~>HG7XX@ITi`C%F6ItiyiUsoO}pfky(C2-R=d zdD2Zd2X#or%>;CWSLbQ)<`K*gNjQlB3KS5G;Zp;EK)j^1<8OpD3^-EoUC8np@zzCh zw9FT8bENL5P!!oSwLU1jAv4$wIi3H2_9+bH0k+N@4MV@6!NyXf=d3do2MuR+{+ckS zg{!B=9P&9r9F@JLrQltm831Ega)N_??b9W33;# zXd;f|UO1&2$8i>S@^+j+rGd%07cqeX;RMqBFl`nIBT}gfUKcTU1;ANYG&a&vB&bK+ zL8#{o>olrECk-BC096F{S&?$%+z2I>fHDQ!838j$+(y~}F`p=e1=Sm_?ChDzNQRFG zgc91jdy}{hsCcQq%ykFy@1jL|%Oa`1SyaWtBB^JE&Ic4hiaU}I)F0R2H?jV{Unf}+ z%Pk50F9xSo)z|GQPqWTPKQq%$T+5%o4_V0juapabn^7nUQePnSF?hw1b8R@+7Ff2B z9F21=aVkzV?n`JMbQUr*T#OeqNN>iLxDB0PFX?r#0?jFlQdBi7mbLc9eje_?7R2`vzay1kYHgr4xW|< zGQ9KKQl?gD+C9j*eAw>u%eRxFK+UNnltfc=sufD1L9ynyrG*OT-XU}bF7loIe)-|^ z_kl90weLz8OQpO@ckn7M_DQp=C>z=Tg*w!*zeN9U!lrxHq_D(jc*At3%K-)xOl+uc z5*X0F!oQcy6!sXHDOe@9G6q;vC)hx+?x7cWF>D&Iv+RpniyhZxc>wrHDi;77Q^ZVM zQWs19(qgEEgZigHFX12Y{61!}X?QG=zN94thh=Fb_(*Ob)D{3RpXI0x%2C5P3WP75 z3!IZ{-2k!K>r=ed6|{g(B6uV680`n-6fdktzna|V)BX4cq5FXlD+sag?!WTNE3>ojG0Tn1Y=&y-O8Bzrpb|@)rgio( z3K|S5Q?s^#3yDvtM2+>9^LV!m1ZL$P1{GUqW0y1oNFC7Rf#6_)c>t<2sZ}THq6}1L zyV7n$sxx(MsJMixlNZ#gv%Y|;lUS{ha|f!kd9Gc3E_ry-ZtYT85mFFD0GCAq2Tb3< z7y;WN6yttpS|oCIQVMY#K^89FjZ`5u+AE;^I^O7&aL(#piKJIhq;k-3_6m@rSWrfK zB|_z8g$vB!f;D<2l3qb=%b|1E?iH9>=^~liQ`C*(YK&6R+@1*~VT4^;RaP3kf&(aM zg|xU703lYWP4<9P1R{+;cjH!wOL?w(gf9FQ_ex`3)=-vydr2aGr@b5N@8(HB5RG#P z(G3fYjcH`&lw$Z9N1iFFSk0QVcbLeF4b9y?PLrt5oH`4+d_4bb#_9H1oG%ZwIYiS2>~mgU9f{}0O|+x)#`WLI}yfPlcOZhW)9MO;Wwr9;8~{C63f zO(oDDooh+`yB*+?KX_I&dB`l;gZjXk%HNLScR(GF>u(D~Y5X|xt9-cx_#|Z5JB9E(CM>Fm-<_E>GVti3`=p^Wdd?RN5N}Km=~5e(KfT@LUF~i zd)ASW2(Rl>|F9}?(Jmf@HrQsbt~=|uQBT`xl{*bCRhB?HPtSVQ zhwDNz5}_+M13^2d&i3EX`#j3#oK3Tdm}F$)ft8~{T_FsL7ORXPaiVQj^*4wUED@{@ znCR~e$Gmn4F*s_Ii}F9vsB(a(=_|8&b+)r^!@1VE4SWy${0h3Q_uwMA?3kbu5#bCKq{x zNOiUO@e~}Hb4np8jN^jwSvEij;zjzw=Jw1K zSc>T+eaeNXOz7JC{U2`I$D(uwwnpNgx`&zOEZ}QpRrL3(SN5=M9Ja4Skzbei&5Kpe z`coDq?J$~G+W@}a%mK;YOKyjV?cd!)B}tebtog+~7B(q-a~JzZPlUo-*yJP#2Q?>4 zR8N*@Fj=CRBH?6NP|rtx#$c)dZicC1u!+79U|(i?yOvB9vP*`FgsxS6+6yKx3vtTme~*vwYVCJUM52%UT;#n_R?eBOPkbWt6(CnK0w&qYSzst&v3H_5VJ zNQa>s5!RyWWd^Us1q-4^i5ztJwcnS|Rfl6u8*};I9;jrbS5yY4CC3G;Bc~|J-<^%V z2VX(ZXry$hby_`4V{}?IQ1o_ipt$*~28zCwQyv99b`+=?PGJ;U8G4}krkY%TOhU2T z5|Z+-IzUB%KDUsHmljk}p;O@UKYLhTsbPJEbw$rVhQ5OFW)PUM@VrfcwL_QvNtc3( zs=v;q{svw4*L&UX1pzUx*C>_)&AfJvoIVh3$XdfoWBQA>14n74*ZZ-gT*ux3UMac~ zHFsmbjs>ttt*{O?w|Vs)dXwl+%s83q!q8OzOF16?SD2P-(F4u+C7q3(jT<}l&&E^x zF&pj1jnT%98{>_MT%k6$HtLNszmi_+|KvLfN!0tXOL~ca-M5o&ui_@fB;+gssl&bk z)QE>a5#|e&QqUB+1}^s8fD@pt}m8Gukq0H z)Ykf;K&B~Yx2tV|s_`vviVrTh*AgzcZjH8Vc-(C{{6krIhR2u&$n0+UU5)gH<`B^L zI%WvgKkgQM=NluvAu4t~R|U~11(^p9xhRIGF#jkbrcl2b4YHgD*~A)MvhJ}PtR{vw2_}*+1;DK zgt+e+E&cKzEW8YVt}$(@=};SeYGCSRtXQmzs2BU+{&oV(jRlVZ&3ONJz^%jzz?>KD z>fI4|?!Y2p%318^u_+ytEufZ+{oG!}*SW~W%&4<~Fs4|%cbM(2?f>ZB80Bke?V*OA zkHt``Puk_${uoc%wFr|C)f;^R8>$sH0?@^OsNp2*nOzQT5)!3WpG7h?S(p>4$7k6dv**S~AU#v{9!9kIr4&zvViZqq$h4dwLqOTwgXjxNOVdGUwVw%l{THbk&6l z1-8>!v%hHj?76)Oe126C`G1GcHopV)I~5l>maSZpB$hH(+O zov(5o*In#)K=HB&3!vk=rTkfDMO$JlZLb%7_gA@q`L1i@kHxG2hjGz?r^&k5@4wu& z%K~(GGLz^IwTKV5Y5H#RuztJ2`rYK({;7Lhm)RvCdFuqzw-Zd?PHpA3%wQHwy=E1jav^cVRO}*Q{*GAipU}uk`(Un_LvF((rr<2=qS%#su5eg1CrTs}rJumLg$W|Gb{5kj zC8d;PjB@S{?gyt3SR#qH*(Gx*s zOzH`Rvo|lw*)GC%GJ(01%&Tp|4>9XDxcL1y_cvrg^kkMb3@4ZSo21J=c*ISuJc9Tr z#|=89?emh@_SkhW5*^Q?guHn}gyY@4jx22&Tx+zPBGxhgV!*nGVeQ9UM0Dow_)dc9 zZ9q2+?*-5P=&G~t_zogt{!TgD4xRn^RcGHRXMaG>MwZFo-m6>k0Xh4Fa&}yw?F61b zXz~Z;#1F}dNqyp1^*;aKRyE>7a`uPi?AFlPpIUYHhvn>#$k}OqwgW7%HWm6Y&d3_# z?_#X61M_sODN^lq#S{s9-TI~j`=hzn9a!1TeyY4>loR7+OVzYY0H=(~uy6ha|2w~cUFdUXNaIW*Vm+_6 zVghU7O;bCq3xpA}+y~QR$bBi>r(o_?wM}8D{_xMm*rhjQwS?8`y&++etS+%i7>OTy z+;J}hSs>$6X+I{UDk{YE)^pwFPDR%_Nb%Gv)} z&K_23f3fP`|5?s{lbk(JYOB=Lo4!4wruO!*g#elwjzKu0{-n&(Lkq2IiqjTau(3{a8(%ad zR@XsyI*lzu&&P;U<9oj$LR+cBu-#`9(CPit>@GLy9imrO zxyd!O&E(qJCPfPm6sPU9SA%7CUaf%4g!U!3v?$h5S(xth1I5M#^?X)_vKIxhMtwIo zQMX}YjW&dc#>Co9u|8#7fi5_>gU2ohU@so)*N8!9*@GR?ld6KED4gI!o z;$K90{~T>dMq`oB#^nLO0lt76UhLPcnM*-^zeTw$J7SsFDY*}`k_iqxo~$hX?nyu; z`c{G}2)`A^AO177=wJKWhc1Ex9qEqb0P1GI2vxgxdhgZRQ7yGbx2m=KQ*j}wfWu%; z(!PsYk=ok}3q9li`=77B9RHMP;T6PsIM$)(%J9e^zwz^`!~#(|{n0}E!}f}gB1!P3 zLsxiHezfE=q}1o%5|$SRcOFR!t+w73j zfj^81xp-+Q++@~lmYhu`+c=_QKyVcKzpY|k%m>wc!fo(EW!WEV#4CXllz1fui+~tJ zEHSE_%7_g_A{h~hK^e|FX}oe_`4gY}?sfnCi~|zsguCPx)Wm?Jb>V^5p@rk0|Lsq{ zF*&eC^wUM7sm>7te^w%(*Tz-K@U9X(3xj6ED}a`Zi}PHV9J9uu^|NMkpA?hReUxaE z6P_)BF$U7f#Z=ih)0KP@U&9>J)`%rDKB1i!%i0mwv9bvh!OW}YR2ixRi>Ae`Xl>DG zr`NJ*5PYpI8lE@aqKQW>nufsw%gE58F(r`ph?+*u3|lntak$#1(X?sevOqD4V9lHx z_B{ACH?QDDnVSc_8318d6tVBT(r+*4RU0W;IFeLHu&s;zLmK-!p3>ny4+pdyXaNcC zjDPBXD)g&m3U+=Kh|E{{?N`nd7Y^vxuT}>DDcAmiR(${{@umyudt%1HS}v-?;)^D5JtexuH>EeQkCb3K*8i{J{MiqY|Nn zjpzoNt%&2cntsw+WkJ9?R}{K>8V1<-`_&cI0TBw^J3I_n#Rpnf^tJEyV}A}3;HiXR z?4|vfQR3Dk2DiK-LoXwW3kO>L49ZM(e^gR|kV#T}J(}-}klTaNyju~u(NHO_EVQUY zEj~e!=1R8RJSBa8UQLK2;%v4BFDg9>skXCbJNkG;{OzoHCdB`WqE(%Zz5^!xA$|Ek zG<)Qw&bDEsjIhgVNZ`qc9uhZY+`bGHH#=c7#%XDe47 z6*kbMRmq*I78LrWNF5Omtt0;$h78sZP}98H#SOi&eDuj!Kak@!v?I2I`d1`1KQS&gVe-BgF?T9;)f&ae{|Q_tPDn)*c83OoodOBQ(%oY0_x#46;C zpl;wT64XswtU3S~kqU*RuLrTP2Ga^z72g7>vf5!D2*2+`ZAG=yd>MMI283L_!`yG2 zMt=ki^j~dBgwy*h3K3cVdlu}Z)?aU>6w#+`+3&#LS#$5{;O#a3X>h9&=gdZ3Mh2+COXVH6#F)+9C}8GYf7w zTz>MuP{Irw6NzbR|Fp%47i^e%j31O#Po}2Qsj2o{dJpCILpr+ zw&kPBf};V(;O!O(=+-Ycj&|UjgPvgqfpzv#Wgj$;W_E?3sU7Av{9dtQ*J1zgpuxW@ z_Q3G?19tbS(7-RVJ02Wr)q1`hE^k0a&$Y{F5Jc^5_dqz8b@NTug${Yd>@bI`6Je*V<%NHU$1-*G~E(p+zM+S{v_{TjRc z_Mw;Ug{j(KhdFr@$jm6;>&EYC@2>83EA_+CcN*g&u@SUSIZbb=g-$qJ}Q(gx70&;Y1HHXlb9tMd(@P0p#3C zNsBL=azW=ubrB<=kK*2bds(D@Co;bEz8NMBe;>mRWEC`|g+i!A1~UEt3iNV`D3LqP zsk4a0lIMYJf#n?jHRoQjZrtp{kPcjt=!8YzDt+sU9cCE|GA5Yx{IY3IFuLzSd!p&2 zt4f&qkFc~wPZH|zlGaT;stgp#aV+RP$hiIxH360a57`)4F`)%P>-X1@e2OC2p9GWA z+iX7`Su1e;AiP3%r-nU&@H=pJ(u%Q82eb$#$cM~=>aPQlm4|y~?bW5_T5GK?t+5Ij=s4-|>0+Gx%-h)t^Eut9kWlI|w^KQ@eFLMRV60d#t?p))f@y!^6UH zct|+@)n!>ErXkUH+_?1+WtktEQoJ(U(3PQvGDIDWnA@upWl6q$=)@Rs27PkPK&TIk zB#lgeq{DIg2Me+EPg^zp5x|>M-!?wlhoWm#6CeaZg(o{}zHENY$5 zy}#RH5BqWWQVm?d;IG%>O!Zj5v6PoZ@C5S^HZkB$Doo`bz9V%8;EjXG$9PrI!mnK~ zL24pE`=ngDuwVwD#>9IC5_NW%3!TZ|ivsPlW(m8DS#$AX0jC~wZfL=&3uj32SYHj2cRMl9h*ghA8}^E!>RyM^WJ zKlp=tztLaXX?N8Fd=l3^0 z|8bgqUDwpiW=qA#OYR}zx|}tyV9d!+U0xQb8H8XhxEr6L!RRRYp0lL6cE*3`H)8+E zKa2hQPJk69-8Ta^iG4H3U3=hN;7vbSm18gp2{rz4k~eVxzg6)#J?=`!kY(iVIh(z~ z=rrVw{{P&)4YVXzb>CZcs;aBItNY%%qbspX#o@bFVZ4ImvpXrJ3&P`Z#qy_Sql*{ogqqIZ*I4y)a2fUH)+- z)_&xzU1)+vJ+k65Sc~{Tq00kDRJ=fW{k-L0W6D039)p;YMpNOya(x7_BMBCPpZLxx zpM^diOBRu?(E=MmrZp1yyBiK{oKmRJ_pSspgOevGJ!JralZsK|@?aWR)o>-Uz>K*+ z(MZIT5kpK0{4;iP8gi@43|==j2`TD}>K!hT9~&Trfo! z(Vczne17Uf4Tk>csXM40u|G+Pm*F4DMnv{5-3b2( z1sb7rm+~Qe5(?U&G{y_r`vhPW1?V$aTgtSA8PHAK0eoX35x;QUX`E{*Ws4^6!Vlf? zU71IZ7w3w-);5%*t?Bc_CWHa@AB{}Lg@CwTWHf(m_T-Z@xW;q<%*zKnsj~|c^lbAOUgyQgxw|@EKAOFyS>koh9 z{SW=zsep0Cd@~<#<8TmPnPSzmBR-jb){Yw{u__J)$sv#;Ae)8!{|TR7mgg1u_(2tC zNH9JLH-Vk}@{%Y#io;(jw?;+y@=`S-$&qE?izXynKfT#cR@t|baQh#%YCpzsy}BQ8 zQ{$kG$$Yc0bH*7jz_W`WnYcsRlt`>~7r2WK`+rJd90k8G45yV8(ztcMRNi>s3r_#c zS3di~AAIri?&R77nv22M?&Ns?PWYLG8EDg7cNbW$k*d*C40%jaRo|~*_Y+^4=2B;p z;&6hCYo~CxclBpe0R|!!x8n)V-mzJ0XEqB8;BHm@@U9J7yaujk<-02FwQIlV;{#3(NB2vyF+DDct2b8Q4F$-;?hZXdzSAzT!#03Bq*;9DzDkahHXj zjmd1)E+fCK$<4^uMjKRBA5Sy#vUlT^UcB7Y0Xn?P>k)p@t7dX3A{8}#fQR_V6R zR-l~8tqDb6TV;e~Onvb&1;~+;RvCd!2CC0v`JtFpwY9ai$$s}tBy3Y0;p6{snEKhb z=U+|!YV(nDGN1aQg<;+i_@LTrhIld`99JF+vO5| z{!~jaisXvdh{EP(=ZYKBhFdE3L#IV&-}FOAboLz@&|%7@A_wI&h4|9VvAY{hsUt4< z>p((n89}cuJ_sI zb-SE}Dih@%3cn~9eY#B|>X);gv&~X{*^ZNZQL;-M!r}Him?Wry^}j3uLnPi=y=uuHs zFN9%Xhl>}R(uiKwMN>hoTukyz0N&jgkgDp7E^{}yU6*>ju>Ua88f^`B=ZVYfFO(?x zAyqyGniv#`Doj$$3zHb}NYN@yQnuDx-dbLejx;|#|LS4p%hR3a>OIHAc%&7VsZxoo z4R_tbLs{ThkOGXg+Hl_>t)l*%U?Xt1WUo)_pH8cBdAc*s?(^jjid>LYM4G=A5<3?N zGSr~&C3RyqYf>4T%xIf5?e+cVA8oJi$C*JpkGUvnRRbu_pspq?GpJitE~&2EWfm+B zSTiBP|SWwPknoK&SrXDhF9p*&T6GH>8Q`#H&)bUN@FBx zrev+qcseU)%|PQ-mOies($r9ScUq11*yJ1N{hs(HW(#JP!)W(Vb~giApY1(F>%4dm zD{}|2b9{?gA)e9G*}bh4PkXa@x+doMY0V60vuRKMr~BDY&%b(@`lp@lG`G@pRi8-6 z)OV$$rxSd4(5*AhXVDMmSgVxF8T;X!c#reXdZ}iJU`xQhv%Qi^Y_rDDz17CpPqjAxEm7O4gHCG*p1HheP8*az9SnC z_`T}~&q~KH;&&G0!_E&Nkn4uQH3W1<%RYy(UgF;8JDPVNhT-m%n}$vD9=V@Cvd$)y zhke9`3g7p8{CRiR{{f19Bhu8*kw`QA1@(q~9!?v3rXL)14?y&hhYy2^@euAB*GbmM zY}_Tcx^XAc>vZ-f&s@sVA}K2LhzmjZuv_uBJxkmevcTHw32B*hQgm)DVm^LNkJY=6`_XR z5D`CdAIA4(#HI`rhwisf#Qh>wxz&w2%$j`SPPvdU4`$7TA$-96Qy>yymkfUYa8j0eJ{#E2=QZeQRx+>vLw4xU)DEe#@TYE(Sv8A zkimX1TZNp}%#s2N1YUu^m)E|{{f~WSNJll#Wg*Xu@7Q%#}_AEjloGtD(D z7qK?2_+^RruV8@}u#}h6(B0rirM_!Q7xhiqUVD!6F~b0dP%NIxgJJtc%eO8-tK>S1 zT1?`Q+~y{3AEQ8+Fy)Sxez$L0f1bL^8n@fs7<~D-(kcimN^I5T^*yLC%rpIt+2GW!okDS1H}3`}XJ& zs{qWR>-nCvRJrdmzDTCUq5;s&&6fE}^XYQwXu6f53mOs1QZoMJj;~ttMUYKa*BQ7;u#h%s2CN-6KY3kE zGK>mA#RObFoB@rqLLLoyop zbVe>vBYHZ1pibL29S;%1EQ`2?Bz}i%FmVkPh{gfOosRmU`T{{!crupb8Y;X9;Bo@J z+Oo;1hU@Kq=L@lhc+>cyeq);ydukOWqNqHFrlhPgKtox1*35}sc}TA>UpCBIa&ahW zI2Q=IbS@B5*bawrU=awZy-a%9u&blnUk3FAahNtQlb-o9>8NL@gqYawSDQ~_aBB<1 zVe?D-9OFcl_pv2Ai&0wn0bMrmD-xjEMd7inU{=(+@Oz+PxV!#O-FaEhY{|IU|D{6TAhB}c{O=wn;pVNI``*3Ej2=h* z)jQs)`m4M&@#a<;R(9rxx;CfTTs3OqcQewPzL{Bl!IcxJf@LCnPT*#J{IonteR%Fa zE!SEf42n0hmFOE$U^yWsZp$5X_bZYufScQ&yi{ZQ zy3=_`>N_kcj*Pfc+~#t&YGD-|{Nh(SGQySXLoV~jm7Q$DY1H9&8GDfC>Vn_Z1)^Bv zbfbxwo&LMC)196EdyA)A&i3r|-SyqfnfOnQG)s*e_eJV5?jk$;c+W>2_~iRdTOsc=N*HZDg6Zbc*hZa5AK?q_o1K`i z{S%)yLIFZlk!FCKcOf9Qo6l}J&tKNf(F5aMVLsUy^>|NTgyc%_r5hu&T$Hu@IozP> zbJAM{3Sp`{@Vs!ZB<@gv|2M0DL)A}Tx{frI`j|}IBh=(fChlwT->>s;X`=zV#rqiz zD2y~3NqaQvH;O?&xiq;)yvZKyMW^}>BvA*eCEU?HmT!i%kZ?MLT~dE8rzY9kU93s6 zM#RW%+K!clyQwMQv+!@|0fw{d(`TjU^F{9dY+@eL0Y|clJ0XA^sJ#;uLeWV_hUs1a z8n#CzYe;TJ_ zmm_;ME`Ub+kaQa?@Q&65UA&nvp ze?=VE;jfa>sS3*bh-n;jfT{LPy_0=w-O7-yg%exu*tI~AWZ11jy~=GTZr{Z13(1vM zX(z|3Letek?J4t~=Bg%p{p*@F`*Z-s%?*_w)PpWu;2^T6A-1o-+!^aoIbxSo!enpa z);u*9XaCh-bk@_Tkt$lZ7E0SkB#YL>v_qSqYsbSO6musidmtY<-w9uT?6JqnW8*?& z64J*@FZNg#kpMESm z7Sd}0M8{c*k4HCPgi87Q=G|wO`)JodjuQ)ACxslla)T=_yQ;{Szu{x7qaFE`@h zL<@wDQR%!V==5EByt#*#foLjJsNsCP9|`_kKBVyw$JrDW2q9yIOg14K5g(NGoF?4deN3Il?UIZx!QVCO5_}!I_XY0yYpLqpWgqHCAvBwPyN!Rg zF!XV}`*Ts|`^{UNoUu+Uc=$t`TL*nRc<@r|a&PDQq#pb1il>3&8{J{;SX*y-pE zlYO|7#<^Vlzt?limV4qi#zO$?`KHfiw5df%s?myfvjgL$_l?ZO1QMYm_tK`y!4C00 z)xHz{@T(m^#9aQZbi9mIMcK?Nzv`D~w?RqTbJC|5x%>t}oK_+71!Lt`t(=r!t#3*z z6U*EHhUAd=)w@quiPGf!>f!XYhtr$K=N`^(cyibKu3u7lVK*Jsu!`M1tzZ@Q&M%Rs zqvfb^Tng#&(K!^^V2Daha+SkE1{V{TfA{*RKz$gDOIt0HcIBAx%ecbZ%;}XJu07oa zLyRj&W$ZZB9?nKRzak#{8Cv=i$uZR$TORfN9+kcoM?GKL zF5I&b?#<>4sW7^!+ZfSKc2IhFfXz;hP&J+UE+T9?9!&j`A6WG-ekvxeyWy9n10gt4@=;o2sNApN_aE+TsvLlbA%KqvE^GNCh^GJ3Yk7UOe z^=*qD$)e?v?6oV)_jai)Uo3bedGqLi@m+adH|u-N>TTYnDGuA^WvIF}uf)bTQfIO4 z8I=SzM^TPhHR~>pR?hq6ndhOM&tI*({nfgyS5w*ZSo+*2yS!Sr^=dLam(knj-rMCT zd&A|{4gJtuLq+vKX1PaxV_fV@3_*p-5B6s}uZWw0r|{AkcuMtrRD{~VQ=q4bfAG)t z>$BPQ>=>36V%K9H*6&UU-(vO2_awhsJ>`J1unJV?0mOkUkixvj#cb(x)(^UF_~j22 zzNmkTtk4!-&Zmd*!*AB_Ooa-w_Q}2k$%uoXnHVV(gNx@AgKvuy18xN*Z$cur+RMb? zyv2#3^Ps$GCI-9KwmF|Z*>OzU<5*}MtGg-6XI?2eySmyo)BVX?g-ES`Gese(|Gn%x zxfpF1b!r&OAF_Q-Haf~)hRp-9%fCoLx5ydU?8FnddeD7MX_)mdv3@yIp0N7!6sQ3d zs=G6VtashE-gOJT>(skgB`mva*1K+7?>hDFw)4<@n%EwY6O4|RF~R607~@27lPDNQO;BHv14~uY z7DG_Zu$;T!#>CO~wxC`chSFwYYRLULSJEONbX1?&mO9>J%`k)!_OGOuRXck-=zrBP zgy-&t!(8>VpM=9a_jISZ)eacKRh}=Xi7w-`m|02o zT(^`j1wU~%aa#n-vvA>5zX(GpSMciLt*!K?7vu5099&WwQlecA{D2^=&%&2MN^zT` z#?W~cDUp<}z!v~avjxK#NIAr}Q|Z$(Zy8}2Ulv0^<`ub!xPvn=7(lMTQbg>|JXD{v z;YlDK1uBKk*;T^$C|^WhEH8w>+UN_sBICXUfPf~T9ccR;JsDe#Uo3{jXg@WXM4;u`vy<(gW<*Ldi?$ZjggAbKnsBvwYiXzKl;5lk47|MAo-0V$M z+?-g^S+%0_J-Ur&M{Hi4f(FmCFgDu+D4ble&TV7Su%0)gG9QbHd&$H-7p;Bmasz$Y z+=@7!j>2!u+g*D-2kX_)IPCPk9nHfh(d)b~tL5{FFo&}hSJYqV($ezv3&nBeCFDHH|pj}*0xq)x<2OCfkzP&q^o z*$fMJ)nb>;BFkq$nTk=4Fz(_uAi(5v1OE@0VJjBhwnM6%lE<=Vls)MjpN3kt*>Le? z4d_FKW|JQ0@UoO?oK4nj8_=f{*Ge3DLoIbAJ;*JZ0eu#wM+5x3DmI@byn-w;Su!iu z;Lq7zmLP!oe#N{cUA!e-Q!Q+x@Jqf9DHtARx-7^vVtCjd&cyHxB8DfMxbMgBv<<^! z6RQF7q`QN7pbu?>c;v(f#DgplhzHJ)If#dEfOygth$oF8p0o|(L3j7A1>&(LJtQdR zQT%;4s13xUvV0)uP<+cah^K5cL+#e48J2SpPuT|Xl#OPn6HGHK8wIy~d=O9C)(q2K zG{XfD4;%>i4fYs_M>LT8lIr)_x!=Qw3@ z)?^yt#1jZFPJacK6u!@;UCVE%#QunAKVgV=XM2Lc0l4Ta&W&veloF9ly<@W&a0^;C zossz{d@n-3MBJB<(g+}Hn5;JJ;oWd=#3TZ+4$=r_r~_j@$3@sc4=nbN6=FaK6o?6x z3wR~lPT~tBCY2J<4gBbHiPy+)dDi#?3QmWldZ3r}HuD=q1bk{+;!~$lmMxcWrYv`N z)pJc*?ru}usaygP`;jWcUwYLgKfGHW;w9yJ{gaz6&OG+i zh0`(Ne>k28G1EsWhu$x#|2sF`d~xO>(2}H9B9C9@Cx6p;U`h1CG#j2Z12O{ni1LMY zM78Xaj-+%KN+TO8|3c+5prz*%_u`OT&cBzSjT`BBK5_5TGjS#iUaTya3(+sE=pp^oZJcGmgCb1 z-&Yu1a;ixKq4N8ipgX8wt_jejQhZ{RX9i4zN!}78 z%CM5HupnZ zuF08Zw${g)x%F|WE$t>bhJN;wB*)U}PIIdrm)cI!Z+Cx#ffyIv7M;ou?}l=Q{^;6^ zxK`e#Uc~JtGiI&bp0~N%iyTfyV2XBp8Psnw_zVm-O8NdU%&0{pM@&PDdRi9F2dr0E z6~H4{^(3rY*zolLrGSGxD;C`y@t|Mm_!XR{eJQ6SN*eq$Oc zlqnvA_9X1@sxNS{KWy7$1I_M;b$?^Yr7lB_7VF+gr**3EQlCV~aiuT^t0A(}-MU=aiG76>sH-ji=|B-J13i}SqF=C^m!d|4F-S+Ot z4*gr{7|wMgj`vA^XdXmpL%^TB_SlSO97cxs9)ZArry7U{RG4rOwpDJx3Q zRx-9l3G|*($%s*c#=2aGj(E?*po-6KtfP? zx3ryNo5hdjvOR65SP-(sHzj1zWu9xgOrOjt2FMa!-rP1E|HSwh_a}MSvdZ<_EnKsZ zBTa^71Xy8Hg5f`kVkE9%gL*J7h3SWDFFC2iwS$xd3AB1zJ;g!ey zAXHbZ14Rh=hVt&lOX1bW;zV7F#c9j1<=`5mq;Gz0{In4sem}^JkRE@bd2S1k#(keo zp@sA%Ozo*JH^y10VRal5cL?dRYlFL1;SUN^Bf~S`6yzQQxEX>CkzDG{C7DPto`SJ>xFBvUQL(N-kdx|}w zGrL{Ji&*e8;{Ae%66pS+UfE~OOY2;PY~!oqJgsO9iaOWRL$Dq8q#m*{lBXF@oQ0_B?_y$__e^E?m2& zfScf{gR#3jKNN1&wR`COBp3YC6PKOg6Mx>so3L*cscYENG5I1QPIVPAcW zRWJaUeZ~4X3!bJGtXLEy{<6d{5tz%4v>Yg-(;w$L)1e?aJ2O=guHZn~b*v@>r5ScO z)2YT+$n4CFEQ2CsJDr)Sd|Jru%-qh*sUvNWc%KR5rINqIH#^+QT=gzw$Hrt1;^GVS z6-X2FTcx3vP&vl5b3VVoB^NUz_|&}^la?ulvQSf#JVS+$3`)2I+W$`1aIC)A)%NS! z#X^63yN}m>K-y$s8FEx^sAQ7)t)O{OAu+I@{H_uYA_h^kH`jZ6h-U3~Ck{HF&x%wy zyPj$oT+ePKzKt5C{~S+w_f?E3J<*w7kR}(qj$M(5 z`Z(SAs%sBAe%%96#L|cvzBHYr)L&ncb+czmw31Vb=>MKTxUsm>oUsmc_D8oDMDEzW?0gB7+OMF^&pzXS%>XW%! zPQ$B@Re2JU>NC!{%?wt~x$zj*!Xq2YIFQSOjD)Jk&f{ z@5kB86*q6gsrU})Q~Z137PZg2EAm5pmOQV*&G+>9UmA#==v}y72VOW#`J@xsSJv7O z9VYfnpBV1y)kWC4^}l{Tt40I_lINtW1|MfvP&7hd@~#n!Lkv`1rR3<1k#gOHK+8Z6 z;8e3R9r~0u$Mz?Q_MQYGOncA}bMiMQ?ojnBxyv@Q8}O^h`w`Oc-+rP9N&Ti2y>M~d z4z$KCZ^rG}&A6f8wZ^Sn7&lzF`N?s)}CC$jS$Xg^N5jiIEvQxeMh&FYe!nX?k~`<-i~>Zf`- zE_Sl(_2U|0CPYzIKLT_Vj$fbFk1&x%dA{m%%lZ_1mRbCo(HKMrUBukVH8po^?N7Q$ z@Fb++_x_6wX$u9~>cX>wmF9D+jG19cnze#wwa=@sb&RDwI%iF7z_+3Xk-HGn!BR9Y z{2}SaSQ&sskLLdgEThCA>{Q%wIzqPjxU__BMfG9&ku}&?U9uTfKo%ZuQ3XiVq~>dC zQ~?iSMc{p6X>U2=Qt$=1IbVSJraqD3vk@G~ixL{=#w5J={^%KdlcMY%O|9z_z$|SX zXlI531-1viXGa`p7ms`6@?XRbJc(R%BA14HXv)+R!J_2Tyn^=ZF=ynWcX9d4x7*4? zr*g>ue1xi;gbyMP%$e*JNntITiazUchYthV{yFlNZ;QxV^5_?kw+?Rx=Pe>{b>1TK z)@}D_xViW_yS3xyYbQ=HCL$T!Jt|ZgBrAE5eBx?>R_!NFQj7jgP)(w<^eka^;&Sd` z{?bxWWgq|DSKWkf!6yR!L*-2jD}_J#mh^D)eX@v?`i%vc&fiQy{31p6@w7{mx28tV zJn>0*%{MaRACQ;jl^4TC4|Oww5QLu=Hn-cKyi~hO>RtVsRL(FQ9L1(jo|V@4?_!ky z>X8d|^o*qxV`hqzd2u3@B8~n!akf?Eq28$PNqzF{G?q)h8l9BOL_Dp?dCLp-vFHPv zxX)u%C0ux0I?E+aZ=(yVzbm#}-4sNjbf07fEXS3y992bywXkQ==(f5j!h6-1-c56$>-iHGyR=>J1!Y#JlcV|ZD z@7Vd|`YlF>FIw>PdDt%RRb8K^n@D->z!SH6k$d(3q4w$mx7Z6|4^!dp5${e0e`^r{ z7%)dc4Go72C-lOO<+;Z@Rvi-MOPYoI8sP zqFGz*5bMI)7BmU#UyxIkNp79YP(W;PkKwxX#b;q{T7Re*x$s3|g`9vEO}CTE=d*Ch zRBZJ=3zHpT3ve&*q|;?E|j}Rk?`YSH1iEonu#J^ zxvmlx&_$L+5Rb0oU&+u@_rY1Rl|Cn($dU~}QuQ)FTZ{$1oj5+@ZQNJBLuG_ES#{nnlYo_Qy3^ds z01r$I%_(+27D|*4XFF;>uWnCMxA~O3xFyS_VN1(MKBUJ!>W+o2t@K)zqj=J-b7+BV z2xx%?It{Zs&TMVz&6GmDVP@|*p){cp;d6~Z)9!40X(sW%R8;>SjZjNP^wr7!n zNTSdLN6$isPbTiTVa7DSll{p{1@2Mw3kuN5HB&30(P@G4;=-W5*&YiI!qC1PYnS7e zQD#uiV_&LdRDna1iF-giStzu3Xj8p3d8vuG_Wvj+X50}W z5VGIr^*d8VU5pV}w3X#_T;M@%wy>Ar!)Eh$rdO*rtt@stx!%F$Muo$Zn)|p^thZIN zj*wYizu6A?ynYSBj{PAxqCU>&((uyipTS*Vd$^;%q6DZV5{^Ngef-CYB0TbAMKwxn z-$r4n-w0$S?(Z)A&aZ2KceDLnd%H7MI7aKOX{4I6YcBCF{4UpA)Le6*r4vx`i3eoi zTr0OJ`X=@> ziT${CuC|pl#yw1y)1q3JH)>rjt46@Wh)`nk9WtXv=1YW3k`1->YRE3|8vk$Ky>7}4 z%VnEk6YGBYy!w9C?`S4f%B0F{Q8lfj1ef4Y&DMP)m8la8;4D=2VQx~R2fq~Bw$CW9 zWNIdaOwF97*;k_?T{AsH*`}bBHyyg30uI#OsIKDEYG4{+x>3KmSHX-H*GP8whMVy0 z99j2ypu}u|!iw^!|})d=@p`w=1>n?K@gx*e_@aJD}v`i$81o$s;- z!<%ByHB{Bz|F|Jg9#0yc3=Z@5UE~G^ ziiP-p@$pXD7pLhRG$ACbky#;8_O#(=B-R!5)J``OG^q(1O#yfUd0N<1+IW3y&&)=f zDQKjw`ap_2-b1A~+uM?Hevx~%cQ0YW+eqYE0L|8JQJ=eMJEFA!E<;UB>ng~lTV@y; z&F0mRhKfv?cT()R`5n|%R=F%t-bgQ*n`2}z`g4Z5SJiFR^I2(KeXr#YSVbJt4o$s9 zryb?N;osMiNMP1b_Og&%p*y8Ce8{mKd@>y;ks~W4)jx{cEZR+4Z?x}?I!O(M$FgHq z?LHqB_7J>l@gd@hC!@&K*GSyU5nx2fcq<4Z&YlWUeZ^Cs9^XtWT>`2@N=&udeq%{< zcceOjZt@+8VyG(2n}+|Q7gM*TW zN3(IAA99=FH6Jc~y+3*8xGS9X-f-tfbl`k|Wn|fRNy2d9KmUADjZ)vo&s^(HKJ|T+ zojc@FS)ps5i@YxEiO=0cR5?hey^Go|%7EH4r-<4^rR)cnG)6H^xiCh6vo=^UIOXo(|#>+`4^Pm&*p|MFU+% zwTnnmR)e@&CP$GP6wglodMw(Yck;^&29-TlXRj9DD?2t`)m#1dKQA8de_#5c?>-CX z){%8zh8vEob0E0snaA=Sj&&A~sTcG*XZFHsNRL|=#5T>J-8-`Gdpuj$yKUXEOw4jw zJyT*N^+8u3K+=St{SPdS0}yg)UF0MQWH21^>+8PU@qMg}_-n>IXT{dHXB5z=p)VD# zNQ*~&vvBqbSCmHU;1+_<*JMKwG6NM@1PmQ>Uhn# zw1Ce1;;?5!<_mEOp6kodnI6^3Eq$+kTV~{U*Qmfs>ybNd79c!pBHQA_I(L_C!dV95PWkmR{HFv?;WB(KxiopPFT+Js1-@I{Pv_G(Y7te6%KgBDplV+Lz(89K!f1 z!`F188vMnbi!y`bQ7b}+VL}G=Chqn$q`rIn+Jo+PmEdMitAGTo`pOTVld2*{8UBSj z19W^DzCwpo%&}Fy(7vezt45!u_^I}FUy{J1bl5VSI}MM<{(K;@^LfEO&A-L@zM_Hk z#qRro4>F_Qp~>B9hc;a50Quba!>8P|zNZRi#4(+LJ{ zPdLG{s9PjV)%H+uOW8WRJeaB-3KsY1rt*{0ed65{W1{ zyf3R?jlQG<#v~?FwI+%CW#cS-S7VqdHZ&pq;z%_{sJ?rc zhEty_!Y_Y}U`u&qom=>9orpIv5n?dt*q?Y`q!E*op{{>1-)kZBIshLPEkN$`XQdbU z&b0LmL@lPyR{A|+78z`W^i@3S)Q>BtoRwf~p!>#xg%RDt^dzL3>`&5AUwdhSmI9<( z`PIDi7%oH(BK!0D7jq0?teRS_rlvf@Q&8DfEd=-Rr}$h%H)+=uKB<4Spg$8KV4x0z zDNXh5jbSYu00o#;591eag;-D%joyGx60lir;LjWTS*%r(wvZc`Z6G&Dsf&<}w4w8g z-87&<++9SEhVePo5h%xzR8p~Z&lbz>$`i^5qZWy+DY%1 zvzL&!TdqPbhb=!^{$o;qia6cO;idEF%e;DvGpA~;5-71F+XWnc{LYT;$xTGmN@21x>yB<$Gy6Qkp827+&$95xTS7r+L@Bvs-W zPzzDS<1>*qurH}T$gE!e{gJ^Uz+I6NYYH|(;z|iCrtj(Rb`TxwG#(Dt>5Vk zKa0O79=uVLdZ zIa|G?8Y^F&nhWu1sk{5s9%^>hclE|NGgL$Jd}6`1TtK$-I2^C_6pR}KhI`1io~&86 z5pb2YmMwv9mMy7)h$>@Qq<^z!^+v!|rq#RcJd)^|MVuMn%C#178*o+N)QFA^U+j`X z)&g8vW=BH$ok5i{#a|WLHZDt7?VXR+^0Ba$UP}=RU(~;l3k0)2d3pGwKQ~a!Kd@0G z2yM$K@hKG^9{SPJ7!;3EFG33{H7m6n4ygVr5=*8Tb&TXjOY48ti50QY@8#P3LET@T ze*umB(e*Kl&bA`iomLPWBDb-H5ZN`j;MixxOCVoY`y|qy@Z*0L?`g{gXMs7k zO`73JrDWJQ`05vwyc_DfAW&> zF{=;oOSy>;!3msgIMI1m1PY=S*RCu07CxnL2g@aKxD5@|VWx(Kg>)fx$nJE&p|daw~Ku9@frg=I0@oYUrwh}JYd@P0DB6? zkN4Uf<0>&T^Fc45xz9>Kw$R)aSQTN>UVUs_gyQJ9bD2Uy7B<2}etZY5S(&5P;ULZ2G>Bdj)0Unu-nV%-^)aiF_TG7uA9R*QW3BgvB9Myb>toKWiXM$J zl*i-@>|SA^{xa6+8BtuZlgTJxX9Ev~$zZ!&2L`97vPSk$93v>zmA4k0RS-Kf*A71*-K|3dlk|CZ&;wH z2#LHhZa6$B`^sJ^Nhp zxoe+sdu)~V8UM;w$37!yw@|`zCe1;&mD^{^if>U)hzx2zt2XvQw?$&u)-qnbJZvcy zuLy3VS@~P(2370`=Mvvi0*Kw@sck?Bmrx$I$j)PQm?U5 z_@ctKhzv;#hT_x9E=}%XXg@z2T3;G?XD4Tx`xJDYIX8&cCC^Dua(#nq`FyMeAFwX( z<(G%zokLQDq7vZuf^1Utq#j{rgEzk*dqw>BmDPXF;Cqwe8TVGn182_rO@pAXvKz{#tc4SQb^YGMGYieBzdy6B18!zh!k z18C1^M*M%}`j||_?C3f`IAQ|@fl%P`3DZBpqpjd}*Ndu(jaa#8`;#vF5AkF?4Ogz4 zQOe+lr2a^+`a&_Q(3vZAYc46Xw)Q7KFm~Y&lZ~amOXDBQxcf+Y1cfpWZ~MAxK%P75 zgmc5Yf0<<=h)vtd`I*;N=$dmYM;7kP!r7nvec>;HVsj758l^)&m#*Z=UNeAW(t!qE zdRdGx4Ptyf(g`y6Y;s^F#=3Tbi(O>Vc01jw`dRI4l|*0TcqCi^XN;=6Hl;8zN(;%0 z-qjC75~CT;31*mK&Qku@XE9wN+?++Fz#2W;em(CXRK;D;OJ#`ny&daZ_VSD!|0m*w8Wgxm~qw;$ELe#>=Kg;_I#0 zt|h@s2L@L**Js)W-&xTMwG2Mm$^#X;ftvBsS8r1A-#R8Nw1|4|yhBPscgu{ws2=a*&80B?07v9ys z_Be$(P&wv3{;ie8Kz!{>A|A25>~BFoI@PTCn_Wn%S6Ts-dIQEBSQF*7Gxwv-&mS0P zJ8X$rvn8gsB_?q@gk6U*$6FblgfC#TpYiS732xX*Wb?#!&cJ*VKAR`j!MulT!rUF) zY}z<{((tGMOq*~zb2Vxco;5G7yNNrhrXI64s$#IKu9pAGnzKl?v@D*beQ(`k(4#6EC=hN_Y++v23bkRwNHg~g?ofI4# zwd~q;Hlw4!9`T92{o7n8g1{1mq0DD`G-fB+w-?BK+S+o*hDmtf5s)UJ>e5k3xznwj z!~aT$vHlAktdHfJ@bo?QkB`}b?zP^9TfPbuW4#)8!|&brvSYvkuAb5hK;Y3dLceGv?x(D5Wqqn8g zaL3K%qe-4TNV&TZv?Axn7Dk}Pb3uHi{uWKWK#pu%H@ACc9}#cR&tJNW1W*uXHcSl8dtg-tB0l zy(jbT@IzZ~+UnvQ+xeW+Jp7>_EzugnzeO2;UAi|O%FaAQSGCT1?42E5coRB@UHIp> z+o7A|A+1(t%i2L!_FRsDI0)sj}P~ z@_5wt?L=EOrznwG`@1xPW-pH?NOLUE$6r9O)nix1;7GD=;m%Bmx>b+2~af4B(V#{4y|8k*-jZ^jaB)o3z$U05Kd78w~n_V|?dHt>q z79}oeD@RV`gH^-U`@;Gos>sjy;gj0N%+xJ z$cSC2RWdeyg)d!r@B2I9)}Lb1-jdd@SGhJ#e3|rQT*|N_ZrqcS6&`@+Cj3*hJc8gW zC6)!xe%Ful!|Trc;dkHP2}$^|?LTDUxzq4^Bl!b2_J;WJqUpF3ems7;a2kFhew<9h ziTH8fbX6ZkdcKy{7tRcEWBAA zn1#2f=d$ot^;{PI>=)&pPWT{*NltVV1$*|!#9nJ4%sc(yt?c@lR~)eV-3vds@R=X$ zgstCW7!pQ7V-O_*Ltv(xjTV@;B-}PLUqVvfXz_zMa`>l}gr@7{&2-^6zRu{}cw&o- z-?r^ynLRe_x|ngtjnyO>Yc3`rGNj7A^Q(mkqVV>!xY3yGa9#Juzgf`wkc4+Usq5bQ z8-4;7e6*E74+Qr5%kKG4xgZ0S3n6i$4}Sm-wutry*f@<=FU zdgGS2KFtMg(v(OVyV3?m!Xs&VcQ0e3MCK9tbzkY}AXmw9OqGs^m$CAR&}%N-{f#`N zdWLTMlWBNo>;p!9r-9R}g+$)bA9*wnpZl4`KA11<5l%KO^v^t4gb)Ah>^w%Rq4<#u zh`n{oSBr4_duONFLj({&rPHjjTjGVa%`3s>89w*(P=2BH&trC9C5xAePk9ptv}woY$zswhvLxa7N&fEPWxDRQ2*2)?4f{usXT~X?82$! zpu0KJ1FFY=V4Un=_$H0vTb$_&7UJxUa5_cYLD86@QOm?ie%l)W54};OZkR3%86zgO zY2pqY8+WiGQ5R4Y0qH2_F8{SSn>&gMbfj^GkKh^tGOAw z8vY|=T!YA6&U88cN&jfMN?KOZH$f9r0B4Y6^4$W}IqF$x5*aNU2~qribbSmL2?KWWCzN^ye_qdgcd&b4p@Mym9^b%dJQ|xtPO*+%XYUi0S}BFxA~oy?Q)& zg-1pO2)~v)nK%dW;lvk5*XvgRj`78YPuHt|n=15g%eAx!g^*C-VY#xsXy8qci2MGT7jGH`wp*NH|`VP7Ou*yQUb$x27I3)+DcVjn_|o7 z9_HV!$VCfm+uHD<6rpwa^9mN27QH7hLo^U*1Y|Z1>dkkme+f`WTa{>ypdrsM8(Phe zSq)mB#$sWLpLNjjWBjb6jImaMu0owF+BV4Pl9PFs)p=ScKpEn%>-Jy}9N!;_BcD^X&sp1XU+oAYmfzX=fa#o8&URA%yF6FP)=85@J{zTcPElJciPS48s+cHk8>Yk@(Pup~{ zf)uQM`=USW73LPZ8dE8#2N9plW($lPA%vW_BtZ3d8apFJ~6#^1IZ`_;Oi{Ur9QZ#&&-Ze@Qis&H1rIu?Q$XR1S&tsiv9?VY;1-xJ@?GUv`HXZZR#(OHie)otpq#W_6H;u(<=e>Sr)tqm(*AFKn5T!f! zd>xW2M>$wFSbEWd)>e{ae2xJK($T6h#_NMW=Y7Zxcp=f)Rte>YFD0T^@++#=)Tt=@ zrJZS{w_NcB**W7ibI9$;!I{0-DY=wAbp3(Ry5W5UYpyTnZZ85NqVhK{vwvQ!Iv#7$ zxs^=Zg@C3rGWSsR4gZ(#PEU@{_rsHZ-JgHm-(DY|e>m~yA5Ok+{qE7S?dn(i za`ZIc8Ic7)(jdOf>Xzny1^8_-aW5bD36+<04GvY1TUV`*hbTKePIxX|D|FfL$h?)x z_gezFK)zqsFTF@uFWAw0R*}dZA1k8ayDySTODgAllwa=kBadKk&t{NY1B_(UsP>+c zQSI$Cs{QGB$fz!nbW6S*8J2NW_u42ft$UI>87z2kHT{l8>oN`^A*S$k>uG!Can4oa z(mzcjpg}(9uGRpNE7^>gZb~BB_*~9;g<&!}9O(F`TY4p(kW&3RoR;r;%Vw}GcCNbm z+SwTo&sVf+8KSn)r9dmKYp>IegCn-N>&@x3zP}HAXOHCjo^|v|k}l_=Sq_-k&CU=fD)bHaqA}Vq%w;r_{8Fm=@qQ%7BqJcPLSr-#`GgWeIX&ZRNmUl66uUE;I zVuEO~TWDSLHy5#tc~4tF_eOrBiNh~M(Jd)=w<$2H_S~M1t1ICg@zqsm=t{i0T_1{x zJ3*mhLQE&(l({uMiW`FQ7bNGllow|?X$qi~6ccwHe{CW7P)qKZiR{@>^80&)Le>z# zrln8=hWZAaeA4JA^TF`5!SEW4oWTIn2F|P?QI`1=(hckSlCMxQ5n_wNWOj-bOEZCT zT?}z>{=3pK79StCQV%PSby-qx=xyzuGwXvSHPYR2U=w2hL7JK_bV2{r}Vj7fvlKpI}V>@M{fRD=g7vC8g3 zvotTe%Qq^WJz;d%QqrQecdvo4PAZ--_ndr7%CKL>1aF-Be$ev5$b}x==swvcF=f#9 z!f1bxGii3iZ>@1fIF$llcypQMO|`^_ES{}i&oHUmI{Hs-)6sX;#N{(TQxor*Cf?so z6Yu%iPtwHur#sE9cF+fVh}!dKW`XbBjXc~%H*eRd-AOlZYSc#E+|tnil`Lq^?J|vv zcQKVbmT8Rj<5;?=W1))W(WA^QNH9PI;W zgBNKVAO(=45=a+vCIbEZ1#zS*;>E{Qy zDa2b{*b*);naP#|yNH&To9&&Mah^zA%tQnd;^oLY#hY9uG%pjry+$|ddzG>(iTLuT zbSVhiuWX^+giyXs2$7vDAtVL-;h!Wtnzw`y-_A-1ne>|tJagtDTzS>c_Oi}qeqD-- z%5RK2c=u4xEF2NTO3z)PIyMub1O3*T{iAXa(KrInileEJImD@L)1vLMHtUF%5{=Ue1{m}F7@XM_)ch!GyqzX{A?@wRw zhClp;JO1nw_xw#?`D#X!{eP~1R|W||mx_vItVeGe8W4r2vFz6y~fc0xWgkkm^ZDv2j#0wyB9Nj>z9Yg6k_dWU98 zLn?-uyDq-nBhTZ?np%D@!f6U-+Z{JpFl?@Uqnj%tn`_^kBgGkPu6>iIdwl_$>xitF z&cv-T-s1ul_L*672i+Jk$TSy~7YB=8)rQ{YgUr%GaT{J7|MFEW+VYXE;`SekZ}Fw) z-RZl}%({2Q@1K6&9it_#ts~u_DuN}WY&NELk0hkWT8I473;u@w{PFYdq`Wr_BCz)< z9i==r&g4!mLwEYj41-+%j)yOJ``h@F9=g*H{P2#u?)0U8$49iokJTM_jLgZZ!gDnU z`581`3Js{r?D#m*V-nxv-|?XJm{vRefCpB0jO6CS@wJ@>F|sQ*T;`Rpeo$pOBA0pf z_V2Q4$qp~@Zo+DN*}}3BRuC6SSW!f8t}*fLtb|pGs#dmyRYyp^>Q72nZc0)e*cf+7 z?)N$CPu10+UCLz>bXf;IrYyCLRNPj&A4|*v3o2@iPr4IMrDSe*!b3kIP6MSQ<-Tu) zbcr;KT{xAJfJ^eZ3(q^|JE3#@+7SgiQR{u*mVKYd&ZCcI&HwNwhOJi~3tRVON7nhA zUco;%91GugEIYDp0UwL~sAv-aK@y-vxkxO${#al2&MHa(bi!i^@yyb9jL9WcnoTD> zrUGm%?B^*>nDSP9>%1zQO0TtZU32bZS-i*WWT%q{uL=*vlkL*hy+Wp|+fT@vCp4$c z?$R8vHtcy-D366lnr7tFiseJM^@3`bYv@9H?TFHN)b+z3qMhKZho{#Xp59;1@F+CL z@Sx#z7RN?gBomM-gub_dgdyoA+ZYdgvC<{hN6|WEpjIRC7u0dq^{ntR{lV+T7daw3 zFy4}mbsGvfl)mTP#&qObv%%SH;1=eVw%I_yc0Lk(o6iP9hVfjR4egUj)9JKkL%g(g zuQ(gp7pYP+PiRh?-K9AI`9x=NHc<49+0ebfbw@+rHJ+!gA6^{GP5Z79c)02p3)hYH zy4o2A-4+aW23{%~6sq_E=OEO<0tj{J2Y{CbJAzP`41}8dB|o^{56=LhZU-+J8DtX5 zShTBUEGVV)1H?~ZLP30HWV#|G> z;ISx3fAI^2KC5yMEOU5mXEED(yzYn`x#nAKU;pNxZeI^&5_itSfenSh-FFX54yDv5 z@_9ILc}S0s!FG%IqKWIypL03&nGL?7G`i+~-$;*z14N8XWMujl2#3sq0#gO2)@(xw?`rH+UH8-Lc^l4H}lr^~t zPD`owNoAF>W*Tick+zoMEYasDsK2Vpc|w)77GiHm4`POn=x=qr;dbW~Rhmme9ti4yWWNA9X>q$YHB}jKt{9mFuIP zVlK{SAi2gUKVqmhM@!;`|G@8K2YPm~15qn~kMF^Vl}M9xn@kc@8{AbxBB3f++Pbwl zE_Bn&K-rK6%ciPSNEE7?9gi-r{1R~Tjd68UbcbDI(!q{l}~yjK-H=Ds%_cf`cm*U=gx`>yY7*rx&e;;7@R@QBfJ_D9QD zw!VBcNs{rNM+bh}pGRNp_xL@J4nDf=&!hkMql3NM{yh3u9{t`&;q+s;N2qnGKD^Az zhg;J3eWmQ!zM7zc6aPcU4{T=${#sx9GJGsCnZEamb@a>*id+X(yB9WpXzKg$lyW7y zx$p4Oed-+dz1tY|GwHJI4Fm2s@PqC5!z@(vXS>BE>NhdCSpX$7QDNwjBx*ns` znI6U|W9Rc1u@bSV7?CF&|FiO-E?DVhpliRFT)tlUVK}t0FR8v#D3gb+Lhvmkgqlsb zu+{Pu zO(1sM-*|gVH$f{$x9OazbC_;pso3r7f9$sQ^+W)fn#pBDvcoZ5$l*9OP7rq{lL}+r zG|-w*sfI(v&5`f;Z0S3OgED=mc?LVgZqFcExzjVyqMd4Wpb43n7b>dtfR-NA8tk*1 z_{BVQu&4)BZ|J7|O*7Dy7r&K`6(PSl9S^3VvoRX_%m9s4Q~#b}tj>vKXUFKHERJ%w z3DGAZHw3yC_#y1|d}gn+pi190dtG$P*z}FPju}L$=i}EBRA*1%Tr;pn<)XbVwvL&- z4s*v9U_@s2x~MYp`c!|Id67ydK4)gpIG=5dmV9s2VNhvmHFmyxMZ>@_`#B;GcK(e97!#CJf$dFEfug%2j0@yWg@8kI{`*HH``p1Vp4|jB~7*|Z~t^YSX^iO=XJ|(FY0}0|#o7BkDa^ylf5{F1>`Lyr= zcW`nn`4WIF{te)tgdq@5Y2nE5z;D`-ZgCT~5J zH=MWYTQjNNagN*;FkdpLYzQoQ{+m4_+`Hq>|G}C0bnmvE|H=C?ik2UvXw_<1@M8es z)poy=-SpIK_xHB9dkfNIGTn>Gv@I{E0H|mw?9$HA5E5pw4fG!5V#OrJEVTdwL`v$ z6e9p=s3qh(ed#V!SSn(_U1-l6{|{@uJJWos=97{XTFL=WAzI@Mi&Qp&JrzKf_AXJ8 zhqtk}BK9exO?{It+ezQHIc;?@nSQCCx%kHlvjV`ES0^3GhKM^yj#6{0rbDxAfIs3V z{dlsmw3d|dPwKKVFU!Kdc1kEOBG;^4C=by%3I8S8SXxO}R@{pH$%>Vgl^h8Yo^koG z_IbGRH8*z;CT|v0@Y3Xi>7bJ9>C)uQ>7XA!z9k*>;>Wk9)XP>wZxe<_A3vy?DBId# zO{Wz!{%>?*6Q1x->Q*vN%8PkB%h*mSgqJ2CU{p~6X>Dnrn#R6;-q?4{8+-e_u|F9} ztaKmbL#y5GejB7iH;jL%m8C8l*bCo5f8FSwle}|&-*?XM`%~PvIg@p^!y|y+4AO!9 zaf89^5%I#inn(Qe`F;QS{JwW`Uwgz|==AMJG?Pbf6P~fLbY;2xP5wzzmn|>ny1H0i zUhXWjhSc@Vl=$g+FTZ=<8+XrJ{hoQNe`en5pOq!joegZi`9w&q)nD^z#AL>X)gM=7 z)odd7XA}9o^Jl$x{;ZRFmZs|agtgMgp92iiepEC1Ursibx@oW1bG_bdSYUriFHL^F z8J73Y@A>}uJ?}AuoSoTdye2f3`gz01WjFOlaNL+OD6Ii|CfthBk7Ns5Kiw7MA-0w8 zP=mG`idRu<3UJ|8`l9u5ft`TI;oEP$(M_NU? zK~Mw)tnAAY9jCD`ae0}2DJD#>z#-Pn%=g_#+$wP`!xtTMl_)1IFt_YAtndiEtGOHp9JGJ?#P`FGyn?7V%t+RT@r1Z z08d2VzEWrqu+ik+_(y8J>8$26CP=bHG|C>C1l1V z;aBc2>M3V{*J{XLQ9o2oGkoEXEsqxbQ1$y=9H`fQ2mHUWf`GeW{=6o&A-%xD?3UQz zQ)2qdE!>*(vv1G8ns28SinPRqR4pO2nAUft)!k!3fkI~^{IKd`5<$MtQWK0zl~fYm zcI;xM`GxAF19Oq8&^)ZrV-d*2Li2P$Bz8oV<(XDyew+p{8$#D-$HChW;qSA`6`66N zd;HpKu0V0A|4?Q6)Ru*x6@MQXDJF)Nnf|^Alc``U1vZE8t2ACQ{cA~b_`qxBM(5)t zuf5r)p`*_eeJ=Ic>vLD1*Yvrk&$T}H^|`Ch6`!xWIP-Y2Fmv(~eYF$P`c4Vg%%}!3@zqU(mmjDAjN>oPYG^F}Bvwv%PwLgPU-7{SBF96|W;X}Vgorw3qosrWbD;|ln1fb@QP1*X3e7B}NiddzXR-cizp#S2y z%RK^duY7lj@E%VCeTU&G4@j&doZcb+N_`)%MhP`;&76 z^hZ9c-)?<1ad|ar@uRfq$Ii+>&>})Jf^i_mG#Snv@7AlB@;8SUhY$Wy0p5uJa4!Hc zcF@cGD#sC1xBpT>8Ne3Dl#3iwZX3sxuyD#56^ht$ajV}~z3v-$z~LLWZrqub@6Ad#Sgl9Ve#hz4L}w;U;M#n*+punRL3Oz`Da@Rc2?;; zK?;eE@vkMRPg%6x*WL`kX$N4EYMp9d$)A(L|3H~jzSkc~tJAS4OWo8XNJF*}t_RZ6 zZo?Qdh1Pv0_%6Dlw2r;?N`T;0!r!@QNjAZSuJLi8zgfY{>|qKpRQ~A zvET5Li86I{t%2L;?gq->EMVIy8mJjacDRsQtoAuli*LyBv^6W!I4j{an=zlMc8qHw zwP>G;M=S)?+76zx3aBj9?J*wp>JQR1y?(|tXC!EY$lfW&II-B*f7ByYk{yf*Pk(gB z?pY}*bDAfs_1h&|7=|ihi%s4!ALX@&fJ;pHk0yEjXr*w4PoF_vqU7b@Gnelr(6pt` zfP~a1p*wwsOa8tz(CiY|)h}!MWxy|7lkm$xznrgMYF&f0(oZ#LA~@@+H&9kMyCJ_` zHH7pIY3m)5_ztKdm8~ti*QldTc!r~JQ)+F3*Y_Dz$7}nYCM~y0+A#n8)yAzdS@f!8 zhoLmU>G7T9=rAu&jaQ5Er}*3_7JRiHj-;!CcHFL7{t zd`TP$ZWdq4R2h_m*npE(oIb_4kPKNSyQX)E>-zLC!t%t{)>aO5CZ!r#=MsJLQ%>R7 z`LkndQ{S~?t5aXvr+Rj55RY+9J`MX+V))+HUekKdC%*(9c4>Bl*a?5@*E;nl5vQ9z z#mUiBfUUPzfUS2LV4#8uDAtPud?x>DdY1gF3vRP*18g3}){B4311yF<8{oVO^?ZOU z8)4%i4W&kTCvH1@CSqCuT^o}6SJS;#MQIR{>X%$v#u_9!IcHs|y_6TciV_L2)&we| zIofPD3EiXcEt31cG4380f0>+a3a0wvTH8s`sjzpzB%tq4EqtkHDA93?oTCN<`NK>a z8HJ zyo<95cdJ#+1z5I7~iJl~1boo|BtzM0k-QZ?>zVW>~r=x z_ijm7_O-5nwKoEeZ7CLsBFn}#)>8c94QXbYnj$HPO^P~KR7Hx0%1q0a00$f%Gj#Jx z21J_ zp0m&1YpuPW-{b%Nzx(6*P8m*;F7F3v!X@=bWX5xe|AgQR8h0x|uEvm2q<{D4&wuxQ zXEb77bB~$9-{1F1%n3gAO4>Z`Kf@=HZ8#oChmhRUX6HN}3XX@uWw*NgEI3!PSkB03 zwtMtM^e4P~Y&;(B@9kDqn@+G8D2aLG@0U*^C59^XpN*7IQ#sVfL{-AuC8M89{qEW6 z@h#ED|HSzji<;0!)gQ1bTvY%0(3ZvysVquwbioPdHq7`gb=A2N&uzHz2e>poL^dNO zw7EN;)wdIfANl?3#y=GKoh*5_!{pL5YSRPB|7vsjKz!?!GI%W3XrGVhf>=N{x@Z{( z>OSu{&=?nmL9`KYos4e$CueL_A&*!;{U6eBFZ~}(y(VHT_NkCZY(o~n?!Jx50FyJb zbw1G^5E-b_%VheKu3CDH75>0T`-yRqTk5G0-xVgKOjr>_2<)YK6_J{scM5tR~Em_r$Zn9_o_%lQ8Z|8#fM%z7m(8 zUx^D%nOE_Fm^eFpnpU=yeyVI0H#H0|l+RVcoc5!GxuLy~UBJd_FGRiuQjJJYiyD4W zZMnUW`uC%uR(LHW5L-OI!r+*E7-e5q8WXtx;+!SSWC`O43wTPt&2ak5n4J3)e^K%Y zOduHx!TG1N`t>E2F=c;J%+T*Ku(I`#)<+>~_3fn}kUDl(^u^dRHJA?bee?SJul;i2 zqd>tNMFCDx*Ttq2Uq2xi+kO{pp$j^l4far_zLv2wm9g^! zaivGOEAl&tS`M|GA9Rv(W`%R>yrjDub~K^HYyak7CVEKG@UTZc;tLA20hQImN|2;v z-n`weC}gvPMvcVF%rl}fF(~?|{??#P{H>T26Mq}MV|UpmQp&t;y;$E3Y+Vwe+I1nl zp0f_VY@`mzCXpYL3p--XX;wzQQZ8t5rIPOqF_b(7+RZQ3#1_738YRdv>qM~G~W6$bifSF}}{ zzfE0?o&0!pv|=LV4pZz8teR?fV6}d|rl5_~-o&xQ7}N7hBFOTG(2p+a~M&wiimSChQnQ=QP>g z#$-o^Jxxpv%Uvn1i2jfOxi?VMAS{~5BTj=wQ3P%iN;6ig_LU+jylkO zs|RP60qWVjuHQ!#CN$hkN1++dx(yTA>Z}4vBC17%f)U#aW&LO}zA{yma!A#+deBeB zcgdPEkc-Pe1jdcRc%Vdtu+d5Z)bgW=q>;q0u4BFw#525B3jK#*Y?_^Pfn<x#XR%%k)ZzPVh&BBe@th+v1Axjn`;Fk}z12#P1s=mSqpgC+p z*px-7PA%8lSZ+aWFY+=F3M!3HVMdD3sH$HGsd;xR(FcO>_xc|I=Fw7uO$9b7MBIog0^H?04F7rHH%Z*$jg>2g)?rX?dWuQ$6@?#aoz6lOi3M zfup-#9AY!4rPEI*t0RKW8e`v#wp-w!?073R$$q<@3@e8ZhRY_^m?)`Ca6WF%3@n4rL4S{o58dV)X%yQdFYO-RFjI$GBK_TF0!i0$%MHoYHEV@XF8v=xH@edQ(2WSE0~ViH z7eJBL-@-dLaHW9~ZhfI*C}2&UZ=&ogLQ8~NQ=hyLG=tNo4f`4#+L;>T*(|t!cr<}yzaD=1wk}S7#A_#+F^11DS*wDVf z8mMRfwZC7uto{}lH(BaL@EN(NM@KoF_xQh#u8y{_O@U1~ULSr|T!S?0_Nxu=u;{eb zs^UH3*%|K<)^dKK7i>YbL62pVi#P>v;1(k-@y#I^-#SJFbkMXEypbPA@KyR=OnZ9^E^ zhCsZ%HiWd>5I|hl8fvn>bsIvcFPU!$sJhc+Ck)9*6Um3e`=A#}PenP^wx?4=n*BYJ z+Jgf_){Efb>pYFDY}-oMW#$g)%-;{ulaHz;8KQnP$#ke!@5wCV98j2S2iET|L>Lc9 zTbF9dabs}8N07DI*-xWvw#0tgZ9G6tE`jbm_LtP} zjQyA2HmKMOu03P|rG#aZXm=R;U+~wm#dy^?` z`FPewz9KwNNUQ5A-hZbtcGDA_gX+S_M+f5i-GY;D$35`N-_BjOC%LBnMFhTk-lT8) z>tE!KX|kK1<-;EPU<*W_WqY{^d5sEO7SIbHDAVLvekc%4-3&5$3p@RVb8{W4(FxNAp^B%y&D z{RBfgY6i$?Lwu(5JB(>;en)}qQFg@>y9RMzr~O1^8)tpa1LxYL?{95|9LO30Rn1v} zGxQxF0m^-r4;C3{z0GBC$hx53B(1}KUCnn zi~R!@z{jLQJLx&(&hSJMNz=vbS$N;`CkdY{W{)_e%aSo1_jLY{KN)$(a5tB8n{NMI zAKkZHZEzlpU4`rM-0|QOoCgzP!N)ic%6>3I&zR3!Dr&d?rgP`P;|*qzm$~bQjlSjU zCrVp=R#&I2#PZQy&B3j_!POib%qw$8bUk-EkGpHjJv(}Dww*Dk=LvP{#ebt??IYIT z{Nf3fLt#R@6f9^#Y3JOJ1yG)Q-1@D!AZvcF6_rzT5I7$r-%m+~r3Uy&5K9DE>F06% zSHtQ7Vrpfa*gtEBJ0eV@@QvcO9Ktq=Z+GtHHD6p-ueargF%y^dI=t`k8;%}0o*iw+ zL~DoO2HOsN8u^0ybs7{!X^d*?r4U@t8JE)r4?Tx;2Y48ohH{`m0Vo&wk=f@`OJ6V`GV&51&#sOxaYGSk@ivA3&mYX)5Pj-y zSj#fZ@(a|TEBqHfh?VtjAbBPGv+ND75E?HWWw3VHQ6`I52g54@46m5O@QMJ#MXq^Lg6e0xu}QzlTS1 zB{~}&x)nLPoLwDV41gLVP<{8PwVwsSvxLGfuRl0~3P2puY8-+OiN%JYWo}Uat&|KnPMO`cmo!r zIlKXSSfLkQ_f2aC9Iq=bJ}~On|B+Aj>W2soStPo80aXAa7#}$C)}r^+*FW*T&seMd zzb9uL??_SxA1SYL{kKCORqsxuRY^<^CZ(MrK?4=6^21GLwc$3tKTi@(G)-RA zeRxUt;icV&mvtXr-hH^G`|zsn!@=&u{_kTv7y?Z|&C11p-qijfAw zWW80 zK&0%2-&X{LG1I*{9%ErDO#XN3;!~W zbD8^jADP19&TTM;keY-9wE*z^=*&%}B$4t$u}4DHlWt^K>U-)nHbJPo5YHYJjoAZZ zzZw(=O#dliNTG#Lk}SHbe?(nSclC}abo*#8a8kYf{`%n|ES_XGLE)c&{ldnt|7PRY zFD<^x?*ILB-cp{j;>>57oFY5SMG;jW9cHY|_-GpNzF`A`gIsw%w>u%Jc4|)U3^PtU zk5pz9tWGnG0b++tB81zRcz1Rtkd;pKC{sTli|4FRT)k8&Bl(9yG&*P*e z7>DCKF?a+rYiZkNQvcl$m&GcLv@>~ylvteiv5V%Uqdux18ZvR=oI(+guZ}ihkdPfe zWW89$h)Z-0X0i0JC5(D=Vqn8`S9G!B=5i&KRfLO+7Qr(Bp1a_gs24mF^@3+WG;RbbYy)Dm8cDHvZJZ8j#+RkJ(AS4YacnYi=-G(8Dk z)6+u;x>grf0L0hlKqpNHVKv~8N&KoH-Yn47$$|jO-`@`upYxFY37nY*T00|Byxg#D z4_yflS$WQFdxYLOa^|^C`KjGdDhURqAArGFc;YOQ)_%_n8sNO%%SQ`5>0- z#X@2oYzOS{aD1qKXlSTu1Y2o2&x$sgn7&}2bNacYK0P$!;7c%0%AU)H z8C{y*5g&;DX90Mk`k!DBum63l=@MZg6o~HKvEe?ZF7mJYUiz%~9b9PY5j%{k#A5g=B1M!`C7|wbzI1LZ6ZWy7O|_lydP^8r%5l z$Q<^f7SihzU4@mgzH3zdZXUAxEwM6Ub>)K+59nRSXSoGU51L;6!|wNL`?cv zoXqBNs%`P9*(}g$QZsIv)rGQ6-WXbm$gFMFyP3G}drXw(L_i7F-jcEP;H1zFgfZKR z4j)`jG)=D2>s`ago6wPYVItM*yM{Q85E>4|KaM+CGe~(6$*+hEknf7(!{jL3Ze2I< z)8_|vGaHjtg4fimDmPnaZCw0n9!&X1ak@7_Q;hJ-z7~30G9;Ci{gGExy!K0rsHVfU zK+aT_!;7u&O|DtUwtX|9PRLG>z5rvT`a7;P(o$D!Dlici@!duBA2z3(k0qAYAY>a6 z;;Y$YOMWvA8#l}RiQ0Nsbdis5FuD@AIdr#n51eBj)_3g>Hc_3?a%0=74MD>a8eAP^ z;72W~1~N;su9fonXx5TEt1Yu*uC%yETy-~+WOtf4U(Ry3sk#&dj2j>bkqL|^Dtw=V zAm}#_4{(2T@Z$mrd!G ziL=%eV|EbkocavtN?ZBgQQTZ=nJsk~RsMGsAJ6N1B~r><^2}(!KQXb{Nhez-&x{g9 zWhc*!QiWtE&x|sOl*u!rT*27MGoyll(4$wTQ?0b^q=8G0x>1iXDcMg*_4?8v&5|t6 zBIFvEOrxET8j}5~Q&*)gwVS6Gbn}Zs(K{~aZdPhCjoby@ygwAZ{etc$i&ip?Rxjx0 z-J$4}3%Z+}P{}kRw)~L;+UY$qHE=g^r;mR3DtK*0whVoJ#EsdYR*xL!>WG;6!=E~( z1=1ZWk9S<}eGJ2T?^TasRPVj&!Uy%_|tupfb zxSTYr3I(+`f)$YbrW~rIFKV_Tzdjrvaua~cQT zu5epXhYL5v##B-u{ZP8evuRrWTF@)=Y*8+)zG=C$kw#_h!tO;U0Vu9rBGJ@@S!6y; zZsmda#k7zlW!uqoN}Npl1H$r(+iQyH)tG0-nvM^uPp!-&~z>n z&38h%+c8bc7QFpDw$uxfH#gW))9O8OoGP+)R|MYLaK%EOTQr~E2-NXH+f7_7RXa9K zS54Z1flQjNkoi*=&o$k6q3P;3j9&{)moQN&EQvH-biVqvBiSmd|6XJx1l@ns%ix85 z9Q3!e--QgSeosy4gcSv=u&S|JbC*?&zs4$(o+urX6TH)%CC^9Nd?Ue{l$_mET6p0} zzYx?JM4~zXLc7PMScJ#1>9Z1{^*J4-7B)$!O-)#atWQ2e5ti8+^MQ)lzpnR~k2BX( z(rE&J#p1{WAs8T$Jj*IiW~~Y$5@GcqiIXhT6PxPlL2~&jEfW9;-O!HMPOF zYukKKx+0ze5^tJ|p&7ZvphX}mu4dV#=B7{%ct9sIn~qQ? zha&l^q)3jXMNXtbE=hk}7L8YZv{81sTiSGkku`P0U}R0r$eOyLEMoL`{norC?dy#k zb!^Q=1=13uDLq0i2|oRt=xUiY%t`?^Ur1)YC=(Hf&f+xSB%K>x z6_57#46io6-D!Z$?sc37DN{-0%h>XmNQt3G6QM^dC$6K!AHOeRcgKnL@TZeQ{`m12 zXn8`%@)`-uVCU==B7tFjJI zyR7GrFz*X2+!^?xsLAPa04K0H`3$!;W)Wq;UL~8BTw@6)6Cx-`Kt&bI+VmjckvWmv z_sJJVKw}`1lU=HES@ow?S{}=86A$42nL z8n4WVssfEyyYwPtR##Zo#f%MA)O}|s!X2XqMQwdPm*&{bf09eHeyHe*GM6S+Kbbvo zX^uOJ_E_8PSq;t3Z0PcfgoWxibiyWlc0(tR($L8{8XAf^M?)u-TfVCq5#xtwE4nRZ z3BZ60h}t&B8|8N`NYD7<2??%)GU;-EO<<9-RejA;5=ROn_$R*y+zv2VX*{-tD<_Ft zx=V!|n&_x2n+b_)EL7;xJV28(JwTJb2WVb~27zf`_B|@4c~qRdDR>MpFADThzlny*PxbK7GQVGgC&RM2Ut{7x z>|w&Tr_sV-m5ZjyK`O8#D?OK|Z+zxB6Yr=6rTN({oiH;&FC4gC)Kte4R~U4kK}vn% zvidzkSnW8GesDHS7oHl8n`|1z(vAL0^VMq2|wo@Q*)DBp?|tXx{ZH3zmU z#28K%&M`Y#)Beet2Crh~Qgkh3c&&pp=vt|H71Oz{l`eEG{T70jcc1fgMFdo$`D;+R z{&vWGxkeWaotaf%8yIU;e}l&?g(%775W*pdY`P%oc5;9L2Z8%iQxL_Hxvyxa{_$nu z^8$20(-6QMKs&LrV{Pw&Ig%cjBk6%T5V$)qN74gxBz-W4{C5dp4$DMG1Hc}*qTf40AkvR6S(7ke?>J3f*21qPAueG>Bpbvs)MN8~CnlC9oiBTuyetg7 z6~Ybx=Mm7K#jvL?kwBH0CtFVU9ZVmo*MiDy&{z{y{DT}Erc?<^26``|!&95DkcX+A@p}C$BRRpjRzJ*$XnIKr z*&a+Z+gWu$7RnPiXBOT@UY>u5d50NlIdD?-y+@`Fq)>%PJ~Jk?Yd;YBE!Iz z)?bV*B5N?m6^tF!(9)+8zmt{!v8+iK{?LBfr3-3)FYS_$APF&k+9B*VcRupLT)^BAzD^8_9w5*n@PsTLm$Hn3M~|-bczWkuRc7^ zmXBpeu2u^{o0vUnyGz#Hrss*3FUJm3 zv8pzs|HQkZ6u$XgQ6fuDK)4d9RUSFqE?uAPKM*w+gV>Fmo(JV3u7=F+k)tT^6l@Z# zyIs-NW|RaV4Cm0U=%RMn)T`0;#&qu{P4319#vZ9@^GmWo$_0$rgja+FVwe`RM4p4W z2cxxGi3KW9w2vTlwwUfFAE$6>$6nEPDhDD|tX7v%4t%LtF*cMl3MD-^B!Uk~yOzlG zT3$cN<`!;Q(m4{(_3kT8nH+V?*DpNb*DgFVr2X^B`cjrAF`ne?G7xB&CA;5#s@a0$ zCA%!y$v@M#pE^0evV$wGe`PyYR>PI-GMrGCCH31+H7mlOgwBOcT<_YhT1??3Zq;4# zlWQ=D(zWf-4GA?=bb`VXb;D1D8_w6q__a=s<0|yWBfj4r-fvH``_@l}x7ow{?MbG$ zezJSNJ-pwZ#PzRi=gMlhlI`LB_9XSKpFG$5F_DL6*{|Ku)z4YOlGeRn_%&b(bi+;I zhDUtAs#*CTJd7IrZXo|h_V1Q{51*{u%AGc0E&0zDxAo>p#Gjv<#BRl1bhG>cz^bS0+$~Q!$IuzmB>%oIgA44x97n^3%D*3T^o%}iwi+(& zVz<>@bh9fDxA9t|FD_ndE|Mc|vAx*&f3brud|%krVY|RfvI#@;CN}wfO+`{{Zcgz> zBR5G-$g7`!dfprx9ZhaaZdK~s2cvZoKcm8!5E;q2*Iu!~qs7udzgJRQ!RW34Q=P$V zl!ZOefM;gN7pTyDZnFUK4*@BiZC7@|Uu8dsf{l2CcA{E>^p;6_+lqL<#qT*N(Ap+D z5i+;QZSlJg0uSEelOt^fU>LHeO$H@*l!SoC8I$|n8~|Gp9p|b7^~r}$8N!++%D)oA z%Qm^Kl;47Vlejc@eh{fIA!2dGduC^ANl?Oi8vS}*Vf62sAQ z<&{B(>o-Y;xHI2$@p_Bam4>O@@TO)!Vp7dfef^hrSoho*EC3mc-8hV)b*uVxnymJ| ziY6qUxyom=gKefMw4xL_w&AQ`mOvTM)k+dhHL&9a!~nh9O8-fU0zFT|kBeAH}lPj)T7(oJr~J&9gp!9Udzp{mum`{1m3 ziiE=UVlpLHZj*Z|%ak9SyO<7!kgT4~n$528``2AIYgQMv>@orJl4i+0x#Kq-n5%5{d=4XI{=ozr-aWn@A5)}Vy64uQ*rC+)y%Z( zh6J`=lI?fowdAF6PK?~L^is|IsI5(nUvf(>h#wj9*OFVJw`vVOpLBx>Fo@HQsoqT< zvP-**HHMcoOUUv4`j)7VdIez0km{akqTdg@XSyFYnNM}rt`5JNaBMf2%y<4e*&LRF zi@sm)$W8k1d6B(m?P_d7h<-`^t70PJKbuN z*>X;fx=T#G%}8}{s<~914X{vF&uJ3f%QgO1&C}@7^(J#W-KFkncb;f=O73ZMER|8O zhD+V%vuaTB#3WcvszJruwKHnC)IH5j?>y05x}gSlads3vZ6T^Z!dc)e~ZiKRQ5z5k<3-3F1BWqCmOm|WE^us>^I{BHWnx``@wS+0sjX1F{ znkLb6cs#V7$NAVkP0#D&(~&9N)0(IAl0ST^c{=Ls)7|vuxe^sc%`>czwJM&$uXmno zo?#e^0Sg1L&@<+Pb|Rm@bJa{2+Ng;tL15uU=eIRaUwnMp;^Q+GPq^)K#jUt3Imkb+ zdi$y7X_b2#pKsxxm!p(*S6;@Im-er`gex!N$}=iHK97HHq7Vv4EzG-V@!(@@(H>f~ zCwbMapA7Zyp+$R=m!n2mi}ui>J;_V^S6;%E7jcCa?V&|`lA9=m7L|h{AF8F-(9(Tx zXK>up`MigJcJnITm7QFX=GtA^!4+w)Pp{lF+|w8_J-nSCP+&nK7JDdtPqOdUPlhKE z$qT8CUFqnJMbI0qbo9m|=#3jn-$Uu9KC%R#;huJ)d738n&^;OEp{ctZm1gQLN2Ph1 z+hr=vw7J|pt=Z*B^Khudte+2OXh z#${_w!!pa*D@?o72l0rAHZZK^D`M9%V@6%KFYx zJD6aA9)mW#;zikQ@AIN=30&=djL8$hfJ^V z<5{v_k6BdEv1U0OAgk>#Xk$ZZW6*2UlI98q_F|o|%jteM4DO!c!rcSeX%}BH?74e} zJ$KKr=k6Ky+&#m-yXOjpYayZzUGA27fn_Zf|7M|~mnCFZ?O?)rU!nS~iq=I$yI?Lyx;F4(z+*_SS_Xccn-&SG~ z`hXcOo*BLr=ZKQaY8CvzhI8?1@V|;M0;kBviD$EzJ?-IPJCGG01}lS^yD`qJfsat| z?VfGg=w#K}kfUu=Z%e%(ko2a-mWsUjUeZ4!aZr=y}Mf z5O4~-6y;e(6+i}mOVQyuj7Vm1Xi1H0Joi>5AW;~>#lV0wRz%D>OqJD)c|smIJb9&k zgH$*pQV}N=MB(r!tS9+jmXD8Yne*L z$O9{(#mys#_C`=S7c8DP4lH!)AwyhJRhxEMDo_Yx(O=+m+7%ICEJ zEwA$Fkq_5DC+|Ej(WG9CtZ73yF`ON_0A!E%1#MQ!@Vx7W?3gEq1TU7mR0w~+TbYhs z4r3itcei3({0OL|rL?q2m?Lf=P&jd^K6&<9Gse^Jqh=^{040b^x}Er|*ZuznL3`H7 zhB@&+MBo%ye=69e;IM|x+|zeJj#QHfb#;#nITQaU@ge`)f$Ly5%>F!;S8k|}Hu8*< zjqQ-;x0%0ueJPFNh&U22ie_|$Mjq41#}XW_ws-ti{6cn9H3d^qS4AcRC}6J<;B-iz ze?8vj#%^?ZT7F!q9_1^@eR6}jQ_q-t7dB5^894UvnWlc{4U^Qv#V}w~N&Q!%8z)7W z6P#u?l^pP=s0xXA)B*NGX9Aspi>)HB)iJ@!}!K!$sBGlO(>4lxvce z;7b{CX!A|%DI<#v5=4f6;~$Xp?-{|?!hWrTmw40X7My2QXhfD z>w>1uhQKq3Fy<4b4{^GnqIwXgs=6~tVkK;Q?41HZ(Yk|Jp_0B*^cnoP{0Tm8*-K1A+M_WR$vC-5Q0#)qh$Ig31lV4QfthZrqpL!TWkg*~HV&`8finYwXe873>L zF4w-WD_UiRoO64C^Tqa%Z_z)_qC%FdjxNGkt}ueaKKM;r*vhn(gc#kV+YXS{yNw~X zI6YYd`z1X1rceT6?fIW%1btkxQyi7guO~ZwoTMhx&aLYA4MEpiL7*`Vx`~lchKK_F*YXZh-77J zz%~hd1X);p2u9MUM#2xIJHU4)%pD`E;m@<;?e=Zer3ZfJI2qr6^Y@;0Wm4PK5^_8_ zs4e8G6|%birC5oSNMzH|^`_Hgb;huZ3~Ys+Fx?eB$!DY`$pT$34^)jFHkb5NLWe@8 z278mKC)u)21dA`EZ#Z-V;1OK>$1-^5&pmrDXfNx{UfL5O%}C#@em->8@pygFSu3HT zNE*{*xBt@!9=UXytQb7}S@BNms_DiKbn|XiYEL}%De#GT2U;#Fh|_vqI9+t%j?jg> z&g#PLl|?)Q)Km@>=0rHUQh(=P7yjg57cO37q$B^vzXQ$w#Td50o%AhoEuD=LuU7)J zu$yi;PR@nu?Nz#T4CG0eH+eTzLm#YkX|JM7$;$9@it(>~U*U3`wB$2@7?+}_6G97w z?mj1p6Ni}d{qCp+zi#zWTh>{%vcaqH$XKa`vI?~NU+SJ*0Ood z1ENp{^(UdL)gLm#5PU>)HF zN__N4gN&FD+2bsuyBK7jy*3H*mM21-=5el zuC%WG#1v;1cGH`?#NSMfZ(5oOz3d|Yr+=2aJmi%C7DBoLpW^<{-4L)#&Yj?LKl)il zxSrX~aG)cr?}}pnfjH`L8~aP)Y0iu>o>L#{dkw$wb$|5Ji1ZX_jTIQBRGq<|Yr+z{JpX^gVxo9uc zNE0yoiX&t7ee_MVcmF!gm5+j~9N0C*u2oxX^_HW}X0cJi(yM^c%`UBf$wCP#_+&QM z?;g2LEMK_(!>?liJ96i-HAa~AXYC|FA=QOLaND_Cf<#6gO0Ix^uNAUeJ^1~Z?~rBK z+4PTNWl1)fQt1$r!Bo+8jI(kde0vWw!Yyp{I@gHYECW$DZq_Kd^ z!VNeXnC=|UVvm$4ptE~NRK0#F868WHJ~i^k6G=gKgcvQ!O!OD`9Qbl{+kyHhML=lo!#q}?b$cN*T`jQK`e-CJ z596Lrj!E1sZ5gNVYD?FNn!+pd4gCihh8?w{8m~RFy^yeyb|1iqDjQcyF4@3sY#-D8 z1e`-Z6knc3ue1pY1q~+)G4_bww(u?8D?|~nEY#4>%4`IAi~uW8KqPDQu#Q2c-+TZw zWuTcXIo)S1oR3DWSqMQOPCv2Q(~h+MzXAAV!ooa^!); z;DXtgV@8k7k!=(c?M)saxFWid{X(yvX?=)@1_~Z^1jJeMj>vEr0P0L!0m7lyjJ0k6 zIZRIwM_(~ilxsu>(1CC8krlrRer-LZ~r(D+j5yb=*OAJ{>-g zy5#b-gP%vyE6j$B2FMWZH_ESx8^1yu2GD9{UqC}N`xf5Qcq;N7mQ$Elqu#tSowONr z^J-x8N&=pC#WUyCL(D5?2= zfizVeoI5G?ITK)q`ZHqs?Wme4OuyK{RVrcyrXMGV-JDL7hp^d~yfJA~?U0;LKh2P~ zl-Z@5e&nqpiPcS%znDifsw;uD%=8;+lx2sE$cbgQmjQC3$D_T;L*xdDZd7k-@oCyw z&a!UW{b2hEvx^#F22?gy`H+~O82Cl`OH5kk25WTkqTk|KTe^_}f5NttyNTQvVOk}M zLXU%i(g4!ksJ}wz>v-mpd1iA=*4uxUeT5aZL+x0UfI7troSfuud7@pqDy?;7*fDr{ zTDN1jbPKaP+ZAnBX*WzFFW1J{EnRqxdJXAIV+myi91ZwgZ2Dy&Ay-dbmDZkAcVrpU zYR1Ny$#g4Pk6^VG4Zqp@Ndt&Zua}!sYGv6mgsL66kI<}CZ`~HQ?$ce-vbb&1UC|P| z)vjnlqCKC=$Olaom644%-d6e8Lunr$UfYoTXZToEi{V1Gkp7!sHHoWtXK8+{xHUN{ z_&mn{0z)-f;N(#AV@vra^eb4T@?|Vq^HDSt!E~jjmZW(jkx&B?V(cc(u*(rQX(~4| zPdc)C3-sFv*_F;Z^$BT!`2xA`6mS4495jW=)e%_=hx^2GUb*4hfqlLGou`;8!DeFQ zQz@oG>&_o{0NBk*X(B4kd@MN>8NL6L5>3t0#i3QmH*M6|XQR5r>25Hhq26Dc2{<&%sgW(L5Z@;(*HJK8T{~r; zf)qpT3;>=+Sb@ytU@#$WLxE<2o0vCn*&1UPT5VJFMP!gPRV;^%A>W{ik%-JxvD~J1 z9!Rp3;>u_^_(*S<0hXE<(-LDP(^3Q+h^Z2J9ydRLba$|QGHt;-V`1(7_bsJ7Z)=(?%B_9iDOHfzTtKk_v~jJ9UTf0L5=nM4|Sl{6a>%zGDbs{xEe?uz0H*&}ah z=+5^gVe@Az+DKm!81_TtRhzxKE%0+H!(Kz*-;4Y#G}Ye^6_ih!a9RUHm;%0wu4!_M844VcC`2vC)a>2U&%dE_!5_Wd>9_( z0b_Ad!%XD$AcjsU$C5{E0(%cOtpvb^IewWRhhSvDmy@)k9b;r_H+Q#g;ns;85*{~9 zj=CCQ4RrabW~-a%??sBJKh>Dj0LG+lBUI&F&$o9#S_MN9`m;DfSE z6CpLceUOfi1@4dC>utG1Ig*%)$`8lJHgqMs&%~8{lc9nI3HA@%Kg?;;Bv)sxLJUKm z4X}9WVFCprH)fuU>8X-r6kz``21z@GrU-;xVAX`{DeZ05fnrlzbs*UkIcA{Q6h&qr z+SFE^_O|Lkwt3VoBaJ4m5Z81+mX#m?0jx=dUd|~+2D}{Zv+5uuB?zM={$iPPYI$)^EicX~J}%Cw<=#29 z+&ib1=jYV&InF5#a=n?wnkv)Jqf^auPz8Foa9#xf7wI3-Ooz8O2Ajk#FerUcHRg*zR$cW_}iz+E!y8BIb^PZ@l zFZTbVWJzWPHWtjKkg9_C#bnlUrrE(Wxxmy62s|8tZW27?kCsQm1gDZiCrIpL_D<{q z!^7=Fk^Zjm*KouVMc0g=zEZGcjxVMDr=xHNX-^01ov^R##FPj#u&sT`4cwCL=?Q!K zKn!*K8e$%GgYD^aQCtxV{dSon;*45sGnXB0v)kl0+n(Ol5`GQQ{kp;S^z+xko+rEY zMAkY?5*c#q8Fa>`xJn_eT{nsNT>T|3OSG#X(#og>V=C#2Z2dhV<8sp9Xgc`9TEJP$w)E3J0h1= zznx}-W8_m1Z?iMePcjc7ZHWen8JOkwWVmkyp76WK$bH92};T>jbPlObPQ=a-# z!EC0`UWWL_?nf`PyiSoc*QVMcAvpbXL0l*^z9DG6fn z^2a1EkO8j75j_BiUOb@^fLQ28)ASaQnJ0u#@5nZ@>QkF# zuz70aPbBMNOqW&$%`zEy;`+aq&A=F$zZPR5*qhuRyMce;w;9Cy<5_*Tj9i!4!65Rz zY;W=@bgFb`H$bv&mZZ2 z26wPGxtlxEy~(eIJ5O-ufBO=5p3t4jR!NLF4f-bO8HAGIx+^_o;Sr}X#0fFgMP__^OW2p zgmyCD53Yka=LKHtyh+wJoo^ZB@a{xY8*q8pX4cyIC@`}`%Y|BZeA zH++6jUQ5@2v1X3TGq)>x{?*Y-Z1ucXsex9pA8YMryP_BPScKY^;^MuKmDQh2gRL2y zYG$@DZC>cb*%6QiyQ(Q^)!%kViG^eT z-hW=WvObY2*@VzB*{wJmp~flV*kg9P*@omA&Ij3Lw#qn<=dB% znbItqn& zP-kyrJY)ag?`9RhEuPhX6oW{9Z{lM%RBevONf=bH2f;43Mx;?ZtKW|E?lJcW&>ecj z640K%N6xtK!h7Vgu9FV-rnD@*3?GGDAZ!8Af#b*TixAukAKzdAQ80O_yE6S5|4aXw zE#|>UrPweuGn)M&cEb~h~O@1+H3v*#|qLG8;&BJC? zxuX7f){yo5L9C@Z?7ps#n+Pv{tQ!LDaw9i7@U`e4uIY6wz?UHPOkk$W6GGx2`h2eK z8SsD`_y_;IWS%n_(!I$ow)ek00NB512=DN+3jaIif8JL2gE8UJ5WR?Zxwo7r{LKEYLl$l(XMEl%ZLcp;uTI4M4`UXGpfI9A1}FLzt!{R*VeYPPU^yc?4!)iyJIbv zuHOosr=q>2A9bwKxWFdO=k8c7!m(P+kJTa^t2|IO$zrbe12ohATt8o=RWI_BmLVr$ z2tbz9gpB;xqd)=Pn~G0Vd^#Z*Oi_J2N#dg*Bc%o_`mq&fD8jr94m#P688Qa-J6>;d z9c+m~WTe7gmPBty_LspGRrjB~<5WQO-GA*#m!6PL6%_b*Jy=S7d<3A73F(+-ftOH@ zXv;cm$=OAjjqa=%!unmBXOWMUq$qXiD})FypB2|=QdgyYN}zs7++z(Ys;a}%N^FyO zvvnB_&aBfcg9fO9`lG$z-6vM%e@v6~SWLu@v)HBnm}QESL0clC=D(JtY76U8!n@XE z?7#8W#d?fUF>3}{>+z3>>oVRJ!%B)fMlsF2)*mn8wGq#{U9_tm!+F0JoIlp!y*Ws5YCH`3aqq21c#?SjY~ zhqW2c*Jd2nCJ$^_oBjUJ`)e~l&;H$uBSZOSO~(GB4P)9DYc6OS^@Lq?X0*@hE)v3N zXtDNX3dV}(B*61oX6O21j8p%G_uIy18*pTg6qhBXC=8(3#2kiG(=cDdQ)Da7{4+Qs zQvVx&l#7p8TO-vdSxq4j9H8!5QHnqH%?2uiX_X4Vw^=Xsy5gFLIaZn8SQ3&m;Ltjl|i1G;}$> z4LXq`(*8V6nWt7VpSyWVT%_)u&(m}fF_~3OzBohDp1c(I=P79{B7gfwhWwB5(gmi` z*G#~QHBZSNappYjpGy}sPkZMQn#ujp^;ExGq{Y$vSbA%J^O0MfaQRjvF!Ci?SbS+3eR9D+A4P6t$+)(;xPE_JHED8`LP{$n*;q(&zCx0Xg=Fx4K%o^>H;B!Q z`Ab>Ij=3RoTX=@`cHCei!vf2;w4{BF?JYn*mru3%PqmZlyL0Unj=xuswQK2AGdR_h zp^hcD{FADbIL-W(lN37{o`3wIOKzEvEWp$JdJBF_H+ZG2YA(MM;F;?CAK3AN8 z&+`=%kaGFW9q>4I6DexN%{W$XR;aBjZh<{Hb0vRCm87FJb|aWUE2e7&258(rVL6=y zcgWq))*!HllB9~3XkhtNTl`d;Tu=3{I5de)H3ME%@44|Ui&sK=rrO2Au<tyUC*Em zI3Oopk6GaWPCTZa-A($o<0o!A#E`quEU3~L^P&UM8H@u2V(L-*x%2NufwSR&qKy`d z^a%lxt0N2f!AJEsh7!7d&=SP!huP>@^^eM<*~ayo+0qN^H{H1O((^67bmP)n-LQTi zXCaqZA!FMF9%mujtdRN%6|!c9tURGYwp$^qPiRkj5|&JKK4D38#0T7zEabvJ#O_I_ zSDH<3+v$~Nvs*j8(rj_tPp>p9ZuRs^v(;U4dZoGO^f-#5jeO^zs~&pz;fDu@+C)&* z81~H5ABx+_={2*ch<7i|Z9K=?W>_#iyHC<&E5=`#8``Ia<@G{)xRq%ay0j-zv1I?h1n{ zeY4ZOTU2SdVP0G5>aL3MqPC@TQ_L?RgRC3HeBx3W_fnUvODai^xNIh%g&dhra->hn z)?3o$oNAwZ(yf`S-ah5zi4&o_w!2mLPa(g(MzwIxcRpqLo_HJ zl10aa8sihh@*mgli>tfZn8f9g>v+MV=#`VmKQNG3{J`J^Yb^t&Oc_eBK;(|NiHx7* zSutkY!OW$`UR5x_9hDVe9jUKP?gsC{)pu{R2v=*0ASaP+>AKZj*S+NLnoIVdbk+OX zarpY(PoJoNb9B;`HqY?CjdfudE|`Q;bn$-Io=`Yv@DLLqjM4o2tuh~uT72IL zpncc5-Jy;Y~z#H9evX4{^lot z=eBE4y6C!p{@mY~9zXq!YfrlPx@Z2uw|?WEJ-_|wlP6A8zmlf;vG`W%Wj+Lx3J^eH z$MC-rSYqDN9IpY{yCswsA`5*CW|iTIX4oxXC=e8m`1CG(q_@XPI{iSj?tkUj^OZ?tfLpqAsEQ?{;%R~@DKWkPhJp|tQJywnw0f=Tl)CO%pcEYZKV}G zRP@}?M^1Tmh{FC(I%_MG@S%V3?jbIUp&PjhqQ-WNpBROG%vY#EhO}U(g0rI=+xY6} zBBvy&nHwnrXUrrmU8yg_DO>|*dNtN$?Go4lM@Q_S$C|G)n+QD)G{+--r+I2wF%#DDG2%UQGJ<7VtLmI6E_yP``( zp-onTwt|G@a-0oA@ikl>UE&5EL-V38^1D<-IF$Yq@6X-9jNt6UlSxdkp@GMVZ3Y5VpP|l44WiSfxz=pubzVjjJ8wHZ{7zO&y2h0&Zrgbh)jyY&K2L5l}knc^L z%~OmvUo`Wni|a4M?a06=Bb2K_aOv+3_X_(!GOM0C)VHl&Q1g1WH8+|tZy6U{2(P(1 zTJePM4lwHoi7r<|N+7-;Jy)-yn1e&$>rwY@6>1u>02ym`#sqxL)zQU{;tQA8caPeU z=73M#>hvK9j`S|&3RFJ_C5`jG(;H%#e zow>P0Y%ZTGfrbAgae^5v+?BqgfC79XwbPYk29SK-|L=5_MXo&eb+bNLU_jIk&1Q?q zQ|OlT`2ex}l!%~VC*tQI-p-c4-4bEIe!LE*h<|phpB_Zf>6b)N%s&bLq^HOG&-m~3 zQ%}D!K7Chw`p26kvlhZSmKo>^fwuoD{C9fa>3^U+0K*&2A1Np=>8)_*Fw=%EhsY=Q zKb0WM_u}z@$etS_Cszi_5J8pLabK>^1Us4z8b!bUwpN z0X+>)2dYTOSB`nPLCb=>7Hc%|mGZF%SW@nf9)j&pC^Lg1Ihg*89E}XP7K#tkUv~{d zAR9EH0Nu56evJgr2xc$`DxY$}=~-AHZr&z7RdVx*PgSu~SnkH^7e&Y#l+LBA-1y=w zWRgGoT#TV)w~OTjJbwK6ui`46il`+$(nZO-#J*e|W#fr3`onpjJr9$N@!~(9k6r)W z7vXCjXjS^pSgdRKtsRrW?ipS)Uhu*2Z8 zAWBr>8uJhXxPclHH%xERj=y2r;`k=zGgt#C*L6}SVDv=pHxN9^CD_e5R&h2*m&&7> zy!yj$Y?n0*Zuyfz{#xCC@~RZhv8=v_m1) z^>>E`f(i!81(iAtzhPlZkhQj_saUoN(^a~8as2qPW?8n}cG6Q^;sXzN|3%HRoB#A< zw@hL;d8p`$+N?<%!#O|~)$fa&Wf?LcPZ7NtF6p@b%0R@z@%mCyM1(abM+0;dn|@Dy zGHq&y49r-Rn+YtibA<^@ayxO8S4@)epr^o0{J(w>3NzR*(#n;}#*1r2*?4uxgrtk) z<0Y`(22xD0sYR-b76zrt51M>D1QZjwoNRXj9JM3>EXe?#9Pt^M1$`+4b0h!|x$D0& zXmW_bn0)QfB|8ak6s*LT>xvB9DROPg1Rwj~iC&-#W=a=qsC2rm{WMPfH*O^8Wut8yAzXCauv zz_-|>L$XiiLKH)*i|Cg#4}vR!>>;i@yg9LG#H*vt3_(c{9^wZn1T#VG zLu9l2{s&Df0uNwY3L(7V!P^WeLCvyrBXJpR6E>Q|n+~D>1)MJHza=-AvB&1sLxZ$D zR^Arh3J8R_rf64`1KZzb`8!vobAsQlpfq4U8KG1}3WIgwvOgcqECgLdWLbhmL!Mh6 z^QH1^M1J?Ix-{jfygof_Q(J2s+UR-6R~g}eyQ0lR_X3^Y5dQc=WHjNHz&T#*%7e2u zS_Ha8y+D_!7w8i80$rkBpi9&Xbcq&$E-nf&=}m|)JHy}PsxzEM01u;j>q>DtACqi*Z&2kro+5k5fT|l#`i$HBQCv5p4u5V0? zYjt4X8*XkEl(IyfzVeZRJuE zbdKK_A=}`Ee)!YLA%FaMd{}5EWj$c^C62&i<6V-D4p&Oxv_*q`oj7d~Gt>HQIdV8& z*yZp_e_e`s&87a&q8rz$+v{w2EV(T{Dkk9lvv#;6@^8p&@ULMe4oOgVwkB7l@X+i& z-%pZHJHd2#&gVro5md6+CpSsb0kMqTa6OPR;1GBDO>2#v^&xkis00%Qkx0GW4m>1o zgz(UBsB`tk-vWjOxiW-u4NRmESYfactS~q~SYaR$bD6WsMbU%T^5MRRAO6O*PS^C` z&1)w=?v_8^)^M|)x_Pawv5V%NO#6f@R(R309cNm1%uO6Qj0Z!DRC$UT>PDwGo-ASz=OTXSH$3NE^@F#-(#iicpm4 zkK(LDh+{%5v87>`DUkPGYz9C)D$z7~$)hlGO*p;#JmeD?Wc6hIR%O6b8PvrrL-AU$ z76A%_wJ3v3t9<-ytzwqSyKwE?m9VuI_EsD=MnADDdYR8jDi*m({f1FfRA-HcbfU$S zvGMr(nltleYBPr^4fR99e!#0k;&y3TclaDp$kum^ga-{jxB_65{nVmf15~PiVML$} zm+`L_=-njFzt;WoBPPMif@L5LHJjRK|DGNd}(t&6FZxHymj7cORc!GvTy zpngLsfT7>Nz9Vu)^?#)? zJiUgC3-d_$sk}o1WA5Wp4XLIyLd>G3>x5iAg&(fBnZNkClNf{T5&!n%$1_xFf6SOU zpM!?A-hiWI4JJ*-oR=q#s5Zg{erSDZ5+#fOBG`SYH)~V=lx?S0j7?Nfat+1N-WR!a z4V0fUrHOla&;&}7SXmN`Cy`Rbjf`q4GY4+*yQz`!m&mjDIlBViDM$s0T#g`qWwpw$Sg+$Iba z*t8BDyHbfRETIQ~>KDk?HJ(9g$_2>7Kxykd(}dMh{9u zu$#AUN}?Gf9qV`5&&j1>d4bes;{xL9YFj*q#G9^=L0R;>t84+GL(D!Vvy9<~!3ru; z&S*S;X=FY1Hp}M`%=Pdli}*U4CVN^S$a|ALJlPFy=_$<(T#4;pbn;PKvTd2%GaoV49+=YZOngGjQp~Zg3Op^aY?5qiM3wWiJR&PtkpfG`glAqG__fVW5(|$-p5e3t06A|n`r??)`AdVMEp=dWD=3012fcBhPoCnrFZ}gf-3nsD-s%h zD{}vO$V#&Donol4FR@`p3UjmkoJq7B_^M-4Lu7DOtv@KV-49IIzjGIS!;4! zFTvbIPBeN~8-{+cye+KLo$A6JoGr?MHs#X`;O?e6IQvrnmKXrf9s2k7c*YN_8&+yM z1-O6-hNYTLliT%eJ0IVokE_%EV$~hu2qx={&7P9ik$odG&(DU{nlPKX?Q4g8~{pq&m`4Lc-+@nW1_9*zVTU;n_}h3QRypIEO}C_*!tYFE83e}qhbNU2dX!;cB8bH zr$}!^21CO29_fw3o1vBUoUpNBC5^WZV| zwi`3Jb5l8P+t}Y1AM(fJ8z%{LaMvCp9P-FvIgI03%k9R-02zLU&HH-~E2#7I!>KVJ zWd3;k>}>yr3qNznAOA>t<0KAOaTeScj+t&{WiuFrBUsi@y-*u6|M0`tJ~t>1F7wN; z@!tahoBQ9tubf>GM=f}#FNM#!;-vk<52uHk${!<@sT+cbn!#uGns)54rrvhQAODOV zes)`|HAA2@u2`ca^FtPEZT=kbU^fMFF*6?9dwir#{qgr(y}8d{LyjgZsZD1}ukzX`kaNY%ze73IT!OzK^S@cF`xm@0+wcG6({H*j^56N~#AW_O z@$5ETqZ9uRfB*mP(zV%Z6bVCzp!W>ShiQ~J9OEt{ST(Sr7?(zyGe&NJc@J7_puj?E zi53J)upCHXrgG@wkd0jlUxHI|BDNi1KyRbs>N81P9Ah5Ycp#B=Ly^vh0w-)Y6e&Z& z-RYsGK&%u)WP)2aV<>VW);A2rfP8~cz7N(YNzyr=(=ZgE4mA`!7$0eq`B0<`MKT|X zL_?7>6!~H(Vwc!ZDEE8b4Mii;m$$_1Z01cy?q!|5EtZ!6;vublt-4pY!alPV7RC({ z|16p2FgmUy8a<~iXg&gBm{l*(YRRg5x*8R3;uPsZMrN5q^ z?e{0+CiTY*GNAeElUKvajKA_rf2;}UBB}~uo{I@gM6D(;)fyg$uKI&NE?Mv{=^k&> znUgO(@xO1s(Q23e>>-oK692vUkiY+{p{|*YBrb7L$piQs9Bu~wwE0r-sTi`E)n=_# zmd0*vnGTDa#gz++Ty z*rbYZ4hGlIg5eB;>_}}G0m;{Y9O)D(lxoIS@?TzC=#g|CLf6u#^{b?@jd=L05hvK- zNEqJuHHYyBwnIN+*1=5~yWtW4&fy`p+?el{6NLSW#3u3#Wb4YdAkK}DK#Sw0Q1KIr zpJ?`U#ZUa``JQ}E6BWIN#le3U5mLn)&8i!-!_L}a*ttZl#Fa*Qbqjh6<{)l)-XzmD>!zAD4MwnQKjOMywXPr$4 zx~-u2Vngu-05>$~LdA>t#afu?B~qN#rT?>k9Rs)o;{!N`T{69Kl5XRTk}DX}FW6!G zNJEr#uab0q=u^7qhDox`r(x}lWl5w1;H%-C-eD#4C$5G;wng4&kn(wkS_lf`7R|8s>-p~VyjoAP5 zWk$>%_^WR$9{9#Lx(DQ}Pqjla#DDQOF}s0kc-L1J54__m;Q`4PcwcqGD{Fe2T<|Ee zv41@Rg}h`dtvupi``Y->nq+lI#_9V1kG!{!vFp6=JI{H}>%HgRnKOLTaE7vlr&c`u#|?4U9S_TO_0Wo+q$WnEK0W_ zvrgNjDUf!RHgMB4sa@Ccx~*e7-6pk*Y}bo8P1?1i#pnBb&bj9fNr{T+rs&!bXXf7X z^1S|@-|P3guU7e~pVWU2Y{D(2C36b!o9q5#WIXC$2XAW&0vRGw!tFq^v9PR3x7}(L zDqq}vXyPkVShf(|{~>xDuA3(=RbBk(DNsSNQvO?91 z``uSmrEd#z^sn+{M}Kq5=oW2VE!1Oi5%F0RBzmg_(EIL&~0IW^RNaaCLa>-Nk3DQ0fOVEy)+b+Ry%#5UciL~diF==lP zMcPk;v`79;_N{8hC-m$kHl0a(6P-wVfE0#BT+GCLxFSL(OA(?pMpDrV1CC)XebN2- zFo}49(9!eyUGrj#1>s%&qt#>lk#t}C^xyrRoFsl0?o4b@MVDQi=UZ9&b&&nhuY=;=uY+>54)S}n4oZaiZ+`wy|D;Ws&6j59H<+C#n6~8v z=UaZl7yZ>aYhSaQ*=0&Wbc?oPe!udy5B=HViFw9I3==$Iqfb^gItRT}I{0w&E)J{K zHu|L3y9Tvx#vAvWIeQO@lt2+Q(?+-bw#F;gR$j5z@3*yHC0SAHf@lOm9q3bOM6Q{z z(fOpA$ilac&b4j%ly$2N8{H(#j2F*VUOekxJhK-MY8`TBxzQ)u=xhBDll!-5U`b468rN zqgK;*Z;#sX^ieXR0_azH)MkiSw-_nklcns~UOyMOdqLQlLoF`;6wPS6kBqWke5UcUOHKfLm4 z%f6OjA;s~>{_+1`6vzDDD-6ZxJh@kjBUJXQ=gDMro^bw4E9i1+TDmWU>C5)z6dEnY zi56oDw7`lQIJ|1MgFvxR1lbuanYgmaxkk2Y>~fAyc* z63c?b%)UtyGykJkO5MF*W65ZZ757MDCakeRx+VMle3dwOy08D}AQBRYM+(i?NI3Sh z|9fBT#f+yQONY=4chO%A)1B|z+Qo$Ci(&mlVg20JQzs97)#5N5d0@5%5Bu;xm!w&QGe#bI z1oEWIpbD5Y`fKVZ!4NNGsD0v3{hwkLx395dF$Ut4jQc4^~gV9~sEXocJMz>KpM0aodk_I*Mu7y~N`O<=S4T_20yQWx5 zsmKE+chS#cCM7`jzQqo1AMP;cvV}WjcFVtpL;i`&b67&9dGvFrQauNR@@Ni~c<_=w zwrucLJT@mxJo5503-aBI&(Z$s{?%`AcCjH4_r*6!+^Y*SoGHUp{ikt+9^P+0ozLghoGg{(;(X={;@7`H zz>16($s|VmL~k-Pygh&3A$8G_zK@Vp*!>t#o!X0yhZoR zKfr<~6~`95`FQ)+S>#n&HaS@~KOC->4+d#Ak2#E5JvKC*Qi)-FCQ>K=uV4N(8@s}M7U&|m+;e5PXSq)Q3){+VAL8aE-AQAs_DIy05O6XN#qx2tQR6`^y<6955d z694ai8$U58J4mghqTf27bsv+vCUjm;=Y6^?MUB%Kr35oaV`Q$(eRk4t}FQv(&!Dh2te}#?^ZBtg9Nxl@0iV8#kM-g6%3MC>Bj?*Kg{PBMfAt>8#_t)~6VkV)ocmFcl zU(1!~7b)*{Vpke(Tledfn{d(N+YL}{s7z!tfHFlfczo#zkRnNNw5&^NBLJWhic!_% zzVwrj>)a=h*p&gfp0CwNRrZp08@?u`_6wgDVpi*rvJv@bjffaHW}@!e@8+xw@*?fG z$AyMv5Pjpv{O*ea(HcL71Px(?zw~1cwdV(Z=Qize4WK@^dyo*3dIiF$xhsxDqwQmR%cXdpVWUw)ZV^i@0<#X?V6S*E1+s3e6EWMRhr zGR-odnf^$gm?+Kxx?EI^$~+c6CM6%qB={9+ngTB+>$wQav~OtrpL;{8w_>?X4O3-O zeSZR-$JH)sRE|etHwpICgIMkM5B-%N{RQed5ln*Yi@F0PXQ$l(Ri-Nc(GH+q1c|Dg zJ4e~6;ncNx=O5n&BuGJ`7>Dv2V)8U|6x$pcs#E*RmnTX-FaR-YY(U@Ltlzk4qRTpy%K>Z?iPe$j#C{=YpeBOD1tevOiu=+b` zPY`+5SaT8zbk5k1U8@K5O0$kmyV;)qJ_xa*?)$G37**lWd0scvE!C-1Ct9)n3ExYd6l1mh_abqGI zkvER%pYDpu#w{5_ILbuZ0xez_KS92Kvg>xOXcnST3M`G-jZual^n;T|MUtB|88Ad; z^JJ%!+80p%W*pC1$W?@uMa3!VlXuvL@Hw#J#4rlgL$VxiTJ-1sTIMNh8Vz<{|HCO~ zmxR@>o5890Igo~Q40A{o6h84p@=95t$an!087!NR>;{U`RBf12GLC`)-ZV0h$0k?6 z#tlnGWGZK^fa)jX*inW1)krJH)(UG)vkcZ{^!oUet)xw#?0w?ewQJX2_#40a`iDf6 z8d+^Fs|e)=dfb)@TUY;iV99^$4% zDX2eGY2q}irS!bCjq5K>dB8apyQ*gWoOkvAI@!{5�uF{bqV0s$oJMyQ^McaD7z& zqbYBFnD#Eu-d6)Ai-Pd*qNOMv4RwInL_^h58BX};N8-hmy0l zHtIn8v+GXL@m;rBe_`t5`ip`ZJ!jtvE%cm^HAr^fv2Y)^kPa-Ay) zT~!jph3R<6#-RMk7#{deymKQSD&Bhvr9Qx31`Gqx=Bs-I@mtwI)l9>wdpp^NCnuDd z^@ozp>bLSZ6+jZNqJFXdM^gy~s=@(l5wZs&UDZLC(gUG0nSurZUl%Vj?<%W50*y#_ zVJk_8Yw#}i|MwYbvEcOqjuMZ^r?P>33V0PsRZ0C&n-2E)UyZ3w`_{OY5kskx;4)Y4 zoHcQG4FCtVR;6#WsisZY?(j9bd3gy8W2CQ_N~^ex-A$*cQs=Edhe9q{DbNK2rZB2l zqHgpf8}%a}`T-2fvLBh%bipU5esE)>I6@Zsv2y3k3du9;0+o2`0zw#SBx`-l-=X!g z*;K~QD`<4p!E;GbutwY+ZmAnu3Zjmnwx~7eTk5Q(lxJ8=4?xlq)S`|1%KbAM5L!$9 zqgpG12iz(eYf=bkZbpek7ek$3D@-DC6I8ba)5eQ`F0KM7tP0Sy$Qr1MHy0I0s~>P< zPX*jqRRAX4kN-~YlWj1OP_t3tB}%~%Z_x+w*l-?Dj67{Kpvq{~yC`>tiUpBkz!HEx zN!6%-MHf=~$#$G^&n{1#?Yf5vB`WVmoj64y@bW?s1?yj(3gE5!hi^M(Fa?Zi3O2ZXgXi0CK8JYT#3abJC4d$aLraknzHS$;@y)3w$6FxH ziDllFeuKIZ4A3_AHK_BCZ#VFnrVD^zfXR+3+xYRqAa$wuzeI_}whi2-c{h%BXR)Xv zj^n8QjcId4E1yDsf!F;?v}j(#@cyv`Gjrm5LW&L`ibc1HTcnv({>VAE*}QJgH9N@# zztcXz6zup1{OjIgrJoP%eK~sGKd|TCe|7Iu(bZOyQZXqzd(roRgMnsphRsuTk>9zF z-CFqvuCpG``Kr4fpLb^gjz)O~K|1cf96hrDeB!*jIF_vI=>@-zZW2Tkw8VVC3d{X5 zsA7+S!g6f;MziDdn_4yFCTrEK`3LIvaK1YK!1d;s2SCcN0f@=cU~pN3UDO=s@yGmq zZ(4co>L~oN)kmf)k38z{(`vh~InHBkAh79qtQOJn)o1(TucXbnUkiq@rj>Nh${eZo zCSN@^PwB%EFo>x;?xJSfPZ(K{l(w7PSNze${&*Mf-)J64F8CvUyPG{jP3P{SX+V#X zgG|Zv8@}uwex`ePzp154Zr~%_uuAa;!B;PuP21f*S9OS{iz=L7F=*fd8ki@yfm@*t zn#p77U-V_jb)5kOuL$Jqal%qavo9MEao)iJ|MkjDBERO3cOP(m)5rC{lFA9Nh0JO} zgs{?Cg$32jvqf9Vt1?q2L-f=fyP0)q%MBuvNFaA!pv|n&8Aeg$XE;>==kW`?EP8pq z>(24dY}b`fhU5MdW7k(>5 zYA|T2#Yjsn$BJEf2i}Kbsijs!JG-a>XCFUh_oaaRoG#iE*6Y}B2AGe{b1rIjl-3&n zvkCyS%5Fox;UAI)Rr;L(q*fYgX6bhXq*ios{hqwi(rCkA)o3FkST?1(RK%jyyleO*^57Ly`s>v2TQX4ttn7B^=GI46rDN2 z6wjp3@EZW+g2ib8nHJT%jm&i6LN{Gksg4SlU5=IQEn7^OB77NU~D-5g&;I z2}Hq`(E&#~(19m;;-j(TaSYecWY>LIetA~J#EhGm6EM~ZSlr<>(QW-H2>(rd-hG&# z4?OBVDv0op#8pE?1Hf!w=n$K5+egrn&byDurB?<4TH)zBgG2d|=nZDSo&n2b*PYUi zmN74$0bg}L$!Sk>MY>USH-^7=9q6^b>TdQQiLb2ceyaZ)g|scZe=Wyp)%|aL?yE5A zu`fY{9(BLGL-(s)=dJg=j?Q}&l1JU{NbE{o`8aQ2s;XVk!Z@)i=1Y1TaJB1-$-H5K z5+}U63;Qhy>nMSufH0AE)Za(8s4x6|=y#>R580{cq4d6DY`4EU@h;8>c}01$*3a@F zKF%D{xWXSlbO_T@4&io!+I-v>`zU{=xmG@IfD6{5&V9{Nt~DksWxqa;tD*X1Isb`I zh)?mtoF6I0??y#dS8d6$?1?hNToOIG4bYGg)D8M#!g>JoG5@;aGVDG9sYus}@lUYG zfU6Cjl?u-a)`RJHCGtIZR+w8l$?g;YvNSM+3%|K5i|*6{t|zmm&OBy{wx)IB*AZ<} zqf0m(yR%{t5iLOENw=&o`;=+!4m#Za#}@U!n+7@2Ld3>sh_xrsfgQeK6ZOhUi zlqp*y@<+PaGi^CAPK@jUrdFy}%v#g%l4x%Cbw3?lQ3>%v&W>*x0q*~fh=*A|PkWqz+uR`L> zIu^rWT|5}osvRqM$i;7s)4{0XgR>jC^e||@BH=gt*FC4uq+j@B8z=3v=5<18^3!f} zQU5=3%W7X#zNo)At-82dXMSeg-;BGzrKx%ie5zr0FbxmiIt}CTXGR$O3_VQQJf8p=96okae1Z^bGBy8on1HSM0T$>QcXMqyVrZFkUoi;RMKQ~FtR zqKmNd{0V>DANMETwB4@z+Q0UhMcV+DalfDW`kA)y>;BZDD=xJjtlJab`xf6t%pLw) z7T)i7$+IwmKq{PEYSu6S0r;8wZSNPDnwXW~|BO5#>PNRr6{Q3JA0&@hX6Z?SCVYxS zot{p7v*V}!q$p{;Kk0Yy@#lVBL~PAT6CFDU|5Z6y34?4M=+M|#N^+1+3P6xAc$I(6 z#x(LxSD<`*CQmjZ6R7MCiF7?RWWcA!TycL_yh)%TMR)LSe*$caVMM?%{&0$eX>Kne z{?jGTw(-rqFGbI{JJ%OpU+?Vs4_rrM_{N4oy_QiF)ln2}!kYF|adTRPJ25zCHnLz= zU0DcFXoF)f@*CYpT~H)9x)*w#asv=dyI|r^-^%^=jh~9!UC5Vb&*-w$PyFsTwl(_P z7XEgJzE8$~pZ>zAq=NVaP8?!tcm z+qvO0LgFm?T%3FMThFhD`?UY|OK9)5Ckqn-q5;`yPMQ~O9{X?cv;F2<{LHtDru8$w zy=b=mDgW9{jM0iN(gEAgJHpfGPQrk59Uf zkvmek3*E;*J~^F4t(?@#=K3}lwZsDztq1K(?go{IB3lT?7B#qSW))@waPoNtUmDZiSk*h?tLGh@SlZe=sXeMkLZK% znCmxPNEGj5Zu~w)VTKWnyQuq^6P25fIRXG<7NO=jjDYj*=lR^c?y6>;||e#JtJyB~7+ZRDScyWh6Yg+v?gx{IAi#gpD)WyF$v^Yd}%F6-5i zclECmepLXIzdmb!iRUGLB&wO(_`!%<%sy)I7+#pk6T#hTYihJyJ90od^XsKTR7ssNj4(yd3hUg z>y?h&)_CM5W2rQGrH`77AH{}mBn6JObNMspc$VkEf=Dpq02WIx`1#U;m=6}j{8kHM zzO*1jCN>`|2z(Z^)&=`v9_)wtU_ZD+?k0B!chh{ZA83}D3iHG6rfKEp!QEtHt@Ly5 zT-pb5a1^p==7FVgZdQR#uw%`fRQq^dbdd3s!8hVoeIqXTMi4Fi;c1B{WGy`rT$WaS z=o8+R2XbW`If`QvEmZUeWJvRyu!7}vaTW`Ln^xL#4bv-_rDgX62?wtW%)>(Jl?4=a z@nt)%e~!8l1<;3&K1B}t>PKG&Ty*uLkLuB>uL)q(gh&M-i)W@bv-dNc`qJEyV#DKc z(#kWk-={nIvi=%Xp5T9~%kpq(-R0yhS0t`+eDu+n)dR=KaP-)LlN>s5jFTmJf0OzPQ)D;MIXia9KMu!E4v+IWpNHcnw;*p0f$Anw9;DRz zcf+(tlFhzL^8=D|Tae~HvHQ8q*06|qj^pc{X}3K5Z`g|p@??G zbj8h{S3ryjO1#98a&fzU7wL4kc;TRwZ^f80&$}66`!@vZAx;UxkkN2qpLK~+$-^F| z5}!Tbj05HdLgl7R@x0AQn8?X~SE>M)`>cMCCH<1q6S2u4fyf@UcD|ufu66>aP^{RC zJgjfNMc34{sJut(|y`UAx&@D$S zx!qPsMq43JR36?6)CM@8JjqtbZ7YzV?R@g)98kyqK2?7rps-SlYGUt5S)6yEfQ@D251K^*)F-#CYV2rDwMSP z9XriKsA6#uCAig%^VyANssyZM!W$eg#1=}+4-n7E48|&Q2!+d_s8F)z;wVAmQt-y7 zfLkr(T3`bR-G2vH=s>uk@*>?E2i9eZJfxN!x;eo~KWk&3097CR?9Fqrufq40z_K7R zGo$gaa2CA}U*7V{+r7cz^)K7XPv1^13=6Qju4mtJyS`odxMD0$wEz?kTR65Ye9MAl z3yYvpdF}lIy>$zZGrW_rzv52m%JK3LVNCOv&Y2@RamuA zT>XWNYNK#GlshUpqL)#dA$K7DN=W^X*-W--n?gLFE7v17o;;+oNrEm`imy+}2Pt$7 z4m7J08uC2y%T3(5;Yx>s#(?Pt>P$o4NIcF5i95JlNCtq&Qu!b$GgbEaAo?m>z)0er zXD~Q%6_Ins&PLrS9f;|GH7=!A((;ukY7X6uY5hd4CyPRDONUk!mz%V6HXt$CEa3{> zFv@yK$E{DhB-}ByUeaY~JzP*s&jOAFwJ=iym=MaWIPqwf4Lr43YF0EfE4$oK8AUM1 zn$^o@9^0lXKIjH{npG z4O1V7FK=E|dVTR;KUD?TSGhQP@B69-whM^!C=MnCjGYD+Z=h%U>4}>Ts>@Vs)7*Gw z(l#^i9#cUB?xb&x(`I5S>5S}G8FOu_oW(&9TUC|OwZrk2LVqLm3=<;7ha9r+QtUwN zKX=J8RNbzt0oXsCc%Qs|PQ|r!kxhyO5N`EnEb2)mlaiiwM-=XOi!yt}O)fe__|V;0-ydr&^RdXA-@!#Y(ppGGmz%o)9ZC5GsZvvl;5kN3@i{4E z%B#yjIieDvEx%O=`z7=oCDa@*CYPjNVRi*`odN~#ltI1Hk1t0H@5l;DyN|`}SML^b z1@fxTP6J8stt0h4NI4DY|IrVqrA0YQG;W<*$&N5?3lOi;(bZ zn#+xZIkp@sAy9R3Wd$+jI>*19F0bwD_WE1sICL=|ix=XnCD(@=6TfCYUVeK!iKtqo z2y|yv!dhKZVyP9*oqo{B-iRwnqBcrpbmDeW%i6 z*L?j}F$Y3yg>Fx~3R?D7tDW|1ym^hpA4J1!zx#6ZKfXR@G@rj@=V+;h;RE{Cn9_ks zlLi%s^5%;@I~_3EoaHPm_o|JeNZ#W>eC4HxeTof zU80>l-xfieJ|r9mBo?sP-5%&)dMYC$a1jq-Kx+D zJ*)m^;^OClWMF|vdw5vJp*%|6LH)O4S*JmfMCo~02iG8>r21Ln=BYIo#a1>%5BJwx zl&ZWWejLMsk|b3|`A}li9du+sE?=?3gFFfw8zeYAQR8$Z?keQ1HxJK`Z4uY3^Mp=Q z>OCa!3aG<9uO?<|NtDO1!jiWAZ6V?GbaJt}Zr}%rv3o{OiKx@sdfSF(sKKX`i{Y@i z-Ft=GA^o%OP(@rOp?E+GVYQi@Tj}fL8Wt%#USvWi^{2z=kH77rYK;>(I-Q&~!n=Q; zh#m`V!37n@yPJD2#_yF%M2U@#Xy+qN=2U;cAf=O;Rn{vq$z=pz=XgCDLkcyNciq~R z%iW9QyNZUYzs-YSq|8~nEwS5@a9c9EZS*#P?usjRN6H-!Mrn%)xhAH(?1f&-g5i@l zT)ydDcjQ?e*t9OQ*JP$**lSdnujdLXBD4_RGqq5ux`!T<+hg+Z7_^|v4QX@X=7b(! zI~Y`w;ThGkDROn3iHh)yEc8LQMwiAfXNhDr7B<2Vts9Izl)7Nle(^}n2=A;=(4wIMivST8B(K4sQc zXRjQ_JJwd-Rkih~wh~W!y|^8LZ*m7>Up{BOcV`#p2nycVd+qyu{B17Xd*J<7uiDw+ zuGXKQw$ca`BA|C$RrOcWI19YN`HpQTj0m}c?xc#`mG!lvibyb*t!PTJ^G@6jR%u)> z%YZqI9Cgob*QI#4`r<)wVRl{lW)X-*KD*)5Z*viSqf6q`s67c%BE!9 z-Vt!vr8q zO!=q!u)m_02|l^Z#lV^1=|P+f7wN90_|WF>6Y9AM+)Zu*>p>P2PE86kgZ_TroueDh zr;CtzZN0=}k4|3wo*R}(W)*v?o1s`)P#=dX4WdV)cUm0z9nihvC)T=}VQ@1ukxwsf zw*X3c*GOs8{z;4#eEF%|Ztp}{vMnf{z{n8Gko`E8aJ{G%qTfSDMLWqNrS@9=%nFpyYROXCpX+>f{obry!&{GMT;@!HZRAWgJT zo~8Zz?b0LBJDG&&2Ia`qjxzAypT@KC-ZatF0t*1?uGZ3Q(Mo*>E#Mg9NCrHaEq%3z zYFI?PttyDpr|f)z3^WL_O_J?+hIyLg*y7Q?T(sb18+{A?;v;dhz-piQ`GPvxMLTKu zp?q~@m!F88bwSDJOn@Rz0>C`@mC;=;Uu-QMjKJ*Pe@G~ zHsLQNE;)#wH_3d*?!D3+aPm@+Xe+rgXv^$WZQfndNSOx$t&-fV;Tv2Ljf8Nb8Uk+P zpIz0}5LFjW2$Gw1j%<&KW@Zm1o5FM3b-cA zZ}qFS;5HfcT~H^3qX%BuP&xcD(qb_p3h&NQF8sZ1 zd7Ys|?UN5gE<{CZPWkc|ArzJ~0ZZ=MwRCZ_S3Z=Dri?Wn1XDEDlbF%+Yn{}dv~^v% z-?^^0si>jLKw*L{xPEH8O>Ta#FP_HeeK3mGtLyT;$(hS#%vhLc-j74V)C`H?!bksg z#F%bgxyi5bUj6>}@r&Cl-~Yb%eJ2jibxxFc9~Y16|0#4&{zPkw$DSE*c$92>Wzj|q zPX4rFgCu)VFu+6gPrP^b{XXA&xPQ;C{gU7W*V^=(qp@C-phaK6OeErbQ3i+5cb@Z2Kuhx9l(C8SgD+nI_< z3~9B!0HMT#B1Bher|_mkVChrSKYqgCYC@fK)SZ*}7lqEH2fcgI`MCbc30kXiK^Qi0 zF`ppDPJLJKLJHwW{>8_#E~+xTaKsvcCQ=%a)*%N1VqXELwEOyJ*!Ypp>K~sp0)tB2 zh3>q(CjBE$=Cm95Jz-;ROAW9@?JAKk74@M5xYw}=(&JXozDg;qh z1XYxp)~9u*JYD&u`pv{u2QI{Nf(a+zQaHfRp;-Im;x?KeT3(^ks-0gqesL=jenNp) zDSreXR43xR4sPJ}*hZ8kn))Rb)6o14X&+8xuGuzIgFD&@E)?5OAm;$g$&P{o7F)-@ z5h515Y#FLZFE?2qCz@dB^qCijly3Y3z0%!eeVm9CC$ns{HVaA~iR3&98bm%WXacHcjZKV!=x((0hmuX7`D)?U9(9qgs0adKJ6<$XxlDiD z3H-|PNg^u_jJi_`lXbf;B5x<=HYsZhXw-L|veUGwkKI!QB~lIU9sJf(eS0k^} ze2K(ttC8{96*9+hay*RmlnwMs?)s`QIts$4ixwF10O;=1?%`+pC&t|;*S}qsQjJ$& z)DBx4Sb?D(MnMdO5%EeI-oysqD}iSbTmVUjgZ$v}c|;Nz>xYlAVnl0i-iin;+bhim z3-00O7(d?DDD&au+mZ?z`jS}?$Ktp7d2Z6`AQKULx1 zU}C#ewIvN)0eaA6K^AgkpmD1Y8C+{bm?P0{S*8NIv?s%Yh_P9v)P?6Snrco4g| znXwO(2O}WC7%;N`7))YaKJBjkL2`+Fjn_wtPysn|;7{(o7`;u==#y{Y?M%+ey%&cM zuVc|N>V-d9|G!Gkw}z-$ z2&05o`>p5Nt>OJ!!~3_E@85d4_iyojd!-G|T#oNy8a0VF9^Qc3X$d-TKc#9DxfDHC zuI(bNFPSqL%OLhyxB;(3z!R6>z{#-oVUiTrVY-zNXZd76-)AA8jD#Z08~X-$M+@Z) zE`rR`;4&s6#k^TbgIk(Gx1_<@9pp!ck#2$JjC=_?er9QP?n7? zl*ytk`F*D?Bsz73(3FD2^kVx)vklh7spNv+ZnsFn-S%7ll#+0@N4M(tv;}UY1k^LI zthXL@J9z5~2-cYQkO6iHXsC#{6ux9dY3LD|~q~%zT(BlKeFK z#2uQsI%KB&lsXf3Ggq)hYM=T!74;G?VRf9s`jdp12MI&cBwgpjhMK4{5jH}kFyq{p z`)%6GtK)iBySdnJPav;OI6RgGArjK|WY=zWPT6|lk3`-g`vYfkn77JOEHkwo0uPP{ zpS9_t&Bjm4&P2;6WoM%PNtm9t0@xO9=4lUy2@>@9LT&P?x|kBKDMhMrJS<||L7NLS z6caA>*QZqJ&V~5`>VFvIlyU$ZuMDA&7rq92Y;@1bX3O?tOUAVGl|6w!)^wEMmWxFy zpymZxhS^+yW6EPo+}4aG8Uo^uSk!D$|AD@pSyB!M0GF6{Ald@XaUcd410Ikq0H#uL z%^l9>*4-gCCMXpL~9!Qz_np1Rb3#yY9d5 z3+q%t4QAdop@8O!ueB#p+je}pxN)`JB5$4Pn5km-6knBX?6;_E-nxd^zV_VpXP)t{ z{*#sP={UD3bp)0&-gWQbY{h+6ue{CRM0y~}p>^Gk@@>&yQlB$>pr4YtY${3@?{`uC zy_N`@po7p&pQ6;7dpMLFk~MAbohk9}8kKfN)z5~@l@{So9%mC!Jg-=Za@MdLHx+_Cj1)LDFpg=x%JM7e3O)cc{ zS(M_BI%3~HyPcdmaL>nAza@f(VbUi8gOUbn?R%?-l6Pnz_`pD5qP=PT`Dt@}SbY9C z>+QI_tf5_M0$to1U1Ydv@z%a`Q>J#bHkOc&otA6N2%}|V;AQ}mxMd~BtRn;*f|3<;s$w` zoWqA2S}Vg(p4Zzos~$J2?hzX~aqiTtdfcqKM{Ha+t6XVTjVnCwKIrh>JBHy?GMDP` zbEBI*0~?JS5FoljIO>+vuEBqzIi);ZffnR`HXjXc77v$ zULdrjjgv3h6A*2a`5!d}?V7V7bAO-g(j&NOP5}9{GTWeng?zvhXg=f#E6y_-Qev~3 zX&`<2kWhNL)#U`L$=@7}>xnSo%S4hf7N*^ngxiKWPyC7T@aigIB9mW{E!wGNE&|F) zodZ>WraJiPa!qnhsin(T$D;y=>8O#mCYSo_?F$^Jh&H%Rp+gR9GBx8c^~IaggYJWl zB*KTEX{VZNI6bD1&0SYi$|@nmtVtNKT56x}H??$bQ1b%FU&fJeq}y3wBmy6|yJ(Ln z_(Xv!F(rym&o*mBy5)3;{Wh98wZeXF(H`s0?l(tmj!VYNLVC<`$f`QJGRGw&XV*+e zR}6DJ#mmsoaLM2ekFmhtbhvap!`u|+mqckBzr-IM&#&AU$GS874ePs~U;bl<4iZK2_m%JdE295-B7 zRsI+rf|H(;JJb3cHJ$KZc;iXs1ofpjw`=;zQS@|rSrNOw9%tP2leaW|oSbF1g`%8g z(_?>frRjtN8Eh>y>8AWuE&Q?0@8e$(P@?|c+;6AhlWfr*9nFhB*46vtJ|_M2$PhZZ zoF42SO-}-1YAAy$!kX$Y9wsMnW{pJ5cyeq_*-gXLK%Fx>DTdy)p;P5*r>s9s?C)2K zR{I)%zkkhtyWQOHPby2yZ}LOz-6dwhTKDjxSqou(5hJR$ZC?5NgtPHM*u+kBGfJYj zeY(vzwzK`r--j>8uHb9Fn?KeRem;-xXV%yJs4Q-XC zDEF^(77#Dp+r7h0iRyKlAIk zpFp$liD{dHqf5b|m=X-w0=q$lIg~KyJQ85NG}QpMGia)WT%@VqG0pg{B0!@wW7Aa2 zK~oJO7p8Pf;WE}#@9GsJO|=K{5sG`-9dy@-<|8qTo+?@Dqhm6fyL!#2CukT)Y|>fg z3T4j>3Y^vTxgRQ;HSn|RuVgA)1WcSD_pYPLejfSRx5hsdcVJeTw|JN!;4-@A9cZfp00Z zss7%$I*;FE3NlAzY6;>nka=SHeE+YG2n|!x(x9&D`4q>@im_k!Q@`dX-R`1(U$upw zGPHC@?L3`C*PA0VPGf4iE~OVn^+e76e(k?XqV~0Kq(tw<7r*!q{)c%yDl%&B_rq8J z%{%9@f9=Pel(2N$>~~?Pu9Y^*^TghO2beGezrs{j{F7PGtSNt(-RE zG&wU0D<2=lS?+3a9UsM6`uvw%VpU%3oJDh+Jsi9lf8?7NV6!15^3`l$ndRr*yR6&& zNGSMa;bKeHMPP~nEw29l{jRq1O~2n(JE99TQU)qW(AP-VGo&{IjVPe@fIZ9z5&rVu z&V9vPtA2BMXD*)=*P+DaJs|?<{y%doA;L=_d)*cK!$^jO+-TK^F#vdYVLUit_0RQ( zCN}HuY>Q>Fl)RoN0^>np6-R|Xp?aja*}!mt^#F}iRqxP3jeCtq|>j&j(3|ja2`3*1{>&9LF&KpPCIeo2^&mQh`ILCp;*84&ecnnK2A0 z80kB-O=Jg32HN%cKz5KX$qw|L-be9+!5jBf2f*cjuK_mu$$581%%;|1GG{i(LQ30X ze#0Moa(m!1$m(AsR3cY~LHm#pux|K1bAFU9#vK}#IoJCV?%w5osCIiWc@ z!aOv8P1adS4DQV-g$Iy_DQZybn>BafYiWe!L5+06HRS`V3*q~u0o01z!nHwj9IJx} z$z=y{Xbc?yzaNhrc}NpK{dRf1Cw>as-yGaV{01fT()htsUw->`Pxok+xt}hY2`9Wd zSCjt*3@<<7j7rt90r@t^y$q*)J0YXBLK!~hkIpgr$|vS2G`%?p{XE1T7D8>`P94w8 zuM_r~F^DqRrWz_Bx5zA;ntwxvnthY%GhD61z@)YglpHB5jk|C>YXFy~@Ga}W?ikj= z$}rL~;D?Kag;e@#=!$`#u^5a%$``}@8)e2+lo<>ZmVkXma6B}rw*&Ia%@j`_1-P95 zm}x6a8q5Ao$Lp7yKt0nrR-U)x7-&8=4%l&=zdw_=%^Dj7{nZNH*_cY%< zRRN_F`>BFSYQ$6Q-~7CLidEGQfPMw7RW}8>D9Koog*Y@(U=nI#$aW6s_9p;F1or&S z4Mh7j^^(%7K2NjNA`x(dE8#Be|js@oLM95>iZNUd6 zheHN_0Yc{@n?x`b@bJXG5D(8A;fBG2v;TAX8!d&=)WfGcQgu}AA(FbbxS@BA%9TX! zrqTh|sSr_>OF-ri>w2j`k3%JQWj%~SCF~B(n(L?zp@-H{l}Kec1acA~RqcUPB?>Lg z601o-@_kg8rT(Ok8}KC=m6%#V9x;rS>8Bb_SWI;Aq`tTzsD8}MLeCDX#ZZR$D-6tD zZ1uEe9ZTAmHaa>0X1;uCB(#>}GfpwPEA4s|E@R0NKh}XVc`%Cow+m86eG8h=7iKzO(oleT<9!lQE0P~d3O*zk168xFM6$3g_X$AY6 ziu8bhWiCRZAE2#ROG}^Pm*6?S7EiH&3k)j7ba(WUmRnK)vUrIB>m#U|qm z%6eFu<437RRF87Xgb$BW&WB)VQWa)}wnX$c0D1XzApsWpTv`Rv+rqFZogm2fn8hOG zrVPbP<&~-!FVw;ir!yUtm;#mBtP(U(GYWCTnumR?fi44bD+b~6U>}DL`Z!!@{niR# z7FmD{IJq{=ucEN|(p04YhL(}*g@HrM%!p0;46$YAS7HS)YF*A|T*(ky_TZKR7`vly zVUw&hF(U+%VrW(xcGiAY{}zZ$h`m{>!(3@QDXATtuvqHHRN0k=6wr97m1-I?9x57^ zg(8uNBQir1GDCR~!Y*Zt#r!x&sTZ0nL5!ferb28(b4^WiO;HTf?HezF2g&8agA|C5 z9(@r_70oq9b8RxfHQ}Z+AV@91rvwBk-BpdJvVHK9jT6l8>X08XyW`~SaUputW3t2g z5@95FC4vH{fVJ)F&ysp&yZX20Bg`}5@%2j|L4gFJOxH$SwisngNI}tPX`iVm3qxeQ z?l6tVYtBq#_?omEmL@Ooc1Rnow-vK^X|t_Hesy)f)>d~zs-=eKJ(hzYMF|otz-UVl zoQBFHY|D5N0wZjVmux07c6qV8yVm*yV>pCfdL+SB(7Tl6BCZk8!DNtA%c8JBM6r>S zqL2c0uCOe7iaR-hC02zh2O?@B?+>`aL2lYK2DG&@068$3ZB8wGqUygT@S2Jv6Q3eD z5HR_|&xp_gry!NURs=s%<*}jzq+uC^kqjOZ^q55zIGSh;+i1ApP)|EBa1Jk-cnZ$prbX#if*3 ziTU6WT&|~)`y}byaOELkdM8oeCE<>t_1TIrC9FY=;T%6qFO$5n3?{XHup?%wTMdvp z*<*l!PIQa`U@&KEg!64IjmCu`cZE_jE({q7BS94{Sy6>$XEJD}RN3%q`7<~KR5_tT zAmI?8u74>a??t05rSw<>NmO#jLxugiyJ!(+<^Ulv(5L%t!m$!`Xk%UWN%zP0&%1LW z?L#c6M9;$Mk_@LyVy8>O>5^>UAUouy=sDNgIew8fKlkhMi|BNj*Y61m5);koG9OMC zkhdox$u^uUVCRhe4djXT01?FM3@5-xH?7jMmjQ9L7po3*u5?f!@3nvD*8_4*$kQ=h zUa~3nsuc|gW#svwSp-ML2$O7ZR4g?HcezRSoJBVSBw08I@F0)M*vF2N7TEHn z9!VRL$rV7b&6{T??Roc9IK)-siy!3HHL1g~p<%8>x7?u z@a`_Y1lod7nBpK!t37`e-I_$l#7~q7P+kQ_7hlMf(gZm|+VZk;u1h(GWhcw1&M~U- z>y|+Yq5^9PDTP6xN<;)1xg^MZ-4pw%(hAn!3qK4Aqu{(h*G{ferDm{Js*~)okjM}> z7x;9~y#))Cpt1m-?pVNmSIN{XPMMb)+B*~xFysJd0k$^;X(86#69os$ zz^8&?VK14t$sB}WSgOW)by!38_VNrK`y%g3F&OzGH6xAS1vT|yo(v;k`HEL_%*zh0 zd{&*1$KiDHHVX43=ifn&{?}udi?>6Lj^%f(T{*;Ntz*Zc9JQrSZ(M!5L9ifY%j=8t z4tU21qI*L!w4!SGI(j5k0i3kCkE~|IGyr*4SEcR5=Qo=v*tubWxcdj zG70r=<)fhMQ~#j-u|r4*+zX#lP!eJLA}W6-uK$-xqS7J{0K|xmw5%qPWt`2bAIsd- z3XdMoO5ZN`~=R)7x7Y24y)2)B>gm9~HX+-pY~M#pe4m%KP{b9o3wu z3+&|cc1HK-#7ES9P*(j6vTY5$u;n8^3+(5qdc~ezZ2+}5D&np`K$oM_iGL_583(FG z);QKFMuKUvnF`b-5v&lI!|sX|NE^mxTRo**I7;fo9rzu>2@cVj_;JB)SXL2g&qXV4 zI78K=oaS8sy@-7Y=b!@Mcoc0+_)Swv5lk%ZfSFiG=rvU`fSK@lJED--(Sfp4nFDIs zJL*AIyJ^1FptHcYVA_gR?2>N1RB4c$hjbQqS*0!%ff5i+M$kc)KfF8|R}~k^jl)ft zZT;z3EWsURz3Raj8U_y%$g3ES0EpC27sVWMGya5g#S(39Sb?hh)e3wI!>F|Vun{Rg zvjD_s6%1=yiwbX0zZqt1MWK~3E~ZSj(8UQ z)VnUFLj95G4f;dn>bS_K%E~jHxc?Va3_SW$okZ#>rtboOdu2R*(5`&PEGbE#=$i(LK8T|^Sqq;)(Kpn-Za_eDB)Q->+M1!<@U=f8nJ8M( z+KvQgJ@J0(YlI@S2P@2cSr;`@d2qN)(3UbkWwaXdhEVC>Y_>vble8l2+KB;kLeWhyL%zJG*@I{*3KRy=b|ewb+J#kCF5XP%Ie>no_A*) z$i_18b1Li@M-{(Nm=jsvLxQV!g2Oae>Lz^2MtO>VRsw`Eg30-Nd)zww-}Z(&nSM(n0wGVnlOEn|fIEjCqB z`Fl3iq-RrAe*YmhRY8D>(r(v9-!accZpVkSyy|A-*O4C{woDgQ4Szr#gyfzHF9ajL z1BbPn+t_BX9TJGkx|BA|n~Be?m@Dnl{bt=~J?ml8e<{Sh^+j8Ci88AuU2^5!e)8^? zUwqX$J{^A`1lG|9;cGcrQx;qTm2#pJQTc!>bC35M*nhZscmtX?#RV7-E|+1Z}Bm0 z_{p6G%7^1#qF{F>=lMV)<)b1n{YYFjGoN_Mbp1rREmL0e5z^w$yN{^GqZi#KUJ@IM z)P9tH3KpiXB%Q^~0lu!E0)N>v^iI3?#S1={@#rijYeQ%G&0^(6KTY~5u1^%Y&_x8V z#lbs;n?a#mlDU~Vejsy}A>FA$x>rnK{`L60yHfeNg>Yk`|M)o)v6+SL%Nf7 zlAF0?;xk~76hW4dEwsy0xicBH`unwpa3yod^U1%OB2D4-WR*X>5B4#uyqL3> z(UIVo_A^r*buQ8SY5lb+CnG3pc1~8GiJcP&slh7%^EnY_l_E3&ULi84_7Kx5@bLgN zP_#KmjM5)<#7Mcx>-Si)lK@ZvpvWI}LTpIQ$CEC=rv_yq0Bb}(9)LB-+85eKl0idRY#=1nq;utGn&x0AE7d-%Irp=z9T_L_S*d5pyKml{N}M6{!b_k@Xq% zFcpY{08_!QqZe3&AobCW`sXtXuL`&_TYVFQSNdH0SHg&h$4XmdJXXsY_&4b&7+Cam z&*WPLU0aUCe$|TtNl|1hXqhxU+TEw3MN1m!DjANcUIR)Und&PXHOWFf!*|v*!pu?B zFSKg|M~&>dxnFxrE4b%}$q7>kSLvpABC9F4TT#X&kk!bPxCjp>2~5#pO)`+x z0F4?CWHl3?Hc3xblifmA1CC^yK~|HPmF&mEzDg<7cacSs^<*_eNI)LTi)}uz)+C{C z##%Gr8H@Kk=y5ED*pF{n0zN0*$8|1|1R~R+{uiW+N|bS+RfCcDwajJD7Xe>a1@=Lb z(O_UHfy%$x8f71<=b)7WiqL;of+n zm9+MIurbK-Y)C%HefeCQ2KniHI>^tom!Hn3*NL5){B(plSLLVkseqBD>YtagcK|a$ zQ^?S!nbE?B-Xd_U!#=-idY>9IbqD1AE*mqwU1O#+X1#CB)UO|I%(QQed!EjIr*{_6 z!v)Vld?2WBEkmNnTeSIb^PI-gXIHDHAU}g3to$$sO(l=8g;cn)TK21wNpm{kM#|q| zHn5;9@T*i9Of3hV>1&-~c#%MkMZ1PI9h1pwejl+Sh&-4G`qCdc&dj(g@9row!(_P* zHdSKoMAe_(a8<%7b7#r^JRY7Ii zDJ>KpN)m*f{;;!p3r%+bU~FV@9+Ww5qJ0sKNM(t_)?JQ zRNlxJ5B0em=}PvoN)Xym)*q~ht^lCy3-D#B$|>+L0xhG0PT2a`yz2s&`5rW4N0VW}OH z4VR?cI@$NbaAmduGWPeO;S$B8ZGailfDM3S#1;!`!LB>YeO!b&!vorNG4KKxwQTcx z{e2{VgUTh09}H-p^(`)B;jyPrg-7f~Zy_KE5M9{IFE!tjmzZxVJlVk&V~o2`9G*;S z)mppN3kmm^X@0Tmj48@BXwcYu-xH-@3el|CJy zf>LL1*XJ(N>k{@+lfSNgOQ^bC!~s*QBVK=N=i&E7d)w^$fz2kq&x%wjk_W zu;x#(!j)GqQJuFYtZ{F!!gV^<-^Fi;*8byRI&KlI(u@8uqIJ@X)`>Qv9NF+;iw>Bc zt4BBh`DfDvgZeZ43B@d}EcocGts}zH42#4|{n$L-S_+WyT0&a25+K8ouqr@Sj)Xf4 z5L;1lI!RHvbEx;O;-4#_-oyC9n*DqE0*2UvfrEaNSoUiVj@x@|iteKY>c}TuYzg7# zfRleS!7L(w9^yZe%P}z|D%ln0*cd1~Z$pf!F+1n81vV70|A#m>^| zcpj2@SA_?e)|Phe7Ou^9uzZXo)$Eo51zl{L!RUlzlT8uaDwaeOET>?$sP4jS7VL16 z+4iFPlQ>^uxU-eRXf{)CF=6r1VmLah>hDe6YzVE+2(1Q2D|2aVhSYKMJ0(cPatww* z{!QGZeyy2{?||5(C`IT3k{K_iSiFIG79}k1R3J{)-UN=hiH?K($%iVB|yUI?&!*AyoYL0;Tx0dhC*AT&{kN6}yMz~q*Z zoiqbMnY=km#8d{I>*6b%u5P=Hy3J?EE<@XVavLl~^&%nAgWFN~emqSmklab|-GgSf z`OE@FH%{6Jrf_0nv=I+@j-^ZuWekt{OheCOj>(f`Dk#=#(VcoGKz<28l2u`$WinHu z^X`}hS0lsxN_gEdi_9u=Ln8o2v;=1h7!webS5&h~6h<*Q z8v>wp0km6zQ(*!SR>ago^P4`i5Sc@WS*72u$A=X8bqsXzz@CyByl%OS>mcLwno^`( zM)B>4%Z6Nr`J--E9&W>4o*5Q%>Ei`y##$=?{kCy;76XX639H1MtH3-;>9Vo{<%+U= zuNV*`6Szal1m@0xFO3tNU~~yI0ogx_HxYeuW2liiRu6IiP5|ISzhNRt3(C6feMsh{ zCvEP-P((j*{j}5VX0AH2snJDs$#Cgyu7h*P4VR+EDrJnG)Zwz+;|C)E ziDDo;hnL2@6u)w;m(E8Wk<#X8EQ&cQrdF5j92wtuMMS6I<6+H+?r>iDid-;GY3=3k ze*9Nh%UZTnI?g* z1noyLLGhp2$!>~PyzAcBWmno11_kfaeGMB~z;9#{&%2968shUB*7%X=5&MgJ4MHJ^ zxP*iq(#%4>a?VyKa*($LCNGEr7CBJXKtXU?hhqX3V(g1Y4c<5?9D1?{B462)XL>SB z2Z5YbPd;V~#~dqBcfo+pqFr}E+mqjVJeSY>O71gZ_bg)ys)dmnz=huRXfDK_52Lxj z2}1mESX?XhrmwsPGw2>p_(cG?@Rrg&8q_`R#IuhjL?FH4fBT$qQ}c1 zdR)=r(QD^XPsNpY5N@$Sl%cRD7a6=goXFE4>e`~$j4fbs%us|2%-`6%`dw&?vhY)1 zkN}4P7U8=f86)9D5dtkK#Q4?Zkx)K~p%Ss4b+l#dAO2J|Cjo5!L4LP0)U0DInR4@s z+x5HJb$;9T>)X9oF9-~&efZjX*HB0<(C4^TF&>Q(}0lG;`V*%L4A`~MlKchCQ|6X-$WOqFn zZOT=9@NWFhF$9d=i(>Ls6=R4>B0qte1I2>R+pJ5vv;)TQ+O=!f;x}#EZ}FS9+XAmp zya<6uwRpDh5uo~8tU1>d{WIL{E`_GK_jj%YJ}u{!^bjprMq=N?tcQN^`gk)`AK58a zSJlr|N{<(H2f(xU6d9ni^K`7TqzT z?QuBN((lkMK`)*XW@p<^+X;u|l$;SM#K?U=!F8x3&GL@@k9qj5NJ1doISxE}V44;erZz0$r z?Q;s{$(>fIb+WKd7Kb|tb_+S1?6Ke+)9Krwlg+OA!=TMKV5KA3w}(7x@P@ zrmFrz7+PM-_ z`pk`!K7G46Cpjs7T7Pm7^14?}tcTn9v6px_(u$6v_?@Btm-}LMI?UAltw6T&Dk0K9`beF_qVSW%- z`J-LrQDcJ?jr>tcDi;2TW&k+Yp-n0cZSO;Pa-jTvv}lj*I>2gu+UmcG; zbZ0|_wsdD>)E)cO=`(qyMum=W*jl`aGF0vo1;lEh?JeC25vsmBj)c!~cVH~G#%r3Y z;57H?^}v1P{Q5=V>8|;8u{}vq zp_PnlDS%FpgHDixPJpC-F~J`{<@`wTLFYtv;Wi5JmMmJ}D|?6*8^yXG z#pPtzmu-5*nXoc~BhHF(sj?OWx%R^A+*%ntBsDhT6kXY$k2bNYXkO$`>W(h0e{hU@ z{c2FHd`c<1 z#xqnxQ6ZM)<651-FS0_9+t`X_*}lp2nc~R(3eA8#3^|NqS>~>kn^~dDjX8#O5J;i| z-()`Q7DkDtlJ;|dIb%8 z;WzfdcTi||;)@~PTlDc>GJ%8Az^{kmy%S$3-V1*uqg(k5HMs!dBlG1#x#SYjd`+^z z{(_E_9Yi8rpbX9p2q#m!u%t@!iqA%rd(M{#>k{y?6mP?|^7RqMdG8u1fea}09#j4* zKN%xFOqRt9O@Zencwc;R2P)9X7~qUWT>9+JEBLM+xe_Z~7nxen8T-?v`&)P-%nR>0 z17&hJzYd6ZBv(By%FG`PRpJ4EiPU*5Fd#fQDO4zKaJQ2Hc01bnlM!OC3dIfXHbC7@ zhE=?}`+LM*@NIg;-m(15s}6QD$)nP{$G4j`vHrE0KN9H6R*zg@Y}X#6drw>?Yy9-`wOyN;r2uD)L9yC4E zWS*F_jae#v)sZ>9i0crtvs&mh@!}6S1v(+Ndeyx@jK6{ZM7B_4?&kn7e8Jp<3iKJ1vWi`jFsFuN_Qvy-dp90z}rOr8SO}Y=;MNp_nBcICL9 zKGln1Fo>(FOKq!aBlz?>%O33_{YAfZwN25bl*b{srrT{2zDYd=Sr)6V2^E7O40A5E zFl&^#h9lpUCeSxvcnV6Xj(qzj?$xO!`b%u+8;X$Y$X`*ZmJCm;=r286&hp4tb<&`e z$_pq)&KS)!F(ydj&kJO&9e6B=IY;Jv<;YZB4&UW|zNmi?4GmB12@PTWXGwIcKW6xJ zN`29~(~jBn&W_@lU&n#A->&Vq>%3uo^$jL@Fk#l^2qGJx_lhS79Vwp0 zma=b1Xpnj|AC4Y(xtaGWz7*6Ip|dR5o{tsOd5>i>D5-~3{JF~IIaS!SkeHeVJzF&7 zzQth16M=hjd%tylf=*(J(cEW)VwClY(VSn+o5HV$4xG8qEDJCGMa)93f#hyJEhdiwGv%#n>@@RodIy*~Rpri$$bc+ld?b z1Wb=3V0y&C4jOP=2;J1$oAnR&&a~mZ02tz4bpfgGje_Wdfi>aqhPE}kLYhrB0Fo+K z>tY)&0tW=)^hULt5|V_f^h`90iWw6+&J3fQ#|Fike3^5IT3poN0tQkPPEKST>uiZofYBK>`MU@ks;$c?w| z=P%bXx$(qYyxKAx;do?ImDb^c#2=daeArXixc!Iyo`T`v(4II9^j<502Nmcf6*)9ywlb+pW4U7d+P^L0O+zeY*=^ zlcJUYhM=gm79x2=D!J2C8c~c-yJHqqpj@bQ&+{FT7}2$t&!H*61MK(zKkD8#+ODg- z?_F!Iwf5WE`@X|ObG$gkJoZw(WY(hzzkc0*iNFbMhO-v|hAlMCgp{2wH!b@Y}JAVIX zuD#bjI|RD`E-6xIjrU0@0U#_D9<`v{t)_bi$7!;SI_14g-RXXuv{5zY2M*leMQr>C!eXaLoL7b zP|N2tRgKdoIUlni&K?JrDr2hVeixRSUA`x-t67lHSMtVdIg~UDM&#Kk>A-qtp;>t8 zZO=`Ja5Ef1dM&;={(Lty44efzZlUsFG zIe8;+dUa<`uUI(_Ldk^DO_IrNm0|1>o^ZCh_@sM|Yv(vFEbtKor#aL0Tkt8ypon$lAuhb~3~rB_MAnL-ZM0wmFWQ@e)fKFuN2aF;$7? zE;g#mvS2^eSj)2f>(UK2JOA2ne#oO>T}AJy$Od*x0!#KUWPBFt%FxMd0nz;#ao@s6 zm?Z%Gd*7_7{m?z5V61+Cpou>lwF|2P#)OZ+4dq?PUP^mQF{@-}Y8SGze9Q4BuU!;UA(1U*>czL z;#%6S=hBw%%UixzTfVPs`98e3=C+i#@O^p9^=ixal`Y?g7uT{a^Ww@IT^#x(N!tHid{+wQ944Pa zdANLu@yYkYHy(fd@$MNR*}HU`b{K~V)@H5?kIp}Se*W>(^N+KHQ!X5jliYe-*QKXVVhf<_h15zjUv8- z)Qnql=C=3m9VHWJ?cc<`WFhedh*(i58G6(^Dfe6$By8sDRdW0QnV-1*wx>td9upAq zsE$bTosnf(>L7w~6}C?p2hWMVJOwLM+_*L_%0Qlw z!uM{h^EGj+6+T}B5(plyi|6phI$Nu4lU7UsCo1DD$MOEU5Uv8U^tG`hLv?t{><8I( zU@F)fJj@J4j4NRH5fVY3Hz{p|RHY%tO@|~snaYZW?G4o+jpKxQEsm2fZ!+?^L!*p1 zN@WXS$%dnvO5zHRQoVRGkAl>Ob08c&Z7M)GJgxh{Y-pFC*2BxN7$6DPqsv{q%#|9w~7;JgX5jfi%X2@qM+dvV{H1o zm>qFHW&}7X>y9t9GRsGHi+IAeI#3x%&=dCpI7twVX7*ChRrAn*yrI zwBN@eW|KvrZn+41I?ZM=ib!}U-{-%Toe3m#M-SIGBCB3{+imL!9v5P?Zc!#9K^5}w z`0eL=>q%o|$b4={-u)uT!p@g@H?#A(#?D8`W3-*~af_WV^Umyips-o|vC!2kRVcKs z;Q|`^CN1jKOEbQjrI`V5sWIR=ztl3|vEsMX*ZHLw^NxoSM6)Dmln_7A_RzB}22{J` z^un7#STeQg%QNij5;kDCgD**)kk4ZcNB6P>fGe?u`)p=Z&1RjQHJweqhuGPC+*0}6 zXN{vmR%)~R+HCRSMeDJn!6wqGzG~i(b2O;k%h%kg=1MeaHOEvfx}c`!Zo8U;fs8fh zRb#Q_nQ^$wb41ENe{)+{Bt zVL+JPX64>y_UH52&&Msz0$Hf(1Qadww+=|A2$sA_-T>Z(S(NQ~7G>McB3SPluynQ^ z<{swx*^<44_aSR|AGWJ>hJDJyZf5iU+kFBxdmT`)gIP6Rp4I$GRxW1o*2*=r_~ln_ z!;Ocu~h?e7W4^1iRtc_r-C`F0_o?+(Xi90|%&|svD zurI5Z(dV!d48mrEE~J)GXmL@w?3miufoGZXN?PW4D20#+?@LvLaRsP`t!x+`Os8YF z-7UToX_@Ph^#J++q;MPY!=p*_=^d$ksxE#!o&9fIc^4k0bOCrHsSWPVp&v;FZY(7# zk7xDBQJxU}i$!`?7r!Ho%-QfR`DK1dPJA9#_sx52cQdwuo6ev|LPyZ^`P#TQiv}&N zp9}ILkc1Aa4eJP`*d3fA0_chhht#qnjGO73u&TnAP(40Sr0LLLd?c+R%Yev~`4-6X zkcIT*8f2upqH>LJG8HFLOfKM;%Fp}*x69xgkGhRIw=||G>T(&MX3|Q3_tWvRC#qfA ze!oF(93^=dwx^<@B#?x(%S)#dinp{Pxfg_~qneHjW!)W4?>TB?JBloPMU^ zO2tDLJTx6gN6sPRz@wMyUgRtb4=$5}ZbY0&XsCU1GqHuo6wEAQr(IFZCaw%B>(kWP z3?lR-kc~K5K|^-@&@kyk3>CwMHDlxpnX;nRSr5Yl%aYu9pC4lAe(dqbA5YIr>&jPB zP5tofZ77MpXK)U*)2H?rHVp_=J0M!LRHIp=!^AcHw=Tj1%g>Wthnf`anlyGOU&NYN z5VPN9vr%;)_gK}U_E6FV8c0?~0;Ne6vU!pq-%VpZh_*IT3n)g~7i0}m_#&V{R1La3 zq#J-^2R^O76{C_A{*?&R4cou@L2NKEbkvf#WfWnLoLew@Hz>D&oDv8s+CpX0xjYNx zVQgg?!|E@`$@m0RkweiU%2c7H$|Uz4=BXL&NVP5#M3=U%>Wg^p5=FuuJzb~MdSs66 z5g~c3Ik+rs?u{$)_Y6rSJ3a3CrI4N)Uqy_5 zNCb9dI`oJg1Gn0nD00jQje8L_D!?h384YP#$8^!k^xW~gd;#A)uqYrr6L-Du-!xK! z-ctFiIP5IEKlMw5DlDNKF0I{1EZF_y)$F+6Q||@Zw8!uDtMUc!@_QRc@ZNjJOWuul zm3|j7ux7|IQ^U-DQLwA_h|Q@v4mFKv59oQY^D=g@oviqh82lAd+-s~;->JSiXsFW< zDVqg7MaWjGFJcVcEMDzaZaTpKtxj*&21sdg80Xg}!8&5M&!3&HV#+&Jcgp?Ve?ri+-i7nOL*^V*)zxVGNd4-}_OaU7K2pEhvVFjqv_+Ip z{i-w&_0GCmWi96iv!mmEPo*8#EVkpCOWHvOP?1zUK|B1um)Cn{joCA6%$|8;_B>f* z_AEALW;Y|2J9%yjY;E|AC^%!W|3VL6-4 z?uDlJ@Gszj216zLY22@{2KU6$y&`FG4DXh$jPcbAm2*#4Irj`JhnVFy{qB&S^;aV+ zm`TKW!Y@Lc%@O#ALt|#?Nw-ci=TkF@v}c)Z2+TM8#J;$q1s8s$$r8o}AxS_Yj0Q&P z`z37`=Xz^kp0w>fVwQ7Gl*NIO<7eA}Y0V{B^H0~k$iCN-<>HQkyj1t9i^`aInqZB% zx3k+V!PLhtH#7rR5FnB~9g<6mN=wNmjk*$Ym01H3VZ1@=tX35LXU6&mO^AfyN zjsj@EBvC$`NMByPF_J@Cj3I&mwfk1yi)K&r^%d@T!!PY(`H6IEhrgbH%M4NTArQCVs1b8Hi0Qkf4c? zi*>Ej5V?m69ZC=orD27mBixk|ahdu(k$@--`Eygw=shAJqN&vzihcSZ7fd|vB)dSm ze=vJZdLsD&cy_@I6`Z?;PpeO*keMS5AE5u9ll){Vy3B0i({RVPvgf96!7nn&JpQ{=xph|{y)VqT2wuXEKl!Nt0V{Ex$HCy`^5R;4z&d6F~n&+qNN5*tI6g? zM0a(>CFkl@0zcs0CvcNC7a<4GRf=MeSVXNlMS9L>Q!ACx&HzB=95HWg?fTJT1Wu%& zcD^8ZbcTJ~1Kx1E!%fjoh&|o>I8u)wbJ2dCnhQKbeCpb%KQD0Mpp0O;ZgbkFD$3Xp zk|d}XW1(HfgRrL#k{vm~1W>dk?E9gATRjkf&$2trU+4$hx4@@*>wtZb29!+9=@`E;8bsBMXCANUPJ+;i61CtkH3s~db z%d}vi76>?NEhq?ET`{?{eX#V$U5ahM$RnFyCYg^@Pu8QC|{b zt4ir*s#K^-eFQhFk|JpbuqCe`4clg>{ebNpS<^gr!pxSw&vw&x;WTx2zKcW>tE0nr z$JI&aW1RZ#G{6ciY()D-z8FW@_0ytQkgm+`(>1B5(~utYt@j2BnQpF~$Crz5hEoyd10HTLIowtwlIHL0h4Xyb=-s8{ zE|q?X(4}2%5Yn!v8WOc@zQLDXU>b|7;2qZ^Vj%xN>Y$W_QdaGx3y0os@F#osGta#56wlrPYyBRB;!VL2M|VtZ~o38^(q?Vikb0zPi_q>g4Y z1}A0ph9Z1kBq=5|SFh_M%<`oXb>ub@yvemi9XAtoG81*+4SSlR4s3AVHKLC51)>fB zc90Fy!pNZNJLUONmT#u#nt@OzC#@={V5e`PdY zo#xZY5pqkuO$opAnL?#K{SrR$=rb+h!_T;c-+0C){LwQm;Tv!5J;R=S%_Hre9Klif z*{+$&T=n%HGXC>NL^TScH{7@FP@fvDMv68BB}8?$#WW^qi@Lgq{FS#UL)j@u`dWW`$ zm*_9j&2ZhF#=bXky><7J;htF&uAWsjfdt9g%YkA|^hp@3yO#~G;*xcM38>AlY8Z<@ z`Mx!7+_g|%bIz(4iLHJ`)7}GUL@=>SD+@h+{ok^r?p^=5{`uSo|K%w6vv2?OMI+~X zk3~brt~1Jy99~i>rpwko%oqDGS09SCV2Ayv zd-Kcnp)d37PaB6J^W`oqKiB{jXYp6OyeFE`dtrTbV?8t6EIy>blL| z(^Ad75Kcp~RTi_H<>kxk!M$XszV8Rt4Exrr^RW4^1Sw9ZMD69loms=>l;1}*;6O3n zzPxv8ZgDIG9P|*AuM83#xN|wN4fTG?p*j)=y1oO!QA6IDM2dYg2>`Gry+90l!%tx|?Vgs+?>YrjJ}U{8oV{@?2NF%p;e=6}%ABa_P&Xl_=BSy2JTA}Ehmjhz3211$Us z1X{T4iXXE4tV&?=D9b2$C4l4hcanA{;+*z{blH%?9iZRHF+=*-r~*XVA*6a;2Dy3M zyA%FL>S7i@UL3O&sYTeKi*X$=@aOSiz~j87nc7$O|gV&L&2~Y-H|RN;V+cjz_G!y-e-Z9 z1e5)I8s3?5b4+Kvw(bttzEJ%^r(1rz0cNRu>NplO8PHK}cU}wzw2q za2CFmo(bto%UT{ebvM&t>O07m6^AEUTKUG#tb^d`w-d)^M=y|#klI2z@OXM=j0xB3 z4bz_biDq|n5qPc+!erLy;lXZQ~t!q$L$y!jfoD%tkO@7(pEf#k6d~?=LfwUFQ0ZMp9y|t%omo`i%g=VNX^nBZW}-dZB>8(gvnFA z85cw#Jj)gZK9cz+05iju)sWk2nD^zv7BH7EFIf$-FM+MG8kAwPwHgc>(<6|&Py`mk zpTm#?(oqXVTb#B|G&oYID<=X!vrz1(25_fi+vePSg$OY(dLG;UMkffuwv7_Pi|0Pra5s&sDQK%z{IKj zL05t1us@|}|ET|w= zuE2k~}?}!7mCH<=g1m5C_)25p@j_DkqbiUw|&V3qv(z*GhbDvsL z9L?~MR&VN?1dZ6$1vNNvFM>RXC>fAq)mu`?HtYl=OVPRx8&|!_;s(8|?np6|MJG2q z4`n08Lja-^SzbfTL5-Wc@GiQ!GlVU2`#H1yy6EP1xUwhS&D`8>?&j7trDR5KvkWnf z?YHpxg6&tMs&?+f+N+h=8RCiwKNZ&in)6DQTuDGqG=x0kC%0F!w8@s zMJq{)+TNX&(G3STmhz<2Nl63ii;cRgCaC8C#?uUmV zcWo^FA6Zcpd65@S@=rRuj+ZA5UrIKX$|P_8Nl~Mud4@0Pbv_NBOV5Ol!@L^)iZt+y zR&BrjfqwmB{ME%@e^1n{$+bx~`{CpW`gGwp)0yjGjahqbl7{a(Mer{fY|_RNLAzWy zyn&z<24b_OmB-W0<^DzZb141M{qxvv~QdJ8y2O6uY@Link zX>e@_!U9>`8O&YpL3- zT!p-kq=B=G_MpTrPPbEQ%(<+5Pd+nU-D;BXjmB04%ziT@5;=~C?53Hp2t6QYpXDrd z!bXcoCL`6sZu3za(H)tniB@({8aA{^=e#psA>SqX1Tq+GILa{24h4S{OC&Q+6w8g; zI|n7~iS%6E_xXL6XF(3M`6-nGn6aH+RsY$qbX579%O+9M@fxSYoBAYYV3&7s`zJ83 z+5Vdgh(sCNf9^B3f3|g>+4i3vakomgBi&r>g0nJ<`+e_3`szF3e&KX-;gLiti9x26 z_QCh!5#>0dUWh<@U3QBc9mj!9S$mFuTb(G!n-fJHers8NC3E@Ay<14QaL{rhTzoPy zn+BCWw~=1L zdCS^PY_oD0;8RotHje3iX05vv^VZ#pdFvuyGzFy1TCBT_Kk??{0vDOKwste=u2 z{(P>IZILIB%|`r~IqDZA%5UBiCCY_CH<+V*`A<0bxWE=+j_o1ek-57y@N(p4Kxy6BRewk zP!7xo8VSz^8g}+i!MLt~p#w;4f=FVu%_ey7BSbQv%Z|E+a3|SO28)*L*j=qjvY zX8|Y+M2BsyKdm#!;b;7~rO)mk53}RGw9IpW4u?i4C=W$kdy19p-wTlqW=sV;5a8Y< z$jY`$kd=`jtJgFr^S!M&uPi3Wg3%d{BJsUNf+pcsu!Psi2_@{}_-7AB9c7BbN8I*o zwQ)N;l7U;eozfTOk9X36h)viWW!6R>dL)6WtqYNd*=TP=SgTzad|yaH>fPD!rv8~V z<*q*0))^6KQ7~j|O`H>D za*7#?42`Ds@ETH~_+sHTMe~}%UUP$YH~2h!GdXoz)YFUv`s658%6J?cPu%q9jnY}u z&Hga2sm@(=GQ${AwwsOzIFQs@O%IMWn{ z^+=ba(SdP9`l;I(J4OjA5yq(Fb4w#I&=^@0G1&UH(=3{qs10+r|I-g>~t0eLw)Kt^0e99^(Ly4 zm_+`n4aQi@yXj+67R+2YR&qukyO8=U_A!wHmR=QSYfAr`GdeF{*-86^S-dH#20 z-ytklyM`s2N-;1+vpR{7F2CWh8W%PkTa`Q8a7ZegZHx!44X4|hhfjOM89d2`BSV7N z_1JRAwnxTwy}75YZYSFg8%}Sw0-kiUdGd{hMbK;$T!HEuqaU;XpxSUjX?RVhIH%zLYq4t-u>=A$b-EL^5m-D>&yG< zz1zx@*H||4p1;QLeYsz~M5ZiTGUYDQOvra6nX(0wp2?KNV`tJ|NY>r!^Tlpm$n)N9 z`c;93e7Pjg`=veGeY!o{ZMBgIt|hQ+zrsDckmr3c?^^O25l|Q&6V74~HrL$? z2-IJ95fY@6F$=}%x)kfDR3>rzkl@PF+jc|ZOJ&&{FqRd6gp?p-Mt4oYjtIHKN}wvp zvEj&6c_kP>dn~uWjqS!Y^y+_~r zL3MX&C3k%fbGn2F67^Xa?3M?4S1+1RsRDaw&v$3&;~~~_I1Bn#t4{d+UoCvU`Y2r% z(&mMVr)^;3!{*Z|8cWj^0u%AG^lvNJR^uufNsqcsL`Lue?5DO(vhHg#Qj=Oe%45RmB82<68 zdWeUgUhPlZc@YE(Q`Y4v?aS(}na`kyP0?P~_5E}e2EplzA8M3ZGE3#4iA1SpVr;*iEoiGDttNN{$aS%uJ=LeFLqdfejZVu^mSd?&_?5{_otJ^lM$J)H7v!n8_ zp()f+`PdfX)2Db;wgeC&x8JRmtsI>XKMM8CN|My0hWK=J1<%Jl$RH+p>?vtw>wTlE zA!!*VN->1S^6a%mK=}aGrv^$rRt-y z=%L0B!Jd+(Lpxc`VNm+z?_2YOaD1aKx!zmJ>8pVsf=C{&yX!ew9RA@neDZyuEa~BqT@y)%gshtv?Rkm;V?_a?u0+TtLD~A zYOp-&@!+;0F`uU0-@Qho|@A0@+#p;bp3L0 zbk>*b5OGtcxzp(q&$Q)`VWjL`n`HX=lYsleSMC06id?>|T%FvK>z3cvEq`ja6V2dtKuyy@s^cKOIeAx?6PGv^u4faN`B^M zdHEq?YdIbwnXTM_Y*O7Y%4RZ|wH-N?E8+EDFQ)ZMc*EC=lOv&NM)aSv+v+{xBY!;GNeCYt_-^>$Pp-QII*28Bygp~|UkU$)m!-^8 zy)3J3x=0T5E8)lN@}I`bdAuws&##2L?DCi5QRId8H2pRhYVrvgmY z;|@ejzY_kfUH!6Mwf(`Km%`##!W-?5zl;^=$IBA^{YrR~UH*E!JcyV7b1eVg+2t?9 z%VoSQ$=$DnpR~)LkC)jeP-#9CpMJA0`(7AHL?u)9sPv+>Cc zxWoyBs}G>?QUybzl>K3&?4zUlu3z%QsLPX#Z5Va=F0zJH_(QX8szMz54O4dB_Zf}r z7ZbmySrOgvpX|rKit@ZoKS8q4Vt&)E{*S2R*wv3U(*FDHs%bz)ygHNTFWA+u%yk@l z{GsOY57-T;NO73$hMAW1TXuuAq<9jjw`W??2kiz^H~R7MnY!^IUG=>n&kS_t91vi^ zS_))29=gr&d1O2dVpJQg$aJyDDm?O#RHXmnVR?S07pE(hoGP{=8jn zRN^g7{!_d9MLpYeY23cL;m>rn(HXaGQRpNfz#REBK^P7fvfOGuLD{gKJz-x zkeLo_H_de5&)U^S2ezxTF8*V?+UUS`bv6S&XIDQT8)H{vFL%Skc9m|5`+(gr>#0ZV z26{Qp54&Ne5dPOz2{u2rl<+6I!NV(uN>MTcls~zd0jB`oZNF)Pzzkn$ZJG$am$_FP z?9VOSJKJCnFKn=*(ks8nJ1}9nb$1VcU-On?-Q7(*Uu8S-zvtIiJyfi_H|OEwe_qJw z^YK3~*4`Bfr@jK1K$PDKmNxq(o!0SZlaBqs zudKVH>LEQ`(&?{mb!30ukw+FflFXP$a-Yn6-!HGb2a)LR`HPP4hX)&p4kd0dabElE zH{UWp|GGBlB2#ti&U2S6Ykq0neTWjf>+VA42fiQPmruj5{S@V1$fnir4A$Mm_p!yj zFRy#jj0Yk%UUz<>g8h<%y$w#Q<_i4IkndF+oHn{)l(&#>Eo;Ee3LDJ}o1G2?qc<~u z`_a1FuMnEaI+6FEzHP5z_u8DWi3$(u9>xjL|!BYx}K_m9hlog z1gPZMnLUJyYcuib=0+Z7#~)1GL|l2a3M1CrUR%h45gzyq8}ne|4r^i$4!V52*Y=J# zvxn@xF1yS2kk{wa@ZLXS%JSTw@J)efJeZ9Cg z5o@W`@JS-KpG-H0qHE-RoC|MmpwimE&S!Si=*W0q<9rBj*Isr$n^r#x9lebeItgm8 z&JHwiwlXa}U3cgbeLh#di$`YNJ>cDaAh^O7<2WaQf71B1@KH4_f&R~dtIWyTl`lKdX z_e9>!%b9H3I7AGgau&Ci?aZa=TFzqEIE(pltFxHjyfReDS)9vc%a6mnq?L1%ZP(bO zyM7mDTV#kX4k8RJa#J2RHz1oUHyoOK24#X_)`Tz01-i>Tm|bP`Lbn)D35|Sj-96~! z$l^r&)7LSU4?0dnYAR2hPf$-3Fr|UHD`#Aj*~$nP+!QC3$_!uqna;X&q@Lzu(^US;5wEEdn7%3c~ zYg6m#`7YST4(E(ev#sWr*;ZTFVdl7#AI?eUNhF*tw)ZYK4X?VpY42}$*4^u=?1vxE z*WGJkd(SVn_xzK#_Zw+b#8>gd=IicJIHs+=kEVZ1?e*R2nBcVoedu$gPVNW4rh5p@dADjy%g!zGJFJ%$q6!6lhS#| zlk0Ak-~r>Cid!^)hjnt_iNPyQ_;YNj5#!oJicP}cG!*p2621M$IF`Paj+^aIi9f21 zW&^Z+?BUdxzPka()HM8-cCm-kY4x?V+7N-EnaJCl*!I)^!E`?TiTCBZ^PlfpLVaT(WcA-Rin<=bJm>lRwk($=bU6_jJGmpFvfM zeR$VmAKtaK54A~Y1L2ScRxm5#PKwCZ-T8SRp5M}kJbh&v&stHU-0gM_Qsd5cj*s46 zKylU&Z~rgFx_hxrl@~6|l!ucoGv$(2-r79>0&C+fvpwU%WO2{9vQ6X+d+o=_i;`C# zFqaQ}Yi}*>wAw9c!TLai2h9H0{vpfqcLuXoSdte=989=C5|J2kqwm{^a=ncu=nhJF z|C>3Vj&D#1&OX`Ia;Y;h?FV}V+!wIW84+(wx;g6m;lF*kv+j=A4sf^;SK6f?*5TKa zmq+u7$@`7B_Dn+*nWMI6KdaTdevevtJ*OlLTGfX0(#erjAHhp_LEc{ls(!I?1n(?u zZ)Ye4dXPG{4q6J$T{dysX!?FjAQ1OaDqs8L$(`9*a zkgmWE;Wz4G6WTr`p^nk{{m4w6=tWYo&;{umV*^1`jV5k5<|@Q4+1ZfZrt^ppT)JZ& zdT}~(wSf}Z)BOhKj{8ZvZ8{zAiZ&OB9%5=UA&;`-ewpmRr>Ejc5qV2~ZITl(h*ukS z-czF#Avf{8=i^7quT6>@lpg`65*&(l$#@1v6L)j?K(YZhB6+U(EJfLU??~&!h2O&J z7(OiJc8N&r@e<{`;7MMvsd zoRp}aj8kR%dZr{jwOVD=s^6}4`n-|DGDCNWQ7C!-D8)o!kq5qi<0wOAY^toZJxn&8 z88?l(3Jyq66d{cW|5#KQ<0S^7@)_o+nD&erMG~oT0qo-U=pgmpNhLcLAD8HwhZdCG zflUlGrI>~<{%FS!?d0FKSmIom32~%t{H#yly-55nRH;fsQ6YY-s$VwRj&apgD~%UE zCaq8PsN|r&q%N#e2_hC@Gp|&g5D*4N-g3POAP~D!Q@@YC6+Ukpy3{2)&@sr z3W2T}V^(0LC#T46tsziHL;r)}g4hp{_K7<^)AkV3=}C(j*yM)pwMm|Da^=V<3l~xn z|CQqv+bujNN_yg_fA^ce@Wn@d=Nr*YvUCblPKWQ>n#-h zhmOY&2afx4?YpxH;1?s9IU3U2*4896d70-hto(D-h54Jtdq#-x@ z7AX;uaOb=G{D+}NKOn1r-p;?2san1na;)Z6FBYkNOhe^j@@p4Tn4rifOdtT_yi$iZ zoOpGoZ3SzSD!8`n>Mm2CHmL&dAW%JHromyM*pysWjCqsWF>5<|#X`(lf?)JwMa3Ph zekI>WmmX_A9B)2c-+Z{S`EYad;nwEE3z`o@>o!qRs)Jb4 zkJ?e|+H0fyMA}+}`)#Z(Wn%oS6o2uvvZ=+-hcM^G&xc!|kF-7?ZGAr0`h1*JIPuvJ ziEDu`^8sr&=_xPw^4Ght!N+~&cwFqs#9cpeIr{;hbAFz`@EH$n?+TYUtX;Fl6i|jF zlG1}$ZVG9aB|bmJpbDL*)|NACDsQ~PZQ6f@%R7CA%X`TcE^l&$%R6+1%R6=j%bPEy zXSR0lC9L#WNjN^OQ)C2A8xE6k!I_dPhWf~q&%K%?4@!nAGvBLgL_;$1i=(`yhCtoSmu`GV73Zf zPyzLA=8Nl?BHA{&k)imog+i-r9CDTvgOJ6@$|&YWGL68ZoGc@&Bk4*Ys9st6>|pl7 z@3;B*;->3G&K$*A0)!DxT}wD6GqkdP5qc!GtY{x-Ma%xPBL_t>7nt=P1k@A(6^dSH zPcIQrUF#J)8cjEUh3L2>pk@;~8_DfZ`KnYrW0ylM4);d6rSm&UkS3hq`L+#pYr9F{%v?I4C@)evHFQPYWH4jx?WIaCQK9e3h zo8j&L*-?#Kx}h-)FgbqMByc!aubePU?_Zl73)LA;i)7+%yn=;O-pv}$?fYU!>QAdL z#FkaDuN+}Y*gO`F!3a*myl6L6L)UB z-`wO!+;Pl1*pQdD*pS6&Lb;ajc$KZLdVA?tXYcVNzr?A0&AK}$;tKODh9gtuA6}OA zF&dMb_|*8!<0k?#Ih+ULBE&aNY9TrA*7%^l<`3<5>6dV={hD8l*C)wFk zGrlqooaxq~Z!@|-Hq+1x*@Cd)AAnxr~fm4Bh z>(?y)DoiW!5c(C1J|#Q^HlN@o*I+IcK9-Q-tvcdvxlU9r1*q>Ys9Nnx{Vr+6>_x3g zeQ28^W&eYmt0`{;W-=vhs+>RGUlShrxxx<+nVUfEfN@==C{t3jZDU=kM`nHob^8^!<%`uK{iQzM6u>77+;qNP9#6&+_A!vNntl z_~E$ld&6T&E>ri^9S3}I@rRS-;_$+je=a`%;t;97{m;b%{=mf%mf5|<=Nmhdd! z+W%dA{>68t;{(1x$P$QbL(ypd0RLVb_4-Pp_6fS%@wZOi<-RyW{Kh)YNATWr9WK%JtVf63#E<>du>1mY4{p`3ErBL`G zMX73_n^lyU7F1NhE4pNlDWM~`O-oNGOs#01n7_QMN`Yy0B#~9NlUpi#(QwF6actaqHZ|yh2P`x3?G?t z&$;Ry!tDw)c7#m0#r(W(r@GvcF}R%%IA1uNcHs<}A4=oz#8=V!?n_^cmxX4VxFgIh zqv^xu=|R zA*s#}(JZ?%OdA%bR~PIF*vyB1H?Vehmiz%2C~jK2zj~R-BD67)bX;!`wAkK)se6z> zHOi}bkOfeL?93SNN{^MR?51(E7_Y!Mrl~vxxzy#w1%^c0ToOMVe+v2h*WUcs-u&0$ z{MTXgE1aI=!V_E%C5l|DUWS5RaA#+4Ze9}K8@HF;gbD1{wC@m=OAl7-%Ed%1=)~>5 zyw_#eHl2_~dpj>s%j?n-)LY=noz1`&CYnL9IUsxiAX+)IY^r8hE=QS?LJ^sju?p5XW9Y5H@UtFReQu7j}8?cdd<`-{rrP z51^s*{P+pWMM#RK@mLBUwu>ecpU)5pV;;T?g2}O#r84mu)ieKWj>kSDwc)c`sSVSG zqW$dBKYh_C_0M{&qoO#5V}?GN@D!4tRc}kjV;V8OF7peok!KV%(X58bE}2;dzC`&J zugfNMNY+ZZID7dA|K$_cE1_mqfU1@$d7!K)sv5ZsXH~b!+PeZrperFaH#9KEyDCc1<<47Cjqh7S|7kELs{n!;qu&y(PN^NWY&9Spd}<_H)q zm5~; zue%$)a7Zs%cQ-SM<@&wZC*jY2v9s=O4Ui$9xT*+ANy?hD*qD}OcoiFQ*X+T0 zNc=L^*4!5Y>I&e*aP_e3a|BQkwCY5bv@f~#x)L|H>p<8j35 zTkbO=x{_xl;XG7|2iVdQ-Y8ow0JY?~=^^`Z;?uTu+}p3@=1#-mseoHyl|(#W|BXWS zai6iKqbo?vj-y}*wsUlJmbg^p}E#y2<*09!wF6smOp=jWXuo-SA@mU&nPsLNt zfrrzAY5pSxr`j*Puka~}RQmjSj^?%xv$mmkN9K>Vp%>S^$O+i_g~?zdC&!V+*SQ_e;=a?0x~(6`RXV8f?OO9j_(t`mp-ZP zw>`!}aH{a*zl;uDp*rUnsIZ!#H$zh68HG4;Y+-FTfW4- z#+R7-z5eQ#`#n#{mq=(>=#JBdFR?dwFGR4<731LUw?bfgV)c_FtHq%#L+;j6|8^QYuoLC~mw7KKQe}sqcgb?SfxAxrVJ0KGgW^L&-nOkK5$){qTi0V-fFDI$EXb z``%Hyj}Jq!)@R3=$UzE|##`u*>Y^S>YCST1o~dNvC(hf*Vy;Q4JY&4%VX1Zl_8uAbk9YR-8ai%5pMw<-22 zd%KF2!f#ofqr#s+K1Jy?EpgvdL*2Q61igg!< znFOSKEH)(aV|LJohx}iqgNnO56QJxslmz3f;JfU zk;p6n2t&TfXWB%>LD7$Hp^g{lPQ82PJ-~q0h14L2B!|sn5Ppkyv%)fLK7-SKw;Wb| z7MskdX8O8HJ;=nMV$99nHl5jxYlJ<>_M4`o6SO|CN3aBonT-9*r(lzeY9J+AW&U6o zZq%eJXV>ajk7{hHS5k3mJG-UsYfxFc+ubT8V9HoakgZIVWb;p9H2j$!4h}`3bcpqk zG}H;Bsb2|?zRg%vWN;@oq=WVOu6y*zRA(AeQ0F{xwpzl=?3zF2HxfcTQ!(t)I8{oWz3B!zb{j&5umF zM)g8XScU+kOF;+tbH87{|Ac?_o!4czlD1BB)iYPELAgg>VpF~dvm;Y%;;tt|ec|k_ zJI^x^Ky)Kyp<+6cmML$}=^^{MhBfsa9o92xzAZNIOabf7Q8}s+vY;d=uy@{JUhYoZ zEm+V}iyxZ3Ec9BML)!3xcN@h4h3pCShV+T_hL+B-KyjM3gm);)*yP%Ph?%m0C$}M2 zC$Wvl)%kI2yu4ss*^FFWk@LC{;jY(yoi-(6s#G!==L^4w|b?94(6E| zJMEUCQ+wJioDJPmqB@8JXPgMI(~1*;%w}wWc!a`gdl1>r(;G%4%dtVa#lVAFj+px9 z^ZRhClV*$%&u%*D@RN0tdph&G%?AO4h35Bm%J5U_q|5R_9By?@5AG_P-|R0$Q)r&k zqYB*XZD&?U2;K6ROC*2-ec7itZG4I`Qm161uV%qWefOZd5!9-FfXOq722QhjG&R-2 ztA1XhFKe*+`BE;Io3EhDg=o@=J1+e{E~ze&(-;`bKXfhB;(-RQ13EJzGZ7gh#y*CV zB@=gG2DA5qd4XuQXB8Hdf-!BL|I@9{bF)1UDEbNltY%Yx0js+sLtY!JJ1^!LdAa~O zlVr#z)gyz5;zPIrX`?tV`tv3ZCOiR_+ufqJ=sF>2i3^#z{qOWki_{ z!Jp!OCs+5CjoJbXD=X-s@|Xyu{lj zf+9RKtIC<4O`H-C8+>o4*zr=2c*HaN^ zNKZ&7?wG0}1^3i#;o}MZ1fP_@u;lui9pG{o(hW=R$yj1kGBhfY>x>Xca}Mjo>~KZ6 z<4y3oq`n9jKGs=>p`r+{x~C(KiX!|6KHbM}Z}?zm-MxcPfBd%2y1R!@|LH@Wb$2(P z{vDtGJ)b_!6K~d4NLAj*r#oPgxl4|Xq{&IxHfs2EsZ3LsWu3fJbP}8~cu|P@>`vSS zp|7CYp4$4tFjC`=yfzazA;pe)U5apggU^R7V4+~UI}H2`2b799YJZOT0@4ZVAB6M* zbJ#~I)ZF2`J(jIX99nm`T0t+c5^kZTzKFyOV`Y@?0#EFP^lV**Y^vPT(Kn+Q>0U^l zpR>k#VOi*-VUi>ug_e>eSx%BJRL|0}WU;;>4Rr2QW-v|WB>`wJ_++Wtc2m$$#L zOcMKzx&!p750kQhLj91Q#k}3~eXkk$$B|FpnY9W#BCs3lT$z}t?e)92HWZdl-V zftZqCm%Y#y;EOL^!Vs*mhPOY4dWd&X?)AOP*l1eE5DI-4z8mdBdi6_u5#IT8<{Hh{ z*4@k47JUH|jBOrn>m$(B6yf}Xh1{t;b3SE%`Ugd!W`XrZ7lqW zWY_t`-6#(98>uO#24!Nt9aFQrX)Rz)AtiJ4c-4jy*MWZHyp!4%${k@_T38=T%1s!K z?UT`!>N7c03&XOO@ImPSLafWh6=(bv?XwPDTwB%3sWJ9NSV0d#2tZ;6_5hR^@>Q*oAFXk+-?h3 zm+g`hBzt)wmMrhgmQ@4?^<`vkP}XIT{k}o=AE{M~ z{kY-n9G$pI+7405t^@ z9c2||U2de6OwtOsw5647CasW+y;S&s-p~Ey3&`+nOkD3^w))Q@bcle6AKyGvr;QxL z^@+r5HLFvx`@Wbj&}8BcS8phM^W(<4 zyHTtipI4Hkc8R*TRzGN&xb`w}?PWqJ@zY!;Br9*n4yDx{sZG`{)ox=Vx@>M4wNk`h zk2JXGgyb0aS^1HUDBRY(fsWJ<@;YX|BM{@%(2h7`s=6S7 z%8dS?Ye0PX)57ZMY1lojho*Y~iyUaKO2Eybe9%f;t^Ro6^XhX0lB0I+`7?M*VJu7T;NKW0x)XYIU4C;IY@>95x>e{4Q8yCELx3aYYN`w|v z`5$QnDvPEyF@BLoGqO*R%-p~-Sb>{aT3Hy^+k2rFeC! zcd?>W#R7@S#Yq3 zr3!DPD>mX*t?o0DS>l#WnWHzxBVX_bawV2B?;iQ39~fo)`5_yakze{3BWJ%`7+B@O=;!OY{7 zy6lFdiopPirbNo=h7zBrgmgGO?oy;3x)Oi{fuGVtNg6K>;w4Xt?Qs6_$h9@&$I*hHQof-C8zJvIO$cLppSi}pj37!SDMtOVrMqf1%ZPhgJ^?)l1p z4Ruup0EA$cL2x&Y0ByMq#qF@?5gMmLiIMji-Oc>wyo$-Q_K}_IV7AIRAYp@OI5w7v zBRE39EEoFII>-M7iLURi-8UZj{QmKBcHEEZr4#9^$0NVwmtUaW+b>md;Vj)d?)n^A z+(G4)yG>tMPF4&)qB3JwEW+bT)XIub2|! zp84L%xJ@HLVTTy7IMf;RiyeSlxyuUxtBToco^7=tRhoeUDD;sT183cA_M6mR-N3 zFMHn6ml=KO+@{ZCUpg_n1!FvD`Vxg8_9Y*;^riE;^<^fSryH;ujyf>H>MXt~c4#tj zx73}63iuG;ZL8t=rAXycDV*yO0Sv1 z)r!J-CNa;R2o^@5cGVjSL`B13=bIqw?@EOR&V)16YN(6IiO3Es)dhj)R$^Sxgx1{; zqt$kZfl=29H2(#%h!epQS1$tBr|;^T>R?WP#}OivW!GN+&xo(_u(vVqH3fTi zaP)L=lop}T3+FE+0G>173y;6woeAg9r?)*ftx8Xy^Bj}GZSwJeg$pVl$q$lj45x0{ zoGDcGR^nFWNsPfQ9b5pwH6(Y@7&8?^WfEI0KGR>x+0}_mLLFU|Bq|RuqLZN3S_|}B z9!0DSXL`Z;megH3zZj)V-=YZ@C5zr)DZKR*E*-oM<0iwlC5Ur3KodzAl8ho)MWhE0 z!h3$X$;%sYs2Z(XqFobuexV}(`hBRzeZ%OxI4{t56XX5r? zlhqQqCH{S3xaXc8BAI3p|92}RH(V((nTsTH-Mx@CG>|30_GRn)BOF7!=Z6xiTa7*o z;UXteF*}jO2BXX9x@4Xv9fo(l_f*d%qM+tc6qz3&sc02wYcSD= zHn1x7&4SmYhs;k{0uAQJOV=9H-Fmhx94xG2V=vqGWT~O9oJ{MydXLp7iu>~Q2Id_zfn z2N5*vseW0^(1%fii}C{<*E9JInxMW}m55_t?Sy~Dw)GfC4MdiBOKHN?oSpCNc;M}@Qatnm19m%bVMkWbR^UQZ zUuC=*j_pWYJw2_9?L5s{&wM0UOo60e^O5kS;Kl0W1LF^PPBOBSnGBt5$x!s*RnxER zvYAW}vKC4$i})(XZjxKdj2?mUT7<}V)l?#9@5Cm_J4H;I7PyOg$tbwBx z@dQzy=|)}Buu<#n-|u_3jU3ZC56h4&Vq2?|@1rnesbuZG@lqp@F4ZXl>5@-PAPwE| zKz2e>_>9I&S`|L6UZ!IJ(Nz8BeELst^|Rc1Q$MPHlohTglq4r7Y?2yiQ;7Yl85$DB zspu+2@!S9q;7+F3j_k(^)2om5{6OLdnr{ZijW^%J&Gs$c56*JUt4VN;R$x^Ww@$6}Q< zYYP*iaSt@IrsCFKe#Q))6dGRVushP@BCx0LG3d?-j^GLV36^#BDL8y3)f_;4Q9mWt zPsyyGl6gPr8i8@@C!JM!W695D$+ve}LniJV*kefutq`sb=1N%`hD33JLMpwMXs1!U z|1X!mB8}lBG+SW4xJLpkDdQNC%FMQ9L`|FX#Kqd?KP{Cn6(26nS zaM#FZzUR#Suen#$Au1UIQ=5^dJB!dnqN9t%M+5LKt=^sz&{DwfhY?DIA@Z2W&h%rC zSnl8qIzgkN(Luo4jVY^=aPOCj>Q_WNjSQYOlDF?QpOo1ccFi&S8bQLN6A#|~fjeZQ z&Fhp_WNO8U^p*F@86Sg_QwDlOl`Xio9hW=uDdE%A<-(^&es)RW)74|0rl?)oUGUN% z>7~Rcb2=a>jp_4u=;I_U2D@xWNlo+<(NrZbP^aDMCd3aku2S5G{E?@dT%*q3yns6 zbwNnS1LoR{biHfnN0F}YtQ&Ted#K$Fg79^>c0>0GAbdNb&Rl?8cQ68ZEJYh+wZd$Q zV6b1-@lNnl-}8g&c4;tvj8h!;)$^rp6(rA`BXOdn25_W=I&JWwTPI4dJyBXv;hh0D z+W`Q)2en~wqNH1TsLkX0l(C_#^g~jr;rt|ss&1I8U?g&lan&Em6GSy<=7+v-px6 zI2GCvW7p$}FsR5Obe19vPs{1v!FJm9E=CTT9SODLyrQ-`;V-&)#PF^8DK-I&=y%A| zdx$c*q9UsEs)%{{^gM)RXnj=H`zy{#L@xv7JP zhyiS^OKca6KdDw_iulX_fzXSeouN0yetp*PD+6-wFGt_!!QiE0b(imIJ0}UynZu>)zG|t!^Z13 zao!KBm&s=py5<@0;6d>S9;86R*M!faaRIvRT!0{0W^SuXT!5}|0phmOv{+S}Yey!e zx&She_|6fxdJeWU43Fv5GB9dsvX0V&S#=^kXI$9>hNu0T9g+uZ8-uESNlUW&Po=N` zPB`=VCAZLKmvne>gwP2+Mqt?~~nvAwON`e*&e(7tI9}a){O?ik)EH_af zmEK*CZXv0!Y4i3V22*)Ma%}{i@JDaygpYp$Yg2lY2zttJSa$ps8&^}}wKD{;1;?bf~U*XbEc z*fhFDF8AUjP^gn&2z-f4bHSj3?XIVMHC2ge&nw zKZ2IkjwauQ4qYa>rcYDMTEZI2)U_Kz+v+zy@X|1 z^Ngni3lp)%%@ec1f+65HHF)#6*&e_xV=n8-?7jI2d|jN;v!^)Tu&7|5)6rE-+*!UC z6L(9%PhyO1 zeMpJVxBb2HIkNa6izFR}p*h8%mJ*o}`HJxg(LdSQnw%5zW_H<{kq^`HstsbM?1k7^ z(Tj&bHlmxr`CU`Z`UzaIq}zxdyCRA~5`uKf!q@V;{`gsjz}@e2|E%jWkRx5CI)MfO zzCu5$K3>)X^i3d7+NgD{S3O|An-AHG#B%t2nIe+$092klSUk4dXGlqGx{NF@%Pgs~ z%2ew2S$XnxASNJ1;rVdLPY#PKw_}Bq`~-0N*HeCq^3$xA^0iP08T(-)pSb;gG;?jA zTEj@W%knkiPe4k^kGTDkQmjhRK0T>Rbs6zUOxWm7p0n@IrH*#|^-|j=vUHhPjg~PW zGVL3vKAg|-cF#+0^Wg4CiEi=WBAX}cZh~Dn1zf}-kHjwE?5u6`K!7K0TiI?~`Apl^ z)&Y0awgnx~+UEOEBjA@X!QD`7=KdTl6fPS?_#L$LDy{Y$xqa%nSe;$P7g0Bv;T8FZ&Rrc-4Q*u{JpX@9fuc zH}e)Xt4(DVCfo+^WbJivIXh@8p&m={`td-uQhW00ScaMnM4nKb8n0!?{UC1Kqcl=V zfs*9wjcbj}XU`k`s1R+63DBq>% zO9qC|mW!wQVlf6vlvQ6W%JJBH`m>a!&6oH2tNu@yocQ~%K39EaP%qywa_+caX8e+n zgKu62HBfjWlIZK#IXZjRwaYCl_Kv*r%X2ICP!T_gFK#vo9XSLkhd5dNVU~sBeVVoR%$k@mG70Zq84MDF7!heEYbr$Qtf7jc)zUqolBV=W zk{Lgo{bDe`a)S+sqb0zY>?)!Py-fPE9Mfrc<^?ldJ#q7h{ z>+rtM`@GNLe(r~$Ll_c&7T~vsK_I3i@NGN+VtXcoDhLmA!GyoW`B^h}!UGwv{yx!n znkFe*_RV_k+-?UQu673K zTWZqgCDK-Z|J@8b$|vNlCX@Ad->ubM)qAhM^TmN_pB=4l+{IDH>^(}z#-cKXCIY#p zToZl;jimwc3ELKzM|zXY$S6YKX{3ov2jS*96DgF(HPOOyXhgdmw3zi5R*XxWnje@J z)4s40xp;V8PhPiEj!X*&0BZ*W;MOLvkW66h(F!xmo(_QfiRmlUra3)*o(=#4Z@T2_ z2Efw+7@6FdiVi9btw8a9*5X_h&05Ixc8v~Kbr*I=j z6_>rd3rj7-y=V$awFb@ntk%X;7a9lIb8Qp*Jf+Dg2jKJUg*t~IDs_4yF(tICMpqOF zM~QzUBa6!Ir2CBO_H2uqW<{Zo9#wY2FgcMKxre&<q_t(J|S zvXaPnHmWQpGVOc#lT6g@%*Gun8=923)0cT~Lh~1>N;&Mx1jv{zEzO#c!!-@OBV^I(LBXI{8=tS8bX(4w#@K)|G?tj1*5I{-Z#0-Z z%0s1F;0VF9D8U~)msyvU1+1OXI4i+!7Ij`*jTKyppW?75+ z+KhihX5^1_l{1Dgf%Y1p|5u?12aW)W?ji9hOjJ+RgghVR5)=4}Tc3kcknmamF|86E(($aty!2ouE1UT*YKS=ai$eu&z1%ImEB6UxA{rY+J01b%E!HvRN(RVLmR8 zj6k=9_p4&l1zrlDMasIBYG!5$anuc%W}sn&wT;y+=s-x2m!t3u#!OH-&wh}$Yj48s zHn{T3q$3ZRK_YT#<%jFQM6z9bZMju;73@x9$^&05{n}Xbz`1V5(Iop~wnd1XYB=UU zpq+@aPMb+XQa8L>mfMiJ8^)?5ON`A>56fY^z-o;$7``$i5Wu38kM$YfhB zZgMDq@l6J)6_7|EGJ-q{TRtJ32$WkEF)=MR>TnHQt|shAWQ~KeM9(!X(FWZcJh3LI zp5pYlSXtaXW_Hws_JoW@Qa_Bc2Z(7}$d=WpKm&wusU^Nc+SmAT*UoLgYY#`|_X_dy zD^U4@T7Oh4XuSfqja)o#x^0e)-WCkI>2=^L`1a-*0IW@NI-t8+laLZj+&(;iL|K!j z+q_KFfSRiSeO&C5XaEs42}bZ(J&_U)qaKSa*}96d@wn+_AeB2RmVOSNv9c3UI}1N*y5hKLq^fqZ3W52 zRw!+D&LM<`=@NB8-eX%v3BS zn(N`i4WKjZaQ&T$!AGPpcBHMuEVL1i#`L3^MsHf5uK z9*_uc8oQo0VNnwTiz0X`>w3MO_M7K;AV8hbY$TReb)D3)>Us$w@?fmG*}6t5aJ{`n zQFg#aD*0VY;6BREF}Y7PQSnj?uHIzJs+(G{*he~CsgQm!Aw&kz011&%#kMLTGCoUK zb$K9Ib$K8Hl$|ay1yaYVi%)@BM5t__L@5=PWi?0OGY;7q!;Aa!+Qli;;f|}e-mML2 zpGi^h`pJl{TsPt1?K5gX#aCTbK&9L#$JGCC#?()Zsh^B#iaxYDrfku4{8i*s&m?1t z4_6&i9(Y0OK#VC5gx2kJThsvUGwCa1$>Z|O_r-l?IVyg!*?7>;Eh)iI&G+homQxB^ zHIbcE>2!-hoC`W#Lqo-xj@n4@6K(J6aHI!$mO8ZlnsVq0{}Xe^$C`o)*=IwuXVWa zcYsX?fY31>Sl5_tQuPf{GjbqFDd@bwqjl;4&oyKlZZ&>b|*ZqSo9Ksg9#Qu2Fx!o07>ba0nXD}%^7U;@D_ZGBz`k!X~b%Elug$J z;5yjJ1qK?T=p6D@t>$#G@@m;Xb=9PB*g;Q%#Ra5@7rUwPM6YnlhNZ}%pO_3PX>tyL z9Bt+8NVLw(DuoM?@SP9Q$J))2y zVOU_i@RSU@nDGV3vJ#LGepqI+$yRkp&STQgc4$7&wGaYHXO>+XrVbtvI= zzY3!i%DWLCbPy2S0jiM#n*2BwP-!+IR2M?kyOZ2rSnUS-X2)n4847#`51PtGBQyo5 z-No}FL-RiF>+>s!`z}B-I*YUiSY*&)#yH_eWNBc_%G!~}EZ}~D|M}sMlO43=qRE6G zlTkpRsQvBQ%05qp2v<;Uj!n5KHn?>VaK!nZ7>ESzSlhH`TQua*$VK3B$F?%yr#V2? zxPy7Nd3G~?DSqU}t+TKpeeBIZ+81b$Blym^aqAH?XQCmrYl5jIiH#!Rc5Ev*sd?bc zik+F|90wzIg0koKVjW?OgK`r%`0Pfd&8F4gb7w^!sT!1K1J-zf2^<>Gu^12byKCX; zb}IDV!DSGwfzvmcA=4w&9+OC>r4ltPE)O@EvS)Hy2Z@nkSQ6eVXkh_LN?!?6(|{{w z2`hxkCw3iOJJH1n3*HBqsmf2sQ);e~1cqdufkWGX3**5#8DfTjBHF$|Xv^oRNIwNx z;MY)>gsAA(;0H25+PRg8$Sfp}d_chl{~7k{%;XZj*@0%?A;b=HKjN#qm zjy1!(`#$#OcXu4S05AKN=jL~}^3%S)51Qt8=Z{^0FY?Eu82N*y`DS=`D_$?p&Bx~* z$Mfen)$t1!EC{0EX2iUO7Geh^q!}K56hv9@5>@9rkE%ovi$x7ER}d`-%sSly0utB= zm_!{8XFQ7Gi|K;5Vk$tiT6vjbIzY4zQcvqJ!xxy0Kgm$qt8%S-Cdy2z7?y_5dcB79 z^{UUDa4^p|*_4A8AmIiPL;YZ0uUAco3PXxRaE4M=Ql>7aVQcKMWhvwD|L z(Nb6E63a8naT2Qo_IaKQFcEcmP#~U?Bs5@xjvuBImRnpN-Yw-3_UQ^{MEz=;trH0$ z;vx%wT)pMtW<<9$I~uV&E#*+dH{7ERK_M9XJgPYl!OWp9K5K?SEmfbe`Ll{db6O<0 zSZ(=hwyM{nO!^9+XAG-P`iaS5N-&+yUM;apF>Pa~^O(#6n_XbmO5$NmKh+8rZJ!QW z8`h{V!Keow1x7tB(=Y6Sc^#+_wo*}-D)K*J)R#W%td-KUM%F!IcmOP9AR8!XC~+dw zT$5+RV4%3GhUM^849iTNVTnf1YFK8}u*|GtSOROU`=N)u=zp0piG9x+llr8+h)giW z85+0j#~s>119l-YiC`?RL=6CA6n7-n$ir)3yMP!6oeZmX1KYwBsQv)(N4Tq|!2}6M zT*lFayvd_!Hp!DDre=&!9>v05nn| z#!2NCTp^Zj5gHkohVTvM-&~>{VbJq#l(u7@KO>%}D8%ZeQ7gs58=3$fFORzjFLTPp z+OL(u+_Pjqbw6h?#$(FA3^5K-Kt=u&@VWNok<1^)t^Hod*3C)4*b(02 z5r`a(=$2_EPd^7~Fbp}L9JZuD>;S123{P;d!_6Oo(8!2xXU{iFs=dxP=XhLs#i81DEpIf}!$ zg5h$beuw@a?XmwY4q^)`OFiT?C5F%>K_GRU`K%}{InzFC2G6=Gvu~ois?5F`2SSkw zqY8Wh?ohj%QhdRQTJwc#V16vXN5QxFUxWEszMJB+0u3Z)7Fi!M#Q!wo`Mr)(B89d; zkw%wLed;GQk!*CjDn}2rK$@c`hpGD{M^8?x?DB(g^yEzSD{}PYmuHhx4ZzXE`d)Bm zL2|Ny{QHwo)kwF5fm?>~GS%5*wHM@6dAWumm>QmmznBwLUo%GziStk|lLU8Ar3$Jn z2UB1x!rSrV&qyhT1)V5WhPlD!?_)P1PzYxIhF%AgR5rAWrvMj~#3c$6b zS`vc51Y+N%&|QYeb^$uFwOVbpsS_gPfW1Fs_rSU{aSsG-wKoF}zwUwPnGu11JJ|y< z;U;7Ezy`YqLe8-VI>z&R@sJpo7S!jXl<<1;Jf!RMiMIjq3C$NYZZA|K5B2$|E7r(e zl~B59yv3kuG{qQ%b-=U}xhaxCv{0J|!jf z-uNKNv}KJ&0L0(C-7#B#XtRdR0Mg)AI z1B}z%8$>4*bufRso6OL1E!Va|k zv71-2QDgJUV$8Tt6YG>{O{W^jMST)0(f(@#g%Ld`(HfwEnMe4u8XUtEwSlocfwn4fQOC5=)EQx6UZEJ1}Y9HLZyE`Lr41Ti-*bi6o?=9Ok-eQyk``XRHRb(lfnUInUHg^ZI}{D0ew6 zv)JiD!mNS^0C%~j{|LG;8Y%78~K!xpLp5wO#er=6BENXy!C;Zb#`LGDwBM^JvT^B0+1gZ55QJ-E3Xf^)(?J#gV9CrVx}EeIm??;iSJwqD2<+Fx^eaWA$pDj zs;5BPzyVM3fK=kko&<74PqCwM_5Bn*#iZ&fCiR3DBSpkz#zg-oA8Skng~HX7jWVvj z@6eM0ldg|sa63dt)m%l4dUb%p3aXW>XDS+3--FE|t3WiK$)3iIFJk9AFKFY=02&JB zr)V%%I$aYOH`vGlAua~da4`dUu7~0U=D`5}heM~^4W<>)GuP76f*wMNY#8|^EiDO! ztF*MpxUfwY;Ci?YN@B39KvdX-V%04Pd(!D1dp^^PXbNoDON;}vB65v`8R2Vfi(`pI z-9$&#C`FY>lu8cInL5KIC)zV0>oxc;A{*1cH`0sFX&J9X21UYWS7dQxqEDf$1Kz*b+;4 zT@YnZ-xuZLKyzd+cu3eE34o7VF3%V*yvG464B(PEnHd#e64(I&wvIJ{993gpVnaXaI|U2zfd!4_39b+V zZ}G$cJD@oUEop>SxLep*MjGL>dXXKsH-=%#4lvu71n~rqCG2qe5x|8gF|hxM3iUGZ z-ss(>2LM{)_Tt!f8w5hFtg^NN$+q^3`6Qba1P{L1@;)1vyrNp`Fq4`whCw}DF zAF2KPa05uEFw#j2FHWaz#3sf^Yyw@XY!?#t{{-)Oqg7uvQ(E=~a8Hy5bHx!{JZ z>(3?VJM4*y2s;9RIHlYWgJ+J*Q^ENO%#jps%r+n>47LNnV>1ee0*0{xy&YjT6Zzg? z{a)#ChqHH?xQ+q`0g{lhZoW*$b3!W{0@H|qkja=Lc%2>!q5zQg;!MQp&}gE&FxTW~ zM_JRbXtv3sS|`ft-|jK9X?y{iL&gPf29RIM`Jg>XmZLtgg#7{>JgoEbY~2??k|@6M z@V{)j<_oVO?~43zP{QLdlZb$waZRbB3DN2gc`!L|sxT1@6S4{>`y-o92#`fuR4#N2 zapf4D$Qg^2#$L=ufo}y&8t^Ot_{=VhN{O?`9y)OI$*IMTC_gFDYxwl3%6($0z+Fj@PI{_hZ-zF8bxlKR zzW8hYVvzaDalT9z5R5Dfwt~PEH53{Ik2tPV4GDSJ&|r((jT`NDP~PB9iZ6L8Yxi%B zx~cO4^R0-P$CB=iVVu*mB^3R{6kwO(s;oheRlTPgLMa%;`$ZqUkx8Q#O4G{sdp7 z&;?LixUhpu#w)!USX;=d5Y#}29y_98zX(_%Xd9_WAMSI9kW-9$8%WUxm>lE0~8NX1?j&qYbfapKUN0xnQV9__wU^<|~%|m%u+X=jJ zV=0C96Z!%me`2#cKvJN1A`kw}NrcjM%U*3kaD(SLbNycDXf;FW$XePcMD55LTD=Op z$vCmV^Wd7LBhqTuaDE3WLeR5#vno4){~*p+c0go3N5JyOYz-P^xSR#GOeVkLJ~ZON zv$JKHt6CLS5dK|J$`=}fVyUr79M|(R+1!fzoG8CyA5R$HL^xzS~OsVYrlaf;A5bYKyjbnJeE z9;EbNH@2=WTCC@h?w9bXAad!C;iQMtJJDkoWZlN9YD{p@D1M-eK-uhYLsAd3PbcR>D_-=tBlh)8e#) zP-*QBR11W8LmViNWI_SQ`e<>f4iDRLd2y+NY$Ksdtw3r*KCnmX?g>Bv+j7k)gQqf= ztbqc%fH@F*ux*vJJTPg~feKA-|4!V?qF`-U?Zs_9%429-^m=h?4^a;Qfzz<|c4cR&!H0i?$h@uF7&2 zoKcsUZ$B>f(C~%b(4!fH>6Hki&Qa4sOJ>D+LkqH|Zgc!XtrMt9s>y3x2>LG;_4=OXI?1BW3Lpfh&@kCH&p>S#Z2nl#Q4`5~-tibK z`r)gLuAw29K{stQMAKpNgP-`eUQ-Y+Ey6{B9`R?;z{n_w1?`EGMqY@9VukX8)h1E| z`@mNS?+Y~RndxvlxMpAp^HF>Y=mKC~yOXiQB!f8m*m)p_SnPt8ov2VKB6QDQyX-;q zK!6%MK(y&F@RdF};5B?o7UOte49)2W9-D|0%|xptqj09Yffv7o=zuDg3CRH;@FN=T zKI^9#7pFM^KSqIR7ykez%=-{cUwqfZ1YC0ha9jpj)a0npTGBOfGA+&Wk{N-;f} zVhY!STRP6FytaA@7QEufyb_{u!_#5X5={!Eh6O4SiMNZm2JREg?tIE*B(22In6euR z!~#XD!?^ddc1|q~twV;bbKB6Qt#{aiYB1c5MBu|`lA{n_O2%J~6e3t3NF~bB=ps-K z#Pc9ngA>`oh_(;Zc7&Dgk=7n)iF z$3D%Htn6J8cm_8(w%lNHgoI!+a;Pp6U2#dA&JHL0ZJG~Bi1Vt`C&8t0g%X@pKP z>rfxPkW@4QMiqZ!5A;aJnoj0bV;Ok%B$gpbECU~|8q44yB}KeRECU}RmeHENcuCUN zMkDxv_H6j=TE8o{-CO40jN$Bk3;U&Mf1mg(Zx*oV_q!2-3d%_T5su%R$U1m|FAYB91g0}afk z$-s=M50L$zIWS{V0#zFrjXVt^O|Z`g{VE*nD~@kIJ-RIejqV!B=(hZyIl5j=?7Sdu zsT*Cz{&9OX=(|c?JoqSwmg@)1^bk%1v@JY(<^46t0aQC?LfQM3$f2$$dJjLtPK^Wt z^@O8saeypyRO{mf`u*BNVX}c-5dY#Lp|%U}aPc`%Q~YpeW=Ft_RD?HDszgED3d_p` zvEg4N`kqHakTQxg1KYisx9XSRCgu?NhZ(7!PwjAg(4tJbg#85x%Q(nEB8ginCWu?G z$D?da;-Ppy)V@am%BcXRO~jx+No_HYN{L%F-o0@mZleYiH!z=m@SCcpbFg;|rs+7w z$=(58nUuCmN4#ZniM`3bP;FNq@g9{HJ7yRMkuq91ASzZJh?T9RSBO6hyvET_iMtWK z8z0u7FmQ+GXP$>O>0lg!g$Q}gnuZNs*$Rn%?CIR#5|Ptb4z`U@gFAV<0n*4MoQSqJ z#8K11d6g$J8*=K=9i{BYH?o;*q3p*uA?JkC(VNX6PdAp^Bh2F~2*7Rm0-U=glC*z^FOYyn|&>)IW%VIbdY+m{sloJZdZ;c*YBR&_e}*1mkm4;Pi)a>vWLL zg<4ySOgAw!Qp{k%BXpdbQSxl_`1p0Ff~iOvNh~Ls)b^nbRo;A zCgNuUYa(X!QSpa`4)+>`PC0r?p}n0-MiNck&2oo`n3ZvY#1%TiT%!7_M*`A^?CG;b-H8|Dvmj?iysN1O68?dw&D4GNX_Q zvwV|i2%ed&ZJw)4la^63${BC(8J2(q>{#GCB_?Y)A|j&zSA|i*$~ixLJFsQYi2UR9 zgd^fkBOgH`UvHSfl!zcKedclS(tSvF9##j$jnUjz@UTsW;&&A!0!d|~^Sp6`2hh{q zoohd!q2hGt)zHMR!?hf_#WJAaEtM#%F+>bX80GZ-azOqSP%_M1Zr&73!yelq_=MDM z0ohgrTu{7#kC;j1>dWFJ8J@&yrv~12RM=g$F`zC;a}{>8Ex+&@r$*d_gjc|%=!5~f zq7z0UlRTMiGs%RJDb#_k2SF$(mLh*LJo6MaXfk01J7G*e;)02yAC49gHU1Lpff-B~ z>b$V(vF-ZCZ$dzpn{zl?!Zv)3jhes))g>?*x971I4~0IDt0RUWibwv#iec`PV(1Pg zh7IRs-JKwV!?ST52u2H*Ke7)9Gf60$)z}G5ji0l$=*I#?~J-4awl%BL?DMD zT^P!L>I{eUkge9qQ;1XxE3jZ~%o2gkH=ZdrMyK^X23vLZdZ7CCNsxmr6(T#0=A>}? zRda~uq>bi$Dv_P>tB4+r=HS;bc+-F$<-4a1<2E}I#5r(s4rE{e#loS*?kgScL1ko_ zy-xQa<#8**`k_W7rY|2$Fhwl`9IWe+Go#pTg{5w%1c^lgQK4XgN<~c`w{*x4#xMA~ zm*zM0 zTEQmLsC1MhNz-U3f5HL7aQNsoj0{s56u?eTRKjXOO*AREU#xP~eY>koM7*-32|@cU%8bOl8Ob`YxY zD#&BOwM$MUh3&R=wSq}JCOK|tFd9vLQA~wFsnqqu2I}xR;?R?XOEJ2fL z80!}IhC#`)wA$8~?sdOfG{`AJHzq^Ju@utgwP0N3!#4wBNtvNhL4UC8LS++-Yz7!v?KOl-e)iF+^$_xBma72S3;?oqy@vcMJXTWh*m4ye z3-2ZmCRh#qB-mIWP92!@71Cv5Gh@mgL!=yxdds#067JOYMxqhUx;)-?p3js8`vH>A zwcF0>Z6`;%NK@hn{vg%T=@YlDzo_1J7BY~&ZL;b?BZC!3n`Jf{V1S5T2RL;h34^oDfWG}ecPpWXj zo+z8(=%L-wC=yK_lN2(ljnTjn#jNfuoQX>WsFR3F22d@W8C8h26_V@W4xpCGSfs0K zU>J=cAz%bJ7>GyN?=o2)NWRln0}KUan-FcylQSLxQ3s}kSJgK~)<@VCgjEf>heK~b z@Y@Q6WflWCTkRC}GuQ8!|wg|QKXiP`SF^oINh zh(D{J=L+^Sg1g$&Zs<)8@7sDXICi2wnG*vF%X>ux1LGk)@77< zQS~7H3eCPP#NAIVOh~K6u_8;XBf`bW3h_kFDKcmo8Ig)uDW0E*yS6-4pT ziLMF|fDcqj^zi-2+kvHjY?7|w2Xkesb1HQ)F zf!mO(IZS@)VuZlz(7Sei#4pT+hL>@-)S zO(=Sy1~hJE5f#o@SGvuyv&)P%h2$KhUjc&^$x`D~V)sx`PEM~x2#rMr^t(rI2*pVk zOv7FvkED^LUlzJ$j}jm&JjR8!7wiR;89Hq78AMy^APDP$Ku{kE7CqGbaH?go4QEtS z0E)n7GO+{o4;Sf^GivA4;+i<2S7+R8Z)WZ)k$o+62VeyiJ_QXx!{o1`dMsZvf^HVB z&p@FvGJp&NW;YGi4ZYTKqejFr;eejz$&A2O;DuT=NA(zJ3jJAZCc`Xh0ozxDh2Ahy z3`K&nTr5x+=J_ZKotH6L*xNpkMF`A(q3O ztWaqvC*lGOQlhW|x&$ z;da==K+8P}a>(~=*nq_k7-aYsAZ7bA#$osR*zak~IwB&D`eW_tz4NwEAA~(Qd<()& zjvm{pCV8Gf(Ur_o*cf1Ni9@p##|*dalvx>WL#+x?PpKbQgY;;!9A$|o-yJD|y#SlX zGlLi!84VtWAp!E%fivo9bQuTP4nR0aiBl4)2ipdOF)a|(8VTQR2!it*%TzI(m<1|v z1eM4d;OiK;SU(ncOwUYj6uMb&6Y2n)w{{=GPa(y&>c)07t8q;rAA^fyzSum6KjYjD z$5vIP+5hPXx1`kB&pB1@>>_Krko$-b--y8Otl1AYZlMe+=RuOM;BA(hj5e-EXpU8SV7xgRFuPEPg_g)B;Pk1sgNLn02$_n)auWdP zMgiW{+eE&sTgoACH_9Zf|DeW-I<7@rx;b^jp=q9GvoxM*uDfA84B8tsl|w9j;@q*Q z;j!VAR;IF(O_K^%IF0biY%JSzAlPD0M-0g{*EHGO48!xx08#XsAbNm7BJ79zB$@lT zs5n_YhfgHaLes=#S`nKZhFoo2l(jL7VF9M20!z7tybD5_`aT(@r0}NtBoO*fnawI+ z1|xGG&___yCHez;VrbjIE zX(L8L$@&q)X;{06yickYHdJ^$vWQG+;R!iH=vNHJXXyI^gFn!^R+DYj(xJ=TC?-sBX73}3?T4FI{qiqds}oez94+VVQ9%TvRuaFMf$ zT_HFp4@7J$Ekq4E*e}tN5VbG`OvI-iWZXEzf}^-~dVO$K^LZ>*L4XzqEDy8=yS+>D zQw!Z@tY|{iEH{7X_S(ouRl}feL~oQgg)_EPz(UuG5+yWZf{7en9bnsqDc~{P=uIwV zYmvfKc3}0OBqTAV*RWVTBg3|DxmsO?fo~kGxJ) z8Yor3XYy_O9)>bnR-Y#P-2BjTP@nkC<48xiW)oU5YvRM z5q1m1>STkpl{zTXDFX9B%T@vm>kt#pG0otN-=nSEOuRmB8x=929g9NaE`knAI6=W* z+HBhdcFXQ<*iz|PHer}y5Jm#~S6~22+ahTE#pNM*tf&e0Z-{Fp9Bg8m*sa=T6*91S zNq~?wO*DaF$F){WGpyj`zFgQ|aM3(dsHPSe3&%r)42#JUedji#QXMJWU3?s>ymfS?D1b_GzirRI*SXlk(M?>SxB{1*OC^ z1=L1W$$!!z_|?K_r9<%O7i|ldlms?WNH$~5`Ls`&CmkMCnQA;2O*zq=0eXZx3Fh48 zoXcX$c52R>6Q9TDoU@z_@SSpw2_;l(0M-K^6;sdQ56~&sGoqkXeq>muz=|?3)v6Mw zEaa%>(eE$EFdbVgkg-1(XAphy>AK|NF-cQBtxLt043tBL&~UcBvpWWnii&KrKpo`HHR=g^ zD7`{NazjtUd0_SDKTAxRfP$?Q2%vuWCk1{+j%nbmzDV}v={WSXJiTx|HQC`93dA`44|jjFm;A5^cH`V{&G8lQ_LJBWfL8UP9sQf}kR#>4U;a4$o(h>ggB?!{w7?1rQvs3zEM zYP7Irikin+EmarwIg=*9Hi=f?Wa0TBWKoEsNYqzG&B*u*Uk))|e44|TYnUd}WY(A# z1#J;t4Ve}d^;PqRgj47fD$o!`MVrZ3%6U^L=fH)* zrO&p|s9dwHE0L*yMs$t02X`^N+6$`-4YE9%Rqy@ zSb{U$1klJJ4Ui)pZb#y3LvaA6h}<#p_KLu%eu8z`TFz7V0ilS0tHY2pdF^N8V=v2@ z{51Q!3};rqN2m#>$2&Vz9y@fvZ*DHI0upyYRprmb4qrxJ4PKI$-yJ(3GT|^YHS_}Y z%W)<=fXWy_VMjM~y3Bcm^&FlBz7MW}^~AyPgHzF2u=Y3uhNTdbtq)6mFqpi-7(qBq zWccfFh7+&3^J?X$QJGW zgmL&Zw%A*^+h&$2#%!bc{nQ8KTugm18ca9& z??cPu8U>RB1qfRbp(7c5*8-_*h$>EBf@p82r<_Z*qMBT8rdXCsXiuklu_Y=h3L_d) zy*(2g9#zvodCNYTA1xhb2&iokXCOk#_Cte6(IJ7I5KylP=WhkQev zq#P%S!=F=SmNP-6ye7O4rdU*^^_17*)T~*F!gzxLrY4Nn$EUTK2;=ou{h&Shpqw{r zaekr#tAui6k5teU37%MC%?2Qr`61fkGE+~@4nX2mp@i0NQHXwLZKrNiX3D92O8QREMz zn2>WGaF=92@1H}q&zPJd1oIWJ+k_DA zPMFAGWWu5(%mn}~LPguQafZ<&R$weiaWvu?6?)>KfTibc(11D6$X~>Sr_Ju>ko4q< zZ62EbbAidRs9^G3UcNRpLsq3+K(MXQgA^}ZIuo@5^vPUm_(8)kd-E6ft(RnS@+p4P#K2g}Vjt>Kakg1GgAPH zCkEQG$heCk2MrA6@a~O^$+hCppidl_=E)_t@8W&7BGqIMbP=JWD#YM>n6KeIAP%a( zEA1BHe)@gs_`B>D0oN1(>*2}mxXJCX=z+@v<=BM^pnP- zI3@sz!0qzsOvpCQaD}Y48n62fgaMac{g-XFauKs(R|a%YHx-~PNbdzyi5s44D<5Hs zXh$KOvejy5BqL#Vd81`S>?8l~hv#wT@dNZ|(U1X9MXcfTz>?lB!7`WpPP)!vdBx2= zjd_3y9M|=brUQ!L3ZMn(zsTg0TXP!HF%edhB>Os>jn^^1Vxvcx_Qf}0jWs)9cT$gt zEeNn}usR628FKaj!T^Cnp*7f}EX`e{KBF-L~Q~Er2lrmV{l*6@Jk$LwM4;H=@ z1i=Ze<6G}o&Dy+17-&%fN`ZK?QZ2d`qYof8BRGx?i~tUc>R^Odu%ZxnNFx7YOA#=V z#$$CAqCiY;l5mTTtVPNqG73;1uq_DP`W2-IAk-ZZJk;d^Oca4?8c;Owz;M18V|iVt zPF6Dun5qjyUCkvqCvPFE4M05C#)I$+Pd6F_FaYplgrSEo;fg~o&`!_NG3N;3zOz_n zv{_tGNGaC@0vxJpK~Bx{W}!X3I09tGbzK*b883cHdK$ChIF2Jo4dW<>#Y6EjL~C&X z8dM|XM!sFmTVPXgW^C{Gnp_4rD42!_db&mzxLXO(@W5m%+};>9MKs7LIlN0h1Yh%Z zs|4uwPLq~*8vfNh1}utvHK&boSr=s@wk(m|OtGut88b`~Mg?rqnaL%R9A~PG_=V)v z_KXB^(pBZU+P?#7ohYXvT;14mjD%7D>UxFj#CaBvgDoCt>!~GV3NkPT8Z`YO@txOX zOhdRF5i1m?*_5^4ok!T2iq5|1F-C4%1#Kj&vkAB5?E5wx*s4n;7u!kX4Uw0vF&nOl1 zVZUJ&7%;Y2#$Eo@8owSqU&dm4a9!=qiZHfaTe)Pdan+?(Qo4c6YAKgA_XMh9F=i}V z`=VB7gW}5FU}@+Ey&l?wm>yK9aq-v%<-~>UqO9>J&tQ*xQmkUjA>Nay;{s#_?Jr#W znf(+=N*|=xII<|^t z6N0TG9FVPI2wHz@${tJ=bqrf+?gd1qv5Tgq#O|s}v76LneioMXdWNku=gg}5bRb8O zv}6hKOcqJiCZ(Q1Il>@65a=v$3fY}5$DANAckZm%PjhS6ynYh4|`NaHA$6l z2CE)ZS*lPM+H*tG=QZ+MZd5KL0lPRhkjAiwFR^6+N6HO^Sfz|f2|p(Ku5&F=#2{QrwFle_}g1vKlGaF}{Tw%83gAJK!R=ERuN+5;!P-F8-j*B}Bp_ zVcsS21S#Pg2$z8f;#G$m!owgWfMy~*cQpxl42;7;j)Ax4m^@(eb!1w7|llY=L zvL=AtP&GVLTs^BNekmp+3WIUO;{Ye~_y?Q9F)qk}gSgK{kV4{xcV$`L`NdrqazMis zELI-=XB#i8y;8rzGa%zq>Q}luwm4W#af*q$J74q)2aK>f%%5+SgHVn=CeQYqOe!Ck zw=JF_gY}4YWMjFxYvKzjB0j4Y{T+koKjADE4vxby3K&^psW})Zk3@U`sV3x}At9jm z(UcSMCH_>U=T$RKy`m_Wa)R*lJniMG^aZ`q!04A-&37w&347jIq*lTX=y9`RUt6^FPIMdVD#ci~sDx_Y z;nvBNEgW@K1jw*L6X+iaF3;8xS#axN3LgJsmnXKNtqMdPrDm0dZ!k(tl2&kY0M{r; zD;$tk48*{fgC&%ZR{RG`2+vJRD1|@$+?G(bM*5fy>L@k65@BR8Gm$h093b-I97y7w zvWL99I@a+RS}z`JNO&|Rg=!PO(Ty7gMJkG)+_)`r5eMtCX7lSoqZl5Z**^E61TU$$ zN=kF4bGoawao4%*ZupYz-#CzsN6%^COsG0l16qdUdQ-4fUaRh4;=8j@&Zu}hAFG!8 z8LnS%#RIy4+#&n10U*LSSM{g}HU&+8);!V=%lFuZLCH@M)sL?f8l;19BQ<9pMTqB2|fhv6fNM3OX`$+2+J#v? zOhMYwg&(Ob;#^y#f63z$pK_^*94bil^dbj~wbwW-T!h+7dm5;yR_02?L)Ddl(@Man zZB$Om8R*3BQSd}!p7(_4AeKCC-BTnL``D-N%(J&;Z*UI~GVzY- zT;^eu`G9-X?j~sqSA#e)#0(}~eFXqiEWsED&6Rn}5kQNq%H#tlEj7dU7$?_9cZ02L z>x2P+f(WH?!uQdAMA!zU`nz{s);40;h_F*H8u3ebPdJ-sFCSpIAQT;F@PPlpq;hA` zvE`y@=SLN^^{d)>=}lp^_LkJ9pxi0#kq>5JDi|$T=WCR4wAbAe925^zj~Rj^VB8+k zq2kd~N78=)BQUyp8h2_NsKkp3%kx00g)Fk$Lu|7<%SZ)sqF7$VZPiZTVV6!$K+cFk zz9T%DlGj7_z#ursb2DUb40{%%exeZ0cX&xUJC6n0$i%elv&CGVo#wC#~9@+ zCY#dXU^-yhkRfM$tr2n$5i&I3AsU-R9w=hRTGfmKcnctJecIZC4r1BnZy`R95HJ_Az3*h0RTQw}&VFo7#9 zX|iLv`omKFsl%UAWuT!_@R#(Qe!gM&UvJV1429G$jKYl=4M$_7*>h~E_IG=qbYwfZfP8qR2k_%3}>Zf7xkhk_YXASc)@Rn$%4BgSNX3 zu1m%d)+oXa1CL~{QWAWQkyKAC%QIDh2e$!;vcrT_txwgR4AEb;A5Ea8cc(o;I4WiO3J7zyV& z!-C*>>(3hYA&Q5p(G1my2GZ1ujBI0S0HD)41Iuy} zcHEG3y8;An?M^b@tsS1}#E+kr3pRDaxNAmy(fdq=qX17!>_pjl+LfhJ;Q!(ohC$p4 zwlOtQs}=f0INUnQ*v=aLj5y@(B$E2X!qy%*PkD?56MyC#jr>iWFv`aB{2B2)6g0sE z7*fp#=n%n`0o)3A5nkqiQJl}CWj4vP_Q9M}K$w$RP{KU@R@rWZjcZvYyQ z9V3~0f{L_;iJGYgMuqsF($m=B0t116fYB+$H#mfiOngIkS`>{j0 z0D)pM;T!lN9X2+c!U6c$Hi-oY3blUlR2!HaOw#|r05a^H4~D>$tArBlz|B@A zD<65AIAOy6=ZEJJSb?BW5V0#dBp>gEgJ@3#o5>BGU~=7=oYa}bo(Z;H5TVct^wa92 z)txDHgHeN?CAlA-1&}AtI-8$*^DKNZZs?;ESxr^kftCUwqNTo=Q1wG&xJ!3B@t+;2 z3suF5x_~!24?2XAteinpQsaTfJ@Fg@WszHfS7HV8a%|t7?&V}3 z?Q@f0&xy#y0ai%*_i1%lN zDVa~yWB_ItA&c0VY5e#u2T5xtt~22=C@3Vx64pqVtIlwGU{?>_mrA?^jNhe%%M0f* z8=4Is{C^T9&46nCUnEfy-|fWrP^|uUh}6UXPo(~@6{&lj_%F`?4x##|=X#ds5&#AO zhZxhzbhs}%tZau5skMzT9q4NL2tZ?3hKsWuiq9C@XBGv;ZPc{{Xit4Ay2ZTeqd=;Fv9nvIB0Zd2Fe9lbvI7pXidbNGC7Uqon`Ax${u%$E0dx@;LqygY3h7<=r8fV3R; zM|6B~JKcO!VB9blY|2eiA%{~V7*FrO>$)#7rCx_qjpz`*deaTY@Li|-Iv}7+(PRks zCGiqZVbwyz>@$!}4jM$yIjdKpIkUVQK zKsUHawG)EV&S)MpBJp2wdN3_Z92zG)f@!XH2|8)tcDNrn$YJs9DDe}D&Y zI`Of@k2;<1awnExFaC@nxX267ELgL&1HB&N4m6Ptufh*7W$gy_72{>JFPH>#>c)jkV_!SAi9Ql5)@w@%4(AP(kdVV~% z5?_;LGkR3(1GzU-*PPuiF5mG~+9(p2K7?aSenc$s~PC^lr$k6SA7Gxo!< z5q;a$)~p%1^r zcxpJA?p~Y^w_`@Z63T)zlq$tI;RIzoLLl13*^(1%lbm2ib*&20BXJmRS8aKWX8@v4 zPX{_s&0d;JIGzniNm_x}UN~9zCuX9#R)HtHa!JNWc#lpIsLa~M`2Xz{C)Qn&*ewSU zLDk=@Tr3+-hVrmiw1d`_JoEVKGiT6VoAbg>EtA+<(Tnd=>Bd#)kJ{Y0(b#}?aL8yj zN!=skhq3M!8p*`OD|XcQ9EMCX4Aq`4dQ;r-!^vSTXs_Os*G?8xJB7vz??QZ%$@$=; zb23kQ5Q@rSj`LpycL4%z3V@Y*Ia{PYLqA0s}N4SzdaUYWR1 z-raJZZ2QMe}G1KDnu>*QzO>5|c}?JqZM@B?}2A9u=^j($tt-#RC! z?|!0u)bpCG``i5`zuze`#@kblUcQ~Yd1a57CHG6`JLky@SB#Kv9QYH-xBXpybndt1 zqL%&Tlu3t3-{UJ~gYWMo+3aR=_nt%LuFJnKM?Zgn^qjm@*1m0f`OCq#O5v5KW$*bL z$w^1tCYKFAOSb*ZGFf}C2j!O&Gm^=zlr83V$t|0|BIoS#jQnxV=Cb~mZk2U<`sL>z zd_%Tecf9oe>u!1LxDhh@w6av%XUks?`?ri4eWVQC=F4(z%N=s-+84^=+isCjJH=9z z{p8p=+sO+vM$7qkoF`j+Q{`M98 z><{FqtM-z$hQ2IIci2<5dGrZcxL};LkGw}7J@H8S`d3Gatb4t@)zBtu-}kKie6yAE zopma5?p_DVqK~$hxxJ4|^zG|p^H)xgzu$a-^xyUunci8D{kk5JHJ^D*zP8OX^33_C z%R^CzbR7GNJpZ+wk+qiYDsPWiD6P-!Bb%%-R^|`+rp%w>$h)WR zBa5%{SbnwaaryQB-BLO5K56K{jCTMj%$hTL_e z`0KaGknPTw+xEUq{(SqkveVT+5n1bPdF0EBWy2j_k`;dqMQ&Oy>zr|~9Dd&$vc|-B z<@LAvTew)%O>tG zllJrEM^$QUnf%qmBq;nzHhTBF((;|Z$_BX$ zWj}vcxn`TE$)J40(IZ;Udd7kk@8hF5kIj2RZ1uFUi@<=F25rXUmt*SRtijPnON%XXNy)FO+46 zpDb%#_p;=l-A0CPwXy8mcaeO(@23(rpDsPiU(EKB$u+0|Ic*Zc^glkfnQ`Ene4rDZKffY&_{?IdrpgW$SBlQZz&5@mu$i zcTc}f?pXX4+4=5Y%EGPwB7eW4Q?5My3|ZEFh^+msXXJ>J`{j_6c9-Q(Z6&|{#g+2x z(9J}C{R=66^rRfL(|6>bnd@Z3Z;z3^_PklPUpZbXA6_p%xan=_`02B9W#1V2&dKkI zf6q|)ca)VUUK}A``rW0nSKE)J^DhP2x%*XFsLo4*<+yL|g{dFW5Sl%T6g z_W9voWaNiO$k8YHa>80O<&MktknbJ#y!8J=B8{~-l@0SPN5%Sx& zM$5_P{z-0Ke54#ZU9C+{+vQg`?veS*P$?snNjr{n~!{p3^x0cOr z-bWTZHburfyQbVZ`pdH3fm_O5hu+4DI$cXvm2owk*%^VjQSi??o*H4fWf_8aq}{AiOaWW?3C$}cwWl>Bv< z%4@%UMAm%2EIoWR-CGyzU&yjcTJX14YCnR2gjWX7^%<*sLbFE=0XuI%*fW^r;oa`IJ+B+8;%vC4{QRgRWW|b`WY5X{ z^2FRudHsnsE}LDxfxLCtt#Zcgo$~nQd&)JJUn}?D+bBa$7%ku1Ypjgg z^8`8fz8BY|9fRcFjfBc&PQ_UHiydd$BvR$A6X#Fuen0{-*`|q z+43&gbn63UTzOk*d*aV>%<=~LXlg|^9saoNam@nR`P`G`tI@r((X^*!=^c+su=ZIp z?C?Lxch5XX*8lne(zW{=(%al5?_Ap~%kOzt#x5BxW0$-kuk3fPEc?a*a`36=%jsM6 z%Ux^zK@R-d`SQ2DmdI}N?~unQhO%tN6d65ynf#{p7g8Rzo$USgWAemN=g7~#x>R;K z_TRE+cDcOtKfjg}qD~pNIFf^Z`m{`Yl%je5(rAy`9Eq*IkJk}`R%}kMR zU;U=^9=}kI-|=ku@WDyzVXQZ za`eHM$_wj$i*Y`Qd~U&|7Yx4fVCrat`o8{Y=Y%E{;>2Kx6*@0ZT z+taeavTwx+n*cR`JB7u@Oj^oxwA(~)AgIn!go)V zmOWpV@>_Yi_QH+idvkY^U#`8IT>IgbGVj?(<*a1~%9@?7+&2E_GXLUV$jpBpDqEj& zl+63X71Fc$zOvm0`-qJFr#vJ_%JO%niL>`k^4yF*iC;L&4;DLo){%{+-pWpW8)*>>=tasEnypQI+(23(my#3uRKc2Vox3?d9@Fct! z8TVz8e}CM)$q_e?-{xma&U?Rj^V|ZPWKX>9y<<;}M7L4Ed{O2B< z)+O=J&hdkXHhZ({u$zw`w#zo>PuOyckFNOAd;MMeJ^lCvYaB3R$$@=GPCW1R{;mhV zb6j}TH%?pfqu#?_8Tvwh*Bb8~yyfl>ojhUIkA^;c)f4?)|33A@k$v}_wB(uhmTlPc zK!4YPfByS{JHLOxgg>AEgVxD+_II88@D}^Lc*$uK=D&T%-4k!>?>hA8`TJdV-q{nH zPWW*8*RJjFTHe3n^kLVYI^q58XKsA@qW-Q2FWB_(FWoe8!o?3hy!q>w^mhe^op!=l zud<}=;g5gwo%8#<8gJZmquuvBbIES|40*u$X@A$W*LFU+zi-zi_ii?Q>5V7$cdhx( zU#}l|*H4x_H~NZSpEs|+>o?Q(ef5Qh`zGvm>M#F%&QbkcQ-1o(*M_$BEjfC(ITQXc zv%l+0n?Ez{&2@jgce;kP`TLx@P2VNQE_v(cr|*CJdvA0dvg=pw z{pZxvm)vv3_^lSc^+wmJ{~v4b0T#s(Hja;qy=#myF}iBb3W&YrbU-{5LlX|WIhMP< zW5eC4B}cJDab_kF+T`6o|q zXXc$g@4S6Tz`#$QRnB&r&sVgqz3;*&yDsQ``$UTK>9U)hFBQ0OgHMiaY}6>*d2@CB z-2wl)uq~qPs{YF+Ynykj7xBwg7oNek9+gmUzO!-WAAY)vExc@IpM-}|JarAz!<6xm`*2DIp)HJrb?ezHGiuej6Qu3a>#`< zDy+Ep&k;_Wr`@^v$9*mwRIfIkbzjCbCv(n)IW1vIQ*yimycYX>g;1a{^P|hE^O#td;7kt z>COq!O9!>v)?y?YGW_D=wbVe(R#q z+L!ykJ6&&$3s==QzW$FfOWQ`@|4QZGUHI<4MlqB3ch+tV8#<%quP)pr*VOJ-#|7H{ z&DJ#FSGn-A)NFddjakmBms4~8`Pqe6M;3m*x6>@=7+CuCG8aDH zd)Qc0({!g~xPRNR%!Su~Ju`QJYl>4E|8xKEm$~qaTk%agHk#r*Xz4TP-ZB?Hb-L=8 zhnHk&@yM&u>~a_W^ykQ&%N<8Mr!TlCxX=MT5A^PTmNMPyI^*@c_98}*#@Wu|k@ z{uW^$uX15-uy8)+^aAIDZ~892u-b*uj9SAjYqGT2<{20E|K`G*?iTHAzje0u((<-_ zZvNrI?t;Bb61FhTS$#Ku*m0c;_cc5kI`PS9ZP&C0*=skr@WftCRyD0U-?`5{uln#! zF5ISxb>?5cj&$DKdvtxbtu8z?Cu>Y;3#+qs?1965x4H0drH`A3hJNFWiMqOM!gd$_ zVvhNo_0VMJw#tX>4|83()vPTZDc&)!p7!_ z&HeUHbuu?cjy4~0;WHhs|2w7eOl_xt0rwZ@xp0HOou^LxJYtQ*K~h;Hp$uV%NNC|ZWrz{V@UFK*F5KdN-KiaS1iOwuNbPGb4%KjJHN!qL2EftyCingc|%DsM> z{F9`ey7I;4kA@WD+RcNhALdPPE}D4o!+#SB@gLvpJQFi)j&^d#OBHUhh4_o)a(@i% zIZ2zasobMxVj*tYdsF3RXC^oei${0*ZcHIQShL^fIc=vq+pH-0AZ$t@J~gdpg5MTN z`vlJ{2+S(PDZY}?k8 zQk>YGv_AA+c8)wbZ{5=g*FIO!+Ek}&l7i)S_g3;>fJ-*SMmZSaWZMK{* z!RW8`Jsv+eC11@Ne%QIK1S9+0zQ?{jC8vLUp_pD;g3f>%(f>BnhHvil? zAz$16=cVSgOE7xYznyL23HjXB&i&>-dW_MtobN3mC* z&N{gX+aF<+ck0N&>xbmOE;`zM_VXi*vg!o}>ki4~^OhZLKKl_yMV}_6tv)C>9yMw2 zaK|Hzg8H+z8VBX;UDt#k8~F&MT-`5J$^kj}bmXFpFo1usY3|wm@@I$L$40b&gpoZ_ z`0Ekl!_T^yNKr+m~Bcm+mOW2%Q}0AGSyS{gWD_8vhFUiOw18cgsPWKS&Vf zLq0q?`;*;r$%?^)pMPD9QSizv{nTBu{Q0MgHzpNhv^?Wk&ZC|3_L8|1m&QTfUvnT~ zr~G-9gTHO6aU)z;y3-DLrm$kWa$B-l6W;7_qDDw;F_QA5e_`j2% zQstok;BTvR+aXu)9~u7m{zHs{OM5SBxLszMxv?eZA7Zrpe3Yg1Z+Xnv!F5~geTY#) zdn)7dHu+A=!_t@y4>8JYU9dYhN8a+We~ofKJj5th*ckT9U-EYQmeK2HJj4k7v9a9j zt@0LUp=L%Jzje^Z4>6(=7lqZ@Ag}8@>EesZ(ErJ=tk2iU z8}P*t-;+fc<<<+HzT#K;<3-oU ztjdLa`A$Qot&&f4-8ZrQx+07+&sx4r`C0B!fBB+IOCisM&lvHOe61jM-S4xY{|2Ui z!&k^JT1T~CHX7jfCgch~$c^T9nYH;V$j8P$jQ?KtFTMY>YbfN$A5ZM{t(@h5;%Im{ z%;&dRFIs;i|GK7V-@fjUFQrCzm?i((F{3`yn!rbPX*5ahFWznS*+-DSzpFyMba~;1 z4=VXpf&6C~Rs9m=kDs@E+BClqBa}7iQ6Z0|yY#NcDXhSK!|_98S+%>%u6vT;A{*KjXzU zmj3v$?4?oV7{-k9pe-#mcxpLr#46FxTLh53&W4=~CdJauTLEqLkGFMsoI1@N!Bq+Z;L zkNPL~=8ErQl+ZXT^!FTGK4aC7wjGeK`S>$Wi`yL8h;!wJ?EYi zd+Rnve>JUVt(Awzpd-tU$+t1en_ug%lssH(_l}FX*|#yWPx&GFbRK@uu*Gk!B!GX> zX=>LKxXSk%|3J~VG3wvKQghJ>9N(L{9p3gfMwv(Q-OoW>x zoKb*L(7G4VXHMZ7_0L{9G^hZh{PzF4I-bVKTcXRMI=8TPc%OOJ@?*6NH$_%JLAP+o zNYwwXJX@Q7dO=MTa|>_%`RtTc>SYq<=w({=Z7zCj3;V)KdE@nU3d$}1zi8wFLSIj z^SddvP@MuCn^85z>>jPHcgOOdyL|!vZ0h!w*U>EPS9QlG-v3XdH=Z!Z{zOvQF)`sPt@)=Ja2;P z+`$z#T}z$Ud%8AfbXa{9bO)!FU-U&(<~S{1*>%eueFq07x-S%so$EYNPkYtP-@)6u zxBT|E33g}A8M?>rxp#2kYSX+=GN(Fw2QIqeUUvs~n?Gzue(+SM`9=5x_i@1A#XGAU z!`gvM9#W|24z4zQTC3mi3}>bFy1J;&T|DUGknH|$N&EcC-<46&UHp-)-HhLN%+h9j z`du9qa~B6?q+dDoYMQoxV9BS*eivW*{d}*~kh#v-PD2}`th=~g>aI!U+fO%c5D4!vd$At)}C`8zl%57v+5P}p5RRP8|ZQu-o-y%t(D)q(pTEyHJMB9+V^ly zwWi$5r zQ-&DuF#yQg->U3c^YT;tV{z1?aJao(JIIS|<&;J&qY zMxM-hc9^kQbv%Q~=o2Bj1&)Np%J-~4r+<*R1 zZko0vtM5&B;R8JVNbJ6oh|?~3^7~PDZ5KYt22jBh$2qSYh`r=)@4}`kpB&nWMr#W{ z>{INHcHx1)FId0YEoke0IpemQcj2Ks3O9&P20LrIf2fPHT)5nU>R-;zoaF4=@aJpp zbuPS_3O!Q^rE6D=tM@U=bK!4B3`%%bKE}DnxverPa^bqSx{T}_JlR=(_NWqf?Lu5} zUSW?8FQT3MXWVRpf(mhbU9tF&@iVjwI!`EZM;GGI{pU`H;2GNFh+qG6^M%-vzNqoz z@`=unJ*EG+=N95G3ep~*_&QBH_TT)I?sbK@!%t(rZ&Z$V{%|9$G0H2%8~!-6@ zw!`lHr|!Z+JZ!4HT(^pxlS^P8yK5KWi`5#3uNgbTIb_t7NAC7TIRDT8cC+KYb2hB` z-%EFN5gxwaaOYR@ciIV&M=KzE5k5TVM(#v3L3{gfajAQ55&qcGZO1jY&BQ4Xyy z!uML63 z+oaQ(bF?EL-+b?z3QR$5An2e6*4o%Wox(o(B_%@_(N=&we0tfrHm7u zp7GdS_z>S4zG|sFZoYQql$kY9onqX4_{(|8nai{lLMK;1LB+U}yzJgC_k8W7Cr-CJ zx)=+Afny8GrD`uUYg6Lpi!oheTq72|dx<+X>(3(oq+_z%$fVqCsy zaij_y9zJ zg2p??8UA?XZvO~3*t!*ubkB1Rviwp5#XQ2>C(LXSz#2Os>i#C`k`?odyz)3f|2?MIu| z-*p#0!s}xfUB{;W+T&+sTGN-dq_@^kZ1h?s@dC_+KBIoo`e-*e3 zORyjNAam>31ANR z-qVpoH3}504k6h`{Hiyk{@Hf@_DT7CyRF$xMz>U;31nP=PPUtxWxW&`_o!d>hKtypiw>NS{qzM%&UH=I-5}!v zboKAQkFcDOi&jtQw6AGXW!(J7uwH=nUt2n};tAP2tm4Z(t6M5q@SnK$inF`X=-|jrGy(8>+hj z&>Vo~MkhBLaa8^!>CoTvwl!1QA~G&Omw&(i>D(i7=Sjz&4b+t zkf!W9EYICKv%~tRMoQaUfaU-+H#6$sqeHSe>nJuxZBy`SIY2J}bp7?F&K8H{yF0@! zn`UX$odjs<`=FU*+!KD)8^%0dGA#C>{O{?Veiu%(RoW(yaRHh~G0VptkgGnwG)~^` z*()+GK=bRApZ@!PdB}h`A?uPawDf&Yii~>_dTvxu`|Lj1eErX#zPeFI85fap0h+bS z9^~FD_lT*pEcAyaKF|yAgXR~*dI7qA#lJUO?v>{bubegWDy6i|1!(puxMhc}?IQQc z+Vn2;&KeEXeFkXG`=HAKdI6wWO{o7;cFR5WtsR2sJp(fC`S(C6GA=+fe{ywY zHpo?fnw$FNm=Dx#2IvI>t(zNEZJj)CPS=S)3=Qysmc9>~Uj+Mn0eWtfa;4Aja)kZt z%~2B?D{Toh=Y3GHvxQs$Xl}btk69j1K`=HCoxEG=AtDLDn{3QSO zs%6R6@+B%g0%*?rpqT)@c+s!=hv~HzPgo}3s5*8`%gi>)umt%0J-`Gq>Lpn1naZE! zd?)w#<@tR5?AA)tV1RDBO8~m9Wg{Fb0*P2)2)-Sg^#1GZ8%miTeRXD9>AN;mpHU7Hgu)KvC z>H>oSnnR#hj*5r>z?EB{?^m4kvATSKULep)xgGAV#iKtz7~Q*WEd`qY0I&q031KHb z|84_b{6oz_BRhIH4?wf8!L2y4s$}~{d@kzJ&tD!;?@R(fa{#*hyU1a8HsdCrj?`9S z)j0e+=7qBFrO60rUbuscI8@?B9X&b+^_f zlS$iD~U0<@^b z*ml3|#GSLd_nos!-Hqh{y#Ub6DK+R5yYPxl;}2VxR!~-y3D8o2W;IyHBfT(0TSc zW{(UG&~5+vRsXQZy>&Yd;1m9Le$)4=tZpO#F9PuCM!f-&!|=i9I~>Np{qslE>@KR@mNDkTeo@vgUPrz zp>2V?R@^Zh^f)A;RlQxx8BhS-cGIu=hgWW_DLIDg&iR{Z(Cm=XF(Si4)AbvzjkAyA zpldY_-9P+^imta|Sbz?ROz57Hhv!8-`0&ya5^#pla)4&v0%WEw0ekZBox9o(XMUs# zm6-s|0qAW&xf8f?(a*+O&D2Yj0MH8nefG!U5y>a;_z!=6dh~mbfJVjzXwooawtNC_ z-QI8DgI`pHq{z4he$^ZPRm-OP;3U4gEo|G>sjASCUjX9*H0$!1q8%skW51XI&zGo@ zLoPrs05sv$KMy}Rh5ynvJ2mRMdVtFTS_;rQJ5nvlr|^NahwG1WbyZXcX!dQt>J5WB zrN!<#g-2|!v_=269vhAu{eAAV=LRuWfz?58ZDXZ{HE9^qhYS)(gtIKrh_!tA22L`Gld@wx2nJFEcBvw5?ukJ3mC1=4?3V|8QlvjIX%E(R z_g{5m`Yze*8rHh~#$5dDj}6JKKYNPN`lBvG)qms_M^2ib{__N*ywvRrCgsaBP6yTM zwc-gz&u*^`EtfA(w&r!bPlHRnde^(;_{!l0;Nsq2 zW3BtL{8x0dVHN)Xm-)-O-Tf}hO}GzhKS%)=dC!iP>X+m$&#Dit+M@)c^#Q}$H@GNY zSwlaFE?0t4!n0xHnq83dM)#ld*CB90r>u_*IxoATJ8kN*@G(aDzm`r9IVZDc)?GUc zDqL2h&F$jP%5gb2qRau{B47Mti{vx%FXun4TkZw8gblTsiKpeRLuzbYco6bWy4_iN zN*?fX%Yeq&;6k1qw_@8#d2@DtjXCTijDimZPrPtK{-zISB_A(wgbwrLiJjQ?hG;sNT z5K*D!Vfkj{0%P+pi!sWNZr^y!L3u-iDZK`D1{ZTCE`N5v+pHkl2j3}KutyH+QGLjOzrh8Y|JjYwT{3DuZc^L=aPhtzHZ*aktUY%ow-N^~ z-0hC32P1i#)ey zUK(`?T&U&#OvfAL3z}0J9zY~-V34r<|^OtzsVCn-+M?SLOv~e z;i#4Jh)-(X_--KNYwTU!VyXP&9BSvhAjqe;Y5CPWx!Kr_nmTnLfBXD1Dosvab*OYK z$Vez=L)@z=c+$fsr!&%rK?f(JM0 z_}fN&tJn6Tvo{|AJa73&Ie24t_xLSo4=@V$yLd;~fkW>^3Y%*@z$jr{%*o3;vC&cM z@}Dd3W3>Egh2K;6;DLwF-kyWOWqCat?-nR}zcDxJpVv&THXVZVb>QO~M70?y!K=Y$?F zfO+J_P+0@d;)g?@PAzxpHbzBvkDg39hiy-C?%kYn8>6hRCoh?E9xuolvgJGNZHyBB zi7&VM0&ZSspS$AY0tmt2KH6~+A8PQyN6puOOK$nv;=D_Emivd&%a#I+g5_Q-^DpB$ z)pF_~>K3lJrEjI*{ibVw={(pECEUWlK25y4*iCEq7Hf99m)*kKvb2YfkDZ|P8KE&e%n3=HeX@v{XKi0*|k3h(^7X?Ph;@$i3_?zFI$G^TA0IoOdg%x#GTd7i$9( z?jEQ&L>t+w=0kVWdw5BDt2I}i&v6d>u-P%U;U4Z*Z(pN^hvT%;t(Xh$W%n>Twjkuf zQ&GEa^^*D^`sFSj*=$cWhc^3K*l;t9P4C2Os{c%<%dnmIh^zQJt>bU7tT*C ztlBV<#t=vKvCbN~4_cw+E?jq!uIj`|vz%iNmsnB03qMY6u&`C; zT<4Lm+odC_5YLU1gbU;6I;-hwv_uJo__DiIE0hxK>~v<4LOCs`KPIJyX zzpN!nD8>^`EXZj4YL@omtzX)Jp=a;RFH^+wc4v>?ZSvjMigD>Le_re{X0dic(dDP^ zrjM{b_A=M$;n&)PnZt_QhDZ2bhocdLD*WI)vbI(r806Acp8BGo$f51}=jVQ4-0L)L z=?90uUZQnRJy9M}k8$f>0ms)>8LQnerh844@EHG+%}nlAWrBA4vO#`e-aP4y&KmK2 zy7N(J7e6qhwux)_KvOr}S$kahBkrap__982()o#twEV`YFWrU`T)A~#hk~k8o!g#% zTI^m{g3FHyG0y@G@~@4W)#wh zftpb$oTFm(0>e;{6?<0Gj6z?DJZG>Aj3i|smNT;)D`lWqhPH;7)l8y8%s>)jvl~Q- z7Z`(y6&Rz$3mJ&xC5oof1XiLGtqfHL5R(L!OE%aT8&u;R62+S-HD{(-D`TQ0o}#T* z-bhOfrEyS+84@G<)aoBHC_JhkWwg=`k)e2wqA8B&0%(gxU@Xuk#lZ*}D4df*d3y$m zBfo?CGNL5#87R_Wl~}tq16df}&Ra7OscqmnYX)*~RyLWjW*}wK%{5|kDv1^;8!s@F zltgnB&oPu;U~ROJK_xR8$jl22YvD+mY?npg9d-jP2y_N1NPMg0HK1D2VP$E~M0+P? zNKE9@Ju_5$AVp$|m6aHSz;LvUF{C(ni7~V3$S#miMEDmviWjJU5z&1pJ1t3!z)=#< zQ%2rqw=(I-!6h;2NaP)yiAhJ%c7_XaNIXk&v_s-;w8R<>W%&Rl$qRN^ypriF@b+-h zf;FY1ScbPVT(~J6+30i>$=D+4bQF}1LSuCydL5+?=@X%&(vgsc1U?Pf`834wX^7<{ z#=-~`pUMbkE1!nUlGvpuWv2y^5hzijjmc%OG?p{*X-G_>;SAGhKw`^2x#L?tT}_yh*TnDB-p}A6f07+QY%!cMVm|%O&NKs!v?$M zfvSDP$HeJk^`1f%X$(eV8j2B^RFq7`#D+(P#15st&<*vcQW2%GS{+DZ)HqO>7o7!$ zHc@s-u+ervm0lVL(ge00)l=g@(Pp#ANPy?+T@Yx_0(c)7sBs{cOQo%>iL%ochTa*ceWtC6?#lCV6j_z}R^~VoVe* zk|}WBHfDigtgI+eR)#iHM&7|m1eFq53*0$R*Xng*D2K?1l$jUEbtQe7y_GZ~>^cnU zkSH;UcUVnSB2+>Sj*th5hxU}&!5PUEp#eupNem@0DGo-285(&`V$vm)NQ+F@&J=Al zGImL%;Nnt=Rz49BpHzs06$w#LcM$nyIR*bEhGIFpLn36$Ah9-vcSxRdHVM4lg9LE# zdr}-v@lp~asHGysq_d)g?2I77SH1d%WG2G^8->XAP?Da7C^;ijMTq#ygLCpw^DA9t1`m#2L69EY|Ccr}@W3`ebM{&G~fxBY@R59{`;IK=m z4=Y9S65MoNLNT<3A$aJ;O@orP(dh=jM>QMEh>R2p2#XvnC!sXjY5?M9Ok#}5cHUFu zlMO9{#L=mYQpbDMc}}8PPBbXRG!@!k7y1RDqbLb9^rjJ`qo~m6sIZ9eP`!sQnRrH| z2oa(sV-iJk8NQSPkTpIkd|1<%q?L0^N43E=?M}_JsFYi?GA&E|9C?=gYO4ba*^Ju~101ASPyp4QAV!|mS zZ3T9n#!5+)ndN9JJBl$;c7cH@n}Cc<%sVSb#YaTEt6n6X9%C_b5;8Gn+F_NbRKRqC zC<=8Rt`SE9e!#wx1E4txMLI;O4-?J-(h{=&8bK$s1Np#HpHe$V{b{R&V3G!qY*0A- z06PrjZ8layG)G%AL{>CdS;&gKSu!Lsv|Y_7(Spf0{}t~LINE9u*->zJp@~A;BS<=q zVUqhpS(FDsyp&p0F1!d-kd;JAU@b`!N@6WZR+5TKV$D*lQV0-|k6_H60pNBq>Yel? z5=jbJzYQ?YbJ9>QZ{q09(939L~HjPcF`ps1!?yR)5x%?hj~y9GvK|tNhT+S{YzxHd;c8 zOlRadBQ2p&d=ZPht~UV9qS}8g+xSev7<` z%6Tfs*18R0?X19v6wOggD#Ll?8{X_CH)<1&*t!kHF^q}Sx~*6g@Hth}M+o;Yd^f6*n-#XGXB=c7eCRfq0q-jFq89=Krx3 zP<2+KB!T8cSUAu9p9UKg5*i;L`9D|X9U=$wGdry0%ce5q3-X&NA`zGcE%Acyl|cey zbO<7w${1Ai6%h!7;Zc2cLv($CUa8uj$_qSKi|45Ln7*KR@B-CW7opSZeD@1@0Z=mK zN+^9sMeDu&{{PZxii5VYW)}2VBHMzd_BIjVIVwB~Gz))q#{T5e{O@xPO=7L8K44<( zQW9li&1MD`!zVMGF^Lv@G1X}0MK3x!K!;bkbQxa5RuKp*CNU-|kv1kf>>{F#yokch zF+9*ILedQiO&5_@Y*2UKhidWyrKoFOSzbh8;SqXWtY>p=G)NR6vlE>`6(76;$lJ|< zcbkBc+E@-)h0&*LB63qcGzrD&B6Oj8!b<^Ttc+PA8%vlfQGGZQ0wfdQkF zhp52KR!kNDd%f3D3;1`eP9Gl|1?DXeWlN=5E1@Gihz-7|=pp%E@hnF@}18CW*MNx&}>c`3;=v5xW0dMyq=1U9vt4+(?TP9 z4i4}?l@?f-8qgk9&;$lmRty$A3oo#6E{;Snv_aD62E9ouV1{1WPwaTmz;mq(qS^uI zb_Qb-7)oGliMD!kCdMK#jEKrM5(o}#0?fEMu}1~AZBC`JR+EtyOuz&{z57lBf`HZE;J;HQq!t*ODM^#r)sQ?c;|YJ3tT-ZN7I+(U{`MNm#01zT^^$o>HbHJsbX45* zYS@&ba}e_4TU)JXVkFv_^csYw21JK@RdrVMC@WyM1a%L_Qd<)i;x$x|TcTX7vSpb; z7gX0ZI3z;hEgBK|n2ssWRpU7?>Xspbf=6YT;1Mf8Di#=+pGOt+nCXewX@Wu2x^YAV z;lLL_4BfyR75H0SsNH*Ocx6S3PUHniq-Y6fiDHcbB#0iQi&YT1uWksGC{h@te06H| z@Zp-r0W3aFN9lEuF;t%beXLGLg+<3gaXaPZjZ_xNJokbMFA3- z1Ox(KNrVC#q|_}FfQWm6$S+A?9sgfxDH-bZ8BnxY0t15r^K6$#f)q##2M71u0wSeH z>@8z`M4}TJb?zVn&|I>YP`$QG9xeT~!ZI@c8oDu@kvFlNh16A#j2Py~9R$IWKq3dI z^p)f!S^$@dLNMAB+Ub=bXuF-E$UUYbr+ys|q| zR#ojcyos_?W*p-PZWo}-INtaY- zjFqS1qWD&OM<*uMM8+~PMl0AfMPm|UqsfqjFDPsQ?k||>NqInkLaHh9Rk;FLfxmLe z$wkr(4AcxPGev8}*3C6(%|Y;O1I^O}#DYl@bO`u39IVLz2{=RpZD$R3NhqU9Uc%!` z?uq%Rd0=z6V8q;nIJnBqHTvc~DKUeS=yZkrcrPfSaY`5ir)EI8vGNwysOC+qm`sJY zk5;p0E6)qy;${Q{mR`Xlw37e{8!bv@MQV;1n+3&Vr8vw`STb>4F@o2{qTFJzTd0mh zSQy7BW;)`OVgzJir6fmUppmz==Zu2G$cXJNe1M`gbP9|CuNNbrL18Q?5J(pn8WN-P zF`Rg<@iYYjKUfHoXfT{Jup_>-yR6u5X9Uk}1J+)qauEisD#RpSK(AQbkjNN$&g3=8 zf<(hfNd`rF@;C%&%Fc@{vA)8rEsTIXHe9&HoCK!%|DiIGw!pw85EjTQ4)(GJAeC|u zDFWUx&j?B*h9mdj|JEd2om?!KKor-d>cS_IIErO#G>~E8|7}K!m;e7|LTU@zWD>ky zOM$TfSwV1KD1)H|6l#UvKCBcC=a7Ozd8;*!NkKZR$mmj#z*v}cJ2Ejw)<#>wM@ZsB z`a0|=is$WUfWu}dp+4R!Mw=ZO9Rj!(i8Ec4Xqz2{SS)c8WA{!3gd{6%_uK`6ksJcY zau$QePKZoTjh)aW)Q^{V2n>NAqd**owAGMG8yyZC#SjkZF&0ubI-L@DULsKlyqOxs z2)yDY9tNqtj8TC3l1x}~xS1qGCWYgXVs!=GD;lUwU~^5HV#I3RS|Lo}i01eD1PTetCwXC8wx4FNuHSe8&Ljo8X-Q}c?E zD9kg0Dm4(Q0R}-IPl)O@c&*S=Uok=faZaPHLBSj_hi7E1x106OF5q3oE`!or$5+$+) zz?(egPLHDGIeFl*eRW|W@ez6uVQK2EiV*mqPT;jr2wywHyWn2?7MYrgXi5}CoIW~M z*Vof(P^ePH%ArIDYn50I(IWgntepr}0YzFRmKK^=rjd;;qdE#=BY$3CEi8uy5dwvP zO(I>o-3niD7bv70D4H5ZM3RHZZ@4**9mVuvq%?-%P!el0F)%*O5j($Vq&X665eJDu z5YrJ($Oo@-l6No{6wVoiL4?nEd7J34A==7{HZQ?U2YVy3+mO$pl3R#WpB>ytB+#6R zx1kV|Dcoj5f+VmuWF^0`A>na4m0G>!|MC@>0*FhOwxJks?b0^l!6$hLDe%tF=CHzw zk(Dc!@XiR>YAXUg8{9D`L&$V|8LA0}0D~LX!jMa)1?J0n^T9J`qAurLa6{0I3`pV-%Q%v=~ zCqTHC&Yr~B7=gATLOEzF$Pi)RsYDzEU_f_7DDgOckigJZ5{QiK$%s!zVc;1|f)E?j z7vdiU#zZn454)p=RmF28lju_v0Z={Y-Xui=<46#M5}+!uQic)~@mjE-QX|E|8k5yD zgs~}sKX0@SQ1W5PB*F~oGwh6#8l-%P2noZ4CL=bLl~|O>TH!;JU)Bg|u)&M)fnP;T zV-dqiRu-J=Ru(Z<7A0C)BtlJ3eHMw3V-YPHSyovaaTp^T&q+xNyC&hs1}no^q$I=} zB}VdQ)W}>CaY(5((o9m45;0|Wh)Kw7gbt){|3Ax2o&`<({ZL zG1Y?TREsxnViFw|#Lj1D$^j~t4Z@p9@Ea+MQ$AoDQ`C5GSfI*1DWF&&50i@nUys=e!h_7L1rRn-VfSj} zD>$?gAx4FVU|bS5#aZBHlbW7ll3~*AG-pym>ebdBI7C}<76WaupjahlB$ly&6QipI zX_^HEcIX0M`b@i9J~ z$ArY{!}a0bh*U&VQ6c*H*pLV%q`>1iSEN*qijIOMD6UI&bgx{YMEHB0P$V9KLB7$R z-kK~~r9&iV77h*vkwKvlri9G2Rb-GoNuWi>V=RQ=Oyciv-L{R_TBwTn;Fs!Lwk+0^ zPK8ECL_qi#$HQ$!@C3$YCw3;F*P!))v!BRco*5xkhb=2mSPWzGmZ(wDAb@-0h2W`y z*Tn!jCrGj07AVRa&hByS5sl3|D2Xri#CVhy6H_7asgqa><%yH}V#3JV7_YB|SYj1l zB#6$ez^I~fC~t%CYv4;jq){9rL9{C*`;{d_t%9eN=a2(zZm_;@}LlEg-~-vWbix%05^q)ws$B~oZ;n?m-eeIB5AuFWP;Y@Jw-roVw`OAIwop7l|6L- zwJ;`7FvSc}Vr*)cuZ13Ym04vSdQbF^%BYBZM04sxld?KyChdQ!GWd=7-T$FC^(B;# zPv=dm((A7;;MTsg4FJ3@B~^Jvt;*DyRs@UQ(%0VDbzUIO6yoSpPs7JkgjxvaM@f;li=<3BIwh?jkm?HceO(7`9j~Q| zvN57aTNso^6T`GO?|FOYF*G9`ILu5S-y02t-yAjG99810f zlqV>WP}Y}CWlcakXe%Qc84vyOM5ue;h7s2q%^4XK4L^E;HhSe?yVa9PGze;roMWPT zj>M_=sUFmG+>aEMqkxc;%*NIFr)u(sf*B@PUYSSEtLw}uju`*ziEQ#x^$p(rLzNx&8YdT2u&UrK>_gwmu1OC^SBL6a7Q2_f~0^Tf$NbqF`NVs zgn^+^ocGl#H7xV(wG5C&*gUv$8WZxT z^nph5CBL-BdwD?wQ#{ie(V#IQT4U;kH74YF4d9W#J#iR`iU)_n9mx9}s9F}42qAbt zltH`-RKja05OI2GOendv#)Q=9F3muXEkFsVA(Y8k0{Y^%b`o=-s+aS)DRb!{b6!)Hvcdhudl4;fA6_5K+Oc2DnTfaY8(Y z=yOI;O9SDc-e<+WqBj-bF=r$)UK~(R0C35X7@$0eq?AAr(U(a03PdIp9@RQ9Ft829 zSV^pi=YES2c#`lJ`q7c40r%kha^dQTq7jo`C9-I|9T#* zR7wd6vmp(4$Mg*HE4MeLi ze3ke%lJpF2qdAEcMcPAc6zdn!nLI^m&hv+aJb6iQz#j$iWIZN4wRQ}c<`vz`yJ}is zyq$XftOi8%h3j<&WFW7c48TFb9nW%r#h~8}*93M55=Wv$7W6%Ey2GD5ddJ@`P@>_{ zaFDs-)6C+pX=WKY5)B`ThNBROcQC}qh8sfS_5Gt`!}UW^gf2`U5E<=*mMAnjGA24c zsxL*b6oo_$MY<^X3p$2=y6Bkbh@oYomDf6=nr{+;SPCRS4hW}@$N!CIUMyUGYliQW zSsLD=zV#FZvG0sd@)aoF6T$DXQNI%$5fZ0|&DHB-Bjc#(Ft0j+JoagL3OxkeokLI@ z?1pX#SwdupK0K5nM(dCW>h(7VJ{8^o1lYdP1a%$KLle|-i1MJy_qh~;7dDEvp~(NO z+fY_Ik+GKjT@-kzgu=pO~|M}_NAWV~KC!~kW00`Q3lhhOsc0XpfYbka{xu*UlU7scx0^s(Wg zdIaYe8>b7^mk}hdM*+lYP2CGQ4-)Agh@dX1?3hXOP{eZ0mnJkeN`rmPRI zT%^%^eB|&4j^L_jL=UC+N;nGUL8y;vn&Hi=JgM>S;p-3AGQ{XSWuWX$yaQvrxXTyXO6dMY;nfK+%!H-DP$2tWs0jfJV~bhKwU9@c zs6tz_uP|5uHE|lrAnauH_Brr&;PeN--xZcR*ggufSWBS6~NQnxTb0Z$a8v zo>JS&Zf8f&nV4OaHNaw-c?Ixh6!{8T7+u`1>Qzb~*zK}pgaRWI)S!Y~pSQrWTQ?yf zcM}vLuMX}zq(n^suM&7t#_B@W{OrsGfhlglXoIzDVW(3w-KYrxLWyh4WGHMPT?vn^5vs2wZednlE-P5(<-(;{BCC2K)t8x-3wOzqd9JZp>FK^)xb4N2?H%i#PAl5pit%t3xmz+hNjUJeS13bJzG&1rtr%qAeznX)}9sWpDH^JNvxW4`sk zBRMCq@OF8#vu$2b;PYl%MpJB+2-}!QL0)!tsyA&s_q(*Qu=+$gzXAK}tl>3n8Ell&KGqhA6yt(OUu3?V6tn@%tu%M{7$d``vtO8$JQ8B9*X7B{6 zz~@a5^hh3OSPLtv+Eru|dJ(IUj7o7?I)chKntg%!i1jMnSD5xkKvXgwIT}GW2jXQI zQwxiHImJ0Vxl8+_@vOygC6A_tJfxaA(NYn(78-LKonnzsdFiN>F>sH;D3(SZjnSh9u@Eq?4MtepN?;E}aWiXg#FgUW(S(U6*WNd=b~Ri@ z{KjH}xfaJ!_DQ2HmN}(y$0#Z)+GT4bMNf%#sf#id>+)q6`mC&dx#dsY=4n|w~46ic8W6#rUqXgB_ zt^!}KH^=8H%42B7QPs`!J_|#;aBQHUg)*p8o-cwW+W1HLSYmw~%d+HjuI_)%LaADE zxtbJ=wR}|HNj^s%*!6Qpbi2!*D}hzlXTn!+dnaS0WxQsjlJOad0VB#rj2M9t1LM`b z{{UBLNPI%#;2}vvhYcTr8B62i<44$nqQm)obvPfAtDgbG)!A}jWl`)c$V^E{92T7H z@PaR|ptvw=nlEKY{D`5!0CgJO_?i!>CwU99)K)_7#GF)LLCWC7VMB*D9$l-*Lz9vQ z54Dcy`FB9VP&FIGqND&{V$&LpOU&THPMsN9R#SP;k>Mm^_QJ#@WMq3Y3!4bR)l|8Z zz>zxKV{v%f{Agf1*xCwANJ9L;q`-L2GGONr)Ctlov50Un!#8!IJnuisA_p4 z79}d7=R{Og%}WesRyZ{`ZEQhaZr;Sgltfj@ft6)r%M4E#k!YRu7ncuCRC_;J*}j65 zITa<9^9CdgADA#OF;D}FwYt@k-tCH6G_SI1^yq|G`>|g_qhuU+;ImkARB7YvCBf|q zLJbH(ammhAnF#@9zJvfCjXWwCpK}a!AbYuEPaJ}&tQXBv*geQfpgbv!@$JP{km*?+ ztA)sCq7)nu&xMpL#Wg0O*>LjTMz@BTrUkZO+f$IHMGj_hcA?LMPs@>&22k;2Z+0^> zOMtx`$jU7$$jU9uN-IoB0MEdSl@3ozLO@PrDA#}mWL)0Ff?NiaHO*&}{D5GyF_~G7 z7A(P8q-D*Oc&lktN|T8;zfcoxm+D8Z{}SftGeLR_&w^;LfYrR`3~bNYMgdf!n<#=d z?qm;)Ub9Bcl+`pMcFme-d)KT{0-|{g#gl>?;lZKElKZj9qv?c#r!@hYoS48Nk~<-G zbbNqvcoRSYZMMB|K6pIg88tdEH41EJSPT6Y@@gV7*eE%sBy)MQu%xD(B_z}`*jf^^ z)O<>OkZf{%IXt*dEaZ0+_?J1%<=dGk|0JrsNkCr~C4Y#@fkq za=iH#8)}kKlXo^;Z-y@=9wQPH2M->W7(aOE@T4Kbh9wPeOp~vcr^URnS~x8wF`h@} z1Pep!^um{(k`S->u0kZR>0WGg-IS#Gc#MgU4+Qc$EG3csf#O*36~i3REP%Tt?>{K3 zsrh+Xxy(RpK=ziQU1IfGaegZso){PBdy9&E1-Zr9zTC{Bv5bcuIw?KXSDcrhlBiw< z-rCU^ly#}n6i&!KM@Ztp_;cyr+xoLM3s#b()ofBZ$c^I%#;eoWgn{uGG)RdC&abMj z8H7QD1|e#~K;#a@fX34fH!7>DFDzfgIxTT5xM0~hjYIXSlM7MSkgra}SsY4zULhrFY48{?B_?XAJ8EexA4CS8+46VrDApCjp{3>3CCUPv zMZWXPN=nO?sG#O6vlVKducWx$YB!8!jopE@f#nPIPc{<3n{euFQ#chqmpb)|;qlJx zDy^F1cFn0gUz9mj75`Q|3lvATlxJg`2$YxeY^%yuy^zgC#zd7`AINAi1}iHmJL^h} zp10hiN}DDt@a1QF(|o)pRcpm&i|c?Eyn9Skb*wd8Ia`W^x8n!KL{*PU;YiIEu58+) znysE$BAWbpiGVyNpAe`JZ|+oeZ#p2Kxo=?O!GO+%Z*YI1Nxa4lNahW!{S=VTP+)R= z1(`nU@l1>Z^GSPS*jiCmmn}Y{Y%5}v-3p#QEva!?nyBvTnyN*O&t-w7@< zUH!*23652hZm!gzY*)a9bWjq^6=UfP^-AGG?mDp=2=6luoyFU(~wW9qjdIL{d^E zL7p}ys+1*AVgnO+CGPI#7fe~Dv6iNvZ6>#w5P}K~%1&6?S+-gwEb1b4(G1`G3?iSV=KrCW-6FwGEbH>Cr z8$FuMSM9E)hG_10v4KHijC+iG%o*cJFs0h$h$%01TN|`Z6P=Nyaj2SGJ}_x{JshV!T1kq}; z6bKAnYGFDkF*eqf;$p)oP4QIRrKNeYHS}EsQS<_xuR&Go*i%Ukk8+YJbc?Hcj z?U+&ntUDz>SW^mv;F^-Fr?*|s1b0AOhR222vkb{Mo`Tm@1<#RP!4EXV{JSaaPbS2~ zHnD7J4a$Q9W}I!Ut(aKXZ2oRhv#3FBL}m^DcBYL=?J=>)vOgyjC=nM(MNpdI&2o!k z3ju-8@vxnhf3sSwxJxUmOXeyA-)h34(g()ArlPnk&}+@qSg|u$LK}*0IlPJd^!eil z&MO0mD|vusr3&T{2x?R&3vOm9`DZ1JuTHUIf_MtT^6+mQLuS<6G15@@@SnL+ks7W%wm+}A`iW4oL6#$jSPXekeZC&P4@9yqh7QU~UD2zf|I-z|b@0(_^gXM<_6AfFSqHAWlt5Y)z(ck* zk82uaZ&aP7TN@p$!sYgy6r!DsX7>@1zcp1

o=R_F!UMYamCD4;#EAoY9e&X z>H$2<8sJ*8w9*yS0Ssi;|_*=RK!=bi%;v#2;*b)~%WoG_W@ zSgpnS=vP%`K;SRhK8tLIKtP`4oH+1-rPY@9H{hBSjmO%k2?4gzB;brP=M|hIkmt;C zsu9jkcxDNk4q!3=4VeVKP#NG3v5f=PAFnx>5}30!E>3JEWn1efDk_N5f54i9C`EIk zXvGr$$I-G_@r>dq^tFjJxPg48pDhpuUN)l-ZsEqAU=_=o>&>1z%@>@;_HHJ08Kk=b zDnf8Z!zwQ-d1=+s*%h`Z2!EeVeb>U4tsH4Z;lZuBMO2qxHC45Z|^p-!O61>saX+&R;~k@(*dFDW^imxlyk?irjaV!Gt9-PcLC8% z_5^3SmZ6H8s^+%4Rebwfq^uKI&ZY|Z<9fEC>1K*>R_x|;@N-0HE^At1iEF7@2Uix4 z1xkR+lKxW&1hh_7ByNuC-oZ&Xub9>F7yNvL9l_2<*-Z=DP_n}U04)LtLTqAiR+*~_ z29&aqzg1=Sr;2&2MNQOu+psU6IETSreHCIYWq7(mYI)0P&3;fftOWt1CNU9TPopfOQdu_)!YG$<`Tu?j|2A zZh4W=xGxo(qU12oK=&Mj`~ORH69T(v0c>q)N2MR2^cq;~I&fn1q@Wd`uO(Jz(;fKZ=)_9c~;0s7c%UR^PU}Of9PrsebsFSxK`VP(~##Y?JdmM&YqV&$soGcK4}JgepK)oU)g zc$dIJ?6`L4b-Q+7f5V==H{SH8eK+57>utB+apzrk z-*fMM_doF9Lw|nwFZ&<)>!Xi7{D5WM?N7|yXm>XIZ$O={|Xrk~$8bsXN*A2((?qKX;Z?8u^*+a?0C@f%UnF!*!n^L2;k-c>`0S4qS^|wM zDV;XnH`S80&dSWqEAVCdrsQXPZ4C{U?oIVti*Xju%3!IHx%F5+W%Ck&T&Bv^bAX}GcvUh4Bi>E54Ve9ayHK5W&QNoYj zTozXh7({FKVYD=FVUZ<{ZRutOKJ=VckXKk3K)Wa_hsEV{f;NsbyjiR+X6+3Ne=r?LPLvY49Nz6EF$WFby*?2W2Qn6_#VQau;JgF2xntf@`n?yRjGha2FoHBX|st z;R!s619$D93VagC8q$VAP z2$2qtZ);;`Zow)%VzgT|pn{{{nYTZ32bwC5~yKW8Tr}(`9>iEt){-qm|`Tup(8r6Rx zVBNzYg{!2DC!uu7#jwb%!0Jxgq za{{U^p%WkokOUY6m|VdG@5h&!PR9rD$lqNz&v_5tZJSKzq{0c=Y@pEk5ZY8|D zl`!NALa+IRo2m)j#}ZbqBCJ_O2wg+CEQ@ec8e!CY!u@H4zoZd1uOiG$CA{DvEJ!6> zx0NvP3c}Zy6D}?x{IrVj{8qy7^@JH|gns3Og|i8J$_ZDN6W(4)*j_?-VJzXjm4pLh z2_LQ`TwOxAJBKi7HeuFm!q)kO@>IgAVnWA_geyx4HFF4`R}=0pCtQ|6cwrV{!*0Si z6@>Y72!AOjTvJYXa3kTf<%B<15-LX%7LF#AjV2slKp5pAY%C#sKA-US6vC}(ggeRz zGpY#&Ft-j3&%kNjNf#@b|HVwDp9!iwLplggZ+Jo5m0#b`e4s5cae`2#)!LhZhm5M-zt5CukQE z=FcZQK94Z8hA=dp&?$qkp_;I~novFw{FuWJiKk&cI_e}+PCk}Au_UK$4;F(cka@qYu9ewx_9r< zqi0W-t5>hyz5Dd(+czqzU%%++n3&kuxVZlP-R=Pc1`ZrFC_X+Rfu$UW3`t5FI&|2u z;loFa2={0ncwgUYIwQWkobbUG!hM$$+HEI1w}s&SX{+fRq(O&-0aLMgH7*zy#@%ZO zU(F>mtIFsf6Da5liugeP_rX6zo2*4 zNY|+y9XfRA5M^a$WgW_Tmb_m6_jhgPFa*_%;eZhi(FNX5t}&gVp74AR z!o#(8?chCmjp>XFAt00hr&R`o5zvZ&)&#U6UgH9731~;~ezwDO8n-A7J4|O} z(i(!+Ua??xEt*fzcfz~vTGQEY=NN)$kKp2HT`EC_54SS{N-7D)PI#ZW)^r+AmJ`ft z;eDMwVJip@4c^bLHJw9WUtoiQOXYF;Qp<0Z4>*=e2np{OV0U;hhY%VL@4lU;bJCcC zKw^t&Et^ZwIst(-cS9;6EPRm7wt+)q1Mw}Q%E+_*HrCx25?Z&1cinZSbLhhh2yGVk z7gpJU=TiycC3Z>xuLXc{J8}D@tIWO6aN^2*3vwW}xzY05-4CCq6557$HLRq-a~Xto z;qb28WjZ6bW!Y%irCu|i5TV1nZ=&0t za5YJG_2@w6RD;8PPbc)J#9+fG+KG8$#+mY#iS`Gjt>34dEeShR?sRS*)^5H6}A zTvJV$T}}vFL>RY#Fm66!>Sc{#7nBnQ&Ll*RB`llWnC`@-gvZJWZ;rO&J~z*T<=#kG zSwX1EBYag(I5>vz*oA}bS7 zqTR}E!#7+tr7)_;#X!a2OV*(`a4BZ@i$?naOD_Os*W4UEGImW`uS-Xd8Kik{?bqqb z@MV8;Ub1icx(*%sCl<82V&>q8)RluSH@a0XyCpNE{qkMD^bI*vi(_^SlFKeyt#_HQ zthFnC_l#@$^jvY{;@d~utEzN12;mX(%H zoDx@bM^@paEz7QLw|(3Nw_mp`EPMR4%eLj_-ImhZvuPE)8~2&cAp-q@1xuL409ygH zV*0omSUv=}A`BRfE3pyi3v|ojpkxe!_rN~W+0UoQ_Zl4|LE{1RRQ|P?4^QeC^*j*a z4tRgwXF7XLD8r%-W*YKlEd$blIc^L?Hi})nMs&tpU^2Y>ZZ@6wYTdipmud3fS(n+y zAw^^-N85HC+IMtz3D<*{j0kJp%CI)&&c2{&A>F!l(;R09sSBFcEwo$bZe9QHUOmpt z&Z^^N<+!%l1@8Q0g6~!Bxc^<4ci$bR^NSLHy}urQ zzrUfOp}y{vAN79JHK4x1-+lju8>Q0?JbqWofYh679 zY&Z?nH-LQ%T7f~s>C>mcz)zp3`@O#I_mgP&s4C6-|w$KRd@3AsS|bn zx_bL+sIP0NuRndNj!WZjsPi|})z{UXuB$uU0C3UjfANF!Lc?jszP|2M-KmC#x?lYb z@cZl0P~T8r-*DpOsru8W{C@xGQ(UHcfBmW7Px(*!PoD%Ai~mmf{q^;Ab*D~qU7hm# z8@T?^(7<=DggSoZZ>WdA!H)+2=?2Ev@8^;+W?bPMLEWv3vGaqmXuxU4#wG!nLyUK2 zDy7m^N~CC+EU%M&O?K(pRth+d{4RXL3Tf$gfmk3G(n1=BNZL<_v_skqKnt(|1(jc$+FC^m|t)JDtFHfrB%*XS?m$Il6OP>cC?*V$Rv zJZ;~%ixaAZiSkS=mue=aqe8dbgI#SgGQ7YWj<&;VmxY$G-J(l+dXJxn}>}@jYo~c z>iuCO);w(R+c%8CDzsSr@|$8aM7c3$g;}9|(hM=jsCSWdA-!T`No6WDyAI#(Cdh>GgXY+vhi}|bh zn|acF&V1KAVcuo_Zk{qd){c&j&WFh_4kUq_T9+R@Ju>xgmm zcMNb0b_{e3a>P3l977yQj-ifM&1cMK&DYIgjzmX_!{^9%0JUgO9Vn9CM=fgcf%@gA z&(yCKb{G+O6)z!{KEQ`~1+U=)yaImisebvXyZVhrFMNQH(I0LMz(5Q_JQ9$I!5E^p z`-fpTMqoHa;5dH7DB|DRV7y2)z{`9Ge$=BDe$>JwjdVIcJPXV2G#B$QALUqp3sHfE zs6-VOVKI{NH#~vA<4HV)r!ka{A%#ZMbC`q|Fa;?Tfic8yUV(!;<1M_6ckl{Uq9u6e z$A-&PgQZx8o6k?4q8q@h!ba6iT=B;&NAh4aI039~cXt^vt- z1y|xKM4&xt5ebtRkE7^}u{L&AZI_`hvRi~ZRkg(oWGTANLIje76j6@}@M9OALub5z zOuO}l(QvY;d=yRigVMBCIF+I^l2N9x<#Mc5?Y36syaWw4+-ziHrHW%xJsjl6D~O=U zh`>}#MJ;Av1`ew5mMP+T+<-mUiyJY)hPqzm@);uV1x8Xb@o}4*k`ak-P>(YB@g2TL z7u4cW)M7WrA`;ti-5->Tdp7?^;7KIoX>>%g!t6P0!VCBcUFj=qMlyO5m!Sr)*w~!E z?Q2_XC2Kt3UoL~yuGX*Bcj}Q0KO*dR9TBOp+k!Bae@irVerzI8t7@uav;NBY*Qz?J z#ZuK@nMPhg@*hCs+T!uEqD>Jgzvy$EzfgR1FH_zFwU1MQE> z=Aa5>qoYlWOw(3 zihjgbs8zh-C;X&h{)7#1+LRfA5ZZ}kBvLJg&{s&Jp^96!1nvCo#wB3NIHstxGa4|6 z;)zG!&bSzflt@Ep2qjV*8bV9aL**2pyg!|X^R|RdGL8jnQcweG?Vk8GUQti4;FucA zT2qM4#W**sPi{r07U^Lb{-k1Zk6vcOw5K*D4xbjZ&;b+ocvKlA`HV7tr;|m-oTp(C8ok!RG=>=qN|D*LXmhBzhWC^qBG*{I;lk*pNddl z<>#K#8Jke6u)i6b6_hr}L@kEX7!_v}agVwL18AVC(TVCU*TOVps(Xa`{T0{1pi=c5 zN+C2+)jVi8O#|aNik4thu;xp#QsuQB)hb>D1}dyOArec1J&J3X3z3RWN^Lx1uv=l>QRTohy!KdvNV{}IwOAR%qZS*Hgudu#3-V;Bkv#=htNyqW zxhm!abi_Z=Sv_BZoS8G#4V|`oD z8PaTYhL-~VzC|n|hJe&g|zj ze7ESBsrVP=za<#bh@BO;@t1tm^C+MB)^# z#wq-UQ8sM(Ty0^bld_Dd8h!_!b#iV1h|6n;k?PNEe~-~bN5Xwrty;#nlnvv?f|G?3l}yVvm#`~&Yo;61#DccCK^ zuj5vT@pB2kT-F&>?f zj;j%iYNVqR+9Lv&;3A|U4YSl35s4@@yXJ8$8ENQ}#27?0lScM5uAl8V^y@E9hKOOc=;^hmV9)wo)T3=(*^ zG?g~lcgvSbSD)bzy#N3A|C2S~(bnN2e1%Cyl;+VwB}tDqmONT}`8EP!soGvj)i#k~ z48~wMwVl*Tw$=FgJ)M6iU;+w73R3XCUMRxk8rp=*F%5^sVG%9jM0e3$Y{W)fX1iEp zFZHCJWP*S1pe~||h!OooFLG<&QHmC(C1@$ykCdQ&LG3hyzQpPWNmwk zZMI!-yKEPAx<|W%>h!&IfKm;MB2qLr60~A?s1nyBNweUEYN1++CexcVo+c|=UjesvfV|oP+Ag*Wq2m0NQnWXbpe4(DWp6D)Sv zM=PR%ygR0S@CW6Ju-#trY5T>u!cY4Z<{o-TzaDqfxAXu#sP7kFQHs1rw%6)~Rd?ra z`!QNetJi?(8tm?VCnK3X4}x=nP|dK=rt3i4>%g~Wbw zg!Dh429HJx*LzzmTLSGDf2Pl|U#urCRa@#!)g-h^(ltSQsf*?^qBMvF^oU*l%k*%g zC2IJicH1L0jbu|Q*`&u~61fbQ;TO{lm$6m%XciBs{DZQEkVku2;oK57UU?WC=Os$S zAk{At@Gwmf7N^=v_fZ926updg#tM1`{j`1>(?Uzo&R@3A$fNBS9!(m1X^%XlC(~+U z3w5I(>5zVr-plA^te_Rt%Xo!W(2L?_%16GUydP;9EjbV8Z3&zG;&aJ4GS`|UN6``L zOr7aRdXNTaE^^T|G)giLilh4}Me}G0s%$CRGD^^1);F|B4@a?7;l(}rsQUG2N2Ny_ zZnxT1NYUJi+cVuiPA}3S{T&R{j>@lPYvYi<6N(m-BLg#a6PvK+JoR=x?WUvh?H22A4=uxfQG-y-q7a(XVtshDbV?+T zmPlWScj%@T;k||z#azMX;_p%_Qn8nK*04m6(DIa^F=(E6!yXwk=`FE9{}Vh~I6k8k z5iZ^o^Tqk*Us8Bf`w0;)ZqPm2YxK78Xm5)b#cy;W_Q*ZD0grYAeJGdcQhY`o&;deZ zQ>n})UW5Fo`1=Cg;zCc7uKG=f7Dx4HZGg?OE<}5+m*V8N34ysH#(0)I+N)$jlef^T zG?Vs7k7j7enoFA}nB)JA1a1?rQwMFYI4DBYn&v2ZwE1EX-9X9OPz=QZ@@T8^knYiL zCXbdw59)KINBa&*nn&wr4AIh&4ig^j04<96p?hwfSiGLVT(bkmZwu^5Yg(J&0d z2eenjP`DV5;fSH{sk`FjH&B$(LmQ#pLMB$=V_|XfHz-^P;n6x6G1|w(9KVgG6JvcG z0xMLG%<(D=u12i7v z(a7;<;eGU>7%fS=O^l|i=qkFB+SBFa(Y~bV`iIn0-ay?ou8UK0h3KYjrQ1Yb`X1{= z2K7TnrW4`O+M_*u)L&tJn>xe#T{7k?AYhg5Wq33pc-6KATj0@BDH_SzM5@;-;h#Px9@0UhF&c~MK$FsWjl0D2lrDZiA4<}0MR!?=M%+Kt zKhme`pVKsQP$eAVYc)?fK_0<-3E#t`WvF@$7Xs61mb{UC3iJIs^T=Y#MK14}+A}Pc zED>Thl`@@Bx~POldsTlxchG7aqPcVo9?jGa(7k%TSdDyfKRlX4^v4*CfmU^qV^MYqytolLi|a3v^#JIJX(m1(>xj^Q*jtP+Arb| zU4v`zv*30K%>OoHGkz6!;!b24`>+pr@;Y3Ho!E(C#O*wsc3=lasM`7z--73Vn~Cw^ z{=n49?IOf!>O4i4PsU^vp$M_upU9(KBMYdIw#)C~rVP4Vu9uDY z_-G34pnuXA=qiU`2u5Ng7SVM1sT`_(qA2VW2)vFkdIOiy$D);So0uV|$*Iz59K&Ns z*1pE=^nobGhqxdAl+U6+reh`^#^3N3{T*-97TL-^`Tvy;(66Y#57n%Rstq5dDvFYA6i8LUfh7e zbXw->|G;OMg6HTx+y^gxgIuv2w~1_#MUUeF927U|PvA#PpkFXUJc9@E5bmOTs06i` zjX5aApHVEPi{EepH=`3yVg|h{98@Gu;TRpn3uuE2^j~N(HQ*%G6SNmG7oG7Jlv5W} z(0r`chtMr3$K|?1TY#Iz0C9Xho%>HT6Y1+#^CnxxNdl5QQtK4TaMv z?GfB9zEbW^-MR;UCn!1XwFbMl1uSbeHY6kVq-AY5%IQs$M%^UyCdy7SQAVqGql_csAMziPV-GQg7%Suo zd6BG=@5&405pj`REmz77GER;+#v7T+tufXZk;Y`hl$M)pbdg=8Lpr1>CpW9v0^^8i zYaEd!E!10MO?PXSuJQei@`(6aM4T0RncO62p8<8J?ca9B`y-;dS^8h(&niaoS@9b4 zjXY0SW30xfRp*UA!dMgxUl7D)Z17zp9DbT@q{^G*5plCTEsltv)H~oQJcTFmG@h~D z(?RzfUcifZ5eIP)&*OO<#$mjQSMeI&Q0`56-Kd+w^J#Ra?)DSo&C(7}qbK!JuD5c1sW0(Lx*tVT zG)38tG5KCOt4(4lwh0`57iat3G>`@nOO1q47!6jArAAnGg=JHQ1>JDvM$u>*O=E~9 zAybIYv%JckMg*}-reqpPA1FymmZfCJ=T^_E)2yd)7%$-^bxO&n=Tp=qa+%})?O}mI1lKo z@X{Bdm#R$YiZD0I5hrI(*0g(0dDE`BuV@TL_wP7?ujx zlB(s!l}i`SE?a`qvdZeR;w2@O7h+M_k~w9SHI`@2(uHNkrRCK%sH`ckC__bANhvBy z<^h;3G-Lyq24EskftK_b#>~{F>*j` zOk8xozOI?G2s>sGelI2*pGD|f!rEb1Ih<`r0O0?j_I(7r>oz-_y(&v;mM$r&D4SbT zwX~*eN%_3_HEhVeyt1-v$($us)z$F+d9%aW-V}z236-waGCo2K?yW^K!xyD~SyuDJ%%yx3@T) zS_HxSuPqLzo>W2b9^c|{wz>6UPOo9#BEGX;<5q{Wd-ZIqSU-)i&_8x5DK$pc#a;*R zV_O|gHYLG;U)<_&YVEl+TOH1*v6^h(8v9?m`rJU zwm;9Gp^x^b`!oGn{sKITr~Mo8E2hyddWmxVDl@YYOAGbd$l%t zmVTL-BKqkU=$Rr{&(uwk=1&sC#SAe@Tq>%?^I{;zi&kQ$*dd-0e-qz`f&Nr~zW-|f zQW&&?zEKu)7HDg=quNIOD*Y+_H@&AQ5_gM_#p~K!CBaOrIglup0HaZx;$S5Pk2s2`g&cWz#HEuU( zW;|}pGG-g=j19)K#(Tybqtw`I++^%G{$^Zj{L|QJJYoFZc-p8nzA{c2Ck-++Qx>(W-;AG)UyNUk-;LA855_U$lu>8Y8-@w*PrDpWpEA{=u@M6yB}g-r=@vH7WL@VE z=PtPRaqh6Y9nOfSW)VCK2vbuD_iQ9ol=C>g+u^*H8m-t_M)N94f_wvEXpK(jiDazc zv0hz9b=jP%%2M=oy)_wFH^x|4Ub(cU3?B^sya-s7D&|)$WyRQsMjW32l=$@0l9ilr z+UZ-z;hDQK<-)4Un)$%G`R=h8(qo)hSysaF@vpLsXRbd!J|wi2i?zia&WANNL&vt_ zEM3$8?E0?~{a?PyvvkQ8k-k_+_~kl6d6q-@|B_<)D|2-oL9ZsPUqQHT1*uAWn1H_!u%CcO2>2@j zj}q`00gn^#Hv*m@;O_)HNx)MCJWaqe1UySXEdd7zc#eSQ33!2kg9N-tz##(EMAUo3 zK8G@k^3Xnqb4L^MvO3dwh?FCokBAij|LJDtwKF4b+G{;R7vKfgS`vXcM!nx3DZUsjr*KW$!q{``D+*WK!H9_l`!{Vb!~ zoY`Hvw3}sgYdd>*mxx(Lx7M>;cj-A}meH+ZcIPe~W*OabXOHaCeU{Pf!r75sI)#Th zsa2~^;T@g5dbcWZbal-(yXb=FYI>oN`WUH?4$~&J3h5CVI&{SN+}xxQZ3+r_6~b3| z`}S<7r>(joBHFc!WRpFeJ9p~Tr3;52KZ%gGod+H}7L_K1L$vSO&zj=BM8L}g946ou z0$wHHH3D8I;0*%aB;YLq-X`E30^TLyJp$e*-~$3aB;X?gJ|^H30zS2=hC~XMVi=ZU zC=!u`5lBJ;CL)I>VkIUbOCO3PBw{$?F&sX9C`KR=BhX8vC>g_%NWyT8L^OtA1bSnn z@{K?qh9e3SF&u*sjlrlw628_(;A=>I1SIt^qtHW(LWBGiK4c*Z7vo}N!G|jlrTpb6 zQ+Ez~3O5B~aLk#ALNsQ=hfRn@45FI-GTfP3;2lGa!c6md8=4Onqa0D_K^q;>DAO~v z-MEb2r#o;G{V<&WwXoJVsppyKhcYB#3SyCja>Obav05yZ3C4kO?4fVOOpLIxDmSDt zQ;fuY7>@?yQ;I?l=ONJ#@8CTEN%L@2%cMIItz^zCuo>AZC71CSMq*r(I93g~ zagTyH4%w;}Eu1s$n9-8!cyp8dtZ=TYgBXWQ&D5hQTGjOeRYRdP4oNT#H!|@ZhT$Ok zA&in#PqoSst=i%!=BcMl)h?MhjBG^XPV~cJV-ccpU#wJuKy2!*g zn1{`{lemVn@lvpcxvr<-kcdSr=1HcmO|hF`(NwhDglG})9i?b<6XrQ8kVIQGQ=3M2 zh9+QB&O}6ODah0r<9Q)VQHeW=^Uc&MFcFzrHtr;*#7zcM)prP0I8+e#2OjshziJ}aIGsEG& zzDcFxoH%s_#$hKmiBJk7uG>e{Xpln(^%pP^9f)W-c|KhH`CUmn5J(8sy{WtGIrMp zHnK4;eLnn^JOVplE{umu;WC)R{*H%p$>lHw9)T-h0>PYnpd4Z_iSb~Jpgqcix#S9V za^n&h2_Hfe%Nq%0M7!HS=2@N~w?Pin!%DIMvSA%$!vdzCiuqG_YUDASPmD#0%nO`f@^`oWSGQim<%3Hv|bCV;1YNVUWRL69m~PoNpKP`%QfsC zOPsXamzj#EA-HScBv3A5*DchL5sm1<{t9-n6-Z6HWA?LQZ?Uk3xCbo| zvr$NM^bFAM9f*ht_rQT=Su@+6S@4c+C2SxwVc&?8)^(06Aq-349pFd;SpYx5KFdl- zwJ&g@r9J5`Sywl3i$*+)*2lSCpXE$bXhY%g1u*d~(&Fxk2f_Bq#aFI!)R$=pfDWY{-&9qew-VBa~sz}H7o>yy}{(Ka6Q}rZSXKW0vn(mI$$Go!X|hE z{svF7IHITF8F&t!hZkTAvz53{h&$W3OZ^4>3Mbe(+DW9CTuGY94dh006S09j8SA`g>CNITg?o*++=XUKEpd9sDPOkN?clGjNWd6T?F-X_~H z0+_r*-X-sm_sPG==cJb$CP&Coa*X^+ej_Kzcy2N`g}Z{A#`(BHZYDR2E9dU!?%_6b zZ*XsOA94G+Z@HhipSd3H5C@j1MYY5%i!Dnm|6{q{ate}n(Feoid`kHQdha_YS(;MjVs`)bwyo^T(Yalb)##kYnf}c>n>NP>lxPz zu2)^Jxn6f|b!~IK>w4eyfoqTJGuIccuUy}`dR<3c$6O~}r(DEsaogM;_aOIZ_c-?i z_eA#;_f)sfJ=1-a`)cptuzJjdI34?l<>&5z?J@Kg9J_^JFfKA-pTGx@9d626L` z$N!11=4<#`KFTlRWqv7t6TggK#ox)_$3MWY=O5)e_$T?-`7VAd|2F>)|32T%@8Q4W zd-gUICrn7VB4KKRFJVT)l?ihb<|iyns7a_zXiT^vVQIp$gjET5 zCUhn|pYUqJYYDF>{5_#7;q8QN3GXDlm#{10^Mw5gUnP8(a5#ZWv?V4brX*gGI5p9i zI3w}O#FE6)#CeGe6KfJ{6QhYuiAxffCN4`{m-uYrYl&Tn?<9Vh_)+5S#J!0}5|1U4 zq_IiklEx=ZOUh50ku)=@EU7$cLDG#$HzzGmT9I^n(%nh-B|V((XVTx2o=$ou z>A9p=l3q*dN_snKds27OXG!289;;`N=N!+uo}r#pPlji-XS`>E=Q7V9JQF>WJd-__ zd!~4<@J#jOdx|}aJU4inJ+h=FRtB=`HtGdFOib=Rk%)8usyLXj$wYSx~ z&ikNuy|>M~&HKK$+k47;+Iz-(WlDL<{FDVL3sY94w5B|hvL&S}<&Bi>DSJ~=1`Qqb z=Rpexg$6YbY8})z2+kqrL9I|uBo*{dOd^+T_A^#e(e@O2T7-}Et7|IV# z8tNUIF|=T4;n3!x8;5od-8>YAWemGuSk|!YVR^&G4ZC#Mq+$8P3Wv=YRz55`ta;ex zVOxi7AJ#o=->}bzLF$myVW}fhGg9ZKR;A8Mou9feRYE!Y2(u-q+OPFdD@h;E7GQ?6{HoVm8UI86VmF_)U@WbrD->%J)HJP+J>}_ zw9d3mX-}j*nYKCY<+Sh84x}AS`yuV;w4Ss>X-Cpdr@`=FSHN1QH$HR9H|8n?O!@nQ?^YEVGM}{98etbBbmvdhId5z~aop=9vUFV%X z4@RsRv3>-kJJY@CL(?jY^%(yNinz1ND%~+E0*Np2kZpdiPSdnpi#;T0gjJAyKjDKZ( znekP|w;7PhWm+;Fna)giCZCy<>CGIJc~0iJnS(QjW)907k$GX}MVTWrvoc3zW@nDh z%+0(cGcR*|=7h{inUgc8WKPY@&n(KEnR#XARhhFhOEb$d%QF{d-jum4^VZDP%zHA| zWj>JEmichzqnVFoZpduUd@}Q?%nvfVGe64wICE#_uFPYZaDnrJgbOk*IRAp|3r1gX z@dbam;I9|lf5CGXe0ITC7r=%27cRW;feU}Q5H5Q0qAeF~z37dLPFw^drIFE*B&#kf zlyzNJeO5Rtk|kzIS<$RTS#p+=rDnyl7H8d^)s^*D*4tUzvcAhYkaaNY`>Y?be$4tQ z>*uVVtV3D7S!lp-%Dy>!S@td2%d=auZ_QqjeOva*?Ax$ha zOvjkbW44Ux8nb;&_n5t7_K$&FF4vN4&9&v)b6vUaTs}7;H!(LUcS!Eg+|=B(+~K+B z<&MbB$j!_>Klg&%3v(~Z9hsYzJ1RFjcXaM0xp}#hbDMMT&xK2RFM&LgXU((a+4CHE z&OBG1JCDyx$V<#i%JbxT^OEyY^3KUSH*av>ki4OJ!}3z|((;DqotHNvFC#BAZ**R6 zUS8hVyvyJ<`||G3dob^zytcfD^G@ZR&g;*!jU79-b?lvE*Nkl& z+cma#ER4$-H+tNxapmK#9=C8@^SBk`o*vi5US{`vNmmJ!zV{Hco!d6Jt@W|)$4)%f z_ZU2$upx1ScSFX8(HrtNh#RC0(GAK5b;I==ZrISg;h_y}8@}7nyWz+NXdlv^(SCk= zUi-NAX#1k}ruOUGTiS1JzpcHsy{-L3`^olG?OcbY!`|WO;5!mJF6g+ZBdcRnM_$Ke z9TPhyb>w%b9nBp}JD%%!v*Uw~?vDK(M>{#2rsPc-n?`Nw+Vu6NZ#R9n>ENdCH~qNjr%gX^>fHoSJi2G| zo_F`a-Wxvc|Fr)L*x!Ewev6)jQ-e;OcWT6`^i$cVMxUB^YU-){Qw67HoOARwD zYTvZJ{61gb^uB_=!oH%unSHbRiu+3X%KPT@&F{OWZ$aN5`~KAT=e}$Es{3mC{C$DG zy1r1~b$!vk#eMyKq~F?a>v#6M`rZ9}e?os^e^S4fhJDzyGWLulv90 z|F-{N|M&er_y5*^x*uQhJc{|*<@hpj;vjz@`6Nb~$O_U*y2%;R%(Zb~$*|;Ee3t2! zD=p2Ihb(QD?UruK4hvXES<9_$R&Y9ps^#%stUP&7JQqcQ17R)qR5-_(^;|Kc8R7H}a6MFbTZG%Xux{iQbvs3U95q znOW@3-exE=Xa#nIq@|fhylJk?Blb0snl>BV+3(0>^-kSVY za(8m`;7x;{82q=vPY&KZ_=UkQ4(=KZ8E|3rLdb(Ukjka2rPb115^RQU*pa=1?g;L< zZbxK?vg3bt+_Yoaj+Hw)cRaZRKAQef(MRy{!jEBR>dx^yOLtc7tlPPC=L0*}?|fwE z<2yI*d~)XtJ73xP+RnFjcJKUQr*)TOSK_Y0yGHDqxa;y=`MYN9x^h>=uJEo~cD3z# zdDqTeJ-eK{&)+>|ch&Cb?iIT`c5mJN#_msdf42Lp-KTb+-hJ+#!Fz`9nXu=wJ!N~! z_f+kX_C)vGx@X0n);$~dbnf}bo;UZrv*&|7&|~Y#?76Tft0%i>e9xqw%X_Bx%G@;NpL?o%gq}c8uqWK3_B8ce-_zW4ThGHikN0fq>Ft5uyxz9n zr+c^d?&yWX-otr^qldpc>^+iyr2GgR%{%&sqs>R#j=p_#|4}%We{A}(-;TlY{Nv@v zPaQA%r5O)n!0rvuKK{l07th-QCkVbypRiXTv!4%=QlE!>(gjIF_Ck^a2+12qh}vpSx$i5N=9?H5g@Bb23bu;l2$T`+(CfcMMjgm zNiJDK#*=$V9>LEVJwV2j2gzmRAu^G)5n|s!rjT|5WFwhEo*+P;BvZ&!1jy55GWRSY z))&cCvV{P7l>m8@aO5olWEQpAlQGdSWdvRp`_7Ghm&0dk8ag|t|_ zmX(%NvdS`)yTg){{HP_#^@at=hZZa8wyY#OEUU;a%bjGmrIqZpkfFVn7IN6Kf*i9f zCC4qz1&)5)7Lm>rLVDHlYXzWE`5!& zC4G%^ZTcE#NBSD)mh?5;%jws1ucqHkUQZ8@?dgB9zL$PC*_*zG+n3(R9ZX-t{gA$x z`zd`bIguXaPNvJ;>GZqF$jlltI`dED;>>HwxXc8yFw^P1DHF&G7rIj+XYeHpbAZ(3 z%qRYwS`x@vK!Q2M9?6+Q#GEQ3<&=|XP6d&3fW&fuT%S|QHRq7Pk2%D8BnL?T29n#n zfebyrog|HGCsUi+iS@7Tv&r@C#C=;kklWj@B(3el+SX3oo$a$oW`7df#m1;YCwRaM zsW2Qe;360aqaY8aK|U10OelkLP~jbT7v9$++A(H$7wl${*q?&KvT*pQ(HD~`%Qw%hl`%U}4udi1MPU#A{E)^|n!pZl-xZ|+~(zr4S-|AGF8 z`aAny=?fvicclUqQf1v+|{=@yh_Mhng&3p~F%`wqY;P5+I9Cth3bsTh@ za3naVIvbtqoqL@<&QY#1SBGnhYp3f%ci8=e`*U|HU&^oGJNPa9PTrZ=oES}NPHIhR zOL`&cMAAghEYEyTou}E;;o0ul>v4NWd$06f=Uw6L^lta=^`7ufOrDjzAh|KQC3$VK zb?9Y78;7dwK!;Obm9^P_z z`{8|uPan=bA|B~H^3IWiM@}4ZAI&*>?a>F0zH{`4qe;hR9b0g$@z}~^n~!Zd)^%+A zG57J@YZEdiw zF7$nU_@{70JQS6W$jVVQc6@OIeyy}lZDG;41>~H39gB0DMaTz9Rq!2*5!C@I3+e0mEevIQdt|$Eyfz zYtcd?3fl=hxsJf6>j*Tp6Y$(mplBU|#C8HVtRrB2+cbO&ekce#1+W;u(Wc-3j~~Be z01WPrVFDX4{+0k74&oOvv~RUNUP$0^J^dSjfECLEKz*2iO|K_)8UZ_}ear2MIsy(( z`^MWJ1@=K6*6z#s1biF6$7o&c8>?O4Yav2yKRWprxWl&)sJfrPIrkB$Tt(o8^#sCe z3Ea1Wz>W74*zzEO%=HAWxtGA)^#q<=K_LHL0-vlU@Zov_Q|>2l;6VbTS_!=OAc6eV z1V*mLugeg4_Fe)PFDLNN6$Cz7O`v)$fr3>8zFtA#-n9hO^#nGoCvfX}0*5dOat|l<<4vlnL)Zxro#aCW6yQH+N{OXFzIdiM#&A(>BfB#(W zK+NLBmmrIA7Vz!=*~zz(rN-|@{231ezqq8SOP4O4Hh#iof0)R=-@A0_(&nX0m$ENn z{l||0YnA->PXV+1fu8A~9QHLIa`NMFsA(L)a1P)+4qyZakj?>QZ~&Pc;0JQx0uJCp z4&WjVU?c~=_YOJv=@{(!e=G=D9Ka|JAe#di&4F)iuai$JzLh}NDgwW>5IDA)z&}?J zc(sK<#jONU$mtmWDl)$j{zIV%UgvgYeL=H%JeiHNfqeBH;KJU0}BY>zp4&kfTEbXVf30sxZH z489YXJq&~Hk2`sf_j&^P6>-^Z$DKTviedN1oqX!9nFN}R(g|JoF>Urs!_a#KSUZ(~ zw_J-1{yjkRFHSx+u!KOX(ZKI&=sM{ACUj3HeoP6@GX#_H+3{;Qg|INw&0n_6fz^AVN&_F!g zjo7`mlTMz?L_Rp_Ir1b89=VD3$rgm`eXLkYYAkOacu02!4G2gNTfby zzY-y^e{x)Q>uEi^?X;8kbj%`XPi8jjVD#kIQi7}3IZ+qxUq8lZJ)})Xr%fXC+|^~5NN(UZfJY4JZ2v>vJL96 z&NcTrdG2C7v)1S2J#SSLm>5^RuLHAl2>7;O_E*;u$kWlvN_S)S7$kk4lTZDhIMN%P zYn{eHF3NttllL@DBhWiFu5)euPM*W;&HYY36|?b&JZF^OKaGP+km~(T-tzn`bcpay z>wB|{PsOy~7n+~XK^_vj*~NPt#ROog#;97Sn_c`6mKrC*_{+=Zz<0td!;U3kPrAU@ zdZUZyPzSZ$=;A%s;Wtdn^eWkTB+MR*B*yH0bp*N%G-39B%pQj%#_alK1V-stV1n7Q z)Wvg`B8iu}c+Za!0)R@nR_vFJdzj77XF2RGd~W& z%{RGtZUU0`CKvBn878nr7b=E|4$Qs`$&1+wqXhO|5jU)E%*F$a`!Rb7eqqfQmwf`W zCn9lgcJZFd3IZAN*=;wwcy1CB7qd$$2psg`7dbWJ`nF^CWF#$SKii~{5&yET^a)eR9@!dJ%6~Cz{Kfs)psp(@f>D%V|mOTAgKO(uazX}P!{6Yfz4UGXB9RB_j zT>KF5H?)w0B0PCW0Pr9i%Fraq-!|hq^gJpv$M# zmFR4S{dmRz0Oo_Q>=an4znpdEhHW8>R)EO2vamNZ5pNgsRWl%lf?@Htq1o$jOpnU3p3XTUj zfdJ=52zU(wI-B9DxEk7SbMdLWuO+a*aG-|SaRA$IbMc+&{i-|zFY^m8np!i{D!*zzW}tXLNs4X zplWDy69>?|+Qp~77bGxo=0FW|;{ZAk z;OaU83ug}W0Z-fF4@c3B03WO+kRM0Md2uzIK!C?f2yBjPVtyPzTdRvteLqNGR2-?V zi38Y<02Orv+Tz%4K^#CY0(?+UAg^d(6n~5ZSbK+yPu*EhVAQMufIr0nbRodLdIBAB z?DFS0fP)C|em#M`aih?+B_nt1oi0B0<9Y(y_Bt6>WQd{R%KHx6K79Ke1A=qMo2 z7zZH40kquZ;#0k41Ulk|SrZ4a`7Rgl`C|ov=At-p-hP*h=lp0vFgw4Vz`{78@%3VM z04>PfE1WOf#DY+})Y0ES-B3{)2wpTI@S+kLl-cN~cjnB`+%8g}sx z7n6*e!Pk4Yi{~gdbB~MnykDzz8&A3s%wBtsi|5d0cVPBW^>^7_m>t9>FuNEPJLqSo zvGhUAu0zigvtOlzo&;qk_{H7?#Vc^U!Vb9hJKM*MQHiyw3y0l)*y zc;UJjz68McnF(`njf+pmBhwN6LKu88z}_o$L}N{@_qzB&A_0)}3!(6J0lp{hHCo$p zuZvH|lW2?dOD4cxA&_@fTuu8CN+AG<2fSiLd$9oiaO|wTmbETEeK7%O&@WW;__~3! zpf)2ED)S`-;C~3fUkE@nfrI9&bQEFV_af9^2>_lm#}jhc_b9Gm#kbgepNpT_OaN|- zgIe-CDEu9qjW46uYBFlW7auG%$bxNl-{<0`{{)3U#$RwFyYKohwc*de2L2Y7{ClhT zdWDwwR=2Nn@t6O;RfNJ{e5(`xOKrZ^`(1nr^S(zlL;ggqtqWG$%8K5b237p!(d5tueUtt;&C0%w;i+HDgkea&K;P&AG24WWq#1bTi(HU zLY&4smf5o2#pB+TZ|!}q;xk>x2MBg0&Zt@eDf0eGNTbj+!cO$?-1bA*4 zffJ>Ml4><;xE($ChtR4{CD0uQV0-`^0k$B3Ri8sK?rG5+-GgNutU|l|kc;=+Tu)%W zv9QpB%QFwTcn-5$+VpJv8RdO#Z7!Z$jXcoi;!}6|wLUQd#SO6)c>n=wXA#Jlt&bpX zhzQX9u!~PsW)YZWEUvUP>o1!C2GD^3Z{0~CTBb{Ii?+CeKdhk}0k+;rV68DdZy7++ z9moR=pn$;Ua-$EeCYlkT?GZG01q3D<0C8x(6S)8ZN@fw*8wX&Z83B3`;2QLluQoer z66!AGg-2a{>KF9{T8w3?I6NRg7Xti%M*HdkUcg`P)g0W7yzr=t_q1O@plzTD@HIc? z;yKK2eaywDVmAI5q2k7H4{`zm{4q+P@oEDL16@RB+m8TiMFQIg08BI?K+EIkUx@_D zO@IMJuR(4=fHrinD~u)v5RCwP5#Vo41Qr^2Y-uIN_v?`o%^O^N>Rg<481mfG3JGjX zzyLZB;M$o4T8-I_&d>8JIlzVb?hP*9bGt&oX-rp;pAT+u@f>EK!19=FqDe2`QOUtt zGD~QAMeFR_~0l1$4Jb>p}aUvK;4})cX z4|iZHfi8o+=1w>1mGwEI7R@D6M$a`z^^zPB>=za*GX*%aEbt&#$|?$E`GroTqY-)1#xiz zZVrIw{=I<&ZlHlgPHP~EJG%jF$iodZu({L4+w}%~U7ap|w3h=&<^WPO6%2zofO9y2 zb2)&)9Ka9`U?>MLj04|eoBl(A)c>VG^Aj$1KE3q`7yl^N^*9IJ*J$lH7jWjck=J(4 z#n%?*9{*fqO~W7nK2j$MDM;@B0a;@B0g;@G8DaqRj_700fd zt2lP8tm4>pcNNF32dX%BJyylB>xn9kUC&i7+TCh`793B&fs9nG!Et$anM@KL7wiBb{BDQqKE_UOb#*-rWp5+ zo^kQS4ZdT~xOkV=HGp%#*Yd1J@z0)h@tK$E;)#7ke+ab^dinTrE%eu&;9L2ei??ij z9Bb?!#FL58+sRo)AwnTBL0J?Isr1s^20@9Cf1DsMCV(R-RoBKuKfxPCrBTj_DIrk^ zTkwv(p33SDO4T88F_jf+8x)kP{ZcqAs3A$L7AXzTfE@w+QY0$K)PeT_eB+YT7zk6@fTvl&TwqFndT!3`*5POs$jT&=M+p>_MqIECs#Twh&XL#ezQ;i&RI2AXSoGtaD+i z(iFTE{C=t^gRnt-GgNg{Q0ov`=U|<(AO@s}NEPK=M1>NjG&(p1`x=vFd|#OwiqPuI z#t%tgE&7EpCJe>CEv{7%iGg7VzE~A%s%xmCR)@p@ZA`_w{V}<^HWXH=oQClDDAGoC zI74JiuBKA$c?O(c3dbVi2y6wjWT_z(NJosYiZ!%kn9nlB0c2ufz2RU;JRb|QN2N*h zUtj|JCB*uLIDVFEVEv+GBSjGw)2I{@)saXx4Ty%RpwKKt@L8p*RcVw-T*QSGQ?n6u z{;;Ib(a2&l6#^GyQiB{)X%3EEm1Ck`Q0W-Pf=!}77du-U3e##Ow1i%Q$zjP~pN9mC ziFiA9FjiVe{q@x{Rj9m}1{laVY@L<#OQI@E;Y)F(3iYd!%qkl{jM0B_$WN>Nb%HGT z@vUcy8uBX>5cNu3q&gx*FT;M}JvR0SR!3c=x<-)Y5S1rl_wYF;Mug}j?4A^*;$#y) zen}LmU%lL(hlP|6Bv#$XQHkQ3QP^sY0Sk zgK|i1nr@E>;jrW{a0bHKU7@vBrnJZ&2;=`V?0%L%GcgcSqLPBRh>7*H5`}OmDAGVR z%PmgP%2vk|p++ahl~!eutX^efG@ETzYUKJ7tKgTEQXBrQE3*npNG!ME-@2=S~!rs-q!kT$Qc5S_z5sEPhFu@2IY(jZtabHCC){frU0k z|7fF)(a6|8S!*Lw;Lnz@6uj0JmV#rW)t0E#u+Yi)AS9}SUGWRy8nMQrgham;dl0Zy zSF3_ZEvg{aT2(=;3)&+z;;)nI*#D7eh_9|zRM}rAi?Q%^cD#*5>+%0E6~hiil|_Ff z8o?U|ggHu7lw8%-f>IY&LzhMqvCAqI1A$t#cHAOIb#<7kwc}(*b#)ML6jyb1SgDiM z(50%gx;hdItF@QL5~?-ES8D^c;}$3Bsu_8wHWX|~HlAw)g;s~eklJW~R8U5prgIFC z0QH9=LYP4>#7LJ8PR8}GN`hlcaD8Q0# zFq4@2L#SqA$9`Tm{zhVC7BC*35flAONs79Wlt_VCnBGnj5GzJStB6MjoEr-Tu;Z&qN?b~%Dq~_R zF%XSl)5B5?rrtr48176ndMDxavS_*PG7B+$%?qh@B}(vav;G+&F@XK3q>Jt`6B+1? zkQh?x){x%Iag6-pWh$D8bV5)&YAa{;Dt3Bg)4-H($WV^j8&DC z%OO#%eb7uPleDg`H;d|BtE6hBDnDeh)ht1&D;5jF z@I_h!9VXRSO*)b`n#}G~rO;*LJ55UaRB6U0lYIJkXUbB;69YV`(PxeEFmX zI$?c*z#J+^LZYBb@{`1?Cs$B0K;_a{VIOKHP1B<*ar zC=u%EJhalN{hu{^8s`>0M+PZWt)NOQjASa487ZDO5rK?~{ElYo1>!+J6K@MO=kP@n zlPDX-Vl<}Ckx)do5H~~k^d^-mFAX%z602*%Qq9XG)sPpNwO&n8vs8;xghpOLeS#j| z9PIe3W-BPsnk>&1Wm@|h@#=NVkSQ%j&+K*LsjZ(Qhk`*W7qTk;PIy1cP$dmhzuHB- zv@t5r4k@aZU0S-84AX1TQdm8eLL?fd${QvkiW;MOg{7tcFnPRE_77C`CUN>z$^`Z; zv)sIp7?2vYn%_24ivt72wvj|lgBNH{$9A)Jvm(mZ}vkYuaq{b-NdEk6-l)_47dX3 z&3;4%BRf`TuB4`%ibeIZ@gEajD^r1P?oP8mGpRa9s;A;ElWY|+v0RW9D(^OvP+%%h zg!hUY--XRuNXRypAy6T z>bmPQ6VF=ZI@kQm>@T)EO9-o&nBkJ{er*F0O}!waYiR=L-mOPqf>`WOiaf*S+{!4 zI#`+kdYB}lBdpm!g~~_B`8usww|wDnrN53w1f{H27q5^c9v#4)D%2E5qLQo@$#O&q z9y7I))-s!0mDdRhozoy4|F4;4BK^fgR<$B(a`&r=?J|Xp0V#gMv?hf$`1~6=SMM-u z(Zv2qGaXSVo9U+rFVY4iCi-iqNixpvG+ZlXB5IX01DFb_i>Y7N)qTVhp^@ukF+4|~ zt@WE4MFZ4)rDiT!r%FI75VRTmjohFHA}(a2gDlGP^c!A;x{Rz-C* zEHtqd07;~5_I@*$gww|>rI;)V;aLHB88=jk)m*1NW8KBI6?9Q7B-6kx+_|h4X252n z15n9kfy+5wtDVh1TexJ#cuXM_&~Uh1m2c(F@zZb^9iMVlW;SFt6>3*-hPlwjTS?X1 zxM5Mok~664uhYydI%~yZNL|S#A-1N+YHO*Cb=}T+Sd#<2U&UQ;cH_RlbumRn;2Dx^ zXrk2|IwM#YWeYEBIm0#9c#5j{KhDta;|wm;NNKRhIxbPGpiHC) zdq0=ZAfU7bm@ayNOB`Uot7y}M93P}AQlVI^mDY2K+Tu*P&=i)0z(brTNL2$XSX~?E zimGx%h|Z+y!(38G^vhLjsRw5bk8s9Jg!Pu8=TXk^V_BM^1s>xJXM!ciiEWukAJ_S) znu>vH6&*2w`7j%}grJ52=JT|3iTY!yBu9jB2bZL=nX$OBkxS4Lv;~Y#Bi~^1O`I3W zP#KF-S!a(2h8+-RRj@L~FK0V1LFbqK{z4~XkbK)d{30OF!sLz^w&ai<@^BkAhAcWC$2(mgWAV1G}8iep1xoJ8sEg>Ji zze8ZJcKA;=E$L_X*rd5Ib$}e znK0&9zQ+yM%>|qOMipw90_JPH&l#@DSw0Pl(+6BaEQ-@EE$2gSkT#i+gS1EthLGlz zb`PK(tq#*yJ2+#WJWtRx*hi)mVhZvHTVDLw9H42fcAE87BIhwxyvwYwNDQ!)-DW|Y zcZnf?B)Z46ExtgYkk$&Zur}M<%Nc9n<>-=s!nsuRDe-=vIe2_34$P6}$T2ztKOgX^ z*<)WIP%eieg4}c!ZTieav8HVq`TxZk^8sHVz(!XfiHa%!jToUwMI%}BoDQUp~M{LJpmk+j6GxxrE97#B*h znlPOm66>{$Z%q2@Qx8q4$FadAP2(1V;Y zi`1K_tP9nuS4tsqPE(YAZ(=~tWjLdO=}kYF73w)SZ}iDQ+yfefpv?}o;=h@I zI7qgtbJE1bOo}tEOUIouiz1LQUq8*Ar|SwfxoM6x7ym1+#U-d_i?QB?U^iL}|3Txsc|y2;h7bxbwWQ#5yHpC$Vw~S9 zsvxR2Sq#IDU|77g^k%aHCf2?{K&xn(#n5*MS4q`b6u)rd69Y_#-(ne}$%!uzKmZMM zC<)6ghL~v{MT_}dE(^hkaI3|QmOvZx3X3u33d%NkZZi{nq7ZIcLRVT0Ul*qgnxn{+ z|Lqo2JBHM{@|aS$$}Ed1zCd8L#ZcrLo(t-Pkl1PtSWEON3XRl+=^eiVW)#2EVkj45 zJ#GP&rMm|37*gvrR=(S89F0t%hTdaw1ZAnAEPCl0OQH}6ltsrEi)v+9YPi=dj&MGf zwbo*|$ha`T^woV9UaNo^n04lG5#0q+fZlI0Tr#W%SDeuDH$8Ax?V7fH(4+x+kcBnt zO{6k4t4-%o=pQnf3$1yZ*(of>I`y!{@MN(IRa6euHfdJ!5i@6oB>QQpq+)l<#BkH2 zXH~=0)?*gKZ)AQDb1}=JsJcUv`1n~PVED<~@b77tw*PzD*p7cs8@JIMZLv~WCj_Jh zHk7j3PK%*%E2tuc7t=}_uC1W8RHmYzZn7lHW-|4wXv?3lc=R+T6)KJX&7^|1I8#Yg z%zDyd_yhV9A63h0HOYF)L=@7k2-l-!DW--*y4kFYbdeMqHIkJ@pEt`GjiIS|!Qw#WQ5cK9Xc?sK ziO$9SI<5Z&HCqrfxKqkf89&y%Wb!|9PMJtwHV30?@Y#}7AB(Y#Zcv5?k^ zg|SGa$(ZxLYT;``jYeCqSzNWDMy-w4P2yr%4dU-6DrV!dyS8xIWpbP^rq(eY-)h2D z(Ap5rlLEBy4HL6vQe+0YI{&a3J_t5BV7zc~`AxI=nUrm2RMJInnWHqxkN)A?CXJam z(@50t7Pnba7KdnqRzoZzqSPtd<7x>|(_{Ik$+*k~tyBfgkyG9=$EP8oFcy8+9G}tg z!dUdZvy?cFMpIVIO$zznH+#vL0(ChyB*vm2m_yYXD3@vULra1pL};b#_k}}((rt>c z;dE&jU<2A=IY$TvLt?E|PGu#eFqtT*`N(o!Af({Tticxy$}}jbR4F#hVnNNvCOaAi z3j4fMvt0ZjA93XfYJKEoll{ah}I)YXxOX2L?Osu?+hUxf*-zHAThnITR%n z6!kTpPt5UR->6(jqqtmwTVe{`XBlB|HEXp9znA3;2GJv6fS>-S(uGtj{n=S$Dvm_M zA*z&0BAWXOso`Ii#F(g1<_Tg?KewbrV@h42)F95QqcTOSt9)Te3Mr_+&`GNb(T4pd zLohYd7!3;{@k_HCSJ5V1DnJj+K;T!Fb1@DjrcyS~tf0Y=qEcD;+GGaS9Mjm}n5=`f zYYJzMR4Pf)Z_mon73*xFhK9ehcr@G><0sN)OH%ZJh1b~x=RgO~l0Du2`+`9v_xF}T zb;9D17@Q-qPG}VS!QvG(H^F45A1x^ZNrof*lPO=i64Z?G&!#J&W23B=sf8Ys4F?8S zKK_uYEVLeofnJN3ZF;eNyjU3blMY*wwV;x?v?G?3+E8O5hDwNj6%|5JwC?l1(!wjT8b#Hl=agj_Zor4MC*7CTF?ZVRqMpFqr!$t6xDu!sE9s&> zlLvL?U^UH@rC4-&Q@>?sP@lt<#nfm_t&G*^*0I@|BC|EKc%Hh^YB&;Fug=nMORa`e zHCqZ6DXK3JxXEfbO$@(EF~)sR`DUx(v}%yctZZ9QwAv)W=cOq|xId9ox1OJi)FsIVokTdjjsZL3USOS(-(jUh!{Vb(RH-dKyh%}gss zQ07mqv>GmPiGU$EY*!ye{&uV3oFU)hD&u&a8mr7M8N6Z|pt4%V9;;16nhvN>L}xAJ zr%J0eMb`{WK76&cm8yKlKx?==sPA~ zGwRF2fry~gud^CcDSbn#Kni2}{Z>P_qlJm7nk3;OgJ$rQ2h4&vfC{NWFaMy4bOQ|o z9cDg>vfgS;e`eIPl!vUw%+V-PAvG|ZYbas6p<`?%Q_LY*a88?3neh~8U3YCw(CZuRJqRuz)O zRslP##&S_LZnSMQo77peG!`k4!b+z#RiUajGt@lgI4@$8)g!Y7fy$5=4AWv!eZu59 zZPP|uyv4Qnzgd%lRILkN*^&@= z-o(pn$&Xuc=xGTueZk~UEub{CguZCiXS35|is>wGG5cN}sHxW1x2q#GBFRlJnRU+- z!unF)EJ><=*_5kFG$)N}k?5BKAu;%hb%+@U$JkM$NQKn!swo2o$6)6fV`}tz&6>zU z3=Dtbb*nM8V%kLmnj=*r2L|ciO=M^pdgr=KWrwX4*WxPTR+Hj}M6gz`qD^mDjRgRV zY{41oKg?=p)N8W=2K%N7SXfi3;k&KjZL@jZG#199+e{oXuUu2w<*K~h zRF^tm7#197G5@rBbRWd<`rol8V$`DP!M|%ll!=;oe$SeKTQ$tyZ zEW_akX#a55RY4I_vo#+gJ7eM;s;F90H%bjA%?LGK9g|e5?64*afq)jj;gf?gRJ1e{7;$6Y#RyT1A`H?zARpRcmuMwQiTGz*sG@=x%GeU&077=8j7-jGV&B z^Bk!%R6g~QA*D(Phji*Nx}g&I%9^0B8?d?4*M=yw`6=4<0xA5Bb&w7d z?+|@!4%I+{E=x^Os(fdfJEo!qRlwu})+8tMZe6*M9(rW&G@f|{SK!{d8By$NR{b0n+}iSe^FF%XMVwnm6*y2sRO zS`CG<=phsP7sQwbCClEWOrN5fxi$!%tSQ3m$<_Rl{CeyoBLL-%`91?@SnM0{4 zs8nSZD|OO(4(p!QwBP9EDQjj>Q0u539;M?`6A5D_kw2}IknKc<24GqREw&W;V1V)ch4+~jtGkhQnM2jfF7Tcf!ATzq^ zR$J0QD!QmEY@UGxwqJIe%{!2yg;cGKFNI-Mw-0m^Pi(BRB@PrRl4@7SSD_)M)n?2X zjCeW5U3bJc&*H7_w4ECg7fbcDT$V77-ay}7Hp5OK6Bad6|J^oYg21#!Nlc}UsI4#x zXn_=|3DJ9OL!_8Gu3Vg+XSIx@qv)qpQuPsY zAlkF$AU|qzu_*)Uq{nPY>|UE5lqkW+ZHX*l4n~U!LAt@LoOMTALub0Y-ImN^8V$RS z&>c1{luQWHaU0G4vXa^cRS8vvPP0m#Y-q6<%qClgzI22+71S@u0bQ>k(JSji(I?Ep ztOT=X8k+xRGj=#wjwaAg+ES1Yv>>q=;h0iaUgCS|Y)B0&n@u@p#Zf?>wrRWh18qs#QHEoq<$Z0I@Lxxd@hW=?qC^R{G6$N63f?s&_kwBdz+s}i-%i#B5> z&H6GUBrEEcvj<)wH5A~)ut-r~vbmT}nk^{m%V$F|G0~Xr6*Ff5w{yj);IDtx#z$j% zFu-NwUo+uY+lw|AY5Uak49HlB4-dwRoEB&hw&lK(JE zVjr2?J){0jv)OW^mv5PHW)#DvXF<>TDx0yOspH6Hy`BIaO*rf$W@{Q8Qzl0Zu+l3WGUQfl{Kc()Z4G;C;KE9~3o#0cp zq^O|cxfGqCr%eZAu39@0sdUE1Vxe`Kp>NQq87amq{WgzZiZ52=hF^y3EtRnvZTfhd z-Ed)zhET1n)N0oecDvp1MGTmVn0RGOQOiU>U1c}ykdawjy4vo+4N+`pI#nAe6gU(D1ut2%$#p62cgBok8rVG_j4GuX#8#mew2b8(; z1M9S%_CXruaUlT-F;@&p;wF0%V?%SP_X+zs8ei(+OZr05-%KKC!A6CW<`F(=PpYlg z_QT5sJP!R7CRF+bk=aC%#jE%UbP0nN+P?gbeNPWhh+<<$41z{nk=;V9W z&SP191MNBc5PZ@ifeO)#Mo_P)A^-FCL{=8tMQ#0peX#kABxRUE_I}asVHvn(=L-aU z;qVrFl9|F(Ni5@P@p%07DnqP#fJSRgH?@~8GReAM_1*GLC2Cy452_C(wY zVKhK4|CcUrI>B`|C===DaR}Bl@)tV&wDrpP&g{3JYYKz5W2zm6`10(^88@+(zp@WE z3l!8*Axhi}6 zzS9N!DvIaCRONsv8TdH9T$b2gCA04b?I~zQ#_Q*_wAh62?JSnjjEi6=aepvHl^H}+ z6BmYL>Q{fXdur=_?6iTdihnXyv7Ug_*h<;|vuRs&6&;KA*o`2s@^P#q${|xM%st3~ z)mT}kc)&E!YtPgt^8W@@Wt@UMY)@wP9>Y7>Fe;>mBlf|H?8oy16;zBxYN$SWRgT&* z%ql*mpyrs}#dhQo_s8u+Wome8%s?q~6-~H*u_vHx88HXH+LJJ1q)b#L&9^^c_u`8X z7$+BNQ6#_Fheu^8Dqu_wOBY1eiXK37Qe*A214_$`ZTnMpZ-XBA5|wAF7$kYxNMg_h^a5wWYM7oKm^jk)Wj@smr1_ItC3qGD*hG zkEM=bIHWRJTdwg1gD7JfUT$)v3r4&V+M=koj9(yy4JN(WagNawA%JaagkR=J)RKL{ zATrD?js#JXqIiqya=C-|V{EM^q%Ds0{|$p>BE8jtA%&qOnj~srmn$63Pyn6gn0lLI zkWAUTOxO!Hszu2!D0HPGAwKnXhfBt{6QHBE%HhQW_^}$~0(|L*OjkRMDjl->{heZFq4&yZi)gooDF*V63Pr{?L66%;yQ;{d)uAVZ6!!Q?wxd^#fH?wz>3>Sx>ptFs{wv*57_m z5(__U!WKmSpSrF(JdP{L@7~^S!;It@0-N0&?x8Px!*DFiPU5}T@mk8#fo?Qvje0%m zmS=hlIWj|xF*7qWGc!Z%Ff%hb_f^$<{c0rNAN_sxtH<^Dy?WJH#fyA~(Boe0Gipj` zkzk3>=uyH+bg3o{EI{wWmB z(_%Xt{4By;2!S^G9(r4Stbv<+5Awy@X1@$IO$FQOvqcLO;r`DW020`$p>CO-x7-#u zCW7Ziz~82I5^b0GISXGeY}W!{G0Aov@U%nIy@K5gk-M!HJ^=#RU#%4XZ? zkr^OOGVIX;#K5k+S2JNDwMU*l|tE)EB|W3C$>g zynsHb870`$0DVd`@@{{kPitBe$@7e6lpH~y)y%95W`Mx^IjvHGYPVpW*Mh|8kI7un zOtQ@co-axOM+BEN(<$ZiHWuWv7A(4@Le?S!9al8dDircADym)8jJ|NM3^82O+UGN9 zoe9PxvafVq)3T_6Tzpci@LVsGhtb5zS;ZSLytL#qSqytq3qEEQTrf%iQ9Y%ZIJIG^ zpVqV@*)VxVlbdxRa|iG6vUN~>&uaJe$uwo0E}8mLu}`L_Rcg-I7S8SXBK|q3Xx~W3 z^N@lxaPor4n`|yGYJp5YnwIRxtd}$cy9ugnb?pwu#YXvt)`77^d~a&sA5d-C`M0z{*0S@qgF=5>`@w(*@~&HWN7F<- zysH__8Lue}!h4Ts9Btm$^ad+e>-Nqxe4y!tOhc{P)yjRS{h-ALuODeTsI>z5u~zxp zqU+F=E-*gP?f?dEya87ga-WhP+O0zAKGWJ!AlU1o&(F0A6$+kK6UKufh~-PI6S&jT zzq$#`s=yupS6Tf7tJ+x5#6?c>4#(RNqp%sA31{ z4$|*|fTAE`uh#{i)C|@!?Rvbdw>?sK8KU0}{(YMvFrkoHjiEZ0IsVHg+p~&rMmL%*< zm5GG%N$wz?Cn!eLGl3W@*-R!#X$b^gVMpr0L1L+3ZJ=OPjh7aED$gBKUQWu5}OfsM)Cp zi(a8oy6@71KsMNoIA`wGjjVLqqgNtb^fI)L+^ZW|nc_ZO9v=5}yk^fF8itbq_W5Xs#(9)B~Oi`fO+;@sQr3;P!`Eu%0G5$oOHsqheyS1?~~OLxbfM zEigOO2l@aokLn#2)AxEz4{}e(^$LK#^9j9Ei&e;i;YRN5q<$x7``%CK9XY?d(+^TU zt#`lk1A7W0duR1Z#C}}o^dK@sBoNxewSj(`4(co zsCQHjew3GVTC`vsBHqh-d&g@kkM@dwr_bZDUezmL7wE@#P4CzU*VW~ruj_X+E)V~t zE|m6YcM2K3p$pAAoKR}|=5OkqNY#$~2*sfKr*xqNNNhaas()Jg6`JwlgyR`qJk!FH zC{kT~R7>!dMgD8DaFR*Uy{^kUfgV;fnM!*@m#gz0WP&X3 zX5ZB1b{KR-?8mIP$~?h>yVUYF_XIn%cXWQS0(;hXb*Ttb2is>fEr`Z4T{MOCp6+k5 z#pNAbPrk1ka3q0eZy)GV>q`14Sj`z5-a3ES#tAOhaEbYmF5e}8XTA&s+jhLq{V_@KV0rz8N{(W6Tw2M0+1fW8 zY<{J;-s8a1J`wx64WF{{jmjOjM0TqU+XmxS->MP{->_R5WXKbH$t7!Mp6k}zxxt2i z5hTLM4M!q2q>USegw5Mf!!HRMX_dtu){1XCb=bv+8y!$Suw27Y0WO3`7)tnH#nuLk zzXQCHhTl|j>O5wY@=Q18by)wSm1n^Qkv^`nA-E*aF>NA2$&GEp#>|du6E{@(_%>Wn zC7xi&dptiyE1L)Z6Aiy($h^q@0Rf*-QWnm*c0jjDhQG5DZf?QOftWJQ)?}3rXyc+O zO)+#pu-KStn4qTdr~`~?t$dR_rW{aFgSK^-wh9QQc9!`yhhv*qx8gys)v%mm1w90O z*S*5e4|y(mpgp&;Ra&Spv#J#n6**S>;VI_@3$A_FD6cK_WTLKLbgkhJve@NZ=tfRs z-p+dK41eTAC*OYPdcz;#I~!3Sa!yei0x3my#Wr^zqQlwClajH zU4~TGMXl4_hMY~wxvpPij~X@5;5e*o-IBLgMGkL8P;+OWDjM8^!RrLy#PWWlJsso1 z1n+^(L~i8y)Geoi2REnac^Iq5ubv@^Z?-3pVBPlFsa~zhK}IPPa_9NobzGXyE=IIKGJi zyl+$pMD_52?+T0G4~ecX1TjrvmGn~qMO*(&)FX)Rb5V6D zq%VvBE^J9cU-}s%*?eULVF3+hz~KDjYojBkQDbEa*`{7#S-+>%{~IF!&7&%d)(dZm zoWa(x)`gJYlDo**Ad^}Y$a=8ZCca*7%Meq?RJys1Lrq^$xJemiQWHce5BD8Osb_?F z2Zp2ej$THZosgw825@gR%7k&nf)^_2ZM11tH@R++j3t^>d31sV8hB$&*_ff1^a5?z zvF4o^gL;xdruD{&5`lp)WpHo2*-_Le3{`UYm|%7gT+lDgXYh1jqS;& z^BE#`WT5utnlCXT_^`f{r2ht~@0%zIR4_cFFq#BK$l*ZZMiR0Ylk zzajpr(JkN&(nep-y{%j+PnTVrAmcb5*VvA=*-x`N+aIFYBH9xTK3heHghFi-WfTL? zcF`F8VP}U3Nu$F~84vc&T>?>O+--J(sinpO4eTtcp<0FN`aLGj*wx+P(%!QQkmz0$ zr%`{#t*+l^1|TrDw(9!*<~@MKY=-u(vKC%q77v&>zY3@B#j3nhYCdSnl_pGV!tDV! z-*Cvp>5eAe>iWYb&h)=W*(}QLX2|jDy4UV`@gKavLiDG`Pm8{_!?|HKld}HNe1A2ix)yOk> zNbkimQNUIb5_^dfVq&WkMSod#OkbdNJzd9og*qV%@M>AaAY*7muTd8KN`k~*r^KKz zU}g)=h7>W@H_RVl30k{k-F(sV-ZbyKjVtIbfn@^Ad+WCGfsf;WIDGqTjA?K{wR)7}kN^(4*bR_@Q4S5@1cO}Yd*Z?3 zuz);q1OZj!A8Bg9>TJGb4G-{%!5?>`kr9-F5Smm;8yUd9UJbt&g4UGrtWhGvR?hY; zcya}U<>-Jsfny^m7{VaYJs!a6P4q0Nsxg#YR~MBIA!-;$#s+Yp5|PqaFS9Z(fP;!~ zqc>gtJLK^J8pYswA7)mDnh?Oo1JXgsP7KKNJ|!EvKhHRQo(SMDj_Pvm6|m7>*jK9|~%+C-7(d4V=_D^Aq& z1HAKz@}SFtvJfOg7{M1(R>UTZVqZk^fp^GsFT9u-YCW-pS{Y2+Y7eoLB*9h;%cu?@ z9a<~Ei#!OwzG?Q>QpuooQ`y&%JTP@g2QKZ_lPA)P1+N>Z+4`%CjiSbJl}Yhz zqUwZrY6d#4Gq4$k+07y*%j!cp+d|o7OOLG-m;Wxf$!*w1EEpubOg-)ow+G~<26Plr z(f(#?M*t^r6v-nkp51Ie1`iZUS#Rg(7tE$D8QaM5ONt~3!GxeUDc@BsHta$$;io99RL`Mz zi%$n|t0(+DLnC=r3*^mic-D`Y@(UdFGf?cKfSgRfB3;6l;DaQUjAts2X#A|3D7eBOgT?{z<4 zA}kDnZ&359F3Kgvn|?_01-W{Qdb=-W3`s?PoAT;Afp>Cw$BzibfQ5dytZI<;9<491 zNNx*QQnbWcqgY-4zLF4~AJMX>_yOa?f2Jh=L*iq^a+3o&`Y0eTUSU{~%bt(RphD2ZXE2dlJOGTJ(X{&=ubKbXQ z8Dq|af;?mB0Q2YIAYCs&JT+(%w`h4of_#-iel76f4Q(3>j6=hMxW1NR#UAiT@8LmQ zUzc&KdUhRg8+Ok}YDcO#@ZSM;9v)GSBF{86Jm}Wf=dEA5g@(~VKEKd!X2gaIf7|tV zP_EO(4ZY{mzh1`#{i{N-OVGy}I5sGk`=|{9H@Uuk$he?f+$cBkzn$^EB$QMCOn!oj zzo8zlg(n7On*|OioZzK;85b^O;VyWRFZjBQGZ5}Ki&kIMLzx_`$T(m(D&G@-IXopO z*J9%0y;zrXrv~9gEj;v)FKVYD2h8M1&h#Mu?H80MC2Izn!Pw z9`pwEr<=llz@ch@zbvbvDijZ0m{&cx6=Pp)lL;f~cIxSIQ z=(JcmELF8Gf8Sh}bC;JlEyim8>|R zd4VsdEr@TGN=vBID7V$BMAR6R(;DR@s;?P;3tE?R*9JR@KQrU4v}`Qvg7~NGQladH z_xj+y@-M#XufQ_MhTz@OQT#7hx@-)>ACKwMpW2par<+s|VZ2hhY!2QhgTVchY9k`# zmLU8KS*Qz$6_a$^ia*?Lb{k}*+f;>8Ch$<8F6U|;zWbea@C#k|RkskAaVvcDd*9K1 zT5w_N`mP~}pN4{I;WbFGs8UKM#CN!T+>OPq=d@^mCeiEU8qQgN&M=BH@ ztyFZZoucD+C_2$z(a8>qPIXjtx|2^wLU$@Ue3zmlcPl!2kD_BgRCN4aMJIlw=;V(T zow`ra>EH9|Xz2G99sUDFNB&UJ(LYjj?2i>4|B0d#f1>E*pDH@_XNpe$xlhMJf1&8` zUn)A%S<%tIQgrNoMaO@t=)_+uIvG-QDy-;q#HZt-sG`F$MMvU_jwTcxODZ~^QgkA% z=;YrhI`y}TPXC=xCqjR(={``bh;s+ z#*L80K)8{CNREMM69X}uf%tD3Nc3SKnP(u?%s|>vARKZT2)8g0>B~U0z(B0XK-^;> zQDPw3kAYNw2GRo*h=c|*5Ppn-$bT{r{VxV$|II-Be;7#oF9XRz-(@H@n4|O%hN7XN z9EFE*6dBG@bOcASksQTGag-R%QSxz)Qe!wuk7Xzp8plz1JV%iU97QK`6nlcB_#}=J zlQ~LG;V3ngqx3X};-Tprg=cUSnaNRf7Dut!9L48wl$gs=avn#i`5dJeFq8-_<4246NISOClC~}pf=rxXF*Exzm$x-44N6DKUrJmv_{WL?7&@&u`pXDg>97oaT zIf}i&QT#=Y5-)L-e3_%vD;%X?WhfeYjid1E97W#XDEcNxv9~yizs*tN9gdRka+G?H zqxAa>#X=u&6#kH-$VVJSKjtX*2}kiyIZAxSQSx(+QeSYC{*s}1=qrxGUvm`shNI{$ zj$+?(6d&|E{`?n-CkAtr9Kum*C`ajG3?)LtISP;9C^C|x=qQe2qdAH{&QW3vN6E1q zrN(iT9?wuRG=Zb=M2;d)a1@=yQEW0t@hKc7rgD^=#!+fIN9h?1r9v|~3eVywGMl65 z9FAghIf~EYC^4U-SE zdW}NSP-rbj;dLBE)^ilyz)@@?NAXP@B{p-E+`>_6D@W;V42476ISTLKD6*5I=q`?8 zyE%&Q;V7|}qvSr0Qu{edA7CgFI>=G@5J!>297T_C6g$dM{1``x;~XVVaFja9QTh}^ z(a>p*!e=;&oaHEbj-%Llj^Yb3iaumMCQRF&D(I+{I-QXyG zlcU5_93`LTDD@0S>1P>=ho0jo{5(gI7dVQ($WiPij^ZzKlz4@sGC>i>QqwvQZMLyvu`YA`T&p3*I z&QanEj*?$;l=_OJ^w$ifLf>!{zQs}GTaKcGzUR+>(Nt_ONAV#XC5Cd89L7;aag;jEQThx+iO^Y&!sj@OoaZQdfuq<(j^dX%N?hhB zd4;3YRgTiv7)pk&a}<7(qsR@8qTlh_dH6H3x<)GlhChm0=XT8{d^LQ3F7MW7^2N>_ z9oO-|?62+Q?r&`I;3ZIJt!Otlgd?fWf%;Ou&;9fdtj=ib=zeQ}*STFm{@M3_c(`2b MY~sHO@wV#!0LM cap { + b.buf = append(b.buf[:cap], make([]byte, size-cap)...) + } else { + b.buf = b.buf[:size] + } + return b.buf +} diff --git a/vendor/github.com/ncruces/go-sqlite3/internal/util/alloc_unix.go b/vendor/github.com/ncruces/go-sqlite3/internal/util/alloc_unix.go new file mode 100644 index 000000000..2b1d3916b --- /dev/null +++ b/vendor/github.com/ncruces/go-sqlite3/internal/util/alloc_unix.go @@ -0,0 +1,67 @@ +//go:build unix && !sqlite3_nosys + +package util + +import ( + "math" + + "github.com/tetratelabs/wazero/experimental" + "golang.org/x/sys/unix" +) + +func virtualAlloc(cap, max uint64) experimental.LinearMemory { + // Round up to the page size. + rnd := uint64(unix.Getpagesize() - 1) + max = (max + rnd) &^ rnd + + if max > math.MaxInt { + // This ensures int(max) overflows to a negative value, + // and unix.Mmap returns EINVAL. + max = math.MaxUint64 + } + + // Reserve max bytes of address space, to ensure we won't need to move it. + // A protected, private, anonymous mapping should not commit memory. + b, err := unix.Mmap(-1, 0, int(max), unix.PROT_NONE, unix.MAP_PRIVATE|unix.MAP_ANON) + if err != nil { + panic(err) + } + return &mmappedMemory{buf: b[:0]} +} + +// The slice covers the entire mmapped memory: +// - len(buf) is the already committed memory, +// - cap(buf) is the reserved address space. +type mmappedMemory struct { + buf []byte +} + +func (m *mmappedMemory) Reallocate(size uint64) []byte { + com := uint64(len(m.buf)) + res := uint64(cap(m.buf)) + if com < size && size < res { + // Round up to the page size. + rnd := uint64(unix.Getpagesize() - 1) + new := (size + rnd) &^ rnd + + // Commit additional memory up to new bytes. + err := unix.Mprotect(m.buf[com:new], unix.PROT_READ|unix.PROT_WRITE) + if err != nil { + panic(err) + } + + // Update committed memory. + m.buf = m.buf[:new] + } + // Limit returned capacity because bytes beyond + // len(m.buf) have not yet been committed. + return m.buf[:size:len(m.buf)] +} + +func (m *mmappedMemory) Free() { + err := unix.Munmap(m.buf[:cap(m.buf)]) + if err != nil { + panic(err) + } + m.buf = nil +} diff --git a/vendor/github.com/ncruces/go-sqlite3/internal/util/alloc_windows.go b/vendor/github.com/ncruces/go-sqlite3/internal/util/alloc_windows.go new file mode 100644 index 000000000..8936173b4 --- /dev/null +++ b/vendor/github.com/ncruces/go-sqlite3/internal/util/alloc_windows.go @@ -0,0 +1,76 @@ +//go:build !sqlite3_nosys + +package util + +import ( + "math" + "reflect" + "unsafe" + + "github.com/tetratelabs/wazero/experimental" + "golang.org/x/sys/windows" +) + +func virtualAlloc(cap, max uint64) experimental.LinearMemory { + // Round up to the page size. + rnd := uint64(windows.Getpagesize() - 1) + max = (max + rnd) &^ rnd + + if max > math.MaxInt { + // This ensures uintptr(max) overflows to a large value, + // and windows.VirtualAlloc returns an error. + max = math.MaxUint64 + } + + // Reserve max bytes of address space, to ensure we won't need to move it. + // This does not commit memory. + r, err := windows.VirtualAlloc(0, uintptr(max), windows.MEM_RESERVE, windows.PAGE_READWRITE) + if err != nil { + panic(err) + } + + mem := virtualMemory{addr: r} + // SliceHeader, although deprecated, avoids a go vet warning. + sh := (*reflect.SliceHeader)(unsafe.Pointer(&mem.buf)) + sh.Cap = int(max) // Not a bug. + sh.Data = r + return &mem +} + +// The slice covers the entire mmapped memory: +// - len(buf) is the already committed memory, +// - cap(buf) is the reserved address space. +type virtualMemory struct { + buf []byte + addr uintptr +} + +func (m *virtualMemory) Reallocate(size uint64) []byte { + com := uint64(len(m.buf)) + res := uint64(cap(m.buf)) + if com < size && size < res { + // Round up to the page size. + rnd := uint64(windows.Getpagesize() - 1) + new := (size + rnd) &^ rnd + + // Commit additional memory up to new bytes. + _, err := windows.VirtualAlloc(m.addr, uintptr(new), windows.MEM_COMMIT, windows.PAGE_READWRITE) + if err != nil { + panic(err) + } + + // Update committed memory. + m.buf = m.buf[:new] + } + // Limit returned capacity because bytes beyond + // len(m.buf) have not yet been committed. + return m.buf[:size:len(m.buf)] +} + +func (m *virtualMemory) Free() { + err := windows.VirtualFree(m.addr, 0, windows.MEM_RELEASE) + if err != nil { + panic(err) + } + m.addr = 0 +} diff --git a/vendor/github.com/ncruces/go-sqlite3/internal/util/bool.go b/vendor/github.com/ncruces/go-sqlite3/internal/util/bool.go new file mode 100644 index 000000000..8427f3085 --- /dev/null +++ b/vendor/github.com/ncruces/go-sqlite3/internal/util/bool.go @@ -0,0 +1,22 @@ +package util + +import "strings" + +func ParseBool(s string) (b, ok bool) { + if len(s) == 0 { + return false, false + } + if s[0] == '0' { + return false, true + } + if '1' <= s[0] && s[0] <= '9' { + return true, true + } + switch strings.ToLower(s) { + case "true", "yes", "on": + return true, true + case "false", "no", "off": + return false, true + } + return false, false +} diff --git a/vendor/github.com/ncruces/go-sqlite3/internal/util/const.go b/vendor/github.com/ncruces/go-sqlite3/internal/util/const.go new file mode 100644 index 000000000..86bb9749d --- /dev/null +++ b/vendor/github.com/ncruces/go-sqlite3/internal/util/const.go @@ -0,0 +1,117 @@ +package util + +// https://sqlite.com/matrix/rescode.html +const ( + OK = 0 /* Successful result */ + + ERROR = 1 /* Generic error */ + INTERNAL = 2 /* Internal logic error in SQLite */ + PERM = 3 /* Access permission denied */ + ABORT = 4 /* Callback routine requested an abort */ + BUSY = 5 /* The database file is locked */ + LOCKED = 6 /* A table in the database is locked */ + NOMEM = 7 /* A malloc() failed */ + READONLY = 8 /* Attempt to write a readonly database */ + INTERRUPT = 9 /* Operation terminated by sqlite3_interrupt() */ + IOERR = 10 /* Some kind of disk I/O error occurred */ + CORRUPT = 11 /* The database disk image is malformed */ + NOTFOUND = 12 /* Unknown opcode in sqlite3_file_control() */ + FULL = 13 /* Insertion failed because database is full */ + CANTOPEN = 14 /* Unable to open the database file */ + PROTOCOL = 15 /* Database lock protocol error */ + EMPTY = 16 /* Internal use only */ + SCHEMA = 17 /* The database schema changed */ + TOOBIG = 18 /* String or BLOB exceeds size limit */ + CONSTRAINT = 19 /* Abort due to constraint violation */ + MISMATCH = 20 /* Data type mismatch */ + MISUSE = 21 /* Library used incorrectly */ + NOLFS = 22 /* Uses OS features not supported on host */ + AUTH = 23 /* Authorization denied */ + FORMAT = 24 /* Not used */ + RANGE = 25 /* 2nd parameter to sqlite3_bind out of range */ + NOTADB = 26 /* File opened that is not a database file */ + NOTICE = 27 /* Notifications from sqlite3_log() */ + WARNING = 28 /* Warnings from sqlite3_log() */ + + ROW = 100 /* sqlite3_step() has another row ready */ + DONE = 101 /* sqlite3_step() has finished executing */ + + ERROR_MISSING_COLLSEQ = ERROR | (1 << 8) + ERROR_RETRY = ERROR | (2 << 8) + ERROR_SNAPSHOT = ERROR | (3 << 8) + IOERR_READ = IOERR | (1 << 8) + IOERR_SHORT_READ = IOERR | (2 << 8) + IOERR_WRITE = IOERR | (3 << 8) + IOERR_FSYNC = IOERR | (4 << 8) + IOERR_DIR_FSYNC = IOERR | (5 << 8) + IOERR_TRUNCATE = IOERR | (6 << 8) + IOERR_FSTAT = IOERR | (7 << 8) + IOERR_UNLOCK = IOERR | (8 << 8) + IOERR_RDLOCK = IOERR | (9 << 8) + IOERR_DELETE = IOERR | (10 << 8) + IOERR_BLOCKED = IOERR | (11 << 8) + IOERR_NOMEM = IOERR | (12 << 8) + IOERR_ACCESS = IOERR | (13 << 8) + IOERR_CHECKRESERVEDLOCK = IOERR | (14 << 8) + IOERR_LOCK = IOERR | (15 << 8) + IOERR_CLOSE = IOERR | (16 << 8) + IOERR_DIR_CLOSE = IOERR | (17 << 8) + IOERR_SHMOPEN = IOERR | (18 << 8) + IOERR_SHMSIZE = IOERR | (19 << 8) + IOERR_SHMLOCK = IOERR | (20 << 8) + IOERR_SHMMAP = IOERR | (21 << 8) + IOERR_SEEK = IOERR | (22 << 8) + IOERR_DELETE_NOENT = IOERR | (23 << 8) + IOERR_MMAP = IOERR | (24 << 8) + IOERR_GETTEMPPATH = IOERR | (25 << 8) + IOERR_CONVPATH = IOERR | (26 << 8) + IOERR_VNODE = IOERR | (27 << 8) + IOERR_AUTH = IOERR | (28 << 8) + IOERR_BEGIN_ATOMIC = IOERR | (29 << 8) + IOERR_COMMIT_ATOMIC = IOERR | (30 << 8) + IOERR_ROLLBACK_ATOMIC = IOERR | (31 << 8) + IOERR_DATA = IOERR | (32 << 8) + IOERR_CORRUPTFS = IOERR | (33 << 8) + IOERR_IN_PAGE = IOERR | (34 << 8) + LOCKED_SHAREDCACHE = LOCKED | (1 << 8) + LOCKED_VTAB = LOCKED | (2 << 8) + BUSY_RECOVERY = BUSY | (1 << 8) + BUSY_SNAPSHOT = BUSY | (2 << 8) + BUSY_TIMEOUT = BUSY | (3 << 8) + CANTOPEN_NOTEMPDIR = CANTOPEN | (1 << 8) + CANTOPEN_ISDIR = CANTOPEN | (2 << 8) + CANTOPEN_FULLPATH = CANTOPEN | (3 << 8) + CANTOPEN_CONVPATH = CANTOPEN | (4 << 8) + CANTOPEN_DIRTYWAL = CANTOPEN | (5 << 8) /* Not Used */ + CANTOPEN_SYMLINK = CANTOPEN | (6 << 8) + CORRUPT_VTAB = CORRUPT | (1 << 8) + CORRUPT_SEQUENCE = CORRUPT | (2 << 8) + CORRUPT_INDEX = CORRUPT | (3 << 8) + READONLY_RECOVERY = READONLY | (1 << 8) + READONLY_CANTLOCK = READONLY | (2 << 8) + READONLY_ROLLBACK = READONLY | (3 << 8) + READONLY_DBMOVED = READONLY | (4 << 8) + READONLY_CANTINIT = READONLY | (5 << 8) + READONLY_DIRECTORY = READONLY | (6 << 8) + ABORT_ROLLBACK = ABORT | (2 << 8) + CONSTRAINT_CHECK = CONSTRAINT | (1 << 8) + CONSTRAINT_COMMITHOOK = CONSTRAINT | (2 << 8) + CONSTRAINT_FOREIGNKEY = CONSTRAINT | (3 << 8) + CONSTRAINT_FUNCTION = CONSTRAINT | (4 << 8) + CONSTRAINT_NOTNULL = CONSTRAINT | (5 << 8) + CONSTRAINT_PRIMARYKEY = CONSTRAINT | (6 << 8) + CONSTRAINT_TRIGGER = CONSTRAINT | (7 << 8) + CONSTRAINT_UNIQUE = CONSTRAINT | (8 << 8) + CONSTRAINT_VTAB = CONSTRAINT | (9 << 8) + CONSTRAINT_ROWID = CONSTRAINT | (10 << 8) + CONSTRAINT_PINNED = CONSTRAINT | (11 << 8) + CONSTRAINT_DATATYPE = CONSTRAINT | (12 << 8) + NOTICE_RECOVER_WAL = NOTICE | (1 << 8) + NOTICE_RECOVER_ROLLBACK = NOTICE | (2 << 8) + NOTICE_RBU = NOTICE | (3 << 8) + WARNING_AUTOINDEX = WARNING | (1 << 8) + AUTH_USER = AUTH | (1 << 8) + + OK_LOAD_PERMANENTLY = OK | (1 << 8) + OK_SYMLINK = OK | (2 << 8) /* internal use only */ +) diff --git a/vendor/github.com/ncruces/go-sqlite3/internal/util/error.go b/vendor/github.com/ncruces/go-sqlite3/internal/util/error.go new file mode 100644 index 000000000..1f5555fd3 --- /dev/null +++ b/vendor/github.com/ncruces/go-sqlite3/internal/util/error.go @@ -0,0 +1,106 @@ +package util + +import ( + "runtime" + "strconv" +) + +type ErrorString string + +func (e ErrorString) Error() string { return string(e) } + +const ( + NilErr = ErrorString("sqlite3: invalid memory address or null pointer dereference") + OOMErr = ErrorString("sqlite3: out of memory") + RangeErr = ErrorString("sqlite3: index out of range") + NoNulErr = ErrorString("sqlite3: missing NUL terminator") + NoBinaryErr = ErrorString("sqlite3: no SQLite binary embed/set/loaded") + BadBinaryErr = ErrorString("sqlite3: invalid SQLite binary embed/set/loaded") + TimeErr = ErrorString("sqlite3: invalid time value") + WhenceErr = ErrorString("sqlite3: invalid whence") + OffsetErr = ErrorString("sqlite3: invalid offset") + TailErr = ErrorString("sqlite3: multiple statements") + IsolationErr = ErrorString("sqlite3: unsupported isolation level") + ValueErr = ErrorString("sqlite3: unsupported value") + NoVFSErr = ErrorString("sqlite3: no such vfs: ") +) + +func AssertErr() ErrorString { + msg := "sqlite3: assertion failed" + if _, file, line, ok := runtime.Caller(1); ok { + msg += " (" + file + ":" + strconv.Itoa(line) + ")" + } + return ErrorString(msg) +} + +func ErrorCodeString(rc uint32) string { + switch rc { + case ABORT_ROLLBACK: + return "sqlite3: abort due to ROLLBACK" + case ROW: + return "sqlite3: another row available" + case DONE: + return "sqlite3: no more rows available" + } + switch rc & 0xff { + case OK: + return "sqlite3: not an error" + case ERROR: + return "sqlite3: SQL logic error" + case INTERNAL: + break + case PERM: + return "sqlite3: access permission denied" + case ABORT: + return "sqlite3: query aborted" + case BUSY: + return "sqlite3: database is locked" + case LOCKED: + return "sqlite3: database table is locked" + case NOMEM: + return "sqlite3: out of memory" + case READONLY: + return "sqlite3: attempt to write a readonly database" + case INTERRUPT: + return "sqlite3: interrupted" + case IOERR: + return "sqlite3: disk I/O error" + case CORRUPT: + return "sqlite3: database disk image is malformed" + case NOTFOUND: + return "sqlite3: unknown operation" + case FULL: + return "sqlite3: database or disk is full" + case CANTOPEN: + return "sqlite3: unable to open database file" + case PROTOCOL: + return "sqlite3: locking protocol" + case FORMAT: + break + case SCHEMA: + return "sqlite3: database schema has changed" + case TOOBIG: + return "sqlite3: string or blob too big" + case CONSTRAINT: + return "sqlite3: constraint failed" + case MISMATCH: + return "sqlite3: datatype mismatch" + case MISUSE: + return "sqlite3: bad parameter or other API misuse" + case NOLFS: + break + case AUTH: + return "sqlite3: authorization denied" + case EMPTY: + break + case RANGE: + return "sqlite3: column index out of range" + case NOTADB: + return "sqlite3: file is not a database" + case NOTICE: + return "sqlite3: notification message" + case WARNING: + return "sqlite3: warning message" + } + return "sqlite3: unknown error" +} diff --git a/vendor/github.com/ncruces/go-sqlite3/internal/util/func.go b/vendor/github.com/ncruces/go-sqlite3/internal/util/func.go new file mode 100644 index 000000000..be7a47c2f --- /dev/null +++ b/vendor/github.com/ncruces/go-sqlite3/internal/util/func.go @@ -0,0 +1,193 @@ +package util + +import ( + "context" + + "github.com/tetratelabs/wazero" + "github.com/tetratelabs/wazero/api" +) + +type i32 interface{ ~int32 | ~uint32 } +type i64 interface{ ~int64 | ~uint64 } + +type funcVI[T0 i32] func(context.Context, api.Module, T0) + +func (fn funcVI[T0]) Call(ctx context.Context, mod api.Module, stack []uint64) { + fn(ctx, mod, T0(stack[0])) +} + +func ExportFuncVI[T0 i32](mod wazero.HostModuleBuilder, name string, fn func(context.Context, api.Module, T0)) { + mod.NewFunctionBuilder(). + WithGoModuleFunction(funcVI[T0](fn), + []api.ValueType{api.ValueTypeI32}, nil). + Export(name) +} + +type funcVII[T0, T1 i32] func(context.Context, api.Module, T0, T1) + +func (fn funcVII[T0, T1]) Call(ctx context.Context, mod api.Module, stack []uint64) { + fn(ctx, mod, T0(stack[0]), T1(stack[1])) +} + +func ExportFuncVII[T0, T1 i32](mod wazero.HostModuleBuilder, name string, fn func(context.Context, api.Module, T0, T1)) { + mod.NewFunctionBuilder(). + WithGoModuleFunction(funcVII[T0, T1](fn), + []api.ValueType{api.ValueTypeI32, api.ValueTypeI32}, nil). + Export(name) +} + +type funcVIII[T0, T1, T2 i32] func(context.Context, api.Module, T0, T1, T2) + +func (fn funcVIII[T0, T1, T2]) Call(ctx context.Context, mod api.Module, stack []uint64) { + fn(ctx, mod, T0(stack[0]), T1(stack[1]), T2(stack[2])) +} + +func ExportFuncVIII[T0, T1, T2 i32](mod wazero.HostModuleBuilder, name string, fn func(context.Context, api.Module, T0, T1, T2)) { + mod.NewFunctionBuilder(). + WithGoModuleFunction(funcVIII[T0, T1, T2](fn), + []api.ValueType{api.ValueTypeI32, api.ValueTypeI32, api.ValueTypeI32}, nil). + Export(name) +} + +type funcVIIII[T0, T1, T2, T3 i32] func(context.Context, api.Module, T0, T1, T2, T3) + +func (fn funcVIIII[T0, T1, T2, T3]) Call(ctx context.Context, mod api.Module, stack []uint64) { + fn(ctx, mod, T0(stack[0]), T1(stack[1]), T2(stack[2]), T3(stack[3])) +} + +func ExportFuncVIIII[T0, T1, T2, T3 i32](mod wazero.HostModuleBuilder, name string, fn func(context.Context, api.Module, T0, T1, T2, T3)) { + mod.NewFunctionBuilder(). + WithGoModuleFunction(funcVIIII[T0, T1, T2, T3](fn), + []api.ValueType{api.ValueTypeI32, api.ValueTypeI32, api.ValueTypeI32, api.ValueTypeI32}, nil). + Export(name) +} + +type funcVIIIII[T0, T1, T2, T3, T4 i32] func(context.Context, api.Module, T0, T1, T2, T3, T4) + +func (fn funcVIIIII[T0, T1, T2, T3, T4]) Call(ctx context.Context, mod api.Module, stack []uint64) { + fn(ctx, mod, T0(stack[0]), T1(stack[1]), T2(stack[2]), T3(stack[3]), T4(stack[4])) +} + +func ExportFuncVIIIII[T0, T1, T2, T3, T4 i32](mod wazero.HostModuleBuilder, name string, fn func(context.Context, api.Module, T0, T1, T2, T3, T4)) { + mod.NewFunctionBuilder(). + WithGoModuleFunction(funcVIIIII[T0, T1, T2, T3, T4](fn), + []api.ValueType{api.ValueTypeI32, api.ValueTypeI32, api.ValueTypeI32, api.ValueTypeI32, api.ValueTypeI32}, nil). + Export(name) +} + +type funcVIIIIJ[T0, T1, T2, T3 i32, T4 i64] func(context.Context, api.Module, T0, T1, T2, T3, T4) + +func (fn funcVIIIIJ[T0, T1, T2, T3, T4]) Call(ctx context.Context, mod api.Module, stack []uint64) { + fn(ctx, mod, T0(stack[0]), T1(stack[1]), T2(stack[2]), T3(stack[3]), T4(stack[4])) +} + +func ExportFuncVIIIIJ[T0, T1, T2, T3 i32, T4 i64](mod wazero.HostModuleBuilder, name string, fn func(context.Context, api.Module, T0, T1, T2, T3, T4)) { + mod.NewFunctionBuilder(). + WithGoModuleFunction(funcVIIIIJ[T0, T1, T2, T3, T4](fn), + []api.ValueType{api.ValueTypeI32, api.ValueTypeI32, api.ValueTypeI32, api.ValueTypeI32, api.ValueTypeI64}, nil). + Export(name) +} + +type funcII[TR, T0 i32] func(context.Context, api.Module, T0) TR + +func (fn funcII[TR, T0]) Call(ctx context.Context, mod api.Module, stack []uint64) { + stack[0] = uint64(fn(ctx, mod, T0(stack[0]))) +} + +func ExportFuncII[TR, T0 i32](mod wazero.HostModuleBuilder, name string, fn func(context.Context, api.Module, T0) TR) { + mod.NewFunctionBuilder(). + WithGoModuleFunction(funcII[TR, T0](fn), + []api.ValueType{api.ValueTypeI32}, []api.ValueType{api.ValueTypeI32}). + Export(name) +} + +type funcIII[TR, T0, T1 i32] func(context.Context, api.Module, T0, T1) TR + +func (fn funcIII[TR, T0, T1]) Call(ctx context.Context, mod api.Module, stack []uint64) { + stack[0] = uint64(fn(ctx, mod, T0(stack[0]), T1(stack[1]))) +} + +func ExportFuncIII[TR, T0, T1 i32](mod wazero.HostModuleBuilder, name string, fn func(context.Context, api.Module, T0, T1) TR) { + mod.NewFunctionBuilder(). + WithGoModuleFunction(funcIII[TR, T0, T1](fn), + []api.ValueType{api.ValueTypeI32, api.ValueTypeI32}, []api.ValueType{api.ValueTypeI32}). + Export(name) +} + +type funcIIII[TR, T0, T1, T2 i32] func(context.Context, api.Module, T0, T1, T2) TR + +func (fn funcIIII[TR, T0, T1, T2]) Call(ctx context.Context, mod api.Module, stack []uint64) { + stack[0] = uint64(fn(ctx, mod, T0(stack[0]), T1(stack[1]), T2(stack[2]))) +} + +func ExportFuncIIII[TR, T0, T1, T2 i32](mod wazero.HostModuleBuilder, name string, fn func(context.Context, api.Module, T0, T1, T2) TR) { + mod.NewFunctionBuilder(). + WithGoModuleFunction(funcIIII[TR, T0, T1, T2](fn), + []api.ValueType{api.ValueTypeI32, api.ValueTypeI32, api.ValueTypeI32}, []api.ValueType{api.ValueTypeI32}). + Export(name) +} + +type funcIIIII[TR, T0, T1, T2, T3 i32] func(context.Context, api.Module, T0, T1, T2, T3) TR + +func (fn funcIIIII[TR, T0, T1, T2, T3]) Call(ctx context.Context, mod api.Module, stack []uint64) { + stack[0] = uint64(fn(ctx, mod, T0(stack[0]), T1(stack[1]), T2(stack[2]), T3(stack[3]))) +} + +func ExportFuncIIIII[TR, T0, T1, T2, T3 i32](mod wazero.HostModuleBuilder, name string, fn func(context.Context, api.Module, T0, T1, T2, T3) TR) { + mod.NewFunctionBuilder(). + WithGoModuleFunction(funcIIIII[TR, T0, T1, T2, T3](fn), + []api.ValueType{api.ValueTypeI32, api.ValueTypeI32, api.ValueTypeI32, api.ValueTypeI32}, []api.ValueType{api.ValueTypeI32}). + Export(name) +} + +type funcIIIIII[TR, T0, T1, T2, T3, T4 i32] func(context.Context, api.Module, T0, T1, T2, T3, T4) TR + +func (fn funcIIIIII[TR, T0, T1, T2, T3, T4]) Call(ctx context.Context, mod api.Module, stack []uint64) { + stack[0] = uint64(fn(ctx, mod, T0(stack[0]), T1(stack[1]), T2(stack[2]), T3(stack[3]), T4(stack[4]))) +} + +func ExportFuncIIIIII[TR, T0, T1, T2, T3, T4 i32](mod wazero.HostModuleBuilder, name string, fn func(context.Context, api.Module, T0, T1, T2, T3, T4) TR) { + mod.NewFunctionBuilder(). + WithGoModuleFunction(funcIIIIII[TR, T0, T1, T2, T3, T4](fn), + []api.ValueType{api.ValueTypeI32, api.ValueTypeI32, api.ValueTypeI32, api.ValueTypeI32, api.ValueTypeI32}, []api.ValueType{api.ValueTypeI32}). + Export(name) +} + +type funcIIIIIII[TR, T0, T1, T2, T3, T4, T5 i32] func(context.Context, api.Module, T0, T1, T2, T3, T4, T5) TR + +func (fn funcIIIIIII[TR, T0, T1, T2, T3, T4, T5]) Call(ctx context.Context, mod api.Module, stack []uint64) { + stack[0] = uint64(fn(ctx, mod, T0(stack[0]), T1(stack[1]), T2(stack[2]), T3(stack[3]), T4(stack[4]), T5(stack[5]))) +} + +func ExportFuncIIIIIII[TR, T0, T1, T2, T3, T4, T5 i32](mod wazero.HostModuleBuilder, name string, fn func(context.Context, api.Module, T0, T1, T2, T3, T4, T5) TR) { + mod.NewFunctionBuilder(). + WithGoModuleFunction(funcIIIIIII[TR, T0, T1, T2, T3, T4, T5](fn), + []api.ValueType{api.ValueTypeI32, api.ValueTypeI32, api.ValueTypeI32, api.ValueTypeI32, api.ValueTypeI32, api.ValueTypeI32}, []api.ValueType{api.ValueTypeI32}). + Export(name) +} + +type funcIIIIJ[TR, T0, T1, T2 i32, T3 i64] func(context.Context, api.Module, T0, T1, T2, T3) TR + +func (fn funcIIIIJ[TR, T0, T1, T2, T3]) Call(ctx context.Context, mod api.Module, stack []uint64) { + stack[0] = uint64(fn(ctx, mod, T0(stack[0]), T1(stack[1]), T2(stack[2]), T3(stack[3]))) +} + +func ExportFuncIIIIJ[TR, T0, T1, T2 i32, T3 i64](mod wazero.HostModuleBuilder, name string, fn func(context.Context, api.Module, T0, T1, T2, T3) TR) { + mod.NewFunctionBuilder(). + WithGoModuleFunction(funcIIIIJ[TR, T0, T1, T2, T3](fn), + []api.ValueType{api.ValueTypeI32, api.ValueTypeI32, api.ValueTypeI32, api.ValueTypeI64}, []api.ValueType{api.ValueTypeI32}). + Export(name) +} + +type funcIIJ[TR, T0 i32, T1 i64] func(context.Context, api.Module, T0, T1) TR + +func (fn funcIIJ[TR, T0, T1]) Call(ctx context.Context, mod api.Module, stack []uint64) { + stack[0] = uint64(fn(ctx, mod, T0(stack[0]), T1(stack[1]))) +} + +func ExportFuncIIJ[TR, T0 i32, T1 i64](mod wazero.HostModuleBuilder, name string, fn func(context.Context, api.Module, T0, T1) TR) { + mod.NewFunctionBuilder(). + WithGoModuleFunction(funcIIJ[TR, T0, T1](fn), + []api.ValueType{api.ValueTypeI32, api.ValueTypeI64}, []api.ValueType{api.ValueTypeI32}). + Export(name) +} diff --git a/vendor/github.com/ncruces/go-sqlite3/internal/util/handle.go b/vendor/github.com/ncruces/go-sqlite3/internal/util/handle.go new file mode 100644 index 000000000..4584324c1 --- /dev/null +++ b/vendor/github.com/ncruces/go-sqlite3/internal/util/handle.go @@ -0,0 +1,65 @@ +package util + +import ( + "context" + "io" +) + +type handleState struct { + handles []any + holes int +} + +func (s *handleState) CloseNotify(ctx context.Context, exitCode uint32) { + for _, h := range s.handles { + if c, ok := h.(io.Closer); ok { + c.Close() + } + } + s.handles = nil + s.holes = 0 +} + +func GetHandle(ctx context.Context, id uint32) any { + if id == 0 { + return nil + } + s := ctx.Value(moduleKey{}).(*moduleState) + return s.handles[^id] +} + +func DelHandle(ctx context.Context, id uint32) error { + if id == 0 { + return nil + } + s := ctx.Value(moduleKey{}).(*moduleState) + a := s.handles[^id] + s.handles[^id] = nil + s.holes++ + if c, ok := a.(io.Closer); ok { + return c.Close() + } + return nil +} + +func AddHandle(ctx context.Context, a any) (id uint32) { + if a == nil { + panic(NilErr) + } + s := ctx.Value(moduleKey{}).(*moduleState) + + // Find an empty slot. + if s.holes > cap(s.handles)-len(s.handles) { + for id, h := range s.handles { + if h == nil { + s.holes-- + s.handles[id] = a + return ^uint32(id) + } + } + } + + // Add a new slot. + s.handles = append(s.handles, a) + return -uint32(len(s.handles)) +} diff --git a/vendor/github.com/ncruces/go-sqlite3/internal/util/json.go b/vendor/github.com/ncruces/go-sqlite3/internal/util/json.go new file mode 100644 index 000000000..c0ba38cf0 --- /dev/null +++ b/vendor/github.com/ncruces/go-sqlite3/internal/util/json.go @@ -0,0 +1,35 @@ +package util + +import ( + "encoding/json" + "strconv" + "time" + "unsafe" +) + +type JSON struct{ Value any } + +func (j JSON) Scan(value any) error { + var buf []byte + + switch v := value.(type) { + case []byte: + buf = v + case string: + buf = unsafe.Slice(unsafe.StringData(v), len(v)) + case int64: + buf = strconv.AppendInt(nil, v, 10) + case float64: + buf = strconv.AppendFloat(nil, v, 'g', -1, 64) + case time.Time: + buf = append(buf, '"') + buf = v.AppendFormat(buf, time.RFC3339Nano) + buf = append(buf, '"') + case nil: + buf = append(buf, "null"...) + default: + panic(AssertErr()) + } + + return json.Unmarshal(buf, j.Value) +} diff --git a/vendor/github.com/ncruces/go-sqlite3/internal/util/mem.go b/vendor/github.com/ncruces/go-sqlite3/internal/util/mem.go new file mode 100644 index 000000000..a09523fd1 --- /dev/null +++ b/vendor/github.com/ncruces/go-sqlite3/internal/util/mem.go @@ -0,0 +1,134 @@ +package util + +import ( + "bytes" + "math" + + "github.com/tetratelabs/wazero/api" +) + +func View(mod api.Module, ptr uint32, size uint64) []byte { + if ptr == 0 { + panic(NilErr) + } + if size > math.MaxUint32 { + panic(RangeErr) + } + if size == 0 { + return nil + } + buf, ok := mod.Memory().Read(ptr, uint32(size)) + if !ok { + panic(RangeErr) + } + return buf +} + +func ReadUint8(mod api.Module, ptr uint32) uint8 { + if ptr == 0 { + panic(NilErr) + } + v, ok := mod.Memory().ReadByte(ptr) + if !ok { + panic(RangeErr) + } + return v +} + +func ReadUint32(mod api.Module, ptr uint32) uint32 { + if ptr == 0 { + panic(NilErr) + } + v, ok := mod.Memory().ReadUint32Le(ptr) + if !ok { + panic(RangeErr) + } + return v +} + +func WriteUint8(mod api.Module, ptr uint32, v uint8) { + if ptr == 0 { + panic(NilErr) + } + ok := mod.Memory().WriteByte(ptr, v) + if !ok { + panic(RangeErr) + } +} + +func WriteUint32(mod api.Module, ptr uint32, v uint32) { + if ptr == 0 { + panic(NilErr) + } + ok := mod.Memory().WriteUint32Le(ptr, v) + if !ok { + panic(RangeErr) + } +} + +func ReadUint64(mod api.Module, ptr uint32) uint64 { + if ptr == 0 { + panic(NilErr) + } + v, ok := mod.Memory().ReadUint64Le(ptr) + if !ok { + panic(RangeErr) + } + return v +} + +func WriteUint64(mod api.Module, ptr uint32, v uint64) { + if ptr == 0 { + panic(NilErr) + } + ok := mod.Memory().WriteUint64Le(ptr, v) + if !ok { + panic(RangeErr) + } +} + +func ReadFloat64(mod api.Module, ptr uint32) float64 { + return math.Float64frombits(ReadUint64(mod, ptr)) +} + +func WriteFloat64(mod api.Module, ptr uint32, v float64) { + WriteUint64(mod, ptr, math.Float64bits(v)) +} + +func ReadString(mod api.Module, ptr, maxlen uint32) string { + if ptr == 0 { + panic(NilErr) + } + switch maxlen { + case 0: + return "" + case math.MaxUint32: + // avoid overflow + default: + maxlen = maxlen + 1 + } + mem := mod.Memory() + buf, ok := mem.Read(ptr, maxlen) + if !ok { + buf, ok = mem.Read(ptr, mem.Size()-ptr) + if !ok { + panic(RangeErr) + } + } + if i := bytes.IndexByte(buf, 0); i < 0 { + panic(NoNulErr) + } else { + return string(buf[:i]) + } +} + +func WriteBytes(mod api.Module, ptr uint32, b []byte) { + buf := View(mod, ptr, uint64(len(b))) + copy(buf, b) +} + +func WriteString(mod api.Module, ptr uint32, s string) { + buf := View(mod, ptr, uint64(len(s)+1)) + buf[len(s)] = 0 + copy(buf, s) +} diff --git a/vendor/github.com/ncruces/go-sqlite3/internal/util/mmap.go b/vendor/github.com/ncruces/go-sqlite3/internal/util/mmap.go new file mode 100644 index 000000000..6783c9612 --- /dev/null +++ b/vendor/github.com/ncruces/go-sqlite3/internal/util/mmap.go @@ -0,0 +1,97 @@ +//go:build (darwin || linux) && (amd64 || arm64 || riscv64) && !(sqlite3_noshm || sqlite3_nosys) + +package util + +import ( + "context" + "os" + "unsafe" + + "github.com/tetratelabs/wazero/api" + "github.com/tetratelabs/wazero/experimental" + "golang.org/x/sys/unix" +) + +func withAllocator(ctx context.Context) context.Context { + return experimental.WithMemoryAllocator(ctx, + experimental.MemoryAllocatorFunc(virtualAlloc)) +} + +type mmapState struct { + regions []*MappedRegion +} + +func (s *mmapState) new(ctx context.Context, mod api.Module, size int32) *MappedRegion { + // Find unused region. + for _, r := range s.regions { + if !r.used && r.size == size { + return r + } + } + + // Allocate page aligned memmory. + alloc := mod.ExportedFunction("aligned_alloc") + stack := [2]uint64{ + uint64(unix.Getpagesize()), + uint64(size), + } + if err := alloc.CallWithStack(ctx, stack[:]); err != nil { + panic(err) + } + if stack[0] == 0 { + panic(OOMErr) + } + + // Save the newly allocated region. + ptr := uint32(stack[0]) + buf := View(mod, ptr, uint64(size)) + addr := uintptr(unsafe.Pointer(&buf[0])) + s.regions = append(s.regions, &MappedRegion{ + Ptr: ptr, + addr: addr, + size: size, + }) + return s.regions[len(s.regions)-1] +} + +type MappedRegion struct { + addr uintptr + Ptr uint32 + size int32 + used bool +} + +func MapRegion(ctx context.Context, mod api.Module, f *os.File, offset int64, size int32, prot int) (*MappedRegion, error) { + s := ctx.Value(moduleKey{}).(*moduleState) + r := s.new(ctx, mod, size) + err := r.mmap(f, offset, prot) + if err != nil { + return nil, err + } + return r, nil +} + +func (r *MappedRegion) Unmap() error { + // We can't munmap the region, otherwise it could be remaped. + // Instead, convert it to a protected, private, anonymous mapping. + // If successful, it can be reused for a subsequent mmap. + _, err := mmap(r.addr, uintptr(r.size), + unix.PROT_NONE, unix.MAP_PRIVATE|unix.MAP_ANON|unix.MAP_FIXED, + -1, 0) + r.used = err != nil + return err +} + +func (r *MappedRegion) mmap(f *os.File, offset int64, prot int) error { + _, err := mmap(r.addr, uintptr(r.size), + prot, unix.MAP_SHARED|unix.MAP_FIXED, + int(f.Fd()), offset) + r.used = err == nil + return err +} + +// We need the low level mmap for MAP_FIXED to work. +// Bind the syscall version hoping that it is more stable. + +//go:linkname mmap syscall.mmap +func mmap(addr, length uintptr, prot, flag, fd int, pos int64) (*byte, error) diff --git a/vendor/github.com/ncruces/go-sqlite3/internal/util/mmap_other.go b/vendor/github.com/ncruces/go-sqlite3/internal/util/mmap_other.go new file mode 100644 index 000000000..1e81c9fd3 --- /dev/null +++ b/vendor/github.com/ncruces/go-sqlite3/internal/util/mmap_other.go @@ -0,0 +1,21 @@ +//go:build !(darwin || linux) || !(amd64 || arm64 || riscv64) || sqlite3_noshm || sqlite3_nosys + +package util + +import ( + "context" + + "github.com/tetratelabs/wazero/experimental" +) + +type mmapState struct{} + +func withAllocator(ctx context.Context) context.Context { + return experimental.WithMemoryAllocator(ctx, + experimental.MemoryAllocatorFunc(func(cap, max uint64) experimental.LinearMemory { + if cap == max { + return virtualAlloc(cap, max) + } + return sliceAlloc(cap, max) + })) +} diff --git a/vendor/github.com/ncruces/go-sqlite3/internal/util/module.go b/vendor/github.com/ncruces/go-sqlite3/internal/util/module.go new file mode 100644 index 000000000..22793e972 --- /dev/null +++ b/vendor/github.com/ncruces/go-sqlite3/internal/util/module.go @@ -0,0 +1,21 @@ +package util + +import ( + "context" + + "github.com/tetratelabs/wazero/experimental" +) + +type moduleKey struct{} +type moduleState struct { + mmapState + handleState +} + +func NewContext(ctx context.Context) context.Context { + state := new(moduleState) + ctx = withAllocator(ctx) + ctx = experimental.WithCloseNotifier(ctx, state) + ctx = context.WithValue(ctx, moduleKey{}, state) + return ctx +} diff --git a/vendor/github.com/ncruces/go-sqlite3/internal/util/pointer.go b/vendor/github.com/ncruces/go-sqlite3/internal/util/pointer.go new file mode 100644 index 000000000..eae4dae17 --- /dev/null +++ b/vendor/github.com/ncruces/go-sqlite3/internal/util/pointer.go @@ -0,0 +1,11 @@ +package util + +type Pointer[T any] struct{ Value T } + +func (p Pointer[T]) unwrap() any { return p.Value } + +type PointerUnwrap interface{ unwrap() any } + +func UnwrapPointer(p PointerUnwrap) any { + return p.unwrap() +} diff --git a/vendor/github.com/ncruces/go-sqlite3/internal/util/reflect.go b/vendor/github.com/ncruces/go-sqlite3/internal/util/reflect.go new file mode 100644 index 000000000..3104a7cf3 --- /dev/null +++ b/vendor/github.com/ncruces/go-sqlite3/internal/util/reflect.go @@ -0,0 +1,10 @@ +package util + +import "reflect" + +func ReflectType(v reflect.Value) reflect.Type { + if v.Kind() != reflect.Invalid { + return v.Type() + } + return nil +} diff --git a/vendor/github.com/ncruces/go-sqlite3/json.go b/vendor/github.com/ncruces/go-sqlite3/json.go new file mode 100644 index 000000000..9b2565e87 --- /dev/null +++ b/vendor/github.com/ncruces/go-sqlite3/json.go @@ -0,0 +1,11 @@ +package sqlite3 + +import "github.com/ncruces/go-sqlite3/internal/util" + +// JSON returns a value that can be used as an argument to +// [database/sql.DB.Exec], [database/sql.Row.Scan] and similar methods to +// store value as JSON, or decode JSON into value. +// JSON should NOT be used with [BindJSON] or [ResultJSON]. +func JSON(value any) any { + return util.JSON{Value: value} +} diff --git a/vendor/github.com/ncruces/go-sqlite3/pointer.go b/vendor/github.com/ncruces/go-sqlite3/pointer.go new file mode 100644 index 000000000..611c1528c --- /dev/null +++ b/vendor/github.com/ncruces/go-sqlite3/pointer.go @@ -0,0 +1,12 @@ +package sqlite3 + +import "github.com/ncruces/go-sqlite3/internal/util" + +// Pointer returns a pointer to a value that can be used as an argument to +// [database/sql.DB.Exec] and similar methods. +// Pointer should NOT be used with [BindPointer] or [ResultPointer]. +// +// https://sqlite.org/bindptr.html +func Pointer[T any](value T) any { + return util.Pointer[T]{Value: value} +} diff --git a/vendor/github.com/ncruces/go-sqlite3/quote.go b/vendor/github.com/ncruces/go-sqlite3/quote.go new file mode 100644 index 000000000..d1cd6fa87 --- /dev/null +++ b/vendor/github.com/ncruces/go-sqlite3/quote.go @@ -0,0 +1,112 @@ +package sqlite3 + +import ( + "bytes" + "math" + "strconv" + "strings" + "time" + "unsafe" + + "github.com/ncruces/go-sqlite3/internal/util" +) + +// Quote escapes and quotes a value +// making it safe to embed in SQL text. +func Quote(value any) string { + switch v := value.(type) { + case nil: + return "NULL" + case bool: + if v { + return "1" + } else { + return "0" + } + + case int: + return strconv.Itoa(v) + case int64: + return strconv.FormatInt(v, 10) + case float64: + switch { + case math.IsNaN(v): + return "NULL" + case math.IsInf(v, 1): + return "9.0e999" + case math.IsInf(v, -1): + return "-9.0e999" + } + return strconv.FormatFloat(v, 'g', -1, 64) + case time.Time: + return "'" + v.Format(time.RFC3339Nano) + "'" + + case string: + if strings.IndexByte(v, 0) >= 0 { + break + } + + buf := make([]byte, 2+len(v)+strings.Count(v, "'")) + buf[0] = '\'' + i := 1 + for _, b := range []byte(v) { + if b == '\'' { + buf[i] = b + i += 1 + } + buf[i] = b + i += 1 + } + buf[i] = '\'' + return unsafe.String(&buf[0], len(buf)) + + case []byte: + buf := make([]byte, 3+2*len(v)) + buf[0] = 'x' + buf[1] = '\'' + i := 2 + for _, b := range v { + const hex = "0123456789ABCDEF" + buf[i+0] = hex[b/16] + buf[i+1] = hex[b%16] + i += 2 + } + buf[i] = '\'' + return unsafe.String(&buf[0], len(buf)) + + case ZeroBlob: + if v > ZeroBlob(1e9-3)/2 { + break + } + + buf := bytes.Repeat([]byte("0"), int(3+2*int64(v))) + buf[0] = 'x' + buf[1] = '\'' + buf[len(buf)-1] = '\'' + return unsafe.String(&buf[0], len(buf)) + } + + panic(util.ValueErr) +} + +// QuoteIdentifier escapes and quotes an identifier +// making it safe to embed in SQL text. +func QuoteIdentifier(id string) string { + if strings.IndexByte(id, 0) >= 0 { + panic(util.ValueErr) + } + + buf := make([]byte, 2+len(id)+strings.Count(id, `"`)) + buf[0] = '"' + i := 1 + for _, b := range []byte(id) { + if b == '"' { + buf[i] = b + i += 1 + } + buf[i] = b + i += 1 + } + buf[i] = '"' + return unsafe.String(&buf[0], len(buf)) +} diff --git a/vendor/github.com/ncruces/go-sqlite3/sqlite.go b/vendor/github.com/ncruces/go-sqlite3/sqlite.go new file mode 100644 index 000000000..61a03652f --- /dev/null +++ b/vendor/github.com/ncruces/go-sqlite3/sqlite.go @@ -0,0 +1,341 @@ +// Package sqlite3 wraps the C SQLite API. +package sqlite3 + +import ( + "context" + "math" + "math/bits" + "os" + "sync" + "unsafe" + + "github.com/ncruces/go-sqlite3/internal/util" + "github.com/ncruces/go-sqlite3/vfs" + "github.com/tetratelabs/wazero" + "github.com/tetratelabs/wazero/api" +) + +// Configure SQLite Wasm. +// +// Importing package embed initializes [Binary] +// with an appropriate build of SQLite: +// +// import _ "github.com/ncruces/go-sqlite3/embed" +var ( + Binary []byte // Wasm binary to load. + Path string // Path to load the binary from. + + RuntimeConfig wazero.RuntimeConfig +) + +// Initialize decodes and compiles the SQLite Wasm binary. +// This is called implicitly when the first connection is openned, +// but is potentially slow, so you may want to call it at a more convenient time. +func Initialize() error { + instance.once.Do(compileSQLite) + return instance.err +} + +var instance struct { + runtime wazero.Runtime + compiled wazero.CompiledModule + err error + once sync.Once +} + +func compileSQLite() { + if RuntimeConfig == nil { + RuntimeConfig = wazero.NewRuntimeConfig() + } + + ctx := context.Background() + instance.runtime = wazero.NewRuntimeWithConfig(ctx, RuntimeConfig) + + env := instance.runtime.NewHostModuleBuilder("env") + env = vfs.ExportHostFunctions(env) + env = exportCallbacks(env) + _, instance.err = env.Instantiate(ctx) + if instance.err != nil { + return + } + + bin := Binary + if bin == nil && Path != "" { + bin, instance.err = os.ReadFile(Path) + if instance.err != nil { + return + } + } + if bin == nil { + instance.err = util.NoBinaryErr + return + } + + instance.compiled, instance.err = instance.runtime.CompileModule(ctx, bin) +} + +type sqlite struct { + ctx context.Context + mod api.Module + funcs struct { + fn [32]api.Function + id [32]*byte + mask uint32 + } + stack [8]uint64 + freer uint32 +} + +func instantiateSQLite() (sqlt *sqlite, err error) { + if err := Initialize(); err != nil { + return nil, err + } + + sqlt = new(sqlite) + sqlt.ctx = util.NewContext(context.Background()) + + sqlt.mod, err = instance.runtime.InstantiateModule(sqlt.ctx, + instance.compiled, wazero.NewModuleConfig().WithName("")) + if err != nil { + return nil, err + } + + global := sqlt.mod.ExportedGlobal("malloc_destructor") + if global == nil { + return nil, util.BadBinaryErr + } + + sqlt.freer = util.ReadUint32(sqlt.mod, uint32(global.Get())) + if sqlt.freer == 0 { + return nil, util.BadBinaryErr + } + return sqlt, nil +} + +func (sqlt *sqlite) close() error { + return sqlt.mod.Close(sqlt.ctx) +} + +func (sqlt *sqlite) error(rc uint64, handle uint32, sql ...string) error { + if rc == _OK { + return nil + } + + err := Error{code: rc} + + if err.Code() == NOMEM || err.ExtendedCode() == IOERR_NOMEM { + panic(util.OOMErr) + } + + if r := sqlt.call("sqlite3_errstr", rc); r != 0 { + err.str = util.ReadString(sqlt.mod, uint32(r), _MAX_NAME) + } + + if handle != 0 { + if r := sqlt.call("sqlite3_errmsg", uint64(handle)); r != 0 { + err.msg = util.ReadString(sqlt.mod, uint32(r), _MAX_LENGTH) + } + + if sql != nil { + if r := sqlt.call("sqlite3_error_offset", uint64(handle)); r != math.MaxUint32 { + err.sql = sql[0][r:] + } + } + } + + switch err.msg { + case err.str, "not an error": + err.msg = "" + } + return &err +} + +func (sqlt *sqlite) getfn(name string) api.Function { + c := &sqlt.funcs + p := unsafe.StringData(name) + for i := range c.id { + if c.id[i] == p { + c.id[i] = nil + c.mask &^= uint32(1) << i + return c.fn[i] + } + } + return sqlt.mod.ExportedFunction(name) +} + +func (sqlt *sqlite) putfn(name string, fn api.Function) { + c := &sqlt.funcs + p := unsafe.StringData(name) + i := bits.TrailingZeros32(^c.mask) + if i < 32 { + c.id[i] = p + c.fn[i] = fn + c.mask |= uint32(1) << i + } else { + c.id[0] = p + c.fn[0] = fn + c.mask = uint32(1) + } +} + +func (sqlt *sqlite) call(name string, params ...uint64) uint64 { + copy(sqlt.stack[:], params) + fn := sqlt.getfn(name) + err := fn.CallWithStack(sqlt.ctx, sqlt.stack[:]) + if err != nil { + panic(err) + } + sqlt.putfn(name, fn) + return sqlt.stack[0] +} + +func (sqlt *sqlite) free(ptr uint32) { + if ptr == 0 { + return + } + sqlt.call("free", uint64(ptr)) +} + +func (sqlt *sqlite) new(size uint64) uint32 { + if size > _MAX_ALLOCATION_SIZE { + panic(util.OOMErr) + } + ptr := uint32(sqlt.call("malloc", size)) + if ptr == 0 && size != 0 { + panic(util.OOMErr) + } + return ptr +} + +func (sqlt *sqlite) newBytes(b []byte) uint32 { + if (*[0]byte)(b) == nil { + return 0 + } + ptr := sqlt.new(uint64(len(b))) + util.WriteBytes(sqlt.mod, ptr, b) + return ptr +} + +func (sqlt *sqlite) newString(s string) uint32 { + ptr := sqlt.new(uint64(len(s) + 1)) + util.WriteString(sqlt.mod, ptr, s) + return ptr +} + +func (sqlt *sqlite) newArena(size uint64) arena { + // Ensure the arena's size is a multiple of 8. + size = (size + 7) &^ 7 + return arena{ + sqlt: sqlt, + size: uint32(size), + base: sqlt.new(size), + } +} + +type arena struct { + sqlt *sqlite + ptrs []uint32 + base uint32 + next uint32 + size uint32 +} + +func (a *arena) free() { + if a.sqlt == nil { + return + } + for _, ptr := range a.ptrs { + a.sqlt.free(ptr) + } + a.sqlt.free(a.base) + a.sqlt = nil +} + +func (a *arena) mark() (reset func()) { + ptrs := len(a.ptrs) + next := a.next + return func() { + for _, ptr := range a.ptrs[ptrs:] { + a.sqlt.free(ptr) + } + a.ptrs = a.ptrs[:ptrs] + a.next = next + } +} + +func (a *arena) new(size uint64) uint32 { + // Align the next address, to 4 or 8 bytes. + if size&7 != 0 { + a.next = (a.next + 3) &^ 3 + } else { + a.next = (a.next + 7) &^ 7 + } + if size <= uint64(a.size-a.next) { + ptr := a.base + a.next + a.next += uint32(size) + return ptr + } + ptr := a.sqlt.new(size) + a.ptrs = append(a.ptrs, ptr) + return ptr +} + +func (a *arena) bytes(b []byte) uint32 { + if (*[0]byte)(b) == nil { + return 0 + } + ptr := a.new(uint64(len(b))) + util.WriteBytes(a.sqlt.mod, ptr, b) + return ptr +} + +func (a *arena) string(s string) uint32 { + ptr := a.new(uint64(len(s) + 1)) + util.WriteString(a.sqlt.mod, ptr, s) + return ptr +} + +func exportCallbacks(env wazero.HostModuleBuilder) wazero.HostModuleBuilder { + util.ExportFuncII(env, "go_progress_handler", progressCallback) + util.ExportFuncIIII(env, "go_busy_timeout", timeoutCallback) + util.ExportFuncIII(env, "go_busy_handler", busyCallback) + util.ExportFuncII(env, "go_commit_hook", commitCallback) + util.ExportFuncVI(env, "go_rollback_hook", rollbackCallback) + util.ExportFuncVIIIIJ(env, "go_update_hook", updateCallback) + util.ExportFuncIIIII(env, "go_wal_hook", walCallback) + util.ExportFuncIIIIII(env, "go_autovacuum_pages", autoVacuumCallback) + util.ExportFuncIIIIIII(env, "go_authorizer", authorizerCallback) + util.ExportFuncVIII(env, "go_log", logCallback) + util.ExportFuncVI(env, "go_destroy", destroyCallback) + util.ExportFuncVIIII(env, "go_func", funcCallback) + util.ExportFuncVIIIII(env, "go_step", stepCallback) + util.ExportFuncVIII(env, "go_final", finalCallback) + util.ExportFuncVII(env, "go_value", valueCallback) + util.ExportFuncVIIII(env, "go_inverse", inverseCallback) + util.ExportFuncVIIII(env, "go_collation_needed", collationCallback) + util.ExportFuncIIIIII(env, "go_compare", compareCallback) + util.ExportFuncIIIIII(env, "go_vtab_create", vtabModuleCallback(xCreate)) + util.ExportFuncIIIIII(env, "go_vtab_connect", vtabModuleCallback(xConnect)) + util.ExportFuncII(env, "go_vtab_disconnect", vtabDisconnectCallback) + util.ExportFuncII(env, "go_vtab_destroy", vtabDestroyCallback) + util.ExportFuncIII(env, "go_vtab_best_index", vtabBestIndexCallback) + util.ExportFuncIIIII(env, "go_vtab_update", vtabUpdateCallback) + util.ExportFuncIII(env, "go_vtab_rename", vtabRenameCallback) + util.ExportFuncIIIII(env, "go_vtab_find_function", vtabFindFuncCallback) + util.ExportFuncII(env, "go_vtab_begin", vtabBeginCallback) + util.ExportFuncII(env, "go_vtab_sync", vtabSyncCallback) + util.ExportFuncII(env, "go_vtab_commit", vtabCommitCallback) + util.ExportFuncII(env, "go_vtab_rollback", vtabRollbackCallback) + util.ExportFuncIII(env, "go_vtab_savepoint", vtabSavepointCallback) + util.ExportFuncIII(env, "go_vtab_release", vtabReleaseCallback) + util.ExportFuncIII(env, "go_vtab_rollback_to", vtabRollbackToCallback) + util.ExportFuncIIIIII(env, "go_vtab_integrity", vtabIntegrityCallback) + util.ExportFuncIII(env, "go_cur_open", cursorOpenCallback) + util.ExportFuncII(env, "go_cur_close", cursorCloseCallback) + util.ExportFuncIIIIII(env, "go_cur_filter", cursorFilterCallback) + util.ExportFuncII(env, "go_cur_next", cursorNextCallback) + util.ExportFuncII(env, "go_cur_eof", cursorEOFCallback) + util.ExportFuncIIII(env, "go_cur_column", cursorColumnCallback) + util.ExportFuncIII(env, "go_cur_rowid", cursorRowIDCallback) + return env +} diff --git a/vendor/github.com/ncruces/go-sqlite3/stmt.go b/vendor/github.com/ncruces/go-sqlite3/stmt.go new file mode 100644 index 000000000..63c2085d0 --- /dev/null +++ b/vendor/github.com/ncruces/go-sqlite3/stmt.go @@ -0,0 +1,639 @@ +package sqlite3 + +import ( + "encoding/json" + "math" + "strconv" + "time" + + "github.com/ncruces/go-sqlite3/internal/util" +) + +// Stmt is a prepared statement object. +// +// https://sqlite.org/c3ref/stmt.html +type Stmt struct { + c *Conn + err error + handle uint32 +} + +// Close destroys the prepared statement object. +// +// It is safe to close a nil, zero or closed Stmt. +// +// https://sqlite.org/c3ref/finalize.html +func (s *Stmt) Close() error { + if s == nil || s.handle == 0 { + return nil + } + + r := s.c.call("sqlite3_finalize", uint64(s.handle)) + + s.handle = 0 + return s.c.error(r) +} + +// Conn returns the database connection to which the prepared statement belongs. +// +// https://sqlite.org/c3ref/db_handle.html +func (s *Stmt) Conn() *Conn { + return s.c +} + +// ReadOnly returns true if and only if the statement +// makes no direct changes to the content of the database file. +// +// https://sqlite.org/c3ref/stmt_readonly.html +func (s *Stmt) ReadOnly() bool { + r := s.c.call("sqlite3_stmt_readonly", uint64(s.handle)) + return r != 0 +} + +// Reset resets the prepared statement object. +// +// https://sqlite.org/c3ref/reset.html +func (s *Stmt) Reset() error { + r := s.c.call("sqlite3_reset", uint64(s.handle)) + s.err = nil + return s.c.error(r) +} + +// Busy determines if a prepared statement has been reset. +// +// https://sqlite.org/c3ref/stmt_busy.html +func (s *Stmt) Busy() bool { + r := s.c.call("sqlite3_stmt_busy", uint64(s.handle)) + return r != 0 +} + +// Step evaluates the SQL statement. +// If the SQL statement being executed returns any data, +// then true is returned each time a new row of data is ready for processing by the caller. +// The values may be accessed using the Column access functions. +// Step is called again to retrieve the next row of data. +// If an error has occurred, Step returns false; +// call [Stmt.Err] or [Stmt.Reset] to get the error. +// +// https://sqlite.org/c3ref/step.html +func (s *Stmt) Step() bool { + s.c.checkInterrupt() + r := s.c.call("sqlite3_step", uint64(s.handle)) + switch r { + case _ROW: + s.err = nil + return true + case _DONE: + s.err = nil + default: + s.err = s.c.error(r) + } + return false +} + +// Err gets the last error occurred during [Stmt.Step]. +// Err returns nil after [Stmt.Reset] is called. +// +// https://sqlite.org/c3ref/step.html +func (s *Stmt) Err() error { + return s.err +} + +// Exec is a convenience function that repeatedly calls [Stmt.Step] until it returns false, +// then calls [Stmt.Reset] to reset the statement and get any error that occurred. +func (s *Stmt) Exec() error { + for s.Step() { + } + return s.Reset() +} + +// Status monitors the performance characteristics of prepared statements. +// +// https://sqlite.org/c3ref/stmt_status.html +func (s *Stmt) Status(op StmtStatus, reset bool) int { + if op > STMTSTATUS_FILTER_HIT && op != STMTSTATUS_MEMUSED { + return 0 + } + var i uint64 + if reset { + i = 1 + } + r := s.c.call("sqlite3_stmt_status", uint64(s.handle), + uint64(op), i) + return int(int32(r)) +} + +// ClearBindings resets all bindings on the prepared statement. +// +// https://sqlite.org/c3ref/clear_bindings.html +func (s *Stmt) ClearBindings() error { + r := s.c.call("sqlite3_clear_bindings", uint64(s.handle)) + return s.c.error(r) +} + +// BindCount returns the number of SQL parameters in the prepared statement. +// +// https://sqlite.org/c3ref/bind_parameter_count.html +func (s *Stmt) BindCount() int { + r := s.c.call("sqlite3_bind_parameter_count", + uint64(s.handle)) + return int(int32(r)) +} + +// BindIndex returns the index of a parameter in the prepared statement +// given its name. +// +// https://sqlite.org/c3ref/bind_parameter_index.html +func (s *Stmt) BindIndex(name string) int { + defer s.c.arena.mark()() + namePtr := s.c.arena.string(name) + r := s.c.call("sqlite3_bind_parameter_index", + uint64(s.handle), uint64(namePtr)) + return int(int32(r)) +} + +// BindName returns the name of a parameter in the prepared statement. +// The leftmost SQL parameter has an index of 1. +// +// https://sqlite.org/c3ref/bind_parameter_name.html +func (s *Stmt) BindName(param int) string { + r := s.c.call("sqlite3_bind_parameter_name", + uint64(s.handle), uint64(param)) + + ptr := uint32(r) + if ptr == 0 { + return "" + } + return util.ReadString(s.c.mod, ptr, _MAX_NAME) +} + +// BindBool binds a bool to the prepared statement. +// The leftmost SQL parameter has an index of 1. +// SQLite does not have a separate boolean storage class. +// Instead, boolean values are stored as integers 0 (false) and 1 (true). +// +// https://sqlite.org/c3ref/bind_blob.html +func (s *Stmt) BindBool(param int, value bool) error { + var i int64 + if value { + i = 1 + } + return s.BindInt64(param, i) +} + +// BindInt binds an int to the prepared statement. +// The leftmost SQL parameter has an index of 1. +// +// https://sqlite.org/c3ref/bind_blob.html +func (s *Stmt) BindInt(param int, value int) error { + return s.BindInt64(param, int64(value)) +} + +// BindInt64 binds an int64 to the prepared statement. +// The leftmost SQL parameter has an index of 1. +// +// https://sqlite.org/c3ref/bind_blob.html +func (s *Stmt) BindInt64(param int, value int64) error { + r := s.c.call("sqlite3_bind_int64", + uint64(s.handle), uint64(param), uint64(value)) + return s.c.error(r) +} + +// BindFloat binds a float64 to the prepared statement. +// The leftmost SQL parameter has an index of 1. +// +// https://sqlite.org/c3ref/bind_blob.html +func (s *Stmt) BindFloat(param int, value float64) error { + r := s.c.call("sqlite3_bind_double", + uint64(s.handle), uint64(param), math.Float64bits(value)) + return s.c.error(r) +} + +// BindText binds a string to the prepared statement. +// The leftmost SQL parameter has an index of 1. +// +// https://sqlite.org/c3ref/bind_blob.html +func (s *Stmt) BindText(param int, value string) error { + if len(value) > _MAX_LENGTH { + return TOOBIG + } + ptr := s.c.newString(value) + r := s.c.call("sqlite3_bind_text64", + uint64(s.handle), uint64(param), + uint64(ptr), uint64(len(value)), + uint64(s.c.freer), _UTF8) + return s.c.error(r) +} + +// BindRawText binds a []byte to the prepared statement as text. +// The leftmost SQL parameter has an index of 1. +// +// https://sqlite.org/c3ref/bind_blob.html +func (s *Stmt) BindRawText(param int, value []byte) error { + if len(value) > _MAX_LENGTH { + return TOOBIG + } + ptr := s.c.newBytes(value) + r := s.c.call("sqlite3_bind_text64", + uint64(s.handle), uint64(param), + uint64(ptr), uint64(len(value)), + uint64(s.c.freer), _UTF8) + return s.c.error(r) +} + +// BindBlob binds a []byte to the prepared statement. +// The leftmost SQL parameter has an index of 1. +// Binding a nil slice is the same as calling [Stmt.BindNull]. +// +// https://sqlite.org/c3ref/bind_blob.html +func (s *Stmt) BindBlob(param int, value []byte) error { + if len(value) > _MAX_LENGTH { + return TOOBIG + } + ptr := s.c.newBytes(value) + r := s.c.call("sqlite3_bind_blob64", + uint64(s.handle), uint64(param), + uint64(ptr), uint64(len(value)), + uint64(s.c.freer)) + return s.c.error(r) +} + +// BindZeroBlob binds a zero-filled, length n BLOB to the prepared statement. +// The leftmost SQL parameter has an index of 1. +// +// https://sqlite.org/c3ref/bind_blob.html +func (s *Stmt) BindZeroBlob(param int, n int64) error { + r := s.c.call("sqlite3_bind_zeroblob64", + uint64(s.handle), uint64(param), uint64(n)) + return s.c.error(r) +} + +// BindNull binds a NULL to the prepared statement. +// The leftmost SQL parameter has an index of 1. +// +// https://sqlite.org/c3ref/bind_blob.html +func (s *Stmt) BindNull(param int) error { + r := s.c.call("sqlite3_bind_null", + uint64(s.handle), uint64(param)) + return s.c.error(r) +} + +// BindTime binds a [time.Time] to the prepared statement. +// The leftmost SQL parameter has an index of 1. +// +// https://sqlite.org/c3ref/bind_blob.html +func (s *Stmt) BindTime(param int, value time.Time, format TimeFormat) error { + if format == TimeFormatDefault { + return s.bindRFC3339Nano(param, value) + } + switch v := format.Encode(value).(type) { + case string: + s.BindText(param, v) + case int64: + s.BindInt64(param, v) + case float64: + s.BindFloat(param, v) + default: + panic(util.AssertErr()) + } + return nil +} + +func (s *Stmt) bindRFC3339Nano(param int, value time.Time) error { + const maxlen = uint64(len(time.RFC3339Nano)) + 5 + + ptr := s.c.new(maxlen) + buf := util.View(s.c.mod, ptr, maxlen) + buf = value.AppendFormat(buf[:0], time.RFC3339Nano) + + r := s.c.call("sqlite3_bind_text64", + uint64(s.handle), uint64(param), + uint64(ptr), uint64(len(buf)), + uint64(s.c.freer), _UTF8) + return s.c.error(r) +} + +// BindPointer binds a NULL to the prepared statement, just like [Stmt.BindNull], +// but it also associates ptr with that NULL value such that it can be retrieved +// within an application-defined SQL function using [Value.Pointer]. +// The leftmost SQL parameter has an index of 1. +// +// https://sqlite.org/c3ref/bind_blob.html +func (s *Stmt) BindPointer(param int, ptr any) error { + valPtr := util.AddHandle(s.c.ctx, ptr) + r := s.c.call("sqlite3_bind_pointer_go", + uint64(s.handle), uint64(param), uint64(valPtr)) + return s.c.error(r) +} + +// BindJSON binds the JSON encoding of value to the prepared statement. +// The leftmost SQL parameter has an index of 1. +// +// https://sqlite.org/c3ref/bind_blob.html +func (s *Stmt) BindJSON(param int, value any) error { + data, err := json.Marshal(value) + if err != nil { + return err + } + return s.BindRawText(param, data) +} + +// BindValue binds a copy of value to the prepared statement. +// The leftmost SQL parameter has an index of 1. +// +// https://sqlite.org/c3ref/bind_blob.html +func (s *Stmt) BindValue(param int, value Value) error { + if value.c != s.c { + return MISUSE + } + r := s.c.call("sqlite3_bind_value", + uint64(s.handle), uint64(param), uint64(value.handle)) + return s.c.error(r) +} + +// ColumnCount returns the number of columns in a result set. +// +// https://sqlite.org/c3ref/column_count.html +func (s *Stmt) ColumnCount() int { + r := s.c.call("sqlite3_column_count", + uint64(s.handle)) + return int(int32(r)) +} + +// ColumnName returns the name of the result column. +// The leftmost column of the result set has the index 0. +// +// https://sqlite.org/c3ref/column_name.html +func (s *Stmt) ColumnName(col int) string { + r := s.c.call("sqlite3_column_name", + uint64(s.handle), uint64(col)) + if r == 0 { + panic(util.OOMErr) + } + return util.ReadString(s.c.mod, uint32(r), _MAX_NAME) +} + +// ColumnType returns the initial [Datatype] of the result column. +// The leftmost column of the result set has the index 0. +// +// https://sqlite.org/c3ref/column_blob.html +func (s *Stmt) ColumnType(col int) Datatype { + r := s.c.call("sqlite3_column_type", + uint64(s.handle), uint64(col)) + return Datatype(r) +} + +// ColumnDeclType returns the declared datatype of the result column. +// The leftmost column of the result set has the index 0. +// +// https://sqlite.org/c3ref/column_decltype.html +func (s *Stmt) ColumnDeclType(col int) string { + r := s.c.call("sqlite3_column_decltype", + uint64(s.handle), uint64(col)) + if r == 0 { + return "" + } + return util.ReadString(s.c.mod, uint32(r), _MAX_NAME) +} + +// ColumnDatabaseName returns the name of the database +// that is the origin of a particular result column. +// The leftmost column of the result set has the index 0. +// +// https://sqlite.org/c3ref/column_database_name.html +func (s *Stmt) ColumnDatabaseName(col int) string { + r := s.c.call("sqlite3_column_database_name", + uint64(s.handle), uint64(col)) + if r == 0 { + return "" + } + return util.ReadString(s.c.mod, uint32(r), _MAX_NAME) +} + +// ColumnTableName returns the name of the table +// that is the origin of a particular result column. +// The leftmost column of the result set has the index 0. +// +// https://sqlite.org/c3ref/column_database_name.html +func (s *Stmt) ColumnTableName(col int) string { + r := s.c.call("sqlite3_column_table_name", + uint64(s.handle), uint64(col)) + if r == 0 { + return "" + } + return util.ReadString(s.c.mod, uint32(r), _MAX_NAME) +} + +// ColumnOriginName returns the name of the table column +// that is the origin of a particular result column. +// The leftmost column of the result set has the index 0. +// +// https://sqlite.org/c3ref/column_database_name.html +func (s *Stmt) ColumnOriginName(col int) string { + r := s.c.call("sqlite3_column_origin_name", + uint64(s.handle), uint64(col)) + if r == 0 { + return "" + } + return util.ReadString(s.c.mod, uint32(r), _MAX_NAME) +} + +// ColumnBool returns the value of the result column as a bool. +// The leftmost column of the result set has the index 0. +// SQLite does not have a separate boolean storage class. +// Instead, boolean values are retrieved as integers, +// with 0 converted to false and any other value to true. +// +// https://sqlite.org/c3ref/column_blob.html +func (s *Stmt) ColumnBool(col int) bool { + return s.ColumnInt64(col) != 0 +} + +// ColumnInt returns the value of the result column as an int. +// The leftmost column of the result set has the index 0. +// +// https://sqlite.org/c3ref/column_blob.html +func (s *Stmt) ColumnInt(col int) int { + return int(s.ColumnInt64(col)) +} + +// ColumnInt64 returns the value of the result column as an int64. +// The leftmost column of the result set has the index 0. +// +// https://sqlite.org/c3ref/column_blob.html +func (s *Stmt) ColumnInt64(col int) int64 { + r := s.c.call("sqlite3_column_int64", + uint64(s.handle), uint64(col)) + return int64(r) +} + +// ColumnFloat returns the value of the result column as a float64. +// The leftmost column of the result set has the index 0. +// +// https://sqlite.org/c3ref/column_blob.html +func (s *Stmt) ColumnFloat(col int) float64 { + r := s.c.call("sqlite3_column_double", + uint64(s.handle), uint64(col)) + return math.Float64frombits(r) +} + +// ColumnTime returns the value of the result column as a [time.Time]. +// The leftmost column of the result set has the index 0. +// +// https://sqlite.org/c3ref/column_blob.html +func (s *Stmt) ColumnTime(col int, format TimeFormat) time.Time { + var v any + switch s.ColumnType(col) { + case INTEGER: + v = s.ColumnInt64(col) + case FLOAT: + v = s.ColumnFloat(col) + case TEXT, BLOB: + v = s.ColumnText(col) + case NULL: + return time.Time{} + default: + panic(util.AssertErr()) + } + t, err := format.Decode(v) + if err != nil { + s.err = err + } + return t +} + +// ColumnText returns the value of the result column as a string. +// The leftmost column of the result set has the index 0. +// +// https://sqlite.org/c3ref/column_blob.html +func (s *Stmt) ColumnText(col int) string { + return string(s.ColumnRawText(col)) +} + +// ColumnBlob appends to buf and returns +// the value of the result column as a []byte. +// The leftmost column of the result set has the index 0. +// +// https://sqlite.org/c3ref/column_blob.html +func (s *Stmt) ColumnBlob(col int, buf []byte) []byte { + return append(buf, s.ColumnRawBlob(col)...) +} + +// ColumnRawText returns the value of the result column as a []byte. +// The []byte is owned by SQLite and may be invalidated by +// subsequent calls to [Stmt] methods. +// The leftmost column of the result set has the index 0. +// +// https://sqlite.org/c3ref/column_blob.html +func (s *Stmt) ColumnRawText(col int) []byte { + r := s.c.call("sqlite3_column_text", + uint64(s.handle), uint64(col)) + return s.columnRawBytes(col, uint32(r)) +} + +// ColumnRawBlob returns the value of the result column as a []byte. +// The []byte is owned by SQLite and may be invalidated by +// subsequent calls to [Stmt] methods. +// The leftmost column of the result set has the index 0. +// +// https://sqlite.org/c3ref/column_blob.html +func (s *Stmt) ColumnRawBlob(col int) []byte { + r := s.c.call("sqlite3_column_blob", + uint64(s.handle), uint64(col)) + return s.columnRawBytes(col, uint32(r)) +} + +func (s *Stmt) columnRawBytes(col int, ptr uint32) []byte { + if ptr == 0 { + r := s.c.call("sqlite3_errcode", uint64(s.c.handle)) + s.err = s.c.error(r) + return nil + } + + r := s.c.call("sqlite3_column_bytes", + uint64(s.handle), uint64(col)) + return util.View(s.c.mod, ptr, r) +} + +// ColumnJSON parses the JSON-encoded value of the result column +// and stores it in the value pointed to by ptr. +// The leftmost column of the result set has the index 0. +// +// https://sqlite.org/c3ref/column_blob.html +func (s *Stmt) ColumnJSON(col int, ptr any) error { + var data []byte + switch s.ColumnType(col) { + case NULL: + data = append(data, "null"...) + case TEXT: + data = s.ColumnRawText(col) + case BLOB: + data = s.ColumnRawBlob(col) + case INTEGER: + data = strconv.AppendInt(nil, s.ColumnInt64(col), 10) + case FLOAT: + data = strconv.AppendFloat(nil, s.ColumnFloat(col), 'g', -1, 64) + default: + panic(util.AssertErr()) + } + return json.Unmarshal(data, ptr) +} + +// ColumnValue returns the unprotected value of the result column. +// The leftmost column of the result set has the index 0. +// +// https://sqlite.org/c3ref/column_blob.html +func (s *Stmt) ColumnValue(col int) Value { + r := s.c.call("sqlite3_column_value", + uint64(s.handle), uint64(col)) + return Value{ + c: s.c, + unprot: true, + handle: uint32(r), + } +} + +// Columns populates result columns into the provided slice. +// The slice must have [Stmt.ColumnCount] length. +// +// [INTEGER] columns will be retrieved as int64 values, +// [FLOAT] as float64, [NULL] as nil, +// [TEXT] as string, and [BLOB] as []byte. +// Any []byte are owned by SQLite and may be invalidated by +// subsequent calls to [Stmt] methods. +func (s *Stmt) Columns(dest []any) error { + defer s.c.arena.mark()() + count := uint64(len(dest)) + typePtr := s.c.arena.new(count) + dataPtr := s.c.arena.new(8 * count) + + r := s.c.call("sqlite3_columns_go", + uint64(s.handle), count, uint64(typePtr), uint64(dataPtr)) + if err := s.c.error(r); err != nil { + return err + } + + types := util.View(s.c.mod, typePtr, count) + for i := range dest { + switch types[i] { + case byte(INTEGER): + dest[i] = int64(util.ReadUint64(s.c.mod, dataPtr+8*uint32(i))) + continue + case byte(FLOAT): + dest[i] = util.ReadFloat64(s.c.mod, dataPtr+8*uint32(i)) + continue + case byte(NULL): + dest[i] = nil + continue + } + ptr := util.ReadUint32(s.c.mod, dataPtr+8*uint32(i)+0) + len := util.ReadUint32(s.c.mod, dataPtr+8*uint32(i)+4) + buf := util.View(s.c.mod, ptr, uint64(len)) + if types[i] == byte(TEXT) { + dest[i] = string(buf) + } else { + dest[i] = buf + } + } + return nil +} diff --git a/vendor/github.com/ncruces/go-sqlite3/time.go b/vendor/github.com/ncruces/go-sqlite3/time.go new file mode 100644 index 000000000..0164a307b --- /dev/null +++ b/vendor/github.com/ncruces/go-sqlite3/time.go @@ -0,0 +1,354 @@ +package sqlite3 + +import ( + "math" + "strconv" + "strings" + "time" + + "github.com/ncruces/go-sqlite3/internal/util" + "github.com/ncruces/julianday" +) + +// TimeFormat specifies how to encode/decode time values. +// +// See the documentation for the [TimeFormatDefault] constant +// for formats recognized by SQLite. +// +// https://sqlite.org/lang_datefunc.html +type TimeFormat string + +// TimeFormats recognized by SQLite to encode/decode time values. +// +// https://sqlite.org/lang_datefunc.html#time_values +const ( + TimeFormatDefault TimeFormat = "" // time.RFC3339Nano + + // Text formats + TimeFormat1 TimeFormat = "2006-01-02" + TimeFormat2 TimeFormat = "2006-01-02 15:04" + TimeFormat3 TimeFormat = "2006-01-02 15:04:05" + TimeFormat4 TimeFormat = "2006-01-02 15:04:05.000" + TimeFormat5 TimeFormat = "2006-01-02T15:04" + TimeFormat6 TimeFormat = "2006-01-02T15:04:05" + TimeFormat7 TimeFormat = "2006-01-02T15:04:05.000" + TimeFormat8 TimeFormat = "15:04" + TimeFormat9 TimeFormat = "15:04:05" + TimeFormat10 TimeFormat = "15:04:05.000" + + TimeFormat2TZ = TimeFormat2 + "Z07:00" + TimeFormat3TZ = TimeFormat3 + "Z07:00" + TimeFormat4TZ = TimeFormat4 + "Z07:00" + TimeFormat5TZ = TimeFormat5 + "Z07:00" + TimeFormat6TZ = TimeFormat6 + "Z07:00" + TimeFormat7TZ = TimeFormat7 + "Z07:00" + TimeFormat8TZ = TimeFormat8 + "Z07:00" + TimeFormat9TZ = TimeFormat9 + "Z07:00" + TimeFormat10TZ = TimeFormat10 + "Z07:00" + + // Numeric formats + TimeFormatJulianDay TimeFormat = "julianday" + TimeFormatUnix TimeFormat = "unixepoch" + TimeFormatUnixFrac TimeFormat = "unixepoch_frac" + TimeFormatUnixMilli TimeFormat = "unixepoch_milli" // not an SQLite format + TimeFormatUnixMicro TimeFormat = "unixepoch_micro" // not an SQLite format + TimeFormatUnixNano TimeFormat = "unixepoch_nano" // not an SQLite format + + // Auto + TimeFormatAuto TimeFormat = "auto" +) + +// Encode encodes a time value using this format. +// +// [TimeFormatDefault] and [TimeFormatAuto] encode using [time.RFC3339Nano], +// with nanosecond accuracy, and preserving any timezone offset. +// +// This is the format used by the [database/sql] driver: +// [database/sql.Row.Scan] will decode as [time.Time] +// values encoded with [time.RFC3339Nano]. +// +// Time values encoded with [time.RFC3339Nano] cannot be sorted as strings +// to produce a time-ordered sequence. +// +// Assuming that the time zones of the time values are the same (e.g., all in UTC), +// and expressed using the same string (e.g., all "Z" or all "+00:00"), +// use the TIME [collating sequence] to produce a time-ordered sequence. +// +// Otherwise, use [TimeFormat7] for time-ordered encoding. +// +// Formats [TimeFormat1] through [TimeFormat10] +// convert time values to UTC before encoding. +// +// Returns a string for the text formats, +// a float64 for [TimeFormatJulianDay] and [TimeFormatUnixFrac], +// or an int64 for the other numeric formats. +// +// https://sqlite.org/lang_datefunc.html +// +// [collating sequence]: https://sqlite.org/datatype3.html#collating_sequences +func (f TimeFormat) Encode(t time.Time) any { + switch f { + // Numeric formats + case TimeFormatJulianDay: + return julianday.Float(t) + case TimeFormatUnix: + return t.Unix() + case TimeFormatUnixFrac: + return float64(t.Unix()) + float64(t.Nanosecond())*1e-9 + case TimeFormatUnixMilli: + return t.UnixMilli() + case TimeFormatUnixMicro: + return t.UnixMicro() + case TimeFormatUnixNano: + return t.UnixNano() + // Special formats. + case TimeFormatDefault, TimeFormatAuto: + f = time.RFC3339Nano + // SQLite assumes UTC if unspecified. + case + TimeFormat1, TimeFormat2, + TimeFormat3, TimeFormat4, + TimeFormat5, TimeFormat6, + TimeFormat7, TimeFormat8, + TimeFormat9, TimeFormat10: + t = t.UTC() + } + return t.Format(string(f)) +} + +// Decode decodes a time value using this format. +// +// The time value can be a string, an int64, or a float64. +// +// Formats [TimeFormat8] through [TimeFormat10] +// (and [TimeFormat8TZ] through [TimeFormat10TZ]) +// assume a date of 2000-01-01. +// +// The timezone indicator and fractional seconds are always optional +// for formats [TimeFormat2] through [TimeFormat10] +// (and [TimeFormat2TZ] through [TimeFormat10TZ]). +// +// [TimeFormatAuto] implements (and extends) the SQLite auto modifier. +// Julian day numbers are safe to use for historical dates, +// from 4712BC through 9999AD. +// Unix timestamps (expressed in seconds, milliseconds, microseconds, or nanoseconds) +// are safe to use for current events, from at least 1980 through at least 2260. +// Unix timestamps before 1980 and after 9999 may be misinterpreted as julian day numbers, +// or have the wrong time unit. +// +// https://sqlite.org/lang_datefunc.html +func (f TimeFormat) Decode(v any) (time.Time, error) { + switch f { + // Numeric formats. + case TimeFormatJulianDay: + switch v := v.(type) { + case string: + return julianday.Parse(v) + case float64: + return julianday.FloatTime(v), nil + case int64: + return julianday.Time(v, 0), nil + default: + return time.Time{}, util.TimeErr + } + + case TimeFormatUnix, TimeFormatUnixFrac: + if s, ok := v.(string); ok { + f, err := strconv.ParseFloat(s, 64) + if err != nil { + return time.Time{}, err + } + v = f + } + switch v := v.(type) { + case float64: + sec, frac := math.Modf(v) + nsec := math.Floor(frac * 1e9) + return time.Unix(int64(sec), int64(nsec)).UTC(), nil + case int64: + return time.Unix(v, 0).UTC(), nil + default: + return time.Time{}, util.TimeErr + } + + case TimeFormatUnixMilli: + if s, ok := v.(string); ok { + i, err := strconv.ParseInt(s, 10, 64) + if err != nil { + return time.Time{}, err + } + v = i + } + switch v := v.(type) { + case float64: + return time.UnixMilli(int64(math.Floor(v))).UTC(), nil + case int64: + return time.UnixMilli(v).UTC(), nil + default: + return time.Time{}, util.TimeErr + } + + case TimeFormatUnixMicro: + if s, ok := v.(string); ok { + i, err := strconv.ParseInt(s, 10, 64) + if err != nil { + return time.Time{}, err + } + v = i + } + switch v := v.(type) { + case float64: + return time.UnixMicro(int64(math.Floor(v))).UTC(), nil + case int64: + return time.UnixMicro(v).UTC(), nil + default: + return time.Time{}, util.TimeErr + } + + case TimeFormatUnixNano: + if s, ok := v.(string); ok { + i, err := strconv.ParseInt(s, 10, 64) + if err != nil { + return time.Time{}, util.TimeErr + } + v = i + } + switch v := v.(type) { + case float64: + return time.Unix(0, int64(math.Floor(v))).UTC(), nil + case int64: + return time.Unix(0, v).UTC(), nil + default: + return time.Time{}, util.TimeErr + } + + // Special formats. + case TimeFormatAuto: + switch s := v.(type) { + case string: + i, err := strconv.ParseInt(s, 10, 64) + if err == nil { + v = i + break + } + f, err := strconv.ParseFloat(s, 64) + if err == nil { + v = f + break + } + + dates := []TimeFormat{ + TimeFormat9, TimeFormat8, + TimeFormat6, TimeFormat5, + TimeFormat3, TimeFormat2, TimeFormat1, + } + for _, f := range dates { + t, err := f.Decode(s) + if err == nil { + return t, nil + } + } + } + switch v := v.(type) { + case float64: + if 0 <= v && v < 5373484.5 { + return TimeFormatJulianDay.Decode(v) + } + if v < 253402300800 { + return TimeFormatUnixFrac.Decode(v) + } + if v < 253402300800_000 { + return TimeFormatUnixMilli.Decode(v) + } + if v < 253402300800_000000 { + return TimeFormatUnixMicro.Decode(v) + } + return TimeFormatUnixNano.Decode(v) + case int64: + if 0 <= v && v < 5373485 { + return TimeFormatJulianDay.Decode(v) + } + if v < 253402300800 { + return TimeFormatUnixFrac.Decode(v) + } + if v < 253402300800_000 { + return TimeFormatUnixMilli.Decode(v) + } + if v < 253402300800_000000 { + return TimeFormatUnixMicro.Decode(v) + } + return TimeFormatUnixNano.Decode(v) + default: + return time.Time{}, util.TimeErr + } + + case + TimeFormat2, TimeFormat2TZ, + TimeFormat3, TimeFormat3TZ, + TimeFormat4, TimeFormat4TZ, + TimeFormat5, TimeFormat5TZ, + TimeFormat6, TimeFormat6TZ, + TimeFormat7, TimeFormat7TZ: + s, ok := v.(string) + if !ok { + return time.Time{}, util.TimeErr + } + return f.parseRelaxed(s) + + case + TimeFormat8, TimeFormat8TZ, + TimeFormat9, TimeFormat9TZ, + TimeFormat10, TimeFormat10TZ: + s, ok := v.(string) + if !ok { + return time.Time{}, util.TimeErr + } + t, err := f.parseRelaxed(s) + if err != nil { + return time.Time{}, err + } + return t.AddDate(2000, 0, 0), nil + + default: + s, ok := v.(string) + if !ok { + return time.Time{}, util.TimeErr + } + if f == "" { + f = time.RFC3339Nano + } + return time.Parse(string(f), s) + } +} + +func (f TimeFormat) parseRelaxed(s string) (time.Time, error) { + fs := string(f) + fs = strings.TrimSuffix(fs, "Z07:00") + fs = strings.TrimSuffix(fs, ".000") + t, err := time.Parse(fs+"Z07:00", s) + if err != nil { + return time.Parse(fs, s) + } + return t, nil +} + +// Scanner returns a [database/sql.Scanner] that can be used as an argument to +// [database/sql.Row.Scan] and similar methods to +// decode a time value into dest using this format. +func (f TimeFormat) Scanner(dest *time.Time) interface{ Scan(any) error } { + return timeScanner{dest, f} +} + +type timeScanner struct { + *time.Time + TimeFormat +} + +func (s timeScanner) Scan(src any) error { + var ok bool + var err error + if *s.Time, ok = src.(time.Time); !ok { + *s.Time, err = s.Decode(src) + } + return err +} diff --git a/vendor/github.com/ncruces/go-sqlite3/txn.go b/vendor/github.com/ncruces/go-sqlite3/txn.go new file mode 100644 index 000000000..0efbc2d80 --- /dev/null +++ b/vendor/github.com/ncruces/go-sqlite3/txn.go @@ -0,0 +1,294 @@ +package sqlite3 + +import ( + "context" + "errors" + "fmt" + "math/rand" + "runtime" + "strconv" + "strings" + + "github.com/ncruces/go-sqlite3/internal/util" + "github.com/tetratelabs/wazero/api" +) + +// Txn is an in-progress database transaction. +// +// https://sqlite.org/lang_transaction.html +type Txn struct { + c *Conn +} + +// Begin starts a deferred transaction. +// +// https://sqlite.org/lang_transaction.html +func (c *Conn) Begin() Txn { + // BEGIN even if interrupted. + err := c.txnExecInterrupted(`BEGIN DEFERRED`) + if err != nil { + panic(err) + } + return Txn{c} +} + +// BeginImmediate starts an immediate transaction. +// +// https://sqlite.org/lang_transaction.html +func (c *Conn) BeginImmediate() (Txn, error) { + err := c.Exec(`BEGIN IMMEDIATE`) + if err != nil { + return Txn{}, err + } + return Txn{c}, nil +} + +// BeginExclusive starts an exclusive transaction. +// +// https://sqlite.org/lang_transaction.html +func (c *Conn) BeginExclusive() (Txn, error) { + err := c.Exec(`BEGIN EXCLUSIVE`) + if err != nil { + return Txn{}, err + } + return Txn{c}, nil +} + +// End calls either [Txn.Commit] or [Txn.Rollback] +// depending on whether *error points to a nil or non-nil error. +// +// This is meant to be deferred: +// +// func doWork(db *sqlite3.Conn) (err error) { +// tx := db.Begin() +// defer tx.End(&err) +// +// // ... do work in the transaction +// } +// +// https://sqlite.org/lang_transaction.html +func (tx Txn) End(errp *error) { + recovered := recover() + if recovered != nil { + defer panic(recovered) + } + + if *errp == nil && recovered == nil { + // Success path. + if tx.c.GetAutocommit() { // There is nothing to commit. + return + } + *errp = tx.Commit() + if *errp == nil { + return + } + // Fall through to the error path. + } + + // Error path. + if tx.c.GetAutocommit() { // There is nothing to rollback. + return + } + err := tx.Rollback() + if err != nil { + panic(err) + } +} + +// Commit commits the transaction. +// +// https://sqlite.org/lang_transaction.html +func (tx Txn) Commit() error { + return tx.c.Exec(`COMMIT`) +} + +// Rollback rolls back the transaction, +// even if the connection has been interrupted. +// +// https://sqlite.org/lang_transaction.html +func (tx Txn) Rollback() error { + return tx.c.txnExecInterrupted(`ROLLBACK`) +} + +// Savepoint is a marker within a transaction +// that allows for partial rollback. +// +// https://sqlite.org/lang_savepoint.html +type Savepoint struct { + c *Conn + name string +} + +// Savepoint establishes a new transaction savepoint. +// +// https://sqlite.org/lang_savepoint.html +func (c *Conn) Savepoint() Savepoint { + // Names can be reused; this makes catching bugs more likely. + name := saveptName() + "_" + strconv.Itoa(int(rand.Int31())) + + err := c.txnExecInterrupted(fmt.Sprintf("SAVEPOINT %q;", name)) + if err != nil { + panic(err) + } + return Savepoint{c: c, name: name} +} + +func saveptName() (name string) { + defer func() { + if name == "" { + name = "sqlite3.Savepoint" + } + }() + + var pc [8]uintptr + n := runtime.Callers(3, pc[:]) + if n <= 0 { + return "" + } + frames := runtime.CallersFrames(pc[:n]) + frame, more := frames.Next() + for more && (strings.HasPrefix(frame.Function, "database/sql.") || + strings.HasPrefix(frame.Function, "github.com/ncruces/go-sqlite3/driver.")) { + frame, more = frames.Next() + } + return frame.Function +} + +// Release releases the savepoint rolling back any changes +// if *error points to a non-nil error. +// +// This is meant to be deferred: +// +// func doWork(db *sqlite3.Conn) (err error) { +// savept := db.Savepoint() +// defer savept.Release(&err) +// +// // ... do work in the transaction +// } +func (s Savepoint) Release(errp *error) { + recovered := recover() + if recovered != nil { + defer panic(recovered) + } + + if *errp == nil && recovered == nil { + // Success path. + if s.c.GetAutocommit() { // There is nothing to commit. + return + } + *errp = s.c.Exec(fmt.Sprintf("RELEASE %q;", s.name)) + if *errp == nil { + return + } + // Fall through to the error path. + } + + // Error path. + if s.c.GetAutocommit() { // There is nothing to rollback. + return + } + // ROLLBACK and RELEASE even if interrupted. + err := s.c.txnExecInterrupted(fmt.Sprintf(` + ROLLBACK TO %[1]q; + RELEASE %[1]q; + `, s.name)) + if err != nil { + panic(err) + } +} + +// Rollback rolls the transaction back to the savepoint, +// even if the connection has been interrupted. +// Rollback does not release the savepoint. +// +// https://sqlite.org/lang_transaction.html +func (s Savepoint) Rollback() error { + // ROLLBACK even if interrupted. + return s.c.txnExecInterrupted(fmt.Sprintf("ROLLBACK TO %q;", s.name)) +} + +func (c *Conn) txnExecInterrupted(sql string) error { + err := c.Exec(sql) + if errors.Is(err, INTERRUPT) { + old := c.SetInterrupt(context.Background()) + defer c.SetInterrupt(old) + err = c.Exec(sql) + } + return err +} + +// TxnState starts a deferred transaction. +// +// https://sqlite.org/c3ref/txn_state.html +func (c *Conn) TxnState(schema string) TxnState { + var ptr uint32 + if schema != "" { + defer c.arena.mark()() + ptr = c.arena.string(schema) + } + r := c.call("sqlite3_txn_state", uint64(c.handle), uint64(ptr)) + return TxnState(r) +} + +// CommitHook registers a callback function to be invoked +// whenever a transaction is committed. +// Return true to allow the commit operation to continue normally. +// +// https://sqlite.org/c3ref/commit_hook.html +func (c *Conn) CommitHook(cb func() (ok bool)) { + var enable uint64 + if cb != nil { + enable = 1 + } + c.call("sqlite3_commit_hook_go", uint64(c.handle), enable) + c.commit = cb +} + +// RollbackHook registers a callback function to be invoked +// whenever a transaction is rolled back. +// +// https://sqlite.org/c3ref/commit_hook.html +func (c *Conn) RollbackHook(cb func()) { + var enable uint64 + if cb != nil { + enable = 1 + } + c.call("sqlite3_rollback_hook_go", uint64(c.handle), enable) + c.rollback = cb +} + +// UpdateHook registers a callback function to be invoked +// whenever a row is updated, inserted or deleted in a rowid table. +// +// https://sqlite.org/c3ref/update_hook.html +func (c *Conn) UpdateHook(cb func(action AuthorizerActionCode, schema, table string, rowid int64)) { + var enable uint64 + if cb != nil { + enable = 1 + } + c.call("sqlite3_update_hook_go", uint64(c.handle), enable) + c.update = cb +} + +func commitCallback(ctx context.Context, mod api.Module, pDB uint32) (rollback uint32) { + if c, ok := ctx.Value(connKey{}).(*Conn); ok && c.handle == pDB && c.commit != nil { + if !c.commit() { + rollback = 1 + } + } + return rollback +} + +func rollbackCallback(ctx context.Context, mod api.Module, pDB uint32) { + if c, ok := ctx.Value(connKey{}).(*Conn); ok && c.handle == pDB && c.rollback != nil { + c.rollback() + } +} + +func updateCallback(ctx context.Context, mod api.Module, pDB uint32, action AuthorizerActionCode, zSchema, zTabName uint32, rowid uint64) { + if c, ok := ctx.Value(connKey{}).(*Conn); ok && c.handle == pDB && c.update != nil { + schema := util.ReadString(mod, zSchema, _MAX_NAME) + table := util.ReadString(mod, zTabName, _MAX_NAME) + c.update(action, schema, table, int64(rowid)) + } +} diff --git a/vendor/github.com/ncruces/go-sqlite3/util/osutil/open.go b/vendor/github.com/ncruces/go-sqlite3/util/osutil/open.go new file mode 100644 index 000000000..0242ad032 --- /dev/null +++ b/vendor/github.com/ncruces/go-sqlite3/util/osutil/open.go @@ -0,0 +1,16 @@ +//go:build !windows + +package osutil + +import ( + "io/fs" + "os" +) + +// OpenFile behaves the same as [os.OpenFile], +// except on Windows it sets [syscall.FILE_SHARE_DELETE]. +// +// See: https://go.dev/issue/32088#issuecomment-502850674 +func OpenFile(name string, flag int, perm fs.FileMode) (*os.File, error) { + return os.OpenFile(name, flag, perm) +} diff --git a/vendor/github.com/ncruces/go-sqlite3/util/osutil/open_windows.go b/vendor/github.com/ncruces/go-sqlite3/util/osutil/open_windows.go new file mode 100644 index 000000000..277f58bc3 --- /dev/null +++ b/vendor/github.com/ncruces/go-sqlite3/util/osutil/open_windows.go @@ -0,0 +1,112 @@ +package osutil + +import ( + "io/fs" + "os" + . "syscall" + "unsafe" +) + +// OpenFile behaves the same as [os.OpenFile], +// except on Windows it sets [syscall.FILE_SHARE_DELETE]. +// +// See: https://go.dev/issue/32088#issuecomment-502850674 +func OpenFile(name string, flag int, perm fs.FileMode) (*os.File, error) { + if name == "" { + return nil, &os.PathError{Op: "open", Path: name, Err: ENOENT} + } + r, e := syscallOpen(name, flag, uint32(perm.Perm())) + if e != nil { + return nil, &os.PathError{Op: "open", Path: name, Err: e} + } + return os.NewFile(uintptr(r), name), nil +} + +// syscallOpen is a copy of [syscall.Open] +// that uses [syscall.FILE_SHARE_DELETE]. +// +// https://go.dev/src/syscall/syscall_windows.go +func syscallOpen(path string, mode int, perm uint32) (fd Handle, err error) { + if len(path) == 0 { + return InvalidHandle, ERROR_FILE_NOT_FOUND + } + pathp, err := UTF16PtrFromString(path) + if err != nil { + return InvalidHandle, err + } + var access uint32 + switch mode & (O_RDONLY | O_WRONLY | O_RDWR) { + case O_RDONLY: + access = GENERIC_READ + case O_WRONLY: + access = GENERIC_WRITE + case O_RDWR: + access = GENERIC_READ | GENERIC_WRITE + } + if mode&O_CREAT != 0 { + access |= GENERIC_WRITE + } + if mode&O_APPEND != 0 { + access &^= GENERIC_WRITE + access |= FILE_APPEND_DATA + } + sharemode := uint32(FILE_SHARE_READ | FILE_SHARE_WRITE | FILE_SHARE_DELETE) + var sa *SecurityAttributes + if mode&O_CLOEXEC == 0 { + sa = makeInheritSa() + } + var createmode uint32 + switch { + case mode&(O_CREAT|O_EXCL) == (O_CREAT | O_EXCL): + createmode = CREATE_NEW + case mode&(O_CREAT|O_TRUNC) == (O_CREAT | O_TRUNC): + createmode = CREATE_ALWAYS + case mode&O_CREAT == O_CREAT: + createmode = OPEN_ALWAYS + case mode&O_TRUNC == O_TRUNC: + createmode = TRUNCATE_EXISTING + default: + createmode = OPEN_EXISTING + } + var attrs uint32 = FILE_ATTRIBUTE_NORMAL + if perm&S_IWRITE == 0 { + attrs = FILE_ATTRIBUTE_READONLY + if createmode == CREATE_ALWAYS { + const _ERROR_BAD_NETPATH = Errno(53) + // We have been asked to create a read-only file. + // If the file already exists, the semantics of + // the Unix open system call is to preserve the + // existing permissions. If we pass CREATE_ALWAYS + // and FILE_ATTRIBUTE_READONLY to CreateFile, + // and the file already exists, CreateFile will + // change the file permissions. + // Avoid that to preserve the Unix semantics. + h, e := CreateFile(pathp, access, sharemode, sa, TRUNCATE_EXISTING, FILE_ATTRIBUTE_NORMAL, 0) + switch e { + case ERROR_FILE_NOT_FOUND, _ERROR_BAD_NETPATH, ERROR_PATH_NOT_FOUND: + // File does not exist. These are the same + // errors as Errno.Is checks for ErrNotExist. + // Carry on to create the file. + default: + // Success or some different error. + return h, e + } + } + } + if createmode == OPEN_EXISTING && access == GENERIC_READ { + // Necessary for opening directory handles. + attrs |= FILE_FLAG_BACKUP_SEMANTICS + } + if mode&O_SYNC != 0 { + const _FILE_FLAG_WRITE_THROUGH = 0x80000000 + attrs |= _FILE_FLAG_WRITE_THROUGH + } + return CreateFile(pathp, access, sharemode, sa, createmode, attrs, 0) +} + +func makeInheritSa() *SecurityAttributes { + var sa SecurityAttributes + sa.Length = uint32(unsafe.Sizeof(sa)) + sa.InheritHandle = 1 + return &sa +} diff --git a/vendor/github.com/ncruces/go-sqlite3/util/osutil/osfs.go b/vendor/github.com/ncruces/go-sqlite3/util/osutil/osfs.go new file mode 100644 index 000000000..2e1195934 --- /dev/null +++ b/vendor/github.com/ncruces/go-sqlite3/util/osutil/osfs.go @@ -0,0 +1,33 @@ +package osutil + +import ( + "io/fs" + "os" +) + +// FS implements [fs.FS], [fs.StatFS], and [fs.ReadFileFS] +// using package [os]. +// +// This filesystem does not respect [fs.ValidPath] rules, +// and fails [testing/fstest.TestFS]! +// +// Still, it can be a useful tool to unify implementations +// that can access either the [os] filesystem or an [fs.FS]. +// It's OK to use this to open files, but you should avoid +// opening directories, resolving paths, or walking the file system. +type FS struct{} + +// Open implements [fs.FS]. +func (FS) Open(name string) (fs.File, error) { + return OpenFile(name, os.O_RDONLY, 0) +} + +// ReadFileFS implements [fs.StatFS]. +func (FS) Stat(name string) (fs.FileInfo, error) { + return os.Stat(name) +} + +// ReadFile implements [fs.ReadFileFS]. +func (FS) ReadFile(name string) ([]byte, error) { + return os.ReadFile(name) +} diff --git a/vendor/github.com/ncruces/go-sqlite3/util/osutil/osutil.go b/vendor/github.com/ncruces/go-sqlite3/util/osutil/osutil.go new file mode 100644 index 000000000..7fbd04787 --- /dev/null +++ b/vendor/github.com/ncruces/go-sqlite3/util/osutil/osutil.go @@ -0,0 +1,2 @@ +// Package osutil implements operating system utility functions. +package osutil diff --git a/vendor/github.com/ncruces/go-sqlite3/value.go b/vendor/github.com/ncruces/go-sqlite3/value.go new file mode 100644 index 000000000..61d3cbf70 --- /dev/null +++ b/vendor/github.com/ncruces/go-sqlite3/value.go @@ -0,0 +1,236 @@ +package sqlite3 + +import ( + "encoding/json" + "math" + "strconv" + "time" + + "github.com/ncruces/go-sqlite3/internal/util" +) + +// Value is any value that can be stored in a database table. +// +// https://sqlite.org/c3ref/value.html +type Value struct { + c *Conn + handle uint32 + unprot bool + copied bool +} + +func (v Value) protected() uint64 { + if v.unprot { + panic(util.ValueErr) + } + return uint64(v.handle) +} + +// Dup makes a copy of the SQL value and returns a pointer to that copy. +// +// https://sqlite.org/c3ref/value_dup.html +func (v Value) Dup() *Value { + r := v.c.call("sqlite3_value_dup", uint64(v.handle)) + return &Value{ + c: v.c, + copied: true, + handle: uint32(r), + } +} + +// Close frees an SQL value previously obtained by [Value.Dup]. +// +// https://sqlite.org/c3ref/value_dup.html +func (dup *Value) Close() error { + if !dup.copied { + panic(util.ValueErr) + } + dup.c.call("sqlite3_value_free", uint64(dup.handle)) + dup.handle = 0 + return nil +} + +// Type returns the initial datatype of the value. +// +// https://sqlite.org/c3ref/value_blob.html +func (v Value) Type() Datatype { + r := v.c.call("sqlite3_value_type", v.protected()) + return Datatype(r) +} + +// Type returns the numeric datatype of the value. +// +// https://sqlite.org/c3ref/value_blob.html +func (v Value) NumericType() Datatype { + r := v.c.call("sqlite3_value_numeric_type", v.protected()) + return Datatype(r) +} + +// Bool returns the value as a bool. +// SQLite does not have a separate boolean storage class. +// Instead, boolean values are retrieved as integers, +// with 0 converted to false and any other value to true. +// +// https://sqlite.org/c3ref/value_blob.html +func (v Value) Bool() bool { + return v.Int64() != 0 +} + +// Int returns the value as an int. +// +// https://sqlite.org/c3ref/value_blob.html +func (v Value) Int() int { + return int(v.Int64()) +} + +// Int64 returns the value as an int64. +// +// https://sqlite.org/c3ref/value_blob.html +func (v Value) Int64() int64 { + r := v.c.call("sqlite3_value_int64", v.protected()) + return int64(r) +} + +// Float returns the value as a float64. +// +// https://sqlite.org/c3ref/value_blob.html +func (v Value) Float() float64 { + r := v.c.call("sqlite3_value_double", v.protected()) + return math.Float64frombits(r) +} + +// Time returns the value as a [time.Time]. +// +// https://sqlite.org/c3ref/value_blob.html +func (v Value) Time(format TimeFormat) time.Time { + var a any + switch v.Type() { + case INTEGER: + a = v.Int64() + case FLOAT: + a = v.Float() + case TEXT, BLOB: + a = v.Text() + case NULL: + return time.Time{} + default: + panic(util.AssertErr()) + } + t, _ := format.Decode(a) + return t +} + +// Text returns the value as a string. +// +// https://sqlite.org/c3ref/value_blob.html +func (v Value) Text() string { + return string(v.RawText()) +} + +// Blob appends to buf and returns +// the value as a []byte. +// +// https://sqlite.org/c3ref/value_blob.html +func (v Value) Blob(buf []byte) []byte { + return append(buf, v.RawBlob()...) +} + +// RawText returns the value as a []byte. +// The []byte is owned by SQLite and may be invalidated by +// subsequent calls to [Value] methods. +// +// https://sqlite.org/c3ref/value_blob.html +func (v Value) RawText() []byte { + r := v.c.call("sqlite3_value_text", v.protected()) + return v.rawBytes(uint32(r)) +} + +// RawBlob returns the value as a []byte. +// The []byte is owned by SQLite and may be invalidated by +// subsequent calls to [Value] methods. +// +// https://sqlite.org/c3ref/value_blob.html +func (v Value) RawBlob() []byte { + r := v.c.call("sqlite3_value_blob", v.protected()) + return v.rawBytes(uint32(r)) +} + +func (v Value) rawBytes(ptr uint32) []byte { + if ptr == 0 { + return nil + } + + r := v.c.call("sqlite3_value_bytes", v.protected()) + return util.View(v.c.mod, ptr, r) +} + +// Pointer gets the pointer associated with this value, +// or nil if it has no associated pointer. +func (v Value) Pointer() any { + r := v.c.call("sqlite3_value_pointer_go", v.protected()) + return util.GetHandle(v.c.ctx, uint32(r)) +} + +// JSON parses a JSON-encoded value +// and stores the result in the value pointed to by ptr. +func (v Value) JSON(ptr any) error { + var data []byte + switch v.Type() { + case NULL: + data = append(data, "null"...) + case TEXT: + data = v.RawText() + case BLOB: + data = v.RawBlob() + case INTEGER: + data = strconv.AppendInt(nil, v.Int64(), 10) + case FLOAT: + data = strconv.AppendFloat(nil, v.Float(), 'g', -1, 64) + default: + panic(util.AssertErr()) + } + return json.Unmarshal(data, ptr) +} + +// NoChange returns true if and only if the value is unchanged +// in a virtual table update operatiom. +// +// https://sqlite.org/c3ref/value_blob.html +func (v Value) NoChange() bool { + r := v.c.call("sqlite3_value_nochange", v.protected()) + return r != 0 +} + +// InFirst returns the first element +// on the right-hand side of an IN constraint. +// +// https://sqlite.org/c3ref/vtab_in_first.html +func (v Value) InFirst() (Value, error) { + defer v.c.arena.mark()() + valPtr := v.c.arena.new(ptrlen) + r := v.c.call("sqlite3_vtab_in_first", uint64(v.handle), uint64(valPtr)) + if err := v.c.error(r); err != nil { + return Value{}, err + } + return Value{ + c: v.c, + handle: util.ReadUint32(v.c.mod, valPtr), + }, nil +} + +// InNext returns the next element +// on the right-hand side of an IN constraint. +// +// https://sqlite.org/c3ref/vtab_in_first.html +func (v Value) InNext() (Value, error) { + defer v.c.arena.mark()() + valPtr := v.c.arena.new(ptrlen) + r := v.c.call("sqlite3_vtab_in_next", uint64(v.handle), uint64(valPtr)) + if err := v.c.error(r); err != nil { + return Value{}, err + } + return Value{ + c: v.c, + handle: util.ReadUint32(v.c.mod, valPtr), + }, nil +} diff --git a/vendor/github.com/ncruces/go-sqlite3/vfs/README.md b/vendor/github.com/ncruces/go-sqlite3/vfs/README.md new file mode 100644 index 000000000..88059a41b --- /dev/null +++ b/vendor/github.com/ncruces/go-sqlite3/vfs/README.md @@ -0,0 +1,86 @@ +# Go SQLite VFS API + +This package implements the SQLite [OS Interface](https://sqlite.org/vfs.html) (aka VFS). + +It replaces the default SQLite VFS with a **pure Go** implementation, +and exposes [interfaces](https://pkg.go.dev/github.com/ncruces/go-sqlite3/vfs#VFS) +that should allow you to implement your own custom VFSes. + +Since it is a from scratch reimplementation, +there are naturally some ways it deviates from the original. + +The main differences are [file locking](#file-locking) and [WAL mode](#write-ahead-logging) support. + +### File Locking + +POSIX advisory locks, which SQLite uses on Unix, are +[broken by design](https://github.com/sqlite/sqlite/blob/b74eb0/src/os_unix.c#L1073-L1161). + +On Linux and macOS, this module uses +[OFD locks](https://www.gnu.org/software/libc/manual/html_node/Open-File-Description-Locks.html) +to synchronize access to database files. +OFD locks are fully compatible with POSIX advisory locks. + +This module can also use +[BSD locks](https://man.freebsd.org/cgi/man.cgi?query=flock&sektion=2), +albeit with reduced concurrency (`BEGIN IMMEDIATE` behaves like `BEGIN EXCLUSIVE`). +On BSD, macOS, and illumos, BSD locks are fully compatible with POSIX advisory locks; +on Linux and z/OS, they are fully functional, but incompatible; +elsewhere, they are very likely broken. +BSD locks are the default on BSD and illumos, +but you can opt into them with the `sqlite3_flock` build tag. + +On Windows, this module uses `LockFileEx` and `UnlockFileEx`, +like SQLite. + +Otherwise, file locking is not supported, and you must use +[`nolock=1`](https://sqlite.org/uri.html#urinolock) +(or [`immutable=1`](https://sqlite.org/uri.html#uriimmutable)) +to open database files. +To use the [`database/sql`](https://pkg.go.dev/database/sql) driver +with `nolock=1` you must disable connection pooling by calling +[`db.SetMaxOpenConns(1)`](https://pkg.go.dev/database/sql#DB.SetMaxOpenConns). + +You can use [`vfs.SupportsFileLocking`](https://pkg.go.dev/github.com/ncruces/go-sqlite3/vfs#SupportsFileLocking) +to check if your build supports file locking. + +### Write-Ahead Logging + +On 64-bit Linux and macOS, this module uses `mmap` to implement +[shared-memory for the WAL-index](https://sqlite.org/wal.html#implementation_of_shared_memory_for_the_wal_index), +like SQLite. + +To allow `mmap` to work, each connection needs to reserve up to 4GB of address space. +To limit the address space each connection reserves, +use [`WithMemoryLimitPages`](../tests/testcfg/testcfg.go). + +Otherwise, [WAL support is limited](https://sqlite.org/wal.html#noshm), +and `EXCLUSIVE` locking mode must be set to create, read, and write WAL databases. +To use `EXCLUSIVE` locking mode with the +[`database/sql`](https://pkg.go.dev/database/sql) driver +you must disable connection pooling by calling +[`db.SetMaxOpenConns(1)`](https://pkg.go.dev/database/sql#DB.SetMaxOpenConns). + +You can use [`vfs.SupportsSharedMemory`](https://pkg.go.dev/github.com/ncruces/go-sqlite3/vfs#SupportsSharedMemory) +to check if your build supports shared memory. + +### Batch-Atomic Write + +On 64-bit Linux, this module supports [batch-atomic writes](https://sqlite.org/cgi/src/technote/714) +on the F2FS filesystem. + +### Build Tags + +The VFS can be customized with a few build tags: +- `sqlite3_flock` forces the use of BSD locks; it can be used on z/OS to enable locking, + and elsewhere to test BSD locks. +- `sqlite3_nosys` prevents importing [`x/sys`](https://pkg.go.dev/golang.org/x/sys); + disables locking _and_ shared memory on all platforms. +- `sqlite3_noshm` disables shared memory on all platforms. + +> [!IMPORTANT] +> The default configuration of this package is compatible with +> the standard [Unix and Windows SQLite VFSes](https://sqlite.org/vfs.html#multiple_vfses); +> `sqlite3_flock` is compatible with the [`unix-flock` VFS](https://sqlite.org/compile.html#enable_locking_style). +> If incompatible file locking is used, accessing databases concurrently with _other_ SQLite libraries +> will eventually corrupt data. diff --git a/vendor/github.com/ncruces/go-sqlite3/vfs/api.go b/vendor/github.com/ncruces/go-sqlite3/vfs/api.go new file mode 100644 index 000000000..19c22ae8f --- /dev/null +++ b/vendor/github.com/ncruces/go-sqlite3/vfs/api.go @@ -0,0 +1,175 @@ +// Package vfs wraps the C SQLite VFS API. +package vfs + +import ( + "context" + "io" + + "github.com/tetratelabs/wazero/api" +) + +// A VFS defines the interface between the SQLite core and the underlying operating system. +// +// Use sqlite3.ErrorCode or sqlite3.ExtendedErrorCode to return specific error codes to SQLite. +// +// https://sqlite.org/c3ref/vfs.html +type VFS interface { + Open(name string, flags OpenFlag) (File, OpenFlag, error) + Delete(name string, syncDir bool) error + Access(name string, flags AccessFlag) (bool, error) + FullPathname(name string) (string, error) +} + +// VFSFilename extends VFS with the ability to use Filename +// objects for opening files. +// +// https://sqlite.org/c3ref/filename.html +type VFSFilename interface { + VFS + OpenFilename(name *Filename, flags OpenFlag) (File, OpenFlag, error) +} + +// A File represents an open file in the OS interface layer. +// +// Use sqlite3.ErrorCode or sqlite3.ExtendedErrorCode to return specific error codes to SQLite. +// In particular, sqlite3.BUSY is necessary to correctly implement lock methods. +// +// https://sqlite.org/c3ref/io_methods.html +type File interface { + Close() error + ReadAt(p []byte, off int64) (n int, err error) + WriteAt(p []byte, off int64) (n int, err error) + Truncate(size int64) error + Sync(flags SyncFlag) error + Size() (int64, error) + Lock(lock LockLevel) error + Unlock(lock LockLevel) error + CheckReservedLock() (bool, error) + SectorSize() int + DeviceCharacteristics() DeviceCharacteristic +} + +// FileLockState extends File to implement the +// SQLITE_FCNTL_LOCKSTATE file control opcode. +// +// https://sqlite.org/c3ref/c_fcntl_begin_atomic_write.html#sqlitefcntllockstate +type FileLockState interface { + File + LockState() LockLevel +} + +// FileChunkSize extends File to implement the +// SQLITE_FCNTL_CHUNK_SIZE file control opcode. +// +// https://sqlite.org/c3ref/c_fcntl_begin_atomic_write.html#sqlitefcntlchunksize +type FileChunkSize interface { + File + ChunkSize(size int) +} + +// FileSizeHint extends File to implement the +// SQLITE_FCNTL_SIZE_HINT file control opcode. +// +// https://sqlite.org/c3ref/c_fcntl_begin_atomic_write.html#sqlitefcntlsizehint +type FileSizeHint interface { + File + SizeHint(size int64) error +} + +// FileHasMoved extends File to implement the +// SQLITE_FCNTL_HAS_MOVED file control opcode. +// +// https://sqlite.org/c3ref/c_fcntl_begin_atomic_write.html#sqlitefcntlhasmoved +type FileHasMoved interface { + File + HasMoved() (bool, error) +} + +// FileOverwrite extends File to implement the +// SQLITE_FCNTL_OVERWRITE file control opcode. +// +// https://sqlite.org/c3ref/c_fcntl_begin_atomic_write.html#sqlitefcntloverwrite +type FileOverwrite interface { + File + Overwrite() error +} + +// FilePersistentWAL extends File to implement the +// SQLITE_FCNTL_PERSIST_WAL file control opcode. +// +// https://sqlite.org/c3ref/c_fcntl_begin_atomic_write.html#sqlitefcntlpersistwal +type FilePersistentWAL interface { + File + PersistentWAL() bool + SetPersistentWAL(bool) +} + +// FilePowersafeOverwrite extends File to implement the +// SQLITE_FCNTL_POWERSAFE_OVERWRITE file control opcode. +// +// https://sqlite.org/c3ref/c_fcntl_begin_atomic_write.html#sqlitefcntlpowersafeoverwrite +type FilePowersafeOverwrite interface { + File + PowersafeOverwrite() bool + SetPowersafeOverwrite(bool) +} + +// FileCommitPhaseTwo extends File to implement the +// SQLITE_FCNTL_COMMIT_PHASETWO file control opcode. +// +// https://sqlite.org/c3ref/c_fcntl_begin_atomic_write.html#sqlitefcntlcommitphasetwo +type FileCommitPhaseTwo interface { + File + CommitPhaseTwo() error +} + +// FileBatchAtomicWrite extends File to implement the +// SQLITE_FCNTL_BEGIN_ATOMIC_WRITE, SQLITE_FCNTL_COMMIT_ATOMIC_WRITE +// and SQLITE_FCNTL_ROLLBACK_ATOMIC_WRITE file control opcodes. +// +// https://sqlite.org/c3ref/c_fcntl_begin_atomic_write.html#sqlitefcntlbeginatomicwrite +type FileBatchAtomicWrite interface { + File + BeginAtomicWrite() error + CommitAtomicWrite() error + RollbackAtomicWrite() error +} + +// FilePragma extends File to implement the +// SQLITE_FCNTL_PRAGMA file control opcode. +// +// https://sqlite.org/c3ref/c_fcntl_begin_atomic_write.html#sqlitefcntlpragma +type FilePragma interface { + File + Pragma(name, value string) (string, error) +} + +// FileCheckpoint extends File to implement the +// SQLITE_FCNTL_CKPT_START and SQLITE_FCNTL_CKPT_DONE +// file control opcodes. +// +// https://sqlite.org/c3ref/c_fcntl_begin_atomic_write.html#sqlitefcntlckptstart +type FileCheckpoint interface { + File + CheckpointDone() error + CheckpointStart() error +} + +// FileSharedMemory extends File to possibly implement +// shared-memory for the WAL-index. +// The same shared-memory instance must be returned +// for the entire life of the file. +// It's OK for SharedMemory to return nil. +type FileSharedMemory interface { + File + SharedMemory() SharedMemory +} + +// SharedMemory is a shared-memory WAL-index implementation. +// Use [NewSharedMemory] to create a shared-memory. +type SharedMemory interface { + shmMap(context.Context, api.Module, int32, int32, bool) (uint32, error) + shmLock(int32, int32, _ShmFlag) error + shmUnmap(bool) + io.Closer +} diff --git a/vendor/github.com/ncruces/go-sqlite3/vfs/const.go b/vendor/github.com/ncruces/go-sqlite3/vfs/const.go new file mode 100644 index 000000000..7f409f35f --- /dev/null +++ b/vendor/github.com/ncruces/go-sqlite3/vfs/const.go @@ -0,0 +1,234 @@ +package vfs + +import "github.com/ncruces/go-sqlite3/internal/util" + +const ( + _MAX_NAME = 1e6 // Self-imposed limit for most NUL terminated strings. + _MAX_SQL_LENGTH = 1e9 + _MAX_PATHNAME = 1024 + _DEFAULT_SECTOR_SIZE = 4096 + + ptrlen = 4 +) + +// https://sqlite.org/rescode.html +type _ErrorCode uint32 + +func (e _ErrorCode) Error() string { + return util.ErrorCodeString(uint32(e)) +} + +const ( + _OK _ErrorCode = util.OK + _ERROR _ErrorCode = util.ERROR + _PERM _ErrorCode = util.PERM + _BUSY _ErrorCode = util.BUSY + _READONLY _ErrorCode = util.READONLY + _IOERR _ErrorCode = util.IOERR + _NOTFOUND _ErrorCode = util.NOTFOUND + _CANTOPEN _ErrorCode = util.CANTOPEN + _IOERR_READ _ErrorCode = util.IOERR_READ + _IOERR_SHORT_READ _ErrorCode = util.IOERR_SHORT_READ + _IOERR_WRITE _ErrorCode = util.IOERR_WRITE + _IOERR_FSYNC _ErrorCode = util.IOERR_FSYNC + _IOERR_DIR_FSYNC _ErrorCode = util.IOERR_DIR_FSYNC + _IOERR_TRUNCATE _ErrorCode = util.IOERR_TRUNCATE + _IOERR_FSTAT _ErrorCode = util.IOERR_FSTAT + _IOERR_UNLOCK _ErrorCode = util.IOERR_UNLOCK + _IOERR_RDLOCK _ErrorCode = util.IOERR_RDLOCK + _IOERR_DELETE _ErrorCode = util.IOERR_DELETE + _IOERR_ACCESS _ErrorCode = util.IOERR_ACCESS + _IOERR_CHECKRESERVEDLOCK _ErrorCode = util.IOERR_CHECKRESERVEDLOCK + _IOERR_LOCK _ErrorCode = util.IOERR_LOCK + _IOERR_CLOSE _ErrorCode = util.IOERR_CLOSE + _IOERR_SHMOPEN _ErrorCode = util.IOERR_SHMOPEN + _IOERR_SHMSIZE _ErrorCode = util.IOERR_SHMSIZE + _IOERR_SHMLOCK _ErrorCode = util.IOERR_SHMLOCK + _IOERR_SHMMAP _ErrorCode = util.IOERR_SHMMAP + _IOERR_SEEK _ErrorCode = util.IOERR_SEEK + _IOERR_DELETE_NOENT _ErrorCode = util.IOERR_DELETE_NOENT + _IOERR_BEGIN_ATOMIC _ErrorCode = util.IOERR_BEGIN_ATOMIC + _IOERR_COMMIT_ATOMIC _ErrorCode = util.IOERR_COMMIT_ATOMIC + _IOERR_ROLLBACK_ATOMIC _ErrorCode = util.IOERR_ROLLBACK_ATOMIC + _CANTOPEN_FULLPATH _ErrorCode = util.CANTOPEN_FULLPATH + _CANTOPEN_ISDIR _ErrorCode = util.CANTOPEN_ISDIR + _READONLY_CANTINIT _ErrorCode = util.READONLY_CANTINIT + _OK_SYMLINK _ErrorCode = util.OK_SYMLINK +) + +// OpenFlag is a flag for the [VFS] Open method. +// +// https://sqlite.org/c3ref/c_open_autoproxy.html +type OpenFlag uint32 + +const ( + OPEN_READONLY OpenFlag = 0x00000001 /* Ok for sqlite3_open_v2() */ + OPEN_READWRITE OpenFlag = 0x00000002 /* Ok for sqlite3_open_v2() */ + OPEN_CREATE OpenFlag = 0x00000004 /* Ok for sqlite3_open_v2() */ + OPEN_DELETEONCLOSE OpenFlag = 0x00000008 /* VFS only */ + OPEN_EXCLUSIVE OpenFlag = 0x00000010 /* VFS only */ + OPEN_AUTOPROXY OpenFlag = 0x00000020 /* VFS only */ + OPEN_URI OpenFlag = 0x00000040 /* Ok for sqlite3_open_v2() */ + OPEN_MEMORY OpenFlag = 0x00000080 /* Ok for sqlite3_open_v2() */ + OPEN_MAIN_DB OpenFlag = 0x00000100 /* VFS only */ + OPEN_TEMP_DB OpenFlag = 0x00000200 /* VFS only */ + OPEN_TRANSIENT_DB OpenFlag = 0x00000400 /* VFS only */ + OPEN_MAIN_JOURNAL OpenFlag = 0x00000800 /* VFS only */ + OPEN_TEMP_JOURNAL OpenFlag = 0x00001000 /* VFS only */ + OPEN_SUBJOURNAL OpenFlag = 0x00002000 /* VFS only */ + OPEN_SUPER_JOURNAL OpenFlag = 0x00004000 /* VFS only */ + OPEN_NOMUTEX OpenFlag = 0x00008000 /* Ok for sqlite3_open_v2() */ + OPEN_FULLMUTEX OpenFlag = 0x00010000 /* Ok for sqlite3_open_v2() */ + OPEN_SHAREDCACHE OpenFlag = 0x00020000 /* Ok for sqlite3_open_v2() */ + OPEN_PRIVATECACHE OpenFlag = 0x00040000 /* Ok for sqlite3_open_v2() */ + OPEN_WAL OpenFlag = 0x00080000 /* VFS only */ + OPEN_NOFOLLOW OpenFlag = 0x01000000 /* Ok for sqlite3_open_v2() */ +) + +// AccessFlag is a flag for the [VFS] Access method. +// +// https://sqlite.org/c3ref/c_access_exists.html +type AccessFlag uint32 + +const ( + ACCESS_EXISTS AccessFlag = 0 + ACCESS_READWRITE AccessFlag = 1 /* Used by PRAGMA temp_store_directory */ + ACCESS_READ AccessFlag = 2 /* Unused */ +) + +// SyncFlag is a flag for the [File] Sync method. +// +// https://sqlite.org/c3ref/c_sync_dataonly.html +type SyncFlag uint32 + +const ( + SYNC_NORMAL SyncFlag = 0x00002 + SYNC_FULL SyncFlag = 0x00003 + SYNC_DATAONLY SyncFlag = 0x00010 +) + +// LockLevel is a value used with [File] Lock and Unlock methods. +// +// https://sqlite.org/c3ref/c_lock_exclusive.html +type LockLevel uint32 + +const ( + // No locks are held on the database. + // The database may be neither read nor written. + // Any internally cached data is considered suspect and subject to + // verification against the database file before being used. + // Other processes can read or write the database as their own locking + // states permit. + // This is the default state. + LOCK_NONE LockLevel = 0 /* xUnlock() only */ + + // The database may be read but not written. + // Any number of processes can hold SHARED locks at the same time, + // hence there can be many simultaneous readers. + // But no other thread or process is allowed to write to the database file + // while one or more SHARED locks are active. + LOCK_SHARED LockLevel = 1 /* xLock() or xUnlock() */ + + // A RESERVED lock means that the process is planning on writing to the + // database file at some point in the future but that it is currently just + // reading from the file. + // Only a single RESERVED lock may be active at one time, + // though multiple SHARED locks can coexist with a single RESERVED lock. + // RESERVED differs from PENDING in that new SHARED locks can be acquired + // while there is a RESERVED lock. + LOCK_RESERVED LockLevel = 2 /* xLock() only */ + + // A PENDING lock means that the process holding the lock wants to write to + // the database as soon as possible and is just waiting on all current + // SHARED locks to clear so that it can get an EXCLUSIVE lock. + // No new SHARED locks are permitted against the database if a PENDING lock + // is active, though existing SHARED locks are allowed to continue. + LOCK_PENDING LockLevel = 3 /* internal use only */ + + // An EXCLUSIVE lock is needed in order to write to the database file. + // Only one EXCLUSIVE lock is allowed on the file and no other locks of any + // kind are allowed to coexist with an EXCLUSIVE lock. + // In order to maximize concurrency, SQLite works to minimize the amount of + // time that EXCLUSIVE locks are held. + LOCK_EXCLUSIVE LockLevel = 4 /* xLock() only */ +) + +// DeviceCharacteristic is a flag retuned by the [File] DeviceCharacteristics method. +// +// https://sqlite.org/c3ref/c_iocap_atomic.html +type DeviceCharacteristic uint32 + +const ( + IOCAP_ATOMIC DeviceCharacteristic = 0x00000001 + IOCAP_ATOMIC512 DeviceCharacteristic = 0x00000002 + IOCAP_ATOMIC1K DeviceCharacteristic = 0x00000004 + IOCAP_ATOMIC2K DeviceCharacteristic = 0x00000008 + IOCAP_ATOMIC4K DeviceCharacteristic = 0x00000010 + IOCAP_ATOMIC8K DeviceCharacteristic = 0x00000020 + IOCAP_ATOMIC16K DeviceCharacteristic = 0x00000040 + IOCAP_ATOMIC32K DeviceCharacteristic = 0x00000080 + IOCAP_ATOMIC64K DeviceCharacteristic = 0x00000100 + IOCAP_SAFE_APPEND DeviceCharacteristic = 0x00000200 + IOCAP_SEQUENTIAL DeviceCharacteristic = 0x00000400 + IOCAP_UNDELETABLE_WHEN_OPEN DeviceCharacteristic = 0x00000800 + IOCAP_POWERSAFE_OVERWRITE DeviceCharacteristic = 0x00001000 + IOCAP_IMMUTABLE DeviceCharacteristic = 0x00002000 + IOCAP_BATCH_ATOMIC DeviceCharacteristic = 0x00004000 +) + +// https://sqlite.org/c3ref/c_fcntl_begin_atomic_write.html +type _FcntlOpcode uint32 + +const ( + _FCNTL_LOCKSTATE _FcntlOpcode = 1 + _FCNTL_GET_LOCKPROXYFILE _FcntlOpcode = 2 + _FCNTL_SET_LOCKPROXYFILE _FcntlOpcode = 3 + _FCNTL_LAST_ERRNO _FcntlOpcode = 4 + _FCNTL_SIZE_HINT _FcntlOpcode = 5 + _FCNTL_CHUNK_SIZE _FcntlOpcode = 6 + _FCNTL_FILE_POINTER _FcntlOpcode = 7 + _FCNTL_SYNC_OMITTED _FcntlOpcode = 8 + _FCNTL_WIN32_AV_RETRY _FcntlOpcode = 9 + _FCNTL_PERSIST_WAL _FcntlOpcode = 10 + _FCNTL_OVERWRITE _FcntlOpcode = 11 + _FCNTL_VFSNAME _FcntlOpcode = 12 + _FCNTL_POWERSAFE_OVERWRITE _FcntlOpcode = 13 + _FCNTL_PRAGMA _FcntlOpcode = 14 + _FCNTL_BUSYHANDLER _FcntlOpcode = 15 + _FCNTL_TEMPFILENAME _FcntlOpcode = 16 + _FCNTL_MMAP_SIZE _FcntlOpcode = 18 + _FCNTL_TRACE _FcntlOpcode = 19 + _FCNTL_HAS_MOVED _FcntlOpcode = 20 + _FCNTL_SYNC _FcntlOpcode = 21 + _FCNTL_COMMIT_PHASETWO _FcntlOpcode = 22 + _FCNTL_WIN32_SET_HANDLE _FcntlOpcode = 23 + _FCNTL_WAL_BLOCK _FcntlOpcode = 24 + _FCNTL_ZIPVFS _FcntlOpcode = 25 + _FCNTL_RBU _FcntlOpcode = 26 + _FCNTL_VFS_POINTER _FcntlOpcode = 27 + _FCNTL_JOURNAL_POINTER _FcntlOpcode = 28 + _FCNTL_WIN32_GET_HANDLE _FcntlOpcode = 29 + _FCNTL_PDB _FcntlOpcode = 30 + _FCNTL_BEGIN_ATOMIC_WRITE _FcntlOpcode = 31 + _FCNTL_COMMIT_ATOMIC_WRITE _FcntlOpcode = 32 + _FCNTL_ROLLBACK_ATOMIC_WRITE _FcntlOpcode = 33 + _FCNTL_LOCK_TIMEOUT _FcntlOpcode = 34 + _FCNTL_DATA_VERSION _FcntlOpcode = 35 + _FCNTL_SIZE_LIMIT _FcntlOpcode = 36 + _FCNTL_CKPT_DONE _FcntlOpcode = 37 + _FCNTL_RESERVE_BYTES _FcntlOpcode = 38 + _FCNTL_CKPT_START _FcntlOpcode = 39 + _FCNTL_EXTERNAL_READER _FcntlOpcode = 40 + _FCNTL_CKSM_FILE _FcntlOpcode = 41 + _FCNTL_RESET_CACHE _FcntlOpcode = 42 +) + +// https://sqlite.org/c3ref/c_shm_exclusive.html +type _ShmFlag uint32 + +const ( + _SHM_UNLOCK _ShmFlag = 1 + _SHM_LOCK _ShmFlag = 2 + _SHM_SHARED _ShmFlag = 4 + _SHM_EXCLUSIVE _ShmFlag = 8 +) diff --git a/vendor/github.com/ncruces/go-sqlite3/vfs/file.go b/vendor/github.com/ncruces/go-sqlite3/vfs/file.go new file mode 100644 index 000000000..ca8cf84f3 --- /dev/null +++ b/vendor/github.com/ncruces/go-sqlite3/vfs/file.go @@ -0,0 +1,217 @@ +package vfs + +import ( + "errors" + "io" + "io/fs" + "os" + "path/filepath" + "runtime" + "syscall" + + "github.com/ncruces/go-sqlite3/util/osutil" +) + +type vfsOS struct{} + +func (vfsOS) FullPathname(path string) (string, error) { + path, err := filepath.Abs(path) + if err != nil { + return "", err + } + fi, err := os.Lstat(path) + if err != nil { + if errors.Is(err, fs.ErrNotExist) { + return path, nil + } + return "", err + } + if fi.Mode()&fs.ModeSymlink != 0 { + err = _OK_SYMLINK + } + return path, err +} + +func (vfsOS) Delete(path string, syncDir bool) error { + err := os.Remove(path) + if err != nil { + if errors.Is(err, fs.ErrNotExist) { + return _IOERR_DELETE_NOENT + } + return err + } + if runtime.GOOS != "windows" && syncDir { + f, err := os.Open(filepath.Dir(path)) + if err != nil { + return _OK + } + defer f.Close() + err = osSync(f, false, false) + if err != nil { + return _IOERR_DIR_FSYNC + } + } + return nil +} + +func (vfsOS) Access(name string, flags AccessFlag) (bool, error) { + err := osAccess(name, flags) + if flags == ACCESS_EXISTS { + if errors.Is(err, fs.ErrNotExist) { + return false, nil + } + } else { + if errors.Is(err, fs.ErrPermission) { + return false, nil + } + } + return err == nil, err +} + +func (vfsOS) Open(name string, flags OpenFlag) (File, OpenFlag, error) { + return nil, 0, _CANTOPEN +} + +func (vfsOS) OpenFilename(name *Filename, flags OpenFlag) (File, OpenFlag, error) { + var oflags int + if flags&OPEN_EXCLUSIVE != 0 { + oflags |= os.O_EXCL + } + if flags&OPEN_CREATE != 0 { + oflags |= os.O_CREATE + } + if flags&OPEN_READONLY != 0 { + oflags |= os.O_RDONLY + } + if flags&OPEN_READWRITE != 0 { + oflags |= os.O_RDWR + } + + var err error + var f *os.File + if name == nil { + f, err = os.CreateTemp("", "*.db") + } else { + f, err = osutil.OpenFile(name.String(), oflags, 0666) + } + if err != nil { + if errors.Is(err, syscall.EISDIR) { + return nil, flags, _CANTOPEN_ISDIR + } + return nil, flags, err + } + + if modeof := name.URIParameter("modeof"); modeof != "" { + if err = osSetMode(f, modeof); err != nil { + f.Close() + return nil, flags, _IOERR_FSTAT + } + } + if flags&OPEN_DELETEONCLOSE != 0 { + os.Remove(f.Name()) + } + + file := vfsFile{ + File: f, + psow: true, + readOnly: flags&OPEN_READONLY != 0, + syncDir: runtime.GOOS != "windows" && + flags&(OPEN_CREATE) != 0 && + flags&(OPEN_MAIN_JOURNAL|OPEN_SUPER_JOURNAL|OPEN_WAL) != 0, + shm: NewSharedMemory(name.String()+"-shm", flags), + } + return &file, flags, nil +} + +type vfsFile struct { + *os.File + shm SharedMemory + lock LockLevel + readOnly bool + keepWAL bool + syncDir bool + psow bool +} + +var ( + // Ensure these interfaces are implemented: + _ FileLockState = &vfsFile{} + _ FileHasMoved = &vfsFile{} + _ FileSizeHint = &vfsFile{} + _ FilePersistentWAL = &vfsFile{} + _ FilePowersafeOverwrite = &vfsFile{} +) + +func (f *vfsFile) Close() error { + if f.shm != nil { + f.shm.Close() + } + return f.File.Close() +} + +func (f *vfsFile) Sync(flags SyncFlag) error { + dataonly := (flags & SYNC_DATAONLY) != 0 + fullsync := (flags & 0x0f) == SYNC_FULL + + err := osSync(f.File, fullsync, dataonly) + if err != nil { + return err + } + if runtime.GOOS != "windows" && f.syncDir { + f.syncDir = false + d, err := os.Open(filepath.Dir(f.File.Name())) + if err != nil { + return nil + } + defer d.Close() + err = osSync(d, false, false) + if err != nil { + return _IOERR_DIR_FSYNC + } + } + return nil +} + +func (f *vfsFile) Size() (int64, error) { + return f.Seek(0, io.SeekEnd) +} + +func (f *vfsFile) SectorSize() int { + return _DEFAULT_SECTOR_SIZE +} + +func (f *vfsFile) DeviceCharacteristics() DeviceCharacteristic { + var res DeviceCharacteristic + if osBatchAtomic(f.File) { + res |= IOCAP_BATCH_ATOMIC + } + if f.psow { + res |= IOCAP_POWERSAFE_OVERWRITE + } + return res +} + +func (f *vfsFile) SizeHint(size int64) error { + return osAllocate(f.File, size) +} + +func (f *vfsFile) HasMoved() (bool, error) { + fi, err := f.Stat() + if err != nil { + return false, err + } + pi, err := os.Stat(f.Name()) + if err != nil { + if errors.Is(err, fs.ErrNotExist) { + return true, nil + } + return false, err + } + return !os.SameFile(fi, pi), nil +} + +func (f *vfsFile) LockState() LockLevel { return f.lock } +func (f *vfsFile) PowersafeOverwrite() bool { return f.psow } +func (f *vfsFile) PersistentWAL() bool { return f.keepWAL } +func (f *vfsFile) SetPowersafeOverwrite(psow bool) { f.psow = psow } +func (f *vfsFile) SetPersistentWAL(keepWAL bool) { f.keepWAL = keepWAL } diff --git a/vendor/github.com/ncruces/go-sqlite3/vfs/filename.go b/vendor/github.com/ncruces/go-sqlite3/vfs/filename.go new file mode 100644 index 000000000..e23575bbb --- /dev/null +++ b/vendor/github.com/ncruces/go-sqlite3/vfs/filename.go @@ -0,0 +1,174 @@ +package vfs + +import ( + "context" + "net/url" + + "github.com/ncruces/go-sqlite3/internal/util" + "github.com/tetratelabs/wazero/api" +) + +// Filename is used by SQLite to pass filenames +// to the Open method of a VFS. +// +// https://sqlite.org/c3ref/filename.html +type Filename struct { + ctx context.Context + mod api.Module + zPath uint32 + flags OpenFlag + stack [2]uint64 +} + +// OpenFilename is an internal API users should not call directly. +func OpenFilename(ctx context.Context, mod api.Module, id uint32, flags OpenFlag) *Filename { + if id == 0 { + return nil + } + return &Filename{ + ctx: ctx, + mod: mod, + zPath: id, + flags: flags, + } +} + +// String returns this filename as a string. +func (n *Filename) String() string { + if n == nil || n.zPath == 0 { + return "" + } + return util.ReadString(n.mod, n.zPath, _MAX_PATHNAME) +} + +// Database returns the name of the corresponding database file. +// +// https://sqlite.org/c3ref/filename_database.html +func (n *Filename) Database() string { + return n.path("sqlite3_filename_database") +} + +// Journal returns the name of the corresponding rollback journal file. +// +// https://sqlite.org/c3ref/filename_database.html +func (n *Filename) Journal() string { + return n.path("sqlite3_filename_journal") +} + +// Journal returns the name of the corresponding WAL file. +// +// https://sqlite.org/c3ref/filename_database.html +func (n *Filename) WAL() string { + return n.path("sqlite3_filename_wal") +} + +func (n *Filename) path(method string) string { + if n == nil || n.zPath == 0 { + return "" + } + n.stack[0] = uint64(n.zPath) + fn := n.mod.ExportedFunction(method) + if err := fn.CallWithStack(n.ctx, n.stack[:]); err != nil { + panic(err) + } + return util.ReadString(n.mod, uint32(n.stack[0]), _MAX_PATHNAME) +} + +// DatabaseFile returns the main database [File] corresponding to a journal. +// +// https://sqlite.org/c3ref/database_file_object.html +func (n *Filename) DatabaseFile() File { + if n == nil || n.zPath == 0 { + return nil + } + if n.flags&(OPEN_MAIN_DB|OPEN_MAIN_JOURNAL|OPEN_WAL) == 0 { + return nil + } + + n.stack[0] = uint64(n.zPath) + fn := n.mod.ExportedFunction("sqlite3_database_file_object") + if err := fn.CallWithStack(n.ctx, n.stack[:]); err != nil { + panic(err) + } + file, _ := vfsFileGet(n.ctx, n.mod, uint32(n.stack[0])).(File) + return file +} + +// URIParameter returns the value of a URI parameter. +// +// https://sqlite.org/c3ref/uri_boolean.html +func (n *Filename) URIParameter(key string) string { + if n == nil || n.zPath == 0 { + return "" + } + + uriKey := n.mod.ExportedFunction("sqlite3_uri_key") + n.stack[0] = uint64(n.zPath) + n.stack[1] = uint64(0) + if err := uriKey.CallWithStack(n.ctx, n.stack[:]); err != nil { + panic(err) + } + + ptr := uint32(n.stack[0]) + if ptr == 0 { + return "" + } + + // Parse the format from: + // https://github.com/sqlite/sqlite/blob/b74eb0/src/pager.c#L4797-L4840 + // This avoids having to alloc/free the key just to find a value. + for { + k := util.ReadString(n.mod, ptr, _MAX_NAME) + if k == "" { + return "" + } + ptr += uint32(len(k)) + 1 + + v := util.ReadString(n.mod, ptr, _MAX_NAME) + if k == key { + return v + } + ptr += uint32(len(v)) + 1 + } +} + +// URIParameters obtains values for URI parameters. +// +// https://sqlite.org/c3ref/uri_boolean.html +func (n *Filename) URIParameters() url.Values { + if n == nil || n.zPath == 0 { + return nil + } + + uriKey := n.mod.ExportedFunction("sqlite3_uri_key") + n.stack[0] = uint64(n.zPath) + n.stack[1] = uint64(0) + if err := uriKey.CallWithStack(n.ctx, n.stack[:]); err != nil { + panic(err) + } + + ptr := uint32(n.stack[0]) + if ptr == 0 { + return nil + } + + var params url.Values + + // Parse the format from: + // https://github.com/sqlite/sqlite/blob/b74eb0/src/pager.c#L4797-L4840 + // This is the only way to support multiple valued keys. + for { + k := util.ReadString(n.mod, ptr, _MAX_NAME) + if k == "" { + return params + } + ptr += uint32(len(k)) + 1 + + v := util.ReadString(n.mod, ptr, _MAX_NAME) + if params == nil { + params = url.Values{} + } + params.Add(k, v) + ptr += uint32(len(v)) + 1 + } +} diff --git a/vendor/github.com/ncruces/go-sqlite3/vfs/lock.go b/vendor/github.com/ncruces/go-sqlite3/vfs/lock.go new file mode 100644 index 000000000..86a988ae8 --- /dev/null +++ b/vendor/github.com/ncruces/go-sqlite3/vfs/lock.go @@ -0,0 +1,144 @@ +//go:build (linux || darwin || windows || freebsd || openbsd || netbsd || dragonfly || illumos || sqlite3_flock) && !sqlite3_nosys + +package vfs + +import "github.com/ncruces/go-sqlite3/internal/util" + +// SupportsFileLocking is false on platforms that do not support file locking. +// To open a database file on those platforms, +// you need to use the [nolock] or [immutable] URI parameters. +// +// [nolock]: https://sqlite.org/uri.html#urinolock +// [immutable]: https://sqlite.org/uri.html#uriimmutable +const SupportsFileLocking = true + +const ( + _PENDING_BYTE = 0x40000000 + _RESERVED_BYTE = (_PENDING_BYTE + 1) + _SHARED_FIRST = (_PENDING_BYTE + 2) + _SHARED_SIZE = 510 +) + +func (f *vfsFile) Lock(lock LockLevel) error { + // Argument check. SQLite never explicitly requests a pending lock. + if lock != LOCK_SHARED && lock != LOCK_RESERVED && lock != LOCK_EXCLUSIVE { + panic(util.AssertErr()) + } + + switch { + case f.lock < LOCK_NONE || f.lock > LOCK_EXCLUSIVE: + // Connection state check. + panic(util.AssertErr()) + case f.lock == LOCK_NONE && lock > LOCK_SHARED: + // We never move from unlocked to anything higher than a shared lock. + panic(util.AssertErr()) + case f.lock != LOCK_SHARED && lock == LOCK_RESERVED: + // A shared lock is always held when a reserved lock is requested. + panic(util.AssertErr()) + } + + // If we already have an equal or more restrictive lock, do nothing. + if f.lock >= lock { + return nil + } + + // Do not allow any kind of write-lock on a read-only database. + if f.readOnly && lock >= LOCK_RESERVED { + return _IOERR_LOCK + } + + switch lock { + case LOCK_SHARED: + // Must be unlocked to get SHARED. + if f.lock != LOCK_NONE { + panic(util.AssertErr()) + } + if rc := osGetSharedLock(f.File); rc != _OK { + return rc + } + f.lock = LOCK_SHARED + return nil + + case LOCK_RESERVED: + // Must be SHARED to get RESERVED. + if f.lock != LOCK_SHARED { + panic(util.AssertErr()) + } + if rc := osGetReservedLock(f.File); rc != _OK { + return rc + } + f.lock = LOCK_RESERVED + return nil + + case LOCK_EXCLUSIVE: + // Must be SHARED, RESERVED or PENDING to get EXCLUSIVE. + if f.lock <= LOCK_NONE || f.lock >= LOCK_EXCLUSIVE { + panic(util.AssertErr()) + } + reserved := f.lock == LOCK_RESERVED + // A PENDING lock is needed before acquiring an EXCLUSIVE lock. + if f.lock < LOCK_PENDING { + // If we're already RESERVED, we can block indefinitely, + // since only new readers may briefly hold the PENDING lock. + if rc := osGetPendingLock(f.File, reserved /* block */); rc != _OK { + return rc + } + f.lock = LOCK_PENDING + } + // We already have PENDING, so we're just waiting for readers to leave. + // If we were RESERVED, we can wait for a little while, before invoking + // the busy handler; we will only do this once. + if rc := osGetExclusiveLock(f.File, reserved /* wait */); rc != _OK { + return rc + } + f.lock = LOCK_EXCLUSIVE + return nil + + default: + panic(util.AssertErr()) + } +} + +func (f *vfsFile) Unlock(lock LockLevel) error { + // Argument check. + if lock != LOCK_NONE && lock != LOCK_SHARED { + panic(util.AssertErr()) + } + + // Connection state check. + if f.lock < LOCK_NONE || f.lock > LOCK_EXCLUSIVE { + panic(util.AssertErr()) + } + + // If we don't have a more restrictive lock, do nothing. + if f.lock <= lock { + return nil + } + + switch lock { + case LOCK_SHARED: + rc := osDowngradeLock(f.File, f.lock) + f.lock = LOCK_SHARED + return rc + + case LOCK_NONE: + rc := osReleaseLock(f.File, f.lock) + f.lock = LOCK_NONE + return rc + + default: + panic(util.AssertErr()) + } +} + +func (f *vfsFile) CheckReservedLock() (bool, error) { + // Connection state check. + if f.lock < LOCK_NONE || f.lock > LOCK_EXCLUSIVE { + panic(util.AssertErr()) + } + + if f.lock >= LOCK_RESERVED { + return true, nil + } + return osCheckReservedLock(f.File) +} diff --git a/vendor/github.com/ncruces/go-sqlite3/vfs/lock_other.go b/vendor/github.com/ncruces/go-sqlite3/vfs/lock_other.go new file mode 100644 index 000000000..c395f34a7 --- /dev/null +++ b/vendor/github.com/ncruces/go-sqlite3/vfs/lock_other.go @@ -0,0 +1,23 @@ +//go:build !(linux || darwin || windows || freebsd || openbsd || netbsd || dragonfly || illumos || sqlite3_flock) || sqlite3_nosys + +package vfs + +// SupportsFileLocking is false on platforms that do not support file locking. +// To open a database file on those platforms, +// you need to use the [nolock] or [immutable] URI parameters. +// +// [nolock]: https://sqlite.org/uri.html#urinolock +// [immutable]: https://sqlite.org/uri.html#uriimmutable +const SupportsFileLocking = false + +func (f *vfsFile) Lock(LockLevel) error { + return _IOERR_LOCK +} + +func (f *vfsFile) Unlock(LockLevel) error { + return _IOERR_UNLOCK +} + +func (f *vfsFile) CheckReservedLock() (bool, error) { + return false, _IOERR_CHECKRESERVEDLOCK +} diff --git a/vendor/github.com/ncruces/go-sqlite3/vfs/memdb/README.md b/vendor/github.com/ncruces/go-sqlite3/vfs/memdb/README.md new file mode 100644 index 000000000..193e29d98 --- /dev/null +++ b/vendor/github.com/ncruces/go-sqlite3/vfs/memdb/README.md @@ -0,0 +1,9 @@ +# Go `"memdb"` SQLite VFS + +This package implements the [`"memdb"`](https://sqlite.org/src/doc/tip/src/memdb.c) +SQLite VFS in pure Go. + +It has some benefits over the C version: +- the memory backing the database needs not be contiguous, +- the database can grow/shrink incrementally without copying, +- reader-writer concurrency is slightly improved. \ No newline at end of file diff --git a/vendor/github.com/ncruces/go-sqlite3/vfs/memdb/api.go b/vendor/github.com/ncruces/go-sqlite3/vfs/memdb/api.go new file mode 100644 index 000000000..5a2b84c71 --- /dev/null +++ b/vendor/github.com/ncruces/go-sqlite3/vfs/memdb/api.go @@ -0,0 +1,68 @@ +// Package memdb implements the "memdb" SQLite VFS. +// +// The "memdb" [vfs.VFS] allows the same in-memory database to be shared +// among multiple database connections in the same process, +// as long as the database name begins with "/". +// +// Importing package memdb registers the VFS: +// +// import _ "github.com/ncruces/go-sqlite3/vfs/memdb" +package memdb + +import ( + "sync" + + "github.com/ncruces/go-sqlite3/vfs" +) + +func init() { + vfs.Register("memdb", memVFS{}) +} + +var ( + memoryMtx sync.Mutex + // +checklocks:memoryMtx + memoryDBs = map[string]*memDB{} +) + +// Create creates a shared memory database, +// using data as its initial contents. +// The new database takes ownership of data, +// and the caller should not use data after this call. +func Create(name string, data []byte) { + memoryMtx.Lock() + defer memoryMtx.Unlock() + + db := &memDB{ + refs: 1, + name: name, + size: int64(len(data)), + } + + // Convert data from WAL to rollback journal. + if len(data) >= 20 && data[18] == 2 && data[19] == 2 { + data[18] = 1 + data[19] = 1 + } + + sectors := divRoundUp(db.size, sectorSize) + db.data = make([]*[sectorSize]byte, sectors) + for i := range db.data { + sector := data[i*sectorSize:] + if len(sector) >= sectorSize { + db.data[i] = (*[sectorSize]byte)(sector) + } else { + db.data[i] = new([sectorSize]byte) + copy((*db.data[i])[:], sector) + } + } + + memoryDBs[name] = db +} + +// Delete deletes a shared memory database. +func Delete(name string) { + memoryMtx.Lock() + defer memoryMtx.Unlock() + delete(memoryDBs, name) +} diff --git a/vendor/github.com/ncruces/go-sqlite3/vfs/memdb/memdb.go b/vendor/github.com/ncruces/go-sqlite3/vfs/memdb/memdb.go new file mode 100644 index 000000000..8dc57ab9c --- /dev/null +++ b/vendor/github.com/ncruces/go-sqlite3/vfs/memdb/memdb.go @@ -0,0 +1,311 @@ +package memdb + +import ( + "io" + "runtime" + "sync" + "time" + + "github.com/ncruces/go-sqlite3" + "github.com/ncruces/go-sqlite3/vfs" +) + +// Must be a multiple of 64K (the largest page size). +const sectorSize = 65536 + +type memVFS struct{} + +func (memVFS) Open(name string, flags vfs.OpenFlag) (vfs.File, vfs.OpenFlag, error) { + // For simplicity, we do not support reading or writing data + // across "sector" boundaries. + // + // This is not a problem for most SQLite file types: + // - databases, which only do page aligned reads/writes; + // - temp journals, as used by the sorter, which does the same: + // https://github.com/sqlite/sqlite/blob/b74eb0/src/vdbesort.c#L409-L412 + // + // We refuse to open all other file types, + // but returning OPEN_MEMORY means SQLite won't ask us to. + const types = vfs.OPEN_MAIN_DB | + vfs.OPEN_TEMP_DB | + vfs.OPEN_TEMP_JOURNAL + if flags&types == 0 { + return nil, flags, sqlite3.CANTOPEN + } + + // A shared database has a name that begins with "/". + shared := len(name) > 1 && name[0] == '/' + + var db *memDB + if shared { + name = name[1:] + memoryMtx.Lock() + defer memoryMtx.Unlock() + db = memoryDBs[name] + } + if db == nil { + if flags&vfs.OPEN_CREATE == 0 { + return nil, flags, sqlite3.CANTOPEN + } + db = &memDB{name: name} + } + if shared { + db.refs++ // +checklocksforce: memoryMtx is held + memoryDBs[name] = db + } + + return &memFile{ + memDB: db, + readOnly: flags&vfs.OPEN_READONLY != 0, + }, flags | vfs.OPEN_MEMORY, nil +} + +func (memVFS) Delete(name string, dirSync bool) error { + return sqlite3.IOERR_DELETE +} + +func (memVFS) Access(name string, flag vfs.AccessFlag) (bool, error) { + return false, nil +} + +func (memVFS) FullPathname(name string) (string, error) { + return name, nil +} + +type memDB struct { + name string + + // +checklocks:lockMtx + pending *memFile + // +checklocks:lockMtx + reserved *memFile + + // +checklocks:dataMtx + data []*[sectorSize]byte + + // +checklocks:dataMtx + size int64 + + // +checklocks:lockMtx + shared int + + // +checklocks:memoryMtx + refs int + + lockMtx sync.Mutex + dataMtx sync.RWMutex +} + +func (m *memDB) release() { + memoryMtx.Lock() + defer memoryMtx.Unlock() + if m.refs--; m.refs == 0 && m == memoryDBs[m.name] { + delete(memoryDBs, m.name) + } +} + +type memFile struct { + *memDB + lock vfs.LockLevel + readOnly bool +} + +var ( + // Ensure these interfaces are implemented: + _ vfs.FileLockState = &memFile{} + _ vfs.FileSizeHint = &memFile{} +) + +func (m *memFile) Close() error { + m.release() + return m.Unlock(vfs.LOCK_NONE) +} + +func (m *memFile) ReadAt(b []byte, off int64) (n int, err error) { + m.dataMtx.RLock() + defer m.dataMtx.RUnlock() + + if off >= m.size { + return 0, io.EOF + } + + base := off / sectorSize + rest := off % sectorSize + have := int64(sectorSize) + if base == int64(len(m.data))-1 { + have = modRoundUp(m.size, sectorSize) + } + n = copy(b, (*m.data[base])[rest:have]) + if n < len(b) { + // Assume reads are page aligned. + return 0, io.ErrNoProgress + } + return n, nil +} + +func (m *memFile) WriteAt(b []byte, off int64) (n int, err error) { + m.dataMtx.Lock() + defer m.dataMtx.Unlock() + + base := off / sectorSize + rest := off % sectorSize + for base >= int64(len(m.data)) { + m.data = append(m.data, new([sectorSize]byte)) + } + n = copy((*m.data[base])[rest:], b) + if n < len(b) { + // Assume writes are page aligned. + return n, io.ErrShortWrite + } + if size := off + int64(len(b)); size > m.size { + m.size = size + } + return n, nil +} + +func (m *memFile) Truncate(size int64) error { + m.dataMtx.Lock() + defer m.dataMtx.Unlock() + return m.truncate(size) +} + +// +checklocks:m.dataMtx +func (m *memFile) truncate(size int64) error { + if size < m.size { + base := size / sectorSize + rest := size % sectorSize + if rest != 0 { + clear((*m.data[base])[rest:]) + } + } + sectors := divRoundUp(size, sectorSize) + for sectors > int64(len(m.data)) { + m.data = append(m.data, new([sectorSize]byte)) + } + clear(m.data[sectors:]) + m.data = m.data[:sectors] + m.size = size + return nil +} + +func (m *memFile) Sync(flag vfs.SyncFlag) error { + return nil +} + +func (m *memFile) Size() (int64, error) { + m.dataMtx.RLock() + defer m.dataMtx.RUnlock() + return m.size, nil +} + +const spinWait = 25 * time.Microsecond + +func (m *memFile) Lock(lock vfs.LockLevel) error { + if m.lock >= lock { + return nil + } + + if m.readOnly && lock >= vfs.LOCK_RESERVED { + return sqlite3.IOERR_LOCK + } + + m.lockMtx.Lock() + defer m.lockMtx.Unlock() + + switch lock { + case vfs.LOCK_SHARED: + if m.pending != nil { + return sqlite3.BUSY + } + m.shared++ + + case vfs.LOCK_RESERVED: + if m.reserved != nil { + return sqlite3.BUSY + } + m.reserved = m + + case vfs.LOCK_EXCLUSIVE: + if m.lock < vfs.LOCK_PENDING { + if m.pending != nil { + return sqlite3.BUSY + } + m.lock = vfs.LOCK_PENDING + m.pending = m + } + + for before := time.Now(); m.shared > 1; { + if time.Since(before) > spinWait { + return sqlite3.BUSY + } + m.lockMtx.Unlock() + runtime.Gosched() + m.lockMtx.Lock() + } + } + + m.lock = lock + return nil +} + +func (m *memFile) Unlock(lock vfs.LockLevel) error { + if m.lock <= lock { + return nil + } + + m.lockMtx.Lock() + defer m.lockMtx.Unlock() + + if m.pending == m { + m.pending = nil + } + if m.reserved == m { + m.reserved = nil + } + if lock < vfs.LOCK_SHARED { + m.shared-- + } + m.lock = lock + return nil +} + +func (m *memFile) CheckReservedLock() (bool, error) { + if m.lock >= vfs.LOCK_RESERVED { + return true, nil + } + m.lockMtx.Lock() + defer m.lockMtx.Unlock() + return m.reserved != nil, nil +} + +func (m *memFile) SectorSize() int { + return sectorSize +} + +func (m *memFile) DeviceCharacteristics() vfs.DeviceCharacteristic { + return vfs.IOCAP_ATOMIC | + vfs.IOCAP_SEQUENTIAL | + vfs.IOCAP_SAFE_APPEND | + vfs.IOCAP_POWERSAFE_OVERWRITE +} + +func (m *memFile) SizeHint(size int64) error { + m.dataMtx.Lock() + defer m.dataMtx.Unlock() + if size > m.size { + return m.truncate(size) + } + return nil +} + +func (m *memFile) LockState() vfs.LockLevel { + return m.lock +} + +func divRoundUp(a, b int64) int64 { + return (a + b - 1) / b +} + +func modRoundUp(a, b int64) int64 { + return b - (b-a%b)%b +} diff --git a/vendor/github.com/ncruces/go-sqlite3/vfs/os_bsd.go b/vendor/github.com/ncruces/go-sqlite3/vfs/os_bsd.go new file mode 100644 index 000000000..48ac5c9c9 --- /dev/null +++ b/vendor/github.com/ncruces/go-sqlite3/vfs/os_bsd.go @@ -0,0 +1,33 @@ +//go:build (freebsd || openbsd || netbsd || dragonfly || illumos || sqlite3_flock) && !sqlite3_nosys + +package vfs + +import ( + "os" + "time" + + "golang.org/x/sys/unix" +) + +func osUnlock(file *os.File, start, len int64) _ErrorCode { + if start == 0 && len == 0 { + err := unix.Flock(int(file.Fd()), unix.LOCK_UN) + if err != nil { + return _IOERR_UNLOCK + } + } + return _OK +} + +func osLock(file *os.File, how int, def _ErrorCode) _ErrorCode { + err := unix.Flock(int(file.Fd()), how) + return osLockErrorCode(err, def) +} + +func osReadLock(file *os.File, _ /*start*/, _ /*len*/ int64, _ /*timeout*/ time.Duration) _ErrorCode { + return osLock(file, unix.LOCK_SH|unix.LOCK_NB, _IOERR_RDLOCK) +} + +func osWriteLock(file *os.File, _ /*start*/, _ /*len*/ int64, _ /*timeout*/ time.Duration) _ErrorCode { + return osLock(file, unix.LOCK_EX|unix.LOCK_NB, _IOERR_LOCK) +} diff --git a/vendor/github.com/ncruces/go-sqlite3/vfs/os_darwin.go b/vendor/github.com/ncruces/go-sqlite3/vfs/os_darwin.go new file mode 100644 index 000000000..8bfe96bb1 --- /dev/null +++ b/vendor/github.com/ncruces/go-sqlite3/vfs/os_darwin.go @@ -0,0 +1,95 @@ +//go:build !(sqlite3_flock || sqlite3_nosys) + +package vfs + +import ( + "io" + "os" + "time" + + "golang.org/x/sys/unix" +) + +const ( + // https://github.com/apple/darwin-xnu/blob/main/bsd/sys/fcntl.h + _F_OFD_SETLK = 90 + _F_OFD_SETLKW = 91 + _F_OFD_SETLKWTIMEOUT = 93 +) + +type flocktimeout_t struct { + fl unix.Flock_t + timeout unix.Timespec +} + +func osSync(file *os.File, fullsync, _ /*dataonly*/ bool) error { + if fullsync { + return file.Sync() + } + return unix.Fsync(int(file.Fd())) +} + +func osAllocate(file *os.File, size int64) error { + off, err := file.Seek(0, io.SeekEnd) + if err != nil { + return err + } + if size <= off { + return nil + } + + store := unix.Fstore_t{ + Flags: unix.F_ALLOCATEALL | unix.F_ALLOCATECONTIG, + Posmode: unix.F_PEOFPOSMODE, + Offset: 0, + Length: size - off, + } + + // Try to get a continuous chunk of disk space. + err = unix.FcntlFstore(file.Fd(), unix.F_PREALLOCATE, &store) + if err != nil { + // OK, perhaps we are too fragmented, allocate non-continuous. + store.Flags = unix.F_ALLOCATEALL + unix.FcntlFstore(file.Fd(), unix.F_PREALLOCATE, &store) + } + return file.Truncate(size) +} + +func osUnlock(file *os.File, start, len int64) _ErrorCode { + err := unix.FcntlFlock(file.Fd(), _F_OFD_SETLK, &unix.Flock_t{ + Type: unix.F_UNLCK, + Start: start, + Len: len, + }) + if err != nil { + return _IOERR_UNLOCK + } + return _OK +} + +func osLock(file *os.File, typ int16, start, len int64, timeout time.Duration, def _ErrorCode) _ErrorCode { + lock := flocktimeout_t{fl: unix.Flock_t{ + Type: typ, + Start: start, + Len: len, + }} + var err error + switch { + case timeout == 0: + err = unix.FcntlFlock(file.Fd(), _F_OFD_SETLK, &lock.fl) + case timeout < 0: + err = unix.FcntlFlock(file.Fd(), _F_OFD_SETLKW, &lock.fl) + default: + lock.timeout = unix.NsecToTimespec(int64(timeout / time.Nanosecond)) + err = unix.FcntlFlock(file.Fd(), _F_OFD_SETLKWTIMEOUT, &lock.fl) + } + return osLockErrorCode(err, def) +} + +func osReadLock(file *os.File, start, len int64, timeout time.Duration) _ErrorCode { + return osLock(file, unix.F_RDLCK, start, len, timeout, _IOERR_RDLOCK) +} + +func osWriteLock(file *os.File, start, len int64, timeout time.Duration) _ErrorCode { + return osLock(file, unix.F_WRLCK, start, len, timeout, _IOERR_LOCK) +} diff --git a/vendor/github.com/ncruces/go-sqlite3/vfs/os_f2fs_linux.go b/vendor/github.com/ncruces/go-sqlite3/vfs/os_f2fs_linux.go new file mode 100644 index 000000000..a9f0e333c --- /dev/null +++ b/vendor/github.com/ncruces/go-sqlite3/vfs/os_f2fs_linux.go @@ -0,0 +1,34 @@ +//go:build (amd64 || arm64 || riscv64) && !sqlite3_nosys + +package vfs + +import ( + "os" + + "golang.org/x/sys/unix" +) + +const ( + _F2FS_IOC_START_ATOMIC_WRITE = 62721 + _F2FS_IOC_COMMIT_ATOMIC_WRITE = 62722 + _F2FS_IOC_ABORT_ATOMIC_WRITE = 62725 + _F2FS_IOC_GET_FEATURES = 2147808524 + _F2FS_FEATURE_ATOMIC_WRITE = 4 +) + +func osBatchAtomic(file *os.File) bool { + flags, err := unix.IoctlGetInt(int(file.Fd()), _F2FS_IOC_GET_FEATURES) + return err == nil && flags&_F2FS_FEATURE_ATOMIC_WRITE != 0 +} + +func (f *vfsFile) BeginAtomicWrite() error { + return unix.IoctlSetInt(int(f.Fd()), _F2FS_IOC_START_ATOMIC_WRITE, 0) +} + +func (f *vfsFile) CommitAtomicWrite() error { + return unix.IoctlSetInt(int(f.Fd()), _F2FS_IOC_COMMIT_ATOMIC_WRITE, 0) +} + +func (f *vfsFile) RollbackAtomicWrite() error { + return unix.IoctlSetInt(int(f.Fd()), _F2FS_IOC_ABORT_ATOMIC_WRITE, 0) +} diff --git a/vendor/github.com/ncruces/go-sqlite3/vfs/os_linux.go b/vendor/github.com/ncruces/go-sqlite3/vfs/os_linux.go new file mode 100644 index 000000000..11e683a04 --- /dev/null +++ b/vendor/github.com/ncruces/go-sqlite3/vfs/os_linux.go @@ -0,0 +1,71 @@ +//go:build !(sqlite3_flock || sqlite3_nosys) + +package vfs + +import ( + "math/rand" + "os" + "time" + + "golang.org/x/sys/unix" +) + +func osSync(file *os.File, _ /*fullsync*/, _ /*dataonly*/ bool) error { + // SQLite trusts Linux's fdatasync for all fsync's. + return unix.Fdatasync(int(file.Fd())) +} + +func osAllocate(file *os.File, size int64) error { + if size == 0 { + return nil + } + return unix.Fallocate(int(file.Fd()), 0, 0, size) +} + +func osUnlock(file *os.File, start, len int64) _ErrorCode { + err := unix.FcntlFlock(file.Fd(), unix.F_OFD_SETLK, &unix.Flock_t{ + Type: unix.F_UNLCK, + Start: start, + Len: len, + }) + if err != nil { + return _IOERR_UNLOCK + } + return _OK +} + +func osLock(file *os.File, typ int16, start, len int64, timeout time.Duration, def _ErrorCode) _ErrorCode { + lock := unix.Flock_t{ + Type: typ, + Start: start, + Len: len, + } + var err error + switch { + case timeout == 0: + err = unix.FcntlFlock(file.Fd(), unix.F_OFD_SETLK, &lock) + case timeout < 0: + err = unix.FcntlFlock(file.Fd(), unix.F_OFD_SETLKW, &lock) + default: + before := time.Now() + for { + err = unix.FcntlFlock(file.Fd(), unix.F_OFD_SETLK, &lock) + if errno, _ := err.(unix.Errno); errno != unix.EAGAIN { + break + } + if timeout < time.Since(before) { + break + } + osSleep(time.Duration(rand.Int63n(int64(time.Millisecond)))) + } + } + return osLockErrorCode(err, def) +} + +func osReadLock(file *os.File, start, len int64, timeout time.Duration) _ErrorCode { + return osLock(file, unix.F_RDLCK, start, len, timeout, _IOERR_RDLOCK) +} + +func osWriteLock(file *os.File, start, len int64, timeout time.Duration) _ErrorCode { + return osLock(file, unix.F_WRLCK, start, len, timeout, _IOERR_LOCK) +} diff --git a/vendor/github.com/ncruces/go-sqlite3/vfs/os_std_access.go b/vendor/github.com/ncruces/go-sqlite3/vfs/os_std_access.go new file mode 100644 index 000000000..1621c0998 --- /dev/null +++ b/vendor/github.com/ncruces/go-sqlite3/vfs/os_std_access.go @@ -0,0 +1,36 @@ +//go:build !unix || sqlite3_nosys + +package vfs + +import ( + "io/fs" + "os" +) + +func osAccess(path string, flags AccessFlag) error { + fi, err := os.Stat(path) + if err != nil { + return err + } + if flags == ACCESS_EXISTS { + return nil + } + + const ( + S_IREAD = 0400 + S_IWRITE = 0200 + S_IEXEC = 0100 + ) + + var want fs.FileMode = S_IREAD + if flags == ACCESS_READWRITE { + want |= S_IWRITE + } + if fi.IsDir() { + want |= S_IEXEC + } + if fi.Mode()&want != want { + return fs.ErrPermission + } + return nil +} diff --git a/vendor/github.com/ncruces/go-sqlite3/vfs/os_std_alloc.go b/vendor/github.com/ncruces/go-sqlite3/vfs/os_std_alloc.go new file mode 100644 index 000000000..60c92182c --- /dev/null +++ b/vendor/github.com/ncruces/go-sqlite3/vfs/os_std_alloc.go @@ -0,0 +1,19 @@ +//go:build !(linux || darwin) || sqlite3_flock || sqlite3_nosys + +package vfs + +import ( + "io" + "os" +) + +func osAllocate(file *os.File, size int64) error { + off, err := file.Seek(0, io.SeekEnd) + if err != nil { + return err + } + if size <= off { + return nil + } + return file.Truncate(size) +} diff --git a/vendor/github.com/ncruces/go-sqlite3/vfs/os_std_atomic.go b/vendor/github.com/ncruces/go-sqlite3/vfs/os_std_atomic.go new file mode 100644 index 000000000..ecaff0245 --- /dev/null +++ b/vendor/github.com/ncruces/go-sqlite3/vfs/os_std_atomic.go @@ -0,0 +1,9 @@ +//go:build !linux || !(amd64 || arm64 || riscv64) || sqlite3_nosys + +package vfs + +import "os" + +func osBatchAtomic(*os.File) bool { + return false +} diff --git a/vendor/github.com/ncruces/go-sqlite3/vfs/os_std_mode.go b/vendor/github.com/ncruces/go-sqlite3/vfs/os_std_mode.go new file mode 100644 index 000000000..ac4904773 --- /dev/null +++ b/vendor/github.com/ncruces/go-sqlite3/vfs/os_std_mode.go @@ -0,0 +1,14 @@ +//go:build !unix || sqlite3_nosys + +package vfs + +import "os" + +func osSetMode(file *os.File, modeof string) error { + fi, err := os.Stat(modeof) + if err != nil { + return err + } + file.Chmod(fi.Mode()) + return nil +} diff --git a/vendor/github.com/ncruces/go-sqlite3/vfs/os_std_sleep.go b/vendor/github.com/ncruces/go-sqlite3/vfs/os_std_sleep.go new file mode 100644 index 000000000..c6bc40769 --- /dev/null +++ b/vendor/github.com/ncruces/go-sqlite3/vfs/os_std_sleep.go @@ -0,0 +1,9 @@ +//go:build !windows || sqlite3_nosys + +package vfs + +import "time" + +func osSleep(d time.Duration) { + time.Sleep(d) +} diff --git a/vendor/github.com/ncruces/go-sqlite3/vfs/os_std_sync.go b/vendor/github.com/ncruces/go-sqlite3/vfs/os_std_sync.go new file mode 100644 index 000000000..84dbd23bc --- /dev/null +++ b/vendor/github.com/ncruces/go-sqlite3/vfs/os_std_sync.go @@ -0,0 +1,9 @@ +//go:build !(linux || darwin) || sqlite3_flock || sqlite3_nosys + +package vfs + +import "os" + +func osSync(file *os.File, _ /*fullsync*/, _ /*dataonly*/ bool) error { + return file.Sync() +} diff --git a/vendor/github.com/ncruces/go-sqlite3/vfs/os_unix.go b/vendor/github.com/ncruces/go-sqlite3/vfs/os_unix.go new file mode 100644 index 000000000..bf4b44efd --- /dev/null +++ b/vendor/github.com/ncruces/go-sqlite3/vfs/os_unix.go @@ -0,0 +1,33 @@ +//go:build unix && !sqlite3_nosys + +package vfs + +import ( + "os" + "syscall" + + "golang.org/x/sys/unix" +) + +func osAccess(path string, flags AccessFlag) error { + var access uint32 // unix.F_OK + switch flags { + case ACCESS_READWRITE: + access = unix.R_OK | unix.W_OK + case ACCESS_READ: + access = unix.R_OK + } + return unix.Access(path, access) +} + +func osSetMode(file *os.File, modeof string) error { + fi, err := os.Stat(modeof) + if err != nil { + return err + } + file.Chmod(fi.Mode()) + if sys, ok := fi.Sys().(*syscall.Stat_t); ok { + file.Chown(int(sys.Uid), int(sys.Gid)) + } + return nil +} diff --git a/vendor/github.com/ncruces/go-sqlite3/vfs/os_unix_lock.go b/vendor/github.com/ncruces/go-sqlite3/vfs/os_unix_lock.go new file mode 100644 index 000000000..d04c1f6a0 --- /dev/null +++ b/vendor/github.com/ncruces/go-sqlite3/vfs/os_unix_lock.go @@ -0,0 +1,106 @@ +//go:build (linux || darwin || freebsd || openbsd || netbsd || dragonfly || illumos || sqlite3_flock) && !sqlite3_nosys + +package vfs + +import ( + "os" + "time" + + "golang.org/x/sys/unix" +) + +func osGetSharedLock(file *os.File) _ErrorCode { + // Test the PENDING lock before acquiring a new SHARED lock. + if lock, _ := osGetLock(file, _PENDING_BYTE, 1); lock == unix.F_WRLCK { + return _BUSY + } + // Acquire the SHARED lock. + return osReadLock(file, _SHARED_FIRST, _SHARED_SIZE, 0) +} + +func osGetReservedLock(file *os.File) _ErrorCode { + // Acquire the RESERVED lock. + return osWriteLock(file, _RESERVED_BYTE, 1, 0) +} + +func osGetPendingLock(file *os.File, block bool) _ErrorCode { + var timeout time.Duration + if block { + timeout = -1 + } + // Acquire the PENDING lock. + return osWriteLock(file, _PENDING_BYTE, 1, timeout) +} + +func osGetExclusiveLock(file *os.File, wait bool) _ErrorCode { + var timeout time.Duration + if wait { + timeout = time.Millisecond + } + // Acquire the EXCLUSIVE lock. + return osWriteLock(file, _SHARED_FIRST, _SHARED_SIZE, timeout) +} + +func osDowngradeLock(file *os.File, state LockLevel) _ErrorCode { + if state >= LOCK_EXCLUSIVE { + // Downgrade to a SHARED lock. + if rc := osReadLock(file, _SHARED_FIRST, _SHARED_SIZE, 0); rc != _OK { + // In theory, the downgrade to a SHARED cannot fail because another + // process is holding an incompatible lock. If it does, this + // indicates that the other process is not following the locking + // protocol. If this happens, return _IOERR_RDLOCK. Returning + // BUSY would confuse the upper layer. + return _IOERR_RDLOCK + } + } + // Release the PENDING and RESERVED locks. + return osUnlock(file, _PENDING_BYTE, 2) +} + +func osReleaseLock(file *os.File, _ LockLevel) _ErrorCode { + // Release all locks. + return osUnlock(file, 0, 0) +} + +func osCheckReservedLock(file *os.File) (bool, _ErrorCode) { + // Test the RESERVED lock. + lock, rc := osGetLock(file, _RESERVED_BYTE, 1) + return lock == unix.F_WRLCK, rc +} + +func osGetLock(file *os.File, start, len int64) (int16, _ErrorCode) { + lock := unix.Flock_t{ + Type: unix.F_WRLCK, + Start: start, + Len: len, + } + if unix.FcntlFlock(file.Fd(), unix.F_GETLK, &lock) != nil { + return 0, _IOERR_CHECKRESERVEDLOCK + } + return lock.Type, _OK +} + +func osLockErrorCode(err error, def _ErrorCode) _ErrorCode { + if err == nil { + return _OK + } + if errno, ok := err.(unix.Errno); ok { + switch errno { + case + unix.EACCES, + unix.EAGAIN, + unix.EBUSY, + unix.EINTR, + unix.ENOLCK, + unix.EDEADLK, + unix.ETIMEDOUT: + return _BUSY + case unix.EPERM: + return _PERM + } + if errno == unix.EWOULDBLOCK && unix.EWOULDBLOCK != unix.EAGAIN { + return _BUSY + } + } + return def +} diff --git a/vendor/github.com/ncruces/go-sqlite3/vfs/os_windows.go b/vendor/github.com/ncruces/go-sqlite3/vfs/os_windows.go new file mode 100644 index 000000000..5c68754f8 --- /dev/null +++ b/vendor/github.com/ncruces/go-sqlite3/vfs/os_windows.go @@ -0,0 +1,186 @@ +//go:build !sqlite3_nosys + +package vfs + +import ( + "math/rand" + "os" + "time" + + "golang.org/x/sys/windows" +) + +func osGetSharedLock(file *os.File) _ErrorCode { + // Acquire the PENDING lock temporarily before acquiring a new SHARED lock. + rc := osReadLock(file, _PENDING_BYTE, 1, 0) + if rc == _OK { + // Acquire the SHARED lock. + rc = osReadLock(file, _SHARED_FIRST, _SHARED_SIZE, 0) + + // Release the PENDING lock. + osUnlock(file, _PENDING_BYTE, 1) + } + return rc +} + +func osGetReservedLock(file *os.File) _ErrorCode { + // Acquire the RESERVED lock. + return osWriteLock(file, _RESERVED_BYTE, 1, 0) +} + +func osGetPendingLock(file *os.File, block bool) _ErrorCode { + var timeout time.Duration + if block { + timeout = -1 + } + + // Acquire the PENDING lock. + return osWriteLock(file, _PENDING_BYTE, 1, timeout) +} + +func osGetExclusiveLock(file *os.File, wait bool) _ErrorCode { + var timeout time.Duration + if wait { + timeout = time.Millisecond + } + + // Release the SHARED lock. + osUnlock(file, _SHARED_FIRST, _SHARED_SIZE) + + // Acquire the EXCLUSIVE lock. + rc := osWriteLock(file, _SHARED_FIRST, _SHARED_SIZE, timeout) + + if rc != _OK { + // Reacquire the SHARED lock. + osReadLock(file, _SHARED_FIRST, _SHARED_SIZE, 0) + } + return rc +} + +func osDowngradeLock(file *os.File, state LockLevel) _ErrorCode { + if state >= LOCK_EXCLUSIVE { + // Release the EXCLUSIVE lock. + osUnlock(file, _SHARED_FIRST, _SHARED_SIZE) + + // Reacquire the SHARED lock. + if rc := osReadLock(file, _SHARED_FIRST, _SHARED_SIZE, 0); rc != _OK { + // This should never happen. + // We should always be able to reacquire the read lock. + return _IOERR_RDLOCK + } + } + + // Release the PENDING and RESERVED locks. + if state >= LOCK_RESERVED { + osUnlock(file, _RESERVED_BYTE, 1) + } + if state >= LOCK_PENDING { + osUnlock(file, _PENDING_BYTE, 1) + } + return _OK +} + +func osReleaseLock(file *os.File, state LockLevel) _ErrorCode { + // Release all locks. + if state >= LOCK_RESERVED { + osUnlock(file, _RESERVED_BYTE, 1) + } + if state >= LOCK_SHARED { + osUnlock(file, _SHARED_FIRST, _SHARED_SIZE) + } + if state >= LOCK_PENDING { + osUnlock(file, _PENDING_BYTE, 1) + } + return _OK +} + +func osCheckReservedLock(file *os.File) (bool, _ErrorCode) { + // Test the RESERVED lock. + rc := osLock(file, 0, _RESERVED_BYTE, 1, 0, _IOERR_CHECKRESERVEDLOCK) + if rc == _BUSY { + return true, _OK + } + if rc == _OK { + // Release the RESERVED lock. + osUnlock(file, _RESERVED_BYTE, 1) + } + return false, rc +} + +func osUnlock(file *os.File, start, len uint32) _ErrorCode { + err := windows.UnlockFileEx(windows.Handle(file.Fd()), + 0, len, 0, &windows.Overlapped{Offset: start}) + if err == windows.ERROR_NOT_LOCKED { + return _OK + } + if err != nil { + return _IOERR_UNLOCK + } + return _OK +} + +func osLock(file *os.File, flags, start, len uint32, timeout time.Duration, def _ErrorCode) _ErrorCode { + var err error + switch { + case timeout == 0: + err = osLockEx(file, flags|windows.LOCKFILE_FAIL_IMMEDIATELY, start, len) + case timeout < 0: + err = osLockEx(file, flags, start, len) + default: + before := time.Now() + for { + err = osLockEx(file, flags|windows.LOCKFILE_FAIL_IMMEDIATELY, start, len) + if errno, _ := err.(windows.Errno); errno != windows.ERROR_LOCK_VIOLATION { + break + } + if timeout < time.Since(before) { + break + } + osSleep(time.Duration(rand.Int63n(int64(time.Millisecond)))) + } + } + return osLockErrorCode(err, def) +} + +func osLockEx(file *os.File, flags, start, len uint32) error { + return windows.LockFileEx(windows.Handle(file.Fd()), flags, + 0, len, 0, &windows.Overlapped{Offset: start}) +} + +func osReadLock(file *os.File, start, len uint32, timeout time.Duration) _ErrorCode { + return osLock(file, 0, start, len, timeout, _IOERR_RDLOCK) +} + +func osWriteLock(file *os.File, start, len uint32, timeout time.Duration) _ErrorCode { + return osLock(file, windows.LOCKFILE_EXCLUSIVE_LOCK, start, len, timeout, _IOERR_LOCK) +} + +func osLockErrorCode(err error, def _ErrorCode) _ErrorCode { + if err == nil { + return _OK + } + if errno, ok := err.(windows.Errno); ok { + // https://devblogs.microsoft.com/oldnewthing/20140905-00/?p=63 + switch errno { + case + windows.ERROR_LOCK_VIOLATION, + windows.ERROR_IO_PENDING, + windows.ERROR_OPERATION_ABORTED: + return _BUSY + } + } + return def +} + +func osSleep(d time.Duration) { + if d > 0 { + period := max(1, d/(5*time.Millisecond)) + if period < 16 { + windows.TimeBeginPeriod(uint32(period)) + } + time.Sleep(d) + if period < 16 { + windows.TimeEndPeriod(uint32(period)) + } + } +} diff --git a/vendor/github.com/ncruces/go-sqlite3/vfs/registry.go b/vendor/github.com/ncruces/go-sqlite3/vfs/registry.go new file mode 100644 index 000000000..42a2106fb --- /dev/null +++ b/vendor/github.com/ncruces/go-sqlite3/vfs/registry.go @@ -0,0 +1,48 @@ +package vfs + +import "sync" + +var ( + // +checklocks:vfsRegistryMtx + vfsRegistry map[string]VFS + vfsRegistryMtx sync.RWMutex +) + +// Find returns a VFS given its name. +// If there is no match, nil is returned. +// If name is empty, the default VFS is returned. +// +// https://sqlite.org/c3ref/vfs_find.html +func Find(name string) VFS { + if name == "" || name == "os" { + return vfsOS{} + } + vfsRegistryMtx.RLock() + defer vfsRegistryMtx.RUnlock() + return vfsRegistry[name] +} + +// Register registers a VFS. +// Empty and "os" are reserved names. +// +// https://sqlite.org/c3ref/vfs_find.html +func Register(name string, vfs VFS) { + if name == "" || name == "os" { + return + } + vfsRegistryMtx.Lock() + defer vfsRegistryMtx.Unlock() + if vfsRegistry == nil { + vfsRegistry = map[string]VFS{} + } + vfsRegistry[name] = vfs +} + +// Unregister unregisters a VFS. +// +// https://sqlite.org/c3ref/vfs_find.html +func Unregister(name string) { + vfsRegistryMtx.Lock() + defer vfsRegistryMtx.Unlock() + delete(vfsRegistry, name) +} diff --git a/vendor/github.com/ncruces/go-sqlite3/vfs/shm.go b/vendor/github.com/ncruces/go-sqlite3/vfs/shm.go new file mode 100644 index 000000000..2b76dd5dc --- /dev/null +++ b/vendor/github.com/ncruces/go-sqlite3/vfs/shm.go @@ -0,0 +1,173 @@ +//go:build (darwin || linux) && (amd64 || arm64 || riscv64) && !(sqlite3_flock || sqlite3_noshm || sqlite3_nosys) + +package vfs + +import ( + "context" + "io" + "os" + + "github.com/ncruces/go-sqlite3/internal/util" + "github.com/tetratelabs/wazero/api" + "golang.org/x/sys/unix" +) + +// SupportsSharedMemory is false on platforms that do not support shared memory. +// To use [WAL without shared-memory], you need to set [EXCLUSIVE locking mode]. +// +// [WAL without shared-memory]: https://sqlite.org/wal.html#noshm +// [EXCLUSIVE locking mode]: https://sqlite.org/pragma.html#pragma_locking_mode +const SupportsSharedMemory = true + +const ( + _SHM_NLOCK = 8 + _SHM_BASE = 120 + _SHM_DMS = _SHM_BASE + _SHM_NLOCK +) + +func (f *vfsFile) SharedMemory() SharedMemory { return f.shm } + +// NewSharedMemory returns a shared-memory WAL-index +// backed by a file with the given path. +// It will return nil if shared-memory is not supported, +// or not appropriate for the given flags. +// Only [OPEN_MAIN_DB] databases may need a WAL-index. +// You must ensure all concurrent accesses to a database +// use shared-memory instances created with the same path. +func NewSharedMemory(path string, flags OpenFlag) SharedMemory { + if flags&OPEN_MAIN_DB == 0 || flags&(OPEN_DELETEONCLOSE|OPEN_MEMORY) != 0 { + return nil + } + return &vfsShm{ + path: path, + readOnly: flags&OPEN_READONLY != 0, + } +} + +type vfsShm struct { + *os.File + path string + regions []*util.MappedRegion + readOnly bool +} + +func (s *vfsShm) shmMap(ctx context.Context, mod api.Module, id, size int32, extend bool) (uint32, error) { + // Ensure size is a multiple of the OS page size. + if int(size)&(unix.Getpagesize()-1) != 0 { + return 0, _IOERR_SHMMAP + } + + if s.File == nil { + var flag int + if s.readOnly { + flag = unix.O_RDONLY + } else { + flag = unix.O_RDWR + } + f, err := os.OpenFile(s.path, + flag|unix.O_CREAT|unix.O_NOFOLLOW, 0666) + if err != nil { + return 0, _CANTOPEN + } + s.File = f + } + + // Dead man's switch. + if lock, rc := osGetLock(s.File, _SHM_DMS, 1); rc != _OK { + return 0, _IOERR_LOCK + } else if lock == unix.F_WRLCK { + return 0, _BUSY + } else if lock == unix.F_UNLCK { + if s.readOnly { + return 0, _READONLY_CANTINIT + } + if rc := osWriteLock(s.File, _SHM_DMS, 1, 0); rc != _OK { + return 0, rc + } + if err := s.Truncate(0); err != nil { + return 0, _IOERR_SHMOPEN + } + } + if rc := osReadLock(s.File, _SHM_DMS, 1, 0); rc != _OK { + return 0, rc + } + + // Check if file is big enough. + o, err := s.Seek(0, io.SeekEnd) + if err != nil { + return 0, _IOERR_SHMSIZE + } + if n := (int64(id) + 1) * int64(size); n > o { + if !extend { + return 0, nil + } + err := osAllocate(s.File, n) + if err != nil { + return 0, _IOERR_SHMSIZE + } + } + + var prot int + if s.readOnly { + prot = unix.PROT_READ + } else { + prot = unix.PROT_READ | unix.PROT_WRITE + } + r, err := util.MapRegion(ctx, mod, s.File, int64(id)*int64(size), size, prot) + if err != nil { + return 0, err + } + s.regions = append(s.regions, r) + return r.Ptr, nil +} + +func (s *vfsShm) shmLock(offset, n int32, flags _ShmFlag) error { + // Argument check. + if n <= 0 || offset < 0 || offset+n > _SHM_NLOCK { + panic(util.AssertErr()) + } + switch flags { + case + _SHM_LOCK | _SHM_SHARED, + _SHM_LOCK | _SHM_EXCLUSIVE, + _SHM_UNLOCK | _SHM_SHARED, + _SHM_UNLOCK | _SHM_EXCLUSIVE: + // + default: + panic(util.AssertErr()) + } + if n != 1 && flags&_SHM_EXCLUSIVE == 0 { + panic(util.AssertErr()) + } + + switch { + case flags&_SHM_UNLOCK != 0: + return osUnlock(s.File, _SHM_BASE+int64(offset), int64(n)) + case flags&_SHM_SHARED != 0: + return osReadLock(s.File, _SHM_BASE+int64(offset), int64(n), 0) + case flags&_SHM_EXCLUSIVE != 0: + return osWriteLock(s.File, _SHM_BASE+int64(offset), int64(n), 0) + default: + panic(util.AssertErr()) + } +} + +func (s *vfsShm) shmUnmap(delete bool) { + if s.File == nil { + return + } + + // Unmap regions. + for _, r := range s.regions { + r.Unmap() + } + clear(s.regions) + s.regions = s.regions[:0] + + // Close the file. + defer s.Close() + if delete { + os.Remove(s.Name()) + } + s.File = nil +} diff --git a/vendor/github.com/ncruces/go-sqlite3/vfs/shm_other.go b/vendor/github.com/ncruces/go-sqlite3/vfs/shm_other.go new file mode 100644 index 000000000..21191979e --- /dev/null +++ b/vendor/github.com/ncruces/go-sqlite3/vfs/shm_other.go @@ -0,0 +1,21 @@ +//go:build !(darwin || linux) || !(amd64 || arm64 || riscv64) || sqlite3_flock || sqlite3_noshm || sqlite3_nosys + +package vfs + +// SupportsSharedMemory is false on platforms that do not support shared memory. +// To use [WAL without shared-memory], you need to set [EXCLUSIVE locking mode]. +// +// [WAL without shared-memory]: https://sqlite.org/wal.html#noshm +// [EXCLUSIVE locking mode]: https://sqlite.org/pragma.html#pragma_locking_mode +const SupportsSharedMemory = false + +// NewSharedMemory returns a shared-memory WAL-index +// backed by a file with the given path. +// It will return nil if shared-memory is not supported, +// or not appropriate for the given flags. +// Only [OPEN_MAIN_DB] databases may need a WAL-index. +// You must ensure all concurrent accesses to a database +// use shared-memory instances created with the same path. +func NewSharedMemory(path string, flags OpenFlag) SharedMemory { + return nil +} diff --git a/vendor/github.com/ncruces/go-sqlite3/vfs/vfs.go b/vendor/github.com/ncruces/go-sqlite3/vfs/vfs.go new file mode 100644 index 000000000..1887e9f22 --- /dev/null +++ b/vendor/github.com/ncruces/go-sqlite3/vfs/vfs.go @@ -0,0 +1,459 @@ +package vfs + +import ( + "context" + "crypto/rand" + "io" + "reflect" + "sync" + "time" + + "github.com/ncruces/go-sqlite3/internal/util" + "github.com/ncruces/julianday" + "github.com/tetratelabs/wazero" + "github.com/tetratelabs/wazero/api" +) + +// ExportHostFunctions is an internal API users need not call directly. +// +// ExportHostFunctions registers the required VFS host functions +// with the provided env module. +func ExportHostFunctions(env wazero.HostModuleBuilder) wazero.HostModuleBuilder { + util.ExportFuncII(env, "go_vfs_find", vfsFind) + util.ExportFuncIIJ(env, "go_localtime", vfsLocaltime) + util.ExportFuncIIII(env, "go_randomness", vfsRandomness) + util.ExportFuncIII(env, "go_sleep", vfsSleep) + util.ExportFuncIII(env, "go_current_time_64", vfsCurrentTime64) + util.ExportFuncIIIII(env, "go_full_pathname", vfsFullPathname) + util.ExportFuncIIII(env, "go_delete", vfsDelete) + util.ExportFuncIIIII(env, "go_access", vfsAccess) + util.ExportFuncIIIIIII(env, "go_open", vfsOpen) + util.ExportFuncII(env, "go_close", vfsClose) + util.ExportFuncIIIIJ(env, "go_read", vfsRead) + util.ExportFuncIIIIJ(env, "go_write", vfsWrite) + util.ExportFuncIIJ(env, "go_truncate", vfsTruncate) + util.ExportFuncIII(env, "go_sync", vfsSync) + util.ExportFuncIII(env, "go_file_size", vfsFileSize) + util.ExportFuncIIII(env, "go_file_control", vfsFileControl) + util.ExportFuncII(env, "go_sector_size", vfsSectorSize) + util.ExportFuncII(env, "go_device_characteristics", vfsDeviceCharacteristics) + util.ExportFuncIII(env, "go_lock", vfsLock) + util.ExportFuncIII(env, "go_unlock", vfsUnlock) + util.ExportFuncIII(env, "go_check_reserved_lock", vfsCheckReservedLock) + util.ExportFuncIIIIII(env, "go_shm_map", vfsShmMap) + util.ExportFuncIIIII(env, "go_shm_lock", vfsShmLock) + util.ExportFuncIII(env, "go_shm_unmap", vfsShmUnmap) + util.ExportFuncVI(env, "go_shm_barrier", vfsShmBarrier) + return env +} + +func vfsFind(ctx context.Context, mod api.Module, zVfsName uint32) uint32 { + name := util.ReadString(mod, zVfsName, _MAX_NAME) + if vfs := Find(name); vfs != nil && vfs != (vfsOS{}) { + return 1 + } + return 0 +} + +func vfsLocaltime(ctx context.Context, mod api.Module, pTm uint32, t int64) _ErrorCode { + tm := time.Unix(t, 0) + var isdst int + if tm.IsDST() { + isdst = 1 + } + + const size = 32 / 8 + // https://pubs.opengroup.org/onlinepubs/7908799/xsh/time.h.html + util.WriteUint32(mod, pTm+0*size, uint32(tm.Second())) + util.WriteUint32(mod, pTm+1*size, uint32(tm.Minute())) + util.WriteUint32(mod, pTm+2*size, uint32(tm.Hour())) + util.WriteUint32(mod, pTm+3*size, uint32(tm.Day())) + util.WriteUint32(mod, pTm+4*size, uint32(tm.Month()-time.January)) + util.WriteUint32(mod, pTm+5*size, uint32(tm.Year()-1900)) + util.WriteUint32(mod, pTm+6*size, uint32(tm.Weekday()-time.Sunday)) + util.WriteUint32(mod, pTm+7*size, uint32(tm.YearDay()-1)) + util.WriteUint32(mod, pTm+8*size, uint32(isdst)) + return _OK +} + +func vfsRandomness(ctx context.Context, mod api.Module, pVfs uint32, nByte int32, zByte uint32) uint32 { + mem := util.View(mod, zByte, uint64(nByte)) + n, _ := rand.Reader.Read(mem) + return uint32(n) +} + +func vfsSleep(ctx context.Context, mod api.Module, pVfs uint32, nMicro int32) _ErrorCode { + osSleep(time.Duration(nMicro) * time.Microsecond) + return _OK +} + +func vfsCurrentTime64(ctx context.Context, mod api.Module, pVfs, piNow uint32) _ErrorCode { + day, nsec := julianday.Date(time.Now()) + msec := day*86_400_000 + nsec/1_000_000 + util.WriteUint64(mod, piNow, uint64(msec)) + return _OK +} + +func vfsFullPathname(ctx context.Context, mod api.Module, pVfs, zRelative uint32, nFull int32, zFull uint32) _ErrorCode { + vfs := vfsGet(mod, pVfs) + path := util.ReadString(mod, zRelative, _MAX_PATHNAME) + + path, err := vfs.FullPathname(path) + + if len(path) >= int(nFull) { + return _CANTOPEN_FULLPATH + } + util.WriteString(mod, zFull, path) + + return vfsErrorCode(err, _CANTOPEN_FULLPATH) +} + +func vfsDelete(ctx context.Context, mod api.Module, pVfs, zPath, syncDir uint32) _ErrorCode { + vfs := vfsGet(mod, pVfs) + path := util.ReadString(mod, zPath, _MAX_PATHNAME) + + err := vfs.Delete(path, syncDir != 0) + return vfsErrorCode(err, _IOERR_DELETE) +} + +func vfsAccess(ctx context.Context, mod api.Module, pVfs, zPath uint32, flags AccessFlag, pResOut uint32) _ErrorCode { + vfs := vfsGet(mod, pVfs) + path := util.ReadString(mod, zPath, _MAX_PATHNAME) + + ok, err := vfs.Access(path, flags) + var res uint32 + if ok { + res = 1 + } + + util.WriteUint32(mod, pResOut, res) + return vfsErrorCode(err, _IOERR_ACCESS) +} + +func vfsOpen(ctx context.Context, mod api.Module, pVfs, zPath, pFile uint32, flags OpenFlag, pOutFlags, pOutVFS uint32) _ErrorCode { + vfs := vfsGet(mod, pVfs) + + var path string + if zPath != 0 { + path = util.ReadString(mod, zPath, _MAX_PATHNAME) + } + + var file File + var err error + if ffs, ok := vfs.(VFSFilename); ok { + name := OpenFilename(ctx, mod, zPath, flags) + file, flags, err = ffs.OpenFilename(name, flags) + } else { + file, flags, err = vfs.Open(path, flags) + } + if err != nil { + return vfsErrorCode(err, _CANTOPEN) + } + + if file, ok := file.(FilePowersafeOverwrite); ok { + name := OpenFilename(ctx, mod, zPath, flags) + if b, ok := util.ParseBool(name.URIParameter("psow")); ok { + file.SetPowersafeOverwrite(b) + } + } + if file, ok := file.(FileSharedMemory); ok && + pOutVFS != 0 && file.SharedMemory() != nil { + util.WriteUint32(mod, pOutVFS, 1) + } + if pOutFlags != 0 { + util.WriteUint32(mod, pOutFlags, uint32(flags)) + } + vfsFileRegister(ctx, mod, pFile, file) + return _OK +} + +func vfsClose(ctx context.Context, mod api.Module, pFile uint32) _ErrorCode { + err := vfsFileClose(ctx, mod, pFile) + if err != nil { + return vfsErrorCode(err, _IOERR_CLOSE) + } + return _OK +} + +func vfsRead(ctx context.Context, mod api.Module, pFile, zBuf uint32, iAmt int32, iOfst int64) _ErrorCode { + file := vfsFileGet(ctx, mod, pFile).(File) + buf := util.View(mod, zBuf, uint64(iAmt)) + + n, err := file.ReadAt(buf, iOfst) + if n == int(iAmt) { + return _OK + } + if err != io.EOF { + return vfsErrorCode(err, _IOERR_READ) + } + clear(buf[n:]) + return _IOERR_SHORT_READ +} + +func vfsWrite(ctx context.Context, mod api.Module, pFile, zBuf uint32, iAmt int32, iOfst int64) _ErrorCode { + file := vfsFileGet(ctx, mod, pFile).(File) + buf := util.View(mod, zBuf, uint64(iAmt)) + + _, err := file.WriteAt(buf, iOfst) + if err != nil { + return vfsErrorCode(err, _IOERR_WRITE) + } + return _OK +} + +func vfsTruncate(ctx context.Context, mod api.Module, pFile uint32, nByte int64) _ErrorCode { + file := vfsFileGet(ctx, mod, pFile).(File) + err := file.Truncate(nByte) + return vfsErrorCode(err, _IOERR_TRUNCATE) +} + +func vfsSync(ctx context.Context, mod api.Module, pFile uint32, flags SyncFlag) _ErrorCode { + file := vfsFileGet(ctx, mod, pFile).(File) + err := file.Sync(flags) + return vfsErrorCode(err, _IOERR_FSYNC) +} + +func vfsFileSize(ctx context.Context, mod api.Module, pFile, pSize uint32) _ErrorCode { + file := vfsFileGet(ctx, mod, pFile).(File) + size, err := file.Size() + util.WriteUint64(mod, pSize, uint64(size)) + return vfsErrorCode(err, _IOERR_SEEK) +} + +func vfsLock(ctx context.Context, mod api.Module, pFile uint32, eLock LockLevel) _ErrorCode { + file := vfsFileGet(ctx, mod, pFile).(File) + err := file.Lock(eLock) + return vfsErrorCode(err, _IOERR_LOCK) +} + +func vfsUnlock(ctx context.Context, mod api.Module, pFile uint32, eLock LockLevel) _ErrorCode { + file := vfsFileGet(ctx, mod, pFile).(File) + err := file.Unlock(eLock) + return vfsErrorCode(err, _IOERR_UNLOCK) +} + +func vfsCheckReservedLock(ctx context.Context, mod api.Module, pFile, pResOut uint32) _ErrorCode { + file := vfsFileGet(ctx, mod, pFile).(File) + locked, err := file.CheckReservedLock() + + var res uint32 + if locked { + res = 1 + } + + util.WriteUint32(mod, pResOut, res) + return vfsErrorCode(err, _IOERR_CHECKRESERVEDLOCK) +} + +func vfsFileControl(ctx context.Context, mod api.Module, pFile uint32, op _FcntlOpcode, pArg uint32) _ErrorCode { + file := vfsFileGet(ctx, mod, pFile).(File) + + switch op { + case _FCNTL_LOCKSTATE: + if file, ok := file.(FileLockState); ok { + util.WriteUint32(mod, pArg, uint32(file.LockState())) + return _OK + } + + case _FCNTL_PERSIST_WAL: + if file, ok := file.(FilePersistentWAL); ok { + if i := util.ReadUint32(mod, pArg); int32(i) >= 0 { + file.SetPersistentWAL(i != 0) + } else if file.PersistentWAL() { + util.WriteUint32(mod, pArg, 1) + } else { + util.WriteUint32(mod, pArg, 0) + } + return _OK + } + + case _FCNTL_POWERSAFE_OVERWRITE: + if file, ok := file.(FilePowersafeOverwrite); ok { + if i := util.ReadUint32(mod, pArg); int32(i) >= 0 { + file.SetPowersafeOverwrite(i != 0) + } else if file.PowersafeOverwrite() { + util.WriteUint32(mod, pArg, 1) + } else { + util.WriteUint32(mod, pArg, 0) + } + return _OK + } + + case _FCNTL_CHUNK_SIZE: + if file, ok := file.(FileChunkSize); ok { + size := util.ReadUint32(mod, pArg) + file.ChunkSize(int(size)) + return _OK + } + + case _FCNTL_SIZE_HINT: + if file, ok := file.(FileSizeHint); ok { + size := util.ReadUint64(mod, pArg) + err := file.SizeHint(int64(size)) + return vfsErrorCode(err, _IOERR_TRUNCATE) + } + + case _FCNTL_HAS_MOVED: + if file, ok := file.(FileHasMoved); ok { + moved, err := file.HasMoved() + var res uint32 + if moved { + res = 1 + } + util.WriteUint32(mod, pArg, res) + return vfsErrorCode(err, _IOERR_FSTAT) + } + + case _FCNTL_OVERWRITE: + if file, ok := file.(FileOverwrite); ok { + err := file.Overwrite() + return vfsErrorCode(err, _IOERR) + } + + case _FCNTL_COMMIT_PHASETWO: + if file, ok := file.(FileCommitPhaseTwo); ok { + err := file.CommitPhaseTwo() + return vfsErrorCode(err, _IOERR) + } + + case _FCNTL_BEGIN_ATOMIC_WRITE: + if file, ok := file.(FileBatchAtomicWrite); ok { + err := file.BeginAtomicWrite() + return vfsErrorCode(err, _IOERR_BEGIN_ATOMIC) + } + case _FCNTL_COMMIT_ATOMIC_WRITE: + if file, ok := file.(FileBatchAtomicWrite); ok { + err := file.CommitAtomicWrite() + return vfsErrorCode(err, _IOERR_COMMIT_ATOMIC) + } + case _FCNTL_ROLLBACK_ATOMIC_WRITE: + if file, ok := file.(FileBatchAtomicWrite); ok { + err := file.RollbackAtomicWrite() + return vfsErrorCode(err, _IOERR_ROLLBACK_ATOMIC) + } + + case _FCNTL_CKPT_DONE: + if file, ok := file.(FileCheckpoint); ok { + err := file.CheckpointDone() + return vfsErrorCode(err, _IOERR) + } + case _FCNTL_CKPT_START: + if file, ok := file.(FileCheckpoint); ok { + err := file.CheckpointStart() + return vfsErrorCode(err, _IOERR) + } + + case _FCNTL_PRAGMA: + if file, ok := file.(FilePragma); ok { + ptr := util.ReadUint32(mod, pArg+1*ptrlen) + name := util.ReadString(mod, ptr, _MAX_SQL_LENGTH) + var value string + if ptr := util.ReadUint32(mod, pArg+2*ptrlen); ptr != 0 { + value = util.ReadString(mod, ptr, _MAX_SQL_LENGTH) + } + + out, err := file.Pragma(name, value) + + ret := vfsErrorCode(err, _ERROR) + if ret == _ERROR { + out = err.Error() + } + if out != "" { + fn := mod.ExportedFunction("malloc") + stack := [...]uint64{uint64(len(out) + 1)} + if err := fn.CallWithStack(ctx, stack[:]); err != nil { + panic(err) + } + util.WriteUint32(mod, pArg, uint32(stack[0])) + util.WriteString(mod, uint32(stack[0]), out) + } + return ret + } + } + + // Consider also implementing these opcodes (in use by SQLite): + // _FCNTL_BUSYHANDLER + // _FCNTL_LAST_ERRNO + // _FCNTL_SYNC + return _NOTFOUND +} + +func vfsSectorSize(ctx context.Context, mod api.Module, pFile uint32) uint32 { + file := vfsFileGet(ctx, mod, pFile).(File) + return uint32(file.SectorSize()) +} + +func vfsDeviceCharacteristics(ctx context.Context, mod api.Module, pFile uint32) DeviceCharacteristic { + file := vfsFileGet(ctx, mod, pFile).(File) + return file.DeviceCharacteristics() +} + +var shmBarrier sync.Mutex + +func vfsShmBarrier(ctx context.Context, mod api.Module, pFile uint32) { + shmBarrier.Lock() + defer shmBarrier.Unlock() +} + +func vfsShmMap(ctx context.Context, mod api.Module, pFile uint32, iRegion, szRegion int32, bExtend, pp uint32) _ErrorCode { + shm := vfsFileGet(ctx, mod, pFile).(FileSharedMemory).SharedMemory() + p, err := shm.shmMap(ctx, mod, iRegion, szRegion, bExtend != 0) + if err != nil { + return vfsErrorCode(err, _IOERR_SHMMAP) + } + util.WriteUint32(mod, pp, p) + return _OK +} + +func vfsShmLock(ctx context.Context, mod api.Module, pFile uint32, offset, n int32, flags _ShmFlag) _ErrorCode { + shm := vfsFileGet(ctx, mod, pFile).(FileSharedMemory).SharedMemory() + err := shm.shmLock(offset, n, flags) + return vfsErrorCode(err, _IOERR_SHMLOCK) +} + +func vfsShmUnmap(ctx context.Context, mod api.Module, pFile, bDelete uint32) _ErrorCode { + shm := vfsFileGet(ctx, mod, pFile).(FileSharedMemory).SharedMemory() + shm.shmUnmap(bDelete != 0) + return _OK +} + +func vfsGet(mod api.Module, pVfs uint32) VFS { + var name string + if pVfs != 0 { + const zNameOffset = 16 + name = util.ReadString(mod, util.ReadUint32(mod, pVfs+zNameOffset), _MAX_NAME) + } + if vfs := Find(name); vfs != nil { + return vfs + } + panic(util.NoVFSErr + util.ErrorString(name)) +} + +func vfsFileRegister(ctx context.Context, mod api.Module, pFile uint32, file File) { + const fileHandleOffset = 4 + id := util.AddHandle(ctx, file) + util.WriteUint32(mod, pFile+fileHandleOffset, id) +} + +func vfsFileGet(ctx context.Context, mod api.Module, pFile uint32) any { + const fileHandleOffset = 4 + id := util.ReadUint32(mod, pFile+fileHandleOffset) + return util.GetHandle(ctx, id) +} + +func vfsFileClose(ctx context.Context, mod api.Module, pFile uint32) error { + const fileHandleOffset = 4 + id := util.ReadUint32(mod, pFile+fileHandleOffset) + return util.DelHandle(ctx, id) +} + +func vfsErrorCode(err error, def _ErrorCode) _ErrorCode { + if err == nil { + return _OK + } + switch v := reflect.ValueOf(err); v.Kind() { + case reflect.Uint8, reflect.Uint16, reflect.Uint32: + return _ErrorCode(v.Uint()) + } + return def +} diff --git a/vendor/github.com/ncruces/go-sqlite3/vtab.go b/vendor/github.com/ncruces/go-sqlite3/vtab.go new file mode 100644 index 000000000..a330c98ff --- /dev/null +++ b/vendor/github.com/ncruces/go-sqlite3/vtab.go @@ -0,0 +1,663 @@ +package sqlite3 + +import ( + "context" + "reflect" + + "github.com/ncruces/go-sqlite3/internal/util" + "github.com/tetratelabs/wazero/api" +) + +// CreateModule registers a new virtual table module name. +// If create is nil, the virtual table is eponymous. +// +// https://sqlite.org/c3ref/create_module.html +func CreateModule[T VTab](db *Conn, name string, create, connect VTabConstructor[T]) error { + var flags int + + const ( + VTAB_CREATOR = 0x01 + VTAB_DESTROYER = 0x02 + VTAB_UPDATER = 0x04 + VTAB_RENAMER = 0x08 + VTAB_OVERLOADER = 0x10 + VTAB_CHECKER = 0x20 + VTAB_TXN = 0x40 + VTAB_SAVEPOINTER = 0x80 + ) + + if create != nil { + flags |= VTAB_CREATOR + } + + vtab := reflect.TypeOf(connect).Out(0) + if implements[VTabDestroyer](vtab) { + flags |= VTAB_DESTROYER + } + if implements[VTabUpdater](vtab) { + flags |= VTAB_UPDATER + } + if implements[VTabRenamer](vtab) { + flags |= VTAB_RENAMER + } + if implements[VTabOverloader](vtab) { + flags |= VTAB_OVERLOADER + } + if implements[VTabChecker](vtab) { + flags |= VTAB_CHECKER + } + if implements[VTabTxn](vtab) { + flags |= VTAB_TXN + } + if implements[VTabSavepointer](vtab) { + flags |= VTAB_SAVEPOINTER + } + + defer db.arena.mark()() + namePtr := db.arena.string(name) + modulePtr := util.AddHandle(db.ctx, module[T]{create, connect}) + r := db.call("sqlite3_create_module_go", uint64(db.handle), + uint64(namePtr), uint64(flags), uint64(modulePtr)) + return db.error(r) +} + +func implements[T any](typ reflect.Type) bool { + var ptr *T + return typ.Implements(reflect.TypeOf(ptr).Elem()) +} + +// DeclareVTab declares the schema of a virtual table. +// +// https://sqlite.org/c3ref/declare_vtab.html +func (c *Conn) DeclareVTab(sql string) error { + defer c.arena.mark()() + sqlPtr := c.arena.string(sql) + r := c.call("sqlite3_declare_vtab", uint64(c.handle), uint64(sqlPtr)) + return c.error(r) +} + +// VTabConflictMode is a virtual table conflict resolution mode. +// +// https://sqlite.org/c3ref/c_fail.html +type VTabConflictMode uint8 + +const ( + VTAB_ROLLBACK VTabConflictMode = 1 + VTAB_IGNORE VTabConflictMode = 2 + VTAB_FAIL VTabConflictMode = 3 + VTAB_ABORT VTabConflictMode = 4 + VTAB_REPLACE VTabConflictMode = 5 +) + +// VTabOnConflict determines the virtual table conflict policy. +// +// https://sqlite.org/c3ref/vtab_on_conflict.html +func (c *Conn) VTabOnConflict() VTabConflictMode { + r := c.call("sqlite3_vtab_on_conflict", uint64(c.handle)) + return VTabConflictMode(r) +} + +// VTabConfigOption is a virtual table configuration option. +// +// https://sqlite.org/c3ref/c_vtab_constraint_support.html +type VTabConfigOption uint8 + +const ( + VTAB_CONSTRAINT_SUPPORT VTabConfigOption = 1 + VTAB_INNOCUOUS VTabConfigOption = 2 + VTAB_DIRECTONLY VTabConfigOption = 3 + VTAB_USES_ALL_SCHEMAS VTabConfigOption = 4 +) + +// VTabConfig configures various facets of the virtual table interface. +// +// https://sqlite.org/c3ref/vtab_config.html +func (c *Conn) VTabConfig(op VTabConfigOption, args ...any) error { + var i uint64 + if op == VTAB_CONSTRAINT_SUPPORT && len(args) > 0 { + if b, ok := args[0].(bool); ok && b { + i = 1 + } + } + r := c.call("sqlite3_vtab_config_go", uint64(c.handle), uint64(op), i) + return c.error(r) +} + +// VTabConstructor is a virtual table constructor function. +type VTabConstructor[T VTab] func(db *Conn, module, schema, table string, arg ...string) (T, error) + +type module[T VTab] [2]VTabConstructor[T] + +type vtabConstructor int + +const ( + xCreate vtabConstructor = 0 + xConnect vtabConstructor = 1 +) + +// A VTab describes a particular instance of the virtual table. +// A VTab may optionally implement [io.Closer] to free resources. +// +// https://sqlite.org/c3ref/vtab.html +type VTab interface { + // https://sqlite.org/vtab.html#xbestindex + BestIndex(*IndexInfo) error + // https://sqlite.org/vtab.html#xopen + Open() (VTabCursor, error) +} + +// A VTabDestroyer allows a virtual table to drop persistent state. +type VTabDestroyer interface { + VTab + // https://sqlite.org/vtab.html#sqlite3_module.xDestroy + Destroy() error +} + +// A VTabUpdater allows a virtual table to be updated. +type VTabUpdater interface { + VTab + // https://sqlite.org/vtab.html#xupdate + Update(arg ...Value) (rowid int64, err error) +} + +// A VTabRenamer allows a virtual table to be renamed. +type VTabRenamer interface { + VTab + // https://sqlite.org/vtab.html#xrename + Rename(new string) error +} + +// A VTabOverloader allows a virtual table to overload SQL functions. +type VTabOverloader interface { + VTab + // https://sqlite.org/vtab.html#xfindfunction + FindFunction(arg int, name string) (ScalarFunction, IndexConstraintOp) +} + +// A VTabChecker allows a virtual table to report errors +// to the PRAGMA integrity_check and PRAGMA quick_check commands. +// +// Integrity should return an error if it finds problems in the content of the virtual table, +// but should avoid returning a (wrapped) [Error], [ErrorCode] or [ExtendedErrorCode], +// as those indicate the Integrity method itself encountered problems +// while trying to evaluate the virtual table content. +type VTabChecker interface { + VTab + // https://sqlite.org/vtab.html#xintegrity + Integrity(schema, table string, flags int) error +} + +// A VTabTxn allows a virtual table to implement +// transactions with two-phase commit. +// +// Anything that is required as part of a commit that may fail +// should be performed in the Sync() callback. +// Current versions of SQLite ignore any errors +// returned by Commit() and Rollback(). +type VTabTxn interface { + VTab + // https://sqlite.org/vtab.html#xBegin + Begin() error + // https://sqlite.org/vtab.html#xsync + Sync() error + // https://sqlite.org/vtab.html#xcommit + Commit() error + // https://sqlite.org/vtab.html#xrollback + Rollback() error +} + +// A VTabSavepointer allows a virtual table to implement +// nested transactions. +// +// https://sqlite.org/vtab.html#xsavepoint +type VTabSavepointer interface { + VTabTxn + Savepoint(id int) error + Release(id int) error + RollbackTo(id int) error +} + +// A VTabCursor describes cursors that point +// into the virtual table and are used +// to loop through the virtual table. +// A VTabCursor may optionally implement +// [io.Closer] to free resources. +// +// http://sqlite.org/c3ref/vtab_cursor.html +type VTabCursor interface { + // https://sqlite.org/vtab.html#xfilter + Filter(idxNum int, idxStr string, arg ...Value) error + // https://sqlite.org/vtab.html#xnext + Next() error + // https://sqlite.org/vtab.html#xeof + EOF() bool + // https://sqlite.org/vtab.html#xcolumn + Column(ctx *Context, n int) error + // https://sqlite.org/vtab.html#xrowid + RowID() (int64, error) +} + +// An IndexInfo describes virtual table indexing information. +// +// https://sqlite.org/c3ref/index_info.html +type IndexInfo struct { + // Inputs + Constraint []IndexConstraint + OrderBy []IndexOrderBy + ColumnsUsed int64 + // Outputs + ConstraintUsage []IndexConstraintUsage + IdxNum int + IdxStr string + IdxFlags IndexScanFlag + OrderByConsumed bool + EstimatedCost float64 + EstimatedRows int64 + // Internal + c *Conn + handle uint32 +} + +// An IndexConstraint describes virtual table indexing constraint information. +// +// https://sqlite.org/c3ref/index_info.html +type IndexConstraint struct { + Column int + Op IndexConstraintOp + Usable bool +} + +// An IndexOrderBy describes virtual table indexing order by information. +// +// https://sqlite.org/c3ref/index_info.html +type IndexOrderBy struct { + Column int + Desc bool +} + +// An IndexConstraintUsage describes how virtual table indexing constraints will be used. +// +// https://sqlite.org/c3ref/index_info.html +type IndexConstraintUsage struct { + ArgvIndex int + Omit bool +} + +// RHSValue returns the value of the right-hand operand of a constraint +// if the right-hand operand is known. +// +// https://sqlite.org/c3ref/vtab_rhs_value.html +func (idx *IndexInfo) RHSValue(column int) (Value, error) { + defer idx.c.arena.mark()() + valPtr := idx.c.arena.new(ptrlen) + r := idx.c.call("sqlite3_vtab_rhs_value", uint64(idx.handle), + uint64(column), uint64(valPtr)) + if err := idx.c.error(r); err != nil { + return Value{}, err + } + return Value{ + c: idx.c, + handle: util.ReadUint32(idx.c.mod, valPtr), + }, nil +} + +// Collation returns the name of the collation for a virtual table constraint. +// +// https://sqlite.org/c3ref/vtab_collation.html +func (idx *IndexInfo) Collation(column int) string { + r := idx.c.call("sqlite3_vtab_collation", uint64(idx.handle), + uint64(column)) + return util.ReadString(idx.c.mod, uint32(r), _MAX_NAME) +} + +// Distinct determines if a virtual table query is DISTINCT. +// +// https://sqlite.org/c3ref/vtab_distinct.html +func (idx *IndexInfo) Distinct() int { + r := idx.c.call("sqlite3_vtab_distinct", uint64(idx.handle)) + return int(r) +} + +// In identifies and handles IN constraints. +// +// https://sqlite.org/c3ref/vtab_in.html +func (idx *IndexInfo) In(column, handle int) bool { + r := idx.c.call("sqlite3_vtab_in", uint64(idx.handle), + uint64(column), uint64(handle)) + return r != 0 +} + +func (idx *IndexInfo) load() { + // https://sqlite.org/c3ref/index_info.html + mod := idx.c.mod + ptr := idx.handle + + idx.Constraint = make([]IndexConstraint, util.ReadUint32(mod, ptr+0)) + idx.ConstraintUsage = make([]IndexConstraintUsage, util.ReadUint32(mod, ptr+0)) + idx.OrderBy = make([]IndexOrderBy, util.ReadUint32(mod, ptr+8)) + + constraintPtr := util.ReadUint32(mod, ptr+4) + for i := range idx.Constraint { + idx.Constraint[i] = IndexConstraint{ + Column: int(int32(util.ReadUint32(mod, constraintPtr+0))), + Op: IndexConstraintOp(util.ReadUint8(mod, constraintPtr+4)), + Usable: util.ReadUint8(mod, constraintPtr+5) != 0, + } + constraintPtr += 12 + } + + orderByPtr := util.ReadUint32(mod, ptr+12) + for i := range idx.OrderBy { + idx.OrderBy[i] = IndexOrderBy{ + Column: int(int32(util.ReadUint32(mod, orderByPtr+0))), + Desc: util.ReadUint8(mod, orderByPtr+4) != 0, + } + orderByPtr += 8 + } + + idx.EstimatedCost = util.ReadFloat64(mod, ptr+40) + idx.EstimatedRows = int64(util.ReadUint64(mod, ptr+48)) + idx.ColumnsUsed = int64(util.ReadUint64(mod, ptr+64)) +} + +func (idx *IndexInfo) save() { + // https://sqlite.org/c3ref/index_info.html + mod := idx.c.mod + ptr := idx.handle + + usagePtr := util.ReadUint32(mod, ptr+16) + for _, usage := range idx.ConstraintUsage { + util.WriteUint32(mod, usagePtr+0, uint32(usage.ArgvIndex)) + if usage.Omit { + util.WriteUint8(mod, usagePtr+4, 1) + } + usagePtr += 8 + } + + util.WriteUint32(mod, ptr+20, uint32(idx.IdxNum)) + if idx.IdxStr != "" { + util.WriteUint32(mod, ptr+24, idx.c.newString(idx.IdxStr)) + util.WriteUint32(mod, ptr+28, 1) // needToFreeIdxStr + } + if idx.OrderByConsumed { + util.WriteUint32(mod, ptr+32, 1) + } + util.WriteFloat64(mod, ptr+40, idx.EstimatedCost) + util.WriteUint64(mod, ptr+48, uint64(idx.EstimatedRows)) + util.WriteUint32(mod, ptr+56, uint32(idx.IdxFlags)) +} + +// IndexConstraintOp is a virtual table constraint operator code. +// +// https://sqlite.org/c3ref/c_index_constraint_eq.html +type IndexConstraintOp uint8 + +const ( + INDEX_CONSTRAINT_EQ IndexConstraintOp = 2 + INDEX_CONSTRAINT_GT IndexConstraintOp = 4 + INDEX_CONSTRAINT_LE IndexConstraintOp = 8 + INDEX_CONSTRAINT_LT IndexConstraintOp = 16 + INDEX_CONSTRAINT_GE IndexConstraintOp = 32 + INDEX_CONSTRAINT_MATCH IndexConstraintOp = 64 + INDEX_CONSTRAINT_LIKE IndexConstraintOp = 65 + INDEX_CONSTRAINT_GLOB IndexConstraintOp = 66 + INDEX_CONSTRAINT_REGEXP IndexConstraintOp = 67 + INDEX_CONSTRAINT_NE IndexConstraintOp = 68 + INDEX_CONSTRAINT_ISNOT IndexConstraintOp = 69 + INDEX_CONSTRAINT_ISNOTNULL IndexConstraintOp = 70 + INDEX_CONSTRAINT_ISNULL IndexConstraintOp = 71 + INDEX_CONSTRAINT_IS IndexConstraintOp = 72 + INDEX_CONSTRAINT_LIMIT IndexConstraintOp = 73 + INDEX_CONSTRAINT_OFFSET IndexConstraintOp = 74 + INDEX_CONSTRAINT_FUNCTION IndexConstraintOp = 150 +) + +// IndexScanFlag is a virtual table scan flag. +// +// https://sqlite.org/c3ref/c_index_scan_unique.html +type IndexScanFlag uint32 + +const ( + INDEX_SCAN_UNIQUE IndexScanFlag = 1 +) + +func vtabModuleCallback(i vtabConstructor) func(_ context.Context, _ api.Module, _, _, _, _, _ uint32) uint32 { + return func(ctx context.Context, mod api.Module, pMod, nArg, pArg, ppVTab, pzErr uint32) uint32 { + arg := make([]reflect.Value, 1+nArg) + arg[0] = reflect.ValueOf(ctx.Value(connKey{})) + + for i := uint32(0); i < nArg; i++ { + ptr := util.ReadUint32(mod, pArg+i*ptrlen) + arg[i+1] = reflect.ValueOf(util.ReadString(mod, ptr, _MAX_SQL_LENGTH)) + } + + module := vtabGetHandle(ctx, mod, pMod) + res := reflect.ValueOf(module).Index(int(i)).Call(arg) + err, _ := res[1].Interface().(error) + if err == nil { + vtabPutHandle(ctx, mod, ppVTab, res[0].Interface()) + } + + return vtabError(ctx, mod, pzErr, _PTR_ERROR, err) + } +} + +func vtabDisconnectCallback(ctx context.Context, mod api.Module, pVTab uint32) uint32 { + err := vtabDelHandle(ctx, mod, pVTab) + return vtabError(ctx, mod, 0, _PTR_ERROR, err) +} + +func vtabDestroyCallback(ctx context.Context, mod api.Module, pVTab uint32) uint32 { + vtab := vtabGetHandle(ctx, mod, pVTab).(VTabDestroyer) + err := vtab.Destroy() + if cerr := vtabDelHandle(ctx, mod, pVTab); err == nil { + err = cerr + } + return vtabError(ctx, mod, 0, _PTR_ERROR, err) +} + +func vtabBestIndexCallback(ctx context.Context, mod api.Module, pVTab, pIdxInfo uint32) uint32 { + var info IndexInfo + info.handle = pIdxInfo + info.c = ctx.Value(connKey{}).(*Conn) + info.load() + + vtab := vtabGetHandle(ctx, mod, pVTab).(VTab) + err := vtab.BestIndex(&info) + + info.save() + return vtabError(ctx, mod, pVTab, _VTAB_ERROR, err) +} + +func vtabUpdateCallback(ctx context.Context, mod api.Module, pVTab, nArg, pArg, pRowID uint32) uint32 { + vtab := vtabGetHandle(ctx, mod, pVTab).(VTabUpdater) + + db := ctx.Value(connKey{}).(*Conn) + args := make([]Value, nArg) + callbackArgs(db, args, pArg) + rowID, err := vtab.Update(args...) + if err == nil { + util.WriteUint64(mod, pRowID, uint64(rowID)) + } + + return vtabError(ctx, mod, pVTab, _VTAB_ERROR, err) +} + +func vtabRenameCallback(ctx context.Context, mod api.Module, pVTab, zNew uint32) uint32 { + vtab := vtabGetHandle(ctx, mod, pVTab).(VTabRenamer) + err := vtab.Rename(util.ReadString(mod, zNew, _MAX_NAME)) + return vtabError(ctx, mod, pVTab, _VTAB_ERROR, err) +} + +func vtabFindFuncCallback(ctx context.Context, mod api.Module, pVTab uint32, nArg int32, zName, pxFunc uint32) uint32 { + vtab := vtabGetHandle(ctx, mod, pVTab).(VTabOverloader) + f, op := vtab.FindFunction(int(nArg), util.ReadString(mod, zName, _MAX_NAME)) + if op != 0 { + var wrapper uint32 + wrapper = util.AddHandle(ctx, func(c Context, arg ...Value) { + defer util.DelHandle(ctx, wrapper) + f(c, arg...) + }) + util.WriteUint32(mod, pxFunc, wrapper) + } + return uint32(op) +} + +func vtabIntegrityCallback(ctx context.Context, mod api.Module, pVTab, zSchema, zTabName, mFlags, pzErr uint32) uint32 { + vtab := vtabGetHandle(ctx, mod, pVTab).(VTabChecker) + schema := util.ReadString(mod, zSchema, _MAX_NAME) + table := util.ReadString(mod, zTabName, _MAX_NAME) + err := vtab.Integrity(schema, table, int(mFlags)) + // xIntegrity should return OK - even if it finds problems in the content of the virtual table. + // https://sqlite.org/vtab.html#xintegrity + vtabError(ctx, mod, pzErr, _PTR_ERROR, err) + _, code := errorCode(err, _OK) + return code +} + +func vtabBeginCallback(ctx context.Context, mod api.Module, pVTab uint32) uint32 { + vtab := vtabGetHandle(ctx, mod, pVTab).(VTabTxn) + err := vtab.Begin() + return vtabError(ctx, mod, pVTab, _VTAB_ERROR, err) +} + +func vtabSyncCallback(ctx context.Context, mod api.Module, pVTab uint32) uint32 { + vtab := vtabGetHandle(ctx, mod, pVTab).(VTabTxn) + err := vtab.Sync() + return vtabError(ctx, mod, pVTab, _VTAB_ERROR, err) +} + +func vtabCommitCallback(ctx context.Context, mod api.Module, pVTab uint32) uint32 { + vtab := vtabGetHandle(ctx, mod, pVTab).(VTabTxn) + err := vtab.Commit() + return vtabError(ctx, mod, pVTab, _VTAB_ERROR, err) +} + +func vtabRollbackCallback(ctx context.Context, mod api.Module, pVTab uint32) uint32 { + vtab := vtabGetHandle(ctx, mod, pVTab).(VTabTxn) + err := vtab.Rollback() + return vtabError(ctx, mod, pVTab, _VTAB_ERROR, err) +} + +func vtabSavepointCallback(ctx context.Context, mod api.Module, pVTab uint32, id int32) uint32 { + vtab := vtabGetHandle(ctx, mod, pVTab).(VTabSavepointer) + err := vtab.Savepoint(int(id)) + return vtabError(ctx, mod, pVTab, _VTAB_ERROR, err) +} + +func vtabReleaseCallback(ctx context.Context, mod api.Module, pVTab uint32, id int32) uint32 { + vtab := vtabGetHandle(ctx, mod, pVTab).(VTabSavepointer) + err := vtab.Release(int(id)) + return vtabError(ctx, mod, pVTab, _VTAB_ERROR, err) +} + +func vtabRollbackToCallback(ctx context.Context, mod api.Module, pVTab uint32, id int32) uint32 { + vtab := vtabGetHandle(ctx, mod, pVTab).(VTabSavepointer) + err := vtab.RollbackTo(int(id)) + return vtabError(ctx, mod, pVTab, _VTAB_ERROR, err) +} + +func cursorOpenCallback(ctx context.Context, mod api.Module, pVTab, ppCur uint32) uint32 { + vtab := vtabGetHandle(ctx, mod, pVTab).(VTab) + + cursor, err := vtab.Open() + if err == nil { + vtabPutHandle(ctx, mod, ppCur, cursor) + } + + return vtabError(ctx, mod, pVTab, _VTAB_ERROR, err) +} + +func cursorCloseCallback(ctx context.Context, mod api.Module, pCur uint32) uint32 { + err := vtabDelHandle(ctx, mod, pCur) + return vtabError(ctx, mod, 0, _VTAB_ERROR, err) +} + +func cursorFilterCallback(ctx context.Context, mod api.Module, pCur uint32, idxNum int32, idxStr, nArg, pArg uint32) uint32 { + cursor := vtabGetHandle(ctx, mod, pCur).(VTabCursor) + db := ctx.Value(connKey{}).(*Conn) + args := make([]Value, nArg) + callbackArgs(db, args, pArg) + var idxName string + if idxStr != 0 { + idxName = util.ReadString(mod, idxStr, _MAX_LENGTH) + } + err := cursor.Filter(int(idxNum), idxName, args...) + return vtabError(ctx, mod, pCur, _CURSOR_ERROR, err) +} + +func cursorEOFCallback(ctx context.Context, mod api.Module, pCur uint32) uint32 { + cursor := vtabGetHandle(ctx, mod, pCur).(VTabCursor) + if cursor.EOF() { + return 1 + } + return 0 +} + +func cursorNextCallback(ctx context.Context, mod api.Module, pCur uint32) uint32 { + cursor := vtabGetHandle(ctx, mod, pCur).(VTabCursor) + err := cursor.Next() + return vtabError(ctx, mod, pCur, _CURSOR_ERROR, err) +} + +func cursorColumnCallback(ctx context.Context, mod api.Module, pCur, pCtx uint32, n int32) uint32 { + cursor := vtabGetHandle(ctx, mod, pCur).(VTabCursor) + db := ctx.Value(connKey{}).(*Conn) + err := cursor.Column(&Context{db, pCtx}, int(n)) + return vtabError(ctx, mod, pCur, _CURSOR_ERROR, err) +} + +func cursorRowIDCallback(ctx context.Context, mod api.Module, pCur, pRowID uint32) uint32 { + cursor := vtabGetHandle(ctx, mod, pCur).(VTabCursor) + + rowID, err := cursor.RowID() + if err == nil { + util.WriteUint64(mod, pRowID, uint64(rowID)) + } + + return vtabError(ctx, mod, pCur, _CURSOR_ERROR, err) +} + +const ( + _PTR_ERROR = iota + _VTAB_ERROR + _CURSOR_ERROR +) + +func vtabError(ctx context.Context, mod api.Module, ptr, kind uint32, err error) uint32 { + const zErrMsgOffset = 8 + msg, code := errorCode(err, ERROR) + if msg != "" && ptr != 0 { + switch kind { + case _VTAB_ERROR: + ptr = ptr + zErrMsgOffset // zErrMsg + case _CURSOR_ERROR: + ptr = util.ReadUint32(mod, ptr) + zErrMsgOffset // pVTab->zErrMsg + } + db := ctx.Value(connKey{}).(*Conn) + if ptr := util.ReadUint32(mod, ptr); ptr != 0 { + db.free(ptr) + } + util.WriteUint32(mod, ptr, db.newString(msg)) + } + return code +} + +func vtabGetHandle(ctx context.Context, mod api.Module, ptr uint32) any { + const handleOffset = 4 + handle := util.ReadUint32(mod, ptr-handleOffset) + return util.GetHandle(ctx, handle) +} + +func vtabDelHandle(ctx context.Context, mod api.Module, ptr uint32) error { + const handleOffset = 4 + handle := util.ReadUint32(mod, ptr-handleOffset) + return util.DelHandle(ctx, handle) +} + +func vtabPutHandle(ctx context.Context, mod api.Module, pptr uint32, val any) { + const handleOffset = 4 + handle := util.AddHandle(ctx, val) + ptr := util.ReadUint32(mod, pptr) + util.WriteUint32(mod, ptr-handleOffset, handle) +} diff --git a/vendor/github.com/ncruces/julianday/.gitignore b/vendor/github.com/ncruces/julianday/.gitignore new file mode 100644 index 000000000..66fd13c90 --- /dev/null +++ b/vendor/github.com/ncruces/julianday/.gitignore @@ -0,0 +1,15 @@ +# Binaries for programs and plugins +*.exe +*.exe~ +*.dll +*.so +*.dylib + +# Test binary, built with `go test -c` +*.test + +# Output of the go coverage tool, specifically when used with LiteIDE +*.out + +# Dependency directories (remove the comment below to include it) +# vendor/ diff --git a/vendor/github.com/ncruces/julianday/LICENSE b/vendor/github.com/ncruces/julianday/LICENSE new file mode 100644 index 000000000..7f0f5534c --- /dev/null +++ b/vendor/github.com/ncruces/julianday/LICENSE @@ -0,0 +1,21 @@ +MIT License + +Copyright (c) 2022 Nuno Cruces + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. diff --git a/vendor/github.com/ncruces/julianday/README.md b/vendor/github.com/ncruces/julianday/README.md new file mode 100644 index 000000000..828ae5749 --- /dev/null +++ b/vendor/github.com/ncruces/julianday/README.md @@ -0,0 +1,9 @@ +# Julian Day calculator + +[![Go Reference](https://pkg.go.dev/badge/image)](https://pkg.go.dev/github.com/ncruces/julianday) +[![Go Report](https://goreportcard.com/badge/github.com/ncruces/julianday)](https://goreportcard.com/report/github.com/ncruces/julianday) +[![Go Coverage](https://github.com/ncruces/julianday/wiki/coverage.svg)](https://raw.githack.com/wiki/ncruces/julianday/coverage.html) + +https://en.wikipedia.org/wiki/Julian_day + +Compatible with [SQLite](https://www.sqlite.org/lang_datefunc.html). diff --git a/vendor/github.com/ncruces/julianday/julianday.go b/vendor/github.com/ncruces/julianday/julianday.go new file mode 100644 index 000000000..d7d0e1960 --- /dev/null +++ b/vendor/github.com/ncruces/julianday/julianday.go @@ -0,0 +1,124 @@ +// Package julianday provides Time to Julian day conversions. +package julianday + +import ( + "bytes" + "errors" + "math" + "strconv" + "time" +) + +const secs_per_day = 86_400 +const nsec_per_sec = 1_000_000_000 +const nsec_per_day = nsec_per_sec * secs_per_day +const epoch_days = 2_440_587 +const epoch_secs = secs_per_day / 2 + +func jd(t time.Time) (day, nsec int64) { + sec := t.Unix() + // guaranteed not to overflow + day, sec = sec/secs_per_day+epoch_days, sec%secs_per_day+epoch_secs + return day, sec*nsec_per_sec + int64(t.Nanosecond()) +} + +// Date returns the Julian day number for t, +// and the nanosecond offset within that day, +// in the range [0, 86399999999999]. +func Date(t time.Time) (day, nsec int64) { + day, nsec = jd(t) + switch { + case nsec < 0: + day -= 1 + nsec += nsec_per_day + case nsec >= nsec_per_day: + day += 1 + nsec -= nsec_per_day + } + return day, nsec +} + +// Float returns the Julian date for t as a float64. +// +// In the XXI century, this has submillisecond precision. +func Float(t time.Time) float64 { + day, nsec := jd(t) + // converting day and nsec to float64 is exact + return float64(day) + float64(nsec)/nsec_per_day +} + +// Format returns the Julian date for t as a string. +// +// This has nanosecond precision. +func Format(t time.Time) string { + var buf [32]byte + return string(AppendFormat(buf[:0], t)) +} + +// AppendFormat is like Format but appends the textual representation to dst +// and returns the extended buffer. +func AppendFormat(dst []byte, t time.Time) []byte { + day, nsec := Date(t) + if day < 0 && nsec != 0 { + dst = append(dst, '-') + day = ^day + nsec = nsec_per_day - nsec + } + var buf [20]byte + dst = strconv.AppendInt(dst, day, 10) + frac := strconv.AppendFloat(buf[:0], float64(nsec)/nsec_per_day, 'f', 15, 64) + return append(dst, bytes.TrimRight(frac[1:], ".0")...) +} + +// Time returns the UTC Time corresponding to the Julian day number +// and nanosecond offset within that day. +// Not all day values have a corresponding time value. +func Time(day, nsec int64) time.Time { + return time.Unix((day-epoch_days)*secs_per_day-epoch_secs, nsec).UTC() +} + +// FloatTime returns the UTC Time corresponding to a Julian date. +// Not all date values have a corresponding time value. +// +// In the XXI century, this has submillisecond precision. +func FloatTime(date float64) time.Time { + day, frac := math.Modf(date) + nsec := math.Floor(frac * nsec_per_day) + return Time(int64(day), int64(nsec)) +} + +// Parse parses a formatted Julian date and returns the UTC Time it represents. +// +// This has nanosecond precision. +func Parse(s string) (time.Time, error) { + digits := 0 + dot := len(s) + for i, b := range []byte(s) { + if '0' <= b && b <= '9' { + digits++ + continue + } + if b == '.' && i < dot { + dot = i + continue + } + if (b == '+' || b == '-') && i == 0 { + continue + } + return time.Time{}, errors.New("julianday: invalid syntax") + } + if digits == 0 { + return time.Time{}, errors.New("julianday: invalid syntax") + } + + day, err := strconv.ParseInt(s[:dot], 10, 64) + if err != nil && dot > 0 { + return time.Time{}, errors.New("julianday: value out of range") + } + frac, _ := strconv.ParseFloat(s[dot:], 64) + nsec := int64(math.Round(frac * nsec_per_day)) + if s[0] == '-' { + nsec = -nsec + } + return Time(day, nsec), nil +} diff --git a/vendor/github.com/tetratelabs/wazero/.editorconfig b/vendor/github.com/tetratelabs/wazero/.editorconfig new file mode 100644 index 000000000..f999431de --- /dev/null +++ b/vendor/github.com/tetratelabs/wazero/.editorconfig @@ -0,0 +1,7 @@ +root = true + +[*] +charset = utf-8 +end_of_line = lf +insert_final_newline = true +trim_trailing_whitespace = true diff --git a/vendor/github.com/tetratelabs/wazero/.gitattributes b/vendor/github.com/tetratelabs/wazero/.gitattributes new file mode 100644 index 000000000..3a08bc389 --- /dev/null +++ b/vendor/github.com/tetratelabs/wazero/.gitattributes @@ -0,0 +1,2 @@ +# Improves experience of commands like `make format` on Windows +* text=auto eol=lf diff --git a/vendor/github.com/tetratelabs/wazero/.gitignore b/vendor/github.com/tetratelabs/wazero/.gitignore new file mode 100644 index 000000000..6a14146d4 --- /dev/null +++ b/vendor/github.com/tetratelabs/wazero/.gitignore @@ -0,0 +1,45 @@ +# If you prefer the allow list template instead of the deny list, see community template: +# https://github.com/github/gitignore/blob/main/community/Golang/Go.AllowList.gitignore +# +# Binaries for programs and plugins +*.exe +*.exe~ +*.dll +*.so +*.dylib +/wazero +build +dist + +# Test binary, built with `go test -c` +*.test + +# Output of the go coverage tool, specifically when used with LiteIDE +*.out + +# Dependency directories (remove the comment below to include it) +# vendor/ + +# Go workspace file +go.work + +# Goland +.idea + +# AssemblyScript +node_modules +package-lock.json + +# codecov.io +/coverage.txt + +.vagrant + +zig-cache/ +zig-out/ + +.DS_Store + +# Ignore compiled stdlib test cases. +/internal/integration_test/stdlibs/testdata +/internal/integration_test/libsodium/testdata diff --git a/vendor/github.com/tetratelabs/wazero/.gitmodules b/vendor/github.com/tetratelabs/wazero/.gitmodules new file mode 100644 index 000000000..410c91f44 --- /dev/null +++ b/vendor/github.com/tetratelabs/wazero/.gitmodules @@ -0,0 +1,3 @@ +[submodule "site/themes/hello-friend"] + path = site/themes/hello-friend + url = https://github.com/panr/hugo-theme-hello-friend.git diff --git a/vendor/github.com/tetratelabs/wazero/CONTRIBUTING.md b/vendor/github.com/tetratelabs/wazero/CONTRIBUTING.md new file mode 100644 index 000000000..8ab866f0e --- /dev/null +++ b/vendor/github.com/tetratelabs/wazero/CONTRIBUTING.md @@ -0,0 +1,75 @@ +# Contributing + +We welcome contributions from the community. Please read the following guidelines carefully to maximize the chances of your PR being merged. + +## Coding Style + +- To ensure your change passes format checks, run `make check`. To format your files, you can run `make format`. +- We follow standard Go table-driven tests and use an internal [testing library](./internal/testing/require) to assert correctness. To verify all tests pass, you can run `make test`. + +## DCO + +We require DCO signoff line in every commit to this repo. + +The sign-off is a simple line at the end of the explanation for the +patch, which certifies that you wrote it or otherwise have the right to +pass it on as an open-source patch. The rules are pretty simple: if you +can certify the below (from +[developercertificate.org](https://developercertificate.org/)): + +``` +Developer Certificate of Origin +Version 1.1 +Copyright (C) 2004, 2006 The Linux Foundation and its contributors. +660 York Street, Suite 102, +San Francisco, CA 94110 USA +Everyone is permitted to copy and distribute verbatim copies of this +license document, but changing it is not allowed. +Developer's Certificate of Origin 1.1 +By making a contribution to this project, I certify that: +(a) The contribution was created in whole or in part by me and I + have the right to submit it under the open source license + indicated in the file; or +(b) The contribution is based upon previous work that, to the best + of my knowledge, is covered under an appropriate open source + license and I have the right under that license to submit that + work with modifications, whether created in whole or in part + by me, under the same open source license (unless I am + permitted to submit under a different license), as indicated + in the file; or +(c) The contribution was provided directly to me by some other + person who certified (a), (b) or (c) and I have not modified + it. +(d) I understand and agree that this project and the contribution + are public and that a record of the contribution (including all + personal information I submit with it, including my sign-off) is + maintained indefinitely and may be redistributed consistent with + this project or the open source license(s) involved. +``` + +then you just add a line to every git commit message: + + Signed-off-by: Joe Smith + +using your real name (sorry, no pseudonyms or anonymous contributions.) + +You can add the sign off when creating the git commit via `git commit -s`. + +## Code Reviews + +* The pull request title should describe what the change does and not embed issue numbers. +The pull request should only be blank when the change is minor. Any feature should include +a description of the change and what motivated it. If the change or design changes through +review, please keep the title and description updated accordingly. +* A single approval is sufficient to merge. If a reviewer asks for +changes in a PR they should be addressed before the PR is merged, +even if another reviewer has already approved the PR. +* During the review, address the comments and commit the changes +_without_ squashing the commits. This facilitates incremental reviews +since the reviewer does not go through all the code again to find out +what has changed since the last review. When a change goes out of sync with main, +please rebase and force push, keeping the original commits where practical. +* Commits are squashed prior to merging a pull request, using the title +as commit message by default. Maintainers may request contributors to +edit the pull request tite to ensure that it remains descriptive as a +commit message. Alternatively, maintainers may change the commit message directly. diff --git a/vendor/github.com/tetratelabs/wazero/LICENSE b/vendor/github.com/tetratelabs/wazero/LICENSE new file mode 100644 index 000000000..e21d69958 --- /dev/null +++ b/vendor/github.com/tetratelabs/wazero/LICENSE @@ -0,0 +1,201 @@ + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "[]" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright 2020-2023 wazero authors + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. diff --git a/vendor/github.com/tetratelabs/wazero/Makefile b/vendor/github.com/tetratelabs/wazero/Makefile new file mode 100644 index 000000000..e5ae8a261 --- /dev/null +++ b/vendor/github.com/tetratelabs/wazero/Makefile @@ -0,0 +1,381 @@ + +gofumpt := mvdan.cc/gofumpt@v0.5.0 +gosimports := github.com/rinchsan/gosimports/cmd/gosimports@v0.3.8 +golangci_lint := github.com/golangci/golangci-lint/cmd/golangci-lint@v1.55.2 +asmfmt := github.com/klauspost/asmfmt/cmd/asmfmt@v1.3.2 +# sync this with netlify.toml! +hugo := github.com/gohugoio/hugo@v0.115.2 + +# Make 3.81 doesn't support '**' globbing: Set explicitly instead of recursion. +all_sources := $(wildcard *.go */*.go */*/*.go */*/*/*.go */*/*/*.go */*/*/*/*.go) +all_testdata := $(wildcard testdata/* */testdata/* */*/testdata/* */*/testdata/*/* */*/*/testdata/*) +all_testing := $(wildcard internal/testing/* internal/testing/*/* internal/testing/*/*/*) +all_examples := $(wildcard examples/* examples/*/* examples/*/*/* */*/example/* */*/example/*/* */*/example/*/*/*) +all_it := $(wildcard internal/integration_test/* internal/integration_test/*/* internal/integration_test/*/*/*) +# main_sources exclude any test or example related code +main_sources := $(wildcard $(filter-out %_test.go $(all_testdata) $(all_testing) $(all_examples) $(all_it), $(all_sources))) +# main_packages collect the unique main source directories (sort will dedupe). +# Paths need to all start with ./, so we do that manually vs foreach which strips it. +main_packages := $(sort $(foreach f,$(dir $(main_sources)),$(if $(findstring ./,$(f)),./,./$(f)))) + +go_test_options ?= -timeout 300s + +ensureCompilerFastest := -ldflags '-X github.com/tetratelabs/wazero/internal/integration_test/vs.ensureCompilerFastest=true' +.PHONY: bench +bench: + @go build ./internal/integration_test/bench/... + @# Don't use -test.benchmem as it isn't accurate when comparing against CGO libs + @for d in vs/time vs/wasmedge vs/wasmtime ; do \ + cd ./internal/integration_test/$$d ; \ + go test -bench=. . -tags='wasmedge' $(ensureCompilerFastest) ; \ + cd - ;\ + done + +bench_testdata_dir := internal/integration_test/bench/testdata +.PHONY: build.bench +build.bench: + @tinygo build -o $(bench_testdata_dir)/case.wasm -scheduler=none --no-debug -target=wasi $(bench_testdata_dir)/case.go + +.PHONY: test.examples +test.examples: + @go test $(go_test_options) ./examples/... ./imports/assemblyscript/example/... ./imports/emscripten/... ./imports/wasi_snapshot_preview1/example/... + +.PHONY: build.examples.as +build.examples.as: + @cd ./imports/assemblyscript/example/testdata && npm install && npm run build + +%.wasm: %.zig + @(cd $(@D); zig build -Doptimize=ReleaseSmall) + @mv $(@D)/zig-out/*/$(@F) $(@D) + +.PHONY: build.examples.zig +build.examples.zig: examples/allocation/zig/testdata/greet.wasm imports/wasi_snapshot_preview1/example/testdata/zig/cat.wasm imports/wasi_snapshot_preview1/testdata/zig/wasi.wasm + @cd internal/testing/dwarftestdata/testdata/zig; zig build; mv zig-out/*/main.wasm ./ # Need DWARF custom sections. + +tinygo_sources := examples/basic/testdata/add.go examples/allocation/tinygo/testdata/greet.go examples/cli/testdata/cli.go imports/wasi_snapshot_preview1/example/testdata/tinygo/cat.go imports/wasi_snapshot_preview1/testdata/tinygo/wasi.go cmd/wazero/testdata/cat/cat.go +.PHONY: build.examples.tinygo +build.examples.tinygo: $(tinygo_sources) + @for f in $^; do \ + tinygo build -o $$(echo $$f | sed -e 's/\.go/\.wasm/') -scheduler=none --no-debug --target=wasi $$f; \ + done + @mv cmd/wazero/testdata/cat/cat.wasm cmd/wazero/testdata/cat/cat-tinygo.wasm + +# We use zig to build C as it is easy to install and embeds a copy of zig-cc. +# Note: Don't use "-Oz" as that breaks our wasi sock example. +c_sources := imports/wasi_snapshot_preview1/example/testdata/zig-cc/cat.c imports/wasi_snapshot_preview1/testdata/zig-cc/wasi.c internal/testing/dwarftestdata/testdata/zig-cc/main.c +.PHONY: build.examples.zig-cc +build.examples.zig-cc: $(c_sources) + @for f in $^; do \ + zig cc --target=wasm32-wasi -o $$(echo $$f | sed -e 's/\.c/\.wasm/') $$f; \ + done + +# Here are the emcc args we use: +# +# * `-Oz` - most optimization for code size. +# * `--profiling` - adds the name section. +# * `-s STANDALONE_WASM` - ensures wasm is built for a non-js runtime. +# * `-s EXPORTED_FUNCTIONS=_malloc,_free` - export allocation functions so that +# they can be used externally as "malloc" and "free". +# * `-s WARN_ON_UNDEFINED_SYMBOLS=0` - imports not defined in JavaScript error +# otherwise. See https://github.com/emscripten-core/emscripten/issues/13641 +# * `-s TOTAL_STACK=8KB -s TOTAL_MEMORY=64KB` - reduce memory default from 16MB +# to one page (64KB). To do this, we have to reduce the stack size. +# * `-s ALLOW_MEMORY_GROWTH` - allows "memory.grow" instructions to succeed, but +# requires a function import "emscripten_notify_memory_growth". +emscripten_sources := $(wildcard imports/emscripten/testdata/*.cc) +.PHONY: build.examples.emscripten +build.examples.emscripten: $(emscripten_sources) + @for f in $^; do \ + em++ -Oz --profiling \ + -s STANDALONE_WASM \ + -s EXPORTED_FUNCTIONS=_malloc,_free \ + -s WARN_ON_UNDEFINED_SYMBOLS=0 \ + -s TOTAL_STACK=8KB -s TOTAL_MEMORY=64KB \ + -s ALLOW_MEMORY_GROWTH \ + --std=c++17 -o $$(echo $$f | sed -e 's/\.cc/\.wasm/') $$f; \ + done + +%/greet.wasm : cargo_target := wasm32-unknown-unknown +%/cat.wasm : cargo_target := wasm32-wasi +%/wasi.wasm : cargo_target := wasm32-wasi + +.PHONY: build.examples.rust +build.examples.rust: examples/allocation/rust/testdata/greet.wasm imports/wasi_snapshot_preview1/example/testdata/cargo-wasi/cat.wasm imports/wasi_snapshot_preview1/testdata/cargo-wasi/wasi.wasm internal/testing/dwarftestdata/testdata/rust/main.wasm.xz + +# Normally, we build release because it is smaller. Testing dwarf requires the debug build. +internal/testing/dwarftestdata/testdata/rust/main.wasm.xz: + cd $(@D) && cargo wasi build + mv $(@D)/target/wasm32-wasi/debug/main.wasm $(@D) + cd $(@D) && xz -k -f ./main.wasm # Rust's DWARF section is huge, so compress it. + +# Builds rust using cargo normally, or cargo-wasi. +%.wasm: %.rs + @(cd $(@D); cargo $(if $(findstring wasi,$(cargo_target)),wasi build,build --target $(cargo_target)) --release) + @mv $(@D)/target/$(cargo_target)/release/$(@F) $(@D) + +spectest_base_dir := internal/integration_test/spectest +spectest_v1_dir := $(spectest_base_dir)/v1 +spectest_v1_testdata_dir := $(spectest_v1_dir)/testdata +spec_version_v1 := wg-1.0 +spectest_v2_dir := $(spectest_base_dir)/v2 +spectest_v2_testdata_dir := $(spectest_v2_dir)/testdata +# Latest draft state as of March 12, 2024. +spec_version_v2 := 1c5e5d178bd75c79b7a12881c529098beaee2a05 +spectest_threads_dir := $(spectest_base_dir)/threads +spectest_threads_testdata_dir := $(spectest_threads_dir)/testdata +# From https://github.com/WebAssembly/threads/tree/upstream-rebuild which has not been merged to main yet. +# It will likely be renamed to main in the future - https://github.com/WebAssembly/threads/issues/216. +spec_version_threads := 3635ca51a17e57e106988846c5b0e0cc48ac04fc + +.PHONY: build.spectest +build.spectest: + @$(MAKE) build.spectest.v1 + @$(MAKE) build.spectest.v2 + +.PHONY: build.spectest.v1 +build.spectest.v1: # Note: wabt by default uses >1.0 features, so wast2json flags might drift as they include more. See WebAssembly/wabt#1878 + @rm -rf $(spectest_v1_testdata_dir) + @mkdir -p $(spectest_v1_testdata_dir) + @cd $(spectest_v1_testdata_dir) \ + && curl -sSL 'https://api.github.com/repos/WebAssembly/spec/contents/test/core?ref=$(spec_version_v1)' | jq -r '.[]| .download_url' | grep -E ".wast" | xargs -Iurl curl -sJL url -O + @cd $(spectest_v1_testdata_dir) && for f in `find . -name '*.wast'`; do \ + perl -pi -e 's/\(assert_return_canonical_nan\s(\(invoke\s"f32.demote_f64"\s\((f[0-9]{2})\.const\s[a-z0-9.+:-]+\)\))\)/\(assert_return $$1 \(f32.const nan:canonical\)\)/g' $$f; \ + perl -pi -e 's/\(assert_return_arithmetic_nan\s(\(invoke\s"f32.demote_f64"\s\((f[0-9]{2})\.const\s[a-z0-9.+:-]+\)\))\)/\(assert_return $$1 \(f32.const nan:arithmetic\)\)/g' $$f; \ + perl -pi -e 's/\(assert_return_canonical_nan\s(\(invoke\s"f64\.promote_f32"\s\((f[0-9]{2})\.const\s[a-z0-9.+:-]+\)\))\)/\(assert_return $$1 \(f64.const nan:canonical\)\)/g' $$f; \ + perl -pi -e 's/\(assert_return_arithmetic_nan\s(\(invoke\s"f64\.promote_f32"\s\((f[0-9]{2})\.const\s[a-z0-9.+:-]+\)\))\)/\(assert_return $$1 \(f64.const nan:arithmetic\)\)/g' $$f; \ + perl -pi -e 's/\(assert_return_canonical_nan\s(\(invoke\s"[a-z._0-9]+"\s\((f[0-9]{2})\.const\s[a-z0-9.+:-]+\)\))\)/\(assert_return $$1 \($$2.const nan:canonical\)\)/g' $$f; \ + perl -pi -e 's/\(assert_return_arithmetic_nan\s(\(invoke\s"[a-z._0-9]+"\s\((f[0-9]{2})\.const\s[a-z0-9.+:-]+\)\))\)/\(assert_return $$1 \($$2.const nan:arithmetic\)\)/g' $$f; \ + perl -pi -e 's/\(assert_return_canonical_nan\s(\(invoke\s"[a-z._0-9]+"\s\((f[0-9]{2})\.const\s[a-z0-9.+:-]+\)\s\([a-z0-9.\s+-:]+\)\))\)/\(assert_return $$1 \($$2.const nan:canonical\)\)/g' $$f; \ + perl -pi -e 's/\(assert_return_arithmetic_nan\s(\(invoke\s"[a-z._0-9]+"\s\((f[0-9]{2})\.const\s[a-z0-9.+:-]+\)\s\([a-z0-9.\s+-:]+\)\))\)/\(assert_return $$1 \($$2.const nan:arithmetic\)\)/g' $$f; \ + perl -pi -e 's/\(assert_return_canonical_nan\s(\(invoke\s"[a-z._0-9]+"\s\((f[0-9]{2})\.const\s[a-z0-9.+:-]+\)\))\)/\(assert_return $$1 \($$2.const nan:canonical\)\)/g' $$f; \ + perl -pi -e 's/\(assert_return_arithmetic_nan\s(\(invoke\s"[a-z._0-9]+"\s\((f[0-9]{2})\.const\s[a-z0-9.+:-]+\)\))\)/\(assert_return $$1 \($$2.const nan:arithmetic\)\)/g' $$f; \ + wast2json \ + --disable-saturating-float-to-int \ + --disable-sign-extension \ + --disable-simd \ + --disable-multi-value \ + --disable-bulk-memory \ + --disable-reference-types \ + --debug-names $$f; \ + done + +.PHONY: build.spectest.v2 +build.spectest.v2: # Note: SIMD cases are placed in the "simd" subdirectory. + @mkdir -p $(spectest_v2_testdata_dir) + @cd $(spectest_v2_testdata_dir) \ + && curl -sSL 'https://api.github.com/repos/WebAssembly/spec/contents/test/core?ref=$(spec_version_v2)' | jq -r '.[]| .download_url' | grep -E ".wast" | xargs -Iurl curl -sJL url -O + @cd $(spectest_v2_testdata_dir) \ + && curl -sSL 'https://api.github.com/repos/WebAssembly/spec/contents/test/core/simd?ref=$(spec_version_v2)' | jq -r '.[]| .download_url' | grep -E ".wast" | xargs -Iurl curl -sJL url -O + @cd $(spectest_v2_testdata_dir) && for f in `find . -name '*.wast'`; do \ + wast2json --debug-names --no-check $$f || true; \ + done # Ignore the error here as some tests (e.g. comments.wast right now) are not supported by wast2json yet. + +# Note: We currently cannot build the "threads" subdirectory that spawns threads due to missing support in wast2json. +# https://github.com/WebAssembly/wabt/issues/2348#issuecomment-1878003959 +.PHONY: build.spectest.threads +build.spectest.threads: + @mkdir -p $(spectest_threads_testdata_dir) + @cd $(spectest_threads_testdata_dir) \ + && curl -sSL 'https://api.github.com/repos/WebAssembly/threads/contents/test/core?ref=$(spec_version_threads)' | jq -r '.[]| .download_url' | grep -E "atomic.wast" | xargs -Iurl curl -sJL url -O + @cd $(spectest_threads_testdata_dir) && for f in `find . -name '*.wast'`; do \ + wast2json --enable-threads --debug-names $$f; \ + done + +.PHONY: test +test: + @go test $(go_test_options) $$(go list ./... | grep -vE '$(spectest_v1_dir)|$(spectest_v2_dir)') + @cd internal/version/testdata && go test $(go_test_options) ./... + @cd internal/integration_test/fuzz/wazerolib && CGO_ENABLED=0 WASM_BINARY_PATH=testdata/test.wasm go test ./... + +.PHONY: coverage +# replace spaces with commas +coverpkg = $(shell echo $(main_packages) | tr ' ' ',') +coverage: ## Generate test coverage + @go test -coverprofile=coverage.txt -covermode=atomic --coverpkg=$(coverpkg) $(main_packages) + @go tool cover -func coverage.txt + +.PHONY: spectest +spectest: + @$(MAKE) spectest.v1 + @$(MAKE) spectest.v2 + +spectest.v1: + @go test $(go_test_options) $$(go list ./... | grep $(spectest_v1_dir)) + +spectest.v2: + @go test $(go_test_options) $$(go list ./... | grep $(spectest_v2_dir)) + +golangci_lint_path := $(shell go env GOPATH)/bin/golangci-lint + +$(golangci_lint_path): + @go install $(golangci_lint) + +golangci_lint_goarch ?= $(shell go env GOARCH) + +.PHONY: lint +lint: $(golangci_lint_path) + @GOARCH=$(golangci_lint_goarch) CGO_ENABLED=0 $(golangci_lint_path) run --timeout 5m + +.PHONY: format +format: + @go run $(gofumpt) -l -w . + @go run $(gosimports) -local github.com/tetratelabs/ -w $(shell find . -name '*.go' -type f) + @go run $(asmfmt) -w $(shell find . -name '*.s' -type f) + +.PHONY: check # Pre-flight check for pull requests +check: +# The following checks help ensure our platform-specific code used for system +# calls safely falls back on a platform unsupported by the compiler engine. +# This makes sure the intepreter can be used. Most often the package that can +# drift here is "platform" or "sysfs": +# +# Ensure we build on plan9. See #1578 + @GOARCH=amd64 GOOS=plan9 go build ./... +# Ensure we build on gojs. See #1526. + @GOARCH=wasm GOOS=js go build ./... +# Ensure we build on wasip1. See #1526. + @GOARCH=wasm GOOS=wasip1 go build ./... +# Ensure we build on aix. See #1723 + @GOARCH=ppc64 GOOS=aix go build ./... +# Ensure we build on windows: + @GOARCH=amd64 GOOS=windows go build ./... +# Ensure we build on an arbitrary operating system: + @GOARCH=amd64 GOOS=dragonfly go build ./... +# Ensure we build on solaris/illumos: + @GOARCH=amd64 GOOS=illumos go build ./... + @GOARCH=amd64 GOOS=solaris go build ./... +# Ensure we build on linux arm for Dapr: +# gh release view -R dapr/dapr --json assets --jq 'first(.assets[] | select(.name = "daprd_linux_arm.tar.gz") | {url, downloadCount})' + @GOARCH=arm GOOS=linux go build ./... +# Ensure we build on linux 386 for Trivy: +# gh release view -R aquasecurity/trivy --json assets --jq 'first(.assets[] | select(.name| test("Linux-32bit.*tar.gz")) | {url, downloadCount})' + @GOARCH=386 GOOS=linux go build ./... +# Ensure we build on FreeBSD amd64 for Trivy: +# gh release view -R aquasecurity/trivy --json assets --jq 'first(.assets[] | select(.name| test("FreeBSD-64bit.*tar.gz")) | {url, downloadCount})' + @GOARCH=amd64 GOOS=freebsd go build ./... + @$(MAKE) lint golangci_lint_goarch=arm64 + @$(MAKE) lint golangci_lint_goarch=amd64 + @$(MAKE) format + @go mod tidy + @if [ ! -z "`git status -s`" ]; then \ + echo "The following differences will fail CI until committed:"; \ + git diff --exit-code; \ + fi + +.PHONY: site +site: ## Serve website content + @git submodule update --init + @cd site && go run $(hugo) server --minify --disableFastRender --baseURL localhost:1313 --cleanDestinationDir -D + +.PHONY: clean +clean: ## Ensure a clean build + @rm -rf dist build coverage.txt + @go clean -testcache + +fuzz_default_flags := --no-trace-compares --sanitizer=none -- -rss_limit_mb=8192 + +fuzz_timeout_seconds ?= 10 +.PHONY: fuzz +fuzz: + @cd internal/integration_test/fuzz && cargo test + @cd internal/integration_test/fuzz && cargo fuzz run logging_no_diff $(fuzz_default_flags) -max_total_time=$(fuzz_timeout_seconds) + @cd internal/integration_test/fuzz && cargo fuzz run no_diff $(fuzz_default_flags) -max_total_time=$(fuzz_timeout_seconds) + @cd internal/integration_test/fuzz && cargo fuzz run memory_no_diff $(fuzz_default_flags) -max_total_time=$(fuzz_timeout_seconds) + @cd internal/integration_test/fuzz && cargo fuzz run validation $(fuzz_default_flags) -max_total_time=$(fuzz_timeout_seconds) + +libsodium: + cd ./internal/integration_test/libsodium/testdata && \ + curl -s "https://api.github.com/repos/jedisct1/webassembly-benchmarks/contents/2022-12/wasm?ref=7e86d68e99e60130899fbe3b3ab6e9dce9187a7c" \ + | jq -r '.[] | .download_url' | xargs -n 1 curl -LO + +#### CLI release related #### + +VERSION ?= dev +# Default to a dummy version 0.0.1.1, which is always lower than a real release. +# Legal version values should look like 'x.x.x.x' where x is an integer from 0 to 65534. +# https://learn.microsoft.com/en-us/windows/win32/msi/productversion?redirectedfrom=MSDN +# https://stackoverflow.com/questions/9312221/msi-version-numbers +MSI_VERSION ?= 0.0.1.1 +non_windows_platforms := darwin_amd64 darwin_arm64 linux_amd64 linux_arm64 +non_windows_archives := $(non_windows_platforms:%=dist/wazero_$(VERSION)_%.tar.gz) +windows_platforms := windows_amd64 # TODO: add arm64 windows once we start testing on it. +windows_archives := $(windows_platforms:%=dist/wazero_$(VERSION)_%.zip) $(windows_platforms:%=dist/wazero_$(VERSION)_%.msi) +checksum_txt := dist/wazero_$(VERSION)_checksums.txt + +# define macros for multi-platform builds. these parse the filename being built +go-arch = $(if $(findstring amd64,$1),amd64,arm64) +go-os = $(if $(findstring .exe,$1),windows,$(if $(findstring linux,$1),linux,darwin)) +# msi-arch is a macro so we can detect it based on the file naming convention +msi-arch = $(if $(findstring amd64,$1),x64,arm64) + +build/wazero_%/wazero: + $(call go-build,$@,$<) + +build/wazero_%/wazero.exe: + $(call go-build,$@,$<) + +dist/wazero_$(VERSION)_%.tar.gz: build/wazero_%/wazero + @echo tar.gz "tarring $@" + @mkdir -p $(@D) +# On Windows, we pass the special flag `--mode='+rx' to ensure that we set the executable flag. +# This is only supported by GNU Tar, so we set it conditionally. + @tar -C $(> $(@F) + +dist: $(non_windows_archives) $(if $(findstring Windows_NT,$(OS)),$(windows_archives),) $(checksum_txt) diff --git a/vendor/github.com/tetratelabs/wazero/NOTICE b/vendor/github.com/tetratelabs/wazero/NOTICE new file mode 100644 index 000000000..2f5ea8ebf --- /dev/null +++ b/vendor/github.com/tetratelabs/wazero/NOTICE @@ -0,0 +1,2 @@ +wazero +Copyright 2020-2023 wazero authors diff --git a/vendor/github.com/tetratelabs/wazero/RATIONALE.md b/vendor/github.com/tetratelabs/wazero/RATIONALE.md new file mode 100644 index 000000000..8d783cb44 --- /dev/null +++ b/vendor/github.com/tetratelabs/wazero/RATIONALE.md @@ -0,0 +1,1587 @@ +# Notable rationale of wazero + +## Zero dependencies + +Wazero has zero dependencies to differentiate itself from other runtimes which +have heavy impact usually due to CGO. By avoiding CGO, wazero avoids +prerequisites such as shared libraries or libc, and lets users keep features +like cross compilation. + +Avoiding go.mod dependencies reduces interference on Go version support, and +size of a statically compiled binary. However, doing so brings some +responsibility into the project. + +Go's native platform support is good: We don't need platform-specific code to +get monotonic time, nor do we need much work to implement certain features +needed by our compiler such as `mmap`. That said, Go does not support all +common operating systems to the same degree. For example, Go 1.18 includes +`Mprotect` on Linux and Darwin, but not FreeBSD. + +The general tradeoff the project takes from a zero dependency policy is more +explicit support of platforms (in the compiler runtime), as well a larger and +more technically difficult codebase. + +At some point, we may allow extensions to supply their own platform-specific +hooks. Until then, one end user impact/tradeoff is some glitches trying +untested platforms (with the Compiler runtime). + +### Why do we use CGO to implement system calls on darwin? + +wazero is dependency and CGO free by design. In some cases, we have code that +can optionally use CGO, but retain a fallback for when that's disabled. The only +operating system (`GOOS`) we use CGO by default in is `darwin`. + +Unlike other operating systems, regardless of `CGO_ENABLED`, Go always uses +"CGO" mechanisms in the runtime layer of `darwin`. This is explained in +[Statically linked binaries on Mac OS X](https://developer.apple.com/library/archive/qa/qa1118/_index.html#//apple_ref/doc/uid/DTS10001666): + +> Apple does not support statically linked binaries on Mac OS X. A statically +> linked binary assumes binary compatibility at the kernel system call +> interface, and we do not make any guarantees on that front. Rather, we strive +> to ensure binary compatibility in each dynamically linked system library and +> framework. + +This plays to our advantage for system calls that aren't yet exposed in the Go +standard library, notably `futimens` for nanosecond-precision timestamp +manipulation. + +### Why not x/sys + +Going beyond Go's SDK limitations can be accomplished with their [x/sys library](https://pkg.go.dev/golang.org/x/sys/unix). +For example, this includes `zsyscall_freebsd_amd64.go` missing from the Go SDK. + +However, like all dependencies, x/sys is a source of conflict. For example, +x/sys had to be in order to upgrade to Go 1.18. + +If we depended on x/sys, we could get more precise functionality needed for +features such as clocks or more platform support for the compiler runtime. + +That said, formally supporting an operating system may still require testing as +even use of x/sys can require platform-specifics. For example, [mmap-go](https://github.com/edsrzf/mmap-go) +uses x/sys, but also mentions limitations, some not surmountable with x/sys +alone. + +Regardless, we may at some point introduce a separate go.mod for users to use +x/sys as a platform plugin without forcing all users to maintain that +dependency. + +## Project structure + +wazero uses internal packages extensively to balance API compatibility desires for end users with the need to safely +share internals between compilers. + +End-user packages include `wazero`, with `Config` structs, `api`, with shared types, and the built-in `wasi` library. +Everything else is internal. + +We put the main program for wazero into a directory of the same name to match conventions used in `go install`, +notably the name of the folder becomes the binary name. We chose to use `cmd/wazero` as it is common practice +and less surprising than `wazero/wazero`. + +### Internal packages + +Most code in wazero is internal, and it is acknowledged that this prevents external implementation of facets such as +compilers or decoding. It also prevents splitting this code into separate repositories, resulting in a larger monorepo. +This also adds work as more code needs to be centrally reviewed. + +However, the alternative is neither secure nor viable. To allow external implementation would require exporting symbols +public, such as the `CodeSection`, which can easily create bugs. Moreover, there's a high drift risk for any attempt at +external implementations, compounded not just by wazero's code organization, but also the fast moving Wasm and WASI +specifications. + +For example, implementing a compiler correctly requires expertise in Wasm, Golang and assembly. This requires deep +insight into how internals are meant to be structured and the various tiers of testing required for `wazero` to result +in a high quality experience. Even if someone had these skills, supporting external code would introduce variables which +are constants in the central one. Supporting an external codebase is harder on the project team, and could starve time +from the already large burden on the central codebase. + +The tradeoffs of internal packages are a larger codebase and responsibility to implement all standard features. It also +implies thinking about extension more as forking is not viable for reasons above also. The primary mitigation of these +realities are friendly OSS licensing, high rigor and a collaborative spirit which aim to make contribution in the shared +codebase productive. + +### Avoiding cyclic dependencies + +wazero shares constants and interfaces with internal code by a sharing pattern described below: +* shared interfaces and constants go in one package under root: `api`. +* user APIs and structs depend on `api` and go into the root package `wazero`. + * e.g. `InstantiateModule` -> `/wasm.go` depends on the type `api.Module`. +* implementation code can also depend on `api` in a corresponding package under `/internal`. + * Ex package `wasm` -> `/internal/wasm/*.go` and can depend on the type `api.Module`. + +The above guarantees no cyclic dependencies at the cost of having to re-define symbols that exist in both packages. +For example, if `wasm.Store` is a type the user needs access to, it is narrowed by a cover type in the `wazero`: + +```go +type runtime struct { + s *wasm.Store +} +``` + +This is not as bad as it sounds as mutations are only available via configuration. This means exported functions are +limited to only a few functions. + +### Avoiding security bugs + +In order to avoid security flaws such as code insertion, nothing in the public API is permitted to write directly to any +mutable symbol in the internal package. For example, the package `api` is shared with internal code. To ensure +immutability, the `api` package cannot contain any mutable public symbol, such as a slice or a struct with an exported +field. + +In practice, this means shared functionality like memory mutation need to be implemented by interfaces. + +Here are some examples: +* `api.Memory` protects access by exposing functions like `WriteFloat64Le` instead of exporting a buffer (`[]byte`). +* There is no exported symbol for the `[]byte` representing the `CodeSection` + +Besides security, this practice prevents other bugs and allows centralization of validation logic such as decoding Wasm. + +## API Design + +### Why is `context.Context` inconsistent? + +It may seem strange that only certain API have an initial `context.Context` +parameter. We originally had a `context.Context` for anything that might be +traced, but it turned out to be only useful for lifecycle and host functions. + +For instruction-scoped aspects like memory updates, a context parameter is too +fine-grained and also invisible in practice. For example, most users will use +the compiler engine, and its memory, global or table access will never use go's +context. + +### Why does `api.ValueType` map to uint64? + +WebAssembly allows functions to be defined either by the guest or the host, +with signatures expressed as WebAssembly types. For example, `i32` is a 32-bit +type which might be interpreted as signed. Function signatures can have zero or +more parameters or results even if WebAssembly 1.0 allows up to one result. + +The guest can export functions, so that the host can call it. In the case of +wazero, the host is Go and an exported function can be called via +`api.Function`. `api.Function` allows users to supply parameters and read +results as a slice of uint64. For example, if there are no results, an empty +slice is returned. The user can learn the signature via `FunctionDescription`, +which returns the `api.ValueType` corresponding to each parameter or result. +`api.ValueType` defines the mapping of WebAssembly types to `uint64` values for +reason described in this section. The special case of `v128` is also mentioned +below. + +wazero maps each value type to a uint64 values because it holds the largest +type in WebAssembly 1.0 (i64). A slice allows you to express empty (e.g. a +nullary signature), for example a start function. + +Here's an example of calling a function, noting this syntax works for both a +signature `(param i32 i32) (result i32)` and `(param i64 i64) (result i64)` +```go +x, y := uint64(1), uint64(2) +results, err := mod.ExportedFunction("add").Call(ctx, x, y) +if err != nil { + log.Panicln(err) +} +fmt.Printf("%d + %d = %d\n", x, y, results[0]) +``` + +WebAssembly does not define an encoding strategy for host defined parameters or +results. This means the encoding rules above are defined by wazero instead. To +address this, we clarified mapping both in `api.ValueType` and added helper +functions like `api.EncodeF64`. This allows users conversions typical in Go +programming, and utilities to avoid ambiguity and edge cases around casting. + +Alternatively, we could have defined a byte buffer based approach and a binary +encoding of value types in and out. For example, an empty byte slice would mean +no values, while a non-empty could use a binary encoding for supported values. +This could work, but it is more difficult for the normal case of i32 and i64. +It also shares a struggle with the current approach, which is that value types +were added after WebAssembly 1.0 and not all of them have an encoding. More on +this below. + +In summary, wazero chose an approach for signature mapping because there was +none, and the one we chose biases towards simplicity with integers and handles +the rest with documentation and utilities. + +#### Post 1.0 value types + +Value types added after WebAssembly 1.0 stressed the current model, as some +have no encoding or are larger than 64 bits. While problematic, these value +types are not commonly used in exported (extern) functions. However, some +decisions were made and detailed below. + +For example `externref` has no guest representation. wazero chose to map +references to uint64 as that's the largest value needed to encode a pointer on +supported platforms. While there are two reference types, `externref` and +`functype`, the latter is an internal detail of function tables, and the former +is rarely if ever used in function signatures as of the end of 2022. + +The only value larger than 64 bits is used for SIMD (`v128`). Vectorizing via +host functions is not used as of the end of 2022. Even if it were, it would be +inefficient vs guest vectorization due to host function overhead. In other +words, the `v128` value type is unlikely to be in an exported function +signature. That it requires two uint64 values to encode is an internal detail +and not worth changing the exported function interface `api.Function`, as doing +so would break all users. + +### Interfaces, not structs + +All exported types in public packages, regardless of configuration vs runtime, are interfaces. The primary benefits are +internal flexibility and avoiding people accidentally mis-initializing by instantiating the types on their own vs using +the `NewXxx` constructor functions. In other words, there's less support load when things can't be done incorrectly. + +Here's an example: +```go +rt := &RuntimeConfig{} // not initialized properly (fields are nil which shouldn't be) +rt := RuntimeConfig{} // not initialized properly (should be a pointer) +rt := wazero.NewRuntimeConfig() // initialized properly +``` + +There are a few drawbacks to this, notably some work for maintainers. +* Interfaces are decoupled from the structs implementing them, which means the signature has to be repeated twice. +* Interfaces have to be documented and guarded at time of use, that 3rd party implementations aren't supported. +* As of Golang 1.21, interfaces are still [not well supported](https://github.com/golang/go/issues/5860) in godoc. + +## Config + +wazero configures scopes such as Runtime and Module using `XxxConfig` types. For example, `RuntimeConfig` configures +`Runtime` and `ModuleConfig` configure `Module` (instantiation). In all cases, config types begin defaults and can be +customized by a user, e.g., selecting features or a module name override. + +### Why don't we make each configuration setting return an error? +No config types create resources that would need to be closed, nor do they return errors on use. This helps reduce +resource leaks, and makes chaining easier. It makes it possible to parse configuration (ex by parsing yaml) independent +of validating it. + +Instead of: +``` +cfg, err = cfg.WithFS(fs) +if err != nil { + return err +} +cfg, err = cfg.WithName(name) +if err != nil { + return err +} +mod, err = rt.InstantiateModuleWithConfig(ctx, code, cfg) +if err != nil { + return err +} +``` + +There's only one call site to handle errors: +``` +cfg = cfg.WithFS(fs).WithName(name) +mod, err = rt.InstantiateModuleWithConfig(ctx, code, cfg) +if err != nil { + return err +} +``` + +This allows users one place to look for errors, and also the benefit that if anything internally opens a resource, but +errs, there's nothing they need to close. In other words, users don't need to track which resources need closing on +partial error, as that is handled internally by the only code that can read configuration fields. + +### Why are configuration immutable? +While it seems certain scopes like `Runtime` won't repeat within a process, they do, possibly in different goroutines. +For example, some users create a new runtime for each module, and some re-use the same base module configuration with +only small updates (ex the name) for each instantiation. Making configuration immutable allows them to be safely used in +any goroutine. + +Since config are immutable, changes apply via return val, similar to `append` in a slice. + +For example, both of these are the same sort of error: +```go +append(slice, element) // bug as only the return value has the updated slice. +cfg.WithName(next) // bug as only the return value has the updated name. +``` + +Here's an example of correct use: re-assigning explicitly or via chaining. +```go +cfg = cfg.WithName(name) // explicit + +mod, err = rt.InstantiateModuleWithConfig(ctx, code, cfg.WithName(name)) // implicit +if err != nil { + return err +} +``` + +### Why aren't configuration assigned with option types? +The option pattern is a familiar one in Go. For example, someone defines a type `func (x X) err` and uses it to update +the target. For example, you could imagine wazero could choose to make `ModuleConfig` from options vs chaining fields. + +Ex instead of: +```go +type ModuleConfig interface { + WithName(string) ModuleConfig + WithFS(fs.FS) ModuleConfig +} + +struct moduleConfig { + name string + fs fs.FS +} + +func (c *moduleConfig) WithName(name string) ModuleConfig { + ret := *c // copy + ret.name = name + return &ret +} + +func (c *moduleConfig) WithFS(fs fs.FS) ModuleConfig { + ret := *c // copy + ret.setFS("/", fs) + return &ret +} + +config := r.NewModuleConfig().WithFS(fs) +configDerived := config.WithName("name") +``` + +An option function could be defined, then refactor each config method into an name prefixed option function: +```go +type ModuleConfig interface { +} +struct moduleConfig { + name string + fs fs.FS +} + +type ModuleConfigOption func(c *moduleConfig) + +func ModuleConfigName(name string) ModuleConfigOption { + return func(c *moduleConfig) { + c.name = name + } +} + +func ModuleConfigFS(fs fs.FS) ModuleConfigOption { + return func(c *moduleConfig) { + c.fs = fs + } +} + +func (r *runtime) NewModuleConfig(opts ...ModuleConfigOption) ModuleConfig { + ret := newModuleConfig() // defaults + for _, opt := range opts { + opt(&ret.config) + } + return ret +} + +func (c *moduleConfig) WithOptions(opts ...ModuleConfigOption) ModuleConfig { + ret := *c // copy base config + for _, opt := range opts { + opt(&ret.config) + } + return ret +} + +config := r.NewModuleConfig(ModuleConfigFS(fs)) +configDerived := config.WithOptions(ModuleConfigName("name")) +``` + +wazero took the path of the former design primarily due to: +* interfaces provide natural namespaces for their methods, which is more direct than functions with name prefixes. +* parsing config into function callbacks is more direct vs parsing config into a slice of functions to do the same. +* in either case derived config is needed and the options pattern is more awkward to achieve that. + +There are other reasons such as test and debug being simpler without options: the above list is constrained to conserve +space. It is accepted that the options pattern is common in Go, which is the main reason for documenting this decision. + +### Why aren't config types deeply structured? +wazero's configuration types cover the two main scopes of WebAssembly use: +* `RuntimeConfig`: This is the broadest scope, so applies also to compilation + and instantiation. e.g. This controls the WebAssembly Specification Version. +* `ModuleConfig`: This affects modules instantiated after compilation and what + resources are allowed. e.g. This defines how or if STDOUT is captured. This + also allows sub-configuration of `FSConfig`. + +These default to a flat definition each, with lazy sub-configuration only after +proven to be necessary. A flat structure is easier to work with and is also +easy to discover. Unlike the option pattern described earlier, more +configuration in the interface doesn't taint the package namespace, only +`ModuleConfig`. + +We default to a flat structure to encourage simplicity. If we eagerly broke out +all possible configurations into sub-types (e.g. ClockConfig), it would be hard +to notice configuration sprawl. By keeping the config flat, it is easy to see +the cognitive load we may be adding to our users. + +In other words, discomfort adding more configuration is a feature, not a bug. +We should only add new configuration rarely, and before doing so, ensure it +will be used. In fact, this is why we support using context fields for +experimental configuration. By letting users practice, we can find out if a +configuration was a good idea or not before committing to it, and potentially +sprawling our types. + +In reflection, this approach worked well for the nearly 1.5 year period leading +to version 1.0. We've only had to create a single sub-configuration, `FSConfig`, +and it was well understood why when it occurred. + +## Why does `ModuleConfig.WithStartFunctions` default to `_start`? + +We formerly had functions like `StartWASICommand` that would verify +preconditions and start WASI's `_start` command. However, this caused confusion +because both many languages compiled a WASI dependency, and many did so +inconsistently. + +The conflict is that exported functions need to use features the language +runtime provides, such as garbage collection. There's a "chicken-egg problem" +where `_start` needs to complete in order for exported behavior to work. + +For example, unlike `GOOS=wasip1` in Go 1.21, TinyGo's "wasi" target supports +function exports. So, the only way to use FFI style is via the "wasi" target. +Not explicitly calling `_start` before an ABI such as wapc-go, would crash, due +to setup not happening (e.g. to implement `panic`). Other embedders such as +Envoy also called `_start` for the same reason. To avoid a common problem for +users unaware of WASI, and also to simplify normal use of WASI (e.g. `main`), +we added `_start` to `ModuleConfig.WithStartFunctions`. + +In cases of multiple initializers, such as in wapc-go, users can override this +to add the others *after* `_start`. Users who want to explicitly control +`_start`, such as some of our unit tests, can clear the start functions and +remove it. + +This decision was made in 2022, and holds true in 2023, even with the +introduction of "wasix". It holds because "wasix" is backwards compatible with +"wasip1". In the future, there will be other ways to start applications, and +may not be backwards compatible with "wasip1". + +Most notably WASI "Preview 2" is not implemented in a way compatible with +wasip1. Its start function is likely to be different, and defined in the +wasi-cli "world". When the design settles, and it is implemented by compilers, +wazero will attempt to support "wasip2". However, it won't do so in a way that +breaks existing compilers. + +In other words, we won't remove `_start` if "wasip2" continues a path of an +alternate function name. If we did, we'd break existing users despite our +compatibility promise saying we don't. The most likely case is that when we +build-in something incompatible with "wasip1", that start function will be +added to the start functions list in addition to `_start`. + +See http://wasix.org +See https://github.com/WebAssembly/wasi-cli + +## Runtime == Engine+Store +wazero defines a single user-type which combines the specification concept of `Store` with the unspecified `Engine` +which manages them. + +### Why not multi-store? +Multi-store isn't supported as the extra tier complicates lifecycle and locking. Moreover, in practice it is unusual for +there to be an engine that has multiple stores which have multiple modules. More often, it is the case that there is +either 1 engine with 1 store and multiple modules, or 1 engine with many stores, each having 1 non-host module. In worst +case, a user can use multiple runtimes until "multi-store" is better understood. + +If later, we have demand for multiple stores, that can be accomplished by overload. e.g. `Runtime.InstantiateInStore` or +`Runtime.Store(name) Store`. + +## Exit + +### Why do we only return a `sys.ExitError` on a non-zero exit code? + +It is reasonable to think an exit error should be returned, even if the code is +success (zero). Even on success, the module is no longer functional. For +example, function exports would error later. However, wazero does not. The only +time `sys.ExitError` is on error (non-zero). + +This decision was to improve performance and ergonomics for guests that both +use WASI (have a `_start` function), and also allow custom exports. +Specifically, Rust, TinyGo and normal wasi-libc, don't exit the module during +`_start`. If they did, it would invalidate their function exports. This means +it is unlikely most compilers will change this behavior. + +`GOOS=waspi1` from Go 1.21 does exit during `_start`. However, it doesn't +support other exports besides `_start`, and `_start` is not defined to be +called multiple times anyway. + +Since `sys.ExitError` is not always returned, we added `Module.IsClosed` for +defensive checks. This helps integrators avoid calling functions which will +always fail. + +### Why panic with `sys.ExitError` after a host function exits? + +Currently, the only portable way to stop processing code is via panic. For +example, WebAssembly "trap" instructions, such as divide by zero, are +implemented via panic. This ensures code isn't executed after it. + +When code reaches the WASI `proc_exit` instruction, we need to stop processing. +Regardless of the exit code, any code invoked after exit would be in an +inconsistent state. This is likely why unreachable instructions are sometimes +inserted after exit: https://github.com/emscripten-core/emscripten/issues/12322 + +## WASI + +Unfortunately, (WASI Snapshot Preview 1)[https://github.com/WebAssembly/WASI/blob/snapshot-01/phases/snapshot/docs.md] is not formally defined enough, and has APIs with ambiguous semantics. +This section describes how Wazero interprets and implements the semantics of several WASI APIs that may be interpreted differently by different wasm runtimes. +Those APIs may affect the portability of a WASI application. + +### Why don't we attempt to pass wasi-testsuite on user-defined `fs.FS`? + +While most cases work fine on an `os.File` based implementation, we won't +promise wasi-testsuite compatibility on user defined wrappers of `os.DirFS`. +The only option for real systems is to use our `sysfs.FS`. + +There are a lot of areas where windows behaves differently, despite the +`os.File` abstraction. This goes well beyond file locking concerns (e.g. +`EBUSY` errors on open files). For example, errors like `ACCESS_DENIED` aren't +properly mapped to `EPERM`. There are trickier parts too. `FileInfo.Sys()` +doesn't return enough information to build inodes needed for WASI. To rebuild +them requires the full path to the underlying file, not just its directory +name, and there's no way for us to get that information. At one point we tried, +but in practice things became tangled and functionality such as read-only +wrappers became untenable. Finally, there are version-specific behaviors which +are difficult to maintain even in our own code. For example, go 1.20 opens +files in a different way than versions before it. + +### Why aren't WASI rules enforced? + +The [snapshot-01](https://github.com/WebAssembly/WASI/blob/snapshot-01/phases/snapshot/docs.md) version of WASI has a +number of rules for a "command module", but only the memory export rule is enforced. If a "_start" function exists, it +is enforced to be the correct signature and succeed, but the export itself isn't enforced. It follows that this means +exports are not required to be contained to a "_start" function invocation. Finally, the "__indirect_function_table" +export is also not enforced. + +The reason for the exceptions are that implementations aren't following the rules. For example, TinyGo doesn't export +"__indirect_function_table", so crashing on this would make wazero unable to run TinyGo modules. Similarly, modules +loaded by wapc-go don't always define a "_start" function. Since "snapshot-01" is not a proper version, and certainly +not a W3C recommendation, there's no sense in breaking users over matters like this. + +### Why is I/O configuration not coupled to WASI? + +WebAssembly System Interfaces (WASI) is a formalization of a practice that can be done anyway: Define a host function to +access a system interface, such as writing to STDOUT. WASI stalled at snapshot-01 and as of early 2023, is being +rewritten entirely. + +This instability implies a need to transition between WASI specs, which places wazero in a position that requires +decoupling. For example, if code uses two different functions to call `fd_write`, the underlying configuration must be +centralized and decoupled. Otherwise, calls using the same file descriptor number will end up writing to different +places. + +In short, wazero defined system configuration in `ModuleConfig`, not a WASI type. This allows end-users to switch from +one spec to another with minimal impact. This has other helpful benefits, as centralized resources are simpler to close +coherently (ex via `Module.Close`). + +In reflection, this worked well as more ABI became usable in wazero. + +### Background on `ModuleConfig` design + +WebAssembly 1.0 (20191205) specifies some aspects to control isolation between modules ([sandboxing](https://en.wikipedia.org/wiki/Sandbox_(computer_security))). +For example, `wasm.Memory` has size constraints and each instance of it is isolated from each other. While `wasm.Memory` +can be shared, by exporting it, it is not exported by default. In fact a WebAssembly Module (Wasm) has no memory by +default. + +While memory is defined in WebAssembly 1.0 (20191205), many aspects are not. Let's use an example of `exec.Cmd` as for +example, a WebAssembly System Interfaces (WASI) command is implemented as a module with a `_start` function, and in many +ways acts similar to a process with a `main` function. + +To capture "hello world" written to the console (stdout a.k.a. file descriptor 1) in `exec.Cmd`, you would set the +`Stdout` field accordingly, perhaps to a buffer. In WebAssembly 1.0 (20191205), the only way to perform something like +this is via a host function (ex `HostModuleFunctionBuilder`) and internally copy memory corresponding to that string +to a buffer. + +WASI implements system interfaces with host functions. Concretely, to write to console, a WASI command `Module` imports +"fd_write" from "wasi_snapshot_preview1" and calls it with the `fd` parameter set to 1 (STDOUT). + +The [snapshot-01](https://github.com/WebAssembly/WASI/blob/snapshot-01/phases/snapshot/docs.md) version of WASI has no +means to declare configuration, although its function definitions imply configuration for example if fd 1 should exist, +and if so where should it write. Moreover, snapshot-01 was last updated in late 2020 and the specification is being +completely rewritten as of early 2022. This means WASI as defined by "snapshot-01" will not clarify aspects like which +file descriptors are required. While it is possible a subsequent version may, it is too early to tell as no version of +WASI has reached a stage near W3C recommendation. Even if it did, module authors are not required to only use WASI to +write to console, as they can define their own host functions, such as they did before WASI existed. + +wazero aims to serve Go developers as a primary function, and help them transition between WASI specifications. In +order to do this, we have to allow top-level configuration. To ensure isolation by default, `ModuleConfig` has WithXXX +that override defaults to no-op or empty. One `ModuleConfig` instance is used regardless of how many times the same WASI +functions are imported. The nil defaults allow safe concurrency in these situations, as well lower the cost when they +are never used. Finally, a one-to-one mapping with `Module` allows the module to close the `ModuleConfig` instead of +confusing users with another API to close. + +Naming, defaults and validation rules of aspects like `STDIN` and `Environ` are intentionally similar to other Go +libraries such as `exec.Cmd` or `syscall.SetEnv`, and differences called out where helpful. For example, there's no goal +to emulate any operating system primitive specific to Windows (such as a 'c:\' drive). Moreover, certain defaults +working with real system calls are neither relevant nor safe to inherit: For example, `exec.Cmd` defaults to read STDIN +from a real file descriptor ("/dev/null"). Defaulting to this, vs reading `io.EOF`, would be unsafe as it can exhaust +file descriptors if resources aren't managed properly. In other words, blind copying of defaults isn't wise as it can +violate isolation or endanger the embedding process. In summary, we try to be similar to normal Go code, but often need +act differently and document `ModuleConfig` is more about emulating, not necessarily performing real system calls. + +## File systems + +### Motivation on `sys.FS` + +The `sys.FS` abstraction in wazero was created because of limitations in +`fs.FS`, and `fs.File` in Go. Compilers targeting `wasip1` may access +functionality that writes new files. The ability to overcome this was requested +even before wazero was named this, via issue #21 in March 2021. + +A month later, golang/go#45757 was raised by someone else on the same topic. As +of July 2023, this has not resolved to a writeable file system abstraction. + +Over the next year more use cases accumulated, consolidated in March 2022 into +#390. This closed in January 2023 with a milestone of providing more +functionality, limited to users giving a real directory. This didn't yet expose +a file abstraction for general purpose use. Internally, this used `os.File`. +However, a wasm module instance is a virtual machine. Only supporting `os.File` +breaks sand-boxing use cases. Moreover, `os.File` is not an interface. Even +though this abstracts functionality, it does allow interception use cases. + +Hence, a few days later in January 2023, we had more issues asking to expose an +abstraction, #1013 and later #1532, on use cases like masking access to files. +In other words, the use case requests never stopped, and aren't solved by +exposing only real files. + +In summary, the primary motivation for exposing a replacement for `fs.FS` and +`fs.File` was around repetitive use case requests for years, around +interception and the ability to create new files, both virtual and real files. +While some use cases are solved with real files, not all are. Regardless, an +interface approach is necessary to ensure users can intercept I/O operations. + +### Why doesn't `sys.File` have a `Fd()` method? + +There are many features we could expose. We could make File expose underlying +file descriptors in case they are supported, for integration of system calls +that accept multiple ones, namely `poll` for multiplexing. This special case is +described in a subsequent section. + +As noted above, users have been asking for a file abstraction for over two +years, and a common answer was to wait. Making users wait is a problem, +especially so long. Good reasons to make people wait are stabilization. Edge +case features are not a great reason to hold abstractions from users. + +Another reason is implementation difficulty. Go did not attempt to abstract +file descriptors. For example, unlike `fs.ReadFile` there is no `fs.FdFile` +interface. Most likely, this is because file descriptors are an implementation +detail of common features. Programming languages, including Go, do not require +end users to know about file descriptors. Types such as `fs.File` can be used +without any knowledge of them. Implementations may or may not have file +descriptors. For example, in Go, `os.DirFS` has underlying file descriptors +while `embed.FS` does not. + +Despite this, some may want to expose a non-standard interface because +`os.File` has `Fd() uintptr` to return a file descriptor. Mainly, this is +handy to integrate with `syscall` package functions (on `GOOS` values that +declare them). Notice, though that `uintptr` is unsafe and not an abstraction. +Close inspection will find some `os.File` types internally use `poll.FD` +instead, yet this is not possible to use abstractly because that type is not +exposed. For example, `plan9` uses a different type than `poll.FD`. In other +words, even in real files, `Fd()` is not wholly portable, despite it being +useful on many operating systems with the `syscall` package. + +The reasons above, why Go doesn't abstract `FdFile` interface are a subset of +reasons why `sys.File` does not. If we exposed `File.Fd()` we not only would +have to declare all the edge cases that Go describes including impact of +finalizers, we would have to describe these in terms of virtualized files. +Then, we would have to reason with this value vs our existing virtualized +`sys.FileTable`, mapping whatever type we return to keys in that table, also +in consideration of garbage collection impact. The combination of issues like +this could lead down a path of not implementing a file system abstraction at +all, and instead a weak key mapped abstraction of the `syscall` package. Once +we finished with all the edge cases, we would have lost context of the original +reason why we started.. simply to allow file write access! + +When wazero attempts to do more than what the Go programming language team, it +has to be carefully evaluated, to: +* Be possible to implement at least for `os.File` backed files +* Not be confusing or cognitively hard for virtual file systems and normal use. +* Affordable: custom code is solely the responsible by the core team, a much + smaller group of individuals than who maintain the Go programming language. + +Due to problems well known in Go, consideration of the end users who constantly +ask for basic file system functionality, and the difficulty virtualizing file +descriptors at multiple levels, we don't expose `Fd()` and likely won't ever +expose `Fd()` on `sys.File`. + +### Why does `sys.File` have a `Poll()` method, while `sys.FS` does not? + +wazero exposes `File.Poll` which allows one-at-a-time poll use cases, +requested by multiple users. This not only includes abstract tests such as +Go 1.21 `GOOS=wasip1`, but real use cases including python and container2wasm +repls, as well listen sockets. The main use cases is non-blocking poll on a +single file. Being a single file, this has no risk of problems such as +head-of-line blocking, even when emulated. + +The main use case of multi-poll are bidirectional network services, something +not used in `GOOS=wasip1` standard libraries, but could be in the future. +Moving forward without a multi-poller allows wazero to expose its file system +abstraction instead of continuing to hold back it back for edge cases. We'll +continue discussion below regardless, as rationale was requested. + +You can loop through multiple `sys.File`, using `File.Poll` to see if an event +is ready, but there is a head-of-line blocking problem. If a long timeout is +used, bad luck could have a file that has nothing to read or write before one +that does. This could cause more blocking than necessary, even if you could +poll the others just after with a zero timeout. What's worse than this is if +unlimited blocking was used (`timeout=-1`). The host implementations could use +goroutines to avoid this, but interrupting a "forever" poll is problematic. All +of these are reasons to consider a multi-poll API, but do not require exporting +`File.Fd()`. + +Should multi-poll becomes critical, `sys.FS` could expose a `Poll` function +like below, despite it being the non-portable, complicated if possible to +implement on all platforms and virtual file systems. +```go +ready, errno := fs.Poll([]sys.PollFile{{f1, sys.POLLIN}, {f2, sys.POLLOUT}}, timeoutMillis) +``` + +A real filesystem could handle this by using an approach like the internal +`unix.Poll` function in Go, passing file descriptors on unix platforms, or +returning `sys.ENOSYS` for unsupported operating systems. Implementation for +virtual files could have a strategy around timeout to avoid the worst case of +head-of-line blocking (unlimited timeout). + +Let's remember that when designing abstractions, it is not best to add an +interface for everything. Certainly, Go doesn't, as evidenced by them not +exposing `poll.FD` in `os.File`! Such a multi-poll could be limited to +built-in filesystems in the wazero repository, avoiding complexity of trying to +support and test this abstractly. This would still permit multiplexing for CLI +users, and also permit single file polling as exists now. + +### Why doesn't wazero implement the working directory? + +An early design of wazero's API included a `WithWorkDirFS` which allowed +control over which file a relative path such as "./config.yml" resolved to, +independent of the root file system. This intended to help separate concerns +like mutability of files, but it didn't work and was removed. + +Compilers that target wasm act differently with regard to the working +directory. For example, wasi-libc, used by TinyGo, +tracks working directory changes in compiled wasm instead: initially "/" until +code calls `chdir`. Zig assumes the first pre-opened file descriptor is the +working directory. + +The only place wazero can standardize a layered concern is via a host function. +Since WASI doesn't use host functions to track the working directory, we can't +standardize the storage and initial value of it. + +Meanwhile, code may be able to affect the working directory by compiling +`chdir` into their main function, using an argument or ENV for the initial +value (possibly `PWD`). Those unable to control the compiled code should only +use absolute paths in configuration. + +See +* https://github.com/golang/go/blob/go1.20/src/syscall/fs_js.go#L324 +* https://github.com/WebAssembly/wasi-libc/pull/214#issue-673090117 +* https://github.com/ziglang/zig/blob/53a9ee699a35a3d245ab6d1dac1f0687a4dcb42c/src/main.zig#L32 + +### Why ignore the error returned by io.Reader when n > 1? + +Per https://pkg.go.dev/io#Reader, if we receive an error, any bytes read should +be processed first. At the syscall abstraction (`fd_read`), the caller is the +processor, so we can't process the bytes inline and also return the error (as +`EIO`). + +Let's assume we want to return the bytes read on error to the caller. This +implies we at least temporarily ignore the error alongside them. The choice +remaining is whether to persist the error returned with the read until a +possible next call, or ignore the error. + +If we persist an error returned, it would be coupled to a file descriptor, but +effectively it is boolean as this case coerces to `EIO`. If we track a "last +error" on a file descriptor, it could be complicated for a couple reasons +including whether the error is transient or permanent, or if the error would +apply to any FD operation, or just read. Finally, there may never be a +subsequent read as perhaps the bytes leading up to the error are enough to +satisfy the processor. + +This decision boils down to whether or not to track an error bit per file +descriptor or not. If not, the assumption is that a subsequent operation would +also error, this time without reading any bytes. + +The current opinion is to go with the simplest path, which is to return the +bytes read and ignore the error the there were any. Assume a subsequent +operation will err if it needs to. This helps reduce the complexity of the code +in wazero and also accommodates the scenario where the bytes read are enough to +satisfy its processor. + +### File descriptor allocation strategy + +File descriptor allocation currently uses a strategy similar the one implemented +by unix systems: when opening a file, the lowest unused number is picked. + +The WASI standard documents that programs cannot expect that file descriptor +numbers will be allocated with a lowest-first strategy, and they should instead +assume the values will be random. Since _random_ is a very imprecise concept in +computers, we technically satisfying the implementation with the descriptor +allocation strategy we use in Wazero. We could imagine adding more _randomness_ +to the descriptor selection process, however this should never be used as a +security measure to prevent applications from guessing the next file number so +there are no strong incentives to complicate the logic. + +### Why does `FSConfig.WithDirMount` not match behaviour with `os.DirFS`? + +It may seem that we should require any feature that seems like a standard +library in Go, to behave the same way as the standard library. Doing so would +present least surprise to Go developers. In the case of how we handle +filesystems, we break from that as it is incompatible with the expectations of +WASI, the most commonly implemented filesystem ABI. + +The main reason is that `os.DirFS` is a virtual filesystem abstraction while +WASI is an abstraction over syscalls. For example, the signature of `fs.Open` +does not permit use of flags. This creates conflict on what default behaviors +to take when Go implemented `os.DirFS`. On the other hand, `path_open` can pass +flags, and in fact tests require them to be honored in specific ways. + +This conflict requires us to choose what to be more compatible with, and which +type of user to surprise the least. We assume there will be more developers +compiling code to wasm than developers of custom filesystem plugins, and those +compiling code to wasm will be better served if we are compatible with WASI. +Hence on conflict, we prefer WASI behavior vs the behavior of `os.DirFS`. + +See https://github.com/WebAssembly/wasi-testsuite +See https://github.com/golang/go/issues/58141 + +## Why is our `Readdir` function more like Go's `os.File` than POSIX `readdir`? + +At one point we attempted to move from a bulk `Readdir` function to something +more like the POSIX `DIR` struct, exposing functions like `telldir`, `seekdir` +and `readdir`. However, we chose the design more like `os.File.Readdir`, +because it performs and fits wasip1 better. + +### wasip1/wasix + +`fd_readdir` in wasip1 (and so also wasix) is like `getdents` in Linux, not +`readdir` in POSIX. `getdents` is more like Go's `os.File.Readdir`. + +We currently have an internal type `sys.DirentCache` which only is used by +wasip1 or wasix. When `HostModuleBuilder` adds support for instantiation state, +we could move this to the `wasi_snapshot_preview1` package. Meanwhile, all +filesystem code is internal anyway, so this special-case is acceptable. + +### wasip2 + +`directory-entry-stream` in wasi-filesystem preview2 is defined in component +model, not an ABI, but in wasmtime it is a consuming iterator. A consuming +iterator is easy to support with anything (like `Readdir(1)`), even if it is +inefficient as you can neither bulk read nor skip. The implementation of the +preview1 adapter (uses preview2) confirms this. They use a dirent cache similar +in some ways to our `sysfs.DirentCache`. As there is no seek concept in +preview2, they interpret the cookie as numeric and read on repeat entries when +a cache wasn't available. Note: we currently do not skip-read like this as it +risks buffering large directories, and no user has requested entries before the +cache, yet. + +Regardless, wasip2 is not complete until the end of 2023. We can defer design +discussion until after it is stable and after the reference impl wasmtime +implements it. + +See + * https://github.com/WebAssembly/wasi-filesystem/blob/ef9fc87c07323a6827632edeb6a7388b31266c8e/example-world.md#directory_entry_stream + * https://github.com/bytecodealliance/wasmtime/blob/b741f7c79d72492d17ab8a29c8ffe4687715938e/crates/wasi/src/preview2/preview2/filesystem.rs#L286-L296 + * https://github.com/bytecodealliance/preview2-prototyping/blob/e4c04bcfbd11c42c27c28984948d501a3e168121/crates/wasi-preview1-component-adapter/src/lib.rs#L2131-L2137 + * https://github.com/bytecodealliance/preview2-prototyping/blob/e4c04bcfbd11c42c27c28984948d501a3e168121/crates/wasi-preview1-component-adapter/src/lib.rs#L936 + +### wasip3 + +`directory-entry-stream` is documented to change significantly in wasip3 moving +from synchronous to synchronous streams. This is dramatically different than +POSIX `readdir` which is synchronous. + +Regardless, wasip3 is not complete until after wasip2, which means 2024 or +later. We can defer design discussion until after it is stable and after the +reference impl wasmtime implements it. + +See + * https://github.com/WebAssembly/WASI/blob/ddfe3d1dda5d1473f37ecebc552ae20ce5fd319a/docs/WitInWasi.md#Streams + * https://docs.google.com/presentation/d/1MNVOZ8hdofO3tI0szg_i-Yoy0N2QPU2C--LzVuoGSlE/edit#slide=id.g1270ef7d5b6_0_662 + +### How do we implement `Pread` with an `fs.File`? + +`ReadAt` is the Go equivalent to `pread`: it does not affect, and is not +affected by, the underlying file offset. Unfortunately, `io.ReaderAt` is not +implemented by all `fs.File`. For example, as of Go 1.19, `embed.openFile` does +not. + +The initial implementation of `fd_pread` instead used `Seek`. To avoid a +regression, we fall back to `io.Seeker` when `io.ReaderAt` is not supported. + +This requires obtaining the initial file offset, seeking to the intended read +offset, and resetting the file offset the initial state. If this final seek +fails, the file offset is left in an undefined state. This is not thread-safe. + +While seeking per read seems expensive, the common case of `embed.openFile` is +only accessing a single int64 field, which is cheap. + +### Pre-opened files + +WASI includes `fd_prestat_get` and `fd_prestat_dir_name` functions used to +learn any directory paths for file descriptors open at initialization time. + +For example, `__wasilibc_register_preopened_fd` scans any file descriptors past +STDERR (1) and invokes `fd_prestat_dir_name` to learn any path prefixes they +correspond to. Zig's `preopensAlloc` does similar. These pre-open functions are +not used again after initialization. + +wazero supports stdio pre-opens followed by any mounts e.g `.:/`. The guest +path is a directory and its name, e.g. "/" is returned by `fd_prestat_dir_name` +for file descriptor 3 (STDERR+1). The first longest match wins on multiple +pre-opens, which allows a path like "/tmp" to match regardless of order vs "/". + +See + * https://github.com/WebAssembly/wasi-libc/blob/a02298043ff551ce1157bc2ee7ab74c3bffe7144/libc-bottom-half/sources/preopens.c + * https://github.com/ziglang/zig/blob/9cb06f3b8bf9ea6b5e5307711bc97328762d6a1d/lib/std/fs/wasi.zig#L50-L53 + +### fd_prestat_dir_name + +`fd_prestat_dir_name` is a WASI function to return the path of the pre-opened +directory of a file descriptor. It has the following three parameters, and the +third `path_len` has ambiguous semantics. + +* `fd`: a file descriptor +* `path`: the offset for the result path +* `path_len`: In wazero, `FdPrestatDirName` writes the result path string to + `path` offset for the exact length of `path_len`. + +Wasmer considers `path_len` to be the maximum length instead of the exact +length that should be written. +See https://github.com/wasmerio/wasmer/blob/3463c51268ed551933392a4063bd4f8e7498b0f6/lib/wasi/src/syscalls/mod.rs#L764 + +The semantics in wazero follows that of wasmtime. +See https://github.com/bytecodealliance/wasmtime/blob/2ca01ae9478f199337cf743a6ab543e8c3f3b238/crates/wasi-common/src/snapshots/preview_1.rs#L578-L582 + +Their semantics match when `path_len` == the length of `path`, so in practice +this difference won't matter match. + +## fd_readdir + +### Why does "wasi_snapshot_preview1" require dot entries when POSIX does not? + +In October 2019, WASI project knew requiring dot entries ("." and "..") was not +documented in preview1, not required by POSIX and problematic to synthesize. +For example, Windows runtimes backed by `FindNextFileW` could not return these. +A year later, the tag representing WASI preview 1 (`snapshot-01`) was made. +This did not include the requested change of making dot entries optional. + +The `phases/snapshot/docs.md` document was altered in subsequent years in +significant ways, often in lock-step with wasmtime or wasi-libc. In January +2022, `sock_accept` was added to `phases/snapshot/docs.md`, a document later +renamed to later renamed to `legacy/preview1/docs.md`. + +As a result, the ABI and behavior remained unstable: The `snapshot-01` tag was +not an effective basis of portability. A test suite was requested well before +this tag, in April 2019. Meanwhile, compliance had no meaning. Developers had +to track changes to the latest doc, while clarifying with wasi-libc or wasmtime +behavior. This lack of stability could have permitted a fix to the dot entries +problem, just as it permitted changes desired by other users. + +In November 2022, the wasi-testsuite project began and started solidifying +expectations. This quickly led to changes in runtimes and the spec doc. WASI +began importing tests from wasmtime as required behaviors for all runtimes. +Some changes implied changes to wasi-libc. For example, `readdir` began to +imply inode fan-outs, which caused performance regressions. Most notably a +test merged in January required dot entries. Tests were merged without running +against any runtime, and even when run ad-hoc only against Linux. Hence, +portability issues mentioned over three years earlier did not trigger any +failure until wazero (which tests Windows) noticed. + +In the same month, wazero requested to revert this change primarily because +Go does not return them from `os.ReadDir`, and materializing them is +complicated due to tests also requiring inodes. Moreover, they are discarded by +not just Go, but other common programming languages. This was rejected by the +WASI lead for preview1, but considered for the completely different ABI named +preview2. + +In February 2023, the WASI chair declared that new rule requiring preview1 to +return dot entries "was decided by the subgroup as a whole", citing meeting +notes. According to these notes, the WASI lead stated incorrectly that POSIX +conformance required returning dot entries, something it explicitly says are +optional. In other words, he said filtering them out would make Preview1 +non-conforming, and asked if anyone objects to this. The co-chair was noted to +say "Because there are existing P1 programs, we shouldn’t make changes like +this." No other were recorded to say anything. + +In summary, preview1 was changed retrospectively to require dot entries and +preview2 was changed to require their absence. This rule was reverse engineered +from wasmtime tests, and affirmed on two false premises: + +* POSIX compliance requires dot entries + * POSIX literally says these are optional +* WASI cannot make changes because there are existing P1 programs. + * Changes to Preview 1 happened before and after this topic. + +As of June 2023, wasi-testsuite still only runs on Linux, so compliance of this +rule on Windows is left to runtimes to decide to validate. The preview2 adapter +uses fake cookies zero and one to refer to dot dirents, uses a real inode for +the dot(".") entry and zero inode for dot-dot(".."). + +See https://github.com/WebAssembly/wasi-filesystem/issues/3 +See https://github.com/WebAssembly/WASI/tree/snapshot-01 +See https://github.com/WebAssembly/WASI/issues/9 +See https://github.com/WebAssembly/WASI/pull/458 +See https://github.com/WebAssembly/wasi-testsuite/pull/32 +See https://github.com/WebAssembly/wasi-libc/pull/345 +See https://github.com/WebAssembly/wasi-testsuite/issues/52 +See https://github.com/WebAssembly/WASI/pull/516 +See https://github.com/WebAssembly/meetings/blob/main/wasi/2023/WASI-02-09.md#should-preview1-fd_readdir-filter-out--and- +See https://github.com/bytecodealliance/preview2-prototyping/blob/e4c04bcfbd11c42c27c28984948d501a3e168121/crates/wasi-preview1-component-adapter/src/lib.rs#L1026-L1041 + +### Why are dot (".") and dot-dot ("..") entries problematic? + +When reading a directory, dot (".") and dot-dot ("..") entries are problematic. +For example, Go does not return them from `os.ReadDir`, and materializing them +is complicated (at least dot-dot is). + +A directory entry has stat information in it. The stat information includes +inode which is used for comparing file equivalence. In the simple case of dot, +we could materialize a special entry to expose the same info as stat on the fd +would return. However, doing this and not doing dot-dot would cause confusion, +and dot-dot is far more tricky. To back-fill inode information about a parent +directory would be costly and subtle. For example, the pre-open (mount) of the +directory may be different than its logical parent. This is easy to understand +when considering the common case of mounting "/" and "/tmp" as pre-opens. To +implement ".." from "/tmp" requires information from a separate pre-open, this +includes state to even know the difference. There are easier edge cases as +well, such as the decision to not return ".." from a root path. In any case, +this should start to explain that faking entries when underlying stdlib doesn't +return them is tricky and requires quite a lot of state. + +Another issue is around the `Dirent.Off` value of a directory entry, sometimes +called a "cookie" in Linux man pagers. When the host operating system or +library function does not return dot entries, to support functions such as +`seekdir`, you still need a value for `Dirent.Off`. Naively, you can synthesize +these by choosing sequential offsets zero and one. However, POSIX strictly says +offsets should be treated opaquely. The backing filesystem could use these to +represent real entries. For example, a directory with one entry could use zero +as the `Dirent.Off` value. If you also used zero for the "." dirent, there +would be a clash. This means if you synthesize `Dirent.Off` for any entry, you +need to synthesize this value for all entries. In practice, the simplest way is +using an incrementing number, such as done in the WASI preview2 adapter. + +Working around these issues causes expense to all users of wazero, so we'd +then look to see if that would be justified or not. However, the most common +compilers involved in end user questions, as of early 2023 are TinyGo, Rust and +Zig. All of these compile code which ignores dot and dot-dot entries. In other +words, faking these entries would not only cost our codebase with complexity, +but it would also add unnecessary overhead as the values aren't commonly used. + +The final reason why we might do this, is an end users or a specification +requiring us to. As of early 2023, no end user has raised concern over Go and +by extension wazero not returning dot and dot-dot. The snapshot-01 spec of WASI +does not mention anything on this point. Also, POSIX has the following to say, +which summarizes to "these are optional" + +> The readdir() function shall not return directory entries containing empty names. If entries for dot or dot-dot exist, one entry shall be returned for dot and one entry shall be returned for dot-dot; otherwise, they shall not be returned. + +Unfortunately, as described above, the WASI project decided in early 2023 to +require dot entries in both the spec and the wasi-testsuite. For only this +reason, wazero adds overhead to synthesize dot entries despite it being +unnecessary for most users. + +See https://pubs.opengroup.org/onlinepubs/9699919799/functions/readdir.html +See https://github.com/golang/go/blob/go1.20/src/os/dir_unix.go#L108-L110 +See https://github.com/bytecodealliance/preview2-prototyping/blob/e4c04bcfbd11c42c27c28984948d501a3e168121/crates/wasi-preview1-component-adapter/src/lib.rs#L1026-L1041 + +### Why don't we pre-populate an inode for the dot-dot ("..") entry? + +We only populate an inode for dot (".") because wasi-testsuite requires it, and +we likely already have it (because we cache it). We could attempt to populate +one for dot-dot (".."), but chose not to. + +Firstly, wasi-testsuite does not require the inode of dot-dot, possibly because +the wasip2 adapter doesn't populate it (but we don't really know why). + +The only other reason to populate it would be to avoid wasi-libc's stat fanout +when it is missing. However, wasi-libc explicitly doesn't fan-out to lstat on +the ".." entry on a zero ino. + +Fetching dot-dot's inode despite the above not only doesn't help wasi-libc, but +it also hurts languages that don't use it, such as Go. These languages would +pay a stat syscall penalty even if they don't need the inode. In fact, Go +discards both dot entries! + +In summary, there are no significant upsides in attempting to pre-fetch +dot-dot's inode, and there are downsides to doing it anyway. + +See + * https://github.com/WebAssembly/wasi-libc/blob/bd950eb128bff337153de217b11270f948d04bb4/libc-bottom-half/cloudlibc/src/libc/dirent/readdir.c#L87-L94 + * https://github.com/WebAssembly/wasi-testsuite/blob/main/tests/rust/src/bin/fd_readdir.rs#L108 + * https://github.com/bytecodealliance/preview2-prototyping/blob/e4c04bcfbd11c42c27c28984948d501a3e168121/crates/wasi-preview1-component-adapter/src/lib.rs#L1037 + +### Why don't we require inodes to be non-zero? + +We don't require a non-zero value for `Dirent.Ino` because doing so can prevent +a real one from resolving later via `Stat_t.Ino`. + +We define `Ino` like `d_ino` in POSIX which doesn't special-case zero. It can +be zero for a few reasons: + +* The file is not a regular file or directory. +* The underlying filesystem does not support inodes. e.g. embed:fs +* A directory doesn't include inodes, but a later stat can. e.g. Windows +* The backend is based on wasi-filesystem (a.k.a wasip2), which has + `directory_entry.inode` optional, and might remove it entirely. + +There are other downsides to returning a zero inode in widely used compilers: + +* File equivalence utilities, like `os.SameFile` will not work. +* wasi-libc's `wasip1` mode will call `lstat` and attempt to retrieve a + non-zero value (unless the entry is named ".."). + +A new compiler may accidentally skip a `Dirent` with a zero `Ino` if emulating +a non-POSIX function and re-using `Dirent.Ino` for `d_fileno`. + +* Linux `getdents` doesn't define `d_fileno` must be non-zero +* BSD `getdirentries` is implementation specific. For example, OpenBSD will + return dirents with a zero `d_fileno`, but Darwin will skip them. + +The above shouldn't be a problem, even in the case of BSD, because `wasip1` is +defined more in terms of `getdents` than `getdirentries`. The bottom half of +either should treat `wasip1` (or any similar ABI such as wasix or wasip2) as a +different operating system and either use different logic that doesn't skip, or +synthesize a fake non-zero `d_fileno` when `d_ino` is zero. + +However, this has been a problem. Go's `syscall.ParseDirent` utility is shared +for all `GOOS=unix`. For simplicity, this abstracts `direntIno` with data from +`d_fileno` or `d_ino`, and drops if either are zero, even if `d_fileno` is the +only field with zero explicitly defined. This led to a change to special case +`GOOS=wasip1` as otherwise virtual files would be unconditionally skipped. + +In practice, this problem is rather unique due to so many compilers relying on +wasi-libc, which tolerates a zero inode. For example, while issues were +reported about the performance regression when wasi-libc began doing a fan-out +on zero `Dirent.Ino`, no issues were reported about dirents being dropped as a +result. + +In summary, rather than complicating implementation and forcing non-zero inodes +for a rare case, we permit zero. We instead document this topic thoroughly, so +that emerging compilers can re-use the research and reference it on conflict. +We also document that `Ino` should be non-zero, so that users implementing that +field will attempt to get it. + +See + * https://github.com/WebAssembly/wasi-filesystem/pull/81 + * https://github.com/WebAssembly/wasi-libc/blob/bd950eb128bff337153de217b11270f948d04bb4/libc-bottom-half/cloudlibc/src/libc/dirent/readdir.c#L87-L94 + * https://linux.die.net/man/3/getdents + * https://www.unix.com/man-page/osx/2/getdirentries/ + * https://man.openbsd.org/OpenBSD-5.4/getdirentries.2 + * https://github.com/golang/go/blob/go1.20/src/syscall/dirent.go#L60-L102 + * https://go-review.googlesource.com/c/go/+/507915 + +## sys.Walltime and Nanotime + +The `sys` package has two function types, `Walltime` and `Nanotime` for real +and monotonic clock exports. The naming matches conventions used in Go. + +```go +func time_now() (sec int64, nsec int32, mono int64) { + sec, nsec = walltime() + return sec, nsec, nanotime() +} +``` + +Splitting functions for wall and clock time allow implementations to choose +whether to implement the clock once (as in Go), or split them out. + +Each can be configured with a `ClockResolution`, although is it usually +incorrect as detailed in a sub-heading below. The only reason for exposing this +is to satisfy WASI: + +See https://github.com/WebAssembly/wasi-clocks + +### Why default to fake time? + +WebAssembly has an implicit design pattern of capabilities based security. By +defaulting to a fake time, we reduce the chance of timing attacks, at the cost +of requiring configuration to opt-into real clocks. + +See https://gruss.cc/files/fantastictimers.pdf for an example attacks. + +### Why does fake time increase on reading? + +Both the fake nanotime and walltime increase by 1ms on reading. Particularly in +the case of nanotime, this prevents spinning. + +### Why not `time.Clock`? + +wazero can't use `time.Clock` as a plugin for clock implementation as it is +only substitutable with build flags (`faketime`) and conflates wall and +monotonic time in the same call. + +Go's `time.Clock` was added monotonic time after the fact. For portability with +prior APIs, a decision was made to combine readings into the same API call. + +See https://go.googlesource.com/proposal/+/master/design/12914-monotonic.md + +WebAssembly time imports do not have the same concern. In fact even Go's +imports for clocks split walltime from nanotime readings. + +See https://github.com/golang/go/blob/go1.20/misc/wasm/wasm_exec.js#L243-L255 + +Finally, Go's clock is not an interface. WebAssembly users who want determinism +or security need to be able to substitute an alternative clock implementation +from the host process one. + +### `ClockResolution` + +A clock's resolution is hardware and OS dependent so requires a system call to retrieve an accurate value. +Go does not provide a function for getting resolution, so without CGO we don't have an easy way to get an actual +value. For now, we return fixed values of 1us for realtime and 1ns for monotonic, assuming that realtime clocks are +often lower precision than monotonic clocks. In the future, this could be improved by having OS+arch specific assembly +to make syscalls. + +For example, Go implements time.Now for linux-amd64 with this [assembly](https://github.com/golang/go/blob/go1.20/src/runtime/time_linux_amd64.s). +Because retrieving resolution is not generally called often, unlike getting time, it could be appropriate to only +implement the fallback logic that does not use VDSO (executing syscalls in user mode). The syscall for clock_getres +is 229 and should be usable. https://pkg.go.dev/syscall#pkg-constants. + +If implementing similar for Windows, [mingw](https://github.com/mirror/mingw-w64/blob/6a0e9165008f731bccadfc41a59719cf7c8efc02/mingw-w64-libraries/winpthreads/src/clock.c#L77 +) is often a good source to find the Windows API calls that correspond +to a POSIX method. + +Writing assembly would allow making syscalls without CGO, but comes with the cost that it will require implementations +across many combinations of OS and architecture. + +## sys.Nanosleep + +All major programming languages have a `sleep` mechanism to block for a +duration. Sleep is typically implemented by a WASI `poll_oneoff` relative clock +subscription. + +For example, the below ends up calling `wasi_snapshot_preview1.poll_oneoff`: + +```zig +const std = @import("std"); +pub fn main() !void { + std.time.sleep(std.time.ns_per_s * 5); +} +``` + +Besides Zig, this is also the case with TinyGo (`-target=wasi`) and Rust +(`--target wasm32-wasi`). + +We decided to expose `sys.Nanosleep` to allow overriding the implementation +used in the common case, even if it isn't used by Go, because this gives an +easy and efficient closure over a common program function. We also documented +`sys.Nanotime` to warn users that some compilers don't optimize sleep. + +## sys.Osyield + +We expose `sys.Osyield`, to allow users to control the behavior of WASI's +`sched_yield` without a new build of wazero. This is mainly for parity with +all other related features which we allow users to implement, including +`sys.Nanosleep`. Unlike others, we don't provide an out-of-box implementation +primarily because it will cause performance problems when accessed. + +For example, the below implementation uses CGO, which might result in a 1us +delay per invocation depending on the platform. + +See https://github.com/golang/go/issues/19409#issuecomment-284788196 +```go +//go:noescape +//go:linkname osyield runtime.osyield +func osyield() +``` + +In practice, a request to customize this is unlikely to happen until other +thread based functions are implemented. That said, as of early 2023, there are +a few signs of implementation interest and cross-referencing: + +See https://github.com/WebAssembly/stack-switching/discussions/38 +See https://github.com/WebAssembly/wasi-threads#what-can-be-skipped +See https://slinkydeveloper.com/Kubernetes-controllers-A-New-Hope/ + +## sys.Stat_t + +We expose `stat` information as `sys.Stat_t`, like `syscall.Stat_t` except +defined without build constraints. For example, you can use `sys.Stat_t` on +`GOOS=windows` which doesn't define `syscall.Stat_t`. + +The first use case of this is to return inodes from `fs.FileInfo` without +relying on platform-specifics. For example, a user could return `*sys.Stat_t` +from `info.Sys()` and define a non-zero inode for a virtual file, or map a +real inode to a virtual one. + +Notable choices per field are listed below, where `sys.Stat_t` is unlike +`syscall.Stat_t` on `GOOS=linux`, or needs clarification. One common issue +not repeated below is that numeric fields are 64-bit when at least one platform +defines it that large. Also, zero values are equivalent to nil or absent. + +* `Dev` and `Ino` (`Inode`) are both defined unsigned as they are defined + opaque, and most `syscall.Stat_t` also defined them unsigned. There are + separate sections in this document discussing the impact of zero in `Ino`. +* `Mode` is defined as a `fs.FileMode` even though that is not defined in POSIX + and will not map to all possible values. This is because the current use is + WASI, which doesn't define any types or features not already supported. By + using `fs.FileMode`, we can re-use routine experience in Go. +* `NLink` is unsigned because it is defined that way in `syscall.Stat_t`: there + can never be less than zero links to a file. We suggest defaulting to 1 in + conversions when information is not knowable because at least that many links + exist. +* `Size` is signed because it is defined that way in `syscall.Stat_t`: while + regular files and directories will always be non-negative, irregular files + are possibly negative or not defined. Notably sparse files are known to + return negative values. +* `Atim`, `Mtim` and `Ctim` are signed because they are defined that way in + `syscall.Stat_t`: Negative values are time before 1970. The resolution is + nanosecond because that's the maximum resolution currently supported in Go. + +### Why do we use `sys.EpochNanos` instead of `time.Time` or similar? + +To simplify documentation, we defined a type alias `sys.EpochNanos` for int64. +`time.Time` is a data structure, and we could have used this for +`syscall.Stat_t` time values. The most important reason we do not is conversion +penalty deriving time from common types. + +The most common ABI used in `wasip2`. This, and compatible ABI such as `wasix`, +encode timestamps in memory as a 64-bit number. If we used `time.Time`, we +would have to convert an underlying type like `syscall.Timespec` to `time.Time` +only to later have to call `.UnixNano()` to convert it back to a 64-bit number. + +In the future, the component model module "wasi-filesystem" may represent stat +timestamps with a type shared with "wasi-clocks", abstractly structured similar +to `time.Time`. However, component model intentionally does not define an ABI. +It is likely that the canonical ABI for timestamp will be in two parts, but it +is not required for it to be intermediately represented this way. A utility +like `syscall.NsecToTimespec` could split an int64 so that it could be written +to memory as 96 bytes (int64, int32), without allocating a struct. + +Finally, some may confuse epoch nanoseconds with 32-bit epoch seconds. While +32-bit epoch seconds has "The year 2038" problem, epoch nanoseconds has +"The Year 2262" problem, which is even less concerning for this library. If +the Go programming language and wazero exist in the 2200's, we can make a major +version increment to adjust the `sys.EpochNanos` approach. Meanwhile, we have +faster code. + +## poll_oneoff + +`poll_oneoff` is a WASI API for waiting for I/O events on multiple handles. +It is conceptually similar to the POSIX `poll(2)` syscall. +The name is not `poll`, because it references [“the fact that this function is not efficient +when used repeatedly with the same large set of handles”][poll_oneoff]. + +We chose to support this API in a handful of cases that work for regular files +and standard input. We currently do not support other types of file descriptors such +as socket handles. + +### Clock Subscriptions + +As detailed above in [sys.Nanosleep](#sysnanosleep), `poll_oneoff` handles +relative clock subscriptions. In our implementation we use `sys.Nanosleep()` +for this purpose in most cases, except when polling for interactive input +from `os.Stdin` (see more details below). + +### FdRead and FdWrite Subscriptions + +When subscribing a file descriptor (except `Stdin`) for reads or writes, +the implementation will generally return immediately with success, unless +the file descriptor is unknown. The file descriptor is not checked further +for new incoming data. Any timeout is cancelled, and the API call is able +to return, unless there are subscriptions to `Stdin`: these are handled +separately. + +### FdRead and FdWrite Subscription to Stdin + +Subscribing `Stdin` for reads (writes make no sense and cause an error), +requires extra care: wazero allows to configure a custom reader for `Stdin`. + +In general, if a custom reader is found, the behavior will be the same +as for regular file descriptors: data is assumed to be present and +a success is written back to the result buffer. + +However, if the reader is detected to read from `os.Stdin`, +a special code path is followed, invoking `sysfs.poll()`. + +`sysfs.poll()` is a wrapper for `poll(2)` on POSIX systems, +and it is emulated on Windows. + +### Poll on POSIX + +On POSIX systems, `poll(2)` allows to wait for incoming data on a file +descriptor, and block until either data becomes available or the timeout +expires. + +Usage of `syfs.poll()` is currently only reserved for standard input, because + +1. it is really only necessary to handle interactive input: otherwise, + there is no way in Go to peek from Standard Input without actually + reading (and thus consuming) from it; + +2. if `Stdin` is connected to a pipe, it is ok in most cases to return + with success immediately; + +3. `syfs.poll()` is currently a blocking call, irrespective of goroutines, + because the underlying syscall is; thus, it is better to limit its usage. + +So, if the subscription is for `os.Stdin` and the handle is detected +to correspond to an interactive session, then `sysfs.poll()` will be +invoked with a the `Stdin` handle *and* the timeout. + +This also means that in this specific case, the timeout is uninterruptible, +unless data becomes available on `Stdin` itself. + +### Select on Windows + +On Windows `sysfs.poll()` cannot be delegated to a single +syscall, because there is no single syscall to handle sockets, +pipes and regular files. + +Instead, we emulate its behavior for the cases that are currently +of interest. + +- For regular files, we _always_ report them as ready, as +[most operating systems do anyway][async-io-windows]. + +- For pipes, we invoke [`PeekNamedPipe`][peeknamedpipe] +for each file handle we detect is a pipe open for reading. +We currently ignore pipes open for writing. + +- Notably, we include also support for sockets using the [WinSock +implementation of `poll`][wsapoll], but instead +of relying on the timeout argument of the `WSAPoll` function, +we set a 0-duration timeout so that it behaves like a peek. + +This way, we can check for regular files all at once, +at the beginning of the function, then we poll pipes and +sockets periodically using a cancellable `time.Tick`, +which plays nicely with the rest of the Go runtime. + +### Impact of blocking + +Because this is a blocking syscall, it will also block the carrier thread of +the goroutine, preventing any means to support context cancellation directly. + +There are ways to obviate this issue. We outline here one idea, that is however +not currently implemented. A common approach to support context cancellation is +to add a signal file descriptor to the set, e.g. the read-end of a pipe or an +eventfd on Linux. When the context is canceled, we may unblock a Select call by +writing to the fd, causing it to return immediately. This however requires to +do a bit of housekeeping to hide the "special" FD from the end-user. + +[poll_oneoff]: https://github.com/WebAssembly/wasi-poll#why-is-the-function-called-poll_oneoff +[async-io-windows]: https://tinyclouds.org/iocp_links +[peeknamedpipe]: https://learn.microsoft.com/en-us/windows/win32/api/namedpipeapi/nf-namedpipeapi-peeknamedpipe +[wsapoll]: https://learn.microsoft.com/en-us/windows/win32/api/winsock2/nf-winsock2-wsapoll + +## Signed encoding of integer global constant initializers + +wazero treats integer global constant initializers signed as their interpretation is not known at declaration time. For +example, there is no signed integer [value type](https://www.w3.org/TR/2019/REC-wasm-core-1-20191205/#value-types%E2%91%A0). + +To get at the problem, let's use an example. +``` +(global (export "start_epoch") i64 (i64.const 1620216263544)) +``` + +In both signed and unsigned LEB128 encoding, this value is the same bit pattern. The problem is that some numbers are +not. For example, 16256 is `807f` encoded as unsigned, but `80ff00` encoded as signed. + +While the specification mentions uninterpreted integers are in abstract [unsigned values](https://www.w3.org/TR/2019/REC-wasm-core-1-20191205/#integers%E2%91%A0), +the binary encoding is clear that they are encoded [signed](https://www.w3.org/TR/2019/REC-wasm-core-1-20191205/#integers%E2%91%A4). + +For consistency, we go with signed encoding in the special case of global constant initializers. + +## Implementation limitations + +WebAssembly 1.0 (20191205) specification allows runtimes to [limit certain aspects of Wasm module or execution](https://www.w3.org/TR/2019/REC-wasm-core-1-20191205/#a2-implementation-limitations). + +wazero limitations are imposed pragmatically and described below. + +### Number of functions in a module + +The possible number of function instances in [a module](https://www.w3.org/TR/2019/REC-wasm-core-1-20191205/#module-instances%E2%91%A0) is not specified in the WebAssembly specifications since [`funcaddr`](https://www.w3.org/TR/2019/REC-wasm-core-1-20191205/#syntax-funcaddr) corresponding to a function instance in a store can be arbitrary number. +wazero limits the maximum function instances to 2^27 as even that number would occupy 1GB in function pointers. + +That is because not only we _believe_ that all use cases are fine with the limitation, but also we have no way to test wazero runtimes under these unusual circumstances. + +### Number of function types in a store + +There's no limitation on the number of function types in [a store](https://www.w3.org/TR/2019/REC-wasm-core-1-20191205/#store%E2%91%A0) according to the spec. In wazero implementation, we assign each function type to a unique ID, and choose to use `uint32` to represent the IDs. +Therefore the maximum number of function types a store can have is limited to 2^27 as even that number would occupy 512MB just to reference the function types. + +This is due to the same reason for the limitation on the number of functions above. + +### Number of values on the stack in a function + +While the the spec does not clarify a limitation of function stack values, wazero limits this to 2^27 = 134,217,728. +The reason is that we internally represent all the values as 64-bit integers regardless of its types (including f32, f64), and 2^27 values means +1 GiB = (2^30). 1 GiB is the reasonable for most applications [as we see a Goroutine has 250 MB as a limit on the stack for 32-bit arch](https://github.com/golang/go/blob/go1.20/src/runtime/proc.go#L152-L159), considering that WebAssembly is (currently) 32-bit environment. + +All the functions are statically analyzed at module instantiation phase, and if a function can potentially reach this limit, an error is returned. + +### Number of globals in a module + +Theoretically, a module can declare globals (including imports) up to 2^32 times. However, wazero limits this to 2^27(134,217,728) per module. +That is because internally we store globals in a slice with pointer types (meaning 8 bytes on 64-bit platforms), and therefore 2^27 globals +means that we have 1 GiB size of slice which seems large enough for most applications. + +### Number of tables in a module + +While the the spec says that a module can have up to 2^32 tables, wazero limits this to 2^27 = 134,217,728. +One of the reasons is even that number would occupy 1GB in the pointers tables alone. Not only that, we access tables slice by +table index by using 32-bit signed offset in the compiler implementation, which means that the table index of 2^27 can reach 2^27 * 8 (pointer size on 64-bit machines) = 2^30 offsets in bytes. + +We _believe_ that all use cases are fine with the limitation, but also note that we have no way to test wazero runtimes under these unusual circumstances. + +If a module reaches this limit, an error is returned at the compilation phase. + +## Compiler engine implementation + +### Why it's safe to execute runtime-generated machine codes against async Goroutine preemption + +Goroutine preemption is the mechanism of the Go runtime to switch goroutines contexts on an OS thread. +There are two types of preemption: cooperative preemption and async preemption. The former happens, for example, +when making a function call, and it is not an issue for our runtime-generated functions as they do not make +direct function calls to Go-implemented functions. On the other hand, the latter, async preemption, can be problematic +since it tries to interrupt the execution of Goroutine at any point of function, and manipulates CPU register states. + +Fortunately, our runtime-generated machine codes do not need to take the async preemption into account. +All the assembly codes are entered via the trampoline implemented as Go Assembler Function (e.g. [arch_amd64.s](./arch_amd64.s)), +and as of Go 1.20, these assembler functions are considered as _unsafe_ for async preemption: +- https://github.com/golang/go/blob/go1.20rc1/src/runtime/preempt.go#L406-L407 +- https://github.com/golang/go/blob/9f0234214473dfb785a5ad84a8fc62a6a395cbc3/src/runtime/traceback.go#L227 + +From the Go runtime point of view, the execution of runtime-generated machine codes is considered as a part of +that trampoline function. Therefore, runtime-generated machine code is also correctly considered unsafe for async preemption. + +## Why context cancellation is handled in Go code rather than native code + +Since [wazero v1.0.0-pre.9](https://github.com/tetratelabs/wazero/releases/tag/v1.0.0-pre.9), the runtime +supports integration with Go contexts to interrupt execution after a timeout, or in response to explicit cancellation. +This support is internally implemented as a special opcode `builtinFunctionCheckExitCode` that triggers the execution of +a Go function (`ModuleInstance.FailIfClosed`) that atomically checks a sentinel value at strategic points in the code. + +[It _is indeed_ possible to check the sentinel value directly, without leaving the native world][native_check], thus sparing some cycles; +however, because native code never preempts (see section above), this may lead to a state where the other goroutines +never get the chance to run, and thus never get the chance to set the sentinel value; effectively preventing +cancellation from taking place. + +[native_check]: https://github.com/tetratelabs/wazero/issues/1409 + +## Golang patterns + +### Hammer tests +Code that uses concurrency primitives, such as locks or atomics, should include "hammer tests", which run large loops +inside a bounded amount of goroutines, run by half that many `GOMAXPROCS`. These are named consistently "hammer", so +they are easy to find. The name inherits from some existing tests in [golang/go](https://github.com/golang/go/search?q=hammer&type=code). + +Here is an annotated description of the key pieces of a hammer test: +1. `P` declares the count of goroutines to use, defaulting to 8 or 4 if `testing.Short`. + * Half this amount are the cores used, and 4 is less than a modern laptop's CPU. This allows multiple "hammer" tests to run in parallel. +2. `N` declares the scale of work (loop) per goroutine, defaulting to value that finishes in ~0.1s on a modern laptop. + * When in doubt, try 1000 or 100 if `testing.Short` + * Remember, there are multiple hammer tests and CI nodes are slow. Slower tests hurt feedback loops. +3. `defer runtime.GOMAXPROCS(runtime.GOMAXPROCS(P/2))` makes goroutines switch cores, testing visibility of shared data. +4. To ensure goroutines execute at the same time, block them with `sync.WaitGroup`, initialized to `Add(P)`. + * `sync.WaitGroup` internally uses `runtime_Semacquire` not available in any other library. + * `sync.WaitGroup.Add` with a negative value can unblock many goroutines at the same time, e.g. without a for loop. +5. Track goroutines progress via `finished := make(chan int)` where each goroutine in `P` defers `finished <- 1`. + 1. Tests use `require.XXX`, so `recover()` into `t.Fail` in a `defer` function before `finished <- 1`. + * This makes it easier to spot larger concurrency problems as you see each failure, not just the first. + 2. After the `defer` function, await unblocked, then run the stateful function `N` times in a normal loop. + * This loop should trigger shared state problems as locks or atomics are contended by `P` goroutines. +6. After all `P` goroutines launch, atomically release all of them with `WaitGroup.Add(-P)`. +7. Block the runner on goroutine completion, by (`<-finished`) for each `P`. +8. When all goroutines complete, `return` if `t.Failed()`, otherwise perform follow-up state checks. + +This is implemented in wazero in [hammer.go](internal/testing/hammer/hammer.go) + +### Lock-free, cross-goroutine observations of updates + +How to achieve cross-goroutine reads of a variable are not explicitly defined in https://go.dev/ref/mem. wazero uses +atomics to implement this following unofficial practice. For example, a `Close` operation can be guarded to happen only +once via compare-and-swap (CAS) against a zero value. When we use this pattern, we consistently use atomics to both +read and update the same numeric field. + +In lieu of formal documentation, we infer this pattern works from other sources (besides tests): + * `sync.WaitGroup` by definition must support calling `Add` from other goroutines. Internally, it uses atomics. + * rsc in golang/go#5045 writes "atomics guarantee sequential consistency among the atomic variables". + +See https://github.com/golang/go/blob/go1.20/src/sync/waitgroup.go#L64 +See https://github.com/golang/go/issues/5045#issuecomment-252730563 +See https://www.youtube.com/watch?v=VmrEG-3bWyM diff --git a/vendor/github.com/tetratelabs/wazero/README.md b/vendor/github.com/tetratelabs/wazero/README.md new file mode 100644 index 000000000..657da2959 --- /dev/null +++ b/vendor/github.com/tetratelabs/wazero/README.md @@ -0,0 +1,132 @@ +# wazero: the zero dependency WebAssembly runtime for Go developers + +[![WebAssembly Core Specification Test](https://github.com/tetratelabs/wazero/actions/workflows/spectest.yaml/badge.svg)](https://github.com/tetratelabs/wazero/actions/workflows/spectest.yaml) [![Go Reference](https://pkg.go.dev/badge/github.com/tetratelabs/wazero.svg)](https://pkg.go.dev/github.com/tetratelabs/wazero) [![License](https://img.shields.io/badge/License-Apache_2.0-blue.svg)](https://opensource.org/licenses/Apache-2.0) + +WebAssembly is a way to safely run code compiled in other languages. Runtimes +execute WebAssembly Modules (Wasm), which are most often binaries with a `.wasm` +extension. + +wazero is a WebAssembly Core Specification [1.0][1] and [2.0][2] compliant +runtime written in Go. It has *zero dependencies*, and doesn't rely on CGO. +This means you can run applications in other languages and still keep cross +compilation. + +Import wazero and extend your Go application with code written in any language! + +## Example + +The best way to learn wazero is by trying one of our [examples](examples/README.md). The +most [basic example](examples/basic) extends a Go application with an addition +function defined in WebAssembly. + +## Runtime + +There are two runtime configurations supported in wazero: _Compiler_ is default: + +By default, ex `wazero.NewRuntime(ctx)`, the Compiler is used if supported. You +can also force the interpreter like so: +```go +r := wazero.NewRuntimeWithConfig(ctx, wazero.NewRuntimeConfigInterpreter()) +``` + +### Interpreter +Interpreter is a naive interpreter-based implementation of Wasm virtual +machine. Its implementation doesn't have any platform (GOARCH, GOOS) specific +code, therefore _interpreter_ can be used for any compilation target available +for Go (such as `riscv64`). + +### Compiler +Compiler compiles WebAssembly modules into machine code ahead of time (AOT), +during `Runtime.CompileModule`. This means your WebAssembly functions execute +natively at runtime. Compiler is faster than Interpreter, often by order of +magnitude (10x) or more. This is done without host-specific dependencies. + +### Conformance + +Both runtimes pass WebAssembly Core [1.0][7] and [2.0][14] specification tests +on supported platforms: + +| Runtime | Usage | amd64 | arm64 | others | +|:-----------:|:--------------------------------------:|:-----:|:-----:|:------:| +| Interpreter | `wazero.NewRuntimeConfigInterpreter()` | ✅ | ✅ | ✅ | +| Compiler | `wazero.NewRuntimeConfigCompiler()` | ✅ | ✅ | ❌ | + +## Support Policy + +The below support policy focuses on compatibility concerns of those embedding +wazero into their Go applications. + +### wazero + +wazero's [1.0 release][15] happened in March 2023, and is [in use][16] by many +projects and production sites. + +We offer an API stability promise with semantic versioning. In other words, we +promise to not break any exported function signature without incrementing the +major version. This does not mean no innovation: New features and behaviors +happen with a minor version increment, e.g. 1.0.11 to 1.2.0. We also fix bugs +or change internal details with a patch version, e.g. 1.0.0 to 1.0.1. + +You can get the latest version of wazero like this. +```bash +go get github.com/tetratelabs/wazero@latest +``` + +Please give us a [star][17] if you end up using wazero! + +### Go + +wazero has no dependencies except Go, so the only source of conflict in your +project's use of wazero is the Go version. + +wazero follows the same version policy as Go's [Release Policy][10]: two +versions. wazero will ensure these versions work and bugs are valid if there's +an issue with a current Go version. + +Additionally, wazero intentionally delays usage of language or standard library +features one additional version. For example, when Go 1.29 is released, wazero +can use language features or standard libraries added in 1.27. This is a +convenience for embedders who have a slower version policy than Go. However, +only supported Go versions may be used to raise support issues. + +### Platform + +wazero has two runtime modes: Interpreter and Compiler. The only supported operating +systems are ones we test, but that doesn't necessarily mean other operating +system versions won't work. + +We currently test Linux (Ubuntu and scratch), MacOS and Windows as packaged by +[GitHub Actions][11], as well compilation of 32-bit Linux and 64-bit FreeBSD. + +* Interpreter + * Linux is tested on amd64 (native) as well arm64 and riscv64 via emulation. + * MacOS and Windows are only tested on amd64. +* Compiler + * Linux is tested on amd64 (native) as well arm64 via emulation. + * MacOS and Windows are only tested on amd64. + +wazero has no dependencies and doesn't require CGO. This means it can also be +embedded in an application that doesn't use an operating system. This is a main +differentiator between wazero and alternatives. + +We verify zero dependencies by running tests in Docker's [scratch image][12]. +This approach ensures compatibility with any parent image. + +----- +wazero is a registered trademark of Tetrate.io, Inc. in the United States and/or other countries + +[1]: https://www.w3.org/TR/2019/REC-wasm-core-1-20191205/ +[2]: https://www.w3.org/TR/2022/WD-wasm-core-2-20220419/ +[4]: https://github.com/WebAssembly/meetings/blob/main/process/subgroups.md +[5]: https://github.com/WebAssembly/WASI +[6]: https://pkg.go.dev/golang.org/x/sys/unix +[7]: https://github.com/WebAssembly/spec/tree/wg-1.0/test/core +[9]: https://github.com/tetratelabs/wazero/issues/506 +[10]: https://go.dev/doc/devel/release +[11]: https://github.com/actions/virtual-environments +[12]: https://docs.docker.com/develop/develop-images/baseimages/#create-a-simple-parent-image-using-scratch +[13]: https://github.com/WebAssembly/WASI/blob/snapshot-01/phases/snapshot/docs.md +[14]: https://github.com/WebAssembly/spec/tree/d39195773112a22b245ffbe864bab6d1182ccb06/test/core +[15]: https://tetrate.io/blog/introducing-wazero-from-tetrate/ +[16]: https://wazero.io/community/users/ +[17]: https://github.com/tetratelabs/wazero/stargazers diff --git a/vendor/github.com/tetratelabs/wazero/api/features.go b/vendor/github.com/tetratelabs/wazero/api/features.go new file mode 100644 index 000000000..c739d3bf7 --- /dev/null +++ b/vendor/github.com/tetratelabs/wazero/api/features.go @@ -0,0 +1,214 @@ +package api + +import ( + "fmt" + "strings" +) + +// CoreFeatures is a bit flag of WebAssembly Core specification features. See +// https://github.com/WebAssembly/proposals for proposals and their status. +// +// Constants define individual features, such as CoreFeatureMultiValue, or +// groups of "finished" features, assigned to a WebAssembly Core Specification +// version, e.g. CoreFeaturesV1 or CoreFeaturesV2. +// +// Note: Numeric values are not intended to be interpreted except as bit flags. +type CoreFeatures uint64 + +// CoreFeaturesV1 are features included in the WebAssembly Core Specification +// 1.0. As of late 2022, this is the only version that is a Web Standard (W3C +// Recommendation). +// +// See https://www.w3.org/TR/2019/REC-wasm-core-1-20191205/ +const CoreFeaturesV1 = CoreFeatureMutableGlobal + +// CoreFeaturesV2 are features included in the WebAssembly Core Specification +// 2.0 (20220419). As of late 2022, version 2.0 is a W3C working draft, not yet +// a Web Standard (W3C Recommendation). +// +// See https://www.w3.org/TR/2022/WD-wasm-core-2-20220419/appendix/changes.html#release-1-1 +const CoreFeaturesV2 = CoreFeaturesV1 | + CoreFeatureBulkMemoryOperations | + CoreFeatureMultiValue | + CoreFeatureNonTrappingFloatToIntConversion | + CoreFeatureReferenceTypes | + CoreFeatureSignExtensionOps | + CoreFeatureSIMD + +const ( + // CoreFeatureBulkMemoryOperations adds instructions modify ranges of + // memory or table entries ("bulk-memory-operations"). This is included in + // CoreFeaturesV2, but not CoreFeaturesV1. + // + // Here are the notable effects: + // - Adds `memory.fill`, `memory.init`, `memory.copy` and `data.drop` + // instructions. + // - Adds `table.init`, `table.copy` and `elem.drop` instructions. + // - Introduces a "passive" form of element and data segments. + // - Stops checking "active" element and data segment boundaries at + // compile-time, meaning they can error at runtime. + // + // Note: "bulk-memory-operations" is mixed with the "reference-types" + // proposal due to the WebAssembly Working Group merging them + // "mutually dependent". Therefore, enabling this feature requires enabling + // CoreFeatureReferenceTypes, and vice-versa. + // + // See https://github.com/WebAssembly/spec/blob/wg-2.0.draft1/proposals/bulk-memory-operations/Overview.md + // https://github.com/WebAssembly/spec/blob/wg-2.0.draft1/proposals/reference-types/Overview.md and + // https://github.com/WebAssembly/spec/pull/1287 + CoreFeatureBulkMemoryOperations CoreFeatures = 1 << iota + + // CoreFeatureMultiValue enables multiple values ("multi-value"). This is + // included in CoreFeaturesV2, but not CoreFeaturesV1. + // + // Here are the notable effects: + // - Function (`func`) types allow more than one result. + // - Block types (`block`, `loop` and `if`) can be arbitrary function + // types. + // + // See https://github.com/WebAssembly/spec/blob/wg-2.0.draft1/proposals/multi-value/Overview.md + CoreFeatureMultiValue + + // CoreFeatureMutableGlobal allows globals to be mutable. This is included + // in both CoreFeaturesV1 and CoreFeaturesV2. + // + // When false, an api.Global can never be cast to an api.MutableGlobal, and + // any wasm that includes global vars will fail to parse. + CoreFeatureMutableGlobal + + // CoreFeatureNonTrappingFloatToIntConversion enables non-trapping + // float-to-int conversions ("nontrapping-float-to-int-conversion"). This + // is included in CoreFeaturesV2, but not CoreFeaturesV1. + // + // The only effect of enabling is allowing the following instructions, + // which return 0 on NaN instead of panicking. + // - `i32.trunc_sat_f32_s` + // - `i32.trunc_sat_f32_u` + // - `i32.trunc_sat_f64_s` + // - `i32.trunc_sat_f64_u` + // - `i64.trunc_sat_f32_s` + // - `i64.trunc_sat_f32_u` + // - `i64.trunc_sat_f64_s` + // - `i64.trunc_sat_f64_u` + // + // See https://github.com/WebAssembly/spec/blob/wg-2.0.draft1/proposals/nontrapping-float-to-int-conversion/Overview.md + CoreFeatureNonTrappingFloatToIntConversion + + // CoreFeatureReferenceTypes enables various instructions and features + // related to table and new reference types. This is included in + // CoreFeaturesV2, but not CoreFeaturesV1. + // + // - Introduction of new value types: `funcref` and `externref`. + // - Support for the following new instructions: + // - `ref.null` + // - `ref.func` + // - `ref.is_null` + // - `table.fill` + // - `table.get` + // - `table.grow` + // - `table.set` + // - `table.size` + // - Support for multiple tables per module: + // - `call_indirect`, `table.init`, `table.copy` and `elem.drop` + // - Support for instructions can take non-zero table index. + // - Element segments can take non-zero table index. + // + // Note: "reference-types" is mixed with the "bulk-memory-operations" + // proposal due to the WebAssembly Working Group merging them + // "mutually dependent". Therefore, enabling this feature requires enabling + // CoreFeatureBulkMemoryOperations, and vice-versa. + // + // See https://github.com/WebAssembly/spec/blob/wg-2.0.draft1/proposals/bulk-memory-operations/Overview.md + // https://github.com/WebAssembly/spec/blob/wg-2.0.draft1/proposals/reference-types/Overview.md and + // https://github.com/WebAssembly/spec/pull/1287 + CoreFeatureReferenceTypes + + // CoreFeatureSignExtensionOps enables sign extension instructions + // ("sign-extension-ops"). This is included in CoreFeaturesV2, but not + // CoreFeaturesV1. + // + // Adds instructions: + // - `i32.extend8_s` + // - `i32.extend16_s` + // - `i64.extend8_s` + // - `i64.extend16_s` + // - `i64.extend32_s` + // + // See https://github.com/WebAssembly/spec/blob/wg-2.0.draft1/proposals/sign-extension-ops/Overview.md + CoreFeatureSignExtensionOps + + // CoreFeatureSIMD enables the vector value type and vector instructions + // (aka SIMD). This is included in CoreFeaturesV2, but not CoreFeaturesV1. + // + // Note: The instruction list is too long to enumerate in godoc. + // See https://github.com/WebAssembly/spec/blob/wg-2.0.draft1/proposals/simd/SIMD.md + CoreFeatureSIMD + + // Update experimental/features.go when adding elements here. +) + +// SetEnabled enables or disables the feature or group of features. +func (f CoreFeatures) SetEnabled(feature CoreFeatures, val bool) CoreFeatures { + if val { + return f | feature + } + return f &^ feature +} + +// IsEnabled returns true if the feature (or group of features) is enabled. +func (f CoreFeatures) IsEnabled(feature CoreFeatures) bool { + return f&feature != 0 +} + +// RequireEnabled returns an error if the feature (or group of features) is not +// enabled. +func (f CoreFeatures) RequireEnabled(feature CoreFeatures) error { + if f&feature == 0 { + return fmt.Errorf("feature %q is disabled", feature) + } + return nil +} + +// String implements fmt.Stringer by returning each enabled feature. +func (f CoreFeatures) String() string { + var builder strings.Builder + for i := 0; i <= 63; i++ { // cycle through all bits to reduce code and maintenance + target := CoreFeatures(1 << i) + if f.IsEnabled(target) { + if name := featureName(target); name != "" { + if builder.Len() > 0 { + builder.WriteByte('|') + } + builder.WriteString(name) + } + } + } + return builder.String() +} + +func featureName(f CoreFeatures) string { + switch f { + case CoreFeatureMutableGlobal: + // match https://github.com/WebAssembly/mutable-global + return "mutable-global" + case CoreFeatureSignExtensionOps: + // match https://github.com/WebAssembly/spec/blob/wg-2.0.draft1/proposals/sign-extension-ops/Overview.md + return "sign-extension-ops" + case CoreFeatureMultiValue: + // match https://github.com/WebAssembly/spec/blob/wg-2.0.draft1/proposals/multi-value/Overview.md + return "multi-value" + case CoreFeatureNonTrappingFloatToIntConversion: + // match https://github.com/WebAssembly/spec/blob/wg-2.0.draft1/proposals/nontrapping-float-to-int-conversion/Overview.md + return "nontrapping-float-to-int-conversion" + case CoreFeatureBulkMemoryOperations: + // match https://github.com/WebAssembly/spec/blob/wg-2.0.draft1/proposals/bulk-memory-operations/Overview.md + return "bulk-memory-operations" + case CoreFeatureReferenceTypes: + // match https://github.com/WebAssembly/spec/blob/wg-2.0.draft1/proposals/reference-types/Overview.md + return "reference-types" + case CoreFeatureSIMD: + // match https://github.com/WebAssembly/spec/blob/wg-2.0.draft1/proposals/simd/SIMD.md + return "simd" + } + return "" +} diff --git a/vendor/github.com/tetratelabs/wazero/api/wasm.go b/vendor/github.com/tetratelabs/wazero/api/wasm.go new file mode 100644 index 000000000..c66b582fa --- /dev/null +++ b/vendor/github.com/tetratelabs/wazero/api/wasm.go @@ -0,0 +1,762 @@ +// Package api includes constants and interfaces used by both end-users and internal implementations. +package api + +import ( + "context" + "fmt" + "math" + + "github.com/tetratelabs/wazero/internal/internalapi" +) + +// ExternType classifies imports and exports with their respective types. +// +// See https://www.w3.org/TR/2019/REC-wasm-core-1-20191205/#external-types%E2%91%A0 +type ExternType = byte + +const ( + ExternTypeFunc ExternType = 0x00 + ExternTypeTable ExternType = 0x01 + ExternTypeMemory ExternType = 0x02 + ExternTypeGlobal ExternType = 0x03 +) + +// The below are exported to consolidate parsing behavior for external types. +const ( + // ExternTypeFuncName is the name of the WebAssembly 1.0 (20191205) Text Format field for ExternTypeFunc. + ExternTypeFuncName = "func" + // ExternTypeTableName is the name of the WebAssembly 1.0 (20191205) Text Format field for ExternTypeTable. + ExternTypeTableName = "table" + // ExternTypeMemoryName is the name of the WebAssembly 1.0 (20191205) Text Format field for ExternTypeMemory. + ExternTypeMemoryName = "memory" + // ExternTypeGlobalName is the name of the WebAssembly 1.0 (20191205) Text Format field for ExternTypeGlobal. + ExternTypeGlobalName = "global" +) + +// ExternTypeName returns the name of the WebAssembly 1.0 (20191205) Text Format field of the given type. +// +// See https://www.w3.org/TR/2019/REC-wasm-core-1-20191205/#exports%E2%91%A4 +func ExternTypeName(et ExternType) string { + switch et { + case ExternTypeFunc: + return ExternTypeFuncName + case ExternTypeTable: + return ExternTypeTableName + case ExternTypeMemory: + return ExternTypeMemoryName + case ExternTypeGlobal: + return ExternTypeGlobalName + } + return fmt.Sprintf("%#x", et) +} + +// ValueType describes a parameter or result type mapped to a WebAssembly +// function signature. +// +// The following describes how to convert between Wasm and Golang types: +// +// - ValueTypeI32 - EncodeU32 DecodeU32 for uint32 / EncodeI32 DecodeI32 for int32 +// - ValueTypeI64 - uint64(int64) +// - ValueTypeF32 - EncodeF32 DecodeF32 from float32 +// - ValueTypeF64 - EncodeF64 DecodeF64 from float64 +// - ValueTypeExternref - unintptr(unsafe.Pointer(p)) where p is any pointer +// type in Go (e.g. *string) +// +// e.g. Given a Text Format type use (param i64) (result i64), no conversion is +// necessary. +// +// results, _ := fn(ctx, input) +// result := result[0] +// +// e.g. Given a Text Format type use (param f64) (result f64), conversion is +// necessary. +// +// results, _ := fn(ctx, api.EncodeF64(input)) +// result := api.DecodeF64(result[0]) +// +// Note: This is a type alias as it is easier to encode and decode in the +// binary format. +// +// See https://www.w3.org/TR/2019/REC-wasm-core-1-20191205/#binary-valtype +type ValueType = byte + +const ( + // ValueTypeI32 is a 32-bit integer. + ValueTypeI32 ValueType = 0x7f + // ValueTypeI64 is a 64-bit integer. + ValueTypeI64 ValueType = 0x7e + // ValueTypeF32 is a 32-bit floating point number. + ValueTypeF32 ValueType = 0x7d + // ValueTypeF64 is a 64-bit floating point number. + ValueTypeF64 ValueType = 0x7c + + // ValueTypeExternref is a externref type. + // + // Note: in wazero, externref type value are opaque raw 64-bit pointers, + // and the ValueTypeExternref type in the signature will be translated as + // uintptr in wazero's API level. + // + // For example, given the import function: + // (func (import "env" "f") (param externref) (result externref)) + // + // This can be defined in Go as: + // r.NewHostModuleBuilder("env"). + // NewFunctionBuilder(). + // WithFunc(func(context.Context, _ uintptr) (_ uintptr) { return }). + // Export("f") + // + // Note: The usage of this type is toggled with api.CoreFeatureBulkMemoryOperations. + ValueTypeExternref ValueType = 0x6f +) + +// ValueTypeName returns the type name of the given ValueType as a string. +// These type names match the names used in the WebAssembly text format. +// +// Note: This returns "unknown", if an undefined ValueType value is passed. +func ValueTypeName(t ValueType) string { + switch t { + case ValueTypeI32: + return "i32" + case ValueTypeI64: + return "i64" + case ValueTypeF32: + return "f32" + case ValueTypeF64: + return "f64" + case ValueTypeExternref: + return "externref" + } + return "unknown" +} + +// Module is a sandboxed, ready to execute Wasm module. This can be used to get exported functions, etc. +// +// In WebAssembly terminology, this corresponds to a "Module Instance", but wazero calls pre-instantiation module as +// "Compiled Module" as in wazero.CompiledModule, therefore we call this post-instantiation module simply "Module". +// See https://www.w3.org/TR/2019/REC-wasm-core-1-20191205/#module-instances%E2%91%A0 +// +// # Notes +// +// - This is an interface for decoupling, not third-party implementations. +// All implementations are in wazero. +// - Closing the wazero.Runtime closes any Module it instantiated. +type Module interface { + fmt.Stringer + + // Name is the name this module was instantiated with. Exported functions can be imported with this name. + Name() string + + // Memory returns a memory defined in this module or nil if there are none wasn't. + Memory() Memory + + // ExportedFunction returns a function exported from this module or nil if it wasn't. + // + // Note: The default wazero.ModuleConfig attempts to invoke `_start`, which + // in rare cases can close the module. When in doubt, check IsClosed prior + // to invoking a function export after instantiation. + ExportedFunction(name string) Function + + // ExportedFunctionDefinitions returns all the exported function + // definitions in this module, keyed on export name. + ExportedFunctionDefinitions() map[string]FunctionDefinition + + // TODO: Table + + // ExportedMemory returns a memory exported from this module or nil if it wasn't. + // + // WASI modules require exporting a Memory named "memory". This means that a module successfully initialized + // as a WASI Command or Reactor will never return nil for this name. + // + // See https://github.com/WebAssembly/WASI/blob/snapshot-01/design/application-abi.md#current-unstable-abi + ExportedMemory(name string) Memory + + // ExportedMemoryDefinitions returns all the exported memory definitions + // in this module, keyed on export name. + // + // Note: As of WebAssembly Core Specification 2.0, there can be at most one + // memory. + ExportedMemoryDefinitions() map[string]MemoryDefinition + + // ExportedGlobal a global exported from this module or nil if it wasn't. + ExportedGlobal(name string) Global + + // CloseWithExitCode releases resources allocated for this Module. Use a non-zero exitCode parameter to indicate a + // failure to ExportedFunction callers. + // + // The error returned here, if present, is about resource de-allocation (such as I/O errors). Only the last error is + // returned, so a non-nil return means at least one error happened. Regardless of error, this Module will + // be removed, making its name available again. + // + // Calling this inside a host function is safe, and may cause ExportedFunction callers to receive a sys.ExitError + // with the exitCode. + CloseWithExitCode(ctx context.Context, exitCode uint32) error + + // Closer closes this module by delegating to CloseWithExitCode with an exit code of zero. + Closer + + // IsClosed returns true if the module is closed, so no longer usable. + // + // This can happen for the following reasons: + // - Closer was called directly. + // - A guest function called Closer indirectly, such as `_start` calling + // `proc_exit`, which internally closed the module. + // - wazero.RuntimeConfig `WithCloseOnContextDone` was enabled and a + // context completion closed the module. + // + // Where any of the above are possible, check this value before calling an + // ExportedFunction, even if you didn't formerly receive a sys.ExitError. + // sys.ExitError is only returned on non-zero code, something that closes + // the module successfully will not result it one. + IsClosed() bool + + internalapi.WazeroOnly +} + +// Closer closes a resource. +// +// # Notes +// +// - This is an interface for decoupling, not third-party implementations. +// All implementations are in wazero. +type Closer interface { + // Close closes the resource. + // + // Note: The context parameter is used for value lookup, such as for + // logging. A canceled or otherwise done context will not prevent Close + // from succeeding. + Close(context.Context) error +} + +// ExportDefinition is a WebAssembly type exported in a module +// (wazero.CompiledModule). +// +// See https://www.w3.org/TR/2019/REC-wasm-core-1-20191205/#exports%E2%91%A0 +// +// # Notes +// +// - This is an interface for decoupling, not third-party implementations. +// All implementations are in wazero. +type ExportDefinition interface { + // ModuleName is the possibly empty name of the module defining this + // export. + // + // Note: This may be different from Module.Name, because a compiled module + // can be instantiated multiple times as different names. + ModuleName() string + + // Index is the position in the module's index, imports first. + Index() uint32 + + // Import returns true with the module and name when this was imported. + // Otherwise, it returns false. + // + // Note: Empty string is valid for both names in the WebAssembly Core + // Specification, so "" "" is possible. + Import() (moduleName, name string, isImport bool) + + // ExportNames include all exported names. + // + // Note: The empty name is allowed in the WebAssembly Core Specification, + // so "" is possible. + ExportNames() []string + + internalapi.WazeroOnly +} + +// MemoryDefinition is a WebAssembly memory exported in a module +// (wazero.CompiledModule). Units are in pages (64KB). +// +// See https://www.w3.org/TR/2019/REC-wasm-core-1-20191205/#exports%E2%91%A0 +// +// # Notes +// +// - This is an interface for decoupling, not third-party implementations. +// All implementations are in wazero. +type MemoryDefinition interface { + ExportDefinition + + // Min returns the possibly zero initial count of 64KB pages. + Min() uint32 + + // Max returns the possibly zero max count of 64KB pages, or false if + // unbounded. + Max() (uint32, bool) + + internalapi.WazeroOnly +} + +// FunctionDefinition is a WebAssembly function exported in a module +// (wazero.CompiledModule). +// +// See https://www.w3.org/TR/2019/REC-wasm-core-1-20191205/#exports%E2%91%A0 +// +// # Notes +// +// - This is an interface for decoupling, not third-party implementations. +// All implementations are in wazero. +type FunctionDefinition interface { + ExportDefinition + + // Name is the module-defined name of the function, which is not necessarily + // the same as its export name. + Name() string + + // DebugName identifies this function based on its Index or Name in the + // module. This is used for errors and stack traces. e.g. "env.abort". + // + // When the function name is empty, a substitute name is generated by + // prefixing '$' to its position in the index. Ex ".$0" is the + // first function (possibly imported) in an unnamed module. + // + // The format is dot-delimited module and function name, but there are no + // restrictions on the module and function name. This means either can be + // empty or include dots. e.g. "x.x.x" could mean module "x" and name "x.x", + // or it could mean module "x.x" and name "x". + // + // Note: This name is stable regardless of import or export. For example, + // if Import returns true, the value is still based on the Name or Index + // and not the imported function name. + DebugName() string + + // GoFunction is non-nil when implemented by the embedder instead of a wasm + // binary, e.g. via wazero.HostModuleBuilder + // + // The expected results are nil, GoFunction or GoModuleFunction. + GoFunction() interface{} + + // ParamTypes are the possibly empty sequence of value types accepted by a + // function with this signature. + // + // See ValueType documentation for encoding rules. + ParamTypes() []ValueType + + // ParamNames are index-correlated with ParamTypes or nil if not available + // for one or more parameters. + ParamNames() []string + + // ResultTypes are the results of the function. + // + // When WebAssembly 1.0 (20191205), there can be at most one result. + // See https://www.w3.org/TR/2019/REC-wasm-core-1-20191205/#result-types%E2%91%A0 + // + // See ValueType documentation for encoding rules. + ResultTypes() []ValueType + + // ResultNames are index-correlated with ResultTypes or nil if not + // available for one or more results. + ResultNames() []string + + internalapi.WazeroOnly +} + +// Function is a WebAssembly function exported from an instantiated module +// (wazero.Runtime InstantiateModule). +// +// See https://www.w3.org/TR/2019/REC-wasm-core-1-20191205/#syntax-func +// +// # Notes +// +// - This is an interface for decoupling, not third-party implementations. +// All implementations are in wazero. +type Function interface { + // Definition is metadata about this function from its defining module. + Definition() FunctionDefinition + + // Call invokes the function with the given parameters and returns any + // results or an error for any failure looking up or invoking the function. + // + // Encoding is described in Definition, and supplying an incorrect count of + // parameters vs FunctionDefinition.ParamTypes is an error. + // + // If the exporting Module was closed during this call, the error returned + // may be a sys.ExitError. See Module.CloseWithExitCode for details. + // + // Call is not goroutine-safe, therefore it is recommended to create + // another Function if you want to invoke the same function concurrently. + // On the other hand, sequential invocations of Call is allowed. + // However, this should not be called multiple times until the previous Call returns. + // + // To safely encode/decode params/results expressed as uint64, users are encouraged to + // use api.EncodeXXX or DecodeXXX functions. See the docs on api.ValueType. + // + // When RuntimeConfig.WithCloseOnContextDone is toggled, the invocation of this Call method is ensured to be closed + // whenever one of the three conditions is met. In the event of close, sys.ExitError will be returned and + // the api.Module from which this api.Function is derived will be made closed. See the documentation of + // WithCloseOnContextDone on wazero.RuntimeConfig for detail. See examples in context_done_example_test.go for + // the end-to-end demonstrations of how these terminations can be performed. + Call(ctx context.Context, params ...uint64) ([]uint64, error) + + // CallWithStack is an optimized variation of Call that saves memory + // allocations when the stack slice is reused across calls. + // + // Stack length must be at least the max of parameter or result length. + // The caller adds parameters in order to the stack, and reads any results + // in order from the stack, except in the error case. + // + // For example, the following reuses the same stack slice to call searchFn + // repeatedly saving one allocation per iteration: + // + // stack := make([]uint64, 4) + // for i, search := range searchParams { + // // copy the next params to the stack + // copy(stack, search) + // if err := searchFn.CallWithStack(ctx, stack); err != nil { + // return err + // } else if stack[0] == 1 { // found + // return i // searchParams[i] matched! + // } + // } + // + // # Notes + // + // - This is similar to GoModuleFunction, except for using calling functions + // instead of implementing them. Moreover, this is used regardless of + // whether the callee is a host or wasm defined function. + CallWithStack(ctx context.Context, stack []uint64) error + + internalapi.WazeroOnly +} + +// GoModuleFunction is a Function implemented in Go instead of a wasm binary. +// The Module parameter is the calling module, used to access memory or +// exported functions. See GoModuleFunc for an example. +// +// The stack is includes any parameters encoded according to their ValueType. +// Its length is the max of parameter or result length. When there are results, +// write them in order beginning at index zero. Do not use the stack after the +// function returns. +// +// Here's a typical way to read three parameters and write back one. +// +// // read parameters off the stack in index order +// argv, argvBuf := api.DecodeU32(stack[0]), api.DecodeU32(stack[1]) +// +// // write results back to the stack in index order +// stack[0] = api.EncodeU32(ErrnoSuccess) +// +// This function can be non-deterministic or cause side effects. It also +// has special properties not defined in the WebAssembly Core specification. +// Notably, this uses the caller's memory (via Module.Memory). See +// https://www.w3.org/TR/wasm-core-1/#host-functions%E2%91%A0 +// +// Most end users will not define functions directly with this, as they will +// use reflection or code generators instead. These approaches are more +// idiomatic as they can map go types to ValueType. This type is exposed for +// those willing to trade usability and safety for performance. +// +// To safely decode/encode values from/to the uint64 stack, users are encouraged to use +// api.EncodeXXX or api.DecodeXXX functions. See the docs on api.ValueType. +type GoModuleFunction interface { + Call(ctx context.Context, mod Module, stack []uint64) +} + +// GoModuleFunc is a convenience for defining an inlined function. +// +// For example, the following returns an uint32 value read from parameter zero: +// +// api.GoModuleFunc(func(ctx context.Context, mod api.Module, stack []uint64) { +// offset := api.DecodeU32(stack[0]) // read the parameter from the stack +// +// ret, ok := mod.Memory().ReadUint32Le(offset) +// if !ok { +// panic("out of memory") +// } +// +// stack[0] = api.EncodeU32(ret) // add the result back to the stack. +// }) +type GoModuleFunc func(ctx context.Context, mod Module, stack []uint64) + +// Call implements GoModuleFunction.Call. +func (f GoModuleFunc) Call(ctx context.Context, mod Module, stack []uint64) { + f(ctx, mod, stack) +} + +// GoFunction is an optimized form of GoModuleFunction which doesn't require +// the Module parameter. See GoFunc for an example. +// +// For example, this function does not need to use the importing module's +// memory or exported functions. +type GoFunction interface { + Call(ctx context.Context, stack []uint64) +} + +// GoFunc is a convenience for defining an inlined function. +// +// For example, the following returns the sum of two uint32 parameters: +// +// api.GoFunc(func(ctx context.Context, stack []uint64) { +// x, y := api.DecodeU32(stack[0]), api.DecodeU32(stack[1]) +// stack[0] = api.EncodeU32(x + y) +// }) +type GoFunc func(ctx context.Context, stack []uint64) + +// Call implements GoFunction.Call. +func (f GoFunc) Call(ctx context.Context, stack []uint64) { + f(ctx, stack) +} + +// Global is a WebAssembly 1.0 (20191205) global exported from an instantiated module (wazero.Runtime InstantiateModule). +// +// For example, if the value is not mutable, you can read it once: +// +// offset := module.ExportedGlobal("memory.offset").Get() +// +// Globals are allowed by specification to be mutable. However, this can be disabled by configuration. When in doubt, +// safe cast to find out if the value can change. Here's an example: +// +// offset := module.ExportedGlobal("memory.offset") +// if _, ok := offset.(api.MutableGlobal); ok { +// // value can change +// } else { +// // value is constant +// } +// +// See https://www.w3.org/TR/2019/REC-wasm-core-1-20191205/#globals%E2%91%A0 +// +// # Notes +// +// - This is an interface for decoupling, not third-party implementations. +// All implementations are in wazero. +type Global interface { + fmt.Stringer + + // Type describes the numeric type of the global. + Type() ValueType + + // Get returns the last known value of this global. + // + // See Type for how to decode this value to a Go type. + Get() uint64 +} + +// MutableGlobal is a Global whose value can be updated at runtime (variable). +// +// # Notes +// +// - This is an interface for decoupling, not third-party implementations. +// All implementations are in wazero. +type MutableGlobal interface { + Global + + // Set updates the value of this global. + // + // See Global.Type for how to encode this value from a Go type. + Set(v uint64) + + internalapi.WazeroOnly +} + +// Memory allows restricted access to a module's memory. Notably, this does not allow growing. +// +// See https://www.w3.org/TR/2019/REC-wasm-core-1-20191205/#storage%E2%91%A0 +// +// # Notes +// +// - This is an interface for decoupling, not third-party implementations. +// All implementations are in wazero. +// - This includes all value types available in WebAssembly 1.0 (20191205) and all are encoded little-endian. +type Memory interface { + // Definition is metadata about this memory from its defining module. + Definition() MemoryDefinition + + // Size returns the memory size in bytes available. + // e.g. If the underlying memory has 1 page: 65536 + // + // # Notes + // + // - This overflows (returns zero) if the memory has the maximum 65536 pages. + // As a workaround until wazero v2 to fix the return type, use Grow(0) to obtain the current pages and + // multiply by 65536. + // + // See https://www.w3.org/TR/2019/REC-wasm-core-1-20191205/#-hrefsyntax-instr-memorymathsfmemorysize%E2%91%A0 + Size() uint32 + + // Grow increases memory by the delta in pages (65536 bytes per page). + // The return val is the previous memory size in pages, or false if the + // delta was ignored as it exceeds MemoryDefinition.Max. + // + // # Notes + // + // - This is the same as the "memory.grow" instruction defined in the + // WebAssembly Core Specification, except returns false instead of -1. + // - When this returns true, any shared views via Read must be refreshed. + // + // See MemorySizer Read and https://www.w3.org/TR/2019/REC-wasm-core-1-20191205/#grow-mem + Grow(deltaPages uint32) (previousPages uint32, ok bool) + + // ReadByte reads a single byte from the underlying buffer at the offset or returns false if out of range. + ReadByte(offset uint32) (byte, bool) + + // ReadUint16Le reads a uint16 in little-endian encoding from the underlying buffer at the offset in or returns + // false if out of range. + ReadUint16Le(offset uint32) (uint16, bool) + + // ReadUint32Le reads a uint32 in little-endian encoding from the underlying buffer at the offset in or returns + // false if out of range. + ReadUint32Le(offset uint32) (uint32, bool) + + // ReadFloat32Le reads a float32 from 32 IEEE 754 little-endian encoded bits in the underlying buffer at the offset + // or returns false if out of range. + // See math.Float32bits + ReadFloat32Le(offset uint32) (float32, bool) + + // ReadUint64Le reads a uint64 in little-endian encoding from the underlying buffer at the offset or returns false + // if out of range. + ReadUint64Le(offset uint32) (uint64, bool) + + // ReadFloat64Le reads a float64 from 64 IEEE 754 little-endian encoded bits in the underlying buffer at the offset + // or returns false if out of range. + // + // See math.Float64bits + ReadFloat64Le(offset uint32) (float64, bool) + + // Read reads byteCount bytes from the underlying buffer at the offset or + // returns false if out of range. + // + // For example, to search for a NUL-terminated string: + // buf, _ = memory.Read(offset, byteCount) + // n := bytes.IndexByte(buf, 0) + // if n < 0 { + // // Not found! + // } + // + // Write-through + // + // This returns a view of the underlying memory, not a copy. This means any + // writes to the slice returned are visible to Wasm, and any updates from + // Wasm are visible reading the returned slice. + // + // For example: + // buf, _ = memory.Read(offset, byteCount) + // buf[1] = 'a' // writes through to memory, meaning Wasm code see 'a'. + // + // If you don't intend-write through, make a copy of the returned slice. + // + // When to refresh Read + // + // The returned slice disconnects on any capacity change. For example, + // `buf = append(buf, 'a')` might result in a slice that is no longer + // shared. The same exists Wasm side. For example, if Wasm changes its + // memory capacity, ex via "memory.grow"), the host slice is no longer + // shared. Those who need a stable view must set Wasm memory min=max, or + // use wazero.RuntimeConfig WithMemoryCapacityPages to ensure max is always + // allocated. + Read(offset, byteCount uint32) ([]byte, bool) + + // WriteByte writes a single byte to the underlying buffer at the offset in or returns false if out of range. + WriteByte(offset uint32, v byte) bool + + // WriteUint16Le writes the value in little-endian encoding to the underlying buffer at the offset in or returns + // false if out of range. + WriteUint16Le(offset uint32, v uint16) bool + + // WriteUint32Le writes the value in little-endian encoding to the underlying buffer at the offset in or returns + // false if out of range. + WriteUint32Le(offset, v uint32) bool + + // WriteFloat32Le writes the value in 32 IEEE 754 little-endian encoded bits to the underlying buffer at the offset + // or returns false if out of range. + // + // See math.Float32bits + WriteFloat32Le(offset uint32, v float32) bool + + // WriteUint64Le writes the value in little-endian encoding to the underlying buffer at the offset in or returns + // false if out of range. + WriteUint64Le(offset uint32, v uint64) bool + + // WriteFloat64Le writes the value in 64 IEEE 754 little-endian encoded bits to the underlying buffer at the offset + // or returns false if out of range. + // + // See math.Float64bits + WriteFloat64Le(offset uint32, v float64) bool + + // Write writes the slice to the underlying buffer at the offset or returns false if out of range. + Write(offset uint32, v []byte) bool + + // WriteString writes the string to the underlying buffer at the offset or returns false if out of range. + WriteString(offset uint32, v string) bool + + internalapi.WazeroOnly +} + +// CustomSection contains the name and raw data of a custom section. +// +// # Notes +// +// - This is an interface for decoupling, not third-party implementations. +// All implementations are in wazero. +type CustomSection interface { + // Name is the name of the custom section + Name() string + // Data is the raw data of the custom section + Data() []byte + + internalapi.WazeroOnly +} + +// EncodeExternref encodes the input as a ValueTypeExternref. +// +// See DecodeExternref +func EncodeExternref(input uintptr) uint64 { + return uint64(input) +} + +// DecodeExternref decodes the input as a ValueTypeExternref. +// +// See EncodeExternref +func DecodeExternref(input uint64) uintptr { + return uintptr(input) +} + +// EncodeI32 encodes the input as a ValueTypeI32. +func EncodeI32(input int32) uint64 { + return uint64(uint32(input)) +} + +// DecodeI32 decodes the input as a ValueTypeI32. +func DecodeI32(input uint64) int32 { + return int32(input) +} + +// EncodeU32 encodes the input as a ValueTypeI32. +func EncodeU32(input uint32) uint64 { + return uint64(input) +} + +// DecodeU32 decodes the input as a ValueTypeI32. +func DecodeU32(input uint64) uint32 { + return uint32(input) +} + +// EncodeI64 encodes the input as a ValueTypeI64. +func EncodeI64(input int64) uint64 { + return uint64(input) +} + +// EncodeF32 encodes the input as a ValueTypeF32. +// +// See DecodeF32 +func EncodeF32(input float32) uint64 { + return uint64(math.Float32bits(input)) +} + +// DecodeF32 decodes the input as a ValueTypeF32. +// +// See EncodeF32 +func DecodeF32(input uint64) float32 { + return math.Float32frombits(uint32(input)) +} + +// EncodeF64 encodes the input as a ValueTypeF64. +// +// See EncodeF32 +func EncodeF64(input float64) uint64 { + return math.Float64bits(input) +} + +// DecodeF64 decodes the input as a ValueTypeF64. +// +// See EncodeF64 +func DecodeF64(input uint64) float64 { + return math.Float64frombits(input) +} diff --git a/vendor/github.com/tetratelabs/wazero/builder.go b/vendor/github.com/tetratelabs/wazero/builder.go new file mode 100644 index 000000000..f64afabdf --- /dev/null +++ b/vendor/github.com/tetratelabs/wazero/builder.go @@ -0,0 +1,352 @@ +package wazero + +import ( + "context" + + "github.com/tetratelabs/wazero/api" + "github.com/tetratelabs/wazero/internal/wasm" +) + +// HostFunctionBuilder defines a host function (in Go), so that a +// WebAssembly binary (e.g. %.wasm file) can import and use it. +// +// Here's an example of an addition function: +// +// hostModuleBuilder.NewFunctionBuilder(). +// WithFunc(func(cxt context.Context, x, y uint32) uint32 { +// return x + y +// }). +// Export("add") +// +// # Memory +// +// All host functions act on the importing api.Module, including any memory +// exported in its binary (%.wasm file). If you are reading or writing memory, +// it is sand-boxed Wasm memory defined by the guest. +// +// Below, `m` is the importing module, defined in Wasm. `fn` is a host function +// added via Export. This means that `x` was read from memory defined in Wasm, +// not arbitrary memory in the process. +// +// fn := func(ctx context.Context, m api.Module, offset uint32) uint32 { +// x, _ := m.Memory().ReadUint32Le(ctx, offset) +// return x +// } +// +// # Notes +// +// - This is an interface for decoupling, not third-party implementations. +// All implementations are in wazero. +type HostFunctionBuilder interface { + // WithGoFunction is an advanced feature for those who need higher + // performance than WithFunc at the cost of more complexity. + // + // Here's an example addition function: + // + // builder.WithGoFunction(api.GoFunc(func(ctx context.Context, stack []uint64) { + // x, y := api.DecodeI32(stack[0]), api.DecodeI32(stack[1]) + // sum := x + y + // stack[0] = api.EncodeI32(sum) + // }), []api.ValueType{api.ValueTypeI32, api.ValueTypeI32}, []api.ValueType{api.ValueTypeI32}) + // + // As you can see above, defining in this way implies knowledge of which + // WebAssembly api.ValueType is appropriate for each parameter and result. + // + // See WithGoModuleFunction if you also need to access the calling module. + WithGoFunction(fn api.GoFunction, params, results []api.ValueType) HostFunctionBuilder + + // WithGoModuleFunction is an advanced feature for those who need higher + // performance than WithFunc at the cost of more complexity. + // + // Here's an example addition function that loads operands from memory: + // + // builder.WithGoModuleFunction(api.GoModuleFunc(func(ctx context.Context, m api.Module, stack []uint64) { + // mem := m.Memory() + // offset := api.DecodeU32(stack[0]) + // + // x, _ := mem.ReadUint32Le(ctx, offset) + // y, _ := mem.ReadUint32Le(ctx, offset + 4) // 32 bits == 4 bytes! + // sum := x + y + // + // stack[0] = api.EncodeU32(sum) + // }), []api.ValueType{api.ValueTypeI32}, []api.ValueType{api.ValueTypeI32}) + // + // As you can see above, defining in this way implies knowledge of which + // WebAssembly api.ValueType is appropriate for each parameter and result. + // + // See WithGoFunction if you don't need access to the calling module. + WithGoModuleFunction(fn api.GoModuleFunction, params, results []api.ValueType) HostFunctionBuilder + + // WithFunc uses reflect.Value to map a go `func` to a WebAssembly + // compatible Signature. An input that isn't a `func` will fail to + // instantiate. + // + // Here's an example of an addition function: + // + // builder.WithFunc(func(cxt context.Context, x, y uint32) uint32 { + // return x + y + // }) + // + // # Defining a function + // + // Except for the context.Context and optional api.Module, all parameters + // or result types must map to WebAssembly numeric value types. This means + // uint32, int32, uint64, int64, float32 or float64. + // + // api.Module may be specified as the second parameter, usually to access + // memory. This is important because there are only numeric types in Wasm. + // The only way to share other data is via writing memory and sharing + // offsets. + // + // builder.WithFunc(func(ctx context.Context, m api.Module, offset uint32) uint32 { + // mem := m.Memory() + // x, _ := mem.ReadUint32Le(ctx, offset) + // y, _ := mem.ReadUint32Le(ctx, offset + 4) // 32 bits == 4 bytes! + // return x + y + // }) + // + // This example propagates context properly when calling other functions + // exported in the api.Module: + // + // builder.WithFunc(func(ctx context.Context, m api.Module, offset, byteCount uint32) uint32 { + // fn = m.ExportedFunction("__read") + // results, err := fn(ctx, offset, byteCount) + // --snip-- + WithFunc(interface{}) HostFunctionBuilder + + // WithName defines the optional module-local name of this function, e.g. + // "random_get" + // + // Note: This is not required to match the Export name. + WithName(name string) HostFunctionBuilder + + // WithParameterNames defines optional parameter names of the function + // signature, e.x. "buf", "buf_len" + // + // Note: When defined, names must be provided for all parameters. + WithParameterNames(names ...string) HostFunctionBuilder + + // WithResultNames defines optional result names of the function + // signature, e.x. "errno" + // + // Note: When defined, names must be provided for all results. + WithResultNames(names ...string) HostFunctionBuilder + + // Export exports this to the HostModuleBuilder as the given name, e.g. + // "random_get" + Export(name string) HostModuleBuilder +} + +// HostModuleBuilder is a way to define host functions (in Go), so that a +// WebAssembly binary (e.g. %.wasm file) can import and use them. +// +// Specifically, this implements the host side of an Application Binary +// Interface (ABI) like WASI or AssemblyScript. +// +// For example, this defines and instantiates a module named "env" with one +// function: +// +// ctx := context.Background() +// r := wazero.NewRuntime(ctx) +// defer r.Close(ctx) // This closes everything this Runtime created. +// +// hello := func() { +// println("hello!") +// } +// env, _ := r.NewHostModuleBuilder("env"). +// NewFunctionBuilder().WithFunc(hello).Export("hello"). +// Instantiate(ctx) +// +// If the same module may be instantiated multiple times, it is more efficient +// to separate steps. Here's an example: +// +// compiled, _ := r.NewHostModuleBuilder("env"). +// NewFunctionBuilder().WithFunc(getRandomString).Export("get_random_string"). +// Compile(ctx) +// +// env1, _ := r.InstantiateModule(ctx, compiled, wazero.NewModuleConfig().WithName("env.1")) +// env2, _ := r.InstantiateModule(ctx, compiled, wazero.NewModuleConfig().WithName("env.2")) +// +// See HostFunctionBuilder for valid host function signatures and other details. +// +// # Notes +// +// - This is an interface for decoupling, not third-party implementations. +// All implementations are in wazero. +// - HostModuleBuilder is mutable: each method returns the same instance for +// chaining. +// - methods do not return errors, to allow chaining. Any validation errors +// are deferred until Compile. +// - Functions are indexed in order of calls to NewFunctionBuilder as +// insertion ordering is needed by ABI such as Emscripten (invoke_*). +type HostModuleBuilder interface { + // Note: until golang/go#5860, we can't use example tests to embed code in interface godocs. + + // NewFunctionBuilder begins the definition of a host function. + NewFunctionBuilder() HostFunctionBuilder + + // Compile returns a CompiledModule that can be instantiated by Runtime. + Compile(context.Context) (CompiledModule, error) + + // Instantiate is a convenience that calls Compile, then Runtime.InstantiateModule. + // This can fail for reasons documented on Runtime.InstantiateModule. + // + // Here's an example: + // + // ctx := context.Background() + // r := wazero.NewRuntime(ctx) + // defer r.Close(ctx) // This closes everything this Runtime created. + // + // hello := func() { + // println("hello!") + // } + // env, _ := r.NewHostModuleBuilder("env"). + // NewFunctionBuilder().WithFunc(hello).Export("hello"). + // Instantiate(ctx) + // + // # Notes + // + // - Closing the Runtime has the same effect as closing the result. + // - Fields in the builder are copied during instantiation: Later changes do not affect the instantiated result. + // - To avoid using configuration defaults, use Compile instead. + Instantiate(context.Context) (api.Module, error) +} + +// hostModuleBuilder implements HostModuleBuilder +type hostModuleBuilder struct { + r *runtime + moduleName string + exportNames []string + nameToHostFunc map[string]*wasm.HostFunc +} + +// NewHostModuleBuilder implements Runtime.NewHostModuleBuilder +func (r *runtime) NewHostModuleBuilder(moduleName string) HostModuleBuilder { + return &hostModuleBuilder{ + r: r, + moduleName: moduleName, + nameToHostFunc: map[string]*wasm.HostFunc{}, + } +} + +// hostFunctionBuilder implements HostFunctionBuilder +type hostFunctionBuilder struct { + b *hostModuleBuilder + fn interface{} + name string + paramNames []string + resultNames []string +} + +// WithGoFunction implements HostFunctionBuilder.WithGoFunction +func (h *hostFunctionBuilder) WithGoFunction(fn api.GoFunction, params, results []api.ValueType) HostFunctionBuilder { + h.fn = &wasm.HostFunc{ParamTypes: params, ResultTypes: results, Code: wasm.Code{GoFunc: fn}} + return h +} + +// WithGoModuleFunction implements HostFunctionBuilder.WithGoModuleFunction +func (h *hostFunctionBuilder) WithGoModuleFunction(fn api.GoModuleFunction, params, results []api.ValueType) HostFunctionBuilder { + h.fn = &wasm.HostFunc{ParamTypes: params, ResultTypes: results, Code: wasm.Code{GoFunc: fn}} + return h +} + +// WithFunc implements HostFunctionBuilder.WithFunc +func (h *hostFunctionBuilder) WithFunc(fn interface{}) HostFunctionBuilder { + h.fn = fn + return h +} + +// WithName implements HostFunctionBuilder.WithName +func (h *hostFunctionBuilder) WithName(name string) HostFunctionBuilder { + h.name = name + return h +} + +// WithParameterNames implements HostFunctionBuilder.WithParameterNames +func (h *hostFunctionBuilder) WithParameterNames(names ...string) HostFunctionBuilder { + h.paramNames = names + return h +} + +// WithResultNames implements HostFunctionBuilder.WithResultNames +func (h *hostFunctionBuilder) WithResultNames(names ...string) HostFunctionBuilder { + h.resultNames = names + return h +} + +// Export implements HostFunctionBuilder.Export +func (h *hostFunctionBuilder) Export(exportName string) HostModuleBuilder { + var hostFn *wasm.HostFunc + if fn, ok := h.fn.(*wasm.HostFunc); ok { + hostFn = fn + } else { + hostFn = &wasm.HostFunc{Code: wasm.Code{GoFunc: h.fn}} + } + + // Assign any names from the builder + hostFn.ExportName = exportName + if h.name != "" { + hostFn.Name = h.name + } + if len(h.paramNames) != 0 { + hostFn.ParamNames = h.paramNames + } + if len(h.resultNames) != 0 { + hostFn.ResultNames = h.resultNames + } + + h.b.ExportHostFunc(hostFn) + return h.b +} + +// ExportHostFunc implements wasm.HostFuncExporter +func (b *hostModuleBuilder) ExportHostFunc(fn *wasm.HostFunc) { + if _, ok := b.nameToHostFunc[fn.ExportName]; !ok { // add a new name + b.exportNames = append(b.exportNames, fn.ExportName) + } + b.nameToHostFunc[fn.ExportName] = fn +} + +// NewFunctionBuilder implements HostModuleBuilder.NewFunctionBuilder +func (b *hostModuleBuilder) NewFunctionBuilder() HostFunctionBuilder { + return &hostFunctionBuilder{b: b} +} + +// Compile implements HostModuleBuilder.Compile +func (b *hostModuleBuilder) Compile(ctx context.Context) (CompiledModule, error) { + module, err := wasm.NewHostModule(b.moduleName, b.exportNames, b.nameToHostFunc, b.r.enabledFeatures) + if err != nil { + return nil, err + } else if err = module.Validate(b.r.enabledFeatures); err != nil { + return nil, err + } + + c := &compiledModule{module: module, compiledEngine: b.r.store.Engine} + listeners, err := buildFunctionListeners(ctx, module) + if err != nil { + return nil, err + } + + if err = b.r.store.Engine.CompileModule(ctx, module, listeners, false); err != nil { + return nil, err + } + + // typeIDs are static and compile-time known. + typeIDs, err := b.r.store.GetFunctionTypeIDs(module.TypeSection) + if err != nil { + return nil, err + } + c.typeIDs = typeIDs + + return c, nil +} + +// Instantiate implements HostModuleBuilder.Instantiate +func (b *hostModuleBuilder) Instantiate(ctx context.Context) (api.Module, error) { + if compiled, err := b.Compile(ctx); err != nil { + return nil, err + } else { + compiled.(*compiledModule).closeWithModule = true + return b.r.InstantiateModule(ctx, compiled, NewModuleConfig()) + } +} diff --git a/vendor/github.com/tetratelabs/wazero/cache.go b/vendor/github.com/tetratelabs/wazero/cache.go new file mode 100644 index 000000000..2d1b4e3b9 --- /dev/null +++ b/vendor/github.com/tetratelabs/wazero/cache.go @@ -0,0 +1,116 @@ +package wazero + +import ( + "context" + "errors" + "fmt" + "os" + "path" + "path/filepath" + goruntime "runtime" + "sync" + + "github.com/tetratelabs/wazero/api" + "github.com/tetratelabs/wazero/internal/filecache" + "github.com/tetratelabs/wazero/internal/version" + "github.com/tetratelabs/wazero/internal/wasm" +) + +// CompilationCache reduces time spent compiling (Runtime.CompileModule) the same wasm module. +// +// # Notes +// +// - This is an interface for decoupling, not third-party implementations. +// All implementations are in wazero. +// - Instances of this can be reused across multiple runtimes, if configured +// via RuntimeConfig. +type CompilationCache interface{ api.Closer } + +// NewCompilationCache returns a new CompilationCache to be passed to RuntimeConfig. +// This configures only in-memory cache, and doesn't persist to the file system. See wazero.NewCompilationCacheWithDir for detail. +// +// The returned CompilationCache can be used to share the in-memory compilation results across multiple instances of wazero.Runtime. +func NewCompilationCache() CompilationCache { + return &cache{} +} + +// NewCompilationCacheWithDir is like wazero.NewCompilationCache except the result also writes +// state into the directory specified by `dirname` parameter. +// +// If the dirname doesn't exist, this creates it or returns an error. +// +// Those running wazero as a CLI or frequently restarting a process using the same wasm should +// use this feature to reduce time waiting to compile the same module a second time. +// +// The contents written into dirname are wazero-version specific, meaning different versions of +// wazero will duplicate entries for the same input wasm. +// +// Note: The embedder must safeguard this directory from external changes. +func NewCompilationCacheWithDir(dirname string) (CompilationCache, error) { + c := &cache{} + err := c.ensuresFileCache(dirname, version.GetWazeroVersion()) + return c, err +} + +// cache implements Cache interface. +type cache struct { + // eng is the engine for this cache. If the cache is configured, the engine is shared across multiple instances of + // Runtime, and its lifetime is not bound to them. Instead, the engine is alive until Cache.Close is called. + engs [engineKindCount]wasm.Engine + fileCache filecache.Cache + initOnces [engineKindCount]sync.Once +} + +func (c *cache) initEngine(ek engineKind, ne newEngine, ctx context.Context, features api.CoreFeatures) wasm.Engine { + c.initOnces[ek].Do(func() { c.engs[ek] = ne(ctx, features, c.fileCache) }) + return c.engs[ek] +} + +// Close implements the same method on the Cache interface. +func (c *cache) Close(_ context.Context) (err error) { + for _, eng := range c.engs { + if eng != nil { + if err = eng.Close(); err != nil { + return + } + } + } + return +} + +func (c *cache) ensuresFileCache(dir string, wazeroVersion string) error { + // Resolve a potentially relative directory into an absolute one. + var err error + dir, err = filepath.Abs(dir) + if err != nil { + return err + } + + // Ensure the user-supplied directory. + if err = mkdir(dir); err != nil { + return err + } + + // Create a version-specific directory to avoid conflicts. + dirname := path.Join(dir, "wazero-"+wazeroVersion+"-"+goruntime.GOARCH+"-"+goruntime.GOOS) + if err = mkdir(dirname); err != nil { + return err + } + + c.fileCache = filecache.New(dirname) + return nil +} + +func mkdir(dirname string) error { + if st, err := os.Stat(dirname); errors.Is(err, os.ErrNotExist) { + // If the directory not found, create the cache dir. + if err = os.MkdirAll(dirname, 0o700); err != nil { + return fmt.Errorf("create directory %s: %v", dirname, err) + } + } else if err != nil { + return err + } else if !st.IsDir() { + return fmt.Errorf("%s is not dir", dirname) + } + return nil +} diff --git a/vendor/github.com/tetratelabs/wazero/codecov.yml b/vendor/github.com/tetratelabs/wazero/codecov.yml new file mode 100644 index 000000000..cf9d94df4 --- /dev/null +++ b/vendor/github.com/tetratelabs/wazero/codecov.yml @@ -0,0 +1,9 @@ +# Codecov for main is visible here https://app.codecov.io/gh/tetratelabs/wazero + +# We use codecov only as a UI, so we disable PR comments and commit status. +# See https://docs.codecov.com/docs/pull-request-comments +comment: false +coverage: + status: + project: off + patch: off diff --git a/vendor/github.com/tetratelabs/wazero/config.go b/vendor/github.com/tetratelabs/wazero/config.go new file mode 100644 index 000000000..819a76df5 --- /dev/null +++ b/vendor/github.com/tetratelabs/wazero/config.go @@ -0,0 +1,876 @@ +package wazero + +import ( + "context" + "errors" + "fmt" + "io" + "io/fs" + "math" + "net" + "time" + + "github.com/tetratelabs/wazero/api" + experimentalsys "github.com/tetratelabs/wazero/experimental/sys" + "github.com/tetratelabs/wazero/internal/engine/interpreter" + "github.com/tetratelabs/wazero/internal/engine/wazevo" + "github.com/tetratelabs/wazero/internal/filecache" + "github.com/tetratelabs/wazero/internal/internalapi" + "github.com/tetratelabs/wazero/internal/platform" + internalsock "github.com/tetratelabs/wazero/internal/sock" + internalsys "github.com/tetratelabs/wazero/internal/sys" + "github.com/tetratelabs/wazero/internal/wasm" + "github.com/tetratelabs/wazero/sys" +) + +// RuntimeConfig controls runtime behavior, with the default implementation as +// NewRuntimeConfig +// +// The example below explicitly limits to Wasm Core 1.0 features as opposed to +// relying on defaults: +// +// rConfig = wazero.NewRuntimeConfig().WithCoreFeatures(api.CoreFeaturesV1) +// +// # Notes +// +// - This is an interface for decoupling, not third-party implementations. +// All implementations are in wazero. +// - RuntimeConfig is immutable. Each WithXXX function returns a new instance +// including the corresponding change. +type RuntimeConfig interface { + // WithCoreFeatures sets the WebAssembly Core specification features this + // runtime supports. Defaults to api.CoreFeaturesV2. + // + // Example of disabling a specific feature: + // features := api.CoreFeaturesV2.SetEnabled(api.CoreFeatureMutableGlobal, false) + // rConfig = wazero.NewRuntimeConfig().WithCoreFeatures(features) + // + // # Why default to version 2.0? + // + // Many compilers that target WebAssembly require features after + // api.CoreFeaturesV1 by default. For example, TinyGo v0.24+ requires + // api.CoreFeatureBulkMemoryOperations. To avoid runtime errors, wazero + // defaults to api.CoreFeaturesV2, even though it is not yet a Web + // Standard (REC). + WithCoreFeatures(api.CoreFeatures) RuntimeConfig + + // WithMemoryLimitPages overrides the maximum pages allowed per memory. The + // default is 65536, allowing 4GB total memory per instance if the maximum is + // not encoded in a Wasm binary. Setting a value larger than default will panic. + // + // This example reduces the largest possible memory size from 4GB to 128KB: + // rConfig = wazero.NewRuntimeConfig().WithMemoryLimitPages(2) + // + // Note: Wasm has 32-bit memory and each page is 65536 (2^16) bytes. This + // implies a max of 65536 (2^16) addressable pages. + // See https://www.w3.org/TR/2019/REC-wasm-core-1-20191205/#grow-mem + WithMemoryLimitPages(memoryLimitPages uint32) RuntimeConfig + + // WithMemoryCapacityFromMax eagerly allocates max memory, unless max is + // not defined. The default is false, which means minimum memory is + // allocated and any call to grow memory results in re-allocations. + // + // This example ensures any memory.grow instruction will never re-allocate: + // rConfig = wazero.NewRuntimeConfig().WithMemoryCapacityFromMax(true) + // + // See https://www.w3.org/TR/2019/REC-wasm-core-1-20191205/#grow-mem + // + // Note: if the memory maximum is not encoded in a Wasm binary, this + // results in allocating 4GB. See the doc on WithMemoryLimitPages for detail. + WithMemoryCapacityFromMax(memoryCapacityFromMax bool) RuntimeConfig + + // WithDebugInfoEnabled toggles DWARF based stack traces in the face of + // runtime errors. Defaults to true. + // + // Those who wish to disable this, can like so: + // + // r := wazero.NewRuntimeWithConfig(wazero.NewRuntimeConfig().WithDebugInfoEnabled(false) + // + // When disabled, a stack trace message looks like: + // + // wasm stack trace: + // .runtime._panic(i32) + // .myFunc() + // .main.main() + // .runtime.run() + // ._start() + // + // When enabled, the stack trace includes source code information: + // + // wasm stack trace: + // .runtime._panic(i32) + // 0x16e2: /opt/homebrew/Cellar/tinygo/0.26.0/src/runtime/runtime_tinygowasm.go:73:6 + // .myFunc() + // 0x190b: /Users/XXXXX/wazero/internal/testing/dwarftestdata/testdata/main.go:19:7 + // .main.main() + // 0x18ed: /Users/XXXXX/wazero/internal/testing/dwarftestdata/testdata/main.go:4:3 + // .runtime.run() + // 0x18cc: /opt/homebrew/Cellar/tinygo/0.26.0/src/runtime/scheduler_none.go:26:10 + // ._start() + // 0x18b6: /opt/homebrew/Cellar/tinygo/0.26.0/src/runtime/runtime_wasm_wasi.go:22:5 + // + // Note: This only takes into effect when the original Wasm binary has the + // DWARF "custom sections" that are often stripped, depending on + // optimization flags passed to the compiler. + WithDebugInfoEnabled(bool) RuntimeConfig + + // WithCompilationCache configures how runtime caches the compiled modules. In the default configuration, compilation results are + // only in-memory until Runtime.Close is closed, and not shareable by multiple Runtime. + // + // Below defines the shared cache across multiple instances of Runtime: + // + // // Creates the new Cache and the runtime configuration with it. + // cache := wazero.NewCompilationCache() + // defer cache.Close() + // config := wazero.NewRuntimeConfig().WithCompilationCache(c) + // + // // Creates two runtimes while sharing compilation caches. + // foo := wazero.NewRuntimeWithConfig(context.Background(), config) + // bar := wazero.NewRuntimeWithConfig(context.Background(), config) + // + // # Cache Key + // + // Cached files are keyed on the version of wazero. This is obtained from go.mod of your application, + // and we use it to verify the compatibility of caches against the currently-running wazero. + // However, if you use this in tests of a package not named as `main`, then wazero cannot obtain the correct + // version of wazero due to the known issue of debug.BuildInfo function: https://github.com/golang/go/issues/33976. + // As a consequence, your cache won't contain the correct version information and always be treated as `dev` version. + // To avoid this issue, you can pass -ldflags "-X github.com/tetratelabs/wazero/internal/version.version=foo" when running tests. + WithCompilationCache(CompilationCache) RuntimeConfig + + // WithCustomSections toggles parsing of "custom sections". Defaults to false. + // + // When enabled, it is possible to retrieve custom sections from a CompiledModule: + // + // config := wazero.NewRuntimeConfig().WithCustomSections(true) + // r := wazero.NewRuntimeWithConfig(ctx, config) + // c, err := r.CompileModule(ctx, wasm) + // customSections := c.CustomSections() + WithCustomSections(bool) RuntimeConfig + + // WithCloseOnContextDone ensures the executions of functions to be closed under one of the following circumstances: + // + // - context.Context passed to the Call method of api.Function is canceled during execution. (i.e. ctx by context.WithCancel) + // - context.Context passed to the Call method of api.Function reaches timeout during execution. (i.e. ctx by context.WithTimeout or context.WithDeadline) + // - Close or CloseWithExitCode of api.Module is explicitly called during execution. + // + // This is especially useful when one wants to run untrusted Wasm binaries since otherwise, any invocation of + // api.Function can potentially block the corresponding Goroutine forever. Moreover, it might block the + // entire underlying OS thread which runs the api.Function call. See "Why it's safe to execute runtime-generated + // machine codes against async Goroutine preemption" section in RATIONALE.md for detail. + // + // Note that this comes with a bit of extra cost when enabled. The reason is that internally this forces + // interpreter and compiler runtimes to insert the periodical checks on the conditions above. For that reason, + // this is disabled by default. + // + // See examples in context_done_example_test.go for the end-to-end demonstrations. + // + // When the invocations of api.Function are closed due to this, sys.ExitError is raised to the callers and + // the api.Module from which the functions are derived is made closed. + WithCloseOnContextDone(bool) RuntimeConfig +} + +// NewRuntimeConfig returns a RuntimeConfig using the compiler if it is supported in this environment, +// or the interpreter otherwise. +func NewRuntimeConfig() RuntimeConfig { + return newRuntimeConfig() +} + +type newEngine func(context.Context, api.CoreFeatures, filecache.Cache) wasm.Engine + +type runtimeConfig struct { + enabledFeatures api.CoreFeatures + memoryLimitPages uint32 + memoryCapacityFromMax bool + engineKind engineKind + dwarfDisabled bool // negative as defaults to enabled + newEngine newEngine + cache CompilationCache + storeCustomSections bool + ensureTermination bool +} + +// engineLessConfig helps avoid copy/pasting the wrong defaults. +var engineLessConfig = &runtimeConfig{ + enabledFeatures: api.CoreFeaturesV2, + memoryLimitPages: wasm.MemoryLimitPages, + memoryCapacityFromMax: false, + dwarfDisabled: false, +} + +type engineKind int + +const ( + engineKindCompiler engineKind = iota + engineKindInterpreter + engineKindCount +) + +// NewRuntimeConfigCompiler compiles WebAssembly modules into +// runtime.GOARCH-specific assembly for optimal performance. +// +// The default implementation is AOT (Ahead of Time) compilation, applied at +// Runtime.CompileModule. This allows consistent runtime performance, as well +// the ability to reduce any first request penalty. +// +// Note: While this is technically AOT, this does not imply any action on your +// part. wazero automatically performs ahead-of-time compilation as needed when +// Runtime.CompileModule is invoked. +// +// Warning: This panics at runtime if the runtime.GOOS or runtime.GOARCH does not +// support compiler. Use NewRuntimeConfig to safely detect and fallback to +// NewRuntimeConfigInterpreter if needed. +func NewRuntimeConfigCompiler() RuntimeConfig { + ret := engineLessConfig.clone() + ret.engineKind = engineKindCompiler + ret.newEngine = wazevo.NewEngine + return ret +} + +// NewRuntimeConfigInterpreter interprets WebAssembly modules instead of compiling them into assembly. +func NewRuntimeConfigInterpreter() RuntimeConfig { + ret := engineLessConfig.clone() + ret.engineKind = engineKindInterpreter + ret.newEngine = interpreter.NewEngine + return ret +} + +// clone makes a deep copy of this runtime config. +func (c *runtimeConfig) clone() *runtimeConfig { + ret := *c // copy except maps which share a ref + return &ret +} + +// WithCoreFeatures implements RuntimeConfig.WithCoreFeatures +func (c *runtimeConfig) WithCoreFeatures(features api.CoreFeatures) RuntimeConfig { + ret := c.clone() + ret.enabledFeatures = features + return ret +} + +// WithCloseOnContextDone implements RuntimeConfig.WithCloseOnContextDone +func (c *runtimeConfig) WithCloseOnContextDone(ensure bool) RuntimeConfig { + ret := c.clone() + ret.ensureTermination = ensure + return ret +} + +// WithMemoryLimitPages implements RuntimeConfig.WithMemoryLimitPages +func (c *runtimeConfig) WithMemoryLimitPages(memoryLimitPages uint32) RuntimeConfig { + ret := c.clone() + // This panics instead of returning an error as it is unlikely. + if memoryLimitPages > wasm.MemoryLimitPages { + panic(fmt.Errorf("memoryLimitPages invalid: %d > %d", memoryLimitPages, wasm.MemoryLimitPages)) + } + ret.memoryLimitPages = memoryLimitPages + return ret +} + +// WithCompilationCache implements RuntimeConfig.WithCompilationCache +func (c *runtimeConfig) WithCompilationCache(ca CompilationCache) RuntimeConfig { + ret := c.clone() + ret.cache = ca + return ret +} + +// WithMemoryCapacityFromMax implements RuntimeConfig.WithMemoryCapacityFromMax +func (c *runtimeConfig) WithMemoryCapacityFromMax(memoryCapacityFromMax bool) RuntimeConfig { + ret := c.clone() + ret.memoryCapacityFromMax = memoryCapacityFromMax + return ret +} + +// WithDebugInfoEnabled implements RuntimeConfig.WithDebugInfoEnabled +func (c *runtimeConfig) WithDebugInfoEnabled(dwarfEnabled bool) RuntimeConfig { + ret := c.clone() + ret.dwarfDisabled = !dwarfEnabled + return ret +} + +// WithCustomSections implements RuntimeConfig.WithCustomSections +func (c *runtimeConfig) WithCustomSections(storeCustomSections bool) RuntimeConfig { + ret := c.clone() + ret.storeCustomSections = storeCustomSections + return ret +} + +// CompiledModule is a WebAssembly module ready to be instantiated (Runtime.InstantiateModule) as an api.Module. +// +// In WebAssembly terminology, this is a decoded, validated, and possibly also compiled module. wazero avoids using +// the name "Module" for both before and after instantiation as the name conflation has caused confusion. +// See https://www.w3.org/TR/2019/REC-wasm-core-1-20191205/#semantic-phases%E2%91%A0 +// +// # Notes +// +// - This is an interface for decoupling, not third-party implementations. +// All implementations are in wazero. +// - Closing the wazero.Runtime closes any CompiledModule it compiled. +type CompiledModule interface { + // Name returns the module name encoded into the binary or empty if not. + Name() string + + // ImportedFunctions returns all the imported functions + // (api.FunctionDefinition) in this module or nil if there are none. + // + // Note: Unlike ExportedFunctions, there is no unique constraint on + // imports. + ImportedFunctions() []api.FunctionDefinition + + // ExportedFunctions returns all the exported functions + // (api.FunctionDefinition) in this module keyed on export name. + ExportedFunctions() map[string]api.FunctionDefinition + + // ImportedMemories returns all the imported memories + // (api.MemoryDefinition) in this module or nil if there are none. + // + // ## Notes + // - As of WebAssembly Core Specification 2.0, there can be at most one + // memory. + // - Unlike ExportedMemories, there is no unique constraint on imports. + ImportedMemories() []api.MemoryDefinition + + // ExportedMemories returns all the exported memories + // (api.MemoryDefinition) in this module keyed on export name. + // + // Note: As of WebAssembly Core Specification 2.0, there can be at most one + // memory. + ExportedMemories() map[string]api.MemoryDefinition + + // CustomSections returns all the custom sections + // (api.CustomSection) in this module keyed on the section name. + CustomSections() []api.CustomSection + + // Close releases all the allocated resources for this CompiledModule. + // + // Note: It is safe to call Close while having outstanding calls from an + // api.Module instantiated from this. + Close(context.Context) error +} + +// compile-time check to ensure compiledModule implements CompiledModule +var _ CompiledModule = &compiledModule{} + +type compiledModule struct { + module *wasm.Module + // compiledEngine holds an engine on which `module` is compiled. + compiledEngine wasm.Engine + // closeWithModule prevents leaking compiled code when a module is compiled implicitly. + closeWithModule bool + typeIDs []wasm.FunctionTypeID +} + +// Name implements CompiledModule.Name +func (c *compiledModule) Name() (moduleName string) { + if ns := c.module.NameSection; ns != nil { + moduleName = ns.ModuleName + } + return +} + +// Close implements CompiledModule.Close +func (c *compiledModule) Close(context.Context) error { + c.compiledEngine.DeleteCompiledModule(c.module) + // It is possible the underlying may need to return an error later, but in any case this matches api.Module.Close. + return nil +} + +// ImportedFunctions implements CompiledModule.ImportedFunctions +func (c *compiledModule) ImportedFunctions() []api.FunctionDefinition { + return c.module.ImportedFunctions() +} + +// ExportedFunctions implements CompiledModule.ExportedFunctions +func (c *compiledModule) ExportedFunctions() map[string]api.FunctionDefinition { + return c.module.ExportedFunctions() +} + +// ImportedMemories implements CompiledModule.ImportedMemories +func (c *compiledModule) ImportedMemories() []api.MemoryDefinition { + return c.module.ImportedMemories() +} + +// ExportedMemories implements CompiledModule.ExportedMemories +func (c *compiledModule) ExportedMemories() map[string]api.MemoryDefinition { + return c.module.ExportedMemories() +} + +// CustomSections implements CompiledModule.CustomSections +func (c *compiledModule) CustomSections() []api.CustomSection { + ret := make([]api.CustomSection, len(c.module.CustomSections)) + for i, d := range c.module.CustomSections { + ret[i] = &customSection{data: d.Data, name: d.Name} + } + return ret +} + +// customSection implements wasm.CustomSection +type customSection struct { + internalapi.WazeroOnlyType + name string + data []byte +} + +// Name implements wasm.CustomSection.Name +func (c *customSection) Name() string { + return c.name +} + +// Data implements wasm.CustomSection.Data +func (c *customSection) Data() []byte { + return c.data +} + +// ModuleConfig configures resources needed by functions that have low-level interactions with the host operating +// system. Using this, resources such as STDIN can be isolated, so that the same module can be safely instantiated +// multiple times. +// +// Here's an example: +// +// // Initialize base configuration: +// config := wazero.NewModuleConfig().WithStdout(buf).WithSysNanotime() +// +// // Assign different configuration on each instantiation +// mod, _ := r.InstantiateModule(ctx, compiled, config.WithName("rotate").WithArgs("rotate", "angle=90", "dir=cw")) +// +// While wazero supports Windows as a platform, host functions using ModuleConfig follow a UNIX dialect. +// See RATIONALE.md for design background and relationship to WebAssembly System Interfaces (WASI). +// +// # Notes +// +// - This is an interface for decoupling, not third-party implementations. +// All implementations are in wazero. +// - ModuleConfig is immutable. Each WithXXX function returns a new instance +// including the corresponding change. +type ModuleConfig interface { + // WithArgs assigns command-line arguments visible to an imported function that reads an arg vector (argv). Defaults to + // none. Runtime.InstantiateModule errs if any arg is empty. + // + // These values are commonly read by the functions like "args_get" in "wasi_snapshot_preview1" although they could be + // read by functions imported from other modules. + // + // Similar to os.Args and exec.Cmd Env, many implementations would expect a program name to be argv[0]. However, neither + // WebAssembly nor WebAssembly System Interfaces (WASI) define this. Regardless, you may choose to set the first + // argument to the same value set via WithName. + // + // Note: This does not default to os.Args as that violates sandboxing. + // + // See https://linux.die.net/man/3/argv and https://en.wikipedia.org/wiki/Null-terminated_string + WithArgs(...string) ModuleConfig + + // WithEnv sets an environment variable visible to a Module that imports functions. Defaults to none. + // Runtime.InstantiateModule errs if the key is empty or contains a NULL(0) or equals("") character. + // + // Validation is the same as os.Setenv on Linux and replaces any existing value. Unlike exec.Cmd Env, this does not + // default to the current process environment as that would violate sandboxing. This also does not preserve order. + // + // Environment variables are commonly read by the functions like "environ_get" in "wasi_snapshot_preview1" although + // they could be read by functions imported from other modules. + // + // While similar to process configuration, there are no assumptions that can be made about anything OS-specific. For + // example, neither WebAssembly nor WebAssembly System Interfaces (WASI) define concerns processes have, such as + // case-sensitivity on environment keys. For portability, define entries with case-insensitively unique keys. + // + // See https://linux.die.net/man/3/environ and https://en.wikipedia.org/wiki/Null-terminated_string + WithEnv(key, value string) ModuleConfig + + // WithFS is a convenience that calls WithFSConfig with an FSConfig of the + // input for the root ("/") guest path. + WithFS(fs.FS) ModuleConfig + + // WithFSConfig configures the filesystem available to each guest + // instantiated with this configuration. By default, no file access is + // allowed, so functions like `path_open` result in unsupported errors + // (e.g. syscall.ENOSYS). + WithFSConfig(FSConfig) ModuleConfig + + // WithName configures the module name. Defaults to what was decoded from + // the name section. Empty string ("") clears any name. + WithName(string) ModuleConfig + + // WithStartFunctions configures the functions to call after the module is + // instantiated. Defaults to "_start". + // + // Clearing the default is supported, via `WithStartFunctions()`. + // + // # Notes + // + // - If a start function doesn't exist, it is skipped. However, any that + // do exist are called in order. + // - Start functions are not intended to be called multiple times. + // Functions that should be called multiple times should be invoked + // manually via api.Module's `ExportedFunction` method. + // - Start functions commonly exit the module during instantiation, + // preventing use of any functions later. This is the case in "wasip1", + // which defines the default value "_start". + // - See /RATIONALE.md for motivation of this feature. + WithStartFunctions(...string) ModuleConfig + + // WithStderr configures where standard error (file descriptor 2) is written. Defaults to io.Discard. + // + // This writer is most commonly used by the functions like "fd_write" in "wasi_snapshot_preview1" although it could + // be used by functions imported from other modules. + // + // # Notes + // + // - The caller is responsible to close any io.Writer they supply: It is not closed on api.Module Close. + // - This does not default to os.Stderr as that both violates sandboxing and prevents concurrent modules. + // + // See https://linux.die.net/man/3/stderr + WithStderr(io.Writer) ModuleConfig + + // WithStdin configures where standard input (file descriptor 0) is read. Defaults to return io.EOF. + // + // This reader is most commonly used by the functions like "fd_read" in "wasi_snapshot_preview1" although it could + // be used by functions imported from other modules. + // + // # Notes + // + // - The caller is responsible to close any io.Reader they supply: It is not closed on api.Module Close. + // - This does not default to os.Stdin as that both violates sandboxing and prevents concurrent modules. + // + // See https://linux.die.net/man/3/stdin + WithStdin(io.Reader) ModuleConfig + + // WithStdout configures where standard output (file descriptor 1) is written. Defaults to io.Discard. + // + // This writer is most commonly used by the functions like "fd_write" in "wasi_snapshot_preview1" although it could + // be used by functions imported from other modules. + // + // # Notes + // + // - The caller is responsible to close any io.Writer they supply: It is not closed on api.Module Close. + // - This does not default to os.Stdout as that both violates sandboxing and prevents concurrent modules. + // + // See https://linux.die.net/man/3/stdout + WithStdout(io.Writer) ModuleConfig + + // WithWalltime configures the wall clock, sometimes referred to as the + // real time clock. sys.Walltime returns the current unix/epoch time, + // seconds since midnight UTC 1 January 1970, with a nanosecond fraction. + // This defaults to a fake result that increases by 1ms on each reading. + // + // Here's an example that uses a custom clock: + // moduleConfig = moduleConfig. + // WithWalltime(func(context.Context) (sec int64, nsec int32) { + // return clock.walltime() + // }, sys.ClockResolution(time.Microsecond.Nanoseconds())) + // + // # Notes: + // - This does not default to time.Now as that violates sandboxing. + // - This is used to implement host functions such as WASI + // `clock_time_get` with the `realtime` clock ID. + // - Use WithSysWalltime for a usable implementation. + WithWalltime(sys.Walltime, sys.ClockResolution) ModuleConfig + + // WithSysWalltime uses time.Now for sys.Walltime with a resolution of 1us + // (1000ns). + // + // See WithWalltime + WithSysWalltime() ModuleConfig + + // WithNanotime configures the monotonic clock, used to measure elapsed + // time in nanoseconds. Defaults to a fake result that increases by 1ms + // on each reading. + // + // Here's an example that uses a custom clock: + // moduleConfig = moduleConfig. + // WithNanotime(func(context.Context) int64 { + // return clock.nanotime() + // }, sys.ClockResolution(time.Microsecond.Nanoseconds())) + // + // # Notes: + // - This does not default to time.Since as that violates sandboxing. + // - This is used to implement host functions such as WASI + // `clock_time_get` with the `monotonic` clock ID. + // - Some compilers implement sleep by looping on sys.Nanotime (e.g. Go). + // - If you set this, you should probably set WithNanosleep also. + // - Use WithSysNanotime for a usable implementation. + WithNanotime(sys.Nanotime, sys.ClockResolution) ModuleConfig + + // WithSysNanotime uses time.Now for sys.Nanotime with a resolution of 1us. + // + // See WithNanotime + WithSysNanotime() ModuleConfig + + // WithNanosleep configures the how to pause the current goroutine for at + // least the configured nanoseconds. Defaults to return immediately. + // + // This example uses a custom sleep function: + // moduleConfig = moduleConfig. + // WithNanosleep(func(ns int64) { + // rel := unix.NsecToTimespec(ns) + // remain := unix.Timespec{} + // for { // loop until no more time remaining + // err := unix.ClockNanosleep(unix.CLOCK_MONOTONIC, 0, &rel, &remain) + // --snip-- + // + // # Notes: + // - This does not default to time.Sleep as that violates sandboxing. + // - This is used to implement host functions such as WASI `poll_oneoff`. + // - Some compilers implement sleep by looping on sys.Nanotime (e.g. Go). + // - If you set this, you should probably set WithNanotime also. + // - Use WithSysNanosleep for a usable implementation. + WithNanosleep(sys.Nanosleep) ModuleConfig + + // WithOsyield yields the processor, typically to implement spin-wait + // loops. Defaults to return immediately. + // + // # Notes: + // - This primarily supports `sched_yield` in WASI + // - This does not default to runtime.osyield as that violates sandboxing. + WithOsyield(sys.Osyield) ModuleConfig + + // WithSysNanosleep uses time.Sleep for sys.Nanosleep. + // + // See WithNanosleep + WithSysNanosleep() ModuleConfig + + // WithRandSource configures a source of random bytes. Defaults to return a + // deterministic source. You might override this with crypto/rand.Reader + // + // This reader is most commonly used by the functions like "random_get" in + // "wasi_snapshot_preview1", "seed" in AssemblyScript standard "env", and + // "getRandomData" when runtime.GOOS is "js". + // + // Note: The caller is responsible to close any io.Reader they supply: It + // is not closed on api.Module Close. + WithRandSource(io.Reader) ModuleConfig +} + +type moduleConfig struct { + name string + nameSet bool + startFunctions []string + stdin io.Reader + stdout io.Writer + stderr io.Writer + randSource io.Reader + walltime sys.Walltime + walltimeResolution sys.ClockResolution + nanotime sys.Nanotime + nanotimeResolution sys.ClockResolution + nanosleep sys.Nanosleep + osyield sys.Osyield + args [][]byte + // environ is pair-indexed to retain order similar to os.Environ. + environ [][]byte + // environKeys allow overwriting of existing values. + environKeys map[string]int + // fsConfig is the file system configuration for ABI like WASI. + fsConfig FSConfig + // sockConfig is the network listener configuration for ABI like WASI. + sockConfig *internalsock.Config +} + +// NewModuleConfig returns a ModuleConfig that can be used for configuring module instantiation. +func NewModuleConfig() ModuleConfig { + return &moduleConfig{ + startFunctions: []string{"_start"}, + environKeys: map[string]int{}, + } +} + +// clone makes a deep copy of this module config. +func (c *moduleConfig) clone() *moduleConfig { + ret := *c // copy except maps which share a ref + ret.environKeys = make(map[string]int, len(c.environKeys)) + for key, value := range c.environKeys { + ret.environKeys[key] = value + } + return &ret +} + +// WithArgs implements ModuleConfig.WithArgs +func (c *moduleConfig) WithArgs(args ...string) ModuleConfig { + ret := c.clone() + ret.args = toByteSlices(args) + return ret +} + +func toByteSlices(strings []string) (result [][]byte) { + if len(strings) == 0 { + return + } + result = make([][]byte, len(strings)) + for i, a := range strings { + result[i] = []byte(a) + } + return +} + +// WithEnv implements ModuleConfig.WithEnv +func (c *moduleConfig) WithEnv(key, value string) ModuleConfig { + ret := c.clone() + // Check to see if this key already exists and update it. + if i, ok := ret.environKeys[key]; ok { + ret.environ[i+1] = []byte(value) // environ is pair-indexed, so the value is 1 after the key. + } else { + ret.environKeys[key] = len(ret.environ) + ret.environ = append(ret.environ, []byte(key), []byte(value)) + } + return ret +} + +// WithFS implements ModuleConfig.WithFS +func (c *moduleConfig) WithFS(fs fs.FS) ModuleConfig { + var config FSConfig + if fs != nil { + config = NewFSConfig().WithFSMount(fs, "") + } + return c.WithFSConfig(config) +} + +// WithFSConfig implements ModuleConfig.WithFSConfig +func (c *moduleConfig) WithFSConfig(config FSConfig) ModuleConfig { + ret := c.clone() + ret.fsConfig = config + return ret +} + +// WithName implements ModuleConfig.WithName +func (c *moduleConfig) WithName(name string) ModuleConfig { + ret := c.clone() + ret.nameSet = true + ret.name = name + return ret +} + +// WithStartFunctions implements ModuleConfig.WithStartFunctions +func (c *moduleConfig) WithStartFunctions(startFunctions ...string) ModuleConfig { + ret := c.clone() + ret.startFunctions = startFunctions + return ret +} + +// WithStderr implements ModuleConfig.WithStderr +func (c *moduleConfig) WithStderr(stderr io.Writer) ModuleConfig { + ret := c.clone() + ret.stderr = stderr + return ret +} + +// WithStdin implements ModuleConfig.WithStdin +func (c *moduleConfig) WithStdin(stdin io.Reader) ModuleConfig { + ret := c.clone() + ret.stdin = stdin + return ret +} + +// WithStdout implements ModuleConfig.WithStdout +func (c *moduleConfig) WithStdout(stdout io.Writer) ModuleConfig { + ret := c.clone() + ret.stdout = stdout + return ret +} + +// WithWalltime implements ModuleConfig.WithWalltime +func (c *moduleConfig) WithWalltime(walltime sys.Walltime, resolution sys.ClockResolution) ModuleConfig { + ret := c.clone() + ret.walltime = walltime + ret.walltimeResolution = resolution + return ret +} + +// We choose arbitrary resolutions here because there's no perfect alternative. For example, according to the +// source in time.go, windows monotonic resolution can be 15ms. This chooses arbitrarily 1us for wall time and +// 1ns for monotonic. See RATIONALE.md for more context. + +// WithSysWalltime implements ModuleConfig.WithSysWalltime +func (c *moduleConfig) WithSysWalltime() ModuleConfig { + return c.WithWalltime(platform.Walltime, sys.ClockResolution(time.Microsecond.Nanoseconds())) +} + +// WithNanotime implements ModuleConfig.WithNanotime +func (c *moduleConfig) WithNanotime(nanotime sys.Nanotime, resolution sys.ClockResolution) ModuleConfig { + ret := c.clone() + ret.nanotime = nanotime + ret.nanotimeResolution = resolution + return ret +} + +// WithSysNanotime implements ModuleConfig.WithSysNanotime +func (c *moduleConfig) WithSysNanotime() ModuleConfig { + return c.WithNanotime(platform.Nanotime, sys.ClockResolution(1)) +} + +// WithNanosleep implements ModuleConfig.WithNanosleep +func (c *moduleConfig) WithNanosleep(nanosleep sys.Nanosleep) ModuleConfig { + ret := *c // copy + ret.nanosleep = nanosleep + return &ret +} + +// WithOsyield implements ModuleConfig.WithOsyield +func (c *moduleConfig) WithOsyield(osyield sys.Osyield) ModuleConfig { + ret := *c // copy + ret.osyield = osyield + return &ret +} + +// WithSysNanosleep implements ModuleConfig.WithSysNanosleep +func (c *moduleConfig) WithSysNanosleep() ModuleConfig { + return c.WithNanosleep(platform.Nanosleep) +} + +// WithRandSource implements ModuleConfig.WithRandSource +func (c *moduleConfig) WithRandSource(source io.Reader) ModuleConfig { + ret := c.clone() + ret.randSource = source + return ret +} + +// toSysContext creates a baseline wasm.Context configured by ModuleConfig. +func (c *moduleConfig) toSysContext() (sysCtx *internalsys.Context, err error) { + var environ [][]byte // Intentionally doesn't pre-allocate to reduce logic to default to nil. + // Same validation as syscall.Setenv for Linux + for i := 0; i < len(c.environ); i += 2 { + key, value := c.environ[i], c.environ[i+1] + keyLen := len(key) + if keyLen == 0 { + err = errors.New("environ invalid: empty key") + return + } + valueLen := len(value) + result := make([]byte, keyLen+valueLen+1) + j := 0 + for ; j < keyLen; j++ { + if k := key[j]; k == '=' { // NUL enforced in NewContext + err = errors.New("environ invalid: key contains '=' character") + return + } else { + result[j] = k + } + } + result[j] = '=' + copy(result[j+1:], value) + environ = append(environ, result) + } + + var fs []experimentalsys.FS + var guestPaths []string + if f, ok := c.fsConfig.(*fsConfig); ok { + fs, guestPaths = f.preopens() + } + + var listeners []*net.TCPListener + if n := c.sockConfig; n != nil { + if listeners, err = n.BuildTCPListeners(); err != nil { + return + } + } + + return internalsys.NewContext( + math.MaxUint32, + c.args, + environ, + c.stdin, + c.stdout, + c.stderr, + c.randSource, + c.walltime, c.walltimeResolution, + c.nanotime, c.nanotimeResolution, + c.nanosleep, c.osyield, + fs, guestPaths, + listeners, + ) +} diff --git a/vendor/github.com/tetratelabs/wazero/config_supported.go b/vendor/github.com/tetratelabs/wazero/config_supported.go new file mode 100644 index 000000000..eb31ab935 --- /dev/null +++ b/vendor/github.com/tetratelabs/wazero/config_supported.go @@ -0,0 +1,14 @@ +// Note: The build constraints here are about the compiler, which is more +// narrow than the architectures supported by the assembler. +// +// Constraints here must match platform.CompilerSupported. +// +// Meanwhile, users who know their runtime.GOOS can operate with the compiler +// may choose to use NewRuntimeConfigCompiler explicitly. +//go:build (amd64 || arm64) && (darwin || linux || freebsd || windows) + +package wazero + +func newRuntimeConfig() RuntimeConfig { + return NewRuntimeConfigCompiler() +} diff --git a/vendor/github.com/tetratelabs/wazero/config_unsupported.go b/vendor/github.com/tetratelabs/wazero/config_unsupported.go new file mode 100644 index 000000000..3e5a53cda --- /dev/null +++ b/vendor/github.com/tetratelabs/wazero/config_unsupported.go @@ -0,0 +1,8 @@ +// This is the opposite constraint of config_supported.go +//go:build !(amd64 || arm64) || !(darwin || linux || freebsd || windows) + +package wazero + +func newRuntimeConfig() RuntimeConfig { + return NewRuntimeConfigInterpreter() +} diff --git a/vendor/github.com/tetratelabs/wazero/experimental/checkpoint.go b/vendor/github.com/tetratelabs/wazero/experimental/checkpoint.go new file mode 100644 index 000000000..443c5a294 --- /dev/null +++ b/vendor/github.com/tetratelabs/wazero/experimental/checkpoint.go @@ -0,0 +1,48 @@ +package experimental + +import ( + "context" + + "github.com/tetratelabs/wazero/internal/expctxkeys" +) + +// Snapshot holds the execution state at the time of a Snapshotter.Snapshot call. +type Snapshot interface { + // Restore sets the Wasm execution state to the capture. Because a host function + // calling this is resetting the pointer to the executation stack, the host function + // will not be able to return values in the normal way. ret is a slice of values the + // host function intends to return from the restored function. + Restore(ret []uint64) +} + +// Snapshotter allows host functions to snapshot the WebAssembly execution environment. +type Snapshotter interface { + // Snapshot captures the current execution state. + Snapshot() Snapshot +} + +// EnableSnapshotterKey is a context key to indicate that snapshotting should be enabled. +// The context.Context passed to a exported function invocation should have this key set +// to a non-nil value, and host functions will be able to retrieve it using SnapshotterKey. +// +// Deprecated: use WithSnapshotter to enable snapshots. +type EnableSnapshotterKey = expctxkeys.EnableSnapshotterKey + +// WithSnapshotter enables snapshots. +// Passing the returned context to a exported function invocation enables snapshots, +// and allows host functions to retrieve the Snapshotter using GetSnapshotter. +func WithSnapshotter(ctx context.Context) context.Context { + return context.WithValue(ctx, expctxkeys.EnableSnapshotterKey{}, struct{}{}) +} + +// SnapshotterKey is a context key to access a Snapshotter from a host function. +// It is only present if EnableSnapshotter was set in the function invocation context. +// +// Deprecated: use GetSnapshotter to get the snapshotter. +type SnapshotterKey = expctxkeys.SnapshotterKey + +// GetSnapshotter gets the Snapshotter from a host function. +// It is only present if WithSnapshotter was called with the function invocation context. +func GetSnapshotter(ctx context.Context) Snapshotter { + return ctx.Value(expctxkeys.SnapshotterKey{}).(Snapshotter) +} diff --git a/vendor/github.com/tetratelabs/wazero/experimental/close.go b/vendor/github.com/tetratelabs/wazero/experimental/close.go new file mode 100644 index 000000000..babecaec4 --- /dev/null +++ b/vendor/github.com/tetratelabs/wazero/experimental/close.go @@ -0,0 +1,63 @@ +package experimental + +import ( + "context" + + "github.com/tetratelabs/wazero/internal/expctxkeys" +) + +// CloseNotifier is a notification hook, invoked when a module is closed. +// +// Note: This is experimental progress towards #1197, and likely to change. Do +// not expose this in shared libraries as it can cause version locks. +type CloseNotifier interface { + // CloseNotify is a notification that occurs *before* an api.Module is + // closed. `exitCode` is zero on success or in the case there was no exit + // code. + // + // Notes: + // - This does not return an error because the module will be closed + // unconditionally. + // - Do not panic from this function as it doing so could cause resource + // leaks. + // - While this is only called once per module, if configured for + // multiple modules, it will be called for each, e.g. on runtime close. + CloseNotify(ctx context.Context, exitCode uint32) +} + +// ^-- Note: This might need to be a part of the listener or become a part of +// host state implementation. For example, if this is used to implement state +// cleanup for host modules, possibly something like below would be better, as +// it could be implemented in a way that allows concurrent module use. +// +// // key is like a context key, stateFactory is invoked per instantiate and +// // is associated with the key (exposed as `Module.State` similar to go +// // context). Using a key is better than the module name because we can +// // de-dupe it for host modules that can be instantiated into different +// // names. Also, you can make the key package private. +// HostModuleBuilder.WithState(key any, stateFactory func() Cleanup)` +// +// Such a design could work to isolate state only needed for wasip1, for +// example the dirent cache. However, if end users use this for different +// things, we may need separate designs. +// +// In summary, the purpose of this iteration is to identify projects that +// would use something like this, and then we can figure out which way it +// should go. + +// CloseNotifyFunc is a convenience for defining inlining a CloseNotifier. +type CloseNotifyFunc func(ctx context.Context, exitCode uint32) + +// CloseNotify implements CloseNotifier.CloseNotify. +func (f CloseNotifyFunc) CloseNotify(ctx context.Context, exitCode uint32) { + f(ctx, exitCode) +} + +// WithCloseNotifier registers the given CloseNotifier into the given +// context.Context. +func WithCloseNotifier(ctx context.Context, notifier CloseNotifier) context.Context { + if notifier != nil { + return context.WithValue(ctx, expctxkeys.CloseNotifierKey{}, notifier) + } + return ctx +} diff --git a/vendor/github.com/tetratelabs/wazero/experimental/experimental.go b/vendor/github.com/tetratelabs/wazero/experimental/experimental.go new file mode 100644 index 000000000..63fd564da --- /dev/null +++ b/vendor/github.com/tetratelabs/wazero/experimental/experimental.go @@ -0,0 +1,41 @@ +// Package experimental includes features we aren't yet sure about. These are enabled with context.Context keys. +// +// Note: All features here may be changed or deleted at any time, so use with caution! +package experimental + +import ( + "github.com/tetratelabs/wazero/api" +) + +// InternalModule is an api.Module that exposes additional +// information. +type InternalModule interface { + api.Module + + // NumGlobal returns the count of all globals in the module. + NumGlobal() int + + // Global provides a read-only view for a given global index. + // + // The methods panics if i is out of bounds. + Global(i int) api.Global +} + +// ProgramCounter is an opaque value representing a specific execution point in +// a module. It is meant to be used with Function.SourceOffsetForPC and +// StackIterator. +type ProgramCounter uint64 + +// InternalFunction exposes some information about a function instance. +type InternalFunction interface { + // Definition provides introspection into the function's names and + // signature. + Definition() api.FunctionDefinition + + // SourceOffsetForPC resolves a program counter into its corresponding + // offset in the Code section of the module this function belongs to. + // The source offset is meant to help map the function calls to their + // location in the original source files. Returns 0 if the offset cannot + // be calculated. + SourceOffsetForPC(pc ProgramCounter) uint64 +} diff --git a/vendor/github.com/tetratelabs/wazero/experimental/features.go b/vendor/github.com/tetratelabs/wazero/experimental/features.go new file mode 100644 index 000000000..b2a5b9069 --- /dev/null +++ b/vendor/github.com/tetratelabs/wazero/experimental/features.go @@ -0,0 +1,15 @@ +package experimental + +import "github.com/tetratelabs/wazero/api" + +// CoreFeaturesThreads enables threads instructions ("threads"). +// +// # Notes +// +// - The instruction list is too long to enumerate in godoc. +// See https://github.com/WebAssembly/threads/blob/main/proposals/threads/Overview.md +// - Atomic operations are guest-only until api.Memory or otherwise expose them to host functions. +// - On systems without mmap available, the memory will pre-allocate to the maximum size. Many +// binaries will use a theroetical maximum like 4GB, so if using such a binary on a system +// without mmap, consider editing the binary to reduce the max size setting of memory. +const CoreFeaturesThreads = api.CoreFeatureSIMD << 1 diff --git a/vendor/github.com/tetratelabs/wazero/experimental/listener.go b/vendor/github.com/tetratelabs/wazero/experimental/listener.go new file mode 100644 index 000000000..b2ba1fe83 --- /dev/null +++ b/vendor/github.com/tetratelabs/wazero/experimental/listener.go @@ -0,0 +1,330 @@ +package experimental + +import ( + "context" + + "github.com/tetratelabs/wazero/api" + "github.com/tetratelabs/wazero/internal/expctxkeys" +) + +// StackIterator allows iterating on each function of the call stack, starting +// from the top. At least one call to Next() is required to start the iteration. +// +// Note: The iterator provides a view of the call stack at the time of +// iteration. As a result, parameter values may be different than the ones their +// function was called with. +type StackIterator interface { + // Next moves the iterator to the next function in the stack. Returns + // false if it reached the bottom of the stack. + Next() bool + // Function describes the function called by the current frame. + Function() InternalFunction + // ProgramCounter returns the program counter associated with the + // function call. + ProgramCounter() ProgramCounter +} + +// FunctionListenerFactoryKey is a context.Context Value key. +// Its associated value should be a FunctionListenerFactory. +// +// Deprecated: use WithFunctionListenerFactory to enable snapshots. +type FunctionListenerFactoryKey = expctxkeys.FunctionListenerFactoryKey + +// WithFunctionListenerFactory registers a FunctionListenerFactory +// with the context. +func WithFunctionListenerFactory(ctx context.Context, factory FunctionListenerFactory) context.Context { + return context.WithValue(ctx, expctxkeys.FunctionListenerFactoryKey{}, factory) +} + +// FunctionListenerFactory returns FunctionListeners to be notified when a +// function is called. +type FunctionListenerFactory interface { + // NewFunctionListener returns a FunctionListener for a defined function. + // If nil is returned, no listener will be notified. + NewFunctionListener(api.FunctionDefinition) FunctionListener + // ^^ A single instance can be returned to avoid instantiating a listener + // per function, especially as they may be thousands of functions. Shared + // listeners use their FunctionDefinition parameter to clarify. +} + +// FunctionListener can be registered for any function via +// FunctionListenerFactory to be notified when the function is called. +type FunctionListener interface { + // Before is invoked before a function is called. + // + // There is always one corresponding call to After or Abort for each call to + // Before. This guarantee allows the listener to maintain an internal stack + // to perform correlations between the entry and exit of functions. + // + // # Params + // + // - ctx: the context of the caller function which must be the same + // instance or parent of the result. + // - mod: the calling module. + // - def: the function definition. + // - params: api.ValueType encoded parameters. + // - stackIterator: iterator on the call stack. At least one entry is + // guaranteed (the called function), whose Args() will be equal to + // params. The iterator will be reused between calls to Before. + // + // Note: api.Memory is meant for inspection, not modification. + // mod can be cast to InternalModule to read non-exported globals. + Before(ctx context.Context, mod api.Module, def api.FunctionDefinition, params []uint64, stackIterator StackIterator) + + // After is invoked after a function is called. + // + // # Params + // + // - ctx: the context of the caller function. + // - mod: the calling module. + // - def: the function definition. + // - results: api.ValueType encoded results. + // + // # Notes + // + // - api.Memory is meant for inspection, not modification. + // - This is not called when a host function panics, or a guest function traps. + // See Abort for more details. + After(ctx context.Context, mod api.Module, def api.FunctionDefinition, results []uint64) + + // Abort is invoked when a function does not return due to a trap or panic. + // + // # Params + // + // - ctx: the context of the caller function. + // - mod: the calling module. + // - def: the function definition. + // - err: the error value representing the reason why the function aborted. + // + // # Notes + // + // - api.Memory is meant for inspection, not modification. + Abort(ctx context.Context, mod api.Module, def api.FunctionDefinition, err error) +} + +// FunctionListenerFunc is a function type implementing the FunctionListener +// interface, making it possible to use regular functions and methods as +// listeners of function invocation. +// +// The FunctionListener interface declares two methods (Before and After), +// but this type invokes its value only when Before is called. It is best +// suites for cases where the host does not need to perform correlation +// between the start and end of the function call. +type FunctionListenerFunc func(context.Context, api.Module, api.FunctionDefinition, []uint64, StackIterator) + +// Before satisfies the FunctionListener interface, calls f. +func (f FunctionListenerFunc) Before(ctx context.Context, mod api.Module, def api.FunctionDefinition, params []uint64, stackIterator StackIterator) { + f(ctx, mod, def, params, stackIterator) +} + +// After is declared to satisfy the FunctionListener interface, but it does +// nothing. +func (f FunctionListenerFunc) After(context.Context, api.Module, api.FunctionDefinition, []uint64) { +} + +// Abort is declared to satisfy the FunctionListener interface, but it does +// nothing. +func (f FunctionListenerFunc) Abort(context.Context, api.Module, api.FunctionDefinition, error) { +} + +// FunctionListenerFactoryFunc is a function type implementing the +// FunctionListenerFactory interface, making it possible to use regular +// functions and methods as factory of function listeners. +type FunctionListenerFactoryFunc func(api.FunctionDefinition) FunctionListener + +// NewFunctionListener satisfies the FunctionListenerFactory interface, calls f. +func (f FunctionListenerFactoryFunc) NewFunctionListener(def api.FunctionDefinition) FunctionListener { + return f(def) +} + +// MultiFunctionListenerFactory constructs a FunctionListenerFactory which +// combines the listeners created by each of the factories passed as arguments. +// +// This function is useful when multiple listeners need to be hooked to a module +// because the propagation mechanism based on installing a listener factory in +// the context.Context used when instantiating modules allows for a single +// listener to be installed. +// +// The stack iterator passed to the Before method is reset so that each listener +// can iterate the call stack independently without impacting the ability of +// other listeners to do so. +func MultiFunctionListenerFactory(factories ...FunctionListenerFactory) FunctionListenerFactory { + multi := make(multiFunctionListenerFactory, len(factories)) + copy(multi, factories) + return multi +} + +type multiFunctionListenerFactory []FunctionListenerFactory + +func (multi multiFunctionListenerFactory) NewFunctionListener(def api.FunctionDefinition) FunctionListener { + var lstns []FunctionListener + for _, factory := range multi { + if lstn := factory.NewFunctionListener(def); lstn != nil { + lstns = append(lstns, lstn) + } + } + switch len(lstns) { + case 0: + return nil + case 1: + return lstns[0] + default: + return &multiFunctionListener{lstns: lstns} + } +} + +type multiFunctionListener struct { + lstns []FunctionListener + stack stackIterator +} + +func (multi *multiFunctionListener) Before(ctx context.Context, mod api.Module, def api.FunctionDefinition, params []uint64, si StackIterator) { + multi.stack.base = si + for _, lstn := range multi.lstns { + multi.stack.index = -1 + lstn.Before(ctx, mod, def, params, &multi.stack) + } +} + +func (multi *multiFunctionListener) After(ctx context.Context, mod api.Module, def api.FunctionDefinition, results []uint64) { + for _, lstn := range multi.lstns { + lstn.After(ctx, mod, def, results) + } +} + +func (multi *multiFunctionListener) Abort(ctx context.Context, mod api.Module, def api.FunctionDefinition, err error) { + for _, lstn := range multi.lstns { + lstn.Abort(ctx, mod, def, err) + } +} + +type stackIterator struct { + base StackIterator + index int + pcs []uint64 + fns []InternalFunction +} + +func (si *stackIterator) Next() bool { + if si.base != nil { + si.pcs = si.pcs[:0] + si.fns = si.fns[:0] + + for si.base.Next() { + si.pcs = append(si.pcs, uint64(si.base.ProgramCounter())) + si.fns = append(si.fns, si.base.Function()) + } + + si.base = nil + } + si.index++ + return si.index < len(si.pcs) +} + +func (si *stackIterator) ProgramCounter() ProgramCounter { + return ProgramCounter(si.pcs[si.index]) +} + +func (si *stackIterator) Function() InternalFunction { + return si.fns[si.index] +} + +// StackFrame represents a frame on the call stack. +type StackFrame struct { + Function api.Function + Params []uint64 + Results []uint64 + PC uint64 + SourceOffset uint64 +} + +type internalFunction struct { + definition api.FunctionDefinition + sourceOffset uint64 +} + +func (f internalFunction) Definition() api.FunctionDefinition { + return f.definition +} + +func (f internalFunction) SourceOffsetForPC(pc ProgramCounter) uint64 { + return f.sourceOffset +} + +// stackFrameIterator is an implementation of the experimental.stackFrameIterator +// interface. +type stackFrameIterator struct { + index int + stack []StackFrame + fndef []api.FunctionDefinition +} + +func (si *stackFrameIterator) Next() bool { + si.index++ + return si.index < len(si.stack) +} + +func (si *stackFrameIterator) Function() InternalFunction { + return internalFunction{ + definition: si.fndef[si.index], + sourceOffset: si.stack[si.index].SourceOffset, + } +} + +func (si *stackFrameIterator) ProgramCounter() ProgramCounter { + return ProgramCounter(si.stack[si.index].PC) +} + +// NewStackIterator constructs a stack iterator from a list of stack frames. +// The top most frame is the last one. +func NewStackIterator(stack ...StackFrame) StackIterator { + si := &stackFrameIterator{ + index: -1, + stack: make([]StackFrame, len(stack)), + fndef: make([]api.FunctionDefinition, len(stack)), + } + for i := range stack { + si.stack[i] = stack[len(stack)-(i+1)] + } + // The size of function definition is only one pointer which should allow + // the compiler to optimize the conversion to api.FunctionDefinition; but + // the presence of internal.WazeroOnlyType, despite being defined as an + // empty struct, forces a heap allocation that we amortize by caching the + // result. + for i, frame := range stack { + si.fndef[i] = frame.Function.Definition() + } + return si +} + +// BenchmarkFunctionListener implements a benchmark for function listeners. +// +// The benchmark calls Before and After methods repeatedly using the provided +// module an stack frames to invoke the methods. +// +// The stack frame is a representation of the call stack that the Before method +// will be invoked with. The top of the stack is stored at index zero. The stack +// must contain at least one frame or the benchmark will fail. +func BenchmarkFunctionListener(n int, module api.Module, stack []StackFrame, listener FunctionListener) { + if len(stack) == 0 { + panic("cannot benchmark function listener with an empty stack") + } + + ctx := context.Background() + def := stack[0].Function.Definition() + params := stack[0].Params + results := stack[0].Results + stackIterator := &stackIterator{base: NewStackIterator(stack...)} + + for i := 0; i < n; i++ { + stackIterator.index = -1 + listener.Before(ctx, module, def, params, stackIterator) + listener.After(ctx, module, def, results) + } +} + +// TODO: the calls to Abort are not yet tested in internal/testing/enginetest, +// but they are validated indirectly in tests which exercise host logging, +// like Test_procExit in imports/wasi_snapshot_preview1. Eventually we should +// add dedicated tests to validate the behavior of the interpreter and compiler +// engines independently. diff --git a/vendor/github.com/tetratelabs/wazero/experimental/memory.go b/vendor/github.com/tetratelabs/wazero/experimental/memory.go new file mode 100644 index 000000000..e379bf053 --- /dev/null +++ b/vendor/github.com/tetratelabs/wazero/experimental/memory.go @@ -0,0 +1,50 @@ +package experimental + +import ( + "context" + + "github.com/tetratelabs/wazero/internal/expctxkeys" +) + +// MemoryAllocator is a memory allocation hook, +// invoked to create a LinearMemory. +type MemoryAllocator interface { + // Allocate should create a new LinearMemory with the given specification: + // cap is the suggested initial capacity for the backing []byte, + // and max the maximum length that will ever be requested. + // + // Notes: + // - To back a shared memory, the address of the backing []byte cannot + // change. This is checked at runtime. Implementations should document + // if the returned LinearMemory meets this requirement. + Allocate(cap, max uint64) LinearMemory +} + +// MemoryAllocatorFunc is a convenience for defining inlining a MemoryAllocator. +type MemoryAllocatorFunc func(cap, max uint64) LinearMemory + +// Allocate implements MemoryAllocator.Allocate. +func (f MemoryAllocatorFunc) Allocate(cap, max uint64) LinearMemory { + return f(cap, max) +} + +// LinearMemory is an expandable []byte that backs a Wasm linear memory. +type LinearMemory interface { + // Reallocates the linear memory to size bytes in length. + // + // Notes: + // - To back a shared memory, Reallocate can't change the address of the + // backing []byte (only its length/capacity may change). + Reallocate(size uint64) []byte + // Free the backing memory buffer. + Free() +} + +// WithMemoryAllocator registers the given MemoryAllocator into the given +// context.Context. +func WithMemoryAllocator(ctx context.Context, allocator MemoryAllocator) context.Context { + if allocator != nil { + return context.WithValue(ctx, expctxkeys.MemoryAllocatorKey{}, allocator) + } + return ctx +} diff --git a/vendor/github.com/tetratelabs/wazero/experimental/sys/dir.go b/vendor/github.com/tetratelabs/wazero/experimental/sys/dir.go new file mode 100644 index 000000000..0b997cb8f --- /dev/null +++ b/vendor/github.com/tetratelabs/wazero/experimental/sys/dir.go @@ -0,0 +1,92 @@ +package sys + +import ( + "fmt" + "io/fs" + + "github.com/tetratelabs/wazero/sys" +) + +// FileType is fs.FileMode masked on fs.ModeType. For example, zero is a +// regular file, fs.ModeDir is a directory and fs.ModeIrregular is unknown. +// +// Note: This is defined by Linux, not POSIX. +type FileType = fs.FileMode + +// Dirent is an entry read from a directory via File.Readdir. +// +// # Notes +// +// - This extends `dirent` defined in POSIX with some fields defined by +// Linux. See https://man7.org/linux/man-pages/man3/readdir.3.html and +// https://pubs.opengroup.org/onlinepubs/9699919799/basedefs/dirent.h.html +// - This has a subset of fields defined in sys.Stat_t. Notably, there is no +// field corresponding to Stat_t.Dev because that value will be constant +// for all files in a directory. To get the Dev value, call File.Stat on +// the directory File.Readdir was called on. +type Dirent struct { + // Ino is the file serial number, or zero if not available. See Ino for + // more details including impact returning a zero value. + Ino sys.Inode + + // Name is the base name of the directory entry. Empty is invalid. + Name string + + // Type is fs.FileMode masked on fs.ModeType. For example, zero is a + // regular file, fs.ModeDir is a directory and fs.ModeIrregular is unknown. + // + // Note: This is defined by Linux, not POSIX. + Type fs.FileMode +} + +func (d *Dirent) String() string { + return fmt.Sprintf("name=%s, type=%v, ino=%d", d.Name, d.Type, d.Ino) +} + +// IsDir returns true if the Type is fs.ModeDir. +func (d *Dirent) IsDir() bool { + return d.Type == fs.ModeDir +} + +// DirFile is embeddable to reduce the amount of functions to implement a file. +type DirFile struct{} + +// IsAppend implements File.IsAppend +func (DirFile) IsAppend() bool { + return false +} + +// SetAppend implements File.SetAppend +func (DirFile) SetAppend(bool) Errno { + return EISDIR +} + +// IsDir implements File.IsDir +func (DirFile) IsDir() (bool, Errno) { + return true, 0 +} + +// Read implements File.Read +func (DirFile) Read([]byte) (int, Errno) { + return 0, EISDIR +} + +// Pread implements File.Pread +func (DirFile) Pread([]byte, int64) (int, Errno) { + return 0, EISDIR +} + +// Write implements File.Write +func (DirFile) Write([]byte) (int, Errno) { + return 0, EISDIR +} + +// Pwrite implements File.Pwrite +func (DirFile) Pwrite([]byte, int64) (int, Errno) { + return 0, EISDIR +} + +// Truncate implements File.Truncate +func (DirFile) Truncate(int64) Errno { + return EISDIR +} diff --git a/vendor/github.com/tetratelabs/wazero/experimental/sys/errno.go b/vendor/github.com/tetratelabs/wazero/experimental/sys/errno.go new file mode 100644 index 000000000..238949496 --- /dev/null +++ b/vendor/github.com/tetratelabs/wazero/experimental/sys/errno.go @@ -0,0 +1,98 @@ +package sys + +import "strconv" + +// Errno is a subset of POSIX errno used by wazero interfaces. Zero is not an +// error. Other values should not be interpreted numerically, rather by constants +// prefixed with 'E'. +// +// See https://pubs.opengroup.org/onlinepubs/9699919799/basedefs/errno.h.html +type Errno uint16 + +// ^-- Note: This will eventually move to the public /sys package. It is +// experimental until we audit the socket related APIs to ensure we have all +// the Errno it returns, and we export fs.FS. This is not in /internal/sys as +// that would introduce a package cycle. + +// This is a subset of errors to reduce implementation burden. `wasip1` defines +// almost all POSIX error numbers, but not all are used in practice. wazero +// will add ones needed in POSIX order, as needed by functions that explicitly +// document returning them. +// +// https://github.com/WebAssembly/WASI/blob/snapshot-01/phases/snapshot/docs.md#-errno-enumu16 +const ( + EACCES Errno = iota + 1 + EAGAIN + EBADF + EEXIST + EFAULT + EINTR + EINVAL + EIO + EISDIR + ELOOP + ENAMETOOLONG + ENOENT + ENOSYS + ENOTDIR + ERANGE + ENOTEMPTY + ENOTSOCK + ENOTSUP + EPERM + EROFS + + // NOTE ENOTCAPABLE is defined in wasip1, but not in POSIX. wasi-libc + // converts it to EBADF, ESPIPE or EINVAL depending on the call site. + // It isn't known if compilers who don't use ENOTCAPABLE would crash on it. +) + +// Error implements error +func (e Errno) Error() string { + switch e { + case 0: // not an error + return "success" + case EACCES: + return "permission denied" + case EAGAIN: + return "resource unavailable, try again" + case EBADF: + return "bad file descriptor" + case EEXIST: + return "file exists" + case EFAULT: + return "bad address" + case EINTR: + return "interrupted function" + case EINVAL: + return "invalid argument" + case EIO: + return "input/output error" + case EISDIR: + return "is a directory" + case ELOOP: + return "too many levels of symbolic links" + case ENAMETOOLONG: + return "filename too long" + case ENOENT: + return "no such file or directory" + case ENOSYS: + return "functionality not supported" + case ENOTDIR: + return "not a directory or a symbolic link to a directory" + case ERANGE: + return "result too large" + case ENOTEMPTY: + return "directory not empty" + case ENOTSOCK: + return "not a socket" + case ENOTSUP: + return "not supported (may be the same value as [EOPNOTSUPP])" + case EPERM: + return "operation not permitted" + case EROFS: + return "read-only file system" + default: + return "Errno(" + strconv.Itoa(int(e)) + ")" + } +} diff --git a/vendor/github.com/tetratelabs/wazero/experimental/sys/error.go b/vendor/github.com/tetratelabs/wazero/experimental/sys/error.go new file mode 100644 index 000000000..a0c76019a --- /dev/null +++ b/vendor/github.com/tetratelabs/wazero/experimental/sys/error.go @@ -0,0 +1,45 @@ +package sys + +import ( + "io" + "io/fs" + "os" +) + +// UnwrapOSError returns an Errno or zero if the input is nil. +func UnwrapOSError(err error) Errno { + if err == nil { + return 0 + } + err = underlyingError(err) + switch err { + case nil, io.EOF: + return 0 // EOF is not a Errno + case fs.ErrInvalid: + return EINVAL + case fs.ErrPermission: + return EPERM + case fs.ErrExist: + return EEXIST + case fs.ErrNotExist: + return ENOENT + case fs.ErrClosed: + return EBADF + } + return errorToErrno(err) +} + +// underlyingError returns the underlying error if a well-known OS error type. +// +// This impl is basically the same as os.underlyingError in os/error.go +func underlyingError(err error) error { + switch err := err.(type) { + case *os.PathError: + return err.Err + case *os.LinkError: + return err.Err + case *os.SyscallError: + return err.Err + } + return err +} diff --git a/vendor/github.com/tetratelabs/wazero/experimental/sys/file.go b/vendor/github.com/tetratelabs/wazero/experimental/sys/file.go new file mode 100644 index 000000000..b6bfbcfeb --- /dev/null +++ b/vendor/github.com/tetratelabs/wazero/experimental/sys/file.go @@ -0,0 +1,316 @@ +package sys + +import "github.com/tetratelabs/wazero/sys" + +// File is a writeable fs.File bridge backed by syscall functions needed for ABI +// including WASI. +// +// Implementations should embed UnimplementedFile for forward compatibility. Any +// unsupported method or parameter should return ENOSYS. +// +// # Errors +// +// All methods that can return an error return a Errno, which is zero +// on success. +// +// Restricting to Errno matches current WebAssembly host functions, +// which are constrained to well-known error codes. For example, WASI maps syscall +// errors to u32 numeric values. +// +// # Notes +// +// - You must call Close to avoid file resource conflicts. For example, +// Windows cannot delete the underlying directory while a handle to it +// remains open. +// - A writable filesystem abstraction is not yet implemented as of Go 1.20. +// See https://github.com/golang/go/issues/45757 +type File interface { + // Dev returns the device ID (Stat_t.Dev) of this file, zero if unknown or + // an error retrieving it. + // + // # Errors + // + // Possible errors are those from Stat, except ENOSYS should not + // be returned. Zero should be returned if there is no implementation. + // + // # Notes + // + // - Implementations should cache this result. + // - This combined with Ino can implement os.SameFile. + Dev() (uint64, Errno) + + // Ino returns the serial number (Stat_t.Ino) of this file, zero if unknown + // or an error retrieving it. + // + // # Errors + // + // Possible errors are those from Stat, except ENOSYS should not + // be returned. Zero should be returned if there is no implementation. + // + // # Notes + // + // - Implementations should cache this result. + // - This combined with Dev can implement os.SameFile. + Ino() (sys.Inode, Errno) + + // IsDir returns true if this file is a directory or an error there was an + // error retrieving this information. + // + // # Errors + // + // Possible errors are those from Stat, except ENOSYS should not + // be returned. false should be returned if there is no implementation. + // + // # Notes + // + // - Implementations should cache this result. + IsDir() (bool, Errno) + + // IsAppend returns true if the file was opened with O_APPEND, or + // SetAppend was successfully enabled on this file. + // + // # Notes + // + // - This might not match the underlying state of the file descriptor if + // the file was not opened via OpenFile. + IsAppend() bool + + // SetAppend toggles the append mode (O_APPEND) of this file. + // + // # Errors + // + // A zero Errno is success. The below are expected otherwise: + // - ENOSYS: the implementation does not support this function. + // - EBADF: the file or directory was closed. + // + // # Notes + // + // - There is no `O_APPEND` for `fcntl` in POSIX, so implementations may + // have to re-open the underlying file to apply this. See + // https://pubs.opengroup.org/onlinepubs/9699919799/functions/open.html + SetAppend(enable bool) Errno + + // Stat is similar to syscall.Fstat. + // + // # Errors + // + // A zero Errno is success. The below are expected otherwise: + // - ENOSYS: the implementation does not support this function. + // - EBADF: the file or directory was closed. + // + // # Notes + // + // - This is like syscall.Fstat and `fstatat` with `AT_FDCWD` in POSIX. + // See https://pubs.opengroup.org/onlinepubs/9699919799/functions/stat.html + // - A fs.FileInfo backed implementation sets atim, mtim and ctim to the + // same value. + // - Windows allows you to stat a closed directory. + Stat() (sys.Stat_t, Errno) + + // Read attempts to read all bytes in the file into `buf`, and returns the + // count read even on error. + // + // # Errors + // + // A zero Errno is success. The below are expected otherwise: + // - ENOSYS: the implementation does not support this function. + // - EBADF: the file or directory was closed or not readable. + // - EISDIR: the file was a directory. + // + // # Notes + // + // - This is like io.Reader and `read` in POSIX, preferring semantics of + // io.Reader. See https://pubs.opengroup.org/onlinepubs/9699919799/functions/read.html + // - Unlike io.Reader, there is no io.EOF returned on end-of-file. To + // read the file completely, the caller must repeat until `n` is zero. + Read(buf []byte) (n int, errno Errno) + + // Pread attempts to read all bytes in the file into `p`, starting at the + // offset `off`, and returns the count read even on error. + // + // # Errors + // + // A zero Errno is success. The below are expected otherwise: + // - ENOSYS: the implementation does not support this function. + // - EBADF: the file or directory was closed or not readable. + // - EINVAL: the offset was negative. + // - EISDIR: the file was a directory. + // + // # Notes + // + // - This is like io.ReaderAt and `pread` in POSIX, preferring semantics + // of io.ReaderAt. See https://pubs.opengroup.org/onlinepubs/9699919799/functions/pread.html + // - Unlike io.ReaderAt, there is no io.EOF returned on end-of-file. To + // read the file completely, the caller must repeat until `n` is zero. + Pread(buf []byte, off int64) (n int, errno Errno) + + // Seek attempts to set the next offset for Read or Write and returns the + // resulting absolute offset or an error. + // + // # Parameters + // + // The `offset` parameters is interpreted in terms of `whence`: + // - io.SeekStart: relative to the start of the file, e.g. offset=0 sets + // the next Read or Write to the beginning of the file. + // - io.SeekCurrent: relative to the current offset, e.g. offset=16 sets + // the next Read or Write 16 bytes past the prior. + // - io.SeekEnd: relative to the end of the file, e.g. offset=-1 sets the + // next Read or Write to the last byte in the file. + // + // # Behavior when a directory + // + // The only supported use case for a directory is seeking to `offset` zero + // (`whence` = io.SeekStart). This should have the same behavior as + // os.File, which resets any internal state used by Readdir. + // + // # Errors + // + // A zero Errno is success. The below are expected otherwise: + // - ENOSYS: the implementation does not support this function. + // - EBADF: the file or directory was closed or not readable. + // - EINVAL: the offset was negative. + // + // # Notes + // + // - This is like io.Seeker and `fseek` in POSIX, preferring semantics + // of io.Seeker. See https://pubs.opengroup.org/onlinepubs/9699919799/functions/fseek.html + Seek(offset int64, whence int) (newOffset int64, errno Errno) + + // Readdir reads the contents of the directory associated with file and + // returns a slice of up to n Dirent values in an arbitrary order. This is + // a stateful function, so subsequent calls return any next values. + // + // If n > 0, Readdir returns at most n entries or an error. + // If n <= 0, Readdir returns all remaining entries or an error. + // + // # Errors + // + // A zero Errno is success. The below are expected otherwise: + // - ENOSYS: the implementation does not support this function. + // - EBADF: the file was closed or not a directory. + // - ENOENT: the directory could not be read (e.g. deleted). + // + // # Notes + // + // - This is like `Readdir` on os.File, but unlike `readdir` in POSIX. + // See https://pubs.opengroup.org/onlinepubs/9699919799/functions/readdir.html + // - Unlike os.File, there is no io.EOF returned on end-of-directory. To + // read the directory completely, the caller must repeat until the + // count read (`len(dirents)`) is less than `n`. + // - See /RATIONALE.md for design notes. + Readdir(n int) (dirents []Dirent, errno Errno) + + // Write attempts to write all bytes in `p` to the file, and returns the + // count written even on error. + // + // # Errors + // + // A zero Errno is success. The below are expected otherwise: + // - ENOSYS: the implementation does not support this function. + // - EBADF: the file was closed, not writeable, or a directory. + // + // # Notes + // + // - This is like io.Writer and `write` in POSIX, preferring semantics of + // io.Writer. See https://pubs.opengroup.org/onlinepubs/9699919799/functions/write.html + Write(buf []byte) (n int, errno Errno) + + // Pwrite attempts to write all bytes in `p` to the file at the given + // offset `off`, and returns the count written even on error. + // + // # Errors + // + // A zero Errno is success. The below are expected otherwise: + // - ENOSYS: the implementation does not support this function. + // - EBADF: the file or directory was closed or not writeable. + // - EINVAL: the offset was negative. + // - EISDIR: the file was a directory. + // + // # Notes + // + // - This is like io.WriterAt and `pwrite` in POSIX, preferring semantics + // of io.WriterAt. See https://pubs.opengroup.org/onlinepubs/9699919799/functions/pwrite.html + Pwrite(buf []byte, off int64) (n int, errno Errno) + + // Truncate truncates a file to a specified length. + // + // # Errors + // + // A zero Errno is success. The below are expected otherwise: + // - ENOSYS: the implementation does not support this function. + // - EBADF: the file or directory was closed. + // - EINVAL: the `size` is negative. + // - EISDIR: the file was a directory. + // + // # Notes + // + // - This is like syscall.Ftruncate and `ftruncate` in POSIX. See + // https://pubs.opengroup.org/onlinepubs/9699919799/functions/ftruncate.html + // - Windows does not error when calling Truncate on a closed file. + Truncate(size int64) Errno + + // Sync synchronizes changes to the file. + // + // # Errors + // + // A zero Errno is success. The below are expected otherwise: + // - EBADF: the file or directory was closed. + // + // # Notes + // + // - This is like syscall.Fsync and `fsync` in POSIX. See + // https://pubs.opengroup.org/onlinepubs/9699919799/functions/fsync.html + // - This returns with no error instead of ENOSYS when + // unimplemented. This prevents fake filesystems from erring. + // - Windows does not error when calling Sync on a closed file. + Sync() Errno + + // Datasync synchronizes the data of a file. + // + // # Errors + // + // A zero Errno is success. The below are expected otherwise: + // - EBADF: the file or directory was closed. + // + // # Notes + // + // - This is like syscall.Fdatasync and `fdatasync` in POSIX. See + // https://pubs.opengroup.org/onlinepubs/9699919799/functions/fdatasync.html + // - This returns with no error instead of ENOSYS when + // unimplemented. This prevents fake filesystems from erring. + // - As this is commonly missing, some implementations dispatch to Sync. + Datasync() Errno + + // Utimens set file access and modification times of this file, at + // nanosecond precision. + // + // # Parameters + // + // The `atim` and `mtim` parameters refer to access and modification time + // stamps as defined in sys.Stat_t. To retain one or the other, substitute + // it with the pseudo-timestamp UTIME_OMIT. + // + // # Errors + // + // A zero Errno is success. The below are expected otherwise: + // - ENOSYS: the implementation does not support this function. + // - EBADF: the file or directory was closed. + // + // # Notes + // + // - This is like syscall.UtimesNano and `futimens` in POSIX. See + // https://pubs.opengroup.org/onlinepubs/9699919799/functions/futimens.html + // - Windows requires files to be open with O_RDWR, which means you + // cannot use this to update timestamps on a directory (EPERM). + Utimens(atim, mtim int64) Errno + + // Close closes the underlying file. + // + // A zero Errno is returned if unimplemented or success. + // + // # Notes + // + // - This is like syscall.Close and `close` in POSIX. See + // https://pubs.opengroup.org/onlinepubs/9699919799/functions/close.html + Close() Errno +} diff --git a/vendor/github.com/tetratelabs/wazero/experimental/sys/fs.go b/vendor/github.com/tetratelabs/wazero/experimental/sys/fs.go new file mode 100644 index 000000000..87810510a --- /dev/null +++ b/vendor/github.com/tetratelabs/wazero/experimental/sys/fs.go @@ -0,0 +1,292 @@ +package sys + +import ( + "io/fs" + + "github.com/tetratelabs/wazero/sys" +) + +// FS is a writeable fs.FS bridge backed by syscall functions needed for ABI +// including WASI. +// +// Implementations should embed UnimplementedFS for forward compatibility. Any +// unsupported method or parameter should return ENO +// +// # Errors +// +// All methods that can return an error return a Errno, which is zero +// on success. +// +// Restricting to Errno matches current WebAssembly host functions, +// which are constrained to well-known error codes. For example, WASI maps syscall +// errors to u32 numeric values. +// +// # Notes +// +// A writable filesystem abstraction is not yet implemented as of Go 1.20. See +// https://github.com/golang/go/issues/45757 +type FS interface { + // OpenFile opens a file. It should be closed via Close on File. + // + // # Errors + // + // A zero Errno is success. The below are expected otherwise: + // - ENOSYS: the implementation does not support this function. + // - EINVAL: `path` or `flag` is invalid. + // - EISDIR: the path was a directory, but flag included O_RDWR or + // O_WRONLY + // - ENOENT: `path` doesn't exist and `flag` doesn't contain O_CREAT. + // + // # Constraints on the returned file + // + // Implementations that can read flags should enforce them regardless of + // the type returned. For example, while os.File implements io.Writer, + // attempts to write to a directory or a file opened with O_RDONLY fail + // with a EBADF. + // + // Some implementations choose whether to enforce read-only opens, namely + // fs.FS. While fs.FS is supported (Adapt), wazero cannot runtime enforce + // open flags. Instead, we encourage good behavior and test our built-in + // implementations. + // + // # Notes + // + // - This is like os.OpenFile, except the path is relative to this file + // system, and Errno is returned instead of os.PathError. + // - Implications of permissions when O_CREAT are described in Chmod notes. + // - This is like `open` in POSIX. See + // https://pubs.opengroup.org/onlinepubs/9699919799/functions/open.html + OpenFile(path string, flag Oflag, perm fs.FileMode) (File, Errno) + + // Lstat gets file status without following symbolic links. + // + // # Errors + // + // A zero Errno is success. The below are expected otherwise: + // - ENOSYS: the implementation does not support this function. + // - ENOENT: `path` doesn't exist. + // + // # Notes + // + // - This is like syscall.Lstat, except the `path` is relative to this + // file system. + // - This is like `lstat` in POSIX. See + // https://pubs.opengroup.org/onlinepubs/9699919799/functions/lstat.html + // - An fs.FileInfo backed implementation sets atim, mtim and ctim to the + // same value. + // - When the path is a symbolic link, the stat returned is for the link, + // not the file it refers to. + Lstat(path string) (sys.Stat_t, Errno) + + // Stat gets file status. + // + // # Errors + // + // A zero Errno is success. The below are expected otherwise: + // - ENOSYS: the implementation does not support this function. + // - ENOENT: `path` doesn't exist. + // + // # Notes + // + // - This is like syscall.Stat, except the `path` is relative to this + // file system. + // - This is like `stat` in POSIX. See + // https://pubs.opengroup.org/onlinepubs/9699919799/functions/stat.html + // - An fs.FileInfo backed implementation sets atim, mtim and ctim to the + // same value. + // - When the path is a symbolic link, the stat returned is for the file + // it refers to. + Stat(path string) (sys.Stat_t, Errno) + + // Mkdir makes a directory. + // + // # Errors + // + // A zero Errno is success. The below are expected otherwise: + // - ENOSYS: the implementation does not support this function. + // - EINVAL: `path` is invalid. + // - EEXIST: `path` exists and is a directory. + // - ENOTDIR: `path` exists and is a file. + // + // # Notes + // + // - This is like syscall.Mkdir, except the `path` is relative to this + // file system. + // - This is like `mkdir` in POSIX. See + // https://pubs.opengroup.org/onlinepubs/9699919799/functions/mkdir.html + // - Implications of permissions are described in Chmod notes. + Mkdir(path string, perm fs.FileMode) Errno + + // Chmod changes the mode of the file. + // + // # Errors + // + // A zero Errno is success. The below are expected otherwise: + // - ENOSYS: the implementation does not support this function. + // - EINVAL: `path` is invalid. + // - ENOENT: `path` does not exist. + // + // # Notes + // + // - This is like syscall.Chmod, except the `path` is relative to this + // file system. + // - This is like `chmod` in POSIX. See + // https://pubs.opengroup.org/onlinepubs/9699919799/functions/chmod.html + // - Windows ignores the execute bit, and any permissions come back as + // group and world. For example, chmod of 0400 reads back as 0444, and + // 0700 0666. Also, permissions on directories aren't supported at all. + Chmod(path string, perm fs.FileMode) Errno + + // Rename renames file or directory. + // + // # Errors + // + // A zero Errno is success. The below are expected otherwise: + // - ENOSYS: the implementation does not support this function. + // - EINVAL: `from` or `to` is invalid. + // - ENOENT: `from` or `to` don't exist. + // - ENOTDIR: `from` is a directory and `to` exists as a file. + // - EISDIR: `from` is a file and `to` exists as a directory. + // - ENOTEMPTY: `both from` and `to` are existing directory, but + // `to` is not empty. + // + // # Notes + // + // - This is like syscall.Rename, except the paths are relative to this + // file system. + // - This is like `rename` in POSIX. See + // https://pubs.opengroup.org/onlinepubs/9699919799/functions/rename.html + // - Windows doesn't let you overwrite an existing directory. + Rename(from, to string) Errno + + // Rmdir removes a directory. + // + // # Errors + // + // A zero Errno is success. The below are expected otherwise: + // - ENOSYS: the implementation does not support this function. + // - EINVAL: `path` is invalid. + // - ENOENT: `path` doesn't exist. + // - ENOTDIR: `path` exists, but isn't a directory. + // - ENOTEMPTY: `path` exists, but isn't empty. + // + // # Notes + // + // - This is like syscall.Rmdir, except the `path` is relative to this + // file system. + // - This is like `rmdir` in POSIX. See + // https://pubs.opengroup.org/onlinepubs/9699919799/functions/rmdir.html + // - As of Go 1.19, Windows maps ENOTDIR to ENOENT. + Rmdir(path string) Errno + + // Unlink removes a directory entry. + // + // # Errors + // + // A zero Errno is success. The below are expected otherwise: + // - ENOSYS: the implementation does not support this function. + // - EINVAL: `path` is invalid. + // - ENOENT: `path` doesn't exist. + // - EISDIR: `path` exists, but is a directory. + // + // # Notes + // + // - This is like syscall.Unlink, except the `path` is relative to this + // file system. + // - This is like `unlink` in POSIX. See + // https://pubs.opengroup.org/onlinepubs/9699919799/functions/unlink.html + // - On Windows, syscall.Unlink doesn't delete symlink to directory unlike other platforms. Implementations might + // want to combine syscall.RemoveDirectory with syscall.Unlink in order to delete such links on Windows. + // See https://learn.microsoft.com/en-us/windows/win32/api/fileapi/nf-fileapi-removedirectorya + Unlink(path string) Errno + + // Link creates a "hard" link from oldPath to newPath, in contrast to a + // soft link (via Symlink). + // + // # Errors + // + // A zero Errno is success. The below are expected otherwise: + // - ENOSYS: the implementation does not support this function. + // - EPERM: `oldPath` is invalid. + // - ENOENT: `oldPath` doesn't exist. + // - EISDIR: `newPath` exists, but is a directory. + // + // # Notes + // + // - This is like syscall.Link, except the `oldPath` is relative to this + // file system. + // - This is like `link` in POSIX. See + // https://pubs.opengroup.org/onlinepubs/9699919799/functions/link.html + Link(oldPath, newPath string) Errno + + // Symlink creates a "soft" link from oldPath to newPath, in contrast to a + // hard link (via Link). + // + // # Errors + // + // A zero Errno is success. The below are expected otherwise: + // - ENOSYS: the implementation does not support this function. + // - EPERM: `oldPath` or `newPath` is invalid. + // - EEXIST: `newPath` exists. + // + // # Notes + // + // - This is like syscall.Symlink, except the `oldPath` is relative to + // this file system. + // - This is like `symlink` in POSIX. See + // https://pubs.opengroup.org/onlinepubs/9699919799/functions/symlink.html + // - Only `newPath` is relative to this file system and `oldPath` is kept + // as-is. That is because the link is only resolved relative to the + // directory when dereferencing it (e.g. ReadLink). + // See https://github.com/bytecodealliance/cap-std/blob/v1.0.4/cap-std/src/fs/dir.rs#L404-L409 + // for how others implement this. + // - Symlinks in Windows requires `SeCreateSymbolicLinkPrivilege`. + // Otherwise, EPERM results. + // See https://learn.microsoft.com/en-us/windows/security/threat-protection/security-policy-settings/create-symbolic-links + Symlink(oldPath, linkName string) Errno + + // Readlink reads the contents of a symbolic link. + // + // # Errors + // + // A zero Errno is success. The below are expected otherwise: + // - ENOSYS: the implementation does not support this function. + // - EINVAL: `path` is invalid. + // + // # Notes + // + // - This is like syscall.Readlink, except the path is relative to this + // filesystem. + // - This is like `readlink` in POSIX. See + // https://pubs.opengroup.org/onlinepubs/9699919799/functions/readlink.html + // - On Windows, the path separator is different from other platforms, + // but to provide consistent results to Wasm, this normalizes to a "/" + // separator. + Readlink(path string) (string, Errno) + + // Utimens set file access and modification times on a path relative to + // this file system, at nanosecond precision. + // + // # Parameters + // + // If the path is a symbolic link, the target of expanding that link is + // updated. + // + // The `atim` and `mtim` parameters refer to access and modification time + // stamps as defined in sys.Stat_t. To retain one or the other, substitute + // it with the pseudo-timestamp UTIME_OMIT. + // + // # Errors + // + // A zero Errno is success. The below are expected otherwise: + // - ENOSYS: the implementation does not support this function. + // - EINVAL: `path` is invalid. + // - EEXIST: `path` exists and is a directory. + // - ENOTDIR: `path` exists and is a file. + // + // # Notes + // + // - This is like syscall.UtimesNano and `utimensat` with `AT_FDCWD` in + // POSIX. See https://pubs.opengroup.org/onlinepubs/9699919799/functions/futimens.html + Utimens(path string, atim, mtim int64) Errno +} diff --git a/vendor/github.com/tetratelabs/wazero/experimental/sys/oflag.go b/vendor/github.com/tetratelabs/wazero/experimental/sys/oflag.go new file mode 100644 index 000000000..39ebd378f --- /dev/null +++ b/vendor/github.com/tetratelabs/wazero/experimental/sys/oflag.go @@ -0,0 +1,70 @@ +package sys + +// Oflag are flags used for FS.OpenFile. Values, including zero, should not be +// interpreted numerically. Instead, use by constants prefixed with 'O_' with +// special casing noted below. +// +// # Notes +// +// - O_RDONLY, O_RDWR and O_WRONLY are mutually exclusive, while the other +// flags can coexist bitwise. +// - This is like `flag` in os.OpenFile and `oflag` in POSIX. See +// https://pubs.opengroup.org/onlinepubs/9699919799/functions/open.html +type Oflag uint32 + +// This is a subset of oflags to reduce implementation burden. `wasip1` splits +// these across `oflags` and `fdflags`. We can't rely on the Go `os` package, +// as it is missing some values. Any flags added will be defined in POSIX +// order, as needed by functions that explicitly document accepting them. +// +// https://github.com/WebAssembly/WASI/blob/snapshot-01/phases/snapshot/docs.md#-oflags-flagsu16 +// https://github.com/WebAssembly/WASI/blob/snapshot-01/phases/snapshot/docs.md#-fdflags-flagsu16 +const ( + // O_RDONLY is like os.O_RDONLY + O_RDONLY Oflag = iota + + // O_RDWR is like os.O_RDWR + O_RDWR + + // O_WRONLY is like os.O_WRONLY + O_WRONLY + + // Define bitflags as they are in POSIX `open`: alphabetically + + // O_APPEND is like os.O_APPEND + O_APPEND Oflag = 1 << iota + + // O_CREAT is link os.O_CREATE + O_CREAT + + // O_DIRECTORY is defined on some platforms as syscall.O_DIRECTORY. + // + // Note: This ensures that the opened file is a directory. Those emulating + // on platforms that don't support the O_DIRECTORY, can double-check the + // result with File.IsDir (or stat) and err if not a directory. + O_DIRECTORY + + // O_DSYNC is defined on some platforms as syscall.O_DSYNC. + O_DSYNC + + // O_EXCL is defined on some platforms as syscall.O_EXCL. + O_EXCL + + // O_NOFOLLOW is defined on some platforms as syscall.O_NOFOLLOW. + // + // Note: This allows programs to ensure that if the opened file is a + // symbolic link, the link itself is opened instead of its target. + O_NOFOLLOW + + // O_NONBLOCK is defined on some platforms as syscall.O_NONBLOCK. + O_NONBLOCK + + // O_RSYNC is defined on some platforms as syscall.O_RSYNC. + O_RSYNC + + // O_SYNC is defined on some platforms as syscall.O_SYNC. + O_SYNC + + // O_TRUNC is defined on some platforms as syscall.O_TRUNC. + O_TRUNC +) diff --git a/vendor/github.com/tetratelabs/wazero/experimental/sys/syscall_errno.go b/vendor/github.com/tetratelabs/wazero/experimental/sys/syscall_errno.go new file mode 100644 index 000000000..ea511ec25 --- /dev/null +++ b/vendor/github.com/tetratelabs/wazero/experimental/sys/syscall_errno.go @@ -0,0 +1,106 @@ +//go:build !plan9 && !aix + +package sys + +import "syscall" + +func syscallToErrno(err error) (Errno, bool) { + errno, ok := err.(syscall.Errno) + if !ok { + return 0, false + } + switch errno { + case 0: + return 0, true + case syscall.EACCES: + return EACCES, true + case syscall.EAGAIN: + return EAGAIN, true + case syscall.EBADF: + return EBADF, true + case syscall.EEXIST: + return EEXIST, true + case syscall.EFAULT: + return EFAULT, true + case syscall.EINTR: + return EINTR, true + case syscall.EINVAL: + return EINVAL, true + case syscall.EIO: + return EIO, true + case syscall.EISDIR: + return EISDIR, true + case syscall.ELOOP: + return ELOOP, true + case syscall.ENAMETOOLONG: + return ENAMETOOLONG, true + case syscall.ENOENT: + return ENOENT, true + case syscall.ENOSYS: + return ENOSYS, true + case syscall.ENOTDIR: + return ENOTDIR, true + case syscall.ERANGE: + return ERANGE, true + case syscall.ENOTEMPTY: + return ENOTEMPTY, true + case syscall.ENOTSOCK: + return ENOTSOCK, true + case syscall.ENOTSUP: + return ENOTSUP, true + case syscall.EPERM: + return EPERM, true + case syscall.EROFS: + return EROFS, true + default: + return EIO, true + } +} + +// Unwrap is a convenience for runtime.GOOS which define syscall.Errno. +func (e Errno) Unwrap() error { + switch e { + case 0: + return nil + case EACCES: + return syscall.EACCES + case EAGAIN: + return syscall.EAGAIN + case EBADF: + return syscall.EBADF + case EEXIST: + return syscall.EEXIST + case EFAULT: + return syscall.EFAULT + case EINTR: + return syscall.EINTR + case EINVAL: + return syscall.EINVAL + case EIO: + return syscall.EIO + case EISDIR: + return syscall.EISDIR + case ELOOP: + return syscall.ELOOP + case ENAMETOOLONG: + return syscall.ENAMETOOLONG + case ENOENT: + return syscall.ENOENT + case ENOSYS: + return syscall.ENOSYS + case ENOTDIR: + return syscall.ENOTDIR + case ENOTEMPTY: + return syscall.ENOTEMPTY + case ENOTSOCK: + return syscall.ENOTSOCK + case ENOTSUP: + return syscall.ENOTSUP + case EPERM: + return syscall.EPERM + case EROFS: + return syscall.EROFS + default: + return syscall.EIO + } +} diff --git a/vendor/github.com/tetratelabs/wazero/experimental/sys/syscall_errno_notwindows.go b/vendor/github.com/tetratelabs/wazero/experimental/sys/syscall_errno_notwindows.go new file mode 100644 index 000000000..8a88ed765 --- /dev/null +++ b/vendor/github.com/tetratelabs/wazero/experimental/sys/syscall_errno_notwindows.go @@ -0,0 +1,13 @@ +//go:build !windows + +package sys + +func errorToErrno(err error) Errno { + if errno, ok := err.(Errno); ok { + return errno + } + if errno, ok := syscallToErrno(err); ok { + return errno + } + return EIO +} diff --git a/vendor/github.com/tetratelabs/wazero/experimental/sys/syscall_errno_unsupported.go b/vendor/github.com/tetratelabs/wazero/experimental/sys/syscall_errno_unsupported.go new file mode 100644 index 000000000..1c6d423d0 --- /dev/null +++ b/vendor/github.com/tetratelabs/wazero/experimental/sys/syscall_errno_unsupported.go @@ -0,0 +1,7 @@ +//go:build plan9 || aix + +package sys + +func syscallToErrno(err error) (Errno, bool) { + return 0, false +} diff --git a/vendor/github.com/tetratelabs/wazero/experimental/sys/syscall_errno_windows.go b/vendor/github.com/tetratelabs/wazero/experimental/sys/syscall_errno_windows.go new file mode 100644 index 000000000..761a1f9dc --- /dev/null +++ b/vendor/github.com/tetratelabs/wazero/experimental/sys/syscall_errno_windows.go @@ -0,0 +1,62 @@ +package sys + +import "syscall" + +// These are errors not defined in the syscall package. They are prefixed with +// underscore to avoid exporting them. +// +// See https://learn.microsoft.com/en-us/windows/win32/debug/system-error-codes--0-499- +const ( + // _ERROR_INVALID_HANDLE is a Windows error returned by syscall.Write + // instead of syscall.EBADF + _ERROR_INVALID_HANDLE = syscall.Errno(6) + + // _ERROR_INVALID_NAME is a Windows error returned by open when a file + // path has a trailing slash + _ERROR_INVALID_NAME = syscall.Errno(0x7B) + + // _ERROR_NEGATIVE_SEEK is a Windows error returned by os.Truncate + // instead of syscall.EINVAL + _ERROR_NEGATIVE_SEEK = syscall.Errno(0x83) + + // _ERROR_DIRECTORY is a Windows error returned by syscall.Rmdir + // instead of syscall.ENOTDIR + _ERROR_DIRECTORY = syscall.Errno(0x10B) + + // _ERROR_INVALID_SOCKET is a Windows error returned by winsock_select + // when a given handle is not a socket. + _ERROR_INVALID_SOCKET = syscall.Errno(0x2736) +) + +func errorToErrno(err error) Errno { + switch err := err.(type) { + case Errno: + return err + case syscall.Errno: + // Note: In windows, _ERROR_PATH_NOT_FOUND(0x3) maps to syscall.ENOTDIR + switch err { + case syscall.ERROR_ALREADY_EXISTS: + return EEXIST + case _ERROR_DIRECTORY: + return ENOTDIR + case syscall.ERROR_DIR_NOT_EMPTY: + return ENOTEMPTY + case syscall.ERROR_FILE_EXISTS: + return EEXIST + case _ERROR_INVALID_HANDLE, _ERROR_INVALID_SOCKET: + return EBADF + case syscall.ERROR_ACCESS_DENIED: + // POSIX read and write functions expect EBADF, not EACCES when not + // open for reading or writing. + return EBADF + case syscall.ERROR_PRIVILEGE_NOT_HELD: + return EPERM + case _ERROR_NEGATIVE_SEEK, _ERROR_INVALID_NAME: + return EINVAL + } + errno, _ := syscallToErrno(err) + return errno + default: + return EIO + } +} diff --git a/vendor/github.com/tetratelabs/wazero/experimental/sys/time.go b/vendor/github.com/tetratelabs/wazero/experimental/sys/time.go new file mode 100644 index 000000000..4f3e01fef --- /dev/null +++ b/vendor/github.com/tetratelabs/wazero/experimental/sys/time.go @@ -0,0 +1,10 @@ +package sys + +import "math" + +// UTIME_OMIT is a special constant for use in updating times via FS.Utimens +// or File.Utimens. When used for atim or mtim, the value is retained. +// +// Note: This may be implemented via a stat when the underlying filesystem +// does not support this value. +const UTIME_OMIT int64 = math.MinInt64 diff --git a/vendor/github.com/tetratelabs/wazero/experimental/sys/unimplemented.go b/vendor/github.com/tetratelabs/wazero/experimental/sys/unimplemented.go new file mode 100644 index 000000000..d853d9e8f --- /dev/null +++ b/vendor/github.com/tetratelabs/wazero/experimental/sys/unimplemented.go @@ -0,0 +1,160 @@ +package sys + +import ( + "io/fs" + + "github.com/tetratelabs/wazero/sys" +) + +// UnimplementedFS is an FS that returns ENOSYS for all functions, +// This should be embedded to have forward compatible implementations. +type UnimplementedFS struct{} + +// OpenFile implements FS.OpenFile +func (UnimplementedFS) OpenFile(path string, flag Oflag, perm fs.FileMode) (File, Errno) { + return nil, ENOSYS +} + +// Lstat implements FS.Lstat +func (UnimplementedFS) Lstat(path string) (sys.Stat_t, Errno) { + return sys.Stat_t{}, ENOSYS +} + +// Stat implements FS.Stat +func (UnimplementedFS) Stat(path string) (sys.Stat_t, Errno) { + return sys.Stat_t{}, ENOSYS +} + +// Readlink implements FS.Readlink +func (UnimplementedFS) Readlink(path string) (string, Errno) { + return "", ENOSYS +} + +// Mkdir implements FS.Mkdir +func (UnimplementedFS) Mkdir(path string, perm fs.FileMode) Errno { + return ENOSYS +} + +// Chmod implements FS.Chmod +func (UnimplementedFS) Chmod(path string, perm fs.FileMode) Errno { + return ENOSYS +} + +// Rename implements FS.Rename +func (UnimplementedFS) Rename(from, to string) Errno { + return ENOSYS +} + +// Rmdir implements FS.Rmdir +func (UnimplementedFS) Rmdir(path string) Errno { + return ENOSYS +} + +// Link implements FS.Link +func (UnimplementedFS) Link(_, _ string) Errno { + return ENOSYS +} + +// Symlink implements FS.Symlink +func (UnimplementedFS) Symlink(_, _ string) Errno { + return ENOSYS +} + +// Unlink implements FS.Unlink +func (UnimplementedFS) Unlink(path string) Errno { + return ENOSYS +} + +// Utimens implements FS.Utimens +func (UnimplementedFS) Utimens(path string, atim, mtim int64) Errno { + return ENOSYS +} + +// UnimplementedFile is a File that returns ENOSYS for all functions, +// except where no-op are otherwise documented. +// +// This should be embedded to have forward compatible implementations. +type UnimplementedFile struct{} + +// Dev implements File.Dev +func (UnimplementedFile) Dev() (uint64, Errno) { + return 0, 0 +} + +// Ino implements File.Ino +func (UnimplementedFile) Ino() (sys.Inode, Errno) { + return 0, 0 +} + +// IsDir implements File.IsDir +func (UnimplementedFile) IsDir() (bool, Errno) { + return false, 0 +} + +// IsAppend implements File.IsAppend +func (UnimplementedFile) IsAppend() bool { + return false +} + +// SetAppend implements File.SetAppend +func (UnimplementedFile) SetAppend(bool) Errno { + return ENOSYS +} + +// Stat implements File.Stat +func (UnimplementedFile) Stat() (sys.Stat_t, Errno) { + return sys.Stat_t{}, ENOSYS +} + +// Read implements File.Read +func (UnimplementedFile) Read([]byte) (int, Errno) { + return 0, ENOSYS +} + +// Pread implements File.Pread +func (UnimplementedFile) Pread([]byte, int64) (int, Errno) { + return 0, ENOSYS +} + +// Seek implements File.Seek +func (UnimplementedFile) Seek(int64, int) (int64, Errno) { + return 0, ENOSYS +} + +// Readdir implements File.Readdir +func (UnimplementedFile) Readdir(int) (dirents []Dirent, errno Errno) { + return nil, ENOSYS +} + +// Write implements File.Write +func (UnimplementedFile) Write([]byte) (int, Errno) { + return 0, ENOSYS +} + +// Pwrite implements File.Pwrite +func (UnimplementedFile) Pwrite([]byte, int64) (int, Errno) { + return 0, ENOSYS +} + +// Truncate implements File.Truncate +func (UnimplementedFile) Truncate(int64) Errno { + return ENOSYS +} + +// Sync implements File.Sync +func (UnimplementedFile) Sync() Errno { + return 0 // not ENOSYS +} + +// Datasync implements File.Datasync +func (UnimplementedFile) Datasync() Errno { + return 0 // not ENOSYS +} + +// Utimens implements File.Utimens +func (UnimplementedFile) Utimens(int64, int64) Errno { + return ENOSYS +} + +// Close implements File.Close +func (UnimplementedFile) Close() (errno Errno) { return } diff --git a/vendor/github.com/tetratelabs/wazero/fsconfig.go b/vendor/github.com/tetratelabs/wazero/fsconfig.go new file mode 100644 index 000000000..c21b6e80b --- /dev/null +++ b/vendor/github.com/tetratelabs/wazero/fsconfig.go @@ -0,0 +1,213 @@ +package wazero + +import ( + "io/fs" + + experimentalsys "github.com/tetratelabs/wazero/experimental/sys" + "github.com/tetratelabs/wazero/internal/sys" + "github.com/tetratelabs/wazero/internal/sysfs" +) + +// FSConfig configures filesystem paths the embedding host allows the wasm +// guest to access. Unconfigured paths are not allowed, so functions like +// `path_open` result in unsupported errors (e.g. syscall.ENOSYS). +// +// # Guest Path +// +// `guestPath` is the name of the path the guest should use a filesystem for, or +// empty for any files. +// +// All `guestPath` paths are normalized, specifically removing any leading or +// trailing slashes. This means "/", "./" or "." all coerce to empty "". +// +// Multiple `guestPath` values can be configured, but the last longest match +// wins. For example, if "tmp", then "" were added, a request to open +// "tmp/foo.txt" use the filesystem associated with "tmp" even though a wider +// path, "" (all files), was added later. +// +// A `guestPath` of "." coerces to the empty string "" because the current +// directory is handled by the guest. In other words, the guest resolves ites +// current directory prior to requesting files. +// +// More notes on `guestPath` +// - Working directories are typically tracked in wasm, though possible some +// relative paths are requested. For example, TinyGo may attempt to resolve +// a path "../.." in unit tests. +// - Zig uses the first path name it sees as the initial working directory of +// the process. +// +// # Scope +// +// Configuration here is module instance scoped. This means you can use the +// same configuration for multiple calls to Runtime.InstantiateModule. Each +// module will have a different file descriptor table. Any errors accessing +// resources allowed here are deferred to instantiation time of each module. +// +// Any host resources present at the time of configuration, but deleted before +// Runtime.InstantiateModule will trap/panic when the guest wasm initializes or +// calls functions like `fd_read`. +// +// # Windows +// +// While wazero supports Windows as a platform, all known compilers use POSIX +// conventions at runtime. For example, even when running on Windows, paths +// used by wasm are separated by forward slash (/), not backslash (\). +// +// # Notes +// +// - This is an interface for decoupling, not third-party implementations. +// All implementations are in wazero. +// - FSConfig is immutable. Each WithXXX function returns a new instance +// including the corresponding change. +// - RATIONALE.md includes design background and relationship to WebAssembly +// System Interfaces (WASI). +type FSConfig interface { + // WithDirMount assigns a directory at `dir` to any paths beginning at + // `guestPath`. + // + // For example, `dirPath` as / (or c:\ in Windows), makes the entire host + // volume writeable to the path on the guest. The `guestPath` is always a + // POSIX style path, slash (/) delimited, even if run on Windows. + // + // If the same `guestPath` was assigned before, this overrides its value, + // retaining the original precedence. See the documentation of FSConfig for + // more details on `guestPath`. + // + // # Isolation + // + // The guest will have full access to this directory including escaping it + // via relative path lookups like "../../". Full access includes operations + // such as creating or deleting files, limited to any host level access + // controls. + // + // # os.DirFS + // + // This configuration optimizes for WASI compatibility which is sometimes + // at odds with the behavior of os.DirFS. Hence, this will not behave + // exactly the same as os.DirFS. See /RATIONALE.md for more. + WithDirMount(dir, guestPath string) FSConfig + + // WithReadOnlyDirMount assigns a directory at `dir` to any paths + // beginning at `guestPath`. + // + // This is the same as WithDirMount except only read operations are + // permitted. However, escaping the directory via relative path lookups + // like "../../" is still allowed. + WithReadOnlyDirMount(dir, guestPath string) FSConfig + + // WithFSMount assigns a fs.FS file system for any paths beginning at + // `guestPath`. + // + // If the same `guestPath` was assigned before, this overrides its value, + // retaining the original precedence. See the documentation of FSConfig for + // more details on `guestPath`. + // + // # Isolation + // + // fs.FS does not restrict the ability to overwrite returned files via + // io.Writer. Moreover, os.DirFS documentation includes important notes + // about isolation, which also applies to fs.Sub. As of Go 1.19, the + // built-in file-systems are not jailed (chroot). See + // https://github.com/golang/go/issues/42322 + // + // # os.DirFS + // + // Due to limited control and functionality available in os.DirFS, we + // advise using WithDirMount instead. There will be behavior differences + // between os.DirFS and WithDirMount, as the latter biases towards what's + // expected from WASI implementations. + // + // # Custom fs.FileInfo + // + // The underlying implementation supports data not usually in fs.FileInfo + // when `info.Sys` returns *sys.Stat_t. For example, a custom fs.FS can use + // this approach to generate or mask sys.Inode data. Such a filesystem + // needs to decorate any functions that can return fs.FileInfo: + // + // - `Stat` as defined on `fs.File` (always) + // - `Readdir` as defined on `os.File` (if defined) + // + // See sys.NewStat_t for examples. + WithFSMount(fs fs.FS, guestPath string) FSConfig +} + +type fsConfig struct { + // fs are the currently configured filesystems. + fs []experimentalsys.FS + // guestPaths are the user-supplied names of the filesystems, retained for + // error messages and fmt.Stringer. + guestPaths []string + // guestPathToFS are the normalized paths to the currently configured + // filesystems, used for de-duplicating. + guestPathToFS map[string]int +} + +// NewFSConfig returns a FSConfig that can be used for configuring module instantiation. +func NewFSConfig() FSConfig { + return &fsConfig{guestPathToFS: map[string]int{}} +} + +// clone makes a deep copy of this module config. +func (c *fsConfig) clone() *fsConfig { + ret := *c // copy except slice and maps which share a ref + ret.fs = make([]experimentalsys.FS, 0, len(c.fs)) + ret.fs = append(ret.fs, c.fs...) + ret.guestPaths = make([]string, 0, len(c.guestPaths)) + ret.guestPaths = append(ret.guestPaths, c.guestPaths...) + ret.guestPathToFS = make(map[string]int, len(c.guestPathToFS)) + for key, value := range c.guestPathToFS { + ret.guestPathToFS[key] = value + } + return &ret +} + +// WithDirMount implements FSConfig.WithDirMount +func (c *fsConfig) WithDirMount(dir, guestPath string) FSConfig { + return c.WithSysFSMount(sysfs.DirFS(dir), guestPath) +} + +// WithReadOnlyDirMount implements FSConfig.WithReadOnlyDirMount +func (c *fsConfig) WithReadOnlyDirMount(dir, guestPath string) FSConfig { + return c.WithSysFSMount(&sysfs.ReadFS{FS: sysfs.DirFS(dir)}, guestPath) +} + +// WithFSMount implements FSConfig.WithFSMount +func (c *fsConfig) WithFSMount(fs fs.FS, guestPath string) FSConfig { + var adapted experimentalsys.FS + if fs != nil { + adapted = &sysfs.AdaptFS{FS: fs} + } + return c.WithSysFSMount(adapted, guestPath) +} + +// WithSysFSMount implements sysfs.FSConfig +func (c *fsConfig) WithSysFSMount(fs experimentalsys.FS, guestPath string) FSConfig { + if _, ok := fs.(experimentalsys.UnimplementedFS); ok { + return c // don't add fake paths. + } + cleaned := sys.StripPrefixesAndTrailingSlash(guestPath) + ret := c.clone() + if i, ok := ret.guestPathToFS[cleaned]; ok { + ret.fs[i] = fs + ret.guestPaths[i] = guestPath + } else if fs != nil { + ret.guestPathToFS[cleaned] = len(ret.fs) + ret.fs = append(ret.fs, fs) + ret.guestPaths = append(ret.guestPaths, guestPath) + } + return ret +} + +// preopens returns the possible nil index-correlated preopened filesystems +// with guest paths. +func (c *fsConfig) preopens() ([]experimentalsys.FS, []string) { + preopenCount := len(c.fs) + if preopenCount == 0 { + return nil, nil + } + fs := make([]experimentalsys.FS, len(c.fs)) + copy(fs, c.fs) + guestPaths := make([]string, len(c.guestPaths)) + copy(guestPaths, c.guestPaths) + return fs, guestPaths +} diff --git a/vendor/github.com/tetratelabs/wazero/internal/descriptor/table.go b/vendor/github.com/tetratelabs/wazero/internal/descriptor/table.go new file mode 100644 index 000000000..542958bc7 --- /dev/null +++ b/vendor/github.com/tetratelabs/wazero/internal/descriptor/table.go @@ -0,0 +1,164 @@ +package descriptor + +import "math/bits" + +// Table is a data structure mapping 32 bit descriptor to items. +// +// # Negative keys are invalid. +// +// Negative keys (e.g. -1) are invalid inputs and will return a corresponding +// not-found value. This matches POSIX behavior of file descriptors. +// See https://pubs.opengroup.org/onlinepubs/9699919799/functions/dirfd.html#tag_16_90 +// +// # Data structure design +// +// The data structure optimizes for memory density and lookup performance, +// trading off compute at insertion time. This is a useful compromise for the +// use cases we employ it with: items are usually accessed a lot more often +// than they are inserted, each operation requires a table lookup, so we are +// better off spending extra compute to insert items in the table in order to +// get cheaper lookups. Memory efficiency is also crucial to support scaling +// with programs that maintain thousands of items: having a high or non-linear +// memory-to-item ratio could otherwise be used as an attack vector by +// malicious applications attempting to damage performance of the host. +type Table[Key ~int32, Item any] struct { + masks []uint64 + items []Item +} + +// Len returns the number of items stored in the table. +func (t *Table[Key, Item]) Len() (n int) { + // We could make this a O(1) operation if we cached the number of items in + // the table. More state usually means more problems, so until we have a + // clear need for this, the simple implementation may be a better trade off. + for _, mask := range t.masks { + n += bits.OnesCount64(mask) + } + return n +} + +// grow ensures that t has enough room for n items, potentially reallocating the +// internal buffers if their capacity was too small to hold this many items. +func (t *Table[Key, Item]) grow(n int) { + // Round up to a multiple of 64 since this is the smallest increment due to + // using 64 bits masks. + n = (n*64 + 63) / 64 + + if n > len(t.masks) { + masks := make([]uint64, n) + copy(masks, t.masks) + + items := make([]Item, n*64) + copy(items, t.items) + + t.masks = masks + t.items = items + } +} + +// Insert inserts the given item to the table, returning the key that it is +// mapped to or false if the table was full. +// +// The method does not perform deduplication, it is possible for the same item +// to be inserted multiple times, each insertion will return a different key. +func (t *Table[Key, Item]) Insert(item Item) (key Key, ok bool) { + offset := 0 +insert: + // Note: this loop could be made a lot more efficient using vectorized + // operations: 256 bits vector registers would yield a theoretical 4x + // speed up (e.g. using AVX2). + for index, mask := range t.masks[offset:] { + if ^mask != 0 { // not full? + shift := bits.TrailingZeros64(^mask) + index += offset + key = Key(index)*64 + Key(shift) + t.items[key] = item + t.masks[index] = mask | uint64(1<= 0 + } + } + + offset = len(t.masks) + n := 2 * len(t.masks) + if n == 0 { + n = 1 + } + + t.grow(n) + goto insert +} + +// Lookup returns the item associated with the given key (may be nil). +func (t *Table[Key, Item]) Lookup(key Key) (item Item, found bool) { + if key < 0 { // invalid key + return + } + if i := int(key); i >= 0 && i < len(t.items) { + index := uint(key) / 64 + shift := uint(key) % 64 + if (t.masks[index] & (1 << shift)) != 0 { + item, found = t.items[i], true + } + } + return +} + +// InsertAt inserts the given `item` at the item descriptor `key`. This returns +// false if the insert was impossible due to negative key. +func (t *Table[Key, Item]) InsertAt(item Item, key Key) bool { + if key < 0 { + return false + } + if diff := int(key) - t.Len(); diff > 0 { + t.grow(diff) + } + index := uint(key) / 64 + shift := uint(key) % 64 + t.masks[index] |= 1 << shift + t.items[key] = item + return true +} + +// Delete deletes the item stored at the given key from the table. +func (t *Table[Key, Item]) Delete(key Key) { + if key < 0 { // invalid key + return + } + if index, shift := key/64, key%64; int(index) < len(t.masks) { + mask := t.masks[index] + if (mask & (1 << shift)) != 0 { + var zero Item + t.items[key] = zero + t.masks[index] = mask & ^uint64(1< 0 { + // We reserve the stack slots for result values below the return call frame slots. + if diff := c.sig.ResultNumInUint64 - c.sig.ParamNumInUint64; diff > 0 { + current += diff + } + } + + // Non-func param locals Start after the return call frame. + current += c.callFrameStackSizeInUint64 + + for _, lt := range c.localTypes { + c.localIndexToStackHeightInUint64 = append(c.localIndexToStackHeightInUint64, current) + if lt == wasm.ValueTypeV128 { + current++ + } + current++ + } + + // Push function arguments. + for _, t := range c.sig.Params { + c.stackPush(wasmValueTypeTounsignedType(t)) + } + + if c.callFrameStackSizeInUint64 > 0 { + // Reserve the stack slots for results. + for i := 0; i < c.sig.ResultNumInUint64-c.sig.ParamNumInUint64; i++ { + c.stackPush(unsignedTypeI64) + } + + // Reserve the stack slots for call frame. + for i := 0; i < c.callFrameStackSizeInUint64; i++ { + c.stackPush(unsignedTypeI64) + } + } +} + +// compiler is in charge of lowering raw Wasm function body to get compilationResult. +// This is created per *wasm.Module and reused for all functions in it to reduce memory allocations. +type compiler struct { + module *wasm.Module + enabledFeatures api.CoreFeatures + callFrameStackSizeInUint64 int + stack []unsignedType + currentFrameID uint32 + controlFrames controlFrames + unreachableState struct { + on bool + depth int + } + pc, currentOpPC uint64 + result compilationResult + + // body holds the code for the function's body where Wasm instructions are stored. + body []byte + // sig is the function type of the target function. + sig *wasm.FunctionType + // localTypes holds the target function locals' value types except function params. + localTypes []wasm.ValueType + // localIndexToStackHeightInUint64 maps the local index (starting with function params) to the stack height + // where the local is places. This is the necessary mapping for functions who contain vector type locals. + localIndexToStackHeightInUint64 []int + + // types hold all the function types in the module where the targe function exists. + types []wasm.FunctionType + // funcs holds the type indexes for all declared functions in the module where the target function exists. + funcs []uint32 + // globals holds the global types for all declared globals in the module where the target function exists. + globals []wasm.GlobalType + + // needSourceOffset is true if this module requires DWARF based stack trace. + needSourceOffset bool + // bodyOffsetInCodeSection is the offset of the body of this function in the original Wasm binary's code section. + bodyOffsetInCodeSection uint64 + + ensureTermination bool + // Pre-allocated bytes.Reader to be used in various places. + br *bytes.Reader + funcTypeToSigs funcTypeToIRSignatures + + next int +} + +//lint:ignore U1000 for debugging only. +func (c *compiler) stackDump() string { + strs := make([]string, 0, len(c.stack)) + for _, s := range c.stack { + strs = append(strs, s.String()) + } + return "[" + strings.Join(strs, ", ") + "]" +} + +func (c *compiler) markUnreachable() { + c.unreachableState.on = true +} + +func (c *compiler) resetUnreachable() { + c.unreachableState.on = false +} + +// memoryType is the type of memory in a compiled module. +type memoryType byte + +const ( + // memoryTypeNone indicates there is no memory. + memoryTypeNone memoryType = iota + // memoryTypeStandard indicates there is a non-shared memory. + memoryTypeStandard + // memoryTypeShared indicates there is a shared memory. + memoryTypeShared +) + +type compilationResult struct { + // Operations holds interpreterir operations compiled from Wasm instructions in a Wasm function. + Operations []unionOperation + + // IROperationSourceOffsetsInWasmBinary is index-correlated with Operation and maps each operation to the corresponding source instruction's + // offset in the original WebAssembly binary. + // Non nil only when the given Wasm module has the DWARF section. + IROperationSourceOffsetsInWasmBinary []uint64 + + // LabelCallers maps label to the number of callers to that label. + // Here "callers" means that the call-sites which jumps to the label with br, br_if or br_table + // instructions. + // + // Note: zero possible and allowed in wasm. e.g. + // + // (block + // (br 0) + // (block i32.const 1111) + // ) + // + // This example the label corresponding to `(block i32.const 1111)` is never be reached at runtime because `br 0` exits the function before we reach there + LabelCallers map[label]uint32 + // UsesMemory is true if this function might use memory. + UsesMemory bool + + // The following fields are per-module values, not per-function. + + // Globals holds all the declarations of globals in the module from which this function is compiled. + Globals []wasm.GlobalType + // Functions holds all the declarations of function in the module from which this function is compiled, including itself. + Functions []wasm.Index + // Types holds all the types in the module from which this function is compiled. + Types []wasm.FunctionType + // Memory indicates the type of memory of the module. + Memory memoryType + // HasTable is true if the module from which this function is compiled has table declaration. + HasTable bool + // HasDataInstances is true if the module has data instances which might be used by memory.init or data.drop instructions. + HasDataInstances bool + // HasDataInstances is true if the module has element instances which might be used by table.init or elem.drop instructions. + HasElementInstances bool +} + +// newCompiler returns the new *compiler for the given parameters. +// Use compiler.Next function to get compilation result per function. +func newCompiler(enabledFeatures api.CoreFeatures, callFrameStackSizeInUint64 int, module *wasm.Module, ensureTermination bool) (*compiler, error) { + functions, globals, mem, tables, err := module.AllDeclarations() + if err != nil { + return nil, err + } + + hasTable, hasDataInstances, hasElementInstances := len(tables) > 0, + len(module.DataSection) > 0, len(module.ElementSection) > 0 + + var mt memoryType + switch { + case mem == nil: + mt = memoryTypeNone + case mem.IsShared: + mt = memoryTypeShared + default: + mt = memoryTypeStandard + } + + types := module.TypeSection + + c := &compiler{ + module: module, + enabledFeatures: enabledFeatures, + controlFrames: controlFrames{}, + callFrameStackSizeInUint64: callFrameStackSizeInUint64, + result: compilationResult{ + Globals: globals, + Functions: functions, + Types: types, + Memory: mt, + HasTable: hasTable, + HasDataInstances: hasDataInstances, + HasElementInstances: hasElementInstances, + LabelCallers: map[label]uint32{}, + }, + globals: globals, + funcs: functions, + types: types, + ensureTermination: ensureTermination, + br: bytes.NewReader(nil), + funcTypeToSigs: funcTypeToIRSignatures{ + indirectCalls: make([]*signature, len(types)), + directCalls: make([]*signature, len(types)), + wasmTypes: types, + }, + needSourceOffset: module.DWARFLines != nil, + } + return c, nil +} + +// Next returns the next compilationResult for this compiler. +func (c *compiler) Next() (*compilationResult, error) { + funcIndex := c.next + code := &c.module.CodeSection[funcIndex] + sig := &c.types[c.module.FunctionSection[funcIndex]] + + // Reset the previous result. + c.result.Operations = c.result.Operations[:0] + c.result.IROperationSourceOffsetsInWasmBinary = c.result.IROperationSourceOffsetsInWasmBinary[:0] + c.result.UsesMemory = false + // Clears the existing entries in LabelCallers. + for frameID := uint32(0); frameID <= c.currentFrameID; frameID++ { + for k := labelKind(0); k < labelKindNum; k++ { + delete(c.result.LabelCallers, newLabel(k, frameID)) + } + } + // Reset the previous states. + c.pc = 0 + c.currentOpPC = 0 + c.currentFrameID = 0 + c.unreachableState.on, c.unreachableState.depth = false, 0 + + if err := c.compile(sig, code.Body, code.LocalTypes, code.BodyOffsetInCodeSection); err != nil { + return nil, err + } + c.next++ + return &c.result, nil +} + +// Compile lowers given function instance into interpreterir operations +// so that the resulting operations can be consumed by the interpreter +// or the compiler compilation engine. +func (c *compiler) compile(sig *wasm.FunctionType, body []byte, localTypes []wasm.ValueType, bodyOffsetInCodeSection uint64) error { + // Set function specific fields. + c.body = body + c.localTypes = localTypes + c.sig = sig + c.bodyOffsetInCodeSection = bodyOffsetInCodeSection + + // Reuses the underlying slices. + c.stack = c.stack[:0] + c.controlFrames.frames = c.controlFrames.frames[:0] + + c.initializeStack() + + // Emit const expressions for locals. + // Note that here we don't take function arguments + // into account, meaning that callers must push + // arguments before entering into the function body. + for _, t := range c.localTypes { + c.emitDefaultValue(t) + } + + // Insert the function control frame. + c.controlFrames.push(controlFrame{ + frameID: c.nextFrameID(), + blockType: c.sig, + kind: controlFrameKindFunction, + }) + + // Now, enter the function body. + for !c.controlFrames.empty() && c.pc < uint64(len(c.body)) { + if err := c.handleInstruction(); err != nil { + return fmt.Errorf("handling instruction: %w", err) + } + } + return nil +} + +// Translate the current Wasm instruction to interpreterir's operations, +// and emit the results into c.results. +func (c *compiler) handleInstruction() error { + op := c.body[c.pc] + c.currentOpPC = c.pc + if false { + var instName string + if op == wasm.OpcodeVecPrefix { + instName = wasm.VectorInstructionName(c.body[c.pc+1]) + } else if op == wasm.OpcodeAtomicPrefix { + instName = wasm.AtomicInstructionName(c.body[c.pc+1]) + } else if op == wasm.OpcodeMiscPrefix { + instName = wasm.MiscInstructionName(c.body[c.pc+1]) + } else { + instName = wasm.InstructionName(op) + } + fmt.Printf("handling %s, unreachable_state(on=%v,depth=%d), stack=%v\n", + instName, c.unreachableState.on, c.unreachableState.depth, c.stack, + ) + } + + var peekValueType unsignedType + if len(c.stack) > 0 { + peekValueType = c.stackPeek() + } + + // Modify the stack according the current instruction. + // Note that some instructions will read "index" in + // applyToStack and advance c.pc inside the function. + index, err := c.applyToStack(op) + if err != nil { + return fmt.Errorf("apply stack failed for %s: %w", wasm.InstructionName(op), err) + } + // Now we handle each instruction, and + // emit the corresponding interpreterir operations to the results. +operatorSwitch: + switch op { + case wasm.OpcodeUnreachable: + c.emit(newOperationUnreachable()) + c.markUnreachable() + case wasm.OpcodeNop: + // Nop is noop! + case wasm.OpcodeBlock: + c.br.Reset(c.body[c.pc+1:]) + bt, num, err := wasm.DecodeBlockType(c.types, c.br, c.enabledFeatures) + if err != nil { + return fmt.Errorf("reading block type for block instruction: %w", err) + } + c.pc += num + + if c.unreachableState.on { + // If it is currently in unreachable, + // just remove the entire block. + c.unreachableState.depth++ + break operatorSwitch + } + + // Create a new frame -- entering this block. + frame := controlFrame{ + frameID: c.nextFrameID(), + originalStackLenWithoutParam: len(c.stack) - len(bt.Params), + kind: controlFrameKindBlockWithoutContinuationLabel, + blockType: bt, + } + c.controlFrames.push(frame) + + case wasm.OpcodeLoop: + c.br.Reset(c.body[c.pc+1:]) + bt, num, err := wasm.DecodeBlockType(c.types, c.br, c.enabledFeatures) + if err != nil { + return fmt.Errorf("reading block type for loop instruction: %w", err) + } + c.pc += num + + if c.unreachableState.on { + // If it is currently in unreachable, + // just remove the entire block. + c.unreachableState.depth++ + break operatorSwitch + } + + // Create a new frame -- entering loop. + frame := controlFrame{ + frameID: c.nextFrameID(), + originalStackLenWithoutParam: len(c.stack) - len(bt.Params), + kind: controlFrameKindLoop, + blockType: bt, + } + c.controlFrames.push(frame) + + // Prep labels for inside and the continuation of this loop. + loopLabel := newLabel(labelKindHeader, frame.frameID) + c.result.LabelCallers[loopLabel]++ + + // Emit the branch operation to enter inside the loop. + c.emit(newOperationBr(loopLabel)) + c.emit(newOperationLabel(loopLabel)) + + // Insert the exit code check on the loop header, which is the only necessary point in the function body + // to prevent infinite loop. + // + // Note that this is a little aggressive: this checks the exit code regardless the loop header is actually + // the loop. In other words, this checks even when no br/br_if/br_table instructions jumping to this loop + // exist. However, in reality, that shouldn't be an issue since such "noop" loop header will highly likely be + // optimized out by almost all guest language compilers which have the control flow optimization passes. + if c.ensureTermination { + c.emit(newOperationBuiltinFunctionCheckExitCode()) + } + case wasm.OpcodeIf: + c.br.Reset(c.body[c.pc+1:]) + bt, num, err := wasm.DecodeBlockType(c.types, c.br, c.enabledFeatures) + if err != nil { + return fmt.Errorf("reading block type for if instruction: %w", err) + } + c.pc += num + + if c.unreachableState.on { + // If it is currently in unreachable, + // just remove the entire block. + c.unreachableState.depth++ + break operatorSwitch + } + + // Create a new frame -- entering if. + frame := controlFrame{ + frameID: c.nextFrameID(), + originalStackLenWithoutParam: len(c.stack) - len(bt.Params), + // Note this will be set to controlFrameKindIfWithElse + // when else opcode found later. + kind: controlFrameKindIfWithoutElse, + blockType: bt, + } + c.controlFrames.push(frame) + + // Prep labels for if and else of this if. + thenLabel := newLabel(labelKindHeader, frame.frameID) + elseLabel := newLabel(labelKindElse, frame.frameID) + c.result.LabelCallers[thenLabel]++ + c.result.LabelCallers[elseLabel]++ + + // Emit the branch operation to enter the then block. + c.emit(newOperationBrIf(thenLabel, elseLabel, nopinclusiveRange)) + c.emit(newOperationLabel(thenLabel)) + case wasm.OpcodeElse: + frame := c.controlFrames.top() + if c.unreachableState.on && c.unreachableState.depth > 0 { + // If it is currently in unreachable, and the nested if, + // just remove the entire else block. + break operatorSwitch + } else if c.unreachableState.on { + // If it is currently in unreachable, and the non-nested if, + // reset the stack so we can correctly handle the else block. + top := c.controlFrames.top() + c.stack = c.stack[:top.originalStackLenWithoutParam] + top.kind = controlFrameKindIfWithElse + + // Re-push the parameters to the if block so that else block can use them. + for _, t := range frame.blockType.Params { + c.stackPush(wasmValueTypeTounsignedType(t)) + } + + // We are no longer unreachable in else frame, + // so emit the correct label, and reset the unreachable state. + elseLabel := newLabel(labelKindElse, frame.frameID) + c.resetUnreachable() + c.emit( + newOperationLabel(elseLabel), + ) + break operatorSwitch + } + + // Change the Kind of this If block, indicating that + // the if has else block. + frame.kind = controlFrameKindIfWithElse + + // We need to reset the stack so that + // the values pushed inside the then block + // do not affect the else block. + dropOp := newOperationDrop(c.getFrameDropRange(frame, false)) + + // Reset the stack manipulated by the then block, and re-push the block param types to the stack. + + c.stack = c.stack[:frame.originalStackLenWithoutParam] + for _, t := range frame.blockType.Params { + c.stackPush(wasmValueTypeTounsignedType(t)) + } + + // Prep labels for else and the continuation of this if block. + elseLabel := newLabel(labelKindElse, frame.frameID) + continuationLabel := newLabel(labelKindContinuation, frame.frameID) + c.result.LabelCallers[continuationLabel]++ + + // Emit the instructions for exiting the if loop, + // and then the initiation of else block. + c.emit(dropOp) + // Jump to the continuation of this block. + c.emit(newOperationBr(continuationLabel)) + // Initiate the else block. + c.emit(newOperationLabel(elseLabel)) + case wasm.OpcodeEnd: + if c.unreachableState.on && c.unreachableState.depth > 0 { + c.unreachableState.depth-- + break operatorSwitch + } else if c.unreachableState.on { + c.resetUnreachable() + + frame := c.controlFrames.pop() + if c.controlFrames.empty() { + return nil + } + + c.stack = c.stack[:frame.originalStackLenWithoutParam] + for _, t := range frame.blockType.Results { + c.stackPush(wasmValueTypeTounsignedType(t)) + } + + continuationLabel := newLabel(labelKindContinuation, frame.frameID) + if frame.kind == controlFrameKindIfWithoutElse { + // Emit the else label. + elseLabel := newLabel(labelKindElse, frame.frameID) + c.result.LabelCallers[continuationLabel]++ + c.emit(newOperationLabel(elseLabel)) + c.emit(newOperationBr(continuationLabel)) + c.emit(newOperationLabel(continuationLabel)) + } else { + c.emit( + newOperationLabel(continuationLabel), + ) + } + + break operatorSwitch + } + + frame := c.controlFrames.pop() + + // We need to reset the stack so that + // the values pushed inside the block. + dropOp := newOperationDrop(c.getFrameDropRange(frame, true)) + c.stack = c.stack[:frame.originalStackLenWithoutParam] + + // Push the result types onto the stack. + for _, t := range frame.blockType.Results { + c.stackPush(wasmValueTypeTounsignedType(t)) + } + + // Emit the instructions according to the Kind of the current control frame. + switch frame.kind { + case controlFrameKindFunction: + if !c.controlFrames.empty() { + // Should never happen. If so, there's a bug in the translation. + panic("bug: found more function control frames") + } + // Return from function. + c.emit(dropOp) + c.emit(newOperationBr(newLabel(labelKindReturn, 0))) + case controlFrameKindIfWithoutElse: + // This case we have to emit "empty" else label. + elseLabel := newLabel(labelKindElse, frame.frameID) + continuationLabel := newLabel(labelKindContinuation, frame.frameID) + c.result.LabelCallers[continuationLabel] += 2 + c.emit(dropOp) + c.emit(newOperationBr(continuationLabel)) + // Emit the else which soon branches into the continuation. + c.emit(newOperationLabel(elseLabel)) + c.emit(newOperationBr(continuationLabel)) + // Initiate the continuation. + c.emit(newOperationLabel(continuationLabel)) + case controlFrameKindBlockWithContinuationLabel, + controlFrameKindIfWithElse: + continuationLabel := newLabel(labelKindContinuation, frame.frameID) + c.result.LabelCallers[continuationLabel]++ + c.emit(dropOp) + c.emit(newOperationBr(continuationLabel)) + c.emit(newOperationLabel(continuationLabel)) + case controlFrameKindLoop, controlFrameKindBlockWithoutContinuationLabel: + c.emit( + dropOp, + ) + default: + // Should never happen. If so, there's a bug in the translation. + panic(fmt.Errorf("bug: invalid control frame Kind: 0x%x", frame.kind)) + } + + case wasm.OpcodeBr: + targetIndex, n, err := leb128.LoadUint32(c.body[c.pc+1:]) + if err != nil { + return fmt.Errorf("read the target for br_if: %w", err) + } + c.pc += n + + if c.unreachableState.on { + // If it is currently in unreachable, br is no-op. + break operatorSwitch + } + + targetFrame := c.controlFrames.get(int(targetIndex)) + targetFrame.ensureContinuation() + dropOp := newOperationDrop(c.getFrameDropRange(targetFrame, false)) + targetID := targetFrame.asLabel() + c.result.LabelCallers[targetID]++ + c.emit(dropOp) + c.emit(newOperationBr(targetID)) + // Br operation is stack-polymorphic, and mark the state as unreachable. + // That means subsequent instructions in the current control frame are "unreachable" + // and can be safely removed. + c.markUnreachable() + case wasm.OpcodeBrIf: + targetIndex, n, err := leb128.LoadUint32(c.body[c.pc+1:]) + if err != nil { + return fmt.Errorf("read the target for br_if: %w", err) + } + c.pc += n + + if c.unreachableState.on { + // If it is currently in unreachable, br-if is no-op. + break operatorSwitch + } + + targetFrame := c.controlFrames.get(int(targetIndex)) + targetFrame.ensureContinuation() + drop := c.getFrameDropRange(targetFrame, false) + target := targetFrame.asLabel() + c.result.LabelCallers[target]++ + + continuationLabel := newLabel(labelKindHeader, c.nextFrameID()) + c.result.LabelCallers[continuationLabel]++ + c.emit(newOperationBrIf(target, continuationLabel, drop)) + // Start emitting else block operations. + c.emit(newOperationLabel(continuationLabel)) + case wasm.OpcodeBrTable: + c.br.Reset(c.body[c.pc+1:]) + r := c.br + numTargets, n, err := leb128.DecodeUint32(r) + if err != nil { + return fmt.Errorf("error reading number of targets in br_table: %w", err) + } + c.pc += n + + if c.unreachableState.on { + // If it is currently in unreachable, br_table is no-op. + // But before proceeding to the next instruction, we must advance the pc + // according to the number of br_table targets. + for i := uint32(0); i <= numTargets; i++ { // inclusive as we also need to read the index of default target. + _, n, err := leb128.DecodeUint32(r) + if err != nil { + return fmt.Errorf("error reading target %d in br_table: %w", i, err) + } + c.pc += n + } + break operatorSwitch + } + + // Read the branch targets. + s := numTargets * 2 + targetLabels := make([]uint64, 2+s) // (label, inclusiveRange) * (default+numTargets) + for i := uint32(0); i < s; i += 2 { + l, n, err := leb128.DecodeUint32(r) + if err != nil { + return fmt.Errorf("error reading target %d in br_table: %w", i, err) + } + c.pc += n + targetFrame := c.controlFrames.get(int(l)) + targetFrame.ensureContinuation() + drop := c.getFrameDropRange(targetFrame, false) + targetLabel := targetFrame.asLabel() + targetLabels[i] = uint64(targetLabel) + targetLabels[i+1] = drop.AsU64() + c.result.LabelCallers[targetLabel]++ + } + + // Prep default target control frame. + l, n, err := leb128.DecodeUint32(r) + if err != nil { + return fmt.Errorf("error reading default target of br_table: %w", err) + } + c.pc += n + defaultTargetFrame := c.controlFrames.get(int(l)) + defaultTargetFrame.ensureContinuation() + defaultTargetDrop := c.getFrameDropRange(defaultTargetFrame, false) + defaultLabel := defaultTargetFrame.asLabel() + c.result.LabelCallers[defaultLabel]++ + targetLabels[s] = uint64(defaultLabel) + targetLabels[s+1] = defaultTargetDrop.AsU64() + c.emit(newOperationBrTable(targetLabels)) + + // br_table operation is stack-polymorphic, and mark the state as unreachable. + // That means subsequent instructions in the current control frame are "unreachable" + // and can be safely removed. + c.markUnreachable() + case wasm.OpcodeReturn: + functionFrame := c.controlFrames.functionFrame() + dropOp := newOperationDrop(c.getFrameDropRange(functionFrame, false)) + + // Cleanup the stack and then jmp to function frame's continuation (meaning return). + c.emit(dropOp) + c.emit(newOperationBr(functionFrame.asLabel())) + + // Return operation is stack-polymorphic, and mark the state as unreachable. + // That means subsequent instructions in the current control frame are "unreachable" + // and can be safely removed. + c.markUnreachable() + case wasm.OpcodeCall: + c.emit( + newOperationCall(index), + ) + case wasm.OpcodeCallIndirect: + typeIndex := index + tableIndex, n, err := leb128.LoadUint32(c.body[c.pc+1:]) + if err != nil { + return fmt.Errorf("read target for br_table: %w", err) + } + c.pc += n + c.emit( + newOperationCallIndirect(typeIndex, tableIndex), + ) + case wasm.OpcodeDrop: + r := inclusiveRange{Start: 0, End: 0} + if peekValueType == unsignedTypeV128 { + // inclusiveRange is the range in uint64 representation, so dropping a vector value on top + // should be translated as drop [0..1] inclusively. + r.End++ + } + c.emit(newOperationDrop(r)) + case wasm.OpcodeSelect: + // If it is on the unreachable state, ignore the instruction. + if c.unreachableState.on { + break operatorSwitch + } + isTargetVector := c.stackPeek() == unsignedTypeV128 + c.emit( + newOperationSelect(isTargetVector), + ) + case wasm.OpcodeTypedSelect: + // Skips two bytes: vector size fixed to 1, and the value type for select. + c.pc += 2 + // If it is on the unreachable state, ignore the instruction. + if c.unreachableState.on { + break operatorSwitch + } + // Typed select is semantically equivalent to select at runtime. + isTargetVector := c.stackPeek() == unsignedTypeV128 + c.emit( + newOperationSelect(isTargetVector), + ) + case wasm.OpcodeLocalGet: + depth := c.localDepth(index) + if isVector := c.localType(index) == wasm.ValueTypeV128; !isVector { + c.emit( + // -1 because we already manipulated the stack before + // called localDepth ^^. + newOperationPick(depth-1, isVector), + ) + } else { + c.emit( + // -2 because we already manipulated the stack before + // called localDepth ^^. + newOperationPick(depth-2, isVector), + ) + } + case wasm.OpcodeLocalSet: + depth := c.localDepth(index) + + isVector := c.localType(index) == wasm.ValueTypeV128 + if isVector { + c.emit( + // +2 because we already popped the operands for this operation from the c.stack before + // called localDepth ^^, + newOperationSet(depth+2, isVector), + ) + } else { + c.emit( + // +1 because we already popped the operands for this operation from the c.stack before + // called localDepth ^^, + newOperationSet(depth+1, isVector), + ) + } + case wasm.OpcodeLocalTee: + depth := c.localDepth(index) + isVector := c.localType(index) == wasm.ValueTypeV128 + if isVector { + c.emit(newOperationPick(1, isVector)) + c.emit(newOperationSet(depth+2, isVector)) + } else { + c.emit( + newOperationPick(0, isVector)) + c.emit(newOperationSet(depth+1, isVector)) + } + case wasm.OpcodeGlobalGet: + c.emit( + newOperationGlobalGet(index), + ) + case wasm.OpcodeGlobalSet: + c.emit( + newOperationGlobalSet(index), + ) + case wasm.OpcodeI32Load: + imm, err := c.readMemoryArg(wasm.OpcodeI32LoadName) + if err != nil { + return err + } + c.emit(newOperationLoad(unsignedTypeI32, imm)) + case wasm.OpcodeI64Load: + imm, err := c.readMemoryArg(wasm.OpcodeI64LoadName) + if err != nil { + return err + } + c.emit(newOperationLoad(unsignedTypeI64, imm)) + case wasm.OpcodeF32Load: + imm, err := c.readMemoryArg(wasm.OpcodeF32LoadName) + if err != nil { + return err + } + c.emit(newOperationLoad(unsignedTypeF32, imm)) + case wasm.OpcodeF64Load: + imm, err := c.readMemoryArg(wasm.OpcodeF64LoadName) + if err != nil { + return err + } + c.emit(newOperationLoad(unsignedTypeF64, imm)) + case wasm.OpcodeI32Load8S: + imm, err := c.readMemoryArg(wasm.OpcodeI32Load8SName) + if err != nil { + return err + } + c.emit(newOperationLoad8(signedInt32, imm)) + case wasm.OpcodeI32Load8U: + imm, err := c.readMemoryArg(wasm.OpcodeI32Load8UName) + if err != nil { + return err + } + c.emit(newOperationLoad8(signedUint32, imm)) + case wasm.OpcodeI32Load16S: + imm, err := c.readMemoryArg(wasm.OpcodeI32Load16SName) + if err != nil { + return err + } + c.emit(newOperationLoad16(signedInt32, imm)) + case wasm.OpcodeI32Load16U: + imm, err := c.readMemoryArg(wasm.OpcodeI32Load16UName) + if err != nil { + return err + } + c.emit(newOperationLoad16(signedUint32, imm)) + case wasm.OpcodeI64Load8S: + imm, err := c.readMemoryArg(wasm.OpcodeI64Load8SName) + if err != nil { + return err + } + c.emit(newOperationLoad8(signedInt64, imm)) + case wasm.OpcodeI64Load8U: + imm, err := c.readMemoryArg(wasm.OpcodeI64Load8UName) + if err != nil { + return err + } + c.emit(newOperationLoad8(signedUint64, imm)) + case wasm.OpcodeI64Load16S: + imm, err := c.readMemoryArg(wasm.OpcodeI64Load16SName) + if err != nil { + return err + } + c.emit(newOperationLoad16(signedInt64, imm)) + case wasm.OpcodeI64Load16U: + imm, err := c.readMemoryArg(wasm.OpcodeI64Load16UName) + if err != nil { + return err + } + c.emit(newOperationLoad16(signedUint64, imm)) + case wasm.OpcodeI64Load32S: + imm, err := c.readMemoryArg(wasm.OpcodeI64Load32SName) + if err != nil { + return err + } + c.emit(newOperationLoad32(true, imm)) + case wasm.OpcodeI64Load32U: + imm, err := c.readMemoryArg(wasm.OpcodeI64Load32UName) + if err != nil { + return err + } + c.emit(newOperationLoad32(false, imm)) + case wasm.OpcodeI32Store: + imm, err := c.readMemoryArg(wasm.OpcodeI32StoreName) + if err != nil { + return err + } + c.emit( + newOperationStore(unsignedTypeI32, imm), + ) + case wasm.OpcodeI64Store: + imm, err := c.readMemoryArg(wasm.OpcodeI64StoreName) + if err != nil { + return err + } + c.emit( + newOperationStore(unsignedTypeI64, imm), + ) + case wasm.OpcodeF32Store: + imm, err := c.readMemoryArg(wasm.OpcodeF32StoreName) + if err != nil { + return err + } + c.emit( + newOperationStore(unsignedTypeF32, imm), + ) + case wasm.OpcodeF64Store: + imm, err := c.readMemoryArg(wasm.OpcodeF64StoreName) + if err != nil { + return err + } + c.emit( + newOperationStore(unsignedTypeF64, imm), + ) + case wasm.OpcodeI32Store8: + imm, err := c.readMemoryArg(wasm.OpcodeI32Store8Name) + if err != nil { + return err + } + c.emit( + newOperationStore8(imm), + ) + case wasm.OpcodeI32Store16: + imm, err := c.readMemoryArg(wasm.OpcodeI32Store16Name) + if err != nil { + return err + } + c.emit( + newOperationStore16(imm), + ) + case wasm.OpcodeI64Store8: + imm, err := c.readMemoryArg(wasm.OpcodeI64Store8Name) + if err != nil { + return err + } + c.emit( + newOperationStore8(imm), + ) + case wasm.OpcodeI64Store16: + imm, err := c.readMemoryArg(wasm.OpcodeI64Store16Name) + if err != nil { + return err + } + c.emit( + newOperationStore16(imm), + ) + case wasm.OpcodeI64Store32: + imm, err := c.readMemoryArg(wasm.OpcodeI64Store32Name) + if err != nil { + return err + } + c.emit( + newOperationStore32(imm), + ) + case wasm.OpcodeMemorySize: + c.result.UsesMemory = true + c.pc++ // Skip the reserved one byte. + c.emit( + newOperationMemorySize(), + ) + case wasm.OpcodeMemoryGrow: + c.result.UsesMemory = true + c.pc++ // Skip the reserved one byte. + c.emit( + newOperationMemoryGrow(), + ) + case wasm.OpcodeI32Const: + val, num, err := leb128.LoadInt32(c.body[c.pc+1:]) + if err != nil { + return fmt.Errorf("reading i32.const value: %v", err) + } + c.pc += num + c.emit( + newOperationConstI32(uint32(val)), + ) + case wasm.OpcodeI64Const: + val, num, err := leb128.LoadInt64(c.body[c.pc+1:]) + if err != nil { + return fmt.Errorf("reading i64.const value: %v", err) + } + c.pc += num + c.emit( + newOperationConstI64(uint64(val)), + ) + case wasm.OpcodeF32Const: + v := math.Float32frombits(binary.LittleEndian.Uint32(c.body[c.pc+1:])) + c.pc += 4 + c.emit( + newOperationConstF32(v), + ) + case wasm.OpcodeF64Const: + v := math.Float64frombits(binary.LittleEndian.Uint64(c.body[c.pc+1:])) + c.pc += 8 + c.emit( + newOperationConstF64(v), + ) + case wasm.OpcodeI32Eqz: + c.emit( + newOperationEqz(unsignedInt32), + ) + case wasm.OpcodeI32Eq: + c.emit( + newOperationEq(unsignedTypeI32), + ) + case wasm.OpcodeI32Ne: + c.emit( + newOperationNe(unsignedTypeI32), + ) + case wasm.OpcodeI32LtS: + c.emit( + newOperationLt(signedTypeInt32), + ) + case wasm.OpcodeI32LtU: + c.emit( + newOperationLt(signedTypeUint32), + ) + case wasm.OpcodeI32GtS: + c.emit( + newOperationGt(signedTypeInt32), + ) + case wasm.OpcodeI32GtU: + c.emit( + newOperationGt(signedTypeUint32), + ) + case wasm.OpcodeI32LeS: + c.emit( + newOperationLe(signedTypeInt32), + ) + case wasm.OpcodeI32LeU: + c.emit( + newOperationLe(signedTypeUint32), + ) + case wasm.OpcodeI32GeS: + c.emit( + newOperationGe(signedTypeInt32), + ) + case wasm.OpcodeI32GeU: + c.emit( + newOperationGe(signedTypeUint32), + ) + case wasm.OpcodeI64Eqz: + c.emit( + newOperationEqz(unsignedInt64), + ) + case wasm.OpcodeI64Eq: + c.emit( + newOperationEq(unsignedTypeI64), + ) + case wasm.OpcodeI64Ne: + c.emit( + newOperationNe(unsignedTypeI64), + ) + case wasm.OpcodeI64LtS: + c.emit( + newOperationLt(signedTypeInt64), + ) + case wasm.OpcodeI64LtU: + c.emit( + newOperationLt(signedTypeUint64), + ) + case wasm.OpcodeI64GtS: + c.emit( + newOperationGt(signedTypeInt64), + ) + case wasm.OpcodeI64GtU: + c.emit( + newOperationGt(signedTypeUint64), + ) + case wasm.OpcodeI64LeS: + c.emit( + newOperationLe(signedTypeInt64), + ) + case wasm.OpcodeI64LeU: + c.emit( + newOperationLe(signedTypeUint64), + ) + case wasm.OpcodeI64GeS: + c.emit( + newOperationGe(signedTypeInt64), + ) + case wasm.OpcodeI64GeU: + c.emit( + newOperationGe(signedTypeUint64), + ) + case wasm.OpcodeF32Eq: + c.emit( + newOperationEq(unsignedTypeF32), + ) + case wasm.OpcodeF32Ne: + c.emit( + newOperationNe(unsignedTypeF32), + ) + case wasm.OpcodeF32Lt: + c.emit( + newOperationLt(signedTypeFloat32), + ) + case wasm.OpcodeF32Gt: + c.emit( + newOperationGt(signedTypeFloat32), + ) + case wasm.OpcodeF32Le: + c.emit( + newOperationLe(signedTypeFloat32), + ) + case wasm.OpcodeF32Ge: + c.emit( + newOperationGe(signedTypeFloat32), + ) + case wasm.OpcodeF64Eq: + c.emit( + newOperationEq(unsignedTypeF64), + ) + case wasm.OpcodeF64Ne: + c.emit( + newOperationNe(unsignedTypeF64), + ) + case wasm.OpcodeF64Lt: + c.emit( + newOperationLt(signedTypeFloat64), + ) + case wasm.OpcodeF64Gt: + c.emit( + newOperationGt(signedTypeFloat64), + ) + case wasm.OpcodeF64Le: + c.emit( + newOperationLe(signedTypeFloat64), + ) + case wasm.OpcodeF64Ge: + c.emit( + newOperationGe(signedTypeFloat64), + ) + case wasm.OpcodeI32Clz: + c.emit( + newOperationClz(unsignedInt32), + ) + case wasm.OpcodeI32Ctz: + c.emit( + newOperationCtz(unsignedInt32), + ) + case wasm.OpcodeI32Popcnt: + c.emit( + newOperationPopcnt(unsignedInt32), + ) + case wasm.OpcodeI32Add: + c.emit( + newOperationAdd(unsignedTypeI32), + ) + case wasm.OpcodeI32Sub: + c.emit( + newOperationSub(unsignedTypeI32), + ) + case wasm.OpcodeI32Mul: + c.emit( + newOperationMul(unsignedTypeI32), + ) + case wasm.OpcodeI32DivS: + c.emit( + newOperationDiv(signedTypeInt32), + ) + case wasm.OpcodeI32DivU: + c.emit( + newOperationDiv(signedTypeUint32), + ) + case wasm.OpcodeI32RemS: + c.emit( + newOperationRem(signedInt32), + ) + case wasm.OpcodeI32RemU: + c.emit( + newOperationRem(signedUint32), + ) + case wasm.OpcodeI32And: + c.emit( + newOperationAnd(unsignedInt32), + ) + case wasm.OpcodeI32Or: + c.emit( + newOperationOr(unsignedInt32), + ) + case wasm.OpcodeI32Xor: + c.emit( + newOperationXor(unsignedInt64), + ) + case wasm.OpcodeI32Shl: + c.emit( + newOperationShl(unsignedInt32), + ) + case wasm.OpcodeI32ShrS: + c.emit( + newOperationShr(signedInt32), + ) + case wasm.OpcodeI32ShrU: + c.emit( + newOperationShr(signedUint32), + ) + case wasm.OpcodeI32Rotl: + c.emit( + newOperationRotl(unsignedInt32), + ) + case wasm.OpcodeI32Rotr: + c.emit( + newOperationRotr(unsignedInt32), + ) + case wasm.OpcodeI64Clz: + c.emit( + newOperationClz(unsignedInt64), + ) + case wasm.OpcodeI64Ctz: + c.emit( + newOperationCtz(unsignedInt64), + ) + case wasm.OpcodeI64Popcnt: + c.emit( + newOperationPopcnt(unsignedInt64), + ) + case wasm.OpcodeI64Add: + c.emit( + newOperationAdd(unsignedTypeI64), + ) + case wasm.OpcodeI64Sub: + c.emit( + newOperationSub(unsignedTypeI64), + ) + case wasm.OpcodeI64Mul: + c.emit( + newOperationMul(unsignedTypeI64), + ) + case wasm.OpcodeI64DivS: + c.emit( + newOperationDiv(signedTypeInt64), + ) + case wasm.OpcodeI64DivU: + c.emit( + newOperationDiv(signedTypeUint64), + ) + case wasm.OpcodeI64RemS: + c.emit( + newOperationRem(signedInt64), + ) + case wasm.OpcodeI64RemU: + c.emit( + newOperationRem(signedUint64), + ) + case wasm.OpcodeI64And: + c.emit( + newOperationAnd(unsignedInt64), + ) + case wasm.OpcodeI64Or: + c.emit( + newOperationOr(unsignedInt64), + ) + case wasm.OpcodeI64Xor: + c.emit( + newOperationXor(unsignedInt64), + ) + case wasm.OpcodeI64Shl: + c.emit( + newOperationShl(unsignedInt64), + ) + case wasm.OpcodeI64ShrS: + c.emit( + newOperationShr(signedInt64), + ) + case wasm.OpcodeI64ShrU: + c.emit( + newOperationShr(signedUint64), + ) + case wasm.OpcodeI64Rotl: + c.emit( + newOperationRotl(unsignedInt64), + ) + case wasm.OpcodeI64Rotr: + c.emit( + newOperationRotr(unsignedInt64), + ) + case wasm.OpcodeF32Abs: + c.emit( + newOperationAbs(f32), + ) + case wasm.OpcodeF32Neg: + c.emit( + newOperationNeg(f32), + ) + case wasm.OpcodeF32Ceil: + c.emit( + newOperationCeil(f32), + ) + case wasm.OpcodeF32Floor: + c.emit( + newOperationFloor(f32), + ) + case wasm.OpcodeF32Trunc: + c.emit( + newOperationTrunc(f32), + ) + case wasm.OpcodeF32Nearest: + c.emit( + newOperationNearest(f32), + ) + case wasm.OpcodeF32Sqrt: + c.emit( + newOperationSqrt(f32), + ) + case wasm.OpcodeF32Add: + c.emit( + newOperationAdd(unsignedTypeF32), + ) + case wasm.OpcodeF32Sub: + c.emit( + newOperationSub(unsignedTypeF32), + ) + case wasm.OpcodeF32Mul: + c.emit( + newOperationMul(unsignedTypeF32), + ) + case wasm.OpcodeF32Div: + c.emit( + newOperationDiv(signedTypeFloat32), + ) + case wasm.OpcodeF32Min: + c.emit( + newOperationMin(f32), + ) + case wasm.OpcodeF32Max: + c.emit( + newOperationMax(f32), + ) + case wasm.OpcodeF32Copysign: + c.emit( + newOperationCopysign(f32), + ) + case wasm.OpcodeF64Abs: + c.emit( + newOperationAbs(f64), + ) + case wasm.OpcodeF64Neg: + c.emit( + newOperationNeg(f64), + ) + case wasm.OpcodeF64Ceil: + c.emit( + newOperationCeil(f64), + ) + case wasm.OpcodeF64Floor: + c.emit( + newOperationFloor(f64), + ) + case wasm.OpcodeF64Trunc: + c.emit( + newOperationTrunc(f64), + ) + case wasm.OpcodeF64Nearest: + c.emit( + newOperationNearest(f64), + ) + case wasm.OpcodeF64Sqrt: + c.emit( + newOperationSqrt(f64), + ) + case wasm.OpcodeF64Add: + c.emit( + newOperationAdd(unsignedTypeF64), + ) + case wasm.OpcodeF64Sub: + c.emit( + newOperationSub(unsignedTypeF64), + ) + case wasm.OpcodeF64Mul: + c.emit( + newOperationMul(unsignedTypeF64), + ) + case wasm.OpcodeF64Div: + c.emit( + newOperationDiv(signedTypeFloat64), + ) + case wasm.OpcodeF64Min: + c.emit( + newOperationMin(f64), + ) + case wasm.OpcodeF64Max: + c.emit( + newOperationMax(f64), + ) + case wasm.OpcodeF64Copysign: + c.emit( + newOperationCopysign(f64), + ) + case wasm.OpcodeI32WrapI64: + c.emit( + newOperationI32WrapFromI64(), + ) + case wasm.OpcodeI32TruncF32S: + c.emit( + newOperationITruncFromF(f32, signedInt32, false), + ) + case wasm.OpcodeI32TruncF32U: + c.emit( + newOperationITruncFromF(f32, signedUint32, false), + ) + case wasm.OpcodeI32TruncF64S: + c.emit( + newOperationITruncFromF(f64, signedInt32, false), + ) + case wasm.OpcodeI32TruncF64U: + c.emit( + newOperationITruncFromF(f64, signedUint32, false), + ) + case wasm.OpcodeI64ExtendI32S: + c.emit( + newOperationExtend(true), + ) + case wasm.OpcodeI64ExtendI32U: + c.emit( + newOperationExtend(false), + ) + case wasm.OpcodeI64TruncF32S: + c.emit( + newOperationITruncFromF(f32, signedInt64, false), + ) + case wasm.OpcodeI64TruncF32U: + c.emit( + newOperationITruncFromF(f32, signedUint64, false), + ) + case wasm.OpcodeI64TruncF64S: + c.emit( + newOperationITruncFromF(f64, signedInt64, false), + ) + case wasm.OpcodeI64TruncF64U: + c.emit( + newOperationITruncFromF(f64, signedUint64, false), + ) + case wasm.OpcodeF32ConvertI32S: + c.emit( + newOperationFConvertFromI(signedInt32, f32), + ) + case wasm.OpcodeF32ConvertI32U: + c.emit( + newOperationFConvertFromI(signedUint32, f32), + ) + case wasm.OpcodeF32ConvertI64S: + c.emit( + newOperationFConvertFromI(signedInt64, f32), + ) + case wasm.OpcodeF32ConvertI64U: + c.emit( + newOperationFConvertFromI(signedUint64, f32), + ) + case wasm.OpcodeF32DemoteF64: + c.emit( + newOperationF32DemoteFromF64(), + ) + case wasm.OpcodeF64ConvertI32S: + c.emit( + newOperationFConvertFromI(signedInt32, f64), + ) + case wasm.OpcodeF64ConvertI32U: + c.emit( + newOperationFConvertFromI(signedUint32, f64), + ) + case wasm.OpcodeF64ConvertI64S: + c.emit( + newOperationFConvertFromI(signedInt64, f64), + ) + case wasm.OpcodeF64ConvertI64U: + c.emit( + newOperationFConvertFromI(signedUint64, f64), + ) + case wasm.OpcodeF64PromoteF32: + c.emit( + newOperationF64PromoteFromF32(), + ) + case wasm.OpcodeI32ReinterpretF32: + c.emit( + newOperationI32ReinterpretFromF32(), + ) + case wasm.OpcodeI64ReinterpretF64: + c.emit( + newOperationI64ReinterpretFromF64(), + ) + case wasm.OpcodeF32ReinterpretI32: + c.emit( + newOperationF32ReinterpretFromI32(), + ) + case wasm.OpcodeF64ReinterpretI64: + c.emit( + newOperationF64ReinterpretFromI64(), + ) + case wasm.OpcodeI32Extend8S: + c.emit( + newOperationSignExtend32From8(), + ) + case wasm.OpcodeI32Extend16S: + c.emit( + newOperationSignExtend32From16(), + ) + case wasm.OpcodeI64Extend8S: + c.emit( + newOperationSignExtend64From8(), + ) + case wasm.OpcodeI64Extend16S: + c.emit( + newOperationSignExtend64From16(), + ) + case wasm.OpcodeI64Extend32S: + c.emit( + newOperationSignExtend64From32(), + ) + case wasm.OpcodeRefFunc: + c.pc++ + index, num, err := leb128.LoadUint32(c.body[c.pc:]) + if err != nil { + return fmt.Errorf("failed to read function index for ref.func: %v", err) + } + c.pc += num - 1 + c.emit( + newOperationRefFunc(index), + ) + case wasm.OpcodeRefNull: + c.pc++ // Skip the type of reftype as every ref value is opaque pointer. + c.emit( + newOperationConstI64(0), + ) + case wasm.OpcodeRefIsNull: + // Simply compare the opaque pointer (i64) with zero. + c.emit( + newOperationEqz(unsignedInt64), + ) + case wasm.OpcodeTableGet: + c.pc++ + tableIndex, num, err := leb128.LoadUint32(c.body[c.pc:]) + if err != nil { + return fmt.Errorf("failed to read function index for table.get: %v", err) + } + c.pc += num - 1 + c.emit( + newOperationTableGet(tableIndex), + ) + case wasm.OpcodeTableSet: + c.pc++ + tableIndex, num, err := leb128.LoadUint32(c.body[c.pc:]) + if err != nil { + return fmt.Errorf("failed to read function index for table.set: %v", err) + } + c.pc += num - 1 + c.emit( + newOperationTableSet(tableIndex), + ) + case wasm.OpcodeMiscPrefix: + c.pc++ + // A misc opcode is encoded as an unsigned variable 32-bit integer. + miscOp, num, err := leb128.LoadUint32(c.body[c.pc:]) + if err != nil { + return fmt.Errorf("failed to read misc opcode: %v", err) + } + c.pc += num - 1 + switch byte(miscOp) { + case wasm.OpcodeMiscI32TruncSatF32S: + c.emit( + newOperationITruncFromF(f32, signedInt32, true), + ) + case wasm.OpcodeMiscI32TruncSatF32U: + c.emit( + newOperationITruncFromF(f32, signedUint32, true), + ) + case wasm.OpcodeMiscI32TruncSatF64S: + c.emit( + newOperationITruncFromF(f64, signedInt32, true), + ) + case wasm.OpcodeMiscI32TruncSatF64U: + c.emit( + newOperationITruncFromF(f64, signedUint32, true), + ) + case wasm.OpcodeMiscI64TruncSatF32S: + c.emit( + newOperationITruncFromF(f32, signedInt64, true), + ) + case wasm.OpcodeMiscI64TruncSatF32U: + c.emit( + newOperationITruncFromF(f32, signedUint64, true), + ) + case wasm.OpcodeMiscI64TruncSatF64S: + c.emit( + newOperationITruncFromF(f64, signedInt64, true), + ) + case wasm.OpcodeMiscI64TruncSatF64U: + c.emit( + newOperationITruncFromF(f64, signedUint64, true), + ) + case wasm.OpcodeMiscMemoryInit: + c.result.UsesMemory = true + dataIndex, num, err := leb128.LoadUint32(c.body[c.pc+1:]) + if err != nil { + return fmt.Errorf("reading i32.const value: %v", err) + } + c.pc += num + 1 // +1 to skip the memory index which is fixed to zero. + c.emit( + newOperationMemoryInit(dataIndex), + ) + case wasm.OpcodeMiscDataDrop: + dataIndex, num, err := leb128.LoadUint32(c.body[c.pc+1:]) + if err != nil { + return fmt.Errorf("reading i32.const value: %v", err) + } + c.pc += num + c.emit( + newOperationDataDrop(dataIndex), + ) + case wasm.OpcodeMiscMemoryCopy: + c.result.UsesMemory = true + c.pc += 2 // +2 to skip two memory indexes which are fixed to zero. + c.emit( + newOperationMemoryCopy(), + ) + case wasm.OpcodeMiscMemoryFill: + c.result.UsesMemory = true + c.pc += 1 // +1 to skip the memory index which is fixed to zero. + c.emit( + newOperationMemoryFill(), + ) + case wasm.OpcodeMiscTableInit: + elemIndex, num, err := leb128.LoadUint32(c.body[c.pc+1:]) + if err != nil { + return fmt.Errorf("reading i32.const value: %v", err) + } + c.pc += num + // Read table index which is fixed to zero currently. + tableIndex, num, err := leb128.LoadUint32(c.body[c.pc+1:]) + if err != nil { + return fmt.Errorf("reading i32.const value: %v", err) + } + c.pc += num + c.emit( + newOperationTableInit(elemIndex, tableIndex), + ) + case wasm.OpcodeMiscElemDrop: + elemIndex, num, err := leb128.LoadUint32(c.body[c.pc+1:]) + if err != nil { + return fmt.Errorf("reading i32.const value: %v", err) + } + c.pc += num + c.emit( + newOperationElemDrop(elemIndex), + ) + case wasm.OpcodeMiscTableCopy: + // Read the source table inde.g. + dst, num, err := leb128.LoadUint32(c.body[c.pc+1:]) + if err != nil { + return fmt.Errorf("reading i32.const value: %v", err) + } + c.pc += num + // Read the destination table inde.g. + src, num, err := leb128.LoadUint32(c.body[c.pc+1:]) + if err != nil { + return fmt.Errorf("reading i32.const value: %v", err) + } + c.pc += num + c.emit( + newOperationTableCopy(src, dst), + ) + case wasm.OpcodeMiscTableGrow: + // Read the source table inde.g. + tableIndex, num, err := leb128.LoadUint32(c.body[c.pc+1:]) + if err != nil { + return fmt.Errorf("reading i32.const value: %v", err) + } + c.pc += num + c.emit( + newOperationTableGrow(tableIndex), + ) + case wasm.OpcodeMiscTableSize: + // Read the source table inde.g. + tableIndex, num, err := leb128.LoadUint32(c.body[c.pc+1:]) + if err != nil { + return fmt.Errorf("reading i32.const value: %v", err) + } + c.pc += num + c.emit( + newOperationTableSize(tableIndex), + ) + case wasm.OpcodeMiscTableFill: + // Read the source table index. + tableIndex, num, err := leb128.LoadUint32(c.body[c.pc+1:]) + if err != nil { + return fmt.Errorf("reading i32.const value: %v", err) + } + c.pc += num + c.emit( + newOperationTableFill(tableIndex), + ) + default: + return fmt.Errorf("unsupported misc instruction in interpreterir: 0x%x", op) + } + case wasm.OpcodeVecPrefix: + c.pc++ + switch vecOp := c.body[c.pc]; vecOp { + case wasm.OpcodeVecV128Const: + c.pc++ + lo := binary.LittleEndian.Uint64(c.body[c.pc : c.pc+8]) + c.pc += 8 + hi := binary.LittleEndian.Uint64(c.body[c.pc : c.pc+8]) + c.emit( + newOperationV128Const(lo, hi), + ) + c.pc += 7 + case wasm.OpcodeVecV128Load: + arg, err := c.readMemoryArg(wasm.OpcodeI32LoadName) + if err != nil { + return err + } + c.emit( + newOperationV128Load(v128LoadType128, arg), + ) + case wasm.OpcodeVecV128Load8x8s: + arg, err := c.readMemoryArg(wasm.OpcodeVecV128Load8x8SName) + if err != nil { + return err + } + c.emit( + newOperationV128Load(v128LoadType8x8s, arg), + ) + case wasm.OpcodeVecV128Load8x8u: + arg, err := c.readMemoryArg(wasm.OpcodeVecV128Load8x8UName) + if err != nil { + return err + } + c.emit( + newOperationV128Load(v128LoadType8x8u, arg), + ) + case wasm.OpcodeVecV128Load16x4s: + arg, err := c.readMemoryArg(wasm.OpcodeVecV128Load16x4SName) + if err != nil { + return err + } + c.emit( + newOperationV128Load(v128LoadType16x4s, arg), + ) + case wasm.OpcodeVecV128Load16x4u: + arg, err := c.readMemoryArg(wasm.OpcodeVecV128Load16x4UName) + if err != nil { + return err + } + c.emit( + newOperationV128Load(v128LoadType16x4u, arg), + ) + case wasm.OpcodeVecV128Load32x2s: + arg, err := c.readMemoryArg(wasm.OpcodeVecV128Load32x2SName) + if err != nil { + return err + } + c.emit( + newOperationV128Load(v128LoadType32x2s, arg), + ) + case wasm.OpcodeVecV128Load32x2u: + arg, err := c.readMemoryArg(wasm.OpcodeVecV128Load32x2UName) + if err != nil { + return err + } + c.emit( + newOperationV128Load(v128LoadType32x2u, arg), + ) + case wasm.OpcodeVecV128Load8Splat: + arg, err := c.readMemoryArg(wasm.OpcodeVecV128Load8SplatName) + if err != nil { + return err + } + c.emit( + newOperationV128Load(v128LoadType8Splat, arg), + ) + case wasm.OpcodeVecV128Load16Splat: + arg, err := c.readMemoryArg(wasm.OpcodeVecV128Load16SplatName) + if err != nil { + return err + } + c.emit( + newOperationV128Load(v128LoadType16Splat, arg), + ) + case wasm.OpcodeVecV128Load32Splat: + arg, err := c.readMemoryArg(wasm.OpcodeVecV128Load32SplatName) + if err != nil { + return err + } + c.emit( + newOperationV128Load(v128LoadType32Splat, arg), + ) + case wasm.OpcodeVecV128Load64Splat: + arg, err := c.readMemoryArg(wasm.OpcodeVecV128Load64SplatName) + if err != nil { + return err + } + c.emit( + newOperationV128Load(v128LoadType64Splat, arg), + ) + case wasm.OpcodeVecV128Load32zero: + arg, err := c.readMemoryArg(wasm.OpcodeVecV128Load32zeroName) + if err != nil { + return err + } + c.emit( + newOperationV128Load(v128LoadType32zero, arg), + ) + case wasm.OpcodeVecV128Load64zero: + arg, err := c.readMemoryArg(wasm.OpcodeVecV128Load64zeroName) + if err != nil { + return err + } + c.emit( + newOperationV128Load(v128LoadType64zero, arg), + ) + case wasm.OpcodeVecV128Load8Lane: + arg, err := c.readMemoryArg(wasm.OpcodeVecV128Load8LaneName) + if err != nil { + return err + } + c.pc++ + laneIndex := c.body[c.pc] + c.emit( + newOperationV128LoadLane(laneIndex, 8, arg), + ) + case wasm.OpcodeVecV128Load16Lane: + arg, err := c.readMemoryArg(wasm.OpcodeVecV128Load16LaneName) + if err != nil { + return err + } + c.pc++ + laneIndex := c.body[c.pc] + c.emit( + newOperationV128LoadLane(laneIndex, 16, arg), + ) + case wasm.OpcodeVecV128Load32Lane: + arg, err := c.readMemoryArg(wasm.OpcodeVecV128Load32LaneName) + if err != nil { + return err + } + c.pc++ + laneIndex := c.body[c.pc] + c.emit( + newOperationV128LoadLane(laneIndex, 32, arg), + ) + case wasm.OpcodeVecV128Load64Lane: + arg, err := c.readMemoryArg(wasm.OpcodeVecV128Load64LaneName) + if err != nil { + return err + } + c.pc++ + laneIndex := c.body[c.pc] + c.emit( + newOperationV128LoadLane(laneIndex, 64, arg), + ) + case wasm.OpcodeVecV128Store: + arg, err := c.readMemoryArg(wasm.OpcodeVecV128StoreName) + if err != nil { + return err + } + c.emit( + newOperationV128Store(arg), + ) + case wasm.OpcodeVecV128Store8Lane: + arg, err := c.readMemoryArg(wasm.OpcodeVecV128Store8LaneName) + if err != nil { + return err + } + c.pc++ + laneIndex := c.body[c.pc] + c.emit( + newOperationV128StoreLane(laneIndex, 8, arg), + ) + case wasm.OpcodeVecV128Store16Lane: + arg, err := c.readMemoryArg(wasm.OpcodeVecV128Store16LaneName) + if err != nil { + return err + } + c.pc++ + laneIndex := c.body[c.pc] + c.emit( + newOperationV128StoreLane(laneIndex, 16, arg), + ) + case wasm.OpcodeVecV128Store32Lane: + arg, err := c.readMemoryArg(wasm.OpcodeVecV128Store32LaneName) + if err != nil { + return err + } + c.pc++ + laneIndex := c.body[c.pc] + c.emit( + newOperationV128StoreLane(laneIndex, 32, arg), + ) + case wasm.OpcodeVecV128Store64Lane: + arg, err := c.readMemoryArg(wasm.OpcodeVecV128Store64LaneName) + if err != nil { + return err + } + c.pc++ + laneIndex := c.body[c.pc] + c.emit( + newOperationV128StoreLane(laneIndex, 64, arg), + ) + case wasm.OpcodeVecI8x16ExtractLaneS: + c.pc++ + laneIndex := c.body[c.pc] + c.emit( + newOperationV128ExtractLane(laneIndex, true, shapeI8x16), + ) + case wasm.OpcodeVecI8x16ExtractLaneU: + c.pc++ + laneIndex := c.body[c.pc] + c.emit( + newOperationV128ExtractLane(laneIndex, false, shapeI8x16), + ) + case wasm.OpcodeVecI16x8ExtractLaneS: + c.pc++ + laneIndex := c.body[c.pc] + c.emit( + newOperationV128ExtractLane(laneIndex, true, shapeI16x8), + ) + case wasm.OpcodeVecI16x8ExtractLaneU: + c.pc++ + laneIndex := c.body[c.pc] + c.emit( + newOperationV128ExtractLane(laneIndex, false, shapeI16x8), + ) + case wasm.OpcodeVecI32x4ExtractLane: + c.pc++ + laneIndex := c.body[c.pc] + c.emit( + newOperationV128ExtractLane(laneIndex, false, shapeI32x4), + ) + case wasm.OpcodeVecI64x2ExtractLane: + c.pc++ + laneIndex := c.body[c.pc] + c.emit( + newOperationV128ExtractLane(laneIndex, false, shapeI64x2), + ) + case wasm.OpcodeVecF32x4ExtractLane: + c.pc++ + laneIndex := c.body[c.pc] + c.emit( + newOperationV128ExtractLane(laneIndex, false, shapeF32x4), + ) + case wasm.OpcodeVecF64x2ExtractLane: + c.pc++ + laneIndex := c.body[c.pc] + c.emit( + newOperationV128ExtractLane(laneIndex, false, shapeF64x2), + ) + case wasm.OpcodeVecI8x16ReplaceLane: + c.pc++ + laneIndex := c.body[c.pc] + c.emit( + newOperationV128ReplaceLane(laneIndex, shapeI8x16), + ) + case wasm.OpcodeVecI16x8ReplaceLane: + c.pc++ + laneIndex := c.body[c.pc] + c.emit( + newOperationV128ReplaceLane(laneIndex, shapeI16x8), + ) + case wasm.OpcodeVecI32x4ReplaceLane: + c.pc++ + laneIndex := c.body[c.pc] + c.emit( + newOperationV128ReplaceLane(laneIndex, shapeI32x4), + ) + case wasm.OpcodeVecI64x2ReplaceLane: + c.pc++ + laneIndex := c.body[c.pc] + c.emit( + newOperationV128ReplaceLane(laneIndex, shapeI64x2), + ) + case wasm.OpcodeVecF32x4ReplaceLane: + c.pc++ + laneIndex := c.body[c.pc] + c.emit( + newOperationV128ReplaceLane(laneIndex, shapeF32x4), + ) + case wasm.OpcodeVecF64x2ReplaceLane: + c.pc++ + laneIndex := c.body[c.pc] + c.emit( + newOperationV128ReplaceLane(laneIndex, shapeF64x2), + ) + case wasm.OpcodeVecI8x16Splat: + c.emit( + newOperationV128Splat(shapeI8x16), + ) + case wasm.OpcodeVecI16x8Splat: + c.emit( + newOperationV128Splat(shapeI16x8), + ) + case wasm.OpcodeVecI32x4Splat: + c.emit( + newOperationV128Splat(shapeI32x4), + ) + case wasm.OpcodeVecI64x2Splat: + c.emit( + newOperationV128Splat(shapeI64x2), + ) + case wasm.OpcodeVecF32x4Splat: + c.emit( + newOperationV128Splat(shapeF32x4), + ) + case wasm.OpcodeVecF64x2Splat: + c.emit( + newOperationV128Splat(shapeF64x2), + ) + case wasm.OpcodeVecI8x16Swizzle: + c.emit( + newOperationV128Swizzle(), + ) + case wasm.OpcodeVecV128i8x16Shuffle: + c.pc++ + lanes := make([]uint64, 16) + for i := uint64(0); i < 16; i++ { + lanes[i] = uint64(c.body[c.pc+i]) + } + op := newOperationV128Shuffle(lanes) + c.emit(op) + c.pc += 15 + case wasm.OpcodeVecV128AnyTrue: + c.emit( + newOperationV128AnyTrue(), + ) + case wasm.OpcodeVecI8x16AllTrue: + c.emit( + newOperationV128AllTrue(shapeI8x16), + ) + case wasm.OpcodeVecI16x8AllTrue: + c.emit( + newOperationV128AllTrue(shapeI16x8), + ) + case wasm.OpcodeVecI32x4AllTrue: + c.emit( + newOperationV128AllTrue(shapeI32x4), + ) + case wasm.OpcodeVecI64x2AllTrue: + c.emit( + newOperationV128AllTrue(shapeI64x2), + ) + case wasm.OpcodeVecI8x16BitMask: + c.emit( + newOperationV128BitMask(shapeI8x16), + ) + case wasm.OpcodeVecI16x8BitMask: + c.emit( + newOperationV128BitMask(shapeI16x8), + ) + case wasm.OpcodeVecI32x4BitMask: + c.emit( + newOperationV128BitMask(shapeI32x4), + ) + case wasm.OpcodeVecI64x2BitMask: + c.emit( + newOperationV128BitMask(shapeI64x2), + ) + case wasm.OpcodeVecV128And: + c.emit( + newOperationV128And(), + ) + case wasm.OpcodeVecV128Not: + c.emit( + newOperationV128Not(), + ) + case wasm.OpcodeVecV128Or: + c.emit( + newOperationV128Or(), + ) + case wasm.OpcodeVecV128Xor: + c.emit( + newOperationV128Xor(), + ) + case wasm.OpcodeVecV128Bitselect: + c.emit( + newOperationV128Bitselect(), + ) + case wasm.OpcodeVecV128AndNot: + c.emit( + newOperationV128AndNot(), + ) + case wasm.OpcodeVecI8x16Shl: + c.emit( + newOperationV128Shl(shapeI8x16), + ) + case wasm.OpcodeVecI8x16ShrS: + c.emit( + newOperationV128Shr(shapeI8x16, true), + ) + case wasm.OpcodeVecI8x16ShrU: + c.emit( + newOperationV128Shr(shapeI8x16, false), + ) + case wasm.OpcodeVecI16x8Shl: + c.emit( + newOperationV128Shl(shapeI16x8), + ) + case wasm.OpcodeVecI16x8ShrS: + c.emit( + newOperationV128Shr(shapeI16x8, true), + ) + case wasm.OpcodeVecI16x8ShrU: + c.emit( + newOperationV128Shr(shapeI16x8, false), + ) + case wasm.OpcodeVecI32x4Shl: + c.emit( + newOperationV128Shl(shapeI32x4), + ) + case wasm.OpcodeVecI32x4ShrS: + c.emit( + newOperationV128Shr(shapeI32x4, true), + ) + case wasm.OpcodeVecI32x4ShrU: + c.emit( + newOperationV128Shr(shapeI32x4, false), + ) + case wasm.OpcodeVecI64x2Shl: + c.emit( + newOperationV128Shl(shapeI64x2), + ) + case wasm.OpcodeVecI64x2ShrS: + c.emit( + newOperationV128Shr(shapeI64x2, true), + ) + case wasm.OpcodeVecI64x2ShrU: + c.emit( + newOperationV128Shr(shapeI64x2, false), + ) + case wasm.OpcodeVecI8x16Eq: + c.emit( + newOperationV128Cmp(v128CmpTypeI8x16Eq), + ) + case wasm.OpcodeVecI8x16Ne: + c.emit( + newOperationV128Cmp(v128CmpTypeI8x16Ne), + ) + case wasm.OpcodeVecI8x16LtS: + c.emit( + newOperationV128Cmp(v128CmpTypeI8x16LtS), + ) + case wasm.OpcodeVecI8x16LtU: + c.emit( + newOperationV128Cmp(v128CmpTypeI8x16LtU), + ) + case wasm.OpcodeVecI8x16GtS: + c.emit( + newOperationV128Cmp(v128CmpTypeI8x16GtS), + ) + case wasm.OpcodeVecI8x16GtU: + c.emit( + newOperationV128Cmp(v128CmpTypeI8x16GtU), + ) + case wasm.OpcodeVecI8x16LeS: + c.emit( + newOperationV128Cmp(v128CmpTypeI8x16LeS), + ) + case wasm.OpcodeVecI8x16LeU: + c.emit( + newOperationV128Cmp(v128CmpTypeI8x16LeU), + ) + case wasm.OpcodeVecI8x16GeS: + c.emit( + newOperationV128Cmp(v128CmpTypeI8x16GeS), + ) + case wasm.OpcodeVecI8x16GeU: + c.emit( + newOperationV128Cmp(v128CmpTypeI8x16GeU), + ) + case wasm.OpcodeVecI16x8Eq: + c.emit( + newOperationV128Cmp(v128CmpTypeI16x8Eq), + ) + case wasm.OpcodeVecI16x8Ne: + c.emit( + newOperationV128Cmp(v128CmpTypeI16x8Ne), + ) + case wasm.OpcodeVecI16x8LtS: + c.emit( + newOperationV128Cmp(v128CmpTypeI16x8LtS), + ) + case wasm.OpcodeVecI16x8LtU: + c.emit( + newOperationV128Cmp(v128CmpTypeI16x8LtU), + ) + case wasm.OpcodeVecI16x8GtS: + c.emit( + newOperationV128Cmp(v128CmpTypeI16x8GtS), + ) + case wasm.OpcodeVecI16x8GtU: + c.emit( + newOperationV128Cmp(v128CmpTypeI16x8GtU), + ) + case wasm.OpcodeVecI16x8LeS: + c.emit( + newOperationV128Cmp(v128CmpTypeI16x8LeS), + ) + case wasm.OpcodeVecI16x8LeU: + c.emit( + newOperationV128Cmp(v128CmpTypeI16x8LeU), + ) + case wasm.OpcodeVecI16x8GeS: + c.emit( + newOperationV128Cmp(v128CmpTypeI16x8GeS), + ) + case wasm.OpcodeVecI16x8GeU: + c.emit( + newOperationV128Cmp(v128CmpTypeI16x8GeU), + ) + case wasm.OpcodeVecI32x4Eq: + c.emit( + newOperationV128Cmp(v128CmpTypeI32x4Eq), + ) + case wasm.OpcodeVecI32x4Ne: + c.emit( + newOperationV128Cmp(v128CmpTypeI32x4Ne), + ) + case wasm.OpcodeVecI32x4LtS: + c.emit( + newOperationV128Cmp(v128CmpTypeI32x4LtS), + ) + case wasm.OpcodeVecI32x4LtU: + c.emit( + newOperationV128Cmp(v128CmpTypeI32x4LtU), + ) + case wasm.OpcodeVecI32x4GtS: + c.emit( + newOperationV128Cmp(v128CmpTypeI32x4GtS), + ) + case wasm.OpcodeVecI32x4GtU: + c.emit( + newOperationV128Cmp(v128CmpTypeI32x4GtU), + ) + case wasm.OpcodeVecI32x4LeS: + c.emit( + newOperationV128Cmp(v128CmpTypeI32x4LeS), + ) + case wasm.OpcodeVecI32x4LeU: + c.emit( + newOperationV128Cmp(v128CmpTypeI32x4LeU), + ) + case wasm.OpcodeVecI32x4GeS: + c.emit( + newOperationV128Cmp(v128CmpTypeI32x4GeS), + ) + case wasm.OpcodeVecI32x4GeU: + c.emit( + newOperationV128Cmp(v128CmpTypeI32x4GeU), + ) + case wasm.OpcodeVecI64x2Eq: + c.emit( + newOperationV128Cmp(v128CmpTypeI64x2Eq), + ) + case wasm.OpcodeVecI64x2Ne: + c.emit( + newOperationV128Cmp(v128CmpTypeI64x2Ne), + ) + case wasm.OpcodeVecI64x2LtS: + c.emit( + newOperationV128Cmp(v128CmpTypeI64x2LtS), + ) + case wasm.OpcodeVecI64x2GtS: + c.emit( + newOperationV128Cmp(v128CmpTypeI64x2GtS), + ) + case wasm.OpcodeVecI64x2LeS: + c.emit( + newOperationV128Cmp(v128CmpTypeI64x2LeS), + ) + case wasm.OpcodeVecI64x2GeS: + c.emit( + newOperationV128Cmp(v128CmpTypeI64x2GeS), + ) + case wasm.OpcodeVecF32x4Eq: + c.emit( + newOperationV128Cmp(v128CmpTypeF32x4Eq), + ) + case wasm.OpcodeVecF32x4Ne: + c.emit( + newOperationV128Cmp(v128CmpTypeF32x4Ne), + ) + case wasm.OpcodeVecF32x4Lt: + c.emit( + newOperationV128Cmp(v128CmpTypeF32x4Lt), + ) + case wasm.OpcodeVecF32x4Gt: + c.emit( + newOperationV128Cmp(v128CmpTypeF32x4Gt), + ) + case wasm.OpcodeVecF32x4Le: + c.emit( + newOperationV128Cmp(v128CmpTypeF32x4Le), + ) + case wasm.OpcodeVecF32x4Ge: + c.emit( + newOperationV128Cmp(v128CmpTypeF32x4Ge), + ) + case wasm.OpcodeVecF64x2Eq: + c.emit( + newOperationV128Cmp(v128CmpTypeF64x2Eq), + ) + case wasm.OpcodeVecF64x2Ne: + c.emit( + newOperationV128Cmp(v128CmpTypeF64x2Ne), + ) + case wasm.OpcodeVecF64x2Lt: + c.emit( + newOperationV128Cmp(v128CmpTypeF64x2Lt), + ) + case wasm.OpcodeVecF64x2Gt: + c.emit( + newOperationV128Cmp(v128CmpTypeF64x2Gt), + ) + case wasm.OpcodeVecF64x2Le: + c.emit( + newOperationV128Cmp(v128CmpTypeF64x2Le), + ) + case wasm.OpcodeVecF64x2Ge: + c.emit( + newOperationV128Cmp(v128CmpTypeF64x2Ge), + ) + case wasm.OpcodeVecI8x16Neg: + c.emit( + newOperationV128Neg(shapeI8x16), + ) + case wasm.OpcodeVecI16x8Neg: + c.emit( + newOperationV128Neg(shapeI16x8), + ) + case wasm.OpcodeVecI32x4Neg: + c.emit( + newOperationV128Neg(shapeI32x4), + ) + case wasm.OpcodeVecI64x2Neg: + c.emit( + newOperationV128Neg(shapeI64x2), + ) + case wasm.OpcodeVecF32x4Neg: + c.emit( + newOperationV128Neg(shapeF32x4), + ) + case wasm.OpcodeVecF64x2Neg: + c.emit( + newOperationV128Neg(shapeF64x2), + ) + case wasm.OpcodeVecI8x16Add: + c.emit( + newOperationV128Add(shapeI8x16), + ) + case wasm.OpcodeVecI16x8Add: + c.emit( + newOperationV128Add(shapeI16x8), + ) + case wasm.OpcodeVecI32x4Add: + c.emit( + newOperationV128Add(shapeI32x4), + ) + case wasm.OpcodeVecI64x2Add: + c.emit( + newOperationV128Add(shapeI64x2), + ) + case wasm.OpcodeVecF32x4Add: + c.emit( + newOperationV128Add(shapeF32x4), + ) + case wasm.OpcodeVecF64x2Add: + c.emit( + newOperationV128Add(shapeF64x2), + ) + case wasm.OpcodeVecI8x16Sub: + c.emit( + newOperationV128Sub(shapeI8x16), + ) + case wasm.OpcodeVecI16x8Sub: + c.emit( + newOperationV128Sub(shapeI16x8), + ) + case wasm.OpcodeVecI32x4Sub: + c.emit( + newOperationV128Sub(shapeI32x4), + ) + case wasm.OpcodeVecI64x2Sub: + c.emit( + newOperationV128Sub(shapeI64x2), + ) + case wasm.OpcodeVecF32x4Sub: + c.emit( + newOperationV128Sub(shapeF32x4), + ) + case wasm.OpcodeVecF64x2Sub: + c.emit( + newOperationV128Sub(shapeF64x2), + ) + case wasm.OpcodeVecI8x16AddSatS: + c.emit( + newOperationV128AddSat(shapeI8x16, true), + ) + case wasm.OpcodeVecI8x16AddSatU: + c.emit( + newOperationV128AddSat(shapeI8x16, false), + ) + case wasm.OpcodeVecI16x8AddSatS: + c.emit( + newOperationV128AddSat(shapeI16x8, true), + ) + case wasm.OpcodeVecI16x8AddSatU: + c.emit( + newOperationV128AddSat(shapeI16x8, false), + ) + case wasm.OpcodeVecI8x16SubSatS: + c.emit( + newOperationV128SubSat(shapeI8x16, true), + ) + case wasm.OpcodeVecI8x16SubSatU: + c.emit( + newOperationV128SubSat(shapeI8x16, false), + ) + case wasm.OpcodeVecI16x8SubSatS: + c.emit( + newOperationV128SubSat(shapeI16x8, true), + ) + case wasm.OpcodeVecI16x8SubSatU: + c.emit( + newOperationV128SubSat(shapeI16x8, false), + ) + case wasm.OpcodeVecI16x8Mul: + c.emit( + newOperationV128Mul(shapeI16x8), + ) + case wasm.OpcodeVecI32x4Mul: + c.emit( + newOperationV128Mul(shapeI32x4), + ) + case wasm.OpcodeVecI64x2Mul: + c.emit( + newOperationV128Mul(shapeI64x2), + ) + case wasm.OpcodeVecF32x4Mul: + c.emit( + newOperationV128Mul(shapeF32x4), + ) + case wasm.OpcodeVecF64x2Mul: + c.emit( + newOperationV128Mul(shapeF64x2), + ) + case wasm.OpcodeVecF32x4Sqrt: + c.emit( + newOperationV128Sqrt(shapeF32x4), + ) + case wasm.OpcodeVecF64x2Sqrt: + c.emit( + newOperationV128Sqrt(shapeF64x2), + ) + case wasm.OpcodeVecF32x4Div: + c.emit( + newOperationV128Div(shapeF32x4), + ) + case wasm.OpcodeVecF64x2Div: + c.emit( + newOperationV128Div(shapeF64x2), + ) + case wasm.OpcodeVecI8x16Abs: + c.emit( + newOperationV128Abs(shapeI8x16), + ) + case wasm.OpcodeVecI8x16Popcnt: + c.emit( + newOperationV128Popcnt(shapeI8x16), + ) + case wasm.OpcodeVecI16x8Abs: + c.emit( + newOperationV128Abs(shapeI16x8), + ) + case wasm.OpcodeVecI32x4Abs: + c.emit( + newOperationV128Abs(shapeI32x4), + ) + case wasm.OpcodeVecI64x2Abs: + c.emit( + newOperationV128Abs(shapeI64x2), + ) + case wasm.OpcodeVecF32x4Abs: + c.emit( + newOperationV128Abs(shapeF32x4), + ) + case wasm.OpcodeVecF64x2Abs: + c.emit( + newOperationV128Abs(shapeF64x2), + ) + case wasm.OpcodeVecI8x16MinS: + c.emit( + newOperationV128Min(shapeI8x16, true), + ) + case wasm.OpcodeVecI8x16MinU: + c.emit( + newOperationV128Min(shapeI8x16, false), + ) + case wasm.OpcodeVecI8x16MaxS: + c.emit( + newOperationV128Max(shapeI8x16, true), + ) + case wasm.OpcodeVecI8x16MaxU: + c.emit( + newOperationV128Max(shapeI8x16, false), + ) + case wasm.OpcodeVecI8x16AvgrU: + c.emit( + newOperationV128AvgrU(shapeI8x16), + ) + case wasm.OpcodeVecI16x8MinS: + c.emit( + newOperationV128Min(shapeI16x8, true), + ) + case wasm.OpcodeVecI16x8MinU: + c.emit( + newOperationV128Min(shapeI16x8, false), + ) + case wasm.OpcodeVecI16x8MaxS: + c.emit( + newOperationV128Max(shapeI16x8, true), + ) + case wasm.OpcodeVecI16x8MaxU: + c.emit( + newOperationV128Max(shapeI16x8, false), + ) + case wasm.OpcodeVecI16x8AvgrU: + c.emit( + newOperationV128AvgrU(shapeI16x8), + ) + case wasm.OpcodeVecI32x4MinS: + c.emit( + newOperationV128Min(shapeI32x4, true), + ) + case wasm.OpcodeVecI32x4MinU: + c.emit( + newOperationV128Min(shapeI32x4, false), + ) + case wasm.OpcodeVecI32x4MaxS: + c.emit( + newOperationV128Max(shapeI32x4, true), + ) + case wasm.OpcodeVecI32x4MaxU: + c.emit( + newOperationV128Max(shapeI32x4, false), + ) + case wasm.OpcodeVecF32x4Min: + c.emit( + newOperationV128Min(shapeF32x4, false), + ) + case wasm.OpcodeVecF32x4Max: + c.emit( + newOperationV128Max(shapeF32x4, false), + ) + case wasm.OpcodeVecF64x2Min: + c.emit( + newOperationV128Min(shapeF64x2, false), + ) + case wasm.OpcodeVecF64x2Max: + c.emit( + newOperationV128Max(shapeF64x2, false), + ) + case wasm.OpcodeVecF32x4Pmin: + c.emit( + newOperationV128Pmin(shapeF32x4), + ) + case wasm.OpcodeVecF32x4Pmax: + c.emit( + newOperationV128Pmax(shapeF32x4), + ) + case wasm.OpcodeVecF64x2Pmin: + c.emit( + newOperationV128Pmin(shapeF64x2), + ) + case wasm.OpcodeVecF64x2Pmax: + c.emit( + newOperationV128Pmax(shapeF64x2), + ) + case wasm.OpcodeVecF32x4Ceil: + c.emit( + newOperationV128Ceil(shapeF32x4), + ) + case wasm.OpcodeVecF32x4Floor: + c.emit( + newOperationV128Floor(shapeF32x4), + ) + case wasm.OpcodeVecF32x4Trunc: + c.emit( + newOperationV128Trunc(shapeF32x4), + ) + case wasm.OpcodeVecF32x4Nearest: + c.emit( + newOperationV128Nearest(shapeF32x4), + ) + case wasm.OpcodeVecF64x2Ceil: + c.emit( + newOperationV128Ceil(shapeF64x2), + ) + case wasm.OpcodeVecF64x2Floor: + c.emit( + newOperationV128Floor(shapeF64x2), + ) + case wasm.OpcodeVecF64x2Trunc: + c.emit( + newOperationV128Trunc(shapeF64x2), + ) + case wasm.OpcodeVecF64x2Nearest: + c.emit( + newOperationV128Nearest(shapeF64x2), + ) + case wasm.OpcodeVecI16x8ExtendLowI8x16S: + c.emit( + newOperationV128Extend(shapeI8x16, true, true), + ) + case wasm.OpcodeVecI16x8ExtendHighI8x16S: + c.emit( + newOperationV128Extend(shapeI8x16, true, false), + ) + case wasm.OpcodeVecI16x8ExtendLowI8x16U: + c.emit( + newOperationV128Extend(shapeI8x16, false, true), + ) + case wasm.OpcodeVecI16x8ExtendHighI8x16U: + c.emit( + newOperationV128Extend(shapeI8x16, false, false), + ) + case wasm.OpcodeVecI32x4ExtendLowI16x8S: + c.emit( + newOperationV128Extend(shapeI16x8, true, true), + ) + case wasm.OpcodeVecI32x4ExtendHighI16x8S: + c.emit( + newOperationV128Extend(shapeI16x8, true, false), + ) + case wasm.OpcodeVecI32x4ExtendLowI16x8U: + c.emit( + newOperationV128Extend(shapeI16x8, false, true), + ) + case wasm.OpcodeVecI32x4ExtendHighI16x8U: + c.emit( + newOperationV128Extend(shapeI16x8, false, false), + ) + case wasm.OpcodeVecI64x2ExtendLowI32x4S: + c.emit( + newOperationV128Extend(shapeI32x4, true, true), + ) + case wasm.OpcodeVecI64x2ExtendHighI32x4S: + c.emit( + newOperationV128Extend(shapeI32x4, true, false), + ) + case wasm.OpcodeVecI64x2ExtendLowI32x4U: + c.emit( + newOperationV128Extend(shapeI32x4, false, true), + ) + case wasm.OpcodeVecI64x2ExtendHighI32x4U: + c.emit( + newOperationV128Extend(shapeI32x4, false, false), + ) + case wasm.OpcodeVecI16x8Q15mulrSatS: + c.emit( + newOperationV128Q15mulrSatS(), + ) + case wasm.OpcodeVecI16x8ExtMulLowI8x16S: + c.emit( + newOperationV128ExtMul(shapeI8x16, true, true), + ) + case wasm.OpcodeVecI16x8ExtMulHighI8x16S: + c.emit( + newOperationV128ExtMul(shapeI8x16, true, false), + ) + case wasm.OpcodeVecI16x8ExtMulLowI8x16U: + c.emit( + newOperationV128ExtMul(shapeI8x16, false, true), + ) + case wasm.OpcodeVecI16x8ExtMulHighI8x16U: + c.emit( + newOperationV128ExtMul(shapeI8x16, false, false), + ) + case wasm.OpcodeVecI32x4ExtMulLowI16x8S: + c.emit( + newOperationV128ExtMul(shapeI16x8, true, true), + ) + case wasm.OpcodeVecI32x4ExtMulHighI16x8S: + c.emit( + newOperationV128ExtMul(shapeI16x8, true, false), + ) + case wasm.OpcodeVecI32x4ExtMulLowI16x8U: + c.emit( + newOperationV128ExtMul(shapeI16x8, false, true), + ) + case wasm.OpcodeVecI32x4ExtMulHighI16x8U: + c.emit( + newOperationV128ExtMul(shapeI16x8, false, false), + ) + case wasm.OpcodeVecI64x2ExtMulLowI32x4S: + c.emit( + newOperationV128ExtMul(shapeI32x4, true, true), + ) + case wasm.OpcodeVecI64x2ExtMulHighI32x4S: + c.emit( + newOperationV128ExtMul(shapeI32x4, true, false), + ) + case wasm.OpcodeVecI64x2ExtMulLowI32x4U: + c.emit( + newOperationV128ExtMul(shapeI32x4, false, true), + ) + case wasm.OpcodeVecI64x2ExtMulHighI32x4U: + c.emit( + newOperationV128ExtMul(shapeI32x4, false, false), + ) + case wasm.OpcodeVecI16x8ExtaddPairwiseI8x16S: + c.emit( + newOperationV128ExtAddPairwise(shapeI8x16, true), + ) + case wasm.OpcodeVecI16x8ExtaddPairwiseI8x16U: + c.emit( + newOperationV128ExtAddPairwise(shapeI8x16, false), + ) + case wasm.OpcodeVecI32x4ExtaddPairwiseI16x8S: + c.emit( + newOperationV128ExtAddPairwise(shapeI16x8, true), + ) + case wasm.OpcodeVecI32x4ExtaddPairwiseI16x8U: + c.emit( + newOperationV128ExtAddPairwise(shapeI16x8, false), + ) + case wasm.OpcodeVecF64x2PromoteLowF32x4Zero: + c.emit( + newOperationV128FloatPromote(), + ) + case wasm.OpcodeVecF32x4DemoteF64x2Zero: + c.emit( + newOperationV128FloatDemote(), + ) + case wasm.OpcodeVecF32x4ConvertI32x4S: + c.emit( + newOperationV128FConvertFromI(shapeF32x4, true), + ) + case wasm.OpcodeVecF32x4ConvertI32x4U: + c.emit( + newOperationV128FConvertFromI(shapeF32x4, false), + ) + case wasm.OpcodeVecF64x2ConvertLowI32x4S: + c.emit( + newOperationV128FConvertFromI(shapeF64x2, true), + ) + case wasm.OpcodeVecF64x2ConvertLowI32x4U: + c.emit( + newOperationV128FConvertFromI(shapeF64x2, false), + ) + case wasm.OpcodeVecI32x4DotI16x8S: + c.emit( + newOperationV128Dot(), + ) + case wasm.OpcodeVecI8x16NarrowI16x8S: + c.emit( + newOperationV128Narrow(shapeI16x8, true), + ) + case wasm.OpcodeVecI8x16NarrowI16x8U: + c.emit( + newOperationV128Narrow(shapeI16x8, false), + ) + case wasm.OpcodeVecI16x8NarrowI32x4S: + c.emit( + newOperationV128Narrow(shapeI32x4, true), + ) + case wasm.OpcodeVecI16x8NarrowI32x4U: + c.emit( + newOperationV128Narrow(shapeI32x4, false), + ) + case wasm.OpcodeVecI32x4TruncSatF32x4S: + c.emit( + newOperationV128ITruncSatFromF(shapeF32x4, true), + ) + case wasm.OpcodeVecI32x4TruncSatF32x4U: + c.emit( + newOperationV128ITruncSatFromF(shapeF32x4, false), + ) + case wasm.OpcodeVecI32x4TruncSatF64x2SZero: + c.emit( + newOperationV128ITruncSatFromF(shapeF64x2, true), + ) + case wasm.OpcodeVecI32x4TruncSatF64x2UZero: + c.emit( + newOperationV128ITruncSatFromF(shapeF64x2, false), + ) + default: + return fmt.Errorf("unsupported vector instruction in interpreterir: %s", wasm.VectorInstructionName(vecOp)) + } + case wasm.OpcodeAtomicPrefix: + c.pc++ + atomicOp := c.body[c.pc] + switch atomicOp { + case wasm.OpcodeAtomicMemoryWait32: + imm, err := c.readMemoryArg(wasm.OpcodeAtomicMemoryWait32Name) + if err != nil { + return err + } + c.emit( + newOperationAtomicMemoryWait(unsignedTypeI32, imm), + ) + case wasm.OpcodeAtomicMemoryWait64: + imm, err := c.readMemoryArg(wasm.OpcodeAtomicMemoryWait64Name) + if err != nil { + return err + } + c.emit( + newOperationAtomicMemoryWait(unsignedTypeI64, imm), + ) + case wasm.OpcodeAtomicMemoryNotify: + imm, err := c.readMemoryArg(wasm.OpcodeAtomicMemoryNotifyName) + if err != nil { + return err + } + c.emit( + newOperationAtomicMemoryNotify(imm), + ) + case wasm.OpcodeAtomicFence: + // Skip immediate value + c.pc++ + _ = c.body[c.pc] + c.emit( + newOperationAtomicFence(), + ) + case wasm.OpcodeAtomicI32Load: + imm, err := c.readMemoryArg(wasm.OpcodeAtomicI32LoadName) + if err != nil { + return err + } + c.emit( + newOperationAtomicLoad(unsignedTypeI32, imm), + ) + case wasm.OpcodeAtomicI64Load: + imm, err := c.readMemoryArg(wasm.OpcodeAtomicI64LoadName) + if err != nil { + return err + } + c.emit( + newOperationAtomicLoad(unsignedTypeI64, imm), + ) + case wasm.OpcodeAtomicI32Load8U: + imm, err := c.readMemoryArg(wasm.OpcodeAtomicI32Load8UName) + if err != nil { + return err + } + c.emit( + newOperationAtomicLoad8(unsignedTypeI32, imm), + ) + case wasm.OpcodeAtomicI32Load16U: + imm, err := c.readMemoryArg(wasm.OpcodeAtomicI32Load16UName) + if err != nil { + return err + } + c.emit( + newOperationAtomicLoad16(unsignedTypeI32, imm), + ) + case wasm.OpcodeAtomicI64Load8U: + imm, err := c.readMemoryArg(wasm.OpcodeAtomicI64Load8UName) + if err != nil { + return err + } + c.emit( + newOperationAtomicLoad8(unsignedTypeI64, imm), + ) + case wasm.OpcodeAtomicI64Load16U: + imm, err := c.readMemoryArg(wasm.OpcodeAtomicI64Load16UName) + if err != nil { + return err + } + c.emit( + newOperationAtomicLoad16(unsignedTypeI64, imm), + ) + case wasm.OpcodeAtomicI64Load32U: + imm, err := c.readMemoryArg(wasm.OpcodeAtomicI64Load32UName) + if err != nil { + return err + } + c.emit( + newOperationAtomicLoad(unsignedTypeI32, imm), + ) + case wasm.OpcodeAtomicI32Store: + imm, err := c.readMemoryArg(wasm.OpcodeAtomicI32StoreName) + if err != nil { + return err + } + c.emit( + newOperationAtomicStore(unsignedTypeI32, imm), + ) + case wasm.OpcodeAtomicI32Store8: + imm, err := c.readMemoryArg(wasm.OpcodeAtomicI32Store8Name) + if err != nil { + return err + } + c.emit( + newOperationAtomicStore8(unsignedTypeI32, imm), + ) + case wasm.OpcodeAtomicI32Store16: + imm, err := c.readMemoryArg(wasm.OpcodeAtomicI32Store16Name) + if err != nil { + return err + } + c.emit( + newOperationAtomicStore16(unsignedTypeI32, imm), + ) + case wasm.OpcodeAtomicI64Store: + imm, err := c.readMemoryArg(wasm.OpcodeAtomicI64StoreName) + if err != nil { + return err + } + c.emit( + newOperationAtomicStore(unsignedTypeI64, imm), + ) + case wasm.OpcodeAtomicI64Store8: + imm, err := c.readMemoryArg(wasm.OpcodeAtomicI64Store8Name) + if err != nil { + return err + } + c.emit( + newOperationAtomicStore8(unsignedTypeI64, imm), + ) + case wasm.OpcodeAtomicI64Store16: + imm, err := c.readMemoryArg(wasm.OpcodeAtomicI64Store16Name) + if err != nil { + return err + } + c.emit( + newOperationAtomicStore16(unsignedTypeI64, imm), + ) + case wasm.OpcodeAtomicI64Store32: + imm, err := c.readMemoryArg(wasm.OpcodeAtomicI64Store32Name) + if err != nil { + return err + } + c.emit( + newOperationAtomicStore(unsignedTypeI32, imm), + ) + case wasm.OpcodeAtomicI32RmwAdd: + imm, err := c.readMemoryArg(wasm.OpcodeAtomicI32RmwAddName) + if err != nil { + return err + } + c.emit( + newOperationAtomicRMW(unsignedTypeI32, imm, atomicArithmeticOpAdd), + ) + case wasm.OpcodeAtomicI64RmwAdd: + imm, err := c.readMemoryArg(wasm.OpcodeAtomicI64RmwAddName) + if err != nil { + return err + } + c.emit( + newOperationAtomicRMW(unsignedTypeI64, imm, atomicArithmeticOpAdd), + ) + case wasm.OpcodeAtomicI32Rmw8AddU: + imm, err := c.readMemoryArg(wasm.OpcodeAtomicI32Rmw8AddUName) + if err != nil { + return err + } + c.emit( + newOperationAtomicRMW8(unsignedTypeI32, imm, atomicArithmeticOpAdd), + ) + case wasm.OpcodeAtomicI64Rmw8AddU: + imm, err := c.readMemoryArg(wasm.OpcodeAtomicI64Rmw8AddUName) + if err != nil { + return err + } + c.emit( + newOperationAtomicRMW8(unsignedTypeI64, imm, atomicArithmeticOpAdd), + ) + case wasm.OpcodeAtomicI32Rmw16AddU: + imm, err := c.readMemoryArg(wasm.OpcodeAtomicI32Rmw16AddUName) + if err != nil { + return err + } + c.emit( + newOperationAtomicRMW16(unsignedTypeI32, imm, atomicArithmeticOpAdd), + ) + case wasm.OpcodeAtomicI64Rmw16AddU: + imm, err := c.readMemoryArg(wasm.OpcodeAtomicI64Rmw16AddUName) + if err != nil { + return err + } + c.emit( + newOperationAtomicRMW16(unsignedTypeI64, imm, atomicArithmeticOpAdd), + ) + case wasm.OpcodeAtomicI64Rmw32AddU: + imm, err := c.readMemoryArg(wasm.OpcodeAtomicI64Rmw32AddUName) + if err != nil { + return err + } + c.emit( + newOperationAtomicRMW(unsignedTypeI32, imm, atomicArithmeticOpAdd), + ) + case wasm.OpcodeAtomicI32RmwSub: + imm, err := c.readMemoryArg(wasm.OpcodeAtomicI32RmwSubName) + if err != nil { + return err + } + c.emit( + newOperationAtomicRMW(unsignedTypeI32, imm, atomicArithmeticOpSub), + ) + case wasm.OpcodeAtomicI64RmwSub: + imm, err := c.readMemoryArg(wasm.OpcodeAtomicI64RmwSubName) + if err != nil { + return err + } + c.emit( + newOperationAtomicRMW(unsignedTypeI64, imm, atomicArithmeticOpSub), + ) + case wasm.OpcodeAtomicI32Rmw8SubU: + imm, err := c.readMemoryArg(wasm.OpcodeAtomicI32Rmw8SubUName) + if err != nil { + return err + } + c.emit( + newOperationAtomicRMW8(unsignedTypeI32, imm, atomicArithmeticOpSub), + ) + case wasm.OpcodeAtomicI64Rmw8SubU: + imm, err := c.readMemoryArg(wasm.OpcodeAtomicI64Rmw8SubUName) + if err != nil { + return err + } + c.emit( + newOperationAtomicRMW8(unsignedTypeI64, imm, atomicArithmeticOpSub), + ) + case wasm.OpcodeAtomicI32Rmw16SubU: + imm, err := c.readMemoryArg(wasm.OpcodeAtomicI32Rmw16SubUName) + if err != nil { + return err + } + c.emit( + newOperationAtomicRMW16(unsignedTypeI32, imm, atomicArithmeticOpSub), + ) + case wasm.OpcodeAtomicI64Rmw16SubU: + imm, err := c.readMemoryArg(wasm.OpcodeAtomicI64Rmw16SubUName) + if err != nil { + return err + } + c.emit( + newOperationAtomicRMW16(unsignedTypeI64, imm, atomicArithmeticOpSub), + ) + case wasm.OpcodeAtomicI64Rmw32SubU: + imm, err := c.readMemoryArg(wasm.OpcodeAtomicI64Rmw32SubUName) + if err != nil { + return err + } + c.emit( + newOperationAtomicRMW(unsignedTypeI32, imm, atomicArithmeticOpSub), + ) + case wasm.OpcodeAtomicI32RmwAnd: + imm, err := c.readMemoryArg(wasm.OpcodeAtomicI32RmwAndName) + if err != nil { + return err + } + c.emit( + newOperationAtomicRMW(unsignedTypeI32, imm, atomicArithmeticOpAnd), + ) + case wasm.OpcodeAtomicI64RmwAnd: + imm, err := c.readMemoryArg(wasm.OpcodeAtomicI64RmwAndName) + if err != nil { + return err + } + c.emit( + newOperationAtomicRMW(unsignedTypeI64, imm, atomicArithmeticOpAnd), + ) + case wasm.OpcodeAtomicI32Rmw8AndU: + imm, err := c.readMemoryArg(wasm.OpcodeAtomicI32Rmw8AndUName) + if err != nil { + return err + } + c.emit( + newOperationAtomicRMW8(unsignedTypeI32, imm, atomicArithmeticOpAnd), + ) + case wasm.OpcodeAtomicI64Rmw8AndU: + imm, err := c.readMemoryArg(wasm.OpcodeAtomicI64Rmw8AndUName) + if err != nil { + return err + } + c.emit( + newOperationAtomicRMW8(unsignedTypeI64, imm, atomicArithmeticOpAnd), + ) + case wasm.OpcodeAtomicI32Rmw16AndU: + imm, err := c.readMemoryArg(wasm.OpcodeAtomicI32Rmw16AndUName) + if err != nil { + return err + } + c.emit( + newOperationAtomicRMW16(unsignedTypeI32, imm, atomicArithmeticOpAnd), + ) + case wasm.OpcodeAtomicI64Rmw16AndU: + imm, err := c.readMemoryArg(wasm.OpcodeAtomicI64Rmw16AndUName) + if err != nil { + return err + } + c.emit( + newOperationAtomicRMW16(unsignedTypeI64, imm, atomicArithmeticOpAnd), + ) + case wasm.OpcodeAtomicI64Rmw32AndU: + imm, err := c.readMemoryArg(wasm.OpcodeAtomicI64Rmw32AndUName) + if err != nil { + return err + } + c.emit( + newOperationAtomicRMW(unsignedTypeI32, imm, atomicArithmeticOpAnd), + ) + case wasm.OpcodeAtomicI32RmwOr: + imm, err := c.readMemoryArg(wasm.OpcodeAtomicI32RmwOrName) + if err != nil { + return err + } + c.emit( + newOperationAtomicRMW(unsignedTypeI32, imm, atomicArithmeticOpOr), + ) + case wasm.OpcodeAtomicI64RmwOr: + imm, err := c.readMemoryArg(wasm.OpcodeAtomicI64RmwOrName) + if err != nil { + return err + } + c.emit( + newOperationAtomicRMW(unsignedTypeI64, imm, atomicArithmeticOpOr), + ) + case wasm.OpcodeAtomicI32Rmw8OrU: + imm, err := c.readMemoryArg(wasm.OpcodeAtomicI32Rmw8OrUName) + if err != nil { + return err + } + c.emit( + newOperationAtomicRMW8(unsignedTypeI32, imm, atomicArithmeticOpOr), + ) + case wasm.OpcodeAtomicI64Rmw8OrU: + imm, err := c.readMemoryArg(wasm.OpcodeAtomicI64Rmw8OrUName) + if err != nil { + return err + } + c.emit( + newOperationAtomicRMW8(unsignedTypeI64, imm, atomicArithmeticOpOr), + ) + case wasm.OpcodeAtomicI32Rmw16OrU: + imm, err := c.readMemoryArg(wasm.OpcodeAtomicI32Rmw16OrUName) + if err != nil { + return err + } + c.emit( + newOperationAtomicRMW16(unsignedTypeI32, imm, atomicArithmeticOpOr), + ) + case wasm.OpcodeAtomicI64Rmw16OrU: + imm, err := c.readMemoryArg(wasm.OpcodeAtomicI64Rmw16OrUName) + if err != nil { + return err + } + c.emit( + newOperationAtomicRMW16(unsignedTypeI64, imm, atomicArithmeticOpOr), + ) + case wasm.OpcodeAtomicI64Rmw32OrU: + imm, err := c.readMemoryArg(wasm.OpcodeAtomicI64Rmw32OrUName) + if err != nil { + return err + } + c.emit( + newOperationAtomicRMW(unsignedTypeI32, imm, atomicArithmeticOpOr), + ) + case wasm.OpcodeAtomicI32RmwXor: + imm, err := c.readMemoryArg(wasm.OpcodeAtomicI32RmwXorName) + if err != nil { + return err + } + c.emit( + newOperationAtomicRMW(unsignedTypeI32, imm, atomicArithmeticOpXor), + ) + case wasm.OpcodeAtomicI64RmwXor: + imm, err := c.readMemoryArg(wasm.OpcodeAtomicI64RmwXorName) + if err != nil { + return err + } + c.emit( + newOperationAtomicRMW(unsignedTypeI64, imm, atomicArithmeticOpXor), + ) + case wasm.OpcodeAtomicI32Rmw8XorU: + imm, err := c.readMemoryArg(wasm.OpcodeAtomicI32Rmw8XorUName) + if err != nil { + return err + } + c.emit( + newOperationAtomicRMW8(unsignedTypeI32, imm, atomicArithmeticOpXor), + ) + case wasm.OpcodeAtomicI64Rmw8XorU: + imm, err := c.readMemoryArg(wasm.OpcodeAtomicI64Rmw8XorUName) + if err != nil { + return err + } + c.emit( + newOperationAtomicRMW8(unsignedTypeI64, imm, atomicArithmeticOpXor), + ) + case wasm.OpcodeAtomicI32Rmw16XorU: + imm, err := c.readMemoryArg(wasm.OpcodeAtomicI32Rmw16XorUName) + if err != nil { + return err + } + c.emit( + newOperationAtomicRMW16(unsignedTypeI32, imm, atomicArithmeticOpXor), + ) + case wasm.OpcodeAtomicI64Rmw16XorU: + imm, err := c.readMemoryArg(wasm.OpcodeAtomicI64Rmw16XorUName) + if err != nil { + return err + } + c.emit( + newOperationAtomicRMW16(unsignedTypeI64, imm, atomicArithmeticOpXor), + ) + case wasm.OpcodeAtomicI64Rmw32XorU: + imm, err := c.readMemoryArg(wasm.OpcodeAtomicI64Rmw32XorUName) + if err != nil { + return err + } + c.emit( + newOperationAtomicRMW(unsignedTypeI32, imm, atomicArithmeticOpXor), + ) + case wasm.OpcodeAtomicI32RmwXchg: + imm, err := c.readMemoryArg(wasm.OpcodeAtomicI32RmwXchgName) + if err != nil { + return err + } + c.emit( + newOperationAtomicRMW(unsignedTypeI32, imm, atomicArithmeticOpNop), + ) + case wasm.OpcodeAtomicI64RmwXchg: + imm, err := c.readMemoryArg(wasm.OpcodeAtomicI64RmwXchgName) + if err != nil { + return err + } + c.emit( + newOperationAtomicRMW(unsignedTypeI64, imm, atomicArithmeticOpNop), + ) + case wasm.OpcodeAtomicI32Rmw8XchgU: + imm, err := c.readMemoryArg(wasm.OpcodeAtomicI32Rmw8XchgUName) + if err != nil { + return err + } + c.emit( + newOperationAtomicRMW8(unsignedTypeI32, imm, atomicArithmeticOpNop), + ) + case wasm.OpcodeAtomicI64Rmw8XchgU: + imm, err := c.readMemoryArg(wasm.OpcodeAtomicI64Rmw8XchgUName) + if err != nil { + return err + } + c.emit( + newOperationAtomicRMW8(unsignedTypeI64, imm, atomicArithmeticOpNop), + ) + case wasm.OpcodeAtomicI32Rmw16XchgU: + imm, err := c.readMemoryArg(wasm.OpcodeAtomicI32Rmw16XchgUName) + if err != nil { + return err + } + c.emit( + newOperationAtomicRMW16(unsignedTypeI32, imm, atomicArithmeticOpNop), + ) + case wasm.OpcodeAtomicI64Rmw16XchgU: + imm, err := c.readMemoryArg(wasm.OpcodeAtomicI64Rmw16XchgUName) + if err != nil { + return err + } + c.emit( + newOperationAtomicRMW16(unsignedTypeI64, imm, atomicArithmeticOpNop), + ) + case wasm.OpcodeAtomicI64Rmw32XchgU: + imm, err := c.readMemoryArg(wasm.OpcodeAtomicI64Rmw32XchgUName) + if err != nil { + return err + } + c.emit( + newOperationAtomicRMW(unsignedTypeI32, imm, atomicArithmeticOpNop), + ) + case wasm.OpcodeAtomicI32RmwCmpxchg: + imm, err := c.readMemoryArg(wasm.OpcodeAtomicI32RmwCmpxchgName) + if err != nil { + return err + } + c.emit( + newOperationAtomicRMWCmpxchg(unsignedTypeI32, imm), + ) + case wasm.OpcodeAtomicI64RmwCmpxchg: + imm, err := c.readMemoryArg(wasm.OpcodeAtomicI64RmwCmpxchgName) + if err != nil { + return err + } + c.emit( + newOperationAtomicRMWCmpxchg(unsignedTypeI64, imm), + ) + case wasm.OpcodeAtomicI32Rmw8CmpxchgU: + imm, err := c.readMemoryArg(wasm.OpcodeAtomicI32Rmw8CmpxchgUName) + if err != nil { + return err + } + c.emit( + newOperationAtomicRMW8Cmpxchg(unsignedTypeI32, imm), + ) + case wasm.OpcodeAtomicI64Rmw8CmpxchgU: + imm, err := c.readMemoryArg(wasm.OpcodeAtomicI64Rmw8CmpxchgUName) + if err != nil { + return err + } + c.emit( + newOperationAtomicRMW8Cmpxchg(unsignedTypeI64, imm), + ) + case wasm.OpcodeAtomicI32Rmw16CmpxchgU: + imm, err := c.readMemoryArg(wasm.OpcodeAtomicI32Rmw16CmpxchgUName) + if err != nil { + return err + } + c.emit( + newOperationAtomicRMW16Cmpxchg(unsignedTypeI32, imm), + ) + case wasm.OpcodeAtomicI64Rmw16CmpxchgU: + imm, err := c.readMemoryArg(wasm.OpcodeAtomicI64Rmw16CmpxchgUName) + if err != nil { + return err + } + c.emit( + newOperationAtomicRMW16Cmpxchg(unsignedTypeI64, imm), + ) + case wasm.OpcodeAtomicI64Rmw32CmpxchgU: + imm, err := c.readMemoryArg(wasm.OpcodeAtomicI64Rmw32CmpxchgUName) + if err != nil { + return err + } + c.emit( + newOperationAtomicRMWCmpxchg(unsignedTypeI32, imm), + ) + default: + return fmt.Errorf("unsupported atomic instruction in interpreterir: %s", wasm.AtomicInstructionName(atomicOp)) + } + default: + return fmt.Errorf("unsupported instruction in interpreterir: 0x%x", op) + } + + // Move the program counter to point to the next instruction. + c.pc++ + return nil +} + +func (c *compiler) nextFrameID() (id uint32) { + id = c.currentFrameID + 1 + c.currentFrameID++ + return +} + +func (c *compiler) applyToStack(opcode wasm.Opcode) (index uint32, err error) { + switch opcode { + case + // These are the opcodes that is coupled with "index" immediate + // and it DOES affect the signature of opcode. + wasm.OpcodeCall, + wasm.OpcodeCallIndirect, + wasm.OpcodeLocalGet, + wasm.OpcodeLocalSet, + wasm.OpcodeLocalTee, + wasm.OpcodeGlobalGet, + wasm.OpcodeGlobalSet: + // Assumes that we are at the opcode now so skip it before read immediates. + v, num, err := leb128.LoadUint32(c.body[c.pc+1:]) + if err != nil { + return 0, fmt.Errorf("reading immediates: %w", err) + } + c.pc += num + index = v + default: + // Note that other opcodes are free of index + // as it doesn't affect the signature of opt code. + // In other words, the "index" argument of wasmOpcodeSignature + // is ignored there. + } + + if c.unreachableState.on { + return 0, nil + } + + // Retrieve the signature of the opcode. + s, err := c.wasmOpcodeSignature(opcode, index) + if err != nil { + return 0, err + } + + // Manipulate the stack according to the signature. + // Note that the following algorithm assumes that + // the unknown type is unique in the signature, + // and is determined by the actual type on the stack. + // The determined type is stored in this typeParam. + var typeParam unsignedType + var typeParamFound bool + for i := range s.in { + want := s.in[len(s.in)-1-i] + actual := c.stackPop() + if want == unsignedTypeUnknown && typeParamFound { + want = typeParam + } else if want == unsignedTypeUnknown { + want = actual + typeParam = want + typeParamFound = true + } + if want != actual { + return 0, fmt.Errorf("input signature mismatch: want %s but have %s", want, actual) + } + } + + for _, target := range s.out { + if target == unsignedTypeUnknown && !typeParamFound { + return 0, fmt.Errorf("cannot determine type of unknown result") + } else if target == unsignedTypeUnknown { + c.stackPush(typeParam) + } else { + c.stackPush(target) + } + } + + return index, nil +} + +func (c *compiler) stackPeek() (ret unsignedType) { + ret = c.stack[len(c.stack)-1] + return +} + +func (c *compiler) stackPop() (ret unsignedType) { + // No need to check stack bound + // as we can assume that all the operations + // are valid thanks to validateFunction + // at module validation phase. + ret = c.stack[len(c.stack)-1] + c.stack = c.stack[:len(c.stack)-1] + return +} + +func (c *compiler) stackPush(ts unsignedType) { + c.stack = append(c.stack, ts) +} + +// emit adds the operations into the result. +func (c *compiler) emit(op unionOperation) { + if !c.unreachableState.on { + switch op.Kind { + case operationKindDrop: + // If the drop range is nil, + // we could remove such operations. + // That happens when drop operation is unnecessary. + // i.e. when there's no need to adjust stack before jmp. + if int64(op.U1) == -1 { + return + } + } + c.result.Operations = append(c.result.Operations, op) + if c.needSourceOffset { + c.result.IROperationSourceOffsetsInWasmBinary = append(c.result.IROperationSourceOffsetsInWasmBinary, + c.currentOpPC+c.bodyOffsetInCodeSection) + } + } +} + +// Emit const expression with default values of the given type. +func (c *compiler) emitDefaultValue(t wasm.ValueType) { + switch t { + case wasm.ValueTypeI32: + c.stackPush(unsignedTypeI32) + c.emit(newOperationConstI32(0)) + case wasm.ValueTypeI64, wasm.ValueTypeExternref, wasm.ValueTypeFuncref: + c.stackPush(unsignedTypeI64) + c.emit(newOperationConstI64(0)) + case wasm.ValueTypeF32: + c.stackPush(unsignedTypeF32) + c.emit(newOperationConstF32(0)) + case wasm.ValueTypeF64: + c.stackPush(unsignedTypeF64) + c.emit(newOperationConstF64(0)) + case wasm.ValueTypeV128: + c.stackPush(unsignedTypeV128) + c.emit(newOperationV128Const(0, 0)) + } +} + +// Returns the "depth" (starting from top of the stack) +// of the n-th local. +func (c *compiler) localDepth(index wasm.Index) int { + height := c.localIndexToStackHeightInUint64[index] + return c.stackLenInUint64(len(c.stack)) - 1 - int(height) +} + +func (c *compiler) localType(index wasm.Index) (t wasm.ValueType) { + if params := uint32(len(c.sig.Params)); index < params { + t = c.sig.Params[index] + } else { + t = c.localTypes[index-params] + } + return +} + +// getFrameDropRange returns the range (starting from top of the stack) that spans across the (uint64) stack. The range is +// supposed to be dropped from the stack when the given frame exists or branch into it. +// +// * frame is the control frame which the call-site is trying to branch into or exit. +// * isEnd true if the call-site is handling wasm.OpcodeEnd. +func (c *compiler) getFrameDropRange(frame *controlFrame, isEnd bool) inclusiveRange { + var start int + if !isEnd && frame.kind == controlFrameKindLoop { + // If this is not End and the call-site is trying to branch into the Loop control frame, + // we have to Start executing from the beginning of the loop block. + // Therefore, we have to pass the inputs to the frame. + start = frame.blockType.ParamNumInUint64 + } else { + start = frame.blockType.ResultNumInUint64 + } + var end int + if frame.kind == controlFrameKindFunction { + // On the function return, we eliminate all the contents on the stack + // including locals (existing below of frame.originalStackLen) + end = c.stackLenInUint64(len(c.stack)) - 1 + } else { + end = c.stackLenInUint64(len(c.stack)) - 1 - c.stackLenInUint64(frame.originalStackLenWithoutParam) + } + if start <= end { + return inclusiveRange{Start: int32(start), End: int32(end)} + } else { + return nopinclusiveRange + } +} + +func (c *compiler) stackLenInUint64(ceil int) (ret int) { + for i := 0; i < ceil; i++ { + if c.stack[i] == unsignedTypeV128 { + ret += 2 + } else { + ret++ + } + } + return +} + +func (c *compiler) readMemoryArg(tag string) (memoryArg, error) { + c.result.UsesMemory = true + alignment, num, err := leb128.LoadUint32(c.body[c.pc+1:]) + if err != nil { + return memoryArg{}, fmt.Errorf("reading alignment for %s: %w", tag, err) + } + c.pc += num + offset, num, err := leb128.LoadUint32(c.body[c.pc+1:]) + if err != nil { + return memoryArg{}, fmt.Errorf("reading offset for %s: %w", tag, err) + } + c.pc += num + return memoryArg{Offset: offset, Alignment: alignment}, nil +} diff --git a/vendor/github.com/tetratelabs/wazero/internal/engine/interpreter/format.go b/vendor/github.com/tetratelabs/wazero/internal/engine/interpreter/format.go new file mode 100644 index 000000000..8af1d94b0 --- /dev/null +++ b/vendor/github.com/tetratelabs/wazero/internal/engine/interpreter/format.go @@ -0,0 +1,22 @@ +package interpreter + +import ( + "bytes" +) + +func format(ops []unionOperation) string { + buf := bytes.NewBuffer(nil) + + _, _ = buf.WriteString(".entrypoint\n") + for i := range ops { + op := &ops[i] + str := op.String() + isLabel := op.Kind == operationKindLabel + if !isLabel { + const indent = "\t" + str = indent + str + } + _, _ = buf.WriteString(str + "\n") + } + return buf.String() +} diff --git a/vendor/github.com/tetratelabs/wazero/internal/engine/interpreter/interpreter.go b/vendor/github.com/tetratelabs/wazero/internal/engine/interpreter/interpreter.go new file mode 100644 index 000000000..a89ddc457 --- /dev/null +++ b/vendor/github.com/tetratelabs/wazero/internal/engine/interpreter/interpreter.go @@ -0,0 +1,4583 @@ +package interpreter + +import ( + "context" + "encoding/binary" + "errors" + "fmt" + "math" + "math/bits" + "sync" + "unsafe" + + "github.com/tetratelabs/wazero/api" + "github.com/tetratelabs/wazero/experimental" + "github.com/tetratelabs/wazero/internal/expctxkeys" + "github.com/tetratelabs/wazero/internal/filecache" + "github.com/tetratelabs/wazero/internal/internalapi" + "github.com/tetratelabs/wazero/internal/moremath" + "github.com/tetratelabs/wazero/internal/wasm" + "github.com/tetratelabs/wazero/internal/wasmdebug" + "github.com/tetratelabs/wazero/internal/wasmruntime" +) + +// callStackCeiling is the maximum WebAssembly call frame stack height. This allows wazero to raise +// wasm.ErrCallStackOverflow instead of overflowing the Go runtime. +// +// The default value should suffice for most use cases. Those wishing to change this can via `go build -ldflags`. +var callStackCeiling = 2000 + +// engine is an interpreter implementation of wasm.Engine +type engine struct { + enabledFeatures api.CoreFeatures + compiledFunctions map[wasm.ModuleID][]compiledFunction // guarded by mutex. + mux sync.RWMutex +} + +func NewEngine(_ context.Context, enabledFeatures api.CoreFeatures, _ filecache.Cache) wasm.Engine { + return &engine{ + enabledFeatures: enabledFeatures, + compiledFunctions: map[wasm.ModuleID][]compiledFunction{}, + } +} + +// Close implements the same method as documented on wasm.Engine. +func (e *engine) Close() (err error) { + return +} + +// CompiledModuleCount implements the same method as documented on wasm.Engine. +func (e *engine) CompiledModuleCount() uint32 { + return uint32(len(e.compiledFunctions)) +} + +// DeleteCompiledModule implements the same method as documented on wasm.Engine. +func (e *engine) DeleteCompiledModule(m *wasm.Module) { + e.deleteCompiledFunctions(m) +} + +func (e *engine) deleteCompiledFunctions(module *wasm.Module) { + e.mux.Lock() + defer e.mux.Unlock() + delete(e.compiledFunctions, module.ID) +} + +func (e *engine) addCompiledFunctions(module *wasm.Module, fs []compiledFunction) { + e.mux.Lock() + defer e.mux.Unlock() + e.compiledFunctions[module.ID] = fs +} + +func (e *engine) getCompiledFunctions(module *wasm.Module) (fs []compiledFunction, ok bool) { + e.mux.RLock() + defer e.mux.RUnlock() + fs, ok = e.compiledFunctions[module.ID] + return +} + +// moduleEngine implements wasm.ModuleEngine +type moduleEngine struct { + // codes are the compiled functions in a module instances. + // The index is module instance-scoped. + functions []function + + // parentEngine holds *engine from which this module engine is created from. + parentEngine *engine +} + +// GetGlobalValue implements the same method as documented on wasm.ModuleEngine. +func (e *moduleEngine) GetGlobalValue(wasm.Index) (lo, hi uint64) { + panic("BUG: GetGlobalValue should never be called on interpreter mode") +} + +// SetGlobalValue implements the same method as documented on wasm.ModuleEngine. +func (e *moduleEngine) SetGlobalValue(idx wasm.Index, lo, hi uint64) { + panic("BUG: SetGlobalValue should never be called on interpreter mode") +} + +// OwnsGlobals implements the same method as documented on wasm.ModuleEngine. +func (e *moduleEngine) OwnsGlobals() bool { return false } + +// callEngine holds context per moduleEngine.Call, and shared across all the +// function calls originating from the same moduleEngine.Call execution. +// +// This implements api.Function. +type callEngine struct { + internalapi.WazeroOnlyType + + // stack contains the operands. + // Note that all the values are represented as uint64. + stack []uint64 + + // frames are the function call stack. + frames []*callFrame + + // f is the initial function for this call engine. + f *function + + // stackiterator for Listeners to walk frames and stack. + stackIterator stackIterator +} + +func (e *moduleEngine) newCallEngine(compiled *function) *callEngine { + return &callEngine{f: compiled} +} + +func (ce *callEngine) pushValue(v uint64) { + ce.stack = append(ce.stack, v) +} + +func (ce *callEngine) pushValues(v []uint64) { + ce.stack = append(ce.stack, v...) +} + +func (ce *callEngine) popValue() (v uint64) { + // No need to check stack bound + // as we can assume that all the operations + // are valid thanks to validateFunction + // at module validation phase + // and interpreterir translation + // before compilation. + stackTopIndex := len(ce.stack) - 1 + v = ce.stack[stackTopIndex] + ce.stack = ce.stack[:stackTopIndex] + return +} + +func (ce *callEngine) popValues(v []uint64) { + stackTopIndex := len(ce.stack) - len(v) + copy(v, ce.stack[stackTopIndex:]) + ce.stack = ce.stack[:stackTopIndex] +} + +// peekValues peeks api.ValueType values from the stack and returns them. +func (ce *callEngine) peekValues(count int) []uint64 { + if count == 0 { + return nil + } + stackLen := len(ce.stack) + return ce.stack[stackLen-count : stackLen] +} + +func (ce *callEngine) drop(raw uint64) { + r := inclusiveRangeFromU64(raw) + if r.Start == -1 { + return + } else if r.Start == 0 { + ce.stack = ce.stack[:int32(len(ce.stack))-1-r.End] + } else { + newStack := ce.stack[:int32(len(ce.stack))-1-r.End] + newStack = append(newStack, ce.stack[int32(len(ce.stack))-r.Start:]...) + ce.stack = newStack + } +} + +func (ce *callEngine) pushFrame(frame *callFrame) { + if callStackCeiling <= len(ce.frames) { + panic(wasmruntime.ErrRuntimeStackOverflow) + } + ce.frames = append(ce.frames, frame) +} + +func (ce *callEngine) popFrame() (frame *callFrame) { + // No need to check stack bound as we can assume that all the operations are valid thanks to validateFunction at + // module validation phase and interpreterir translation before compilation. + oneLess := len(ce.frames) - 1 + frame = ce.frames[oneLess] + ce.frames = ce.frames[:oneLess] + return +} + +type callFrame struct { + // pc is the program counter representing the current position in code.body. + pc uint64 + // f is the compiled function used in this function frame. + f *function + // base index in the frame of this function, used to detect the count of + // values on the stack. + base int +} + +type compiledFunction struct { + source *wasm.Module + body []unionOperation + listener experimental.FunctionListener + offsetsInWasmBinary []uint64 + hostFn interface{} + ensureTermination bool + index wasm.Index +} + +type function struct { + funcType *wasm.FunctionType + moduleInstance *wasm.ModuleInstance + typeID wasm.FunctionTypeID + parent *compiledFunction +} + +// functionFromUintptr resurrects the original *function from the given uintptr +// which comes from either funcref table or OpcodeRefFunc instruction. +func functionFromUintptr(ptr uintptr) *function { + // Wraps ptrs as the double pointer in order to avoid the unsafe access as detected by race detector. + // + // For example, if we have (*function)(unsafe.Pointer(ptr)) instead, then the race detector's "checkptr" + // subroutine wanrs as "checkptr: pointer arithmetic result points to invalid allocation" + // https://github.com/golang/go/blob/1ce7fcf139417d618c2730010ede2afb41664211/src/runtime/checkptr.go#L69 + var wrapped *uintptr = &ptr + return *(**function)(unsafe.Pointer(wrapped)) +} + +type snapshot struct { + stack []uint64 + frames []*callFrame + pc uint64 + + ret []uint64 + + ce *callEngine +} + +// Snapshot implements the same method as documented on experimental.Snapshotter. +func (ce *callEngine) Snapshot() experimental.Snapshot { + stack := make([]uint64, len(ce.stack)) + copy(stack, ce.stack) + + frames := make([]*callFrame, len(ce.frames)) + copy(frames, ce.frames) + + return &snapshot{ + stack: stack, + frames: frames, + ce: ce, + } +} + +// Restore implements the same method as documented on experimental.Snapshot. +func (s *snapshot) Restore(ret []uint64) { + s.ret = ret + panic(s) +} + +func (s *snapshot) doRestore() { + ce := s.ce + + ce.stack = s.stack + ce.frames = s.frames + ce.frames[len(ce.frames)-1].pc = s.pc + + copy(ce.stack[len(ce.stack)-len(s.ret):], s.ret) +} + +// Error implements the same method on error. +func (s *snapshot) Error() string { + return "unhandled snapshot restore, this generally indicates restore was called from a different " + + "exported function invocation than snapshot" +} + +// stackIterator implements experimental.StackIterator. +type stackIterator struct { + stack []uint64 + frames []*callFrame + started bool + fn *function + pc uint64 +} + +func (si *stackIterator) reset(stack []uint64, frames []*callFrame, f *function) { + si.fn = f + si.pc = 0 + si.stack = stack + si.frames = frames + si.started = false +} + +func (si *stackIterator) clear() { + si.stack = nil + si.frames = nil + si.started = false + si.fn = nil +} + +// Next implements the same method as documented on experimental.StackIterator. +func (si *stackIterator) Next() bool { + if !si.started { + si.started = true + return true + } + + if len(si.frames) == 0 { + return false + } + + frame := si.frames[len(si.frames)-1] + si.stack = si.stack[:frame.base] + si.fn = frame.f + si.pc = frame.pc + si.frames = si.frames[:len(si.frames)-1] + return true +} + +// Function implements the same method as documented on +// experimental.StackIterator. +func (si *stackIterator) Function() experimental.InternalFunction { + return internalFunction{si.fn} +} + +// ProgramCounter implements the same method as documented on +// experimental.StackIterator. +func (si *stackIterator) ProgramCounter() experimental.ProgramCounter { + return experimental.ProgramCounter(si.pc) +} + +// internalFunction implements experimental.InternalFunction. +type internalFunction struct{ *function } + +// Definition implements the same method as documented on +// experimental.InternalFunction. +func (f internalFunction) Definition() api.FunctionDefinition { + return f.definition() +} + +// SourceOffsetForPC implements the same method as documented on +// experimental.InternalFunction. +func (f internalFunction) SourceOffsetForPC(pc experimental.ProgramCounter) uint64 { + offsetsMap := f.parent.offsetsInWasmBinary + if uint64(pc) < uint64(len(offsetsMap)) { + return offsetsMap[pc] + } + return 0 +} + +// interpreter mode doesn't maintain call frames in the stack, so pass the zero size to the IR. +const callFrameStackSize = 0 + +// CompileModule implements the same method as documented on wasm.Engine. +func (e *engine) CompileModule(_ context.Context, module *wasm.Module, listeners []experimental.FunctionListener, ensureTermination bool) error { + if _, ok := e.getCompiledFunctions(module); ok { // cache hit! + return nil + } + + funcs := make([]compiledFunction, len(module.FunctionSection)) + irCompiler, err := newCompiler(e.enabledFeatures, callFrameStackSize, module, ensureTermination) + if err != nil { + return err + } + imported := module.ImportFunctionCount + for i := range module.CodeSection { + var lsn experimental.FunctionListener + if i < len(listeners) { + lsn = listeners[i] + } + + compiled := &funcs[i] + // If this is the host function, there's nothing to do as the runtime representation of + // host function in interpreter is its Go function itself as opposed to Wasm functions, + // which need to be compiled down to + if codeSeg := &module.CodeSection[i]; codeSeg.GoFunc != nil { + compiled.hostFn = codeSeg.GoFunc + } else { + ir, err := irCompiler.Next() + if err != nil { + return err + } + err = e.lowerIR(ir, compiled) + if err != nil { + def := module.FunctionDefinition(uint32(i) + module.ImportFunctionCount) + return fmt.Errorf("failed to lower func[%s] to interpreterir: %w", def.DebugName(), err) + } + } + compiled.source = module + compiled.ensureTermination = ensureTermination + compiled.listener = lsn + compiled.index = imported + uint32(i) + } + e.addCompiledFunctions(module, funcs) + return nil +} + +// NewModuleEngine implements the same method as documented on wasm.Engine. +func (e *engine) NewModuleEngine(module *wasm.Module, instance *wasm.ModuleInstance) (wasm.ModuleEngine, error) { + me := &moduleEngine{ + parentEngine: e, + functions: make([]function, len(module.FunctionSection)+int(module.ImportFunctionCount)), + } + + codes, ok := e.getCompiledFunctions(module) + if !ok { + return nil, errors.New("source module must be compiled before instantiation") + } + + for i := range codes { + c := &codes[i] + offset := i + int(module.ImportFunctionCount) + typeIndex := module.FunctionSection[i] + me.functions[offset] = function{ + moduleInstance: instance, + typeID: instance.TypeIDs[typeIndex], + funcType: &module.TypeSection[typeIndex], + parent: c, + } + } + return me, nil +} + +// lowerIR lowers the interpreterir operations to engine friendly struct. +func (e *engine) lowerIR(ir *compilationResult, ret *compiledFunction) error { + // Copy the body from the result. + ret.body = make([]unionOperation, len(ir.Operations)) + copy(ret.body, ir.Operations) + // Also copy the offsets if necessary. + if offsets := ir.IROperationSourceOffsetsInWasmBinary; len(offsets) > 0 { + ret.offsetsInWasmBinary = make([]uint64, len(offsets)) + copy(ret.offsetsInWasmBinary, offsets) + } + + labelAddressResolutions := [labelKindNum][]uint64{} + + // First, we iterate all labels, and resolve the address. + for i := range ret.body { + op := &ret.body[i] + switch op.Kind { + case operationKindLabel: + label := label(op.U1) + address := uint64(i) + + kind, fid := label.Kind(), label.FrameID() + frameToAddresses := labelAddressResolutions[label.Kind()] + // Expand the slice if necessary. + if diff := fid - len(frameToAddresses) + 1; diff > 0 { + for j := 0; j < diff; j++ { + frameToAddresses = append(frameToAddresses, 0) + } + } + frameToAddresses[fid] = address + labelAddressResolutions[kind] = frameToAddresses + } + } + + // Then resolve the label as the index to the body. + for i := range ret.body { + op := &ret.body[i] + switch op.Kind { + case operationKindBr: + e.setLabelAddress(&op.U1, label(op.U1), labelAddressResolutions) + case operationKindBrIf: + e.setLabelAddress(&op.U1, label(op.U1), labelAddressResolutions) + e.setLabelAddress(&op.U2, label(op.U2), labelAddressResolutions) + case operationKindBrTable: + for j := 0; j < len(op.Us); j += 2 { + target := op.Us[j] + e.setLabelAddress(&op.Us[j], label(target), labelAddressResolutions) + } + } + } + return nil +} + +func (e *engine) setLabelAddress(op *uint64, label label, labelAddressResolutions [labelKindNum][]uint64) { + if label.IsReturnTarget() { + // Jmp to the end of the possible binary. + *op = math.MaxUint64 + } else { + *op = labelAddressResolutions[label.Kind()][label.FrameID()] + } +} + +// ResolveImportedFunction implements wasm.ModuleEngine. +func (e *moduleEngine) ResolveImportedFunction(index, indexInImportedModule wasm.Index, importedModuleEngine wasm.ModuleEngine) { + imported := importedModuleEngine.(*moduleEngine) + e.functions[index] = imported.functions[indexInImportedModule] +} + +// ResolveImportedMemory implements wasm.ModuleEngine. +func (e *moduleEngine) ResolveImportedMemory(wasm.ModuleEngine) {} + +// DoneInstantiation implements wasm.ModuleEngine. +func (e *moduleEngine) DoneInstantiation() {} + +// FunctionInstanceReference implements the same method as documented on wasm.ModuleEngine. +func (e *moduleEngine) FunctionInstanceReference(funcIndex wasm.Index) wasm.Reference { + return uintptr(unsafe.Pointer(&e.functions[funcIndex])) +} + +// NewFunction implements the same method as documented on wasm.ModuleEngine. +func (e *moduleEngine) NewFunction(index wasm.Index) (ce api.Function) { + // Note: The input parameters are pre-validated, so a compiled function is only absent on close. Updates to + // code on close aren't locked, neither is this read. + compiled := &e.functions[index] + return e.newCallEngine(compiled) +} + +// LookupFunction implements the same method as documented on wasm.ModuleEngine. +func (e *moduleEngine) LookupFunction(t *wasm.TableInstance, typeId wasm.FunctionTypeID, tableOffset wasm.Index) (*wasm.ModuleInstance, wasm.Index) { + if tableOffset >= uint32(len(t.References)) { + panic(wasmruntime.ErrRuntimeInvalidTableAccess) + } + rawPtr := t.References[tableOffset] + if rawPtr == 0 { + panic(wasmruntime.ErrRuntimeInvalidTableAccess) + } + + tf := functionFromUintptr(rawPtr) + if tf.typeID != typeId { + panic(wasmruntime.ErrRuntimeIndirectCallTypeMismatch) + } + return tf.moduleInstance, tf.parent.index +} + +// Definition implements the same method as documented on api.Function. +func (ce *callEngine) Definition() api.FunctionDefinition { + return ce.f.definition() +} + +func (f *function) definition() api.FunctionDefinition { + compiled := f.parent + return compiled.source.FunctionDefinition(compiled.index) +} + +// Call implements the same method as documented on api.Function. +func (ce *callEngine) Call(ctx context.Context, params ...uint64) (results []uint64, err error) { + ft := ce.f.funcType + if n := ft.ParamNumInUint64; n != len(params) { + return nil, fmt.Errorf("expected %d params, but passed %d", n, len(params)) + } + return ce.call(ctx, params, nil) +} + +// CallWithStack implements the same method as documented on api.Function. +func (ce *callEngine) CallWithStack(ctx context.Context, stack []uint64) error { + params, results, err := wasm.SplitCallStack(ce.f.funcType, stack) + if err != nil { + return err + } + _, err = ce.call(ctx, params, results) + return err +} + +func (ce *callEngine) call(ctx context.Context, params, results []uint64) (_ []uint64, err error) { + m := ce.f.moduleInstance + if ce.f.parent.ensureTermination { + select { + case <-ctx.Done(): + // If the provided context is already done, close the call context + // and return the error. + m.CloseWithCtxErr(ctx) + return nil, m.FailIfClosed() + default: + } + } + + if ctx.Value(expctxkeys.EnableSnapshotterKey{}) != nil { + ctx = context.WithValue(ctx, expctxkeys.SnapshotterKey{}, ce) + } + + defer func() { + // If the module closed during the call, and the call didn't err for another reason, set an ExitError. + if err == nil { + err = m.FailIfClosed() + } + // TODO: ^^ Will not fail if the function was imported from a closed module. + + if v := recover(); v != nil { + err = ce.recoverOnCall(ctx, m, v) + } + }() + + ce.pushValues(params) + + if ce.f.parent.ensureTermination { + done := m.CloseModuleOnCanceledOrTimeout(ctx) + defer done() + } + + ce.callFunction(ctx, m, ce.f) + + // This returns a safe copy of the results, instead of a slice view. If we + // returned a re-slice, the caller could accidentally or purposefully + // corrupt the stack of subsequent calls. + ft := ce.f.funcType + if results == nil && ft.ResultNumInUint64 > 0 { + results = make([]uint64, ft.ResultNumInUint64) + } + ce.popValues(results) + return results, nil +} + +// functionListenerInvocation captures arguments needed to perform function +// listener invocations when unwinding the call stack. +type functionListenerInvocation struct { + experimental.FunctionListener + def api.FunctionDefinition +} + +// recoverOnCall takes the recovered value `recoverOnCall`, and wraps it +// with the call frame stack traces. Also, reset the state of callEngine +// so that it can be used for the subsequent calls. +func (ce *callEngine) recoverOnCall(ctx context.Context, m *wasm.ModuleInstance, v interface{}) (err error) { + if s, ok := v.(*snapshot); ok { + // A snapshot that wasn't handled was created by a different call engine possibly from a nested wasm invocation, + // let it propagate up to be handled by the caller. + panic(s) + } + + builder := wasmdebug.NewErrorBuilder() + frameCount := len(ce.frames) + functionListeners := make([]functionListenerInvocation, 0, 16) + + if frameCount > wasmdebug.MaxFrames { + frameCount = wasmdebug.MaxFrames + } + for i := 0; i < frameCount; i++ { + frame := ce.popFrame() + f := frame.f + def := f.definition() + var sources []string + if parent := frame.f.parent; parent.body != nil && len(parent.offsetsInWasmBinary) > 0 { + sources = parent.source.DWARFLines.Line(parent.offsetsInWasmBinary[frame.pc]) + } + builder.AddFrame(def.DebugName(), def.ParamTypes(), def.ResultTypes(), sources) + if f.parent.listener != nil { + functionListeners = append(functionListeners, functionListenerInvocation{ + FunctionListener: f.parent.listener, + def: f.definition(), + }) + } + } + + err = builder.FromRecovered(v) + for i := range functionListeners { + functionListeners[i].Abort(ctx, m, functionListeners[i].def, err) + } + + // Allows the reuse of CallEngine. + ce.stack, ce.frames = ce.stack[:0], ce.frames[:0] + return +} + +func (ce *callEngine) callFunction(ctx context.Context, m *wasm.ModuleInstance, f *function) { + if f.parent.hostFn != nil { + ce.callGoFuncWithStack(ctx, m, f) + } else if lsn := f.parent.listener; lsn != nil { + ce.callNativeFuncWithListener(ctx, m, f, lsn) + } else { + ce.callNativeFunc(ctx, m, f) + } +} + +func (ce *callEngine) callGoFunc(ctx context.Context, m *wasm.ModuleInstance, f *function, stack []uint64) { + typ := f.funcType + lsn := f.parent.listener + if lsn != nil { + params := stack[:typ.ParamNumInUint64] + ce.stackIterator.reset(ce.stack, ce.frames, f) + lsn.Before(ctx, m, f.definition(), params, &ce.stackIterator) + ce.stackIterator.clear() + } + frame := &callFrame{f: f, base: len(ce.stack)} + ce.pushFrame(frame) + + fn := f.parent.hostFn + switch fn := fn.(type) { + case api.GoModuleFunction: + fn.Call(ctx, m, stack) + case api.GoFunction: + fn.Call(ctx, stack) + } + + ce.popFrame() + if lsn != nil { + // TODO: This doesn't get the error due to use of panic to propagate them. + results := stack[:typ.ResultNumInUint64] + lsn.After(ctx, m, f.definition(), results) + } +} + +func (ce *callEngine) callNativeFunc(ctx context.Context, m *wasm.ModuleInstance, f *function) { + frame := &callFrame{f: f, base: len(ce.stack)} + moduleInst := f.moduleInstance + functions := moduleInst.Engine.(*moduleEngine).functions + memoryInst := moduleInst.MemoryInstance + globals := moduleInst.Globals + tables := moduleInst.Tables + typeIDs := moduleInst.TypeIDs + dataInstances := moduleInst.DataInstances + elementInstances := moduleInst.ElementInstances + ce.pushFrame(frame) + body := frame.f.parent.body + bodyLen := uint64(len(body)) + for frame.pc < bodyLen { + op := &body[frame.pc] + // TODO: add description of each operation/case + // on, for example, how many args are used, + // how the stack is modified, etc. + switch op.Kind { + case operationKindBuiltinFunctionCheckExitCode: + if err := m.FailIfClosed(); err != nil { + panic(err) + } + frame.pc++ + case operationKindUnreachable: + panic(wasmruntime.ErrRuntimeUnreachable) + case operationKindBr: + frame.pc = op.U1 + case operationKindBrIf: + if ce.popValue() > 0 { + ce.drop(op.U3) + frame.pc = op.U1 + } else { + frame.pc = op.U2 + } + case operationKindBrTable: + v := ce.popValue() + defaultAt := uint64(len(op.Us))/2 - 1 + if v > defaultAt { + v = defaultAt + } + v *= 2 + ce.drop(op.Us[v+1]) + frame.pc = op.Us[v] + case operationKindCall: + func() { + if ctx.Value(expctxkeys.EnableSnapshotterKey{}) != nil { + defer func() { + if r := recover(); r != nil { + if s, ok := r.(*snapshot); ok && s.ce == ce { + s.doRestore() + frame = ce.frames[len(ce.frames)-1] + body = frame.f.parent.body + bodyLen = uint64(len(body)) + } else { + panic(r) + } + } + }() + } + ce.callFunction(ctx, f.moduleInstance, &functions[op.U1]) + }() + frame.pc++ + case operationKindCallIndirect: + offset := ce.popValue() + table := tables[op.U2] + if offset >= uint64(len(table.References)) { + panic(wasmruntime.ErrRuntimeInvalidTableAccess) + } + rawPtr := table.References[offset] + if rawPtr == 0 { + panic(wasmruntime.ErrRuntimeInvalidTableAccess) + } + + tf := functionFromUintptr(rawPtr) + if tf.typeID != typeIDs[op.U1] { + panic(wasmruntime.ErrRuntimeIndirectCallTypeMismatch) + } + + ce.callFunction(ctx, f.moduleInstance, tf) + frame.pc++ + case operationKindDrop: + ce.drop(op.U1) + frame.pc++ + case operationKindSelect: + c := ce.popValue() + if op.B3 { // Target is vector. + x2Hi, x2Lo := ce.popValue(), ce.popValue() + if c == 0 { + _, _ = ce.popValue(), ce.popValue() // discard the x1's lo and hi bits. + ce.pushValue(x2Lo) + ce.pushValue(x2Hi) + } + } else { + v2 := ce.popValue() + if c == 0 { + _ = ce.popValue() + ce.pushValue(v2) + } + } + frame.pc++ + case operationKindPick: + index := len(ce.stack) - 1 - int(op.U1) + ce.pushValue(ce.stack[index]) + if op.B3 { // V128 value target. + ce.pushValue(ce.stack[index+1]) + } + frame.pc++ + case operationKindSet: + if op.B3 { // V128 value target. + lowIndex := len(ce.stack) - 1 - int(op.U1) + highIndex := lowIndex + 1 + hi, lo := ce.popValue(), ce.popValue() + ce.stack[lowIndex], ce.stack[highIndex] = lo, hi + } else { + index := len(ce.stack) - 1 - int(op.U1) + ce.stack[index] = ce.popValue() + } + frame.pc++ + case operationKindGlobalGet: + g := globals[op.U1] + ce.pushValue(g.Val) + if g.Type.ValType == wasm.ValueTypeV128 { + ce.pushValue(g.ValHi) + } + frame.pc++ + case operationKindGlobalSet: + g := globals[op.U1] + if g.Type.ValType == wasm.ValueTypeV128 { + g.ValHi = ce.popValue() + } + g.Val = ce.popValue() + frame.pc++ + case operationKindLoad: + offset := ce.popMemoryOffset(op) + switch unsignedType(op.B1) { + case unsignedTypeI32, unsignedTypeF32: + if val, ok := memoryInst.ReadUint32Le(offset); !ok { + panic(wasmruntime.ErrRuntimeOutOfBoundsMemoryAccess) + } else { + ce.pushValue(uint64(val)) + } + case unsignedTypeI64, unsignedTypeF64: + if val, ok := memoryInst.ReadUint64Le(offset); !ok { + panic(wasmruntime.ErrRuntimeOutOfBoundsMemoryAccess) + } else { + ce.pushValue(val) + } + } + frame.pc++ + case operationKindLoad8: + val, ok := memoryInst.ReadByte(ce.popMemoryOffset(op)) + if !ok { + panic(wasmruntime.ErrRuntimeOutOfBoundsMemoryAccess) + } + + switch signedInt(op.B1) { + case signedInt32: + ce.pushValue(uint64(uint32(int8(val)))) + case signedInt64: + ce.pushValue(uint64(int8(val))) + case signedUint32, signedUint64: + ce.pushValue(uint64(val)) + } + frame.pc++ + case operationKindLoad16: + + val, ok := memoryInst.ReadUint16Le(ce.popMemoryOffset(op)) + if !ok { + panic(wasmruntime.ErrRuntimeOutOfBoundsMemoryAccess) + } + + switch signedInt(op.B1) { + case signedInt32: + ce.pushValue(uint64(uint32(int16(val)))) + case signedInt64: + ce.pushValue(uint64(int16(val))) + case signedUint32, signedUint64: + ce.pushValue(uint64(val)) + } + frame.pc++ + case operationKindLoad32: + val, ok := memoryInst.ReadUint32Le(ce.popMemoryOffset(op)) + if !ok { + panic(wasmruntime.ErrRuntimeOutOfBoundsMemoryAccess) + } + + if op.B1 == 1 { // Signed + ce.pushValue(uint64(int32(val))) + } else { + ce.pushValue(uint64(val)) + } + frame.pc++ + case operationKindStore: + val := ce.popValue() + offset := ce.popMemoryOffset(op) + switch unsignedType(op.B1) { + case unsignedTypeI32, unsignedTypeF32: + if !memoryInst.WriteUint32Le(offset, uint32(val)) { + panic(wasmruntime.ErrRuntimeOutOfBoundsMemoryAccess) + } + case unsignedTypeI64, unsignedTypeF64: + if !memoryInst.WriteUint64Le(offset, val) { + panic(wasmruntime.ErrRuntimeOutOfBoundsMemoryAccess) + } + } + frame.pc++ + case operationKindStore8: + val := byte(ce.popValue()) + offset := ce.popMemoryOffset(op) + if !memoryInst.WriteByte(offset, val) { + panic(wasmruntime.ErrRuntimeOutOfBoundsMemoryAccess) + } + frame.pc++ + case operationKindStore16: + val := uint16(ce.popValue()) + offset := ce.popMemoryOffset(op) + if !memoryInst.WriteUint16Le(offset, val) { + panic(wasmruntime.ErrRuntimeOutOfBoundsMemoryAccess) + } + frame.pc++ + case operationKindStore32: + val := uint32(ce.popValue()) + offset := ce.popMemoryOffset(op) + if !memoryInst.WriteUint32Le(offset, val) { + panic(wasmruntime.ErrRuntimeOutOfBoundsMemoryAccess) + } + frame.pc++ + case operationKindMemorySize: + ce.pushValue(uint64(memoryInst.Pages())) + frame.pc++ + case operationKindMemoryGrow: + n := ce.popValue() + if res, ok := memoryInst.Grow(uint32(n)); !ok { + ce.pushValue(uint64(0xffffffff)) // = -1 in signed 32-bit integer. + } else { + ce.pushValue(uint64(res)) + } + frame.pc++ + case operationKindConstI32, operationKindConstI64, + operationKindConstF32, operationKindConstF64: + ce.pushValue(op.U1) + frame.pc++ + case operationKindEq: + var b bool + switch unsignedType(op.B1) { + case unsignedTypeI32: + v2, v1 := ce.popValue(), ce.popValue() + b = uint32(v1) == uint32(v2) + case unsignedTypeI64: + v2, v1 := ce.popValue(), ce.popValue() + b = v1 == v2 + case unsignedTypeF32: + v2, v1 := ce.popValue(), ce.popValue() + b = math.Float32frombits(uint32(v2)) == math.Float32frombits(uint32(v1)) + case unsignedTypeF64: + v2, v1 := ce.popValue(), ce.popValue() + b = math.Float64frombits(v2) == math.Float64frombits(v1) + } + if b { + ce.pushValue(1) + } else { + ce.pushValue(0) + } + frame.pc++ + case operationKindNe: + var b bool + switch unsignedType(op.B1) { + case unsignedTypeI32, unsignedTypeI64: + v2, v1 := ce.popValue(), ce.popValue() + b = v1 != v2 + case unsignedTypeF32: + v2, v1 := ce.popValue(), ce.popValue() + b = math.Float32frombits(uint32(v2)) != math.Float32frombits(uint32(v1)) + case unsignedTypeF64: + v2, v1 := ce.popValue(), ce.popValue() + b = math.Float64frombits(v2) != math.Float64frombits(v1) + } + if b { + ce.pushValue(1) + } else { + ce.pushValue(0) + } + frame.pc++ + case operationKindEqz: + if ce.popValue() == 0 { + ce.pushValue(1) + } else { + ce.pushValue(0) + } + frame.pc++ + case operationKindLt: + v2 := ce.popValue() + v1 := ce.popValue() + var b bool + switch signedType(op.B1) { + case signedTypeInt32: + b = int32(v1) < int32(v2) + case signedTypeInt64: + b = int64(v1) < int64(v2) + case signedTypeUint32, signedTypeUint64: + b = v1 < v2 + case signedTypeFloat32: + b = math.Float32frombits(uint32(v1)) < math.Float32frombits(uint32(v2)) + case signedTypeFloat64: + b = math.Float64frombits(v1) < math.Float64frombits(v2) + } + if b { + ce.pushValue(1) + } else { + ce.pushValue(0) + } + frame.pc++ + case operationKindGt: + v2 := ce.popValue() + v1 := ce.popValue() + var b bool + switch signedType(op.B1) { + case signedTypeInt32: + b = int32(v1) > int32(v2) + case signedTypeInt64: + b = int64(v1) > int64(v2) + case signedTypeUint32, signedTypeUint64: + b = v1 > v2 + case signedTypeFloat32: + b = math.Float32frombits(uint32(v1)) > math.Float32frombits(uint32(v2)) + case signedTypeFloat64: + b = math.Float64frombits(v1) > math.Float64frombits(v2) + } + if b { + ce.pushValue(1) + } else { + ce.pushValue(0) + } + frame.pc++ + case operationKindLe: + v2 := ce.popValue() + v1 := ce.popValue() + var b bool + switch signedType(op.B1) { + case signedTypeInt32: + b = int32(v1) <= int32(v2) + case signedTypeInt64: + b = int64(v1) <= int64(v2) + case signedTypeUint32, signedTypeUint64: + b = v1 <= v2 + case signedTypeFloat32: + b = math.Float32frombits(uint32(v1)) <= math.Float32frombits(uint32(v2)) + case signedTypeFloat64: + b = math.Float64frombits(v1) <= math.Float64frombits(v2) + } + if b { + ce.pushValue(1) + } else { + ce.pushValue(0) + } + frame.pc++ + case operationKindGe: + v2 := ce.popValue() + v1 := ce.popValue() + var b bool + switch signedType(op.B1) { + case signedTypeInt32: + b = int32(v1) >= int32(v2) + case signedTypeInt64: + b = int64(v1) >= int64(v2) + case signedTypeUint32, signedTypeUint64: + b = v1 >= v2 + case signedTypeFloat32: + b = math.Float32frombits(uint32(v1)) >= math.Float32frombits(uint32(v2)) + case signedTypeFloat64: + b = math.Float64frombits(v1) >= math.Float64frombits(v2) + } + if b { + ce.pushValue(1) + } else { + ce.pushValue(0) + } + frame.pc++ + case operationKindAdd: + v2 := ce.popValue() + v1 := ce.popValue() + switch unsignedType(op.B1) { + case unsignedTypeI32: + v := uint32(v1) + uint32(v2) + ce.pushValue(uint64(v)) + case unsignedTypeI64: + ce.pushValue(v1 + v2) + case unsignedTypeF32: + ce.pushValue(addFloat32bits(uint32(v1), uint32(v2))) + case unsignedTypeF64: + v := math.Float64frombits(v1) + math.Float64frombits(v2) + ce.pushValue(math.Float64bits(v)) + } + frame.pc++ + case operationKindSub: + v2 := ce.popValue() + v1 := ce.popValue() + switch unsignedType(op.B1) { + case unsignedTypeI32: + ce.pushValue(uint64(uint32(v1) - uint32(v2))) + case unsignedTypeI64: + ce.pushValue(v1 - v2) + case unsignedTypeF32: + ce.pushValue(subFloat32bits(uint32(v1), uint32(v2))) + case unsignedTypeF64: + v := math.Float64frombits(v1) - math.Float64frombits(v2) + ce.pushValue(math.Float64bits(v)) + } + frame.pc++ + case operationKindMul: + v2 := ce.popValue() + v1 := ce.popValue() + switch unsignedType(op.B1) { + case unsignedTypeI32: + ce.pushValue(uint64(uint32(v1) * uint32(v2))) + case unsignedTypeI64: + ce.pushValue(v1 * v2) + case unsignedTypeF32: + ce.pushValue(mulFloat32bits(uint32(v1), uint32(v2))) + case unsignedTypeF64: + v := math.Float64frombits(v2) * math.Float64frombits(v1) + ce.pushValue(math.Float64bits(v)) + } + frame.pc++ + case operationKindClz: + v := ce.popValue() + if op.B1 == 0 { + // unsignedInt32 + ce.pushValue(uint64(bits.LeadingZeros32(uint32(v)))) + } else { + // unsignedInt64 + ce.pushValue(uint64(bits.LeadingZeros64(v))) + } + frame.pc++ + case operationKindCtz: + v := ce.popValue() + if op.B1 == 0 { + // unsignedInt32 + ce.pushValue(uint64(bits.TrailingZeros32(uint32(v)))) + } else { + // unsignedInt64 + ce.pushValue(uint64(bits.TrailingZeros64(v))) + } + frame.pc++ + case operationKindPopcnt: + v := ce.popValue() + if op.B1 == 0 { + // unsignedInt32 + ce.pushValue(uint64(bits.OnesCount32(uint32(v)))) + } else { + // unsignedInt64 + ce.pushValue(uint64(bits.OnesCount64(v))) + } + frame.pc++ + case operationKindDiv: + // If an integer, check we won't divide by zero. + t := signedType(op.B1) + v2, v1 := ce.popValue(), ce.popValue() + switch t { + case signedTypeFloat32, signedTypeFloat64: // not integers + default: + if v2 == 0 { + panic(wasmruntime.ErrRuntimeIntegerDivideByZero) + } + } + + switch t { + case signedTypeInt32: + d := int32(v2) + n := int32(v1) + if n == math.MinInt32 && d == -1 { + panic(wasmruntime.ErrRuntimeIntegerOverflow) + } + ce.pushValue(uint64(uint32(n / d))) + case signedTypeInt64: + d := int64(v2) + n := int64(v1) + if n == math.MinInt64 && d == -1 { + panic(wasmruntime.ErrRuntimeIntegerOverflow) + } + ce.pushValue(uint64(n / d)) + case signedTypeUint32: + d := uint32(v2) + n := uint32(v1) + ce.pushValue(uint64(n / d)) + case signedTypeUint64: + d := v2 + n := v1 + ce.pushValue(n / d) + case signedTypeFloat32: + ce.pushValue(divFloat32bits(uint32(v1), uint32(v2))) + case signedTypeFloat64: + ce.pushValue(math.Float64bits(math.Float64frombits(v1) / math.Float64frombits(v2))) + } + frame.pc++ + case operationKindRem: + v2, v1 := ce.popValue(), ce.popValue() + if v2 == 0 { + panic(wasmruntime.ErrRuntimeIntegerDivideByZero) + } + switch signedInt(op.B1) { + case signedInt32: + d := int32(v2) + n := int32(v1) + ce.pushValue(uint64(uint32(n % d))) + case signedInt64: + d := int64(v2) + n := int64(v1) + ce.pushValue(uint64(n % d)) + case signedUint32: + d := uint32(v2) + n := uint32(v1) + ce.pushValue(uint64(n % d)) + case signedUint64: + d := v2 + n := v1 + ce.pushValue(n % d) + } + frame.pc++ + case operationKindAnd: + v2 := ce.popValue() + v1 := ce.popValue() + if op.B1 == 0 { + // unsignedInt32 + ce.pushValue(uint64(uint32(v2) & uint32(v1))) + } else { + // unsignedInt64 + ce.pushValue(uint64(v2 & v1)) + } + frame.pc++ + case operationKindOr: + v2 := ce.popValue() + v1 := ce.popValue() + if op.B1 == 0 { + // unsignedInt32 + ce.pushValue(uint64(uint32(v2) | uint32(v1))) + } else { + // unsignedInt64 + ce.pushValue(uint64(v2 | v1)) + } + frame.pc++ + case operationKindXor: + v2 := ce.popValue() + v1 := ce.popValue() + if op.B1 == 0 { + // unsignedInt32 + ce.pushValue(uint64(uint32(v2) ^ uint32(v1))) + } else { + // unsignedInt64 + ce.pushValue(uint64(v2 ^ v1)) + } + frame.pc++ + case operationKindShl: + v2 := ce.popValue() + v1 := ce.popValue() + if op.B1 == 0 { + // unsignedInt32 + ce.pushValue(uint64(uint32(v1) << (uint32(v2) % 32))) + } else { + // unsignedInt64 + ce.pushValue(v1 << (v2 % 64)) + } + frame.pc++ + case operationKindShr: + v2 := ce.popValue() + v1 := ce.popValue() + switch signedInt(op.B1) { + case signedInt32: + ce.pushValue(uint64(uint32(int32(v1) >> (uint32(v2) % 32)))) + case signedInt64: + ce.pushValue(uint64(int64(v1) >> (v2 % 64))) + case signedUint32: + ce.pushValue(uint64(uint32(v1) >> (uint32(v2) % 32))) + case signedUint64: + ce.pushValue(v1 >> (v2 % 64)) + } + frame.pc++ + case operationKindRotl: + v2 := ce.popValue() + v1 := ce.popValue() + if op.B1 == 0 { + // unsignedInt32 + ce.pushValue(uint64(bits.RotateLeft32(uint32(v1), int(v2)))) + } else { + // unsignedInt64 + ce.pushValue(uint64(bits.RotateLeft64(v1, int(v2)))) + } + frame.pc++ + case operationKindRotr: + v2 := ce.popValue() + v1 := ce.popValue() + if op.B1 == 0 { + // unsignedInt32 + ce.pushValue(uint64(bits.RotateLeft32(uint32(v1), -int(v2)))) + } else { + // unsignedInt64 + ce.pushValue(uint64(bits.RotateLeft64(v1, -int(v2)))) + } + frame.pc++ + case operationKindAbs: + if op.B1 == 0 { + // float32 + const mask uint32 = 1 << 31 + ce.pushValue(uint64(uint32(ce.popValue()) &^ mask)) + } else { + // float64 + const mask uint64 = 1 << 63 + ce.pushValue(ce.popValue() &^ mask) + } + frame.pc++ + case operationKindNeg: + if op.B1 == 0 { + // float32 + v := -math.Float32frombits(uint32(ce.popValue())) + ce.pushValue(uint64(math.Float32bits(v))) + } else { + // float64 + v := -math.Float64frombits(ce.popValue()) + ce.pushValue(math.Float64bits(v)) + } + frame.pc++ + case operationKindCeil: + if op.B1 == 0 { + // float32 + v := moremath.WasmCompatCeilF32(math.Float32frombits(uint32(ce.popValue()))) + ce.pushValue(uint64(math.Float32bits(v))) + } else { + // float64 + v := moremath.WasmCompatCeilF64(math.Float64frombits(ce.popValue())) + ce.pushValue(math.Float64bits(v)) + } + frame.pc++ + case operationKindFloor: + if op.B1 == 0 { + // float32 + v := moremath.WasmCompatFloorF32(math.Float32frombits(uint32(ce.popValue()))) + ce.pushValue(uint64(math.Float32bits(v))) + } else { + // float64 + v := moremath.WasmCompatFloorF64(math.Float64frombits(ce.popValue())) + ce.pushValue(math.Float64bits(v)) + } + frame.pc++ + case operationKindTrunc: + if op.B1 == 0 { + // float32 + v := moremath.WasmCompatTruncF32(math.Float32frombits(uint32(ce.popValue()))) + ce.pushValue(uint64(math.Float32bits(v))) + } else { + // float64 + v := moremath.WasmCompatTruncF64(math.Float64frombits(ce.popValue())) + ce.pushValue(math.Float64bits(v)) + } + frame.pc++ + case operationKindNearest: + if op.B1 == 0 { + // float32 + f := math.Float32frombits(uint32(ce.popValue())) + ce.pushValue(uint64(math.Float32bits(moremath.WasmCompatNearestF32(f)))) + } else { + // float64 + f := math.Float64frombits(ce.popValue()) + ce.pushValue(math.Float64bits(moremath.WasmCompatNearestF64(f))) + } + frame.pc++ + case operationKindSqrt: + if op.B1 == 0 { + // float32 + v := math.Sqrt(float64(math.Float32frombits(uint32(ce.popValue())))) + ce.pushValue(uint64(math.Float32bits(float32(v)))) + } else { + // float64 + v := math.Sqrt(math.Float64frombits(ce.popValue())) + ce.pushValue(math.Float64bits(v)) + } + frame.pc++ + case operationKindMin: + if op.B1 == 0 { + // float32 + ce.pushValue(wasmCompatMin32bits(uint32(ce.popValue()), uint32(ce.popValue()))) + } else { + v2 := math.Float64frombits(ce.popValue()) + v1 := math.Float64frombits(ce.popValue()) + ce.pushValue(math.Float64bits(moremath.WasmCompatMin64(v1, v2))) + } + frame.pc++ + case operationKindMax: + if op.B1 == 0 { + ce.pushValue(wasmCompatMax32bits(uint32(ce.popValue()), uint32(ce.popValue()))) + } else { + // float64 + v2 := math.Float64frombits(ce.popValue()) + v1 := math.Float64frombits(ce.popValue()) + ce.pushValue(math.Float64bits(moremath.WasmCompatMax64(v1, v2))) + } + frame.pc++ + case operationKindCopysign: + if op.B1 == 0 { + // float32 + v2 := uint32(ce.popValue()) + v1 := uint32(ce.popValue()) + const signbit = 1 << 31 + ce.pushValue(uint64(v1&^signbit | v2&signbit)) + } else { + // float64 + v2 := ce.popValue() + v1 := ce.popValue() + const signbit = 1 << 63 + ce.pushValue(v1&^signbit | v2&signbit) + } + frame.pc++ + case operationKindI32WrapFromI64: + ce.pushValue(uint64(uint32(ce.popValue()))) + frame.pc++ + case operationKindITruncFromF: + if op.B1 == 0 { + // float32 + switch signedInt(op.B2) { + case signedInt32: + v := math.Trunc(float64(math.Float32frombits(uint32(ce.popValue())))) + if math.IsNaN(v) { // NaN cannot be compared with themselves, so we have to use IsNaN + if op.B3 { + // non-trapping conversion must cast nan to zero. + v = 0 + } else { + panic(wasmruntime.ErrRuntimeInvalidConversionToInteger) + } + } else if v < math.MinInt32 || v > math.MaxInt32 { + if op.B3 { + // non-trapping conversion must "saturate" the value for overflowing sources. + if v < 0 { + v = math.MinInt32 + } else { + v = math.MaxInt32 + } + } else { + panic(wasmruntime.ErrRuntimeIntegerOverflow) + } + } + ce.pushValue(uint64(uint32(int32(v)))) + case signedInt64: + v := math.Trunc(float64(math.Float32frombits(uint32(ce.popValue())))) + res := int64(v) + if math.IsNaN(v) { // NaN cannot be compared with themselves, so we have to use IsNaN + if op.B3 { + // non-trapping conversion must cast nan to zero. + res = 0 + } else { + panic(wasmruntime.ErrRuntimeInvalidConversionToInteger) + } + } else if v < math.MinInt64 || v >= math.MaxInt64 { + // Note: math.MaxInt64 is rounded up to math.MaxInt64+1 in 64-bit float representation, + // and that's why we use '>=' not '>' to check overflow. + if op.B3 { + // non-trapping conversion must "saturate" the value for overflowing sources. + if v < 0 { + res = math.MinInt64 + } else { + res = math.MaxInt64 + } + } else { + panic(wasmruntime.ErrRuntimeIntegerOverflow) + } + } + ce.pushValue(uint64(res)) + case signedUint32: + v := math.Trunc(float64(math.Float32frombits(uint32(ce.popValue())))) + if math.IsNaN(v) { // NaN cannot be compared with themselves, so we have to use IsNaN + if op.B3 { + // non-trapping conversion must cast nan to zero. + v = 0 + } else { + panic(wasmruntime.ErrRuntimeInvalidConversionToInteger) + } + } else if v < 0 || v > math.MaxUint32 { + if op.B3 { + // non-trapping conversion must "saturate" the value for overflowing source. + if v < 0 { + v = 0 + } else { + v = math.MaxUint32 + } + } else { + panic(wasmruntime.ErrRuntimeIntegerOverflow) + } + } + ce.pushValue(uint64(uint32(v))) + case signedUint64: + v := math.Trunc(float64(math.Float32frombits(uint32(ce.popValue())))) + res := uint64(v) + if math.IsNaN(v) { // NaN cannot be compared with themselves, so we have to use IsNaN + if op.B3 { + // non-trapping conversion must cast nan to zero. + res = 0 + } else { + panic(wasmruntime.ErrRuntimeInvalidConversionToInteger) + } + } else if v < 0 || v >= math.MaxUint64 { + // Note: math.MaxUint64 is rounded up to math.MaxUint64+1 in 64-bit float representation, + // and that's why we use '>=' not '>' to check overflow. + if op.B3 { + // non-trapping conversion must "saturate" the value for overflowing source. + if v < 0 { + res = 0 + } else { + res = math.MaxUint64 + } + } else { + panic(wasmruntime.ErrRuntimeIntegerOverflow) + } + } + ce.pushValue(res) + } + } else { + // float64 + switch signedInt(op.B2) { + case signedInt32: + v := math.Trunc(math.Float64frombits(ce.popValue())) + if math.IsNaN(v) { // NaN cannot be compared with themselves, so we have to use IsNaN + if op.B3 { + // non-trapping conversion must cast nan to zero. + v = 0 + } else { + panic(wasmruntime.ErrRuntimeInvalidConversionToInteger) + } + } else if v < math.MinInt32 || v > math.MaxInt32 { + if op.B3 { + // non-trapping conversion must "saturate" the value for overflowing source. + if v < 0 { + v = math.MinInt32 + } else { + v = math.MaxInt32 + } + } else { + panic(wasmruntime.ErrRuntimeIntegerOverflow) + } + } + ce.pushValue(uint64(uint32(int32(v)))) + case signedInt64: + v := math.Trunc(math.Float64frombits(ce.popValue())) + res := int64(v) + if math.IsNaN(v) { // NaN cannot be compared with themselves, so we have to use IsNaN + if op.B3 { + // non-trapping conversion must cast nan to zero. + res = 0 + } else { + panic(wasmruntime.ErrRuntimeInvalidConversionToInteger) + } + } else if v < math.MinInt64 || v >= math.MaxInt64 { + // Note: math.MaxInt64 is rounded up to math.MaxInt64+1 in 64-bit float representation, + // and that's why we use '>=' not '>' to check overflow. + if op.B3 { + // non-trapping conversion must "saturate" the value for overflowing source. + if v < 0 { + res = math.MinInt64 + } else { + res = math.MaxInt64 + } + } else { + panic(wasmruntime.ErrRuntimeIntegerOverflow) + } + } + ce.pushValue(uint64(res)) + case signedUint32: + v := math.Trunc(math.Float64frombits(ce.popValue())) + if math.IsNaN(v) { // NaN cannot be compared with themselves, so we have to use IsNaN + if op.B3 { + // non-trapping conversion must cast nan to zero. + v = 0 + } else { + panic(wasmruntime.ErrRuntimeInvalidConversionToInteger) + } + } else if v < 0 || v > math.MaxUint32 { + if op.B3 { + // non-trapping conversion must "saturate" the value for overflowing source. + if v < 0 { + v = 0 + } else { + v = math.MaxUint32 + } + } else { + panic(wasmruntime.ErrRuntimeIntegerOverflow) + } + } + ce.pushValue(uint64(uint32(v))) + case signedUint64: + v := math.Trunc(math.Float64frombits(ce.popValue())) + res := uint64(v) + if math.IsNaN(v) { // NaN cannot be compared with themselves, so we have to use IsNaN + if op.B3 { + // non-trapping conversion must cast nan to zero. + res = 0 + } else { + panic(wasmruntime.ErrRuntimeInvalidConversionToInteger) + } + } else if v < 0 || v >= math.MaxUint64 { + // Note: math.MaxUint64 is rounded up to math.MaxUint64+1 in 64-bit float representation, + // and that's why we use '>=' not '>' to check overflow. + if op.B3 { + // non-trapping conversion must "saturate" the value for overflowing source. + if v < 0 { + res = 0 + } else { + res = math.MaxUint64 + } + } else { + panic(wasmruntime.ErrRuntimeIntegerOverflow) + } + } + ce.pushValue(res) + } + } + frame.pc++ + case operationKindFConvertFromI: + switch signedInt(op.B1) { + case signedInt32: + if op.B2 == 0 { + // float32 + v := float32(int32(ce.popValue())) + ce.pushValue(uint64(math.Float32bits(v))) + } else { + // float64 + v := float64(int32(ce.popValue())) + ce.pushValue(math.Float64bits(v)) + } + case signedInt64: + if op.B2 == 0 { + // float32 + v := float32(int64(ce.popValue())) + ce.pushValue(uint64(math.Float32bits(v))) + } else { + // float64 + v := float64(int64(ce.popValue())) + ce.pushValue(math.Float64bits(v)) + } + case signedUint32: + if op.B2 == 0 { + // float32 + v := float32(uint32(ce.popValue())) + ce.pushValue(uint64(math.Float32bits(v))) + } else { + // float64 + v := float64(uint32(ce.popValue())) + ce.pushValue(math.Float64bits(v)) + } + case signedUint64: + if op.B2 == 0 { + // float32 + v := float32(ce.popValue()) + ce.pushValue(uint64(math.Float32bits(v))) + } else { + // float64 + v := float64(ce.popValue()) + ce.pushValue(math.Float64bits(v)) + } + } + frame.pc++ + case operationKindF32DemoteFromF64: + v := float32(math.Float64frombits(ce.popValue())) + ce.pushValue(uint64(math.Float32bits(v))) + frame.pc++ + case operationKindF64PromoteFromF32: + v := float64(math.Float32frombits(uint32(ce.popValue()))) + ce.pushValue(math.Float64bits(v)) + frame.pc++ + case operationKindExtend: + if op.B1 == 1 { + // Signed. + v := int64(int32(ce.popValue())) + ce.pushValue(uint64(v)) + } else { + v := uint64(uint32(ce.popValue())) + ce.pushValue(v) + } + frame.pc++ + case operationKindSignExtend32From8: + v := uint32(int8(ce.popValue())) + ce.pushValue(uint64(v)) + frame.pc++ + case operationKindSignExtend32From16: + v := uint32(int16(ce.popValue())) + ce.pushValue(uint64(v)) + frame.pc++ + case operationKindSignExtend64From8: + v := int64(int8(ce.popValue())) + ce.pushValue(uint64(v)) + frame.pc++ + case operationKindSignExtend64From16: + v := int64(int16(ce.popValue())) + ce.pushValue(uint64(v)) + frame.pc++ + case operationKindSignExtend64From32: + v := int64(int32(ce.popValue())) + ce.pushValue(uint64(v)) + frame.pc++ + case operationKindMemoryInit: + dataInstance := dataInstances[op.U1] + copySize := ce.popValue() + inDataOffset := ce.popValue() + inMemoryOffset := ce.popValue() + if inDataOffset+copySize > uint64(len(dataInstance)) || + inMemoryOffset+copySize > uint64(len(memoryInst.Buffer)) { + panic(wasmruntime.ErrRuntimeOutOfBoundsMemoryAccess) + } else if copySize != 0 { + copy(memoryInst.Buffer[inMemoryOffset:inMemoryOffset+copySize], dataInstance[inDataOffset:]) + } + frame.pc++ + case operationKindDataDrop: + dataInstances[op.U1] = nil + frame.pc++ + case operationKindMemoryCopy: + memLen := uint64(len(memoryInst.Buffer)) + copySize := ce.popValue() + sourceOffset := ce.popValue() + destinationOffset := ce.popValue() + if sourceOffset+copySize > memLen || destinationOffset+copySize > memLen { + panic(wasmruntime.ErrRuntimeOutOfBoundsMemoryAccess) + } else if copySize != 0 { + copy(memoryInst.Buffer[destinationOffset:], + memoryInst.Buffer[sourceOffset:sourceOffset+copySize]) + } + frame.pc++ + case operationKindMemoryFill: + fillSize := ce.popValue() + value := byte(ce.popValue()) + offset := ce.popValue() + if fillSize+offset > uint64(len(memoryInst.Buffer)) { + panic(wasmruntime.ErrRuntimeOutOfBoundsMemoryAccess) + } else if fillSize != 0 { + // Uses the copy trick for faster filling buffer. + // https://gist.github.com/taylorza/df2f89d5f9ab3ffd06865062a4cf015d + buf := memoryInst.Buffer[offset : offset+fillSize] + buf[0] = value + for i := 1; i < len(buf); i *= 2 { + copy(buf[i:], buf[:i]) + } + } + frame.pc++ + case operationKindTableInit: + elementInstance := elementInstances[op.U1] + copySize := ce.popValue() + inElementOffset := ce.popValue() + inTableOffset := ce.popValue() + table := tables[op.U2] + if inElementOffset+copySize > uint64(len(elementInstance)) || + inTableOffset+copySize > uint64(len(table.References)) { + panic(wasmruntime.ErrRuntimeInvalidTableAccess) + } else if copySize != 0 { + copy(table.References[inTableOffset:inTableOffset+copySize], elementInstance[inElementOffset:]) + } + frame.pc++ + case operationKindElemDrop: + elementInstances[op.U1] = nil + frame.pc++ + case operationKindTableCopy: + srcTable, dstTable := tables[op.U1].References, tables[op.U2].References + copySize := ce.popValue() + sourceOffset := ce.popValue() + destinationOffset := ce.popValue() + if sourceOffset+copySize > uint64(len(srcTable)) || destinationOffset+copySize > uint64(len(dstTable)) { + panic(wasmruntime.ErrRuntimeInvalidTableAccess) + } else if copySize != 0 { + copy(dstTable[destinationOffset:], srcTable[sourceOffset:sourceOffset+copySize]) + } + frame.pc++ + case operationKindRefFunc: + ce.pushValue(uint64(uintptr(unsafe.Pointer(&functions[op.U1])))) + frame.pc++ + case operationKindTableGet: + table := tables[op.U1] + + offset := ce.popValue() + if offset >= uint64(len(table.References)) { + panic(wasmruntime.ErrRuntimeInvalidTableAccess) + } + + ce.pushValue(uint64(table.References[offset])) + frame.pc++ + case operationKindTableSet: + table := tables[op.U1] + ref := ce.popValue() + + offset := ce.popValue() + if offset >= uint64(len(table.References)) { + panic(wasmruntime.ErrRuntimeInvalidTableAccess) + } + + table.References[offset] = uintptr(ref) // externrefs are opaque uint64. + frame.pc++ + case operationKindTableSize: + table := tables[op.U1] + ce.pushValue(uint64(len(table.References))) + frame.pc++ + case operationKindTableGrow: + table := tables[op.U1] + num, ref := ce.popValue(), ce.popValue() + ret := table.Grow(uint32(num), uintptr(ref)) + ce.pushValue(uint64(ret)) + frame.pc++ + case operationKindTableFill: + table := tables[op.U1] + num := ce.popValue() + ref := uintptr(ce.popValue()) + offset := ce.popValue() + if num+offset > uint64(len(table.References)) { + panic(wasmruntime.ErrRuntimeInvalidTableAccess) + } else if num > 0 { + // Uses the copy trick for faster filling the region with the value. + // https://gist.github.com/taylorza/df2f89d5f9ab3ffd06865062a4cf015d + targetRegion := table.References[offset : offset+num] + targetRegion[0] = ref + for i := 1; i < len(targetRegion); i *= 2 { + copy(targetRegion[i:], targetRegion[:i]) + } + } + frame.pc++ + case operationKindV128Const: + lo, hi := op.U1, op.U2 + ce.pushValue(lo) + ce.pushValue(hi) + frame.pc++ + case operationKindV128Add: + yHigh, yLow := ce.popValue(), ce.popValue() + xHigh, xLow := ce.popValue(), ce.popValue() + switch op.B1 { + case shapeI8x16: + ce.pushValue( + uint64(uint8(xLow>>8)+uint8(yLow>>8))<<8 | uint64(uint8(xLow)+uint8(yLow)) | + uint64(uint8(xLow>>24)+uint8(yLow>>24))<<24 | uint64(uint8(xLow>>16)+uint8(yLow>>16))<<16 | + uint64(uint8(xLow>>40)+uint8(yLow>>40))<<40 | uint64(uint8(xLow>>32)+uint8(yLow>>32))<<32 | + uint64(uint8(xLow>>56)+uint8(yLow>>56))<<56 | uint64(uint8(xLow>>48)+uint8(yLow>>48))<<48, + ) + ce.pushValue( + uint64(uint8(xHigh>>8)+uint8(yHigh>>8))<<8 | uint64(uint8(xHigh)+uint8(yHigh)) | + uint64(uint8(xHigh>>24)+uint8(yHigh>>24))<<24 | uint64(uint8(xHigh>>16)+uint8(yHigh>>16))<<16 | + uint64(uint8(xHigh>>40)+uint8(yHigh>>40))<<40 | uint64(uint8(xHigh>>32)+uint8(yHigh>>32))<<32 | + uint64(uint8(xHigh>>56)+uint8(yHigh>>56))<<56 | uint64(uint8(xHigh>>48)+uint8(yHigh>>48))<<48, + ) + case shapeI16x8: + ce.pushValue( + uint64(uint16(xLow>>16+yLow>>16))<<16 | uint64(uint16(xLow)+uint16(yLow)) | + uint64(uint16(xLow>>48+yLow>>48))<<48 | uint64(uint16(xLow>>32+yLow>>32))<<32, + ) + ce.pushValue( + uint64(uint16(xHigh>>16)+uint16(yHigh>>16))<<16 | uint64(uint16(xHigh)+uint16(yHigh)) | + uint64(uint16(xHigh>>48)+uint16(yHigh>>48))<<48 | uint64(uint16(xHigh>>32)+uint16(yHigh>>32))<<32, + ) + case shapeI32x4: + ce.pushValue(uint64(uint32(xLow>>32)+uint32(yLow>>32))<<32 | uint64(uint32(xLow)+uint32(yLow))) + ce.pushValue(uint64(uint32(xHigh>>32)+uint32(yHigh>>32))<<32 | uint64(uint32(xHigh)+uint32(yHigh))) + case shapeI64x2: + ce.pushValue(xLow + yLow) + ce.pushValue(xHigh + yHigh) + case shapeF32x4: + ce.pushValue( + addFloat32bits(uint32(xLow), uint32(yLow)) | addFloat32bits(uint32(xLow>>32), uint32(yLow>>32))<<32, + ) + ce.pushValue( + addFloat32bits(uint32(xHigh), uint32(yHigh)) | addFloat32bits(uint32(xHigh>>32), uint32(yHigh>>32))<<32, + ) + case shapeF64x2: + ce.pushValue(math.Float64bits(math.Float64frombits(xLow) + math.Float64frombits(yLow))) + ce.pushValue(math.Float64bits(math.Float64frombits(xHigh) + math.Float64frombits(yHigh))) + } + frame.pc++ + case operationKindV128Sub: + yHigh, yLow := ce.popValue(), ce.popValue() + xHigh, xLow := ce.popValue(), ce.popValue() + switch op.B1 { + case shapeI8x16: + ce.pushValue( + uint64(uint8(xLow>>8)-uint8(yLow>>8))<<8 | uint64(uint8(xLow)-uint8(yLow)) | + uint64(uint8(xLow>>24)-uint8(yLow>>24))<<24 | uint64(uint8(xLow>>16)-uint8(yLow>>16))<<16 | + uint64(uint8(xLow>>40)-uint8(yLow>>40))<<40 | uint64(uint8(xLow>>32)-uint8(yLow>>32))<<32 | + uint64(uint8(xLow>>56)-uint8(yLow>>56))<<56 | uint64(uint8(xLow>>48)-uint8(yLow>>48))<<48, + ) + ce.pushValue( + uint64(uint8(xHigh>>8)-uint8(yHigh>>8))<<8 | uint64(uint8(xHigh)-uint8(yHigh)) | + uint64(uint8(xHigh>>24)-uint8(yHigh>>24))<<24 | uint64(uint8(xHigh>>16)-uint8(yHigh>>16))<<16 | + uint64(uint8(xHigh>>40)-uint8(yHigh>>40))<<40 | uint64(uint8(xHigh>>32)-uint8(yHigh>>32))<<32 | + uint64(uint8(xHigh>>56)-uint8(yHigh>>56))<<56 | uint64(uint8(xHigh>>48)-uint8(yHigh>>48))<<48, + ) + case shapeI16x8: + ce.pushValue( + uint64(uint16(xLow>>16)-uint16(yLow>>16))<<16 | uint64(uint16(xLow)-uint16(yLow)) | + uint64(uint16(xLow>>48)-uint16(yLow>>48))<<48 | uint64(uint16(xLow>>32)-uint16(yLow>>32))<<32, + ) + ce.pushValue( + uint64(uint16(xHigh>>16)-uint16(yHigh>>16))<<16 | uint64(uint16(xHigh)-uint16(yHigh)) | + uint64(uint16(xHigh>>48)-uint16(yHigh>>48))<<48 | uint64(uint16(xHigh>>32)-uint16(yHigh>>32))<<32, + ) + case shapeI32x4: + ce.pushValue(uint64(uint32(xLow>>32-yLow>>32))<<32 | uint64(uint32(xLow)-uint32(yLow))) + ce.pushValue(uint64(uint32(xHigh>>32-yHigh>>32))<<32 | uint64(uint32(xHigh)-uint32(yHigh))) + case shapeI64x2: + ce.pushValue(xLow - yLow) + ce.pushValue(xHigh - yHigh) + case shapeF32x4: + ce.pushValue( + subFloat32bits(uint32(xLow), uint32(yLow)) | subFloat32bits(uint32(xLow>>32), uint32(yLow>>32))<<32, + ) + ce.pushValue( + subFloat32bits(uint32(xHigh), uint32(yHigh)) | subFloat32bits(uint32(xHigh>>32), uint32(yHigh>>32))<<32, + ) + case shapeF64x2: + ce.pushValue(math.Float64bits(math.Float64frombits(xLow) - math.Float64frombits(yLow))) + ce.pushValue(math.Float64bits(math.Float64frombits(xHigh) - math.Float64frombits(yHigh))) + } + frame.pc++ + case operationKindV128Load: + offset := ce.popMemoryOffset(op) + switch op.B1 { + case v128LoadType128: + lo, ok := memoryInst.ReadUint64Le(offset) + if !ok { + panic(wasmruntime.ErrRuntimeOutOfBoundsMemoryAccess) + } + ce.pushValue(lo) + hi, ok := memoryInst.ReadUint64Le(offset + 8) + if !ok { + panic(wasmruntime.ErrRuntimeOutOfBoundsMemoryAccess) + } + ce.pushValue(hi) + case v128LoadType8x8s: + data, ok := memoryInst.Read(offset, 8) + if !ok { + panic(wasmruntime.ErrRuntimeOutOfBoundsMemoryAccess) + } + ce.pushValue( + uint64(uint16(int8(data[3])))<<48 | uint64(uint16(int8(data[2])))<<32 | uint64(uint16(int8(data[1])))<<16 | uint64(uint16(int8(data[0]))), + ) + ce.pushValue( + uint64(uint16(int8(data[7])))<<48 | uint64(uint16(int8(data[6])))<<32 | uint64(uint16(int8(data[5])))<<16 | uint64(uint16(int8(data[4]))), + ) + case v128LoadType8x8u: + data, ok := memoryInst.Read(offset, 8) + if !ok { + panic(wasmruntime.ErrRuntimeOutOfBoundsMemoryAccess) + } + ce.pushValue( + uint64(data[3])<<48 | uint64(data[2])<<32 | uint64(data[1])<<16 | uint64(data[0]), + ) + ce.pushValue( + uint64(data[7])<<48 | uint64(data[6])<<32 | uint64(data[5])<<16 | uint64(data[4]), + ) + case v128LoadType16x4s: + data, ok := memoryInst.Read(offset, 8) + if !ok { + panic(wasmruntime.ErrRuntimeOutOfBoundsMemoryAccess) + } + ce.pushValue( + uint64(int16(binary.LittleEndian.Uint16(data[2:])))<<32 | + uint64(uint32(int16(binary.LittleEndian.Uint16(data)))), + ) + ce.pushValue( + uint64(uint32(int16(binary.LittleEndian.Uint16(data[6:]))))<<32 | + uint64(uint32(int16(binary.LittleEndian.Uint16(data[4:])))), + ) + case v128LoadType16x4u: + data, ok := memoryInst.Read(offset, 8) + if !ok { + panic(wasmruntime.ErrRuntimeOutOfBoundsMemoryAccess) + } + ce.pushValue( + uint64(binary.LittleEndian.Uint16(data[2:]))<<32 | uint64(binary.LittleEndian.Uint16(data)), + ) + ce.pushValue( + uint64(binary.LittleEndian.Uint16(data[6:]))<<32 | uint64(binary.LittleEndian.Uint16(data[4:])), + ) + case v128LoadType32x2s: + data, ok := memoryInst.Read(offset, 8) + if !ok { + panic(wasmruntime.ErrRuntimeOutOfBoundsMemoryAccess) + } + ce.pushValue(uint64(int32(binary.LittleEndian.Uint32(data)))) + ce.pushValue(uint64(int32(binary.LittleEndian.Uint32(data[4:])))) + case v128LoadType32x2u: + data, ok := memoryInst.Read(offset, 8) + if !ok { + panic(wasmruntime.ErrRuntimeOutOfBoundsMemoryAccess) + } + ce.pushValue(uint64(binary.LittleEndian.Uint32(data))) + ce.pushValue(uint64(binary.LittleEndian.Uint32(data[4:]))) + case v128LoadType8Splat: + v, ok := memoryInst.ReadByte(offset) + if !ok { + panic(wasmruntime.ErrRuntimeOutOfBoundsMemoryAccess) + } + v8 := uint64(v)<<56 | uint64(v)<<48 | uint64(v)<<40 | uint64(v)<<32 | + uint64(v)<<24 | uint64(v)<<16 | uint64(v)<<8 | uint64(v) + ce.pushValue(v8) + ce.pushValue(v8) + case v128LoadType16Splat: + v, ok := memoryInst.ReadUint16Le(offset) + if !ok { + panic(wasmruntime.ErrRuntimeOutOfBoundsMemoryAccess) + } + v4 := uint64(v)<<48 | uint64(v)<<32 | uint64(v)<<16 | uint64(v) + ce.pushValue(v4) + ce.pushValue(v4) + case v128LoadType32Splat: + v, ok := memoryInst.ReadUint32Le(offset) + if !ok { + panic(wasmruntime.ErrRuntimeOutOfBoundsMemoryAccess) + } + vv := uint64(v)<<32 | uint64(v) + ce.pushValue(vv) + ce.pushValue(vv) + case v128LoadType64Splat: + lo, ok := memoryInst.ReadUint64Le(offset) + if !ok { + panic(wasmruntime.ErrRuntimeOutOfBoundsMemoryAccess) + } + ce.pushValue(lo) + ce.pushValue(lo) + case v128LoadType32zero: + lo, ok := memoryInst.ReadUint32Le(offset) + if !ok { + panic(wasmruntime.ErrRuntimeOutOfBoundsMemoryAccess) + } + ce.pushValue(uint64(lo)) + ce.pushValue(0) + case v128LoadType64zero: + lo, ok := memoryInst.ReadUint64Le(offset) + if !ok { + panic(wasmruntime.ErrRuntimeOutOfBoundsMemoryAccess) + } + ce.pushValue(lo) + ce.pushValue(0) + } + frame.pc++ + case operationKindV128LoadLane: + hi, lo := ce.popValue(), ce.popValue() + offset := ce.popMemoryOffset(op) + switch op.B1 { + case 8: + b, ok := memoryInst.ReadByte(offset) + if !ok { + panic(wasmruntime.ErrRuntimeOutOfBoundsMemoryAccess) + } + if op.B2 < 8 { + s := op.B2 << 3 + lo = (lo & ^(0xff << s)) | uint64(b)< math.MaxUint32 { + panic(wasmruntime.ErrRuntimeOutOfBoundsMemoryAccess) + } + if ok := memoryInst.WriteUint64Le(offset+8, hi); !ok { + panic(wasmruntime.ErrRuntimeOutOfBoundsMemoryAccess) + } + if ok := memoryInst.WriteUint64Le(offset, lo); !ok { + panic(wasmruntime.ErrRuntimeOutOfBoundsMemoryAccess) + } + frame.pc++ + case operationKindV128StoreLane: + hi, lo := ce.popValue(), ce.popValue() + offset := ce.popMemoryOffset(op) + var ok bool + switch op.B1 { + case 8: + if op.B2 < 8 { + ok = memoryInst.WriteByte(offset, byte(lo>>(op.B2*8))) + } else { + ok = memoryInst.WriteByte(offset, byte(hi>>((op.B2-8)*8))) + } + case 16: + if op.B2 < 4 { + ok = memoryInst.WriteUint16Le(offset, uint16(lo>>(op.B2*16))) + } else { + ok = memoryInst.WriteUint16Le(offset, uint16(hi>>((op.B2-4)*16))) + } + case 32: + if op.B2 < 2 { + ok = memoryInst.WriteUint32Le(offset, uint32(lo>>(op.B2*32))) + } else { + ok = memoryInst.WriteUint32Le(offset, uint32(hi>>((op.B2-2)*32))) + } + case 64: + if op.B2 == 0 { + ok = memoryInst.WriteUint64Le(offset, lo) + } else { + ok = memoryInst.WriteUint64Le(offset, hi) + } + } + if !ok { + panic(wasmruntime.ErrRuntimeOutOfBoundsMemoryAccess) + } + frame.pc++ + case operationKindV128ReplaceLane: + v := ce.popValue() + hi, lo := ce.popValue(), ce.popValue() + switch op.B1 { + case shapeI8x16: + if op.B2 < 8 { + s := op.B2 << 3 + lo = (lo & ^(0xff << s)) | uint64(byte(v))<> (op.B2 * 8)) + } else { + u8 = byte(hi >> ((op.B2 - 8) * 8)) + } + if op.B3 { + // sign-extend. + v = uint64(uint32(int8(u8))) + } else { + v = uint64(u8) + } + case shapeI16x8: + var u16 uint16 + if op.B2 < 4 { + u16 = uint16(lo >> (op.B2 * 16)) + } else { + u16 = uint16(hi >> ((op.B2 - 4) * 16)) + } + if op.B3 { + // sign-extend. + v = uint64(uint32(int16(u16))) + } else { + v = uint64(u16) + } + case shapeI32x4, shapeF32x4: + if op.B2 < 2 { + v = uint64(uint32(lo >> (op.B2 * 32))) + } else { + v = uint64(uint32(hi >> ((op.B2 - 2) * 32))) + } + case shapeI64x2, shapeF64x2: + if op.B2 == 0 { + v = lo + } else { + v = hi + } + } + ce.pushValue(v) + frame.pc++ + case operationKindV128Splat: + v := ce.popValue() + var hi, lo uint64 + switch op.B1 { + case shapeI8x16: + v8 := uint64(byte(v))<<56 | uint64(byte(v))<<48 | uint64(byte(v))<<40 | uint64(byte(v))<<32 | + uint64(byte(v))<<24 | uint64(byte(v))<<16 | uint64(byte(v))<<8 | uint64(byte(v)) + hi, lo = v8, v8 + case shapeI16x8: + v4 := uint64(uint16(v))<<48 | uint64(uint16(v))<<32 | uint64(uint16(v))<<16 | uint64(uint16(v)) + hi, lo = v4, v4 + case shapeI32x4, shapeF32x4: + v2 := uint64(uint32(v))<<32 | uint64(uint32(v)) + lo, hi = v2, v2 + case shapeI64x2, shapeF64x2: + lo, hi = v, v + } + ce.pushValue(lo) + ce.pushValue(hi) + frame.pc++ + case operationKindV128Swizzle: + idxHi, idxLo := ce.popValue(), ce.popValue() + baseHi, baseLo := ce.popValue(), ce.popValue() + var newVal [16]byte + for i := 0; i < 16; i++ { + var id byte + if i < 8 { + id = byte(idxLo >> (i * 8)) + } else { + id = byte(idxHi >> ((i - 8) * 8)) + } + if id < 8 { + newVal[i] = byte(baseLo >> (id * 8)) + } else if id < 16 { + newVal[i] = byte(baseHi >> ((id - 8) * 8)) + } + } + ce.pushValue(binary.LittleEndian.Uint64(newVal[:8])) + ce.pushValue(binary.LittleEndian.Uint64(newVal[8:])) + frame.pc++ + case operationKindV128Shuffle: + xHi, xLo, yHi, yLo := ce.popValue(), ce.popValue(), ce.popValue(), ce.popValue() + var newVal [16]byte + for i, l := range op.Us { + if l < 8 { + newVal[i] = byte(yLo >> (l * 8)) + } else if l < 16 { + newVal[i] = byte(yHi >> ((l - 8) * 8)) + } else if l < 24 { + newVal[i] = byte(xLo >> ((l - 16) * 8)) + } else if l < 32 { + newVal[i] = byte(xHi >> ((l - 24) * 8)) + } + } + ce.pushValue(binary.LittleEndian.Uint64(newVal[:8])) + ce.pushValue(binary.LittleEndian.Uint64(newVal[8:])) + frame.pc++ + case operationKindV128AnyTrue: + hi, lo := ce.popValue(), ce.popValue() + if hi != 0 || lo != 0 { + ce.pushValue(1) + } else { + ce.pushValue(0) + } + frame.pc++ + case operationKindV128AllTrue: + hi, lo := ce.popValue(), ce.popValue() + var ret bool + switch op.B1 { + case shapeI8x16: + ret = (uint8(lo) != 0) && (uint8(lo>>8) != 0) && (uint8(lo>>16) != 0) && (uint8(lo>>24) != 0) && + (uint8(lo>>32) != 0) && (uint8(lo>>40) != 0) && (uint8(lo>>48) != 0) && (uint8(lo>>56) != 0) && + (uint8(hi) != 0) && (uint8(hi>>8) != 0) && (uint8(hi>>16) != 0) && (uint8(hi>>24) != 0) && + (uint8(hi>>32) != 0) && (uint8(hi>>40) != 0) && (uint8(hi>>48) != 0) && (uint8(hi>>56) != 0) + case shapeI16x8: + ret = (uint16(lo) != 0) && (uint16(lo>>16) != 0) && (uint16(lo>>32) != 0) && (uint16(lo>>48) != 0) && + (uint16(hi) != 0) && (uint16(hi>>16) != 0) && (uint16(hi>>32) != 0) && (uint16(hi>>48) != 0) + case shapeI32x4: + ret = (uint32(lo) != 0) && (uint32(lo>>32) != 0) && + (uint32(hi) != 0) && (uint32(hi>>32) != 0) + case shapeI64x2: + ret = (lo != 0) && + (hi != 0) + } + if ret { + ce.pushValue(1) + } else { + ce.pushValue(0) + } + frame.pc++ + case operationKindV128BitMask: + // https://github.com/WebAssembly/spec/blob/wg-2.0.draft1/proposals/simd/SIMD.md#bitmask-extraction + hi, lo := ce.popValue(), ce.popValue() + var res uint64 + switch op.B1 { + case shapeI8x16: + for i := 0; i < 8; i++ { + if int8(lo>>(i*8)) < 0 { + res |= 1 << i + } + } + for i := 0; i < 8; i++ { + if int8(hi>>(i*8)) < 0 { + res |= 1 << (i + 8) + } + } + case shapeI16x8: + for i := 0; i < 4; i++ { + if int16(lo>>(i*16)) < 0 { + res |= 1 << i + } + } + for i := 0; i < 4; i++ { + if int16(hi>>(i*16)) < 0 { + res |= 1 << (i + 4) + } + } + case shapeI32x4: + for i := 0; i < 2; i++ { + if int32(lo>>(i*32)) < 0 { + res |= 1 << i + } + } + for i := 0; i < 2; i++ { + if int32(hi>>(i*32)) < 0 { + res |= 1 << (i + 2) + } + } + case shapeI64x2: + if int64(lo) < 0 { + res |= 0b01 + } + if int(hi) < 0 { + res |= 0b10 + } + } + ce.pushValue(res) + frame.pc++ + case operationKindV128And: + x2Hi, x2Lo := ce.popValue(), ce.popValue() + x1Hi, x1Lo := ce.popValue(), ce.popValue() + ce.pushValue(x1Lo & x2Lo) + ce.pushValue(x1Hi & x2Hi) + frame.pc++ + case operationKindV128Not: + hi, lo := ce.popValue(), ce.popValue() + ce.pushValue(^lo) + ce.pushValue(^hi) + frame.pc++ + case operationKindV128Or: + x2Hi, x2Lo := ce.popValue(), ce.popValue() + x1Hi, x1Lo := ce.popValue(), ce.popValue() + ce.pushValue(x1Lo | x2Lo) + ce.pushValue(x1Hi | x2Hi) + frame.pc++ + case operationKindV128Xor: + x2Hi, x2Lo := ce.popValue(), ce.popValue() + x1Hi, x1Lo := ce.popValue(), ce.popValue() + ce.pushValue(x1Lo ^ x2Lo) + ce.pushValue(x1Hi ^ x2Hi) + frame.pc++ + case operationKindV128Bitselect: + // https://github.com/WebAssembly/spec/blob/wg-2.0.draft1/proposals/simd/SIMD.md#bitwise-select + cHi, cLo := ce.popValue(), ce.popValue() + x2Hi, x2Lo := ce.popValue(), ce.popValue() + x1Hi, x1Lo := ce.popValue(), ce.popValue() + // v128.or(v128.and(v1, c), v128.and(v2, v128.not(c))) + ce.pushValue((x1Lo & cLo) | (x2Lo & (^cLo))) + ce.pushValue((x1Hi & cHi) | (x2Hi & (^cHi))) + frame.pc++ + case operationKindV128AndNot: + x2Hi, x2Lo := ce.popValue(), ce.popValue() + x1Hi, x1Lo := ce.popValue(), ce.popValue() + ce.pushValue(x1Lo & (^x2Lo)) + ce.pushValue(x1Hi & (^x2Hi)) + frame.pc++ + case operationKindV128Shl: + s := ce.popValue() + hi, lo := ce.popValue(), ce.popValue() + switch op.B1 { + case shapeI8x16: + s = s % 8 + lo = uint64(uint8(lo<>8)<>16)<>24)<>32)<>40)<>48)<>56)<>8)<>16)<>24)<>32)<>40)<>48)<>56)<>16)<>32)<>48)<>16)<>32)<>48)<>32)<>32)<>s)) | + uint64(uint8(int8(lo>>8)>>s))<<8 | + uint64(uint8(int8(lo>>16)>>s))<<16 | + uint64(uint8(int8(lo>>24)>>s))<<24 | + uint64(uint8(int8(lo>>32)>>s))<<32 | + uint64(uint8(int8(lo>>40)>>s))<<40 | + uint64(uint8(int8(lo>>48)>>s))<<48 | + uint64(uint8(int8(lo>>56)>>s))<<56 + hi = uint64(uint8(int8(hi)>>s)) | + uint64(uint8(int8(hi>>8)>>s))<<8 | + uint64(uint8(int8(hi>>16)>>s))<<16 | + uint64(uint8(int8(hi>>24)>>s))<<24 | + uint64(uint8(int8(hi>>32)>>s))<<32 | + uint64(uint8(int8(hi>>40)>>s))<<40 | + uint64(uint8(int8(hi>>48)>>s))<<48 | + uint64(uint8(int8(hi>>56)>>s))<<56 + } else { + lo = uint64(uint8(lo)>>s) | + uint64(uint8(lo>>8)>>s)<<8 | + uint64(uint8(lo>>16)>>s)<<16 | + uint64(uint8(lo>>24)>>s)<<24 | + uint64(uint8(lo>>32)>>s)<<32 | + uint64(uint8(lo>>40)>>s)<<40 | + uint64(uint8(lo>>48)>>s)<<48 | + uint64(uint8(lo>>56)>>s)<<56 + hi = uint64(uint8(hi)>>s) | + uint64(uint8(hi>>8)>>s)<<8 | + uint64(uint8(hi>>16)>>s)<<16 | + uint64(uint8(hi>>24)>>s)<<24 | + uint64(uint8(hi>>32)>>s)<<32 | + uint64(uint8(hi>>40)>>s)<<40 | + uint64(uint8(hi>>48)>>s)<<48 | + uint64(uint8(hi>>56)>>s)<<56 + } + case shapeI16x8: + s = s % 16 + if op.B3 { // signed + lo = uint64(uint16(int16(lo)>>s)) | + uint64(uint16(int16(lo>>16)>>s))<<16 | + uint64(uint16(int16(lo>>32)>>s))<<32 | + uint64(uint16(int16(lo>>48)>>s))<<48 + hi = uint64(uint16(int16(hi)>>s)) | + uint64(uint16(int16(hi>>16)>>s))<<16 | + uint64(uint16(int16(hi>>32)>>s))<<32 | + uint64(uint16(int16(hi>>48)>>s))<<48 + } else { + lo = uint64(uint16(lo)>>s) | + uint64(uint16(lo>>16)>>s)<<16 | + uint64(uint16(lo>>32)>>s)<<32 | + uint64(uint16(lo>>48)>>s)<<48 + hi = uint64(uint16(hi)>>s) | + uint64(uint16(hi>>16)>>s)<<16 | + uint64(uint16(hi>>32)>>s)<<32 | + uint64(uint16(hi>>48)>>s)<<48 + } + case shapeI32x4: + s = s % 32 + if op.B3 { + lo = uint64(uint32(int32(lo)>>s)) | uint64(uint32(int32(lo>>32)>>s))<<32 + hi = uint64(uint32(int32(hi)>>s)) | uint64(uint32(int32(hi>>32)>>s))<<32 + } else { + lo = uint64(uint32(lo)>>s) | uint64(uint32(lo>>32)>>s)<<32 + hi = uint64(uint32(hi)>>s) | uint64(uint32(hi>>32)>>s)<<32 + } + case shapeI64x2: + s = s % 64 + if op.B3 { // signed + lo = uint64(int64(lo) >> s) + hi = uint64(int64(hi) >> s) + } else { + lo = lo >> s + hi = hi >> s + } + + } + ce.pushValue(lo) + ce.pushValue(hi) + frame.pc++ + case operationKindV128Cmp: + x2Hi, x2Lo := ce.popValue(), ce.popValue() + x1Hi, x1Lo := ce.popValue(), ce.popValue() + var result []bool + switch op.B1 { + case v128CmpTypeI8x16Eq: + result = []bool{ + byte(x1Lo>>0) == byte(x2Lo>>0), byte(x1Lo>>8) == byte(x2Lo>>8), + byte(x1Lo>>16) == byte(x2Lo>>16), byte(x1Lo>>24) == byte(x2Lo>>24), + byte(x1Lo>>32) == byte(x2Lo>>32), byte(x1Lo>>40) == byte(x2Lo>>40), + byte(x1Lo>>48) == byte(x2Lo>>48), byte(x1Lo>>56) == byte(x2Lo>>56), + byte(x1Hi>>0) == byte(x2Hi>>0), byte(x1Hi>>8) == byte(x2Hi>>8), + byte(x1Hi>>16) == byte(x2Hi>>16), byte(x1Hi>>24) == byte(x2Hi>>24), + byte(x1Hi>>32) == byte(x2Hi>>32), byte(x1Hi>>40) == byte(x2Hi>>40), + byte(x1Hi>>48) == byte(x2Hi>>48), byte(x1Hi>>56) == byte(x2Hi>>56), + } + case v128CmpTypeI8x16Ne: + result = []bool{ + byte(x1Lo>>0) != byte(x2Lo>>0), byte(x1Lo>>8) != byte(x2Lo>>8), + byte(x1Lo>>16) != byte(x2Lo>>16), byte(x1Lo>>24) != byte(x2Lo>>24), + byte(x1Lo>>32) != byte(x2Lo>>32), byte(x1Lo>>40) != byte(x2Lo>>40), + byte(x1Lo>>48) != byte(x2Lo>>48), byte(x1Lo>>56) != byte(x2Lo>>56), + byte(x1Hi>>0) != byte(x2Hi>>0), byte(x1Hi>>8) != byte(x2Hi>>8), + byte(x1Hi>>16) != byte(x2Hi>>16), byte(x1Hi>>24) != byte(x2Hi>>24), + byte(x1Hi>>32) != byte(x2Hi>>32), byte(x1Hi>>40) != byte(x2Hi>>40), + byte(x1Hi>>48) != byte(x2Hi>>48), byte(x1Hi>>56) != byte(x2Hi>>56), + } + case v128CmpTypeI8x16LtS: + result = []bool{ + int8(x1Lo>>0) < int8(x2Lo>>0), int8(x1Lo>>8) < int8(x2Lo>>8), + int8(x1Lo>>16) < int8(x2Lo>>16), int8(x1Lo>>24) < int8(x2Lo>>24), + int8(x1Lo>>32) < int8(x2Lo>>32), int8(x1Lo>>40) < int8(x2Lo>>40), + int8(x1Lo>>48) < int8(x2Lo>>48), int8(x1Lo>>56) < int8(x2Lo>>56), + int8(x1Hi>>0) < int8(x2Hi>>0), int8(x1Hi>>8) < int8(x2Hi>>8), + int8(x1Hi>>16) < int8(x2Hi>>16), int8(x1Hi>>24) < int8(x2Hi>>24), + int8(x1Hi>>32) < int8(x2Hi>>32), int8(x1Hi>>40) < int8(x2Hi>>40), + int8(x1Hi>>48) < int8(x2Hi>>48), int8(x1Hi>>56) < int8(x2Hi>>56), + } + case v128CmpTypeI8x16LtU: + result = []bool{ + byte(x1Lo>>0) < byte(x2Lo>>0), byte(x1Lo>>8) < byte(x2Lo>>8), + byte(x1Lo>>16) < byte(x2Lo>>16), byte(x1Lo>>24) < byte(x2Lo>>24), + byte(x1Lo>>32) < byte(x2Lo>>32), byte(x1Lo>>40) < byte(x2Lo>>40), + byte(x1Lo>>48) < byte(x2Lo>>48), byte(x1Lo>>56) < byte(x2Lo>>56), + byte(x1Hi>>0) < byte(x2Hi>>0), byte(x1Hi>>8) < byte(x2Hi>>8), + byte(x1Hi>>16) < byte(x2Hi>>16), byte(x1Hi>>24) < byte(x2Hi>>24), + byte(x1Hi>>32) < byte(x2Hi>>32), byte(x1Hi>>40) < byte(x2Hi>>40), + byte(x1Hi>>48) < byte(x2Hi>>48), byte(x1Hi>>56) < byte(x2Hi>>56), + } + case v128CmpTypeI8x16GtS: + result = []bool{ + int8(x1Lo>>0) > int8(x2Lo>>0), int8(x1Lo>>8) > int8(x2Lo>>8), + int8(x1Lo>>16) > int8(x2Lo>>16), int8(x1Lo>>24) > int8(x2Lo>>24), + int8(x1Lo>>32) > int8(x2Lo>>32), int8(x1Lo>>40) > int8(x2Lo>>40), + int8(x1Lo>>48) > int8(x2Lo>>48), int8(x1Lo>>56) > int8(x2Lo>>56), + int8(x1Hi>>0) > int8(x2Hi>>0), int8(x1Hi>>8) > int8(x2Hi>>8), + int8(x1Hi>>16) > int8(x2Hi>>16), int8(x1Hi>>24) > int8(x2Hi>>24), + int8(x1Hi>>32) > int8(x2Hi>>32), int8(x1Hi>>40) > int8(x2Hi>>40), + int8(x1Hi>>48) > int8(x2Hi>>48), int8(x1Hi>>56) > int8(x2Hi>>56), + } + case v128CmpTypeI8x16GtU: + result = []bool{ + byte(x1Lo>>0) > byte(x2Lo>>0), byte(x1Lo>>8) > byte(x2Lo>>8), + byte(x1Lo>>16) > byte(x2Lo>>16), byte(x1Lo>>24) > byte(x2Lo>>24), + byte(x1Lo>>32) > byte(x2Lo>>32), byte(x1Lo>>40) > byte(x2Lo>>40), + byte(x1Lo>>48) > byte(x2Lo>>48), byte(x1Lo>>56) > byte(x2Lo>>56), + byte(x1Hi>>0) > byte(x2Hi>>0), byte(x1Hi>>8) > byte(x2Hi>>8), + byte(x1Hi>>16) > byte(x2Hi>>16), byte(x1Hi>>24) > byte(x2Hi>>24), + byte(x1Hi>>32) > byte(x2Hi>>32), byte(x1Hi>>40) > byte(x2Hi>>40), + byte(x1Hi>>48) > byte(x2Hi>>48), byte(x1Hi>>56) > byte(x2Hi>>56), + } + case v128CmpTypeI8x16LeS: + result = []bool{ + int8(x1Lo>>0) <= int8(x2Lo>>0), int8(x1Lo>>8) <= int8(x2Lo>>8), + int8(x1Lo>>16) <= int8(x2Lo>>16), int8(x1Lo>>24) <= int8(x2Lo>>24), + int8(x1Lo>>32) <= int8(x2Lo>>32), int8(x1Lo>>40) <= int8(x2Lo>>40), + int8(x1Lo>>48) <= int8(x2Lo>>48), int8(x1Lo>>56) <= int8(x2Lo>>56), + int8(x1Hi>>0) <= int8(x2Hi>>0), int8(x1Hi>>8) <= int8(x2Hi>>8), + int8(x1Hi>>16) <= int8(x2Hi>>16), int8(x1Hi>>24) <= int8(x2Hi>>24), + int8(x1Hi>>32) <= int8(x2Hi>>32), int8(x1Hi>>40) <= int8(x2Hi>>40), + int8(x1Hi>>48) <= int8(x2Hi>>48), int8(x1Hi>>56) <= int8(x2Hi>>56), + } + case v128CmpTypeI8x16LeU: + result = []bool{ + byte(x1Lo>>0) <= byte(x2Lo>>0), byte(x1Lo>>8) <= byte(x2Lo>>8), + byte(x1Lo>>16) <= byte(x2Lo>>16), byte(x1Lo>>24) <= byte(x2Lo>>24), + byte(x1Lo>>32) <= byte(x2Lo>>32), byte(x1Lo>>40) <= byte(x2Lo>>40), + byte(x1Lo>>48) <= byte(x2Lo>>48), byte(x1Lo>>56) <= byte(x2Lo>>56), + byte(x1Hi>>0) <= byte(x2Hi>>0), byte(x1Hi>>8) <= byte(x2Hi>>8), + byte(x1Hi>>16) <= byte(x2Hi>>16), byte(x1Hi>>24) <= byte(x2Hi>>24), + byte(x1Hi>>32) <= byte(x2Hi>>32), byte(x1Hi>>40) <= byte(x2Hi>>40), + byte(x1Hi>>48) <= byte(x2Hi>>48), byte(x1Hi>>56) <= byte(x2Hi>>56), + } + case v128CmpTypeI8x16GeS: + result = []bool{ + int8(x1Lo>>0) >= int8(x2Lo>>0), int8(x1Lo>>8) >= int8(x2Lo>>8), + int8(x1Lo>>16) >= int8(x2Lo>>16), int8(x1Lo>>24) >= int8(x2Lo>>24), + int8(x1Lo>>32) >= int8(x2Lo>>32), int8(x1Lo>>40) >= int8(x2Lo>>40), + int8(x1Lo>>48) >= int8(x2Lo>>48), int8(x1Lo>>56) >= int8(x2Lo>>56), + int8(x1Hi>>0) >= int8(x2Hi>>0), int8(x1Hi>>8) >= int8(x2Hi>>8), + int8(x1Hi>>16) >= int8(x2Hi>>16), int8(x1Hi>>24) >= int8(x2Hi>>24), + int8(x1Hi>>32) >= int8(x2Hi>>32), int8(x1Hi>>40) >= int8(x2Hi>>40), + int8(x1Hi>>48) >= int8(x2Hi>>48), int8(x1Hi>>56) >= int8(x2Hi>>56), + } + case v128CmpTypeI8x16GeU: + result = []bool{ + byte(x1Lo>>0) >= byte(x2Lo>>0), byte(x1Lo>>8) >= byte(x2Lo>>8), + byte(x1Lo>>16) >= byte(x2Lo>>16), byte(x1Lo>>24) >= byte(x2Lo>>24), + byte(x1Lo>>32) >= byte(x2Lo>>32), byte(x1Lo>>40) >= byte(x2Lo>>40), + byte(x1Lo>>48) >= byte(x2Lo>>48), byte(x1Lo>>56) >= byte(x2Lo>>56), + byte(x1Hi>>0) >= byte(x2Hi>>0), byte(x1Hi>>8) >= byte(x2Hi>>8), + byte(x1Hi>>16) >= byte(x2Hi>>16), byte(x1Hi>>24) >= byte(x2Hi>>24), + byte(x1Hi>>32) >= byte(x2Hi>>32), byte(x1Hi>>40) >= byte(x2Hi>>40), + byte(x1Hi>>48) >= byte(x2Hi>>48), byte(x1Hi>>56) >= byte(x2Hi>>56), + } + case v128CmpTypeI16x8Eq: + result = []bool{ + uint16(x1Lo>>0) == uint16(x2Lo>>0), uint16(x1Lo>>16) == uint16(x2Lo>>16), + uint16(x1Lo>>32) == uint16(x2Lo>>32), uint16(x1Lo>>48) == uint16(x2Lo>>48), + uint16(x1Hi>>0) == uint16(x2Hi>>0), uint16(x1Hi>>16) == uint16(x2Hi>>16), + uint16(x1Hi>>32) == uint16(x2Hi>>32), uint16(x1Hi>>48) == uint16(x2Hi>>48), + } + case v128CmpTypeI16x8Ne: + result = []bool{ + uint16(x1Lo>>0) != uint16(x2Lo>>0), uint16(x1Lo>>16) != uint16(x2Lo>>16), + uint16(x1Lo>>32) != uint16(x2Lo>>32), uint16(x1Lo>>48) != uint16(x2Lo>>48), + uint16(x1Hi>>0) != uint16(x2Hi>>0), uint16(x1Hi>>16) != uint16(x2Hi>>16), + uint16(x1Hi>>32) != uint16(x2Hi>>32), uint16(x1Hi>>48) != uint16(x2Hi>>48), + } + case v128CmpTypeI16x8LtS: + result = []bool{ + int16(x1Lo>>0) < int16(x2Lo>>0), int16(x1Lo>>16) < int16(x2Lo>>16), + int16(x1Lo>>32) < int16(x2Lo>>32), int16(x1Lo>>48) < int16(x2Lo>>48), + int16(x1Hi>>0) < int16(x2Hi>>0), int16(x1Hi>>16) < int16(x2Hi>>16), + int16(x1Hi>>32) < int16(x2Hi>>32), int16(x1Hi>>48) < int16(x2Hi>>48), + } + case v128CmpTypeI16x8LtU: + result = []bool{ + uint16(x1Lo>>0) < uint16(x2Lo>>0), uint16(x1Lo>>16) < uint16(x2Lo>>16), + uint16(x1Lo>>32) < uint16(x2Lo>>32), uint16(x1Lo>>48) < uint16(x2Lo>>48), + uint16(x1Hi>>0) < uint16(x2Hi>>0), uint16(x1Hi>>16) < uint16(x2Hi>>16), + uint16(x1Hi>>32) < uint16(x2Hi>>32), uint16(x1Hi>>48) < uint16(x2Hi>>48), + } + case v128CmpTypeI16x8GtS: + result = []bool{ + int16(x1Lo>>0) > int16(x2Lo>>0), int16(x1Lo>>16) > int16(x2Lo>>16), + int16(x1Lo>>32) > int16(x2Lo>>32), int16(x1Lo>>48) > int16(x2Lo>>48), + int16(x1Hi>>0) > int16(x2Hi>>0), int16(x1Hi>>16) > int16(x2Hi>>16), + int16(x1Hi>>32) > int16(x2Hi>>32), int16(x1Hi>>48) > int16(x2Hi>>48), + } + case v128CmpTypeI16x8GtU: + result = []bool{ + uint16(x1Lo>>0) > uint16(x2Lo>>0), uint16(x1Lo>>16) > uint16(x2Lo>>16), + uint16(x1Lo>>32) > uint16(x2Lo>>32), uint16(x1Lo>>48) > uint16(x2Lo>>48), + uint16(x1Hi>>0) > uint16(x2Hi>>0), uint16(x1Hi>>16) > uint16(x2Hi>>16), + uint16(x1Hi>>32) > uint16(x2Hi>>32), uint16(x1Hi>>48) > uint16(x2Hi>>48), + } + case v128CmpTypeI16x8LeS: + result = []bool{ + int16(x1Lo>>0) <= int16(x2Lo>>0), int16(x1Lo>>16) <= int16(x2Lo>>16), + int16(x1Lo>>32) <= int16(x2Lo>>32), int16(x1Lo>>48) <= int16(x2Lo>>48), + int16(x1Hi>>0) <= int16(x2Hi>>0), int16(x1Hi>>16) <= int16(x2Hi>>16), + int16(x1Hi>>32) <= int16(x2Hi>>32), int16(x1Hi>>48) <= int16(x2Hi>>48), + } + case v128CmpTypeI16x8LeU: + result = []bool{ + uint16(x1Lo>>0) <= uint16(x2Lo>>0), uint16(x1Lo>>16) <= uint16(x2Lo>>16), + uint16(x1Lo>>32) <= uint16(x2Lo>>32), uint16(x1Lo>>48) <= uint16(x2Lo>>48), + uint16(x1Hi>>0) <= uint16(x2Hi>>0), uint16(x1Hi>>16) <= uint16(x2Hi>>16), + uint16(x1Hi>>32) <= uint16(x2Hi>>32), uint16(x1Hi>>48) <= uint16(x2Hi>>48), + } + case v128CmpTypeI16x8GeS: + result = []bool{ + int16(x1Lo>>0) >= int16(x2Lo>>0), int16(x1Lo>>16) >= int16(x2Lo>>16), + int16(x1Lo>>32) >= int16(x2Lo>>32), int16(x1Lo>>48) >= int16(x2Lo>>48), + int16(x1Hi>>0) >= int16(x2Hi>>0), int16(x1Hi>>16) >= int16(x2Hi>>16), + int16(x1Hi>>32) >= int16(x2Hi>>32), int16(x1Hi>>48) >= int16(x2Hi>>48), + } + case v128CmpTypeI16x8GeU: + result = []bool{ + uint16(x1Lo>>0) >= uint16(x2Lo>>0), uint16(x1Lo>>16) >= uint16(x2Lo>>16), + uint16(x1Lo>>32) >= uint16(x2Lo>>32), uint16(x1Lo>>48) >= uint16(x2Lo>>48), + uint16(x1Hi>>0) >= uint16(x2Hi>>0), uint16(x1Hi>>16) >= uint16(x2Hi>>16), + uint16(x1Hi>>32) >= uint16(x2Hi>>32), uint16(x1Hi>>48) >= uint16(x2Hi>>48), + } + case v128CmpTypeI32x4Eq: + result = []bool{ + uint32(x1Lo>>0) == uint32(x2Lo>>0), uint32(x1Lo>>32) == uint32(x2Lo>>32), + uint32(x1Hi>>0) == uint32(x2Hi>>0), uint32(x1Hi>>32) == uint32(x2Hi>>32), + } + case v128CmpTypeI32x4Ne: + result = []bool{ + uint32(x1Lo>>0) != uint32(x2Lo>>0), uint32(x1Lo>>32) != uint32(x2Lo>>32), + uint32(x1Hi>>0) != uint32(x2Hi>>0), uint32(x1Hi>>32) != uint32(x2Hi>>32), + } + case v128CmpTypeI32x4LtS: + result = []bool{ + int32(x1Lo>>0) < int32(x2Lo>>0), int32(x1Lo>>32) < int32(x2Lo>>32), + int32(x1Hi>>0) < int32(x2Hi>>0), int32(x1Hi>>32) < int32(x2Hi>>32), + } + case v128CmpTypeI32x4LtU: + result = []bool{ + uint32(x1Lo>>0) < uint32(x2Lo>>0), uint32(x1Lo>>32) < uint32(x2Lo>>32), + uint32(x1Hi>>0) < uint32(x2Hi>>0), uint32(x1Hi>>32) < uint32(x2Hi>>32), + } + case v128CmpTypeI32x4GtS: + result = []bool{ + int32(x1Lo>>0) > int32(x2Lo>>0), int32(x1Lo>>32) > int32(x2Lo>>32), + int32(x1Hi>>0) > int32(x2Hi>>0), int32(x1Hi>>32) > int32(x2Hi>>32), + } + case v128CmpTypeI32x4GtU: + result = []bool{ + uint32(x1Lo>>0) > uint32(x2Lo>>0), uint32(x1Lo>>32) > uint32(x2Lo>>32), + uint32(x1Hi>>0) > uint32(x2Hi>>0), uint32(x1Hi>>32) > uint32(x2Hi>>32), + } + case v128CmpTypeI32x4LeS: + result = []bool{ + int32(x1Lo>>0) <= int32(x2Lo>>0), int32(x1Lo>>32) <= int32(x2Lo>>32), + int32(x1Hi>>0) <= int32(x2Hi>>0), int32(x1Hi>>32) <= int32(x2Hi>>32), + } + case v128CmpTypeI32x4LeU: + result = []bool{ + uint32(x1Lo>>0) <= uint32(x2Lo>>0), uint32(x1Lo>>32) <= uint32(x2Lo>>32), + uint32(x1Hi>>0) <= uint32(x2Hi>>0), uint32(x1Hi>>32) <= uint32(x2Hi>>32), + } + case v128CmpTypeI32x4GeS: + result = []bool{ + int32(x1Lo>>0) >= int32(x2Lo>>0), int32(x1Lo>>32) >= int32(x2Lo>>32), + int32(x1Hi>>0) >= int32(x2Hi>>0), int32(x1Hi>>32) >= int32(x2Hi>>32), + } + case v128CmpTypeI32x4GeU: + result = []bool{ + uint32(x1Lo>>0) >= uint32(x2Lo>>0), uint32(x1Lo>>32) >= uint32(x2Lo>>32), + uint32(x1Hi>>0) >= uint32(x2Hi>>0), uint32(x1Hi>>32) >= uint32(x2Hi>>32), + } + case v128CmpTypeI64x2Eq: + result = []bool{x1Lo == x2Lo, x1Hi == x2Hi} + case v128CmpTypeI64x2Ne: + result = []bool{x1Lo != x2Lo, x1Hi != x2Hi} + case v128CmpTypeI64x2LtS: + result = []bool{int64(x1Lo) < int64(x2Lo), int64(x1Hi) < int64(x2Hi)} + case v128CmpTypeI64x2GtS: + result = []bool{int64(x1Lo) > int64(x2Lo), int64(x1Hi) > int64(x2Hi)} + case v128CmpTypeI64x2LeS: + result = []bool{int64(x1Lo) <= int64(x2Lo), int64(x1Hi) <= int64(x2Hi)} + case v128CmpTypeI64x2GeS: + result = []bool{int64(x1Lo) >= int64(x2Lo), int64(x1Hi) >= int64(x2Hi)} + case v128CmpTypeF32x4Eq: + result = []bool{ + math.Float32frombits(uint32(x1Lo>>0)) == math.Float32frombits(uint32(x2Lo>>0)), + math.Float32frombits(uint32(x1Lo>>32)) == math.Float32frombits(uint32(x2Lo>>32)), + math.Float32frombits(uint32(x1Hi>>0)) == math.Float32frombits(uint32(x2Hi>>0)), + math.Float32frombits(uint32(x1Hi>>32)) == math.Float32frombits(uint32(x2Hi>>32)), + } + case v128CmpTypeF32x4Ne: + result = []bool{ + math.Float32frombits(uint32(x1Lo>>0)) != math.Float32frombits(uint32(x2Lo>>0)), + math.Float32frombits(uint32(x1Lo>>32)) != math.Float32frombits(uint32(x2Lo>>32)), + math.Float32frombits(uint32(x1Hi>>0)) != math.Float32frombits(uint32(x2Hi>>0)), + math.Float32frombits(uint32(x1Hi>>32)) != math.Float32frombits(uint32(x2Hi>>32)), + } + case v128CmpTypeF32x4Lt: + result = []bool{ + math.Float32frombits(uint32(x1Lo>>0)) < math.Float32frombits(uint32(x2Lo>>0)), + math.Float32frombits(uint32(x1Lo>>32)) < math.Float32frombits(uint32(x2Lo>>32)), + math.Float32frombits(uint32(x1Hi>>0)) < math.Float32frombits(uint32(x2Hi>>0)), + math.Float32frombits(uint32(x1Hi>>32)) < math.Float32frombits(uint32(x2Hi>>32)), + } + case v128CmpTypeF32x4Gt: + result = []bool{ + math.Float32frombits(uint32(x1Lo>>0)) > math.Float32frombits(uint32(x2Lo>>0)), + math.Float32frombits(uint32(x1Lo>>32)) > math.Float32frombits(uint32(x2Lo>>32)), + math.Float32frombits(uint32(x1Hi>>0)) > math.Float32frombits(uint32(x2Hi>>0)), + math.Float32frombits(uint32(x1Hi>>32)) > math.Float32frombits(uint32(x2Hi>>32)), + } + case v128CmpTypeF32x4Le: + result = []bool{ + math.Float32frombits(uint32(x1Lo>>0)) <= math.Float32frombits(uint32(x2Lo>>0)), + math.Float32frombits(uint32(x1Lo>>32)) <= math.Float32frombits(uint32(x2Lo>>32)), + math.Float32frombits(uint32(x1Hi>>0)) <= math.Float32frombits(uint32(x2Hi>>0)), + math.Float32frombits(uint32(x1Hi>>32)) <= math.Float32frombits(uint32(x2Hi>>32)), + } + case v128CmpTypeF32x4Ge: + result = []bool{ + math.Float32frombits(uint32(x1Lo>>0)) >= math.Float32frombits(uint32(x2Lo>>0)), + math.Float32frombits(uint32(x1Lo>>32)) >= math.Float32frombits(uint32(x2Lo>>32)), + math.Float32frombits(uint32(x1Hi>>0)) >= math.Float32frombits(uint32(x2Hi>>0)), + math.Float32frombits(uint32(x1Hi>>32)) >= math.Float32frombits(uint32(x2Hi>>32)), + } + case v128CmpTypeF64x2Eq: + result = []bool{ + math.Float64frombits(x1Lo) == math.Float64frombits(x2Lo), + math.Float64frombits(x1Hi) == math.Float64frombits(x2Hi), + } + case v128CmpTypeF64x2Ne: + result = []bool{ + math.Float64frombits(x1Lo) != math.Float64frombits(x2Lo), + math.Float64frombits(x1Hi) != math.Float64frombits(x2Hi), + } + case v128CmpTypeF64x2Lt: + result = []bool{ + math.Float64frombits(x1Lo) < math.Float64frombits(x2Lo), + math.Float64frombits(x1Hi) < math.Float64frombits(x2Hi), + } + case v128CmpTypeF64x2Gt: + result = []bool{ + math.Float64frombits(x1Lo) > math.Float64frombits(x2Lo), + math.Float64frombits(x1Hi) > math.Float64frombits(x2Hi), + } + case v128CmpTypeF64x2Le: + result = []bool{ + math.Float64frombits(x1Lo) <= math.Float64frombits(x2Lo), + math.Float64frombits(x1Hi) <= math.Float64frombits(x2Hi), + } + case v128CmpTypeF64x2Ge: + result = []bool{ + math.Float64frombits(x1Lo) >= math.Float64frombits(x2Lo), + math.Float64frombits(x1Hi) >= math.Float64frombits(x2Hi), + } + } + + var retLo, retHi uint64 + laneNum := len(result) + switch laneNum { + case 16: + for i, b := range result { + if b { + if i < 8 { + retLo |= 0xff << (i * 8) + } else { + retHi |= 0xff << ((i - 8) * 8) + } + } + } + case 8: + for i, b := range result { + if b { + if i < 4 { + retLo |= 0xffff << (i * 16) + } else { + retHi |= 0xffff << ((i - 4) * 16) + } + } + } + case 4: + for i, b := range result { + if b { + if i < 2 { + retLo |= 0xffff_ffff << (i * 32) + } else { + retHi |= 0xffff_ffff << ((i - 2) * 32) + } + } + } + case 2: + if result[0] { + retLo = ^uint64(0) + } + if result[1] { + retHi = ^uint64(0) + } + } + + ce.pushValue(retLo) + ce.pushValue(retHi) + frame.pc++ + case operationKindV128AddSat: + x2hi, x2Lo := ce.popValue(), ce.popValue() + x1hi, x1Lo := ce.popValue(), ce.popValue() + + var retLo, retHi uint64 + + // Lane-wise addition while saturating the overflowing values. + // https://github.com/WebAssembly/spec/blob/wg-2.0.draft1/proposals/simd/SIMD.md#saturating-integer-addition + switch op.B1 { + case shapeI8x16: + for i := 0; i < 16; i++ { + var v, w byte + if i < 8 { + v, w = byte(x1Lo>>(i*8)), byte(x2Lo>>(i*8)) + } else { + v, w = byte(x1hi>>((i-8)*8)), byte(x2hi>>((i-8)*8)) + } + + var uv uint64 + if op.B3 { // signed + if subbed := int64(int8(v)) + int64(int8(w)); subbed < math.MinInt8 { + uv = uint64(byte(0x80)) + } else if subbed > math.MaxInt8 { + uv = uint64(byte(0x7f)) + } else { + uv = uint64(byte(int8(subbed))) + } + } else { + if subbed := int64(v) + int64(w); subbed < 0 { + uv = uint64(byte(0)) + } else if subbed > math.MaxUint8 { + uv = uint64(byte(0xff)) + } else { + uv = uint64(byte(subbed)) + } + } + + if i < 8 { // first 8 lanes are on lower 64bits. + retLo |= uv << (i * 8) + } else { + retHi |= uv << ((i - 8) * 8) + } + } + case shapeI16x8: + for i := 0; i < 8; i++ { + var v, w uint16 + if i < 4 { + v, w = uint16(x1Lo>>(i*16)), uint16(x2Lo>>(i*16)) + } else { + v, w = uint16(x1hi>>((i-4)*16)), uint16(x2hi>>((i-4)*16)) + } + + var uv uint64 + if op.B3 { // signed + if added := int64(int16(v)) + int64(int16(w)); added < math.MinInt16 { + uv = uint64(uint16(0x8000)) + } else if added > math.MaxInt16 { + uv = uint64(uint16(0x7fff)) + } else { + uv = uint64(uint16(int16(added))) + } + } else { + if added := int64(v) + int64(w); added < 0 { + uv = uint64(uint16(0)) + } else if added > math.MaxUint16 { + uv = uint64(uint16(0xffff)) + } else { + uv = uint64(uint16(added)) + } + } + + if i < 4 { // first 4 lanes are on lower 64bits. + retLo |= uv << (i * 16) + } else { + retHi |= uv << ((i - 4) * 16) + } + } + } + + ce.pushValue(retLo) + ce.pushValue(retHi) + frame.pc++ + case operationKindV128SubSat: + x2hi, x2Lo := ce.popValue(), ce.popValue() + x1hi, x1Lo := ce.popValue(), ce.popValue() + + var retLo, retHi uint64 + + // Lane-wise subtraction while saturating the overflowing values. + // https://github.com/WebAssembly/spec/blob/wg-2.0.draft1/proposals/simd/SIMD.md#saturating-integer-subtraction + switch op.B1 { + case shapeI8x16: + for i := 0; i < 16; i++ { + var v, w byte + if i < 8 { + v, w = byte(x1Lo>>(i*8)), byte(x2Lo>>(i*8)) + } else { + v, w = byte(x1hi>>((i-8)*8)), byte(x2hi>>((i-8)*8)) + } + + var uv uint64 + if op.B3 { // signed + if subbed := int64(int8(v)) - int64(int8(w)); subbed < math.MinInt8 { + uv = uint64(byte(0x80)) + } else if subbed > math.MaxInt8 { + uv = uint64(byte(0x7f)) + } else { + uv = uint64(byte(int8(subbed))) + } + } else { + if subbed := int64(v) - int64(w); subbed < 0 { + uv = uint64(byte(0)) + } else if subbed > math.MaxUint8 { + uv = uint64(byte(0xff)) + } else { + uv = uint64(byte(subbed)) + } + } + + if i < 8 { + retLo |= uv << (i * 8) + } else { + retHi |= uv << ((i - 8) * 8) + } + } + case shapeI16x8: + for i := 0; i < 8; i++ { + var v, w uint16 + if i < 4 { + v, w = uint16(x1Lo>>(i*16)), uint16(x2Lo>>(i*16)) + } else { + v, w = uint16(x1hi>>((i-4)*16)), uint16(x2hi>>((i-4)*16)) + } + + var uv uint64 + if op.B3 { // signed + if subbed := int64(int16(v)) - int64(int16(w)); subbed < math.MinInt16 { + uv = uint64(uint16(0x8000)) + } else if subbed > math.MaxInt16 { + uv = uint64(uint16(0x7fff)) + } else { + uv = uint64(uint16(int16(subbed))) + } + } else { + if subbed := int64(v) - int64(w); subbed < 0 { + uv = uint64(uint16(0)) + } else if subbed > math.MaxUint16 { + uv = uint64(uint16(0xffff)) + } else { + uv = uint64(uint16(subbed)) + } + } + + if i < 4 { + retLo |= uv << (i * 16) + } else { + retHi |= uv << ((i - 4) * 16) + } + } + } + + ce.pushValue(retLo) + ce.pushValue(retHi) + frame.pc++ + case operationKindV128Mul: + x2hi, x2lo := ce.popValue(), ce.popValue() + x1hi, x1lo := ce.popValue(), ce.popValue() + var retLo, retHi uint64 + switch op.B1 { + case shapeI16x8: + retHi = uint64(uint16(x1hi)*uint16(x2hi)) | (uint64(uint16(x1hi>>16)*uint16(x2hi>>16)) << 16) | + (uint64(uint16(x1hi>>32)*uint16(x2hi>>32)) << 32) | (uint64(uint16(x1hi>>48)*uint16(x2hi>>48)) << 48) + retLo = uint64(uint16(x1lo)*uint16(x2lo)) | (uint64(uint16(x1lo>>16)*uint16(x2lo>>16)) << 16) | + (uint64(uint16(x1lo>>32)*uint16(x2lo>>32)) << 32) | (uint64(uint16(x1lo>>48)*uint16(x2lo>>48)) << 48) + case shapeI32x4: + retHi = uint64(uint32(x1hi)*uint32(x2hi)) | (uint64(uint32(x1hi>>32)*uint32(x2hi>>32)) << 32) + retLo = uint64(uint32(x1lo)*uint32(x2lo)) | (uint64(uint32(x1lo>>32)*uint32(x2lo>>32)) << 32) + case shapeI64x2: + retHi = x1hi * x2hi + retLo = x1lo * x2lo + case shapeF32x4: + retHi = mulFloat32bits(uint32(x1hi), uint32(x2hi)) | mulFloat32bits(uint32(x1hi>>32), uint32(x2hi>>32))<<32 + retLo = mulFloat32bits(uint32(x1lo), uint32(x2lo)) | mulFloat32bits(uint32(x1lo>>32), uint32(x2lo>>32))<<32 + case shapeF64x2: + retHi = math.Float64bits(math.Float64frombits(x1hi) * math.Float64frombits(x2hi)) + retLo = math.Float64bits(math.Float64frombits(x1lo) * math.Float64frombits(x2lo)) + } + ce.pushValue(retLo) + ce.pushValue(retHi) + frame.pc++ + case operationKindV128Div: + x2hi, x2lo := ce.popValue(), ce.popValue() + x1hi, x1lo := ce.popValue(), ce.popValue() + var retLo, retHi uint64 + if op.B1 == shapeF64x2 { + retHi = math.Float64bits(math.Float64frombits(x1hi) / math.Float64frombits(x2hi)) + retLo = math.Float64bits(math.Float64frombits(x1lo) / math.Float64frombits(x2lo)) + } else { + retHi = divFloat32bits(uint32(x1hi), uint32(x2hi)) | divFloat32bits(uint32(x1hi>>32), uint32(x2hi>>32))<<32 + retLo = divFloat32bits(uint32(x1lo), uint32(x2lo)) | divFloat32bits(uint32(x1lo>>32), uint32(x2lo>>32))<<32 + } + ce.pushValue(retLo) + ce.pushValue(retHi) + frame.pc++ + case operationKindV128Neg: + hi, lo := ce.popValue(), ce.popValue() + switch op.B1 { + case shapeI8x16: + lo = uint64(-byte(lo)) | (uint64(-byte(lo>>8)) << 8) | + (uint64(-byte(lo>>16)) << 16) | (uint64(-byte(lo>>24)) << 24) | + (uint64(-byte(lo>>32)) << 32) | (uint64(-byte(lo>>40)) << 40) | + (uint64(-byte(lo>>48)) << 48) | (uint64(-byte(lo>>56)) << 56) + hi = uint64(-byte(hi)) | (uint64(-byte(hi>>8)) << 8) | + (uint64(-byte(hi>>16)) << 16) | (uint64(-byte(hi>>24)) << 24) | + (uint64(-byte(hi>>32)) << 32) | (uint64(-byte(hi>>40)) << 40) | + (uint64(-byte(hi>>48)) << 48) | (uint64(-byte(hi>>56)) << 56) + case shapeI16x8: + hi = uint64(-uint16(hi)) | (uint64(-uint16(hi>>16)) << 16) | + (uint64(-uint16(hi>>32)) << 32) | (uint64(-uint16(hi>>48)) << 48) + lo = uint64(-uint16(lo)) | (uint64(-uint16(lo>>16)) << 16) | + (uint64(-uint16(lo>>32)) << 32) | (uint64(-uint16(lo>>48)) << 48) + case shapeI32x4: + hi = uint64(-uint32(hi)) | (uint64(-uint32(hi>>32)) << 32) + lo = uint64(-uint32(lo)) | (uint64(-uint32(lo>>32)) << 32) + case shapeI64x2: + hi = -hi + lo = -lo + case shapeF32x4: + hi = uint64(math.Float32bits(-math.Float32frombits(uint32(hi)))) | + (uint64(math.Float32bits(-math.Float32frombits(uint32(hi>>32)))) << 32) + lo = uint64(math.Float32bits(-math.Float32frombits(uint32(lo)))) | + (uint64(math.Float32bits(-math.Float32frombits(uint32(lo>>32)))) << 32) + case shapeF64x2: + hi = math.Float64bits(-math.Float64frombits(hi)) + lo = math.Float64bits(-math.Float64frombits(lo)) + } + ce.pushValue(lo) + ce.pushValue(hi) + frame.pc++ + case operationKindV128Sqrt: + hi, lo := ce.popValue(), ce.popValue() + if op.B1 == shapeF64x2 { + hi = math.Float64bits(math.Sqrt(math.Float64frombits(hi))) + lo = math.Float64bits(math.Sqrt(math.Float64frombits(lo))) + } else { + hi = uint64(math.Float32bits(float32(math.Sqrt(float64(math.Float32frombits(uint32(hi))))))) | + (uint64(math.Float32bits(float32(math.Sqrt(float64(math.Float32frombits(uint32(hi>>32))))))) << 32) + lo = uint64(math.Float32bits(float32(math.Sqrt(float64(math.Float32frombits(uint32(lo))))))) | + (uint64(math.Float32bits(float32(math.Sqrt(float64(math.Float32frombits(uint32(lo>>32))))))) << 32) + } + ce.pushValue(lo) + ce.pushValue(hi) + frame.pc++ + case operationKindV128Abs: + hi, lo := ce.popValue(), ce.popValue() + switch op.B1 { + case shapeI8x16: + lo = uint64(i8Abs(byte(lo))) | (uint64(i8Abs(byte(lo>>8))) << 8) | + (uint64(i8Abs(byte(lo>>16))) << 16) | (uint64(i8Abs(byte(lo>>24))) << 24) | + (uint64(i8Abs(byte(lo>>32))) << 32) | (uint64(i8Abs(byte(lo>>40))) << 40) | + (uint64(i8Abs(byte(lo>>48))) << 48) | (uint64(i8Abs(byte(lo>>56))) << 56) + hi = uint64(i8Abs(byte(hi))) | (uint64(i8Abs(byte(hi>>8))) << 8) | + (uint64(i8Abs(byte(hi>>16))) << 16) | (uint64(i8Abs(byte(hi>>24))) << 24) | + (uint64(i8Abs(byte(hi>>32))) << 32) | (uint64(i8Abs(byte(hi>>40))) << 40) | + (uint64(i8Abs(byte(hi>>48))) << 48) | (uint64(i8Abs(byte(hi>>56))) << 56) + case shapeI16x8: + hi = uint64(i16Abs(uint16(hi))) | (uint64(i16Abs(uint16(hi>>16))) << 16) | + (uint64(i16Abs(uint16(hi>>32))) << 32) | (uint64(i16Abs(uint16(hi>>48))) << 48) + lo = uint64(i16Abs(uint16(lo))) | (uint64(i16Abs(uint16(lo>>16))) << 16) | + (uint64(i16Abs(uint16(lo>>32))) << 32) | (uint64(i16Abs(uint16(lo>>48))) << 48) + case shapeI32x4: + hi = uint64(i32Abs(uint32(hi))) | (uint64(i32Abs(uint32(hi>>32))) << 32) + lo = uint64(i32Abs(uint32(lo))) | (uint64(i32Abs(uint32(lo>>32))) << 32) + case shapeI64x2: + if int64(hi) < 0 { + hi = -hi + } + if int64(lo) < 0 { + lo = -lo + } + case shapeF32x4: + hi = hi &^ (1<<31 | 1<<63) + lo = lo &^ (1<<31 | 1<<63) + case shapeF64x2: + hi = hi &^ (1 << 63) + lo = lo &^ (1 << 63) + } + ce.pushValue(lo) + ce.pushValue(hi) + frame.pc++ + case operationKindV128Popcnt: + hi, lo := ce.popValue(), ce.popValue() + var retLo, retHi uint64 + for i := 0; i < 16; i++ { + var v byte + if i < 8 { + v = byte(lo >> (i * 8)) + } else { + v = byte(hi >> ((i - 8) * 8)) + } + + var cnt uint64 + for i := 0; i < 8; i++ { + if (v>>i)&0b1 != 0 { + cnt++ + } + } + + if i < 8 { + retLo |= cnt << (i * 8) + } else { + retHi |= cnt << ((i - 8) * 8) + } + } + ce.pushValue(retLo) + ce.pushValue(retHi) + frame.pc++ + case operationKindV128Min: + x2hi, x2lo := ce.popValue(), ce.popValue() + x1hi, x1lo := ce.popValue(), ce.popValue() + var retLo, retHi uint64 + switch op.B1 { + case shapeI8x16: + if op.B3 { // signed + retLo = uint64(i8MinS(uint8(x1lo>>8), uint8(x2lo>>8)))<<8 | uint64(i8MinS(uint8(x1lo), uint8(x2lo))) | + uint64(i8MinS(uint8(x1lo>>24), uint8(x2lo>>24)))<<24 | uint64(i8MinS(uint8(x1lo>>16), uint8(x2lo>>16)))<<16 | + uint64(i8MinS(uint8(x1lo>>40), uint8(x2lo>>40)))<<40 | uint64(i8MinS(uint8(x1lo>>32), uint8(x2lo>>32)))<<32 | + uint64(i8MinS(uint8(x1lo>>56), uint8(x2lo>>56)))<<56 | uint64(i8MinS(uint8(x1lo>>48), uint8(x2lo>>48)))<<48 + retHi = uint64(i8MinS(uint8(x1hi>>8), uint8(x2hi>>8)))<<8 | uint64(i8MinS(uint8(x1hi), uint8(x2hi))) | + uint64(i8MinS(uint8(x1hi>>24), uint8(x2hi>>24)))<<24 | uint64(i8MinS(uint8(x1hi>>16), uint8(x2hi>>16)))<<16 | + uint64(i8MinS(uint8(x1hi>>40), uint8(x2hi>>40)))<<40 | uint64(i8MinS(uint8(x1hi>>32), uint8(x2hi>>32)))<<32 | + uint64(i8MinS(uint8(x1hi>>56), uint8(x2hi>>56)))<<56 | uint64(i8MinS(uint8(x1hi>>48), uint8(x2hi>>48)))<<48 + } else { + retLo = uint64(i8MinU(uint8(x1lo>>8), uint8(x2lo>>8)))<<8 | uint64(i8MinU(uint8(x1lo), uint8(x2lo))) | + uint64(i8MinU(uint8(x1lo>>24), uint8(x2lo>>24)))<<24 | uint64(i8MinU(uint8(x1lo>>16), uint8(x2lo>>16)))<<16 | + uint64(i8MinU(uint8(x1lo>>40), uint8(x2lo>>40)))<<40 | uint64(i8MinU(uint8(x1lo>>32), uint8(x2lo>>32)))<<32 | + uint64(i8MinU(uint8(x1lo>>56), uint8(x2lo>>56)))<<56 | uint64(i8MinU(uint8(x1lo>>48), uint8(x2lo>>48)))<<48 + retHi = uint64(i8MinU(uint8(x1hi>>8), uint8(x2hi>>8)))<<8 | uint64(i8MinU(uint8(x1hi), uint8(x2hi))) | + uint64(i8MinU(uint8(x1hi>>24), uint8(x2hi>>24)))<<24 | uint64(i8MinU(uint8(x1hi>>16), uint8(x2hi>>16)))<<16 | + uint64(i8MinU(uint8(x1hi>>40), uint8(x2hi>>40)))<<40 | uint64(i8MinU(uint8(x1hi>>32), uint8(x2hi>>32)))<<32 | + uint64(i8MinU(uint8(x1hi>>56), uint8(x2hi>>56)))<<56 | uint64(i8MinU(uint8(x1hi>>48), uint8(x2hi>>48)))<<48 + } + case shapeI16x8: + if op.B3 { // signed + retLo = uint64(i16MinS(uint16(x1lo), uint16(x2lo))) | + uint64(i16MinS(uint16(x1lo>>16), uint16(x2lo>>16)))<<16 | + uint64(i16MinS(uint16(x1lo>>32), uint16(x2lo>>32)))<<32 | + uint64(i16MinS(uint16(x1lo>>48), uint16(x2lo>>48)))<<48 + retHi = uint64(i16MinS(uint16(x1hi), uint16(x2hi))) | + uint64(i16MinS(uint16(x1hi>>16), uint16(x2hi>>16)))<<16 | + uint64(i16MinS(uint16(x1hi>>32), uint16(x2hi>>32)))<<32 | + uint64(i16MinS(uint16(x1hi>>48), uint16(x2hi>>48)))<<48 + } else { + retLo = uint64(i16MinU(uint16(x1lo), uint16(x2lo))) | + uint64(i16MinU(uint16(x1lo>>16), uint16(x2lo>>16)))<<16 | + uint64(i16MinU(uint16(x1lo>>32), uint16(x2lo>>32)))<<32 | + uint64(i16MinU(uint16(x1lo>>48), uint16(x2lo>>48)))<<48 + retHi = uint64(i16MinU(uint16(x1hi), uint16(x2hi))) | + uint64(i16MinU(uint16(x1hi>>16), uint16(x2hi>>16)))<<16 | + uint64(i16MinU(uint16(x1hi>>32), uint16(x2hi>>32)))<<32 | + uint64(i16MinU(uint16(x1hi>>48), uint16(x2hi>>48)))<<48 + } + case shapeI32x4: + if op.B3 { // signed + retLo = uint64(i32MinS(uint32(x1lo), uint32(x2lo))) | + uint64(i32MinS(uint32(x1lo>>32), uint32(x2lo>>32)))<<32 + retHi = uint64(i32MinS(uint32(x1hi), uint32(x2hi))) | + uint64(i32MinS(uint32(x1hi>>32), uint32(x2hi>>32)))<<32 + } else { + retLo = uint64(i32MinU(uint32(x1lo), uint32(x2lo))) | + uint64(i32MinU(uint32(x1lo>>32), uint32(x2lo>>32)))<<32 + retHi = uint64(i32MinU(uint32(x1hi), uint32(x2hi))) | + uint64(i32MinU(uint32(x1hi>>32), uint32(x2hi>>32)))<<32 + } + case shapeF32x4: + retHi = wasmCompatMin32bits(uint32(x1hi), uint32(x2hi)) | + wasmCompatMin32bits(uint32(x1hi>>32), uint32(x2hi>>32))<<32 + retLo = wasmCompatMin32bits(uint32(x1lo), uint32(x2lo)) | + wasmCompatMin32bits(uint32(x1lo>>32), uint32(x2lo>>32))<<32 + case shapeF64x2: + retHi = math.Float64bits(moremath.WasmCompatMin64( + math.Float64frombits(x1hi), + math.Float64frombits(x2hi), + )) + retLo = math.Float64bits(moremath.WasmCompatMin64( + math.Float64frombits(x1lo), + math.Float64frombits(x2lo), + )) + } + ce.pushValue(retLo) + ce.pushValue(retHi) + frame.pc++ + case operationKindV128Max: + x2hi, x2lo := ce.popValue(), ce.popValue() + x1hi, x1lo := ce.popValue(), ce.popValue() + var retLo, retHi uint64 + switch op.B1 { + case shapeI8x16: + if op.B3 { // signed + retLo = uint64(i8MaxS(uint8(x1lo>>8), uint8(x2lo>>8)))<<8 | uint64(i8MaxS(uint8(x1lo), uint8(x2lo))) | + uint64(i8MaxS(uint8(x1lo>>24), uint8(x2lo>>24)))<<24 | uint64(i8MaxS(uint8(x1lo>>16), uint8(x2lo>>16)))<<16 | + uint64(i8MaxS(uint8(x1lo>>40), uint8(x2lo>>40)))<<40 | uint64(i8MaxS(uint8(x1lo>>32), uint8(x2lo>>32)))<<32 | + uint64(i8MaxS(uint8(x1lo>>56), uint8(x2lo>>56)))<<56 | uint64(i8MaxS(uint8(x1lo>>48), uint8(x2lo>>48)))<<48 + retHi = uint64(i8MaxS(uint8(x1hi>>8), uint8(x2hi>>8)))<<8 | uint64(i8MaxS(uint8(x1hi), uint8(x2hi))) | + uint64(i8MaxS(uint8(x1hi>>24), uint8(x2hi>>24)))<<24 | uint64(i8MaxS(uint8(x1hi>>16), uint8(x2hi>>16)))<<16 | + uint64(i8MaxS(uint8(x1hi>>40), uint8(x2hi>>40)))<<40 | uint64(i8MaxS(uint8(x1hi>>32), uint8(x2hi>>32)))<<32 | + uint64(i8MaxS(uint8(x1hi>>56), uint8(x2hi>>56)))<<56 | uint64(i8MaxS(uint8(x1hi>>48), uint8(x2hi>>48)))<<48 + } else { + retLo = uint64(i8MaxU(uint8(x1lo>>8), uint8(x2lo>>8)))<<8 | uint64(i8MaxU(uint8(x1lo), uint8(x2lo))) | + uint64(i8MaxU(uint8(x1lo>>24), uint8(x2lo>>24)))<<24 | uint64(i8MaxU(uint8(x1lo>>16), uint8(x2lo>>16)))<<16 | + uint64(i8MaxU(uint8(x1lo>>40), uint8(x2lo>>40)))<<40 | uint64(i8MaxU(uint8(x1lo>>32), uint8(x2lo>>32)))<<32 | + uint64(i8MaxU(uint8(x1lo>>56), uint8(x2lo>>56)))<<56 | uint64(i8MaxU(uint8(x1lo>>48), uint8(x2lo>>48)))<<48 + retHi = uint64(i8MaxU(uint8(x1hi>>8), uint8(x2hi>>8)))<<8 | uint64(i8MaxU(uint8(x1hi), uint8(x2hi))) | + uint64(i8MaxU(uint8(x1hi>>24), uint8(x2hi>>24)))<<24 | uint64(i8MaxU(uint8(x1hi>>16), uint8(x2hi>>16)))<<16 | + uint64(i8MaxU(uint8(x1hi>>40), uint8(x2hi>>40)))<<40 | uint64(i8MaxU(uint8(x1hi>>32), uint8(x2hi>>32)))<<32 | + uint64(i8MaxU(uint8(x1hi>>56), uint8(x2hi>>56)))<<56 | uint64(i8MaxU(uint8(x1hi>>48), uint8(x2hi>>48)))<<48 + } + case shapeI16x8: + if op.B3 { // signed + retLo = uint64(i16MaxS(uint16(x1lo), uint16(x2lo))) | + uint64(i16MaxS(uint16(x1lo>>16), uint16(x2lo>>16)))<<16 | + uint64(i16MaxS(uint16(x1lo>>32), uint16(x2lo>>32)))<<32 | + uint64(i16MaxS(uint16(x1lo>>48), uint16(x2lo>>48)))<<48 + retHi = uint64(i16MaxS(uint16(x1hi), uint16(x2hi))) | + uint64(i16MaxS(uint16(x1hi>>16), uint16(x2hi>>16)))<<16 | + uint64(i16MaxS(uint16(x1hi>>32), uint16(x2hi>>32)))<<32 | + uint64(i16MaxS(uint16(x1hi>>48), uint16(x2hi>>48)))<<48 + } else { + retLo = uint64(i16MaxU(uint16(x1lo), uint16(x2lo))) | + uint64(i16MaxU(uint16(x1lo>>16), uint16(x2lo>>16)))<<16 | + uint64(i16MaxU(uint16(x1lo>>32), uint16(x2lo>>32)))<<32 | + uint64(i16MaxU(uint16(x1lo>>48), uint16(x2lo>>48)))<<48 + retHi = uint64(i16MaxU(uint16(x1hi), uint16(x2hi))) | + uint64(i16MaxU(uint16(x1hi>>16), uint16(x2hi>>16)))<<16 | + uint64(i16MaxU(uint16(x1hi>>32), uint16(x2hi>>32)))<<32 | + uint64(i16MaxU(uint16(x1hi>>48), uint16(x2hi>>48)))<<48 + } + case shapeI32x4: + if op.B3 { // signed + retLo = uint64(i32MaxS(uint32(x1lo), uint32(x2lo))) | + uint64(i32MaxS(uint32(x1lo>>32), uint32(x2lo>>32)))<<32 + retHi = uint64(i32MaxS(uint32(x1hi), uint32(x2hi))) | + uint64(i32MaxS(uint32(x1hi>>32), uint32(x2hi>>32)))<<32 + } else { + retLo = uint64(i32MaxU(uint32(x1lo), uint32(x2lo))) | + uint64(i32MaxU(uint32(x1lo>>32), uint32(x2lo>>32)))<<32 + retHi = uint64(i32MaxU(uint32(x1hi), uint32(x2hi))) | + uint64(i32MaxU(uint32(x1hi>>32), uint32(x2hi>>32)))<<32 + } + case shapeF32x4: + retHi = wasmCompatMax32bits(uint32(x1hi), uint32(x2hi)) | + wasmCompatMax32bits(uint32(x1hi>>32), uint32(x2hi>>32))<<32 + retLo = wasmCompatMax32bits(uint32(x1lo), uint32(x2lo)) | + wasmCompatMax32bits(uint32(x1lo>>32), uint32(x2lo>>32))<<32 + case shapeF64x2: + retHi = math.Float64bits(moremath.WasmCompatMax64( + math.Float64frombits(x1hi), + math.Float64frombits(x2hi), + )) + retLo = math.Float64bits(moremath.WasmCompatMax64( + math.Float64frombits(x1lo), + math.Float64frombits(x2lo), + )) + } + ce.pushValue(retLo) + ce.pushValue(retHi) + frame.pc++ + case operationKindV128AvgrU: + x2hi, x2lo := ce.popValue(), ce.popValue() + x1hi, x1lo := ce.popValue(), ce.popValue() + var retLo, retHi uint64 + switch op.B1 { + case shapeI8x16: + retLo = uint64(i8RoundingAverage(uint8(x1lo>>8), uint8(x2lo>>8)))<<8 | uint64(i8RoundingAverage(uint8(x1lo), uint8(x2lo))) | + uint64(i8RoundingAverage(uint8(x1lo>>24), uint8(x2lo>>24)))<<24 | uint64(i8RoundingAverage(uint8(x1lo>>16), uint8(x2lo>>16)))<<16 | + uint64(i8RoundingAverage(uint8(x1lo>>40), uint8(x2lo>>40)))<<40 | uint64(i8RoundingAverage(uint8(x1lo>>32), uint8(x2lo>>32)))<<32 | + uint64(i8RoundingAverage(uint8(x1lo>>56), uint8(x2lo>>56)))<<56 | uint64(i8RoundingAverage(uint8(x1lo>>48), uint8(x2lo>>48)))<<48 + retHi = uint64(i8RoundingAverage(uint8(x1hi>>8), uint8(x2hi>>8)))<<8 | uint64(i8RoundingAverage(uint8(x1hi), uint8(x2hi))) | + uint64(i8RoundingAverage(uint8(x1hi>>24), uint8(x2hi>>24)))<<24 | uint64(i8RoundingAverage(uint8(x1hi>>16), uint8(x2hi>>16)))<<16 | + uint64(i8RoundingAverage(uint8(x1hi>>40), uint8(x2hi>>40)))<<40 | uint64(i8RoundingAverage(uint8(x1hi>>32), uint8(x2hi>>32)))<<32 | + uint64(i8RoundingAverage(uint8(x1hi>>56), uint8(x2hi>>56)))<<56 | uint64(i8RoundingAverage(uint8(x1hi>>48), uint8(x2hi>>48)))<<48 + case shapeI16x8: + retLo = uint64(i16RoundingAverage(uint16(x1lo), uint16(x2lo))) | + uint64(i16RoundingAverage(uint16(x1lo>>16), uint16(x2lo>>16)))<<16 | + uint64(i16RoundingAverage(uint16(x1lo>>32), uint16(x2lo>>32)))<<32 | + uint64(i16RoundingAverage(uint16(x1lo>>48), uint16(x2lo>>48)))<<48 + retHi = uint64(i16RoundingAverage(uint16(x1hi), uint16(x2hi))) | + uint64(i16RoundingAverage(uint16(x1hi>>16), uint16(x2hi>>16)))<<16 | + uint64(i16RoundingAverage(uint16(x1hi>>32), uint16(x2hi>>32)))<<32 | + uint64(i16RoundingAverage(uint16(x1hi>>48), uint16(x2hi>>48)))<<48 + } + ce.pushValue(retLo) + ce.pushValue(retHi) + frame.pc++ + case operationKindV128Pmin: + x2hi, x2lo := ce.popValue(), ce.popValue() + x1hi, x1lo := ce.popValue(), ce.popValue() + var retLo, retHi uint64 + if op.B1 == shapeF32x4 { + if flt32(math.Float32frombits(uint32(x2lo)), math.Float32frombits(uint32(x1lo))) { + retLo = x2lo & 0x00000000_ffffffff + } else { + retLo = x1lo & 0x00000000_ffffffff + } + if flt32(math.Float32frombits(uint32(x2lo>>32)), math.Float32frombits(uint32(x1lo>>32))) { + retLo |= x2lo & 0xffffffff_00000000 + } else { + retLo |= x1lo & 0xffffffff_00000000 + } + if flt32(math.Float32frombits(uint32(x2hi)), math.Float32frombits(uint32(x1hi))) { + retHi = x2hi & 0x00000000_ffffffff + } else { + retHi = x1hi & 0x00000000_ffffffff + } + if flt32(math.Float32frombits(uint32(x2hi>>32)), math.Float32frombits(uint32(x1hi>>32))) { + retHi |= x2hi & 0xffffffff_00000000 + } else { + retHi |= x1hi & 0xffffffff_00000000 + } + } else { + if flt64(math.Float64frombits(x2lo), math.Float64frombits(x1lo)) { + retLo = x2lo + } else { + retLo = x1lo + } + if flt64(math.Float64frombits(x2hi), math.Float64frombits(x1hi)) { + retHi = x2hi + } else { + retHi = x1hi + } + } + ce.pushValue(retLo) + ce.pushValue(retHi) + frame.pc++ + case operationKindV128Pmax: + x2hi, x2lo := ce.popValue(), ce.popValue() + x1hi, x1lo := ce.popValue(), ce.popValue() + var retLo, retHi uint64 + if op.B1 == shapeF32x4 { + if flt32(math.Float32frombits(uint32(x1lo)), math.Float32frombits(uint32(x2lo))) { + retLo = x2lo & 0x00000000_ffffffff + } else { + retLo = x1lo & 0x00000000_ffffffff + } + if flt32(math.Float32frombits(uint32(x1lo>>32)), math.Float32frombits(uint32(x2lo>>32))) { + retLo |= x2lo & 0xffffffff_00000000 + } else { + retLo |= x1lo & 0xffffffff_00000000 + } + if flt32(math.Float32frombits(uint32(x1hi)), math.Float32frombits(uint32(x2hi))) { + retHi = x2hi & 0x00000000_ffffffff + } else { + retHi = x1hi & 0x00000000_ffffffff + } + if flt32(math.Float32frombits(uint32(x1hi>>32)), math.Float32frombits(uint32(x2hi>>32))) { + retHi |= x2hi & 0xffffffff_00000000 + } else { + retHi |= x1hi & 0xffffffff_00000000 + } + } else { + if flt64(math.Float64frombits(x1lo), math.Float64frombits(x2lo)) { + retLo = x2lo + } else { + retLo = x1lo + } + if flt64(math.Float64frombits(x1hi), math.Float64frombits(x2hi)) { + retHi = x2hi + } else { + retHi = x1hi + } + } + ce.pushValue(retLo) + ce.pushValue(retHi) + frame.pc++ + case operationKindV128Ceil: + hi, lo := ce.popValue(), ce.popValue() + if op.B1 == shapeF32x4 { + lo = uint64(math.Float32bits(moremath.WasmCompatCeilF32(math.Float32frombits(uint32(lo))))) | + (uint64(math.Float32bits(moremath.WasmCompatCeilF32(math.Float32frombits(uint32(lo>>32))))) << 32) + hi = uint64(math.Float32bits(moremath.WasmCompatCeilF32(math.Float32frombits(uint32(hi))))) | + (uint64(math.Float32bits(moremath.WasmCompatCeilF32(math.Float32frombits(uint32(hi>>32))))) << 32) + } else { + lo = math.Float64bits(moremath.WasmCompatCeilF64(math.Float64frombits(lo))) + hi = math.Float64bits(moremath.WasmCompatCeilF64(math.Float64frombits(hi))) + } + ce.pushValue(lo) + ce.pushValue(hi) + frame.pc++ + case operationKindV128Floor: + hi, lo := ce.popValue(), ce.popValue() + if op.B1 == shapeF32x4 { + lo = uint64(math.Float32bits(moremath.WasmCompatFloorF32(math.Float32frombits(uint32(lo))))) | + (uint64(math.Float32bits(moremath.WasmCompatFloorF32(math.Float32frombits(uint32(lo>>32))))) << 32) + hi = uint64(math.Float32bits(moremath.WasmCompatFloorF32(math.Float32frombits(uint32(hi))))) | + (uint64(math.Float32bits(moremath.WasmCompatFloorF32(math.Float32frombits(uint32(hi>>32))))) << 32) + } else { + lo = math.Float64bits(moremath.WasmCompatFloorF64(math.Float64frombits(lo))) + hi = math.Float64bits(moremath.WasmCompatFloorF64(math.Float64frombits(hi))) + } + ce.pushValue(lo) + ce.pushValue(hi) + frame.pc++ + case operationKindV128Trunc: + hi, lo := ce.popValue(), ce.popValue() + if op.B1 == shapeF32x4 { + lo = uint64(math.Float32bits(moremath.WasmCompatTruncF32(math.Float32frombits(uint32(lo))))) | + (uint64(math.Float32bits(moremath.WasmCompatTruncF32(math.Float32frombits(uint32(lo>>32))))) << 32) + hi = uint64(math.Float32bits(moremath.WasmCompatTruncF32(math.Float32frombits(uint32(hi))))) | + (uint64(math.Float32bits(moremath.WasmCompatTruncF32(math.Float32frombits(uint32(hi>>32))))) << 32) + } else { + lo = math.Float64bits(moremath.WasmCompatTruncF64(math.Float64frombits(lo))) + hi = math.Float64bits(moremath.WasmCompatTruncF64(math.Float64frombits(hi))) + } + ce.pushValue(lo) + ce.pushValue(hi) + frame.pc++ + case operationKindV128Nearest: + hi, lo := ce.popValue(), ce.popValue() + if op.B1 == shapeF32x4 { + lo = uint64(math.Float32bits(moremath.WasmCompatNearestF32(math.Float32frombits(uint32(lo))))) | + (uint64(math.Float32bits(moremath.WasmCompatNearestF32(math.Float32frombits(uint32(lo>>32))))) << 32) + hi = uint64(math.Float32bits(moremath.WasmCompatNearestF32(math.Float32frombits(uint32(hi))))) | + (uint64(math.Float32bits(moremath.WasmCompatNearestF32(math.Float32frombits(uint32(hi>>32))))) << 32) + } else { + lo = math.Float64bits(moremath.WasmCompatNearestF64(math.Float64frombits(lo))) + hi = math.Float64bits(moremath.WasmCompatNearestF64(math.Float64frombits(hi))) + } + ce.pushValue(lo) + ce.pushValue(hi) + frame.pc++ + case operationKindV128Extend: + hi, lo := ce.popValue(), ce.popValue() + var origin uint64 + if op.B3 { // use lower 64 bits + origin = lo + } else { + origin = hi + } + + signed := op.B2 == 1 + + var retHi, retLo uint64 + switch op.B1 { + case shapeI8x16: + for i := 0; i < 8; i++ { + v8 := byte(origin >> (i * 8)) + + var v16 uint16 + if signed { + v16 = uint16(int8(v8)) + } else { + v16 = uint16(v8) + } + + if i < 4 { + retLo |= uint64(v16) << (i * 16) + } else { + retHi |= uint64(v16) << ((i - 4) * 16) + } + } + case shapeI16x8: + for i := 0; i < 4; i++ { + v16 := uint16(origin >> (i * 16)) + + var v32 uint32 + if signed { + v32 = uint32(int16(v16)) + } else { + v32 = uint32(v16) + } + + if i < 2 { + retLo |= uint64(v32) << (i * 32) + } else { + retHi |= uint64(v32) << ((i - 2) * 32) + } + } + case shapeI32x4: + v32Lo := uint32(origin) + v32Hi := uint32(origin >> 32) + if signed { + retLo = uint64(int32(v32Lo)) + retHi = uint64(int32(v32Hi)) + } else { + retLo = uint64(v32Lo) + retHi = uint64(v32Hi) + } + } + ce.pushValue(retLo) + ce.pushValue(retHi) + frame.pc++ + case operationKindV128ExtMul: + x2Hi, x2Lo := ce.popValue(), ce.popValue() + x1Hi, x1Lo := ce.popValue(), ce.popValue() + var x1, x2 uint64 + if op.B3 { // use lower 64 bits + x1, x2 = x1Lo, x2Lo + } else { + x1, x2 = x1Hi, x2Hi + } + + signed := op.B2 == 1 + + var retLo, retHi uint64 + switch op.B1 { + case shapeI8x16: + for i := 0; i < 8; i++ { + v1, v2 := byte(x1>>(i*8)), byte(x2>>(i*8)) + + var v16 uint16 + if signed { + v16 = uint16(int16(int8(v1)) * int16(int8(v2))) + } else { + v16 = uint16(v1) * uint16(v2) + } + + if i < 4 { + retLo |= uint64(v16) << (i * 16) + } else { + retHi |= uint64(v16) << ((i - 4) * 16) + } + } + case shapeI16x8: + for i := 0; i < 4; i++ { + v1, v2 := uint16(x1>>(i*16)), uint16(x2>>(i*16)) + + var v32 uint32 + if signed { + v32 = uint32(int32(int16(v1)) * int32(int16(v2))) + } else { + v32 = uint32(v1) * uint32(v2) + } + + if i < 2 { + retLo |= uint64(v32) << (i * 32) + } else { + retHi |= uint64(v32) << ((i - 2) * 32) + } + } + case shapeI32x4: + v1Lo, v2Lo := uint32(x1), uint32(x2) + v1Hi, v2Hi := uint32(x1>>32), uint32(x2>>32) + if signed { + retLo = uint64(int64(int32(v1Lo)) * int64(int32(v2Lo))) + retHi = uint64(int64(int32(v1Hi)) * int64(int32(v2Hi))) + } else { + retLo = uint64(v1Lo) * uint64(v2Lo) + retHi = uint64(v1Hi) * uint64(v2Hi) + } + } + + ce.pushValue(retLo) + ce.pushValue(retHi) + frame.pc++ + case operationKindV128Q15mulrSatS: + x2hi, x2Lo := ce.popValue(), ce.popValue() + x1hi, x1Lo := ce.popValue(), ce.popValue() + var retLo, retHi uint64 + for i := 0; i < 8; i++ { + var v, w int16 + if i < 4 { + v, w = int16(uint16(x1Lo>>(i*16))), int16(uint16(x2Lo>>(i*16))) + } else { + v, w = int16(uint16(x1hi>>((i-4)*16))), int16(uint16(x2hi>>((i-4)*16))) + } + + var uv uint64 + // https://github.com/WebAssembly/spec/blob/wg-2.0.draft1/proposals/simd/SIMD.md#saturating-integer-q-format-rounding-multiplication + if calc := ((int32(v) * int32(w)) + 0x4000) >> 15; calc < math.MinInt16 { + uv = uint64(uint16(0x8000)) + } else if calc > math.MaxInt16 { + uv = uint64(uint16(0x7fff)) + } else { + uv = uint64(uint16(int16(calc))) + } + + if i < 4 { + retLo |= uv << (i * 16) + } else { + retHi |= uv << ((i - 4) * 16) + } + } + + ce.pushValue(retLo) + ce.pushValue(retHi) + frame.pc++ + case operationKindV128ExtAddPairwise: + hi, lo := ce.popValue(), ce.popValue() + + signed := op.B3 + + var retLo, retHi uint64 + switch op.B1 { + case shapeI8x16: + for i := 0; i < 8; i++ { + var v1, v2 byte + if i < 4 { + v1, v2 = byte(lo>>((i*2)*8)), byte(lo>>((i*2+1)*8)) + } else { + v1, v2 = byte(hi>>(((i-4)*2)*8)), byte(hi>>(((i-4)*2+1)*8)) + } + + var v16 uint16 + if signed { + v16 = uint16(int16(int8(v1)) + int16(int8(v2))) + } else { + v16 = uint16(v1) + uint16(v2) + } + + if i < 4 { + retLo |= uint64(v16) << (i * 16) + } else { + retHi |= uint64(v16) << ((i - 4) * 16) + } + } + case shapeI16x8: + for i := 0; i < 4; i++ { + var v1, v2 uint16 + if i < 2 { + v1, v2 = uint16(lo>>((i*2)*16)), uint16(lo>>((i*2+1)*16)) + } else { + v1, v2 = uint16(hi>>(((i-2)*2)*16)), uint16(hi>>(((i-2)*2+1)*16)) + } + + var v32 uint32 + if signed { + v32 = uint32(int32(int16(v1)) + int32(int16(v2))) + } else { + v32 = uint32(v1) + uint32(v2) + } + + if i < 2 { + retLo |= uint64(v32) << (i * 32) + } else { + retHi |= uint64(v32) << ((i - 2) * 32) + } + } + } + ce.pushValue(retLo) + ce.pushValue(retHi) + frame.pc++ + case operationKindV128FloatPromote: + _, toPromote := ce.popValue(), ce.popValue() + ce.pushValue(math.Float64bits(float64(math.Float32frombits(uint32(toPromote))))) + ce.pushValue(math.Float64bits(float64(math.Float32frombits(uint32(toPromote >> 32))))) + frame.pc++ + case operationKindV128FloatDemote: + hi, lo := ce.popValue(), ce.popValue() + ce.pushValue( + uint64(math.Float32bits(float32(math.Float64frombits(lo)))) | + (uint64(math.Float32bits(float32(math.Float64frombits(hi)))) << 32), + ) + ce.pushValue(0) + frame.pc++ + case operationKindV128FConvertFromI: + hi, lo := ce.popValue(), ce.popValue() + v1, v2, v3, v4 := uint32(lo), uint32(lo>>32), uint32(hi), uint32(hi>>32) + signed := op.B3 + + var retLo, retHi uint64 + switch op.B1 { // Destination shape. + case shapeF32x4: // f32x4 from signed/unsigned i32x4 + if signed { + retLo = uint64(math.Float32bits(float32(int32(v1)))) | + (uint64(math.Float32bits(float32(int32(v2)))) << 32) + retHi = uint64(math.Float32bits(float32(int32(v3)))) | + (uint64(math.Float32bits(float32(int32(v4)))) << 32) + } else { + retLo = uint64(math.Float32bits(float32(v1))) | + (uint64(math.Float32bits(float32(v2))) << 32) + retHi = uint64(math.Float32bits(float32(v3))) | + (uint64(math.Float32bits(float32(v4))) << 32) + } + case shapeF64x2: // f64x2 from signed/unsigned i32x4 + if signed { + retLo, retHi = math.Float64bits(float64(int32(v1))), math.Float64bits(float64(int32(v2))) + } else { + retLo, retHi = math.Float64bits(float64(v1)), math.Float64bits(float64(v2)) + } + } + + ce.pushValue(retLo) + ce.pushValue(retHi) + frame.pc++ + case operationKindV128Narrow: + x2Hi, x2Lo := ce.popValue(), ce.popValue() + x1Hi, x1Lo := ce.popValue(), ce.popValue() + signed := op.B3 + + var retLo, retHi uint64 + switch op.B1 { + case shapeI16x8: // signed/unsigned i16x8 to i8x16 + for i := 0; i < 8; i++ { + var v16 uint16 + if i < 4 { + v16 = uint16(x1Lo >> (i * 16)) + } else { + v16 = uint16(x1Hi >> ((i - 4) * 16)) + } + + var v byte + if signed { + if s := int16(v16); s > math.MaxInt8 { + v = math.MaxInt8 + } else if s < math.MinInt8 { + s = math.MinInt8 + v = byte(s) + } else { + v = byte(v16) + } + } else { + if s := int16(v16); s > math.MaxUint8 { + v = math.MaxUint8 + } else if s < 0 { + v = 0 + } else { + v = byte(v16) + } + } + retLo |= uint64(v) << (i * 8) + } + for i := 0; i < 8; i++ { + var v16 uint16 + if i < 4 { + v16 = uint16(x2Lo >> (i * 16)) + } else { + v16 = uint16(x2Hi >> ((i - 4) * 16)) + } + + var v byte + if signed { + if s := int16(v16); s > math.MaxInt8 { + v = math.MaxInt8 + } else if s < math.MinInt8 { + s = math.MinInt8 + v = byte(s) + } else { + v = byte(v16) + } + } else { + if s := int16(v16); s > math.MaxUint8 { + v = math.MaxUint8 + } else if s < 0 { + v = 0 + } else { + v = byte(v16) + } + } + retHi |= uint64(v) << (i * 8) + } + case shapeI32x4: // signed/unsigned i32x4 to i16x8 + for i := 0; i < 4; i++ { + var v32 uint32 + if i < 2 { + v32 = uint32(x1Lo >> (i * 32)) + } else { + v32 = uint32(x1Hi >> ((i - 2) * 32)) + } + + var v uint16 + if signed { + if s := int32(v32); s > math.MaxInt16 { + v = math.MaxInt16 + } else if s < math.MinInt16 { + s = math.MinInt16 + v = uint16(s) + } else { + v = uint16(v32) + } + } else { + if s := int32(v32); s > math.MaxUint16 { + v = math.MaxUint16 + } else if s < 0 { + v = 0 + } else { + v = uint16(v32) + } + } + retLo |= uint64(v) << (i * 16) + } + + for i := 0; i < 4; i++ { + var v32 uint32 + if i < 2 { + v32 = uint32(x2Lo >> (i * 32)) + } else { + v32 = uint32(x2Hi >> ((i - 2) * 32)) + } + + var v uint16 + if signed { + if s := int32(v32); s > math.MaxInt16 { + v = math.MaxInt16 + } else if s < math.MinInt16 { + s = math.MinInt16 + v = uint16(s) + } else { + v = uint16(v32) + } + } else { + if s := int32(v32); s > math.MaxUint16 { + v = math.MaxUint16 + } else if s < 0 { + v = 0 + } else { + v = uint16(v32) + } + } + retHi |= uint64(v) << (i * 16) + } + } + ce.pushValue(retLo) + ce.pushValue(retHi) + frame.pc++ + case operationKindV128Dot: + x2Hi, x2Lo := ce.popValue(), ce.popValue() + x1Hi, x1Lo := ce.popValue(), ce.popValue() + ce.pushValue( + uint64(uint32(int32(int16(x1Lo>>0))*int32(int16(x2Lo>>0))+int32(int16(x1Lo>>16))*int32(int16(x2Lo>>16)))) | + (uint64(uint32(int32(int16(x1Lo>>32))*int32(int16(x2Lo>>32))+int32(int16(x1Lo>>48))*int32(int16(x2Lo>>48)))) << 32), + ) + ce.pushValue( + uint64(uint32(int32(int16(x1Hi>>0))*int32(int16(x2Hi>>0))+int32(int16(x1Hi>>16))*int32(int16(x2Hi>>16)))) | + (uint64(uint32(int32(int16(x1Hi>>32))*int32(int16(x2Hi>>32))+int32(int16(x1Hi>>48))*int32(int16(x2Hi>>48)))) << 32), + ) + frame.pc++ + case operationKindV128ITruncSatFromF: + hi, lo := ce.popValue(), ce.popValue() + signed := op.B3 + var retLo, retHi uint64 + + switch op.B1 { + case shapeF32x4: // f32x4 to i32x4 + for i, f64 := range [4]float64{ + math.Trunc(float64(math.Float32frombits(uint32(lo)))), + math.Trunc(float64(math.Float32frombits(uint32(lo >> 32)))), + math.Trunc(float64(math.Float32frombits(uint32(hi)))), + math.Trunc(float64(math.Float32frombits(uint32(hi >> 32)))), + } { + + var v uint32 + if math.IsNaN(f64) { + v = 0 + } else if signed { + if f64 < math.MinInt32 { + f64 = math.MinInt32 + } else if f64 > math.MaxInt32 { + f64 = math.MaxInt32 + } + v = uint32(int32(f64)) + } else { + if f64 < 0 { + f64 = 0 + } else if f64 > math.MaxUint32 { + f64 = math.MaxUint32 + } + v = uint32(f64) + } + + if i < 2 { + retLo |= uint64(v) << (i * 32) + } else { + retHi |= uint64(v) << ((i - 2) * 32) + } + } + + case shapeF64x2: // f64x2 to i32x4 + for i, f := range [2]float64{ + math.Trunc(math.Float64frombits(lo)), + math.Trunc(math.Float64frombits(hi)), + } { + var v uint32 + if math.IsNaN(f) { + v = 0 + } else if signed { + if f < math.MinInt32 { + f = math.MinInt32 + } else if f > math.MaxInt32 { + f = math.MaxInt32 + } + v = uint32(int32(f)) + } else { + if f < 0 { + f = 0 + } else if f > math.MaxUint32 { + f = math.MaxUint32 + } + v = uint32(f) + } + + retLo |= uint64(v) << (i * 32) + } + } + + ce.pushValue(retLo) + ce.pushValue(retHi) + frame.pc++ + case operationKindAtomicMemoryWait: + timeout := int64(ce.popValue()) + exp := ce.popValue() + offset := ce.popMemoryOffset(op) + // Runtime instead of validation error because the spec intends to allow binaries to include + // such instructions as long as they are not executed. + if !memoryInst.Shared { + panic(wasmruntime.ErrRuntimeExpectedSharedMemory) + } + + switch unsignedType(op.B1) { + case unsignedTypeI32: + if offset%4 != 0 { + panic(wasmruntime.ErrRuntimeUnalignedAtomic) + } + if int(offset) > len(memoryInst.Buffer)-4 { + panic(wasmruntime.ErrRuntimeOutOfBoundsMemoryAccess) + } + ce.pushValue(memoryInst.Wait32(offset, uint32(exp), timeout, func(mem *wasm.MemoryInstance, offset uint32) uint32 { + mem.Mux.Lock() + defer mem.Mux.Unlock() + value, _ := mem.ReadUint32Le(offset) + return value + })) + case unsignedTypeI64: + if offset%8 != 0 { + panic(wasmruntime.ErrRuntimeUnalignedAtomic) + } + if int(offset) > len(memoryInst.Buffer)-8 { + panic(wasmruntime.ErrRuntimeOutOfBoundsMemoryAccess) + } + ce.pushValue(memoryInst.Wait64(offset, exp, timeout, func(mem *wasm.MemoryInstance, offset uint32) uint64 { + mem.Mux.Lock() + defer mem.Mux.Unlock() + value, _ := mem.ReadUint64Le(offset) + return value + })) + } + frame.pc++ + case operationKindAtomicMemoryNotify: + count := ce.popValue() + offset := ce.popMemoryOffset(op) + if offset%4 != 0 { + panic(wasmruntime.ErrRuntimeUnalignedAtomic) + } + // Just a bounds check + if offset >= memoryInst.Size() { + panic(wasmruntime.ErrRuntimeOutOfBoundsMemoryAccess) + } + res := memoryInst.Notify(offset, uint32(count)) + ce.pushValue(uint64(res)) + frame.pc++ + case operationKindAtomicFence: + // Memory not required for fence only + if memoryInst != nil { + // An empty critical section can be used as a synchronization primitive, which is what + // fence is. Probably, there are no spectests or defined behavior to confirm this yet. + memoryInst.Mux.Lock() + memoryInst.Mux.Unlock() //nolint:staticcheck + } + frame.pc++ + case operationKindAtomicLoad: + offset := ce.popMemoryOffset(op) + switch unsignedType(op.B1) { + case unsignedTypeI32: + if offset%4 != 0 { + panic(wasmruntime.ErrRuntimeUnalignedAtomic) + } + memoryInst.Mux.Lock() + val, ok := memoryInst.ReadUint32Le(offset) + memoryInst.Mux.Unlock() + if !ok { + panic(wasmruntime.ErrRuntimeOutOfBoundsMemoryAccess) + } + ce.pushValue(uint64(val)) + case unsignedTypeI64: + if offset%8 != 0 { + panic(wasmruntime.ErrRuntimeUnalignedAtomic) + } + memoryInst.Mux.Lock() + val, ok := memoryInst.ReadUint64Le(offset) + memoryInst.Mux.Unlock() + if !ok { + panic(wasmruntime.ErrRuntimeOutOfBoundsMemoryAccess) + } + ce.pushValue(val) + } + frame.pc++ + case operationKindAtomicLoad8: + offset := ce.popMemoryOffset(op) + memoryInst.Mux.Lock() + val, ok := memoryInst.ReadByte(offset) + memoryInst.Mux.Unlock() + if !ok { + panic(wasmruntime.ErrRuntimeOutOfBoundsMemoryAccess) + } + ce.pushValue(uint64(val)) + frame.pc++ + case operationKindAtomicLoad16: + offset := ce.popMemoryOffset(op) + if offset%2 != 0 { + panic(wasmruntime.ErrRuntimeUnalignedAtomic) + } + memoryInst.Mux.Lock() + val, ok := memoryInst.ReadUint16Le(offset) + memoryInst.Mux.Unlock() + if !ok { + panic(wasmruntime.ErrRuntimeOutOfBoundsMemoryAccess) + } + ce.pushValue(uint64(val)) + frame.pc++ + case operationKindAtomicStore: + val := ce.popValue() + offset := ce.popMemoryOffset(op) + switch unsignedType(op.B1) { + case unsignedTypeI32: + if offset%4 != 0 { + panic(wasmruntime.ErrRuntimeUnalignedAtomic) + } + memoryInst.Mux.Lock() + ok := memoryInst.WriteUint32Le(offset, uint32(val)) + memoryInst.Mux.Unlock() + if !ok { + panic(wasmruntime.ErrRuntimeOutOfBoundsMemoryAccess) + } + case unsignedTypeI64: + if offset%8 != 0 { + panic(wasmruntime.ErrRuntimeUnalignedAtomic) + } + memoryInst.Mux.Lock() + ok := memoryInst.WriteUint64Le(offset, val) + memoryInst.Mux.Unlock() + if !ok { + panic(wasmruntime.ErrRuntimeOutOfBoundsMemoryAccess) + } + } + frame.pc++ + case operationKindAtomicStore8: + val := byte(ce.popValue()) + offset := ce.popMemoryOffset(op) + memoryInst.Mux.Lock() + ok := memoryInst.WriteByte(offset, val) + memoryInst.Mux.Unlock() + if !ok { + panic(wasmruntime.ErrRuntimeOutOfBoundsMemoryAccess) + } + frame.pc++ + case operationKindAtomicStore16: + val := uint16(ce.popValue()) + offset := ce.popMemoryOffset(op) + if offset%2 != 0 { + panic(wasmruntime.ErrRuntimeUnalignedAtomic) + } + memoryInst.Mux.Lock() + ok := memoryInst.WriteUint16Le(offset, val) + memoryInst.Mux.Unlock() + if !ok { + panic(wasmruntime.ErrRuntimeOutOfBoundsMemoryAccess) + } + frame.pc++ + case operationKindAtomicRMW: + val := ce.popValue() + offset := ce.popMemoryOffset(op) + switch unsignedType(op.B1) { + case unsignedTypeI32: + if offset%4 != 0 { + panic(wasmruntime.ErrRuntimeUnalignedAtomic) + } + memoryInst.Mux.Lock() + old, ok := memoryInst.ReadUint32Le(offset) + if !ok { + memoryInst.Mux.Unlock() + panic(wasmruntime.ErrRuntimeOutOfBoundsMemoryAccess) + } + var newVal uint32 + switch atomicArithmeticOp(op.B2) { + case atomicArithmeticOpAdd: + newVal = old + uint32(val) + case atomicArithmeticOpSub: + newVal = old - uint32(val) + case atomicArithmeticOpAnd: + newVal = old & uint32(val) + case atomicArithmeticOpOr: + newVal = old | uint32(val) + case atomicArithmeticOpXor: + newVal = old ^ uint32(val) + case atomicArithmeticOpNop: + newVal = uint32(val) + } + memoryInst.WriteUint32Le(offset, newVal) + memoryInst.Mux.Unlock() + ce.pushValue(uint64(old)) + case unsignedTypeI64: + if offset%8 != 0 { + panic(wasmruntime.ErrRuntimeUnalignedAtomic) + } + memoryInst.Mux.Lock() + old, ok := memoryInst.ReadUint64Le(offset) + if !ok { + memoryInst.Mux.Unlock() + panic(wasmruntime.ErrRuntimeOutOfBoundsMemoryAccess) + } + var newVal uint64 + switch atomicArithmeticOp(op.B2) { + case atomicArithmeticOpAdd: + newVal = old + val + case atomicArithmeticOpSub: + newVal = old - val + case atomicArithmeticOpAnd: + newVal = old & val + case atomicArithmeticOpOr: + newVal = old | val + case atomicArithmeticOpXor: + newVal = old ^ val + case atomicArithmeticOpNop: + newVal = val + } + memoryInst.WriteUint64Le(offset, newVal) + memoryInst.Mux.Unlock() + ce.pushValue(old) + } + frame.pc++ + case operationKindAtomicRMW8: + val := ce.popValue() + offset := ce.popMemoryOffset(op) + memoryInst.Mux.Lock() + old, ok := memoryInst.ReadByte(offset) + if !ok { + memoryInst.Mux.Unlock() + panic(wasmruntime.ErrRuntimeOutOfBoundsMemoryAccess) + } + arg := byte(val) + var newVal byte + switch atomicArithmeticOp(op.B2) { + case atomicArithmeticOpAdd: + newVal = old + arg + case atomicArithmeticOpSub: + newVal = old - arg + case atomicArithmeticOpAnd: + newVal = old & arg + case atomicArithmeticOpOr: + newVal = old | arg + case atomicArithmeticOpXor: + newVal = old ^ arg + case atomicArithmeticOpNop: + newVal = arg + } + memoryInst.WriteByte(offset, newVal) + memoryInst.Mux.Unlock() + ce.pushValue(uint64(old)) + frame.pc++ + case operationKindAtomicRMW16: + val := ce.popValue() + offset := ce.popMemoryOffset(op) + if offset%2 != 0 { + panic(wasmruntime.ErrRuntimeUnalignedAtomic) + } + memoryInst.Mux.Lock() + old, ok := memoryInst.ReadUint16Le(offset) + if !ok { + memoryInst.Mux.Unlock() + panic(wasmruntime.ErrRuntimeOutOfBoundsMemoryAccess) + } + arg := uint16(val) + var newVal uint16 + switch atomicArithmeticOp(op.B2) { + case atomicArithmeticOpAdd: + newVal = old + arg + case atomicArithmeticOpSub: + newVal = old - arg + case atomicArithmeticOpAnd: + newVal = old & arg + case atomicArithmeticOpOr: + newVal = old | arg + case atomicArithmeticOpXor: + newVal = old ^ arg + case atomicArithmeticOpNop: + newVal = arg + } + memoryInst.WriteUint16Le(offset, newVal) + memoryInst.Mux.Unlock() + ce.pushValue(uint64(old)) + frame.pc++ + case operationKindAtomicRMWCmpxchg: + rep := ce.popValue() + exp := ce.popValue() + offset := ce.popMemoryOffset(op) + switch unsignedType(op.B1) { + case unsignedTypeI32: + if offset%4 != 0 { + panic(wasmruntime.ErrRuntimeUnalignedAtomic) + } + memoryInst.Mux.Lock() + old, ok := memoryInst.ReadUint32Le(offset) + if !ok { + memoryInst.Mux.Unlock() + panic(wasmruntime.ErrRuntimeOutOfBoundsMemoryAccess) + } + if old == uint32(exp) { + memoryInst.WriteUint32Le(offset, uint32(rep)) + } + memoryInst.Mux.Unlock() + ce.pushValue(uint64(old)) + case unsignedTypeI64: + if offset%8 != 0 { + panic(wasmruntime.ErrRuntimeUnalignedAtomic) + } + memoryInst.Mux.Lock() + old, ok := memoryInst.ReadUint64Le(offset) + if !ok { + memoryInst.Mux.Unlock() + panic(wasmruntime.ErrRuntimeOutOfBoundsMemoryAccess) + } + if old == exp { + memoryInst.WriteUint64Le(offset, rep) + } + memoryInst.Mux.Unlock() + ce.pushValue(old) + } + frame.pc++ + case operationKindAtomicRMW8Cmpxchg: + rep := byte(ce.popValue()) + exp := byte(ce.popValue()) + offset := ce.popMemoryOffset(op) + memoryInst.Mux.Lock() + old, ok := memoryInst.ReadByte(offset) + if !ok { + memoryInst.Mux.Unlock() + panic(wasmruntime.ErrRuntimeOutOfBoundsMemoryAccess) + } + if old == exp { + memoryInst.WriteByte(offset, rep) + } + memoryInst.Mux.Unlock() + ce.pushValue(uint64(old)) + frame.pc++ + case operationKindAtomicRMW16Cmpxchg: + rep := uint16(ce.popValue()) + exp := uint16(ce.popValue()) + offset := ce.popMemoryOffset(op) + if offset%2 != 0 { + panic(wasmruntime.ErrRuntimeUnalignedAtomic) + } + memoryInst.Mux.Lock() + old, ok := memoryInst.ReadUint16Le(offset) + if !ok { + memoryInst.Mux.Unlock() + panic(wasmruntime.ErrRuntimeOutOfBoundsMemoryAccess) + } + if old == exp { + memoryInst.WriteUint16Le(offset, rep) + } + memoryInst.Mux.Unlock() + ce.pushValue(uint64(old)) + frame.pc++ + default: + frame.pc++ + } + } + ce.popFrame() +} + +func wasmCompatMax32bits(v1, v2 uint32) uint64 { + return uint64(math.Float32bits(moremath.WasmCompatMax32( + math.Float32frombits(v1), + math.Float32frombits(v2), + ))) +} + +func wasmCompatMin32bits(v1, v2 uint32) uint64 { + return uint64(math.Float32bits(moremath.WasmCompatMin32( + math.Float32frombits(v1), + math.Float32frombits(v2), + ))) +} + +func addFloat32bits(v1, v2 uint32) uint64 { + return uint64(math.Float32bits(math.Float32frombits(v1) + math.Float32frombits(v2))) +} + +func subFloat32bits(v1, v2 uint32) uint64 { + return uint64(math.Float32bits(math.Float32frombits(v1) - math.Float32frombits(v2))) +} + +func mulFloat32bits(v1, v2 uint32) uint64 { + return uint64(math.Float32bits(math.Float32frombits(v1) * math.Float32frombits(v2))) +} + +func divFloat32bits(v1, v2 uint32) uint64 { + return uint64(math.Float32bits(math.Float32frombits(v1) / math.Float32frombits(v2))) +} + +// https://www.w3.org/TR/2022/WD-wasm-core-2-20220419/exec/numerics.html#xref-exec-numerics-op-flt-mathrm-flt-n-z-1-z-2 +func flt32(z1, z2 float32) bool { + if z1 != z1 || z2 != z2 { + return false + } else if z1 == z2 { + return false + } else if math.IsInf(float64(z1), 1) { + return false + } else if math.IsInf(float64(z1), -1) { + return true + } else if math.IsInf(float64(z2), 1) { + return true + } else if math.IsInf(float64(z2), -1) { + return false + } + return z1 < z2 +} + +// https://www.w3.org/TR/2022/WD-wasm-core-2-20220419/exec/numerics.html#xref-exec-numerics-op-flt-mathrm-flt-n-z-1-z-2 +func flt64(z1, z2 float64) bool { + if z1 != z1 || z2 != z2 { + return false + } else if z1 == z2 { + return false + } else if math.IsInf(z1, 1) { + return false + } else if math.IsInf(z1, -1) { + return true + } else if math.IsInf(z2, 1) { + return true + } else if math.IsInf(z2, -1) { + return false + } + return z1 < z2 +} + +func i8RoundingAverage(v1, v2 byte) byte { + // https://github.com/WebAssembly/spec/blob/wg-2.0.draft1/proposals/simd/SIMD.md#lane-wise-integer-rounding-average + return byte((uint16(v1) + uint16(v2) + uint16(1)) / 2) +} + +func i16RoundingAverage(v1, v2 uint16) uint16 { + // https://github.com/WebAssembly/spec/blob/wg-2.0.draft1/proposals/simd/SIMD.md#lane-wise-integer-rounding-average + return uint16((uint32(v1) + uint32(v2) + 1) / 2) +} + +func i8Abs(v byte) byte { + if i := int8(v); i < 0 { + return byte(-i) + } else { + return byte(i) + } +} + +func i8MaxU(v1, v2 byte) byte { + if v1 < v2 { + return v2 + } else { + return v1 + } +} + +func i8MinU(v1, v2 byte) byte { + if v1 > v2 { + return v2 + } else { + return v1 + } +} + +func i8MaxS(v1, v2 byte) byte { + if int8(v1) < int8(v2) { + return v2 + } else { + return v1 + } +} + +func i8MinS(v1, v2 byte) byte { + if int8(v1) > int8(v2) { + return v2 + } else { + return v1 + } +} + +func i16MaxU(v1, v2 uint16) uint16 { + if v1 < v2 { + return v2 + } else { + return v1 + } +} + +func i16MinU(v1, v2 uint16) uint16 { + if v1 > v2 { + return v2 + } else { + return v1 + } +} + +func i16MaxS(v1, v2 uint16) uint16 { + if int16(v1) < int16(v2) { + return v2 + } else { + return v1 + } +} + +func i16MinS(v1, v2 uint16) uint16 { + if int16(v1) > int16(v2) { + return v2 + } else { + return v1 + } +} + +func i32MaxU(v1, v2 uint32) uint32 { + if v1 < v2 { + return v2 + } else { + return v1 + } +} + +func i32MinU(v1, v2 uint32) uint32 { + if v1 > v2 { + return v2 + } else { + return v1 + } +} + +func i32MaxS(v1, v2 uint32) uint32 { + if int32(v1) < int32(v2) { + return v2 + } else { + return v1 + } +} + +func i32MinS(v1, v2 uint32) uint32 { + if int32(v1) > int32(v2) { + return v2 + } else { + return v1 + } +} + +func i16Abs(v uint16) uint16 { + if i := int16(v); i < 0 { + return uint16(-i) + } else { + return uint16(i) + } +} + +func i32Abs(v uint32) uint32 { + if i := int32(v); i < 0 { + return uint32(-i) + } else { + return uint32(i) + } +} + +func (ce *callEngine) callNativeFuncWithListener(ctx context.Context, m *wasm.ModuleInstance, f *function, fnl experimental.FunctionListener) context.Context { + def, typ := f.definition(), f.funcType + + ce.stackIterator.reset(ce.stack, ce.frames, f) + fnl.Before(ctx, m, def, ce.peekValues(typ.ParamNumInUint64), &ce.stackIterator) + ce.stackIterator.clear() + ce.callNativeFunc(ctx, m, f) + fnl.After(ctx, m, def, ce.peekValues(typ.ResultNumInUint64)) + return ctx +} + +// popMemoryOffset takes a memory offset off the stack for use in load and store instructions. +// As the top of stack value is 64-bit, this ensures it is in range before returning it. +func (ce *callEngine) popMemoryOffset(op *unionOperation) uint32 { + offset := op.U2 + ce.popValue() + if offset > math.MaxUint32 { + panic(wasmruntime.ErrRuntimeOutOfBoundsMemoryAccess) + } + return uint32(offset) +} + +func (ce *callEngine) callGoFuncWithStack(ctx context.Context, m *wasm.ModuleInstance, f *function) { + typ := f.funcType + paramLen := typ.ParamNumInUint64 + resultLen := typ.ResultNumInUint64 + stackLen := paramLen + + // In the interpreter engine, ce.stack may only have capacity to store + // parameters. Grow when there are more results than parameters. + if growLen := resultLen - paramLen; growLen > 0 { + for i := 0; i < growLen; i++ { + ce.stack = append(ce.stack, 0) + } + stackLen += growLen + } + + // Pass the stack elements to the go function. + stack := ce.stack[len(ce.stack)-stackLen:] + ce.callGoFunc(ctx, m, f, stack) + + // Shrink the stack when there were more parameters than results. + if shrinkLen := paramLen - resultLen; shrinkLen > 0 { + ce.stack = ce.stack[0 : len(ce.stack)-shrinkLen] + } +} diff --git a/vendor/github.com/tetratelabs/wazero/internal/engine/interpreter/operations.go b/vendor/github.com/tetratelabs/wazero/internal/engine/interpreter/operations.go new file mode 100644 index 000000000..3087a718f --- /dev/null +++ b/vendor/github.com/tetratelabs/wazero/internal/engine/interpreter/operations.go @@ -0,0 +1,2812 @@ +package interpreter + +import ( + "fmt" + "math" + "strings" +) + +// unsignedInt represents unsigned 32-bit or 64-bit integers. +type unsignedInt byte + +const ( + unsignedInt32 unsignedInt = iota + unsignedInt64 +) + +// String implements fmt.Stringer. +func (s unsignedInt) String() (ret string) { + switch s { + case unsignedInt32: + ret = "i32" + case unsignedInt64: + ret = "i64" + } + return +} + +// signedInt represents signed or unsigned integers. +type signedInt byte + +const ( + signedInt32 signedInt = iota + signedInt64 + signedUint32 + signedUint64 +) + +// String implements fmt.Stringer. +func (s signedInt) String() (ret string) { + switch s { + case signedUint32: + ret = "u32" + case signedUint64: + ret = "u64" + case signedInt32: + ret = "s32" + case signedInt64: + ret = "s64" + } + return +} + +// float represents the scalar double or single precision floating points. +type float byte + +const ( + f32 float = iota + f64 +) + +// String implements fmt.Stringer. +func (s float) String() (ret string) { + switch s { + case f32: + ret = "f32" + case f64: + ret = "f64" + } + return +} + +// unsignedType is the union of unsignedInt, float and V128 vector type. +type unsignedType byte + +const ( + unsignedTypeI32 unsignedType = iota + unsignedTypeI64 + unsignedTypeF32 + unsignedTypeF64 + unsignedTypeV128 + unsignedTypeUnknown +) + +// String implements fmt.Stringer. +func (s unsignedType) String() (ret string) { + switch s { + case unsignedTypeI32: + ret = "i32" + case unsignedTypeI64: + ret = "i64" + case unsignedTypeF32: + ret = "f32" + case unsignedTypeF64: + ret = "f64" + case unsignedTypeV128: + ret = "v128" + case unsignedTypeUnknown: + ret = "unknown" + } + return +} + +// signedType is the union of signedInt and float types. +type signedType byte + +const ( + signedTypeInt32 signedType = iota + signedTypeUint32 + signedTypeInt64 + signedTypeUint64 + signedTypeFloat32 + signedTypeFloat64 +) + +// String implements fmt.Stringer. +func (s signedType) String() (ret string) { + switch s { + case signedTypeInt32: + ret = "s32" + case signedTypeUint32: + ret = "u32" + case signedTypeInt64: + ret = "s64" + case signedTypeUint64: + ret = "u64" + case signedTypeFloat32: + ret = "f32" + case signedTypeFloat64: + ret = "f64" + } + return +} + +// operationKind is the Kind of each implementation of Operation interface. +type operationKind uint16 + +// String implements fmt.Stringer. +func (o operationKind) String() (ret string) { + switch o { + case operationKindUnreachable: + ret = "Unreachable" + case operationKindLabel: + ret = "label" + case operationKindBr: + ret = "Br" + case operationKindBrIf: + ret = "BrIf" + case operationKindBrTable: + ret = "BrTable" + case operationKindCall: + ret = "Call" + case operationKindCallIndirect: + ret = "CallIndirect" + case operationKindDrop: + ret = "Drop" + case operationKindSelect: + ret = "Select" + case operationKindPick: + ret = "Pick" + case operationKindSet: + ret = "Swap" + case operationKindGlobalGet: + ret = "GlobalGet" + case operationKindGlobalSet: + ret = "GlobalSet" + case operationKindLoad: + ret = "Load" + case operationKindLoad8: + ret = "Load8" + case operationKindLoad16: + ret = "Load16" + case operationKindLoad32: + ret = "Load32" + case operationKindStore: + ret = "Store" + case operationKindStore8: + ret = "Store8" + case operationKindStore16: + ret = "Store16" + case operationKindStore32: + ret = "Store32" + case operationKindMemorySize: + ret = "MemorySize" + case operationKindMemoryGrow: + ret = "MemoryGrow" + case operationKindConstI32: + ret = "ConstI32" + case operationKindConstI64: + ret = "ConstI64" + case operationKindConstF32: + ret = "ConstF32" + case operationKindConstF64: + ret = "ConstF64" + case operationKindEq: + ret = "Eq" + case operationKindNe: + ret = "Ne" + case operationKindEqz: + ret = "Eqz" + case operationKindLt: + ret = "Lt" + case operationKindGt: + ret = "Gt" + case operationKindLe: + ret = "Le" + case operationKindGe: + ret = "Ge" + case operationKindAdd: + ret = "Add" + case operationKindSub: + ret = "Sub" + case operationKindMul: + ret = "Mul" + case operationKindClz: + ret = "Clz" + case operationKindCtz: + ret = "Ctz" + case operationKindPopcnt: + ret = "Popcnt" + case operationKindDiv: + ret = "Div" + case operationKindRem: + ret = "Rem" + case operationKindAnd: + ret = "And" + case operationKindOr: + ret = "Or" + case operationKindXor: + ret = "Xor" + case operationKindShl: + ret = "Shl" + case operationKindShr: + ret = "Shr" + case operationKindRotl: + ret = "Rotl" + case operationKindRotr: + ret = "Rotr" + case operationKindAbs: + ret = "Abs" + case operationKindNeg: + ret = "Neg" + case operationKindCeil: + ret = "Ceil" + case operationKindFloor: + ret = "Floor" + case operationKindTrunc: + ret = "Trunc" + case operationKindNearest: + ret = "Nearest" + case operationKindSqrt: + ret = "Sqrt" + case operationKindMin: + ret = "Min" + case operationKindMax: + ret = "Max" + case operationKindCopysign: + ret = "Copysign" + case operationKindI32WrapFromI64: + ret = "I32WrapFromI64" + case operationKindITruncFromF: + ret = "ITruncFromF" + case operationKindFConvertFromI: + ret = "FConvertFromI" + case operationKindF32DemoteFromF64: + ret = "F32DemoteFromF64" + case operationKindF64PromoteFromF32: + ret = "F64PromoteFromF32" + case operationKindI32ReinterpretFromF32: + ret = "I32ReinterpretFromF32" + case operationKindI64ReinterpretFromF64: + ret = "I64ReinterpretFromF64" + case operationKindF32ReinterpretFromI32: + ret = "F32ReinterpretFromI32" + case operationKindF64ReinterpretFromI64: + ret = "F64ReinterpretFromI64" + case operationKindExtend: + ret = "Extend" + case operationKindMemoryInit: + ret = "MemoryInit" + case operationKindDataDrop: + ret = "DataDrop" + case operationKindMemoryCopy: + ret = "MemoryCopy" + case operationKindMemoryFill: + ret = "MemoryFill" + case operationKindTableInit: + ret = "TableInit" + case operationKindElemDrop: + ret = "ElemDrop" + case operationKindTableCopy: + ret = "TableCopy" + case operationKindRefFunc: + ret = "RefFunc" + case operationKindTableGet: + ret = "TableGet" + case operationKindTableSet: + ret = "TableSet" + case operationKindTableSize: + ret = "TableSize" + case operationKindTableGrow: + ret = "TableGrow" + case operationKindTableFill: + ret = "TableFill" + case operationKindV128Const: + ret = "ConstV128" + case operationKindV128Add: + ret = "V128Add" + case operationKindV128Sub: + ret = "V128Sub" + case operationKindV128Load: + ret = "V128Load" + case operationKindV128LoadLane: + ret = "V128LoadLane" + case operationKindV128Store: + ret = "V128Store" + case operationKindV128StoreLane: + ret = "V128StoreLane" + case operationKindV128ExtractLane: + ret = "V128ExtractLane" + case operationKindV128ReplaceLane: + ret = "V128ReplaceLane" + case operationKindV128Splat: + ret = "V128Splat" + case operationKindV128Shuffle: + ret = "V128Shuffle" + case operationKindV128Swizzle: + ret = "V128Swizzle" + case operationKindV128AnyTrue: + ret = "V128AnyTrue" + case operationKindV128AllTrue: + ret = "V128AllTrue" + case operationKindV128And: + ret = "V128And" + case operationKindV128Not: + ret = "V128Not" + case operationKindV128Or: + ret = "V128Or" + case operationKindV128Xor: + ret = "V128Xor" + case operationKindV128Bitselect: + ret = "V128Bitselect" + case operationKindV128AndNot: + ret = "V128AndNot" + case operationKindV128BitMask: + ret = "V128BitMask" + case operationKindV128Shl: + ret = "V128Shl" + case operationKindV128Shr: + ret = "V128Shr" + case operationKindV128Cmp: + ret = "V128Cmp" + case operationKindSignExtend32From8: + ret = "SignExtend32From8" + case operationKindSignExtend32From16: + ret = "SignExtend32From16" + case operationKindSignExtend64From8: + ret = "SignExtend64From8" + case operationKindSignExtend64From16: + ret = "SignExtend64From16" + case operationKindSignExtend64From32: + ret = "SignExtend64From32" + case operationKindV128AddSat: + ret = "V128AddSat" + case operationKindV128SubSat: + ret = "V128SubSat" + case operationKindV128Mul: + ret = "V128Mul" + case operationKindV128Div: + ret = "V128Div" + case operationKindV128Neg: + ret = "V128Neg" + case operationKindV128Sqrt: + ret = "V128Sqrt" + case operationKindV128Abs: + ret = "V128Abs" + case operationKindV128Popcnt: + ret = "V128Popcnt" + case operationKindV128Min: + ret = "V128Min" + case operationKindV128Max: + ret = "V128Max" + case operationKindV128AvgrU: + ret = "V128AvgrU" + case operationKindV128Ceil: + ret = "V128Ceil" + case operationKindV128Floor: + ret = "V128Floor" + case operationKindV128Trunc: + ret = "V128Trunc" + case operationKindV128Nearest: + ret = "V128Nearest" + case operationKindV128Pmin: + ret = "V128Pmin" + case operationKindV128Pmax: + ret = "V128Pmax" + case operationKindV128Extend: + ret = "V128Extend" + case operationKindV128ExtMul: + ret = "V128ExtMul" + case operationKindV128Q15mulrSatS: + ret = "V128Q15mulrSatS" + case operationKindV128ExtAddPairwise: + ret = "V128ExtAddPairwise" + case operationKindV128FloatPromote: + ret = "V128FloatPromote" + case operationKindV128FloatDemote: + ret = "V128FloatDemote" + case operationKindV128FConvertFromI: + ret = "V128FConvertFromI" + case operationKindV128Dot: + ret = "V128Dot" + case operationKindV128Narrow: + ret = "V128Narrow" + case operationKindV128ITruncSatFromF: + ret = "V128ITruncSatFromF" + case operationKindBuiltinFunctionCheckExitCode: + ret = "BuiltinFunctionCheckExitCode" + case operationKindAtomicMemoryWait: + ret = "operationKindAtomicMemoryWait" + case operationKindAtomicMemoryNotify: + ret = "operationKindAtomicMemoryNotify" + case operationKindAtomicFence: + ret = "operationKindAtomicFence" + case operationKindAtomicLoad: + ret = "operationKindAtomicLoad" + case operationKindAtomicLoad8: + ret = "operationKindAtomicLoad8" + case operationKindAtomicLoad16: + ret = "operationKindAtomicLoad16" + case operationKindAtomicStore: + ret = "operationKindAtomicStore" + case operationKindAtomicStore8: + ret = "operationKindAtomicStore8" + case operationKindAtomicStore16: + ret = "operationKindAtomicStore16" + case operationKindAtomicRMW: + ret = "operationKindAtomicRMW" + case operationKindAtomicRMW8: + ret = "operationKindAtomicRMW8" + case operationKindAtomicRMW16: + ret = "operationKindAtomicRMW16" + case operationKindAtomicRMWCmpxchg: + ret = "operationKindAtomicRMWCmpxchg" + case operationKindAtomicRMW8Cmpxchg: + ret = "operationKindAtomicRMW8Cmpxchg" + case operationKindAtomicRMW16Cmpxchg: + ret = "operationKindAtomicRMW16Cmpxchg" + default: + panic(fmt.Errorf("unknown operation %d", o)) + } + return +} + +const ( + // operationKindUnreachable is the Kind for NewOperationUnreachable. + operationKindUnreachable operationKind = iota + // operationKindLabel is the Kind for NewOperationLabel. + operationKindLabel + // operationKindBr is the Kind for NewOperationBr. + operationKindBr + // operationKindBrIf is the Kind for NewOperationBrIf. + operationKindBrIf + // operationKindBrTable is the Kind for NewOperationBrTable. + operationKindBrTable + // operationKindCall is the Kind for NewOperationCall. + operationKindCall + // operationKindCallIndirect is the Kind for NewOperationCallIndirect. + operationKindCallIndirect + // operationKindDrop is the Kind for NewOperationDrop. + operationKindDrop + // operationKindSelect is the Kind for NewOperationSelect. + operationKindSelect + // operationKindPick is the Kind for NewOperationPick. + operationKindPick + // operationKindSet is the Kind for NewOperationSet. + operationKindSet + // operationKindGlobalGet is the Kind for NewOperationGlobalGet. + operationKindGlobalGet + // operationKindGlobalSet is the Kind for NewOperationGlobalSet. + operationKindGlobalSet + // operationKindLoad is the Kind for NewOperationLoad. + operationKindLoad + // operationKindLoad8 is the Kind for NewOperationLoad8. + operationKindLoad8 + // operationKindLoad16 is the Kind for NewOperationLoad16. + operationKindLoad16 + // operationKindLoad32 is the Kind for NewOperationLoad32. + operationKindLoad32 + // operationKindStore is the Kind for NewOperationStore. + operationKindStore + // operationKindStore8 is the Kind for NewOperationStore8. + operationKindStore8 + // operationKindStore16 is the Kind for NewOperationStore16. + operationKindStore16 + // operationKindStore32 is the Kind for NewOperationStore32. + operationKindStore32 + // operationKindMemorySize is the Kind for NewOperationMemorySize. + operationKindMemorySize + // operationKindMemoryGrow is the Kind for NewOperationMemoryGrow. + operationKindMemoryGrow + // operationKindConstI32 is the Kind for NewOperationConstI32. + operationKindConstI32 + // operationKindConstI64 is the Kind for NewOperationConstI64. + operationKindConstI64 + // operationKindConstF32 is the Kind for NewOperationConstF32. + operationKindConstF32 + // operationKindConstF64 is the Kind for NewOperationConstF64. + operationKindConstF64 + // operationKindEq is the Kind for NewOperationEq. + operationKindEq + // operationKindNe is the Kind for NewOperationNe. + operationKindNe + // operationKindEqz is the Kind for NewOperationEqz. + operationKindEqz + // operationKindLt is the Kind for NewOperationLt. + operationKindLt + // operationKindGt is the Kind for NewOperationGt. + operationKindGt + // operationKindLe is the Kind for NewOperationLe. + operationKindLe + // operationKindGe is the Kind for NewOperationGe. + operationKindGe + // operationKindAdd is the Kind for NewOperationAdd. + operationKindAdd + // operationKindSub is the Kind for NewOperationSub. + operationKindSub + // operationKindMul is the Kind for NewOperationMul. + operationKindMul + // operationKindClz is the Kind for NewOperationClz. + operationKindClz + // operationKindCtz is the Kind for NewOperationCtz. + operationKindCtz + // operationKindPopcnt is the Kind for NewOperationPopcnt. + operationKindPopcnt + // operationKindDiv is the Kind for NewOperationDiv. + operationKindDiv + // operationKindRem is the Kind for NewOperationRem. + operationKindRem + // operationKindAnd is the Kind for NewOperationAnd. + operationKindAnd + // operationKindOr is the Kind for NewOperationOr. + operationKindOr + // operationKindXor is the Kind for NewOperationXor. + operationKindXor + // operationKindShl is the Kind for NewOperationShl. + operationKindShl + // operationKindShr is the Kind for NewOperationShr. + operationKindShr + // operationKindRotl is the Kind for NewOperationRotl. + operationKindRotl + // operationKindRotr is the Kind for NewOperationRotr. + operationKindRotr + // operationKindAbs is the Kind for NewOperationAbs. + operationKindAbs + // operationKindNeg is the Kind for NewOperationNeg. + operationKindNeg + // operationKindCeil is the Kind for NewOperationCeil. + operationKindCeil + // operationKindFloor is the Kind for NewOperationFloor. + operationKindFloor + // operationKindTrunc is the Kind for NewOperationTrunc. + operationKindTrunc + // operationKindNearest is the Kind for NewOperationNearest. + operationKindNearest + // operationKindSqrt is the Kind for NewOperationSqrt. + operationKindSqrt + // operationKindMin is the Kind for NewOperationMin. + operationKindMin + // operationKindMax is the Kind for NewOperationMax. + operationKindMax + // operationKindCopysign is the Kind for NewOperationCopysign. + operationKindCopysign + // operationKindI32WrapFromI64 is the Kind for NewOperationI32WrapFromI64. + operationKindI32WrapFromI64 + // operationKindITruncFromF is the Kind for NewOperationITruncFromF. + operationKindITruncFromF + // operationKindFConvertFromI is the Kind for NewOperationFConvertFromI. + operationKindFConvertFromI + // operationKindF32DemoteFromF64 is the Kind for NewOperationF32DemoteFromF64. + operationKindF32DemoteFromF64 + // operationKindF64PromoteFromF32 is the Kind for NewOperationF64PromoteFromF32. + operationKindF64PromoteFromF32 + // operationKindI32ReinterpretFromF32 is the Kind for NewOperationI32ReinterpretFromF32. + operationKindI32ReinterpretFromF32 + // operationKindI64ReinterpretFromF64 is the Kind for NewOperationI64ReinterpretFromF64. + operationKindI64ReinterpretFromF64 + // operationKindF32ReinterpretFromI32 is the Kind for NewOperationF32ReinterpretFromI32. + operationKindF32ReinterpretFromI32 + // operationKindF64ReinterpretFromI64 is the Kind for NewOperationF64ReinterpretFromI64. + operationKindF64ReinterpretFromI64 + // operationKindExtend is the Kind for NewOperationExtend. + operationKindExtend + // operationKindSignExtend32From8 is the Kind for NewOperationSignExtend32From8. + operationKindSignExtend32From8 + // operationKindSignExtend32From16 is the Kind for NewOperationSignExtend32From16. + operationKindSignExtend32From16 + // operationKindSignExtend64From8 is the Kind for NewOperationSignExtend64From8. + operationKindSignExtend64From8 + // operationKindSignExtend64From16 is the Kind for NewOperationSignExtend64From16. + operationKindSignExtend64From16 + // operationKindSignExtend64From32 is the Kind for NewOperationSignExtend64From32. + operationKindSignExtend64From32 + // operationKindMemoryInit is the Kind for NewOperationMemoryInit. + operationKindMemoryInit + // operationKindDataDrop is the Kind for NewOperationDataDrop. + operationKindDataDrop + // operationKindMemoryCopy is the Kind for NewOperationMemoryCopy. + operationKindMemoryCopy + // operationKindMemoryFill is the Kind for NewOperationMemoryFill. + operationKindMemoryFill + // operationKindTableInit is the Kind for NewOperationTableInit. + operationKindTableInit + // operationKindElemDrop is the Kind for NewOperationElemDrop. + operationKindElemDrop + // operationKindTableCopy is the Kind for NewOperationTableCopy. + operationKindTableCopy + // operationKindRefFunc is the Kind for NewOperationRefFunc. + operationKindRefFunc + // operationKindTableGet is the Kind for NewOperationTableGet. + operationKindTableGet + // operationKindTableSet is the Kind for NewOperationTableSet. + operationKindTableSet + // operationKindTableSize is the Kind for NewOperationTableSize. + operationKindTableSize + // operationKindTableGrow is the Kind for NewOperationTableGrow. + operationKindTableGrow + // operationKindTableFill is the Kind for NewOperationTableFill. + operationKindTableFill + + // Vector value related instructions are prefixed by V128. + + // operationKindV128Const is the Kind for NewOperationV128Const. + operationKindV128Const + // operationKindV128Add is the Kind for NewOperationV128Add. + operationKindV128Add + // operationKindV128Sub is the Kind for NewOperationV128Sub. + operationKindV128Sub + // operationKindV128Load is the Kind for NewOperationV128Load. + operationKindV128Load + // operationKindV128LoadLane is the Kind for NewOperationV128LoadLane. + operationKindV128LoadLane + // operationKindV128Store is the Kind for NewOperationV128Store. + operationKindV128Store + // operationKindV128StoreLane is the Kind for NewOperationV128StoreLane. + operationKindV128StoreLane + // operationKindV128ExtractLane is the Kind for NewOperationV128ExtractLane. + operationKindV128ExtractLane + // operationKindV128ReplaceLane is the Kind for NewOperationV128ReplaceLane. + operationKindV128ReplaceLane + // operationKindV128Splat is the Kind for NewOperationV128Splat. + operationKindV128Splat + // operationKindV128Shuffle is the Kind for NewOperationV128Shuffle. + operationKindV128Shuffle + // operationKindV128Swizzle is the Kind for NewOperationV128Swizzle. + operationKindV128Swizzle + // operationKindV128AnyTrue is the Kind for NewOperationV128AnyTrue. + operationKindV128AnyTrue + // operationKindV128AllTrue is the Kind for NewOperationV128AllTrue. + operationKindV128AllTrue + // operationKindV128BitMask is the Kind for NewOperationV128BitMask. + operationKindV128BitMask + // operationKindV128And is the Kind for NewOperationV128And. + operationKindV128And + // operationKindV128Not is the Kind for NewOperationV128Not. + operationKindV128Not + // operationKindV128Or is the Kind for NewOperationV128Or. + operationKindV128Or + // operationKindV128Xor is the Kind for NewOperationV128Xor. + operationKindV128Xor + // operationKindV128Bitselect is the Kind for NewOperationV128Bitselect. + operationKindV128Bitselect + // operationKindV128AndNot is the Kind for NewOperationV128AndNot. + operationKindV128AndNot + // operationKindV128Shl is the Kind for NewOperationV128Shl. + operationKindV128Shl + // operationKindV128Shr is the Kind for NewOperationV128Shr. + operationKindV128Shr + // operationKindV128Cmp is the Kind for NewOperationV128Cmp. + operationKindV128Cmp + // operationKindV128AddSat is the Kind for NewOperationV128AddSat. + operationKindV128AddSat + // operationKindV128SubSat is the Kind for NewOperationV128SubSat. + operationKindV128SubSat + // operationKindV128Mul is the Kind for NewOperationV128Mul. + operationKindV128Mul + // operationKindV128Div is the Kind for NewOperationV128Div. + operationKindV128Div + // operationKindV128Neg is the Kind for NewOperationV128Neg. + operationKindV128Neg + // operationKindV128Sqrt is the Kind for NewOperationV128Sqrt. + operationKindV128Sqrt + // operationKindV128Abs is the Kind for NewOperationV128Abs. + operationKindV128Abs + // operationKindV128Popcnt is the Kind for NewOperationV128Popcnt. + operationKindV128Popcnt + // operationKindV128Min is the Kind for NewOperationV128Min. + operationKindV128Min + // operationKindV128Max is the Kind for NewOperationV128Max. + operationKindV128Max + // operationKindV128AvgrU is the Kind for NewOperationV128AvgrU. + operationKindV128AvgrU + // operationKindV128Pmin is the Kind for NewOperationV128Pmin. + operationKindV128Pmin + // operationKindV128Pmax is the Kind for NewOperationV128Pmax. + operationKindV128Pmax + // operationKindV128Ceil is the Kind for NewOperationV128Ceil. + operationKindV128Ceil + // operationKindV128Floor is the Kind for NewOperationV128Floor. + operationKindV128Floor + // operationKindV128Trunc is the Kind for NewOperationV128Trunc. + operationKindV128Trunc + // operationKindV128Nearest is the Kind for NewOperationV128Nearest. + operationKindV128Nearest + // operationKindV128Extend is the Kind for NewOperationV128Extend. + operationKindV128Extend + // operationKindV128ExtMul is the Kind for NewOperationV128ExtMul. + operationKindV128ExtMul + // operationKindV128Q15mulrSatS is the Kind for NewOperationV128Q15mulrSatS. + operationKindV128Q15mulrSatS + // operationKindV128ExtAddPairwise is the Kind for NewOperationV128ExtAddPairwise. + operationKindV128ExtAddPairwise + // operationKindV128FloatPromote is the Kind for NewOperationV128FloatPromote. + operationKindV128FloatPromote + // operationKindV128FloatDemote is the Kind for NewOperationV128FloatDemote. + operationKindV128FloatDemote + // operationKindV128FConvertFromI is the Kind for NewOperationV128FConvertFromI. + operationKindV128FConvertFromI + // operationKindV128Dot is the Kind for NewOperationV128Dot. + operationKindV128Dot + // operationKindV128Narrow is the Kind for NewOperationV128Narrow. + operationKindV128Narrow + // operationKindV128ITruncSatFromF is the Kind for NewOperationV128ITruncSatFromF. + operationKindV128ITruncSatFromF + + // operationKindBuiltinFunctionCheckExitCode is the Kind for NewOperationBuiltinFunctionCheckExitCode. + operationKindBuiltinFunctionCheckExitCode + + // operationKindAtomicMemoryWait is the kind for NewOperationAtomicMemoryWait. + operationKindAtomicMemoryWait + // operationKindAtomicMemoryNotify is the kind for NewOperationAtomicMemoryNotify. + operationKindAtomicMemoryNotify + // operationKindAtomicFence is the kind for NewOperationAtomicFence. + operationKindAtomicFence + // operationKindAtomicLoad is the kind for NewOperationAtomicLoad. + operationKindAtomicLoad + // operationKindAtomicLoad8 is the kind for NewOperationAtomicLoad8. + operationKindAtomicLoad8 + // operationKindAtomicLoad16 is the kind for NewOperationAtomicLoad16. + operationKindAtomicLoad16 + // operationKindAtomicStore is the kind for NewOperationAtomicStore. + operationKindAtomicStore + // operationKindAtomicStore8 is the kind for NewOperationAtomicStore8. + operationKindAtomicStore8 + // operationKindAtomicStore16 is the kind for NewOperationAtomicStore16. + operationKindAtomicStore16 + + // operationKindAtomicRMW is the kind for NewOperationAtomicRMW. + operationKindAtomicRMW + // operationKindAtomicRMW8 is the kind for NewOperationAtomicRMW8. + operationKindAtomicRMW8 + // operationKindAtomicRMW16 is the kind for NewOperationAtomicRMW16. + operationKindAtomicRMW16 + + // operationKindAtomicRMWCmpxchg is the kind for NewOperationAtomicRMWCmpxchg. + operationKindAtomicRMWCmpxchg + // operationKindAtomicRMW8Cmpxchg is the kind for NewOperationAtomicRMW8Cmpxchg. + operationKindAtomicRMW8Cmpxchg + // operationKindAtomicRMW16Cmpxchg is the kind for NewOperationAtomicRMW16Cmpxchg. + operationKindAtomicRMW16Cmpxchg + + // operationKindEnd is always placed at the bottom of this iota definition to be used in the test. + operationKindEnd +) + +// NewOperationBuiltinFunctionCheckExitCode is a constructor for unionOperation with Kind operationKindBuiltinFunctionCheckExitCode. +// +// OperationBuiltinFunctionCheckExitCode corresponds to the instruction to check the api.Module is already closed due to +// context.DeadlineExceeded, context.Canceled, or the explicit call of CloseWithExitCode on api.Module. +func newOperationBuiltinFunctionCheckExitCode() unionOperation { + return unionOperation{Kind: operationKindBuiltinFunctionCheckExitCode} +} + +// label is the unique identifier for each block in a single function in interpreterir +// where "block" consists of multiple operations, and must End with branching operations +// (e.g. operationKindBr or operationKindBrIf). +type label uint64 + +// Kind returns the labelKind encoded in this label. +func (l label) Kind() labelKind { + return labelKind(uint32(l)) +} + +// FrameID returns the frame id encoded in this label. +func (l label) FrameID() int { + return int(uint32(l >> 32)) +} + +// NewLabel is a constructor for a label. +func newLabel(kind labelKind, frameID uint32) label { + return label(kind) | label(frameID)<<32 +} + +// String implements fmt.Stringer. +func (l label) String() (ret string) { + frameID := l.FrameID() + switch l.Kind() { + case labelKindHeader: + ret = fmt.Sprintf(".L%d", frameID) + case labelKindElse: + ret = fmt.Sprintf(".L%d_else", frameID) + case labelKindContinuation: + ret = fmt.Sprintf(".L%d_cont", frameID) + case labelKindReturn: + return ".return" + } + return +} + +func (l label) IsReturnTarget() bool { + return l.Kind() == labelKindReturn +} + +// labelKind is the Kind of the label. +type labelKind = byte + +const ( + // labelKindHeader is the header for various blocks. For example, the "then" block of + // wasm.OpcodeIfName in Wasm has the label of this Kind. + labelKindHeader labelKind = iota + // labelKindElse is the Kind of label for "else" block of wasm.OpcodeIfName in Wasm. + labelKindElse + // labelKindContinuation is the Kind of label which is the continuation of blocks. + // For example, for wasm text like + // (func + // .... + // (if (local.get 0) (then (nop)) (else (nop))) + // return + // ) + // we have the continuation block (of if-block) corresponding to "return" opcode. + labelKindContinuation + labelKindReturn + labelKindNum +) + +// unionOperation implements Operation and is the compilation (engine.lowerIR) result of a interpreterir.Operation. +// +// Not all operations result in a unionOperation, e.g. interpreterir.OperationI32ReinterpretFromF32, and some operations are +// more complex than others, e.g. interpreterir.NewOperationBrTable. +// +// Note: This is a form of union type as it can store fields needed for any operation. Hence, most fields are opaque and +// only relevant when in context of its kind. +type unionOperation struct { + // Kind determines how to interpret the other fields in this struct. + Kind operationKind + B1, B2 byte + B3 bool + U1, U2 uint64 + U3 uint64 + Us []uint64 +} + +// String implements fmt.Stringer. +func (o unionOperation) String() string { + switch o.Kind { + case operationKindUnreachable, + operationKindSelect, + operationKindMemorySize, + operationKindMemoryGrow, + operationKindI32WrapFromI64, + operationKindF32DemoteFromF64, + operationKindF64PromoteFromF32, + operationKindI32ReinterpretFromF32, + operationKindI64ReinterpretFromF64, + operationKindF32ReinterpretFromI32, + operationKindF64ReinterpretFromI64, + operationKindSignExtend32From8, + operationKindSignExtend32From16, + operationKindSignExtend64From8, + operationKindSignExtend64From16, + operationKindSignExtend64From32, + operationKindMemoryInit, + operationKindDataDrop, + operationKindMemoryCopy, + operationKindMemoryFill, + operationKindTableInit, + operationKindElemDrop, + operationKindTableCopy, + operationKindRefFunc, + operationKindTableGet, + operationKindTableSet, + operationKindTableSize, + operationKindTableGrow, + operationKindTableFill, + operationKindBuiltinFunctionCheckExitCode: + return o.Kind.String() + + case operationKindCall, + operationKindGlobalGet, + operationKindGlobalSet: + return fmt.Sprintf("%s %d", o.Kind, o.B1) + + case operationKindLabel: + return label(o.U1).String() + + case operationKindBr: + return fmt.Sprintf("%s %s", o.Kind, label(o.U1).String()) + + case operationKindBrIf: + thenTarget := label(o.U1) + elseTarget := label(o.U2) + return fmt.Sprintf("%s %s, %s", o.Kind, thenTarget, elseTarget) + + case operationKindBrTable: + var targets []string + var defaultLabel label + if len(o.Us) > 0 { + targets = make([]string, len(o.Us)-1) + for i, t := range o.Us[1:] { + targets[i] = label(t).String() + } + defaultLabel = label(o.Us[0]) + } + return fmt.Sprintf("%s [%s] %s", o.Kind, strings.Join(targets, ","), defaultLabel) + + case operationKindCallIndirect: + return fmt.Sprintf("%s: type=%d, table=%d", o.Kind, o.U1, o.U2) + + case operationKindDrop: + start := int64(o.U1) + end := int64(o.U2) + return fmt.Sprintf("%s %d..%d", o.Kind, start, end) + + case operationKindPick, operationKindSet: + return fmt.Sprintf("%s %d (is_vector=%v)", o.Kind, o.U1, o.B3) + + case operationKindLoad, operationKindStore: + return fmt.Sprintf("%s.%s (align=%d, offset=%d)", unsignedType(o.B1), o.Kind, o.U1, o.U2) + + case operationKindLoad8, + operationKindLoad16: + return fmt.Sprintf("%s.%s (align=%d, offset=%d)", signedType(o.B1), o.Kind, o.U1, o.U2) + + case operationKindStore8, + operationKindStore16, + operationKindStore32: + return fmt.Sprintf("%s (align=%d, offset=%d)", o.Kind, o.U1, o.U2) + + case operationKindLoad32: + var t string + if o.B1 == 1 { + t = "i64" + } else { + t = "u64" + } + return fmt.Sprintf("%s.%s (align=%d, offset=%d)", t, o.Kind, o.U1, o.U2) + + case operationKindEq, + operationKindNe, + operationKindAdd, + operationKindSub, + operationKindMul: + return fmt.Sprintf("%s.%s", unsignedType(o.B1), o.Kind) + + case operationKindEqz, + operationKindClz, + operationKindCtz, + operationKindPopcnt, + operationKindAnd, + operationKindOr, + operationKindXor, + operationKindShl, + operationKindRotl, + operationKindRotr: + return fmt.Sprintf("%s.%s", unsignedInt(o.B1), o.Kind) + + case operationKindRem, operationKindShr: + return fmt.Sprintf("%s.%s", signedInt(o.B1), o.Kind) + + case operationKindLt, + operationKindGt, + operationKindLe, + operationKindGe, + operationKindDiv: + return fmt.Sprintf("%s.%s", signedType(o.B1), o.Kind) + + case operationKindAbs, + operationKindNeg, + operationKindCeil, + operationKindFloor, + operationKindTrunc, + operationKindNearest, + operationKindSqrt, + operationKindMin, + operationKindMax, + operationKindCopysign: + return fmt.Sprintf("%s.%s", float(o.B1), o.Kind) + + case operationKindConstI32, + operationKindConstI64: + return fmt.Sprintf("%s %#x", o.Kind, o.U1) + + case operationKindConstF32: + return fmt.Sprintf("%s %f", o.Kind, math.Float32frombits(uint32(o.U1))) + case operationKindConstF64: + return fmt.Sprintf("%s %f", o.Kind, math.Float64frombits(o.U1)) + + case operationKindITruncFromF: + return fmt.Sprintf("%s.%s.%s (non_trapping=%v)", signedInt(o.B2), o.Kind, float(o.B1), o.B3) + case operationKindFConvertFromI: + return fmt.Sprintf("%s.%s.%s", float(o.B2), o.Kind, signedInt(o.B1)) + case operationKindExtend: + var in, out string + if o.B3 { + in = "i32" + out = "i64" + } else { + in = "u32" + out = "u64" + } + return fmt.Sprintf("%s.%s.%s", out, o.Kind, in) + + case operationKindV128Const: + return fmt.Sprintf("%s [%#x, %#x]", o.Kind, o.U1, o.U2) + case operationKindV128Add, + operationKindV128Sub: + return fmt.Sprintf("%s (shape=%s)", o.Kind, shapeName(o.B1)) + case operationKindV128Load, + operationKindV128LoadLane, + operationKindV128Store, + operationKindV128StoreLane, + operationKindV128ExtractLane, + operationKindV128ReplaceLane, + operationKindV128Splat, + operationKindV128Shuffle, + operationKindV128Swizzle, + operationKindV128AnyTrue, + operationKindV128AllTrue, + operationKindV128BitMask, + operationKindV128And, + operationKindV128Not, + operationKindV128Or, + operationKindV128Xor, + operationKindV128Bitselect, + operationKindV128AndNot, + operationKindV128Shl, + operationKindV128Shr, + operationKindV128Cmp, + operationKindV128AddSat, + operationKindV128SubSat, + operationKindV128Mul, + operationKindV128Div, + operationKindV128Neg, + operationKindV128Sqrt, + operationKindV128Abs, + operationKindV128Popcnt, + operationKindV128Min, + operationKindV128Max, + operationKindV128AvgrU, + operationKindV128Pmin, + operationKindV128Pmax, + operationKindV128Ceil, + operationKindV128Floor, + operationKindV128Trunc, + operationKindV128Nearest, + operationKindV128Extend, + operationKindV128ExtMul, + operationKindV128Q15mulrSatS, + operationKindV128ExtAddPairwise, + operationKindV128FloatPromote, + operationKindV128FloatDemote, + operationKindV128FConvertFromI, + operationKindV128Dot, + operationKindV128Narrow: + return o.Kind.String() + + case operationKindV128ITruncSatFromF: + if o.B3 { + return fmt.Sprintf("%s.%sS", o.Kind, shapeName(o.B1)) + } else { + return fmt.Sprintf("%s.%sU", o.Kind, shapeName(o.B1)) + } + + case operationKindAtomicMemoryWait, + operationKindAtomicMemoryNotify, + operationKindAtomicFence, + operationKindAtomicLoad, + operationKindAtomicLoad8, + operationKindAtomicLoad16, + operationKindAtomicStore, + operationKindAtomicStore8, + operationKindAtomicStore16, + operationKindAtomicRMW, + operationKindAtomicRMW8, + operationKindAtomicRMW16, + operationKindAtomicRMWCmpxchg, + operationKindAtomicRMW8Cmpxchg, + operationKindAtomicRMW16Cmpxchg: + return o.Kind.String() + + default: + panic(fmt.Sprintf("TODO: %v", o.Kind)) + } +} + +// NewOperationUnreachable is a constructor for unionOperation with operationKindUnreachable +// +// This corresponds to wasm.OpcodeUnreachable. +// +// The engines are expected to exit the execution with wasmruntime.ErrRuntimeUnreachable error. +func newOperationUnreachable() unionOperation { + return unionOperation{Kind: operationKindUnreachable} +} + +// NewOperationLabel is a constructor for unionOperation with operationKindLabel. +// +// This is used to inform the engines of the beginning of a label. +func newOperationLabel(label label) unionOperation { + return unionOperation{Kind: operationKindLabel, U1: uint64(label)} +} + +// NewOperationBr is a constructor for unionOperation with operationKindBr. +// +// The engines are expected to branch into U1 label. +func newOperationBr(target label) unionOperation { + return unionOperation{Kind: operationKindBr, U1: uint64(target)} +} + +// NewOperationBrIf is a constructor for unionOperation with operationKindBrIf. +// +// The engines are expected to pop a value and branch into U1 label if the value equals 1. +// Otherwise, the code branches into U2 label. +func newOperationBrIf(thenTarget, elseTarget label, thenDrop inclusiveRange) unionOperation { + return unionOperation{ + Kind: operationKindBrIf, + U1: uint64(thenTarget), + U2: uint64(elseTarget), + U3: thenDrop.AsU64(), + } +} + +// NewOperationBrTable is a constructor for unionOperation with operationKindBrTable. +// +// This corresponds to wasm.OpcodeBrTableName except that the label +// here means the interpreterir level, not the ones of Wasm. +// +// The engines are expected to do the br_table operation based on the default (Us[len(Us)-1], Us[len(Us)-2]) and +// targets (Us[:len(Us)-1], Rs[:len(Us)-1]). More precisely, this pops a value from the stack (called "index") +// and decides which branch we go into next based on the value. +// +// For example, assume we have operations like {default: L_DEFAULT, targets: [L0, L1, L2]}. +// If "index" >= len(defaults), then branch into the L_DEFAULT label. +// Otherwise, we enter label of targets[index]. +func newOperationBrTable(targetLabelsAndRanges []uint64) unionOperation { + return unionOperation{ + Kind: operationKindBrTable, + Us: targetLabelsAndRanges, + } +} + +// NewOperationCall is a constructor for unionOperation with operationKindCall. +// +// This corresponds to wasm.OpcodeCallName, and engines are expected to +// enter into a function whose index equals OperationCall.FunctionIndex. +func newOperationCall(functionIndex uint32) unionOperation { + return unionOperation{Kind: operationKindCall, U1: uint64(functionIndex)} +} + +// NewOperationCallIndirect implements Operation. +// +// This corresponds to wasm.OpcodeCallIndirectName, and engines are expected to +// consume the one value from the top of stack (called "offset"), +// and make a function call against the function whose function address equals +// Tables[OperationCallIndirect.TableIndex][offset]. +// +// Note: This is called indirect function call in the sense that the target function is indirectly +// determined by the current state (top value) of the stack. +// Therefore, two checks are performed at runtime before entering the target function: +// 1) whether "offset" exceeds the length of table Tables[OperationCallIndirect.TableIndex]. +// 2) whether the type of the function table[offset] matches the function type specified by OperationCallIndirect.TypeIndex. +func newOperationCallIndirect(typeIndex, tableIndex uint32) unionOperation { + return unionOperation{Kind: operationKindCallIndirect, U1: uint64(typeIndex), U2: uint64(tableIndex)} +} + +// inclusiveRange is the range which spans across the value stack starting from the top to the bottom, and +// both boundary are included in the range. +type inclusiveRange struct { + Start, End int32 +} + +// AsU64 is be used to convert inclusiveRange to uint64 so that it can be stored in unionOperation. +func (i inclusiveRange) AsU64() uint64 { + return uint64(uint32(i.Start))<<32 | uint64(uint32(i.End)) +} + +// inclusiveRangeFromU64 retrieves inclusiveRange from the given uint64 which is stored in unionOperation. +func inclusiveRangeFromU64(v uint64) inclusiveRange { + return inclusiveRange{ + Start: int32(uint32(v >> 32)), + End: int32(uint32(v)), + } +} + +// nopinclusiveRange is inclusiveRange which corresponds to no-operation. +var nopinclusiveRange = inclusiveRange{Start: -1, End: -1} + +// NewOperationDrop is a constructor for unionOperation with operationKindDrop. +// +// The engines are expected to discard the values selected by NewOperationDrop.Depth which +// starts from the top of the stack to the bottom. +// +// depth spans across the uint64 value stack at runtime to be dropped by this operation. +func newOperationDrop(depth inclusiveRange) unionOperation { + return unionOperation{Kind: operationKindDrop, U1: depth.AsU64()} +} + +// NewOperationSelect is a constructor for unionOperation with operationKindSelect. +// +// This corresponds to wasm.OpcodeSelect. +// +// The engines are expected to pop three values, say [..., x2, x1, c], then if the value "c" equals zero, +// "x1" is pushed back onto the stack and, otherwise "x2" is pushed back. +// +// isTargetVector true if the selection target value's type is wasm.ValueTypeV128. +func newOperationSelect(isTargetVector bool) unionOperation { + return unionOperation{Kind: operationKindSelect, B3: isTargetVector} +} + +// NewOperationPick is a constructor for unionOperation with operationKindPick. +// +// The engines are expected to copy a value pointed by depth, and push the +// copied value onto the top of the stack. +// +// depth is the location of the pick target in the uint64 value stack at runtime. +// If isTargetVector=true, this points to the location of the lower 64-bits of the vector. +func newOperationPick(depth int, isTargetVector bool) unionOperation { + return unionOperation{Kind: operationKindPick, U1: uint64(depth), B3: isTargetVector} +} + +// NewOperationSet is a constructor for unionOperation with operationKindSet. +// +// The engines are expected to set the top value of the stack to the location specified by +// depth. +// +// depth is the location of the set target in the uint64 value stack at runtime. +// If isTargetVector=true, this points the location of the lower 64-bits of the vector. +func newOperationSet(depth int, isTargetVector bool) unionOperation { + return unionOperation{Kind: operationKindSet, U1: uint64(depth), B3: isTargetVector} +} + +// NewOperationGlobalGet is a constructor for unionOperation with operationKindGlobalGet. +// +// The engines are expected to read the global value specified by OperationGlobalGet.Index, +// and push the copy of the value onto the stack. +// +// See wasm.OpcodeGlobalGet. +func newOperationGlobalGet(index uint32) unionOperation { + return unionOperation{Kind: operationKindGlobalGet, U1: uint64(index)} +} + +// NewOperationGlobalSet is a constructor for unionOperation with operationKindGlobalSet. +// +// The engines are expected to consume the value from the top of the stack, +// and write the value into the global specified by OperationGlobalSet.Index. +// +// See wasm.OpcodeGlobalSet. +func newOperationGlobalSet(index uint32) unionOperation { + return unionOperation{Kind: operationKindGlobalSet, U1: uint64(index)} +} + +// memoryArg is the "memarg" to all memory instructions. +// +// See https://www.w3.org/TR/2019/REC-wasm-core-1-20191205/#memory-instructions%E2%91%A0 +type memoryArg struct { + // Alignment the expected alignment (expressed as the exponent of a power of 2). Default to the natural alignment. + // + // "Natural alignment" is defined here as the smallest power of two that can hold the size of the value type. Ex + // wasm.ValueTypeI64 is encoded in 8 little-endian bytes. 2^3 = 8, so the natural alignment is three. + Alignment uint32 + + // Offset is the address offset added to the instruction's dynamic address operand, yielding a 33-bit effective + // address that is the zero-based index at which the memory is accessed. Default to zero. + Offset uint32 +} + +// NewOperationLoad is a constructor for unionOperation with operationKindLoad. +// +// This corresponds to wasm.OpcodeI32LoadName wasm.OpcodeI64LoadName wasm.OpcodeF32LoadName and wasm.OpcodeF64LoadName. +// +// The engines are expected to check the boundary of memory length, and exit the execution if this exceeds the boundary, +// otherwise load the corresponding value following the semantics of the corresponding WebAssembly instruction. +func newOperationLoad(unsignedType unsignedType, arg memoryArg) unionOperation { + return unionOperation{Kind: operationKindLoad, B1: byte(unsignedType), U1: uint64(arg.Alignment), U2: uint64(arg.Offset)} +} + +// NewOperationLoad8 is a constructor for unionOperation with operationKindLoad8. +// +// This corresponds to wasm.OpcodeI32Load8SName wasm.OpcodeI32Load8UName wasm.OpcodeI64Load8SName wasm.OpcodeI64Load8UName. +// +// The engines are expected to check the boundary of memory length, and exit the execution if this exceeds the boundary, +// otherwise load the corresponding value following the semantics of the corresponding WebAssembly instruction. +func newOperationLoad8(signedInt signedInt, arg memoryArg) unionOperation { + return unionOperation{Kind: operationKindLoad8, B1: byte(signedInt), U1: uint64(arg.Alignment), U2: uint64(arg.Offset)} +} + +// NewOperationLoad16 is a constructor for unionOperation with operationKindLoad16. +// +// This corresponds to wasm.OpcodeI32Load16SName wasm.OpcodeI32Load16UName wasm.OpcodeI64Load16SName wasm.OpcodeI64Load16UName. +// +// The engines are expected to check the boundary of memory length, and exit the execution if this exceeds the boundary, +// otherwise load the corresponding value following the semantics of the corresponding WebAssembly instruction. +func newOperationLoad16(signedInt signedInt, arg memoryArg) unionOperation { + return unionOperation{Kind: operationKindLoad16, B1: byte(signedInt), U1: uint64(arg.Alignment), U2: uint64(arg.Offset)} +} + +// NewOperationLoad32 is a constructor for unionOperation with operationKindLoad32. +// +// This corresponds to wasm.OpcodeI64Load32SName wasm.OpcodeI64Load32UName. +// +// The engines are expected to check the boundary of memory length, and exit the execution if this exceeds the boundary, +// otherwise load the corresponding value following the semantics of the corresponding WebAssembly instruction. +func newOperationLoad32(signed bool, arg memoryArg) unionOperation { + sigB := byte(0) + if signed { + sigB = 1 + } + return unionOperation{Kind: operationKindLoad32, B1: sigB, U1: uint64(arg.Alignment), U2: uint64(arg.Offset)} +} + +// NewOperationStore is a constructor for unionOperation with operationKindStore. +// +// # This corresponds to wasm.OpcodeI32StoreName wasm.OpcodeI64StoreName wasm.OpcodeF32StoreName wasm.OpcodeF64StoreName +// +// The engines are expected to check the boundary of memory length, and exit the execution if this exceeds the boundary, +// otherwise store the corresponding value following the semantics of the corresponding WebAssembly instruction. +func newOperationStore(unsignedType unsignedType, arg memoryArg) unionOperation { + return unionOperation{Kind: operationKindStore, B1: byte(unsignedType), U1: uint64(arg.Alignment), U2: uint64(arg.Offset)} +} + +// NewOperationStore8 is a constructor for unionOperation with operationKindStore8. +// +// # This corresponds to wasm.OpcodeI32Store8Name wasm.OpcodeI64Store8Name +// +// The engines are expected to check the boundary of memory length, and exit the execution if this exceeds the boundary, +// otherwise store the corresponding value following the semantics of the corresponding WebAssembly instruction. +func newOperationStore8(arg memoryArg) unionOperation { + return unionOperation{Kind: operationKindStore8, U1: uint64(arg.Alignment), U2: uint64(arg.Offset)} +} + +// NewOperationStore16 is a constructor for unionOperation with operationKindStore16. +// +// # This corresponds to wasm.OpcodeI32Store16Name wasm.OpcodeI64Store16Name +// +// The engines are expected to check the boundary of memory length, and exit the execution if this exceeds the boundary, +// otherwise store the corresponding value following the semantics of the corresponding WebAssembly instruction. +func newOperationStore16(arg memoryArg) unionOperation { + return unionOperation{Kind: operationKindStore16, U1: uint64(arg.Alignment), U2: uint64(arg.Offset)} +} + +// NewOperationStore32 is a constructor for unionOperation with operationKindStore32. +// +// # This corresponds to wasm.OpcodeI64Store32Name +// +// The engines are expected to check the boundary of memory length, and exit the execution if this exceeds the boundary, +// otherwise store the corresponding value following the semantics of the corresponding WebAssembly instruction. +func newOperationStore32(arg memoryArg) unionOperation { + return unionOperation{Kind: operationKindStore32, U1: uint64(arg.Alignment), U2: uint64(arg.Offset)} +} + +// NewOperationMemorySize is a constructor for unionOperation with operationKindMemorySize. +// +// This corresponds to wasm.OpcodeMemorySize. +// +// The engines are expected to push the current page size of the memory onto the stack. +func newOperationMemorySize() unionOperation { + return unionOperation{Kind: operationKindMemorySize} +} + +// NewOperationMemoryGrow is a constructor for unionOperation with operationKindMemoryGrow. +// +// This corresponds to wasm.OpcodeMemoryGrow. +// +// The engines are expected to pop one value from the top of the stack, then +// execute wasm.MemoryInstance Grow with the value, and push the previous +// page size of the memory onto the stack. +func newOperationMemoryGrow() unionOperation { + return unionOperation{Kind: operationKindMemoryGrow} +} + +// NewOperationConstI32 is a constructor for unionOperation with OperationConstI32. +// +// This corresponds to wasm.OpcodeI32Const. +func newOperationConstI32(value uint32) unionOperation { + return unionOperation{Kind: operationKindConstI32, U1: uint64(value)} +} + +// NewOperationConstI64 is a constructor for unionOperation with OperationConstI64. +// +// This corresponds to wasm.OpcodeI64Const. +func newOperationConstI64(value uint64) unionOperation { + return unionOperation{Kind: operationKindConstI64, U1: value} +} + +// NewOperationConstF32 is a constructor for unionOperation with OperationConstF32. +// +// This corresponds to wasm.OpcodeF32Const. +func newOperationConstF32(value float32) unionOperation { + return unionOperation{Kind: operationKindConstF32, U1: uint64(math.Float32bits(value))} +} + +// NewOperationConstF64 is a constructor for unionOperation with OperationConstF64. +// +// This corresponds to wasm.OpcodeF64Const. +func newOperationConstF64(value float64) unionOperation { + return unionOperation{Kind: operationKindConstF64, U1: math.Float64bits(value)} +} + +// NewOperationEq is a constructor for unionOperation with operationKindEq. +// +// This corresponds to wasm.OpcodeI32EqName wasm.OpcodeI64EqName wasm.OpcodeF32EqName wasm.OpcodeF64EqName +func newOperationEq(b unsignedType) unionOperation { + return unionOperation{Kind: operationKindEq, B1: byte(b)} +} + +// NewOperationNe is a constructor for unionOperation with operationKindNe. +// +// This corresponds to wasm.OpcodeI32NeName wasm.OpcodeI64NeName wasm.OpcodeF32NeName wasm.OpcodeF64NeName +func newOperationNe(b unsignedType) unionOperation { + return unionOperation{Kind: operationKindNe, B1: byte(b)} +} + +// NewOperationEqz is a constructor for unionOperation with operationKindEqz. +// +// This corresponds to wasm.OpcodeI32EqzName wasm.OpcodeI64EqzName +func newOperationEqz(b unsignedInt) unionOperation { + return unionOperation{Kind: operationKindEqz, B1: byte(b)} +} + +// NewOperationLt is a constructor for unionOperation with operationKindLt. +// +// This corresponds to wasm.OpcodeI32LtS wasm.OpcodeI32LtU wasm.OpcodeI64LtS wasm.OpcodeI64LtU wasm.OpcodeF32Lt wasm.OpcodeF64Lt +func newOperationLt(b signedType) unionOperation { + return unionOperation{Kind: operationKindLt, B1: byte(b)} +} + +// NewOperationGt is a constructor for unionOperation with operationKindGt. +// +// This corresponds to wasm.OpcodeI32GtS wasm.OpcodeI32GtU wasm.OpcodeI64GtS wasm.OpcodeI64GtU wasm.OpcodeF32Gt wasm.OpcodeF64Gt +func newOperationGt(b signedType) unionOperation { + return unionOperation{Kind: operationKindGt, B1: byte(b)} +} + +// NewOperationLe is a constructor for unionOperation with operationKindLe. +// +// This corresponds to wasm.OpcodeI32LeS wasm.OpcodeI32LeU wasm.OpcodeI64LeS wasm.OpcodeI64LeU wasm.OpcodeF32Le wasm.OpcodeF64Le +func newOperationLe(b signedType) unionOperation { + return unionOperation{Kind: operationKindLe, B1: byte(b)} +} + +// NewOperationGe is a constructor for unionOperation with operationKindGe. +// +// This corresponds to wasm.OpcodeI32GeS wasm.OpcodeI32GeU wasm.OpcodeI64GeS wasm.OpcodeI64GeU wasm.OpcodeF32Ge wasm.OpcodeF64Ge +// NewOperationGe is the constructor for OperationGe +func newOperationGe(b signedType) unionOperation { + return unionOperation{Kind: operationKindGe, B1: byte(b)} +} + +// NewOperationAdd is a constructor for unionOperation with operationKindAdd. +// +// This corresponds to wasm.OpcodeI32AddName wasm.OpcodeI64AddName wasm.OpcodeF32AddName wasm.OpcodeF64AddName. +func newOperationAdd(b unsignedType) unionOperation { + return unionOperation{Kind: operationKindAdd, B1: byte(b)} +} + +// NewOperationSub is a constructor for unionOperation with operationKindSub. +// +// This corresponds to wasm.OpcodeI32SubName wasm.OpcodeI64SubName wasm.OpcodeF32SubName wasm.OpcodeF64SubName. +func newOperationSub(b unsignedType) unionOperation { + return unionOperation{Kind: operationKindSub, B1: byte(b)} +} + +// NewOperationMul is a constructor for unionOperation with wperationKindMul. +// +// This corresponds to wasm.OpcodeI32MulName wasm.OpcodeI64MulName wasm.OpcodeF32MulName wasm.OpcodeF64MulName. +// NewOperationMul is the constructor for OperationMul +func newOperationMul(b unsignedType) unionOperation { + return unionOperation{Kind: operationKindMul, B1: byte(b)} +} + +// NewOperationClz is a constructor for unionOperation with operationKindClz. +// +// This corresponds to wasm.OpcodeI32ClzName wasm.OpcodeI64ClzName. +// +// The engines are expected to count up the leading zeros in the +// current top of the stack, and push the count result. +// For example, stack of [..., 0x00_ff_ff_ff] results in [..., 8]. +// See wasm.OpcodeI32Clz wasm.OpcodeI64Clz +func newOperationClz(b unsignedInt) unionOperation { + return unionOperation{Kind: operationKindClz, B1: byte(b)} +} + +// NewOperationCtz is a constructor for unionOperation with operationKindCtz. +// +// This corresponds to wasm.OpcodeI32CtzName wasm.OpcodeI64CtzName. +// +// The engines are expected to count up the trailing zeros in the +// current top of the stack, and push the count result. +// For example, stack of [..., 0xff_ff_ff_00] results in [..., 8]. +func newOperationCtz(b unsignedInt) unionOperation { + return unionOperation{Kind: operationKindCtz, B1: byte(b)} +} + +// NewOperationPopcnt is a constructor for unionOperation with operationKindPopcnt. +// +// This corresponds to wasm.OpcodeI32PopcntName wasm.OpcodeI64PopcntName. +// +// The engines are expected to count up the number of set bits in the +// current top of the stack, and push the count result. +// For example, stack of [..., 0b00_00_00_11] results in [..., 2]. +func newOperationPopcnt(b unsignedInt) unionOperation { + return unionOperation{Kind: operationKindPopcnt, B1: byte(b)} +} + +// NewOperationDiv is a constructor for unionOperation with operationKindDiv. +// +// This corresponds to wasm.OpcodeI32DivS wasm.OpcodeI32DivU wasm.OpcodeI64DivS +// +// wasm.OpcodeI64DivU wasm.OpcodeF32Div wasm.OpcodeF64Div. +func newOperationDiv(b signedType) unionOperation { + return unionOperation{Kind: operationKindDiv, B1: byte(b)} +} + +// NewOperationRem is a constructor for unionOperation with operationKindRem. +// +// This corresponds to wasm.OpcodeI32RemS wasm.OpcodeI32RemU wasm.OpcodeI64RemS wasm.OpcodeI64RemU. +// +// The engines are expected to perform division on the top +// two values of integer type on the stack and puts the remainder of the result +// onto the stack. For example, stack [..., 10, 3] results in [..., 1] where +// the quotient is discarded. +// NewOperationRem is the constructor for OperationRem +func newOperationRem(b signedInt) unionOperation { + return unionOperation{Kind: operationKindRem, B1: byte(b)} +} + +// NewOperationAnd is a constructor for unionOperation with operationKindAnd. +// +// # This corresponds to wasm.OpcodeI32AndName wasm.OpcodeI64AndName +// +// The engines are expected to perform "And" operation on +// top two values on the stack, and pushes the result. +func newOperationAnd(b unsignedInt) unionOperation { + return unionOperation{Kind: operationKindAnd, B1: byte(b)} +} + +// NewOperationOr is a constructor for unionOperation with operationKindOr. +// +// # This corresponds to wasm.OpcodeI32OrName wasm.OpcodeI64OrName +// +// The engines are expected to perform "Or" operation on +// top two values on the stack, and pushes the result. +func newOperationOr(b unsignedInt) unionOperation { + return unionOperation{Kind: operationKindOr, B1: byte(b)} +} + +// NewOperationXor is a constructor for unionOperation with operationKindXor. +// +// # This corresponds to wasm.OpcodeI32XorName wasm.OpcodeI64XorName +// +// The engines are expected to perform "Xor" operation on +// top two values on the stack, and pushes the result. +func newOperationXor(b unsignedInt) unionOperation { + return unionOperation{Kind: operationKindXor, B1: byte(b)} +} + +// NewOperationShl is a constructor for unionOperation with operationKindShl. +// +// # This corresponds to wasm.OpcodeI32ShlName wasm.OpcodeI64ShlName +// +// The engines are expected to perform "Shl" operation on +// top two values on the stack, and pushes the result. +func newOperationShl(b unsignedInt) unionOperation { + return unionOperation{Kind: operationKindShl, B1: byte(b)} +} + +// NewOperationShr is a constructor for unionOperation with operationKindShr. +// +// # This corresponds to wasm.OpcodeI32ShrSName wasm.OpcodeI32ShrUName wasm.OpcodeI64ShrSName wasm.OpcodeI64ShrUName +// +// If OperationShr.Type is signed integer, then, the engines are expected to perform arithmetic right shift on the two +// top values on the stack, otherwise do the logical right shift. +func newOperationShr(b signedInt) unionOperation { + return unionOperation{Kind: operationKindShr, B1: byte(b)} +} + +// NewOperationRotl is a constructor for unionOperation with operationKindRotl. +// +// # This corresponds to wasm.OpcodeI32RotlName wasm.OpcodeI64RotlName +// +// The engines are expected to perform "Rotl" operation on +// top two values on the stack, and pushes the result. +func newOperationRotl(b unsignedInt) unionOperation { + return unionOperation{Kind: operationKindRotl, B1: byte(b)} +} + +// NewOperationRotr is a constructor for unionOperation with operationKindRotr. +// +// # This corresponds to wasm.OpcodeI32RotrName wasm.OpcodeI64RotrName +// +// The engines are expected to perform "Rotr" operation on +// top two values on the stack, and pushes the result. +func newOperationRotr(b unsignedInt) unionOperation { + return unionOperation{Kind: operationKindRotr, B1: byte(b)} +} + +// NewOperationAbs is a constructor for unionOperation with operationKindAbs. +// +// This corresponds to wasm.OpcodeF32Abs wasm.OpcodeF64Abs +func newOperationAbs(b float) unionOperation { + return unionOperation{Kind: operationKindAbs, B1: byte(b)} +} + +// NewOperationNeg is a constructor for unionOperation with operationKindNeg. +// +// This corresponds to wasm.OpcodeF32Neg wasm.OpcodeF64Neg +func newOperationNeg(b float) unionOperation { + return unionOperation{Kind: operationKindNeg, B1: byte(b)} +} + +// NewOperationCeil is a constructor for unionOperation with operationKindCeil. +// +// This corresponds to wasm.OpcodeF32CeilName wasm.OpcodeF64CeilName +func newOperationCeil(b float) unionOperation { + return unionOperation{Kind: operationKindCeil, B1: byte(b)} +} + +// NewOperationFloor is a constructor for unionOperation with operationKindFloor. +// +// This corresponds to wasm.OpcodeF32FloorName wasm.OpcodeF64FloorName +func newOperationFloor(b float) unionOperation { + return unionOperation{Kind: operationKindFloor, B1: byte(b)} +} + +// NewOperationTrunc is a constructor for unionOperation with operationKindTrunc. +// +// This corresponds to wasm.OpcodeF32TruncName wasm.OpcodeF64TruncName +func newOperationTrunc(b float) unionOperation { + return unionOperation{Kind: operationKindTrunc, B1: byte(b)} +} + +// NewOperationNearest is a constructor for unionOperation with operationKindNearest. +// +// # This corresponds to wasm.OpcodeF32NearestName wasm.OpcodeF64NearestName +// +// Note: this is *not* equivalent to math.Round and instead has the same +// the semantics of LLVM's rint intrinsic. See https://llvm.org/docs/LangRef.html#llvm-rint-intrinsic. +// For example, math.Round(-4.5) produces -5 while we want to produce -4. +func newOperationNearest(b float) unionOperation { + return unionOperation{Kind: operationKindNearest, B1: byte(b)} +} + +// NewOperationSqrt is a constructor for unionOperation with operationKindSqrt. +// +// This corresponds to wasm.OpcodeF32SqrtName wasm.OpcodeF64SqrtName +func newOperationSqrt(b float) unionOperation { + return unionOperation{Kind: operationKindSqrt, B1: byte(b)} +} + +// NewOperationMin is a constructor for unionOperation with operationKindMin. +// +// # This corresponds to wasm.OpcodeF32MinName wasm.OpcodeF64MinName +// +// The engines are expected to pop two values from the stack, and push back the maximum of +// these two values onto the stack. For example, stack [..., 100.1, 1.9] results in [..., 1.9]. +// +// Note: WebAssembly specifies that min/max must always return NaN if one of values is NaN, +// which is a different behavior different from math.Min. +func newOperationMin(b float) unionOperation { + return unionOperation{Kind: operationKindMin, B1: byte(b)} +} + +// NewOperationMax is a constructor for unionOperation with operationKindMax. +// +// # This corresponds to wasm.OpcodeF32MaxName wasm.OpcodeF64MaxName +// +// The engines are expected to pop two values from the stack, and push back the maximum of +// these two values onto the stack. For example, stack [..., 100.1, 1.9] results in [..., 100.1]. +// +// Note: WebAssembly specifies that min/max must always return NaN if one of values is NaN, +// which is a different behavior different from math.Max. +func newOperationMax(b float) unionOperation { + return unionOperation{Kind: operationKindMax, B1: byte(b)} +} + +// NewOperationCopysign is a constructor for unionOperation with operationKindCopysign. +// +// # This corresponds to wasm.OpcodeF32CopysignName wasm.OpcodeF64CopysignName +// +// The engines are expected to pop two float values from the stack, and copy the signbit of +// the first-popped value to the last one. +// For example, stack [..., 1.213, -5.0] results in [..., -1.213]. +func newOperationCopysign(b float) unionOperation { + return unionOperation{Kind: operationKindCopysign, B1: byte(b)} +} + +// NewOperationI32WrapFromI64 is a constructor for unionOperation with operationKindI32WrapFromI64. +// +// This corresponds to wasm.OpcodeI32WrapI64 and equivalent to uint64(uint32(v)) in Go. +// +// The engines are expected to replace the 64-bit int on top of the stack +// with the corresponding 32-bit integer. +func newOperationI32WrapFromI64() unionOperation { + return unionOperation{Kind: operationKindI32WrapFromI64} +} + +// NewOperationITruncFromF is a constructor for unionOperation with operationKindITruncFromF. +// +// This corresponds to +// +// wasm.OpcodeI32TruncF32SName wasm.OpcodeI32TruncF32UName wasm.OpcodeI32TruncF64SName +// wasm.OpcodeI32TruncF64UName wasm.OpcodeI64TruncF32SName wasm.OpcodeI64TruncF32UName wasm.OpcodeI64TruncF64SName +// wasm.OpcodeI64TruncF64UName. wasm.OpcodeI32TruncSatF32SName wasm.OpcodeI32TruncSatF32UName +// wasm.OpcodeI32TruncSatF64SName wasm.OpcodeI32TruncSatF64UName wasm.OpcodeI64TruncSatF32SName +// wasm.OpcodeI64TruncSatF32UName wasm.OpcodeI64TruncSatF64SName wasm.OpcodeI64TruncSatF64UName +// +// See [1] and [2] for when we encounter undefined behavior in the WebAssembly specification if NewOperationITruncFromF.NonTrapping == false. +// To summarize, if the source float value is NaN or doesn't fit in the destination range of integers (incl. +=Inf), +// then the runtime behavior is undefined. In wazero, the engines are expected to exit the execution in these undefined cases with +// wasmruntime.ErrRuntimeInvalidConversionToInteger error. +// +// [1] https://www.w3.org/TR/2019/REC-wasm-core-1-20191205/#-hrefop-trunc-umathrmtruncmathsfu_m-n-z for unsigned integers. +// [2] https://www.w3.org/TR/2019/REC-wasm-core-1-20191205/#-hrefop-trunc-smathrmtruncmathsfs_m-n-z for signed integers. +// +// nonTrapping true if this conversion is "nontrapping" in the sense of the +// https://github.com/WebAssembly/spec/blob/ce4b6c4d47eb06098cc7ab2e81f24748da822f20/proposals/nontrapping-float-to-int-conversion/Overview.md +func newOperationITruncFromF(inputType float, outputType signedInt, nonTrapping bool) unionOperation { + return unionOperation{ + Kind: operationKindITruncFromF, + B1: byte(inputType), + B2: byte(outputType), + B3: nonTrapping, + } +} + +// NewOperationFConvertFromI is a constructor for unionOperation with operationKindFConvertFromI. +// +// This corresponds to +// +// wasm.OpcodeF32ConvertI32SName wasm.OpcodeF32ConvertI32UName wasm.OpcodeF32ConvertI64SName wasm.OpcodeF32ConvertI64UName +// wasm.OpcodeF64ConvertI32SName wasm.OpcodeF64ConvertI32UName wasm.OpcodeF64ConvertI64SName wasm.OpcodeF64ConvertI64UName +// +// and equivalent to float32(uint32(x)), float32(int32(x)), etc in Go. +func newOperationFConvertFromI(inputType signedInt, outputType float) unionOperation { + return unionOperation{ + Kind: operationKindFConvertFromI, + B1: byte(inputType), + B2: byte(outputType), + } +} + +// NewOperationF32DemoteFromF64 is a constructor for unionOperation with operationKindF32DemoteFromF64. +// +// This corresponds to wasm.OpcodeF32DemoteF64 and is equivalent float32(float64(v)). +func newOperationF32DemoteFromF64() unionOperation { + return unionOperation{Kind: operationKindF32DemoteFromF64} +} + +// NewOperationF64PromoteFromF32 is a constructor for unionOperation with operationKindF64PromoteFromF32. +// +// This corresponds to wasm.OpcodeF64PromoteF32 and is equivalent float64(float32(v)). +func newOperationF64PromoteFromF32() unionOperation { + return unionOperation{Kind: operationKindF64PromoteFromF32} +} + +// NewOperationI32ReinterpretFromF32 is a constructor for unionOperation with operationKindI32ReinterpretFromF32. +// +// This corresponds to wasm.OpcodeI32ReinterpretF32Name. +func newOperationI32ReinterpretFromF32() unionOperation { + return unionOperation{Kind: operationKindI32ReinterpretFromF32} +} + +// NewOperationI64ReinterpretFromF64 is a constructor for unionOperation with operationKindI64ReinterpretFromF64. +// +// This corresponds to wasm.OpcodeI64ReinterpretF64Name. +func newOperationI64ReinterpretFromF64() unionOperation { + return unionOperation{Kind: operationKindI64ReinterpretFromF64} +} + +// NewOperationF32ReinterpretFromI32 is a constructor for unionOperation with operationKindF32ReinterpretFromI32. +// +// This corresponds to wasm.OpcodeF32ReinterpretI32Name. +func newOperationF32ReinterpretFromI32() unionOperation { + return unionOperation{Kind: operationKindF32ReinterpretFromI32} +} + +// NewOperationF64ReinterpretFromI64 is a constructor for unionOperation with operationKindF64ReinterpretFromI64. +// +// This corresponds to wasm.OpcodeF64ReinterpretI64Name. +func newOperationF64ReinterpretFromI64() unionOperation { + return unionOperation{Kind: operationKindF64ReinterpretFromI64} +} + +// NewOperationExtend is a constructor for unionOperation with operationKindExtend. +// +// # This corresponds to wasm.OpcodeI64ExtendI32SName wasm.OpcodeI64ExtendI32UName +// +// The engines are expected to extend the 32-bit signed or unsigned int on top of the stack +// as a 64-bit integer of corresponding signedness. For unsigned case, this is just reinterpreting the +// underlying bit pattern as 64-bit integer. For signed case, this is sign-extension which preserves the +// original integer's sign. +func newOperationExtend(signed bool) unionOperation { + op := unionOperation{Kind: operationKindExtend} + if signed { + op.B1 = 1 + } + return op +} + +// NewOperationSignExtend32From8 is a constructor for unionOperation with operationKindSignExtend32From8. +// +// This corresponds to wasm.OpcodeI32Extend8SName. +// +// The engines are expected to sign-extend the first 8-bits of 32-bit in as signed 32-bit int. +func newOperationSignExtend32From8() unionOperation { + return unionOperation{Kind: operationKindSignExtend32From8} +} + +// NewOperationSignExtend32From16 is a constructor for unionOperation with operationKindSignExtend32From16. +// +// This corresponds to wasm.OpcodeI32Extend16SName. +// +// The engines are expected to sign-extend the first 16-bits of 32-bit in as signed 32-bit int. +func newOperationSignExtend32From16() unionOperation { + return unionOperation{Kind: operationKindSignExtend32From16} +} + +// NewOperationSignExtend64From8 is a constructor for unionOperation with operationKindSignExtend64From8. +// +// This corresponds to wasm.OpcodeI64Extend8SName. +// +// The engines are expected to sign-extend the first 8-bits of 64-bit in as signed 32-bit int. +func newOperationSignExtend64From8() unionOperation { + return unionOperation{Kind: operationKindSignExtend64From8} +} + +// NewOperationSignExtend64From16 is a constructor for unionOperation with operationKindSignExtend64From16. +// +// This corresponds to wasm.OpcodeI64Extend16SName. +// +// The engines are expected to sign-extend the first 16-bits of 64-bit in as signed 32-bit int. +func newOperationSignExtend64From16() unionOperation { + return unionOperation{Kind: operationKindSignExtend64From16} +} + +// NewOperationSignExtend64From32 is a constructor for unionOperation with operationKindSignExtend64From32. +// +// This corresponds to wasm.OpcodeI64Extend32SName. +// +// The engines are expected to sign-extend the first 32-bits of 64-bit in as signed 32-bit int. +func newOperationSignExtend64From32() unionOperation { + return unionOperation{Kind: operationKindSignExtend64From32} +} + +// NewOperationMemoryInit is a constructor for unionOperation with operationKindMemoryInit. +// +// This corresponds to wasm.OpcodeMemoryInitName. +// +// dataIndex is the index of the data instance in ModuleInstance.DataInstances +// by which this operation instantiates a part of the memory. +func newOperationMemoryInit(dataIndex uint32) unionOperation { + return unionOperation{Kind: operationKindMemoryInit, U1: uint64(dataIndex)} +} + +// NewOperationDataDrop implements Operation. +// +// This corresponds to wasm.OpcodeDataDropName. +// +// dataIndex is the index of the data instance in ModuleInstance.DataInstances +// which this operation drops. +func newOperationDataDrop(dataIndex uint32) unionOperation { + return unionOperation{Kind: operationKindDataDrop, U1: uint64(dataIndex)} +} + +// NewOperationMemoryCopy is a consuctor for unionOperation with operationKindMemoryCopy. +// +// This corresponds to wasm.OpcodeMemoryCopyName. +func newOperationMemoryCopy() unionOperation { + return unionOperation{Kind: operationKindMemoryCopy} +} + +// NewOperationMemoryFill is a consuctor for unionOperation with operationKindMemoryFill. +func newOperationMemoryFill() unionOperation { + return unionOperation{Kind: operationKindMemoryFill} +} + +// NewOperationTableInit is a constructor for unionOperation with operationKindTableInit. +// +// This corresponds to wasm.OpcodeTableInitName. +// +// elemIndex is the index of the element by which this operation initializes a part of the table. +// tableIndex is the index of the table on which this operation initialize by the target element. +func newOperationTableInit(elemIndex, tableIndex uint32) unionOperation { + return unionOperation{Kind: operationKindTableInit, U1: uint64(elemIndex), U2: uint64(tableIndex)} +} + +// NewOperationElemDrop is a constructor for unionOperation with operationKindElemDrop. +// +// This corresponds to wasm.OpcodeElemDropName. +// +// elemIndex is the index of the element which this operation drops. +func newOperationElemDrop(elemIndex uint32) unionOperation { + return unionOperation{Kind: operationKindElemDrop, U1: uint64(elemIndex)} +} + +// NewOperationTableCopy implements Operation. +// +// This corresponds to wasm.OpcodeTableCopyName. +func newOperationTableCopy(srcTableIndex, dstTableIndex uint32) unionOperation { + return unionOperation{Kind: operationKindTableCopy, U1: uint64(srcTableIndex), U2: uint64(dstTableIndex)} +} + +// NewOperationRefFunc constructor for unionOperation with operationKindRefFunc. +// +// This corresponds to wasm.OpcodeRefFuncName, and engines are expected to +// push the opaque pointer value of engine specific func for the given FunctionIndex. +// +// Note: in wazero, we express any reference types (funcref or externref) as opaque pointers which is uint64. +// Therefore, the engine implementations emit instructions to push the address of *function onto the stack. +func newOperationRefFunc(functionIndex uint32) unionOperation { + return unionOperation{Kind: operationKindRefFunc, U1: uint64(functionIndex)} +} + +// NewOperationTableGet constructor for unionOperation with operationKindTableGet. +// +// This corresponds to wasm.OpcodeTableGetName. +func newOperationTableGet(tableIndex uint32) unionOperation { + return unionOperation{Kind: operationKindTableGet, U1: uint64(tableIndex)} +} + +// NewOperationTableSet constructor for unionOperation with operationKindTableSet. +// +// This corresponds to wasm.OpcodeTableSetName. +func newOperationTableSet(tableIndex uint32) unionOperation { + return unionOperation{Kind: operationKindTableSet, U1: uint64(tableIndex)} +} + +// NewOperationTableSize constructor for unionOperation with operationKindTableSize. +// +// This corresponds to wasm.OpcodeTableSizeName. +func newOperationTableSize(tableIndex uint32) unionOperation { + return unionOperation{Kind: operationKindTableSize, U1: uint64(tableIndex)} +} + +// NewOperationTableGrow constructor for unionOperation with operationKindTableGrow. +// +// This corresponds to wasm.OpcodeTableGrowName. +func newOperationTableGrow(tableIndex uint32) unionOperation { + return unionOperation{Kind: operationKindTableGrow, U1: uint64(tableIndex)} +} + +// NewOperationTableFill constructor for unionOperation with operationKindTableFill. +// +// This corresponds to wasm.OpcodeTableFillName. +func newOperationTableFill(tableIndex uint32) unionOperation { + return unionOperation{Kind: operationKindTableFill, U1: uint64(tableIndex)} +} + +// NewOperationV128Const constructor for unionOperation with operationKindV128Const +func newOperationV128Const(lo, hi uint64) unionOperation { + return unionOperation{Kind: operationKindV128Const, U1: lo, U2: hi} +} + +// shape corresponds to a shape of v128 values. +// https://webassembly.github.io/spec/core/syntax/instructions.html#syntax-shape +type shape = byte + +const ( + shapeI8x16 shape = iota + shapeI16x8 + shapeI32x4 + shapeI64x2 + shapeF32x4 + shapeF64x2 +) + +func shapeName(s shape) (ret string) { + switch s { + case shapeI8x16: + ret = "I8x16" + case shapeI16x8: + ret = "I16x8" + case shapeI32x4: + ret = "I32x4" + case shapeI64x2: + ret = "I64x2" + case shapeF32x4: + ret = "F32x4" + case shapeF64x2: + ret = "F64x2" + } + return +} + +// NewOperationV128Add constructor for unionOperation with operationKindV128Add. +// +// This corresponds to wasm.OpcodeVecI8x16AddName wasm.OpcodeVecI16x8AddName wasm.OpcodeVecI32x4AddName +// +// wasm.OpcodeVecI64x2AddName wasm.OpcodeVecF32x4AddName wasm.OpcodeVecF64x2AddName +func newOperationV128Add(shape shape) unionOperation { + return unionOperation{Kind: operationKindV128Add, B1: shape} +} + +// NewOperationV128Sub constructor for unionOperation with operationKindV128Sub. +// +// This corresponds to wasm.OpcodeVecI8x16SubName wasm.OpcodeVecI16x8SubName wasm.OpcodeVecI32x4SubName +// +// wasm.OpcodeVecI64x2SubName wasm.OpcodeVecF32x4SubName wasm.OpcodeVecF64x2SubName +func newOperationV128Sub(shape shape) unionOperation { + return unionOperation{Kind: operationKindV128Sub, B1: shape} +} + +// v128LoadType represents a type of wasm.OpcodeVecV128Load* instructions. +type v128LoadType = byte + +const ( + // v128LoadType128 corresponds to wasm.OpcodeVecV128LoadName. + v128LoadType128 v128LoadType = iota + // v128LoadType8x8s corresponds to wasm.OpcodeVecV128Load8x8SName. + v128LoadType8x8s + // v128LoadType8x8u corresponds to wasm.OpcodeVecV128Load8x8UName. + v128LoadType8x8u + // v128LoadType16x4s corresponds to wasm.OpcodeVecV128Load16x4SName + v128LoadType16x4s + // v128LoadType16x4u corresponds to wasm.OpcodeVecV128Load16x4UName + v128LoadType16x4u + // v128LoadType32x2s corresponds to wasm.OpcodeVecV128Load32x2SName + v128LoadType32x2s + // v128LoadType32x2u corresponds to wasm.OpcodeVecV128Load32x2UName + v128LoadType32x2u + // v128LoadType8Splat corresponds to wasm.OpcodeVecV128Load8SplatName + v128LoadType8Splat + // v128LoadType16Splat corresponds to wasm.OpcodeVecV128Load16SplatName + v128LoadType16Splat + // v128LoadType32Splat corresponds to wasm.OpcodeVecV128Load32SplatName + v128LoadType32Splat + // v128LoadType64Splat corresponds to wasm.OpcodeVecV128Load64SplatName + v128LoadType64Splat + // v128LoadType32zero corresponds to wasm.OpcodeVecV128Load32zeroName + v128LoadType32zero + // v128LoadType64zero corresponds to wasm.OpcodeVecV128Load64zeroName + v128LoadType64zero +) + +// NewOperationV128Load is a constructor for unionOperation with operationKindV128Load. +// +// This corresponds to +// +// wasm.OpcodeVecV128LoadName wasm.OpcodeVecV128Load8x8SName wasm.OpcodeVecV128Load8x8UName +// wasm.OpcodeVecV128Load16x4SName wasm.OpcodeVecV128Load16x4UName wasm.OpcodeVecV128Load32x2SName +// wasm.OpcodeVecV128Load32x2UName wasm.OpcodeVecV128Load8SplatName wasm.OpcodeVecV128Load16SplatName +// wasm.OpcodeVecV128Load32SplatName wasm.OpcodeVecV128Load64SplatName wasm.OpcodeVecV128Load32zeroName +// wasm.OpcodeVecV128Load64zeroName +func newOperationV128Load(loadType v128LoadType, arg memoryArg) unionOperation { + return unionOperation{Kind: operationKindV128Load, B1: loadType, U1: uint64(arg.Alignment), U2: uint64(arg.Offset)} +} + +// NewOperationV128LoadLane is a constructor for unionOperation with operationKindV128LoadLane. +// +// This corresponds to wasm.OpcodeVecV128Load8LaneName wasm.OpcodeVecV128Load16LaneName +// +// wasm.OpcodeVecV128Load32LaneName wasm.OpcodeVecV128Load64LaneName. +// +// laneIndex is >=0 && <(128/LaneSize). +// laneSize is either 8, 16, 32, or 64. +func newOperationV128LoadLane(laneIndex, laneSize byte, arg memoryArg) unionOperation { + return unionOperation{Kind: operationKindV128LoadLane, B1: laneSize, B2: laneIndex, U1: uint64(arg.Alignment), U2: uint64(arg.Offset)} +} + +// NewOperationV128Store is a constructor for unionOperation with operationKindV128Store. +// +// This corresponds to wasm.OpcodeVecV128Load8LaneName wasm.OpcodeVecV128Load16LaneName +// +// wasm.OpcodeVecV128Load32LaneName wasm.OpcodeVecV128Load64LaneName. +func newOperationV128Store(arg memoryArg) unionOperation { + return unionOperation{ + Kind: operationKindV128Store, + U1: uint64(arg.Alignment), + U2: uint64(arg.Offset), + } +} + +// NewOperationV128StoreLane implements Operation. +// +// This corresponds to wasm.OpcodeVecV128Load8LaneName wasm.OpcodeVecV128Load16LaneName +// +// wasm.OpcodeVecV128Load32LaneName wasm.OpcodeVecV128Load64LaneName. +// +// laneIndex is >=0 && <(128/LaneSize). +// laneSize is either 8, 16, 32, or 64. +func newOperationV128StoreLane(laneIndex byte, laneSize byte, arg memoryArg) unionOperation { + return unionOperation{ + Kind: operationKindV128StoreLane, + B1: laneSize, + B2: laneIndex, + U1: uint64(arg.Alignment), + U2: uint64(arg.Offset), + } +} + +// NewOperationV128ExtractLane is a constructor for unionOperation with operationKindV128ExtractLane. +// +// This corresponds to +// +// wasm.OpcodeVecI8x16ExtractLaneSName wasm.OpcodeVecI8x16ExtractLaneUName +// wasm.OpcodeVecI16x8ExtractLaneSName wasm.OpcodeVecI16x8ExtractLaneUName +// wasm.OpcodeVecI32x4ExtractLaneName wasm.OpcodeVecI64x2ExtractLaneName +// wasm.OpcodeVecF32x4ExtractLaneName wasm.OpcodeVecF64x2ExtractLaneName. +// +// laneIndex is >=0 && =0 && = l { + return nil, fmt.Errorf("invalid local index for local.get %d >= %d", index, l) + } + var t wasm.ValueType + if index < inputLen { + t = c.sig.Params[index] + } else { + t = c.localTypes[index-inputLen] + } + return wasmValueTypeToUnsignedOutSignature(t), nil + case wasm.OpcodeLocalSet: + inputLen := uint32(len(c.sig.Params)) + if l := uint32(len(c.localTypes)) + inputLen; index >= l { + return nil, fmt.Errorf("invalid local index for local.get %d >= %d", index, l) + } + var t wasm.ValueType + if index < inputLen { + t = c.sig.Params[index] + } else { + t = c.localTypes[index-inputLen] + } + return wasmValueTypeToUnsignedInSignature(t), nil + case wasm.OpcodeLocalTee: + inputLen := uint32(len(c.sig.Params)) + if l := uint32(len(c.localTypes)) + inputLen; index >= l { + return nil, fmt.Errorf("invalid local index for local.get %d >= %d", index, l) + } + var t wasm.ValueType + if index < inputLen { + t = c.sig.Params[index] + } else { + t = c.localTypes[index-inputLen] + } + return wasmValueTypeToUnsignedInOutSignature(t), nil + case wasm.OpcodeGlobalGet: + if len(c.globals) <= int(index) { + return nil, fmt.Errorf("invalid global index for global.get %d >= %d", index, len(c.globals)) + } + return wasmValueTypeToUnsignedOutSignature(c.globals[index].ValType), nil + case wasm.OpcodeGlobalSet: + if len(c.globals) <= int(index) { + return nil, fmt.Errorf("invalid global index for global.get %d >= %d", index, len(c.globals)) + } + return wasmValueTypeToUnsignedInSignature(c.globals[index].ValType), nil + case wasm.OpcodeI32Load: + return signature_I32_I32, nil + case wasm.OpcodeI64Load: + return signature_I32_I64, nil + case wasm.OpcodeF32Load: + return signature_I32_F32, nil + case wasm.OpcodeF64Load: + return signature_I32_F64, nil + case wasm.OpcodeI32Load8S, wasm.OpcodeI32Load8U, wasm.OpcodeI32Load16S, wasm.OpcodeI32Load16U: + return signature_I32_I32, nil + case wasm.OpcodeI64Load8S, wasm.OpcodeI64Load8U, wasm.OpcodeI64Load16S, wasm.OpcodeI64Load16U, + wasm.OpcodeI64Load32S, wasm.OpcodeI64Load32U: + return signature_I32_I64, nil + case wasm.OpcodeI32Store: + return signature_I32I32_None, nil + case wasm.OpcodeI64Store: + return signature_I32I64_None, nil + case wasm.OpcodeF32Store: + return signature_I32F32_None, nil + case wasm.OpcodeF64Store: + return signature_I32F64_None, nil + case wasm.OpcodeI32Store8: + return signature_I32I32_None, nil + case wasm.OpcodeI32Store16: + return signature_I32I32_None, nil + case wasm.OpcodeI64Store8: + return signature_I32I64_None, nil + case wasm.OpcodeI64Store16: + return signature_I32I64_None, nil + case wasm.OpcodeI64Store32: + return signature_I32I64_None, nil + case wasm.OpcodeMemorySize: + return signature_None_I32, nil + case wasm.OpcodeMemoryGrow: + return signature_I32_I32, nil + case wasm.OpcodeI32Const: + return signature_None_I32, nil + case wasm.OpcodeI64Const: + return signature_None_I64, nil + case wasm.OpcodeF32Const: + return signature_None_F32, nil + case wasm.OpcodeF64Const: + return signature_None_F64, nil + case wasm.OpcodeI32Eqz: + return signature_I32_I32, nil + case wasm.OpcodeI32Eq, wasm.OpcodeI32Ne, wasm.OpcodeI32LtS, + wasm.OpcodeI32LtU, wasm.OpcodeI32GtS, wasm.OpcodeI32GtU, + wasm.OpcodeI32LeS, wasm.OpcodeI32LeU, wasm.OpcodeI32GeS, + wasm.OpcodeI32GeU: + return signature_I32I32_I32, nil + case wasm.OpcodeI64Eqz: + return signature_I64_I32, nil + case wasm.OpcodeI64Eq, wasm.OpcodeI64Ne, wasm.OpcodeI64LtS, + wasm.OpcodeI64LtU, wasm.OpcodeI64GtS, wasm.OpcodeI64GtU, + wasm.OpcodeI64LeS, wasm.OpcodeI64LeU, wasm.OpcodeI64GeS, + wasm.OpcodeI64GeU: + return signature_I64I64_I32, nil + case wasm.OpcodeF32Eq, wasm.OpcodeF32Ne, wasm.OpcodeF32Lt, + wasm.OpcodeF32Gt, wasm.OpcodeF32Le, wasm.OpcodeF32Ge: + return signature_F32F32_I32, nil + case wasm.OpcodeF64Eq, wasm.OpcodeF64Ne, wasm.OpcodeF64Lt, + wasm.OpcodeF64Gt, wasm.OpcodeF64Le, wasm.OpcodeF64Ge: + return signature_F64F64_I32, nil + case wasm.OpcodeI32Clz, wasm.OpcodeI32Ctz, wasm.OpcodeI32Popcnt: + return signature_I32_I32, nil + case wasm.OpcodeI32Add, wasm.OpcodeI32Sub, wasm.OpcodeI32Mul, + wasm.OpcodeI32DivS, wasm.OpcodeI32DivU, wasm.OpcodeI32RemS, + wasm.OpcodeI32RemU, wasm.OpcodeI32And, wasm.OpcodeI32Or, + wasm.OpcodeI32Xor, wasm.OpcodeI32Shl, wasm.OpcodeI32ShrS, + wasm.OpcodeI32ShrU, wasm.OpcodeI32Rotl, wasm.OpcodeI32Rotr: + return signature_I32I32_I32, nil + case wasm.OpcodeI64Clz, wasm.OpcodeI64Ctz, wasm.OpcodeI64Popcnt: + return signature_I64_I64, nil + case wasm.OpcodeI64Add, wasm.OpcodeI64Sub, wasm.OpcodeI64Mul, + wasm.OpcodeI64DivS, wasm.OpcodeI64DivU, wasm.OpcodeI64RemS, + wasm.OpcodeI64RemU, wasm.OpcodeI64And, wasm.OpcodeI64Or, + wasm.OpcodeI64Xor, wasm.OpcodeI64Shl, wasm.OpcodeI64ShrS, + wasm.OpcodeI64ShrU, wasm.OpcodeI64Rotl, wasm.OpcodeI64Rotr: + return signature_I64I64_I64, nil + case wasm.OpcodeF32Abs, wasm.OpcodeF32Neg, wasm.OpcodeF32Ceil, + wasm.OpcodeF32Floor, wasm.OpcodeF32Trunc, wasm.OpcodeF32Nearest, + wasm.OpcodeF32Sqrt: + return signature_F32_F32, nil + case wasm.OpcodeF32Add, wasm.OpcodeF32Sub, wasm.OpcodeF32Mul, + wasm.OpcodeF32Div, wasm.OpcodeF32Min, wasm.OpcodeF32Max, + wasm.OpcodeF32Copysign: + return signature_F32F32_F32, nil + case wasm.OpcodeF64Abs, wasm.OpcodeF64Neg, wasm.OpcodeF64Ceil, + wasm.OpcodeF64Floor, wasm.OpcodeF64Trunc, wasm.OpcodeF64Nearest, + wasm.OpcodeF64Sqrt: + return signature_F64_F64, nil + case wasm.OpcodeF64Add, wasm.OpcodeF64Sub, wasm.OpcodeF64Mul, + wasm.OpcodeF64Div, wasm.OpcodeF64Min, wasm.OpcodeF64Max, + wasm.OpcodeF64Copysign: + return signature_F64F64_F64, nil + case wasm.OpcodeI32WrapI64: + return signature_I64_I32, nil + case wasm.OpcodeI32TruncF32S, wasm.OpcodeI32TruncF32U: + return signature_F32_I32, nil + case wasm.OpcodeI32TruncF64S, wasm.OpcodeI32TruncF64U: + return signature_F64_I32, nil + case wasm.OpcodeI64ExtendI32S, wasm.OpcodeI64ExtendI32U: + return signature_I32_I64, nil + case wasm.OpcodeI64TruncF32S, wasm.OpcodeI64TruncF32U: + return signature_F32_I64, nil + case wasm.OpcodeI64TruncF64S, wasm.OpcodeI64TruncF64U: + return signature_F64_I64, nil + case wasm.OpcodeF32ConvertI32S, wasm.OpcodeF32ConvertI32U: + return signature_I32_F32, nil + case wasm.OpcodeF32ConvertI64S, wasm.OpcodeF32ConvertI64U: + return signature_I64_F32, nil + case wasm.OpcodeF32DemoteF64: + return signature_F64_F32, nil + case wasm.OpcodeF64ConvertI32S, wasm.OpcodeF64ConvertI32U: + return signature_I32_F64, nil + case wasm.OpcodeF64ConvertI64S, wasm.OpcodeF64ConvertI64U: + return signature_I64_F64, nil + case wasm.OpcodeF64PromoteF32: + return signature_F32_F64, nil + case wasm.OpcodeI32ReinterpretF32: + return signature_F32_I32, nil + case wasm.OpcodeI64ReinterpretF64: + return signature_F64_I64, nil + case wasm.OpcodeF32ReinterpretI32: + return signature_I32_F32, nil + case wasm.OpcodeF64ReinterpretI64: + return signature_I64_F64, nil + case wasm.OpcodeI32Extend8S, wasm.OpcodeI32Extend16S: + return signature_I32_I32, nil + case wasm.OpcodeI64Extend8S, wasm.OpcodeI64Extend16S, wasm.OpcodeI64Extend32S: + return signature_I64_I64, nil + case wasm.OpcodeTableGet: + // table.get takes table's offset and pushes the ref type value of opaque pointer as i64 value onto the stack. + return signature_I32_I64, nil + case wasm.OpcodeTableSet: + // table.set takes table's offset and the ref type value of opaque pointer as i64 value. + return signature_I32I64_None, nil + case wasm.OpcodeRefFunc: + // ref.func is translated as pushing the compiled function's opaque pointer (uint64) at interpreterir layer. + return signature_None_I64, nil + case wasm.OpcodeRefIsNull: + // ref.is_null is translated as checking if the uint64 on the top of the stack (opaque pointer) is zero or not. + return signature_I64_I32, nil + case wasm.OpcodeRefNull: + // ref.null is translated as i64.const 0. + return signature_None_I64, nil + case wasm.OpcodeMiscPrefix: + switch miscOp := c.body[c.pc+1]; miscOp { + case wasm.OpcodeMiscI32TruncSatF32S, wasm.OpcodeMiscI32TruncSatF32U: + return signature_F32_I32, nil + case wasm.OpcodeMiscI32TruncSatF64S, wasm.OpcodeMiscI32TruncSatF64U: + return signature_F64_I32, nil + case wasm.OpcodeMiscI64TruncSatF32S, wasm.OpcodeMiscI64TruncSatF32U: + return signature_F32_I64, nil + case wasm.OpcodeMiscI64TruncSatF64S, wasm.OpcodeMiscI64TruncSatF64U: + return signature_F64_I64, nil + case wasm.OpcodeMiscMemoryInit, wasm.OpcodeMiscMemoryCopy, wasm.OpcodeMiscMemoryFill, + wasm.OpcodeMiscTableInit, wasm.OpcodeMiscTableCopy: + return signature_I32I32I32_None, nil + case wasm.OpcodeMiscDataDrop, wasm.OpcodeMiscElemDrop: + return signature_None_None, nil + case wasm.OpcodeMiscTableGrow: + return signature_I64I32_I32, nil + case wasm.OpcodeMiscTableSize: + return signature_None_I32, nil + case wasm.OpcodeMiscTableFill: + return signature_I32I64I32_None, nil + default: + return nil, fmt.Errorf("unsupported misc instruction in interpreterir: 0x%x", op) + } + case wasm.OpcodeVecPrefix: + switch vecOp := c.body[c.pc+1]; vecOp { + case wasm.OpcodeVecV128Const: + return signature_None_V128, nil + case wasm.OpcodeVecV128Load, wasm.OpcodeVecV128Load8x8s, wasm.OpcodeVecV128Load8x8u, + wasm.OpcodeVecV128Load16x4s, wasm.OpcodeVecV128Load16x4u, wasm.OpcodeVecV128Load32x2s, + wasm.OpcodeVecV128Load32x2u, wasm.OpcodeVecV128Load8Splat, wasm.OpcodeVecV128Load16Splat, + wasm.OpcodeVecV128Load32Splat, wasm.OpcodeVecV128Load64Splat, wasm.OpcodeVecV128Load32zero, + wasm.OpcodeVecV128Load64zero: + return signature_I32_V128, nil + case wasm.OpcodeVecV128Load8Lane, wasm.OpcodeVecV128Load16Lane, + wasm.OpcodeVecV128Load32Lane, wasm.OpcodeVecV128Load64Lane: + return signature_I32V128_V128, nil + case wasm.OpcodeVecV128Store, + wasm.OpcodeVecV128Store8Lane, + wasm.OpcodeVecV128Store16Lane, + wasm.OpcodeVecV128Store32Lane, + wasm.OpcodeVecV128Store64Lane: + return signature_I32V128_None, nil + case wasm.OpcodeVecI8x16ExtractLaneS, + wasm.OpcodeVecI8x16ExtractLaneU, + wasm.OpcodeVecI16x8ExtractLaneS, + wasm.OpcodeVecI16x8ExtractLaneU, + wasm.OpcodeVecI32x4ExtractLane: + return signature_V128_I32, nil + case wasm.OpcodeVecI64x2ExtractLane: + return signature_V128_I64, nil + case wasm.OpcodeVecF32x4ExtractLane: + return signature_V128_F32, nil + case wasm.OpcodeVecF64x2ExtractLane: + return signature_V128_F64, nil + case wasm.OpcodeVecI8x16ReplaceLane, wasm.OpcodeVecI16x8ReplaceLane, wasm.OpcodeVecI32x4ReplaceLane, + wasm.OpcodeVecI8x16Shl, wasm.OpcodeVecI8x16ShrS, wasm.OpcodeVecI8x16ShrU, + wasm.OpcodeVecI16x8Shl, wasm.OpcodeVecI16x8ShrS, wasm.OpcodeVecI16x8ShrU, + wasm.OpcodeVecI32x4Shl, wasm.OpcodeVecI32x4ShrS, wasm.OpcodeVecI32x4ShrU, + wasm.OpcodeVecI64x2Shl, wasm.OpcodeVecI64x2ShrS, wasm.OpcodeVecI64x2ShrU: + return signature_V128I32_V128, nil + case wasm.OpcodeVecI64x2ReplaceLane: + return signature_V128I64_V128, nil + case wasm.OpcodeVecF32x4ReplaceLane: + return signature_V128F32_V128, nil + case wasm.OpcodeVecF64x2ReplaceLane: + return signature_V128F64_V128, nil + case wasm.OpcodeVecI8x16Splat, + wasm.OpcodeVecI16x8Splat, + wasm.OpcodeVecI32x4Splat: + return signature_I32_V128, nil + case wasm.OpcodeVecI64x2Splat: + return signature_I64_V128, nil + case wasm.OpcodeVecF32x4Splat: + return signature_F32_V128, nil + case wasm.OpcodeVecF64x2Splat: + return signature_F64_V128, nil + case wasm.OpcodeVecV128i8x16Shuffle, wasm.OpcodeVecI8x16Swizzle, wasm.OpcodeVecV128And, wasm.OpcodeVecV128Or, wasm.OpcodeVecV128Xor, wasm.OpcodeVecV128AndNot: + return signature_V128V128_V128, nil + case wasm.OpcodeVecI8x16AllTrue, wasm.OpcodeVecI16x8AllTrue, wasm.OpcodeVecI32x4AllTrue, wasm.OpcodeVecI64x2AllTrue, + wasm.OpcodeVecV128AnyTrue, + wasm.OpcodeVecI8x16BitMask, wasm.OpcodeVecI16x8BitMask, wasm.OpcodeVecI32x4BitMask, wasm.OpcodeVecI64x2BitMask: + return signature_V128_I32, nil + case wasm.OpcodeVecV128Not, wasm.OpcodeVecI8x16Neg, wasm.OpcodeVecI16x8Neg, wasm.OpcodeVecI32x4Neg, wasm.OpcodeVecI64x2Neg, + wasm.OpcodeVecF32x4Neg, wasm.OpcodeVecF64x2Neg, wasm.OpcodeVecF32x4Sqrt, wasm.OpcodeVecF64x2Sqrt, + wasm.OpcodeVecI8x16Abs, wasm.OpcodeVecI8x16Popcnt, wasm.OpcodeVecI16x8Abs, wasm.OpcodeVecI32x4Abs, wasm.OpcodeVecI64x2Abs, + wasm.OpcodeVecF32x4Abs, wasm.OpcodeVecF64x2Abs, + wasm.OpcodeVecF32x4Ceil, wasm.OpcodeVecF32x4Floor, wasm.OpcodeVecF32x4Trunc, wasm.OpcodeVecF32x4Nearest, + wasm.OpcodeVecF64x2Ceil, wasm.OpcodeVecF64x2Floor, wasm.OpcodeVecF64x2Trunc, wasm.OpcodeVecF64x2Nearest, + wasm.OpcodeVecI16x8ExtendLowI8x16S, wasm.OpcodeVecI16x8ExtendHighI8x16S, wasm.OpcodeVecI16x8ExtendLowI8x16U, wasm.OpcodeVecI16x8ExtendHighI8x16U, + wasm.OpcodeVecI32x4ExtendLowI16x8S, wasm.OpcodeVecI32x4ExtendHighI16x8S, wasm.OpcodeVecI32x4ExtendLowI16x8U, wasm.OpcodeVecI32x4ExtendHighI16x8U, + wasm.OpcodeVecI64x2ExtendLowI32x4S, wasm.OpcodeVecI64x2ExtendHighI32x4S, wasm.OpcodeVecI64x2ExtendLowI32x4U, wasm.OpcodeVecI64x2ExtendHighI32x4U, + wasm.OpcodeVecI16x8ExtaddPairwiseI8x16S, wasm.OpcodeVecI16x8ExtaddPairwiseI8x16U, wasm.OpcodeVecI32x4ExtaddPairwiseI16x8S, wasm.OpcodeVecI32x4ExtaddPairwiseI16x8U, + wasm.OpcodeVecF64x2PromoteLowF32x4Zero, wasm.OpcodeVecF32x4DemoteF64x2Zero, + wasm.OpcodeVecF32x4ConvertI32x4S, wasm.OpcodeVecF32x4ConvertI32x4U, + wasm.OpcodeVecF64x2ConvertLowI32x4S, wasm.OpcodeVecF64x2ConvertLowI32x4U, + wasm.OpcodeVecI32x4TruncSatF32x4S, wasm.OpcodeVecI32x4TruncSatF32x4U, + wasm.OpcodeVecI32x4TruncSatF64x2SZero, wasm.OpcodeVecI32x4TruncSatF64x2UZero: + return signature_V128_V128, nil + case wasm.OpcodeVecV128Bitselect: + return signature_V128V128V128_V32, nil + case wasm.OpcodeVecI8x16Eq, wasm.OpcodeVecI8x16Ne, wasm.OpcodeVecI8x16LtS, wasm.OpcodeVecI8x16LtU, wasm.OpcodeVecI8x16GtS, + wasm.OpcodeVecI8x16GtU, wasm.OpcodeVecI8x16LeS, wasm.OpcodeVecI8x16LeU, wasm.OpcodeVecI8x16GeS, wasm.OpcodeVecI8x16GeU, + wasm.OpcodeVecI16x8Eq, wasm.OpcodeVecI16x8Ne, wasm.OpcodeVecI16x8LtS, wasm.OpcodeVecI16x8LtU, wasm.OpcodeVecI16x8GtS, + wasm.OpcodeVecI16x8GtU, wasm.OpcodeVecI16x8LeS, wasm.OpcodeVecI16x8LeU, wasm.OpcodeVecI16x8GeS, wasm.OpcodeVecI16x8GeU, + wasm.OpcodeVecI32x4Eq, wasm.OpcodeVecI32x4Ne, wasm.OpcodeVecI32x4LtS, wasm.OpcodeVecI32x4LtU, wasm.OpcodeVecI32x4GtS, + wasm.OpcodeVecI32x4GtU, wasm.OpcodeVecI32x4LeS, wasm.OpcodeVecI32x4LeU, wasm.OpcodeVecI32x4GeS, wasm.OpcodeVecI32x4GeU, + wasm.OpcodeVecI64x2Eq, wasm.OpcodeVecI64x2Ne, wasm.OpcodeVecI64x2LtS, wasm.OpcodeVecI64x2GtS, wasm.OpcodeVecI64x2LeS, + wasm.OpcodeVecI64x2GeS, wasm.OpcodeVecF32x4Eq, wasm.OpcodeVecF32x4Ne, wasm.OpcodeVecF32x4Lt, wasm.OpcodeVecF32x4Gt, + wasm.OpcodeVecF32x4Le, wasm.OpcodeVecF32x4Ge, wasm.OpcodeVecF64x2Eq, wasm.OpcodeVecF64x2Ne, wasm.OpcodeVecF64x2Lt, + wasm.OpcodeVecF64x2Gt, wasm.OpcodeVecF64x2Le, wasm.OpcodeVecF64x2Ge, + wasm.OpcodeVecI8x16Add, wasm.OpcodeVecI8x16AddSatS, wasm.OpcodeVecI8x16AddSatU, wasm.OpcodeVecI8x16Sub, + wasm.OpcodeVecI8x16SubSatS, wasm.OpcodeVecI8x16SubSatU, + wasm.OpcodeVecI16x8Add, wasm.OpcodeVecI16x8AddSatS, wasm.OpcodeVecI16x8AddSatU, wasm.OpcodeVecI16x8Sub, + wasm.OpcodeVecI16x8SubSatS, wasm.OpcodeVecI16x8SubSatU, wasm.OpcodeVecI16x8Mul, + wasm.OpcodeVecI32x4Add, wasm.OpcodeVecI32x4Sub, wasm.OpcodeVecI32x4Mul, + wasm.OpcodeVecI64x2Add, wasm.OpcodeVecI64x2Sub, wasm.OpcodeVecI64x2Mul, + wasm.OpcodeVecF32x4Add, wasm.OpcodeVecF32x4Sub, wasm.OpcodeVecF32x4Mul, wasm.OpcodeVecF32x4Div, + wasm.OpcodeVecF64x2Add, wasm.OpcodeVecF64x2Sub, wasm.OpcodeVecF64x2Mul, wasm.OpcodeVecF64x2Div, + wasm.OpcodeVecI8x16MinS, wasm.OpcodeVecI8x16MinU, wasm.OpcodeVecI8x16MaxS, wasm.OpcodeVecI8x16MaxU, wasm.OpcodeVecI8x16AvgrU, + wasm.OpcodeVecI16x8MinS, wasm.OpcodeVecI16x8MinU, wasm.OpcodeVecI16x8MaxS, wasm.OpcodeVecI16x8MaxU, wasm.OpcodeVecI16x8AvgrU, + wasm.OpcodeVecI32x4MinS, wasm.OpcodeVecI32x4MinU, wasm.OpcodeVecI32x4MaxS, wasm.OpcodeVecI32x4MaxU, + wasm.OpcodeVecF32x4Min, wasm.OpcodeVecF32x4Max, wasm.OpcodeVecF64x2Min, wasm.OpcodeVecF64x2Max, + wasm.OpcodeVecF32x4Pmin, wasm.OpcodeVecF32x4Pmax, wasm.OpcodeVecF64x2Pmin, wasm.OpcodeVecF64x2Pmax, + wasm.OpcodeVecI16x8Q15mulrSatS, + wasm.OpcodeVecI16x8ExtMulLowI8x16S, wasm.OpcodeVecI16x8ExtMulHighI8x16S, wasm.OpcodeVecI16x8ExtMulLowI8x16U, wasm.OpcodeVecI16x8ExtMulHighI8x16U, + wasm.OpcodeVecI32x4ExtMulLowI16x8S, wasm.OpcodeVecI32x4ExtMulHighI16x8S, wasm.OpcodeVecI32x4ExtMulLowI16x8U, wasm.OpcodeVecI32x4ExtMulHighI16x8U, + wasm.OpcodeVecI64x2ExtMulLowI32x4S, wasm.OpcodeVecI64x2ExtMulHighI32x4S, wasm.OpcodeVecI64x2ExtMulLowI32x4U, wasm.OpcodeVecI64x2ExtMulHighI32x4U, + wasm.OpcodeVecI32x4DotI16x8S, + wasm.OpcodeVecI8x16NarrowI16x8S, wasm.OpcodeVecI8x16NarrowI16x8U, wasm.OpcodeVecI16x8NarrowI32x4S, wasm.OpcodeVecI16x8NarrowI32x4U: + return signature_V128V128_V128, nil + default: + return nil, fmt.Errorf("unsupported vector instruction in interpreterir: %s", wasm.VectorInstructionName(vecOp)) + } + case wasm.OpcodeAtomicPrefix: + switch atomicOp := c.body[c.pc+1]; atomicOp { + case wasm.OpcodeAtomicMemoryNotify: + return signature_I32I32_I32, nil + case wasm.OpcodeAtomicMemoryWait32: + return signature_I32I32I64_I32, nil + case wasm.OpcodeAtomicMemoryWait64: + return signature_I32I64I64_I32, nil + case wasm.OpcodeAtomicFence: + return signature_None_None, nil + case wasm.OpcodeAtomicI32Load, wasm.OpcodeAtomicI32Load8U, wasm.OpcodeAtomicI32Load16U: + return signature_I32_I32, nil + case wasm.OpcodeAtomicI64Load, wasm.OpcodeAtomicI64Load8U, wasm.OpcodeAtomicI64Load16U, wasm.OpcodeAtomicI64Load32U: + return signature_I32_I64, nil + case wasm.OpcodeAtomicI32Store, wasm.OpcodeAtomicI32Store8, wasm.OpcodeAtomicI32Store16: + return signature_I32I32_None, nil + case wasm.OpcodeAtomicI64Store, wasm.OpcodeAtomicI64Store8, wasm.OpcodeAtomicI64Store16, wasm.OpcodeAtomicI64Store32: + return signature_I32I64_None, nil + case wasm.OpcodeAtomicI32RmwAdd, wasm.OpcodeAtomicI32RmwSub, wasm.OpcodeAtomicI32RmwAnd, wasm.OpcodeAtomicI32RmwOr, wasm.OpcodeAtomicI32RmwXor, wasm.OpcodeAtomicI32RmwXchg, + wasm.OpcodeAtomicI32Rmw8AddU, wasm.OpcodeAtomicI32Rmw8SubU, wasm.OpcodeAtomicI32Rmw8AndU, wasm.OpcodeAtomicI32Rmw8OrU, wasm.OpcodeAtomicI32Rmw8XorU, wasm.OpcodeAtomicI32Rmw8XchgU, + wasm.OpcodeAtomicI32Rmw16AddU, wasm.OpcodeAtomicI32Rmw16SubU, wasm.OpcodeAtomicI32Rmw16AndU, wasm.OpcodeAtomicI32Rmw16OrU, wasm.OpcodeAtomicI32Rmw16XorU, wasm.OpcodeAtomicI32Rmw16XchgU: + return signature_I32I32_I32, nil + case wasm.OpcodeAtomicI64RmwAdd, wasm.OpcodeAtomicI64RmwSub, wasm.OpcodeAtomicI64RmwAnd, wasm.OpcodeAtomicI64RmwOr, wasm.OpcodeAtomicI64RmwXor, wasm.OpcodeAtomicI64RmwXchg, + wasm.OpcodeAtomicI64Rmw8AddU, wasm.OpcodeAtomicI64Rmw8SubU, wasm.OpcodeAtomicI64Rmw8AndU, wasm.OpcodeAtomicI64Rmw8OrU, wasm.OpcodeAtomicI64Rmw8XorU, wasm.OpcodeAtomicI64Rmw8XchgU, + wasm.OpcodeAtomicI64Rmw16AddU, wasm.OpcodeAtomicI64Rmw16SubU, wasm.OpcodeAtomicI64Rmw16AndU, wasm.OpcodeAtomicI64Rmw16OrU, wasm.OpcodeAtomicI64Rmw16XorU, wasm.OpcodeAtomicI64Rmw16XchgU, + wasm.OpcodeAtomicI64Rmw32AddU, wasm.OpcodeAtomicI64Rmw32SubU, wasm.OpcodeAtomicI64Rmw32AndU, wasm.OpcodeAtomicI64Rmw32OrU, wasm.OpcodeAtomicI64Rmw32XorU, wasm.OpcodeAtomicI64Rmw32XchgU: + return signature_I32I64_I64, nil + case wasm.OpcodeAtomicI32RmwCmpxchg, wasm.OpcodeAtomicI32Rmw8CmpxchgU, wasm.OpcodeAtomicI32Rmw16CmpxchgU: + return signature_I32I32I32_I32, nil + case wasm.OpcodeAtomicI64RmwCmpxchg, wasm.OpcodeAtomicI64Rmw8CmpxchgU, wasm.OpcodeAtomicI64Rmw16CmpxchgU, wasm.OpcodeAtomicI64Rmw32CmpxchgU: + return signature_I32I64I64_I64, nil + default: + return nil, fmt.Errorf("unsupported atomic instruction in interpreterir: %s", wasm.AtomicInstructionName(atomicOp)) + } + default: + return nil, fmt.Errorf("unsupported instruction in interpreterir: 0x%x", op) + } +} + +// funcTypeToIRSignatures is the central cache for a module to get the *signature +// for function calls. +type funcTypeToIRSignatures struct { + directCalls []*signature + indirectCalls []*signature + wasmTypes []wasm.FunctionType +} + +// get returns the *signature for the direct or indirect function call against functions whose type is at `typeIndex`. +func (f *funcTypeToIRSignatures) get(typeIndex wasm.Index, indirect bool) *signature { + var sig *signature + if indirect { + sig = f.indirectCalls[typeIndex] + } else { + sig = f.directCalls[typeIndex] + } + if sig != nil { + return sig + } + + tp := &f.wasmTypes[typeIndex] + if indirect { + sig = &signature{ + in: make([]unsignedType, 0, len(tp.Params)+1), // +1 to reserve space for call indirect index. + out: make([]unsignedType, 0, len(tp.Results)), + } + } else { + sig = &signature{ + in: make([]unsignedType, 0, len(tp.Params)), + out: make([]unsignedType, 0, len(tp.Results)), + } + } + + for _, vt := range tp.Params { + sig.in = append(sig.in, wasmValueTypeTounsignedType(vt)) + } + for _, vt := range tp.Results { + sig.out = append(sig.out, wasmValueTypeTounsignedType(vt)) + } + + if indirect { + sig.in = append(sig.in, unsignedTypeI32) + f.indirectCalls[typeIndex] = sig + } else { + f.directCalls[typeIndex] = sig + } + return sig +} + +func wasmValueTypeTounsignedType(vt wasm.ValueType) unsignedType { + switch vt { + case wasm.ValueTypeI32: + return unsignedTypeI32 + case wasm.ValueTypeI64, + // From interpreterir layer, ref type values are opaque 64-bit pointers. + wasm.ValueTypeExternref, wasm.ValueTypeFuncref: + return unsignedTypeI64 + case wasm.ValueTypeF32: + return unsignedTypeF32 + case wasm.ValueTypeF64: + return unsignedTypeF64 + case wasm.ValueTypeV128: + return unsignedTypeV128 + } + panic("unreachable") +} + +func wasmValueTypeToUnsignedOutSignature(vt wasm.ValueType) *signature { + switch vt { + case wasm.ValueTypeI32: + return signature_None_I32 + case wasm.ValueTypeI64, + // From interpreterir layer, ref type values are opaque 64-bit pointers. + wasm.ValueTypeExternref, wasm.ValueTypeFuncref: + return signature_None_I64 + case wasm.ValueTypeF32: + return signature_None_F32 + case wasm.ValueTypeF64: + return signature_None_F64 + case wasm.ValueTypeV128: + return signature_None_V128 + } + panic("unreachable") +} + +func wasmValueTypeToUnsignedInSignature(vt wasm.ValueType) *signature { + switch vt { + case wasm.ValueTypeI32: + return signature_I32_None + case wasm.ValueTypeI64, + // From interpreterir layer, ref type values are opaque 64-bit pointers. + wasm.ValueTypeExternref, wasm.ValueTypeFuncref: + return signature_I64_None + case wasm.ValueTypeF32: + return signature_F32_None + case wasm.ValueTypeF64: + return signature_F64_None + case wasm.ValueTypeV128: + return signature_V128_None + } + panic("unreachable") +} + +func wasmValueTypeToUnsignedInOutSignature(vt wasm.ValueType) *signature { + switch vt { + case wasm.ValueTypeI32: + return signature_I32_I32 + case wasm.ValueTypeI64, + // At interpreterir layer, ref type values are opaque 64-bit pointers. + wasm.ValueTypeExternref, wasm.ValueTypeFuncref: + return signature_I64_I64 + case wasm.ValueTypeF32: + return signature_F32_F32 + case wasm.ValueTypeF64: + return signature_F64_F64 + case wasm.ValueTypeV128: + return signature_V128_V128 + } + panic("unreachable") +} diff --git a/vendor/github.com/tetratelabs/wazero/internal/engine/wazevo/backend/abi.go b/vendor/github.com/tetratelabs/wazero/internal/engine/wazevo/backend/abi.go new file mode 100644 index 000000000..cf91c6b7a --- /dev/null +++ b/vendor/github.com/tetratelabs/wazero/internal/engine/wazevo/backend/abi.go @@ -0,0 +1,170 @@ +package backend + +import ( + "fmt" + + "github.com/tetratelabs/wazero/internal/engine/wazevo/backend/regalloc" + "github.com/tetratelabs/wazero/internal/engine/wazevo/ssa" +) + +type ( + // FunctionABI represents the ABI information for a function which corresponds to a ssa.Signature. + FunctionABI struct { + Initialized bool + + Args, Rets []ABIArg + ArgStackSize, RetStackSize int64 + + ArgIntRealRegs byte + ArgFloatRealRegs byte + RetIntRealRegs byte + RetFloatRealRegs byte + } + + // ABIArg represents either argument or return value's location. + ABIArg struct { + // Index is the index of the argument. + Index int + // Kind is the kind of the argument. + Kind ABIArgKind + // Reg is valid if Kind == ABIArgKindReg. + // This VReg must be based on RealReg. + Reg regalloc.VReg + // Offset is valid if Kind == ABIArgKindStack. + // This is the offset from the beginning of either arg or ret stack slot. + Offset int64 + // Type is the type of the argument. + Type ssa.Type + } + + // ABIArgKind is the kind of ABI argument. + ABIArgKind byte +) + +const ( + // ABIArgKindReg represents an argument passed in a register. + ABIArgKindReg = iota + // ABIArgKindStack represents an argument passed in the stack. + ABIArgKindStack +) + +// String implements fmt.Stringer. +func (a *ABIArg) String() string { + return fmt.Sprintf("args[%d]: %s", a.Index, a.Kind) +} + +// String implements fmt.Stringer. +func (a ABIArgKind) String() string { + switch a { + case ABIArgKindReg: + return "reg" + case ABIArgKindStack: + return "stack" + default: + panic("BUG") + } +} + +// Init initializes the abiImpl for the given signature. +func (a *FunctionABI) Init(sig *ssa.Signature, argResultInts, argResultFloats []regalloc.RealReg) { + if len(a.Rets) < len(sig.Results) { + a.Rets = make([]ABIArg, len(sig.Results)) + } + a.Rets = a.Rets[:len(sig.Results)] + a.RetStackSize = a.setABIArgs(a.Rets, sig.Results, argResultInts, argResultFloats) + if argsNum := len(sig.Params); len(a.Args) < argsNum { + a.Args = make([]ABIArg, argsNum) + } + a.Args = a.Args[:len(sig.Params)] + a.ArgStackSize = a.setABIArgs(a.Args, sig.Params, argResultInts, argResultFloats) + + // Gather the real registers usages in arg/return. + a.ArgIntRealRegs, a.ArgFloatRealRegs = 0, 0 + a.RetIntRealRegs, a.RetFloatRealRegs = 0, 0 + for i := range a.Rets { + r := &a.Rets[i] + if r.Kind == ABIArgKindReg { + if r.Type.IsInt() { + a.RetIntRealRegs++ + } else { + a.RetFloatRealRegs++ + } + } + } + for i := range a.Args { + arg := &a.Args[i] + if arg.Kind == ABIArgKindReg { + if arg.Type.IsInt() { + a.ArgIntRealRegs++ + } else { + a.ArgFloatRealRegs++ + } + } + } + + a.Initialized = true +} + +// setABIArgs sets the ABI arguments in the given slice. This assumes that len(s) >= len(types) +// where if len(s) > len(types), the last elements of s is for the multi-return slot. +func (a *FunctionABI) setABIArgs(s []ABIArg, types []ssa.Type, ints, floats []regalloc.RealReg) (stackSize int64) { + il, fl := len(ints), len(floats) + + var stackOffset int64 + intParamIndex, floatParamIndex := 0, 0 + for i, typ := range types { + arg := &s[i] + arg.Index = i + arg.Type = typ + if typ.IsInt() { + if intParamIndex >= il { + arg.Kind = ABIArgKindStack + const slotSize = 8 // Align 8 bytes. + arg.Offset = stackOffset + stackOffset += slotSize + } else { + arg.Kind = ABIArgKindReg + arg.Reg = regalloc.FromRealReg(ints[intParamIndex], regalloc.RegTypeInt) + intParamIndex++ + } + } else { + if floatParamIndex >= fl { + arg.Kind = ABIArgKindStack + slotSize := int64(8) // Align at least 8 bytes. + if typ.Bits() == 128 { // Vector. + slotSize = 16 + } + arg.Offset = stackOffset + stackOffset += slotSize + } else { + arg.Kind = ABIArgKindReg + arg.Reg = regalloc.FromRealReg(floats[floatParamIndex], regalloc.RegTypeFloat) + floatParamIndex++ + } + } + } + return stackOffset +} + +func (a *FunctionABI) AlignedArgResultStackSlotSize() uint32 { + stackSlotSize := a.RetStackSize + a.ArgStackSize + // Align stackSlotSize to 16 bytes. + stackSlotSize = (stackSlotSize + 15) &^ 15 + // Check overflow 32-bit. + if stackSlotSize > 0xFFFFFFFF { + panic("ABI stack slot size overflow") + } + return uint32(stackSlotSize) +} + +func (a *FunctionABI) ABIInfoAsUint64() uint64 { + return uint64(a.ArgIntRealRegs)<<56 | + uint64(a.ArgFloatRealRegs)<<48 | + uint64(a.RetIntRealRegs)<<40 | + uint64(a.RetFloatRealRegs)<<32 | + uint64(a.AlignedArgResultStackSlotSize()) +} + +func ABIInfoFromUint64(info uint64) (argIntRealRegs, argFloatRealRegs, retIntRealRegs, retFloatRealRegs byte, stackSlotSize uint32) { + return byte(info >> 56), byte(info >> 48), byte(info >> 40), byte(info >> 32), uint32(info) +} diff --git a/vendor/github.com/tetratelabs/wazero/internal/engine/wazevo/backend/backend.go b/vendor/github.com/tetratelabs/wazero/internal/engine/wazevo/backend/backend.go new file mode 100644 index 000000000..dd67da43e --- /dev/null +++ b/vendor/github.com/tetratelabs/wazero/internal/engine/wazevo/backend/backend.go @@ -0,0 +1,3 @@ +// Package backend must be free of Wasm-specific concept. In other words, +// this package must not import internal/wasm package. +package backend diff --git a/vendor/github.com/tetratelabs/wazero/internal/engine/wazevo/backend/compiler.go b/vendor/github.com/tetratelabs/wazero/internal/engine/wazevo/backend/compiler.go new file mode 100644 index 000000000..59bbfe02d --- /dev/null +++ b/vendor/github.com/tetratelabs/wazero/internal/engine/wazevo/backend/compiler.go @@ -0,0 +1,417 @@ +package backend + +import ( + "context" + "fmt" + + "github.com/tetratelabs/wazero/internal/engine/wazevo/backend/regalloc" + "github.com/tetratelabs/wazero/internal/engine/wazevo/ssa" + "github.com/tetratelabs/wazero/internal/engine/wazevo/wazevoapi" +) + +// NewCompiler returns a new Compiler that can generate a machine code. +func NewCompiler(ctx context.Context, mach Machine, builder ssa.Builder) Compiler { + return newCompiler(ctx, mach, builder) +} + +func newCompiler(_ context.Context, mach Machine, builder ssa.Builder) *compiler { + argResultInts, argResultFloats := mach.ArgsResultsRegs() + c := &compiler{ + mach: mach, ssaBuilder: builder, + nextVRegID: regalloc.VRegIDNonReservedBegin, + argResultInts: argResultInts, + argResultFloats: argResultFloats, + } + mach.SetCompiler(c) + return c +} + +// Compiler is the backend of wazevo which takes ssa.Builder and Machine, +// use the information there to emit the final machine code. +type Compiler interface { + // SSABuilder returns the ssa.Builder used by this compiler. + SSABuilder() ssa.Builder + + // Compile executes the following steps: + // 1. Lower() + // 2. RegAlloc() + // 3. Finalize() + // 4. Encode() + // + // Each step can be called individually for testing purpose, therefore they are exposed in this interface too. + // + // The returned byte slices are the machine code and the relocation information for the machine code. + // The caller is responsible for copying them immediately since the compiler may reuse the buffer. + Compile(ctx context.Context) (_ []byte, _ []RelocationInfo, _ error) + + // Lower lowers the given ssa.Instruction to the machine-specific instructions. + Lower() + + // RegAlloc performs the register allocation after Lower is called. + RegAlloc() + + // Finalize performs the finalization of the compilation, including machine code emission. + // This must be called after RegAlloc. + Finalize(ctx context.Context) error + + // Buf returns the buffer of the encoded machine code. This is only used for testing purpose. + Buf() []byte + + BufPtr() *[]byte + + // Format returns the debug string of the current state of the compiler. + Format() string + + // Init initializes the internal state of the compiler for the next compilation. + Init() + + // AllocateVReg allocates a new virtual register of the given type. + AllocateVReg(typ ssa.Type) regalloc.VReg + + // ValueDefinition returns the definition of the given value. + ValueDefinition(ssa.Value) *SSAValueDefinition + + // VRegOf returns the virtual register of the given ssa.Value. + VRegOf(value ssa.Value) regalloc.VReg + + // TypeOf returns the ssa.Type of the given virtual register. + TypeOf(regalloc.VReg) ssa.Type + + // MatchInstr returns true if the given definition is from an instruction with the given opcode, the current group ID, + // and a refcount of 1. That means, the instruction can be merged/swapped within the current instruction group. + MatchInstr(def *SSAValueDefinition, opcode ssa.Opcode) bool + + // MatchInstrOneOf is the same as MatchInstr but for multiple opcodes. If it matches one of ssa.Opcode, + // this returns the opcode. Otherwise, this returns ssa.OpcodeInvalid. + // + // Note: caller should be careful to avoid excessive allocation on opcodes slice. + MatchInstrOneOf(def *SSAValueDefinition, opcodes []ssa.Opcode) ssa.Opcode + + // AddRelocationInfo appends the relocation information for the function reference at the current buffer offset. + AddRelocationInfo(funcRef ssa.FuncRef) + + // AddSourceOffsetInfo appends the source offset information for the given offset. + AddSourceOffsetInfo(executableOffset int64, sourceOffset ssa.SourceOffset) + + // SourceOffsetInfo returns the source offset information for the current buffer offset. + SourceOffsetInfo() []SourceOffsetInfo + + // EmitByte appends a byte to the buffer. Used during the code emission. + EmitByte(b byte) + + // Emit4Bytes appends 4 bytes to the buffer. Used during the code emission. + Emit4Bytes(b uint32) + + // Emit8Bytes appends 8 bytes to the buffer. Used during the code emission. + Emit8Bytes(b uint64) + + // GetFunctionABI returns the ABI information for the given signature. + GetFunctionABI(sig *ssa.Signature) *FunctionABI +} + +// RelocationInfo represents the relocation information for a call instruction. +type RelocationInfo struct { + // Offset represents the offset from the beginning of the machine code of either a function or the entire module. + Offset int64 + // Target is the target function of the call instruction. + FuncRef ssa.FuncRef +} + +// compiler implements Compiler. +type compiler struct { + mach Machine + currentGID ssa.InstructionGroupID + ssaBuilder ssa.Builder + // nextVRegID is the next virtual register ID to be allocated. + nextVRegID regalloc.VRegID + // ssaValueToVRegs maps ssa.ValueID to regalloc.VReg. + ssaValueToVRegs [] /* VRegID to */ regalloc.VReg + // ssaValueDefinitions maps ssa.ValueID to its definition. + ssaValueDefinitions []SSAValueDefinition + // ssaValueRefCounts is a cached list obtained by ssa.Builder.ValueRefCounts(). + ssaValueRefCounts []int + // returnVRegs is the list of virtual registers that store the return values. + returnVRegs []regalloc.VReg + varEdges [][2]regalloc.VReg + varEdgeTypes []ssa.Type + constEdges []struct { + cInst *ssa.Instruction + dst regalloc.VReg + } + vRegSet []bool + vRegIDs []regalloc.VRegID + tempRegs []regalloc.VReg + tmpVals []ssa.Value + ssaTypeOfVRegID [] /* VRegID to */ ssa.Type + buf []byte + relocations []RelocationInfo + sourceOffsets []SourceOffsetInfo + // abis maps ssa.SignatureID to the ABI implementation. + abis []FunctionABI + argResultInts, argResultFloats []regalloc.RealReg +} + +// SourceOffsetInfo is a data to associate the source offset with the executable offset. +type SourceOffsetInfo struct { + // SourceOffset is the source offset in the original source code. + SourceOffset ssa.SourceOffset + // ExecutableOffset is the offset in the compiled executable. + ExecutableOffset int64 +} + +// Compile implements Compiler.Compile. +func (c *compiler) Compile(ctx context.Context) ([]byte, []RelocationInfo, error) { + c.Lower() + if wazevoapi.PrintSSAToBackendIRLowering && wazevoapi.PrintEnabledIndex(ctx) { + fmt.Printf("[[[after lowering for %s ]]]%s\n", wazevoapi.GetCurrentFunctionName(ctx), c.Format()) + } + if wazevoapi.DeterministicCompilationVerifierEnabled { + wazevoapi.VerifyOrSetDeterministicCompilationContextValue(ctx, "After lowering to ISA specific IR", c.Format()) + } + c.RegAlloc() + if wazevoapi.PrintRegisterAllocated && wazevoapi.PrintEnabledIndex(ctx) { + fmt.Printf("[[[after regalloc for %s]]]%s\n", wazevoapi.GetCurrentFunctionName(ctx), c.Format()) + } + if wazevoapi.DeterministicCompilationVerifierEnabled { + wazevoapi.VerifyOrSetDeterministicCompilationContextValue(ctx, "After Register Allocation", c.Format()) + } + if err := c.Finalize(ctx); err != nil { + return nil, nil, err + } + if wazevoapi.PrintFinalizedMachineCode && wazevoapi.PrintEnabledIndex(ctx) { + fmt.Printf("[[[after finalize for %s]]]%s\n", wazevoapi.GetCurrentFunctionName(ctx), c.Format()) + } + if wazevoapi.DeterministicCompilationVerifierEnabled { + wazevoapi.VerifyOrSetDeterministicCompilationContextValue(ctx, "After Finalization", c.Format()) + } + return c.buf, c.relocations, nil +} + +// RegAlloc implements Compiler.RegAlloc. +func (c *compiler) RegAlloc() { + c.mach.RegAlloc() +} + +// Finalize implements Compiler.Finalize. +func (c *compiler) Finalize(ctx context.Context) error { + c.mach.PostRegAlloc() + return c.mach.Encode(ctx) +} + +// setCurrentGroupID sets the current instruction group ID. +func (c *compiler) setCurrentGroupID(gid ssa.InstructionGroupID) { + c.currentGID = gid +} + +// assignVirtualRegisters assigns a virtual register to each ssa.ValueID Valid in the ssa.Builder. +func (c *compiler) assignVirtualRegisters() { + builder := c.ssaBuilder + refCounts := builder.ValueRefCounts() + c.ssaValueRefCounts = refCounts + + need := len(refCounts) + if need >= len(c.ssaValueToVRegs) { + c.ssaValueToVRegs = append(c.ssaValueToVRegs, make([]regalloc.VReg, need+1)...) + } + if need >= len(c.ssaValueDefinitions) { + c.ssaValueDefinitions = append(c.ssaValueDefinitions, make([]SSAValueDefinition, need+1)...) + } + + for blk := builder.BlockIteratorReversePostOrderBegin(); blk != nil; blk = builder.BlockIteratorReversePostOrderNext() { + // First we assign a virtual register to each parameter. + for i := 0; i < blk.Params(); i++ { + p := blk.Param(i) + pid := p.ID() + typ := p.Type() + vreg := c.AllocateVReg(typ) + c.ssaValueToVRegs[pid] = vreg + c.ssaValueDefinitions[pid] = SSAValueDefinition{BlockParamValue: p, BlkParamVReg: vreg} + c.ssaTypeOfVRegID[vreg.ID()] = p.Type() + } + + // Assigns each value to a virtual register produced by instructions. + for cur := blk.Root(); cur != nil; cur = cur.Next() { + r, rs := cur.Returns() + var N int + if r.Valid() { + id := r.ID() + ssaTyp := r.Type() + typ := r.Type() + vReg := c.AllocateVReg(typ) + c.ssaValueToVRegs[id] = vReg + c.ssaValueDefinitions[id] = SSAValueDefinition{ + Instr: cur, + N: 0, + RefCount: refCounts[id], + } + c.ssaTypeOfVRegID[vReg.ID()] = ssaTyp + N++ + } + for _, r := range rs { + id := r.ID() + ssaTyp := r.Type() + vReg := c.AllocateVReg(ssaTyp) + c.ssaValueToVRegs[id] = vReg + c.ssaValueDefinitions[id] = SSAValueDefinition{ + Instr: cur, + N: N, + RefCount: refCounts[id], + } + c.ssaTypeOfVRegID[vReg.ID()] = ssaTyp + N++ + } + } + } + + for i, retBlk := 0, builder.ReturnBlock(); i < retBlk.Params(); i++ { + typ := retBlk.Param(i).Type() + vReg := c.AllocateVReg(typ) + c.returnVRegs = append(c.returnVRegs, vReg) + c.ssaTypeOfVRegID[vReg.ID()] = typ + } +} + +// AllocateVReg implements Compiler.AllocateVReg. +func (c *compiler) AllocateVReg(typ ssa.Type) regalloc.VReg { + regType := regalloc.RegTypeOf(typ) + r := regalloc.VReg(c.nextVRegID).SetRegType(regType) + + id := r.ID() + if int(id) >= len(c.ssaTypeOfVRegID) { + c.ssaTypeOfVRegID = append(c.ssaTypeOfVRegID, make([]ssa.Type, id+1)...) + } + c.ssaTypeOfVRegID[id] = typ + c.nextVRegID++ + return r +} + +// Init implements Compiler.Init. +func (c *compiler) Init() { + c.currentGID = 0 + c.nextVRegID = regalloc.VRegIDNonReservedBegin + c.returnVRegs = c.returnVRegs[:0] + c.mach.Reset() + c.varEdges = c.varEdges[:0] + c.constEdges = c.constEdges[:0] + c.buf = c.buf[:0] + c.sourceOffsets = c.sourceOffsets[:0] + c.relocations = c.relocations[:0] +} + +// ValueDefinition implements Compiler.ValueDefinition. +func (c *compiler) ValueDefinition(value ssa.Value) *SSAValueDefinition { + return &c.ssaValueDefinitions[value.ID()] +} + +// VRegOf implements Compiler.VRegOf. +func (c *compiler) VRegOf(value ssa.Value) regalloc.VReg { + return c.ssaValueToVRegs[value.ID()] +} + +// Format implements Compiler.Format. +func (c *compiler) Format() string { + return c.mach.Format() +} + +// TypeOf implements Compiler.Format. +func (c *compiler) TypeOf(v regalloc.VReg) ssa.Type { + return c.ssaTypeOfVRegID[v.ID()] +} + +// MatchInstr implements Compiler.MatchInstr. +func (c *compiler) MatchInstr(def *SSAValueDefinition, opcode ssa.Opcode) bool { + instr := def.Instr + return def.IsFromInstr() && + instr.Opcode() == opcode && + instr.GroupID() == c.currentGID && + def.RefCount < 2 +} + +// MatchInstrOneOf implements Compiler.MatchInstrOneOf. +func (c *compiler) MatchInstrOneOf(def *SSAValueDefinition, opcodes []ssa.Opcode) ssa.Opcode { + instr := def.Instr + if !def.IsFromInstr() { + return ssa.OpcodeInvalid + } + + if instr.GroupID() != c.currentGID { + return ssa.OpcodeInvalid + } + + if def.RefCount >= 2 { + return ssa.OpcodeInvalid + } + + opcode := instr.Opcode() + for _, op := range opcodes { + if opcode == op { + return opcode + } + } + return ssa.OpcodeInvalid +} + +// SSABuilder implements Compiler .SSABuilder. +func (c *compiler) SSABuilder() ssa.Builder { + return c.ssaBuilder +} + +// AddSourceOffsetInfo implements Compiler.AddSourceOffsetInfo. +func (c *compiler) AddSourceOffsetInfo(executableOffset int64, sourceOffset ssa.SourceOffset) { + c.sourceOffsets = append(c.sourceOffsets, SourceOffsetInfo{ + SourceOffset: sourceOffset, + ExecutableOffset: executableOffset, + }) +} + +// SourceOffsetInfo implements Compiler.SourceOffsetInfo. +func (c *compiler) SourceOffsetInfo() []SourceOffsetInfo { + return c.sourceOffsets +} + +// AddRelocationInfo implements Compiler.AddRelocationInfo. +func (c *compiler) AddRelocationInfo(funcRef ssa.FuncRef) { + c.relocations = append(c.relocations, RelocationInfo{ + Offset: int64(len(c.buf)), + FuncRef: funcRef, + }) +} + +// Emit8Bytes implements Compiler.Emit8Bytes. +func (c *compiler) Emit8Bytes(b uint64) { + c.buf = append(c.buf, byte(b), byte(b>>8), byte(b>>16), byte(b>>24), byte(b>>32), byte(b>>40), byte(b>>48), byte(b>>56)) +} + +// Emit4Bytes implements Compiler.Emit4Bytes. +func (c *compiler) Emit4Bytes(b uint32) { + c.buf = append(c.buf, byte(b), byte(b>>8), byte(b>>16), byte(b>>24)) +} + +// EmitByte implements Compiler.EmitByte. +func (c *compiler) EmitByte(b byte) { + c.buf = append(c.buf, b) +} + +// Buf implements Compiler.Buf. +func (c *compiler) Buf() []byte { + return c.buf +} + +// BufPtr implements Compiler.BufPtr. +func (c *compiler) BufPtr() *[]byte { + return &c.buf +} + +func (c *compiler) GetFunctionABI(sig *ssa.Signature) *FunctionABI { + if int(sig.ID) >= len(c.abis) { + c.abis = append(c.abis, make([]FunctionABI, int(sig.ID)+1)...) + } + + abi := &c.abis[sig.ID] + if abi.Initialized { + return abi + } + + abi.Init(sig, c.argResultInts, c.argResultFloats) + return abi +} diff --git a/vendor/github.com/tetratelabs/wazero/internal/engine/wazevo/backend/compiler_lower.go b/vendor/github.com/tetratelabs/wazero/internal/engine/wazevo/backend/compiler_lower.go new file mode 100644 index 000000000..80e65668a --- /dev/null +++ b/vendor/github.com/tetratelabs/wazero/internal/engine/wazevo/backend/compiler_lower.go @@ -0,0 +1,226 @@ +package backend + +import ( + "github.com/tetratelabs/wazero/internal/engine/wazevo/backend/regalloc" + "github.com/tetratelabs/wazero/internal/engine/wazevo/ssa" +) + +// Lower implements Compiler.Lower. +func (c *compiler) Lower() { + c.assignVirtualRegisters() + c.mach.SetCurrentABI(c.GetFunctionABI(c.ssaBuilder.Signature())) + c.mach.ExecutableContext().StartLoweringFunction(c.ssaBuilder.BlockIDMax()) + c.lowerBlocks() +} + +// lowerBlocks lowers each block in the ssa.Builder. +func (c *compiler) lowerBlocks() { + builder := c.ssaBuilder + for blk := builder.BlockIteratorReversePostOrderBegin(); blk != nil; blk = builder.BlockIteratorReversePostOrderNext() { + c.lowerBlock(blk) + } + + ectx := c.mach.ExecutableContext() + // After lowering all blocks, we need to link adjacent blocks to layout one single instruction list. + var prev ssa.BasicBlock + for next := builder.BlockIteratorReversePostOrderBegin(); next != nil; next = builder.BlockIteratorReversePostOrderNext() { + if prev != nil { + ectx.LinkAdjacentBlocks(prev, next) + } + prev = next + } +} + +func (c *compiler) lowerBlock(blk ssa.BasicBlock) { + mach := c.mach + ectx := mach.ExecutableContext() + ectx.StartBlock(blk) + + // We traverse the instructions in reverse order because we might want to lower multiple + // instructions together. + cur := blk.Tail() + + // First gather the branching instructions at the end of the blocks. + var br0, br1 *ssa.Instruction + if cur.IsBranching() { + br0 = cur + cur = cur.Prev() + if cur != nil && cur.IsBranching() { + br1 = cur + cur = cur.Prev() + } + } + + if br0 != nil { + c.lowerBranches(br0, br1) + } + + if br1 != nil && br0 == nil { + panic("BUG? when a block has conditional branch but doesn't end with an unconditional branch?") + } + + // Now start lowering the non-branching instructions. + for ; cur != nil; cur = cur.Prev() { + c.setCurrentGroupID(cur.GroupID()) + if cur.Lowered() { + continue + } + + switch cur.Opcode() { + case ssa.OpcodeReturn: + rets := cur.ReturnVals() + if len(rets) > 0 { + c.mach.LowerReturns(rets) + } + c.mach.InsertReturn() + default: + mach.LowerInstr(cur) + } + ectx.FlushPendingInstructions() + } + + // Finally, if this is the entry block, we have to insert copies of arguments from the real location to the VReg. + if blk.EntryBlock() { + c.lowerFunctionArguments(blk) + } + + ectx.EndBlock() +} + +// lowerBranches is called right after StartBlock and before any LowerInstr call if +// there are branches to the given block. br0 is the very end of the block and b1 is the before the br0 if it exists. +// At least br0 is not nil, but br1 can be nil if there's no branching before br0. +// +// See ssa.Instruction IsBranching, and the comment on ssa.BasicBlock. +func (c *compiler) lowerBranches(br0, br1 *ssa.Instruction) { + ectx := c.mach.ExecutableContext() + + c.setCurrentGroupID(br0.GroupID()) + c.mach.LowerSingleBranch(br0) + ectx.FlushPendingInstructions() + if br1 != nil { + c.setCurrentGroupID(br1.GroupID()) + c.mach.LowerConditionalBranch(br1) + ectx.FlushPendingInstructions() + } + + if br0.Opcode() == ssa.OpcodeJump { + _, args, target := br0.BranchData() + argExists := len(args) != 0 + if argExists && br1 != nil { + panic("BUG: critical edge split failed") + } + if argExists && target.ReturnBlock() { + if len(args) > 0 { + c.mach.LowerReturns(args) + } + } else if argExists { + c.lowerBlockArguments(args, target) + } + } + ectx.FlushPendingInstructions() +} + +func (c *compiler) lowerFunctionArguments(entry ssa.BasicBlock) { + ectx := c.mach.ExecutableContext() + + c.tmpVals = c.tmpVals[:0] + for i := 0; i < entry.Params(); i++ { + p := entry.Param(i) + if c.ssaValueRefCounts[p.ID()] > 0 { + c.tmpVals = append(c.tmpVals, p) + } else { + // If the argument is not used, we can just pass an invalid value. + c.tmpVals = append(c.tmpVals, ssa.ValueInvalid) + } + } + c.mach.LowerParams(c.tmpVals) + ectx.FlushPendingInstructions() +} + +// lowerBlockArguments lowers how to pass arguments to the given successor block. +func (c *compiler) lowerBlockArguments(args []ssa.Value, succ ssa.BasicBlock) { + if len(args) != succ.Params() { + panic("BUG: mismatched number of arguments") + } + + c.varEdges = c.varEdges[:0] + c.varEdgeTypes = c.varEdgeTypes[:0] + c.constEdges = c.constEdges[:0] + for i := 0; i < len(args); i++ { + dst := succ.Param(i) + src := args[i] + + dstReg := c.VRegOf(dst) + srcDef := c.ssaValueDefinitions[src.ID()] + if srcDef.IsFromInstr() && srcDef.Instr.Constant() { + c.constEdges = append(c.constEdges, struct { + cInst *ssa.Instruction + dst regalloc.VReg + }{cInst: srcDef.Instr, dst: dstReg}) + } else { + srcReg := c.VRegOf(src) + // Even when the src=dst, insert the move so that we can keep such registers keep-alive. + c.varEdges = append(c.varEdges, [2]regalloc.VReg{srcReg, dstReg}) + c.varEdgeTypes = append(c.varEdgeTypes, src.Type()) + } + } + + // Check if there's an overlap among the dsts and srcs in varEdges. + c.vRegIDs = c.vRegIDs[:0] + for _, edge := range c.varEdges { + src := edge[0].ID() + if int(src) >= len(c.vRegSet) { + c.vRegSet = append(c.vRegSet, make([]bool, src+1)...) + } + c.vRegSet[src] = true + c.vRegIDs = append(c.vRegIDs, src) + } + separated := true + for _, edge := range c.varEdges { + dst := edge[1].ID() + if int(dst) >= len(c.vRegSet) { + c.vRegSet = append(c.vRegSet, make([]bool, dst+1)...) + } else { + if c.vRegSet[dst] { + separated = false + break + } + } + } + for _, id := range c.vRegIDs { + c.vRegSet[id] = false // reset for the next use. + } + + if separated { + // If there's no overlap, we can simply move the source to destination. + for i, edge := range c.varEdges { + src, dst := edge[0], edge[1] + c.mach.InsertMove(dst, src, c.varEdgeTypes[i]) + } + } else { + // Otherwise, we allocate a temporary registers and move the source to the temporary register, + // + // First move all of them to temporary registers. + c.tempRegs = c.tempRegs[:0] + for i, edge := range c.varEdges { + src := edge[0] + typ := c.varEdgeTypes[i] + temp := c.AllocateVReg(typ) + c.tempRegs = append(c.tempRegs, temp) + c.mach.InsertMove(temp, src, typ) + } + // Then move the temporary registers to the destination. + for i, edge := range c.varEdges { + temp := c.tempRegs[i] + dst := edge[1] + c.mach.InsertMove(dst, temp, c.varEdgeTypes[i]) + } + } + + // Finally, move the constants. + for _, edge := range c.constEdges { + cInst, dst := edge.cInst, edge.dst + c.mach.InsertLoadConstantBlockArg(cInst, dst) + } +} diff --git a/vendor/github.com/tetratelabs/wazero/internal/engine/wazevo/backend/executable_context.go b/vendor/github.com/tetratelabs/wazero/internal/engine/wazevo/backend/executable_context.go new file mode 100644 index 000000000..81c6a6b62 --- /dev/null +++ b/vendor/github.com/tetratelabs/wazero/internal/engine/wazevo/backend/executable_context.go @@ -0,0 +1,219 @@ +package backend + +import ( + "fmt" + "math" + + "github.com/tetratelabs/wazero/internal/engine/wazevo/ssa" + "github.com/tetratelabs/wazero/internal/engine/wazevo/wazevoapi" +) + +type ExecutableContext interface { + // StartLoweringFunction is called when the lowering of the given function is started. + // maximumBlockID is the maximum value of ssa.BasicBlockID existing in the function. + StartLoweringFunction(maximumBlockID ssa.BasicBlockID) + + // LinkAdjacentBlocks is called after finished lowering all blocks in order to create one single instruction list. + LinkAdjacentBlocks(prev, next ssa.BasicBlock) + + // StartBlock is called when the compilation of the given block is started. + // The order of this being called is the reverse post order of the ssa.BasicBlock(s) as we iterate with + // ssa.Builder BlockIteratorReversePostOrderBegin and BlockIteratorReversePostOrderEnd. + StartBlock(ssa.BasicBlock) + + // EndBlock is called when the compilation of the current block is finished. + EndBlock() + + // FlushPendingInstructions flushes the pending instructions to the buffer. + // This will be called after the lowering of each SSA Instruction. + FlushPendingInstructions() +} + +type ExecutableContextT[Instr any] struct { + CurrentSSABlk ssa.BasicBlock + + // InstrPool is the InstructionPool of instructions. + InstructionPool wazevoapi.Pool[Instr] + asNop func(*Instr) + setNext func(*Instr, *Instr) + setPrev func(*Instr, *Instr) + + // RootInstr is the root instruction of the executable. + RootInstr *Instr + labelPositionPool wazevoapi.Pool[LabelPosition[Instr]] + NextLabel Label + // LabelPositions maps a label to the instructions of the region which the label represents. + LabelPositions map[Label]*LabelPosition[Instr] + OrderedBlockLabels []*LabelPosition[Instr] + + // PerBlockHead and PerBlockEnd are the head and tail of the instruction list per currently-compiled ssa.BasicBlock. + PerBlockHead, PerBlockEnd *Instr + // PendingInstructions are the instructions which are not yet emitted into the instruction list. + PendingInstructions []*Instr + + // SsaBlockIDToLabels maps an SSA block ID to the label. + SsaBlockIDToLabels []Label +} + +func NewExecutableContextT[Instr any]( + resetInstruction func(*Instr), + setNext func(*Instr, *Instr), + setPrev func(*Instr, *Instr), + asNop func(*Instr), +) *ExecutableContextT[Instr] { + return &ExecutableContextT[Instr]{ + InstructionPool: wazevoapi.NewPool[Instr](resetInstruction), + asNop: asNop, + setNext: setNext, + setPrev: setPrev, + labelPositionPool: wazevoapi.NewPool[LabelPosition[Instr]](resetLabelPosition[Instr]), + LabelPositions: make(map[Label]*LabelPosition[Instr]), + NextLabel: LabelInvalid, + } +} + +func resetLabelPosition[T any](l *LabelPosition[T]) { + *l = LabelPosition[T]{} +} + +// StartLoweringFunction implements ExecutableContext. +func (e *ExecutableContextT[Instr]) StartLoweringFunction(max ssa.BasicBlockID) { + imax := int(max) + if len(e.SsaBlockIDToLabels) <= imax { + // Eagerly allocate labels for the blocks since the underlying slice will be used for the next iteration. + e.SsaBlockIDToLabels = append(e.SsaBlockIDToLabels, make([]Label, imax+1)...) + } +} + +func (e *ExecutableContextT[Instr]) StartBlock(blk ssa.BasicBlock) { + e.CurrentSSABlk = blk + + l := e.SsaBlockIDToLabels[e.CurrentSSABlk.ID()] + if l == LabelInvalid { + l = e.AllocateLabel() + e.SsaBlockIDToLabels[blk.ID()] = l + } + + end := e.allocateNop0() + e.PerBlockHead, e.PerBlockEnd = end, end + + labelPos, ok := e.LabelPositions[l] + if !ok { + labelPos = e.AllocateLabelPosition(l) + e.LabelPositions[l] = labelPos + } + e.OrderedBlockLabels = append(e.OrderedBlockLabels, labelPos) + labelPos.Begin, labelPos.End = end, end + labelPos.SB = blk +} + +// EndBlock implements ExecutableContext. +func (e *ExecutableContextT[T]) EndBlock() { + // Insert nop0 as the head of the block for convenience to simplify the logic of inserting instructions. + e.insertAtPerBlockHead(e.allocateNop0()) + + l := e.SsaBlockIDToLabels[e.CurrentSSABlk.ID()] + e.LabelPositions[l].Begin = e.PerBlockHead + + if e.CurrentSSABlk.EntryBlock() { + e.RootInstr = e.PerBlockHead + } +} + +func (e *ExecutableContextT[T]) insertAtPerBlockHead(i *T) { + if e.PerBlockHead == nil { + e.PerBlockHead = i + e.PerBlockEnd = i + return + } + e.setNext(i, e.PerBlockHead) + e.setPrev(e.PerBlockHead, i) + e.PerBlockHead = i +} + +// FlushPendingInstructions implements ExecutableContext. +func (e *ExecutableContextT[T]) FlushPendingInstructions() { + l := len(e.PendingInstructions) + if l == 0 { + return + } + for i := l - 1; i >= 0; i-- { // reverse because we lower instructions in reverse order. + e.insertAtPerBlockHead(e.PendingInstructions[i]) + } + e.PendingInstructions = e.PendingInstructions[:0] +} + +func (e *ExecutableContextT[T]) Reset() { + e.labelPositionPool.Reset() + e.InstructionPool.Reset() + for l := Label(0); l <= e.NextLabel; l++ { + delete(e.LabelPositions, l) + } + e.PendingInstructions = e.PendingInstructions[:0] + e.OrderedBlockLabels = e.OrderedBlockLabels[:0] + e.RootInstr = nil + e.SsaBlockIDToLabels = e.SsaBlockIDToLabels[:0] + e.PerBlockHead, e.PerBlockEnd = nil, nil + e.NextLabel = LabelInvalid +} + +// AllocateLabel allocates an unused label. +func (e *ExecutableContextT[T]) AllocateLabel() Label { + e.NextLabel++ + return e.NextLabel +} + +func (e *ExecutableContextT[T]) AllocateLabelPosition(la Label) *LabelPosition[T] { + l := e.labelPositionPool.Allocate() + l.L = la + return l +} + +func (e *ExecutableContextT[T]) GetOrAllocateSSABlockLabel(blk ssa.BasicBlock) Label { + if blk.ReturnBlock() { + return LabelReturn + } + l := e.SsaBlockIDToLabels[blk.ID()] + if l == LabelInvalid { + l = e.AllocateLabel() + e.SsaBlockIDToLabels[blk.ID()] = l + } + return l +} + +func (e *ExecutableContextT[T]) allocateNop0() *T { + i := e.InstructionPool.Allocate() + e.asNop(i) + return i +} + +// LinkAdjacentBlocks implements backend.Machine. +func (e *ExecutableContextT[T]) LinkAdjacentBlocks(prev, next ssa.BasicBlock) { + prevLabelPos := e.LabelPositions[e.GetOrAllocateSSABlockLabel(prev)] + nextLabelPos := e.LabelPositions[e.GetOrAllocateSSABlockLabel(next)] + e.setNext(prevLabelPos.End, nextLabelPos.Begin) +} + +// LabelPosition represents the regions of the generated code which the label represents. +type LabelPosition[Instr any] struct { + SB ssa.BasicBlock + L Label + Begin, End *Instr + BinaryOffset int64 +} + +// Label represents a position in the generated code which is either +// a real instruction or the constant InstructionPool (e.g. jump tables). +// +// This is exactly the same as the traditional "label" in assembly code. +type Label uint32 + +const ( + LabelInvalid Label = 0 + LabelReturn Label = math.MaxUint32 +) + +// String implements backend.Machine. +func (l Label) String() string { + return fmt.Sprintf("L%d", l) +} diff --git a/vendor/github.com/tetratelabs/wazero/internal/engine/wazevo/backend/go_call.go b/vendor/github.com/tetratelabs/wazero/internal/engine/wazevo/backend/go_call.go new file mode 100644 index 000000000..6fe6d7b3c --- /dev/null +++ b/vendor/github.com/tetratelabs/wazero/internal/engine/wazevo/backend/go_call.go @@ -0,0 +1,33 @@ +package backend + +import "github.com/tetratelabs/wazero/internal/engine/wazevo/ssa" + +// GoFunctionCallRequiredStackSize returns the size of the stack required for the Go function call. +// argBegin is the index of the first argument in the signature which is not either execution context or module context. +func GoFunctionCallRequiredStackSize(sig *ssa.Signature, argBegin int) (ret, retUnaligned int64) { + var paramNeededInBytes, resultNeededInBytes int64 + for _, p := range sig.Params[argBegin:] { + s := int64(p.Size()) + if s < 8 { + s = 8 // We use uint64 for all basic types, except SIMD v128. + } + paramNeededInBytes += s + } + for _, r := range sig.Results { + s := int64(r.Size()) + if s < 8 { + s = 8 // We use uint64 for all basic types, except SIMD v128. + } + resultNeededInBytes += s + } + + if paramNeededInBytes > resultNeededInBytes { + ret = paramNeededInBytes + } else { + ret = resultNeededInBytes + } + retUnaligned = ret + // Align to 16 bytes. + ret = (ret + 15) &^ 15 + return +} diff --git a/vendor/github.com/tetratelabs/wazero/internal/engine/wazevo/backend/isa/amd64/abi.go b/vendor/github.com/tetratelabs/wazero/internal/engine/wazevo/backend/isa/amd64/abi.go new file mode 100644 index 000000000..130f8c621 --- /dev/null +++ b/vendor/github.com/tetratelabs/wazero/internal/engine/wazevo/backend/isa/amd64/abi.go @@ -0,0 +1,186 @@ +package amd64 + +import ( + "github.com/tetratelabs/wazero/internal/engine/wazevo/backend" + "github.com/tetratelabs/wazero/internal/engine/wazevo/backend/regalloc" + "github.com/tetratelabs/wazero/internal/engine/wazevo/ssa" +) + +// For the details of the ABI, see: +// https://github.com/golang/go/blob/49d42128fd8594c172162961ead19ac95e247d24/src/cmd/compile/abi-internal.md#amd64-architecture + +var ( + intArgResultRegs = []regalloc.RealReg{rax, rbx, rcx, rdi, rsi, r8, r9, r10, r11} + floatArgResultRegs = []regalloc.RealReg{xmm0, xmm1, xmm2, xmm3, xmm4, xmm5, xmm6, xmm7} +) + +var regInfo = ®alloc.RegisterInfo{ + AllocatableRegisters: [regalloc.NumRegType][]regalloc.RealReg{ + regalloc.RegTypeInt: { + rax, rcx, rdx, rbx, rsi, rdi, r8, r9, r10, r11, r12, r13, r14, r15, + }, + regalloc.RegTypeFloat: { + xmm0, xmm1, xmm2, xmm3, xmm4, xmm5, xmm6, xmm7, xmm8, xmm9, xmm10, xmm11, xmm12, xmm13, xmm14, xmm15, + }, + }, + CalleeSavedRegisters: regalloc.NewRegSet( + rdx, r12, r13, r14, r15, + xmm8, xmm9, xmm10, xmm11, xmm12, xmm13, xmm14, xmm15, + ), + CallerSavedRegisters: regalloc.NewRegSet( + rax, rcx, rbx, rsi, rdi, r8, r9, r10, r11, + xmm0, xmm1, xmm2, xmm3, xmm4, xmm5, xmm6, xmm7, + ), + RealRegToVReg: []regalloc.VReg{ + rax: raxVReg, rcx: rcxVReg, rdx: rdxVReg, rbx: rbxVReg, rsp: rspVReg, rbp: rbpVReg, rsi: rsiVReg, rdi: rdiVReg, + r8: r8VReg, r9: r9VReg, r10: r10VReg, r11: r11VReg, r12: r12VReg, r13: r13VReg, r14: r14VReg, r15: r15VReg, + xmm0: xmm0VReg, xmm1: xmm1VReg, xmm2: xmm2VReg, xmm3: xmm3VReg, xmm4: xmm4VReg, xmm5: xmm5VReg, xmm6: xmm6VReg, + xmm7: xmm7VReg, xmm8: xmm8VReg, xmm9: xmm9VReg, xmm10: xmm10VReg, xmm11: xmm11VReg, xmm12: xmm12VReg, + xmm13: xmm13VReg, xmm14: xmm14VReg, xmm15: xmm15VReg, + }, + RealRegName: func(r regalloc.RealReg) string { return regNames[r] }, + RealRegType: func(r regalloc.RealReg) regalloc.RegType { + if r < xmm0 { + return regalloc.RegTypeInt + } + return regalloc.RegTypeFloat + }, +} + +// ArgsResultsRegs implements backend.Machine. +func (m *machine) ArgsResultsRegs() (argResultInts, argResultFloats []regalloc.RealReg) { + return intArgResultRegs, floatArgResultRegs +} + +// LowerParams implements backend.Machine. +func (m *machine) LowerParams(args []ssa.Value) { + a := m.currentABI + + for i, ssaArg := range args { + if !ssaArg.Valid() { + continue + } + reg := m.c.VRegOf(ssaArg) + arg := &a.Args[i] + if arg.Kind == backend.ABIArgKindReg { + m.InsertMove(reg, arg.Reg, arg.Type) + } else { + // + // (high address) + // +-----------------+ + // | ....... | + // | ret Y | + // | ....... | + // | ret 0 | + // | arg X | + // | ....... | + // | arg 1 | + // | arg 0 | + // | ReturnAddress | + // | Caller_RBP | + // +-----------------+ <-- RBP + // | ........... | + // | clobbered M | + // | ............ | + // | clobbered 0 | + // | spill slot N | + // | ........... | + // | spill slot 0 | + // RSP--> +-----------------+ + // (low address) + + // Load the value from the arg stack slot above the current RBP. + load := m.allocateInstr() + mem := newOperandMem(m.newAmodeImmRBPReg(uint32(arg.Offset + 16))) + switch arg.Type { + case ssa.TypeI32: + load.asMovzxRmR(extModeLQ, mem, reg) + case ssa.TypeI64: + load.asMov64MR(mem, reg) + case ssa.TypeF32: + load.asXmmUnaryRmR(sseOpcodeMovss, mem, reg) + case ssa.TypeF64: + load.asXmmUnaryRmR(sseOpcodeMovsd, mem, reg) + case ssa.TypeV128: + load.asXmmUnaryRmR(sseOpcodeMovdqu, mem, reg) + default: + panic("BUG") + } + m.insert(load) + } + } +} + +// LowerReturns implements backend.Machine. +func (m *machine) LowerReturns(rets []ssa.Value) { + // Load the XMM registers first as it might need a temporary register to inline + // constant return. + a := m.currentABI + for i, ret := range rets { + r := &a.Rets[i] + if !r.Type.IsInt() { + m.LowerReturn(ret, r) + } + } + // Then load the GPR registers. + for i, ret := range rets { + r := &a.Rets[i] + if r.Type.IsInt() { + m.LowerReturn(ret, r) + } + } +} + +func (m *machine) LowerReturn(ret ssa.Value, r *backend.ABIArg) { + reg := m.c.VRegOf(ret) + if def := m.c.ValueDefinition(ret); def.IsFromInstr() { + // Constant instructions are inlined. + if inst := def.Instr; inst.Constant() { + m.insertLoadConstant(inst, reg) + } + } + if r.Kind == backend.ABIArgKindReg { + m.InsertMove(r.Reg, reg, ret.Type()) + } else { + // + // (high address) + // +-----------------+ + // | ....... | + // | ret Y | + // | ....... | + // | ret 0 | + // | arg X | + // | ....... | + // | arg 1 | + // | arg 0 | + // | ReturnAddress | + // | Caller_RBP | + // +-----------------+ <-- RBP + // | ........... | + // | clobbered M | + // | ............ | + // | clobbered 0 | + // | spill slot N | + // | ........... | + // | spill slot 0 | + // RSP--> +-----------------+ + // (low address) + + // Store the value to the return stack slot above the current RBP. + store := m.allocateInstr() + mem := newOperandMem(m.newAmodeImmRBPReg(uint32(m.currentABI.ArgStackSize + 16 + r.Offset))) + switch r.Type { + case ssa.TypeI32: + store.asMovRM(reg, mem, 4) + case ssa.TypeI64: + store.asMovRM(reg, mem, 8) + case ssa.TypeF32: + store.asXmmMovRM(sseOpcodeMovss, reg, mem) + case ssa.TypeF64: + store.asXmmMovRM(sseOpcodeMovsd, reg, mem) + case ssa.TypeV128: + store.asXmmMovRM(sseOpcodeMovdqu, reg, mem) + } + m.insert(store) + } +} diff --git a/vendor/github.com/tetratelabs/wazero/internal/engine/wazevo/backend/isa/amd64/abi_entry_amd64.go b/vendor/github.com/tetratelabs/wazero/internal/engine/wazevo/backend/isa/amd64/abi_entry_amd64.go new file mode 100644 index 000000000..cbf1cfdc5 --- /dev/null +++ b/vendor/github.com/tetratelabs/wazero/internal/engine/wazevo/backend/isa/amd64/abi_entry_amd64.go @@ -0,0 +1,9 @@ +package amd64 + +// entrypoint enters the machine code generated by this backend which begins with the preamble generated by functionABI.EmitGoEntryPreamble below. +// This implements wazevo.entrypoint, and see the comments there for detail. +func entrypoint(preambleExecutable, functionExecutable *byte, executionContextPtr uintptr, moduleContextPtr *byte, paramResultPtr *uint64, goAllocatedStackSlicePtr uintptr) + +// afterGoFunctionCallEntrypoint enters the machine code after growing the stack. +// This implements wazevo.afterGoFunctionCallEntrypoint, and see the comments there for detail. +func afterGoFunctionCallEntrypoint(executable *byte, executionContextPtr uintptr, stackPointer, framePointer uintptr) diff --git a/vendor/github.com/tetratelabs/wazero/internal/engine/wazevo/backend/isa/amd64/abi_entry_amd64.s b/vendor/github.com/tetratelabs/wazero/internal/engine/wazevo/backend/isa/amd64/abi_entry_amd64.s new file mode 100644 index 000000000..e9cb131d1 --- /dev/null +++ b/vendor/github.com/tetratelabs/wazero/internal/engine/wazevo/backend/isa/amd64/abi_entry_amd64.s @@ -0,0 +1,29 @@ +#include "funcdata.h" +#include "textflag.h" + +// entrypoint(preambleExecutable, functionExecutable *byte, executionContextPtr uintptr, moduleContextPtr *byte, paramResultPtr *uint64, goAllocatedStackSlicePtr uintptr +TEXT ·entrypoint(SB), NOSPLIT|NOFRAME, $0-48 + MOVQ preambleExecutable+0(FP), R11 + MOVQ functionExectuable+8(FP), R14 + MOVQ executionContextPtr+16(FP), AX // First argument is passed in AX. + MOVQ moduleContextPtr+24(FP), BX // Second argument is passed in BX. + MOVQ paramResultSlicePtr+32(FP), R12 + MOVQ goAllocatedStackSlicePtr+40(FP), R13 + JMP R11 + +// afterGoFunctionCallEntrypoint(executable *byte, executionContextPtr uintptr, stackPointer, framePointer uintptr) +TEXT ·afterGoFunctionCallEntrypoint(SB), NOSPLIT|NOFRAME, $0-32 + MOVQ executable+0(FP), CX + MOVQ executionContextPtr+8(FP), AX // First argument is passed in AX. + + // Save the stack pointer and frame pointer. + MOVQ BP, 16(AX) // 16 == ExecutionContextOffsetOriginalFramePointer + MOVQ SP, 24(AX) // 24 == ExecutionContextOffsetOriginalStackPointer + + // Then set the stack pointer and frame pointer to the values we got from the Go runtime. + MOVQ framePointer+24(FP), BP + + // WARNING: do not update SP before BP, because the Go translates (FP) as (SP) + 8. + MOVQ stackPointer+16(FP), SP + + JMP CX diff --git a/vendor/github.com/tetratelabs/wazero/internal/engine/wazevo/backend/isa/amd64/abi_entry_preamble.go b/vendor/github.com/tetratelabs/wazero/internal/engine/wazevo/backend/isa/amd64/abi_entry_preamble.go new file mode 100644 index 000000000..882d06c06 --- /dev/null +++ b/vendor/github.com/tetratelabs/wazero/internal/engine/wazevo/backend/isa/amd64/abi_entry_preamble.go @@ -0,0 +1,248 @@ +package amd64 + +import ( + "github.com/tetratelabs/wazero/internal/engine/wazevo/backend" + "github.com/tetratelabs/wazero/internal/engine/wazevo/backend/regalloc" + "github.com/tetratelabs/wazero/internal/engine/wazevo/ssa" + "github.com/tetratelabs/wazero/internal/engine/wazevo/wazevoapi" +) + +var ( + executionContextPtrReg = raxVReg + + // Followings are callee saved registers. They can be used freely in the entry preamble + // since the preamble is called via Go assembly function which has stack-based ABI. + + // savedExecutionContextPtr also must be a callee-saved reg so that they can be used in the prologue and epilogue. + savedExecutionContextPtr = rdxVReg + // paramResultSlicePtr must match with entrypoint function in abi_entry_amd64.s. + paramResultSlicePtr = r12VReg + // goAllocatedStackPtr must match with entrypoint function in abi_entry_amd64.s. + goAllocatedStackPtr = r13VReg + // functionExecutable must match with entrypoint function in abi_entry_amd64.s. + functionExecutable = r14VReg + tmpIntReg = r15VReg + tmpXmmReg = xmm15VReg +) + +// CompileEntryPreamble implements backend.Machine. +func (m *machine) CompileEntryPreamble(sig *ssa.Signature) []byte { + root := m.compileEntryPreamble(sig) + m.encodeWithoutSSA(root) + buf := m.c.Buf() + return buf +} + +func (m *machine) compileEntryPreamble(sig *ssa.Signature) *instruction { + abi := backend.FunctionABI{} + abi.Init(sig, intArgResultRegs, floatArgResultRegs) + + root := m.allocateNop() + + //// ----------------------------------- prologue ----------------------------------- //// + + // First, we save executionContextPtrReg into a callee-saved register so that it can be used in epilogue as well. + // mov %executionContextPtrReg, %savedExecutionContextPtr + cur := m.move64(executionContextPtrReg, savedExecutionContextPtr, root) + + // Next is to save the original RBP and RSP into the execution context. + cur = m.saveOriginalRSPRBP(cur) + + // Now set the RSP to the Go-allocated stack pointer. + // mov %goAllocatedStackPtr, %rsp + cur = m.move64(goAllocatedStackPtr, rspVReg, cur) + + if stackSlotSize := abi.AlignedArgResultStackSlotSize(); stackSlotSize > 0 { + // Allocate stack slots for the arguments and return values. + // sub $stackSlotSize, %rsp + spDec := m.allocateInstr().asAluRmiR(aluRmiROpcodeSub, newOperandImm32(uint32(stackSlotSize)), rspVReg, true) + cur = linkInstr(cur, spDec) + } + + var offset uint32 + for i := range abi.Args { + if i < 2 { + // module context ptr and execution context ptr are passed in rax and rbx by the Go assembly function. + continue + } + arg := &abi.Args[i] + cur = m.goEntryPreamblePassArg(cur, paramResultSlicePtr, offset, arg) + if arg.Type == ssa.TypeV128 { + offset += 16 + } else { + offset += 8 + } + } + + // Zero out RBP so that the unwind/stack growth code can correctly detect the end of the stack. + zerosRbp := m.allocateInstr().asAluRmiR(aluRmiROpcodeXor, newOperandReg(rbpVReg), rbpVReg, true) + cur = linkInstr(cur, zerosRbp) + + // Now ready to call the real function. Note that at this point stack pointer is already set to the Go-allocated, + // which is aligned to 16 bytes. + call := m.allocateInstr().asCallIndirect(newOperandReg(functionExecutable), &abi) + cur = linkInstr(cur, call) + + //// ----------------------------------- epilogue ----------------------------------- //// + + // Read the results from regs and the stack, and set them correctly into the paramResultSlicePtr. + offset = 0 + for i := range abi.Rets { + r := &abi.Rets[i] + cur = m.goEntryPreamblePassResult(cur, paramResultSlicePtr, offset, r, uint32(abi.ArgStackSize)) + if r.Type == ssa.TypeV128 { + offset += 16 + } else { + offset += 8 + } + } + + // Finally, restore the original RBP and RSP. + cur = m.restoreOriginalRSPRBP(cur) + + ret := m.allocateInstr().asRet() + linkInstr(cur, ret) + return root +} + +// saveOriginalRSPRBP saves the original RSP and RBP into the execution context. +func (m *machine) saveOriginalRSPRBP(cur *instruction) *instruction { + // mov %rbp, wazevoapi.ExecutionContextOffsetOriginalFramePointer(%executionContextPtrReg) + // mov %rsp, wazevoapi.ExecutionContextOffsetOriginalStackPointer(%executionContextPtrReg) + cur = m.loadOrStore64AtExecutionCtx(executionContextPtrReg, wazevoapi.ExecutionContextOffsetOriginalFramePointer, rbpVReg, true, cur) + cur = m.loadOrStore64AtExecutionCtx(executionContextPtrReg, wazevoapi.ExecutionContextOffsetOriginalStackPointer, rspVReg, true, cur) + return cur +} + +// restoreOriginalRSPRBP restores the original RSP and RBP from the execution context. +func (m *machine) restoreOriginalRSPRBP(cur *instruction) *instruction { + // mov wazevoapi.ExecutionContextOffsetOriginalFramePointer(%executionContextPtrReg), %rbp + // mov wazevoapi.ExecutionContextOffsetOriginalStackPointer(%executionContextPtrReg), %rsp + cur = m.loadOrStore64AtExecutionCtx(savedExecutionContextPtr, wazevoapi.ExecutionContextOffsetOriginalFramePointer, rbpVReg, false, cur) + cur = m.loadOrStore64AtExecutionCtx(savedExecutionContextPtr, wazevoapi.ExecutionContextOffsetOriginalStackPointer, rspVReg, false, cur) + return cur +} + +func (m *machine) move64(src, dst regalloc.VReg, prev *instruction) *instruction { + mov := m.allocateInstr().asMovRR(src, dst, true) + return linkInstr(prev, mov) +} + +func (m *machine) loadOrStore64AtExecutionCtx(execCtx regalloc.VReg, offset wazevoapi.Offset, r regalloc.VReg, store bool, prev *instruction) *instruction { + mem := newOperandMem(m.newAmodeImmReg(offset.U32(), execCtx)) + instr := m.allocateInstr() + if store { + instr.asMovRM(r, mem, 8) + } else { + instr.asMov64MR(mem, r) + } + return linkInstr(prev, instr) +} + +// This is for debugging. +func (m *machine) linkUD2(cur *instruction) *instruction { //nolint + return linkInstr(cur, m.allocateInstr().asUD2()) +} + +func (m *machine) goEntryPreamblePassArg(cur *instruction, paramSlicePtr regalloc.VReg, offsetInParamSlice uint32, arg *backend.ABIArg) *instruction { + var dst regalloc.VReg + argTyp := arg.Type + if arg.Kind == backend.ABIArgKindStack { + // Caller saved registers ca + switch argTyp { + case ssa.TypeI32, ssa.TypeI64: + dst = tmpIntReg + case ssa.TypeF32, ssa.TypeF64, ssa.TypeV128: + dst = tmpXmmReg + default: + panic("BUG") + } + } else { + dst = arg.Reg + } + + load := m.allocateInstr() + a := newOperandMem(m.newAmodeImmReg(offsetInParamSlice, paramSlicePtr)) + switch arg.Type { + case ssa.TypeI32: + load.asMovzxRmR(extModeLQ, a, dst) + case ssa.TypeI64: + load.asMov64MR(a, dst) + case ssa.TypeF32: + load.asXmmUnaryRmR(sseOpcodeMovss, a, dst) + case ssa.TypeF64: + load.asXmmUnaryRmR(sseOpcodeMovsd, a, dst) + case ssa.TypeV128: + load.asXmmUnaryRmR(sseOpcodeMovdqu, a, dst) + } + + cur = linkInstr(cur, load) + if arg.Kind == backend.ABIArgKindStack { + // Store back to the stack. + store := m.allocateInstr() + a := newOperandMem(m.newAmodeImmReg(uint32(arg.Offset), rspVReg)) + switch arg.Type { + case ssa.TypeI32: + store.asMovRM(dst, a, 4) + case ssa.TypeI64: + store.asMovRM(dst, a, 8) + case ssa.TypeF32: + store.asXmmMovRM(sseOpcodeMovss, dst, a) + case ssa.TypeF64: + store.asXmmMovRM(sseOpcodeMovsd, dst, a) + case ssa.TypeV128: + store.asXmmMovRM(sseOpcodeMovdqu, dst, a) + } + cur = linkInstr(cur, store) + } + return cur +} + +func (m *machine) goEntryPreamblePassResult(cur *instruction, resultSlicePtr regalloc.VReg, offsetInResultSlice uint32, result *backend.ABIArg, resultStackSlotBeginOffset uint32) *instruction { + var r regalloc.VReg + if result.Kind == backend.ABIArgKindStack { + // Load the value to the temporary. + load := m.allocateInstr() + offset := resultStackSlotBeginOffset + uint32(result.Offset) + a := newOperandMem(m.newAmodeImmReg(offset, rspVReg)) + switch result.Type { + case ssa.TypeI32: + r = tmpIntReg + load.asMovzxRmR(extModeLQ, a, r) + case ssa.TypeI64: + r = tmpIntReg + load.asMov64MR(a, r) + case ssa.TypeF32: + r = tmpXmmReg + load.asXmmUnaryRmR(sseOpcodeMovss, a, r) + case ssa.TypeF64: + r = tmpXmmReg + load.asXmmUnaryRmR(sseOpcodeMovsd, a, r) + case ssa.TypeV128: + r = tmpXmmReg + load.asXmmUnaryRmR(sseOpcodeMovdqu, a, r) + default: + panic("BUG") + } + cur = linkInstr(cur, load) + } else { + r = result.Reg + } + + store := m.allocateInstr() + a := newOperandMem(m.newAmodeImmReg(offsetInResultSlice, resultSlicePtr)) + switch result.Type { + case ssa.TypeI32: + store.asMovRM(r, a, 4) + case ssa.TypeI64: + store.asMovRM(r, a, 8) + case ssa.TypeF32: + store.asXmmMovRM(sseOpcodeMovss, r, a) + case ssa.TypeF64: + store.asXmmMovRM(sseOpcodeMovsd, r, a) + case ssa.TypeV128: + store.asXmmMovRM(sseOpcodeMovdqu, r, a) + } + + return linkInstr(cur, store) +} diff --git a/vendor/github.com/tetratelabs/wazero/internal/engine/wazevo/backend/isa/amd64/abi_go_call.go b/vendor/github.com/tetratelabs/wazero/internal/engine/wazevo/backend/isa/amd64/abi_go_call.go new file mode 100644 index 000000000..751050aff --- /dev/null +++ b/vendor/github.com/tetratelabs/wazero/internal/engine/wazevo/backend/isa/amd64/abi_go_call.go @@ -0,0 +1,443 @@ +package amd64 + +import ( + "github.com/tetratelabs/wazero/internal/engine/wazevo/backend" + "github.com/tetratelabs/wazero/internal/engine/wazevo/backend/regalloc" + "github.com/tetratelabs/wazero/internal/engine/wazevo/ssa" + "github.com/tetratelabs/wazero/internal/engine/wazevo/wazevoapi" +) + +var calleeSavedVRegs = []regalloc.VReg{ + rdxVReg, r12VReg, r13VReg, r14VReg, r15VReg, + xmm8VReg, xmm9VReg, xmm10VReg, xmm11VReg, xmm12VReg, xmm13VReg, xmm14VReg, xmm15VReg, +} + +// CompileGoFunctionTrampoline implements backend.Machine. +func (m *machine) CompileGoFunctionTrampoline(exitCode wazevoapi.ExitCode, sig *ssa.Signature, needModuleContextPtr bool) []byte { + ectx := m.ectx + argBegin := 1 // Skips exec context by default. + if needModuleContextPtr { + argBegin++ + } + + abi := &backend.FunctionABI{} + abi.Init(sig, intArgResultRegs, floatArgResultRegs) + m.currentABI = abi + + cur := m.allocateNop() + ectx.RootInstr = cur + + // Execution context is always the first argument. + execCtrPtr := raxVReg + + // First we update RBP and RSP just like the normal prologue. + // + // (high address) (high address) + // RBP ----> +-----------------+ +-----------------+ + // | ....... | | ....... | + // | ret Y | | ret Y | + // | ....... | | ....... | + // | ret 0 | | ret 0 | + // | arg X | | arg X | + // | ....... | ====> | ....... | + // | arg 1 | | arg 1 | + // | arg 0 | | arg 0 | + // | Return Addr | | Return Addr | + // RSP ----> +-----------------+ | Caller_RBP | + // (low address) +-----------------+ <----- RSP, RBP + // + cur = m.setupRBPRSP(cur) + + goSliceSizeAligned, goSliceSizeAlignedUnaligned := backend.GoFunctionCallRequiredStackSize(sig, argBegin) + cur = m.insertStackBoundsCheck(goSliceSizeAligned+8 /* size of the Go slice */, cur) + + // Save the callee saved registers. + cur = m.saveRegistersInExecutionContext(cur, execCtrPtr, calleeSavedVRegs) + + if needModuleContextPtr { + moduleCtrPtr := rbxVReg // Module context is always the second argument. + mem := m.newAmodeImmReg( + wazevoapi.ExecutionContextOffsetGoFunctionCallCalleeModuleContextOpaque.U32(), + execCtrPtr) + store := m.allocateInstr().asMovRM(moduleCtrPtr, newOperandMem(mem), 8) + cur = linkInstr(cur, store) + } + + // Now let's advance the RSP to the stack slot for the arguments. + // + // (high address) (high address) + // +-----------------+ +-----------------+ + // | ....... | | ....... | + // | ret Y | | ret Y | + // | ....... | | ....... | + // | ret 0 | | ret 0 | + // | arg X | | arg X | + // | ....... | =======> | ....... | + // | arg 1 | | arg 1 | + // | arg 0 | | arg 0 | + // | Return Addr | | Return Addr | + // | Caller_RBP | | Caller_RBP | + // RBP,RSP --> +-----------------+ +-----------------+ <----- RBP + // (low address) | arg[N]/ret[M] | + // | .......... | + // | arg[1]/ret[1] | + // | arg[0]/ret[0] | + // +-----------------+ <----- RSP + // (low address) + // + // where the region of "arg[0]/ret[0] ... arg[N]/ret[M]" is the stack used by the Go functions, + // therefore will be accessed as the usual []uint64. So that's where we need to pass/receive + // the arguments/return values to/from Go function. + cur = m.addRSP(-int32(goSliceSizeAligned), cur) + + // Next, we need to store all the arguments to the stack in the typical Wasm stack style. + var offsetInGoSlice int32 + for i := range abi.Args[argBegin:] { + arg := &abi.Args[argBegin+i] + var v regalloc.VReg + if arg.Kind == backend.ABIArgKindReg { + v = arg.Reg + } else { + // We have saved callee saved registers, so we can use them. + if arg.Type.IsInt() { + v = r15VReg + } else { + v = xmm15VReg + } + mem := newOperandMem(m.newAmodeImmReg(uint32(arg.Offset+16 /* to skip caller_rbp and ret_addr */), rbpVReg)) + load := m.allocateInstr() + switch arg.Type { + case ssa.TypeI32: + load.asMovzxRmR(extModeLQ, mem, v) + case ssa.TypeI64: + load.asMov64MR(mem, v) + case ssa.TypeF32: + load.asXmmUnaryRmR(sseOpcodeMovss, mem, v) + case ssa.TypeF64: + load.asXmmUnaryRmR(sseOpcodeMovsd, mem, v) + case ssa.TypeV128: + load.asXmmUnaryRmR(sseOpcodeMovdqu, mem, v) + default: + panic("BUG") + } + cur = linkInstr(cur, load) + } + + store := m.allocateInstr() + mem := newOperandMem(m.newAmodeImmReg(uint32(offsetInGoSlice), rspVReg)) + switch arg.Type { + case ssa.TypeI32: + store.asMovRM(v, mem, 4) + offsetInGoSlice += 8 // always uint64 rep. + case ssa.TypeI64: + store.asMovRM(v, mem, 8) + offsetInGoSlice += 8 + case ssa.TypeF32: + store.asXmmMovRM(sseOpcodeMovss, v, mem) + offsetInGoSlice += 8 // always uint64 rep. + case ssa.TypeF64: + store.asXmmMovRM(sseOpcodeMovsd, v, mem) + offsetInGoSlice += 8 + case ssa.TypeV128: + store.asXmmMovRM(sseOpcodeMovdqu, v, mem) + offsetInGoSlice += 16 + default: + panic("BUG") + } + cur = linkInstr(cur, store) + } + + // Finally we push the size of the slice to the stack so the stack looks like: + // + // (high address) + // +-----------------+ + // | ....... | + // | ret Y | + // | ....... | + // | ret 0 | + // | arg X | + // | ....... | + // | arg 1 | + // | arg 0 | + // | Return Addr | + // | Caller_RBP | + // +-----------------+ <----- RBP + // | arg[N]/ret[M] | + // | .......... | + // | arg[1]/ret[1] | + // | arg[0]/ret[0] | + // | slice size | + // +-----------------+ <----- RSP + // (low address) + // + // push $sliceSize + cur = linkInstr(cur, m.allocateInstr().asPush64(newOperandImm32(uint32(goSliceSizeAlignedUnaligned)))) + + // Load the exitCode to the register. + exitCodeReg := r12VReg // Callee saved which is already saved. + cur = linkInstr(cur, m.allocateInstr().asImm(exitCodeReg, uint64(exitCode), false)) + + saveRsp, saveRbp, setExitCode := m.allocateExitInstructions(execCtrPtr, exitCodeReg) + cur = linkInstr(cur, setExitCode) + cur = linkInstr(cur, saveRsp) + cur = linkInstr(cur, saveRbp) + + // Ready to exit the execution. + cur = m.storeReturnAddressAndExit(cur, execCtrPtr) + + // We don't need the slice size anymore, so pop it. + cur = m.addRSP(8, cur) + + // Ready to set up the results. + offsetInGoSlice = 0 + // To avoid overwriting with the execution context pointer by the result, we need to track the offset, + // and defer the restoration of the result to the end of this function. + var argOverlapWithExecCtxOffset int32 = -1 + for i := range abi.Rets { + r := &abi.Rets[i] + var v regalloc.VReg + isRegResult := r.Kind == backend.ABIArgKindReg + if isRegResult { + v = r.Reg + if v.RealReg() == execCtrPtr.RealReg() { + argOverlapWithExecCtxOffset = offsetInGoSlice + offsetInGoSlice += 8 // always uint64 rep. + continue + } + } else { + if r.Type.IsInt() { + v = r15VReg + } else { + v = xmm15VReg + } + } + + load := m.allocateInstr() + mem := newOperandMem(m.newAmodeImmReg(uint32(offsetInGoSlice), rspVReg)) + switch r.Type { + case ssa.TypeI32: + load.asMovzxRmR(extModeLQ, mem, v) + offsetInGoSlice += 8 // always uint64 rep. + case ssa.TypeI64: + load.asMov64MR(mem, v) + offsetInGoSlice += 8 + case ssa.TypeF32: + load.asXmmUnaryRmR(sseOpcodeMovss, mem, v) + offsetInGoSlice += 8 // always uint64 rep. + case ssa.TypeF64: + load.asXmmUnaryRmR(sseOpcodeMovsd, mem, v) + offsetInGoSlice += 8 + case ssa.TypeV128: + load.asXmmUnaryRmR(sseOpcodeMovdqu, mem, v) + offsetInGoSlice += 16 + default: + panic("BUG") + } + cur = linkInstr(cur, load) + + if !isRegResult { + // We need to store it back to the result slot above rbp. + store := m.allocateInstr() + mem := newOperandMem(m.newAmodeImmReg(uint32(abi.ArgStackSize+r.Offset+16 /* to skip caller_rbp and ret_addr */), rbpVReg)) + switch r.Type { + case ssa.TypeI32: + store.asMovRM(v, mem, 4) + case ssa.TypeI64: + store.asMovRM(v, mem, 8) + case ssa.TypeF32: + store.asXmmMovRM(sseOpcodeMovss, v, mem) + case ssa.TypeF64: + store.asXmmMovRM(sseOpcodeMovsd, v, mem) + case ssa.TypeV128: + store.asXmmMovRM(sseOpcodeMovdqu, v, mem) + default: + panic("BUG") + } + cur = linkInstr(cur, store) + } + } + + // Before return, we need to restore the callee saved registers. + cur = m.restoreRegistersInExecutionContext(cur, execCtrPtr, calleeSavedVRegs) + + if argOverlapWithExecCtxOffset >= 0 { + // At this point execCtt is not used anymore, so we can finally store the + // result to the register which overlaps with the execution context pointer. + mem := newOperandMem(m.newAmodeImmReg(uint32(argOverlapWithExecCtxOffset), rspVReg)) + load := m.allocateInstr().asMov64MR(mem, execCtrPtr) + cur = linkInstr(cur, load) + } + + // Finally ready to return. + cur = m.revertRBPRSP(cur) + linkInstr(cur, m.allocateInstr().asRet()) + + m.encodeWithoutSSA(ectx.RootInstr) + return m.c.Buf() +} + +func (m *machine) saveRegistersInExecutionContext(cur *instruction, execCtx regalloc.VReg, regs []regalloc.VReg) *instruction { + offset := wazevoapi.ExecutionContextOffsetSavedRegistersBegin.I64() + for _, v := range regs { + store := m.allocateInstr() + mem := newOperandMem(m.newAmodeImmReg(uint32(offset), execCtx)) + switch v.RegType() { + case regalloc.RegTypeInt: + store.asMovRM(v, mem, 8) + case regalloc.RegTypeFloat: + store.asXmmMovRM(sseOpcodeMovdqu, v, mem) + default: + panic("BUG") + } + cur = linkInstr(cur, store) + offset += 16 // See execution context struct. Each register is 16 bytes-aligned unconditionally. + } + return cur +} + +func (m *machine) restoreRegistersInExecutionContext(cur *instruction, execCtx regalloc.VReg, regs []regalloc.VReg) *instruction { + offset := wazevoapi.ExecutionContextOffsetSavedRegistersBegin.I64() + for _, v := range regs { + load := m.allocateInstr() + mem := newOperandMem(m.newAmodeImmReg(uint32(offset), execCtx)) + switch v.RegType() { + case regalloc.RegTypeInt: + load.asMov64MR(mem, v) + case regalloc.RegTypeFloat: + load.asXmmUnaryRmR(sseOpcodeMovdqu, mem, v) + default: + panic("BUG") + } + cur = linkInstr(cur, load) + offset += 16 // See execution context struct. Each register is 16 bytes-aligned unconditionally. + } + return cur +} + +func (m *machine) storeReturnAddressAndExit(cur *instruction, execCtx regalloc.VReg) *instruction { + readRip := m.allocateInstr() + cur = linkInstr(cur, readRip) + + ripReg := r12VReg // Callee saved which is already saved. + saveRip := m.allocateInstr().asMovRM( + ripReg, + newOperandMem(m.newAmodeImmReg(wazevoapi.ExecutionContextOffsetGoCallReturnAddress.U32(), execCtx)), + 8, + ) + cur = linkInstr(cur, saveRip) + + exit := m.allocateExitSeq(execCtx) + cur = linkInstr(cur, exit) + + nop, l := m.allocateBrTarget() + cur = linkInstr(cur, nop) + readRip.asLEA(newOperandLabel(l), ripReg) + return cur +} + +// saveRequiredRegs is the set of registers that must be saved/restored during growing stack when there's insufficient +// stack space left. Basically this is the all allocatable registers except for RSP and RBP, and RAX which contains the +// execution context pointer. ExecCtx pointer is always the first argument so we don't need to save it. +var stackGrowSaveVRegs = []regalloc.VReg{ + rdxVReg, r12VReg, r13VReg, r14VReg, r15VReg, + rcxVReg, rbxVReg, rsiVReg, rdiVReg, r8VReg, r9VReg, r10VReg, r11VReg, + xmm8VReg, xmm9VReg, xmm10VReg, xmm11VReg, xmm12VReg, xmm13VReg, xmm14VReg, xmm15VReg, + xmm0VReg, xmm1VReg, xmm2VReg, xmm3VReg, xmm4VReg, xmm5VReg, xmm6VReg, xmm7VReg, +} + +// CompileStackGrowCallSequence implements backend.Machine. +func (m *machine) CompileStackGrowCallSequence() []byte { + ectx := m.ectx + + cur := m.allocateNop() + ectx.RootInstr = cur + + cur = m.setupRBPRSP(cur) + + // Execution context is always the first argument. + execCtrPtr := raxVReg + + // Save the callee saved and argument registers. + cur = m.saveRegistersInExecutionContext(cur, execCtrPtr, stackGrowSaveVRegs) + + // Load the exitCode to the register. + exitCodeReg := r12VReg // Already saved. + cur = linkInstr(cur, m.allocateInstr().asImm(exitCodeReg, uint64(wazevoapi.ExitCodeGrowStack), false)) + + saveRsp, saveRbp, setExitCode := m.allocateExitInstructions(execCtrPtr, exitCodeReg) + cur = linkInstr(cur, setExitCode) + cur = linkInstr(cur, saveRsp) + cur = linkInstr(cur, saveRbp) + + // Ready to exit the execution. + cur = m.storeReturnAddressAndExit(cur, execCtrPtr) + + // After the exit, restore the saved registers. + cur = m.restoreRegistersInExecutionContext(cur, execCtrPtr, stackGrowSaveVRegs) + + // Finally ready to return. + cur = m.revertRBPRSP(cur) + linkInstr(cur, m.allocateInstr().asRet()) + + m.encodeWithoutSSA(ectx.RootInstr) + return m.c.Buf() +} + +// insertStackBoundsCheck will insert the instructions after `cur` to check the +// stack bounds, and if there's no sufficient spaces required for the function, +// exit the execution and try growing it in Go world. +func (m *machine) insertStackBoundsCheck(requiredStackSize int64, cur *instruction) *instruction { + // add $requiredStackSize, %rsp ;; Temporarily update the sp. + // cmp ExecutionContextOffsetStackBottomPtr(%rax), %rsp ;; Compare the stack bottom and the sp. + // ja .ok + // sub $requiredStackSize, %rsp ;; Reverse the temporary update. + // pushq r15 ;; save the temporary. + // mov $requiredStackSize, %r15 + // mov %15, ExecutionContextOffsetStackGrowRequiredSize(%rax) ;; Set the required size in the execution context. + // popq r15 ;; restore the temporary. + // callq *ExecutionContextOffsetStackGrowCallTrampolineAddress(%rax) ;; Call the Go function to grow the stack. + // jmp .cont + // .ok: + // sub $requiredStackSize, %rsp ;; Reverse the temporary update. + // .cont: + cur = m.addRSP(-int32(requiredStackSize), cur) + cur = linkInstr(cur, m.allocateInstr().asCmpRmiR(true, + newOperandMem(m.newAmodeImmReg(wazevoapi.ExecutionContextOffsetStackBottomPtr.U32(), raxVReg)), + rspVReg, true)) + + ja := m.allocateInstr() + cur = linkInstr(cur, ja) + + cur = m.addRSP(int32(requiredStackSize), cur) + + // Save the temporary. + + cur = linkInstr(cur, m.allocateInstr().asPush64(newOperandReg(r15VReg))) + // Load the required size to the temporary. + cur = linkInstr(cur, m.allocateInstr().asImm(r15VReg, uint64(requiredStackSize), true)) + // Set the required size in the execution context. + cur = linkInstr(cur, m.allocateInstr().asMovRM(r15VReg, + newOperandMem(m.newAmodeImmReg(wazevoapi.ExecutionContextOffsetStackGrowRequiredSize.U32(), raxVReg)), 8)) + // Restore the temporary. + cur = linkInstr(cur, m.allocateInstr().asPop64(r15VReg)) + // Call the Go function to grow the stack. + cur = linkInstr(cur, m.allocateInstr().asCallIndirect(newOperandMem(m.newAmodeImmReg( + wazevoapi.ExecutionContextOffsetStackGrowCallTrampolineAddress.U32(), raxVReg)), nil)) + // Jump to the continuation. + jmpToCont := m.allocateInstr() + cur = linkInstr(cur, jmpToCont) + + // .ok: + okInstr, ok := m.allocateBrTarget() + cur = linkInstr(cur, okInstr) + ja.asJmpIf(condNBE, newOperandLabel(ok)) + // On the ok path, we only need to reverse the temporary update. + cur = m.addRSP(int32(requiredStackSize), cur) + + // .cont: + contInstr, cont := m.allocateBrTarget() + cur = linkInstr(cur, contInstr) + jmpToCont.asJmp(newOperandLabel(cont)) + + return cur +} diff --git a/vendor/github.com/tetratelabs/wazero/internal/engine/wazevo/backend/isa/amd64/cond.go b/vendor/github.com/tetratelabs/wazero/internal/engine/wazevo/backend/isa/amd64/cond.go new file mode 100644 index 000000000..75cbeab75 --- /dev/null +++ b/vendor/github.com/tetratelabs/wazero/internal/engine/wazevo/backend/isa/amd64/cond.go @@ -0,0 +1,168 @@ +package amd64 + +import ( + "fmt" + + "github.com/tetratelabs/wazero/internal/engine/wazevo/ssa" +) + +type cond byte + +const ( + // condO represents (overflow) condition. + condO cond = iota + // condNO represents (no overflow) condition. + condNO + // condB represents (< unsigned) condition. + condB + // condNB represents (>= unsigned) condition. + condNB + // condZ represents (zero) condition. + condZ + // condNZ represents (not-zero) condition. + condNZ + // condBE represents (<= unsigned) condition. + condBE + // condNBE represents (> unsigned) condition. + condNBE + // condS represents (negative) condition. + condS + // condNS represents (not-negative) condition. + condNS + // condP represents (parity) condition. + condP + // condNP represents (not parity) condition. + condNP + // condL represents (< signed) condition. + condL + // condNL represents (>= signed) condition. + condNL + // condLE represents (<= signed) condition. + condLE + // condNLE represents (> signed) condition. + condNLE + + condInvalid +) + +func (c cond) String() string { + switch c { + case condO: + return "o" + case condNO: + return "no" + case condB: + return "b" + case condNB: + return "nb" + case condZ: + return "z" + case condNZ: + return "nz" + case condBE: + return "be" + case condNBE: + return "nbe" + case condS: + return "s" + case condNS: + return "ns" + case condL: + return "l" + case condNL: + return "nl" + case condLE: + return "le" + case condNLE: + return "nle" + case condP: + return "p" + case condNP: + return "np" + default: + panic("unreachable") + } +} + +func condFromSSAIntCmpCond(origin ssa.IntegerCmpCond) cond { + switch origin { + case ssa.IntegerCmpCondEqual: + return condZ + case ssa.IntegerCmpCondNotEqual: + return condNZ + case ssa.IntegerCmpCondSignedLessThan: + return condL + case ssa.IntegerCmpCondSignedGreaterThanOrEqual: + return condNL + case ssa.IntegerCmpCondSignedGreaterThan: + return condNLE + case ssa.IntegerCmpCondSignedLessThanOrEqual: + return condLE + case ssa.IntegerCmpCondUnsignedLessThan: + return condB + case ssa.IntegerCmpCondUnsignedGreaterThanOrEqual: + return condNB + case ssa.IntegerCmpCondUnsignedGreaterThan: + return condNBE + case ssa.IntegerCmpCondUnsignedLessThanOrEqual: + return condBE + default: + panic("unreachable") + } +} + +func condFromSSAFloatCmpCond(origin ssa.FloatCmpCond) cond { + switch origin { + case ssa.FloatCmpCondGreaterThanOrEqual: + return condNB + case ssa.FloatCmpCondGreaterThan: + return condNBE + case ssa.FloatCmpCondEqual, ssa.FloatCmpCondNotEqual, ssa.FloatCmpCondLessThan, ssa.FloatCmpCondLessThanOrEqual: + panic(fmt.Sprintf("cond %s must be treated as a special case", origin)) + default: + panic("unreachable") + } +} + +func (c cond) encoding() byte { + return byte(c) +} + +func (c cond) invert() cond { + switch c { + case condO: + return condNO + case condNO: + return condO + case condB: + return condNB + case condNB: + return condB + case condZ: + return condNZ + case condNZ: + return condZ + case condBE: + return condNBE + case condNBE: + return condBE + case condS: + return condNS + case condNS: + return condS + case condP: + return condNP + case condNP: + return condP + case condL: + return condNL + case condNL: + return condL + case condLE: + return condNLE + case condNLE: + return condLE + default: + panic("unreachable") + } +} diff --git a/vendor/github.com/tetratelabs/wazero/internal/engine/wazevo/backend/isa/amd64/ext.go b/vendor/github.com/tetratelabs/wazero/internal/engine/wazevo/backend/isa/amd64/ext.go new file mode 100644 index 000000000..5e731e822 --- /dev/null +++ b/vendor/github.com/tetratelabs/wazero/internal/engine/wazevo/backend/isa/amd64/ext.go @@ -0,0 +1,35 @@ +package amd64 + +// extMode represents the mode of extension in movzx/movsx. +type extMode byte + +const ( + // extModeBL represents Byte -> Longword. + extModeBL extMode = iota + // extModeBQ represents Byte -> Quadword. + extModeBQ + // extModeWL represents Word -> Longword. + extModeWL + // extModeWQ represents Word -> Quadword. + extModeWQ + // extModeLQ represents Longword -> Quadword. + extModeLQ +) + +// String implements fmt.Stringer. +func (e extMode) String() string { + switch e { + case extModeBL: + return "bl" + case extModeBQ: + return "bq" + case extModeWL: + return "wl" + case extModeWQ: + return "wq" + case extModeLQ: + return "lq" + default: + panic("BUG: invalid ext mode") + } +} diff --git a/vendor/github.com/tetratelabs/wazero/internal/engine/wazevo/backend/isa/amd64/instr.go b/vendor/github.com/tetratelabs/wazero/internal/engine/wazevo/backend/isa/amd64/instr.go new file mode 100644 index 000000000..d27e79c0e --- /dev/null +++ b/vendor/github.com/tetratelabs/wazero/internal/engine/wazevo/backend/isa/amd64/instr.go @@ -0,0 +1,2472 @@ +package amd64 + +import ( + "fmt" + + "github.com/tetratelabs/wazero/internal/engine/wazevo/backend" + "github.com/tetratelabs/wazero/internal/engine/wazevo/backend/regalloc" + "github.com/tetratelabs/wazero/internal/engine/wazevo/ssa" +) + +type instruction struct { + prev, next *instruction + op1, op2 operand + u1, u2 uint64 + b1 bool + addedBeforeRegAlloc bool + kind instructionKind +} + +// Next implements regalloc.Instr. +func (i *instruction) Next() regalloc.Instr { + return i.next +} + +// Prev implements regalloc.Instr. +func (i *instruction) Prev() regalloc.Instr { + return i.prev +} + +// IsCall implements regalloc.Instr. +func (i *instruction) IsCall() bool { return i.kind == call } + +// IsIndirectCall implements regalloc.Instr. +func (i *instruction) IsIndirectCall() bool { return i.kind == callIndirect } + +// IsReturn implements regalloc.Instr. +func (i *instruction) IsReturn() bool { return i.kind == ret } + +// AddedBeforeRegAlloc implements regalloc.Instr. +func (i *instruction) AddedBeforeRegAlloc() bool { return i.addedBeforeRegAlloc } + +// String implements regalloc.Instr. +func (i *instruction) String() string { + switch i.kind { + case nop0: + return "nop" + case sourceOffsetInfo: + return fmt.Sprintf("source_offset_info %d", i.u1) + case ret: + return "ret" + case imm: + if i.b1 { + return fmt.Sprintf("movabsq $%d, %s", int64(i.u1), i.op2.format(true)) + } else { + return fmt.Sprintf("movl $%d, %s", int32(i.u1), i.op2.format(false)) + } + case aluRmiR: + return fmt.Sprintf("%s %s, %s", aluRmiROpcode(i.u1), i.op1.format(i.b1), i.op2.format(i.b1)) + case movRR: + if i.b1 { + return fmt.Sprintf("movq %s, %s", i.op1.format(true), i.op2.format(true)) + } else { + return fmt.Sprintf("movl %s, %s", i.op1.format(false), i.op2.format(false)) + } + case xmmRmR: + return fmt.Sprintf("%s %s, %s", sseOpcode(i.u1), i.op1.format(false), i.op2.format(false)) + case gprToXmm: + return fmt.Sprintf("%s %s, %s", sseOpcode(i.u1), i.op1.format(i.b1), i.op2.format(i.b1)) + case xmmUnaryRmR: + return fmt.Sprintf("%s %s, %s", sseOpcode(i.u1), i.op1.format(false), i.op2.format(false)) + case xmmUnaryRmRImm: + return fmt.Sprintf("%s $%d, %s, %s", sseOpcode(i.u1), roundingMode(i.u2), i.op1.format(false), i.op2.format(false)) + case unaryRmR: + var suffix string + if i.b1 { + suffix = "q" + } else { + suffix = "l" + } + return fmt.Sprintf("%s%s %s, %s", unaryRmROpcode(i.u1), suffix, i.op1.format(i.b1), i.op2.format(i.b1)) + case not: + var op string + if i.b1 { + op = "notq" + } else { + op = "notl" + } + return fmt.Sprintf("%s %s", op, i.op1.format(i.b1)) + case neg: + var op string + if i.b1 { + op = "negq" + } else { + op = "negl" + } + return fmt.Sprintf("%s %s", op, i.op1.format(i.b1)) + case div: + var prefix string + var op string + if i.b1 { + op = "divq" + } else { + op = "divl" + } + if i.u1 != 0 { + prefix = "i" + } + return fmt.Sprintf("%s%s %s", prefix, op, i.op1.format(i.b1)) + case mulHi: + signed, _64 := i.u1 != 0, i.b1 + var op string + switch { + case signed && _64: + op = "imulq" + case !signed && _64: + op = "mulq" + case signed && !_64: + op = "imull" + case !signed && !_64: + op = "mull" + } + return fmt.Sprintf("%s %s", op, i.op1.format(i.b1)) + case signExtendData: + var op string + if i.b1 { + op = "cqo" + } else { + op = "cdq" + } + return op + case movzxRmR: + return fmt.Sprintf("movzx.%s %s, %s", extMode(i.u1), i.op1.format(true), i.op2.format(true)) + case mov64MR: + return fmt.Sprintf("movq %s, %s", i.op1.format(true), i.op2.format(true)) + case lea: + return fmt.Sprintf("lea %s, %s", i.op1.format(true), i.op2.format(true)) + case movsxRmR: + return fmt.Sprintf("movsx.%s %s, %s", extMode(i.u1), i.op1.format(true), i.op2.format(true)) + case movRM: + var suffix string + switch i.u1 { + case 1: + suffix = "b" + case 2: + suffix = "w" + case 4: + suffix = "l" + case 8: + suffix = "q" + } + return fmt.Sprintf("mov.%s %s, %s", suffix, i.op1.format(true), i.op2.format(true)) + case shiftR: + var suffix string + if i.b1 { + suffix = "q" + } else { + suffix = "l" + } + return fmt.Sprintf("%s%s %s, %s", shiftROp(i.u1), suffix, i.op1.format(false), i.op2.format(i.b1)) + case xmmRmiReg: + return fmt.Sprintf("%s %s, %s", sseOpcode(i.u1), i.op1.format(true), i.op2.format(true)) + case cmpRmiR: + var op, suffix string + if i.u1 != 0 { + op = "cmp" + } else { + op = "test" + } + if i.b1 { + suffix = "q" + } else { + suffix = "l" + } + if op == "test" && i.op1.kind == operandKindMem { + // Print consistently with AT&T syntax. + return fmt.Sprintf("%s%s %s, %s", op, suffix, i.op2.format(i.b1), i.op1.format(i.b1)) + } + return fmt.Sprintf("%s%s %s, %s", op, suffix, i.op1.format(i.b1), i.op2.format(i.b1)) + case setcc: + return fmt.Sprintf("set%s %s", cond(i.u1), i.op2.format(true)) + case cmove: + var suffix string + if i.b1 { + suffix = "q" + } else { + suffix = "l" + } + return fmt.Sprintf("cmov%s%s %s, %s", cond(i.u1), suffix, i.op1.format(i.b1), i.op2.format(i.b1)) + case push64: + return fmt.Sprintf("pushq %s", i.op1.format(true)) + case pop64: + return fmt.Sprintf("popq %s", i.op1.format(true)) + case xmmMovRM: + return fmt.Sprintf("%s %s, %s", sseOpcode(i.u1), i.op1.format(true), i.op2.format(true)) + case xmmLoadConst: + panic("TODO") + case xmmToGpr: + return fmt.Sprintf("%s %s, %s", sseOpcode(i.u1), i.op1.format(i.b1), i.op2.format(i.b1)) + case cvtUint64ToFloatSeq: + panic("TODO") + case cvtFloatToSintSeq: + panic("TODO") + case cvtFloatToUintSeq: + panic("TODO") + case xmmMinMaxSeq: + panic("TODO") + case xmmCmpRmR: + return fmt.Sprintf("%s %s, %s", sseOpcode(i.u1), i.op1.format(false), i.op2.format(false)) + case xmmRmRImm: + op := sseOpcode(i.u1) + r1, r2 := i.op1.format(op == sseOpcodePextrq || op == sseOpcodePinsrq), + i.op2.format(op == sseOpcodePextrq || op == sseOpcodePinsrq) + return fmt.Sprintf("%s $%d, %s, %s", op, i.u2, r1, r2) + case jmp: + return fmt.Sprintf("jmp %s", i.op1.format(true)) + case jmpIf: + return fmt.Sprintf("j%s %s", cond(i.u1), i.op1.format(true)) + case jmpTableIsland: + return fmt.Sprintf("jump_table_island: jmp_table_index=%d", i.u1) + case exitSequence: + return fmt.Sprintf("exit_sequence %s", i.op1.format(true)) + case ud2: + return "ud2" + case call: + return fmt.Sprintf("call %s", ssa.FuncRef(i.u1)) + case callIndirect: + return fmt.Sprintf("callq *%s", i.op1.format(true)) + case xchg: + var suffix string + switch i.u1 { + case 1: + suffix = "b" + case 2: + suffix = "w" + case 4: + suffix = "l" + case 8: + suffix = "q" + } + return fmt.Sprintf("xchg.%s %s, %s", suffix, i.op1.format(true), i.op2.format(true)) + case zeros: + return fmt.Sprintf("xor %s, %s", i.op2.format(true), i.op2.format(true)) + case fcvtToSintSequence: + execCtx, src, tmpGp, tmpGp2, tmpXmm, src64, dst64, sat := i.fcvtToSintSequenceData() + return fmt.Sprintf( + "fcvtToSintSequence execCtx=%s, src=%s, tmpGp=%s, tmpGp2=%s, tmpXmm=%s, src64=%v, dst64=%v, sat=%v", + formatVRegSized(execCtx, true), + formatVRegSized(src, true), + formatVRegSized(tmpGp, true), + formatVRegSized(tmpGp2, true), + formatVRegSized(tmpXmm, true), src64, dst64, sat) + case fcvtToUintSequence: + execCtx, src, tmpGp, tmpGp2, tmpXmm, tmpXmm2, src64, dst64, sat := i.fcvtToUintSequenceData() + return fmt.Sprintf( + "fcvtToUintSequence execCtx=%s, src=%s, tmpGp=%s, tmpGp2=%s, tmpXmm=%s, tmpXmm2=%s, src64=%v, dst64=%v, sat=%v", + formatVRegSized(execCtx, true), + formatVRegSized(src, true), + formatVRegSized(tmpGp, true), + formatVRegSized(tmpGp2, true), + formatVRegSized(tmpXmm, true), + formatVRegSized(tmpXmm2, true), src64, dst64, sat) + case idivRemSequence: + execCtx, divisor, tmpGp, isDiv, signed, _64 := i.idivRemSequenceData() + return fmt.Sprintf("idivRemSequence execCtx=%s, divisor=%s, tmpGp=%s, isDiv=%v, signed=%v, _64=%v", + formatVRegSized(execCtx, true), formatVRegSized(divisor, _64), formatVRegSized(tmpGp, _64), isDiv, signed, _64) + case defineUninitializedReg: + return fmt.Sprintf("defineUninitializedReg %s", i.op2.format(true)) + case xmmCMov: + return fmt.Sprintf("xmmcmov%s %s, %s", cond(i.u1), i.op1.format(true), i.op2.format(true)) + case blendvpd: + return fmt.Sprintf("blendvpd %s, %s, %%xmm0", i.op1.format(false), i.op2.format(false)) + case mfence: + return "mfence" + case lockcmpxchg: + var suffix string + switch i.u1 { + case 1: + suffix = "b" + case 2: + suffix = "w" + case 4: + suffix = "l" + case 8: + suffix = "q" + } + return fmt.Sprintf("lock cmpxchg.%s %s, %s", suffix, i.op1.format(true), i.op2.format(true)) + case lockxadd: + var suffix string + switch i.u1 { + case 1: + suffix = "b" + case 2: + suffix = "w" + case 4: + suffix = "l" + case 8: + suffix = "q" + } + return fmt.Sprintf("lock xadd.%s %s, %s", suffix, i.op1.format(true), i.op2.format(true)) + + case nopUseReg: + return fmt.Sprintf("nop_use_reg %s", i.op1.format(true)) + + default: + panic(fmt.Sprintf("BUG: %d", int(i.kind))) + } +} + +// Defs implements regalloc.Instr. +func (i *instruction) Defs(regs *[]regalloc.VReg) []regalloc.VReg { + *regs = (*regs)[:0] + switch dk := defKinds[i.kind]; dk { + case defKindNone: + case defKindOp2: + *regs = append(*regs, i.op2.reg()) + case defKindCall: + _, _, retIntRealRegs, retFloatRealRegs, _ := backend.ABIInfoFromUint64(i.u2) + for i := byte(0); i < retIntRealRegs; i++ { + *regs = append(*regs, regInfo.RealRegToVReg[intArgResultRegs[i]]) + } + for i := byte(0); i < retFloatRealRegs; i++ { + *regs = append(*regs, regInfo.RealRegToVReg[floatArgResultRegs[i]]) + } + case defKindDivRem: + _, _, _, isDiv, _, _ := i.idivRemSequenceData() + if isDiv { + *regs = append(*regs, raxVReg) + } else { + *regs = append(*regs, rdxVReg) + } + default: + panic(fmt.Sprintf("BUG: invalid defKind \"%s\" for %s", dk, i)) + } + return *regs +} + +// Uses implements regalloc.Instr. +func (i *instruction) Uses(regs *[]regalloc.VReg) []regalloc.VReg { + *regs = (*regs)[:0] + switch uk := useKinds[i.kind]; uk { + case useKindNone: + case useKindOp1Op2Reg, useKindOp1RegOp2: + opAny, opReg := &i.op1, &i.op2 + if uk == useKindOp1RegOp2 { + opAny, opReg = opReg, opAny + } + // The destination operand (op2) can be only reg, + // the source operand (op1) can be imm32, reg or mem. + switch opAny.kind { + case operandKindReg: + *regs = append(*regs, opAny.reg()) + case operandKindMem: + opAny.addressMode().uses(regs) + case operandKindImm32: + default: + panic(fmt.Sprintf("BUG: invalid operand: %s", i)) + } + if opReg.kind != operandKindReg { + panic(fmt.Sprintf("BUG: invalid operand: %s", i)) + } + *regs = append(*regs, opReg.reg()) + case useKindOp1: + op := i.op1 + switch op.kind { + case operandKindReg: + *regs = append(*regs, op.reg()) + case operandKindMem: + op.addressMode().uses(regs) + case operandKindImm32, operandKindLabel: + default: + panic(fmt.Sprintf("BUG: invalid operand: %s", i)) + } + case useKindCallInd: + op := i.op1 + switch op.kind { + case operandKindReg: + *regs = append(*regs, op.reg()) + case operandKindMem: + op.addressMode().uses(regs) + default: + panic(fmt.Sprintf("BUG: invalid operand: %s", i)) + } + fallthrough + case useKindCall: + argIntRealRegs, argFloatRealRegs, _, _, _ := backend.ABIInfoFromUint64(i.u2) + for i := byte(0); i < argIntRealRegs; i++ { + *regs = append(*regs, regInfo.RealRegToVReg[intArgResultRegs[i]]) + } + for i := byte(0); i < argFloatRealRegs; i++ { + *regs = append(*regs, regInfo.RealRegToVReg[floatArgResultRegs[i]]) + } + case useKindFcvtToSintSequence: + execCtx, src, tmpGp, tmpGp2, tmpXmm, _, _, _ := i.fcvtToSintSequenceData() + *regs = append(*regs, execCtx, src, tmpGp, tmpGp2, tmpXmm) + case useKindFcvtToUintSequence: + execCtx, src, tmpGp, tmpGp2, tmpXmm, tmpXmm2, _, _, _ := i.fcvtToUintSequenceData() + *regs = append(*regs, execCtx, src, tmpGp, tmpGp2, tmpXmm, tmpXmm2) + case useKindDivRem: + execCtx, divisor, tmpGp, _, _, _ := i.idivRemSequenceData() + // idiv uses rax and rdx as implicit operands. + *regs = append(*regs, raxVReg, rdxVReg, execCtx, divisor, tmpGp) + case useKindBlendvpd: + *regs = append(*regs, xmm0VReg) + + opAny, opReg := &i.op1, &i.op2 + switch opAny.kind { + case operandKindReg: + *regs = append(*regs, opAny.reg()) + case operandKindMem: + opAny.addressMode().uses(regs) + default: + panic(fmt.Sprintf("BUG: invalid operand: %s", i)) + } + if opReg.kind != operandKindReg { + panic(fmt.Sprintf("BUG: invalid operand: %s", i)) + } + *regs = append(*regs, opReg.reg()) + + case useKindRaxOp1RegOp2: + opReg, opAny := &i.op1, &i.op2 + *regs = append(*regs, raxVReg, opReg.reg()) + switch opAny.kind { + case operandKindReg: + *regs = append(*regs, opAny.reg()) + case operandKindMem: + opAny.addressMode().uses(regs) + default: + panic(fmt.Sprintf("BUG: invalid operand: %s", i)) + } + if opReg.kind != operandKindReg { + panic(fmt.Sprintf("BUG: invalid operand: %s", i)) + } + + default: + panic(fmt.Sprintf("BUG: invalid useKind %s for %s", uk, i)) + } + return *regs +} + +// AssignUse implements regalloc.Instr. +func (i *instruction) AssignUse(index int, v regalloc.VReg) { + switch uk := useKinds[i.kind]; uk { + case useKindNone: + case useKindCallInd: + if index != 0 { + panic("BUG") + } + op := &i.op1 + switch op.kind { + case operandKindReg: + op.setReg(v) + case operandKindMem: + op.addressMode().assignUses(index, v) + default: + panic("BUG") + } + case useKindOp1Op2Reg, useKindOp1RegOp2: + op, opMustBeReg := &i.op1, &i.op2 + if uk == useKindOp1RegOp2 { + op, opMustBeReg = opMustBeReg, op + } + switch op.kind { + case operandKindReg: + if index == 0 { + op.setReg(v) + } else if index == 1 { + opMustBeReg.setReg(v) + } else { + panic("BUG") + } + case operandKindMem: + nregs := op.addressMode().nregs() + if index < nregs { + op.addressMode().assignUses(index, v) + } else if index == nregs { + opMustBeReg.setReg(v) + } else { + panic("BUG") + } + case operandKindImm32: + if index == 0 { + opMustBeReg.setReg(v) + } else { + panic("BUG") + } + default: + panic(fmt.Sprintf("BUG: invalid operand pair: %s", i)) + } + case useKindOp1: + op := &i.op1 + switch op.kind { + case operandKindReg: + if index != 0 { + panic("BUG") + } + op.setReg(v) + case operandKindMem: + op.addressMode().assignUses(index, v) + default: + panic(fmt.Sprintf("BUG: invalid operand: %s", i)) + } + case useKindFcvtToSintSequence: + switch index { + case 0: + i.op1.addressMode().base = v + case 1: + i.op1.addressMode().index = v + case 2: + i.op2.addressMode().base = v + case 3: + i.op2.addressMode().index = v + case 4: + i.u1 = uint64(v) + default: + panic("BUG") + } + case useKindFcvtToUintSequence: + switch index { + case 0: + i.op1.addressMode().base = v + case 1: + i.op1.addressMode().index = v + case 2: + i.op2.addressMode().base = v + case 3: + i.op2.addressMode().index = v + case 4: + i.u1 = uint64(v) + case 5: + i.u2 = uint64(v) + default: + panic("BUG") + } + case useKindDivRem: + switch index { + case 0: + if v != raxVReg { + panic("BUG") + } + case 1: + if v != rdxVReg { + panic("BUG") + } + case 2: + i.op1.setReg(v) + case 3: + i.op2.setReg(v) + case 4: + i.u1 = uint64(v) + default: + panic("BUG") + } + case useKindBlendvpd: + op, opMustBeReg := &i.op1, &i.op2 + if index == 0 { + if v.RealReg() != xmm0 { + panic("BUG") + } + } else { + switch op.kind { + case operandKindReg: + switch index { + case 1: + op.setReg(v) + case 2: + opMustBeReg.setReg(v) + default: + panic("BUG") + } + case operandKindMem: + nregs := op.addressMode().nregs() + index-- + if index < nregs { + op.addressMode().assignUses(index, v) + } else if index == nregs { + opMustBeReg.setReg(v) + } else { + panic("BUG") + } + default: + panic(fmt.Sprintf("BUG: invalid operand pair: %s", i)) + } + } + + case useKindRaxOp1RegOp2: + switch index { + case 0: + if v.RealReg() != rax { + panic("BUG") + } + case 1: + i.op1.setReg(v) + default: + op := &i.op2 + switch op.kind { + case operandKindReg: + switch index { + case 1: + op.setReg(v) + case 2: + op.setReg(v) + default: + panic("BUG") + } + case operandKindMem: + nregs := op.addressMode().nregs() + index -= 2 + if index < nregs { + op.addressMode().assignUses(index, v) + } else if index == nregs { + op.setReg(v) + } else { + panic("BUG") + } + default: + panic(fmt.Sprintf("BUG: invalid operand pair: %s", i)) + } + } + default: + panic(fmt.Sprintf("BUG: invalid useKind %s for %s", uk, i)) + } +} + +// AssignDef implements regalloc.Instr. +func (i *instruction) AssignDef(reg regalloc.VReg) { + switch dk := defKinds[i.kind]; dk { + case defKindNone: + case defKindOp2: + i.op2.setReg(reg) + default: + panic(fmt.Sprintf("BUG: invalid defKind \"%s\" for %s", dk, i)) + } +} + +// IsCopy implements regalloc.Instr. +func (i *instruction) IsCopy() bool { + k := i.kind + if k == movRR { + return true + } + if k == xmmUnaryRmR { + if i.op1.kind == operandKindReg { + sse := sseOpcode(i.u1) + return sse == sseOpcodeMovss || sse == sseOpcodeMovsd || sse == sseOpcodeMovdqu + } + } + return false +} + +func resetInstruction(i *instruction) { + *i = instruction{} +} + +func setNext(i *instruction, next *instruction) { + i.next = next +} + +func setPrev(i *instruction, prev *instruction) { + i.prev = prev +} + +func asNop(i *instruction) { + i.kind = nop0 +} + +func (i *instruction) asNop0WithLabel(label backend.Label) *instruction { //nolint + i.kind = nop0 + i.u1 = uint64(label) + return i +} + +func (i *instruction) nop0Label() backend.Label { + return backend.Label(i.u1) +} + +type instructionKind byte + +const ( + nop0 instructionKind = iota + 1 + + // Integer arithmetic/bit-twiddling: (add sub and or xor mul, etc.) (32 64) (reg addr imm) reg + aluRmiR + + // Instructions on GPR that only read src and defines dst (dst is not modified): bsr, etc. + unaryRmR + + // Bitwise not + not + + // Integer negation + neg + + // Integer quotient and remainder: (div idiv) $rax $rdx (reg addr) + div + + // The high bits (RDX) of a (un)signed multiply: RDX:RAX := RAX * rhs. + mulHi + + // Do a sign-extend based on the sign of the value in rax into rdx: (cwd cdq cqo) + // or al into ah: (cbw) + signExtendData + + // Constant materialization: (imm32 imm64) reg. + // Either: movl $imm32, %reg32 or movabsq $imm64, %reg64. + imm + + // GPR to GPR move: mov (64 32) reg reg. + movRR + + // movzxRmR is zero-extended loads or move (R to R), except for 64 bits: movz (bl bq wl wq lq) addr reg. + // Note that the lq variant doesn't really exist since the default zero-extend rule makes it + // unnecessary. For that case we emit the equivalent "movl AM, reg32". + movzxRmR + + // mov64MR is a plain 64-bit integer load, since movzxRmR can't represent that. + mov64MR + + // Loads the memory address of addr into dst. + lea + + // Sign-extended loads and moves: movs (bl bq wl wq lq) addr reg. + movsxRmR + + // Integer stores: mov (b w l q) reg addr. + movRM + + // Arithmetic shifts: (shl shr sar) (b w l q) imm reg. + shiftR + + // Arithmetic SIMD shifts. + xmmRmiReg + + // Integer comparisons/tests: cmp or test (b w l q) (reg addr imm) reg. + cmpRmiR + + // Materializes the requested condition code in the destination reg. + setcc + + // Integer conditional move. + // Overwrites the destination register. + cmove + + // pushq (reg addr imm) + push64 + + // popq reg + pop64 + + // XMM (scalar or vector) binary op: (add sub and or xor mul adc? sbb?) (32 64) (reg addr) reg + xmmRmR + + // XMM (scalar or vector) unary op: mov between XMM registers (32 64) (reg addr) reg. + // + // This differs from xmmRmR in that the dst register of xmmUnaryRmR is not used in the + // computation of the instruction dst value and so does not have to be a previously valid + // value. This is characteristic of mov instructions. + xmmUnaryRmR + + // XMM (scalar or vector) unary op with immediate: roundss, roundsd, etc. + // + // This differs from XMM_RM_R_IMM in that the dst register of + // XmmUnaryRmRImm is not used in the computation of the instruction dst + // value and so does not have to be a previously valid value. + xmmUnaryRmRImm + + // XMM (scalar or vector) unary op (from xmm to mem): stores, movd, movq + xmmMovRM + + // XMM (vector) unary op (to move a constant value into an xmm register): movups + xmmLoadConst + + // XMM (scalar) unary op (from xmm to integer reg): movd, movq, cvtts{s,d}2si + xmmToGpr + + // XMM (scalar) unary op (from integer to float reg): movd, movq, cvtsi2s{s,d} + gprToXmm + + // Converts an unsigned int64 to a float32/float64. + cvtUint64ToFloatSeq + + // Converts a scalar xmm to a signed int32/int64. + cvtFloatToSintSeq + + // Converts a scalar xmm to an unsigned int32/int64. + cvtFloatToUintSeq + + // A sequence to compute min/max with the proper NaN semantics for xmm registers. + xmmMinMaxSeq + + // Float comparisons/tests: cmp (b w l q) (reg addr imm) reg. + xmmCmpRmR + + // A binary XMM instruction with an 8-bit immediate: e.g. cmp (ps pd) imm (reg addr) reg + xmmRmRImm + + // Direct call: call simm32. + // Note that the offset is the relative to the *current RIP*, which points to the first byte of the next instruction. + call + + // Indirect call: callq (reg mem). + callIndirect + + // Return. + ret + + // Jump: jmp (reg, mem, imm32 or label) + jmp + + // Jump conditionally: jcond cond label. + jmpIf + + // jmpTableIsland is to emit the jump table. + jmpTableIsland + + // exitSequence exits the execution and go back to the Go world. + exitSequence + + // An instruction that will always trigger the illegal instruction exception. + ud2 + + // xchg is described in https://www.felixcloutier.com/x86/xchg. + // This instruction uses two operands, where one of them can be a memory address, and swaps their values. + // If the dst is a memory address, the execution is atomic. + xchg + + // lockcmpxchg is the cmpxchg instruction https://www.felixcloutier.com/x86/cmpxchg with a lock prefix. + lockcmpxchg + + // zeros puts zeros into the destination register. This is implemented as xor reg, reg for + // either integer or XMM registers. The reason why we have this instruction instead of using aluRmiR + // is that it requires the already-defined registers. From reg alloc's perspective, this defines + // the destination register and takes no inputs. + zeros + + // sourceOffsetInfo is a dummy instruction to emit source offset info. + // The existence of this instruction does not affect the execution. + sourceOffsetInfo + + // defineUninitializedReg is a no-op instruction that defines a register without a defining instruction. + defineUninitializedReg + + // fcvtToSintSequence is a sequence of instructions to convert a float to a signed integer. + fcvtToSintSequence + + // fcvtToUintSequence is a sequence of instructions to convert a float to an unsigned integer. + fcvtToUintSequence + + // xmmCMov is a conditional move instruction for XMM registers. Lowered after register allocation. + xmmCMov + + // idivRemSequence is a sequence of instructions to compute both the quotient and remainder of a division. + idivRemSequence + + // blendvpd is https://www.felixcloutier.com/x86/blendvpd. + blendvpd + + // mfence is https://www.felixcloutier.com/x86/mfence + mfence + + // lockxadd is xadd https://www.felixcloutier.com/x86/xadd with a lock prefix. + lockxadd + + // nopUseReg is a meta instruction that uses one register and does nothing. + nopUseReg + + instrMax +) + +func (i *instruction) asMFence() *instruction { + i.kind = mfence + return i +} + +func (i *instruction) asNopUseReg(r regalloc.VReg) *instruction { + i.kind = nopUseReg + i.op1 = newOperandReg(r) + return i +} + +func (i *instruction) asIdivRemSequence(execCtx, divisor, tmpGp regalloc.VReg, isDiv, signed, _64 bool) *instruction { + i.kind = idivRemSequence + i.op1 = newOperandReg(execCtx) + i.op2 = newOperandReg(divisor) + i.u1 = uint64(tmpGp) + if isDiv { + i.u2 |= 1 + } + if signed { + i.u2 |= 2 + } + if _64 { + i.u2 |= 4 + } + return i +} + +func (i *instruction) idivRemSequenceData() ( + execCtx, divisor, tmpGp regalloc.VReg, isDiv, signed, _64 bool, +) { + if i.kind != idivRemSequence { + panic("BUG") + } + return i.op1.reg(), i.op2.reg(), regalloc.VReg(i.u1), i.u2&1 != 0, i.u2&2 != 0, i.u2&4 != 0 +} + +func (i *instruction) asXmmCMov(cc cond, x operand, rd regalloc.VReg, size byte) *instruction { + i.kind = xmmCMov + i.op1 = x + i.op2 = newOperandReg(rd) + i.u1 = uint64(cc) + i.u2 = uint64(size) + return i +} + +func (i *instruction) asDefineUninitializedReg(r regalloc.VReg) *instruction { + i.kind = defineUninitializedReg + i.op2 = newOperandReg(r) + return i +} + +func (m *machine) allocateFcvtToUintSequence( + execCtx, src, tmpGp, tmpGp2, tmpXmm, tmpXmm2 regalloc.VReg, + src64, dst64, sat bool, +) *instruction { + i := m.allocateInstr() + i.kind = fcvtToUintSequence + op1a := m.amodePool.Allocate() + op2a := m.amodePool.Allocate() + i.op1 = newOperandMem(op1a) + i.op2 = newOperandMem(op2a) + if src64 { + op1a.imm32 = 1 + } else { + op1a.imm32 = 0 + } + if dst64 { + op1a.imm32 |= 2 + } + if sat { + op1a.imm32 |= 4 + } + + op1a.base = execCtx + op1a.index = src + op2a.base = tmpGp + op2a.index = tmpGp2 + i.u1 = uint64(tmpXmm) + i.u2 = uint64(tmpXmm2) + return i +} + +func (i *instruction) fcvtToUintSequenceData() ( + execCtx, src, tmpGp, tmpGp2, tmpXmm, tmpXmm2 regalloc.VReg, src64, dst64, sat bool, +) { + if i.kind != fcvtToUintSequence { + panic("BUG") + } + op1a := i.op1.addressMode() + op2a := i.op2.addressMode() + return op1a.base, op1a.index, op2a.base, op2a.index, regalloc.VReg(i.u1), regalloc.VReg(i.u2), + op1a.imm32&1 != 0, op1a.imm32&2 != 0, op1a.imm32&4 != 0 +} + +func (m *machine) allocateFcvtToSintSequence( + execCtx, src, tmpGp, tmpGp2, tmpXmm regalloc.VReg, + src64, dst64, sat bool, +) *instruction { + i := m.allocateInstr() + i.kind = fcvtToSintSequence + op1a := m.amodePool.Allocate() + op2a := m.amodePool.Allocate() + i.op1 = newOperandMem(op1a) + i.op2 = newOperandMem(op2a) + op1a.base = execCtx + op1a.index = src + op2a.base = tmpGp + op2a.index = tmpGp2 + i.u1 = uint64(tmpXmm) + if src64 { + i.u2 = 1 + } else { + i.u2 = 0 + } + if dst64 { + i.u2 |= 2 + } + if sat { + i.u2 |= 4 + } + return i +} + +func (i *instruction) fcvtToSintSequenceData() ( + execCtx, src, tmpGp, tmpGp2, tmpXmm regalloc.VReg, src64, dst64, sat bool, +) { + if i.kind != fcvtToSintSequence { + panic("BUG") + } + op1a := i.op1.addressMode() + op2a := i.op2.addressMode() + return op1a.base, op1a.index, op2a.base, op2a.index, regalloc.VReg(i.u1), + i.u2&1 != 0, i.u2&2 != 0, i.u2&4 != 0 +} + +func (k instructionKind) String() string { + switch k { + case nop0: + return "nop" + case ret: + return "ret" + case imm: + return "imm" + case aluRmiR: + return "aluRmiR" + case movRR: + return "movRR" + case xmmRmR: + return "xmmRmR" + case gprToXmm: + return "gprToXmm" + case xmmUnaryRmR: + return "xmmUnaryRmR" + case xmmUnaryRmRImm: + return "xmmUnaryRmRImm" + case unaryRmR: + return "unaryRmR" + case not: + return "not" + case neg: + return "neg" + case div: + return "div" + case mulHi: + return "mulHi" + case signExtendData: + return "signExtendData" + case movzxRmR: + return "movzxRmR" + case mov64MR: + return "mov64MR" + case lea: + return "lea" + case movsxRmR: + return "movsxRmR" + case movRM: + return "movRM" + case shiftR: + return "shiftR" + case xmmRmiReg: + return "xmmRmiReg" + case cmpRmiR: + return "cmpRmiR" + case setcc: + return "setcc" + case cmove: + return "cmove" + case push64: + return "push64" + case pop64: + return "pop64" + case xmmMovRM: + return "xmmMovRM" + case xmmLoadConst: + return "xmmLoadConst" + case xmmToGpr: + return "xmmToGpr" + case cvtUint64ToFloatSeq: + return "cvtUint64ToFloatSeq" + case cvtFloatToSintSeq: + return "cvtFloatToSintSeq" + case cvtFloatToUintSeq: + return "cvtFloatToUintSeq" + case xmmMinMaxSeq: + return "xmmMinMaxSeq" + case xmmCmpRmR: + return "xmmCmpRmR" + case xmmRmRImm: + return "xmmRmRImm" + case jmpIf: + return "jmpIf" + case jmp: + return "jmp" + case jmpTableIsland: + return "jmpTableIsland" + case exitSequence: + return "exit_sequence" + case ud2: + return "ud2" + case xchg: + return "xchg" + case zeros: + return "zeros" + case fcvtToSintSequence: + return "fcvtToSintSequence" + case fcvtToUintSequence: + return "fcvtToUintSequence" + case xmmCMov: + return "xmmCMov" + case idivRemSequence: + return "idivRemSequence" + case mfence: + return "mfence" + case lockcmpxchg: + return "lockcmpxchg" + case lockxadd: + return "lockxadd" + default: + panic("BUG") + } +} + +type aluRmiROpcode byte + +const ( + aluRmiROpcodeAdd aluRmiROpcode = iota + 1 + aluRmiROpcodeSub + aluRmiROpcodeAnd + aluRmiROpcodeOr + aluRmiROpcodeXor + aluRmiROpcodeMul +) + +func (a aluRmiROpcode) String() string { + switch a { + case aluRmiROpcodeAdd: + return "add" + case aluRmiROpcodeSub: + return "sub" + case aluRmiROpcodeAnd: + return "and" + case aluRmiROpcodeOr: + return "or" + case aluRmiROpcodeXor: + return "xor" + case aluRmiROpcodeMul: + return "imul" + default: + panic("BUG") + } +} + +func (i *instruction) asJmpIf(cond cond, target operand) *instruction { + i.kind = jmpIf + i.u1 = uint64(cond) + i.op1 = target + return i +} + +// asJmpTableSequence is used to emit the jump table. +// targetSliceIndex is the index of the target slice in machine.jmpTableTargets. +func (i *instruction) asJmpTableSequence(targetSliceIndex int, targetCount int) *instruction { + i.kind = jmpTableIsland + i.u1 = uint64(targetSliceIndex) + i.u2 = uint64(targetCount) + return i +} + +func (i *instruction) asJmp(target operand) *instruction { + i.kind = jmp + i.op1 = target + return i +} + +func (i *instruction) jmpLabel() backend.Label { + switch i.kind { + case jmp, jmpIf, lea, xmmUnaryRmR: + return i.op1.label() + default: + panic("BUG") + } +} + +func (i *instruction) asLEA(target operand, rd regalloc.VReg) *instruction { + i.kind = lea + i.op1 = target + i.op2 = newOperandReg(rd) + return i +} + +func (i *instruction) asCall(ref ssa.FuncRef, abi *backend.FunctionABI) *instruction { + i.kind = call + i.u1 = uint64(ref) + if abi != nil { + i.u2 = abi.ABIInfoAsUint64() + } + return i +} + +func (i *instruction) asCallIndirect(ptr operand, abi *backend.FunctionABI) *instruction { + if ptr.kind != operandKindReg && ptr.kind != operandKindMem { + panic("BUG") + } + i.kind = callIndirect + i.op1 = ptr + if abi != nil { + i.u2 = abi.ABIInfoAsUint64() + } + return i +} + +func (i *instruction) asRet() *instruction { + i.kind = ret + return i +} + +func (i *instruction) asImm(dst regalloc.VReg, value uint64, _64 bool) *instruction { + i.kind = imm + i.op2 = newOperandReg(dst) + i.u1 = value + i.b1 = _64 + return i +} + +func (i *instruction) asAluRmiR(op aluRmiROpcode, rm operand, rd regalloc.VReg, _64 bool) *instruction { + if rm.kind != operandKindReg && rm.kind != operandKindMem && rm.kind != operandKindImm32 { + panic("BUG") + } + i.kind = aluRmiR + i.op1 = rm + i.op2 = newOperandReg(rd) + i.u1 = uint64(op) + i.b1 = _64 + return i +} + +func (i *instruction) asZeros(dst regalloc.VReg) *instruction { + i.kind = zeros + i.op2 = newOperandReg(dst) + return i +} + +func (i *instruction) asBlendvpd(rm operand, rd regalloc.VReg) *instruction { + if rm.kind != operandKindReg && rm.kind != operandKindMem { + panic("BUG") + } + i.kind = blendvpd + i.op1 = rm + i.op2 = newOperandReg(rd) + return i +} + +func (i *instruction) asXmmRmR(op sseOpcode, rm operand, rd regalloc.VReg) *instruction { + if rm.kind != operandKindReg && rm.kind != operandKindMem { + panic("BUG") + } + i.kind = xmmRmR + i.op1 = rm + i.op2 = newOperandReg(rd) + i.u1 = uint64(op) + return i +} + +func (i *instruction) asXmmRmRImm(op sseOpcode, imm uint8, rm operand, rd regalloc.VReg) *instruction { + if rm.kind != operandKindReg && rm.kind != operandKindMem { + panic("BUG") + } + i.kind = xmmRmRImm + i.op1 = rm + i.op2 = newOperandReg(rd) + i.u1 = uint64(op) + i.u2 = uint64(imm) + return i +} + +func (i *instruction) asGprToXmm(op sseOpcode, rm operand, rd regalloc.VReg, _64 bool) *instruction { + if rm.kind != operandKindReg && rm.kind != operandKindMem { + panic("BUG") + } + i.kind = gprToXmm + i.op1 = rm + i.op2 = newOperandReg(rd) + i.u1 = uint64(op) + i.b1 = _64 + return i +} + +func (i *instruction) asEmitSourceOffsetInfo(l ssa.SourceOffset) *instruction { + i.kind = sourceOffsetInfo + i.u1 = uint64(l) + return i +} + +func (i *instruction) sourceOffsetInfo() ssa.SourceOffset { + return ssa.SourceOffset(i.u1) +} + +func (i *instruction) asXmmToGpr(op sseOpcode, rm, rd regalloc.VReg, _64 bool) *instruction { + i.kind = xmmToGpr + i.op1 = newOperandReg(rm) + i.op2 = newOperandReg(rd) + i.u1 = uint64(op) + i.b1 = _64 + return i +} + +func (i *instruction) asMovRM(rm regalloc.VReg, rd operand, size byte) *instruction { + if rd.kind != operandKindMem { + panic("BUG") + } + i.kind = movRM + i.op1 = newOperandReg(rm) + i.op2 = rd + i.u1 = uint64(size) + return i +} + +func (i *instruction) asMovsxRmR(ext extMode, src operand, rd regalloc.VReg) *instruction { + if src.kind != operandKindReg && src.kind != operandKindMem { + panic("BUG") + } + i.kind = movsxRmR + i.op1 = src + i.op2 = newOperandReg(rd) + i.u1 = uint64(ext) + return i +} + +func (i *instruction) asMovzxRmR(ext extMode, src operand, rd regalloc.VReg) *instruction { + if src.kind != operandKindReg && src.kind != operandKindMem { + panic("BUG") + } + i.kind = movzxRmR + i.op1 = src + i.op2 = newOperandReg(rd) + i.u1 = uint64(ext) + return i +} + +func (i *instruction) asSignExtendData(_64 bool) *instruction { + i.kind = signExtendData + i.b1 = _64 + return i +} + +func (i *instruction) asUD2() *instruction { + i.kind = ud2 + return i +} + +func (i *instruction) asDiv(rn operand, signed bool, _64 bool) *instruction { + i.kind = div + i.op1 = rn + i.b1 = _64 + if signed { + i.u1 = 1 + } + return i +} + +func (i *instruction) asMov64MR(rm operand, rd regalloc.VReg) *instruction { + if rm.kind != operandKindMem { + panic("BUG") + } + i.kind = mov64MR + i.op1 = rm + i.op2 = newOperandReg(rd) + return i +} + +func (i *instruction) asMovRR(rm, rd regalloc.VReg, _64 bool) *instruction { + i.kind = movRR + i.op1 = newOperandReg(rm) + i.op2 = newOperandReg(rd) + i.b1 = _64 + return i +} + +func (i *instruction) asNot(rm operand, _64 bool) *instruction { + if rm.kind != operandKindReg && rm.kind != operandKindMem { + panic("BUG") + } + i.kind = not + i.op1 = rm + i.b1 = _64 + return i +} + +func (i *instruction) asNeg(rm operand, _64 bool) *instruction { + if rm.kind != operandKindReg && rm.kind != operandKindMem { + panic("BUG") + } + i.kind = neg + i.op1 = rm + i.b1 = _64 + return i +} + +func (i *instruction) asMulHi(rm operand, signed, _64 bool) *instruction { + if rm.kind != operandKindReg && (rm.kind != operandKindMem) { + panic("BUG") + } + i.kind = mulHi + i.op1 = rm + i.b1 = _64 + if signed { + i.u1 = 1 + } + return i +} + +func (i *instruction) asUnaryRmR(op unaryRmROpcode, rm operand, rd regalloc.VReg, _64 bool) *instruction { + if rm.kind != operandKindReg && rm.kind != operandKindMem { + panic("BUG") + } + i.kind = unaryRmR + i.op1 = rm + i.op2 = newOperandReg(rd) + i.u1 = uint64(op) + i.b1 = _64 + return i +} + +func (i *instruction) asShiftR(op shiftROp, amount operand, rd regalloc.VReg, _64 bool) *instruction { + if amount.kind != operandKindReg && amount.kind != operandKindImm32 { + panic("BUG") + } + i.kind = shiftR + i.op1 = amount + i.op2 = newOperandReg(rd) + i.u1 = uint64(op) + i.b1 = _64 + return i +} + +func (i *instruction) asXmmRmiReg(op sseOpcode, rm operand, rd regalloc.VReg) *instruction { + if rm.kind != operandKindReg && rm.kind != operandKindImm32 && rm.kind != operandKindMem { + panic("BUG") + } + i.kind = xmmRmiReg + i.op1 = rm + i.op2 = newOperandReg(rd) + i.u1 = uint64(op) + return i +} + +func (i *instruction) asCmpRmiR(cmp bool, rm operand, rn regalloc.VReg, _64 bool) *instruction { + if rm.kind != operandKindReg && rm.kind != operandKindImm32 && rm.kind != operandKindMem { + panic("BUG") + } + i.kind = cmpRmiR + i.op1 = rm + i.op2 = newOperandReg(rn) + if cmp { + i.u1 = 1 + } + i.b1 = _64 + return i +} + +func (i *instruction) asSetcc(c cond, rd regalloc.VReg) *instruction { + i.kind = setcc + i.op2 = newOperandReg(rd) + i.u1 = uint64(c) + return i +} + +func (i *instruction) asCmove(c cond, rm operand, rd regalloc.VReg, _64 bool) *instruction { + i.kind = cmove + i.op1 = rm + i.op2 = newOperandReg(rd) + i.u1 = uint64(c) + i.b1 = _64 + return i +} + +func (m *machine) allocateExitSeq(execCtx regalloc.VReg) *instruction { + i := m.allocateInstr() + i.kind = exitSequence + i.op1 = newOperandReg(execCtx) + // Allocate the address mode that will be used in encoding the exit sequence. + i.op2 = newOperandMem(m.amodePool.Allocate()) + return i +} + +func (i *instruction) asXmmUnaryRmR(op sseOpcode, rm operand, rd regalloc.VReg) *instruction { + if rm.kind != operandKindReg && rm.kind != operandKindMem { + panic("BUG") + } + i.kind = xmmUnaryRmR + i.op1 = rm + i.op2 = newOperandReg(rd) + i.u1 = uint64(op) + return i +} + +func (i *instruction) asXmmUnaryRmRImm(op sseOpcode, imm byte, rm operand, rd regalloc.VReg) *instruction { + if rm.kind != operandKindReg && rm.kind != operandKindMem { + panic("BUG") + } + i.kind = xmmUnaryRmRImm + i.op1 = rm + i.op2 = newOperandReg(rd) + i.u1 = uint64(op) + i.u2 = uint64(imm) + return i +} + +func (i *instruction) asXmmCmpRmR(op sseOpcode, rm operand, rd regalloc.VReg) *instruction { + if rm.kind != operandKindReg && rm.kind != operandKindMem { + panic("BUG") + } + i.kind = xmmCmpRmR + i.op1 = rm + i.op2 = newOperandReg(rd) + i.u1 = uint64(op) + return i +} + +func (i *instruction) asXmmMovRM(op sseOpcode, rm regalloc.VReg, rd operand) *instruction { + if rd.kind != operandKindMem { + panic("BUG") + } + i.kind = xmmMovRM + i.op1 = newOperandReg(rm) + i.op2 = rd + i.u1 = uint64(op) + return i +} + +func (i *instruction) asPop64(rm regalloc.VReg) *instruction { + i.kind = pop64 + i.op1 = newOperandReg(rm) + return i +} + +func (i *instruction) asPush64(op operand) *instruction { + if op.kind != operandKindReg && op.kind != operandKindMem && op.kind != operandKindImm32 { + panic("BUG") + } + i.kind = push64 + i.op1 = op + return i +} + +func (i *instruction) asXCHG(rm regalloc.VReg, rd operand, size byte) *instruction { + i.kind = xchg + i.op1 = newOperandReg(rm) + i.op2 = rd + i.u1 = uint64(size) + return i +} + +func (i *instruction) asLockCmpXCHG(rm regalloc.VReg, rd *amode, size byte) *instruction { + i.kind = lockcmpxchg + i.op1 = newOperandReg(rm) + i.op2 = newOperandMem(rd) + i.u1 = uint64(size) + return i +} + +func (i *instruction) asLockXAdd(rm regalloc.VReg, rd *amode, size byte) *instruction { + i.kind = lockxadd + i.op1 = newOperandReg(rm) + i.op2 = newOperandMem(rd) + i.u1 = uint64(size) + return i +} + +type unaryRmROpcode byte + +const ( + unaryRmROpcodeBsr unaryRmROpcode = iota + unaryRmROpcodeBsf + unaryRmROpcodeLzcnt + unaryRmROpcodeTzcnt + unaryRmROpcodePopcnt +) + +func (u unaryRmROpcode) String() string { + switch u { + case unaryRmROpcodeBsr: + return "bsr" + case unaryRmROpcodeBsf: + return "bsf" + case unaryRmROpcodeLzcnt: + return "lzcnt" + case unaryRmROpcodeTzcnt: + return "tzcnt" + case unaryRmROpcodePopcnt: + return "popcnt" + default: + panic("BUG") + } +} + +type shiftROp byte + +const ( + shiftROpRotateLeft shiftROp = 0 + shiftROpRotateRight shiftROp = 1 + shiftROpShiftLeft shiftROp = 4 + shiftROpShiftRightLogical shiftROp = 5 + shiftROpShiftRightArithmetic shiftROp = 7 +) + +func (s shiftROp) String() string { + switch s { + case shiftROpRotateLeft: + return "rol" + case shiftROpRotateRight: + return "ror" + case shiftROpShiftLeft: + return "shl" + case shiftROpShiftRightLogical: + return "shr" + case shiftROpShiftRightArithmetic: + return "sar" + default: + panic("BUG") + } +} + +type sseOpcode byte + +const ( + sseOpcodeInvalid sseOpcode = iota + sseOpcodeAddps + sseOpcodeAddpd + sseOpcodeAddss + sseOpcodeAddsd + sseOpcodeAndps + sseOpcodeAndpd + sseOpcodeAndnps + sseOpcodeAndnpd + sseOpcodeBlendvps + sseOpcodeBlendvpd + sseOpcodeComiss + sseOpcodeComisd + sseOpcodeCmpps + sseOpcodeCmppd + sseOpcodeCmpss + sseOpcodeCmpsd + sseOpcodeCvtdq2ps + sseOpcodeCvtdq2pd + sseOpcodeCvtsd2ss + sseOpcodeCvtsd2si + sseOpcodeCvtsi2ss + sseOpcodeCvtsi2sd + sseOpcodeCvtss2si + sseOpcodeCvtss2sd + sseOpcodeCvttps2dq + sseOpcodeCvttss2si + sseOpcodeCvttsd2si + sseOpcodeDivps + sseOpcodeDivpd + sseOpcodeDivss + sseOpcodeDivsd + sseOpcodeInsertps + sseOpcodeMaxps + sseOpcodeMaxpd + sseOpcodeMaxss + sseOpcodeMaxsd + sseOpcodeMinps + sseOpcodeMinpd + sseOpcodeMinss + sseOpcodeMinsd + sseOpcodeMovaps + sseOpcodeMovapd + sseOpcodeMovd + sseOpcodeMovdqa + sseOpcodeMovdqu + sseOpcodeMovlhps + sseOpcodeMovmskps + sseOpcodeMovmskpd + sseOpcodeMovq + sseOpcodeMovss + sseOpcodeMovsd + sseOpcodeMovups + sseOpcodeMovupd + sseOpcodeMulps + sseOpcodeMulpd + sseOpcodeMulss + sseOpcodeMulsd + sseOpcodeOrps + sseOpcodeOrpd + sseOpcodePabsb + sseOpcodePabsw + sseOpcodePabsd + sseOpcodePackssdw + sseOpcodePacksswb + sseOpcodePackusdw + sseOpcodePackuswb + sseOpcodePaddb + sseOpcodePaddd + sseOpcodePaddq + sseOpcodePaddw + sseOpcodePaddsb + sseOpcodePaddsw + sseOpcodePaddusb + sseOpcodePaddusw + sseOpcodePalignr + sseOpcodePand + sseOpcodePandn + sseOpcodePavgb + sseOpcodePavgw + sseOpcodePcmpeqb + sseOpcodePcmpeqw + sseOpcodePcmpeqd + sseOpcodePcmpeqq + sseOpcodePcmpgtb + sseOpcodePcmpgtw + sseOpcodePcmpgtd + sseOpcodePcmpgtq + sseOpcodePextrb + sseOpcodePextrw + sseOpcodePextrd + sseOpcodePextrq + sseOpcodePinsrb + sseOpcodePinsrw + sseOpcodePinsrd + sseOpcodePinsrq + sseOpcodePmaddwd + sseOpcodePmaxsb + sseOpcodePmaxsw + sseOpcodePmaxsd + sseOpcodePmaxub + sseOpcodePmaxuw + sseOpcodePmaxud + sseOpcodePminsb + sseOpcodePminsw + sseOpcodePminsd + sseOpcodePminub + sseOpcodePminuw + sseOpcodePminud + sseOpcodePmovmskb + sseOpcodePmovsxbd + sseOpcodePmovsxbw + sseOpcodePmovsxbq + sseOpcodePmovsxwd + sseOpcodePmovsxwq + sseOpcodePmovsxdq + sseOpcodePmovzxbd + sseOpcodePmovzxbw + sseOpcodePmovzxbq + sseOpcodePmovzxwd + sseOpcodePmovzxwq + sseOpcodePmovzxdq + sseOpcodePmulld + sseOpcodePmullw + sseOpcodePmuludq + sseOpcodePor + sseOpcodePshufb + sseOpcodePshufd + sseOpcodePsllw + sseOpcodePslld + sseOpcodePsllq + sseOpcodePsraw + sseOpcodePsrad + sseOpcodePsrlw + sseOpcodePsrld + sseOpcodePsrlq + sseOpcodePsubb + sseOpcodePsubd + sseOpcodePsubq + sseOpcodePsubw + sseOpcodePsubsb + sseOpcodePsubsw + sseOpcodePsubusb + sseOpcodePsubusw + sseOpcodePtest + sseOpcodePunpckhbw + sseOpcodePunpcklbw + sseOpcodePxor + sseOpcodeRcpss + sseOpcodeRoundps + sseOpcodeRoundpd + sseOpcodeRoundss + sseOpcodeRoundsd + sseOpcodeRsqrtss + sseOpcodeSqrtps + sseOpcodeSqrtpd + sseOpcodeSqrtss + sseOpcodeSqrtsd + sseOpcodeSubps + sseOpcodeSubpd + sseOpcodeSubss + sseOpcodeSubsd + sseOpcodeUcomiss + sseOpcodeUcomisd + sseOpcodeXorps + sseOpcodeXorpd + sseOpcodePmulhrsw + sseOpcodeUnpcklps + sseOpcodeCvtps2pd + sseOpcodeCvtpd2ps + sseOpcodeCvttpd2dq + sseOpcodeShufps + sseOpcodePmaddubsw +) + +func (s sseOpcode) String() string { + switch s { + case sseOpcodeInvalid: + return "invalid" + case sseOpcodeAddps: + return "addps" + case sseOpcodeAddpd: + return "addpd" + case sseOpcodeAddss: + return "addss" + case sseOpcodeAddsd: + return "addsd" + case sseOpcodeAndps: + return "andps" + case sseOpcodeAndpd: + return "andpd" + case sseOpcodeAndnps: + return "andnps" + case sseOpcodeAndnpd: + return "andnpd" + case sseOpcodeBlendvps: + return "blendvps" + case sseOpcodeBlendvpd: + return "blendvpd" + case sseOpcodeComiss: + return "comiss" + case sseOpcodeComisd: + return "comisd" + case sseOpcodeCmpps: + return "cmpps" + case sseOpcodeCmppd: + return "cmppd" + case sseOpcodeCmpss: + return "cmpss" + case sseOpcodeCmpsd: + return "cmpsd" + case sseOpcodeCvtdq2ps: + return "cvtdq2ps" + case sseOpcodeCvtdq2pd: + return "cvtdq2pd" + case sseOpcodeCvtsd2ss: + return "cvtsd2ss" + case sseOpcodeCvtsd2si: + return "cvtsd2si" + case sseOpcodeCvtsi2ss: + return "cvtsi2ss" + case sseOpcodeCvtsi2sd: + return "cvtsi2sd" + case sseOpcodeCvtss2si: + return "cvtss2si" + case sseOpcodeCvtss2sd: + return "cvtss2sd" + case sseOpcodeCvttps2dq: + return "cvttps2dq" + case sseOpcodeCvttss2si: + return "cvttss2si" + case sseOpcodeCvttsd2si: + return "cvttsd2si" + case sseOpcodeDivps: + return "divps" + case sseOpcodeDivpd: + return "divpd" + case sseOpcodeDivss: + return "divss" + case sseOpcodeDivsd: + return "divsd" + case sseOpcodeInsertps: + return "insertps" + case sseOpcodeMaxps: + return "maxps" + case sseOpcodeMaxpd: + return "maxpd" + case sseOpcodeMaxss: + return "maxss" + case sseOpcodeMaxsd: + return "maxsd" + case sseOpcodeMinps: + return "minps" + case sseOpcodeMinpd: + return "minpd" + case sseOpcodeMinss: + return "minss" + case sseOpcodeMinsd: + return "minsd" + case sseOpcodeMovaps: + return "movaps" + case sseOpcodeMovapd: + return "movapd" + case sseOpcodeMovd: + return "movd" + case sseOpcodeMovdqa: + return "movdqa" + case sseOpcodeMovdqu: + return "movdqu" + case sseOpcodeMovlhps: + return "movlhps" + case sseOpcodeMovmskps: + return "movmskps" + case sseOpcodeMovmskpd: + return "movmskpd" + case sseOpcodeMovq: + return "movq" + case sseOpcodeMovss: + return "movss" + case sseOpcodeMovsd: + return "movsd" + case sseOpcodeMovups: + return "movups" + case sseOpcodeMovupd: + return "movupd" + case sseOpcodeMulps: + return "mulps" + case sseOpcodeMulpd: + return "mulpd" + case sseOpcodeMulss: + return "mulss" + case sseOpcodeMulsd: + return "mulsd" + case sseOpcodeOrps: + return "orps" + case sseOpcodeOrpd: + return "orpd" + case sseOpcodePabsb: + return "pabsb" + case sseOpcodePabsw: + return "pabsw" + case sseOpcodePabsd: + return "pabsd" + case sseOpcodePackssdw: + return "packssdw" + case sseOpcodePacksswb: + return "packsswb" + case sseOpcodePackusdw: + return "packusdw" + case sseOpcodePackuswb: + return "packuswb" + case sseOpcodePaddb: + return "paddb" + case sseOpcodePaddd: + return "paddd" + case sseOpcodePaddq: + return "paddq" + case sseOpcodePaddw: + return "paddw" + case sseOpcodePaddsb: + return "paddsb" + case sseOpcodePaddsw: + return "paddsw" + case sseOpcodePaddusb: + return "paddusb" + case sseOpcodePaddusw: + return "paddusw" + case sseOpcodePalignr: + return "palignr" + case sseOpcodePand: + return "pand" + case sseOpcodePandn: + return "pandn" + case sseOpcodePavgb: + return "pavgb" + case sseOpcodePavgw: + return "pavgw" + case sseOpcodePcmpeqb: + return "pcmpeqb" + case sseOpcodePcmpeqw: + return "pcmpeqw" + case sseOpcodePcmpeqd: + return "pcmpeqd" + case sseOpcodePcmpeqq: + return "pcmpeqq" + case sseOpcodePcmpgtb: + return "pcmpgtb" + case sseOpcodePcmpgtw: + return "pcmpgtw" + case sseOpcodePcmpgtd: + return "pcmpgtd" + case sseOpcodePcmpgtq: + return "pcmpgtq" + case sseOpcodePextrb: + return "pextrb" + case sseOpcodePextrw: + return "pextrw" + case sseOpcodePextrd: + return "pextrd" + case sseOpcodePextrq: + return "pextrq" + case sseOpcodePinsrb: + return "pinsrb" + case sseOpcodePinsrw: + return "pinsrw" + case sseOpcodePinsrd: + return "pinsrd" + case sseOpcodePinsrq: + return "pinsrq" + case sseOpcodePmaddwd: + return "pmaddwd" + case sseOpcodePmaxsb: + return "pmaxsb" + case sseOpcodePmaxsw: + return "pmaxsw" + case sseOpcodePmaxsd: + return "pmaxsd" + case sseOpcodePmaxub: + return "pmaxub" + case sseOpcodePmaxuw: + return "pmaxuw" + case sseOpcodePmaxud: + return "pmaxud" + case sseOpcodePminsb: + return "pminsb" + case sseOpcodePminsw: + return "pminsw" + case sseOpcodePminsd: + return "pminsd" + case sseOpcodePminub: + return "pminub" + case sseOpcodePminuw: + return "pminuw" + case sseOpcodePminud: + return "pminud" + case sseOpcodePmovmskb: + return "pmovmskb" + case sseOpcodePmovsxbd: + return "pmovsxbd" + case sseOpcodePmovsxbw: + return "pmovsxbw" + case sseOpcodePmovsxbq: + return "pmovsxbq" + case sseOpcodePmovsxwd: + return "pmovsxwd" + case sseOpcodePmovsxwq: + return "pmovsxwq" + case sseOpcodePmovsxdq: + return "pmovsxdq" + case sseOpcodePmovzxbd: + return "pmovzxbd" + case sseOpcodePmovzxbw: + return "pmovzxbw" + case sseOpcodePmovzxbq: + return "pmovzxbq" + case sseOpcodePmovzxwd: + return "pmovzxwd" + case sseOpcodePmovzxwq: + return "pmovzxwq" + case sseOpcodePmovzxdq: + return "pmovzxdq" + case sseOpcodePmulld: + return "pmulld" + case sseOpcodePmullw: + return "pmullw" + case sseOpcodePmuludq: + return "pmuludq" + case sseOpcodePor: + return "por" + case sseOpcodePshufb: + return "pshufb" + case sseOpcodePshufd: + return "pshufd" + case sseOpcodePsllw: + return "psllw" + case sseOpcodePslld: + return "pslld" + case sseOpcodePsllq: + return "psllq" + case sseOpcodePsraw: + return "psraw" + case sseOpcodePsrad: + return "psrad" + case sseOpcodePsrlw: + return "psrlw" + case sseOpcodePsrld: + return "psrld" + case sseOpcodePsrlq: + return "psrlq" + case sseOpcodePsubb: + return "psubb" + case sseOpcodePsubd: + return "psubd" + case sseOpcodePsubq: + return "psubq" + case sseOpcodePsubw: + return "psubw" + case sseOpcodePsubsb: + return "psubsb" + case sseOpcodePsubsw: + return "psubsw" + case sseOpcodePsubusb: + return "psubusb" + case sseOpcodePsubusw: + return "psubusw" + case sseOpcodePtest: + return "ptest" + case sseOpcodePunpckhbw: + return "punpckhbw" + case sseOpcodePunpcklbw: + return "punpcklbw" + case sseOpcodePxor: + return "pxor" + case sseOpcodeRcpss: + return "rcpss" + case sseOpcodeRoundps: + return "roundps" + case sseOpcodeRoundpd: + return "roundpd" + case sseOpcodeRoundss: + return "roundss" + case sseOpcodeRoundsd: + return "roundsd" + case sseOpcodeRsqrtss: + return "rsqrtss" + case sseOpcodeSqrtps: + return "sqrtps" + case sseOpcodeSqrtpd: + return "sqrtpd" + case sseOpcodeSqrtss: + return "sqrtss" + case sseOpcodeSqrtsd: + return "sqrtsd" + case sseOpcodeSubps: + return "subps" + case sseOpcodeSubpd: + return "subpd" + case sseOpcodeSubss: + return "subss" + case sseOpcodeSubsd: + return "subsd" + case sseOpcodeUcomiss: + return "ucomiss" + case sseOpcodeUcomisd: + return "ucomisd" + case sseOpcodeXorps: + return "xorps" + case sseOpcodeXorpd: + return "xorpd" + case sseOpcodePmulhrsw: + return "pmulhrsw" + case sseOpcodeUnpcklps: + return "unpcklps" + case sseOpcodeCvtps2pd: + return "cvtps2pd" + case sseOpcodeCvtpd2ps: + return "cvtpd2ps" + case sseOpcodeCvttpd2dq: + return "cvttpd2dq" + case sseOpcodeShufps: + return "shufps" + case sseOpcodePmaddubsw: + return "pmaddubsw" + default: + panic("BUG") + } +} + +type roundingMode uint8 + +const ( + roundingModeNearest roundingMode = iota + roundingModeDown + roundingModeUp + roundingModeZero +) + +func (r roundingMode) String() string { + switch r { + case roundingModeNearest: + return "nearest" + case roundingModeDown: + return "down" + case roundingModeUp: + return "up" + case roundingModeZero: + return "zero" + default: + panic("BUG") + } +} + +// cmpPred is the immediate value for a comparison operation in xmmRmRImm. +type cmpPred uint8 + +const ( + // cmpPredEQ_OQ is Equal (ordered, non-signaling) + cmpPredEQ_OQ cmpPred = iota + // cmpPredLT_OS is Less-than (ordered, signaling) + cmpPredLT_OS + // cmpPredLE_OS is Less-than-or-equal (ordered, signaling) + cmpPredLE_OS + // cmpPredUNORD_Q is Unordered (non-signaling) + cmpPredUNORD_Q + // cmpPredNEQ_UQ is Not-equal (unordered, non-signaling) + cmpPredNEQ_UQ + // cmpPredNLT_US is Not-less-than (unordered, signaling) + cmpPredNLT_US + // cmpPredNLE_US is Not-less-than-or-equal (unordered, signaling) + cmpPredNLE_US + // cmpPredORD_Q is Ordered (non-signaling) + cmpPredORD_Q + // cmpPredEQ_UQ is Equal (unordered, non-signaling) + cmpPredEQ_UQ + // cmpPredNGE_US is Not-greater-than-or-equal (unordered, signaling) + cmpPredNGE_US + // cmpPredNGT_US is Not-greater-than (unordered, signaling) + cmpPredNGT_US + // cmpPredFALSE_OQ is False (ordered, non-signaling) + cmpPredFALSE_OQ + // cmpPredNEQ_OQ is Not-equal (ordered, non-signaling) + cmpPredNEQ_OQ + // cmpPredGE_OS is Greater-than-or-equal (ordered, signaling) + cmpPredGE_OS + // cmpPredGT_OS is Greater-than (ordered, signaling) + cmpPredGT_OS + // cmpPredTRUE_UQ is True (unordered, non-signaling) + cmpPredTRUE_UQ + // Equal (ordered, signaling) + cmpPredEQ_OS + // Less-than (ordered, nonsignaling) + cmpPredLT_OQ + // Less-than-or-equal (ordered, nonsignaling) + cmpPredLE_OQ + // Unordered (signaling) + cmpPredUNORD_S + // Not-equal (unordered, signaling) + cmpPredNEQ_US + // Not-less-than (unordered, nonsignaling) + cmpPredNLT_UQ + // Not-less-than-or-equal (unordered, nonsignaling) + cmpPredNLE_UQ + // Ordered (signaling) + cmpPredORD_S + // Equal (unordered, signaling) + cmpPredEQ_US + // Not-greater-than-or-equal (unordered, non-signaling) + cmpPredNGE_UQ + // Not-greater-than (unordered, nonsignaling) + cmpPredNGT_UQ + // False (ordered, signaling) + cmpPredFALSE_OS + // Not-equal (ordered, signaling) + cmpPredNEQ_OS + // Greater-than-or-equal (ordered, nonsignaling) + cmpPredGE_OQ + // Greater-than (ordered, nonsignaling) + cmpPredGT_OQ + // True (unordered, signaling) + cmpPredTRUE_US +) + +func (r cmpPred) String() string { + switch r { + case cmpPredEQ_OQ: + return "eq_oq" + case cmpPredLT_OS: + return "lt_os" + case cmpPredLE_OS: + return "le_os" + case cmpPredUNORD_Q: + return "unord_q" + case cmpPredNEQ_UQ: + return "neq_uq" + case cmpPredNLT_US: + return "nlt_us" + case cmpPredNLE_US: + return "nle_us" + case cmpPredORD_Q: + return "ord_q" + case cmpPredEQ_UQ: + return "eq_uq" + case cmpPredNGE_US: + return "nge_us" + case cmpPredNGT_US: + return "ngt_us" + case cmpPredFALSE_OQ: + return "false_oq" + case cmpPredNEQ_OQ: + return "neq_oq" + case cmpPredGE_OS: + return "ge_os" + case cmpPredGT_OS: + return "gt_os" + case cmpPredTRUE_UQ: + return "true_uq" + case cmpPredEQ_OS: + return "eq_os" + case cmpPredLT_OQ: + return "lt_oq" + case cmpPredLE_OQ: + return "le_oq" + case cmpPredUNORD_S: + return "unord_s" + case cmpPredNEQ_US: + return "neq_us" + case cmpPredNLT_UQ: + return "nlt_uq" + case cmpPredNLE_UQ: + return "nle_uq" + case cmpPredORD_S: + return "ord_s" + case cmpPredEQ_US: + return "eq_us" + case cmpPredNGE_UQ: + return "nge_uq" + case cmpPredNGT_UQ: + return "ngt_uq" + case cmpPredFALSE_OS: + return "false_os" + case cmpPredNEQ_OS: + return "neq_os" + case cmpPredGE_OQ: + return "ge_oq" + case cmpPredGT_OQ: + return "gt_oq" + case cmpPredTRUE_US: + return "true_us" + default: + panic("BUG") + } +} + +func linkInstr(prev, next *instruction) *instruction { + prev.next = next + next.prev = prev + return next +} + +type defKind byte + +const ( + defKindNone defKind = iota + 1 + defKindOp2 + defKindCall + defKindDivRem +) + +var defKinds = [instrMax]defKind{ + nop0: defKindNone, + ret: defKindNone, + movRR: defKindOp2, + movRM: defKindNone, + xmmMovRM: defKindNone, + aluRmiR: defKindNone, + shiftR: defKindNone, + imm: defKindOp2, + unaryRmR: defKindOp2, + xmmRmiReg: defKindNone, + xmmUnaryRmR: defKindOp2, + xmmUnaryRmRImm: defKindOp2, + xmmCmpRmR: defKindNone, + xmmRmR: defKindNone, + xmmRmRImm: defKindNone, + mov64MR: defKindOp2, + movsxRmR: defKindOp2, + movzxRmR: defKindOp2, + gprToXmm: defKindOp2, + xmmToGpr: defKindOp2, + cmove: defKindNone, + call: defKindCall, + callIndirect: defKindCall, + ud2: defKindNone, + jmp: defKindNone, + jmpIf: defKindNone, + jmpTableIsland: defKindNone, + cmpRmiR: defKindNone, + exitSequence: defKindNone, + lea: defKindOp2, + setcc: defKindOp2, + zeros: defKindOp2, + sourceOffsetInfo: defKindNone, + fcvtToSintSequence: defKindNone, + defineUninitializedReg: defKindOp2, + fcvtToUintSequence: defKindNone, + xmmCMov: defKindOp2, + idivRemSequence: defKindDivRem, + blendvpd: defKindNone, + mfence: defKindNone, + xchg: defKindNone, + lockcmpxchg: defKindNone, + lockxadd: defKindNone, + neg: defKindNone, + nopUseReg: defKindNone, +} + +// String implements fmt.Stringer. +func (d defKind) String() string { + switch d { + case defKindNone: + return "none" + case defKindOp2: + return "op2" + case defKindCall: + return "call" + case defKindDivRem: + return "divrem" + default: + return "invalid" + } +} + +type useKind byte + +const ( + useKindNone useKind = iota + 1 + useKindOp1 + // useKindOp1Op2Reg is Op1 can be any operand, Op2 must be a register. + useKindOp1Op2Reg + // useKindOp1RegOp2 is Op1 must be a register, Op2 can be any operand. + useKindOp1RegOp2 + // useKindRaxOp1RegOp2 is Op1 must be a register, Op2 can be any operand, and RAX is used. + useKindRaxOp1RegOp2 + useKindDivRem + useKindBlendvpd + useKindCall + useKindCallInd + useKindFcvtToSintSequence + useKindFcvtToUintSequence +) + +var useKinds = [instrMax]useKind{ + nop0: useKindNone, + ret: useKindNone, + movRR: useKindOp1, + movRM: useKindOp1RegOp2, + xmmMovRM: useKindOp1RegOp2, + cmove: useKindOp1Op2Reg, + aluRmiR: useKindOp1Op2Reg, + shiftR: useKindOp1Op2Reg, + imm: useKindNone, + unaryRmR: useKindOp1, + xmmRmiReg: useKindOp1Op2Reg, + xmmUnaryRmR: useKindOp1, + xmmUnaryRmRImm: useKindOp1, + xmmCmpRmR: useKindOp1Op2Reg, + xmmRmR: useKindOp1Op2Reg, + xmmRmRImm: useKindOp1Op2Reg, + mov64MR: useKindOp1, + movzxRmR: useKindOp1, + movsxRmR: useKindOp1, + gprToXmm: useKindOp1, + xmmToGpr: useKindOp1, + call: useKindCall, + callIndirect: useKindCallInd, + ud2: useKindNone, + jmpIf: useKindOp1, + jmp: useKindOp1, + cmpRmiR: useKindOp1Op2Reg, + exitSequence: useKindOp1, + lea: useKindOp1, + jmpTableIsland: useKindNone, + setcc: useKindNone, + zeros: useKindNone, + sourceOffsetInfo: useKindNone, + fcvtToSintSequence: useKindFcvtToSintSequence, + defineUninitializedReg: useKindNone, + fcvtToUintSequence: useKindFcvtToUintSequence, + xmmCMov: useKindOp1, + idivRemSequence: useKindDivRem, + blendvpd: useKindBlendvpd, + mfence: useKindNone, + xchg: useKindOp1RegOp2, + lockcmpxchg: useKindRaxOp1RegOp2, + lockxadd: useKindOp1RegOp2, + neg: useKindOp1, + nopUseReg: useKindOp1, +} + +func (u useKind) String() string { + switch u { + case useKindNone: + return "none" + case useKindOp1: + return "op1" + case useKindOp1Op2Reg: + return "op1op2Reg" + case useKindOp1RegOp2: + return "op1RegOp2" + case useKindCall: + return "call" + case useKindCallInd: + return "callInd" + default: + return "invalid" + } +} diff --git a/vendor/github.com/tetratelabs/wazero/internal/engine/wazevo/backend/isa/amd64/instr_encoding.go b/vendor/github.com/tetratelabs/wazero/internal/engine/wazevo/backend/isa/amd64/instr_encoding.go new file mode 100644 index 000000000..6637b428c --- /dev/null +++ b/vendor/github.com/tetratelabs/wazero/internal/engine/wazevo/backend/isa/amd64/instr_encoding.go @@ -0,0 +1,1683 @@ +package amd64 + +import ( + "fmt" + + "github.com/tetratelabs/wazero/internal/engine/wazevo/backend" + "github.com/tetratelabs/wazero/internal/engine/wazevo/backend/regalloc" + "github.com/tetratelabs/wazero/internal/engine/wazevo/ssa" + "github.com/tetratelabs/wazero/internal/engine/wazevo/wazevoapi" +) + +func (i *instruction) encode(c backend.Compiler) (needsLabelResolution bool) { + switch kind := i.kind; kind { + case nop0, sourceOffsetInfo, defineUninitializedReg, fcvtToSintSequence, fcvtToUintSequence, nopUseReg: + case ret: + encodeRet(c) + case imm: + dst := regEncodings[i.op2.reg().RealReg()] + con := i.u1 + if i.b1 { // 64 bit. + if lower32willSignExtendTo64(con) { + // Sign extend mov(imm32). + encodeRegReg(c, + legacyPrefixesNone, + 0xc7, 1, + 0, + dst, + rexInfo(0).setW(), + ) + c.Emit4Bytes(uint32(con)) + } else { + c.EmitByte(rexEncodingW | dst.rexBit()) + c.EmitByte(0xb8 | dst.encoding()) + c.Emit8Bytes(con) + } + } else { + if dst.rexBit() > 0 { + c.EmitByte(rexEncodingDefault | 0x1) + } + c.EmitByte(0xb8 | dst.encoding()) + c.Emit4Bytes(uint32(con)) + } + + case aluRmiR: + var rex rexInfo + if i.b1 { + rex = rex.setW() + } else { + rex = rex.clearW() + } + + dst := regEncodings[i.op2.reg().RealReg()] + + aluOp := aluRmiROpcode(i.u1) + if aluOp == aluRmiROpcodeMul { + op1 := i.op1 + const regMemOpc, regMemOpcNum = 0x0FAF, 2 + switch op1.kind { + case operandKindReg: + src := regEncodings[op1.reg().RealReg()] + encodeRegReg(c, legacyPrefixesNone, regMemOpc, regMemOpcNum, dst, src, rex) + case operandKindMem: + m := i.op1.addressMode() + encodeRegMem(c, legacyPrefixesNone, regMemOpc, regMemOpcNum, dst, m, rex) + case operandKindImm32: + imm8 := lower8willSignExtendTo32(op1.imm32()) + var opc uint32 + if imm8 { + opc = 0x6b + } else { + opc = 0x69 + } + encodeRegReg(c, legacyPrefixesNone, opc, 1, dst, dst, rex) + if imm8 { + c.EmitByte(byte(op1.imm32())) + } else { + c.Emit4Bytes(op1.imm32()) + } + default: + panic("BUG: invalid operand kind") + } + } else { + const opcodeNum = 1 + var opcR, opcM, subOpcImm uint32 + switch aluOp { + case aluRmiROpcodeAdd: + opcR, opcM, subOpcImm = 0x01, 0x03, 0x0 + case aluRmiROpcodeSub: + opcR, opcM, subOpcImm = 0x29, 0x2b, 0x5 + case aluRmiROpcodeAnd: + opcR, opcM, subOpcImm = 0x21, 0x23, 0x4 + case aluRmiROpcodeOr: + opcR, opcM, subOpcImm = 0x09, 0x0b, 0x1 + case aluRmiROpcodeXor: + opcR, opcM, subOpcImm = 0x31, 0x33, 0x6 + default: + panic("BUG: invalid aluRmiROpcode") + } + + op1 := i.op1 + switch op1.kind { + case operandKindReg: + src := regEncodings[op1.reg().RealReg()] + encodeRegReg(c, legacyPrefixesNone, opcR, opcodeNum, src, dst, rex) + case operandKindMem: + m := i.op1.addressMode() + encodeRegMem(c, legacyPrefixesNone, opcM, opcodeNum, dst, m, rex) + case operandKindImm32: + imm8 := lower8willSignExtendTo32(op1.imm32()) + var opc uint32 + if imm8 { + opc = 0x83 + } else { + opc = 0x81 + } + encodeRegReg(c, legacyPrefixesNone, opc, opcodeNum, regEnc(subOpcImm), dst, rex) + if imm8 { + c.EmitByte(byte(op1.imm32())) + } else { + c.Emit4Bytes(op1.imm32()) + } + default: + panic("BUG: invalid operand kind") + } + } + + case movRR: + src := regEncodings[i.op1.reg().RealReg()] + dst := regEncodings[i.op2.reg().RealReg()] + var rex rexInfo + if i.b1 { + rex = rex.setW() + } else { + rex = rex.clearW() + } + encodeRegReg(c, legacyPrefixesNone, 0x89, 1, src, dst, rex) + + case xmmRmR, blendvpd: + op := sseOpcode(i.u1) + var legPrex legacyPrefixes + var opcode uint32 + var opcodeNum uint32 + switch op { + case sseOpcodeAddps: + legPrex, opcode, opcodeNum = legacyPrefixesNone, 0x0F58, 2 + case sseOpcodeAddpd: + legPrex, opcode, opcodeNum = legacyPrefixes0x66, 0x0F58, 2 + case sseOpcodeAddss: + legPrex, opcode, opcodeNum = legacyPrefixes0xF3, 0x0F58, 2 + case sseOpcodeAddsd: + legPrex, opcode, opcodeNum = legacyPrefixes0xF2, 0x0F58, 2 + case sseOpcodeAndps: + legPrex, opcode, opcodeNum = legacyPrefixesNone, 0x0F54, 2 + case sseOpcodeAndpd: + legPrex, opcode, opcodeNum = legacyPrefixes0x66, 0x0F54, 2 + case sseOpcodeAndnps: + legPrex, opcode, opcodeNum = legacyPrefixesNone, 0x0F55, 2 + case sseOpcodeAndnpd: + legPrex, opcode, opcodeNum = legacyPrefixes0x66, 0x0F55, 2 + case sseOpcodeBlendvps: + legPrex, opcode, opcodeNum = legacyPrefixes0x66, 0x0F3814, 3 + case sseOpcodeBlendvpd: + legPrex, opcode, opcodeNum = legacyPrefixes0x66, 0x0F3815, 3 + case sseOpcodeDivps: + legPrex, opcode, opcodeNum = legacyPrefixesNone, 0x0F5E, 2 + case sseOpcodeDivpd: + legPrex, opcode, opcodeNum = legacyPrefixes0x66, 0x0F5E, 2 + case sseOpcodeDivss: + legPrex, opcode, opcodeNum = legacyPrefixes0xF3, 0x0F5E, 2 + case sseOpcodeDivsd: + legPrex, opcode, opcodeNum = legacyPrefixes0xF2, 0x0F5E, 2 + case sseOpcodeMaxps: + legPrex, opcode, opcodeNum = legacyPrefixesNone, 0x0F5F, 2 + case sseOpcodeMaxpd: + legPrex, opcode, opcodeNum = legacyPrefixes0x66, 0x0F5F, 2 + case sseOpcodeMaxss: + legPrex, opcode, opcodeNum = legacyPrefixes0xF3, 0x0F5F, 2 + case sseOpcodeMaxsd: + legPrex, opcode, opcodeNum = legacyPrefixes0xF2, 0x0F5F, 2 + case sseOpcodeMinps: + legPrex, opcode, opcodeNum = legacyPrefixesNone, 0x0F5D, 2 + case sseOpcodeMinpd: + legPrex, opcode, opcodeNum = legacyPrefixes0x66, 0x0F5D, 2 + case sseOpcodeMinss: + legPrex, opcode, opcodeNum = legacyPrefixes0xF3, 0x0F5D, 2 + case sseOpcodeMinsd: + legPrex, opcode, opcodeNum = legacyPrefixes0xF2, 0x0F5D, 2 + case sseOpcodeMovlhps: + legPrex, opcode, opcodeNum = legacyPrefixesNone, 0x0F16, 2 + case sseOpcodeMovsd: + legPrex, opcode, opcodeNum = legacyPrefixes0xF2, 0x0F10, 2 + case sseOpcodeMulps: + legPrex, opcode, opcodeNum = legacyPrefixesNone, 0x0F59, 2 + case sseOpcodeMulpd: + legPrex, opcode, opcodeNum = legacyPrefixes0x66, 0x0F59, 2 + case sseOpcodeMulss: + legPrex, opcode, opcodeNum = legacyPrefixes0xF3, 0x0F59, 2 + case sseOpcodeMulsd: + legPrex, opcode, opcodeNum = legacyPrefixes0xF2, 0x0F59, 2 + case sseOpcodeOrpd: + legPrex, opcode, opcodeNum = legacyPrefixes0x66, 0x0F56, 2 + case sseOpcodeOrps: + legPrex, opcode, opcodeNum = legacyPrefixesNone, 0x0F56, 2 + case sseOpcodePackssdw: + legPrex, opcode, opcodeNum = legacyPrefixes0x66, 0x0F6B, 2 + case sseOpcodePacksswb: + legPrex, opcode, opcodeNum = legacyPrefixes0x66, 0x0F63, 2 + case sseOpcodePackusdw: + legPrex, opcode, opcodeNum = legacyPrefixes0x66, 0x0F382B, 3 + case sseOpcodePackuswb: + legPrex, opcode, opcodeNum = legacyPrefixes0x66, 0x0F67, 2 + case sseOpcodePaddb: + legPrex, opcode, opcodeNum = legacyPrefixes0x66, 0x0FFC, 2 + case sseOpcodePaddd: + legPrex, opcode, opcodeNum = legacyPrefixes0x66, 0x0FFE, 2 + case sseOpcodePaddq: + legPrex, opcode, opcodeNum = legacyPrefixes0x66, 0x0FD4, 2 + case sseOpcodePaddw: + legPrex, opcode, opcodeNum = legacyPrefixes0x66, 0x0FFD, 2 + case sseOpcodePaddsb: + legPrex, opcode, opcodeNum = legacyPrefixes0x66, 0x0FEC, 2 + case sseOpcodePaddsw: + legPrex, opcode, opcodeNum = legacyPrefixes0x66, 0x0FED, 2 + case sseOpcodePaddusb: + legPrex, opcode, opcodeNum = legacyPrefixes0x66, 0x0FDC, 2 + case sseOpcodePaddusw: + legPrex, opcode, opcodeNum = legacyPrefixes0x66, 0x0FDD, 2 + case sseOpcodePand: + legPrex, opcode, opcodeNum = legacyPrefixes0x66, 0x0FDB, 2 + case sseOpcodePandn: + legPrex, opcode, opcodeNum = legacyPrefixes0x66, 0x0FDF, 2 + case sseOpcodePavgb: + legPrex, opcode, opcodeNum = legacyPrefixes0x66, 0x0FE0, 2 + case sseOpcodePavgw: + legPrex, opcode, opcodeNum = legacyPrefixes0x66, 0x0FE3, 2 + case sseOpcodePcmpeqb: + legPrex, opcode, opcodeNum = legacyPrefixes0x66, 0x0F74, 2 + case sseOpcodePcmpeqw: + legPrex, opcode, opcodeNum = legacyPrefixes0x66, 0x0F75, 2 + case sseOpcodePcmpeqd: + legPrex, opcode, opcodeNum = legacyPrefixes0x66, 0x0F76, 2 + case sseOpcodePcmpeqq: + legPrex, opcode, opcodeNum = legacyPrefixes0x66, 0x0F3829, 3 + case sseOpcodePcmpgtb: + legPrex, opcode, opcodeNum = legacyPrefixes0x66, 0x0F64, 2 + case sseOpcodePcmpgtw: + legPrex, opcode, opcodeNum = legacyPrefixes0x66, 0x0F65, 2 + case sseOpcodePcmpgtd: + legPrex, opcode, opcodeNum = legacyPrefixes0x66, 0x0F66, 2 + case sseOpcodePcmpgtq: + legPrex, opcode, opcodeNum = legacyPrefixes0x66, 0x0F3837, 3 + case sseOpcodePmaddwd: + legPrex, opcode, opcodeNum = legacyPrefixes0x66, 0x0FF5, 2 + case sseOpcodePmaxsb: + legPrex, opcode, opcodeNum = legacyPrefixes0x66, 0x0F383C, 3 + case sseOpcodePmaxsw: + legPrex, opcode, opcodeNum = legacyPrefixes0x66, 0x0FEE, 2 + case sseOpcodePmaxsd: + legPrex, opcode, opcodeNum = legacyPrefixes0x66, 0x0F383D, 3 + case sseOpcodePmaxub: + legPrex, opcode, opcodeNum = legacyPrefixes0x66, 0x0FDE, 2 + case sseOpcodePmaxuw: + legPrex, opcode, opcodeNum = legacyPrefixes0x66, 0x0F383E, 3 + case sseOpcodePmaxud: + legPrex, opcode, opcodeNum = legacyPrefixes0x66, 0x0F383F, 3 + case sseOpcodePminsb: + legPrex, opcode, opcodeNum = legacyPrefixes0x66, 0x0F3838, 3 + case sseOpcodePminsw: + legPrex, opcode, opcodeNum = legacyPrefixes0x66, 0x0FEA, 2 + case sseOpcodePminsd: + legPrex, opcode, opcodeNum = legacyPrefixes0x66, 0x0F3839, 3 + case sseOpcodePminub: + legPrex, opcode, opcodeNum = legacyPrefixes0x66, 0x0FDA, 2 + case sseOpcodePminuw: + legPrex, opcode, opcodeNum = legacyPrefixes0x66, 0x0F383A, 3 + case sseOpcodePminud: + legPrex, opcode, opcodeNum = legacyPrefixes0x66, 0x0F383B, 3 + case sseOpcodePmulld: + legPrex, opcode, opcodeNum = legacyPrefixes0x66, 0x0F3840, 3 + case sseOpcodePmullw: + legPrex, opcode, opcodeNum = legacyPrefixes0x66, 0x0FD5, 2 + case sseOpcodePmuludq: + legPrex, opcode, opcodeNum = legacyPrefixes0x66, 0x0FF4, 2 + case sseOpcodePor: + legPrex, opcode, opcodeNum = legacyPrefixes0x66, 0x0FEB, 2 + case sseOpcodePshufb: + legPrex, opcode, opcodeNum = legacyPrefixes0x66, 0x0F3800, 3 + case sseOpcodePsubb: + legPrex, opcode, opcodeNum = legacyPrefixes0x66, 0x0FF8, 2 + case sseOpcodePsubd: + legPrex, opcode, opcodeNum = legacyPrefixes0x66, 0x0FFA, 2 + case sseOpcodePsubq: + legPrex, opcode, opcodeNum = legacyPrefixes0x66, 0x0FFB, 2 + case sseOpcodePsubw: + legPrex, opcode, opcodeNum = legacyPrefixes0x66, 0x0FF9, 2 + case sseOpcodePsubsb: + legPrex, opcode, opcodeNum = legacyPrefixes0x66, 0x0FE8, 2 + case sseOpcodePsubsw: + legPrex, opcode, opcodeNum = legacyPrefixes0x66, 0x0FE9, 2 + case sseOpcodePsubusb: + legPrex, opcode, opcodeNum = legacyPrefixes0x66, 0x0FD8, 2 + case sseOpcodePsubusw: + legPrex, opcode, opcodeNum = legacyPrefixes0x66, 0x0FD9, 2 + case sseOpcodePunpckhbw: + legPrex, opcode, opcodeNum = legacyPrefixes0x66, 0x0F68, 2 + case sseOpcodePunpcklbw: + legPrex, opcode, opcodeNum = legacyPrefixes0x66, 0x0F60, 2 + case sseOpcodePxor: + legPrex, opcode, opcodeNum = legacyPrefixes0x66, 0x0FEF, 2 + case sseOpcodeSubps: + legPrex, opcode, opcodeNum = legacyPrefixesNone, 0x0F5C, 2 + case sseOpcodeSubpd: + legPrex, opcode, opcodeNum = legacyPrefixes0x66, 0x0F5C, 2 + case sseOpcodeSubss: + legPrex, opcode, opcodeNum = legacyPrefixes0xF3, 0x0F5C, 2 + case sseOpcodeSubsd: + legPrex, opcode, opcodeNum = legacyPrefixes0xF2, 0x0F5C, 2 + case sseOpcodeXorps: + legPrex, opcode, opcodeNum = legacyPrefixesNone, 0x0F57, 2 + case sseOpcodeXorpd: + legPrex, opcode, opcodeNum = legacyPrefixes0x66, 0x0F57, 2 + case sseOpcodePmulhrsw: + legPrex, opcode, opcodeNum = legacyPrefixes0x66, 0x0F380B, 3 + case sseOpcodeUnpcklps: + legPrex, opcode, opcodeNum = legacyPrefixesNone, 0x0F14, 2 + case sseOpcodePmaddubsw: + legPrex, opcode, opcodeNum = legacyPrefixes0x66, 0x0F3804, 3 + default: + if kind == blendvpd { + legPrex, opcode, opcodeNum = legacyPrefixes0x66, 0x0F3815, 3 + } else { + panic(fmt.Sprintf("Unsupported sseOpcode: %s", op)) + } + } + + dst := regEncodings[i.op2.reg().RealReg()] + + rex := rexInfo(0).clearW() + op1 := i.op1 + if op1.kind == operandKindReg { + src := regEncodings[op1.reg().RealReg()] + encodeRegReg(c, legPrex, opcode, opcodeNum, dst, src, rex) + } else if i.op1.kind == operandKindMem { + m := i.op1.addressMode() + encodeRegMem(c, legPrex, opcode, opcodeNum, dst, m, rex) + } else { + panic("BUG: invalid operand kind") + } + + case gprToXmm: + var legPrefix legacyPrefixes + var opcode uint32 + const opcodeNum = 2 + switch sseOpcode(i.u1) { + case sseOpcodeMovd, sseOpcodeMovq: + legPrefix, opcode = legacyPrefixes0x66, 0x0f6e + case sseOpcodeCvtsi2ss: + legPrefix, opcode = legacyPrefixes0xF3, 0x0f2a + case sseOpcodeCvtsi2sd: + legPrefix, opcode = legacyPrefixes0xF2, 0x0f2a + default: + panic(fmt.Sprintf("Unsupported sseOpcode: %s", sseOpcode(i.u1))) + } + + var rex rexInfo + if i.b1 { + rex = rex.setW() + } else { + rex = rex.clearW() + } + dst := regEncodings[i.op2.reg().RealReg()] + + op1 := i.op1 + if op1.kind == operandKindReg { + src := regEncodings[op1.reg().RealReg()] + encodeRegReg(c, legPrefix, opcode, opcodeNum, dst, src, rex) + } else if i.op1.kind == operandKindMem { + m := i.op1.addressMode() + encodeRegMem(c, legPrefix, opcode, opcodeNum, dst, m, rex) + } else { + panic("BUG: invalid operand kind") + } + + case xmmUnaryRmR: + var prefix legacyPrefixes + var opcode uint32 + var opcodeNum uint32 + op := sseOpcode(i.u1) + switch op { + case sseOpcodeCvtss2sd: + prefix, opcode, opcodeNum = legacyPrefixes0xF3, 0x0F5A, 2 + case sseOpcodeCvtsd2ss: + prefix, opcode, opcodeNum = legacyPrefixes0xF2, 0x0F5A, 2 + case sseOpcodeMovaps: + prefix, opcode, opcodeNum = legacyPrefixesNone, 0x0F28, 2 + case sseOpcodeMovapd: + prefix, opcode, opcodeNum = legacyPrefixes0x66, 0x0F28, 2 + case sseOpcodeMovdqa: + prefix, opcode, opcodeNum = legacyPrefixes0x66, 0x0F6F, 2 + case sseOpcodeMovdqu: + prefix, opcode, opcodeNum = legacyPrefixes0xF3, 0x0F6F, 2 + case sseOpcodeMovsd: + prefix, opcode, opcodeNum = legacyPrefixes0xF2, 0x0F10, 2 + case sseOpcodeMovss: + prefix, opcode, opcodeNum = legacyPrefixes0xF3, 0x0F10, 2 + case sseOpcodeMovups: + prefix, opcode, opcodeNum = legacyPrefixesNone, 0x0F10, 2 + case sseOpcodeMovupd: + prefix, opcode, opcodeNum = legacyPrefixes0x66, 0x0F10, 2 + case sseOpcodePabsb: + prefix, opcode, opcodeNum = legacyPrefixes0x66, 0x0F381C, 3 + case sseOpcodePabsw: + prefix, opcode, opcodeNum = legacyPrefixes0x66, 0x0F381D, 3 + case sseOpcodePabsd: + prefix, opcode, opcodeNum = legacyPrefixes0x66, 0x0F381E, 3 + case sseOpcodePmovsxbd: + prefix, opcode, opcodeNum = legacyPrefixes0x66, 0x0F3821, 3 + case sseOpcodePmovsxbw: + prefix, opcode, opcodeNum = legacyPrefixes0x66, 0x0F3820, 3 + case sseOpcodePmovsxbq: + prefix, opcode, opcodeNum = legacyPrefixes0x66, 0x0F3822, 3 + case sseOpcodePmovsxwd: + prefix, opcode, opcodeNum = legacyPrefixes0x66, 0x0F3823, 3 + case sseOpcodePmovsxwq: + prefix, opcode, opcodeNum = legacyPrefixes0x66, 0x0F3824, 3 + case sseOpcodePmovsxdq: + prefix, opcode, opcodeNum = legacyPrefixes0x66, 0x0F3825, 3 + case sseOpcodePmovzxbd: + prefix, opcode, opcodeNum = legacyPrefixes0x66, 0x0F3831, 3 + case sseOpcodePmovzxbw: + prefix, opcode, opcodeNum = legacyPrefixes0x66, 0x0F3830, 3 + case sseOpcodePmovzxbq: + prefix, opcode, opcodeNum = legacyPrefixes0x66, 0x0F3832, 3 + case sseOpcodePmovzxwd: + prefix, opcode, opcodeNum = legacyPrefixes0x66, 0x0F3833, 3 + case sseOpcodePmovzxwq: + prefix, opcode, opcodeNum = legacyPrefixes0x66, 0x0F3834, 3 + case sseOpcodePmovzxdq: + prefix, opcode, opcodeNum = legacyPrefixes0x66, 0x0F3835, 3 + case sseOpcodeSqrtps: + prefix, opcode, opcodeNum = legacyPrefixesNone, 0x0F51, 2 + case sseOpcodeSqrtpd: + prefix, opcode, opcodeNum = legacyPrefixes0x66, 0x0F51, 2 + case sseOpcodeSqrtss: + prefix, opcode, opcodeNum = legacyPrefixes0xF3, 0x0F51, 2 + case sseOpcodeSqrtsd: + prefix, opcode, opcodeNum = legacyPrefixes0xF2, 0x0F51, 2 + case sseOpcodeXorps: + prefix, opcode, opcodeNum = legacyPrefixesNone, 0x0F57, 2 + case sseOpcodeXorpd: + prefix, opcode, opcodeNum = legacyPrefixes0x66, 0x0F57, 2 + case sseOpcodeCvtdq2ps: + prefix, opcode, opcodeNum = legacyPrefixesNone, 0x0F5B, 2 + case sseOpcodeCvtdq2pd: + prefix, opcode, opcodeNum = legacyPrefixes0xF3, 0x0FE6, 2 + case sseOpcodeCvtps2pd: + prefix, opcode, opcodeNum = legacyPrefixesNone, 0x0F5A, 2 + case sseOpcodeCvtpd2ps: + prefix, opcode, opcodeNum = legacyPrefixes0x66, 0x0F5A, 2 + case sseOpcodeCvttps2dq: + prefix, opcode, opcodeNum = legacyPrefixes0xF3, 0x0F5B, 2 + case sseOpcodeCvttpd2dq: + prefix, opcode, opcodeNum = legacyPrefixes0x66, 0x0FE6, 2 + default: + panic(fmt.Sprintf("Unsupported sseOpcode: %s", op)) + } + + dst := regEncodings[i.op2.reg().RealReg()] + + rex := rexInfo(0).clearW() + op1 := i.op1 + if op1.kind == operandKindReg { + src := regEncodings[op1.reg().RealReg()] + encodeRegReg(c, prefix, opcode, opcodeNum, dst, src, rex) + } else if i.op1.kind == operandKindMem { + m := i.op1.addressMode() + needsLabelResolution = encodeRegMem(c, prefix, opcode, opcodeNum, dst, m, rex) + } else { + panic("BUG: invalid operand kind") + } + + case xmmUnaryRmRImm: + var prefix legacyPrefixes + var opcode uint32 + var opcodeNum uint32 + op := sseOpcode(i.u1) + switch op { + case sseOpcodeRoundps: + prefix, opcode, opcodeNum = legacyPrefixes0x66, 0x0f3a08, 3 + case sseOpcodeRoundss: + prefix, opcode, opcodeNum = legacyPrefixes0x66, 0x0f3a0a, 3 + case sseOpcodeRoundpd: + prefix, opcode, opcodeNum = legacyPrefixes0x66, 0x0f3a09, 3 + case sseOpcodeRoundsd: + prefix, opcode, opcodeNum = legacyPrefixes0x66, 0x0f3a0b, 3 + } + rex := rexInfo(0).clearW() + dst := regEncodings[i.op2.reg().RealReg()] + op1 := i.op1 + if op1.kind == operandKindReg { + src := regEncodings[op1.reg().RealReg()] + encodeRegReg(c, prefix, opcode, opcodeNum, dst, src, rex) + } else if i.op1.kind == operandKindMem { + m := i.op1.addressMode() + encodeRegMem(c, prefix, opcode, opcodeNum, dst, m, rex) + } else { + panic("BUG: invalid operand kind") + } + + c.EmitByte(byte(i.u2)) + + case unaryRmR: + var prefix legacyPrefixes + var opcode uint32 + var opcodeNum uint32 + op := unaryRmROpcode(i.u1) + // We assume size is either 32 or 64. + switch op { + case unaryRmROpcodeBsr: + prefix, opcode, opcodeNum = legacyPrefixesNone, 0x0fbd, 2 + case unaryRmROpcodeBsf: + prefix, opcode, opcodeNum = legacyPrefixesNone, 0x0fbc, 2 + case unaryRmROpcodeLzcnt: + prefix, opcode, opcodeNum = legacyPrefixes0xF3, 0x0fbd, 2 + case unaryRmROpcodeTzcnt: + prefix, opcode, opcodeNum = legacyPrefixes0xF3, 0x0fbc, 2 + case unaryRmROpcodePopcnt: + prefix, opcode, opcodeNum = legacyPrefixes0xF3, 0x0fb8, 2 + default: + panic(fmt.Sprintf("Unsupported unaryRmROpcode: %s", op)) + } + + dst := regEncodings[i.op2.reg().RealReg()] + + rex := rexInfo(0) + if i.b1 { // 64 bit. + rex = rexInfo(0).setW() + } else { + rex = rexInfo(0).clearW() + } + op1 := i.op1 + if op1.kind == operandKindReg { + src := regEncodings[op1.reg().RealReg()] + encodeRegReg(c, prefix, opcode, opcodeNum, dst, src, rex) + } else if i.op1.kind == operandKindMem { + m := i.op1.addressMode() + encodeRegMem(c, prefix, opcode, opcodeNum, dst, m, rex) + } else { + panic("BUG: invalid operand kind") + } + + case not: + var prefix legacyPrefixes + src := regEncodings[i.op1.reg().RealReg()] + rex := rexInfo(0) + if i.b1 { // 64 bit. + rex = rexInfo(0).setW() + } else { + rex = rexInfo(0).clearW() + } + subopcode := uint8(2) + encodeEncEnc(c, prefix, 0xf7, 1, subopcode, uint8(src), rex) + + case neg: + var prefix legacyPrefixes + src := regEncodings[i.op1.reg().RealReg()] + rex := rexInfo(0) + if i.b1 { // 64 bit. + rex = rexInfo(0).setW() + } else { + rex = rexInfo(0).clearW() + } + subopcode := uint8(3) + encodeEncEnc(c, prefix, 0xf7, 1, subopcode, uint8(src), rex) + + case div: + rex := rexInfo(0) + if i.b1 { // 64 bit. + rex = rexInfo(0).setW() + } else { + rex = rexInfo(0).clearW() + } + var subopcode uint8 + if i.u1 != 0 { // Signed. + subopcode = 7 + } else { + subopcode = 6 + } + + divisor := i.op1 + if divisor.kind == operandKindReg { + src := regEncodings[divisor.reg().RealReg()] + encodeEncEnc(c, legacyPrefixesNone, 0xf7, 1, subopcode, uint8(src), rex) + } else if divisor.kind == operandKindMem { + m := divisor.addressMode() + encodeEncMem(c, legacyPrefixesNone, 0xf7, 1, subopcode, m, rex) + } else { + panic("BUG: invalid operand kind") + } + + case mulHi: + var prefix legacyPrefixes + rex := rexInfo(0) + if i.b1 { // 64 bit. + rex = rexInfo(0).setW() + } else { + rex = rexInfo(0).clearW() + } + + signed := i.u1 != 0 + var subopcode uint8 + if signed { + subopcode = 5 + } else { + subopcode = 4 + } + + // src1 is implicitly rax, + // dst_lo is implicitly rax, + // dst_hi is implicitly rdx. + src2 := i.op1 + if src2.kind == operandKindReg { + src := regEncodings[src2.reg().RealReg()] + encodeEncEnc(c, prefix, 0xf7, 1, subopcode, uint8(src), rex) + } else if src2.kind == operandKindMem { + m := src2.addressMode() + encodeEncMem(c, prefix, 0xf7, 1, subopcode, m, rex) + } else { + panic("BUG: invalid operand kind") + } + + case signExtendData: + if i.b1 { // 64 bit. + c.EmitByte(0x48) + c.EmitByte(0x99) + } else { + c.EmitByte(0x99) + } + case movzxRmR, movsxRmR: + signed := i.kind == movsxRmR + + ext := extMode(i.u1) + var opcode uint32 + var opcodeNum uint32 + var rex rexInfo + switch ext { + case extModeBL: + if signed { + opcode, opcodeNum, rex = 0x0fbe, 2, rex.clearW() + } else { + opcode, opcodeNum, rex = 0x0fb6, 2, rex.clearW() + } + case extModeBQ: + if signed { + opcode, opcodeNum, rex = 0x0fbe, 2, rex.setW() + } else { + opcode, opcodeNum, rex = 0x0fb6, 2, rex.setW() + } + case extModeWL: + if signed { + opcode, opcodeNum, rex = 0x0fbf, 2, rex.clearW() + } else { + opcode, opcodeNum, rex = 0x0fb7, 2, rex.clearW() + } + case extModeWQ: + if signed { + opcode, opcodeNum, rex = 0x0fbf, 2, rex.setW() + } else { + opcode, opcodeNum, rex = 0x0fb7, 2, rex.setW() + } + case extModeLQ: + if signed { + opcode, opcodeNum, rex = 0x63, 1, rex.setW() + } else { + opcode, opcodeNum, rex = 0x8b, 1, rex.clearW() + } + default: + panic("BUG: invalid extMode") + } + + op := i.op1 + dst := regEncodings[i.op2.reg().RealReg()] + switch op.kind { + case operandKindReg: + src := regEncodings[op.reg().RealReg()] + if ext == extModeBL || ext == extModeBQ { + // Some destinations must be encoded with REX.R = 1. + if e := src.encoding(); e >= 4 && e <= 7 { + rex = rex.always() + } + } + encodeRegReg(c, legacyPrefixesNone, opcode, opcodeNum, dst, src, rex) + case operandKindMem: + m := op.addressMode() + encodeRegMem(c, legacyPrefixesNone, opcode, opcodeNum, dst, m, rex) + default: + panic("BUG: invalid operand kind") + } + + case mov64MR: + m := i.op1.addressMode() + encodeLoad64(c, m, i.op2.reg().RealReg()) + + case lea: + needsLabelResolution = true + dst := regEncodings[i.op2.reg().RealReg()] + rex := rexInfo(0).setW() + const opcode, opcodeNum = 0x8d, 1 + switch i.op1.kind { + case operandKindMem: + a := i.op1.addressMode() + encodeRegMem(c, legacyPrefixesNone, opcode, opcodeNum, dst, a, rex) + case operandKindLabel: + rex.encode(c, regRexBit(byte(dst)), 0) + c.EmitByte(byte((opcode) & 0xff)) + + // Indicate "LEAQ [RIP + 32bit displacement]. + // https://wiki.osdev.org/X86-64_Instruction_Encoding#32.2F64-bit_addressing + c.EmitByte(encodeModRM(0b00, dst.encoding(), 0b101)) + + // This will be resolved later, so we just emit a placeholder (0xffffffff for testing). + c.Emit4Bytes(0xffffffff) + default: + panic("BUG: invalid operand kind") + } + + case movRM: + m := i.op2.addressMode() + src := regEncodings[i.op1.reg().RealReg()] + + var rex rexInfo + switch i.u1 { + case 1: + if e := src.encoding(); e >= 4 && e <= 7 { + rex = rex.always() + } + encodeRegMem(c, legacyPrefixesNone, 0x88, 1, src, m, rex.clearW()) + case 2: + encodeRegMem(c, legacyPrefixes0x66, 0x89, 1, src, m, rex.clearW()) + case 4: + encodeRegMem(c, legacyPrefixesNone, 0x89, 1, src, m, rex.clearW()) + case 8: + encodeRegMem(c, legacyPrefixesNone, 0x89, 1, src, m, rex.setW()) + default: + panic(fmt.Sprintf("BUG: invalid size %d: %s", i.u1, i.String())) + } + + case shiftR: + src := regEncodings[i.op2.reg().RealReg()] + amount := i.op1 + + var opcode uint32 + var prefix legacyPrefixes + rex := rexInfo(0) + if i.b1 { // 64 bit. + rex = rexInfo(0).setW() + } else { + rex = rexInfo(0).clearW() + } + + switch amount.kind { + case operandKindReg: + if amount.reg() != rcxVReg { + panic("BUG: invalid reg operand: must be rcx") + } + opcode, prefix = 0xd3, legacyPrefixesNone + encodeEncEnc(c, prefix, opcode, 1, uint8(i.u1), uint8(src), rex) + case operandKindImm32: + opcode, prefix = 0xc1, legacyPrefixesNone + encodeEncEnc(c, prefix, opcode, 1, uint8(i.u1), uint8(src), rex) + c.EmitByte(byte(amount.imm32())) + default: + panic("BUG: invalid operand kind") + } + case xmmRmiReg: + const legPrefix = legacyPrefixes0x66 + rex := rexInfo(0).clearW() + dst := regEncodings[i.op2.reg().RealReg()] + + var opcode uint32 + var regDigit uint8 + + op := sseOpcode(i.u1) + op1 := i.op1 + if i.op1.kind == operandKindImm32 { + switch op { + case sseOpcodePsllw: + opcode, regDigit = 0x0f71, 6 + case sseOpcodePslld: + opcode, regDigit = 0x0f72, 6 + case sseOpcodePsllq: + opcode, regDigit = 0x0f73, 6 + case sseOpcodePsraw: + opcode, regDigit = 0x0f71, 4 + case sseOpcodePsrad: + opcode, regDigit = 0x0f72, 4 + case sseOpcodePsrlw: + opcode, regDigit = 0x0f71, 2 + case sseOpcodePsrld: + opcode, regDigit = 0x0f72, 2 + case sseOpcodePsrlq: + opcode, regDigit = 0x0f73, 2 + default: + panic("invalid opcode") + } + + encodeEncEnc(c, legPrefix, opcode, 2, regDigit, uint8(dst), rex) + imm32 := op1.imm32() + if imm32 > 0xff&imm32 { + panic("immediate value does not fit 1 byte") + } + c.EmitByte(uint8(imm32)) + } else { + switch op { + case sseOpcodePsllw: + opcode = 0x0ff1 + case sseOpcodePslld: + opcode = 0x0ff2 + case sseOpcodePsllq: + opcode = 0x0ff3 + case sseOpcodePsraw: + opcode = 0x0fe1 + case sseOpcodePsrad: + opcode = 0x0fe2 + case sseOpcodePsrlw: + opcode = 0x0fd1 + case sseOpcodePsrld: + opcode = 0x0fd2 + case sseOpcodePsrlq: + opcode = 0x0fd3 + default: + panic("invalid opcode") + } + + if op1.kind == operandKindReg { + reg := regEncodings[op1.reg().RealReg()] + encodeRegReg(c, legPrefix, opcode, 2, dst, reg, rex) + } else if op1.kind == operandKindMem { + m := op1.addressMode() + encodeRegMem(c, legPrefix, opcode, 2, dst, m, rex) + } else { + panic("BUG: invalid operand kind") + } + } + + case cmpRmiR: + var opcode uint32 + isCmp := i.u1 != 0 + rex := rexInfo(0) + _64 := i.b1 + if _64 { // 64 bit. + rex = rex.setW() + } else { + rex = rex.clearW() + } + dst := regEncodings[i.op2.reg().RealReg()] + op1 := i.op1 + switch op1.kind { + case operandKindReg: + reg := regEncodings[op1.reg().RealReg()] + if isCmp { + opcode = 0x39 + } else { + opcode = 0x85 + } + // Here we swap the encoding of the operands for CMP to be consistent with the output of LLVM/GCC. + encodeRegReg(c, legacyPrefixesNone, opcode, 1, reg, dst, rex) + + case operandKindMem: + if isCmp { + opcode = 0x3b + } else { + opcode = 0x85 + } + m := op1.addressMode() + encodeRegMem(c, legacyPrefixesNone, opcode, 1, dst, m, rex) + + case operandKindImm32: + imm32 := op1.imm32() + useImm8 := isCmp && lower8willSignExtendTo32(imm32) + var subopcode uint8 + + switch { + case isCmp && useImm8: + opcode, subopcode = 0x83, 7 + case isCmp && !useImm8: + opcode, subopcode = 0x81, 7 + default: + opcode, subopcode = 0xf7, 0 + } + encodeEncEnc(c, legacyPrefixesNone, opcode, 1, subopcode, uint8(dst), rex) + if useImm8 { + c.EmitByte(uint8(imm32)) + } else { + c.Emit4Bytes(imm32) + } + + default: + panic("BUG: invalid operand kind") + } + case setcc: + cc := cond(i.u1) + dst := regEncodings[i.op2.reg().RealReg()] + rex := rexInfo(0).clearW().always() + opcode := uint32(0x0f90) + uint32(cc) + encodeEncEnc(c, legacyPrefixesNone, opcode, 2, 0, uint8(dst), rex) + case cmove: + cc := cond(i.u1) + dst := regEncodings[i.op2.reg().RealReg()] + rex := rexInfo(0) + if i.b1 { // 64 bit. + rex = rex.setW() + } else { + rex = rex.clearW() + } + opcode := uint32(0x0f40) + uint32(cc) + src := i.op1 + switch src.kind { + case operandKindReg: + srcReg := regEncodings[src.reg().RealReg()] + encodeRegReg(c, legacyPrefixesNone, opcode, 2, dst, srcReg, rex) + case operandKindMem: + m := src.addressMode() + encodeRegMem(c, legacyPrefixesNone, opcode, 2, dst, m, rex) + default: + panic("BUG: invalid operand kind") + } + case push64: + op := i.op1 + + switch op.kind { + case operandKindReg: + dst := regEncodings[op.reg().RealReg()] + if dst.rexBit() > 0 { + c.EmitByte(rexEncodingDefault | 0x1) + } + c.EmitByte(0x50 | dst.encoding()) + case operandKindMem: + m := op.addressMode() + encodeRegMem( + c, legacyPrefixesNone, 0xff, 1, regEnc(6), m, rexInfo(0).clearW(), + ) + case operandKindImm32: + c.EmitByte(0x68) + c.Emit4Bytes(op.imm32()) + default: + panic("BUG: invalid operand kind") + } + + case pop64: + dst := regEncodings[i.op1.reg().RealReg()] + if dst.rexBit() > 0 { + c.EmitByte(rexEncodingDefault | 0x1) + } + c.EmitByte(0x58 | dst.encoding()) + + case xmmMovRM: + var legPrefix legacyPrefixes + var opcode uint32 + const opcodeNum = 2 + switch sseOpcode(i.u1) { + case sseOpcodeMovaps: + legPrefix, opcode = legacyPrefixesNone, 0x0f29 + case sseOpcodeMovapd: + legPrefix, opcode = legacyPrefixes0x66, 0x0f29 + case sseOpcodeMovdqa: + legPrefix, opcode = legacyPrefixes0x66, 0x0f7f + case sseOpcodeMovdqu: + legPrefix, opcode = legacyPrefixes0xF3, 0x0f7f + case sseOpcodeMovss: + legPrefix, opcode = legacyPrefixes0xF3, 0x0f11 + case sseOpcodeMovsd: + legPrefix, opcode = legacyPrefixes0xF2, 0x0f11 + case sseOpcodeMovups: + legPrefix, opcode = legacyPrefixesNone, 0x0f11 + case sseOpcodeMovupd: + legPrefix, opcode = legacyPrefixes0x66, 0x0f11 + default: + panic(fmt.Sprintf("Unsupported sseOpcode: %s", sseOpcode(i.u1))) + } + + dst := regEncodings[i.op1.reg().RealReg()] + encodeRegMem(c, legPrefix, opcode, opcodeNum, dst, i.op2.addressMode(), rexInfo(0).clearW()) + case xmmLoadConst: + panic("TODO") + case xmmToGpr: + var legPrefix legacyPrefixes + var opcode uint32 + var argSwap bool + const opcodeNum = 2 + switch sseOpcode(i.u1) { + case sseOpcodeMovd, sseOpcodeMovq: + legPrefix, opcode, argSwap = legacyPrefixes0x66, 0x0f7e, false + case sseOpcodeMovmskps: + legPrefix, opcode, argSwap = legacyPrefixesNone, 0x0f50, true + case sseOpcodeMovmskpd: + legPrefix, opcode, argSwap = legacyPrefixes0x66, 0x0f50, true + case sseOpcodePmovmskb: + legPrefix, opcode, argSwap = legacyPrefixes0x66, 0x0fd7, true + case sseOpcodeCvttss2si: + legPrefix, opcode, argSwap = legacyPrefixes0xF3, 0x0f2c, true + case sseOpcodeCvttsd2si: + legPrefix, opcode, argSwap = legacyPrefixes0xF2, 0x0f2c, true + default: + panic(fmt.Sprintf("Unsupported sseOpcode: %s", sseOpcode(i.u1))) + } + + var rex rexInfo + if i.b1 { + rex = rex.setW() + } else { + rex = rex.clearW() + } + src := regEncodings[i.op1.reg().RealReg()] + dst := regEncodings[i.op2.reg().RealReg()] + if argSwap { + src, dst = dst, src + } + encodeRegReg(c, legPrefix, opcode, opcodeNum, src, dst, rex) + + case cvtUint64ToFloatSeq: + panic("TODO") + case cvtFloatToSintSeq: + panic("TODO") + case cvtFloatToUintSeq: + panic("TODO") + case xmmMinMaxSeq: + panic("TODO") + case xmmCmpRmR: + var prefix legacyPrefixes + var opcode uint32 + var opcodeNum uint32 + rex := rexInfo(0) + _64 := i.b1 + if _64 { // 64 bit. + rex = rex.setW() + } else { + rex = rex.clearW() + } + + op := sseOpcode(i.u1) + switch op { + case sseOpcodePtest: + prefix, opcode, opcodeNum = legacyPrefixes0x66, 0x0f3817, 3 + case sseOpcodeUcomisd: + prefix, opcode, opcodeNum = legacyPrefixes0x66, 0x0f2e, 2 + case sseOpcodeUcomiss: + prefix, opcode, opcodeNum = legacyPrefixesNone, 0x0f2e, 2 + default: + panic(fmt.Sprintf("Unsupported sseOpcode: %s", op)) + } + + dst := regEncodings[i.op2.reg().RealReg()] + op1 := i.op1 + switch op1.kind { + case operandKindReg: + reg := regEncodings[op1.reg().RealReg()] + encodeRegReg(c, prefix, opcode, opcodeNum, dst, reg, rex) + + case operandKindMem: + m := op1.addressMode() + encodeRegMem(c, prefix, opcode, opcodeNum, dst, m, rex) + + default: + panic("BUG: invalid operand kind") + } + case xmmRmRImm: + op := sseOpcode(i.u1) + var legPrex legacyPrefixes + var opcode uint32 + var opcodeNum uint32 + var swap bool + switch op { + case sseOpcodeCmpps: + legPrex, opcode, opcodeNum = legacyPrefixesNone, 0x0FC2, 2 + case sseOpcodeCmppd: + legPrex, opcode, opcodeNum = legacyPrefixes0x66, 0x0FC2, 2 + case sseOpcodeCmpss: + legPrex, opcode, opcodeNum = legacyPrefixes0xF3, 0x0FC2, 2 + case sseOpcodeCmpsd: + legPrex, opcode, opcodeNum = legacyPrefixes0xF2, 0x0FC2, 2 + case sseOpcodeInsertps: + legPrex, opcode, opcodeNum = legacyPrefixes0x66, 0x0F3A21, 3 + case sseOpcodePalignr: + legPrex, opcode, opcodeNum = legacyPrefixes0x66, 0x0F3A0F, 3 + case sseOpcodePinsrb: + legPrex, opcode, opcodeNum = legacyPrefixes0x66, 0x0F3A20, 3 + case sseOpcodePinsrw: + legPrex, opcode, opcodeNum = legacyPrefixes0x66, 0x0FC4, 2 + case sseOpcodePinsrd, sseOpcodePinsrq: + legPrex, opcode, opcodeNum = legacyPrefixes0x66, 0x0F3A22, 3 + case sseOpcodePextrb: + swap = true + legPrex, opcode, opcodeNum = legacyPrefixes0x66, 0x0F3A14, 3 + case sseOpcodePextrw: + legPrex, opcode, opcodeNum = legacyPrefixes0x66, 0x0FC5, 2 + case sseOpcodePextrd, sseOpcodePextrq: + swap = true + legPrex, opcode, opcodeNum = legacyPrefixes0x66, 0x0F3A16, 3 + case sseOpcodePshufd: + legPrex, opcode, opcodeNum = legacyPrefixes0x66, 0x0F70, 2 + case sseOpcodeRoundps: + legPrex, opcode, opcodeNum = legacyPrefixes0x66, 0x0F3A08, 3 + case sseOpcodeRoundpd: + legPrex, opcode, opcodeNum = legacyPrefixes0x66, 0x0F3A09, 3 + case sseOpcodeShufps: + legPrex, opcode, opcodeNum = legacyPrefixesNone, 0x0FC6, 2 + default: + panic(fmt.Sprintf("Unsupported sseOpcode: %s", op)) + } + + dst := regEncodings[i.op2.reg().RealReg()] + + var rex rexInfo + if op == sseOpcodePextrq || op == sseOpcodePinsrq { + rex = rexInfo(0).setW() + } else { + rex = rexInfo(0).clearW() + } + op1 := i.op1 + if op1.kind == operandKindReg { + src := regEncodings[op1.reg().RealReg()] + if swap { + src, dst = dst, src + } + encodeRegReg(c, legPrex, opcode, opcodeNum, dst, src, rex) + } else if i.op1.kind == operandKindMem { + if swap { + panic("BUG: this is not possible to encode") + } + m := i.op1.addressMode() + encodeRegMem(c, legPrex, opcode, opcodeNum, dst, m, rex) + } else { + panic("BUG: invalid operand kind") + } + + c.EmitByte(byte(i.u2)) + + case jmp: + const ( + regMemOpcode = 0xff + regMemOpcodeNum = 1 + regMemSubOpcode = 4 + ) + op := i.op1 + switch op.kind { + case operandKindLabel: + needsLabelResolution = true + fallthrough + case operandKindImm32: + c.EmitByte(0xe9) + c.Emit4Bytes(op.imm32()) + case operandKindMem: + m := op.addressMode() + encodeRegMem(c, + legacyPrefixesNone, + regMemOpcode, regMemOpcodeNum, + regMemSubOpcode, m, rexInfo(0).clearW(), + ) + case operandKindReg: + r := op.reg().RealReg() + encodeRegReg( + c, + legacyPrefixesNone, + regMemOpcode, regMemOpcodeNum, + regMemSubOpcode, + regEncodings[r], rexInfo(0).clearW(), + ) + default: + panic("BUG: invalid operand kind") + } + + case jmpIf: + op := i.op1 + switch op.kind { + case operandKindLabel: + needsLabelResolution = true + fallthrough + case operandKindImm32: + c.EmitByte(0x0f) + c.EmitByte(0x80 | cond(i.u1).encoding()) + c.Emit4Bytes(op.imm32()) + default: + panic("BUG: invalid operand kind") + } + + case jmpTableIsland: + needsLabelResolution = true + for tc := uint64(0); tc < i.u2; tc++ { + c.Emit8Bytes(0) + } + + case exitSequence: + execCtx := i.op1.reg() + allocatedAmode := i.op2.addressMode() + + // Restore the RBP, RSP, and return to the Go code: + *allocatedAmode = amode{ + kindWithShift: uint32(amodeImmReg), base: execCtx, + imm32: wazevoapi.ExecutionContextOffsetOriginalFramePointer.U32(), + } + encodeLoad64(c, allocatedAmode, rbp) + allocatedAmode.imm32 = wazevoapi.ExecutionContextOffsetOriginalStackPointer.U32() + encodeLoad64(c, allocatedAmode, rsp) + encodeRet(c) + + case ud2: + c.EmitByte(0x0f) + c.EmitByte(0x0b) + + case call: + c.EmitByte(0xe8) + // Meaning that the call target is a function value, and requires relocation. + c.AddRelocationInfo(ssa.FuncRef(i.u1)) + // Note that this is zero as a placeholder for the call target if it's a function value. + c.Emit4Bytes(uint32(i.u2)) + + case callIndirect: + op := i.op1 + + const opcodeNum = 1 + const opcode = 0xff + rex := rexInfo(0).clearW() + switch op.kind { + case operandKindReg: + dst := regEncodings[op.reg().RealReg()] + encodeRegReg(c, + legacyPrefixesNone, + opcode, opcodeNum, + regEnc(2), + dst, + rex, + ) + case operandKindMem: + m := op.addressMode() + encodeRegMem(c, + legacyPrefixesNone, + opcode, opcodeNum, + regEnc(2), + m, + rex, + ) + default: + panic("BUG: invalid operand kind") + } + + case xchg: + src, dst := regEncodings[i.op1.reg().RealReg()], i.op2 + size := i.u1 + + var rex rexInfo + var opcode uint32 + lp := legacyPrefixesNone + switch size { + case 8: + opcode = 0x87 + rex = rexInfo(0).setW() + case 4: + opcode = 0x87 + rex = rexInfo(0).clearW() + case 2: + lp = legacyPrefixes0x66 + opcode = 0x87 + rex = rexInfo(0).clearW() + case 1: + opcode = 0x86 + if i.op2.kind == operandKindReg { + panic("TODO?: xchg on two 1-byte registers") + } + // Some destinations must be encoded with REX.R = 1. + if e := src.encoding(); e >= 4 && e <= 7 { + rex = rexInfo(0).always() + } + default: + panic(fmt.Sprintf("BUG: invalid size %d: %s", size, i.String())) + } + + switch dst.kind { + case operandKindMem: + m := dst.addressMode() + encodeRegMem(c, lp, opcode, 1, src, m, rex) + case operandKindReg: + r := dst.reg().RealReg() + encodeRegReg(c, lp, opcode, 1, src, regEncodings[r], rex) + default: + panic("BUG: invalid operand kind") + } + + case lockcmpxchg: + src, dst := regEncodings[i.op1.reg().RealReg()], i.op2 + size := i.u1 + + var rex rexInfo + var opcode uint32 + lp := legacyPrefixes0xF0 // Lock prefix. + switch size { + case 8: + opcode = 0x0FB1 + rex = rexInfo(0).setW() + case 4: + opcode = 0x0FB1 + rex = rexInfo(0).clearW() + case 2: + lp = legacyPrefixes0x660xF0 // Legacy prefix + Lock prefix. + opcode = 0x0FB1 + rex = rexInfo(0).clearW() + case 1: + opcode = 0x0FB0 + // Some destinations must be encoded with REX.R = 1. + if e := src.encoding(); e >= 4 && e <= 7 { + rex = rexInfo(0).always() + } + default: + panic(fmt.Sprintf("BUG: invalid size %d: %s", size, i.String())) + } + + switch dst.kind { + case operandKindMem: + m := dst.addressMode() + encodeRegMem(c, lp, opcode, 2, src, m, rex) + default: + panic("BUG: invalid operand kind") + } + + case lockxadd: + src, dst := regEncodings[i.op1.reg().RealReg()], i.op2 + size := i.u1 + + var rex rexInfo + var opcode uint32 + lp := legacyPrefixes0xF0 // Lock prefix. + switch size { + case 8: + opcode = 0x0FC1 + rex = rexInfo(0).setW() + case 4: + opcode = 0x0FC1 + rex = rexInfo(0).clearW() + case 2: + lp = legacyPrefixes0x660xF0 // Legacy prefix + Lock prefix. + opcode = 0x0FC1 + rex = rexInfo(0).clearW() + case 1: + opcode = 0x0FC0 + // Some destinations must be encoded with REX.R = 1. + if e := src.encoding(); e >= 4 && e <= 7 { + rex = rexInfo(0).always() + } + default: + panic(fmt.Sprintf("BUG: invalid size %d: %s", size, i.String())) + } + + switch dst.kind { + case operandKindMem: + m := dst.addressMode() + encodeRegMem(c, lp, opcode, 2, src, m, rex) + default: + panic("BUG: invalid operand kind") + } + + case zeros: + r := i.op2.reg() + if r.RegType() == regalloc.RegTypeInt { + i.asAluRmiR(aluRmiROpcodeXor, newOperandReg(r), r, true) + } else { + i.asXmmRmR(sseOpcodePxor, newOperandReg(r), r) + } + i.encode(c) + + case mfence: + // https://www.felixcloutier.com/x86/mfence + c.EmitByte(0x0f) + c.EmitByte(0xae) + c.EmitByte(0xf0) + + default: + panic(fmt.Sprintf("TODO: %v", i.kind)) + } + return +} + +func encodeLoad64(c backend.Compiler, m *amode, rd regalloc.RealReg) { + dst := regEncodings[rd] + encodeRegMem(c, legacyPrefixesNone, 0x8b, 1, dst, m, rexInfo(0).setW()) +} + +func encodeRet(c backend.Compiler) { + c.EmitByte(0xc3) +} + +func encodeEncEnc( + c backend.Compiler, + legPrefixes legacyPrefixes, + opcodes uint32, + opcodeNum uint32, + r uint8, + rm uint8, + rex rexInfo, +) { + legPrefixes.encode(c) + rex.encode(c, r>>3, rm>>3) + + for opcodeNum > 0 { + opcodeNum-- + c.EmitByte(byte((opcodes >> (opcodeNum << 3)) & 0xff)) + } + c.EmitByte(encodeModRM(3, r&7, rm&7)) +} + +func encodeRegReg( + c backend.Compiler, + legPrefixes legacyPrefixes, + opcodes uint32, + opcodeNum uint32, + r regEnc, + rm regEnc, + rex rexInfo, +) { + encodeEncEnc(c, legPrefixes, opcodes, opcodeNum, uint8(r), uint8(rm), rex) +} + +func encodeModRM(mod byte, reg byte, rm byte) byte { + return mod<<6 | reg<<3 | rm +} + +func encodeSIB(shift byte, encIndex byte, encBase byte) byte { + return shift<<6 | encIndex<<3 | encBase +} + +func encodeRegMem( + c backend.Compiler, legPrefixes legacyPrefixes, opcodes uint32, opcodeNum uint32, r regEnc, m *amode, rex rexInfo, +) (needsLabelResolution bool) { + needsLabelResolution = encodeEncMem(c, legPrefixes, opcodes, opcodeNum, uint8(r), m, rex) + return +} + +func encodeEncMem( + c backend.Compiler, legPrefixes legacyPrefixes, opcodes uint32, opcodeNum uint32, r uint8, m *amode, rex rexInfo, +) (needsLabelResolution bool) { + legPrefixes.encode(c) + + const ( + modNoDisplacement = 0b00 + modShortDisplacement = 0b01 + modLongDisplacement = 0b10 + + useSBI = 4 // the encoding of rsp or r12 register. + ) + + switch m.kind() { + case amodeImmReg, amodeImmRBP: + base := m.base.RealReg() + baseEnc := regEncodings[base] + + rex.encode(c, regRexBit(r), baseEnc.rexBit()) + + for opcodeNum > 0 { + opcodeNum-- + c.EmitByte(byte((opcodes >> (opcodeNum << 3)) & 0xff)) + } + + // SIB byte is the last byte of the memory encoding before the displacement + const sibByte = 0x24 // == encodeSIB(0, 4, 4) + + immZero, baseRbp, baseR13 := m.imm32 == 0, base == rbp, base == r13 + short := lower8willSignExtendTo32(m.imm32) + rspOrR12 := base == rsp || base == r12 + + if immZero && !baseRbp && !baseR13 { // rbp or r13 can't be used as base for without displacement encoding. + c.EmitByte(encodeModRM(modNoDisplacement, regEncoding(r), baseEnc.encoding())) + if rspOrR12 { + c.EmitByte(sibByte) + } + } else if short { // Note: this includes the case where m.imm32 == 0 && base == rbp || base == r13. + c.EmitByte(encodeModRM(modShortDisplacement, regEncoding(r), baseEnc.encoding())) + if rspOrR12 { + c.EmitByte(sibByte) + } + c.EmitByte(byte(m.imm32)) + } else { + c.EmitByte(encodeModRM(modLongDisplacement, regEncoding(r), baseEnc.encoding())) + if rspOrR12 { + c.EmitByte(sibByte) + } + c.Emit4Bytes(m.imm32) + } + + case amodeRegRegShift: + base := m.base.RealReg() + baseEnc := regEncodings[base] + index := m.index.RealReg() + indexEnc := regEncodings[index] + + if index == rsp { + panic("BUG: rsp can't be used as index of addressing mode") + } + + rex.encodeForIndex(c, regEnc(r), indexEnc, baseEnc) + + for opcodeNum > 0 { + opcodeNum-- + c.EmitByte(byte((opcodes >> (opcodeNum << 3)) & 0xff)) + } + + immZero, baseRbp, baseR13 := m.imm32 == 0, base == rbp, base == r13 + if immZero && !baseRbp && !baseR13 { // rbp or r13 can't be used as base for without displacement encoding. (curious why? because it's interpreted as RIP relative addressing). + c.EmitByte(encodeModRM(modNoDisplacement, regEncoding(r), useSBI)) + c.EmitByte(encodeSIB(m.shift(), indexEnc.encoding(), baseEnc.encoding())) + } else if lower8willSignExtendTo32(m.imm32) { + c.EmitByte(encodeModRM(modShortDisplacement, regEncoding(r), useSBI)) + c.EmitByte(encodeSIB(m.shift(), indexEnc.encoding(), baseEnc.encoding())) + c.EmitByte(byte(m.imm32)) + } else { + c.EmitByte(encodeModRM(modLongDisplacement, regEncoding(r), useSBI)) + c.EmitByte(encodeSIB(m.shift(), indexEnc.encoding(), baseEnc.encoding())) + c.Emit4Bytes(m.imm32) + } + + case amodeRipRel: + rex.encode(c, regRexBit(r), 0) + for opcodeNum > 0 { + opcodeNum-- + c.EmitByte(byte((opcodes >> (opcodeNum << 3)) & 0xff)) + } + + // Indicate "LEAQ [RIP + 32bit displacement]. + // https://wiki.osdev.org/X86-64_Instruction_Encoding#32.2F64-bit_addressing + c.EmitByte(encodeModRM(0b00, regEncoding(r), 0b101)) + + // This will be resolved later, so we just emit a placeholder. + needsLabelResolution = true + c.Emit4Bytes(0) + + default: + panic("BUG: invalid addressing mode") + } + return +} + +const ( + rexEncodingDefault byte = 0x40 + rexEncodingW = rexEncodingDefault | 0x08 +) + +// rexInfo is a bit set to indicate: +// +// 0x01: W bit must be cleared. +// 0x02: REX prefix must be emitted. +type rexInfo byte + +func (ri rexInfo) setW() rexInfo { + return ri | 0x01 +} + +func (ri rexInfo) clearW() rexInfo { + return ri & 0x02 +} + +func (ri rexInfo) always() rexInfo { + return ri | 0x02 +} + +func (ri rexInfo) notAlways() rexInfo { //nolint + return ri & 0x01 +} + +func (ri rexInfo) encode(c backend.Compiler, r uint8, b uint8) { + var w byte = 0 + if ri&0x01 != 0 { + w = 0x01 + } + rex := rexEncodingDefault | w<<3 | r<<2 | b + if rex != rexEncodingDefault || ri&0x02 != 0 { + c.EmitByte(rex) + } +} + +func (ri rexInfo) encodeForIndex(c backend.Compiler, encR regEnc, encIndex regEnc, encBase regEnc) { + var w byte = 0 + if ri&0x01 != 0 { + w = 0x01 + } + r := encR.rexBit() + x := encIndex.rexBit() + b := encBase.rexBit() + rex := byte(0x40) | w<<3 | r<<2 | x<<1 | b + if rex != 0x40 || ri&0x02 != 0 { + c.EmitByte(rex) + } +} + +type regEnc byte + +func (r regEnc) rexBit() byte { + return regRexBit(byte(r)) +} + +func (r regEnc) encoding() byte { + return regEncoding(byte(r)) +} + +func regRexBit(r byte) byte { + return r >> 3 +} + +func regEncoding(r byte) byte { + return r & 0x07 +} + +var regEncodings = [...]regEnc{ + rax: 0b000, + rcx: 0b001, + rdx: 0b010, + rbx: 0b011, + rsp: 0b100, + rbp: 0b101, + rsi: 0b110, + rdi: 0b111, + r8: 0b1000, + r9: 0b1001, + r10: 0b1010, + r11: 0b1011, + r12: 0b1100, + r13: 0b1101, + r14: 0b1110, + r15: 0b1111, + xmm0: 0b000, + xmm1: 0b001, + xmm2: 0b010, + xmm3: 0b011, + xmm4: 0b100, + xmm5: 0b101, + xmm6: 0b110, + xmm7: 0b111, + xmm8: 0b1000, + xmm9: 0b1001, + xmm10: 0b1010, + xmm11: 0b1011, + xmm12: 0b1100, + xmm13: 0b1101, + xmm14: 0b1110, + xmm15: 0b1111, +} + +type legacyPrefixes byte + +const ( + legacyPrefixesNone legacyPrefixes = iota + legacyPrefixes0x66 + legacyPrefixes0xF0 + legacyPrefixes0x660xF0 + legacyPrefixes0xF2 + legacyPrefixes0xF3 +) + +func (p legacyPrefixes) encode(c backend.Compiler) { + switch p { + case legacyPrefixesNone: + case legacyPrefixes0x66: + c.EmitByte(0x66) + case legacyPrefixes0xF0: + c.EmitByte(0xf0) + case legacyPrefixes0x660xF0: + c.EmitByte(0x66) + c.EmitByte(0xf0) + case legacyPrefixes0xF2: + c.EmitByte(0xf2) + case legacyPrefixes0xF3: + c.EmitByte(0xf3) + default: + panic("BUG: invalid legacy prefix") + } +} + +func lower32willSignExtendTo64(x uint64) bool { + xs := int64(x) + return xs == int64(uint64(int32(xs))) +} + +func lower8willSignExtendTo32(x uint32) bool { + xs := int32(x) + return xs == ((xs << 24) >> 24) +} diff --git a/vendor/github.com/tetratelabs/wazero/internal/engine/wazevo/backend/isa/amd64/lower_constant.go b/vendor/github.com/tetratelabs/wazero/internal/engine/wazevo/backend/isa/amd64/lower_constant.go new file mode 100644 index 000000000..55d05ef63 --- /dev/null +++ b/vendor/github.com/tetratelabs/wazero/internal/engine/wazevo/backend/isa/amd64/lower_constant.go @@ -0,0 +1,71 @@ +package amd64 + +import ( + "github.com/tetratelabs/wazero/internal/engine/wazevo/backend/regalloc" + "github.com/tetratelabs/wazero/internal/engine/wazevo/ssa" +) + +// lowerConstant allocates a new VReg and inserts the instruction to load the constant value. +func (m *machine) lowerConstant(instr *ssa.Instruction) (vr regalloc.VReg) { + val := instr.Return() + valType := val.Type() + + vr = m.c.AllocateVReg(valType) + m.insertLoadConstant(instr, vr) + return +} + +// InsertLoadConstantBlockArg implements backend.Machine. +func (m *machine) InsertLoadConstantBlockArg(instr *ssa.Instruction, vr regalloc.VReg) { + m.insertLoadConstant(instr, vr) +} + +func (m *machine) insertLoadConstant(instr *ssa.Instruction, vr regalloc.VReg) { + val := instr.Return() + valType := val.Type() + v := instr.ConstantVal() + + bits := valType.Bits() + if bits < 64 { // Clear the redundant bits just in case it's unexpectedly sign-extended, etc. + v = v & ((1 << valType.Bits()) - 1) + } + + switch valType { + case ssa.TypeF32, ssa.TypeF64: + m.lowerFconst(vr, v, bits == 64) + case ssa.TypeI32, ssa.TypeI64: + m.lowerIconst(vr, v, bits == 64) + default: + panic("BUG") + } +} + +func (m *machine) lowerFconst(dst regalloc.VReg, c uint64, _64 bool) { + if c == 0 { + xor := m.allocateInstr().asZeros(dst) + m.insert(xor) + } else { + var tmpType ssa.Type + if _64 { + tmpType = ssa.TypeI64 + } else { + tmpType = ssa.TypeI32 + } + tmpInt := m.c.AllocateVReg(tmpType) + loadToGP := m.allocateInstr().asImm(tmpInt, c, _64) + m.insert(loadToGP) + + movToXmm := m.allocateInstr().asGprToXmm(sseOpcodeMovq, newOperandReg(tmpInt), dst, _64) + m.insert(movToXmm) + } +} + +func (m *machine) lowerIconst(dst regalloc.VReg, c uint64, _64 bool) { + i := m.allocateInstr() + if c == 0 { + i.asZeros(dst) + } else { + i.asImm(dst, c, _64) + } + m.insert(i) +} diff --git a/vendor/github.com/tetratelabs/wazero/internal/engine/wazevo/backend/isa/amd64/lower_mem.go b/vendor/github.com/tetratelabs/wazero/internal/engine/wazevo/backend/isa/amd64/lower_mem.go new file mode 100644 index 000000000..bee673d25 --- /dev/null +++ b/vendor/github.com/tetratelabs/wazero/internal/engine/wazevo/backend/isa/amd64/lower_mem.go @@ -0,0 +1,187 @@ +package amd64 + +import ( + "fmt" + + "github.com/tetratelabs/wazero/internal/engine/wazevo/backend" + "github.com/tetratelabs/wazero/internal/engine/wazevo/backend/regalloc" + "github.com/tetratelabs/wazero/internal/engine/wazevo/ssa" +) + +var addendsMatchOpcodes = [...]ssa.Opcode{ssa.OpcodeUExtend, ssa.OpcodeSExtend, ssa.OpcodeIadd, ssa.OpcodeIconst, ssa.OpcodeIshl} + +type addend struct { + r regalloc.VReg + off int64 + shift byte +} + +func (a addend) String() string { + return fmt.Sprintf("addend{r=%s, off=%d, shift=%d}", a.r, a.off, a.shift) +} + +// lowerToAddressMode converts a pointer to an addressMode that can be used as an operand for load/store instructions. +func (m *machine) lowerToAddressMode(ptr ssa.Value, offsetBase uint32) (am *amode) { + def := m.c.ValueDefinition(ptr) + + if offsetBase&0x80000000 != 0 { + // Special casing the huge base offset whose MSB is set. In x64, the immediate is always + // sign-extended, but our IR semantics requires the offset base is always unsigned. + // Note that this should be extremely rare or even this shouldn't hit in the real application, + // therefore we don't need to optimize this case in my opinion. + + a := m.lowerAddend(def) + off64 := a.off + int64(offsetBase) + offsetBaseReg := m.c.AllocateVReg(ssa.TypeI64) + m.lowerIconst(offsetBaseReg, uint64(off64), true) + if a.r != regalloc.VRegInvalid { + return m.newAmodeRegRegShift(0, offsetBaseReg, a.r, a.shift) + } else { + return m.newAmodeImmReg(0, offsetBaseReg) + } + } + + if op := m.c.MatchInstrOneOf(def, addendsMatchOpcodes[:]); op == ssa.OpcodeIadd { + add := def.Instr + x, y := add.Arg2() + xDef, yDef := m.c.ValueDefinition(x), m.c.ValueDefinition(y) + ax := m.lowerAddend(xDef) + ay := m.lowerAddend(yDef) + add.MarkLowered() + return m.lowerAddendsToAmode(ax, ay, offsetBase) + } else { + // If it is not an Iadd, then we lower the one addend. + a := m.lowerAddend(def) + // off is always 0 if r is valid. + if a.r != regalloc.VRegInvalid { + if a.shift != 0 { + tmpReg := m.c.AllocateVReg(ssa.TypeI64) + m.lowerIconst(tmpReg, 0, true) + return m.newAmodeRegRegShift(offsetBase, tmpReg, a.r, a.shift) + } + return m.newAmodeImmReg(offsetBase, a.r) + } else { + off64 := a.off + int64(offsetBase) + tmpReg := m.c.AllocateVReg(ssa.TypeI64) + m.lowerIconst(tmpReg, uint64(off64), true) + return m.newAmodeImmReg(0, tmpReg) + } + } +} + +func (m *machine) lowerAddendsToAmode(x, y addend, offBase uint32) *amode { + if x.r != regalloc.VRegInvalid && x.off != 0 || y.r != regalloc.VRegInvalid && y.off != 0 { + panic("invalid input") + } + + u64 := uint64(x.off+y.off) + uint64(offBase) + if u64 != 0 { + if _, ok := asImm32(u64, false); !ok { + tmpReg := m.c.AllocateVReg(ssa.TypeI64) + m.lowerIconst(tmpReg, u64, true) + // Blank u64 as it has been already lowered. + u64 = 0 + + if x.r == regalloc.VRegInvalid { + x.r = tmpReg + } else if y.r == regalloc.VRegInvalid { + y.r = tmpReg + } else { + // We already know that either rx or ry is invalid, + // so we overwrite it with the temporary register. + panic("BUG") + } + } + } + + u32 := uint32(u64) + switch { + // We assume rx, ry are valid iff offx, offy are 0. + case x.r != regalloc.VRegInvalid && y.r != regalloc.VRegInvalid: + switch { + case x.shift != 0 && y.shift != 0: + // Cannot absorb two shifted registers, must lower one to a shift instruction. + shifted := m.allocateInstr() + shifted.asShiftR(shiftROpShiftLeft, newOperandImm32(uint32(x.shift)), x.r, true) + m.insert(shifted) + + return m.newAmodeRegRegShift(u32, x.r, y.r, y.shift) + case x.shift != 0 && y.shift == 0: + // Swap base and index. + x, y = y, x + fallthrough + default: + return m.newAmodeRegRegShift(u32, x.r, y.r, y.shift) + } + case x.r == regalloc.VRegInvalid && y.r != regalloc.VRegInvalid: + x, y = y, x + fallthrough + case x.r != regalloc.VRegInvalid && y.r == regalloc.VRegInvalid: + if x.shift != 0 { + zero := m.c.AllocateVReg(ssa.TypeI64) + m.lowerIconst(zero, 0, true) + return m.newAmodeRegRegShift(u32, zero, x.r, x.shift) + } + return m.newAmodeImmReg(u32, x.r) + default: // Both are invalid: use the offset. + tmpReg := m.c.AllocateVReg(ssa.TypeI64) + m.lowerIconst(tmpReg, u64, true) + return m.newAmodeImmReg(0, tmpReg) + } +} + +func (m *machine) lowerAddend(x *backend.SSAValueDefinition) addend { + if x.IsFromBlockParam() { + return addend{x.BlkParamVReg, 0, 0} + } + // Ensure the addend is not referenced in multiple places; we will discard nested Iadds. + op := m.c.MatchInstrOneOf(x, addendsMatchOpcodes[:]) + if op != ssa.OpcodeInvalid && op != ssa.OpcodeIadd { + return m.lowerAddendFromInstr(x.Instr) + } + p := m.getOperand_Reg(x) + return addend{p.reg(), 0, 0} +} + +// lowerAddendFromInstr takes an instruction returns a Vreg and an offset that can be used in an address mode. +// The Vreg is regalloc.VRegInvalid if the addend cannot be lowered to a register. +// The offset is 0 if the addend can be lowered to a register. +func (m *machine) lowerAddendFromInstr(instr *ssa.Instruction) addend { + instr.MarkLowered() + switch op := instr.Opcode(); op { + case ssa.OpcodeIconst: + u64 := instr.ConstantVal() + if instr.Return().Type().Bits() == 32 { + return addend{regalloc.VRegInvalid, int64(int32(u64)), 0} // sign-extend. + } else { + return addend{regalloc.VRegInvalid, int64(u64), 0} + } + case ssa.OpcodeUExtend, ssa.OpcodeSExtend: + input := instr.Arg() + inputDef := m.c.ValueDefinition(input) + if input.Type().Bits() != 32 { + panic("BUG: invalid input type " + input.Type().String()) + } + constInst := inputDef.IsFromInstr() && inputDef.Instr.Constant() + switch { + case constInst && op == ssa.OpcodeSExtend: + return addend{regalloc.VRegInvalid, int64(uint32(inputDef.Instr.ConstantVal())), 0} + case constInst && op == ssa.OpcodeUExtend: + return addend{regalloc.VRegInvalid, int64(int32(inputDef.Instr.ConstantVal())), 0} // sign-extend! + default: + r := m.getOperand_Reg(inputDef) + return addend{r.reg(), 0, 0} + } + case ssa.OpcodeIshl: + // If the addend is a shift, we can only handle it if the shift amount is a constant. + x, amount := instr.Arg2() + amountDef := m.c.ValueDefinition(amount) + if amountDef.IsFromInstr() && amountDef.Instr.Constant() && amountDef.Instr.ConstantVal() <= 3 { + r := m.getOperand_Reg(m.c.ValueDefinition(x)) + return addend{r.reg(), 0, uint8(amountDef.Instr.ConstantVal())} + } + r := m.getOperand_Reg(m.c.ValueDefinition(x)) + return addend{r.reg(), 0, 0} + } + panic("BUG: invalid opcode") +} diff --git a/vendor/github.com/tetratelabs/wazero/internal/engine/wazevo/backend/isa/amd64/machine.go b/vendor/github.com/tetratelabs/wazero/internal/engine/wazevo/backend/isa/amd64/machine.go new file mode 100644 index 000000000..310ad2203 --- /dev/null +++ b/vendor/github.com/tetratelabs/wazero/internal/engine/wazevo/backend/isa/amd64/machine.go @@ -0,0 +1,3611 @@ +package amd64 + +import ( + "context" + "encoding/binary" + "fmt" + "math" + "strings" + + "github.com/tetratelabs/wazero/internal/engine/wazevo/backend" + "github.com/tetratelabs/wazero/internal/engine/wazevo/backend/regalloc" + "github.com/tetratelabs/wazero/internal/engine/wazevo/ssa" + "github.com/tetratelabs/wazero/internal/engine/wazevo/wazevoapi" + "github.com/tetratelabs/wazero/internal/platform" +) + +// NewBackend returns a new backend for arm64. +func NewBackend() backend.Machine { + ectx := backend.NewExecutableContextT[instruction]( + resetInstruction, + setNext, + setPrev, + asNop, + ) + return &machine{ + ectx: ectx, + cpuFeatures: platform.CpuFeatures, + regAlloc: regalloc.NewAllocator(regInfo), + spillSlots: map[regalloc.VRegID]int64{}, + amodePool: wazevoapi.NewPool[amode](nil), + constSwizzleMaskConstIndex: -1, + constSqmulRoundSatIndex: -1, + constI8x16SHLMaskTableIndex: -1, + constI8x16LogicalSHRMaskTableIndex: -1, + constF64x2CvtFromIMaskIndex: -1, + constTwop52Index: -1, + constI32sMaxOnF64x2Index: -1, + constI32uMaxOnF64x2Index: -1, + constAllOnesI8x16Index: -1, + constAllOnesI16x8Index: -1, + constExtAddPairwiseI16x8uMask1Index: -1, + constExtAddPairwiseI16x8uMask2Index: -1, + } +} + +type ( + // machine implements backend.Machine for amd64. + machine struct { + c backend.Compiler + ectx *backend.ExecutableContextT[instruction] + stackBoundsCheckDisabled bool + + amodePool wazevoapi.Pool[amode] + + cpuFeatures platform.CpuFeatureFlags + + regAlloc regalloc.Allocator + regAllocFn *backend.RegAllocFunction[*instruction, *machine] + regAllocStarted bool + + spillSlotSize int64 + spillSlots map[regalloc.VRegID]int64 + currentABI *backend.FunctionABI + clobberedRegs []regalloc.VReg + + maxRequiredStackSizeForCalls int64 + + labelResolutionPends []labelResolutionPend + + jmpTableTargets [][]uint32 + consts []_const + + constSwizzleMaskConstIndex, constSqmulRoundSatIndex, + constI8x16SHLMaskTableIndex, constI8x16LogicalSHRMaskTableIndex, + constF64x2CvtFromIMaskIndex, constTwop52Index, + constI32sMaxOnF64x2Index, constI32uMaxOnF64x2Index, + constAllOnesI8x16Index, constAllOnesI16x8Index, + constExtAddPairwiseI16x8uMask1Index, constExtAddPairwiseI16x8uMask2Index int + } + + _const struct { + lo, hi uint64 + _var []byte + label *labelPosition + } + + labelResolutionPend struct { + instr *instruction + instrOffset int64 + // imm32Offset is the offset of the last 4 bytes of the instruction. + imm32Offset int64 + } + + labelPosition = backend.LabelPosition[instruction] +) + +func (m *machine) getOrAllocateConstLabel(i *int, _var []byte) backend.Label { + index := *i + if index == -1 { + label := m.allocateLabel() + index = len(m.consts) + m.consts = append(m.consts, _const{ + _var: _var, + label: label, + }) + *i = index + } + return m.consts[index].label.L +} + +// Reset implements backend.Machine. +func (m *machine) Reset() { + m.consts = m.consts[:0] + m.clobberedRegs = m.clobberedRegs[:0] + for key := range m.spillSlots { + m.clobberedRegs = append(m.clobberedRegs, regalloc.VReg(key)) + } + for _, key := range m.clobberedRegs { + delete(m.spillSlots, regalloc.VRegID(key)) + } + + m.stackBoundsCheckDisabled = false + m.ectx.Reset() + + m.regAllocFn.Reset() + m.regAlloc.Reset() + m.regAllocStarted = false + m.clobberedRegs = m.clobberedRegs[:0] + + m.spillSlotSize = 0 + m.maxRequiredStackSizeForCalls = 0 + + m.amodePool.Reset() + m.jmpTableTargets = m.jmpTableTargets[:0] + m.constSwizzleMaskConstIndex = -1 + m.constSqmulRoundSatIndex = -1 + m.constI8x16SHLMaskTableIndex = -1 + m.constI8x16LogicalSHRMaskTableIndex = -1 + m.constF64x2CvtFromIMaskIndex = -1 + m.constTwop52Index = -1 + m.constI32sMaxOnF64x2Index = -1 + m.constI32uMaxOnF64x2Index = -1 + m.constAllOnesI8x16Index = -1 + m.constAllOnesI16x8Index = -1 + m.constExtAddPairwiseI16x8uMask1Index = -1 + m.constExtAddPairwiseI16x8uMask2Index = -1 +} + +// ExecutableContext implements backend.Machine. +func (m *machine) ExecutableContext() backend.ExecutableContext { return m.ectx } + +// DisableStackCheck implements backend.Machine. +func (m *machine) DisableStackCheck() { m.stackBoundsCheckDisabled = true } + +// SetCompiler implements backend.Machine. +func (m *machine) SetCompiler(c backend.Compiler) { + m.c = c + m.regAllocFn = backend.NewRegAllocFunction[*instruction, *machine](m, c.SSABuilder(), c) +} + +// SetCurrentABI implements backend.Machine. +func (m *machine) SetCurrentABI(abi *backend.FunctionABI) { + m.currentABI = abi +} + +// RegAlloc implements backend.Machine. +func (m *machine) RegAlloc() { + rf := m.regAllocFn + for _, pos := range m.ectx.OrderedBlockLabels { + rf.AddBlock(pos.SB, pos.L, pos.Begin, pos.End) + } + + m.regAllocStarted = true + m.regAlloc.DoAllocation(rf) + // Now that we know the final spill slot size, we must align spillSlotSize to 16 bytes. + m.spillSlotSize = (m.spillSlotSize + 15) &^ 15 +} + +// InsertReturn implements backend.Machine. +func (m *machine) InsertReturn() { + i := m.allocateInstr().asRet() + m.insert(i) +} + +// LowerSingleBranch implements backend.Machine. +func (m *machine) LowerSingleBranch(b *ssa.Instruction) { + ectx := m.ectx + switch b.Opcode() { + case ssa.OpcodeJump: + _, _, targetBlk := b.BranchData() + if b.IsFallthroughJump() { + return + } + jmp := m.allocateInstr() + target := ectx.GetOrAllocateSSABlockLabel(targetBlk) + if target == backend.LabelReturn { + jmp.asRet() + } else { + jmp.asJmp(newOperandLabel(target)) + } + m.insert(jmp) + case ssa.OpcodeBrTable: + index, target := b.BrTableData() + m.lowerBrTable(index, target) + default: + panic("BUG: unexpected branch opcode" + b.Opcode().String()) + } +} + +func (m *machine) addJmpTableTarget(targets []ssa.BasicBlock) (index int) { + // TODO: reuse the slice! + labels := make([]uint32, len(targets)) + for j, target := range targets { + labels[j] = uint32(m.ectx.GetOrAllocateSSABlockLabel(target)) + } + index = len(m.jmpTableTargets) + m.jmpTableTargets = append(m.jmpTableTargets, labels) + return +} + +var condBranchMatches = [...]ssa.Opcode{ssa.OpcodeIcmp, ssa.OpcodeFcmp} + +func (m *machine) lowerBrTable(index ssa.Value, targets []ssa.BasicBlock) { + _v := m.getOperand_Reg(m.c.ValueDefinition(index)) + v := m.copyToTmp(_v.reg()) + + // First, we need to do the bounds check. + maxIndex := m.c.AllocateVReg(ssa.TypeI32) + m.lowerIconst(maxIndex, uint64(len(targets)-1), false) + cmp := m.allocateInstr().asCmpRmiR(true, newOperandReg(maxIndex), v, false) + m.insert(cmp) + + // Then do the conditional move maxIndex to v if v > maxIndex. + cmov := m.allocateInstr().asCmove(condNB, newOperandReg(maxIndex), v, false) + m.insert(cmov) + + // Now that v has the correct index. Load the address of the jump table into the addr. + addr := m.c.AllocateVReg(ssa.TypeI64) + leaJmpTableAddr := m.allocateInstr() + m.insert(leaJmpTableAddr) + + // Then add the target's offset into jmpTableAddr. + loadTargetOffsetFromJmpTable := m.allocateInstr().asAluRmiR(aluRmiROpcodeAdd, + // Shift by 3 because each entry is 8 bytes. + newOperandMem(m.newAmodeRegRegShift(0, addr, v, 3)), addr, true) + m.insert(loadTargetOffsetFromJmpTable) + + // Now ready to jump. + jmp := m.allocateInstr().asJmp(newOperandReg(addr)) + m.insert(jmp) + + jmpTableBegin, jmpTableBeginLabel := m.allocateBrTarget() + m.insert(jmpTableBegin) + leaJmpTableAddr.asLEA(newOperandLabel(jmpTableBeginLabel), addr) + + jmpTable := m.allocateInstr() + targetSliceIndex := m.addJmpTableTarget(targets) + jmpTable.asJmpTableSequence(targetSliceIndex, len(targets)) + m.insert(jmpTable) +} + +// LowerConditionalBranch implements backend.Machine. +func (m *machine) LowerConditionalBranch(b *ssa.Instruction) { + exctx := m.ectx + cval, args, targetBlk := b.BranchData() + if len(args) > 0 { + panic(fmt.Sprintf( + "conditional branch shouldn't have args; likely a bug in critical edge splitting: from %s to %s", + exctx.CurrentSSABlk, + targetBlk, + )) + } + + target := exctx.GetOrAllocateSSABlockLabel(targetBlk) + cvalDef := m.c.ValueDefinition(cval) + + switch m.c.MatchInstrOneOf(cvalDef, condBranchMatches[:]) { + case ssa.OpcodeIcmp: + cvalInstr := cvalDef.Instr + x, y, c := cvalInstr.IcmpData() + + cc := condFromSSAIntCmpCond(c) + if b.Opcode() == ssa.OpcodeBrz { + cc = cc.invert() + } + + // First, perform the comparison and set the flag. + xd, yd := m.c.ValueDefinition(x), m.c.ValueDefinition(y) + if !m.tryLowerBandToFlag(xd, yd) { + m.lowerIcmpToFlag(xd, yd, x.Type() == ssa.TypeI64) + } + + // Then perform the conditional branch. + m.insert(m.allocateInstr().asJmpIf(cc, newOperandLabel(target))) + cvalDef.Instr.MarkLowered() + case ssa.OpcodeFcmp: + cvalInstr := cvalDef.Instr + + f1, f2, and := m.lowerFcmpToFlags(cvalInstr) + isBrz := b.Opcode() == ssa.OpcodeBrz + if isBrz { + f1 = f1.invert() + } + if f2 == condInvalid { + m.insert(m.allocateInstr().asJmpIf(f1, newOperandLabel(target))) + } else { + if isBrz { + f2 = f2.invert() + and = !and + } + jmp1, jmp2 := m.allocateInstr(), m.allocateInstr() + m.insert(jmp1) + m.insert(jmp2) + notTaken, notTakenLabel := m.allocateBrTarget() + m.insert(notTaken) + if and { + jmp1.asJmpIf(f1.invert(), newOperandLabel(notTakenLabel)) + jmp2.asJmpIf(f2, newOperandLabel(target)) + } else { + jmp1.asJmpIf(f1, newOperandLabel(target)) + jmp2.asJmpIf(f2, newOperandLabel(target)) + } + } + + cvalDef.Instr.MarkLowered() + default: + v := m.getOperand_Reg(cvalDef) + + var cc cond + if b.Opcode() == ssa.OpcodeBrz { + cc = condZ + } else { + cc = condNZ + } + + // Perform test %v, %v to set the flag. + cmp := m.allocateInstr().asCmpRmiR(false, v, v.reg(), false) + m.insert(cmp) + m.insert(m.allocateInstr().asJmpIf(cc, newOperandLabel(target))) + } +} + +// LowerInstr implements backend.Machine. +func (m *machine) LowerInstr(instr *ssa.Instruction) { + if l := instr.SourceOffset(); l.Valid() { + info := m.allocateInstr().asEmitSourceOffsetInfo(l) + m.insert(info) + } + + switch op := instr.Opcode(); op { + case ssa.OpcodeBrz, ssa.OpcodeBrnz, ssa.OpcodeJump, ssa.OpcodeBrTable: + panic("BUG: branching instructions are handled by LowerBranches") + case ssa.OpcodeReturn: + panic("BUG: return must be handled by backend.Compiler") + case ssa.OpcodeIconst, ssa.OpcodeF32const, ssa.OpcodeF64const: // Constant instructions are inlined. + case ssa.OpcodeCall, ssa.OpcodeCallIndirect: + m.lowerCall(instr) + case ssa.OpcodeStore, ssa.OpcodeIstore8, ssa.OpcodeIstore16, ssa.OpcodeIstore32: + m.lowerStore(instr) + case ssa.OpcodeIadd: + m.lowerAluRmiROp(instr, aluRmiROpcodeAdd) + case ssa.OpcodeIsub: + m.lowerAluRmiROp(instr, aluRmiROpcodeSub) + case ssa.OpcodeImul: + m.lowerAluRmiROp(instr, aluRmiROpcodeMul) + case ssa.OpcodeSdiv, ssa.OpcodeUdiv, ssa.OpcodeSrem, ssa.OpcodeUrem: + isDiv := op == ssa.OpcodeSdiv || op == ssa.OpcodeUdiv + isSigned := op == ssa.OpcodeSdiv || op == ssa.OpcodeSrem + m.lowerIDivRem(instr, isDiv, isSigned) + case ssa.OpcodeBand: + m.lowerAluRmiROp(instr, aluRmiROpcodeAnd) + case ssa.OpcodeBor: + m.lowerAluRmiROp(instr, aluRmiROpcodeOr) + case ssa.OpcodeBxor: + m.lowerAluRmiROp(instr, aluRmiROpcodeXor) + case ssa.OpcodeIshl: + m.lowerShiftR(instr, shiftROpShiftLeft) + case ssa.OpcodeSshr: + m.lowerShiftR(instr, shiftROpShiftRightArithmetic) + case ssa.OpcodeUshr: + m.lowerShiftR(instr, shiftROpShiftRightLogical) + case ssa.OpcodeRotl: + m.lowerShiftR(instr, shiftROpRotateLeft) + case ssa.OpcodeRotr: + m.lowerShiftR(instr, shiftROpRotateRight) + case ssa.OpcodeClz: + m.lowerClz(instr) + case ssa.OpcodeCtz: + m.lowerCtz(instr) + case ssa.OpcodePopcnt: + m.lowerUnaryRmR(instr, unaryRmROpcodePopcnt) + case ssa.OpcodeFadd, ssa.OpcodeFsub, ssa.OpcodeFmul, ssa.OpcodeFdiv: + m.lowerXmmRmR(instr) + case ssa.OpcodeFabs: + m.lowerFabsFneg(instr) + case ssa.OpcodeFneg: + m.lowerFabsFneg(instr) + case ssa.OpcodeCeil: + m.lowerRound(instr, roundingModeUp) + case ssa.OpcodeFloor: + m.lowerRound(instr, roundingModeDown) + case ssa.OpcodeTrunc: + m.lowerRound(instr, roundingModeZero) + case ssa.OpcodeNearest: + m.lowerRound(instr, roundingModeNearest) + case ssa.OpcodeFmin, ssa.OpcodeFmax: + m.lowerFminFmax(instr) + case ssa.OpcodeFcopysign: + m.lowerFcopysign(instr) + case ssa.OpcodeBitcast: + m.lowerBitcast(instr) + case ssa.OpcodeSqrt: + m.lowerSqrt(instr) + case ssa.OpcodeFpromote: + v := instr.Arg() + rn := m.getOperand_Reg(m.c.ValueDefinition(v)) + rd := m.c.VRegOf(instr.Return()) + cnt := m.allocateInstr() + cnt.asXmmUnaryRmR(sseOpcodeCvtss2sd, rn, rd) + m.insert(cnt) + case ssa.OpcodeFdemote: + v := instr.Arg() + rn := m.getOperand_Reg(m.c.ValueDefinition(v)) + rd := m.c.VRegOf(instr.Return()) + cnt := m.allocateInstr() + cnt.asXmmUnaryRmR(sseOpcodeCvtsd2ss, rn, rd) + m.insert(cnt) + case ssa.OpcodeFcvtToSint, ssa.OpcodeFcvtToSintSat: + x, ctx := instr.Arg2() + rn := m.getOperand_Reg(m.c.ValueDefinition(x)) + rd := m.c.VRegOf(instr.Return()) + ctxVReg := m.c.VRegOf(ctx) + m.lowerFcvtToSint(ctxVReg, rn.reg(), rd, x.Type() == ssa.TypeF64, + instr.Return().Type().Bits() == 64, op == ssa.OpcodeFcvtToSintSat) + case ssa.OpcodeFcvtToUint, ssa.OpcodeFcvtToUintSat: + x, ctx := instr.Arg2() + rn := m.getOperand_Reg(m.c.ValueDefinition(x)) + rd := m.c.VRegOf(instr.Return()) + ctxVReg := m.c.VRegOf(ctx) + m.lowerFcvtToUint(ctxVReg, rn.reg(), rd, x.Type() == ssa.TypeF64, + instr.Return().Type().Bits() == 64, op == ssa.OpcodeFcvtToUintSat) + case ssa.OpcodeFcvtFromSint: + x := instr.Arg() + rn := m.getOperand_Reg(m.c.ValueDefinition(x)) + rd := newOperandReg(m.c.VRegOf(instr.Return())) + m.lowerFcvtFromSint(rn, rd, + x.Type() == ssa.TypeI64, instr.Return().Type().Bits() == 64) + case ssa.OpcodeFcvtFromUint: + x := instr.Arg() + rn := m.getOperand_Reg(m.c.ValueDefinition(x)) + rd := newOperandReg(m.c.VRegOf(instr.Return())) + m.lowerFcvtFromUint(rn, rd, x.Type() == ssa.TypeI64, + instr.Return().Type().Bits() == 64) + case ssa.OpcodeVanyTrue: + m.lowerVanyTrue(instr) + case ssa.OpcodeVallTrue: + m.lowerVallTrue(instr) + case ssa.OpcodeVhighBits: + m.lowerVhighBits(instr) + case ssa.OpcodeVbnot: + m.lowerVbnot(instr) + case ssa.OpcodeVband: + x, y := instr.Arg2() + m.lowerVbBinOp(sseOpcodePand, x, y, instr.Return()) + case ssa.OpcodeVbor: + x, y := instr.Arg2() + m.lowerVbBinOp(sseOpcodePor, x, y, instr.Return()) + case ssa.OpcodeVbxor: + x, y := instr.Arg2() + m.lowerVbBinOp(sseOpcodePxor, x, y, instr.Return()) + case ssa.OpcodeVbandnot: + m.lowerVbandnot(instr, sseOpcodePandn) + case ssa.OpcodeVbitselect: + m.lowerVbitselect(instr) + case ssa.OpcodeVIadd: + x, y, lane := instr.Arg2WithLane() + var vecOp sseOpcode + switch lane { + case ssa.VecLaneI8x16: + vecOp = sseOpcodePaddb + case ssa.VecLaneI16x8: + vecOp = sseOpcodePaddw + case ssa.VecLaneI32x4: + vecOp = sseOpcodePaddd + case ssa.VecLaneI64x2: + vecOp = sseOpcodePaddq + } + m.lowerVbBinOp(vecOp, x, y, instr.Return()) + + case ssa.OpcodeVSaddSat: + x, y, lane := instr.Arg2WithLane() + var vecOp sseOpcode + switch lane { + case ssa.VecLaneI8x16: + vecOp = sseOpcodePaddsb + case ssa.VecLaneI16x8: + vecOp = sseOpcodePaddsw + } + m.lowerVbBinOp(vecOp, x, y, instr.Return()) + + case ssa.OpcodeVUaddSat: + x, y, lane := instr.Arg2WithLane() + var vecOp sseOpcode + switch lane { + case ssa.VecLaneI8x16: + vecOp = sseOpcodePaddusb + case ssa.VecLaneI16x8: + vecOp = sseOpcodePaddusw + } + m.lowerVbBinOp(vecOp, x, y, instr.Return()) + + case ssa.OpcodeVIsub: + x, y, lane := instr.Arg2WithLane() + var vecOp sseOpcode + switch lane { + case ssa.VecLaneI8x16: + vecOp = sseOpcodePsubb + case ssa.VecLaneI16x8: + vecOp = sseOpcodePsubw + case ssa.VecLaneI32x4: + vecOp = sseOpcodePsubd + case ssa.VecLaneI64x2: + vecOp = sseOpcodePsubq + } + m.lowerVbBinOp(vecOp, x, y, instr.Return()) + + case ssa.OpcodeVSsubSat: + x, y, lane := instr.Arg2WithLane() + var vecOp sseOpcode + switch lane { + case ssa.VecLaneI8x16: + vecOp = sseOpcodePsubsb + case ssa.VecLaneI16x8: + vecOp = sseOpcodePsubsw + } + m.lowerVbBinOp(vecOp, x, y, instr.Return()) + + case ssa.OpcodeVUsubSat: + x, y, lane := instr.Arg2WithLane() + var vecOp sseOpcode + switch lane { + case ssa.VecLaneI8x16: + vecOp = sseOpcodePsubusb + case ssa.VecLaneI16x8: + vecOp = sseOpcodePsubusw + } + m.lowerVbBinOp(vecOp, x, y, instr.Return()) + + case ssa.OpcodeVImul: + m.lowerVImul(instr) + case ssa.OpcodeVIneg: + x, lane := instr.ArgWithLane() + rn := m.getOperand_Reg(m.c.ValueDefinition(x)) + rd := m.c.VRegOf(instr.Return()) + var vecOp sseOpcode + switch lane { + case ssa.VecLaneI8x16: + vecOp = sseOpcodePsubb + case ssa.VecLaneI16x8: + vecOp = sseOpcodePsubw + case ssa.VecLaneI32x4: + vecOp = sseOpcodePsubd + case ssa.VecLaneI64x2: + vecOp = sseOpcodePsubq + default: + panic("BUG") + } + + tmp := m.c.AllocateVReg(ssa.TypeV128) + m.insert(m.allocateInstr().asZeros(tmp)) + + i := m.allocateInstr() + i.asXmmRmR(vecOp, rn, tmp) + m.insert(i) + + m.copyTo(tmp, rd) + case ssa.OpcodeVFadd: + x, y, lane := instr.Arg2WithLane() + var vecOp sseOpcode + switch lane { + case ssa.VecLaneF32x4: + vecOp = sseOpcodeAddps + case ssa.VecLaneF64x2: + vecOp = sseOpcodeAddpd + } + m.lowerVbBinOp(vecOp, x, y, instr.Return()) + + case ssa.OpcodeVFsub: + x, y, lane := instr.Arg2WithLane() + var vecOp sseOpcode + switch lane { + case ssa.VecLaneF32x4: + vecOp = sseOpcodeSubps + case ssa.VecLaneF64x2: + vecOp = sseOpcodeSubpd + } + m.lowerVbBinOp(vecOp, x, y, instr.Return()) + + case ssa.OpcodeVFdiv: + x, y, lane := instr.Arg2WithLane() + var vecOp sseOpcode + switch lane { + case ssa.VecLaneF32x4: + vecOp = sseOpcodeDivps + case ssa.VecLaneF64x2: + vecOp = sseOpcodeDivpd + } + m.lowerVbBinOp(vecOp, x, y, instr.Return()) + + case ssa.OpcodeVFmul: + x, y, lane := instr.Arg2WithLane() + var vecOp sseOpcode + switch lane { + case ssa.VecLaneF32x4: + vecOp = sseOpcodeMulps + case ssa.VecLaneF64x2: + vecOp = sseOpcodeMulpd + } + m.lowerVbBinOp(vecOp, x, y, instr.Return()) + + case ssa.OpcodeVFneg: + x, lane := instr.ArgWithLane() + rn := m.getOperand_Reg(m.c.ValueDefinition(x)) + rd := m.c.VRegOf(instr.Return()) + + tmp := m.c.AllocateVReg(ssa.TypeV128) + + var shiftOp, xorOp sseOpcode + var shiftAmt uint32 + switch lane { + case ssa.VecLaneF32x4: + shiftOp, shiftAmt, xorOp = sseOpcodePslld, 31, sseOpcodeXorps + case ssa.VecLaneF64x2: + shiftOp, shiftAmt, xorOp = sseOpcodePsllq, 63, sseOpcodeXorpd + } + + zero := m.allocateInstr() + zero.asZeros(tmp) + m.insert(zero) + + // Set all bits on tmp by CMPPD with arg=0 (== pseudo CMPEQPD instruction). + // See https://www.felixcloutier.com/x86/cmpps + // + // Note: if we do not clear all the bits ^ with XORPS, this might end up not setting ones on some lane + // if the lane is NaN. + cmp := m.allocateInstr() + cmp.asXmmRmRImm(sseOpcodeCmppd, uint8(cmpPredEQ_UQ), newOperandReg(tmp), tmp) + m.insert(cmp) + + // Do the left shift on each lane to set only the most significant bit in each. + i := m.allocateInstr() + i.asXmmRmiReg(shiftOp, newOperandImm32(shiftAmt), tmp) + m.insert(i) + + // Get the negated result by XOR on each lane with tmp. + i = m.allocateInstr() + i.asXmmRmR(xorOp, rn, tmp) + m.insert(i) + + m.copyTo(tmp, rd) + + case ssa.OpcodeVSqrt: + x, lane := instr.ArgWithLane() + rn := m.getOperand_Reg(m.c.ValueDefinition(x)) + rd := m.c.VRegOf(instr.Return()) + + var vecOp sseOpcode + switch lane { + case ssa.VecLaneF32x4: + vecOp = sseOpcodeSqrtps + case ssa.VecLaneF64x2: + vecOp = sseOpcodeSqrtpd + } + i := m.allocateInstr() + i.asXmmUnaryRmR(vecOp, rn, rd) + m.insert(i) + + case ssa.OpcodeVImin: + x, y, lane := instr.Arg2WithLane() + var vecOp sseOpcode + switch lane { + case ssa.VecLaneI8x16: + vecOp = sseOpcodePminsb + case ssa.VecLaneI16x8: + vecOp = sseOpcodePminsw + case ssa.VecLaneI32x4: + vecOp = sseOpcodePminsd + } + m.lowerVbBinOp(vecOp, x, y, instr.Return()) + + case ssa.OpcodeVUmin: + x, y, lane := instr.Arg2WithLane() + var vecOp sseOpcode + switch lane { + case ssa.VecLaneI8x16: + vecOp = sseOpcodePminub + case ssa.VecLaneI16x8: + vecOp = sseOpcodePminuw + case ssa.VecLaneI32x4: + vecOp = sseOpcodePminud + } + m.lowerVbBinOp(vecOp, x, y, instr.Return()) + + case ssa.OpcodeVImax: + x, y, lane := instr.Arg2WithLane() + var vecOp sseOpcode + switch lane { + case ssa.VecLaneI8x16: + vecOp = sseOpcodePmaxsb + case ssa.VecLaneI16x8: + vecOp = sseOpcodePmaxsw + case ssa.VecLaneI32x4: + vecOp = sseOpcodePmaxsd + } + m.lowerVbBinOp(vecOp, x, y, instr.Return()) + + case ssa.OpcodeVUmax: + x, y, lane := instr.Arg2WithLane() + var vecOp sseOpcode + switch lane { + case ssa.VecLaneI8x16: + vecOp = sseOpcodePmaxub + case ssa.VecLaneI16x8: + vecOp = sseOpcodePmaxuw + case ssa.VecLaneI32x4: + vecOp = sseOpcodePmaxud + } + m.lowerVbBinOp(vecOp, x, y, instr.Return()) + + case ssa.OpcodeVAvgRound: + x, y, lane := instr.Arg2WithLane() + var vecOp sseOpcode + switch lane { + case ssa.VecLaneI8x16: + vecOp = sseOpcodePavgb + case ssa.VecLaneI16x8: + vecOp = sseOpcodePavgw + } + m.lowerVbBinOp(vecOp, x, y, instr.Return()) + + case ssa.OpcodeVIcmp: + x, y, c, lane := instr.VIcmpData() + m.lowerVIcmp(x, y, c, instr.Return(), lane) + + case ssa.OpcodeVFcmp: + x, y, c, lane := instr.VFcmpData() + m.lowerVFcmp(x, y, c, instr.Return(), lane) + + case ssa.OpcodeExtractlane: + x, index, signed, lane := instr.ExtractlaneData() + m.lowerExtractLane(x, index, signed, instr.Return(), lane) + + case ssa.OpcodeInsertlane: + x, y, index, lane := instr.InsertlaneData() + m.lowerInsertLane(x, y, index, instr.Return(), lane) + + case ssa.OpcodeSwizzle: + x, y, _ := instr.Arg2WithLane() + m.lowerSwizzle(x, y, instr.Return()) + + case ssa.OpcodeShuffle: + x, y, lo, hi := instr.ShuffleData() + m.lowerShuffle(x, y, lo, hi, instr.Return()) + + case ssa.OpcodeSplat: + x, lane := instr.ArgWithLane() + m.lowerSplat(x, instr.Return(), lane) + + case ssa.OpcodeSqmulRoundSat: + x, y := instr.Arg2() + m.lowerSqmulRoundSat(x, y, instr.Return()) + + case ssa.OpcodeVZeroExtLoad: + ptr, offset, typ := instr.VZeroExtLoadData() + var sseOp sseOpcode + // Both movss and movsd clears the higher bits of the destination register upt 128 bits. + // https://www.felixcloutier.com/x86/movss + // https://www.felixcloutier.com/x86/movsd + if typ == ssa.TypeF32 { + sseOp = sseOpcodeMovss + } else { + sseOp = sseOpcodeMovsd + } + mem := m.lowerToAddressMode(ptr, offset) + dst := m.c.VRegOf(instr.Return()) + m.insert(m.allocateInstr().asXmmUnaryRmR(sseOp, newOperandMem(mem), dst)) + + case ssa.OpcodeVMinPseudo: + x, y, lane := instr.Arg2WithLane() + var vecOp sseOpcode + switch lane { + case ssa.VecLaneF32x4: + vecOp = sseOpcodeMinps + case ssa.VecLaneF64x2: + vecOp = sseOpcodeMinpd + default: + panic("BUG: unexpected lane type") + } + m.lowerVbBinOpUnaligned(vecOp, y, x, instr.Return()) + + case ssa.OpcodeVMaxPseudo: + x, y, lane := instr.Arg2WithLane() + var vecOp sseOpcode + switch lane { + case ssa.VecLaneF32x4: + vecOp = sseOpcodeMaxps + case ssa.VecLaneF64x2: + vecOp = sseOpcodeMaxpd + default: + panic("BUG: unexpected lane type") + } + m.lowerVbBinOpUnaligned(vecOp, y, x, instr.Return()) + + case ssa.OpcodeVIshl: + x, y, lane := instr.Arg2WithLane() + m.lowerVIshl(x, y, instr.Return(), lane) + + case ssa.OpcodeVSshr: + x, y, lane := instr.Arg2WithLane() + m.lowerVSshr(x, y, instr.Return(), lane) + + case ssa.OpcodeVUshr: + x, y, lane := instr.Arg2WithLane() + m.lowerVUshr(x, y, instr.Return(), lane) + + case ssa.OpcodeVCeil: + x, lane := instr.ArgWithLane() + m.lowerVRound(x, instr.Return(), 0x2, lane == ssa.VecLaneF64x2) + + case ssa.OpcodeVFloor: + x, lane := instr.ArgWithLane() + m.lowerVRound(x, instr.Return(), 0x1, lane == ssa.VecLaneF64x2) + + case ssa.OpcodeVTrunc: + x, lane := instr.ArgWithLane() + m.lowerVRound(x, instr.Return(), 0x3, lane == ssa.VecLaneF64x2) + + case ssa.OpcodeVNearest: + x, lane := instr.ArgWithLane() + m.lowerVRound(x, instr.Return(), 0x0, lane == ssa.VecLaneF64x2) + + case ssa.OpcodeExtIaddPairwise: + x, lane, signed := instr.ExtIaddPairwiseData() + m.lowerExtIaddPairwise(x, instr.Return(), lane, signed) + + case ssa.OpcodeUwidenLow, ssa.OpcodeSwidenLow: + x, lane := instr.ArgWithLane() + m.lowerWidenLow(x, instr.Return(), lane, op == ssa.OpcodeSwidenLow) + + case ssa.OpcodeUwidenHigh, ssa.OpcodeSwidenHigh: + x, lane := instr.ArgWithLane() + m.lowerWidenHigh(x, instr.Return(), lane, op == ssa.OpcodeSwidenHigh) + + case ssa.OpcodeLoadSplat: + ptr, offset, lane := instr.LoadSplatData() + m.lowerLoadSplat(ptr, offset, instr.Return(), lane) + + case ssa.OpcodeVFcvtFromUint, ssa.OpcodeVFcvtFromSint: + x, lane := instr.ArgWithLane() + m.lowerVFcvtFromInt(x, instr.Return(), lane, op == ssa.OpcodeVFcvtFromSint) + + case ssa.OpcodeVFcvtToSintSat, ssa.OpcodeVFcvtToUintSat: + x, lane := instr.ArgWithLane() + m.lowerVFcvtToIntSat(x, instr.Return(), lane, op == ssa.OpcodeVFcvtToSintSat) + + case ssa.OpcodeSnarrow, ssa.OpcodeUnarrow: + x, y, lane := instr.Arg2WithLane() + m.lowerNarrow(x, y, instr.Return(), lane, op == ssa.OpcodeSnarrow) + + case ssa.OpcodeFvpromoteLow: + x := instr.Arg() + src := m.getOperand_Reg(m.c.ValueDefinition(x)) + dst := m.c.VRegOf(instr.Return()) + m.insert(m.allocateInstr().asXmmUnaryRmR(sseOpcodeCvtps2pd, src, dst)) + + case ssa.OpcodeFvdemote: + x := instr.Arg() + src := m.getOperand_Reg(m.c.ValueDefinition(x)) + dst := m.c.VRegOf(instr.Return()) + m.insert(m.allocateInstr().asXmmUnaryRmR(sseOpcodeCvtpd2ps, src, dst)) + + case ssa.OpcodeWideningPairwiseDotProductS: + x, y := instr.Arg2() + m.lowerWideningPairwiseDotProductS(x, y, instr.Return()) + + case ssa.OpcodeVIabs: + m.lowerVIabs(instr) + case ssa.OpcodeVIpopcnt: + m.lowerVIpopcnt(instr) + case ssa.OpcodeVFmin: + m.lowerVFmin(instr) + case ssa.OpcodeVFmax: + m.lowerVFmax(instr) + case ssa.OpcodeVFabs: + m.lowerVFabs(instr) + case ssa.OpcodeUndefined: + m.insert(m.allocateInstr().asUD2()) + case ssa.OpcodeExitWithCode: + execCtx, code := instr.ExitWithCodeData() + m.lowerExitWithCode(m.c.VRegOf(execCtx), code) + case ssa.OpcodeExitIfTrueWithCode: + execCtx, c, code := instr.ExitIfTrueWithCodeData() + m.lowerExitIfTrueWithCode(m.c.VRegOf(execCtx), c, code) + case ssa.OpcodeLoad: + ptr, offset, typ := instr.LoadData() + dst := m.c.VRegOf(instr.Return()) + m.lowerLoad(ptr, offset, typ, dst) + case ssa.OpcodeUload8, ssa.OpcodeUload16, ssa.OpcodeUload32, ssa.OpcodeSload8, ssa.OpcodeSload16, ssa.OpcodeSload32: + ptr, offset, _ := instr.LoadData() + ret := m.c.VRegOf(instr.Return()) + m.lowerExtLoad(op, ptr, offset, ret) + case ssa.OpcodeVconst: + result := m.c.VRegOf(instr.Return()) + lo, hi := instr.VconstData() + m.lowerVconst(result, lo, hi) + case ssa.OpcodeSExtend, ssa.OpcodeUExtend: + from, to, signed := instr.ExtendData() + m.lowerExtend(instr.Arg(), instr.Return(), from, to, signed) + case ssa.OpcodeIcmp: + m.lowerIcmp(instr) + case ssa.OpcodeFcmp: + m.lowerFcmp(instr) + case ssa.OpcodeSelect: + cval, x, y := instr.SelectData() + m.lowerSelect(x, y, cval, instr.Return()) + case ssa.OpcodeIreduce: + rn := m.getOperand_Mem_Reg(m.c.ValueDefinition(instr.Arg())) + retVal := instr.Return() + rd := m.c.VRegOf(retVal) + + if retVal.Type() != ssa.TypeI32 { + panic("TODO?: Ireduce to non-i32") + } + m.insert(m.allocateInstr().asMovzxRmR(extModeLQ, rn, rd)) + + case ssa.OpcodeAtomicLoad: + ptr := instr.Arg() + size := instr.AtomicTargetSize() + dst := m.c.VRegOf(instr.Return()) + + // At this point, the ptr is ensured to be aligned, so using a normal load is atomic. + // https://github.com/golang/go/blob/adead1a93f472affa97c494ef19f2f492ee6f34a/src/runtime/internal/atomic/atomic_amd64.go#L30 + mem := newOperandMem(m.lowerToAddressMode(ptr, 0)) + load := m.allocateInstr() + switch size { + case 8: + load.asMov64MR(mem, dst) + case 4: + load.asMovzxRmR(extModeLQ, mem, dst) + case 2: + load.asMovzxRmR(extModeWQ, mem, dst) + case 1: + load.asMovzxRmR(extModeBQ, mem, dst) + default: + panic("BUG") + } + m.insert(load) + + case ssa.OpcodeFence: + m.insert(m.allocateInstr().asMFence()) + + case ssa.OpcodeAtomicStore: + ptr, _val := instr.Arg2() + size := instr.AtomicTargetSize() + + val := m.getOperand_Reg(m.c.ValueDefinition(_val)) + // The content on the val register will be overwritten by xchg, so we need to copy it to a temporary register. + copied := m.copyToTmp(val.reg()) + + mem := newOperandMem(m.lowerToAddressMode(ptr, 0)) + store := m.allocateInstr().asXCHG(copied, mem, byte(size)) + m.insert(store) + + case ssa.OpcodeAtomicCas: + addr, exp, repl := instr.Arg3() + size := instr.AtomicTargetSize() + m.lowerAtomicCas(addr, exp, repl, size, instr.Return()) + + case ssa.OpcodeAtomicRmw: + addr, val := instr.Arg2() + atomicOp, size := instr.AtomicRmwData() + m.lowerAtomicRmw(atomicOp, addr, val, size, instr.Return()) + + default: + panic("TODO: lowering " + op.String()) + } +} + +func (m *machine) lowerAtomicRmw(op ssa.AtomicRmwOp, addr, val ssa.Value, size uint64, ret ssa.Value) { + mem := m.lowerToAddressMode(addr, 0) + _val := m.getOperand_Reg(m.c.ValueDefinition(val)) + + switch op { + case ssa.AtomicRmwOpAdd, ssa.AtomicRmwOpSub: + valCopied := m.copyToTmp(_val.reg()) + if op == ssa.AtomicRmwOpSub { + // Negate the value. + m.insert(m.allocateInstr().asNeg(newOperandReg(valCopied), true)) + } + m.insert(m.allocateInstr().asLockXAdd(valCopied, mem, byte(size))) + m.clearHigherBitsForAtomic(valCopied, size, ret.Type()) + m.copyTo(valCopied, m.c.VRegOf(ret)) + + case ssa.AtomicRmwOpAnd, ssa.AtomicRmwOpOr, ssa.AtomicRmwOpXor: + accumulator := raxVReg + // Reserve rax for the accumulator to make regalloc happy. + // Note: do this initialization before defining valCopied, because it might be the same register and + // if that happens, the unnecessary load/store will be performed inside the loop. + // This can be mitigated in any way once the register allocator is clever enough. + m.insert(m.allocateInstr().asDefineUninitializedReg(accumulator)) + + // Copy the value to a temporary register. + valCopied := m.copyToTmp(_val.reg()) + m.clearHigherBitsForAtomic(valCopied, size, ret.Type()) + + memOp := newOperandMem(mem) + tmp := m.c.AllocateVReg(ssa.TypeI64) + beginLoop, beginLoopLabel := m.allocateBrTarget() + { + m.insert(beginLoop) + // Reset the value on tmp by the original value. + m.copyTo(valCopied, tmp) + // Load the current value at the memory location into accumulator. + switch size { + case 1: + m.insert(m.allocateInstr().asMovzxRmR(extModeBQ, memOp, accumulator)) + case 2: + m.insert(m.allocateInstr().asMovzxRmR(extModeWQ, memOp, accumulator)) + case 4: + m.insert(m.allocateInstr().asMovzxRmR(extModeLQ, memOp, accumulator)) + case 8: + m.insert(m.allocateInstr().asMov64MR(memOp, accumulator)) + default: + panic("BUG") + } + // Then perform the logical operation on the accumulator and the value on tmp. + switch op { + case ssa.AtomicRmwOpAnd: + m.insert(m.allocateInstr().asAluRmiR(aluRmiROpcodeAnd, newOperandReg(accumulator), tmp, true)) + case ssa.AtomicRmwOpOr: + m.insert(m.allocateInstr().asAluRmiR(aluRmiROpcodeOr, newOperandReg(accumulator), tmp, true)) + case ssa.AtomicRmwOpXor: + m.insert(m.allocateInstr().asAluRmiR(aluRmiROpcodeXor, newOperandReg(accumulator), tmp, true)) + default: + panic("BUG") + } + // Finally, try compare-exchange the value at the memory location with the tmp. + m.insert(m.allocateInstr().asLockCmpXCHG(tmp, memOp.addressMode(), byte(size))) + // If it succeeds, ZF will be set, and we can break the loop. + m.insert(m.allocateInstr().asJmpIf(condNZ, newOperandLabel(beginLoopLabel))) + } + + // valCopied must be alive at the end of the loop. + m.insert(m.allocateInstr().asNopUseReg(valCopied)) + + // At this point, accumulator contains the result. + m.clearHigherBitsForAtomic(accumulator, size, ret.Type()) + m.copyTo(accumulator, m.c.VRegOf(ret)) + + case ssa.AtomicRmwOpXchg: + valCopied := m.copyToTmp(_val.reg()) + + m.insert(m.allocateInstr().asXCHG(valCopied, newOperandMem(mem), byte(size))) + m.clearHigherBitsForAtomic(valCopied, size, ret.Type()) + m.copyTo(valCopied, m.c.VRegOf(ret)) + + default: + panic("BUG") + } +} + +func (m *machine) lowerAtomicCas(addr, exp, repl ssa.Value, size uint64, ret ssa.Value) { + mem := m.lowerToAddressMode(addr, 0) + expOp := m.getOperand_Reg(m.c.ValueDefinition(exp)) + replOp := m.getOperand_Reg(m.c.ValueDefinition(repl)) + + accumulator := raxVReg + m.copyTo(expOp.reg(), accumulator) + m.insert(m.allocateInstr().asLockCmpXCHG(replOp.reg(), mem, byte(size))) + m.clearHigherBitsForAtomic(accumulator, size, ret.Type()) + m.copyTo(accumulator, m.c.VRegOf(ret)) +} + +func (m *machine) clearHigherBitsForAtomic(r regalloc.VReg, valSize uint64, resultType ssa.Type) { + switch resultType { + case ssa.TypeI32: + switch valSize { + case 1: + m.insert(m.allocateInstr().asMovzxRmR(extModeBL, newOperandReg(r), r)) + case 2: + m.insert(m.allocateInstr().asMovzxRmR(extModeWL, newOperandReg(r), r)) + } + case ssa.TypeI64: + switch valSize { + case 1: + m.insert(m.allocateInstr().asMovzxRmR(extModeBQ, newOperandReg(r), r)) + case 2: + m.insert(m.allocateInstr().asMovzxRmR(extModeWQ, newOperandReg(r), r)) + case 4: + m.insert(m.allocateInstr().asMovzxRmR(extModeLQ, newOperandReg(r), r)) + } + } +} + +func (m *machine) lowerFcmp(instr *ssa.Instruction) { + f1, f2, and := m.lowerFcmpToFlags(instr) + rd := m.c.VRegOf(instr.Return()) + if f2 == condInvalid { + tmp := m.c.AllocateVReg(ssa.TypeI32) + m.insert(m.allocateInstr().asSetcc(f1, tmp)) + // On amd64, setcc only sets the first byte of the register, so we need to zero extend it to match + // the semantics of Icmp that sets either 0 or 1. + m.insert(m.allocateInstr().asMovzxRmR(extModeBQ, newOperandReg(tmp), rd)) + } else { + tmp1, tmp2 := m.c.AllocateVReg(ssa.TypeI32), m.c.AllocateVReg(ssa.TypeI32) + m.insert(m.allocateInstr().asSetcc(f1, tmp1)) + m.insert(m.allocateInstr().asSetcc(f2, tmp2)) + var op aluRmiROpcode + if and { + op = aluRmiROpcodeAnd + } else { + op = aluRmiROpcodeOr + } + m.insert(m.allocateInstr().asAluRmiR(op, newOperandReg(tmp1), tmp2, false)) + m.insert(m.allocateInstr().asMovzxRmR(extModeBQ, newOperandReg(tmp2), rd)) + } +} + +func (m *machine) lowerIcmp(instr *ssa.Instruction) { + x, y, c := instr.IcmpData() + m.lowerIcmpToFlag(m.c.ValueDefinition(x), m.c.ValueDefinition(y), x.Type() == ssa.TypeI64) + rd := m.c.VRegOf(instr.Return()) + tmp := m.c.AllocateVReg(ssa.TypeI32) + m.insert(m.allocateInstr().asSetcc(condFromSSAIntCmpCond(c), tmp)) + // On amd64, setcc only sets the first byte of the register, so we need to zero extend it to match + // the semantics of Icmp that sets either 0 or 1. + m.insert(m.allocateInstr().asMovzxRmR(extModeBQ, newOperandReg(tmp), rd)) +} + +func (m *machine) lowerSelect(x, y, cval, ret ssa.Value) { + xo, yo := m.getOperand_Mem_Reg(m.c.ValueDefinition(x)), m.getOperand_Reg(m.c.ValueDefinition(y)) + rd := m.c.VRegOf(ret) + + var cond cond + cvalDef := m.c.ValueDefinition(cval) + switch m.c.MatchInstrOneOf(cvalDef, condBranchMatches[:]) { + case ssa.OpcodeIcmp: + icmp := cvalDef.Instr + xc, yc, cc := icmp.IcmpData() + m.lowerIcmpToFlag(m.c.ValueDefinition(xc), m.c.ValueDefinition(yc), xc.Type() == ssa.TypeI64) + cond = condFromSSAIntCmpCond(cc) + icmp.Lowered() + default: // TODO: match ssa.OpcodeFcmp for optimization, but seems a bit complex. + cv := m.getOperand_Reg(cvalDef) + test := m.allocateInstr().asCmpRmiR(false, cv, cv.reg(), false) + m.insert(test) + cond = condNZ + } + + if typ := x.Type(); typ.IsInt() { + _64 := typ.Bits() == 64 + mov := m.allocateInstr() + tmp := m.c.AllocateVReg(typ) + switch yo.kind { + case operandKindReg: + mov.asMovRR(yo.reg(), tmp, _64) + case operandKindMem: + if _64 { + mov.asMov64MR(yo, tmp) + } else { + mov.asMovzxRmR(extModeLQ, yo, tmp) + } + default: + panic("BUG") + } + m.insert(mov) + cmov := m.allocateInstr().asCmove(cond, xo, tmp, _64) + m.insert(cmov) + m.insert(m.allocateInstr().asMovRR(tmp, rd, _64)) + } else { + mov := m.allocateInstr() + tmp := m.c.AllocateVReg(typ) + switch typ { + case ssa.TypeF32: + mov.asXmmUnaryRmR(sseOpcodeMovss, yo, tmp) + case ssa.TypeF64: + mov.asXmmUnaryRmR(sseOpcodeMovsd, yo, tmp) + case ssa.TypeV128: + mov.asXmmUnaryRmR(sseOpcodeMovdqu, yo, tmp) + default: + panic("BUG") + } + m.insert(mov) + + cmov := m.allocateInstr().asXmmCMov(cond, xo, tmp, typ.Size()) + m.insert(cmov) + + m.copyTo(tmp, rd) + } +} + +func (m *machine) lowerXmmCmovAfterRegAlloc(i *instruction) { + x := i.op1 + rd := i.op2.reg() + cond := cond(i.u1) + + jcc := m.allocateInstr() + m.insert(jcc) + + mov := m.allocateInstr() + switch i.u2 { + case 4: + mov.asXmmUnaryRmR(sseOpcodeMovss, x, rd) + case 8: + mov.asXmmUnaryRmR(sseOpcodeMovsd, x, rd) + case 16: + mov.asXmmUnaryRmR(sseOpcodeMovdqu, x, rd) + default: + panic("BUG") + } + m.insert(mov) + + nop, end := m.allocateBrTarget() + m.insert(nop) + jcc.asJmpIf(cond.invert(), newOperandLabel(end)) +} + +func (m *machine) lowerExtend(_arg, ret ssa.Value, from, to byte, signed bool) { + rd0 := m.c.VRegOf(ret) + arg := m.getOperand_Mem_Reg(m.c.ValueDefinition(_arg)) + + rd := m.c.AllocateVReg(ret.Type()) + + ext := m.allocateInstr() + switch { + case from == 8 && to == 16 && signed: + ext.asMovsxRmR(extModeBQ, arg, rd) + case from == 8 && to == 16 && !signed: + ext.asMovzxRmR(extModeBL, arg, rd) + case from == 8 && to == 32 && signed: + ext.asMovsxRmR(extModeBL, arg, rd) + case from == 8 && to == 32 && !signed: + ext.asMovzxRmR(extModeBQ, arg, rd) + case from == 8 && to == 64 && signed: + ext.asMovsxRmR(extModeBQ, arg, rd) + case from == 8 && to == 64 && !signed: + ext.asMovzxRmR(extModeBQ, arg, rd) + case from == 16 && to == 32 && signed: + ext.asMovsxRmR(extModeWL, arg, rd) + case from == 16 && to == 32 && !signed: + ext.asMovzxRmR(extModeWL, arg, rd) + case from == 16 && to == 64 && signed: + ext.asMovsxRmR(extModeWQ, arg, rd) + case from == 16 && to == 64 && !signed: + ext.asMovzxRmR(extModeWQ, arg, rd) + case from == 32 && to == 64 && signed: + ext.asMovsxRmR(extModeLQ, arg, rd) + case from == 32 && to == 64 && !signed: + ext.asMovzxRmR(extModeLQ, arg, rd) + default: + panic(fmt.Sprintf("BUG: unhandled extend: from=%d, to=%d, signed=%t", from, to, signed)) + } + m.insert(ext) + + m.copyTo(rd, rd0) +} + +func (m *machine) lowerVconst(dst regalloc.VReg, lo, hi uint64) { + if lo == 0 && hi == 0 { + m.insert(m.allocateInstr().asZeros(dst)) + return + } + + load := m.allocateInstr() + constLabel := m.allocateLabel() + m.consts = append(m.consts, _const{label: constLabel, lo: lo, hi: hi}) + load.asXmmUnaryRmR(sseOpcodeMovdqu, newOperandMem(m.newAmodeRipRel(constLabel.L)), dst) + m.insert(load) +} + +func (m *machine) lowerCtz(instr *ssa.Instruction) { + if m.cpuFeatures.HasExtra(platform.CpuExtraFeatureAmd64ABM) { + m.lowerUnaryRmR(instr, unaryRmROpcodeTzcnt) + } else { + // On processors that do not support TZCNT, the BSF instruction is + // executed instead. The key difference between TZCNT and BSF + // instruction is that if source operand is zero, the content of + // destination operand is undefined. + // https://www.felixcloutier.com/x86/tzcnt.html + + x := instr.Arg() + if !x.Type().IsInt() { + panic("BUG?") + } + _64 := x.Type().Bits() == 64 + + xDef := m.c.ValueDefinition(x) + tmp := m.c.AllocateVReg(x.Type()) + rm := m.getOperand_Reg(xDef) + + // First, we have to check if the target is non-zero. + test := m.allocateInstr() + test.asCmpRmiR(false, rm, rm.reg(), _64) + m.insert(test) + + jmpNz := m.allocateInstr() + m.insert(jmpNz) + + // If the value is zero, we just push the const value. + m.lowerIconst(tmp, uint64(x.Type().Bits()), _64) + + // Now jump right after the non-zero case. + jmpAtEnd := m.allocateInstr() + m.insert(jmpAtEnd) + + // jmpNz target label is set here. + nop, nz := m.allocateBrTarget() + jmpNz.asJmpIf(condNZ, newOperandLabel(nz)) + m.insert(nop) + + // Emit the non-zero case. + bsr := m.allocateInstr() + bsr.asUnaryRmR(unaryRmROpcodeBsf, rm, tmp, _64) + m.insert(bsr) + + // jmpAtEnd target label is set here. + nopEnd, end := m.allocateBrTarget() + jmpAtEnd.asJmp(newOperandLabel(end)) + m.insert(nopEnd) + + m.copyTo(tmp, m.c.VRegOf(instr.Return())) + } +} + +func (m *machine) lowerClz(instr *ssa.Instruction) { + if m.cpuFeatures.HasExtra(platform.CpuExtraFeatureAmd64ABM) { + m.lowerUnaryRmR(instr, unaryRmROpcodeLzcnt) + } else { + // On processors that do not support LZCNT, we combine BSR (calculating + // most significant set bit) with XOR. This logic is described in + // "Replace Raw Assembly Code with Builtin Intrinsics" section in: + // https://developer.apple.com/documentation/apple-silicon/addressing-architectural-differences-in-your-macos-code. + + x := instr.Arg() + if !x.Type().IsInt() { + panic("BUG?") + } + _64 := x.Type().Bits() == 64 + + xDef := m.c.ValueDefinition(x) + rm := m.getOperand_Reg(xDef) + tmp := m.c.AllocateVReg(x.Type()) + + // First, we have to check if the rm is non-zero as BSR is undefined + // on zero. See https://www.felixcloutier.com/x86/bsr. + test := m.allocateInstr() + test.asCmpRmiR(false, rm, rm.reg(), _64) + m.insert(test) + + jmpNz := m.allocateInstr() + m.insert(jmpNz) + + // If the value is zero, we just push the const value. + m.lowerIconst(tmp, uint64(x.Type().Bits()), _64) + + // Now jump right after the non-zero case. + jmpAtEnd := m.allocateInstr() + m.insert(jmpAtEnd) + + // jmpNz target label is set here. + nop, nz := m.allocateBrTarget() + jmpNz.asJmpIf(condNZ, newOperandLabel(nz)) + m.insert(nop) + + // Emit the non-zero case. + bsr := m.allocateInstr() + bsr.asUnaryRmR(unaryRmROpcodeBsr, rm, tmp, _64) + m.insert(bsr) + + // Now we XOR the value with the bit length minus one. + xor := m.allocateInstr() + xor.asAluRmiR(aluRmiROpcodeXor, newOperandImm32(uint32(x.Type().Bits()-1)), tmp, _64) + m.insert(xor) + + // jmpAtEnd target label is set here. + nopEnd, end := m.allocateBrTarget() + jmpAtEnd.asJmp(newOperandLabel(end)) + m.insert(nopEnd) + + m.copyTo(tmp, m.c.VRegOf(instr.Return())) + } +} + +func (m *machine) lowerUnaryRmR(si *ssa.Instruction, op unaryRmROpcode) { + x := si.Arg() + if !x.Type().IsInt() { + panic("BUG?") + } + _64 := x.Type().Bits() == 64 + + xDef := m.c.ValueDefinition(x) + rm := m.getOperand_Mem_Reg(xDef) + rd := m.c.VRegOf(si.Return()) + + instr := m.allocateInstr() + instr.asUnaryRmR(op, rm, rd, _64) + m.insert(instr) +} + +func (m *machine) lowerLoad(ptr ssa.Value, offset uint32, typ ssa.Type, dst regalloc.VReg) { + mem := newOperandMem(m.lowerToAddressMode(ptr, offset)) + load := m.allocateInstr() + switch typ { + case ssa.TypeI32: + load.asMovzxRmR(extModeLQ, mem, dst) + case ssa.TypeI64: + load.asMov64MR(mem, dst) + case ssa.TypeF32: + load.asXmmUnaryRmR(sseOpcodeMovss, mem, dst) + case ssa.TypeF64: + load.asXmmUnaryRmR(sseOpcodeMovsd, mem, dst) + case ssa.TypeV128: + load.asXmmUnaryRmR(sseOpcodeMovdqu, mem, dst) + default: + panic("BUG") + } + m.insert(load) +} + +func (m *machine) lowerExtLoad(op ssa.Opcode, ptr ssa.Value, offset uint32, dst regalloc.VReg) { + mem := newOperandMem(m.lowerToAddressMode(ptr, offset)) + load := m.allocateInstr() + switch op { + case ssa.OpcodeUload8: + load.asMovzxRmR(extModeBQ, mem, dst) + case ssa.OpcodeUload16: + load.asMovzxRmR(extModeWQ, mem, dst) + case ssa.OpcodeUload32: + load.asMovzxRmR(extModeLQ, mem, dst) + case ssa.OpcodeSload8: + load.asMovsxRmR(extModeBQ, mem, dst) + case ssa.OpcodeSload16: + load.asMovsxRmR(extModeWQ, mem, dst) + case ssa.OpcodeSload32: + load.asMovsxRmR(extModeLQ, mem, dst) + default: + panic("BUG") + } + m.insert(load) +} + +func (m *machine) lowerExitIfTrueWithCode(execCtx regalloc.VReg, cond ssa.Value, code wazevoapi.ExitCode) { + condDef := m.c.ValueDefinition(cond) + if !m.c.MatchInstr(condDef, ssa.OpcodeIcmp) { + panic("TODO: ExitIfTrue must come after Icmp at the moment: " + condDef.Instr.Opcode().String()) + } + cvalInstr := condDef.Instr + cvalInstr.MarkLowered() + + // We need to copy the execution context to a temp register, because if it's spilled, + // it might end up being reloaded inside the exiting branch. + execCtxTmp := m.copyToTmp(execCtx) + + x, y, c := cvalInstr.IcmpData() + xx, yy := m.c.ValueDefinition(x), m.c.ValueDefinition(y) + if !m.tryLowerBandToFlag(xx, yy) { + m.lowerIcmpToFlag(xx, yy, x.Type() == ssa.TypeI64) + } + + jmpIf := m.allocateInstr() + m.insert(jmpIf) + l := m.lowerExitWithCode(execCtxTmp, code) + jmpIf.asJmpIf(condFromSSAIntCmpCond(c).invert(), newOperandLabel(l)) +} + +func (m *machine) tryLowerBandToFlag(x, y *backend.SSAValueDefinition) (ok bool) { + var target *backend.SSAValueDefinition + if x.IsFromInstr() && x.Instr.Constant() && x.Instr.ConstantVal() == 0 { + if m.c.MatchInstr(y, ssa.OpcodeBand) { + target = y + } + } + + if y.IsFromInstr() && y.Instr.Constant() && y.Instr.ConstantVal() == 0 { + if m.c.MatchInstr(x, ssa.OpcodeBand) { + target = x + } + } + + if target == nil { + return false + } + + bandInstr := target.Instr + bandX, bandY := bandInstr.Arg2() + + xx := m.getOperand_Reg(m.c.ValueDefinition(bandX)) + yy := m.getOperand_Mem_Imm32_Reg(m.c.ValueDefinition(bandY)) + test := m.allocateInstr().asCmpRmiR(false, yy, xx.reg(), bandX.Type() == ssa.TypeI64) + m.insert(test) + bandInstr.MarkLowered() + return true +} + +func (m *machine) allocateExitInstructions(execCtx, exitCodeReg regalloc.VReg) (saveRsp, saveRbp, setExitCode *instruction) { + saveRsp = m.allocateInstr().asMovRM( + rspVReg, + newOperandMem(m.newAmodeImmReg(wazevoapi.ExecutionContextOffsetStackPointerBeforeGoCall.U32(), execCtx)), + 8, + ) + + saveRbp = m.allocateInstr().asMovRM( + rbpVReg, + newOperandMem(m.newAmodeImmReg(wazevoapi.ExecutionContextOffsetFramePointerBeforeGoCall.U32(), execCtx)), + 8, + ) + setExitCode = m.allocateInstr().asMovRM( + exitCodeReg, + newOperandMem(m.newAmodeImmReg(wazevoapi.ExecutionContextOffsetExitCodeOffset.U32(), execCtx)), + 4, + ) + return +} + +func (m *machine) lowerExitWithCode(execCtx regalloc.VReg, code wazevoapi.ExitCode) (afterLabel backend.Label) { + exitCodeReg := rbpVReg + saveRsp, saveRbp, setExitCode := m.allocateExitInstructions(execCtx, exitCodeReg) + + // Set save RSP, RBP, and write exit code. + m.insert(saveRsp) + m.insert(saveRbp) + m.lowerIconst(exitCodeReg, uint64(code), false) + m.insert(setExitCode) + + ripReg := rbpVReg + + // Next is to save the current address for stack unwinding. + nop, currentAddrLabel := m.allocateBrTarget() + m.insert(nop) + readRip := m.allocateInstr().asLEA(newOperandLabel(currentAddrLabel), ripReg) + m.insert(readRip) + saveRip := m.allocateInstr().asMovRM( + ripReg, + newOperandMem(m.newAmodeImmReg(wazevoapi.ExecutionContextOffsetGoCallReturnAddress.U32(), execCtx)), + 8, + ) + m.insert(saveRip) + + // Finally exit. + exitSq := m.allocateExitSeq(execCtx) + m.insert(exitSq) + + // Return the label for continuation. + continuation, afterLabel := m.allocateBrTarget() + m.insert(continuation) + return afterLabel +} + +func (m *machine) lowerAluRmiROp(si *ssa.Instruction, op aluRmiROpcode) { + x, y := si.Arg2() + if !x.Type().IsInt() { + panic("BUG?") + } + + _64 := x.Type().Bits() == 64 + + xDef, yDef := m.c.ValueDefinition(x), m.c.ValueDefinition(y) + + // TODO: commutative args can be swapped if one of them is an immediate. + rn := m.getOperand_Reg(xDef) + rm := m.getOperand_Mem_Imm32_Reg(yDef) + rd := m.c.VRegOf(si.Return()) + + // rn is being overwritten, so we first copy its value to a temp register, + // in case it is referenced again later. + tmp := m.copyToTmp(rn.reg()) + + alu := m.allocateInstr() + alu.asAluRmiR(op, rm, tmp, _64) + m.insert(alu) + + // tmp now contains the result, we copy it to the dest register. + m.copyTo(tmp, rd) +} + +func (m *machine) lowerShiftR(si *ssa.Instruction, op shiftROp) { + x, amt := si.Arg2() + if !x.Type().IsInt() { + panic("BUG?") + } + _64 := x.Type().Bits() == 64 + + xDef, amtDef := m.c.ValueDefinition(x), m.c.ValueDefinition(amt) + + opAmt := m.getOperand_Imm32_Reg(amtDef) + rx := m.getOperand_Reg(xDef) + rd := m.c.VRegOf(si.Return()) + + // rx is being overwritten, so we first copy its value to a temp register, + // in case it is referenced again later. + tmpDst := m.copyToTmp(rx.reg()) + + if opAmt.kind == operandKindReg { + // If opAmt is a register we must copy its value to rcx, + // because shiftR encoding mandates that the shift amount is in rcx. + m.copyTo(opAmt.reg(), rcxVReg) + + alu := m.allocateInstr() + alu.asShiftR(op, newOperandReg(rcxVReg), tmpDst, _64) + m.insert(alu) + + } else { + alu := m.allocateInstr() + alu.asShiftR(op, opAmt, tmpDst, _64) + m.insert(alu) + } + + // tmp now contains the result, we copy it to the dest register. + m.copyTo(tmpDst, rd) +} + +func (m *machine) lowerXmmRmR(instr *ssa.Instruction) { + x, y := instr.Arg2() + if !x.Type().IsFloat() { + panic("BUG?") + } + _64 := x.Type().Bits() == 64 + + var op sseOpcode + if _64 { + switch instr.Opcode() { + case ssa.OpcodeFadd: + op = sseOpcodeAddsd + case ssa.OpcodeFsub: + op = sseOpcodeSubsd + case ssa.OpcodeFmul: + op = sseOpcodeMulsd + case ssa.OpcodeFdiv: + op = sseOpcodeDivsd + default: + panic("BUG") + } + } else { + switch instr.Opcode() { + case ssa.OpcodeFadd: + op = sseOpcodeAddss + case ssa.OpcodeFsub: + op = sseOpcodeSubss + case ssa.OpcodeFmul: + op = sseOpcodeMulss + case ssa.OpcodeFdiv: + op = sseOpcodeDivss + default: + panic("BUG") + } + } + + xDef, yDef := m.c.ValueDefinition(x), m.c.ValueDefinition(y) + rn := m.getOperand_Reg(yDef) + rm := m.getOperand_Reg(xDef) + rd := m.c.VRegOf(instr.Return()) + + // rm is being overwritten, so we first copy its value to a temp register, + // in case it is referenced again later. + tmp := m.copyToTmp(rm.reg()) + + xmm := m.allocateInstr().asXmmRmR(op, rn, tmp) + m.insert(xmm) + + m.copyTo(tmp, rd) +} + +func (m *machine) lowerSqrt(instr *ssa.Instruction) { + x := instr.Arg() + if !x.Type().IsFloat() { + panic("BUG") + } + _64 := x.Type().Bits() == 64 + var op sseOpcode + if _64 { + op = sseOpcodeSqrtsd + } else { + op = sseOpcodeSqrtss + } + + xDef := m.c.ValueDefinition(x) + rm := m.getOperand_Mem_Reg(xDef) + rd := m.c.VRegOf(instr.Return()) + + xmm := m.allocateInstr().asXmmUnaryRmR(op, rm, rd) + m.insert(xmm) +} + +func (m *machine) lowerFabsFneg(instr *ssa.Instruction) { + x := instr.Arg() + if !x.Type().IsFloat() { + panic("BUG") + } + _64 := x.Type().Bits() == 64 + var op sseOpcode + var mask uint64 + if _64 { + switch instr.Opcode() { + case ssa.OpcodeFabs: + mask, op = 0x7fffffffffffffff, sseOpcodeAndpd + case ssa.OpcodeFneg: + mask, op = 0x8000000000000000, sseOpcodeXorpd + } + } else { + switch instr.Opcode() { + case ssa.OpcodeFabs: + mask, op = 0x7fffffff, sseOpcodeAndps + case ssa.OpcodeFneg: + mask, op = 0x80000000, sseOpcodeXorps + } + } + + tmp := m.c.AllocateVReg(x.Type()) + + xDef := m.c.ValueDefinition(x) + rm := m.getOperand_Reg(xDef) + rd := m.c.VRegOf(instr.Return()) + + m.lowerFconst(tmp, mask, _64) + + xmm := m.allocateInstr().asXmmRmR(op, rm, tmp) + m.insert(xmm) + + m.copyTo(tmp, rd) +} + +func (m *machine) lowerStore(si *ssa.Instruction) { + value, ptr, offset, storeSizeInBits := si.StoreData() + rm := m.getOperand_Reg(m.c.ValueDefinition(value)) + mem := newOperandMem(m.lowerToAddressMode(ptr, offset)) + + store := m.allocateInstr() + switch value.Type() { + case ssa.TypeI32: + store.asMovRM(rm.reg(), mem, storeSizeInBits/8) + case ssa.TypeI64: + store.asMovRM(rm.reg(), mem, storeSizeInBits/8) + case ssa.TypeF32: + store.asXmmMovRM(sseOpcodeMovss, rm.reg(), mem) + case ssa.TypeF64: + store.asXmmMovRM(sseOpcodeMovsd, rm.reg(), mem) + case ssa.TypeV128: + store.asXmmMovRM(sseOpcodeMovdqu, rm.reg(), mem) + default: + panic("BUG") + } + m.insert(store) +} + +func (m *machine) lowerCall(si *ssa.Instruction) { + isDirectCall := si.Opcode() == ssa.OpcodeCall + var indirectCalleePtr ssa.Value + var directCallee ssa.FuncRef + var sigID ssa.SignatureID + var args []ssa.Value + var isMemmove bool + if isDirectCall { + directCallee, sigID, args = si.CallData() + } else { + indirectCalleePtr, sigID, args, isMemmove = si.CallIndirectData() + } + calleeABI := m.c.GetFunctionABI(m.c.SSABuilder().ResolveSignature(sigID)) + + stackSlotSize := int64(calleeABI.AlignedArgResultStackSlotSize()) + if m.maxRequiredStackSizeForCalls < stackSlotSize+16 { + m.maxRequiredStackSizeForCalls = stackSlotSize + 16 // 16 == return address + RBP. + } + + // Note: See machine.SetupPrologue for the stack layout. + // The stack pointer decrease/increase will be inserted later in the compilation. + + for i, arg := range args { + reg := m.c.VRegOf(arg) + def := m.c.ValueDefinition(arg) + m.callerGenVRegToFunctionArg(calleeABI, i, reg, def, stackSlotSize) + } + + if isMemmove { + // Go's memmove *might* use all xmm0-xmm15, so we need to release them. + // https://github.com/golang/go/blob/49d42128fd8594c172162961ead19ac95e247d24/src/cmd/compile/abi-internal.md#architecture-specifics + // https://github.com/golang/go/blob/49d42128fd8594c172162961ead19ac95e247d24/src/runtime/memmove_amd64.s#L271-L286 + for i := regalloc.RealReg(0); i < 16; i++ { + m.insert(m.allocateInstr().asDefineUninitializedReg(regInfo.RealRegToVReg[xmm0+i])) + } + } + + if isDirectCall { + call := m.allocateInstr().asCall(directCallee, calleeABI) + m.insert(call) + } else { + ptrOp := m.getOperand_Mem_Reg(m.c.ValueDefinition(indirectCalleePtr)) + callInd := m.allocateInstr().asCallIndirect(ptrOp, calleeABI) + m.insert(callInd) + } + + if isMemmove { + for i := regalloc.RealReg(0); i < 16; i++ { + m.insert(m.allocateInstr().asNopUseReg(regInfo.RealRegToVReg[xmm0+i])) + } + } + + var index int + r1, rs := si.Returns() + if r1.Valid() { + m.callerGenFunctionReturnVReg(calleeABI, 0, m.c.VRegOf(r1), stackSlotSize) + index++ + } + + for _, r := range rs { + m.callerGenFunctionReturnVReg(calleeABI, index, m.c.VRegOf(r), stackSlotSize) + index++ + } +} + +// callerGenVRegToFunctionArg is the opposite of GenFunctionArgToVReg, which is used to generate the +// caller side of the function call. +func (m *machine) callerGenVRegToFunctionArg(a *backend.FunctionABI, argIndex int, reg regalloc.VReg, def *backend.SSAValueDefinition, stackSlotSize int64) { + arg := &a.Args[argIndex] + if def != nil && def.IsFromInstr() { + // Constant instructions are inlined. + if inst := def.Instr; inst.Constant() { + m.insertLoadConstant(inst, reg) + } + } + if arg.Kind == backend.ABIArgKindReg { + m.InsertMove(arg.Reg, reg, arg.Type) + } else { + store := m.allocateInstr() + mem := newOperandMem(m.newAmodeImmReg( + // -stackSlotSize because the stack pointer is not yet decreased. + uint32(arg.Offset-stackSlotSize), rspVReg)) + switch arg.Type { + case ssa.TypeI32: + store.asMovRM(reg, mem, 4) + case ssa.TypeI64: + store.asMovRM(reg, mem, 8) + case ssa.TypeF32: + store.asXmmMovRM(sseOpcodeMovss, reg, mem) + case ssa.TypeF64: + store.asXmmMovRM(sseOpcodeMovsd, reg, mem) + case ssa.TypeV128: + store.asXmmMovRM(sseOpcodeMovdqu, reg, mem) + default: + panic("BUG") + } + m.insert(store) + } +} + +func (m *machine) callerGenFunctionReturnVReg(a *backend.FunctionABI, retIndex int, reg regalloc.VReg, stackSlotSize int64) { + r := &a.Rets[retIndex] + if r.Kind == backend.ABIArgKindReg { + m.InsertMove(reg, r.Reg, r.Type) + } else { + load := m.allocateInstr() + mem := newOperandMem(m.newAmodeImmReg( + // -stackSlotSize because the stack pointer is not yet decreased. + uint32(a.ArgStackSize+r.Offset-stackSlotSize), rspVReg)) + switch r.Type { + case ssa.TypeI32: + load.asMovzxRmR(extModeLQ, mem, reg) + case ssa.TypeI64: + load.asMov64MR(mem, reg) + case ssa.TypeF32: + load.asXmmUnaryRmR(sseOpcodeMovss, mem, reg) + case ssa.TypeF64: + load.asXmmUnaryRmR(sseOpcodeMovsd, mem, reg) + case ssa.TypeV128: + load.asXmmUnaryRmR(sseOpcodeMovdqu, mem, reg) + default: + panic("BUG") + } + m.insert(load) + } +} + +// InsertMove implements backend.Machine. +func (m *machine) InsertMove(dst, src regalloc.VReg, typ ssa.Type) { + switch typ { + case ssa.TypeI32, ssa.TypeI64: + i := m.allocateInstr().asMovRR(src, dst, typ.Bits() == 64) + m.insert(i) + case ssa.TypeF32, ssa.TypeF64, ssa.TypeV128: + var op sseOpcode + switch typ { + case ssa.TypeF32: + op = sseOpcodeMovss + case ssa.TypeF64: + op = sseOpcodeMovsd + case ssa.TypeV128: + op = sseOpcodeMovdqa + } + i := m.allocateInstr().asXmmUnaryRmR(op, newOperandReg(src), dst) + m.insert(i) + default: + panic("BUG") + } +} + +// Format implements backend.Machine. +func (m *machine) Format() string { + ectx := m.ectx + begins := map[*instruction]backend.Label{} + for l, pos := range ectx.LabelPositions { + begins[pos.Begin] = l + } + + irBlocks := map[backend.Label]ssa.BasicBlockID{} + for i, l := range ectx.SsaBlockIDToLabels { + irBlocks[l] = ssa.BasicBlockID(i) + } + + var lines []string + for cur := ectx.RootInstr; cur != nil; cur = cur.next { + if l, ok := begins[cur]; ok { + var labelStr string + if blkID, ok := irBlocks[l]; ok { + labelStr = fmt.Sprintf("%s (SSA Block: %s):", l, blkID) + } else { + labelStr = fmt.Sprintf("%s:", l) + } + lines = append(lines, labelStr) + } + if cur.kind == nop0 { + continue + } + lines = append(lines, "\t"+cur.String()) + } + for _, vc := range m.consts { + if vc._var == nil { + lines = append(lines, fmt.Sprintf("%s: const [%d %d]", vc.label.L, vc.lo, vc.hi)) + } else { + lines = append(lines, fmt.Sprintf("%s: const %#x", vc.label.L, vc._var)) + } + } + return "\n" + strings.Join(lines, "\n") + "\n" +} + +func (m *machine) encodeWithoutSSA(root *instruction) { + m.labelResolutionPends = m.labelResolutionPends[:0] + ectx := m.ectx + + bufPtr := m.c.BufPtr() + for cur := root; cur != nil; cur = cur.next { + offset := int64(len(*bufPtr)) + if cur.kind == nop0 { + l := cur.nop0Label() + if pos, ok := ectx.LabelPositions[l]; ok { + pos.BinaryOffset = offset + } + } + + needLabelResolution := cur.encode(m.c) + if needLabelResolution { + m.labelResolutionPends = append(m.labelResolutionPends, + labelResolutionPend{instr: cur, imm32Offset: int64(len(*bufPtr)) - 4}, + ) + } + } + + for i := range m.labelResolutionPends { + p := &m.labelResolutionPends[i] + switch p.instr.kind { + case jmp, jmpIf, lea: + target := p.instr.jmpLabel() + targetOffset := ectx.LabelPositions[target].BinaryOffset + imm32Offset := p.imm32Offset + jmpOffset := int32(targetOffset - (p.imm32Offset + 4)) // +4 because RIP points to the next instruction. + binary.LittleEndian.PutUint32((*bufPtr)[imm32Offset:], uint32(jmpOffset)) + default: + panic("BUG") + } + } +} + +// Encode implements backend.Machine Encode. +func (m *machine) Encode(ctx context.Context) (err error) { + ectx := m.ectx + bufPtr := m.c.BufPtr() + + var fn string + var fnIndex int + var labelToSSABlockID map[backend.Label]ssa.BasicBlockID + if wazevoapi.PerfMapEnabled { + fn = wazevoapi.GetCurrentFunctionName(ctx) + labelToSSABlockID = make(map[backend.Label]ssa.BasicBlockID) + for i, l := range ectx.SsaBlockIDToLabels { + labelToSSABlockID[l] = ssa.BasicBlockID(i) + } + fnIndex = wazevoapi.GetCurrentFunctionIndex(ctx) + } + + m.labelResolutionPends = m.labelResolutionPends[:0] + for _, pos := range ectx.OrderedBlockLabels { + offset := int64(len(*bufPtr)) + pos.BinaryOffset = offset + for cur := pos.Begin; cur != pos.End.next; cur = cur.next { + offset := int64(len(*bufPtr)) + + switch cur.kind { + case nop0: + l := cur.nop0Label() + if pos, ok := ectx.LabelPositions[l]; ok { + pos.BinaryOffset = offset + } + case sourceOffsetInfo: + m.c.AddSourceOffsetInfo(offset, cur.sourceOffsetInfo()) + } + + needLabelResolution := cur.encode(m.c) + if needLabelResolution { + m.labelResolutionPends = append(m.labelResolutionPends, + labelResolutionPend{instr: cur, instrOffset: offset, imm32Offset: int64(len(*bufPtr)) - 4}, + ) + } + } + + if wazevoapi.PerfMapEnabled { + l := pos.L + var labelStr string + if blkID, ok := labelToSSABlockID[l]; ok { + labelStr = fmt.Sprintf("%s::SSA_Block[%s]", l, blkID) + } else { + labelStr = l.String() + } + size := int64(len(*bufPtr)) - offset + wazevoapi.PerfMap.AddModuleEntry(fnIndex, offset, uint64(size), fmt.Sprintf("%s:::::%s", fn, labelStr)) + } + } + + for i := range m.consts { + offset := int64(len(*bufPtr)) + vc := &m.consts[i] + vc.label.BinaryOffset = offset + if vc._var == nil { + lo, hi := vc.lo, vc.hi + m.c.Emit8Bytes(lo) + m.c.Emit8Bytes(hi) + } else { + for _, b := range vc._var { + m.c.EmitByte(b) + } + } + } + + buf := *bufPtr + for i := range m.labelResolutionPends { + p := &m.labelResolutionPends[i] + switch p.instr.kind { + case jmp, jmpIf, lea, xmmUnaryRmR: + target := p.instr.jmpLabel() + targetOffset := ectx.LabelPositions[target].BinaryOffset + imm32Offset := p.imm32Offset + jmpOffset := int32(targetOffset - (p.imm32Offset + 4)) // +4 because RIP points to the next instruction. + binary.LittleEndian.PutUint32(buf[imm32Offset:], uint32(jmpOffset)) + case jmpTableIsland: + tableBegin := p.instrOffset + // Each entry is the offset from the beginning of the jmpTableIsland instruction in 8 bytes. + targets := m.jmpTableTargets[p.instr.u1] + for i, l := range targets { + targetOffset := ectx.LabelPositions[backend.Label(l)].BinaryOffset + jmpOffset := targetOffset - tableBegin + binary.LittleEndian.PutUint64(buf[tableBegin+int64(i)*8:], uint64(jmpOffset)) + } + default: + panic("BUG") + } + } + return +} + +// ResolveRelocations implements backend.Machine. +func (m *machine) ResolveRelocations(refToBinaryOffset []int, binary []byte, relocations []backend.RelocationInfo, _ []int) { + for _, r := range relocations { + offset := r.Offset + calleeFnOffset := refToBinaryOffset[r.FuncRef] + // offset is the offset of the last 4 bytes of the call instruction. + callInstrOffsetBytes := binary[offset : offset+4] + diff := int64(calleeFnOffset) - (offset + 4) // +4 because we want the offset of the next instruction (In x64, RIP always points to the next instruction). + callInstrOffsetBytes[0] = byte(diff) + callInstrOffsetBytes[1] = byte(diff >> 8) + callInstrOffsetBytes[2] = byte(diff >> 16) + callInstrOffsetBytes[3] = byte(diff >> 24) + } +} + +// CallTrampolineIslandInfo implements backend.Machine CallTrampolineIslandInfo. +func (m *machine) CallTrampolineIslandInfo(_ int) (_, _ int, _ error) { return } + +func (m *machine) lowerIcmpToFlag(xd, yd *backend.SSAValueDefinition, _64 bool) { + x := m.getOperand_Reg(xd) + y := m.getOperand_Mem_Imm32_Reg(yd) + cmp := m.allocateInstr().asCmpRmiR(true, y, x.reg(), _64) + m.insert(cmp) +} + +func (m *machine) lowerFcmpToFlags(instr *ssa.Instruction) (f1, f2 cond, and bool) { + x, y, c := instr.FcmpData() + switch c { + case ssa.FloatCmpCondEqual: + f1, f2 = condNP, condZ + and = true + case ssa.FloatCmpCondNotEqual: + f1, f2 = condP, condNZ + case ssa.FloatCmpCondLessThan: + f1 = condFromSSAFloatCmpCond(ssa.FloatCmpCondGreaterThan) + f2 = condInvalid + x, y = y, x + case ssa.FloatCmpCondLessThanOrEqual: + f1 = condFromSSAFloatCmpCond(ssa.FloatCmpCondGreaterThanOrEqual) + f2 = condInvalid + x, y = y, x + default: + f1 = condFromSSAFloatCmpCond(c) + f2 = condInvalid + } + + var opc sseOpcode + if x.Type() == ssa.TypeF32 { + opc = sseOpcodeUcomiss + } else { + opc = sseOpcodeUcomisd + } + + xr := m.getOperand_Reg(m.c.ValueDefinition(x)) + yr := m.getOperand_Mem_Reg(m.c.ValueDefinition(y)) + m.insert(m.allocateInstr().asXmmCmpRmR(opc, yr, xr.reg())) + return +} + +// allocateInstr allocates an instruction. +func (m *machine) allocateInstr() *instruction { + instr := m.ectx.InstructionPool.Allocate() + if !m.regAllocStarted { + instr.addedBeforeRegAlloc = true + } + return instr +} + +func (m *machine) allocateNop() *instruction { + instr := m.allocateInstr() + instr.kind = nop0 + return instr +} + +func (m *machine) insert(i *instruction) { + ectx := m.ectx + ectx.PendingInstructions = append(ectx.PendingInstructions, i) +} + +func (m *machine) allocateBrTarget() (nop *instruction, l backend.Label) { //nolint + pos := m.allocateLabel() + l = pos.L + nop = m.allocateInstr() + nop.asNop0WithLabel(l) + pos.Begin, pos.End = nop, nop + return +} + +func (m *machine) allocateLabel() *labelPosition { + ectx := m.ectx + l := ectx.AllocateLabel() + pos := ectx.AllocateLabelPosition(l) + ectx.LabelPositions[l] = pos + return pos +} + +func (m *machine) getVRegSpillSlotOffsetFromSP(id regalloc.VRegID, size byte) int64 { + offset, ok := m.spillSlots[id] + if !ok { + offset = m.spillSlotSize + m.spillSlots[id] = offset + m.spillSlotSize += int64(size) + } + return offset +} + +func (m *machine) copyTo(src regalloc.VReg, dst regalloc.VReg) { + mov := m.allocateInstr() + if src.RegType() == regalloc.RegTypeInt { + mov.asMovRR(src, dst, true) + } else { + mov.asXmmUnaryRmR(sseOpcodeMovdqu, newOperandReg(src), dst) + } + m.insert(mov) +} + +func (m *machine) copyToTmp(v regalloc.VReg) regalloc.VReg { + typ := m.c.TypeOf(v) + tmp := m.c.AllocateVReg(typ) + m.copyTo(v, tmp) + return tmp +} + +func (m *machine) requiredStackSize() int64 { + return m.maxRequiredStackSizeForCalls + + m.frameSize() + + 16 + // Need for stack checking. + 16 // return address and the caller RBP. +} + +func (m *machine) frameSize() int64 { + s := m.clobberedRegSlotSize() + m.spillSlotSize + if s&0xf != 0 { + panic(fmt.Errorf("BUG: frame size %d is not 16-byte aligned", s)) + } + return s +} + +func (m *machine) clobberedRegSlotSize() int64 { + return int64(len(m.clobberedRegs) * 16) +} + +func (m *machine) lowerIDivRem(si *ssa.Instruction, isDiv bool, signed bool) { + x, y, execCtx := si.Arg3() + + dividend := m.getOperand_Reg(m.c.ValueDefinition(x)) + divisor := m.getOperand_Reg(m.c.ValueDefinition(y)) + ctxVReg := m.c.VRegOf(execCtx) + tmpGp := m.c.AllocateVReg(si.Return().Type()) + + m.copyTo(dividend.reg(), raxVReg) + m.insert(m.allocateInstr().asDefineUninitializedReg(rdxVReg)) + m.insert(m.allocateInstr().asDefineUninitializedReg(tmpGp)) + seq := m.allocateInstr().asIdivRemSequence(ctxVReg, divisor.reg(), tmpGp, isDiv, signed, x.Type().Bits() == 64) + m.insert(seq) + rd := m.c.VRegOf(si.Return()) + if isDiv { + m.copyTo(raxVReg, rd) + } else { + m.copyTo(rdxVReg, rd) + } +} + +func (m *machine) lowerIDivRemSequenceAfterRegAlloc(i *instruction) { + execCtx, divisor, tmpGp, isDiv, signed, _64 := i.idivRemSequenceData() + + dividend := raxVReg + + // Ensure yr is not zero. + test := m.allocateInstr() + test.asCmpRmiR(false, newOperandReg(divisor), divisor, _64) + m.insert(test) + + jnz := m.allocateInstr() + m.insert(jnz) + + nz := m.lowerExitWithCode(execCtx, wazevoapi.ExitCodeIntegerDivisionByZero) + + // If not zero, we can proceed with the division. + jnz.asJmpIf(condNZ, newOperandLabel(nz)) + + var ifRemNeg1 *instruction + if signed { + var neg1 uint64 + if _64 { + neg1 = 0xffffffffffffffff + } else { + neg1 = 0xffffffff + } + m.lowerIconst(tmpGp, neg1, _64) + + if isDiv { + // For signed division, we have to have branches for "math.MinInt{32,64} / -1" + // case which results in the floating point exception via division error as + // the resulting value exceeds the maximum of signed int. + + // First, we check if the divisor is -1. + cmp := m.allocateInstr() + cmp.asCmpRmiR(true, newOperandReg(tmpGp), divisor, _64) + m.insert(cmp) + + ifNotNeg1 := m.allocateInstr() + m.insert(ifNotNeg1) + + var minInt uint64 + if _64 { + minInt = 0x8000000000000000 + } else { + minInt = 0x80000000 + } + m.lowerIconst(tmpGp, minInt, _64) + + // Next we check if the quotient is the most negative value for the signed integer, i.e. + // if we are trying to do (math.MinInt32 / -1) or (math.MinInt64 / -1) respectively. + cmp2 := m.allocateInstr() + cmp2.asCmpRmiR(true, newOperandReg(tmpGp), dividend, _64) + m.insert(cmp2) + + ifNotMinInt := m.allocateInstr() + m.insert(ifNotMinInt) + + // Trap if we are trying to do (math.MinInt32 / -1) or (math.MinInt64 / -1), + // as that is the overflow in division as the result becomes 2^31 which is larger than + // the maximum of signed 32-bit int (2^31-1). + end := m.lowerExitWithCode(execCtx, wazevoapi.ExitCodeIntegerOverflow) + ifNotNeg1.asJmpIf(condNZ, newOperandLabel(end)) + ifNotMinInt.asJmpIf(condNZ, newOperandLabel(end)) + } else { + // If it is remainder, zeros DX register and compare the divisor to -1. + xor := m.allocateInstr().asZeros(rdxVReg) + m.insert(xor) + + // We check if the divisor is -1. + cmp := m.allocateInstr() + cmp.asCmpRmiR(true, newOperandReg(tmpGp), divisor, _64) + m.insert(cmp) + + ifRemNeg1 = m.allocateInstr() + m.insert(ifRemNeg1) + } + + // Sign-extend DX register to have 2*x.Type().Bits() dividend over DX and AX registers. + sed := m.allocateInstr() + sed.asSignExtendData(_64) + m.insert(sed) + } else { + // Zeros DX register to have 2*x.Type().Bits() dividend over DX and AX registers. + zeros := m.allocateInstr().asZeros(rdxVReg) + m.insert(zeros) + } + + div := m.allocateInstr() + div.asDiv(newOperandReg(divisor), signed, _64) + m.insert(div) + + nop, end := m.allocateBrTarget() + m.insert(nop) + // If we are compiling a Rem instruction, when the divisor is -1 we land at the end of the function. + if ifRemNeg1 != nil { + ifRemNeg1.asJmpIf(condZ, newOperandLabel(end)) + } +} + +func (m *machine) lowerRound(instr *ssa.Instruction, imm roundingMode) { + x := instr.Arg() + if !x.Type().IsFloat() { + panic("BUG?") + } + var op sseOpcode + if x.Type().Bits() == 64 { + op = sseOpcodeRoundsd + } else { + op = sseOpcodeRoundss + } + + xDef := m.c.ValueDefinition(x) + rm := m.getOperand_Mem_Reg(xDef) + rd := m.c.VRegOf(instr.Return()) + + xmm := m.allocateInstr().asXmmUnaryRmRImm(op, uint8(imm), rm, rd) + m.insert(xmm) +} + +func (m *machine) lowerFminFmax(instr *ssa.Instruction) { + x, y := instr.Arg2() + if !x.Type().IsFloat() { + panic("BUG?") + } + + _64 := x.Type().Bits() == 64 + isMin := instr.Opcode() == ssa.OpcodeFmin + var minMaxOp sseOpcode + + switch { + case _64 && isMin: + minMaxOp = sseOpcodeMinpd + case _64 && !isMin: + minMaxOp = sseOpcodeMaxpd + case !_64 && isMin: + minMaxOp = sseOpcodeMinps + case !_64 && !isMin: + minMaxOp = sseOpcodeMaxps + } + + xDef, yDef := m.c.ValueDefinition(x), m.c.ValueDefinition(y) + rm := m.getOperand_Reg(xDef) + // We cannot ensure that y is aligned to 16 bytes, so we have to use it on reg. + rn := m.getOperand_Reg(yDef) + rd := m.c.VRegOf(instr.Return()) + + tmp := m.copyToTmp(rm.reg()) + + // Check if this is (either x1 or x2 is NaN) or (x1 equals x2) case. + cmp := m.allocateInstr() + if _64 { + cmp.asXmmCmpRmR(sseOpcodeUcomisd, rn, tmp) + } else { + cmp.asXmmCmpRmR(sseOpcodeUcomiss, rn, tmp) + } + m.insert(cmp) + + // At this point, we have the three cases of conditional flags below + // (See https://www.felixcloutier.com/x86/ucomiss#operation for detail.) + // + // 1) Two values are NaN-free and different: All flags are cleared. + // 2) Two values are NaN-free and equal: Only ZF flags is set. + // 3) One of Two values is NaN: ZF, PF and CF flags are set. + + // Jump instruction to handle 1) case by checking the ZF flag + // as ZF is only set for 2) and 3) cases. + nanFreeOrDiffJump := m.allocateInstr() + m.insert(nanFreeOrDiffJump) + + // Start handling 2) and 3). + + // Jump if one of two values is NaN by checking the parity flag (PF). + ifIsNan := m.allocateInstr() + m.insert(ifIsNan) + + // Start handling 2) NaN-free and equal. + + // Before we exit this case, we have to ensure that positive zero (or negative zero for min instruction) is + // returned if two values are positive and negative zeros. + var op sseOpcode + switch { + case !_64 && isMin: + op = sseOpcodeOrps + case _64 && isMin: + op = sseOpcodeOrpd + case !_64 && !isMin: + op = sseOpcodeAndps + case _64 && !isMin: + op = sseOpcodeAndpd + } + orAnd := m.allocateInstr() + orAnd.asXmmRmR(op, rn, tmp) + m.insert(orAnd) + + // Done, jump to end. + sameExitJump := m.allocateInstr() + m.insert(sameExitJump) + + // Start handling 3) either is NaN. + isNanTarget, isNan := m.allocateBrTarget() + m.insert(isNanTarget) + ifIsNan.asJmpIf(condP, newOperandLabel(isNan)) + + // We emit the ADD instruction to produce the NaN in tmp. + add := m.allocateInstr() + if _64 { + add.asXmmRmR(sseOpcodeAddsd, rn, tmp) + } else { + add.asXmmRmR(sseOpcodeAddss, rn, tmp) + } + m.insert(add) + + // Exit from the NaN case branch. + nanExitJmp := m.allocateInstr() + m.insert(nanExitJmp) + + // Start handling 1). + doMinMaxTarget, doMinMax := m.allocateBrTarget() + m.insert(doMinMaxTarget) + nanFreeOrDiffJump.asJmpIf(condNZ, newOperandLabel(doMinMax)) + + // Now handle the NaN-free and different values case. + minMax := m.allocateInstr() + minMax.asXmmRmR(minMaxOp, rn, tmp) + m.insert(minMax) + + endNop, end := m.allocateBrTarget() + m.insert(endNop) + nanExitJmp.asJmp(newOperandLabel(end)) + sameExitJump.asJmp(newOperandLabel(end)) + + m.copyTo(tmp, rd) +} + +func (m *machine) lowerFcopysign(instr *ssa.Instruction) { + x, y := instr.Arg2() + if !x.Type().IsFloat() { + panic("BUG") + } + + _64 := x.Type().Bits() == 64 + + xDef, yDef := m.c.ValueDefinition(x), m.c.ValueDefinition(y) + rm := m.getOperand_Reg(xDef) + rn := m.getOperand_Reg(yDef) + rd := m.c.VRegOf(instr.Return()) + + // Clear the non-sign bits of src via AND with the mask. + var opAnd, opOr sseOpcode + var signMask uint64 + if _64 { + signMask, opAnd, opOr = 0x8000000000000000, sseOpcodeAndpd, sseOpcodeOrpd + } else { + signMask, opAnd, opOr = 0x80000000, sseOpcodeAndps, sseOpcodeOrps + } + + signBitReg := m.c.AllocateVReg(x.Type()) + m.lowerFconst(signBitReg, signMask, _64) + nonSignBitReg := m.c.AllocateVReg(x.Type()) + m.lowerFconst(nonSignBitReg, ^signMask, _64) + + // Extract the sign bits of rn. + and := m.allocateInstr().asXmmRmR(opAnd, rn, signBitReg) + m.insert(and) + + // Clear the sign bit of dst via AND with the non-sign bit mask. + xor := m.allocateInstr().asXmmRmR(opAnd, rm, nonSignBitReg) + m.insert(xor) + + // Copy the sign bits of src to dst via OR. + or := m.allocateInstr().asXmmRmR(opOr, newOperandReg(signBitReg), nonSignBitReg) + m.insert(or) + + m.copyTo(nonSignBitReg, rd) +} + +func (m *machine) lowerBitcast(instr *ssa.Instruction) { + x, dstTyp := instr.BitcastData() + srcTyp := x.Type() + rn := m.getOperand_Reg(m.c.ValueDefinition(x)) + rd := m.c.VRegOf(instr.Return()) + switch { + case srcTyp == ssa.TypeF32 && dstTyp == ssa.TypeI32: + cvt := m.allocateInstr().asXmmToGpr(sseOpcodeMovd, rn.reg(), rd, false) + m.insert(cvt) + case srcTyp == ssa.TypeI32 && dstTyp == ssa.TypeF32: + cvt := m.allocateInstr().asGprToXmm(sseOpcodeMovd, rn, rd, false) + m.insert(cvt) + case srcTyp == ssa.TypeF64 && dstTyp == ssa.TypeI64: + cvt := m.allocateInstr().asXmmToGpr(sseOpcodeMovq, rn.reg(), rd, true) + m.insert(cvt) + case srcTyp == ssa.TypeI64 && dstTyp == ssa.TypeF64: + cvt := m.allocateInstr().asGprToXmm(sseOpcodeMovq, rn, rd, true) + m.insert(cvt) + default: + panic(fmt.Sprintf("invalid bitcast from %s to %s", srcTyp, dstTyp)) + } +} + +func (m *machine) lowerFcvtToSint(ctxVReg, rn, rd regalloc.VReg, src64, dst64, sat bool) { + var tmpXmm regalloc.VReg + if dst64 { + tmpXmm = m.c.AllocateVReg(ssa.TypeF64) + } else { + tmpXmm = m.c.AllocateVReg(ssa.TypeF32) + } + + m.insert(m.allocateInstr().asDefineUninitializedReg(tmpXmm)) + tmpGp, tmpGp2 := m.c.AllocateVReg(ssa.TypeI64), m.c.AllocateVReg(ssa.TypeI64) + m.insert(m.allocateInstr().asDefineUninitializedReg(tmpGp)) + m.insert(m.allocateInstr().asDefineUninitializedReg(tmpGp2)) + + m.insert(m.allocateFcvtToSintSequence(ctxVReg, rn, tmpGp, tmpGp2, tmpXmm, src64, dst64, sat)) + m.copyTo(tmpGp, rd) +} + +func (m *machine) lowerFcvtToSintSequenceAfterRegalloc(i *instruction) { + execCtx, src, tmpGp, tmpGp2, tmpXmm, src64, dst64, sat := i.fcvtToSintSequenceData() + var cmpOp, truncOp sseOpcode + if src64 { + cmpOp, truncOp = sseOpcodeUcomisd, sseOpcodeCvttsd2si + } else { + cmpOp, truncOp = sseOpcodeUcomiss, sseOpcodeCvttss2si + } + + trunc := m.allocateInstr() + trunc.asXmmToGpr(truncOp, src, tmpGp, dst64) + m.insert(trunc) + + // Check if the dst operand was INT_MIN, by checking it against 1. + cmp1 := m.allocateInstr() + cmp1.asCmpRmiR(true, newOperandImm32(1), tmpGp, dst64) + m.insert(cmp1) + + // If no overflow, then we are done. + doneTarget, done := m.allocateBrTarget() + ifNoOverflow := m.allocateInstr() + ifNoOverflow.asJmpIf(condNO, newOperandLabel(done)) + m.insert(ifNoOverflow) + + // Now, check for NaN. + cmpNan := m.allocateInstr() + cmpNan.asXmmCmpRmR(cmpOp, newOperandReg(src), src) + m.insert(cmpNan) + + // We allocate the "non-nan target" here, but we will insert it later. + notNanTarget, notNaN := m.allocateBrTarget() + ifNotNan := m.allocateInstr() + ifNotNan.asJmpIf(condNP, newOperandLabel(notNaN)) + m.insert(ifNotNan) + + if sat { + // If NaN and saturating, return 0. + zeroDst := m.allocateInstr().asZeros(tmpGp) + m.insert(zeroDst) + + jmpEnd := m.allocateInstr() + jmpEnd.asJmp(newOperandLabel(done)) + m.insert(jmpEnd) + + // Otherwise: + m.insert(notNanTarget) + + // Zero-out the tmp register. + zero := m.allocateInstr().asZeros(tmpXmm) + m.insert(zero) + + cmpXmm := m.allocateInstr().asXmmCmpRmR(cmpOp, newOperandReg(tmpXmm), src) + m.insert(cmpXmm) + + // if >= jump to end. + jmpEnd2 := m.allocateInstr() + jmpEnd2.asJmpIf(condB, newOperandLabel(done)) + m.insert(jmpEnd2) + + // Otherwise, saturate to INT_MAX. + if dst64 { + m.lowerIconst(tmpGp, math.MaxInt64, dst64) + } else { + m.lowerIconst(tmpGp, math.MaxInt32, dst64) + } + + } else { + + // If non-sat, NaN, trap. + m.lowerExitWithCode(execCtx, wazevoapi.ExitCodeInvalidConversionToInteger) + + // Otherwise, we will jump here. + m.insert(notNanTarget) + + // jump over trap if src larger than threshold + condAboveThreshold := condNB + + // The magic constants are various combination of minInt for int[32|64] represented as float[32|64]. + var minInt uint64 + switch { + case src64 && dst64: + minInt = 0xc3e0000000000000 + case src64 && !dst64: + condAboveThreshold = condNBE + minInt = 0xC1E0_0000_0020_0000 + case !src64 && dst64: + minInt = 0xDF00_0000 + case !src64 && !dst64: + minInt = 0xCF00_0000 + } + + loadToGP := m.allocateInstr().asImm(tmpGp2, minInt, src64) + m.insert(loadToGP) + + movToXmm := m.allocateInstr().asGprToXmm(sseOpcodeMovq, newOperandReg(tmpGp2), tmpXmm, src64) + m.insert(movToXmm) + + cmpXmm := m.allocateInstr().asXmmCmpRmR(cmpOp, newOperandReg(tmpXmm), src) + m.insert(cmpXmm) + + jmpIfLarger := m.allocateInstr() + checkPositiveTarget, checkPositive := m.allocateBrTarget() + jmpIfLarger.asJmpIf(condAboveThreshold, newOperandLabel(checkPositive)) + m.insert(jmpIfLarger) + + m.lowerExitWithCode(execCtx, wazevoapi.ExitCodeIntegerOverflow) + + // If positive, it was a real overflow. + m.insert(checkPositiveTarget) + + // Zero out the temp register. + xorpd := m.allocateInstr() + xorpd.asXmmRmR(sseOpcodeXorpd, newOperandReg(tmpXmm), tmpXmm) + m.insert(xorpd) + + pos := m.allocateInstr() + pos.asXmmCmpRmR(cmpOp, newOperandReg(src), tmpXmm) + m.insert(pos) + + // If >= jump to end. + jmp := m.allocateInstr().asJmpIf(condNB, newOperandLabel(done)) + m.insert(jmp) + m.lowerExitWithCode(execCtx, wazevoapi.ExitCodeIntegerOverflow) + } + + m.insert(doneTarget) +} + +func (m *machine) lowerFcvtToUint(ctxVReg, rn, rd regalloc.VReg, src64, dst64, sat bool) { + tmpXmm, tmpXmm2 := m.c.AllocateVReg(ssa.TypeF64), m.c.AllocateVReg(ssa.TypeF64) + m.insert(m.allocateInstr().asDefineUninitializedReg(tmpXmm)) + m.insert(m.allocateInstr().asDefineUninitializedReg(tmpXmm2)) + tmpGp, tmpGp2 := m.c.AllocateVReg(ssa.TypeI64), m.c.AllocateVReg(ssa.TypeI64) + m.insert(m.allocateInstr().asDefineUninitializedReg(tmpGp)) + m.insert(m.allocateInstr().asDefineUninitializedReg(tmpGp2)) + + m.insert(m.allocateFcvtToUintSequence( + ctxVReg, rn, tmpGp, tmpGp2, tmpXmm, tmpXmm2, src64, dst64, sat, + )) + m.copyTo(tmpGp, rd) +} + +func (m *machine) lowerFcvtToUintSequenceAfterRegalloc(i *instruction) { + execCtx, src, tmpGp, tmpGp2, tmpXmm, tmpXmm2, src64, dst64, sat := i.fcvtToUintSequenceData() + + var subOp, cmpOp, truncOp sseOpcode + if src64 { + subOp, cmpOp, truncOp = sseOpcodeSubsd, sseOpcodeUcomisd, sseOpcodeCvttsd2si + } else { + subOp, cmpOp, truncOp = sseOpcodeSubss, sseOpcodeUcomiss, sseOpcodeCvttss2si + } + + doneTarget, done := m.allocateBrTarget() + + switch { + case src64 && dst64: + loadToGP := m.allocateInstr().asImm(tmpGp, 0x43e0000000000000, true) + m.insert(loadToGP) + movToXmm := m.allocateInstr().asGprToXmm(sseOpcodeMovq, newOperandReg(tmpGp), tmpXmm, true) + m.insert(movToXmm) + case src64 && !dst64: + loadToGP := m.allocateInstr().asImm(tmpGp, 0x41e0000000000000, true) + m.insert(loadToGP) + movToXmm := m.allocateInstr().asGprToXmm(sseOpcodeMovq, newOperandReg(tmpGp), tmpXmm, true) + m.insert(movToXmm) + case !src64 && dst64: + loadToGP := m.allocateInstr().asImm(tmpGp, 0x5f000000, false) + m.insert(loadToGP) + movToXmm := m.allocateInstr().asGprToXmm(sseOpcodeMovq, newOperandReg(tmpGp), tmpXmm, false) + m.insert(movToXmm) + case !src64 && !dst64: + loadToGP := m.allocateInstr().asImm(tmpGp, 0x4f000000, false) + m.insert(loadToGP) + movToXmm := m.allocateInstr().asGprToXmm(sseOpcodeMovq, newOperandReg(tmpGp), tmpXmm, false) + m.insert(movToXmm) + } + + cmp := m.allocateInstr() + cmp.asXmmCmpRmR(cmpOp, newOperandReg(tmpXmm), src) + m.insert(cmp) + + // If above `tmp` ("large threshold"), jump to `ifAboveThreshold` + ifAboveThresholdTarget, ifAboveThreshold := m.allocateBrTarget() + jmpIfAboveThreshold := m.allocateInstr() + jmpIfAboveThreshold.asJmpIf(condNB, newOperandLabel(ifAboveThreshold)) + m.insert(jmpIfAboveThreshold) + + ifNotNaNTarget, ifNotNaN := m.allocateBrTarget() + jmpIfNotNaN := m.allocateInstr() + jmpIfNotNaN.asJmpIf(condNP, newOperandLabel(ifNotNaN)) + m.insert(jmpIfNotNaN) + + // If NaN, handle the error condition. + if sat { + // On NaN, saturating, we just return 0. + zeros := m.allocateInstr().asZeros(tmpGp) + m.insert(zeros) + + jmpEnd := m.allocateInstr() + jmpEnd.asJmp(newOperandLabel(done)) + m.insert(jmpEnd) + } else { + // On NaN, non-saturating, we trap. + m.lowerExitWithCode(execCtx, wazevoapi.ExitCodeInvalidConversionToInteger) + } + + // If not NaN, land here. + m.insert(ifNotNaNTarget) + + // Truncation happens here. + + trunc := m.allocateInstr() + trunc.asXmmToGpr(truncOp, src, tmpGp, dst64) + m.insert(trunc) + + // Check if the result is negative. + cmpNeg := m.allocateInstr() + cmpNeg.asCmpRmiR(true, newOperandImm32(0), tmpGp, dst64) + m.insert(cmpNeg) + + // If non-neg, jump to end. + jmpIfNonNeg := m.allocateInstr() + jmpIfNonNeg.asJmpIf(condNL, newOperandLabel(done)) + m.insert(jmpIfNonNeg) + + if sat { + // If the input was "small" (< 2**(width -1)), the only way to get an integer + // overflow is because the input was too small: saturate to the min value, i.e. 0. + zeros := m.allocateInstr().asZeros(tmpGp) + m.insert(zeros) + + jmpEnd := m.allocateInstr() + jmpEnd.asJmp(newOperandLabel(done)) + m.insert(jmpEnd) + } else { + // If not saturating, trap. + m.lowerExitWithCode(execCtx, wazevoapi.ExitCodeIntegerOverflow) + } + + // If above the threshold, land here. + m.insert(ifAboveThresholdTarget) + + // tmpDiff := threshold - rn. + copySrc := m.allocateInstr().asXmmUnaryRmR(sseOpcodeMovdqu, newOperandReg(src), tmpXmm2) + m.insert(copySrc) + + sub := m.allocateInstr() + sub.asXmmRmR(subOp, newOperandReg(tmpXmm), tmpXmm2) // must be -0x8000000000000000 + m.insert(sub) + + trunc2 := m.allocateInstr() + trunc2.asXmmToGpr(truncOp, tmpXmm2, tmpGp, dst64) + m.insert(trunc2) + + // Check if the result is negative. + cmpNeg2 := m.allocateInstr().asCmpRmiR(true, newOperandImm32(0), tmpGp, dst64) + m.insert(cmpNeg2) + + ifNextLargeTarget, ifNextLarge := m.allocateBrTarget() + jmpIfNextLarge := m.allocateInstr() + jmpIfNextLarge.asJmpIf(condNL, newOperandLabel(ifNextLarge)) + m.insert(jmpIfNextLarge) + + if sat { + // The input was "large" (>= maxInt), so the only way to get an integer + // overflow is because the input was too large: saturate to the max value. + var maxInt uint64 + if dst64 { + maxInt = math.MaxUint64 + } else { + maxInt = math.MaxUint32 + } + m.lowerIconst(tmpGp, maxInt, dst64) + + jmpToEnd := m.allocateInstr() + jmpToEnd.asJmp(newOperandLabel(done)) + m.insert(jmpToEnd) + } else { + // If not saturating, trap. + m.lowerExitWithCode(execCtx, wazevoapi.ExitCodeIntegerOverflow) + } + + m.insert(ifNextLargeTarget) + + var op operand + if dst64 { + m.lowerIconst(tmpGp2, 0x8000000000000000, true) + op = newOperandReg(tmpGp2) + } else { + op = newOperandImm32(0x80000000) + } + + add := m.allocateInstr() + add.asAluRmiR(aluRmiROpcodeAdd, op, tmpGp, dst64) + m.insert(add) + + m.insert(doneTarget) +} + +func (m *machine) lowerFcvtFromSint(rn, rd operand, src64, dst64 bool) { + var op sseOpcode + if dst64 { + op = sseOpcodeCvtsi2sd + } else { + op = sseOpcodeCvtsi2ss + } + + trunc := m.allocateInstr() + trunc.asGprToXmm(op, rn, rd.reg(), src64) + m.insert(trunc) +} + +func (m *machine) lowerFcvtFromUint(rn, rd operand, src64, dst64 bool) { + var op sseOpcode + if dst64 { + op = sseOpcodeCvtsi2sd + } else { + op = sseOpcodeCvtsi2ss + } + + // Src is 32 bit, then we just perform the conversion with 64 bit width. + // + // See the following link for why we use 64bit conversion for unsigned 32bit integer sources: + // https://stackoverflow.com/questions/41495498/fpu-operations-generated-by-gcc-during-casting-integer-to-float. + // + // Here's the summary: + // >> CVTSI2SS is indeed designed for converting a signed integer to a scalar single-precision float, + // >> not an unsigned integer like you have here. So what gives? Well, a 64-bit processor has 64-bit wide + // >> registers available, so the unsigned 32-bit input values can be stored as signed 64-bit intermediate values, + // >> which allows CVTSI2SS to be used after all. + // + if !src64 { + // Before we convert, we have to clear the higher 32-bits of the 64-bit register + // to get the correct result. + tmp := m.c.AllocateVReg(ssa.TypeI32) + m.insert(m.allocateInstr().asMovzxRmR(extModeLQ, rn, tmp)) + m.insert(m.allocateInstr().asGprToXmm(op, newOperandReg(tmp), rd.reg(), true)) + return + } + + // If uint64, we have to do a bit more work. + endTarget, end := m.allocateBrTarget() + + var tmpXmm regalloc.VReg + if dst64 { + tmpXmm = m.c.AllocateVReg(ssa.TypeF64) + } else { + tmpXmm = m.c.AllocateVReg(ssa.TypeF32) + } + + // Check if the most significant bit (sign bit) is set. + test := m.allocateInstr() + test.asCmpRmiR(false, rn, rn.reg(), src64) + m.insert(test) + + // Jump if the sign bit is set. + ifSignTarget, ifSign := m.allocateBrTarget() + jmpIfNeg := m.allocateInstr() + jmpIfNeg.asJmpIf(condS, newOperandLabel(ifSign)) + m.insert(jmpIfNeg) + + // If the sign bit is not set, we could fit the unsigned int into float32/float64. + // So, we convert it to float and emit jump instruction to exit from this branch. + cvt := m.allocateInstr() + cvt.asGprToXmm(op, rn, tmpXmm, src64) + m.insert(cvt) + + // We are done, jump to end. + jmpEnd := m.allocateInstr() + jmpEnd.asJmp(newOperandLabel(end)) + m.insert(jmpEnd) + + // Now handling the case where sign-bit is set. + // We emit the following sequences: + // mov %rn, %tmp + // shr 1, %tmp + // mov %rn, %tmp2 + // and 1, %tmp2 + // or %tmp2, %tmp + // cvtsi2ss %tmp, %xmm0 + // addsd %xmm0, %xmm0 + m.insert(ifSignTarget) + + tmp := m.copyToTmp(rn.reg()) + shr := m.allocateInstr() + shr.asShiftR(shiftROpShiftRightLogical, newOperandImm32(1), tmp, src64) + m.insert(shr) + + tmp2 := m.copyToTmp(rn.reg()) + and := m.allocateInstr() + and.asAluRmiR(aluRmiROpcodeAnd, newOperandImm32(1), tmp2, src64) + m.insert(and) + + or := m.allocateInstr() + or.asAluRmiR(aluRmiROpcodeOr, newOperandReg(tmp2), tmp, src64) + m.insert(or) + + cvt2 := m.allocateInstr() + cvt2.asGprToXmm(op, newOperandReg(tmp), tmpXmm, src64) + m.insert(cvt2) + + addsd := m.allocateInstr() + if dst64 { + addsd.asXmmRmR(sseOpcodeAddsd, newOperandReg(tmpXmm), tmpXmm) + } else { + addsd.asXmmRmR(sseOpcodeAddss, newOperandReg(tmpXmm), tmpXmm) + } + m.insert(addsd) + + m.insert(endTarget) + m.copyTo(tmpXmm, rd.reg()) +} + +func (m *machine) lowerVanyTrue(instr *ssa.Instruction) { + x := instr.Arg() + rm := m.getOperand_Reg(m.c.ValueDefinition(x)) + rd := m.c.VRegOf(instr.Return()) + + tmp := m.c.AllocateVReg(ssa.TypeI32) + + cmp := m.allocateInstr() + cmp.asXmmCmpRmR(sseOpcodePtest, rm, rm.reg()) + m.insert(cmp) + + setcc := m.allocateInstr() + setcc.asSetcc(condNZ, tmp) + m.insert(setcc) + + // Clear the irrelevant bits. + and := m.allocateInstr() + and.asAluRmiR(aluRmiROpcodeAnd, newOperandImm32(1), tmp, false) + m.insert(and) + + m.copyTo(tmp, rd) +} + +func (m *machine) lowerVallTrue(instr *ssa.Instruction) { + x, lane := instr.ArgWithLane() + var op sseOpcode + switch lane { + case ssa.VecLaneI8x16: + op = sseOpcodePcmpeqb + case ssa.VecLaneI16x8: + op = sseOpcodePcmpeqw + case ssa.VecLaneI32x4: + op = sseOpcodePcmpeqd + case ssa.VecLaneI64x2: + op = sseOpcodePcmpeqq + } + rm := m.getOperand_Reg(m.c.ValueDefinition(x)) + rd := m.c.VRegOf(instr.Return()) + + tmp := m.c.AllocateVReg(ssa.TypeV128) + + zeros := m.allocateInstr() + zeros.asZeros(tmp) + m.insert(zeros) + + pcmp := m.allocateInstr() + pcmp.asXmmRmR(op, rm, tmp) + m.insert(pcmp) + + test := m.allocateInstr() + test.asXmmCmpRmR(sseOpcodePtest, newOperandReg(tmp), tmp) + m.insert(test) + + tmp2 := m.c.AllocateVReg(ssa.TypeI32) + + setcc := m.allocateInstr() + setcc.asSetcc(condZ, tmp2) + m.insert(setcc) + + // Clear the irrelevant bits. + and := m.allocateInstr() + and.asAluRmiR(aluRmiROpcodeAnd, newOperandImm32(1), tmp2, false) + m.insert(and) + + m.copyTo(tmp2, rd) +} + +func (m *machine) lowerVhighBits(instr *ssa.Instruction) { + x, lane := instr.ArgWithLane() + rm := m.getOperand_Reg(m.c.ValueDefinition(x)) + rd := m.c.VRegOf(instr.Return()) + switch lane { + case ssa.VecLaneI8x16: + mov := m.allocateInstr() + mov.asXmmToGpr(sseOpcodePmovmskb, rm.reg(), rd, false) + m.insert(mov) + + case ssa.VecLaneI16x8: + // When we have: + // R1 = [R1(w1), R1(w2), R1(w3), R1(w4), R1(w5), R1(w6), R1(w7), R1(v8)] + // R2 = [R2(w1), R2(w2), R2(w3), R2(v4), R2(w5), R2(w6), R2(w7), R2(v8)] + // where RX(wn) is n-th signed word (16-bit) of RX register, + // + // "PACKSSWB R1, R2" produces + // R1 = [ + // byte_sat(R1(w1)), byte_sat(R1(w2)), byte_sat(R1(w3)), byte_sat(R1(w4)), + // byte_sat(R1(w5)), byte_sat(R1(w6)), byte_sat(R1(w7)), byte_sat(R1(w8)), + // byte_sat(R2(w1)), byte_sat(R2(w2)), byte_sat(R2(w3)), byte_sat(R2(w4)), + // byte_sat(R2(w5)), byte_sat(R2(w6)), byte_sat(R2(w7)), byte_sat(R2(w8)), + // ] + // where R1 is the destination register, and + // byte_sat(w) = int8(w) if w fits as signed 8-bit, + // 0x80 if w is less than 0x80 + // 0x7F if w is greater than 0x7f + // + // See https://www.felixcloutier.com/x86/packsswb:packssdw for detail. + // + // Therefore, v.register ends up having i-th and (i+8)-th bit set if i-th lane is negative (for i in 0..8). + tmp := m.copyToTmp(rm.reg()) + res := m.c.AllocateVReg(ssa.TypeI32) + + pak := m.allocateInstr() + pak.asXmmRmR(sseOpcodePacksswb, rm, tmp) + m.insert(pak) + + mov := m.allocateInstr() + mov.asXmmToGpr(sseOpcodePmovmskb, tmp, res, false) + m.insert(mov) + + // Clear the higher bits than 8. + shr := m.allocateInstr() + shr.asShiftR(shiftROpShiftRightLogical, newOperandImm32(8), res, false) + m.insert(shr) + + m.copyTo(res, rd) + + case ssa.VecLaneI32x4: + mov := m.allocateInstr() + mov.asXmmToGpr(sseOpcodeMovmskps, rm.reg(), rd, true) + m.insert(mov) + + case ssa.VecLaneI64x2: + mov := m.allocateInstr() + mov.asXmmToGpr(sseOpcodeMovmskpd, rm.reg(), rd, true) + m.insert(mov) + } +} + +func (m *machine) lowerVbnot(instr *ssa.Instruction) { + x := instr.Arg() + xDef := m.c.ValueDefinition(x) + rm := m.getOperand_Reg(xDef) + rd := m.c.VRegOf(instr.Return()) + + tmp := m.copyToTmp(rm.reg()) + tmp2 := m.c.AllocateVReg(ssa.TypeV128) + + // Ensure tmp2 is considered defined by regalloc. + m.insert(m.allocateInstr().asDefineUninitializedReg(tmp2)) + + // Set all bits on tmp register. + pak := m.allocateInstr() + pak.asXmmRmR(sseOpcodePcmpeqd, newOperandReg(tmp2), tmp2) + m.insert(pak) + + // Then XOR with tmp to reverse all bits on v.register. + xor := m.allocateInstr() + xor.asXmmRmR(sseOpcodePxor, newOperandReg(tmp2), tmp) + m.insert(xor) + + m.copyTo(tmp, rd) +} + +func (m *machine) lowerSplat(x, ret ssa.Value, lane ssa.VecLane) { + tmpDst := m.c.AllocateVReg(ssa.TypeV128) + m.insert(m.allocateInstr().asDefineUninitializedReg(tmpDst)) + + switch lane { + case ssa.VecLaneI8x16: + tmp := m.c.AllocateVReg(ssa.TypeV128) + m.insert(m.allocateInstr().asDefineUninitializedReg(tmp)) + xx := m.getOperand_Mem_Reg(m.c.ValueDefinition(x)) + m.insert(m.allocateInstr().asXmmRmRImm(sseOpcodePinsrb, 0, xx, tmpDst)) + m.insert(m.allocateInstr().asXmmRmR(sseOpcodePxor, newOperandReg(tmp), tmp)) + m.insert(m.allocateInstr().asXmmRmR(sseOpcodePshufb, newOperandReg(tmp), tmpDst)) + case ssa.VecLaneI16x8: + xx := m.getOperand_Reg(m.c.ValueDefinition(x)) + m.insert(m.allocateInstr().asXmmRmRImm(sseOpcodePinsrw, 0, xx, tmpDst)) + m.insert(m.allocateInstr().asXmmRmRImm(sseOpcodePinsrw, 1, xx, tmpDst)) + m.insert(m.allocateInstr().asXmmRmRImm(sseOpcodePshufd, 0, newOperandReg(tmpDst), tmpDst)) + case ssa.VecLaneI32x4: + xx := m.getOperand_Mem_Reg(m.c.ValueDefinition(x)) + m.insert(m.allocateInstr().asXmmRmRImm(sseOpcodePinsrd, 0, xx, tmpDst)) + m.insert(m.allocateInstr().asXmmRmRImm(sseOpcodePshufd, 0, newOperandReg(tmpDst), tmpDst)) + case ssa.VecLaneI64x2: + xx := m.getOperand_Reg(m.c.ValueDefinition(x)) + m.insert(m.allocateInstr().asXmmRmRImm(sseOpcodePinsrq, 0, xx, tmpDst)) + m.insert(m.allocateInstr().asXmmRmRImm(sseOpcodePinsrq, 1, xx, tmpDst)) + case ssa.VecLaneF32x4: + xx := m.getOperand_Mem_Reg(m.c.ValueDefinition(x)) + m.insert(m.allocateInstr().asXmmRmRImm(sseOpcodeInsertps, 0, xx, tmpDst)) + m.insert(m.allocateInstr().asXmmRmRImm(sseOpcodePshufd, 0, newOperandReg(tmpDst), tmpDst)) + case ssa.VecLaneF64x2: + xx := m.getOperand_Reg(m.c.ValueDefinition(x)) + m.insert(m.allocateInstr().asXmmUnaryRmR(sseOpcodeMovsd, xx, tmpDst)) + m.insert(m.allocateInstr().asXmmRmR(sseOpcodeMovlhps, xx, tmpDst)) + default: + panic(fmt.Sprintf("invalid lane type: %s", lane)) + } + + m.copyTo(tmpDst, m.c.VRegOf(ret)) +} + +func (m *machine) lowerShuffle(x, y ssa.Value, lo, hi uint64, ret ssa.Value) { + var xMask, yMask [2]uint64 + for i := 0; i < 8; i++ { + loLane := byte(lo >> (i * 8)) + if loLane < 16 { + xMask[0] |= uint64(loLane) << (i * 8) + yMask[0] |= uint64(0x80) << (i * 8) + } else { + xMask[0] |= uint64(0x80) << (i * 8) + yMask[0] |= uint64(loLane-16) << (i * 8) + } + hiLane := byte(hi >> (i * 8)) + if hiLane < 16 { + xMask[1] |= uint64(hiLane) << (i * 8) + yMask[1] |= uint64(0x80) << (i * 8) + } else { + xMask[1] |= uint64(0x80) << (i * 8) + yMask[1] |= uint64(hiLane-16) << (i * 8) + } + } + + xmaskLabel := m.allocateLabel() + m.consts = append(m.consts, _const{lo: xMask[0], hi: xMask[1], label: xmaskLabel}) + ymaskLabel := m.allocateLabel() + m.consts = append(m.consts, _const{lo: yMask[0], hi: yMask[1], label: ymaskLabel}) + + xx, yy := m.getOperand_Reg(m.c.ValueDefinition(x)), m.getOperand_Reg(m.c.ValueDefinition(y)) + tmpX, tmpY := m.copyToTmp(xx.reg()), m.copyToTmp(yy.reg()) + + // Apply mask to X. + tmp := m.c.AllocateVReg(ssa.TypeV128) + loadMaskLo := m.allocateInstr().asXmmUnaryRmR(sseOpcodeMovdqu, newOperandMem(m.newAmodeRipRel(xmaskLabel.L)), tmp) + m.insert(loadMaskLo) + m.insert(m.allocateInstr().asXmmRmR(sseOpcodePshufb, newOperandReg(tmp), tmpX)) + + // Apply mask to Y. + loadMaskHi := m.allocateInstr().asXmmUnaryRmR(sseOpcodeMovdqu, newOperandMem(m.newAmodeRipRel(ymaskLabel.L)), tmp) + m.insert(loadMaskHi) + m.insert(m.allocateInstr().asXmmRmR(sseOpcodePshufb, newOperandReg(tmp), tmpY)) + + // Combine the results. + m.insert(m.allocateInstr().asXmmRmR(sseOpcodeOrps, newOperandReg(tmpX), tmpY)) + + m.copyTo(tmpY, m.c.VRegOf(ret)) +} + +func (m *machine) lowerVbBinOpUnaligned(op sseOpcode, x, y, ret ssa.Value) { + rn := m.getOperand_Reg(m.c.ValueDefinition(x)) + rm := m.getOperand_Reg(m.c.ValueDefinition(y)) + rd := m.c.VRegOf(ret) + + tmp := m.copyToTmp(rn.reg()) + + binOp := m.allocateInstr() + binOp.asXmmRmR(op, rm, tmp) + m.insert(binOp) + + m.copyTo(tmp, rd) +} + +func (m *machine) lowerVbBinOp(op sseOpcode, x, y, ret ssa.Value) { + rn := m.getOperand_Reg(m.c.ValueDefinition(x)) + rm := m.getOperand_Mem_Reg(m.c.ValueDefinition(y)) + rd := m.c.VRegOf(ret) + + tmp := m.copyToTmp(rn.reg()) + + binOp := m.allocateInstr() + binOp.asXmmRmR(op, rm, tmp) + m.insert(binOp) + + m.copyTo(tmp, rd) +} + +func (m *machine) lowerVFcmp(x, y ssa.Value, c ssa.FloatCmpCond, ret ssa.Value, lane ssa.VecLane) { + var cmpOp sseOpcode + switch lane { + case ssa.VecLaneF32x4: + cmpOp = sseOpcodeCmpps + case ssa.VecLaneF64x2: + cmpOp = sseOpcodeCmppd + default: + panic(fmt.Sprintf("invalid lane type: %s", lane)) + } + + xx, yy := m.c.ValueDefinition(x), m.c.ValueDefinition(y) + var cmpImm cmpPred + switch c { + case ssa.FloatCmpCondGreaterThan: + yy, xx = xx, yy + cmpImm = cmpPredLT_OS + case ssa.FloatCmpCondGreaterThanOrEqual: + yy, xx = xx, yy + cmpImm = cmpPredLE_OS + case ssa.FloatCmpCondEqual: + cmpImm = cmpPredEQ_OQ + case ssa.FloatCmpCondNotEqual: + cmpImm = cmpPredNEQ_UQ + case ssa.FloatCmpCondLessThan: + cmpImm = cmpPredLT_OS + case ssa.FloatCmpCondLessThanOrEqual: + cmpImm = cmpPredLE_OS + default: + panic(fmt.Sprintf("invalid float comparison condition: %s", c)) + } + + tmp := m.c.AllocateVReg(ssa.TypeV128) + xxx := m.getOperand_Mem_Reg(xx) + m.insert(m.allocateInstr().asXmmUnaryRmR(sseOpcodeMovdqu, xxx, tmp)) + + rm := m.getOperand_Mem_Reg(yy) + m.insert(m.allocateInstr().asXmmRmRImm(cmpOp, byte(cmpImm), rm, tmp)) + + m.copyTo(tmp, m.c.VRegOf(ret)) +} + +func (m *machine) lowerVIcmp(x, y ssa.Value, c ssa.IntegerCmpCond, ret ssa.Value, lane ssa.VecLane) { + var eq, gt, maxu, minu, mins sseOpcode + switch lane { + case ssa.VecLaneI8x16: + eq, gt, maxu, minu, mins = sseOpcodePcmpeqb, sseOpcodePcmpgtb, sseOpcodePmaxub, sseOpcodePminub, sseOpcodePminsb + case ssa.VecLaneI16x8: + eq, gt, maxu, minu, mins = sseOpcodePcmpeqw, sseOpcodePcmpgtw, sseOpcodePmaxuw, sseOpcodePminuw, sseOpcodePminsw + case ssa.VecLaneI32x4: + eq, gt, maxu, minu, mins = sseOpcodePcmpeqd, sseOpcodePcmpgtd, sseOpcodePmaxud, sseOpcodePminud, sseOpcodePminsd + case ssa.VecLaneI64x2: + eq, gt = sseOpcodePcmpeqq, sseOpcodePcmpgtq + default: + panic(fmt.Sprintf("invalid lane type: %s", lane)) + } + + tmp := m.c.AllocateVReg(ssa.TypeV128) + var op operand + switch c { + case ssa.IntegerCmpCondSignedLessThanOrEqual: + if lane == ssa.VecLaneI64x2 { + x := m.getOperand_Mem_Reg(m.c.ValueDefinition(x)) + // Copy x to tmp. + m.insert(m.allocateInstr().asXmmUnaryRmR(sseOpcodeMovdqu, x, tmp)) + op = m.getOperand_Mem_Reg(m.c.ValueDefinition(y)) + } else { + y := m.getOperand_Mem_Reg(m.c.ValueDefinition(y)) + // Copy y to tmp. + m.insert(m.allocateInstr().asXmmUnaryRmR(sseOpcodeMovdqu, y, tmp)) + op = m.getOperand_Mem_Reg(m.c.ValueDefinition(x)) + } + case ssa.IntegerCmpCondSignedGreaterThanOrEqual: + if lane == ssa.VecLaneI64x2 { + y := m.getOperand_Mem_Reg(m.c.ValueDefinition(y)) + // Copy y to tmp. + m.insert(m.allocateInstr().asXmmUnaryRmR(sseOpcodeMovdqu, y, tmp)) + op = m.getOperand_Mem_Reg(m.c.ValueDefinition(x)) + } else { + x := m.getOperand_Mem_Reg(m.c.ValueDefinition(x)) + // Copy x to tmp. + m.insert(m.allocateInstr().asXmmUnaryRmR(sseOpcodeMovdqu, x, tmp)) + op = m.getOperand_Mem_Reg(m.c.ValueDefinition(y)) + } + case ssa.IntegerCmpCondSignedLessThan, ssa.IntegerCmpCondUnsignedLessThan, ssa.IntegerCmpCondUnsignedLessThanOrEqual: + y := m.getOperand_Mem_Reg(m.c.ValueDefinition(y)) + // Copy y to tmp. + m.insert(m.allocateInstr().asXmmUnaryRmR(sseOpcodeMovdqu, y, tmp)) + op = m.getOperand_Mem_Reg(m.c.ValueDefinition(x)) + default: + x := m.getOperand_Mem_Reg(m.c.ValueDefinition(x)) + // Copy x to tmp. + m.insert(m.allocateInstr().asXmmUnaryRmR(sseOpcodeMovdqu, x, tmp)) + op = m.getOperand_Mem_Reg(m.c.ValueDefinition(y)) + } + + switch c { + case ssa.IntegerCmpCondEqual: + m.insert(m.allocateInstr().asXmmRmR(eq, op, tmp)) + case ssa.IntegerCmpCondNotEqual: + // First we compare for equality. + m.insert(m.allocateInstr().asXmmRmR(eq, op, tmp)) + // Then flip the bits. To do so, we set all bits on tmp2. + tmp2 := m.c.AllocateVReg(ssa.TypeV128) + m.insert(m.allocateInstr().asDefineUninitializedReg(tmp2)) + m.insert(m.allocateInstr().asXmmRmR(eq, newOperandReg(tmp2), tmp2)) + // And then xor with tmp. + m.insert(m.allocateInstr().asXmmRmR(sseOpcodePxor, newOperandReg(tmp2), tmp)) + case ssa.IntegerCmpCondSignedGreaterThan, ssa.IntegerCmpCondSignedLessThan: + m.insert(m.allocateInstr().asXmmRmR(gt, op, tmp)) + case ssa.IntegerCmpCondSignedGreaterThanOrEqual, ssa.IntegerCmpCondSignedLessThanOrEqual: + if lane == ssa.VecLaneI64x2 { + m.insert(m.allocateInstr().asXmmRmR(gt, op, tmp)) + // Then flip the bits. To do so, we set all bits on tmp2. + tmp2 := m.c.AllocateVReg(ssa.TypeV128) + m.insert(m.allocateInstr().asDefineUninitializedReg(tmp2)) + m.insert(m.allocateInstr().asXmmRmR(eq, newOperandReg(tmp2), tmp2)) + // And then xor with tmp. + m.insert(m.allocateInstr().asXmmRmR(sseOpcodePxor, newOperandReg(tmp2), tmp)) + } else { + // First take min of x and y. + m.insert(m.allocateInstr().asXmmRmR(mins, op, tmp)) + // Then compare for equality. + m.insert(m.allocateInstr().asXmmRmR(eq, op, tmp)) + } + case ssa.IntegerCmpCondUnsignedGreaterThan, ssa.IntegerCmpCondUnsignedLessThan: + // First maxu of x and y. + m.insert(m.allocateInstr().asXmmRmR(maxu, op, tmp)) + // Then compare for equality. + m.insert(m.allocateInstr().asXmmRmR(eq, op, tmp)) + // Then flip the bits. To do so, we set all bits on tmp2. + tmp2 := m.c.AllocateVReg(ssa.TypeV128) + m.insert(m.allocateInstr().asDefineUninitializedReg(tmp2)) + m.insert(m.allocateInstr().asXmmRmR(eq, newOperandReg(tmp2), tmp2)) + // And then xor with tmp. + m.insert(m.allocateInstr().asXmmRmR(sseOpcodePxor, newOperandReg(tmp2), tmp)) + case ssa.IntegerCmpCondUnsignedGreaterThanOrEqual, ssa.IntegerCmpCondUnsignedLessThanOrEqual: + m.insert(m.allocateInstr().asXmmRmR(minu, op, tmp)) + m.insert(m.allocateInstr().asXmmRmR(eq, op, tmp)) + default: + panic("BUG") + } + + m.copyTo(tmp, m.c.VRegOf(ret)) +} + +func (m *machine) lowerVbandnot(instr *ssa.Instruction, op sseOpcode) { + x, y := instr.Arg2() + xDef := m.c.ValueDefinition(x) + yDef := m.c.ValueDefinition(y) + rm, rn := m.getOperand_Reg(xDef), m.getOperand_Reg(yDef) + rd := m.c.VRegOf(instr.Return()) + + tmp := m.copyToTmp(rn.reg()) + + // pandn between rn, rm. + pand := m.allocateInstr() + pand.asXmmRmR(sseOpcodePandn, rm, tmp) + m.insert(pand) + + m.copyTo(tmp, rd) +} + +func (m *machine) lowerVbitselect(instr *ssa.Instruction) { + c, x, y := instr.SelectData() + xDef := m.c.ValueDefinition(x) + yDef := m.c.ValueDefinition(y) + rm, rn := m.getOperand_Reg(xDef), m.getOperand_Reg(yDef) + creg := m.getOperand_Reg(m.c.ValueDefinition(c)) + rd := m.c.VRegOf(instr.Return()) + + tmpC := m.copyToTmp(creg.reg()) + tmpX := m.copyToTmp(rm.reg()) + + // And between c, x (overwrites x). + pand := m.allocateInstr() + pand.asXmmRmR(sseOpcodePand, creg, tmpX) + m.insert(pand) + + // Andn between y, c (overwrites c). + pandn := m.allocateInstr() + pandn.asXmmRmR(sseOpcodePandn, rn, tmpC) + m.insert(pandn) + + por := m.allocateInstr() + por.asXmmRmR(sseOpcodePor, newOperandReg(tmpC), tmpX) + m.insert(por) + + m.copyTo(tmpX, rd) +} + +func (m *machine) lowerVFmin(instr *ssa.Instruction) { + x, y, lane := instr.Arg2WithLane() + rn := m.getOperand_Reg(m.c.ValueDefinition(x)) + rm := m.getOperand_Reg(m.c.ValueDefinition(y)) + rd := m.c.VRegOf(instr.Return()) + + var min, cmp, andn, or, srl /* shift right logical */ sseOpcode + var shiftNumToInverseNaN uint32 + if lane == ssa.VecLaneF32x4 { + min, cmp, andn, or, srl, shiftNumToInverseNaN = sseOpcodeMinps, sseOpcodeCmpps, sseOpcodeAndnps, sseOpcodeOrps, sseOpcodePsrld, 0xa + } else { + min, cmp, andn, or, srl, shiftNumToInverseNaN = sseOpcodeMinpd, sseOpcodeCmppd, sseOpcodeAndnpd, sseOpcodeOrpd, sseOpcodePsrlq, 0xd + } + + tmp1 := m.copyToTmp(rn.reg()) + tmp2 := m.copyToTmp(rm.reg()) + + // tmp1=min(rn, rm) + minIns1 := m.allocateInstr() + minIns1.asXmmRmR(min, rn, tmp2) + m.insert(minIns1) + + // tmp2=min(rm, rn) + minIns2 := m.allocateInstr() + minIns2.asXmmRmR(min, rm, tmp1) + m.insert(minIns2) + + // tmp3:=tmp1=min(rn, rm) + tmp3 := m.copyToTmp(tmp1) + + // tmp1 = -0 if (rn == -0 || rm == -0) && rn != NaN && rm !=NaN + // NaN if rn == NaN || rm == NaN + // min(rm, rm) otherwise + orIns := m.allocateInstr() + orIns.asXmmRmR(or, newOperandReg(tmp2), tmp1) + m.insert(orIns) + + // tmp3 is originally min(rn,rm). + // tmp3 = 0^ (set all bits) if rn == NaN || rm == NaN + // 0 otherwise + cmpIns := m.allocateInstr() + cmpIns.asXmmRmRImm(cmp, uint8(cmpPredUNORD_Q), newOperandReg(tmp2), tmp3) + m.insert(cmpIns) + + // tmp1 = -0 if (rn == -0 || rm == -0) && rn != NaN && rm !=NaN + // ^0 if rn == NaN || rm == NaN + // min(v1, v2) otherwise + orIns2 := m.allocateInstr() + orIns2.asXmmRmR(or, newOperandReg(tmp3), tmp1) + m.insert(orIns2) + + // tmp3 = set all bits on the mantissa bits + // 0 otherwise + shift := m.allocateInstr() + shift.asXmmRmiReg(srl, newOperandImm32(shiftNumToInverseNaN), tmp3) + m.insert(shift) + + // tmp3 = tmp1 and !tmp3 + // = -0 if (rn == -0 || rm == -0) && rn != NaN && rm !=NaN + // set all bits on exponential and sign bit (== NaN) if rn == NaN || rm == NaN + // min(rn, rm) otherwise + andnIns := m.allocateInstr() + andnIns.asXmmRmR(andn, newOperandReg(tmp1), tmp3) + m.insert(andnIns) + + m.copyTo(tmp3, rd) +} + +func (m *machine) lowerVFmax(instr *ssa.Instruction) { + x, y, lane := instr.Arg2WithLane() + rn := m.getOperand_Reg(m.c.ValueDefinition(x)) + rm := m.getOperand_Reg(m.c.ValueDefinition(y)) + rd := m.c.VRegOf(instr.Return()) + + var max, cmp, andn, or, xor, sub, srl /* shift right logical */ sseOpcode + var shiftNumToInverseNaN uint32 + if lane == ssa.VecLaneF32x4 { + max, cmp, andn, or, xor, sub, srl, shiftNumToInverseNaN = sseOpcodeMaxps, sseOpcodeCmpps, sseOpcodeAndnps, sseOpcodeOrps, sseOpcodeXorps, sseOpcodeSubps, sseOpcodePsrld, 0xa + } else { + max, cmp, andn, or, xor, sub, srl, shiftNumToInverseNaN = sseOpcodeMaxpd, sseOpcodeCmppd, sseOpcodeAndnpd, sseOpcodeOrpd, sseOpcodeXorpd, sseOpcodeSubpd, sseOpcodePsrlq, 0xd + } + + tmp0 := m.copyToTmp(rm.reg()) + tmp1 := m.copyToTmp(rn.reg()) + + // tmp0=max(rn, rm) + maxIns1 := m.allocateInstr() + maxIns1.asXmmRmR(max, rn, tmp0) + m.insert(maxIns1) + + // tmp1=max(rm, rn) + maxIns2 := m.allocateInstr() + maxIns2.asXmmRmR(max, rm, tmp1) + m.insert(maxIns2) + + // tmp2=max(rm, rn) + tmp2 := m.copyToTmp(tmp1) + + // tmp2 = -0 if (rn == -0 && rm == 0) || (rn == 0 && rm == -0) + // 0 if (rn == 0 && rm == 0) + // -0 if (rn == -0 && rm == -0) + // v1^v2 if rn == NaN || rm == NaN + // 0 otherwise + xorInstr := m.allocateInstr() + xorInstr.asXmmRmR(xor, newOperandReg(tmp0), tmp2) + m.insert(xorInstr) + // tmp1 = -0 if (rn == -0 && rm == 0) || (rn == 0 && rm == -0) + // 0 if (rn == 0 && rm == 0) + // -0 if (rn == -0 && rm == -0) + // NaN if rn == NaN || rm == NaN + // max(v1, v2) otherwise + orInstr := m.allocateInstr() + orInstr.asXmmRmR(or, newOperandReg(tmp2), tmp1) + m.insert(orInstr) + + tmp3 := m.copyToTmp(tmp1) + + // tmp3 = 0 if (rn == -0 && rm == 0) || (rn == 0 && rm == -0) || (rn == 0 && rm == 0) + // -0 if (rn == -0 && rm == -0) + // NaN if rn == NaN || rm == NaN + // max(v1, v2) otherwise + // + // Note: -0 - (-0) = 0 (!= -0) in floating point operation. + subIns := m.allocateInstr() + subIns.asXmmRmR(sub, newOperandReg(tmp2), tmp3) + m.insert(subIns) + + // tmp1 = 0^ if rn == NaN || rm == NaN + cmpIns := m.allocateInstr() + cmpIns.asXmmRmRImm(cmp, uint8(cmpPredUNORD_Q), newOperandReg(tmp1), tmp1) + m.insert(cmpIns) + + // tmp1 = set all bits on the mantissa bits + // 0 otherwise + shift := m.allocateInstr() + shift.asXmmRmiReg(srl, newOperandImm32(shiftNumToInverseNaN), tmp1) + m.insert(shift) + + andnIns := m.allocateInstr() + andnIns.asXmmRmR(andn, newOperandReg(tmp3), tmp1) + m.insert(andnIns) + + m.copyTo(tmp1, rd) +} + +func (m *machine) lowerVFabs(instr *ssa.Instruction) { + x, lane := instr.ArgWithLane() + rm := m.getOperand_Mem_Reg(m.c.ValueDefinition(x)) + rd := m.c.VRegOf(instr.Return()) + + tmp := m.c.AllocateVReg(ssa.TypeV128) + + def := m.allocateInstr() + def.asDefineUninitializedReg(tmp) + m.insert(def) + + // Set all bits on tmp. + pcmp := m.allocateInstr() + pcmp.asXmmRmR(sseOpcodePcmpeqd, newOperandReg(tmp), tmp) + m.insert(pcmp) + + switch lane { + case ssa.VecLaneF32x4: + // Shift right packed single floats by 1 to clear the sign bits. + shift := m.allocateInstr() + shift.asXmmRmiReg(sseOpcodePsrld, newOperandImm32(1), tmp) + m.insert(shift) + // Clear the sign bit of rm. + andp := m.allocateInstr() + andp.asXmmRmR(sseOpcodeAndpd, rm, tmp) + m.insert(andp) + case ssa.VecLaneF64x2: + // Shift right packed single floats by 1 to clear the sign bits. + shift := m.allocateInstr() + shift.asXmmRmiReg(sseOpcodePsrlq, newOperandImm32(1), tmp) + m.insert(shift) + // Clear the sign bit of rm. + andp := m.allocateInstr() + andp.asXmmRmR(sseOpcodeAndps, rm, tmp) + m.insert(andp) + } + + m.copyTo(tmp, rd) +} diff --git a/vendor/github.com/tetratelabs/wazero/internal/engine/wazevo/backend/isa/amd64/machine_pro_epi_logue.go b/vendor/github.com/tetratelabs/wazero/internal/engine/wazevo/backend/isa/amd64/machine_pro_epi_logue.go new file mode 100644 index 000000000..8fa974c66 --- /dev/null +++ b/vendor/github.com/tetratelabs/wazero/internal/engine/wazevo/backend/isa/amd64/machine_pro_epi_logue.go @@ -0,0 +1,304 @@ +package amd64 + +import ( + "github.com/tetratelabs/wazero/internal/engine/wazevo/backend" + "github.com/tetratelabs/wazero/internal/engine/wazevo/backend/regalloc" +) + +// PostRegAlloc implements backend.Machine. +func (m *machine) PostRegAlloc() { + m.setupPrologue() + m.postRegAlloc() +} + +func (m *machine) setupPrologue() { + cur := m.ectx.RootInstr + prevInitInst := cur.next + + // At this point, we have the stack layout as follows: + // + // (high address) + // +-----------------+ <----- RBP (somewhere in the middle of the stack) + // | ....... | + // | ret Y | + // | ....... | + // | ret 0 | + // | arg X | + // | ....... | + // | arg 1 | + // | arg 0 | + // | Return Addr | + // RSP ----> +-----------------+ + // (low address) + + // First, we push the RBP, and update the RBP to the current RSP. + // + // (high address) (high address) + // RBP ----> +-----------------+ +-----------------+ + // | ....... | | ....... | + // | ret Y | | ret Y | + // | ....... | | ....... | + // | ret 0 | | ret 0 | + // | arg X | | arg X | + // | ....... | ====> | ....... | + // | arg 1 | | arg 1 | + // | arg 0 | | arg 0 | + // | Return Addr | | Return Addr | + // RSP ----> +-----------------+ | Caller_RBP | + // (low address) +-----------------+ <----- RSP, RBP + // + cur = m.setupRBPRSP(cur) + + if !m.stackBoundsCheckDisabled { + cur = m.insertStackBoundsCheck(m.requiredStackSize(), cur) + } + + // + // (high address) + // +-----------------+ +-----------------+ + // | ....... | | ....... | + // | ret Y | | ret Y | + // | ....... | | ....... | + // | ret 0 | | ret 0 | + // | arg X | | arg X | + // | ....... | | ....... | + // | arg 1 | | arg 1 | + // | arg 0 | | arg 0 | + // | xxxxx | | xxxxx | + // | Return Addr | | Return Addr | + // | Caller_RBP | ====> | Caller_RBP | + // RBP,RSP->+-----------------+ +-----------------+ <----- RBP + // (low address) | clobbered M | + // | clobbered 1 | + // | ........... | + // | clobbered 0 | + // +-----------------+ <----- RSP + // + if regs := m.clobberedRegs; len(regs) > 0 { + for i := range regs { + r := regs[len(regs)-1-i] // Reverse order. + if r.RegType() == regalloc.RegTypeInt { + cur = linkInstr(cur, m.allocateInstr().asPush64(newOperandReg(r))) + } else { + // Push the XMM register is not supported by the PUSH instruction. + cur = m.addRSP(-16, cur) + push := m.allocateInstr().asXmmMovRM( + sseOpcodeMovdqu, r, newOperandMem(m.newAmodeImmReg(0, rspVReg)), + ) + cur = linkInstr(cur, push) + } + } + } + + if size := m.spillSlotSize; size > 0 { + // Simply decrease the RSP to allocate the spill slots. + // sub $size, %rsp + cur = linkInstr(cur, m.allocateInstr().asAluRmiR(aluRmiROpcodeSub, newOperandImm32(uint32(size)), rspVReg, true)) + + // At this point, we have the stack layout as follows: + // + // (high address) + // +-----------------+ + // | ....... | + // | ret Y | + // | ....... | + // | ret 0 | + // | arg X | + // | ....... | + // | arg 1 | + // | arg 0 | + // | ReturnAddress | + // | Caller_RBP | + // +-----------------+ <--- RBP + // | clobbered M | + // | ............ | + // | clobbered 1 | + // | clobbered 0 | + // | spill slot N | + // | ............ | + // | spill slot 0 | + // +-----------------+ <--- RSP + // (low address) + } + + linkInstr(cur, prevInitInst) +} + +// postRegAlloc does multiple things while walking through the instructions: +// 1. Inserts the epilogue code. +// 2. Removes the redundant copy instruction. +// 3. Inserts the dec/inc RSP instruction right before/after the call instruction. +// 4. Lowering that is supposed to be done after regalloc. +func (m *machine) postRegAlloc() { + ectx := m.ectx + for cur := ectx.RootInstr; cur != nil; cur = cur.next { + switch k := cur.kind; k { + case ret: + m.setupEpilogueAfter(cur.prev) + continue + case fcvtToSintSequence, fcvtToUintSequence: + m.ectx.PendingInstructions = m.ectx.PendingInstructions[:0] + if k == fcvtToSintSequence { + m.lowerFcvtToSintSequenceAfterRegalloc(cur) + } else { + m.lowerFcvtToUintSequenceAfterRegalloc(cur) + } + prev := cur.prev + next := cur.next + cur := prev + for _, instr := range m.ectx.PendingInstructions { + cur = linkInstr(cur, instr) + } + linkInstr(cur, next) + continue + case xmmCMov: + m.ectx.PendingInstructions = m.ectx.PendingInstructions[:0] + m.lowerXmmCmovAfterRegAlloc(cur) + prev := cur.prev + next := cur.next + cur := prev + for _, instr := range m.ectx.PendingInstructions { + cur = linkInstr(cur, instr) + } + linkInstr(cur, next) + continue + case idivRemSequence: + m.ectx.PendingInstructions = m.ectx.PendingInstructions[:0] + m.lowerIDivRemSequenceAfterRegAlloc(cur) + prev := cur.prev + next := cur.next + cur := prev + for _, instr := range m.ectx.PendingInstructions { + cur = linkInstr(cur, instr) + } + linkInstr(cur, next) + continue + case call, callIndirect: + // At this point, reg alloc is done, therefore we can safely insert dec/inc RPS instruction + // right before/after the call instruction. If this is done before reg alloc, the stack slot + // can point to the wrong location and therefore results in a wrong value. + call := cur + next := call.next + _, _, _, _, size := backend.ABIInfoFromUint64(call.u2) + if size > 0 { + dec := m.allocateInstr().asAluRmiR(aluRmiROpcodeSub, newOperandImm32(size), rspVReg, true) + linkInstr(call.prev, dec) + linkInstr(dec, call) + inc := m.allocateInstr().asAluRmiR(aluRmiROpcodeAdd, newOperandImm32(size), rspVReg, true) + linkInstr(call, inc) + linkInstr(inc, next) + } + continue + } + + // Removes the redundant copy instruction. + if cur.IsCopy() && cur.op1.reg().RealReg() == cur.op2.reg().RealReg() { + prev, next := cur.prev, cur.next + // Remove the copy instruction. + prev.next = next + if next != nil { + next.prev = prev + } + } + } +} + +func (m *machine) setupEpilogueAfter(cur *instruction) { + prevNext := cur.next + + // At this point, we have the stack layout as follows: + // + // (high address) + // +-----------------+ + // | ....... | + // | ret Y | + // | ....... | + // | ret 0 | + // | arg X | + // | ....... | + // | arg 1 | + // | arg 0 | + // | ReturnAddress | + // | Caller_RBP | + // +-----------------+ <--- RBP + // | clobbered M | + // | ............ | + // | clobbered 1 | + // | clobbered 0 | + // | spill slot N | + // | ............ | + // | spill slot 0 | + // +-----------------+ <--- RSP + // (low address) + + if size := m.spillSlotSize; size > 0 { + // Simply increase the RSP to free the spill slots. + // add $size, %rsp + cur = linkInstr(cur, m.allocateInstr().asAluRmiR(aluRmiROpcodeAdd, newOperandImm32(uint32(size)), rspVReg, true)) + } + + // + // (high address) + // +-----------------+ +-----------------+ + // | ....... | | ....... | + // | ret Y | | ret Y | + // | ....... | | ....... | + // | ret 0 | | ret 0 | + // | arg X | | arg X | + // | ....... | | ....... | + // | arg 1 | | arg 1 | + // | arg 0 | | arg 0 | + // | ReturnAddress | | ReturnAddress | + // | Caller_RBP | | Caller_RBP | + // RBP ---> +-----------------+ ========> +-----------------+ <---- RSP, RBP + // | clobbered M | + // | ............ | + // | clobbered 1 | + // | clobbered 0 | + // RSP ---> +-----------------+ + // (low address) + // + if regs := m.clobberedRegs; len(regs) > 0 { + for _, r := range regs { + if r.RegType() == regalloc.RegTypeInt { + cur = linkInstr(cur, m.allocateInstr().asPop64(r)) + } else { + // Pop the XMM register is not supported by the POP instruction. + pop := m.allocateInstr().asXmmUnaryRmR( + sseOpcodeMovdqu, newOperandMem(m.newAmodeImmReg(0, rspVReg)), r, + ) + cur = linkInstr(cur, pop) + cur = m.addRSP(16, cur) + } + } + } + + // Now roll back the RSP to RBP, and pop the caller's RBP. + cur = m.revertRBPRSP(cur) + + linkInstr(cur, prevNext) +} + +func (m *machine) addRSP(offset int32, cur *instruction) *instruction { + if offset == 0 { + return cur + } + opcode := aluRmiROpcodeAdd + if offset < 0 { + opcode = aluRmiROpcodeSub + offset = -offset + } + return linkInstr(cur, m.allocateInstr().asAluRmiR(opcode, newOperandImm32(uint32(offset)), rspVReg, true)) +} + +func (m *machine) setupRBPRSP(cur *instruction) *instruction { + cur = linkInstr(cur, m.allocateInstr().asPush64(newOperandReg(rbpVReg))) + cur = linkInstr(cur, m.allocateInstr().asMovRR(rspVReg, rbpVReg, true)) + return cur +} + +func (m *machine) revertRBPRSP(cur *instruction) *instruction { + cur = linkInstr(cur, m.allocateInstr().asMovRR(rbpVReg, rspVReg, true)) + cur = linkInstr(cur, m.allocateInstr().asPop64(rbpVReg)) + return cur +} diff --git a/vendor/github.com/tetratelabs/wazero/internal/engine/wazevo/backend/isa/amd64/machine_regalloc.go b/vendor/github.com/tetratelabs/wazero/internal/engine/wazevo/backend/isa/amd64/machine_regalloc.go new file mode 100644 index 000000000..0bb28ee9e --- /dev/null +++ b/vendor/github.com/tetratelabs/wazero/internal/engine/wazevo/backend/isa/amd64/machine_regalloc.go @@ -0,0 +1,153 @@ +package amd64 + +import ( + "github.com/tetratelabs/wazero/internal/engine/wazevo/backend" + "github.com/tetratelabs/wazero/internal/engine/wazevo/backend/regalloc" + "github.com/tetratelabs/wazero/internal/engine/wazevo/ssa" +) + +// InsertMoveBefore implements backend.RegAllocFunctionMachine. +func (m *machine) InsertMoveBefore(dst, src regalloc.VReg, instr *instruction) { + typ := src.RegType() + if typ != dst.RegType() { + panic("BUG: src and dst must have the same type") + } + + mov := m.allocateInstr() + if typ == regalloc.RegTypeInt { + mov.asMovRR(src, dst, true) + } else { + mov.asXmmUnaryRmR(sseOpcodeMovdqu, newOperandReg(src), dst) + } + + cur := instr.prev + prevNext := cur.next + cur = linkInstr(cur, mov) + linkInstr(cur, prevNext) +} + +// InsertStoreRegisterAt implements backend.RegAllocFunctionMachine. +func (m *machine) InsertStoreRegisterAt(v regalloc.VReg, instr *instruction, after bool) *instruction { + if !v.IsRealReg() { + panic("BUG: VReg must be backed by real reg to be stored") + } + + typ := m.c.TypeOf(v) + + var prevNext, cur *instruction + if after { + cur, prevNext = instr, instr.next + } else { + cur, prevNext = instr.prev, instr + } + + offsetFromSP := m.getVRegSpillSlotOffsetFromSP(v.ID(), typ.Size()) + store := m.allocateInstr() + mem := newOperandMem(m.newAmodeImmReg(uint32(offsetFromSP), rspVReg)) + switch typ { + case ssa.TypeI32: + store.asMovRM(v, mem, 4) + case ssa.TypeI64: + store.asMovRM(v, mem, 8) + case ssa.TypeF32: + store.asXmmMovRM(sseOpcodeMovss, v, mem) + case ssa.TypeF64: + store.asXmmMovRM(sseOpcodeMovsd, v, mem) + case ssa.TypeV128: + store.asXmmMovRM(sseOpcodeMovdqu, v, mem) + } + + cur = linkInstr(cur, store) + return linkInstr(cur, prevNext) +} + +// InsertReloadRegisterAt implements backend.RegAllocFunctionMachine. +func (m *machine) InsertReloadRegisterAt(v regalloc.VReg, instr *instruction, after bool) *instruction { + if !v.IsRealReg() { + panic("BUG: VReg must be backed by real reg to be stored") + } + + typ := m.c.TypeOf(v) + var prevNext, cur *instruction + if after { + cur, prevNext = instr, instr.next + } else { + cur, prevNext = instr.prev, instr + } + + // Load the value to the temporary. + load := m.allocateInstr() + offsetFromSP := m.getVRegSpillSlotOffsetFromSP(v.ID(), typ.Size()) + a := newOperandMem(m.newAmodeImmReg(uint32(offsetFromSP), rspVReg)) + switch typ { + case ssa.TypeI32: + load.asMovzxRmR(extModeLQ, a, v) + case ssa.TypeI64: + load.asMov64MR(a, v) + case ssa.TypeF32: + load.asXmmUnaryRmR(sseOpcodeMovss, a, v) + case ssa.TypeF64: + load.asXmmUnaryRmR(sseOpcodeMovsd, a, v) + case ssa.TypeV128: + load.asXmmUnaryRmR(sseOpcodeMovdqu, a, v) + default: + panic("BUG") + } + + cur = linkInstr(cur, load) + return linkInstr(cur, prevNext) +} + +// ClobberedRegisters implements backend.RegAllocFunctionMachine. +func (m *machine) ClobberedRegisters(regs []regalloc.VReg) { + m.clobberedRegs = append(m.clobberedRegs[:0], regs...) +} + +// Swap implements backend.RegAllocFunctionMachine. +func (m *machine) Swap(cur *instruction, x1, x2, tmp regalloc.VReg) { + if x1.RegType() == regalloc.RegTypeInt { + prevNext := cur.next + xc := m.allocateInstr().asXCHG(x1, newOperandReg(x2), 8) + cur = linkInstr(cur, xc) + linkInstr(cur, prevNext) + } else { + if tmp.Valid() { + prevNext := cur.next + m.InsertMoveBefore(tmp, x1, prevNext) + m.InsertMoveBefore(x1, x2, prevNext) + m.InsertMoveBefore(x2, tmp, prevNext) + } else { + prevNext := cur.next + r2 := x2.RealReg() + // Temporarily spill x1 to stack. + cur = m.InsertStoreRegisterAt(x1, cur, true).prev + // Then move x2 to x1. + cur = linkInstr(cur, m.allocateInstr().asXmmUnaryRmR(sseOpcodeMovdqa, newOperandReg(x2), x1)) + linkInstr(cur, prevNext) + // Then reload the original value on x1 from stack to r2. + m.InsertReloadRegisterAt(x1.SetRealReg(r2), cur, true) + } + } +} + +// LastInstrForInsertion implements backend.RegAllocFunctionMachine. +func (m *machine) LastInstrForInsertion(begin, end *instruction) *instruction { + cur := end + for cur.kind == nop0 { + cur = cur.prev + if cur == begin { + return end + } + } + switch cur.kind { + case jmp: + return cur + default: + return end + } +} + +// SSABlockLabel implements backend.RegAllocFunctionMachine. +func (m *machine) SSABlockLabel(id ssa.BasicBlockID) backend.Label { + return m.ectx.SsaBlockIDToLabels[id] +} diff --git a/vendor/github.com/tetratelabs/wazero/internal/engine/wazevo/backend/isa/amd64/machine_vec.go b/vendor/github.com/tetratelabs/wazero/internal/engine/wazevo/backend/isa/amd64/machine_vec.go new file mode 100644 index 000000000..539a8b754 --- /dev/null +++ b/vendor/github.com/tetratelabs/wazero/internal/engine/wazevo/backend/isa/amd64/machine_vec.go @@ -0,0 +1,992 @@ +package amd64 + +import ( + "fmt" + + "github.com/tetratelabs/wazero/internal/engine/wazevo/backend/regalloc" + "github.com/tetratelabs/wazero/internal/engine/wazevo/ssa" +) + +var swizzleMask = [16]byte{ + 0x70, 0x70, 0x70, 0x70, 0x70, 0x70, 0x70, 0x70, + 0x70, 0x70, 0x70, 0x70, 0x70, 0x70, 0x70, 0x70, +} + +func (m *machine) lowerSwizzle(x, y ssa.Value, ret ssa.Value) { + masklabel := m.getOrAllocateConstLabel(&m.constSwizzleMaskConstIndex, swizzleMask[:]) + + // Load mask to maskReg. + maskReg := m.c.AllocateVReg(ssa.TypeV128) + loadMask := m.allocateInstr().asXmmUnaryRmR(sseOpcodeMovdqu, newOperandMem(m.newAmodeRipRel(masklabel)), maskReg) + m.insert(loadMask) + + // Copy x and y to tmp registers. + xx := m.getOperand_Reg(m.c.ValueDefinition(x)) + tmpDst := m.copyToTmp(xx.reg()) + yy := m.getOperand_Reg(m.c.ValueDefinition(y)) + tmpX := m.copyToTmp(yy.reg()) + + m.insert(m.allocateInstr().asXmmRmR(sseOpcodePaddusb, newOperandReg(maskReg), tmpX)) + m.insert(m.allocateInstr().asXmmRmR(sseOpcodePshufb, newOperandReg(tmpX), tmpDst)) + + // Copy the result to the destination register. + m.copyTo(tmpDst, m.c.VRegOf(ret)) +} + +func (m *machine) lowerInsertLane(x, y ssa.Value, index byte, ret ssa.Value, lane ssa.VecLane) { + // Copy x to tmp. + tmpDst := m.c.AllocateVReg(ssa.TypeV128) + m.insert(m.allocateInstr().asXmmUnaryRmR(sseOpcodeMovdqu, m.getOperand_Mem_Reg(m.c.ValueDefinition(x)), tmpDst)) + + yy := m.getOperand_Reg(m.c.ValueDefinition(y)) + switch lane { + case ssa.VecLaneI8x16: + m.insert(m.allocateInstr().asXmmRmRImm(sseOpcodePinsrb, index, yy, tmpDst)) + case ssa.VecLaneI16x8: + m.insert(m.allocateInstr().asXmmRmRImm(sseOpcodePinsrw, index, yy, tmpDst)) + case ssa.VecLaneI32x4: + m.insert(m.allocateInstr().asXmmRmRImm(sseOpcodePinsrd, index, yy, tmpDst)) + case ssa.VecLaneI64x2: + m.insert(m.allocateInstr().asXmmRmRImm(sseOpcodePinsrq, index, yy, tmpDst)) + case ssa.VecLaneF32x4: + // In INSERTPS instruction, the destination index is encoded at 4 and 5 bits of the argument. + // See https://www.felixcloutier.com/x86/insertps + m.insert(m.allocateInstr().asXmmRmRImm(sseOpcodeInsertps, index<<4, yy, tmpDst)) + case ssa.VecLaneF64x2: + if index == 0 { + m.insert(m.allocateInstr().asXmmUnaryRmR(sseOpcodeMovsd, yy, tmpDst)) + } else { + m.insert(m.allocateInstr().asXmmRmR(sseOpcodeMovlhps, yy, tmpDst)) + } + default: + panic(fmt.Sprintf("invalid lane type: %s", lane)) + } + + m.copyTo(tmpDst, m.c.VRegOf(ret)) +} + +func (m *machine) lowerExtractLane(x ssa.Value, index byte, signed bool, ret ssa.Value, lane ssa.VecLane) { + // Pextr variants are used to extract a lane from a vector register. + xx := m.getOperand_Reg(m.c.ValueDefinition(x)) + + tmpDst := m.c.AllocateVReg(ret.Type()) + m.insert(m.allocateInstr().asDefineUninitializedReg(tmpDst)) + switch lane { + case ssa.VecLaneI8x16: + m.insert(m.allocateInstr().asXmmRmRImm(sseOpcodePextrb, index, xx, tmpDst)) + if signed { + m.insert(m.allocateInstr().asMovsxRmR(extModeBL, newOperandReg(tmpDst), tmpDst)) + } else { + m.insert(m.allocateInstr().asMovzxRmR(extModeBL, newOperandReg(tmpDst), tmpDst)) + } + case ssa.VecLaneI16x8: + m.insert(m.allocateInstr().asXmmRmRImm(sseOpcodePextrw, index, xx, tmpDst)) + if signed { + m.insert(m.allocateInstr().asMovsxRmR(extModeWL, newOperandReg(tmpDst), tmpDst)) + } else { + m.insert(m.allocateInstr().asMovzxRmR(extModeWL, newOperandReg(tmpDst), tmpDst)) + } + case ssa.VecLaneI32x4: + m.insert(m.allocateInstr().asXmmRmRImm(sseOpcodePextrd, index, xx, tmpDst)) + case ssa.VecLaneI64x2: + m.insert(m.allocateInstr().asXmmRmRImm(sseOpcodePextrq, index, xx, tmpDst)) + case ssa.VecLaneF32x4: + if index == 0 { + m.insert(m.allocateInstr().asXmmUnaryRmR(sseOpcodeMovss, xx, tmpDst)) + } else { + m.insert(m.allocateInstr().asXmmRmRImm(sseOpcodePshufd, index, xx, tmpDst)) + } + case ssa.VecLaneF64x2: + if index == 0 { + m.insert(m.allocateInstr().asXmmUnaryRmR(sseOpcodeMovsd, xx, tmpDst)) + } else { + m.copyTo(xx.reg(), tmpDst) + m.insert(m.allocateInstr().asXmmRmRImm(sseOpcodePshufd, 0b00_00_11_10, newOperandReg(tmpDst), tmpDst)) + } + default: + panic(fmt.Sprintf("invalid lane type: %s", lane)) + } + + m.copyTo(tmpDst, m.c.VRegOf(ret)) +} + +var sqmulRoundSat = [16]byte{ + 0x00, 0x80, 0x00, 0x80, 0x00, 0x80, 0x00, 0x80, + 0x00, 0x80, 0x00, 0x80, 0x00, 0x80, 0x00, 0x80, +} + +func (m *machine) lowerSqmulRoundSat(x, y, ret ssa.Value) { + // See https://github.com/WebAssembly/simd/pull/365 for the following logic. + maskLabel := m.getOrAllocateConstLabel(&m.constSqmulRoundSatIndex, sqmulRoundSat[:]) + + tmp := m.c.AllocateVReg(ssa.TypeV128) + loadMask := m.allocateInstr().asXmmUnaryRmR(sseOpcodeMovdqu, newOperandMem(m.newAmodeRipRel(maskLabel)), tmp) + m.insert(loadMask) + + xx, yy := m.getOperand_Reg(m.c.ValueDefinition(x)), m.getOperand_Mem_Reg(m.c.ValueDefinition(y)) + tmpX := m.copyToTmp(xx.reg()) + + m.insert(m.allocateInstr().asXmmRmR(sseOpcodePmulhrsw, yy, tmpX)) + m.insert(m.allocateInstr().asXmmRmR(sseOpcodePcmpeqd, newOperandReg(tmpX), tmp)) + m.insert(m.allocateInstr().asXmmRmR(sseOpcodePxor, newOperandReg(tmp), tmpX)) + + m.copyTo(tmpX, m.c.VRegOf(ret)) +} + +func (m *machine) lowerVUshr(x, y, ret ssa.Value, lane ssa.VecLane) { + switch lane { + case ssa.VecLaneI8x16: + m.lowerVUshri8x16(x, y, ret) + case ssa.VecLaneI16x8, ssa.VecLaneI32x4, ssa.VecLaneI64x2: + m.lowerShr(x, y, ret, lane, false) + default: + panic(fmt.Sprintf("invalid lane type: %s", lane)) + } +} + +// i8x16LogicalSHRMaskTable is necessary for emulating non-existent packed bytes logical right shifts on amd64. +// The mask is applied after performing packed word shifts on the value to clear out the unnecessary bits. +var i8x16LogicalSHRMaskTable = [8 * 16]byte{ // (the number of possible shift amount 0, 1, ..., 7.) * 16 bytes. + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, // for 0 shift + 0x7f, 0x7f, 0x7f, 0x7f, 0x7f, 0x7f, 0x7f, 0x7f, 0x7f, 0x7f, 0x7f, 0x7f, 0x7f, 0x7f, 0x7f, 0x7f, // for 1 shift + 0x3f, 0x3f, 0x3f, 0x3f, 0x3f, 0x3f, 0x3f, 0x3f, 0x3f, 0x3f, 0x3f, 0x3f, 0x3f, 0x3f, 0x3f, 0x3f, // for 2 shift + 0x1f, 0x1f, 0x1f, 0x1f, 0x1f, 0x1f, 0x1f, 0x1f, 0x1f, 0x1f, 0x1f, 0x1f, 0x1f, 0x1f, 0x1f, 0x1f, // for 3 shift + 0x0f, 0x0f, 0x0f, 0x0f, 0x0f, 0x0f, 0x0f, 0x0f, 0x0f, 0x0f, 0x0f, 0x0f, 0x0f, 0x0f, 0x0f, 0x0f, // for 4 shift + 0x07, 0x07, 0x07, 0x07, 0x07, 0x07, 0x07, 0x07, 0x07, 0x07, 0x07, 0x07, 0x07, 0x07, 0x07, 0x07, // for 5 shift + 0x03, 0x03, 0x03, 0x03, 0x03, 0x03, 0x03, 0x03, 0x03, 0x03, 0x03, 0x03, 0x03, 0x03, 0x03, 0x03, // for 6 shift + 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, // for 7 shift +} + +func (m *machine) lowerVUshri8x16(x, y, ret ssa.Value) { + tmpGpReg := m.c.AllocateVReg(ssa.TypeI32) + // Load the modulo 8 mask to tmpReg. + m.lowerIconst(tmpGpReg, 0x7, false) + // Take the modulo 8 of the shift amount. + shiftAmt := m.getOperand_Mem_Imm32_Reg(m.c.ValueDefinition(y)) + m.insert(m.allocateInstr().asAluRmiR(aluRmiROpcodeAnd, shiftAmt, tmpGpReg, false)) + + _xx := m.getOperand_Reg(m.c.ValueDefinition(x)) + xx := m.copyToTmp(_xx.reg()) + + vecTmp := m.c.AllocateVReg(ssa.TypeV128) + m.insert(m.allocateInstr().asGprToXmm(sseOpcodeMovd, newOperandReg(tmpGpReg), vecTmp, false)) + m.insert(m.allocateInstr().asXmmRmiReg(sseOpcodePsrlw, newOperandReg(vecTmp), xx)) + + maskTableLabel := m.getOrAllocateConstLabel(&m.constI8x16LogicalSHRMaskTableIndex, i8x16LogicalSHRMaskTable[:]) + base := m.c.AllocateVReg(ssa.TypeI64) + lea := m.allocateInstr().asLEA(newOperandLabel(maskTableLabel), base) + m.insert(lea) + + // Shift tmpGpReg by 4 to multiply the shift amount by 16. + m.insert(m.allocateInstr().asShiftR(shiftROpShiftLeft, newOperandImm32(4), tmpGpReg, false)) + + mem := m.newAmodeRegRegShift(0, base, tmpGpReg, 0) + loadMask := m.allocateInstr().asXmmUnaryRmR(sseOpcodeMovdqu, newOperandMem(mem), vecTmp) + m.insert(loadMask) + + m.insert(m.allocateInstr().asXmmRmR(sseOpcodePand, newOperandReg(vecTmp), xx)) + m.copyTo(xx, m.c.VRegOf(ret)) +} + +func (m *machine) lowerVSshr(x, y, ret ssa.Value, lane ssa.VecLane) { + switch lane { + case ssa.VecLaneI8x16: + m.lowerVSshri8x16(x, y, ret) + case ssa.VecLaneI16x8, ssa.VecLaneI32x4: + m.lowerShr(x, y, ret, lane, true) + case ssa.VecLaneI64x2: + m.lowerVSshri64x2(x, y, ret) + default: + panic(fmt.Sprintf("invalid lane type: %s", lane)) + } +} + +func (m *machine) lowerVSshri8x16(x, y, ret ssa.Value) { + shiftAmtReg := m.c.AllocateVReg(ssa.TypeI32) + // Load the modulo 8 mask to tmpReg. + m.lowerIconst(shiftAmtReg, 0x7, false) + // Take the modulo 8 of the shift amount. + shiftAmt := m.getOperand_Mem_Imm32_Reg(m.c.ValueDefinition(y)) + m.insert(m.allocateInstr().asAluRmiR(aluRmiROpcodeAnd, shiftAmt, shiftAmtReg, false)) + + // Copy the x value to two temporary registers. + _xx := m.getOperand_Reg(m.c.ValueDefinition(x)) + xx := m.copyToTmp(_xx.reg()) + vecTmp := m.c.AllocateVReg(ssa.TypeV128) + m.copyTo(xx, vecTmp) + + // Assuming that we have + // xx = [b1, ..., b16] + // vecTmp = [b1, ..., b16] + // at this point, then we use PUNPCKLBW and PUNPCKHBW to produce: + // xx = [b1, b1, b2, b2, ..., b8, b8] + // vecTmp = [b9, b9, b10, b10, ..., b16, b16] + m.insert(m.allocateInstr().asXmmRmR(sseOpcodePunpcklbw, newOperandReg(xx), xx)) + m.insert(m.allocateInstr().asXmmRmR(sseOpcodePunpckhbw, newOperandReg(vecTmp), vecTmp)) + + // Adding 8 to the shift amount, and then move the amount to vecTmp2. + vecTmp2 := m.c.AllocateVReg(ssa.TypeV128) + m.insert(m.allocateInstr().asAluRmiR(aluRmiROpcodeAdd, newOperandImm32(8), shiftAmtReg, false)) + m.insert(m.allocateInstr().asGprToXmm(sseOpcodeMovd, newOperandReg(shiftAmtReg), vecTmp2, false)) + + // Perform the word packed arithmetic right shifts on vreg and vecTmp. + // This changes these two registers as: + // xx = [xxx, b1 >> s, xxx, b2 >> s, ..., xxx, b8 >> s] + // vecTmp = [xxx, b9 >> s, xxx, b10 >> s, ..., xxx, b16 >> s] + // where xxx is 1 or 0 depending on each byte's sign, and ">>" is the arithmetic shift on a byte. + m.insert(m.allocateInstr().asXmmRmiReg(sseOpcodePsraw, newOperandReg(vecTmp2), xx)) + m.insert(m.allocateInstr().asXmmRmiReg(sseOpcodePsraw, newOperandReg(vecTmp2), vecTmp)) + + // Finally, we can get the result by packing these two word vectors. + m.insert(m.allocateInstr().asXmmRmR(sseOpcodePacksswb, newOperandReg(vecTmp), xx)) + + m.copyTo(xx, m.c.VRegOf(ret)) +} + +func (m *machine) lowerVSshri64x2(x, y, ret ssa.Value) { + // Load the shift amount to RCX. + shiftAmt := m.getOperand_Mem_Reg(m.c.ValueDefinition(y)) + m.insert(m.allocateInstr().asMovzxRmR(extModeBQ, shiftAmt, rcxVReg)) + + tmpGp := m.c.AllocateVReg(ssa.TypeI64) + + _xx := m.getOperand_Reg(m.c.ValueDefinition(x)) + xxReg := m.copyToTmp(_xx.reg()) + + m.insert(m.allocateInstr().asDefineUninitializedReg(tmpGp)) + m.insert(m.allocateInstr().asXmmRmRImm(sseOpcodePextrq, 0, newOperandReg(xxReg), tmpGp)) + m.insert(m.allocateInstr().asShiftR(shiftROpShiftRightArithmetic, newOperandReg(rcxVReg), tmpGp, true)) + m.insert(m.allocateInstr().asXmmRmRImm(sseOpcodePinsrq, 0, newOperandReg(tmpGp), xxReg)) + m.insert(m.allocateInstr().asXmmRmRImm(sseOpcodePextrq, 1, newOperandReg(xxReg), tmpGp)) + m.insert(m.allocateInstr().asShiftR(shiftROpShiftRightArithmetic, newOperandReg(rcxVReg), tmpGp, true)) + m.insert(m.allocateInstr().asXmmRmRImm(sseOpcodePinsrq, 1, newOperandReg(tmpGp), xxReg)) + + m.copyTo(xxReg, m.c.VRegOf(ret)) +} + +func (m *machine) lowerShr(x, y, ret ssa.Value, lane ssa.VecLane, signed bool) { + var modulo uint64 + var shiftOp sseOpcode + switch lane { + case ssa.VecLaneI16x8: + modulo = 0xf + if signed { + shiftOp = sseOpcodePsraw + } else { + shiftOp = sseOpcodePsrlw + } + case ssa.VecLaneI32x4: + modulo = 0x1f + if signed { + shiftOp = sseOpcodePsrad + } else { + shiftOp = sseOpcodePsrld + } + case ssa.VecLaneI64x2: + modulo = 0x3f + if signed { + panic("BUG") + } + shiftOp = sseOpcodePsrlq + default: + panic(fmt.Sprintf("invalid lane type: %s", lane)) + } + + _xx := m.getOperand_Reg(m.c.ValueDefinition(x)) + xx := m.copyToTmp(_xx.reg()) + + tmpGpReg := m.c.AllocateVReg(ssa.TypeI32) + // Load the modulo 8 mask to tmpReg. + m.lowerIconst(tmpGpReg, modulo, false) + // Take the modulo 8 of the shift amount. + m.insert(m.allocateInstr().asAluRmiR(aluRmiROpcodeAnd, + m.getOperand_Mem_Imm32_Reg(m.c.ValueDefinition(y)), tmpGpReg, false)) + // And move it to a xmm register. + tmpVec := m.c.AllocateVReg(ssa.TypeV128) + m.insert(m.allocateInstr().asGprToXmm(sseOpcodeMovd, newOperandReg(tmpGpReg), tmpVec, false)) + + // Then do the actual shift. + m.insert(m.allocateInstr().asXmmRmiReg(shiftOp, newOperandReg(tmpVec), xx)) + + m.copyTo(xx, m.c.VRegOf(ret)) +} + +func (m *machine) lowerVIshl(x, y, ret ssa.Value, lane ssa.VecLane) { + var modulo uint64 + var shiftOp sseOpcode + var isI8x16 bool + switch lane { + case ssa.VecLaneI8x16: + isI8x16 = true + modulo = 0x7 + shiftOp = sseOpcodePsllw + case ssa.VecLaneI16x8: + modulo = 0xf + shiftOp = sseOpcodePsllw + case ssa.VecLaneI32x4: + modulo = 0x1f + shiftOp = sseOpcodePslld + case ssa.VecLaneI64x2: + modulo = 0x3f + shiftOp = sseOpcodePsllq + default: + panic(fmt.Sprintf("invalid lane type: %s", lane)) + } + + _xx := m.getOperand_Reg(m.c.ValueDefinition(x)) + xx := m.copyToTmp(_xx.reg()) + + tmpGpReg := m.c.AllocateVReg(ssa.TypeI32) + // Load the modulo 8 mask to tmpReg. + m.lowerIconst(tmpGpReg, modulo, false) + // Take the modulo 8 of the shift amount. + m.insert(m.allocateInstr().asAluRmiR(aluRmiROpcodeAnd, + m.getOperand_Mem_Imm32_Reg(m.c.ValueDefinition(y)), tmpGpReg, false)) + // And move it to a xmm register. + tmpVec := m.c.AllocateVReg(ssa.TypeV128) + m.insert(m.allocateInstr().asGprToXmm(sseOpcodeMovd, newOperandReg(tmpGpReg), tmpVec, false)) + + // Then do the actual shift. + m.insert(m.allocateInstr().asXmmRmiReg(shiftOp, newOperandReg(tmpVec), xx)) + + if isI8x16 { + maskTableLabel := m.getOrAllocateConstLabel(&m.constI8x16SHLMaskTableIndex, i8x16SHLMaskTable[:]) + base := m.c.AllocateVReg(ssa.TypeI64) + lea := m.allocateInstr().asLEA(newOperandLabel(maskTableLabel), base) + m.insert(lea) + + // Shift tmpGpReg by 4 to multiply the shift amount by 16. + m.insert(m.allocateInstr().asShiftR(shiftROpShiftLeft, newOperandImm32(4), tmpGpReg, false)) + + mem := m.newAmodeRegRegShift(0, base, tmpGpReg, 0) + loadMask := m.allocateInstr().asXmmUnaryRmR(sseOpcodeMovdqu, newOperandMem(mem), tmpVec) + m.insert(loadMask) + + m.insert(m.allocateInstr().asXmmRmR(sseOpcodePand, newOperandReg(tmpVec), xx)) + } + + m.copyTo(xx, m.c.VRegOf(ret)) +} + +// i8x16SHLMaskTable is necessary for emulating non-existent packed bytes left shifts on amd64. +// The mask is applied after performing packed word shifts on the value to clear out the unnecessary bits. +var i8x16SHLMaskTable = [8 * 16]byte{ // (the number of possible shift amount 0, 1, ..., 7.) * 16 bytes. + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, // for 0 shift + 0xfe, 0xfe, 0xfe, 0xfe, 0xfe, 0xfe, 0xfe, 0xfe, 0xfe, 0xfe, 0xfe, 0xfe, 0xfe, 0xfe, 0xfe, 0xfe, // for 1 shift + 0xfc, 0xfc, 0xfc, 0xfc, 0xfc, 0xfc, 0xfc, 0xfc, 0xfc, 0xfc, 0xfc, 0xfc, 0xfc, 0xfc, 0xfc, 0xfc, // for 2 shift + 0xf8, 0xf8, 0xf8, 0xf8, 0xf8, 0xf8, 0xf8, 0xf8, 0xf8, 0xf8, 0xf8, 0xf8, 0xf8, 0xf8, 0xf8, 0xf8, // for 3 shift + 0xf0, 0xf0, 0xf0, 0xf0, 0xf0, 0xf0, 0xf0, 0xf0, 0xf0, 0xf0, 0xf0, 0xf0, 0xf0, 0xf0, 0xf0, 0xf0, // for 4 shift + 0xe0, 0xe0, 0xe0, 0xe0, 0xe0, 0xe0, 0xe0, 0xe0, 0xe0, 0xe0, 0xe0, 0xe0, 0xe0, 0xe0, 0xe0, 0xe0, // for 5 shift + 0xc0, 0xc0, 0xc0, 0xc0, 0xc0, 0xc0, 0xc0, 0xc0, 0xc0, 0xc0, 0xc0, 0xc0, 0xc0, 0xc0, 0xc0, 0xc0, // for 6 shift + 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, // for 7 shift +} + +func (m *machine) lowerVRound(x, ret ssa.Value, imm byte, _64 bool) { + xx := m.getOperand_Mem_Reg(m.c.ValueDefinition(x)) + var round sseOpcode + if _64 { + round = sseOpcodeRoundpd + } else { + round = sseOpcodeRoundps + } + m.insert(m.allocateInstr().asXmmUnaryRmRImm(round, imm, xx, m.c.VRegOf(ret))) +} + +var ( + allOnesI8x16 = [16]byte{0x1, 0x1, 0x1, 0x1, 0x1, 0x1, 0x1, 0x1, 0x1, 0x1, 0x1, 0x1, 0x1, 0x1, 0x1, 0x1} + allOnesI16x8 = [16]byte{0x1, 0x0, 0x1, 0x0, 0x1, 0x0, 0x1, 0x0, 0x1, 0x0, 0x1, 0x0, 0x1, 0x0, 0x1, 0x0} + extAddPairwiseI16x8uMask1 = [16]byte{0x00, 0x80, 0x00, 0x80, 0x00, 0x80, 0x00, 0x80, 0x00, 0x80, 0x00, 0x80, 0x00, 0x80, 0x00, 0x80} + extAddPairwiseI16x8uMask2 = [16]byte{0x00, 0x00, 0x01, 0x00, 0x00, 0x00, 0x01, 0x00, 0x00, 0x00, 0x01, 0x00, 0x00, 0x00, 0x01, 0x00} +) + +func (m *machine) lowerExtIaddPairwise(x, ret ssa.Value, srcLane ssa.VecLane, signed bool) { + _xx := m.getOperand_Reg(m.c.ValueDefinition(x)) + xx := m.copyToTmp(_xx.reg()) + switch srcLane { + case ssa.VecLaneI8x16: + allOneReg := m.c.AllocateVReg(ssa.TypeV128) + mask := m.getOrAllocateConstLabel(&m.constAllOnesI8x16Index, allOnesI8x16[:]) + m.insert(m.allocateInstr().asXmmUnaryRmR(sseOpcodeMovdqu, newOperandMem(m.newAmodeRipRel(mask)), allOneReg)) + + var resultReg regalloc.VReg + if signed { + resultReg = allOneReg + m.insert(m.allocateInstr().asXmmRmR(sseOpcodePmaddubsw, newOperandReg(xx), resultReg)) + } else { + // Interpreter tmp (all ones) as signed byte meaning that all the multiply-add is unsigned. + resultReg = xx + m.insert(m.allocateInstr().asXmmRmR(sseOpcodePmaddubsw, newOperandReg(allOneReg), resultReg)) + } + m.copyTo(resultReg, m.c.VRegOf(ret)) + + case ssa.VecLaneI16x8: + if signed { + allOnesReg := m.c.AllocateVReg(ssa.TypeV128) + mask := m.getOrAllocateConstLabel(&m.constAllOnesI16x8Index, allOnesI16x8[:]) + m.insert(m.allocateInstr().asXmmUnaryRmR(sseOpcodeMovdqu, newOperandMem(m.newAmodeRipRel(mask)), allOnesReg)) + m.insert(m.allocateInstr().asXmmRmR(sseOpcodePmaddwd, newOperandReg(allOnesReg), xx)) + m.copyTo(xx, m.c.VRegOf(ret)) + } else { + maskReg := m.c.AllocateVReg(ssa.TypeV128) + mask := m.getOrAllocateConstLabel(&m.constExtAddPairwiseI16x8uMask1Index, extAddPairwiseI16x8uMask1[:]) + m.insert(m.allocateInstr().asXmmUnaryRmR(sseOpcodeMovdqu, newOperandMem(m.newAmodeRipRel(mask)), maskReg)) + + // Flip the sign bits on xx. + // + // Assuming that xx = [w1, ..., w8], now we have, + // xx[i] = int8(-w1) for i = 0...8 + m.insert(m.allocateInstr().asXmmRmR(sseOpcodePxor, newOperandReg(maskReg), xx)) + + mask = m.getOrAllocateConstLabel(&m.constAllOnesI16x8Index, allOnesI16x8[:]) + m.insert(m.allocateInstr().asXmmUnaryRmR(sseOpcodeMovdqu, newOperandMem(m.newAmodeRipRel(mask)), maskReg)) + + // For i = 0,..4 (as this results in i32x4 lanes), now we have + // xx[i] = int32(-wn + -w(n+1)) = int32(-(wn + w(n+1))) + // c.assembler.CompileRegisterToRegister(amd64.PMADDWD, tmp, vr) + m.insert(m.allocateInstr().asXmmRmR(sseOpcodePmaddwd, newOperandReg(maskReg), xx)) + + mask = m.getOrAllocateConstLabel(&m.constExtAddPairwiseI16x8uMask2Index, extAddPairwiseI16x8uMask2[:]) + m.insert(m.allocateInstr().asXmmUnaryRmR(sseOpcodeMovdqu, newOperandMem(m.newAmodeRipRel(mask)), maskReg)) + + // vr[i] = int32(-(wn + w(n+1))) + int32(math.MaxInt16+1) = int32((wn + w(n+1))) = uint32(wn + w(n+1)). + // c.assembler.CompileRegisterToRegister(amd64.PADDD, tmp, vr) + m.insert(m.allocateInstr().asXmmRmR(sseOpcodePaddd, newOperandReg(maskReg), xx)) + + m.copyTo(xx, m.c.VRegOf(ret)) + } + default: + panic(fmt.Sprintf("invalid lane type: %s", srcLane)) + } +} + +func (m *machine) lowerWidenLow(x, ret ssa.Value, lane ssa.VecLane, signed bool) { + var sseOp sseOpcode + switch lane { + case ssa.VecLaneI8x16: + if signed { + sseOp = sseOpcodePmovsxbw + } else { + sseOp = sseOpcodePmovzxbw + } + case ssa.VecLaneI16x8: + if signed { + sseOp = sseOpcodePmovsxwd + } else { + sseOp = sseOpcodePmovzxwd + } + case ssa.VecLaneI32x4: + if signed { + sseOp = sseOpcodePmovsxdq + } else { + sseOp = sseOpcodePmovzxdq + } + default: + panic(fmt.Sprintf("invalid lane type: %s", lane)) + } + + xx := m.getOperand_Mem_Reg(m.c.ValueDefinition(x)) + m.insert(m.allocateInstr().asXmmUnaryRmR(sseOp, xx, m.c.VRegOf(ret))) +} + +func (m *machine) lowerWidenHigh(x, ret ssa.Value, lane ssa.VecLane, signed bool) { + tmp := m.c.AllocateVReg(ssa.TypeV128) + xx := m.getOperand_Reg(m.c.ValueDefinition(x)) + m.copyTo(xx.reg(), tmp) + m.insert(m.allocateInstr().asXmmRmRImm(sseOpcodePalignr, 8, newOperandReg(tmp), tmp)) + + var sseOp sseOpcode + switch lane { + case ssa.VecLaneI8x16: + if signed { + sseOp = sseOpcodePmovsxbw + } else { + sseOp = sseOpcodePmovzxbw + } + case ssa.VecLaneI16x8: + if signed { + sseOp = sseOpcodePmovsxwd + } else { + sseOp = sseOpcodePmovzxwd + } + case ssa.VecLaneI32x4: + if signed { + sseOp = sseOpcodePmovsxdq + } else { + sseOp = sseOpcodePmovzxdq + } + default: + panic(fmt.Sprintf("invalid lane type: %s", lane)) + } + + m.insert(m.allocateInstr().asXmmUnaryRmR(sseOp, newOperandReg(tmp), m.c.VRegOf(ret))) +} + +func (m *machine) lowerLoadSplat(ptr ssa.Value, offset uint32, ret ssa.Value, lane ssa.VecLane) { + tmpDst, tmpGp := m.c.AllocateVReg(ssa.TypeV128), m.c.AllocateVReg(ssa.TypeI64) + am := newOperandMem(m.lowerToAddressMode(ptr, offset)) + + m.insert(m.allocateInstr().asDefineUninitializedReg(tmpDst)) + switch lane { + case ssa.VecLaneI8x16: + m.insert(m.allocateInstr().asMovzxRmR(extModeBQ, am, tmpGp)) + m.insert(m.allocateInstr().asXmmRmRImm(sseOpcodePinsrb, 0, newOperandReg(tmpGp), tmpDst)) + tmpZeroVec := m.c.AllocateVReg(ssa.TypeV128) + m.insert(m.allocateInstr().asZeros(tmpZeroVec)) + m.insert(m.allocateInstr().asXmmRmR(sseOpcodePshufb, newOperandReg(tmpZeroVec), tmpDst)) + case ssa.VecLaneI16x8: + m.insert(m.allocateInstr().asMovzxRmR(extModeWQ, am, tmpGp)) + m.insert(m.allocateInstr().asXmmRmRImm(sseOpcodePinsrw, 0, newOperandReg(tmpGp), tmpDst)) + m.insert(m.allocateInstr().asXmmRmRImm(sseOpcodePinsrw, 1, newOperandReg(tmpGp), tmpDst)) + m.insert(m.allocateInstr().asXmmRmRImm(sseOpcodePshufd, 0, newOperandReg(tmpDst), tmpDst)) + case ssa.VecLaneI32x4: + m.insert(m.allocateInstr().asMovzxRmR(extModeLQ, am, tmpGp)) + m.insert(m.allocateInstr().asXmmRmRImm(sseOpcodePinsrd, 0, newOperandReg(tmpGp), tmpDst)) + m.insert(m.allocateInstr().asXmmRmRImm(sseOpcodePshufd, 0, newOperandReg(tmpDst), tmpDst)) + case ssa.VecLaneI64x2: + m.insert(m.allocateInstr().asMov64MR(am, tmpGp)) + m.insert(m.allocateInstr().asXmmRmRImm(sseOpcodePinsrq, 0, newOperandReg(tmpGp), tmpDst)) + m.insert(m.allocateInstr().asXmmRmRImm(sseOpcodePinsrq, 1, newOperandReg(tmpGp), tmpDst)) + default: + panic(fmt.Sprintf("invalid lane type: %s", lane)) + } + + m.copyTo(tmpDst, m.c.VRegOf(ret)) +} + +var f64x2CvtFromIMask = [16]byte{ + 0x00, 0x00, 0x30, 0x43, 0x00, 0x00, 0x30, 0x43, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, +} + +func (m *machine) lowerVFcvtFromInt(x, ret ssa.Value, lane ssa.VecLane, signed bool) { + switch lane { + case ssa.VecLaneF32x4: + if signed { + xx := m.getOperand_Reg(m.c.ValueDefinition(x)) + m.insert(m.allocateInstr().asXmmUnaryRmR(sseOpcodeCvtdq2ps, xx, m.c.VRegOf(ret))) + } else { + xx := m.getOperand_Reg(m.c.ValueDefinition(x)) + // Copy the value to two temporary registers. + tmp := m.copyToTmp(xx.reg()) + tmp2 := m.copyToTmp(xx.reg()) + + // Clear the higher 16 bits of each 32-bit element. + m.insert(m.allocateInstr().asXmmRmiReg(sseOpcodePslld, newOperandImm32(0xa), tmp)) + m.insert(m.allocateInstr().asXmmRmiReg(sseOpcodePsrld, newOperandImm32(0xa), tmp)) + + // Subtract the higher 16-bits from tmp2: clear the lower 16-bits of tmp2. + m.insert(m.allocateInstr().asXmmRmR(sseOpcodePsubd, newOperandReg(tmp), tmp2)) + + // Convert the lower 16-bits in tmp. + m.insert(m.allocateInstr().asXmmUnaryRmR(sseOpcodeCvtdq2ps, newOperandReg(tmp), tmp)) + + // Left shift by one and convert tmp2, meaning that halved conversion result of higher 16-bits in tmp2. + m.insert(m.allocateInstr().asXmmRmiReg(sseOpcodePsrld, newOperandImm32(1), tmp2)) + m.insert(m.allocateInstr().asXmmUnaryRmR(sseOpcodeCvtdq2ps, newOperandReg(tmp2), tmp2)) + + // Double the converted halved higher 16bits. + m.insert(m.allocateInstr().asXmmRmR(sseOpcodeAddps, newOperandReg(tmp2), tmp2)) + + // Get the conversion result by add tmp (holding lower 16-bit conversion) into tmp2. + m.insert(m.allocateInstr().asXmmRmR(sseOpcodeAddps, newOperandReg(tmp), tmp2)) + + m.copyTo(tmp2, m.c.VRegOf(ret)) + } + case ssa.VecLaneF64x2: + if signed { + xx := m.getOperand_Mem_Reg(m.c.ValueDefinition(x)) + m.insert(m.allocateInstr().asXmmUnaryRmR(sseOpcodeCvtdq2pd, xx, m.c.VRegOf(ret))) + } else { + maskReg := m.c.AllocateVReg(ssa.TypeV128) + maskLabel := m.getOrAllocateConstLabel(&m.constF64x2CvtFromIMaskIndex, f64x2CvtFromIMask[:]) + // maskReg = [0x00, 0x00, 0x30, 0x43, 0x00, 0x00, 0x30, 0x43, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00] + m.insert(m.allocateInstr().asXmmUnaryRmR(sseOpcodeMovdqu, newOperandMem(m.newAmodeRipRel(maskLabel)), maskReg)) + + _xx := m.getOperand_Reg(m.c.ValueDefinition(x)) + xx := m.copyToTmp(_xx.reg()) + + // Given that we have xx = [d1, d2, d3, d4], this results in + // xx = [d1, [0x00, 0x00, 0x30, 0x43], d2, [0x00, 0x00, 0x30, 0x43]] + // = [float64(uint32(d1)) + 0x1.0p52, float64(uint32(d2)) + 0x1.0p52] + // ^See https://stackoverflow.com/questions/13269523/can-all-32-bit-ints-be-exactly-represented-as-a-double + m.insert(m.allocateInstr().asXmmRmR(sseOpcodeUnpcklps, newOperandReg(maskReg), xx)) + + // maskReg = [float64(0x1.0p52), float64(0x1.0p52)] + maskLabel = m.getOrAllocateConstLabel(&m.constTwop52Index, twop52[:]) + m.insert(m.allocateInstr().asXmmUnaryRmR(sseOpcodeMovdqu, newOperandMem(m.newAmodeRipRel(maskLabel)), maskReg)) + + // Now, we get the result as + // xx = [float64(uint32(d1)), float64(uint32(d2))] + // because the following equality always satisfies: + // float64(0x1.0p52 + float64(uint32(x))) - float64(0x1.0p52 + float64(uint32(y))) = float64(uint32(x)) - float64(uint32(y)) + m.insert(m.allocateInstr().asXmmRmR(sseOpcodeSubpd, newOperandReg(maskReg), xx)) + + m.copyTo(xx, m.c.VRegOf(ret)) + } + default: + panic(fmt.Sprintf("invalid lane type: %s", lane)) + } +} + +var ( + // i32sMaxOnF64x2 holds math.MaxInt32(=2147483647.0) on two f64 lanes. + i32sMaxOnF64x2 = [16]byte{ + 0x00, 0x00, 0xc0, 0xff, 0xff, 0xff, 0xdf, 0x41, // float64(2147483647.0) + 0x00, 0x00, 0xc0, 0xff, 0xff, 0xff, 0xdf, 0x41, // float64(2147483647.0) + } + + // i32sMaxOnF64x2 holds math.MaxUint32(=4294967295.0) on two f64 lanes. + i32uMaxOnF64x2 = [16]byte{ + 0x00, 0x00, 0xe0, 0xff, 0xff, 0xff, 0xef, 0x41, // float64(4294967295.0) + 0x00, 0x00, 0xe0, 0xff, 0xff, 0xff, 0xef, 0x41, // float64(4294967295.0) + } + + // twop52 holds two float64(0x1.0p52) on two f64 lanes. 0x1.0p52 is special in the sense that + // with this exponent, the mantissa represents a corresponding uint32 number, and arithmetics, + // like addition or subtraction, the resulted floating point holds exactly the same + // bit representations in 32-bit integer on its mantissa. + // + // Note: the name twop52 is common across various compiler ecosystem. + // E.g. https://github.com/llvm/llvm-project/blob/92ab024f81e5b64e258b7c3baaf213c7c26fcf40/compiler-rt/lib/builtins/floatdidf.c#L28 + // E.g. https://opensource.apple.com/source/clang/clang-425.0.24/src/projects/compiler-rt/lib/floatdidf.c.auto.html + twop52 = [16]byte{ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x30, 0x43, // float64(0x1.0p52) + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x30, 0x43, // float64(0x1.0p52) + } +) + +func (m *machine) lowerVFcvtToIntSat(x, ret ssa.Value, lane ssa.VecLane, signed bool) { + _xx := m.getOperand_Reg(m.c.ValueDefinition(x)) + xx := m.copyToTmp(_xx.reg()) + + switch lane { + case ssa.VecLaneF32x4: + if signed { + tmp := m.copyToTmp(xx) + + // Assuming we have xx = [v1, v2, v3, v4]. + // + // Set all bits if lane is not NaN on tmp. + // tmp[i] = 0xffffffff if vi != NaN + // = 0 if vi == NaN + m.insert(m.allocateInstr().asXmmRmRImm(sseOpcodeCmpps, uint8(cmpPredEQ_OQ), newOperandReg(tmp), tmp)) + + // Clear NaN lanes on xx, meaning that + // xx[i] = vi if vi != NaN + // 0 if vi == NaN + m.insert(m.allocateInstr().asXmmRmR(sseOpcodeAndps, newOperandReg(tmp), xx)) + + // tmp[i] = ^vi if vi != NaN + // = 0xffffffff if vi == NaN + // which means that tmp[i] & 0x80000000 != 0 if and only if vi is negative. + m.insert(m.allocateInstr().asXmmRmR(sseOpcodeXorps, newOperandReg(xx), tmp)) + + // xx[i] = int32(vi) if vi != NaN and xx is not overflowing. + // = 0x80000000 if vi != NaN and xx is overflowing (See https://www.felixcloutier.com/x86/cvttps2dq) + // = 0 if vi == NaN + m.insert(m.allocateInstr().asXmmUnaryRmR(sseOpcodeCvttps2dq, newOperandReg(xx), xx)) + + // Below, we have to convert 0x80000000 into 0x7FFFFFFF for positive overflowing lane. + // + // tmp[i] = 0x80000000 if vi is positive + // = any satisfying any&0x80000000 = 0 if vi is negative or zero. + m.insert(m.allocateInstr().asXmmRmR(sseOpcodeAndps, newOperandReg(xx), tmp)) + + // Arithmetic right shifting tmp by 31, meaning that we have + // tmp[i] = 0xffffffff if vi is positive, 0 otherwise. + m.insert(m.allocateInstr().asXmmRmiReg(sseOpcodePsrad, newOperandImm32(0x1f), tmp)) + + // Flipping 0x80000000 if vi is positive, otherwise keep intact. + m.insert(m.allocateInstr().asXmmRmR(sseOpcodePxor, newOperandReg(tmp), xx)) + } else { + tmp := m.c.AllocateVReg(ssa.TypeV128) + m.insert(m.allocateInstr().asZeros(tmp)) + m.insert(m.allocateInstr().asXmmRmR(sseOpcodeMaxps, newOperandReg(tmp), xx)) + m.insert(m.allocateInstr().asXmmRmR(sseOpcodePcmpeqd, newOperandReg(tmp), tmp)) + m.insert(m.allocateInstr().asXmmRmiReg(sseOpcodePsrld, newOperandImm32(0x1), tmp)) + m.insert(m.allocateInstr().asXmmUnaryRmR(sseOpcodeCvtdq2ps, newOperandReg(tmp), tmp)) + tmp2 := m.copyToTmp(xx) + m.insert(m.allocateInstr().asXmmUnaryRmR(sseOpcodeCvttps2dq, newOperandReg(xx), xx)) + m.insert(m.allocateInstr().asXmmRmR(sseOpcodeSubps, newOperandReg(tmp), tmp2)) + m.insert(m.allocateInstr().asXmmRmRImm(sseOpcodeCmpps, uint8(cmpPredLE_OS), newOperandReg(tmp2), tmp)) + m.insert(m.allocateInstr().asXmmUnaryRmR(sseOpcodeCvttps2dq, newOperandReg(tmp2), tmp2)) + m.insert(m.allocateInstr().asXmmRmR(sseOpcodePxor, newOperandReg(tmp), tmp2)) + m.insert(m.allocateInstr().asXmmRmR(sseOpcodePxor, newOperandReg(tmp), tmp)) + m.insert(m.allocateInstr().asXmmRmR(sseOpcodePmaxsd, newOperandReg(tmp), tmp2)) + m.insert(m.allocateInstr().asXmmRmR(sseOpcodePaddd, newOperandReg(tmp2), xx)) + } + + case ssa.VecLaneF64x2: + tmp2 := m.c.AllocateVReg(ssa.TypeV128) + if signed { + tmp := m.copyToTmp(xx) + + // Set all bits for non-NaN lanes, zeros otherwise. + // I.e. tmp[i] = 0xffffffff_ffffffff if vi != NaN, 0 otherwise. + m.insert(m.allocateInstr().asXmmRmRImm(sseOpcodeCmppd, uint8(cmpPredEQ_OQ), newOperandReg(tmp), tmp)) + + maskLabel := m.getOrAllocateConstLabel(&m.constI32sMaxOnF64x2Index, i32sMaxOnF64x2[:]) + // Load the 2147483647 into tmp2's each lane. + m.insert(m.allocateInstr().asXmmUnaryRmR(sseOpcodeMovdqu, newOperandMem(m.newAmodeRipRel(maskLabel)), tmp2)) + + // tmp[i] = 2147483647 if vi != NaN, 0 otherwise. + m.insert(m.allocateInstr().asXmmRmR(sseOpcodeAndps, newOperandReg(tmp2), tmp)) + + // MINPD returns the source register's value as-is, so we have + // xx[i] = vi if vi != NaN + // = 0 if vi == NaN + m.insert(m.allocateInstr().asXmmRmR(sseOpcodeMinpd, newOperandReg(tmp), xx)) + + m.insert(m.allocateInstr().asXmmUnaryRmR(sseOpcodeCvttpd2dq, newOperandReg(xx), xx)) + } else { + tmp := m.c.AllocateVReg(ssa.TypeV128) + m.insert(m.allocateInstr().asZeros(tmp)) + + // xx[i] = vi if vi != NaN && vi > 0 + // = 0 if vi == NaN || vi <= 0 + m.insert(m.allocateInstr().asXmmRmR(sseOpcodeMaxpd, newOperandReg(tmp), xx)) + + // tmp2[i] = float64(math.MaxUint32) = math.MaxUint32 + maskIndex := m.getOrAllocateConstLabel(&m.constI32uMaxOnF64x2Index, i32uMaxOnF64x2[:]) + m.insert(m.allocateInstr().asXmmUnaryRmR(sseOpcodeMovdqu, newOperandMem(m.newAmodeRipRel(maskIndex)), tmp2)) + + // xx[i] = vi if vi != NaN && vi > 0 && vi <= math.MaxUint32 + // = 0 otherwise + m.insert(m.allocateInstr().asXmmRmR(sseOpcodeMinpd, newOperandReg(tmp2), xx)) + + // Round the floating points into integer. + m.insert(m.allocateInstr().asXmmRmRImm(sseOpcodeRoundpd, 0x3, newOperandReg(xx), xx)) + + // tmp2[i] = float64(0x1.0p52) + maskIndex = m.getOrAllocateConstLabel(&m.constTwop52Index, twop52[:]) + m.insert(m.allocateInstr().asXmmUnaryRmR(sseOpcodeMovdqu, newOperandMem(m.newAmodeRipRel(maskIndex)), tmp2)) + + // xx[i] = float64(0x1.0p52) + float64(uint32(vi)) if vi != NaN && vi > 0 && vi <= math.MaxUint32 + // = 0 otherwise + // + // This means that xx[i] holds exactly the same bit of uint32(vi) in its lower 32-bits. + m.insert(m.allocateInstr().asXmmRmR(sseOpcodeAddpd, newOperandReg(tmp2), xx)) + + // At this point, we have + // xx = [uint32(v0), float64(0x1.0p52), uint32(v1), float64(0x1.0p52)] + // tmp = [0, 0, 0, 0] + // as 32x4 lanes. Therefore, SHUFPS with 0b00_00_10_00 results in + // xx = [xx[00], xx[10], tmp[00], tmp[00]] = [xx[00], xx[10], 0, 0] + // meaning that for i = 0 and 1, we have + // xx[i] = uint32(vi) if vi != NaN && vi > 0 && vi <= math.MaxUint32 + // = 0 otherwise. + m.insert(m.allocateInstr().asXmmRmRImm(sseOpcodeShufps, 0b00_00_10_00, newOperandReg(tmp), xx)) + } + default: + panic(fmt.Sprintf("invalid lane type: %s", lane)) + } + + m.copyTo(xx, m.c.VRegOf(ret)) +} + +func (m *machine) lowerNarrow(x, y, ret ssa.Value, lane ssa.VecLane, signed bool) { + _xx := m.getOperand_Reg(m.c.ValueDefinition(x)) + xx := m.copyToTmp(_xx.reg()) + yy := m.getOperand_Mem_Reg(m.c.ValueDefinition(y)) + + var sseOp sseOpcode + switch lane { + case ssa.VecLaneI16x8: + if signed { + sseOp = sseOpcodePacksswb + } else { + sseOp = sseOpcodePackuswb + } + case ssa.VecLaneI32x4: + if signed { + sseOp = sseOpcodePackssdw + } else { + sseOp = sseOpcodePackusdw + } + default: + panic(fmt.Sprintf("invalid lane type: %s", lane)) + } + m.insert(m.allocateInstr().asXmmRmR(sseOp, yy, xx)) + m.copyTo(xx, m.c.VRegOf(ret)) +} + +func (m *machine) lowerWideningPairwiseDotProductS(x, y, ret ssa.Value) { + _xx := m.getOperand_Reg(m.c.ValueDefinition(x)) + xx := m.copyToTmp(_xx.reg()) + yy := m.getOperand_Mem_Reg(m.c.ValueDefinition(y)) + m.insert(m.allocateInstr().asXmmRmR(sseOpcodePmaddwd, yy, xx)) + m.copyTo(xx, m.c.VRegOf(ret)) +} + +func (m *machine) lowerVIabs(instr *ssa.Instruction) { + x, lane := instr.ArgWithLane() + rd := m.c.VRegOf(instr.Return()) + + if lane == ssa.VecLaneI64x2 { + _xx := m.getOperand_Reg(m.c.ValueDefinition(x)) + + blendReg := xmm0VReg + m.insert(m.allocateInstr().asDefineUninitializedReg(blendReg)) + + tmp := m.copyToTmp(_xx.reg()) + xx := m.copyToTmp(_xx.reg()) + + // Clear all bits on blendReg. + m.insert(m.allocateInstr().asXmmRmR(sseOpcodePxor, newOperandReg(blendReg), blendReg)) + // Subtract xx from blendMaskReg. + m.insert(m.allocateInstr().asXmmRmR(sseOpcodePsubq, newOperandReg(xx), blendReg)) + // Copy the subtracted value ^^ back into tmp. + m.copyTo(blendReg, xx) + + m.insert(m.allocateInstr().asBlendvpd(newOperandReg(tmp), xx)) + + m.copyTo(xx, rd) + } else { + var vecOp sseOpcode + switch lane { + case ssa.VecLaneI8x16: + vecOp = sseOpcodePabsb + case ssa.VecLaneI16x8: + vecOp = sseOpcodePabsw + case ssa.VecLaneI32x4: + vecOp = sseOpcodePabsd + } + rn := m.getOperand_Reg(m.c.ValueDefinition(x)) + + i := m.allocateInstr() + i.asXmmUnaryRmR(vecOp, rn, rd) + m.insert(i) + } +} + +func (m *machine) lowerVIpopcnt(instr *ssa.Instruction) { + x := instr.Arg() + rn := m.getOperand_Reg(m.c.ValueDefinition(x)) + rd := m.c.VRegOf(instr.Return()) + + tmp1 := m.c.AllocateVReg(ssa.TypeV128) + m.lowerVconst(tmp1, 0x0f0f0f0f0f0f0f0f, 0x0f0f0f0f0f0f0f0f) + + // Copy input into tmp2. + tmp2 := m.copyToTmp(rn.reg()) + + // Given that we have: + // rm = [b1, ..., b16] where bn = hn:ln and hn and ln are higher and lower 4-bits of bn. + // + // Take PAND on tmp1 and tmp2, so that we mask out all the higher bits. + // tmp2 = [l1, ..., l16]. + pand := m.allocateInstr() + pand.asXmmRmR(sseOpcodePand, newOperandReg(tmp1), tmp2) + m.insert(pand) + + // Do logical (packed word) right shift by 4 on rm and PAND against the mask (tmp1); meaning that we have + // tmp3 = [h1, ...., h16]. + tmp3 := m.copyToTmp(rn.reg()) + psrlw := m.allocateInstr() + psrlw.asXmmRmiReg(sseOpcodePsrlw, newOperandImm32(4), tmp3) + m.insert(psrlw) + + pand2 := m.allocateInstr() + pand2.asXmmRmR(sseOpcodePand, newOperandReg(tmp1), tmp3) + m.insert(pand2) + + // Read the popcntTable into tmp4, and we have + // tmp4 = [0x00, 0x01, 0x01, 0x02, 0x01, 0x02, 0x02, 0x03, 0x01, 0x02, 0x02, 0x03, 0x02, 0x03, 0x03, 0x04] + tmp4 := m.c.AllocateVReg(ssa.TypeV128) + m.lowerVconst(tmp4, 0x03_02_02_01_02_01_01_00, 0x04_03_03_02_03_02_02_01) + + // Make a copy for later. + tmp5 := m.copyToTmp(tmp4) + + // tmp4 = [popcnt(l1), ..., popcnt(l16)]. + pshufb := m.allocateInstr() + pshufb.asXmmRmR(sseOpcodePshufb, newOperandReg(tmp2), tmp4) + m.insert(pshufb) + + pshufb2 := m.allocateInstr() + pshufb2.asXmmRmR(sseOpcodePshufb, newOperandReg(tmp3), tmp5) + m.insert(pshufb2) + + // tmp4 + tmp5 is the result. + paddb := m.allocateInstr() + paddb.asXmmRmR(sseOpcodePaddb, newOperandReg(tmp4), tmp5) + m.insert(paddb) + + m.copyTo(tmp5, rd) +} + +func (m *machine) lowerVImul(instr *ssa.Instruction) { + x, y, lane := instr.Arg2WithLane() + rd := m.c.VRegOf(instr.Return()) + if lane == ssa.VecLaneI64x2 { + rn := m.getOperand_Reg(m.c.ValueDefinition(x)) + rm := m.getOperand_Reg(m.c.ValueDefinition(y)) + // Assuming that we have + // rm = [p1, p2] = [p1_lo, p1_hi, p2_lo, p2_high] + // rn = [q1, q2] = [q1_lo, q1_hi, q2_lo, q2_high] + // where pN and qN are 64-bit (quad word) lane, and pN_lo, pN_hi, qN_lo and qN_hi are 32-bit (double word) lane. + + // Copy rn into tmp1. + tmp1 := m.copyToTmp(rn.reg()) + + // And do the logical right shift by 32-bit on tmp1, which makes tmp1 = [0, p1_high, 0, p2_high] + shift := m.allocateInstr() + shift.asXmmRmiReg(sseOpcodePsrlq, newOperandImm32(32), tmp1) + m.insert(shift) + + // Execute "pmuludq rm,tmp1", which makes tmp1 = [p1_high*q1_lo, p2_high*q2_lo] where each lane is 64-bit. + mul := m.allocateInstr() + mul.asXmmRmR(sseOpcodePmuludq, rm, tmp1) + m.insert(mul) + + // Copy rm value into tmp2. + tmp2 := m.copyToTmp(rm.reg()) + + // And do the logical right shift by 32-bit on tmp2, which makes tmp2 = [0, q1_high, 0, q2_high] + shift2 := m.allocateInstr() + shift2.asXmmRmiReg(sseOpcodePsrlq, newOperandImm32(32), tmp2) + m.insert(shift2) + + // Execute "pmuludq rm,tmp2", which makes tmp2 = [p1_lo*q1_high, p2_lo*q2_high] where each lane is 64-bit. + mul2 := m.allocateInstr() + mul2.asXmmRmR(sseOpcodePmuludq, rn, tmp2) + m.insert(mul2) + + // Adds tmp1 and tmp2 and do the logical left shift by 32-bit, + // which makes tmp1 = [(p1_lo*q1_high+p1_high*q1_lo)<<32, (p2_lo*q2_high+p2_high*q2_lo)<<32] + add := m.allocateInstr() + add.asXmmRmR(sseOpcodePaddq, newOperandReg(tmp2), tmp1) + m.insert(add) + + shift3 := m.allocateInstr() + shift3.asXmmRmiReg(sseOpcodePsllq, newOperandImm32(32), tmp1) + m.insert(shift3) + + // Copy rm value into tmp3. + tmp3 := m.copyToTmp(rm.reg()) + + // "pmuludq rm,tmp3" makes tmp3 = [p1_lo*q1_lo, p2_lo*q2_lo] where each lane is 64-bit. + mul3 := m.allocateInstr() + mul3.asXmmRmR(sseOpcodePmuludq, rn, tmp3) + m.insert(mul3) + + // Finally, we get the result by computing tmp1 + tmp3, + // which makes tmp1 = [(p1_lo*q1_high+p1_high*q1_lo)<<32+p1_lo*q1_lo, (p2_lo*q2_high+p2_high*q2_lo)<<32+p2_lo*q2_lo] + add2 := m.allocateInstr() + add2.asXmmRmR(sseOpcodePaddq, newOperandReg(tmp3), tmp1) + m.insert(add2) + + m.copyTo(tmp1, rd) + + } else { + var vecOp sseOpcode + switch lane { + case ssa.VecLaneI16x8: + vecOp = sseOpcodePmullw + case ssa.VecLaneI32x4: + vecOp = sseOpcodePmulld + default: + panic("unsupported: " + lane.String()) + } + m.lowerVbBinOp(vecOp, x, y, instr.Return()) + } +} diff --git a/vendor/github.com/tetratelabs/wazero/internal/engine/wazevo/backend/isa/amd64/operands.go b/vendor/github.com/tetratelabs/wazero/internal/engine/wazevo/backend/isa/amd64/operands.go new file mode 100644 index 000000000..c6fcb8673 --- /dev/null +++ b/vendor/github.com/tetratelabs/wazero/internal/engine/wazevo/backend/isa/amd64/operands.go @@ -0,0 +1,346 @@ +package amd64 + +import ( + "fmt" + "unsafe" + + "github.com/tetratelabs/wazero/internal/engine/wazevo/backend" + "github.com/tetratelabs/wazero/internal/engine/wazevo/backend/regalloc" + "github.com/tetratelabs/wazero/internal/engine/wazevo/ssa" + "github.com/tetratelabs/wazero/internal/engine/wazevo/wazevoapi" +) + +type operand struct { + kind operandKind + data uint64 +} + +type operandKind byte + +const ( + // operandKindReg is an operand which is an integer Register. + operandKindReg operandKind = iota + 1 + + // operandKindMem is a value in Memory. + // 32, 64, or 128 bit value. + operandKindMem + + // operandKindImm32 is a signed-32-bit integer immediate value. + operandKindImm32 + + // operandKindLabel is a label. + operandKindLabel +) + +// String implements fmt.Stringer. +func (o operandKind) String() string { + switch o { + case operandKindReg: + return "reg" + case operandKindMem: + return "mem" + case operandKindImm32: + return "imm32" + case operandKindLabel: + return "label" + default: + panic("BUG: invalid operand kind") + } +} + +// format returns the string representation of the operand. +// _64 is only for the case where the operand is a register, and it's integer. +func (o *operand) format(_64 bool) string { + switch o.kind { + case operandKindReg: + return formatVRegSized(o.reg(), _64) + case operandKindMem: + return o.addressMode().String() + case operandKindImm32: + return fmt.Sprintf("$%d", int32(o.imm32())) + case operandKindLabel: + return backend.Label(o.imm32()).String() + default: + panic(fmt.Sprintf("BUG: invalid operand: %s", o.kind)) + } +} + +//go:inline +func (o *operand) reg() regalloc.VReg { + return regalloc.VReg(o.data) +} + +//go:inline +func (o *operand) setReg(r regalloc.VReg) { + o.data = uint64(r) +} + +//go:inline +func (o *operand) addressMode() *amode { + return wazevoapi.PtrFromUintptr[amode](uintptr(o.data)) +} + +//go:inline +func (o *operand) imm32() uint32 { + return uint32(o.data) +} + +func (o *operand) label() backend.Label { + switch o.kind { + case operandKindLabel: + return backend.Label(o.data) + case operandKindMem: + mem := o.addressMode() + if mem.kind() != amodeRipRel { + panic("BUG: invalid label") + } + return backend.Label(mem.imm32) + default: + panic("BUG: invalid operand kind") + } +} + +func newOperandLabel(label backend.Label) operand { + return operand{kind: operandKindLabel, data: uint64(label)} +} + +func newOperandReg(r regalloc.VReg) operand { + return operand{kind: operandKindReg, data: uint64(r)} +} + +func newOperandImm32(imm32 uint32) operand { + return operand{kind: operandKindImm32, data: uint64(imm32)} +} + +func newOperandMem(amode *amode) operand { + return operand{kind: operandKindMem, data: uint64(uintptr(unsafe.Pointer(amode)))} +} + +// amode is a memory operand (addressing mode). +type amode struct { + kindWithShift uint32 + imm32 uint32 + base regalloc.VReg + + // For amodeRegRegShift: + index regalloc.VReg +} + +type amodeKind byte + +const ( + // amodeRegRegShift calculates sign-extend-32-to-64(Immediate) + base + amodeImmReg amodeKind = iota + 1 + + // amodeImmRBP is the same as amodeImmReg, but the base register is fixed to RBP. + // The only differece is that it doesn't tell the register allocator to use RBP which is distracting for the + // register allocator. + amodeImmRBP + + // amodeRegRegShift calculates sign-extend-32-to-64(Immediate) + base + (Register2 << Shift) + amodeRegRegShift + + // amodeRipRel is a RIP-relative addressing mode specified by the label. + amodeRipRel + + // TODO: there are other addressing modes such as the one without base register. +) + +func (a *amode) kind() amodeKind { + return amodeKind(a.kindWithShift & 0xff) +} + +func (a *amode) shift() byte { + return byte(a.kindWithShift >> 8) +} + +func (a *amode) uses(rs *[]regalloc.VReg) { + switch a.kind() { + case amodeImmReg: + *rs = append(*rs, a.base) + case amodeRegRegShift: + *rs = append(*rs, a.base, a.index) + case amodeImmRBP, amodeRipRel: + default: + panic("BUG: invalid amode kind") + } +} + +func (a *amode) nregs() int { + switch a.kind() { + case amodeImmReg: + return 1 + case amodeRegRegShift: + return 2 + case amodeImmRBP, amodeRipRel: + return 0 + default: + panic("BUG: invalid amode kind") + } +} + +func (a *amode) assignUses(i int, reg regalloc.VReg) { + switch a.kind() { + case amodeImmReg: + if i == 0 { + a.base = reg + } else { + panic("BUG: invalid amode assignment") + } + case amodeRegRegShift: + if i == 0 { + a.base = reg + } else if i == 1 { + a.index = reg + } else { + panic("BUG: invalid amode assignment") + } + default: + panic("BUG: invalid amode assignment") + } +} + +func (m *machine) newAmodeImmReg(imm32 uint32, base regalloc.VReg) *amode { + ret := m.amodePool.Allocate() + *ret = amode{kindWithShift: uint32(amodeImmReg), imm32: imm32, base: base} + return ret +} + +func (m *machine) newAmodeImmRBPReg(imm32 uint32) *amode { + ret := m.amodePool.Allocate() + *ret = amode{kindWithShift: uint32(amodeImmRBP), imm32: imm32, base: rbpVReg} + return ret +} + +func (m *machine) newAmodeRegRegShift(imm32 uint32, base, index regalloc.VReg, shift byte) *amode { + if shift > 3 { + panic(fmt.Sprintf("BUG: invalid shift (must be 3>=): %d", shift)) + } + ret := m.amodePool.Allocate() + *ret = amode{kindWithShift: uint32(amodeRegRegShift) | uint32(shift)<<8, imm32: imm32, base: base, index: index} + return ret +} + +func (m *machine) newAmodeRipRel(label backend.Label) *amode { + ret := m.amodePool.Allocate() + *ret = amode{kindWithShift: uint32(amodeRipRel), imm32: uint32(label)} + return ret +} + +// String implements fmt.Stringer. +func (a *amode) String() string { + switch a.kind() { + case amodeImmReg, amodeImmRBP: + if a.imm32 == 0 { + return fmt.Sprintf("(%s)", formatVRegSized(a.base, true)) + } + return fmt.Sprintf("%d(%s)", int32(a.imm32), formatVRegSized(a.base, true)) + case amodeRegRegShift: + shift := 1 << a.shift() + if a.imm32 == 0 { + return fmt.Sprintf( + "(%s,%s,%d)", + formatVRegSized(a.base, true), formatVRegSized(a.index, true), shift) + } + return fmt.Sprintf( + "%d(%s,%s,%d)", + int32(a.imm32), formatVRegSized(a.base, true), formatVRegSized(a.index, true), shift) + case amodeRipRel: + return fmt.Sprintf("%s(%%rip)", backend.Label(a.imm32)) + default: + panic("BUG: invalid amode kind") + } +} + +func (m *machine) getOperand_Mem_Reg(def *backend.SSAValueDefinition) (op operand) { + if def.IsFromBlockParam() { + return newOperandReg(def.BlkParamVReg) + } + + if def.SSAValue().Type() == ssa.TypeV128 { + // SIMD instructions require strict memory alignment, so we don't support the memory operand for V128 at the moment. + return m.getOperand_Reg(def) + } + + if m.c.MatchInstr(def, ssa.OpcodeLoad) { + instr := def.Instr + ptr, offset, _ := instr.LoadData() + op = newOperandMem(m.lowerToAddressMode(ptr, offset)) + instr.MarkLowered() + return op + } + return m.getOperand_Reg(def) +} + +func (m *machine) getOperand_Mem_Imm32_Reg(def *backend.SSAValueDefinition) (op operand) { + if def.IsFromBlockParam() { + return newOperandReg(def.BlkParamVReg) + } + + if m.c.MatchInstr(def, ssa.OpcodeLoad) { + instr := def.Instr + ptr, offset, _ := instr.LoadData() + op = newOperandMem(m.lowerToAddressMode(ptr, offset)) + instr.MarkLowered() + return op + } + return m.getOperand_Imm32_Reg(def) +} + +func (m *machine) getOperand_Imm32_Reg(def *backend.SSAValueDefinition) (op operand) { + if def.IsFromBlockParam() { + return newOperandReg(def.BlkParamVReg) + } + + instr := def.Instr + if instr.Constant() { + // If the operation is 64-bit, x64 sign-extends the 32-bit immediate value. + // Therefore, we need to check if the immediate value is within the 32-bit range and if the sign bit is set, + // we should not use the immediate value. + if op, ok := asImm32Operand(instr.ConstantVal(), instr.Return().Type() == ssa.TypeI32); ok { + instr.MarkLowered() + return op + } + } + return m.getOperand_Reg(def) +} + +func asImm32Operand(val uint64, allowSignExt bool) (operand, bool) { + if imm32, ok := asImm32(val, allowSignExt); ok { + return newOperandImm32(imm32), true + } + return operand{}, false +} + +func asImm32(val uint64, allowSignExt bool) (uint32, bool) { + u32val := uint32(val) + if uint64(u32val) != val { + return 0, false + } + if !allowSignExt && u32val&0x80000000 != 0 { + return 0, false + } + return u32val, true +} + +func (m *machine) getOperand_Reg(def *backend.SSAValueDefinition) (op operand) { + var v regalloc.VReg + if def.IsFromBlockParam() { + v = def.BlkParamVReg + } else { + instr := def.Instr + if instr.Constant() { + // We inline all the constant instructions so that we could reduce the register usage. + v = m.lowerConstant(instr) + instr.MarkLowered() + } else { + if n := def.N; n == 0 { + v = m.c.VRegOf(instr.Return()) + } else { + _, rs := instr.Returns() + v = m.c.VRegOf(rs[n-1]) + } + } + } + return newOperandReg(v) +} diff --git a/vendor/github.com/tetratelabs/wazero/internal/engine/wazevo/backend/isa/amd64/reflect.go b/vendor/github.com/tetratelabs/wazero/internal/engine/wazevo/backend/isa/amd64/reflect.go new file mode 100644 index 000000000..5219837e3 --- /dev/null +++ b/vendor/github.com/tetratelabs/wazero/internal/engine/wazevo/backend/isa/amd64/reflect.go @@ -0,0 +1,11 @@ +//go:build !tinygo + +package amd64 + +import "reflect" + +// setSliceLimits sets both Cap and Len for the given reflected slice. +func setSliceLimits(s *reflect.SliceHeader, limit uintptr) { + s.Len = int(limit) + s.Cap = int(limit) +} diff --git a/vendor/github.com/tetratelabs/wazero/internal/engine/wazevo/backend/isa/amd64/reflect_tinygo.go b/vendor/github.com/tetratelabs/wazero/internal/engine/wazevo/backend/isa/amd64/reflect_tinygo.go new file mode 100644 index 000000000..df4cf46ec --- /dev/null +++ b/vendor/github.com/tetratelabs/wazero/internal/engine/wazevo/backend/isa/amd64/reflect_tinygo.go @@ -0,0 +1,11 @@ +//go:build tinygo + +package amd64 + +import "reflect" + +// setSliceLimits sets both Cap and Len for the given reflected slice. +func setSliceLimits(s *reflect.SliceHeader, limit uintptr) { + s.Len = limit + s.Len = limit +} diff --git a/vendor/github.com/tetratelabs/wazero/internal/engine/wazevo/backend/isa/amd64/reg.go b/vendor/github.com/tetratelabs/wazero/internal/engine/wazevo/backend/isa/amd64/reg.go new file mode 100644 index 000000000..4aec856fa --- /dev/null +++ b/vendor/github.com/tetratelabs/wazero/internal/engine/wazevo/backend/isa/amd64/reg.go @@ -0,0 +1,181 @@ +package amd64 + +import ( + "fmt" + + "github.com/tetratelabs/wazero/internal/engine/wazevo/backend/regalloc" +) + +// Amd64-specific registers. +const ( + // rax is a gp register. + rax = regalloc.RealRegInvalid + 1 + iota + // rcx is a gp register. + rcx + // rdx is a gp register. + rdx + // rbx is a gp register. + rbx + // rsp is a gp register. + rsp + // rbp is a gp register. + rbp + // rsi is a gp register. + rsi + // rdi is a gp register. + rdi + // r8 is a gp register. + r8 + // r9 is a gp register. + r9 + // r10 is a gp register. + r10 + // r11 is a gp register. + r11 + // r12 is a gp register. + r12 + // r13 is a gp register. + r13 + // r14 is a gp register. + r14 + // r15 is a gp register. + r15 + + // xmm0 is a vector register. + xmm0 + // xmm1 is a vector register. + xmm1 + // xmm2 is a vector register. + xmm2 + // xmm3 is a vector register. + xmm3 + // xmm4 is a vector register. + xmm4 + // xmm5 is a vector register. + xmm5 + // xmm6 is a vector register. + xmm6 + // xmm7 is a vector register. + xmm7 + // xmm8 is a vector register. + xmm8 + // xmm9 is a vector register. + xmm9 + // xmm10 is a vector register. + xmm10 + // xmm11 is a vector register. + xmm11 + // xmm12 is a vector register. + xmm12 + // xmm13 is a vector register. + xmm13 + // xmm14 is a vector register. + xmm14 + // xmm15 is a vector register. + xmm15 +) + +var ( + raxVReg = regalloc.FromRealReg(rax, regalloc.RegTypeInt) + rcxVReg = regalloc.FromRealReg(rcx, regalloc.RegTypeInt) + rdxVReg = regalloc.FromRealReg(rdx, regalloc.RegTypeInt) + rbxVReg = regalloc.FromRealReg(rbx, regalloc.RegTypeInt) + rspVReg = regalloc.FromRealReg(rsp, regalloc.RegTypeInt) + rbpVReg = regalloc.FromRealReg(rbp, regalloc.RegTypeInt) + rsiVReg = regalloc.FromRealReg(rsi, regalloc.RegTypeInt) + rdiVReg = regalloc.FromRealReg(rdi, regalloc.RegTypeInt) + r8VReg = regalloc.FromRealReg(r8, regalloc.RegTypeInt) + r9VReg = regalloc.FromRealReg(r9, regalloc.RegTypeInt) + r10VReg = regalloc.FromRealReg(r10, regalloc.RegTypeInt) + r11VReg = regalloc.FromRealReg(r11, regalloc.RegTypeInt) + r12VReg = regalloc.FromRealReg(r12, regalloc.RegTypeInt) + r13VReg = regalloc.FromRealReg(r13, regalloc.RegTypeInt) + r14VReg = regalloc.FromRealReg(r14, regalloc.RegTypeInt) + r15VReg = regalloc.FromRealReg(r15, regalloc.RegTypeInt) + + xmm0VReg = regalloc.FromRealReg(xmm0, regalloc.RegTypeFloat) + xmm1VReg = regalloc.FromRealReg(xmm1, regalloc.RegTypeFloat) + xmm2VReg = regalloc.FromRealReg(xmm2, regalloc.RegTypeFloat) + xmm3VReg = regalloc.FromRealReg(xmm3, regalloc.RegTypeFloat) + xmm4VReg = regalloc.FromRealReg(xmm4, regalloc.RegTypeFloat) + xmm5VReg = regalloc.FromRealReg(xmm5, regalloc.RegTypeFloat) + xmm6VReg = regalloc.FromRealReg(xmm6, regalloc.RegTypeFloat) + xmm7VReg = regalloc.FromRealReg(xmm7, regalloc.RegTypeFloat) + xmm8VReg = regalloc.FromRealReg(xmm8, regalloc.RegTypeFloat) + xmm9VReg = regalloc.FromRealReg(xmm9, regalloc.RegTypeFloat) + xmm10VReg = regalloc.FromRealReg(xmm10, regalloc.RegTypeFloat) + xmm11VReg = regalloc.FromRealReg(xmm11, regalloc.RegTypeFloat) + xmm12VReg = regalloc.FromRealReg(xmm12, regalloc.RegTypeFloat) + xmm13VReg = regalloc.FromRealReg(xmm13, regalloc.RegTypeFloat) + xmm14VReg = regalloc.FromRealReg(xmm14, regalloc.RegTypeFloat) + xmm15VReg = regalloc.FromRealReg(xmm15, regalloc.RegTypeFloat) +) + +var regNames = [...]string{ + rax: "rax", + rcx: "rcx", + rdx: "rdx", + rbx: "rbx", + rsp: "rsp", + rbp: "rbp", + rsi: "rsi", + rdi: "rdi", + r8: "r8", + r9: "r9", + r10: "r10", + r11: "r11", + r12: "r12", + r13: "r13", + r14: "r14", + r15: "r15", + xmm0: "xmm0", + xmm1: "xmm1", + xmm2: "xmm2", + xmm3: "xmm3", + xmm4: "xmm4", + xmm5: "xmm5", + xmm6: "xmm6", + xmm7: "xmm7", + xmm8: "xmm8", + xmm9: "xmm9", + xmm10: "xmm10", + xmm11: "xmm11", + xmm12: "xmm12", + xmm13: "xmm13", + xmm14: "xmm14", + xmm15: "xmm15", +} + +func formatVRegSized(r regalloc.VReg, _64 bool) string { + if r.IsRealReg() { + if r.RegType() == regalloc.RegTypeInt { + rr := r.RealReg() + orig := regNames[rr] + if rr <= rdi { + if _64 { + return "%" + orig + } else { + return "%e" + orig[1:] + } + } else { + if _64 { + return "%" + orig + } else { + return "%" + orig + "d" + } + } + } else { + return "%" + regNames[r.RealReg()] + } + } else { + if r.RegType() == regalloc.RegTypeInt { + if _64 { + return fmt.Sprintf("%%r%d?", r.ID()) + } else { + return fmt.Sprintf("%%r%dd?", r.ID()) + } + } else { + return fmt.Sprintf("%%xmm%d?", r.ID()) + } + } +} diff --git a/vendor/github.com/tetratelabs/wazero/internal/engine/wazevo/backend/isa/amd64/stack.go b/vendor/github.com/tetratelabs/wazero/internal/engine/wazevo/backend/isa/amd64/stack.go new file mode 100644 index 000000000..05ba5f027 --- /dev/null +++ b/vendor/github.com/tetratelabs/wazero/internal/engine/wazevo/backend/isa/amd64/stack.go @@ -0,0 +1,128 @@ +package amd64 + +import ( + "encoding/binary" + "reflect" + "unsafe" + + "github.com/tetratelabs/wazero/internal/wasmdebug" +) + +func stackView(rbp, top uintptr) []byte { + var stackBuf []byte + { + // TODO: use unsafe.Slice after floor version is set to Go 1.20. + hdr := (*reflect.SliceHeader)(unsafe.Pointer(&stackBuf)) + hdr.Data = rbp + setSliceLimits(hdr, top-rbp) + } + return stackBuf +} + +// UnwindStack implements wazevo.unwindStack. +func UnwindStack(_, rbp, top uintptr, returnAddresses []uintptr) []uintptr { + stackBuf := stackView(rbp, top) + + for i := uint64(0); i < uint64(len(stackBuf)); { + // (high address) + // +-----------------+ + // | ....... | + // | ret Y | + // | ....... | + // | ret 0 | + // | arg X | + // | ....... | + // | arg 1 | + // | arg 0 | + // | ReturnAddress | + // | Caller_RBP | + // +-----------------+ <---- Caller_RBP + // | ........... | + // | clobbered M | + // | ............ | + // | clobbered 0 | + // | spill slot N | + // | ............ | + // | spill slot 0 | + // | ReturnAddress | + // | Caller_RBP | + // +-----------------+ <---- RBP + // (low address) + + callerRBP := binary.LittleEndian.Uint64(stackBuf[i:]) + retAddr := binary.LittleEndian.Uint64(stackBuf[i+8:]) + returnAddresses = append(returnAddresses, uintptr(retAddr)) + i = callerRBP - uint64(rbp) + if len(returnAddresses) == wasmdebug.MaxFrames { + break + } + } + return returnAddresses +} + +// GoCallStackView implements wazevo.goCallStackView. +func GoCallStackView(stackPointerBeforeGoCall *uint64) []uint64 { + // (high address) + // +-----------------+ <----+ + // | xxxxxxxxxxx | | ;; optional unused space to make it 16-byte aligned. + // ^ | arg[N]/ret[M] | | + // sliceSize | | ............ | | SizeInBytes/8 + // | | arg[1]/ret[1] | | + // v | arg[0]/ret[0] | <----+ + // | SizeInBytes | + // +-----------------+ <---- stackPointerBeforeGoCall + // (low address) + data := unsafe.Pointer(uintptr(unsafe.Pointer(stackPointerBeforeGoCall)) + 8) + size := *stackPointerBeforeGoCall / 8 + return unsafe.Slice((*uint64)(data), int(size)) +} + +func AdjustClonedStack(oldRsp, oldTop, rsp, rbp, top uintptr) { + diff := uint64(rsp - oldRsp) + + newBuf := stackView(rbp, top) + for i := uint64(0); i < uint64(len(newBuf)); { + // (high address) + // +-----------------+ + // | ....... | + // | ret Y | + // | ....... | + // | ret 0 | + // | arg X | + // | ....... | + // | arg 1 | + // | arg 0 | + // | ReturnAddress | + // | Caller_RBP | + // +-----------------+ <---- Caller_RBP + // | ........... | + // | clobbered M | + // | ............ | + // | clobbered 0 | + // | spill slot N | + // | ............ | + // | spill slot 0 | + // | ReturnAddress | + // | Caller_RBP | + // +-----------------+ <---- RBP + // (low address) + + callerRBP := binary.LittleEndian.Uint64(newBuf[i:]) + if callerRBP == 0 { + // End of stack. + break + } + if i64 := int64(callerRBP); i64 < int64(oldRsp) || i64 >= int64(oldTop) { + panic("BUG: callerRBP is out of range") + } + if int(callerRBP) < 0 { + panic("BUG: callerRBP is negative") + } + adjustedCallerRBP := callerRBP + diff + if int(adjustedCallerRBP) < 0 { + panic("BUG: adjustedCallerRBP is negative") + } + binary.LittleEndian.PutUint64(newBuf[i:], adjustedCallerRBP) + i = adjustedCallerRBP - uint64(rbp) + } +} diff --git a/vendor/github.com/tetratelabs/wazero/internal/engine/wazevo/backend/isa/arm64/abi.go b/vendor/github.com/tetratelabs/wazero/internal/engine/wazevo/backend/isa/arm64/abi.go new file mode 100644 index 000000000..6615471c6 --- /dev/null +++ b/vendor/github.com/tetratelabs/wazero/internal/engine/wazevo/backend/isa/arm64/abi.go @@ -0,0 +1,332 @@ +package arm64 + +import ( + "github.com/tetratelabs/wazero/internal/engine/wazevo/backend" + "github.com/tetratelabs/wazero/internal/engine/wazevo/backend/regalloc" + "github.com/tetratelabs/wazero/internal/engine/wazevo/ssa" +) + +// References: +// * https://github.com/golang/go/blob/49d42128fd8594c172162961ead19ac95e247d24/src/cmd/compile/abi-internal.md#arm64-architecture +// * https://developer.arm.com/documentation/102374/0101/Procedure-Call-Standard + +var ( + intParamResultRegs = []regalloc.RealReg{x0, x1, x2, x3, x4, x5, x6, x7} + floatParamResultRegs = []regalloc.RealReg{v0, v1, v2, v3, v4, v5, v6, v7} +) + +var regInfo = ®alloc.RegisterInfo{ + AllocatableRegisters: [regalloc.NumRegType][]regalloc.RealReg{ + // We don't allocate: + // - x18: Reserved by the macOS: https://developer.apple.com/documentation/xcode/writing-arm64-code-for-apple-platforms#Respect-the-purpose-of-specific-CPU-registers + // - x28: Reserved by Go runtime. + // - x27(=tmpReg): because of the reason described on tmpReg. + regalloc.RegTypeInt: { + x8, x9, x10, x11, x12, x13, x14, x15, + x16, x17, x19, x20, x21, x22, x23, x24, x25, + x26, x29, x30, + // These are the argument/return registers. Less preferred in the allocation. + x7, x6, x5, x4, x3, x2, x1, x0, + }, + regalloc.RegTypeFloat: { + v8, v9, v10, v11, v12, v13, v14, v15, v16, v17, v18, v19, + v20, v21, v22, v23, v24, v25, v26, v27, v28, v29, v30, + // These are the argument/return registers. Less preferred in the allocation. + v7, v6, v5, v4, v3, v2, v1, v0, + }, + }, + CalleeSavedRegisters: regalloc.NewRegSet( + x19, x20, x21, x22, x23, x24, x25, x26, x28, + v18, v19, v20, v21, v22, v23, v24, v25, v26, v27, v28, v29, v30, v31, + ), + CallerSavedRegisters: regalloc.NewRegSet( + x0, x1, x2, x3, x4, x5, x6, x7, x8, x9, x10, x11, x12, x13, x14, x15, x16, x17, x29, x30, + v0, v1, v2, v3, v4, v5, v6, v7, v8, v9, v10, v11, v12, v13, v14, v15, v16, v17, + ), + RealRegToVReg: []regalloc.VReg{ + x0: x0VReg, x1: x1VReg, x2: x2VReg, x3: x3VReg, x4: x4VReg, x5: x5VReg, x6: x6VReg, x7: x7VReg, x8: x8VReg, x9: x9VReg, x10: x10VReg, x11: x11VReg, x12: x12VReg, x13: x13VReg, x14: x14VReg, x15: x15VReg, x16: x16VReg, x17: x17VReg, x18: x18VReg, x19: x19VReg, x20: x20VReg, x21: x21VReg, x22: x22VReg, x23: x23VReg, x24: x24VReg, x25: x25VReg, x26: x26VReg, x27: x27VReg, x28: x28VReg, x29: x29VReg, x30: x30VReg, + v0: v0VReg, v1: v1VReg, v2: v2VReg, v3: v3VReg, v4: v4VReg, v5: v5VReg, v6: v6VReg, v7: v7VReg, v8: v8VReg, v9: v9VReg, v10: v10VReg, v11: v11VReg, v12: v12VReg, v13: v13VReg, v14: v14VReg, v15: v15VReg, v16: v16VReg, v17: v17VReg, v18: v18VReg, v19: v19VReg, v20: v20VReg, v21: v21VReg, v22: v22VReg, v23: v23VReg, v24: v24VReg, v25: v25VReg, v26: v26VReg, v27: v27VReg, v28: v28VReg, v29: v29VReg, v30: v30VReg, v31: v31VReg, + }, + RealRegName: func(r regalloc.RealReg) string { return regNames[r] }, + RealRegType: func(r regalloc.RealReg) regalloc.RegType { + if r < v0 { + return regalloc.RegTypeInt + } + return regalloc.RegTypeFloat + }, +} + +// ArgsResultsRegs implements backend.Machine. +func (m *machine) ArgsResultsRegs() (argResultInts, argResultFloats []regalloc.RealReg) { + return intParamResultRegs, floatParamResultRegs +} + +// LowerParams implements backend.FunctionABI. +func (m *machine) LowerParams(args []ssa.Value) { + a := m.currentABI + + for i, ssaArg := range args { + if !ssaArg.Valid() { + continue + } + reg := m.compiler.VRegOf(ssaArg) + arg := &a.Args[i] + if arg.Kind == backend.ABIArgKindReg { + m.InsertMove(reg, arg.Reg, arg.Type) + } else { + // TODO: we could use pair load if there's consecutive loads for the same type. + // + // (high address) + // +-----------------+ + // | ....... | + // | ret Y | + // | ....... | + // | ret 0 | + // | arg X | + // | ....... | + // | arg 1 | + // | arg 0 | <-| + // | ReturnAddress | | + // +-----------------+ | + // | ........... | | + // | clobbered M | | argStackOffset: is unknown at this point of compilation. + // | ............ | | + // | clobbered 0 | | + // | spill slot N | | + // | ........... | | + // | spill slot 0 | | + // SP---> +-----------------+ <-+ + // (low address) + + bits := arg.Type.Bits() + // At this point of compilation, we don't yet know how much space exist below the return address. + // So we instruct the address mode to add the `argStackOffset` to the offset at the later phase of compilation. + amode := addressMode{imm: arg.Offset, rn: spVReg, kind: addressModeKindArgStackSpace} + load := m.allocateInstr() + switch arg.Type { + case ssa.TypeI32, ssa.TypeI64: + load.asULoad(operandNR(reg), amode, bits) + case ssa.TypeF32, ssa.TypeF64, ssa.TypeV128: + load.asFpuLoad(operandNR(reg), amode, bits) + default: + panic("BUG") + } + m.insert(load) + m.unresolvedAddressModes = append(m.unresolvedAddressModes, load) + } + } +} + +// LowerReturns lowers the given returns. +func (m *machine) LowerReturns(rets []ssa.Value) { + a := m.currentABI + + l := len(rets) - 1 + for i := range rets { + // Reverse order in order to avoid overwriting the stack returns existing in the return registers. + ret := rets[l-i] + r := &a.Rets[l-i] + reg := m.compiler.VRegOf(ret) + if def := m.compiler.ValueDefinition(ret); def.IsFromInstr() { + // Constant instructions are inlined. + if inst := def.Instr; inst.Constant() { + val := inst.Return() + valType := val.Type() + v := inst.ConstantVal() + m.insertLoadConstant(v, valType, reg) + } + } + if r.Kind == backend.ABIArgKindReg { + m.InsertMove(r.Reg, reg, ret.Type()) + } else { + // TODO: we could use pair store if there's consecutive stores for the same type. + // + // (high address) + // +-----------------+ + // | ....... | + // | ret Y | + // | ....... | + // | ret 0 | <-+ + // | arg X | | + // | ....... | | + // | arg 1 | | + // | arg 0 | | + // | ReturnAddress | | + // +-----------------+ | + // | ........... | | + // | spill slot M | | retStackOffset: is unknown at this point of compilation. + // | ............ | | + // | spill slot 2 | | + // | spill slot 1 | | + // | clobbered 0 | | + // | clobbered 1 | | + // | ........... | | + // | clobbered N | | + // SP---> +-----------------+ <-+ + // (low address) + + bits := r.Type.Bits() + + // At this point of compilation, we don't yet know how much space exist below the return address. + // So we instruct the address mode to add the `retStackOffset` to the offset at the later phase of compilation. + amode := addressMode{imm: r.Offset, rn: spVReg, kind: addressModeKindResultStackSpace} + store := m.allocateInstr() + store.asStore(operandNR(reg), amode, bits) + m.insert(store) + m.unresolvedAddressModes = append(m.unresolvedAddressModes, store) + } + } +} + +// callerGenVRegToFunctionArg is the opposite of GenFunctionArgToVReg, which is used to generate the +// caller side of the function call. +func (m *machine) callerGenVRegToFunctionArg(a *backend.FunctionABI, argIndex int, reg regalloc.VReg, def *backend.SSAValueDefinition, slotBegin int64) { + arg := &a.Args[argIndex] + if def != nil && def.IsFromInstr() { + // Constant instructions are inlined. + if inst := def.Instr; inst.Constant() { + val := inst.Return() + valType := val.Type() + v := inst.ConstantVal() + m.insertLoadConstant(v, valType, reg) + } + } + if arg.Kind == backend.ABIArgKindReg { + m.InsertMove(arg.Reg, reg, arg.Type) + } else { + // TODO: we could use pair store if there's consecutive stores for the same type. + // + // Note that at this point, stack pointer is already adjusted. + bits := arg.Type.Bits() + amode := m.resolveAddressModeForOffset(arg.Offset-slotBegin, bits, spVReg, false) + store := m.allocateInstr() + store.asStore(operandNR(reg), amode, bits) + m.insert(store) + } +} + +func (m *machine) callerGenFunctionReturnVReg(a *backend.FunctionABI, retIndex int, reg regalloc.VReg, slotBegin int64) { + r := &a.Rets[retIndex] + if r.Kind == backend.ABIArgKindReg { + m.InsertMove(reg, r.Reg, r.Type) + } else { + // TODO: we could use pair load if there's consecutive loads for the same type. + amode := m.resolveAddressModeForOffset(a.ArgStackSize+r.Offset-slotBegin, r.Type.Bits(), spVReg, false) + ldr := m.allocateInstr() + switch r.Type { + case ssa.TypeI32, ssa.TypeI64: + ldr.asULoad(operandNR(reg), amode, r.Type.Bits()) + case ssa.TypeF32, ssa.TypeF64, ssa.TypeV128: + ldr.asFpuLoad(operandNR(reg), amode, r.Type.Bits()) + default: + panic("BUG") + } + m.insert(ldr) + } +} + +func (m *machine) resolveAddressModeForOffsetAndInsert(cur *instruction, offset int64, dstBits byte, rn regalloc.VReg, allowTmpRegUse bool) (*instruction, addressMode) { + exct := m.executableContext + exct.PendingInstructions = exct.PendingInstructions[:0] + mode := m.resolveAddressModeForOffset(offset, dstBits, rn, allowTmpRegUse) + for _, instr := range exct.PendingInstructions { + cur = linkInstr(cur, instr) + } + return cur, mode +} + +func (m *machine) resolveAddressModeForOffset(offset int64, dstBits byte, rn regalloc.VReg, allowTmpRegUse bool) addressMode { + if rn.RegType() != regalloc.RegTypeInt { + panic("BUG: rn should be a pointer: " + formatVRegSized(rn, 64)) + } + var amode addressMode + if offsetFitsInAddressModeKindRegUnsignedImm12(dstBits, offset) { + amode = addressMode{kind: addressModeKindRegUnsignedImm12, rn: rn, imm: offset} + } else if offsetFitsInAddressModeKindRegSignedImm9(offset) { + amode = addressMode{kind: addressModeKindRegSignedImm9, rn: rn, imm: offset} + } else { + var indexReg regalloc.VReg + if allowTmpRegUse { + m.lowerConstantI64(tmpRegVReg, offset) + indexReg = tmpRegVReg + } else { + indexReg = m.compiler.AllocateVReg(ssa.TypeI64) + m.lowerConstantI64(indexReg, offset) + } + amode = addressMode{kind: addressModeKindRegReg, rn: rn, rm: indexReg, extOp: extendOpUXTX /* indicates index rm is 64-bit */} + } + return amode +} + +func (m *machine) lowerCall(si *ssa.Instruction) { + isDirectCall := si.Opcode() == ssa.OpcodeCall + var indirectCalleePtr ssa.Value + var directCallee ssa.FuncRef + var sigID ssa.SignatureID + var args []ssa.Value + if isDirectCall { + directCallee, sigID, args = si.CallData() + } else { + indirectCalleePtr, sigID, args, _ /* on arm64, the calling convention is compatible with the Go runtime */ = si.CallIndirectData() + } + calleeABI := m.compiler.GetFunctionABI(m.compiler.SSABuilder().ResolveSignature(sigID)) + + stackSlotSize := int64(calleeABI.AlignedArgResultStackSlotSize()) + if m.maxRequiredStackSizeForCalls < stackSlotSize+16 { + m.maxRequiredStackSizeForCalls = stackSlotSize + 16 // return address frame. + } + + for i, arg := range args { + reg := m.compiler.VRegOf(arg) + def := m.compiler.ValueDefinition(arg) + m.callerGenVRegToFunctionArg(calleeABI, i, reg, def, stackSlotSize) + } + + if isDirectCall { + call := m.allocateInstr() + call.asCall(directCallee, calleeABI) + m.insert(call) + } else { + ptr := m.compiler.VRegOf(indirectCalleePtr) + callInd := m.allocateInstr() + callInd.asCallIndirect(ptr, calleeABI) + m.insert(callInd) + } + + var index int + r1, rs := si.Returns() + if r1.Valid() { + m.callerGenFunctionReturnVReg(calleeABI, 0, m.compiler.VRegOf(r1), stackSlotSize) + index++ + } + + for _, r := range rs { + m.callerGenFunctionReturnVReg(calleeABI, index, m.compiler.VRegOf(r), stackSlotSize) + index++ + } +} + +func (m *machine) insertAddOrSubStackPointer(rd regalloc.VReg, diff int64, add bool) { + if imm12Operand, ok := asImm12Operand(uint64(diff)); ok { + alu := m.allocateInstr() + var ao aluOp + if add { + ao = aluOpAdd + } else { + ao = aluOpSub + } + alu.asALU(ao, operandNR(rd), operandNR(spVReg), imm12Operand, true) + m.insert(alu) + } else { + m.lowerConstantI64(tmpRegVReg, diff) + alu := m.allocateInstr() + var ao aluOp + if add { + ao = aluOpAdd + } else { + ao = aluOpSub + } + alu.asALU(ao, operandNR(rd), operandNR(spVReg), operandNR(tmpRegVReg), true) + m.insert(alu) + } +} diff --git a/vendor/github.com/tetratelabs/wazero/internal/engine/wazevo/backend/isa/arm64/abi_entry_arm64.go b/vendor/github.com/tetratelabs/wazero/internal/engine/wazevo/backend/isa/arm64/abi_entry_arm64.go new file mode 100644 index 000000000..5f0c613df --- /dev/null +++ b/vendor/github.com/tetratelabs/wazero/internal/engine/wazevo/backend/isa/arm64/abi_entry_arm64.go @@ -0,0 +1,9 @@ +package arm64 + +// entrypoint enters the machine code generated by this backend which begins with the preamble generated by functionABI.EmitGoEntryPreamble below. +// This implements wazevo.entrypoint, and see the comments there for detail. +func entrypoint(preambleExecutable, functionExecutable *byte, executionContextPtr uintptr, moduleContextPtr *byte, paramResultPtr *uint64, goAllocatedStackSlicePtr uintptr) + +// afterGoFunctionCallEntrypoint enters the machine code after growing the stack. +// This implements wazevo.afterGoFunctionCallEntrypoint, and see the comments there for detail. +func afterGoFunctionCallEntrypoint(executable *byte, executionContextPtr uintptr, stackPointer, framePointer uintptr) diff --git a/vendor/github.com/tetratelabs/wazero/internal/engine/wazevo/backend/isa/arm64/abi_entry_arm64.s b/vendor/github.com/tetratelabs/wazero/internal/engine/wazevo/backend/isa/arm64/abi_entry_arm64.s new file mode 100644 index 000000000..0b579f852 --- /dev/null +++ b/vendor/github.com/tetratelabs/wazero/internal/engine/wazevo/backend/isa/arm64/abi_entry_arm64.s @@ -0,0 +1,29 @@ +//go:build arm64 + +#include "funcdata.h" +#include "textflag.h" + +// See the comments on EmitGoEntryPreamble for what this function is supposed to do. +TEXT ·entrypoint(SB), NOSPLIT|NOFRAME, $0-48 + MOVD preambleExecutable+0(FP), R27 + MOVD functionExectuable+8(FP), R24 + MOVD executionContextPtr+16(FP), R0 + MOVD moduleContextPtr+24(FP), R1 + MOVD paramResultSlicePtr+32(FP), R19 + MOVD goAllocatedStackSlicePtr+40(FP), R26 + JMP (R27) + +TEXT ·afterGoFunctionCallEntrypoint(SB), NOSPLIT|NOFRAME, $0-32 + MOVD goCallReturnAddress+0(FP), R20 + MOVD executionContextPtr+8(FP), R0 + MOVD stackPointer+16(FP), R19 + + // Save the current FP(R29), SP and LR(R30) into the wazevo.executionContext (stored in R0). + MOVD R29, 16(R0) // Store FP(R29) into [RO, #ExecutionContextOffsets.OriginalFramePointer] + MOVD RSP, R27 // Move SP to R27 (temporary register) since SP cannot be stored directly in str instructions. + MOVD R27, 24(R0) // Store R27 into [RO, #ExecutionContextOffsets.OriginalFramePointer] + MOVD R30, 32(R0) // Store R30 into [R0, #ExecutionContextOffsets.GoReturnAddress] + + // Load the new stack pointer (which sits somewhere in Go-allocated stack) into SP. + MOVD R19, RSP + JMP (R20) diff --git a/vendor/github.com/tetratelabs/wazero/internal/engine/wazevo/backend/isa/arm64/abi_entry_preamble.go b/vendor/github.com/tetratelabs/wazero/internal/engine/wazevo/backend/isa/arm64/abi_entry_preamble.go new file mode 100644 index 000000000..7a9cceb33 --- /dev/null +++ b/vendor/github.com/tetratelabs/wazero/internal/engine/wazevo/backend/isa/arm64/abi_entry_preamble.go @@ -0,0 +1,230 @@ +package arm64 + +import ( + "github.com/tetratelabs/wazero/internal/engine/wazevo/backend" + "github.com/tetratelabs/wazero/internal/engine/wazevo/backend/regalloc" + "github.com/tetratelabs/wazero/internal/engine/wazevo/ssa" + "github.com/tetratelabs/wazero/internal/engine/wazevo/wazevoapi" +) + +// CompileEntryPreamble implements backend.Machine. This assumes `entrypoint` function (in abi_go_entry_arm64.s) passes: +// +// 1. First (execution context ptr) and Second arguments are already passed in x0, and x1. +// 2. param/result slice ptr in x19; the pointer to []uint64{} which is used to pass arguments and accept return values. +// 3. Go-allocated stack slice ptr in x26. +// 4. Function executable in x24. +// +// also SP and FP are correct Go-runtime-based values, and LR is the return address to the Go-side caller. +func (m *machine) CompileEntryPreamble(signature *ssa.Signature) []byte { + root := m.constructEntryPreamble(signature) + m.encode(root) + return m.compiler.Buf() +} + +var ( + executionContextPtrReg = x0VReg + // callee-saved regs so that they can be used in the prologue and epilogue. + paramResultSlicePtr = x19VReg + savedExecutionContextPtr = x20VReg + // goAllocatedStackPtr is not used in the epilogue. + goAllocatedStackPtr = x26VReg + // paramResultSliceCopied is not used in the epilogue. + paramResultSliceCopied = x25VReg + // tmpRegVReg is not used in the epilogue. + functionExecutable = x24VReg +) + +func (m *machine) goEntryPreamblePassArg(cur *instruction, paramSlicePtr regalloc.VReg, arg *backend.ABIArg, argStartOffsetFromSP int64) *instruction { + typ := arg.Type + bits := typ.Bits() + isStackArg := arg.Kind == backend.ABIArgKindStack + + var loadTargetReg operand + if !isStackArg { + loadTargetReg = operandNR(arg.Reg) + } else { + switch typ { + case ssa.TypeI32, ssa.TypeI64: + loadTargetReg = operandNR(x15VReg) + case ssa.TypeF32, ssa.TypeF64, ssa.TypeV128: + loadTargetReg = operandNR(v15VReg) + default: + panic("TODO?") + } + } + + var postIndexImm int64 + if typ == ssa.TypeV128 { + postIndexImm = 16 // v128 is represented as 2x64-bit in Go slice. + } else { + postIndexImm = 8 + } + loadMode := addressMode{kind: addressModeKindPostIndex, rn: paramSlicePtr, imm: postIndexImm} + + instr := m.allocateInstr() + switch typ { + case ssa.TypeI32: + instr.asULoad(loadTargetReg, loadMode, 32) + case ssa.TypeI64: + instr.asULoad(loadTargetReg, loadMode, 64) + case ssa.TypeF32: + instr.asFpuLoad(loadTargetReg, loadMode, 32) + case ssa.TypeF64: + instr.asFpuLoad(loadTargetReg, loadMode, 64) + case ssa.TypeV128: + instr.asFpuLoad(loadTargetReg, loadMode, 128) + } + cur = linkInstr(cur, instr) + + if isStackArg { + var storeMode addressMode + cur, storeMode = m.resolveAddressModeForOffsetAndInsert(cur, argStartOffsetFromSP+arg.Offset, bits, spVReg, true) + toStack := m.allocateInstr() + toStack.asStore(loadTargetReg, storeMode, bits) + cur = linkInstr(cur, toStack) + } + return cur +} + +func (m *machine) goEntryPreamblePassResult(cur *instruction, resultSlicePtr regalloc.VReg, result *backend.ABIArg, resultStartOffsetFromSP int64) *instruction { + isStackArg := result.Kind == backend.ABIArgKindStack + typ := result.Type + bits := typ.Bits() + + var storeTargetReg operand + if !isStackArg { + storeTargetReg = operandNR(result.Reg) + } else { + switch typ { + case ssa.TypeI32, ssa.TypeI64: + storeTargetReg = operandNR(x15VReg) + case ssa.TypeF32, ssa.TypeF64, ssa.TypeV128: + storeTargetReg = operandNR(v15VReg) + default: + panic("TODO?") + } + } + + var postIndexImm int64 + if typ == ssa.TypeV128 { + postIndexImm = 16 // v128 is represented as 2x64-bit in Go slice. + } else { + postIndexImm = 8 + } + + if isStackArg { + var loadMode addressMode + cur, loadMode = m.resolveAddressModeForOffsetAndInsert(cur, resultStartOffsetFromSP+result.Offset, bits, spVReg, true) + toReg := m.allocateInstr() + switch typ { + case ssa.TypeI32, ssa.TypeI64: + toReg.asULoad(storeTargetReg, loadMode, bits) + case ssa.TypeF32, ssa.TypeF64, ssa.TypeV128: + toReg.asFpuLoad(storeTargetReg, loadMode, bits) + default: + panic("TODO?") + } + cur = linkInstr(cur, toReg) + } + + mode := addressMode{kind: addressModeKindPostIndex, rn: resultSlicePtr, imm: postIndexImm} + instr := m.allocateInstr() + instr.asStore(storeTargetReg, mode, bits) + cur = linkInstr(cur, instr) + return cur +} + +func (m *machine) constructEntryPreamble(sig *ssa.Signature) (root *instruction) { + abi := backend.FunctionABI{} + abi.Init(sig, intParamResultRegs, floatParamResultRegs) + + root = m.allocateNop() + + //// ----------------------------------- prologue ----------------------------------- //// + + // First, we save executionContextPtrReg into a callee-saved register so that it can be used in epilogue as well. + // mov savedExecutionContextPtr, x0 + cur := m.move64(savedExecutionContextPtr, executionContextPtrReg, root) + + // Next, save the current FP, SP and LR into the wazevo.executionContext: + // str fp, [savedExecutionContextPtr, #OriginalFramePointer] + // mov tmp, sp ;; sp cannot be str'ed directly. + // str sp, [savedExecutionContextPtr, #OriginalStackPointer] + // str lr, [savedExecutionContextPtr, #GoReturnAddress] + cur = m.loadOrStoreAtExecutionContext(fpVReg, wazevoapi.ExecutionContextOffsetOriginalFramePointer, true, cur) + cur = m.move64(tmpRegVReg, spVReg, cur) + cur = m.loadOrStoreAtExecutionContext(tmpRegVReg, wazevoapi.ExecutionContextOffsetOriginalStackPointer, true, cur) + cur = m.loadOrStoreAtExecutionContext(lrVReg, wazevoapi.ExecutionContextOffsetGoReturnAddress, true, cur) + + // Then, move the Go-allocated stack pointer to SP: + // mov sp, goAllocatedStackPtr + cur = m.move64(spVReg, goAllocatedStackPtr, cur) + + prReg := paramResultSlicePtr + if len(abi.Args) > 2 && len(abi.Rets) > 0 { + // paramResultSlicePtr is modified during the execution of goEntryPreamblePassArg, + // so copy it to another reg. + cur = m.move64(paramResultSliceCopied, paramResultSlicePtr, cur) + prReg = paramResultSliceCopied + } + + stackSlotSize := int64(abi.AlignedArgResultStackSlotSize()) + for i := range abi.Args { + if i < 2 { + // module context ptr and execution context ptr are passed in x0 and x1 by the Go assembly function. + continue + } + arg := &abi.Args[i] + cur = m.goEntryPreamblePassArg(cur, prReg, arg, -stackSlotSize) + } + + // Call the real function. + bl := m.allocateInstr() + bl.asCallIndirect(functionExecutable, &abi) + cur = linkInstr(cur, bl) + + ///// ----------------------------------- epilogue ----------------------------------- ///// + + // Store the register results into paramResultSlicePtr. + for i := range abi.Rets { + cur = m.goEntryPreamblePassResult(cur, paramResultSlicePtr, &abi.Rets[i], abi.ArgStackSize-stackSlotSize) + } + + // Finally, restore the FP, SP and LR, and return to the Go code. + // ldr fp, [savedExecutionContextPtr, #OriginalFramePointer] + // ldr tmp, [savedExecutionContextPtr, #OriginalStackPointer] + // mov sp, tmp ;; sp cannot be str'ed directly. + // ldr lr, [savedExecutionContextPtr, #GoReturnAddress] + // ret ;; --> return to the Go code + cur = m.loadOrStoreAtExecutionContext(fpVReg, wazevoapi.ExecutionContextOffsetOriginalFramePointer, false, cur) + cur = m.loadOrStoreAtExecutionContext(tmpRegVReg, wazevoapi.ExecutionContextOffsetOriginalStackPointer, false, cur) + cur = m.move64(spVReg, tmpRegVReg, cur) + cur = m.loadOrStoreAtExecutionContext(lrVReg, wazevoapi.ExecutionContextOffsetGoReturnAddress, false, cur) + retInst := m.allocateInstr() + retInst.asRet() + linkInstr(cur, retInst) + return +} + +func (m *machine) move64(dst, src regalloc.VReg, prev *instruction) *instruction { + instr := m.allocateInstr() + instr.asMove64(dst, src) + return linkInstr(prev, instr) +} + +func (m *machine) loadOrStoreAtExecutionContext(d regalloc.VReg, offset wazevoapi.Offset, store bool, prev *instruction) *instruction { + instr := m.allocateInstr() + mode := addressMode{kind: addressModeKindRegUnsignedImm12, rn: savedExecutionContextPtr, imm: offset.I64()} + if store { + instr.asStore(operandNR(d), mode, 64) + } else { + instr.asULoad(operandNR(d), mode, 64) + } + return linkInstr(prev, instr) +} + +func linkInstr(prev, next *instruction) *instruction { + prev.next = next + next.prev = prev + return next +} diff --git a/vendor/github.com/tetratelabs/wazero/internal/engine/wazevo/backend/isa/arm64/abi_go_call.go b/vendor/github.com/tetratelabs/wazero/internal/engine/wazevo/backend/isa/arm64/abi_go_call.go new file mode 100644 index 000000000..466b1f960 --- /dev/null +++ b/vendor/github.com/tetratelabs/wazero/internal/engine/wazevo/backend/isa/arm64/abi_go_call.go @@ -0,0 +1,428 @@ +package arm64 + +import ( + "github.com/tetratelabs/wazero/internal/engine/wazevo/backend" + "github.com/tetratelabs/wazero/internal/engine/wazevo/backend/regalloc" + "github.com/tetratelabs/wazero/internal/engine/wazevo/ssa" + "github.com/tetratelabs/wazero/internal/engine/wazevo/wazevoapi" +) + +var calleeSavedRegistersSorted = []regalloc.VReg{ + x19VReg, x20VReg, x21VReg, x22VReg, x23VReg, x24VReg, x25VReg, x26VReg, x28VReg, + v18VReg, v19VReg, v20VReg, v21VReg, v22VReg, v23VReg, v24VReg, v25VReg, v26VReg, v27VReg, v28VReg, v29VReg, v30VReg, v31VReg, +} + +// CompileGoFunctionTrampoline implements backend.Machine. +func (m *machine) CompileGoFunctionTrampoline(exitCode wazevoapi.ExitCode, sig *ssa.Signature, needModuleContextPtr bool) []byte { + exct := m.executableContext + argBegin := 1 // Skips exec context by default. + if needModuleContextPtr { + argBegin++ + } + + abi := &backend.FunctionABI{} + abi.Init(sig, intParamResultRegs, floatParamResultRegs) + m.currentABI = abi + + cur := m.allocateInstr() + cur.asNop0() + exct.RootInstr = cur + + // Execution context is always the first argument. + execCtrPtr := x0VReg + + // In the following, we create the following stack layout: + // + // (high address) + // SP ------> +-----------------+ <----+ + // | ....... | | + // | ret Y | | + // | ....... | | + // | ret 0 | | + // | arg X | | size_of_arg_ret + // | ....... | | + // | arg 1 | | + // | arg 0 | <----+ <-------- originalArg0Reg + // | size_of_arg_ret | + // | ReturnAddress | + // +-----------------+ <----+ + // | xxxx | | ;; might be padded to make it 16-byte aligned. + // +--->| arg[N]/ret[M] | | + // sliceSize| | ............ | | goCallStackSize + // | | arg[1]/ret[1] | | + // +--->| arg[0]/ret[0] | <----+ <-------- arg0ret0AddrReg + // | sliceSize | + // | frame_size | + // +-----------------+ + // (low address) + // + // where the region of "arg[0]/ret[0] ... arg[N]/ret[M]" is the stack used by the Go functions, + // therefore will be accessed as the usual []uint64. So that's where we need to pass/receive + // the arguments/return values. + + // First of all, to update the SP, and create "ReturnAddress + size_of_arg_ret". + cur = m.createReturnAddrAndSizeOfArgRetSlot(cur) + + const frameInfoSize = 16 // == frame_size + sliceSize. + + // Next, we should allocate the stack for the Go function call if necessary. + goCallStackSize, sliceSizeInBytes := backend.GoFunctionCallRequiredStackSize(sig, argBegin) + cur = m.insertStackBoundsCheck(goCallStackSize+frameInfoSize, cur) + + originalArg0Reg := x17VReg // Caller save, so we can use it for whatever we want. + if m.currentABI.AlignedArgResultStackSlotSize() > 0 { + // At this point, SP points to `ReturnAddress`, so add 16 to get the original arg 0 slot. + cur = m.addsAddOrSubStackPointer(cur, originalArg0Reg, frameInfoSize, true) + } + + // Save the callee saved registers. + cur = m.saveRegistersInExecutionContext(cur, calleeSavedRegistersSorted) + + if needModuleContextPtr { + offset := wazevoapi.ExecutionContextOffsetGoFunctionCallCalleeModuleContextOpaque.I64() + if !offsetFitsInAddressModeKindRegUnsignedImm12(64, offset) { + panic("BUG: too large or un-aligned offset for goFunctionCallCalleeModuleContextOpaque in execution context") + } + + // Module context is always the second argument. + moduleCtrPtr := x1VReg + store := m.allocateInstr() + amode := addressMode{kind: addressModeKindRegUnsignedImm12, rn: execCtrPtr, imm: offset} + store.asStore(operandNR(moduleCtrPtr), amode, 64) + cur = linkInstr(cur, store) + } + + // Advances the stack pointer. + cur = m.addsAddOrSubStackPointer(cur, spVReg, goCallStackSize, false) + + // Copy the pointer to x15VReg. + arg0ret0AddrReg := x15VReg // Caller save, so we can use it for whatever we want. + copySp := m.allocateInstr() + copySp.asMove64(arg0ret0AddrReg, spVReg) + cur = linkInstr(cur, copySp) + + // Next, we need to store all the arguments to the stack in the typical Wasm stack style. + for i := range abi.Args[argBegin:] { + arg := &abi.Args[argBegin+i] + store := m.allocateInstr() + var v regalloc.VReg + if arg.Kind == backend.ABIArgKindReg { + v = arg.Reg + } else { + cur, v = m.goFunctionCallLoadStackArg(cur, originalArg0Reg, arg, + // Caller save, so we can use it for whatever we want. + x11VReg, v11VReg) + } + + var sizeInBits byte + if arg.Type == ssa.TypeV128 { + sizeInBits = 128 + } else { + sizeInBits = 64 + } + store.asStore(operandNR(v), + addressMode{ + kind: addressModeKindPostIndex, + rn: arg0ret0AddrReg, imm: int64(sizeInBits / 8), + }, sizeInBits) + cur = linkInstr(cur, store) + } + + // Finally, now that we've advanced SP to arg[0]/ret[0], we allocate `frame_size + sliceSize`. + var frameSizeReg, sliceSizeReg regalloc.VReg + if goCallStackSize > 0 { + cur = m.lowerConstantI64AndInsert(cur, tmpRegVReg, goCallStackSize) + frameSizeReg = tmpRegVReg + cur = m.lowerConstantI64AndInsert(cur, x16VReg, sliceSizeInBytes/8) + sliceSizeReg = x16VReg + } else { + frameSizeReg = xzrVReg + sliceSizeReg = xzrVReg + } + _amode := addressModePreOrPostIndex(spVReg, -16, true) + storeP := m.allocateInstr() + storeP.asStorePair64(frameSizeReg, sliceSizeReg, _amode) + cur = linkInstr(cur, storeP) + + // Set the exit status on the execution context. + cur = m.setExitCode(cur, x0VReg, exitCode) + + // Save the current stack pointer. + cur = m.saveCurrentStackPointer(cur, x0VReg) + + // Exit the execution. + cur = m.storeReturnAddressAndExit(cur) + + // After the call, we need to restore the callee saved registers. + cur = m.restoreRegistersInExecutionContext(cur, calleeSavedRegistersSorted) + + // Get the pointer to the arg[0]/ret[0]: We need to skip `frame_size + sliceSize`. + if len(abi.Rets) > 0 { + cur = m.addsAddOrSubStackPointer(cur, arg0ret0AddrReg, frameInfoSize, true) + } + + // Advances the SP so that it points to `ReturnAddress`. + cur = m.addsAddOrSubStackPointer(cur, spVReg, frameInfoSize+goCallStackSize, true) + ldr := m.allocateInstr() + // And load the return address. + ldr.asULoad(operandNR(lrVReg), + addressModePreOrPostIndex(spVReg, 16 /* stack pointer must be 16-byte aligned. */, false /* increment after loads */), 64) + cur = linkInstr(cur, ldr) + + originalRet0Reg := x17VReg // Caller save, so we can use it for whatever we want. + if m.currentABI.RetStackSize > 0 { + cur = m.addsAddOrSubStackPointer(cur, originalRet0Reg, m.currentABI.ArgStackSize, true) + } + + // Make the SP point to the original address (above the result slot). + if s := int64(m.currentABI.AlignedArgResultStackSlotSize()); s > 0 { + cur = m.addsAddOrSubStackPointer(cur, spVReg, s, true) + } + + for i := range abi.Rets { + r := &abi.Rets[i] + if r.Kind == backend.ABIArgKindReg { + loadIntoReg := m.allocateInstr() + mode := addressMode{kind: addressModeKindPostIndex, rn: arg0ret0AddrReg} + switch r.Type { + case ssa.TypeI32: + mode.imm = 8 // We use uint64 for all basic types, except SIMD v128. + loadIntoReg.asULoad(operandNR(r.Reg), mode, 32) + case ssa.TypeI64: + mode.imm = 8 // We use uint64 for all basic types, except SIMD v128. + loadIntoReg.asULoad(operandNR(r.Reg), mode, 64) + case ssa.TypeF32: + mode.imm = 8 // We use uint64 for all basic types, except SIMD v128. + loadIntoReg.asFpuLoad(operandNR(r.Reg), mode, 32) + case ssa.TypeF64: + mode.imm = 8 // We use uint64 for all basic types, except SIMD v128. + loadIntoReg.asFpuLoad(operandNR(r.Reg), mode, 64) + case ssa.TypeV128: + mode.imm = 16 + loadIntoReg.asFpuLoad(operandNR(r.Reg), mode, 128) + default: + panic("TODO") + } + cur = linkInstr(cur, loadIntoReg) + } else { + // First we need to load the value to a temporary just like ^^. + intTmp, floatTmp := x11VReg, v11VReg + loadIntoTmpReg := m.allocateInstr() + mode := addressMode{kind: addressModeKindPostIndex, rn: arg0ret0AddrReg} + var resultReg regalloc.VReg + switch r.Type { + case ssa.TypeI32: + mode.imm = 8 // We use uint64 for all basic types, except SIMD v128. + loadIntoTmpReg.asULoad(operandNR(intTmp), mode, 32) + resultReg = intTmp + case ssa.TypeI64: + mode.imm = 8 // We use uint64 for all basic types, except SIMD v128. + loadIntoTmpReg.asULoad(operandNR(intTmp), mode, 64) + resultReg = intTmp + case ssa.TypeF32: + mode.imm = 8 // We use uint64 for all basic types, except SIMD v128. + loadIntoTmpReg.asFpuLoad(operandNR(floatTmp), mode, 32) + resultReg = floatTmp + case ssa.TypeF64: + mode.imm = 8 // We use uint64 for all basic types, except SIMD v128. + loadIntoTmpReg.asFpuLoad(operandNR(floatTmp), mode, 64) + resultReg = floatTmp + case ssa.TypeV128: + mode.imm = 16 + loadIntoTmpReg.asFpuLoad(operandNR(floatTmp), mode, 128) + resultReg = floatTmp + default: + panic("TODO") + } + cur = linkInstr(cur, loadIntoTmpReg) + cur = m.goFunctionCallStoreStackResult(cur, originalRet0Reg, r, resultReg) + } + } + + ret := m.allocateInstr() + ret.asRet() + linkInstr(cur, ret) + + m.encode(m.executableContext.RootInstr) + return m.compiler.Buf() +} + +func (m *machine) saveRegistersInExecutionContext(cur *instruction, regs []regalloc.VReg) *instruction { + offset := wazevoapi.ExecutionContextOffsetSavedRegistersBegin.I64() + for _, v := range regs { + store := m.allocateInstr() + var sizeInBits byte + switch v.RegType() { + case regalloc.RegTypeInt: + sizeInBits = 64 + case regalloc.RegTypeFloat: + sizeInBits = 128 + } + store.asStore(operandNR(v), + addressMode{ + kind: addressModeKindRegUnsignedImm12, + // Execution context is always the first argument. + rn: x0VReg, imm: offset, + }, sizeInBits) + store.prev = cur + cur.next = store + cur = store + offset += 16 // Imm12 must be aligned 16 for vector regs, so we unconditionally store regs at the offset of multiple of 16. + } + return cur +} + +func (m *machine) restoreRegistersInExecutionContext(cur *instruction, regs []regalloc.VReg) *instruction { + offset := wazevoapi.ExecutionContextOffsetSavedRegistersBegin.I64() + for _, v := range regs { + load := m.allocateInstr() + var as func(dst operand, amode addressMode, sizeInBits byte) + var sizeInBits byte + switch v.RegType() { + case regalloc.RegTypeInt: + as = load.asULoad + sizeInBits = 64 + case regalloc.RegTypeFloat: + as = load.asFpuLoad + sizeInBits = 128 + } + as(operandNR(v), + addressMode{ + kind: addressModeKindRegUnsignedImm12, + // Execution context is always the first argument. + rn: x0VReg, imm: offset, + }, sizeInBits) + cur = linkInstr(cur, load) + offset += 16 // Imm12 must be aligned 16 for vector regs, so we unconditionally load regs at the offset of multiple of 16. + } + return cur +} + +func (m *machine) lowerConstantI64AndInsert(cur *instruction, dst regalloc.VReg, v int64) *instruction { + exct := m.executableContext + exct.PendingInstructions = exct.PendingInstructions[:0] + m.lowerConstantI64(dst, v) + for _, instr := range exct.PendingInstructions { + cur = linkInstr(cur, instr) + } + return cur +} + +func (m *machine) lowerConstantI32AndInsert(cur *instruction, dst regalloc.VReg, v int32) *instruction { + exct := m.executableContext + exct.PendingInstructions = exct.PendingInstructions[:0] + m.lowerConstantI32(dst, v) + for _, instr := range exct.PendingInstructions { + cur = linkInstr(cur, instr) + } + return cur +} + +func (m *machine) setExitCode(cur *instruction, execCtr regalloc.VReg, exitCode wazevoapi.ExitCode) *instruction { + constReg := x17VReg // caller-saved, so we can use it. + cur = m.lowerConstantI32AndInsert(cur, constReg, int32(exitCode)) + + // Set the exit status on the execution context. + setExistStatus := m.allocateInstr() + setExistStatus.asStore(operandNR(constReg), + addressMode{ + kind: addressModeKindRegUnsignedImm12, + rn: execCtr, imm: wazevoapi.ExecutionContextOffsetExitCodeOffset.I64(), + }, 32) + cur = linkInstr(cur, setExistStatus) + return cur +} + +func (m *machine) storeReturnAddressAndExit(cur *instruction) *instruction { + // Read the return address into tmp, and store it in the execution context. + adr := m.allocateInstr() + adr.asAdr(tmpRegVReg, exitSequenceSize+8) + cur = linkInstr(cur, adr) + + storeReturnAddr := m.allocateInstr() + storeReturnAddr.asStore(operandNR(tmpRegVReg), + addressMode{ + kind: addressModeKindRegUnsignedImm12, + // Execution context is always the first argument. + rn: x0VReg, imm: wazevoapi.ExecutionContextOffsetGoCallReturnAddress.I64(), + }, 64) + cur = linkInstr(cur, storeReturnAddr) + + // Exit the execution. + trapSeq := m.allocateInstr() + trapSeq.asExitSequence(x0VReg) + cur = linkInstr(cur, trapSeq) + return cur +} + +func (m *machine) saveCurrentStackPointer(cur *instruction, execCtr regalloc.VReg) *instruction { + // Save the current stack pointer: + // mov tmp, sp, + // str tmp, [exec_ctx, #stackPointerBeforeGoCall] + movSp := m.allocateInstr() + movSp.asMove64(tmpRegVReg, spVReg) + cur = linkInstr(cur, movSp) + + strSp := m.allocateInstr() + strSp.asStore(operandNR(tmpRegVReg), + addressMode{ + kind: addressModeKindRegUnsignedImm12, + rn: execCtr, imm: wazevoapi.ExecutionContextOffsetStackPointerBeforeGoCall.I64(), + }, 64) + cur = linkInstr(cur, strSp) + return cur +} + +func (m *machine) goFunctionCallLoadStackArg(cur *instruction, originalArg0Reg regalloc.VReg, arg *backend.ABIArg, intVReg, floatVReg regalloc.VReg) (*instruction, regalloc.VReg) { + load := m.allocateInstr() + var result regalloc.VReg + mode := addressMode{kind: addressModeKindPostIndex, rn: originalArg0Reg} + switch arg.Type { + case ssa.TypeI32: + mode.imm = 8 // We use uint64 for all basic types, except SIMD v128. + load.asULoad(operandNR(intVReg), mode, 32) + result = intVReg + case ssa.TypeI64: + mode.imm = 8 // We use uint64 for all basic types, except SIMD v128. + load.asULoad(operandNR(intVReg), mode, 64) + result = intVReg + case ssa.TypeF32: + mode.imm = 8 // We use uint64 for all basic types, except SIMD v128. + load.asFpuLoad(operandNR(floatVReg), mode, 32) + result = floatVReg + case ssa.TypeF64: + mode.imm = 8 // We use uint64 for all basic types, except SIMD v128. + load.asFpuLoad(operandNR(floatVReg), mode, 64) + result = floatVReg + case ssa.TypeV128: + mode.imm = 16 + load.asFpuLoad(operandNR(floatVReg), mode, 128) + result = floatVReg + default: + panic("TODO") + } + + cur = linkInstr(cur, load) + return cur, result +} + +func (m *machine) goFunctionCallStoreStackResult(cur *instruction, originalRet0Reg regalloc.VReg, result *backend.ABIArg, resultVReg regalloc.VReg) *instruction { + store := m.allocateInstr() + mode := addressMode{kind: addressModeKindPostIndex, rn: originalRet0Reg} + var sizeInBits byte + switch result.Type { + case ssa.TypeI32, ssa.TypeF32: + mode.imm = 8 + sizeInBits = 32 + case ssa.TypeI64, ssa.TypeF64: + mode.imm = 8 + sizeInBits = 64 + case ssa.TypeV128: + mode.imm = 16 + sizeInBits = 128 + default: + panic("TODO") + } + store.asStore(operandNR(resultVReg), mode, sizeInBits) + return linkInstr(cur, store) +} diff --git a/vendor/github.com/tetratelabs/wazero/internal/engine/wazevo/backend/isa/arm64/cond.go b/vendor/github.com/tetratelabs/wazero/internal/engine/wazevo/backend/isa/arm64/cond.go new file mode 100644 index 000000000..6f6cdd1b2 --- /dev/null +++ b/vendor/github.com/tetratelabs/wazero/internal/engine/wazevo/backend/isa/arm64/cond.go @@ -0,0 +1,215 @@ +package arm64 + +import ( + "strconv" + + "github.com/tetratelabs/wazero/internal/engine/wazevo/backend/regalloc" + "github.com/tetratelabs/wazero/internal/engine/wazevo/ssa" +) + +type ( + cond uint64 + condKind byte +) + +const ( + // condKindRegisterZero represents a condition which checks if the register is zero. + // This indicates that the instruction must be encoded as CBZ: + // https://developer.arm.com/documentation/ddi0596/2020-12/Base-Instructions/CBZ--Compare-and-Branch-on-Zero- + condKindRegisterZero condKind = iota + // condKindRegisterNotZero indicates that the instruction must be encoded as CBNZ: + // https://developer.arm.com/documentation/ddi0596/2020-12/Base-Instructions/CBNZ--Compare-and-Branch-on-Nonzero- + condKindRegisterNotZero + // condKindCondFlagSet indicates that the instruction must be encoded as B.cond: + // https://developer.arm.com/documentation/ddi0596/2020-12/Base-Instructions/B-cond--Branch-conditionally- + condKindCondFlagSet +) + +// kind returns the kind of condition which is stored in the first two bits. +func (c cond) kind() condKind { + return condKind(c & 0b11) +} + +func (c cond) asUint64() uint64 { + return uint64(c) +} + +// register returns the register for register conditions. +// This panics if the condition is not a register condition (condKindRegisterZero or condKindRegisterNotZero). +func (c cond) register() regalloc.VReg { + if c.kind() != condKindRegisterZero && c.kind() != condKindRegisterNotZero { + panic("condition is not a register") + } + return regalloc.VReg(c >> 2) +} + +func registerAsRegZeroCond(r regalloc.VReg) cond { + return cond(r)<<2 | cond(condKindRegisterZero) +} + +func registerAsRegNotZeroCond(r regalloc.VReg) cond { + return cond(r)<<2 | cond(condKindRegisterNotZero) +} + +func (c cond) flag() condFlag { + if c.kind() != condKindCondFlagSet { + panic("condition is not a flag") + } + return condFlag(c >> 2) +} + +func (c condFlag) asCond() cond { + return cond(c)<<2 | cond(condKindCondFlagSet) +} + +// condFlag represents a condition flag for conditional branches. +// The value matches the encoding of condition flags in the ARM64 instruction set. +// https://developer.arm.com/documentation/den0024/a/The-A64-instruction-set/Data-processing-instructions/Conditional-instructions +type condFlag uint8 + +const ( + eq condFlag = iota // eq represents "equal" + ne // ne represents "not equal" + hs // hs represents "higher or same" + lo // lo represents "lower" + mi // mi represents "minus or negative result" + pl // pl represents "plus or positive result" + vs // vs represents "overflow set" + vc // vc represents "overflow clear" + hi // hi represents "higher" + ls // ls represents "lower or same" + ge // ge represents "greater or equal" + lt // lt represents "less than" + gt // gt represents "greater than" + le // le represents "less than or equal" + al // al represents "always" + nv // nv represents "never" +) + +// invert returns the inverted condition. +func (c condFlag) invert() condFlag { + switch c { + case eq: + return ne + case ne: + return eq + case hs: + return lo + case lo: + return hs + case mi: + return pl + case pl: + return mi + case vs: + return vc + case vc: + return vs + case hi: + return ls + case ls: + return hi + case ge: + return lt + case lt: + return ge + case gt: + return le + case le: + return gt + case al: + return nv + case nv: + return al + default: + panic(c) + } +} + +// String implements fmt.Stringer. +func (c condFlag) String() string { + switch c { + case eq: + return "eq" + case ne: + return "ne" + case hs: + return "hs" + case lo: + return "lo" + case mi: + return "mi" + case pl: + return "pl" + case vs: + return "vs" + case vc: + return "vc" + case hi: + return "hi" + case ls: + return "ls" + case ge: + return "ge" + case lt: + return "lt" + case gt: + return "gt" + case le: + return "le" + case al: + return "al" + case nv: + return "nv" + default: + panic(strconv.Itoa(int(c))) + } +} + +// condFlagFromSSAIntegerCmpCond returns the condition flag for the given ssa.IntegerCmpCond. +func condFlagFromSSAIntegerCmpCond(c ssa.IntegerCmpCond) condFlag { + switch c { + case ssa.IntegerCmpCondEqual: + return eq + case ssa.IntegerCmpCondNotEqual: + return ne + case ssa.IntegerCmpCondSignedLessThan: + return lt + case ssa.IntegerCmpCondSignedGreaterThanOrEqual: + return ge + case ssa.IntegerCmpCondSignedGreaterThan: + return gt + case ssa.IntegerCmpCondSignedLessThanOrEqual: + return le + case ssa.IntegerCmpCondUnsignedLessThan: + return lo + case ssa.IntegerCmpCondUnsignedGreaterThanOrEqual: + return hs + case ssa.IntegerCmpCondUnsignedGreaterThan: + return hi + case ssa.IntegerCmpCondUnsignedLessThanOrEqual: + return ls + default: + panic(c) + } +} + +// condFlagFromSSAFloatCmpCond returns the condition flag for the given ssa.FloatCmpCond. +func condFlagFromSSAFloatCmpCond(c ssa.FloatCmpCond) condFlag { + switch c { + case ssa.FloatCmpCondEqual: + return eq + case ssa.FloatCmpCondNotEqual: + return ne + case ssa.FloatCmpCondLessThan: + return mi + case ssa.FloatCmpCondLessThanOrEqual: + return ls + case ssa.FloatCmpCondGreaterThan: + return gt + case ssa.FloatCmpCondGreaterThanOrEqual: + return ge + default: + panic(c) + } +} diff --git a/vendor/github.com/tetratelabs/wazero/internal/engine/wazevo/backend/isa/arm64/instr.go b/vendor/github.com/tetratelabs/wazero/internal/engine/wazevo/backend/isa/arm64/instr.go new file mode 100644 index 000000000..8aabc5997 --- /dev/null +++ b/vendor/github.com/tetratelabs/wazero/internal/engine/wazevo/backend/isa/arm64/instr.go @@ -0,0 +1,2545 @@ +package arm64 + +import ( + "fmt" + "math" + + "github.com/tetratelabs/wazero/internal/engine/wazevo/backend" + "github.com/tetratelabs/wazero/internal/engine/wazevo/backend/regalloc" + "github.com/tetratelabs/wazero/internal/engine/wazevo/ssa" +) + +type ( + // instruction represents either a real instruction in arm64, or the meta instructions + // that are convenient for code generation. For example, inline constants are also treated + // as instructions. + // + // Basically, each instruction knows how to get encoded in binaries. Hence, the final output of compilation + // can be considered equivalent to the sequence of such instructions. + // + // Each field is interpreted depending on the kind. + // + // TODO: optimize the layout later once the impl settles. + instruction struct { + prev, next *instruction + u1, u2, u3 uint64 + rd, rm, rn, ra operand + amode addressMode + kind instructionKind + addedBeforeRegAlloc bool + } + + // instructionKind represents the kind of instruction. + // This controls how the instruction struct is interpreted. + instructionKind byte +) + +func asNop0(i *instruction) { + i.kind = nop0 +} + +func setNext(i, next *instruction) { + i.next = next +} + +func setPrev(i, prev *instruction) { + i.prev = prev +} + +// IsCall implements regalloc.Instr IsCall. +func (i *instruction) IsCall() bool { + return i.kind == call +} + +// IsIndirectCall implements regalloc.Instr IsIndirectCall. +func (i *instruction) IsIndirectCall() bool { + return i.kind == callInd +} + +// IsReturn implements regalloc.Instr IsReturn. +func (i *instruction) IsReturn() bool { + return i.kind == ret +} + +// Next implements regalloc.Instr Next. +func (i *instruction) Next() regalloc.Instr { + return i.next +} + +// Prev implements regalloc.Instr Prev. +func (i *instruction) Prev() regalloc.Instr { + return i.prev +} + +// AddedBeforeRegAlloc implements regalloc.Instr AddedBeforeRegAlloc. +func (i *instruction) AddedBeforeRegAlloc() bool { + return i.addedBeforeRegAlloc +} + +type defKind byte + +const ( + defKindNone defKind = iota + 1 + defKindRD + defKindCall +) + +var defKinds = [numInstructionKinds]defKind{ + adr: defKindRD, + aluRRR: defKindRD, + aluRRRR: defKindRD, + aluRRImm12: defKindRD, + aluRRBitmaskImm: defKindRD, + aluRRRShift: defKindRD, + aluRRImmShift: defKindRD, + aluRRRExtend: defKindRD, + bitRR: defKindRD, + movZ: defKindRD, + movK: defKindRD, + movN: defKindRD, + mov32: defKindRD, + mov64: defKindRD, + fpuMov64: defKindRD, + fpuMov128: defKindRD, + fpuRR: defKindRD, + fpuRRR: defKindRD, + nop0: defKindNone, + call: defKindCall, + callInd: defKindCall, + ret: defKindNone, + store8: defKindNone, + store16: defKindNone, + store32: defKindNone, + store64: defKindNone, + exitSequence: defKindNone, + condBr: defKindNone, + br: defKindNone, + brTableSequence: defKindNone, + cSet: defKindRD, + extend: defKindRD, + fpuCmp: defKindNone, + uLoad8: defKindRD, + uLoad16: defKindRD, + uLoad32: defKindRD, + sLoad8: defKindRD, + sLoad16: defKindRD, + sLoad32: defKindRD, + uLoad64: defKindRD, + fpuLoad32: defKindRD, + fpuLoad64: defKindRD, + fpuLoad128: defKindRD, + vecLoad1R: defKindRD, + loadFpuConst32: defKindRD, + loadFpuConst64: defKindRD, + loadFpuConst128: defKindRD, + fpuStore32: defKindNone, + fpuStore64: defKindNone, + fpuStore128: defKindNone, + udf: defKindNone, + cSel: defKindRD, + fpuCSel: defKindRD, + movToVec: defKindRD, + movFromVec: defKindRD, + movFromVecSigned: defKindRD, + vecDup: defKindRD, + vecDupElement: defKindRD, + vecExtract: defKindRD, + vecMisc: defKindRD, + vecMovElement: defKindRD, + vecLanes: defKindRD, + vecShiftImm: defKindRD, + vecTbl: defKindRD, + vecTbl2: defKindRD, + vecPermute: defKindRD, + vecRRR: defKindRD, + vecRRRRewrite: defKindNone, + fpuToInt: defKindRD, + intToFpu: defKindRD, + cCmpImm: defKindNone, + movToFPSR: defKindNone, + movFromFPSR: defKindRD, + emitSourceOffsetInfo: defKindNone, + atomicRmw: defKindRD, + atomicCas: defKindNone, + atomicLoad: defKindRD, + atomicStore: defKindNone, + dmb: defKindNone, + loadConstBlockArg: defKindRD, +} + +// Defs returns the list of regalloc.VReg that are defined by the instruction. +// In order to reduce the number of allocations, the caller can pass the slice to be used. +func (i *instruction) Defs(regs *[]regalloc.VReg) []regalloc.VReg { + *regs = (*regs)[:0] + switch defKinds[i.kind] { + case defKindNone: + case defKindRD: + *regs = append(*regs, i.rd.nr()) + case defKindCall: + _, _, retIntRealRegs, retFloatRealRegs, _ := backend.ABIInfoFromUint64(i.u2) + for i := byte(0); i < retIntRealRegs; i++ { + *regs = append(*regs, regInfo.RealRegToVReg[intParamResultRegs[i]]) + } + for i := byte(0); i < retFloatRealRegs; i++ { + *regs = append(*regs, regInfo.RealRegToVReg[floatParamResultRegs[i]]) + } + default: + panic(fmt.Sprintf("defKind for %v not defined", i)) + } + return *regs +} + +// AssignDef implements regalloc.Instr AssignDef. +func (i *instruction) AssignDef(reg regalloc.VReg) { + switch defKinds[i.kind] { + case defKindNone: + case defKindRD: + i.rd = i.rd.assignReg(reg) + case defKindCall: + panic("BUG: call instructions shouldn't be assigned") + default: + panic(fmt.Sprintf("defKind for %v not defined", i)) + } +} + +type useKind byte + +const ( + useKindNone useKind = iota + 1 + useKindRN + useKindRNRM + useKindRNRMRA + useKindRNRN1RM + useKindCall + useKindCallInd + useKindAMode + useKindRNAMode + useKindCond + // useKindRDRewrite indicates an instruction where RD is used both as a source and destination. + // A temporary register for RD must be allocated explicitly with the source copied to this + // register before the instruction and the value copied from this register to the instruction + // return register. + useKindRDRewrite +) + +var useKinds = [numInstructionKinds]useKind{ + udf: useKindNone, + aluRRR: useKindRNRM, + aluRRRR: useKindRNRMRA, + aluRRImm12: useKindRN, + aluRRBitmaskImm: useKindRN, + aluRRRShift: useKindRNRM, + aluRRImmShift: useKindRN, + aluRRRExtend: useKindRNRM, + bitRR: useKindRN, + movZ: useKindNone, + movK: useKindNone, + movN: useKindNone, + mov32: useKindRN, + mov64: useKindRN, + fpuMov64: useKindRN, + fpuMov128: useKindRN, + fpuRR: useKindRN, + fpuRRR: useKindRNRM, + nop0: useKindNone, + call: useKindCall, + callInd: useKindCallInd, + ret: useKindNone, + store8: useKindRNAMode, + store16: useKindRNAMode, + store32: useKindRNAMode, + store64: useKindRNAMode, + exitSequence: useKindRN, + condBr: useKindCond, + br: useKindNone, + brTableSequence: useKindRN, + cSet: useKindNone, + extend: useKindRN, + fpuCmp: useKindRNRM, + uLoad8: useKindAMode, + uLoad16: useKindAMode, + uLoad32: useKindAMode, + sLoad8: useKindAMode, + sLoad16: useKindAMode, + sLoad32: useKindAMode, + uLoad64: useKindAMode, + fpuLoad32: useKindAMode, + fpuLoad64: useKindAMode, + fpuLoad128: useKindAMode, + fpuStore32: useKindRNAMode, + fpuStore64: useKindRNAMode, + fpuStore128: useKindRNAMode, + loadFpuConst32: useKindNone, + loadFpuConst64: useKindNone, + loadFpuConst128: useKindNone, + vecLoad1R: useKindRN, + cSel: useKindRNRM, + fpuCSel: useKindRNRM, + movToVec: useKindRN, + movFromVec: useKindRN, + movFromVecSigned: useKindRN, + vecDup: useKindRN, + vecDupElement: useKindRN, + vecExtract: useKindRNRM, + cCmpImm: useKindRN, + vecMisc: useKindRN, + vecMovElement: useKindRN, + vecLanes: useKindRN, + vecShiftImm: useKindRN, + vecTbl: useKindRNRM, + vecTbl2: useKindRNRN1RM, + vecRRR: useKindRNRM, + vecRRRRewrite: useKindRDRewrite, + vecPermute: useKindRNRM, + fpuToInt: useKindRN, + intToFpu: useKindRN, + movToFPSR: useKindRN, + movFromFPSR: useKindNone, + adr: useKindNone, + emitSourceOffsetInfo: useKindNone, + atomicRmw: useKindRNRM, + atomicCas: useKindRDRewrite, + atomicLoad: useKindRN, + atomicStore: useKindRNRM, + loadConstBlockArg: useKindNone, + dmb: useKindNone, +} + +// Uses returns the list of regalloc.VReg that are used by the instruction. +// In order to reduce the number of allocations, the caller can pass the slice to be used. +func (i *instruction) Uses(regs *[]regalloc.VReg) []regalloc.VReg { + *regs = (*regs)[:0] + switch useKinds[i.kind] { + case useKindNone: + case useKindRN: + if rn := i.rn.reg(); rn.Valid() { + *regs = append(*regs, rn) + } + case useKindRNRM: + if rn := i.rn.reg(); rn.Valid() { + *regs = append(*regs, rn) + } + if rm := i.rm.reg(); rm.Valid() { + *regs = append(*regs, rm) + } + case useKindRNRMRA: + if rn := i.rn.reg(); rn.Valid() { + *regs = append(*regs, rn) + } + if rm := i.rm.reg(); rm.Valid() { + *regs = append(*regs, rm) + } + if ra := i.ra.reg(); ra.Valid() { + *regs = append(*regs, ra) + } + case useKindRNRN1RM: + if rn := i.rn.reg(); rn.Valid() && rn.IsRealReg() { + rn1 := regalloc.FromRealReg(rn.RealReg()+1, rn.RegType()) + *regs = append(*regs, rn, rn1) + } + if rm := i.rm.reg(); rm.Valid() { + *regs = append(*regs, rm) + } + case useKindAMode: + if amodeRN := i.amode.rn; amodeRN.Valid() { + *regs = append(*regs, amodeRN) + } + if amodeRM := i.amode.rm; amodeRM.Valid() { + *regs = append(*regs, amodeRM) + } + case useKindRNAMode: + *regs = append(*regs, i.rn.reg()) + if amodeRN := i.amode.rn; amodeRN.Valid() { + *regs = append(*regs, amodeRN) + } + if amodeRM := i.amode.rm; amodeRM.Valid() { + *regs = append(*regs, amodeRM) + } + case useKindCond: + cnd := cond(i.u1) + if cnd.kind() != condKindCondFlagSet { + *regs = append(*regs, cnd.register()) + } + case useKindCallInd: + *regs = append(*regs, i.rn.nr()) + fallthrough + case useKindCall: + argIntRealRegs, argFloatRealRegs, _, _, _ := backend.ABIInfoFromUint64(i.u2) + for i := byte(0); i < argIntRealRegs; i++ { + *regs = append(*regs, regInfo.RealRegToVReg[intParamResultRegs[i]]) + } + for i := byte(0); i < argFloatRealRegs; i++ { + *regs = append(*regs, regInfo.RealRegToVReg[floatParamResultRegs[i]]) + } + case useKindRDRewrite: + *regs = append(*regs, i.rn.reg()) + *regs = append(*regs, i.rm.reg()) + *regs = append(*regs, i.rd.reg()) + default: + panic(fmt.Sprintf("useKind for %v not defined", i)) + } + return *regs +} + +func (i *instruction) AssignUse(index int, reg regalloc.VReg) { + switch useKinds[i.kind] { + case useKindNone: + case useKindRN: + if rn := i.rn.reg(); rn.Valid() { + i.rn = i.rn.assignReg(reg) + } + case useKindRNRM: + if index == 0 { + if rn := i.rn.reg(); rn.Valid() { + i.rn = i.rn.assignReg(reg) + } + } else { + if rm := i.rm.reg(); rm.Valid() { + i.rm = i.rm.assignReg(reg) + } + } + case useKindRDRewrite: + if index == 0 { + if rn := i.rn.reg(); rn.Valid() { + i.rn = i.rn.assignReg(reg) + } + } else if index == 1 { + if rm := i.rm.reg(); rm.Valid() { + i.rm = i.rm.assignReg(reg) + } + } else { + if rd := i.rd.reg(); rd.Valid() { + i.rd = i.rd.assignReg(reg) + } + } + case useKindRNRN1RM: + if index == 0 { + if rn := i.rn.reg(); rn.Valid() { + i.rn = i.rn.assignReg(reg) + } + if rn1 := i.rn.reg() + 1; rn1.Valid() { + i.rm = i.rm.assignReg(reg + 1) + } + } else { + if rm := i.rm.reg(); rm.Valid() { + i.rm = i.rm.assignReg(reg) + } + } + case useKindRNRMRA: + if index == 0 { + if rn := i.rn.reg(); rn.Valid() { + i.rn = i.rn.assignReg(reg) + } + } else if index == 1 { + if rm := i.rm.reg(); rm.Valid() { + i.rm = i.rm.assignReg(reg) + } + } else { + if ra := i.ra.reg(); ra.Valid() { + i.ra = i.ra.assignReg(reg) + } + } + case useKindAMode: + if index == 0 { + if amodeRN := i.amode.rn; amodeRN.Valid() { + i.amode.rn = reg + } + } else { + if amodeRM := i.amode.rm; amodeRM.Valid() { + i.amode.rm = reg + } + } + case useKindRNAMode: + if index == 0 { + i.rn = i.rn.assignReg(reg) + } else if index == 1 { + if amodeRN := i.amode.rn; amodeRN.Valid() { + i.amode.rn = reg + } else { + panic("BUG") + } + } else { + if amodeRM := i.amode.rm; amodeRM.Valid() { + i.amode.rm = reg + } else { + panic("BUG") + } + } + case useKindCond: + c := cond(i.u1) + switch c.kind() { + case condKindRegisterZero: + i.u1 = uint64(registerAsRegZeroCond(reg)) + case condKindRegisterNotZero: + i.u1 = uint64(registerAsRegNotZeroCond(reg)) + } + case useKindCall: + panic("BUG: call instructions shouldn't be assigned") + case useKindCallInd: + i.rn = i.rn.assignReg(reg) + default: + panic(fmt.Sprintf("useKind for %v not defined", i)) + } +} + +func (i *instruction) asCall(ref ssa.FuncRef, abi *backend.FunctionABI) { + i.kind = call + i.u1 = uint64(ref) + if abi != nil { + i.u2 = abi.ABIInfoAsUint64() + } +} + +func (i *instruction) asCallIndirect(ptr regalloc.VReg, abi *backend.FunctionABI) { + i.kind = callInd + i.rn = operandNR(ptr) + if abi != nil { + i.u2 = abi.ABIInfoAsUint64() + } +} + +func (i *instruction) callFuncRef() ssa.FuncRef { + return ssa.FuncRef(i.u1) +} + +// shift must be divided by 16 and must be in range 0-3 (if dst64bit is true) or 0-1 (if dst64bit is false) +func (i *instruction) asMOVZ(dst regalloc.VReg, imm uint64, shift uint64, dst64bit bool) { + i.kind = movZ + i.rd = operandNR(dst) + i.u1 = imm + i.u2 = shift + if dst64bit { + i.u3 = 1 + } +} + +// shift must be divided by 16 and must be in range 0-3 (if dst64bit is true) or 0-1 (if dst64bit is false) +func (i *instruction) asMOVK(dst regalloc.VReg, imm uint64, shift uint64, dst64bit bool) { + i.kind = movK + i.rd = operandNR(dst) + i.u1 = imm + i.u2 = shift + if dst64bit { + i.u3 = 1 + } +} + +// shift must be divided by 16 and must be in range 0-3 (if dst64bit is true) or 0-1 (if dst64bit is false) +func (i *instruction) asMOVN(dst regalloc.VReg, imm uint64, shift uint64, dst64bit bool) { + i.kind = movN + i.rd = operandNR(dst) + i.u1 = imm + i.u2 = shift + if dst64bit { + i.u3 = 1 + } +} + +func (i *instruction) asNop0() *instruction { + i.kind = nop0 + return i +} + +func (i *instruction) asNop0WithLabel(l label) { + i.kind = nop0 + i.u1 = uint64(l) +} + +func (i *instruction) nop0Label() label { + return label(i.u1) +} + +func (i *instruction) asRet() { + i.kind = ret +} + +func (i *instruction) asStorePair64(src1, src2 regalloc.VReg, amode addressMode) { + i.kind = storeP64 + i.rn = operandNR(src1) + i.rm = operandNR(src2) + i.amode = amode +} + +func (i *instruction) asLoadPair64(src1, src2 regalloc.VReg, amode addressMode) { + i.kind = loadP64 + i.rn = operandNR(src1) + i.rm = operandNR(src2) + i.amode = amode +} + +func (i *instruction) asStore(src operand, amode addressMode, sizeInBits byte) { + switch sizeInBits { + case 8: + i.kind = store8 + case 16: + i.kind = store16 + case 32: + if src.reg().RegType() == regalloc.RegTypeInt { + i.kind = store32 + } else { + i.kind = fpuStore32 + } + case 64: + if src.reg().RegType() == regalloc.RegTypeInt { + i.kind = store64 + } else { + i.kind = fpuStore64 + } + case 128: + i.kind = fpuStore128 + } + i.rn = src + i.amode = amode +} + +func (i *instruction) asSLoad(dst operand, amode addressMode, sizeInBits byte) { + switch sizeInBits { + case 8: + i.kind = sLoad8 + case 16: + i.kind = sLoad16 + case 32: + i.kind = sLoad32 + default: + panic("BUG") + } + i.rd = dst + i.amode = amode +} + +func (i *instruction) asULoad(dst operand, amode addressMode, sizeInBits byte) { + switch sizeInBits { + case 8: + i.kind = uLoad8 + case 16: + i.kind = uLoad16 + case 32: + i.kind = uLoad32 + case 64: + i.kind = uLoad64 + } + i.rd = dst + i.amode = amode +} + +func (i *instruction) asFpuLoad(dst operand, amode addressMode, sizeInBits byte) { + switch sizeInBits { + case 32: + i.kind = fpuLoad32 + case 64: + i.kind = fpuLoad64 + case 128: + i.kind = fpuLoad128 + } + i.rd = dst + i.amode = amode +} + +func (i *instruction) asVecLoad1R(rd, rn operand, arr vecArrangement) { + // NOTE: currently only has support for no-offset loads, though it is suspicious that + // we would need to support offset load (that is only available for post-index). + i.kind = vecLoad1R + i.rd = rd + i.rn = rn + i.u1 = uint64(arr) +} + +func (i *instruction) asCSet(rd regalloc.VReg, mask bool, c condFlag) { + i.kind = cSet + i.rd = operandNR(rd) + i.u1 = uint64(c) + if mask { + i.u2 = 1 + } +} + +func (i *instruction) asCSel(rd, rn, rm operand, c condFlag, _64bit bool) { + i.kind = cSel + i.rd = rd + i.rn = rn + i.rm = rm + i.u1 = uint64(c) + if _64bit { + i.u3 = 1 + } +} + +func (i *instruction) asFpuCSel(rd, rn, rm operand, c condFlag, _64bit bool) { + i.kind = fpuCSel + i.rd = rd + i.rn = rn + i.rm = rm + i.u1 = uint64(c) + if _64bit { + i.u3 = 1 + } +} + +func (i *instruction) asBr(target label) { + if target == labelReturn { + panic("BUG: call site should special case for returnLabel") + } + i.kind = br + i.u1 = uint64(target) +} + +func (i *instruction) asBrTableSequence(indexReg regalloc.VReg, targetIndex, targetCounts int) { + i.kind = brTableSequence + i.rn = operandNR(indexReg) + i.u1 = uint64(targetIndex) + i.u2 = uint64(targetCounts) +} + +func (i *instruction) brTableSequenceOffsetsResolved() { + i.u3 = 1 // indicate that the offsets are resolved, for debugging. +} + +func (i *instruction) brLabel() label { + return label(i.u1) +} + +// brOffsetResolved is called when the target label is resolved. +func (i *instruction) brOffsetResolve(offset int64) { + i.u2 = uint64(offset) + i.u3 = 1 // indicate that the offset is resolved, for debugging. +} + +func (i *instruction) brOffset() int64 { + return int64(i.u2) +} + +// asCondBr encodes a conditional branch instruction. is64bit is only needed when cond is not flag. +func (i *instruction) asCondBr(c cond, target label, is64bit bool) { + i.kind = condBr + i.u1 = c.asUint64() + i.u2 = uint64(target) + if is64bit { + i.u3 = 1 + } +} + +func (i *instruction) setCondBrTargets(target label) { + i.u2 = uint64(target) +} + +func (i *instruction) condBrLabel() label { + return label(i.u2) +} + +// condBrOffsetResolve is called when the target label is resolved. +func (i *instruction) condBrOffsetResolve(offset int64) { + i.rd.data = uint64(offset) + i.rd.data2 = 1 // indicate that the offset is resolved, for debugging. +} + +// condBrOffsetResolved returns true if condBrOffsetResolve is already called. +func (i *instruction) condBrOffsetResolved() bool { + return i.rd.data2 == 1 +} + +func (i *instruction) condBrOffset() int64 { + return int64(i.rd.data) +} + +func (i *instruction) condBrCond() cond { + return cond(i.u1) +} + +func (i *instruction) condBr64bit() bool { + return i.u3 == 1 +} + +func (i *instruction) asLoadFpuConst32(rd regalloc.VReg, raw uint64) { + i.kind = loadFpuConst32 + i.u1 = raw + i.rd = operandNR(rd) +} + +func (i *instruction) asLoadFpuConst64(rd regalloc.VReg, raw uint64) { + i.kind = loadFpuConst64 + i.u1 = raw + i.rd = operandNR(rd) +} + +func (i *instruction) asLoadFpuConst128(rd regalloc.VReg, lo, hi uint64) { + i.kind = loadFpuConst128 + i.u1 = lo + i.u2 = hi + i.rd = operandNR(rd) +} + +func (i *instruction) asFpuCmp(rn, rm operand, is64bit bool) { + i.kind = fpuCmp + i.rn, i.rm = rn, rm + if is64bit { + i.u3 = 1 + } +} + +func (i *instruction) asCCmpImm(rn operand, imm uint64, c condFlag, flag byte, is64bit bool) { + i.kind = cCmpImm + i.rn = rn + i.rm.data = imm + i.u1 = uint64(c) + i.u2 = uint64(flag) + if is64bit { + i.u3 = 1 + } +} + +// asALU setups a basic ALU instruction. +func (i *instruction) asALU(aluOp aluOp, rd, rn, rm operand, dst64bit bool) { + switch rm.kind { + case operandKindNR: + i.kind = aluRRR + case operandKindSR: + i.kind = aluRRRShift + case operandKindER: + i.kind = aluRRRExtend + case operandKindImm12: + i.kind = aluRRImm12 + default: + panic("BUG") + } + i.u1 = uint64(aluOp) + i.rd, i.rn, i.rm = rd, rn, rm + if dst64bit { + i.u3 = 1 + } +} + +// asALU setups a basic ALU instruction. +func (i *instruction) asALURRRR(aluOp aluOp, rd, rn, rm, ra operand, dst64bit bool) { + i.kind = aluRRRR + i.u1 = uint64(aluOp) + i.rd, i.rn, i.rm, i.ra = rd, rn, rm, ra + if dst64bit { + i.u3 = 1 + } +} + +// asALUShift setups a shift based ALU instruction. +func (i *instruction) asALUShift(aluOp aluOp, rd, rn, rm operand, dst64bit bool) { + switch rm.kind { + case operandKindNR: + i.kind = aluRRR // If the shift amount op is a register, then the instruction is encoded as a normal ALU instruction with two register operands. + case operandKindShiftImm: + i.kind = aluRRImmShift + default: + panic("BUG") + } + i.u1 = uint64(aluOp) + i.rd, i.rn, i.rm = rd, rn, rm + if dst64bit { + i.u3 = 1 + } +} + +func (i *instruction) asALUBitmaskImm(aluOp aluOp, rd, rn regalloc.VReg, imm uint64, dst64bit bool) { + i.kind = aluRRBitmaskImm + i.u1 = uint64(aluOp) + i.rn, i.rd = operandNR(rn), operandNR(rd) + i.u2 = imm + if dst64bit { + i.u3 = 1 + } +} + +func (i *instruction) asMovToFPSR(rn regalloc.VReg) { + i.kind = movToFPSR + i.rn = operandNR(rn) +} + +func (i *instruction) asMovFromFPSR(rd regalloc.VReg) { + i.kind = movFromFPSR + i.rd = operandNR(rd) +} + +func (i *instruction) asBitRR(bitOp bitOp, rd, rn regalloc.VReg, is64bit bool) { + i.kind = bitRR + i.rn, i.rd = operandNR(rn), operandNR(rd) + i.u1 = uint64(bitOp) + if is64bit { + i.u2 = 1 + } +} + +func (i *instruction) asFpuRRR(op fpuBinOp, rd, rn, rm operand, dst64bit bool) { + i.kind = fpuRRR + i.u1 = uint64(op) + i.rd, i.rn, i.rm = rd, rn, rm + if dst64bit { + i.u3 = 1 + } +} + +func (i *instruction) asFpuRR(op fpuUniOp, rd, rn operand, dst64bit bool) { + i.kind = fpuRR + i.u1 = uint64(op) + i.rd, i.rn = rd, rn + if dst64bit { + i.u3 = 1 + } +} + +func (i *instruction) asExtend(rd, rn regalloc.VReg, fromBits, toBits byte, signed bool) { + i.kind = extend + i.rn, i.rd = operandNR(rn), operandNR(rd) + i.u1 = uint64(fromBits) + i.u2 = uint64(toBits) + if signed { + i.u3 = 1 + } +} + +func (i *instruction) asMove32(rd, rn regalloc.VReg) { + i.kind = mov32 + i.rn, i.rd = operandNR(rn), operandNR(rd) +} + +func (i *instruction) asMove64(rd, rn regalloc.VReg) *instruction { + i.kind = mov64 + i.rn, i.rd = operandNR(rn), operandNR(rd) + return i +} + +func (i *instruction) asFpuMov64(rd, rn regalloc.VReg) { + i.kind = fpuMov64 + i.rn, i.rd = operandNR(rn), operandNR(rd) +} + +func (i *instruction) asFpuMov128(rd, rn regalloc.VReg) *instruction { + i.kind = fpuMov128 + i.rn, i.rd = operandNR(rn), operandNR(rd) + return i +} + +func (i *instruction) asMovToVec(rd, rn operand, arr vecArrangement, index vecIndex) { + i.kind = movToVec + i.rd = rd + i.rn = rn + i.u1, i.u2 = uint64(arr), uint64(index) +} + +func (i *instruction) asMovFromVec(rd, rn operand, arr vecArrangement, index vecIndex, signed bool) { + if signed { + i.kind = movFromVecSigned + } else { + i.kind = movFromVec + } + i.rd = rd + i.rn = rn + i.u1, i.u2 = uint64(arr), uint64(index) +} + +func (i *instruction) asVecDup(rd, rn operand, arr vecArrangement) { + i.kind = vecDup + i.u1 = uint64(arr) + i.rn, i.rd = rn, rd +} + +func (i *instruction) asVecDupElement(rd, rn operand, arr vecArrangement, index vecIndex) { + i.kind = vecDupElement + i.u1 = uint64(arr) + i.rn, i.rd = rn, rd + i.u2 = uint64(index) +} + +func (i *instruction) asVecExtract(rd, rn, rm operand, arr vecArrangement, index uint32) { + i.kind = vecExtract + i.u1 = uint64(arr) + i.rn, i.rm, i.rd = rn, rm, rd + i.u2 = uint64(index) +} + +func (i *instruction) asVecMovElement(rd, rn operand, arr vecArrangement, rdIndex, rnIndex vecIndex) { + i.kind = vecMovElement + i.u1 = uint64(arr) + i.u2, i.u3 = uint64(rdIndex), uint64(rnIndex) + i.rn, i.rd = rn, rd +} + +func (i *instruction) asVecMisc(op vecOp, rd, rn operand, arr vecArrangement) { + i.kind = vecMisc + i.u1 = uint64(op) + i.rn, i.rd = rn, rd + i.u2 = uint64(arr) +} + +func (i *instruction) asVecLanes(op vecOp, rd, rn operand, arr vecArrangement) { + i.kind = vecLanes + i.u1 = uint64(op) + i.rn, i.rd = rn, rd + i.u2 = uint64(arr) +} + +func (i *instruction) asVecShiftImm(op vecOp, rd, rn, rm operand, arr vecArrangement) *instruction { + i.kind = vecShiftImm + i.u1 = uint64(op) + i.rn, i.rm, i.rd = rn, rm, rd + i.u2 = uint64(arr) + return i +} + +func (i *instruction) asVecTbl(nregs byte, rd, rn, rm operand, arr vecArrangement) { + switch nregs { + case 0, 1: + i.kind = vecTbl + case 2: + i.kind = vecTbl2 + if !rn.reg().IsRealReg() { + panic("rn is not a RealReg") + } + if rn.realReg() == v31 { + panic("rn cannot be v31") + } + default: + panic(fmt.Sprintf("unsupported number of registers %d", nregs)) + } + i.rn, i.rm, i.rd = rn, rm, rd + i.u2 = uint64(arr) +} + +func (i *instruction) asVecPermute(op vecOp, rd, rn, rm operand, arr vecArrangement) { + i.kind = vecPermute + i.u1 = uint64(op) + i.rn, i.rm, i.rd = rn, rm, rd + i.u2 = uint64(arr) +} + +func (i *instruction) asVecRRR(op vecOp, rd, rn, rm operand, arr vecArrangement) *instruction { + i.kind = vecRRR + i.u1 = uint64(op) + i.rn, i.rd, i.rm = rn, rd, rm + i.u2 = uint64(arr) + return i +} + +// asVecRRRRewrite encodes a vector instruction that rewrites the destination register. +// IMPORTANT: the destination register must be already defined before this instruction. +func (i *instruction) asVecRRRRewrite(op vecOp, rd, rn, rm operand, arr vecArrangement) { + i.kind = vecRRRRewrite + i.u1 = uint64(op) + i.rn, i.rd, i.rm = rn, rd, rm + i.u2 = uint64(arr) +} + +func (i *instruction) IsCopy() bool { + op := i.kind + // We do not include mov32 as it is not a copy instruction in the sense that it does not preserve the upper 32 bits, + // and it is only used in the translation of IReduce, not the actual copy indeed. + return op == mov64 || op == fpuMov64 || op == fpuMov128 +} + +// String implements fmt.Stringer. +func (i *instruction) String() (str string) { + is64SizeBitToSize := func(u3 uint64) byte { + if u3 == 0 { + return 32 + } + return 64 + } + + switch i.kind { + case nop0: + if i.u1 != 0 { + l := label(i.u1) + str = fmt.Sprintf("%s:", l) + } else { + str = "nop0" + } + case aluRRR: + size := is64SizeBitToSize(i.u3) + str = fmt.Sprintf("%s %s, %s, %s", aluOp(i.u1).String(), + formatVRegSized(i.rd.nr(), size), formatVRegSized(i.rn.nr(), size), + i.rm.format(size)) + case aluRRRR: + size := is64SizeBitToSize(i.u3) + str = fmt.Sprintf("%s %s, %s, %s, %s", aluOp(i.u1).String(), + formatVRegSized(i.rd.nr(), size), formatVRegSized(i.rn.nr(), size), formatVRegSized(i.rm.nr(), size), formatVRegSized(i.ra.nr(), size)) + case aluRRImm12: + size := is64SizeBitToSize(i.u3) + str = fmt.Sprintf("%s %s, %s, %s", aluOp(i.u1).String(), + formatVRegSized(i.rd.nr(), size), formatVRegSized(i.rn.nr(), size), i.rm.format(size)) + case aluRRBitmaskImm: + size := is64SizeBitToSize(i.u3) + rd, rn := formatVRegSized(i.rd.nr(), size), formatVRegSized(i.rn.nr(), size) + if size == 32 { + str = fmt.Sprintf("%s %s, %s, #%#x", aluOp(i.u1).String(), rd, rn, uint32(i.u2)) + } else { + str = fmt.Sprintf("%s %s, %s, #%#x", aluOp(i.u1).String(), rd, rn, i.u2) + } + case aluRRImmShift: + size := is64SizeBitToSize(i.u3) + str = fmt.Sprintf("%s %s, %s, %#x", + aluOp(i.u1).String(), + formatVRegSized(i.rd.nr(), size), + formatVRegSized(i.rn.nr(), size), + i.rm.shiftImm(), + ) + case aluRRRShift: + size := is64SizeBitToSize(i.u3) + str = fmt.Sprintf("%s %s, %s, %s", + aluOp(i.u1).String(), + formatVRegSized(i.rd.nr(), size), + formatVRegSized(i.rn.nr(), size), + i.rm.format(size), + ) + case aluRRRExtend: + size := is64SizeBitToSize(i.u3) + str = fmt.Sprintf("%s %s, %s, %s", aluOp(i.u1).String(), + formatVRegSized(i.rd.nr(), size), + formatVRegSized(i.rn.nr(), size), + // Regardless of the source size, the register is formatted in 32-bit. + i.rm.format(32), + ) + case bitRR: + size := is64SizeBitToSize(i.u2) + str = fmt.Sprintf("%s %s, %s", + bitOp(i.u1), + formatVRegSized(i.rd.nr(), size), + formatVRegSized(i.rn.nr(), size), + ) + case uLoad8: + str = fmt.Sprintf("ldrb %s, %s", formatVRegSized(i.rd.nr(), 32), i.amode.format(32)) + case sLoad8: + str = fmt.Sprintf("ldrsb %s, %s", formatVRegSized(i.rd.nr(), 32), i.amode.format(32)) + case uLoad16: + str = fmt.Sprintf("ldrh %s, %s", formatVRegSized(i.rd.nr(), 32), i.amode.format(32)) + case sLoad16: + str = fmt.Sprintf("ldrsh %s, %s", formatVRegSized(i.rd.nr(), 32), i.amode.format(32)) + case uLoad32: + str = fmt.Sprintf("ldr %s, %s", formatVRegSized(i.rd.nr(), 32), i.amode.format(32)) + case sLoad32: + str = fmt.Sprintf("ldrs %s, %s", formatVRegSized(i.rd.nr(), 32), i.amode.format(32)) + case uLoad64: + str = fmt.Sprintf("ldr %s, %s", formatVRegSized(i.rd.nr(), 64), i.amode.format(64)) + case store8: + str = fmt.Sprintf("strb %s, %s", formatVRegSized(i.rn.nr(), 32), i.amode.format(8)) + case store16: + str = fmt.Sprintf("strh %s, %s", formatVRegSized(i.rn.nr(), 32), i.amode.format(16)) + case store32: + str = fmt.Sprintf("str %s, %s", formatVRegSized(i.rn.nr(), 32), i.amode.format(32)) + case store64: + str = fmt.Sprintf("str %s, %s", formatVRegSized(i.rn.nr(), 64), i.amode.format(64)) + case storeP64: + str = fmt.Sprintf("stp %s, %s, %s", + formatVRegSized(i.rn.nr(), 64), formatVRegSized(i.rm.nr(), 64), i.amode.format(64)) + case loadP64: + str = fmt.Sprintf("ldp %s, %s, %s", + formatVRegSized(i.rn.nr(), 64), formatVRegSized(i.rm.nr(), 64), i.amode.format(64)) + case mov64: + str = fmt.Sprintf("mov %s, %s", + formatVRegSized(i.rd.nr(), 64), + formatVRegSized(i.rn.nr(), 64)) + case mov32: + str = fmt.Sprintf("mov %s, %s", formatVRegSized(i.rd.nr(), 32), formatVRegSized(i.rn.nr(), 32)) + case movZ: + size := is64SizeBitToSize(i.u3) + str = fmt.Sprintf("movz %s, #%#x, lsl %d", formatVRegSized(i.rd.nr(), size), uint16(i.u1), i.u2*16) + case movN: + size := is64SizeBitToSize(i.u3) + str = fmt.Sprintf("movn %s, #%#x, lsl %d", formatVRegSized(i.rd.nr(), size), uint16(i.u1), i.u2*16) + case movK: + size := is64SizeBitToSize(i.u3) + str = fmt.Sprintf("movk %s, #%#x, lsl %d", formatVRegSized(i.rd.nr(), size), uint16(i.u1), i.u2*16) + case extend: + fromBits, toBits := byte(i.u1), byte(i.u2) + + var signedStr string + if i.u3 == 1 { + signedStr = "s" + } else { + signedStr = "u" + } + var fromStr string + switch fromBits { + case 8: + fromStr = "b" + case 16: + fromStr = "h" + case 32: + fromStr = "w" + } + str = fmt.Sprintf("%sxt%s %s, %s", signedStr, fromStr, formatVRegSized(i.rd.nr(), toBits), formatVRegSized(i.rn.nr(), 32)) + case cSel: + size := is64SizeBitToSize(i.u3) + str = fmt.Sprintf("csel %s, %s, %s, %s", + formatVRegSized(i.rd.nr(), size), + formatVRegSized(i.rn.nr(), size), + formatVRegSized(i.rm.nr(), size), + condFlag(i.u1), + ) + case cSet: + if i.u2 != 0 { + str = fmt.Sprintf("csetm %s, %s", formatVRegSized(i.rd.nr(), 64), condFlag(i.u1)) + } else { + str = fmt.Sprintf("cset %s, %s", formatVRegSized(i.rd.nr(), 64), condFlag(i.u1)) + } + case cCmpImm: + size := is64SizeBitToSize(i.u3) + str = fmt.Sprintf("ccmp %s, #%#x, #%#x, %s", + formatVRegSized(i.rn.nr(), size), i.rm.data, + i.u2&0b1111, + condFlag(i.u1)) + case fpuMov64: + str = fmt.Sprintf("mov %s, %s", + formatVRegVec(i.rd.nr(), vecArrangement8B, vecIndexNone), + formatVRegVec(i.rn.nr(), vecArrangement8B, vecIndexNone)) + case fpuMov128: + str = fmt.Sprintf("mov %s, %s", + formatVRegVec(i.rd.nr(), vecArrangement16B, vecIndexNone), + formatVRegVec(i.rn.nr(), vecArrangement16B, vecIndexNone)) + case fpuMovFromVec: + panic("TODO") + case fpuRR: + dstSz := is64SizeBitToSize(i.u3) + srcSz := dstSz + op := fpuUniOp(i.u1) + switch op { + case fpuUniOpCvt32To64: + srcSz = 32 + case fpuUniOpCvt64To32: + srcSz = 64 + } + str = fmt.Sprintf("%s %s, %s", op.String(), + formatVRegSized(i.rd.nr(), dstSz), formatVRegSized(i.rn.nr(), srcSz)) + case fpuRRR: + size := is64SizeBitToSize(i.u3) + str = fmt.Sprintf("%s %s, %s, %s", fpuBinOp(i.u1).String(), + formatVRegSized(i.rd.nr(), size), formatVRegSized(i.rn.nr(), size), formatVRegSized(i.rm.nr(), size)) + case fpuRRI: + panic("TODO") + case fpuRRRR: + panic("TODO") + case fpuCmp: + size := is64SizeBitToSize(i.u3) + str = fmt.Sprintf("fcmp %s, %s", + formatVRegSized(i.rn.nr(), size), formatVRegSized(i.rm.nr(), size)) + case fpuLoad32: + str = fmt.Sprintf("ldr %s, %s", formatVRegSized(i.rd.nr(), 32), i.amode.format(32)) + case fpuStore32: + str = fmt.Sprintf("str %s, %s", formatVRegSized(i.rn.nr(), 32), i.amode.format(64)) + case fpuLoad64: + str = fmt.Sprintf("ldr %s, %s", formatVRegSized(i.rd.nr(), 64), i.amode.format(64)) + case fpuStore64: + str = fmt.Sprintf("str %s, %s", formatVRegSized(i.rn.nr(), 64), i.amode.format(64)) + case fpuLoad128: + str = fmt.Sprintf("ldr %s, %s", formatVRegSized(i.rd.nr(), 128), i.amode.format(64)) + case fpuStore128: + str = fmt.Sprintf("str %s, %s", formatVRegSized(i.rn.nr(), 128), i.amode.format(64)) + case loadFpuConst32: + str = fmt.Sprintf("ldr %s, #8; b 8; data.f32 %f", formatVRegSized(i.rd.nr(), 32), math.Float32frombits(uint32(i.u1))) + case loadFpuConst64: + str = fmt.Sprintf("ldr %s, #8; b 16; data.f64 %f", formatVRegSized(i.rd.nr(), 64), math.Float64frombits(i.u1)) + case loadFpuConst128: + str = fmt.Sprintf("ldr %s, #8; b 32; data.v128 %016x %016x", + formatVRegSized(i.rd.nr(), 128), i.u1, i.u2) + case fpuToInt: + var op, src, dst string + if signed := i.u1 == 1; signed { + op = "fcvtzs" + } else { + op = "fcvtzu" + } + if src64 := i.u2 == 1; src64 { + src = formatVRegWidthVec(i.rn.nr(), vecArrangementD) + } else { + src = formatVRegWidthVec(i.rn.nr(), vecArrangementS) + } + if dst64 := i.u3 == 1; dst64 { + dst = formatVRegSized(i.rd.nr(), 64) + } else { + dst = formatVRegSized(i.rd.nr(), 32) + } + str = fmt.Sprintf("%s %s, %s", op, dst, src) + + case intToFpu: + var op, src, dst string + if signed := i.u1 == 1; signed { + op = "scvtf" + } else { + op = "ucvtf" + } + if src64 := i.u2 == 1; src64 { + src = formatVRegSized(i.rn.nr(), 64) + } else { + src = formatVRegSized(i.rn.nr(), 32) + } + if dst64 := i.u3 == 1; dst64 { + dst = formatVRegWidthVec(i.rd.nr(), vecArrangementD) + } else { + dst = formatVRegWidthVec(i.rd.nr(), vecArrangementS) + } + str = fmt.Sprintf("%s %s, %s", op, dst, src) + case fpuCSel: + size := is64SizeBitToSize(i.u3) + str = fmt.Sprintf("fcsel %s, %s, %s, %s", + formatVRegSized(i.rd.nr(), size), + formatVRegSized(i.rn.nr(), size), + formatVRegSized(i.rm.nr(), size), + condFlag(i.u1), + ) + case movToVec: + var size byte + arr := vecArrangement(i.u1) + switch arr { + case vecArrangementB, vecArrangementH, vecArrangementS: + size = 32 + case vecArrangementD: + size = 64 + default: + panic("unsupported arrangement " + arr.String()) + } + str = fmt.Sprintf("ins %s, %s", formatVRegVec(i.rd.nr(), arr, vecIndex(i.u2)), formatVRegSized(i.rn.nr(), size)) + case movFromVec, movFromVecSigned: + var size byte + var opcode string + arr := vecArrangement(i.u1) + signed := i.kind == movFromVecSigned + switch arr { + case vecArrangementB, vecArrangementH, vecArrangementS: + size = 32 + if signed { + opcode = "smov" + } else { + opcode = "umov" + } + case vecArrangementD: + size = 64 + if signed { + opcode = "smov" + } else { + opcode = "mov" + } + default: + panic("unsupported arrangement " + arr.String()) + } + str = fmt.Sprintf("%s %s, %s", opcode, formatVRegSized(i.rd.nr(), size), formatVRegVec(i.rn.nr(), arr, vecIndex(i.u2))) + case vecDup: + str = fmt.Sprintf("dup %s, %s", + formatVRegVec(i.rd.nr(), vecArrangement(i.u1), vecIndexNone), + formatVRegSized(i.rn.nr(), 64), + ) + case vecDupElement: + arr := vecArrangement(i.u1) + str = fmt.Sprintf("dup %s, %s", + formatVRegVec(i.rd.nr(), arr, vecIndexNone), + formatVRegVec(i.rn.nr(), arr, vecIndex(i.u2)), + ) + case vecDupFromFpu: + panic("TODO") + case vecExtract: + str = fmt.Sprintf("ext %s, %s, %s, #%d", + formatVRegVec(i.rd.nr(), vecArrangement(i.u1), vecIndexNone), + formatVRegVec(i.rn.nr(), vecArrangement(i.u1), vecIndexNone), + formatVRegVec(i.rm.nr(), vecArrangement(i.u1), vecIndexNone), + uint32(i.u2), + ) + case vecExtend: + panic("TODO") + case vecMovElement: + str = fmt.Sprintf("mov %s, %s", + formatVRegVec(i.rd.nr(), vecArrangement(i.u1), vecIndex(i.u2)), + formatVRegVec(i.rn.nr(), vecArrangement(i.u1), vecIndex(i.u3)), + ) + case vecMiscNarrow: + panic("TODO") + case vecRRR, vecRRRRewrite: + str = fmt.Sprintf("%s %s, %s, %s", + vecOp(i.u1), + formatVRegVec(i.rd.nr(), vecArrangement(i.u2), vecIndexNone), + formatVRegVec(i.rn.nr(), vecArrangement(i.u2), vecIndexNone), + formatVRegVec(i.rm.nr(), vecArrangement(i.u2), vecIndexNone), + ) + case vecMisc: + vop := vecOp(i.u1) + if vop == vecOpCmeq0 { + str = fmt.Sprintf("cmeq %s, %s, #0", + formatVRegVec(i.rd.nr(), vecArrangement(i.u2), vecIndexNone), + formatVRegVec(i.rn.nr(), vecArrangement(i.u2), vecIndexNone)) + } else { + str = fmt.Sprintf("%s %s, %s", + vop, + formatVRegVec(i.rd.nr(), vecArrangement(i.u2), vecIndexNone), + formatVRegVec(i.rn.nr(), vecArrangement(i.u2), vecIndexNone)) + } + case vecLanes: + arr := vecArrangement(i.u2) + var destArr vecArrangement + switch arr { + case vecArrangement8B, vecArrangement16B: + destArr = vecArrangementH + case vecArrangement4H, vecArrangement8H: + destArr = vecArrangementS + case vecArrangement4S: + destArr = vecArrangementD + default: + panic("invalid arrangement " + arr.String()) + } + str = fmt.Sprintf("%s %s, %s", + vecOp(i.u1), + formatVRegWidthVec(i.rd.nr(), destArr), + formatVRegVec(i.rn.nr(), arr, vecIndexNone)) + case vecShiftImm: + arr := vecArrangement(i.u2) + str = fmt.Sprintf("%s %s, %s, #%d", + vecOp(i.u1), + formatVRegVec(i.rd.nr(), arr, vecIndexNone), + formatVRegVec(i.rn.nr(), arr, vecIndexNone), + i.rm.shiftImm()) + case vecTbl: + arr := vecArrangement(i.u2) + str = fmt.Sprintf("tbl %s, { %s }, %s", + formatVRegVec(i.rd.nr(), arr, vecIndexNone), + formatVRegVec(i.rn.nr(), vecArrangement16B, vecIndexNone), + formatVRegVec(i.rm.nr(), arr, vecIndexNone)) + case vecTbl2: + arr := vecArrangement(i.u2) + rd, rn, rm := i.rd.nr(), i.rn.nr(), i.rm.nr() + rn1 := regalloc.FromRealReg(rn.RealReg()+1, rn.RegType()) + str = fmt.Sprintf("tbl %s, { %s, %s }, %s", + formatVRegVec(rd, arr, vecIndexNone), + formatVRegVec(rn, vecArrangement16B, vecIndexNone), + formatVRegVec(rn1, vecArrangement16B, vecIndexNone), + formatVRegVec(rm, arr, vecIndexNone)) + case vecPermute: + arr := vecArrangement(i.u2) + str = fmt.Sprintf("%s %s, %s, %s", + vecOp(i.u1), + formatVRegVec(i.rd.nr(), arr, vecIndexNone), + formatVRegVec(i.rn.nr(), arr, vecIndexNone), + formatVRegVec(i.rm.nr(), arr, vecIndexNone)) + case movToFPSR: + str = fmt.Sprintf("msr fpsr, %s", formatVRegSized(i.rn.nr(), 64)) + case movFromFPSR: + str = fmt.Sprintf("mrs %s fpsr", formatVRegSized(i.rd.nr(), 64)) + case call: + str = fmt.Sprintf("bl %s", ssa.FuncRef(i.u1)) + case callInd: + str = fmt.Sprintf("bl %s", formatVRegSized(i.rn.nr(), 64)) + case ret: + str = "ret" + case br: + target := label(i.u1) + if i.u3 != 0 { + str = fmt.Sprintf("b #%#x (%s)", i.brOffset(), target.String()) + } else { + str = fmt.Sprintf("b %s", target.String()) + } + case condBr: + size := is64SizeBitToSize(i.u3) + c := cond(i.u1) + target := label(i.u2) + switch c.kind() { + case condKindRegisterZero: + if !i.condBrOffsetResolved() { + str = fmt.Sprintf("cbz %s, (%s)", formatVRegSized(c.register(), size), target.String()) + } else { + str = fmt.Sprintf("cbz %s, #%#x %s", formatVRegSized(c.register(), size), i.condBrOffset(), target.String()) + } + case condKindRegisterNotZero: + if offset := i.condBrOffset(); offset != 0 { + str = fmt.Sprintf("cbnz %s, #%#x (%s)", formatVRegSized(c.register(), size), offset, target.String()) + } else { + str = fmt.Sprintf("cbnz %s, %s", formatVRegSized(c.register(), size), target.String()) + } + case condKindCondFlagSet: + if offset := i.condBrOffset(); offset != 0 { + if target == labelInvalid { + str = fmt.Sprintf("b.%s #%#x", c.flag(), offset) + } else { + str = fmt.Sprintf("b.%s #%#x, (%s)", c.flag(), offset, target.String()) + } + } else { + str = fmt.Sprintf("b.%s %s", c.flag(), target.String()) + } + } + case adr: + str = fmt.Sprintf("adr %s, #%#x", formatVRegSized(i.rd.nr(), 64), int64(i.u1)) + case brTableSequence: + targetIndex := i.u1 + str = fmt.Sprintf("br_table_sequence %s, table_index=%d", formatVRegSized(i.rn.nr(), 64), targetIndex) + case exitSequence: + str = fmt.Sprintf("exit_sequence %s", formatVRegSized(i.rn.nr(), 64)) + case atomicRmw: + m := atomicRmwOp(i.u1).String() + size := byte(32) + switch i.u2 { + case 8: + size = 64 + case 2: + m = m + "h" + case 1: + m = m + "b" + } + str = fmt.Sprintf("%s %s, %s, %s", m, formatVRegSized(i.rm.nr(), size), formatVRegSized(i.rd.nr(), size), formatVRegSized(i.rn.nr(), 64)) + case atomicCas: + m := "casal" + size := byte(32) + switch i.u2 { + case 8: + size = 64 + case 2: + m = m + "h" + case 1: + m = m + "b" + } + str = fmt.Sprintf("%s %s, %s, %s", m, formatVRegSized(i.rd.nr(), size), formatVRegSized(i.rm.nr(), size), formatVRegSized(i.rn.nr(), 64)) + case atomicLoad: + m := "ldar" + size := byte(32) + switch i.u2 { + case 8: + size = 64 + case 2: + m = m + "h" + case 1: + m = m + "b" + } + str = fmt.Sprintf("%s %s, %s", m, formatVRegSized(i.rd.nr(), size), formatVRegSized(i.rn.nr(), 64)) + case atomicStore: + m := "stlr" + size := byte(32) + switch i.u2 { + case 8: + size = 64 + case 2: + m = m + "h" + case 1: + m = m + "b" + } + str = fmt.Sprintf("%s %s, %s", m, formatVRegSized(i.rm.nr(), size), formatVRegSized(i.rn.nr(), 64)) + case dmb: + str = "dmb" + case udf: + str = "udf" + case emitSourceOffsetInfo: + str = fmt.Sprintf("source_offset_info %d", ssa.SourceOffset(i.u1)) + case vecLoad1R: + str = fmt.Sprintf("ld1r {%s}, [%s]", formatVRegVec(i.rd.nr(), vecArrangement(i.u1), vecIndexNone), formatVRegSized(i.rn.nr(), 64)) + case loadConstBlockArg: + str = fmt.Sprintf("load_const_block_arg %s, %#x", formatVRegSized(i.rd.nr(), 64), i.u1) + default: + panic(i.kind) + } + return +} + +func (i *instruction) asAdr(rd regalloc.VReg, offset int64) { + i.kind = adr + i.rd = operandNR(rd) + i.u1 = uint64(offset) +} + +func (i *instruction) asAtomicRmw(op atomicRmwOp, rn, rs, rt operand, size uint64) { + i.kind = atomicRmw + i.rd, i.rn, i.rm = rt, rn, rs + i.u1 = uint64(op) + i.u2 = size +} + +func (i *instruction) asAtomicCas(rn, rs, rt operand, size uint64) { + i.kind = atomicCas + i.rm, i.rn, i.rd = rt, rn, rs + i.u2 = size +} + +func (i *instruction) asAtomicLoad(rn, rt operand, size uint64) { + i.kind = atomicLoad + i.rn, i.rd = rn, rt + i.u2 = size +} + +func (i *instruction) asAtomicStore(rn, rt operand, size uint64) { + i.kind = atomicStore + i.rn, i.rm = rn, rt + i.u2 = size +} + +func (i *instruction) asDMB() { + i.kind = dmb +} + +// TODO: delete unnecessary things. +const ( + // nop0 represents a no-op of zero size. + nop0 instructionKind = iota + 1 + // aluRRR represents an ALU operation with two register sources and a register destination. + aluRRR + // aluRRRR represents an ALU operation with three register sources and a register destination. + aluRRRR + // aluRRImm12 represents an ALU operation with a register source and an immediate-12 source, with a register destination. + aluRRImm12 + // aluRRBitmaskImm represents an ALU operation with a register source and a bitmask immediate, with a register destination. + aluRRBitmaskImm + // aluRRImmShift represents an ALU operation with a register source and an immediate-shifted source, with a register destination. + aluRRImmShift + // aluRRRShift represents an ALU operation with two register sources, one of which can be shifted, with a register destination. + aluRRRShift + // aluRRRExtend represents an ALU operation with two register sources, one of which can be extended, with a register destination. + aluRRRExtend + // bitRR represents a bit op instruction with a single register source. + bitRR + // uLoad8 represents an unsigned 8-bit load. + uLoad8 + // sLoad8 represents a signed 8-bit load into 64-bit register. + sLoad8 + // uLoad16 represents an unsigned 16-bit load into 64-bit register. + uLoad16 + // sLoad16 represents a signed 16-bit load into 64-bit register. + sLoad16 + // uLoad32 represents an unsigned 32-bit load into 64-bit register. + uLoad32 + // sLoad32 represents a signed 32-bit load into 64-bit register. + sLoad32 + // uLoad64 represents a 64-bit load. + uLoad64 + // store8 represents an 8-bit store. + store8 + // store16 represents a 16-bit store. + store16 + // store32 represents a 32-bit store. + store32 + // store64 represents a 64-bit store. + store64 + // storeP64 represents a store of a pair of registers. + storeP64 + // loadP64 represents a load of a pair of registers. + loadP64 + // mov64 represents a MOV instruction. These are encoded as ORR's but we keep them separate for better handling. + mov64 + // mov32 represents a 32-bit MOV. This zeroes the top 32 bits of the destination. + mov32 + // movZ represents a MOVZ with a 16-bit immediate. + movZ + // movN represents a MOVN with a 16-bit immediate. + movN + // movK represents a MOVK with a 16-bit immediate. + movK + // extend represents a sign- or zero-extend operation. + extend + // cSel represents a conditional-select operation. + cSel + // cSet represents a conditional-set operation. + cSet + // cCmpImm represents a conditional comparison with an immediate. + cCmpImm + // fpuMov64 represents a FPU move. Distinct from a vector-register move; moving just 64 bits appears to be significantly faster. + fpuMov64 + // fpuMov128 represents a vector register move. + fpuMov128 + // fpuMovFromVec represents a move to scalar from a vector element. + fpuMovFromVec + // fpuRR represents a 1-op FPU instruction. + fpuRR + // fpuRRR represents a 2-op FPU instruction. + fpuRRR + // fpuRRI represents a 2-op FPU instruction with immediate value. + fpuRRI + // fpuRRRR represents a 3-op FPU instruction. + fpuRRRR + // fpuCmp represents a FPU comparison, either 32 or 64 bit. + fpuCmp + // fpuLoad32 represents a floating-point load, single-precision (32 bit). + fpuLoad32 + // fpuStore32 represents a floating-point store, single-precision (32 bit). + fpuStore32 + // fpuLoad64 represents a floating-point load, double-precision (64 bit). + fpuLoad64 + // fpuStore64 represents a floating-point store, double-precision (64 bit). + fpuStore64 + // fpuLoad128 represents a floating-point/vector load, 128 bit. + fpuLoad128 + // fpuStore128 represents a floating-point/vector store, 128 bit. + fpuStore128 + // loadFpuConst32 represents a load of a 32-bit floating-point constant. + loadFpuConst32 + // loadFpuConst64 represents a load of a 64-bit floating-point constant. + loadFpuConst64 + // loadFpuConst128 represents a load of a 128-bit floating-point constant. + loadFpuConst128 + // vecLoad1R represents a load of a one single-element structure that replicates to all lanes of a vector. + vecLoad1R + // fpuToInt represents a conversion from FP to integer. + fpuToInt + // intToFpu represents a conversion from integer to FP. + intToFpu + // fpuCSel represents a 32/64-bit FP conditional select. + fpuCSel + // movToVec represents a move to a vector element from a GPR. + movToVec + // movFromVec represents an unsigned move from a vector element to a GPR. + movFromVec + // movFromVecSigned represents a signed move from a vector element to a GPR. + movFromVecSigned + // vecDup represents a duplication of general-purpose register to vector. + vecDup + // vecDupElement represents a duplication of a vector element to vector or scalar. + vecDupElement + // vecDupFromFpu represents a duplication of scalar to vector. + vecDupFromFpu + // vecExtract represents a vector extraction operation. + vecExtract + // vecExtend represents a vector extension operation. + vecExtend + // vecMovElement represents a move vector element to another vector element operation. + vecMovElement + // vecMiscNarrow represents a vector narrowing operation. + vecMiscNarrow + // vecRRR represents a vector ALU operation. + vecRRR + // vecRRRRewrite is exactly the same as vecRRR except that this rewrites the destination register. + // For example, BSL instruction rewrites the destination register, and the existing value influences the result. + // Therefore, the "destination" register in vecRRRRewrite will be treated as "use" which makes the register outlive + // the instruction while this instruction doesn't have "def" in the context of register allocation. + vecRRRRewrite + // vecMisc represents a vector two register miscellaneous instruction. + vecMisc + // vecLanes represents a vector instruction across lanes. + vecLanes + // vecShiftImm represents a SIMD scalar shift by immediate instruction. + vecShiftImm + // vecTbl represents a table vector lookup - single register table. + vecTbl + // vecTbl2 represents a table vector lookup - two register table. + vecTbl2 + // vecPermute represents a vector permute instruction. + vecPermute + // movToNZCV represents a move to the FPSR. + movToFPSR + // movFromNZCV represents a move from the FPSR. + movFromFPSR + // call represents a machine call instruction. + call + // callInd represents a machine indirect-call instruction. + callInd + // ret represents a machine return instruction. + ret + // br represents an unconditional branch. + br + // condBr represents a conditional branch. + condBr + // adr represents a compute the address (using a PC-relative offset) of a memory location. + adr + // brTableSequence represents a jump-table sequence. + brTableSequence + // exitSequence consists of multiple instructions, and exits the execution immediately. + // See encodeExitSequence. + exitSequence + // atomicRmw represents an atomic read-modify-write operation with two register sources and a register destination. + atomicRmw + // atomicCas represents an atomic compare-and-swap operation with three register sources. The value is loaded to + // the source register containing the comparison value. + atomicCas + // atomicLoad represents an atomic load with one source register and a register destination. + atomicLoad + // atomicStore represents an atomic store with two source registers and no destination. + atomicStore + // dmb represents the data memory barrier instruction in inner-shareable (ish) mode. + dmb + // UDF is the undefined instruction. For debugging only. + udf + // loadConstBlockArg represents a load of a constant block argument. + loadConstBlockArg + + // emitSourceOffsetInfo is a dummy instruction to emit source offset info. + // The existence of this instruction does not affect the execution. + emitSourceOffsetInfo + + // ------------------- do not define below this line ------------------- + numInstructionKinds +) + +func (i *instruction) asLoadConstBlockArg(v uint64, typ ssa.Type, dst regalloc.VReg) *instruction { + i.kind = loadConstBlockArg + i.u1 = v + i.u2 = uint64(typ) + i.rd = operandNR(dst) + return i +} + +func (i *instruction) loadConstBlockArgData() (v uint64, typ ssa.Type, dst regalloc.VReg) { + return i.u1, ssa.Type(i.u2), i.rd.nr() +} + +func (i *instruction) asEmitSourceOffsetInfo(l ssa.SourceOffset) *instruction { + i.kind = emitSourceOffsetInfo + i.u1 = uint64(l) + return i +} + +func (i *instruction) sourceOffsetInfo() ssa.SourceOffset { + return ssa.SourceOffset(i.u1) +} + +func (i *instruction) asUDF() *instruction { + i.kind = udf + return i +} + +func (i *instruction) asFpuToInt(rd, rn operand, rdSigned, src64bit, dst64bit bool) { + i.kind = fpuToInt + i.rn = rn + i.rd = rd + if rdSigned { + i.u1 = 1 + } + if src64bit { + i.u2 = 1 + } + if dst64bit { + i.u3 = 1 + } +} + +func (i *instruction) asIntToFpu(rd, rn operand, rnSigned, src64bit, dst64bit bool) { + i.kind = intToFpu + i.rn = rn + i.rd = rd + if rnSigned { + i.u1 = 1 + } + if src64bit { + i.u2 = 1 + } + if dst64bit { + i.u3 = 1 + } +} + +func (i *instruction) asExitSequence(ctx regalloc.VReg) *instruction { + i.kind = exitSequence + i.rn = operandNR(ctx) + return i +} + +// aluOp determines the type of ALU operation. Instructions whose kind is one of +// aluRRR, aluRRRR, aluRRImm12, aluRRBitmaskImm, aluRRImmShift, aluRRRShift and aluRRRExtend +// would use this type. +type aluOp int + +func (a aluOp) String() string { + switch a { + case aluOpAdd: + return "add" + case aluOpSub: + return "sub" + case aluOpOrr: + return "orr" + case aluOpOrn: + return "orn" + case aluOpAnd: + return "and" + case aluOpAnds: + return "ands" + case aluOpBic: + return "bic" + case aluOpEor: + return "eor" + case aluOpAddS: + return "adds" + case aluOpSubS: + return "subs" + case aluOpSMulH: + return "sMulH" + case aluOpUMulH: + return "uMulH" + case aluOpSDiv: + return "sdiv" + case aluOpUDiv: + return "udiv" + case aluOpRotR: + return "ror" + case aluOpLsr: + return "lsr" + case aluOpAsr: + return "asr" + case aluOpLsl: + return "lsl" + case aluOpMAdd: + return "madd" + case aluOpMSub: + return "msub" + } + panic(int(a)) +} + +const ( + // 32/64-bit Add. + aluOpAdd aluOp = iota + // 32/64-bit Subtract. + aluOpSub + // 32/64-bit Bitwise OR. + aluOpOrr + // 32/64-bit Bitwise OR NOT. + aluOpOrn + // 32/64-bit Bitwise AND. + aluOpAnd + // 32/64-bit Bitwise ANDS. + aluOpAnds + // 32/64-bit Bitwise AND NOT. + aluOpBic + // 32/64-bit Bitwise XOR (Exclusive OR). + aluOpEor + // 32/64-bit Add setting flags. + aluOpAddS + // 32/64-bit Subtract setting flags. + aluOpSubS + // Signed multiply, high-word result. + aluOpSMulH + // Unsigned multiply, high-word result. + aluOpUMulH + // 64-bit Signed divide. + aluOpSDiv + // 64-bit Unsigned divide. + aluOpUDiv + // 32/64-bit Rotate right. + aluOpRotR + // 32/64-bit Logical shift right. + aluOpLsr + // 32/64-bit Arithmetic shift right. + aluOpAsr + // 32/64-bit Logical shift left. + aluOpLsl /// Multiply-add + + // MAdd and MSub are only applicable for aluRRRR. + aluOpMAdd + aluOpMSub +) + +// vecOp determines the type of vector operation. Instructions whose kind is one of +// vecOpCnt would use this type. +type vecOp int + +// String implements fmt.Stringer. +func (b vecOp) String() string { + switch b { + case vecOpCnt: + return "cnt" + case vecOpCmeq: + return "cmeq" + case vecOpCmgt: + return "cmgt" + case vecOpCmhi: + return "cmhi" + case vecOpCmge: + return "cmge" + case vecOpCmhs: + return "cmhs" + case vecOpFcmeq: + return "fcmeq" + case vecOpFcmgt: + return "fcmgt" + case vecOpFcmge: + return "fcmge" + case vecOpCmeq0: + return "cmeq0" + case vecOpUaddlv: + return "uaddlv" + case vecOpBit: + return "bit" + case vecOpBic: + return "bic" + case vecOpBsl: + return "bsl" + case vecOpNot: + return "not" + case vecOpAnd: + return "and" + case vecOpOrr: + return "orr" + case vecOpEOR: + return "eor" + case vecOpFadd: + return "fadd" + case vecOpAdd: + return "add" + case vecOpAddp: + return "addp" + case vecOpAddv: + return "addv" + case vecOpSub: + return "sub" + case vecOpFsub: + return "fsub" + case vecOpSmin: + return "smin" + case vecOpUmin: + return "umin" + case vecOpUminv: + return "uminv" + case vecOpSmax: + return "smax" + case vecOpUmax: + return "umax" + case vecOpUmaxp: + return "umaxp" + case vecOpUrhadd: + return "urhadd" + case vecOpFmul: + return "fmul" + case vecOpSqrdmulh: + return "sqrdmulh" + case vecOpMul: + return "mul" + case vecOpUmlal: + return "umlal" + case vecOpFdiv: + return "fdiv" + case vecOpFsqrt: + return "fsqrt" + case vecOpAbs: + return "abs" + case vecOpFabs: + return "fabs" + case vecOpNeg: + return "neg" + case vecOpFneg: + return "fneg" + case vecOpFrintp: + return "frintp" + case vecOpFrintm: + return "frintm" + case vecOpFrintn: + return "frintn" + case vecOpFrintz: + return "frintz" + case vecOpFcvtl: + return "fcvtl" + case vecOpFcvtn: + return "fcvtn" + case vecOpFcvtzu: + return "fcvtzu" + case vecOpFcvtzs: + return "fcvtzs" + case vecOpScvtf: + return "scvtf" + case vecOpUcvtf: + return "ucvtf" + case vecOpSqxtn: + return "sqxtn" + case vecOpUqxtn: + return "uqxtn" + case vecOpSqxtun: + return "sqxtun" + case vecOpRev64: + return "rev64" + case vecOpXtn: + return "xtn" + case vecOpShll: + return "shll" + case vecOpSshl: + return "sshl" + case vecOpSshll: + return "sshll" + case vecOpUshl: + return "ushl" + case vecOpUshll: + return "ushll" + case vecOpSshr: + return "sshr" + case vecOpZip1: + return "zip1" + case vecOpFmin: + return "fmin" + case vecOpFmax: + return "fmax" + case vecOpSmull: + return "smull" + case vecOpSmull2: + return "smull2" + } + panic(int(b)) +} + +const ( + vecOpCnt vecOp = iota + vecOpCmeq0 + vecOpCmeq + vecOpCmgt + vecOpCmhi + vecOpCmge + vecOpCmhs + vecOpFcmeq + vecOpFcmgt + vecOpFcmge + vecOpUaddlv + vecOpBit + vecOpBic + vecOpBsl + vecOpNot + vecOpAnd + vecOpOrr + vecOpEOR + vecOpAdd + vecOpFadd + vecOpAddv + vecOpSqadd + vecOpUqadd + vecOpAddp + vecOpSub + vecOpFsub + vecOpSqsub + vecOpUqsub + vecOpSmin + vecOpUmin + vecOpUminv + vecOpFmin + vecOpSmax + vecOpUmax + vecOpUmaxp + vecOpFmax + vecOpUrhadd + vecOpMul + vecOpFmul + vecOpSqrdmulh + vecOpUmlal + vecOpFdiv + vecOpFsqrt + vecOpAbs + vecOpFabs + vecOpNeg + vecOpFneg + vecOpFrintm + vecOpFrintn + vecOpFrintp + vecOpFrintz + vecOpFcvtl + vecOpFcvtn + vecOpFcvtzs + vecOpFcvtzu + vecOpScvtf + vecOpUcvtf + vecOpSqxtn + vecOpSqxtun + vecOpUqxtn + vecOpRev64 + vecOpXtn + vecOpShll + vecOpSshl + vecOpSshll + vecOpUshl + vecOpUshll + vecOpSshr + vecOpZip1 + vecOpSmull + vecOpSmull2 +) + +// bitOp determines the type of bitwise operation. Instructions whose kind is one of +// bitOpRbit and bitOpClz would use this type. +type bitOp int + +// String implements fmt.Stringer. +func (b bitOp) String() string { + switch b { + case bitOpRbit: + return "rbit" + case bitOpClz: + return "clz" + } + panic(int(b)) +} + +const ( + // 32/64-bit Rbit. + bitOpRbit bitOp = iota + // 32/64-bit Clz. + bitOpClz +) + +// fpuUniOp represents a unary floating-point unit (FPU) operation. +type fpuUniOp byte + +const ( + fpuUniOpNeg fpuUniOp = iota + fpuUniOpCvt32To64 + fpuUniOpCvt64To32 + fpuUniOpSqrt + fpuUniOpRoundPlus + fpuUniOpRoundMinus + fpuUniOpRoundZero + fpuUniOpRoundNearest + fpuUniOpAbs +) + +// String implements the fmt.Stringer. +func (f fpuUniOp) String() string { + switch f { + case fpuUniOpNeg: + return "fneg" + case fpuUniOpCvt32To64: + return "fcvt" + case fpuUniOpCvt64To32: + return "fcvt" + case fpuUniOpSqrt: + return "fsqrt" + case fpuUniOpRoundPlus: + return "frintp" + case fpuUniOpRoundMinus: + return "frintm" + case fpuUniOpRoundZero: + return "frintz" + case fpuUniOpRoundNearest: + return "frintn" + case fpuUniOpAbs: + return "fabs" + } + panic(int(f)) +} + +// fpuBinOp represents a binary floating-point unit (FPU) operation. +type fpuBinOp byte + +const ( + fpuBinOpAdd = iota + fpuBinOpSub + fpuBinOpMul + fpuBinOpDiv + fpuBinOpMax + fpuBinOpMin +) + +// String implements the fmt.Stringer. +func (f fpuBinOp) String() string { + switch f { + case fpuBinOpAdd: + return "fadd" + case fpuBinOpSub: + return "fsub" + case fpuBinOpMul: + return "fmul" + case fpuBinOpDiv: + return "fdiv" + case fpuBinOpMax: + return "fmax" + case fpuBinOpMin: + return "fmin" + } + panic(int(f)) +} + +// extMode represents the mode of a register operand extension. +// For example, aluRRRExtend instructions need this info to determine the extensions. +type extMode byte + +const ( + extModeNone extMode = iota + // extModeZeroExtend64 suggests a zero-extension to 32 bits if the original bit size is less than 32. + extModeZeroExtend32 + // extModeSignExtend64 stands for a sign-extension to 32 bits if the original bit size is less than 32. + extModeSignExtend32 + // extModeZeroExtend64 suggests a zero-extension to 64 bits if the original bit size is less than 64. + extModeZeroExtend64 + // extModeSignExtend64 stands for a sign-extension to 64 bits if the original bit size is less than 64. + extModeSignExtend64 +) + +func (e extMode) bits() byte { + switch e { + case extModeZeroExtend32, extModeSignExtend32: + return 32 + case extModeZeroExtend64, extModeSignExtend64: + return 64 + default: + return 0 + } +} + +func (e extMode) signed() bool { + switch e { + case extModeSignExtend32, extModeSignExtend64: + return true + default: + return false + } +} + +func extModeOf(t ssa.Type, signed bool) extMode { + switch t.Bits() { + case 32: + if signed { + return extModeSignExtend32 + } + return extModeZeroExtend32 + case 64: + if signed { + return extModeSignExtend64 + } + return extModeZeroExtend64 + default: + panic("TODO? do we need narrower than 32 bits?") + } +} + +type extendOp byte + +const ( + extendOpUXTB extendOp = 0b000 + extendOpUXTH extendOp = 0b001 + extendOpUXTW extendOp = 0b010 + // extendOpUXTX does nothing, but convenient symbol that officially exists. See: + // https://stackoverflow.com/questions/72041372/what-do-the-uxtx-and-sxtx-extensions-mean-for-32-bit-aarch64-adds-instruct + extendOpUXTX extendOp = 0b011 + extendOpSXTB extendOp = 0b100 + extendOpSXTH extendOp = 0b101 + extendOpSXTW extendOp = 0b110 + // extendOpSXTX does nothing, but convenient symbol that officially exists. See: + // https://stackoverflow.com/questions/72041372/what-do-the-uxtx-and-sxtx-extensions-mean-for-32-bit-aarch64-adds-instruct + extendOpSXTX extendOp = 0b111 + extendOpNone extendOp = 0xff +) + +func (e extendOp) srcBits() byte { + switch e { + case extendOpUXTB, extendOpSXTB: + return 8 + case extendOpUXTH, extendOpSXTH: + return 16 + case extendOpUXTW, extendOpSXTW: + return 32 + case extendOpUXTX, extendOpSXTX: + return 64 + } + panic(int(e)) +} + +func (e extendOp) String() string { + switch e { + case extendOpUXTB: + return "UXTB" + case extendOpUXTH: + return "UXTH" + case extendOpUXTW: + return "UXTW" + case extendOpUXTX: + return "UXTX" + case extendOpSXTB: + return "SXTB" + case extendOpSXTH: + return "SXTH" + case extendOpSXTW: + return "SXTW" + case extendOpSXTX: + return "SXTX" + } + panic(int(e)) +} + +func extendOpFrom(signed bool, from byte) extendOp { + switch from { + case 8: + if signed { + return extendOpSXTB + } + return extendOpUXTB + case 16: + if signed { + return extendOpSXTH + } + return extendOpUXTH + case 32: + if signed { + return extendOpSXTW + } + return extendOpUXTW + case 64: + if signed { + return extendOpSXTX + } + return extendOpUXTX + } + panic("invalid extendOpFrom") +} + +type shiftOp byte + +const ( + shiftOpLSL shiftOp = 0b00 + shiftOpLSR shiftOp = 0b01 + shiftOpASR shiftOp = 0b10 + shiftOpROR shiftOp = 0b11 +) + +func (s shiftOp) String() string { + switch s { + case shiftOpLSL: + return "lsl" + case shiftOpLSR: + return "lsr" + case shiftOpASR: + return "asr" + case shiftOpROR: + return "ror" + } + panic(int(s)) +} + +const exitSequenceSize = 6 * 4 // 6 instructions as in encodeExitSequence. + +// size returns the size of the instruction in encoded bytes. +func (i *instruction) size() int64 { + switch i.kind { + case exitSequence: + return exitSequenceSize // 5 instructions as in encodeExitSequence. + case nop0, loadConstBlockArg: + return 0 + case emitSourceOffsetInfo: + return 0 + case loadFpuConst32: + if i.u1 == 0 { + return 4 // zero loading can be encoded as a single instruction. + } + return 4 + 4 + 4 + case loadFpuConst64: + if i.u1 == 0 { + return 4 // zero loading can be encoded as a single instruction. + } + return 4 + 4 + 8 + case loadFpuConst128: + if i.u1 == 0 && i.u2 == 0 { + return 4 // zero loading can be encoded as a single instruction. + } + return 4 + 4 + 16 + case brTableSequence: + return 4*4 + int64(i.u2)*4 + default: + return 4 + } +} + +// vecArrangement is the arrangement of data within a vector register. +type vecArrangement byte + +const ( + // vecArrangementNone is an arrangement indicating no data is stored. + vecArrangementNone vecArrangement = iota + // vecArrangement8B is an arrangement of 8 bytes (64-bit vector) + vecArrangement8B + // vecArrangement16B is an arrangement of 16 bytes (128-bit vector) + vecArrangement16B + // vecArrangement4H is an arrangement of 4 half precisions (64-bit vector) + vecArrangement4H + // vecArrangement8H is an arrangement of 8 half precisions (128-bit vector) + vecArrangement8H + // vecArrangement2S is an arrangement of 2 single precisions (64-bit vector) + vecArrangement2S + // vecArrangement4S is an arrangement of 4 single precisions (128-bit vector) + vecArrangement4S + // vecArrangement1D is an arrangement of 1 double precision (64-bit vector) + vecArrangement1D + // vecArrangement2D is an arrangement of 2 double precisions (128-bit vector) + vecArrangement2D + + // Assign each vector size specifier to a vector arrangement ID. + // Instructions can only have an arrangement or a size specifier, but not both, so it + // simplifies the internal representation of vector instructions by being able to + // store either into the same field. + + // vecArrangementB is a size specifier of byte + vecArrangementB + // vecArrangementH is a size specifier of word (16-bit) + vecArrangementH + // vecArrangementS is a size specifier of double word (32-bit) + vecArrangementS + // vecArrangementD is a size specifier of quad word (64-bit) + vecArrangementD + // vecArrangementQ is a size specifier of the entire vector (128-bit) + vecArrangementQ +) + +// String implements fmt.Stringer +func (v vecArrangement) String() (ret string) { + switch v { + case vecArrangement8B: + ret = "8B" + case vecArrangement16B: + ret = "16B" + case vecArrangement4H: + ret = "4H" + case vecArrangement8H: + ret = "8H" + case vecArrangement2S: + ret = "2S" + case vecArrangement4S: + ret = "4S" + case vecArrangement1D: + ret = "1D" + case vecArrangement2D: + ret = "2D" + case vecArrangementB: + ret = "B" + case vecArrangementH: + ret = "H" + case vecArrangementS: + ret = "S" + case vecArrangementD: + ret = "D" + case vecArrangementQ: + ret = "Q" + case vecArrangementNone: + ret = "none" + default: + panic(v) + } + return +} + +// vecIndex is the index of an element of a vector register +type vecIndex byte + +// vecIndexNone indicates no vector index specified. +const vecIndexNone = ^vecIndex(0) + +func ssaLaneToArrangement(lane ssa.VecLane) vecArrangement { + switch lane { + case ssa.VecLaneI8x16: + return vecArrangement16B + case ssa.VecLaneI16x8: + return vecArrangement8H + case ssa.VecLaneI32x4: + return vecArrangement4S + case ssa.VecLaneI64x2: + return vecArrangement2D + case ssa.VecLaneF32x4: + return vecArrangement4S + case ssa.VecLaneF64x2: + return vecArrangement2D + default: + panic(lane) + } +} + +// atomicRmwOp is the type of atomic read-modify-write operation. +type atomicRmwOp byte + +const ( + // atomicRmwOpAdd is an atomic add operation. + atomicRmwOpAdd atomicRmwOp = iota + // atomicRmwOpClr is an atomic clear operation, i.e. AND NOT. + atomicRmwOpClr + // atomicRmwOpSet is an atomic set operation, i.e. OR. + atomicRmwOpSet + // atomicRmwOpEor is an atomic exclusive OR operation. + atomicRmwOpEor + // atomicRmwOpSwp is an atomic swap operation. + atomicRmwOpSwp +) + +// String implements fmt.Stringer +func (a atomicRmwOp) String() string { + switch a { + case atomicRmwOpAdd: + return "ldaddal" + case atomicRmwOpClr: + return "ldclral" + case atomicRmwOpSet: + return "ldsetal" + case atomicRmwOpEor: + return "ldeoral" + case atomicRmwOpSwp: + return "swpal" + } + panic(fmt.Sprintf("unknown atomicRmwOp: %d", a)) +} diff --git a/vendor/github.com/tetratelabs/wazero/internal/engine/wazevo/backend/isa/arm64/instr_encoding.go b/vendor/github.com/tetratelabs/wazero/internal/engine/wazevo/backend/isa/arm64/instr_encoding.go new file mode 100644 index 000000000..227a96474 --- /dev/null +++ b/vendor/github.com/tetratelabs/wazero/internal/engine/wazevo/backend/isa/arm64/instr_encoding.go @@ -0,0 +1,2351 @@ +package arm64 + +import ( + "context" + "fmt" + + "github.com/tetratelabs/wazero/internal/engine/wazevo/backend" + "github.com/tetratelabs/wazero/internal/engine/wazevo/backend/regalloc" + "github.com/tetratelabs/wazero/internal/engine/wazevo/wazevoapi" +) + +// Encode implements backend.Machine Encode. +func (m *machine) Encode(ctx context.Context) error { + m.resolveRelativeAddresses(ctx) + m.encode(m.executableContext.RootInstr) + if l := len(m.compiler.Buf()); l > maxFunctionExecutableSize { + return fmt.Errorf("function size exceeds the limit: %d > %d", l, maxFunctionExecutableSize) + } + return nil +} + +func (m *machine) encode(root *instruction) { + for cur := root; cur != nil; cur = cur.next { + cur.encode(m) + } +} + +func (i *instruction) encode(m *machine) { + c := m.compiler + switch kind := i.kind; kind { + case nop0, emitSourceOffsetInfo, loadConstBlockArg: + case exitSequence: + encodeExitSequence(c, i.rn.reg()) + case ret: + // https://developer.arm.com/documentation/ddi0596/2020-12/Base-Instructions/RET--Return-from-subroutine-?lang=en + c.Emit4Bytes(encodeRet()) + case br: + imm := i.brOffset() + c.Emit4Bytes(encodeUnconditionalBranch(false, imm)) + case call: + // We still don't know the exact address of the function to call, so we emit a placeholder. + c.AddRelocationInfo(i.callFuncRef()) + c.Emit4Bytes(encodeUnconditionalBranch(true, 0)) // 0 = placeholder + case callInd: + c.Emit4Bytes(encodeUnconditionalBranchReg(regNumberInEncoding[i.rn.realReg()], true)) + case store8, store16, store32, store64, fpuStore32, fpuStore64, fpuStore128: + c.Emit4Bytes(encodeLoadOrStore(i.kind, regNumberInEncoding[i.rn.realReg()], i.amode)) + case uLoad8, uLoad16, uLoad32, uLoad64, sLoad8, sLoad16, sLoad32, fpuLoad32, fpuLoad64, fpuLoad128: + c.Emit4Bytes(encodeLoadOrStore(i.kind, regNumberInEncoding[i.rd.realReg()], i.amode)) + case vecLoad1R: + c.Emit4Bytes(encodeVecLoad1R( + regNumberInEncoding[i.rd.realReg()], + regNumberInEncoding[i.rn.realReg()], + vecArrangement(i.u1))) + case condBr: + imm19 := i.condBrOffset() + if imm19%4 != 0 { + panic("imm26 for branch must be a multiple of 4") + } + + imm19U32 := uint32(imm19/4) & 0b111_11111111_11111111 + brCond := i.condBrCond() + switch brCond.kind() { + case condKindRegisterZero: + rt := regNumberInEncoding[brCond.register().RealReg()] + c.Emit4Bytes(encodeCBZCBNZ(rt, false, imm19U32, i.condBr64bit())) + case condKindRegisterNotZero: + rt := regNumberInEncoding[brCond.register().RealReg()] + c.Emit4Bytes(encodeCBZCBNZ(rt, true, imm19U32, i.condBr64bit())) + case condKindCondFlagSet: + // https://developer.arm.com/documentation/ddi0596/2021-12/Base-Instructions/B-cond--Branch-conditionally- + fl := brCond.flag() + c.Emit4Bytes(0b01010100<<24 | (imm19U32 << 5) | uint32(fl)) + default: + panic("BUG") + } + case movN: + c.Emit4Bytes(encodeMoveWideImmediate(0b00, regNumberInEncoding[i.rd.realReg()], i.u1, i.u2, i.u3)) + case movZ: + c.Emit4Bytes(encodeMoveWideImmediate(0b10, regNumberInEncoding[i.rd.realReg()], i.u1, i.u2, i.u3)) + case movK: + c.Emit4Bytes(encodeMoveWideImmediate(0b11, regNumberInEncoding[i.rd.realReg()], i.u1, i.u2, i.u3)) + case mov32: + to, from := i.rd.realReg(), i.rn.realReg() + c.Emit4Bytes(encodeAsMov32(regNumberInEncoding[from], regNumberInEncoding[to])) + case mov64: + to, from := i.rd.realReg(), i.rn.realReg() + toIsSp := to == sp + fromIsSp := from == sp + c.Emit4Bytes(encodeMov64(regNumberInEncoding[to], regNumberInEncoding[from], toIsSp, fromIsSp)) + case loadP64, storeP64: + rt, rt2 := regNumberInEncoding[i.rn.realReg()], regNumberInEncoding[i.rm.realReg()] + amode := i.amode + rn := regNumberInEncoding[amode.rn.RealReg()] + var pre bool + switch amode.kind { + case addressModeKindPostIndex: + case addressModeKindPreIndex: + pre = true + default: + panic("BUG") + } + c.Emit4Bytes(encodePreOrPostIndexLoadStorePair64(pre, kind == loadP64, rn, rt, rt2, amode.imm)) + case loadFpuConst32: + rd := regNumberInEncoding[i.rd.realReg()] + if i.u1 == 0 { + c.Emit4Bytes(encodeVecRRR(vecOpEOR, rd, rd, rd, vecArrangement8B)) + } else { + encodeLoadFpuConst32(c, rd, i.u1) + } + case loadFpuConst64: + rd := regNumberInEncoding[i.rd.realReg()] + if i.u1 == 0 { + c.Emit4Bytes(encodeVecRRR(vecOpEOR, rd, rd, rd, vecArrangement8B)) + } else { + encodeLoadFpuConst64(c, regNumberInEncoding[i.rd.realReg()], i.u1) + } + case loadFpuConst128: + rd := regNumberInEncoding[i.rd.realReg()] + lo, hi := i.u1, i.u2 + if lo == 0 && hi == 0 { + c.Emit4Bytes(encodeVecRRR(vecOpEOR, rd, rd, rd, vecArrangement16B)) + } else { + encodeLoadFpuConst128(c, rd, lo, hi) + } + case aluRRRR: + c.Emit4Bytes(encodeAluRRRR( + aluOp(i.u1), + regNumberInEncoding[i.rd.realReg()], + regNumberInEncoding[i.rn.realReg()], + regNumberInEncoding[i.rm.realReg()], + regNumberInEncoding[i.ra.realReg()], + uint32(i.u3), + )) + case aluRRImmShift: + c.Emit4Bytes(encodeAluRRImm( + aluOp(i.u1), + regNumberInEncoding[i.rd.realReg()], + regNumberInEncoding[i.rn.realReg()], + uint32(i.rm.shiftImm()), + uint32(i.u3), + )) + case aluRRR: + rn := i.rn.realReg() + c.Emit4Bytes(encodeAluRRR( + aluOp(i.u1), + regNumberInEncoding[i.rd.realReg()], + regNumberInEncoding[rn], + regNumberInEncoding[i.rm.realReg()], + i.u3 == 1, + rn == sp, + )) + case aluRRRExtend: + rm, exo, to := i.rm.er() + c.Emit4Bytes(encodeAluRRRExtend( + aluOp(i.u1), + regNumberInEncoding[i.rd.realReg()], + regNumberInEncoding[i.rn.realReg()], + regNumberInEncoding[rm.RealReg()], + exo, + to, + )) + case aluRRRShift: + r, amt, sop := i.rm.sr() + c.Emit4Bytes(encodeAluRRRShift( + aluOp(i.u1), + regNumberInEncoding[i.rd.realReg()], + regNumberInEncoding[i.rn.realReg()], + regNumberInEncoding[r.RealReg()], + uint32(amt), + sop, + i.u3 == 1, + )) + case aluRRBitmaskImm: + c.Emit4Bytes(encodeAluBitmaskImmediate( + aluOp(i.u1), + regNumberInEncoding[i.rd.realReg()], + regNumberInEncoding[i.rn.realReg()], + i.u2, + i.u3 == 1, + )) + case bitRR: + c.Emit4Bytes(encodeBitRR( + bitOp(i.u1), + regNumberInEncoding[i.rd.realReg()], + regNumberInEncoding[i.rn.realReg()], + uint32(i.u2)), + ) + case aluRRImm12: + imm12, shift := i.rm.imm12() + c.Emit4Bytes(encodeAluRRImm12( + aluOp(i.u1), + regNumberInEncoding[i.rd.realReg()], + regNumberInEncoding[i.rn.realReg()], + imm12, shift, + i.u3 == 1, + )) + case fpuRRR: + c.Emit4Bytes(encodeFpuRRR( + fpuBinOp(i.u1), + regNumberInEncoding[i.rd.realReg()], + regNumberInEncoding[i.rn.realReg()], + regNumberInEncoding[i.rm.realReg()], + i.u3 == 1, + )) + case fpuMov64, fpuMov128: + // https://developer.arm.com/documentation/ddi0596/2021-12/SIMD-FP-Instructions/MOV--vector---Move-vector--an-alias-of-ORR--vector--register-- + rd := regNumberInEncoding[i.rd.realReg()] + rn := regNumberInEncoding[i.rn.realReg()] + var q uint32 + if kind == fpuMov128 { + q = 0b1 + } + c.Emit4Bytes(q<<30 | 0b1110101<<21 | rn<<16 | 0b000111<<10 | rn<<5 | rd) + case cSet: + rd := regNumberInEncoding[i.rd.realReg()] + cf := condFlag(i.u1) + if i.u2 == 1 { + // https://developer.arm.com/documentation/ddi0602/2022-03/Base-Instructions/CSETM--Conditional-Set-Mask--an-alias-of-CSINV- + // Note that we set 64bit version here. + c.Emit4Bytes(0b1101101010011111<<16 | uint32(cf.invert())<<12 | 0b011111<<5 | rd) + } else { + // https://developer.arm.com/documentation/ddi0602/2022-06/Base-Instructions/CSET--Conditional-Set--an-alias-of-CSINC- + // Note that we set 64bit version here. + c.Emit4Bytes(0b1001101010011111<<16 | uint32(cf.invert())<<12 | 0b111111<<5 | rd) + } + case extend: + c.Emit4Bytes(encodeExtend(i.u3 == 1, byte(i.u1), byte(i.u2), regNumberInEncoding[i.rd.realReg()], regNumberInEncoding[i.rn.realReg()])) + case fpuCmp: + // https://developer.arm.com/documentation/ddi0596/2020-12/SIMD-FP-Instructions/FCMP--Floating-point-quiet-Compare--scalar--?lang=en + rn, rm := regNumberInEncoding[i.rn.realReg()], regNumberInEncoding[i.rm.realReg()] + var ftype uint32 + if i.u3 == 1 { + ftype = 0b01 // double precision. + } + c.Emit4Bytes(0b1111<<25 | ftype<<22 | 1<<21 | rm<<16 | 0b1<<13 | rn<<5) + case udf: + // https://developer.arm.com/documentation/ddi0596/2020-12/Base-Instructions/UDF--Permanently-Undefined-?lang=en + if wazevoapi.PrintMachineCodeHexPerFunctionDisassemblable { + c.Emit4Bytes(dummyInstruction) + } else { + c.Emit4Bytes(0) + } + case adr: + c.Emit4Bytes(encodeAdr(regNumberInEncoding[i.rd.realReg()], uint32(i.u1))) + case cSel: + c.Emit4Bytes(encodeConditionalSelect( + kind, + regNumberInEncoding[i.rd.realReg()], + regNumberInEncoding[i.rn.realReg()], + regNumberInEncoding[i.rm.realReg()], + condFlag(i.u1), + i.u3 == 1, + )) + case fpuCSel: + c.Emit4Bytes(encodeFpuCSel( + regNumberInEncoding[i.rd.realReg()], + regNumberInEncoding[i.rn.realReg()], + regNumberInEncoding[i.rm.realReg()], + condFlag(i.u1), + i.u3 == 1, + )) + case movToVec: + c.Emit4Bytes(encodeMoveToVec( + regNumberInEncoding[i.rd.realReg()], + regNumberInEncoding[i.rn.realReg()], + vecArrangement(byte(i.u1)), + vecIndex(i.u2), + )) + case movFromVec, movFromVecSigned: + c.Emit4Bytes(encodeMoveFromVec( + regNumberInEncoding[i.rd.realReg()], + regNumberInEncoding[i.rn.realReg()], + vecArrangement(byte(i.u1)), + vecIndex(i.u2), + i.kind == movFromVecSigned, + )) + case vecDup: + c.Emit4Bytes(encodeVecDup( + regNumberInEncoding[i.rd.realReg()], + regNumberInEncoding[i.rn.realReg()], + vecArrangement(byte(i.u1)))) + case vecDupElement: + c.Emit4Bytes(encodeVecDupElement( + regNumberInEncoding[i.rd.realReg()], + regNumberInEncoding[i.rn.realReg()], + vecArrangement(byte(i.u1)), + vecIndex(i.u2))) + case vecExtract: + c.Emit4Bytes(encodeVecExtract( + regNumberInEncoding[i.rd.realReg()], + regNumberInEncoding[i.rn.realReg()], + regNumberInEncoding[i.rm.realReg()], + vecArrangement(byte(i.u1)), + uint32(i.u2))) + case vecPermute: + c.Emit4Bytes(encodeVecPermute( + vecOp(i.u1), + regNumberInEncoding[i.rd.realReg()], + regNumberInEncoding[i.rn.realReg()], + regNumberInEncoding[i.rm.realReg()], + vecArrangement(byte(i.u2)))) + case vecMovElement: + c.Emit4Bytes(encodeVecMovElement( + regNumberInEncoding[i.rd.realReg()], + regNumberInEncoding[i.rn.realReg()], + vecArrangement(i.u1), + uint32(i.u2), uint32(i.u3), + )) + case vecMisc: + c.Emit4Bytes(encodeAdvancedSIMDTwoMisc( + vecOp(i.u1), + regNumberInEncoding[i.rd.realReg()], + regNumberInEncoding[i.rn.realReg()], + vecArrangement(i.u2), + )) + case vecLanes: + c.Emit4Bytes(encodeVecLanes( + vecOp(i.u1), + regNumberInEncoding[i.rd.realReg()], + regNumberInEncoding[i.rn.realReg()], + vecArrangement(i.u2), + )) + case vecShiftImm: + c.Emit4Bytes(encodeVecShiftImm( + vecOp(i.u1), + regNumberInEncoding[i.rd.realReg()], + regNumberInEncoding[i.rn.realReg()], + uint32(i.rm.shiftImm()), + vecArrangement(i.u2), + )) + case vecTbl: + c.Emit4Bytes(encodeVecTbl( + 1, + regNumberInEncoding[i.rd.realReg()], + regNumberInEncoding[i.rn.realReg()], + regNumberInEncoding[i.rm.realReg()], + vecArrangement(i.u2)), + ) + case vecTbl2: + c.Emit4Bytes(encodeVecTbl( + 2, + regNumberInEncoding[i.rd.realReg()], + regNumberInEncoding[i.rn.realReg()], + regNumberInEncoding[i.rm.realReg()], + vecArrangement(i.u2)), + ) + case brTableSequence: + targets := m.jmpTableTargets[i.u1] + encodeBrTableSequence(c, i.rn.reg(), targets) + case fpuToInt, intToFpu: + c.Emit4Bytes(encodeCnvBetweenFloatInt(i)) + case fpuRR: + c.Emit4Bytes(encodeFloatDataOneSource( + fpuUniOp(i.u1), + regNumberInEncoding[i.rd.realReg()], + regNumberInEncoding[i.rn.realReg()], + i.u3 == 1, + )) + case vecRRR: + if op := vecOp(i.u1); op == vecOpBsl || op == vecOpBit || op == vecOpUmlal { + panic(fmt.Sprintf("vecOp %s must use vecRRRRewrite instead of vecRRR", op.String())) + } + fallthrough + case vecRRRRewrite: + c.Emit4Bytes(encodeVecRRR( + vecOp(i.u1), + regNumberInEncoding[i.rd.realReg()], + regNumberInEncoding[i.rn.realReg()], + regNumberInEncoding[i.rm.realReg()], + vecArrangement(i.u2), + )) + case cCmpImm: + // Conditional compare (immediate) in https://developer.arm.com/documentation/ddi0596/2020-12/Index-by-Encoding/Data-Processing----Register?lang=en + sf := uint32(i.u3 & 0b1) + nzcv := uint32(i.u2 & 0b1111) + cond := uint32(condFlag(i.u1)) + imm := uint32(i.rm.data & 0b11111) + rn := regNumberInEncoding[i.rn.realReg()] + c.Emit4Bytes( + sf<<31 | 0b111101001<<22 | imm<<16 | cond<<12 | 0b1<<11 | rn<<5 | nzcv, + ) + case movFromFPSR: + rt := regNumberInEncoding[i.rd.realReg()] + c.Emit4Bytes(encodeSystemRegisterMove(rt, true)) + case movToFPSR: + rt := regNumberInEncoding[i.rn.realReg()] + c.Emit4Bytes(encodeSystemRegisterMove(rt, false)) + case atomicRmw: + c.Emit4Bytes(encodeAtomicRmw( + atomicRmwOp(i.u1), + regNumberInEncoding[i.rm.realReg()], + regNumberInEncoding[i.rd.realReg()], + regNumberInEncoding[i.rn.realReg()], + uint32(i.u2), + )) + case atomicCas: + c.Emit4Bytes(encodeAtomicCas( + regNumberInEncoding[i.rd.realReg()], + regNumberInEncoding[i.rm.realReg()], + regNumberInEncoding[i.rn.realReg()], + uint32(i.u2), + )) + case atomicLoad: + c.Emit4Bytes(encodeAtomicLoadStore( + regNumberInEncoding[i.rn.realReg()], + regNumberInEncoding[i.rd.realReg()], + uint32(i.u2), + 1, + )) + case atomicStore: + c.Emit4Bytes(encodeAtomicLoadStore( + regNumberInEncoding[i.rn.realReg()], + regNumberInEncoding[i.rm.realReg()], + uint32(i.u2), + 0, + )) + case dmb: + c.Emit4Bytes(encodeDMB()) + default: + panic(i.String()) + } +} + +func encodeMov64(rd, rn uint32, toIsSp, fromIsSp bool) uint32 { + if toIsSp || fromIsSp { + // This is an alias of ADD (immediate): + // https://developer.arm.com/documentation/ddi0596/2021-12/Base-Instructions/MOV--to-from-SP---Move-between-register-and-stack-pointer--an-alias-of-ADD--immediate-- + return encodeAddSubtractImmediate(0b100, 0, 0, rn, rd) + } else { + // This is an alias of ORR (shifted register): + // https://developer.arm.com/documentation/ddi0596/2021-12/Base-Instructions/MOV--register---Move--register---an-alias-of-ORR--shifted-register-- + return encodeLogicalShiftedRegister(0b101, 0, rn, 0, regNumberInEncoding[xzr], rd) + } +} + +// encodeSystemRegisterMove encodes as "System register move" in +// https://developer.arm.com/documentation/ddi0596/2020-12/Index-by-Encoding/Branches--Exception-Generating-and-System-instructions?lang=en +// +// Note that currently we only supports read/write of FPSR. +func encodeSystemRegisterMove(rt uint32, fromSystem bool) uint32 { + ret := 0b11010101<<24 | 0b11011<<16 | 0b01000100<<8 | 0b001<<5 | rt + if fromSystem { + ret |= 0b1 << 21 + } + return ret +} + +// encodeVecRRR encodes as either "Advanced SIMD three *" in +// https://developer.arm.com/documentation/ddi0596/2020-12/Index-by-Encoding/Data-Processing----Scalar-Floating-Point-and-Advanced-SIMD?lang=en +func encodeVecRRR(op vecOp, rd, rn, rm uint32, arr vecArrangement) uint32 { + switch op { + case vecOpBit: + _, q := arrToSizeQEncoded(arr) + return encodeAdvancedSIMDThreeSame(rd, rn, rm, 0b00011, 0b10 /* always has size 0b10 */, 0b1, q) + case vecOpBic: + if arr > vecArrangement16B { + panic("unsupported arrangement: " + arr.String()) + } + _, q := arrToSizeQEncoded(arr) + return encodeAdvancedSIMDThreeSame(rd, rn, rm, 0b00011, 0b01 /* always has size 0b01 */, 0b0, q) + case vecOpBsl: + if arr > vecArrangement16B { + panic("unsupported arrangement: " + arr.String()) + } + _, q := arrToSizeQEncoded(arr) + return encodeAdvancedSIMDThreeSame(rd, rn, rm, 0b00011, 0b01 /* always has size 0b01 */, 0b1, q) + case vecOpAnd: + if arr > vecArrangement16B { + panic("unsupported arrangement: " + arr.String()) + } + _, q := arrToSizeQEncoded(arr) + return encodeAdvancedSIMDThreeSame(rd, rn, rm, 0b00011, 0b00 /* always has size 0b00 */, 0b0, q) + case vecOpOrr: + _, q := arrToSizeQEncoded(arr) + return encodeAdvancedSIMDThreeSame(rd, rn, rm, 0b00011, 0b10 /* always has size 0b10 */, 0b0, q) + case vecOpEOR: + size, q := arrToSizeQEncoded(arr) + return encodeAdvancedSIMDThreeSame(rd, rn, rm, 0b00011, size, 0b1, q) + case vecOpCmeq: + size, q := arrToSizeQEncoded(arr) + return encodeAdvancedSIMDThreeSame(rd, rn, rm, 0b10001, size, 0b1, q) + case vecOpCmgt: + size, q := arrToSizeQEncoded(arr) + return encodeAdvancedSIMDThreeSame(rd, rn, rm, 0b00110, size, 0b0, q) + case vecOpCmhi: + size, q := arrToSizeQEncoded(arr) + return encodeAdvancedSIMDThreeSame(rd, rn, rm, 0b00110, size, 0b1, q) + case vecOpCmge: + size, q := arrToSizeQEncoded(arr) + return encodeAdvancedSIMDThreeSame(rd, rn, rm, 0b00111, size, 0b0, q) + case vecOpCmhs: + size, q := arrToSizeQEncoded(arr) + return encodeAdvancedSIMDThreeSame(rd, rn, rm, 0b00111, size, 0b1, q) + case vecOpFcmeq: + var size, q uint32 + switch arr { + case vecArrangement4S: + size, q = 0b00, 0b1 + case vecArrangement2S: + size, q = 0b00, 0b0 + case vecArrangement2D: + size, q = 0b01, 0b1 + default: + panic("unsupported arrangement: " + arr.String()) + } + return encodeAdvancedSIMDThreeSame(rd, rn, rm, 0b11100, size, 0b0, q) + case vecOpFcmgt: + if arr < vecArrangement2S || arr == vecArrangement1D { + panic("unsupported arrangement: " + arr.String()) + } + size, q := arrToSizeQEncoded(arr) + return encodeAdvancedSIMDThreeSame(rd, rn, rm, 0b11100, size, 0b1, q) + case vecOpFcmge: + var size, q uint32 + switch arr { + case vecArrangement4S: + size, q = 0b00, 0b1 + case vecArrangement2S: + size, q = 0b00, 0b0 + case vecArrangement2D: + size, q = 0b01, 0b1 + default: + panic("unsupported arrangement: " + arr.String()) + } + return encodeAdvancedSIMDThreeSame(rd, rn, rm, 0b11100, size, 0b1, q) + case vecOpAdd: + if arr == vecArrangement1D { + panic("unsupported arrangement: " + arr.String()) + } + size, q := arrToSizeQEncoded(arr) + return encodeAdvancedSIMDThreeSame(rd, rn, rm, 0b10000, size, 0b0, q) + case vecOpSqadd: + if arr == vecArrangement1D { + panic("unsupported arrangement: " + arr.String()) + } + size, q := arrToSizeQEncoded(arr) + return encodeAdvancedSIMDThreeSame(rd, rn, rm, 0b00001, size, 0b0, q) + case vecOpUqadd: + if arr == vecArrangement1D { + panic("unsupported arrangement: " + arr.String()) + } + size, q := arrToSizeQEncoded(arr) + return encodeAdvancedSIMDThreeSame(rd, rn, rm, 0b00001, size, 0b1, q) + case vecOpAddp: + if arr == vecArrangement1D { + panic("unsupported arrangement: " + arr.String()) + } + size, q := arrToSizeQEncoded(arr) + return encodeAdvancedSIMDThreeSame(rd, rn, rm, 0b10111, size, 0b0, q) + case vecOpSqsub: + if arr == vecArrangement1D { + panic("unsupported arrangement: " + arr.String()) + } + size, q := arrToSizeQEncoded(arr) + return encodeAdvancedSIMDThreeSame(rd, rn, rm, 0b00101, size, 0b0, q) + case vecOpUqsub: + if arr == vecArrangement1D { + panic("unsupported arrangement: " + arr.String()) + } + size, q := arrToSizeQEncoded(arr) + return encodeAdvancedSIMDThreeSame(rd, rn, rm, 0b00101, size, 0b1, q) + case vecOpSub: + if arr == vecArrangement1D { + panic("unsupported arrangement: " + arr.String()) + } + size, q := arrToSizeQEncoded(arr) + return encodeAdvancedSIMDThreeSame(rd, rn, rm, 0b10000, size, 0b1, q) + case vecOpFmin: + if arr < vecArrangement2S || arr == vecArrangement1D { + panic("unsupported arrangement: " + arr.String()) + } + size, q := arrToSizeQEncoded(arr) + return encodeAdvancedSIMDThreeSame(rd, rn, rm, 0b11110, size, 0b0, q) + case vecOpSmin: + if arr > vecArrangement4S { + panic("unsupported arrangement: " + arr.String()) + } + size, q := arrToSizeQEncoded(arr) + return encodeAdvancedSIMDThreeSame(rd, rn, rm, 0b01101, size, 0b0, q) + case vecOpUmin: + if arr > vecArrangement4S { + panic("unsupported arrangement: " + arr.String()) + } + size, q := arrToSizeQEncoded(arr) + return encodeAdvancedSIMDThreeSame(rd, rn, rm, 0b01101, size, 0b1, q) + case vecOpFmax: + var size, q uint32 + switch arr { + case vecArrangement4S: + size, q = 0b00, 0b1 + case vecArrangement2S: + size, q = 0b00, 0b0 + case vecArrangement2D: + size, q = 0b01, 0b1 + default: + panic("unsupported arrangement: " + arr.String()) + } + return encodeAdvancedSIMDThreeSame(rd, rn, rm, 0b11110, size, 0b0, q) + case vecOpFadd: + var size, q uint32 + switch arr { + case vecArrangement4S: + size, q = 0b00, 0b1 + case vecArrangement2S: + size, q = 0b00, 0b0 + case vecArrangement2D: + size, q = 0b01, 0b1 + default: + panic("unsupported arrangement: " + arr.String()) + } + return encodeAdvancedSIMDThreeSame(rd, rn, rm, 0b11010, size, 0b0, q) + case vecOpFsub: + if arr < vecArrangement2S || arr == vecArrangement1D { + panic("unsupported arrangement: " + arr.String()) + } + size, q := arrToSizeQEncoded(arr) + return encodeAdvancedSIMDThreeSame(rd, rn, rm, 0b11010, size, 0b0, q) + case vecOpFmul: + var size, q uint32 + switch arr { + case vecArrangement4S: + size, q = 0b00, 0b1 + case vecArrangement2S: + size, q = 0b00, 0b0 + case vecArrangement2D: + size, q = 0b01, 0b1 + default: + panic("unsupported arrangement: " + arr.String()) + } + return encodeAdvancedSIMDThreeSame(rd, rn, rm, 0b11011, size, 0b1, q) + case vecOpSqrdmulh: + if arr < vecArrangement4H || arr > vecArrangement4S { + panic("unsupported arrangement: " + arr.String()) + } + size, q := arrToSizeQEncoded(arr) + return encodeAdvancedSIMDThreeSame(rd, rn, rm, 0b10110, size, 0b1, q) + case vecOpFdiv: + var size, q uint32 + switch arr { + case vecArrangement4S: + size, q = 0b00, 0b1 + case vecArrangement2S: + size, q = 0b00, 0b0 + case vecArrangement2D: + size, q = 0b01, 0b1 + default: + panic("unsupported arrangement: " + arr.String()) + } + return encodeAdvancedSIMDThreeSame(rd, rn, rm, 0b11111, size, 0b1, q) + case vecOpSmax: + if arr > vecArrangement4S { + panic("unsupported arrangement: " + arr.String()) + } + size, q := arrToSizeQEncoded(arr) + return encodeAdvancedSIMDThreeSame(rd, rn, rm, 0b01100, size, 0b0, q) + case vecOpUmax: + if arr > vecArrangement4S { + panic("unsupported arrangement: " + arr.String()) + } + size, q := arrToSizeQEncoded(arr) + return encodeAdvancedSIMDThreeSame(rd, rn, rm, 0b01100, size, 0b1, q) + case vecOpUmaxp: + if arr > vecArrangement4S { + panic("unsupported arrangement: " + arr.String()) + } + size, q := arrToSizeQEncoded(arr) + return encodeAdvancedSIMDThreeSame(rd, rn, rm, 0b10100, size, 0b1, q) + case vecOpUrhadd: + if arr > vecArrangement4S { + panic("unsupported arrangement: " + arr.String()) + } + size, q := arrToSizeQEncoded(arr) + return encodeAdvancedSIMDThreeSame(rd, rn, rm, 0b00010, size, 0b1, q) + case vecOpMul: + if arr > vecArrangement4S { + panic("unsupported arrangement: " + arr.String()) + } + size, q := arrToSizeQEncoded(arr) + return encodeAdvancedSIMDThreeSame(rd, rn, rm, 0b10011, size, 0b0, q) + case vecOpUmlal: + if arr > vecArrangement4S { + panic("unsupported arrangement: " + arr.String()) + } + size, q := arrToSizeQEncoded(arr) + return encodeAdvancedSIMDThreeDifferent(rd, rn, rm, 0b1000, size, 0b1, q) + case vecOpSshl: + if arr == vecArrangement1D { + panic("unsupported arrangement: " + arr.String()) + } + size, q := arrToSizeQEncoded(arr) + return encodeAdvancedSIMDThreeSame(rd, rn, rm, 0b01000, size, 0b0, q) + case vecOpUshl: + if arr == vecArrangement1D { + panic("unsupported arrangement: " + arr.String()) + } + size, q := arrToSizeQEncoded(arr) + return encodeAdvancedSIMDThreeSame(rd, rn, rm, 0b01000, size, 0b1, q) + + case vecOpSmull: + if arr > vecArrangement4S { + panic("unsupported arrangement: " + arr.String()) + } + size, _ := arrToSizeQEncoded(arr) + return encodeAdvancedSIMDThreeDifferent(rd, rn, rm, 0b1100, size, 0b0, 0b0) + + case vecOpSmull2: + if arr > vecArrangement4S { + panic("unsupported arrangement: " + arr.String()) + } + size, _ := arrToSizeQEncoded(arr) + return encodeAdvancedSIMDThreeDifferent(rd, rn, rm, 0b1100, size, 0b0, 0b1) + + default: + panic("TODO: " + op.String()) + } +} + +func arrToSizeQEncoded(arr vecArrangement) (size, q uint32) { + switch arr { + case vecArrangement16B: + q = 0b1 + fallthrough + case vecArrangement8B: + size = 0b00 + case vecArrangement8H: + q = 0b1 + fallthrough + case vecArrangement4H: + size = 0b01 + case vecArrangement4S: + q = 0b1 + fallthrough + case vecArrangement2S: + size = 0b10 + case vecArrangement2D: + q = 0b1 + fallthrough + case vecArrangement1D: + size = 0b11 + default: + panic("BUG") + } + return +} + +// encodeAdvancedSIMDThreeSame encodes as "Advanced SIMD three same" in +// https://developer.arm.com/documentation/ddi0596/2020-12/Index-by-Encoding/Data-Processing----Scalar-Floating-Point-and-Advanced-SIMD?lang=en +func encodeAdvancedSIMDThreeSame(rd, rn, rm, opcode, size, U, Q uint32) uint32 { + return Q<<30 | U<<29 | 0b111<<25 | size<<22 | 0b1<<21 | rm<<16 | opcode<<11 | 0b1<<10 | rn<<5 | rd +} + +// encodeAdvancedSIMDThreeDifferent encodes as "Advanced SIMD three different" in +// https://developer.arm.com/documentation/ddi0596/2020-12/Index-by-Encoding/Data-Processing----Scalar-Floating-Point-and-Advanced-SIMD?lang=en +func encodeAdvancedSIMDThreeDifferent(rd, rn, rm, opcode, size, U, Q uint32) uint32 { + return Q<<30 | U<<29 | 0b111<<25 | size<<22 | 0b1<<21 | rm<<16 | opcode<<12 | rn<<5 | rd +} + +// encodeFloatDataOneSource encodes as "Floating-point data-processing (1 source)" in +// https://developer.arm.com/documentation/ddi0596/2020-12/Index-by-Encoding/Data-Processing----Scalar-Floating-Point-and-Advanced-SIMD?lang=en#simd-dp +func encodeFloatDataOneSource(op fpuUniOp, rd, rn uint32, dst64bit bool) uint32 { + var opcode, ptype uint32 + switch op { + case fpuUniOpCvt32To64: + opcode = 0b000101 + case fpuUniOpCvt64To32: + opcode = 0b000100 + ptype = 0b01 + case fpuUniOpNeg: + opcode = 0b000010 + if dst64bit { + ptype = 0b01 + } + case fpuUniOpSqrt: + opcode = 0b000011 + if dst64bit { + ptype = 0b01 + } + case fpuUniOpRoundPlus: + opcode = 0b001001 + if dst64bit { + ptype = 0b01 + } + case fpuUniOpRoundMinus: + opcode = 0b001010 + if dst64bit { + ptype = 0b01 + } + case fpuUniOpRoundZero: + opcode = 0b001011 + if dst64bit { + ptype = 0b01 + } + case fpuUniOpRoundNearest: + opcode = 0b001000 + if dst64bit { + ptype = 0b01 + } + case fpuUniOpAbs: + opcode = 0b000001 + if dst64bit { + ptype = 0b01 + } + default: + panic("BUG") + } + return 0b1111<<25 | ptype<<22 | 0b1<<21 | opcode<<15 | 0b1<<14 | rn<<5 | rd +} + +// encodeCnvBetweenFloatInt encodes as "Conversion between floating-point and integer" in +// https://developer.arm.com/documentation/ddi0596/2020-12/Index-by-Encoding/Data-Processing----Scalar-Floating-Point-and-Advanced-SIMD?lang=en +func encodeCnvBetweenFloatInt(i *instruction) uint32 { + rd := regNumberInEncoding[i.rd.realReg()] + rn := regNumberInEncoding[i.rn.realReg()] + + var opcode uint32 + var rmode uint32 + var ptype uint32 + var sf uint32 + switch i.kind { + case intToFpu: // Either UCVTF or SCVTF. + rmode = 0b00 + + signed := i.u1 == 1 + src64bit := i.u2 == 1 + dst64bit := i.u3 == 1 + if signed { + opcode = 0b010 + } else { + opcode = 0b011 + } + if src64bit { + sf = 0b1 + } + if dst64bit { + ptype = 0b01 + } else { + ptype = 0b00 + } + case fpuToInt: // Either FCVTZU or FCVTZS. + rmode = 0b11 + + signed := i.u1 == 1 + src64bit := i.u2 == 1 + dst64bit := i.u3 == 1 + + if signed { + opcode = 0b000 + } else { + opcode = 0b001 + } + if dst64bit { + sf = 0b1 + } + if src64bit { + ptype = 0b01 + } else { + ptype = 0b00 + } + } + return sf<<31 | 0b1111<<25 | ptype<<22 | 0b1<<21 | rmode<<19 | opcode<<16 | rn<<5 | rd +} + +// encodeAdr encodes a PC-relative ADR instruction. +// https://developer.arm.com/documentation/ddi0602/2022-06/Base-Instructions/ADR--Form-PC-relative-address- +func encodeAdr(rd uint32, offset uint32) uint32 { + if offset >= 1<<20 { + panic("BUG: too large adr instruction") + } + return offset&0b11<<29 | 0b1<<28 | offset&0b1111111111_1111111100<<3 | rd +} + +// encodeFpuCSel encodes as "Floating-point conditional select" in +// https://developer.arm.com/documentation/ddi0596/2020-12/Index-by-Encoding/Data-Processing----Scalar-Floating-Point-and-Advanced-SIMD?lang=en +func encodeFpuCSel(rd, rn, rm uint32, c condFlag, _64bit bool) uint32 { + var ftype uint32 + if _64bit { + ftype = 0b01 // double precision. + } + return 0b1111<<25 | ftype<<22 | 0b1<<21 | rm<<16 | uint32(c)<<12 | 0b11<<10 | rn<<5 | rd +} + +// encodeMoveToVec encodes as "Move general-purpose register to a vector element" (represented as `ins`) in +// https://developer.arm.com/documentation/dui0801/g/A64-SIMD-Vector-Instructions/MOV--vector--from-general- +// https://developer.arm.com/documentation/ddi0596/2020-12/SIMD-FP-Instructions/MOV--from-general---Move-general-purpose-register-to-a-vector-element--an-alias-of-INS--general--?lang=en +func encodeMoveToVec(rd, rn uint32, arr vecArrangement, index vecIndex) uint32 { + var imm5 uint32 + switch arr { + case vecArrangementB: + imm5 |= 0b1 + imm5 |= uint32(index) << 1 + if index > 0b1111 { + panic(fmt.Sprintf("vector index is larger than the allowed bound: %d > 15", index)) + } + case vecArrangementH: + imm5 |= 0b10 + imm5 |= uint32(index) << 2 + if index > 0b111 { + panic(fmt.Sprintf("vector index is larger than the allowed bound: %d > 7", index)) + } + case vecArrangementS: + imm5 |= 0b100 + imm5 |= uint32(index) << 3 + if index > 0b11 { + panic(fmt.Sprintf("vector index is larger than the allowed bound: %d > 3", index)) + } + case vecArrangementD: + imm5 |= 0b1000 + imm5 |= uint32(index) << 4 + if index > 0b1 { + panic(fmt.Sprintf("vector index is larger than the allowed bound: %d > 1", index)) + } + default: + panic("Unsupported arrangement " + arr.String()) + } + + return 0b01001110000<<21 | imm5<<16 | 0b000111<<10 | rn<<5 | rd +} + +// encodeMoveToVec encodes as "Move vector element to another vector element, mov (element)" (represented as `ins`) in +// https://developer.arm.com/documentation/ddi0596/2020-12/SIMD-FP-Instructions/MOV--element---Move-vector-element-to-another-vector-element--an-alias-of-INS--element--?lang=en +// https://developer.arm.com/documentation/ddi0596/2020-12/SIMD-FP-Instructions/INS--element---Insert-vector-element-from-another-vector-element-?lang=en +func encodeVecMovElement(rd, rn uint32, arr vecArrangement, srcIndex, dstIndex uint32) uint32 { + var imm4, imm5 uint32 + switch arr { + case vecArrangementB: + imm5 |= 0b1 + imm5 |= srcIndex << 1 + imm4 = dstIndex + if srcIndex > 0b1111 { + panic(fmt.Sprintf("vector index is larger than the allowed bound: %d > 15", srcIndex)) + } + case vecArrangementH: + imm5 |= 0b10 + imm5 |= srcIndex << 2 + imm4 = dstIndex << 1 + if srcIndex > 0b111 { + panic(fmt.Sprintf("vector index is larger than the allowed bound: %d > 7", srcIndex)) + } + case vecArrangementS: + imm5 |= 0b100 + imm5 |= srcIndex << 3 + imm4 = dstIndex << 2 + if srcIndex > 0b11 { + panic(fmt.Sprintf("vector index is larger than the allowed bound: %d > 3", srcIndex)) + } + case vecArrangementD: + imm5 |= 0b1000 + imm5 |= srcIndex << 4 + imm4 = dstIndex << 3 + if srcIndex > 0b1 { + panic(fmt.Sprintf("vector index is larger than the allowed bound: %d > 1", srcIndex)) + } + default: + panic("Unsupported arrangement " + arr.String()) + } + + return 0b01101110000<<21 | imm5<<16 | imm4<<11 | 0b1<<10 | rn<<5 | rd +} + +// encodeUnconditionalBranchReg encodes as "Unconditional branch (register)" in: +// https://developer.arm.com/documentation/ddi0596/2020-12/Index-by-Encoding/Branches--Exception-Generating-and-System-instructions?lang=en +func encodeUnconditionalBranchReg(rn uint32, link bool) uint32 { + var opc uint32 + if link { + opc = 0b0001 + } + return 0b1101011<<25 | opc<<21 | 0b11111<<16 | rn<<5 +} + +// encodeMoveFromVec encodes as "Move vector element to a general-purpose register" +// (represented as `umov` when dest is 32-bit, `umov` otherwise) in +// https://developer.arm.com/documentation/ddi0596/2020-12/SIMD-FP-Instructions/UMOV--Unsigned-Move-vector-element-to-general-purpose-register-?lang=en +// https://developer.arm.com/documentation/ddi0596/2020-12/SIMD-FP-Instructions/MOV--to-general---Move-vector-element-to-general-purpose-register--an-alias-of-UMOV-?lang=en +func encodeMoveFromVec(rd, rn uint32, arr vecArrangement, index vecIndex, signed bool) uint32 { + var op, imm4, q, imm5 uint32 + switch { + case arr == vecArrangementB: + imm5 |= 0b1 + imm5 |= uint32(index) << 1 + if index > 0b1111 { + panic(fmt.Sprintf("vector index is larger than the allowed bound: %d > 15", index)) + } + case arr == vecArrangementH: + imm5 |= 0b10 + imm5 |= uint32(index) << 2 + if index > 0b111 { + panic(fmt.Sprintf("vector index is larger than the allowed bound: %d > 7", index)) + } + case arr == vecArrangementS && signed: + q = 0b1 + fallthrough + case arr == vecArrangementS: + imm5 |= 0b100 + imm5 |= uint32(index) << 3 + if index > 0b11 { + panic(fmt.Sprintf("vector index is larger than the allowed bound: %d > 3", index)) + } + case arr == vecArrangementD && !signed: + imm5 |= 0b1000 + imm5 |= uint32(index) << 4 + q = 0b1 + if index > 0b1 { + panic(fmt.Sprintf("vector index is larger than the allowed bound: %d > 1", index)) + } + default: + panic("Unsupported arrangement " + arr.String()) + } + if signed { + op, imm4 = 0, 0b0101 + } else { + op, imm4 = 0, 0b0111 + } + return op<<29 | 0b01110000<<21 | q<<30 | imm5<<16 | imm4<<11 | 1<<10 | rn<<5 | rd +} + +// encodeVecDup encodes as "Duplicate general-purpose register to vector" DUP (general) +// (represented as `dup`) +// https://developer.arm.com/documentation/ddi0596/2020-12/SIMD-FP-Instructions/DUP--general---Duplicate-general-purpose-register-to-vector-?lang=en +func encodeVecDup(rd, rn uint32, arr vecArrangement) uint32 { + var q, imm5 uint32 + switch arr { + case vecArrangement8B: + q, imm5 = 0b0, 0b1 + case vecArrangement16B: + q, imm5 = 0b1, 0b1 + case vecArrangement4H: + q, imm5 = 0b0, 0b10 + case vecArrangement8H: + q, imm5 = 0b1, 0b10 + case vecArrangement2S: + q, imm5 = 0b0, 0b100 + case vecArrangement4S: + q, imm5 = 0b1, 0b100 + case vecArrangement2D: + q, imm5 = 0b1, 0b1000 + default: + panic("Unsupported arrangement " + arr.String()) + } + return q<<30 | 0b001110000<<21 | imm5<<16 | 0b000011<<10 | rn<<5 | rd +} + +// encodeVecDup encodes as "Duplicate vector element to vector or scalar" DUP (element). +// (represented as `dup`) +// https://developer.arm.com/documentation/ddi0596/2020-12/SIMD-FP-Instructions/DUP--element---Duplicate-vector-element-to-vector-or-scalar- +func encodeVecDupElement(rd, rn uint32, arr vecArrangement, srcIndex vecIndex) uint32 { + var q, imm5 uint32 + q = 0b1 + switch arr { + case vecArrangementB: + imm5 |= 0b1 + imm5 |= uint32(srcIndex) << 1 + case vecArrangementH: + imm5 |= 0b10 + imm5 |= uint32(srcIndex) << 2 + case vecArrangementS: + imm5 |= 0b100 + imm5 |= uint32(srcIndex) << 3 + case vecArrangementD: + imm5 |= 0b1000 + imm5 |= uint32(srcIndex) << 4 + default: + panic("unsupported arrangement" + arr.String()) + } + + return q<<30 | 0b001110000<<21 | imm5<<16 | 0b1<<10 | rn<<5 | rd +} + +// encodeVecExtract encodes as "Advanced SIMD extract." +// Currently only `ext` is defined. +// https://developer.arm.com/documentation/ddi0602/2023-06/Index-by-Encoding/Data-Processing----Scalar-Floating-Point-and-Advanced-SIMD?lang=en#simd-dp +// https://developer.arm.com/documentation/ddi0602/2023-06/SIMD-FP-Instructions/EXT--Extract-vector-from-pair-of-vectors-?lang=en +func encodeVecExtract(rd, rn, rm uint32, arr vecArrangement, index uint32) uint32 { + var q, imm4 uint32 + switch arr { + case vecArrangement8B: + q, imm4 = 0, 0b0111&uint32(index) + case vecArrangement16B: + q, imm4 = 1, 0b1111&uint32(index) + default: + panic("Unsupported arrangement " + arr.String()) + } + return q<<30 | 0b101110000<<21 | rm<<16 | imm4<<11 | rn<<5 | rd +} + +// encodeVecPermute encodes as "Advanced SIMD permute." +// https://developer.arm.com/documentation/ddi0602/2023-06/Index-by-Encoding/Data-Processing----Scalar-Floating-Point-and-Advanced-SIMD?lang=en#simd-dp +func encodeVecPermute(op vecOp, rd, rn, rm uint32, arr vecArrangement) uint32 { + var q, size, opcode uint32 + switch op { + case vecOpZip1: + opcode = 0b011 + if arr == vecArrangement1D { + panic("unsupported arrangement: " + arr.String()) + } + size, q = arrToSizeQEncoded(arr) + default: + panic("TODO: " + op.String()) + } + return q<<30 | 0b001110<<24 | size<<22 | rm<<16 | opcode<<12 | 0b10<<10 | rn<<5 | rd +} + +// encodeConditionalSelect encodes as "Conditional select" in +// https://developer.arm.com/documentation/ddi0596/2020-12/Index-by-Encoding/Data-Processing----Register?lang=en#condsel +func encodeConditionalSelect(kind instructionKind, rd, rn, rm uint32, c condFlag, _64bit bool) uint32 { + if kind != cSel { + panic("TODO: support other conditional select") + } + + ret := 0b110101<<23 | rm<<16 | uint32(c)<<12 | rn<<5 | rd + if _64bit { + ret |= 0b1 << 31 + } + return ret +} + +const dummyInstruction uint32 = 0x14000000 // "b 0" + +// encodeLoadFpuConst32 encodes the following three instructions: +// +// ldr s8, #8 ;; literal load of data.f32 +// b 8 ;; skip the data +// data.f32 xxxxxxx +func encodeLoadFpuConst32(c backend.Compiler, rd uint32, rawF32 uint64) { + c.Emit4Bytes( + // https://developer.arm.com/documentation/ddi0596/2020-12/SIMD-FP-Instructions/LDR--literal--SIMD-FP---Load-SIMD-FP-Register--PC-relative-literal--?lang=en + 0b111<<26 | (0x8/4)<<5 | rd, + ) + c.Emit4Bytes(encodeUnconditionalBranch(false, 8)) // b 8 + if wazevoapi.PrintMachineCodeHexPerFunctionDisassemblable { + // Inlined data.f32 cannot be disassembled, so we add a dummy instruction here. + c.Emit4Bytes(dummyInstruction) + } else { + c.Emit4Bytes(uint32(rawF32)) // data.f32 xxxxxxx + } +} + +// encodeLoadFpuConst64 encodes the following three instructions: +// +// ldr d8, #8 ;; literal load of data.f64 +// b 12 ;; skip the data +// data.f64 xxxxxxx +func encodeLoadFpuConst64(c backend.Compiler, rd uint32, rawF64 uint64) { + c.Emit4Bytes( + // https://developer.arm.com/documentation/ddi0596/2020-12/SIMD-FP-Instructions/LDR--literal--SIMD-FP---Load-SIMD-FP-Register--PC-relative-literal--?lang=en + 0b1<<30 | 0b111<<26 | (0x8/4)<<5 | rd, + ) + c.Emit4Bytes(encodeUnconditionalBranch(false, 12)) // b 12 + if wazevoapi.PrintMachineCodeHexPerFunctionDisassemblable { + // Inlined data.f64 cannot be disassembled, so we add dummy instructions here. + c.Emit4Bytes(dummyInstruction) + c.Emit4Bytes(dummyInstruction) + } else { + // data.f64 xxxxxxx + c.Emit4Bytes(uint32(rawF64)) + c.Emit4Bytes(uint32(rawF64 >> 32)) + } +} + +// encodeLoadFpuConst128 encodes the following three instructions: +// +// ldr v8, #8 ;; literal load of data.f64 +// b 20 ;; skip the data +// data.v128 xxxxxxx +func encodeLoadFpuConst128(c backend.Compiler, rd uint32, lo, hi uint64) { + c.Emit4Bytes( + // https://developer.arm.com/documentation/ddi0596/2020-12/SIMD-FP-Instructions/LDR--literal--SIMD-FP---Load-SIMD-FP-Register--PC-relative-literal--?lang=en + 0b1<<31 | 0b111<<26 | (0x8/4)<<5 | rd, + ) + c.Emit4Bytes(encodeUnconditionalBranch(false, 20)) // b 20 + if wazevoapi.PrintMachineCodeHexPerFunctionDisassemblable { + // Inlined data.v128 cannot be disassembled, so we add dummy instructions here. + c.Emit4Bytes(dummyInstruction) + c.Emit4Bytes(dummyInstruction) + c.Emit4Bytes(dummyInstruction) + c.Emit4Bytes(dummyInstruction) + } else { + // data.v128 xxxxxxx + c.Emit4Bytes(uint32(lo)) + c.Emit4Bytes(uint32(lo >> 32)) + c.Emit4Bytes(uint32(hi)) + c.Emit4Bytes(uint32(hi >> 32)) + } +} + +// encodeAluRRRR encodes as Data-processing (3 source) in +// https://developer.arm.com/documentation/ddi0596/2020-12/Index-by-Encoding/Data-Processing----Register?lang=en +func encodeAluRRRR(op aluOp, rd, rn, rm, ra, _64bit uint32) uint32 { + var oO, op31 uint32 + switch op { + case aluOpMAdd: + op31, oO = 0b000, 0b0 + case aluOpMSub: + op31, oO = 0b000, 0b1 + default: + panic("TODO/BUG") + } + return _64bit<<31 | 0b11011<<24 | op31<<21 | rm<<16 | oO<<15 | ra<<10 | rn<<5 | rd +} + +// encodeBitRR encodes as Data-processing (1 source) in +// https://developer.arm.com/documentation/ddi0596/2020-12/Index-by-Encoding/Data-Processing----Register?lang=en +func encodeBitRR(op bitOp, rd, rn, _64bit uint32) uint32 { + var opcode2, opcode uint32 + switch op { + case bitOpRbit: + opcode2, opcode = 0b00000, 0b000000 + case bitOpClz: + opcode2, opcode = 0b00000, 0b000100 + default: + panic("TODO/BUG") + } + return _64bit<<31 | 0b1_0_11010110<<21 | opcode2<<15 | opcode<<10 | rn<<5 | rd +} + +func encodeAsMov32(rn, rd uint32) uint32 { + // This is an alias of ORR (shifted register): + // https://developer.arm.com/documentation/ddi0596/2021-12/Base-Instructions/MOV--register---Move--register---an-alias-of-ORR--shifted-register-- + return encodeLogicalShiftedRegister(0b001, 0, rn, 0, regNumberInEncoding[xzr], rd) +} + +// encodeExtend encodes extension instructions. +func encodeExtend(signed bool, from, to byte, rd, rn uint32) uint32 { + // UTXB: https://developer.arm.com/documentation/ddi0596/2020-12/Base-Instructions/UXTB--Unsigned-Extend-Byte--an-alias-of-UBFM-?lang=en + // UTXH: https://developer.arm.com/documentation/ddi0596/2020-12/Base-Instructions/UXTH--Unsigned-Extend-Halfword--an-alias-of-UBFM-?lang=en + // STXB: https://developer.arm.com/documentation/ddi0596/2020-12/Base-Instructions/SXTB--Signed-Extend-Byte--an-alias-of-SBFM- + // STXH: https://developer.arm.com/documentation/ddi0596/2020-12/Base-Instructions/SXTH--Sign-Extend-Halfword--an-alias-of-SBFM- + // STXW: https://developer.arm.com/documentation/ddi0596/2020-12/Base-Instructions/SXTW--Sign-Extend-Word--an-alias-of-SBFM- + var _31to10 uint32 + switch { + case !signed && from == 8 && to == 32: + // 32-bit UXTB + _31to10 = 0b0101001100000000000111 + case !signed && from == 16 && to == 32: + // 32-bit UXTH + _31to10 = 0b0101001100000000001111 + case !signed && from == 8 && to == 64: + // 64-bit UXTB + _31to10 = 0b0101001100000000000111 + case !signed && from == 16 && to == 64: + // 64-bit UXTH + _31to10 = 0b0101001100000000001111 + case !signed && from == 32 && to == 64: + return encodeAsMov32(rn, rd) + case signed && from == 8 && to == 32: + // 32-bit SXTB + _31to10 = 0b0001001100000000000111 + case signed && from == 16 && to == 32: + // 32-bit SXTH + _31to10 = 0b0001001100000000001111 + case signed && from == 8 && to == 64: + // 64-bit SXTB + _31to10 = 0b1001001101000000000111 + case signed && from == 16 && to == 64: + // 64-bit SXTH + _31to10 = 0b1001001101000000001111 + case signed && from == 32 && to == 64: + // SXTW + _31to10 = 0b1001001101000000011111 + default: + panic("BUG") + } + return _31to10<<10 | rn<<5 | rd +} + +func encodeLoadOrStore(kind instructionKind, rt uint32, amode addressMode) uint32 { + var _22to31 uint32 + var bits int64 + switch kind { + case uLoad8: + _22to31 = 0b0011100001 + bits = 8 + case sLoad8: + _22to31 = 0b0011100010 + bits = 8 + case uLoad16: + _22to31 = 0b0111100001 + bits = 16 + case sLoad16: + _22to31 = 0b0111100010 + bits = 16 + case uLoad32: + _22to31 = 0b1011100001 + bits = 32 + case sLoad32: + _22to31 = 0b1011100010 + bits = 32 + case uLoad64: + _22to31 = 0b1111100001 + bits = 64 + case fpuLoad32: + _22to31 = 0b1011110001 + bits = 32 + case fpuLoad64: + _22to31 = 0b1111110001 + bits = 64 + case fpuLoad128: + _22to31 = 0b0011110011 + bits = 128 + case store8: + _22to31 = 0b0011100000 + bits = 8 + case store16: + _22to31 = 0b0111100000 + bits = 16 + case store32: + _22to31 = 0b1011100000 + bits = 32 + case store64: + _22to31 = 0b1111100000 + bits = 64 + case fpuStore32: + _22to31 = 0b1011110000 + bits = 32 + case fpuStore64: + _22to31 = 0b1111110000 + bits = 64 + case fpuStore128: + _22to31 = 0b0011110010 + bits = 128 + default: + panic("BUG") + } + + switch amode.kind { + case addressModeKindRegScaledExtended: + return encodeLoadOrStoreExtended(_22to31, + regNumberInEncoding[amode.rn.RealReg()], + regNumberInEncoding[amode.rm.RealReg()], + rt, true, amode.extOp) + case addressModeKindRegScaled: + return encodeLoadOrStoreExtended(_22to31, + regNumberInEncoding[amode.rn.RealReg()], regNumberInEncoding[amode.rm.RealReg()], + rt, true, extendOpNone) + case addressModeKindRegExtended: + return encodeLoadOrStoreExtended(_22to31, + regNumberInEncoding[amode.rn.RealReg()], regNumberInEncoding[amode.rm.RealReg()], + rt, false, amode.extOp) + case addressModeKindRegReg: + return encodeLoadOrStoreExtended(_22to31, + regNumberInEncoding[amode.rn.RealReg()], regNumberInEncoding[amode.rm.RealReg()], + rt, false, extendOpNone) + case addressModeKindRegSignedImm9: + // e.g. https://developer.arm.com/documentation/ddi0596/2021-12/Base-Instructions/LDUR--Load-Register--unscaled-- + return encodeLoadOrStoreSIMM9(_22to31, 0b00 /* unscaled */, regNumberInEncoding[amode.rn.RealReg()], rt, amode.imm) + case addressModeKindPostIndex: + return encodeLoadOrStoreSIMM9(_22to31, 0b01 /* post index */, regNumberInEncoding[amode.rn.RealReg()], rt, amode.imm) + case addressModeKindPreIndex: + return encodeLoadOrStoreSIMM9(_22to31, 0b11 /* pre index */, regNumberInEncoding[amode.rn.RealReg()], rt, amode.imm) + case addressModeKindRegUnsignedImm12: + // "unsigned immediate" in https://developer.arm.com/documentation/ddi0596/2020-12/Index-by-Encoding/Loads-and-Stores?lang=en + rn := regNumberInEncoding[amode.rn.RealReg()] + imm := amode.imm + div := bits / 8 + if imm != 0 && !offsetFitsInAddressModeKindRegUnsignedImm12(byte(bits), imm) { + panic("BUG") + } + imm /= div + return _22to31<<22 | 0b1<<24 | uint32(imm&0b111111111111)<<10 | rn<<5 | rt + default: + panic("BUG") + } +} + +// encodeVecLoad1R encodes as Load one single-element structure and Replicate to all lanes (of one register) in +// https://developer.arm.com/documentation/ddi0596/2021-12/SIMD-FP-Instructions/LD1R--Load-one-single-element-structure-and-Replicate-to-all-lanes--of-one-register--?lang=en#sa_imm +func encodeVecLoad1R(rt, rn uint32, arr vecArrangement) uint32 { + size, q := arrToSizeQEncoded(arr) + return q<<30 | 0b001101010000001100<<12 | size<<10 | rn<<5 | rt +} + +// encodeAluBitmaskImmediate encodes as Logical (immediate) in +// https://developer.arm.com/documentation/ddi0596/2020-12/Index-by-Encoding/Data-Processing----Immediate?lang=en +func encodeAluBitmaskImmediate(op aluOp, rd, rn uint32, imm uint64, _64bit bool) uint32 { + var _31to23 uint32 + switch op { + case aluOpAnd: + _31to23 = 0b00_100100 + case aluOpOrr: + _31to23 = 0b01_100100 + case aluOpEor: + _31to23 = 0b10_100100 + case aluOpAnds: + _31to23 = 0b11_100100 + default: + panic("BUG") + } + if _64bit { + _31to23 |= 0b1 << 8 + } + immr, imms, N := bitmaskImmediate(imm, _64bit) + return _31to23<<23 | uint32(N)<<22 | uint32(immr)<<16 | uint32(imms)<<10 | rn<<5 | rd +} + +func bitmaskImmediate(c uint64, is64bit bool) (immr, imms, N byte) { + var size uint32 + switch { + case c != c>>32|c<<32: + size = 64 + case c != c>>16|c<<48: + size = 32 + c = uint64(int32(c)) + case c != c>>8|c<<56: + size = 16 + c = uint64(int16(c)) + case c != c>>4|c<<60: + size = 8 + c = uint64(int8(c)) + case c != c>>2|c<<62: + size = 4 + c = uint64(int64(c<<60) >> 60) + default: + size = 2 + c = uint64(int64(c<<62) >> 62) + } + + neg := false + if int64(c) < 0 { + c = ^c + neg = true + } + + onesSize, nonZeroPos := getOnesSequenceSize(c) + if neg { + nonZeroPos = onesSize + nonZeroPos + onesSize = size - onesSize + } + + var mode byte = 32 + if is64bit && size == 64 { + N, mode = 0b1, 64 + } + + immr = byte((size - nonZeroPos) & (size - 1) & uint32(mode-1)) + imms = byte((onesSize - 1) | 63&^(size<<1-1)) + return +} + +func getOnesSequenceSize(x uint64) (size, nonZeroPos uint32) { + // Take 0b00111000 for example: + y := getLowestBit(x) // = 0b0000100 + nonZeroPos = setBitPos(y) // = 2 + size = setBitPos(x+y) - nonZeroPos // = setBitPos(0b0100000) - 2 = 5 - 2 = 3 + return +} + +func setBitPos(x uint64) (ret uint32) { + for ; ; ret++ { + if x == 0b1 { + break + } + x = x >> 1 + } + return +} + +// encodeLoadOrStoreExtended encodes store/load instruction as "extended register offset" in Load/store register (register offset): +// https://developer.arm.com/documentation/ddi0596/2020-12/Index-by-Encoding/Loads-and-Stores?lang=en +func encodeLoadOrStoreExtended(_22to32 uint32, rn, rm, rt uint32, scaled bool, extOp extendOp) uint32 { + var option uint32 + switch extOp { + case extendOpUXTW: + option = 0b010 + case extendOpSXTW: + option = 0b110 + case extendOpNone: + option = 0b111 + default: + panic("BUG") + } + var s uint32 + if scaled { + s = 0b1 + } + return _22to32<<22 | 0b1<<21 | rm<<16 | option<<13 | s<<12 | 0b10<<10 | rn<<5 | rt +} + +// encodeLoadOrStoreSIMM9 encodes store/load instruction as one of post-index, pre-index or unscaled immediate as in +// https://developer.arm.com/documentation/ddi0596/2020-12/Index-by-Encoding/Loads-and-Stores?lang=en +func encodeLoadOrStoreSIMM9(_22to32, _1011 uint32, rn, rt uint32, imm9 int64) uint32 { + return _22to32<<22 | (uint32(imm9)&0b111111111)<<12 | _1011<<10 | rn<<5 | rt +} + +// encodeFpuRRR encodes as single or double precision (depending on `_64bit`) of Floating-point data-processing (2 source) in +// https://developer.arm.com/documentation/ddi0596/2020-12/Index-by-Encoding/Data-Processing----Scalar-Floating-Point-and-Advanced-SIMD?lang=en +func encodeFpuRRR(op fpuBinOp, rd, rn, rm uint32, _64bit bool) (ret uint32) { + // https://developer.arm.com/documentation/ddi0596/2021-12/SIMD-FP-Instructions/ADD--vector--Add-vectors--scalar--floating-point-and-integer- + var opcode uint32 + switch op { + case fpuBinOpAdd: + opcode = 0b0010 + case fpuBinOpSub: + opcode = 0b0011 + case fpuBinOpMul: + opcode = 0b0000 + case fpuBinOpDiv: + opcode = 0b0001 + case fpuBinOpMax: + opcode = 0b0100 + case fpuBinOpMin: + opcode = 0b0101 + default: + panic("BUG") + } + var ptype uint32 + if _64bit { + ptype = 0b01 + } + return 0b1111<<25 | ptype<<22 | 0b1<<21 | rm<<16 | opcode<<12 | 0b1<<11 | rn<<5 | rd +} + +// encodeAluRRImm12 encodes as Add/subtract (immediate) in +// https://developer.arm.com/documentation/ddi0596/2020-12/Index-by-Encoding/Data-Processing----Immediate?lang=en +func encodeAluRRImm12(op aluOp, rd, rn uint32, imm12 uint16, shiftBit byte, _64bit bool) uint32 { + var _31to24 uint32 + switch op { + case aluOpAdd: + _31to24 = 0b00_10001 + case aluOpAddS: + _31to24 = 0b01_10001 + case aluOpSub: + _31to24 = 0b10_10001 + case aluOpSubS: + _31to24 = 0b11_10001 + default: + panic("BUG") + } + if _64bit { + _31to24 |= 0b1 << 7 + } + return _31to24<<24 | uint32(shiftBit)<<22 | uint32(imm12&0b111111111111)<<10 | rn<<5 | rd +} + +// encodeAluRRR encodes as Data Processing (shifted register), depending on aluOp. +// https://developer.arm.com/documentation/ddi0596/2020-12/Index-by-Encoding/Data-Processing----Register?lang=en#addsub_shift +func encodeAluRRRShift(op aluOp, rd, rn, rm, amount uint32, shiftOp shiftOp, _64bit bool) uint32 { + var _31to24 uint32 + var opc, n uint32 + switch op { + case aluOpAdd: + _31to24 = 0b00001011 + case aluOpAddS: + _31to24 = 0b00101011 + case aluOpSub: + _31to24 = 0b01001011 + case aluOpSubS: + _31to24 = 0b01101011 + case aluOpAnd, aluOpOrr, aluOpEor, aluOpAnds: + // "Logical (shifted register)". + switch op { + case aluOpAnd: + // all zeros + case aluOpOrr: + opc = 0b01 + case aluOpEor: + opc = 0b10 + case aluOpAnds: + opc = 0b11 + } + _31to24 = 0b000_01010 + default: + panic(op.String()) + } + + if _64bit { + _31to24 |= 0b1 << 7 + } + + var shift uint32 + switch shiftOp { + case shiftOpLSL: + shift = 0b00 + case shiftOpLSR: + shift = 0b01 + case shiftOpASR: + shift = 0b10 + default: + panic(shiftOp.String()) + } + return opc<<29 | n<<21 | _31to24<<24 | shift<<22 | rm<<16 | (amount << 10) | (rn << 5) | rd +} + +// "Add/subtract (extended register)" in +// https://developer.arm.com/documentation/ddi0596/2020-12/Index-by-Encoding/Data-Processing----Register?lang=en#addsub_ext +func encodeAluRRRExtend(ao aluOp, rd, rn, rm uint32, extOp extendOp, to byte) uint32 { + var s, op uint32 + switch ao { + case aluOpAdd: + op = 0b0 + case aluOpAddS: + op, s = 0b0, 0b1 + case aluOpSub: + op = 0b1 + case aluOpSubS: + op, s = 0b1, 0b1 + default: + panic("BUG: extended register operand can be used only for add/sub") + } + + var sf uint32 + if to == 64 { + sf = 0b1 + } + + var option uint32 + switch extOp { + case extendOpUXTB: + option = 0b000 + case extendOpUXTH: + option = 0b001 + case extendOpUXTW: + option = 0b010 + case extendOpSXTB: + option = 0b100 + case extendOpSXTH: + option = 0b101 + case extendOpSXTW: + option = 0b110 + case extendOpSXTX, extendOpUXTX: + panic(fmt.Sprintf("%s is essentially noop, and should be handled much earlier than encoding", extOp.String())) + } + return sf<<31 | op<<30 | s<<29 | 0b1011001<<21 | rm<<16 | option<<13 | rn<<5 | rd +} + +// encodeAluRRR encodes as Data Processing (register), depending on aluOp. +// https://developer.arm.com/documentation/ddi0596/2020-12/Index-by-Encoding/Data-Processing----Register?lang=en +func encodeAluRRR(op aluOp, rd, rn, rm uint32, _64bit, isRnSp bool) uint32 { + var _31to21, _15to10 uint32 + switch op { + case aluOpAdd: + if isRnSp { + // "Extended register" with UXTW. + _31to21 = 0b00001011_001 + _15to10 = 0b011000 + } else { + // "Shifted register" with shift = 0 + _31to21 = 0b00001011_000 + } + case aluOpAddS: + if isRnSp { + panic("TODO") + } + // "Shifted register" with shift = 0 + _31to21 = 0b00101011_000 + case aluOpSub: + if isRnSp { + // "Extended register" with UXTW. + _31to21 = 0b01001011_001 + _15to10 = 0b011000 + } else { + // "Shifted register" with shift = 0 + _31to21 = 0b01001011_000 + } + case aluOpSubS: + if isRnSp { + panic("TODO") + } + // "Shifted register" with shift = 0 + _31to21 = 0b01101011_000 + case aluOpAnd, aluOpOrr, aluOpOrn, aluOpEor, aluOpAnds: + // "Logical (shifted register)". + var opc, n uint32 + switch op { + case aluOpAnd: + // all zeros + case aluOpOrr: + opc = 0b01 + case aluOpOrn: + opc = 0b01 + n = 1 + case aluOpEor: + opc = 0b10 + case aluOpAnds: + opc = 0b11 + } + _31to21 = 0b000_01010_000 | opc<<8 | n + case aluOpLsl, aluOpAsr, aluOpLsr, aluOpRotR: + // "Data-processing (2 source)". + _31to21 = 0b00011010_110 + switch op { + case aluOpLsl: + _15to10 = 0b001000 + case aluOpLsr: + _15to10 = 0b001001 + case aluOpAsr: + _15to10 = 0b001010 + case aluOpRotR: + _15to10 = 0b001011 + } + case aluOpSDiv: + // "Data-processing (2 source)". + _31to21 = 0b11010110 + _15to10 = 0b000011 + case aluOpUDiv: + // "Data-processing (2 source)". + _31to21 = 0b11010110 + _15to10 = 0b000010 + default: + panic(op.String()) + } + if _64bit { + _31to21 |= 0b1 << 10 + } + return _31to21<<21 | rm<<16 | (_15to10 << 10) | (rn << 5) | rd +} + +// encodeLogicalShiftedRegister encodes as Logical (shifted register) in +// https://developer.arm.com/documentation/ddi0596/2020-12/Index-by-Encoding/Data-Processing----Register?lang=en +func encodeLogicalShiftedRegister(sf_opc uint32, shift_N uint32, rm uint32, imm6 uint32, rn, rd uint32) (ret uint32) { + ret = sf_opc << 29 + ret |= 0b01010 << 24 + ret |= shift_N << 21 + ret |= rm << 16 + ret |= imm6 << 10 + ret |= rn << 5 + ret |= rd + return +} + +// encodeAddSubtractImmediate encodes as Add/subtract (immediate) in +// https://developer.arm.com/documentation/ddi0596/2020-12/Index-by-Encoding/Data-Processing----Immediate?lang=en +func encodeAddSubtractImmediate(sf_op_s uint32, sh uint32, imm12 uint32, rn, rd uint32) (ret uint32) { + ret = sf_op_s << 29 + ret |= 0b100010 << 23 + ret |= sh << 22 + ret |= imm12 << 10 + ret |= rn << 5 + ret |= rd + return +} + +// encodePreOrPostIndexLoadStorePair64 encodes as Load/store pair (pre/post-indexed) in +// https://developer.arm.com/documentation/ddi0596/2021-12/Base-Instructions/LDP--Load-Pair-of-Registers- +// https://developer.arm.com/documentation/ddi0596/2021-12/Base-Instructions/STP--Store-Pair-of-Registers- +func encodePreOrPostIndexLoadStorePair64(pre bool, load bool, rn, rt, rt2 uint32, imm7 int64) (ret uint32) { + if imm7%8 != 0 { + panic("imm7 for pair load/store must be a multiple of 8") + } + imm7 /= 8 + ret = rt + ret |= rn << 5 + ret |= rt2 << 10 + ret |= (uint32(imm7) & 0b1111111) << 15 + if load { + ret |= 0b1 << 22 + } + ret |= 0b101010001 << 23 + if pre { + ret |= 0b1 << 24 + } + return +} + +// encodeUnconditionalBranch encodes as B or BL instructions: +// https://developer.arm.com/documentation/ddi0596/2021-12/Base-Instructions/B--Branch- +// https://developer.arm.com/documentation/ddi0596/2021-12/Base-Instructions/BL--Branch-with-Link- +func encodeUnconditionalBranch(link bool, imm26 int64) (ret uint32) { + if imm26%4 != 0 { + panic("imm26 for branch must be a multiple of 4") + } + imm26 /= 4 + ret = uint32(imm26 & 0b11_11111111_11111111_11111111) + ret |= 0b101 << 26 + if link { + ret |= 0b1 << 31 + } + return +} + +// encodeCBZCBNZ encodes as either CBZ or CBNZ: +// https://developer.arm.com/documentation/ddi0596/2021-12/Base-Instructions/CBZ--Compare-and-Branch-on-Zero- +// https://developer.arm.com/documentation/ddi0596/2021-12/Base-Instructions/CBNZ--Compare-and-Branch-on-Nonzero- +func encodeCBZCBNZ(rt uint32, nz bool, imm19 uint32, _64bit bool) (ret uint32) { + ret = rt + ret |= imm19 << 5 + if nz { + ret |= 1 << 24 + } + ret |= 0b11010 << 25 + if _64bit { + ret |= 1 << 31 + } + return +} + +// encodeMoveWideImmediate encodes as either MOVZ, MOVN or MOVK, as Move wide (immediate) in +// https://developer.arm.com/documentation/ddi0596/2020-12/Index-by-Encoding/Data-Processing----Immediate?lang=en +// +// "shift" must have been divided by 16 at this point. +func encodeMoveWideImmediate(opc uint32, rd uint32, imm, shift, _64bit uint64) (ret uint32) { + ret = rd + ret |= uint32(imm&0xffff) << 5 + ret |= (uint32(shift)) << 21 + ret |= 0b100101 << 23 + ret |= opc << 29 + ret |= uint32(_64bit) << 31 + return +} + +// encodeAluRRImm encodes as "Bitfield" in +// https://developer.arm.com/documentation/ddi0596/2020-12/Index-by-Encoding/Data-Processing----Immediate?lang=en#log_imm +func encodeAluRRImm(op aluOp, rd, rn, amount, _64bit uint32) uint32 { + var opc uint32 + var immr, imms uint32 + switch op { + case aluOpLsl: + // LSL (immediate) is an alias for UBFM. + // https://developer.arm.com/documentation/ddi0596/2021-12/Base-Instructions/UBFM--Unsigned-Bitfield-Move-?lang=en + opc = 0b10 + if amount == 0 { + // This can be encoded as NOP, but we don't do it for consistency: lsr xn, xm, #0 + immr = 0 + if _64bit == 1 { + imms = 0b111111 + } else { + imms = 0b11111 + } + } else { + if _64bit == 1 { + immr = 64 - amount + } else { + immr = (32 - amount) & 0b11111 + } + imms = immr - 1 + } + case aluOpLsr: + // LSR (immediate) is an alias for UBFM. + // https://developer.arm.com/documentation/ddi0596/2021-12/Base-Instructions/LSR--immediate---Logical-Shift-Right--immediate---an-alias-of-UBFM-?lang=en + opc = 0b10 + imms, immr = 0b011111|_64bit<<5, amount + case aluOpAsr: + // ASR (immediate) is an alias for SBFM. + // https://developer.arm.com/documentation/ddi0596/2020-12/Base-Instructions/SBFM--Signed-Bitfield-Move-?lang=en + opc = 0b00 + imms, immr = 0b011111|_64bit<<5, amount + default: + panic(op.String()) + } + return _64bit<<31 | opc<<29 | 0b100110<<23 | _64bit<<22 | immr<<16 | imms<<10 | rn<<5 | rd +} + +// encodeVecLanes encodes as Data Processing (Advanced SIMD across lanes) depending on vecOp in +// https://developer.arm.com/documentation/ddi0596/2020-12/Index-by-Encoding/Data-Processing----Scalar-Floating-Point-and-Advanced-SIMD?lang=en +func encodeVecLanes(op vecOp, rd uint32, rn uint32, arr vecArrangement) uint32 { + var u, q, size, opcode uint32 + switch arr { + case vecArrangement8B: + q, size = 0b0, 0b00 + case vecArrangement16B: + q, size = 0b1, 0b00 + case vecArrangement4H: + q, size = 0, 0b01 + case vecArrangement8H: + q, size = 1, 0b01 + case vecArrangement4S: + q, size = 1, 0b10 + default: + panic("unsupported arrangement: " + arr.String()) + } + switch op { + case vecOpUaddlv: + u, opcode = 1, 0b00011 + case vecOpUminv: + u, opcode = 1, 0b11010 + case vecOpAddv: + u, opcode = 0, 0b11011 + default: + panic("unsupported or illegal vecOp: " + op.String()) + } + return q<<30 | u<<29 | 0b1110<<24 | size<<22 | 0b11000<<17 | opcode<<12 | 0b10<<10 | rn<<5 | rd +} + +// encodeVecLanes encodes as Data Processing (Advanced SIMD scalar shift by immediate) depending on vecOp in +// https://developer.arm.com/documentation/ddi0596/2020-12/Index-by-Encoding/Data-Processing----Scalar-Floating-Point-and-Advanced-SIMD?lang=en +func encodeVecShiftImm(op vecOp, rd uint32, rn, amount uint32, arr vecArrangement) uint32 { + var u, q, immh, immb, opcode uint32 + switch op { + case vecOpSshll: + u, opcode = 0b0, 0b10100 + case vecOpUshll: + u, opcode = 0b1, 0b10100 + case vecOpSshr: + u, opcode = 0, 0b00000 + default: + panic("unsupported or illegal vecOp: " + op.String()) + } + switch arr { + case vecArrangement16B: + q = 0b1 + fallthrough + case vecArrangement8B: + immh = 0b0001 + immb = 8 - uint32(amount&0b111) + case vecArrangement8H: + q = 0b1 + fallthrough + case vecArrangement4H: + v := 16 - uint32(amount&0b1111) + immb = v & 0b111 + immh = 0b0010 | (v >> 3) + case vecArrangement4S: + q = 0b1 + fallthrough + case vecArrangement2S: + v := 32 - uint32(amount&0b11111) + immb = v & 0b111 + immh = 0b0100 | (v >> 3) + case vecArrangement2D: + q = 0b1 + v := 64 - uint32(amount&0b111111) + immb = v & 0b111 + immh = 0b1000 | (v >> 3) + default: + panic("unsupported arrangement: " + arr.String()) + } + return q<<30 | u<<29 | 0b011110<<23 | immh<<19 | immb<<16 | 0b000001<<10 | opcode<<11 | 0b1<<10 | rn<<5 | rd +} + +// encodeVecTbl encodes as Data Processing (Advanced SIMD table lookup) in +// https://developer.arm.com/documentation/ddi0596/2020-12/Index-by-Encoding/Data-Processing----Scalar-Floating-Point-and-Advanced-SIMD?lang=en#simd-dp +// +// Note: tblOp may encode tbl1, tbl2... in the future. Currently, it is ignored. +func encodeVecTbl(nregs, rd, rn, rm uint32, arr vecArrangement) uint32 { + var q, op2, len, op uint32 + + switch nregs { + case 1: + // tbl: single-register + len = 0b00 + case 2: + // tbl2: 2-register table + len = 0b01 + default: + panic(fmt.Sprintf("unsupported number or registers %d", nregs)) + } + switch arr { + case vecArrangement8B: + q = 0b0 + case vecArrangement16B: + q = 0b1 + default: + panic("unsupported arrangement: " + arr.String()) + } + + return q<<30 | 0b001110<<24 | op2<<22 | rm<<16 | len<<13 | op<<12 | rn<<5 | rd +} + +// encodeVecMisc encodes as Data Processing (Advanced SIMD two-register miscellaneous) depending on vecOp in +// https://developer.arm.com/documentation/ddi0596/2020-12/Index-by-Encoding/Data-Processing----Scalar-Floating-Point-and-Advanced-SIMD?lang=en#simd-dp +func encodeAdvancedSIMDTwoMisc(op vecOp, rd, rn uint32, arr vecArrangement) uint32 { + var q, u, size, opcode uint32 + switch op { + case vecOpCnt: + opcode = 0b00101 + switch arr { + case vecArrangement8B: + q, size = 0b0, 0b00 + case vecArrangement16B: + q, size = 0b1, 0b00 + default: + panic("unsupported arrangement: " + arr.String()) + } + case vecOpCmeq0: + if arr == vecArrangement1D { + panic("unsupported arrangement: " + arr.String()) + } + opcode = 0b01001 + size, q = arrToSizeQEncoded(arr) + case vecOpNot: + u = 1 + opcode = 0b00101 + switch arr { + case vecArrangement8B: + q, size = 0b0, 0b00 + case vecArrangement16B: + q, size = 0b1, 0b00 + default: + panic("unsupported arrangement: " + arr.String()) + } + case vecOpAbs: + if arr == vecArrangement1D { + panic("unsupported arrangement: " + arr.String()) + } + opcode = 0b01011 + u = 0b0 + size, q = arrToSizeQEncoded(arr) + case vecOpNeg: + if arr == vecArrangement1D { + panic("unsupported arrangement: " + arr.String()) + } + opcode = 0b01011 + u = 0b1 + size, q = arrToSizeQEncoded(arr) + case vecOpFabs: + if arr < vecArrangement2S || arr == vecArrangement1D { + panic("unsupported arrangement: " + arr.String()) + } + opcode = 0b01111 + u = 0b0 + size, q = arrToSizeQEncoded(arr) + case vecOpFneg: + if arr < vecArrangement2S || arr == vecArrangement1D { + panic("unsupported arrangement: " + arr.String()) + } + opcode = 0b01111 + u = 0b1 + size, q = arrToSizeQEncoded(arr) + case vecOpFrintm: + u = 0b0 + opcode = 0b11001 + switch arr { + case vecArrangement2S: + q, size = 0b0, 0b00 + case vecArrangement4S: + q, size = 0b1, 0b00 + case vecArrangement2D: + q, size = 0b1, 0b01 + default: + panic("unsupported arrangement: " + arr.String()) + } + case vecOpFrintn: + u = 0b0 + opcode = 0b11000 + switch arr { + case vecArrangement2S: + q, size = 0b0, 0b00 + case vecArrangement4S: + q, size = 0b1, 0b00 + case vecArrangement2D: + q, size = 0b1, 0b01 + default: + panic("unsupported arrangement: " + arr.String()) + } + case vecOpFrintp: + u = 0b0 + opcode = 0b11000 + if arr < vecArrangement2S || arr == vecArrangement1D { + panic("unsupported arrangement: " + arr.String()) + } + size, q = arrToSizeQEncoded(arr) + case vecOpFrintz: + u = 0b0 + opcode = 0b11001 + if arr < vecArrangement2S || arr == vecArrangement1D { + panic("unsupported arrangement: " + arr.String()) + } + size, q = arrToSizeQEncoded(arr) + case vecOpFsqrt: + if arr < vecArrangement2S || arr == vecArrangement1D { + panic("unsupported arrangement: " + arr.String()) + } + opcode = 0b11111 + u = 0b1 + size, q = arrToSizeQEncoded(arr) + case vecOpFcvtl: + opcode = 0b10111 + u = 0b0 + switch arr { + case vecArrangement2S: + size, q = 0b01, 0b0 + case vecArrangement4H: + size, q = 0b00, 0b0 + default: + panic("unsupported arrangement: " + arr.String()) + } + case vecOpFcvtn: + opcode = 0b10110 + u = 0b0 + switch arr { + case vecArrangement2S: + size, q = 0b01, 0b0 + case vecArrangement4H: + size, q = 0b00, 0b0 + default: + panic("unsupported arrangement: " + arr.String()) + } + case vecOpFcvtzs: + opcode = 0b11011 + u = 0b0 + switch arr { + case vecArrangement2S: + q, size = 0b0, 0b10 + case vecArrangement4S: + q, size = 0b1, 0b10 + case vecArrangement2D: + q, size = 0b1, 0b11 + default: + panic("unsupported arrangement: " + arr.String()) + } + case vecOpFcvtzu: + opcode = 0b11011 + u = 0b1 + switch arr { + case vecArrangement2S: + q, size = 0b0, 0b10 + case vecArrangement4S: + q, size = 0b1, 0b10 + case vecArrangement2D: + q, size = 0b1, 0b11 + default: + panic("unsupported arrangement: " + arr.String()) + } + case vecOpScvtf: + opcode = 0b11101 + u = 0b0 + switch arr { + case vecArrangement4S: + q, size = 0b1, 0b00 + case vecArrangement2S: + q, size = 0b0, 0b00 + case vecArrangement2D: + q, size = 0b1, 0b01 + default: + panic("unsupported arrangement: " + arr.String()) + } + case vecOpUcvtf: + opcode = 0b11101 + u = 0b1 + switch arr { + case vecArrangement4S: + q, size = 0b1, 0b00 + case vecArrangement2S: + q, size = 0b0, 0b00 + case vecArrangement2D: + q, size = 0b1, 0b01 + default: + panic("unsupported arrangement: " + arr.String()) + } + case vecOpSqxtn: + // When q == 1 it encodes sqxtn2 (operates on upper 64 bits). + opcode = 0b10100 + u = 0b0 + if arr > vecArrangement4S { + panic("unsupported arrangement: " + arr.String()) + } + size, q = arrToSizeQEncoded(arr) + case vecOpUqxtn: + // When q == 1 it encodes uqxtn2 (operates on upper 64 bits). + opcode = 0b10100 + u = 0b1 + if arr > vecArrangement4S { + panic("unsupported arrangement: " + arr.String()) + } + size, q = arrToSizeQEncoded(arr) + case vecOpSqxtun: + // When q == 1 it encodes sqxtun2 (operates on upper 64 bits). + opcode = 0b10010 // 0b10100 + u = 0b1 + if arr > vecArrangement4S { + panic("unsupported arrangement: " + arr.String()) + } + size, q = arrToSizeQEncoded(arr) + case vecOpRev64: + opcode = 0b00000 + size, q = arrToSizeQEncoded(arr) + case vecOpXtn: + u = 0b0 + opcode = 0b10010 + size, q = arrToSizeQEncoded(arr) + case vecOpShll: + u = 0b1 + opcode = 0b10011 + switch arr { + case vecArrangement8B: + q, size = 0b0, 0b00 + case vecArrangement4H: + q, size = 0b0, 0b01 + case vecArrangement2S: + q, size = 0b0, 0b10 + default: + panic("unsupported arrangement: " + arr.String()) + } + default: + panic("unsupported or illegal vecOp: " + op.String()) + } + return q<<30 | u<<29 | 0b01110<<24 | size<<22 | 0b10000<<17 | opcode<<12 | 0b10<<10 | rn<<5 | rd +} + +// brTableSequenceOffsetTableBegin is the offset inside the brTableSequence where the table begins after 4 instructions +const brTableSequenceOffsetTableBegin = 16 + +func encodeBrTableSequence(c backend.Compiler, index regalloc.VReg, targets []uint32) { + tmpRegNumber := regNumberInEncoding[tmp] + indexNumber := regNumberInEncoding[index.RealReg()] + + // adr tmpReg, PC+16 (PC+16 is the address of the first label offset) + // ldrsw index, [tmpReg, index, UXTW 2] ;; index = int64(*(tmpReg + index*8)) + // add tmpReg, tmpReg, index + // br tmpReg + // [offset_to_l1, offset_to_l2, ..., offset_to_lN] + c.Emit4Bytes(encodeAdr(tmpRegNumber, 16)) + c.Emit4Bytes(encodeLoadOrStore(sLoad32, indexNumber, + addressMode{kind: addressModeKindRegScaledExtended, rn: tmpRegVReg, rm: index, extOp: extendOpUXTW}, + )) + c.Emit4Bytes(encodeAluRRR(aluOpAdd, tmpRegNumber, tmpRegNumber, indexNumber, true, false)) + c.Emit4Bytes(encodeUnconditionalBranchReg(tmpRegNumber, false)) + + // Offsets are resolved in ResolveRelativeAddress phase. + for _, offset := range targets { + if wazevoapi.PrintMachineCodeHexPerFunctionDisassemblable { + // Inlined offset tables cannot be disassembled properly, so pad dummy instructions to make the debugging easier. + c.Emit4Bytes(dummyInstruction) + } else { + c.Emit4Bytes(offset) + } + } +} + +// encodeExitSequence matches the implementation detail of functionABI.emitGoEntryPreamble. +func encodeExitSequence(c backend.Compiler, ctxReg regalloc.VReg) { + // Restore the FP, SP and LR, and return to the Go code: + // ldr lr, [ctxReg, #GoReturnAddress] + // ldr fp, [ctxReg, #OriginalFramePointer] + // ldr tmp, [ctxReg, #OriginalStackPointer] + // mov sp, tmp ;; sp cannot be str'ed directly. + // ret ;; --> return to the Go code + + var ctxEvicted bool + if ctx := ctxReg.RealReg(); ctx == fp || ctx == lr { + // In order to avoid overwriting the context register, we move ctxReg to tmp. + c.Emit4Bytes(encodeMov64(regNumberInEncoding[tmp], regNumberInEncoding[ctx], false, false)) + ctxReg = tmpRegVReg + ctxEvicted = true + } + + restoreLr := encodeLoadOrStore( + uLoad64, + regNumberInEncoding[lr], + addressMode{ + kind: addressModeKindRegUnsignedImm12, + rn: ctxReg, + imm: wazevoapi.ExecutionContextOffsetGoReturnAddress.I64(), + }, + ) + + restoreFp := encodeLoadOrStore( + uLoad64, + regNumberInEncoding[fp], + addressMode{ + kind: addressModeKindRegUnsignedImm12, + rn: ctxReg, + imm: wazevoapi.ExecutionContextOffsetOriginalFramePointer.I64(), + }, + ) + + restoreSpToTmp := encodeLoadOrStore( + uLoad64, + regNumberInEncoding[tmp], + addressMode{ + kind: addressModeKindRegUnsignedImm12, + rn: ctxReg, + imm: wazevoapi.ExecutionContextOffsetOriginalStackPointer.I64(), + }, + ) + + movTmpToSp := encodeAddSubtractImmediate(0b100, 0, 0, + regNumberInEncoding[tmp], regNumberInEncoding[sp]) + + c.Emit4Bytes(restoreFp) + c.Emit4Bytes(restoreLr) + c.Emit4Bytes(restoreSpToTmp) + c.Emit4Bytes(movTmpToSp) + c.Emit4Bytes(encodeRet()) + if !ctxEvicted { + // In order to have the fixed-length exit sequence, we need to padd the binary. + // Since this will never be reached, we insert a dummy instruction. + c.Emit4Bytes(dummyInstruction) + } +} + +func encodeRet() uint32 { + // https://developer.arm.com/documentation/ddi0596/2020-12/Base-Instructions/RET--Return-from-subroutine-?lang=en + return 0b1101011001011111<<16 | regNumberInEncoding[lr]<<5 +} + +func encodeAtomicRmw(op atomicRmwOp, rs, rt, rn uint32, size uint32) uint32 { + var _31to21, _15to10, sz uint32 + + switch size { + case 8: + sz = 0b11 + case 4: + sz = 0b10 + case 2: + sz = 0b01 + case 1: + sz = 0b00 + } + + _31to21 = 0b00111000_111 | sz<<9 + + switch op { + case atomicRmwOpAdd: + _15to10 = 0b000000 + case atomicRmwOpClr: + _15to10 = 0b000100 + case atomicRmwOpSet: + _15to10 = 0b001100 + case atomicRmwOpEor: + _15to10 = 0b001000 + case atomicRmwOpSwp: + _15to10 = 0b100000 + } + + return _31to21<<21 | rs<<16 | _15to10<<10 | rn<<5 | rt +} + +func encodeAtomicCas(rs, rt, rn uint32, size uint32) uint32 { + var _31to21, _15to10, sz uint32 + + switch size { + case 8: + sz = 0b11 + case 4: + sz = 0b10 + case 2: + sz = 0b01 + case 1: + sz = 0b00 + } + + _31to21 = 0b00001000_111 | sz<<9 + _15to10 = 0b111111 + + return _31to21<<21 | rs<<16 | _15to10<<10 | rn<<5 | rt +} + +func encodeAtomicLoadStore(rn, rt, size, l uint32) uint32 { + var _31to21, _20to16, _15to10, sz uint32 + + switch size { + case 8: + sz = 0b11 + case 4: + sz = 0b10 + case 2: + sz = 0b01 + case 1: + sz = 0b00 + } + + _31to21 = 0b00001000_100 | sz<<9 | l<<1 + _20to16 = 0b11111 + _15to10 = 0b111111 + + return _31to21<<21 | _20to16<<16 | _15to10<<10 | rn<<5 | rt +} + +func encodeDMB() uint32 { + return 0b11010101000000110011101110111111 +} diff --git a/vendor/github.com/tetratelabs/wazero/internal/engine/wazevo/backend/isa/arm64/lower_constant.go b/vendor/github.com/tetratelabs/wazero/internal/engine/wazevo/backend/isa/arm64/lower_constant.go new file mode 100644 index 000000000..698b382d4 --- /dev/null +++ b/vendor/github.com/tetratelabs/wazero/internal/engine/wazevo/backend/isa/arm64/lower_constant.go @@ -0,0 +1,301 @@ +package arm64 + +import ( + "github.com/tetratelabs/wazero/internal/engine/wazevo/backend/regalloc" + "github.com/tetratelabs/wazero/internal/engine/wazevo/ssa" +) + +// lowerConstant allocates a new VReg and inserts the instruction to load the constant value. +func (m *machine) lowerConstant(instr *ssa.Instruction) (vr regalloc.VReg) { + val := instr.Return() + valType := val.Type() + + vr = m.compiler.AllocateVReg(valType) + v := instr.ConstantVal() + m.insertLoadConstant(v, valType, vr) + return +} + +// InsertLoadConstantBlockArg implements backend.Machine. +func (m *machine) InsertLoadConstantBlockArg(instr *ssa.Instruction, vr regalloc.VReg) { + val := instr.Return() + valType := val.Type() + v := instr.ConstantVal() + load := m.allocateInstr() + load.asLoadConstBlockArg(v, valType, vr) + m.insert(load) +} + +func (m *machine) lowerLoadConstantBlockArgAfterRegAlloc(i *instruction) { + v, typ, dst := i.loadConstBlockArgData() + m.insertLoadConstant(v, typ, dst) +} + +func (m *machine) insertLoadConstant(v uint64, valType ssa.Type, vr regalloc.VReg) { + if valType.Bits() < 64 { // Clear the redundant bits just in case it's unexpectedly sign-extended, etc. + v = v & ((1 << valType.Bits()) - 1) + } + + switch valType { + case ssa.TypeF32: + loadF := m.allocateInstr() + loadF.asLoadFpuConst32(vr, v) + m.insert(loadF) + case ssa.TypeF64: + loadF := m.allocateInstr() + loadF.asLoadFpuConst64(vr, v) + m.insert(loadF) + case ssa.TypeI32: + if v == 0 { + m.InsertMove(vr, xzrVReg, ssa.TypeI32) + } else { + m.lowerConstantI32(vr, int32(v)) + } + case ssa.TypeI64: + if v == 0 { + m.InsertMove(vr, xzrVReg, ssa.TypeI64) + } else { + m.lowerConstantI64(vr, int64(v)) + } + default: + panic("TODO") + } +} + +// The following logics are based on the old asm/arm64 package. +// https://github.com/tetratelabs/wazero/blob/39f2ff23a6d609e10c82b9cc0b981f6de5b87a9c/internal/asm/arm64/impl.go + +func (m *machine) lowerConstantI32(dst regalloc.VReg, c int32) { + // Following the logic here: + // https://github.com/golang/go/blob/release-branch.go1.15/src/cmd/internal/obj/arm64/asm7.go#L1637 + ic := int64(uint32(c)) + if ic >= 0 && (ic <= 0xfff || (ic&0xfff) == 0 && (uint64(ic>>12) <= 0xfff)) { + if isBitMaskImmediate(uint64(c), false) { + m.lowerConstViaBitMaskImmediate(uint64(uint32(c)), dst, false) + return + } + } + + if t := const16bitAligned(int64(uint32(c))); t >= 0 { + // If the const can fit within 16-bit alignment, for example, 0xffff, 0xffff_0000 or 0xffff_0000_0000_0000 + // We could load it into temporary with movk. + m.insertMOVZ(dst, uint64(uint32(c)>>(16*t)), t, false) + } else if t := const16bitAligned(int64(^c)); t >= 0 { + // Also, if the inverse of the const can fit within 16-bit range, do the same ^^. + m.insertMOVN(dst, uint64(^c>>(16*t)), t, false) + } else if isBitMaskImmediate(uint64(uint32(c)), false) { + m.lowerConstViaBitMaskImmediate(uint64(c), dst, false) + } else { + // Otherwise, we use MOVZ and MOVK to load it. + c16 := uint16(c) + m.insertMOVZ(dst, uint64(c16), 0, false) + c16 = uint16(uint32(c) >> 16) + m.insertMOVK(dst, uint64(c16), 1, false) + } +} + +func (m *machine) lowerConstantI64(dst regalloc.VReg, c int64) { + // Following the logic here: + // https://github.com/golang/go/blob/release-branch.go1.15/src/cmd/internal/obj/arm64/asm7.go#L1798-L1852 + if c >= 0 && (c <= 0xfff || (c&0xfff) == 0 && (uint64(c>>12) <= 0xfff)) { + if isBitMaskImmediate(uint64(c), true) { + m.lowerConstViaBitMaskImmediate(uint64(c), dst, true) + return + } + } + + if t := const16bitAligned(c); t >= 0 { + // If the const can fit within 16-bit alignment, for example, 0xffff, 0xffff_0000 or 0xffff_0000_0000_0000 + // We could load it into temporary with movk. + m.insertMOVZ(dst, uint64(c)>>(16*t), t, true) + } else if t := const16bitAligned(^c); t >= 0 { + // Also, if the reverse of the const can fit within 16-bit range, do the same ^^. + m.insertMOVN(dst, uint64(^c)>>(16*t), t, true) + } else if isBitMaskImmediate(uint64(c), true) { + m.lowerConstViaBitMaskImmediate(uint64(c), dst, true) + } else { + m.load64bitConst(c, dst) + } +} + +func (m *machine) lowerConstViaBitMaskImmediate(c uint64, dst regalloc.VReg, b64 bool) { + instr := m.allocateInstr() + instr.asALUBitmaskImm(aluOpOrr, dst, xzrVReg, c, b64) + m.insert(instr) +} + +// isBitMaskImmediate determines if the value can be encoded as "bitmask immediate". +// +// Such an immediate is a 32-bit or 64-bit pattern viewed as a vector of identical elements of size e = 2, 4, 8, 16, 32, or 64 bits. +// Each element contains the same sub-pattern: a single run of 1 to e-1 non-zero bits, rotated by 0 to e-1 bits. +// +// See https://developer.arm.com/documentation/dui0802/b/A64-General-Instructions/MOV--bitmask-immediate- +func isBitMaskImmediate(x uint64, _64 bool) bool { + // All zeros and ones are not "bitmask immediate" by definition. + if x == 0 || (_64 && x == 0xffff_ffff_ffff_ffff) || (!_64 && x == 0xffff_ffff) { + return false + } + + switch { + case x != x>>32|x<<32: + // e = 64 + case x != x>>16|x<<48: + // e = 32 (x == x>>32|x<<32). + // e.g. 0x00ff_ff00_00ff_ff00 + x = uint64(int32(x)) + case x != x>>8|x<<56: + // e = 16 (x == x>>16|x<<48). + // e.g. 0x00ff_00ff_00ff_00ff + x = uint64(int16(x)) + case x != x>>4|x<<60: + // e = 8 (x == x>>8|x<<56). + // e.g. 0x0f0f_0f0f_0f0f_0f0f + x = uint64(int8(x)) + default: + // e = 4 or 2. + return true + } + return sequenceOfSetbits(x) || sequenceOfSetbits(^x) +} + +// sequenceOfSetbits returns true if the number's binary representation is the sequence set bit (1). +// For example: 0b1110 -> true, 0b1010 -> false +func sequenceOfSetbits(x uint64) bool { + y := getLowestBit(x) + // If x is a sequence of set bit, this should results in the number + // with only one set bit (i.e. power of two). + y += x + return (y-1)&y == 0 +} + +func getLowestBit(x uint64) uint64 { + return x & (^x + 1) +} + +// const16bitAligned check if the value is on the 16-bit alignment. +// If so, returns the shift num divided by 16, and otherwise -1. +func const16bitAligned(v int64) (ret int) { + ret = -1 + for s := 0; s < 64; s += 16 { + if (uint64(v) &^ (uint64(0xffff) << uint(s))) == 0 { + ret = s / 16 + break + } + } + return +} + +// load64bitConst loads a 64-bit constant into the register, following the same logic to decide how to load large 64-bit +// consts as in the Go assembler. +// +// See https://github.com/golang/go/blob/release-branch.go1.15/src/cmd/internal/obj/arm64/asm7.go#L6632-L6759 +func (m *machine) load64bitConst(c int64, dst regalloc.VReg) { + var bits [4]uint64 + var zeros, negs int + for i := 0; i < 4; i++ { + bits[i] = uint64(c) >> uint(i*16) & 0xffff + if v := bits[i]; v == 0 { + zeros++ + } else if v == 0xffff { + negs++ + } + } + + if zeros == 3 { + // one MOVZ instruction. + for i, v := range bits { + if v != 0 { + m.insertMOVZ(dst, v, i, true) + } + } + } else if negs == 3 { + // one MOVN instruction. + for i, v := range bits { + if v != 0xffff { + v = ^v + m.insertMOVN(dst, v, i, true) + } + } + } else if zeros == 2 { + // one MOVZ then one OVK. + var movz bool + for i, v := range bits { + if !movz && v != 0 { // MOVZ. + m.insertMOVZ(dst, v, i, true) + movz = true + } else if v != 0 { + m.insertMOVK(dst, v, i, true) + } + } + + } else if negs == 2 { + // one MOVN then one or two MOVK. + var movn bool + for i, v := range bits { // Emit MOVN. + if !movn && v != 0xffff { + v = ^v + // https://developer.arm.com/documentation/dui0802/a/A64-General-Instructions/MOVN + m.insertMOVN(dst, v, i, true) + movn = true + } else if v != 0xffff { + m.insertMOVK(dst, v, i, true) + } + } + + } else if zeros == 1 { + // one MOVZ then two MOVK. + var movz bool + for i, v := range bits { + if !movz && v != 0 { // MOVZ. + m.insertMOVZ(dst, v, i, true) + movz = true + } else if v != 0 { + m.insertMOVK(dst, v, i, true) + } + } + + } else if negs == 1 { + // one MOVN then two MOVK. + var movn bool + for i, v := range bits { // Emit MOVN. + if !movn && v != 0xffff { + v = ^v + // https://developer.arm.com/documentation/dui0802/a/A64-General-Instructions/MOVN + m.insertMOVN(dst, v, i, true) + movn = true + } else if v != 0xffff { + m.insertMOVK(dst, v, i, true) + } + } + + } else { + // one MOVZ then up to three MOVK. + var movz bool + for i, v := range bits { + if !movz && v != 0 { // MOVZ. + m.insertMOVZ(dst, v, i, true) + movz = true + } else if v != 0 { + m.insertMOVK(dst, v, i, true) + } + } + } +} + +func (m *machine) insertMOVZ(dst regalloc.VReg, v uint64, shift int, dst64 bool) { + instr := m.allocateInstr() + instr.asMOVZ(dst, v, uint64(shift), dst64) + m.insert(instr) +} + +func (m *machine) insertMOVK(dst regalloc.VReg, v uint64, shift int, dst64 bool) { + instr := m.allocateInstr() + instr.asMOVK(dst, v, uint64(shift), dst64) + m.insert(instr) +} + +func (m *machine) insertMOVN(dst regalloc.VReg, v uint64, shift int, dst64 bool) { + instr := m.allocateInstr() + instr.asMOVN(dst, v, uint64(shift), dst64) + m.insert(instr) +} diff --git a/vendor/github.com/tetratelabs/wazero/internal/engine/wazevo/backend/isa/arm64/lower_instr.go b/vendor/github.com/tetratelabs/wazero/internal/engine/wazevo/backend/isa/arm64/lower_instr.go new file mode 100644 index 000000000..2bb234e8c --- /dev/null +++ b/vendor/github.com/tetratelabs/wazero/internal/engine/wazevo/backend/isa/arm64/lower_instr.go @@ -0,0 +1,2221 @@ +package arm64 + +// Files prefixed as lower_instr** do the instruction selection, meaning that lowering SSA level instructions +// into machine specific instructions. +// +// Importantly, what the lower** functions does includes tree-matching; find the pattern from the given instruction tree, +// and merge the multiple instructions if possible. It can be considered as "N:1" instruction selection. + +import ( + "fmt" + "math" + + "github.com/tetratelabs/wazero/internal/engine/wazevo/backend/regalloc" + "github.com/tetratelabs/wazero/internal/engine/wazevo/ssa" + "github.com/tetratelabs/wazero/internal/engine/wazevo/wazevoapi" +) + +// LowerSingleBranch implements backend.Machine. +func (m *machine) LowerSingleBranch(br *ssa.Instruction) { + ectx := m.executableContext + switch br.Opcode() { + case ssa.OpcodeJump: + _, _, targetBlk := br.BranchData() + if br.IsFallthroughJump() { + return + } + b := m.allocateInstr() + target := ectx.GetOrAllocateSSABlockLabel(targetBlk) + if target == labelReturn { + b.asRet() + } else { + b.asBr(target) + } + m.insert(b) + case ssa.OpcodeBrTable: + m.lowerBrTable(br) + default: + panic("BUG: unexpected branch opcode" + br.Opcode().String()) + } +} + +func (m *machine) lowerBrTable(i *ssa.Instruction) { + index, targets := i.BrTableData() + indexOperand := m.getOperand_NR(m.compiler.ValueDefinition(index), extModeNone) + + // Firstly, we have to do the bounds check of the index, and + // set it to the default target (sitting at the end of the list) if it's out of bounds. + + // mov maxIndexReg #maximum_index + // subs wzr, index, maxIndexReg + // csel adjustedIndex, maxIndexReg, index, hs ;; if index is higher or equal than maxIndexReg. + maxIndexReg := m.compiler.AllocateVReg(ssa.TypeI32) + m.lowerConstantI32(maxIndexReg, int32(len(targets)-1)) + subs := m.allocateInstr() + subs.asALU(aluOpSubS, operandNR(xzrVReg), indexOperand, operandNR(maxIndexReg), false) + m.insert(subs) + csel := m.allocateInstr() + adjustedIndex := m.compiler.AllocateVReg(ssa.TypeI32) + csel.asCSel(operandNR(adjustedIndex), operandNR(maxIndexReg), indexOperand, hs, false) + m.insert(csel) + + brSequence := m.allocateInstr() + + tableIndex := m.addJmpTableTarget(targets) + brSequence.asBrTableSequence(adjustedIndex, tableIndex, len(targets)) + m.insert(brSequence) +} + +// LowerConditionalBranch implements backend.Machine. +func (m *machine) LowerConditionalBranch(b *ssa.Instruction) { + exctx := m.executableContext + cval, args, targetBlk := b.BranchData() + if len(args) > 0 { + panic(fmt.Sprintf( + "conditional branch shouldn't have args; likely a bug in critical edge splitting: from %s to %s", + exctx.CurrentSSABlk, + targetBlk, + )) + } + + target := exctx.GetOrAllocateSSABlockLabel(targetBlk) + cvalDef := m.compiler.ValueDefinition(cval) + + switch { + case m.compiler.MatchInstr(cvalDef, ssa.OpcodeIcmp): // This case, we can use the ALU flag set by SUBS instruction. + cvalInstr := cvalDef.Instr + x, y, c := cvalInstr.IcmpData() + cc, signed := condFlagFromSSAIntegerCmpCond(c), c.Signed() + if b.Opcode() == ssa.OpcodeBrz { + cc = cc.invert() + } + + if !m.tryLowerBandToFlag(x, y) { + m.lowerIcmpToFlag(x, y, signed) + } + cbr := m.allocateInstr() + cbr.asCondBr(cc.asCond(), target, false /* ignored */) + m.insert(cbr) + cvalDef.Instr.MarkLowered() + case m.compiler.MatchInstr(cvalDef, ssa.OpcodeFcmp): // This case we can use the Fpu flag directly. + cvalInstr := cvalDef.Instr + x, y, c := cvalInstr.FcmpData() + cc := condFlagFromSSAFloatCmpCond(c) + if b.Opcode() == ssa.OpcodeBrz { + cc = cc.invert() + } + m.lowerFcmpToFlag(x, y) + cbr := m.allocateInstr() + cbr.asCondBr(cc.asCond(), target, false /* ignored */) + m.insert(cbr) + cvalDef.Instr.MarkLowered() + default: + rn := m.getOperand_NR(cvalDef, extModeNone) + var c cond + if b.Opcode() == ssa.OpcodeBrz { + c = registerAsRegZeroCond(rn.nr()) + } else { + c = registerAsRegNotZeroCond(rn.nr()) + } + cbr := m.allocateInstr() + cbr.asCondBr(c, target, false) + m.insert(cbr) + } +} + +func (m *machine) tryLowerBandToFlag(x, y ssa.Value) (ok bool) { + xx := m.compiler.ValueDefinition(x) + yy := m.compiler.ValueDefinition(y) + if xx.IsFromInstr() && xx.Instr.Constant() && xx.Instr.ConstantVal() == 0 { + if m.compiler.MatchInstr(yy, ssa.OpcodeBand) { + bandInstr := yy.Instr + m.lowerBitwiseAluOp(bandInstr, aluOpAnds, true) + ok = true + bandInstr.MarkLowered() + return + } + } + + if yy.IsFromInstr() && yy.Instr.Constant() && yy.Instr.ConstantVal() == 0 { + if m.compiler.MatchInstr(xx, ssa.OpcodeBand) { + bandInstr := xx.Instr + m.lowerBitwiseAluOp(bandInstr, aluOpAnds, true) + ok = true + bandInstr.MarkLowered() + return + } + } + return +} + +// LowerInstr implements backend.Machine. +func (m *machine) LowerInstr(instr *ssa.Instruction) { + if l := instr.SourceOffset(); l.Valid() { + info := m.allocateInstr().asEmitSourceOffsetInfo(l) + m.insert(info) + } + + switch op := instr.Opcode(); op { + case ssa.OpcodeBrz, ssa.OpcodeBrnz, ssa.OpcodeJump, ssa.OpcodeBrTable: + panic("BUG: branching instructions are handled by LowerBranches") + case ssa.OpcodeReturn: + panic("BUG: return must be handled by backend.Compiler") + case ssa.OpcodeIadd, ssa.OpcodeIsub: + m.lowerSubOrAdd(instr, op == ssa.OpcodeIadd) + case ssa.OpcodeFadd, ssa.OpcodeFsub, ssa.OpcodeFmul, ssa.OpcodeFdiv, ssa.OpcodeFmax, ssa.OpcodeFmin: + m.lowerFpuBinOp(instr) + case ssa.OpcodeIconst, ssa.OpcodeF32const, ssa.OpcodeF64const: // Constant instructions are inlined. + case ssa.OpcodeExitWithCode: + execCtx, code := instr.ExitWithCodeData() + m.lowerExitWithCode(m.compiler.VRegOf(execCtx), code) + case ssa.OpcodeExitIfTrueWithCode: + execCtx, c, code := instr.ExitIfTrueWithCodeData() + m.lowerExitIfTrueWithCode(m.compiler.VRegOf(execCtx), c, code) + case ssa.OpcodeStore, ssa.OpcodeIstore8, ssa.OpcodeIstore16, ssa.OpcodeIstore32: + m.lowerStore(instr) + case ssa.OpcodeLoad: + dst := instr.Return() + ptr, offset, typ := instr.LoadData() + m.lowerLoad(ptr, offset, typ, dst) + case ssa.OpcodeVZeroExtLoad: + dst := instr.Return() + ptr, offset, typ := instr.VZeroExtLoadData() + m.lowerLoad(ptr, offset, typ, dst) + case ssa.OpcodeUload8, ssa.OpcodeUload16, ssa.OpcodeUload32, ssa.OpcodeSload8, ssa.OpcodeSload16, ssa.OpcodeSload32: + ptr, offset, _ := instr.LoadData() + ret := m.compiler.VRegOf(instr.Return()) + m.lowerExtLoad(op, ptr, offset, ret) + case ssa.OpcodeCall, ssa.OpcodeCallIndirect: + m.lowerCall(instr) + case ssa.OpcodeIcmp: + m.lowerIcmp(instr) + case ssa.OpcodeVIcmp: + m.lowerVIcmp(instr) + case ssa.OpcodeVFcmp: + m.lowerVFcmp(instr) + case ssa.OpcodeVCeil: + m.lowerVecMisc(vecOpFrintp, instr) + case ssa.OpcodeVFloor: + m.lowerVecMisc(vecOpFrintm, instr) + case ssa.OpcodeVTrunc: + m.lowerVecMisc(vecOpFrintz, instr) + case ssa.OpcodeVNearest: + m.lowerVecMisc(vecOpFrintn, instr) + case ssa.OpcodeVMaxPseudo: + m.lowerVMinMaxPseudo(instr, true) + case ssa.OpcodeVMinPseudo: + m.lowerVMinMaxPseudo(instr, false) + case ssa.OpcodeBand: + m.lowerBitwiseAluOp(instr, aluOpAnd, false) + case ssa.OpcodeBor: + m.lowerBitwiseAluOp(instr, aluOpOrr, false) + case ssa.OpcodeBxor: + m.lowerBitwiseAluOp(instr, aluOpEor, false) + case ssa.OpcodeIshl: + m.lowerShifts(instr, extModeNone, aluOpLsl) + case ssa.OpcodeSshr: + if instr.Return().Type().Bits() == 64 { + m.lowerShifts(instr, extModeSignExtend64, aluOpAsr) + } else { + m.lowerShifts(instr, extModeSignExtend32, aluOpAsr) + } + case ssa.OpcodeUshr: + if instr.Return().Type().Bits() == 64 { + m.lowerShifts(instr, extModeZeroExtend64, aluOpLsr) + } else { + m.lowerShifts(instr, extModeZeroExtend32, aluOpLsr) + } + case ssa.OpcodeRotl: + m.lowerRotl(instr) + case ssa.OpcodeRotr: + m.lowerRotr(instr) + case ssa.OpcodeSExtend, ssa.OpcodeUExtend: + from, to, signed := instr.ExtendData() + m.lowerExtend(instr.Arg(), instr.Return(), from, to, signed) + case ssa.OpcodeFcmp: + x, y, c := instr.FcmpData() + m.lowerFcmp(x, y, instr.Return(), c) + case ssa.OpcodeImul: + x, y := instr.Arg2() + result := instr.Return() + m.lowerImul(x, y, result) + case ssa.OpcodeUndefined: + undef := m.allocateInstr() + undef.asUDF() + m.insert(undef) + case ssa.OpcodeSelect: + c, x, y := instr.SelectData() + if x.Type() == ssa.TypeV128 { + rc := m.getOperand_NR(m.compiler.ValueDefinition(c), extModeNone) + rn := m.getOperand_NR(m.compiler.ValueDefinition(x), extModeNone) + rm := m.getOperand_NR(m.compiler.ValueDefinition(y), extModeNone) + rd := operandNR(m.compiler.VRegOf(instr.Return())) + m.lowerSelectVec(rc, rn, rm, rd) + } else { + m.lowerSelect(c, x, y, instr.Return()) + } + case ssa.OpcodeClz: + x := instr.Arg() + result := instr.Return() + m.lowerClz(x, result) + case ssa.OpcodeCtz: + x := instr.Arg() + result := instr.Return() + m.lowerCtz(x, result) + case ssa.OpcodePopcnt: + x := instr.Arg() + result := instr.Return() + m.lowerPopcnt(x, result) + case ssa.OpcodeFcvtToSint, ssa.OpcodeFcvtToSintSat: + x, ctx := instr.Arg2() + result := instr.Return() + rn := m.getOperand_NR(m.compiler.ValueDefinition(x), extModeNone) + rd := operandNR(m.compiler.VRegOf(result)) + ctxVReg := m.compiler.VRegOf(ctx) + m.lowerFpuToInt(rd, rn, ctxVReg, true, x.Type() == ssa.TypeF64, + result.Type().Bits() == 64, op == ssa.OpcodeFcvtToSintSat) + case ssa.OpcodeFcvtToUint, ssa.OpcodeFcvtToUintSat: + x, ctx := instr.Arg2() + result := instr.Return() + rn := m.getOperand_NR(m.compiler.ValueDefinition(x), extModeNone) + rd := operandNR(m.compiler.VRegOf(result)) + ctxVReg := m.compiler.VRegOf(ctx) + m.lowerFpuToInt(rd, rn, ctxVReg, false, x.Type() == ssa.TypeF64, + result.Type().Bits() == 64, op == ssa.OpcodeFcvtToUintSat) + case ssa.OpcodeFcvtFromSint: + x := instr.Arg() + result := instr.Return() + rn := m.getOperand_NR(m.compiler.ValueDefinition(x), extModeNone) + rd := operandNR(m.compiler.VRegOf(result)) + m.lowerIntToFpu(rd, rn, true, x.Type() == ssa.TypeI64, result.Type().Bits() == 64) + case ssa.OpcodeFcvtFromUint: + x := instr.Arg() + result := instr.Return() + rn := m.getOperand_NR(m.compiler.ValueDefinition(x), extModeNone) + rd := operandNR(m.compiler.VRegOf(result)) + m.lowerIntToFpu(rd, rn, false, x.Type() == ssa.TypeI64, result.Type().Bits() == 64) + case ssa.OpcodeFdemote: + v := instr.Arg() + rn := m.getOperand_NR(m.compiler.ValueDefinition(v), extModeNone) + rd := operandNR(m.compiler.VRegOf(instr.Return())) + cnt := m.allocateInstr() + cnt.asFpuRR(fpuUniOpCvt64To32, rd, rn, false) + m.insert(cnt) + case ssa.OpcodeFpromote: + v := instr.Arg() + rn := m.getOperand_NR(m.compiler.ValueDefinition(v), extModeNone) + rd := operandNR(m.compiler.VRegOf(instr.Return())) + cnt := m.allocateInstr() + cnt.asFpuRR(fpuUniOpCvt32To64, rd, rn, true) + m.insert(cnt) + case ssa.OpcodeIreduce: + rn := m.getOperand_NR(m.compiler.ValueDefinition(instr.Arg()), extModeNone) + retVal := instr.Return() + rd := m.compiler.VRegOf(retVal) + + if retVal.Type() != ssa.TypeI32 { + panic("TODO?: Ireduce to non-i32") + } + mov := m.allocateInstr() + mov.asMove32(rd, rn.reg()) + m.insert(mov) + case ssa.OpcodeFneg: + m.lowerFpuUniOp(fpuUniOpNeg, instr.Arg(), instr.Return()) + case ssa.OpcodeSqrt: + m.lowerFpuUniOp(fpuUniOpSqrt, instr.Arg(), instr.Return()) + case ssa.OpcodeCeil: + m.lowerFpuUniOp(fpuUniOpRoundPlus, instr.Arg(), instr.Return()) + case ssa.OpcodeFloor: + m.lowerFpuUniOp(fpuUniOpRoundMinus, instr.Arg(), instr.Return()) + case ssa.OpcodeTrunc: + m.lowerFpuUniOp(fpuUniOpRoundZero, instr.Arg(), instr.Return()) + case ssa.OpcodeNearest: + m.lowerFpuUniOp(fpuUniOpRoundNearest, instr.Arg(), instr.Return()) + case ssa.OpcodeFabs: + m.lowerFpuUniOp(fpuUniOpAbs, instr.Arg(), instr.Return()) + case ssa.OpcodeBitcast: + m.lowerBitcast(instr) + case ssa.OpcodeFcopysign: + x, y := instr.Arg2() + m.lowerFcopysign(x, y, instr.Return()) + case ssa.OpcodeSdiv, ssa.OpcodeUdiv: + x, y, ctx := instr.Arg3() + ctxVReg := m.compiler.VRegOf(ctx) + rn := m.getOperand_NR(m.compiler.ValueDefinition(x), extModeNone) + rm := m.getOperand_NR(m.compiler.ValueDefinition(y), extModeNone) + rd := operandNR(m.compiler.VRegOf(instr.Return())) + m.lowerIDiv(ctxVReg, rd, rn, rm, x.Type() == ssa.TypeI64, op == ssa.OpcodeSdiv) + case ssa.OpcodeSrem, ssa.OpcodeUrem: + x, y, ctx := instr.Arg3() + ctxVReg := m.compiler.VRegOf(ctx) + rn := m.getOperand_NR(m.compiler.ValueDefinition(x), extModeNone) + rm := m.getOperand_NR(m.compiler.ValueDefinition(y), extModeNone) + rd := operandNR(m.compiler.VRegOf(instr.Return())) + m.lowerIRem(ctxVReg, rd, rn, rm, x.Type() == ssa.TypeI64, op == ssa.OpcodeSrem) + case ssa.OpcodeVconst: + result := m.compiler.VRegOf(instr.Return()) + lo, hi := instr.VconstData() + v := m.allocateInstr() + v.asLoadFpuConst128(result, lo, hi) + m.insert(v) + case ssa.OpcodeVbnot: + x := instr.Arg() + ins := m.allocateInstr() + rn := m.getOperand_NR(m.compiler.ValueDefinition(x), extModeNone) + rd := operandNR(m.compiler.VRegOf(instr.Return())) + ins.asVecMisc(vecOpNot, rd, rn, vecArrangement16B) + m.insert(ins) + case ssa.OpcodeVbxor: + x, y := instr.Arg2() + m.lowerVecRRR(vecOpEOR, x, y, instr.Return(), vecArrangement16B) + case ssa.OpcodeVbor: + x, y := instr.Arg2() + m.lowerVecRRR(vecOpOrr, x, y, instr.Return(), vecArrangement16B) + case ssa.OpcodeVband: + x, y := instr.Arg2() + m.lowerVecRRR(vecOpAnd, x, y, instr.Return(), vecArrangement16B) + case ssa.OpcodeVbandnot: + x, y := instr.Arg2() + m.lowerVecRRR(vecOpBic, x, y, instr.Return(), vecArrangement16B) + case ssa.OpcodeVbitselect: + c, x, y := instr.SelectData() + rn := m.getOperand_NR(m.compiler.ValueDefinition(x), extModeNone) + rm := m.getOperand_NR(m.compiler.ValueDefinition(y), extModeNone) + creg := m.getOperand_NR(m.compiler.ValueDefinition(c), extModeNone) + tmp := operandNR(m.compiler.AllocateVReg(ssa.TypeV128)) + + // creg is overwritten by BSL, so we need to move it to the result register before the instruction + // in case when it is used somewhere else. + mov := m.allocateInstr() + mov.asFpuMov128(tmp.nr(), creg.nr()) + m.insert(mov) + + ins := m.allocateInstr() + ins.asVecRRRRewrite(vecOpBsl, tmp, rn, rm, vecArrangement16B) + m.insert(ins) + + mov2 := m.allocateInstr() + rd := m.compiler.VRegOf(instr.Return()) + mov2.asFpuMov128(rd, tmp.nr()) + m.insert(mov2) + case ssa.OpcodeVanyTrue, ssa.OpcodeVallTrue: + x, lane := instr.ArgWithLane() + var arr vecArrangement + if op == ssa.OpcodeVallTrue { + arr = ssaLaneToArrangement(lane) + } + rm := m.getOperand_NR(m.compiler.ValueDefinition(x), extModeNone) + rd := operandNR(m.compiler.VRegOf(instr.Return())) + m.lowerVcheckTrue(op, rm, rd, arr) + case ssa.OpcodeVhighBits: + x, lane := instr.ArgWithLane() + rm := m.getOperand_NR(m.compiler.ValueDefinition(x), extModeNone) + rd := operandNR(m.compiler.VRegOf(instr.Return())) + arr := ssaLaneToArrangement(lane) + m.lowerVhighBits(rm, rd, arr) + case ssa.OpcodeVIadd: + x, y, lane := instr.Arg2WithLane() + arr := ssaLaneToArrangement(lane) + m.lowerVecRRR(vecOpAdd, x, y, instr.Return(), arr) + case ssa.OpcodeExtIaddPairwise: + v, lane, signed := instr.ExtIaddPairwiseData() + vv := m.getOperand_NR(m.compiler.ValueDefinition(v), extModeNone) + + tmpLo, tmpHi := operandNR(m.compiler.AllocateVReg(ssa.TypeV128)), operandNR(m.compiler.AllocateVReg(ssa.TypeV128)) + var widen vecOp + if signed { + widen = vecOpSshll + } else { + widen = vecOpUshll + } + + var loArr, hiArr, dstArr vecArrangement + switch lane { + case ssa.VecLaneI8x16: + loArr, hiArr, dstArr = vecArrangement8B, vecArrangement16B, vecArrangement8H + case ssa.VecLaneI16x8: + loArr, hiArr, dstArr = vecArrangement4H, vecArrangement8H, vecArrangement4S + case ssa.VecLaneI32x4: + loArr, hiArr, dstArr = vecArrangement2S, vecArrangement4S, vecArrangement2D + default: + panic("unsupported lane " + lane.String()) + } + + widenLo := m.allocateInstr().asVecShiftImm(widen, tmpLo, vv, operandShiftImm(0), loArr) + widenHi := m.allocateInstr().asVecShiftImm(widen, tmpHi, vv, operandShiftImm(0), hiArr) + addp := m.allocateInstr().asVecRRR(vecOpAddp, operandNR(m.compiler.VRegOf(instr.Return())), tmpLo, tmpHi, dstArr) + m.insert(widenLo) + m.insert(widenHi) + m.insert(addp) + + case ssa.OpcodeVSaddSat: + x, y, lane := instr.Arg2WithLane() + arr := ssaLaneToArrangement(lane) + m.lowerVecRRR(vecOpSqadd, x, y, instr.Return(), arr) + case ssa.OpcodeVUaddSat: + x, y, lane := instr.Arg2WithLane() + arr := ssaLaneToArrangement(lane) + m.lowerVecRRR(vecOpUqadd, x, y, instr.Return(), arr) + case ssa.OpcodeVIsub: + x, y, lane := instr.Arg2WithLane() + arr := ssaLaneToArrangement(lane) + m.lowerVecRRR(vecOpSub, x, y, instr.Return(), arr) + case ssa.OpcodeVSsubSat: + x, y, lane := instr.Arg2WithLane() + arr := ssaLaneToArrangement(lane) + m.lowerVecRRR(vecOpSqsub, x, y, instr.Return(), arr) + case ssa.OpcodeVUsubSat: + x, y, lane := instr.Arg2WithLane() + arr := ssaLaneToArrangement(lane) + m.lowerVecRRR(vecOpUqsub, x, y, instr.Return(), arr) + case ssa.OpcodeVImin: + x, y, lane := instr.Arg2WithLane() + arr := ssaLaneToArrangement(lane) + m.lowerVecRRR(vecOpSmin, x, y, instr.Return(), arr) + case ssa.OpcodeVUmin: + x, y, lane := instr.Arg2WithLane() + arr := ssaLaneToArrangement(lane) + m.lowerVecRRR(vecOpUmin, x, y, instr.Return(), arr) + case ssa.OpcodeVImax: + x, y, lane := instr.Arg2WithLane() + arr := ssaLaneToArrangement(lane) + m.lowerVecRRR(vecOpSmax, x, y, instr.Return(), arr) + case ssa.OpcodeVUmax: + x, y, lane := instr.Arg2WithLane() + arr := ssaLaneToArrangement(lane) + m.lowerVecRRR(vecOpUmax, x, y, instr.Return(), arr) + case ssa.OpcodeVAvgRound: + x, y, lane := instr.Arg2WithLane() + arr := ssaLaneToArrangement(lane) + m.lowerVecRRR(vecOpUrhadd, x, y, instr.Return(), arr) + case ssa.OpcodeVImul: + x, y, lane := instr.Arg2WithLane() + arr := ssaLaneToArrangement(lane) + rn := m.getOperand_NR(m.compiler.ValueDefinition(x), extModeNone) + rm := m.getOperand_NR(m.compiler.ValueDefinition(y), extModeNone) + rd := operandNR(m.compiler.VRegOf(instr.Return())) + m.lowerVIMul(rd, rn, rm, arr) + case ssa.OpcodeVIabs: + m.lowerVecMisc(vecOpAbs, instr) + case ssa.OpcodeVIneg: + m.lowerVecMisc(vecOpNeg, instr) + case ssa.OpcodeVIpopcnt: + m.lowerVecMisc(vecOpCnt, instr) + case ssa.OpcodeVIshl, + ssa.OpcodeVSshr, ssa.OpcodeVUshr: + x, y, lane := instr.Arg2WithLane() + arr := ssaLaneToArrangement(lane) + rn := m.getOperand_NR(m.compiler.ValueDefinition(x), extModeNone) + rm := m.getOperand_NR(m.compiler.ValueDefinition(y), extModeNone) + rd := operandNR(m.compiler.VRegOf(instr.Return())) + m.lowerVShift(op, rd, rn, rm, arr) + case ssa.OpcodeVSqrt: + m.lowerVecMisc(vecOpFsqrt, instr) + case ssa.OpcodeVFabs: + m.lowerVecMisc(vecOpFabs, instr) + case ssa.OpcodeVFneg: + m.lowerVecMisc(vecOpFneg, instr) + case ssa.OpcodeVFmin: + x, y, lane := instr.Arg2WithLane() + arr := ssaLaneToArrangement(lane) + m.lowerVecRRR(vecOpFmin, x, y, instr.Return(), arr) + case ssa.OpcodeVFmax: + x, y, lane := instr.Arg2WithLane() + arr := ssaLaneToArrangement(lane) + m.lowerVecRRR(vecOpFmax, x, y, instr.Return(), arr) + case ssa.OpcodeVFadd: + x, y, lane := instr.Arg2WithLane() + arr := ssaLaneToArrangement(lane) + m.lowerVecRRR(vecOpFadd, x, y, instr.Return(), arr) + case ssa.OpcodeVFsub: + x, y, lane := instr.Arg2WithLane() + arr := ssaLaneToArrangement(lane) + m.lowerVecRRR(vecOpFsub, x, y, instr.Return(), arr) + case ssa.OpcodeVFmul: + x, y, lane := instr.Arg2WithLane() + arr := ssaLaneToArrangement(lane) + m.lowerVecRRR(vecOpFmul, x, y, instr.Return(), arr) + case ssa.OpcodeSqmulRoundSat: + x, y, lane := instr.Arg2WithLane() + arr := ssaLaneToArrangement(lane) + m.lowerVecRRR(vecOpSqrdmulh, x, y, instr.Return(), arr) + case ssa.OpcodeVFdiv: + x, y, lane := instr.Arg2WithLane() + arr := ssaLaneToArrangement(lane) + m.lowerVecRRR(vecOpFdiv, x, y, instr.Return(), arr) + case ssa.OpcodeVFcvtToSintSat, ssa.OpcodeVFcvtToUintSat: + x, lane := instr.ArgWithLane() + arr := ssaLaneToArrangement(lane) + rn := m.getOperand_NR(m.compiler.ValueDefinition(x), extModeNone) + rd := operandNR(m.compiler.VRegOf(instr.Return())) + m.lowerVfpuToInt(rd, rn, arr, op == ssa.OpcodeVFcvtToSintSat) + case ssa.OpcodeVFcvtFromSint, ssa.OpcodeVFcvtFromUint: + x, lane := instr.ArgWithLane() + arr := ssaLaneToArrangement(lane) + rn := m.getOperand_NR(m.compiler.ValueDefinition(x), extModeNone) + rd := operandNR(m.compiler.VRegOf(instr.Return())) + m.lowerVfpuFromInt(rd, rn, arr, op == ssa.OpcodeVFcvtFromSint) + case ssa.OpcodeSwidenLow, ssa.OpcodeUwidenLow: + x, lane := instr.ArgWithLane() + rn := m.getOperand_NR(m.compiler.ValueDefinition(x), extModeNone) + rd := operandNR(m.compiler.VRegOf(instr.Return())) + + var arr vecArrangement + switch lane { + case ssa.VecLaneI8x16: + arr = vecArrangement8B + case ssa.VecLaneI16x8: + arr = vecArrangement4H + case ssa.VecLaneI32x4: + arr = vecArrangement2S + } + + shll := m.allocateInstr() + if signed := op == ssa.OpcodeSwidenLow; signed { + shll.asVecShiftImm(vecOpSshll, rd, rn, operandShiftImm(0), arr) + } else { + shll.asVecShiftImm(vecOpUshll, rd, rn, operandShiftImm(0), arr) + } + m.insert(shll) + case ssa.OpcodeSwidenHigh, ssa.OpcodeUwidenHigh: + x, lane := instr.ArgWithLane() + rn := m.getOperand_NR(m.compiler.ValueDefinition(x), extModeNone) + rd := operandNR(m.compiler.VRegOf(instr.Return())) + + arr := ssaLaneToArrangement(lane) + + shll := m.allocateInstr() + if signed := op == ssa.OpcodeSwidenHigh; signed { + shll.asVecShiftImm(vecOpSshll, rd, rn, operandShiftImm(0), arr) + } else { + shll.asVecShiftImm(vecOpUshll, rd, rn, operandShiftImm(0), arr) + } + m.insert(shll) + + case ssa.OpcodeSnarrow, ssa.OpcodeUnarrow: + x, y, lane := instr.Arg2WithLane() + var arr, arr2 vecArrangement + switch lane { + case ssa.VecLaneI16x8: // I16x8 + arr = vecArrangement8B + arr2 = vecArrangement16B // Implies sqxtn2. + case ssa.VecLaneI32x4: + arr = vecArrangement4H + arr2 = vecArrangement8H // Implies sqxtn2. + default: + panic("unsupported lane " + lane.String()) + } + rn := m.getOperand_NR(m.compiler.ValueDefinition(x), extModeNone) + rm := m.getOperand_NR(m.compiler.ValueDefinition(y), extModeNone) + rd := operandNR(m.compiler.VRegOf(instr.Return())) + + tmp := operandNR(m.compiler.AllocateVReg(ssa.TypeV128)) + + loQxtn := m.allocateInstr() + hiQxtn := m.allocateInstr() + if signed := op == ssa.OpcodeSnarrow; signed { + // Narrow lanes on rn and write them into lower-half of rd. + loQxtn.asVecMisc(vecOpSqxtn, tmp, rn, arr) // low + // Narrow lanes on rm and write them into higher-half of rd. + hiQxtn.asVecMisc(vecOpSqxtn, tmp, rm, arr2) // high (sqxtn2) + } else { + // Narrow lanes on rn and write them into lower-half of rd. + loQxtn.asVecMisc(vecOpSqxtun, tmp, rn, arr) // low + // Narrow lanes on rm and write them into higher-half of rd. + hiQxtn.asVecMisc(vecOpSqxtun, tmp, rm, arr2) // high (sqxtn2) + } + m.insert(loQxtn) + m.insert(hiQxtn) + + mov := m.allocateInstr() + mov.asFpuMov128(rd.nr(), tmp.nr()) + m.insert(mov) + case ssa.OpcodeFvpromoteLow: + x, lane := instr.ArgWithLane() + if lane != ssa.VecLaneF32x4 { + panic("unsupported lane type " + lane.String()) + } + ins := m.allocateInstr() + rn := m.getOperand_NR(m.compiler.ValueDefinition(x), extModeNone) + rd := operandNR(m.compiler.VRegOf(instr.Return())) + ins.asVecMisc(vecOpFcvtl, rd, rn, vecArrangement2S) + m.insert(ins) + case ssa.OpcodeFvdemote: + x, lane := instr.ArgWithLane() + if lane != ssa.VecLaneF64x2 { + panic("unsupported lane type " + lane.String()) + } + ins := m.allocateInstr() + rn := m.getOperand_NR(m.compiler.ValueDefinition(x), extModeNone) + rd := operandNR(m.compiler.VRegOf(instr.Return())) + ins.asVecMisc(vecOpFcvtn, rd, rn, vecArrangement2S) + m.insert(ins) + case ssa.OpcodeExtractlane: + x, index, signed, lane := instr.ExtractlaneData() + + rn := m.getOperand_NR(m.compiler.ValueDefinition(x), extModeNone) + rd := operandNR(m.compiler.VRegOf(instr.Return())) + + mov := m.allocateInstr() + switch lane { + case ssa.VecLaneI8x16: + mov.asMovFromVec(rd, rn, vecArrangementB, vecIndex(index), signed) + case ssa.VecLaneI16x8: + mov.asMovFromVec(rd, rn, vecArrangementH, vecIndex(index), signed) + case ssa.VecLaneI32x4: + mov.asMovFromVec(rd, rn, vecArrangementS, vecIndex(index), signed) + case ssa.VecLaneI64x2: + mov.asMovFromVec(rd, rn, vecArrangementD, vecIndex(index), signed) + case ssa.VecLaneF32x4: + mov.asVecMovElement(rd, rn, vecArrangementS, vecIndex(0), vecIndex(index)) + case ssa.VecLaneF64x2: + mov.asVecMovElement(rd, rn, vecArrangementD, vecIndex(0), vecIndex(index)) + default: + panic("unsupported lane: " + lane.String()) + } + + m.insert(mov) + + case ssa.OpcodeInsertlane: + x, y, index, lane := instr.InsertlaneData() + rn := m.getOperand_NR(m.compiler.ValueDefinition(x), extModeNone) + rm := m.getOperand_NR(m.compiler.ValueDefinition(y), extModeNone) + rd := operandNR(m.compiler.VRegOf(instr.Return())) + tmpReg := operandNR(m.compiler.AllocateVReg(ssa.TypeV128)) + + // Initially mov rn to tmp. + mov1 := m.allocateInstr() + mov1.asFpuMov128(tmpReg.nr(), rn.nr()) + m.insert(mov1) + + // movToVec and vecMovElement do not clear the remaining bits to zero, + // thus, we can mov rm in-place to tmp. + mov2 := m.allocateInstr() + switch lane { + case ssa.VecLaneI8x16: + mov2.asMovToVec(tmpReg, rm, vecArrangementB, vecIndex(index)) + case ssa.VecLaneI16x8: + mov2.asMovToVec(tmpReg, rm, vecArrangementH, vecIndex(index)) + case ssa.VecLaneI32x4: + mov2.asMovToVec(tmpReg, rm, vecArrangementS, vecIndex(index)) + case ssa.VecLaneI64x2: + mov2.asMovToVec(tmpReg, rm, vecArrangementD, vecIndex(index)) + case ssa.VecLaneF32x4: + mov2.asVecMovElement(tmpReg, rm, vecArrangementS, vecIndex(index), vecIndex(0)) + case ssa.VecLaneF64x2: + mov2.asVecMovElement(tmpReg, rm, vecArrangementD, vecIndex(index), vecIndex(0)) + } + m.insert(mov2) + + // Finally mov tmp to rd. + mov3 := m.allocateInstr() + mov3.asFpuMov128(rd.nr(), tmpReg.nr()) + m.insert(mov3) + + case ssa.OpcodeSwizzle: + x, y, lane := instr.Arg2WithLane() + rn := m.getOperand_NR(m.compiler.ValueDefinition(x), extModeNone) + rm := m.getOperand_NR(m.compiler.ValueDefinition(y), extModeNone) + rd := operandNR(m.compiler.VRegOf(instr.Return())) + + arr := ssaLaneToArrangement(lane) + + // tbl ., { . }, . + tbl1 := m.allocateInstr() + tbl1.asVecTbl(1, rd, rn, rm, arr) + m.insert(tbl1) + + case ssa.OpcodeShuffle: + x, y, lane1, lane2 := instr.ShuffleData() + rn := m.getOperand_NR(m.compiler.ValueDefinition(x), extModeNone) + rm := m.getOperand_NR(m.compiler.ValueDefinition(y), extModeNone) + rd := operandNR(m.compiler.VRegOf(instr.Return())) + + m.lowerShuffle(rd, rn, rm, lane1, lane2) + + case ssa.OpcodeSplat: + x, lane := instr.ArgWithLane() + rn := m.getOperand_NR(m.compiler.ValueDefinition(x), extModeNone) + rd := operandNR(m.compiler.VRegOf(instr.Return())) + + dup := m.allocateInstr() + switch lane { + case ssa.VecLaneI8x16: + dup.asVecDup(rd, rn, vecArrangement16B) + case ssa.VecLaneI16x8: + dup.asVecDup(rd, rn, vecArrangement8H) + case ssa.VecLaneI32x4: + dup.asVecDup(rd, rn, vecArrangement4S) + case ssa.VecLaneI64x2: + dup.asVecDup(rd, rn, vecArrangement2D) + case ssa.VecLaneF32x4: + dup.asVecDupElement(rd, rn, vecArrangementS, vecIndex(0)) + case ssa.VecLaneF64x2: + dup.asVecDupElement(rd, rn, vecArrangementD, vecIndex(0)) + } + m.insert(dup) + + case ssa.OpcodeWideningPairwiseDotProductS: + x, y := instr.Arg2() + xx, yy := m.getOperand_NR(m.compiler.ValueDefinition(x), extModeNone), + m.getOperand_NR(m.compiler.ValueDefinition(y), extModeNone) + tmp, tmp2 := operandNR(m.compiler.AllocateVReg(ssa.TypeV128)), operandNR(m.compiler.AllocateVReg(ssa.TypeV128)) + m.insert(m.allocateInstr().asVecRRR(vecOpSmull, tmp, xx, yy, vecArrangement8H)) + m.insert(m.allocateInstr().asVecRRR(vecOpSmull2, tmp2, xx, yy, vecArrangement8H)) + m.insert(m.allocateInstr().asVecRRR(vecOpAddp, tmp, tmp, tmp2, vecArrangement4S)) + + rd := operandNR(m.compiler.VRegOf(instr.Return())) + m.insert(m.allocateInstr().asFpuMov128(rd.nr(), tmp.nr())) + + case ssa.OpcodeLoadSplat: + ptr, offset, lane := instr.LoadSplatData() + m.lowerLoadSplat(ptr, offset, lane, instr.Return()) + + case ssa.OpcodeAtomicRmw: + m.lowerAtomicRmw(instr) + + case ssa.OpcodeAtomicCas: + m.lowerAtomicCas(instr) + + case ssa.OpcodeAtomicLoad: + m.lowerAtomicLoad(instr) + + case ssa.OpcodeAtomicStore: + m.lowerAtomicStore(instr) + + case ssa.OpcodeFence: + instr := m.allocateInstr() + instr.asDMB() + m.insert(instr) + + default: + panic("TODO: lowering " + op.String()) + } + m.executableContext.FlushPendingInstructions() +} + +func (m *machine) lowerShuffle(rd, rn, rm operand, lane1, lane2 uint64) { + // `tbl2` requires 2 consecutive registers, so we arbitrarily pick v29, v30. + vReg, wReg := v29VReg, v30VReg + + // Initialize v29, v30 to rn, rm. + movv := m.allocateInstr() + movv.asFpuMov128(vReg, rn.nr()) + m.insert(movv) + + movw := m.allocateInstr() + movw.asFpuMov128(wReg, rm.nr()) + m.insert(movw) + + // `lane1`, `lane2` are already encoded as two u64s with the right layout: + // lane1 := lane[7]<<56 | ... | lane[1]<<8 | lane[0] + // lane2 := lane[15]<<56 | ... | lane[9]<<8 | lane[8] + // Thus, we can use loadFpuConst128. + tmp := operandNR(m.compiler.AllocateVReg(ssa.TypeV128)) + lfc := m.allocateInstr() + lfc.asLoadFpuConst128(tmp.nr(), lane1, lane2) + m.insert(lfc) + + // tbl .16b, { .16B, .16b }, .16b + tbl2 := m.allocateInstr() + tbl2.asVecTbl(2, rd, operandNR(vReg), tmp, vecArrangement16B) + m.insert(tbl2) +} + +func (m *machine) lowerVShift(op ssa.Opcode, rd, rn, rm operand, arr vecArrangement) { + var modulo byte + switch arr { + case vecArrangement16B: + modulo = 0x7 // Modulo 8. + case vecArrangement8H: + modulo = 0xf // Modulo 16. + case vecArrangement4S: + modulo = 0x1f // Modulo 32. + case vecArrangement2D: + modulo = 0x3f // Modulo 64. + default: + panic("unsupported arrangment " + arr.String()) + } + + rtmp := operandNR(m.compiler.AllocateVReg(ssa.TypeI64)) + vtmp := operandNR(m.compiler.AllocateVReg(ssa.TypeV128)) + + and := m.allocateInstr() + and.asALUBitmaskImm(aluOpAnd, rtmp.nr(), rm.nr(), uint64(modulo), true) + m.insert(and) + + if op != ssa.OpcodeVIshl { + // Negate the amount to make this as right shift. + neg := m.allocateInstr() + neg.asALU(aluOpSub, rtmp, operandNR(xzrVReg), rtmp, true) + m.insert(neg) + } + + // Copy the shift amount into a vector register as sshl/ushl requires it to be there. + dup := m.allocateInstr() + dup.asVecDup(vtmp, rtmp, arr) + m.insert(dup) + + if op == ssa.OpcodeVIshl || op == ssa.OpcodeVSshr { + sshl := m.allocateInstr() + sshl.asVecRRR(vecOpSshl, rd, rn, vtmp, arr) + m.insert(sshl) + } else { + ushl := m.allocateInstr() + ushl.asVecRRR(vecOpUshl, rd, rn, vtmp, arr) + m.insert(ushl) + } +} + +func (m *machine) lowerVcheckTrue(op ssa.Opcode, rm, rd operand, arr vecArrangement) { + tmp := operandNR(m.compiler.AllocateVReg(ssa.TypeV128)) + + // Special case VallTrue for i64x2. + if op == ssa.OpcodeVallTrue && arr == vecArrangement2D { + // cmeq v3?.2d, v2?.2d, #0 + // addp v3?.2d, v3?.2d, v3?.2d + // fcmp v3?, v3? + // cset dst, eq + + ins := m.allocateInstr() + ins.asVecMisc(vecOpCmeq0, tmp, rm, vecArrangement2D) + m.insert(ins) + + addp := m.allocateInstr() + addp.asVecRRR(vecOpAddp, tmp, tmp, tmp, vecArrangement2D) + m.insert(addp) + + fcmp := m.allocateInstr() + fcmp.asFpuCmp(tmp, tmp, true) + m.insert(fcmp) + + cset := m.allocateInstr() + cset.asCSet(rd.nr(), false, eq) + m.insert(cset) + + return + } + + // Create a scalar value with umaxp or uminv, then compare it against zero. + ins := m.allocateInstr() + if op == ssa.OpcodeVanyTrue { + // umaxp v4?.16b, v2?.16b, v2?.16b + ins.asVecRRR(vecOpUmaxp, tmp, rm, rm, vecArrangement16B) + } else { + // uminv d4?, v2?.4s + ins.asVecLanes(vecOpUminv, tmp, rm, arr) + } + m.insert(ins) + + // mov x3?, v4?.d[0] + // ccmp x3?, #0x0, #0x0, al + // cset x3?, ne + // mov x0, x3? + + movv := m.allocateInstr() + movv.asMovFromVec(rd, tmp, vecArrangementD, vecIndex(0), false) + m.insert(movv) + + fc := m.allocateInstr() + fc.asCCmpImm(rd, uint64(0), al, 0, true) + m.insert(fc) + + cset := m.allocateInstr() + cset.asCSet(rd.nr(), false, ne) + m.insert(cset) +} + +func (m *machine) lowerVhighBits(rm, rd operand, arr vecArrangement) { + r0 := operandNR(m.compiler.AllocateVReg(ssa.TypeI64)) + v0 := operandNR(m.compiler.AllocateVReg(ssa.TypeV128)) + v1 := operandNR(m.compiler.AllocateVReg(ssa.TypeV128)) + + switch arr { + case vecArrangement16B: + // sshr v6?.16b, v2?.16b, #7 + // movz x4?, #0x201, lsl 0 + // movk x4?, #0x804, lsl 16 + // movk x4?, #0x2010, lsl 32 + // movk x4?, #0x8040, lsl 48 + // dup v5?.2d, x4? + // and v6?.16b, v6?.16b, v5?.16b + // ext v5?.16b, v6?.16b, v6?.16b, #8 + // zip1 v5?.16b, v6?.16b, v5?.16b + // addv s5?, v5?.8h + // umov s3?, v5?.h[0] + + // Right arithmetic shift on the original vector and store the result into v1. So we have: + // v1[i] = 0xff if vi<0, 0 otherwise. + sshr := m.allocateInstr() + sshr.asVecShiftImm(vecOpSshr, v1, rm, operandShiftImm(7), vecArrangement16B) + m.insert(sshr) + + // Load the bit mask into r0. + m.insertMOVZ(r0.nr(), 0x0201, 0, true) + m.insertMOVK(r0.nr(), 0x0804, 1, true) + m.insertMOVK(r0.nr(), 0x2010, 2, true) + m.insertMOVK(r0.nr(), 0x8040, 3, true) + + // dup r0 to v0. + dup := m.allocateInstr() + dup.asVecDup(v0, r0, vecArrangement2D) + m.insert(dup) + + // Lane-wise logical AND with the bit mask, meaning that we have + // v[i] = (1 << i) if vi<0, 0 otherwise. + // + // Below, we use the following notation: + // wi := (1 << i) if vi<0, 0 otherwise. + and := m.allocateInstr() + and.asVecRRR(vecOpAnd, v1, v1, v0, vecArrangement16B) + m.insert(and) + + // Swap the lower and higher 8 byte elements, and write it into v0, meaning that we have + // v0[i] = w(i+8) if i < 8, w(i-8) otherwise. + ext := m.allocateInstr() + ext.asVecExtract(v0, v1, v1, vecArrangement16B, uint32(8)) + m.insert(ext) + + // v = [w0, w8, ..., w7, w15] + zip1 := m.allocateInstr() + zip1.asVecPermute(vecOpZip1, v0, v1, v0, vecArrangement16B) + m.insert(zip1) + + // v.h[0] = w0 + ... + w15 + addv := m.allocateInstr() + addv.asVecLanes(vecOpAddv, v0, v0, vecArrangement8H) + m.insert(addv) + + // Extract the v.h[0] as the result. + movfv := m.allocateInstr() + movfv.asMovFromVec(rd, v0, vecArrangementH, vecIndex(0), false) + m.insert(movfv) + case vecArrangement8H: + // sshr v6?.8h, v2?.8h, #15 + // movz x4?, #0x1, lsl 0 + // movk x4?, #0x2, lsl 16 + // movk x4?, #0x4, lsl 32 + // movk x4?, #0x8, lsl 48 + // dup v5?.2d, x4? + // lsl x4?, x4?, 0x4 + // ins v5?.d[1], x4? + // and v5?.16b, v6?.16b, v5?.16b + // addv s5?, v5?.8h + // umov s3?, v5?.h[0] + + // Right arithmetic shift on the original vector and store the result into v1. So we have: + // v[i] = 0xffff if vi<0, 0 otherwise. + sshr := m.allocateInstr() + sshr.asVecShiftImm(vecOpSshr, v1, rm, operandShiftImm(15), vecArrangement8H) + m.insert(sshr) + + // Load the bit mask into r0. + m.lowerConstantI64(r0.nr(), 0x0008000400020001) + + // dup r0 to vector v0. + dup := m.allocateInstr() + dup.asVecDup(v0, r0, vecArrangement2D) + m.insert(dup) + + lsl := m.allocateInstr() + lsl.asALUShift(aluOpLsl, r0, r0, operandShiftImm(4), true) + m.insert(lsl) + + movv := m.allocateInstr() + movv.asMovToVec(v0, r0, vecArrangementD, vecIndex(1)) + m.insert(movv) + + // Lane-wise logical AND with the bitmask, meaning that we have + // v[i] = (1 << i) if vi<0, 0 otherwise for i=0..3 + // = (1 << (i+4)) if vi<0, 0 otherwise for i=3..7 + and := m.allocateInstr() + and.asVecRRR(vecOpAnd, v0, v1, v0, vecArrangement16B) + m.insert(and) + + addv := m.allocateInstr() + addv.asVecLanes(vecOpAddv, v0, v0, vecArrangement8H) + m.insert(addv) + + movfv := m.allocateInstr() + movfv.asMovFromVec(rd, v0, vecArrangementH, vecIndex(0), false) + m.insert(movfv) + case vecArrangement4S: + // sshr v6?.8h, v2?.8h, #15 + // movz x4?, #0x1, lsl 0 + // movk x4?, #0x2, lsl 16 + // movk x4?, #0x4, lsl 32 + // movk x4?, #0x8, lsl 48 + // dup v5?.2d, x4? + // lsl x4?, x4?, 0x4 + // ins v5?.d[1], x4? + // and v5?.16b, v6?.16b, v5?.16b + // addv s5?, v5?.8h + // umov s3?, v5?.h[0] + + // Right arithmetic shift on the original vector and store the result into v1. So we have: + // v[i] = 0xffffffff if vi<0, 0 otherwise. + sshr := m.allocateInstr() + sshr.asVecShiftImm(vecOpSshr, v1, rm, operandShiftImm(31), vecArrangement4S) + m.insert(sshr) + + // Load the bit mask into r0. + m.lowerConstantI64(r0.nr(), 0x0000000200000001) + + // dup r0 to vector v0. + dup := m.allocateInstr() + dup.asVecDup(v0, r0, vecArrangement2D) + m.insert(dup) + + lsl := m.allocateInstr() + lsl.asALUShift(aluOpLsl, r0, r0, operandShiftImm(2), true) + m.insert(lsl) + + movv := m.allocateInstr() + movv.asMovToVec(v0, r0, vecArrangementD, vecIndex(1)) + m.insert(movv) + + // Lane-wise logical AND with the bitmask, meaning that we have + // v[i] = (1 << i) if vi<0, 0 otherwise for i in [0, 1] + // = (1 << (i+4)) if vi<0, 0 otherwise for i in [2, 3] + and := m.allocateInstr() + and.asVecRRR(vecOpAnd, v0, v1, v0, vecArrangement16B) + m.insert(and) + + addv := m.allocateInstr() + addv.asVecLanes(vecOpAddv, v0, v0, vecArrangement4S) + m.insert(addv) + + movfv := m.allocateInstr() + movfv.asMovFromVec(rd, v0, vecArrangementS, vecIndex(0), false) + m.insert(movfv) + case vecArrangement2D: + // mov d3?, v2?.d[0] + // mov x4?, v2?.d[1] + // lsr x4?, x4?, 0x3f + // lsr d3?, d3?, 0x3f + // add s3?, s3?, w4?, lsl #1 + + // Move the lower 64-bit int into result. + movv0 := m.allocateInstr() + movv0.asMovFromVec(rd, rm, vecArrangementD, vecIndex(0), false) + m.insert(movv0) + + // Move the higher 64-bit int into r0. + movv1 := m.allocateInstr() + movv1.asMovFromVec(r0, rm, vecArrangementD, vecIndex(1), false) + m.insert(movv1) + + // Move the sign bit into the least significant bit. + lsr1 := m.allocateInstr() + lsr1.asALUShift(aluOpLsr, r0, r0, operandShiftImm(63), true) + m.insert(lsr1) + + lsr2 := m.allocateInstr() + lsr2.asALUShift(aluOpLsr, rd, rd, operandShiftImm(63), true) + m.insert(lsr2) + + // rd = (r0<<1) | rd + lsl := m.allocateInstr() + lsl.asALU(aluOpAdd, rd, rd, operandSR(r0.nr(), 1, shiftOpLSL), false) + m.insert(lsl) + default: + panic("Unsupported " + arr.String()) + } +} + +func (m *machine) lowerVecMisc(op vecOp, instr *ssa.Instruction) { + x, lane := instr.ArgWithLane() + arr := ssaLaneToArrangement(lane) + ins := m.allocateInstr() + rn := m.getOperand_NR(m.compiler.ValueDefinition(x), extModeNone) + rd := operandNR(m.compiler.VRegOf(instr.Return())) + ins.asVecMisc(op, rd, rn, arr) + m.insert(ins) +} + +func (m *machine) lowerVecRRR(op vecOp, x, y, ret ssa.Value, arr vecArrangement) { + ins := m.allocateInstr() + rn := m.getOperand_NR(m.compiler.ValueDefinition(x), extModeNone) + rm := m.getOperand_NR(m.compiler.ValueDefinition(y), extModeNone) + rd := operandNR(m.compiler.VRegOf(ret)) + ins.asVecRRR(op, rd, rn, rm, arr) + m.insert(ins) +} + +func (m *machine) lowerVIMul(rd, rn, rm operand, arr vecArrangement) { + if arr != vecArrangement2D { + mul := m.allocateInstr() + mul.asVecRRR(vecOpMul, rd, rn, rm, arr) + m.insert(mul) + } else { + tmp1 := operandNR(m.compiler.AllocateVReg(ssa.TypeV128)) + tmp2 := operandNR(m.compiler.AllocateVReg(ssa.TypeV128)) + tmp3 := operandNR(m.compiler.AllocateVReg(ssa.TypeV128)) + + tmpRes := operandNR(m.compiler.AllocateVReg(ssa.TypeV128)) + + // Following the algorithm in https://chromium-review.googlesource.com/c/v8/v8/+/1781696 + rev64 := m.allocateInstr() + rev64.asVecMisc(vecOpRev64, tmp2, rm, vecArrangement4S) + m.insert(rev64) + + mul := m.allocateInstr() + mul.asVecRRR(vecOpMul, tmp2, tmp2, rn, vecArrangement4S) + m.insert(mul) + + xtn1 := m.allocateInstr() + xtn1.asVecMisc(vecOpXtn, tmp1, rn, vecArrangement2S) + m.insert(xtn1) + + addp := m.allocateInstr() + addp.asVecRRR(vecOpAddp, tmp2, tmp2, tmp2, vecArrangement4S) + m.insert(addp) + + xtn2 := m.allocateInstr() + xtn2.asVecMisc(vecOpXtn, tmp3, rm, vecArrangement2S) + m.insert(xtn2) + + // Note: do not write the result directly into result yet. This is the same reason as in bsl. + // In short, in UMLAL instruction, the result register is also one of the source register, and + // the value on the result register is significant. + shll := m.allocateInstr() + shll.asVecMisc(vecOpShll, tmpRes, tmp2, vecArrangement2S) + m.insert(shll) + + umlal := m.allocateInstr() + umlal.asVecRRRRewrite(vecOpUmlal, tmpRes, tmp3, tmp1, vecArrangement2S) + m.insert(umlal) + + mov := m.allocateInstr() + mov.asFpuMov128(rd.nr(), tmpRes.nr()) + m.insert(mov) + } +} + +func (m *machine) lowerVMinMaxPseudo(instr *ssa.Instruction, max bool) { + x, y, lane := instr.Arg2WithLane() + arr := ssaLaneToArrangement(lane) + + rn := m.getOperand_NR(m.compiler.ValueDefinition(x), extModeNone) + rm := m.getOperand_NR(m.compiler.ValueDefinition(y), extModeNone) + + // Note: this usage of tmp is important. + // BSL modifies the destination register, so we need to use a temporary register so that + // the actual definition of the destination register happens *after* the BSL instruction. + // That way, we can force the spill instruction to be inserted after the BSL instruction. + tmp := operandNR(m.compiler.AllocateVReg(ssa.TypeV128)) + + fcmgt := m.allocateInstr() + if max { + fcmgt.asVecRRR(vecOpFcmgt, tmp, rm, rn, arr) + } else { + // If min, swap the args. + fcmgt.asVecRRR(vecOpFcmgt, tmp, rn, rm, arr) + } + m.insert(fcmgt) + + bsl := m.allocateInstr() + bsl.asVecRRRRewrite(vecOpBsl, tmp, rm, rn, vecArrangement16B) + m.insert(bsl) + + res := operandNR(m.compiler.VRegOf(instr.Return())) + mov2 := m.allocateInstr() + mov2.asFpuMov128(res.nr(), tmp.nr()) + m.insert(mov2) +} + +func (m *machine) lowerIRem(execCtxVReg regalloc.VReg, rd, rn, rm operand, _64bit, signed bool) { + div := m.allocateInstr() + + if signed { + div.asALU(aluOpSDiv, rd, rn, rm, _64bit) + } else { + div.asALU(aluOpUDiv, rd, rn, rm, _64bit) + } + m.insert(div) + + // Check if rm is zero: + m.exitIfNot(execCtxVReg, registerAsRegNotZeroCond(rm.nr()), _64bit, wazevoapi.ExitCodeIntegerDivisionByZero) + + // rd = rn-rd*rm by MSUB instruction. + msub := m.allocateInstr() + msub.asALURRRR(aluOpMSub, rd, rd, rm, rn, _64bit) + m.insert(msub) +} + +func (m *machine) lowerIDiv(execCtxVReg regalloc.VReg, rd, rn, rm operand, _64bit, signed bool) { + div := m.allocateInstr() + + if signed { + div.asALU(aluOpSDiv, rd, rn, rm, _64bit) + } else { + div.asALU(aluOpUDiv, rd, rn, rm, _64bit) + } + m.insert(div) + + // Check if rm is zero: + m.exitIfNot(execCtxVReg, registerAsRegNotZeroCond(rm.nr()), _64bit, wazevoapi.ExitCodeIntegerDivisionByZero) + + if signed { + // We need to check the signed overflow which happens iff "math.MinInt{32,64} / -1" + minusOneCheck := m.allocateInstr() + // Sets eq condition if rm == -1. + minusOneCheck.asALU(aluOpAddS, operandNR(xzrVReg), rm, operandImm12(1, 0), _64bit) + m.insert(minusOneCheck) + + ccmp := m.allocateInstr() + // If eq condition is set, sets the flag by the result based on "rn - 1", otherwise clears the flag. + ccmp.asCCmpImm(rn, 1, eq, 0, _64bit) + m.insert(ccmp) + + // Check the overflow flag. + m.exitIfNot(execCtxVReg, vs.invert().asCond(), false, wazevoapi.ExitCodeIntegerOverflow) + } +} + +// exitIfNot emits a conditional branch to exit if the condition is not met. +// If `c` (cond type) is a register, `cond64bit` must be chosen to indicate whether the register is 32-bit or 64-bit. +// Otherwise, `cond64bit` is ignored. +func (m *machine) exitIfNot(execCtxVReg regalloc.VReg, c cond, cond64bit bool, code wazevoapi.ExitCode) { + execCtxTmp := m.copyToTmp(execCtxVReg) + + cbr := m.allocateInstr() + m.insert(cbr) + m.lowerExitWithCode(execCtxTmp, code) + // Conditional branch target is after exit. + l := m.insertBrTargetLabel() + cbr.asCondBr(c, l, cond64bit) +} + +func (m *machine) lowerFcopysign(x, y, ret ssa.Value) { + rn := m.getOperand_NR(m.compiler.ValueDefinition(x), extModeNone) + rm := m.getOperand_NR(m.compiler.ValueDefinition(y), extModeNone) + var tmpI, tmpF operand + _64 := x.Type() == ssa.TypeF64 + if _64 { + tmpF = operandNR(m.compiler.AllocateVReg(ssa.TypeF64)) + tmpI = operandNR(m.compiler.AllocateVReg(ssa.TypeI64)) + } else { + tmpF = operandNR(m.compiler.AllocateVReg(ssa.TypeF32)) + tmpI = operandNR(m.compiler.AllocateVReg(ssa.TypeI32)) + } + rd := m.compiler.VRegOf(ret) + m.lowerFcopysignImpl(operandNR(rd), rn, rm, tmpI, tmpF, _64) +} + +func (m *machine) lowerFcopysignImpl(rd, rn, rm, tmpI, tmpF operand, _64bit bool) { + // This is exactly the same code emitted by GCC for "__builtin_copysign": + // + // mov x0, -9223372036854775808 + // fmov d2, x0 + // vbit v0.8b, v1.8b, v2.8b + // + + setMSB := m.allocateInstr() + if _64bit { + m.lowerConstantI64(tmpI.nr(), math.MinInt64) + setMSB.asMovToVec(tmpF, tmpI, vecArrangementD, vecIndex(0)) + } else { + m.lowerConstantI32(tmpI.nr(), math.MinInt32) + setMSB.asMovToVec(tmpF, tmpI, vecArrangementS, vecIndex(0)) + } + m.insert(setMSB) + + tmpReg := operandNR(m.compiler.AllocateVReg(ssa.TypeF64)) + + mov := m.allocateInstr() + mov.asFpuMov64(tmpReg.nr(), rn.nr()) + m.insert(mov) + + vbit := m.allocateInstr() + vbit.asVecRRRRewrite(vecOpBit, tmpReg, rm, tmpF, vecArrangement8B) + m.insert(vbit) + + movDst := m.allocateInstr() + movDst.asFpuMov64(rd.nr(), tmpReg.nr()) + m.insert(movDst) +} + +func (m *machine) lowerBitcast(instr *ssa.Instruction) { + v, dstType := instr.BitcastData() + srcType := v.Type() + rn := m.getOperand_NR(m.compiler.ValueDefinition(v), extModeNone) + rd := operandNR(m.compiler.VRegOf(instr.Return())) + srcInt := srcType.IsInt() + dstInt := dstType.IsInt() + switch { + case srcInt && !dstInt: // Int to Float: + mov := m.allocateInstr() + var arr vecArrangement + if srcType.Bits() == 64 { + arr = vecArrangementD + } else { + arr = vecArrangementS + } + mov.asMovToVec(rd, rn, arr, vecIndex(0)) + m.insert(mov) + case !srcInt && dstInt: // Float to Int: + mov := m.allocateInstr() + var arr vecArrangement + if dstType.Bits() == 64 { + arr = vecArrangementD + } else { + arr = vecArrangementS + } + mov.asMovFromVec(rd, rn, arr, vecIndex(0), false) + m.insert(mov) + default: + panic("TODO?BUG?") + } +} + +func (m *machine) lowerFpuUniOp(op fpuUniOp, in, out ssa.Value) { + rn := m.getOperand_NR(m.compiler.ValueDefinition(in), extModeNone) + rd := operandNR(m.compiler.VRegOf(out)) + + neg := m.allocateInstr() + neg.asFpuRR(op, rd, rn, in.Type().Bits() == 64) + m.insert(neg) +} + +func (m *machine) lowerFpuToInt(rd, rn operand, ctx regalloc.VReg, signed, src64bit, dst64bit, nonTrapping bool) { + if !nonTrapping { + // First of all, we have to clear the FPU flags. + flagClear := m.allocateInstr() + flagClear.asMovToFPSR(xzrVReg) + m.insert(flagClear) + } + + // Then, do the conversion which doesn't trap inherently. + cvt := m.allocateInstr() + cvt.asFpuToInt(rd, rn, signed, src64bit, dst64bit) + m.insert(cvt) + + if !nonTrapping { + tmpReg := m.compiler.AllocateVReg(ssa.TypeI64) + + // After the conversion, check the FPU flags. + getFlag := m.allocateInstr() + getFlag.asMovFromFPSR(tmpReg) + m.insert(getFlag) + + execCtx := m.copyToTmp(ctx) + _rn := operandNR(m.copyToTmp(rn.nr())) + + // Check if the conversion was undefined by comparing the status with 1. + // See https://developer.arm.com/documentation/ddi0595/2020-12/AArch64-Registers/FPSR--Floating-point-Status-Register + alu := m.allocateInstr() + alu.asALU(aluOpSubS, operandNR(xzrVReg), operandNR(tmpReg), operandImm12(1, 0), true) + m.insert(alu) + + // If it is not undefined, we can return the result. + ok := m.allocateInstr() + m.insert(ok) + + // Otherwise, we have to choose the status depending on it is overflow or NaN conversion. + + // Comparing itself to check if it is a NaN. + fpuCmp := m.allocateInstr() + fpuCmp.asFpuCmp(_rn, _rn, src64bit) + m.insert(fpuCmp) + // If the VC flag is not set (== VS flag is set), it is a NaN. + m.exitIfNot(execCtx, vc.asCond(), false, wazevoapi.ExitCodeInvalidConversionToInteger) + // Otherwise, it is an overflow. + m.lowerExitWithCode(execCtx, wazevoapi.ExitCodeIntegerOverflow) + + // Conditional branch target is after exit. + l := m.insertBrTargetLabel() + ok.asCondBr(ne.asCond(), l, false /* ignored */) + } +} + +func (m *machine) lowerIntToFpu(rd, rn operand, signed, src64bit, dst64bit bool) { + cvt := m.allocateInstr() + cvt.asIntToFpu(rd, rn, signed, src64bit, dst64bit) + m.insert(cvt) +} + +func (m *machine) lowerFpuBinOp(si *ssa.Instruction) { + instr := m.allocateInstr() + var op fpuBinOp + switch si.Opcode() { + case ssa.OpcodeFadd: + op = fpuBinOpAdd + case ssa.OpcodeFsub: + op = fpuBinOpSub + case ssa.OpcodeFmul: + op = fpuBinOpMul + case ssa.OpcodeFdiv: + op = fpuBinOpDiv + case ssa.OpcodeFmax: + op = fpuBinOpMax + case ssa.OpcodeFmin: + op = fpuBinOpMin + } + x, y := si.Arg2() + xDef, yDef := m.compiler.ValueDefinition(x), m.compiler.ValueDefinition(y) + rn := m.getOperand_NR(xDef, extModeNone) + rm := m.getOperand_NR(yDef, extModeNone) + rd := operandNR(m.compiler.VRegOf(si.Return())) + instr.asFpuRRR(op, rd, rn, rm, x.Type().Bits() == 64) + m.insert(instr) +} + +func (m *machine) lowerSubOrAdd(si *ssa.Instruction, add bool) { + x, y := si.Arg2() + if !x.Type().IsInt() { + panic("BUG?") + } + + xDef, yDef := m.compiler.ValueDefinition(x), m.compiler.ValueDefinition(y) + rn := m.getOperand_NR(xDef, extModeNone) + rm, yNegated := m.getOperand_MaybeNegatedImm12_ER_SR_NR(yDef, extModeNone) + + var aop aluOp + switch { + case add && !yNegated: // rn+rm = x+y + aop = aluOpAdd + case add && yNegated: // rn-rm = x-(-y) = x+y + aop = aluOpSub + case !add && !yNegated: // rn-rm = x-y + aop = aluOpSub + case !add && yNegated: // rn+rm = x-(-y) = x-y + aop = aluOpAdd + } + rd := operandNR(m.compiler.VRegOf(si.Return())) + alu := m.allocateInstr() + alu.asALU(aop, rd, rn, rm, x.Type().Bits() == 64) + m.insert(alu) +} + +// InsertMove implements backend.Machine. +func (m *machine) InsertMove(dst, src regalloc.VReg, typ ssa.Type) { + instr := m.allocateInstr() + switch typ { + case ssa.TypeI32, ssa.TypeI64: + instr.asMove64(dst, src) + case ssa.TypeF32, ssa.TypeF64: + instr.asFpuMov64(dst, src) + case ssa.TypeV128: + instr.asFpuMov128(dst, src) + default: + panic("TODO") + } + m.insert(instr) +} + +func (m *machine) lowerIcmp(si *ssa.Instruction) { + x, y, c := si.IcmpData() + flag := condFlagFromSSAIntegerCmpCond(c) + + in64bit := x.Type().Bits() == 64 + var ext extMode + if in64bit { + if c.Signed() { + ext = extModeSignExtend64 + } else { + ext = extModeZeroExtend64 + } + } else { + if c.Signed() { + ext = extModeSignExtend32 + } else { + ext = extModeZeroExtend32 + } + } + + rn := m.getOperand_NR(m.compiler.ValueDefinition(x), ext) + rm := m.getOperand_Imm12_ER_SR_NR(m.compiler.ValueDefinition(y), ext) + alu := m.allocateInstr() + alu.asALU(aluOpSubS, operandNR(xzrVReg), rn, rm, in64bit) + m.insert(alu) + + cset := m.allocateInstr() + cset.asCSet(m.compiler.VRegOf(si.Return()), false, flag) + m.insert(cset) +} + +func (m *machine) lowerVIcmp(si *ssa.Instruction) { + x, y, c, lane := si.VIcmpData() + flag := condFlagFromSSAIntegerCmpCond(c) + arr := ssaLaneToArrangement(lane) + + rn := m.getOperand_NR(m.compiler.ValueDefinition(x), extModeNone) + rm := m.getOperand_NR(m.compiler.ValueDefinition(y), extModeNone) + rd := operandNR(m.compiler.VRegOf(si.Return())) + + switch flag { + case eq: + cmp := m.allocateInstr() + cmp.asVecRRR(vecOpCmeq, rd, rn, rm, arr) + m.insert(cmp) + case ne: + cmp := m.allocateInstr() + cmp.asVecRRR(vecOpCmeq, rd, rn, rm, arr) + m.insert(cmp) + not := m.allocateInstr() + not.asVecMisc(vecOpNot, rd, rd, vecArrangement16B) + m.insert(not) + case ge: + cmp := m.allocateInstr() + cmp.asVecRRR(vecOpCmge, rd, rn, rm, arr) + m.insert(cmp) + case gt: + cmp := m.allocateInstr() + cmp.asVecRRR(vecOpCmgt, rd, rn, rm, arr) + m.insert(cmp) + case le: + cmp := m.allocateInstr() + cmp.asVecRRR(vecOpCmge, rd, rm, rn, arr) // rm, rn are swapped + m.insert(cmp) + case lt: + cmp := m.allocateInstr() + cmp.asVecRRR(vecOpCmgt, rd, rm, rn, arr) // rm, rn are swapped + m.insert(cmp) + case hs: + cmp := m.allocateInstr() + cmp.asVecRRR(vecOpCmhs, rd, rn, rm, arr) + m.insert(cmp) + case hi: + cmp := m.allocateInstr() + cmp.asVecRRR(vecOpCmhi, rd, rn, rm, arr) + m.insert(cmp) + case ls: + cmp := m.allocateInstr() + cmp.asVecRRR(vecOpCmhs, rd, rm, rn, arr) // rm, rn are swapped + m.insert(cmp) + case lo: + cmp := m.allocateInstr() + cmp.asVecRRR(vecOpCmhi, rd, rm, rn, arr) // rm, rn are swapped + m.insert(cmp) + } +} + +func (m *machine) lowerVFcmp(si *ssa.Instruction) { + x, y, c, lane := si.VFcmpData() + flag := condFlagFromSSAFloatCmpCond(c) + arr := ssaLaneToArrangement(lane) + + rn := m.getOperand_NR(m.compiler.ValueDefinition(x), extModeNone) + rm := m.getOperand_NR(m.compiler.ValueDefinition(y), extModeNone) + rd := operandNR(m.compiler.VRegOf(si.Return())) + + switch flag { + case eq: + cmp := m.allocateInstr() + cmp.asVecRRR(vecOpFcmeq, rd, rn, rm, arr) + m.insert(cmp) + case ne: + cmp := m.allocateInstr() + cmp.asVecRRR(vecOpFcmeq, rd, rn, rm, arr) + m.insert(cmp) + not := m.allocateInstr() + not.asVecMisc(vecOpNot, rd, rd, vecArrangement16B) + m.insert(not) + case ge: + cmp := m.allocateInstr() + cmp.asVecRRR(vecOpFcmge, rd, rn, rm, arr) + m.insert(cmp) + case gt: + cmp := m.allocateInstr() + cmp.asVecRRR(vecOpFcmgt, rd, rn, rm, arr) + m.insert(cmp) + case mi: + cmp := m.allocateInstr() + cmp.asVecRRR(vecOpFcmgt, rd, rm, rn, arr) // rm, rn are swapped + m.insert(cmp) + case ls: + cmp := m.allocateInstr() + cmp.asVecRRR(vecOpFcmge, rd, rm, rn, arr) // rm, rn are swapped + m.insert(cmp) + } +} + +func (m *machine) lowerVfpuToInt(rd, rn operand, arr vecArrangement, signed bool) { + cvt := m.allocateInstr() + if signed { + cvt.asVecMisc(vecOpFcvtzs, rd, rn, arr) + } else { + cvt.asVecMisc(vecOpFcvtzu, rd, rn, arr) + } + m.insert(cvt) + + if arr == vecArrangement2D { + narrow := m.allocateInstr() + if signed { + narrow.asVecMisc(vecOpSqxtn, rd, rd, vecArrangement2S) + } else { + narrow.asVecMisc(vecOpUqxtn, rd, rd, vecArrangement2S) + } + m.insert(narrow) + } +} + +func (m *machine) lowerVfpuFromInt(rd, rn operand, arr vecArrangement, signed bool) { + cvt := m.allocateInstr() + if signed { + cvt.asVecMisc(vecOpScvtf, rd, rn, arr) + } else { + cvt.asVecMisc(vecOpUcvtf, rd, rn, arr) + } + m.insert(cvt) +} + +func (m *machine) lowerShifts(si *ssa.Instruction, ext extMode, aluOp aluOp) { + x, amount := si.Arg2() + rn := m.getOperand_NR(m.compiler.ValueDefinition(x), ext) + rm := m.getOperand_ShiftImm_NR(m.compiler.ValueDefinition(amount), ext, x.Type().Bits()) + rd := operandNR(m.compiler.VRegOf(si.Return())) + + alu := m.allocateInstr() + alu.asALUShift(aluOp, rd, rn, rm, x.Type().Bits() == 64) + m.insert(alu) +} + +func (m *machine) lowerBitwiseAluOp(si *ssa.Instruction, op aluOp, ignoreResult bool) { + x, y := si.Arg2() + + xDef, yDef := m.compiler.ValueDefinition(x), m.compiler.ValueDefinition(y) + rn := m.getOperand_NR(xDef, extModeNone) + + var rd operand + if ignoreResult { + rd = operandNR(xzrVReg) + } else { + rd = operandNR(m.compiler.VRegOf(si.Return())) + } + + _64 := x.Type().Bits() == 64 + alu := m.allocateInstr() + if instr := yDef.Instr; instr != nil && instr.Constant() { + c := instr.ConstantVal() + if isBitMaskImmediate(c, _64) { + // Constant bit wise operations can be lowered to a single instruction. + alu.asALUBitmaskImm(op, rd.nr(), rn.nr(), c, _64) + m.insert(alu) + return + } + } + + rm := m.getOperand_SR_NR(yDef, extModeNone) + alu.asALU(op, rd, rn, rm, _64) + m.insert(alu) +} + +func (m *machine) lowerRotl(si *ssa.Instruction) { + x, y := si.Arg2() + r := si.Return() + _64 := r.Type().Bits() == 64 + + rn := m.getOperand_NR(m.compiler.ValueDefinition(x), extModeNone) + rm := m.getOperand_NR(m.compiler.ValueDefinition(y), extModeNone) + var tmp operand + if _64 { + tmp = operandNR(m.compiler.AllocateVReg(ssa.TypeI64)) + } else { + tmp = operandNR(m.compiler.AllocateVReg(ssa.TypeI32)) + } + rd := operandNR(m.compiler.VRegOf(r)) + + // Encode rotl as neg + rotr: neg is a sub against the zero-reg. + m.lowerRotlImpl(rd, rn, rm, tmp, _64) +} + +func (m *machine) lowerRotlImpl(rd, rn, rm, tmp operand, is64bit bool) { + // Encode rotl as neg + rotr: neg is a sub against the zero-reg. + neg := m.allocateInstr() + neg.asALU(aluOpSub, tmp, operandNR(xzrVReg), rm, is64bit) + m.insert(neg) + alu := m.allocateInstr() + alu.asALU(aluOpRotR, rd, rn, tmp, is64bit) + m.insert(alu) +} + +func (m *machine) lowerRotr(si *ssa.Instruction) { + x, y := si.Arg2() + + xDef, yDef := m.compiler.ValueDefinition(x), m.compiler.ValueDefinition(y) + rn := m.getOperand_NR(xDef, extModeNone) + rm := m.getOperand_NR(yDef, extModeNone) + rd := operandNR(m.compiler.VRegOf(si.Return())) + + alu := m.allocateInstr() + alu.asALU(aluOpRotR, rd, rn, rm, si.Return().Type().Bits() == 64) + m.insert(alu) +} + +func (m *machine) lowerExtend(arg, ret ssa.Value, from, to byte, signed bool) { + rd := m.compiler.VRegOf(ret) + def := m.compiler.ValueDefinition(arg) + + if instr := def.Instr; !signed && from == 32 && instr != nil { + // We can optimize out the unsigned extend because: + // Writes to the W register set bits [63:32] of the X register to zero + // https://developer.arm.com/documentation/den0024/a/An-Introduction-to-the-ARMv8-Instruction-Sets/The-ARMv8-instruction-sets/Distinguishing-between-32-bit-and-64-bit-A64-instructions + switch instr.Opcode() { + case + ssa.OpcodeIadd, ssa.OpcodeIsub, ssa.OpcodeLoad, + ssa.OpcodeBand, ssa.OpcodeBor, ssa.OpcodeBnot, + ssa.OpcodeIshl, ssa.OpcodeUshr, ssa.OpcodeSshr, + ssa.OpcodeRotl, ssa.OpcodeRotr, + ssa.OpcodeUload8, ssa.OpcodeUload16, ssa.OpcodeUload32: + // So, if the argument is the result of a 32-bit operation, we can just copy the register. + // It is highly likely that this copy will be optimized out after register allocation. + rn := m.compiler.VRegOf(arg) + mov := m.allocateInstr() + // Note: do not use move32 as it will be lowered to a 32-bit move, which is not copy (that is actually the impl of UExtend). + mov.asMove64(rd, rn) + m.insert(mov) + return + default: + } + } + rn := m.getOperand_NR(def, extModeNone) + + ext := m.allocateInstr() + ext.asExtend(rd, rn.nr(), from, to, signed) + m.insert(ext) +} + +func (m *machine) lowerFcmp(x, y, result ssa.Value, c ssa.FloatCmpCond) { + rn, rm := m.getOperand_NR(m.compiler.ValueDefinition(x), extModeNone), m.getOperand_NR(m.compiler.ValueDefinition(y), extModeNone) + + fc := m.allocateInstr() + fc.asFpuCmp(rn, rm, x.Type().Bits() == 64) + m.insert(fc) + + cset := m.allocateInstr() + cset.asCSet(m.compiler.VRegOf(result), false, condFlagFromSSAFloatCmpCond(c)) + m.insert(cset) +} + +func (m *machine) lowerImul(x, y, result ssa.Value) { + rd := m.compiler.VRegOf(result) + rn := m.getOperand_NR(m.compiler.ValueDefinition(x), extModeNone) + rm := m.getOperand_NR(m.compiler.ValueDefinition(y), extModeNone) + + // TODO: if this comes before Add/Sub, we could merge it by putting it into the place of xzrVReg. + + mul := m.allocateInstr() + mul.asALURRRR(aluOpMAdd, operandNR(rd), rn, rm, operandNR(xzrVReg), x.Type().Bits() == 64) + m.insert(mul) +} + +func (m *machine) lowerClz(x, result ssa.Value) { + rd := m.compiler.VRegOf(result) + rn := m.getOperand_NR(m.compiler.ValueDefinition(x), extModeNone) + clz := m.allocateInstr() + clz.asBitRR(bitOpClz, rd, rn.nr(), x.Type().Bits() == 64) + m.insert(clz) +} + +func (m *machine) lowerCtz(x, result ssa.Value) { + rd := m.compiler.VRegOf(result) + rn := m.getOperand_NR(m.compiler.ValueDefinition(x), extModeNone) + rbit := m.allocateInstr() + _64 := x.Type().Bits() == 64 + var tmpReg regalloc.VReg + if _64 { + tmpReg = m.compiler.AllocateVReg(ssa.TypeI64) + } else { + tmpReg = m.compiler.AllocateVReg(ssa.TypeI32) + } + rbit.asBitRR(bitOpRbit, tmpReg, rn.nr(), _64) + m.insert(rbit) + + clz := m.allocateInstr() + clz.asBitRR(bitOpClz, rd, tmpReg, _64) + m.insert(clz) +} + +func (m *machine) lowerPopcnt(x, result ssa.Value) { + // arm64 doesn't have an instruction for population count on scalar register, + // so we use the vector instruction `cnt`. + // This is exactly what the official Go implements bits.OneCount. + // For example, "func () int { return bits.OneCount(10) }" is compiled as + // + // MOVD $10, R0 ;; Load 10. + // FMOVD R0, F0 + // VCNT V0.B8, V0.B8 + // UADDLV V0.B8, V0 + // + // In aarch64 asm, FMOVD is encoded as `ins`, VCNT is `cnt`, + // and the registers may use different names. In our encoding we use the following + // instructions: + // + // ins v0.d[0], x0 ;; mov from GPR to vec (FMOV above) is encoded as INS + // cnt v0.16b, v0.16b ;; we use vec arrangement 16b + // uaddlv h0, v0.8b ;; h0 is still v0 with the dest width specifier 'H', implied when src arrangement is 8b + // mov x5, v0.d[0] ;; finally we mov the result back to a GPR + // + + rd := operandNR(m.compiler.VRegOf(result)) + rn := m.getOperand_NR(m.compiler.ValueDefinition(x), extModeNone) + + rf1 := operandNR(m.compiler.AllocateVReg(ssa.TypeF64)) + ins := m.allocateInstr() + ins.asMovToVec(rf1, rn, vecArrangementD, vecIndex(0)) + m.insert(ins) + + rf2 := operandNR(m.compiler.AllocateVReg(ssa.TypeF64)) + cnt := m.allocateInstr() + cnt.asVecMisc(vecOpCnt, rf2, rf1, vecArrangement16B) + m.insert(cnt) + + rf3 := operandNR(m.compiler.AllocateVReg(ssa.TypeF64)) + uaddlv := m.allocateInstr() + uaddlv.asVecLanes(vecOpUaddlv, rf3, rf2, vecArrangement8B) + m.insert(uaddlv) + + mov := m.allocateInstr() + mov.asMovFromVec(rd, rf3, vecArrangementD, vecIndex(0), false) + m.insert(mov) +} + +// lowerExitWithCode lowers the lowerExitWithCode takes a context pointer as argument. +func (m *machine) lowerExitWithCode(execCtxVReg regalloc.VReg, code wazevoapi.ExitCode) { + tmpReg1 := m.compiler.AllocateVReg(ssa.TypeI32) + loadExitCodeConst := m.allocateInstr() + loadExitCodeConst.asMOVZ(tmpReg1, uint64(code), 0, true) + + setExitCode := m.allocateInstr() + setExitCode.asStore(operandNR(tmpReg1), + addressMode{ + kind: addressModeKindRegUnsignedImm12, + rn: execCtxVReg, imm: wazevoapi.ExecutionContextOffsetExitCodeOffset.I64(), + }, 32) + + // In order to unwind the stack, we also need to push the current stack pointer: + tmp2 := m.compiler.AllocateVReg(ssa.TypeI64) + movSpToTmp := m.allocateInstr() + movSpToTmp.asMove64(tmp2, spVReg) + strSpToExecCtx := m.allocateInstr() + strSpToExecCtx.asStore(operandNR(tmp2), + addressMode{ + kind: addressModeKindRegUnsignedImm12, + rn: execCtxVReg, imm: wazevoapi.ExecutionContextOffsetStackPointerBeforeGoCall.I64(), + }, 64) + // Also the address of this exit. + tmp3 := m.compiler.AllocateVReg(ssa.TypeI64) + currentAddrToTmp := m.allocateInstr() + currentAddrToTmp.asAdr(tmp3, 0) + storeCurrentAddrToExecCtx := m.allocateInstr() + storeCurrentAddrToExecCtx.asStore(operandNR(tmp3), + addressMode{ + kind: addressModeKindRegUnsignedImm12, + rn: execCtxVReg, imm: wazevoapi.ExecutionContextOffsetGoCallReturnAddress.I64(), + }, 64) + + exitSeq := m.allocateInstr() + exitSeq.asExitSequence(execCtxVReg) + + m.insert(loadExitCodeConst) + m.insert(setExitCode) + m.insert(movSpToTmp) + m.insert(strSpToExecCtx) + m.insert(currentAddrToTmp) + m.insert(storeCurrentAddrToExecCtx) + m.insert(exitSeq) +} + +func (m *machine) lowerIcmpToFlag(x, y ssa.Value, signed bool) { + if x.Type() != y.Type() { + panic( + fmt.Sprintf("TODO(maybe): support icmp with different types: v%d=%s != v%d=%s", + x.ID(), x.Type(), y.ID(), y.Type())) + } + + extMod := extModeOf(x.Type(), signed) + + // First operand must be in pure register form. + rn := m.getOperand_NR(m.compiler.ValueDefinition(x), extMod) + // Second operand can be in any of Imm12, ER, SR, or NR form supported by the SUBS instructions. + rm := m.getOperand_Imm12_ER_SR_NR(m.compiler.ValueDefinition(y), extMod) + + alu := m.allocateInstr() + // subs zr, rn, rm + alu.asALU( + aluOpSubS, + // We don't need the result, just need to set flags. + operandNR(xzrVReg), + rn, + rm, + x.Type().Bits() == 64, + ) + m.insert(alu) +} + +func (m *machine) lowerFcmpToFlag(x, y ssa.Value) { + if x.Type() != y.Type() { + panic("TODO(maybe): support icmp with different types") + } + + rn := m.getOperand_NR(m.compiler.ValueDefinition(x), extModeNone) + rm := m.getOperand_NR(m.compiler.ValueDefinition(y), extModeNone) + cmp := m.allocateInstr() + cmp.asFpuCmp(rn, rm, x.Type().Bits() == 64) + m.insert(cmp) +} + +func (m *machine) lowerExitIfTrueWithCode(execCtxVReg regalloc.VReg, cond ssa.Value, code wazevoapi.ExitCode) { + condDef := m.compiler.ValueDefinition(cond) + if !m.compiler.MatchInstr(condDef, ssa.OpcodeIcmp) { + panic("TODO: OpcodeExitIfTrueWithCode must come after Icmp at the moment: " + condDef.Instr.Opcode().String()) + } + condDef.Instr.MarkLowered() + + cvalInstr := condDef.Instr + x, y, c := cvalInstr.IcmpData() + signed := c.Signed() + + if !m.tryLowerBandToFlag(x, y) { + m.lowerIcmpToFlag(x, y, signed) + } + + // We need to copy the execution context to a temp register, because if it's spilled, + // it might end up being reloaded inside the exiting branch. + execCtxTmp := m.copyToTmp(execCtxVReg) + + // We have to skip the entire exit sequence if the condition is false. + cbr := m.allocateInstr() + m.insert(cbr) + m.lowerExitWithCode(execCtxTmp, code) + // conditional branch target is after exit. + l := m.insertBrTargetLabel() + cbr.asCondBr(condFlagFromSSAIntegerCmpCond(c).invert().asCond(), l, false /* ignored */) +} + +func (m *machine) lowerSelect(c, x, y, result ssa.Value) { + cvalDef := m.compiler.ValueDefinition(c) + + var cc condFlag + switch { + case m.compiler.MatchInstr(cvalDef, ssa.OpcodeIcmp): // This case, we can use the ALU flag set by SUBS instruction. + cvalInstr := cvalDef.Instr + x, y, c := cvalInstr.IcmpData() + cc = condFlagFromSSAIntegerCmpCond(c) + m.lowerIcmpToFlag(x, y, c.Signed()) + cvalDef.Instr.MarkLowered() + case m.compiler.MatchInstr(cvalDef, ssa.OpcodeFcmp): // This case we can use the Fpu flag directly. + cvalInstr := cvalDef.Instr + x, y, c := cvalInstr.FcmpData() + cc = condFlagFromSSAFloatCmpCond(c) + m.lowerFcmpToFlag(x, y) + cvalDef.Instr.MarkLowered() + default: + rn := m.getOperand_NR(cvalDef, extModeNone) + if c.Type() != ssa.TypeI32 && c.Type() != ssa.TypeI64 { + panic("TODO?BUG?: support select with non-integer condition") + } + alu := m.allocateInstr() + // subs zr, rn, zr + alu.asALU( + aluOpSubS, + // We don't need the result, just need to set flags. + operandNR(xzrVReg), + rn, + operandNR(xzrVReg), + c.Type().Bits() == 64, + ) + m.insert(alu) + cc = ne + } + + rn := m.getOperand_NR(m.compiler.ValueDefinition(x), extModeNone) + rm := m.getOperand_NR(m.compiler.ValueDefinition(y), extModeNone) + + rd := operandNR(m.compiler.VRegOf(result)) + switch x.Type() { + case ssa.TypeI32, ssa.TypeI64: + // csel rd, rn, rm, cc + csel := m.allocateInstr() + csel.asCSel(rd, rn, rm, cc, x.Type().Bits() == 64) + m.insert(csel) + case ssa.TypeF32, ssa.TypeF64: + // fcsel rd, rn, rm, cc + fcsel := m.allocateInstr() + fcsel.asFpuCSel(rd, rn, rm, cc, x.Type().Bits() == 64) + m.insert(fcsel) + default: + panic("BUG") + } +} + +func (m *machine) lowerSelectVec(rc, rn, rm, rd operand) { + // First check if `rc` is zero or not. + checkZero := m.allocateInstr() + checkZero.asALU(aluOpSubS, operandNR(xzrVReg), rc, operandNR(xzrVReg), false) + m.insert(checkZero) + + // Then use CSETM to set all bits to one if `rc` is zero. + allOnesOrZero := m.compiler.AllocateVReg(ssa.TypeI64) + cset := m.allocateInstr() + cset.asCSet(allOnesOrZero, true, ne) + m.insert(cset) + + // Then move the bits to the result vector register. + tmp2 := operandNR(m.compiler.AllocateVReg(ssa.TypeV128)) + dup := m.allocateInstr() + dup.asVecDup(tmp2, operandNR(allOnesOrZero), vecArrangement2D) + m.insert(dup) + + // Now that `tmp2` has either all bits one or zero depending on `rc`, + // we can use bsl to select between `rn` and `rm`. + ins := m.allocateInstr() + ins.asVecRRRRewrite(vecOpBsl, tmp2, rn, rm, vecArrangement16B) + m.insert(ins) + + // Finally, move the result to the destination register. + mov2 := m.allocateInstr() + mov2.asFpuMov128(rd.nr(), tmp2.nr()) + m.insert(mov2) +} + +func (m *machine) lowerAtomicRmw(si *ssa.Instruction) { + ssaOp, size := si.AtomicRmwData() + + var op atomicRmwOp + var negateArg bool + var flipArg bool + switch ssaOp { + case ssa.AtomicRmwOpAdd: + op = atomicRmwOpAdd + case ssa.AtomicRmwOpSub: + op = atomicRmwOpAdd + negateArg = true + case ssa.AtomicRmwOpAnd: + op = atomicRmwOpClr + flipArg = true + case ssa.AtomicRmwOpOr: + op = atomicRmwOpSet + case ssa.AtomicRmwOpXor: + op = atomicRmwOpEor + case ssa.AtomicRmwOpXchg: + op = atomicRmwOpSwp + default: + panic(fmt.Sprintf("unknown ssa atomic rmw op: %s", ssaOp)) + } + + addr, val := si.Arg2() + addrDef, valDef := m.compiler.ValueDefinition(addr), m.compiler.ValueDefinition(val) + rn := m.getOperand_NR(addrDef, extModeNone) + rt := operandNR(m.compiler.VRegOf(si.Return())) + rs := m.getOperand_NR(valDef, extModeNone) + + _64 := si.Return().Type().Bits() == 64 + var tmp operand + if _64 { + tmp = operandNR(m.compiler.AllocateVReg(ssa.TypeI64)) + } else { + tmp = operandNR(m.compiler.AllocateVReg(ssa.TypeI32)) + } + m.lowerAtomicRmwImpl(op, rn, rs, rt, tmp, size, negateArg, flipArg, _64) +} + +func (m *machine) lowerAtomicRmwImpl(op atomicRmwOp, rn, rs, rt, tmp operand, size uint64, negateArg, flipArg, dst64bit bool) { + switch { + case negateArg: + neg := m.allocateInstr() + neg.asALU(aluOpSub, tmp, operandNR(xzrVReg), rs, dst64bit) + m.insert(neg) + case flipArg: + flip := m.allocateInstr() + flip.asALU(aluOpOrn, tmp, operandNR(xzrVReg), rs, dst64bit) + m.insert(flip) + default: + tmp = rs + } + + rmw := m.allocateInstr() + rmw.asAtomicRmw(op, rn, tmp, rt, size) + m.insert(rmw) +} + +func (m *machine) lowerAtomicCas(si *ssa.Instruction) { + addr, exp, repl := si.Arg3() + size := si.AtomicTargetSize() + + addrDef, expDef, replDef := m.compiler.ValueDefinition(addr), m.compiler.ValueDefinition(exp), m.compiler.ValueDefinition(repl) + rn := m.getOperand_NR(addrDef, extModeNone) + rt := m.getOperand_NR(replDef, extModeNone) + rs := m.getOperand_NR(expDef, extModeNone) + tmp := operandNR(m.compiler.AllocateVReg(si.Return().Type())) + + _64 := si.Return().Type().Bits() == 64 + // rs is overwritten by CAS, so we need to move it to the result register before the instruction + // in case when it is used somewhere else. + mov := m.allocateInstr() + if _64 { + mov.asMove64(tmp.nr(), rs.nr()) + } else { + mov.asMove32(tmp.nr(), rs.nr()) + } + m.insert(mov) + + m.lowerAtomicCasImpl(rn, tmp, rt, size) + + mov2 := m.allocateInstr() + rd := m.compiler.VRegOf(si.Return()) + if _64 { + mov2.asMove64(rd, tmp.nr()) + } else { + mov2.asMove32(rd, tmp.nr()) + } + m.insert(mov2) +} + +func (m *machine) lowerAtomicCasImpl(rn, rs, rt operand, size uint64) { + cas := m.allocateInstr() + cas.asAtomicCas(rn, rs, rt, size) + m.insert(cas) +} + +func (m *machine) lowerAtomicLoad(si *ssa.Instruction) { + addr := si.Arg() + size := si.AtomicTargetSize() + + addrDef := m.compiler.ValueDefinition(addr) + rn := m.getOperand_NR(addrDef, extModeNone) + rt := operandNR(m.compiler.VRegOf(si.Return())) + + m.lowerAtomicLoadImpl(rn, rt, size) +} + +func (m *machine) lowerAtomicLoadImpl(rn, rt operand, size uint64) { + ld := m.allocateInstr() + ld.asAtomicLoad(rn, rt, size) + m.insert(ld) +} + +func (m *machine) lowerAtomicStore(si *ssa.Instruction) { + addr, val := si.Arg2() + size := si.AtomicTargetSize() + + addrDef := m.compiler.ValueDefinition(addr) + valDef := m.compiler.ValueDefinition(val) + rn := m.getOperand_NR(addrDef, extModeNone) + rt := m.getOperand_NR(valDef, extModeNone) + + m.lowerAtomicStoreImpl(rn, rt, size) +} + +func (m *machine) lowerAtomicStoreImpl(rn, rt operand, size uint64) { + ld := m.allocateInstr() + ld.asAtomicStore(rn, rt, size) + m.insert(ld) +} + +// copyToTmp copies the given regalloc.VReg to a temporary register. This is called before cbr to avoid the regalloc issue +// e.g. reload happening in the middle of the exit sequence which is not the path the normal path executes +func (m *machine) copyToTmp(v regalloc.VReg) regalloc.VReg { + typ := m.compiler.TypeOf(v) + mov := m.allocateInstr() + tmp := m.compiler.AllocateVReg(typ) + if typ.IsInt() { + mov.asMove64(tmp, v) + } else { + mov.asFpuMov128(tmp, v) + } + m.insert(mov) + return tmp +} diff --git a/vendor/github.com/tetratelabs/wazero/internal/engine/wazevo/backend/isa/arm64/lower_instr_operands.go b/vendor/github.com/tetratelabs/wazero/internal/engine/wazevo/backend/isa/arm64/lower_instr_operands.go new file mode 100644 index 000000000..d9fbf1789 --- /dev/null +++ b/vendor/github.com/tetratelabs/wazero/internal/engine/wazevo/backend/isa/arm64/lower_instr_operands.go @@ -0,0 +1,350 @@ +package arm64 + +// This file contains the logic to "find and determine operands" for instructions. +// In order to finalize the form of an operand, we might end up merging/eliminating +// the source instructions into an operand whenever possible. + +import ( + "fmt" + + "github.com/tetratelabs/wazero/internal/engine/wazevo/backend" + "github.com/tetratelabs/wazero/internal/engine/wazevo/backend/regalloc" + "github.com/tetratelabs/wazero/internal/engine/wazevo/ssa" +) + +type ( + // operand represents an operand of an instruction whose type is determined by the kind. + operand struct { + kind operandKind + data, data2 uint64 + } + operandKind byte +) + +// Here's the list of operand kinds. We use the abbreviation of the kind name not only for these consts, +// but also names of functions which return the operand of the kind. +const ( + // operandKindNR represents "NormalRegister" (NR). This is literally the register without any special operation unlike others. + operandKindNR operandKind = iota + // operandKindSR represents "Shifted Register" (SR). This is a register which is shifted by a constant. + // Some of the arm64 instructions can take this kind of operand. + operandKindSR + // operandKindER represents "Extended Register (ER). This is a register which is sign/zero-extended to a larger size. + // Some of the arm64 instructions can take this kind of operand. + operandKindER + // operandKindImm12 represents "Immediate 12" (Imm12). This is a 12-bit immediate value which can be either shifted by 12 or not. + // See asImm12 function for detail. + operandKindImm12 + // operandKindShiftImm represents "Shifted Immediate" (ShiftImm) used by shift operations. + operandKindShiftImm +) + +// String implements fmt.Stringer for debugging. +func (o operand) format(size byte) string { + switch o.kind { + case operandKindNR: + return formatVRegSized(o.nr(), size) + case operandKindSR: + r, amt, sop := o.sr() + return fmt.Sprintf("%s, %s #%d", formatVRegSized(r, size), sop, amt) + case operandKindER: + r, eop, _ := o.er() + return fmt.Sprintf("%s %s", formatVRegSized(r, size), eop) + case operandKindImm12: + imm12, shiftBit := o.imm12() + if shiftBit == 1 { + return fmt.Sprintf("#%#x", uint64(imm12)<<12) + } else { + return fmt.Sprintf("#%#x", imm12) + } + default: + panic(fmt.Sprintf("unknown operand kind: %d", o.kind)) + } +} + +// operandNR encodes the given VReg as an operand of operandKindNR. +func operandNR(r regalloc.VReg) operand { + return operand{kind: operandKindNR, data: uint64(r)} +} + +// nr decodes the underlying VReg assuming the operand is of operandKindNR. +func (o operand) nr() regalloc.VReg { + return regalloc.VReg(o.data) +} + +// operandER encodes the given VReg as an operand of operandKindER. +func operandER(r regalloc.VReg, eop extendOp, to byte) operand { + if to < 32 { + panic("TODO?BUG?: when we need to extend to less than 32 bits?") + } + return operand{kind: operandKindER, data: uint64(r), data2: uint64(eop)<<32 | uint64(to)} +} + +// er decodes the underlying VReg, extend operation, and the target size assuming the operand is of operandKindER. +func (o operand) er() (r regalloc.VReg, eop extendOp, to byte) { + return regalloc.VReg(o.data), extendOp(o.data2>>32) & 0xff, byte(o.data2 & 0xff) +} + +// operandSR encodes the given VReg as an operand of operandKindSR. +func operandSR(r regalloc.VReg, amt byte, sop shiftOp) operand { + return operand{kind: operandKindSR, data: uint64(r), data2: uint64(amt)<<32 | uint64(sop)} +} + +// sr decodes the underlying VReg, shift amount, and shift operation assuming the operand is of operandKindSR. +func (o operand) sr() (r regalloc.VReg, amt byte, sop shiftOp) { + return regalloc.VReg(o.data), byte(o.data2>>32) & 0xff, shiftOp(o.data2) & 0xff +} + +// operandImm12 encodes the given imm12 as an operand of operandKindImm12. +func operandImm12(imm12 uint16, shiftBit byte) operand { + return operand{kind: operandKindImm12, data: uint64(imm12) | uint64(shiftBit)<<32} +} + +// imm12 decodes the underlying imm12 data assuming the operand is of operandKindImm12. +func (o operand) imm12() (v uint16, shiftBit byte) { + return uint16(o.data), byte(o.data >> 32) +} + +// operandShiftImm encodes the given amount as an operand of operandKindShiftImm. +func operandShiftImm(amount byte) operand { + return operand{kind: operandKindShiftImm, data: uint64(amount)} +} + +// shiftImm decodes the underlying shift amount data assuming the operand is of operandKindShiftImm. +func (o operand) shiftImm() byte { + return byte(o.data) +} + +// reg returns the register of the operand if applicable. +func (o operand) reg() regalloc.VReg { + switch o.kind { + case operandKindNR: + return o.nr() + case operandKindSR: + r, _, _ := o.sr() + return r + case operandKindER: + r, _, _ := o.er() + return r + case operandKindImm12: + // Does not have a register. + case operandKindShiftImm: + // Does not have a register. + default: + panic(o.kind) + } + return regalloc.VRegInvalid +} + +func (o operand) realReg() regalloc.RealReg { + return o.nr().RealReg() +} + +func (o operand) assignReg(v regalloc.VReg) operand { + switch o.kind { + case operandKindNR: + return operandNR(v) + case operandKindSR: + _, amt, sop := o.sr() + return operandSR(v, amt, sop) + case operandKindER: + _, eop, to := o.er() + return operandER(v, eop, to) + case operandKindImm12: + // Does not have a register. + case operandKindShiftImm: + // Does not have a register. + } + panic(o.kind) +} + +// ensureValueNR returns an operand of either operandKindER, operandKindSR, or operandKindNR from the given value (defined by `def). +// +// `mode` is used to extend the operand if the bit length is smaller than mode.bits(). +// If the operand can be expressed as operandKindImm12, `mode` is ignored. +func (m *machine) getOperand_Imm12_ER_SR_NR(def *backend.SSAValueDefinition, mode extMode) (op operand) { + if def.IsFromBlockParam() { + return operandNR(def.BlkParamVReg) + } + + instr := def.Instr + if instr.Opcode() == ssa.OpcodeIconst { + if imm12Op, ok := asImm12Operand(instr.ConstantVal()); ok { + instr.MarkLowered() + return imm12Op + } + } + return m.getOperand_ER_SR_NR(def, mode) +} + +// getOperand_MaybeNegatedImm12_ER_SR_NR is almost the same as getOperand_Imm12_ER_SR_NR, but this might negate the immediate value. +// If the immediate value is negated, the second return value is true, otherwise always false. +func (m *machine) getOperand_MaybeNegatedImm12_ER_SR_NR(def *backend.SSAValueDefinition, mode extMode) (op operand, negatedImm12 bool) { + if def.IsFromBlockParam() { + return operandNR(def.BlkParamVReg), false + } + + instr := def.Instr + if instr.Opcode() == ssa.OpcodeIconst { + c := instr.ConstantVal() + if imm12Op, ok := asImm12Operand(c); ok { + instr.MarkLowered() + return imm12Op, false + } + + signExtended := int64(c) + if def.SSAValue().Type().Bits() == 32 { + signExtended = (signExtended << 32) >> 32 + } + negatedWithoutSign := -signExtended + if imm12Op, ok := asImm12Operand(uint64(negatedWithoutSign)); ok { + instr.MarkLowered() + return imm12Op, true + } + } + return m.getOperand_ER_SR_NR(def, mode), false +} + +// ensureValueNR returns an operand of either operandKindER, operandKindSR, or operandKindNR from the given value (defined by `def). +// +// `mode` is used to extend the operand if the bit length is smaller than mode.bits(). +func (m *machine) getOperand_ER_SR_NR(def *backend.SSAValueDefinition, mode extMode) (op operand) { + if def.IsFromBlockParam() { + return operandNR(def.BlkParamVReg) + } + + if m.compiler.MatchInstr(def, ssa.OpcodeSExtend) || m.compiler.MatchInstr(def, ssa.OpcodeUExtend) { + extInstr := def.Instr + + signed := extInstr.Opcode() == ssa.OpcodeSExtend + innerExtFromBits, innerExtToBits := extInstr.ExtendFromToBits() + modeBits, modeSigned := mode.bits(), mode.signed() + if mode == extModeNone || innerExtToBits == modeBits { + eop := extendOpFrom(signed, innerExtFromBits) + extArg := m.getOperand_NR(m.compiler.ValueDefinition(extInstr.Arg()), extModeNone) + op = operandER(extArg.nr(), eop, innerExtToBits) + extInstr.MarkLowered() + return + } + + if innerExtToBits > modeBits { + panic("BUG?TODO?: need the results of inner extension to be larger than the mode") + } + + switch { + case (!signed && !modeSigned) || (signed && modeSigned): + // Two sign/zero extensions are equivalent to one sign/zero extension for the larger size. + eop := extendOpFrom(modeSigned, innerExtFromBits) + op = operandER(m.compiler.VRegOf(extInstr.Arg()), eop, modeBits) + extInstr.MarkLowered() + case (signed && !modeSigned) || (!signed && modeSigned): + // We need to {sign, zero}-extend the result of the {zero,sign} extension. + eop := extendOpFrom(modeSigned, innerExtToBits) + op = operandER(m.compiler.VRegOf(extInstr.Return()), eop, modeBits) + // Note that we failed to merge the inner extension instruction this case. + } + return + } + return m.getOperand_SR_NR(def, mode) +} + +// ensureValueNR returns an operand of either operandKindSR or operandKindNR from the given value (defined by `def). +// +// `mode` is used to extend the operand if the bit length is smaller than mode.bits(). +func (m *machine) getOperand_SR_NR(def *backend.SSAValueDefinition, mode extMode) (op operand) { + if def.IsFromBlockParam() { + return operandNR(def.BlkParamVReg) + } + + if m.compiler.MatchInstr(def, ssa.OpcodeIshl) { + // Check if the shift amount is constant instruction. + targetVal, amountVal := def.Instr.Arg2() + targetVReg := m.getOperand_NR(m.compiler.ValueDefinition(targetVal), extModeNone).nr() + amountDef := m.compiler.ValueDefinition(amountVal) + if amountDef.IsFromInstr() && amountDef.Instr.Constant() { + // If that is the case, we can use the shifted register operand (SR). + c := byte(amountDef.Instr.ConstantVal()) & (targetVal.Type().Bits() - 1) // Clears the unnecessary bits. + def.Instr.MarkLowered() + amountDef.Instr.MarkLowered() + return operandSR(targetVReg, c, shiftOpLSL) + } + } + return m.getOperand_NR(def, mode) +} + +// getOperand_ShiftImm_NR returns an operand of either operandKindShiftImm or operandKindNR from the given value (defined by `def). +func (m *machine) getOperand_ShiftImm_NR(def *backend.SSAValueDefinition, mode extMode, shiftBitWidth byte) (op operand) { + if def.IsFromBlockParam() { + return operandNR(def.BlkParamVReg) + } + + instr := def.Instr + if instr.Constant() { + amount := byte(instr.ConstantVal()) & (shiftBitWidth - 1) // Clears the unnecessary bits. + return operandShiftImm(amount) + } + return m.getOperand_NR(def, mode) +} + +// ensureValueNR returns an operand of operandKindNR from the given value (defined by `def). +// +// `mode` is used to extend the operand if the bit length is smaller than mode.bits(). +func (m *machine) getOperand_NR(def *backend.SSAValueDefinition, mode extMode) (op operand) { + var v regalloc.VReg + if def.IsFromBlockParam() { + v = def.BlkParamVReg + } else { + instr := def.Instr + if instr.Constant() { + // We inline all the constant instructions so that we could reduce the register usage. + v = m.lowerConstant(instr) + instr.MarkLowered() + } else { + if n := def.N; n == 0 { + v = m.compiler.VRegOf(instr.Return()) + } else { + _, rs := instr.Returns() + v = m.compiler.VRegOf(rs[n-1]) + } + } + } + + r := v + switch inBits := def.SSAValue().Type().Bits(); { + case mode == extModeNone: + case inBits == 32 && (mode == extModeZeroExtend32 || mode == extModeSignExtend32): + case inBits == 32 && mode == extModeZeroExtend64: + extended := m.compiler.AllocateVReg(ssa.TypeI64) + ext := m.allocateInstr() + ext.asExtend(extended, v, 32, 64, false) + m.insert(ext) + r = extended + case inBits == 32 && mode == extModeSignExtend64: + extended := m.compiler.AllocateVReg(ssa.TypeI64) + ext := m.allocateInstr() + ext.asExtend(extended, v, 32, 64, true) + m.insert(ext) + r = extended + case inBits == 64 && (mode == extModeZeroExtend64 || mode == extModeSignExtend64): + } + return operandNR(r) +} + +func asImm12Operand(val uint64) (op operand, ok bool) { + v, shiftBit, ok := asImm12(val) + if !ok { + return operand{}, false + } + return operandImm12(v, shiftBit), true +} + +func asImm12(val uint64) (v uint16, shiftBit byte, ok bool) { + const mask1, mask2 uint64 = 0xfff, 0xfff_000 + if val&^mask1 == 0 { + return uint16(val), 0, true + } else if val&^mask2 == 0 { + return uint16(val >> 12), 1, true + } else { + return 0, 0, false + } +} diff --git a/vendor/github.com/tetratelabs/wazero/internal/engine/wazevo/backend/isa/arm64/lower_mem.go b/vendor/github.com/tetratelabs/wazero/internal/engine/wazevo/backend/isa/arm64/lower_mem.go new file mode 100644 index 000000000..4842eaa38 --- /dev/null +++ b/vendor/github.com/tetratelabs/wazero/internal/engine/wazevo/backend/isa/arm64/lower_mem.go @@ -0,0 +1,440 @@ +package arm64 + +import ( + "fmt" + + "github.com/tetratelabs/wazero/internal/engine/wazevo/backend/regalloc" + "github.com/tetratelabs/wazero/internal/engine/wazevo/ssa" + "github.com/tetratelabs/wazero/internal/engine/wazevo/wazevoapi" +) + +type ( + // addressMode represents an ARM64 addressing mode. + // + // https://developer.arm.com/documentation/102374/0101/Loads-and-stores---addressing + // TODO: use the bit-packed layout like operand struct. + addressMode struct { + kind addressModeKind + rn, rm regalloc.VReg + extOp extendOp + imm int64 + } + + // addressModeKind represents the kind of ARM64 addressing mode. + addressModeKind byte +) + +const ( + // addressModeKindRegExtended takes a base register and an index register. The index register is sign/zero-extended, + // and then scaled by bits(type)/8. + // + // e.g. + // - ldrh w1, [x2, w3, SXTW #1] ;; sign-extended and scaled by 2 (== LSL #1) + // - strh w1, [x2, w3, UXTW #1] ;; zero-extended and scaled by 2 (== LSL #1) + // - ldr w1, [x2, w3, SXTW #2] ;; sign-extended and scaled by 4 (== LSL #2) + // - str x1, [x2, w3, UXTW #3] ;; zero-extended and scaled by 8 (== LSL #3) + // + // See the following pages: + // https://developer.arm.com/documentation/ddi0596/2021-12/Base-Instructions/LDRH--register---Load-Register-Halfword--register-- + // https://developer.arm.com/documentation/ddi0596/2021-12/Base-Instructions/LDR--register---Load-Register--register-- + addressModeKindRegScaledExtended addressModeKind = iota + + // addressModeKindRegScaled is the same as addressModeKindRegScaledExtended, but without extension factor. + addressModeKindRegScaled + + // addressModeKindRegScaled is the same as addressModeKindRegScaledExtended, but without scale factor. + addressModeKindRegExtended + + // addressModeKindRegReg takes a base register and an index register. The index register is not either scaled or extended. + addressModeKindRegReg + + // addressModeKindRegSignedImm9 takes a base register and a 9-bit "signed" immediate offset (-256 to 255). + // The immediate will be sign-extended, and be added to the base register. + // This is a.k.a. "unscaled" since the immediate is not scaled. + // https://developer.arm.com/documentation/ddi0596/2021-12/Base-Instructions/LDUR--Load-Register--unscaled-- + addressModeKindRegSignedImm9 + + // addressModeKindRegUnsignedImm12 takes a base register and a 12-bit "unsigned" immediate offset. scaled by + // the size of the type. In other words, the actual offset will be imm12 * bits(type)/8. + // See "Unsigned offset" in the following pages: + // https://developer.arm.com/documentation/ddi0596/2021-12/Base-Instructions/LDRB--immediate---Load-Register-Byte--immediate-- + // https://developer.arm.com/documentation/ddi0596/2021-12/Base-Instructions/LDRH--immediate---Load-Register-Halfword--immediate-- + // https://developer.arm.com/documentation/ddi0596/2021-12/Base-Instructions/LDR--immediate---Load-Register--immediate-- + addressModeKindRegUnsignedImm12 + + // addressModePostIndex takes a base register and a 9-bit "signed" immediate offset. + // After the load/store, the base register will be updated by the offset. + // + // Note that when this is used for pair load/store, the offset will be 7-bit "signed" immediate offset. + // + // See "Post-index" in the following pages for examples: + // https://developer.arm.com/documentation/ddi0596/2021-12/Base-Instructions/LDRB--immediate---Load-Register-Byte--immediate-- + // https://developer.arm.com/documentation/ddi0596/2021-12/Base-Instructions/LDRH--immediate---Load-Register-Halfword--immediate-- + // https://developer.arm.com/documentation/ddi0596/2021-12/Base-Instructions/LDR--immediate---Load-Register--immediate-- + // https://developer.arm.com/documentation/ddi0596/2021-12/Base-Instructions/LDP--Load-Pair-of-Registers- + addressModeKindPostIndex + + // addressModePostIndex takes a base register and a 9-bit "signed" immediate offset. + // Before the load/store, the base register will be updated by the offset. + // + // Note that when this is used for pair load/store, the offset will be 7-bit "signed" immediate offset. + // + // See "Pre-index" in the following pages for examples: + // https://developer.arm.com/documentation/ddi0596/2021-12/Base-Instructions/LDRB--immediate---Load-Register-Byte--immediate-- + // https://developer.arm.com/documentation/ddi0596/2021-12/Base-Instructions/LDRH--immediate---Load-Register-Halfword--immediate-- + // https://developer.arm.com/documentation/ddi0596/2021-12/Base-Instructions/LDR--immediate---Load-Register--immediate-- + // https://developer.arm.com/documentation/ddi0596/2021-12/Base-Instructions/LDP--Load-Pair-of-Registers- + addressModeKindPreIndex + + // addressModeKindArgStackSpace is used to resolve the address of the argument stack space + // exiting right above the stack pointer. Since we don't know the exact stack space needed for a function + // at a compilation phase, this is used as a placeholder and further lowered to a real addressing mode like above. + addressModeKindArgStackSpace + + // addressModeKindResultStackSpace is used to resolve the address of the result stack space + // exiting right above the stack pointer. Since we don't know the exact stack space needed for a function + // at a compilation phase, this is used as a placeholder and further lowered to a real addressing mode like above. + addressModeKindResultStackSpace +) + +func (a addressMode) format(dstSizeBits byte) (ret string) { + base := formatVRegSized(a.rn, 64) + if rn := a.rn; rn.RegType() != regalloc.RegTypeInt { + panic("invalid base register type: " + a.rn.RegType().String()) + } else if rn.IsRealReg() && v0 <= a.rn.RealReg() && a.rn.RealReg() <= v30 { + panic("BUG: likely a bug in reg alloc or reset behavior") + } + + switch a.kind { + case addressModeKindRegScaledExtended: + amount := a.sizeInBitsToShiftAmount(dstSizeBits) + ret = fmt.Sprintf("[%s, %s, %s #%#x]", base, formatVRegSized(a.rm, a.indexRegBits()), a.extOp, amount) + case addressModeKindRegScaled: + amount := a.sizeInBitsToShiftAmount(dstSizeBits) + ret = fmt.Sprintf("[%s, %s, lsl #%#x]", base, formatVRegSized(a.rm, a.indexRegBits()), amount) + case addressModeKindRegExtended: + ret = fmt.Sprintf("[%s, %s, %s]", base, formatVRegSized(a.rm, a.indexRegBits()), a.extOp) + case addressModeKindRegReg: + ret = fmt.Sprintf("[%s, %s]", base, formatVRegSized(a.rm, a.indexRegBits())) + case addressModeKindRegSignedImm9: + if a.imm != 0 { + ret = fmt.Sprintf("[%s, #%#x]", base, a.imm) + } else { + ret = fmt.Sprintf("[%s]", base) + } + case addressModeKindRegUnsignedImm12: + if a.imm != 0 { + ret = fmt.Sprintf("[%s, #%#x]", base, a.imm) + } else { + ret = fmt.Sprintf("[%s]", base) + } + case addressModeKindPostIndex: + ret = fmt.Sprintf("[%s], #%#x", base, a.imm) + case addressModeKindPreIndex: + ret = fmt.Sprintf("[%s, #%#x]!", base, a.imm) + case addressModeKindArgStackSpace: + ret = fmt.Sprintf("[#arg_space, #%#x]", a.imm) + case addressModeKindResultStackSpace: + ret = fmt.Sprintf("[#ret_space, #%#x]", a.imm) + } + return +} + +func addressModePreOrPostIndex(rn regalloc.VReg, imm int64, preIndex bool) addressMode { + if !offsetFitsInAddressModeKindRegSignedImm9(imm) { + panic(fmt.Sprintf("BUG: offset %#x does not fit in addressModeKindRegSignedImm9", imm)) + } + if preIndex { + return addressMode{kind: addressModeKindPreIndex, rn: rn, imm: imm} + } else { + return addressMode{kind: addressModeKindPostIndex, rn: rn, imm: imm} + } +} + +func offsetFitsInAddressModeKindRegUnsignedImm12(dstSizeInBits byte, offset int64) bool { + divisor := int64(dstSizeInBits) / 8 + return 0 < offset && offset%divisor == 0 && offset/divisor < 4096 +} + +func offsetFitsInAddressModeKindRegSignedImm9(offset int64) bool { + return -256 <= offset && offset <= 255 +} + +func (a addressMode) indexRegBits() byte { + bits := a.extOp.srcBits() + if bits != 32 && bits != 64 { + panic("invalid index register for address mode. it must be either 32 or 64 bits") + } + return bits +} + +func (a addressMode) sizeInBitsToShiftAmount(sizeInBits byte) (lsl byte) { + switch sizeInBits { + case 8: + lsl = 0 + case 16: + lsl = 1 + case 32: + lsl = 2 + case 64: + lsl = 3 + } + return +} + +func extLoadSignSize(op ssa.Opcode) (size byte, signed bool) { + switch op { + case ssa.OpcodeUload8: + size, signed = 8, false + case ssa.OpcodeUload16: + size, signed = 16, false + case ssa.OpcodeUload32: + size, signed = 32, false + case ssa.OpcodeSload8: + size, signed = 8, true + case ssa.OpcodeSload16: + size, signed = 16, true + case ssa.OpcodeSload32: + size, signed = 32, true + default: + panic("BUG") + } + return +} + +func (m *machine) lowerExtLoad(op ssa.Opcode, ptr ssa.Value, offset uint32, ret regalloc.VReg) { + size, signed := extLoadSignSize(op) + amode := m.lowerToAddressMode(ptr, offset, size) + load := m.allocateInstr() + if signed { + load.asSLoad(operandNR(ret), amode, size) + } else { + load.asULoad(operandNR(ret), amode, size) + } + m.insert(load) +} + +func (m *machine) lowerLoad(ptr ssa.Value, offset uint32, typ ssa.Type, ret ssa.Value) { + amode := m.lowerToAddressMode(ptr, offset, typ.Bits()) + + dst := m.compiler.VRegOf(ret) + load := m.allocateInstr() + switch typ { + case ssa.TypeI32, ssa.TypeI64: + load.asULoad(operandNR(dst), amode, typ.Bits()) + case ssa.TypeF32, ssa.TypeF64: + load.asFpuLoad(operandNR(dst), amode, typ.Bits()) + case ssa.TypeV128: + load.asFpuLoad(operandNR(dst), amode, 128) + default: + panic("TODO") + } + m.insert(load) +} + +func (m *machine) lowerLoadSplat(ptr ssa.Value, offset uint32, lane ssa.VecLane, ret ssa.Value) { + // vecLoad1R has offset address mode (base+imm) only for post index, so we simply add the offset to the base. + base := m.getOperand_NR(m.compiler.ValueDefinition(ptr), extModeNone).nr() + offsetReg := m.compiler.AllocateVReg(ssa.TypeI64) + m.lowerConstantI64(offsetReg, int64(offset)) + addedBase := m.addReg64ToReg64(base, offsetReg) + + rd := operandNR(m.compiler.VRegOf(ret)) + + ld1r := m.allocateInstr() + ld1r.asVecLoad1R(rd, operandNR(addedBase), ssaLaneToArrangement(lane)) + m.insert(ld1r) +} + +func (m *machine) lowerStore(si *ssa.Instruction) { + // TODO: merge consecutive stores into a single pair store instruction. + value, ptr, offset, storeSizeInBits := si.StoreData() + amode := m.lowerToAddressMode(ptr, offset, storeSizeInBits) + + valueOp := m.getOperand_NR(m.compiler.ValueDefinition(value), extModeNone) + store := m.allocateInstr() + store.asStore(valueOp, amode, storeSizeInBits) + m.insert(store) +} + +// lowerToAddressMode converts a pointer to an addressMode that can be used as an operand for load/store instructions. +func (m *machine) lowerToAddressMode(ptr ssa.Value, offsetBase uint32, size byte) (amode addressMode) { + // TODO: currently the instruction selection logic doesn't support addressModeKindRegScaledExtended and + // addressModeKindRegScaled since collectAddends doesn't take ssa.OpcodeIshl into account. This should be fixed + // to support more efficient address resolution. + + a32s, a64s, offset := m.collectAddends(ptr) + offset += int64(offsetBase) + return m.lowerToAddressModeFromAddends(a32s, a64s, size, offset) +} + +// lowerToAddressModeFromAddends creates an addressMode from a list of addends collected by collectAddends. +// During the construction, this might emit additional instructions. +// +// Extracted as a separate function for easy testing. +func (m *machine) lowerToAddressModeFromAddends(a32s *wazevoapi.Queue[addend32], a64s *wazevoapi.Queue[regalloc.VReg], size byte, offset int64) (amode addressMode) { + switch a64sExist, a32sExist := !a64s.Empty(), !a32s.Empty(); { + case a64sExist && a32sExist: + var base regalloc.VReg + base = a64s.Dequeue() + var a32 addend32 + a32 = a32s.Dequeue() + amode = addressMode{kind: addressModeKindRegExtended, rn: base, rm: a32.r, extOp: a32.ext} + case a64sExist && offsetFitsInAddressModeKindRegUnsignedImm12(size, offset): + var base regalloc.VReg + base = a64s.Dequeue() + amode = addressMode{kind: addressModeKindRegUnsignedImm12, rn: base, imm: offset} + offset = 0 + case a64sExist && offsetFitsInAddressModeKindRegSignedImm9(offset): + var base regalloc.VReg + base = a64s.Dequeue() + amode = addressMode{kind: addressModeKindRegSignedImm9, rn: base, imm: offset} + offset = 0 + case a64sExist: + var base regalloc.VReg + base = a64s.Dequeue() + if !a64s.Empty() { + index := a64s.Dequeue() + amode = addressMode{kind: addressModeKindRegReg, rn: base, rm: index, extOp: extendOpUXTX /* indicates index reg is 64-bit */} + } else { + amode = addressMode{kind: addressModeKindRegUnsignedImm12, rn: base, imm: 0} + } + case a32sExist: + base32 := a32s.Dequeue() + + // First we need 64-bit base. + base := m.compiler.AllocateVReg(ssa.TypeI64) + baseExt := m.allocateInstr() + var signed bool + if base32.ext == extendOpSXTW { + signed = true + } + baseExt.asExtend(base, base32.r, 32, 64, signed) + m.insert(baseExt) + + if !a32s.Empty() { + index := a32s.Dequeue() + amode = addressMode{kind: addressModeKindRegExtended, rn: base, rm: index.r, extOp: index.ext} + } else { + amode = addressMode{kind: addressModeKindRegUnsignedImm12, rn: base, imm: 0} + } + default: // Only static offsets. + tmpReg := m.compiler.AllocateVReg(ssa.TypeI64) + m.lowerConstantI64(tmpReg, offset) + amode = addressMode{kind: addressModeKindRegUnsignedImm12, rn: tmpReg, imm: 0} + offset = 0 + } + + baseReg := amode.rn + if offset > 0 { + baseReg = m.addConstToReg64(baseReg, offset) // baseReg += offset + } + + for !a64s.Empty() { + a64 := a64s.Dequeue() + baseReg = m.addReg64ToReg64(baseReg, a64) // baseReg += a64 + } + + for !a32s.Empty() { + a32 := a32s.Dequeue() + baseReg = m.addRegToReg64Ext(baseReg, a32.r, a32.ext) // baseReg += (a32 extended to 64-bit) + } + amode.rn = baseReg + return +} + +var addendsMatchOpcodes = [4]ssa.Opcode{ssa.OpcodeUExtend, ssa.OpcodeSExtend, ssa.OpcodeIadd, ssa.OpcodeIconst} + +func (m *machine) collectAddends(ptr ssa.Value) (addends32 *wazevoapi.Queue[addend32], addends64 *wazevoapi.Queue[regalloc.VReg], offset int64) { + m.addendsWorkQueue.Reset() + m.addends32.Reset() + m.addends64.Reset() + m.addendsWorkQueue.Enqueue(ptr) + + for !m.addendsWorkQueue.Empty() { + v := m.addendsWorkQueue.Dequeue() + + def := m.compiler.ValueDefinition(v) + switch op := m.compiler.MatchInstrOneOf(def, addendsMatchOpcodes[:]); op { + case ssa.OpcodeIadd: + // If the addend is an add, we recursively collect its operands. + x, y := def.Instr.Arg2() + m.addendsWorkQueue.Enqueue(x) + m.addendsWorkQueue.Enqueue(y) + def.Instr.MarkLowered() + case ssa.OpcodeIconst: + // If the addend is constant, we just statically merge it into the offset. + ic := def.Instr + u64 := ic.ConstantVal() + if ic.Return().Type().Bits() == 32 { + offset += int64(int32(u64)) // sign-extend. + } else { + offset += int64(u64) + } + def.Instr.MarkLowered() + case ssa.OpcodeUExtend, ssa.OpcodeSExtend: + input := def.Instr.Arg() + if input.Type().Bits() != 32 { + panic("illegal size: " + input.Type().String()) + } + + var ext extendOp + if op == ssa.OpcodeUExtend { + ext = extendOpUXTW + } else { + ext = extendOpSXTW + } + + inputDef := m.compiler.ValueDefinition(input) + constInst := inputDef.IsFromInstr() && inputDef.Instr.Constant() + switch { + case constInst && ext == extendOpUXTW: + // Zero-extension of a 32-bit constant can be merged into the offset. + offset += int64(uint32(inputDef.Instr.ConstantVal())) + case constInst && ext == extendOpSXTW: + // Sign-extension of a 32-bit constant can be merged into the offset. + offset += int64(int32(inputDef.Instr.ConstantVal())) // sign-extend! + default: + m.addends32.Enqueue(addend32{r: m.getOperand_NR(inputDef, extModeNone).nr(), ext: ext}) + } + def.Instr.MarkLowered() + continue + default: + // If the addend is not one of them, we simply use it as-is (without merging!), optionally zero-extending it. + m.addends64.Enqueue(m.getOperand_NR(def, extModeZeroExtend64 /* optional zero ext */).nr()) + } + } + return &m.addends32, &m.addends64, offset +} + +func (m *machine) addConstToReg64(r regalloc.VReg, c int64) (rd regalloc.VReg) { + rd = m.compiler.AllocateVReg(ssa.TypeI64) + alu := m.allocateInstr() + if imm12Op, ok := asImm12Operand(uint64(c)); ok { + alu.asALU(aluOpAdd, operandNR(rd), operandNR(r), imm12Op, true) + } else if imm12Op, ok = asImm12Operand(uint64(-c)); ok { + alu.asALU(aluOpSub, operandNR(rd), operandNR(r), imm12Op, true) + } else { + tmp := m.compiler.AllocateVReg(ssa.TypeI64) + m.load64bitConst(c, tmp) + alu.asALU(aluOpAdd, operandNR(rd), operandNR(r), operandNR(tmp), true) + } + m.insert(alu) + return +} + +func (m *machine) addReg64ToReg64(rn, rm regalloc.VReg) (rd regalloc.VReg) { + rd = m.compiler.AllocateVReg(ssa.TypeI64) + alu := m.allocateInstr() + alu.asALU(aluOpAdd, operandNR(rd), operandNR(rn), operandNR(rm), true) + m.insert(alu) + return +} + +func (m *machine) addRegToReg64Ext(rn, rm regalloc.VReg, ext extendOp) (rd regalloc.VReg) { + rd = m.compiler.AllocateVReg(ssa.TypeI64) + alu := m.allocateInstr() + alu.asALU(aluOpAdd, operandNR(rd), operandNR(rn), operandER(rm, ext, 64), true) + m.insert(alu) + return +} diff --git a/vendor/github.com/tetratelabs/wazero/internal/engine/wazevo/backend/isa/arm64/machine.go b/vendor/github.com/tetratelabs/wazero/internal/engine/wazevo/backend/isa/arm64/machine.go new file mode 100644 index 000000000..b435d9ba9 --- /dev/null +++ b/vendor/github.com/tetratelabs/wazero/internal/engine/wazevo/backend/isa/arm64/machine.go @@ -0,0 +1,515 @@ +package arm64 + +import ( + "context" + "fmt" + "strings" + + "github.com/tetratelabs/wazero/internal/engine/wazevo/backend" + "github.com/tetratelabs/wazero/internal/engine/wazevo/backend/regalloc" + "github.com/tetratelabs/wazero/internal/engine/wazevo/ssa" + "github.com/tetratelabs/wazero/internal/engine/wazevo/wazevoapi" +) + +type ( + // machine implements backend.Machine. + machine struct { + compiler backend.Compiler + executableContext *backend.ExecutableContextT[instruction] + currentABI *backend.FunctionABI + + regAlloc regalloc.Allocator + regAllocFn *backend.RegAllocFunction[*instruction, *machine] + + // addendsWorkQueue is used during address lowering, defined here for reuse. + addendsWorkQueue wazevoapi.Queue[ssa.Value] + addends32 wazevoapi.Queue[addend32] + // addends64 is used during address lowering, defined here for reuse. + addends64 wazevoapi.Queue[regalloc.VReg] + unresolvedAddressModes []*instruction + + // condBrRelocs holds the conditional branches which need offset relocation. + condBrRelocs []condBrReloc + + // jmpTableTargets holds the labels of the jump table targets. + jmpTableTargets [][]uint32 + + // spillSlotSize is the size of the stack slot in bytes used for spilling registers. + // During the execution of the function, the stack looks like: + // + // + // (high address) + // +-----------------+ + // | ....... | + // | ret Y | + // | ....... | + // | ret 0 | + // | arg X | + // | ....... | + // | arg 1 | + // | arg 0 | + // | xxxxx | + // | ReturnAddress | + // +-----------------+ <<-| + // | ........... | | + // | spill slot M | | <--- spillSlotSize + // | ............ | | + // | spill slot 2 | | + // | spill slot 1 | <<-+ + // | clobbered N | + // | ........... | + // | clobbered 1 | + // | clobbered 0 | + // SP---> +-----------------+ + // (low address) + // + // and it represents the size of the space between FP and the first spilled slot. This must be a multiple of 16. + // Also note that this is only known after register allocation. + spillSlotSize int64 + spillSlots map[regalloc.VRegID]int64 // regalloc.VRegID to offset. + // clobberedRegs holds real-register backed VRegs saved at the function prologue, and restored at the epilogue. + clobberedRegs []regalloc.VReg + + maxRequiredStackSizeForCalls int64 + stackBoundsCheckDisabled bool + + regAllocStarted bool + } + + addend32 struct { + r regalloc.VReg + ext extendOp + } + + condBrReloc struct { + cbr *instruction + // currentLabelPos is the labelPosition within which condBr is defined. + currentLabelPos *labelPosition + // Next block's labelPosition. + nextLabel label + offset int64 + } + + labelPosition = backend.LabelPosition[instruction] + label = backend.Label +) + +const ( + labelReturn = backend.LabelReturn + labelInvalid = backend.LabelInvalid +) + +// NewBackend returns a new backend for arm64. +func NewBackend() backend.Machine { + m := &machine{ + spillSlots: make(map[regalloc.VRegID]int64), + executableContext: newExecutableContext(), + regAlloc: regalloc.NewAllocator(regInfo), + } + return m +} + +func newExecutableContext() *backend.ExecutableContextT[instruction] { + return backend.NewExecutableContextT[instruction](resetInstruction, setNext, setPrev, asNop0) +} + +// ExecutableContext implements backend.Machine. +func (m *machine) ExecutableContext() backend.ExecutableContext { + return m.executableContext +} + +// RegAlloc implements backend.Machine Function. +func (m *machine) RegAlloc() { + rf := m.regAllocFn + for _, pos := range m.executableContext.OrderedBlockLabels { + rf.AddBlock(pos.SB, pos.L, pos.Begin, pos.End) + } + + m.regAllocStarted = true + m.regAlloc.DoAllocation(rf) + // Now that we know the final spill slot size, we must align spillSlotSize to 16 bytes. + m.spillSlotSize = (m.spillSlotSize + 15) &^ 15 +} + +// Reset implements backend.Machine. +func (m *machine) Reset() { + m.clobberedRegs = m.clobberedRegs[:0] + for key := range m.spillSlots { + m.clobberedRegs = append(m.clobberedRegs, regalloc.VReg(key)) + } + for _, key := range m.clobberedRegs { + delete(m.spillSlots, regalloc.VRegID(key)) + } + m.clobberedRegs = m.clobberedRegs[:0] + m.regAllocStarted = false + m.regAlloc.Reset() + m.regAllocFn.Reset() + m.spillSlotSize = 0 + m.unresolvedAddressModes = m.unresolvedAddressModes[:0] + m.maxRequiredStackSizeForCalls = 0 + m.executableContext.Reset() + m.jmpTableTargets = m.jmpTableTargets[:0] +} + +// SetCurrentABI implements backend.Machine SetCurrentABI. +func (m *machine) SetCurrentABI(abi *backend.FunctionABI) { + m.currentABI = abi +} + +// DisableStackCheck implements backend.Machine DisableStackCheck. +func (m *machine) DisableStackCheck() { + m.stackBoundsCheckDisabled = true +} + +// SetCompiler implements backend.Machine. +func (m *machine) SetCompiler(ctx backend.Compiler) { + m.compiler = ctx + m.regAllocFn = backend.NewRegAllocFunction[*instruction, *machine](m, ctx.SSABuilder(), ctx) +} + +func (m *machine) insert(i *instruction) { + ectx := m.executableContext + ectx.PendingInstructions = append(ectx.PendingInstructions, i) +} + +func (m *machine) insertBrTargetLabel() label { + nop, l := m.allocateBrTarget() + m.insert(nop) + return l +} + +func (m *machine) allocateBrTarget() (nop *instruction, l label) { + ectx := m.executableContext + l = ectx.AllocateLabel() + nop = m.allocateInstr() + nop.asNop0WithLabel(l) + pos := ectx.AllocateLabelPosition(l) + pos.Begin, pos.End = nop, nop + ectx.LabelPositions[l] = pos + return +} + +// allocateInstr allocates an instruction. +func (m *machine) allocateInstr() *instruction { + instr := m.executableContext.InstructionPool.Allocate() + if !m.regAllocStarted { + instr.addedBeforeRegAlloc = true + } + return instr +} + +func resetInstruction(i *instruction) { + *i = instruction{} +} + +func (m *machine) allocateNop() *instruction { + instr := m.allocateInstr() + instr.asNop0() + return instr +} + +func (m *machine) resolveAddressingMode(arg0offset, ret0offset int64, i *instruction) { + amode := &i.amode + switch amode.kind { + case addressModeKindResultStackSpace: + amode.imm += ret0offset + case addressModeKindArgStackSpace: + amode.imm += arg0offset + default: + panic("BUG") + } + + var sizeInBits byte + switch i.kind { + case store8, uLoad8: + sizeInBits = 8 + case store16, uLoad16: + sizeInBits = 16 + case store32, fpuStore32, uLoad32, fpuLoad32: + sizeInBits = 32 + case store64, fpuStore64, uLoad64, fpuLoad64: + sizeInBits = 64 + case fpuStore128, fpuLoad128: + sizeInBits = 128 + default: + panic("BUG") + } + + if offsetFitsInAddressModeKindRegUnsignedImm12(sizeInBits, amode.imm) { + amode.kind = addressModeKindRegUnsignedImm12 + } else { + // This case, we load the offset into the temporary register, + // and then use it as the index register. + newPrev := m.lowerConstantI64AndInsert(i.prev, tmpRegVReg, amode.imm) + linkInstr(newPrev, i) + *amode = addressMode{kind: addressModeKindRegReg, rn: amode.rn, rm: tmpRegVReg, extOp: extendOpUXTX /* indicates rm reg is 64-bit */} + } +} + +// resolveRelativeAddresses resolves the relative addresses before encoding. +func (m *machine) resolveRelativeAddresses(ctx context.Context) { + ectx := m.executableContext + for { + if len(m.unresolvedAddressModes) > 0 { + arg0offset, ret0offset := m.arg0OffsetFromSP(), m.ret0OffsetFromSP() + for _, i := range m.unresolvedAddressModes { + m.resolveAddressingMode(arg0offset, ret0offset, i) + } + } + + // Reuse the slice to gather the unresolved conditional branches. + m.condBrRelocs = m.condBrRelocs[:0] + + var fn string + var fnIndex int + var labelToSSABlockID map[label]ssa.BasicBlockID + if wazevoapi.PerfMapEnabled { + fn = wazevoapi.GetCurrentFunctionName(ctx) + labelToSSABlockID = make(map[label]ssa.BasicBlockID) + for i, l := range ectx.SsaBlockIDToLabels { + labelToSSABlockID[l] = ssa.BasicBlockID(i) + } + fnIndex = wazevoapi.GetCurrentFunctionIndex(ctx) + } + + // Next, in order to determine the offsets of relative jumps, we have to calculate the size of each label. + var offset int64 + for i, pos := range ectx.OrderedBlockLabels { + pos.BinaryOffset = offset + var size int64 + for cur := pos.Begin; ; cur = cur.next { + switch cur.kind { + case nop0: + l := cur.nop0Label() + if pos, ok := ectx.LabelPositions[l]; ok { + pos.BinaryOffset = offset + size + } + case condBr: + if !cur.condBrOffsetResolved() { + var nextLabel label + if i < len(ectx.OrderedBlockLabels)-1 { + // Note: this is only used when the block ends with fallthrough, + // therefore can be safely assumed that the next block exists when it's needed. + nextLabel = ectx.OrderedBlockLabels[i+1].L + } + m.condBrRelocs = append(m.condBrRelocs, condBrReloc{ + cbr: cur, currentLabelPos: pos, offset: offset + size, + nextLabel: nextLabel, + }) + } + } + size += cur.size() + if cur == pos.End { + break + } + } + + if wazevoapi.PerfMapEnabled { + if size > 0 { + l := pos.L + var labelStr string + if blkID, ok := labelToSSABlockID[l]; ok { + labelStr = fmt.Sprintf("%s::SSA_Block[%s]", l, blkID) + } else { + labelStr = l.String() + } + wazevoapi.PerfMap.AddModuleEntry(fnIndex, offset, uint64(size), fmt.Sprintf("%s:::::%s", fn, labelStr)) + } + } + offset += size + } + + // Before resolving any offsets, we need to check if all the conditional branches can be resolved. + var needRerun bool + for i := range m.condBrRelocs { + reloc := &m.condBrRelocs[i] + cbr := reloc.cbr + offset := reloc.offset + + target := cbr.condBrLabel() + offsetOfTarget := ectx.LabelPositions[target].BinaryOffset + diff := offsetOfTarget - offset + if divided := diff >> 2; divided < minSignedInt19 || divided > maxSignedInt19 { + // This case the conditional branch is too huge. We place the trampoline instructions at the end of the current block, + // and jump to it. + m.insertConditionalJumpTrampoline(cbr, reloc.currentLabelPos, reloc.nextLabel) + // Then, we need to recall this function to fix up the label offsets + // as they have changed after the trampoline is inserted. + needRerun = true + } + } + if needRerun { + if wazevoapi.PerfMapEnabled { + wazevoapi.PerfMap.Clear() + } + } else { + break + } + } + + var currentOffset int64 + for cur := ectx.RootInstr; cur != nil; cur = cur.next { + switch cur.kind { + case br: + target := cur.brLabel() + offsetOfTarget := ectx.LabelPositions[target].BinaryOffset + diff := offsetOfTarget - currentOffset + divided := diff >> 2 + if divided < minSignedInt26 || divided > maxSignedInt26 { + // This means the currently compiled single function is extremely large. + panic("too large function that requires branch relocation of large unconditional branch larger than 26-bit range") + } + cur.brOffsetResolve(diff) + case condBr: + if !cur.condBrOffsetResolved() { + target := cur.condBrLabel() + offsetOfTarget := ectx.LabelPositions[target].BinaryOffset + diff := offsetOfTarget - currentOffset + if divided := diff >> 2; divided < minSignedInt19 || divided > maxSignedInt19 { + panic("BUG: branch relocation for large conditional branch larger than 19-bit range must be handled properly") + } + cur.condBrOffsetResolve(diff) + } + case brTableSequence: + tableIndex := cur.u1 + targets := m.jmpTableTargets[tableIndex] + for i := range targets { + l := label(targets[i]) + offsetOfTarget := ectx.LabelPositions[l].BinaryOffset + diff := offsetOfTarget - (currentOffset + brTableSequenceOffsetTableBegin) + targets[i] = uint32(diff) + } + cur.brTableSequenceOffsetsResolved() + case emitSourceOffsetInfo: + m.compiler.AddSourceOffsetInfo(currentOffset, cur.sourceOffsetInfo()) + } + currentOffset += cur.size() + } +} + +const ( + maxSignedInt26 = 1<<25 - 1 + minSignedInt26 = -(1 << 25) + + maxSignedInt19 = 1<<18 - 1 + minSignedInt19 = -(1 << 18) +) + +func (m *machine) insertConditionalJumpTrampoline(cbr *instruction, currentBlk *labelPosition, nextLabel label) { + cur := currentBlk.End + originalTarget := cbr.condBrLabel() + endNext := cur.next + + if cur.kind != br { + // If the current block ends with a conditional branch, we can just insert the trampoline after it. + // Otherwise, we need to insert "skip" instruction to skip the trampoline instructions. + skip := m.allocateInstr() + skip.asBr(nextLabel) + cur = linkInstr(cur, skip) + } + + cbrNewTargetInstr, cbrNewTargetLabel := m.allocateBrTarget() + cbr.setCondBrTargets(cbrNewTargetLabel) + cur = linkInstr(cur, cbrNewTargetInstr) + + // Then insert the unconditional branch to the original, which should be possible to get encoded + // as 26-bit offset should be enough for any practical application. + br := m.allocateInstr() + br.asBr(originalTarget) + cur = linkInstr(cur, br) + + // Update the end of the current block. + currentBlk.End = cur + + linkInstr(cur, endNext) +} + +// Format implements backend.Machine. +func (m *machine) Format() string { + ectx := m.executableContext + begins := map[*instruction]label{} + for l, pos := range ectx.LabelPositions { + begins[pos.Begin] = l + } + + irBlocks := map[label]ssa.BasicBlockID{} + for i, l := range ectx.SsaBlockIDToLabels { + irBlocks[l] = ssa.BasicBlockID(i) + } + + var lines []string + for cur := ectx.RootInstr; cur != nil; cur = cur.next { + if l, ok := begins[cur]; ok { + var labelStr string + if blkID, ok := irBlocks[l]; ok { + labelStr = fmt.Sprintf("%s (SSA Block: %s):", l, blkID) + } else { + labelStr = fmt.Sprintf("%s:", l) + } + lines = append(lines, labelStr) + } + if cur.kind == nop0 { + continue + } + lines = append(lines, "\t"+cur.String()) + } + return "\n" + strings.Join(lines, "\n") + "\n" +} + +// InsertReturn implements backend.Machine. +func (m *machine) InsertReturn() { + i := m.allocateInstr() + i.asRet() + m.insert(i) +} + +func (m *machine) getVRegSpillSlotOffsetFromSP(id regalloc.VRegID, size byte) int64 { + offset, ok := m.spillSlots[id] + if !ok { + offset = m.spillSlotSize + // TODO: this should be aligned depending on the `size` to use Imm12 offset load/store as much as possible. + m.spillSlots[id] = offset + m.spillSlotSize += int64(size) + } + return offset + 16 // spill slot starts above the clobbered registers and the frame size. +} + +func (m *machine) clobberedRegSlotSize() int64 { + return int64(len(m.clobberedRegs) * 16) +} + +func (m *machine) arg0OffsetFromSP() int64 { + return m.frameSize() + + 16 + // 16-byte aligned return address + 16 // frame size saved below the clobbered registers. +} + +func (m *machine) ret0OffsetFromSP() int64 { + return m.arg0OffsetFromSP() + m.currentABI.ArgStackSize +} + +func (m *machine) requiredStackSize() int64 { + return m.maxRequiredStackSizeForCalls + + m.frameSize() + + 16 + // 16-byte aligned return address. + 16 // frame size saved below the clobbered registers. +} + +func (m *machine) frameSize() int64 { + s := m.clobberedRegSlotSize() + m.spillSlotSize + if s&0xf != 0 { + panic(fmt.Errorf("BUG: frame size %d is not 16-byte aligned", s)) + } + return s +} + +func (m *machine) addJmpTableTarget(targets []ssa.BasicBlock) (index int) { + // TODO: reuse the slice! + labels := make([]uint32, len(targets)) + for j, target := range targets { + labels[j] = uint32(m.executableContext.GetOrAllocateSSABlockLabel(target)) + } + index = len(m.jmpTableTargets) + m.jmpTableTargets = append(m.jmpTableTargets, labels) + return +} diff --git a/vendor/github.com/tetratelabs/wazero/internal/engine/wazevo/backend/isa/arm64/machine_pro_epi_logue.go b/vendor/github.com/tetratelabs/wazero/internal/engine/wazevo/backend/isa/arm64/machine_pro_epi_logue.go new file mode 100644 index 000000000..466fac464 --- /dev/null +++ b/vendor/github.com/tetratelabs/wazero/internal/engine/wazevo/backend/isa/arm64/machine_pro_epi_logue.go @@ -0,0 +1,469 @@ +package arm64 + +import ( + "fmt" + + "github.com/tetratelabs/wazero/internal/engine/wazevo/backend/regalloc" + "github.com/tetratelabs/wazero/internal/engine/wazevo/wazevoapi" +) + +// PostRegAlloc implements backend.Machine. +func (m *machine) PostRegAlloc() { + m.setupPrologue() + m.postRegAlloc() +} + +// setupPrologue initializes the prologue of the function. +func (m *machine) setupPrologue() { + ectx := m.executableContext + + cur := ectx.RootInstr + prevInitInst := cur.next + + // + // (high address) (high address) + // SP----> +-----------------+ +------------------+ <----+ + // | ....... | | ....... | | + // | ret Y | | ret Y | | + // | ....... | | ....... | | + // | ret 0 | | ret 0 | | + // | arg X | | arg X | | size_of_arg_ret. + // | ....... | ====> | ....... | | + // | arg 1 | | arg 1 | | + // | arg 0 | | arg 0 | <----+ + // |-----------------| | size_of_arg_ret | + // | return address | + // +------------------+ <---- SP + // (low address) (low address) + + // Saves the return address (lr) and the size_of_arg_ret below the SP. + // size_of_arg_ret is used for stack unwinding. + cur = m.createReturnAddrAndSizeOfArgRetSlot(cur) + + if !m.stackBoundsCheckDisabled { + cur = m.insertStackBoundsCheck(m.requiredStackSize(), cur) + } + + // Decrement SP if spillSlotSize > 0. + if m.spillSlotSize == 0 && len(m.spillSlots) != 0 { + panic(fmt.Sprintf("BUG: spillSlotSize=%d, spillSlots=%v\n", m.spillSlotSize, m.spillSlots)) + } + + if regs := m.clobberedRegs; len(regs) > 0 { + // + // (high address) (high address) + // +-----------------+ +-----------------+ + // | ....... | | ....... | + // | ret Y | | ret Y | + // | ....... | | ....... | + // | ret 0 | | ret 0 | + // | arg X | | arg X | + // | ....... | | ....... | + // | arg 1 | | arg 1 | + // | arg 0 | | arg 0 | + // | size_of_arg_ret | | size_of_arg_ret | + // | ReturnAddress | | ReturnAddress | + // SP----> +-----------------+ ====> +-----------------+ + // (low address) | clobbered M | + // | ............ | + // | clobbered 0 | + // +-----------------+ <----- SP + // (low address) + // + _amode := addressModePreOrPostIndex(spVReg, + -16, // stack pointer must be 16-byte aligned. + true, // Decrement before store. + ) + for _, vr := range regs { + // TODO: pair stores to reduce the number of instructions. + store := m.allocateInstr() + store.asStore(operandNR(vr), _amode, regTypeToRegisterSizeInBits(vr.RegType())) + cur = linkInstr(cur, store) + } + } + + if size := m.spillSlotSize; size > 0 { + // Check if size is 16-byte aligned. + if size&0xf != 0 { + panic(fmt.Errorf("BUG: spill slot size %d is not 16-byte aligned", size)) + } + + cur = m.addsAddOrSubStackPointer(cur, spVReg, size, false) + + // At this point, the stack looks like: + // + // (high address) + // +------------------+ + // | ....... | + // | ret Y | + // | ....... | + // | ret 0 | + // | arg X | + // | ....... | + // | arg 1 | + // | arg 0 | + // | size_of_arg_ret | + // | ReturnAddress | + // +------------------+ + // | clobbered M | + // | ............ | + // | clobbered 0 | + // | spill slot N | + // | ............ | + // | spill slot 2 | + // | spill slot 0 | + // SP----> +------------------+ + // (low address) + } + + // We push the frame size into the stack to make it possible to unwind stack: + // + // + // (high address) (high address) + // +-----------------+ +-----------------+ + // | ....... | | ....... | + // | ret Y | | ret Y | + // | ....... | | ....... | + // | ret 0 | | ret 0 | + // | arg X | | arg X | + // | ....... | | ....... | + // | arg 1 | | arg 1 | + // | arg 0 | | arg 0 | + // | size_of_arg_ret | | size_of_arg_ret | + // | ReturnAddress | | ReturnAddress | + // +-----------------+ ==> +-----------------+ <----+ + // | clobbered M | | clobbered M | | + // | ............ | | ............ | | + // | clobbered 2 | | clobbered 2 | | + // | clobbered 1 | | clobbered 1 | | frame size + // | clobbered 0 | | clobbered 0 | | + // | spill slot N | | spill slot N | | + // | ............ | | ............ | | + // | spill slot 0 | | spill slot 0 | <----+ + // SP---> +-----------------+ | xxxxxx | ;; unused space to make it 16-byte aligned. + // | frame_size | + // +-----------------+ <---- SP + // (low address) + // + cur = m.createFrameSizeSlot(cur, m.frameSize()) + + linkInstr(cur, prevInitInst) +} + +func (m *machine) createReturnAddrAndSizeOfArgRetSlot(cur *instruction) *instruction { + // First we decrement the stack pointer to point the arg0 slot. + var sizeOfArgRetReg regalloc.VReg + s := int64(m.currentABI.AlignedArgResultStackSlotSize()) + if s > 0 { + cur = m.lowerConstantI64AndInsert(cur, tmpRegVReg, s) + sizeOfArgRetReg = tmpRegVReg + + subSp := m.allocateInstr() + subSp.asALU(aluOpSub, operandNR(spVReg), operandNR(spVReg), operandNR(sizeOfArgRetReg), true) + cur = linkInstr(cur, subSp) + } else { + sizeOfArgRetReg = xzrVReg + } + + // Saves the return address (lr) and the size_of_arg_ret below the SP. + // size_of_arg_ret is used for stack unwinding. + pstr := m.allocateInstr() + amode := addressModePreOrPostIndex(spVReg, -16, true /* decrement before store */) + pstr.asStorePair64(lrVReg, sizeOfArgRetReg, amode) + cur = linkInstr(cur, pstr) + return cur +} + +func (m *machine) createFrameSizeSlot(cur *instruction, s int64) *instruction { + var frameSizeReg regalloc.VReg + if s > 0 { + cur = m.lowerConstantI64AndInsert(cur, tmpRegVReg, s) + frameSizeReg = tmpRegVReg + } else { + frameSizeReg = xzrVReg + } + _amode := addressModePreOrPostIndex(spVReg, + -16, // stack pointer must be 16-byte aligned. + true, // Decrement before store. + ) + store := m.allocateInstr() + store.asStore(operandNR(frameSizeReg), _amode, 64) + cur = linkInstr(cur, store) + return cur +} + +// postRegAlloc does multiple things while walking through the instructions: +// 1. Removes the redundant copy instruction. +// 2. Inserts the epilogue. +func (m *machine) postRegAlloc() { + ectx := m.executableContext + for cur := ectx.RootInstr; cur != nil; cur = cur.next { + switch cur.kind { + case ret: + m.setupEpilogueAfter(cur.prev) + case loadConstBlockArg: + lc := cur + next := lc.next + m.executableContext.PendingInstructions = m.executableContext.PendingInstructions[:0] + m.lowerLoadConstantBlockArgAfterRegAlloc(lc) + for _, instr := range m.executableContext.PendingInstructions { + cur = linkInstr(cur, instr) + } + linkInstr(cur, next) + m.executableContext.PendingInstructions = m.executableContext.PendingInstructions[:0] + default: + // Removes the redundant copy instruction. + if cur.IsCopy() && cur.rn.realReg() == cur.rd.realReg() { + prev, next := cur.prev, cur.next + // Remove the copy instruction. + prev.next = next + if next != nil { + next.prev = prev + } + } + } + } +} + +func (m *machine) setupEpilogueAfter(cur *instruction) { + prevNext := cur.next + + // We've stored the frame size in the prologue, and now that we are about to return from this function, we won't need it anymore. + cur = m.addsAddOrSubStackPointer(cur, spVReg, 16, true) + + if s := m.spillSlotSize; s > 0 { + // Adjust SP to the original value: + // + // (high address) (high address) + // +-----------------+ +-----------------+ + // | ....... | | ....... | + // | ret Y | | ret Y | + // | ....... | | ....... | + // | ret 0 | | ret 0 | + // | arg X | | arg X | + // | ....... | | ....... | + // | arg 1 | | arg 1 | + // | arg 0 | | arg 0 | + // | xxxxx | | xxxxx | + // | ReturnAddress | | ReturnAddress | + // +-----------------+ ====> +-----------------+ + // | clobbered M | | clobbered M | + // | ............ | | ............ | + // | clobbered 1 | | clobbered 1 | + // | clobbered 0 | | clobbered 0 | + // | spill slot N | +-----------------+ <---- SP + // | ............ | + // | spill slot 0 | + // SP---> +-----------------+ + // (low address) + // + cur = m.addsAddOrSubStackPointer(cur, spVReg, s, true) + } + + // First we need to restore the clobbered registers. + if len(m.clobberedRegs) > 0 { + // (high address) + // +-----------------+ +-----------------+ + // | ....... | | ....... | + // | ret Y | | ret Y | + // | ....... | | ....... | + // | ret 0 | | ret 0 | + // | arg X | | arg X | + // | ....... | | ....... | + // | arg 1 | | arg 1 | + // | arg 0 | | arg 0 | + // | xxxxx | | xxxxx | + // | ReturnAddress | | ReturnAddress | + // +-----------------+ ========> +-----------------+ <---- SP + // | clobbered M | + // | ........... | + // | clobbered 1 | + // | clobbered 0 | + // SP---> +-----------------+ + // (low address) + + l := len(m.clobberedRegs) - 1 + for i := range m.clobberedRegs { + vr := m.clobberedRegs[l-i] // reverse order to restore. + load := m.allocateInstr() + amode := addressModePreOrPostIndex(spVReg, + 16, // stack pointer must be 16-byte aligned. + false, // Increment after store. + ) + // TODO: pair loads to reduce the number of instructions. + switch regTypeToRegisterSizeInBits(vr.RegType()) { + case 64: // save int reg. + load.asULoad(operandNR(vr), amode, 64) + case 128: // save vector reg. + load.asFpuLoad(operandNR(vr), amode, 128) + } + cur = linkInstr(cur, load) + } + } + + // Reload the return address (lr). + // + // +-----------------+ +-----------------+ + // | ....... | | ....... | + // | ret Y | | ret Y | + // | ....... | | ....... | + // | ret 0 | | ret 0 | + // | arg X | | arg X | + // | ....... | ===> | ....... | + // | arg 1 | | arg 1 | + // | arg 0 | | arg 0 | + // | xxxxx | +-----------------+ <---- SP + // | ReturnAddress | + // SP----> +-----------------+ + + ldr := m.allocateInstr() + ldr.asULoad(operandNR(lrVReg), + addressModePreOrPostIndex(spVReg, 16 /* stack pointer must be 16-byte aligned. */, false /* increment after loads */), 64) + cur = linkInstr(cur, ldr) + + if s := int64(m.currentABI.AlignedArgResultStackSlotSize()); s > 0 { + cur = m.addsAddOrSubStackPointer(cur, spVReg, s, true) + } + + linkInstr(cur, prevNext) +} + +// saveRequiredRegs is the set of registers that must be saved/restored during growing stack when there's insufficient +// stack space left. Basically this is the combination of CalleeSavedRegisters plus argument registers execpt for x0, +// which always points to the execution context whenever the native code is entered from Go. +var saveRequiredRegs = []regalloc.VReg{ + x1VReg, x2VReg, x3VReg, x4VReg, x5VReg, x6VReg, x7VReg, + x19VReg, x20VReg, x21VReg, x22VReg, x23VReg, x24VReg, x25VReg, x26VReg, x28VReg, lrVReg, + v0VReg, v1VReg, v2VReg, v3VReg, v4VReg, v5VReg, v6VReg, v7VReg, + v18VReg, v19VReg, v20VReg, v21VReg, v22VReg, v23VReg, v24VReg, v25VReg, v26VReg, v27VReg, v28VReg, v29VReg, v30VReg, v31VReg, +} + +// insertStackBoundsCheck will insert the instructions after `cur` to check the +// stack bounds, and if there's no sufficient spaces required for the function, +// exit the execution and try growing it in Go world. +// +// TODO: we should be able to share the instructions across all the functions to reduce the size of compiled executable. +func (m *machine) insertStackBoundsCheck(requiredStackSize int64, cur *instruction) *instruction { + if requiredStackSize%16 != 0 { + panic("BUG") + } + + if immm12op, ok := asImm12Operand(uint64(requiredStackSize)); ok { + // sub tmp, sp, #requiredStackSize + sub := m.allocateInstr() + sub.asALU(aluOpSub, operandNR(tmpRegVReg), operandNR(spVReg), immm12op, true) + cur = linkInstr(cur, sub) + } else { + // This case, we first load the requiredStackSize into the temporary register, + cur = m.lowerConstantI64AndInsert(cur, tmpRegVReg, requiredStackSize) + // Then subtract it. + sub := m.allocateInstr() + sub.asALU(aluOpSub, operandNR(tmpRegVReg), operandNR(spVReg), operandNR(tmpRegVReg), true) + cur = linkInstr(cur, sub) + } + + tmp2 := x11VReg // Caller save, so it is safe to use it here in the prologue. + + // ldr tmp2, [executionContext #StackBottomPtr] + ldr := m.allocateInstr() + ldr.asULoad(operandNR(tmp2), addressMode{ + kind: addressModeKindRegUnsignedImm12, + rn: x0VReg, // execution context is always the first argument. + imm: wazevoapi.ExecutionContextOffsetStackBottomPtr.I64(), + }, 64) + cur = linkInstr(cur, ldr) + + // subs xzr, tmp, tmp2 + subs := m.allocateInstr() + subs.asALU(aluOpSubS, operandNR(xzrVReg), operandNR(tmpRegVReg), operandNR(tmp2), true) + cur = linkInstr(cur, subs) + + // b.ge #imm + cbr := m.allocateInstr() + cbr.asCondBr(ge.asCond(), labelInvalid, false /* ignored */) + cur = linkInstr(cur, cbr) + + // Set the required stack size and set it to the exec context. + { + // First load the requiredStackSize into the temporary register, + cur = m.lowerConstantI64AndInsert(cur, tmpRegVReg, requiredStackSize) + setRequiredStackSize := m.allocateInstr() + setRequiredStackSize.asStore(operandNR(tmpRegVReg), + addressMode{ + kind: addressModeKindRegUnsignedImm12, + // Execution context is always the first argument. + rn: x0VReg, imm: wazevoapi.ExecutionContextOffsetStackGrowRequiredSize.I64(), + }, 64) + + cur = linkInstr(cur, setRequiredStackSize) + } + + ldrAddress := m.allocateInstr() + ldrAddress.asULoad(operandNR(tmpRegVReg), addressMode{ + kind: addressModeKindRegUnsignedImm12, + rn: x0VReg, // execution context is always the first argument + imm: wazevoapi.ExecutionContextOffsetStackGrowCallTrampolineAddress.I64(), + }, 64) + cur = linkInstr(cur, ldrAddress) + + // Then jumps to the stack grow call sequence's address, meaning + // transferring the control to the code compiled by CompileStackGrowCallSequence. + bl := m.allocateInstr() + bl.asCallIndirect(tmpRegVReg, nil) + cur = linkInstr(cur, bl) + + // Now that we know the entire code, we can finalize how many bytes + // we have to skip when the stack size is sufficient. + var cbrOffset int64 + for _cur := cbr; ; _cur = _cur.next { + cbrOffset += _cur.size() + if _cur == cur { + break + } + } + cbr.condBrOffsetResolve(cbrOffset) + return cur +} + +// CompileStackGrowCallSequence implements backend.Machine. +func (m *machine) CompileStackGrowCallSequence() []byte { + ectx := m.executableContext + + cur := m.allocateInstr() + cur.asNop0() + ectx.RootInstr = cur + + // Save the callee saved and argument registers. + cur = m.saveRegistersInExecutionContext(cur, saveRequiredRegs) + + // Save the current stack pointer. + cur = m.saveCurrentStackPointer(cur, x0VReg) + + // Set the exit status on the execution context. + cur = m.setExitCode(cur, x0VReg, wazevoapi.ExitCodeGrowStack) + + // Exit the execution. + cur = m.storeReturnAddressAndExit(cur) + + // After the exit, restore the saved registers. + cur = m.restoreRegistersInExecutionContext(cur, saveRequiredRegs) + + // Then goes back the original address of this stack grow call. + ret := m.allocateInstr() + ret.asRet() + linkInstr(cur, ret) + + m.encode(ectx.RootInstr) + return m.compiler.Buf() +} + +func (m *machine) addsAddOrSubStackPointer(cur *instruction, rd regalloc.VReg, diff int64, add bool) *instruction { + ectx := m.executableContext + + ectx.PendingInstructions = ectx.PendingInstructions[:0] + m.insertAddOrSubStackPointer(rd, diff, add) + for _, inserted := range ectx.PendingInstructions { + cur = linkInstr(cur, inserted) + } + return cur +} diff --git a/vendor/github.com/tetratelabs/wazero/internal/engine/wazevo/backend/isa/arm64/machine_regalloc.go b/vendor/github.com/tetratelabs/wazero/internal/engine/wazevo/backend/isa/arm64/machine_regalloc.go new file mode 100644 index 000000000..1c8793b73 --- /dev/null +++ b/vendor/github.com/tetratelabs/wazero/internal/engine/wazevo/backend/isa/arm64/machine_regalloc.go @@ -0,0 +1,152 @@ +package arm64 + +// This file implements the interfaces required for register allocations. See backend.RegAllocFunctionMachine. + +import ( + "github.com/tetratelabs/wazero/internal/engine/wazevo/backend" + "github.com/tetratelabs/wazero/internal/engine/wazevo/backend/regalloc" + "github.com/tetratelabs/wazero/internal/engine/wazevo/ssa" +) + +// ClobberedRegisters implements backend.RegAllocFunctionMachine. +func (m *machine) ClobberedRegisters(regs []regalloc.VReg) { + m.clobberedRegs = append(m.clobberedRegs[:0], regs...) +} + +// Swap implements backend.RegAllocFunctionMachine. +func (m *machine) Swap(cur *instruction, x1, x2, tmp regalloc.VReg) { + prevNext := cur.next + var mov1, mov2, mov3 *instruction + if x1.RegType() == regalloc.RegTypeInt { + if !tmp.Valid() { + tmp = tmpRegVReg + } + mov1 = m.allocateInstr().asMove64(tmp, x1) + mov2 = m.allocateInstr().asMove64(x1, x2) + mov3 = m.allocateInstr().asMove64(x2, tmp) + cur = linkInstr(cur, mov1) + cur = linkInstr(cur, mov2) + cur = linkInstr(cur, mov3) + linkInstr(cur, prevNext) + } else { + if !tmp.Valid() { + r2 := x2.RealReg() + // Temporarily spill x1 to stack. + cur = m.InsertStoreRegisterAt(x1, cur, true).prev + // Then move x2 to x1. + cur = linkInstr(cur, m.allocateInstr().asFpuMov128(x1, x2)) + linkInstr(cur, prevNext) + // Then reload the original value on x1 from stack to r2. + m.InsertReloadRegisterAt(x1.SetRealReg(r2), cur, true) + } else { + mov1 = m.allocateInstr().asFpuMov128(tmp, x1) + mov2 = m.allocateInstr().asFpuMov128(x1, x2) + mov3 = m.allocateInstr().asFpuMov128(x2, tmp) + cur = linkInstr(cur, mov1) + cur = linkInstr(cur, mov2) + cur = linkInstr(cur, mov3) + linkInstr(cur, prevNext) + } + } +} + +// InsertMoveBefore implements backend.RegAllocFunctionMachine. +func (m *machine) InsertMoveBefore(dst, src regalloc.VReg, instr *instruction) { + typ := src.RegType() + if typ != dst.RegType() { + panic("BUG: src and dst must have the same type") + } + + mov := m.allocateInstr() + if typ == regalloc.RegTypeInt { + mov.asMove64(dst, src) + } else { + mov.asFpuMov128(dst, src) + } + + cur := instr.prev + prevNext := cur.next + cur = linkInstr(cur, mov) + linkInstr(cur, prevNext) +} + +// SSABlockLabel implements backend.RegAllocFunctionMachine. +func (m *machine) SSABlockLabel(id ssa.BasicBlockID) backend.Label { + return m.executableContext.SsaBlockIDToLabels[id] +} + +// InsertStoreRegisterAt implements backend.RegAllocFunctionMachine. +func (m *machine) InsertStoreRegisterAt(v regalloc.VReg, instr *instruction, after bool) *instruction { + if !v.IsRealReg() { + panic("BUG: VReg must be backed by real reg to be stored") + } + + typ := m.compiler.TypeOf(v) + + var prevNext, cur *instruction + if after { + cur, prevNext = instr, instr.next + } else { + cur, prevNext = instr.prev, instr + } + + offsetFromSP := m.getVRegSpillSlotOffsetFromSP(v.ID(), typ.Size()) + var amode addressMode + cur, amode = m.resolveAddressModeForOffsetAndInsert(cur, offsetFromSP, typ.Bits(), spVReg, true) + store := m.allocateInstr() + store.asStore(operandNR(v), amode, typ.Bits()) + + cur = linkInstr(cur, store) + return linkInstr(cur, prevNext) +} + +// InsertReloadRegisterAt implements backend.RegAllocFunctionMachine. +func (m *machine) InsertReloadRegisterAt(v regalloc.VReg, instr *instruction, after bool) *instruction { + if !v.IsRealReg() { + panic("BUG: VReg must be backed by real reg to be stored") + } + + typ := m.compiler.TypeOf(v) + + var prevNext, cur *instruction + if after { + cur, prevNext = instr, instr.next + } else { + cur, prevNext = instr.prev, instr + } + + offsetFromSP := m.getVRegSpillSlotOffsetFromSP(v.ID(), typ.Size()) + var amode addressMode + cur, amode = m.resolveAddressModeForOffsetAndInsert(cur, offsetFromSP, typ.Bits(), spVReg, true) + load := m.allocateInstr() + switch typ { + case ssa.TypeI32, ssa.TypeI64: + load.asULoad(operandNR(v), amode, typ.Bits()) + case ssa.TypeF32, ssa.TypeF64: + load.asFpuLoad(operandNR(v), amode, typ.Bits()) + case ssa.TypeV128: + load.asFpuLoad(operandNR(v), amode, 128) + default: + panic("TODO") + } + + cur = linkInstr(cur, load) + return linkInstr(cur, prevNext) +} + +// LastInstrForInsertion implements backend.RegAllocFunctionMachine. +func (m *machine) LastInstrForInsertion(begin, end *instruction) *instruction { + cur := end + for cur.kind == nop0 { + cur = cur.prev + if cur == begin { + return end + } + } + switch cur.kind { + case br: + return cur + default: + return end + } +} diff --git a/vendor/github.com/tetratelabs/wazero/internal/engine/wazevo/backend/isa/arm64/machine_relocation.go b/vendor/github.com/tetratelabs/wazero/internal/engine/wazevo/backend/isa/arm64/machine_relocation.go new file mode 100644 index 000000000..83902d927 --- /dev/null +++ b/vendor/github.com/tetratelabs/wazero/internal/engine/wazevo/backend/isa/arm64/machine_relocation.go @@ -0,0 +1,117 @@ +package arm64 + +import ( + "encoding/binary" + "fmt" + "math" + "sort" + + "github.com/tetratelabs/wazero/internal/engine/wazevo/backend" +) + +const ( + // trampolineCallSize is the size of the trampoline instruction sequence for each function in an island. + trampolineCallSize = 4*4 + 4 // Four instructions + 32-bit immediate. + + // Unconditional branch offset is encoded as divided by 4 in imm26. + // https://developer.arm.com/documentation/ddi0596/2021-12/Base-Instructions/BL--Branch-with-Link-?lang=en + + maxUnconditionalBranchOffset = maxSignedInt26 * 4 + minUnconditionalBranchOffset = minSignedInt26 * 4 + + // trampolineIslandInterval is the range of the trampoline island. + // Half of the range is used for the trampoline island, and the other half is used for the function. + trampolineIslandInterval = maxUnconditionalBranchOffset / 2 + + // maxNumFunctions explicitly specifies the maximum number of functions that can be allowed in a single executable. + maxNumFunctions = trampolineIslandInterval >> 6 + + // maxFunctionExecutableSize is the maximum size of a function that can exist in a trampoline island. + // Conservatively set to 1/4 of the trampoline island interval. + maxFunctionExecutableSize = trampolineIslandInterval >> 2 +) + +// CallTrampolineIslandInfo implements backend.Machine CallTrampolineIslandInfo. +func (m *machine) CallTrampolineIslandInfo(numFunctions int) (interval, size int, err error) { + if numFunctions > maxNumFunctions { + return 0, 0, fmt.Errorf("too many functions: %d > %d", numFunctions, maxNumFunctions) + } + return trampolineIslandInterval, trampolineCallSize * numFunctions, nil +} + +// ResolveRelocations implements backend.Machine ResolveRelocations. +func (m *machine) ResolveRelocations( + refToBinaryOffset []int, + executable []byte, + relocations []backend.RelocationInfo, + callTrampolineIslandOffsets []int, +) { + for _, islandOffset := range callTrampolineIslandOffsets { + encodeCallTrampolineIsland(refToBinaryOffset, islandOffset, executable) + } + + for _, r := range relocations { + instrOffset := r.Offset + calleeFnOffset := refToBinaryOffset[r.FuncRef] + diff := int64(calleeFnOffset) - (instrOffset) + // Check if the diff is within the range of the branch instruction. + if diff < minUnconditionalBranchOffset || diff > maxUnconditionalBranchOffset { + // Find the near trampoline island from callTrampolineIslandOffsets. + islandOffset := searchTrampolineIsland(callTrampolineIslandOffsets, int(instrOffset)) + islandTargetOffset := islandOffset + trampolineCallSize*int(r.FuncRef) + diff = int64(islandTargetOffset) - (instrOffset) + if diff < minUnconditionalBranchOffset || diff > maxUnconditionalBranchOffset { + panic("BUG in trampoline placement") + } + } + binary.LittleEndian.PutUint32(executable[instrOffset:instrOffset+4], encodeUnconditionalBranch(true, diff)) + } +} + +// encodeCallTrampolineIsland encodes a trampoline island for the given functions. +// Each island consists of a trampoline instruction sequence for each function. +// Each trampoline instruction sequence consists of 4 instructions + 32-bit immediate. +func encodeCallTrampolineIsland(refToBinaryOffset []int, islandOffset int, executable []byte) { + for i := 0; i < len(refToBinaryOffset); i++ { + trampolineOffset := islandOffset + trampolineCallSize*i + + fnOffset := refToBinaryOffset[i] + diff := fnOffset - (trampolineOffset + 16) + if diff > math.MaxInt32 || diff < math.MinInt32 { + // This case even amd64 can't handle. 4GB is too big. + panic("too big binary") + } + + // The tmpReg, tmpReg2 is safe to overwrite (in fact any caller-saved register is safe to use). + tmpReg, tmpReg2 := regNumberInEncoding[tmpRegVReg.RealReg()], regNumberInEncoding[x11] + + // adr tmpReg, PC+16: load the address of #diff into tmpReg. + binary.LittleEndian.PutUint32(executable[trampolineOffset:], encodeAdr(tmpReg, 16)) + // ldrsw tmpReg2, [tmpReg]: Load #diff into tmpReg2. + binary.LittleEndian.PutUint32(executable[trampolineOffset+4:], + encodeLoadOrStore(sLoad32, tmpReg2, addressMode{kind: addressModeKindRegUnsignedImm12, rn: tmpRegVReg})) + // add tmpReg, tmpReg2, tmpReg: add #diff to the address of #diff, getting the absolute address of the function. + binary.LittleEndian.PutUint32(executable[trampolineOffset+8:], + encodeAluRRR(aluOpAdd, tmpReg, tmpReg, tmpReg2, true, false)) + // br tmpReg: branch to the function without overwriting the link register. + binary.LittleEndian.PutUint32(executable[trampolineOffset+12:], encodeUnconditionalBranchReg(tmpReg, false)) + // #diff + binary.LittleEndian.PutUint32(executable[trampolineOffset+16:], uint32(diff)) + } +} + +// searchTrampolineIsland finds the nearest trampoline island from callTrampolineIslandOffsets. +// Note that even if the offset is in the middle of two islands, it returns the latter one. +// That is ok because the island is always placed in the middle of the range. +// +// precondition: callTrampolineIslandOffsets is sorted in ascending order. +func searchTrampolineIsland(callTrampolineIslandOffsets []int, offset int) int { + l := len(callTrampolineIslandOffsets) + n := sort.Search(l, func(i int) bool { + return callTrampolineIslandOffsets[i] >= offset + }) + if n == l { + n = l - 1 + } + return callTrampolineIslandOffsets[n] +} diff --git a/vendor/github.com/tetratelabs/wazero/internal/engine/wazevo/backend/isa/arm64/reg.go b/vendor/github.com/tetratelabs/wazero/internal/engine/wazevo/backend/isa/arm64/reg.go new file mode 100644 index 000000000..45737516d --- /dev/null +++ b/vendor/github.com/tetratelabs/wazero/internal/engine/wazevo/backend/isa/arm64/reg.go @@ -0,0 +1,397 @@ +package arm64 + +import ( + "fmt" + "strconv" + "strings" + + "github.com/tetratelabs/wazero/internal/engine/wazevo/backend/regalloc" +) + +// Arm64-specific registers. +// +// See https://developer.arm.com/documentation/dui0801/a/Overview-of-AArch64-state/Predeclared-core-register-names-in-AArch64-state + +const ( + // General purpose registers. Note that we do not distinguish wn and xn registers + // because they are the same from the perspective of register allocator, and + // the size can be determined by the type of the instruction. + + x0 = regalloc.RealRegInvalid + 1 + iota + x1 + x2 + x3 + x4 + x5 + x6 + x7 + x8 + x9 + x10 + x11 + x12 + x13 + x14 + x15 + x16 + x17 + x18 + x19 + x20 + x21 + x22 + x23 + x24 + x25 + x26 + x27 + x28 + x29 + x30 + + // Vector registers. Note that we do not distinguish vn and dn, ... registers + // because they are the same from the perspective of register allocator, and + // the size can be determined by the type of the instruction. + + v0 + v1 + v2 + v3 + v4 + v5 + v6 + v7 + v8 + v9 + v10 + v11 + v12 + v13 + v14 + v15 + v16 + v17 + v18 + v19 + v20 + v21 + v22 + v23 + v24 + v25 + v26 + v27 + v28 + v29 + v30 + v31 + + // Special registers + + xzr + sp + lr = x30 + fp = x29 + tmp = x27 +) + +var ( + x0VReg = regalloc.FromRealReg(x0, regalloc.RegTypeInt) + x1VReg = regalloc.FromRealReg(x1, regalloc.RegTypeInt) + x2VReg = regalloc.FromRealReg(x2, regalloc.RegTypeInt) + x3VReg = regalloc.FromRealReg(x3, regalloc.RegTypeInt) + x4VReg = regalloc.FromRealReg(x4, regalloc.RegTypeInt) + x5VReg = regalloc.FromRealReg(x5, regalloc.RegTypeInt) + x6VReg = regalloc.FromRealReg(x6, regalloc.RegTypeInt) + x7VReg = regalloc.FromRealReg(x7, regalloc.RegTypeInt) + x8VReg = regalloc.FromRealReg(x8, regalloc.RegTypeInt) + x9VReg = regalloc.FromRealReg(x9, regalloc.RegTypeInt) + x10VReg = regalloc.FromRealReg(x10, regalloc.RegTypeInt) + x11VReg = regalloc.FromRealReg(x11, regalloc.RegTypeInt) + x12VReg = regalloc.FromRealReg(x12, regalloc.RegTypeInt) + x13VReg = regalloc.FromRealReg(x13, regalloc.RegTypeInt) + x14VReg = regalloc.FromRealReg(x14, regalloc.RegTypeInt) + x15VReg = regalloc.FromRealReg(x15, regalloc.RegTypeInt) + x16VReg = regalloc.FromRealReg(x16, regalloc.RegTypeInt) + x17VReg = regalloc.FromRealReg(x17, regalloc.RegTypeInt) + x18VReg = regalloc.FromRealReg(x18, regalloc.RegTypeInt) + x19VReg = regalloc.FromRealReg(x19, regalloc.RegTypeInt) + x20VReg = regalloc.FromRealReg(x20, regalloc.RegTypeInt) + x21VReg = regalloc.FromRealReg(x21, regalloc.RegTypeInt) + x22VReg = regalloc.FromRealReg(x22, regalloc.RegTypeInt) + x23VReg = regalloc.FromRealReg(x23, regalloc.RegTypeInt) + x24VReg = regalloc.FromRealReg(x24, regalloc.RegTypeInt) + x25VReg = regalloc.FromRealReg(x25, regalloc.RegTypeInt) + x26VReg = regalloc.FromRealReg(x26, regalloc.RegTypeInt) + x27VReg = regalloc.FromRealReg(x27, regalloc.RegTypeInt) + x28VReg = regalloc.FromRealReg(x28, regalloc.RegTypeInt) + x29VReg = regalloc.FromRealReg(x29, regalloc.RegTypeInt) + x30VReg = regalloc.FromRealReg(x30, regalloc.RegTypeInt) + v0VReg = regalloc.FromRealReg(v0, regalloc.RegTypeFloat) + v1VReg = regalloc.FromRealReg(v1, regalloc.RegTypeFloat) + v2VReg = regalloc.FromRealReg(v2, regalloc.RegTypeFloat) + v3VReg = regalloc.FromRealReg(v3, regalloc.RegTypeFloat) + v4VReg = regalloc.FromRealReg(v4, regalloc.RegTypeFloat) + v5VReg = regalloc.FromRealReg(v5, regalloc.RegTypeFloat) + v6VReg = regalloc.FromRealReg(v6, regalloc.RegTypeFloat) + v7VReg = regalloc.FromRealReg(v7, regalloc.RegTypeFloat) + v8VReg = regalloc.FromRealReg(v8, regalloc.RegTypeFloat) + v9VReg = regalloc.FromRealReg(v9, regalloc.RegTypeFloat) + v10VReg = regalloc.FromRealReg(v10, regalloc.RegTypeFloat) + v11VReg = regalloc.FromRealReg(v11, regalloc.RegTypeFloat) + v12VReg = regalloc.FromRealReg(v12, regalloc.RegTypeFloat) + v13VReg = regalloc.FromRealReg(v13, regalloc.RegTypeFloat) + v14VReg = regalloc.FromRealReg(v14, regalloc.RegTypeFloat) + v15VReg = regalloc.FromRealReg(v15, regalloc.RegTypeFloat) + v16VReg = regalloc.FromRealReg(v16, regalloc.RegTypeFloat) + v17VReg = regalloc.FromRealReg(v17, regalloc.RegTypeFloat) + v18VReg = regalloc.FromRealReg(v18, regalloc.RegTypeFloat) + v19VReg = regalloc.FromRealReg(v19, regalloc.RegTypeFloat) + v20VReg = regalloc.FromRealReg(v20, regalloc.RegTypeFloat) + v21VReg = regalloc.FromRealReg(v21, regalloc.RegTypeFloat) + v22VReg = regalloc.FromRealReg(v22, regalloc.RegTypeFloat) + v23VReg = regalloc.FromRealReg(v23, regalloc.RegTypeFloat) + v24VReg = regalloc.FromRealReg(v24, regalloc.RegTypeFloat) + v25VReg = regalloc.FromRealReg(v25, regalloc.RegTypeFloat) + v26VReg = regalloc.FromRealReg(v26, regalloc.RegTypeFloat) + v27VReg = regalloc.FromRealReg(v27, regalloc.RegTypeFloat) + // lr (link register) holds the return address at the function entry. + lrVReg = x30VReg + // tmpReg is used to perform spill/load on large stack offsets, and load large constants. + // Therefore, be cautious to use this register in the middle of the compilation, especially before the register allocation. + // This is the same as golang/go, but it's only described in the source code: + // https://github.com/golang/go/blob/18e17e2cb12837ea2c8582ecdb0cc780f49a1aac/src/cmd/compile/internal/ssa/_gen/ARM64Ops.go#L59 + // https://github.com/golang/go/blob/18e17e2cb12837ea2c8582ecdb0cc780f49a1aac/src/cmd/compile/internal/ssa/_gen/ARM64Ops.go#L13-L15 + tmpRegVReg = regalloc.FromRealReg(tmp, regalloc.RegTypeInt) + v28VReg = regalloc.FromRealReg(v28, regalloc.RegTypeFloat) + v29VReg = regalloc.FromRealReg(v29, regalloc.RegTypeFloat) + v30VReg = regalloc.FromRealReg(v30, regalloc.RegTypeFloat) + v31VReg = regalloc.FromRealReg(v31, regalloc.RegTypeFloat) + xzrVReg = regalloc.FromRealReg(xzr, regalloc.RegTypeInt) + spVReg = regalloc.FromRealReg(sp, regalloc.RegTypeInt) + fpVReg = regalloc.FromRealReg(fp, regalloc.RegTypeInt) +) + +var regNames = [...]string{ + x0: "x0", + x1: "x1", + x2: "x2", + x3: "x3", + x4: "x4", + x5: "x5", + x6: "x6", + x7: "x7", + x8: "x8", + x9: "x9", + x10: "x10", + x11: "x11", + x12: "x12", + x13: "x13", + x14: "x14", + x15: "x15", + x16: "x16", + x17: "x17", + x18: "x18", + x19: "x19", + x20: "x20", + x21: "x21", + x22: "x22", + x23: "x23", + x24: "x24", + x25: "x25", + x26: "x26", + x27: "x27", + x28: "x28", + x29: "x29", + x30: "x30", + xzr: "xzr", + sp: "sp", + v0: "v0", + v1: "v1", + v2: "v2", + v3: "v3", + v4: "v4", + v5: "v5", + v6: "v6", + v7: "v7", + v8: "v8", + v9: "v9", + v10: "v10", + v11: "v11", + v12: "v12", + v13: "v13", + v14: "v14", + v15: "v15", + v16: "v16", + v17: "v17", + v18: "v18", + v19: "v19", + v20: "v20", + v21: "v21", + v22: "v22", + v23: "v23", + v24: "v24", + v25: "v25", + v26: "v26", + v27: "v27", + v28: "v28", + v29: "v29", + v30: "v30", + v31: "v31", +} + +func formatVRegSized(r regalloc.VReg, size byte) (ret string) { + if r.IsRealReg() { + ret = regNames[r.RealReg()] + switch ret[0] { + case 'x': + switch size { + case 32: + ret = strings.Replace(ret, "x", "w", 1) + case 64: + default: + panic("BUG: invalid register size: " + strconv.Itoa(int(size))) + } + case 'v': + switch size { + case 32: + ret = strings.Replace(ret, "v", "s", 1) + case 64: + ret = strings.Replace(ret, "v", "d", 1) + case 128: + ret = strings.Replace(ret, "v", "q", 1) + default: + panic("BUG: invalid register size") + } + } + } else { + switch r.RegType() { + case regalloc.RegTypeInt: + switch size { + case 32: + ret = fmt.Sprintf("w%d?", r.ID()) + case 64: + ret = fmt.Sprintf("x%d?", r.ID()) + default: + panic("BUG: invalid register size: " + strconv.Itoa(int(size))) + } + case regalloc.RegTypeFloat: + switch size { + case 32: + ret = fmt.Sprintf("s%d?", r.ID()) + case 64: + ret = fmt.Sprintf("d%d?", r.ID()) + case 128: + ret = fmt.Sprintf("q%d?", r.ID()) + default: + panic("BUG: invalid register size") + } + default: + panic(fmt.Sprintf("BUG: invalid register type: %d for %s", r.RegType(), r)) + } + } + return +} + +func formatVRegWidthVec(r regalloc.VReg, width vecArrangement) (ret string) { + var id string + wspec := strings.ToLower(width.String()) + if r.IsRealReg() { + id = regNames[r.RealReg()][1:] + } else { + id = fmt.Sprintf("%d?", r.ID()) + } + ret = fmt.Sprintf("%s%s", wspec, id) + return +} + +func formatVRegVec(r regalloc.VReg, arr vecArrangement, index vecIndex) (ret string) { + id := fmt.Sprintf("v%d?", r.ID()) + if r.IsRealReg() { + id = regNames[r.RealReg()] + } + ret = fmt.Sprintf("%s.%s", id, strings.ToLower(arr.String())) + if index != vecIndexNone { + ret += fmt.Sprintf("[%d]", index) + } + return +} + +func regTypeToRegisterSizeInBits(r regalloc.RegType) byte { + switch r { + case regalloc.RegTypeInt: + return 64 + case regalloc.RegTypeFloat: + return 128 + default: + panic("BUG: invalid register type") + } +} + +var regNumberInEncoding = [...]uint32{ + x0: 0, + x1: 1, + x2: 2, + x3: 3, + x4: 4, + x5: 5, + x6: 6, + x7: 7, + x8: 8, + x9: 9, + x10: 10, + x11: 11, + x12: 12, + x13: 13, + x14: 14, + x15: 15, + x16: 16, + x17: 17, + x18: 18, + x19: 19, + x20: 20, + x21: 21, + x22: 22, + x23: 23, + x24: 24, + x25: 25, + x26: 26, + x27: 27, + x28: 28, + x29: 29, + x30: 30, + xzr: 31, + sp: 31, + v0: 0, + v1: 1, + v2: 2, + v3: 3, + v4: 4, + v5: 5, + v6: 6, + v7: 7, + v8: 8, + v9: 9, + v10: 10, + v11: 11, + v12: 12, + v13: 13, + v14: 14, + v15: 15, + v16: 16, + v17: 17, + v18: 18, + v19: 19, + v20: 20, + v21: 21, + v22: 22, + v23: 23, + v24: 24, + v25: 25, + v26: 26, + v27: 27, + v28: 28, + v29: 29, + v30: 30, + v31: 31, +} diff --git a/vendor/github.com/tetratelabs/wazero/internal/engine/wazevo/backend/isa/arm64/unwind_stack.go b/vendor/github.com/tetratelabs/wazero/internal/engine/wazevo/backend/isa/arm64/unwind_stack.go new file mode 100644 index 000000000..edb0e36e3 --- /dev/null +++ b/vendor/github.com/tetratelabs/wazero/internal/engine/wazevo/backend/isa/arm64/unwind_stack.go @@ -0,0 +1,90 @@ +package arm64 + +import ( + "encoding/binary" + "reflect" + "unsafe" + + "github.com/tetratelabs/wazero/internal/wasmdebug" +) + +// UnwindStack implements wazevo.unwindStack. +func UnwindStack(sp, _, top uintptr, returnAddresses []uintptr) []uintptr { + l := int(top - sp) + + var stackBuf []byte + { + // TODO: use unsafe.Slice after floor version is set to Go 1.20. + hdr := (*reflect.SliceHeader)(unsafe.Pointer(&stackBuf)) + hdr.Data = sp + hdr.Len = l + hdr.Cap = l + } + + for i := uint64(0); i < uint64(l); { + // (high address) + // +-----------------+ + // | ....... | + // | ret Y | <----+ + // | ....... | | + // | ret 0 | | + // | arg X | | size_of_arg_ret + // | ....... | | + // | arg 1 | | + // | arg 0 | <----+ + // | size_of_arg_ret | + // | ReturnAddress | + // +-----------------+ <----+ + // | ........... | | + // | spill slot M | | + // | ............ | | + // | spill slot 2 | | + // | spill slot 1 | | frame size + // | spill slot 1 | | + // | clobbered N | | + // | ............ | | + // | clobbered 0 | <----+ + // | xxxxxx | ;; unused space to make it 16-byte aligned. + // | frame_size | + // +-----------------+ <---- SP + // (low address) + + frameSize := binary.LittleEndian.Uint64(stackBuf[i:]) + i += frameSize + + 16 // frame size + aligned space. + retAddr := binary.LittleEndian.Uint64(stackBuf[i:]) + i += 8 // ret addr. + sizeOfArgRet := binary.LittleEndian.Uint64(stackBuf[i:]) + i += 8 + sizeOfArgRet + returnAddresses = append(returnAddresses, uintptr(retAddr)) + if len(returnAddresses) == wasmdebug.MaxFrames { + break + } + } + return returnAddresses +} + +// GoCallStackView implements wazevo.goCallStackView. +func GoCallStackView(stackPointerBeforeGoCall *uint64) []uint64 { + // (high address) + // +-----------------+ <----+ + // | xxxxxxxxxxx | | ;; optional unused space to make it 16-byte aligned. + // ^ | arg[N]/ret[M] | | + // sliceSize | | ............ | | sliceSize + // | | arg[1]/ret[1] | | + // v | arg[0]/ret[0] | <----+ + // | sliceSize | + // | frame_size | + // +-----------------+ <---- stackPointerBeforeGoCall + // (low address) + ptr := unsafe.Pointer(stackPointerBeforeGoCall) + size := *(*uint64)(unsafe.Add(ptr, 8)) + var view []uint64 + { + sh := (*reflect.SliceHeader)(unsafe.Pointer(&view)) + sh.Data = uintptr(unsafe.Add(ptr, 16)) // skips the (frame_size, sliceSize). + sh.Len = int(size) + sh.Cap = int(size) + } + return view +} diff --git a/vendor/github.com/tetratelabs/wazero/internal/engine/wazevo/backend/machine.go b/vendor/github.com/tetratelabs/wazero/internal/engine/wazevo/backend/machine.go new file mode 100644 index 000000000..54ce89e46 --- /dev/null +++ b/vendor/github.com/tetratelabs/wazero/internal/engine/wazevo/backend/machine.go @@ -0,0 +1,100 @@ +package backend + +import ( + "context" + + "github.com/tetratelabs/wazero/internal/engine/wazevo/backend/regalloc" + "github.com/tetratelabs/wazero/internal/engine/wazevo/ssa" + "github.com/tetratelabs/wazero/internal/engine/wazevo/wazevoapi" +) + +type ( + // Machine is a backend for a specific ISA machine. + Machine interface { + ExecutableContext() ExecutableContext + + // DisableStackCheck disables the stack check for the current compilation for debugging/testing. + DisableStackCheck() + + // SetCurrentABI initializes the FunctionABI for the given signature. + SetCurrentABI(abi *FunctionABI) + + // SetCompiler sets the compilation context used for the lifetime of Machine. + // This is only called once per Machine, i.e. before the first compilation. + SetCompiler(Compiler) + + // LowerSingleBranch is called when the compilation of the given single branch is started. + LowerSingleBranch(b *ssa.Instruction) + + // LowerConditionalBranch is called when the compilation of the given conditional branch is started. + LowerConditionalBranch(b *ssa.Instruction) + + // LowerInstr is called for each instruction in the given block except for the ones marked as already lowered + // via Compiler.MarkLowered. The order is reverse, i.e. from the last instruction to the first one. + // + // Note: this can lower multiple instructions (which produce the inputs) at once whenever it's possible + // for optimization. + LowerInstr(*ssa.Instruction) + + // Reset resets the machine state for the next compilation. + Reset() + + // InsertMove inserts a move instruction from src to dst whose type is typ. + InsertMove(dst, src regalloc.VReg, typ ssa.Type) + + // InsertReturn inserts the return instruction to return from the current function. + InsertReturn() + + // InsertLoadConstantBlockArg inserts the instruction(s) to load the constant value into the given regalloc.VReg. + InsertLoadConstantBlockArg(instr *ssa.Instruction, vr regalloc.VReg) + + // Format returns the string representation of the currently compiled machine code. + // This is only for testing purpose. + Format() string + + // RegAlloc does the register allocation after lowering. + RegAlloc() + + // PostRegAlloc does the post register allocation, e.g. setting up prologue/epilogue, redundant move elimination, etc. + PostRegAlloc() + + // ResolveRelocations resolves the relocations after emitting machine code. + // * refToBinaryOffset: the map from the function reference (ssa.FuncRef) to the executable offset. + // * executable: the binary to resolve the relocations. + // * relocations: the relocations to resolve. + // * callTrampolineIslandOffsets: the offsets of the trampoline islands in the executable. + ResolveRelocations( + refToBinaryOffset []int, + executable []byte, + relocations []RelocationInfo, + callTrampolineIslandOffsets []int, + ) + + // Encode encodes the machine instructions to the Compiler. + Encode(ctx context.Context) error + + // CompileGoFunctionTrampoline compiles the trampoline function to call a Go function of the given exit code and signature. + CompileGoFunctionTrampoline(exitCode wazevoapi.ExitCode, sig *ssa.Signature, needModuleContextPtr bool) []byte + + // CompileStackGrowCallSequence returns the sequence of instructions shared by all functions to + // call the stack grow builtin function. + CompileStackGrowCallSequence() []byte + + // CompileEntryPreamble returns the sequence of instructions shared by multiple functions to + // enter the function from Go. + CompileEntryPreamble(signature *ssa.Signature) []byte + + // LowerParams lowers the given parameters. + LowerParams(params []ssa.Value) + + // LowerReturns lowers the given returns. + LowerReturns(returns []ssa.Value) + + // ArgsResultsRegs returns the registers used for arguments and return values. + ArgsResultsRegs() (argResultInts, argResultFloats []regalloc.RealReg) + + // CallTrampolineIslandInfo returns the interval of the offset where the trampoline island is placed, and + // the size of the trampoline island. If islandSize is zero, the trampoline island is not used on this machine. + CallTrampolineIslandInfo(numFunctions int) (interval, islandSize int, err error) + } +) diff --git a/vendor/github.com/tetratelabs/wazero/internal/engine/wazevo/backend/regalloc.go b/vendor/github.com/tetratelabs/wazero/internal/engine/wazevo/backend/regalloc.go new file mode 100644 index 000000000..3f36c84e5 --- /dev/null +++ b/vendor/github.com/tetratelabs/wazero/internal/engine/wazevo/backend/regalloc.go @@ -0,0 +1,319 @@ +package backend + +import ( + "github.com/tetratelabs/wazero/internal/engine/wazevo/backend/regalloc" + "github.com/tetratelabs/wazero/internal/engine/wazevo/ssa" +) + +// RegAllocFunctionMachine is the interface for the machine specific logic that will be used in RegAllocFunction. +type RegAllocFunctionMachine[I regalloc.InstrConstraint] interface { + // InsertMoveBefore inserts the move instruction from src to dst before the given instruction. + InsertMoveBefore(dst, src regalloc.VReg, instr I) + // InsertStoreRegisterAt inserts the instruction(s) to store the given virtual register at the given instruction. + // If after is true, the instruction(s) will be inserted after the given instruction, otherwise before. + InsertStoreRegisterAt(v regalloc.VReg, instr I, after bool) I + // InsertReloadRegisterAt inserts the instruction(s) to reload the given virtual register at the given instruction. + // If after is true, the instruction(s) will be inserted after the given instruction, otherwise before. + InsertReloadRegisterAt(v regalloc.VReg, instr I, after bool) I + // ClobberedRegisters is called when the register allocation is done and the clobbered registers are known. + ClobberedRegisters(regs []regalloc.VReg) + // Swap swaps the two virtual registers after the given instruction. + Swap(cur I, x1, x2, tmp regalloc.VReg) + // LastInstrForInsertion implements LastInstrForInsertion of regalloc.Function. See its comment for details. + LastInstrForInsertion(begin, end I) I + // SSABlockLabel returns the label of the given ssa.BasicBlockID. + SSABlockLabel(id ssa.BasicBlockID) Label +} + +type ( + // RegAllocFunction implements regalloc.Function. + RegAllocFunction[I regalloc.InstrConstraint, m RegAllocFunctionMachine[I]] struct { + m m + ssb ssa.Builder + c Compiler + // iter is the iterator for reversePostOrderBlocks + iter int + reversePostOrderBlocks []RegAllocBlock[I, m] + // labelToRegAllocBlockIndex maps label to the index of reversePostOrderBlocks. + labelToRegAllocBlockIndex map[Label]int + loopNestingForestRoots []ssa.BasicBlock + } + + // RegAllocBlock implements regalloc.Block. + RegAllocBlock[I regalloc.InstrConstraint, m RegAllocFunctionMachine[I]] struct { + // f is the function this instruction belongs to. Used to reuse the regAllocFunctionImpl.predsSlice slice for Defs() and Uses(). + f *RegAllocFunction[I, m] + sb ssa.BasicBlock + l Label + begin, end I + loopNestingForestChildren []ssa.BasicBlock + cur I + id int + cachedLastInstrForInsertion I + } +) + +// NewRegAllocFunction returns a new RegAllocFunction. +func NewRegAllocFunction[I regalloc.InstrConstraint, M RegAllocFunctionMachine[I]](m M, ssb ssa.Builder, c Compiler) *RegAllocFunction[I, M] { + return &RegAllocFunction[I, M]{ + m: m, + ssb: ssb, + c: c, + labelToRegAllocBlockIndex: make(map[Label]int), + } +} + +// AddBlock adds a new block to the function. +func (f *RegAllocFunction[I, M]) AddBlock(sb ssa.BasicBlock, l Label, begin, end I) { + i := len(f.reversePostOrderBlocks) + f.reversePostOrderBlocks = append(f.reversePostOrderBlocks, RegAllocBlock[I, M]{ + f: f, + sb: sb, + l: l, + begin: begin, + end: end, + id: int(sb.ID()), + }) + f.labelToRegAllocBlockIndex[l] = i +} + +// Reset resets the function for the next compilation. +func (f *RegAllocFunction[I, M]) Reset() { + f.reversePostOrderBlocks = f.reversePostOrderBlocks[:0] + f.iter = 0 +} + +// StoreRegisterAfter implements regalloc.Function StoreRegisterAfter. +func (f *RegAllocFunction[I, M]) StoreRegisterAfter(v regalloc.VReg, instr regalloc.Instr) { + m := f.m + m.InsertStoreRegisterAt(v, instr.(I), true) +} + +// ReloadRegisterBefore implements regalloc.Function ReloadRegisterBefore. +func (f *RegAllocFunction[I, M]) ReloadRegisterBefore(v regalloc.VReg, instr regalloc.Instr) { + m := f.m + m.InsertReloadRegisterAt(v, instr.(I), false) +} + +// ReloadRegisterAfter implements regalloc.Function ReloadRegisterAfter. +func (f *RegAllocFunction[I, M]) ReloadRegisterAfter(v regalloc.VReg, instr regalloc.Instr) { + m := f.m + m.InsertReloadRegisterAt(v, instr.(I), true) +} + +// StoreRegisterBefore implements regalloc.Function StoreRegisterBefore. +func (f *RegAllocFunction[I, M]) StoreRegisterBefore(v regalloc.VReg, instr regalloc.Instr) { + m := f.m + m.InsertStoreRegisterAt(v, instr.(I), false) +} + +// ClobberedRegisters implements regalloc.Function ClobberedRegisters. +func (f *RegAllocFunction[I, M]) ClobberedRegisters(regs []regalloc.VReg) { + f.m.ClobberedRegisters(regs) +} + +// SwapBefore implements regalloc.Function SwapBefore. +func (f *RegAllocFunction[I, M]) SwapBefore(x1, x2, tmp regalloc.VReg, instr regalloc.Instr) { + f.m.Swap(instr.Prev().(I), x1, x2, tmp) +} + +// PostOrderBlockIteratorBegin implements regalloc.Function PostOrderBlockIteratorBegin. +func (f *RegAllocFunction[I, M]) PostOrderBlockIteratorBegin() regalloc.Block { + f.iter = len(f.reversePostOrderBlocks) - 1 + return f.PostOrderBlockIteratorNext() +} + +// PostOrderBlockIteratorNext implements regalloc.Function PostOrderBlockIteratorNext. +func (f *RegAllocFunction[I, M]) PostOrderBlockIteratorNext() regalloc.Block { + if f.iter < 0 { + return nil + } + b := &f.reversePostOrderBlocks[f.iter] + f.iter-- + return b +} + +// ReversePostOrderBlockIteratorBegin implements regalloc.Function ReversePostOrderBlockIteratorBegin. +func (f *RegAllocFunction[I, M]) ReversePostOrderBlockIteratorBegin() regalloc.Block { + f.iter = 0 + return f.ReversePostOrderBlockIteratorNext() +} + +// ReversePostOrderBlockIteratorNext implements regalloc.Function ReversePostOrderBlockIteratorNext. +func (f *RegAllocFunction[I, M]) ReversePostOrderBlockIteratorNext() regalloc.Block { + if f.iter >= len(f.reversePostOrderBlocks) { + return nil + } + b := &f.reversePostOrderBlocks[f.iter] + f.iter++ + return b +} + +// LoopNestingForestRoots implements regalloc.Function LoopNestingForestRoots. +func (f *RegAllocFunction[I, M]) LoopNestingForestRoots() int { + f.loopNestingForestRoots = f.ssb.LoopNestingForestRoots() + return len(f.loopNestingForestRoots) +} + +// LoopNestingForestRoot implements regalloc.Function LoopNestingForestRoot. +func (f *RegAllocFunction[I, M]) LoopNestingForestRoot(i int) regalloc.Block { + blk := f.loopNestingForestRoots[i] + l := f.m.SSABlockLabel(blk.ID()) + index := f.labelToRegAllocBlockIndex[l] + return &f.reversePostOrderBlocks[index] +} + +// InsertMoveBefore implements regalloc.Function InsertMoveBefore. +func (f *RegAllocFunction[I, M]) InsertMoveBefore(dst, src regalloc.VReg, instr regalloc.Instr) { + f.m.InsertMoveBefore(dst, src, instr.(I)) +} + +// LowestCommonAncestor implements regalloc.Function LowestCommonAncestor. +func (f *RegAllocFunction[I, M]) LowestCommonAncestor(blk1, blk2 regalloc.Block) regalloc.Block { + ret := f.ssb.LowestCommonAncestor(blk1.(*RegAllocBlock[I, M]).sb, blk2.(*RegAllocBlock[I, M]).sb) + l := f.m.SSABlockLabel(ret.ID()) + index := f.labelToRegAllocBlockIndex[l] + return &f.reversePostOrderBlocks[index] +} + +// Idom implements regalloc.Function Idom. +func (f *RegAllocFunction[I, M]) Idom(blk regalloc.Block) regalloc.Block { + builder := f.ssb + idom := builder.Idom(blk.(*RegAllocBlock[I, M]).sb) + if idom == nil { + panic("BUG: idom must not be nil") + } + l := f.m.SSABlockLabel(idom.ID()) + index := f.labelToRegAllocBlockIndex[l] + return &f.reversePostOrderBlocks[index] +} + +// ID implements regalloc.Block. +func (r *RegAllocBlock[I, m]) ID() int32 { return int32(r.id) } + +// BlockParams implements regalloc.Block. +func (r *RegAllocBlock[I, m]) BlockParams(regs *[]regalloc.VReg) []regalloc.VReg { + c := r.f.c + *regs = (*regs)[:0] + for i := 0; i < r.sb.Params(); i++ { + v := c.VRegOf(r.sb.Param(i)) + *regs = append(*regs, v) + } + return *regs +} + +// InstrIteratorBegin implements regalloc.Block. +func (r *RegAllocBlock[I, m]) InstrIteratorBegin() regalloc.Instr { + r.cur = r.begin + return r.cur +} + +// InstrIteratorNext implements regalloc.Block. +func (r *RegAllocBlock[I, m]) InstrIteratorNext() regalloc.Instr { + for { + if r.cur == r.end { + return nil + } + instr := r.cur.Next() + r.cur = instr.(I) + if instr == nil { + return nil + } else if instr.AddedBeforeRegAlloc() { + // Only concerned about the instruction added before regalloc. + return instr + } + } +} + +// InstrRevIteratorBegin implements regalloc.Block. +func (r *RegAllocBlock[I, m]) InstrRevIteratorBegin() regalloc.Instr { + r.cur = r.end + return r.cur +} + +// InstrRevIteratorNext implements regalloc.Block. +func (r *RegAllocBlock[I, m]) InstrRevIteratorNext() regalloc.Instr { + for { + if r.cur == r.begin { + return nil + } + instr := r.cur.Prev() + r.cur = instr.(I) + if instr == nil { + return nil + } else if instr.AddedBeforeRegAlloc() { + // Only concerned about the instruction added before regalloc. + return instr + } + } +} + +// FirstInstr implements regalloc.Block. +func (r *RegAllocBlock[I, m]) FirstInstr() regalloc.Instr { + return r.begin +} + +// EndInstr implements regalloc.Block. +func (r *RegAllocBlock[I, m]) EndInstr() regalloc.Instr { + return r.end +} + +// LastInstrForInsertion implements regalloc.Block. +func (r *RegAllocBlock[I, m]) LastInstrForInsertion() regalloc.Instr { + var nil I + if r.cachedLastInstrForInsertion == nil { + r.cachedLastInstrForInsertion = r.f.m.LastInstrForInsertion(r.begin, r.end) + } + return r.cachedLastInstrForInsertion +} + +// Preds implements regalloc.Block. +func (r *RegAllocBlock[I, m]) Preds() int { return r.sb.Preds() } + +// Pred implements regalloc.Block. +func (r *RegAllocBlock[I, m]) Pred(i int) regalloc.Block { + sb := r.sb + pred := sb.Pred(i) + l := r.f.m.SSABlockLabel(pred.ID()) + index := r.f.labelToRegAllocBlockIndex[l] + return &r.f.reversePostOrderBlocks[index] +} + +// Entry implements regalloc.Block. +func (r *RegAllocBlock[I, m]) Entry() bool { return r.sb.EntryBlock() } + +// Succs implements regalloc.Block. +func (r *RegAllocBlock[I, m]) Succs() int { + return r.sb.Succs() +} + +// Succ implements regalloc.Block. +func (r *RegAllocBlock[I, m]) Succ(i int) regalloc.Block { + sb := r.sb + succ := sb.Succ(i) + if succ.ReturnBlock() { + return nil + } + l := r.f.m.SSABlockLabel(succ.ID()) + index := r.f.labelToRegAllocBlockIndex[l] + return &r.f.reversePostOrderBlocks[index] +} + +// LoopHeader implements regalloc.Block. +func (r *RegAllocBlock[I, m]) LoopHeader() bool { + return r.sb.LoopHeader() +} + +// LoopNestingForestChildren implements regalloc.Block. +func (r *RegAllocBlock[I, m]) LoopNestingForestChildren() int { + r.loopNestingForestChildren = r.sb.LoopNestingForestChildren() + return len(r.loopNestingForestChildren) +} + +// LoopNestingForestChild implements regalloc.Block. +func (r *RegAllocBlock[I, m]) LoopNestingForestChild(i int) regalloc.Block { + blk := r.loopNestingForestChildren[i] + l := r.f.m.SSABlockLabel(blk.ID()) + index := r.f.labelToRegAllocBlockIndex[l] + return &r.f.reversePostOrderBlocks[index] +} diff --git a/vendor/github.com/tetratelabs/wazero/internal/engine/wazevo/backend/regalloc/api.go b/vendor/github.com/tetratelabs/wazero/internal/engine/wazevo/backend/regalloc/api.go new file mode 100644 index 000000000..23157b478 --- /dev/null +++ b/vendor/github.com/tetratelabs/wazero/internal/engine/wazevo/backend/regalloc/api.go @@ -0,0 +1,136 @@ +package regalloc + +import "fmt" + +// These interfaces are implemented by ISA-specific backends to abstract away the details, and allow the register +// allocators to work on any ISA. +// +// TODO: the interfaces are not stabilized yet, especially x64 will need some changes. E.g. x64 has an addressing mode +// where index can be in memory. That kind of info will be useful to reduce the register pressure, and should be leveraged +// by the register allocators, like https://docs.rs/regalloc2/latest/regalloc2/enum.OperandConstraint.html + +type ( + // Function is the top-level interface to do register allocation, which corresponds to a CFG containing + // Blocks(s). + Function interface { + // PostOrderBlockIteratorBegin returns the first block in the post-order traversal of the CFG. + // In other words, the last blocks in the CFG will be returned first. + PostOrderBlockIteratorBegin() Block + // PostOrderBlockIteratorNext returns the next block in the post-order traversal of the CFG. + PostOrderBlockIteratorNext() Block + // ReversePostOrderBlockIteratorBegin returns the first block in the reverse post-order traversal of the CFG. + // In other words, the first blocks in the CFG will be returned first. + ReversePostOrderBlockIteratorBegin() Block + // ReversePostOrderBlockIteratorNext returns the next block in the reverse post-order traversal of the CFG. + ReversePostOrderBlockIteratorNext() Block + // ClobberedRegisters tell the clobbered registers by this function. + ClobberedRegisters([]VReg) + // LoopNestingForestRoots returns the number of roots of the loop nesting forest in a function. + LoopNestingForestRoots() int + // LoopNestingForestRoot returns the i-th root of the loop nesting forest in a function. + LoopNestingForestRoot(i int) Block + // LowestCommonAncestor returns the lowest common ancestor of two blocks in the dominator tree. + LowestCommonAncestor(blk1, blk2 Block) Block + // Idom returns the immediate dominator of the given block. + Idom(blk Block) Block + + // Followings are for rewriting the function. + + // SwapAtEndOfBlock swaps the two virtual registers at the end of the given block. + SwapBefore(x1, x2, tmp VReg, instr Instr) + // StoreRegisterBefore inserts store instruction(s) before the given instruction for the given virtual register. + StoreRegisterBefore(v VReg, instr Instr) + // StoreRegisterAfter inserts store instruction(s) after the given instruction for the given virtual register. + StoreRegisterAfter(v VReg, instr Instr) + // ReloadRegisterBefore inserts reload instruction(s) before the given instruction for the given virtual register. + ReloadRegisterBefore(v VReg, instr Instr) + // ReloadRegisterAfter inserts reload instruction(s) after the given instruction for the given virtual register. + ReloadRegisterAfter(v VReg, instr Instr) + // InsertMoveBefore inserts move instruction(s) before the given instruction for the given virtual registers. + InsertMoveBefore(dst, src VReg, instr Instr) + } + + // Block is a basic block in the CFG of a function, and it consists of multiple instructions, and predecessor Block(s). + Block interface { + // ID returns the unique identifier of this block which is ordered in the reverse post-order traversal of the CFG. + ID() int32 + // BlockParams returns the virtual registers used as the parameters of this block. + BlockParams(*[]VReg) []VReg + // InstrIteratorBegin returns the first instruction in this block. Instructions added after lowering must be skipped. + // Note: multiple Instr(s) will not be held at the same time, so it's safe to use the same impl for the return Instr. + InstrIteratorBegin() Instr + // InstrIteratorNext returns the next instruction in this block. Instructions added after lowering must be skipped. + // Note: multiple Instr(s) will not be held at the same time, so it's safe to use the same impl for the return Instr. + InstrIteratorNext() Instr + // InstrRevIteratorBegin is the same as InstrIteratorBegin, but in the reverse order. + InstrRevIteratorBegin() Instr + // InstrRevIteratorNext is the same as InstrIteratorNext, but in the reverse order. + InstrRevIteratorNext() Instr + // FirstInstr returns the fist instruction in this block where instructions will be inserted after it. + FirstInstr() Instr + // EndInstr returns the end instruction in this block. + EndInstr() Instr + // LastInstrForInsertion returns the last instruction in this block where instructions will be inserted before it. + // Such insertions only happen when we need to insert spill/reload instructions to adjust the merge edges. + // At the time of register allocation, all the critical edges are already split, so there is no need + // to worry about the case where branching instruction has multiple successors. + // Therefore, usually, it is the nop instruction, but if the block ends with an unconditional branching, then it returns + // the unconditional branch, not the nop. In other words it is either nop or unconditional branch. + LastInstrForInsertion() Instr + // Preds returns the number of predecessors of this block in the CFG. + Preds() int + // Pred returns the i-th predecessor of this block in the CFG. + Pred(i int) Block + // Entry returns true if the block is for the entry block. + Entry() bool + // Succs returns the number of successors of this block in the CFG. + Succs() int + // Succ returns the i-th successor of this block in the CFG. + Succ(i int) Block + // LoopHeader returns true if this block is a loop header. + LoopHeader() bool + // LoopNestingForestChildren returns the number of children of this block in the loop nesting forest. + LoopNestingForestChildren() int + // LoopNestingForestChild returns the i-th child of this block in the loop nesting forest. + LoopNestingForestChild(i int) Block + } + + // Instr is an instruction in a block, abstracting away the underlying ISA. + Instr interface { + fmt.Stringer + // Next returns the next instruction in the same block. + Next() Instr + // Prev returns the previous instruction in the same block. + Prev() Instr + // Defs returns the virtual registers defined by this instruction. + Defs(*[]VReg) []VReg + // Uses returns the virtual registers used by this instruction. + // Note: multiple returned []VReg will not be held at the same time, so it's safe to use the same slice for this. + Uses(*[]VReg) []VReg + // AssignUse assigns the RealReg-allocated virtual register used by this instruction at the given index. + AssignUse(index int, v VReg) + // AssignDef assigns a RealReg-allocated virtual register defined by this instruction. + // This only accepts one register because we don't allocate registers for multi-def instructions (i.e. call instruction) + AssignDef(VReg) + // IsCopy returns true if this instruction is a move instruction between two registers. + // If true, the instruction is of the form of dst = src, and if the src and dst do not interfere with each other, + // we could coalesce them, and hence the copy can be eliminated from the final code. + IsCopy() bool + // IsCall returns true if this instruction is a call instruction. The result is used to insert + // caller saved register spills and restores. + IsCall() bool + // IsIndirectCall returns true if this instruction is an indirect call instruction which calls a function pointer. + // The result is used to insert caller saved register spills and restores. + IsIndirectCall() bool + // IsReturn returns true if this instruction is a return instruction. + IsReturn() bool + // AddedBeforeRegAlloc returns true if this instruction is added before register allocation. + AddedBeforeRegAlloc() bool + } + + // InstrConstraint is an interface for arch-specific instruction constraints. + InstrConstraint interface { + comparable + Instr + } +) diff --git a/vendor/github.com/tetratelabs/wazero/internal/engine/wazevo/backend/regalloc/reg.go b/vendor/github.com/tetratelabs/wazero/internal/engine/wazevo/backend/regalloc/reg.go new file mode 100644 index 000000000..46df807e6 --- /dev/null +++ b/vendor/github.com/tetratelabs/wazero/internal/engine/wazevo/backend/regalloc/reg.go @@ -0,0 +1,123 @@ +package regalloc + +import ( + "fmt" + + "github.com/tetratelabs/wazero/internal/engine/wazevo/ssa" +) + +// VReg represents a register which is assigned to an SSA value. This is used to represent a register in the backend. +// A VReg may or may not be a physical register, and the info of physical register can be obtained by RealReg. +type VReg uint64 + +// VRegID is the lower 32bit of VReg, which is the pure identifier of VReg without RealReg info. +type VRegID uint32 + +// RealReg returns the RealReg of this VReg. +func (v VReg) RealReg() RealReg { + return RealReg(v >> 32) +} + +// IsRealReg returns true if this VReg is backed by a physical register. +func (v VReg) IsRealReg() bool { + return v.RealReg() != RealRegInvalid +} + +// FromRealReg returns a VReg from the given RealReg and RegType. +// This is used to represent a specific pre-colored register in the backend. +func FromRealReg(r RealReg, typ RegType) VReg { + rid := VRegID(r) + if rid > vRegIDReservedForRealNum { + panic(fmt.Sprintf("invalid real reg %d", r)) + } + return VReg(r).SetRealReg(r).SetRegType(typ) +} + +// SetRealReg sets the RealReg of this VReg and returns the updated VReg. +func (v VReg) SetRealReg(r RealReg) VReg { + return VReg(r)<<32 | (v & 0xff_00_ffffffff) +} + +// RegType returns the RegType of this VReg. +func (v VReg) RegType() RegType { + return RegType(v >> 40) +} + +// SetRegType sets the RegType of this VReg and returns the updated VReg. +func (v VReg) SetRegType(t RegType) VReg { + return VReg(t)<<40 | (v & 0x00_ff_ffffffff) +} + +// ID returns the VRegID of this VReg. +func (v VReg) ID() VRegID { + return VRegID(v & 0xffffffff) +} + +// Valid returns true if this VReg is Valid. +func (v VReg) Valid() bool { + return v.ID() != vRegIDInvalid && v.RegType() != RegTypeInvalid +} + +// RealReg represents a physical register. +type RealReg byte + +const RealRegInvalid RealReg = 0 + +const ( + vRegIDInvalid VRegID = 1 << 31 + VRegIDNonReservedBegin = vRegIDReservedForRealNum + vRegIDReservedForRealNum VRegID = 128 + VRegInvalid = VReg(vRegIDInvalid) +) + +// String implements fmt.Stringer. +func (r RealReg) String() string { + switch r { + case RealRegInvalid: + return "invalid" + default: + return fmt.Sprintf("r%d", r) + } +} + +// String implements fmt.Stringer. +func (v VReg) String() string { + if v.IsRealReg() { + return fmt.Sprintf("r%d", v.ID()) + } + return fmt.Sprintf("v%d?", v.ID()) +} + +// RegType represents the type of a register. +type RegType byte + +const ( + RegTypeInvalid RegType = iota + RegTypeInt + RegTypeFloat + NumRegType +) + +// String implements fmt.Stringer. +func (r RegType) String() string { + switch r { + case RegTypeInt: + return "int" + case RegTypeFloat: + return "float" + default: + return "invalid" + } +} + +// RegTypeOf returns the RegType of the given ssa.Type. +func RegTypeOf(p ssa.Type) RegType { + switch p { + case ssa.TypeI32, ssa.TypeI64: + return RegTypeInt + case ssa.TypeF32, ssa.TypeF64, ssa.TypeV128: + return RegTypeFloat + default: + panic("invalid type") + } +} diff --git a/vendor/github.com/tetratelabs/wazero/internal/engine/wazevo/backend/regalloc/regalloc.go b/vendor/github.com/tetratelabs/wazero/internal/engine/wazevo/backend/regalloc/regalloc.go new file mode 100644 index 000000000..b4450d56f --- /dev/null +++ b/vendor/github.com/tetratelabs/wazero/internal/engine/wazevo/backend/regalloc/regalloc.go @@ -0,0 +1,1212 @@ +// Package regalloc performs register allocation. The algorithm can work on any ISA by implementing the interfaces in +// api.go. +// +// References: +// - https://web.stanford.edu/class/archive/cs/cs143/cs143.1128/lectures/17/Slides17.pdf +// - https://en.wikipedia.org/wiki/Chaitin%27s_algorithm +// - https://llvm.org/ProjectsWithLLVM/2004-Fall-CS426-LS.pdf +// - https://pfalcon.github.io/ssabook/latest/book-full.pdf: Chapter 9. for liveness analysis. +// - https://github.com/golang/go/blob/release-branch.go1.21/src/cmd/compile/internal/ssa/regalloc.go +package regalloc + +import ( + "fmt" + "math" + "strings" + + "github.com/tetratelabs/wazero/internal/engine/wazevo/wazevoapi" +) + +// NewAllocator returns a new Allocator. +func NewAllocator(allocatableRegs *RegisterInfo) Allocator { + a := Allocator{ + regInfo: allocatableRegs, + phiDefInstListPool: wazevoapi.NewPool[phiDefInstList](resetPhiDefInstList), + blockStates: wazevoapi.NewIDedPool[blockState](resetBlockState), + } + a.state.vrStates = wazevoapi.NewIDedPool[vrState](resetVrState) + a.state.reset() + for _, regs := range allocatableRegs.AllocatableRegisters { + for _, r := range regs { + a.allocatableSet = a.allocatableSet.add(r) + } + } + return a +} + +type ( + // RegisterInfo holds the statically-known ISA-specific register information. + RegisterInfo struct { + // AllocatableRegisters is a 2D array of allocatable RealReg, indexed by regTypeNum and regNum. + // The order matters: the first element is the most preferred one when allocating. + AllocatableRegisters [NumRegType][]RealReg + CalleeSavedRegisters RegSet + CallerSavedRegisters RegSet + RealRegToVReg []VReg + // RealRegName returns the name of the given RealReg for debugging. + RealRegName func(r RealReg) string + RealRegType func(r RealReg) RegType + } + + // Allocator is a register allocator. + Allocator struct { + // regInfo is static per ABI/ISA, and is initialized by the machine during Machine.PrepareRegisterAllocator. + regInfo *RegisterInfo + // allocatableSet is a set of allocatable RealReg derived from regInfo. Static per ABI/ISA. + allocatableSet RegSet + allocatedCalleeSavedRegs []VReg + vs []VReg + vs2 []VRegID + phiDefInstListPool wazevoapi.Pool[phiDefInstList] + + // Followings are re-used during various places. + blks []Block + reals []RealReg + currentOccupants regInUseSet + + // Following two fields are updated while iterating the blocks in the reverse postorder. + state state + blockStates wazevoapi.IDedPool[blockState] + } + + // programCounter represents an opaque index into the program which is used to represents a LiveInterval of a VReg. + programCounter int32 + + state struct { + argRealRegs []VReg + regsInUse regInUseSet + vrStates wazevoapi.IDedPool[vrState] + + currentBlockID int32 + + // allocatedRegSet is a set of RealReg that are allocated during the allocation phase. This is reset per function. + allocatedRegSet RegSet + } + + blockState struct { + // liveIns is a list of VReg that are live at the beginning of the block. + liveIns []VRegID + // seen is true if the block is visited during the liveness analysis. + seen bool + // visited is true if the block is visited during the allocation phase. + visited bool + startFromPredIndex int + // startRegs is a list of RealReg that are used at the beginning of the block. This is used to fix the merge edges. + startRegs regInUseSet + // endRegs is a list of RealReg that are used at the end of the block. This is used to fix the merge edges. + endRegs regInUseSet + } + + vrState struct { + v VReg + r RealReg + // defInstr is the instruction that defines this value. If this is the phi value and not the entry block, this is nil. + defInstr Instr + // defBlk is the block that defines this value. If this is the phi value, this is the block whose arguments contain this value. + defBlk Block + // lca = lowest common ancestor. This is the block that is the lowest common ancestor of all the blocks that + // reloads this value. This is used to determine the spill location. Only valid if spilled=true. + lca Block + // lastUse is the program counter of the last use of this value. This changes while iterating the block, and + // should not be used across the blocks as it becomes invalid. To check the validity, use lastUseUpdatedAtBlockID. + lastUse programCounter + lastUseUpdatedAtBlockID int32 + // spilled is true if this value is spilled i.e. the value is reload from the stack somewhere in the program. + // + // Note that this field is used during liveness analysis for different purpose. This is used to determine the + // value is live-in or not. + spilled bool + // isPhi is true if this is a phi value. + isPhi bool + desiredLoc desiredLoc + // phiDefInstList is a list of instructions that defines this phi value. + // This is used to determine the spill location, and only valid if isPhi=true. + *phiDefInstList + } + + // phiDefInstList is a linked list of instructions that defines a phi value. + phiDefInstList struct { + instr Instr + v VReg + next *phiDefInstList + } + + // desiredLoc represents a desired location for a VReg. + desiredLoc uint16 + // desiredLocKind is a kind of desired location for a VReg. + desiredLocKind uint16 +) + +const ( + // desiredLocKindUnspecified is a kind of desired location for a VReg that is not specified. + desiredLocKindUnspecified desiredLocKind = iota + // desiredLocKindStack is a kind of desired location for a VReg that is on the stack, only used for the phi values. + desiredLocKindStack + // desiredLocKindReg is a kind of desired location for a VReg that is in a register. + desiredLocKindReg + desiredLocUnspecified = desiredLoc(desiredLocKindUnspecified) + desiredLocStack = desiredLoc(desiredLocKindStack) +) + +func newDesiredLocReg(r RealReg) desiredLoc { + return desiredLoc(desiredLocKindReg) | desiredLoc(r<<2) +} + +func (d desiredLoc) realReg() RealReg { + return RealReg(d >> 2) +} + +func (d desiredLoc) stack() bool { + return d&3 == desiredLoc(desiredLocKindStack) +} + +func resetPhiDefInstList(l *phiDefInstList) { + l.instr = nil + l.next = nil + l.v = VRegInvalid +} + +func (s *state) dump(info *RegisterInfo) { //nolint:unused + fmt.Println("\t\tstate:") + fmt.Println("\t\t\targRealRegs:", s.argRealRegs) + fmt.Println("\t\t\tregsInUse", s.regsInUse.format(info)) + fmt.Println("\t\t\tallocatedRegSet:", s.allocatedRegSet.format(info)) + fmt.Println("\t\t\tused:", s.regsInUse.format(info)) + var strs []string + for i := 0; i <= s.vrStates.MaxIDEncountered(); i++ { + vs := s.vrStates.Get(i) + if vs == nil { + continue + } + if vs.r != RealRegInvalid { + strs = append(strs, fmt.Sprintf("(v%d: %s)", vs.v.ID(), info.RealRegName(vs.r))) + } + } + fmt.Println("\t\t\tvrStates:", strings.Join(strs, ", ")) +} + +func (s *state) reset() { + s.argRealRegs = s.argRealRegs[:0] + s.vrStates.Reset() + s.allocatedRegSet = RegSet(0) + s.regsInUse.reset() + s.currentBlockID = -1 +} + +func (s *state) setVRegState(v VReg, r RealReg) { + id := int(v.ID()) + st := s.vrStates.GetOrAllocate(id) + st.r = r + st.v = v +} + +func resetVrState(vs *vrState) { + vs.v = VRegInvalid + vs.r = RealRegInvalid + vs.defInstr = nil + vs.defBlk = nil + vs.spilled = false + vs.lastUse = -1 + vs.lastUseUpdatedAtBlockID = -1 + vs.lca = nil + vs.isPhi = false + vs.phiDefInstList = nil + vs.desiredLoc = desiredLocUnspecified +} + +func (s *state) getVRegState(v VRegID) *vrState { + return s.vrStates.GetOrAllocate(int(v)) +} + +func (s *state) useRealReg(r RealReg, v VReg) { + if s.regsInUse.has(r) { + panic("BUG: useRealReg: the given real register is already used") + } + s.regsInUse.add(r, v) + s.setVRegState(v, r) + s.allocatedRegSet = s.allocatedRegSet.add(r) +} + +func (s *state) releaseRealReg(r RealReg) { + current := s.regsInUse.get(r) + if current.Valid() { + s.regsInUse.remove(r) + s.setVRegState(current, RealRegInvalid) + } +} + +// recordReload records that the given VReg is reloaded in the given block. +// This is used to determine the spill location by tracking the lowest common ancestor of all the blocks that reloads the value. +func (vs *vrState) recordReload(f Function, blk Block) { + vs.spilled = true + if vs.lca == nil { + if wazevoapi.RegAllocLoggingEnabled { + fmt.Printf("\t\tv%d is reloaded in blk%d,\n", vs.v.ID(), blk.ID()) + } + vs.lca = blk + } else { + if wazevoapi.RegAllocLoggingEnabled { + fmt.Printf("\t\tv%d is reloaded in blk%d, lca=%d\n", vs.v.ID(), blk.ID(), vs.lca.ID()) + } + vs.lca = f.LowestCommonAncestor(vs.lca, blk) + if wazevoapi.RegAllocLoggingEnabled { + fmt.Printf("updated lca=%d\n", vs.lca.ID()) + } + } +} + +func (s *state) findOrSpillAllocatable(a *Allocator, allocatable []RealReg, forbiddenMask RegSet, preferred RealReg) (r RealReg) { + r = RealRegInvalid + // First, check if the preferredMask has any allocatable register. + if preferred != RealRegInvalid && !forbiddenMask.has(preferred) && !s.regsInUse.has(preferred) { + for _, candidateReal := range allocatable { + // TODO: we should ensure the preferred register is in the allocatable set in the first place, + // but right now, just in case, we check it here. + if candidateReal == preferred { + return preferred + } + } + } + + var lastUseAt programCounter + var spillVReg VReg + for _, candidateReal := range allocatable { + if forbiddenMask.has(candidateReal) { + continue + } + + using := s.regsInUse.get(candidateReal) + if using == VRegInvalid { + // This is not used at this point. + return candidateReal + } + + // Real registers in use should not be spilled, so we skip them. + // For example, if the register is used as an argument register, and it might be + // spilled and not reloaded when it ends up being used as a temporary to pass + // stack based argument. + if using.IsRealReg() { + continue + } + + isPreferred := candidateReal == preferred + + // last == -1 means the value won't be used anymore. + if last := s.getVRegState(using.ID()).lastUse; r == RealRegInvalid || isPreferred || last == -1 || (lastUseAt != -1 && last > lastUseAt) { + lastUseAt = last + r = candidateReal + spillVReg = using + if isPreferred { + break + } + } + } + + if r == RealRegInvalid { + panic("not found any allocatable register") + } + + if wazevoapi.RegAllocLoggingEnabled { + fmt.Printf("\tspilling v%d when lastUseAt=%d and regsInUse=%s\n", spillVReg.ID(), lastUseAt, s.regsInUse.format(a.regInfo)) + } + s.releaseRealReg(r) + return r +} + +func (s *state) findAllocatable(allocatable []RealReg, forbiddenMask RegSet) RealReg { + for _, r := range allocatable { + if !s.regsInUse.has(r) && !forbiddenMask.has(r) { + return r + } + } + return RealRegInvalid +} + +func (s *state) resetAt(bs *blockState) { + s.regsInUse.range_(func(_ RealReg, vr VReg) { + s.setVRegState(vr, RealRegInvalid) + }) + s.regsInUse.reset() + bs.endRegs.range_(func(r RealReg, v VReg) { + id := int(v.ID()) + st := s.vrStates.GetOrAllocate(id) + if st.lastUseUpdatedAtBlockID == s.currentBlockID && st.lastUse == programCounterLiveIn { + s.regsInUse.add(r, v) + s.setVRegState(v, r) + } + }) +} + +func resetBlockState(b *blockState) { + b.seen = false + b.visited = false + b.endRegs.reset() + b.startRegs.reset() + b.startFromPredIndex = -1 + b.liveIns = b.liveIns[:0] +} + +func (b *blockState) dump(a *RegisterInfo) { + fmt.Println("\t\tblockState:") + fmt.Println("\t\t\tstartRegs:", b.startRegs.format(a)) + fmt.Println("\t\t\tendRegs:", b.endRegs.format(a)) + fmt.Println("\t\t\tstartFromPredIndex:", b.startFromPredIndex) + fmt.Println("\t\t\tvisited:", b.visited) +} + +// DoAllocation performs register allocation on the given Function. +func (a *Allocator) DoAllocation(f Function) { + a.livenessAnalysis(f) + a.alloc(f) + a.determineCalleeSavedRealRegs(f) +} + +func (a *Allocator) determineCalleeSavedRealRegs(f Function) { + a.allocatedCalleeSavedRegs = a.allocatedCalleeSavedRegs[:0] + a.state.allocatedRegSet.Range(func(allocatedRealReg RealReg) { + if a.regInfo.CalleeSavedRegisters.has(allocatedRealReg) { + a.allocatedCalleeSavedRegs = append(a.allocatedCalleeSavedRegs, a.regInfo.RealRegToVReg[allocatedRealReg]) + } + }) + f.ClobberedRegisters(a.allocatedCalleeSavedRegs) +} + +func (a *Allocator) getOrAllocateBlockState(blockID int32) *blockState { + return a.blockStates.GetOrAllocate(int(blockID)) +} + +// phiBlk returns the block that defines the given phi value, nil otherwise. +func (s *state) phiBlk(v VRegID) Block { + vs := s.getVRegState(v) + if vs.isPhi { + return vs.defBlk + } + return nil +} + +const ( + programCounterLiveIn = math.MinInt32 + programCounterLiveOut = math.MaxInt32 +) + +// liveAnalysis constructs Allocator.blockLivenessData. +// The algorithm here is described in https://pfalcon.github.io/ssabook/latest/book-full.pdf Chapter 9.2. +func (a *Allocator) livenessAnalysis(f Function) { + s := &a.state + for blk := f.PostOrderBlockIteratorBegin(); blk != nil; blk = f.PostOrderBlockIteratorNext() { // Order doesn't matter. + + // We should gather phi value data. + for _, p := range blk.BlockParams(&a.vs) { + vs := s.getVRegState(p.ID()) + vs.isPhi = true + vs.defBlk = blk + } + } + + for blk := f.PostOrderBlockIteratorBegin(); blk != nil; blk = f.PostOrderBlockIteratorNext() { + blkID := blk.ID() + info := a.getOrAllocateBlockState(blkID) + + a.vs2 = a.vs2[:0] + const ( + flagDeleted = false + flagLive = true + ) + ns := blk.Succs() + for i := 0; i < ns; i++ { + succ := blk.Succ(i) + if succ == nil { + continue + } + + succID := succ.ID() + succInfo := a.getOrAllocateBlockState(succID) + if !succInfo.seen { // This means the back edge. + continue + } + + for _, v := range succInfo.liveIns { + if s.phiBlk(v) != succ { + st := s.getVRegState(v) + // We use .spilled field to store the flag. + st.spilled = flagLive + a.vs2 = append(a.vs2, v) + } + } + } + + for instr := blk.InstrRevIteratorBegin(); instr != nil; instr = blk.InstrRevIteratorNext() { + + var use, def VReg + for _, def = range instr.Defs(&a.vs) { + if !def.IsRealReg() { + id := def.ID() + st := s.getVRegState(id) + // We use .spilled field to store the flag. + st.spilled = flagDeleted + a.vs2 = append(a.vs2, id) + } + } + for _, use = range instr.Uses(&a.vs) { + if !use.IsRealReg() { + id := use.ID() + st := s.getVRegState(id) + // We use .spilled field to store the flag. + st.spilled = flagLive + a.vs2 = append(a.vs2, id) + } + } + + if def.Valid() && s.phiBlk(def.ID()) != nil { + if use.Valid() && use.IsRealReg() { + // If the destination is a phi value, and the source is a real register, this is the beginning of the function. + a.state.argRealRegs = append(a.state.argRealRegs, use) + } + } + } + + for _, v := range a.vs2 { + st := s.getVRegState(v) + // We use .spilled field to store the flag. + if st.spilled == flagLive { //nolint:gosimple + info.liveIns = append(info.liveIns, v) + st.spilled = false + } + } + + info.seen = true + } + + nrs := f.LoopNestingForestRoots() + for i := 0; i < nrs; i++ { + root := f.LoopNestingForestRoot(i) + a.loopTreeDFS(root) + } +} + +// loopTreeDFS implements the Algorithm 9.3 in the book in an iterative way. +func (a *Allocator) loopTreeDFS(entry Block) { + a.blks = a.blks[:0] + a.blks = append(a.blks, entry) + + s := &a.state + for len(a.blks) > 0 { + tail := len(a.blks) - 1 + loop := a.blks[tail] + a.blks = a.blks[:tail] + a.vs2 = a.vs2[:0] + const ( + flagDone = false + flagPending = true + ) + info := a.getOrAllocateBlockState(loop.ID()) + for _, v := range info.liveIns { + if s.phiBlk(v) != loop { + a.vs2 = append(a.vs2, v) + st := s.getVRegState(v) + // We use .spilled field to store the flag. + st.spilled = flagPending + } + } + + var siblingAddedView []VRegID + cn := loop.LoopNestingForestChildren() + for i := 0; i < cn; i++ { + child := loop.LoopNestingForestChild(i) + childID := child.ID() + childInfo := a.getOrAllocateBlockState(childID) + + if i == 0 { + begin := len(childInfo.liveIns) + for _, v := range a.vs2 { + st := s.getVRegState(v) + // We use .spilled field to store the flag. + if st.spilled == flagPending { //nolint:gosimple + st.spilled = flagDone + // TODO: deduplicate, though I don't think it has much impact. + childInfo.liveIns = append(childInfo.liveIns, v) + } + } + siblingAddedView = childInfo.liveIns[begin:] + } else { + // TODO: deduplicate, though I don't think it has much impact. + childInfo.liveIns = append(childInfo.liveIns, siblingAddedView...) + } + + if child.LoopHeader() { + a.blks = append(a.blks, child) + } + } + + if cn == 0 { + // If there's no forest child, we haven't cleared the .spilled field at this point. + for _, v := range a.vs2 { + st := s.getVRegState(v) + st.spilled = false + } + } + } +} + +// alloc allocates registers for the given function by iterating the blocks in the reverse postorder. +// The algorithm here is derived from the Go compiler's allocator https://github.com/golang/go/blob/release-branch.go1.21/src/cmd/compile/internal/ssa/regalloc.go +// In short, this is a simply linear scan register allocation where each block inherits the register allocation state from +// one of its predecessors. Each block inherits the selected state and starts allocation from there. +// If there's a discrepancy in the end states between predecessors, the adjustments are made to ensure consistency after allocation is done (which we call "fixing merge state"). +// The spill instructions (store into the dedicated slots) are inserted after all the allocations and fixing merge states. That is because +// at the point, we all know where the reloads happen, and therefore we can know the best place to spill the values. More precisely, +// the spill happens in the block that is the lowest common ancestor of all the blocks that reloads the value. +// +// All of these logics are almost the same as Go's compiler which has a dedicated description in the source file ^^. +func (a *Allocator) alloc(f Function) { + // First we allocate each block in the reverse postorder (at least one predecessor should be allocated for each block). + for blk := f.ReversePostOrderBlockIteratorBegin(); blk != nil; blk = f.ReversePostOrderBlockIteratorNext() { + if wazevoapi.RegAllocLoggingEnabled { + fmt.Printf("========== allocating blk%d ========\n", blk.ID()) + } + if blk.Entry() { + a.finalizeStartReg(blk) + } + a.allocBlock(f, blk) + } + // After the allocation, we all know the start and end state of each block. So we can fix the merge states. + for blk := f.ReversePostOrderBlockIteratorBegin(); blk != nil; blk = f.ReversePostOrderBlockIteratorNext() { + a.fixMergeState(f, blk) + } + // Finally, we insert the spill instructions as we know all the places where the reloads happen. + a.scheduleSpills(f) +} + +func (a *Allocator) updateLiveInVRState(liveness *blockState) { + currentBlockID := a.state.currentBlockID + for _, v := range liveness.liveIns { + vs := a.state.getVRegState(v) + vs.lastUse = programCounterLiveIn + vs.lastUseUpdatedAtBlockID = currentBlockID + } +} + +func (a *Allocator) finalizeStartReg(blk Block) { + bID := blk.ID() + liveness := a.getOrAllocateBlockState(bID) + s := &a.state + currentBlkState := a.getOrAllocateBlockState(bID) + if currentBlkState.startFromPredIndex > -1 { + return + } + + s.currentBlockID = bID + a.updateLiveInVRState(liveness) + + preds := blk.Preds() + var predState *blockState + switch preds { + case 0: // This is the entry block. + case 1: + predID := blk.Pred(0).ID() + predState = a.getOrAllocateBlockState(predID) + currentBlkState.startFromPredIndex = 0 + default: + // TODO: there should be some better heuristic to choose the predecessor. + for i := 0; i < preds; i++ { + predID := blk.Pred(i).ID() + if _predState := a.getOrAllocateBlockState(predID); _predState.visited { + predState = _predState + currentBlkState.startFromPredIndex = i + break + } + } + } + if predState == nil { + if !blk.Entry() { + panic(fmt.Sprintf("BUG: at lease one predecessor should be visited for blk%d", blk.ID())) + } + for _, u := range s.argRealRegs { + s.useRealReg(u.RealReg(), u) + } + currentBlkState.startFromPredIndex = 0 + } else if predState != nil { + if wazevoapi.RegAllocLoggingEnabled { + fmt.Printf("allocating blk%d starting from blk%d (on index=%d) \n", + bID, blk.Pred(currentBlkState.startFromPredIndex).ID(), currentBlkState.startFromPredIndex) + } + s.resetAt(predState) + } + + s.regsInUse.range_(func(allocated RealReg, v VReg) { + currentBlkState.startRegs.add(allocated, v) + }) + if wazevoapi.RegAllocLoggingEnabled { + fmt.Printf("finalized start reg for blk%d: %s\n", blk.ID(), currentBlkState.startRegs.format(a.regInfo)) + } +} + +func (a *Allocator) allocBlock(f Function, blk Block) { + bID := blk.ID() + s := &a.state + currentBlkState := a.getOrAllocateBlockState(bID) + s.currentBlockID = bID + + if currentBlkState.startFromPredIndex < 0 { + panic("BUG: startFromPredIndex should be set in finalizeStartReg prior to allocBlock") + } + + // Clears the previous state. + s.regsInUse.range_(func(allocatedRealReg RealReg, vr VReg) { + s.setVRegState(vr, RealRegInvalid) + }) + s.regsInUse.reset() + // Then set the start state. + currentBlkState.startRegs.range_(func(allocatedRealReg RealReg, vr VReg) { + s.useRealReg(allocatedRealReg, vr) + }) + + desiredUpdated := a.vs2[:0] + + // Update the last use of each VReg. + var pc programCounter + for instr := blk.InstrIteratorBegin(); instr != nil; instr = blk.InstrIteratorNext() { + var use, def VReg + for _, use = range instr.Uses(&a.vs) { + if !use.IsRealReg() { + s.getVRegState(use.ID()).lastUse = pc + } + } + + if instr.IsCopy() { + def = instr.Defs(&a.vs)[0] + r := def.RealReg() + if r != RealRegInvalid { + useID := use.ID() + vs := s.getVRegState(useID) + if !vs.isPhi { // TODO: no idea why do we need this. + vs.desiredLoc = newDesiredLocReg(r) + desiredUpdated = append(desiredUpdated, useID) + } + } + } + pc++ + } + + // Mark all live-out values by checking live-in of the successors. + // While doing so, we also update the desired register values. + var succ Block + for i, ns := 0, blk.Succs(); i < ns; i++ { + succ = blk.Succ(i) + if succ == nil { + continue + } + + succID := succ.ID() + succState := a.getOrAllocateBlockState(succID) + for _, v := range succState.liveIns { + if s.phiBlk(v) != succ { + st := s.getVRegState(v) + st.lastUse = programCounterLiveOut + } + } + + if succState.startFromPredIndex > -1 { + if wazevoapi.RegAllocLoggingEnabled { + fmt.Printf("blk%d -> blk%d: start_regs: %s\n", bID, succID, succState.startRegs.format(a.regInfo)) + } + succState.startRegs.range_(func(allocatedRealReg RealReg, vr VReg) { + vs := s.getVRegState(vr.ID()) + vs.desiredLoc = newDesiredLocReg(allocatedRealReg) + desiredUpdated = append(desiredUpdated, vr.ID()) + }) + for _, p := range succ.BlockParams(&a.vs) { + vs := s.getVRegState(p.ID()) + if vs.desiredLoc.realReg() == RealRegInvalid { + vs.desiredLoc = desiredLocStack + desiredUpdated = append(desiredUpdated, p.ID()) + } + } + } + } + + // Propagate the desired register values from the end of the block to the beginning. + for instr := blk.InstrRevIteratorBegin(); instr != nil; instr = blk.InstrRevIteratorNext() { + if instr.IsCopy() { + def := instr.Defs(&a.vs)[0] + defState := s.getVRegState(def.ID()) + desired := defState.desiredLoc.realReg() + if desired == RealRegInvalid { + continue + } + + use := instr.Uses(&a.vs)[0] + useID := use.ID() + useState := s.getVRegState(useID) + if s.phiBlk(useID) != succ && useState.desiredLoc == desiredLocUnspecified { + useState.desiredLoc = newDesiredLocReg(desired) + desiredUpdated = append(desiredUpdated, useID) + } + } + } + + pc = 0 + for instr := blk.InstrIteratorBegin(); instr != nil; instr = blk.InstrIteratorNext() { + if wazevoapi.RegAllocLoggingEnabled { + fmt.Println(instr) + } + + var currentUsedSet RegSet + killSet := a.reals[:0] + + // Gather the set of registers that will be used in the current instruction. + for _, use := range instr.Uses(&a.vs) { + if use.IsRealReg() { + r := use.RealReg() + currentUsedSet = currentUsedSet.add(r) + if a.allocatableSet.has(r) { + killSet = append(killSet, r) + } + } else { + vs := s.getVRegState(use.ID()) + if r := vs.r; r != RealRegInvalid { + currentUsedSet = currentUsedSet.add(r) + } + } + } + + for i, use := range instr.Uses(&a.vs) { + if !use.IsRealReg() { + vs := s.getVRegState(use.ID()) + killed := vs.lastUse == pc + r := vs.r + + if r == RealRegInvalid { + r = s.findOrSpillAllocatable(a, a.regInfo.AllocatableRegisters[use.RegType()], currentUsedSet, + // Prefer the desired register if it's available. + vs.desiredLoc.realReg()) + vs.recordReload(f, blk) + f.ReloadRegisterBefore(use.SetRealReg(r), instr) + s.useRealReg(r, use) + } + if wazevoapi.RegAllocLoggingEnabled { + fmt.Printf("\ttrying to use v%v on %s\n", use.ID(), a.regInfo.RealRegName(r)) + } + instr.AssignUse(i, use.SetRealReg(r)) + currentUsedSet = currentUsedSet.add(r) + if killed { + if wazevoapi.RegAllocLoggingEnabled { + fmt.Printf("\tkill v%d with %s\n", use.ID(), a.regInfo.RealRegName(r)) + } + killSet = append(killSet, r) + } + } + } + + isIndirect := instr.IsIndirectCall() + call := instr.IsCall() || isIndirect + if call { + addr := RealRegInvalid + if instr.IsIndirectCall() { + addr = a.vs[0].RealReg() + } + a.releaseCallerSavedRegs(addr) + } + + for _, r := range killSet { + s.releaseRealReg(r) + } + a.reals = killSet + + defs := instr.Defs(&a.vs) + switch { + case len(defs) > 1: + // Some instructions define multiple values on real registers. + // E.g. call instructions (following calling convention) / div instruction on x64 that defines both rax and rdx. + // + // Note that currently I assume that such instructions define only the pre colored real registers, not the VRegs + // that require allocations. If we need to support such case, we need to add the logic to handle it here, + // though is there any such instruction? + for _, def := range defs { + if !def.IsRealReg() { + panic("BUG: multiple defs should be on real registers") + } + r := def.RealReg() + if s.regsInUse.has(r) { + s.releaseRealReg(r) + } + s.useRealReg(r, def) + } + case len(defs) == 1: + def := defs[0] + if def.IsRealReg() { + r := def.RealReg() + if a.allocatableSet.has(r) { + if s.regsInUse.has(r) { + s.releaseRealReg(r) + } + s.useRealReg(r, def) + } + } else { + vState := s.getVRegState(def.ID()) + r := vState.r + + if desired := vState.desiredLoc.realReg(); desired != RealRegInvalid { + if r != desired { + if (vState.isPhi && vState.defBlk == succ) || + // If this is not a phi and it's already assigned a real reg, + // this value has multiple definitions, hence we cannot assign the desired register. + (!s.regsInUse.has(desired) && r == RealRegInvalid) { + // If the phi value is passed via a real register, we force the value to be in the desired register. + if wazevoapi.RegAllocLoggingEnabled { + fmt.Printf("\t\tv%d is phi and desiredReg=%s\n", def.ID(), a.regInfo.RealRegName(desired)) + } + if r != RealRegInvalid { + // If the value is already in a different real register, we release it to change the state. + // Otherwise, multiple registers might have the same values at the end, which results in + // messing up the merge state reconciliation. + s.releaseRealReg(r) + } + r = desired + s.releaseRealReg(r) + s.useRealReg(r, def) + } + } + } + + // Allocate a new real register if `def` is not currently assigned one. + // It can happen when multiple instructions define the same VReg (e.g. const loads). + if r == RealRegInvalid { + if instr.IsCopy() { + copySrc := instr.Uses(&a.vs)[0].RealReg() + if a.allocatableSet.has(copySrc) && !s.regsInUse.has(copySrc) { + r = copySrc + } + } + if r == RealRegInvalid { + typ := def.RegType() + r = s.findOrSpillAllocatable(a, a.regInfo.AllocatableRegisters[typ], RegSet(0), RealRegInvalid) + } + s.useRealReg(r, def) + } + dr := def.SetRealReg(r) + instr.AssignDef(dr) + if wazevoapi.RegAllocLoggingEnabled { + fmt.Printf("\tdefining v%d with %s\n", def.ID(), a.regInfo.RealRegName(r)) + } + if vState.isPhi { + if vState.desiredLoc.stack() { // Stack based phi value. + f.StoreRegisterAfter(dr, instr) + // Release the real register as it's not used anymore. + s.releaseRealReg(r) + } else { + // Only the register based phis are necessary to track the defining instructions + // since the stack-based phis are already having stores inserted ^. + n := a.phiDefInstListPool.Allocate() + n.instr = instr + n.next = vState.phiDefInstList + n.v = dr + vState.phiDefInstList = n + } + } else { + vState.defInstr = instr + vState.defBlk = blk + } + } + } + if wazevoapi.RegAllocLoggingEnabled { + fmt.Println(instr) + } + pc++ + } + + s.regsInUse.range_(func(allocated RealReg, v VReg) { + currentBlkState.endRegs.add(allocated, v) + }) + + currentBlkState.visited = true + if wazevoapi.RegAllocLoggingEnabled { + currentBlkState.dump(a.regInfo) + } + + // Reset the desired end location. + for _, v := range desiredUpdated { + vs := s.getVRegState(v) + vs.desiredLoc = desiredLocUnspecified + } + a.vs2 = desiredUpdated[:0] + + for i := 0; i < blk.Succs(); i++ { + succ := blk.Succ(i) + if succ == nil { + continue + } + // If the successor is not visited yet, finalize the start state. + a.finalizeStartReg(succ) + } +} + +func (a *Allocator) releaseCallerSavedRegs(addrReg RealReg) { + s := &a.state + + for i := 0; i < 64; i++ { + allocated := RealReg(i) + if allocated == addrReg { // If this is the call indirect, we should not touch the addr register. + continue + } + if v := s.regsInUse.get(allocated); v.Valid() { + if v.IsRealReg() { + continue // This is the argument register as it's already used by VReg backed by the corresponding RealReg. + } + if !a.regInfo.CallerSavedRegisters.has(allocated) { + // If this is not a caller-saved register, it is safe to keep it across the call. + continue + } + s.releaseRealReg(allocated) + } + } +} + +func (a *Allocator) fixMergeState(f Function, blk Block) { + preds := blk.Preds() + if preds <= 1 { + return + } + + s := &a.state + + // Restores the state at the beginning of the block. + bID := blk.ID() + blkSt := a.getOrAllocateBlockState(bID) + desiredOccupants := &blkSt.startRegs + aliveOnRegVRegs := make(map[VReg]RealReg) + for i := 0; i < 64; i++ { + r := RealReg(i) + if v := blkSt.startRegs.get(r); v.Valid() { + aliveOnRegVRegs[v] = r + } + } + + if wazevoapi.RegAllocLoggingEnabled { + fmt.Println("fixMergeState", blk.ID(), ":", desiredOccupants.format(a.regInfo)) + } + + s.currentBlockID = bID + a.updateLiveInVRState(a.getOrAllocateBlockState(bID)) + + currentOccupants := &a.currentOccupants + for i := 0; i < preds; i++ { + currentOccupants.reset() + if i == blkSt.startFromPredIndex { + continue + } + + currentOccupantsRev := make(map[VReg]RealReg) + pred := blk.Pred(i) + predSt := a.getOrAllocateBlockState(pred.ID()) + for ii := 0; ii < 64; ii++ { + r := RealReg(ii) + if v := predSt.endRegs.get(r); v.Valid() { + if _, ok := aliveOnRegVRegs[v]; !ok { + continue + } + currentOccupants.add(r, v) + currentOccupantsRev[v] = r + } + } + + s.resetAt(predSt) + + // Finds the free registers if any. + intTmp, floatTmp := VRegInvalid, VRegInvalid + if intFree := s.findAllocatable( + a.regInfo.AllocatableRegisters[RegTypeInt], desiredOccupants.set, + ); intFree != RealRegInvalid { + intTmp = FromRealReg(intFree, RegTypeInt) + } + if floatFree := s.findAllocatable( + a.regInfo.AllocatableRegisters[RegTypeFloat], desiredOccupants.set, + ); floatFree != RealRegInvalid { + floatTmp = FromRealReg(floatFree, RegTypeFloat) + } + + if wazevoapi.RegAllocLoggingEnabled { + fmt.Println("\t", pred.ID(), ":", currentOccupants.format(a.regInfo)) + } + + for ii := 0; ii < 64; ii++ { + r := RealReg(ii) + desiredVReg := desiredOccupants.get(r) + if !desiredVReg.Valid() { + continue + } + + currentVReg := currentOccupants.get(r) + if desiredVReg.ID() == currentVReg.ID() { + continue + } + + typ := desiredVReg.RegType() + var tmpRealReg VReg + if typ == RegTypeInt { + tmpRealReg = intTmp + } else { + tmpRealReg = floatTmp + } + a.reconcileEdge(f, r, pred, currentOccupants, currentOccupantsRev, currentVReg, desiredVReg, tmpRealReg, typ) + } + } +} + +func (a *Allocator) reconcileEdge(f Function, + r RealReg, + pred Block, + currentOccupants *regInUseSet, + currentOccupantsRev map[VReg]RealReg, + currentVReg, desiredVReg VReg, + freeReg VReg, + typ RegType, +) { + s := &a.state + if currentVReg.Valid() { + // Both are on reg. + er, ok := currentOccupantsRev[desiredVReg] + if !ok { + if wazevoapi.RegAllocLoggingEnabled { + fmt.Printf("\t\tv%d is desired to be on %s, but currently on the stack\n", + desiredVReg.ID(), a.regInfo.RealRegName(r), + ) + } + // This case is that the desired value is on the stack, but currentVReg is on the target register. + // We need to move the current value to the stack, and reload the desired value. + // TODO: we can do better here. + f.StoreRegisterBefore(currentVReg.SetRealReg(r), pred.LastInstrForInsertion()) + delete(currentOccupantsRev, currentVReg) + + s.getVRegState(desiredVReg.ID()).recordReload(f, pred) + f.ReloadRegisterBefore(desiredVReg.SetRealReg(r), pred.LastInstrForInsertion()) + currentOccupants.add(r, desiredVReg) + currentOccupantsRev[desiredVReg] = r + return + } + + if wazevoapi.RegAllocLoggingEnabled { + fmt.Printf("\t\tv%d is desired to be on %s, but currently on %s\n", + desiredVReg.ID(), a.regInfo.RealRegName(r), a.regInfo.RealRegName(er), + ) + } + f.SwapBefore( + currentVReg.SetRealReg(r), + desiredVReg.SetRealReg(er), + freeReg, + pred.LastInstrForInsertion(), + ) + s.allocatedRegSet = s.allocatedRegSet.add(freeReg.RealReg()) + currentOccupantsRev[desiredVReg] = r + currentOccupantsRev[currentVReg] = er + currentOccupants.add(r, desiredVReg) + currentOccupants.add(er, currentVReg) + if wazevoapi.RegAllocLoggingEnabled { + fmt.Printf("\t\tv%d previously on %s moved to %s\n", currentVReg.ID(), a.regInfo.RealRegName(r), a.regInfo.RealRegName(er)) + } + } else { + // Desired is on reg, but currently the target register is not used. + if wazevoapi.RegAllocLoggingEnabled { + fmt.Printf("\t\tv%d is desired to be on %s, current not used\n", + desiredVReg.ID(), a.regInfo.RealRegName(r), + ) + } + if currentReg, ok := currentOccupantsRev[desiredVReg]; ok { + f.InsertMoveBefore( + FromRealReg(r, typ), + desiredVReg.SetRealReg(currentReg), + pred.LastInstrForInsertion(), + ) + currentOccupants.remove(currentReg) + } else { + s.getVRegState(desiredVReg.ID()).recordReload(f, pred) + f.ReloadRegisterBefore(desiredVReg.SetRealReg(r), pred.LastInstrForInsertion()) + } + currentOccupantsRev[desiredVReg] = r + currentOccupants.add(r, desiredVReg) + } + + if wazevoapi.RegAllocLoggingEnabled { + fmt.Println("\t", pred.ID(), ":", currentOccupants.format(a.regInfo)) + } +} + +func (a *Allocator) scheduleSpills(f Function) { + states := a.state.vrStates + for i := 0; i <= states.MaxIDEncountered(); i++ { + vs := states.Get(i) + if vs == nil { + continue + } + if vs.spilled { + a.scheduleSpill(f, vs) + } + } +} + +func (a *Allocator) scheduleSpill(f Function, vs *vrState) { + v := vs.v + // If the value is the phi value, we need to insert a spill after each phi definition. + if vs.isPhi { + for defInstr := vs.phiDefInstList; defInstr != nil; defInstr = defInstr.next { + f.StoreRegisterAfter(defInstr.v, defInstr.instr) + } + return + } + + pos := vs.lca + definingBlk := vs.defBlk + r := RealRegInvalid + if definingBlk == nil { + panic(fmt.Sprintf("BUG: definingBlk should not be nil for %s. This is likley a bug in backend lowering logic", vs.v.String())) + } + if pos == nil { + panic(fmt.Sprintf("BUG: pos should not be nil for %s. This is likley a bug in backend lowering logic", vs.v.String())) + } + + if wazevoapi.RegAllocLoggingEnabled { + fmt.Printf("v%d is spilled in blk%d, lca=blk%d\n", v.ID(), definingBlk.ID(), pos.ID()) + } + for pos != definingBlk { + st := a.getOrAllocateBlockState(pos.ID()) + for ii := 0; ii < 64; ii++ { + rr := RealReg(ii) + if st.startRegs.get(rr) == v { + r = rr + // Already in the register, so we can place the spill at the beginning of the block. + break + } + } + + if r != RealRegInvalid { + break + } + + pos = f.Idom(pos) + } + + if pos == definingBlk { + defInstr := vs.defInstr + defInstr.Defs(&a.vs) + if wazevoapi.RegAllocLoggingEnabled { + fmt.Printf("schedule spill v%d after %v\n", v.ID(), defInstr) + } + f.StoreRegisterAfter(a.vs[0], defInstr) + } else { + // Found an ancestor block that holds the value in the register at the beginning of the block. + // We need to insert a spill before the last use. + first := pos.FirstInstr() + if wazevoapi.RegAllocLoggingEnabled { + fmt.Printf("schedule spill v%d before %v\n", v.ID(), first) + } + f.StoreRegisterAfter(v.SetRealReg(r), first) + } +} + +// Reset resets the allocator's internal state so that it can be reused. +func (a *Allocator) Reset() { + a.state.reset() + a.blockStates.Reset() + a.phiDefInstListPool.Reset() + a.vs = a.vs[:0] +} diff --git a/vendor/github.com/tetratelabs/wazero/internal/engine/wazevo/backend/regalloc/regset.go b/vendor/github.com/tetratelabs/wazero/internal/engine/wazevo/backend/regalloc/regset.go new file mode 100644 index 000000000..e9bf60661 --- /dev/null +++ b/vendor/github.com/tetratelabs/wazero/internal/engine/wazevo/backend/regalloc/regset.go @@ -0,0 +1,108 @@ +package regalloc + +import ( + "fmt" + "strings" +) + +// NewRegSet returns a new RegSet with the given registers. +func NewRegSet(regs ...RealReg) RegSet { + var ret RegSet + for _, r := range regs { + ret = ret.add(r) + } + return ret +} + +// RegSet represents a set of registers. +type RegSet uint64 + +func (rs RegSet) format(info *RegisterInfo) string { //nolint:unused + var ret []string + for i := 0; i < 64; i++ { + if rs&(1<= 64 { + return rs + } + return rs | 1<v%d)", info.RealRegName(RealReg(i)), vr.ID())) + } + } + return strings.Join(ret, ", ") +} + +func (rs *regInUseSet) has(r RealReg) bool { + if r >= 64 { + return false + } + return rs.set&(1<= 64 { + return VRegInvalid + } + return rs.vrs[r] +} + +func (rs *regInUseSet) remove(r RealReg) { + if r >= 64 { + return + } + rs.set &= ^(1 << uint(r)) + rs.vrs[r] = VRegInvalid +} + +func (rs *regInUseSet) add(r RealReg, vr VReg) { + if r >= 64 { + return + } + rs.set |= 1 << uint(r) + rs.vrs[r] = vr +} + +func (rs *regInUseSet) range_(f func(allocatedRealReg RealReg, vr VReg)) { + for i := 0; i < 64; i++ { + if rs.set&(1< stackSize { + stackSize = required + } + return stackSize +} + +func (c *callEngine) init() { + stackSize := c.requiredInitialStackSize() + if wazevoapi.StackGuardCheckEnabled { + stackSize += wazevoapi.StackGuardCheckGuardPageSize + } + c.stack = make([]byte, stackSize) + c.stackTop = alignedStackTop(c.stack) + if wazevoapi.StackGuardCheckEnabled { + c.execCtx.stackBottomPtr = &c.stack[wazevoapi.StackGuardCheckGuardPageSize] + } else { + c.execCtx.stackBottomPtr = &c.stack[0] + } + c.execCtxPtr = uintptr(unsafe.Pointer(&c.execCtx)) +} + +// alignedStackTop returns 16-bytes aligned stack top of given stack. +// 16 bytes should be good for all platform (arm64/amd64). +func alignedStackTop(s []byte) uintptr { + stackAddr := uintptr(unsafe.Pointer(&s[len(s)-1])) + return stackAddr - (stackAddr & (16 - 1)) +} + +// Definition implements api.Function. +func (c *callEngine) Definition() api.FunctionDefinition { + return c.parent.module.Source.FunctionDefinition(c.indexInModule) +} + +// Call implements api.Function. +func (c *callEngine) Call(ctx context.Context, params ...uint64) ([]uint64, error) { + if c.requiredParams != len(params) { + return nil, fmt.Errorf("expected %d params, but passed %d", c.requiredParams, len(params)) + } + paramResultSlice := make([]uint64, c.sizeOfParamResultSlice) + copy(paramResultSlice, params) + if err := c.callWithStack(ctx, paramResultSlice); err != nil { + return nil, err + } + return paramResultSlice[:c.numberOfResults], nil +} + +func (c *callEngine) addFrame(builder wasmdebug.ErrorBuilder, addr uintptr) (def api.FunctionDefinition, listener experimental.FunctionListener) { + eng := c.parent.parent.parent + cm := eng.compiledModuleOfAddr(addr) + if cm == nil { + // This case, the module might have been closed and deleted from the engine. + // We fall back to searching the imported modules that can be referenced from this callEngine. + + // First, we check itself. + if checkAddrInBytes(addr, c.parent.parent.executable) { + cm = c.parent.parent + } else { + // Otherwise, search all imported modules. TODO: maybe recursive, but not sure it's useful in practice. + p := c.parent + for i := range p.importedFunctions { + candidate := p.importedFunctions[i].me.parent + if checkAddrInBytes(addr, candidate.executable) { + cm = candidate + break + } + } + } + } + + if cm != nil { + index := cm.functionIndexOf(addr) + def = cm.module.FunctionDefinition(cm.module.ImportFunctionCount + index) + var sources []string + if dw := cm.module.DWARFLines; dw != nil { + sourceOffset := cm.getSourceOffset(addr) + sources = dw.Line(sourceOffset) + } + builder.AddFrame(def.DebugName(), def.ParamTypes(), def.ResultTypes(), sources) + if len(cm.listeners) > 0 { + listener = cm.listeners[index] + } + } + return +} + +// CallWithStack implements api.Function. +func (c *callEngine) CallWithStack(ctx context.Context, paramResultStack []uint64) (err error) { + if c.sizeOfParamResultSlice > len(paramResultStack) { + return fmt.Errorf("need %d params, but stack size is %d", c.sizeOfParamResultSlice, len(paramResultStack)) + } + return c.callWithStack(ctx, paramResultStack) +} + +// CallWithStack implements api.Function. +func (c *callEngine) callWithStack(ctx context.Context, paramResultStack []uint64) (err error) { + snapshotEnabled := ctx.Value(expctxkeys.EnableSnapshotterKey{}) != nil + if snapshotEnabled { + ctx = context.WithValue(ctx, expctxkeys.SnapshotterKey{}, c) + } + + if wazevoapi.StackGuardCheckEnabled { + defer func() { + wazevoapi.CheckStackGuardPage(c.stack) + }() + } + + p := c.parent + ensureTermination := p.parent.ensureTermination + m := p.module + if ensureTermination { + select { + case <-ctx.Done(): + // If the provided context is already done, close the module and return the error. + m.CloseWithCtxErr(ctx) + return m.FailIfClosed() + default: + } + } + + var paramResultPtr *uint64 + if len(paramResultStack) > 0 { + paramResultPtr = ¶mResultStack[0] + } + defer func() { + r := recover() + if s, ok := r.(*snapshot); ok { + // A snapshot that wasn't handled was created by a different call engine possibly from a nested wasm invocation, + // let it propagate up to be handled by the caller. + panic(s) + } + if r != nil { + type listenerForAbort struct { + def api.FunctionDefinition + lsn experimental.FunctionListener + } + + var listeners []listenerForAbort + builder := wasmdebug.NewErrorBuilder() + def, lsn := c.addFrame(builder, uintptr(unsafe.Pointer(c.execCtx.goCallReturnAddress))) + if lsn != nil { + listeners = append(listeners, listenerForAbort{def, lsn}) + } + returnAddrs := unwindStack( + uintptr(unsafe.Pointer(c.execCtx.stackPointerBeforeGoCall)), + c.execCtx.framePointerBeforeGoCall, + c.stackTop, + nil, + ) + for _, retAddr := range returnAddrs[:len(returnAddrs)-1] { // the last return addr is the trampoline, so we skip it. + def, lsn = c.addFrame(builder, retAddr) + if lsn != nil { + listeners = append(listeners, listenerForAbort{def, lsn}) + } + } + err = builder.FromRecovered(r) + + for _, lsn := range listeners { + lsn.lsn.Abort(ctx, m, lsn.def, err) + } + } else { + if err != wasmruntime.ErrRuntimeStackOverflow { // Stackoverflow case shouldn't be panic (to avoid extreme stack unwinding). + err = c.parent.module.FailIfClosed() + } + } + + if err != nil { + // Ensures that we can reuse this callEngine even after an error. + c.execCtx.exitCode = wazevoapi.ExitCodeOK + } + }() + + if ensureTermination { + done := m.CloseModuleOnCanceledOrTimeout(ctx) + defer done() + } + + if c.stackTop&(16-1) != 0 { + panic("BUG: stack must be aligned to 16 bytes") + } + entrypoint(c.preambleExecutable, c.executable, c.execCtxPtr, c.parent.opaquePtr, paramResultPtr, c.stackTop) + for { + switch ec := c.execCtx.exitCode; ec & wazevoapi.ExitCodeMask { + case wazevoapi.ExitCodeOK: + return nil + case wazevoapi.ExitCodeGrowStack: + oldsp := uintptr(unsafe.Pointer(c.execCtx.stackPointerBeforeGoCall)) + oldTop := c.stackTop + oldStack := c.stack + var newsp, newfp uintptr + if wazevoapi.StackGuardCheckEnabled { + newsp, newfp, err = c.growStackWithGuarded() + } else { + newsp, newfp, err = c.growStack() + } + if err != nil { + return err + } + adjustClonedStack(oldsp, oldTop, newsp, newfp, c.stackTop) + // Old stack must be alive until the new stack is adjusted. + runtime.KeepAlive(oldStack) + c.execCtx.exitCode = wazevoapi.ExitCodeOK + afterGoFunctionCallEntrypoint(c.execCtx.goCallReturnAddress, c.execCtxPtr, newsp, newfp) + case wazevoapi.ExitCodeGrowMemory: + mod := c.callerModuleInstance() + mem := mod.MemoryInstance + s := goCallStackView(c.execCtx.stackPointerBeforeGoCall) + argRes := &s[0] + if res, ok := mem.Grow(uint32(*argRes)); !ok { + *argRes = uint64(0xffffffff) // = -1 in signed 32-bit integer. + } else { + *argRes = uint64(res) + calleeOpaque := opaqueViewFromPtr(uintptr(unsafe.Pointer(c.execCtx.callerModuleContextPtr))) + if mod.Source.MemorySection != nil { // Local memory. + putLocalMemory(calleeOpaque, 8 /* local memory begins at 8 */, mem) + } else { + // Imported memory's owner at offset 16 of the callerModuleContextPtr. + opaquePtr := uintptr(binary.LittleEndian.Uint64(calleeOpaque[16:])) + importedMemOwner := opaqueViewFromPtr(opaquePtr) + putLocalMemory(importedMemOwner, 8 /* local memory begins at 8 */, mem) + } + } + c.execCtx.exitCode = wazevoapi.ExitCodeOK + afterGoFunctionCallEntrypoint(c.execCtx.goCallReturnAddress, c.execCtxPtr, uintptr(unsafe.Pointer(c.execCtx.stackPointerBeforeGoCall)), c.execCtx.framePointerBeforeGoCall) + case wazevoapi.ExitCodeTableGrow: + mod := c.callerModuleInstance() + s := goCallStackView(c.execCtx.stackPointerBeforeGoCall) + tableIndex, num, ref := uint32(s[0]), uint32(s[1]), uintptr(s[2]) + table := mod.Tables[tableIndex] + s[0] = uint64(uint32(int32(table.Grow(num, ref)))) + c.execCtx.exitCode = wazevoapi.ExitCodeOK + afterGoFunctionCallEntrypoint(c.execCtx.goCallReturnAddress, c.execCtxPtr, + uintptr(unsafe.Pointer(c.execCtx.stackPointerBeforeGoCall)), c.execCtx.framePointerBeforeGoCall) + case wazevoapi.ExitCodeCallGoFunction: + index := wazevoapi.GoFunctionIndexFromExitCode(ec) + f := hostModuleGoFuncFromOpaque[api.GoFunction](index, c.execCtx.goFunctionCallCalleeModuleContextOpaque) + func() { + if snapshotEnabled { + defer snapshotRecoverFn(c) + } + f.Call(ctx, goCallStackView(c.execCtx.stackPointerBeforeGoCall)) + }() + // Back to the native code. + c.execCtx.exitCode = wazevoapi.ExitCodeOK + afterGoFunctionCallEntrypoint(c.execCtx.goCallReturnAddress, c.execCtxPtr, + uintptr(unsafe.Pointer(c.execCtx.stackPointerBeforeGoCall)), c.execCtx.framePointerBeforeGoCall) + case wazevoapi.ExitCodeCallGoFunctionWithListener: + index := wazevoapi.GoFunctionIndexFromExitCode(ec) + f := hostModuleGoFuncFromOpaque[api.GoFunction](index, c.execCtx.goFunctionCallCalleeModuleContextOpaque) + listeners := hostModuleListenersSliceFromOpaque(c.execCtx.goFunctionCallCalleeModuleContextOpaque) + s := goCallStackView(c.execCtx.stackPointerBeforeGoCall) + // Call Listener.Before. + callerModule := c.callerModuleInstance() + listener := listeners[index] + hostModule := hostModuleFromOpaque(c.execCtx.goFunctionCallCalleeModuleContextOpaque) + def := hostModule.FunctionDefinition(wasm.Index(index)) + listener.Before(ctx, callerModule, def, s, c.stackIterator(true)) + // Call into the Go function. + func() { + if snapshotEnabled { + defer snapshotRecoverFn(c) + } + f.Call(ctx, s) + }() + // Call Listener.After. + listener.After(ctx, callerModule, def, s) + // Back to the native code. + c.execCtx.exitCode = wazevoapi.ExitCodeOK + afterGoFunctionCallEntrypoint(c.execCtx.goCallReturnAddress, c.execCtxPtr, + uintptr(unsafe.Pointer(c.execCtx.stackPointerBeforeGoCall)), c.execCtx.framePointerBeforeGoCall) + case wazevoapi.ExitCodeCallGoModuleFunction: + index := wazevoapi.GoFunctionIndexFromExitCode(ec) + f := hostModuleGoFuncFromOpaque[api.GoModuleFunction](index, c.execCtx.goFunctionCallCalleeModuleContextOpaque) + mod := c.callerModuleInstance() + func() { + if snapshotEnabled { + defer snapshotRecoverFn(c) + } + f.Call(ctx, mod, goCallStackView(c.execCtx.stackPointerBeforeGoCall)) + }() + // Back to the native code. + c.execCtx.exitCode = wazevoapi.ExitCodeOK + afterGoFunctionCallEntrypoint(c.execCtx.goCallReturnAddress, c.execCtxPtr, + uintptr(unsafe.Pointer(c.execCtx.stackPointerBeforeGoCall)), c.execCtx.framePointerBeforeGoCall) + case wazevoapi.ExitCodeCallGoModuleFunctionWithListener: + index := wazevoapi.GoFunctionIndexFromExitCode(ec) + f := hostModuleGoFuncFromOpaque[api.GoModuleFunction](index, c.execCtx.goFunctionCallCalleeModuleContextOpaque) + listeners := hostModuleListenersSliceFromOpaque(c.execCtx.goFunctionCallCalleeModuleContextOpaque) + s := goCallStackView(c.execCtx.stackPointerBeforeGoCall) + // Call Listener.Before. + callerModule := c.callerModuleInstance() + listener := listeners[index] + hostModule := hostModuleFromOpaque(c.execCtx.goFunctionCallCalleeModuleContextOpaque) + def := hostModule.FunctionDefinition(wasm.Index(index)) + listener.Before(ctx, callerModule, def, s, c.stackIterator(true)) + // Call into the Go function. + func() { + if snapshotEnabled { + defer snapshotRecoverFn(c) + } + f.Call(ctx, callerModule, s) + }() + // Call Listener.After. + listener.After(ctx, callerModule, def, s) + // Back to the native code. + c.execCtx.exitCode = wazevoapi.ExitCodeOK + afterGoFunctionCallEntrypoint(c.execCtx.goCallReturnAddress, c.execCtxPtr, + uintptr(unsafe.Pointer(c.execCtx.stackPointerBeforeGoCall)), c.execCtx.framePointerBeforeGoCall) + case wazevoapi.ExitCodeCallListenerBefore: + stack := goCallStackView(c.execCtx.stackPointerBeforeGoCall) + index := wasm.Index(stack[0]) + mod := c.callerModuleInstance() + listener := mod.Engine.(*moduleEngine).listeners[index] + def := mod.Source.FunctionDefinition(index + mod.Source.ImportFunctionCount) + listener.Before(ctx, mod, def, stack[1:], c.stackIterator(false)) + c.execCtx.exitCode = wazevoapi.ExitCodeOK + afterGoFunctionCallEntrypoint(c.execCtx.goCallReturnAddress, c.execCtxPtr, + uintptr(unsafe.Pointer(c.execCtx.stackPointerBeforeGoCall)), c.execCtx.framePointerBeforeGoCall) + case wazevoapi.ExitCodeCallListenerAfter: + stack := goCallStackView(c.execCtx.stackPointerBeforeGoCall) + index := wasm.Index(stack[0]) + mod := c.callerModuleInstance() + listener := mod.Engine.(*moduleEngine).listeners[index] + def := mod.Source.FunctionDefinition(index + mod.Source.ImportFunctionCount) + listener.After(ctx, mod, def, stack[1:]) + c.execCtx.exitCode = wazevoapi.ExitCodeOK + afterGoFunctionCallEntrypoint(c.execCtx.goCallReturnAddress, c.execCtxPtr, + uintptr(unsafe.Pointer(c.execCtx.stackPointerBeforeGoCall)), c.execCtx.framePointerBeforeGoCall) + case wazevoapi.ExitCodeCheckModuleExitCode: + // Note: this operation must be done in Go, not native code. The reason is that + // native code cannot be preempted and that means it can block forever if there are not + // enough OS threads (which we don't have control over). + if err := m.FailIfClosed(); err != nil { + panic(err) + } + c.execCtx.exitCode = wazevoapi.ExitCodeOK + afterGoFunctionCallEntrypoint(c.execCtx.goCallReturnAddress, c.execCtxPtr, + uintptr(unsafe.Pointer(c.execCtx.stackPointerBeforeGoCall)), c.execCtx.framePointerBeforeGoCall) + case wazevoapi.ExitCodeRefFunc: + mod := c.callerModuleInstance() + s := goCallStackView(c.execCtx.stackPointerBeforeGoCall) + funcIndex := wasm.Index(s[0]) + ref := mod.Engine.FunctionInstanceReference(funcIndex) + s[0] = uint64(ref) + c.execCtx.exitCode = wazevoapi.ExitCodeOK + afterGoFunctionCallEntrypoint(c.execCtx.goCallReturnAddress, c.execCtxPtr, + uintptr(unsafe.Pointer(c.execCtx.stackPointerBeforeGoCall)), c.execCtx.framePointerBeforeGoCall) + case wazevoapi.ExitCodeMemoryWait32: + mod := c.callerModuleInstance() + mem := mod.MemoryInstance + if !mem.Shared { + panic(wasmruntime.ErrRuntimeExpectedSharedMemory) + } + + s := goCallStackView(c.execCtx.stackPointerBeforeGoCall) + timeout, exp, addr := int64(s[0]), uint32(s[1]), uintptr(s[2]) + base := uintptr(unsafe.Pointer(&mem.Buffer[0])) + + offset := uint32(addr - base) + res := mem.Wait32(offset, exp, timeout, func(mem *wasm.MemoryInstance, offset uint32) uint32 { + addr := unsafe.Add(unsafe.Pointer(&mem.Buffer[0]), offset) + return atomic.LoadUint32((*uint32)(addr)) + }) + s[0] = res + c.execCtx.exitCode = wazevoapi.ExitCodeOK + afterGoFunctionCallEntrypoint(c.execCtx.goCallReturnAddress, c.execCtxPtr, + uintptr(unsafe.Pointer(c.execCtx.stackPointerBeforeGoCall)), c.execCtx.framePointerBeforeGoCall) + case wazevoapi.ExitCodeMemoryWait64: + mod := c.callerModuleInstance() + mem := mod.MemoryInstance + if !mem.Shared { + panic(wasmruntime.ErrRuntimeExpectedSharedMemory) + } + + s := goCallStackView(c.execCtx.stackPointerBeforeGoCall) + timeout, exp, addr := int64(s[0]), uint64(s[1]), uintptr(s[2]) + base := uintptr(unsafe.Pointer(&mem.Buffer[0])) + + offset := uint32(addr - base) + res := mem.Wait64(offset, exp, timeout, func(mem *wasm.MemoryInstance, offset uint32) uint64 { + addr := unsafe.Add(unsafe.Pointer(&mem.Buffer[0]), offset) + return atomic.LoadUint64((*uint64)(addr)) + }) + s[0] = uint64(res) + c.execCtx.exitCode = wazevoapi.ExitCodeOK + afterGoFunctionCallEntrypoint(c.execCtx.goCallReturnAddress, c.execCtxPtr, + uintptr(unsafe.Pointer(c.execCtx.stackPointerBeforeGoCall)), c.execCtx.framePointerBeforeGoCall) + case wazevoapi.ExitCodeMemoryNotify: + mod := c.callerModuleInstance() + mem := mod.MemoryInstance + + s := goCallStackView(c.execCtx.stackPointerBeforeGoCall) + count, addr := uint32(s[0]), s[1] + offset := uint32(uintptr(addr) - uintptr(unsafe.Pointer(&mem.Buffer[0]))) + res := mem.Notify(offset, count) + s[0] = uint64(res) + c.execCtx.exitCode = wazevoapi.ExitCodeOK + afterGoFunctionCallEntrypoint(c.execCtx.goCallReturnAddress, c.execCtxPtr, + uintptr(unsafe.Pointer(c.execCtx.stackPointerBeforeGoCall)), c.execCtx.framePointerBeforeGoCall) + case wazevoapi.ExitCodeUnreachable: + panic(wasmruntime.ErrRuntimeUnreachable) + case wazevoapi.ExitCodeMemoryOutOfBounds: + panic(wasmruntime.ErrRuntimeOutOfBoundsMemoryAccess) + case wazevoapi.ExitCodeTableOutOfBounds: + panic(wasmruntime.ErrRuntimeInvalidTableAccess) + case wazevoapi.ExitCodeIndirectCallNullPointer: + panic(wasmruntime.ErrRuntimeInvalidTableAccess) + case wazevoapi.ExitCodeIndirectCallTypeMismatch: + panic(wasmruntime.ErrRuntimeIndirectCallTypeMismatch) + case wazevoapi.ExitCodeIntegerOverflow: + panic(wasmruntime.ErrRuntimeIntegerOverflow) + case wazevoapi.ExitCodeIntegerDivisionByZero: + panic(wasmruntime.ErrRuntimeIntegerDivideByZero) + case wazevoapi.ExitCodeInvalidConversionToInteger: + panic(wasmruntime.ErrRuntimeInvalidConversionToInteger) + case wazevoapi.ExitCodeUnalignedAtomic: + panic(wasmruntime.ErrRuntimeUnalignedAtomic) + default: + panic("BUG") + } + } +} + +func (c *callEngine) callerModuleInstance() *wasm.ModuleInstance { + return moduleInstanceFromOpaquePtr(c.execCtx.callerModuleContextPtr) +} + +func opaqueViewFromPtr(ptr uintptr) []byte { + var opaque []byte + sh := (*reflect.SliceHeader)(unsafe.Pointer(&opaque)) + sh.Data = ptr + setSliceLimits(sh, 24, 24) + return opaque +} + +const callStackCeiling = uintptr(50000000) // in uint64 (8 bytes) == 400000000 bytes in total == 400mb. + +func (c *callEngine) growStackWithGuarded() (newSP uintptr, newFP uintptr, err error) { + if wazevoapi.StackGuardCheckEnabled { + wazevoapi.CheckStackGuardPage(c.stack) + } + newSP, newFP, err = c.growStack() + if err != nil { + return + } + if wazevoapi.StackGuardCheckEnabled { + c.execCtx.stackBottomPtr = &c.stack[wazevoapi.StackGuardCheckGuardPageSize] + } + return +} + +// growStack grows the stack, and returns the new stack pointer. +func (c *callEngine) growStack() (newSP, newFP uintptr, err error) { + currentLen := uintptr(len(c.stack)) + if callStackCeiling < currentLen { + err = wasmruntime.ErrRuntimeStackOverflow + return + } + + newLen := 2*currentLen + c.execCtx.stackGrowRequiredSize + 16 // Stack might be aligned to 16 bytes, so add 16 bytes just in case. + newSP, newFP, c.stackTop, c.stack = c.cloneStack(newLen) + c.execCtx.stackBottomPtr = &c.stack[0] + return +} + +func (c *callEngine) cloneStack(l uintptr) (newSP, newFP, newTop uintptr, newStack []byte) { + newStack = make([]byte, l) + + relSp := c.stackTop - uintptr(unsafe.Pointer(c.execCtx.stackPointerBeforeGoCall)) + relFp := c.stackTop - c.execCtx.framePointerBeforeGoCall + + // Copy the existing contents in the previous Go-allocated stack into the new one. + var prevStackAligned, newStackAligned []byte + { + sh := (*reflect.SliceHeader)(unsafe.Pointer(&prevStackAligned)) + sh.Data = c.stackTop - relSp + setSliceLimits(sh, relSp, relSp) + } + newTop = alignedStackTop(newStack) + { + newSP = newTop - relSp + newFP = newTop - relFp + sh := (*reflect.SliceHeader)(unsafe.Pointer(&newStackAligned)) + sh.Data = newSP + setSliceLimits(sh, relSp, relSp) + } + copy(newStackAligned, prevStackAligned) + return +} + +func (c *callEngine) stackIterator(onHostCall bool) experimental.StackIterator { + c.stackIteratorImpl.reset(c, onHostCall) + return &c.stackIteratorImpl +} + +// stackIterator implements experimental.StackIterator. +type stackIterator struct { + retAddrs []uintptr + retAddrCursor int + eng *engine + pc uint64 + + currentDef *wasm.FunctionDefinition +} + +func (si *stackIterator) reset(c *callEngine, onHostCall bool) { + if onHostCall { + si.retAddrs = append(si.retAddrs[:0], uintptr(unsafe.Pointer(c.execCtx.goCallReturnAddress))) + } else { + si.retAddrs = si.retAddrs[:0] + } + si.retAddrs = unwindStack(uintptr(unsafe.Pointer(c.execCtx.stackPointerBeforeGoCall)), c.execCtx.framePointerBeforeGoCall, c.stackTop, si.retAddrs) + si.retAddrs = si.retAddrs[:len(si.retAddrs)-1] // the last return addr is the trampoline, so we skip it. + si.retAddrCursor = 0 + si.eng = c.parent.parent.parent +} + +// Next implements the same method as documented on experimental.StackIterator. +func (si *stackIterator) Next() bool { + if si.retAddrCursor >= len(si.retAddrs) { + return false + } + + addr := si.retAddrs[si.retAddrCursor] + cm := si.eng.compiledModuleOfAddr(addr) + if cm != nil { + index := cm.functionIndexOf(addr) + def := cm.module.FunctionDefinition(cm.module.ImportFunctionCount + index) + si.currentDef = def + si.retAddrCursor++ + si.pc = uint64(addr) + return true + } + return false +} + +// ProgramCounter implements the same method as documented on experimental.StackIterator. +func (si *stackIterator) ProgramCounter() experimental.ProgramCounter { + return experimental.ProgramCounter(si.pc) +} + +// Function implements the same method as documented on experimental.StackIterator. +func (si *stackIterator) Function() experimental.InternalFunction { + return si +} + +// Definition implements the same method as documented on experimental.InternalFunction. +func (si *stackIterator) Definition() api.FunctionDefinition { + return si.currentDef +} + +// SourceOffsetForPC implements the same method as documented on experimental.InternalFunction. +func (si *stackIterator) SourceOffsetForPC(pc experimental.ProgramCounter) uint64 { + upc := uintptr(pc) + cm := si.eng.compiledModuleOfAddr(upc) + return cm.getSourceOffset(upc) +} + +// snapshot implements experimental.Snapshot +type snapshot struct { + sp, fp, top uintptr + returnAddress *byte + stack []byte + savedRegisters [64][2]uint64 + ret []uint64 + c *callEngine +} + +// Snapshot implements the same method as documented on experimental.Snapshotter. +func (c *callEngine) Snapshot() experimental.Snapshot { + returnAddress := c.execCtx.goCallReturnAddress + oldTop, oldSp := c.stackTop, uintptr(unsafe.Pointer(c.execCtx.stackPointerBeforeGoCall)) + newSP, newFP, newTop, newStack := c.cloneStack(uintptr(len(c.stack)) + 16) + adjustClonedStack(oldSp, oldTop, newSP, newFP, newTop) + return &snapshot{ + sp: newSP, + fp: newFP, + top: newTop, + savedRegisters: c.execCtx.savedRegisters, + returnAddress: returnAddress, + stack: newStack, + c: c, + } +} + +// Restore implements the same method as documented on experimental.Snapshot. +func (s *snapshot) Restore(ret []uint64) { + s.ret = ret + panic(s) +} + +func (s *snapshot) doRestore() { + spp := *(**uint64)(unsafe.Pointer(&s.sp)) + view := goCallStackView(spp) + copy(view, s.ret) + + c := s.c + c.stack = s.stack + c.stackTop = s.top + ec := &c.execCtx + ec.stackBottomPtr = &c.stack[0] + ec.stackPointerBeforeGoCall = spp + ec.framePointerBeforeGoCall = s.fp + ec.goCallReturnAddress = s.returnAddress + ec.savedRegisters = s.savedRegisters +} + +// Error implements the same method on error. +func (s *snapshot) Error() string { + return "unhandled snapshot restore, this generally indicates restore was called from a different " + + "exported function invocation than snapshot" +} + +func snapshotRecoverFn(c *callEngine) { + if r := recover(); r != nil { + if s, ok := r.(*snapshot); ok && s.c == c { + s.doRestore() + } else { + panic(r) + } + } +} diff --git a/vendor/github.com/tetratelabs/wazero/internal/engine/wazevo/engine.go b/vendor/github.com/tetratelabs/wazero/internal/engine/wazevo/engine.go new file mode 100644 index 000000000..f02b905fc --- /dev/null +++ b/vendor/github.com/tetratelabs/wazero/internal/engine/wazevo/engine.go @@ -0,0 +1,843 @@ +package wazevo + +import ( + "context" + "encoding/hex" + "errors" + "fmt" + "runtime" + "sort" + "sync" + "unsafe" + + "github.com/tetratelabs/wazero/api" + "github.com/tetratelabs/wazero/experimental" + "github.com/tetratelabs/wazero/internal/engine/wazevo/backend" + "github.com/tetratelabs/wazero/internal/engine/wazevo/frontend" + "github.com/tetratelabs/wazero/internal/engine/wazevo/ssa" + "github.com/tetratelabs/wazero/internal/engine/wazevo/wazevoapi" + "github.com/tetratelabs/wazero/internal/filecache" + "github.com/tetratelabs/wazero/internal/platform" + "github.com/tetratelabs/wazero/internal/version" + "github.com/tetratelabs/wazero/internal/wasm" +) + +type ( + // engine implements wasm.Engine. + engine struct { + wazeroVersion string + fileCache filecache.Cache + compiledModules map[wasm.ModuleID]*compiledModule + // sortedCompiledModules is a list of compiled modules sorted by the initial address of the executable. + sortedCompiledModules []*compiledModule + mux sync.RWMutex + // sharedFunctions is compiled functions shared by all modules. + sharedFunctions *sharedFunctions + // setFinalizer defaults to runtime.SetFinalizer, but overridable for tests. + setFinalizer func(obj interface{}, finalizer interface{}) + + // The followings are reused for compiling shared functions. + machine backend.Machine + be backend.Compiler + } + + sharedFunctions struct { + // memoryGrowExecutable is a compiled trampoline executable for memory.grow builtin function. + memoryGrowExecutable []byte + // checkModuleExitCode is a compiled trampoline executable for checking module instance exit code. This + // is used when ensureTermination is true. + checkModuleExitCode []byte + // stackGrowExecutable is a compiled executable for growing stack builtin function. + stackGrowExecutable []byte + // tableGrowExecutable is a compiled trampoline executable for table.grow builtin function. + tableGrowExecutable []byte + // refFuncExecutable is a compiled trampoline executable for ref.func builtin function. + refFuncExecutable []byte + // memoryWait32Executable is a compiled trampoline executable for memory.wait32 builtin function + memoryWait32Executable []byte + // memoryWait64Executable is a compiled trampoline executable for memory.wait64 builtin function + memoryWait64Executable []byte + // memoryNotifyExecutable is a compiled trampoline executable for memory.notify builtin function + memoryNotifyExecutable []byte + listenerBeforeTrampolines map[*wasm.FunctionType][]byte + listenerAfterTrampolines map[*wasm.FunctionType][]byte + } + + // compiledModule is a compiled variant of a wasm.Module and ready to be used for instantiation. + compiledModule struct { + *executables + // functionOffsets maps a local function index to the offset in the executable. + functionOffsets []int + parent *engine + module *wasm.Module + ensureTermination bool + listeners []experimental.FunctionListener + listenerBeforeTrampolines []*byte + listenerAfterTrampolines []*byte + + // The followings are only available for non host modules. + + offsets wazevoapi.ModuleContextOffsetData + sharedFunctions *sharedFunctions + sourceMap sourceMap + } + + executables struct { + executable []byte + entryPreambles [][]byte + } +) + +// sourceMap is a mapping from the offset of the executable to the offset of the original wasm binary. +type sourceMap struct { + // executableOffsets is a sorted list of offsets of the executable. This is index-correlated with wasmBinaryOffsets, + // in other words executableOffsets[i] is the offset of the executable which corresponds to the offset of a Wasm + // binary pointed by wasmBinaryOffsets[i]. + executableOffsets []uintptr + // wasmBinaryOffsets is the counterpart of executableOffsets. + wasmBinaryOffsets []uint64 +} + +var _ wasm.Engine = (*engine)(nil) + +// NewEngine returns the implementation of wasm.Engine. +func NewEngine(ctx context.Context, _ api.CoreFeatures, fc filecache.Cache) wasm.Engine { + machine := newMachine() + be := backend.NewCompiler(ctx, machine, ssa.NewBuilder()) + e := &engine{ + compiledModules: make(map[wasm.ModuleID]*compiledModule), + setFinalizer: runtime.SetFinalizer, + machine: machine, + be: be, + fileCache: fc, + wazeroVersion: version.GetWazeroVersion(), + } + e.compileSharedFunctions() + return e +} + +// CompileModule implements wasm.Engine. +func (e *engine) CompileModule(ctx context.Context, module *wasm.Module, listeners []experimental.FunctionListener, ensureTermination bool) (err error) { + if wazevoapi.PerfMapEnabled { + wazevoapi.PerfMap.Lock() + defer wazevoapi.PerfMap.Unlock() + } + + if _, ok, err := e.getCompiledModule(module, listeners, ensureTermination); ok { // cache hit! + return nil + } else if err != nil { + return err + } + + if wazevoapi.DeterministicCompilationVerifierEnabled { + ctx = wazevoapi.NewDeterministicCompilationVerifierContext(ctx, len(module.CodeSection)) + } + cm, err := e.compileModule(ctx, module, listeners, ensureTermination) + if err != nil { + return err + } + if err = e.addCompiledModule(module, cm); err != nil { + return err + } + + if wazevoapi.DeterministicCompilationVerifierEnabled { + for i := 0; i < wazevoapi.DeterministicCompilationVerifyingIter; i++ { + _, err := e.compileModule(ctx, module, listeners, ensureTermination) + if err != nil { + return err + } + } + } + + if len(listeners) > 0 { + cm.listeners = listeners + cm.listenerBeforeTrampolines = make([]*byte, len(module.TypeSection)) + cm.listenerAfterTrampolines = make([]*byte, len(module.TypeSection)) + for i := range module.TypeSection { + typ := &module.TypeSection[i] + before, after := e.getListenerTrampolineForType(typ) + cm.listenerBeforeTrampolines[i] = before + cm.listenerAfterTrampolines[i] = after + } + } + return nil +} + +func (exec *executables) compileEntryPreambles(m *wasm.Module, machine backend.Machine, be backend.Compiler) { + exec.entryPreambles = make([][]byte, len(m.TypeSection)) + for i := range m.TypeSection { + typ := &m.TypeSection[i] + sig := frontend.SignatureForWasmFunctionType(typ) + be.Init() + buf := machine.CompileEntryPreamble(&sig) + executable := mmapExecutable(buf) + exec.entryPreambles[i] = executable + + if wazevoapi.PerfMapEnabled { + wazevoapi.PerfMap.AddEntry(uintptr(unsafe.Pointer(&executable[0])), + uint64(len(executable)), fmt.Sprintf("entry_preamble::type=%s", typ.String())) + } + } +} + +func (e *engine) compileModule(ctx context.Context, module *wasm.Module, listeners []experimental.FunctionListener, ensureTermination bool) (*compiledModule, error) { + withListener := len(listeners) > 0 + cm := &compiledModule{ + offsets: wazevoapi.NewModuleContextOffsetData(module, withListener), parent: e, module: module, + ensureTermination: ensureTermination, + executables: &executables{}, + } + + if module.IsHostModule { + return e.compileHostModule(ctx, module, listeners) + } + + importedFns, localFns := int(module.ImportFunctionCount), len(module.FunctionSection) + if localFns == 0 { + return cm, nil + } + + rels := make([]backend.RelocationInfo, 0) + refToBinaryOffset := make([]int, importedFns+localFns) + + if wazevoapi.DeterministicCompilationVerifierEnabled { + // The compilation must be deterministic regardless of the order of functions being compiled. + wazevoapi.DeterministicCompilationVerifierRandomizeIndexes(ctx) + } + + needSourceInfo := module.DWARFLines != nil + + // Creates new compiler instances which are reused for each function. + ssaBuilder := ssa.NewBuilder() + fe := frontend.NewFrontendCompiler(module, ssaBuilder, &cm.offsets, ensureTermination, withListener, needSourceInfo) + machine := newMachine() + be := backend.NewCompiler(ctx, machine, ssaBuilder) + + cm.executables.compileEntryPreambles(module, machine, be) + + totalSize := 0 // Total binary size of the executable. + cm.functionOffsets = make([]int, localFns) + bodies := make([][]byte, localFns) + + // Trampoline relocation related variables. + trampolineInterval, callTrampolineIslandSize, err := machine.CallTrampolineIslandInfo(localFns) + if err != nil { + return nil, err + } + needCallTrampoline := callTrampolineIslandSize > 0 + var callTrampolineIslandOffsets []int // Holds the offsets of trampoline islands. + + for i := range module.CodeSection { + if wazevoapi.DeterministicCompilationVerifierEnabled { + i = wazevoapi.DeterministicCompilationVerifierGetRandomizedLocalFunctionIndex(ctx, i) + } + + fidx := wasm.Index(i + importedFns) + + if wazevoapi.NeedFunctionNameInContext { + def := module.FunctionDefinition(fidx) + name := def.DebugName() + if len(def.ExportNames()) > 0 { + name = def.ExportNames()[0] + } + ctx = wazevoapi.SetCurrentFunctionName(ctx, i, fmt.Sprintf("[%d/%d]%s", i, len(module.CodeSection)-1, name)) + } + + needListener := len(listeners) > 0 && listeners[i] != nil + body, relsPerFunc, err := e.compileLocalWasmFunction(ctx, module, wasm.Index(i), fe, ssaBuilder, be, needListener) + if err != nil { + return nil, fmt.Errorf("compile function %d/%d: %v", i, len(module.CodeSection)-1, err) + } + + // Align 16-bytes boundary. + totalSize = (totalSize + 15) &^ 15 + cm.functionOffsets[i] = totalSize + + if needSourceInfo { + // At the beginning of the function, we add the offset of the function body so that + // we can resolve the source location of the call site of before listener call. + cm.sourceMap.executableOffsets = append(cm.sourceMap.executableOffsets, uintptr(totalSize)) + cm.sourceMap.wasmBinaryOffsets = append(cm.sourceMap.wasmBinaryOffsets, module.CodeSection[i].BodyOffsetInCodeSection) + + for _, info := range be.SourceOffsetInfo() { + cm.sourceMap.executableOffsets = append(cm.sourceMap.executableOffsets, uintptr(totalSize)+uintptr(info.ExecutableOffset)) + cm.sourceMap.wasmBinaryOffsets = append(cm.sourceMap.wasmBinaryOffsets, uint64(info.SourceOffset)) + } + } + + fref := frontend.FunctionIndexToFuncRef(fidx) + refToBinaryOffset[fref] = totalSize + + // At this point, relocation offsets are relative to the start of the function body, + // so we adjust it to the start of the executable. + for _, r := range relsPerFunc { + r.Offset += int64(totalSize) + rels = append(rels, r) + } + + bodies[i] = body + totalSize += len(body) + if wazevoapi.PrintMachineCodeHexPerFunction { + fmt.Printf("[[[machine code for %s]]]\n%s\n\n", wazevoapi.GetCurrentFunctionName(ctx), hex.EncodeToString(body)) + } + + if needCallTrampoline { + // If the total size exceeds the trampoline interval, we need to add a trampoline island. + if totalSize/trampolineInterval > len(callTrampolineIslandOffsets) { + callTrampolineIslandOffsets = append(callTrampolineIslandOffsets, totalSize) + totalSize += callTrampolineIslandSize + } + } + } + + // Allocate executable memory and then copy the generated machine code. + executable, err := platform.MmapCodeSegment(totalSize) + if err != nil { + panic(err) + } + cm.executable = executable + + for i, b := range bodies { + offset := cm.functionOffsets[i] + copy(executable[offset:], b) + } + + if wazevoapi.PerfMapEnabled { + wazevoapi.PerfMap.Flush(uintptr(unsafe.Pointer(&executable[0])), cm.functionOffsets) + } + + if needSourceInfo { + for i := range cm.sourceMap.executableOffsets { + cm.sourceMap.executableOffsets[i] += uintptr(unsafe.Pointer(&cm.executable[0])) + } + } + + // Resolve relocations for local function calls. + if len(rels) > 0 { + machine.ResolveRelocations(refToBinaryOffset, executable, rels, callTrampolineIslandOffsets) + } + + if runtime.GOARCH == "arm64" { + // On arm64, we cannot give all of rwx at the same time, so we change it to exec. + if err = platform.MprotectRX(executable); err != nil { + return nil, err + } + } + cm.sharedFunctions = e.sharedFunctions + e.setFinalizer(cm.executables, executablesFinalizer) + return cm, nil +} + +func (e *engine) compileLocalWasmFunction( + ctx context.Context, + module *wasm.Module, + localFunctionIndex wasm.Index, + fe *frontend.Compiler, + ssaBuilder ssa.Builder, + be backend.Compiler, + needListener bool, +) (body []byte, rels []backend.RelocationInfo, err error) { + typIndex := module.FunctionSection[localFunctionIndex] + typ := &module.TypeSection[typIndex] + codeSeg := &module.CodeSection[localFunctionIndex] + + // Initializes both frontend and backend compilers. + fe.Init(localFunctionIndex, typIndex, typ, codeSeg.LocalTypes, codeSeg.Body, needListener, codeSeg.BodyOffsetInCodeSection) + be.Init() + + // Lower Wasm to SSA. + fe.LowerToSSA() + if wazevoapi.PrintSSA && wazevoapi.PrintEnabledIndex(ctx) { + fmt.Printf("[[[SSA for %s]]]%s\n", wazevoapi.GetCurrentFunctionName(ctx), ssaBuilder.Format()) + } + + if wazevoapi.DeterministicCompilationVerifierEnabled { + wazevoapi.VerifyOrSetDeterministicCompilationContextValue(ctx, "SSA", ssaBuilder.Format()) + } + + // Run SSA-level optimization passes. + ssaBuilder.RunPasses() + + if wazevoapi.PrintOptimizedSSA && wazevoapi.PrintEnabledIndex(ctx) { + fmt.Printf("[[[Optimized SSA for %s]]]%s\n", wazevoapi.GetCurrentFunctionName(ctx), ssaBuilder.Format()) + } + + if wazevoapi.DeterministicCompilationVerifierEnabled { + wazevoapi.VerifyOrSetDeterministicCompilationContextValue(ctx, "Optimized SSA", ssaBuilder.Format()) + } + + // Now our ssaBuilder contains the necessary information to further lower them to + // machine code. + original, rels, err := be.Compile(ctx) + if err != nil { + return nil, nil, fmt.Errorf("ssa->machine code: %v", err) + } + + // TODO: optimize as zero copy. + copied := make([]byte, len(original)) + copy(copied, original) + return copied, rels, nil +} + +func (e *engine) compileHostModule(ctx context.Context, module *wasm.Module, listeners []experimental.FunctionListener) (*compiledModule, error) { + machine := newMachine() + be := backend.NewCompiler(ctx, machine, ssa.NewBuilder()) + + num := len(module.CodeSection) + cm := &compiledModule{module: module, listeners: listeners, executables: &executables{}} + cm.functionOffsets = make([]int, num) + totalSize := 0 // Total binary size of the executable. + bodies := make([][]byte, num) + var sig ssa.Signature + for i := range module.CodeSection { + totalSize = (totalSize + 15) &^ 15 + cm.functionOffsets[i] = totalSize + + typIndex := module.FunctionSection[i] + typ := &module.TypeSection[typIndex] + + // We can relax until the index fits together in ExitCode as we do in wazevoapi.ExitCodeCallGoModuleFunctionWithIndex. + // However, 1 << 16 should be large enough for a real use case. + const hostFunctionNumMaximum = 1 << 16 + if i >= hostFunctionNumMaximum { + return nil, fmt.Errorf("too many host functions (maximum %d)", hostFunctionNumMaximum) + } + + sig.ID = ssa.SignatureID(typIndex) // This is important since we reuse the `machine` which caches the ABI based on the SignatureID. + sig.Params = append(sig.Params[:0], + ssa.TypeI64, // First argument must be exec context. + ssa.TypeI64, // The second argument is the moduleContextOpaque of this host module. + ) + for _, t := range typ.Params { + sig.Params = append(sig.Params, frontend.WasmTypeToSSAType(t)) + } + + sig.Results = sig.Results[:0] + for _, t := range typ.Results { + sig.Results = append(sig.Results, frontend.WasmTypeToSSAType(t)) + } + + c := &module.CodeSection[i] + if c.GoFunc == nil { + panic("BUG: GoFunc must be set for host module") + } + + withListener := len(listeners) > 0 && listeners[i] != nil + var exitCode wazevoapi.ExitCode + fn := c.GoFunc + switch fn.(type) { + case api.GoModuleFunction: + exitCode = wazevoapi.ExitCodeCallGoModuleFunctionWithIndex(i, withListener) + case api.GoFunction: + exitCode = wazevoapi.ExitCodeCallGoFunctionWithIndex(i, withListener) + } + + be.Init() + machine.CompileGoFunctionTrampoline(exitCode, &sig, true) + if err := be.Finalize(ctx); err != nil { + return nil, err + } + body := be.Buf() + + if wazevoapi.PerfMapEnabled { + name := module.FunctionDefinition(wasm.Index(i)).DebugName() + wazevoapi.PerfMap.AddModuleEntry(i, + int64(totalSize), + uint64(len(body)), + fmt.Sprintf("trampoline:%s", name)) + } + + // TODO: optimize as zero copy. + copied := make([]byte, len(body)) + copy(copied, body) + bodies[i] = copied + totalSize += len(body) + } + + if totalSize == 0 { + // Empty module. + return cm, nil + } + + // Allocate executable memory and then copy the generated machine code. + executable, err := platform.MmapCodeSegment(totalSize) + if err != nil { + panic(err) + } + cm.executable = executable + + for i, b := range bodies { + offset := cm.functionOffsets[i] + copy(executable[offset:], b) + } + + if wazevoapi.PerfMapEnabled { + wazevoapi.PerfMap.Flush(uintptr(unsafe.Pointer(&executable[0])), cm.functionOffsets) + } + + if runtime.GOARCH == "arm64" { + // On arm64, we cannot give all of rwx at the same time, so we change it to exec. + if err = platform.MprotectRX(executable); err != nil { + return nil, err + } + } + e.setFinalizer(cm.executables, executablesFinalizer) + return cm, nil +} + +// Close implements wasm.Engine. +func (e *engine) Close() (err error) { + e.mux.Lock() + defer e.mux.Unlock() + e.sortedCompiledModules = nil + e.compiledModules = nil + e.sharedFunctions = nil + return nil +} + +// CompiledModuleCount implements wasm.Engine. +func (e *engine) CompiledModuleCount() uint32 { + e.mux.RLock() + defer e.mux.RUnlock() + return uint32(len(e.compiledModules)) +} + +// DeleteCompiledModule implements wasm.Engine. +func (e *engine) DeleteCompiledModule(m *wasm.Module) { + e.mux.Lock() + defer e.mux.Unlock() + cm, ok := e.compiledModules[m.ID] + if ok { + if len(cm.executable) > 0 { + e.deleteCompiledModuleFromSortedList(cm) + } + delete(e.compiledModules, m.ID) + } +} + +func (e *engine) addCompiledModuleToSortedList(cm *compiledModule) { + ptr := uintptr(unsafe.Pointer(&cm.executable[0])) + + index := sort.Search(len(e.sortedCompiledModules), func(i int) bool { + return uintptr(unsafe.Pointer(&e.sortedCompiledModules[i].executable[0])) >= ptr + }) + e.sortedCompiledModules = append(e.sortedCompiledModules, nil) + copy(e.sortedCompiledModules[index+1:], e.sortedCompiledModules[index:]) + e.sortedCompiledModules[index] = cm +} + +func (e *engine) deleteCompiledModuleFromSortedList(cm *compiledModule) { + ptr := uintptr(unsafe.Pointer(&cm.executable[0])) + + index := sort.Search(len(e.sortedCompiledModules), func(i int) bool { + return uintptr(unsafe.Pointer(&e.sortedCompiledModules[i].executable[0])) >= ptr + }) + if index >= len(e.sortedCompiledModules) { + return + } + copy(e.sortedCompiledModules[index:], e.sortedCompiledModules[index+1:]) + e.sortedCompiledModules = e.sortedCompiledModules[:len(e.sortedCompiledModules)-1] +} + +func (e *engine) compiledModuleOfAddr(addr uintptr) *compiledModule { + e.mux.RLock() + defer e.mux.RUnlock() + + index := sort.Search(len(e.sortedCompiledModules), func(i int) bool { + return uintptr(unsafe.Pointer(&e.sortedCompiledModules[i].executable[0])) > addr + }) + index -= 1 + if index < 0 { + return nil + } + candidate := e.sortedCompiledModules[index] + if checkAddrInBytes(addr, candidate.executable) { + // If a module is already deleted, the found module may have been wrong. + return candidate + } + return nil +} + +func checkAddrInBytes(addr uintptr, b []byte) bool { + return uintptr(unsafe.Pointer(&b[0])) <= addr && addr <= uintptr(unsafe.Pointer(&b[len(b)-1])) +} + +// NewModuleEngine implements wasm.Engine. +func (e *engine) NewModuleEngine(m *wasm.Module, mi *wasm.ModuleInstance) (wasm.ModuleEngine, error) { + me := &moduleEngine{} + + // Note: imported functions are resolved in moduleEngine.ResolveImportedFunction. + me.importedFunctions = make([]importedFunction, m.ImportFunctionCount) + + compiled, ok := e.getCompiledModuleFromMemory(m) + if !ok { + return nil, errors.New("source module must be compiled before instantiation") + } + me.parent = compiled + me.module = mi + me.listeners = compiled.listeners + + if m.IsHostModule { + me.opaque = buildHostModuleOpaque(m, compiled.listeners) + me.opaquePtr = &me.opaque[0] + } else { + if size := compiled.offsets.TotalSize; size != 0 { + opaque := newAlignedOpaque(size) + me.opaque = opaque + me.opaquePtr = &opaque[0] + } + } + return me, nil +} + +func (e *engine) compileSharedFunctions() { + e.sharedFunctions = &sharedFunctions{ + listenerBeforeTrampolines: make(map[*wasm.FunctionType][]byte), + listenerAfterTrampolines: make(map[*wasm.FunctionType][]byte), + } + + e.be.Init() + { + src := e.machine.CompileGoFunctionTrampoline(wazevoapi.ExitCodeGrowMemory, &ssa.Signature{ + Params: []ssa.Type{ssa.TypeI64 /* exec context */, ssa.TypeI32}, + Results: []ssa.Type{ssa.TypeI32}, + }, false) + e.sharedFunctions.memoryGrowExecutable = mmapExecutable(src) + if wazevoapi.PerfMapEnabled { + exe := e.sharedFunctions.memoryGrowExecutable + wazevoapi.PerfMap.AddEntry(uintptr(unsafe.Pointer(&exe[0])), uint64(len(exe)), "memory_grow_trampoline") + } + } + + e.be.Init() + { + src := e.machine.CompileGoFunctionTrampoline(wazevoapi.ExitCodeTableGrow, &ssa.Signature{ + Params: []ssa.Type{ssa.TypeI64 /* exec context */, ssa.TypeI32 /* table index */, ssa.TypeI32 /* num */, ssa.TypeI64 /* ref */}, + Results: []ssa.Type{ssa.TypeI32}, + }, false) + e.sharedFunctions.tableGrowExecutable = mmapExecutable(src) + if wazevoapi.PerfMapEnabled { + exe := e.sharedFunctions.tableGrowExecutable + wazevoapi.PerfMap.AddEntry(uintptr(unsafe.Pointer(&exe[0])), uint64(len(exe)), "table_grow_trampoline") + } + } + + e.be.Init() + { + src := e.machine.CompileGoFunctionTrampoline(wazevoapi.ExitCodeCheckModuleExitCode, &ssa.Signature{ + Params: []ssa.Type{ssa.TypeI32 /* exec context */}, + Results: []ssa.Type{ssa.TypeI32}, + }, false) + e.sharedFunctions.checkModuleExitCode = mmapExecutable(src) + if wazevoapi.PerfMapEnabled { + exe := e.sharedFunctions.checkModuleExitCode + wazevoapi.PerfMap.AddEntry(uintptr(unsafe.Pointer(&exe[0])), uint64(len(exe)), "check_module_exit_code_trampoline") + } + } + + e.be.Init() + { + src := e.machine.CompileGoFunctionTrampoline(wazevoapi.ExitCodeRefFunc, &ssa.Signature{ + Params: []ssa.Type{ssa.TypeI64 /* exec context */, ssa.TypeI32 /* function index */}, + Results: []ssa.Type{ssa.TypeI64}, // returns the function reference. + }, false) + e.sharedFunctions.refFuncExecutable = mmapExecutable(src) + if wazevoapi.PerfMapEnabled { + exe := e.sharedFunctions.refFuncExecutable + wazevoapi.PerfMap.AddEntry(uintptr(unsafe.Pointer(&exe[0])), uint64(len(exe)), "ref_func_trampoline") + } + } + + e.be.Init() + { + src := e.machine.CompileStackGrowCallSequence() + e.sharedFunctions.stackGrowExecutable = mmapExecutable(src) + if wazevoapi.PerfMapEnabled { + exe := e.sharedFunctions.stackGrowExecutable + wazevoapi.PerfMap.AddEntry(uintptr(unsafe.Pointer(&exe[0])), uint64(len(exe)), "stack_grow_trampoline") + } + } + + e.be.Init() + { + src := e.machine.CompileGoFunctionTrampoline(wazevoapi.ExitCodeMemoryWait32, &ssa.Signature{ + // exec context, timeout, expected, addr + Params: []ssa.Type{ssa.TypeI64, ssa.TypeI64, ssa.TypeI32, ssa.TypeI64}, + // Returns the status. + Results: []ssa.Type{ssa.TypeI32}, + }, false) + e.sharedFunctions.memoryWait32Executable = mmapExecutable(src) + if wazevoapi.PerfMapEnabled { + exe := e.sharedFunctions.memoryWait32Executable + wazevoapi.PerfMap.AddEntry(uintptr(unsafe.Pointer(&exe[0])), uint64(len(exe)), "memory_wait32_trampoline") + } + } + + e.be.Init() + { + src := e.machine.CompileGoFunctionTrampoline(wazevoapi.ExitCodeMemoryWait64, &ssa.Signature{ + // exec context, timeout, expected, addr + Params: []ssa.Type{ssa.TypeI64, ssa.TypeI64, ssa.TypeI64, ssa.TypeI64}, + // Returns the status. + Results: []ssa.Type{ssa.TypeI32}, + }, false) + e.sharedFunctions.memoryWait64Executable = mmapExecutable(src) + if wazevoapi.PerfMapEnabled { + exe := e.sharedFunctions.memoryWait64Executable + wazevoapi.PerfMap.AddEntry(uintptr(unsafe.Pointer(&exe[0])), uint64(len(exe)), "memory_wait64_trampoline") + } + } + + e.be.Init() + { + src := e.machine.CompileGoFunctionTrampoline(wazevoapi.ExitCodeMemoryNotify, &ssa.Signature{ + // exec context, count, addr + Params: []ssa.Type{ssa.TypeI64, ssa.TypeI32, ssa.TypeI64}, + // Returns the number notified. + Results: []ssa.Type{ssa.TypeI32}, + }, false) + e.sharedFunctions.memoryNotifyExecutable = mmapExecutable(src) + if wazevoapi.PerfMapEnabled { + exe := e.sharedFunctions.memoryNotifyExecutable + wazevoapi.PerfMap.AddEntry(uintptr(unsafe.Pointer(&exe[0])), uint64(len(exe)), "memory_notify_trampoline") + } + } + + e.setFinalizer(e.sharedFunctions, sharedFunctionsFinalizer) +} + +func sharedFunctionsFinalizer(sf *sharedFunctions) { + if err := platform.MunmapCodeSegment(sf.memoryGrowExecutable); err != nil { + panic(err) + } + if err := platform.MunmapCodeSegment(sf.checkModuleExitCode); err != nil { + panic(err) + } + if err := platform.MunmapCodeSegment(sf.stackGrowExecutable); err != nil { + panic(err) + } + if err := platform.MunmapCodeSegment(sf.tableGrowExecutable); err != nil { + panic(err) + } + if err := platform.MunmapCodeSegment(sf.refFuncExecutable); err != nil { + panic(err) + } + if err := platform.MunmapCodeSegment(sf.memoryWait32Executable); err != nil { + panic(err) + } + if err := platform.MunmapCodeSegment(sf.memoryWait64Executable); err != nil { + panic(err) + } + if err := platform.MunmapCodeSegment(sf.memoryNotifyExecutable); err != nil { + panic(err) + } + for _, f := range sf.listenerBeforeTrampolines { + if err := platform.MunmapCodeSegment(f); err != nil { + panic(err) + } + } + for _, f := range sf.listenerAfterTrampolines { + if err := platform.MunmapCodeSegment(f); err != nil { + panic(err) + } + } + + sf.memoryGrowExecutable = nil + sf.checkModuleExitCode = nil + sf.stackGrowExecutable = nil + sf.tableGrowExecutable = nil + sf.refFuncExecutable = nil + sf.memoryWait32Executable = nil + sf.memoryWait64Executable = nil + sf.memoryNotifyExecutable = nil + sf.listenerBeforeTrampolines = nil + sf.listenerAfterTrampolines = nil +} + +func executablesFinalizer(exec *executables) { + if len(exec.executable) > 0 { + if err := platform.MunmapCodeSegment(exec.executable); err != nil { + panic(err) + } + } + exec.executable = nil + + for _, f := range exec.entryPreambles { + if err := platform.MunmapCodeSegment(f); err != nil { + panic(err) + } + } + exec.entryPreambles = nil +} + +func mmapExecutable(src []byte) []byte { + executable, err := platform.MmapCodeSegment(len(src)) + if err != nil { + panic(err) + } + + copy(executable, src) + + if runtime.GOARCH == "arm64" { + // On arm64, we cannot give all of rwx at the same time, so we change it to exec. + if err = platform.MprotectRX(executable); err != nil { + panic(err) + } + } + return executable +} + +func (cm *compiledModule) functionIndexOf(addr uintptr) wasm.Index { + addr -= uintptr(unsafe.Pointer(&cm.executable[0])) + offset := cm.functionOffsets + index := sort.Search(len(offset), func(i int) bool { + return offset[i] > int(addr) + }) + index-- + if index < 0 { + panic("BUG") + } + return wasm.Index(index) +} + +func (e *engine) getListenerTrampolineForType(functionType *wasm.FunctionType) (before, after *byte) { + e.mux.Lock() + defer e.mux.Unlock() + + beforeBuf, ok := e.sharedFunctions.listenerBeforeTrampolines[functionType] + afterBuf := e.sharedFunctions.listenerAfterTrampolines[functionType] + if ok { + return &beforeBuf[0], &afterBuf[0] + } + + beforeSig, afterSig := frontend.SignatureForListener(functionType) + + e.be.Init() + buf := e.machine.CompileGoFunctionTrampoline(wazevoapi.ExitCodeCallListenerBefore, beforeSig, false) + beforeBuf = mmapExecutable(buf) + + e.be.Init() + buf = e.machine.CompileGoFunctionTrampoline(wazevoapi.ExitCodeCallListenerAfter, afterSig, false) + afterBuf = mmapExecutable(buf) + + e.sharedFunctions.listenerBeforeTrampolines[functionType] = beforeBuf + e.sharedFunctions.listenerAfterTrampolines[functionType] = afterBuf + return &beforeBuf[0], &afterBuf[0] +} + +func (cm *compiledModule) getSourceOffset(pc uintptr) uint64 { + offsets := cm.sourceMap.executableOffsets + if len(offsets) == 0 { + return 0 + } + + index := sort.Search(len(offsets), func(i int) bool { + return offsets[i] >= pc + }) + + index-- + if index < 0 { + return 0 + } + return cm.sourceMap.wasmBinaryOffsets[index] +} diff --git a/vendor/github.com/tetratelabs/wazero/internal/engine/wazevo/engine_cache.go b/vendor/github.com/tetratelabs/wazero/internal/engine/wazevo/engine_cache.go new file mode 100644 index 000000000..f7c0450ae --- /dev/null +++ b/vendor/github.com/tetratelabs/wazero/internal/engine/wazevo/engine_cache.go @@ -0,0 +1,296 @@ +package wazevo + +import ( + "bytes" + "context" + "crypto/sha256" + "encoding/binary" + "fmt" + "hash/crc32" + "io" + "runtime" + "unsafe" + + "github.com/tetratelabs/wazero/experimental" + "github.com/tetratelabs/wazero/internal/engine/wazevo/backend" + "github.com/tetratelabs/wazero/internal/engine/wazevo/ssa" + "github.com/tetratelabs/wazero/internal/engine/wazevo/wazevoapi" + "github.com/tetratelabs/wazero/internal/filecache" + "github.com/tetratelabs/wazero/internal/platform" + "github.com/tetratelabs/wazero/internal/u32" + "github.com/tetratelabs/wazero/internal/u64" + "github.com/tetratelabs/wazero/internal/wasm" +) + +var crc = crc32.MakeTable(crc32.Castagnoli) + +// fileCacheKey returns a key for the file cache. +// In order to avoid collisions with the existing compiler, we do not use m.ID directly, +// but instead we rehash it with magic. +func fileCacheKey(m *wasm.Module) (ret filecache.Key) { + s := sha256.New() + s.Write(m.ID[:]) + s.Write(magic) + s.Sum(ret[:0]) + return +} + +func (e *engine) addCompiledModule(module *wasm.Module, cm *compiledModule) (err error) { + e.addCompiledModuleToMemory(module, cm) + if !module.IsHostModule && e.fileCache != nil { + err = e.addCompiledModuleToCache(module, cm) + } + return +} + +func (e *engine) getCompiledModule(module *wasm.Module, listeners []experimental.FunctionListener, ensureTermination bool) (cm *compiledModule, ok bool, err error) { + cm, ok = e.getCompiledModuleFromMemory(module) + if ok { + return + } + cm, ok, err = e.getCompiledModuleFromCache(module) + if ok { + cm.parent = e + cm.module = module + cm.sharedFunctions = e.sharedFunctions + cm.ensureTermination = ensureTermination + cm.offsets = wazevoapi.NewModuleContextOffsetData(module, len(listeners) > 0) + if len(listeners) > 0 { + cm.listeners = listeners + cm.listenerBeforeTrampolines = make([]*byte, len(module.TypeSection)) + cm.listenerAfterTrampolines = make([]*byte, len(module.TypeSection)) + for i := range module.TypeSection { + typ := &module.TypeSection[i] + before, after := e.getListenerTrampolineForType(typ) + cm.listenerBeforeTrampolines[i] = before + cm.listenerAfterTrampolines[i] = after + } + } + e.addCompiledModuleToMemory(module, cm) + ssaBuilder := ssa.NewBuilder() + machine := newMachine() + be := backend.NewCompiler(context.Background(), machine, ssaBuilder) + cm.executables.compileEntryPreambles(module, machine, be) + + // Set the finalizer. + e.setFinalizer(cm.executables, executablesFinalizer) + } + return +} + +func (e *engine) addCompiledModuleToMemory(m *wasm.Module, cm *compiledModule) { + e.mux.Lock() + defer e.mux.Unlock() + e.compiledModules[m.ID] = cm + if len(cm.executable) > 0 { + e.addCompiledModuleToSortedList(cm) + } +} + +func (e *engine) getCompiledModuleFromMemory(module *wasm.Module) (cm *compiledModule, ok bool) { + e.mux.RLock() + defer e.mux.RUnlock() + cm, ok = e.compiledModules[module.ID] + return +} + +func (e *engine) addCompiledModuleToCache(module *wasm.Module, cm *compiledModule) (err error) { + if e.fileCache == nil || module.IsHostModule { + return + } + err = e.fileCache.Add(fileCacheKey(module), serializeCompiledModule(e.wazeroVersion, cm)) + return +} + +func (e *engine) getCompiledModuleFromCache(module *wasm.Module) (cm *compiledModule, hit bool, err error) { + if e.fileCache == nil || module.IsHostModule { + return + } + + // Check if the entries exist in the external cache. + var cached io.ReadCloser + cached, hit, err = e.fileCache.Get(fileCacheKey(module)) + if !hit || err != nil { + return + } + + // Otherwise, we hit the cache on external cache. + // We retrieve *code structures from `cached`. + var staleCache bool + // Note: cached.Close is ensured to be called in deserializeCodes. + cm, staleCache, err = deserializeCompiledModule(e.wazeroVersion, cached) + if err != nil { + hit = false + return + } else if staleCache { + return nil, false, e.fileCache.Delete(fileCacheKey(module)) + } + return +} + +var magic = []byte{'W', 'A', 'Z', 'E', 'V', 'O'} + +func serializeCompiledModule(wazeroVersion string, cm *compiledModule) io.Reader { + buf := bytes.NewBuffer(nil) + // First 6 byte: WAZEVO header. + buf.Write(magic) + // Next 1 byte: length of version: + buf.WriteByte(byte(len(wazeroVersion))) + // Version of wazero. + buf.WriteString(wazeroVersion) + // Number of *code (== locally defined functions in the module): 4 bytes. + buf.Write(u32.LeBytes(uint32(len(cm.functionOffsets)))) + for _, offset := range cm.functionOffsets { + // The offset of this function in the executable (8 bytes). + buf.Write(u64.LeBytes(uint64(offset))) + } + // The length of code segment (8 bytes). + buf.Write(u64.LeBytes(uint64(len(cm.executable)))) + // Append the native code. + buf.Write(cm.executable) + // Append checksum. + checksum := crc32.Checksum(cm.executable, crc) + buf.Write(u32.LeBytes(checksum)) + if sm := cm.sourceMap; len(sm.executableOffsets) > 0 { + buf.WriteByte(1) // indicates that source map is present. + l := len(sm.wasmBinaryOffsets) + buf.Write(u64.LeBytes(uint64(l))) + executableAddr := uintptr(unsafe.Pointer(&cm.executable[0])) + for i := 0; i < l; i++ { + buf.Write(u64.LeBytes(sm.wasmBinaryOffsets[i])) + // executableOffsets is absolute address, so we need to subtract executableAddr. + buf.Write(u64.LeBytes(uint64(sm.executableOffsets[i] - executableAddr))) + } + } else { + buf.WriteByte(0) // indicates that source map is not present. + } + return bytes.NewReader(buf.Bytes()) +} + +func deserializeCompiledModule(wazeroVersion string, reader io.ReadCloser) (cm *compiledModule, staleCache bool, err error) { + defer reader.Close() + cacheHeaderSize := len(magic) + 1 /* version size */ + len(wazeroVersion) + 4 /* number of functions */ + + // Read the header before the native code. + header := make([]byte, cacheHeaderSize) + n, err := reader.Read(header) + if err != nil { + return nil, false, fmt.Errorf("compilationcache: error reading header: %v", err) + } + + if n != cacheHeaderSize { + return nil, false, fmt.Errorf("compilationcache: invalid header length: %d", n) + } + + if !bytes.Equal(header[:len(magic)], magic) { + return nil, false, fmt.Errorf( + "compilationcache: invalid magic number: got %s but want %s", magic, header[:len(magic)]) + } + + // Check the version compatibility. + versionSize := int(header[len(magic)]) + + cachedVersionBegin, cachedVersionEnd := len(magic)+1, len(magic)+1+versionSize + if cachedVersionEnd >= len(header) { + staleCache = true + return + } else if cachedVersion := string(header[cachedVersionBegin:cachedVersionEnd]); cachedVersion != wazeroVersion { + staleCache = true + return + } + + functionsNum := binary.LittleEndian.Uint32(header[len(header)-4:]) + cm = &compiledModule{functionOffsets: make([]int, functionsNum), executables: &executables{}} + + var eightBytes [8]byte + for i := uint32(0); i < functionsNum; i++ { + // Read the offset of each function in the executable. + var offset uint64 + if offset, err = readUint64(reader, &eightBytes); err != nil { + err = fmt.Errorf("compilationcache: error reading func[%d] executable offset: %v", i, err) + return + } + cm.functionOffsets[i] = int(offset) + } + + executableLen, err := readUint64(reader, &eightBytes) + if err != nil { + err = fmt.Errorf("compilationcache: error reading executable size: %v", err) + return + } + + if executableLen > 0 { + executable, err := platform.MmapCodeSegment(int(executableLen)) + if err != nil { + err = fmt.Errorf("compilationcache: error mmapping executable (len=%d): %v", executableLen, err) + return nil, false, err + } + + _, err = io.ReadFull(reader, executable) + if err != nil { + err = fmt.Errorf("compilationcache: error reading executable (len=%d): %v", executableLen, err) + return nil, false, err + } + + expected := crc32.Checksum(executable, crc) + if _, err = io.ReadFull(reader, eightBytes[:4]); err != nil { + return nil, false, fmt.Errorf("compilationcache: could not read checksum: %v", err) + } else if checksum := binary.LittleEndian.Uint32(eightBytes[:4]); expected != checksum { + return nil, false, fmt.Errorf("compilationcache: checksum mismatch (expected %d, got %d)", expected, checksum) + } + + if runtime.GOARCH == "arm64" { + // On arm64, we cannot give all of rwx at the same time, so we change it to exec. + if err = platform.MprotectRX(executable); err != nil { + return nil, false, err + } + } + cm.executable = executable + } + + if _, err := io.ReadFull(reader, eightBytes[:1]); err != nil { + return nil, false, fmt.Errorf("compilationcache: error reading source map presence: %v", err) + } + + if eightBytes[0] == 1 { + sm := &cm.sourceMap + sourceMapLen, err := readUint64(reader, &eightBytes) + if err != nil { + err = fmt.Errorf("compilationcache: error reading source map length: %v", err) + return nil, false, err + } + executableOffset := uintptr(unsafe.Pointer(&cm.executable[0])) + for i := uint64(0); i < sourceMapLen; i++ { + wasmBinaryOffset, err := readUint64(reader, &eightBytes) + if err != nil { + err = fmt.Errorf("compilationcache: error reading source map[%d] wasm binary offset: %v", i, err) + return nil, false, err + } + executableRelativeOffset, err := readUint64(reader, &eightBytes) + if err != nil { + err = fmt.Errorf("compilationcache: error reading source map[%d] executable offset: %v", i, err) + return nil, false, err + } + sm.wasmBinaryOffsets = append(sm.wasmBinaryOffsets, wasmBinaryOffset) + // executableOffsets is absolute address, so we need to add executableOffset. + sm.executableOffsets = append(sm.executableOffsets, uintptr(executableRelativeOffset)+executableOffset) + } + } + return +} + +// readUint64 strictly reads an uint64 in little-endian byte order, using the +// given array as a buffer. This returns io.EOF if less than 8 bytes were read. +func readUint64(reader io.Reader, b *[8]byte) (uint64, error) { + s := b[0:8] + n, err := reader.Read(s) + if err != nil { + return 0, err + } else if n < 8 { // more strict than reader.Read + return 0, io.EOF + } + + // Read the u64 from the underlying buffer. + ret := binary.LittleEndian.Uint64(s) + return ret, nil +} diff --git a/vendor/github.com/tetratelabs/wazero/internal/engine/wazevo/entrypoint_amd64.go b/vendor/github.com/tetratelabs/wazero/internal/engine/wazevo/entrypoint_amd64.go new file mode 100644 index 000000000..18f60af3a --- /dev/null +++ b/vendor/github.com/tetratelabs/wazero/internal/engine/wazevo/entrypoint_amd64.go @@ -0,0 +1,15 @@ +//go:build amd64 && !tinygo + +package wazevo + +import _ "unsafe" + +// entrypoint is implemented by the backend. +// +//go:linkname entrypoint github.com/tetratelabs/wazero/internal/engine/wazevo/backend/isa/amd64.entrypoint +func entrypoint(preambleExecutable, functionExecutable *byte, executionContextPtr uintptr, moduleContextPtr *byte, paramResultStackPtr *uint64, goAllocatedStackSlicePtr uintptr) + +// entrypoint is implemented by the backend. +// +//go:linkname afterGoFunctionCallEntrypoint github.com/tetratelabs/wazero/internal/engine/wazevo/backend/isa/amd64.afterGoFunctionCallEntrypoint +func afterGoFunctionCallEntrypoint(executable *byte, executionContextPtr uintptr, stackPointer, framePointer uintptr) diff --git a/vendor/github.com/tetratelabs/wazero/internal/engine/wazevo/entrypoint_arm64.go b/vendor/github.com/tetratelabs/wazero/internal/engine/wazevo/entrypoint_arm64.go new file mode 100644 index 000000000..e16d64f65 --- /dev/null +++ b/vendor/github.com/tetratelabs/wazero/internal/engine/wazevo/entrypoint_arm64.go @@ -0,0 +1,15 @@ +//go:build arm64 && !tinygo + +package wazevo + +import _ "unsafe" + +// entrypoint is implemented by the backend. +// +//go:linkname entrypoint github.com/tetratelabs/wazero/internal/engine/wazevo/backend/isa/arm64.entrypoint +func entrypoint(preambleExecutable, functionExecutable *byte, executionContextPtr uintptr, moduleContextPtr *byte, paramResultStackPtr *uint64, goAllocatedStackSlicePtr uintptr) + +// entrypoint is implemented by the backend. +// +//go:linkname afterGoFunctionCallEntrypoint github.com/tetratelabs/wazero/internal/engine/wazevo/backend/isa/arm64.afterGoFunctionCallEntrypoint +func afterGoFunctionCallEntrypoint(executable *byte, executionContextPtr uintptr, stackPointer, framePointer uintptr) diff --git a/vendor/github.com/tetratelabs/wazero/internal/engine/wazevo/entrypoint_others.go b/vendor/github.com/tetratelabs/wazero/internal/engine/wazevo/entrypoint_others.go new file mode 100644 index 000000000..8f9d64b2b --- /dev/null +++ b/vendor/github.com/tetratelabs/wazero/internal/engine/wazevo/entrypoint_others.go @@ -0,0 +1,15 @@ +//go:build (!arm64 && !amd64) || tinygo + +package wazevo + +import ( + "runtime" +) + +func entrypoint(preambleExecutable, functionExecutable *byte, executionContextPtr uintptr, moduleContextPtr *byte, paramResultStackPtr *uint64, goAllocatedStackSlicePtr uintptr) { + panic(runtime.GOARCH) +} + +func afterGoFunctionCallEntrypoint(executable *byte, executionContextPtr uintptr, stackPointer, framePointer uintptr) { + panic(runtime.GOARCH) +} diff --git a/vendor/github.com/tetratelabs/wazero/internal/engine/wazevo/frontend/frontend.go b/vendor/github.com/tetratelabs/wazero/internal/engine/wazevo/frontend/frontend.go new file mode 100644 index 000000000..873a35a55 --- /dev/null +++ b/vendor/github.com/tetratelabs/wazero/internal/engine/wazevo/frontend/frontend.go @@ -0,0 +1,594 @@ +// Package frontend implements the translation of WebAssembly to SSA IR using the ssa package. +package frontend + +import ( + "bytes" + "math" + + "github.com/tetratelabs/wazero/internal/engine/wazevo/ssa" + "github.com/tetratelabs/wazero/internal/engine/wazevo/wazevoapi" + "github.com/tetratelabs/wazero/internal/wasm" +) + +// Compiler is in charge of lowering Wasm to SSA IR, and does the optimization +// on top of it in architecture-independent way. +type Compiler struct { + // Per-module data that is used across all functions. + + m *wasm.Module + offset *wazevoapi.ModuleContextOffsetData + // ssaBuilder is a ssa.Builder used by this frontend. + ssaBuilder ssa.Builder + signatures map[*wasm.FunctionType]*ssa.Signature + listenerSignatures map[*wasm.FunctionType][2]*ssa.Signature + memoryGrowSig ssa.Signature + memoryWait32Sig ssa.Signature + memoryWait64Sig ssa.Signature + memoryNotifySig ssa.Signature + checkModuleExitCodeSig ssa.Signature + tableGrowSig ssa.Signature + refFuncSig ssa.Signature + memmoveSig ssa.Signature + ensureTermination bool + + // Followings are reset by per function. + + // wasmLocalToVariable maps the index (considered as wasm.Index of locals) + // to the corresponding ssa.Variable. + wasmLocalToVariable [] /* local index to */ ssa.Variable + wasmLocalFunctionIndex wasm.Index + wasmFunctionTypeIndex wasm.Index + wasmFunctionTyp *wasm.FunctionType + wasmFunctionLocalTypes []wasm.ValueType + wasmFunctionBody []byte + wasmFunctionBodyOffsetInCodeSection uint64 + memoryBaseVariable, memoryLenVariable ssa.Variable + needMemory bool + memoryShared bool + globalVariables []ssa.Variable + globalVariablesTypes []ssa.Type + mutableGlobalVariablesIndexes []wasm.Index // index to ^. + needListener bool + needSourceOffsetInfo bool + // br is reused during lowering. + br *bytes.Reader + loweringState loweringState + + knownSafeBounds [] /* ssa.ValueID to */ knownSafeBound + knownSafeBoundsSet []ssa.ValueID + + knownSafeBoundsAtTheEndOfBlocks [] /* ssa.BlockID to */ knownSafeBoundsAtTheEndOfBlock + varLengthKnownSafeBoundWithIDPool wazevoapi.VarLengthPool[knownSafeBoundWithID] + + execCtxPtrValue, moduleCtxPtrValue ssa.Value + + // Following are reused for the known safe bounds analysis. + + pointers []int + bounds [][]knownSafeBoundWithID +} + +type ( + // knownSafeBound represents a known safe bound for a value. + knownSafeBound struct { + // bound is a constant upper bound for the value. + bound uint64 + // absoluteAddr is the absolute address of the value. + absoluteAddr ssa.Value + } + // knownSafeBoundWithID is a knownSafeBound with the ID of the value. + knownSafeBoundWithID struct { + knownSafeBound + id ssa.ValueID + } + knownSafeBoundsAtTheEndOfBlock = wazevoapi.VarLength[knownSafeBoundWithID] +) + +var knownSafeBoundsAtTheEndOfBlockNil = wazevoapi.NewNilVarLength[knownSafeBoundWithID]() + +// NewFrontendCompiler returns a frontend Compiler. +func NewFrontendCompiler(m *wasm.Module, ssaBuilder ssa.Builder, offset *wazevoapi.ModuleContextOffsetData, ensureTermination bool, listenerOn bool, sourceInfo bool) *Compiler { + c := &Compiler{ + m: m, + ssaBuilder: ssaBuilder, + br: bytes.NewReader(nil), + offset: offset, + ensureTermination: ensureTermination, + needSourceOffsetInfo: sourceInfo, + varLengthKnownSafeBoundWithIDPool: wazevoapi.NewVarLengthPool[knownSafeBoundWithID](), + } + c.declareSignatures(listenerOn) + return c +} + +func (c *Compiler) declareSignatures(listenerOn bool) { + m := c.m + c.signatures = make(map[*wasm.FunctionType]*ssa.Signature, len(m.TypeSection)+2) + if listenerOn { + c.listenerSignatures = make(map[*wasm.FunctionType][2]*ssa.Signature, len(m.TypeSection)) + } + for i := range m.TypeSection { + wasmSig := &m.TypeSection[i] + sig := SignatureForWasmFunctionType(wasmSig) + sig.ID = ssa.SignatureID(i) + c.signatures[wasmSig] = &sig + c.ssaBuilder.DeclareSignature(&sig) + + if listenerOn { + beforeSig, afterSig := SignatureForListener(wasmSig) + beforeSig.ID = ssa.SignatureID(i) + ssa.SignatureID(len(m.TypeSection)) + afterSig.ID = ssa.SignatureID(i) + ssa.SignatureID(len(m.TypeSection))*2 + c.listenerSignatures[wasmSig] = [2]*ssa.Signature{beforeSig, afterSig} + c.ssaBuilder.DeclareSignature(beforeSig) + c.ssaBuilder.DeclareSignature(afterSig) + } + } + + begin := ssa.SignatureID(len(m.TypeSection)) + if listenerOn { + begin *= 3 + } + c.memoryGrowSig = ssa.Signature{ + ID: begin, + // Takes execution context and the page size to grow. + Params: []ssa.Type{ssa.TypeI64, ssa.TypeI32}, + // Returns the previous page size. + Results: []ssa.Type{ssa.TypeI32}, + } + c.ssaBuilder.DeclareSignature(&c.memoryGrowSig) + + c.checkModuleExitCodeSig = ssa.Signature{ + ID: c.memoryGrowSig.ID + 1, + // Only takes execution context. + Params: []ssa.Type{ssa.TypeI64}, + } + c.ssaBuilder.DeclareSignature(&c.checkModuleExitCodeSig) + + c.tableGrowSig = ssa.Signature{ + ID: c.checkModuleExitCodeSig.ID + 1, + Params: []ssa.Type{ssa.TypeI64 /* exec context */, ssa.TypeI32 /* table index */, ssa.TypeI32 /* num */, ssa.TypeI64 /* ref */}, + // Returns the previous size. + Results: []ssa.Type{ssa.TypeI32}, + } + c.ssaBuilder.DeclareSignature(&c.tableGrowSig) + + c.refFuncSig = ssa.Signature{ + ID: c.tableGrowSig.ID + 1, + Params: []ssa.Type{ssa.TypeI64 /* exec context */, ssa.TypeI32 /* func index */}, + // Returns the function reference. + Results: []ssa.Type{ssa.TypeI64}, + } + c.ssaBuilder.DeclareSignature(&c.refFuncSig) + + c.memmoveSig = ssa.Signature{ + ID: c.refFuncSig.ID + 1, + // dst, src, and the byte count. + Params: []ssa.Type{ssa.TypeI64, ssa.TypeI64, ssa.TypeI64}, + } + + c.ssaBuilder.DeclareSignature(&c.memmoveSig) + + c.memoryWait32Sig = ssa.Signature{ + ID: c.memmoveSig.ID + 1, + // exec context, timeout, expected, addr + Params: []ssa.Type{ssa.TypeI64, ssa.TypeI64, ssa.TypeI32, ssa.TypeI64}, + // Returns the status. + Results: []ssa.Type{ssa.TypeI32}, + } + c.ssaBuilder.DeclareSignature(&c.memoryWait32Sig) + + c.memoryWait64Sig = ssa.Signature{ + ID: c.memoryWait32Sig.ID + 1, + // exec context, timeout, expected, addr + Params: []ssa.Type{ssa.TypeI64, ssa.TypeI64, ssa.TypeI64, ssa.TypeI64}, + // Returns the status. + Results: []ssa.Type{ssa.TypeI32}, + } + c.ssaBuilder.DeclareSignature(&c.memoryWait64Sig) + + c.memoryNotifySig = ssa.Signature{ + ID: c.memoryWait64Sig.ID + 1, + // exec context, count, addr + Params: []ssa.Type{ssa.TypeI64, ssa.TypeI32, ssa.TypeI64}, + // Returns the number notified. + Results: []ssa.Type{ssa.TypeI32}, + } + c.ssaBuilder.DeclareSignature(&c.memoryNotifySig) +} + +// SignatureForWasmFunctionType returns the ssa.Signature for the given wasm.FunctionType. +func SignatureForWasmFunctionType(typ *wasm.FunctionType) ssa.Signature { + sig := ssa.Signature{ + // +2 to pass moduleContextPtr and executionContextPtr. See the inline comment LowerToSSA. + Params: make([]ssa.Type, len(typ.Params)+2), + Results: make([]ssa.Type, len(typ.Results)), + } + sig.Params[0] = executionContextPtrTyp + sig.Params[1] = moduleContextPtrTyp + for j, typ := range typ.Params { + sig.Params[j+2] = WasmTypeToSSAType(typ) + } + for j, typ := range typ.Results { + sig.Results[j] = WasmTypeToSSAType(typ) + } + return sig +} + +// Init initializes the state of frontendCompiler and make it ready for a next function. +func (c *Compiler) Init(idx, typIndex wasm.Index, typ *wasm.FunctionType, localTypes []wasm.ValueType, body []byte, needListener bool, bodyOffsetInCodeSection uint64) { + c.ssaBuilder.Init(c.signatures[typ]) + c.loweringState.reset() + + c.wasmFunctionTypeIndex = typIndex + c.wasmLocalFunctionIndex = idx + c.wasmFunctionTyp = typ + c.wasmFunctionLocalTypes = localTypes + c.wasmFunctionBody = body + c.wasmFunctionBodyOffsetInCodeSection = bodyOffsetInCodeSection + c.needListener = needListener + c.clearSafeBounds() + c.varLengthKnownSafeBoundWithIDPool.Reset() + c.knownSafeBoundsAtTheEndOfBlocks = c.knownSafeBoundsAtTheEndOfBlocks[:0] +} + +// Note: this assumes 64-bit platform (I believe we won't have 32-bit backend ;)). +const executionContextPtrTyp, moduleContextPtrTyp = ssa.TypeI64, ssa.TypeI64 + +// LowerToSSA lowers the current function to SSA function which will be held by ssaBuilder. +// After calling this, the caller will be able to access the SSA info in *Compiler.ssaBuilder. +// +// Note that this only does the naive lowering, and do not do any optimization, instead the caller is expected to do so. +func (c *Compiler) LowerToSSA() { + builder := c.ssaBuilder + + // Set up the entry block. + entryBlock := builder.AllocateBasicBlock() + builder.SetCurrentBlock(entryBlock) + + // Functions always take two parameters in addition to Wasm-level parameters: + // + // 1. executionContextPtr: pointer to the *executionContext in wazevo package. + // This will be used to exit the execution in the face of trap, plus used for host function calls. + // + // 2. moduleContextPtr: pointer to the *moduleContextOpaque in wazevo package. + // This will be used to access memory, etc. Also, this will be used during host function calls. + // + // Note: it's clear that sometimes a function won't need them. For example, + // if the function doesn't trap and doesn't make function call, then + // we might be able to eliminate the parameter. However, if that function + // can be called via call_indirect, then we cannot eliminate because the + // signature won't match with the expected one. + // TODO: maybe there's some way to do this optimization without glitches, but so far I have no clue about the feasibility. + // + // Note: In Wasmtime or many other runtimes, moduleContextPtr is called "vmContext". Also note that `moduleContextPtr` + // is wazero-specific since other runtimes can naturally use the OS-level signal to do this job thanks to the fact that + // they can use native stack vs wazero cannot use Go-routine stack and have to use Go-runtime allocated []byte as a stack. + c.execCtxPtrValue = entryBlock.AddParam(builder, executionContextPtrTyp) + c.moduleCtxPtrValue = entryBlock.AddParam(builder, moduleContextPtrTyp) + builder.AnnotateValue(c.execCtxPtrValue, "exec_ctx") + builder.AnnotateValue(c.moduleCtxPtrValue, "module_ctx") + + for i, typ := range c.wasmFunctionTyp.Params { + st := WasmTypeToSSAType(typ) + variable := builder.DeclareVariable(st) + value := entryBlock.AddParam(builder, st) + builder.DefineVariable(variable, value, entryBlock) + c.setWasmLocalVariable(wasm.Index(i), variable) + } + c.declareWasmLocals(entryBlock) + c.declareNecessaryVariables() + + c.lowerBody(entryBlock) +} + +// localVariable returns the SSA variable for the given Wasm local index. +func (c *Compiler) localVariable(index wasm.Index) ssa.Variable { + return c.wasmLocalToVariable[index] +} + +func (c *Compiler) setWasmLocalVariable(index wasm.Index, variable ssa.Variable) { + idx := int(index) + if idx >= len(c.wasmLocalToVariable) { + c.wasmLocalToVariable = append(c.wasmLocalToVariable, make([]ssa.Variable, idx+1-len(c.wasmLocalToVariable))...) + } + c.wasmLocalToVariable[idx] = variable +} + +// declareWasmLocals declares the SSA variables for the Wasm locals. +func (c *Compiler) declareWasmLocals(entry ssa.BasicBlock) { + localCount := wasm.Index(len(c.wasmFunctionTyp.Params)) + for i, typ := range c.wasmFunctionLocalTypes { + st := WasmTypeToSSAType(typ) + variable := c.ssaBuilder.DeclareVariable(st) + c.setWasmLocalVariable(wasm.Index(i)+localCount, variable) + + zeroInst := c.ssaBuilder.AllocateInstruction() + switch st { + case ssa.TypeI32: + zeroInst.AsIconst32(0) + case ssa.TypeI64: + zeroInst.AsIconst64(0) + case ssa.TypeF32: + zeroInst.AsF32const(0) + case ssa.TypeF64: + zeroInst.AsF64const(0) + case ssa.TypeV128: + zeroInst.AsVconst(0, 0) + default: + panic("TODO: " + wasm.ValueTypeName(typ)) + } + + c.ssaBuilder.InsertInstruction(zeroInst) + value := zeroInst.Return() + c.ssaBuilder.DefineVariable(variable, value, entry) + } +} + +func (c *Compiler) declareNecessaryVariables() { + if c.needMemory = c.m.MemorySection != nil; c.needMemory { + c.memoryShared = c.m.MemorySection.IsShared + } else if c.needMemory = c.m.ImportMemoryCount > 0; c.needMemory { + for _, imp := range c.m.ImportSection { + if imp.Type == wasm.ExternTypeMemory { + c.memoryShared = imp.DescMem.IsShared + break + } + } + } + + if c.needMemory { + c.memoryBaseVariable = c.ssaBuilder.DeclareVariable(ssa.TypeI64) + c.memoryLenVariable = c.ssaBuilder.DeclareVariable(ssa.TypeI64) + } + + c.globalVariables = c.globalVariables[:0] + c.mutableGlobalVariablesIndexes = c.mutableGlobalVariablesIndexes[:0] + c.globalVariablesTypes = c.globalVariablesTypes[:0] + for _, imp := range c.m.ImportSection { + if imp.Type == wasm.ExternTypeGlobal { + desc := imp.DescGlobal + c.declareWasmGlobal(desc.ValType, desc.Mutable) + } + } + for _, g := range c.m.GlobalSection { + desc := g.Type + c.declareWasmGlobal(desc.ValType, desc.Mutable) + } + + // TODO: add tables. +} + +func (c *Compiler) declareWasmGlobal(typ wasm.ValueType, mutable bool) { + var st ssa.Type + switch typ { + case wasm.ValueTypeI32: + st = ssa.TypeI32 + case wasm.ValueTypeI64, + // Both externref and funcref are represented as I64 since we only support 64-bit platforms. + wasm.ValueTypeExternref, wasm.ValueTypeFuncref: + st = ssa.TypeI64 + case wasm.ValueTypeF32: + st = ssa.TypeF32 + case wasm.ValueTypeF64: + st = ssa.TypeF64 + case wasm.ValueTypeV128: + st = ssa.TypeV128 + default: + panic("TODO: " + wasm.ValueTypeName(typ)) + } + v := c.ssaBuilder.DeclareVariable(st) + index := wasm.Index(len(c.globalVariables)) + c.globalVariables = append(c.globalVariables, v) + c.globalVariablesTypes = append(c.globalVariablesTypes, st) + if mutable { + c.mutableGlobalVariablesIndexes = append(c.mutableGlobalVariablesIndexes, index) + } +} + +// WasmTypeToSSAType converts wasm.ValueType to ssa.Type. +func WasmTypeToSSAType(vt wasm.ValueType) ssa.Type { + switch vt { + case wasm.ValueTypeI32: + return ssa.TypeI32 + case wasm.ValueTypeI64, + // Both externref and funcref are represented as I64 since we only support 64-bit platforms. + wasm.ValueTypeExternref, wasm.ValueTypeFuncref: + return ssa.TypeI64 + case wasm.ValueTypeF32: + return ssa.TypeF32 + case wasm.ValueTypeF64: + return ssa.TypeF64 + case wasm.ValueTypeV128: + return ssa.TypeV128 + default: + panic("TODO: " + wasm.ValueTypeName(vt)) + } +} + +// addBlockParamsFromWasmTypes adds the block parameters to the given block. +func (c *Compiler) addBlockParamsFromWasmTypes(tps []wasm.ValueType, blk ssa.BasicBlock) { + for _, typ := range tps { + st := WasmTypeToSSAType(typ) + blk.AddParam(c.ssaBuilder, st) + } +} + +// formatBuilder outputs the constructed SSA function as a string with a source information. +func (c *Compiler) formatBuilder() string { + return c.ssaBuilder.Format() +} + +// SignatureForListener returns the signatures for the listener functions. +func SignatureForListener(wasmSig *wasm.FunctionType) (*ssa.Signature, *ssa.Signature) { + beforeSig := &ssa.Signature{} + beforeSig.Params = make([]ssa.Type, len(wasmSig.Params)+2) + beforeSig.Params[0] = ssa.TypeI64 // Execution context. + beforeSig.Params[1] = ssa.TypeI32 // Function index. + for i, p := range wasmSig.Params { + beforeSig.Params[i+2] = WasmTypeToSSAType(p) + } + afterSig := &ssa.Signature{} + afterSig.Params = make([]ssa.Type, len(wasmSig.Results)+2) + afterSig.Params[0] = ssa.TypeI64 // Execution context. + afterSig.Params[1] = ssa.TypeI32 // Function index. + for i, p := range wasmSig.Results { + afterSig.Params[i+2] = WasmTypeToSSAType(p) + } + return beforeSig, afterSig +} + +// isBoundSafe returns true if the given value is known to be safe to access up to the given bound. +func (c *Compiler) getKnownSafeBound(v ssa.ValueID) *knownSafeBound { + if int(v) >= len(c.knownSafeBounds) { + return nil + } + return &c.knownSafeBounds[v] +} + +// recordKnownSafeBound records the given safe bound for the given value. +func (c *Compiler) recordKnownSafeBound(v ssa.ValueID, safeBound uint64, absoluteAddr ssa.Value) { + if int(v) >= len(c.knownSafeBounds) { + c.knownSafeBounds = append(c.knownSafeBounds, make([]knownSafeBound, v+1)...) + } + + if exiting := c.knownSafeBounds[v]; exiting.bound == 0 { + c.knownSafeBounds[v] = knownSafeBound{ + bound: safeBound, + absoluteAddr: absoluteAddr, + } + c.knownSafeBoundsSet = append(c.knownSafeBoundsSet, v) + } else if safeBound > exiting.bound { + c.knownSafeBounds[v].bound = safeBound + } +} + +// clearSafeBounds clears the known safe bounds. +func (c *Compiler) clearSafeBounds() { + for _, v := range c.knownSafeBoundsSet { + ptr := &c.knownSafeBounds[v] + ptr.bound = 0 + ptr.absoluteAddr = ssa.ValueInvalid + } + c.knownSafeBoundsSet = c.knownSafeBoundsSet[:0] +} + +// resetAbsoluteAddressInSafeBounds resets the absolute addresses recorded in the known safe bounds. +func (c *Compiler) resetAbsoluteAddressInSafeBounds() { + for _, v := range c.knownSafeBoundsSet { + ptr := &c.knownSafeBounds[v] + ptr.absoluteAddr = ssa.ValueInvalid + } +} + +func (k *knownSafeBound) valid() bool { + return k != nil && k.bound > 0 +} + +func (c *Compiler) allocateVarLengthValues(_cap int, vs ...ssa.Value) ssa.Values { + builder := c.ssaBuilder + pool := builder.VarLengthPool() + args := pool.Allocate(_cap) + args = args.Append(builder.VarLengthPool(), vs...) + return args +} + +func (c *Compiler) finalizeKnownSafeBoundsAtTheEndOfBlock(bID ssa.BasicBlockID) { + _bID := int(bID) + if l := len(c.knownSafeBoundsAtTheEndOfBlocks); _bID >= l { + c.knownSafeBoundsAtTheEndOfBlocks = append(c.knownSafeBoundsAtTheEndOfBlocks, + make([]knownSafeBoundsAtTheEndOfBlock, _bID+1-len(c.knownSafeBoundsAtTheEndOfBlocks))...) + for i := l; i < len(c.knownSafeBoundsAtTheEndOfBlocks); i++ { + c.knownSafeBoundsAtTheEndOfBlocks[i] = knownSafeBoundsAtTheEndOfBlockNil + } + } + p := &c.varLengthKnownSafeBoundWithIDPool + size := len(c.knownSafeBoundsSet) + allocated := c.varLengthKnownSafeBoundWithIDPool.Allocate(size) + // Sort the known safe bounds by the value ID so that we can use the intersection algorithm in initializeCurrentBlockKnownBounds. + sortSSAValueIDs(c.knownSafeBoundsSet) + for _, vID := range c.knownSafeBoundsSet { + kb := c.knownSafeBounds[vID] + allocated = allocated.Append(p, knownSafeBoundWithID{ + knownSafeBound: kb, + id: vID, + }) + } + c.knownSafeBoundsAtTheEndOfBlocks[bID] = allocated + c.clearSafeBounds() +} + +func (c *Compiler) initializeCurrentBlockKnownBounds() { + currentBlk := c.ssaBuilder.CurrentBlock() + switch preds := currentBlk.Preds(); preds { + case 0: + case 1: + pred := currentBlk.Pred(0).ID() + for _, kb := range c.getKnownSafeBoundsAtTheEndOfBlocks(pred).View() { + // Unless the block is sealed, we cannot assume the absolute address is valid: + // later we might add another predecessor that has no visibility of that value. + addr := ssa.ValueInvalid + if currentBlk.Sealed() { + addr = kb.absoluteAddr + } + c.recordKnownSafeBound(kb.id, kb.bound, addr) + } + default: + c.pointers = c.pointers[:0] + c.bounds = c.bounds[:0] + for i := 0; i < preds; i++ { + c.bounds = append(c.bounds, c.getKnownSafeBoundsAtTheEndOfBlocks(currentBlk.Pred(i).ID()).View()) + c.pointers = append(c.pointers, 0) + } + + // If there are multiple predecessors, we need to find the intersection of the known safe bounds. + + outer: + for { + smallestID := ssa.ValueID(math.MaxUint32) + for i, ptr := range c.pointers { + if ptr >= len(c.bounds[i]) { + break outer + } + cb := &c.bounds[i][ptr] + if id := cb.id; id < smallestID { + smallestID = cb.id + } + } + + // Check if current elements are the same across all lists. + same := true + minBound := uint64(math.MaxUint64) + for i := 0; i < preds; i++ { + cb := &c.bounds[i][c.pointers[i]] + if cb.id != smallestID { + same = false + break + } else { + if cb.bound < minBound { + minBound = cb.bound + } + } + } + + if same { // All elements are the same. + // Absolute address cannot be used in the intersection since the value might be only defined in one of the predecessors. + c.recordKnownSafeBound(smallestID, minBound, ssa.ValueInvalid) + } + + // Move pointer(s) for the smallest ID forward (if same, move all). + for i := 0; i < preds; i++ { + cb := &c.bounds[i][c.pointers[i]] + if cb.id == smallestID { + c.pointers[i]++ + } + } + } + } +} + +func (c *Compiler) getKnownSafeBoundsAtTheEndOfBlocks(id ssa.BasicBlockID) knownSafeBoundsAtTheEndOfBlock { + if int(id) >= len(c.knownSafeBoundsAtTheEndOfBlocks) { + return knownSafeBoundsAtTheEndOfBlockNil + } + return c.knownSafeBoundsAtTheEndOfBlocks[id] +} diff --git a/vendor/github.com/tetratelabs/wazero/internal/engine/wazevo/frontend/lower.go b/vendor/github.com/tetratelabs/wazero/internal/engine/wazevo/frontend/lower.go new file mode 100644 index 000000000..5096a6365 --- /dev/null +++ b/vendor/github.com/tetratelabs/wazero/internal/engine/wazevo/frontend/lower.go @@ -0,0 +1,4268 @@ +package frontend + +import ( + "encoding/binary" + "fmt" + "math" + "runtime" + "strings" + + "github.com/tetratelabs/wazero/api" + "github.com/tetratelabs/wazero/internal/engine/wazevo/ssa" + "github.com/tetratelabs/wazero/internal/engine/wazevo/wazevoapi" + "github.com/tetratelabs/wazero/internal/leb128" + "github.com/tetratelabs/wazero/internal/wasm" +) + +type ( + // loweringState is used to keep the state of lowering. + loweringState struct { + // values holds the values on the Wasm stack. + values []ssa.Value + controlFrames []controlFrame + unreachable bool + unreachableDepth int + tmpForBrTable []uint32 + pc int + } + controlFrame struct { + kind controlFrameKind + // originalStackLen holds the number of values on the Wasm stack + // when start executing this control frame minus params for the block. + originalStackLenWithoutParam int + // blk is the loop header if this is loop, and is the else-block if this is an if frame. + blk, + // followingBlock is the basic block we enter if we reach "end" of block. + followingBlock ssa.BasicBlock + blockType *wasm.FunctionType + // clonedArgs hold the arguments to Else block. + clonedArgs ssa.Values + } + + controlFrameKind byte +) + +// String implements fmt.Stringer for debugging. +func (l *loweringState) String() string { + var str []string + for _, v := range l.values { + str = append(str, fmt.Sprintf("v%v", v.ID())) + } + var frames []string + for i := range l.controlFrames { + frames = append(frames, l.controlFrames[i].kind.String()) + } + return fmt.Sprintf("\n\tunreachable=%v(depth=%d)\n\tstack: %s\n\tcontrol frames: %s", + l.unreachable, l.unreachableDepth, + strings.Join(str, ", "), + strings.Join(frames, ", "), + ) +} + +const ( + controlFrameKindFunction = iota + 1 + controlFrameKindLoop + controlFrameKindIfWithElse + controlFrameKindIfWithoutElse + controlFrameKindBlock +) + +// String implements fmt.Stringer for debugging. +func (k controlFrameKind) String() string { + switch k { + case controlFrameKindFunction: + return "function" + case controlFrameKindLoop: + return "loop" + case controlFrameKindIfWithElse: + return "if_with_else" + case controlFrameKindIfWithoutElse: + return "if_without_else" + case controlFrameKindBlock: + return "block" + default: + panic(k) + } +} + +// isLoop returns true if this is a loop frame. +func (ctrl *controlFrame) isLoop() bool { + return ctrl.kind == controlFrameKindLoop +} + +// reset resets the state of loweringState for reuse. +func (l *loweringState) reset() { + l.values = l.values[:0] + l.controlFrames = l.controlFrames[:0] + l.pc = 0 + l.unreachable = false + l.unreachableDepth = 0 +} + +func (l *loweringState) peek() (ret ssa.Value) { + tail := len(l.values) - 1 + return l.values[tail] +} + +func (l *loweringState) pop() (ret ssa.Value) { + tail := len(l.values) - 1 + ret = l.values[tail] + l.values = l.values[:tail] + return +} + +func (l *loweringState) push(ret ssa.Value) { + l.values = append(l.values, ret) +} + +func (c *Compiler) nPeekDup(n int) ssa.Values { + if n == 0 { + return ssa.ValuesNil + } + + l := c.state() + tail := len(l.values) + + args := c.allocateVarLengthValues(n) + args = args.Append(c.ssaBuilder.VarLengthPool(), l.values[tail-n:tail]...) + return args +} + +func (l *loweringState) ctrlPop() (ret controlFrame) { + tail := len(l.controlFrames) - 1 + ret = l.controlFrames[tail] + l.controlFrames = l.controlFrames[:tail] + return +} + +func (l *loweringState) ctrlPush(ret controlFrame) { + l.controlFrames = append(l.controlFrames, ret) +} + +func (l *loweringState) ctrlPeekAt(n int) (ret *controlFrame) { + tail := len(l.controlFrames) - 1 + return &l.controlFrames[tail-n] +} + +// lowerBody lowers the body of the Wasm function to the SSA form. +func (c *Compiler) lowerBody(entryBlk ssa.BasicBlock) { + c.ssaBuilder.Seal(entryBlk) + + if c.needListener { + c.callListenerBefore() + } + + // Pushes the empty control frame which corresponds to the function return. + c.loweringState.ctrlPush(controlFrame{ + kind: controlFrameKindFunction, + blockType: c.wasmFunctionTyp, + followingBlock: c.ssaBuilder.ReturnBlock(), + }) + + for c.loweringState.pc < len(c.wasmFunctionBody) { + blkBeforeLowering := c.ssaBuilder.CurrentBlock() + c.lowerCurrentOpcode() + blkAfterLowering := c.ssaBuilder.CurrentBlock() + if blkBeforeLowering != blkAfterLowering { + // In Wasm, once a block exits, that means we've done compiling the block. + // Therefore, we finalize the known bounds at the end of the block for the exiting block. + c.finalizeKnownSafeBoundsAtTheEndOfBlock(blkBeforeLowering.ID()) + // After that, we initialize the known bounds for the new compilation target block. + c.initializeCurrentBlockKnownBounds() + } + } +} + +func (c *Compiler) state() *loweringState { + return &c.loweringState +} + +func (c *Compiler) lowerCurrentOpcode() { + op := c.wasmFunctionBody[c.loweringState.pc] + + if c.needSourceOffsetInfo { + c.ssaBuilder.SetCurrentSourceOffset( + ssa.SourceOffset(c.loweringState.pc) + ssa.SourceOffset(c.wasmFunctionBodyOffsetInCodeSection), + ) + } + + builder := c.ssaBuilder + state := c.state() + switch op { + case wasm.OpcodeI32Const: + c := c.readI32s() + if state.unreachable { + break + } + + iconst := builder.AllocateInstruction().AsIconst32(uint32(c)).Insert(builder) + value := iconst.Return() + state.push(value) + case wasm.OpcodeI64Const: + c := c.readI64s() + if state.unreachable { + break + } + iconst := builder.AllocateInstruction().AsIconst64(uint64(c)).Insert(builder) + value := iconst.Return() + state.push(value) + case wasm.OpcodeF32Const: + f32 := c.readF32() + if state.unreachable { + break + } + f32const := builder.AllocateInstruction(). + AsF32const(f32). + Insert(builder). + Return() + state.push(f32const) + case wasm.OpcodeF64Const: + f64 := c.readF64() + if state.unreachable { + break + } + f64const := builder.AllocateInstruction(). + AsF64const(f64). + Insert(builder). + Return() + state.push(f64const) + case wasm.OpcodeI32Add, wasm.OpcodeI64Add: + if state.unreachable { + break + } + y, x := state.pop(), state.pop() + iadd := builder.AllocateInstruction() + iadd.AsIadd(x, y) + builder.InsertInstruction(iadd) + value := iadd.Return() + state.push(value) + case wasm.OpcodeI32Sub, wasm.OpcodeI64Sub: + if state.unreachable { + break + } + y, x := state.pop(), state.pop() + isub := builder.AllocateInstruction() + isub.AsIsub(x, y) + builder.InsertInstruction(isub) + value := isub.Return() + state.push(value) + case wasm.OpcodeF32Add, wasm.OpcodeF64Add: + if state.unreachable { + break + } + y, x := state.pop(), state.pop() + iadd := builder.AllocateInstruction() + iadd.AsFadd(x, y) + builder.InsertInstruction(iadd) + value := iadd.Return() + state.push(value) + case wasm.OpcodeI32Mul, wasm.OpcodeI64Mul: + if state.unreachable { + break + } + y, x := state.pop(), state.pop() + imul := builder.AllocateInstruction() + imul.AsImul(x, y) + builder.InsertInstruction(imul) + value := imul.Return() + state.push(value) + case wasm.OpcodeF32Sub, wasm.OpcodeF64Sub: + if state.unreachable { + break + } + y, x := state.pop(), state.pop() + isub := builder.AllocateInstruction() + isub.AsFsub(x, y) + builder.InsertInstruction(isub) + value := isub.Return() + state.push(value) + case wasm.OpcodeF32Mul, wasm.OpcodeF64Mul: + if state.unreachable { + break + } + y, x := state.pop(), state.pop() + isub := builder.AllocateInstruction() + isub.AsFmul(x, y) + builder.InsertInstruction(isub) + value := isub.Return() + state.push(value) + case wasm.OpcodeF32Div, wasm.OpcodeF64Div: + if state.unreachable { + break + } + y, x := state.pop(), state.pop() + isub := builder.AllocateInstruction() + isub.AsFdiv(x, y) + builder.InsertInstruction(isub) + value := isub.Return() + state.push(value) + case wasm.OpcodeF32Max, wasm.OpcodeF64Max: + if state.unreachable { + break + } + y, x := state.pop(), state.pop() + isub := builder.AllocateInstruction() + isub.AsFmax(x, y) + builder.InsertInstruction(isub) + value := isub.Return() + state.push(value) + case wasm.OpcodeF32Min, wasm.OpcodeF64Min: + if state.unreachable { + break + } + y, x := state.pop(), state.pop() + isub := builder.AllocateInstruction() + isub.AsFmin(x, y) + builder.InsertInstruction(isub) + value := isub.Return() + state.push(value) + case wasm.OpcodeI64Extend8S: + if state.unreachable { + break + } + c.insertIntegerExtend(true, 8, 64) + case wasm.OpcodeI64Extend16S: + if state.unreachable { + break + } + c.insertIntegerExtend(true, 16, 64) + case wasm.OpcodeI64Extend32S, wasm.OpcodeI64ExtendI32S: + if state.unreachable { + break + } + c.insertIntegerExtend(true, 32, 64) + case wasm.OpcodeI64ExtendI32U: + if state.unreachable { + break + } + c.insertIntegerExtend(false, 32, 64) + case wasm.OpcodeI32Extend8S: + if state.unreachable { + break + } + c.insertIntegerExtend(true, 8, 32) + case wasm.OpcodeI32Extend16S: + if state.unreachable { + break + } + c.insertIntegerExtend(true, 16, 32) + case wasm.OpcodeI32Eqz, wasm.OpcodeI64Eqz: + if state.unreachable { + break + } + x := state.pop() + zero := builder.AllocateInstruction() + if op == wasm.OpcodeI32Eqz { + zero.AsIconst32(0) + } else { + zero.AsIconst64(0) + } + builder.InsertInstruction(zero) + icmp := builder.AllocateInstruction(). + AsIcmp(x, zero.Return(), ssa.IntegerCmpCondEqual). + Insert(builder). + Return() + state.push(icmp) + case wasm.OpcodeI32Eq, wasm.OpcodeI64Eq: + if state.unreachable { + break + } + c.insertIcmp(ssa.IntegerCmpCondEqual) + case wasm.OpcodeI32Ne, wasm.OpcodeI64Ne: + if state.unreachable { + break + } + c.insertIcmp(ssa.IntegerCmpCondNotEqual) + case wasm.OpcodeI32LtS, wasm.OpcodeI64LtS: + if state.unreachable { + break + } + c.insertIcmp(ssa.IntegerCmpCondSignedLessThan) + case wasm.OpcodeI32LtU, wasm.OpcodeI64LtU: + if state.unreachable { + break + } + c.insertIcmp(ssa.IntegerCmpCondUnsignedLessThan) + case wasm.OpcodeI32GtS, wasm.OpcodeI64GtS: + if state.unreachable { + break + } + c.insertIcmp(ssa.IntegerCmpCondSignedGreaterThan) + case wasm.OpcodeI32GtU, wasm.OpcodeI64GtU: + if state.unreachable { + break + } + c.insertIcmp(ssa.IntegerCmpCondUnsignedGreaterThan) + case wasm.OpcodeI32LeS, wasm.OpcodeI64LeS: + if state.unreachable { + break + } + c.insertIcmp(ssa.IntegerCmpCondSignedLessThanOrEqual) + case wasm.OpcodeI32LeU, wasm.OpcodeI64LeU: + if state.unreachable { + break + } + c.insertIcmp(ssa.IntegerCmpCondUnsignedLessThanOrEqual) + case wasm.OpcodeI32GeS, wasm.OpcodeI64GeS: + if state.unreachable { + break + } + c.insertIcmp(ssa.IntegerCmpCondSignedGreaterThanOrEqual) + case wasm.OpcodeI32GeU, wasm.OpcodeI64GeU: + if state.unreachable { + break + } + c.insertIcmp(ssa.IntegerCmpCondUnsignedGreaterThanOrEqual) + + case wasm.OpcodeF32Eq, wasm.OpcodeF64Eq: + if state.unreachable { + break + } + c.insertFcmp(ssa.FloatCmpCondEqual) + case wasm.OpcodeF32Ne, wasm.OpcodeF64Ne: + if state.unreachable { + break + } + c.insertFcmp(ssa.FloatCmpCondNotEqual) + case wasm.OpcodeF32Lt, wasm.OpcodeF64Lt: + if state.unreachable { + break + } + c.insertFcmp(ssa.FloatCmpCondLessThan) + case wasm.OpcodeF32Gt, wasm.OpcodeF64Gt: + if state.unreachable { + break + } + c.insertFcmp(ssa.FloatCmpCondGreaterThan) + case wasm.OpcodeF32Le, wasm.OpcodeF64Le: + if state.unreachable { + break + } + c.insertFcmp(ssa.FloatCmpCondLessThanOrEqual) + case wasm.OpcodeF32Ge, wasm.OpcodeF64Ge: + if state.unreachable { + break + } + c.insertFcmp(ssa.FloatCmpCondGreaterThanOrEqual) + case wasm.OpcodeF32Neg, wasm.OpcodeF64Neg: + if state.unreachable { + break + } + x := state.pop() + v := builder.AllocateInstruction().AsFneg(x).Insert(builder).Return() + state.push(v) + case wasm.OpcodeF32Sqrt, wasm.OpcodeF64Sqrt: + if state.unreachable { + break + } + x := state.pop() + v := builder.AllocateInstruction().AsSqrt(x).Insert(builder).Return() + state.push(v) + case wasm.OpcodeF32Abs, wasm.OpcodeF64Abs: + if state.unreachable { + break + } + x := state.pop() + v := builder.AllocateInstruction().AsFabs(x).Insert(builder).Return() + state.push(v) + case wasm.OpcodeF32Copysign, wasm.OpcodeF64Copysign: + if state.unreachable { + break + } + y, x := state.pop(), state.pop() + v := builder.AllocateInstruction().AsFcopysign(x, y).Insert(builder).Return() + state.push(v) + + case wasm.OpcodeF32Ceil, wasm.OpcodeF64Ceil: + if state.unreachable { + break + } + x := state.pop() + v := builder.AllocateInstruction().AsCeil(x).Insert(builder).Return() + state.push(v) + case wasm.OpcodeF32Floor, wasm.OpcodeF64Floor: + if state.unreachable { + break + } + x := state.pop() + v := builder.AllocateInstruction().AsFloor(x).Insert(builder).Return() + state.push(v) + case wasm.OpcodeF32Trunc, wasm.OpcodeF64Trunc: + if state.unreachable { + break + } + x := state.pop() + v := builder.AllocateInstruction().AsTrunc(x).Insert(builder).Return() + state.push(v) + case wasm.OpcodeF32Nearest, wasm.OpcodeF64Nearest: + if state.unreachable { + break + } + x := state.pop() + v := builder.AllocateInstruction().AsNearest(x).Insert(builder).Return() + state.push(v) + case wasm.OpcodeI64TruncF64S, wasm.OpcodeI64TruncF32S, + wasm.OpcodeI32TruncF64S, wasm.OpcodeI32TruncF32S, + wasm.OpcodeI64TruncF64U, wasm.OpcodeI64TruncF32U, + wasm.OpcodeI32TruncF64U, wasm.OpcodeI32TruncF32U: + if state.unreachable { + break + } + ret := builder.AllocateInstruction().AsFcvtToInt( + state.pop(), + c.execCtxPtrValue, + op == wasm.OpcodeI64TruncF64S || op == wasm.OpcodeI64TruncF32S || op == wasm.OpcodeI32TruncF32S || op == wasm.OpcodeI32TruncF64S, + op == wasm.OpcodeI64TruncF64S || op == wasm.OpcodeI64TruncF32S || op == wasm.OpcodeI64TruncF64U || op == wasm.OpcodeI64TruncF32U, + false, + ).Insert(builder).Return() + state.push(ret) + case wasm.OpcodeMiscPrefix: + state.pc++ + // A misc opcode is encoded as an unsigned variable 32-bit integer. + miscOpUint, num, err := leb128.LoadUint32(c.wasmFunctionBody[state.pc:]) + if err != nil { + // In normal conditions this should never happen because the function has passed validation. + panic(fmt.Sprintf("failed to read misc opcode: %v", err)) + } + state.pc += int(num - 1) + miscOp := wasm.OpcodeMisc(miscOpUint) + switch miscOp { + case wasm.OpcodeMiscI64TruncSatF64S, wasm.OpcodeMiscI64TruncSatF32S, + wasm.OpcodeMiscI32TruncSatF64S, wasm.OpcodeMiscI32TruncSatF32S, + wasm.OpcodeMiscI64TruncSatF64U, wasm.OpcodeMiscI64TruncSatF32U, + wasm.OpcodeMiscI32TruncSatF64U, wasm.OpcodeMiscI32TruncSatF32U: + if state.unreachable { + break + } + ret := builder.AllocateInstruction().AsFcvtToInt( + state.pop(), + c.execCtxPtrValue, + miscOp == wasm.OpcodeMiscI64TruncSatF64S || miscOp == wasm.OpcodeMiscI64TruncSatF32S || miscOp == wasm.OpcodeMiscI32TruncSatF32S || miscOp == wasm.OpcodeMiscI32TruncSatF64S, + miscOp == wasm.OpcodeMiscI64TruncSatF64S || miscOp == wasm.OpcodeMiscI64TruncSatF32S || miscOp == wasm.OpcodeMiscI64TruncSatF64U || miscOp == wasm.OpcodeMiscI64TruncSatF32U, + true, + ).Insert(builder).Return() + state.push(ret) + + case wasm.OpcodeMiscTableSize: + tableIndex := c.readI32u() + if state.unreachable { + break + } + + // Load the table. + loadTableInstancePtr := builder.AllocateInstruction() + loadTableInstancePtr.AsLoad(c.moduleCtxPtrValue, c.offset.TableOffset(int(tableIndex)).U32(), ssa.TypeI64) + builder.InsertInstruction(loadTableInstancePtr) + tableInstancePtr := loadTableInstancePtr.Return() + + // Load the table's length. + loadTableLen := builder.AllocateInstruction(). + AsLoad(tableInstancePtr, tableInstanceLenOffset, ssa.TypeI32). + Insert(builder) + state.push(loadTableLen.Return()) + + case wasm.OpcodeMiscTableGrow: + tableIndex := c.readI32u() + if state.unreachable { + break + } + + c.storeCallerModuleContext() + + tableIndexVal := builder.AllocateInstruction().AsIconst32(tableIndex).Insert(builder).Return() + + num := state.pop() + r := state.pop() + + tableGrowPtr := builder.AllocateInstruction(). + AsLoad(c.execCtxPtrValue, + wazevoapi.ExecutionContextOffsetTableGrowTrampolineAddress.U32(), + ssa.TypeI64, + ).Insert(builder).Return() + + args := c.allocateVarLengthValues(4, c.execCtxPtrValue, tableIndexVal, num, r) + callGrowRet := builder. + AllocateInstruction(). + AsCallIndirect(tableGrowPtr, &c.tableGrowSig, args). + Insert(builder).Return() + state.push(callGrowRet) + + case wasm.OpcodeMiscTableCopy: + dstTableIndex := c.readI32u() + srcTableIndex := c.readI32u() + if state.unreachable { + break + } + + copySize := builder. + AllocateInstruction().AsUExtend(state.pop(), 32, 64).Insert(builder).Return() + srcOffset := builder. + AllocateInstruction().AsUExtend(state.pop(), 32, 64).Insert(builder).Return() + dstOffset := builder. + AllocateInstruction().AsUExtend(state.pop(), 32, 64).Insert(builder).Return() + + // Out of bounds check. + dstTableInstancePtr := c.boundsCheckInTable(dstTableIndex, dstOffset, copySize) + srcTableInstancePtr := c.boundsCheckInTable(srcTableIndex, srcOffset, copySize) + + dstTableBaseAddr := c.loadTableBaseAddr(dstTableInstancePtr) + srcTableBaseAddr := c.loadTableBaseAddr(srcTableInstancePtr) + + three := builder.AllocateInstruction().AsIconst64(3).Insert(builder).Return() + + dstOffsetInBytes := builder.AllocateInstruction().AsIshl(dstOffset, three).Insert(builder).Return() + dstAddr := builder.AllocateInstruction().AsIadd(dstTableBaseAddr, dstOffsetInBytes).Insert(builder).Return() + srcOffsetInBytes := builder.AllocateInstruction().AsIshl(srcOffset, three).Insert(builder).Return() + srcAddr := builder.AllocateInstruction().AsIadd(srcTableBaseAddr, srcOffsetInBytes).Insert(builder).Return() + + copySizeInBytes := builder.AllocateInstruction().AsIshl(copySize, three).Insert(builder).Return() + c.callMemmove(dstAddr, srcAddr, copySizeInBytes) + + case wasm.OpcodeMiscMemoryCopy: + state.pc += 2 // +2 to skip two memory indexes which are fixed to zero. + if state.unreachable { + break + } + + copySize := builder. + AllocateInstruction().AsUExtend(state.pop(), 32, 64).Insert(builder).Return() + srcOffset := builder. + AllocateInstruction().AsUExtend(state.pop(), 32, 64).Insert(builder).Return() + dstOffset := builder. + AllocateInstruction().AsUExtend(state.pop(), 32, 64).Insert(builder).Return() + + // Out of bounds check. + memLen := c.getMemoryLenValue(false) + c.boundsCheckInMemory(memLen, dstOffset, copySize) + c.boundsCheckInMemory(memLen, srcOffset, copySize) + + memBase := c.getMemoryBaseValue(false) + dstAddr := builder.AllocateInstruction().AsIadd(memBase, dstOffset).Insert(builder).Return() + srcAddr := builder.AllocateInstruction().AsIadd(memBase, srcOffset).Insert(builder).Return() + + c.callMemmove(dstAddr, srcAddr, copySize) + + case wasm.OpcodeMiscTableFill: + tableIndex := c.readI32u() + if state.unreachable { + break + } + fillSize := state.pop() + value := state.pop() + offset := state.pop() + + fillSizeExt := builder. + AllocateInstruction().AsUExtend(fillSize, 32, 64).Insert(builder).Return() + offsetExt := builder. + AllocateInstruction().AsUExtend(offset, 32, 64).Insert(builder).Return() + tableInstancePtr := c.boundsCheckInTable(tableIndex, offsetExt, fillSizeExt) + + three := builder.AllocateInstruction().AsIconst64(3).Insert(builder).Return() + offsetInBytes := builder.AllocateInstruction().AsIshl(offsetExt, three).Insert(builder).Return() + fillSizeInBytes := builder.AllocateInstruction().AsIshl(fillSizeExt, three).Insert(builder).Return() + + // Calculate the base address of the table. + tableBaseAddr := c.loadTableBaseAddr(tableInstancePtr) + addr := builder.AllocateInstruction().AsIadd(tableBaseAddr, offsetInBytes).Insert(builder).Return() + + // Prepare the loop and following block. + beforeLoop := builder.AllocateBasicBlock() + loopBlk := builder.AllocateBasicBlock() + loopVar := loopBlk.AddParam(builder, ssa.TypeI64) + followingBlk := builder.AllocateBasicBlock() + + // Uses the copy trick for faster filling buffer like memory.fill, but in this case we copy 8 bytes at a time. + // buf := memoryInst.Buffer[offset : offset+fillSize] + // buf[0:8] = value + // for i := 8; i < fillSize; i *= 2 { Begin with 8 bytes. + // copy(buf[i:], buf[:i]) + // } + + // Insert the jump to the beforeLoop block; If the fillSize is zero, then jump to the following block to skip entire logics. + zero := builder.AllocateInstruction().AsIconst64(0).Insert(builder).Return() + ifFillSizeZero := builder.AllocateInstruction().AsIcmp(fillSizeExt, zero, ssa.IntegerCmpCondEqual). + Insert(builder).Return() + builder.AllocateInstruction().AsBrnz(ifFillSizeZero, ssa.ValuesNil, followingBlk).Insert(builder) + c.insertJumpToBlock(ssa.ValuesNil, beforeLoop) + + // buf[0:8] = value + builder.SetCurrentBlock(beforeLoop) + builder.AllocateInstruction().AsStore(ssa.OpcodeStore, value, addr, 0).Insert(builder) + initValue := builder.AllocateInstruction().AsIconst64(8).Insert(builder).Return() + c.insertJumpToBlock(c.allocateVarLengthValues(1, initValue), loopBlk) + + builder.SetCurrentBlock(loopBlk) + dstAddr := builder.AllocateInstruction().AsIadd(addr, loopVar).Insert(builder).Return() + + // If loopVar*2 > fillSizeInBytes, then count must be fillSizeInBytes-loopVar. + var count ssa.Value + { + loopVarDoubled := builder.AllocateInstruction().AsIadd(loopVar, loopVar).Insert(builder).Return() + loopVarDoubledLargerThanFillSize := builder. + AllocateInstruction().AsIcmp(loopVarDoubled, fillSizeInBytes, ssa.IntegerCmpCondUnsignedGreaterThanOrEqual). + Insert(builder).Return() + diff := builder.AllocateInstruction().AsIsub(fillSizeInBytes, loopVar).Insert(builder).Return() + count = builder.AllocateInstruction().AsSelect(loopVarDoubledLargerThanFillSize, diff, loopVar).Insert(builder).Return() + } + + c.callMemmove(dstAddr, addr, count) + + shiftAmount := builder.AllocateInstruction().AsIconst64(1).Insert(builder).Return() + newLoopVar := builder.AllocateInstruction().AsIshl(loopVar, shiftAmount).Insert(builder).Return() + loopVarLessThanFillSize := builder.AllocateInstruction(). + AsIcmp(newLoopVar, fillSizeInBytes, ssa.IntegerCmpCondUnsignedLessThan).Insert(builder).Return() + + builder.AllocateInstruction(). + AsBrnz(loopVarLessThanFillSize, c.allocateVarLengthValues(1, newLoopVar), loopBlk). + Insert(builder) + + c.insertJumpToBlock(ssa.ValuesNil, followingBlk) + builder.SetCurrentBlock(followingBlk) + + builder.Seal(beforeLoop) + builder.Seal(loopBlk) + builder.Seal(followingBlk) + + case wasm.OpcodeMiscMemoryFill: + state.pc++ // Skip the memory index which is fixed to zero. + if state.unreachable { + break + } + + fillSize := builder. + AllocateInstruction().AsUExtend(state.pop(), 32, 64).Insert(builder).Return() + value := state.pop() + offset := builder. + AllocateInstruction().AsUExtend(state.pop(), 32, 64).Insert(builder).Return() + + // Out of bounds check. + c.boundsCheckInMemory(c.getMemoryLenValue(false), offset, fillSize) + + // Calculate the base address: + addr := builder.AllocateInstruction().AsIadd(c.getMemoryBaseValue(false), offset).Insert(builder).Return() + + // Uses the copy trick for faster filling buffer: https://gist.github.com/taylorza/df2f89d5f9ab3ffd06865062a4cf015d + // buf := memoryInst.Buffer[offset : offset+fillSize] + // buf[0] = value + // for i := 1; i < fillSize; i *= 2 { + // copy(buf[i:], buf[:i]) + // } + + // Prepare the loop and following block. + beforeLoop := builder.AllocateBasicBlock() + loopBlk := builder.AllocateBasicBlock() + loopVar := loopBlk.AddParam(builder, ssa.TypeI64) + followingBlk := builder.AllocateBasicBlock() + + // Insert the jump to the beforeLoop block; If the fillSize is zero, then jump to the following block to skip entire logics. + zero := builder.AllocateInstruction().AsIconst64(0).Insert(builder).Return() + ifFillSizeZero := builder.AllocateInstruction().AsIcmp(fillSize, zero, ssa.IntegerCmpCondEqual). + Insert(builder).Return() + builder.AllocateInstruction().AsBrnz(ifFillSizeZero, ssa.ValuesNil, followingBlk).Insert(builder) + c.insertJumpToBlock(ssa.ValuesNil, beforeLoop) + + // buf[0] = value + builder.SetCurrentBlock(beforeLoop) + builder.AllocateInstruction().AsStore(ssa.OpcodeIstore8, value, addr, 0).Insert(builder) + initValue := builder.AllocateInstruction().AsIconst64(1).Insert(builder).Return() + c.insertJumpToBlock(c.allocateVarLengthValues(1, initValue), loopBlk) + + builder.SetCurrentBlock(loopBlk) + dstAddr := builder.AllocateInstruction().AsIadd(addr, loopVar).Insert(builder).Return() + + // If loopVar*2 > fillSizeExt, then count must be fillSizeExt-loopVar. + var count ssa.Value + { + loopVarDoubled := builder.AllocateInstruction().AsIadd(loopVar, loopVar).Insert(builder).Return() + loopVarDoubledLargerThanFillSize := builder. + AllocateInstruction().AsIcmp(loopVarDoubled, fillSize, ssa.IntegerCmpCondUnsignedGreaterThanOrEqual). + Insert(builder).Return() + diff := builder.AllocateInstruction().AsIsub(fillSize, loopVar).Insert(builder).Return() + count = builder.AllocateInstruction().AsSelect(loopVarDoubledLargerThanFillSize, diff, loopVar).Insert(builder).Return() + } + + c.callMemmove(dstAddr, addr, count) + + shiftAmount := builder.AllocateInstruction().AsIconst64(1).Insert(builder).Return() + newLoopVar := builder.AllocateInstruction().AsIshl(loopVar, shiftAmount).Insert(builder).Return() + loopVarLessThanFillSize := builder.AllocateInstruction(). + AsIcmp(newLoopVar, fillSize, ssa.IntegerCmpCondUnsignedLessThan).Insert(builder).Return() + + builder.AllocateInstruction(). + AsBrnz(loopVarLessThanFillSize, c.allocateVarLengthValues(1, newLoopVar), loopBlk). + Insert(builder) + + c.insertJumpToBlock(ssa.ValuesNil, followingBlk) + builder.SetCurrentBlock(followingBlk) + + builder.Seal(beforeLoop) + builder.Seal(loopBlk) + builder.Seal(followingBlk) + + case wasm.OpcodeMiscMemoryInit: + index := c.readI32u() + state.pc++ // Skip the memory index which is fixed to zero. + if state.unreachable { + break + } + + copySize := builder. + AllocateInstruction().AsUExtend(state.pop(), 32, 64).Insert(builder).Return() + offsetInDataInstance := builder. + AllocateInstruction().AsUExtend(state.pop(), 32, 64).Insert(builder).Return() + offsetInMemory := builder. + AllocateInstruction().AsUExtend(state.pop(), 32, 64).Insert(builder).Return() + + dataInstPtr := c.dataOrElementInstanceAddr(index, c.offset.DataInstances1stElement) + + // Bounds check. + c.boundsCheckInMemory(c.getMemoryLenValue(false), offsetInMemory, copySize) + c.boundsCheckInDataOrElementInstance(dataInstPtr, offsetInDataInstance, copySize, wazevoapi.ExitCodeMemoryOutOfBounds) + + dataInstBaseAddr := builder.AllocateInstruction().AsLoad(dataInstPtr, 0, ssa.TypeI64).Insert(builder).Return() + srcAddr := builder.AllocateInstruction().AsIadd(dataInstBaseAddr, offsetInDataInstance).Insert(builder).Return() + + memBase := c.getMemoryBaseValue(false) + dstAddr := builder.AllocateInstruction().AsIadd(memBase, offsetInMemory).Insert(builder).Return() + + c.callMemmove(dstAddr, srcAddr, copySize) + + case wasm.OpcodeMiscTableInit: + elemIndex := c.readI32u() + tableIndex := c.readI32u() + if state.unreachable { + break + } + + copySize := builder. + AllocateInstruction().AsUExtend(state.pop(), 32, 64).Insert(builder).Return() + offsetInElementInstance := builder. + AllocateInstruction().AsUExtend(state.pop(), 32, 64).Insert(builder).Return() + offsetInTable := builder. + AllocateInstruction().AsUExtend(state.pop(), 32, 64).Insert(builder).Return() + + elemInstPtr := c.dataOrElementInstanceAddr(elemIndex, c.offset.ElementInstances1stElement) + + // Bounds check. + tableInstancePtr := c.boundsCheckInTable(tableIndex, offsetInTable, copySize) + c.boundsCheckInDataOrElementInstance(elemInstPtr, offsetInElementInstance, copySize, wazevoapi.ExitCodeTableOutOfBounds) + + three := builder.AllocateInstruction().AsIconst64(3).Insert(builder).Return() + // Calculates the destination address in the table. + tableOffsetInBytes := builder.AllocateInstruction().AsIshl(offsetInTable, three).Insert(builder).Return() + tableBaseAddr := c.loadTableBaseAddr(tableInstancePtr) + dstAddr := builder.AllocateInstruction().AsIadd(tableBaseAddr, tableOffsetInBytes).Insert(builder).Return() + + // Calculates the source address in the element instance. + srcOffsetInBytes := builder.AllocateInstruction().AsIshl(offsetInElementInstance, three).Insert(builder).Return() + elemInstBaseAddr := builder.AllocateInstruction().AsLoad(elemInstPtr, 0, ssa.TypeI64).Insert(builder).Return() + srcAddr := builder.AllocateInstruction().AsIadd(elemInstBaseAddr, srcOffsetInBytes).Insert(builder).Return() + + copySizeInBytes := builder.AllocateInstruction().AsIshl(copySize, three).Insert(builder).Return() + c.callMemmove(dstAddr, srcAddr, copySizeInBytes) + + case wasm.OpcodeMiscElemDrop: + index := c.readI32u() + if state.unreachable { + break + } + + c.dropDataOrElementInstance(index, c.offset.ElementInstances1stElement) + + case wasm.OpcodeMiscDataDrop: + index := c.readI32u() + if state.unreachable { + break + } + c.dropDataOrElementInstance(index, c.offset.DataInstances1stElement) + + default: + panic("Unknown MiscOp " + wasm.MiscInstructionName(miscOp)) + } + + case wasm.OpcodeI32ReinterpretF32: + if state.unreachable { + break + } + reinterpret := builder.AllocateInstruction(). + AsBitcast(state.pop(), ssa.TypeI32). + Insert(builder).Return() + state.push(reinterpret) + + case wasm.OpcodeI64ReinterpretF64: + if state.unreachable { + break + } + reinterpret := builder.AllocateInstruction(). + AsBitcast(state.pop(), ssa.TypeI64). + Insert(builder).Return() + state.push(reinterpret) + + case wasm.OpcodeF32ReinterpretI32: + if state.unreachable { + break + } + reinterpret := builder.AllocateInstruction(). + AsBitcast(state.pop(), ssa.TypeF32). + Insert(builder).Return() + state.push(reinterpret) + + case wasm.OpcodeF64ReinterpretI64: + if state.unreachable { + break + } + reinterpret := builder.AllocateInstruction(). + AsBitcast(state.pop(), ssa.TypeF64). + Insert(builder).Return() + state.push(reinterpret) + + case wasm.OpcodeI32DivS, wasm.OpcodeI64DivS: + if state.unreachable { + break + } + y, x := state.pop(), state.pop() + result := builder.AllocateInstruction().AsSDiv(x, y, c.execCtxPtrValue).Insert(builder).Return() + state.push(result) + + case wasm.OpcodeI32DivU, wasm.OpcodeI64DivU: + if state.unreachable { + break + } + y, x := state.pop(), state.pop() + result := builder.AllocateInstruction().AsUDiv(x, y, c.execCtxPtrValue).Insert(builder).Return() + state.push(result) + + case wasm.OpcodeI32RemS, wasm.OpcodeI64RemS: + if state.unreachable { + break + } + y, x := state.pop(), state.pop() + result := builder.AllocateInstruction().AsSRem(x, y, c.execCtxPtrValue).Insert(builder).Return() + state.push(result) + + case wasm.OpcodeI32RemU, wasm.OpcodeI64RemU: + if state.unreachable { + break + } + y, x := state.pop(), state.pop() + result := builder.AllocateInstruction().AsURem(x, y, c.execCtxPtrValue).Insert(builder).Return() + state.push(result) + + case wasm.OpcodeI32And, wasm.OpcodeI64And: + if state.unreachable { + break + } + y, x := state.pop(), state.pop() + and := builder.AllocateInstruction() + and.AsBand(x, y) + builder.InsertInstruction(and) + value := and.Return() + state.push(value) + case wasm.OpcodeI32Or, wasm.OpcodeI64Or: + if state.unreachable { + break + } + y, x := state.pop(), state.pop() + or := builder.AllocateInstruction() + or.AsBor(x, y) + builder.InsertInstruction(or) + value := or.Return() + state.push(value) + case wasm.OpcodeI32Xor, wasm.OpcodeI64Xor: + if state.unreachable { + break + } + y, x := state.pop(), state.pop() + xor := builder.AllocateInstruction() + xor.AsBxor(x, y) + builder.InsertInstruction(xor) + value := xor.Return() + state.push(value) + case wasm.OpcodeI32Shl, wasm.OpcodeI64Shl: + if state.unreachable { + break + } + y, x := state.pop(), state.pop() + ishl := builder.AllocateInstruction() + ishl.AsIshl(x, y) + builder.InsertInstruction(ishl) + value := ishl.Return() + state.push(value) + case wasm.OpcodeI32ShrU, wasm.OpcodeI64ShrU: + if state.unreachable { + break + } + y, x := state.pop(), state.pop() + ishl := builder.AllocateInstruction() + ishl.AsUshr(x, y) + builder.InsertInstruction(ishl) + value := ishl.Return() + state.push(value) + case wasm.OpcodeI32ShrS, wasm.OpcodeI64ShrS: + if state.unreachable { + break + } + y, x := state.pop(), state.pop() + ishl := builder.AllocateInstruction() + ishl.AsSshr(x, y) + builder.InsertInstruction(ishl) + value := ishl.Return() + state.push(value) + case wasm.OpcodeI32Rotl, wasm.OpcodeI64Rotl: + if state.unreachable { + break + } + y, x := state.pop(), state.pop() + rotl := builder.AllocateInstruction() + rotl.AsRotl(x, y) + builder.InsertInstruction(rotl) + value := rotl.Return() + state.push(value) + case wasm.OpcodeI32Rotr, wasm.OpcodeI64Rotr: + if state.unreachable { + break + } + y, x := state.pop(), state.pop() + rotr := builder.AllocateInstruction() + rotr.AsRotr(x, y) + builder.InsertInstruction(rotr) + value := rotr.Return() + state.push(value) + case wasm.OpcodeI32Clz, wasm.OpcodeI64Clz: + if state.unreachable { + break + } + x := state.pop() + clz := builder.AllocateInstruction() + clz.AsClz(x) + builder.InsertInstruction(clz) + value := clz.Return() + state.push(value) + case wasm.OpcodeI32Ctz, wasm.OpcodeI64Ctz: + if state.unreachable { + break + } + x := state.pop() + ctz := builder.AllocateInstruction() + ctz.AsCtz(x) + builder.InsertInstruction(ctz) + value := ctz.Return() + state.push(value) + case wasm.OpcodeI32Popcnt, wasm.OpcodeI64Popcnt: + if state.unreachable { + break + } + x := state.pop() + popcnt := builder.AllocateInstruction() + popcnt.AsPopcnt(x) + builder.InsertInstruction(popcnt) + value := popcnt.Return() + state.push(value) + + case wasm.OpcodeI32WrapI64: + if state.unreachable { + break + } + x := state.pop() + wrap := builder.AllocateInstruction().AsIreduce(x, ssa.TypeI32).Insert(builder).Return() + state.push(wrap) + case wasm.OpcodeGlobalGet: + index := c.readI32u() + if state.unreachable { + break + } + v := c.getWasmGlobalValue(index, false) + state.push(v) + case wasm.OpcodeGlobalSet: + index := c.readI32u() + if state.unreachable { + break + } + v := state.pop() + c.setWasmGlobalValue(index, v) + case wasm.OpcodeLocalGet: + index := c.readI32u() + if state.unreachable { + break + } + variable := c.localVariable(index) + if _, ok := c.m.NonStaticLocals[c.wasmLocalFunctionIndex][index]; ok { + state.push(builder.MustFindValue(variable)) + } else { + // If a local is static, we can simply find it in the entry block which is either a function param + // or a zero value. This fast pass helps to avoid the overhead of searching the entire function plus + // avoid adding unnecessary block arguments. + // TODO: I think this optimization should be done in a SSA pass like passRedundantPhiEliminationOpt, + // but somehow there's some corner cases that it fails to optimize. + state.push(builder.MustFindValueInBlk(variable, c.ssaBuilder.EntryBlock())) + } + case wasm.OpcodeLocalSet: + index := c.readI32u() + if state.unreachable { + break + } + variable := c.localVariable(index) + newValue := state.pop() + builder.DefineVariableInCurrentBB(variable, newValue) + + case wasm.OpcodeLocalTee: + index := c.readI32u() + if state.unreachable { + break + } + variable := c.localVariable(index) + newValue := state.peek() + builder.DefineVariableInCurrentBB(variable, newValue) + + case wasm.OpcodeSelect, wasm.OpcodeTypedSelect: + if op == wasm.OpcodeTypedSelect { + state.pc += 2 // ignores the type which is only needed during validation. + } + + if state.unreachable { + break + } + + cond := state.pop() + v2 := state.pop() + v1 := state.pop() + + sl := builder.AllocateInstruction(). + AsSelect(cond, v1, v2). + Insert(builder). + Return() + state.push(sl) + + case wasm.OpcodeMemorySize: + state.pc++ // skips the memory index. + if state.unreachable { + break + } + + var memSizeInBytes ssa.Value + if c.offset.LocalMemoryBegin < 0 { + memInstPtr := builder.AllocateInstruction(). + AsLoad(c.moduleCtxPtrValue, c.offset.ImportedMemoryBegin.U32(), ssa.TypeI64). + Insert(builder). + Return() + + memSizeInBytes = builder.AllocateInstruction(). + AsLoad(memInstPtr, memoryInstanceBufSizeOffset, ssa.TypeI32). + Insert(builder). + Return() + } else { + memSizeInBytes = builder.AllocateInstruction(). + AsLoad(c.moduleCtxPtrValue, c.offset.LocalMemoryLen().U32(), ssa.TypeI32). + Insert(builder). + Return() + } + + amount := builder.AllocateInstruction() + amount.AsIconst32(uint32(wasm.MemoryPageSizeInBits)) + builder.InsertInstruction(amount) + memSize := builder.AllocateInstruction(). + AsUshr(memSizeInBytes, amount.Return()). + Insert(builder). + Return() + state.push(memSize) + + case wasm.OpcodeMemoryGrow: + state.pc++ // skips the memory index. + if state.unreachable { + break + } + + c.storeCallerModuleContext() + + pages := state.pop() + memoryGrowPtr := builder.AllocateInstruction(). + AsLoad(c.execCtxPtrValue, + wazevoapi.ExecutionContextOffsetMemoryGrowTrampolineAddress.U32(), + ssa.TypeI64, + ).Insert(builder).Return() + + args := c.allocateVarLengthValues(1, c.execCtxPtrValue, pages) + callGrowRet := builder. + AllocateInstruction(). + AsCallIndirect(memoryGrowPtr, &c.memoryGrowSig, args). + Insert(builder).Return() + state.push(callGrowRet) + + // After the memory grow, reload the cached memory base and len. + c.reloadMemoryBaseLen() + + case wasm.OpcodeI32Store, + wasm.OpcodeI64Store, + wasm.OpcodeF32Store, + wasm.OpcodeF64Store, + wasm.OpcodeI32Store8, + wasm.OpcodeI32Store16, + wasm.OpcodeI64Store8, + wasm.OpcodeI64Store16, + wasm.OpcodeI64Store32: + + _, offset := c.readMemArg() + if state.unreachable { + break + } + var opSize uint64 + var opcode ssa.Opcode + switch op { + case wasm.OpcodeI32Store, wasm.OpcodeF32Store: + opcode = ssa.OpcodeStore + opSize = 4 + case wasm.OpcodeI64Store, wasm.OpcodeF64Store: + opcode = ssa.OpcodeStore + opSize = 8 + case wasm.OpcodeI32Store8, wasm.OpcodeI64Store8: + opcode = ssa.OpcodeIstore8 + opSize = 1 + case wasm.OpcodeI32Store16, wasm.OpcodeI64Store16: + opcode = ssa.OpcodeIstore16 + opSize = 2 + case wasm.OpcodeI64Store32: + opcode = ssa.OpcodeIstore32 + opSize = 4 + default: + panic("BUG") + } + + value := state.pop() + baseAddr := state.pop() + addr := c.memOpSetup(baseAddr, uint64(offset), opSize) + builder.AllocateInstruction(). + AsStore(opcode, value, addr, offset). + Insert(builder) + + case wasm.OpcodeI32Load, + wasm.OpcodeI64Load, + wasm.OpcodeF32Load, + wasm.OpcodeF64Load, + wasm.OpcodeI32Load8S, + wasm.OpcodeI32Load8U, + wasm.OpcodeI32Load16S, + wasm.OpcodeI32Load16U, + wasm.OpcodeI64Load8S, + wasm.OpcodeI64Load8U, + wasm.OpcodeI64Load16S, + wasm.OpcodeI64Load16U, + wasm.OpcodeI64Load32S, + wasm.OpcodeI64Load32U: + _, offset := c.readMemArg() + if state.unreachable { + break + } + + var opSize uint64 + switch op { + case wasm.OpcodeI32Load, wasm.OpcodeF32Load: + opSize = 4 + case wasm.OpcodeI64Load, wasm.OpcodeF64Load: + opSize = 8 + case wasm.OpcodeI32Load8S, wasm.OpcodeI32Load8U: + opSize = 1 + case wasm.OpcodeI32Load16S, wasm.OpcodeI32Load16U: + opSize = 2 + case wasm.OpcodeI64Load8S, wasm.OpcodeI64Load8U: + opSize = 1 + case wasm.OpcodeI64Load16S, wasm.OpcodeI64Load16U: + opSize = 2 + case wasm.OpcodeI64Load32S, wasm.OpcodeI64Load32U: + opSize = 4 + default: + panic("BUG") + } + + baseAddr := state.pop() + addr := c.memOpSetup(baseAddr, uint64(offset), opSize) + load := builder.AllocateInstruction() + switch op { + case wasm.OpcodeI32Load: + load.AsLoad(addr, offset, ssa.TypeI32) + case wasm.OpcodeI64Load: + load.AsLoad(addr, offset, ssa.TypeI64) + case wasm.OpcodeF32Load: + load.AsLoad(addr, offset, ssa.TypeF32) + case wasm.OpcodeF64Load: + load.AsLoad(addr, offset, ssa.TypeF64) + case wasm.OpcodeI32Load8S: + load.AsExtLoad(ssa.OpcodeSload8, addr, offset, false) + case wasm.OpcodeI32Load8U: + load.AsExtLoad(ssa.OpcodeUload8, addr, offset, false) + case wasm.OpcodeI32Load16S: + load.AsExtLoad(ssa.OpcodeSload16, addr, offset, false) + case wasm.OpcodeI32Load16U: + load.AsExtLoad(ssa.OpcodeUload16, addr, offset, false) + case wasm.OpcodeI64Load8S: + load.AsExtLoad(ssa.OpcodeSload8, addr, offset, true) + case wasm.OpcodeI64Load8U: + load.AsExtLoad(ssa.OpcodeUload8, addr, offset, true) + case wasm.OpcodeI64Load16S: + load.AsExtLoad(ssa.OpcodeSload16, addr, offset, true) + case wasm.OpcodeI64Load16U: + load.AsExtLoad(ssa.OpcodeUload16, addr, offset, true) + case wasm.OpcodeI64Load32S: + load.AsExtLoad(ssa.OpcodeSload32, addr, offset, true) + case wasm.OpcodeI64Load32U: + load.AsExtLoad(ssa.OpcodeUload32, addr, offset, true) + default: + panic("BUG") + } + builder.InsertInstruction(load) + state.push(load.Return()) + case wasm.OpcodeBlock: + // Note: we do not need to create a BB for this as that would always have only one predecessor + // which is the current BB, and therefore it's always ok to merge them in any way. + + bt := c.readBlockType() + + if state.unreachable { + state.unreachableDepth++ + break + } + + followingBlk := builder.AllocateBasicBlock() + c.addBlockParamsFromWasmTypes(bt.Results, followingBlk) + + state.ctrlPush(controlFrame{ + kind: controlFrameKindBlock, + originalStackLenWithoutParam: len(state.values) - len(bt.Params), + followingBlock: followingBlk, + blockType: bt, + }) + case wasm.OpcodeLoop: + bt := c.readBlockType() + + if state.unreachable { + state.unreachableDepth++ + break + } + + loopHeader, afterLoopBlock := builder.AllocateBasicBlock(), builder.AllocateBasicBlock() + c.addBlockParamsFromWasmTypes(bt.Params, loopHeader) + c.addBlockParamsFromWasmTypes(bt.Results, afterLoopBlock) + + originalLen := len(state.values) - len(bt.Params) + state.ctrlPush(controlFrame{ + originalStackLenWithoutParam: originalLen, + kind: controlFrameKindLoop, + blk: loopHeader, + followingBlock: afterLoopBlock, + blockType: bt, + }) + + args := c.allocateVarLengthValues(originalLen) + args = args.Append(builder.VarLengthPool(), state.values[originalLen:]...) + + // Insert the jump to the header of loop. + br := builder.AllocateInstruction() + br.AsJump(args, loopHeader) + builder.InsertInstruction(br) + + c.switchTo(originalLen, loopHeader) + + if c.ensureTermination { + checkModuleExitCodePtr := builder.AllocateInstruction(). + AsLoad(c.execCtxPtrValue, + wazevoapi.ExecutionContextOffsetCheckModuleExitCodeTrampolineAddress.U32(), + ssa.TypeI64, + ).Insert(builder).Return() + + args := c.allocateVarLengthValues(1, c.execCtxPtrValue) + builder.AllocateInstruction(). + AsCallIndirect(checkModuleExitCodePtr, &c.checkModuleExitCodeSig, args). + Insert(builder) + } + case wasm.OpcodeIf: + bt := c.readBlockType() + + if state.unreachable { + state.unreachableDepth++ + break + } + + v := state.pop() + thenBlk, elseBlk, followingBlk := builder.AllocateBasicBlock(), builder.AllocateBasicBlock(), builder.AllocateBasicBlock() + + // We do not make the Wasm-level block parameters as SSA-level block params for if-else blocks + // since they won't be PHI and the definition is unique. + + // On the other hand, the following block after if-else-end will likely have + // multiple definitions (one in Then and another in Else blocks). + c.addBlockParamsFromWasmTypes(bt.Results, followingBlk) + + args := c.allocateVarLengthValues(len(bt.Params)) + args = args.Append(builder.VarLengthPool(), state.values[len(state.values)-len(bt.Params):]...) + + // Insert the conditional jump to the Else block. + brz := builder.AllocateInstruction() + brz.AsBrz(v, ssa.ValuesNil, elseBlk) + builder.InsertInstruction(brz) + + // Then, insert the jump to the Then block. + br := builder.AllocateInstruction() + br.AsJump(ssa.ValuesNil, thenBlk) + builder.InsertInstruction(br) + + state.ctrlPush(controlFrame{ + kind: controlFrameKindIfWithoutElse, + originalStackLenWithoutParam: len(state.values) - len(bt.Params), + blk: elseBlk, + followingBlock: followingBlk, + blockType: bt, + clonedArgs: args, + }) + + builder.SetCurrentBlock(thenBlk) + + // Then and Else (if exists) have only one predecessor. + builder.Seal(thenBlk) + builder.Seal(elseBlk) + case wasm.OpcodeElse: + ifctrl := state.ctrlPeekAt(0) + if unreachable := state.unreachable; unreachable && state.unreachableDepth > 0 { + // If it is currently in unreachable and is a nested if, + // we just remove the entire else block. + break + } + + ifctrl.kind = controlFrameKindIfWithElse + if !state.unreachable { + // If this Then block is currently reachable, we have to insert the branching to the following BB. + followingBlk := ifctrl.followingBlock // == the BB after if-then-else. + args := c.nPeekDup(len(ifctrl.blockType.Results)) + c.insertJumpToBlock(args, followingBlk) + } else { + state.unreachable = false + } + + // Reset the stack so that we can correctly handle the else block. + state.values = state.values[:ifctrl.originalStackLenWithoutParam] + elseBlk := ifctrl.blk + for _, arg := range ifctrl.clonedArgs.View() { + state.push(arg) + } + + builder.SetCurrentBlock(elseBlk) + + case wasm.OpcodeEnd: + if state.unreachableDepth > 0 { + state.unreachableDepth-- + break + } + + ctrl := state.ctrlPop() + followingBlk := ctrl.followingBlock + + unreachable := state.unreachable + if !unreachable { + // Top n-th args will be used as a result of the current control frame. + args := c.nPeekDup(len(ctrl.blockType.Results)) + + // Insert the unconditional branch to the target. + c.insertJumpToBlock(args, followingBlk) + } else { // recover from the unreachable state. + state.unreachable = false + } + + switch ctrl.kind { + case controlFrameKindFunction: + break // This is the very end of function. + case controlFrameKindLoop: + // Loop header block can be reached from any br/br_table contained in the loop, + // so now that we've reached End of it, we can seal it. + builder.Seal(ctrl.blk) + case controlFrameKindIfWithoutElse: + // If this is the end of Then block, we have to emit the empty Else block. + elseBlk := ctrl.blk + builder.SetCurrentBlock(elseBlk) + c.insertJumpToBlock(ctrl.clonedArgs, followingBlk) + } + + builder.Seal(followingBlk) + + // Ready to start translating the following block. + c.switchTo(ctrl.originalStackLenWithoutParam, followingBlk) + + case wasm.OpcodeBr: + labelIndex := c.readI32u() + if state.unreachable { + break + } + + targetBlk, argNum := state.brTargetArgNumFor(labelIndex) + args := c.nPeekDup(argNum) + c.insertJumpToBlock(args, targetBlk) + + state.unreachable = true + + case wasm.OpcodeBrIf: + labelIndex := c.readI32u() + if state.unreachable { + break + } + + v := state.pop() + + targetBlk, argNum := state.brTargetArgNumFor(labelIndex) + args := c.nPeekDup(argNum) + var sealTargetBlk bool + if c.needListener && targetBlk.ReturnBlock() { // In this case, we have to call the listener before returning. + // Save the currently active block. + current := builder.CurrentBlock() + + // Allocate the trampoline block to the return where we call the listener. + targetBlk = builder.AllocateBasicBlock() + builder.SetCurrentBlock(targetBlk) + sealTargetBlk = true + + c.callListenerAfter() + + instr := builder.AllocateInstruction() + instr.AsReturn(args) + builder.InsertInstruction(instr) + + args = ssa.ValuesNil + + // Revert the current block. + builder.SetCurrentBlock(current) + } + + // Insert the conditional jump to the target block. + brnz := builder.AllocateInstruction() + brnz.AsBrnz(v, args, targetBlk) + builder.InsertInstruction(brnz) + + if sealTargetBlk { + builder.Seal(targetBlk) + } + + // Insert the unconditional jump to the Else block which corresponds to after br_if. + elseBlk := builder.AllocateBasicBlock() + c.insertJumpToBlock(ssa.ValuesNil, elseBlk) + + // Now start translating the instructions after br_if. + builder.Seal(elseBlk) // Else of br_if has the current block as the only one successor. + builder.SetCurrentBlock(elseBlk) + + case wasm.OpcodeBrTable: + labels := state.tmpForBrTable + labels = labels[:0] + labelCount := c.readI32u() + for i := 0; i < int(labelCount); i++ { + labels = append(labels, c.readI32u()) + } + labels = append(labels, c.readI32u()) // default label. + if state.unreachable { + break + } + + index := state.pop() + if labelCount == 0 { // If this br_table is empty, we can just emit the unconditional jump. + targetBlk, argNum := state.brTargetArgNumFor(labels[0]) + args := c.nPeekDup(argNum) + c.insertJumpToBlock(args, targetBlk) + } else { + c.lowerBrTable(labels, index) + } + state.unreachable = true + + case wasm.OpcodeNop: + case wasm.OpcodeReturn: + if state.unreachable { + break + } + if c.needListener { + c.callListenerAfter() + } + + results := c.nPeekDup(c.results()) + instr := builder.AllocateInstruction() + + instr.AsReturn(results) + builder.InsertInstruction(instr) + state.unreachable = true + + case wasm.OpcodeUnreachable: + if state.unreachable { + break + } + exit := builder.AllocateInstruction() + exit.AsExitWithCode(c.execCtxPtrValue, wazevoapi.ExitCodeUnreachable) + builder.InsertInstruction(exit) + state.unreachable = true + + case wasm.OpcodeCallIndirect: + typeIndex := c.readI32u() + tableIndex := c.readI32u() + if state.unreachable { + break + } + c.lowerCallIndirect(typeIndex, tableIndex) + + case wasm.OpcodeCall: + fnIndex := c.readI32u() + if state.unreachable { + break + } + + var typIndex wasm.Index + if fnIndex < c.m.ImportFunctionCount { + // Before transfer the control to the callee, we have to store the current module's moduleContextPtr + // into execContext.callerModuleContextPtr in case when the callee is a Go function. + c.storeCallerModuleContext() + var fi int + for i := range c.m.ImportSection { + imp := &c.m.ImportSection[i] + if imp.Type == wasm.ExternTypeFunc { + if fi == int(fnIndex) { + typIndex = imp.DescFunc + break + } + fi++ + } + } + } else { + typIndex = c.m.FunctionSection[fnIndex-c.m.ImportFunctionCount] + } + typ := &c.m.TypeSection[typIndex] + + argN := len(typ.Params) + tail := len(state.values) - argN + vs := state.values[tail:] + state.values = state.values[:tail] + args := c.allocateVarLengthValues(2+len(vs), c.execCtxPtrValue) + + sig := c.signatures[typ] + call := builder.AllocateInstruction() + if fnIndex >= c.m.ImportFunctionCount { + args = args.Append(builder.VarLengthPool(), c.moduleCtxPtrValue) // This case the callee module is itself. + args = args.Append(builder.VarLengthPool(), vs...) + call.AsCall(FunctionIndexToFuncRef(fnIndex), sig, args) + builder.InsertInstruction(call) + } else { + // This case we have to read the address of the imported function from the module context. + moduleCtx := c.moduleCtxPtrValue + loadFuncPtr, loadModuleCtxPtr := builder.AllocateInstruction(), builder.AllocateInstruction() + funcPtrOffset, moduleCtxPtrOffset, _ := c.offset.ImportedFunctionOffset(fnIndex) + loadFuncPtr.AsLoad(moduleCtx, funcPtrOffset.U32(), ssa.TypeI64) + loadModuleCtxPtr.AsLoad(moduleCtx, moduleCtxPtrOffset.U32(), ssa.TypeI64) + builder.InsertInstruction(loadFuncPtr) + builder.InsertInstruction(loadModuleCtxPtr) + + args = args.Append(builder.VarLengthPool(), loadModuleCtxPtr.Return()) + args = args.Append(builder.VarLengthPool(), vs...) + call.AsCallIndirect(loadFuncPtr.Return(), sig, args) + builder.InsertInstruction(call) + } + + first, rest := call.Returns() + if first.Valid() { + state.push(first) + } + for _, v := range rest { + state.push(v) + } + + c.reloadAfterCall() + + case wasm.OpcodeDrop: + if state.unreachable { + break + } + _ = state.pop() + case wasm.OpcodeF64ConvertI32S, wasm.OpcodeF64ConvertI64S, wasm.OpcodeF64ConvertI32U, wasm.OpcodeF64ConvertI64U: + if state.unreachable { + break + } + result := builder.AllocateInstruction().AsFcvtFromInt( + state.pop(), + op == wasm.OpcodeF64ConvertI32S || op == wasm.OpcodeF64ConvertI64S, + true, + ).Insert(builder).Return() + state.push(result) + case wasm.OpcodeF32ConvertI32S, wasm.OpcodeF32ConvertI64S, wasm.OpcodeF32ConvertI32U, wasm.OpcodeF32ConvertI64U: + if state.unreachable { + break + } + result := builder.AllocateInstruction().AsFcvtFromInt( + state.pop(), + op == wasm.OpcodeF32ConvertI32S || op == wasm.OpcodeF32ConvertI64S, + false, + ).Insert(builder).Return() + state.push(result) + case wasm.OpcodeF32DemoteF64: + if state.unreachable { + break + } + cvt := builder.AllocateInstruction() + cvt.AsFdemote(state.pop()) + builder.InsertInstruction(cvt) + state.push(cvt.Return()) + case wasm.OpcodeF64PromoteF32: + if state.unreachable { + break + } + cvt := builder.AllocateInstruction() + cvt.AsFpromote(state.pop()) + builder.InsertInstruction(cvt) + state.push(cvt.Return()) + + case wasm.OpcodeVecPrefix: + state.pc++ + vecOp := c.wasmFunctionBody[state.pc] + switch vecOp { + case wasm.OpcodeVecV128Const: + state.pc++ + lo := binary.LittleEndian.Uint64(c.wasmFunctionBody[state.pc:]) + state.pc += 8 + hi := binary.LittleEndian.Uint64(c.wasmFunctionBody[state.pc:]) + state.pc += 7 + if state.unreachable { + break + } + ret := builder.AllocateInstruction().AsVconst(lo, hi).Insert(builder).Return() + state.push(ret) + case wasm.OpcodeVecV128Load: + _, offset := c.readMemArg() + if state.unreachable { + break + } + baseAddr := state.pop() + addr := c.memOpSetup(baseAddr, uint64(offset), 16) + load := builder.AllocateInstruction() + load.AsLoad(addr, offset, ssa.TypeV128) + builder.InsertInstruction(load) + state.push(load.Return()) + case wasm.OpcodeVecV128Load8Lane, wasm.OpcodeVecV128Load16Lane, wasm.OpcodeVecV128Load32Lane: + _, offset := c.readMemArg() + state.pc++ + if state.unreachable { + break + } + var lane ssa.VecLane + var loadOp ssa.Opcode + var opSize uint64 + switch vecOp { + case wasm.OpcodeVecV128Load8Lane: + loadOp, lane, opSize = ssa.OpcodeUload8, ssa.VecLaneI8x16, 1 + case wasm.OpcodeVecV128Load16Lane: + loadOp, lane, opSize = ssa.OpcodeUload16, ssa.VecLaneI16x8, 2 + case wasm.OpcodeVecV128Load32Lane: + loadOp, lane, opSize = ssa.OpcodeUload32, ssa.VecLaneI32x4, 4 + } + laneIndex := c.wasmFunctionBody[state.pc] + vector := state.pop() + baseAddr := state.pop() + addr := c.memOpSetup(baseAddr, uint64(offset), opSize) + load := builder.AllocateInstruction(). + AsExtLoad(loadOp, addr, offset, false). + Insert(builder).Return() + ret := builder.AllocateInstruction(). + AsInsertlane(vector, load, laneIndex, lane). + Insert(builder).Return() + state.push(ret) + case wasm.OpcodeVecV128Load64Lane: + _, offset := c.readMemArg() + state.pc++ + if state.unreachable { + break + } + laneIndex := c.wasmFunctionBody[state.pc] + vector := state.pop() + baseAddr := state.pop() + addr := c.memOpSetup(baseAddr, uint64(offset), 8) + load := builder.AllocateInstruction(). + AsLoad(addr, offset, ssa.TypeI64). + Insert(builder).Return() + ret := builder.AllocateInstruction(). + AsInsertlane(vector, load, laneIndex, ssa.VecLaneI64x2). + Insert(builder).Return() + state.push(ret) + + case wasm.OpcodeVecV128Load32zero, wasm.OpcodeVecV128Load64zero: + _, offset := c.readMemArg() + if state.unreachable { + break + } + + var scalarType ssa.Type + switch vecOp { + case wasm.OpcodeVecV128Load32zero: + scalarType = ssa.TypeF32 + case wasm.OpcodeVecV128Load64zero: + scalarType = ssa.TypeF64 + } + + baseAddr := state.pop() + addr := c.memOpSetup(baseAddr, uint64(offset), uint64(scalarType.Size())) + + ret := builder.AllocateInstruction(). + AsVZeroExtLoad(addr, offset, scalarType). + Insert(builder).Return() + state.push(ret) + + case wasm.OpcodeVecV128Load8x8u, wasm.OpcodeVecV128Load8x8s, + wasm.OpcodeVecV128Load16x4u, wasm.OpcodeVecV128Load16x4s, + wasm.OpcodeVecV128Load32x2u, wasm.OpcodeVecV128Load32x2s: + _, offset := c.readMemArg() + if state.unreachable { + break + } + var lane ssa.VecLane + var signed bool + switch vecOp { + case wasm.OpcodeVecV128Load8x8s: + signed = true + fallthrough + case wasm.OpcodeVecV128Load8x8u: + lane = ssa.VecLaneI8x16 + case wasm.OpcodeVecV128Load16x4s: + signed = true + fallthrough + case wasm.OpcodeVecV128Load16x4u: + lane = ssa.VecLaneI16x8 + case wasm.OpcodeVecV128Load32x2s: + signed = true + fallthrough + case wasm.OpcodeVecV128Load32x2u: + lane = ssa.VecLaneI32x4 + } + baseAddr := state.pop() + addr := c.memOpSetup(baseAddr, uint64(offset), 8) + load := builder.AllocateInstruction(). + AsLoad(addr, offset, ssa.TypeF64). + Insert(builder).Return() + ret := builder.AllocateInstruction(). + AsWiden(load, lane, signed, true). + Insert(builder).Return() + state.push(ret) + case wasm.OpcodeVecV128Load8Splat, wasm.OpcodeVecV128Load16Splat, + wasm.OpcodeVecV128Load32Splat, wasm.OpcodeVecV128Load64Splat: + _, offset := c.readMemArg() + if state.unreachable { + break + } + var lane ssa.VecLane + var opSize uint64 + switch vecOp { + case wasm.OpcodeVecV128Load8Splat: + lane, opSize = ssa.VecLaneI8x16, 1 + case wasm.OpcodeVecV128Load16Splat: + lane, opSize = ssa.VecLaneI16x8, 2 + case wasm.OpcodeVecV128Load32Splat: + lane, opSize = ssa.VecLaneI32x4, 4 + case wasm.OpcodeVecV128Load64Splat: + lane, opSize = ssa.VecLaneI64x2, 8 + } + baseAddr := state.pop() + addr := c.memOpSetup(baseAddr, uint64(offset), opSize) + ret := builder.AllocateInstruction(). + AsLoadSplat(addr, offset, lane). + Insert(builder).Return() + state.push(ret) + case wasm.OpcodeVecV128Store: + _, offset := c.readMemArg() + if state.unreachable { + break + } + value := state.pop() + baseAddr := state.pop() + addr := c.memOpSetup(baseAddr, uint64(offset), 16) + builder.AllocateInstruction(). + AsStore(ssa.OpcodeStore, value, addr, offset). + Insert(builder) + case wasm.OpcodeVecV128Store8Lane, wasm.OpcodeVecV128Store16Lane, + wasm.OpcodeVecV128Store32Lane, wasm.OpcodeVecV128Store64Lane: + _, offset := c.readMemArg() + state.pc++ + if state.unreachable { + break + } + laneIndex := c.wasmFunctionBody[state.pc] + var storeOp ssa.Opcode + var lane ssa.VecLane + var opSize uint64 + switch vecOp { + case wasm.OpcodeVecV128Store8Lane: + storeOp, lane, opSize = ssa.OpcodeIstore8, ssa.VecLaneI8x16, 1 + case wasm.OpcodeVecV128Store16Lane: + storeOp, lane, opSize = ssa.OpcodeIstore16, ssa.VecLaneI16x8, 2 + case wasm.OpcodeVecV128Store32Lane: + storeOp, lane, opSize = ssa.OpcodeIstore32, ssa.VecLaneI32x4, 4 + case wasm.OpcodeVecV128Store64Lane: + storeOp, lane, opSize = ssa.OpcodeStore, ssa.VecLaneI64x2, 8 + } + vector := state.pop() + baseAddr := state.pop() + addr := c.memOpSetup(baseAddr, uint64(offset), opSize) + value := builder.AllocateInstruction(). + AsExtractlane(vector, laneIndex, lane, false). + Insert(builder).Return() + builder.AllocateInstruction(). + AsStore(storeOp, value, addr, offset). + Insert(builder) + case wasm.OpcodeVecV128Not: + if state.unreachable { + break + } + v1 := state.pop() + ret := builder.AllocateInstruction().AsVbnot(v1).Insert(builder).Return() + state.push(ret) + case wasm.OpcodeVecV128And: + if state.unreachable { + break + } + v2 := state.pop() + v1 := state.pop() + ret := builder.AllocateInstruction().AsVband(v1, v2).Insert(builder).Return() + state.push(ret) + case wasm.OpcodeVecV128AndNot: + if state.unreachable { + break + } + v2 := state.pop() + v1 := state.pop() + ret := builder.AllocateInstruction().AsVbandnot(v1, v2).Insert(builder).Return() + state.push(ret) + case wasm.OpcodeVecV128Or: + if state.unreachable { + break + } + v2 := state.pop() + v1 := state.pop() + ret := builder.AllocateInstruction().AsVbor(v1, v2).Insert(builder).Return() + state.push(ret) + case wasm.OpcodeVecV128Xor: + if state.unreachable { + break + } + v2 := state.pop() + v1 := state.pop() + ret := builder.AllocateInstruction().AsVbxor(v1, v2).Insert(builder).Return() + state.push(ret) + case wasm.OpcodeVecV128Bitselect: + if state.unreachable { + break + } + c := state.pop() + v2 := state.pop() + v1 := state.pop() + ret := builder.AllocateInstruction().AsVbitselect(c, v1, v2).Insert(builder).Return() + state.push(ret) + case wasm.OpcodeVecV128AnyTrue: + if state.unreachable { + break + } + v1 := state.pop() + ret := builder.AllocateInstruction().AsVanyTrue(v1).Insert(builder).Return() + state.push(ret) + case wasm.OpcodeVecI8x16AllTrue, wasm.OpcodeVecI16x8AllTrue, wasm.OpcodeVecI32x4AllTrue, wasm.OpcodeVecI64x2AllTrue: + if state.unreachable { + break + } + var lane ssa.VecLane + switch vecOp { + case wasm.OpcodeVecI8x16AllTrue: + lane = ssa.VecLaneI8x16 + case wasm.OpcodeVecI16x8AllTrue: + lane = ssa.VecLaneI16x8 + case wasm.OpcodeVecI32x4AllTrue: + lane = ssa.VecLaneI32x4 + case wasm.OpcodeVecI64x2AllTrue: + lane = ssa.VecLaneI64x2 + } + v1 := state.pop() + ret := builder.AllocateInstruction().AsVallTrue(v1, lane).Insert(builder).Return() + state.push(ret) + case wasm.OpcodeVecI8x16BitMask, wasm.OpcodeVecI16x8BitMask, wasm.OpcodeVecI32x4BitMask, wasm.OpcodeVecI64x2BitMask: + if state.unreachable { + break + } + var lane ssa.VecLane + switch vecOp { + case wasm.OpcodeVecI8x16BitMask: + lane = ssa.VecLaneI8x16 + case wasm.OpcodeVecI16x8BitMask: + lane = ssa.VecLaneI16x8 + case wasm.OpcodeVecI32x4BitMask: + lane = ssa.VecLaneI32x4 + case wasm.OpcodeVecI64x2BitMask: + lane = ssa.VecLaneI64x2 + } + v1 := state.pop() + ret := builder.AllocateInstruction().AsVhighBits(v1, lane).Insert(builder).Return() + state.push(ret) + case wasm.OpcodeVecI8x16Abs, wasm.OpcodeVecI16x8Abs, wasm.OpcodeVecI32x4Abs, wasm.OpcodeVecI64x2Abs: + if state.unreachable { + break + } + var lane ssa.VecLane + switch vecOp { + case wasm.OpcodeVecI8x16Abs: + lane = ssa.VecLaneI8x16 + case wasm.OpcodeVecI16x8Abs: + lane = ssa.VecLaneI16x8 + case wasm.OpcodeVecI32x4Abs: + lane = ssa.VecLaneI32x4 + case wasm.OpcodeVecI64x2Abs: + lane = ssa.VecLaneI64x2 + } + v1 := state.pop() + ret := builder.AllocateInstruction().AsVIabs(v1, lane).Insert(builder).Return() + state.push(ret) + case wasm.OpcodeVecI8x16Neg, wasm.OpcodeVecI16x8Neg, wasm.OpcodeVecI32x4Neg, wasm.OpcodeVecI64x2Neg: + if state.unreachable { + break + } + var lane ssa.VecLane + switch vecOp { + case wasm.OpcodeVecI8x16Neg: + lane = ssa.VecLaneI8x16 + case wasm.OpcodeVecI16x8Neg: + lane = ssa.VecLaneI16x8 + case wasm.OpcodeVecI32x4Neg: + lane = ssa.VecLaneI32x4 + case wasm.OpcodeVecI64x2Neg: + lane = ssa.VecLaneI64x2 + } + v1 := state.pop() + ret := builder.AllocateInstruction().AsVIneg(v1, lane).Insert(builder).Return() + state.push(ret) + case wasm.OpcodeVecI8x16Popcnt: + if state.unreachable { + break + } + lane := ssa.VecLaneI8x16 + v1 := state.pop() + + ret := builder.AllocateInstruction().AsVIpopcnt(v1, lane).Insert(builder).Return() + state.push(ret) + case wasm.OpcodeVecI8x16Add, wasm.OpcodeVecI16x8Add, wasm.OpcodeVecI32x4Add, wasm.OpcodeVecI64x2Add: + if state.unreachable { + break + } + var lane ssa.VecLane + switch vecOp { + case wasm.OpcodeVecI8x16Add: + lane = ssa.VecLaneI8x16 + case wasm.OpcodeVecI16x8Add: + lane = ssa.VecLaneI16x8 + case wasm.OpcodeVecI32x4Add: + lane = ssa.VecLaneI32x4 + case wasm.OpcodeVecI64x2Add: + lane = ssa.VecLaneI64x2 + } + v2 := state.pop() + v1 := state.pop() + ret := builder.AllocateInstruction().AsVIadd(v1, v2, lane).Insert(builder).Return() + state.push(ret) + case wasm.OpcodeVecI8x16AddSatS, wasm.OpcodeVecI16x8AddSatS: + if state.unreachable { + break + } + var lane ssa.VecLane + switch vecOp { + case wasm.OpcodeVecI8x16AddSatS: + lane = ssa.VecLaneI8x16 + case wasm.OpcodeVecI16x8AddSatS: + lane = ssa.VecLaneI16x8 + } + v2 := state.pop() + v1 := state.pop() + ret := builder.AllocateInstruction().AsVSaddSat(v1, v2, lane).Insert(builder).Return() + state.push(ret) + case wasm.OpcodeVecI8x16AddSatU, wasm.OpcodeVecI16x8AddSatU: + if state.unreachable { + break + } + var lane ssa.VecLane + switch vecOp { + case wasm.OpcodeVecI8x16AddSatU: + lane = ssa.VecLaneI8x16 + case wasm.OpcodeVecI16x8AddSatU: + lane = ssa.VecLaneI16x8 + } + v2 := state.pop() + v1 := state.pop() + ret := builder.AllocateInstruction().AsVUaddSat(v1, v2, lane).Insert(builder).Return() + state.push(ret) + case wasm.OpcodeVecI8x16SubSatS, wasm.OpcodeVecI16x8SubSatS: + if state.unreachable { + break + } + var lane ssa.VecLane + switch vecOp { + case wasm.OpcodeVecI8x16SubSatS: + lane = ssa.VecLaneI8x16 + case wasm.OpcodeVecI16x8SubSatS: + lane = ssa.VecLaneI16x8 + } + v2 := state.pop() + v1 := state.pop() + ret := builder.AllocateInstruction().AsVSsubSat(v1, v2, lane).Insert(builder).Return() + state.push(ret) + case wasm.OpcodeVecI8x16SubSatU, wasm.OpcodeVecI16x8SubSatU: + if state.unreachable { + break + } + var lane ssa.VecLane + switch vecOp { + case wasm.OpcodeVecI8x16SubSatU: + lane = ssa.VecLaneI8x16 + case wasm.OpcodeVecI16x8SubSatU: + lane = ssa.VecLaneI16x8 + } + v2 := state.pop() + v1 := state.pop() + ret := builder.AllocateInstruction().AsVUsubSat(v1, v2, lane).Insert(builder).Return() + state.push(ret) + + case wasm.OpcodeVecI8x16Sub, wasm.OpcodeVecI16x8Sub, wasm.OpcodeVecI32x4Sub, wasm.OpcodeVecI64x2Sub: + if state.unreachable { + break + } + var lane ssa.VecLane + switch vecOp { + case wasm.OpcodeVecI8x16Sub: + lane = ssa.VecLaneI8x16 + case wasm.OpcodeVecI16x8Sub: + lane = ssa.VecLaneI16x8 + case wasm.OpcodeVecI32x4Sub: + lane = ssa.VecLaneI32x4 + case wasm.OpcodeVecI64x2Sub: + lane = ssa.VecLaneI64x2 + } + v2 := state.pop() + v1 := state.pop() + ret := builder.AllocateInstruction().AsVIsub(v1, v2, lane).Insert(builder).Return() + state.push(ret) + case wasm.OpcodeVecI8x16MinS, wasm.OpcodeVecI16x8MinS, wasm.OpcodeVecI32x4MinS: + if state.unreachable { + break + } + var lane ssa.VecLane + switch vecOp { + case wasm.OpcodeVecI8x16MinS: + lane = ssa.VecLaneI8x16 + case wasm.OpcodeVecI16x8MinS: + lane = ssa.VecLaneI16x8 + case wasm.OpcodeVecI32x4MinS: + lane = ssa.VecLaneI32x4 + } + v2 := state.pop() + v1 := state.pop() + ret := builder.AllocateInstruction().AsVImin(v1, v2, lane).Insert(builder).Return() + state.push(ret) + case wasm.OpcodeVecI8x16MinU, wasm.OpcodeVecI16x8MinU, wasm.OpcodeVecI32x4MinU: + if state.unreachable { + break + } + var lane ssa.VecLane + switch vecOp { + case wasm.OpcodeVecI8x16MinU: + lane = ssa.VecLaneI8x16 + case wasm.OpcodeVecI16x8MinU: + lane = ssa.VecLaneI16x8 + case wasm.OpcodeVecI32x4MinU: + lane = ssa.VecLaneI32x4 + } + v2 := state.pop() + v1 := state.pop() + ret := builder.AllocateInstruction().AsVUmin(v1, v2, lane).Insert(builder).Return() + state.push(ret) + case wasm.OpcodeVecI8x16MaxS, wasm.OpcodeVecI16x8MaxS, wasm.OpcodeVecI32x4MaxS: + if state.unreachable { + break + } + var lane ssa.VecLane + switch vecOp { + case wasm.OpcodeVecI8x16MaxS: + lane = ssa.VecLaneI8x16 + case wasm.OpcodeVecI16x8MaxS: + lane = ssa.VecLaneI16x8 + case wasm.OpcodeVecI32x4MaxS: + lane = ssa.VecLaneI32x4 + } + v2 := state.pop() + v1 := state.pop() + ret := builder.AllocateInstruction().AsVImax(v1, v2, lane).Insert(builder).Return() + state.push(ret) + case wasm.OpcodeVecI8x16MaxU, wasm.OpcodeVecI16x8MaxU, wasm.OpcodeVecI32x4MaxU: + if state.unreachable { + break + } + var lane ssa.VecLane + switch vecOp { + case wasm.OpcodeVecI8x16MaxU: + lane = ssa.VecLaneI8x16 + case wasm.OpcodeVecI16x8MaxU: + lane = ssa.VecLaneI16x8 + case wasm.OpcodeVecI32x4MaxU: + lane = ssa.VecLaneI32x4 + } + v2 := state.pop() + v1 := state.pop() + ret := builder.AllocateInstruction().AsVUmax(v1, v2, lane).Insert(builder).Return() + state.push(ret) + case wasm.OpcodeVecI8x16AvgrU, wasm.OpcodeVecI16x8AvgrU: + if state.unreachable { + break + } + var lane ssa.VecLane + switch vecOp { + case wasm.OpcodeVecI8x16AvgrU: + lane = ssa.VecLaneI8x16 + case wasm.OpcodeVecI16x8AvgrU: + lane = ssa.VecLaneI16x8 + } + v2 := state.pop() + v1 := state.pop() + ret := builder.AllocateInstruction().AsVAvgRound(v1, v2, lane).Insert(builder).Return() + state.push(ret) + case wasm.OpcodeVecI16x8Mul, wasm.OpcodeVecI32x4Mul, wasm.OpcodeVecI64x2Mul: + if state.unreachable { + break + } + var lane ssa.VecLane + switch vecOp { + case wasm.OpcodeVecI16x8Mul: + lane = ssa.VecLaneI16x8 + case wasm.OpcodeVecI32x4Mul: + lane = ssa.VecLaneI32x4 + case wasm.OpcodeVecI64x2Mul: + lane = ssa.VecLaneI64x2 + } + v2 := state.pop() + v1 := state.pop() + ret := builder.AllocateInstruction().AsVImul(v1, v2, lane).Insert(builder).Return() + state.push(ret) + case wasm.OpcodeVecI16x8Q15mulrSatS: + if state.unreachable { + break + } + v2 := state.pop() + v1 := state.pop() + ret := builder.AllocateInstruction().AsSqmulRoundSat(v1, v2, ssa.VecLaneI16x8).Insert(builder).Return() + state.push(ret) + case wasm.OpcodeVecI8x16Eq, wasm.OpcodeVecI16x8Eq, wasm.OpcodeVecI32x4Eq, wasm.OpcodeVecI64x2Eq: + if state.unreachable { + break + } + var lane ssa.VecLane + switch vecOp { + case wasm.OpcodeVecI8x16Eq: + lane = ssa.VecLaneI8x16 + case wasm.OpcodeVecI16x8Eq: + lane = ssa.VecLaneI16x8 + case wasm.OpcodeVecI32x4Eq: + lane = ssa.VecLaneI32x4 + case wasm.OpcodeVecI64x2Eq: + lane = ssa.VecLaneI64x2 + } + v2 := state.pop() + v1 := state.pop() + ret := builder.AllocateInstruction(). + AsVIcmp(v1, v2, ssa.IntegerCmpCondEqual, lane).Insert(builder).Return() + state.push(ret) + case wasm.OpcodeVecI8x16Ne, wasm.OpcodeVecI16x8Ne, wasm.OpcodeVecI32x4Ne, wasm.OpcodeVecI64x2Ne: + if state.unreachable { + break + } + var lane ssa.VecLane + switch vecOp { + case wasm.OpcodeVecI8x16Ne: + lane = ssa.VecLaneI8x16 + case wasm.OpcodeVecI16x8Ne: + lane = ssa.VecLaneI16x8 + case wasm.OpcodeVecI32x4Ne: + lane = ssa.VecLaneI32x4 + case wasm.OpcodeVecI64x2Ne: + lane = ssa.VecLaneI64x2 + } + v2 := state.pop() + v1 := state.pop() + ret := builder.AllocateInstruction(). + AsVIcmp(v1, v2, ssa.IntegerCmpCondNotEqual, lane).Insert(builder).Return() + state.push(ret) + case wasm.OpcodeVecI8x16LtS, wasm.OpcodeVecI16x8LtS, wasm.OpcodeVecI32x4LtS, wasm.OpcodeVecI64x2LtS: + if state.unreachable { + break + } + var lane ssa.VecLane + switch vecOp { + case wasm.OpcodeVecI8x16LtS: + lane = ssa.VecLaneI8x16 + case wasm.OpcodeVecI16x8LtS: + lane = ssa.VecLaneI16x8 + case wasm.OpcodeVecI32x4LtS: + lane = ssa.VecLaneI32x4 + case wasm.OpcodeVecI64x2LtS: + lane = ssa.VecLaneI64x2 + } + v2 := state.pop() + v1 := state.pop() + ret := builder.AllocateInstruction(). + AsVIcmp(v1, v2, ssa.IntegerCmpCondSignedLessThan, lane).Insert(builder).Return() + state.push(ret) + case wasm.OpcodeVecI8x16LtU, wasm.OpcodeVecI16x8LtU, wasm.OpcodeVecI32x4LtU: + if state.unreachable { + break + } + var lane ssa.VecLane + switch vecOp { + case wasm.OpcodeVecI8x16LtU: + lane = ssa.VecLaneI8x16 + case wasm.OpcodeVecI16x8LtU: + lane = ssa.VecLaneI16x8 + case wasm.OpcodeVecI32x4LtU: + lane = ssa.VecLaneI32x4 + } + v2 := state.pop() + v1 := state.pop() + ret := builder.AllocateInstruction(). + AsVIcmp(v1, v2, ssa.IntegerCmpCondUnsignedLessThan, lane).Insert(builder).Return() + state.push(ret) + case wasm.OpcodeVecI8x16LeS, wasm.OpcodeVecI16x8LeS, wasm.OpcodeVecI32x4LeS, wasm.OpcodeVecI64x2LeS: + if state.unreachable { + break + } + var lane ssa.VecLane + switch vecOp { + case wasm.OpcodeVecI8x16LeS: + lane = ssa.VecLaneI8x16 + case wasm.OpcodeVecI16x8LeS: + lane = ssa.VecLaneI16x8 + case wasm.OpcodeVecI32x4LeS: + lane = ssa.VecLaneI32x4 + case wasm.OpcodeVecI64x2LeS: + lane = ssa.VecLaneI64x2 + } + v2 := state.pop() + v1 := state.pop() + ret := builder.AllocateInstruction(). + AsVIcmp(v1, v2, ssa.IntegerCmpCondSignedLessThanOrEqual, lane).Insert(builder).Return() + state.push(ret) + case wasm.OpcodeVecI8x16LeU, wasm.OpcodeVecI16x8LeU, wasm.OpcodeVecI32x4LeU: + if state.unreachable { + break + } + var lane ssa.VecLane + switch vecOp { + case wasm.OpcodeVecI8x16LeU: + lane = ssa.VecLaneI8x16 + case wasm.OpcodeVecI16x8LeU: + lane = ssa.VecLaneI16x8 + case wasm.OpcodeVecI32x4LeU: + lane = ssa.VecLaneI32x4 + } + v2 := state.pop() + v1 := state.pop() + ret := builder.AllocateInstruction(). + AsVIcmp(v1, v2, ssa.IntegerCmpCondUnsignedLessThanOrEqual, lane).Insert(builder).Return() + state.push(ret) + case wasm.OpcodeVecI8x16GtS, wasm.OpcodeVecI16x8GtS, wasm.OpcodeVecI32x4GtS, wasm.OpcodeVecI64x2GtS: + if state.unreachable { + break + } + var lane ssa.VecLane + switch vecOp { + case wasm.OpcodeVecI8x16GtS: + lane = ssa.VecLaneI8x16 + case wasm.OpcodeVecI16x8GtS: + lane = ssa.VecLaneI16x8 + case wasm.OpcodeVecI32x4GtS: + lane = ssa.VecLaneI32x4 + case wasm.OpcodeVecI64x2GtS: + lane = ssa.VecLaneI64x2 + } + v2 := state.pop() + v1 := state.pop() + ret := builder.AllocateInstruction(). + AsVIcmp(v1, v2, ssa.IntegerCmpCondSignedGreaterThan, lane).Insert(builder).Return() + state.push(ret) + case wasm.OpcodeVecI8x16GtU, wasm.OpcodeVecI16x8GtU, wasm.OpcodeVecI32x4GtU: + if state.unreachable { + break + } + var lane ssa.VecLane + switch vecOp { + case wasm.OpcodeVecI8x16GtU: + lane = ssa.VecLaneI8x16 + case wasm.OpcodeVecI16x8GtU: + lane = ssa.VecLaneI16x8 + case wasm.OpcodeVecI32x4GtU: + lane = ssa.VecLaneI32x4 + } + v2 := state.pop() + v1 := state.pop() + ret := builder.AllocateInstruction(). + AsVIcmp(v1, v2, ssa.IntegerCmpCondUnsignedGreaterThan, lane).Insert(builder).Return() + state.push(ret) + case wasm.OpcodeVecI8x16GeS, wasm.OpcodeVecI16x8GeS, wasm.OpcodeVecI32x4GeS, wasm.OpcodeVecI64x2GeS: + if state.unreachable { + break + } + var lane ssa.VecLane + switch vecOp { + case wasm.OpcodeVecI8x16GeS: + lane = ssa.VecLaneI8x16 + case wasm.OpcodeVecI16x8GeS: + lane = ssa.VecLaneI16x8 + case wasm.OpcodeVecI32x4GeS: + lane = ssa.VecLaneI32x4 + case wasm.OpcodeVecI64x2GeS: + lane = ssa.VecLaneI64x2 + } + v2 := state.pop() + v1 := state.pop() + ret := builder.AllocateInstruction(). + AsVIcmp(v1, v2, ssa.IntegerCmpCondSignedGreaterThanOrEqual, lane).Insert(builder).Return() + state.push(ret) + case wasm.OpcodeVecI8x16GeU, wasm.OpcodeVecI16x8GeU, wasm.OpcodeVecI32x4GeU: + if state.unreachable { + break + } + var lane ssa.VecLane + switch vecOp { + case wasm.OpcodeVecI8x16GeU: + lane = ssa.VecLaneI8x16 + case wasm.OpcodeVecI16x8GeU: + lane = ssa.VecLaneI16x8 + case wasm.OpcodeVecI32x4GeU: + lane = ssa.VecLaneI32x4 + } + v2 := state.pop() + v1 := state.pop() + ret := builder.AllocateInstruction(). + AsVIcmp(v1, v2, ssa.IntegerCmpCondUnsignedGreaterThanOrEqual, lane).Insert(builder).Return() + state.push(ret) + case wasm.OpcodeVecF32x4Max, wasm.OpcodeVecF64x2Max: + if state.unreachable { + break + } + var lane ssa.VecLane + switch vecOp { + case wasm.OpcodeVecF32x4Max: + lane = ssa.VecLaneF32x4 + case wasm.OpcodeVecF64x2Max: + lane = ssa.VecLaneF64x2 + } + v2 := state.pop() + v1 := state.pop() + ret := builder.AllocateInstruction().AsVFmax(v1, v2, lane).Insert(builder).Return() + state.push(ret) + case wasm.OpcodeVecF32x4Abs, wasm.OpcodeVecF64x2Abs: + if state.unreachable { + break + } + var lane ssa.VecLane + switch vecOp { + case wasm.OpcodeVecF32x4Abs: + lane = ssa.VecLaneF32x4 + case wasm.OpcodeVecF64x2Abs: + lane = ssa.VecLaneF64x2 + } + v1 := state.pop() + ret := builder.AllocateInstruction().AsVFabs(v1, lane).Insert(builder).Return() + state.push(ret) + case wasm.OpcodeVecF32x4Min, wasm.OpcodeVecF64x2Min: + if state.unreachable { + break + } + var lane ssa.VecLane + switch vecOp { + case wasm.OpcodeVecF32x4Min: + lane = ssa.VecLaneF32x4 + case wasm.OpcodeVecF64x2Min: + lane = ssa.VecLaneF64x2 + } + v2 := state.pop() + v1 := state.pop() + ret := builder.AllocateInstruction().AsVFmin(v1, v2, lane).Insert(builder).Return() + state.push(ret) + case wasm.OpcodeVecF32x4Neg, wasm.OpcodeVecF64x2Neg: + if state.unreachable { + break + } + var lane ssa.VecLane + switch vecOp { + case wasm.OpcodeVecF32x4Neg: + lane = ssa.VecLaneF32x4 + case wasm.OpcodeVecF64x2Neg: + lane = ssa.VecLaneF64x2 + } + v1 := state.pop() + ret := builder.AllocateInstruction().AsVFneg(v1, lane).Insert(builder).Return() + state.push(ret) + case wasm.OpcodeVecF32x4Sqrt, wasm.OpcodeVecF64x2Sqrt: + if state.unreachable { + break + } + var lane ssa.VecLane + switch vecOp { + case wasm.OpcodeVecF32x4Sqrt: + lane = ssa.VecLaneF32x4 + case wasm.OpcodeVecF64x2Sqrt: + lane = ssa.VecLaneF64x2 + } + v1 := state.pop() + ret := builder.AllocateInstruction().AsVSqrt(v1, lane).Insert(builder).Return() + state.push(ret) + + case wasm.OpcodeVecF32x4Add, wasm.OpcodeVecF64x2Add: + if state.unreachable { + break + } + var lane ssa.VecLane + switch vecOp { + case wasm.OpcodeVecF32x4Add: + lane = ssa.VecLaneF32x4 + case wasm.OpcodeVecF64x2Add: + lane = ssa.VecLaneF64x2 + } + v2 := state.pop() + v1 := state.pop() + ret := builder.AllocateInstruction().AsVFadd(v1, v2, lane).Insert(builder).Return() + state.push(ret) + case wasm.OpcodeVecF32x4Sub, wasm.OpcodeVecF64x2Sub: + if state.unreachable { + break + } + var lane ssa.VecLane + switch vecOp { + case wasm.OpcodeVecF32x4Sub: + lane = ssa.VecLaneF32x4 + case wasm.OpcodeVecF64x2Sub: + lane = ssa.VecLaneF64x2 + } + v2 := state.pop() + v1 := state.pop() + ret := builder.AllocateInstruction().AsVFsub(v1, v2, lane).Insert(builder).Return() + state.push(ret) + case wasm.OpcodeVecF32x4Mul, wasm.OpcodeVecF64x2Mul: + if state.unreachable { + break + } + var lane ssa.VecLane + switch vecOp { + case wasm.OpcodeVecF32x4Mul: + lane = ssa.VecLaneF32x4 + case wasm.OpcodeVecF64x2Mul: + lane = ssa.VecLaneF64x2 + } + v2 := state.pop() + v1 := state.pop() + ret := builder.AllocateInstruction().AsVFmul(v1, v2, lane).Insert(builder).Return() + state.push(ret) + case wasm.OpcodeVecF32x4Div, wasm.OpcodeVecF64x2Div: + if state.unreachable { + break + } + var lane ssa.VecLane + switch vecOp { + case wasm.OpcodeVecF32x4Div: + lane = ssa.VecLaneF32x4 + case wasm.OpcodeVecF64x2Div: + lane = ssa.VecLaneF64x2 + } + v2 := state.pop() + v1 := state.pop() + ret := builder.AllocateInstruction().AsVFdiv(v1, v2, lane).Insert(builder).Return() + state.push(ret) + + case wasm.OpcodeVecI16x8ExtaddPairwiseI8x16S, wasm.OpcodeVecI16x8ExtaddPairwiseI8x16U: + if state.unreachable { + break + } + v := state.pop() + signed := vecOp == wasm.OpcodeVecI16x8ExtaddPairwiseI8x16S + ret := builder.AllocateInstruction().AsExtIaddPairwise(v, ssa.VecLaneI8x16, signed).Insert(builder).Return() + state.push(ret) + + case wasm.OpcodeVecI32x4ExtaddPairwiseI16x8S, wasm.OpcodeVecI32x4ExtaddPairwiseI16x8U: + if state.unreachable { + break + } + v := state.pop() + signed := vecOp == wasm.OpcodeVecI32x4ExtaddPairwiseI16x8S + ret := builder.AllocateInstruction().AsExtIaddPairwise(v, ssa.VecLaneI16x8, signed).Insert(builder).Return() + state.push(ret) + + case wasm.OpcodeVecI16x8ExtMulLowI8x16S, wasm.OpcodeVecI16x8ExtMulLowI8x16U: + if state.unreachable { + break + } + v2 := state.pop() + v1 := state.pop() + ret := c.lowerExtMul( + v1, v2, + ssa.VecLaneI8x16, ssa.VecLaneI16x8, + vecOp == wasm.OpcodeVecI16x8ExtMulLowI8x16S, true) + state.push(ret) + + case wasm.OpcodeVecI16x8ExtMulHighI8x16S, wasm.OpcodeVecI16x8ExtMulHighI8x16U: + if state.unreachable { + break + } + v2 := state.pop() + v1 := state.pop() + ret := c.lowerExtMul( + v1, v2, + ssa.VecLaneI8x16, ssa.VecLaneI16x8, + vecOp == wasm.OpcodeVecI16x8ExtMulHighI8x16S, false) + state.push(ret) + + case wasm.OpcodeVecI32x4ExtMulLowI16x8S, wasm.OpcodeVecI32x4ExtMulLowI16x8U: + if state.unreachable { + break + } + v2 := state.pop() + v1 := state.pop() + ret := c.lowerExtMul( + v1, v2, + ssa.VecLaneI16x8, ssa.VecLaneI32x4, + vecOp == wasm.OpcodeVecI32x4ExtMulLowI16x8S, true) + state.push(ret) + + case wasm.OpcodeVecI32x4ExtMulHighI16x8S, wasm.OpcodeVecI32x4ExtMulHighI16x8U: + if state.unreachable { + break + } + v2 := state.pop() + v1 := state.pop() + ret := c.lowerExtMul( + v1, v2, + ssa.VecLaneI16x8, ssa.VecLaneI32x4, + vecOp == wasm.OpcodeVecI32x4ExtMulHighI16x8S, false) + state.push(ret) + case wasm.OpcodeVecI64x2ExtMulLowI32x4S, wasm.OpcodeVecI64x2ExtMulLowI32x4U: + if state.unreachable { + break + } + v2 := state.pop() + v1 := state.pop() + ret := c.lowerExtMul( + v1, v2, + ssa.VecLaneI32x4, ssa.VecLaneI64x2, + vecOp == wasm.OpcodeVecI64x2ExtMulLowI32x4S, true) + state.push(ret) + + case wasm.OpcodeVecI64x2ExtMulHighI32x4S, wasm.OpcodeVecI64x2ExtMulHighI32x4U: + if state.unreachable { + break + } + v2 := state.pop() + v1 := state.pop() + ret := c.lowerExtMul( + v1, v2, + ssa.VecLaneI32x4, ssa.VecLaneI64x2, + vecOp == wasm.OpcodeVecI64x2ExtMulHighI32x4S, false) + state.push(ret) + + case wasm.OpcodeVecI32x4DotI16x8S: + if state.unreachable { + break + } + v2 := state.pop() + v1 := state.pop() + + ret := builder.AllocateInstruction().AsWideningPairwiseDotProductS(v1, v2).Insert(builder).Return() + state.push(ret) + + case wasm.OpcodeVecF32x4Eq, wasm.OpcodeVecF64x2Eq: + if state.unreachable { + break + } + var lane ssa.VecLane + switch vecOp { + case wasm.OpcodeVecF32x4Eq: + lane = ssa.VecLaneF32x4 + case wasm.OpcodeVecF64x2Eq: + lane = ssa.VecLaneF64x2 + } + v2 := state.pop() + v1 := state.pop() + ret := builder.AllocateInstruction(). + AsVFcmp(v1, v2, ssa.FloatCmpCondEqual, lane).Insert(builder).Return() + state.push(ret) + case wasm.OpcodeVecF32x4Ne, wasm.OpcodeVecF64x2Ne: + if state.unreachable { + break + } + var lane ssa.VecLane + switch vecOp { + case wasm.OpcodeVecF32x4Ne: + lane = ssa.VecLaneF32x4 + case wasm.OpcodeVecF64x2Ne: + lane = ssa.VecLaneF64x2 + } + v2 := state.pop() + v1 := state.pop() + ret := builder.AllocateInstruction(). + AsVFcmp(v1, v2, ssa.FloatCmpCondNotEqual, lane).Insert(builder).Return() + state.push(ret) + case wasm.OpcodeVecF32x4Lt, wasm.OpcodeVecF64x2Lt: + if state.unreachable { + break + } + var lane ssa.VecLane + switch vecOp { + case wasm.OpcodeVecF32x4Lt: + lane = ssa.VecLaneF32x4 + case wasm.OpcodeVecF64x2Lt: + lane = ssa.VecLaneF64x2 + } + v2 := state.pop() + v1 := state.pop() + ret := builder.AllocateInstruction(). + AsVFcmp(v1, v2, ssa.FloatCmpCondLessThan, lane).Insert(builder).Return() + state.push(ret) + case wasm.OpcodeVecF32x4Le, wasm.OpcodeVecF64x2Le: + if state.unreachable { + break + } + var lane ssa.VecLane + switch vecOp { + case wasm.OpcodeVecF32x4Le: + lane = ssa.VecLaneF32x4 + case wasm.OpcodeVecF64x2Le: + lane = ssa.VecLaneF64x2 + } + v2 := state.pop() + v1 := state.pop() + ret := builder.AllocateInstruction(). + AsVFcmp(v1, v2, ssa.FloatCmpCondLessThanOrEqual, lane).Insert(builder).Return() + state.push(ret) + case wasm.OpcodeVecF32x4Gt, wasm.OpcodeVecF64x2Gt: + if state.unreachable { + break + } + var lane ssa.VecLane + switch vecOp { + case wasm.OpcodeVecF32x4Gt: + lane = ssa.VecLaneF32x4 + case wasm.OpcodeVecF64x2Gt: + lane = ssa.VecLaneF64x2 + } + v2 := state.pop() + v1 := state.pop() + ret := builder.AllocateInstruction(). + AsVFcmp(v1, v2, ssa.FloatCmpCondGreaterThan, lane).Insert(builder).Return() + state.push(ret) + case wasm.OpcodeVecF32x4Ge, wasm.OpcodeVecF64x2Ge: + if state.unreachable { + break + } + var lane ssa.VecLane + switch vecOp { + case wasm.OpcodeVecF32x4Ge: + lane = ssa.VecLaneF32x4 + case wasm.OpcodeVecF64x2Ge: + lane = ssa.VecLaneF64x2 + } + v2 := state.pop() + v1 := state.pop() + ret := builder.AllocateInstruction(). + AsVFcmp(v1, v2, ssa.FloatCmpCondGreaterThanOrEqual, lane).Insert(builder).Return() + state.push(ret) + case wasm.OpcodeVecF32x4Ceil, wasm.OpcodeVecF64x2Ceil: + if state.unreachable { + break + } + var lane ssa.VecLane + switch vecOp { + case wasm.OpcodeVecF32x4Ceil: + lane = ssa.VecLaneF32x4 + case wasm.OpcodeVecF64x2Ceil: + lane = ssa.VecLaneF64x2 + } + v1 := state.pop() + ret := builder.AllocateInstruction().AsVCeil(v1, lane).Insert(builder).Return() + state.push(ret) + case wasm.OpcodeVecF32x4Floor, wasm.OpcodeVecF64x2Floor: + if state.unreachable { + break + } + var lane ssa.VecLane + switch vecOp { + case wasm.OpcodeVecF32x4Floor: + lane = ssa.VecLaneF32x4 + case wasm.OpcodeVecF64x2Floor: + lane = ssa.VecLaneF64x2 + } + v1 := state.pop() + ret := builder.AllocateInstruction().AsVFloor(v1, lane).Insert(builder).Return() + state.push(ret) + case wasm.OpcodeVecF32x4Trunc, wasm.OpcodeVecF64x2Trunc: + if state.unreachable { + break + } + var lane ssa.VecLane + switch vecOp { + case wasm.OpcodeVecF32x4Trunc: + lane = ssa.VecLaneF32x4 + case wasm.OpcodeVecF64x2Trunc: + lane = ssa.VecLaneF64x2 + } + v1 := state.pop() + ret := builder.AllocateInstruction().AsVTrunc(v1, lane).Insert(builder).Return() + state.push(ret) + case wasm.OpcodeVecF32x4Nearest, wasm.OpcodeVecF64x2Nearest: + if state.unreachable { + break + } + var lane ssa.VecLane + switch vecOp { + case wasm.OpcodeVecF32x4Nearest: + lane = ssa.VecLaneF32x4 + case wasm.OpcodeVecF64x2Nearest: + lane = ssa.VecLaneF64x2 + } + v1 := state.pop() + ret := builder.AllocateInstruction().AsVNearest(v1, lane).Insert(builder).Return() + state.push(ret) + case wasm.OpcodeVecF32x4Pmin, wasm.OpcodeVecF64x2Pmin: + if state.unreachable { + break + } + var lane ssa.VecLane + switch vecOp { + case wasm.OpcodeVecF32x4Pmin: + lane = ssa.VecLaneF32x4 + case wasm.OpcodeVecF64x2Pmin: + lane = ssa.VecLaneF64x2 + } + v2 := state.pop() + v1 := state.pop() + ret := builder.AllocateInstruction().AsVMinPseudo(v1, v2, lane).Insert(builder).Return() + state.push(ret) + case wasm.OpcodeVecF32x4Pmax, wasm.OpcodeVecF64x2Pmax: + if state.unreachable { + break + } + var lane ssa.VecLane + switch vecOp { + case wasm.OpcodeVecF32x4Pmax: + lane = ssa.VecLaneF32x4 + case wasm.OpcodeVecF64x2Pmax: + lane = ssa.VecLaneF64x2 + } + v2 := state.pop() + v1 := state.pop() + ret := builder.AllocateInstruction().AsVMaxPseudo(v1, v2, lane).Insert(builder).Return() + state.push(ret) + case wasm.OpcodeVecI32x4TruncSatF32x4S, wasm.OpcodeVecI32x4TruncSatF32x4U: + if state.unreachable { + break + } + v1 := state.pop() + ret := builder.AllocateInstruction(). + AsVFcvtToIntSat(v1, ssa.VecLaneF32x4, vecOp == wasm.OpcodeVecI32x4TruncSatF32x4S).Insert(builder).Return() + state.push(ret) + case wasm.OpcodeVecI32x4TruncSatF64x2SZero, wasm.OpcodeVecI32x4TruncSatF64x2UZero: + if state.unreachable { + break + } + v1 := state.pop() + ret := builder.AllocateInstruction(). + AsVFcvtToIntSat(v1, ssa.VecLaneF64x2, vecOp == wasm.OpcodeVecI32x4TruncSatF64x2SZero).Insert(builder).Return() + state.push(ret) + case wasm.OpcodeVecF32x4ConvertI32x4S, wasm.OpcodeVecF32x4ConvertI32x4U: + if state.unreachable { + break + } + v1 := state.pop() + ret := builder.AllocateInstruction(). + AsVFcvtFromInt(v1, ssa.VecLaneF32x4, vecOp == wasm.OpcodeVecF32x4ConvertI32x4S).Insert(builder).Return() + state.push(ret) + case wasm.OpcodeVecF64x2ConvertLowI32x4S, wasm.OpcodeVecF64x2ConvertLowI32x4U: + if state.unreachable { + break + } + v1 := state.pop() + if runtime.GOARCH == "arm64" { + // TODO: this is weird. fix. + v1 = builder.AllocateInstruction(). + AsWiden(v1, ssa.VecLaneI32x4, vecOp == wasm.OpcodeVecF64x2ConvertLowI32x4S, true).Insert(builder).Return() + } + ret := builder.AllocateInstruction(). + AsVFcvtFromInt(v1, ssa.VecLaneF64x2, vecOp == wasm.OpcodeVecF64x2ConvertLowI32x4S). + Insert(builder).Return() + state.push(ret) + case wasm.OpcodeVecI8x16NarrowI16x8S, wasm.OpcodeVecI8x16NarrowI16x8U: + if state.unreachable { + break + } + v2 := state.pop() + v1 := state.pop() + ret := builder.AllocateInstruction(). + AsNarrow(v1, v2, ssa.VecLaneI16x8, vecOp == wasm.OpcodeVecI8x16NarrowI16x8S). + Insert(builder).Return() + state.push(ret) + case wasm.OpcodeVecI16x8NarrowI32x4S, wasm.OpcodeVecI16x8NarrowI32x4U: + if state.unreachable { + break + } + v2 := state.pop() + v1 := state.pop() + ret := builder.AllocateInstruction(). + AsNarrow(v1, v2, ssa.VecLaneI32x4, vecOp == wasm.OpcodeVecI16x8NarrowI32x4S). + Insert(builder).Return() + state.push(ret) + case wasm.OpcodeVecI16x8ExtendLowI8x16S, wasm.OpcodeVecI16x8ExtendLowI8x16U: + if state.unreachable { + break + } + v1 := state.pop() + ret := builder.AllocateInstruction(). + AsWiden(v1, ssa.VecLaneI8x16, vecOp == wasm.OpcodeVecI16x8ExtendLowI8x16S, true). + Insert(builder).Return() + state.push(ret) + case wasm.OpcodeVecI16x8ExtendHighI8x16S, wasm.OpcodeVecI16x8ExtendHighI8x16U: + if state.unreachable { + break + } + v1 := state.pop() + ret := builder.AllocateInstruction(). + AsWiden(v1, ssa.VecLaneI8x16, vecOp == wasm.OpcodeVecI16x8ExtendHighI8x16S, false). + Insert(builder).Return() + state.push(ret) + case wasm.OpcodeVecI32x4ExtendLowI16x8S, wasm.OpcodeVecI32x4ExtendLowI16x8U: + if state.unreachable { + break + } + v1 := state.pop() + ret := builder.AllocateInstruction(). + AsWiden(v1, ssa.VecLaneI16x8, vecOp == wasm.OpcodeVecI32x4ExtendLowI16x8S, true). + Insert(builder).Return() + state.push(ret) + case wasm.OpcodeVecI32x4ExtendHighI16x8S, wasm.OpcodeVecI32x4ExtendHighI16x8U: + if state.unreachable { + break + } + v1 := state.pop() + ret := builder.AllocateInstruction(). + AsWiden(v1, ssa.VecLaneI16x8, vecOp == wasm.OpcodeVecI32x4ExtendHighI16x8S, false). + Insert(builder).Return() + state.push(ret) + case wasm.OpcodeVecI64x2ExtendLowI32x4S, wasm.OpcodeVecI64x2ExtendLowI32x4U: + if state.unreachable { + break + } + v1 := state.pop() + ret := builder.AllocateInstruction(). + AsWiden(v1, ssa.VecLaneI32x4, vecOp == wasm.OpcodeVecI64x2ExtendLowI32x4S, true). + Insert(builder).Return() + state.push(ret) + case wasm.OpcodeVecI64x2ExtendHighI32x4S, wasm.OpcodeVecI64x2ExtendHighI32x4U: + if state.unreachable { + break + } + v1 := state.pop() + ret := builder.AllocateInstruction(). + AsWiden(v1, ssa.VecLaneI32x4, vecOp == wasm.OpcodeVecI64x2ExtendHighI32x4S, false). + Insert(builder).Return() + state.push(ret) + + case wasm.OpcodeVecF64x2PromoteLowF32x4Zero: + if state.unreachable { + break + } + v1 := state.pop() + ret := builder.AllocateInstruction(). + AsFvpromoteLow(v1, ssa.VecLaneF32x4). + Insert(builder).Return() + state.push(ret) + case wasm.OpcodeVecF32x4DemoteF64x2Zero: + if state.unreachable { + break + } + v1 := state.pop() + ret := builder.AllocateInstruction(). + AsFvdemote(v1, ssa.VecLaneF64x2). + Insert(builder).Return() + state.push(ret) + case wasm.OpcodeVecI8x16Shl, wasm.OpcodeVecI16x8Shl, wasm.OpcodeVecI32x4Shl, wasm.OpcodeVecI64x2Shl: + if state.unreachable { + break + } + var lane ssa.VecLane + switch vecOp { + case wasm.OpcodeVecI8x16Shl: + lane = ssa.VecLaneI8x16 + case wasm.OpcodeVecI16x8Shl: + lane = ssa.VecLaneI16x8 + case wasm.OpcodeVecI32x4Shl: + lane = ssa.VecLaneI32x4 + case wasm.OpcodeVecI64x2Shl: + lane = ssa.VecLaneI64x2 + } + v2 := state.pop() + v1 := state.pop() + ret := builder.AllocateInstruction().AsVIshl(v1, v2, lane).Insert(builder).Return() + state.push(ret) + case wasm.OpcodeVecI8x16ShrS, wasm.OpcodeVecI16x8ShrS, wasm.OpcodeVecI32x4ShrS, wasm.OpcodeVecI64x2ShrS: + if state.unreachable { + break + } + var lane ssa.VecLane + switch vecOp { + case wasm.OpcodeVecI8x16ShrS: + lane = ssa.VecLaneI8x16 + case wasm.OpcodeVecI16x8ShrS: + lane = ssa.VecLaneI16x8 + case wasm.OpcodeVecI32x4ShrS: + lane = ssa.VecLaneI32x4 + case wasm.OpcodeVecI64x2ShrS: + lane = ssa.VecLaneI64x2 + } + v2 := state.pop() + v1 := state.pop() + ret := builder.AllocateInstruction().AsVSshr(v1, v2, lane).Insert(builder).Return() + state.push(ret) + case wasm.OpcodeVecI8x16ShrU, wasm.OpcodeVecI16x8ShrU, wasm.OpcodeVecI32x4ShrU, wasm.OpcodeVecI64x2ShrU: + if state.unreachable { + break + } + var lane ssa.VecLane + switch vecOp { + case wasm.OpcodeVecI8x16ShrU: + lane = ssa.VecLaneI8x16 + case wasm.OpcodeVecI16x8ShrU: + lane = ssa.VecLaneI16x8 + case wasm.OpcodeVecI32x4ShrU: + lane = ssa.VecLaneI32x4 + case wasm.OpcodeVecI64x2ShrU: + lane = ssa.VecLaneI64x2 + } + v2 := state.pop() + v1 := state.pop() + ret := builder.AllocateInstruction().AsVUshr(v1, v2, lane).Insert(builder).Return() + state.push(ret) + case wasm.OpcodeVecI8x16ExtractLaneS, wasm.OpcodeVecI16x8ExtractLaneS: + state.pc++ + if state.unreachable { + break + } + var lane ssa.VecLane + switch vecOp { + case wasm.OpcodeVecI8x16ExtractLaneS: + lane = ssa.VecLaneI8x16 + case wasm.OpcodeVecI16x8ExtractLaneS: + lane = ssa.VecLaneI16x8 + } + v1 := state.pop() + index := c.wasmFunctionBody[state.pc] + ext := builder.AllocateInstruction().AsExtractlane(v1, index, lane, true).Insert(builder).Return() + state.push(ext) + case wasm.OpcodeVecI8x16ExtractLaneU, wasm.OpcodeVecI16x8ExtractLaneU, + wasm.OpcodeVecI32x4ExtractLane, wasm.OpcodeVecI64x2ExtractLane, + wasm.OpcodeVecF32x4ExtractLane, wasm.OpcodeVecF64x2ExtractLane: + state.pc++ // Skip the immediate value. + if state.unreachable { + break + } + var lane ssa.VecLane + switch vecOp { + case wasm.OpcodeVecI8x16ExtractLaneU: + lane = ssa.VecLaneI8x16 + case wasm.OpcodeVecI16x8ExtractLaneU: + lane = ssa.VecLaneI16x8 + case wasm.OpcodeVecI32x4ExtractLane: + lane = ssa.VecLaneI32x4 + case wasm.OpcodeVecI64x2ExtractLane: + lane = ssa.VecLaneI64x2 + case wasm.OpcodeVecF32x4ExtractLane: + lane = ssa.VecLaneF32x4 + case wasm.OpcodeVecF64x2ExtractLane: + lane = ssa.VecLaneF64x2 + } + v1 := state.pop() + index := c.wasmFunctionBody[state.pc] + ext := builder.AllocateInstruction().AsExtractlane(v1, index, lane, false).Insert(builder).Return() + state.push(ext) + case wasm.OpcodeVecI8x16ReplaceLane, wasm.OpcodeVecI16x8ReplaceLane, + wasm.OpcodeVecI32x4ReplaceLane, wasm.OpcodeVecI64x2ReplaceLane, + wasm.OpcodeVecF32x4ReplaceLane, wasm.OpcodeVecF64x2ReplaceLane: + state.pc++ + if state.unreachable { + break + } + var lane ssa.VecLane + switch vecOp { + case wasm.OpcodeVecI8x16ReplaceLane: + lane = ssa.VecLaneI8x16 + case wasm.OpcodeVecI16x8ReplaceLane: + lane = ssa.VecLaneI16x8 + case wasm.OpcodeVecI32x4ReplaceLane: + lane = ssa.VecLaneI32x4 + case wasm.OpcodeVecI64x2ReplaceLane: + lane = ssa.VecLaneI64x2 + case wasm.OpcodeVecF32x4ReplaceLane: + lane = ssa.VecLaneF32x4 + case wasm.OpcodeVecF64x2ReplaceLane: + lane = ssa.VecLaneF64x2 + } + v2 := state.pop() + v1 := state.pop() + index := c.wasmFunctionBody[state.pc] + ret := builder.AllocateInstruction().AsInsertlane(v1, v2, index, lane).Insert(builder).Return() + state.push(ret) + case wasm.OpcodeVecV128i8x16Shuffle: + state.pc++ + laneIndexes := c.wasmFunctionBody[state.pc : state.pc+16] + state.pc += 15 + if state.unreachable { + break + } + v2 := state.pop() + v1 := state.pop() + ret := builder.AllocateInstruction().AsShuffle(v1, v2, laneIndexes).Insert(builder).Return() + state.push(ret) + + case wasm.OpcodeVecI8x16Swizzle: + if state.unreachable { + break + } + v2 := state.pop() + v1 := state.pop() + ret := builder.AllocateInstruction().AsSwizzle(v1, v2, ssa.VecLaneI8x16).Insert(builder).Return() + state.push(ret) + + case wasm.OpcodeVecI8x16Splat, + wasm.OpcodeVecI16x8Splat, + wasm.OpcodeVecI32x4Splat, + wasm.OpcodeVecI64x2Splat, + wasm.OpcodeVecF32x4Splat, + wasm.OpcodeVecF64x2Splat: + if state.unreachable { + break + } + var lane ssa.VecLane + switch vecOp { + case wasm.OpcodeVecI8x16Splat: + lane = ssa.VecLaneI8x16 + case wasm.OpcodeVecI16x8Splat: + lane = ssa.VecLaneI16x8 + case wasm.OpcodeVecI32x4Splat: + lane = ssa.VecLaneI32x4 + case wasm.OpcodeVecI64x2Splat: + lane = ssa.VecLaneI64x2 + case wasm.OpcodeVecF32x4Splat: + lane = ssa.VecLaneF32x4 + case wasm.OpcodeVecF64x2Splat: + lane = ssa.VecLaneF64x2 + } + v1 := state.pop() + ret := builder.AllocateInstruction().AsSplat(v1, lane).Insert(builder).Return() + state.push(ret) + + default: + panic("TODO: unsupported vector instruction: " + wasm.VectorInstructionName(vecOp)) + } + case wasm.OpcodeAtomicPrefix: + state.pc++ + atomicOp := c.wasmFunctionBody[state.pc] + switch atomicOp { + case wasm.OpcodeAtomicMemoryWait32, wasm.OpcodeAtomicMemoryWait64: + _, offset := c.readMemArg() + if state.unreachable { + break + } + + c.storeCallerModuleContext() + + var opSize uint64 + var trampoline wazevoapi.Offset + var sig *ssa.Signature + switch atomicOp { + case wasm.OpcodeAtomicMemoryWait32: + opSize = 4 + trampoline = wazevoapi.ExecutionContextOffsetMemoryWait32TrampolineAddress + sig = &c.memoryWait32Sig + case wasm.OpcodeAtomicMemoryWait64: + opSize = 8 + trampoline = wazevoapi.ExecutionContextOffsetMemoryWait64TrampolineAddress + sig = &c.memoryWait64Sig + } + + timeout := state.pop() + exp := state.pop() + baseAddr := state.pop() + addr := c.atomicMemOpSetup(baseAddr, uint64(offset), opSize) + + memoryWaitPtr := builder.AllocateInstruction(). + AsLoad(c.execCtxPtrValue, + trampoline.U32(), + ssa.TypeI64, + ).Insert(builder).Return() + + args := c.allocateVarLengthValues(3, c.execCtxPtrValue, timeout, exp, addr) + memoryWaitRet := builder.AllocateInstruction(). + AsCallIndirect(memoryWaitPtr, sig, args). + Insert(builder).Return() + state.push(memoryWaitRet) + case wasm.OpcodeAtomicMemoryNotify: + _, offset := c.readMemArg() + if state.unreachable { + break + } + + c.storeCallerModuleContext() + count := state.pop() + baseAddr := state.pop() + addr := c.atomicMemOpSetup(baseAddr, uint64(offset), 4) + + memoryNotifyPtr := builder.AllocateInstruction(). + AsLoad(c.execCtxPtrValue, + wazevoapi.ExecutionContextOffsetMemoryNotifyTrampolineAddress.U32(), + ssa.TypeI64, + ).Insert(builder).Return() + args := c.allocateVarLengthValues(2, c.execCtxPtrValue, count, addr) + memoryNotifyRet := builder.AllocateInstruction(). + AsCallIndirect(memoryNotifyPtr, &c.memoryNotifySig, args). + Insert(builder).Return() + state.push(memoryNotifyRet) + case wasm.OpcodeAtomicI32Load, wasm.OpcodeAtomicI64Load, wasm.OpcodeAtomicI32Load8U, wasm.OpcodeAtomicI32Load16U, wasm.OpcodeAtomicI64Load8U, wasm.OpcodeAtomicI64Load16U, wasm.OpcodeAtomicI64Load32U: + _, offset := c.readMemArg() + if state.unreachable { + break + } + + baseAddr := state.pop() + + var size uint64 + switch atomicOp { + case wasm.OpcodeAtomicI64Load: + size = 8 + case wasm.OpcodeAtomicI32Load, wasm.OpcodeAtomicI64Load32U: + size = 4 + case wasm.OpcodeAtomicI32Load16U, wasm.OpcodeAtomicI64Load16U: + size = 2 + case wasm.OpcodeAtomicI32Load8U, wasm.OpcodeAtomicI64Load8U: + size = 1 + } + + var typ ssa.Type + switch atomicOp { + case wasm.OpcodeAtomicI64Load, wasm.OpcodeAtomicI64Load32U, wasm.OpcodeAtomicI64Load16U, wasm.OpcodeAtomicI64Load8U: + typ = ssa.TypeI64 + case wasm.OpcodeAtomicI32Load, wasm.OpcodeAtomicI32Load16U, wasm.OpcodeAtomicI32Load8U: + typ = ssa.TypeI32 + } + + addr := c.atomicMemOpSetup(baseAddr, uint64(offset), size) + res := builder.AllocateInstruction().AsAtomicLoad(addr, size, typ).Insert(builder).Return() + state.push(res) + case wasm.OpcodeAtomicI32Store, wasm.OpcodeAtomicI64Store, wasm.OpcodeAtomicI32Store8, wasm.OpcodeAtomicI32Store16, wasm.OpcodeAtomicI64Store8, wasm.OpcodeAtomicI64Store16, wasm.OpcodeAtomicI64Store32: + _, offset := c.readMemArg() + if state.unreachable { + break + } + + val := state.pop() + baseAddr := state.pop() + + var size uint64 + switch atomicOp { + case wasm.OpcodeAtomicI64Store: + size = 8 + case wasm.OpcodeAtomicI32Store, wasm.OpcodeAtomicI64Store32: + size = 4 + case wasm.OpcodeAtomicI32Store16, wasm.OpcodeAtomicI64Store16: + size = 2 + case wasm.OpcodeAtomicI32Store8, wasm.OpcodeAtomicI64Store8: + size = 1 + } + + addr := c.atomicMemOpSetup(baseAddr, uint64(offset), size) + builder.AllocateInstruction().AsAtomicStore(addr, val, size).Insert(builder) + case wasm.OpcodeAtomicI32RmwAdd, wasm.OpcodeAtomicI64RmwAdd, wasm.OpcodeAtomicI32Rmw8AddU, wasm.OpcodeAtomicI32Rmw16AddU, wasm.OpcodeAtomicI64Rmw8AddU, wasm.OpcodeAtomicI64Rmw16AddU, wasm.OpcodeAtomicI64Rmw32AddU, + wasm.OpcodeAtomicI32RmwSub, wasm.OpcodeAtomicI64RmwSub, wasm.OpcodeAtomicI32Rmw8SubU, wasm.OpcodeAtomicI32Rmw16SubU, wasm.OpcodeAtomicI64Rmw8SubU, wasm.OpcodeAtomicI64Rmw16SubU, wasm.OpcodeAtomicI64Rmw32SubU, + wasm.OpcodeAtomicI32RmwAnd, wasm.OpcodeAtomicI64RmwAnd, wasm.OpcodeAtomicI32Rmw8AndU, wasm.OpcodeAtomicI32Rmw16AndU, wasm.OpcodeAtomicI64Rmw8AndU, wasm.OpcodeAtomicI64Rmw16AndU, wasm.OpcodeAtomicI64Rmw32AndU, + wasm.OpcodeAtomicI32RmwOr, wasm.OpcodeAtomicI64RmwOr, wasm.OpcodeAtomicI32Rmw8OrU, wasm.OpcodeAtomicI32Rmw16OrU, wasm.OpcodeAtomicI64Rmw8OrU, wasm.OpcodeAtomicI64Rmw16OrU, wasm.OpcodeAtomicI64Rmw32OrU, + wasm.OpcodeAtomicI32RmwXor, wasm.OpcodeAtomicI64RmwXor, wasm.OpcodeAtomicI32Rmw8XorU, wasm.OpcodeAtomicI32Rmw16XorU, wasm.OpcodeAtomicI64Rmw8XorU, wasm.OpcodeAtomicI64Rmw16XorU, wasm.OpcodeAtomicI64Rmw32XorU, + wasm.OpcodeAtomicI32RmwXchg, wasm.OpcodeAtomicI64RmwXchg, wasm.OpcodeAtomicI32Rmw8XchgU, wasm.OpcodeAtomicI32Rmw16XchgU, wasm.OpcodeAtomicI64Rmw8XchgU, wasm.OpcodeAtomicI64Rmw16XchgU, wasm.OpcodeAtomicI64Rmw32XchgU: + _, offset := c.readMemArg() + if state.unreachable { + break + } + + val := state.pop() + baseAddr := state.pop() + + var rmwOp ssa.AtomicRmwOp + var size uint64 + switch atomicOp { + case wasm.OpcodeAtomicI32RmwAdd, wasm.OpcodeAtomicI64RmwAdd, wasm.OpcodeAtomicI32Rmw8AddU, wasm.OpcodeAtomicI32Rmw16AddU, wasm.OpcodeAtomicI64Rmw8AddU, wasm.OpcodeAtomicI64Rmw16AddU, wasm.OpcodeAtomicI64Rmw32AddU: + rmwOp = ssa.AtomicRmwOpAdd + switch atomicOp { + case wasm.OpcodeAtomicI64RmwAdd: + size = 8 + case wasm.OpcodeAtomicI32RmwAdd, wasm.OpcodeAtomicI64Rmw32AddU: + size = 4 + case wasm.OpcodeAtomicI32Rmw16AddU, wasm.OpcodeAtomicI64Rmw16AddU: + size = 2 + case wasm.OpcodeAtomicI32Rmw8AddU, wasm.OpcodeAtomicI64Rmw8AddU: + size = 1 + } + case wasm.OpcodeAtomicI32RmwSub, wasm.OpcodeAtomicI64RmwSub, wasm.OpcodeAtomicI32Rmw8SubU, wasm.OpcodeAtomicI32Rmw16SubU, wasm.OpcodeAtomicI64Rmw8SubU, wasm.OpcodeAtomicI64Rmw16SubU, wasm.OpcodeAtomicI64Rmw32SubU: + rmwOp = ssa.AtomicRmwOpSub + switch atomicOp { + case wasm.OpcodeAtomicI64RmwSub: + size = 8 + case wasm.OpcodeAtomicI32RmwSub, wasm.OpcodeAtomicI64Rmw32SubU: + size = 4 + case wasm.OpcodeAtomicI32Rmw16SubU, wasm.OpcodeAtomicI64Rmw16SubU: + size = 2 + case wasm.OpcodeAtomicI32Rmw8SubU, wasm.OpcodeAtomicI64Rmw8SubU: + size = 1 + } + case wasm.OpcodeAtomicI32RmwAnd, wasm.OpcodeAtomicI64RmwAnd, wasm.OpcodeAtomicI32Rmw8AndU, wasm.OpcodeAtomicI32Rmw16AndU, wasm.OpcodeAtomicI64Rmw8AndU, wasm.OpcodeAtomicI64Rmw16AndU, wasm.OpcodeAtomicI64Rmw32AndU: + rmwOp = ssa.AtomicRmwOpAnd + switch atomicOp { + case wasm.OpcodeAtomicI64RmwAnd: + size = 8 + case wasm.OpcodeAtomicI32RmwAnd, wasm.OpcodeAtomicI64Rmw32AndU: + size = 4 + case wasm.OpcodeAtomicI32Rmw16AndU, wasm.OpcodeAtomicI64Rmw16AndU: + size = 2 + case wasm.OpcodeAtomicI32Rmw8AndU, wasm.OpcodeAtomicI64Rmw8AndU: + size = 1 + } + case wasm.OpcodeAtomicI32RmwOr, wasm.OpcodeAtomicI64RmwOr, wasm.OpcodeAtomicI32Rmw8OrU, wasm.OpcodeAtomicI32Rmw16OrU, wasm.OpcodeAtomicI64Rmw8OrU, wasm.OpcodeAtomicI64Rmw16OrU, wasm.OpcodeAtomicI64Rmw32OrU: + rmwOp = ssa.AtomicRmwOpOr + switch atomicOp { + case wasm.OpcodeAtomicI64RmwOr: + size = 8 + case wasm.OpcodeAtomicI32RmwOr, wasm.OpcodeAtomicI64Rmw32OrU: + size = 4 + case wasm.OpcodeAtomicI32Rmw16OrU, wasm.OpcodeAtomicI64Rmw16OrU: + size = 2 + case wasm.OpcodeAtomicI32Rmw8OrU, wasm.OpcodeAtomicI64Rmw8OrU: + size = 1 + } + case wasm.OpcodeAtomicI32RmwXor, wasm.OpcodeAtomicI64RmwXor, wasm.OpcodeAtomicI32Rmw8XorU, wasm.OpcodeAtomicI32Rmw16XorU, wasm.OpcodeAtomicI64Rmw8XorU, wasm.OpcodeAtomicI64Rmw16XorU, wasm.OpcodeAtomicI64Rmw32XorU: + rmwOp = ssa.AtomicRmwOpXor + switch atomicOp { + case wasm.OpcodeAtomicI64RmwXor: + size = 8 + case wasm.OpcodeAtomicI32RmwXor, wasm.OpcodeAtomicI64Rmw32XorU: + size = 4 + case wasm.OpcodeAtomicI32Rmw16XorU, wasm.OpcodeAtomicI64Rmw16XorU: + size = 2 + case wasm.OpcodeAtomicI32Rmw8XorU, wasm.OpcodeAtomicI64Rmw8XorU: + size = 1 + } + case wasm.OpcodeAtomicI32RmwXchg, wasm.OpcodeAtomicI64RmwXchg, wasm.OpcodeAtomicI32Rmw8XchgU, wasm.OpcodeAtomicI32Rmw16XchgU, wasm.OpcodeAtomicI64Rmw8XchgU, wasm.OpcodeAtomicI64Rmw16XchgU, wasm.OpcodeAtomicI64Rmw32XchgU: + rmwOp = ssa.AtomicRmwOpXchg + switch atomicOp { + case wasm.OpcodeAtomicI64RmwXchg: + size = 8 + case wasm.OpcodeAtomicI32RmwXchg, wasm.OpcodeAtomicI64Rmw32XchgU: + size = 4 + case wasm.OpcodeAtomicI32Rmw16XchgU, wasm.OpcodeAtomicI64Rmw16XchgU: + size = 2 + case wasm.OpcodeAtomicI32Rmw8XchgU, wasm.OpcodeAtomicI64Rmw8XchgU: + size = 1 + } + } + + addr := c.atomicMemOpSetup(baseAddr, uint64(offset), size) + res := builder.AllocateInstruction().AsAtomicRmw(rmwOp, addr, val, size).Insert(builder).Return() + state.push(res) + case wasm.OpcodeAtomicI32RmwCmpxchg, wasm.OpcodeAtomicI64RmwCmpxchg, wasm.OpcodeAtomicI32Rmw8CmpxchgU, wasm.OpcodeAtomicI32Rmw16CmpxchgU, wasm.OpcodeAtomicI64Rmw8CmpxchgU, wasm.OpcodeAtomicI64Rmw16CmpxchgU, wasm.OpcodeAtomicI64Rmw32CmpxchgU: + _, offset := c.readMemArg() + if state.unreachable { + break + } + + repl := state.pop() + exp := state.pop() + baseAddr := state.pop() + + var size uint64 + switch atomicOp { + case wasm.OpcodeAtomicI64RmwCmpxchg: + size = 8 + case wasm.OpcodeAtomicI32RmwCmpxchg, wasm.OpcodeAtomicI64Rmw32CmpxchgU: + size = 4 + case wasm.OpcodeAtomicI32Rmw16CmpxchgU, wasm.OpcodeAtomicI64Rmw16CmpxchgU: + size = 2 + case wasm.OpcodeAtomicI32Rmw8CmpxchgU, wasm.OpcodeAtomicI64Rmw8CmpxchgU: + size = 1 + } + addr := c.atomicMemOpSetup(baseAddr, uint64(offset), size) + res := builder.AllocateInstruction().AsAtomicCas(addr, exp, repl, size).Insert(builder).Return() + state.push(res) + case wasm.OpcodeAtomicFence: + order := c.readByte() + if state.unreachable { + break + } + if c.needMemory { + builder.AllocateInstruction().AsFence(order).Insert(builder) + } + default: + panic("TODO: unsupported atomic instruction: " + wasm.AtomicInstructionName(atomicOp)) + } + case wasm.OpcodeRefFunc: + funcIndex := c.readI32u() + if state.unreachable { + break + } + + c.storeCallerModuleContext() + + funcIndexVal := builder.AllocateInstruction().AsIconst32(funcIndex).Insert(builder).Return() + + refFuncPtr := builder.AllocateInstruction(). + AsLoad(c.execCtxPtrValue, + wazevoapi.ExecutionContextOffsetRefFuncTrampolineAddress.U32(), + ssa.TypeI64, + ).Insert(builder).Return() + + args := c.allocateVarLengthValues(2, c.execCtxPtrValue, funcIndexVal) + refFuncRet := builder. + AllocateInstruction(). + AsCallIndirect(refFuncPtr, &c.refFuncSig, args). + Insert(builder).Return() + state.push(refFuncRet) + + case wasm.OpcodeRefNull: + c.loweringState.pc++ // skips the reference type as we treat both of them as i64(0). + if state.unreachable { + break + } + ret := builder.AllocateInstruction().AsIconst64(0).Insert(builder).Return() + state.push(ret) + case wasm.OpcodeRefIsNull: + if state.unreachable { + break + } + r := state.pop() + zero := builder.AllocateInstruction().AsIconst64(0).Insert(builder) + icmp := builder.AllocateInstruction(). + AsIcmp(r, zero.Return(), ssa.IntegerCmpCondEqual). + Insert(builder). + Return() + state.push(icmp) + case wasm.OpcodeTableSet: + tableIndex := c.readI32u() + if state.unreachable { + break + } + r := state.pop() + targetOffsetInTable := state.pop() + + elementAddr := c.lowerAccessTableWithBoundsCheck(tableIndex, targetOffsetInTable) + builder.AllocateInstruction().AsStore(ssa.OpcodeStore, r, elementAddr, 0).Insert(builder) + + case wasm.OpcodeTableGet: + tableIndex := c.readI32u() + if state.unreachable { + break + } + targetOffsetInTable := state.pop() + elementAddr := c.lowerAccessTableWithBoundsCheck(tableIndex, targetOffsetInTable) + loaded := builder.AllocateInstruction().AsLoad(elementAddr, 0, ssa.TypeI64).Insert(builder).Return() + state.push(loaded) + default: + panic("TODO: unsupported in wazevo yet: " + wasm.InstructionName(op)) + } + + if wazevoapi.FrontEndLoggingEnabled { + fmt.Println("--------- Translated " + wasm.InstructionName(op) + " --------") + fmt.Println("state: " + c.loweringState.String()) + fmt.Println(c.formatBuilder()) + fmt.Println("--------------------------") + } + c.loweringState.pc++ +} + +func (c *Compiler) lowerExtMul(v1, v2 ssa.Value, from, to ssa.VecLane, signed, low bool) ssa.Value { + // TODO: The sequence `Widen; Widen; VIMul` can be substituted for a single instruction on some ISAs. + builder := c.ssaBuilder + + v1lo := builder.AllocateInstruction().AsWiden(v1, from, signed, low).Insert(builder).Return() + v2lo := builder.AllocateInstruction().AsWiden(v2, from, signed, low).Insert(builder).Return() + + return builder.AllocateInstruction().AsVImul(v1lo, v2lo, to).Insert(builder).Return() +} + +const ( + tableInstanceBaseAddressOffset = 0 + tableInstanceLenOffset = tableInstanceBaseAddressOffset + 8 +) + +func (c *Compiler) lowerAccessTableWithBoundsCheck(tableIndex uint32, elementOffsetInTable ssa.Value) (elementAddress ssa.Value) { + builder := c.ssaBuilder + + // Load the table. + loadTableInstancePtr := builder.AllocateInstruction() + loadTableInstancePtr.AsLoad(c.moduleCtxPtrValue, c.offset.TableOffset(int(tableIndex)).U32(), ssa.TypeI64) + builder.InsertInstruction(loadTableInstancePtr) + tableInstancePtr := loadTableInstancePtr.Return() + + // Load the table's length. + loadTableLen := builder.AllocateInstruction() + loadTableLen.AsLoad(tableInstancePtr, tableInstanceLenOffset, ssa.TypeI32) + builder.InsertInstruction(loadTableLen) + tableLen := loadTableLen.Return() + + // Compare the length and the target, and trap if out of bounds. + checkOOB := builder.AllocateInstruction() + checkOOB.AsIcmp(elementOffsetInTable, tableLen, ssa.IntegerCmpCondUnsignedGreaterThanOrEqual) + builder.InsertInstruction(checkOOB) + exitIfOOB := builder.AllocateInstruction() + exitIfOOB.AsExitIfTrueWithCode(c.execCtxPtrValue, checkOOB.Return(), wazevoapi.ExitCodeTableOutOfBounds) + builder.InsertInstruction(exitIfOOB) + + // Get the base address of wasm.TableInstance.References. + loadTableBaseAddress := builder.AllocateInstruction() + loadTableBaseAddress.AsLoad(tableInstancePtr, tableInstanceBaseAddressOffset, ssa.TypeI64) + builder.InsertInstruction(loadTableBaseAddress) + tableBase := loadTableBaseAddress.Return() + + // Calculate the address of the target function. First we need to multiply targetOffsetInTable by 8 (pointer size). + multiplyBy8 := builder.AllocateInstruction() + three := builder.AllocateInstruction() + three.AsIconst64(3) + builder.InsertInstruction(three) + multiplyBy8.AsIshl(elementOffsetInTable, three.Return()) + builder.InsertInstruction(multiplyBy8) + targetOffsetInTableMultipliedBy8 := multiplyBy8.Return() + + // Then add the multiplied value to the base which results in the address of the target function (*wazevo.functionInstance) + calcElementAddressInTable := builder.AllocateInstruction() + calcElementAddressInTable.AsIadd(tableBase, targetOffsetInTableMultipliedBy8) + builder.InsertInstruction(calcElementAddressInTable) + return calcElementAddressInTable.Return() +} + +func (c *Compiler) lowerCallIndirect(typeIndex, tableIndex uint32) { + builder := c.ssaBuilder + state := c.state() + + elementOffsetInTable := state.pop() + functionInstancePtrAddress := c.lowerAccessTableWithBoundsCheck(tableIndex, elementOffsetInTable) + loadFunctionInstancePtr := builder.AllocateInstruction() + loadFunctionInstancePtr.AsLoad(functionInstancePtrAddress, 0, ssa.TypeI64) + builder.InsertInstruction(loadFunctionInstancePtr) + functionInstancePtr := loadFunctionInstancePtr.Return() + + // Check if it is not the null pointer. + zero := builder.AllocateInstruction() + zero.AsIconst64(0) + builder.InsertInstruction(zero) + checkNull := builder.AllocateInstruction() + checkNull.AsIcmp(functionInstancePtr, zero.Return(), ssa.IntegerCmpCondEqual) + builder.InsertInstruction(checkNull) + exitIfNull := builder.AllocateInstruction() + exitIfNull.AsExitIfTrueWithCode(c.execCtxPtrValue, checkNull.Return(), wazevoapi.ExitCodeIndirectCallNullPointer) + builder.InsertInstruction(exitIfNull) + + // We need to do the type check. First, load the target function instance's typeID. + loadTypeID := builder.AllocateInstruction() + loadTypeID.AsLoad(functionInstancePtr, wazevoapi.FunctionInstanceTypeIDOffset, ssa.TypeI32) + builder.InsertInstruction(loadTypeID) + actualTypeID := loadTypeID.Return() + + // Next, we load the expected TypeID: + loadTypeIDsBegin := builder.AllocateInstruction() + loadTypeIDsBegin.AsLoad(c.moduleCtxPtrValue, c.offset.TypeIDs1stElement.U32(), ssa.TypeI64) + builder.InsertInstruction(loadTypeIDsBegin) + typeIDsBegin := loadTypeIDsBegin.Return() + + loadExpectedTypeID := builder.AllocateInstruction() + loadExpectedTypeID.AsLoad(typeIDsBegin, uint32(typeIndex)*4 /* size of wasm.FunctionTypeID */, ssa.TypeI32) + builder.InsertInstruction(loadExpectedTypeID) + expectedTypeID := loadExpectedTypeID.Return() + + // Check if the type ID matches. + checkTypeID := builder.AllocateInstruction() + checkTypeID.AsIcmp(actualTypeID, expectedTypeID, ssa.IntegerCmpCondNotEqual) + builder.InsertInstruction(checkTypeID) + exitIfNotMatch := builder.AllocateInstruction() + exitIfNotMatch.AsExitIfTrueWithCode(c.execCtxPtrValue, checkTypeID.Return(), wazevoapi.ExitCodeIndirectCallTypeMismatch) + builder.InsertInstruction(exitIfNotMatch) + + // Now ready to call the function. Load the executable and moduleContextOpaquePtr from the function instance. + loadExecutablePtr := builder.AllocateInstruction() + loadExecutablePtr.AsLoad(functionInstancePtr, wazevoapi.FunctionInstanceExecutableOffset, ssa.TypeI64) + builder.InsertInstruction(loadExecutablePtr) + executablePtr := loadExecutablePtr.Return() + loadModuleContextOpaquePtr := builder.AllocateInstruction() + loadModuleContextOpaquePtr.AsLoad(functionInstancePtr, wazevoapi.FunctionInstanceModuleContextOpaquePtrOffset, ssa.TypeI64) + builder.InsertInstruction(loadModuleContextOpaquePtr) + moduleContextOpaquePtr := loadModuleContextOpaquePtr.Return() + + typ := &c.m.TypeSection[typeIndex] + tail := len(state.values) - len(typ.Params) + vs := state.values[tail:] + state.values = state.values[:tail] + args := c.allocateVarLengthValues(2+len(vs), c.execCtxPtrValue, moduleContextOpaquePtr) + args = args.Append(builder.VarLengthPool(), vs...) + + // Before transfer the control to the callee, we have to store the current module's moduleContextPtr + // into execContext.callerModuleContextPtr in case when the callee is a Go function. + c.storeCallerModuleContext() + + call := builder.AllocateInstruction() + call.AsCallIndirect(executablePtr, c.signatures[typ], args) + builder.InsertInstruction(call) + + first, rest := call.Returns() + if first.Valid() { + state.push(first) + } + for _, v := range rest { + state.push(v) + } + + c.reloadAfterCall() +} + +// memOpSetup inserts the bounds check and calculates the address of the memory operation (loads/stores). +func (c *Compiler) memOpSetup(baseAddr ssa.Value, constOffset, operationSizeInBytes uint64) (address ssa.Value) { + address = ssa.ValueInvalid + builder := c.ssaBuilder + + baseAddrID := baseAddr.ID() + ceil := constOffset + operationSizeInBytes + if known := c.getKnownSafeBound(baseAddrID); known.valid() { + // We reuse the calculated absolute address even if the bound is not known to be safe. + address = known.absoluteAddr + if ceil <= known.bound { + if !address.Valid() { + // This means that, the bound is known to be safe, but the memory base might have changed. + // So, we re-calculate the address. + memBase := c.getMemoryBaseValue(false) + extBaseAddr := builder.AllocateInstruction(). + AsUExtend(baseAddr, 32, 64). + Insert(builder). + Return() + address = builder.AllocateInstruction(). + AsIadd(memBase, extBaseAddr).Insert(builder).Return() + known.absoluteAddr = address // Update the absolute address for the subsequent memory access. + } + return + } + } + + ceilConst := builder.AllocateInstruction() + ceilConst.AsIconst64(ceil) + builder.InsertInstruction(ceilConst) + + // We calculate the offset in 64-bit space. + extBaseAddr := builder.AllocateInstruction(). + AsUExtend(baseAddr, 32, 64). + Insert(builder). + Return() + + // Note: memLen is already zero extended to 64-bit space at the load time. + memLen := c.getMemoryLenValue(false) + + // baseAddrPlusCeil = baseAddr + ceil + baseAddrPlusCeil := builder.AllocateInstruction() + baseAddrPlusCeil.AsIadd(extBaseAddr, ceilConst.Return()) + builder.InsertInstruction(baseAddrPlusCeil) + + // Check for out of bounds memory access: `memLen >= baseAddrPlusCeil`. + cmp := builder.AllocateInstruction() + cmp.AsIcmp(memLen, baseAddrPlusCeil.Return(), ssa.IntegerCmpCondUnsignedLessThan) + builder.InsertInstruction(cmp) + exitIfNZ := builder.AllocateInstruction() + exitIfNZ.AsExitIfTrueWithCode(c.execCtxPtrValue, cmp.Return(), wazevoapi.ExitCodeMemoryOutOfBounds) + builder.InsertInstruction(exitIfNZ) + + // Load the value from memBase + extBaseAddr. + if address == ssa.ValueInvalid { // Reuse the value if the memBase is already calculated at this point. + memBase := c.getMemoryBaseValue(false) + address = builder.AllocateInstruction(). + AsIadd(memBase, extBaseAddr).Insert(builder).Return() + } + + // Record the bound ceil for this baseAddr is known to be safe for the subsequent memory access in the same block. + c.recordKnownSafeBound(baseAddrID, ceil, address) + return +} + +// atomicMemOpSetup inserts the bounds check and calculates the address of the memory operation (loads/stores), including +// the constant offset and performs an alignment check on the final address. +func (c *Compiler) atomicMemOpSetup(baseAddr ssa.Value, constOffset, operationSizeInBytes uint64) (address ssa.Value) { + builder := c.ssaBuilder + + addrWithoutOffset := c.memOpSetup(baseAddr, constOffset, operationSizeInBytes) + var addr ssa.Value + if constOffset == 0 { + addr = addrWithoutOffset + } else { + offset := builder.AllocateInstruction().AsIconst64(constOffset).Insert(builder).Return() + addr = builder.AllocateInstruction().AsIadd(addrWithoutOffset, offset).Insert(builder).Return() + } + + c.memAlignmentCheck(addr, operationSizeInBytes) + + return addr +} + +func (c *Compiler) memAlignmentCheck(addr ssa.Value, operationSizeInBytes uint64) { + if operationSizeInBytes == 1 { + return // No alignment restrictions when accessing a byte + } + var checkBits uint64 + switch operationSizeInBytes { + case 2: + checkBits = 0b1 + case 4: + checkBits = 0b11 + case 8: + checkBits = 0b111 + } + + builder := c.ssaBuilder + + mask := builder.AllocateInstruction().AsIconst64(checkBits).Insert(builder).Return() + masked := builder.AllocateInstruction().AsBand(addr, mask).Insert(builder).Return() + zero := builder.AllocateInstruction().AsIconst64(0).Insert(builder).Return() + cmp := builder.AllocateInstruction().AsIcmp(masked, zero, ssa.IntegerCmpCondNotEqual).Insert(builder).Return() + builder.AllocateInstruction().AsExitIfTrueWithCode(c.execCtxPtrValue, cmp, wazevoapi.ExitCodeUnalignedAtomic).Insert(builder) +} + +func (c *Compiler) callMemmove(dst, src, size ssa.Value) { + args := c.allocateVarLengthValues(3, dst, src, size) + if size.Type() != ssa.TypeI64 { + panic("TODO: memmove size must be i64") + } + + builder := c.ssaBuilder + memmovePtr := builder.AllocateInstruction(). + AsLoad(c.execCtxPtrValue, + wazevoapi.ExecutionContextOffsetMemmoveAddress.U32(), + ssa.TypeI64, + ).Insert(builder).Return() + builder.AllocateInstruction().AsCallGoRuntimeMemmove(memmovePtr, &c.memmoveSig, args).Insert(builder) +} + +func (c *Compiler) reloadAfterCall() { + // Note that when these are not used in the following instructions, they will be optimized out. + // So in any ways, we define them! + + // After calling any function, memory buffer might have changed. So we need to re-define the variable. + // However, if the memory is shared, we don't need to reload the memory base and length as the base will never change. + if c.needMemory && !c.memoryShared { + c.reloadMemoryBaseLen() + } + + // Also, any mutable Global can change. + for _, index := range c.mutableGlobalVariablesIndexes { + _ = c.getWasmGlobalValue(index, true) + } +} + +func (c *Compiler) reloadMemoryBaseLen() { + _ = c.getMemoryBaseValue(true) + _ = c.getMemoryLenValue(true) + + // This function being called means that the memory base might have changed. + // Therefore, we need to clear the absolute addresses recorded in the known safe bounds + // because we cache the absolute address of the memory access per each base offset. + c.resetAbsoluteAddressInSafeBounds() +} + +func (c *Compiler) setWasmGlobalValue(index wasm.Index, v ssa.Value) { + variable := c.globalVariables[index] + opaqueOffset := c.offset.GlobalInstanceOffset(index) + + builder := c.ssaBuilder + if index < c.m.ImportGlobalCount { + loadGlobalInstPtr := builder.AllocateInstruction() + loadGlobalInstPtr.AsLoad(c.moduleCtxPtrValue, uint32(opaqueOffset), ssa.TypeI64) + builder.InsertInstruction(loadGlobalInstPtr) + + store := builder.AllocateInstruction() + store.AsStore(ssa.OpcodeStore, v, loadGlobalInstPtr.Return(), uint32(0)) + builder.InsertInstruction(store) + + } else { + store := builder.AllocateInstruction() + store.AsStore(ssa.OpcodeStore, v, c.moduleCtxPtrValue, uint32(opaqueOffset)) + builder.InsertInstruction(store) + } + + // The value has changed to `v`, so we record it. + builder.DefineVariableInCurrentBB(variable, v) +} + +func (c *Compiler) getWasmGlobalValue(index wasm.Index, forceLoad bool) ssa.Value { + variable := c.globalVariables[index] + typ := c.globalVariablesTypes[index] + opaqueOffset := c.offset.GlobalInstanceOffset(index) + + builder := c.ssaBuilder + if !forceLoad { + if v := builder.FindValueInLinearPath(variable); v.Valid() { + return v + } + } + + var load *ssa.Instruction + if index < c.m.ImportGlobalCount { + loadGlobalInstPtr := builder.AllocateInstruction() + loadGlobalInstPtr.AsLoad(c.moduleCtxPtrValue, uint32(opaqueOffset), ssa.TypeI64) + builder.InsertInstruction(loadGlobalInstPtr) + load = builder.AllocateInstruction(). + AsLoad(loadGlobalInstPtr.Return(), uint32(0), typ) + } else { + load = builder.AllocateInstruction(). + AsLoad(c.moduleCtxPtrValue, uint32(opaqueOffset), typ) + } + + v := load.Insert(builder).Return() + builder.DefineVariableInCurrentBB(variable, v) + return v +} + +const ( + memoryInstanceBufOffset = 0 + memoryInstanceBufSizeOffset = memoryInstanceBufOffset + 8 +) + +func (c *Compiler) getMemoryBaseValue(forceReload bool) ssa.Value { + builder := c.ssaBuilder + variable := c.memoryBaseVariable + if !forceReload { + if v := builder.FindValueInLinearPath(variable); v.Valid() { + return v + } + } + + var ret ssa.Value + if c.offset.LocalMemoryBegin < 0 { + loadMemInstPtr := builder.AllocateInstruction() + loadMemInstPtr.AsLoad(c.moduleCtxPtrValue, c.offset.ImportedMemoryBegin.U32(), ssa.TypeI64) + builder.InsertInstruction(loadMemInstPtr) + memInstPtr := loadMemInstPtr.Return() + + loadBufPtr := builder.AllocateInstruction() + loadBufPtr.AsLoad(memInstPtr, memoryInstanceBufOffset, ssa.TypeI64) + builder.InsertInstruction(loadBufPtr) + ret = loadBufPtr.Return() + } else { + load := builder.AllocateInstruction() + load.AsLoad(c.moduleCtxPtrValue, c.offset.LocalMemoryBase().U32(), ssa.TypeI64) + builder.InsertInstruction(load) + ret = load.Return() + } + + builder.DefineVariableInCurrentBB(variable, ret) + return ret +} + +func (c *Compiler) getMemoryLenValue(forceReload bool) ssa.Value { + variable := c.memoryLenVariable + builder := c.ssaBuilder + if !forceReload && !c.memoryShared { + if v := builder.FindValueInLinearPath(variable); v.Valid() { + return v + } + } + + var ret ssa.Value + if c.offset.LocalMemoryBegin < 0 { + loadMemInstPtr := builder.AllocateInstruction() + loadMemInstPtr.AsLoad(c.moduleCtxPtrValue, c.offset.ImportedMemoryBegin.U32(), ssa.TypeI64) + builder.InsertInstruction(loadMemInstPtr) + memInstPtr := loadMemInstPtr.Return() + + loadBufSizePtr := builder.AllocateInstruction() + if c.memoryShared { + sizeOffset := builder.AllocateInstruction().AsIconst64(memoryInstanceBufSizeOffset).Insert(builder).Return() + addr := builder.AllocateInstruction().AsIadd(memInstPtr, sizeOffset).Insert(builder).Return() + loadBufSizePtr.AsAtomicLoad(addr, 8, ssa.TypeI64) + } else { + loadBufSizePtr.AsLoad(memInstPtr, memoryInstanceBufSizeOffset, ssa.TypeI64) + } + builder.InsertInstruction(loadBufSizePtr) + + ret = loadBufSizePtr.Return() + } else { + load := builder.AllocateInstruction() + if c.memoryShared { + lenOffset := builder.AllocateInstruction().AsIconst64(c.offset.LocalMemoryLen().U64()).Insert(builder).Return() + addr := builder.AllocateInstruction().AsIadd(c.moduleCtxPtrValue, lenOffset).Insert(builder).Return() + load.AsAtomicLoad(addr, 8, ssa.TypeI64) + } else { + load.AsExtLoad(ssa.OpcodeUload32, c.moduleCtxPtrValue, c.offset.LocalMemoryLen().U32(), true) + } + builder.InsertInstruction(load) + ret = load.Return() + } + + builder.DefineVariableInCurrentBB(variable, ret) + return ret +} + +func (c *Compiler) insertIcmp(cond ssa.IntegerCmpCond) { + state, builder := c.state(), c.ssaBuilder + y, x := state.pop(), state.pop() + cmp := builder.AllocateInstruction() + cmp.AsIcmp(x, y, cond) + builder.InsertInstruction(cmp) + value := cmp.Return() + state.push(value) +} + +func (c *Compiler) insertFcmp(cond ssa.FloatCmpCond) { + state, builder := c.state(), c.ssaBuilder + y, x := state.pop(), state.pop() + cmp := builder.AllocateInstruction() + cmp.AsFcmp(x, y, cond) + builder.InsertInstruction(cmp) + value := cmp.Return() + state.push(value) +} + +// storeCallerModuleContext stores the current module's moduleContextPtr into execContext.callerModuleContextPtr. +func (c *Compiler) storeCallerModuleContext() { + builder := c.ssaBuilder + execCtx := c.execCtxPtrValue + store := builder.AllocateInstruction() + store.AsStore(ssa.OpcodeStore, + c.moduleCtxPtrValue, execCtx, wazevoapi.ExecutionContextOffsetCallerModuleContextPtr.U32()) + builder.InsertInstruction(store) +} + +func (c *Compiler) readByte() byte { + v := c.wasmFunctionBody[c.loweringState.pc+1] + c.loweringState.pc++ + return v +} + +func (c *Compiler) readI32u() uint32 { + v, n, err := leb128.LoadUint32(c.wasmFunctionBody[c.loweringState.pc+1:]) + if err != nil { + panic(err) // shouldn't be reached since compilation comes after validation. + } + c.loweringState.pc += int(n) + return v +} + +func (c *Compiler) readI32s() int32 { + v, n, err := leb128.LoadInt32(c.wasmFunctionBody[c.loweringState.pc+1:]) + if err != nil { + panic(err) // shouldn't be reached since compilation comes after validation. + } + c.loweringState.pc += int(n) + return v +} + +func (c *Compiler) readI64s() int64 { + v, n, err := leb128.LoadInt64(c.wasmFunctionBody[c.loweringState.pc+1:]) + if err != nil { + panic(err) // shouldn't be reached since compilation comes after validation. + } + c.loweringState.pc += int(n) + return v +} + +func (c *Compiler) readF32() float32 { + v := math.Float32frombits(binary.LittleEndian.Uint32(c.wasmFunctionBody[c.loweringState.pc+1:])) + c.loweringState.pc += 4 + return v +} + +func (c *Compiler) readF64() float64 { + v := math.Float64frombits(binary.LittleEndian.Uint64(c.wasmFunctionBody[c.loweringState.pc+1:])) + c.loweringState.pc += 8 + return v +} + +// readBlockType reads the block type from the current position of the bytecode reader. +func (c *Compiler) readBlockType() *wasm.FunctionType { + state := c.state() + + c.br.Reset(c.wasmFunctionBody[state.pc+1:]) + bt, num, err := wasm.DecodeBlockType(c.m.TypeSection, c.br, api.CoreFeaturesV2) + if err != nil { + panic(err) // shouldn't be reached since compilation comes after validation. + } + state.pc += int(num) + + return bt +} + +func (c *Compiler) readMemArg() (align, offset uint32) { + state := c.state() + + align, num, err := leb128.LoadUint32(c.wasmFunctionBody[state.pc+1:]) + if err != nil { + panic(fmt.Errorf("read memory align: %v", err)) + } + + state.pc += int(num) + offset, num, err = leb128.LoadUint32(c.wasmFunctionBody[state.pc+1:]) + if err != nil { + panic(fmt.Errorf("read memory offset: %v", err)) + } + + state.pc += int(num) + return align, offset +} + +// insertJumpToBlock inserts a jump instruction to the given block in the current block. +func (c *Compiler) insertJumpToBlock(args ssa.Values, targetBlk ssa.BasicBlock) { + if targetBlk.ReturnBlock() { + if c.needListener { + c.callListenerAfter() + } + } + + builder := c.ssaBuilder + jmp := builder.AllocateInstruction() + jmp.AsJump(args, targetBlk) + builder.InsertInstruction(jmp) +} + +func (c *Compiler) insertIntegerExtend(signed bool, from, to byte) { + state := c.state() + builder := c.ssaBuilder + v := state.pop() + extend := builder.AllocateInstruction() + if signed { + extend.AsSExtend(v, from, to) + } else { + extend.AsUExtend(v, from, to) + } + builder.InsertInstruction(extend) + value := extend.Return() + state.push(value) +} + +func (c *Compiler) switchTo(originalStackLen int, targetBlk ssa.BasicBlock) { + if targetBlk.Preds() == 0 { + c.loweringState.unreachable = true + } + + // Now we should adjust the stack and start translating the continuation block. + c.loweringState.values = c.loweringState.values[:originalStackLen] + + c.ssaBuilder.SetCurrentBlock(targetBlk) + + // At this point, blocks params consist only of the Wasm-level parameters, + // (since it's added only when we are trying to resolve variable *inside* this block). + for i := 0; i < targetBlk.Params(); i++ { + value := targetBlk.Param(i) + c.loweringState.push(value) + } +} + +// results returns the number of results of the current function. +func (c *Compiler) results() int { + return len(c.wasmFunctionTyp.Results) +} + +func (c *Compiler) lowerBrTable(labels []uint32, index ssa.Value) { + state := c.state() + builder := c.ssaBuilder + + f := state.ctrlPeekAt(int(labels[0])) + var numArgs int + if f.isLoop() { + numArgs = len(f.blockType.Params) + } else { + numArgs = len(f.blockType.Results) + } + + targets := make([]ssa.BasicBlock, len(labels)) + + // We need trampoline blocks since depending on the target block structure, we might end up inserting moves before jumps, + // which cannot be done with br_table. Instead, we can do such per-block moves in the trampoline blocks. + // At the linking phase (very end of the backend), we can remove the unnecessary jumps, and therefore no runtime overhead. + currentBlk := builder.CurrentBlock() + for i, l := range labels { + // Args are always on the top of the stack. Note that we should not share the args slice + // among the jump instructions since the args are modified during passes (e.g. redundant phi elimination). + args := c.nPeekDup(numArgs) + targetBlk, _ := state.brTargetArgNumFor(l) + trampoline := builder.AllocateBasicBlock() + builder.SetCurrentBlock(trampoline) + c.insertJumpToBlock(args, targetBlk) + targets[i] = trampoline + } + builder.SetCurrentBlock(currentBlk) + + // If the target block has no arguments, we can just jump to the target block. + brTable := builder.AllocateInstruction() + brTable.AsBrTable(index, targets) + builder.InsertInstruction(brTable) + + for _, trampoline := range targets { + builder.Seal(trampoline) + } +} + +func (l *loweringState) brTargetArgNumFor(labelIndex uint32) (targetBlk ssa.BasicBlock, argNum int) { + targetFrame := l.ctrlPeekAt(int(labelIndex)) + if targetFrame.isLoop() { + targetBlk, argNum = targetFrame.blk, len(targetFrame.blockType.Params) + } else { + targetBlk, argNum = targetFrame.followingBlock, len(targetFrame.blockType.Results) + } + return +} + +func (c *Compiler) callListenerBefore() { + c.storeCallerModuleContext() + + builder := c.ssaBuilder + beforeListeners1stElement := builder.AllocateInstruction(). + AsLoad(c.moduleCtxPtrValue, + c.offset.BeforeListenerTrampolines1stElement.U32(), + ssa.TypeI64, + ).Insert(builder).Return() + + beforeListenerPtr := builder.AllocateInstruction(). + AsLoad(beforeListeners1stElement, uint32(c.wasmFunctionTypeIndex)*8 /* 8 bytes per index */, ssa.TypeI64).Insert(builder).Return() + + entry := builder.EntryBlock() + ps := entry.Params() + + args := c.allocateVarLengthValues(ps, c.execCtxPtrValue, + builder.AllocateInstruction().AsIconst32(c.wasmLocalFunctionIndex).Insert(builder).Return()) + for i := 2; i < ps; i++ { + args = args.Append(builder.VarLengthPool(), entry.Param(i)) + } + + beforeSig := c.listenerSignatures[c.wasmFunctionTyp][0] + builder.AllocateInstruction(). + AsCallIndirect(beforeListenerPtr, beforeSig, args). + Insert(builder) +} + +func (c *Compiler) callListenerAfter() { + c.storeCallerModuleContext() + + builder := c.ssaBuilder + afterListeners1stElement := builder.AllocateInstruction(). + AsLoad(c.moduleCtxPtrValue, + c.offset.AfterListenerTrampolines1stElement.U32(), + ssa.TypeI64, + ).Insert(builder).Return() + + afterListenerPtr := builder.AllocateInstruction(). + AsLoad(afterListeners1stElement, + uint32(c.wasmFunctionTypeIndex)*8 /* 8 bytes per index */, ssa.TypeI64). + Insert(builder). + Return() + + afterSig := c.listenerSignatures[c.wasmFunctionTyp][1] + args := c.allocateVarLengthValues( + c.results()+2, + c.execCtxPtrValue, + builder.AllocateInstruction().AsIconst32(c.wasmLocalFunctionIndex).Insert(builder).Return(), + ) + + l := c.state() + tail := len(l.values) + args = args.Append(c.ssaBuilder.VarLengthPool(), l.values[tail-c.results():tail]...) + builder.AllocateInstruction(). + AsCallIndirect(afterListenerPtr, afterSig, args). + Insert(builder) +} + +const ( + elementOrDataInstanceLenOffset = 8 + elementOrDataInstanceSize = 24 +) + +// dropInstance inserts instructions to drop the element/data instance specified by the given index. +func (c *Compiler) dropDataOrElementInstance(index uint32, firstItemOffset wazevoapi.Offset) { + builder := c.ssaBuilder + instPtr := c.dataOrElementInstanceAddr(index, firstItemOffset) + + zero := builder.AllocateInstruction().AsIconst64(0).Insert(builder).Return() + + // Clear the instance. + builder.AllocateInstruction().AsStore(ssa.OpcodeStore, zero, instPtr, 0).Insert(builder) + builder.AllocateInstruction().AsStore(ssa.OpcodeStore, zero, instPtr, elementOrDataInstanceLenOffset).Insert(builder) + builder.AllocateInstruction().AsStore(ssa.OpcodeStore, zero, instPtr, elementOrDataInstanceLenOffset+8).Insert(builder) +} + +func (c *Compiler) dataOrElementInstanceAddr(index uint32, firstItemOffset wazevoapi.Offset) ssa.Value { + builder := c.ssaBuilder + + _1stItemPtr := builder. + AllocateInstruction(). + AsLoad(c.moduleCtxPtrValue, firstItemOffset.U32(), ssa.TypeI64). + Insert(builder).Return() + + // Each data/element instance is a slice, so we need to multiply index by 16 to get the offset of the target instance. + index = index * elementOrDataInstanceSize + indexExt := builder.AllocateInstruction().AsIconst64(uint64(index)).Insert(builder).Return() + // Then, add the offset to the address of the instance. + instPtr := builder.AllocateInstruction().AsIadd(_1stItemPtr, indexExt).Insert(builder).Return() + return instPtr +} + +func (c *Compiler) boundsCheckInDataOrElementInstance(instPtr, offsetInInstance, copySize ssa.Value, exitCode wazevoapi.ExitCode) { + builder := c.ssaBuilder + dataInstLen := builder.AllocateInstruction(). + AsLoad(instPtr, elementOrDataInstanceLenOffset, ssa.TypeI64). + Insert(builder).Return() + ceil := builder.AllocateInstruction().AsIadd(offsetInInstance, copySize).Insert(builder).Return() + cmp := builder.AllocateInstruction(). + AsIcmp(dataInstLen, ceil, ssa.IntegerCmpCondUnsignedLessThan). + Insert(builder). + Return() + builder.AllocateInstruction(). + AsExitIfTrueWithCode(c.execCtxPtrValue, cmp, exitCode). + Insert(builder) +} + +func (c *Compiler) boundsCheckInTable(tableIndex uint32, offset, size ssa.Value) (tableInstancePtr ssa.Value) { + builder := c.ssaBuilder + dstCeil := builder.AllocateInstruction().AsIadd(offset, size).Insert(builder).Return() + + // Load the table. + tableInstancePtr = builder.AllocateInstruction(). + AsLoad(c.moduleCtxPtrValue, c.offset.TableOffset(int(tableIndex)).U32(), ssa.TypeI64). + Insert(builder).Return() + + // Load the table's length. + tableLen := builder.AllocateInstruction(). + AsLoad(tableInstancePtr, tableInstanceLenOffset, ssa.TypeI32).Insert(builder).Return() + tableLenExt := builder.AllocateInstruction().AsUExtend(tableLen, 32, 64).Insert(builder).Return() + + // Compare the length and the target, and trap if out of bounds. + checkOOB := builder.AllocateInstruction() + checkOOB.AsIcmp(tableLenExt, dstCeil, ssa.IntegerCmpCondUnsignedLessThan) + builder.InsertInstruction(checkOOB) + exitIfOOB := builder.AllocateInstruction() + exitIfOOB.AsExitIfTrueWithCode(c.execCtxPtrValue, checkOOB.Return(), wazevoapi.ExitCodeTableOutOfBounds) + builder.InsertInstruction(exitIfOOB) + return +} + +func (c *Compiler) loadTableBaseAddr(tableInstancePtr ssa.Value) ssa.Value { + builder := c.ssaBuilder + loadTableBaseAddress := builder. + AllocateInstruction(). + AsLoad(tableInstancePtr, tableInstanceBaseAddressOffset, ssa.TypeI64). + Insert(builder) + return loadTableBaseAddress.Return() +} + +func (c *Compiler) boundsCheckInMemory(memLen, offset, size ssa.Value) { + builder := c.ssaBuilder + ceil := builder.AllocateInstruction().AsIadd(offset, size).Insert(builder).Return() + cmp := builder.AllocateInstruction(). + AsIcmp(memLen, ceil, ssa.IntegerCmpCondUnsignedLessThan). + Insert(builder). + Return() + builder.AllocateInstruction(). + AsExitIfTrueWithCode(c.execCtxPtrValue, cmp, wazevoapi.ExitCodeMemoryOutOfBounds). + Insert(builder) +} diff --git a/vendor/github.com/tetratelabs/wazero/internal/engine/wazevo/frontend/misc.go b/vendor/github.com/tetratelabs/wazero/internal/engine/wazevo/frontend/misc.go new file mode 100644 index 000000000..2db2b892c --- /dev/null +++ b/vendor/github.com/tetratelabs/wazero/internal/engine/wazevo/frontend/misc.go @@ -0,0 +1,10 @@ +package frontend + +import ( + "github.com/tetratelabs/wazero/internal/engine/wazevo/ssa" + "github.com/tetratelabs/wazero/internal/wasm" +) + +func FunctionIndexToFuncRef(idx wasm.Index) ssa.FuncRef { + return ssa.FuncRef(idx) +} diff --git a/vendor/github.com/tetratelabs/wazero/internal/engine/wazevo/frontend/sort_id.go b/vendor/github.com/tetratelabs/wazero/internal/engine/wazevo/frontend/sort_id.go new file mode 100644 index 000000000..1296706f5 --- /dev/null +++ b/vendor/github.com/tetratelabs/wazero/internal/engine/wazevo/frontend/sort_id.go @@ -0,0 +1,15 @@ +//go:build go1.21 + +package frontend + +import ( + "slices" + + "github.com/tetratelabs/wazero/internal/engine/wazevo/ssa" +) + +func sortSSAValueIDs(IDs []ssa.ValueID) { + slices.SortFunc(IDs, func(i, j ssa.ValueID) int { + return int(i) - int(j) + }) +} diff --git a/vendor/github.com/tetratelabs/wazero/internal/engine/wazevo/frontend/sort_id_old.go b/vendor/github.com/tetratelabs/wazero/internal/engine/wazevo/frontend/sort_id_old.go new file mode 100644 index 000000000..2e786a160 --- /dev/null +++ b/vendor/github.com/tetratelabs/wazero/internal/engine/wazevo/frontend/sort_id_old.go @@ -0,0 +1,17 @@ +//go:build !go1.21 + +// TODO: delete after the floor Go version is 1.21 + +package frontend + +import ( + "sort" + + "github.com/tetratelabs/wazero/internal/engine/wazevo/ssa" +) + +func sortSSAValueIDs(IDs []ssa.ValueID) { + sort.SliceStable(IDs, func(i, j int) bool { + return int(IDs[i]) < int(IDs[j]) + }) +} diff --git a/vendor/github.com/tetratelabs/wazero/internal/engine/wazevo/hostmodule.go b/vendor/github.com/tetratelabs/wazero/internal/engine/wazevo/hostmodule.go new file mode 100644 index 000000000..8da7347a9 --- /dev/null +++ b/vendor/github.com/tetratelabs/wazero/internal/engine/wazevo/hostmodule.go @@ -0,0 +1,82 @@ +package wazevo + +import ( + "encoding/binary" + "reflect" + "unsafe" + + "github.com/tetratelabs/wazero/experimental" + "github.com/tetratelabs/wazero/internal/wasm" +) + +func buildHostModuleOpaque(m *wasm.Module, listeners []experimental.FunctionListener) moduleContextOpaque { + size := len(m.CodeSection)*16 + 32 + ret := newAlignedOpaque(size) + + binary.LittleEndian.PutUint64(ret[0:], uint64(uintptr(unsafe.Pointer(m)))) + + if len(listeners) > 0 { + sliceHeader := (*reflect.SliceHeader)(unsafe.Pointer(&listeners)) + binary.LittleEndian.PutUint64(ret[8:], uint64(sliceHeader.Data)) + binary.LittleEndian.PutUint64(ret[16:], uint64(sliceHeader.Len)) + binary.LittleEndian.PutUint64(ret[24:], uint64(sliceHeader.Cap)) + } + + offset := 32 + for i := range m.CodeSection { + goFn := m.CodeSection[i].GoFunc + writeIface(goFn, ret[offset:]) + offset += 16 + } + return ret +} + +func hostModuleFromOpaque(opaqueBegin uintptr) *wasm.Module { + var opaqueViewOverSlice []byte + sh := (*reflect.SliceHeader)(unsafe.Pointer(&opaqueViewOverSlice)) + sh.Data = opaqueBegin + sh.Len = 32 + sh.Cap = 32 + return *(**wasm.Module)(unsafe.Pointer(&opaqueViewOverSlice[0])) +} + +func hostModuleListenersSliceFromOpaque(opaqueBegin uintptr) []experimental.FunctionListener { + var opaqueViewOverSlice []byte + sh := (*reflect.SliceHeader)(unsafe.Pointer(&opaqueViewOverSlice)) + sh.Data = opaqueBegin + sh.Len = 32 + sh.Cap = 32 + + b := binary.LittleEndian.Uint64(opaqueViewOverSlice[8:]) + l := binary.LittleEndian.Uint64(opaqueViewOverSlice[16:]) + c := binary.LittleEndian.Uint64(opaqueViewOverSlice[24:]) + var ret []experimental.FunctionListener + sh = (*reflect.SliceHeader)(unsafe.Pointer(&ret)) + sh.Data = uintptr(b) + setSliceLimits(sh, uintptr(l), uintptr(c)) + return ret +} + +func hostModuleGoFuncFromOpaque[T any](index int, opaqueBegin uintptr) T { + offset := uintptr(index*16) + 32 + ptr := opaqueBegin + offset + + var opaqueViewOverFunction []byte + sh := (*reflect.SliceHeader)(unsafe.Pointer(&opaqueViewOverFunction)) + sh.Data = ptr + sh.Len = 16 + sh.Cap = 16 + return readIface(opaqueViewOverFunction).(T) +} + +func writeIface(goFn interface{}, buf []byte) { + goFnIface := *(*[2]uint64)(unsafe.Pointer(&goFn)) + binary.LittleEndian.PutUint64(buf, goFnIface[0]) + binary.LittleEndian.PutUint64(buf[8:], goFnIface[1]) +} + +func readIface(buf []byte) interface{} { + b := binary.LittleEndian.Uint64(buf) + s := binary.LittleEndian.Uint64(buf[8:]) + return *(*interface{})(unsafe.Pointer(&[2]uint64{b, s})) +} diff --git a/vendor/github.com/tetratelabs/wazero/internal/engine/wazevo/isa_amd64.go b/vendor/github.com/tetratelabs/wazero/internal/engine/wazevo/isa_amd64.go new file mode 100644 index 000000000..da27cc108 --- /dev/null +++ b/vendor/github.com/tetratelabs/wazero/internal/engine/wazevo/isa_amd64.go @@ -0,0 +1,30 @@ +//go:build amd64 + +package wazevo + +import ( + "github.com/tetratelabs/wazero/internal/engine/wazevo/backend" + "github.com/tetratelabs/wazero/internal/engine/wazevo/backend/isa/amd64" +) + +func newMachine() backend.Machine { + return amd64.NewBackend() +} + +// unwindStack is a function to unwind the stack, and appends return addresses to `returnAddresses` slice. +// The implementation must be aligned with the ABI/Calling convention. +func unwindStack(sp, fp, top uintptr, returnAddresses []uintptr) []uintptr { + return amd64.UnwindStack(sp, fp, top, returnAddresses) +} + +// goCallStackView is a function to get a view of the stack before a Go call, which +// is the view of the stack allocated in CompileGoFunctionTrampoline. +func goCallStackView(stackPointerBeforeGoCall *uint64) []uint64 { + return amd64.GoCallStackView(stackPointerBeforeGoCall) +} + +// adjustClonedStack is a function to adjust the stack after it is grown. +// More precisely, absolute addresses (frame pointers) in the stack must be adjusted. +func adjustClonedStack(oldsp, oldTop, sp, fp, top uintptr) { + amd64.AdjustClonedStack(oldsp, oldTop, sp, fp, top) +} diff --git a/vendor/github.com/tetratelabs/wazero/internal/engine/wazevo/isa_arm64.go b/vendor/github.com/tetratelabs/wazero/internal/engine/wazevo/isa_arm64.go new file mode 100644 index 000000000..e7a846548 --- /dev/null +++ b/vendor/github.com/tetratelabs/wazero/internal/engine/wazevo/isa_arm64.go @@ -0,0 +1,32 @@ +//go:build arm64 + +package wazevo + +import ( + "github.com/tetratelabs/wazero/internal/engine/wazevo/backend" + "github.com/tetratelabs/wazero/internal/engine/wazevo/backend/isa/arm64" +) + +func newMachine() backend.Machine { + return arm64.NewBackend() +} + +// unwindStack is a function to unwind the stack, and appends return addresses to `returnAddresses` slice. +// The implementation must be aligned with the ABI/Calling convention. +func unwindStack(sp, fp, top uintptr, returnAddresses []uintptr) []uintptr { + return arm64.UnwindStack(sp, fp, top, returnAddresses) +} + +// goCallStackView is a function to get a view of the stack before a Go call, which +// is the view of the stack allocated in CompileGoFunctionTrampoline. +func goCallStackView(stackPointerBeforeGoCall *uint64) []uint64 { + return arm64.GoCallStackView(stackPointerBeforeGoCall) +} + +// adjustClonedStack is a function to adjust the stack after it is grown. +// More precisely, absolute addresses (frame pointers) in the stack must be adjusted. +func adjustClonedStack(oldsp, oldTop, sp, fp, top uintptr) { + // TODO: currently, the frame pointers are not used, and saved old sps are relative to the current stack pointer, + // so no need to adjustment on arm64. However, when we make it absolute, which in my opinion is better perf-wise + // at the expense of slightly costly stack growth, we need to adjust the pushed frame pointers. +} diff --git a/vendor/github.com/tetratelabs/wazero/internal/engine/wazevo/isa_other.go b/vendor/github.com/tetratelabs/wazero/internal/engine/wazevo/isa_other.go new file mode 100644 index 000000000..c5afc6314 --- /dev/null +++ b/vendor/github.com/tetratelabs/wazero/internal/engine/wazevo/isa_other.go @@ -0,0 +1,29 @@ +//go:build !(amd64 || arm64) + +package wazevo + +import ( + "github.com/tetratelabs/wazero/internal/engine/wazevo/backend" +) + +func newMachine() backend.Machine { + panic("unsupported architecture") +} + +// unwindStack is a function to unwind the stack, and appends return addresses to `returnAddresses` slice. +// The implementation must be aligned with the ABI/Calling convention. +func unwindStack(sp, fp, top uintptr, returnAddresses []uintptr) []uintptr { + panic("unsupported architecture") +} + +// goCallStackView is a function to get a view of the stack before a Go call, which +// is the view of the stack allocated in CompileGoFunctionTrampoline. +func goCallStackView(stackPointerBeforeGoCall *uint64) []uint64 { + panic("unsupported architecture") +} + +// adjustClonedStack is a function to adjust the stack after it is grown. +// More precisely, absolute addresses (frame pointers) in the stack must be adjusted. +func adjustClonedStack(oldsp, oldTop, sp, fp, top uintptr) { + panic("unsupported architecture") +} diff --git a/vendor/github.com/tetratelabs/wazero/internal/engine/wazevo/memmove.go b/vendor/github.com/tetratelabs/wazero/internal/engine/wazevo/memmove.go new file mode 100644 index 000000000..889922107 --- /dev/null +++ b/vendor/github.com/tetratelabs/wazero/internal/engine/wazevo/memmove.go @@ -0,0 +1,11 @@ +package wazevo + +import ( + "reflect" + "unsafe" +) + +//go:linkname memmove runtime.memmove +func memmove(_, _ unsafe.Pointer, _ uintptr) + +var memmovPtr = reflect.ValueOf(memmove).Pointer() diff --git a/vendor/github.com/tetratelabs/wazero/internal/engine/wazevo/module_engine.go b/vendor/github.com/tetratelabs/wazero/internal/engine/wazevo/module_engine.go new file mode 100644 index 000000000..ba8f546c0 --- /dev/null +++ b/vendor/github.com/tetratelabs/wazero/internal/engine/wazevo/module_engine.go @@ -0,0 +1,344 @@ +package wazevo + +import ( + "encoding/binary" + "unsafe" + + "github.com/tetratelabs/wazero/api" + "github.com/tetratelabs/wazero/experimental" + "github.com/tetratelabs/wazero/internal/engine/wazevo/wazevoapi" + "github.com/tetratelabs/wazero/internal/wasm" + "github.com/tetratelabs/wazero/internal/wasmruntime" +) + +type ( + // moduleEngine implements wasm.ModuleEngine. + moduleEngine struct { + // opaquePtr equals &opaque[0]. + opaquePtr *byte + parent *compiledModule + module *wasm.ModuleInstance + opaque moduleContextOpaque + localFunctionInstances []*functionInstance + importedFunctions []importedFunction + listeners []experimental.FunctionListener + } + + functionInstance struct { + executable *byte + moduleContextOpaquePtr *byte + typeID wasm.FunctionTypeID + indexInModule wasm.Index + } + + importedFunction struct { + me *moduleEngine + indexInModule wasm.Index + } + + // moduleContextOpaque is the opaque byte slice of Module instance specific contents whose size + // is only Wasm-compile-time known, hence dynamic. Its contents are basically the pointers to the module instance, + // specific objects as well as functions. This is sometimes called "VMContext" in other Wasm runtimes. + // + // Internally, the buffer is structured as follows: + // + // type moduleContextOpaque struct { + // moduleInstance *wasm.ModuleInstance + // localMemoryBufferPtr *byte (optional) + // localMemoryLength uint64 (optional) + // importedMemoryInstance *wasm.MemoryInstance (optional) + // importedMemoryOwnerOpaqueCtx *byte (optional) + // importedFunctions [# of importedFunctions]functionInstance + // importedGlobals []ImportedGlobal (optional) + // localGlobals []Global (optional) + // typeIDsBegin &wasm.ModuleInstance.TypeIDs[0] (optional) + // tables []*wasm.TableInstance (optional) + // beforeListenerTrampolines1stElement **byte (optional) + // afterListenerTrampolines1stElement **byte (optional) + // dataInstances1stElement []wasm.DataInstance (optional) + // elementInstances1stElement []wasm.ElementInstance (optional) + // } + // + // type ImportedGlobal struct { + // *Global + // _ uint64 // padding + // } + // + // type Global struct { + // Val, ValHi uint64 + // } + // + // See wazevoapi.NewModuleContextOffsetData for the details of the offsets. + // + // Note that for host modules, the structure is entirely different. See buildHostModuleOpaque. + moduleContextOpaque []byte +) + +func newAlignedOpaque(size int) moduleContextOpaque { + // Check if the size is a multiple of 16. + if size%16 != 0 { + panic("size must be a multiple of 16") + } + buf := make([]byte, size+16) + // Align the buffer to 16 bytes. + rem := uintptr(unsafe.Pointer(&buf[0])) % 16 + buf = buf[16-rem:] + return buf +} + +func putLocalMemory(opaque []byte, offset wazevoapi.Offset, mem *wasm.MemoryInstance) { + s := uint64(len(mem.Buffer)) + var b uint64 + if len(mem.Buffer) > 0 { + b = uint64(uintptr(unsafe.Pointer(&mem.Buffer[0]))) + } + binary.LittleEndian.PutUint64(opaque[offset:], b) + binary.LittleEndian.PutUint64(opaque[offset+8:], s) +} + +func (m *moduleEngine) setupOpaque() { + inst := m.module + offsets := &m.parent.offsets + opaque := m.opaque + + binary.LittleEndian.PutUint64(opaque[offsets.ModuleInstanceOffset:], + uint64(uintptr(unsafe.Pointer(m.module))), + ) + + if lm := offsets.LocalMemoryBegin; lm >= 0 { + putLocalMemory(opaque, lm, inst.MemoryInstance) + } + + // Note: imported memory is resolved in ResolveImportedFunction. + + // Note: imported functions are resolved in ResolveImportedFunction. + + if globalOffset := offsets.GlobalsBegin; globalOffset >= 0 { + for i, g := range inst.Globals { + if i < int(inst.Source.ImportGlobalCount) { + importedME := g.Me.(*moduleEngine) + offset := importedME.parent.offsets.GlobalInstanceOffset(g.Index) + importedMEOpaque := importedME.opaque + binary.LittleEndian.PutUint64(opaque[globalOffset:], + uint64(uintptr(unsafe.Pointer(&importedMEOpaque[offset])))) + } else { + binary.LittleEndian.PutUint64(opaque[globalOffset:], g.Val) + binary.LittleEndian.PutUint64(opaque[globalOffset+8:], g.ValHi) + } + globalOffset += 16 + } + } + + if tableOffset := offsets.TablesBegin; tableOffset >= 0 { + // First we write the first element's address of typeIDs. + if len(inst.TypeIDs) > 0 { + binary.LittleEndian.PutUint64(opaque[offsets.TypeIDs1stElement:], uint64(uintptr(unsafe.Pointer(&inst.TypeIDs[0])))) + } + + // Then we write the table addresses. + for _, table := range inst.Tables { + binary.LittleEndian.PutUint64(opaque[tableOffset:], uint64(uintptr(unsafe.Pointer(table)))) + tableOffset += 8 + } + } + + if beforeListenerOffset := offsets.BeforeListenerTrampolines1stElement; beforeListenerOffset >= 0 { + binary.LittleEndian.PutUint64(opaque[beforeListenerOffset:], uint64(uintptr(unsafe.Pointer(&m.parent.listenerBeforeTrampolines[0])))) + } + if afterListenerOffset := offsets.AfterListenerTrampolines1stElement; afterListenerOffset >= 0 { + binary.LittleEndian.PutUint64(opaque[afterListenerOffset:], uint64(uintptr(unsafe.Pointer(&m.parent.listenerAfterTrampolines[0])))) + } + if len(inst.DataInstances) > 0 { + binary.LittleEndian.PutUint64(opaque[offsets.DataInstances1stElement:], uint64(uintptr(unsafe.Pointer(&inst.DataInstances[0])))) + } + if len(inst.ElementInstances) > 0 { + binary.LittleEndian.PutUint64(opaque[offsets.ElementInstances1stElement:], uint64(uintptr(unsafe.Pointer(&inst.ElementInstances[0])))) + } +} + +// NewFunction implements wasm.ModuleEngine. +func (m *moduleEngine) NewFunction(index wasm.Index) api.Function { + if wazevoapi.PrintMachineCodeHexPerFunctionDisassemblable { + panic("When PrintMachineCodeHexPerFunctionDisassemblable enabled, functions must not be called") + } + + localIndex := index + if importedFnCount := m.module.Source.ImportFunctionCount; index < importedFnCount { + imported := &m.importedFunctions[index] + return imported.me.NewFunction(imported.indexInModule) + } else { + localIndex -= importedFnCount + } + + src := m.module.Source + typIndex := src.FunctionSection[localIndex] + typ := src.TypeSection[typIndex] + sizeOfParamResultSlice := typ.ResultNumInUint64 + if ps := typ.ParamNumInUint64; ps > sizeOfParamResultSlice { + sizeOfParamResultSlice = ps + } + p := m.parent + offset := p.functionOffsets[localIndex] + + ce := &callEngine{ + indexInModule: index, + executable: &p.executable[offset], + parent: m, + preambleExecutable: &m.parent.entryPreambles[typIndex][0], + sizeOfParamResultSlice: sizeOfParamResultSlice, + requiredParams: typ.ParamNumInUint64, + numberOfResults: typ.ResultNumInUint64, + } + + ce.execCtx.memoryGrowTrampolineAddress = &m.parent.sharedFunctions.memoryGrowExecutable[0] + ce.execCtx.stackGrowCallTrampolineAddress = &m.parent.sharedFunctions.stackGrowExecutable[0] + ce.execCtx.checkModuleExitCodeTrampolineAddress = &m.parent.sharedFunctions.checkModuleExitCode[0] + ce.execCtx.tableGrowTrampolineAddress = &m.parent.sharedFunctions.tableGrowExecutable[0] + ce.execCtx.refFuncTrampolineAddress = &m.parent.sharedFunctions.refFuncExecutable[0] + ce.execCtx.memoryWait32TrampolineAddress = &m.parent.sharedFunctions.memoryWait32Executable[0] + ce.execCtx.memoryWait64TrampolineAddress = &m.parent.sharedFunctions.memoryWait64Executable[0] + ce.execCtx.memoryNotifyTrampolineAddress = &m.parent.sharedFunctions.memoryNotifyExecutable[0] + ce.execCtx.memmoveAddress = memmovPtr + ce.init() + return ce +} + +// GetGlobalValue implements the same method as documented on wasm.ModuleEngine. +func (m *moduleEngine) GetGlobalValue(i wasm.Index) (lo, hi uint64) { + offset := m.parent.offsets.GlobalInstanceOffset(i) + buf := m.opaque[offset:] + if i < m.module.Source.ImportGlobalCount { + panic("GetGlobalValue should not be called for imported globals") + } + return binary.LittleEndian.Uint64(buf), binary.LittleEndian.Uint64(buf[8:]) +} + +// SetGlobalValue implements the same method as documented on wasm.ModuleEngine. +func (m *moduleEngine) SetGlobalValue(i wasm.Index, lo, hi uint64) { + offset := m.parent.offsets.GlobalInstanceOffset(i) + buf := m.opaque[offset:] + if i < m.module.Source.ImportGlobalCount { + panic("GetGlobalValue should not be called for imported globals") + } + binary.LittleEndian.PutUint64(buf, lo) + binary.LittleEndian.PutUint64(buf[8:], hi) +} + +// OwnsGlobals implements the same method as documented on wasm.ModuleEngine. +func (m *moduleEngine) OwnsGlobals() bool { return true } + +// ResolveImportedFunction implements wasm.ModuleEngine. +func (m *moduleEngine) ResolveImportedFunction(index, indexInImportedModule wasm.Index, importedModuleEngine wasm.ModuleEngine) { + executableOffset, moduleCtxOffset, typeIDOffset := m.parent.offsets.ImportedFunctionOffset(index) + importedME := importedModuleEngine.(*moduleEngine) + + if int(indexInImportedModule) >= len(importedME.importedFunctions) { + indexInImportedModule -= wasm.Index(len(importedME.importedFunctions)) + } else { + imported := &importedME.importedFunctions[indexInImportedModule] + m.ResolveImportedFunction(index, imported.indexInModule, imported.me) + return // Recursively resolve the imported function. + } + + offset := importedME.parent.functionOffsets[indexInImportedModule] + typeID := getTypeIDOf(indexInImportedModule, importedME.module) + executable := &importedME.parent.executable[offset] + // Write functionInstance. + binary.LittleEndian.PutUint64(m.opaque[executableOffset:], uint64(uintptr(unsafe.Pointer(executable)))) + binary.LittleEndian.PutUint64(m.opaque[moduleCtxOffset:], uint64(uintptr(unsafe.Pointer(importedME.opaquePtr)))) + binary.LittleEndian.PutUint64(m.opaque[typeIDOffset:], uint64(typeID)) + + // Write importedFunction so that it can be used by NewFunction. + m.importedFunctions[index] = importedFunction{me: importedME, indexInModule: indexInImportedModule} +} + +func getTypeIDOf(funcIndex wasm.Index, m *wasm.ModuleInstance) wasm.FunctionTypeID { + source := m.Source + + var typeIndex wasm.Index + if funcIndex >= source.ImportFunctionCount { + funcIndex -= source.ImportFunctionCount + typeIndex = source.FunctionSection[funcIndex] + } else { + var cnt wasm.Index + for i := range source.ImportSection { + if source.ImportSection[i].Type == wasm.ExternTypeFunc { + if cnt == funcIndex { + typeIndex = source.ImportSection[i].DescFunc + break + } + cnt++ + } + } + } + return m.TypeIDs[typeIndex] +} + +// ResolveImportedMemory implements wasm.ModuleEngine. +func (m *moduleEngine) ResolveImportedMemory(importedModuleEngine wasm.ModuleEngine) { + importedME := importedModuleEngine.(*moduleEngine) + inst := importedME.module + + var memInstPtr uint64 + var memOwnerOpaquePtr uint64 + if offs := importedME.parent.offsets; offs.ImportedMemoryBegin >= 0 { + offset := offs.ImportedMemoryBegin + memInstPtr = binary.LittleEndian.Uint64(importedME.opaque[offset:]) + memOwnerOpaquePtr = binary.LittleEndian.Uint64(importedME.opaque[offset+8:]) + } else { + memInstPtr = uint64(uintptr(unsafe.Pointer(inst.MemoryInstance))) + memOwnerOpaquePtr = uint64(uintptr(unsafe.Pointer(importedME.opaquePtr))) + } + offset := m.parent.offsets.ImportedMemoryBegin + binary.LittleEndian.PutUint64(m.opaque[offset:], memInstPtr) + binary.LittleEndian.PutUint64(m.opaque[offset+8:], memOwnerOpaquePtr) +} + +// DoneInstantiation implements wasm.ModuleEngine. +func (m *moduleEngine) DoneInstantiation() { + if !m.module.Source.IsHostModule { + m.setupOpaque() + } +} + +// FunctionInstanceReference implements wasm.ModuleEngine. +func (m *moduleEngine) FunctionInstanceReference(funcIndex wasm.Index) wasm.Reference { + if funcIndex < m.module.Source.ImportFunctionCount { + begin, _, _ := m.parent.offsets.ImportedFunctionOffset(funcIndex) + return uintptr(unsafe.Pointer(&m.opaque[begin])) + } + localIndex := funcIndex - m.module.Source.ImportFunctionCount + p := m.parent + executable := &p.executable[p.functionOffsets[localIndex]] + typeID := m.module.TypeIDs[m.module.Source.FunctionSection[localIndex]] + + lf := &functionInstance{ + executable: executable, + moduleContextOpaquePtr: m.opaquePtr, + typeID: typeID, + indexInModule: funcIndex, + } + m.localFunctionInstances = append(m.localFunctionInstances, lf) + return uintptr(unsafe.Pointer(lf)) +} + +// LookupFunction implements wasm.ModuleEngine. +func (m *moduleEngine) LookupFunction(t *wasm.TableInstance, typeId wasm.FunctionTypeID, tableOffset wasm.Index) (*wasm.ModuleInstance, wasm.Index) { + if tableOffset >= uint32(len(t.References)) || t.Type != wasm.RefTypeFuncref { + panic(wasmruntime.ErrRuntimeInvalidTableAccess) + } + rawPtr := t.References[tableOffset] + if rawPtr == 0 { + panic(wasmruntime.ErrRuntimeInvalidTableAccess) + } + + tf := wazevoapi.PtrFromUintptr[functionInstance](rawPtr) + if tf.typeID != typeId { + panic(wasmruntime.ErrRuntimeIndirectCallTypeMismatch) + } + return moduleInstanceFromOpaquePtr(tf.moduleContextOpaquePtr), tf.indexInModule +} + +func moduleInstanceFromOpaquePtr(ptr *byte) *wasm.ModuleInstance { + return *(**wasm.ModuleInstance)(unsafe.Pointer(ptr)) +} diff --git a/vendor/github.com/tetratelabs/wazero/internal/engine/wazevo/reflect.go b/vendor/github.com/tetratelabs/wazero/internal/engine/wazevo/reflect.go new file mode 100644 index 000000000..6a03fc65c --- /dev/null +++ b/vendor/github.com/tetratelabs/wazero/internal/engine/wazevo/reflect.go @@ -0,0 +1,11 @@ +//go:build !tinygo + +package wazevo + +import "reflect" + +// setSliceLimits sets both Cap and Len for the given reflected slice. +func setSliceLimits(s *reflect.SliceHeader, l, c uintptr) { + s.Len = int(l) + s.Cap = int(c) +} diff --git a/vendor/github.com/tetratelabs/wazero/internal/engine/wazevo/reflect_tinygo.go b/vendor/github.com/tetratelabs/wazero/internal/engine/wazevo/reflect_tinygo.go new file mode 100644 index 000000000..eda3e706a --- /dev/null +++ b/vendor/github.com/tetratelabs/wazero/internal/engine/wazevo/reflect_tinygo.go @@ -0,0 +1,11 @@ +//go:build tinygo + +package wazevo + +import "reflect" + +// setSliceLimits sets both Cap and Len for the given reflected slice. +func setSliceLimits(s *reflect.SliceHeader, l, c uintptr) { + s.Len = l + s.Cap = c +} diff --git a/vendor/github.com/tetratelabs/wazero/internal/engine/wazevo/ssa/basic_block.go b/vendor/github.com/tetratelabs/wazero/internal/engine/wazevo/ssa/basic_block.go new file mode 100644 index 000000000..10b6b4b62 --- /dev/null +++ b/vendor/github.com/tetratelabs/wazero/internal/engine/wazevo/ssa/basic_block.go @@ -0,0 +1,407 @@ +package ssa + +import ( + "fmt" + "strconv" + "strings" + + "github.com/tetratelabs/wazero/internal/engine/wazevo/wazevoapi" +) + +// BasicBlock represents the Basic Block of an SSA function. +// Each BasicBlock always ends with branching instructions (e.g. Branch, Return, etc.), +// and at most two branches are allowed. If there's two branches, these two are placed together at the end of the block. +// In other words, there's no branching instruction in the middle of the block. +// +// Note: we use the "block argument" variant of SSA, instead of PHI functions. See the package level doc comments. +// +// Note: we use "parameter/param" as a placeholder which represents a variant of PHI, and "argument/arg" as an actual +// Value passed to that "parameter/param". +type BasicBlock interface { + // ID returns the unique ID of this block. + ID() BasicBlockID + + // Name returns the unique string ID of this block. e.g. blk0, blk1, ... + Name() string + + // AddParam adds the parameter to the block whose type specified by `t`. + AddParam(b Builder, t Type) Value + + // Params returns the number of parameters to this block. + Params() int + + // Param returns (Variable, Value) which corresponds to the i-th parameter of this block. + // The returned Value is the definition of the param in this block. + Param(i int) Value + + // InsertInstruction inserts an instruction that implements Value into the tail of this block. + InsertInstruction(raw *Instruction) + + // Root returns the root instruction of this block. + Root() *Instruction + + // Tail returns the tail instruction of this block. + Tail() *Instruction + + // EntryBlock returns true if this block represents the function entry. + EntryBlock() bool + + // ReturnBlock returns ture if this block represents the function return. + ReturnBlock() bool + + // FormatHeader returns the debug string of this block, not including instruction. + FormatHeader(b Builder) string + + // Valid is true if this block is still valid even after optimizations. + Valid() bool + + // Sealed is true if this block has been sealed. + Sealed() bool + + // BeginPredIterator returns the first predecessor of this block. + BeginPredIterator() BasicBlock + + // NextPredIterator returns the next predecessor of this block. + NextPredIterator() BasicBlock + + // Preds returns the number of predecessors of this block. + Preds() int + + // Pred returns the i-th predecessor of this block. + Pred(i int) BasicBlock + + // Succs returns the number of successors of this block. + Succs() int + + // Succ returns the i-th successor of this block. + Succ(i int) BasicBlock + + // LoopHeader returns true if this block is a loop header. + LoopHeader() bool + + // LoopNestingForestChildren returns the children of this block in the loop nesting forest. + LoopNestingForestChildren() []BasicBlock +} + +type ( + // basicBlock is a basic block in a SSA-transformed function. + basicBlock struct { + id BasicBlockID + rootInstr, currentInstr *Instruction + params []blockParam + predIter int + preds []basicBlockPredecessorInfo + success []*basicBlock + // singlePred is the alias to preds[0] for fast lookup, and only set after Seal is called. + singlePred *basicBlock + // lastDefinitions maps Variable to its last definition in this block. + lastDefinitions map[Variable]Value + // unknownsValues are used in builder.findValue. The usage is well-described in the paper. + unknownValues []unknownValue + // invalid is true if this block is made invalid during optimizations. + invalid bool + // sealed is true if this is sealed (all the predecessors are known). + sealed bool + // loopHeader is true if this block is a loop header: + // + // > A loop header (sometimes called the entry point of the loop) is a dominator that is the target + // > of a loop-forming back edge. The loop header dominates all blocks in the loop body. + // > A block may be a loop header for more than one loop. A loop may have multiple entry points, + // > in which case it has no "loop header". + // + // See https://en.wikipedia.org/wiki/Control-flow_graph for more details. + // + // This is modified during the subPassLoopDetection pass. + loopHeader bool + + // loopNestingForestChildren holds the children of this block in the loop nesting forest. + // Non-empty if and only if this block is a loop header (i.e. loopHeader=true) + loopNestingForestChildren []BasicBlock + + // reversePostOrder is used to sort all the blocks in the function in reverse post order. + // This is used in builder.LayoutBlocks. + reversePostOrder int + + // child and sibling are the ones in the dominator tree. + child, sibling *basicBlock + } + // BasicBlockID is the unique ID of a basicBlock. + BasicBlockID uint32 + + // blockParam implements Value and represents a parameter to a basicBlock. + blockParam struct { + // value is the Value that corresponds to the parameter in this block, + // and can be considered as an output of PHI instruction in traditional SSA. + value Value + // typ is the type of the parameter. + typ Type + } + + unknownValue struct { + // variable is the variable that this unknownValue represents. + variable Variable + // value is the value that this unknownValue represents. + value Value + } +) + +const basicBlockIDReturnBlock = 0xffffffff + +// Name implements BasicBlock.Name. +func (bb *basicBlock) Name() string { + if bb.id == basicBlockIDReturnBlock { + return "blk_ret" + } else { + return fmt.Sprintf("blk%d", bb.id) + } +} + +// String implements fmt.Stringer for debugging. +func (bid BasicBlockID) String() string { + if bid == basicBlockIDReturnBlock { + return "blk_ret" + } else { + return fmt.Sprintf("blk%d", bid) + } +} + +// ID implements BasicBlock.ID. +func (bb *basicBlock) ID() BasicBlockID { + return bb.id +} + +// basicBlockPredecessorInfo is the information of a predecessor of a basicBlock. +// predecessor is determined by a pair of block and the branch instruction used to jump to the successor. +type basicBlockPredecessorInfo struct { + blk *basicBlock + branch *Instruction +} + +// EntryBlock implements BasicBlock.EntryBlock. +func (bb *basicBlock) EntryBlock() bool { + return bb.id == 0 +} + +// ReturnBlock implements BasicBlock.ReturnBlock. +func (bb *basicBlock) ReturnBlock() bool { + return bb.id == basicBlockIDReturnBlock +} + +// AddParam implements BasicBlock.AddParam. +func (bb *basicBlock) AddParam(b Builder, typ Type) Value { + paramValue := b.allocateValue(typ) + bb.params = append(bb.params, blockParam{typ: typ, value: paramValue}) + return paramValue +} + +// addParamOn adds a parameter to this block whose value is already allocated. +func (bb *basicBlock) addParamOn(typ Type, value Value) { + bb.params = append(bb.params, blockParam{typ: typ, value: value}) +} + +// Params implements BasicBlock.Params. +func (bb *basicBlock) Params() int { + return len(bb.params) +} + +// Param implements BasicBlock.Param. +func (bb *basicBlock) Param(i int) Value { + p := &bb.params[i] + return p.value +} + +// Valid implements BasicBlock.Valid. +func (bb *basicBlock) Valid() bool { + return !bb.invalid +} + +// Sealed implements BasicBlock.Sealed. +func (bb *basicBlock) Sealed() bool { + return bb.sealed +} + +// InsertInstruction implements BasicBlock.InsertInstruction. +func (bb *basicBlock) InsertInstruction(next *Instruction) { + current := bb.currentInstr + if current != nil { + current.next = next + next.prev = current + } else { + bb.rootInstr = next + } + bb.currentInstr = next + + switch next.opcode { + case OpcodeJump, OpcodeBrz, OpcodeBrnz: + target := next.blk.(*basicBlock) + target.addPred(bb, next) + case OpcodeBrTable: + for _, _target := range next.targets { + target := _target.(*basicBlock) + target.addPred(bb, next) + } + } +} + +// NumPreds implements BasicBlock.NumPreds. +func (bb *basicBlock) NumPreds() int { + return len(bb.preds) +} + +// BeginPredIterator implements BasicBlock.BeginPredIterator. +func (bb *basicBlock) BeginPredIterator() BasicBlock { + bb.predIter = 0 + return bb.NextPredIterator() +} + +// NextPredIterator implements BasicBlock.NextPredIterator. +func (bb *basicBlock) NextPredIterator() BasicBlock { + if bb.predIter >= len(bb.preds) { + return nil + } + pred := bb.preds[bb.predIter].blk + bb.predIter++ + return pred +} + +// Preds implements BasicBlock.Preds. +func (bb *basicBlock) Preds() int { + return len(bb.preds) +} + +// Pred implements BasicBlock.Pred. +func (bb *basicBlock) Pred(i int) BasicBlock { + return bb.preds[i].blk +} + +// Succs implements BasicBlock.Succs. +func (bb *basicBlock) Succs() int { + return len(bb.success) +} + +// Succ implements BasicBlock.Succ. +func (bb *basicBlock) Succ(i int) BasicBlock { + return bb.success[i] +} + +// Root implements BasicBlock.Root. +func (bb *basicBlock) Root() *Instruction { + return bb.rootInstr +} + +// Tail implements BasicBlock.Tail. +func (bb *basicBlock) Tail() *Instruction { + return bb.currentInstr +} + +// reset resets the basicBlock to its initial state so that it can be reused for another function. +func resetBasicBlock(bb *basicBlock) { + bb.params = bb.params[:0] + bb.rootInstr, bb.currentInstr = nil, nil + bb.preds = bb.preds[:0] + bb.success = bb.success[:0] + bb.invalid, bb.sealed = false, false + bb.singlePred = nil + bb.unknownValues = bb.unknownValues[:0] + bb.lastDefinitions = wazevoapi.ResetMap(bb.lastDefinitions) + bb.reversePostOrder = -1 + bb.loopNestingForestChildren = bb.loopNestingForestChildren[:0] + bb.loopHeader = false + bb.sibling = nil + bb.child = nil +} + +// addPred adds a predecessor to this block specified by the branch instruction. +func (bb *basicBlock) addPred(blk BasicBlock, branch *Instruction) { + if bb.sealed { + panic("BUG: trying to add predecessor to a sealed block: " + bb.Name()) + } + + pred := blk.(*basicBlock) + for i := range bb.preds { + existingPred := &bb.preds[i] + if existingPred.blk == pred && existingPred.branch != branch { + // If the target is already added, then this must come from the same BrTable, + // otherwise such redundant branch should be eliminated by the frontend. (which should be simpler). + panic(fmt.Sprintf("BUG: redundant non BrTable jumps in %s whose targes are the same", bb.Name())) + } + } + + bb.preds = append(bb.preds, basicBlockPredecessorInfo{ + blk: pred, + branch: branch, + }) + + pred.success = append(pred.success, bb) +} + +// FormatHeader implements BasicBlock.FormatHeader. +func (bb *basicBlock) FormatHeader(b Builder) string { + ps := make([]string, len(bb.params)) + for i, p := range bb.params { + ps[i] = p.value.formatWithType(b) + } + + if len(bb.preds) > 0 { + preds := make([]string, 0, len(bb.preds)) + for _, pred := range bb.preds { + if pred.blk.invalid { + continue + } + preds = append(preds, fmt.Sprintf("blk%d", pred.blk.id)) + + } + return fmt.Sprintf("blk%d: (%s) <-- (%s)", + bb.id, strings.Join(ps, ","), strings.Join(preds, ",")) + } else { + return fmt.Sprintf("blk%d: (%s)", bb.id, strings.Join(ps, ", ")) + } +} + +// validates validates the basicBlock for debugging purpose. +func (bb *basicBlock) validate(b *builder) { + if bb.invalid { + panic("BUG: trying to validate an invalid block: " + bb.Name()) + } + if len(bb.preds) > 0 { + for _, pred := range bb.preds { + if pred.branch.opcode != OpcodeBrTable { + if target := pred.branch.blk; target != bb { + panic(fmt.Sprintf("BUG: '%s' is not branch to %s, but to %s", + pred.branch.Format(b), bb.Name(), target.Name())) + } + } + + var exp int + if bb.ReturnBlock() { + exp = len(b.currentSignature.Results) + } else { + exp = len(bb.params) + } + + if len(pred.branch.vs.View()) != exp { + panic(fmt.Sprintf( + "BUG: len(argument at %s) != len(params at %s): %d != %d: %s", + pred.blk.Name(), bb.Name(), + len(pred.branch.vs.View()), len(bb.params), pred.branch.Format(b), + )) + } + + } + } +} + +// String implements fmt.Stringer for debugging purpose only. +func (bb *basicBlock) String() string { + return strconv.Itoa(int(bb.id)) +} + +// LoopNestingForestChildren implements BasicBlock.LoopNestingForestChildren. +func (bb *basicBlock) LoopNestingForestChildren() []BasicBlock { + return bb.loopNestingForestChildren +} + +// LoopHeader implements BasicBlock.LoopHeader. +func (bb *basicBlock) LoopHeader() bool { + return bb.loopHeader +} diff --git a/vendor/github.com/tetratelabs/wazero/internal/engine/wazevo/ssa/basic_block_sort.go b/vendor/github.com/tetratelabs/wazero/internal/engine/wazevo/ssa/basic_block_sort.go new file mode 100644 index 000000000..e1471edc3 --- /dev/null +++ b/vendor/github.com/tetratelabs/wazero/internal/engine/wazevo/ssa/basic_block_sort.go @@ -0,0 +1,34 @@ +//go:build go1.21 + +package ssa + +import ( + "slices" +) + +func sortBlocks(blocks []*basicBlock) { + slices.SortFunc(blocks, func(i, j *basicBlock) int { + jIsReturn := j.ReturnBlock() + iIsReturn := i.ReturnBlock() + if iIsReturn && jIsReturn { + return 0 + } + if jIsReturn { + return 1 + } + if iIsReturn { + return -1 + } + iRoot, jRoot := i.rootInstr, j.rootInstr + if iRoot == nil && jRoot == nil { // For testing. + return 0 + } + if jRoot == nil { + return 1 + } + if iRoot == nil { + return -1 + } + return i.rootInstr.id - j.rootInstr.id + }) +} diff --git a/vendor/github.com/tetratelabs/wazero/internal/engine/wazevo/ssa/basic_block_sort_old.go b/vendor/github.com/tetratelabs/wazero/internal/engine/wazevo/ssa/basic_block_sort_old.go new file mode 100644 index 000000000..9dc881dae --- /dev/null +++ b/vendor/github.com/tetratelabs/wazero/internal/engine/wazevo/ssa/basic_block_sort_old.go @@ -0,0 +1,24 @@ +//go:build !go1.21 + +// TODO: delete after the floor Go version is 1.21 + +package ssa + +import "sort" + +func sortBlocks(blocks []*basicBlock) { + sort.SliceStable(blocks, func(i, j int) bool { + iBlk, jBlk := blocks[i], blocks[j] + if jBlk.ReturnBlock() { + return true + } + if iBlk.ReturnBlock() { + return false + } + iRoot, jRoot := iBlk.rootInstr, jBlk.rootInstr + if iRoot == nil || jRoot == nil { // For testing. + return true + } + return iBlk.rootInstr.id < jBlk.rootInstr.id + }) +} diff --git a/vendor/github.com/tetratelabs/wazero/internal/engine/wazevo/ssa/builder.go b/vendor/github.com/tetratelabs/wazero/internal/engine/wazevo/ssa/builder.go new file mode 100644 index 000000000..1fc84d2ea --- /dev/null +++ b/vendor/github.com/tetratelabs/wazero/internal/engine/wazevo/ssa/builder.go @@ -0,0 +1,731 @@ +package ssa + +import ( + "fmt" + "sort" + "strings" + + "github.com/tetratelabs/wazero/internal/engine/wazevo/wazevoapi" +) + +// Builder is used to builds SSA consisting of Basic Blocks per function. +type Builder interface { + // Init must be called to reuse this builder for the next function. + Init(typ *Signature) + + // Signature returns the Signature of the currently-compiled function. + Signature() *Signature + + // BlockIDMax returns the maximum value of BasicBlocksID existing in the currently-compiled function. + BlockIDMax() BasicBlockID + + // AllocateBasicBlock creates a basic block in SSA function. + AllocateBasicBlock() BasicBlock + + // CurrentBlock returns the currently handled BasicBlock which is set by the latest call to SetCurrentBlock. + CurrentBlock() BasicBlock + + // EntryBlock returns the entry BasicBlock of the currently-compiled function. + EntryBlock() BasicBlock + + // SetCurrentBlock sets the instruction insertion target to the BasicBlock `b`. + SetCurrentBlock(b BasicBlock) + + // DeclareVariable declares a Variable of the given Type. + DeclareVariable(Type) Variable + + // DefineVariable defines a variable in the `block` with value. + // The defining instruction will be inserted into the `block`. + DefineVariable(variable Variable, value Value, block BasicBlock) + + // DefineVariableInCurrentBB is the same as DefineVariable except the definition is + // inserted into the current BasicBlock. Alias to DefineVariable(x, y, CurrentBlock()). + DefineVariableInCurrentBB(variable Variable, value Value) + + // AllocateInstruction returns a new Instruction. + AllocateInstruction() *Instruction + + // InsertInstruction executes BasicBlock.InsertInstruction for the currently handled basic block. + InsertInstruction(raw *Instruction) + + // allocateValue allocates an unused Value. + allocateValue(typ Type) Value + + // MustFindValue searches the latest definition of the given Variable and returns the result. + MustFindValue(variable Variable) Value + + // MustFindValueInBlk is the same as MustFindValue except it searches the latest definition from the given BasicBlock. + MustFindValueInBlk(variable Variable, blk BasicBlock) Value + + // FindValueInLinearPath tries to find the latest definition of the given Variable in the linear path to the current BasicBlock. + // If it cannot find the definition, or it's not sealed yet, it returns ValueInvalid. + FindValueInLinearPath(variable Variable) Value + + // Seal declares that we've known all the predecessors to this block and were added via AddPred. + // After calling this, AddPred will be forbidden. + Seal(blk BasicBlock) + + // AnnotateValue is for debugging purpose. + AnnotateValue(value Value, annotation string) + + // DeclareSignature appends the *Signature to be referenced by various instructions (e.g. OpcodeCall). + DeclareSignature(signature *Signature) + + // Signatures returns the slice of declared Signatures. + Signatures() []*Signature + + // ResolveSignature returns the Signature which corresponds to SignatureID. + ResolveSignature(id SignatureID) *Signature + + // RunPasses runs various passes on the constructed SSA function. + RunPasses() + + // Format returns the debugging string of the SSA function. + Format() string + + // BlockIteratorBegin initializes the state to iterate over all the valid BasicBlock(s) compiled. + // Combined with BlockIteratorNext, we can use this like: + // + // for blk := builder.BlockIteratorBegin(); blk != nil; blk = builder.BlockIteratorNext() { + // // ... + // } + // + // The returned blocks are ordered in the order of AllocateBasicBlock being called. + BlockIteratorBegin() BasicBlock + + // BlockIteratorNext advances the state for iteration initialized by BlockIteratorBegin. + // Returns nil if there's no unseen BasicBlock. + BlockIteratorNext() BasicBlock + + // ValueRefCounts returns the map of ValueID to its reference count. + // The returned slice must not be modified. + ValueRefCounts() []int + + // BlockIteratorReversePostOrderBegin is almost the same as BlockIteratorBegin except it returns the BasicBlock in the reverse post-order. + // This is available after RunPasses is run. + BlockIteratorReversePostOrderBegin() BasicBlock + + // BlockIteratorReversePostOrderNext is almost the same as BlockIteratorPostOrderNext except it returns the BasicBlock in the reverse post-order. + // This is available after RunPasses is run. + BlockIteratorReversePostOrderNext() BasicBlock + + // ReturnBlock returns the BasicBlock which is used to return from the function. + ReturnBlock() BasicBlock + + // InsertUndefined inserts an undefined instruction at the current position. + InsertUndefined() + + // SetCurrentSourceOffset sets the current source offset. The incoming instruction will be annotated with this offset. + SetCurrentSourceOffset(line SourceOffset) + + // LoopNestingForestRoots returns the roots of the loop nesting forest. + LoopNestingForestRoots() []BasicBlock + + // LowestCommonAncestor returns the lowest common ancestor in the dominator tree of the given BasicBlock(s). + LowestCommonAncestor(blk1, blk2 BasicBlock) BasicBlock + + // Idom returns the immediate dominator of the given BasicBlock. + Idom(blk BasicBlock) BasicBlock + + VarLengthPool() *wazevoapi.VarLengthPool[Value] +} + +// NewBuilder returns a new Builder implementation. +func NewBuilder() Builder { + return &builder{ + instructionsPool: wazevoapi.NewPool[Instruction](resetInstruction), + basicBlocksPool: wazevoapi.NewPool[basicBlock](resetBasicBlock), + varLengthPool: wazevoapi.NewVarLengthPool[Value](), + valueAnnotations: make(map[ValueID]string), + signatures: make(map[SignatureID]*Signature), + blkVisited: make(map[*basicBlock]int), + valueIDAliases: make(map[ValueID]Value), + redundantParameterIndexToValue: make(map[int]Value), + returnBlk: &basicBlock{id: basicBlockIDReturnBlock}, + } +} + +// builder implements Builder interface. +type builder struct { + basicBlocksPool wazevoapi.Pool[basicBlock] + instructionsPool wazevoapi.Pool[Instruction] + varLengthPool wazevoapi.VarLengthPool[Value] + signatures map[SignatureID]*Signature + currentSignature *Signature + + // reversePostOrderedBasicBlocks are the BasicBlock(s) ordered in the reverse post-order after passCalculateImmediateDominators. + reversePostOrderedBasicBlocks []*basicBlock + currentBB *basicBlock + returnBlk *basicBlock + + // variables track the types for Variable with the index regarded Variable. + variables []Type + // nextValueID is used by builder.AllocateValue. + nextValueID ValueID + // nextVariable is used by builder.AllocateVariable. + nextVariable Variable + + valueIDAliases map[ValueID]Value + valueAnnotations map[ValueID]string + + // valueRefCounts is used to lower the SSA in backend, and will be calculated + // by the last SSA-level optimization pass. + valueRefCounts []int + + // dominators stores the immediate dominator of each BasicBlock. + // The index is blockID of the BasicBlock. + dominators []*basicBlock + sparseTree dominatorSparseTree + + // loopNestingForestRoots are the roots of the loop nesting forest. + loopNestingForestRoots []BasicBlock + + // The followings are used for optimization passes/deterministic compilation. + instStack []*Instruction + blkVisited map[*basicBlock]int + valueIDToInstruction []*Instruction + blkStack []*basicBlock + blkStack2 []*basicBlock + ints []int + redundantParameterIndexToValue map[int]Value + + // blockIterCur is used to implement blockIteratorBegin and blockIteratorNext. + blockIterCur int + + // donePreBlockLayoutPasses is true if all the passes before LayoutBlocks are called. + donePreBlockLayoutPasses bool + // doneBlockLayout is true if LayoutBlocks is called. + doneBlockLayout bool + // donePostBlockLayoutPasses is true if all the passes after LayoutBlocks are called. + donePostBlockLayoutPasses bool + + currentSourceOffset SourceOffset +} + +func (b *builder) VarLengthPool() *wazevoapi.VarLengthPool[Value] { + return &b.varLengthPool +} + +// ReturnBlock implements Builder.ReturnBlock. +func (b *builder) ReturnBlock() BasicBlock { + return b.returnBlk +} + +// Init implements Builder.Reset. +func (b *builder) Init(s *Signature) { + b.nextVariable = 0 + b.currentSignature = s + resetBasicBlock(b.returnBlk) + b.instructionsPool.Reset() + b.basicBlocksPool.Reset() + b.varLengthPool.Reset() + b.donePreBlockLayoutPasses = false + b.doneBlockLayout = false + b.donePostBlockLayoutPasses = false + for _, sig := range b.signatures { + sig.used = false + } + + b.ints = b.ints[:0] + b.blkStack = b.blkStack[:0] + b.blkStack2 = b.blkStack2[:0] + b.dominators = b.dominators[:0] + b.loopNestingForestRoots = b.loopNestingForestRoots[:0] + + for i := 0; i < b.basicBlocksPool.Allocated(); i++ { + blk := b.basicBlocksPool.View(i) + delete(b.blkVisited, blk) + } + b.basicBlocksPool.Reset() + + for v := ValueID(0); v < b.nextValueID; v++ { + delete(b.valueAnnotations, v) + delete(b.valueIDAliases, v) + b.valueRefCounts[v] = 0 + b.valueIDToInstruction[v] = nil + } + b.nextValueID = 0 + b.reversePostOrderedBasicBlocks = b.reversePostOrderedBasicBlocks[:0] + b.doneBlockLayout = false + for i := range b.valueRefCounts { + b.valueRefCounts[i] = 0 + } + + b.currentSourceOffset = sourceOffsetUnknown +} + +// Signature implements Builder.Signature. +func (b *builder) Signature() *Signature { + return b.currentSignature +} + +// AnnotateValue implements Builder.AnnotateValue. +func (b *builder) AnnotateValue(value Value, a string) { + b.valueAnnotations[value.ID()] = a +} + +// AllocateInstruction implements Builder.AllocateInstruction. +func (b *builder) AllocateInstruction() *Instruction { + instr := b.instructionsPool.Allocate() + instr.id = b.instructionsPool.Allocated() + return instr +} + +// DeclareSignature implements Builder.AnnotateValue. +func (b *builder) DeclareSignature(s *Signature) { + b.signatures[s.ID] = s + s.used = false +} + +// Signatures implements Builder.Signatures. +func (b *builder) Signatures() (ret []*Signature) { + for _, sig := range b.signatures { + ret = append(ret, sig) + } + sort.Slice(ret, func(i, j int) bool { + return ret[i].ID < ret[j].ID + }) + return +} + +// SetCurrentSourceOffset implements Builder.SetCurrentSourceOffset. +func (b *builder) SetCurrentSourceOffset(l SourceOffset) { + b.currentSourceOffset = l +} + +func (b *builder) usedSignatures() (ret []*Signature) { + for _, sig := range b.signatures { + if sig.used { + ret = append(ret, sig) + } + } + sort.Slice(ret, func(i, j int) bool { + return ret[i].ID < ret[j].ID + }) + return +} + +// ResolveSignature implements Builder.ResolveSignature. +func (b *builder) ResolveSignature(id SignatureID) *Signature { + return b.signatures[id] +} + +// AllocateBasicBlock implements Builder.AllocateBasicBlock. +func (b *builder) AllocateBasicBlock() BasicBlock { + return b.allocateBasicBlock() +} + +// allocateBasicBlock allocates a new basicBlock. +func (b *builder) allocateBasicBlock() *basicBlock { + id := BasicBlockID(b.basicBlocksPool.Allocated()) + blk := b.basicBlocksPool.Allocate() + blk.id = id + return blk +} + +// Idom implements Builder.Idom. +func (b *builder) Idom(blk BasicBlock) BasicBlock { + return b.dominators[blk.ID()] +} + +// InsertInstruction implements Builder.InsertInstruction. +func (b *builder) InsertInstruction(instr *Instruction) { + b.currentBB.InsertInstruction(instr) + + if l := b.currentSourceOffset; l.Valid() { + // Emit the source offset info only when the instruction has side effect because + // these are the only instructions that are accessed by stack unwinding. + // This reduces the significant amount of the offset info in the binary. + if instr.sideEffect() != sideEffectNone { + instr.annotateSourceOffset(l) + } + } + + resultTypesFn := instructionReturnTypes[instr.opcode] + if resultTypesFn == nil { + panic("TODO: " + instr.Format(b)) + } + + t1, ts := resultTypesFn(b, instr) + if t1.invalid() { + return + } + + r1 := b.allocateValue(t1) + instr.rValue = r1 + + tsl := len(ts) + if tsl == 0 { + return + } + + rValues := b.varLengthPool.Allocate(tsl) + for i := 0; i < tsl; i++ { + rValues = rValues.Append(&b.varLengthPool, b.allocateValue(ts[i])) + } + instr.rValues = rValues +} + +// DefineVariable implements Builder.DefineVariable. +func (b *builder) DefineVariable(variable Variable, value Value, block BasicBlock) { + if b.variables[variable].invalid() { + panic("BUG: trying to define variable " + variable.String() + " but is not declared yet") + } + + if b.variables[variable] != value.Type() { + panic(fmt.Sprintf("BUG: inconsistent type for variable %d: expected %s but got %s", variable, b.variables[variable], value.Type())) + } + bb := block.(*basicBlock) + bb.lastDefinitions[variable] = value +} + +// DefineVariableInCurrentBB implements Builder.DefineVariableInCurrentBB. +func (b *builder) DefineVariableInCurrentBB(variable Variable, value Value) { + b.DefineVariable(variable, value, b.currentBB) +} + +// SetCurrentBlock implements Builder.SetCurrentBlock. +func (b *builder) SetCurrentBlock(bb BasicBlock) { + b.currentBB = bb.(*basicBlock) +} + +// CurrentBlock implements Builder.CurrentBlock. +func (b *builder) CurrentBlock() BasicBlock { + return b.currentBB +} + +// EntryBlock implements Builder.EntryBlock. +func (b *builder) EntryBlock() BasicBlock { + return b.entryBlk() +} + +// DeclareVariable implements Builder.DeclareVariable. +func (b *builder) DeclareVariable(typ Type) Variable { + v := b.allocateVariable() + iv := int(v) + if l := len(b.variables); l <= iv { + b.variables = append(b.variables, make([]Type, 2*(l+1))...) + } + b.variables[v] = typ + return v +} + +// allocateVariable allocates a new variable. +func (b *builder) allocateVariable() (ret Variable) { + ret = b.nextVariable + b.nextVariable++ + return +} + +// allocateValue implements Builder.AllocateValue. +func (b *builder) allocateValue(typ Type) (v Value) { + v = Value(b.nextValueID) + v = v.setType(typ) + b.nextValueID++ + return +} + +// FindValueInLinearPath implements Builder.FindValueInLinearPath. +func (b *builder) FindValueInLinearPath(variable Variable) Value { + return b.findValueInLinearPath(variable, b.currentBB) +} + +func (b *builder) findValueInLinearPath(variable Variable, blk *basicBlock) Value { + if val, ok := blk.lastDefinitions[variable]; ok { + return val + } else if !blk.sealed { + return ValueInvalid + } + + if pred := blk.singlePred; pred != nil { + // If this block is sealed and have only one predecessor, + // we can use the value in that block without ambiguity on definition. + return b.findValueInLinearPath(variable, pred) + } + if len(blk.preds) == 1 { + panic("BUG") + } + return ValueInvalid +} + +func (b *builder) MustFindValueInBlk(variable Variable, blk BasicBlock) Value { + typ := b.definedVariableType(variable) + return b.findValue(typ, variable, blk.(*basicBlock)) +} + +// MustFindValue implements Builder.MustFindValue. +func (b *builder) MustFindValue(variable Variable) Value { + typ := b.definedVariableType(variable) + return b.findValue(typ, variable, b.currentBB) +} + +// findValue recursively tries to find the latest definition of a `variable`. The algorithm is described in +// the section 2 of the paper https://link.springer.com/content/pdf/10.1007/978-3-642-37051-9_6.pdf. +// +// TODO: reimplement this in iterative, not recursive, to avoid stack overflow. +func (b *builder) findValue(typ Type, variable Variable, blk *basicBlock) Value { + if val, ok := blk.lastDefinitions[variable]; ok { + // The value is already defined in this block! + return val + } else if !blk.sealed { // Incomplete CFG as in the paper. + // If this is not sealed, that means it might have additional unknown predecessor later on. + // So we temporarily define the placeholder value here (not add as a parameter yet!), + // and record it as unknown. + // The unknown values are resolved when we call seal this block via BasicBlock.Seal(). + value := b.allocateValue(typ) + if wazevoapi.SSALoggingEnabled { + fmt.Printf("adding unknown value placeholder for %s at %d\n", variable, blk.id) + } + blk.lastDefinitions[variable] = value + blk.unknownValues = append(blk.unknownValues, unknownValue{ + variable: variable, + value: value, + }) + return value + } + + if pred := blk.singlePred; pred != nil { + // If this block is sealed and have only one predecessor, + // we can use the value in that block without ambiguity on definition. + return b.findValue(typ, variable, pred) + } else if len(blk.preds) == 0 { + panic("BUG: value is not defined for " + variable.String()) + } + + // If this block has multiple predecessors, we have to gather the definitions, + // and treat them as an argument to this block. + // + // The first thing is to define a new parameter to this block which may or may not be redundant, but + // later we eliminate trivial params in an optimization pass. This must be done before finding the + // definitions in the predecessors so that we can break the cycle. + paramValue := blk.AddParam(b, typ) + b.DefineVariable(variable, paramValue, blk) + + // After the new param is added, we have to manipulate the original branching instructions + // in predecessors so that they would pass the definition of `variable` as the argument to + // the newly added PHI. + for i := range blk.preds { + pred := &blk.preds[i] + value := b.findValue(typ, variable, pred.blk) + pred.branch.addArgumentBranchInst(b, value) + } + return paramValue +} + +// Seal implements Builder.Seal. +func (b *builder) Seal(raw BasicBlock) { + blk := raw.(*basicBlock) + if len(blk.preds) == 1 { + blk.singlePred = blk.preds[0].blk + } + blk.sealed = true + + for _, v := range blk.unknownValues { + variable, phiValue := v.variable, v.value + typ := b.definedVariableType(variable) + blk.addParamOn(typ, phiValue) + for i := range blk.preds { + pred := &blk.preds[i] + predValue := b.findValue(typ, variable, pred.blk) + if !predValue.Valid() { + panic("BUG: value is not defined anywhere in the predecessors in the CFG") + } + pred.branch.addArgumentBranchInst(b, predValue) + } + } +} + +// definedVariableType returns the type of the given variable. If the variable is not defined yet, it panics. +func (b *builder) definedVariableType(variable Variable) Type { + typ := b.variables[variable] + if typ.invalid() { + panic(fmt.Sprintf("%s is not defined yet", variable)) + } + return typ +} + +// Format implements Builder.Format. +func (b *builder) Format() string { + str := strings.Builder{} + usedSigs := b.usedSignatures() + if len(usedSigs) > 0 { + str.WriteByte('\n') + str.WriteString("signatures:\n") + for _, sig := range usedSigs { + str.WriteByte('\t') + str.WriteString(sig.String()) + str.WriteByte('\n') + } + } + + var iterBegin, iterNext func() *basicBlock + if b.doneBlockLayout { + iterBegin, iterNext = b.blockIteratorReversePostOrderBegin, b.blockIteratorReversePostOrderNext + } else { + iterBegin, iterNext = b.blockIteratorBegin, b.blockIteratorNext + } + for bb := iterBegin(); bb != nil; bb = iterNext() { + str.WriteByte('\n') + str.WriteString(bb.FormatHeader(b)) + str.WriteByte('\n') + + for cur := bb.Root(); cur != nil; cur = cur.Next() { + str.WriteByte('\t') + str.WriteString(cur.Format(b)) + str.WriteByte('\n') + } + } + return str.String() +} + +// BlockIteratorNext implements Builder.BlockIteratorNext. +func (b *builder) BlockIteratorNext() BasicBlock { + if blk := b.blockIteratorNext(); blk == nil { + return nil // BasicBlock((*basicBlock)(nil)) != BasicBlock(nil) + } else { + return blk + } +} + +// BlockIteratorNext implements Builder.BlockIteratorNext. +func (b *builder) blockIteratorNext() *basicBlock { + index := b.blockIterCur + for { + if index == b.basicBlocksPool.Allocated() { + return nil + } + ret := b.basicBlocksPool.View(index) + index++ + if !ret.invalid { + b.blockIterCur = index + return ret + } + } +} + +// BlockIteratorBegin implements Builder.BlockIteratorBegin. +func (b *builder) BlockIteratorBegin() BasicBlock { + return b.blockIteratorBegin() +} + +// BlockIteratorBegin implements Builder.BlockIteratorBegin. +func (b *builder) blockIteratorBegin() *basicBlock { + b.blockIterCur = 0 + return b.blockIteratorNext() +} + +// BlockIteratorReversePostOrderBegin implements Builder.BlockIteratorReversePostOrderBegin. +func (b *builder) BlockIteratorReversePostOrderBegin() BasicBlock { + return b.blockIteratorReversePostOrderBegin() +} + +// BlockIteratorBegin implements Builder.BlockIteratorBegin. +func (b *builder) blockIteratorReversePostOrderBegin() *basicBlock { + b.blockIterCur = 0 + return b.blockIteratorReversePostOrderNext() +} + +// BlockIteratorReversePostOrderNext implements Builder.BlockIteratorReversePostOrderNext. +func (b *builder) BlockIteratorReversePostOrderNext() BasicBlock { + if blk := b.blockIteratorReversePostOrderNext(); blk == nil { + return nil // BasicBlock((*basicBlock)(nil)) != BasicBlock(nil) + } else { + return blk + } +} + +// BlockIteratorNext implements Builder.BlockIteratorNext. +func (b *builder) blockIteratorReversePostOrderNext() *basicBlock { + if b.blockIterCur >= len(b.reversePostOrderedBasicBlocks) { + return nil + } else { + ret := b.reversePostOrderedBasicBlocks[b.blockIterCur] + b.blockIterCur++ + return ret + } +} + +// ValueRefCounts implements Builder.ValueRefCounts. +func (b *builder) ValueRefCounts() []int { + return b.valueRefCounts +} + +// alias records the alias of the given values. The alias(es) will be +// eliminated in the optimization pass via resolveArgumentAlias. +func (b *builder) alias(dst, src Value) { + b.valueIDAliases[dst.ID()] = src +} + +// resolveArgumentAlias resolves the alias of the arguments of the given instruction. +func (b *builder) resolveArgumentAlias(instr *Instruction) { + if instr.v.Valid() { + instr.v = b.resolveAlias(instr.v) + } + + if instr.v2.Valid() { + instr.v2 = b.resolveAlias(instr.v2) + } + + if instr.v3.Valid() { + instr.v3 = b.resolveAlias(instr.v3) + } + + view := instr.vs.View() + for i, v := range view { + view[i] = b.resolveAlias(v) + } +} + +// resolveAlias resolves the alias of the given value. +func (b *builder) resolveAlias(v Value) Value { + // Some aliases are chained, so we need to resolve them recursively. + for { + if src, ok := b.valueIDAliases[v.ID()]; ok { + v = src + } else { + break + } + } + return v +} + +// entryBlk returns the entry block of the function. +func (b *builder) entryBlk() *basicBlock { + return b.basicBlocksPool.View(0) +} + +// isDominatedBy returns true if the given block `n` is dominated by the given block `d`. +// Before calling this, the builder must pass by passCalculateImmediateDominators. +func (b *builder) isDominatedBy(n *basicBlock, d *basicBlock) bool { + if len(b.dominators) == 0 { + panic("BUG: passCalculateImmediateDominators must be called before calling isDominatedBy") + } + ent := b.entryBlk() + doms := b.dominators + for n != d && n != ent { + n = doms[n.id] + } + return n == d +} + +// BlockIDMax implements Builder.BlockIDMax. +func (b *builder) BlockIDMax() BasicBlockID { + return BasicBlockID(b.basicBlocksPool.Allocated()) +} + +// InsertUndefined implements Builder.InsertUndefined. +func (b *builder) InsertUndefined() { + instr := b.AllocateInstruction() + instr.opcode = OpcodeUndefined + b.InsertInstruction(instr) +} + +// LoopNestingForestRoots implements Builder.LoopNestingForestRoots. +func (b *builder) LoopNestingForestRoots() []BasicBlock { + return b.loopNestingForestRoots +} + +// LowestCommonAncestor implements Builder.LowestCommonAncestor. +func (b *builder) LowestCommonAncestor(blk1, blk2 BasicBlock) BasicBlock { + return b.sparseTree.findLCA(blk1.ID(), blk2.ID()) +} diff --git a/vendor/github.com/tetratelabs/wazero/internal/engine/wazevo/ssa/cmp.go b/vendor/github.com/tetratelabs/wazero/internal/engine/wazevo/ssa/cmp.go new file mode 100644 index 000000000..15b62ca8e --- /dev/null +++ b/vendor/github.com/tetratelabs/wazero/internal/engine/wazevo/ssa/cmp.go @@ -0,0 +1,107 @@ +package ssa + +// IntegerCmpCond represents a condition for integer comparison. +type IntegerCmpCond byte + +const ( + // IntegerCmpCondInvalid represents an invalid condition. + IntegerCmpCondInvalid IntegerCmpCond = iota + // IntegerCmpCondEqual represents "==". + IntegerCmpCondEqual + // IntegerCmpCondNotEqual represents "!=". + IntegerCmpCondNotEqual + // IntegerCmpCondSignedLessThan represents Signed "<". + IntegerCmpCondSignedLessThan + // IntegerCmpCondSignedGreaterThanOrEqual represents Signed ">=". + IntegerCmpCondSignedGreaterThanOrEqual + // IntegerCmpCondSignedGreaterThan represents Signed ">". + IntegerCmpCondSignedGreaterThan + // IntegerCmpCondSignedLessThanOrEqual represents Signed "<=". + IntegerCmpCondSignedLessThanOrEqual + // IntegerCmpCondUnsignedLessThan represents Unsigned "<". + IntegerCmpCondUnsignedLessThan + // IntegerCmpCondUnsignedGreaterThanOrEqual represents Unsigned ">=". + IntegerCmpCondUnsignedGreaterThanOrEqual + // IntegerCmpCondUnsignedGreaterThan represents Unsigned ">". + IntegerCmpCondUnsignedGreaterThan + // IntegerCmpCondUnsignedLessThanOrEqual represents Unsigned "<=". + IntegerCmpCondUnsignedLessThanOrEqual +) + +// String implements fmt.Stringer. +func (i IntegerCmpCond) String() string { + switch i { + case IntegerCmpCondEqual: + return "eq" + case IntegerCmpCondNotEqual: + return "neq" + case IntegerCmpCondSignedLessThan: + return "lt_s" + case IntegerCmpCondSignedGreaterThanOrEqual: + return "ge_s" + case IntegerCmpCondSignedGreaterThan: + return "gt_s" + case IntegerCmpCondSignedLessThanOrEqual: + return "le_s" + case IntegerCmpCondUnsignedLessThan: + return "lt_u" + case IntegerCmpCondUnsignedGreaterThanOrEqual: + return "ge_u" + case IntegerCmpCondUnsignedGreaterThan: + return "gt_u" + case IntegerCmpCondUnsignedLessThanOrEqual: + return "le_u" + default: + panic("invalid integer comparison condition") + } +} + +// Signed returns true if the condition is signed integer comparison. +func (i IntegerCmpCond) Signed() bool { + switch i { + case IntegerCmpCondSignedLessThan, IntegerCmpCondSignedGreaterThanOrEqual, + IntegerCmpCondSignedGreaterThan, IntegerCmpCondSignedLessThanOrEqual: + return true + default: + return false + } +} + +type FloatCmpCond byte + +const ( + // FloatCmpCondInvalid represents an invalid condition. + FloatCmpCondInvalid FloatCmpCond = iota + // FloatCmpCondEqual represents "==". + FloatCmpCondEqual + // FloatCmpCondNotEqual represents "!=". + FloatCmpCondNotEqual + // FloatCmpCondLessThan represents "<". + FloatCmpCondLessThan + // FloatCmpCondLessThanOrEqual represents "<=". + FloatCmpCondLessThanOrEqual + // FloatCmpCondGreaterThan represents ">". + FloatCmpCondGreaterThan + // FloatCmpCondGreaterThanOrEqual represents ">=". + FloatCmpCondGreaterThanOrEqual +) + +// String implements fmt.Stringer. +func (f FloatCmpCond) String() string { + switch f { + case FloatCmpCondEqual: + return "eq" + case FloatCmpCondNotEqual: + return "neq" + case FloatCmpCondLessThan: + return "lt" + case FloatCmpCondLessThanOrEqual: + return "le" + case FloatCmpCondGreaterThan: + return "gt" + case FloatCmpCondGreaterThanOrEqual: + return "ge" + default: + panic("invalid float comparison condition") + } +} diff --git a/vendor/github.com/tetratelabs/wazero/internal/engine/wazevo/ssa/funcref.go b/vendor/github.com/tetratelabs/wazero/internal/engine/wazevo/ssa/funcref.go new file mode 100644 index 000000000..d9620762a --- /dev/null +++ b/vendor/github.com/tetratelabs/wazero/internal/engine/wazevo/ssa/funcref.go @@ -0,0 +1,12 @@ +package ssa + +import "fmt" + +// FuncRef is a unique identifier for a function of the frontend, +// and is used to reference the function in function call. +type FuncRef uint32 + +// String implements fmt.Stringer. +func (r FuncRef) String() string { + return fmt.Sprintf("f%d", r) +} diff --git a/vendor/github.com/tetratelabs/wazero/internal/engine/wazevo/ssa/instructions.go b/vendor/github.com/tetratelabs/wazero/internal/engine/wazevo/ssa/instructions.go new file mode 100644 index 000000000..3e3482efc --- /dev/null +++ b/vendor/github.com/tetratelabs/wazero/internal/engine/wazevo/ssa/instructions.go @@ -0,0 +1,2967 @@ +package ssa + +import ( + "fmt" + "math" + "strings" + + "github.com/tetratelabs/wazero/internal/engine/wazevo/wazevoapi" +) + +// Opcode represents a SSA instruction. +type Opcode uint32 + +// Instruction represents an instruction whose opcode is specified by +// Opcode. Since Go doesn't have union type, we use this flattened type +// for all instructions, and therefore each field has different meaning +// depending on Opcode. +type Instruction struct { + // id is the unique ID of this instruction which ascends from 0 following the order of program. + id int + opcode Opcode + u1, u2 uint64 + v Value + v2 Value + v3 Value + vs Values + typ Type + blk BasicBlock + targets []BasicBlock + prev, next *Instruction + + rValue Value + rValues Values + gid InstructionGroupID + sourceOffset SourceOffset + live bool + alreadyLowered bool +} + +// SourceOffset represents the offset of the source of an instruction. +type SourceOffset int64 + +const sourceOffsetUnknown = -1 + +// Valid returns true if this source offset is valid. +func (l SourceOffset) Valid() bool { + return l != sourceOffsetUnknown +} + +func (i *Instruction) annotateSourceOffset(line SourceOffset) { + i.sourceOffset = line +} + +// SourceOffset returns the source offset of this instruction. +func (i *Instruction) SourceOffset() SourceOffset { + return i.sourceOffset +} + +// Opcode returns the opcode of this instruction. +func (i *Instruction) Opcode() Opcode { + return i.opcode +} + +// GroupID returns the InstructionGroupID of this instruction. +func (i *Instruction) GroupID() InstructionGroupID { + return i.gid +} + +// MarkLowered marks this instruction as already lowered. +func (i *Instruction) MarkLowered() { + i.alreadyLowered = true +} + +// Lowered returns true if this instruction is already lowered. +func (i *Instruction) Lowered() bool { + return i.alreadyLowered +} + +// resetInstruction resets this instruction to the initial state. +func resetInstruction(i *Instruction) { + *i = Instruction{} + i.v = ValueInvalid + i.v2 = ValueInvalid + i.v3 = ValueInvalid + i.rValue = ValueInvalid + i.typ = typeInvalid + i.vs = ValuesNil + i.sourceOffset = sourceOffsetUnknown +} + +// InstructionGroupID is assigned to each instruction and represents a group of instructions +// where each instruction is interchangeable with others except for the last instruction +// in the group which has side effects. In short, InstructionGroupID is determined by the side effects of instructions. +// That means, if there's an instruction with side effect between two instructions, then these two instructions +// will have different instructionGroupID. Note that each block always ends with branching, which is with side effects, +// therefore, instructions in different blocks always have different InstructionGroupID(s). +// +// The notable application of this is used in lowering SSA-level instruction to a ISA specific instruction, +// where we eagerly try to merge multiple instructions into single operation etc. Such merging cannot be done +// if these instruction have different InstructionGroupID since it will change the semantics of a program. +// +// See passDeadCodeElimination. +type InstructionGroupID uint32 + +// Returns Value(s) produced by this instruction if any. +// The `first` is the first return value, and `rest` is the rest of the values. +func (i *Instruction) Returns() (first Value, rest []Value) { + return i.rValue, i.rValues.View() +} + +// Return returns a Value(s) produced by this instruction if any. +// If there's multiple return values, only the first one is returned. +func (i *Instruction) Return() (first Value) { + return i.rValue +} + +// Args returns the arguments to this instruction. +func (i *Instruction) Args() (v1, v2, v3 Value, vs []Value) { + return i.v, i.v2, i.v3, i.vs.View() +} + +// Arg returns the first argument to this instruction. +func (i *Instruction) Arg() Value { + return i.v +} + +// Arg2 returns the first two arguments to this instruction. +func (i *Instruction) Arg2() (Value, Value) { + return i.v, i.v2 +} + +// ArgWithLane returns the first argument to this instruction, and the lane type. +func (i *Instruction) ArgWithLane() (Value, VecLane) { + return i.v, VecLane(i.u1) +} + +// Arg2WithLane returns the first two arguments to this instruction, and the lane type. +func (i *Instruction) Arg2WithLane() (Value, Value, VecLane) { + return i.v, i.v2, VecLane(i.u1) +} + +// ShuffleData returns the first two arguments to this instruction and 2 uint64s `lo`, `hi`. +// +// Note: Each uint64 encodes a sequence of 8 bytes where each byte encodes a VecLane, +// so that the 128bit integer `hi<<64|lo` packs a slice `[16]VecLane`, +// where `lane[0]` is the least significant byte, and `lane[n]` is shifted to offset `n*8`. +func (i *Instruction) ShuffleData() (v Value, v2 Value, lo uint64, hi uint64) { + return i.v, i.v2, i.u1, i.u2 +} + +// Arg3 returns the first three arguments to this instruction. +func (i *Instruction) Arg3() (Value, Value, Value) { + return i.v, i.v2, i.v3 +} + +// Next returns the next instruction laid out next to itself. +func (i *Instruction) Next() *Instruction { + return i.next +} + +// Prev returns the previous instruction laid out prior to itself. +func (i *Instruction) Prev() *Instruction { + return i.prev +} + +// IsBranching returns true if this instruction is a branching instruction. +func (i *Instruction) IsBranching() bool { + switch i.opcode { + case OpcodeJump, OpcodeBrz, OpcodeBrnz, OpcodeBrTable: + return true + default: + return false + } +} + +// TODO: complete opcode comments. +const ( + OpcodeInvalid Opcode = iota + + // OpcodeUndefined is a placeholder for undefined opcode. This can be used for debugging to intentionally + // cause a crash at certain point. + OpcodeUndefined + + // OpcodeJump takes the list of args to the `block` and unconditionally jumps to it. + OpcodeJump + + // OpcodeBrz branches into `blk` with `args` if the value `c` equals zero: `Brz c, blk, args`. + OpcodeBrz + + // OpcodeBrnz branches into `blk` with `args` if the value `c` is not zero: `Brnz c, blk, args`. + OpcodeBrnz + + // OpcodeBrTable takes the index value `index`, and branches into `labelX`. If the `index` is out of range, + // it branches into the last labelN: `BrTable index, [label1, label2, ... labelN]`. + OpcodeBrTable + + // OpcodeExitWithCode exit the execution immediately. + OpcodeExitWithCode + + // OpcodeExitIfTrueWithCode exits the execution immediately if the value `c` is not zero. + OpcodeExitIfTrueWithCode + + // OpcodeReturn returns from the function: `return rvalues`. + OpcodeReturn + + // OpcodeCall calls a function specified by the symbol FN with arguments `args`: `returnvals = Call FN, args...` + // This is a "near" call, which means the call target is known at compile time, and the target is relatively close + // to this function. If the target cannot be reached by near call, the backend fails to compile. + OpcodeCall + + // OpcodeCallIndirect calls a function specified by `callee` which is a function address: `returnvals = call_indirect SIG, callee, args`. + // Note that this is different from call_indirect in Wasm, which also does type checking, etc. + OpcodeCallIndirect + + // OpcodeSplat performs a vector splat operation: `v = Splat.lane x`. + OpcodeSplat + + // OpcodeSwizzle performs a vector swizzle operation: `v = Swizzle.lane x, y`. + OpcodeSwizzle + + // OpcodeInsertlane inserts a lane value into a vector: `v = InsertLane x, y, Idx`. + OpcodeInsertlane + + // OpcodeExtractlane extracts a lane value from a vector: `v = ExtractLane x, Idx`. + OpcodeExtractlane + + // OpcodeLoad loads a Type value from the [base + offset] address: `v = Load base, offset`. + OpcodeLoad + + // OpcodeStore stores a Type value to the [base + offset] address: `Store v, base, offset`. + OpcodeStore + + // OpcodeUload8 loads the 8-bit value from the [base + offset] address, zero-extended to 64 bits: `v = Uload8 base, offset`. + OpcodeUload8 + + // OpcodeSload8 loads the 8-bit value from the [base + offset] address, sign-extended to 64 bits: `v = Sload8 base, offset`. + OpcodeSload8 + + // OpcodeIstore8 stores the 8-bit value to the [base + offset] address, sign-extended to 64 bits: `Istore8 v, base, offset`. + OpcodeIstore8 + + // OpcodeUload16 loads the 16-bit value from the [base + offset] address, zero-extended to 64 bits: `v = Uload16 base, offset`. + OpcodeUload16 + + // OpcodeSload16 loads the 16-bit value from the [base + offset] address, sign-extended to 64 bits: `v = Sload16 base, offset`. + OpcodeSload16 + + // OpcodeIstore16 stores the 16-bit value to the [base + offset] address, zero-extended to 64 bits: `Istore16 v, base, offset`. + OpcodeIstore16 + + // OpcodeUload32 loads the 32-bit value from the [base + offset] address, zero-extended to 64 bits: `v = Uload32 base, offset`. + OpcodeUload32 + + // OpcodeSload32 loads the 32-bit value from the [base + offset] address, sign-extended to 64 bits: `v = Sload32 base, offset`. + OpcodeSload32 + + // OpcodeIstore32 stores the 32-bit value to the [base + offset] address, zero-extended to 64 bits: `Istore16 v, base, offset`. + OpcodeIstore32 + + // OpcodeLoadSplat represents a load that replicates the loaded value to all lanes `v = LoadSplat.lane p, Offset`. + OpcodeLoadSplat + + // OpcodeVZeroExtLoad loads a scalar single/double precision floating point value from the [p + Offset] address, + // and zero-extend it to the V128 value: `v = VExtLoad p, Offset`. + OpcodeVZeroExtLoad + + // OpcodeIconst represents the integer const. + OpcodeIconst + + // OpcodeF32const represents the single-precision const. + OpcodeF32const + + // OpcodeF64const represents the double-precision const. + OpcodeF64const + + // OpcodeVconst represents the 128bit vector const. + OpcodeVconst + + // OpcodeVbor computes binary or between two 128bit vectors: `v = bor x, y`. + OpcodeVbor + + // OpcodeVbxor computes binary xor between two 128bit vectors: `v = bxor x, y`. + OpcodeVbxor + + // OpcodeVband computes binary and between two 128bit vectors: `v = band x, y`. + OpcodeVband + + // OpcodeVbandnot computes binary and-not between two 128bit vectors: `v = bandnot x, y`. + OpcodeVbandnot + + // OpcodeVbnot negates a 128bit vector: `v = bnot x`. + OpcodeVbnot + + // OpcodeVbitselect uses the bits in the control mask c to select the corresponding bit from x when 1 + // and y when 0: `v = bitselect c, x, y`. + OpcodeVbitselect + + // OpcodeShuffle shuffles two vectors using the given 128-bit immediate: `v = shuffle imm, x, y`. + // For each byte in the immediate, a value i in [0, 15] selects the i-th byte in vector x; + // i in [16, 31] selects the (i-16)-th byte in vector y. + OpcodeShuffle + + // OpcodeSelect chooses between two values based on a condition `c`: `v = Select c, x, y`. + OpcodeSelect + + // OpcodeVanyTrue performs a any true operation: `s = VanyTrue a`. + OpcodeVanyTrue + + // OpcodeVallTrue performs a lane-wise all true operation: `s = VallTrue.lane a`. + OpcodeVallTrue + + // OpcodeVhighBits performs a lane-wise extract of the high bits: `v = VhighBits.lane a`. + OpcodeVhighBits + + // OpcodeIcmp compares two integer values with the given condition: `v = icmp Cond, x, y`. + OpcodeIcmp + + // OpcodeVIcmp compares two integer values with the given condition: `v = vicmp Cond, x, y` on vector. + OpcodeVIcmp + + // OpcodeIcmpImm compares an integer value with the immediate value on the given condition: `v = icmp_imm Cond, x, Y`. + OpcodeIcmpImm + + // OpcodeIadd performs an integer addition: `v = Iadd x, y`. + OpcodeIadd + + // OpcodeVIadd performs an integer addition: `v = VIadd.lane x, y` on vector. + OpcodeVIadd + + // OpcodeVSaddSat performs a signed saturating vector addition: `v = VSaddSat.lane x, y` on vector. + OpcodeVSaddSat + + // OpcodeVUaddSat performs an unsigned saturating vector addition: `v = VUaddSat.lane x, y` on vector. + OpcodeVUaddSat + + // OpcodeIsub performs an integer subtraction: `v = Isub x, y`. + OpcodeIsub + + // OpcodeVIsub performs an integer subtraction: `v = VIsub.lane x, y` on vector. + OpcodeVIsub + + // OpcodeVSsubSat performs a signed saturating vector subtraction: `v = VSsubSat.lane x, y` on vector. + OpcodeVSsubSat + + // OpcodeVUsubSat performs an unsigned saturating vector subtraction: `v = VUsubSat.lane x, y` on vector. + OpcodeVUsubSat + + // OpcodeVImin performs a signed integer min: `v = VImin.lane x, y` on vector. + OpcodeVImin + + // OpcodeVUmin performs an unsigned integer min: `v = VUmin.lane x, y` on vector. + OpcodeVUmin + + // OpcodeVImax performs a signed integer max: `v = VImax.lane x, y` on vector. + OpcodeVImax + + // OpcodeVUmax performs an unsigned integer max: `v = VUmax.lane x, y` on vector. + OpcodeVUmax + + // OpcodeVAvgRound performs an unsigned integer avg, truncating to zero: `v = VAvgRound.lane x, y` on vector. + OpcodeVAvgRound + + // OpcodeVImul performs an integer multiplication: `v = VImul.lane x, y` on vector. + OpcodeVImul + + // OpcodeVIneg negates the given integer vector value: `v = VIneg x`. + OpcodeVIneg + + // OpcodeVIpopcnt counts the number of 1-bits in the given vector: `v = VIpopcnt x`. + OpcodeVIpopcnt + + // OpcodeVIabs returns the absolute value for the given vector value: `v = VIabs.lane x`. + OpcodeVIabs + + // OpcodeVIshl shifts x left by (y mod lane-width): `v = VIshl.lane x, y` on vector. + OpcodeVIshl + + // OpcodeVUshr shifts x right by (y mod lane-width), unsigned: `v = VUshr.lane x, y` on vector. + OpcodeVUshr + + // OpcodeVSshr shifts x right by (y mod lane-width), signed: `v = VSshr.lane x, y` on vector. + OpcodeVSshr + + // OpcodeVFabs takes the absolute value of a floating point value: `v = VFabs.lane x on vector. + OpcodeVFabs + + // OpcodeVFmax takes the maximum of two floating point values: `v = VFmax.lane x, y on vector. + OpcodeVFmax + + // OpcodeVFmin takes the minimum of two floating point values: `v = VFmin.lane x, y on vector. + OpcodeVFmin + + // OpcodeVFneg negates the given floating point vector value: `v = VFneg x`. + OpcodeVFneg + + // OpcodeVFadd performs a floating point addition: `v = VFadd.lane x, y` on vector. + OpcodeVFadd + + // OpcodeVFsub performs a floating point subtraction: `v = VFsub.lane x, y` on vector. + OpcodeVFsub + + // OpcodeVFmul performs a floating point multiplication: `v = VFmul.lane x, y` on vector. + OpcodeVFmul + + // OpcodeVFdiv performs a floating point division: `v = VFdiv.lane x, y` on vector. + OpcodeVFdiv + + // OpcodeVFcmp compares two float values with the given condition: `v = VFcmp.lane Cond, x, y` on float. + OpcodeVFcmp + + // OpcodeVCeil takes the ceiling of the given floating point value: `v = ceil.lane x` on vector. + OpcodeVCeil + + // OpcodeVFloor takes the floor of the given floating point value: `v = floor.lane x` on vector. + OpcodeVFloor + + // OpcodeVTrunc takes the truncation of the given floating point value: `v = trunc.lane x` on vector. + OpcodeVTrunc + + // OpcodeVNearest takes the nearest integer of the given floating point value: `v = nearest.lane x` on vector. + OpcodeVNearest + + // OpcodeVMaxPseudo computes the lane-wise maximum value `v = VMaxPseudo.lane x, y` on vector defined as `x < y ? x : y`. + OpcodeVMaxPseudo + + // OpcodeVMinPseudo computes the lane-wise minimum value `v = VMinPseudo.lane x, y` on vector defined as `y < x ? x : y`. + OpcodeVMinPseudo + + // OpcodeVSqrt takes the minimum of two floating point values: `v = VFmin.lane x, y` on vector. + OpcodeVSqrt + + // OpcodeVFcvtToUintSat converts a floating point value to an unsigned integer: `v = FcvtToUintSat.lane x` on vector. + OpcodeVFcvtToUintSat + + // OpcodeVFcvtToSintSat converts a floating point value to a signed integer: `v = VFcvtToSintSat.lane x` on vector. + OpcodeVFcvtToSintSat + + // OpcodeVFcvtFromUint converts a floating point value from an unsigned integer: `v = FcvtFromUint.lane x` on vector. + // x is always a 32-bit integer lane, and the result is either a 32-bit or 64-bit floating point-sized vector. + OpcodeVFcvtFromUint + + // OpcodeVFcvtFromSint converts a floating point value from a signed integer: `v = VFcvtFromSint.lane x` on vector. + // x is always a 32-bit integer lane, and the result is either a 32-bit or 64-bit floating point-sized vector. + OpcodeVFcvtFromSint + + // OpcodeImul performs an integer multiplication: `v = Imul x, y`. + OpcodeImul + + // OpcodeUdiv performs the unsigned integer division `v = Udiv x, y`. + OpcodeUdiv + + // OpcodeSdiv performs the signed integer division `v = Sdiv x, y`. + OpcodeSdiv + + // OpcodeUrem computes the remainder of the unsigned integer division `v = Urem x, y`. + OpcodeUrem + + // OpcodeSrem computes the remainder of the signed integer division `v = Srem x, y`. + OpcodeSrem + + // OpcodeBand performs a binary and: `v = Band x, y`. + OpcodeBand + + // OpcodeBor performs a binary or: `v = Bor x, y`. + OpcodeBor + + // OpcodeBxor performs a binary xor: `v = Bxor x, y`. + OpcodeBxor + + // OpcodeBnot performs a binary not: `v = Bnot x`. + OpcodeBnot + + // OpcodeRotl rotates the given integer value to the left: `v = Rotl x, y`. + OpcodeRotl + + // OpcodeRotr rotates the given integer value to the right: `v = Rotr x, y`. + OpcodeRotr + + // OpcodeIshl does logical shift left: `v = Ishl x, y`. + OpcodeIshl + + // OpcodeUshr does logical shift right: `v = Ushr x, y`. + OpcodeUshr + + // OpcodeSshr does arithmetic shift right: `v = Sshr x, y`. + OpcodeSshr + + // OpcodeClz counts the number of leading zeros: `v = clz x`. + OpcodeClz + + // OpcodeCtz counts the number of trailing zeros: `v = ctz x`. + OpcodeCtz + + // OpcodePopcnt counts the number of 1-bits: `v = popcnt x`. + OpcodePopcnt + + // OpcodeFcmp compares two floating point values: `v = fcmp Cond, x, y`. + OpcodeFcmp + + // OpcodeFadd performs a floating point addition: / `v = Fadd x, y`. + OpcodeFadd + + // OpcodeFsub performs a floating point subtraction: `v = Fsub x, y`. + OpcodeFsub + + // OpcodeFmul performs a floating point multiplication: `v = Fmul x, y`. + OpcodeFmul + + // OpcodeSqmulRoundSat performs a lane-wise saturating rounding multiplication + // in Q15 format: `v = SqmulRoundSat.lane x,y` on vector. + OpcodeSqmulRoundSat + + // OpcodeFdiv performs a floating point division: `v = Fdiv x, y`. + OpcodeFdiv + + // OpcodeSqrt takes the square root of the given floating point value: `v = sqrt x`. + OpcodeSqrt + + // OpcodeFneg negates the given floating point value: `v = Fneg x`. + OpcodeFneg + + // OpcodeFabs takes the absolute value of the given floating point value: `v = fabs x`. + OpcodeFabs + + // OpcodeFcopysign copies the sign of the second floating point value to the first floating point value: + // `v = Fcopysign x, y`. + OpcodeFcopysign + + // OpcodeFmin takes the minimum of two floating point values: `v = fmin x, y`. + OpcodeFmin + + // OpcodeFmax takes the maximum of two floating point values: `v = fmax x, y`. + OpcodeFmax + + // OpcodeCeil takes the ceiling of the given floating point value: `v = ceil x`. + OpcodeCeil + + // OpcodeFloor takes the floor of the given floating point value: `v = floor x`. + OpcodeFloor + + // OpcodeTrunc takes the truncation of the given floating point value: `v = trunc x`. + OpcodeTrunc + + // OpcodeNearest takes the nearest integer of the given floating point value: `v = nearest x`. + OpcodeNearest + + // OpcodeBitcast is a bitcast operation: `v = bitcast x`. + OpcodeBitcast + + // OpcodeIreduce narrow the given integer: `v = Ireduce x`. + OpcodeIreduce + + // OpcodeSnarrow converts two input vectors x, y into a smaller lane vector by narrowing each lane, signed `v = Snarrow.lane x, y`. + OpcodeSnarrow + + // OpcodeUnarrow converts two input vectors x, y into a smaller lane vector by narrowing each lane, unsigned `v = Unarrow.lane x, y`. + OpcodeUnarrow + + // OpcodeSwidenLow converts low half of the smaller lane vector to a larger lane vector, sign extended: `v = SwidenLow.lane x`. + OpcodeSwidenLow + + // OpcodeSwidenHigh converts high half of the smaller lane vector to a larger lane vector, sign extended: `v = SwidenHigh.lane x`. + OpcodeSwidenHigh + + // OpcodeUwidenLow converts low half of the smaller lane vector to a larger lane vector, zero (unsigned) extended: `v = UwidenLow.lane x`. + OpcodeUwidenLow + + // OpcodeUwidenHigh converts high half of the smaller lane vector to a larger lane vector, zero (unsigned) extended: `v = UwidenHigh.lane x`. + OpcodeUwidenHigh + + // OpcodeExtIaddPairwise is a lane-wise integer extended pairwise addition producing extended results (twice wider results than the inputs): `v = extiadd_pairwise x, y` on vector. + OpcodeExtIaddPairwise + + // OpcodeWideningPairwiseDotProductS is a lane-wise widening pairwise dot product with signed saturation: `v = WideningPairwiseDotProductS x, y` on vector. + // Currently, the only lane is i16, and the result is i32. + OpcodeWideningPairwiseDotProductS + + // OpcodeUExtend zero-extends the given integer: `v = UExtend x, from->to`. + OpcodeUExtend + + // OpcodeSExtend sign-extends the given integer: `v = SExtend x, from->to`. + OpcodeSExtend + + // OpcodeFpromote promotes the given floating point value: `v = Fpromote x`. + OpcodeFpromote + + // OpcodeFvpromoteLow converts the two lower single-precision floating point lanes + // to the two double-precision lanes of the result: `v = FvpromoteLow.lane x` on vector. + OpcodeFvpromoteLow + + // OpcodeFdemote demotes the given float point value: `v = Fdemote x`. + OpcodeFdemote + + // OpcodeFvdemote converts the two double-precision floating point lanes + // to two lower single-precision lanes of the result `v = Fvdemote.lane x`. + OpcodeFvdemote + + // OpcodeFcvtToUint converts a floating point value to an unsigned integer: `v = FcvtToUint x`. + OpcodeFcvtToUint + + // OpcodeFcvtToSint converts a floating point value to a signed integer: `v = FcvtToSint x`. + OpcodeFcvtToSint + + // OpcodeFcvtToUintSat converts a floating point value to an unsigned integer: `v = FcvtToUintSat x` which saturates on overflow. + OpcodeFcvtToUintSat + + // OpcodeFcvtToSintSat converts a floating point value to a signed integer: `v = FcvtToSintSat x` which saturates on overflow. + OpcodeFcvtToSintSat + + // OpcodeFcvtFromUint converts an unsigned integer to a floating point value: `v = FcvtFromUint x`. + OpcodeFcvtFromUint + + // OpcodeFcvtFromSint converts a signed integer to a floating point value: `v = FcvtFromSint x`. + OpcodeFcvtFromSint + + // OpcodeAtomicRmw is atomic read-modify-write operation: `v = atomic_rmw op, p, offset, value`. + OpcodeAtomicRmw + + // OpcodeAtomicCas is atomic compare-and-swap operation. + OpcodeAtomicCas + + // OpcodeAtomicLoad is atomic load operation. + OpcodeAtomicLoad + + // OpcodeAtomicStore is atomic store operation. + OpcodeAtomicStore + + // OpcodeFence is a memory fence operation. + OpcodeFence + + // opcodeEnd marks the end of the opcode list. + opcodeEnd +) + +// AtomicRmwOp represents the atomic read-modify-write operation. +type AtomicRmwOp byte + +const ( + // AtomicRmwOpAdd is an atomic add operation. + AtomicRmwOpAdd AtomicRmwOp = iota + // AtomicRmwOpSub is an atomic sub operation. + AtomicRmwOpSub + // AtomicRmwOpAnd is an atomic and operation. + AtomicRmwOpAnd + // AtomicRmwOpOr is an atomic or operation. + AtomicRmwOpOr + // AtomicRmwOpXor is an atomic xor operation. + AtomicRmwOpXor + // AtomicRmwOpXchg is an atomic swap operation. + AtomicRmwOpXchg +) + +// String implements the fmt.Stringer. +func (op AtomicRmwOp) String() string { + switch op { + case AtomicRmwOpAdd: + return "add" + case AtomicRmwOpSub: + return "sub" + case AtomicRmwOpAnd: + return "and" + case AtomicRmwOpOr: + return "or" + case AtomicRmwOpXor: + return "xor" + case AtomicRmwOpXchg: + return "xchg" + } + panic(fmt.Sprintf("unknown AtomicRmwOp: %d", op)) +} + +// returnTypesFn provides the info to determine the type of instruction. +// t1 is the type of the first result, ts are the types of the remaining results. +type returnTypesFn func(b *builder, instr *Instruction) (t1 Type, ts []Type) + +var ( + returnTypesFnNoReturns returnTypesFn = func(b *builder, instr *Instruction) (t1 Type, ts []Type) { return typeInvalid, nil } + returnTypesFnSingle = func(b *builder, instr *Instruction) (t1 Type, ts []Type) { return instr.typ, nil } + returnTypesFnI32 = func(b *builder, instr *Instruction) (t1 Type, ts []Type) { return TypeI32, nil } + returnTypesFnF32 = func(b *builder, instr *Instruction) (t1 Type, ts []Type) { return TypeF32, nil } + returnTypesFnF64 = func(b *builder, instr *Instruction) (t1 Type, ts []Type) { return TypeF64, nil } + returnTypesFnV128 = func(b *builder, instr *Instruction) (t1 Type, ts []Type) { return TypeV128, nil } +) + +// sideEffect provides the info to determine if an instruction has side effects which +// is used to determine if it can be optimized out, interchanged with others, etc. +type sideEffect byte + +const ( + sideEffectUnknown sideEffect = iota + // sideEffectStrict represents an instruction with side effects, and should be always alive plus cannot be reordered. + sideEffectStrict + // sideEffectTraps represents an instruction that can trap, and should be always alive but can be reordered within the group. + sideEffectTraps + // sideEffectNone represents an instruction without side effects, and can be eliminated if the result is not used, plus can be reordered within the group. + sideEffectNone +) + +// instructionSideEffects provides the info to determine if an instruction has side effects. +// Instructions with side effects must not be eliminated regardless whether the result is used or not. +var instructionSideEffects = [opcodeEnd]sideEffect{ + OpcodeUndefined: sideEffectStrict, + OpcodeJump: sideEffectStrict, + OpcodeIconst: sideEffectNone, + OpcodeCall: sideEffectStrict, + OpcodeCallIndirect: sideEffectStrict, + OpcodeIadd: sideEffectNone, + OpcodeImul: sideEffectNone, + OpcodeIsub: sideEffectNone, + OpcodeIcmp: sideEffectNone, + OpcodeExtractlane: sideEffectNone, + OpcodeInsertlane: sideEffectNone, + OpcodeBand: sideEffectNone, + OpcodeBor: sideEffectNone, + OpcodeBxor: sideEffectNone, + OpcodeRotl: sideEffectNone, + OpcodeRotr: sideEffectNone, + OpcodeFcmp: sideEffectNone, + OpcodeFadd: sideEffectNone, + OpcodeClz: sideEffectNone, + OpcodeCtz: sideEffectNone, + OpcodePopcnt: sideEffectNone, + OpcodeLoad: sideEffectNone, + OpcodeLoadSplat: sideEffectNone, + OpcodeUload8: sideEffectNone, + OpcodeUload16: sideEffectNone, + OpcodeUload32: sideEffectNone, + OpcodeSload8: sideEffectNone, + OpcodeSload16: sideEffectNone, + OpcodeSload32: sideEffectNone, + OpcodeSExtend: sideEffectNone, + OpcodeUExtend: sideEffectNone, + OpcodeSwidenLow: sideEffectNone, + OpcodeUwidenLow: sideEffectNone, + OpcodeSwidenHigh: sideEffectNone, + OpcodeUwidenHigh: sideEffectNone, + OpcodeSnarrow: sideEffectNone, + OpcodeUnarrow: sideEffectNone, + OpcodeSwizzle: sideEffectNone, + OpcodeShuffle: sideEffectNone, + OpcodeSplat: sideEffectNone, + OpcodeFsub: sideEffectNone, + OpcodeF32const: sideEffectNone, + OpcodeF64const: sideEffectNone, + OpcodeIshl: sideEffectNone, + OpcodeSshr: sideEffectNone, + OpcodeUshr: sideEffectNone, + OpcodeStore: sideEffectStrict, + OpcodeIstore8: sideEffectStrict, + OpcodeIstore16: sideEffectStrict, + OpcodeIstore32: sideEffectStrict, + OpcodeExitWithCode: sideEffectStrict, + OpcodeExitIfTrueWithCode: sideEffectStrict, + OpcodeReturn: sideEffectStrict, + OpcodeBrz: sideEffectStrict, + OpcodeBrnz: sideEffectStrict, + OpcodeBrTable: sideEffectStrict, + OpcodeFdiv: sideEffectNone, + OpcodeFmul: sideEffectNone, + OpcodeFmax: sideEffectNone, + OpcodeSqmulRoundSat: sideEffectNone, + OpcodeSelect: sideEffectNone, + OpcodeFmin: sideEffectNone, + OpcodeFneg: sideEffectNone, + OpcodeFcvtToSint: sideEffectTraps, + OpcodeFcvtToUint: sideEffectTraps, + OpcodeFcvtFromSint: sideEffectNone, + OpcodeFcvtFromUint: sideEffectNone, + OpcodeFcvtToSintSat: sideEffectNone, + OpcodeFcvtToUintSat: sideEffectNone, + OpcodeVFcvtFromUint: sideEffectNone, + OpcodeVFcvtFromSint: sideEffectNone, + OpcodeFdemote: sideEffectNone, + OpcodeFvpromoteLow: sideEffectNone, + OpcodeFvdemote: sideEffectNone, + OpcodeFpromote: sideEffectNone, + OpcodeBitcast: sideEffectNone, + OpcodeIreduce: sideEffectNone, + OpcodeSqrt: sideEffectNone, + OpcodeCeil: sideEffectNone, + OpcodeFloor: sideEffectNone, + OpcodeTrunc: sideEffectNone, + OpcodeNearest: sideEffectNone, + OpcodeSdiv: sideEffectTraps, + OpcodeSrem: sideEffectTraps, + OpcodeUdiv: sideEffectTraps, + OpcodeUrem: sideEffectTraps, + OpcodeFabs: sideEffectNone, + OpcodeFcopysign: sideEffectNone, + OpcodeExtIaddPairwise: sideEffectNone, + OpcodeVconst: sideEffectNone, + OpcodeVbor: sideEffectNone, + OpcodeVbxor: sideEffectNone, + OpcodeVband: sideEffectNone, + OpcodeVbandnot: sideEffectNone, + OpcodeVbnot: sideEffectNone, + OpcodeVbitselect: sideEffectNone, + OpcodeVanyTrue: sideEffectNone, + OpcodeVallTrue: sideEffectNone, + OpcodeVhighBits: sideEffectNone, + OpcodeVIadd: sideEffectNone, + OpcodeVSaddSat: sideEffectNone, + OpcodeVUaddSat: sideEffectNone, + OpcodeVIsub: sideEffectNone, + OpcodeVSsubSat: sideEffectNone, + OpcodeVUsubSat: sideEffectNone, + OpcodeVIcmp: sideEffectNone, + OpcodeVImin: sideEffectNone, + OpcodeVUmin: sideEffectNone, + OpcodeVImax: sideEffectNone, + OpcodeVUmax: sideEffectNone, + OpcodeVAvgRound: sideEffectNone, + OpcodeVImul: sideEffectNone, + OpcodeVIabs: sideEffectNone, + OpcodeVIneg: sideEffectNone, + OpcodeVIpopcnt: sideEffectNone, + OpcodeVIshl: sideEffectNone, + OpcodeVSshr: sideEffectNone, + OpcodeVUshr: sideEffectNone, + OpcodeVSqrt: sideEffectNone, + OpcodeVFabs: sideEffectNone, + OpcodeVFmin: sideEffectNone, + OpcodeVFmax: sideEffectNone, + OpcodeVFneg: sideEffectNone, + OpcodeVFadd: sideEffectNone, + OpcodeVFsub: sideEffectNone, + OpcodeVFmul: sideEffectNone, + OpcodeVFdiv: sideEffectNone, + OpcodeVFcmp: sideEffectNone, + OpcodeVCeil: sideEffectNone, + OpcodeVFloor: sideEffectNone, + OpcodeVTrunc: sideEffectNone, + OpcodeVNearest: sideEffectNone, + OpcodeVMaxPseudo: sideEffectNone, + OpcodeVMinPseudo: sideEffectNone, + OpcodeVFcvtToUintSat: sideEffectNone, + OpcodeVFcvtToSintSat: sideEffectNone, + OpcodeVZeroExtLoad: sideEffectNone, + OpcodeAtomicRmw: sideEffectStrict, + OpcodeAtomicLoad: sideEffectStrict, + OpcodeAtomicStore: sideEffectStrict, + OpcodeAtomicCas: sideEffectStrict, + OpcodeFence: sideEffectStrict, + OpcodeWideningPairwiseDotProductS: sideEffectNone, +} + +// sideEffect returns true if this instruction has side effects. +func (i *Instruction) sideEffect() sideEffect { + if e := instructionSideEffects[i.opcode]; e == sideEffectUnknown { + panic("BUG: side effect info not registered for " + i.opcode.String()) + } else { + return e + } +} + +// instructionReturnTypes provides the function to determine the return types of an instruction. +var instructionReturnTypes = [opcodeEnd]returnTypesFn{ + OpcodeExtIaddPairwise: returnTypesFnV128, + OpcodeVbor: returnTypesFnV128, + OpcodeVbxor: returnTypesFnV128, + OpcodeVband: returnTypesFnV128, + OpcodeVbnot: returnTypesFnV128, + OpcodeVbandnot: returnTypesFnV128, + OpcodeVbitselect: returnTypesFnV128, + OpcodeVanyTrue: returnTypesFnI32, + OpcodeVallTrue: returnTypesFnI32, + OpcodeVhighBits: returnTypesFnI32, + OpcodeVIadd: returnTypesFnV128, + OpcodeVSaddSat: returnTypesFnV128, + OpcodeVUaddSat: returnTypesFnV128, + OpcodeVIsub: returnTypesFnV128, + OpcodeVSsubSat: returnTypesFnV128, + OpcodeVUsubSat: returnTypesFnV128, + OpcodeVIcmp: returnTypesFnV128, + OpcodeVImin: returnTypesFnV128, + OpcodeVUmin: returnTypesFnV128, + OpcodeVImax: returnTypesFnV128, + OpcodeVUmax: returnTypesFnV128, + OpcodeVImul: returnTypesFnV128, + OpcodeVAvgRound: returnTypesFnV128, + OpcodeVIabs: returnTypesFnV128, + OpcodeVIneg: returnTypesFnV128, + OpcodeVIpopcnt: returnTypesFnV128, + OpcodeVIshl: returnTypesFnV128, + OpcodeVSshr: returnTypesFnV128, + OpcodeVUshr: returnTypesFnV128, + OpcodeExtractlane: returnTypesFnSingle, + OpcodeInsertlane: returnTypesFnV128, + OpcodeBand: returnTypesFnSingle, + OpcodeFcopysign: returnTypesFnSingle, + OpcodeBitcast: returnTypesFnSingle, + OpcodeBor: returnTypesFnSingle, + OpcodeBxor: returnTypesFnSingle, + OpcodeRotl: returnTypesFnSingle, + OpcodeRotr: returnTypesFnSingle, + OpcodeIshl: returnTypesFnSingle, + OpcodeSshr: returnTypesFnSingle, + OpcodeSdiv: returnTypesFnSingle, + OpcodeSrem: returnTypesFnSingle, + OpcodeUdiv: returnTypesFnSingle, + OpcodeUrem: returnTypesFnSingle, + OpcodeUshr: returnTypesFnSingle, + OpcodeJump: returnTypesFnNoReturns, + OpcodeUndefined: returnTypesFnNoReturns, + OpcodeIconst: returnTypesFnSingle, + OpcodeSelect: returnTypesFnSingle, + OpcodeSExtend: returnTypesFnSingle, + OpcodeUExtend: returnTypesFnSingle, + OpcodeSwidenLow: returnTypesFnV128, + OpcodeUwidenLow: returnTypesFnV128, + OpcodeSwidenHigh: returnTypesFnV128, + OpcodeUwidenHigh: returnTypesFnV128, + OpcodeSnarrow: returnTypesFnV128, + OpcodeUnarrow: returnTypesFnV128, + OpcodeSwizzle: returnTypesFnSingle, + OpcodeShuffle: returnTypesFnV128, + OpcodeSplat: returnTypesFnV128, + OpcodeIreduce: returnTypesFnSingle, + OpcodeFabs: returnTypesFnSingle, + OpcodeSqrt: returnTypesFnSingle, + OpcodeCeil: returnTypesFnSingle, + OpcodeFloor: returnTypesFnSingle, + OpcodeTrunc: returnTypesFnSingle, + OpcodeNearest: returnTypesFnSingle, + OpcodeCallIndirect: func(b *builder, instr *Instruction) (t1 Type, ts []Type) { + sigID := SignatureID(instr.u1) + sig, ok := b.signatures[sigID] + if !ok { + panic("BUG") + } + switch len(sig.Results) { + case 0: + t1 = typeInvalid + case 1: + t1 = sig.Results[0] + default: + t1, ts = sig.Results[0], sig.Results[1:] + } + return + }, + OpcodeCall: func(b *builder, instr *Instruction) (t1 Type, ts []Type) { + sigID := SignatureID(instr.u2) + sig, ok := b.signatures[sigID] + if !ok { + panic("BUG") + } + switch len(sig.Results) { + case 0: + t1 = typeInvalid + case 1: + t1 = sig.Results[0] + default: + t1, ts = sig.Results[0], sig.Results[1:] + } + return + }, + OpcodeLoad: returnTypesFnSingle, + OpcodeVZeroExtLoad: returnTypesFnV128, + OpcodeLoadSplat: returnTypesFnV128, + OpcodeIadd: returnTypesFnSingle, + OpcodeIsub: returnTypesFnSingle, + OpcodeImul: returnTypesFnSingle, + OpcodeIcmp: returnTypesFnI32, + OpcodeFcmp: returnTypesFnI32, + OpcodeFadd: returnTypesFnSingle, + OpcodeFsub: returnTypesFnSingle, + OpcodeFdiv: returnTypesFnSingle, + OpcodeFmul: returnTypesFnSingle, + OpcodeFmax: returnTypesFnSingle, + OpcodeFmin: returnTypesFnSingle, + OpcodeSqmulRoundSat: returnTypesFnV128, + OpcodeF32const: returnTypesFnF32, + OpcodeF64const: returnTypesFnF64, + OpcodeClz: returnTypesFnSingle, + OpcodeCtz: returnTypesFnSingle, + OpcodePopcnt: returnTypesFnSingle, + OpcodeStore: returnTypesFnNoReturns, + OpcodeIstore8: returnTypesFnNoReturns, + OpcodeIstore16: returnTypesFnNoReturns, + OpcodeIstore32: returnTypesFnNoReturns, + OpcodeExitWithCode: returnTypesFnNoReturns, + OpcodeExitIfTrueWithCode: returnTypesFnNoReturns, + OpcodeReturn: returnTypesFnNoReturns, + OpcodeBrz: returnTypesFnNoReturns, + OpcodeBrnz: returnTypesFnNoReturns, + OpcodeBrTable: returnTypesFnNoReturns, + OpcodeUload8: returnTypesFnSingle, + OpcodeUload16: returnTypesFnSingle, + OpcodeUload32: returnTypesFnSingle, + OpcodeSload8: returnTypesFnSingle, + OpcodeSload16: returnTypesFnSingle, + OpcodeSload32: returnTypesFnSingle, + OpcodeFcvtToSint: returnTypesFnSingle, + OpcodeFcvtToUint: returnTypesFnSingle, + OpcodeFcvtFromSint: returnTypesFnSingle, + OpcodeFcvtFromUint: returnTypesFnSingle, + OpcodeFcvtToSintSat: returnTypesFnSingle, + OpcodeFcvtToUintSat: returnTypesFnSingle, + OpcodeVFcvtFromUint: returnTypesFnV128, + OpcodeVFcvtFromSint: returnTypesFnV128, + OpcodeFneg: returnTypesFnSingle, + OpcodeFdemote: returnTypesFnF32, + OpcodeFvdemote: returnTypesFnV128, + OpcodeFvpromoteLow: returnTypesFnV128, + OpcodeFpromote: returnTypesFnF64, + OpcodeVconst: returnTypesFnV128, + OpcodeVFabs: returnTypesFnV128, + OpcodeVSqrt: returnTypesFnV128, + OpcodeVFmax: returnTypesFnV128, + OpcodeVFmin: returnTypesFnV128, + OpcodeVFneg: returnTypesFnV128, + OpcodeVFadd: returnTypesFnV128, + OpcodeVFsub: returnTypesFnV128, + OpcodeVFmul: returnTypesFnV128, + OpcodeVFdiv: returnTypesFnV128, + OpcodeVFcmp: returnTypesFnV128, + OpcodeVCeil: returnTypesFnV128, + OpcodeVFloor: returnTypesFnV128, + OpcodeVTrunc: returnTypesFnV128, + OpcodeVNearest: returnTypesFnV128, + OpcodeVMaxPseudo: returnTypesFnV128, + OpcodeVMinPseudo: returnTypesFnV128, + OpcodeVFcvtToUintSat: returnTypesFnV128, + OpcodeVFcvtToSintSat: returnTypesFnV128, + OpcodeAtomicRmw: returnTypesFnSingle, + OpcodeAtomicLoad: returnTypesFnSingle, + OpcodeAtomicStore: returnTypesFnNoReturns, + OpcodeAtomicCas: returnTypesFnSingle, + OpcodeFence: returnTypesFnNoReturns, + OpcodeWideningPairwiseDotProductS: returnTypesFnV128, +} + +// AsLoad initializes this instruction as a store instruction with OpcodeLoad. +func (i *Instruction) AsLoad(ptr Value, offset uint32, typ Type) *Instruction { + i.opcode = OpcodeLoad + i.v = ptr + i.u1 = uint64(offset) + i.typ = typ + return i +} + +// AsExtLoad initializes this instruction as a store instruction with OpcodeLoad. +func (i *Instruction) AsExtLoad(op Opcode, ptr Value, offset uint32, dst64bit bool) *Instruction { + i.opcode = op + i.v = ptr + i.u1 = uint64(offset) + if dst64bit { + i.typ = TypeI64 + } else { + i.typ = TypeI32 + } + return i +} + +// AsVZeroExtLoad initializes this instruction as a store instruction with OpcodeVExtLoad. +func (i *Instruction) AsVZeroExtLoad(ptr Value, offset uint32, scalarType Type) *Instruction { + i.opcode = OpcodeVZeroExtLoad + i.v = ptr + i.u1 = uint64(offset) + i.u2 = uint64(scalarType) + i.typ = TypeV128 + return i +} + +// VZeroExtLoadData returns the operands for a load instruction. The returned `typ` is the scalar type of the load target. +func (i *Instruction) VZeroExtLoadData() (ptr Value, offset uint32, typ Type) { + return i.v, uint32(i.u1), Type(i.u2) +} + +// AsLoadSplat initializes this instruction as a store instruction with OpcodeLoadSplat. +func (i *Instruction) AsLoadSplat(ptr Value, offset uint32, lane VecLane) *Instruction { + i.opcode = OpcodeLoadSplat + i.v = ptr + i.u1 = uint64(offset) + i.u2 = uint64(lane) + i.typ = TypeV128 + return i +} + +// LoadData returns the operands for a load instruction. +func (i *Instruction) LoadData() (ptr Value, offset uint32, typ Type) { + return i.v, uint32(i.u1), i.typ +} + +// LoadSplatData returns the operands for a load splat instruction. +func (i *Instruction) LoadSplatData() (ptr Value, offset uint32, lane VecLane) { + return i.v, uint32(i.u1), VecLane(i.u2) +} + +// AsStore initializes this instruction as a store instruction with OpcodeStore. +func (i *Instruction) AsStore(storeOp Opcode, value, ptr Value, offset uint32) *Instruction { + i.opcode = storeOp + i.v = value + i.v2 = ptr + + var dstSize uint64 + switch storeOp { + case OpcodeStore: + dstSize = uint64(value.Type().Bits()) + case OpcodeIstore8: + dstSize = 8 + case OpcodeIstore16: + dstSize = 16 + case OpcodeIstore32: + dstSize = 32 + default: + panic("invalid store opcode" + storeOp.String()) + } + i.u1 = uint64(offset) | dstSize<<32 + return i +} + +// StoreData returns the operands for a store instruction. +func (i *Instruction) StoreData() (value, ptr Value, offset uint32, storeSizeInBits byte) { + return i.v, i.v2, uint32(i.u1), byte(i.u1 >> 32) +} + +// AsIconst64 initializes this instruction as a 64-bit integer constant instruction with OpcodeIconst. +func (i *Instruction) AsIconst64(v uint64) *Instruction { + i.opcode = OpcodeIconst + i.typ = TypeI64 + i.u1 = v + return i +} + +// AsIconst32 initializes this instruction as a 32-bit integer constant instruction with OpcodeIconst. +func (i *Instruction) AsIconst32(v uint32) *Instruction { + i.opcode = OpcodeIconst + i.typ = TypeI32 + i.u1 = uint64(v) + return i +} + +// AsIadd initializes this instruction as an integer addition instruction with OpcodeIadd. +func (i *Instruction) AsIadd(x, y Value) *Instruction { + i.opcode = OpcodeIadd + i.v = x + i.v2 = y + i.typ = x.Type() + return i +} + +// AsVIadd initializes this instruction as an integer addition instruction with OpcodeVIadd on a vector. +func (i *Instruction) AsVIadd(x, y Value, lane VecLane) *Instruction { + i.opcode = OpcodeVIadd + i.v = x + i.v2 = y + i.u1 = uint64(lane) + i.typ = TypeV128 + return i +} + +// AsWideningPairwiseDotProductS initializes this instruction as a lane-wise integer extended pairwise addition instruction +// with OpcodeIaddPairwise on a vector. +func (i *Instruction) AsWideningPairwiseDotProductS(x, y Value) *Instruction { + i.opcode = OpcodeWideningPairwiseDotProductS + i.v = x + i.v2 = y + i.typ = TypeV128 + return i +} + +// AsExtIaddPairwise initializes this instruction as a lane-wise integer extended pairwise addition instruction +// with OpcodeIaddPairwise on a vector. +func (i *Instruction) AsExtIaddPairwise(x Value, srcLane VecLane, signed bool) *Instruction { + i.opcode = OpcodeExtIaddPairwise + i.v = x + i.u1 = uint64(srcLane) + if signed { + i.u2 = 1 + } + i.typ = TypeV128 + return i +} + +// ExtIaddPairwiseData returns the operands for a lane-wise integer extended pairwise addition instruction. +func (i *Instruction) ExtIaddPairwiseData() (x Value, srcLane VecLane, signed bool) { + return i.v, VecLane(i.u1), i.u2 != 0 +} + +// AsVSaddSat initializes this instruction as a vector addition with saturation instruction with OpcodeVSaddSat on a vector. +func (i *Instruction) AsVSaddSat(x, y Value, lane VecLane) *Instruction { + i.opcode = OpcodeVSaddSat + i.v = x + i.v2 = y + i.u1 = uint64(lane) + i.typ = TypeV128 + return i +} + +// AsVUaddSat initializes this instruction as a vector addition with saturation instruction with OpcodeVUaddSat on a vector. +func (i *Instruction) AsVUaddSat(x, y Value, lane VecLane) *Instruction { + i.opcode = OpcodeVUaddSat + i.v = x + i.v2 = y + i.u1 = uint64(lane) + i.typ = TypeV128 + return i +} + +// AsVIsub initializes this instruction as an integer subtraction instruction with OpcodeVIsub on a vector. +func (i *Instruction) AsVIsub(x, y Value, lane VecLane) *Instruction { + i.opcode = OpcodeVIsub + i.v = x + i.v2 = y + i.u1 = uint64(lane) + i.typ = TypeV128 + return i +} + +// AsVSsubSat initializes this instruction as a vector addition with saturation instruction with OpcodeVSsubSat on a vector. +func (i *Instruction) AsVSsubSat(x, y Value, lane VecLane) *Instruction { + i.opcode = OpcodeVSsubSat + i.v = x + i.v2 = y + i.u1 = uint64(lane) + i.typ = TypeV128 + return i +} + +// AsVUsubSat initializes this instruction as a vector addition with saturation instruction with OpcodeVUsubSat on a vector. +func (i *Instruction) AsVUsubSat(x, y Value, lane VecLane) *Instruction { + i.opcode = OpcodeVUsubSat + i.v = x + i.v2 = y + i.u1 = uint64(lane) + i.typ = TypeV128 + return i +} + +// AsVImin initializes this instruction as a signed integer min instruction with OpcodeVImin on a vector. +func (i *Instruction) AsVImin(x, y Value, lane VecLane) *Instruction { + i.opcode = OpcodeVImin + i.v = x + i.v2 = y + i.u1 = uint64(lane) + i.typ = TypeV128 + return i +} + +// AsVUmin initializes this instruction as an unsigned integer min instruction with OpcodeVUmin on a vector. +func (i *Instruction) AsVUmin(x, y Value, lane VecLane) *Instruction { + i.opcode = OpcodeVUmin + i.v = x + i.v2 = y + i.u1 = uint64(lane) + i.typ = TypeV128 + return i +} + +// AsVImax initializes this instruction as a signed integer max instruction with OpcodeVImax on a vector. +func (i *Instruction) AsVImax(x, y Value, lane VecLane) *Instruction { + i.opcode = OpcodeVImax + i.v = x + i.v2 = y + i.u1 = uint64(lane) + i.typ = TypeV128 + return i +} + +// AsVUmax initializes this instruction as an unsigned integer max instruction with OpcodeVUmax on a vector. +func (i *Instruction) AsVUmax(x, y Value, lane VecLane) *Instruction { + i.opcode = OpcodeVUmax + i.v = x + i.v2 = y + i.u1 = uint64(lane) + i.typ = TypeV128 + return i +} + +// AsVAvgRound initializes this instruction as an unsigned integer avg instruction, truncating to zero with OpcodeVAvgRound on a vector. +func (i *Instruction) AsVAvgRound(x, y Value, lane VecLane) *Instruction { + i.opcode = OpcodeVAvgRound + i.v = x + i.v2 = y + i.u1 = uint64(lane) + i.typ = TypeV128 + return i +} + +// AsVImul initializes this instruction as an integer multiplication with OpcodeVImul on a vector. +func (i *Instruction) AsVImul(x, y Value, lane VecLane) *Instruction { + i.opcode = OpcodeVImul + i.v = x + i.v2 = y + i.u1 = uint64(lane) + i.typ = TypeV128 + return i +} + +// AsSqmulRoundSat initializes this instruction as a lane-wise saturating rounding multiplication +// in Q15 format with OpcodeSqmulRoundSat on a vector. +func (i *Instruction) AsSqmulRoundSat(x, y Value, lane VecLane) *Instruction { + i.opcode = OpcodeSqmulRoundSat + i.v = x + i.v2 = y + i.u1 = uint64(lane) + i.typ = TypeV128 + return i +} + +// AsVIabs initializes this instruction as a vector absolute value with OpcodeVIabs. +func (i *Instruction) AsVIabs(x Value, lane VecLane) *Instruction { + i.opcode = OpcodeVIabs + i.v = x + i.u1 = uint64(lane) + i.typ = TypeV128 + return i +} + +// AsVIneg initializes this instruction as a vector negation with OpcodeVIneg. +func (i *Instruction) AsVIneg(x Value, lane VecLane) *Instruction { + i.opcode = OpcodeVIneg + i.v = x + i.u1 = uint64(lane) + i.typ = TypeV128 + return i +} + +// AsVIpopcnt initializes this instruction as a Population Count instruction with OpcodeVIpopcnt on a vector. +func (i *Instruction) AsVIpopcnt(x Value, lane VecLane) *Instruction { + if lane != VecLaneI8x16 { + panic("Unsupported lane type " + lane.String()) + } + i.opcode = OpcodeVIpopcnt + i.v = x + i.u1 = uint64(lane) + i.typ = TypeV128 + return i +} + +// AsVSqrt initializes this instruction as a sqrt instruction with OpcodeVSqrt on a vector. +func (i *Instruction) AsVSqrt(x Value, lane VecLane) *Instruction { + i.opcode = OpcodeVSqrt + i.v = x + i.u1 = uint64(lane) + i.typ = TypeV128 + return i +} + +// AsVFabs initializes this instruction as a float abs instruction with OpcodeVFabs on a vector. +func (i *Instruction) AsVFabs(x Value, lane VecLane) *Instruction { + i.opcode = OpcodeVFabs + i.v = x + i.u1 = uint64(lane) + i.typ = TypeV128 + return i +} + +// AsVFneg initializes this instruction as a float neg instruction with OpcodeVFneg on a vector. +func (i *Instruction) AsVFneg(x Value, lane VecLane) *Instruction { + i.opcode = OpcodeVFneg + i.v = x + i.u1 = uint64(lane) + i.typ = TypeV128 + return i +} + +// AsVFmax initializes this instruction as a float max instruction with OpcodeVFmax on a vector. +func (i *Instruction) AsVFmax(x, y Value, lane VecLane) *Instruction { + i.opcode = OpcodeVFmax + i.v = x + i.v2 = y + i.u1 = uint64(lane) + i.typ = TypeV128 + return i +} + +// AsVFmin initializes this instruction as a float min instruction with OpcodeVFmin on a vector. +func (i *Instruction) AsVFmin(x, y Value, lane VecLane) *Instruction { + i.opcode = OpcodeVFmin + i.v = x + i.v2 = y + i.u1 = uint64(lane) + i.typ = TypeV128 + return i +} + +// AsVFadd initializes this instruction as a floating point add instruction with OpcodeVFadd on a vector. +func (i *Instruction) AsVFadd(x, y Value, lane VecLane) *Instruction { + i.opcode = OpcodeVFadd + i.v = x + i.v2 = y + i.u1 = uint64(lane) + i.typ = TypeV128 + return i +} + +// AsVFsub initializes this instruction as a floating point subtraction instruction with OpcodeVFsub on a vector. +func (i *Instruction) AsVFsub(x, y Value, lane VecLane) *Instruction { + i.opcode = OpcodeVFsub + i.v = x + i.v2 = y + i.u1 = uint64(lane) + i.typ = TypeV128 + return i +} + +// AsVFmul initializes this instruction as a floating point multiplication instruction with OpcodeVFmul on a vector. +func (i *Instruction) AsVFmul(x, y Value, lane VecLane) *Instruction { + i.opcode = OpcodeVFmul + i.v = x + i.v2 = y + i.u1 = uint64(lane) + i.typ = TypeV128 + return i +} + +// AsVFdiv initializes this instruction as a floating point division instruction with OpcodeVFdiv on a vector. +func (i *Instruction) AsVFdiv(x, y Value, lane VecLane) *Instruction { + i.opcode = OpcodeVFdiv + i.v = x + i.v2 = y + i.u1 = uint64(lane) + i.typ = TypeV128 + return i +} + +// AsImul initializes this instruction as an integer addition instruction with OpcodeImul. +func (i *Instruction) AsImul(x, y Value) *Instruction { + i.opcode = OpcodeImul + i.v = x + i.v2 = y + i.typ = x.Type() + return i +} + +func (i *Instruction) Insert(b Builder) *Instruction { + b.InsertInstruction(i) + return i +} + +// AsIsub initializes this instruction as an integer subtraction instruction with OpcodeIsub. +func (i *Instruction) AsIsub(x, y Value) *Instruction { + i.opcode = OpcodeIsub + i.v = x + i.v2 = y + i.typ = x.Type() + return i +} + +// AsIcmp initializes this instruction as an integer comparison instruction with OpcodeIcmp. +func (i *Instruction) AsIcmp(x, y Value, c IntegerCmpCond) *Instruction { + i.opcode = OpcodeIcmp + i.v = x + i.v2 = y + i.u1 = uint64(c) + i.typ = TypeI32 + return i +} + +// AsFcmp initializes this instruction as an integer comparison instruction with OpcodeFcmp. +func (i *Instruction) AsFcmp(x, y Value, c FloatCmpCond) { + i.opcode = OpcodeFcmp + i.v = x + i.v2 = y + i.u1 = uint64(c) + i.typ = TypeI32 +} + +// AsVIcmp initializes this instruction as an integer vector comparison instruction with OpcodeVIcmp. +func (i *Instruction) AsVIcmp(x, y Value, c IntegerCmpCond, lane VecLane) *Instruction { + i.opcode = OpcodeVIcmp + i.v = x + i.v2 = y + i.u1 = uint64(c) + i.u2 = uint64(lane) + i.typ = TypeV128 + return i +} + +// AsVFcmp initializes this instruction as a float comparison instruction with OpcodeVFcmp on Vector. +func (i *Instruction) AsVFcmp(x, y Value, c FloatCmpCond, lane VecLane) *Instruction { + i.opcode = OpcodeVFcmp + i.v = x + i.v2 = y + i.u1 = uint64(c) + i.typ = TypeV128 + i.u2 = uint64(lane) + return i +} + +// AsVCeil initializes this instruction as an instruction with OpcodeCeil. +func (i *Instruction) AsVCeil(x Value, lane VecLane) *Instruction { + i.opcode = OpcodeVCeil + i.v = x + i.typ = x.Type() + i.u1 = uint64(lane) + return i +} + +// AsVFloor initializes this instruction as an instruction with OpcodeFloor. +func (i *Instruction) AsVFloor(x Value, lane VecLane) *Instruction { + i.opcode = OpcodeVFloor + i.v = x + i.typ = x.Type() + i.u1 = uint64(lane) + return i +} + +// AsVTrunc initializes this instruction as an instruction with OpcodeTrunc. +func (i *Instruction) AsVTrunc(x Value, lane VecLane) *Instruction { + i.opcode = OpcodeVTrunc + i.v = x + i.typ = x.Type() + i.u1 = uint64(lane) + return i +} + +// AsVNearest initializes this instruction as an instruction with OpcodeNearest. +func (i *Instruction) AsVNearest(x Value, lane VecLane) *Instruction { + i.opcode = OpcodeVNearest + i.v = x + i.typ = x.Type() + i.u1 = uint64(lane) + return i +} + +// AsVMaxPseudo initializes this instruction as an instruction with OpcodeVMaxPseudo. +func (i *Instruction) AsVMaxPseudo(x, y Value, lane VecLane) *Instruction { + i.opcode = OpcodeVMaxPseudo + i.typ = x.Type() + i.v = x + i.v2 = y + i.u1 = uint64(lane) + return i +} + +// AsVMinPseudo initializes this instruction as an instruction with OpcodeVMinPseudo. +func (i *Instruction) AsVMinPseudo(x, y Value, lane VecLane) *Instruction { + i.opcode = OpcodeVMinPseudo + i.typ = x.Type() + i.v = x + i.v2 = y + i.u1 = uint64(lane) + return i +} + +// AsSDiv initializes this instruction as an integer bitwise and instruction with OpcodeSdiv. +func (i *Instruction) AsSDiv(x, y, ctx Value) *Instruction { + i.opcode = OpcodeSdiv + i.v = x + i.v2 = y + i.v3 = ctx + i.typ = x.Type() + return i +} + +// AsUDiv initializes this instruction as an integer bitwise and instruction with OpcodeUdiv. +func (i *Instruction) AsUDiv(x, y, ctx Value) *Instruction { + i.opcode = OpcodeUdiv + i.v = x + i.v2 = y + i.v3 = ctx + i.typ = x.Type() + return i +} + +// AsSRem initializes this instruction as an integer bitwise and instruction with OpcodeSrem. +func (i *Instruction) AsSRem(x, y, ctx Value) *Instruction { + i.opcode = OpcodeSrem + i.v = x + i.v2 = y + i.v3 = ctx + i.typ = x.Type() + return i +} + +// AsURem initializes this instruction as an integer bitwise and instruction with OpcodeUrem. +func (i *Instruction) AsURem(x, y, ctx Value) *Instruction { + i.opcode = OpcodeUrem + i.v = x + i.v2 = y + i.v3 = ctx + i.typ = x.Type() + return i +} + +// AsBand initializes this instruction as an integer bitwise and instruction with OpcodeBand. +func (i *Instruction) AsBand(x, amount Value) *Instruction { + i.opcode = OpcodeBand + i.v = x + i.v2 = amount + i.typ = x.Type() + return i +} + +// AsBor initializes this instruction as an integer bitwise or instruction with OpcodeBor. +func (i *Instruction) AsBor(x, amount Value) { + i.opcode = OpcodeBor + i.v = x + i.v2 = amount + i.typ = x.Type() +} + +// AsBxor initializes this instruction as an integer bitwise xor instruction with OpcodeBxor. +func (i *Instruction) AsBxor(x, amount Value) { + i.opcode = OpcodeBxor + i.v = x + i.v2 = amount + i.typ = x.Type() +} + +// AsIshl initializes this instruction as an integer shift left instruction with OpcodeIshl. +func (i *Instruction) AsIshl(x, amount Value) *Instruction { + i.opcode = OpcodeIshl + i.v = x + i.v2 = amount + i.typ = x.Type() + return i +} + +// AsVIshl initializes this instruction as an integer shift left instruction with OpcodeVIshl on vector. +func (i *Instruction) AsVIshl(x, amount Value, lane VecLane) *Instruction { + i.opcode = OpcodeVIshl + i.v = x + i.v2 = amount + i.u1 = uint64(lane) + i.typ = x.Type() + return i +} + +// AsUshr initializes this instruction as an integer unsigned shift right (logical shift right) instruction with OpcodeUshr. +func (i *Instruction) AsUshr(x, amount Value) *Instruction { + i.opcode = OpcodeUshr + i.v = x + i.v2 = amount + i.typ = x.Type() + return i +} + +// AsVUshr initializes this instruction as an integer unsigned shift right (logical shift right) instruction with OpcodeVUshr on vector. +func (i *Instruction) AsVUshr(x, amount Value, lane VecLane) *Instruction { + i.opcode = OpcodeVUshr + i.v = x + i.v2 = amount + i.u1 = uint64(lane) + i.typ = x.Type() + return i +} + +// AsSshr initializes this instruction as an integer signed shift right (arithmetic shift right) instruction with OpcodeSshr. +func (i *Instruction) AsSshr(x, amount Value) *Instruction { + i.opcode = OpcodeSshr + i.v = x + i.v2 = amount + i.typ = x.Type() + return i +} + +// AsVSshr initializes this instruction as an integer signed shift right (arithmetic shift right) instruction with OpcodeVSshr on vector. +func (i *Instruction) AsVSshr(x, amount Value, lane VecLane) *Instruction { + i.opcode = OpcodeVSshr + i.v = x + i.v2 = amount + i.u1 = uint64(lane) + i.typ = x.Type() + return i +} + +// AsExtractlane initializes this instruction as an extract lane instruction with OpcodeExtractlane on vector. +func (i *Instruction) AsExtractlane(x Value, index byte, lane VecLane, signed bool) *Instruction { + i.opcode = OpcodeExtractlane + i.v = x + // We do not have a field for signedness, but `index` is a byte, + // so we just encode the flag in the high bits of `u1`. + i.u1 = uint64(index) + if signed { + i.u1 = i.u1 | 1<<32 + } + i.u2 = uint64(lane) + switch lane { + case VecLaneI8x16, VecLaneI16x8, VecLaneI32x4: + i.typ = TypeI32 + case VecLaneI64x2: + i.typ = TypeI64 + case VecLaneF32x4: + i.typ = TypeF32 + case VecLaneF64x2: + i.typ = TypeF64 + } + return i +} + +// AsInsertlane initializes this instruction as an insert lane instruction with OpcodeInsertlane on vector. +func (i *Instruction) AsInsertlane(x, y Value, index byte, lane VecLane) *Instruction { + i.opcode = OpcodeInsertlane + i.v = x + i.v2 = y + i.u1 = uint64(index) + i.u2 = uint64(lane) + i.typ = TypeV128 + return i +} + +// AsShuffle initializes this instruction as a shuffle instruction with OpcodeShuffle on vector. +func (i *Instruction) AsShuffle(x, y Value, lane []byte) *Instruction { + i.opcode = OpcodeShuffle + i.v = x + i.v2 = y + // Encode the 16 bytes as 8 bytes in u1, and 8 bytes in u2. + i.u1 = uint64(lane[7])<<56 | uint64(lane[6])<<48 | uint64(lane[5])<<40 | uint64(lane[4])<<32 | uint64(lane[3])<<24 | uint64(lane[2])<<16 | uint64(lane[1])<<8 | uint64(lane[0]) + i.u2 = uint64(lane[15])<<56 | uint64(lane[14])<<48 | uint64(lane[13])<<40 | uint64(lane[12])<<32 | uint64(lane[11])<<24 | uint64(lane[10])<<16 | uint64(lane[9])<<8 | uint64(lane[8]) + i.typ = TypeV128 + return i +} + +// AsSwizzle initializes this instruction as an insert lane instruction with OpcodeSwizzle on vector. +func (i *Instruction) AsSwizzle(x, y Value, lane VecLane) *Instruction { + i.opcode = OpcodeSwizzle + i.v = x + i.v2 = y + i.u1 = uint64(lane) + i.typ = TypeV128 + return i +} + +// AsSplat initializes this instruction as an insert lane instruction with OpcodeSplat on vector. +func (i *Instruction) AsSplat(x Value, lane VecLane) *Instruction { + i.opcode = OpcodeSplat + i.v = x + i.u1 = uint64(lane) + i.typ = TypeV128 + return i +} + +// AsRotl initializes this instruction as a word rotate left instruction with OpcodeRotl. +func (i *Instruction) AsRotl(x, amount Value) { + i.opcode = OpcodeRotl + i.v = x + i.v2 = amount + i.typ = x.Type() +} + +// AsRotr initializes this instruction as a word rotate right instruction with OpcodeRotr. +func (i *Instruction) AsRotr(x, amount Value) { + i.opcode = OpcodeRotr + i.v = x + i.v2 = amount + i.typ = x.Type() +} + +// IcmpData returns the operands and comparison condition of this integer comparison instruction. +func (i *Instruction) IcmpData() (x, y Value, c IntegerCmpCond) { + return i.v, i.v2, IntegerCmpCond(i.u1) +} + +// FcmpData returns the operands and comparison condition of this floating-point comparison instruction. +func (i *Instruction) FcmpData() (x, y Value, c FloatCmpCond) { + return i.v, i.v2, FloatCmpCond(i.u1) +} + +// VIcmpData returns the operands and comparison condition of this integer comparison instruction on vector. +func (i *Instruction) VIcmpData() (x, y Value, c IntegerCmpCond, l VecLane) { + return i.v, i.v2, IntegerCmpCond(i.u1), VecLane(i.u2) +} + +// VFcmpData returns the operands and comparison condition of this float comparison instruction on vector. +func (i *Instruction) VFcmpData() (x, y Value, c FloatCmpCond, l VecLane) { + return i.v, i.v2, FloatCmpCond(i.u1), VecLane(i.u2) +} + +// ExtractlaneData returns the operands and sign flag of Extractlane on vector. +func (i *Instruction) ExtractlaneData() (x Value, index byte, signed bool, l VecLane) { + x = i.v + index = byte(0b00001111 & i.u1) + signed = i.u1>>32 != 0 + l = VecLane(i.u2) + return +} + +// InsertlaneData returns the operands and sign flag of Insertlane on vector. +func (i *Instruction) InsertlaneData() (x, y Value, index byte, l VecLane) { + x = i.v + y = i.v2 + index = byte(i.u1) + l = VecLane(i.u2) + return +} + +// AsFadd initializes this instruction as a floating-point addition instruction with OpcodeFadd. +func (i *Instruction) AsFadd(x, y Value) { + i.opcode = OpcodeFadd + i.v = x + i.v2 = y + i.typ = x.Type() +} + +// AsFsub initializes this instruction as a floating-point subtraction instruction with OpcodeFsub. +func (i *Instruction) AsFsub(x, y Value) { + i.opcode = OpcodeFsub + i.v = x + i.v2 = y + i.typ = x.Type() +} + +// AsFmul initializes this instruction as a floating-point multiplication instruction with OpcodeFmul. +func (i *Instruction) AsFmul(x, y Value) { + i.opcode = OpcodeFmul + i.v = x + i.v2 = y + i.typ = x.Type() +} + +// AsFdiv initializes this instruction as a floating-point division instruction with OpcodeFdiv. +func (i *Instruction) AsFdiv(x, y Value) { + i.opcode = OpcodeFdiv + i.v = x + i.v2 = y + i.typ = x.Type() +} + +// AsFmin initializes this instruction to take the minimum of two floating-points with OpcodeFmin. +func (i *Instruction) AsFmin(x, y Value) { + i.opcode = OpcodeFmin + i.v = x + i.v2 = y + i.typ = x.Type() +} + +// AsFmax initializes this instruction to take the maximum of two floating-points with OpcodeFmax. +func (i *Instruction) AsFmax(x, y Value) { + i.opcode = OpcodeFmax + i.v = x + i.v2 = y + i.typ = x.Type() +} + +// AsF32const initializes this instruction as a 32-bit floating-point constant instruction with OpcodeF32const. +func (i *Instruction) AsF32const(f float32) *Instruction { + i.opcode = OpcodeF32const + i.typ = TypeF64 + i.u1 = uint64(math.Float32bits(f)) + return i +} + +// AsF64const initializes this instruction as a 64-bit floating-point constant instruction with OpcodeF64const. +func (i *Instruction) AsF64const(f float64) *Instruction { + i.opcode = OpcodeF64const + i.typ = TypeF64 + i.u1 = math.Float64bits(f) + return i +} + +// AsVconst initializes this instruction as a vector constant instruction with OpcodeVconst. +func (i *Instruction) AsVconst(lo, hi uint64) *Instruction { + i.opcode = OpcodeVconst + i.typ = TypeV128 + i.u1 = lo + i.u2 = hi + return i +} + +// AsVbnot initializes this instruction as a vector negation instruction with OpcodeVbnot. +func (i *Instruction) AsVbnot(v Value) *Instruction { + i.opcode = OpcodeVbnot + i.typ = TypeV128 + i.v = v + return i +} + +// AsVband initializes this instruction as an and vector instruction with OpcodeVband. +func (i *Instruction) AsVband(x, y Value) *Instruction { + i.opcode = OpcodeVband + i.typ = TypeV128 + i.v = x + i.v2 = y + return i +} + +// AsVbor initializes this instruction as an or vector instruction with OpcodeVbor. +func (i *Instruction) AsVbor(x, y Value) *Instruction { + i.opcode = OpcodeVbor + i.typ = TypeV128 + i.v = x + i.v2 = y + return i +} + +// AsVbxor initializes this instruction as a xor vector instruction with OpcodeVbxor. +func (i *Instruction) AsVbxor(x, y Value) *Instruction { + i.opcode = OpcodeVbxor + i.typ = TypeV128 + i.v = x + i.v2 = y + return i +} + +// AsVbandnot initializes this instruction as an and-not vector instruction with OpcodeVbandnot. +func (i *Instruction) AsVbandnot(x, y Value) *Instruction { + i.opcode = OpcodeVbandnot + i.typ = TypeV128 + i.v = x + i.v2 = y + return i +} + +// AsVbitselect initializes this instruction as a bit select vector instruction with OpcodeVbitselect. +func (i *Instruction) AsVbitselect(c, x, y Value) *Instruction { + i.opcode = OpcodeVbitselect + i.typ = TypeV128 + i.v = c + i.v2 = x + i.v3 = y + return i +} + +// AsVanyTrue initializes this instruction as an anyTrue vector instruction with OpcodeVanyTrue. +func (i *Instruction) AsVanyTrue(x Value) *Instruction { + i.opcode = OpcodeVanyTrue + i.typ = TypeI32 + i.v = x + return i +} + +// AsVallTrue initializes this instruction as an allTrue vector instruction with OpcodeVallTrue. +func (i *Instruction) AsVallTrue(x Value, lane VecLane) *Instruction { + i.opcode = OpcodeVallTrue + i.typ = TypeI32 + i.v = x + i.u1 = uint64(lane) + return i +} + +// AsVhighBits initializes this instruction as a highBits vector instruction with OpcodeVhighBits. +func (i *Instruction) AsVhighBits(x Value, lane VecLane) *Instruction { + i.opcode = OpcodeVhighBits + i.typ = TypeI32 + i.v = x + i.u1 = uint64(lane) + return i +} + +// VconstData returns the operands of this vector constant instruction. +func (i *Instruction) VconstData() (lo, hi uint64) { + return i.u1, i.u2 +} + +// AsReturn initializes this instruction as a return instruction with OpcodeReturn. +func (i *Instruction) AsReturn(vs wazevoapi.VarLength[Value]) *Instruction { + i.opcode = OpcodeReturn + i.vs = vs + return i +} + +// AsIreduce initializes this instruction as a reduction instruction with OpcodeIreduce. +func (i *Instruction) AsIreduce(v Value, dstType Type) *Instruction { + i.opcode = OpcodeIreduce + i.v = v + i.typ = dstType + return i +} + +// AsWiden initializes this instruction as a signed or unsigned widen instruction +// on low half or high half of the given vector with OpcodeSwidenLow, OpcodeUwidenLow, OpcodeSwidenHigh, OpcodeUwidenHigh. +func (i *Instruction) AsWiden(v Value, lane VecLane, signed, low bool) *Instruction { + switch { + case signed && low: + i.opcode = OpcodeSwidenLow + case !signed && low: + i.opcode = OpcodeUwidenLow + case signed && !low: + i.opcode = OpcodeSwidenHigh + case !signed && !low: + i.opcode = OpcodeUwidenHigh + } + i.v = v + i.u1 = uint64(lane) + return i +} + +// AsAtomicLoad initializes this instruction as an atomic load. +// The size is in bytes and must be 1, 2, 4, or 8. +func (i *Instruction) AsAtomicLoad(addr Value, size uint64, typ Type) *Instruction { + i.opcode = OpcodeAtomicLoad + i.u1 = size + i.v = addr + i.typ = typ + return i +} + +// AsAtomicLoad initializes this instruction as an atomic store. +// The size is in bytes and must be 1, 2, 4, or 8. +func (i *Instruction) AsAtomicStore(addr, val Value, size uint64) *Instruction { + i.opcode = OpcodeAtomicStore + i.u1 = size + i.v = addr + i.v2 = val + i.typ = val.Type() + return i +} + +// AsAtomicRmw initializes this instruction as an atomic read-modify-write. +// The size is in bytes and must be 1, 2, 4, or 8. +func (i *Instruction) AsAtomicRmw(op AtomicRmwOp, addr, val Value, size uint64) *Instruction { + i.opcode = OpcodeAtomicRmw + i.u1 = uint64(op) + i.u2 = size + i.v = addr + i.v2 = val + i.typ = val.Type() + return i +} + +// AsAtomicCas initializes this instruction as an atomic compare-and-swap. +// The size is in bytes and must be 1, 2, 4, or 8. +func (i *Instruction) AsAtomicCas(addr, exp, repl Value, size uint64) *Instruction { + i.opcode = OpcodeAtomicCas + i.u1 = size + i.v = addr + i.v2 = exp + i.v3 = repl + i.typ = repl.Type() + return i +} + +// AsFence initializes this instruction as a memory fence. +// A single byte immediate may be used to indicate fence ordering in the future +// but is currently always 0 and ignored. +func (i *Instruction) AsFence(order byte) *Instruction { + i.opcode = OpcodeFence + i.u1 = uint64(order) + return i +} + +// AtomicRmwData returns the data for this atomic read-modify-write instruction. +func (i *Instruction) AtomicRmwData() (op AtomicRmwOp, size uint64) { + return AtomicRmwOp(i.u1), i.u2 +} + +// AtomicTargetSize returns the target memory size of the atomic instruction. +func (i *Instruction) AtomicTargetSize() (size uint64) { + return i.u1 +} + +// ReturnVals returns the return values of OpcodeReturn. +func (i *Instruction) ReturnVals() []Value { + return i.vs.View() +} + +// AsExitWithCode initializes this instruction as a trap instruction with OpcodeExitWithCode. +func (i *Instruction) AsExitWithCode(ctx Value, code wazevoapi.ExitCode) { + i.opcode = OpcodeExitWithCode + i.v = ctx + i.u1 = uint64(code) +} + +// AsExitIfTrueWithCode initializes this instruction as a trap instruction with OpcodeExitIfTrueWithCode. +func (i *Instruction) AsExitIfTrueWithCode(ctx, c Value, code wazevoapi.ExitCode) *Instruction { + i.opcode = OpcodeExitIfTrueWithCode + i.v = ctx + i.v2 = c + i.u1 = uint64(code) + return i +} + +// ExitWithCodeData returns the context and exit code of OpcodeExitWithCode. +func (i *Instruction) ExitWithCodeData() (ctx Value, code wazevoapi.ExitCode) { + return i.v, wazevoapi.ExitCode(i.u1) +} + +// ExitIfTrueWithCodeData returns the context and exit code of OpcodeExitWithCode. +func (i *Instruction) ExitIfTrueWithCodeData() (ctx, c Value, code wazevoapi.ExitCode) { + return i.v, i.v2, wazevoapi.ExitCode(i.u1) +} + +// InvertBrx inverts either OpcodeBrz or OpcodeBrnz to the other. +func (i *Instruction) InvertBrx() { + switch i.opcode { + case OpcodeBrz: + i.opcode = OpcodeBrnz + case OpcodeBrnz: + i.opcode = OpcodeBrz + default: + panic("BUG") + } +} + +// BranchData returns the branch data for this instruction necessary for backends. +func (i *Instruction) BranchData() (condVal Value, blockArgs []Value, target BasicBlock) { + switch i.opcode { + case OpcodeJump: + condVal = ValueInvalid + case OpcodeBrz, OpcodeBrnz: + condVal = i.v + default: + panic("BUG") + } + blockArgs = i.vs.View() + target = i.blk + return +} + +// BrTableData returns the branch table data for this instruction necessary for backends. +func (i *Instruction) BrTableData() (index Value, targets []BasicBlock) { + if i.opcode != OpcodeBrTable { + panic("BUG: BrTableData only available for OpcodeBrTable") + } + index = i.v + targets = i.targets + return +} + +// AsJump initializes this instruction as a jump instruction with OpcodeJump. +func (i *Instruction) AsJump(vs Values, target BasicBlock) *Instruction { + i.opcode = OpcodeJump + i.vs = vs + i.blk = target + return i +} + +// IsFallthroughJump returns true if this instruction is a fallthrough jump. +func (i *Instruction) IsFallthroughJump() bool { + if i.opcode != OpcodeJump { + panic("BUG: IsFallthrough only available for OpcodeJump") + } + return i.opcode == OpcodeJump && i.u1 != 0 +} + +// AsFallthroughJump marks this instruction as a fallthrough jump. +func (i *Instruction) AsFallthroughJump() { + if i.opcode != OpcodeJump { + panic("BUG: AsFallthroughJump only available for OpcodeJump") + } + i.u1 = 1 +} + +// AsBrz initializes this instruction as a branch-if-zero instruction with OpcodeBrz. +func (i *Instruction) AsBrz(v Value, args Values, target BasicBlock) { + i.opcode = OpcodeBrz + i.v = v + i.vs = args + i.blk = target +} + +// AsBrnz initializes this instruction as a branch-if-not-zero instruction with OpcodeBrnz. +func (i *Instruction) AsBrnz(v Value, args Values, target BasicBlock) *Instruction { + i.opcode = OpcodeBrnz + i.v = v + i.vs = args + i.blk = target + return i +} + +// AsBrTable initializes this instruction as a branch-table instruction with OpcodeBrTable. +func (i *Instruction) AsBrTable(index Value, targets []BasicBlock) { + i.opcode = OpcodeBrTable + i.v = index + i.targets = targets +} + +// AsCall initializes this instruction as a call instruction with OpcodeCall. +func (i *Instruction) AsCall(ref FuncRef, sig *Signature, args Values) { + i.opcode = OpcodeCall + i.u1 = uint64(ref) + i.vs = args + i.u2 = uint64(sig.ID) + sig.used = true +} + +// CallData returns the call data for this instruction necessary for backends. +func (i *Instruction) CallData() (ref FuncRef, sigID SignatureID, args []Value) { + if i.opcode != OpcodeCall { + panic("BUG: CallData only available for OpcodeCall") + } + ref = FuncRef(i.u1) + sigID = SignatureID(i.u2) + args = i.vs.View() + return +} + +// AsCallIndirect initializes this instruction as a call-indirect instruction with OpcodeCallIndirect. +func (i *Instruction) AsCallIndirect(funcPtr Value, sig *Signature, args Values) *Instruction { + i.opcode = OpcodeCallIndirect + i.typ = TypeF64 + i.vs = args + i.v = funcPtr + i.u1 = uint64(sig.ID) + sig.used = true + return i +} + +// AsCallGoRuntimeMemmove is the same as AsCallIndirect, but with a special flag set to indicate that it is a call to the Go runtime memmove function. +func (i *Instruction) AsCallGoRuntimeMemmove(funcPtr Value, sig *Signature, args Values) *Instruction { + i.AsCallIndirect(funcPtr, sig, args) + i.u2 = 1 + return i +} + +// CallIndirectData returns the call indirect data for this instruction necessary for backends. +func (i *Instruction) CallIndirectData() (funcPtr Value, sigID SignatureID, args []Value, isGoMemmove bool) { + if i.opcode != OpcodeCallIndirect { + panic("BUG: CallIndirectData only available for OpcodeCallIndirect") + } + funcPtr = i.v + sigID = SignatureID(i.u1) + args = i.vs.View() + isGoMemmove = i.u2 == 1 + return +} + +// AsClz initializes this instruction as a Count Leading Zeroes instruction with OpcodeClz. +func (i *Instruction) AsClz(x Value) { + i.opcode = OpcodeClz + i.v = x + i.typ = x.Type() +} + +// AsCtz initializes this instruction as a Count Trailing Zeroes instruction with OpcodeCtz. +func (i *Instruction) AsCtz(x Value) { + i.opcode = OpcodeCtz + i.v = x + i.typ = x.Type() +} + +// AsPopcnt initializes this instruction as a Population Count instruction with OpcodePopcnt. +func (i *Instruction) AsPopcnt(x Value) { + i.opcode = OpcodePopcnt + i.v = x + i.typ = x.Type() +} + +// AsFneg initializes this instruction as an instruction with OpcodeFneg. +func (i *Instruction) AsFneg(x Value) *Instruction { + i.opcode = OpcodeFneg + i.v = x + i.typ = x.Type() + return i +} + +// AsSqrt initializes this instruction as an instruction with OpcodeSqrt. +func (i *Instruction) AsSqrt(x Value) *Instruction { + i.opcode = OpcodeSqrt + i.v = x + i.typ = x.Type() + return i +} + +// AsFabs initializes this instruction as an instruction with OpcodeFabs. +func (i *Instruction) AsFabs(x Value) *Instruction { + i.opcode = OpcodeFabs + i.v = x + i.typ = x.Type() + return i +} + +// AsFcopysign initializes this instruction as an instruction with OpcodeFcopysign. +func (i *Instruction) AsFcopysign(x, y Value) *Instruction { + i.opcode = OpcodeFcopysign + i.v = x + i.v2 = y + i.typ = x.Type() + return i +} + +// AsCeil initializes this instruction as an instruction with OpcodeCeil. +func (i *Instruction) AsCeil(x Value) *Instruction { + i.opcode = OpcodeCeil + i.v = x + i.typ = x.Type() + return i +} + +// AsFloor initializes this instruction as an instruction with OpcodeFloor. +func (i *Instruction) AsFloor(x Value) *Instruction { + i.opcode = OpcodeFloor + i.v = x + i.typ = x.Type() + return i +} + +// AsTrunc initializes this instruction as an instruction with OpcodeTrunc. +func (i *Instruction) AsTrunc(x Value) *Instruction { + i.opcode = OpcodeTrunc + i.v = x + i.typ = x.Type() + return i +} + +// AsNearest initializes this instruction as an instruction with OpcodeNearest. +func (i *Instruction) AsNearest(x Value) *Instruction { + i.opcode = OpcodeNearest + i.v = x + i.typ = x.Type() + return i +} + +// AsBitcast initializes this instruction as an instruction with OpcodeBitcast. +func (i *Instruction) AsBitcast(x Value, dstType Type) *Instruction { + i.opcode = OpcodeBitcast + i.v = x + i.typ = dstType + return i +} + +// BitcastData returns the operands for a bitcast instruction. +func (i *Instruction) BitcastData() (x Value, dstType Type) { + return i.v, i.typ +} + +// AsFdemote initializes this instruction as an instruction with OpcodeFdemote. +func (i *Instruction) AsFdemote(x Value) { + i.opcode = OpcodeFdemote + i.v = x + i.typ = TypeF32 +} + +// AsFpromote initializes this instruction as an instruction with OpcodeFpromote. +func (i *Instruction) AsFpromote(x Value) { + i.opcode = OpcodeFpromote + i.v = x + i.typ = TypeF64 +} + +// AsFcvtFromInt initializes this instruction as an instruction with either OpcodeFcvtFromUint or OpcodeFcvtFromSint +func (i *Instruction) AsFcvtFromInt(x Value, signed bool, dst64bit bool) *Instruction { + if signed { + i.opcode = OpcodeFcvtFromSint + } else { + i.opcode = OpcodeFcvtFromUint + } + i.v = x + if dst64bit { + i.typ = TypeF64 + } else { + i.typ = TypeF32 + } + return i +} + +// AsFcvtToInt initializes this instruction as an instruction with either OpcodeFcvtToUint or OpcodeFcvtToSint +func (i *Instruction) AsFcvtToInt(x, ctx Value, signed bool, dst64bit bool, sat bool) *Instruction { + switch { + case signed && !sat: + i.opcode = OpcodeFcvtToSint + case !signed && !sat: + i.opcode = OpcodeFcvtToUint + case signed && sat: + i.opcode = OpcodeFcvtToSintSat + case !signed && sat: + i.opcode = OpcodeFcvtToUintSat + } + i.v = x + i.v2 = ctx + if dst64bit { + i.typ = TypeI64 + } else { + i.typ = TypeI32 + } + return i +} + +// AsVFcvtToIntSat initializes this instruction as an instruction with either OpcodeVFcvtToSintSat or OpcodeVFcvtToUintSat +func (i *Instruction) AsVFcvtToIntSat(x Value, lane VecLane, signed bool) *Instruction { + if signed { + i.opcode = OpcodeVFcvtToSintSat + } else { + i.opcode = OpcodeVFcvtToUintSat + } + i.v = x + i.u1 = uint64(lane) + return i +} + +// AsVFcvtFromInt initializes this instruction as an instruction with either OpcodeVFcvtToSintSat or OpcodeVFcvtToUintSat +func (i *Instruction) AsVFcvtFromInt(x Value, lane VecLane, signed bool) *Instruction { + if signed { + i.opcode = OpcodeVFcvtFromSint + } else { + i.opcode = OpcodeVFcvtFromUint + } + i.v = x + i.u1 = uint64(lane) + return i +} + +// AsNarrow initializes this instruction as an instruction with either OpcodeSnarrow or OpcodeUnarrow +func (i *Instruction) AsNarrow(x, y Value, lane VecLane, signed bool) *Instruction { + if signed { + i.opcode = OpcodeSnarrow + } else { + i.opcode = OpcodeUnarrow + } + i.v = x + i.v2 = y + i.u1 = uint64(lane) + return i +} + +// AsFvpromoteLow initializes this instruction as an instruction with OpcodeFvpromoteLow +func (i *Instruction) AsFvpromoteLow(x Value, lane VecLane) *Instruction { + i.opcode = OpcodeFvpromoteLow + i.v = x + i.u1 = uint64(lane) + return i +} + +// AsFvdemote initializes this instruction as an instruction with OpcodeFvdemote +func (i *Instruction) AsFvdemote(x Value, lane VecLane) *Instruction { + i.opcode = OpcodeFvdemote + i.v = x + i.u1 = uint64(lane) + return i +} + +// AsSExtend initializes this instruction as a sign extension instruction with OpcodeSExtend. +func (i *Instruction) AsSExtend(v Value, from, to byte) *Instruction { + i.opcode = OpcodeSExtend + i.v = v + i.u1 = uint64(from)<<8 | uint64(to) + if to == 64 { + i.typ = TypeI64 + } else { + i.typ = TypeI32 + } + return i +} + +// AsUExtend initializes this instruction as an unsigned extension instruction with OpcodeUExtend. +func (i *Instruction) AsUExtend(v Value, from, to byte) *Instruction { + i.opcode = OpcodeUExtend + i.v = v + i.u1 = uint64(from)<<8 | uint64(to) + if to == 64 { + i.typ = TypeI64 + } else { + i.typ = TypeI32 + } + return i +} + +func (i *Instruction) ExtendData() (from, to byte, signed bool) { + if i.opcode != OpcodeSExtend && i.opcode != OpcodeUExtend { + panic("BUG: ExtendData only available for OpcodeSExtend and OpcodeUExtend") + } + from = byte(i.u1 >> 8) + to = byte(i.u1) + signed = i.opcode == OpcodeSExtend + return +} + +// AsSelect initializes this instruction as an unsigned extension instruction with OpcodeSelect. +func (i *Instruction) AsSelect(c, x, y Value) *Instruction { + i.opcode = OpcodeSelect + i.v = c + i.v2 = x + i.v3 = y + i.typ = x.Type() + return i +} + +// SelectData returns the select data for this instruction necessary for backends. +func (i *Instruction) SelectData() (c, x, y Value) { + c = i.v + x = i.v2 + y = i.v3 + return +} + +// ExtendFromToBits returns the from and to bit size for the extension instruction. +func (i *Instruction) ExtendFromToBits() (from, to byte) { + from = byte(i.u1 >> 8) + to = byte(i.u1) + return +} + +// Format returns a string representation of this instruction with the given builder. +// For debugging purposes only. +func (i *Instruction) Format(b Builder) string { + var instSuffix string + switch i.opcode { + case OpcodeExitWithCode: + instSuffix = fmt.Sprintf(" %s, %s", i.v.Format(b), wazevoapi.ExitCode(i.u1)) + case OpcodeExitIfTrueWithCode: + instSuffix = fmt.Sprintf(" %s, %s, %s", i.v2.Format(b), i.v.Format(b), wazevoapi.ExitCode(i.u1)) + case OpcodeIadd, OpcodeIsub, OpcodeImul, OpcodeFadd, OpcodeFsub, OpcodeFmin, OpcodeFmax, OpcodeFdiv, OpcodeFmul: + instSuffix = fmt.Sprintf(" %s, %s", i.v.Format(b), i.v2.Format(b)) + case OpcodeIcmp: + instSuffix = fmt.Sprintf(" %s, %s, %s", IntegerCmpCond(i.u1), i.v.Format(b), i.v2.Format(b)) + case OpcodeFcmp: + instSuffix = fmt.Sprintf(" %s, %s, %s", FloatCmpCond(i.u1), i.v.Format(b), i.v2.Format(b)) + case OpcodeSExtend, OpcodeUExtend: + instSuffix = fmt.Sprintf(" %s, %d->%d", i.v.Format(b), i.u1>>8, i.u1&0xff) + case OpcodeCall, OpcodeCallIndirect: + view := i.vs.View() + vs := make([]string, len(view)) + for idx := range vs { + vs[idx] = view[idx].Format(b) + } + if i.opcode == OpcodeCallIndirect { + instSuffix = fmt.Sprintf(" %s:%s, %s", i.v.Format(b), SignatureID(i.u1), strings.Join(vs, ", ")) + } else { + instSuffix = fmt.Sprintf(" %s:%s, %s", FuncRef(i.u1), SignatureID(i.u2), strings.Join(vs, ", ")) + } + case OpcodeStore, OpcodeIstore8, OpcodeIstore16, OpcodeIstore32: + instSuffix = fmt.Sprintf(" %s, %s, %#x", i.v.Format(b), i.v2.Format(b), uint32(i.u1)) + case OpcodeLoad, OpcodeVZeroExtLoad: + instSuffix = fmt.Sprintf(" %s, %#x", i.v.Format(b), int32(i.u1)) + case OpcodeLoadSplat: + instSuffix = fmt.Sprintf(".%s %s, %#x", VecLane(i.u2), i.v.Format(b), int32(i.u1)) + case OpcodeUload8, OpcodeUload16, OpcodeUload32, OpcodeSload8, OpcodeSload16, OpcodeSload32: + instSuffix = fmt.Sprintf(" %s, %#x", i.v.Format(b), int32(i.u1)) + case OpcodeSelect, OpcodeVbitselect: + instSuffix = fmt.Sprintf(" %s, %s, %s", i.v.Format(b), i.v2.Format(b), i.v3.Format(b)) + case OpcodeIconst: + switch i.typ { + case TypeI32: + instSuffix = fmt.Sprintf("_32 %#x", uint32(i.u1)) + case TypeI64: + instSuffix = fmt.Sprintf("_64 %#x", i.u1) + } + case OpcodeVconst: + instSuffix = fmt.Sprintf(" %016x %016x", i.u1, i.u2) + case OpcodeF32const: + instSuffix = fmt.Sprintf(" %f", math.Float32frombits(uint32(i.u1))) + case OpcodeF64const: + instSuffix = fmt.Sprintf(" %f", math.Float64frombits(i.u1)) + case OpcodeReturn: + view := i.vs.View() + if len(view) == 0 { + break + } + vs := make([]string, len(view)) + for idx := range vs { + vs[idx] = view[idx].Format(b) + } + instSuffix = fmt.Sprintf(" %s", strings.Join(vs, ", ")) + case OpcodeJump: + view := i.vs.View() + vs := make([]string, len(view)+1) + if i.IsFallthroughJump() { + vs[0] = " fallthrough" + } else { + vs[0] = " " + i.blk.(*basicBlock).Name() + } + for idx := range view { + vs[idx+1] = view[idx].Format(b) + } + + instSuffix = strings.Join(vs, ", ") + case OpcodeBrz, OpcodeBrnz: + view := i.vs.View() + vs := make([]string, len(view)+2) + vs[0] = " " + i.v.Format(b) + vs[1] = i.blk.(*basicBlock).Name() + for idx := range view { + vs[idx+2] = view[idx].Format(b) + } + instSuffix = strings.Join(vs, ", ") + case OpcodeBrTable: + // `BrTable index, [label1, label2, ... labelN]` + instSuffix = fmt.Sprintf(" %s", i.v.Format(b)) + instSuffix += ", [" + for i, target := range i.targets { + blk := target.(*basicBlock) + if i == 0 { + instSuffix += blk.Name() + } else { + instSuffix += ", " + blk.Name() + } + } + instSuffix += "]" + case OpcodeBand, OpcodeBor, OpcodeBxor, OpcodeRotr, OpcodeRotl, OpcodeIshl, OpcodeSshr, OpcodeUshr, + OpcodeSdiv, OpcodeUdiv, OpcodeFcopysign, OpcodeSrem, OpcodeUrem, + OpcodeVbnot, OpcodeVbxor, OpcodeVbor, OpcodeVband, OpcodeVbandnot, OpcodeVIcmp, OpcodeVFcmp: + instSuffix = fmt.Sprintf(" %s, %s", i.v.Format(b), i.v2.Format(b)) + case OpcodeUndefined: + case OpcodeClz, OpcodeCtz, OpcodePopcnt, OpcodeFneg, OpcodeFcvtToSint, OpcodeFcvtToUint, OpcodeFcvtFromSint, + OpcodeFcvtFromUint, OpcodeFcvtToSintSat, OpcodeFcvtToUintSat, OpcodeFdemote, OpcodeFpromote, OpcodeIreduce, OpcodeBitcast, OpcodeSqrt, OpcodeFabs, + OpcodeCeil, OpcodeFloor, OpcodeTrunc, OpcodeNearest: + instSuffix = " " + i.v.Format(b) + case OpcodeVIadd, OpcodeExtIaddPairwise, OpcodeVSaddSat, OpcodeVUaddSat, OpcodeVIsub, OpcodeVSsubSat, OpcodeVUsubSat, + OpcodeVImin, OpcodeVUmin, OpcodeVImax, OpcodeVUmax, OpcodeVImul, OpcodeVAvgRound, + OpcodeVFadd, OpcodeVFsub, OpcodeVFmul, OpcodeVFdiv, + OpcodeVIshl, OpcodeVSshr, OpcodeVUshr, + OpcodeVFmin, OpcodeVFmax, OpcodeVMinPseudo, OpcodeVMaxPseudo, + OpcodeSnarrow, OpcodeUnarrow, OpcodeSwizzle, OpcodeSqmulRoundSat: + instSuffix = fmt.Sprintf(".%s %s, %s", VecLane(i.u1), i.v.Format(b), i.v2.Format(b)) + case OpcodeVIabs, OpcodeVIneg, OpcodeVIpopcnt, OpcodeVhighBits, OpcodeVallTrue, OpcodeVanyTrue, + OpcodeVFabs, OpcodeVFneg, OpcodeVSqrt, OpcodeVCeil, OpcodeVFloor, OpcodeVTrunc, OpcodeVNearest, + OpcodeVFcvtToUintSat, OpcodeVFcvtToSintSat, OpcodeVFcvtFromUint, OpcodeVFcvtFromSint, + OpcodeFvpromoteLow, OpcodeFvdemote, OpcodeSwidenLow, OpcodeUwidenLow, OpcodeSwidenHigh, OpcodeUwidenHigh, + OpcodeSplat: + instSuffix = fmt.Sprintf(".%s %s", VecLane(i.u1), i.v.Format(b)) + case OpcodeExtractlane: + var signedness string + if i.u1 != 0 { + signedness = "signed" + } else { + signedness = "unsigned" + } + instSuffix = fmt.Sprintf(".%s %d, %s (%s)", VecLane(i.u2), 0x0000FFFF&i.u1, i.v.Format(b), signedness) + case OpcodeInsertlane: + instSuffix = fmt.Sprintf(".%s %d, %s, %s", VecLane(i.u2), i.u1, i.v.Format(b), i.v2.Format(b)) + case OpcodeShuffle: + lanes := make([]byte, 16) + for idx := 0; idx < 8; idx++ { + lanes[idx] = byte(i.u1 >> (8 * idx)) + } + for idx := 0; idx < 8; idx++ { + lanes[idx+8] = byte(i.u2 >> (8 * idx)) + } + // Prints Shuffle.[0 1 2 3 4 5 6 7 ...] v2, v3 + instSuffix = fmt.Sprintf(".%v %s, %s", lanes, i.v.Format(b), i.v2.Format(b)) + case OpcodeAtomicRmw: + instSuffix = fmt.Sprintf(" %s_%d, %s, %s", AtomicRmwOp(i.u1), 8*i.u2, i.v.Format(b), i.v2.Format(b)) + case OpcodeAtomicLoad: + instSuffix = fmt.Sprintf("_%d, %s", 8*i.u1, i.v.Format(b)) + case OpcodeAtomicStore: + instSuffix = fmt.Sprintf("_%d, %s, %s", 8*i.u1, i.v.Format(b), i.v2.Format(b)) + case OpcodeAtomicCas: + instSuffix = fmt.Sprintf("_%d, %s, %s, %s", 8*i.u1, i.v.Format(b), i.v2.Format(b), i.v3.Format(b)) + case OpcodeFence: + instSuffix = fmt.Sprintf(" %d", i.u1) + case OpcodeWideningPairwiseDotProductS: + instSuffix = fmt.Sprintf(" %s, %s", i.v.Format(b), i.v2.Format(b)) + default: + panic(fmt.Sprintf("TODO: format for %s", i.opcode)) + } + + instr := i.opcode.String() + instSuffix + + var rvs []string + if rv := i.rValue; rv.Valid() { + rvs = append(rvs, rv.formatWithType(b)) + } + + for _, v := range i.rValues.View() { + rvs = append(rvs, v.formatWithType(b)) + } + + if len(rvs) > 0 { + return fmt.Sprintf("%s = %s", strings.Join(rvs, ", "), instr) + } else { + return instr + } +} + +// addArgumentBranchInst adds an argument to this instruction. +func (i *Instruction) addArgumentBranchInst(b *builder, v Value) { + switch i.opcode { + case OpcodeJump, OpcodeBrz, OpcodeBrnz: + i.vs = i.vs.Append(&b.varLengthPool, v) + default: + panic("BUG: " + i.opcode.String()) + } +} + +// Constant returns true if this instruction is a constant instruction. +func (i *Instruction) Constant() bool { + switch i.opcode { + case OpcodeIconst, OpcodeF32const, OpcodeF64const: + return true + } + return false +} + +// ConstantVal returns the constant value of this instruction. +// How to interpret the return value depends on the opcode. +func (i *Instruction) ConstantVal() (ret uint64) { + switch i.opcode { + case OpcodeIconst, OpcodeF32const, OpcodeF64const: + ret = i.u1 + default: + panic("TODO") + } + return +} + +// String implements fmt.Stringer. +func (o Opcode) String() (ret string) { + switch o { + case OpcodeInvalid: + return "invalid" + case OpcodeUndefined: + return "Undefined" + case OpcodeJump: + return "Jump" + case OpcodeBrz: + return "Brz" + case OpcodeBrnz: + return "Brnz" + case OpcodeBrTable: + return "BrTable" + case OpcodeExitWithCode: + return "Exit" + case OpcodeExitIfTrueWithCode: + return "ExitIfTrue" + case OpcodeReturn: + return "Return" + case OpcodeCall: + return "Call" + case OpcodeCallIndirect: + return "CallIndirect" + case OpcodeSplat: + return "Splat" + case OpcodeSwizzle: + return "Swizzle" + case OpcodeInsertlane: + return "Insertlane" + case OpcodeExtractlane: + return "Extractlane" + case OpcodeLoad: + return "Load" + case OpcodeLoadSplat: + return "LoadSplat" + case OpcodeStore: + return "Store" + case OpcodeUload8: + return "Uload8" + case OpcodeSload8: + return "Sload8" + case OpcodeIstore8: + return "Istore8" + case OpcodeUload16: + return "Uload16" + case OpcodeSload16: + return "Sload16" + case OpcodeIstore16: + return "Istore16" + case OpcodeUload32: + return "Uload32" + case OpcodeSload32: + return "Sload32" + case OpcodeIstore32: + return "Istore32" + case OpcodeIconst: + return "Iconst" + case OpcodeF32const: + return "F32const" + case OpcodeF64const: + return "F64const" + case OpcodeVconst: + return "Vconst" + case OpcodeShuffle: + return "Shuffle" + case OpcodeSelect: + return "Select" + case OpcodeVanyTrue: + return "VanyTrue" + case OpcodeVallTrue: + return "VallTrue" + case OpcodeVhighBits: + return "VhighBits" + case OpcodeIcmp: + return "Icmp" + case OpcodeIcmpImm: + return "IcmpImm" + case OpcodeVIcmp: + return "VIcmp" + case OpcodeIadd: + return "Iadd" + case OpcodeIsub: + return "Isub" + case OpcodeImul: + return "Imul" + case OpcodeUdiv: + return "Udiv" + case OpcodeSdiv: + return "Sdiv" + case OpcodeUrem: + return "Urem" + case OpcodeSrem: + return "Srem" + case OpcodeBand: + return "Band" + case OpcodeBor: + return "Bor" + case OpcodeBxor: + return "Bxor" + case OpcodeBnot: + return "Bnot" + case OpcodeRotl: + return "Rotl" + case OpcodeRotr: + return "Rotr" + case OpcodeIshl: + return "Ishl" + case OpcodeUshr: + return "Ushr" + case OpcodeSshr: + return "Sshr" + case OpcodeClz: + return "Clz" + case OpcodeCtz: + return "Ctz" + case OpcodePopcnt: + return "Popcnt" + case OpcodeFcmp: + return "Fcmp" + case OpcodeFadd: + return "Fadd" + case OpcodeFsub: + return "Fsub" + case OpcodeFmul: + return "Fmul" + case OpcodeFdiv: + return "Fdiv" + case OpcodeSqmulRoundSat: + return "SqmulRoundSat" + case OpcodeSqrt: + return "Sqrt" + case OpcodeFneg: + return "Fneg" + case OpcodeFabs: + return "Fabs" + case OpcodeFcopysign: + return "Fcopysign" + case OpcodeFmin: + return "Fmin" + case OpcodeFmax: + return "Fmax" + case OpcodeCeil: + return "Ceil" + case OpcodeFloor: + return "Floor" + case OpcodeTrunc: + return "Trunc" + case OpcodeNearest: + return "Nearest" + case OpcodeBitcast: + return "Bitcast" + case OpcodeIreduce: + return "Ireduce" + case OpcodeSnarrow: + return "Snarrow" + case OpcodeUnarrow: + return "Unarrow" + case OpcodeSwidenLow: + return "SwidenLow" + case OpcodeSwidenHigh: + return "SwidenHigh" + case OpcodeUwidenLow: + return "UwidenLow" + case OpcodeUwidenHigh: + return "UwidenHigh" + case OpcodeExtIaddPairwise: + return "IaddPairwise" + case OpcodeWideningPairwiseDotProductS: + return "WideningPairwiseDotProductS" + case OpcodeUExtend: + return "UExtend" + case OpcodeSExtend: + return "SExtend" + case OpcodeFpromote: + return "Fpromote" + case OpcodeFdemote: + return "Fdemote" + case OpcodeFvdemote: + return "Fvdemote" + case OpcodeFcvtToUint: + return "FcvtToUint" + case OpcodeFcvtToSint: + return "FcvtToSint" + case OpcodeFcvtToUintSat: + return "FcvtToUintSat" + case OpcodeFcvtToSintSat: + return "FcvtToSintSat" + case OpcodeFcvtFromUint: + return "FcvtFromUint" + case OpcodeFcvtFromSint: + return "FcvtFromSint" + case OpcodeAtomicRmw: + return "AtomicRmw" + case OpcodeAtomicCas: + return "AtomicCas" + case OpcodeAtomicLoad: + return "AtomicLoad" + case OpcodeAtomicStore: + return "AtomicStore" + case OpcodeFence: + return "Fence" + case OpcodeVbor: + return "Vbor" + case OpcodeVbxor: + return "Vbxor" + case OpcodeVband: + return "Vband" + case OpcodeVbandnot: + return "Vbandnot" + case OpcodeVbnot: + return "Vbnot" + case OpcodeVbitselect: + return "Vbitselect" + case OpcodeVIadd: + return "VIadd" + case OpcodeVSaddSat: + return "VSaddSat" + case OpcodeVUaddSat: + return "VUaddSat" + case OpcodeVSsubSat: + return "VSsubSat" + case OpcodeVUsubSat: + return "VUsubSat" + case OpcodeVAvgRound: + return "OpcodeVAvgRound" + case OpcodeVIsub: + return "VIsub" + case OpcodeVImin: + return "VImin" + case OpcodeVUmin: + return "VUmin" + case OpcodeVImax: + return "VImax" + case OpcodeVUmax: + return "VUmax" + case OpcodeVImul: + return "VImul" + case OpcodeVIabs: + return "VIabs" + case OpcodeVIneg: + return "VIneg" + case OpcodeVIpopcnt: + return "VIpopcnt" + case OpcodeVIshl: + return "VIshl" + case OpcodeVUshr: + return "VUshr" + case OpcodeVSshr: + return "VSshr" + case OpcodeVFabs: + return "VFabs" + case OpcodeVFmax: + return "VFmax" + case OpcodeVFmin: + return "VFmin" + case OpcodeVFneg: + return "VFneg" + case OpcodeVFadd: + return "VFadd" + case OpcodeVFsub: + return "VFsub" + case OpcodeVFmul: + return "VFmul" + case OpcodeVFdiv: + return "VFdiv" + case OpcodeVFcmp: + return "VFcmp" + case OpcodeVCeil: + return "VCeil" + case OpcodeVFloor: + return "VFloor" + case OpcodeVTrunc: + return "VTrunc" + case OpcodeVNearest: + return "VNearest" + case OpcodeVMaxPseudo: + return "VMaxPseudo" + case OpcodeVMinPseudo: + return "VMinPseudo" + case OpcodeVSqrt: + return "VSqrt" + case OpcodeVFcvtToUintSat: + return "VFcvtToUintSat" + case OpcodeVFcvtToSintSat: + return "VFcvtToSintSat" + case OpcodeVFcvtFromUint: + return "VFcvtFromUint" + case OpcodeVFcvtFromSint: + return "VFcvtFromSint" + case OpcodeFvpromoteLow: + return "FvpromoteLow" + case OpcodeVZeroExtLoad: + return "VZeroExtLoad" + } + panic(fmt.Sprintf("unknown opcode %d", o)) +} diff --git a/vendor/github.com/tetratelabs/wazero/internal/engine/wazevo/ssa/pass.go b/vendor/github.com/tetratelabs/wazero/internal/engine/wazevo/ssa/pass.go new file mode 100644 index 000000000..a2e986cd1 --- /dev/null +++ b/vendor/github.com/tetratelabs/wazero/internal/engine/wazevo/ssa/pass.go @@ -0,0 +1,417 @@ +package ssa + +import ( + "fmt" + + "github.com/tetratelabs/wazero/internal/engine/wazevo/wazevoapi" +) + +// RunPasses implements Builder.RunPasses. +// +// The order here matters; some pass depends on the previous ones. +// +// Note that passes suffixed with "Opt" are the optimization passes, meaning that they edit the instructions and blocks +// while the other passes are not, like passEstimateBranchProbabilities does not edit them, but only calculates the additional information. +func (b *builder) RunPasses() { + b.runPreBlockLayoutPasses() + b.runBlockLayoutPass() + b.runPostBlockLayoutPasses() + b.runFinalizingPasses() +} + +func (b *builder) runPreBlockLayoutPasses() { + passSortSuccessors(b) + passDeadBlockEliminationOpt(b) + passRedundantPhiEliminationOpt(b) + // The result of passCalculateImmediateDominators will be used by various passes below. + passCalculateImmediateDominators(b) + passNopInstElimination(b) + + // TODO: implement either conversion of irreducible CFG into reducible one, or irreducible CFG detection where we panic. + // WebAssembly program shouldn't result in irreducible CFG, but we should handle it properly in just in case. + // See FixIrreducible pass in LLVM: https://llvm.org/doxygen/FixIrreducible_8cpp_source.html + + // TODO: implement more optimization passes like: + // block coalescing. + // Copy-propagation. + // Constant folding. + // Common subexpression elimination. + // Arithmetic simplifications. + // and more! + + // passDeadCodeEliminationOpt could be more accurate if we do this after other optimizations. + passDeadCodeEliminationOpt(b) + b.donePreBlockLayoutPasses = true +} + +func (b *builder) runBlockLayoutPass() { + if !b.donePreBlockLayoutPasses { + panic("runBlockLayoutPass must be called after all pre passes are done") + } + passLayoutBlocks(b) + b.doneBlockLayout = true +} + +// runPostBlockLayoutPasses runs the post block layout passes. After this point, CFG is somewhat stable, +// but still can be modified before finalizing passes. At this point, critical edges are split by passLayoutBlocks. +func (b *builder) runPostBlockLayoutPasses() { + if !b.doneBlockLayout { + panic("runPostBlockLayoutPasses must be called after block layout pass is done") + } + // TODO: Do more. e.g. tail duplication, loop unrolling, etc. + + b.donePostBlockLayoutPasses = true +} + +// runFinalizingPasses runs the finalizing passes. After this point, CFG should not be modified. +func (b *builder) runFinalizingPasses() { + if !b.donePostBlockLayoutPasses { + panic("runFinalizingPasses must be called after post block layout passes are done") + } + // Critical edges are split, so we fix the loop nesting forest. + passBuildLoopNestingForest(b) + passBuildDominatorTree(b) + // Now that we know the final placement of the blocks, we can explicitly mark the fallthrough jumps. + b.markFallthroughJumps() +} + +// passDeadBlockEliminationOpt searches the unreachable blocks, and sets the basicBlock.invalid flag true if so. +func passDeadBlockEliminationOpt(b *builder) { + entryBlk := b.entryBlk() + b.clearBlkVisited() + b.blkStack = append(b.blkStack, entryBlk) + for len(b.blkStack) > 0 { + reachableBlk := b.blkStack[len(b.blkStack)-1] + b.blkStack = b.blkStack[:len(b.blkStack)-1] + b.blkVisited[reachableBlk] = 0 // the value won't be used in this pass. + + if !reachableBlk.sealed && !reachableBlk.ReturnBlock() { + panic(fmt.Sprintf("%s is not sealed", reachableBlk)) + } + + if wazevoapi.SSAValidationEnabled { + reachableBlk.validate(b) + } + + for _, succ := range reachableBlk.success { + if _, ok := b.blkVisited[succ]; ok { + continue + } + b.blkStack = append(b.blkStack, succ) + } + } + + for blk := b.blockIteratorBegin(); blk != nil; blk = b.blockIteratorNext() { + if _, ok := b.blkVisited[blk]; !ok { + blk.invalid = true + } + } +} + +// passRedundantPhiEliminationOpt eliminates the redundant PHIs (in our terminology, parameters of a block). +func passRedundantPhiEliminationOpt(b *builder) { + redundantParameterIndexes := b.ints[:0] // reuse the slice from previous iterations. + + // TODO: this might be costly for large programs, but at least, as far as I did the experiment, it's almost the + // same as the single iteration version in terms of the overall compilation time. That *might be* mostly thanks to the fact + // that removing many PHIs results in the reduction of the total instructions, not because of this indefinite iteration is + // relatively small. For example, sqlite speedtest binary results in the large number of redundant PHIs, + // the maximum number of iteration was 22, which seems to be acceptable but not that small either since the + // complexity here is O(BlockNum * Iterations) at the worst case where BlockNum might be the order of thousands. + for { + changed := false + _ = b.blockIteratorBegin() // skip entry block! + // Below, we intentionally use the named iteration variable name, as this comes with inevitable nested for loops! + for blk := b.blockIteratorNext(); blk != nil; blk = b.blockIteratorNext() { + paramNum := len(blk.params) + + for paramIndex := 0; paramIndex < paramNum; paramIndex++ { + phiValue := blk.params[paramIndex].value + redundant := true + + nonSelfReferencingValue := ValueInvalid + for predIndex := range blk.preds { + br := blk.preds[predIndex].branch + // Resolve the alias in the arguments so that we could use the previous iteration's result. + b.resolveArgumentAlias(br) + pred := br.vs.View()[paramIndex] + if pred == phiValue { + // This is self-referencing: PHI from the same PHI. + continue + } + + if !nonSelfReferencingValue.Valid() { + nonSelfReferencingValue = pred + continue + } + + if nonSelfReferencingValue != pred { + redundant = false + break + } + } + + if !nonSelfReferencingValue.Valid() { + // This shouldn't happen, and must be a bug in builder.go. + panic("BUG: params added but only self-referencing") + } + + if redundant { + b.redundantParameterIndexToValue[paramIndex] = nonSelfReferencingValue + redundantParameterIndexes = append(redundantParameterIndexes, paramIndex) + } + } + + if len(b.redundantParameterIndexToValue) == 0 { + continue + } + changed = true + + // Remove the redundant PHIs from the argument list of branching instructions. + for predIndex := range blk.preds { + var cur int + predBlk := blk.preds[predIndex] + branchInst := predBlk.branch + view := branchInst.vs.View() + for argIndex, value := range view { + if _, ok := b.redundantParameterIndexToValue[argIndex]; !ok { + view[cur] = value + cur++ + } + } + branchInst.vs.Cut(cur) + } + + // Still need to have the definition of the value of the PHI (previously as the parameter). + for _, redundantParamIndex := range redundantParameterIndexes { + phiValue := blk.params[redundantParamIndex].value + onlyValue := b.redundantParameterIndexToValue[redundantParamIndex] + // Create an alias in this block from the only phi argument to the phi value. + b.alias(phiValue, onlyValue) + } + + // Finally, Remove the param from the blk. + var cur int + for paramIndex := 0; paramIndex < paramNum; paramIndex++ { + param := blk.params[paramIndex] + if _, ok := b.redundantParameterIndexToValue[paramIndex]; !ok { + blk.params[cur] = param + cur++ + } + } + blk.params = blk.params[:cur] + + // Clears the map for the next iteration. + for _, paramIndex := range redundantParameterIndexes { + delete(b.redundantParameterIndexToValue, paramIndex) + } + redundantParameterIndexes = redundantParameterIndexes[:0] + } + + if !changed { + break + } + } + + // Reuse the slice for the future passes. + b.ints = redundantParameterIndexes +} + +// passDeadCodeEliminationOpt traverses all the instructions, and calculates the reference count of each Value, and +// eliminates all the unnecessary instructions whose ref count is zero. +// The results are stored at builder.valueRefCounts. This also assigns a InstructionGroupID to each Instruction +// during the process. This is the last SSA-level optimization pass and after this, +// the SSA function is ready to be used by backends. +// +// TODO: the algorithm here might not be efficient. Get back to this later. +func passDeadCodeEliminationOpt(b *builder) { + nvid := int(b.nextValueID) + if nvid >= len(b.valueRefCounts) { + b.valueRefCounts = append(b.valueRefCounts, make([]int, b.nextValueID)...) + } + if nvid >= len(b.valueIDToInstruction) { + b.valueIDToInstruction = append(b.valueIDToInstruction, make([]*Instruction, b.nextValueID)...) + } + + // First, we gather all the instructions with side effects. + liveInstructions := b.instStack[:0] + // During the process, we will assign InstructionGroupID to each instruction, which is not + // relevant to dead code elimination, but we need in the backend. + var gid InstructionGroupID + for blk := b.blockIteratorBegin(); blk != nil; blk = b.blockIteratorNext() { + for cur := blk.rootInstr; cur != nil; cur = cur.next { + cur.gid = gid + switch cur.sideEffect() { + case sideEffectTraps: + // The trappable should always be alive. + liveInstructions = append(liveInstructions, cur) + case sideEffectStrict: + liveInstructions = append(liveInstructions, cur) + // The strict side effect should create different instruction groups. + gid++ + } + + r1, rs := cur.Returns() + if r1.Valid() { + b.valueIDToInstruction[r1.ID()] = cur + } + for _, r := range rs { + b.valueIDToInstruction[r.ID()] = cur + } + } + } + + // Find all the instructions referenced by live instructions transitively. + for len(liveInstructions) > 0 { + tail := len(liveInstructions) - 1 + live := liveInstructions[tail] + liveInstructions = liveInstructions[:tail] + if live.live { + // If it's already marked alive, this is referenced multiple times, + // so we can skip it. + continue + } + live.live = true + + // Before we walk, we need to resolve the alias first. + b.resolveArgumentAlias(live) + + v1, v2, v3, vs := live.Args() + if v1.Valid() { + producingInst := b.valueIDToInstruction[v1.ID()] + if producingInst != nil { + liveInstructions = append(liveInstructions, producingInst) + } + } + + if v2.Valid() { + producingInst := b.valueIDToInstruction[v2.ID()] + if producingInst != nil { + liveInstructions = append(liveInstructions, producingInst) + } + } + + if v3.Valid() { + producingInst := b.valueIDToInstruction[v3.ID()] + if producingInst != nil { + liveInstructions = append(liveInstructions, producingInst) + } + } + + for _, v := range vs { + producingInst := b.valueIDToInstruction[v.ID()] + if producingInst != nil { + liveInstructions = append(liveInstructions, producingInst) + } + } + } + + // Now that all the live instructions are flagged as live=true, we eliminate all dead instructions. + for blk := b.blockIteratorBegin(); blk != nil; blk = b.blockIteratorNext() { + for cur := blk.rootInstr; cur != nil; cur = cur.next { + if !cur.live { + // Remove the instruction from the list. + if prev := cur.prev; prev != nil { + prev.next = cur.next + } else { + blk.rootInstr = cur.next + } + if next := cur.next; next != nil { + next.prev = cur.prev + } + continue + } + + // If the value alive, we can be sure that arguments are used definitely. + // Hence, we can increment the value reference counts. + v1, v2, v3, vs := cur.Args() + if v1.Valid() { + b.incRefCount(v1.ID(), cur) + } + if v2.Valid() { + b.incRefCount(v2.ID(), cur) + } + if v3.Valid() { + b.incRefCount(v3.ID(), cur) + } + for _, v := range vs { + b.incRefCount(v.ID(), cur) + } + } + } + + b.instStack = liveInstructions // we reuse the stack for the next iteration. +} + +func (b *builder) incRefCount(id ValueID, from *Instruction) { + if wazevoapi.SSALoggingEnabled { + fmt.Printf("v%d referenced from %v\n", id, from.Format(b)) + } + b.valueRefCounts[id]++ +} + +// clearBlkVisited clears the b.blkVisited map so that we can reuse it for multiple places. +func (b *builder) clearBlkVisited() { + b.blkStack2 = b.blkStack2[:0] + for key := range b.blkVisited { + b.blkStack2 = append(b.blkStack2, key) + } + for _, blk := range b.blkStack2 { + delete(b.blkVisited, blk) + } + b.blkStack2 = b.blkStack2[:0] +} + +// passNopInstElimination eliminates the instructions which is essentially a no-op. +func passNopInstElimination(b *builder) { + if int(b.nextValueID) >= len(b.valueIDToInstruction) { + b.valueIDToInstruction = append(b.valueIDToInstruction, make([]*Instruction, b.nextValueID)...) + } + + for blk := b.blockIteratorBegin(); blk != nil; blk = b.blockIteratorNext() { + for cur := blk.rootInstr; cur != nil; cur = cur.next { + r1, rs := cur.Returns() + if r1.Valid() { + b.valueIDToInstruction[r1.ID()] = cur + } + for _, r := range rs { + b.valueIDToInstruction[r.ID()] = cur + } + } + } + + for blk := b.blockIteratorBegin(); blk != nil; blk = b.blockIteratorNext() { + for cur := blk.rootInstr; cur != nil; cur = cur.next { + switch cur.Opcode() { + // TODO: add more logics here. + case OpcodeIshl, OpcodeSshr, OpcodeUshr: + x, amount := cur.Arg2() + definingInst := b.valueIDToInstruction[amount.ID()] + if definingInst == nil { + // If there's no defining instruction, that means the amount is coming from the parameter. + continue + } + if definingInst.Constant() { + v := definingInst.ConstantVal() + + if x.Type().Bits() == 64 { + v = v % 64 + } else { + v = v % 32 + } + if v == 0 { + b.alias(cur.Return(), x) + } + } + } + } + } +} + +// passSortSuccessors sorts the successors of each block in the natural program order. +func passSortSuccessors(b *builder) { + for i := 0; i < b.basicBlocksPool.Allocated(); i++ { + blk := b.basicBlocksPool.View(i) + sortBlocks(blk.success) + } +} diff --git a/vendor/github.com/tetratelabs/wazero/internal/engine/wazevo/ssa/pass_blk_layouts.go b/vendor/github.com/tetratelabs/wazero/internal/engine/wazevo/ssa/pass_blk_layouts.go new file mode 100644 index 000000000..9068180a0 --- /dev/null +++ b/vendor/github.com/tetratelabs/wazero/internal/engine/wazevo/ssa/pass_blk_layouts.go @@ -0,0 +1,335 @@ +package ssa + +import ( + "fmt" + "strings" + + "github.com/tetratelabs/wazero/internal/engine/wazevo/wazevoapi" +) + +// passLayoutBlocks implements Builder.LayoutBlocks. This re-organizes builder.reversePostOrderedBasicBlocks. +// +// TODO: there are tons of room for improvement here. e.g. LLVM has BlockPlacementPass using BlockFrequencyInfo, +// BranchProbabilityInfo, and LoopInfo to do a much better job. Also, if we have the profiling instrumentation +// like ball-larus algorithm, then we could do profile-guided optimization. Basically all of them are trying +// to maximize the fall-through opportunities which is most efficient. +// +// Here, fallthrough happens when a block ends with jump instruction whose target is the right next block in the +// builder.reversePostOrderedBasicBlocks. +// +// Currently, we just place blocks using the DFS reverse post-order of the dominator tree with the heuristics: +// 1. a split edge trampoline towards a loop header will be placed as a fallthrough. +// 2. we invert the brz and brnz if it makes the fallthrough more likely. +// +// This heuristic is done in maybeInvertBranches function. +func passLayoutBlocks(b *builder) { + b.clearBlkVisited() + + // We might end up splitting critical edges which adds more basic blocks, + // so we store the currently existing basic blocks in nonSplitBlocks temporarily. + // That way we can iterate over the original basic blocks while appending new ones into reversePostOrderedBasicBlocks. + nonSplitBlocks := b.blkStack[:0] + for i, blk := range b.reversePostOrderedBasicBlocks { + if !blk.Valid() { + continue + } + nonSplitBlocks = append(nonSplitBlocks, blk) + if i != len(b.reversePostOrderedBasicBlocks)-1 { + _ = maybeInvertBranches(blk, b.reversePostOrderedBasicBlocks[i+1]) + } + } + + var trampolines []*basicBlock + + // Reset the order slice since we update on the fly by splitting critical edges. + b.reversePostOrderedBasicBlocks = b.reversePostOrderedBasicBlocks[:0] + uninsertedTrampolines := b.blkStack2[:0] + for _, blk := range nonSplitBlocks { + for i := range blk.preds { + pred := blk.preds[i].blk + if _, ok := b.blkVisited[pred]; ok || !pred.Valid() { + continue + } else if pred.reversePostOrder < blk.reversePostOrder { + // This means the edge is critical, and this pred is the trampoline and yet to be inserted. + // Split edge trampolines must come before the destination in reverse post-order. + b.reversePostOrderedBasicBlocks = append(b.reversePostOrderedBasicBlocks, pred) + b.blkVisited[pred] = 0 // mark as inserted, the value is not used. + } + } + + // Now that we've already added all the potential trampoline blocks incoming to this block, + // we can add this block itself. + b.reversePostOrderedBasicBlocks = append(b.reversePostOrderedBasicBlocks, blk) + b.blkVisited[blk] = 0 // mark as inserted, the value is not used. + + if len(blk.success) < 2 { + // There won't be critical edge originating from this block. + continue + } else if blk.currentInstr.opcode == OpcodeBrTable { + // We don't split critical edges here, because at the construction site of BrTable, we already split the edges. + continue + } + + for sidx, succ := range blk.success { + if !succ.ReturnBlock() && // If the successor is a return block, we need to split the edge any way because we need "epilogue" to be inserted. + // Plus if there's no multiple incoming edges to this successor, (pred, succ) is not critical. + len(succ.preds) < 2 { + continue + } + + // Otherwise, we are sure this is a critical edge. To modify the CFG, we need to find the predecessor info + // from the successor. + var predInfo *basicBlockPredecessorInfo + for i := range succ.preds { // This linear search should not be a problem since the number of predecessors should almost always small. + pred := &succ.preds[i] + if pred.blk == blk { + predInfo = pred + break + } + } + + if predInfo == nil { + // This must be a bug in somewhere around branch manipulation. + panic("BUG: predecessor info not found while the successor exists in successors list") + } + + if wazevoapi.SSALoggingEnabled { + fmt.Printf("trying to split edge from %d->%d at %s\n", + blk.ID(), succ.ID(), predInfo.branch.Format(b)) + } + + trampoline := b.splitCriticalEdge(blk, succ, predInfo) + // Update the successors slice because the target is no longer the original `succ`. + blk.success[sidx] = trampoline + + if wazevoapi.SSAValidationEnabled { + trampolines = append(trampolines, trampoline) + } + + if wazevoapi.SSALoggingEnabled { + fmt.Printf("edge split from %d->%d at %s as %d->%d->%d \n", + blk.ID(), succ.ID(), predInfo.branch.Format(b), + blk.ID(), trampoline.ID(), succ.ID()) + } + + fallthroughBranch := blk.currentInstr + if fallthroughBranch.opcode == OpcodeJump && fallthroughBranch.blk == trampoline { + // This can be lowered as fallthrough at the end of the block. + b.reversePostOrderedBasicBlocks = append(b.reversePostOrderedBasicBlocks, trampoline) + b.blkVisited[trampoline] = 0 // mark as inserted, the value is not used. + } else { + uninsertedTrampolines = append(uninsertedTrampolines, trampoline) + } + } + + for _, trampoline := range uninsertedTrampolines { + if trampoline.success[0].reversePostOrder <= trampoline.reversePostOrder { // "<=", not "<" because the target might be itself. + // This means the critical edge was backward, so we insert after the current block immediately. + b.reversePostOrderedBasicBlocks = append(b.reversePostOrderedBasicBlocks, trampoline) + b.blkVisited[trampoline] = 0 // mark as inserted, the value is not used. + } // If the target is forward, we can wait to insert until the target is inserted. + } + uninsertedTrampolines = uninsertedTrampolines[:0] // Reuse the stack for the next block. + } + + if wazevoapi.SSALoggingEnabled { + var bs []string + for _, blk := range b.reversePostOrderedBasicBlocks { + bs = append(bs, blk.Name()) + } + fmt.Println("ordered blocks: ", strings.Join(bs, ", ")) + } + + if wazevoapi.SSAValidationEnabled { + for _, trampoline := range trampolines { + if _, ok := b.blkVisited[trampoline]; !ok { + panic("BUG: trampoline block not inserted: " + trampoline.FormatHeader(b)) + } + trampoline.validate(b) + } + } + + // Reuse the stack for the next iteration. + b.blkStack2 = uninsertedTrampolines[:0] +} + +// markFallthroughJumps finds the fallthrough jumps and marks them as such. +func (b *builder) markFallthroughJumps() { + l := len(b.reversePostOrderedBasicBlocks) - 1 + for i, blk := range b.reversePostOrderedBasicBlocks { + if i < l { + cur := blk.currentInstr + if cur.opcode == OpcodeJump && cur.blk == b.reversePostOrderedBasicBlocks[i+1] { + cur.AsFallthroughJump() + } + } + } +} + +// maybeInvertBranches inverts the branch instructions if it is likely possible to the fallthrough more likely with simple heuristics. +// nextInRPO is the next block in the reverse post-order. +// +// Returns true if the branch is inverted for testing purpose. +func maybeInvertBranches(now *basicBlock, nextInRPO *basicBlock) bool { + fallthroughBranch := now.currentInstr + if fallthroughBranch.opcode == OpcodeBrTable { + return false + } + + condBranch := fallthroughBranch.prev + if condBranch == nil || (condBranch.opcode != OpcodeBrnz && condBranch.opcode != OpcodeBrz) { + return false + } + + if len(fallthroughBranch.vs.View()) != 0 || len(condBranch.vs.View()) != 0 { + // If either one of them has arguments, we don't invert the branches. + return false + } + + // So this block has two branches (a conditional branch followed by an unconditional branch) at the end. + // We can invert the condition of the branch if it makes the fallthrough more likely. + + fallthroughTarget, condTarget := fallthroughBranch.blk.(*basicBlock), condBranch.blk.(*basicBlock) + + if fallthroughTarget.loopHeader { + // First, if the tail's target is loopHeader, we don't need to do anything here, + // because the edge is likely to be critical edge for complex loops (e.g. loop with branches inside it). + // That means, we will split the edge in the end of LayoutBlocks function, and insert the trampoline block + // right after this block, which will be fallthrough in any way. + return false + } else if condTarget.loopHeader { + // On the other hand, if the condBranch's target is loopHeader, we invert the condition of the branch + // so that we could get the fallthrough to the trampoline block. + goto invert + } + + if fallthroughTarget == nextInRPO { + // Also, if the tail's target is the next block in the reverse post-order, we don't need to do anything here, + // because if this is not critical edge, we would end up placing these two blocks adjacent to each other. + // Even if it is the critical edge, we place the trampoline block right after this block, which will be fallthrough in any way. + return false + } else if condTarget == nextInRPO { + // If the condBranch's target is the next block in the reverse post-order, we invert the condition of the branch + // so that we could get the fallthrough to the block. + goto invert + } else { + return false + } + +invert: + for i := range fallthroughTarget.preds { + pred := &fallthroughTarget.preds[i] + if pred.branch == fallthroughBranch { + pred.branch = condBranch + break + } + } + for i := range condTarget.preds { + pred := &condTarget.preds[i] + if pred.branch == condBranch { + pred.branch = fallthroughBranch + break + } + } + + condBranch.InvertBrx() + condBranch.blk = fallthroughTarget + fallthroughBranch.blk = condTarget + if wazevoapi.SSALoggingEnabled { + fmt.Printf("inverting branches at %d->%d and %d->%d\n", + now.ID(), fallthroughTarget.ID(), now.ID(), condTarget.ID()) + } + + return true +} + +// splitCriticalEdge splits the critical edge between the given predecessor (`pred`) and successor (owning `predInfo`). +// +// - `pred` is the source of the critical edge, +// - `succ` is the destination of the critical edge, +// - `predInfo` is the predecessor info in the succ.preds slice which represents the critical edge. +// +// Why splitting critical edges is important? See following links: +// +// - https://en.wikipedia.org/wiki/Control-flow_graph +// - https://nickdesaulniers.github.io/blog/2023/01/27/critical-edge-splitting/ +// +// The returned basic block is the trampoline block which is inserted to split the critical edge. +func (b *builder) splitCriticalEdge(pred, succ *basicBlock, predInfo *basicBlockPredecessorInfo) *basicBlock { + // In the following, we convert the following CFG: + // + // pred --(originalBranch)--> succ + // + // to the following CFG: + // + // pred --(newBranch)--> trampoline --(originalBranch)-> succ + // + // where trampoline is a new basic block which is created to split the critical edge. + + trampoline := b.allocateBasicBlock() + if int(trampoline.id) >= len(b.dominators) { + b.dominators = append(b.dominators, make([]*basicBlock, trampoline.id+1)...) + } + b.dominators[trampoline.id] = pred + + originalBranch := predInfo.branch + + // Replace originalBranch with the newBranch. + newBranch := b.AllocateInstruction() + newBranch.opcode = originalBranch.opcode + newBranch.blk = trampoline + switch originalBranch.opcode { + case OpcodeJump: + case OpcodeBrz, OpcodeBrnz: + originalBranch.opcode = OpcodeJump // Trampoline consists of one unconditional branch. + newBranch.v = originalBranch.v + originalBranch.v = ValueInvalid + default: + panic("BUG: critical edge shouldn't be originated from br_table") + } + swapInstruction(pred, originalBranch, newBranch) + + // Replace the original branch with the new branch. + trampoline.rootInstr = originalBranch + trampoline.currentInstr = originalBranch + trampoline.success = append(trampoline.success, succ) // Do not use []*basicBlock{pred} because we might have already allocated the slice. + trampoline.preds = append(trampoline.preds, // same as ^. + basicBlockPredecessorInfo{blk: pred, branch: newBranch}) + b.Seal(trampoline) + + // Update the original branch to point to the trampoline. + predInfo.blk = trampoline + predInfo.branch = originalBranch + + if wazevoapi.SSAValidationEnabled { + trampoline.validate(b) + } + + if len(trampoline.params) > 0 { + panic("trampoline should not have params") + } + + // Assign the same order as the original block so that this will be placed before the actual destination. + trampoline.reversePostOrder = pred.reversePostOrder + return trampoline +} + +// swapInstruction replaces `old` in the block `blk` with `New`. +func swapInstruction(blk *basicBlock, old, New *Instruction) { + if blk.rootInstr == old { + blk.rootInstr = New + next := old.next + New.next = next + next.prev = New + } else { + if blk.currentInstr == old { + blk.currentInstr = New + } + prev := old.prev + prev.next, New.prev = New, prev + if next := old.next; next != nil { + New.next, next.prev = next, New + } + } + old.prev, old.next = nil, nil +} diff --git a/vendor/github.com/tetratelabs/wazero/internal/engine/wazevo/ssa/pass_cfg.go b/vendor/github.com/tetratelabs/wazero/internal/engine/wazevo/ssa/pass_cfg.go new file mode 100644 index 000000000..50cb9c475 --- /dev/null +++ b/vendor/github.com/tetratelabs/wazero/internal/engine/wazevo/ssa/pass_cfg.go @@ -0,0 +1,312 @@ +package ssa + +import ( + "fmt" + "math" + "strings" + + "github.com/tetratelabs/wazero/internal/engine/wazevo/wazevoapi" +) + +// passCalculateImmediateDominators calculates immediate dominators for each basic block. +// The result is stored in b.dominators. This make it possible for the following passes to +// use builder.isDominatedBy to check if a block is dominated by another block. +// +// At the last of pass, this function also does the loop detection and sets the basicBlock.loop flag. +func passCalculateImmediateDominators(b *builder) { + reversePostOrder := b.reversePostOrderedBasicBlocks[:0] + exploreStack := b.blkStack[:0] + b.clearBlkVisited() + + entryBlk := b.entryBlk() + + // Store the reverse postorder from the entrypoint into reversePostOrder slice. + // This calculation of reverse postorder is not described in the paper, + // so we use heuristic to calculate it so that we could potentially handle arbitrary + // complex CFGs under the assumption that success is sorted in program's natural order. + // That means blk.success[i] always appears before blk.success[i+1] in the source program, + // which is a reasonable assumption as long as SSA Builder is properly used. + // + // First we push blocks in postorder iteratively visit successors of the entry block. + exploreStack = append(exploreStack, entryBlk) + const visitStateUnseen, visitStateSeen, visitStateDone = 0, 1, 2 + b.blkVisited[entryBlk] = visitStateSeen + for len(exploreStack) > 0 { + tail := len(exploreStack) - 1 + blk := exploreStack[tail] + exploreStack = exploreStack[:tail] + switch b.blkVisited[blk] { + case visitStateUnseen: + // This is likely a bug in the frontend. + panic("BUG: unsupported CFG") + case visitStateSeen: + // This is the first time to pop this block, and we have to see the successors first. + // So push this block again to the stack. + exploreStack = append(exploreStack, blk) + // And push the successors to the stack if necessary. + for _, succ := range blk.success { + if succ.ReturnBlock() || succ.invalid { + continue + } + if b.blkVisited[succ] == visitStateUnseen { + b.blkVisited[succ] = visitStateSeen + exploreStack = append(exploreStack, succ) + } + } + // Finally, we could pop this block once we pop all of its successors. + b.blkVisited[blk] = visitStateDone + case visitStateDone: + // Note: at this point we push blk in postorder despite its name. + reversePostOrder = append(reversePostOrder, blk) + } + } + // At this point, reversePostOrder has postorder actually, so we reverse it. + for i := len(reversePostOrder)/2 - 1; i >= 0; i-- { + j := len(reversePostOrder) - 1 - i + reversePostOrder[i], reversePostOrder[j] = reversePostOrder[j], reversePostOrder[i] + } + + for i, blk := range reversePostOrder { + blk.reversePostOrder = i + } + + // Reuse the dominators slice if possible from the previous computation of function. + b.dominators = b.dominators[:cap(b.dominators)] + if len(b.dominators) < b.basicBlocksPool.Allocated() { + // Generously reserve space in the slice because the slice will be reused future allocation. + b.dominators = append(b.dominators, make([]*basicBlock, b.basicBlocksPool.Allocated())...) + } + calculateDominators(reversePostOrder, b.dominators) + + // Reuse the slices for the future use. + b.blkStack = exploreStack + + // For the following passes. + b.reversePostOrderedBasicBlocks = reversePostOrder + + // Ready to detect loops! + subPassLoopDetection(b) +} + +// calculateDominators calculates the immediate dominator of each node in the CFG, and store the result in `doms`. +// The algorithm is based on the one described in the paper "A Simple, Fast Dominance Algorithm" +// https://www.cs.rice.edu/~keith/EMBED/dom.pdf which is a faster/simple alternative to the well known Lengauer-Tarjan algorithm. +// +// The following code almost matches the pseudocode in the paper with one exception (see the code comment below). +// +// The result slice `doms` must be pre-allocated with the size larger than the size of dfsBlocks. +func calculateDominators(reversePostOrderedBlks []*basicBlock, doms []*basicBlock) { + entry, reversePostOrderedBlks := reversePostOrderedBlks[0], reversePostOrderedBlks[1: /* skips entry point */] + for _, blk := range reversePostOrderedBlks { + doms[blk.id] = nil + } + doms[entry.id] = entry + + changed := true + for changed { + changed = false + for _, blk := range reversePostOrderedBlks { + var u *basicBlock + for i := range blk.preds { + pred := blk.preds[i].blk + // Skip if this pred is not reachable yet. Note that this is not described in the paper, + // but it is necessary to handle nested loops etc. + if doms[pred.id] == nil { + continue + } + + if u == nil { + u = pred + continue + } else { + u = intersect(doms, u, pred) + } + } + if doms[blk.id] != u { + doms[blk.id] = u + changed = true + } + } + } +} + +// intersect returns the common dominator of blk1 and blk2. +// +// This is the `intersect` function in the paper. +func intersect(doms []*basicBlock, blk1 *basicBlock, blk2 *basicBlock) *basicBlock { + finger1, finger2 := blk1, blk2 + for finger1 != finger2 { + // Move the 'finger1' upwards to its immediate dominator. + for finger1.reversePostOrder > finger2.reversePostOrder { + finger1 = doms[finger1.id] + } + // Move the 'finger2' upwards to its immediate dominator. + for finger2.reversePostOrder > finger1.reversePostOrder { + finger2 = doms[finger2.id] + } + } + return finger1 +} + +// subPassLoopDetection detects loops in the function using the immediate dominators. +// +// This is run at the last of passCalculateImmediateDominators. +func subPassLoopDetection(b *builder) { + for blk := b.blockIteratorBegin(); blk != nil; blk = b.blockIteratorNext() { + for i := range blk.preds { + pred := blk.preds[i].blk + if pred.invalid { + continue + } + if b.isDominatedBy(pred, blk) { + blk.loopHeader = true + } + } + } +} + +// buildLoopNestingForest builds the loop nesting forest for the function. +// This must be called after branch splitting since it relies on the CFG. +func passBuildLoopNestingForest(b *builder) { + ent := b.entryBlk() + doms := b.dominators + for _, blk := range b.reversePostOrderedBasicBlocks { + n := doms[blk.id] + for !n.loopHeader && n != ent { + n = doms[n.id] + } + + if n == ent && blk.loopHeader { + b.loopNestingForestRoots = append(b.loopNestingForestRoots, blk) + } else if n == ent { + } else if n.loopHeader { + n.loopNestingForestChildren = append(n.loopNestingForestChildren, blk) + } + } + + if wazevoapi.SSALoggingEnabled { + for _, root := range b.loopNestingForestRoots { + printLoopNestingForest(root.(*basicBlock), 0) + } + } +} + +func printLoopNestingForest(root *basicBlock, depth int) { + fmt.Println(strings.Repeat("\t", depth), "loop nesting forest root:", root.ID()) + for _, child := range root.loopNestingForestChildren { + fmt.Println(strings.Repeat("\t", depth+1), "child:", child.ID()) + if child.LoopHeader() { + printLoopNestingForest(child.(*basicBlock), depth+2) + } + } +} + +type dominatorSparseTree struct { + time int + euler []*basicBlock + first, depth []int + table [][]int +} + +// passBuildDominatorTree builds the dominator tree for the function, and constructs builder.sparseTree. +func passBuildDominatorTree(b *builder) { + // First we materialize the children of each node in the dominator tree. + idoms := b.dominators + for _, blk := range b.reversePostOrderedBasicBlocks { + parent := idoms[blk.id] + if parent == nil { + panic("BUG") + } else if parent == blk { + // This is the entry block. + continue + } + if prev := parent.child; prev == nil { + parent.child = blk + } else { + parent.child = blk + blk.sibling = prev + } + } + + // Reset the state from the previous computation. + n := b.basicBlocksPool.Allocated() + st := &b.sparseTree + st.euler = append(st.euler[:0], make([]*basicBlock, 2*n-1)...) + st.first = append(st.first[:0], make([]int, n)...) + for i := range st.first { + st.first[i] = -1 + } + st.depth = append(st.depth[:0], make([]int, 2*n-1)...) + st.time = 0 + + // Start building the sparse tree. + st.eulerTour(b.entryBlk(), 0) + st.buildSparseTable() +} + +func (dt *dominatorSparseTree) eulerTour(node *basicBlock, height int) { + if wazevoapi.SSALoggingEnabled { + fmt.Println(strings.Repeat("\t", height), "euler tour:", node.ID()) + } + dt.euler[dt.time] = node + dt.depth[dt.time] = height + if dt.first[node.id] == -1 { + dt.first[node.id] = dt.time + } + dt.time++ + + for child := node.child; child != nil; child = child.sibling { + dt.eulerTour(child, height+1) + dt.euler[dt.time] = node // add the current node again after visiting a child + dt.depth[dt.time] = height + dt.time++ + } +} + +// buildSparseTable builds a sparse table for RMQ queries. +func (dt *dominatorSparseTree) buildSparseTable() { + n := len(dt.depth) + k := int(math.Log2(float64(n))) + 1 + table := dt.table + + if n >= len(table) { + table = append(table, make([][]int, n+1)...) + } + for i := range table { + if len(table[i]) < k { + table[i] = append(table[i], make([]int, k)...) + } + table[i][0] = i + } + + for j := 1; 1< first[v] { + u, v = v, u + } + return dt.euler[dt.rmq(first[u], first[v])] +} diff --git a/vendor/github.com/tetratelabs/wazero/internal/engine/wazevo/ssa/signature.go b/vendor/github.com/tetratelabs/wazero/internal/engine/wazevo/ssa/signature.go new file mode 100644 index 000000000..43483395a --- /dev/null +++ b/vendor/github.com/tetratelabs/wazero/internal/engine/wazevo/ssa/signature.go @@ -0,0 +1,49 @@ +package ssa + +import ( + "fmt" + "strings" +) + +// Signature is a function prototype. +type Signature struct { + // ID is a unique identifier for this signature used to lookup. + ID SignatureID + // Params and Results are the types of the parameters and results of the function. + Params, Results []Type + + // used is true if this is used by the currently-compiled function. + // Debugging only. + used bool +} + +// String implements fmt.Stringer. +func (s *Signature) String() string { + str := strings.Builder{} + str.WriteString(s.ID.String()) + str.WriteString(": ") + if len(s.Params) > 0 { + for _, typ := range s.Params { + str.WriteString(typ.String()) + } + } else { + str.WriteByte('v') + } + str.WriteByte('_') + if len(s.Results) > 0 { + for _, typ := range s.Results { + str.WriteString(typ.String()) + } + } else { + str.WriteByte('v') + } + return str.String() +} + +// SignatureID is an unique identifier used to lookup. +type SignatureID int + +// String implements fmt.Stringer. +func (s SignatureID) String() string { + return fmt.Sprintf("sig%d", s) +} diff --git a/vendor/github.com/tetratelabs/wazero/internal/engine/wazevo/ssa/ssa.go b/vendor/github.com/tetratelabs/wazero/internal/engine/wazevo/ssa/ssa.go new file mode 100644 index 000000000..b477e58bd --- /dev/null +++ b/vendor/github.com/tetratelabs/wazero/internal/engine/wazevo/ssa/ssa.go @@ -0,0 +1,14 @@ +// Package ssa is used to construct SSA function. By nature this is free of Wasm specific thing +// and ISA. +// +// We use the "block argument" variant of SSA: https://en.wikipedia.org/wiki/Static_single-assignment_form#Block_arguments +// which is equivalent to the traditional PHI function based one, but more convenient during optimizations. +// However, in this package's source code comment, we might use PHI whenever it seems necessary in order to be aligned with +// existing literatures, e.g. SSA level optimization algorithms are often described using PHI nodes. +// +// The rationale doc for the choice of "block argument" by MLIR of LLVM is worth a read: +// https://mlir.llvm.org/docs/Rationale/Rationale/#block-arguments-vs-phi-nodes +// +// The algorithm to resolve variable definitions used here is based on the paper +// "Simple and Efficient Construction of Static Single Assignment Form": https://link.springer.com/content/pdf/10.1007/978-3-642-37051-9_6.pdf. +package ssa diff --git a/vendor/github.com/tetratelabs/wazero/internal/engine/wazevo/ssa/type.go b/vendor/github.com/tetratelabs/wazero/internal/engine/wazevo/ssa/type.go new file mode 100644 index 000000000..e8c8cd9de --- /dev/null +++ b/vendor/github.com/tetratelabs/wazero/internal/engine/wazevo/ssa/type.go @@ -0,0 +1,112 @@ +package ssa + +type Type byte + +const ( + typeInvalid Type = iota + + // TODO: add 8, 16 bit types when it's needed for optimizations. + + // TypeI32 represents an integer type with 32 bits. + TypeI32 + + // TypeI64 represents an integer type with 64 bits. + TypeI64 + + // TypeF32 represents 32-bit floats in the IEEE 754. + TypeF32 + + // TypeF64 represents 64-bit floats in the IEEE 754. + TypeF64 + + // TypeV128 represents 128-bit SIMD vectors. + TypeV128 +) + +// String implements fmt.Stringer. +func (t Type) String() (ret string) { + switch t { + case typeInvalid: + return "invalid" + case TypeI32: + return "i32" + case TypeI64: + return "i64" + case TypeF32: + return "f32" + case TypeF64: + return "f64" + case TypeV128: + return "v128" + default: + panic(int(t)) + } +} + +// IsInt returns true if the type is an integer type. +func (t Type) IsInt() bool { + return t == TypeI32 || t == TypeI64 +} + +// IsFloat returns true if the type is a floating point type. +func (t Type) IsFloat() bool { + return t == TypeF32 || t == TypeF64 +} + +// Bits returns the number of bits required to represent the type. +func (t Type) Bits() byte { + switch t { + case TypeI32, TypeF32: + return 32 + case TypeI64, TypeF64: + return 64 + case TypeV128: + return 128 + default: + panic(int(t)) + } +} + +// Size returns the number of bytes required to represent the type. +func (t Type) Size() byte { + return t.Bits() / 8 +} + +func (t Type) invalid() bool { + return t == typeInvalid +} + +// VecLane represents a lane in a SIMD vector. +type VecLane byte + +const ( + VecLaneInvalid VecLane = 1 + iota + VecLaneI8x16 + VecLaneI16x8 + VecLaneI32x4 + VecLaneI64x2 + VecLaneF32x4 + VecLaneF64x2 +) + +// String implements fmt.Stringer. +func (vl VecLane) String() (ret string) { + switch vl { + case VecLaneInvalid: + return "invalid" + case VecLaneI8x16: + return "i8x16" + case VecLaneI16x8: + return "i16x8" + case VecLaneI32x4: + return "i32x4" + case VecLaneI64x2: + return "i64x2" + case VecLaneF32x4: + return "f32x4" + case VecLaneF64x2: + return "f64x2" + default: + panic(int(vl)) + } +} diff --git a/vendor/github.com/tetratelabs/wazero/internal/engine/wazevo/ssa/vs.go b/vendor/github.com/tetratelabs/wazero/internal/engine/wazevo/ssa/vs.go new file mode 100644 index 000000000..bcf83cbf8 --- /dev/null +++ b/vendor/github.com/tetratelabs/wazero/internal/engine/wazevo/ssa/vs.go @@ -0,0 +1,87 @@ +package ssa + +import ( + "fmt" + "math" + + "github.com/tetratelabs/wazero/internal/engine/wazevo/wazevoapi" +) + +// Variable is a unique identifier for a source program's variable and will correspond to +// multiple ssa Value(s). +// +// For example, `Local 1` is a Variable in WebAssembly, and Value(s) will be created for it +// whenever it executes `local.set 1`. +// +// Variable is useful to track the SSA Values of a variable in the source program, and +// can be used to find the corresponding latest SSA Value via Builder.FindValue. +type Variable uint32 + +// String implements fmt.Stringer. +func (v Variable) String() string { + return fmt.Sprintf("var%d", v) +} + +// Value represents an SSA value with a type information. The relationship with Variable is 1: N (including 0), +// that means there might be multiple Variable(s) for a Value. +// +// Higher 32-bit is used to store Type for this value. +type Value uint64 + +// ValueID is the lower 32bit of Value, which is the pure identifier of Value without type info. +type ValueID uint32 + +const ( + valueIDInvalid ValueID = math.MaxUint32 + ValueInvalid Value = Value(valueIDInvalid) +) + +// Format creates a debug string for this Value using the data stored in Builder. +func (v Value) Format(b Builder) string { + if annotation, ok := b.(*builder).valueAnnotations[v.ID()]; ok { + return annotation + } + return fmt.Sprintf("v%d", v.ID()) +} + +func (v Value) formatWithType(b Builder) (ret string) { + if annotation, ok := b.(*builder).valueAnnotations[v.ID()]; ok { + ret = annotation + ":" + v.Type().String() + } else { + ret = fmt.Sprintf("v%d:%s", v.ID(), v.Type()) + } + + if wazevoapi.SSALoggingEnabled { // This is useful to check live value analysis bugs. + if bd := b.(*builder); bd.donePostBlockLayoutPasses { + id := v.ID() + ret += fmt.Sprintf("(ref=%d)", bd.valueRefCounts[id]) + } + } + return ret +} + +// Valid returns true if this value is valid. +func (v Value) Valid() bool { + return v.ID() != valueIDInvalid +} + +// Type returns the Type of this value. +func (v Value) Type() Type { + return Type(v >> 32) +} + +// ID returns the valueID of this value. +func (v Value) ID() ValueID { + return ValueID(v) +} + +// setType sets a type to this Value and returns the updated Value. +func (v Value) setType(typ Type) Value { + return v | Value(typ)<<32 +} + +// Values is a slice of Value. Use this instead of []Value to reuse the underlying memory. +type Values = wazevoapi.VarLength[Value] + +// ValuesNil is a nil Values. +var ValuesNil = wazevoapi.NewNilVarLength[Value]() diff --git a/vendor/github.com/tetratelabs/wazero/internal/engine/wazevo/wazevoapi/debug_options.go b/vendor/github.com/tetratelabs/wazero/internal/engine/wazevo/wazevoapi/debug_options.go new file mode 100644 index 000000000..2db61e219 --- /dev/null +++ b/vendor/github.com/tetratelabs/wazero/internal/engine/wazevo/wazevoapi/debug_options.go @@ -0,0 +1,196 @@ +package wazevoapi + +import ( + "context" + "encoding/hex" + "fmt" + "math/rand" + "os" + "time" +) + +// These consts are used various places in the wazevo implementations. +// Instead of defining them in each file, we define them here so that we can quickly iterate on +// debugging without spending "where do we have debug logging?" time. + +// ----- Debug logging ----- +// These consts must be disabled by default. Enable them only when debugging. + +const ( + FrontEndLoggingEnabled = false + SSALoggingEnabled = false + RegAllocLoggingEnabled = false +) + +// ----- Output prints ----- +// These consts must be disabled by default. Enable them only when debugging. + +const ( + PrintSSA = false + PrintOptimizedSSA = false + PrintSSAToBackendIRLowering = false + PrintRegisterAllocated = false + PrintFinalizedMachineCode = false + PrintMachineCodeHexPerFunction = printMachineCodeHexPerFunctionUnmodified || PrintMachineCodeHexPerFunctionDisassemblable //nolint + printMachineCodeHexPerFunctionUnmodified = false + // PrintMachineCodeHexPerFunctionDisassemblable prints the machine code while modifying the actual result + // to make it disassemblable. This is useful when debugging the final machine code. See the places where this is used for detail. + // When this is enabled, functions must not be called. + PrintMachineCodeHexPerFunctionDisassemblable = false +) + +// printTarget is the function index to print the machine code. This is used for debugging to print the machine code +// of a specific function. +const printTarget = -1 + +// PrintEnabledIndex returns true if the current function index is the print target. +func PrintEnabledIndex(ctx context.Context) bool { + if printTarget == -1 { + return true + } + return GetCurrentFunctionIndex(ctx) == printTarget +} + +// ----- Validations ----- +const ( + // SSAValidationEnabled enables the SSA validation. This is disabled by default since the operation is expensive. + SSAValidationEnabled = false +) + +// ----- Stack Guard Check ----- +const ( + // StackGuardCheckEnabled enables the stack guard check to ensure that our stack bounds check works correctly. + StackGuardCheckEnabled = false + StackGuardCheckGuardPageSize = 8096 +) + +// CheckStackGuardPage checks the given stack guard page is not corrupted. +func CheckStackGuardPage(s []byte) { + for i := 0; i < StackGuardCheckGuardPageSize; i++ { + if s[i] != 0 { + panic( + fmt.Sprintf("BUG: stack guard page is corrupted:\n\tguard_page=%s\n\tstack=%s", + hex.EncodeToString(s[:StackGuardCheckGuardPageSize]), + hex.EncodeToString(s[StackGuardCheckGuardPageSize:]), + )) + } + } +} + +// ----- Deterministic compilation verifier ----- + +const ( + // DeterministicCompilationVerifierEnabled enables the deterministic compilation verifier. This is disabled by default + // since the operation is expensive. But when in doubt, enable this to make sure the compilation is deterministic. + DeterministicCompilationVerifierEnabled = false + DeterministicCompilationVerifyingIter = 5 +) + +type ( + verifierState struct { + initialCompilationDone bool + maybeRandomizedIndexes []int + r *rand.Rand + values map[string]string + } + verifierStateContextKey struct{} + currentFunctionNameKey struct{} + currentFunctionIndexKey struct{} +) + +// NewDeterministicCompilationVerifierContext creates a new context with the deterministic compilation verifier used per wasm.Module. +func NewDeterministicCompilationVerifierContext(ctx context.Context, localFunctions int) context.Context { + maybeRandomizedIndexes := make([]int, localFunctions) + for i := range maybeRandomizedIndexes { + maybeRandomizedIndexes[i] = i + } + r := rand.New(rand.NewSource(time.Now().UnixNano())) + return context.WithValue(ctx, verifierStateContextKey{}, &verifierState{ + r: r, maybeRandomizedIndexes: maybeRandomizedIndexes, values: map[string]string{}, + }) +} + +// DeterministicCompilationVerifierRandomizeIndexes randomizes the indexes for the deterministic compilation verifier. +// To get the randomized index, use DeterministicCompilationVerifierGetRandomizedLocalFunctionIndex. +func DeterministicCompilationVerifierRandomizeIndexes(ctx context.Context) { + state := ctx.Value(verifierStateContextKey{}).(*verifierState) + if !state.initialCompilationDone { + // If this is the first attempt, we use the index as-is order. + state.initialCompilationDone = true + return + } + r := state.r + r.Shuffle(len(state.maybeRandomizedIndexes), func(i, j int) { + state.maybeRandomizedIndexes[i], state.maybeRandomizedIndexes[j] = state.maybeRandomizedIndexes[j], state.maybeRandomizedIndexes[i] + }) +} + +// DeterministicCompilationVerifierGetRandomizedLocalFunctionIndex returns the randomized index for the given `index` +// which is assigned by DeterministicCompilationVerifierRandomizeIndexes. +func DeterministicCompilationVerifierGetRandomizedLocalFunctionIndex(ctx context.Context, index int) int { + state := ctx.Value(verifierStateContextKey{}).(*verifierState) + ret := state.maybeRandomizedIndexes[index] + return ret +} + +// VerifyOrSetDeterministicCompilationContextValue verifies that the `newValue` is the same as the previous value for the given `scope` +// and the current function name. If the previous value doesn't exist, it sets the value to the given `newValue`. +// +// If the verification fails, this prints the diff and exits the process. +func VerifyOrSetDeterministicCompilationContextValue(ctx context.Context, scope string, newValue string) { + fn := ctx.Value(currentFunctionNameKey{}).(string) + key := fn + ": " + scope + verifierCtx := ctx.Value(verifierStateContextKey{}).(*verifierState) + oldValue, ok := verifierCtx.values[key] + if !ok { + verifierCtx.values[key] = newValue + return + } + if oldValue != newValue { + fmt.Printf( + `BUG: Deterministic compilation failed for function%s at scope="%s". + +This is mostly due to (but might not be limited to): + * Resetting ssa.Builder, backend.Compiler or frontend.Compiler, etc doens't work as expected, and the compilation has been affected by the previous iterations. + * Using a map with non-deterministic iteration order. + +---------- [old] ---------- +%s + +---------- [new] ---------- +%s +`, + fn, scope, oldValue, newValue, + ) + os.Exit(1) + } +} + +// nolint +const NeedFunctionNameInContext = PrintSSA || + PrintOptimizedSSA || + PrintSSAToBackendIRLowering || + PrintRegisterAllocated || + PrintFinalizedMachineCode || + PrintMachineCodeHexPerFunction || + DeterministicCompilationVerifierEnabled || + PerfMapEnabled + +// SetCurrentFunctionName sets the current function name to the given `functionName`. +func SetCurrentFunctionName(ctx context.Context, index int, functionName string) context.Context { + ctx = context.WithValue(ctx, currentFunctionNameKey{}, functionName) + ctx = context.WithValue(ctx, currentFunctionIndexKey{}, index) + return ctx +} + +// GetCurrentFunctionName returns the current function name. +func GetCurrentFunctionName(ctx context.Context) string { + ret, _ := ctx.Value(currentFunctionNameKey{}).(string) + return ret +} + +// GetCurrentFunctionIndex returns the current function index. +func GetCurrentFunctionIndex(ctx context.Context) int { + ret, _ := ctx.Value(currentFunctionIndexKey{}).(int) + return ret +} diff --git a/vendor/github.com/tetratelabs/wazero/internal/engine/wazevo/wazevoapi/exitcode.go b/vendor/github.com/tetratelabs/wazero/internal/engine/wazevo/wazevoapi/exitcode.go new file mode 100644 index 000000000..5ad594982 --- /dev/null +++ b/vendor/github.com/tetratelabs/wazero/internal/engine/wazevo/wazevoapi/exitcode.go @@ -0,0 +1,109 @@ +package wazevoapi + +// ExitCode is an exit code of an execution of a function. +type ExitCode uint32 + +const ( + ExitCodeOK ExitCode = iota + ExitCodeGrowStack + ExitCodeGrowMemory + ExitCodeUnreachable + ExitCodeMemoryOutOfBounds + // ExitCodeCallGoModuleFunction is an exit code for a call to an api.GoModuleFunction. + ExitCodeCallGoModuleFunction + // ExitCodeCallGoFunction is an exit code for a call to an api.GoFunction. + ExitCodeCallGoFunction + ExitCodeTableOutOfBounds + ExitCodeIndirectCallNullPointer + ExitCodeIndirectCallTypeMismatch + ExitCodeIntegerDivisionByZero + ExitCodeIntegerOverflow + ExitCodeInvalidConversionToInteger + ExitCodeCheckModuleExitCode + ExitCodeCallListenerBefore + ExitCodeCallListenerAfter + ExitCodeCallGoModuleFunctionWithListener + ExitCodeCallGoFunctionWithListener + ExitCodeTableGrow + ExitCodeRefFunc + ExitCodeMemoryWait32 + ExitCodeMemoryWait64 + ExitCodeMemoryNotify + ExitCodeUnalignedAtomic + exitCodeMax +) + +const ExitCodeMask = 0xff + +// String implements fmt.Stringer. +func (e ExitCode) String() string { + switch e { + case ExitCodeOK: + return "ok" + case ExitCodeGrowStack: + return "grow_stack" + case ExitCodeCallGoModuleFunction: + return "call_go_module_function" + case ExitCodeCallGoFunction: + return "call_go_function" + case ExitCodeUnreachable: + return "unreachable" + case ExitCodeMemoryOutOfBounds: + return "memory_out_of_bounds" + case ExitCodeUnalignedAtomic: + return "unaligned_atomic" + case ExitCodeTableOutOfBounds: + return "table_out_of_bounds" + case ExitCodeIndirectCallNullPointer: + return "indirect_call_null_pointer" + case ExitCodeIndirectCallTypeMismatch: + return "indirect_call_type_mismatch" + case ExitCodeIntegerDivisionByZero: + return "integer_division_by_zero" + case ExitCodeIntegerOverflow: + return "integer_overflow" + case ExitCodeInvalidConversionToInteger: + return "invalid_conversion_to_integer" + case ExitCodeCheckModuleExitCode: + return "check_module_exit_code" + case ExitCodeCallListenerBefore: + return "call_listener_before" + case ExitCodeCallListenerAfter: + return "call_listener_after" + case ExitCodeCallGoModuleFunctionWithListener: + return "call_go_module_function_with_listener" + case ExitCodeCallGoFunctionWithListener: + return "call_go_function_with_listener" + case ExitCodeGrowMemory: + return "grow_memory" + case ExitCodeTableGrow: + return "table_grow" + case ExitCodeRefFunc: + return "ref_func" + case ExitCodeMemoryWait32: + return "memory_wait32" + case ExitCodeMemoryWait64: + return "memory_wait64" + case ExitCodeMemoryNotify: + return "memory_notify" + } + panic("TODO") +} + +func ExitCodeCallGoModuleFunctionWithIndex(index int, withListener bool) ExitCode { + if withListener { + return ExitCodeCallGoModuleFunctionWithListener | ExitCode(index<<8) + } + return ExitCodeCallGoModuleFunction | ExitCode(index<<8) +} + +func ExitCodeCallGoFunctionWithIndex(index int, withListener bool) ExitCode { + if withListener { + return ExitCodeCallGoFunctionWithListener | ExitCode(index<<8) + } + return ExitCodeCallGoFunction | ExitCode(index<<8) +} + +func GoFunctionIndexFromExitCode(exitCode ExitCode) int { + return int(exitCode >> 8) +} diff --git a/vendor/github.com/tetratelabs/wazero/internal/engine/wazevo/wazevoapi/offsetdata.go b/vendor/github.com/tetratelabs/wazero/internal/engine/wazevo/wazevoapi/offsetdata.go new file mode 100644 index 000000000..fe6161b04 --- /dev/null +++ b/vendor/github.com/tetratelabs/wazero/internal/engine/wazevo/wazevoapi/offsetdata.go @@ -0,0 +1,216 @@ +package wazevoapi + +import ( + "github.com/tetratelabs/wazero/internal/wasm" +) + +const ( + // FunctionInstanceSize is the size of wazevo.functionInstance. + FunctionInstanceSize = 24 + // FunctionInstanceExecutableOffset is an offset of `executable` field in wazevo.functionInstance + FunctionInstanceExecutableOffset = 0 + // FunctionInstanceModuleContextOpaquePtrOffset is an offset of `moduleContextOpaquePtr` field in wazevo.functionInstance + FunctionInstanceModuleContextOpaquePtrOffset = 8 + // FunctionInstanceTypeIDOffset is an offset of `typeID` field in wazevo.functionInstance + FunctionInstanceTypeIDOffset = 16 +) + +const ( + // ExecutionContextOffsetExitCodeOffset is an offset of `exitCode` field in wazevo.executionContext + ExecutionContextOffsetExitCodeOffset Offset = 0 + // ExecutionContextOffsetCallerModuleContextPtr is an offset of `callerModuleContextPtr` field in wazevo.executionContext + ExecutionContextOffsetCallerModuleContextPtr Offset = 8 + // ExecutionContextOffsetOriginalFramePointer is an offset of `originalFramePointer` field in wazevo.executionContext + ExecutionContextOffsetOriginalFramePointer Offset = 16 + // ExecutionContextOffsetOriginalStackPointer is an offset of `originalStackPointer` field in wazevo.executionContext + ExecutionContextOffsetOriginalStackPointer Offset = 24 + // ExecutionContextOffsetGoReturnAddress is an offset of `goReturnAddress` field in wazevo.executionContext + ExecutionContextOffsetGoReturnAddress Offset = 32 + // ExecutionContextOffsetStackBottomPtr is an offset of `stackBottomPtr` field in wazevo.executionContext + ExecutionContextOffsetStackBottomPtr Offset = 40 + // ExecutionContextOffsetGoCallReturnAddress is an offset of `goCallReturnAddress` field in wazevo.executionContext + ExecutionContextOffsetGoCallReturnAddress Offset = 48 + // ExecutionContextOffsetStackPointerBeforeGoCall is an offset of `StackPointerBeforeGoCall` field in wazevo.executionContext + ExecutionContextOffsetStackPointerBeforeGoCall Offset = 56 + // ExecutionContextOffsetStackGrowRequiredSize is an offset of `stackGrowRequiredSize` field in wazevo.executionContext + ExecutionContextOffsetStackGrowRequiredSize Offset = 64 + // ExecutionContextOffsetMemoryGrowTrampolineAddress is an offset of `memoryGrowTrampolineAddress` field in wazevo.executionContext + ExecutionContextOffsetMemoryGrowTrampolineAddress Offset = 72 + // ExecutionContextOffsetStackGrowCallTrampolineAddress is an offset of `stackGrowCallTrampolineAddress` field in wazevo.executionContext. + ExecutionContextOffsetStackGrowCallTrampolineAddress Offset = 80 + // ExecutionContextOffsetCheckModuleExitCodeTrampolineAddress is an offset of `checkModuleExitCodeTrampolineAddress` field in wazevo.executionContext. + ExecutionContextOffsetCheckModuleExitCodeTrampolineAddress Offset = 88 + // ExecutionContextOffsetSavedRegistersBegin is an offset of the first element of `savedRegisters` field in wazevo.executionContext + ExecutionContextOffsetSavedRegistersBegin Offset = 96 + // ExecutionContextOffsetGoFunctionCallCalleeModuleContextOpaque is an offset of `goFunctionCallCalleeModuleContextOpaque` field in wazevo.executionContext + ExecutionContextOffsetGoFunctionCallCalleeModuleContextOpaque Offset = 1120 + // ExecutionContextOffsetTableGrowTrampolineAddress is an offset of `tableGrowTrampolineAddress` field in wazevo.executionContext + ExecutionContextOffsetTableGrowTrampolineAddress Offset = 1128 + // ExecutionContextOffsetRefFuncTrampolineAddress is an offset of `refFuncTrampolineAddress` field in wazevo.executionContext + ExecutionContextOffsetRefFuncTrampolineAddress Offset = 1136 + ExecutionContextOffsetMemmoveAddress Offset = 1144 + ExecutionContextOffsetFramePointerBeforeGoCall Offset = 1152 + ExecutionContextOffsetMemoryWait32TrampolineAddress Offset = 1160 + ExecutionContextOffsetMemoryWait64TrampolineAddress Offset = 1168 + ExecutionContextOffsetMemoryNotifyTrampolineAddress Offset = 1176 +) + +// ModuleContextOffsetData allows the compilers to get the information about offsets to the fields of wazevo.moduleContextOpaque, +// This is unique per module. +type ModuleContextOffsetData struct { + TotalSize int + ModuleInstanceOffset, + LocalMemoryBegin, + ImportedMemoryBegin, + ImportedFunctionsBegin, + GlobalsBegin, + TypeIDs1stElement, + TablesBegin, + BeforeListenerTrampolines1stElement, + AfterListenerTrampolines1stElement, + DataInstances1stElement, + ElementInstances1stElement Offset +} + +// ImportedFunctionOffset returns an offset of the i-th imported function. +// Each item is stored as wazevo.functionInstance whose size matches FunctionInstanceSize. +func (m *ModuleContextOffsetData) ImportedFunctionOffset(i wasm.Index) ( + executableOffset, moduleCtxOffset, typeIDOffset Offset, +) { + base := m.ImportedFunctionsBegin + Offset(i)*FunctionInstanceSize + return base, base + 8, base + 16 +} + +// GlobalInstanceOffset returns an offset of the i-th global instance. +func (m *ModuleContextOffsetData) GlobalInstanceOffset(i wasm.Index) Offset { + return m.GlobalsBegin + Offset(i)*16 +} + +// Offset represents an offset of a field of a struct. +type Offset int32 + +// U32 encodes an Offset as uint32 for convenience. +func (o Offset) U32() uint32 { + return uint32(o) +} + +// I64 encodes an Offset as int64 for convenience. +func (o Offset) I64() int64 { + return int64(o) +} + +// U64 encodes an Offset as int64 for convenience. +func (o Offset) U64() uint64 { + return uint64(o) +} + +// LocalMemoryBase returns an offset of the first byte of the local memory. +func (m *ModuleContextOffsetData) LocalMemoryBase() Offset { + return m.LocalMemoryBegin +} + +// LocalMemoryLen returns an offset of the length of the local memory buffer. +func (m *ModuleContextOffsetData) LocalMemoryLen() Offset { + if l := m.LocalMemoryBegin; l >= 0 { + return l + 8 + } + return -1 +} + +// TableOffset returns an offset of the i-th table instance. +func (m *ModuleContextOffsetData) TableOffset(tableIndex int) Offset { + return m.TablesBegin + Offset(tableIndex)*8 +} + +// NewModuleContextOffsetData creates a ModuleContextOffsetData determining the structure of moduleContextOpaque for the given Module. +// The structure is described in the comment of wazevo.moduleContextOpaque. +func NewModuleContextOffsetData(m *wasm.Module, withListener bool) ModuleContextOffsetData { + ret := ModuleContextOffsetData{} + var offset Offset + + ret.ModuleInstanceOffset = 0 + offset += 8 + + if m.MemorySection != nil { + ret.LocalMemoryBegin = offset + // buffer base + memory size. + const localMemorySizeInOpaqueModuleContext = 16 + offset += localMemorySizeInOpaqueModuleContext + } else { + // Indicates that there's no local memory + ret.LocalMemoryBegin = -1 + } + + if m.ImportMemoryCount > 0 { + offset = align8(offset) + // *wasm.MemoryInstance + imported memory's owner (moduleContextOpaque) + const importedMemorySizeInOpaqueModuleContext = 16 + ret.ImportedMemoryBegin = offset + offset += importedMemorySizeInOpaqueModuleContext + } else { + // Indicates that there's no imported memory + ret.ImportedMemoryBegin = -1 + } + + if m.ImportFunctionCount > 0 { + offset = align8(offset) + ret.ImportedFunctionsBegin = offset + // Each function is stored wazevo.functionInstance. + size := int(m.ImportFunctionCount) * FunctionInstanceSize + offset += Offset(size) + } else { + ret.ImportedFunctionsBegin = -1 + } + + if globals := int(m.ImportGlobalCount) + len(m.GlobalSection); globals > 0 { + // Align to 16 bytes for globals, as f32/f64/v128 might be loaded via SIMD instructions. + offset = align16(offset) + ret.GlobalsBegin = offset + // Pointers to *wasm.GlobalInstance. + offset += Offset(globals) * 16 + } else { + ret.GlobalsBegin = -1 + } + + if tables := len(m.TableSection) + int(m.ImportTableCount); tables > 0 { + offset = align8(offset) + ret.TypeIDs1stElement = offset + offset += 8 // First element of TypeIDs. + + ret.TablesBegin = offset + // Pointers to *wasm.TableInstance. + offset += Offset(tables) * 8 + } else { + ret.TypeIDs1stElement = -1 + ret.TablesBegin = -1 + } + + if withListener { + offset = align8(offset) + ret.BeforeListenerTrampolines1stElement = offset + offset += 8 // First element of BeforeListenerTrampolines. + + ret.AfterListenerTrampolines1stElement = offset + offset += 8 // First element of AfterListenerTrampolines. + } else { + ret.BeforeListenerTrampolines1stElement = -1 + ret.AfterListenerTrampolines1stElement = -1 + } + + ret.DataInstances1stElement = offset + offset += 8 // First element of DataInstances. + + ret.ElementInstances1stElement = offset + offset += 8 // First element of ElementInstances. + + ret.TotalSize = int(align16(offset)) + return ret +} + +func align16(o Offset) Offset { + return (o + 15) &^ 15 +} + +func align8(o Offset) Offset { + return (o + 7) &^ 7 +} diff --git a/vendor/github.com/tetratelabs/wazero/internal/engine/wazevo/wazevoapi/perfmap.go b/vendor/github.com/tetratelabs/wazero/internal/engine/wazevo/wazevoapi/perfmap.go new file mode 100644 index 000000000..642c7f75d --- /dev/null +++ b/vendor/github.com/tetratelabs/wazero/internal/engine/wazevo/wazevoapi/perfmap.go @@ -0,0 +1,96 @@ +package wazevoapi + +import ( + "fmt" + "os" + "strconv" + "sync" +) + +var PerfMap *Perfmap + +func init() { + if PerfMapEnabled { + pid := os.Getpid() + filename := "/tmp/perf-" + strconv.Itoa(pid) + ".map" + + fh, err := os.OpenFile(filename, os.O_APPEND|os.O_RDWR|os.O_CREATE, 0o644) + if err != nil { + panic(err) + } + + PerfMap = &Perfmap{fh: fh} + } +} + +// Perfmap holds perfmap entries to be flushed into a perfmap file. +type Perfmap struct { + entries []entry + mux sync.Mutex + fh *os.File +} + +type entry struct { + index int + offset int64 + size uint64 + name string +} + +func (f *Perfmap) Lock() { + f.mux.Lock() +} + +func (f *Perfmap) Unlock() { + f.mux.Unlock() +} + +// AddModuleEntry adds a perfmap entry into the perfmap file. +// index is the index of the function in the module, offset is the offset of the function in the module, +// size is the size of the function, and name is the name of the function. +// +// Note that the entries are not flushed into the perfmap file until Flush is called, +// and the entries are module-scoped; Perfmap must be locked until Flush is called. +func (f *Perfmap) AddModuleEntry(index int, offset int64, size uint64, name string) { + e := entry{index: index, offset: offset, size: size, name: name} + if f.entries == nil { + f.entries = []entry{e} + return + } + f.entries = append(f.entries, e) +} + +// Flush writes the perfmap entries into the perfmap file where the entries are adjusted by the given `addr` and `functionOffsets`. +func (f *Perfmap) Flush(addr uintptr, functionOffsets []int) { + defer func() { + _ = f.fh.Sync() + }() + + for _, e := range f.entries { + if _, err := f.fh.WriteString(fmt.Sprintf("%x %s %s\n", + uintptr(e.offset)+addr+uintptr(functionOffsets[e.index]), + strconv.FormatUint(e.size, 16), + e.name, + )); err != nil { + panic(err) + } + } + f.entries = f.entries[:0] +} + +// Clear clears the perfmap entries not yet flushed. +func (f *Perfmap) Clear() { + f.entries = f.entries[:0] +} + +// AddEntry writes a perfmap entry directly into the perfmap file, not using the entries. +func (f *Perfmap) AddEntry(addr uintptr, size uint64, name string) { + _, err := f.fh.WriteString(fmt.Sprintf("%x %s %s\n", + addr, + strconv.FormatUint(size, 16), + name, + )) + if err != nil { + panic(err) + } +} diff --git a/vendor/github.com/tetratelabs/wazero/internal/engine/wazevo/wazevoapi/perfmap_disabled.go b/vendor/github.com/tetratelabs/wazero/internal/engine/wazevo/wazevoapi/perfmap_disabled.go new file mode 100644 index 000000000..bcc4e545c --- /dev/null +++ b/vendor/github.com/tetratelabs/wazero/internal/engine/wazevo/wazevoapi/perfmap_disabled.go @@ -0,0 +1,5 @@ +//go:build !perfmap + +package wazevoapi + +const PerfMapEnabled = false diff --git a/vendor/github.com/tetratelabs/wazero/internal/engine/wazevo/wazevoapi/perfmap_enabled.go b/vendor/github.com/tetratelabs/wazero/internal/engine/wazevo/wazevoapi/perfmap_enabled.go new file mode 100644 index 000000000..2a39879ec --- /dev/null +++ b/vendor/github.com/tetratelabs/wazero/internal/engine/wazevo/wazevoapi/perfmap_enabled.go @@ -0,0 +1,5 @@ +//go:build perfmap + +package wazevoapi + +const PerfMapEnabled = true diff --git a/vendor/github.com/tetratelabs/wazero/internal/engine/wazevo/wazevoapi/pool.go b/vendor/github.com/tetratelabs/wazero/internal/engine/wazevo/wazevoapi/pool.go new file mode 100644 index 000000000..3149fdc9e --- /dev/null +++ b/vendor/github.com/tetratelabs/wazero/internal/engine/wazevo/wazevoapi/pool.go @@ -0,0 +1,215 @@ +package wazevoapi + +const poolPageSize = 128 + +// Pool is a pool of T that can be allocated and reset. +// This is useful to avoid unnecessary allocations. +type Pool[T any] struct { + pages []*[poolPageSize]T + resetFn func(*T) + allocated, index int +} + +// NewPool returns a new Pool. +// resetFn is called when a new T is allocated in Pool.Allocate. +func NewPool[T any](resetFn func(*T)) Pool[T] { + var ret Pool[T] + ret.resetFn = resetFn + ret.Reset() + return ret +} + +// Allocated returns the number of allocated T currently in the pool. +func (p *Pool[T]) Allocated() int { + return p.allocated +} + +// Allocate allocates a new T from the pool. +func (p *Pool[T]) Allocate() *T { + if p.index == poolPageSize { + if len(p.pages) == cap(p.pages) { + p.pages = append(p.pages, new([poolPageSize]T)) + } else { + i := len(p.pages) + p.pages = p.pages[:i+1] + if p.pages[i] == nil { + p.pages[i] = new([poolPageSize]T) + } + } + p.index = 0 + } + ret := &p.pages[len(p.pages)-1][p.index] + if p.resetFn != nil { + p.resetFn(ret) + } + p.index++ + p.allocated++ + return ret +} + +// View returns the pointer to i-th item from the pool. +func (p *Pool[T]) View(i int) *T { + page, index := i/poolPageSize, i%poolPageSize + return &p.pages[page][index] +} + +// Reset resets the pool. +func (p *Pool[T]) Reset() { + p.pages = p.pages[:0] + p.index = poolPageSize + p.allocated = 0 +} + +// IDedPool is a pool of T that can be allocated and reset, with a way to get T by an ID. +type IDedPool[T any] struct { + pool Pool[T] + idToItems []*T + maxIDEncountered int +} + +// NewIDedPool returns a new IDedPool. +func NewIDedPool[T any](resetFn func(*T)) IDedPool[T] { + return IDedPool[T]{pool: NewPool[T](resetFn)} +} + +// GetOrAllocate returns the T with the given id. +func (p *IDedPool[T]) GetOrAllocate(id int) *T { + if p.maxIDEncountered < id { + p.maxIDEncountered = id + } + if id >= len(p.idToItems) { + p.idToItems = append(p.idToItems, make([]*T, id-len(p.idToItems)+1)...) + } + if p.idToItems[id] == nil { + p.idToItems[id] = p.pool.Allocate() + } + return p.idToItems[id] +} + +// Get returns the T with the given id, or nil if it's not allocated. +func (p *IDedPool[T]) Get(id int) *T { + if id >= len(p.idToItems) { + return nil + } + return p.idToItems[id] +} + +// Reset resets the pool. +func (p *IDedPool[T]) Reset() { + p.pool.Reset() + for i := range p.idToItems { + p.idToItems[i] = nil + } + p.maxIDEncountered = -1 +} + +// MaxIDEncountered returns the maximum id encountered so far. +func (p *IDedPool[T]) MaxIDEncountered() int { + return p.maxIDEncountered +} + +// arraySize is the size of the array used in VarLengthPool's arrayPool. +// This is chosen to be 8, which is empirically a good number among 8, 12, 16 and 20. +const arraySize = 8 + +// VarLengthPool is a pool of VarLength[T] that can be allocated and reset. +type ( + VarLengthPool[T any] struct { + arrayPool Pool[varLengthPoolArray[T]] + slicePool Pool[[]T] + } + // varLengthPoolArray wraps an array and keeps track of the next index to be used to avoid the heap allocation. + varLengthPoolArray[T any] struct { + arr [arraySize]T + next int + } +) + +// VarLength is a variable length array that can be reused via a pool. +type VarLength[T any] struct { + arr *varLengthPoolArray[T] + slc *[]T +} + +// NewVarLengthPool returns a new VarLengthPool. +func NewVarLengthPool[T any]() VarLengthPool[T] { + return VarLengthPool[T]{ + arrayPool: NewPool[varLengthPoolArray[T]](func(v *varLengthPoolArray[T]) { + v.next = 0 + }), + slicePool: NewPool[[]T](func(i *[]T) { + *i = (*i)[:0] + }), + } +} + +// NewNilVarLength returns a new VarLength[T] with a nil backing. +func NewNilVarLength[T any]() VarLength[T] { + return VarLength[T]{} +} + +// Allocate allocates a new VarLength[T] from the pool. +func (p *VarLengthPool[T]) Allocate(knownMin int) VarLength[T] { + if knownMin <= arraySize { + arr := p.arrayPool.Allocate() + return VarLength[T]{arr: arr} + } + slc := p.slicePool.Allocate() + return VarLength[T]{slc: slc} +} + +// Reset resets the pool. +func (p *VarLengthPool[T]) Reset() { + p.arrayPool.Reset() + p.slicePool.Reset() +} + +// Append appends items to the backing slice just like the `append` builtin function in Go. +func (i VarLength[T]) Append(p *VarLengthPool[T], items ...T) VarLength[T] { + if i.slc != nil { + *i.slc = append(*i.slc, items...) + return i + } + + if i.arr == nil { + i.arr = p.arrayPool.Allocate() + } + + arr := i.arr + if arr.next+len(items) <= arraySize { + for _, item := range items { + arr.arr[arr.next] = item + arr.next++ + } + } else { + slc := p.slicePool.Allocate() + // Copy the array to the slice. + for ptr := 0; ptr < arr.next; ptr++ { + *slc = append(*slc, arr.arr[ptr]) + } + i.slc = slc + *i.slc = append(*i.slc, items...) + } + return i +} + +// View returns the backing slice. +func (i VarLength[T]) View() []T { + if i.slc != nil { + return *i.slc + } else if i.arr != nil { + arr := i.arr + return arr.arr[:arr.next] + } + return nil +} + +// Cut cuts the backing slice to the given length. +// Precondition: n <= len(i.backing). +func (i VarLength[T]) Cut(n int) { + if i.slc != nil { + *i.slc = (*i.slc)[:n] + } else if i.arr != nil { + i.arr.next = n + } +} diff --git a/vendor/github.com/tetratelabs/wazero/internal/engine/wazevo/wazevoapi/ptr.go b/vendor/github.com/tetratelabs/wazero/internal/engine/wazevo/wazevoapi/ptr.go new file mode 100644 index 000000000..f21e1a5d8 --- /dev/null +++ b/vendor/github.com/tetratelabs/wazero/internal/engine/wazevo/wazevoapi/ptr.go @@ -0,0 +1,15 @@ +package wazevoapi + +import "unsafe" + +// PtrFromUintptr resurrects the original *T from the given uintptr. +// The caller of this function MUST be sure that ptr is valid. +func PtrFromUintptr[T any](ptr uintptr) *T { + // Wraps ptrs as the double pointer in order to avoid the unsafe access as detected by race detector. + // + // For example, if we have (*function)(unsafe.Pointer(ptr)) instead, then the race detector's "checkptr" + // subroutine wanrs as "checkptr: pointer arithmetic result points to invalid allocation" + // https://github.com/golang/go/blob/1ce7fcf139417d618c2730010ede2afb41664211/src/runtime/checkptr.go#L69 + var wrapped *uintptr = &ptr + return *(**T)(unsafe.Pointer(wrapped)) +} diff --git a/vendor/github.com/tetratelabs/wazero/internal/engine/wazevo/wazevoapi/queue.go b/vendor/github.com/tetratelabs/wazero/internal/engine/wazevo/wazevoapi/queue.go new file mode 100644 index 000000000..e3118fa69 --- /dev/null +++ b/vendor/github.com/tetratelabs/wazero/internal/engine/wazevo/wazevoapi/queue.go @@ -0,0 +1,26 @@ +package wazevoapi + +// Queue is the resettable queue where the underlying slice is reused. +type Queue[T any] struct { + index int + Data []T +} + +func (q *Queue[T]) Enqueue(v T) { + q.Data = append(q.Data, v) +} + +func (q *Queue[T]) Dequeue() (ret T) { + ret = q.Data[q.index] + q.index++ + return +} + +func (q *Queue[T]) Empty() bool { + return q.index >= len(q.Data) +} + +func (q *Queue[T]) Reset() { + q.index = 0 + q.Data = q.Data[:0] +} diff --git a/vendor/github.com/tetratelabs/wazero/internal/engine/wazevo/wazevoapi/resetmap.go b/vendor/github.com/tetratelabs/wazero/internal/engine/wazevo/wazevoapi/resetmap.go new file mode 100644 index 000000000..7177fbb4b --- /dev/null +++ b/vendor/github.com/tetratelabs/wazero/internal/engine/wazevo/wazevoapi/resetmap.go @@ -0,0 +1,13 @@ +package wazevoapi + +// ResetMap resets the map to an empty state, or creates a new map if it is nil. +func ResetMap[K comparable, V any](m map[K]V) map[K]V { + if m == nil { + m = make(map[K]V) + } else { + for v := range m { + delete(m, v) + } + } + return m +} diff --git a/vendor/github.com/tetratelabs/wazero/internal/expctxkeys/checkpoint.go b/vendor/github.com/tetratelabs/wazero/internal/expctxkeys/checkpoint.go new file mode 100644 index 000000000..fc62e83f3 --- /dev/null +++ b/vendor/github.com/tetratelabs/wazero/internal/expctxkeys/checkpoint.go @@ -0,0 +1,10 @@ +package expctxkeys + +// EnableSnapshotterKey is a context key to indicate that snapshotting should be enabled. +// The context.Context passed to a exported function invocation should have this key set +// to a non-nil value, and host functions will be able to retrieve it using SnapshotterKey. +type EnableSnapshotterKey struct{} + +// SnapshotterKey is a context key to access a Snapshotter from a host function. +// It is only present if EnableSnapshotter was set in the function invocation context. +type SnapshotterKey struct{} diff --git a/vendor/github.com/tetratelabs/wazero/internal/expctxkeys/close.go b/vendor/github.com/tetratelabs/wazero/internal/expctxkeys/close.go new file mode 100644 index 000000000..75e5134e5 --- /dev/null +++ b/vendor/github.com/tetratelabs/wazero/internal/expctxkeys/close.go @@ -0,0 +1,5 @@ +package expctxkeys + +// CloseNotifierKey is a context.Context Value key. Its associated value should be a +// Notifier. +type CloseNotifierKey struct{} diff --git a/vendor/github.com/tetratelabs/wazero/internal/expctxkeys/expctxkeys.go b/vendor/github.com/tetratelabs/wazero/internal/expctxkeys/expctxkeys.go new file mode 100644 index 000000000..6800005b9 --- /dev/null +++ b/vendor/github.com/tetratelabs/wazero/internal/expctxkeys/expctxkeys.go @@ -0,0 +1,2 @@ +// Package expctxkeys provides keys for the context used to store the experimental APIs. +package expctxkeys diff --git a/vendor/github.com/tetratelabs/wazero/internal/expctxkeys/listener.go b/vendor/github.com/tetratelabs/wazero/internal/expctxkeys/listener.go new file mode 100644 index 000000000..9565db8e9 --- /dev/null +++ b/vendor/github.com/tetratelabs/wazero/internal/expctxkeys/listener.go @@ -0,0 +1,7 @@ +package expctxkeys + +// FunctionListenerFactoryKey is a context.Context Value key. +// Its associated value should be a FunctionListenerFactory. +// +// See https://github.com/tetratelabs/wazero/issues/451 +type FunctionListenerFactoryKey struct{} diff --git a/vendor/github.com/tetratelabs/wazero/internal/expctxkeys/memory.go b/vendor/github.com/tetratelabs/wazero/internal/expctxkeys/memory.go new file mode 100644 index 000000000..d41c01914 --- /dev/null +++ b/vendor/github.com/tetratelabs/wazero/internal/expctxkeys/memory.go @@ -0,0 +1,4 @@ +package expctxkeys + +// MemoryAllocatorKey is a context.Context key for the experimental memory allocator. +type MemoryAllocatorKey struct{} diff --git a/vendor/github.com/tetratelabs/wazero/internal/filecache/compilationcache.go b/vendor/github.com/tetratelabs/wazero/internal/filecache/compilationcache.go new file mode 100644 index 000000000..b2dbd4650 --- /dev/null +++ b/vendor/github.com/tetratelabs/wazero/internal/filecache/compilationcache.go @@ -0,0 +1,42 @@ +package filecache + +import ( + "crypto/sha256" + "io" +) + +// Cache allows the compiler engine to skip compilation of wasm to machine code +// where doing so is redundant for the same wasm binary and version of wazero. +// +// This augments the default in-memory cache of compiled functions, by +// decoupling it from a wazero.Runtime instance. Concretely, a runtime loses +// its cache once closed. This cache allows the runtime to rebuild its +// in-memory cache quicker, significantly reducing first-hit penalty on a hit. +// +// See New for the example implementation. +type Cache interface { + // Get is called when the runtime is trying to get the cached compiled functions. + // Implementations are supposed to return compiled function in io.Reader with ok=true + // if the key exists on the cache. In the case of not-found, this should return + // ok=false with err=nil. content.Close() is automatically called by + // the caller of this Get. + // + // Note: the returned content won't go through the validation pass of Wasm binary + // which is applied when the binary is compiled from scratch without cache hit. + Get(key Key) (content io.ReadCloser, ok bool, err error) + // + // Add is called when the runtime is trying to add the new cache entry. + // The given `content` must be un-modified, and returned as-is in Get method. + // + // Note: the `content` is ensured to be safe through the validation phase applied on the Wasm binary. + Add(key Key, content io.Reader) (err error) + // + // Delete is called when the cache on the `key` returned by Get is no longer usable, and + // must be purged. Specifically, this is called happens when the wazero's version has been changed. + // For example, that is when there's a difference between the version of compiling wazero and the + // version of the currently used wazero. + Delete(key Key) (err error) +} + +// Key represents the 256-bit unique identifier assigned to each cache entry. +type Key = [sha256.Size]byte diff --git a/vendor/github.com/tetratelabs/wazero/internal/filecache/file_cache.go b/vendor/github.com/tetratelabs/wazero/internal/filecache/file_cache.go new file mode 100644 index 000000000..940a79a8d --- /dev/null +++ b/vendor/github.com/tetratelabs/wazero/internal/filecache/file_cache.go @@ -0,0 +1,76 @@ +package filecache + +import ( + "encoding/hex" + "errors" + "io" + "os" + "path" + "path/filepath" +) + +// New returns a new Cache implemented by fileCache. +func New(dir string) Cache { + return newFileCache(dir) +} + +func newFileCache(dir string) *fileCache { + return &fileCache{dirPath: dir} +} + +// fileCache persists compiled functions into dirPath. +// +// Note: this can be expanded to do binary signing/verification, set TTL on each entry, etc. +type fileCache struct { + dirPath string +} + +func (fc *fileCache) path(key Key) string { + return path.Join(fc.dirPath, hex.EncodeToString(key[:])) +} + +func (fc *fileCache) Get(key Key) (content io.ReadCloser, ok bool, err error) { + f, err := os.Open(fc.path(key)) + if errors.Is(err, os.ErrNotExist) { + return nil, false, nil + } else if err != nil { + return nil, false, err + } else { + return f, true, nil + } +} + +func (fc *fileCache) Add(key Key, content io.Reader) (err error) { + path := fc.path(key) + dirPath, fileName := filepath.Split(path) + + file, err := os.CreateTemp(dirPath, fileName+".*.tmp") + if err != nil { + return + } + defer func() { + file.Close() + if err != nil { + _ = os.Remove(file.Name()) + } + }() + if _, err = io.Copy(file, content); err != nil { + return + } + if err = file.Sync(); err != nil { + return + } + if err = file.Close(); err != nil { + return + } + err = os.Rename(file.Name(), path) + return +} + +func (fc *fileCache) Delete(key Key) (err error) { + err = os.Remove(fc.path(key)) + if errors.Is(err, os.ErrNotExist) { + err = nil + } + return +} diff --git a/vendor/github.com/tetratelabs/wazero/internal/fsapi/file.go b/vendor/github.com/tetratelabs/wazero/internal/fsapi/file.go new file mode 100644 index 000000000..0640b2271 --- /dev/null +++ b/vendor/github.com/tetratelabs/wazero/internal/fsapi/file.go @@ -0,0 +1,69 @@ +package fsapi + +import experimentalsys "github.com/tetratelabs/wazero/experimental/sys" + +// File includes methods not yet ready to document for end users, notably +// non-blocking functionality. +// +// Particularly, Poll is subject to debate. For example, whether a user should +// be able to choose how to implement timeout or not. Currently, this interface +// allows the user to choose to sleep or use native polling, and which choice +// they make impacts thread behavior as summarized here: +// https://github.com/tetratelabs/wazero/pull/1606#issuecomment-1665475516 +type File interface { + experimentalsys.File + + // IsNonblock returns true if the file was opened with O_NONBLOCK, or + // SetNonblock was successfully enabled on this file. + // + // # Notes + // + // - This might not match the underlying state of the file descriptor if + // the file was not opened via OpenFile. + IsNonblock() bool + + // SetNonblock toggles the non-blocking mode (O_NONBLOCK) of this file. + // + // # Errors + // + // A zero Errno is success. The below are expected otherwise: + // - ENOSYS: the implementation does not support this function. + // - EBADF: the file or directory was closed. + // + // # Notes + // + // - This is like syscall.SetNonblock and `fcntl` with O_NONBLOCK in + // POSIX. See https://pubs.opengroup.org/onlinepubs/9699919799/functions/fcntl.html + SetNonblock(enable bool) experimentalsys.Errno + + // Poll returns if the file has data ready to be read or written. + // + // # Parameters + // + // The `flag` parameter determines which event to await, such as POLLIN, + // POLLOUT, or a combination like `POLLIN|POLLOUT`. + // + // The `timeoutMillis` parameter is how long to block for an event, or + // interrupted, in milliseconds. There are two special values: + // - zero returns immediately + // - any negative value blocks any amount of time + // + // # Results + // + // `ready` means there was data ready to read or written. False can mean no + // event was ready or `errno` is not zero. + // + // A zero `errno` is success. The below are expected otherwise: + // - ENOSYS: the implementation does not support this function. + // - ENOTSUP: the implementation does not the flag combination. + // - EINTR: the call was interrupted prior to an event. + // + // # Notes + // + // - This is like `poll` in POSIX, for a single file. + // See https://pubs.opengroup.org/onlinepubs/9699919799/functions/poll.html + // - No-op files, such as those which read from /dev/null, should return + // immediately true, as data will never become available. + // - See /RATIONALE.md for detailed notes including impact of blocking. + Poll(flag Pflag, timeoutMillis int32) (ready bool, errno experimentalsys.Errno) +} diff --git a/vendor/github.com/tetratelabs/wazero/internal/fsapi/poll.go b/vendor/github.com/tetratelabs/wazero/internal/fsapi/poll.go new file mode 100644 index 000000000..25f7c5711 --- /dev/null +++ b/vendor/github.com/tetratelabs/wazero/internal/fsapi/poll.go @@ -0,0 +1,20 @@ +package fsapi + +// Pflag are bit flags used for File.Poll. Values, including zero, should not +// be interpreted numerically. Instead, use by constants prefixed with 'POLL'. +// +// # Notes +// +// - This is like `pollfd.events` flags for `poll` in POSIX. See +// https://pubs.opengroup.org/onlinepubs/9699919799/basedefs/poll.h.html +type Pflag uint32 + +// Only define bitflags we support and are needed by `poll_oneoff` in wasip1 +// See https://github.com/WebAssembly/WASI/blob/snapshot-01/phases/snapshot/docs.md#eventrwflags +const ( + // POLLIN is a read event. + POLLIN Pflag = 1 << iota + + // POLLOUT is a write event. + POLLOUT +) diff --git a/vendor/github.com/tetratelabs/wazero/internal/fsapi/unimplemented.go b/vendor/github.com/tetratelabs/wazero/internal/fsapi/unimplemented.go new file mode 100644 index 000000000..99d9c2db3 --- /dev/null +++ b/vendor/github.com/tetratelabs/wazero/internal/fsapi/unimplemented.go @@ -0,0 +1,27 @@ +package fsapi + +import experimentalsys "github.com/tetratelabs/wazero/experimental/sys" + +func Adapt(f experimentalsys.File) File { + if f, ok := f.(File); ok { + return f + } + return unimplementedFile{f} +} + +type unimplementedFile struct{ experimentalsys.File } + +// IsNonblock implements File.IsNonblock +func (unimplementedFile) IsNonblock() bool { + return false +} + +// SetNonblock implements File.SetNonblock +func (unimplementedFile) SetNonblock(bool) experimentalsys.Errno { + return experimentalsys.ENOSYS +} + +// Poll implements File.Poll +func (unimplementedFile) Poll(Pflag, int32) (ready bool, errno experimentalsys.Errno) { + return false, experimentalsys.ENOSYS +} diff --git a/vendor/github.com/tetratelabs/wazero/internal/ieee754/ieee754.go b/vendor/github.com/tetratelabs/wazero/internal/ieee754/ieee754.go new file mode 100644 index 000000000..0c9298957 --- /dev/null +++ b/vendor/github.com/tetratelabs/wazero/internal/ieee754/ieee754.go @@ -0,0 +1,29 @@ +package ieee754 + +import ( + "encoding/binary" + "io" + "math" +) + +// DecodeFloat32 decodes a float32 in IEEE 754 binary representation. +// See https://www.w3.org/TR/2019/REC-wasm-core-1-20191205/#floating-point%E2%91%A2 +func DecodeFloat32(buf []byte) (float32, error) { + if len(buf) < 4 { + return 0, io.ErrUnexpectedEOF + } + + raw := binary.LittleEndian.Uint32(buf[:4]) + return math.Float32frombits(raw), nil +} + +// DecodeFloat64 decodes a float64 in IEEE 754 binary representation. +// See https://www.w3.org/TR/2019/REC-wasm-core-1-20191205/#floating-point%E2%91%A2 +func DecodeFloat64(buf []byte) (float64, error) { + if len(buf) < 8 { + return 0, io.ErrUnexpectedEOF + } + + raw := binary.LittleEndian.Uint64(buf) + return math.Float64frombits(raw), nil +} diff --git a/vendor/github.com/tetratelabs/wazero/internal/internalapi/internal.go b/vendor/github.com/tetratelabs/wazero/internal/internalapi/internal.go new file mode 100644 index 000000000..a4f354355 --- /dev/null +++ b/vendor/github.com/tetratelabs/wazero/internal/internalapi/internal.go @@ -0,0 +1,9 @@ +package internalapi + +type WazeroOnly interface { + wazeroOnly() +} + +type WazeroOnlyType struct{} + +func (WazeroOnlyType) wazeroOnly() {} diff --git a/vendor/github.com/tetratelabs/wazero/internal/leb128/leb128.go b/vendor/github.com/tetratelabs/wazero/internal/leb128/leb128.go new file mode 100644 index 000000000..a31051724 --- /dev/null +++ b/vendor/github.com/tetratelabs/wazero/internal/leb128/leb128.go @@ -0,0 +1,285 @@ +package leb128 + +import ( + "errors" + "fmt" + "io" +) + +const ( + maxVarintLen32 = 5 + maxVarintLen33 = maxVarintLen32 + maxVarintLen64 = 10 + + int33Mask int64 = 1 << 7 + int33Mask2 = ^int33Mask + int33Mask3 = 1 << 6 + int33Mask4 = 8589934591 // 2^33-1 + int33Mask5 = 1 << 32 + int33Mask6 = int33Mask4 + 1 // 2^33 + + int64Mask3 = 1 << 6 + int64Mask4 = ^0 +) + +var ( + errOverflow32 = errors.New("overflows a 32-bit integer") + errOverflow33 = errors.New("overflows a 33-bit integer") + errOverflow64 = errors.New("overflows a 64-bit integer") +) + +// EncodeInt32 encodes the signed value into a buffer in LEB128 format +// +// See https://en.wikipedia.org/wiki/LEB128#Encode_signed_integer +func EncodeInt32(value int32) []byte { + return EncodeInt64(int64(value)) +} + +// EncodeInt64 encodes the signed value into a buffer in LEB128 format +// +// See https://en.wikipedia.org/wiki/LEB128#Encode_signed_integer +func EncodeInt64(value int64) (buf []byte) { + for { + // Take 7 remaining low-order bits from the value into b. + b := uint8(value & 0x7f) + // Extract the sign bit. + s := uint8(value & 0x40) + value >>= 7 + + // The encoding unsigned numbers is simpler as it only needs to check if the value is non-zero to tell if there + // are more bits to encode. Signed is a little more complicated as you have to double-check the sign bit. + // If either case, set the high-order bit to tell the reader there are more bytes in this int. + if (value != -1 || s == 0) && (value != 0 || s != 0) { + b |= 0x80 + } + + // Append b into the buffer + buf = append(buf, b) + if b&0x80 == 0 { + break + } + } + return buf +} + +// EncodeUint32 encodes the value into a buffer in LEB128 format +// +// See https://en.wikipedia.org/wiki/LEB128#Encode_unsigned_integer +func EncodeUint32(value uint32) []byte { + return EncodeUint64(uint64(value)) +} + +// EncodeUint64 encodes the value into a buffer in LEB128 format +// +// See https://en.wikipedia.org/wiki/LEB128#Encode_unsigned_integer +func EncodeUint64(value uint64) (buf []byte) { + // This is effectively a do/while loop where we take 7 bits of the value and encode them until it is zero. + for { + // Take 7 remaining low-order bits from the value into b. + b := uint8(value & 0x7f) + value = value >> 7 + + // If there are remaining bits, the value won't be zero: Set the high- + // order bit to tell the reader there are more bytes in this uint. + if value != 0 { + b |= 0x80 + } + + // Append b into the buffer + buf = append(buf, b) + if b&0x80 == 0 { + return buf + } + } +} + +type nextByte func(i int) (byte, error) + +func DecodeUint32(r io.ByteReader) (ret uint32, bytesRead uint64, err error) { + return decodeUint32(func(_ int) (byte, error) { return r.ReadByte() }) +} + +func LoadUint32(buf []byte) (ret uint32, bytesRead uint64, err error) { + return decodeUint32(func(i int) (byte, error) { + if i >= len(buf) { + return 0, io.EOF + } + return buf[i], nil + }) +} + +func decodeUint32(next nextByte) (ret uint32, bytesRead uint64, err error) { + // Derived from https://github.com/golang/go/blob/go1.20/src/encoding/binary/varint.go + // with the modification on the overflow handling tailored for 32-bits. + var s uint32 + for i := 0; i < maxVarintLen32; i++ { + b, err := next(i) + if err != nil { + return 0, 0, err + } + if b < 0x80 { + // Unused bits must be all zero. + if i == maxVarintLen32-1 && (b&0xf0) > 0 { + return 0, 0, errOverflow32 + } + return ret | uint32(b)<= bufLen { + return 0, 0, io.EOF + } + b := buf[i] + if b < 0x80 { + // Unused bits (non first bit) must all be zero. + if i == maxVarintLen64-1 && b > 1 { + return 0, 0, errOverflow64 + } + return ret | uint64(b)<= len(buf) { + return 0, io.EOF + } + return buf[i], nil + }) +} + +func decodeInt32(next nextByte) (ret int32, bytesRead uint64, err error) { + var shift int + var b byte + for { + b, err = next(int(bytesRead)) + if err != nil { + return 0, 0, fmt.Errorf("readByte failed: %w", err) + } + ret |= (int32(b) & 0x7f) << shift + shift += 7 + bytesRead++ + if b&0x80 == 0 { + if shift < 32 && (b&0x40) != 0 { + ret |= ^0 << shift + } + // Over flow checks. + // fixme: can be optimized. + if bytesRead > maxVarintLen32 { + return 0, 0, errOverflow32 + } else if unused := b & 0b00110000; bytesRead == maxVarintLen32 && ret < 0 && unused != 0b00110000 { + return 0, 0, errOverflow32 + } else if bytesRead == maxVarintLen32 && ret >= 0 && unused != 0x00 { + return 0, 0, errOverflow32 + } + return + } + } +} + +// DecodeInt33AsInt64 is a special cased decoder for wasm.BlockType which is encoded as a positive signed integer, yet +// still needs to fit the 32-bit range of allowed indices. Hence, this is 33, not 32-bit! +// +// See https://webassembly.github.io/spec/core/binary/instructions.html#control-instructions +func DecodeInt33AsInt64(r io.ByteReader) (ret int64, bytesRead uint64, err error) { + var shift int + var b int64 + var rb byte + for shift < 35 { + rb, err = r.ReadByte() + if err != nil { + return 0, 0, fmt.Errorf("readByte failed: %w", err) + } + b = int64(rb) + ret |= (b & int33Mask2) << shift + shift += 7 + bytesRead++ + if b&int33Mask == 0 { + break + } + } + + // fixme: can be optimized + if shift < 33 && (b&int33Mask3) == int33Mask3 { + ret |= int33Mask4 << shift + } + ret = ret & int33Mask4 + + // if 33rd bit == 1, we translate it as a corresponding signed-33bit minus value + if ret&int33Mask5 > 0 { + ret = ret - int33Mask6 + } + // Over flow checks. + // fixme: can be optimized. + if bytesRead > maxVarintLen33 { + return 0, 0, errOverflow33 + } else if unused := b & 0b00100000; bytesRead == maxVarintLen33 && ret < 0 && unused != 0b00100000 { + return 0, 0, errOverflow33 + } else if bytesRead == maxVarintLen33 && ret >= 0 && unused != 0x00 { + return 0, 0, errOverflow33 + } + return ret, bytesRead, nil +} + +func DecodeInt64(r io.ByteReader) (ret int64, bytesRead uint64, err error) { + return decodeInt64(func(_ int) (byte, error) { return r.ReadByte() }) +} + +func LoadInt64(buf []byte) (ret int64, bytesRead uint64, err error) { + return decodeInt64(func(i int) (byte, error) { + if i >= len(buf) { + return 0, io.EOF + } + return buf[i], nil + }) +} + +func decodeInt64(next nextByte) (ret int64, bytesRead uint64, err error) { + var shift int + var b byte + for { + b, err = next(int(bytesRead)) + if err != nil { + return 0, 0, fmt.Errorf("readByte failed: %w", err) + } + ret |= (int64(b) & 0x7f) << shift + shift += 7 + bytesRead++ + if b&0x80 == 0 { + if shift < 64 && (b&int64Mask3) == int64Mask3 { + ret |= int64Mask4 << shift + } + // Over flow checks. + // fixme: can be optimized. + if bytesRead > maxVarintLen64 { + return 0, 0, errOverflow64 + } else if unused := b & 0b00111110; bytesRead == maxVarintLen64 && ret < 0 && unused != 0b00111110 { + return 0, 0, errOverflow64 + } else if bytesRead == maxVarintLen64 && ret >= 0 && unused != 0x00 { + return 0, 0, errOverflow64 + } + return + } + } +} diff --git a/vendor/github.com/tetratelabs/wazero/internal/moremath/moremath.go b/vendor/github.com/tetratelabs/wazero/internal/moremath/moremath.go new file mode 100644 index 000000000..4741f07bb --- /dev/null +++ b/vendor/github.com/tetratelabs/wazero/internal/moremath/moremath.go @@ -0,0 +1,271 @@ +package moremath + +import ( + "math" +) + +// https://www.w3.org/TR/2022/WD-wasm-core-2-20220419/syntax/values.html#floating-point +const ( + // F32CanonicalNaNBits is the 32-bit float where payload's MSB equals 1 and others are all zero. + F32CanonicalNaNBits = uint32(0x7fc0_0000) + // F32CanonicalNaNBitsMask can be used to judge the value `v` is canonical nan as "v&F32CanonicalNaNBitsMask == F32CanonicalNaNBits" + F32CanonicalNaNBitsMask = uint32(0x7fff_ffff) + // F64CanonicalNaNBits is the 64-bit float where payload's MSB equals 1 and others are all zero. + F64CanonicalNaNBits = uint64(0x7ff8_0000_0000_0000) + // F64CanonicalNaNBitsMask can be used to judge the value `v` is canonical nan as "v&F64CanonicalNaNBitsMask == F64CanonicalNaNBits" + F64CanonicalNaNBitsMask = uint64(0x7fff_ffff_ffff_ffff) + // F32ArithmeticNaNPayloadMSB is used to extract the most significant bit of payload of 32-bit arithmetic NaN values + F32ArithmeticNaNPayloadMSB = uint32(0x0040_0000) + // F32ExponentMask is used to extract the exponent of 32-bit floating point. + F32ExponentMask = uint32(0x7f80_0000) + // F32ArithmeticNaNBits is an example 32-bit arithmetic NaN. + F32ArithmeticNaNBits = F32CanonicalNaNBits | 0b1 // Set first bit to make this different from the canonical NaN. + // F64ArithmeticNaNPayloadMSB is used to extract the most significant bit of payload of 64-bit arithmetic NaN values + F64ArithmeticNaNPayloadMSB = uint64(0x0008_0000_0000_0000) + // F64ExponentMask is used to extract the exponent of 64-bit floating point. + F64ExponentMask = uint64(0x7ff0_0000_0000_0000) + // F64ArithmeticNaNBits is an example 64-bit arithmetic NaN. + F64ArithmeticNaNBits = F64CanonicalNaNBits | 0b1 // Set first bit to make this different from the canonical NaN. +) + +// WasmCompatMin64 is the Wasm spec compatible variant of math.Min for 64-bit floating points. +func WasmCompatMin64(x, y float64) float64 { + switch { + case math.IsNaN(x) || math.IsNaN(y): + return returnF64NaNBinOp(x, y) + case math.IsInf(x, -1) || math.IsInf(y, -1): + return math.Inf(-1) + case x == 0 && x == y: + if math.Signbit(x) { + return x + } + return y + } + if x < y { + return x + } + return y +} + +// WasmCompatMin32 is the Wasm spec compatible variant of math.Min for 32-bit floating points. +func WasmCompatMin32(x, y float32) float32 { + x64, y64 := float64(x), float64(y) + switch { + case math.IsNaN(x64) || math.IsNaN(y64): + return returnF32NaNBinOp(x, y) + case math.IsInf(x64, -1) || math.IsInf(y64, -1): + return float32(math.Inf(-1)) + case x == 0 && x == y: + if math.Signbit(x64) { + return x + } + return y + } + if x < y { + return x + } + return y +} + +// WasmCompatMax64 is the Wasm spec compatible variant of math.Max for 64-bit floating points. +func WasmCompatMax64(x, y float64) float64 { + switch { + case math.IsNaN(x) || math.IsNaN(y): + return returnF64NaNBinOp(x, y) + case math.IsInf(x, 1) || math.IsInf(y, 1): + return math.Inf(1) + case x == 0 && x == y: + if math.Signbit(x) { + return y + } + return x + } + if x > y { + return x + } + return y +} + +// WasmCompatMax32 is the Wasm spec compatible variant of math.Max for 32-bit floating points. +func WasmCompatMax32(x, y float32) float32 { + x64, y64 := float64(x), float64(y) + switch { + case math.IsNaN(x64) || math.IsNaN(y64): + return returnF32NaNBinOp(x, y) + case math.IsInf(x64, 1) || math.IsInf(y64, 1): + return float32(math.Inf(1)) + case x == 0 && x == y: + if math.Signbit(x64) { + return y + } + return x + } + if x > y { + return x + } + return y +} + +// WasmCompatNearestF32 is the Wasm spec compatible variant of math.Round, used for Nearest instruction. +// For example, this converts 1.9 to 2.0, and this has the semantics of LLVM's rint intrinsic. +// +// e.g. math.Round(-4.5) results in -5 while this results in -4. +// +// See https://llvm.org/docs/LangRef.html#llvm-rint-intrinsic. +func WasmCompatNearestF32(f float32) float32 { + var res float32 + // TODO: look at https://github.com/bytecodealliance/wasmtime/pull/2171 and reconsider this algorithm + if f != 0 { + ceil := float32(math.Ceil(float64(f))) + floor := float32(math.Floor(float64(f))) + distToCeil := math.Abs(float64(f - ceil)) + distToFloor := math.Abs(float64(f - floor)) + h := ceil / 2.0 + if distToCeil < distToFloor { + res = ceil + } else if distToCeil == distToFloor && float32(math.Floor(float64(h))) == h { + res = ceil + } else { + res = floor + } + } else { + res = f + } + return returnF32UniOp(f, res) +} + +// WasmCompatNearestF64 is the Wasm spec compatible variant of math.Round, used for Nearest instruction. +// For example, this converts 1.9 to 2.0, and this has the semantics of LLVM's rint intrinsic. +// +// e.g. math.Round(-4.5) results in -5 while this results in -4. +// +// See https://llvm.org/docs/LangRef.html#llvm-rint-intrinsic. +func WasmCompatNearestF64(f float64) float64 { + // TODO: look at https://github.com/bytecodealliance/wasmtime/pull/2171 and reconsider this algorithm + var res float64 + if f != 0 { + ceil := math.Ceil(f) + floor := math.Floor(f) + distToCeil := math.Abs(f - ceil) + distToFloor := math.Abs(f - floor) + h := ceil / 2.0 + if distToCeil < distToFloor { + res = ceil + } else if distToCeil == distToFloor && math.Floor(h) == h { + res = ceil + } else { + res = floor + } + } else { + res = f + } + return returnF64UniOp(f, res) +} + +// WasmCompatCeilF32 is the same as math.Ceil on 32-bit except that +// the returned NaN value follows the Wasm specification on NaN +// propagation. +// https://www.w3.org/TR/2022/WD-wasm-core-2-20220419/exec/numerics.html#nan-propagation +func WasmCompatCeilF32(f float32) float32 { + return returnF32UniOp(f, float32(math.Ceil(float64(f)))) +} + +// WasmCompatCeilF64 is the same as math.Ceil on 64-bit except that +// the returned NaN value follows the Wasm specification on NaN +// propagation. +// https://www.w3.org/TR/2022/WD-wasm-core-2-20220419/exec/numerics.html#nan-propagation +func WasmCompatCeilF64(f float64) float64 { + return returnF64UniOp(f, math.Ceil(f)) +} + +// WasmCompatFloorF32 is the same as math.Floor on 32-bit except that +// the returned NaN value follows the Wasm specification on NaN +// propagation. +// https://www.w3.org/TR/2022/WD-wasm-core-2-20220419/exec/numerics.html#nan-propagation +func WasmCompatFloorF32(f float32) float32 { + return returnF32UniOp(f, float32(math.Floor(float64(f)))) +} + +// WasmCompatFloorF64 is the same as math.Floor on 64-bit except that +// the returned NaN value follows the Wasm specification on NaN +// propagation. +// https://www.w3.org/TR/2022/WD-wasm-core-2-20220419/exec/numerics.html#nan-propagation +func WasmCompatFloorF64(f float64) float64 { + return returnF64UniOp(f, math.Floor(f)) +} + +// WasmCompatTruncF32 is the same as math.Trunc on 32-bit except that +// the returned NaN value follows the Wasm specification on NaN +// propagation. +// https://www.w3.org/TR/2022/WD-wasm-core-2-20220419/exec/numerics.html#nan-propagation +func WasmCompatTruncF32(f float32) float32 { + return returnF32UniOp(f, float32(math.Trunc(float64(f)))) +} + +// WasmCompatTruncF64 is the same as math.Trunc on 64-bit except that +// the returned NaN value follows the Wasm specification on NaN +// propagation. +// https://www.w3.org/TR/2022/WD-wasm-core-2-20220419/exec/numerics.html#nan-propagation +func WasmCompatTruncF64(f float64) float64 { + return returnF64UniOp(f, math.Trunc(f)) +} + +func f32IsNaN(v float32) bool { + return v != v // this is how NaN is defined. +} + +func f64IsNaN(v float64) bool { + return v != v // this is how NaN is defined. +} + +// returnF32UniOp returns the result of 32-bit unary operation. This accepts `original` which is the operand, +// and `result` which is its result. This returns the `result` as-is if the result is not NaN. Otherwise, this follows +// the same logic as in the reference interpreter as well as the amd64 and arm64 floating point handling. +func returnF32UniOp(original, result float32) float32 { + // Following the same logic as in the reference interpreter: + // https://github.com/WebAssembly/spec/blob/d48af683f5e6d00c13f775ab07d29a15daf92203/interpreter/exec/fxx.ml#L115-L122 + if !f32IsNaN(result) { + return result + } + if !f32IsNaN(original) { + return math.Float32frombits(F32CanonicalNaNBits) + } + return math.Float32frombits(math.Float32bits(original) | F32CanonicalNaNBits) +} + +// returnF32UniOp returns the result of 64-bit unary operation. This accepts `original` which is the operand, +// and `result` which is its result. This returns the `result` as-is if the result is not NaN. Otherwise, this follows +// the same logic as in the reference interpreter as well as the amd64 and arm64 floating point handling. +func returnF64UniOp(original, result float64) float64 { + // Following the same logic as in the reference interpreter (== amd64 and arm64's behavior): + // https://github.com/WebAssembly/spec/blob/d48af683f5e6d00c13f775ab07d29a15daf92203/interpreter/exec/fxx.ml#L115-L122 + if !f64IsNaN(result) { + return result + } + if !f64IsNaN(original) { + return math.Float64frombits(F64CanonicalNaNBits) + } + return math.Float64frombits(math.Float64bits(original) | F64CanonicalNaNBits) +} + +// returnF64NaNBinOp returns a NaN for 64-bit binary operations. `x` and `y` are original floats +// and at least one of them is NaN. The returned NaN is guaranteed to comply with the NaN propagation +// procedure: https://www.w3.org/TR/2022/WD-wasm-core-2-20220419/exec/numerics.html#nan-propagation +func returnF64NaNBinOp(x, y float64) float64 { + if f64IsNaN(x) { + return math.Float64frombits(math.Float64bits(x) | F64CanonicalNaNBits) + } else { + return math.Float64frombits(math.Float64bits(y) | F64CanonicalNaNBits) + } +} + +// returnF64NaNBinOp returns a NaN for 32-bit binary operations. `x` and `y` are original floats +// and at least one of them is NaN. The returned NaN is guaranteed to comply with the NaN propagation +// procedure: https://www.w3.org/TR/2022/WD-wasm-core-2-20220419/exec/numerics.html#nan-propagation +func returnF32NaNBinOp(x, y float32) float32 { + if f32IsNaN(x) { + return math.Float32frombits(math.Float32bits(x) | F32CanonicalNaNBits) + } else { + return math.Float32frombits(math.Float32bits(y) | F32CanonicalNaNBits) + } +} diff --git a/vendor/github.com/tetratelabs/wazero/internal/platform/cpuid.go b/vendor/github.com/tetratelabs/wazero/internal/platform/cpuid.go new file mode 100644 index 000000000..25d7d3fdc --- /dev/null +++ b/vendor/github.com/tetratelabs/wazero/internal/platform/cpuid.go @@ -0,0 +1,25 @@ +package platform + +// CpuFeatureFlags exposes methods for querying CPU capabilities +type CpuFeatureFlags interface { + // Has returns true when the specified flag (represented as uint64) is supported + Has(cpuFeature CpuFeature) bool + // HasExtra returns true when the specified extraFlag (represented as uint64) is supported + HasExtra(cpuFeature CpuFeature) bool +} + +type CpuFeature uint64 + +const ( + // CpuFeatureAmd64SSE3 is the flag to query CpuFeatureFlags.Has for SSEv3 capabilities on amd64 + CpuFeatureAmd64SSE3 CpuFeature = 1 + // CpuFeatureAmd64SSE4_1 is the flag to query CpuFeatureFlags.Has for SSEv4.1 capabilities on amd64 + CpuFeatureAmd64SSE4_1 CpuFeature = 1 << 19 + // CpuFeatureAmd64SSE4_2 is the flag to query CpuFeatureFlags.Has for SSEv4.2 capabilities on amd64 + CpuFeatureAmd64SSE4_2 CpuFeature = 1 << 20 +) + +const ( + // CpuExtraFeatureAmd64ABM is the flag to query CpuFeatureFlags.HasExtra for Advanced Bit Manipulation capabilities (e.g. LZCNT) on amd64 + CpuExtraFeatureAmd64ABM CpuFeature = 1 << 5 +) diff --git a/vendor/github.com/tetratelabs/wazero/internal/platform/cpuid_amd64.go b/vendor/github.com/tetratelabs/wazero/internal/platform/cpuid_amd64.go new file mode 100644 index 000000000..8c9f1a9f3 --- /dev/null +++ b/vendor/github.com/tetratelabs/wazero/internal/platform/cpuid_amd64.go @@ -0,0 +1,59 @@ +//go:build amd64 && !tinygo + +package platform + +// CpuFeatures exposes the capabilities for this CPU, queried via the Has, HasExtra methods +var CpuFeatures CpuFeatureFlags = loadCpuFeatureFlags() + +// cpuFeatureFlags implements CpuFeatureFlags interface +type cpuFeatureFlags struct { + flags uint64 + extraFlags uint64 +} + +// cpuid exposes the CPUID instruction to the Go layer (https://www.amd.com/system/files/TechDocs/25481.pdf) +// implemented in impl_amd64.s +func cpuid(arg1, arg2 uint32) (eax, ebx, ecx, edx uint32) + +// cpuidAsBitmap combines the result of invoking cpuid to uint64 bitmap +func cpuidAsBitmap(arg1, arg2 uint32) uint64 { + _ /* eax */, _ /* ebx */, ecx, edx := cpuid(arg1, arg2) + return (uint64(edx) << 32) | uint64(ecx) +} + +// loadStandardRange load flags from the standard range, panics otherwise +func loadStandardRange(id uint32) uint64 { + // ensure that the id is in the valid range, returned by cpuid(0,0) + maxRange, _, _, _ := cpuid(0, 0) + if id > maxRange { + panic("cannot query standard CPU flags") + } + return cpuidAsBitmap(id, 0) +} + +// loadStandardRange load flags from the extended range, panics otherwise +func loadExtendedRange(id uint32) uint64 { + // ensure that the id is in the valid range, returned by cpuid(0x80000000,0) + maxRange, _, _, _ := cpuid(0x80000000, 0) + if id > maxRange { + panic("cannot query extended CPU flags") + } + return cpuidAsBitmap(id, 0) +} + +func loadCpuFeatureFlags() CpuFeatureFlags { + return &cpuFeatureFlags{ + flags: loadStandardRange(1), + extraFlags: loadExtendedRange(0x80000001), + } +} + +// Has implements the same method on the CpuFeatureFlags interface +func (f *cpuFeatureFlags) Has(cpuFeature CpuFeature) bool { + return (f.flags & uint64(cpuFeature)) != 0 +} + +// HasExtra implements the same method on the CpuFeatureFlags interface +func (f *cpuFeatureFlags) HasExtra(cpuFeature CpuFeature) bool { + return (f.extraFlags & uint64(cpuFeature)) != 0 +} diff --git a/vendor/github.com/tetratelabs/wazero/internal/platform/cpuid_amd64.s b/vendor/github.com/tetratelabs/wazero/internal/platform/cpuid_amd64.s new file mode 100644 index 000000000..8d483f3a6 --- /dev/null +++ b/vendor/github.com/tetratelabs/wazero/internal/platform/cpuid_amd64.s @@ -0,0 +1,14 @@ +#include "textflag.h" + +// lifted from github.com/intel-go/cpuid and src/internal/cpu/cpu_x86.s +// func cpuid(arg1, arg2 uint32) (eax, ebx, ecx, edx uint32) +TEXT ·cpuid(SB), NOSPLIT, $0-24 + MOVL arg1+0(FP), AX + MOVL arg2+4(FP), CX + CPUID + MOVL AX, eax+8(FP) + MOVL BX, ebx+12(FP) + MOVL CX, ecx+16(FP) + MOVL DX, edx+20(FP) + RET + diff --git a/vendor/github.com/tetratelabs/wazero/internal/platform/cpuid_unsupported.go b/vendor/github.com/tetratelabs/wazero/internal/platform/cpuid_unsupported.go new file mode 100644 index 000000000..8ae826d36 --- /dev/null +++ b/vendor/github.com/tetratelabs/wazero/internal/platform/cpuid_unsupported.go @@ -0,0 +1,14 @@ +//go:build !amd64 || tinygo + +package platform + +var CpuFeatures CpuFeatureFlags = &cpuFeatureFlags{} + +// cpuFeatureFlags implements CpuFeatureFlags for unsupported platforms +type cpuFeatureFlags struct{} + +// Has implements the same method on the CpuFeatureFlags interface +func (c *cpuFeatureFlags) Has(cpuFeature CpuFeature) bool { return false } + +// HasExtra implements the same method on the CpuFeatureFlags interface +func (c *cpuFeatureFlags) HasExtra(cpuFeature CpuFeature) bool { return false } diff --git a/vendor/github.com/tetratelabs/wazero/internal/platform/crypto.go b/vendor/github.com/tetratelabs/wazero/internal/platform/crypto.go new file mode 100644 index 000000000..c141f00f0 --- /dev/null +++ b/vendor/github.com/tetratelabs/wazero/internal/platform/crypto.go @@ -0,0 +1,17 @@ +package platform + +import ( + "io" + "math/rand" +) + +// seed is a fixed seed value for NewFakeRandSource. +// +// Trivia: While arbitrary, 42 was chosen as it is the "Ultimate Answer" in +// the Douglas Adams novel "The Hitchhiker's Guide to the Galaxy." +const seed = int64(42) + +// NewFakeRandSource returns a deterministic source of random values. +func NewFakeRandSource() io.Reader { + return rand.New(rand.NewSource(seed)) +} diff --git a/vendor/github.com/tetratelabs/wazero/internal/platform/mmap_linux.go b/vendor/github.com/tetratelabs/wazero/internal/platform/mmap_linux.go new file mode 100644 index 000000000..55906e827 --- /dev/null +++ b/vendor/github.com/tetratelabs/wazero/internal/platform/mmap_linux.go @@ -0,0 +1,76 @@ +package platform + +import ( + "math/bits" + "os" + "sort" + "strconv" + "strings" + "syscall" +) + +const ( + // https://man7.org/linux/man-pages/man2/mmap.2.html + __MAP_HUGE_SHIFT = 26 + __MAP_HUGETLB = 0x40000 +) + +var hugePagesConfigs []hugePagesConfig + +type hugePagesConfig struct { + size int + flag int +} + +func (hpc *hugePagesConfig) match(size int) bool { + return (size & (hpc.size - 1)) == 0 +} + +func init() { + dirents, err := os.ReadDir("/sys/kernel/mm/hugepages/") + if err != nil { + return + } + + for _, dirent := range dirents { + name := dirent.Name() + if !strings.HasPrefix(name, "hugepages-") { + continue + } + if !strings.HasSuffix(name, "kB") { + continue + } + n, err := strconv.ParseUint(name[10:len(name)-2], 10, 64) + if err != nil { + continue + } + if bits.OnesCount64(n) != 1 { + continue + } + n *= 1024 + hugePagesConfigs = append(hugePagesConfigs, hugePagesConfig{ + size: int(n), + flag: int(bits.TrailingZeros64(n)<<__MAP_HUGE_SHIFT) | __MAP_HUGETLB, + }) + } + + sort.Slice(hugePagesConfigs, func(i, j int) bool { + return hugePagesConfigs[i].size > hugePagesConfigs[j].size + }) +} + +func mmapCodeSegment(size, prot int) ([]byte, error) { + flags := syscall.MAP_ANON | syscall.MAP_PRIVATE + + for _, hugePagesConfig := range hugePagesConfigs { + if hugePagesConfig.match(size) { + b, err := syscall.Mmap(-1, 0, size, prot, flags|hugePagesConfig.flag) + if err != nil { + continue + } + return b, nil + } + } + + return syscall.Mmap(-1, 0, size, prot, flags) +} diff --git a/vendor/github.com/tetratelabs/wazero/internal/platform/mmap_other.go b/vendor/github.com/tetratelabs/wazero/internal/platform/mmap_other.go new file mode 100644 index 000000000..ed5c40a4d --- /dev/null +++ b/vendor/github.com/tetratelabs/wazero/internal/platform/mmap_other.go @@ -0,0 +1,18 @@ +// Separated from linux which has support for huge pages. +//go:build darwin || freebsd + +package platform + +import "syscall" + +func mmapCodeSegment(size, prot int) ([]byte, error) { + return syscall.Mmap( + -1, + 0, + size, + prot, + // Anonymous as this is not an actual file, but a memory, + // Private as this is in-process memory region. + syscall.MAP_ANON|syscall.MAP_PRIVATE, + ) +} diff --git a/vendor/github.com/tetratelabs/wazero/internal/platform/mmap_unix.go b/vendor/github.com/tetratelabs/wazero/internal/platform/mmap_unix.go new file mode 100644 index 000000000..a61996d58 --- /dev/null +++ b/vendor/github.com/tetratelabs/wazero/internal/platform/mmap_unix.go @@ -0,0 +1,49 @@ +//go:build (darwin || linux || freebsd) && !tinygo + +package platform + +import ( + "syscall" + "unsafe" +) + +const ( + mmapProtAMD64 = syscall.PROT_READ | syscall.PROT_WRITE | syscall.PROT_EXEC + mmapProtARM64 = syscall.PROT_READ | syscall.PROT_WRITE +) + +const MmapSupported = true + +func munmapCodeSegment(code []byte) error { + return syscall.Munmap(code) +} + +// mmapCodeSegmentAMD64 gives all read-write-exec permission to the mmap region +// to enter the function. Otherwise, segmentation fault exception is raised. +func mmapCodeSegmentAMD64(size int) ([]byte, error) { + // The region must be RWX: RW for writing native codes, X for executing the region. + return mmapCodeSegment(size, mmapProtAMD64) +} + +// mmapCodeSegmentARM64 cannot give all read-write-exec permission to the mmap region. +// Otherwise, the mmap systemcall would raise an error. Here we give read-write +// to the region so that we can write contents at call-sites. Callers are responsible to +// execute MprotectRX on the returned buffer. +func mmapCodeSegmentARM64(size int) ([]byte, error) { + // The region must be RW: RW for writing native codes. + return mmapCodeSegment(size, mmapProtARM64) +} + +// MprotectRX is like syscall.Mprotect with RX permission, defined locally so that freebsd compiles. +func MprotectRX(b []byte) (err error) { + var _p0 unsafe.Pointer + if len(b) > 0 { + _p0 = unsafe.Pointer(&b[0]) + } + const prot = syscall.PROT_READ | syscall.PROT_EXEC + _, _, e1 := syscall.Syscall(syscall.SYS_MPROTECT, uintptr(_p0), uintptr(len(b)), uintptr(prot)) + if e1 != 0 { + err = syscall.Errno(e1) + } + return +} diff --git a/vendor/github.com/tetratelabs/wazero/internal/platform/mmap_unsupported.go b/vendor/github.com/tetratelabs/wazero/internal/platform/mmap_unsupported.go new file mode 100644 index 000000000..27833db37 --- /dev/null +++ b/vendor/github.com/tetratelabs/wazero/internal/platform/mmap_unsupported.go @@ -0,0 +1,28 @@ +//go:build !(darwin || linux || freebsd || windows) || tinygo + +package platform + +import ( + "fmt" + "runtime" +) + +var errUnsupported = fmt.Errorf("mmap unsupported on GOOS=%s. Use interpreter instead.", runtime.GOOS) + +const MmapSupported = false + +func munmapCodeSegment(code []byte) error { + panic(errUnsupported) +} + +func mmapCodeSegmentAMD64(size int) ([]byte, error) { + panic(errUnsupported) +} + +func mmapCodeSegmentARM64(size int) ([]byte, error) { + panic(errUnsupported) +} + +func MprotectRX(b []byte) (err error) { + panic(errUnsupported) +} diff --git a/vendor/github.com/tetratelabs/wazero/internal/platform/mmap_windows.go b/vendor/github.com/tetratelabs/wazero/internal/platform/mmap_windows.go new file mode 100644 index 000000000..69fcb6d6b --- /dev/null +++ b/vendor/github.com/tetratelabs/wazero/internal/platform/mmap_windows.go @@ -0,0 +1,97 @@ +package platform + +import ( + "fmt" + "syscall" + "unsafe" +) + +var ( + kernel32 = syscall.NewLazyDLL("kernel32.dll") + procVirtualAlloc = kernel32.NewProc("VirtualAlloc") + procVirtualProtect = kernel32.NewProc("VirtualProtect") + procVirtualFree = kernel32.NewProc("VirtualFree") +) + +const ( + windows_MEM_COMMIT uintptr = 0x00001000 + windows_MEM_RELEASE uintptr = 0x00008000 + windows_PAGE_READWRITE uintptr = 0x00000004 + windows_PAGE_EXECUTE_READ uintptr = 0x00000020 + windows_PAGE_EXECUTE_READWRITE uintptr = 0x00000040 +) + +const MmapSupported = true + +func munmapCodeSegment(code []byte) error { + return freeMemory(code) +} + +// allocateMemory commits the memory region via the "VirtualAlloc" function. +// See https://docs.microsoft.com/en-us/windows/win32/api/memoryapi/nf-memoryapi-virtualalloc +func allocateMemory(size uintptr, protect uintptr) (uintptr, error) { + address := uintptr(0) // system determines where to allocate the region. + alloctype := windows_MEM_COMMIT + if r, _, err := procVirtualAlloc.Call(address, size, alloctype, protect); r == 0 { + return 0, fmt.Errorf("compiler: VirtualAlloc error: %w", ensureErr(err)) + } else { + return r, nil + } +} + +// freeMemory releases the memory region via the "VirtualFree" function. +// See https://docs.microsoft.com/en-us/windows/win32/api/memoryapi/nf-memoryapi-virtualfree +func freeMemory(code []byte) error { + address := unsafe.Pointer(&code[0]) + size := uintptr(0) // size must be 0 because we're using MEM_RELEASE. + freetype := windows_MEM_RELEASE + if r, _, err := procVirtualFree.Call(uintptr(address), size, freetype); r == 0 { + return fmt.Errorf("compiler: VirtualFree error: %w", ensureErr(err)) + } + return nil +} + +func virtualProtect(address, size, newprotect uintptr, oldprotect *uint32) error { + if r, _, err := procVirtualProtect.Call(address, size, newprotect, uintptr(unsafe.Pointer(oldprotect))); r == 0 { + return fmt.Errorf("compiler: VirtualProtect error: %w", ensureErr(err)) + } + return nil +} + +func mmapCodeSegmentAMD64(size int) ([]byte, error) { + p, err := allocateMemory(uintptr(size), windows_PAGE_EXECUTE_READWRITE) + if err != nil { + return nil, err + } + + return unsafe.Slice((*byte)(unsafe.Pointer(p)), size), nil +} + +func mmapCodeSegmentARM64(size int) ([]byte, error) { + p, err := allocateMemory(uintptr(size), windows_PAGE_READWRITE) + if err != nil { + return nil, err + } + + return unsafe.Slice((*byte)(unsafe.Pointer(p)), size), nil +} + +var old = uint32(windows_PAGE_READWRITE) + +func MprotectRX(b []byte) (err error) { + err = virtualProtect(uintptr(unsafe.Pointer(&b[0])), uintptr(len(b)), windows_PAGE_EXECUTE_READ, &old) + return +} + +// ensureErr returns syscall.EINVAL when the input error is nil. +// +// We are supposed to use "GetLastError" which is more precise, but it is not safe to execute in goroutines. While +// "GetLastError" is thread-local, goroutines are not pinned to threads. +// +// See https://docs.microsoft.com/en-us/windows/win32/api/errhandlingapi/nf-errhandlingapi-getlasterror +func ensureErr(err error) error { + if err != nil { + return err + } + return syscall.EINVAL +} diff --git a/vendor/github.com/tetratelabs/wazero/internal/platform/mremap_other.go b/vendor/github.com/tetratelabs/wazero/internal/platform/mremap_other.go new file mode 100644 index 000000000..5cba99fb2 --- /dev/null +++ b/vendor/github.com/tetratelabs/wazero/internal/platform/mremap_other.go @@ -0,0 +1,23 @@ +//go:build !(darwin || linux || freebsd) || tinygo + +package platform + +func remapCodeSegmentAMD64(code []byte, size int) ([]byte, error) { + b, err := mmapCodeSegmentAMD64(size) + if err != nil { + return nil, err + } + copy(b, code) + mustMunmapCodeSegment(code) + return b, nil +} + +func remapCodeSegmentARM64(code []byte, size int) ([]byte, error) { + b, err := mmapCodeSegmentARM64(size) + if err != nil { + return nil, err + } + copy(b, code) + mustMunmapCodeSegment(code) + return b, nil +} diff --git a/vendor/github.com/tetratelabs/wazero/internal/platform/mremap_unix.go b/vendor/github.com/tetratelabs/wazero/internal/platform/mremap_unix.go new file mode 100644 index 000000000..8f42d44fd --- /dev/null +++ b/vendor/github.com/tetratelabs/wazero/internal/platform/mremap_unix.go @@ -0,0 +1,21 @@ +//go:build (darwin || linux || freebsd) && !tinygo + +package platform + +func remapCodeSegmentAMD64(code []byte, size int) ([]byte, error) { + return remapCodeSegment(code, size, mmapProtAMD64) +} + +func remapCodeSegmentARM64(code []byte, size int) ([]byte, error) { + return remapCodeSegment(code, size, mmapProtARM64) +} + +func remapCodeSegment(code []byte, size, prot int) ([]byte, error) { + b, err := mmapCodeSegment(size, prot) + if err != nil { + return nil, err + } + copy(b, code) + mustMunmapCodeSegment(code) + return b, nil +} diff --git a/vendor/github.com/tetratelabs/wazero/internal/platform/path.go b/vendor/github.com/tetratelabs/wazero/internal/platform/path.go new file mode 100644 index 000000000..361049ae2 --- /dev/null +++ b/vendor/github.com/tetratelabs/wazero/internal/platform/path.go @@ -0,0 +1,6 @@ +//go:build !windows + +package platform + +// ToPosixPath returns the input, as only windows might return backslashes. +func ToPosixPath(in string) string { return in } diff --git a/vendor/github.com/tetratelabs/wazero/internal/platform/path_windows.go b/vendor/github.com/tetratelabs/wazero/internal/platform/path_windows.go new file mode 100644 index 000000000..77c4187d9 --- /dev/null +++ b/vendor/github.com/tetratelabs/wazero/internal/platform/path_windows.go @@ -0,0 +1,17 @@ +package platform + +import "strings" + +// ToPosixPath returns the input, converting any backslashes to forward ones. +func ToPosixPath(in string) string { + // strings.Map only allocates on change, which is good enough especially as + // path.Join uses forward slash even on windows. + return strings.Map(windowsToPosixSeparator, in) +} + +func windowsToPosixSeparator(r rune) rune { + if r == '\\' { + return '/' + } + return r +} diff --git a/vendor/github.com/tetratelabs/wazero/internal/platform/platform.go b/vendor/github.com/tetratelabs/wazero/internal/platform/platform.go new file mode 100644 index 000000000..c6dc0f857 --- /dev/null +++ b/vendor/github.com/tetratelabs/wazero/internal/platform/platform.go @@ -0,0 +1,81 @@ +// Package platform includes runtime-specific code needed for the compiler or otherwise. +// +// Note: This is a dependency-free alternative to depending on parts of Go's x/sys. +// See /RATIONALE.md for more context. +package platform + +import ( + "runtime" +) + +// archRequirementsVerified is set by platform-specific init to true if the platform is supported +var archRequirementsVerified bool + +// CompilerSupported is exported for tests and includes constraints here and also the assembler. +func CompilerSupported() bool { + switch runtime.GOOS { + case "darwin", "windows", "linux", "freebsd": + default: + return false + } + + return archRequirementsVerified +} + +// MmapCodeSegment copies the code into the executable region and returns the byte slice of the region. +// +// See https://man7.org/linux/man-pages/man2/mmap.2.html for mmap API and flags. +func MmapCodeSegment(size int) ([]byte, error) { + if size == 0 { + panic("BUG: MmapCodeSegment with zero length") + } + if runtime.GOARCH == "amd64" { + return mmapCodeSegmentAMD64(size) + } else { + return mmapCodeSegmentARM64(size) + } +} + +// RemapCodeSegment reallocates the memory mapping of an existing code segment +// to increase its size. The previous code mapping is unmapped and must not be +// reused after the function returns. +// +// This is similar to mremap(2) on linux, and emulated on platforms which do not +// have this syscall. +// +// See https://man7.org/linux/man-pages/man2/mremap.2.html +func RemapCodeSegment(code []byte, size int) ([]byte, error) { + if size < len(code) { + panic("BUG: RemapCodeSegment with size less than code") + } + if code == nil { + return MmapCodeSegment(size) + } + if runtime.GOARCH == "amd64" { + return remapCodeSegmentAMD64(code, size) + } else { + return remapCodeSegmentARM64(code, size) + } +} + +// MunmapCodeSegment unmaps the given memory region. +func MunmapCodeSegment(code []byte) error { + if len(code) == 0 { + panic("BUG: MunmapCodeSegment with zero length") + } + return munmapCodeSegment(code) +} + +// mustMunmapCodeSegment panics instead of returning an error to the +// application. +// +// # Why panic? +// +// It is less disruptive to the application to leak the previous block if it +// could be unmapped than to leak the new block and return an error. +// Realistically, either scenarios are pretty hard to debug, so we panic. +func mustMunmapCodeSegment(code []byte) { + if err := munmapCodeSegment(code); err != nil { + panic(err) + } +} diff --git a/vendor/github.com/tetratelabs/wazero/internal/platform/platform_amd64.go b/vendor/github.com/tetratelabs/wazero/internal/platform/platform_amd64.go new file mode 100644 index 000000000..59aaf5eae --- /dev/null +++ b/vendor/github.com/tetratelabs/wazero/internal/platform/platform_amd64.go @@ -0,0 +1,7 @@ +package platform + +// init verifies that the current CPU supports the required AMD64 instructions +func init() { + // Ensure SSE4.1 is supported. + archRequirementsVerified = CpuFeatures.Has(CpuFeatureAmd64SSE4_1) +} diff --git a/vendor/github.com/tetratelabs/wazero/internal/platform/platform_arm64.go b/vendor/github.com/tetratelabs/wazero/internal/platform/platform_arm64.go new file mode 100644 index 000000000..caac58a3d --- /dev/null +++ b/vendor/github.com/tetratelabs/wazero/internal/platform/platform_arm64.go @@ -0,0 +1,7 @@ +package platform + +// init verifies that the current CPU supports the required ARM64 features +func init() { + // No further checks currently needed. + archRequirementsVerified = true +} diff --git a/vendor/github.com/tetratelabs/wazero/internal/platform/time.go b/vendor/github.com/tetratelabs/wazero/internal/platform/time.go new file mode 100644 index 000000000..fa9da1acb --- /dev/null +++ b/vendor/github.com/tetratelabs/wazero/internal/platform/time.go @@ -0,0 +1,76 @@ +package platform + +import ( + "sync/atomic" + "time" + + "github.com/tetratelabs/wazero/sys" +) + +const ( + ms = int64(time.Millisecond) + // FakeEpochNanos is midnight UTC 2022-01-01 and exposed for testing + FakeEpochNanos = 1640995200000 * ms +) + +// NewFakeWalltime implements sys.Walltime with FakeEpochNanos that increases by 1ms each reading. +// See /RATIONALE.md +func NewFakeWalltime() sys.Walltime { + // AddInt64 returns the new value. Adjust so the first reading will be FakeEpochNanos + t := FakeEpochNanos - ms + return func() (sec int64, nsec int32) { + wt := atomic.AddInt64(&t, ms) + return wt / 1e9, int32(wt % 1e9) + } +} + +// NewFakeNanotime implements sys.Nanotime that increases by 1ms each reading. +// See /RATIONALE.md +func NewFakeNanotime() sys.Nanotime { + // AddInt64 returns the new value. Adjust so the first reading will be zero. + t := int64(0) - ms + return func() int64 { + return atomic.AddInt64(&t, ms) + } +} + +// FakeNanosleep implements sys.Nanosleep by returning without sleeping. +var FakeNanosleep = sys.Nanosleep(func(int64) {}) + +// FakeOsyield implements sys.Osyield by returning without yielding. +var FakeOsyield = sys.Osyield(func() {}) + +// Walltime implements sys.Walltime with time.Now. +// +// Note: This is only notably less efficient than it could be is reading +// runtime.walltime(). time.Now defensively reads nanotime also, just in case +// time.Since is used. This doubles the performance impact. However, wall time +// is likely to be read less frequently than Nanotime. Also, doubling the cost +// matters less on fast platforms that can return both in <=100ns. +func Walltime() (sec int64, nsec int32) { + t := time.Now() + return t.Unix(), int32(t.Nanosecond()) +} + +// nanoBase uses time.Now to ensure a monotonic clock reading on all platforms +// via time.Since. +var nanoBase = time.Now() + +// nanotimePortable implements sys.Nanotime with time.Since. +// +// Note: This is less efficient than it could be is reading runtime.nanotime(), +// Just to do that requires CGO. +func nanotimePortable() int64 { + return time.Since(nanoBase).Nanoseconds() +} + +// Nanotime implements sys.Nanotime with runtime.nanotime() if CGO is available +// and time.Since if not. +func Nanotime() int64 { + return nanotime() +} + +// Nanosleep implements sys.Nanosleep with time.Sleep. +func Nanosleep(ns int64) { + time.Sleep(time.Duration(ns)) +} diff --git a/vendor/github.com/tetratelabs/wazero/internal/platform/time_cgo.go b/vendor/github.com/tetratelabs/wazero/internal/platform/time_cgo.go new file mode 100644 index 000000000..ff01d90ce --- /dev/null +++ b/vendor/github.com/tetratelabs/wazero/internal/platform/time_cgo.go @@ -0,0 +1,11 @@ +//go:build cgo && !windows + +package platform + +import _ "unsafe" // for go:linkname + +// nanotime uses runtime.nanotime as it is available on all platforms and +// benchmarks faster than using time.Since. +// +//go:linkname nanotime runtime.nanotime +func nanotime() int64 diff --git a/vendor/github.com/tetratelabs/wazero/internal/platform/time_notcgo.go b/vendor/github.com/tetratelabs/wazero/internal/platform/time_notcgo.go new file mode 100644 index 000000000..0697b7c70 --- /dev/null +++ b/vendor/github.com/tetratelabs/wazero/internal/platform/time_notcgo.go @@ -0,0 +1,7 @@ +//go:build !cgo && !windows + +package platform + +func nanotime() int64 { + return nanotimePortable() +} diff --git a/vendor/github.com/tetratelabs/wazero/internal/platform/time_windows.go b/vendor/github.com/tetratelabs/wazero/internal/platform/time_windows.go new file mode 100644 index 000000000..58731fc8e --- /dev/null +++ b/vendor/github.com/tetratelabs/wazero/internal/platform/time_windows.go @@ -0,0 +1,40 @@ +//go:build windows + +package platform + +import ( + "math/bits" + "time" + "unsafe" +) + +var ( + _QueryPerformanceCounter = kernel32.NewProc("QueryPerformanceCounter") + _QueryPerformanceFrequency = kernel32.NewProc("QueryPerformanceFrequency") +) + +var qpcfreq uint64 + +func init() { + _, _, _ = _QueryPerformanceFrequency.Call(uintptr(unsafe.Pointer(&qpcfreq))) +} + +// On Windows, time.Time handled in time package cannot have the nanosecond precision. +// The reason is that by default, it doesn't use QueryPerformanceCounter[1], but instead, use "interrupt time" +// which doesn't support nanoseconds precision (though it is a monotonic) [2, 3, 4, 5]. +// +// [1] https://learn.microsoft.com/en-us/windows/win32/api/profileapi/nf-profileapi-queryperformancecounter +// [2] https://github.com/golang/go/blob/0cd309e12818f988693bf8e4d9f1453331dcf9f2/src/runtime/sys_windows_amd64.s#L297-L298 +// [3] https://github.com/golang/go/blob/0cd309e12818f988693bf8e4d9f1453331dcf9f2/src/runtime/os_windows.go#L549-L551 +// [4] https://github.com/golang/go/blob/master/src/runtime/time_windows.h#L7-L13 +// [5] http://web.archive.org/web/20210411000829/https://wrkhpi.wordpress.com/2007/08/09/getting-os-information-the-kuser_shared_data-structure/ +// +// Therefore, on Windows, we directly invoke the syscall for QPC instead of time.Now or runtime.nanotime. +// See https://github.com/golang/go/issues/31160 for example. +func nanotime() int64 { + var counter uint64 + _, _, _ = _QueryPerformanceCounter.Call(uintptr(unsafe.Pointer(&counter))) + hi, lo := bits.Mul64(counter, uint64(time.Second)) + nanos, _ := bits.Div64(hi, lo, qpcfreq) + return int64(nanos) +} diff --git a/vendor/github.com/tetratelabs/wazero/internal/sock/sock.go b/vendor/github.com/tetratelabs/wazero/internal/sock/sock.go new file mode 100644 index 000000000..ca17aa39e --- /dev/null +++ b/vendor/github.com/tetratelabs/wazero/internal/sock/sock.go @@ -0,0 +1,89 @@ +package sock + +import ( + "fmt" + "net" + + "github.com/tetratelabs/wazero/experimental/sys" +) + +// TCPSock is a pseudo-file representing a TCP socket. +type TCPSock interface { + sys.File + + Accept() (TCPConn, sys.Errno) +} + +// TCPConn is a pseudo-file representing a TCP connection. +type TCPConn interface { + sys.File + + // Recvfrom only supports the flag sysfs.MSG_PEEK + // TODO: document this like sys.File with known sys.Errno + Recvfrom(p []byte, flags int) (n int, errno sys.Errno) + + // TODO: document this like sys.File with known sys.Errno + Shutdown(how int) sys.Errno +} + +// ConfigKey is a context.Context Value key. Its associated value should be a Config. +type ConfigKey struct{} + +// Config is an internal struct meant to implement +// the interface in experimental/sock/Config. +type Config struct { + // TCPAddresses is a slice of the configured host:port pairs. + TCPAddresses []TCPAddress +} + +// TCPAddress is a host:port pair to pre-open. +type TCPAddress struct { + // Host is the host name for this listener. + Host string + // Port is the port number for this listener. + Port int +} + +// WithTCPListener implements the method of the same name in experimental/sock/Config. +// +// However, to avoid cyclic dependencies, this is returning the *Config in this scope. +// The interface is implemented in experimental/sock/Config via delegation. +func (c *Config) WithTCPListener(host string, port int) *Config { + ret := c.clone() + ret.TCPAddresses = append(ret.TCPAddresses, TCPAddress{host, port}) + return &ret +} + +// Makes a deep copy of this sockConfig. +func (c *Config) clone() Config { + ret := *c + ret.TCPAddresses = make([]TCPAddress, 0, len(c.TCPAddresses)) + ret.TCPAddresses = append(ret.TCPAddresses, c.TCPAddresses...) + return ret +} + +// BuildTCPListeners build listeners from the current configuration. +func (c *Config) BuildTCPListeners() (tcpListeners []*net.TCPListener, err error) { + for _, tcpAddr := range c.TCPAddresses { + var ln net.Listener + ln, err = net.Listen("tcp", tcpAddr.String()) + if err != nil { + break + } + if tcpln, ok := ln.(*net.TCPListener); ok { + tcpListeners = append(tcpListeners, tcpln) + } + } + if err != nil { + // An error occurred, cleanup. + for _, l := range tcpListeners { + _ = l.Close() // Ignore errors, we are already cleaning. + } + tcpListeners = nil + } + return +} + +func (t TCPAddress) String() string { + return fmt.Sprintf("%s:%d", t.Host, t.Port) +} diff --git a/vendor/github.com/tetratelabs/wazero/internal/sock/sock_supported.go b/vendor/github.com/tetratelabs/wazero/internal/sock/sock_supported.go new file mode 100644 index 000000000..e317be832 --- /dev/null +++ b/vendor/github.com/tetratelabs/wazero/internal/sock/sock_supported.go @@ -0,0 +1,11 @@ +//go:build !plan9 && !js && !tinygo + +package sock + +import "syscall" + +const ( + SHUT_RD = syscall.SHUT_RD + SHUT_RDWR = syscall.SHUT_RDWR + SHUT_WR = syscall.SHUT_WR +) diff --git a/vendor/github.com/tetratelabs/wazero/internal/sock/sock_unsupported.go b/vendor/github.com/tetratelabs/wazero/internal/sock/sock_unsupported.go new file mode 100644 index 000000000..77026754f --- /dev/null +++ b/vendor/github.com/tetratelabs/wazero/internal/sock/sock_unsupported.go @@ -0,0 +1,10 @@ +//go:build plan9 || js || tinygo + +package sock + +// plan9/js doesn't declare these constants +const ( + SHUT_RD = 1 << iota + SHUT_WR + SHUT_RDWR = SHUT_RD | SHUT_WR +) diff --git a/vendor/github.com/tetratelabs/wazero/internal/sys/fs.go b/vendor/github.com/tetratelabs/wazero/internal/sys/fs.go new file mode 100644 index 000000000..157de788f --- /dev/null +++ b/vendor/github.com/tetratelabs/wazero/internal/sys/fs.go @@ -0,0 +1,457 @@ +package sys + +import ( + "io" + "io/fs" + "net" + + "github.com/tetratelabs/wazero/experimental/sys" + "github.com/tetratelabs/wazero/internal/descriptor" + "github.com/tetratelabs/wazero/internal/fsapi" + socketapi "github.com/tetratelabs/wazero/internal/sock" + "github.com/tetratelabs/wazero/internal/sysfs" +) + +const ( + FdStdin int32 = iota + FdStdout + FdStderr + // FdPreopen is the file descriptor of the first pre-opened directory. + // + // # Why file descriptor 3? + // + // While not specified, the most common WASI implementation, wasi-libc, + // expects POSIX style file descriptor allocation, where the lowest + // available number is used to open the next file. Since 1 and 2 are taken + // by stdout and stderr, the next is 3. + // - https://github.com/WebAssembly/WASI/issues/122 + // - https://pubs.opengroup.org/onlinepubs/9699919799/functions/V2_chap02.html#tag_15_14 + // - https://github.com/WebAssembly/wasi-libc/blob/wasi-sdk-16/libc-bottom-half/sources/preopens.c#L215 + FdPreopen +) + +const modeDevice = fs.ModeDevice | 0o640 + +// FileEntry maps a path to an open file in a file system. +type FileEntry struct { + // Name is the name of the directory up to its pre-open, or the pre-open + // name itself when IsPreopen. + // + // # Notes + // + // - This can drift on rename. + // - This relates to the guest path, which is not the real file path + // except if the entire host filesystem was made available. + Name string + + // IsPreopen is a directory that is lazily opened. + IsPreopen bool + + // FS is the filesystem associated with the pre-open. + FS sys.FS + + // File is always non-nil. + File fsapi.File + + // direntCache is nil until DirentCache was called. + direntCache *DirentCache +} + +// DirentCache gets or creates a DirentCache for this file or returns an error. +// +// # Errors +// +// A zero sys.Errno is success. The below are expected otherwise: +// - sys.ENOSYS: the implementation does not support this function. +// - sys.EBADF: the dir was closed or not readable. +// - sys.ENOTDIR: the file was not a directory. +// +// # Notes +// +// - See /RATIONALE.md for design notes. +func (f *FileEntry) DirentCache() (*DirentCache, sys.Errno) { + if dir := f.direntCache; dir != nil { + return dir, 0 + } + + // Require the file to be a directory vs a late error on the same. + if isDir, errno := f.File.IsDir(); errno != 0 { + return nil, errno + } else if !isDir { + return nil, sys.ENOTDIR + } + + // Generate the dotEntries only once. + if dotEntries, errno := synthesizeDotEntries(f); errno != 0 { + return nil, errno + } else { + f.direntCache = &DirentCache{f: f.File, dotEntries: dotEntries} + } + + return f.direntCache, 0 +} + +// DirentCache is a caching abstraction of sys.File Readdir. +// +// This is special-cased for "wasi_snapshot_preview1.fd_readdir", and may be +// unneeded, or require changes, to support preview1 or preview2. +// - The position of the dirents are serialized as `d_next`. For reasons +// described below, any may need to be re-read. This accepts any positions +// in the cache, rather than track the position of the last dirent. +// - dot entries ("." and "..") must be returned. See /RATIONALE.md for why. +// - An sys.Dirent Name is variable length, it could exceed memory size and +// need to be re-read. +// - Multiple dirents may be returned. It is more efficient to read from the +// underlying file in bulk vs one-at-a-time. +// +// The last results returned by Read are cached, but entries before that +// position are not. This support re-reading entries that couldn't fit into +// memory without accidentally caching all entries in a large directory. This +// approach is sometimes called a sliding window. +type DirentCache struct { + // f is the underlying file + f sys.File + + // dotEntries are the "." and ".." entries added when the directory is + // initialized. + dotEntries []sys.Dirent + + // dirents are the potentially unread directory entries. + // + // Internal detail: nil is different from zero length. Zero length is an + // exhausted directory (eof). nil means the re-read. + dirents []sys.Dirent + + // countRead is the total count of dirents read since last rewind. + countRead uint64 + + // eof is true when the underlying file is at EOF. This avoids re-reading + // the directory when it is exhausted. Entires in an exhausted directory + // are not visible until it is rewound via calling Read with `pos==0`. + eof bool +} + +// synthesizeDotEntries generates a slice of the two elements "." and "..". +func synthesizeDotEntries(f *FileEntry) ([]sys.Dirent, sys.Errno) { + dotIno, errno := f.File.Ino() + if errno != 0 { + return nil, errno + } + result := [2]sys.Dirent{} + result[0] = sys.Dirent{Name: ".", Ino: dotIno, Type: fs.ModeDir} + // See /RATIONALE.md for why we don't attempt to get an inode for ".." and + // why in wasi-libc this won't fan-out either. + result[1] = sys.Dirent{Name: "..", Ino: 0, Type: fs.ModeDir} + return result[:], 0 +} + +// exhaustedDirents avoids allocating empty slices. +var exhaustedDirents = [0]sys.Dirent{} + +// Read is similar to and returns the same errors as `Readdir` on sys.File. +// The main difference is this caches entries returned, resulting in multiple +// valid positions to read from. +// +// When zero, `pos` means rewind to the beginning of this directory. This +// implies a rewind (Seek to zero on the underlying sys.File), unless the +// initial entries are still cached. +// +// When non-zero, `pos` is the zero based index of all dirents returned since +// last rewind. Only entries beginning at `pos` are cached for subsequent +// calls. A non-zero `pos` before the cache returns sys.ENOENT for reasons +// described on DirentCache documentation. +// +// Up to `n` entries are cached and returned. When `n` exceeds the cache, the +// difference are read from the underlying sys.File via `Readdir`. EOF is +// when `len(dirents)` returned are less than `n`. +func (d *DirentCache) Read(pos uint64, n uint32) (dirents []sys.Dirent, errno sys.Errno) { + switch { + case pos > d.countRead: // farther than read or negative coerced to uint64. + return nil, sys.ENOENT + case pos == 0 && d.dirents != nil: + // Rewind if we have already read entries. This allows us to see new + // entries added after the directory was opened. + if _, errno = d.f.Seek(0, io.SeekStart); errno != 0 { + return + } + d.dirents = nil // dump cache + d.countRead = 0 + } + + if n == 0 { + return // special case no entries. + } + + if d.dirents == nil { + // Always populate dot entries, which makes min len(dirents) == 2. + d.dirents = d.dotEntries + d.countRead = 2 + d.eof = false + + if countToRead := int(n - 2); countToRead <= 0 { + return + } else if dirents, errno = d.f.Readdir(countToRead); errno != 0 { + return + } else if countRead := len(dirents); countRead > 0 { + d.eof = countRead < countToRead + d.dirents = append(d.dotEntries, dirents...) + d.countRead += uint64(countRead) + } + + return d.cachedDirents(n), 0 + } + + // Reset our cache to the first entry being read. + cacheStart := d.countRead - uint64(len(d.dirents)) + if pos < cacheStart { + // We don't currently allow reads before our cache because Seek(0) is + // the only portable way. Doing otherwise requires skipping, which we + // won't do unless wasi-testsuite starts requiring it. Implementing + // this would allow re-reading a large directory, so care would be + // needed to not buffer the entire directory in memory while skipping. + errno = sys.ENOENT + return + } else if posInCache := pos - cacheStart; posInCache != 0 { + if uint64(len(d.dirents)) == posInCache { + // Avoid allocation re-slicing to zero length. + d.dirents = exhaustedDirents[:] + } else { + d.dirents = d.dirents[posInCache:] + } + } + + // See if we need more entries. + if countToRead := int(n) - len(d.dirents); countToRead > 0 && !d.eof { + // Try to read more, which could fail. + if dirents, errno = d.f.Readdir(countToRead); errno != 0 { + return + } + + // Append the next read entries if we weren't at EOF. + if countRead := len(dirents); countRead > 0 { + d.eof = countRead < countToRead + d.dirents = append(d.dirents, dirents...) + d.countRead += uint64(countRead) + } + } + + return d.cachedDirents(n), 0 +} + +// cachedDirents returns up to `n` dirents from the cache. +func (d *DirentCache) cachedDirents(n uint32) []sys.Dirent { + direntCount := uint32(len(d.dirents)) + switch { + case direntCount == 0: + return nil + case direntCount > n: + return d.dirents[:n] + } + return d.dirents +} + +type FSContext struct { + // openedFiles is a map of file descriptor numbers (>=FdPreopen) to open files + // (or directories) and defaults to empty. + // TODO: This is unguarded, so not goroutine-safe! + openedFiles FileTable +} + +// FileTable is a specialization of the descriptor.Table type used to map file +// descriptors to file entries. +type FileTable = descriptor.Table[int32, *FileEntry] + +// LookupFile returns a file if it is in the table. +func (c *FSContext) LookupFile(fd int32) (*FileEntry, bool) { + return c.openedFiles.Lookup(fd) +} + +// OpenFile opens the file into the table and returns its file descriptor. +// The result must be closed by CloseFile or Close. +func (c *FSContext) OpenFile(fs sys.FS, path string, flag sys.Oflag, perm fs.FileMode) (int32, sys.Errno) { + if f, errno := fs.OpenFile(path, flag, perm); errno != 0 { + return 0, errno + } else { + fe := &FileEntry{FS: fs, File: fsapi.Adapt(f)} + if path == "/" || path == "." { + fe.Name = "" + } else { + fe.Name = path + } + if newFD, ok := c.openedFiles.Insert(fe); !ok { + return 0, sys.EBADF + } else { + return newFD, 0 + } + } +} + +// Renumber assigns the file pointed by the descriptor `from` to `to`. +func (c *FSContext) Renumber(from, to int32) sys.Errno { + fromFile, ok := c.openedFiles.Lookup(from) + if !ok || to < 0 { + return sys.EBADF + } else if fromFile.IsPreopen { + return sys.ENOTSUP + } + + // If toFile is already open, we close it to prevent windows lock issues. + // + // The doc is unclear and other implementations do nothing for already-opened To FDs. + // https://github.com/WebAssembly/WASI/blob/snapshot-01/phases/snapshot/docs.md#-fd_renumberfd-fd-to-fd---errno + // https://github.com/bytecodealliance/wasmtime/blob/main/crates/wasi-common/src/snapshots/preview_1.rs#L531-L546 + if toFile, ok := c.openedFiles.Lookup(to); ok { + if toFile.IsPreopen { + return sys.ENOTSUP + } + _ = toFile.File.Close() + } + + c.openedFiles.Delete(from) + if !c.openedFiles.InsertAt(fromFile, to) { + return sys.EBADF + } + return 0 +} + +// SockAccept accepts a sock.TCPConn into the file table and returns its file +// descriptor. +func (c *FSContext) SockAccept(sockFD int32, nonblock bool) (int32, sys.Errno) { + var sock socketapi.TCPSock + if e, ok := c.LookupFile(sockFD); !ok || !e.IsPreopen { + return 0, sys.EBADF // Not a preopen + } else if sock, ok = e.File.(socketapi.TCPSock); !ok { + return 0, sys.EBADF // Not a sock + } + + conn, errno := sock.Accept() + if errno != 0 { + return 0, errno + } + + fe := &FileEntry{File: fsapi.Adapt(conn)} + + if nonblock { + if errno = fe.File.SetNonblock(true); errno != 0 { + _ = conn.Close() + return 0, errno + } + } + + if newFD, ok := c.openedFiles.Insert(fe); !ok { + return 0, sys.EBADF + } else { + return newFD, 0 + } +} + +// CloseFile returns any error closing the existing file. +func (c *FSContext) CloseFile(fd int32) (errno sys.Errno) { + f, ok := c.openedFiles.Lookup(fd) + if !ok { + return sys.EBADF + } + if errno = f.File.Close(); errno != 0 { + return errno + } + c.openedFiles.Delete(fd) + return errno +} + +// Close implements io.Closer +func (c *FSContext) Close() (err error) { + // Close any files opened in this context + c.openedFiles.Range(func(fd int32, entry *FileEntry) bool { + if errno := entry.File.Close(); errno != 0 { + err = errno // This means err returned == the last non-nil error. + } + return true + }) + // A closed FSContext cannot be reused so clear the state. + c.openedFiles = FileTable{} + return +} + +// InitFSContext initializes a FSContext with stdio streams and optional +// pre-opened filesystems and TCP listeners. +func (c *Context) InitFSContext( + stdin io.Reader, + stdout, stderr io.Writer, + fs []sys.FS, guestPaths []string, + tcpListeners []*net.TCPListener, +) (err error) { + inFile, err := stdinFileEntry(stdin) + if err != nil { + return err + } + c.fsc.openedFiles.Insert(inFile) + outWriter, err := stdioWriterFileEntry("stdout", stdout) + if err != nil { + return err + } + c.fsc.openedFiles.Insert(outWriter) + errWriter, err := stdioWriterFileEntry("stderr", stderr) + if err != nil { + return err + } + c.fsc.openedFiles.Insert(errWriter) + + for i, f := range fs { + guestPath := guestPaths[i] + + if StripPrefixesAndTrailingSlash(guestPath) == "" { + // Default to bind to '/' when guestPath is effectively empty. + guestPath = "/" + } + c.fsc.openedFiles.Insert(&FileEntry{ + FS: f, + Name: guestPath, + IsPreopen: true, + File: &lazyDir{fs: f}, + }) + } + + for _, tl := range tcpListeners { + c.fsc.openedFiles.Insert(&FileEntry{IsPreopen: true, File: fsapi.Adapt(sysfs.NewTCPListenerFile(tl))}) + } + return nil +} + +// StripPrefixesAndTrailingSlash skips any leading "./" or "/" such that the +// result index begins with another string. A result of "." coerces to the +// empty string "" because the current directory is handled by the guest. +// +// Results are the offset/len pair which is an optimization to avoid re-slicing +// overhead, as this function is called for every path operation. +// +// Note: Relative paths should be handled by the guest, as that's what knows +// what the current directory is. However, paths that escape the current +// directory e.g. "../.." have been found in `tinygo test` and this +// implementation takes care to avoid it. +func StripPrefixesAndTrailingSlash(path string) string { + // strip trailing slashes + pathLen := len(path) + for ; pathLen > 0 && path[pathLen-1] == '/'; pathLen-- { + } + + pathI := 0 +loop: + for pathI < pathLen { + switch path[pathI] { + case '/': + pathI++ + case '.': + nextI := pathI + 1 + if nextI < pathLen && path[nextI] == '/' { + pathI = nextI + 1 + } else if nextI == pathLen { + pathI = nextI + } else { + break loop + } + default: + break loop + } + } + return path[pathI:pathLen] +} diff --git a/vendor/github.com/tetratelabs/wazero/internal/sys/lazy.go b/vendor/github.com/tetratelabs/wazero/internal/sys/lazy.go new file mode 100644 index 000000000..fe233d29e --- /dev/null +++ b/vendor/github.com/tetratelabs/wazero/internal/sys/lazy.go @@ -0,0 +1,151 @@ +package sys + +import ( + experimentalsys "github.com/tetratelabs/wazero/experimental/sys" + "github.com/tetratelabs/wazero/internal/fsapi" + "github.com/tetratelabs/wazero/sys" +) + +// compile-time check to ensure lazyDir implements sys.File. +var _ experimentalsys.File = (*lazyDir)(nil) + +type lazyDir struct { + experimentalsys.DirFile + + fs experimentalsys.FS + f experimentalsys.File +} + +// Dev implements the same method as documented on sys.File +func (d *lazyDir) Dev() (uint64, experimentalsys.Errno) { + if f, ok := d.file(); !ok { + return 0, experimentalsys.EBADF + } else { + return f.Dev() + } +} + +// Ino implements the same method as documented on sys.File +func (d *lazyDir) Ino() (sys.Inode, experimentalsys.Errno) { + if f, ok := d.file(); !ok { + return 0, experimentalsys.EBADF + } else { + return f.Ino() + } +} + +// IsDir implements the same method as documented on sys.File +func (d *lazyDir) IsDir() (bool, experimentalsys.Errno) { + // Note: we don't return a constant because we don't know if this is really + // backed by a dir, until the first call. + if f, ok := d.file(); !ok { + return false, experimentalsys.EBADF + } else { + return f.IsDir() + } +} + +// IsAppend implements the same method as documented on sys.File +func (d *lazyDir) IsAppend() bool { + return false +} + +// SetAppend implements the same method as documented on sys.File +func (d *lazyDir) SetAppend(bool) experimentalsys.Errno { + return experimentalsys.EISDIR +} + +// Seek implements the same method as documented on sys.File +func (d *lazyDir) Seek(offset int64, whence int) (newOffset int64, errno experimentalsys.Errno) { + if f, ok := d.file(); !ok { + return 0, experimentalsys.EBADF + } else { + return f.Seek(offset, whence) + } +} + +// Stat implements the same method as documented on sys.File +func (d *lazyDir) Stat() (sys.Stat_t, experimentalsys.Errno) { + if f, ok := d.file(); !ok { + return sys.Stat_t{}, experimentalsys.EBADF + } else { + return f.Stat() + } +} + +// Readdir implements the same method as documented on sys.File +func (d *lazyDir) Readdir(n int) (dirents []experimentalsys.Dirent, errno experimentalsys.Errno) { + if f, ok := d.file(); !ok { + return nil, experimentalsys.EBADF + } else { + return f.Readdir(n) + } +} + +// Sync implements the same method as documented on sys.File +func (d *lazyDir) Sync() experimentalsys.Errno { + if f, ok := d.file(); !ok { + return experimentalsys.EBADF + } else { + return f.Sync() + } +} + +// Datasync implements the same method as documented on sys.File +func (d *lazyDir) Datasync() experimentalsys.Errno { + if f, ok := d.file(); !ok { + return experimentalsys.EBADF + } else { + return f.Datasync() + } +} + +// Utimens implements the same method as documented on sys.File +func (d *lazyDir) Utimens(atim, mtim int64) experimentalsys.Errno { + if f, ok := d.file(); !ok { + return experimentalsys.EBADF + } else { + return f.Utimens(atim, mtim) + } +} + +// file returns the underlying file or false if it doesn't exist. +func (d *lazyDir) file() (experimentalsys.File, bool) { + if f := d.f; d.f != nil { + return f, true + } + var errno experimentalsys.Errno + d.f, errno = d.fs.OpenFile(".", experimentalsys.O_RDONLY, 0) + switch errno { + case 0: + return d.f, true + case experimentalsys.ENOENT: + return nil, false + default: + panic(errno) // unexpected + } +} + +// Close implements fs.File +func (d *lazyDir) Close() experimentalsys.Errno { + f := d.f + if f == nil { + return 0 // never opened + } + return f.Close() +} + +// IsNonblock implements the same method as documented on fsapi.File +func (d *lazyDir) IsNonblock() bool { + return false +} + +// SetNonblock implements the same method as documented on fsapi.File +func (d *lazyDir) SetNonblock(bool) experimentalsys.Errno { + return experimentalsys.EISDIR +} + +// Poll implements the same method as documented on fsapi.File +func (d *lazyDir) Poll(fsapi.Pflag, int32) (ready bool, errno experimentalsys.Errno) { + return false, experimentalsys.ENOSYS +} diff --git a/vendor/github.com/tetratelabs/wazero/internal/sys/stdio.go b/vendor/github.com/tetratelabs/wazero/internal/sys/stdio.go new file mode 100644 index 000000000..32c33661e --- /dev/null +++ b/vendor/github.com/tetratelabs/wazero/internal/sys/stdio.go @@ -0,0 +1,128 @@ +package sys + +import ( + "io" + "os" + + experimentalsys "github.com/tetratelabs/wazero/experimental/sys" + "github.com/tetratelabs/wazero/internal/fsapi" + "github.com/tetratelabs/wazero/internal/sysfs" + "github.com/tetratelabs/wazero/sys" +) + +// StdinFile is a fs.ModeDevice file for use implementing FdStdin. +// This is safer than reading from os.DevNull as it can never overrun +// operating system file descriptors. +type StdinFile struct { + noopStdinFile + io.Reader +} + +// Read implements the same method as documented on sys.File +func (f *StdinFile) Read(buf []byte) (int, experimentalsys.Errno) { + n, err := f.Reader.Read(buf) + return n, experimentalsys.UnwrapOSError(err) +} + +type writerFile struct { + noopStdoutFile + + w io.Writer +} + +// Write implements the same method as documented on sys.File +func (f *writerFile) Write(buf []byte) (int, experimentalsys.Errno) { + n, err := f.w.Write(buf) + return n, experimentalsys.UnwrapOSError(err) +} + +// noopStdinFile is a fs.ModeDevice file for use implementing FdStdin. This is +// safer than reading from os.DevNull as it can never overrun operating system +// file descriptors. +type noopStdinFile struct { + noopStdioFile +} + +// Read implements the same method as documented on sys.File +func (noopStdinFile) Read([]byte) (int, experimentalsys.Errno) { + return 0, 0 // Always EOF +} + +// Poll implements the same method as documented on fsapi.File +func (noopStdinFile) Poll(flag fsapi.Pflag, timeoutMillis int32) (ready bool, errno experimentalsys.Errno) { + if flag != fsapi.POLLIN { + return false, experimentalsys.ENOTSUP + } + return true, 0 // always ready to read nothing +} + +// noopStdoutFile is a fs.ModeDevice file for use implementing FdStdout and +// FdStderr. +type noopStdoutFile struct { + noopStdioFile +} + +// Write implements the same method as documented on sys.File +func (noopStdoutFile) Write(buf []byte) (int, experimentalsys.Errno) { + return len(buf), 0 // same as io.Discard +} + +type noopStdioFile struct { + experimentalsys.UnimplementedFile +} + +// Stat implements the same method as documented on sys.File +func (noopStdioFile) Stat() (sys.Stat_t, experimentalsys.Errno) { + return sys.Stat_t{Mode: modeDevice, Nlink: 1}, 0 +} + +// IsDir implements the same method as documented on sys.File +func (noopStdioFile) IsDir() (bool, experimentalsys.Errno) { + return false, 0 +} + +// Close implements the same method as documented on sys.File +func (noopStdioFile) Close() (errno experimentalsys.Errno) { return } + +// IsNonblock implements the same method as documented on fsapi.File +func (noopStdioFile) IsNonblock() bool { + return false +} + +// SetNonblock implements the same method as documented on fsapi.File +func (noopStdioFile) SetNonblock(bool) experimentalsys.Errno { + return experimentalsys.ENOSYS +} + +// Poll implements the same method as documented on fsapi.File +func (noopStdioFile) Poll(fsapi.Pflag, int32) (ready bool, errno experimentalsys.Errno) { + return false, experimentalsys.ENOSYS +} + +func stdinFileEntry(r io.Reader) (*FileEntry, error) { + if r == nil { + return &FileEntry{Name: "stdin", IsPreopen: true, File: &noopStdinFile{}}, nil + } else if f, ok := r.(*os.File); ok { + if f, err := sysfs.NewStdioFile(true, f); err != nil { + return nil, err + } else { + return &FileEntry{Name: "stdin", IsPreopen: true, File: f}, nil + } + } else { + return &FileEntry{Name: "stdin", IsPreopen: true, File: &StdinFile{Reader: r}}, nil + } +} + +func stdioWriterFileEntry(name string, w io.Writer) (*FileEntry, error) { + if w == nil { + return &FileEntry{Name: name, IsPreopen: true, File: &noopStdoutFile{}}, nil + } else if f, ok := w.(*os.File); ok { + if f, err := sysfs.NewStdioFile(false, f); err != nil { + return nil, err + } else { + return &FileEntry{Name: name, IsPreopen: true, File: f}, nil + } + } else { + return &FileEntry{Name: name, IsPreopen: true, File: &writerFile{w: w}}, nil + } +} diff --git a/vendor/github.com/tetratelabs/wazero/internal/sys/sys.go b/vendor/github.com/tetratelabs/wazero/internal/sys/sys.go new file mode 100644 index 000000000..12279ee49 --- /dev/null +++ b/vendor/github.com/tetratelabs/wazero/internal/sys/sys.go @@ -0,0 +1,228 @@ +package sys + +import ( + "errors" + "fmt" + "io" + "net" + "time" + + experimentalsys "github.com/tetratelabs/wazero/experimental/sys" + "github.com/tetratelabs/wazero/internal/platform" + "github.com/tetratelabs/wazero/sys" +) + +// Context holds module-scoped system resources currently only supported by +// built-in host functions. +type Context struct { + args, environ [][]byte + argsSize, environSize uint32 + + walltime sys.Walltime + walltimeResolution sys.ClockResolution + nanotime sys.Nanotime + nanotimeResolution sys.ClockResolution + nanosleep sys.Nanosleep + osyield sys.Osyield + randSource io.Reader + fsc FSContext +} + +// Args is like os.Args and defaults to nil. +// +// Note: The count will never be more than math.MaxUint32. +// See wazero.ModuleConfig WithArgs +func (c *Context) Args() [][]byte { + return c.args +} + +// ArgsSize is the size to encode Args as Null-terminated strings. +// +// Note: To get the size without null-terminators, subtract the length of Args from this value. +// See wazero.ModuleConfig WithArgs +// See https://en.wikipedia.org/wiki/Null-terminated_string +func (c *Context) ArgsSize() uint32 { + return c.argsSize +} + +// Environ are "key=value" entries like os.Environ and default to nil. +// +// Note: The count will never be more than math.MaxUint32. +// See wazero.ModuleConfig WithEnv +func (c *Context) Environ() [][]byte { + return c.environ +} + +// EnvironSize is the size to encode Environ as Null-terminated strings. +// +// Note: To get the size without null-terminators, subtract the length of Environ from this value. +// See wazero.ModuleConfig WithEnv +// See https://en.wikipedia.org/wiki/Null-terminated_string +func (c *Context) EnvironSize() uint32 { + return c.environSize +} + +// Walltime implements platform.Walltime. +func (c *Context) Walltime() (sec int64, nsec int32) { + return c.walltime() +} + +// WalltimeNanos returns platform.Walltime as epoch nanoseconds. +func (c *Context) WalltimeNanos() int64 { + sec, nsec := c.Walltime() + return (sec * time.Second.Nanoseconds()) + int64(nsec) +} + +// WalltimeResolution returns resolution of Walltime. +func (c *Context) WalltimeResolution() sys.ClockResolution { + return c.walltimeResolution +} + +// Nanotime implements sys.Nanotime. +func (c *Context) Nanotime() int64 { + return c.nanotime() +} + +// NanotimeResolution returns resolution of Nanotime. +func (c *Context) NanotimeResolution() sys.ClockResolution { + return c.nanotimeResolution +} + +// Nanosleep implements sys.Nanosleep. +func (c *Context) Nanosleep(ns int64) { + c.nanosleep(ns) +} + +// Osyield implements sys.Osyield. +func (c *Context) Osyield() { + c.osyield() +} + +// FS returns the possibly empty (UnimplementedFS) file system context. +func (c *Context) FS() *FSContext { + return &c.fsc +} + +// RandSource is a source of random bytes and defaults to a deterministic source. +// see wazero.ModuleConfig WithRandSource +func (c *Context) RandSource() io.Reader { + return c.randSource +} + +// DefaultContext returns Context with no values set except a possible nil +// sys.FS. +// +// Note: This is only used for testing. +func DefaultContext(fs experimentalsys.FS) *Context { + if sysCtx, err := NewContext(0, nil, nil, nil, nil, nil, nil, nil, 0, nil, 0, nil, nil, []experimentalsys.FS{fs}, []string{""}, nil); err != nil { + panic(fmt.Errorf("BUG: DefaultContext should never error: %w", err)) + } else { + return sysCtx + } +} + +// NewContext is a factory function which helps avoid needing to know defaults or exporting all fields. +// Note: max is exposed for testing. max is only used for env/args validation. +func NewContext( + max uint32, + args, environ [][]byte, + stdin io.Reader, + stdout, stderr io.Writer, + randSource io.Reader, + walltime sys.Walltime, + walltimeResolution sys.ClockResolution, + nanotime sys.Nanotime, + nanotimeResolution sys.ClockResolution, + nanosleep sys.Nanosleep, + osyield sys.Osyield, + fs []experimentalsys.FS, guestPaths []string, + tcpListeners []*net.TCPListener, +) (sysCtx *Context, err error) { + sysCtx = &Context{args: args, environ: environ} + + if sysCtx.argsSize, err = nullTerminatedByteCount(max, args); err != nil { + return nil, fmt.Errorf("args invalid: %w", err) + } + + if sysCtx.environSize, err = nullTerminatedByteCount(max, environ); err != nil { + return nil, fmt.Errorf("environ invalid: %w", err) + } + + if randSource == nil { + sysCtx.randSource = platform.NewFakeRandSource() + } else { + sysCtx.randSource = randSource + } + + if walltime != nil { + if clockResolutionInvalid(walltimeResolution) { + return nil, fmt.Errorf("invalid Walltime resolution: %d", walltimeResolution) + } + sysCtx.walltime = walltime + sysCtx.walltimeResolution = walltimeResolution + } else { + sysCtx.walltime = platform.NewFakeWalltime() + sysCtx.walltimeResolution = sys.ClockResolution(time.Microsecond.Nanoseconds()) + } + + if nanotime != nil { + if clockResolutionInvalid(nanotimeResolution) { + return nil, fmt.Errorf("invalid Nanotime resolution: %d", nanotimeResolution) + } + sysCtx.nanotime = nanotime + sysCtx.nanotimeResolution = nanotimeResolution + } else { + sysCtx.nanotime = platform.NewFakeNanotime() + sysCtx.nanotimeResolution = sys.ClockResolution(time.Nanosecond) + } + + if nanosleep != nil { + sysCtx.nanosleep = nanosleep + } else { + sysCtx.nanosleep = platform.FakeNanosleep + } + + if osyield != nil { + sysCtx.osyield = osyield + } else { + sysCtx.osyield = platform.FakeOsyield + } + + err = sysCtx.InitFSContext(stdin, stdout, stderr, fs, guestPaths, tcpListeners) + + return +} + +// clockResolutionInvalid returns true if the value stored isn't reasonable. +func clockResolutionInvalid(resolution sys.ClockResolution) bool { + return resolution < 1 || resolution > sys.ClockResolution(time.Hour.Nanoseconds()) +} + +// nullTerminatedByteCount ensures the count or Nul-terminated length of the elements doesn't exceed max, and that no +// element includes the nul character. +func nullTerminatedByteCount(max uint32, elements [][]byte) (uint32, error) { + count := uint32(len(elements)) + if count > max { + return 0, errors.New("exceeds maximum count") + } + + // The buffer size is the total size including null terminators. The null terminator count == value count, sum + // count with each value length. This works because in Go, the length of a string is the same as its byte count. + bufSize, maxSize := uint64(count), uint64(max) // uint64 to allow summing without overflow + for _, e := range elements { + // As this is null-terminated, We have to validate there are no null characters in the string. + for _, c := range e { + if c == 0 { + return 0, errors.New("contains NUL character") + } + } + + nextSize := bufSize + uint64(len(e)) + if nextSize > maxSize { + return 0, errors.New("exceeds maximum size") + } + bufSize = nextSize + + } + return uint32(bufSize), nil +} diff --git a/vendor/github.com/tetratelabs/wazero/internal/sysfs/adapter.go b/vendor/github.com/tetratelabs/wazero/internal/sysfs/adapter.go new file mode 100644 index 000000000..51a9a5480 --- /dev/null +++ b/vendor/github.com/tetratelabs/wazero/internal/sysfs/adapter.go @@ -0,0 +1,105 @@ +package sysfs + +import ( + "fmt" + "io/fs" + "path" + + experimentalsys "github.com/tetratelabs/wazero/experimental/sys" + "github.com/tetratelabs/wazero/sys" +) + +type AdaptFS struct { + FS fs.FS +} + +// String implements fmt.Stringer +func (a *AdaptFS) String() string { + return fmt.Sprintf("%v", a.FS) +} + +// OpenFile implements the same method as documented on sys.FS +func (a *AdaptFS) OpenFile(path string, flag experimentalsys.Oflag, perm fs.FileMode) (experimentalsys.File, experimentalsys.Errno) { + return OpenFSFile(a.FS, cleanPath(path), flag, perm) +} + +// Lstat implements the same method as documented on sys.FS +func (a *AdaptFS) Lstat(path string) (sys.Stat_t, experimentalsys.Errno) { + // At this time, we make the assumption sys.FS instances do not support + // symbolic links, therefore Lstat is the same as Stat. This is obviously + // not true, but until FS.FS has a solid story for how to handle symlinks, + // we are better off not making a decision that would be difficult to + // revert later on. + // + // For further discussions on the topic, see: + // https://github.com/golang/go/issues/49580 + return a.Stat(path) +} + +// Stat implements the same method as documented on sys.FS +func (a *AdaptFS) Stat(path string) (sys.Stat_t, experimentalsys.Errno) { + f, errno := a.OpenFile(path, experimentalsys.O_RDONLY, 0) + if errno != 0 { + return sys.Stat_t{}, errno + } + defer f.Close() + return f.Stat() +} + +// Readlink implements the same method as documented on sys.FS +func (a *AdaptFS) Readlink(string) (string, experimentalsys.Errno) { + return "", experimentalsys.ENOSYS +} + +// Mkdir implements the same method as documented on sys.FS +func (a *AdaptFS) Mkdir(string, fs.FileMode) experimentalsys.Errno { + return experimentalsys.ENOSYS +} + +// Chmod implements the same method as documented on sys.FS +func (a *AdaptFS) Chmod(string, fs.FileMode) experimentalsys.Errno { + return experimentalsys.ENOSYS +} + +// Rename implements the same method as documented on sys.FS +func (a *AdaptFS) Rename(string, string) experimentalsys.Errno { + return experimentalsys.ENOSYS +} + +// Rmdir implements the same method as documented on sys.FS +func (a *AdaptFS) Rmdir(string) experimentalsys.Errno { + return experimentalsys.ENOSYS +} + +// Link implements the same method as documented on sys.FS +func (a *AdaptFS) Link(string, string) experimentalsys.Errno { + return experimentalsys.ENOSYS +} + +// Symlink implements the same method as documented on sys.FS +func (a *AdaptFS) Symlink(string, string) experimentalsys.Errno { + return experimentalsys.ENOSYS +} + +// Unlink implements the same method as documented on sys.FS +func (a *AdaptFS) Unlink(string) experimentalsys.Errno { + return experimentalsys.ENOSYS +} + +// Utimens implements the same method as documented on sys.FS +func (a *AdaptFS) Utimens(string, int64, int64) experimentalsys.Errno { + return experimentalsys.ENOSYS +} + +func cleanPath(name string) string { + if len(name) == 0 { + return name + } + // fs.ValidFile cannot be rooted (start with '/') + cleaned := name + if name[0] == '/' { + cleaned = name[1:] + } + cleaned = path.Clean(cleaned) // e.g. "sub/." -> "sub" + return cleaned +} diff --git a/vendor/github.com/tetratelabs/wazero/internal/sysfs/datasync_linux.go b/vendor/github.com/tetratelabs/wazero/internal/sysfs/datasync_linux.go new file mode 100644 index 000000000..5a8a415c5 --- /dev/null +++ b/vendor/github.com/tetratelabs/wazero/internal/sysfs/datasync_linux.go @@ -0,0 +1,14 @@ +//go:build linux && !tinygo + +package sysfs + +import ( + "os" + "syscall" + + "github.com/tetratelabs/wazero/experimental/sys" +) + +func datasync(f *os.File) sys.Errno { + return sys.UnwrapOSError(syscall.Fdatasync(int(f.Fd()))) +} diff --git a/vendor/github.com/tetratelabs/wazero/internal/sysfs/datasync_tinygo.go b/vendor/github.com/tetratelabs/wazero/internal/sysfs/datasync_tinygo.go new file mode 100644 index 000000000..e58fc9142 --- /dev/null +++ b/vendor/github.com/tetratelabs/wazero/internal/sysfs/datasync_tinygo.go @@ -0,0 +1,13 @@ +//go:build tinygo + +package sysfs + +import ( + "os" + + "github.com/tetratelabs/wazero/experimental/sys" +) + +func datasync(f *os.File) sys.Errno { + return sys.ENOSYS +} diff --git a/vendor/github.com/tetratelabs/wazero/internal/sysfs/datasync_unsupported.go b/vendor/github.com/tetratelabs/wazero/internal/sysfs/datasync_unsupported.go new file mode 100644 index 000000000..aa05719be --- /dev/null +++ b/vendor/github.com/tetratelabs/wazero/internal/sysfs/datasync_unsupported.go @@ -0,0 +1,14 @@ +//go:build !linux + +package sysfs + +import ( + "os" + + "github.com/tetratelabs/wazero/experimental/sys" +) + +func datasync(f *os.File) sys.Errno { + // Attempt to sync everything, even if we only need to sync the data. + return fsync(f) +} diff --git a/vendor/github.com/tetratelabs/wazero/internal/sysfs/dir.go b/vendor/github.com/tetratelabs/wazero/internal/sysfs/dir.go new file mode 100644 index 000000000..f9823287c --- /dev/null +++ b/vendor/github.com/tetratelabs/wazero/internal/sysfs/dir.go @@ -0,0 +1,24 @@ +package sysfs + +import ( + "io" + + "github.com/tetratelabs/wazero/experimental/sys" +) + +func adjustReaddirErr(f sys.File, isClosed bool, err error) sys.Errno { + if err == io.EOF { + return 0 // e.g. Readdir on darwin returns io.EOF, but linux doesn't. + } else if errno := sys.UnwrapOSError(err); errno != 0 { + errno = dirError(f, isClosed, errno) + // Comply with errors allowed on sys.File Readdir + switch errno { + case sys.EINVAL: // os.File Readdir can return this + return sys.EBADF + case sys.ENOTDIR: // dirError can return this + return sys.EBADF + } + return errno + } + return 0 +} diff --git a/vendor/github.com/tetratelabs/wazero/internal/sysfs/dirfs.go b/vendor/github.com/tetratelabs/wazero/internal/sysfs/dirfs.go new file mode 100644 index 000000000..04384038f --- /dev/null +++ b/vendor/github.com/tetratelabs/wazero/internal/sysfs/dirfs.go @@ -0,0 +1,99 @@ +package sysfs + +import ( + "io/fs" + "os" + + experimentalsys "github.com/tetratelabs/wazero/experimental/sys" + "github.com/tetratelabs/wazero/internal/platform" + "github.com/tetratelabs/wazero/sys" +) + +func DirFS(dir string) experimentalsys.FS { + return &dirFS{ + dir: dir, + cleanedDir: ensureTrailingPathSeparator(dir), + } +} + +func ensureTrailingPathSeparator(dir string) string { + if !os.IsPathSeparator(dir[len(dir)-1]) { + return dir + string(os.PathSeparator) + } + return dir +} + +// dirFS is not exported because the input fields must be maintained together. +// This is likely why os.DirFS doesn't, either! +type dirFS struct { + experimentalsys.UnimplementedFS + + dir string + // cleanedDir is for easier OS-specific concatenation, as it always has + // a trailing path separator. + cleanedDir string +} + +// String implements fmt.Stringer +func (d *dirFS) String() string { + return d.dir +} + +// OpenFile implements the same method as documented on sys.FS +func (d *dirFS) OpenFile(path string, flag experimentalsys.Oflag, perm fs.FileMode) (experimentalsys.File, experimentalsys.Errno) { + return OpenOSFile(d.join(path), flag, perm) +} + +// Lstat implements the same method as documented on sys.FS +func (d *dirFS) Lstat(path string) (sys.Stat_t, experimentalsys.Errno) { + return lstat(d.join(path)) +} + +// Stat implements the same method as documented on sys.FS +func (d *dirFS) Stat(path string) (sys.Stat_t, experimentalsys.Errno) { + return stat(d.join(path)) +} + +// Mkdir implements the same method as documented on sys.FS +func (d *dirFS) Mkdir(path string, perm fs.FileMode) (errno experimentalsys.Errno) { + err := os.Mkdir(d.join(path), perm) + if errno = experimentalsys.UnwrapOSError(err); errno == experimentalsys.ENOTDIR { + errno = experimentalsys.ENOENT + } + return +} + +// Readlink implements the same method as documented on sys.FS +func (d *dirFS) Readlink(path string) (string, experimentalsys.Errno) { + // Note: do not use syscall.Readlink as that causes race on Windows. + // In any case, syscall.Readlink does almost the same logic as os.Readlink. + dst, err := os.Readlink(d.join(path)) + if err != nil { + return "", experimentalsys.UnwrapOSError(err) + } + return platform.ToPosixPath(dst), 0 +} + +// Rmdir implements the same method as documented on sys.FS +func (d *dirFS) Rmdir(path string) experimentalsys.Errno { + return rmdir(d.join(path)) +} + +// Utimens implements the same method as documented on sys.FS +func (d *dirFS) Utimens(path string, atim, mtim int64) experimentalsys.Errno { + return utimens(d.join(path), atim, mtim) +} + +func (d *dirFS) join(path string) string { + switch path { + case "", ".", "/": + if d.cleanedDir == "/" { + return "/" + } + // cleanedDir includes an unnecessary delimiter for the root path. + return d.cleanedDir[:len(d.cleanedDir)-1] + } + // TODO: Enforce similar to safefilepath.FromFS(path), but be careful as + // relative path inputs are allowed. e.g. dir or path == ../ + return d.cleanedDir + path +} diff --git a/vendor/github.com/tetratelabs/wazero/internal/sysfs/dirfs_supported.go b/vendor/github.com/tetratelabs/wazero/internal/sysfs/dirfs_supported.go new file mode 100644 index 000000000..ff93415b9 --- /dev/null +++ b/vendor/github.com/tetratelabs/wazero/internal/sysfs/dirfs_supported.go @@ -0,0 +1,42 @@ +//go:build !tinygo + +package sysfs + +import ( + "io/fs" + "os" + + experimentalsys "github.com/tetratelabs/wazero/experimental/sys" +) + +// Link implements the same method as documented on sys.FS +func (d *dirFS) Link(oldName, newName string) experimentalsys.Errno { + err := os.Link(d.join(oldName), d.join(newName)) + return experimentalsys.UnwrapOSError(err) +} + +// Unlink implements the same method as documented on sys.FS +func (d *dirFS) Unlink(path string) (err experimentalsys.Errno) { + return unlink(d.join(path)) +} + +// Rename implements the same method as documented on sys.FS +func (d *dirFS) Rename(from, to string) experimentalsys.Errno { + from, to = d.join(from), d.join(to) + return rename(from, to) +} + +// Chmod implements the same method as documented on sys.FS +func (d *dirFS) Chmod(path string, perm fs.FileMode) experimentalsys.Errno { + err := os.Chmod(d.join(path), perm) + return experimentalsys.UnwrapOSError(err) +} + +// Symlink implements the same method as documented on sys.FS +func (d *dirFS) Symlink(oldName, link string) experimentalsys.Errno { + // Note: do not resolve `oldName` relative to this dirFS. The link result is always resolved + // when dereference the `link` on its usage (e.g. readlink, read, etc). + // https://github.com/bytecodealliance/cap-std/blob/v1.0.4/cap-std/src/fs/dir.rs#L404-L409 + err := os.Symlink(oldName, d.join(link)) + return experimentalsys.UnwrapOSError(err) +} diff --git a/vendor/github.com/tetratelabs/wazero/internal/sysfs/dirfs_unsupported.go b/vendor/github.com/tetratelabs/wazero/internal/sysfs/dirfs_unsupported.go new file mode 100644 index 000000000..98b1a3b84 --- /dev/null +++ b/vendor/github.com/tetratelabs/wazero/internal/sysfs/dirfs_unsupported.go @@ -0,0 +1,34 @@ +//go:build tinygo + +package sysfs + +import ( + "io/fs" + + experimentalsys "github.com/tetratelabs/wazero/experimental/sys" +) + +// Link implements the same method as documented on sys.FS +func (d *dirFS) Link(oldName, newName string) experimentalsys.Errno { + return experimentalsys.ENOSYS +} + +// Unlink implements the same method as documented on sys.FS +func (d *dirFS) Unlink(path string) (err experimentalsys.Errno) { + return experimentalsys.ENOSYS +} + +// Rename implements the same method as documented on sys.FS +func (d *dirFS) Rename(from, to string) experimentalsys.Errno { + return experimentalsys.ENOSYS +} + +// Chmod implements the same method as documented on sys.FS +func (d *dirFS) Chmod(path string, perm fs.FileMode) experimentalsys.Errno { + return experimentalsys.ENOSYS +} + +// Symlink implements the same method as documented on sys.FS +func (d *dirFS) Symlink(oldName, link string) experimentalsys.Errno { + return experimentalsys.ENOSYS +} diff --git a/vendor/github.com/tetratelabs/wazero/internal/sysfs/file.go b/vendor/github.com/tetratelabs/wazero/internal/sysfs/file.go new file mode 100644 index 000000000..9a77205bb --- /dev/null +++ b/vendor/github.com/tetratelabs/wazero/internal/sysfs/file.go @@ -0,0 +1,520 @@ +package sysfs + +import ( + "io" + "io/fs" + "os" + "time" + + experimentalsys "github.com/tetratelabs/wazero/experimental/sys" + "github.com/tetratelabs/wazero/internal/fsapi" + "github.com/tetratelabs/wazero/sys" +) + +func NewStdioFile(stdin bool, f fs.File) (fsapi.File, error) { + // Return constant stat, which has fake times, but keep the underlying + // file mode. Fake times are needed to pass wasi-testsuite. + // https://github.com/WebAssembly/wasi-testsuite/blob/af57727/tests/rust/src/bin/fd_filestat_get.rs#L1-L19 + var mode fs.FileMode + if st, err := f.Stat(); err != nil { + return nil, err + } else { + mode = st.Mode() + } + var flag experimentalsys.Oflag + if stdin { + flag = experimentalsys.O_RDONLY + } else { + flag = experimentalsys.O_WRONLY + } + var file fsapi.File + if of, ok := f.(*os.File); ok { + // This is ok because functions that need path aren't used by stdioFile + file = newOsFile("", flag, 0, of) + } else { + file = &fsFile{file: f} + } + return &stdioFile{File: file, st: sys.Stat_t{Mode: mode, Nlink: 1}}, nil +} + +func OpenFile(path string, flag experimentalsys.Oflag, perm fs.FileMode) (*os.File, experimentalsys.Errno) { + if flag&experimentalsys.O_DIRECTORY != 0 && flag&(experimentalsys.O_WRONLY|experimentalsys.O_RDWR) != 0 { + return nil, experimentalsys.EISDIR // invalid to open a directory writeable + } + return openFile(path, flag, perm) +} + +func OpenOSFile(path string, flag experimentalsys.Oflag, perm fs.FileMode) (experimentalsys.File, experimentalsys.Errno) { + f, errno := OpenFile(path, flag, perm) + if errno != 0 { + return nil, errno + } + return newOsFile(path, flag, perm, f), 0 +} + +func OpenFSFile(fs fs.FS, path string, flag experimentalsys.Oflag, perm fs.FileMode) (experimentalsys.File, experimentalsys.Errno) { + if flag&experimentalsys.O_DIRECTORY != 0 && flag&(experimentalsys.O_WRONLY|experimentalsys.O_RDWR) != 0 { + return nil, experimentalsys.EISDIR // invalid to open a directory writeable + } + f, err := fs.Open(path) + if errno := experimentalsys.UnwrapOSError(err); errno != 0 { + return nil, errno + } + // Don't return an os.File because the path is not absolute. osFile needs + // the path to be real and certain FS.File impls are subrooted. + return &fsFile{fs: fs, name: path, file: f}, 0 +} + +type stdioFile struct { + fsapi.File + st sys.Stat_t +} + +// SetAppend implements File.SetAppend +func (f *stdioFile) SetAppend(bool) experimentalsys.Errno { + // Ignore for stdio. + return 0 +} + +// IsAppend implements File.SetAppend +func (f *stdioFile) IsAppend() bool { + return true +} + +// Stat implements File.Stat +func (f *stdioFile) Stat() (sys.Stat_t, experimentalsys.Errno) { + return f.st, 0 +} + +// Close implements File.Close +func (f *stdioFile) Close() experimentalsys.Errno { + return 0 +} + +// fsFile is used for wrapped fs.File, like os.Stdin or any fs.File +// implementation. Notably, this does not have access to the full file path. +// so certain operations can't be supported, such as inode lookups on Windows. +type fsFile struct { + experimentalsys.UnimplementedFile + + // fs is the file-system that opened the file, or nil when wrapped for + // pre-opens like stdio. + fs fs.FS + + // name is what was used in fs for Open, so it may not be the actual path. + name string + + // file is always set, possibly an os.File like os.Stdin. + file fs.File + + // reopenDir is true if reopen should be called before Readdir. This flag + // is deferred until Readdir to prevent redundant rewinds. This could + // happen if Seek(0) was called twice, or if in Windows, Seek(0) was called + // before Readdir. + reopenDir bool + + // closed is true when closed was called. This ensures proper sys.EBADF + closed bool + + // cachedStat includes fields that won't change while a file is open. + cachedSt *cachedStat +} + +type cachedStat struct { + // dev is the same as sys.Stat_t Dev. + dev uint64 + + // dev is the same as sys.Stat_t Ino. + ino sys.Inode + + // isDir is sys.Stat_t Mode masked with fs.ModeDir + isDir bool +} + +// cachedStat returns the cacheable parts of sys.Stat_t or an error if they +// couldn't be retrieved. +func (f *fsFile) cachedStat() (dev uint64, ino sys.Inode, isDir bool, errno experimentalsys.Errno) { + if f.cachedSt == nil { + if _, errno = f.Stat(); errno != 0 { + return + } + } + return f.cachedSt.dev, f.cachedSt.ino, f.cachedSt.isDir, 0 +} + +// Dev implements the same method as documented on sys.File +func (f *fsFile) Dev() (uint64, experimentalsys.Errno) { + dev, _, _, errno := f.cachedStat() + return dev, errno +} + +// Ino implements the same method as documented on sys.File +func (f *fsFile) Ino() (sys.Inode, experimentalsys.Errno) { + _, ino, _, errno := f.cachedStat() + return ino, errno +} + +// IsDir implements the same method as documented on sys.File +func (f *fsFile) IsDir() (bool, experimentalsys.Errno) { + _, _, isDir, errno := f.cachedStat() + return isDir, errno +} + +// IsAppend implements the same method as documented on sys.File +func (f *fsFile) IsAppend() bool { + return false +} + +// SetAppend implements the same method as documented on sys.File +func (f *fsFile) SetAppend(bool) (errno experimentalsys.Errno) { + return fileError(f, f.closed, experimentalsys.ENOSYS) +} + +// Stat implements the same method as documented on sys.File +func (f *fsFile) Stat() (sys.Stat_t, experimentalsys.Errno) { + if f.closed { + return sys.Stat_t{}, experimentalsys.EBADF + } + + st, errno := statFile(f.file) + switch errno { + case 0: + f.cachedSt = &cachedStat{dev: st.Dev, ino: st.Ino, isDir: st.Mode&fs.ModeDir == fs.ModeDir} + case experimentalsys.EIO: + errno = experimentalsys.EBADF + } + return st, errno +} + +// Read implements the same method as documented on sys.File +func (f *fsFile) Read(buf []byte) (n int, errno experimentalsys.Errno) { + if n, errno = read(f.file, buf); errno != 0 { + // Defer validation overhead until we've already had an error. + errno = fileError(f, f.closed, errno) + } + return +} + +// Pread implements the same method as documented on sys.File +func (f *fsFile) Pread(buf []byte, off int64) (n int, errno experimentalsys.Errno) { + if ra, ok := f.file.(io.ReaderAt); ok { + if n, errno = pread(ra, buf, off); errno != 0 { + // Defer validation overhead until we've already had an error. + errno = fileError(f, f.closed, errno) + } + return + } + + // See /RATIONALE.md "fd_pread: io.Seeker fallback when io.ReaderAt is not supported" + if rs, ok := f.file.(io.ReadSeeker); ok { + // Determine the current position in the file, as we need to revert it. + currentOffset, err := rs.Seek(0, io.SeekCurrent) + if err != nil { + return 0, fileError(f, f.closed, experimentalsys.UnwrapOSError(err)) + } + + // Put the read position back when complete. + defer func() { _, _ = rs.Seek(currentOffset, io.SeekStart) }() + + // If the current offset isn't in sync with this reader, move it. + if off != currentOffset { + if _, err = rs.Seek(off, io.SeekStart); err != nil { + return 0, fileError(f, f.closed, experimentalsys.UnwrapOSError(err)) + } + } + + n, err = rs.Read(buf) + if errno = experimentalsys.UnwrapOSError(err); errno != 0 { + // Defer validation overhead until we've already had an error. + errno = fileError(f, f.closed, errno) + } + } else { + errno = experimentalsys.ENOSYS // unsupported + } + return +} + +// Seek implements the same method as documented on sys.File +func (f *fsFile) Seek(offset int64, whence int) (newOffset int64, errno experimentalsys.Errno) { + // If this is a directory, and we're attempting to seek to position zero, + // we have to re-open the file to ensure the directory state is reset. + var isDir bool + if offset == 0 && whence == io.SeekStart { + if isDir, errno = f.IsDir(); errno == 0 && isDir { + f.reopenDir = true + return + } + } + + if s, ok := f.file.(io.Seeker); ok { + if newOffset, errno = seek(s, offset, whence); errno != 0 { + // Defer validation overhead until we've already had an error. + errno = fileError(f, f.closed, errno) + } + } else { + errno = experimentalsys.ENOSYS // unsupported + } + return +} + +// Readdir implements the same method as documented on sys.File +// +// Notably, this uses readdirFile or fs.ReadDirFile if available. This does not +// return inodes on windows. +func (f *fsFile) Readdir(n int) (dirents []experimentalsys.Dirent, errno experimentalsys.Errno) { + // Windows lets you Readdir after close, FS.File also may not implement + // close in a meaningful way. read our closed field to return consistent + // results. + if f.closed { + errno = experimentalsys.EBADF + return + } + + if f.reopenDir { // re-open the directory if needed. + f.reopenDir = false + if errno = adjustReaddirErr(f, f.closed, f.reopen()); errno != 0 { + return + } + } + + if of, ok := f.file.(readdirFile); ok { + // We can't use f.name here because it is the path up to the sys.FS, + // not necessarily the real path. For this reason, Windows may not be + // able to populate inodes. However, Darwin and Linux will. + if dirents, errno = readdir(of, "", n); errno != 0 { + errno = adjustReaddirErr(f, f.closed, errno) + } + return + } + + // Try with FS.ReadDirFile which is available on api.FS implementations + // like embed:FS. + if rdf, ok := f.file.(fs.ReadDirFile); ok { + entries, e := rdf.ReadDir(n) + if errno = adjustReaddirErr(f, f.closed, e); errno != 0 { + return + } + dirents = make([]experimentalsys.Dirent, 0, len(entries)) + for _, e := range entries { + // By default, we don't attempt to read inode data + dirents = append(dirents, experimentalsys.Dirent{Name: e.Name(), Type: e.Type()}) + } + } else { + errno = experimentalsys.EBADF // not a directory + } + return +} + +// Write implements the same method as documented on sys.File. +func (f *fsFile) Write(buf []byte) (n int, errno experimentalsys.Errno) { + if w, ok := f.file.(io.Writer); ok { + if n, errno = write(w, buf); errno != 0 { + // Defer validation overhead until we've already had an error. + errno = fileError(f, f.closed, errno) + } + } else { + errno = experimentalsys.ENOSYS // unsupported + } + return +} + +// Pwrite implements the same method as documented on sys.File. +func (f *fsFile) Pwrite(buf []byte, off int64) (n int, errno experimentalsys.Errno) { + if wa, ok := f.file.(io.WriterAt); ok { + if n, errno = pwrite(wa, buf, off); errno != 0 { + // Defer validation overhead until we've already had an error. + errno = fileError(f, f.closed, errno) + } + } else { + errno = experimentalsys.ENOSYS // unsupported + } + return +} + +// Close implements the same method as documented on sys.File. +func (f *fsFile) Close() experimentalsys.Errno { + if f.closed { + return 0 + } + f.closed = true + return f.close() +} + +func (f *fsFile) close() experimentalsys.Errno { + return experimentalsys.UnwrapOSError(f.file.Close()) +} + +// IsNonblock implements the same method as documented on fsapi.File +func (f *fsFile) IsNonblock() bool { + return false +} + +// SetNonblock implements the same method as documented on fsapi.File +func (f *fsFile) SetNonblock(bool) experimentalsys.Errno { + return experimentalsys.ENOSYS +} + +// Poll implements the same method as documented on fsapi.File +func (f *fsFile) Poll(fsapi.Pflag, int32) (ready bool, errno experimentalsys.Errno) { + return false, experimentalsys.ENOSYS +} + +// dirError is used for commands that work against a directory, but not a file. +func dirError(f experimentalsys.File, isClosed bool, errno experimentalsys.Errno) experimentalsys.Errno { + if vErrno := validate(f, isClosed, false, true); vErrno != 0 { + return vErrno + } + return errno +} + +// fileError is used for commands that work against a file, but not a directory. +func fileError(f experimentalsys.File, isClosed bool, errno experimentalsys.Errno) experimentalsys.Errno { + if vErrno := validate(f, isClosed, true, false); vErrno != 0 { + return vErrno + } + return errno +} + +// validate is used to making syscalls which will fail. +func validate(f experimentalsys.File, isClosed, wantFile, wantDir bool) experimentalsys.Errno { + if isClosed { + return experimentalsys.EBADF + } + + isDir, errno := f.IsDir() + if errno != 0 { + return errno + } + + if wantFile && isDir { + return experimentalsys.EISDIR + } else if wantDir && !isDir { + return experimentalsys.ENOTDIR + } + return 0 +} + +func read(r io.Reader, buf []byte) (n int, errno experimentalsys.Errno) { + if len(buf) == 0 { + return 0, 0 // less overhead on zero-length reads. + } + + n, err := r.Read(buf) + return n, experimentalsys.UnwrapOSError(err) +} + +func pread(ra io.ReaderAt, buf []byte, off int64) (n int, errno experimentalsys.Errno) { + if len(buf) == 0 { + return 0, 0 // less overhead on zero-length reads. + } + + n, err := ra.ReadAt(buf, off) + return n, experimentalsys.UnwrapOSError(err) +} + +func seek(s io.Seeker, offset int64, whence int) (int64, experimentalsys.Errno) { + if uint(whence) > io.SeekEnd { + return 0, experimentalsys.EINVAL // negative or exceeds the largest valid whence + } + + newOffset, err := s.Seek(offset, whence) + return newOffset, experimentalsys.UnwrapOSError(err) +} + +// reopenFile allows re-opening a file for reasons such as applying flags or +// directory iteration. +type reopenFile func() experimentalsys.Errno + +// compile-time check to ensure fsFile.reopen implements reopenFile. +var _ reopenFile = (*fsFile)(nil).reopen + +// reopen implements the same method as documented on reopenFile. +func (f *fsFile) reopen() experimentalsys.Errno { + _ = f.close() + var err error + f.file, err = f.fs.Open(f.name) + return experimentalsys.UnwrapOSError(err) +} + +// readdirFile allows masking the `Readdir` function on os.File. +type readdirFile interface { + Readdir(n int) ([]fs.FileInfo, error) +} + +// readdir uses readdirFile.Readdir, special casing windows when path !="". +func readdir(f readdirFile, path string, n int) (dirents []experimentalsys.Dirent, errno experimentalsys.Errno) { + fis, e := f.Readdir(n) + if errno = experimentalsys.UnwrapOSError(e); errno != 0 { + return + } + + dirents = make([]experimentalsys.Dirent, 0, len(fis)) + + // linux/darwin won't have to fan out to lstat, but windows will. + var ino sys.Inode + for fi := range fis { + t := fis[fi] + // inoFromFileInfo is more efficient than sys.NewStat_t, as it gets the + // inode without allocating an instance and filling other fields. + if ino, errno = inoFromFileInfo(path, t); errno != 0 { + return + } + dirents = append(dirents, experimentalsys.Dirent{Name: t.Name(), Ino: ino, Type: t.Mode().Type()}) + } + return +} + +func write(w io.Writer, buf []byte) (n int, errno experimentalsys.Errno) { + if len(buf) == 0 { + return 0, 0 // less overhead on zero-length writes. + } + + n, err := w.Write(buf) + return n, experimentalsys.UnwrapOSError(err) +} + +func pwrite(w io.WriterAt, buf []byte, off int64) (n int, errno experimentalsys.Errno) { + if len(buf) == 0 { + return 0, 0 // less overhead on zero-length writes. + } + + n, err := w.WriteAt(buf, off) + return n, experimentalsys.UnwrapOSError(err) +} + +func chtimes(path string, atim, mtim int64) (errno experimentalsys.Errno) { //nolint:unused + // When both inputs are omitted, there is nothing to change. + if atim == experimentalsys.UTIME_OMIT && mtim == experimentalsys.UTIME_OMIT { + return + } + + // UTIME_OMIT is expensive until progress is made in Go, as it requires a + // stat to read-back the value to re-apply. + // - https://github.com/golang/go/issues/32558. + // - https://go-review.googlesource.com/c/go/+/219638 (unmerged) + var st sys.Stat_t + if atim == experimentalsys.UTIME_OMIT || mtim == experimentalsys.UTIME_OMIT { + if st, errno = stat(path); errno != 0 { + return + } + } + + var atime, mtime time.Time + if atim == experimentalsys.UTIME_OMIT { + atime = epochNanosToTime(st.Atim) + mtime = epochNanosToTime(mtim) + } else if mtim == experimentalsys.UTIME_OMIT { + atime = epochNanosToTime(atim) + mtime = epochNanosToTime(st.Mtim) + } else { + atime = epochNanosToTime(atim) + mtime = epochNanosToTime(mtim) + } + return experimentalsys.UnwrapOSError(os.Chtimes(path, atime, mtime)) +} + +func epochNanosToTime(epochNanos int64) time.Time { //nolint:unused + seconds := epochNanos / 1e9 + nanos := epochNanos % 1e9 + return time.Unix(seconds, nanos) +} diff --git a/vendor/github.com/tetratelabs/wazero/internal/sysfs/file_unix.go b/vendor/github.com/tetratelabs/wazero/internal/sysfs/file_unix.go new file mode 100644 index 000000000..f201e813d --- /dev/null +++ b/vendor/github.com/tetratelabs/wazero/internal/sysfs/file_unix.go @@ -0,0 +1,39 @@ +//go:build unix && !tinygo + +package sysfs + +import ( + "syscall" + + "github.com/tetratelabs/wazero/experimental/sys" +) + +const ( + nonBlockingFileReadSupported = true + nonBlockingFileWriteSupported = true +) + +func rmdir(path string) sys.Errno { + err := syscall.Rmdir(path) + return sys.UnwrapOSError(err) +} + +// readFd exposes syscall.Read. +func readFd(fd uintptr, buf []byte) (int, sys.Errno) { + if len(buf) == 0 { + return 0, 0 // Short-circuit 0-len reads. + } + n, err := syscall.Read(int(fd), buf) + errno := sys.UnwrapOSError(err) + return n, errno +} + +// writeFd exposes syscall.Write. +func writeFd(fd uintptr, buf []byte) (int, sys.Errno) { + if len(buf) == 0 { + return 0, 0 // Short-circuit 0-len writes. + } + n, err := syscall.Write(int(fd), buf) + errno := sys.UnwrapOSError(err) + return n, errno +} diff --git a/vendor/github.com/tetratelabs/wazero/internal/sysfs/file_unsupported.go b/vendor/github.com/tetratelabs/wazero/internal/sysfs/file_unsupported.go new file mode 100644 index 000000000..a028b9479 --- /dev/null +++ b/vendor/github.com/tetratelabs/wazero/internal/sysfs/file_unsupported.go @@ -0,0 +1,28 @@ +//go:build !(unix || windows) || tinygo + +package sysfs + +import ( + "os" + + "github.com/tetratelabs/wazero/experimental/sys" +) + +const ( + nonBlockingFileReadSupported = false + nonBlockingFileWriteSupported = false +) + +func rmdir(path string) sys.Errno { + return sys.UnwrapOSError(os.Remove(path)) +} + +// readFd returns ENOSYS on unsupported platforms. +func readFd(fd uintptr, buf []byte) (int, sys.Errno) { + return -1, sys.ENOSYS +} + +// writeFd returns ENOSYS on unsupported platforms. +func writeFd(fd uintptr, buf []byte) (int, sys.Errno) { + return -1, sys.ENOSYS +} diff --git a/vendor/github.com/tetratelabs/wazero/internal/sysfs/file_windows.go b/vendor/github.com/tetratelabs/wazero/internal/sysfs/file_windows.go new file mode 100644 index 000000000..37870ea36 --- /dev/null +++ b/vendor/github.com/tetratelabs/wazero/internal/sysfs/file_windows.go @@ -0,0 +1,175 @@ +package sysfs + +import ( + "errors" + "syscall" + "unsafe" + + "github.com/tetratelabs/wazero/experimental/sys" +) + +const ( + nonBlockingFileReadSupported = true + nonBlockingFileWriteSupported = false + + _ERROR_IO_INCOMPLETE = syscall.Errno(996) +) + +var kernel32 = syscall.NewLazyDLL("kernel32.dll") + +// procPeekNamedPipe is the syscall.LazyProc in kernel32 for PeekNamedPipe +var ( + // procPeekNamedPipe is the syscall.LazyProc in kernel32 for PeekNamedPipe + procPeekNamedPipe = kernel32.NewProc("PeekNamedPipe") + // procGetOverlappedResult is the syscall.LazyProc in kernel32 for GetOverlappedResult + procGetOverlappedResult = kernel32.NewProc("GetOverlappedResult") + // procCreateEventW is the syscall.LazyProc in kernel32 for CreateEventW + procCreateEventW = kernel32.NewProc("CreateEventW") +) + +// readFd returns ENOSYS on unsupported platforms. +// +// PeekNamedPipe: https://learn.microsoft.com/en-us/windows/win32/api/namedpipeapi/nf-namedpipeapi-peeknamedpipe +// "GetFileType can assist in determining what device type the handle refers to. A console handle presents as FILE_TYPE_CHAR." +// https://learn.microsoft.com/en-us/windows/console/console-handles +func readFd(fd uintptr, buf []byte) (int, sys.Errno) { + handle := syscall.Handle(fd) + fileType, err := syscall.GetFileType(handle) + if err != nil { + return 0, sys.UnwrapOSError(err) + } + if fileType&syscall.FILE_TYPE_CHAR == 0 { + return -1, sys.ENOSYS + } + n, errno := peekNamedPipe(handle) + if errno == syscall.ERROR_BROKEN_PIPE { + return 0, 0 + } + if n == 0 { + return -1, sys.EAGAIN + } + un, err := syscall.Read(handle, buf[0:n]) + return un, sys.UnwrapOSError(err) +} + +func writeFd(fd uintptr, buf []byte) (int, sys.Errno) { + return -1, sys.ENOSYS +} + +func readSocket(h uintptr, buf []byte) (int, sys.Errno) { + // Poll the socket to ensure that we never perform a blocking/overlapped Read. + // + // When the socket is closed by the remote peer, wsaPoll will return n=1 and + // errno=0, and syscall.ReadFile will return n=0 and errno=0 -- which indicates + // io.EOF. + if n, errno := wsaPoll( + []pollFd{newPollFd(h, _POLLIN, 0)}, 0); !errors.Is(errno, sys.Errno(0)) { + return 0, sys.UnwrapOSError(errno) + } else if n <= 0 { + return 0, sys.EAGAIN + } + + // Properly use overlapped result. + // + // If hFile was opened with FILE_FLAG_OVERLAPPED, the following conditions are in effect: + // - The lpOverlapped parameter must point to a valid and unique OVERLAPPED structure, + // otherwise the function can incorrectly report that the read operation is complete. + // - The lpNumberOfBytesRead parameter should be set to NULL. Use the GetOverlappedResult + // function to get the actual number of bytes read. If the hFile parameter is associated + // with an I/O completion port, you can also get the number of bytes read by calling the + // GetQueuedCompletionStatus function. + // + // We are currently skipping checking if hFile was opened with FILE_FLAG_OVERLAPPED but using + // both lpOverlapped and lpNumberOfBytesRead. + var overlapped syscall.Overlapped + + // Create an event to wait on. + if hEvent, err := createEventW(nil, true, false, nil); err != 0 { + return 0, sys.UnwrapOSError(err) + } else { + overlapped.HEvent = syscall.Handle(hEvent) + } + + var done uint32 + errno := syscall.ReadFile(syscall.Handle(h), buf, &done, &overlapped) + if errors.Is(errno, syscall.ERROR_IO_PENDING) { + errno = syscall.CancelIo(syscall.Handle(h)) + if errno != nil { + return 0, sys.UnwrapOSError(errno) // This is a fatal error. CancelIo failed. + } + + done, errno = getOverlappedResult(syscall.Handle(h), &overlapped, true) // wait for I/O to complete(cancel or finish). Overwrite done and errno. + if errors.Is(errno, syscall.ERROR_OPERATION_ABORTED) { + return int(done), sys.EAGAIN // This is one of the expected behavior, I/O was cancelled(completed) before finished. + } + } + + return int(done), sys.UnwrapOSError(errno) +} + +func writeSocket(fd uintptr, buf []byte) (int, sys.Errno) { + var done uint32 + var overlapped syscall.Overlapped + errno := syscall.WriteFile(syscall.Handle(fd), buf, &done, &overlapped) + if errors.Is(errno, syscall.ERROR_IO_PENDING) { + errno = syscall.EAGAIN + } + return int(done), sys.UnwrapOSError(errno) +} + +// peekNamedPipe partially exposes PeekNamedPipe from the Win32 API +// see https://learn.microsoft.com/en-us/windows/win32/api/namedpipeapi/nf-namedpipeapi-peeknamedpipe +func peekNamedPipe(handle syscall.Handle) (uint32, syscall.Errno) { + var totalBytesAvail uint32 + totalBytesPtr := unsafe.Pointer(&totalBytesAvail) + _, _, errno := syscall.SyscallN( + procPeekNamedPipe.Addr(), + uintptr(handle), // [in] HANDLE hNamedPipe, + 0, // [out, optional] LPVOID lpBuffer, + 0, // [in] DWORD nBufferSize, + 0, // [out, optional] LPDWORD lpBytesRead + uintptr(totalBytesPtr), // [out, optional] LPDWORD lpTotalBytesAvail, + 0) // [out, optional] LPDWORD lpBytesLeftThisMessage + return totalBytesAvail, errno +} + +func rmdir(path string) sys.Errno { + err := syscall.Rmdir(path) + return sys.UnwrapOSError(err) +} + +func getOverlappedResult(handle syscall.Handle, overlapped *syscall.Overlapped, wait bool) (uint32, syscall.Errno) { + var totalBytesAvail uint32 + var bwait uintptr + if wait { + bwait = 0xFFFFFFFF + } + totalBytesPtr := unsafe.Pointer(&totalBytesAvail) + _, _, errno := syscall.SyscallN( + procGetOverlappedResult.Addr(), + uintptr(handle), // [in] HANDLE hFile, + uintptr(unsafe.Pointer(overlapped)), // [in] LPOVERLAPPED lpOverlapped, + uintptr(totalBytesPtr), // [out] LPDWORD lpNumberOfBytesTransferred, + bwait) // [in] BOOL bWait + return totalBytesAvail, errno +} + +func createEventW(lpEventAttributes *syscall.SecurityAttributes, bManualReset bool, bInitialState bool, lpName *uint16) (uintptr, syscall.Errno) { + var manualReset uintptr + var initialState uintptr + if bManualReset { + manualReset = 1 + } + if bInitialState { + initialState = 1 + } + handle, _, errno := syscall.SyscallN( + procCreateEventW.Addr(), + uintptr(unsafe.Pointer(lpEventAttributes)), // [in] LPSECURITY_ATTRIBUTES lpEventAttributes, + manualReset, // [in] BOOL bManualReset, + initialState, // [in] BOOL bInitialState, + uintptr(unsafe.Pointer(lpName)), // [in, opt]LPCWSTR lpName, + ) + + return handle, errno +} diff --git a/vendor/github.com/tetratelabs/wazero/internal/sysfs/futimens.go b/vendor/github.com/tetratelabs/wazero/internal/sysfs/futimens.go new file mode 100644 index 000000000..7f6b11094 --- /dev/null +++ b/vendor/github.com/tetratelabs/wazero/internal/sysfs/futimens.go @@ -0,0 +1,37 @@ +//go:build (linux || darwin) && !tinygo + +package sysfs + +import ( + "syscall" + "unsafe" + + "github.com/tetratelabs/wazero/experimental/sys" +) + +func timesToPtr(times *[2]syscall.Timespec) unsafe.Pointer { //nolint:unused + if times != nil { + return unsafe.Pointer(×[0]) + } + return unsafe.Pointer(nil) +} + +func timesToTimespecs(atim int64, mtim int64) (times *[2]syscall.Timespec) { + // When both inputs are omitted, there is nothing to change. + if atim == sys.UTIME_OMIT && mtim == sys.UTIME_OMIT { + return + } + + times = &[2]syscall.Timespec{} + if atim == sys.UTIME_OMIT { + times[0] = syscall.Timespec{Nsec: _UTIME_OMIT} + times[1] = syscall.NsecToTimespec(mtim) + } else if mtim == sys.UTIME_OMIT { + times[0] = syscall.NsecToTimespec(atim) + times[1] = syscall.Timespec{Nsec: _UTIME_OMIT} + } else { + times[0] = syscall.NsecToTimespec(atim) + times[1] = syscall.NsecToTimespec(mtim) + } + return +} diff --git a/vendor/github.com/tetratelabs/wazero/internal/sysfs/futimens_darwin.go b/vendor/github.com/tetratelabs/wazero/internal/sysfs/futimens_darwin.go new file mode 100644 index 000000000..88e4008f0 --- /dev/null +++ b/vendor/github.com/tetratelabs/wazero/internal/sysfs/futimens_darwin.go @@ -0,0 +1,51 @@ +package sysfs + +import ( + "syscall" + _ "unsafe" + + experimentalsys "github.com/tetratelabs/wazero/experimental/sys" +) + +const ( + _AT_FDCWD = -0x2 + _AT_SYMLINK_NOFOLLOW = 0x0020 + _UTIME_OMIT = -2 +) + +//go:noescape +//go:linkname utimensat syscall.utimensat +func utimensat(dirfd int, path string, times *[2]syscall.Timespec, flags int) error + +func utimens(path string, atim, mtim int64) experimentalsys.Errno { + times := timesToTimespecs(atim, mtim) + if times == nil { + return 0 + } + var flags int + return experimentalsys.UnwrapOSError(utimensat(_AT_FDCWD, path, times, flags)) +} + +func futimens(fd uintptr, atim, mtim int64) experimentalsys.Errno { + times := timesToTimespecs(atim, mtim) + if times == nil { + return 0 + } + _p0 := timesToPtr(times) + + // Warning: futimens only exists since High Sierra (10.13). + _, _, e1 := syscall_syscall6(libc_futimens_trampoline_addr, fd, uintptr(_p0), 0, 0, 0, 0) + return experimentalsys.UnwrapOSError(e1) +} + +// libc_futimens_trampoline_addr is the address of the +// `libc_futimens_trampoline` symbol, defined in `futimens_darwin.s`. +// +// We use this to invoke the syscall through syscall_syscall6 imported below. +var libc_futimens_trampoline_addr uintptr + +// Imports the futimens symbol from libc as `libc_futimens`. +// +// Note: CGO mechanisms are used in darwin regardless of the CGO_ENABLED value +// or the "cgo" build flag. See /RATIONALE.md for why. +//go:cgo_import_dynamic libc_futimens futimens "/usr/lib/libSystem.B.dylib" diff --git a/vendor/github.com/tetratelabs/wazero/internal/sysfs/futimens_darwin.s b/vendor/github.com/tetratelabs/wazero/internal/sysfs/futimens_darwin.s new file mode 100644 index 000000000..b86aecdf0 --- /dev/null +++ b/vendor/github.com/tetratelabs/wazero/internal/sysfs/futimens_darwin.s @@ -0,0 +1,8 @@ +// lifted from golang.org/x/sys unix +#include "textflag.h" + +TEXT libc_futimens_trampoline<>(SB), NOSPLIT, $0-0 + JMP libc_futimens(SB) + +GLOBL ·libc_futimens_trampoline_addr(SB), RODATA, $8 +DATA ·libc_futimens_trampoline_addr(SB)/8, $libc_futimens_trampoline<>(SB) diff --git a/vendor/github.com/tetratelabs/wazero/internal/sysfs/futimens_linux.go b/vendor/github.com/tetratelabs/wazero/internal/sysfs/futimens_linux.go new file mode 100644 index 000000000..db3b1b8b6 --- /dev/null +++ b/vendor/github.com/tetratelabs/wazero/internal/sysfs/futimens_linux.go @@ -0,0 +1,49 @@ +//go:build !tinygo + +package sysfs + +import ( + "syscall" + "unsafe" + _ "unsafe" + + experimentalsys "github.com/tetratelabs/wazero/experimental/sys" +) + +const ( + _AT_FDCWD = -0x64 + _UTIME_OMIT = (1 << 30) - 2 +) + +func utimens(path string, atim, mtim int64) experimentalsys.Errno { + times := timesToTimespecs(atim, mtim) + if times == nil { + return 0 + } + + var flags int + var _p0 *byte + _p0, err := syscall.BytePtrFromString(path) + if err == nil { + err = utimensat(_AT_FDCWD, uintptr(unsafe.Pointer(_p0)), times, flags) + } + return experimentalsys.UnwrapOSError(err) +} + +// On linux, implement futimens via utimensat with the NUL path. +func futimens(fd uintptr, atim, mtim int64) experimentalsys.Errno { + times := timesToTimespecs(atim, mtim) + if times == nil { + return 0 + } + return experimentalsys.UnwrapOSError(utimensat(int(fd), 0 /* NUL */, times, 0)) +} + +// utimensat is like syscall.utimensat special-cased to accept a NUL string for the path value. +func utimensat(dirfd int, strPtr uintptr, times *[2]syscall.Timespec, flags int) (err error) { + _, _, e1 := syscall.Syscall6(syscall.SYS_UTIMENSAT, uintptr(dirfd), strPtr, uintptr(unsafe.Pointer(times)), uintptr(flags), 0, 0) + if e1 != 0 { + err = e1 + } + return +} diff --git a/vendor/github.com/tetratelabs/wazero/internal/sysfs/futimens_unsupported.go b/vendor/github.com/tetratelabs/wazero/internal/sysfs/futimens_unsupported.go new file mode 100644 index 000000000..69d564942 --- /dev/null +++ b/vendor/github.com/tetratelabs/wazero/internal/sysfs/futimens_unsupported.go @@ -0,0 +1,18 @@ +//go:build (!windows && !linux && !darwin) || tinygo + +package sysfs + +import ( + "github.com/tetratelabs/wazero/experimental/sys" +) + +func utimens(path string, atim, mtim int64) sys.Errno { + return chtimes(path, atim, mtim) +} + +func futimens(fd uintptr, atim, mtim int64) error { + // Go exports syscall.Futimes, which is microsecond granularity, and + // WASI tests expect nanosecond. We don't yet have a way to invoke the + // futimens syscall portably. + return sys.ENOSYS +} diff --git a/vendor/github.com/tetratelabs/wazero/internal/sysfs/futimens_windows.go b/vendor/github.com/tetratelabs/wazero/internal/sysfs/futimens_windows.go new file mode 100644 index 000000000..e0c89f303 --- /dev/null +++ b/vendor/github.com/tetratelabs/wazero/internal/sysfs/futimens_windows.go @@ -0,0 +1,42 @@ +package sysfs + +import ( + "syscall" + + "github.com/tetratelabs/wazero/experimental/sys" +) + +func utimens(path string, atim, mtim int64) sys.Errno { + return chtimes(path, atim, mtim) +} + +func futimens(fd uintptr, atim, mtim int64) error { + // Per docs, zero isn't a valid timestamp as it cannot be differentiated + // from nil. In both cases, it is a marker like sys.UTIME_OMIT. + // See https://learn.microsoft.com/en-us/windows/win32/api/fileapi/nf-fileapi-setfiletime + a, w := timespecToFiletime(atim, mtim) + + if a == nil && w == nil { + return nil // both omitted, so nothing to change + } + + // Attempt to get the stat by handle, which works for normal files + h := syscall.Handle(fd) + + // Note: This returns ERROR_ACCESS_DENIED when the input is a directory. + return syscall.SetFileTime(h, nil, a, w) +} + +func timespecToFiletime(atim, mtim int64) (a, w *syscall.Filetime) { + a = timespecToFileTime(atim) + w = timespecToFileTime(mtim) + return +} + +func timespecToFileTime(tim int64) *syscall.Filetime { + if tim == sys.UTIME_OMIT { + return nil + } + ft := syscall.NsecToFiletime(tim) + return &ft +} diff --git a/vendor/github.com/tetratelabs/wazero/internal/sysfs/ino.go b/vendor/github.com/tetratelabs/wazero/internal/sysfs/ino.go new file mode 100644 index 000000000..8344cd16f --- /dev/null +++ b/vendor/github.com/tetratelabs/wazero/internal/sysfs/ino.go @@ -0,0 +1,22 @@ +//go:build !windows && !plan9 && !tinygo + +package sysfs + +import ( + "io/fs" + "syscall" + + experimentalsys "github.com/tetratelabs/wazero/experimental/sys" + "github.com/tetratelabs/wazero/sys" +) + +func inoFromFileInfo(_ string, info fs.FileInfo) (sys.Inode, experimentalsys.Errno) { + switch v := info.Sys().(type) { + case *sys.Stat_t: + return v.Ino, 0 + case *syscall.Stat_t: + return v.Ino, 0 + default: + return 0, 0 + } +} diff --git a/vendor/github.com/tetratelabs/wazero/internal/sysfs/ino_plan9.go b/vendor/github.com/tetratelabs/wazero/internal/sysfs/ino_plan9.go new file mode 100644 index 000000000..9c669a475 --- /dev/null +++ b/vendor/github.com/tetratelabs/wazero/internal/sysfs/ino_plan9.go @@ -0,0 +1,15 @@ +package sysfs + +import ( + "io/fs" + + experimentalsys "github.com/tetratelabs/wazero/experimental/sys" + "github.com/tetratelabs/wazero/sys" +) + +func inoFromFileInfo(_ string, info fs.FileInfo) (sys.Inode, experimentalsys.Errno) { + if v, ok := info.Sys().(*sys.Stat_t); ok { + return v.Ino, 0 + } + return 0, 0 +} diff --git a/vendor/github.com/tetratelabs/wazero/internal/sysfs/ino_tinygo.go b/vendor/github.com/tetratelabs/wazero/internal/sysfs/ino_tinygo.go new file mode 100644 index 000000000..2099231cf --- /dev/null +++ b/vendor/github.com/tetratelabs/wazero/internal/sysfs/ino_tinygo.go @@ -0,0 +1,14 @@ +//go:build tinygo + +package sysfs + +import ( + "io/fs" + + experimentalsys "github.com/tetratelabs/wazero/experimental/sys" + "github.com/tetratelabs/wazero/sys" +) + +func inoFromFileInfo(_ string, info fs.FileInfo) (sys.Inode, experimentalsys.Errno) { + return 0, experimentalsys.ENOTSUP +} diff --git a/vendor/github.com/tetratelabs/wazero/internal/sysfs/ino_windows.go b/vendor/github.com/tetratelabs/wazero/internal/sysfs/ino_windows.go new file mode 100644 index 000000000..d163b3601 --- /dev/null +++ b/vendor/github.com/tetratelabs/wazero/internal/sysfs/ino_windows.go @@ -0,0 +1,28 @@ +package sysfs + +import ( + "io/fs" + "path" + + experimentalsys "github.com/tetratelabs/wazero/experimental/sys" + "github.com/tetratelabs/wazero/sys" +) + +// inoFromFileInfo uses stat to get the inode information of the file. +func inoFromFileInfo(dirPath string, info fs.FileInfo) (ino sys.Inode, errno experimentalsys.Errno) { + if v, ok := info.Sys().(*sys.Stat_t); ok { + return v.Ino, 0 + } + if dirPath == "" { + // This is a FS.File backed implementation which doesn't have access to + // the original file path. + return + } + // Ino is no not in Win32FileAttributeData + inoPath := path.Clean(path.Join(dirPath, info.Name())) + var st sys.Stat_t + if st, errno = lstat(inoPath); errno == 0 { + ino = st.Ino + } + return +} diff --git a/vendor/github.com/tetratelabs/wazero/internal/sysfs/nonblock_unix.go b/vendor/github.com/tetratelabs/wazero/internal/sysfs/nonblock_unix.go new file mode 100644 index 000000000..4477ee977 --- /dev/null +++ b/vendor/github.com/tetratelabs/wazero/internal/sysfs/nonblock_unix.go @@ -0,0 +1,17 @@ +//go:build !windows && !plan9 && !tinygo + +package sysfs + +import ( + "syscall" + + "github.com/tetratelabs/wazero/experimental/sys" +) + +func setNonblock(fd uintptr, enable bool) sys.Errno { + return sys.UnwrapOSError(syscall.SetNonblock(int(fd), enable)) +} + +func isNonblock(f *osFile) bool { + return f.flag&sys.O_NONBLOCK == sys.O_NONBLOCK +} diff --git a/vendor/github.com/tetratelabs/wazero/internal/sysfs/nonblock_unsupported.go b/vendor/github.com/tetratelabs/wazero/internal/sysfs/nonblock_unsupported.go new file mode 100644 index 000000000..3e141a7b5 --- /dev/null +++ b/vendor/github.com/tetratelabs/wazero/internal/sysfs/nonblock_unsupported.go @@ -0,0 +1,13 @@ +//go:build plan9 || tinygo + +package sysfs + +import "github.com/tetratelabs/wazero/experimental/sys" + +func setNonblock(fd uintptr, enable bool) sys.Errno { + return sys.ENOSYS +} + +func isNonblock(f *osFile) bool { + return false +} diff --git a/vendor/github.com/tetratelabs/wazero/internal/sysfs/nonblock_windows.go b/vendor/github.com/tetratelabs/wazero/internal/sysfs/nonblock_windows.go new file mode 100644 index 000000000..eb38ea5af --- /dev/null +++ b/vendor/github.com/tetratelabs/wazero/internal/sysfs/nonblock_windows.go @@ -0,0 +1,23 @@ +package sysfs + +import ( + "io/fs" + "syscall" + + "github.com/tetratelabs/wazero/experimental/sys" +) + +func setNonblock(fd uintptr, enable bool) sys.Errno { + // We invoke the syscall, but this is currently no-op. + return sys.UnwrapOSError(syscall.SetNonblock(syscall.Handle(fd), enable)) +} + +func isNonblock(f *osFile) bool { + // On Windows, we support non-blocking reads only on named pipes. + isValid := false + st, errno := f.Stat() + if errno == 0 { + isValid = st.Mode&fs.ModeNamedPipe != 0 + } + return isValid && f.flag&sys.O_NONBLOCK == sys.O_NONBLOCK +} diff --git a/vendor/github.com/tetratelabs/wazero/internal/sysfs/oflag.go b/vendor/github.com/tetratelabs/wazero/internal/sysfs/oflag.go new file mode 100644 index 000000000..be6d2c35f --- /dev/null +++ b/vendor/github.com/tetratelabs/wazero/internal/sysfs/oflag.go @@ -0,0 +1,38 @@ +package sysfs + +import ( + "os" + + "github.com/tetratelabs/wazero/experimental/sys" +) + +// toOsOpenFlag converts the input to the flag parameter of os.OpenFile +func toOsOpenFlag(oflag sys.Oflag) (flag int) { + // First flags are exclusive + switch oflag & (sys.O_RDONLY | sys.O_RDWR | sys.O_WRONLY) { + case sys.O_RDONLY: + flag |= os.O_RDONLY + case sys.O_RDWR: + flag |= os.O_RDWR + case sys.O_WRONLY: + flag |= os.O_WRONLY + } + + // Run down the flags defined in the os package + if oflag&sys.O_APPEND != 0 { + flag |= os.O_APPEND + } + if oflag&sys.O_CREAT != 0 { + flag |= os.O_CREATE + } + if oflag&sys.O_EXCL != 0 { + flag |= os.O_EXCL + } + if oflag&sys.O_SYNC != 0 { + flag |= os.O_SYNC + } + if oflag&sys.O_TRUNC != 0 { + flag |= os.O_TRUNC + } + return withSyscallOflag(oflag, flag) +} diff --git a/vendor/github.com/tetratelabs/wazero/internal/sysfs/open_file_darwin.go b/vendor/github.com/tetratelabs/wazero/internal/sysfs/open_file_darwin.go new file mode 100644 index 000000000..a4f54ca2c --- /dev/null +++ b/vendor/github.com/tetratelabs/wazero/internal/sysfs/open_file_darwin.go @@ -0,0 +1,26 @@ +package sysfs + +import ( + "syscall" + + "github.com/tetratelabs/wazero/experimental/sys" +) + +const supportedSyscallOflag = sys.O_DIRECTORY | sys.O_DSYNC | sys.O_NOFOLLOW | sys.O_NONBLOCK + +func withSyscallOflag(oflag sys.Oflag, flag int) int { + if oflag&sys.O_DIRECTORY != 0 { + flag |= syscall.O_DIRECTORY + } + if oflag&sys.O_DSYNC != 0 { + flag |= syscall.O_DSYNC + } + if oflag&sys.O_NOFOLLOW != 0 { + flag |= syscall.O_NOFOLLOW + } + if oflag&sys.O_NONBLOCK != 0 { + flag |= syscall.O_NONBLOCK + } + // syscall.O_RSYNC not defined on darwin + return flag +} diff --git a/vendor/github.com/tetratelabs/wazero/internal/sysfs/open_file_freebsd.go b/vendor/github.com/tetratelabs/wazero/internal/sysfs/open_file_freebsd.go new file mode 100644 index 000000000..42adaa214 --- /dev/null +++ b/vendor/github.com/tetratelabs/wazero/internal/sysfs/open_file_freebsd.go @@ -0,0 +1,24 @@ +package sysfs + +import ( + "syscall" + + "github.com/tetratelabs/wazero/experimental/sys" +) + +const supportedSyscallOflag = sys.O_DIRECTORY | sys.O_NOFOLLOW | sys.O_NONBLOCK + +func withSyscallOflag(oflag sys.Oflag, flag int) int { + if oflag&sys.O_DIRECTORY != 0 { + flag |= syscall.O_DIRECTORY + } + // syscall.O_DSYNC not defined on darwin + if oflag&sys.O_NOFOLLOW != 0 { + flag |= syscall.O_NOFOLLOW + } + if oflag&sys.O_NONBLOCK != 0 { + flag |= syscall.O_NONBLOCK + } + // syscall.O_RSYNC not defined on darwin + return flag +} diff --git a/vendor/github.com/tetratelabs/wazero/internal/sysfs/open_file_linux.go b/vendor/github.com/tetratelabs/wazero/internal/sysfs/open_file_linux.go new file mode 100644 index 000000000..3fe2bb6e1 --- /dev/null +++ b/vendor/github.com/tetratelabs/wazero/internal/sysfs/open_file_linux.go @@ -0,0 +1,30 @@ +//go:build !tinygo + +package sysfs + +import ( + "syscall" + + "github.com/tetratelabs/wazero/experimental/sys" +) + +const supportedSyscallOflag = sys.O_DIRECTORY | sys.O_DSYNC | sys.O_NOFOLLOW | sys.O_NONBLOCK | sys.O_RSYNC + +func withSyscallOflag(oflag sys.Oflag, flag int) int { + if oflag&sys.O_DIRECTORY != 0 { + flag |= syscall.O_DIRECTORY + } + if oflag&sys.O_DSYNC != 0 { + flag |= syscall.O_DSYNC + } + if oflag&sys.O_NOFOLLOW != 0 { + flag |= syscall.O_NOFOLLOW + } + if oflag&sys.O_NONBLOCK != 0 { + flag |= syscall.O_NONBLOCK + } + if oflag&sys.O_RSYNC != 0 { + flag |= syscall.O_RSYNC + } + return flag +} diff --git a/vendor/github.com/tetratelabs/wazero/internal/sysfs/open_file_notwindows.go b/vendor/github.com/tetratelabs/wazero/internal/sysfs/open_file_notwindows.go new file mode 100644 index 000000000..670e35910 --- /dev/null +++ b/vendor/github.com/tetratelabs/wazero/internal/sysfs/open_file_notwindows.go @@ -0,0 +1,20 @@ +//go:build !windows && !tinygo + +package sysfs + +import ( + "io/fs" + "os" + + "github.com/tetratelabs/wazero/experimental/sys" +) + +// openFile is like os.OpenFile except it accepts a sys.Oflag and returns +// sys.Errno. A zero sys.Errno is success. +func openFile(path string, oflag sys.Oflag, perm fs.FileMode) (*os.File, sys.Errno) { + f, err := os.OpenFile(path, toOsOpenFlag(oflag), perm) + // Note: This does not return a sys.File because sys.FS that returns + // one may want to hide the real OS path. For example, this is needed for + // pre-opens. + return f, sys.UnwrapOSError(err) +} diff --git a/vendor/github.com/tetratelabs/wazero/internal/sysfs/open_file_sun.go b/vendor/github.com/tetratelabs/wazero/internal/sysfs/open_file_sun.go new file mode 100644 index 000000000..bdf7dd84d --- /dev/null +++ b/vendor/github.com/tetratelabs/wazero/internal/sysfs/open_file_sun.go @@ -0,0 +1,31 @@ +//go:build illumos || solaris + +package sysfs + +import ( + "syscall" + + "github.com/tetratelabs/wazero/experimental/sys" +) + +const supportedSyscallOflag = sys.O_DIRECTORY | sys.O_DSYNC | sys.O_NOFOLLOW | sys.O_NONBLOCK | sys.O_RSYNC + +func withSyscallOflag(oflag sys.Oflag, flag int) int { + if oflag&sys.O_DIRECTORY != 0 { + // See https://github.com/illumos/illumos-gate/blob/edd580643f2cf1434e252cd7779e83182ea84945/usr/src/uts/common/sys/fcntl.h#L90 + flag |= 0x1000000 + } + if oflag&sys.O_DSYNC != 0 { + flag |= syscall.O_DSYNC + } + if oflag&sys.O_NOFOLLOW != 0 { + flag |= syscall.O_NOFOLLOW + } + if oflag&sys.O_NONBLOCK != 0 { + flag |= syscall.O_NONBLOCK + } + if oflag&sys.O_RSYNC != 0 { + flag |= syscall.O_RSYNC + } + return flag +} diff --git a/vendor/github.com/tetratelabs/wazero/internal/sysfs/open_file_tinygo.go b/vendor/github.com/tetratelabs/wazero/internal/sysfs/open_file_tinygo.go new file mode 100644 index 000000000..ccf6847c0 --- /dev/null +++ b/vendor/github.com/tetratelabs/wazero/internal/sysfs/open_file_tinygo.go @@ -0,0 +1,25 @@ +//go:build tinygo + +package sysfs + +import ( + "io/fs" + "os" + + "github.com/tetratelabs/wazero/experimental/sys" +) + +const supportedSyscallOflag = sys.Oflag(0) + +func withSyscallOflag(oflag sys.Oflag, flag int) int { + // O_DIRECTORY not defined + // O_DSYNC not defined + // O_NOFOLLOW not defined + // O_NONBLOCK not defined + // O_RSYNC not defined + return flag +} + +func openFile(path string, oflag sys.Oflag, perm fs.FileMode) (*os.File, sys.Errno) { + return nil, sys.ENOSYS +} diff --git a/vendor/github.com/tetratelabs/wazero/internal/sysfs/open_file_unsupported.go b/vendor/github.com/tetratelabs/wazero/internal/sysfs/open_file_unsupported.go new file mode 100644 index 000000000..9f7a6d088 --- /dev/null +++ b/vendor/github.com/tetratelabs/wazero/internal/sysfs/open_file_unsupported.go @@ -0,0 +1,18 @@ +//go:build !darwin && !linux && !windows && !illumos && !solaris && !freebsd + +package sysfs + +import ( + "github.com/tetratelabs/wazero/experimental/sys" +) + +const supportedSyscallOflag = sys.Oflag(0) + +func withSyscallOflag(oflag sys.Oflag, flag int) int { + // O_DIRECTORY not defined + // O_DSYNC not defined + // O_NOFOLLOW not defined + // O_NONBLOCK not defined + // O_RSYNC not defined + return flag +} diff --git a/vendor/github.com/tetratelabs/wazero/internal/sysfs/open_file_windows.go b/vendor/github.com/tetratelabs/wazero/internal/sysfs/open_file_windows.go new file mode 100644 index 000000000..717f8598a --- /dev/null +++ b/vendor/github.com/tetratelabs/wazero/internal/sysfs/open_file_windows.go @@ -0,0 +1,161 @@ +package sysfs + +import ( + "io/fs" + "os" + "strings" + "syscall" + "unsafe" + + "github.com/tetratelabs/wazero/experimental/sys" +) + +func openFile(path string, oflag sys.Oflag, perm fs.FileMode) (*os.File, sys.Errno) { + isDir := oflag&sys.O_DIRECTORY > 0 + flag := toOsOpenFlag(oflag) + + // TODO: document why we are opening twice + fd, err := open(path, flag|syscall.O_CLOEXEC, uint32(perm)) + if err == nil { + return os.NewFile(uintptr(fd), path), 0 + } + + // TODO: Set FILE_SHARE_DELETE for directory as well. + f, err := os.OpenFile(path, flag, perm) + errno := sys.UnwrapOSError(err) + if errno == 0 { + return f, 0 + } + + switch errno { + case sys.EINVAL: + // WASI expects ENOTDIR for a file path with a trailing slash. + if strings.HasSuffix(path, "/") { + errno = sys.ENOTDIR + } + // To match expectations of WASI, e.g. TinyGo TestStatBadDir, return + // ENOENT, not ENOTDIR. + case sys.ENOTDIR: + errno = sys.ENOENT + case sys.ENOENT: + if isSymlink(path) { + // Either symlink or hard link not found. We change the returned + // errno depending on if it is symlink or not to have consistent + // behavior across OSes. + if isDir { + // Dangling symlink dir must raise ENOTDIR. + errno = sys.ENOTDIR + } else { + errno = sys.ELOOP + } + } + } + return f, errno +} + +const supportedSyscallOflag = sys.O_NONBLOCK + +// Map to synthetic values here https://github.com/golang/go/blob/go1.20/src/syscall/types_windows.go#L34-L48 +func withSyscallOflag(oflag sys.Oflag, flag int) int { + // O_DIRECTORY not defined in windows + // O_DSYNC not defined in windows + // O_NOFOLLOW not defined in windows + if oflag&sys.O_NONBLOCK != 0 { + flag |= syscall.O_NONBLOCK + } + // O_RSYNC not defined in windows + return flag +} + +func isSymlink(path string) bool { + if st, e := os.Lstat(path); e == nil && st.Mode()&os.ModeSymlink != 0 { + return true + } + return false +} + +// # Differences from syscall.Open +// +// This code is based on syscall.Open from the below link with some differences +// https://github.com/golang/go/blame/go1.20/src/syscall/syscall_windows.go#L308-L379 +// +// - syscall.O_CREAT doesn't imply syscall.GENERIC_WRITE as that breaks +// flag expectations in wasi. +// - add support for setting FILE_SHARE_DELETE. +func open(path string, mode int, perm uint32) (fd syscall.Handle, err error) { + if len(path) == 0 { + return syscall.InvalidHandle, syscall.ERROR_FILE_NOT_FOUND + } + pathp, err := syscall.UTF16PtrFromString(path) + if err != nil { + return syscall.InvalidHandle, err + } + var access uint32 + switch mode & (syscall.O_RDONLY | syscall.O_WRONLY | syscall.O_RDWR) { + case syscall.O_RDONLY: + access = syscall.GENERIC_READ + case syscall.O_WRONLY: + access = syscall.GENERIC_WRITE + case syscall.O_RDWR: + access = syscall.GENERIC_READ | syscall.GENERIC_WRITE + } + if mode&syscall.O_APPEND != 0 { + access &^= syscall.GENERIC_WRITE + access |= syscall.FILE_APPEND_DATA + } + sharemode := uint32(syscall.FILE_SHARE_READ | syscall.FILE_SHARE_WRITE | syscall.FILE_SHARE_DELETE) + var sa *syscall.SecurityAttributes + if mode&syscall.O_CLOEXEC == 0 { + var _sa syscall.SecurityAttributes + _sa.Length = uint32(unsafe.Sizeof(sa)) + _sa.InheritHandle = 1 + sa = &_sa + } + var createmode uint32 + switch { + case mode&(syscall.O_CREAT|syscall.O_EXCL) == (syscall.O_CREAT | syscall.O_EXCL): + createmode = syscall.CREATE_NEW + case mode&(syscall.O_CREAT|syscall.O_TRUNC) == (syscall.O_CREAT | syscall.O_TRUNC): + createmode = syscall.CREATE_ALWAYS + case mode&syscall.O_CREAT == syscall.O_CREAT: + createmode = syscall.OPEN_ALWAYS + case mode&syscall.O_TRUNC == syscall.O_TRUNC: + createmode = syscall.TRUNCATE_EXISTING + default: + createmode = syscall.OPEN_EXISTING + } + var attrs uint32 = syscall.FILE_ATTRIBUTE_NORMAL + if perm&syscall.S_IWRITE == 0 { + attrs = syscall.FILE_ATTRIBUTE_READONLY + if createmode == syscall.CREATE_ALWAYS { + // We have been asked to create a read-only file. + // If the file already exists, the semantics of + // the Unix open system call is to preserve the + // existing permissions. If we pass CREATE_ALWAYS + // and FILE_ATTRIBUTE_READONLY to CreateFile, + // and the file already exists, CreateFile will + // change the file permissions. + // Avoid that to preserve the Unix semantics. + h, e := syscall.CreateFile(pathp, access, sharemode, sa, syscall.TRUNCATE_EXISTING, syscall.FILE_ATTRIBUTE_NORMAL, 0) + switch e { + case syscall.ERROR_FILE_NOT_FOUND, syscall.ERROR_PATH_NOT_FOUND: + // File does not exist. These are the same + // errors as Errno.Is checks for ErrNotExist. + // Carry on to create the file. + default: + // Success or some different error. + return h, e + } + } + } + + // This shouldn't be included before 1.20 to have consistent behavior. + // https://github.com/golang/go/commit/0f0aa5d8a6a0253627d58b3aa083b24a1091933f + if createmode == syscall.OPEN_EXISTING && access == syscall.GENERIC_READ { + // Necessary for opening directory handles. + attrs |= syscall.FILE_FLAG_BACKUP_SEMANTICS + } + + h, e := syscall.CreateFile(pathp, access, sharemode, sa, createmode, attrs, 0) + return h, e +} diff --git a/vendor/github.com/tetratelabs/wazero/internal/sysfs/osfile.go b/vendor/github.com/tetratelabs/wazero/internal/sysfs/osfile.go new file mode 100644 index 000000000..490f0fa68 --- /dev/null +++ b/vendor/github.com/tetratelabs/wazero/internal/sysfs/osfile.go @@ -0,0 +1,295 @@ +package sysfs + +import ( + "io" + "io/fs" + "os" + "runtime" + + experimentalsys "github.com/tetratelabs/wazero/experimental/sys" + "github.com/tetratelabs/wazero/internal/fsapi" + "github.com/tetratelabs/wazero/sys" +) + +func newOsFile(path string, flag experimentalsys.Oflag, perm fs.FileMode, f *os.File) fsapi.File { + // Windows cannot read files written to a directory after it was opened. + // This was noticed in #1087 in zig tests. Use a flag instead of a + // different type. + reopenDir := runtime.GOOS == "windows" + return &osFile{path: path, flag: flag, perm: perm, reopenDir: reopenDir, file: f, fd: f.Fd()} +} + +// osFile is a file opened with this package, and uses os.File or syscalls to +// implement api.File. +type osFile struct { + path string + flag experimentalsys.Oflag + perm fs.FileMode + file *os.File + fd uintptr + + // reopenDir is true if reopen should be called before Readdir. This flag + // is deferred until Readdir to prevent redundant rewinds. This could + // happen if Seek(0) was called twice, or if in Windows, Seek(0) was called + // before Readdir. + reopenDir bool + + // closed is true when closed was called. This ensures proper sys.EBADF + closed bool + + // cachedStat includes fields that won't change while a file is open. + cachedSt *cachedStat +} + +// cachedStat returns the cacheable parts of sys.Stat_t or an error if they +// couldn't be retrieved. +func (f *osFile) cachedStat() (dev uint64, ino sys.Inode, isDir bool, errno experimentalsys.Errno) { + if f.cachedSt == nil { + if _, errno = f.Stat(); errno != 0 { + return + } + } + return f.cachedSt.dev, f.cachedSt.ino, f.cachedSt.isDir, 0 +} + +// Dev implements the same method as documented on sys.File +func (f *osFile) Dev() (uint64, experimentalsys.Errno) { + dev, _, _, errno := f.cachedStat() + return dev, errno +} + +// Ino implements the same method as documented on sys.File +func (f *osFile) Ino() (sys.Inode, experimentalsys.Errno) { + _, ino, _, errno := f.cachedStat() + return ino, errno +} + +// IsDir implements the same method as documented on sys.File +func (f *osFile) IsDir() (bool, experimentalsys.Errno) { + _, _, isDir, errno := f.cachedStat() + return isDir, errno +} + +// IsAppend implements File.IsAppend +func (f *osFile) IsAppend() bool { + return f.flag&experimentalsys.O_APPEND == experimentalsys.O_APPEND +} + +// SetAppend implements the same method as documented on sys.File +func (f *osFile) SetAppend(enable bool) (errno experimentalsys.Errno) { + if enable { + f.flag |= experimentalsys.O_APPEND + } else { + f.flag &= ^experimentalsys.O_APPEND + } + + // Clear any create or trunc flag, as we are re-opening, not re-creating. + f.flag &= ^(experimentalsys.O_CREAT | experimentalsys.O_TRUNC) + + // appendMode (bool) cannot be changed later, so we have to re-open the + // file. https://github.com/golang/go/blob/go1.20/src/os/file_unix.go#L60 + return fileError(f, f.closed, f.reopen()) +} + +// compile-time check to ensure osFile.reopen implements reopenFile. +var _ reopenFile = (*osFile)(nil).reopen + +func (f *osFile) reopen() (errno experimentalsys.Errno) { + // Clear any create flag, as we are re-opening, not re-creating. + f.flag &= ^experimentalsys.O_CREAT + + var ( + isDir bool + offset int64 + err error + ) + + isDir, errno = f.IsDir() + if errno != 0 { + return errno + } + + if !isDir { + offset, err = f.file.Seek(0, io.SeekCurrent) + if err != nil { + return experimentalsys.UnwrapOSError(err) + } + } + + _ = f.close() + f.file, errno = OpenFile(f.path, f.flag, f.perm) + if errno != 0 { + return errno + } + + if !isDir { + _, err = f.file.Seek(offset, io.SeekStart) + if err != nil { + return experimentalsys.UnwrapOSError(err) + } + } + + return 0 +} + +// IsNonblock implements the same method as documented on fsapi.File +func (f *osFile) IsNonblock() bool { + return isNonblock(f) +} + +// SetNonblock implements the same method as documented on fsapi.File +func (f *osFile) SetNonblock(enable bool) (errno experimentalsys.Errno) { + if enable { + f.flag |= experimentalsys.O_NONBLOCK + } else { + f.flag &= ^experimentalsys.O_NONBLOCK + } + if errno = setNonblock(f.fd, enable); errno != 0 { + return fileError(f, f.closed, errno) + } + return 0 +} + +// Stat implements the same method as documented on sys.File +func (f *osFile) Stat() (sys.Stat_t, experimentalsys.Errno) { + if f.closed { + return sys.Stat_t{}, experimentalsys.EBADF + } + + st, errno := statFile(f.file) + switch errno { + case 0: + f.cachedSt = &cachedStat{dev: st.Dev, ino: st.Ino, isDir: st.Mode&fs.ModeDir == fs.ModeDir} + case experimentalsys.EIO: + errno = experimentalsys.EBADF + } + return st, errno +} + +// Read implements the same method as documented on sys.File +func (f *osFile) Read(buf []byte) (n int, errno experimentalsys.Errno) { + if len(buf) == 0 { + return 0, 0 // Short-circuit 0-len reads. + } + if nonBlockingFileReadSupported && f.IsNonblock() { + n, errno = readFd(f.fd, buf) + } else { + n, errno = read(f.file, buf) + } + if errno != 0 { + // Defer validation overhead until we've already had an error. + errno = fileError(f, f.closed, errno) + } + return +} + +// Pread implements the same method as documented on sys.File +func (f *osFile) Pread(buf []byte, off int64) (n int, errno experimentalsys.Errno) { + if n, errno = pread(f.file, buf, off); errno != 0 { + // Defer validation overhead until we've already had an error. + errno = fileError(f, f.closed, errno) + } + return +} + +// Seek implements the same method as documented on sys.File +func (f *osFile) Seek(offset int64, whence int) (newOffset int64, errno experimentalsys.Errno) { + if newOffset, errno = seek(f.file, offset, whence); errno != 0 { + // Defer validation overhead until we've already had an error. + errno = fileError(f, f.closed, errno) + + // If the error was trying to rewind a directory, re-open it. Notably, + // seeking to zero on a directory doesn't work on Windows with Go 1.19. + if errno == experimentalsys.EISDIR && offset == 0 && whence == io.SeekStart { + errno = 0 + f.reopenDir = true + } + } + return +} + +// Poll implements the same method as documented on fsapi.File +func (f *osFile) Poll(flag fsapi.Pflag, timeoutMillis int32) (ready bool, errno experimentalsys.Errno) { + return poll(f.fd, flag, timeoutMillis) +} + +// Readdir implements File.Readdir. Notably, this uses "Readdir", not +// "ReadDir", from os.File. +func (f *osFile) Readdir(n int) (dirents []experimentalsys.Dirent, errno experimentalsys.Errno) { + if f.reopenDir { // re-open the directory if needed. + f.reopenDir = false + if errno = adjustReaddirErr(f, f.closed, f.reopen()); errno != 0 { + return + } + } + + if dirents, errno = readdir(f.file, f.path, n); errno != 0 { + errno = adjustReaddirErr(f, f.closed, errno) + } + return +} + +// Write implements the same method as documented on sys.File +func (f *osFile) Write(buf []byte) (n int, errno experimentalsys.Errno) { + if len(buf) == 0 { + return 0, 0 // Short-circuit 0-len writes. + } + if nonBlockingFileWriteSupported && f.IsNonblock() { + n, errno = writeFd(f.fd, buf) + } else if n, errno = write(f.file, buf); errno != 0 { + // Defer validation overhead until we've already had an error. + errno = fileError(f, f.closed, errno) + } + return +} + +// Pwrite implements the same method as documented on sys.File +func (f *osFile) Pwrite(buf []byte, off int64) (n int, errno experimentalsys.Errno) { + if n, errno = pwrite(f.file, buf, off); errno != 0 { + // Defer validation overhead until we've already had an error. + errno = fileError(f, f.closed, errno) + } + return +} + +// Truncate implements the same method as documented on sys.File +func (f *osFile) Truncate(size int64) (errno experimentalsys.Errno) { + if errno = experimentalsys.UnwrapOSError(f.file.Truncate(size)); errno != 0 { + // Defer validation overhead until we've already had an error. + errno = fileError(f, f.closed, errno) + } + return +} + +// Sync implements the same method as documented on sys.File +func (f *osFile) Sync() experimentalsys.Errno { + return fsync(f.file) +} + +// Datasync implements the same method as documented on sys.File +func (f *osFile) Datasync() experimentalsys.Errno { + return datasync(f.file) +} + +// Utimens implements the same method as documented on sys.File +func (f *osFile) Utimens(atim, mtim int64) experimentalsys.Errno { + if f.closed { + return experimentalsys.EBADF + } + + err := futimens(f.fd, atim, mtim) + return experimentalsys.UnwrapOSError(err) +} + +// Close implements the same method as documented on sys.File +func (f *osFile) Close() experimentalsys.Errno { + if f.closed { + return 0 + } + f.closed = true + return f.close() +} + +func (f *osFile) close() experimentalsys.Errno { + return experimentalsys.UnwrapOSError(f.file.Close()) +} diff --git a/vendor/github.com/tetratelabs/wazero/internal/sysfs/poll.go b/vendor/github.com/tetratelabs/wazero/internal/sysfs/poll.go new file mode 100644 index 000000000..a2e1103e0 --- /dev/null +++ b/vendor/github.com/tetratelabs/wazero/internal/sysfs/poll.go @@ -0,0 +1,18 @@ +//go:build windows || (linux && !tinygo) || darwin + +package sysfs + +import ( + "github.com/tetratelabs/wazero/experimental/sys" + "github.com/tetratelabs/wazero/internal/fsapi" +) + +// poll implements `Poll` as documented on sys.File via a file descriptor. +func poll(fd uintptr, flag fsapi.Pflag, timeoutMillis int32) (ready bool, errno sys.Errno) { + if flag != fsapi.POLLIN { + return false, sys.ENOTSUP + } + fds := []pollFd{newPollFd(fd, _POLLIN, 0)} + count, errno := _poll(fds, timeoutMillis) + return count > 0, errno +} diff --git a/vendor/github.com/tetratelabs/wazero/internal/sysfs/poll_darwin.go b/vendor/github.com/tetratelabs/wazero/internal/sysfs/poll_darwin.go new file mode 100644 index 000000000..1f7f89093 --- /dev/null +++ b/vendor/github.com/tetratelabs/wazero/internal/sysfs/poll_darwin.go @@ -0,0 +1,55 @@ +package sysfs + +import ( + "unsafe" + + "github.com/tetratelabs/wazero/experimental/sys" +) + +// pollFd is the struct to query for file descriptor events using poll. +type pollFd struct { + // fd is the file descriptor. + fd int32 + // events is a bitmap containing the requested events. + events int16 + // revents is a bitmap containing the returned events. + revents int16 +} + +// newPollFd is a constructor for pollFd that abstracts the platform-specific type of file descriptors. +func newPollFd(fd uintptr, events, revents int16) pollFd { + return pollFd{fd: int32(fd), events: events, revents: revents} +} + +// _POLLIN subscribes a notification when any readable data is available. +const _POLLIN = 0x0001 + +// _poll implements poll on Darwin via the corresponding libc function. +func _poll(fds []pollFd, timeoutMillis int32) (n int, errno sys.Errno) { + var fdptr *pollFd + nfds := len(fds) + if nfds > 0 { + fdptr = &fds[0] + } + n1, _, err := syscall_syscall6( + libc_poll_trampoline_addr, + uintptr(unsafe.Pointer(fdptr)), + uintptr(nfds), + uintptr(int(timeoutMillis)), + uintptr(unsafe.Pointer(nil)), + uintptr(unsafe.Pointer(nil)), + uintptr(unsafe.Pointer(nil))) + return int(n1), sys.UnwrapOSError(err) +} + +// libc_poll_trampoline_addr is the address of the +// `libc_poll_trampoline` symbol, defined in `poll_darwin.s`. +// +// We use this to invoke the syscall through syscall_syscall6 imported below. +var libc_poll_trampoline_addr uintptr + +// Imports the select symbol from libc as `libc_poll`. +// +// Note: CGO mechanisms are used in darwin regardless of the CGO_ENABLED value +// or the "cgo" build flag. See /RATIONALE.md for why. +//go:cgo_import_dynamic libc_poll poll "/usr/lib/libSystem.B.dylib" diff --git a/vendor/github.com/tetratelabs/wazero/internal/sysfs/poll_darwin.s b/vendor/github.com/tetratelabs/wazero/internal/sysfs/poll_darwin.s new file mode 100644 index 000000000..e04fca583 --- /dev/null +++ b/vendor/github.com/tetratelabs/wazero/internal/sysfs/poll_darwin.s @@ -0,0 +1,8 @@ +// lifted from golang.org/x/sys unix +#include "textflag.h" + +TEXT libc_poll_trampoline<>(SB), NOSPLIT, $0-0 + JMP libc_poll(SB) + +GLOBL ·libc_poll_trampoline_addr(SB), RODATA, $8 +DATA ·libc_poll_trampoline_addr(SB)/8, $libc_poll_trampoline<>(SB) diff --git a/vendor/github.com/tetratelabs/wazero/internal/sysfs/poll_linux.go b/vendor/github.com/tetratelabs/wazero/internal/sysfs/poll_linux.go new file mode 100644 index 000000000..49bf4fd06 --- /dev/null +++ b/vendor/github.com/tetratelabs/wazero/internal/sysfs/poll_linux.go @@ -0,0 +1,59 @@ +//go:build !tinygo + +package sysfs + +import ( + "syscall" + "time" + "unsafe" + + "github.com/tetratelabs/wazero/experimental/sys" +) + +// pollFd is the struct to query for file descriptor events using poll. +type pollFd struct { + // fd is the file descriptor. + fd int32 + // events is a bitmap containing the requested events. + events int16 + // revents is a bitmap containing the returned events. + revents int16 +} + +// newPollFd is a constructor for pollFd that abstracts the platform-specific type of file descriptors. +func newPollFd(fd uintptr, events, revents int16) pollFd { + return pollFd{fd: int32(fd), events: events, revents: revents} +} + +// _POLLIN subscribes a notification when any readable data is available. +const _POLLIN = 0x0001 + +// _poll implements poll on Linux via ppoll. +func _poll(fds []pollFd, timeoutMillis int32) (n int, errno sys.Errno) { + var ts syscall.Timespec + if timeoutMillis >= 0 { + ts = syscall.NsecToTimespec(int64(time.Duration(timeoutMillis) * time.Millisecond)) + } + return ppoll(fds, &ts) +} + +// ppoll is a poll variant that allows to subscribe to a mask of signals. +// However, we do not need such mask, so the corresponding argument is always nil. +func ppoll(fds []pollFd, timespec *syscall.Timespec) (n int, err sys.Errno) { + var fdptr *pollFd + nfd := len(fds) + if nfd != 0 { + fdptr = &fds[0] + } + + n1, _, errno := syscall.Syscall6( + uintptr(syscall.SYS_PPOLL), + uintptr(unsafe.Pointer(fdptr)), + uintptr(nfd), + uintptr(unsafe.Pointer(timespec)), + uintptr(unsafe.Pointer(nil)), // sigmask is currently always ignored + uintptr(unsafe.Pointer(nil)), + uintptr(unsafe.Pointer(nil))) + + return int(n1), sys.UnwrapOSError(errno) +} diff --git a/vendor/github.com/tetratelabs/wazero/internal/sysfs/poll_unsupported.go b/vendor/github.com/tetratelabs/wazero/internal/sysfs/poll_unsupported.go new file mode 100644 index 000000000..2301a067e --- /dev/null +++ b/vendor/github.com/tetratelabs/wazero/internal/sysfs/poll_unsupported.go @@ -0,0 +1,13 @@ +//go:build !(linux || darwin || windows) || tinygo + +package sysfs + +import ( + "github.com/tetratelabs/wazero/experimental/sys" + "github.com/tetratelabs/wazero/internal/fsapi" +) + +// poll implements `Poll` as documented on fsapi.File via a file descriptor. +func poll(uintptr, fsapi.Pflag, int32) (bool, sys.Errno) { + return false, sys.ENOSYS +} diff --git a/vendor/github.com/tetratelabs/wazero/internal/sysfs/poll_windows.go b/vendor/github.com/tetratelabs/wazero/internal/sysfs/poll_windows.go new file mode 100644 index 000000000..82c8b2baf --- /dev/null +++ b/vendor/github.com/tetratelabs/wazero/internal/sysfs/poll_windows.go @@ -0,0 +1,224 @@ +package sysfs + +import ( + "syscall" + "time" + "unsafe" + + "github.com/tetratelabs/wazero/experimental/sys" +) + +var ( + procWSAPoll = modws2_32.NewProc("WSAPoll") + procGetNamedPipeInfo = kernel32.NewProc("GetNamedPipeInfo") +) + +const ( + // _POLLRDNORM subscribes to normal data for read. + _POLLRDNORM = 0x0100 + // _POLLRDBAND subscribes to priority band (out-of-band) data for read. + _POLLRDBAND = 0x0200 + // _POLLIN subscribes a notification when any readable data is available. + _POLLIN = (_POLLRDNORM | _POLLRDBAND) +) + +// pollFd is the struct to query for file descriptor events using poll. +type pollFd struct { + // fd is the file descriptor. + fd uintptr + // events is a bitmap containing the requested events. + events int16 + // revents is a bitmap containing the returned events. + revents int16 +} + +// newPollFd is a constructor for pollFd that abstracts the platform-specific type of file descriptors. +func newPollFd(fd uintptr, events, revents int16) pollFd { + return pollFd{fd: fd, events: events, revents: revents} +} + +// pollInterval is the interval between each calls to peekNamedPipe in selectAllHandles +const pollInterval = 100 * time.Millisecond + +// _poll implements poll on Windows, for a subset of cases. +// +// fds may contain any number of file handles, but regular files and pipes are only processed for _POLLIN. +// Stdin is a pipe, thus it is checked for readiness when present. Pipes are checked using PeekNamedPipe. +// Regular files always immediately reported as ready, regardless their actual state and timeouts. +// +// If n==0 it will wait for the given timeout duration, but it will return sys.ENOSYS if timeout is nil, +// i.e. it won't block indefinitely. The given ctx is used to allow for cancellation, +// and it is currently used only in tests. +// +// The implementation actually polls every 100 milliseconds (pollInterval) until it reaches the +// given timeout (in millis). +// +// The duration may be negative, in which case it will wait indefinitely. The given ctx is +// used to allow for cancellation, and it is currently used only in tests. +func _poll(fds []pollFd, timeoutMillis int32) (n int, errno sys.Errno) { + if fds == nil { + return -1, sys.ENOSYS + } + + regular, pipes, sockets, errno := partionByFtype(fds) + nregular := len(regular) + if errno != 0 { + return -1, errno + } + + // Ticker that emits at every pollInterval. + tick := time.NewTicker(pollInterval) + tickCh := tick.C + defer tick.Stop() + + // Timer that expires after the given duration. + // Initialize afterCh as nil: the select below will wait forever. + var afterCh <-chan time.Time + if timeoutMillis >= 0 { + // If duration is not nil, instantiate the timer. + after := time.NewTimer(time.Duration(timeoutMillis) * time.Millisecond) + defer after.Stop() + afterCh = after.C + } + + npipes, nsockets, errno := peekAll(pipes, sockets) + if errno != 0 { + return -1, errno + } + count := nregular + npipes + nsockets + if count > 0 { + return count, 0 + } + + for { + select { + case <-afterCh: + return 0, 0 + case <-tickCh: + npipes, nsockets, errno := peekAll(pipes, sockets) + if errno != 0 { + return -1, errno + } + count = nregular + npipes + nsockets + if count > 0 { + return count, 0 + } + } + } +} + +func peekAll(pipes, sockets []pollFd) (npipes, nsockets int, errno sys.Errno) { + npipes, errno = peekPipes(pipes) + if errno != 0 { + return + } + + // Invoke wsaPoll with a 0-timeout to avoid blocking. + // Timeouts are handled in pollWithContext instead. + nsockets, errno = wsaPoll(sockets, 0) + if errno != 0 { + return + } + + count := npipes + nsockets + if count > 0 { + return + } + + return +} + +func peekPipes(fds []pollFd) (n int, errno sys.Errno) { + for _, fd := range fds { + bytes, errno := peekNamedPipe(syscall.Handle(fd.fd)) + if errno != 0 { + return -1, sys.UnwrapOSError(errno) + } + if bytes > 0 { + n++ + } + } + return +} + +// wsaPoll is the WSAPoll function from winsock2. +// +// See https://learn.microsoft.com/en-us/windows/win32/api/winsock2/nf-winsock2-wsapoll +func wsaPoll(fds []pollFd, timeout int) (n int, errno sys.Errno) { + if len(fds) > 0 { + sockptr := &fds[0] + ns, _, e := syscall.SyscallN( + procWSAPoll.Addr(), + uintptr(unsafe.Pointer(sockptr)), + uintptr(len(fds)), + uintptr(timeout)) + if e != 0 { + return -1, sys.UnwrapOSError(e) + } + n = int(ns) + } + return +} + +// ftype is a type of file that can be handled by poll. +type ftype uint8 + +const ( + ftype_regular ftype = iota + ftype_pipe + ftype_socket +) + +// partionByFtype checks the type of each fd in fds and returns 3 distinct partitions +// for regular files, named pipes and sockets. +func partionByFtype(fds []pollFd) (regular, pipe, socket []pollFd, errno sys.Errno) { + for _, pfd := range fds { + t, errno := ftypeOf(pfd.fd) + if errno != 0 { + return nil, nil, nil, errno + } + switch t { + case ftype_regular: + regular = append(regular, pfd) + case ftype_pipe: + pipe = append(pipe, pfd) + case ftype_socket: + socket = append(socket, pfd) + } + } + return +} + +// ftypeOf checks the type of fd and return the corresponding ftype. +func ftypeOf(fd uintptr) (ftype, sys.Errno) { + h := syscall.Handle(fd) + t, err := syscall.GetFileType(h) + if err != nil { + return 0, sys.UnwrapOSError(err) + } + switch t { + case syscall.FILE_TYPE_CHAR, syscall.FILE_TYPE_DISK: + return ftype_regular, 0 + case syscall.FILE_TYPE_PIPE: + if isSocket(h) { + return ftype_socket, 0 + } else { + return ftype_pipe, 0 + } + default: + return ftype_regular, 0 + } +} + +// isSocket returns true if the given file handle +// is a pipe. +func isSocket(fd syscall.Handle) bool { + r, _, errno := syscall.SyscallN( + procGetNamedPipeInfo.Addr(), + uintptr(fd), + uintptr(unsafe.Pointer(nil)), + uintptr(unsafe.Pointer(nil)), + uintptr(unsafe.Pointer(nil)), + uintptr(unsafe.Pointer(nil))) + return r == 0 || errno != 0 +} diff --git a/vendor/github.com/tetratelabs/wazero/internal/sysfs/readfs.go b/vendor/github.com/tetratelabs/wazero/internal/sysfs/readfs.go new file mode 100644 index 000000000..59e331a29 --- /dev/null +++ b/vendor/github.com/tetratelabs/wazero/internal/sysfs/readfs.go @@ -0,0 +1,117 @@ +package sysfs + +import ( + "io/fs" + + experimentalsys "github.com/tetratelabs/wazero/experimental/sys" +) + +type ReadFS struct { + experimentalsys.FS +} + +// OpenFile implements the same method as documented on sys.FS +func (r *ReadFS) OpenFile(path string, flag experimentalsys.Oflag, perm fs.FileMode) (experimentalsys.File, experimentalsys.Errno) { + // Mask the mutually exclusive bits as they determine write mode. + switch flag & (experimentalsys.O_RDONLY | experimentalsys.O_WRONLY | experimentalsys.O_RDWR) { + case experimentalsys.O_WRONLY, experimentalsys.O_RDWR: + // Return the correct error if a directory was opened for write. + if flag&experimentalsys.O_DIRECTORY != 0 { + return nil, experimentalsys.EISDIR + } + return nil, experimentalsys.ENOSYS + default: // sys.O_RDONLY (integer zero) so we are ok! + } + + f, errno := r.FS.OpenFile(path, flag, perm) + if errno != 0 { + return nil, errno + } + return &readFile{f}, 0 +} + +// Mkdir implements the same method as documented on sys.FS +func (r *ReadFS) Mkdir(path string, perm fs.FileMode) experimentalsys.Errno { + return experimentalsys.EROFS +} + +// Chmod implements the same method as documented on sys.FS +func (r *ReadFS) Chmod(path string, perm fs.FileMode) experimentalsys.Errno { + return experimentalsys.EROFS +} + +// Rename implements the same method as documented on sys.FS +func (r *ReadFS) Rename(from, to string) experimentalsys.Errno { + return experimentalsys.EROFS +} + +// Rmdir implements the same method as documented on sys.FS +func (r *ReadFS) Rmdir(path string) experimentalsys.Errno { + return experimentalsys.EROFS +} + +// Link implements the same method as documented on sys.FS +func (r *ReadFS) Link(_, _ string) experimentalsys.Errno { + return experimentalsys.EROFS +} + +// Symlink implements the same method as documented on sys.FS +func (r *ReadFS) Symlink(_, _ string) experimentalsys.Errno { + return experimentalsys.EROFS +} + +// Unlink implements the same method as documented on sys.FS +func (r *ReadFS) Unlink(path string) experimentalsys.Errno { + return experimentalsys.EROFS +} + +// Utimens implements the same method as documented on sys.FS +func (r *ReadFS) Utimens(path string, atim, mtim int64) experimentalsys.Errno { + return experimentalsys.EROFS +} + +// compile-time check to ensure readFile implements api.File. +var _ experimentalsys.File = (*readFile)(nil) + +type readFile struct { + experimentalsys.File +} + +// Write implements the same method as documented on sys.File. +func (r *readFile) Write([]byte) (int, experimentalsys.Errno) { + return 0, r.writeErr() +} + +// Pwrite implements the same method as documented on sys.File. +func (r *readFile) Pwrite([]byte, int64) (n int, errno experimentalsys.Errno) { + return 0, r.writeErr() +} + +// Truncate implements the same method as documented on sys.File. +func (r *readFile) Truncate(int64) experimentalsys.Errno { + return r.writeErr() +} + +// Sync implements the same method as documented on sys.File. +func (r *readFile) Sync() experimentalsys.Errno { + return experimentalsys.EBADF +} + +// Datasync implements the same method as documented on sys.File. +func (r *readFile) Datasync() experimentalsys.Errno { + return experimentalsys.EBADF +} + +// Utimens implements the same method as documented on sys.File. +func (r *readFile) Utimens(int64, int64) experimentalsys.Errno { + return experimentalsys.EBADF +} + +func (r *readFile) writeErr() experimentalsys.Errno { + if isDir, errno := r.IsDir(); errno != 0 { + return errno + } else if isDir { + return experimentalsys.EISDIR + } + return experimentalsys.EBADF +} diff --git a/vendor/github.com/tetratelabs/wazero/internal/sysfs/rename.go b/vendor/github.com/tetratelabs/wazero/internal/sysfs/rename.go new file mode 100644 index 000000000..37c53571d --- /dev/null +++ b/vendor/github.com/tetratelabs/wazero/internal/sysfs/rename.go @@ -0,0 +1,16 @@ +//go:build !windows && !plan9 && !tinygo + +package sysfs + +import ( + "syscall" + + "github.com/tetratelabs/wazero/experimental/sys" +) + +func rename(from, to string) sys.Errno { + if from == to { + return 0 + } + return sys.UnwrapOSError(syscall.Rename(from, to)) +} diff --git a/vendor/github.com/tetratelabs/wazero/internal/sysfs/rename_plan9.go b/vendor/github.com/tetratelabs/wazero/internal/sysfs/rename_plan9.go new file mode 100644 index 000000000..474cc7595 --- /dev/null +++ b/vendor/github.com/tetratelabs/wazero/internal/sysfs/rename_plan9.go @@ -0,0 +1,14 @@ +package sysfs + +import ( + "os" + + "github.com/tetratelabs/wazero/experimental/sys" +) + +func rename(from, to string) sys.Errno { + if from == to { + return 0 + } + return sys.UnwrapOSError(os.Rename(from, to)) +} diff --git a/vendor/github.com/tetratelabs/wazero/internal/sysfs/rename_windows.go b/vendor/github.com/tetratelabs/wazero/internal/sysfs/rename_windows.go new file mode 100644 index 000000000..5e8102239 --- /dev/null +++ b/vendor/github.com/tetratelabs/wazero/internal/sysfs/rename_windows.go @@ -0,0 +1,55 @@ +package sysfs + +import ( + "os" + "syscall" + + "github.com/tetratelabs/wazero/experimental/sys" +) + +func rename(from, to string) sys.Errno { + if from == to { + return 0 + } + + var fromIsDir, toIsDir bool + if fromStat, errno := stat(from); errno != 0 { + return errno // failed to stat from + } else { + fromIsDir = fromStat.Mode.IsDir() + } + if toStat, errno := stat(to); errno == sys.ENOENT { + return syscallRename(from, to) // file or dir to not-exist is ok + } else if errno != 0 { + return errno // failed to stat to + } else { + toIsDir = toStat.Mode.IsDir() + } + + // Now, handle known cases + switch { + case !fromIsDir && toIsDir: // file to dir + return sys.EISDIR + case !fromIsDir && !toIsDir: // file to file + // Use os.Rename instead of syscall.Rename to overwrite a file. + // This uses MoveFileEx instead of MoveFile (used by syscall.Rename). + return sys.UnwrapOSError(os.Rename(from, to)) + case fromIsDir && !toIsDir: // dir to file + return sys.ENOTDIR + default: // dir to dir + + // We can't tell if a directory is empty or not, via stat information. + // Reading the directory is expensive, as it can buffer large amounts + // of data on fail. Instead, speculatively try to remove the directory. + // This is only one syscall and won't buffer anything. + if errno := rmdir(to); errno == 0 || errno == sys.ENOENT { + return syscallRename(from, to) + } else { + return errno + } + } +} + +func syscallRename(from string, to string) sys.Errno { + return sys.UnwrapOSError(syscall.Rename(from, to)) +} diff --git a/vendor/github.com/tetratelabs/wazero/internal/sysfs/sock.go b/vendor/github.com/tetratelabs/wazero/internal/sysfs/sock.go new file mode 100644 index 000000000..ab9bb1ffa --- /dev/null +++ b/vendor/github.com/tetratelabs/wazero/internal/sysfs/sock.go @@ -0,0 +1,187 @@ +package sysfs + +import ( + "net" + "os" + + experimentalsys "github.com/tetratelabs/wazero/experimental/sys" + "github.com/tetratelabs/wazero/internal/fsapi" + socketapi "github.com/tetratelabs/wazero/internal/sock" + "github.com/tetratelabs/wazero/sys" +) + +// NewTCPListenerFile creates a socketapi.TCPSock for a given *net.TCPListener. +func NewTCPListenerFile(tl *net.TCPListener) socketapi.TCPSock { + return newTCPListenerFile(tl) +} + +// baseSockFile implements base behavior for all TCPSock, TCPConn files, +// regardless the platform. +type baseSockFile struct { + experimentalsys.UnimplementedFile +} + +var _ experimentalsys.File = (*baseSockFile)(nil) + +// IsDir implements the same method as documented on File.IsDir +func (*baseSockFile) IsDir() (bool, experimentalsys.Errno) { + // We need to override this method because WASI-libc prestats the FD + // and the default impl returns ENOSYS otherwise. + return false, 0 +} + +// Stat implements the same method as documented on File.Stat +func (f *baseSockFile) Stat() (fs sys.Stat_t, errno experimentalsys.Errno) { + // The mode is not really important, but it should be neither a regular file nor a directory. + fs.Mode = os.ModeIrregular + return +} + +var _ socketapi.TCPSock = (*tcpListenerFile)(nil) + +type tcpListenerFile struct { + baseSockFile + + tl *net.TCPListener + closed bool + nonblock bool +} + +// newTCPListenerFile is a constructor for a socketapi.TCPSock. +// +// The current strategy is to wrap a net.TCPListener +// and invoking raw syscalls using syscallConnControl: +// this internal calls RawConn.Control(func(fd)), making sure +// that the underlying file descriptor is valid throughout +// the duration of the syscall. +func newDefaultTCPListenerFile(tl *net.TCPListener) socketapi.TCPSock { + return &tcpListenerFile{tl: tl} +} + +// Close implements the same method as documented on experimentalsys.File +func (f *tcpListenerFile) Close() experimentalsys.Errno { + if !f.closed { + return experimentalsys.UnwrapOSError(f.tl.Close()) + } + return 0 +} + +// Addr is exposed for testing. +func (f *tcpListenerFile) Addr() *net.TCPAddr { + return f.tl.Addr().(*net.TCPAddr) +} + +// IsNonblock implements the same method as documented on fsapi.File +func (f *tcpListenerFile) IsNonblock() bool { + return f.nonblock +} + +// Poll implements the same method as documented on fsapi.File +func (f *tcpListenerFile) Poll(flag fsapi.Pflag, timeoutMillis int32) (ready bool, errno experimentalsys.Errno) { + return false, experimentalsys.ENOSYS +} + +var _ socketapi.TCPConn = (*tcpConnFile)(nil) + +type tcpConnFile struct { + baseSockFile + + tc *net.TCPConn + + // nonblock is true when the underlying connection is flagged as non-blocking. + // This ensures that reads and writes return experimentalsys.EAGAIN without blocking the caller. + nonblock bool + // closed is true when closed was called. This ensures proper experimentalsys.EBADF + closed bool +} + +func newTcpConn(tc *net.TCPConn) socketapi.TCPConn { + return &tcpConnFile{tc: tc} +} + +// Read implements the same method as documented on experimentalsys.File +func (f *tcpConnFile) Read(buf []byte) (n int, errno experimentalsys.Errno) { + if len(buf) == 0 { + return 0, 0 // Short-circuit 0-len reads. + } + if nonBlockingFileReadSupported && f.IsNonblock() { + n, errno = syscallConnControl(f.tc, func(fd uintptr) (int, experimentalsys.Errno) { + n, err := readSocket(fd, buf) + errno = experimentalsys.UnwrapOSError(err) + errno = fileError(f, f.closed, errno) + return n, errno + }) + } else { + n, errno = read(f.tc, buf) + } + if errno != 0 { + // Defer validation overhead until we've already had an error. + errno = fileError(f, f.closed, errno) + } + return +} + +// Write implements the same method as documented on experimentalsys.File +func (f *tcpConnFile) Write(buf []byte) (n int, errno experimentalsys.Errno) { + if nonBlockingFileWriteSupported && f.IsNonblock() { + return syscallConnControl(f.tc, func(fd uintptr) (int, experimentalsys.Errno) { + n, err := writeSocket(fd, buf) + errno = experimentalsys.UnwrapOSError(err) + errno = fileError(f, f.closed, errno) + return n, errno + }) + } else { + n, errno = write(f.tc, buf) + } + if errno != 0 { + // Defer validation overhead until we've already had an error. + errno = fileError(f, f.closed, errno) + } + return +} + +// Recvfrom implements the same method as documented on socketapi.TCPConn +func (f *tcpConnFile) Recvfrom(p []byte, flags int) (n int, errno experimentalsys.Errno) { + if flags != MSG_PEEK { + errno = experimentalsys.EINVAL + return + } + return syscallConnControl(f.tc, func(fd uintptr) (int, experimentalsys.Errno) { + n, err := recvfrom(fd, p, MSG_PEEK) + errno = experimentalsys.UnwrapOSError(err) + errno = fileError(f, f.closed, errno) + return n, errno + }) +} + +// Close implements the same method as documented on experimentalsys.File +func (f *tcpConnFile) Close() experimentalsys.Errno { + return f.close() +} + +func (f *tcpConnFile) close() experimentalsys.Errno { + if f.closed { + return 0 + } + f.closed = true + return f.Shutdown(socketapi.SHUT_RDWR) +} + +// SetNonblock implements the same method as documented on fsapi.File +func (f *tcpConnFile) SetNonblock(enabled bool) (errno experimentalsys.Errno) { + f.nonblock = enabled + _, errno = syscallConnControl(f.tc, func(fd uintptr) (int, experimentalsys.Errno) { + return 0, experimentalsys.UnwrapOSError(setNonblockSocket(fd, enabled)) + }) + return +} + +// IsNonblock implements the same method as documented on fsapi.File +func (f *tcpConnFile) IsNonblock() bool { + return f.nonblock +} + +// Poll implements the same method as documented on fsapi.File +func (f *tcpConnFile) Poll(flag fsapi.Pflag, timeoutMillis int32) (ready bool, errno experimentalsys.Errno) { + return false, experimentalsys.ENOSYS +} diff --git a/vendor/github.com/tetratelabs/wazero/internal/sysfs/sock_supported.go b/vendor/github.com/tetratelabs/wazero/internal/sysfs/sock_supported.go new file mode 100644 index 000000000..6c976fb86 --- /dev/null +++ b/vendor/github.com/tetratelabs/wazero/internal/sysfs/sock_supported.go @@ -0,0 +1,77 @@ +//go:build (linux || darwin || windows) && !tinygo + +package sysfs + +import ( + "net" + "syscall" + + experimentalsys "github.com/tetratelabs/wazero/experimental/sys" + "github.com/tetratelabs/wazero/internal/fsapi" + socketapi "github.com/tetratelabs/wazero/internal/sock" +) + +// Accept implements the same method as documented on socketapi.TCPSock +func (f *tcpListenerFile) Accept() (socketapi.TCPConn, experimentalsys.Errno) { + // Ensure we have an incoming connection, otherwise return immediately. + if f.nonblock { + if ready, errno := _pollSock(f.tl, fsapi.POLLIN, 0); !ready || errno != 0 { + return nil, experimentalsys.EAGAIN + } + } + + // Accept normally blocks goroutines, but we + // made sure that we have an incoming connection, + // so we should be safe. + if conn, err := f.tl.Accept(); err != nil { + return nil, experimentalsys.UnwrapOSError(err) + } else { + return newTcpConn(conn.(*net.TCPConn)), 0 + } +} + +// SetNonblock implements the same method as documented on fsapi.File +func (f *tcpListenerFile) SetNonblock(enabled bool) (errno experimentalsys.Errno) { + f.nonblock = enabled + _, errno = syscallConnControl(f.tl, func(fd uintptr) (int, experimentalsys.Errno) { + return 0, setNonblockSocket(fd, enabled) + }) + return +} + +// Shutdown implements the same method as documented on experimentalsys.Conn +func (f *tcpConnFile) Shutdown(how int) experimentalsys.Errno { + // FIXME: can userland shutdown listeners? + var err error + switch how { + case socketapi.SHUT_RD: + err = f.tc.CloseRead() + case socketapi.SHUT_WR: + err = f.tc.CloseWrite() + case socketapi.SHUT_RDWR: + return f.close() + default: + return experimentalsys.EINVAL + } + return experimentalsys.UnwrapOSError(err) +} + +// syscallConnControl extracts a syscall.RawConn from the given syscall.Conn and applies +// the given fn to a file descriptor, returning an integer or a nonzero syscall.Errno on failure. +// +// syscallConnControl streamlines the pattern of extracting the syscall.Rawconn, +// invoking its syscall.RawConn.Control method, then handling properly the errors that may occur +// within fn or returned by syscall.RawConn.Control itself. +func syscallConnControl(conn syscall.Conn, fn func(fd uintptr) (int, experimentalsys.Errno)) (n int, errno experimentalsys.Errno) { + syscallConn, err := conn.SyscallConn() + if err != nil { + return 0, experimentalsys.UnwrapOSError(err) + } + // Prioritize the inner errno over Control + if controlErr := syscallConn.Control(func(fd uintptr) { + n, errno = fn(fd) + }); errno == 0 { + errno = experimentalsys.UnwrapOSError(controlErr) + } + return +} diff --git a/vendor/github.com/tetratelabs/wazero/internal/sysfs/sock_unix.go b/vendor/github.com/tetratelabs/wazero/internal/sysfs/sock_unix.go new file mode 100644 index 000000000..99ef018a4 --- /dev/null +++ b/vendor/github.com/tetratelabs/wazero/internal/sysfs/sock_unix.go @@ -0,0 +1,49 @@ +//go:build (linux || darwin) && !tinygo + +package sysfs + +import ( + "net" + "syscall" + + "github.com/tetratelabs/wazero/experimental/sys" + "github.com/tetratelabs/wazero/internal/fsapi" + socketapi "github.com/tetratelabs/wazero/internal/sock" +) + +// MSG_PEEK is the constant syscall.MSG_PEEK +const MSG_PEEK = syscall.MSG_PEEK + +func newTCPListenerFile(tl *net.TCPListener) socketapi.TCPSock { + return newDefaultTCPListenerFile(tl) +} + +func _pollSock(conn syscall.Conn, flag fsapi.Pflag, timeoutMillis int32) (bool, sys.Errno) { + n, errno := syscallConnControl(conn, func(fd uintptr) (int, sys.Errno) { + if ready, errno := poll(fd, fsapi.POLLIN, 0); !ready || errno != 0 { + return -1, errno + } else { + return 0, errno + } + }) + return n >= 0, errno +} + +func setNonblockSocket(fd uintptr, enabled bool) sys.Errno { + return sys.UnwrapOSError(setNonblock(fd, enabled)) +} + +func readSocket(fd uintptr, buf []byte) (int, sys.Errno) { + n, err := syscall.Read(int(fd), buf) + return n, sys.UnwrapOSError(err) +} + +func writeSocket(fd uintptr, buf []byte) (int, sys.Errno) { + n, err := syscall.Write(int(fd), buf) + return n, sys.UnwrapOSError(err) +} + +func recvfrom(fd uintptr, buf []byte, flags int32) (n int, errno sys.Errno) { + n, _, err := syscall.Recvfrom(int(fd), buf, int(flags)) + return n, sys.UnwrapOSError(err) +} diff --git a/vendor/github.com/tetratelabs/wazero/internal/sysfs/sock_unsupported.go b/vendor/github.com/tetratelabs/wazero/internal/sysfs/sock_unsupported.go new file mode 100644 index 000000000..8c27fed7f --- /dev/null +++ b/vendor/github.com/tetratelabs/wazero/internal/sysfs/sock_unsupported.go @@ -0,0 +1,81 @@ +//go:build (!linux && !darwin && !windows) || tinygo + +package sysfs + +import ( + "net" + "syscall" + + "github.com/tetratelabs/wazero/experimental/sys" + experimentalsys "github.com/tetratelabs/wazero/experimental/sys" + "github.com/tetratelabs/wazero/internal/fsapi" + socketapi "github.com/tetratelabs/wazero/internal/sock" +) + +// MSG_PEEK is a filler value. +const MSG_PEEK = 0x2 + +func newTCPListenerFile(tl *net.TCPListener) socketapi.TCPSock { + return &unsupportedSockFile{} +} + +type unsupportedSockFile struct { + baseSockFile +} + +// Accept implements the same method as documented on socketapi.TCPSock +func (f *unsupportedSockFile) Accept() (socketapi.TCPConn, sys.Errno) { + return nil, sys.ENOSYS +} + +func _pollSock(conn syscall.Conn, flag fsapi.Pflag, timeoutMillis int32) (bool, sys.Errno) { + return false, sys.ENOTSUP +} + +func setNonblockSocket(fd uintptr, enabled bool) sys.Errno { + return sys.ENOTSUP +} + +func readSocket(fd uintptr, buf []byte) (int, sys.Errno) { + return -1, sys.ENOTSUP +} + +func writeSocket(fd uintptr, buf []byte) (int, sys.Errno) { + return -1, sys.ENOTSUP +} + +func recvfrom(fd uintptr, buf []byte, flags int32) (n int, errno sys.Errno) { + return -1, sys.ENOTSUP +} + +// syscallConnControl extracts a syscall.RawConn from the given syscall.Conn and applies +// the given fn to a file descriptor, returning an integer or a nonzero syscall.Errno on failure. +// +// syscallConnControl streamlines the pattern of extracting the syscall.Rawconn, +// invoking its syscall.RawConn.Control method, then handling properly the errors that may occur +// within fn or returned by syscall.RawConn.Control itself. +func syscallConnControl(conn syscall.Conn, fn func(fd uintptr) (int, experimentalsys.Errno)) (n int, errno sys.Errno) { + return -1, sys.ENOTSUP +} + +// Accept implements the same method as documented on socketapi.TCPSock +func (f *tcpListenerFile) Accept() (socketapi.TCPConn, experimentalsys.Errno) { + return nil, experimentalsys.ENOSYS +} + +// Shutdown implements the same method as documented on experimentalsys.Conn +func (f *tcpConnFile) Shutdown(how int) experimentalsys.Errno { + // FIXME: can userland shutdown listeners? + var err error + switch how { + case socketapi.SHUT_RD: + err = f.tc.Close() + case socketapi.SHUT_WR: + err = f.tc.Close() + case socketapi.SHUT_RDWR: + return f.close() + default: + return experimentalsys.EINVAL + } + return experimentalsys.UnwrapOSError(err) +} diff --git a/vendor/github.com/tetratelabs/wazero/internal/sysfs/sock_windows.go b/vendor/github.com/tetratelabs/wazero/internal/sysfs/sock_windows.go new file mode 100644 index 000000000..703df42fc --- /dev/null +++ b/vendor/github.com/tetratelabs/wazero/internal/sysfs/sock_windows.go @@ -0,0 +1,80 @@ +//go:build windows + +package sysfs + +import ( + "net" + "syscall" + "unsafe" + + "github.com/tetratelabs/wazero/experimental/sys" + "github.com/tetratelabs/wazero/internal/fsapi" + socketapi "github.com/tetratelabs/wazero/internal/sock" +) + +const ( + // MSG_PEEK is the flag PEEK for syscall.Recvfrom on Windows. + // This constant is not exported on this platform. + MSG_PEEK = 0x2 + // _FIONBIO is the flag to set the O_NONBLOCK flag on socket handles using ioctlsocket. + _FIONBIO = 0x8004667e +) + +var ( + // modws2_32 is WinSock. + modws2_32 = syscall.NewLazyDLL("ws2_32.dll") + // procrecvfrom exposes recvfrom from WinSock. + procrecvfrom = modws2_32.NewProc("recvfrom") + // procioctlsocket exposes ioctlsocket from WinSock. + procioctlsocket = modws2_32.NewProc("ioctlsocket") +) + +func newTCPListenerFile(tl *net.TCPListener) socketapi.TCPSock { + return newDefaultTCPListenerFile(tl) +} + +// recvfrom exposes the underlying syscall in Windows. +// +// Note: since we are only using this to expose MSG_PEEK, +// we do not need really need all the parameters that are actually +// allowed in WinSock. +// We ignore `from *sockaddr` and `fromlen *int`. +func recvfrom(s uintptr, buf []byte, flags int32) (n int, errno sys.Errno) { + var _p0 *byte + if len(buf) > 0 { + _p0 = &buf[0] + } + r0, _, e1 := syscall.SyscallN( + procrecvfrom.Addr(), + s, + uintptr(unsafe.Pointer(_p0)), + uintptr(len(buf)), + uintptr(flags), + 0, // from *sockaddr (optional) + 0) // fromlen *int (optional) + return int(r0), sys.UnwrapOSError(e1) +} + +func setNonblockSocket(fd uintptr, enabled bool) sys.Errno { + opt := uint64(0) + if enabled { + opt = 1 + } + // ioctlsocket(fd, FIONBIO, &opt) + _, _, errno := syscall.SyscallN( + procioctlsocket.Addr(), + uintptr(fd), + uintptr(_FIONBIO), + uintptr(unsafe.Pointer(&opt))) + return sys.UnwrapOSError(errno) +} + +func _pollSock(conn syscall.Conn, flag fsapi.Pflag, timeoutMillis int32) (bool, sys.Errno) { + if flag != fsapi.POLLIN { + return false, sys.ENOTSUP + } + n, errno := syscallConnControl(conn, func(fd uintptr) (int, sys.Errno) { + return _poll([]pollFd{newPollFd(fd, _POLLIN, 0)}, timeoutMillis) + }) + return n > 0, errno +} diff --git a/vendor/github.com/tetratelabs/wazero/internal/sysfs/stat.go b/vendor/github.com/tetratelabs/wazero/internal/sysfs/stat.go new file mode 100644 index 000000000..2d973b16c --- /dev/null +++ b/vendor/github.com/tetratelabs/wazero/internal/sysfs/stat.go @@ -0,0 +1,16 @@ +package sysfs + +import ( + "io/fs" + + experimentalsys "github.com/tetratelabs/wazero/experimental/sys" + "github.com/tetratelabs/wazero/sys" +) + +func defaultStatFile(f fs.File) (sys.Stat_t, experimentalsys.Errno) { + if info, err := f.Stat(); err != nil { + return sys.Stat_t{}, experimentalsys.UnwrapOSError(err) + } else { + return sys.NewStat_t(info), 0 + } +} diff --git a/vendor/github.com/tetratelabs/wazero/internal/sysfs/stat_bsd.go b/vendor/github.com/tetratelabs/wazero/internal/sysfs/stat_bsd.go new file mode 100644 index 000000000..254e204cd --- /dev/null +++ b/vendor/github.com/tetratelabs/wazero/internal/sysfs/stat_bsd.go @@ -0,0 +1,37 @@ +//go:build (amd64 || arm64) && (darwin || freebsd) + +package sysfs + +import ( + "io/fs" + "os" + + experimentalsys "github.com/tetratelabs/wazero/experimental/sys" + "github.com/tetratelabs/wazero/sys" +) + +// dirNlinkIncludesDot is true because even though os.File filters out dot +// entries, the underlying syscall.Stat includes them. +// +// Note: this is only used in tests +const dirNlinkIncludesDot = true + +func lstat(path string) (sys.Stat_t, experimentalsys.Errno) { + if info, err := os.Lstat(path); err != nil { + return sys.Stat_t{}, experimentalsys.UnwrapOSError(err) + } else { + return sys.NewStat_t(info), 0 + } +} + +func stat(path string) (sys.Stat_t, experimentalsys.Errno) { + if info, err := os.Stat(path); err != nil { + return sys.Stat_t{}, experimentalsys.UnwrapOSError(err) + } else { + return sys.NewStat_t(info), 0 + } +} + +func statFile(f fs.File) (sys.Stat_t, experimentalsys.Errno) { + return defaultStatFile(f) +} diff --git a/vendor/github.com/tetratelabs/wazero/internal/sysfs/stat_linux.go b/vendor/github.com/tetratelabs/wazero/internal/sysfs/stat_linux.go new file mode 100644 index 000000000..fd289756d --- /dev/null +++ b/vendor/github.com/tetratelabs/wazero/internal/sysfs/stat_linux.go @@ -0,0 +1,40 @@ +//go:build (amd64 || arm64 || riscv64) && linux + +// Note: This expression is not the same as compiler support, even if it looks +// similar. Platform functions here are used in interpreter mode as well. + +package sysfs + +import ( + "io/fs" + "os" + + experimentalsys "github.com/tetratelabs/wazero/experimental/sys" + "github.com/tetratelabs/wazero/sys" +) + +// dirNlinkIncludesDot is true because even though os.File filters out dot +// entries, the underlying syscall.Stat includes them. +// +// Note: this is only used in tests +const dirNlinkIncludesDot = true + +func lstat(path string) (sys.Stat_t, experimentalsys.Errno) { + if info, err := os.Lstat(path); err != nil { + return sys.Stat_t{}, experimentalsys.UnwrapOSError(err) + } else { + return sys.NewStat_t(info), 0 + } +} + +func stat(path string) (sys.Stat_t, experimentalsys.Errno) { + if info, err := os.Stat(path); err != nil { + return sys.Stat_t{}, experimentalsys.UnwrapOSError(err) + } else { + return sys.NewStat_t(info), 0 + } +} + +func statFile(f fs.File) (sys.Stat_t, experimentalsys.Errno) { + return defaultStatFile(f) +} diff --git a/vendor/github.com/tetratelabs/wazero/internal/sysfs/stat_unsupported.go b/vendor/github.com/tetratelabs/wazero/internal/sysfs/stat_unsupported.go new file mode 100644 index 000000000..4b05a8977 --- /dev/null +++ b/vendor/github.com/tetratelabs/wazero/internal/sysfs/stat_unsupported.go @@ -0,0 +1,40 @@ +//go:build (!((amd64 || arm64 || riscv64) && linux) && !((amd64 || arm64) && (darwin || freebsd)) && !((amd64 || arm64) && windows)) || js + +package sysfs + +import ( + "io/fs" + "os" + + experimentalsys "github.com/tetratelabs/wazero/experimental/sys" + "github.com/tetratelabs/wazero/sys" +) + +// Note: go:build constraints must be the same as /sys.stat_unsupported.go for +// the same reasons. + +// dirNlinkIncludesDot might be true for some operating systems, which can have +// new stat_XX.go files as necessary. +// +// Note: this is only used in tests +const dirNlinkIncludesDot = false + +func lstat(path string) (sys.Stat_t, experimentalsys.Errno) { + if info, err := os.Lstat(path); err != nil { + return sys.Stat_t{}, experimentalsys.UnwrapOSError(err) + } else { + return sys.NewStat_t(info), 0 + } +} + +func stat(path string) (sys.Stat_t, experimentalsys.Errno) { + if info, err := os.Stat(path); err != nil { + return sys.Stat_t{}, experimentalsys.UnwrapOSError(err) + } else { + return sys.NewStat_t(info), 0 + } +} + +func statFile(f fs.File) (sys.Stat_t, experimentalsys.Errno) { + return defaultStatFile(f) +} diff --git a/vendor/github.com/tetratelabs/wazero/internal/sysfs/stat_windows.go b/vendor/github.com/tetratelabs/wazero/internal/sysfs/stat_windows.go new file mode 100644 index 000000000..4456dd782 --- /dev/null +++ b/vendor/github.com/tetratelabs/wazero/internal/sysfs/stat_windows.go @@ -0,0 +1,120 @@ +//go:build (amd64 || arm64) && windows + +package sysfs + +import ( + "io/fs" + "syscall" + + experimentalsys "github.com/tetratelabs/wazero/experimental/sys" + "github.com/tetratelabs/wazero/sys" +) + +// dirNlinkIncludesDot is false because Windows does not return dot entries. +// +// Note: this is only used in tests +const dirNlinkIncludesDot = false + +func lstat(path string) (sys.Stat_t, experimentalsys.Errno) { + attrs := uint32(syscall.FILE_FLAG_BACKUP_SEMANTICS) + // Use FILE_FLAG_OPEN_REPARSE_POINT, otherwise CreateFile will follow symlink. + // See https://docs.microsoft.com/en-us/windows/desktop/FileIO/symbolic-link-effects-on-file-systems-functions#createfile-and-createfiletransacted + attrs |= syscall.FILE_FLAG_OPEN_REPARSE_POINT + return statPath(attrs, path) +} + +func stat(path string) (sys.Stat_t, experimentalsys.Errno) { + attrs := uint32(syscall.FILE_FLAG_BACKUP_SEMANTICS) + return statPath(attrs, path) +} + +func statPath(createFileAttrs uint32, path string) (sys.Stat_t, experimentalsys.Errno) { + if len(path) == 0 { + return sys.Stat_t{}, experimentalsys.ENOENT + } + pathp, err := syscall.UTF16PtrFromString(path) + if err != nil { + return sys.Stat_t{}, experimentalsys.EINVAL + } + + // open the file handle + h, err := syscall.CreateFile(pathp, 0, 0, nil, + syscall.OPEN_EXISTING, createFileAttrs, 0) + if err != nil { + errno := experimentalsys.UnwrapOSError(err) + // To match expectations of WASI, e.g. TinyGo TestStatBadDir, return + // ENOENT, not ENOTDIR. + if errno == experimentalsys.ENOTDIR { + errno = experimentalsys.ENOENT + } + return sys.Stat_t{}, errno + } + defer syscall.CloseHandle(h) + + return statHandle(h) +} + +// fdFile allows masking the `Fd` function on os.File. +type fdFile interface { + Fd() uintptr +} + +func statFile(f fs.File) (sys.Stat_t, experimentalsys.Errno) { + if osF, ok := f.(fdFile); ok { + // Attempt to get the stat by handle, which works for normal files + st, err := statHandle(syscall.Handle(osF.Fd())) + + // ERROR_INVALID_HANDLE happens before Go 1.20. Don't fail as we only + // use that approach to fill in inode data, which is not critical. + // + // Note: statHandle uses UnwrapOSError which coerces + // ERROR_INVALID_HANDLE to EBADF. + if err != experimentalsys.EBADF { + return st, err + } + } + return defaultStatFile(f) +} + +func statHandle(h syscall.Handle) (sys.Stat_t, experimentalsys.Errno) { + winFt, err := syscall.GetFileType(h) + if err != nil { + return sys.Stat_t{}, experimentalsys.UnwrapOSError(err) + } + + var fi syscall.ByHandleFileInformation + if err = syscall.GetFileInformationByHandle(h, &fi); err != nil { + return sys.Stat_t{}, experimentalsys.UnwrapOSError(err) + } + + var m fs.FileMode + if fi.FileAttributes&syscall.FILE_ATTRIBUTE_READONLY != 0 { + m |= 0o444 + } else { + m |= 0o666 + } + + switch { // check whether this is a symlink first + case fi.FileAttributes&syscall.FILE_ATTRIBUTE_REPARSE_POINT != 0: + m |= fs.ModeSymlink + case winFt == syscall.FILE_TYPE_PIPE: + m |= fs.ModeNamedPipe + case winFt == syscall.FILE_TYPE_CHAR: + m |= fs.ModeDevice | fs.ModeCharDevice + case fi.FileAttributes&syscall.FILE_ATTRIBUTE_DIRECTORY != 0: + m |= fs.ModeDir | 0o111 // e.g. 0o444 -> 0o555 + } + + st := sys.Stat_t{} + // FileIndex{High,Low} can be combined and used as a unique identifier like inode. + // https://learn.microsoft.com/en-us/windows/win32/api/fileapi/ns-fileapi-by_handle_file_information + st.Dev = uint64(fi.VolumeSerialNumber) + st.Ino = (uint64(fi.FileIndexHigh) << 32) | uint64(fi.FileIndexLow) + st.Mode = m + st.Nlink = uint64(fi.NumberOfLinks) + st.Size = int64(fi.FileSizeHigh)<<32 + int64(fi.FileSizeLow) + st.Atim = fi.LastAccessTime.Nanoseconds() + st.Mtim = fi.LastWriteTime.Nanoseconds() + st.Ctim = fi.CreationTime.Nanoseconds() + return st, 0 +} diff --git a/vendor/github.com/tetratelabs/wazero/internal/sysfs/sync.go b/vendor/github.com/tetratelabs/wazero/internal/sysfs/sync.go new file mode 100644 index 000000000..86f9a0865 --- /dev/null +++ b/vendor/github.com/tetratelabs/wazero/internal/sysfs/sync.go @@ -0,0 +1,13 @@ +//go:build !windows + +package sysfs + +import ( + "os" + + "github.com/tetratelabs/wazero/experimental/sys" +) + +func fsync(f *os.File) sys.Errno { + return sys.UnwrapOSError(f.Sync()) +} diff --git a/vendor/github.com/tetratelabs/wazero/internal/sysfs/sync_windows.go b/vendor/github.com/tetratelabs/wazero/internal/sysfs/sync_windows.go new file mode 100644 index 000000000..f288eb25b --- /dev/null +++ b/vendor/github.com/tetratelabs/wazero/internal/sysfs/sync_windows.go @@ -0,0 +1,20 @@ +package sysfs + +import ( + "os" + + "github.com/tetratelabs/wazero/experimental/sys" +) + +func fsync(f *os.File) sys.Errno { + errno := sys.UnwrapOSError(f.Sync()) + // Coerce error performing stat on a directory to 0, as it won't work + // on Windows. + switch errno { + case sys.EACCES /* Go 1.20 */, sys.EBADF /* Go 1.19 */ : + if st, err := f.Stat(); err == nil && st.IsDir() { + errno = 0 + } + } + return errno +} diff --git a/vendor/github.com/tetratelabs/wazero/internal/sysfs/syscall6_darwin.go b/vendor/github.com/tetratelabs/wazero/internal/sysfs/syscall6_darwin.go new file mode 100644 index 000000000..9fde5baa5 --- /dev/null +++ b/vendor/github.com/tetratelabs/wazero/internal/sysfs/syscall6_darwin.go @@ -0,0 +1,13 @@ +package sysfs + +import ( + "syscall" + _ "unsafe" +) + +// syscall_syscall6 is a private symbol that we link below. We need to use this +// instead of syscall.Syscall6 because the public syscall.Syscall6 won't work +// when fn is an address. +// +//go:linkname syscall_syscall6 syscall.syscall6 +func syscall_syscall6(fn, a1, a2, a3, a4, a5, a6 uintptr) (r1, r2 uintptr, err syscall.Errno) diff --git a/vendor/github.com/tetratelabs/wazero/internal/sysfs/sysfs.go b/vendor/github.com/tetratelabs/wazero/internal/sysfs/sysfs.go new file mode 100644 index 000000000..dd0a8882e --- /dev/null +++ b/vendor/github.com/tetratelabs/wazero/internal/sysfs/sysfs.go @@ -0,0 +1,6 @@ +// Package sysfs includes a low-level filesystem interface and utilities needed +// for WebAssembly host functions (ABI) such as WASI. +// +// The name sysfs was chosen because wazero's public API has a "sys" package, +// which was named after https://github.com/golang/sys. +package sysfs diff --git a/vendor/github.com/tetratelabs/wazero/internal/sysfs/unlink.go b/vendor/github.com/tetratelabs/wazero/internal/sysfs/unlink.go new file mode 100644 index 000000000..e3f051008 --- /dev/null +++ b/vendor/github.com/tetratelabs/wazero/internal/sysfs/unlink.go @@ -0,0 +1,17 @@ +//go:build !windows && !plan9 && !tinygo + +package sysfs + +import ( + "syscall" + + "github.com/tetratelabs/wazero/experimental/sys" +) + +func unlink(name string) (errno sys.Errno) { + err := syscall.Unlink(name) + if errno = sys.UnwrapOSError(err); errno == sys.EPERM { + errno = sys.EISDIR + } + return errno +} diff --git a/vendor/github.com/tetratelabs/wazero/internal/sysfs/unlink_plan9.go b/vendor/github.com/tetratelabs/wazero/internal/sysfs/unlink_plan9.go new file mode 100644 index 000000000..16ed06ab2 --- /dev/null +++ b/vendor/github.com/tetratelabs/wazero/internal/sysfs/unlink_plan9.go @@ -0,0 +1,12 @@ +package sysfs + +import ( + "syscall" + + "github.com/tetratelabs/wazero/experimental/sys" +) + +func unlink(name string) sys.Errno { + err := syscall.Remove(name) + return sys.UnwrapOSError(err) +} diff --git a/vendor/github.com/tetratelabs/wazero/internal/sysfs/unlink_windows.go b/vendor/github.com/tetratelabs/wazero/internal/sysfs/unlink_windows.go new file mode 100644 index 000000000..be31c7b91 --- /dev/null +++ b/vendor/github.com/tetratelabs/wazero/internal/sysfs/unlink_windows.go @@ -0,0 +1,25 @@ +package sysfs + +import ( + "os" + "syscall" + + "github.com/tetratelabs/wazero/experimental/sys" +) + +func unlink(name string) sys.Errno { + err := syscall.Unlink(name) + if err == nil { + return 0 + } + errno := sys.UnwrapOSError(err) + if errno == sys.EBADF { + lstat, errLstat := os.Lstat(name) + if errLstat == nil && lstat.Mode()&os.ModeSymlink != 0 { + errno = sys.UnwrapOSError(os.Remove(name)) + } else { + errno = sys.EISDIR + } + } + return errno +} diff --git a/vendor/github.com/tetratelabs/wazero/internal/u32/u32.go b/vendor/github.com/tetratelabs/wazero/internal/u32/u32.go new file mode 100644 index 000000000..5960a6f0c --- /dev/null +++ b/vendor/github.com/tetratelabs/wazero/internal/u32/u32.go @@ -0,0 +1,11 @@ +package u32 + +// LeBytes returns a byte slice corresponding to the 4 bytes in the uint32 in little-endian byte order. +func LeBytes(v uint32) []byte { + return []byte{ + byte(v), + byte(v >> 8), + byte(v >> 16), + byte(v >> 24), + } +} diff --git a/vendor/github.com/tetratelabs/wazero/internal/u64/u64.go b/vendor/github.com/tetratelabs/wazero/internal/u64/u64.go new file mode 100644 index 000000000..65c7cd124 --- /dev/null +++ b/vendor/github.com/tetratelabs/wazero/internal/u64/u64.go @@ -0,0 +1,15 @@ +package u64 + +// LeBytes returns a byte slice corresponding to the 8 bytes in the uint64 in little-endian byte order. +func LeBytes(v uint64) []byte { + return []byte{ + byte(v), + byte(v >> 8), + byte(v >> 16), + byte(v >> 24), + byte(v >> 32), + byte(v >> 40), + byte(v >> 48), + byte(v >> 56), + } +} diff --git a/vendor/github.com/tetratelabs/wazero/internal/version/version.go b/vendor/github.com/tetratelabs/wazero/internal/version/version.go new file mode 100644 index 000000000..9261df0f7 --- /dev/null +++ b/vendor/github.com/tetratelabs/wazero/internal/version/version.go @@ -0,0 +1,52 @@ +package version + +import ( + "runtime/debug" + "strings" +) + +// Default is the default version value used when none was found. +const Default = "dev" + +// version holds the current version from the go.mod of downstream users or set by ldflag for wazero CLI. +var version string + +// GetWazeroVersion returns the current version of wazero either in the go.mod or set by ldflag for wazero CLI. +// +// If this is not CLI, this assumes that downstream users of wazero imports wazero as "github.com/tetratelabs/wazero". +// To be precise, the returned string matches the require statement there. +// For example, if the go.mod has "require github.com/tetratelabs/wazero 0.1.2-12314124-abcd", +// then this returns "0.1.2-12314124-abcd". +// +// Note: this is tested in ./testdata/main_test.go with a separate go.mod to pretend as the wazero user. +func GetWazeroVersion() (ret string) { + if len(version) != 0 { + return version + } + + info, ok := debug.ReadBuildInfo() + if ok { + for _, dep := range info.Deps { + // Note: here's the assumption that wazero is imported as github.com/tetratelabs/wazero. + if strings.Contains(dep.Path, "github.com/tetratelabs/wazero") { + ret = dep.Version + } + } + + // In wazero CLI, wazero is a main module, so we have to get the version info from info.Main. + if versionMissing(ret) { + ret = info.Main.Version + } + } + if versionMissing(ret) { + return Default // don't return parens + } + + // Cache for the subsequent calls. + version = ret + return ret +} + +func versionMissing(ret string) bool { + return ret == "" || ret == "(devel)" // pkg.go defaults to (devel) +} diff --git a/vendor/github.com/tetratelabs/wazero/internal/wasm/binary/code.go b/vendor/github.com/tetratelabs/wazero/internal/wasm/binary/code.go new file mode 100644 index 000000000..2fac9196c --- /dev/null +++ b/vendor/github.com/tetratelabs/wazero/internal/wasm/binary/code.go @@ -0,0 +1,100 @@ +package binary + +import ( + "bytes" + "fmt" + "io" + "math" + + "github.com/tetratelabs/wazero/internal/leb128" + "github.com/tetratelabs/wazero/internal/wasm" +) + +func decodeCode(r *bytes.Reader, codeSectionStart uint64, ret *wasm.Code) (err error) { + ss, _, err := leb128.DecodeUint32(r) + if err != nil { + return fmt.Errorf("get the size of code: %w", err) + } + remaining := int64(ss) + + // Parse #locals. + ls, bytesRead, err := leb128.DecodeUint32(r) + remaining -= int64(bytesRead) + if err != nil { + return fmt.Errorf("get the size locals: %v", err) + } else if remaining < 0 { + return io.EOF + } + + // Validate the locals. + bytesRead = 0 + var sum uint64 + for i := uint32(0); i < ls; i++ { + num, n, err := leb128.DecodeUint32(r) + if err != nil { + return fmt.Errorf("read n of locals: %v", err) + } else if remaining < 0 { + return io.EOF + } + + sum += uint64(num) + + b, err := r.ReadByte() + if err != nil { + return fmt.Errorf("read type of local: %v", err) + } + + bytesRead += n + 1 + switch vt := b; vt { + case wasm.ValueTypeI32, wasm.ValueTypeF32, wasm.ValueTypeI64, wasm.ValueTypeF64, + wasm.ValueTypeFuncref, wasm.ValueTypeExternref, wasm.ValueTypeV128: + default: + return fmt.Errorf("invalid local type: 0x%x", vt) + } + } + + if sum > math.MaxUint32 { + return fmt.Errorf("too many locals: %d", sum) + } + + // Rewind the buffer. + _, err = r.Seek(-int64(bytesRead), io.SeekCurrent) + if err != nil { + return err + } + + localTypes := make([]wasm.ValueType, 0, sum) + for i := uint32(0); i < ls; i++ { + num, bytesRead, err := leb128.DecodeUint32(r) + remaining -= int64(bytesRead) + 1 // +1 for the subsequent ReadByte + if err != nil { + return fmt.Errorf("read n of locals: %v", err) + } else if remaining < 0 { + return io.EOF + } + + b, err := r.ReadByte() + if err != nil { + return fmt.Errorf("read type of local: %v", err) + } + + for j := uint32(0); j < num; j++ { + localTypes = append(localTypes, b) + } + } + + bodyOffsetInCodeSection := codeSectionStart - uint64(r.Len()) + body := make([]byte, remaining) + if _, err = io.ReadFull(r, body); err != nil { + return fmt.Errorf("read body: %w", err) + } + + if endIndex := len(body) - 1; endIndex < 0 || body[endIndex] != wasm.OpcodeEnd { + return fmt.Errorf("expr not end with OpcodeEnd") + } + + ret.BodyOffsetInCodeSection = bodyOffsetInCodeSection + ret.LocalTypes = localTypes + ret.Body = body + return nil +} diff --git a/vendor/github.com/tetratelabs/wazero/internal/wasm/binary/const_expr.go b/vendor/github.com/tetratelabs/wazero/internal/wasm/binary/const_expr.go new file mode 100644 index 000000000..edfc0a086 --- /dev/null +++ b/vendor/github.com/tetratelabs/wazero/internal/wasm/binary/const_expr.go @@ -0,0 +1,105 @@ +package binary + +import ( + "bytes" + "fmt" + "io" + + "github.com/tetratelabs/wazero/api" + "github.com/tetratelabs/wazero/internal/ieee754" + "github.com/tetratelabs/wazero/internal/leb128" + "github.com/tetratelabs/wazero/internal/wasm" +) + +func decodeConstantExpression(r *bytes.Reader, enabledFeatures api.CoreFeatures, ret *wasm.ConstantExpression) error { + b, err := r.ReadByte() + if err != nil { + return fmt.Errorf("read opcode: %v", err) + } + + remainingBeforeData := int64(r.Len()) + offsetAtData := r.Size() - remainingBeforeData + + opcode := b + switch opcode { + case wasm.OpcodeI32Const: + // Treat constants as signed as their interpretation is not yet known per /RATIONALE.md + _, _, err = leb128.DecodeInt32(r) + case wasm.OpcodeI64Const: + // Treat constants as signed as their interpretation is not yet known per /RATIONALE.md + _, _, err = leb128.DecodeInt64(r) + case wasm.OpcodeF32Const: + buf := make([]byte, 4) + if _, err := io.ReadFull(r, buf); err != nil { + return fmt.Errorf("read f32 constant: %v", err) + } + _, err = ieee754.DecodeFloat32(buf) + case wasm.OpcodeF64Const: + buf := make([]byte, 8) + if _, err := io.ReadFull(r, buf); err != nil { + return fmt.Errorf("read f64 constant: %v", err) + } + _, err = ieee754.DecodeFloat64(buf) + case wasm.OpcodeGlobalGet: + _, _, err = leb128.DecodeUint32(r) + case wasm.OpcodeRefNull: + if err := enabledFeatures.RequireEnabled(api.CoreFeatureBulkMemoryOperations); err != nil { + return fmt.Errorf("ref.null is not supported as %w", err) + } + reftype, err := r.ReadByte() + if err != nil { + return fmt.Errorf("read reference type for ref.null: %w", err) + } else if reftype != wasm.RefTypeFuncref && reftype != wasm.RefTypeExternref { + return fmt.Errorf("invalid type for ref.null: 0x%x", reftype) + } + case wasm.OpcodeRefFunc: + if err := enabledFeatures.RequireEnabled(api.CoreFeatureBulkMemoryOperations); err != nil { + return fmt.Errorf("ref.func is not supported as %w", err) + } + // Parsing index. + _, _, err = leb128.DecodeUint32(r) + case wasm.OpcodeVecPrefix: + if err := enabledFeatures.RequireEnabled(api.CoreFeatureSIMD); err != nil { + return fmt.Errorf("vector instructions are not supported as %w", err) + } + opcode, err = r.ReadByte() + if err != nil { + return fmt.Errorf("read vector instruction opcode suffix: %w", err) + } + + if opcode != wasm.OpcodeVecV128Const { + return fmt.Errorf("invalid vector opcode for const expression: %#x", opcode) + } + + remainingBeforeData = int64(r.Len()) + offsetAtData = r.Size() - remainingBeforeData + + n, err := r.Read(make([]byte, 16)) + if err != nil { + return fmt.Errorf("read vector const instruction immediates: %w", err) + } else if n != 16 { + return fmt.Errorf("read vector const instruction immediates: needs 16 bytes but was %d bytes", n) + } + default: + return fmt.Errorf("%v for const expression opt code: %#x", ErrInvalidByte, b) + } + + if err != nil { + return fmt.Errorf("read value: %v", err) + } + + if b, err = r.ReadByte(); err != nil { + return fmt.Errorf("look for end opcode: %v", err) + } + + if b != wasm.OpcodeEnd { + return fmt.Errorf("constant expression has been not terminated") + } + + ret.Data = make([]byte, remainingBeforeData-int64(r.Len())-1) + if _, err = r.ReadAt(ret.Data, offsetAtData); err != nil { + return fmt.Errorf("error re-buffering ConstantExpression.Data") + } + ret.Opcode = opcode + return nil +} diff --git a/vendor/github.com/tetratelabs/wazero/internal/wasm/binary/custom.go b/vendor/github.com/tetratelabs/wazero/internal/wasm/binary/custom.go new file mode 100644 index 000000000..771f8c327 --- /dev/null +++ b/vendor/github.com/tetratelabs/wazero/internal/wasm/binary/custom.go @@ -0,0 +1,22 @@ +package binary + +import ( + "bytes" + + "github.com/tetratelabs/wazero/internal/wasm" +) + +// decodeCustomSection deserializes the data **not** associated with the "name" key in SectionIDCustom. +// +// See https://www.w3.org/TR/2019/REC-wasm-core-1-20191205/#custom-section%E2%91%A0 +func decodeCustomSection(r *bytes.Reader, name string, limit uint64) (result *wasm.CustomSection, err error) { + buf := make([]byte, limit) + _, err = r.Read(buf) + + result = &wasm.CustomSection{ + Name: name, + Data: buf, + } + + return +} diff --git a/vendor/github.com/tetratelabs/wazero/internal/wasm/binary/data.go b/vendor/github.com/tetratelabs/wazero/internal/wasm/binary/data.go new file mode 100644 index 000000000..054ccb3c6 --- /dev/null +++ b/vendor/github.com/tetratelabs/wazero/internal/wasm/binary/data.go @@ -0,0 +1,79 @@ +package binary + +import ( + "bytes" + "fmt" + "io" + + "github.com/tetratelabs/wazero/api" + "github.com/tetratelabs/wazero/internal/leb128" + "github.com/tetratelabs/wazero/internal/wasm" +) + +// dataSegmentPrefix represents three types of data segments. +// +// https://www.w3.org/TR/2022/WD-wasm-core-2-20220419/binary/modules.html#data-section +type dataSegmentPrefix = uint32 + +const ( + // dataSegmentPrefixActive is the prefix for the version 1.0 compatible data segment, which is classified as "active" in 2.0. + dataSegmentPrefixActive dataSegmentPrefix = 0x0 + // dataSegmentPrefixPassive prefixes the "passive" data segment as in version 2.0 specification. + dataSegmentPrefixPassive dataSegmentPrefix = 0x1 + // dataSegmentPrefixActiveWithMemoryIndex is the active prefix with memory index encoded which is defined for futur use as of 2.0. + dataSegmentPrefixActiveWithMemoryIndex dataSegmentPrefix = 0x2 +) + +func decodeDataSegment(r *bytes.Reader, enabledFeatures api.CoreFeatures, ret *wasm.DataSegment) (err error) { + dataSegmentPrefx, _, err := leb128.DecodeUint32(r) + if err != nil { + err = fmt.Errorf("read data segment prefix: %w", err) + return + } + + if dataSegmentPrefx != dataSegmentPrefixActive { + if err = enabledFeatures.RequireEnabled(api.CoreFeatureBulkMemoryOperations); err != nil { + err = fmt.Errorf("non-zero prefix for data segment is invalid as %w", err) + return + } + } + + switch dataSegmentPrefx { + case dataSegmentPrefixActive, + dataSegmentPrefixActiveWithMemoryIndex: + // Active data segment as in + // https://www.w3.org/TR/2022/WD-wasm-core-2-20220419/binary/modules.html#data-section + if dataSegmentPrefx == 0x2 { + d, _, err := leb128.DecodeUint32(r) + if err != nil { + return fmt.Errorf("read memory index: %v", err) + } else if d != 0 { + return fmt.Errorf("memory index must be zero but was %d", d) + } + } + + err = decodeConstantExpression(r, enabledFeatures, &ret.OffsetExpression) + if err != nil { + return fmt.Errorf("read offset expression: %v", err) + } + case dataSegmentPrefixPassive: + // Passive data segment doesn't need const expr nor memory index encoded. + // https://www.w3.org/TR/2022/WD-wasm-core-2-20220419/binary/modules.html#data-section + ret.Passive = true + default: + err = fmt.Errorf("invalid data segment prefix: 0x%x", dataSegmentPrefx) + return + } + + vs, _, err := leb128.DecodeUint32(r) + if err != nil { + err = fmt.Errorf("get the size of vector: %v", err) + return + } + + ret.Init = make([]byte, vs) + if _, err = io.ReadFull(r, ret.Init); err != nil { + err = fmt.Errorf("read bytes for init: %v", err) + } + return +} diff --git a/vendor/github.com/tetratelabs/wazero/internal/wasm/binary/decoder.go b/vendor/github.com/tetratelabs/wazero/internal/wasm/binary/decoder.go new file mode 100644 index 000000000..c4191dae9 --- /dev/null +++ b/vendor/github.com/tetratelabs/wazero/internal/wasm/binary/decoder.go @@ -0,0 +1,193 @@ +package binary + +import ( + "bytes" + "debug/dwarf" + "errors" + "fmt" + "io" + + "github.com/tetratelabs/wazero/api" + "github.com/tetratelabs/wazero/internal/leb128" + "github.com/tetratelabs/wazero/internal/wasm" + "github.com/tetratelabs/wazero/internal/wasmdebug" +) + +// DecodeModule implements wasm.DecodeModule for the WebAssembly 1.0 (20191205) Binary Format +// See https://www.w3.org/TR/2019/REC-wasm-core-1-20191205/#binary-format%E2%91%A0 +func DecodeModule( + binary []byte, + enabledFeatures api.CoreFeatures, + memoryLimitPages uint32, + memoryCapacityFromMax, + dwarfEnabled, storeCustomSections bool, +) (*wasm.Module, error) { + r := bytes.NewReader(binary) + + // Magic number. + buf := make([]byte, 4) + if _, err := io.ReadFull(r, buf); err != nil || !bytes.Equal(buf, Magic) { + return nil, ErrInvalidMagicNumber + } + + // Version. + if _, err := io.ReadFull(r, buf); err != nil || !bytes.Equal(buf, version) { + return nil, ErrInvalidVersion + } + + memSizer := newMemorySizer(memoryLimitPages, memoryCapacityFromMax) + + m := &wasm.Module{} + var info, line, str, abbrev, ranges []byte // For DWARF Data. + for { + // TODO: except custom sections, all others are required to be in order, but we aren't checking yet. + // See https://www.w3.org/TR/2019/REC-wasm-core-1-20191205/#modules%E2%91%A0%E2%93%AA + sectionID, err := r.ReadByte() + if err == io.EOF { + break + } else if err != nil { + return nil, fmt.Errorf("read section id: %w", err) + } + + sectionSize, _, err := leb128.DecodeUint32(r) + if err != nil { + return nil, fmt.Errorf("get size of section %s: %v", wasm.SectionIDName(sectionID), err) + } + + sectionContentStart := r.Len() + switch sectionID { + case wasm.SectionIDCustom: + // First, validate the section and determine if the section for this name has already been set + name, nameSize, decodeErr := decodeUTF8(r, "custom section name") + if decodeErr != nil { + err = decodeErr + break + } else if sectionSize < nameSize { + err = fmt.Errorf("malformed custom section %s", name) + break + } else if name == "name" && m.NameSection != nil { + err = fmt.Errorf("redundant custom section %s", name) + break + } + + // Now, either decode the NameSection or CustomSection + limit := sectionSize - nameSize + + var c *wasm.CustomSection + if name != "name" { + if storeCustomSections || dwarfEnabled { + c, err = decodeCustomSection(r, name, uint64(limit)) + if err != nil { + return nil, fmt.Errorf("failed to read custom section name[%s]: %w", name, err) + } + m.CustomSections = append(m.CustomSections, c) + if dwarfEnabled { + switch name { + case ".debug_info": + info = c.Data + case ".debug_line": + line = c.Data + case ".debug_str": + str = c.Data + case ".debug_abbrev": + abbrev = c.Data + case ".debug_ranges": + ranges = c.Data + } + } + } else { + if _, err = io.CopyN(io.Discard, r, int64(limit)); err != nil { + return nil, fmt.Errorf("failed to skip name[%s]: %w", name, err) + } + } + } else { + m.NameSection, err = decodeNameSection(r, uint64(limit)) + } + case wasm.SectionIDType: + m.TypeSection, err = decodeTypeSection(enabledFeatures, r) + case wasm.SectionIDImport: + m.ImportSection, m.ImportPerModule, m.ImportFunctionCount, m.ImportGlobalCount, m.ImportMemoryCount, m.ImportTableCount, err = decodeImportSection(r, memSizer, memoryLimitPages, enabledFeatures) + if err != nil { + return nil, err // avoid re-wrapping the error. + } + case wasm.SectionIDFunction: + m.FunctionSection, err = decodeFunctionSection(r) + case wasm.SectionIDTable: + m.TableSection, err = decodeTableSection(r, enabledFeatures) + case wasm.SectionIDMemory: + m.MemorySection, err = decodeMemorySection(r, enabledFeatures, memSizer, memoryLimitPages) + case wasm.SectionIDGlobal: + if m.GlobalSection, err = decodeGlobalSection(r, enabledFeatures); err != nil { + return nil, err // avoid re-wrapping the error. + } + case wasm.SectionIDExport: + m.ExportSection, m.Exports, err = decodeExportSection(r) + case wasm.SectionIDStart: + if m.StartSection != nil { + return nil, errors.New("multiple start sections are invalid") + } + m.StartSection, err = decodeStartSection(r) + case wasm.SectionIDElement: + m.ElementSection, err = decodeElementSection(r, enabledFeatures) + case wasm.SectionIDCode: + m.CodeSection, err = decodeCodeSection(r) + case wasm.SectionIDData: + m.DataSection, err = decodeDataSection(r, enabledFeatures) + case wasm.SectionIDDataCount: + if err := enabledFeatures.RequireEnabled(api.CoreFeatureBulkMemoryOperations); err != nil { + return nil, fmt.Errorf("data count section not supported as %v", err) + } + m.DataCountSection, err = decodeDataCountSection(r) + default: + err = ErrInvalidSectionID + } + + readBytes := sectionContentStart - r.Len() + if err == nil && int(sectionSize) != readBytes { + err = fmt.Errorf("invalid section length: expected to be %d but got %d", sectionSize, readBytes) + } + + if err != nil { + return nil, fmt.Errorf("section %s: %v", wasm.SectionIDName(sectionID), err) + } + } + + if dwarfEnabled { + d, _ := dwarf.New(abbrev, nil, nil, info, line, nil, ranges, str) + m.DWARFLines = wasmdebug.NewDWARFLines(d) + } + + functionCount, codeCount := m.SectionElementCount(wasm.SectionIDFunction), m.SectionElementCount(wasm.SectionIDCode) + if functionCount != codeCount { + return nil, fmt.Errorf("function and code section have inconsistent lengths: %d != %d", functionCount, codeCount) + } + return m, nil +} + +// memorySizer derives min, capacity and max pages from decoded wasm. +type memorySizer func(minPages uint32, maxPages *uint32) (min uint32, capacity uint32, max uint32) + +// newMemorySizer sets capacity to minPages unless max is defined and +// memoryCapacityFromMax is true. +func newMemorySizer(memoryLimitPages uint32, memoryCapacityFromMax bool) memorySizer { + return func(minPages uint32, maxPages *uint32) (min, capacity, max uint32) { + if maxPages != nil { + if memoryCapacityFromMax { + return minPages, *maxPages, *maxPages + } + // This is an invalid value: let it propagate, we will fail later. + if *maxPages > wasm.MemoryLimitPages { + return minPages, minPages, *maxPages + } + // This is a valid value, but it goes over the run-time limit: return the limit. + if *maxPages > memoryLimitPages { + return minPages, minPages, memoryLimitPages + } + return minPages, minPages, *maxPages + } + if memoryCapacityFromMax { + return minPages, memoryLimitPages, memoryLimitPages + } + return minPages, minPages, memoryLimitPages + } +} diff --git a/vendor/github.com/tetratelabs/wazero/internal/wasm/binary/element.go b/vendor/github.com/tetratelabs/wazero/internal/wasm/binary/element.go new file mode 100644 index 000000000..7ab4b48eb --- /dev/null +++ b/vendor/github.com/tetratelabs/wazero/internal/wasm/binary/element.go @@ -0,0 +1,269 @@ +package binary + +import ( + "bytes" + "errors" + "fmt" + + "github.com/tetratelabs/wazero/api" + "github.com/tetratelabs/wazero/internal/leb128" + "github.com/tetratelabs/wazero/internal/wasm" +) + +func ensureElementKindFuncRef(r *bytes.Reader) error { + elemKind, err := r.ReadByte() + if err != nil { + return fmt.Errorf("read element prefix: %w", err) + } + if elemKind != 0x0 { // ElemKind is fixed to 0x0 now: https://www.w3.org/TR/2022/WD-wasm-core-2-20220419/binary/modules.html#element-section + return fmt.Errorf("element kind must be zero but was 0x%x", elemKind) + } + return nil +} + +func decodeElementInitValueVector(r *bytes.Reader) ([]wasm.Index, error) { + vs, _, err := leb128.DecodeUint32(r) + if err != nil { + return nil, fmt.Errorf("get size of vector: %w", err) + } + + vec := make([]wasm.Index, vs) + for i := range vec { + u32, _, err := leb128.DecodeUint32(r) + if err != nil { + return nil, fmt.Errorf("read function index: %w", err) + } + + if u32 >= wasm.MaximumFunctionIndex { + return nil, fmt.Errorf("too large function index in Element init: %d", u32) + } + vec[i] = u32 + } + return vec, nil +} + +func decodeElementConstExprVector(r *bytes.Reader, elemType wasm.RefType, enabledFeatures api.CoreFeatures) ([]wasm.Index, error) { + vs, _, err := leb128.DecodeUint32(r) + if err != nil { + return nil, fmt.Errorf("failed to get the size of constexpr vector: %w", err) + } + vec := make([]wasm.Index, vs) + for i := range vec { + var expr wasm.ConstantExpression + err := decodeConstantExpression(r, enabledFeatures, &expr) + if err != nil { + return nil, err + } + switch expr.Opcode { + case wasm.OpcodeRefFunc: + if elemType != wasm.RefTypeFuncref { + return nil, fmt.Errorf("element type mismatch: want %s, but constexpr has funcref", wasm.RefTypeName(elemType)) + } + v, _, _ := leb128.LoadUint32(expr.Data) + if v >= wasm.MaximumFunctionIndex { + return nil, fmt.Errorf("too large function index in Element init: %d", v) + } + vec[i] = v + case wasm.OpcodeRefNull: + if elemType != expr.Data[0] { + return nil, fmt.Errorf("element type mismatch: want %s, but constexpr has %s", + wasm.RefTypeName(elemType), wasm.RefTypeName(expr.Data[0])) + } + vec[i] = wasm.ElementInitNullReference + case wasm.OpcodeGlobalGet: + i32, _, _ := leb128.LoadInt32(expr.Data) + // Resolving the reference type from globals is done at instantiation phase. See the comment on + // wasm.elementInitImportedGlobalReferenceType. + vec[i] = wasm.WrapGlobalIndexAsElementInit(wasm.Index(i32)) + default: + return nil, fmt.Errorf("const expr must be either ref.null or ref.func but was %s", wasm.InstructionName(expr.Opcode)) + } + } + return vec, nil +} + +func decodeElementRefType(r *bytes.Reader) (ret wasm.RefType, err error) { + ret, err = r.ReadByte() + if err != nil { + err = fmt.Errorf("read element ref type: %w", err) + return + } + if ret != wasm.RefTypeFuncref && ret != wasm.RefTypeExternref { + return 0, errors.New("ref type must be funcref or externref for element as of WebAssembly 2.0") + } + return +} + +const ( + // The prefix is explained at https://www.w3.org/TR/2022/WD-wasm-core-2-20220419/binary/modules.html#element-section + + // elementSegmentPrefixLegacy is the legacy prefix and is only valid one before CoreFeatureBulkMemoryOperations. + elementSegmentPrefixLegacy = iota + // elementSegmentPrefixPassiveFuncrefValueVector is the passive element whose indexes are encoded as vec(varint), and reftype is fixed to funcref. + elementSegmentPrefixPassiveFuncrefValueVector + // elementSegmentPrefixActiveFuncrefValueVectorWithTableIndex is the same as elementSegmentPrefixPassiveFuncrefValueVector but active and table index is encoded. + elementSegmentPrefixActiveFuncrefValueVectorWithTableIndex + // elementSegmentPrefixDeclarativeFuncrefValueVector is the same as elementSegmentPrefixPassiveFuncrefValueVector but declarative. + elementSegmentPrefixDeclarativeFuncrefValueVector + // elementSegmentPrefixActiveFuncrefConstExprVector is active whoce reftype is fixed to funcref and indexes are encoded as vec(const_expr). + elementSegmentPrefixActiveFuncrefConstExprVector + // elementSegmentPrefixPassiveConstExprVector is passive whoce indexes are encoded as vec(const_expr), and reftype is encoded. + elementSegmentPrefixPassiveConstExprVector + // elementSegmentPrefixPassiveConstExprVector is active whoce indexes are encoded as vec(const_expr), and reftype and table index are encoded. + elementSegmentPrefixActiveConstExprVector + // elementSegmentPrefixDeclarativeConstExprVector is declarative whoce indexes are encoded as vec(const_expr), and reftype is encoded. + elementSegmentPrefixDeclarativeConstExprVector +) + +func decodeElementSegment(r *bytes.Reader, enabledFeatures api.CoreFeatures, ret *wasm.ElementSegment) error { + prefix, _, err := leb128.DecodeUint32(r) + if err != nil { + return fmt.Errorf("read element prefix: %w", err) + } + + if prefix != elementSegmentPrefixLegacy { + if err := enabledFeatures.RequireEnabled(api.CoreFeatureBulkMemoryOperations); err != nil { + return fmt.Errorf("non-zero prefix for element segment is invalid as %w", err) + } + } + + // Encoding depends on the prefix and described at https://www.w3.org/TR/2022/WD-wasm-core-2-20220419/binary/modules.html#element-section + switch prefix { + case elementSegmentPrefixLegacy: + // Legacy prefix which is WebAssembly 1.0 compatible. + err = decodeConstantExpression(r, enabledFeatures, &ret.OffsetExpr) + if err != nil { + return fmt.Errorf("read expr for offset: %w", err) + } + + ret.Init, err = decodeElementInitValueVector(r) + if err != nil { + return err + } + + ret.Mode = wasm.ElementModeActive + ret.Type = wasm.RefTypeFuncref + return nil + case elementSegmentPrefixPassiveFuncrefValueVector: + // Prefix 1 requires funcref. + if err = ensureElementKindFuncRef(r); err != nil { + return err + } + + ret.Init, err = decodeElementInitValueVector(r) + if err != nil { + return err + } + ret.Mode = wasm.ElementModePassive + ret.Type = wasm.RefTypeFuncref + return nil + case elementSegmentPrefixActiveFuncrefValueVectorWithTableIndex: + ret.TableIndex, _, err = leb128.DecodeUint32(r) + if err != nil { + return fmt.Errorf("get size of vector: %w", err) + } + + if ret.TableIndex != 0 { + if err := enabledFeatures.RequireEnabled(api.CoreFeatureReferenceTypes); err != nil { + return fmt.Errorf("table index must be zero but was %d: %w", ret.TableIndex, err) + } + } + + err := decodeConstantExpression(r, enabledFeatures, &ret.OffsetExpr) + if err != nil { + return fmt.Errorf("read expr for offset: %w", err) + } + + // Prefix 2 requires funcref. + if err = ensureElementKindFuncRef(r); err != nil { + return err + } + + ret.Init, err = decodeElementInitValueVector(r) + if err != nil { + return err + } + + ret.Mode = wasm.ElementModeActive + ret.Type = wasm.RefTypeFuncref + return nil + case elementSegmentPrefixDeclarativeFuncrefValueVector: + // Prefix 3 requires funcref. + if err = ensureElementKindFuncRef(r); err != nil { + return err + } + ret.Init, err = decodeElementInitValueVector(r) + if err != nil { + return err + } + ret.Type = wasm.RefTypeFuncref + ret.Mode = wasm.ElementModeDeclarative + return nil + case elementSegmentPrefixActiveFuncrefConstExprVector: + err := decodeConstantExpression(r, enabledFeatures, &ret.OffsetExpr) + if err != nil { + return fmt.Errorf("read expr for offset: %w", err) + } + + ret.Init, err = decodeElementConstExprVector(r, wasm.RefTypeFuncref, enabledFeatures) + if err != nil { + return err + } + ret.Mode = wasm.ElementModeActive + ret.Type = wasm.RefTypeFuncref + return nil + case elementSegmentPrefixPassiveConstExprVector: + ret.Type, err = decodeElementRefType(r) + if err != nil { + return err + } + ret.Init, err = decodeElementConstExprVector(r, ret.Type, enabledFeatures) + if err != nil { + return err + } + ret.Mode = wasm.ElementModePassive + return nil + case elementSegmentPrefixActiveConstExprVector: + ret.TableIndex, _, err = leb128.DecodeUint32(r) + if err != nil { + return fmt.Errorf("get size of vector: %w", err) + } + + if ret.TableIndex != 0 { + if err := enabledFeatures.RequireEnabled(api.CoreFeatureReferenceTypes); err != nil { + return fmt.Errorf("table index must be zero but was %d: %w", ret.TableIndex, err) + } + } + err := decodeConstantExpression(r, enabledFeatures, &ret.OffsetExpr) + if err != nil { + return fmt.Errorf("read expr for offset: %w", err) + } + + ret.Type, err = decodeElementRefType(r) + if err != nil { + return err + } + + ret.Init, err = decodeElementConstExprVector(r, ret.Type, enabledFeatures) + if err != nil { + return err + } + + ret.Mode = wasm.ElementModeActive + return nil + case elementSegmentPrefixDeclarativeConstExprVector: + ret.Type, err = decodeElementRefType(r) + if err != nil { + return err + } + ret.Init, err = decodeElementConstExprVector(r, ret.Type, enabledFeatures) + if err != nil { + return err + } + + ret.Mode = wasm.ElementModeDeclarative + return nil + default: + return fmt.Errorf("invalid element segment prefix: 0x%x", prefix) + } +} diff --git a/vendor/github.com/tetratelabs/wazero/internal/wasm/binary/errors.go b/vendor/github.com/tetratelabs/wazero/internal/wasm/binary/errors.go new file mode 100644 index 000000000..b9125b038 --- /dev/null +++ b/vendor/github.com/tetratelabs/wazero/internal/wasm/binary/errors.go @@ -0,0 +1,11 @@ +package binary + +import "errors" + +var ( + ErrInvalidByte = errors.New("invalid byte") + ErrInvalidMagicNumber = errors.New("invalid magic number") + ErrInvalidVersion = errors.New("invalid version header") + ErrInvalidSectionID = errors.New("invalid section id") + ErrCustomSectionNotFound = errors.New("custom section not found") +) diff --git a/vendor/github.com/tetratelabs/wazero/internal/wasm/binary/export.go b/vendor/github.com/tetratelabs/wazero/internal/wasm/binary/export.go new file mode 100644 index 000000000..925e9c499 --- /dev/null +++ b/vendor/github.com/tetratelabs/wazero/internal/wasm/binary/export.go @@ -0,0 +1,32 @@ +package binary + +import ( + "bytes" + "fmt" + + "github.com/tetratelabs/wazero/internal/leb128" + "github.com/tetratelabs/wazero/internal/wasm" +) + +func decodeExport(r *bytes.Reader, ret *wasm.Export) (err error) { + if ret.Name, _, err = decodeUTF8(r, "export name"); err != nil { + return + } + + b, err := r.ReadByte() + if err != nil { + err = fmt.Errorf("error decoding export kind: %w", err) + return + } + + ret.Type = b + switch ret.Type { + case wasm.ExternTypeFunc, wasm.ExternTypeTable, wasm.ExternTypeMemory, wasm.ExternTypeGlobal: + if ret.Index, _, err = leb128.DecodeUint32(r); err != nil { + err = fmt.Errorf("error decoding export index: %w", err) + } + default: + err = fmt.Errorf("%w: invalid byte for exportdesc: %#x", ErrInvalidByte, b) + } + return +} diff --git a/vendor/github.com/tetratelabs/wazero/internal/wasm/binary/function.go b/vendor/github.com/tetratelabs/wazero/internal/wasm/binary/function.go new file mode 100644 index 000000000..bb9e2b649 --- /dev/null +++ b/vendor/github.com/tetratelabs/wazero/internal/wasm/binary/function.go @@ -0,0 +1,56 @@ +package binary + +import ( + "bytes" + "fmt" + + "github.com/tetratelabs/wazero/api" + "github.com/tetratelabs/wazero/internal/leb128" + "github.com/tetratelabs/wazero/internal/wasm" +) + +func decodeFunctionType(enabledFeatures api.CoreFeatures, r *bytes.Reader, ret *wasm.FunctionType) (err error) { + b, err := r.ReadByte() + if err != nil { + return fmt.Errorf("read leading byte: %w", err) + } + + if b != 0x60 { + return fmt.Errorf("%w: %#x != 0x60", ErrInvalidByte, b) + } + + paramCount, _, err := leb128.DecodeUint32(r) + if err != nil { + return fmt.Errorf("could not read parameter count: %w", err) + } + + paramTypes, err := decodeValueTypes(r, paramCount) + if err != nil { + return fmt.Errorf("could not read parameter types: %w", err) + } + + resultCount, _, err := leb128.DecodeUint32(r) + if err != nil { + return fmt.Errorf("could not read result count: %w", err) + } + + // Guard >1.0 feature multi-value + if resultCount > 1 { + if err = enabledFeatures.RequireEnabled(api.CoreFeatureMultiValue); err != nil { + return fmt.Errorf("multiple result types invalid as %v", err) + } + } + + resultTypes, err := decodeValueTypes(r, resultCount) + if err != nil { + return fmt.Errorf("could not read result types: %w", err) + } + + ret.Params = paramTypes + ret.Results = resultTypes + + // cache the key for the function type + _ = ret.String() + + return nil +} diff --git a/vendor/github.com/tetratelabs/wazero/internal/wasm/binary/global.go b/vendor/github.com/tetratelabs/wazero/internal/wasm/binary/global.go new file mode 100644 index 000000000..4e1c16fda --- /dev/null +++ b/vendor/github.com/tetratelabs/wazero/internal/wasm/binary/global.go @@ -0,0 +1,50 @@ +package binary + +import ( + "bytes" + "fmt" + + "github.com/tetratelabs/wazero/api" + "github.com/tetratelabs/wazero/internal/wasm" +) + +// decodeGlobal returns the api.Global decoded with the WebAssembly 1.0 (20191205) Binary Format. +// +// See https://www.w3.org/TR/2019/REC-wasm-core-1-20191205/#binary-global +func decodeGlobal(r *bytes.Reader, enabledFeatures api.CoreFeatures, ret *wasm.Global) (err error) { + ret.Type, err = decodeGlobalType(r) + if err != nil { + return err + } + + err = decodeConstantExpression(r, enabledFeatures, &ret.Init) + return +} + +// decodeGlobalType returns the wasm.GlobalType decoded with the WebAssembly 1.0 (20191205) Binary Format. +// +// See https://www.w3.org/TR/2019/REC-wasm-core-1-20191205/#binary-globaltype +func decodeGlobalType(r *bytes.Reader) (wasm.GlobalType, error) { + vt, err := decodeValueTypes(r, 1) + if err != nil { + return wasm.GlobalType{}, fmt.Errorf("read value type: %w", err) + } + + ret := wasm.GlobalType{ + ValType: vt[0], + } + + b, err := r.ReadByte() + if err != nil { + return wasm.GlobalType{}, fmt.Errorf("read mutablity: %w", err) + } + + switch mut := b; mut { + case 0x00: // not mutable + case 0x01: // mutable + ret.Mutable = true + default: + return wasm.GlobalType{}, fmt.Errorf("%w for mutability: %#x != 0x00 or 0x01", ErrInvalidByte, mut) + } + return ret, nil +} diff --git a/vendor/github.com/tetratelabs/wazero/internal/wasm/binary/header.go b/vendor/github.com/tetratelabs/wazero/internal/wasm/binary/header.go new file mode 100644 index 000000000..29ba1b599 --- /dev/null +++ b/vendor/github.com/tetratelabs/wazero/internal/wasm/binary/header.go @@ -0,0 +1,9 @@ +package binary + +// Magic is the 4 byte preamble (literally "\0asm") of the binary format +// See https://www.w3.org/TR/2019/REC-wasm-core-1-20191205/#binary-magic +var Magic = []byte{0x00, 0x61, 0x73, 0x6D} + +// version is format version and doesn't change between known specification versions +// See https://www.w3.org/TR/2019/REC-wasm-core-1-20191205/#binary-version +var version = []byte{0x01, 0x00, 0x00, 0x00} diff --git a/vendor/github.com/tetratelabs/wazero/internal/wasm/binary/import.go b/vendor/github.com/tetratelabs/wazero/internal/wasm/binary/import.go new file mode 100644 index 000000000..39d310c55 --- /dev/null +++ b/vendor/github.com/tetratelabs/wazero/internal/wasm/binary/import.go @@ -0,0 +1,52 @@ +package binary + +import ( + "bytes" + "fmt" + + "github.com/tetratelabs/wazero/api" + "github.com/tetratelabs/wazero/internal/leb128" + "github.com/tetratelabs/wazero/internal/wasm" +) + +func decodeImport( + r *bytes.Reader, + idx uint32, + memorySizer memorySizer, + memoryLimitPages uint32, + enabledFeatures api.CoreFeatures, + ret *wasm.Import, +) (err error) { + if ret.Module, _, err = decodeUTF8(r, "import module"); err != nil { + err = fmt.Errorf("import[%d] error decoding module: %w", idx, err) + return + } + + if ret.Name, _, err = decodeUTF8(r, "import name"); err != nil { + err = fmt.Errorf("import[%d] error decoding name: %w", idx, err) + return + } + + b, err := r.ReadByte() + if err != nil { + err = fmt.Errorf("import[%d] error decoding type: %w", idx, err) + return + } + ret.Type = b + switch ret.Type { + case wasm.ExternTypeFunc: + ret.DescFunc, _, err = leb128.DecodeUint32(r) + case wasm.ExternTypeTable: + err = decodeTable(r, enabledFeatures, &ret.DescTable) + case wasm.ExternTypeMemory: + ret.DescMem, err = decodeMemory(r, enabledFeatures, memorySizer, memoryLimitPages) + case wasm.ExternTypeGlobal: + ret.DescGlobal, err = decodeGlobalType(r) + default: + err = fmt.Errorf("%w: invalid byte for importdesc: %#x", ErrInvalidByte, b) + } + if err != nil { + err = fmt.Errorf("import[%d] %s[%s.%s]: %w", idx, wasm.ExternTypeName(ret.Type), ret.Module, ret.Name, err) + } + return +} diff --git a/vendor/github.com/tetratelabs/wazero/internal/wasm/binary/limits.go b/vendor/github.com/tetratelabs/wazero/internal/wasm/binary/limits.go new file mode 100644 index 000000000..ff2d73b5f --- /dev/null +++ b/vendor/github.com/tetratelabs/wazero/internal/wasm/binary/limits.go @@ -0,0 +1,47 @@ +package binary + +import ( + "bytes" + "fmt" + + "github.com/tetratelabs/wazero/internal/leb128" +) + +// decodeLimitsType returns the `limitsType` (min, max) decoded with the WebAssembly 1.0 (20191205) Binary Format. +// +// See https://www.w3.org/TR/2019/REC-wasm-core-1-20191205/#limits%E2%91%A6 +// +// Extended in threads proposal: https://webassembly.github.io/threads/core/binary/types.html#limits +func decodeLimitsType(r *bytes.Reader) (min uint32, max *uint32, shared bool, err error) { + var flag byte + if flag, err = r.ReadByte(); err != nil { + err = fmt.Errorf("read leading byte: %v", err) + return + } + + switch flag { + case 0x00, 0x02: + min, _, err = leb128.DecodeUint32(r) + if err != nil { + err = fmt.Errorf("read min of limit: %v", err) + } + case 0x01, 0x03: + min, _, err = leb128.DecodeUint32(r) + if err != nil { + err = fmt.Errorf("read min of limit: %v", err) + return + } + var m uint32 + if m, _, err = leb128.DecodeUint32(r); err != nil { + err = fmt.Errorf("read max of limit: %v", err) + } else { + max = &m + } + default: + err = fmt.Errorf("%v for limits: %#x not in (0x00, 0x01, 0x02, 0x03)", ErrInvalidByte, flag) + } + + shared = flag == 0x02 || flag == 0x03 + + return +} diff --git a/vendor/github.com/tetratelabs/wazero/internal/wasm/binary/memory.go b/vendor/github.com/tetratelabs/wazero/internal/wasm/binary/memory.go new file mode 100644 index 000000000..e1b175123 --- /dev/null +++ b/vendor/github.com/tetratelabs/wazero/internal/wasm/binary/memory.go @@ -0,0 +1,42 @@ +package binary + +import ( + "bytes" + "fmt" + + "github.com/tetratelabs/wazero/api" + "github.com/tetratelabs/wazero/experimental" + "github.com/tetratelabs/wazero/internal/wasm" +) + +// decodeMemory returns the api.Memory decoded with the WebAssembly 1.0 (20191205) Binary Format. +// +// See https://www.w3.org/TR/2019/REC-wasm-core-1-20191205/#binary-memory +func decodeMemory( + r *bytes.Reader, + enabledFeatures api.CoreFeatures, + memorySizer func(minPages uint32, maxPages *uint32) (min, capacity, max uint32), + memoryLimitPages uint32, +) (*wasm.Memory, error) { + min, maxP, shared, err := decodeLimitsType(r) + if err != nil { + return nil, err + } + + if shared { + if !enabledFeatures.IsEnabled(experimental.CoreFeaturesThreads) { + return nil, fmt.Errorf("shared memory requested but threads feature not enabled") + } + + // This restriction may be lifted in the future. + // https://webassembly.github.io/threads/core/binary/types.html#memory-types + if maxP == nil { + return nil, fmt.Errorf("shared memory requires a maximum size to be specified") + } + } + + min, capacity, max := memorySizer(min, maxP) + mem := &wasm.Memory{Min: min, Cap: capacity, Max: max, IsMaxEncoded: maxP != nil, IsShared: shared} + + return mem, mem.Validate(memoryLimitPages) +} diff --git a/vendor/github.com/tetratelabs/wazero/internal/wasm/binary/names.go b/vendor/github.com/tetratelabs/wazero/internal/wasm/binary/names.go new file mode 100644 index 000000000..56fb96dc8 --- /dev/null +++ b/vendor/github.com/tetratelabs/wazero/internal/wasm/binary/names.go @@ -0,0 +1,151 @@ +package binary + +import ( + "bytes" + "fmt" + "io" + + "github.com/tetratelabs/wazero/internal/leb128" + "github.com/tetratelabs/wazero/internal/wasm" +) + +const ( + // subsectionIDModuleName contains only the module name. + subsectionIDModuleName = uint8(0) + // subsectionIDFunctionNames is a map of indices to function names, in ascending order by function index + subsectionIDFunctionNames = uint8(1) + // subsectionIDLocalNames contain a map of function indices to a map of local indices to their names, in ascending + // order by function and local index + subsectionIDLocalNames = uint8(2) +) + +// decodeNameSection deserializes the data associated with the "name" key in SectionIDCustom according to the +// standard: +// +// * ModuleName decode from subsection 0 +// * FunctionNames decode from subsection 1 +// * LocalNames decode from subsection 2 +// +// See https://www.w3.org/TR/2019/REC-wasm-core-1-20191205/#binary-namesec +func decodeNameSection(r *bytes.Reader, limit uint64) (result *wasm.NameSection, err error) { + // TODO: add leb128 functions that work on []byte and offset. While using a reader allows us to reuse reader-based + // leb128 functions, it is less efficient, causes untestable code and in some cases more complex vs plain []byte. + result = &wasm.NameSection{} + + // subsectionID is decoded if known, and skipped if not + var subsectionID uint8 + // subsectionSize is the length to skip when the subsectionID is unknown + var subsectionSize uint32 + var bytesRead uint64 + for limit > 0 { + if subsectionID, err = r.ReadByte(); err != nil { + if err == io.EOF { + return result, nil + } + // TODO: untestable as this can't fail for a reason beside EOF reading a byte from a buffer + return nil, fmt.Errorf("failed to read a subsection ID: %w", err) + } + limit-- + + if subsectionSize, bytesRead, err = leb128.DecodeUint32(r); err != nil { + return nil, fmt.Errorf("failed to read the size of subsection[%d]: %w", subsectionID, err) + } + limit -= bytesRead + + switch subsectionID { + case subsectionIDModuleName: + if result.ModuleName, _, err = decodeUTF8(r, "module name"); err != nil { + return nil, err + } + case subsectionIDFunctionNames: + if result.FunctionNames, err = decodeFunctionNames(r); err != nil { + return nil, err + } + case subsectionIDLocalNames: + if result.LocalNames, err = decodeLocalNames(r); err != nil { + return nil, err + } + default: // Skip other subsections. + // Note: Not Seek because it doesn't err when given an offset past EOF. Rather, it leads to undefined state. + if _, err = io.CopyN(io.Discard, r, int64(subsectionSize)); err != nil { + return nil, fmt.Errorf("failed to skip subsection[%d]: %w", subsectionID, err) + } + } + limit -= uint64(subsectionSize) + } + return +} + +func decodeFunctionNames(r *bytes.Reader) (wasm.NameMap, error) { + functionCount, err := decodeFunctionCount(r, subsectionIDFunctionNames) + if err != nil { + return nil, err + } + + result := make(wasm.NameMap, functionCount) + for i := uint32(0); i < functionCount; i++ { + functionIndex, err := decodeFunctionIndex(r, subsectionIDFunctionNames) + if err != nil { + return nil, err + } + + name, _, err := decodeUTF8(r, "function[%d] name", functionIndex) + if err != nil { + return nil, err + } + result[i] = wasm.NameAssoc{Index: functionIndex, Name: name} + } + return result, nil +} + +func decodeLocalNames(r *bytes.Reader) (wasm.IndirectNameMap, error) { + functionCount, err := decodeFunctionCount(r, subsectionIDLocalNames) + if err != nil { + return nil, err + } + + result := make(wasm.IndirectNameMap, functionCount) + for i := uint32(0); i < functionCount; i++ { + functionIndex, err := decodeFunctionIndex(r, subsectionIDLocalNames) + if err != nil { + return nil, err + } + + localCount, _, err := leb128.DecodeUint32(r) + if err != nil { + return nil, fmt.Errorf("failed to read the local count for function[%d]: %w", functionIndex, err) + } + + locals := make(wasm.NameMap, localCount) + for j := uint32(0); j < localCount; j++ { + localIndex, _, err := leb128.DecodeUint32(r) + if err != nil { + return nil, fmt.Errorf("failed to read a local index of function[%d]: %w", functionIndex, err) + } + + name, _, err := decodeUTF8(r, "function[%d] local[%d] name", functionIndex, localIndex) + if err != nil { + return nil, err + } + locals[j] = wasm.NameAssoc{Index: localIndex, Name: name} + } + result[i] = wasm.NameMapAssoc{Index: functionIndex, NameMap: locals} + } + return result, nil +} + +func decodeFunctionIndex(r *bytes.Reader, subsectionID uint8) (uint32, error) { + functionIndex, _, err := leb128.DecodeUint32(r) + if err != nil { + return 0, fmt.Errorf("failed to read a function index in subsection[%d]: %w", subsectionID, err) + } + return functionIndex, nil +} + +func decodeFunctionCount(r *bytes.Reader, subsectionID uint8) (uint32, error) { + functionCount, _, err := leb128.DecodeUint32(r) + if err != nil { + return 0, fmt.Errorf("failed to read the function count of subsection[%d]: %w", subsectionID, err) + } + return functionCount, nil +} diff --git a/vendor/github.com/tetratelabs/wazero/internal/wasm/binary/section.go b/vendor/github.com/tetratelabs/wazero/internal/wasm/binary/section.go new file mode 100644 index 000000000..622ee5923 --- /dev/null +++ b/vendor/github.com/tetratelabs/wazero/internal/wasm/binary/section.go @@ -0,0 +1,226 @@ +package binary + +import ( + "bytes" + "fmt" + "io" + + "github.com/tetratelabs/wazero/api" + "github.com/tetratelabs/wazero/internal/leb128" + "github.com/tetratelabs/wazero/internal/wasm" +) + +func decodeTypeSection(enabledFeatures api.CoreFeatures, r *bytes.Reader) ([]wasm.FunctionType, error) { + vs, _, err := leb128.DecodeUint32(r) + if err != nil { + return nil, fmt.Errorf("get size of vector: %w", err) + } + + result := make([]wasm.FunctionType, vs) + for i := uint32(0); i < vs; i++ { + if err = decodeFunctionType(enabledFeatures, r, &result[i]); err != nil { + return nil, fmt.Errorf("read %d-th type: %v", i, err) + } + } + return result, nil +} + +// decodeImportSection decodes the decoded import segments plus the count per wasm.ExternType. +func decodeImportSection( + r *bytes.Reader, + memorySizer memorySizer, + memoryLimitPages uint32, + enabledFeatures api.CoreFeatures, +) (result []wasm.Import, + perModule map[string][]*wasm.Import, + funcCount, globalCount, memoryCount, tableCount wasm.Index, err error, +) { + vs, _, err := leb128.DecodeUint32(r) + if err != nil { + err = fmt.Errorf("get size of vector: %w", err) + return + } + + perModule = make(map[string][]*wasm.Import) + result = make([]wasm.Import, vs) + for i := uint32(0); i < vs; i++ { + imp := &result[i] + if err = decodeImport(r, i, memorySizer, memoryLimitPages, enabledFeatures, imp); err != nil { + return + } + switch imp.Type { + case wasm.ExternTypeFunc: + imp.IndexPerType = funcCount + funcCount++ + case wasm.ExternTypeGlobal: + imp.IndexPerType = globalCount + globalCount++ + case wasm.ExternTypeMemory: + imp.IndexPerType = memoryCount + memoryCount++ + case wasm.ExternTypeTable: + imp.IndexPerType = tableCount + tableCount++ + } + perModule[imp.Module] = append(perModule[imp.Module], imp) + } + return +} + +func decodeFunctionSection(r *bytes.Reader) ([]uint32, error) { + vs, _, err := leb128.DecodeUint32(r) + if err != nil { + return nil, fmt.Errorf("get size of vector: %w", err) + } + + result := make([]uint32, vs) + for i := uint32(0); i < vs; i++ { + if result[i], _, err = leb128.DecodeUint32(r); err != nil { + return nil, fmt.Errorf("get type index: %w", err) + } + } + return result, err +} + +func decodeTableSection(r *bytes.Reader, enabledFeatures api.CoreFeatures) ([]wasm.Table, error) { + vs, _, err := leb128.DecodeUint32(r) + if err != nil { + return nil, fmt.Errorf("error reading size") + } + if vs > 1 { + if err := enabledFeatures.RequireEnabled(api.CoreFeatureReferenceTypes); err != nil { + return nil, fmt.Errorf("at most one table allowed in module as %w", err) + } + } + + ret := make([]wasm.Table, vs) + for i := range ret { + err = decodeTable(r, enabledFeatures, &ret[i]) + if err != nil { + return nil, err + } + } + return ret, nil +} + +func decodeMemorySection( + r *bytes.Reader, + enabledFeatures api.CoreFeatures, + memorySizer memorySizer, + memoryLimitPages uint32, +) (*wasm.Memory, error) { + vs, _, err := leb128.DecodeUint32(r) + if err != nil { + return nil, fmt.Errorf("error reading size") + } + if vs > 1 { + return nil, fmt.Errorf("at most one memory allowed in module, but read %d", vs) + } else if vs == 0 { + // memory count can be zero. + return nil, nil + } + + return decodeMemory(r, enabledFeatures, memorySizer, memoryLimitPages) +} + +func decodeGlobalSection(r *bytes.Reader, enabledFeatures api.CoreFeatures) ([]wasm.Global, error) { + vs, _, err := leb128.DecodeUint32(r) + if err != nil { + return nil, fmt.Errorf("get size of vector: %w", err) + } + + result := make([]wasm.Global, vs) + for i := uint32(0); i < vs; i++ { + if err = decodeGlobal(r, enabledFeatures, &result[i]); err != nil { + return nil, fmt.Errorf("global[%d]: %w", i, err) + } + } + return result, nil +} + +func decodeExportSection(r *bytes.Reader) ([]wasm.Export, map[string]*wasm.Export, error) { + vs, _, sizeErr := leb128.DecodeUint32(r) + if sizeErr != nil { + return nil, nil, fmt.Errorf("get size of vector: %v", sizeErr) + } + + exportMap := make(map[string]*wasm.Export, vs) + exportSection := make([]wasm.Export, vs) + for i := wasm.Index(0); i < vs; i++ { + export := &exportSection[i] + err := decodeExport(r, export) + if err != nil { + return nil, nil, fmt.Errorf("read export: %w", err) + } + if _, ok := exportMap[export.Name]; ok { + return nil, nil, fmt.Errorf("export[%d] duplicates name %q", i, export.Name) + } else { + exportMap[export.Name] = export + } + } + return exportSection, exportMap, nil +} + +func decodeStartSection(r *bytes.Reader) (*wasm.Index, error) { + vs, _, err := leb128.DecodeUint32(r) + if err != nil { + return nil, fmt.Errorf("get function index: %w", err) + } + return &vs, nil +} + +func decodeElementSection(r *bytes.Reader, enabledFeatures api.CoreFeatures) ([]wasm.ElementSegment, error) { + vs, _, err := leb128.DecodeUint32(r) + if err != nil { + return nil, fmt.Errorf("get size of vector: %w", err) + } + + result := make([]wasm.ElementSegment, vs) + for i := uint32(0); i < vs; i++ { + if err = decodeElementSegment(r, enabledFeatures, &result[i]); err != nil { + return nil, fmt.Errorf("read element: %w", err) + } + } + return result, nil +} + +func decodeCodeSection(r *bytes.Reader) ([]wasm.Code, error) { + codeSectionStart := uint64(r.Len()) + vs, _, err := leb128.DecodeUint32(r) + if err != nil { + return nil, fmt.Errorf("get size of vector: %w", err) + } + + result := make([]wasm.Code, vs) + for i := uint32(0); i < vs; i++ { + err = decodeCode(r, codeSectionStart, &result[i]) + if err != nil { + return nil, fmt.Errorf("read %d-th code segment: %v", i, err) + } + } + return result, nil +} + +func decodeDataSection(r *bytes.Reader, enabledFeatures api.CoreFeatures) ([]wasm.DataSegment, error) { + vs, _, err := leb128.DecodeUint32(r) + if err != nil { + return nil, fmt.Errorf("get size of vector: %w", err) + } + + result := make([]wasm.DataSegment, vs) + for i := uint32(0); i < vs; i++ { + if err = decodeDataSegment(r, enabledFeatures, &result[i]); err != nil { + return nil, fmt.Errorf("read data segment: %w", err) + } + } + return result, nil +} + +func decodeDataCountSection(r *bytes.Reader) (count *uint32, err error) { + v, _, err := leb128.DecodeUint32(r) + if err != nil && err != io.EOF { + // data count is optional, so EOF is fine. + return nil, err + } + return &v, nil +} diff --git a/vendor/github.com/tetratelabs/wazero/internal/wasm/binary/table.go b/vendor/github.com/tetratelabs/wazero/internal/wasm/binary/table.go new file mode 100644 index 000000000..353ec7566 --- /dev/null +++ b/vendor/github.com/tetratelabs/wazero/internal/wasm/binary/table.go @@ -0,0 +1,43 @@ +package binary + +import ( + "bytes" + "fmt" + + "github.com/tetratelabs/wazero/api" + "github.com/tetratelabs/wazero/internal/wasm" +) + +// decodeTable returns the wasm.Table decoded with the WebAssembly 1.0 (20191205) Binary Format. +// +// See https://www.w3.org/TR/2019/REC-wasm-core-1-20191205/#binary-table +func decodeTable(r *bytes.Reader, enabledFeatures api.CoreFeatures, ret *wasm.Table) (err error) { + ret.Type, err = r.ReadByte() + if err != nil { + return fmt.Errorf("read leading byte: %v", err) + } + + if ret.Type != wasm.RefTypeFuncref { + if err = enabledFeatures.RequireEnabled(api.CoreFeatureReferenceTypes); err != nil { + return fmt.Errorf("table type funcref is invalid: %w", err) + } + } + + var shared bool + ret.Min, ret.Max, shared, err = decodeLimitsType(r) + if err != nil { + return fmt.Errorf("read limits: %v", err) + } + if ret.Min > wasm.MaximumFunctionIndex { + return fmt.Errorf("table min must be at most %d", wasm.MaximumFunctionIndex) + } + if ret.Max != nil { + if *ret.Max < ret.Min { + return fmt.Errorf("table size minimum must not be greater than maximum") + } + } + if shared { + return fmt.Errorf("tables cannot be marked as shared") + } + return +} diff --git a/vendor/github.com/tetratelabs/wazero/internal/wasm/binary/value.go b/vendor/github.com/tetratelabs/wazero/internal/wasm/binary/value.go new file mode 100644 index 000000000..755ee5ea3 --- /dev/null +++ b/vendor/github.com/tetratelabs/wazero/internal/wasm/binary/value.go @@ -0,0 +1,60 @@ +package binary + +import ( + "bytes" + "fmt" + "io" + "unicode/utf8" + "unsafe" + + "github.com/tetratelabs/wazero/internal/leb128" + "github.com/tetratelabs/wazero/internal/wasm" +) + +func decodeValueTypes(r *bytes.Reader, num uint32) ([]wasm.ValueType, error) { + if num == 0 { + return nil, nil + } + + ret := make([]wasm.ValueType, num) + _, err := io.ReadFull(r, ret) + if err != nil { + return nil, err + } + + for _, v := range ret { + switch v { + case wasm.ValueTypeI32, wasm.ValueTypeF32, wasm.ValueTypeI64, wasm.ValueTypeF64, + wasm.ValueTypeExternref, wasm.ValueTypeFuncref, wasm.ValueTypeV128: + default: + return nil, fmt.Errorf("invalid value type: %d", v) + } + } + return ret, nil +} + +// decodeUTF8 decodes a size prefixed string from the reader, returning it and the count of bytes read. +// contextFormat and contextArgs apply an error format when present +func decodeUTF8(r *bytes.Reader, contextFormat string, contextArgs ...interface{}) (string, uint32, error) { + size, sizeOfSize, err := leb128.DecodeUint32(r) + if err != nil { + return "", 0, fmt.Errorf("failed to read %s size: %w", fmt.Sprintf(contextFormat, contextArgs...), err) + } + + if size == 0 { + return "", uint32(sizeOfSize), nil + } + + buf := make([]byte, size) + if _, err = io.ReadFull(r, buf); err != nil { + return "", 0, fmt.Errorf("failed to read %s: %w", fmt.Sprintf(contextFormat, contextArgs...), err) + } + + if !utf8.Valid(buf) { + return "", 0, fmt.Errorf("%s is not valid UTF-8", fmt.Sprintf(contextFormat, contextArgs...)) + } + + // TODO: use unsafe.String after flooring Go 1.20. + ret := *(*string)(unsafe.Pointer(&buf)) + return ret, size + uint32(sizeOfSize), nil +} diff --git a/vendor/github.com/tetratelabs/wazero/internal/wasm/counts.go b/vendor/github.com/tetratelabs/wazero/internal/wasm/counts.go new file mode 100644 index 000000000..685a40941 --- /dev/null +++ b/vendor/github.com/tetratelabs/wazero/internal/wasm/counts.go @@ -0,0 +1,51 @@ +package wasm + +import "fmt" + +// SectionElementCount returns the count of elements in a given section ID +// +// For example... +// * SectionIDType returns the count of FunctionType +// * SectionIDCustom returns the count of CustomSections plus one if NameSection is present +// * SectionIDHostFunction returns the count of HostFunctionSection +// * SectionIDExport returns the count of unique export names +func (m *Module) SectionElementCount(sectionID SectionID) uint32 { // element as in vector elements! + switch sectionID { + case SectionIDCustom: + numCustomSections := uint32(len(m.CustomSections)) + if m.NameSection != nil { + numCustomSections++ + } + return numCustomSections + case SectionIDType: + return uint32(len(m.TypeSection)) + case SectionIDImport: + return uint32(len(m.ImportSection)) + case SectionIDFunction: + return uint32(len(m.FunctionSection)) + case SectionIDTable: + return uint32(len(m.TableSection)) + case SectionIDMemory: + if m.MemorySection != nil { + return 1 + } + return 0 + case SectionIDGlobal: + return uint32(len(m.GlobalSection)) + case SectionIDExport: + return uint32(len(m.ExportSection)) + case SectionIDStart: + if m.StartSection != nil { + return 1 + } + return 0 + case SectionIDElement: + return uint32(len(m.ElementSection)) + case SectionIDCode: + return uint32(len(m.CodeSection)) + case SectionIDData: + return uint32(len(m.DataSection)) + default: + panic(fmt.Errorf("BUG: unknown section: %d", sectionID)) + } +} diff --git a/vendor/github.com/tetratelabs/wazero/internal/wasm/engine.go b/vendor/github.com/tetratelabs/wazero/internal/wasm/engine.go new file mode 100644 index 000000000..58a458217 --- /dev/null +++ b/vendor/github.com/tetratelabs/wazero/internal/wasm/engine.go @@ -0,0 +1,72 @@ +package wasm + +import ( + "context" + + "github.com/tetratelabs/wazero/api" + "github.com/tetratelabs/wazero/experimental" +) + +// Engine is a Store-scoped mechanism to compile functions declared or imported by a module. +// This is a top-level type implemented by an interpreter or compiler. +type Engine interface { + // Close closes this engine, and releases all the compiled cache. + Close() (err error) + + // CompileModule implements the same method as documented on wasm.Engine. + CompileModule(ctx context.Context, module *Module, listeners []experimental.FunctionListener, ensureTermination bool) error + + // CompiledModuleCount is exported for testing, to track the size of the compilation cache. + CompiledModuleCount() uint32 + + // DeleteCompiledModule releases compilation caches for the given module (source). + // Note: it is safe to call this function for a module from which module instances are instantiated even when these + // module instances have outstanding calls. + DeleteCompiledModule(module *Module) + + // NewModuleEngine compiles down the function instances in a module, and returns ModuleEngine for the module. + // + // * module is the source module from which moduleFunctions are instantiated. This is used for caching. + // * instance is the *ModuleInstance which is created from `module`. + // + // Note: Input parameters must be pre-validated with wasm.Module Validate, to ensure no fields are invalid + // due to reasons such as out-of-bounds. + NewModuleEngine(module *Module, instance *ModuleInstance) (ModuleEngine, error) +} + +// ModuleEngine implements function calls for a given module. +type ModuleEngine interface { + // DoneInstantiation is called at the end of the instantiation of the module. + DoneInstantiation() + + // NewFunction returns an api.Function for the given function pointed by the given Index. + NewFunction(index Index) api.Function + + // ResolveImportedFunction is used to add imported functions needed to make this ModuleEngine fully functional. + // - `index` is the function Index of this imported function. + // - `indexInImportedModule` is the function Index of the imported function in the imported module. + // - `importedModuleEngine` is the ModuleEngine for the imported ModuleInstance. + ResolveImportedFunction(index, indexInImportedModule Index, importedModuleEngine ModuleEngine) + + // ResolveImportedMemory is called when this module imports a memory from another module. + ResolveImportedMemory(importedModuleEngine ModuleEngine) + + // LookupFunction returns the FunctionModule and the Index of the function in the returned ModuleInstance at the given offset in the table. + LookupFunction(t *TableInstance, typeId FunctionTypeID, tableOffset Index) (*ModuleInstance, Index) + + // GetGlobalValue returns the value of the global variable at the given Index. + // Only called when OwnsGlobals() returns true, and must not be called for imported globals + GetGlobalValue(idx Index) (lo, hi uint64) + + // SetGlobalValue sets the value of the global variable at the given Index. + // Only called when OwnsGlobals() returns true, and must not be called for imported globals + SetGlobalValue(idx Index, lo, hi uint64) + + // OwnsGlobals returns true if this ModuleEngine owns the global variables. If true, wasm.GlobalInstance's Val,ValHi should + // not be accessed directly. + OwnsGlobals() bool + + // FunctionInstanceReference returns Reference for the given Index for a FunctionInstance. The returned values are used by + // the initialization via ElementSegment. + FunctionInstanceReference(funcIndex Index) Reference +} diff --git a/vendor/github.com/tetratelabs/wazero/internal/wasm/func_validation.go b/vendor/github.com/tetratelabs/wazero/internal/wasm/func_validation.go new file mode 100644 index 000000000..8da689076 --- /dev/null +++ b/vendor/github.com/tetratelabs/wazero/internal/wasm/func_validation.go @@ -0,0 +1,2340 @@ +package wasm + +import ( + "bytes" + "errors" + "fmt" + "strconv" + "strings" + + "github.com/tetratelabs/wazero/api" + "github.com/tetratelabs/wazero/experimental" + "github.com/tetratelabs/wazero/internal/leb128" +) + +// The wazero specific limitation described at RATIONALE.md. +const maximumValuesOnStack = 1 << 27 + +// validateFunction validates the instruction sequence of a function. +// following the specification https://www.w3.org/TR/2019/REC-wasm-core-1-20191205/#instructions%E2%91%A2. +// +// * idx is the index in the FunctionSection +// * functions are the function index, which is prefixed by imports. The value is the TypeSection index. +// * globals are the global index, which is prefixed by imports. +// * memory is the potentially imported memory and can be nil. +// * table is the potentially imported table and can be nil. +// * declaredFunctionIndexes is the set of function indexes declared by declarative element segments which can be acceed by OpcodeRefFunc instruction. +// +// Returns an error if the instruction sequence is not valid, +// or potentially it can exceed the maximum number of values on the stack. +func (m *Module) validateFunction(sts *stacks, enabledFeatures api.CoreFeatures, idx Index, functions []Index, + globals []GlobalType, memory *Memory, tables []Table, declaredFunctionIndexes map[Index]struct{}, br *bytes.Reader, +) error { + return m.validateFunctionWithMaxStackValues(sts, enabledFeatures, idx, functions, globals, memory, tables, maximumValuesOnStack, declaredFunctionIndexes, br) +} + +func readMemArg(pc uint64, body []byte) (align, offset uint32, read uint64, err error) { + align, num, err := leb128.LoadUint32(body[pc:]) + if err != nil { + err = fmt.Errorf("read memory align: %v", err) + return + } + read += num + + offset, num, err = leb128.LoadUint32(body[pc+num:]) + if err != nil { + err = fmt.Errorf("read memory offset: %v", err) + return + } + + read += num + return align, offset, read, nil +} + +// validateFunctionWithMaxStackValues is like validateFunction, but allows overriding maxStackValues for testing. +// +// * stacks is to track the state of Wasm value and control frame stacks at anypoint of execution, and reused to reduce allocation. +// * maxStackValues is the maximum height of values stack which the target is allowed to reach. +func (m *Module) validateFunctionWithMaxStackValues( + sts *stacks, + enabledFeatures api.CoreFeatures, + idx Index, + functions []Index, + globals []GlobalType, + memory *Memory, + tables []Table, + maxStackValues int, + declaredFunctionIndexes map[Index]struct{}, + br *bytes.Reader, +) error { + nonStaticLocals := make(map[Index]struct{}) + if len(m.NonStaticLocals) > 0 { + m.NonStaticLocals[idx] = nonStaticLocals + } + + functionType := &m.TypeSection[m.FunctionSection[idx]] + code := &m.CodeSection[idx] + body := code.Body + localTypes := code.LocalTypes + + sts.reset(functionType) + valueTypeStack := &sts.vs + // We start with the outermost control block which is for function return if the code branches into it. + controlBlockStack := &sts.cs + + // Now start walking through all the instructions in the body while tracking + // control blocks and value types to check the validity of all instructions. + for pc := uint64(0); pc < uint64(len(body)); pc++ { + op := body[pc] + if false { + var instName string + if op == OpcodeMiscPrefix { + instName = MiscInstructionName(body[pc+1]) + } else if op == OpcodeVecPrefix { + instName = VectorInstructionName(body[pc+1]) + } else if op == OpcodeAtomicPrefix { + instName = AtomicInstructionName(body[pc+1]) + } else { + instName = InstructionName(op) + } + fmt.Printf("handling %s, stack=%s, blocks: %v\n", instName, valueTypeStack.stack, controlBlockStack) + } + + if len(controlBlockStack.stack) == 0 { + return fmt.Errorf("unexpected end of function at pc=%#x", pc) + } + + if OpcodeI32Load <= op && op <= OpcodeI64Store32 { + if memory == nil { + return fmt.Errorf("memory must exist for %s", InstructionName(op)) + } + pc++ + align, _, read, err := readMemArg(pc, body) + if err != nil { + return err + } + pc += read - 1 + switch op { + case OpcodeI32Load: + if 1< 32/8 { + return fmt.Errorf("invalid memory alignment") + } + if err := valueTypeStack.popAndVerifyType(ValueTypeI32); err != nil { + return err + } + valueTypeStack.push(ValueTypeI32) + case OpcodeF32Load: + if 1< 32/8 { + return fmt.Errorf("invalid memory alignment") + } + if err := valueTypeStack.popAndVerifyType(ValueTypeI32); err != nil { + return err + } + valueTypeStack.push(ValueTypeF32) + case OpcodeI32Store: + if 1< 32/8 { + return fmt.Errorf("invalid memory alignment") + } + if err := valueTypeStack.popAndVerifyType(ValueTypeI32); err != nil { + return err + } + if err := valueTypeStack.popAndVerifyType(ValueTypeI32); err != nil { + return err + } + case OpcodeF32Store: + if 1< 32/8 { + return fmt.Errorf("invalid memory alignment") + } + if err := valueTypeStack.popAndVerifyType(ValueTypeF32); err != nil { + return err + } + if err := valueTypeStack.popAndVerifyType(ValueTypeI32); err != nil { + return err + } + case OpcodeI64Load: + if 1< 64/8 { + return fmt.Errorf("invalid memory alignment") + } + if err := valueTypeStack.popAndVerifyType(ValueTypeI32); err != nil { + return err + } + valueTypeStack.push(ValueTypeI64) + case OpcodeF64Load: + if 1< 64/8 { + return fmt.Errorf("invalid memory alignment") + } + if err := valueTypeStack.popAndVerifyType(ValueTypeI32); err != nil { + return err + } + valueTypeStack.push(ValueTypeF64) + case OpcodeI64Store: + if 1< 64/8 { + return fmt.Errorf("invalid memory alignment") + } + if err := valueTypeStack.popAndVerifyType(ValueTypeI64); err != nil { + return err + } + if err := valueTypeStack.popAndVerifyType(ValueTypeI32); err != nil { + return err + } + case OpcodeF64Store: + if 1< 64/8 { + return fmt.Errorf("invalid memory alignment") + } + if err := valueTypeStack.popAndVerifyType(ValueTypeF64); err != nil { + return err + } + if err := valueTypeStack.popAndVerifyType(ValueTypeI32); err != nil { + return err + } + case OpcodeI32Load8S: + if 1< 1 { + return fmt.Errorf("invalid memory alignment") + } + if err := valueTypeStack.popAndVerifyType(ValueTypeI32); err != nil { + return err + } + valueTypeStack.push(ValueTypeI32) + case OpcodeI32Load8U: + if 1< 1 { + return fmt.Errorf("invalid memory alignment") + } + if err := valueTypeStack.popAndVerifyType(ValueTypeI32); err != nil { + return err + } + valueTypeStack.push(ValueTypeI32) + case OpcodeI64Load8S, OpcodeI64Load8U: + if 1< 1 { + return fmt.Errorf("invalid memory alignment") + } + if err := valueTypeStack.popAndVerifyType(ValueTypeI32); err != nil { + return err + } + valueTypeStack.push(ValueTypeI64) + case OpcodeI32Store8: + if 1< 1 { + return fmt.Errorf("invalid memory alignment") + } + if err := valueTypeStack.popAndVerifyType(ValueTypeI32); err != nil { + return err + } + if err := valueTypeStack.popAndVerifyType(ValueTypeI32); err != nil { + return err + } + case OpcodeI64Store8: + if 1< 1 { + return fmt.Errorf("invalid memory alignment") + } + if err := valueTypeStack.popAndVerifyType(ValueTypeI64); err != nil { + return err + } + if err := valueTypeStack.popAndVerifyType(ValueTypeI32); err != nil { + return err + } + case OpcodeI32Load16S, OpcodeI32Load16U: + if 1< 16/8 { + return fmt.Errorf("invalid memory alignment") + } + if err := valueTypeStack.popAndVerifyType(ValueTypeI32); err != nil { + return err + } + valueTypeStack.push(ValueTypeI32) + case OpcodeI64Load16S, OpcodeI64Load16U: + if 1< 16/8 { + return fmt.Errorf("invalid memory alignment") + } + if err := valueTypeStack.popAndVerifyType(ValueTypeI32); err != nil { + return err + } + valueTypeStack.push(ValueTypeI64) + case OpcodeI32Store16: + if 1< 16/8 { + return fmt.Errorf("invalid memory alignment") + } + if err := valueTypeStack.popAndVerifyType(ValueTypeI32); err != nil { + return err + } + if err := valueTypeStack.popAndVerifyType(ValueTypeI32); err != nil { + return err + } + case OpcodeI64Store16: + if 1< 16/8 { + return fmt.Errorf("invalid memory alignment") + } + if err := valueTypeStack.popAndVerifyType(ValueTypeI64); err != nil { + return err + } + if err := valueTypeStack.popAndVerifyType(ValueTypeI32); err != nil { + return err + } + case OpcodeI64Load32S, OpcodeI64Load32U: + if 1< 32/8 { + return fmt.Errorf("invalid memory alignment") + } + if err := valueTypeStack.popAndVerifyType(ValueTypeI32); err != nil { + return err + } + valueTypeStack.push(ValueTypeI64) + case OpcodeI64Store32: + if 1< 32/8 { + return fmt.Errorf("invalid memory alignment") + } + if err := valueTypeStack.popAndVerifyType(ValueTypeI64); err != nil { + return err + } + if err := valueTypeStack.popAndVerifyType(ValueTypeI32); err != nil { + return err + } + } + } else if OpcodeMemorySize <= op && op <= OpcodeMemoryGrow { + if memory == nil { + return fmt.Errorf("memory must exist for %s", InstructionName(op)) + } + pc++ + val, num, err := leb128.LoadUint32(body[pc:]) + if err != nil { + return fmt.Errorf("read immediate: %v", err) + } + if val != 0 || num != 1 { + return fmt.Errorf("memory instruction reserved bytes not zero with 1 byte") + } + switch Opcode(op) { + case OpcodeMemoryGrow: + if err := valueTypeStack.popAndVerifyType(ValueTypeI32); err != nil { + return err + } + valueTypeStack.push(ValueTypeI32) + case OpcodeMemorySize: + valueTypeStack.push(ValueTypeI32) + } + pc += num - 1 + } else if OpcodeI32Const <= op && op <= OpcodeF64Const { + pc++ + switch Opcode(op) { + case OpcodeI32Const: + _, num, err := leb128.LoadInt32(body[pc:]) + if err != nil { + return fmt.Errorf("read i32 immediate: %s", err) + } + pc += num - 1 + valueTypeStack.push(ValueTypeI32) + case OpcodeI64Const: + _, num, err := leb128.LoadInt64(body[pc:]) + if err != nil { + return fmt.Errorf("read i64 immediate: %v", err) + } + valueTypeStack.push(ValueTypeI64) + pc += num - 1 + case OpcodeF32Const: + valueTypeStack.push(ValueTypeF32) + pc += 3 + case OpcodeF64Const: + valueTypeStack.push(ValueTypeF64) + pc += 7 + } + } else if OpcodeLocalGet <= op && op <= OpcodeGlobalSet { + pc++ + index, num, err := leb128.LoadUint32(body[pc:]) + if err != nil { + return fmt.Errorf("read immediate: %v", err) + } + pc += num - 1 + switch op { + case OpcodeLocalGet: + inputLen := uint32(len(functionType.Params)) + if l := uint32(len(localTypes)) + inputLen; index >= l { + return fmt.Errorf("invalid local index for %s %d >= %d(=len(locals)+len(parameters))", + OpcodeLocalGetName, index, l) + } + if index < inputLen { + valueTypeStack.push(functionType.Params[index]) + } else { + valueTypeStack.push(localTypes[index-inputLen]) + } + case OpcodeLocalSet: + inputLen := uint32(len(functionType.Params)) + if l := uint32(len(localTypes)) + inputLen; index >= l { + return fmt.Errorf("invalid local index for %s %d >= %d(=len(locals)+len(parameters))", + OpcodeLocalSetName, index, l) + } + nonStaticLocals[index] = struct{}{} + var expType ValueType + if index < inputLen { + expType = functionType.Params[index] + } else { + expType = localTypes[index-inputLen] + } + if err := valueTypeStack.popAndVerifyType(expType); err != nil { + return err + } + case OpcodeLocalTee: + inputLen := uint32(len(functionType.Params)) + if l := uint32(len(localTypes)) + inputLen; index >= l { + return fmt.Errorf("invalid local index for %s %d >= %d(=len(locals)+len(parameters))", + OpcodeLocalTeeName, index, l) + } + nonStaticLocals[index] = struct{}{} + var expType ValueType + if index < inputLen { + expType = functionType.Params[index] + } else { + expType = localTypes[index-inputLen] + } + if err := valueTypeStack.popAndVerifyType(expType); err != nil { + return err + } + valueTypeStack.push(expType) + case OpcodeGlobalGet: + if index >= uint32(len(globals)) { + return fmt.Errorf("invalid index for %s", OpcodeGlobalGetName) + } + valueTypeStack.push(globals[index].ValType) + case OpcodeGlobalSet: + if index >= uint32(len(globals)) { + return fmt.Errorf("invalid global index") + } else if !globals[index].Mutable { + return fmt.Errorf("%s when not mutable", OpcodeGlobalSetName) + } else if err := valueTypeStack.popAndVerifyType( + globals[index].ValType); err != nil { + return err + } + } + } else if op == OpcodeBr { + pc++ + index, num, err := leb128.LoadUint32(body[pc:]) + if err != nil { + return fmt.Errorf("read immediate: %v", err) + } else if int(index) >= len(controlBlockStack.stack) { + return fmt.Errorf("invalid %s operation: index out of range", OpcodeBrName) + } + pc += num - 1 + // Check type soundness. + target := &controlBlockStack.stack[len(controlBlockStack.stack)-int(index)-1] + var targetResultType []ValueType + if target.op == OpcodeLoop { + targetResultType = target.blockType.Params + } else { + targetResultType = target.blockType.Results + } + if err = valueTypeStack.popResults(op, targetResultType, false); err != nil { + return err + } + // br instruction is stack-polymorphic. + valueTypeStack.unreachable() + } else if op == OpcodeBrIf { + pc++ + index, num, err := leb128.LoadUint32(body[pc:]) + if err != nil { + return fmt.Errorf("read immediate: %v", err) + } else if int(index) >= len(controlBlockStack.stack) { + return fmt.Errorf( + "invalid ln param given for %s: index=%d with %d for the current label stack length", + OpcodeBrIfName, index, len(controlBlockStack.stack)) + } + pc += num - 1 + if err := valueTypeStack.popAndVerifyType(ValueTypeI32); err != nil { + return fmt.Errorf("cannot pop the required operand for %s", OpcodeBrIfName) + } + // Check type soundness. + target := &controlBlockStack.stack[len(controlBlockStack.stack)-int(index)-1] + var targetResultType []ValueType + if target.op == OpcodeLoop { + targetResultType = target.blockType.Params + } else { + targetResultType = target.blockType.Results + } + if err := valueTypeStack.popResults(op, targetResultType, false); err != nil { + return err + } + // Push back the result + for _, t := range targetResultType { + valueTypeStack.push(t) + } + } else if op == OpcodeBrTable { + pc++ + br.Reset(body[pc:]) + nl, num, err := leb128.DecodeUint32(br) + if err != nil { + return fmt.Errorf("read immediate: %w", err) + } + + list := make([]uint32, nl) + for i := uint32(0); i < nl; i++ { + l, n, err := leb128.DecodeUint32(br) + if err != nil { + return fmt.Errorf("read immediate: %w", err) + } + num += n + list[i] = l + } + ln, n, err := leb128.DecodeUint32(br) + if err != nil { + return fmt.Errorf("read immediate: %w", err) + } else if int(ln) >= len(controlBlockStack.stack) { + return fmt.Errorf( + "invalid ln param given for %s: ln=%d with %d for the current label stack length", + OpcodeBrTableName, ln, len(controlBlockStack.stack)) + } + pc += n + num - 1 + // Check type soundness. + if err := valueTypeStack.popAndVerifyType(ValueTypeI32); err != nil { + return fmt.Errorf("cannot pop the required operand for %s", OpcodeBrTableName) + } + lnLabel := &controlBlockStack.stack[len(controlBlockStack.stack)-1-int(ln)] + var defaultLabelType []ValueType + // Below, we might modify the slice in case of unreachable. Therefore, + // we have to copy the content of block result types, otherwise the original + // function type might result in invalid value types if the block is the outermost label + // which equals the function's type. + if lnLabel.op != OpcodeLoop { // Loop operation doesn't require results since the continuation is the beginning of the loop. + defaultLabelType = make([]ValueType, len(lnLabel.blockType.Results)) + copy(defaultLabelType, lnLabel.blockType.Results) + } else { + defaultLabelType = make([]ValueType, len(lnLabel.blockType.Params)) + copy(defaultLabelType, lnLabel.blockType.Params) + } + + if enabledFeatures.IsEnabled(api.CoreFeatureReferenceTypes) { + // As of reference-types proposal, br_table on unreachable state + // can choose unknown types for expected parameter types for each label. + // https://github.com/WebAssembly/reference-types/pull/116 + for i := range defaultLabelType { + index := len(defaultLabelType) - 1 - i + exp := defaultLabelType[index] + actual, err := valueTypeStack.pop() + if err != nil { + return err + } + if actual == valueTypeUnknown { + // Re-assign the expected type to unknown. + defaultLabelType[index] = valueTypeUnknown + } else if actual != exp { + return typeMismatchError(true, OpcodeBrTableName, actual, exp, i) + } + } + } else { + if err = valueTypeStack.popResults(op, defaultLabelType, false); err != nil { + return err + } + } + + for _, l := range list { + if int(l) >= len(controlBlockStack.stack) { + return fmt.Errorf("invalid l param given for %s", OpcodeBrTableName) + } + label := &controlBlockStack.stack[len(controlBlockStack.stack)-1-int(l)] + var tableLabelType []ValueType + if label.op != OpcodeLoop { + tableLabelType = label.blockType.Results + } else { + tableLabelType = label.blockType.Params + } + if len(defaultLabelType) != len(tableLabelType) { + return fmt.Errorf("inconsistent block type length for %s at %d; %v (ln=%d) != %v (l=%d)", OpcodeBrTableName, l, defaultLabelType, ln, tableLabelType, l) + } + for i := range defaultLabelType { + if defaultLabelType[i] != valueTypeUnknown && defaultLabelType[i] != tableLabelType[i] { + return fmt.Errorf("incosistent block type for %s at %d", OpcodeBrTableName, l) + } + } + } + + // br_table instruction is stack-polymorphic. + valueTypeStack.unreachable() + } else if op == OpcodeCall { + pc++ + index, num, err := leb128.LoadUint32(body[pc:]) + if err != nil { + return fmt.Errorf("read immediate: %v", err) + } + pc += num - 1 + if int(index) >= len(functions) { + return fmt.Errorf("invalid function index") + } + funcType := &m.TypeSection[functions[index]] + for i := 0; i < len(funcType.Params); i++ { + if err := valueTypeStack.popAndVerifyType(funcType.Params[len(funcType.Params)-1-i]); err != nil { + return fmt.Errorf("type mismatch on %s operation param type: %v", OpcodeCallName, err) + } + } + for _, exp := range funcType.Results { + valueTypeStack.push(exp) + } + } else if op == OpcodeCallIndirect { + pc++ + typeIndex, num, err := leb128.LoadUint32(body[pc:]) + if err != nil { + return fmt.Errorf("read immediate: %v", err) + } + pc += num + + if int(typeIndex) >= len(m.TypeSection) { + return fmt.Errorf("invalid type index at %s: %d", OpcodeCallIndirectName, typeIndex) + } + + tableIndex, num, err := leb128.LoadUint32(body[pc:]) + if err != nil { + return fmt.Errorf("read table index: %v", err) + } + pc += num - 1 + if tableIndex != 0 { + if err := enabledFeatures.RequireEnabled(api.CoreFeatureReferenceTypes); err != nil { + return fmt.Errorf("table index must be zero but was %d: %w", tableIndex, err) + } + } + + if tableIndex >= uint32(len(tables)) { + return fmt.Errorf("unknown table index: %d", tableIndex) + } + + table := tables[tableIndex] + if table.Type != RefTypeFuncref { + return fmt.Errorf("table is not funcref type but was %s for %s", RefTypeName(table.Type), OpcodeCallIndirectName) + } + + if err = valueTypeStack.popAndVerifyType(ValueTypeI32); err != nil { + return fmt.Errorf("cannot pop the offset in table for %s", OpcodeCallIndirectName) + } + funcType := &m.TypeSection[typeIndex] + for i := 0; i < len(funcType.Params); i++ { + if err = valueTypeStack.popAndVerifyType(funcType.Params[len(funcType.Params)-1-i]); err != nil { + return fmt.Errorf("type mismatch on %s operation input type", OpcodeCallIndirectName) + } + } + for _, exp := range funcType.Results { + valueTypeStack.push(exp) + } + } else if OpcodeI32Eqz <= op && op <= OpcodeI64Extend32S { + switch op { + case OpcodeI32Eqz: + if err := valueTypeStack.popAndVerifyType(ValueTypeI32); err != nil { + return fmt.Errorf("cannot pop the operand for %s: %v", OpcodeI32EqzName, err) + } + valueTypeStack.push(ValueTypeI32) + case OpcodeI32Eq, OpcodeI32Ne, OpcodeI32LtS, + OpcodeI32LtU, OpcodeI32GtS, OpcodeI32GtU, OpcodeI32LeS, + OpcodeI32LeU, OpcodeI32GeS, OpcodeI32GeU: + if err := valueTypeStack.popAndVerifyType(ValueTypeI32); err != nil { + return fmt.Errorf("cannot pop the 1st i32 operand for %s: %v", InstructionName(op), err) + } + if err := valueTypeStack.popAndVerifyType(ValueTypeI32); err != nil { + return fmt.Errorf("cannot pop the 2nd i32 operand for %s: %v", InstructionName(op), err) + } + valueTypeStack.push(ValueTypeI32) + case OpcodeI64Eqz: + if err := valueTypeStack.popAndVerifyType(ValueTypeI64); err != nil { + return fmt.Errorf("cannot pop the operand for %s: %v", OpcodeI64EqzName, err) + } + valueTypeStack.push(ValueTypeI32) + case OpcodeI64Eq, OpcodeI64Ne, OpcodeI64LtS, + OpcodeI64LtU, OpcodeI64GtS, OpcodeI64GtU, + OpcodeI64LeS, OpcodeI64LeU, OpcodeI64GeS, OpcodeI64GeU: + if err := valueTypeStack.popAndVerifyType(ValueTypeI64); err != nil { + return fmt.Errorf("cannot pop the 1st i64 operand for %s: %v", InstructionName(op), err) + } + if err := valueTypeStack.popAndVerifyType(ValueTypeI64); err != nil { + return fmt.Errorf("cannot pop the 2nd i64 operand for %s: %v", InstructionName(op), err) + } + valueTypeStack.push(ValueTypeI32) + case OpcodeF32Eq, OpcodeF32Ne, OpcodeF32Lt, OpcodeF32Gt, OpcodeF32Le, OpcodeF32Ge: + if err := valueTypeStack.popAndVerifyType(ValueTypeF32); err != nil { + return fmt.Errorf("cannot pop the 1st f32 operand for %s: %v", InstructionName(op), err) + } + if err := valueTypeStack.popAndVerifyType(ValueTypeF32); err != nil { + return fmt.Errorf("cannot pop the 2nd f32 operand for %s: %v", InstructionName(op), err) + } + valueTypeStack.push(ValueTypeI32) + case OpcodeF64Eq, OpcodeF64Ne, OpcodeF64Lt, OpcodeF64Gt, OpcodeF64Le, OpcodeF64Ge: + if err := valueTypeStack.popAndVerifyType(ValueTypeF64); err != nil { + return fmt.Errorf("cannot pop the 1st f64 operand for %s: %v", InstructionName(op), err) + } + if err := valueTypeStack.popAndVerifyType(ValueTypeF64); err != nil { + return fmt.Errorf("cannot pop the 2nd f64 operand for %s: %v", InstructionName(op), err) + } + valueTypeStack.push(ValueTypeI32) + case OpcodeI32Clz, OpcodeI32Ctz, OpcodeI32Popcnt: + if err := valueTypeStack.popAndVerifyType(ValueTypeI32); err != nil { + return fmt.Errorf("cannot pop the i32 operand for %s: %v", InstructionName(op), err) + } + valueTypeStack.push(ValueTypeI32) + case OpcodeI32Add, OpcodeI32Sub, OpcodeI32Mul, OpcodeI32DivS, + OpcodeI32DivU, OpcodeI32RemS, OpcodeI32RemU, OpcodeI32And, + OpcodeI32Or, OpcodeI32Xor, OpcodeI32Shl, OpcodeI32ShrS, + OpcodeI32ShrU, OpcodeI32Rotl, OpcodeI32Rotr: + if err := valueTypeStack.popAndVerifyType(ValueTypeI32); err != nil { + return fmt.Errorf("cannot pop the 1st operand for %s: %v", InstructionName(op), err) + } + if err := valueTypeStack.popAndVerifyType(ValueTypeI32); err != nil { + return fmt.Errorf("cannot pop the 2nd operand for %s: %v", InstructionName(op), err) + } + valueTypeStack.push(ValueTypeI32) + case OpcodeI64Clz, OpcodeI64Ctz, OpcodeI64Popcnt: + if err := valueTypeStack.popAndVerifyType(ValueTypeI64); err != nil { + return fmt.Errorf("cannot pop the i64 operand for %s: %v", InstructionName(op), err) + } + valueTypeStack.push(ValueTypeI64) + case OpcodeI64Add, OpcodeI64Sub, OpcodeI64Mul, OpcodeI64DivS, + OpcodeI64DivU, OpcodeI64RemS, OpcodeI64RemU, OpcodeI64And, + OpcodeI64Or, OpcodeI64Xor, OpcodeI64Shl, OpcodeI64ShrS, + OpcodeI64ShrU, OpcodeI64Rotl, OpcodeI64Rotr: + if err := valueTypeStack.popAndVerifyType(ValueTypeI64); err != nil { + return fmt.Errorf("cannot pop the 1st i64 operand for %s: %v", InstructionName(op), err) + } + if err := valueTypeStack.popAndVerifyType(ValueTypeI64); err != nil { + return fmt.Errorf("cannot pop the 2nd i64 operand for %s: %v", InstructionName(op), err) + } + valueTypeStack.push(ValueTypeI64) + case OpcodeF32Abs, OpcodeF32Neg, OpcodeF32Ceil, + OpcodeF32Floor, OpcodeF32Trunc, OpcodeF32Nearest, + OpcodeF32Sqrt: + if err := valueTypeStack.popAndVerifyType(ValueTypeF32); err != nil { + return fmt.Errorf("cannot pop the 1st f32 operand for %s: %v", InstructionName(op), err) + } + valueTypeStack.push(ValueTypeF32) + case OpcodeF32Add, OpcodeF32Sub, OpcodeF32Mul, + OpcodeF32Div, OpcodeF32Min, OpcodeF32Max, + OpcodeF32Copysign: + if err := valueTypeStack.popAndVerifyType(ValueTypeF32); err != nil { + return fmt.Errorf("cannot pop the 1st f32 operand for %s: %v", InstructionName(op), err) + } + if err := valueTypeStack.popAndVerifyType(ValueTypeF32); err != nil { + return fmt.Errorf("cannot pop the 2nd f32 operand for %s: %v", InstructionName(op), err) + } + valueTypeStack.push(ValueTypeF32) + case OpcodeF64Abs, OpcodeF64Neg, OpcodeF64Ceil, + OpcodeF64Floor, OpcodeF64Trunc, OpcodeF64Nearest, + OpcodeF64Sqrt: + if err := valueTypeStack.popAndVerifyType(ValueTypeF64); err != nil { + return fmt.Errorf("cannot pop the 1st f64 operand for %s: %v", InstructionName(op), err) + } + valueTypeStack.push(ValueTypeF64) + case OpcodeF64Add, OpcodeF64Sub, OpcodeF64Mul, + OpcodeF64Div, OpcodeF64Min, OpcodeF64Max, + OpcodeF64Copysign: + if err := valueTypeStack.popAndVerifyType(ValueTypeF64); err != nil { + return fmt.Errorf("cannot pop the 1st f64 operand for %s: %v", InstructionName(op), err) + } + if err := valueTypeStack.popAndVerifyType(ValueTypeF64); err != nil { + return fmt.Errorf("cannot pop the 2nd f64 operand for %s: %v", InstructionName(op), err) + } + valueTypeStack.push(ValueTypeF64) + case OpcodeI32WrapI64: + if err := valueTypeStack.popAndVerifyType(ValueTypeI64); err != nil { + return fmt.Errorf("cannot pop the operand for %s: %v", OpcodeI32WrapI64Name, err) + } + valueTypeStack.push(ValueTypeI32) + case OpcodeI32TruncF32S, OpcodeI32TruncF32U: + if err := valueTypeStack.popAndVerifyType(ValueTypeF32); err != nil { + return fmt.Errorf("cannot pop the f32 operand for %s: %v", InstructionName(op), err) + } + valueTypeStack.push(ValueTypeI32) + case OpcodeI32TruncF64S, OpcodeI32TruncF64U: + if err := valueTypeStack.popAndVerifyType(ValueTypeF64); err != nil { + return fmt.Errorf("cannot pop the f64 operand for %s: %v", InstructionName(op), err) + } + valueTypeStack.push(ValueTypeI32) + case OpcodeI64ExtendI32S, OpcodeI64ExtendI32U: + if err := valueTypeStack.popAndVerifyType(ValueTypeI32); err != nil { + return fmt.Errorf("cannot pop the i32 operand for %s: %v", InstructionName(op), err) + } + valueTypeStack.push(ValueTypeI64) + case OpcodeI64TruncF32S, OpcodeI64TruncF32U: + if err := valueTypeStack.popAndVerifyType(ValueTypeF32); err != nil { + return fmt.Errorf("cannot pop the f32 operand for %s: %v", InstructionName(op), err) + } + valueTypeStack.push(ValueTypeI64) + case OpcodeI64TruncF64S, OpcodeI64TruncF64U: + if err := valueTypeStack.popAndVerifyType(ValueTypeF64); err != nil { + return fmt.Errorf("cannot pop the f64 operand for %s: %v", InstructionName(op), err) + } + valueTypeStack.push(ValueTypeI64) + case OpcodeF32ConvertI32S, OpcodeF32ConvertI32U: + if err := valueTypeStack.popAndVerifyType(ValueTypeI32); err != nil { + return fmt.Errorf("cannot pop the i32 operand for %s: %v", InstructionName(op), err) + } + valueTypeStack.push(ValueTypeF32) + case OpcodeF32ConvertI64S, OpcodeF32ConvertI64U: + if err := valueTypeStack.popAndVerifyType(ValueTypeI64); err != nil { + return fmt.Errorf("cannot pop the i64 operand for %s: %v", InstructionName(op), err) + } + valueTypeStack.push(ValueTypeF32) + case OpcodeF32DemoteF64: + if err := valueTypeStack.popAndVerifyType(ValueTypeF64); err != nil { + return fmt.Errorf("cannot pop the operand for %s: %v", OpcodeF32DemoteF64Name, err) + } + valueTypeStack.push(ValueTypeF32) + case OpcodeF64ConvertI32S, OpcodeF64ConvertI32U: + if err := valueTypeStack.popAndVerifyType(ValueTypeI32); err != nil { + return fmt.Errorf("cannot pop the i32 operand for %s: %v", InstructionName(op), err) + } + valueTypeStack.push(ValueTypeF64) + case OpcodeF64ConvertI64S, OpcodeF64ConvertI64U: + if err := valueTypeStack.popAndVerifyType(ValueTypeI64); err != nil { + return fmt.Errorf("cannot pop the i64 operand for %s: %v", InstructionName(op), err) + } + valueTypeStack.push(ValueTypeF64) + case OpcodeF64PromoteF32: + if err := valueTypeStack.popAndVerifyType(ValueTypeF32); err != nil { + return fmt.Errorf("cannot pop the operand for %s: %v", OpcodeF64PromoteF32Name, err) + } + valueTypeStack.push(ValueTypeF64) + case OpcodeI32ReinterpretF32: + if err := valueTypeStack.popAndVerifyType(ValueTypeF32); err != nil { + return fmt.Errorf("cannot pop the operand for %s: %v", OpcodeI32ReinterpretF32Name, err) + } + valueTypeStack.push(ValueTypeI32) + case OpcodeI64ReinterpretF64: + if err := valueTypeStack.popAndVerifyType(ValueTypeF64); err != nil { + return fmt.Errorf("cannot pop the operand for %s: %v", OpcodeI64ReinterpretF64Name, err) + } + valueTypeStack.push(ValueTypeI64) + case OpcodeF32ReinterpretI32: + if err := valueTypeStack.popAndVerifyType(ValueTypeI32); err != nil { + return fmt.Errorf("cannot pop the operand for %s: %v", OpcodeF32ReinterpretI32Name, err) + } + valueTypeStack.push(ValueTypeF32) + case OpcodeF64ReinterpretI64: + if err := valueTypeStack.popAndVerifyType(ValueTypeI64); err != nil { + return fmt.Errorf("cannot pop the operand for %s: %v", OpcodeF64ReinterpretI64Name, err) + } + valueTypeStack.push(ValueTypeF64) + case OpcodeI32Extend8S, OpcodeI32Extend16S: + if err := enabledFeatures.RequireEnabled(api.CoreFeatureSignExtensionOps); err != nil { + return fmt.Errorf("%s invalid as %v", instructionNames[op], err) + } + if err := valueTypeStack.popAndVerifyType(ValueTypeI32); err != nil { + return fmt.Errorf("cannot pop the operand for %s: %v", instructionNames[op], err) + } + valueTypeStack.push(ValueTypeI32) + case OpcodeI64Extend8S, OpcodeI64Extend16S, OpcodeI64Extend32S: + if err := enabledFeatures.RequireEnabled(api.CoreFeatureSignExtensionOps); err != nil { + return fmt.Errorf("%s invalid as %v", instructionNames[op], err) + } + if err := valueTypeStack.popAndVerifyType(ValueTypeI64); err != nil { + return fmt.Errorf("cannot pop the operand for %s: %v", instructionNames[op], err) + } + valueTypeStack.push(ValueTypeI64) + default: + return fmt.Errorf("invalid numeric instruction 0x%x", op) + } + } else if op >= OpcodeRefNull && op <= OpcodeRefFunc { + if err := enabledFeatures.RequireEnabled(api.CoreFeatureReferenceTypes); err != nil { + return fmt.Errorf("%s invalid as %v", instructionNames[op], err) + } + switch op { + case OpcodeRefNull: + pc++ + switch reftype := body[pc]; reftype { + case ValueTypeExternref: + valueTypeStack.push(ValueTypeExternref) + case ValueTypeFuncref: + valueTypeStack.push(ValueTypeFuncref) + default: + return fmt.Errorf("unknown type for ref.null: 0x%x", reftype) + } + case OpcodeRefIsNull: + tp, err := valueTypeStack.pop() + if err != nil { + return fmt.Errorf("cannot pop the operand for ref.is_null: %v", err) + } else if !isReferenceValueType(tp) && tp != valueTypeUnknown { + return fmt.Errorf("type mismatch: expected reference type but was %s", ValueTypeName(tp)) + } + valueTypeStack.push(ValueTypeI32) + case OpcodeRefFunc: + pc++ + index, num, err := leb128.LoadUint32(body[pc:]) + if err != nil { + return fmt.Errorf("failed to read function index for ref.func: %v", err) + } + if _, ok := declaredFunctionIndexes[index]; !ok { + return fmt.Errorf("undeclared function index %d for ref.func", index) + } + pc += num - 1 + valueTypeStack.push(ValueTypeFuncref) + } + } else if op == OpcodeTableGet || op == OpcodeTableSet { + if err := enabledFeatures.RequireEnabled(api.CoreFeatureReferenceTypes); err != nil { + return fmt.Errorf("%s is invalid as %v", InstructionName(op), err) + } + pc++ + tableIndex, num, err := leb128.LoadUint32(body[pc:]) + if err != nil { + return fmt.Errorf("read immediate: %v", err) + } + if tableIndex >= uint32(len(tables)) { + return fmt.Errorf("table of index %d not found", tableIndex) + } + + refType := tables[tableIndex].Type + if op == OpcodeTableGet { + if err := valueTypeStack.popAndVerifyType(api.ValueTypeI32); err != nil { + return fmt.Errorf("cannot pop the operand for table.get: %v", err) + } + valueTypeStack.push(refType) + } else { + if err := valueTypeStack.popAndVerifyType(refType); err != nil { + return fmt.Errorf("cannot pop the operand for table.set: %v", err) + } + if err := valueTypeStack.popAndVerifyType(api.ValueTypeI32); err != nil { + return fmt.Errorf("cannot pop the operand for table.set: %v", err) + } + } + pc += num - 1 + } else if op == OpcodeMiscPrefix { + pc++ + // A misc opcode is encoded as an unsigned variable 32-bit integer. + miscOp32, num, err := leb128.LoadUint32(body[pc:]) + if err != nil { + return fmt.Errorf("failed to read misc opcode: %v", err) + } + pc += num - 1 + miscOpcode := byte(miscOp32) + // If the misc opcode is beyond byte range, it is highly likely this is an invalid binary, or + // it is due to the new opcode from a new proposal. In the latter case, we have to + // change the alias type of OpcodeMisc (which is currently byte) to uint32. + if uint32(byte(miscOp32)) != miscOp32 { + return fmt.Errorf("invalid misc opcode: %#x", miscOp32) + } + if miscOpcode >= OpcodeMiscI32TruncSatF32S && miscOpcode <= OpcodeMiscI64TruncSatF64U { + if err := enabledFeatures.RequireEnabled(api.CoreFeatureNonTrappingFloatToIntConversion); err != nil { + return fmt.Errorf("%s invalid as %v", miscInstructionNames[miscOpcode], err) + } + var inType, outType ValueType + switch miscOpcode { + case OpcodeMiscI32TruncSatF32S, OpcodeMiscI32TruncSatF32U: + inType, outType = ValueTypeF32, ValueTypeI32 + case OpcodeMiscI32TruncSatF64S, OpcodeMiscI32TruncSatF64U: + inType, outType = ValueTypeF64, ValueTypeI32 + case OpcodeMiscI64TruncSatF32S, OpcodeMiscI64TruncSatF32U: + inType, outType = ValueTypeF32, ValueTypeI64 + case OpcodeMiscI64TruncSatF64S, OpcodeMiscI64TruncSatF64U: + inType, outType = ValueTypeF64, ValueTypeI64 + } + if err := valueTypeStack.popAndVerifyType(inType); err != nil { + return fmt.Errorf("cannot pop the operand for %s: %v", miscInstructionNames[miscOpcode], err) + } + valueTypeStack.push(outType) + } else if miscOpcode >= OpcodeMiscMemoryInit && miscOpcode <= OpcodeMiscTableCopy { + if err := enabledFeatures.RequireEnabled(api.CoreFeatureBulkMemoryOperations); err != nil { + return fmt.Errorf("%s invalid as %v", miscInstructionNames[miscOpcode], err) + } + var params []ValueType + // Handle opcodes added in bulk-memory-operations/WebAssembly 2.0. + switch miscOpcode { + case OpcodeMiscDataDrop: + if m.DataCountSection == nil { + return fmt.Errorf("%s requires data count section", MiscInstructionName(miscOpcode)) + } + + // We need to read the index to the data section. + pc++ + index, num, err := leb128.LoadUint32(body[pc:]) + if err != nil { + return fmt.Errorf("failed to read data segment index for %s: %v", MiscInstructionName(miscOpcode), err) + } + if int(index) >= len(m.DataSection) { + return fmt.Errorf("index %d out of range of data section(len=%d)", index, len(m.DataSection)) + } + pc += num - 1 + case OpcodeMiscMemoryInit, OpcodeMiscMemoryCopy, OpcodeMiscMemoryFill: + if memory == nil { + return fmt.Errorf("memory must exist for %s", MiscInstructionName(miscOpcode)) + } + params = []ValueType{ValueTypeI32, ValueTypeI32, ValueTypeI32} + + if miscOpcode == OpcodeMiscMemoryInit { + if m.DataCountSection == nil { + return fmt.Errorf("%s requires data count section", MiscInstructionName(miscOpcode)) + } + + // We need to read the index to the data section. + pc++ + index, num, err := leb128.LoadUint32(body[pc:]) + if err != nil { + return fmt.Errorf("failed to read data segment index for %s: %v", MiscInstructionName(miscOpcode), err) + } + if int(index) >= len(m.DataSection) { + return fmt.Errorf("index %d out of range of data section(len=%d)", index, len(m.DataSection)) + } + pc += num - 1 + } + + pc++ + val, num, err := leb128.LoadUint32(body[pc:]) + if err != nil { + return fmt.Errorf("failed to read memory index for %s: %v", MiscInstructionName(miscOpcode), err) + } + if val != 0 || num != 1 { + return fmt.Errorf("%s reserved byte must be zero encoded with 1 byte", MiscInstructionName(miscOpcode)) + } + if miscOpcode == OpcodeMiscMemoryCopy { + pc++ + // memory.copy needs two memory index which are reserved as zero. + val, num, err := leb128.LoadUint32(body[pc:]) + if err != nil { + return fmt.Errorf("failed to read memory index for %s: %v", MiscInstructionName(miscOpcode), err) + } + if val != 0 || num != 1 { + return fmt.Errorf("%s reserved byte must be zero encoded with 1 byte", MiscInstructionName(miscOpcode)) + } + } + + case OpcodeMiscTableInit: + params = []ValueType{ValueTypeI32, ValueTypeI32, ValueTypeI32} + pc++ + elementIndex, num, err := leb128.LoadUint32(body[pc:]) + if err != nil { + return fmt.Errorf("failed to read element segment index for %s: %v", MiscInstructionName(miscOpcode), err) + } + if int(elementIndex) >= len(m.ElementSection) { + return fmt.Errorf("index %d out of range of element section(len=%d)", elementIndex, len(m.ElementSection)) + } + pc += num + + tableIndex, num, err := leb128.LoadUint32(body[pc:]) + if err != nil { + return fmt.Errorf("failed to read source table index for %s: %v", MiscInstructionName(miscOpcode), err) + } + if tableIndex != 0 { + if err := enabledFeatures.RequireEnabled(api.CoreFeatureReferenceTypes); err != nil { + return fmt.Errorf("source table index must be zero for %s as %v", MiscInstructionName(miscOpcode), err) + } + } + if tableIndex >= uint32(len(tables)) { + return fmt.Errorf("table of index %d not found", tableIndex) + } + + if m.ElementSection[elementIndex].Type != tables[tableIndex].Type { + return fmt.Errorf("type mismatch for table.init: element type %s does not match table type %s", + RefTypeName(m.ElementSection[elementIndex].Type), + RefTypeName(tables[tableIndex].Type), + ) + } + pc += num - 1 + case OpcodeMiscElemDrop: + pc++ + elementIndex, num, err := leb128.LoadUint32(body[pc:]) + if err != nil { + return fmt.Errorf("failed to read element segment index for %s: %v", MiscInstructionName(miscOpcode), err) + } else if int(elementIndex) >= len(m.ElementSection) { + return fmt.Errorf("index %d out of range of element section(len=%d)", elementIndex, len(m.ElementSection)) + } + pc += num - 1 + case OpcodeMiscTableCopy: + params = []ValueType{ValueTypeI32, ValueTypeI32, ValueTypeI32} + pc++ + + dstTableIndex, num, err := leb128.LoadUint32(body[pc:]) + if err != nil { + return fmt.Errorf("failed to read destination table index for %s: %v", MiscInstructionName(miscOpcode), err) + } + if dstTableIndex != 0 { + if err := enabledFeatures.RequireEnabled(api.CoreFeatureReferenceTypes); err != nil { + return fmt.Errorf("destination table index must be zero for %s as %v", MiscInstructionName(miscOpcode), err) + } + } + if dstTableIndex >= uint32(len(tables)) { + return fmt.Errorf("table of index %d not found", dstTableIndex) + } + pc += num + + srcTableIndex, num, err := leb128.LoadUint32(body[pc:]) + if err != nil { + return fmt.Errorf("failed to read source table index for %s: %v", MiscInstructionName(miscOpcode), err) + } + if srcTableIndex != 0 { + if err := enabledFeatures.RequireEnabled(api.CoreFeatureReferenceTypes); err != nil { + return fmt.Errorf("source table index must be zero for %s as %v", MiscInstructionName(miscOpcode), err) + } + } + if srcTableIndex >= uint32(len(tables)) { + return fmt.Errorf("table of index %d not found", srcTableIndex) + } + + if tables[srcTableIndex].Type != tables[dstTableIndex].Type { + return fmt.Errorf("table type mismatch for table.copy: %s (src) != %s (dst)", + RefTypeName(tables[srcTableIndex].Type), RefTypeName(tables[dstTableIndex].Type)) + } + + pc += num - 1 + } + for _, p := range params { + if err := valueTypeStack.popAndVerifyType(p); err != nil { + return fmt.Errorf("cannot pop the operand for %s: %v", miscInstructionNames[miscOpcode], err) + } + } + } else if miscOpcode >= OpcodeMiscTableGrow && miscOpcode <= OpcodeMiscTableFill { + if err := enabledFeatures.RequireEnabled(api.CoreFeatureReferenceTypes); err != nil { + return fmt.Errorf("%s invalid as %v", miscInstructionNames[miscOpcode], err) + } + + pc++ + tableIndex, num, err := leb128.LoadUint32(body[pc:]) + if err != nil { + return fmt.Errorf("failed to read table index for %s: %v", MiscInstructionName(miscOpcode), err) + } + if tableIndex >= uint32(len(tables)) { + return fmt.Errorf("table of index %d not found", tableIndex) + } + pc += num - 1 + + var params, results []ValueType + reftype := tables[tableIndex].Type + if miscOpcode == OpcodeMiscTableGrow { + params = []ValueType{ValueTypeI32, reftype} + results = []ValueType{ValueTypeI32} + } else if miscOpcode == OpcodeMiscTableSize { + results = []ValueType{ValueTypeI32} + } else if miscOpcode == OpcodeMiscTableFill { + params = []ValueType{ValueTypeI32, reftype, ValueTypeI32} + } + + for _, p := range params { + if err := valueTypeStack.popAndVerifyType(p); err != nil { + return fmt.Errorf("cannot pop the operand for %s: %v", miscInstructionNames[miscOpcode], err) + } + } + for _, r := range results { + valueTypeStack.push(r) + } + } else { + return fmt.Errorf("unknown misc opcode %#x", miscOpcode) + } + } else if op == OpcodeVecPrefix { + pc++ + // Vector instructions come with two bytes where the first byte is always OpcodeVecPrefix, + // and the second byte determines the actual instruction. + vecOpcode := body[pc] + if err := enabledFeatures.RequireEnabled(api.CoreFeatureSIMD); err != nil { + return fmt.Errorf("%s invalid as %v", vectorInstructionName[vecOpcode], err) + } + + switch vecOpcode { + case OpcodeVecV128Const: + // Read 128-bit = 16 bytes constants + if int(pc+16) >= len(body) { + return fmt.Errorf("cannot read constant vector value for %s", vectorInstructionName[vecOpcode]) + } + pc += 16 + valueTypeStack.push(ValueTypeV128) + case OpcodeVecV128AnyTrue, OpcodeVecI8x16AllTrue, OpcodeVecI16x8AllTrue, OpcodeVecI32x4AllTrue, OpcodeVecI64x2AllTrue, + OpcodeVecI8x16BitMask, OpcodeVecI16x8BitMask, OpcodeVecI32x4BitMask, OpcodeVecI64x2BitMask: + if err := valueTypeStack.popAndVerifyType(ValueTypeV128); err != nil { + return fmt.Errorf("cannot pop the operand for %s: %v", vectorInstructionName[vecOpcode], err) + } + valueTypeStack.push(ValueTypeI32) + case OpcodeVecV128Load, OpcodeVecV128Load8x8s, OpcodeVecV128Load8x8u, OpcodeVecV128Load16x4s, OpcodeVecV128Load16x4u, + OpcodeVecV128Load32x2s, OpcodeVecV128Load32x2u, OpcodeVecV128Load8Splat, OpcodeVecV128Load16Splat, + OpcodeVecV128Load32Splat, OpcodeVecV128Load64Splat, + OpcodeVecV128Load32zero, OpcodeVecV128Load64zero: + if memory == nil { + return fmt.Errorf("memory must exist for %s", VectorInstructionName(vecOpcode)) + } + pc++ + align, _, read, err := readMemArg(pc, body) + if err != nil { + return err + } + pc += read - 1 + var maxAlign uint32 + switch vecOpcode { + case OpcodeVecV128Load: + maxAlign = 128 / 8 + case OpcodeVecV128Load8x8s, OpcodeVecV128Load8x8u, OpcodeVecV128Load16x4s, OpcodeVecV128Load16x4u, + OpcodeVecV128Load32x2s, OpcodeVecV128Load32x2u: + maxAlign = 64 / 8 + case OpcodeVecV128Load8Splat: + maxAlign = 1 + case OpcodeVecV128Load16Splat: + maxAlign = 16 / 8 + case OpcodeVecV128Load32Splat: + maxAlign = 32 / 8 + case OpcodeVecV128Load64Splat: + maxAlign = 64 / 8 + case OpcodeVecV128Load32zero: + maxAlign = 32 / 8 + case OpcodeVecV128Load64zero: + maxAlign = 64 / 8 + } + + if 1< maxAlign { + return fmt.Errorf("invalid memory alignment %d for %s", align, VectorInstructionName(vecOpcode)) + } + if err := valueTypeStack.popAndVerifyType(ValueTypeI32); err != nil { + return fmt.Errorf("cannot pop the operand for %s: %v", VectorInstructionName(vecOpcode), err) + } + valueTypeStack.push(ValueTypeV128) + case OpcodeVecV128Store: + if memory == nil { + return fmt.Errorf("memory must exist for %s", VectorInstructionName(vecOpcode)) + } + pc++ + align, _, read, err := readMemArg(pc, body) + if err != nil { + return err + } + pc += read - 1 + if 1< 128/8 { + return fmt.Errorf("invalid memory alignment %d for %s", align, OpcodeVecV128StoreName) + } + if err := valueTypeStack.popAndVerifyType(ValueTypeV128); err != nil { + return fmt.Errorf("cannot pop the operand for %s: %v", OpcodeVecV128StoreName, err) + } + if err := valueTypeStack.popAndVerifyType(ValueTypeI32); err != nil { + return fmt.Errorf("cannot pop the operand for %s: %v", OpcodeVecV128StoreName, err) + } + case OpcodeVecV128Load8Lane, OpcodeVecV128Load16Lane, OpcodeVecV128Load32Lane, OpcodeVecV128Load64Lane: + if memory == nil { + return fmt.Errorf("memory must exist for %s", VectorInstructionName(vecOpcode)) + } + attr := vecLoadLanes[vecOpcode] + pc++ + align, _, read, err := readMemArg(pc, body) + if err != nil { + return err + } + if 1< attr.alignMax { + return fmt.Errorf("invalid memory alignment %d for %s", align, vectorInstructionName[vecOpcode]) + } + pc += read + if pc >= uint64(len(body)) { + return fmt.Errorf("lane for %s not found", OpcodeVecV128Load64LaneName) + } + lane := body[pc] + if lane >= attr.laneCeil { + return fmt.Errorf("invalid lane index %d >= %d for %s", lane, attr.laneCeil, vectorInstructionName[vecOpcode]) + } + if err := valueTypeStack.popAndVerifyType(ValueTypeV128); err != nil { + return fmt.Errorf("cannot pop the operand for %s: %v", vectorInstructionName[vecOpcode], err) + } + if err := valueTypeStack.popAndVerifyType(ValueTypeI32); err != nil { + return fmt.Errorf("cannot pop the operand for %s: %v", vectorInstructionName[vecOpcode], err) + } + valueTypeStack.push(ValueTypeV128) + case OpcodeVecV128Store8Lane, OpcodeVecV128Store16Lane, OpcodeVecV128Store32Lane, OpcodeVecV128Store64Lane: + if memory == nil { + return fmt.Errorf("memory must exist for %s", VectorInstructionName(vecOpcode)) + } + attr := vecStoreLanes[vecOpcode] + pc++ + align, _, read, err := readMemArg(pc, body) + if err != nil { + return err + } + if 1< attr.alignMax { + return fmt.Errorf("invalid memory alignment %d for %s", align, vectorInstructionName[vecOpcode]) + } + pc += read + if pc >= uint64(len(body)) { + return fmt.Errorf("lane for %s not found", vectorInstructionName[vecOpcode]) + } + lane := body[pc] + if lane >= attr.laneCeil { + return fmt.Errorf("invalid lane index %d >= %d for %s", lane, attr.laneCeil, vectorInstructionName[vecOpcode]) + } + if err := valueTypeStack.popAndVerifyType(ValueTypeV128); err != nil { + return fmt.Errorf("cannot pop the operand for %s: %v", vectorInstructionName[vecOpcode], err) + } + if err := valueTypeStack.popAndVerifyType(ValueTypeI32); err != nil { + return fmt.Errorf("cannot pop the operand for %s: %v", vectorInstructionName[vecOpcode], err) + } + case OpcodeVecI8x16ExtractLaneS, + OpcodeVecI8x16ExtractLaneU, + OpcodeVecI16x8ExtractLaneS, + OpcodeVecI16x8ExtractLaneU, + OpcodeVecI32x4ExtractLane, + OpcodeVecI64x2ExtractLane, + OpcodeVecF32x4ExtractLane, + OpcodeVecF64x2ExtractLane: + pc++ + if pc >= uint64(len(body)) { + return fmt.Errorf("lane for %s not found", vectorInstructionName[vecOpcode]) + } + attr := vecExtractLanes[vecOpcode] + lane := body[pc] + if lane >= attr.laneCeil { + return fmt.Errorf("invalid lane index %d >= %d for %s", lane, attr.laneCeil, vectorInstructionName[vecOpcode]) + } + if err := valueTypeStack.popAndVerifyType(ValueTypeV128); err != nil { + return fmt.Errorf("cannot pop the operand for %s: %v", vectorInstructionName[vecOpcode], err) + } + valueTypeStack.push(attr.resultType) + case OpcodeVecI8x16ReplaceLane, OpcodeVecI16x8ReplaceLane, OpcodeVecI32x4ReplaceLane, + OpcodeVecI64x2ReplaceLane, OpcodeVecF32x4ReplaceLane, OpcodeVecF64x2ReplaceLane: + pc++ + if pc >= uint64(len(body)) { + return fmt.Errorf("lane for %s not found", vectorInstructionName[vecOpcode]) + } + attr := vecReplaceLanes[vecOpcode] + lane := body[pc] + if lane >= attr.laneCeil { + return fmt.Errorf("invalid lane index %d >= %d for %s", lane, attr.laneCeil, vectorInstructionName[vecOpcode]) + } + if err := valueTypeStack.popAndVerifyType(attr.paramType); err != nil { + return fmt.Errorf("cannot pop the operand for %s: %v", vectorInstructionName[vecOpcode], err) + } + if err := valueTypeStack.popAndVerifyType(ValueTypeV128); err != nil { + return fmt.Errorf("cannot pop the operand for %s: %v", vectorInstructionName[vecOpcode], err) + } + valueTypeStack.push(ValueTypeV128) + case OpcodeVecI8x16Splat, OpcodeVecI16x8Splat, OpcodeVecI32x4Splat, + OpcodeVecI64x2Splat, OpcodeVecF32x4Splat, OpcodeVecF64x2Splat: + tp := vecSplatValueTypes[vecOpcode] + if err := valueTypeStack.popAndVerifyType(tp); err != nil { + return fmt.Errorf("cannot pop the operand for %s: %v", vectorInstructionName[vecOpcode], err) + } + valueTypeStack.push(ValueTypeV128) + case OpcodeVecI8x16Swizzle, OpcodeVecV128And, OpcodeVecV128Or, OpcodeVecV128Xor, OpcodeVecV128AndNot: + if err := valueTypeStack.popAndVerifyType(ValueTypeV128); err != nil { + return fmt.Errorf("cannot pop the operand for %s: %v", vectorInstructionName[vecOpcode], err) + } + if err := valueTypeStack.popAndVerifyType(ValueTypeV128); err != nil { + return fmt.Errorf("cannot pop the operand for %s: %v", vectorInstructionName[vecOpcode], err) + } + valueTypeStack.push(ValueTypeV128) + case OpcodeVecV128Bitselect: + if err := valueTypeStack.popAndVerifyType(ValueTypeV128); err != nil { + return fmt.Errorf("cannot pop the operand for %s: %v", vectorInstructionName[vecOpcode], err) + } + if err := valueTypeStack.popAndVerifyType(ValueTypeV128); err != nil { + return fmt.Errorf("cannot pop the operand for %s: %v", vectorInstructionName[vecOpcode], err) + } + if err := valueTypeStack.popAndVerifyType(ValueTypeV128); err != nil { + return fmt.Errorf("cannot pop the operand for %s: %v", vectorInstructionName[vecOpcode], err) + } + valueTypeStack.push(ValueTypeV128) + case OpcodeVecV128Not: + if err := valueTypeStack.popAndVerifyType(ValueTypeV128); err != nil { + return fmt.Errorf("cannot pop the operand for %s: %v", vectorInstructionName[vecOpcode], err) + } + valueTypeStack.push(ValueTypeV128) + case OpcodeVecV128i8x16Shuffle: + pc++ + if pc+15 >= uint64(len(body)) { + return fmt.Errorf("16 lane indexes for %s not found", vectorInstructionName[vecOpcode]) + } + lanes := body[pc : pc+16] + for i, l := range lanes { + if l >= 32 { + return fmt.Errorf("invalid lane index[%d] %d >= %d for %s", i, l, 32, vectorInstructionName[vecOpcode]) + } + } + if err := valueTypeStack.popAndVerifyType(ValueTypeV128); err != nil { + return fmt.Errorf("cannot pop the operand for %s: %v", vectorInstructionName[vecOpcode], err) + } + if err := valueTypeStack.popAndVerifyType(ValueTypeV128); err != nil { + return fmt.Errorf("cannot pop the operand for %s: %v", vectorInstructionName[vecOpcode], err) + } + valueTypeStack.push(ValueTypeV128) + pc += 15 + case OpcodeVecI8x16Shl, OpcodeVecI8x16ShrS, OpcodeVecI8x16ShrU, + OpcodeVecI16x8Shl, OpcodeVecI16x8ShrS, OpcodeVecI16x8ShrU, + OpcodeVecI32x4Shl, OpcodeVecI32x4ShrS, OpcodeVecI32x4ShrU, + OpcodeVecI64x2Shl, OpcodeVecI64x2ShrS, OpcodeVecI64x2ShrU: + if err := valueTypeStack.popAndVerifyType(ValueTypeI32); err != nil { + return fmt.Errorf("cannot pop the operand for %s: %v", vectorInstructionName[vecOpcode], err) + } + if err := valueTypeStack.popAndVerifyType(ValueTypeV128); err != nil { + return fmt.Errorf("cannot pop the operand for %s: %v", vectorInstructionName[vecOpcode], err) + } + valueTypeStack.push(ValueTypeV128) + case OpcodeVecI8x16Eq, OpcodeVecI8x16Ne, OpcodeVecI8x16LtS, OpcodeVecI8x16LtU, OpcodeVecI8x16GtS, + OpcodeVecI8x16GtU, OpcodeVecI8x16LeS, OpcodeVecI8x16LeU, OpcodeVecI8x16GeS, OpcodeVecI8x16GeU, + OpcodeVecI16x8Eq, OpcodeVecI16x8Ne, OpcodeVecI16x8LtS, OpcodeVecI16x8LtU, OpcodeVecI16x8GtS, + OpcodeVecI16x8GtU, OpcodeVecI16x8LeS, OpcodeVecI16x8LeU, OpcodeVecI16x8GeS, OpcodeVecI16x8GeU, + OpcodeVecI32x4Eq, OpcodeVecI32x4Ne, OpcodeVecI32x4LtS, OpcodeVecI32x4LtU, OpcodeVecI32x4GtS, + OpcodeVecI32x4GtU, OpcodeVecI32x4LeS, OpcodeVecI32x4LeU, OpcodeVecI32x4GeS, OpcodeVecI32x4GeU, + OpcodeVecI64x2Eq, OpcodeVecI64x2Ne, OpcodeVecI64x2LtS, OpcodeVecI64x2GtS, OpcodeVecI64x2LeS, + OpcodeVecI64x2GeS, OpcodeVecF32x4Eq, OpcodeVecF32x4Ne, OpcodeVecF32x4Lt, OpcodeVecF32x4Gt, + OpcodeVecF32x4Le, OpcodeVecF32x4Ge, OpcodeVecF64x2Eq, OpcodeVecF64x2Ne, OpcodeVecF64x2Lt, + OpcodeVecF64x2Gt, OpcodeVecF64x2Le, OpcodeVecF64x2Ge, + OpcodeVecI32x4DotI16x8S, + OpcodeVecI8x16NarrowI16x8S, OpcodeVecI8x16NarrowI16x8U, OpcodeVecI16x8NarrowI32x4S, OpcodeVecI16x8NarrowI32x4U: + if err := valueTypeStack.popAndVerifyType(ValueTypeV128); err != nil { + return fmt.Errorf("cannot pop the operand for %s: %v", vectorInstructionName[vecOpcode], err) + } + if err := valueTypeStack.popAndVerifyType(ValueTypeV128); err != nil { + return fmt.Errorf("cannot pop the operand for %s: %v", vectorInstructionName[vecOpcode], err) + } + valueTypeStack.push(ValueTypeV128) + case OpcodeVecI8x16Neg, OpcodeVecI16x8Neg, OpcodeVecI32x4Neg, OpcodeVecI64x2Neg, OpcodeVecF32x4Neg, OpcodeVecF64x2Neg, + OpcodeVecF32x4Sqrt, OpcodeVecF64x2Sqrt, + OpcodeVecI8x16Abs, OpcodeVecI8x16Popcnt, OpcodeVecI16x8Abs, OpcodeVecI32x4Abs, OpcodeVecI64x2Abs, + OpcodeVecF32x4Abs, OpcodeVecF64x2Abs, + OpcodeVecF32x4Ceil, OpcodeVecF32x4Floor, OpcodeVecF32x4Trunc, OpcodeVecF32x4Nearest, + OpcodeVecF64x2Ceil, OpcodeVecF64x2Floor, OpcodeVecF64x2Trunc, OpcodeVecF64x2Nearest, + OpcodeVecI16x8ExtendLowI8x16S, OpcodeVecI16x8ExtendHighI8x16S, OpcodeVecI16x8ExtendLowI8x16U, OpcodeVecI16x8ExtendHighI8x16U, + OpcodeVecI32x4ExtendLowI16x8S, OpcodeVecI32x4ExtendHighI16x8S, OpcodeVecI32x4ExtendLowI16x8U, OpcodeVecI32x4ExtendHighI16x8U, + OpcodeVecI64x2ExtendLowI32x4S, OpcodeVecI64x2ExtendHighI32x4S, OpcodeVecI64x2ExtendLowI32x4U, OpcodeVecI64x2ExtendHighI32x4U, + OpcodeVecI16x8ExtaddPairwiseI8x16S, OpcodeVecI16x8ExtaddPairwiseI8x16U, + OpcodeVecI32x4ExtaddPairwiseI16x8S, OpcodeVecI32x4ExtaddPairwiseI16x8U, + OpcodeVecF64x2PromoteLowF32x4Zero, OpcodeVecF32x4DemoteF64x2Zero, + OpcodeVecF32x4ConvertI32x4S, OpcodeVecF32x4ConvertI32x4U, + OpcodeVecF64x2ConvertLowI32x4S, OpcodeVecF64x2ConvertLowI32x4U, + OpcodeVecI32x4TruncSatF32x4S, OpcodeVecI32x4TruncSatF32x4U, OpcodeVecI32x4TruncSatF64x2SZero, OpcodeVecI32x4TruncSatF64x2UZero: + if err := valueTypeStack.popAndVerifyType(ValueTypeV128); err != nil { + return fmt.Errorf("cannot pop the operand for %s: %v", vectorInstructionName[vecOpcode], err) + } + valueTypeStack.push(ValueTypeV128) + + case OpcodeVecI8x16Add, OpcodeVecI8x16AddSatS, OpcodeVecI8x16AddSatU, OpcodeVecI8x16Sub, OpcodeVecI8x16SubSatS, OpcodeVecI8x16SubSatU, + OpcodeVecI16x8Add, OpcodeVecI16x8AddSatS, OpcodeVecI16x8AddSatU, OpcodeVecI16x8Sub, OpcodeVecI16x8SubSatS, OpcodeVecI16x8SubSatU, OpcodeVecI16x8Mul, + OpcodeVecI32x4Add, OpcodeVecI32x4Sub, OpcodeVecI32x4Mul, + OpcodeVecI64x2Add, OpcodeVecI64x2Sub, OpcodeVecI64x2Mul, + OpcodeVecF32x4Add, OpcodeVecF32x4Sub, OpcodeVecF32x4Mul, OpcodeVecF32x4Div, + OpcodeVecF64x2Add, OpcodeVecF64x2Sub, OpcodeVecF64x2Mul, OpcodeVecF64x2Div, + OpcodeVecI8x16MinS, OpcodeVecI8x16MinU, OpcodeVecI8x16MaxS, OpcodeVecI8x16MaxU, + OpcodeVecI8x16AvgrU, + OpcodeVecI16x8MinS, OpcodeVecI16x8MinU, OpcodeVecI16x8MaxS, OpcodeVecI16x8MaxU, + OpcodeVecI16x8AvgrU, + OpcodeVecI32x4MinS, OpcodeVecI32x4MinU, OpcodeVecI32x4MaxS, OpcodeVecI32x4MaxU, + OpcodeVecF32x4Min, OpcodeVecF32x4Max, OpcodeVecF64x2Min, OpcodeVecF64x2Max, + OpcodeVecF32x4Pmin, OpcodeVecF32x4Pmax, OpcodeVecF64x2Pmin, OpcodeVecF64x2Pmax, + OpcodeVecI16x8Q15mulrSatS, + OpcodeVecI16x8ExtMulLowI8x16S, OpcodeVecI16x8ExtMulHighI8x16S, OpcodeVecI16x8ExtMulLowI8x16U, OpcodeVecI16x8ExtMulHighI8x16U, + OpcodeVecI32x4ExtMulLowI16x8S, OpcodeVecI32x4ExtMulHighI16x8S, OpcodeVecI32x4ExtMulLowI16x8U, OpcodeVecI32x4ExtMulHighI16x8U, + OpcodeVecI64x2ExtMulLowI32x4S, OpcodeVecI64x2ExtMulHighI32x4S, OpcodeVecI64x2ExtMulLowI32x4U, OpcodeVecI64x2ExtMulHighI32x4U: + if err := valueTypeStack.popAndVerifyType(ValueTypeV128); err != nil { + return fmt.Errorf("cannot pop the operand for %s: %v", vectorInstructionName[vecOpcode], err) + } + if err := valueTypeStack.popAndVerifyType(ValueTypeV128); err != nil { + return fmt.Errorf("cannot pop the operand for %s: %v", vectorInstructionName[vecOpcode], err) + } + valueTypeStack.push(ValueTypeV128) + default: + return fmt.Errorf("unknown SIMD instruction %s", vectorInstructionName[vecOpcode]) + } + } else if op == OpcodeBlock { + br.Reset(body[pc+1:]) + bt, num, err := DecodeBlockType(m.TypeSection, br, enabledFeatures) + if err != nil { + return fmt.Errorf("read block: %w", err) + } + controlBlockStack.push(pc, 0, 0, bt, num, 0) + if err = valueTypeStack.popParams(op, bt.Params, false); err != nil { + return err + } + // Plus we have to push any block params again. + for _, p := range bt.Params { + valueTypeStack.push(p) + } + valueTypeStack.pushStackLimit(len(bt.Params)) + pc += num + } else if op == OpcodeAtomicPrefix { + pc++ + // Atomic instructions come with two bytes where the first byte is always OpcodeAtomicPrefix, + // and the second byte determines the actual instruction. + atomicOpcode := body[pc] + if err := enabledFeatures.RequireEnabled(experimental.CoreFeaturesThreads); err != nil { + return fmt.Errorf("%s invalid as %v", atomicInstructionName[atomicOpcode], err) + } + pc++ + + if atomicOpcode == OpcodeAtomicFence { + // No memory requirement and no arguments or return, however the immediate byte value must be 0. + imm := body[pc] + if imm != 0x0 { + return fmt.Errorf("invalid immediate value for %s", AtomicInstructionName(atomicOpcode)) + } + continue + } + + // All atomic operations except fence (checked above) require memory + if memory == nil { + return fmt.Errorf("memory must exist for %s", AtomicInstructionName(atomicOpcode)) + } + align, _, read, err := readMemArg(pc, body) + if err != nil { + return err + } + pc += read - 1 + switch atomicOpcode { + case OpcodeAtomicMemoryNotify: + if 1< 32/8 { + return fmt.Errorf("invalid memory alignment") + } + if err := valueTypeStack.popAndVerifyType(ValueTypeI32); err != nil { + return err + } + if err := valueTypeStack.popAndVerifyType(ValueTypeI32); err != nil { + return err + } + valueTypeStack.push(ValueTypeI32) + case OpcodeAtomicMemoryWait32: + if 1< 32/8 { + return fmt.Errorf("invalid memory alignment") + } + if err := valueTypeStack.popAndVerifyType(ValueTypeI64); err != nil { + return err + } + if err := valueTypeStack.popAndVerifyType(ValueTypeI32); err != nil { + return err + } + if err := valueTypeStack.popAndVerifyType(ValueTypeI32); err != nil { + return err + } + valueTypeStack.push(ValueTypeI32) + case OpcodeAtomicMemoryWait64: + if 1< 64/8 { + return fmt.Errorf("invalid memory alignment") + } + if err := valueTypeStack.popAndVerifyType(ValueTypeI64); err != nil { + return err + } + if err := valueTypeStack.popAndVerifyType(ValueTypeI64); err != nil { + return err + } + if err := valueTypeStack.popAndVerifyType(ValueTypeI32); err != nil { + return err + } + valueTypeStack.push(ValueTypeI32) + case OpcodeAtomicI32Load: + if 1< 32/8 { + return fmt.Errorf("invalid memory alignment") + } + if err := valueTypeStack.popAndVerifyType(ValueTypeI32); err != nil { + return err + } + valueTypeStack.push(ValueTypeI32) + case OpcodeAtomicI64Load: + if 1< 64/8 { + return fmt.Errorf("invalid memory alignment") + } + if err := valueTypeStack.popAndVerifyType(ValueTypeI32); err != nil { + return err + } + valueTypeStack.push(ValueTypeI64) + case OpcodeAtomicI32Load8U: + if 1< 16/8 { + return fmt.Errorf("invalid memory alignment") + } + if err := valueTypeStack.popAndVerifyType(ValueTypeI32); err != nil { + return err + } + valueTypeStack.push(ValueTypeI64) + case OpcodeAtomicI64Load32U: + if 1< 32/8 { + return fmt.Errorf("invalid memory alignment") + } + if err := valueTypeStack.popAndVerifyType(ValueTypeI32); err != nil { + return err + } + valueTypeStack.push(ValueTypeI64) + case OpcodeAtomicI32Store: + if 1< 32/8 { + return fmt.Errorf("invalid memory alignment") + } + if err := valueTypeStack.popAndVerifyType(ValueTypeI32); err != nil { + return err + } + if err := valueTypeStack.popAndVerifyType(ValueTypeI32); err != nil { + return err + } + case OpcodeAtomicI64Store: + if 1< 64/8 { + return fmt.Errorf("invalid memory alignment") + } + if err := valueTypeStack.popAndVerifyType(ValueTypeI64); err != nil { + return err + } + if err := valueTypeStack.popAndVerifyType(ValueTypeI32); err != nil { + return err + } + case OpcodeAtomicI32Store8: + if 1< 1 { + return fmt.Errorf("invalid memory alignment") + } + if err := valueTypeStack.popAndVerifyType(ValueTypeI32); err != nil { + return err + } + if err := valueTypeStack.popAndVerifyType(ValueTypeI32); err != nil { + return err + } + case OpcodeAtomicI32Store16: + if 1< 16/8 { + return fmt.Errorf("invalid memory alignment") + } + if err := valueTypeStack.popAndVerifyType(ValueTypeI32); err != nil { + return err + } + if err := valueTypeStack.popAndVerifyType(ValueTypeI32); err != nil { + return err + } + case OpcodeAtomicI64Store8: + if 1< 1 { + return fmt.Errorf("invalid memory alignment") + } + if err := valueTypeStack.popAndVerifyType(ValueTypeI64); err != nil { + return err + } + if err := valueTypeStack.popAndVerifyType(ValueTypeI32); err != nil { + return err + } + case OpcodeAtomicI64Store16: + if 1< 16/8 { + return fmt.Errorf("invalid memory alignment") + } + if err := valueTypeStack.popAndVerifyType(ValueTypeI64); err != nil { + return err + } + if err := valueTypeStack.popAndVerifyType(ValueTypeI32); err != nil { + return err + } + case OpcodeAtomicI64Store32: + if 1< 32/8 { + return fmt.Errorf("invalid memory alignment") + } + if err := valueTypeStack.popAndVerifyType(ValueTypeI64); err != nil { + return err + } + if err := valueTypeStack.popAndVerifyType(ValueTypeI32); err != nil { + return err + } + case OpcodeAtomicI32RmwAdd, OpcodeAtomicI32RmwSub, OpcodeAtomicI32RmwAnd, OpcodeAtomicI32RmwOr, OpcodeAtomicI32RmwXor, OpcodeAtomicI32RmwXchg: + if 1< 32/8 { + return fmt.Errorf("invalid memory alignment") + } + if err := valueTypeStack.popAndVerifyType(ValueTypeI32); err != nil { + return err + } + if err := valueTypeStack.popAndVerifyType(ValueTypeI32); err != nil { + return err + } + valueTypeStack.push(ValueTypeI32) + case OpcodeAtomicI32Rmw8AddU, OpcodeAtomicI32Rmw8SubU, OpcodeAtomicI32Rmw8AndU, OpcodeAtomicI32Rmw8OrU, OpcodeAtomicI32Rmw8XorU, OpcodeAtomicI32Rmw8XchgU: + if 1< 1 { + return fmt.Errorf("invalid memory alignment") + } + if err := valueTypeStack.popAndVerifyType(ValueTypeI32); err != nil { + return err + } + if err := valueTypeStack.popAndVerifyType(ValueTypeI32); err != nil { + return err + } + valueTypeStack.push(ValueTypeI32) + case OpcodeAtomicI32Rmw16AddU, OpcodeAtomicI32Rmw16SubU, OpcodeAtomicI32Rmw16AndU, OpcodeAtomicI32Rmw16OrU, OpcodeAtomicI32Rmw16XorU, OpcodeAtomicI32Rmw16XchgU: + if 1< 16/8 { + return fmt.Errorf("invalid memory alignment") + } + if err := valueTypeStack.popAndVerifyType(ValueTypeI32); err != nil { + return err + } + if err := valueTypeStack.popAndVerifyType(ValueTypeI32); err != nil { + return err + } + valueTypeStack.push(ValueTypeI32) + case OpcodeAtomicI64RmwAdd, OpcodeAtomicI64RmwSub, OpcodeAtomicI64RmwAnd, OpcodeAtomicI64RmwOr, OpcodeAtomicI64RmwXor, OpcodeAtomicI64RmwXchg: + if 1< 64/8 { + return fmt.Errorf("invalid memory alignment") + } + if err := valueTypeStack.popAndVerifyType(ValueTypeI64); err != nil { + return err + } + if err := valueTypeStack.popAndVerifyType(ValueTypeI32); err != nil { + return err + } + valueTypeStack.push(ValueTypeI64) + case OpcodeAtomicI64Rmw8AddU, OpcodeAtomicI64Rmw8SubU, OpcodeAtomicI64Rmw8AndU, OpcodeAtomicI64Rmw8OrU, OpcodeAtomicI64Rmw8XorU, OpcodeAtomicI64Rmw8XchgU: + if 1< 1 { + return fmt.Errorf("invalid memory alignment") + } + if err := valueTypeStack.popAndVerifyType(ValueTypeI64); err != nil { + return err + } + if err := valueTypeStack.popAndVerifyType(ValueTypeI32); err != nil { + return err + } + valueTypeStack.push(ValueTypeI64) + case OpcodeAtomicI64Rmw16AddU, OpcodeAtomicI64Rmw16SubU, OpcodeAtomicI64Rmw16AndU, OpcodeAtomicI64Rmw16OrU, OpcodeAtomicI64Rmw16XorU, OpcodeAtomicI64Rmw16XchgU: + if 1< 16/8 { + return fmt.Errorf("invalid memory alignment") + } + if err := valueTypeStack.popAndVerifyType(ValueTypeI64); err != nil { + return err + } + if err := valueTypeStack.popAndVerifyType(ValueTypeI32); err != nil { + return err + } + valueTypeStack.push(ValueTypeI64) + case OpcodeAtomicI64Rmw32AddU, OpcodeAtomicI64Rmw32SubU, OpcodeAtomicI64Rmw32AndU, OpcodeAtomicI64Rmw32OrU, OpcodeAtomicI64Rmw32XorU, OpcodeAtomicI64Rmw32XchgU: + if 1< 32/8 { + return fmt.Errorf("invalid memory alignment") + } + if err := valueTypeStack.popAndVerifyType(ValueTypeI64); err != nil { + return err + } + if err := valueTypeStack.popAndVerifyType(ValueTypeI32); err != nil { + return err + } + valueTypeStack.push(ValueTypeI64) + case OpcodeAtomicI32RmwCmpxchg: + if 1< 32/8 { + return fmt.Errorf("invalid memory alignment") + } + if err := valueTypeStack.popAndVerifyType(ValueTypeI32); err != nil { + return err + } + if err := valueTypeStack.popAndVerifyType(ValueTypeI32); err != nil { + return err + } + if err := valueTypeStack.popAndVerifyType(ValueTypeI32); err != nil { + return err + } + valueTypeStack.push(ValueTypeI32) + case OpcodeAtomicI32Rmw8CmpxchgU: + if 1< 1 { + return fmt.Errorf("invalid memory alignment") + } + if err := valueTypeStack.popAndVerifyType(ValueTypeI32); err != nil { + return err + } + if err := valueTypeStack.popAndVerifyType(ValueTypeI32); err != nil { + return err + } + if err := valueTypeStack.popAndVerifyType(ValueTypeI32); err != nil { + return err + } + valueTypeStack.push(ValueTypeI32) + case OpcodeAtomicI32Rmw16CmpxchgU: + if 1< 16/8 { + return fmt.Errorf("invalid memory alignment") + } + if err := valueTypeStack.popAndVerifyType(ValueTypeI32); err != nil { + return err + } + if err := valueTypeStack.popAndVerifyType(ValueTypeI32); err != nil { + return err + } + if err := valueTypeStack.popAndVerifyType(ValueTypeI32); err != nil { + return err + } + valueTypeStack.push(ValueTypeI32) + case OpcodeAtomicI64RmwCmpxchg: + if 1< 64/8 { + return fmt.Errorf("invalid memory alignment") + } + if err := valueTypeStack.popAndVerifyType(ValueTypeI64); err != nil { + return err + } + if err := valueTypeStack.popAndVerifyType(ValueTypeI64); err != nil { + return err + } + if err := valueTypeStack.popAndVerifyType(ValueTypeI32); err != nil { + return err + } + valueTypeStack.push(ValueTypeI64) + case OpcodeAtomicI64Rmw8CmpxchgU: + if 1< 1 { + return fmt.Errorf("invalid memory alignment") + } + if err := valueTypeStack.popAndVerifyType(ValueTypeI64); err != nil { + return err + } + if err := valueTypeStack.popAndVerifyType(ValueTypeI64); err != nil { + return err + } + if err := valueTypeStack.popAndVerifyType(ValueTypeI32); err != nil { + return err + } + valueTypeStack.push(ValueTypeI64) + case OpcodeAtomicI64Rmw16CmpxchgU: + if 1< 16/8 { + return fmt.Errorf("invalid memory alignment") + } + if err := valueTypeStack.popAndVerifyType(ValueTypeI64); err != nil { + return err + } + if err := valueTypeStack.popAndVerifyType(ValueTypeI64); err != nil { + return err + } + if err := valueTypeStack.popAndVerifyType(ValueTypeI32); err != nil { + return err + } + valueTypeStack.push(ValueTypeI64) + case OpcodeAtomicI64Rmw32CmpxchgU: + if 1< 32/8 { + return fmt.Errorf("invalid memory alignment") + } + if err := valueTypeStack.popAndVerifyType(ValueTypeI64); err != nil { + return err + } + if err := valueTypeStack.popAndVerifyType(ValueTypeI64); err != nil { + return err + } + if err := valueTypeStack.popAndVerifyType(ValueTypeI32); err != nil { + return err + } + valueTypeStack.push(ValueTypeI64) + default: + return fmt.Errorf("invalid atomic opcode: 0x%x", atomicOpcode) + } + } else if op == OpcodeLoop { + br.Reset(body[pc+1:]) + bt, num, err := DecodeBlockType(m.TypeSection, br, enabledFeatures) + if err != nil { + return fmt.Errorf("read block: %w", err) + } + controlBlockStack.push(pc, 0, 0, bt, num, op) + if err = valueTypeStack.popParams(op, bt.Params, false); err != nil { + return err + } + // Plus we have to push any block params again. + for _, p := range bt.Params { + valueTypeStack.push(p) + } + valueTypeStack.pushStackLimit(len(bt.Params)) + pc += num + } else if op == OpcodeIf { + br.Reset(body[pc+1:]) + bt, num, err := DecodeBlockType(m.TypeSection, br, enabledFeatures) + if err != nil { + return fmt.Errorf("read block: %w", err) + } + controlBlockStack.push(pc, 0, 0, bt, num, op) + if err = valueTypeStack.popAndVerifyType(ValueTypeI32); err != nil { + return fmt.Errorf("cannot pop the operand for 'if': %v", err) + } + if err = valueTypeStack.popParams(op, bt.Params, false); err != nil { + return err + } + // Plus we have to push any block params again. + for _, p := range bt.Params { + valueTypeStack.push(p) + } + valueTypeStack.pushStackLimit(len(bt.Params)) + pc += num + } else if op == OpcodeElse { + bl := &controlBlockStack.stack[len(controlBlockStack.stack)-1] + if bl.op != OpcodeIf { + return fmt.Errorf("else instruction must be used in if block: %#x", pc) + } + bl.op = OpcodeElse + bl.elseAt = pc + // Check the type soundness of the instructions *before* entering this else Op. + if err := valueTypeStack.popResults(OpcodeIf, bl.blockType.Results, true); err != nil { + return err + } + // Before entering instructions inside else, we pop all the values pushed by then block. + valueTypeStack.resetAtStackLimit() + // Plus we have to push any block params again. + for _, p := range bl.blockType.Params { + valueTypeStack.push(p) + } + } else if op == OpcodeEnd { + bl := controlBlockStack.pop() + bl.endAt = pc + + // OpcodeEnd can end a block or the function itself. Check to see what it is: + + ifMissingElse := bl.op == OpcodeIf && bl.elseAt <= bl.startAt + if ifMissingElse { + // If this is the end of block without else, the number of block's results and params must be same. + // Otherwise, the value stack would result in the inconsistent state at runtime. + if !bytes.Equal(bl.blockType.Results, bl.blockType.Params) { + return typeCountError(false, OpcodeElseName, bl.blockType.Params, bl.blockType.Results) + } + // -1 skips else, to handle if block without else properly. + bl.elseAt = bl.endAt - 1 + } + + // Determine the block context + ctx := "" // the outer-most block: the function return + if bl.op == OpcodeIf && !ifMissingElse && bl.elseAt > 0 { + ctx = OpcodeElseName + } else if bl.op != 0 { + ctx = InstructionName(bl.op) + } + + // Check return types match + if err := valueTypeStack.requireStackValues(false, ctx, bl.blockType.Results, true); err != nil { + return err + } + + // Put the result types at the end after resetting at the stack limit + // since we might have Any type between the limit and the current top. + valueTypeStack.resetAtStackLimit() + for _, exp := range bl.blockType.Results { + valueTypeStack.push(exp) + } + // We exit if/loop/block, so reset the constraints on the stack manipulation + // on values previously pushed by outer blocks. + valueTypeStack.popStackLimit() + } else if op == OpcodeReturn { + // Same formatting as OpcodeEnd on the outer-most block + if err := valueTypeStack.requireStackValues(false, "", functionType.Results, false); err != nil { + return err + } + // return instruction is stack-polymorphic. + valueTypeStack.unreachable() + } else if op == OpcodeDrop { + _, err := valueTypeStack.pop() + if err != nil { + return fmt.Errorf("invalid drop: %v", err) + } + } else if op == OpcodeSelect || op == OpcodeTypedSelect { + if err := valueTypeStack.popAndVerifyType(ValueTypeI32); err != nil { + return fmt.Errorf("type mismatch on 3rd select operand: %v", err) + } + v1, err := valueTypeStack.pop() + if err != nil { + return fmt.Errorf("invalid select: %v", err) + } + v2, err := valueTypeStack.pop() + if err != nil { + return fmt.Errorf("invalid select: %v", err) + } + + if op == OpcodeTypedSelect { + if err := enabledFeatures.RequireEnabled(api.CoreFeatureReferenceTypes); err != nil { + return fmt.Errorf("%s is invalid as %w", InstructionName(op), err) + } + pc++ + if numTypeImmeidates := body[pc]; numTypeImmeidates != 1 { + return fmt.Errorf("too many type immediates for %s", InstructionName(op)) + } + pc++ + tp := body[pc] + if tp != ValueTypeI32 && tp != ValueTypeI64 && tp != ValueTypeF32 && tp != ValueTypeF64 && + tp != api.ValueTypeExternref && tp != ValueTypeFuncref && tp != ValueTypeV128 { + return fmt.Errorf("invalid type %s for %s", ValueTypeName(tp), OpcodeTypedSelectName) + } + } else if isReferenceValueType(v1) || isReferenceValueType(v2) { + return fmt.Errorf("reference types cannot be used for non typed select instruction") + } + + if v1 != v2 && v1 != valueTypeUnknown && v2 != valueTypeUnknown { + return fmt.Errorf("type mismatch on 1st and 2nd select operands") + } + if v1 == valueTypeUnknown { + valueTypeStack.push(v2) + } else { + valueTypeStack.push(v1) + } + } else if op == OpcodeUnreachable { + // unreachable instruction is stack-polymorphic. + valueTypeStack.unreachable() + } else if op == OpcodeNop { + } else { + return fmt.Errorf("invalid instruction 0x%x", op) + } + } + + if len(controlBlockStack.stack) > 0 { + return fmt.Errorf("ill-nested block exists") + } + if valueTypeStack.maximumStackPointer > maxStackValues { + return fmt.Errorf("function may have %d stack values, which exceeds limit %d", valueTypeStack.maximumStackPointer, maxStackValues) + } + return nil +} + +var vecExtractLanes = [...]struct { + laneCeil byte + resultType ValueType +}{ + OpcodeVecI8x16ExtractLaneS: {laneCeil: 16, resultType: ValueTypeI32}, + OpcodeVecI8x16ExtractLaneU: {laneCeil: 16, resultType: ValueTypeI32}, + OpcodeVecI16x8ExtractLaneS: {laneCeil: 8, resultType: ValueTypeI32}, + OpcodeVecI16x8ExtractLaneU: {laneCeil: 8, resultType: ValueTypeI32}, + OpcodeVecI32x4ExtractLane: {laneCeil: 4, resultType: ValueTypeI32}, + OpcodeVecI64x2ExtractLane: {laneCeil: 2, resultType: ValueTypeI64}, + OpcodeVecF32x4ExtractLane: {laneCeil: 4, resultType: ValueTypeF32}, + OpcodeVecF64x2ExtractLane: {laneCeil: 2, resultType: ValueTypeF64}, +} + +var vecReplaceLanes = [...]struct { + laneCeil byte + paramType ValueType +}{ + OpcodeVecI8x16ReplaceLane: {laneCeil: 16, paramType: ValueTypeI32}, + OpcodeVecI16x8ReplaceLane: {laneCeil: 8, paramType: ValueTypeI32}, + OpcodeVecI32x4ReplaceLane: {laneCeil: 4, paramType: ValueTypeI32}, + OpcodeVecI64x2ReplaceLane: {laneCeil: 2, paramType: ValueTypeI64}, + OpcodeVecF32x4ReplaceLane: {laneCeil: 4, paramType: ValueTypeF32}, + OpcodeVecF64x2ReplaceLane: {laneCeil: 2, paramType: ValueTypeF64}, +} + +var vecStoreLanes = [...]struct { + alignMax uint32 + laneCeil byte +}{ + OpcodeVecV128Store64Lane: {alignMax: 64 / 8, laneCeil: 128 / 64}, + OpcodeVecV128Store32Lane: {alignMax: 32 / 8, laneCeil: 128 / 32}, + OpcodeVecV128Store16Lane: {alignMax: 16 / 8, laneCeil: 128 / 16}, + OpcodeVecV128Store8Lane: {alignMax: 1, laneCeil: 128 / 8}, +} + +var vecLoadLanes = [...]struct { + alignMax uint32 + laneCeil byte +}{ + OpcodeVecV128Load64Lane: {alignMax: 64 / 8, laneCeil: 128 / 64}, + OpcodeVecV128Load32Lane: {alignMax: 32 / 8, laneCeil: 128 / 32}, + OpcodeVecV128Load16Lane: {alignMax: 16 / 8, laneCeil: 128 / 16}, + OpcodeVecV128Load8Lane: {alignMax: 1, laneCeil: 128 / 8}, +} + +var vecSplatValueTypes = [...]ValueType{ + OpcodeVecI8x16Splat: ValueTypeI32, + OpcodeVecI16x8Splat: ValueTypeI32, + OpcodeVecI32x4Splat: ValueTypeI32, + OpcodeVecI64x2Splat: ValueTypeI64, + OpcodeVecF32x4Splat: ValueTypeF32, + OpcodeVecF64x2Splat: ValueTypeF64, +} + +type stacks struct { + vs valueTypeStack + cs controlBlockStack +} + +func (sts *stacks) reset(functionType *FunctionType) { + // Reset valueStack for reuse. + sts.vs.stack = sts.vs.stack[:0] + sts.vs.stackLimits = sts.vs.stackLimits[:0] + sts.vs.maximumStackPointer = 0 + sts.cs.stack = sts.cs.stack[:0] + sts.cs.stack = append(sts.cs.stack, controlBlock{blockType: functionType}) +} + +type controlBlockStack struct { + stack []controlBlock +} + +func (s *controlBlockStack) pop() *controlBlock { + tail := len(s.stack) - 1 + ret := &s.stack[tail] + s.stack = s.stack[:tail] + return ret +} + +func (s *controlBlockStack) push(startAt, elseAt, endAt uint64, blockType *FunctionType, blockTypeBytes uint64, op Opcode) { + s.stack = append(s.stack, controlBlock{ + startAt: startAt, + elseAt: elseAt, + endAt: endAt, + blockType: blockType, + blockTypeBytes: blockTypeBytes, + op: op, + }) +} + +type valueTypeStack struct { + stack []ValueType + stackLimits []int + maximumStackPointer int + // requireStackValuesTmp is used in requireStackValues function to reduce the allocation. + requireStackValuesTmp []ValueType +} + +// Only used in the analyzeFunction below. +const valueTypeUnknown = ValueType(0xFF) + +func (s *valueTypeStack) tryPop() (vt ValueType, limit int, ok bool) { + if len(s.stackLimits) > 0 { + limit = s.stackLimits[len(s.stackLimits)-1] + } + stackLen := len(s.stack) + if stackLen <= limit { + return + } else if stackLen == limit+1 && s.stack[limit] == valueTypeUnknown { + vt = valueTypeUnknown + ok = true + return + } else { + vt = s.stack[stackLen-1] + s.stack = s.stack[:stackLen-1] + ok = true + return + } +} + +func (s *valueTypeStack) pop() (ValueType, error) { + if vt, limit, ok := s.tryPop(); ok { + return vt, nil + } else { + return 0, fmt.Errorf("invalid operation: trying to pop at %d with limit %d", len(s.stack), limit) + } +} + +// popAndVerifyType returns an error if the stack value is unexpected. +func (s *valueTypeStack) popAndVerifyType(expected ValueType) error { + have, _, ok := s.tryPop() + if !ok { + return fmt.Errorf("%s missing", ValueTypeName(expected)) + } + if have != expected && have != valueTypeUnknown && expected != valueTypeUnknown { + return fmt.Errorf("type mismatch: expected %s, but was %s", ValueTypeName(expected), ValueTypeName(have)) + } + return nil +} + +func (s *valueTypeStack) push(v ValueType) { + s.stack = append(s.stack, v) + if sp := len(s.stack); sp > s.maximumStackPointer { + s.maximumStackPointer = sp + } +} + +func (s *valueTypeStack) unreachable() { + s.resetAtStackLimit() + s.stack = append(s.stack, valueTypeUnknown) +} + +func (s *valueTypeStack) resetAtStackLimit() { + if len(s.stackLimits) != 0 { + s.stack = s.stack[:s.stackLimits[len(s.stackLimits)-1]] + } else { + s.stack = s.stack[:0] + } +} + +func (s *valueTypeStack) popStackLimit() { + if len(s.stackLimits) != 0 { + s.stackLimits = s.stackLimits[:len(s.stackLimits)-1] + } +} + +// pushStackLimit pushes the control frame's bottom of the stack. +func (s *valueTypeStack) pushStackLimit(params int) { + limit := len(s.stack) - params + s.stackLimits = append(s.stackLimits, limit) +} + +func (s *valueTypeStack) popParams(oc Opcode, want []ValueType, checkAboveLimit bool) error { + return s.requireStackValues(true, InstructionName(oc), want, checkAboveLimit) +} + +func (s *valueTypeStack) popResults(oc Opcode, want []ValueType, checkAboveLimit bool) error { + return s.requireStackValues(false, InstructionName(oc), want, checkAboveLimit) +} + +func (s *valueTypeStack) requireStackValues( + isParam bool, + context string, + want []ValueType, + checkAboveLimit bool, +) error { + limit := 0 + if len(s.stackLimits) > 0 { + limit = s.stackLimits[len(s.stackLimits)-1] + } + // Iterate backwards as we are comparing the desired slice against stack value types. + countWanted := len(want) + + // First, check if there are enough values on the stack. + s.requireStackValuesTmp = s.requireStackValuesTmp[:0] + for i := countWanted - 1; i >= 0; i-- { + popped, _, ok := s.tryPop() + if !ok { + if len(s.requireStackValuesTmp) > len(want) { + return typeCountError(isParam, context, s.requireStackValuesTmp, want) + } + return typeCountError(isParam, context, s.requireStackValuesTmp, want) + } + s.requireStackValuesTmp = append(s.requireStackValuesTmp, popped) + } + + // Now, check if there are too many values. + if checkAboveLimit { + if !(limit == len(s.stack) || (limit+1 == len(s.stack) && s.stack[limit] == valueTypeUnknown)) { + return typeCountError(isParam, context, append(s.stack, want...), want) + } + } + + // Finally, check the types of the values: + for i, v := range s.requireStackValuesTmp { + nextWant := want[countWanted-i-1] // have is in reverse order (stack) + if v != nextWant && v != valueTypeUnknown && nextWant != valueTypeUnknown { + return typeMismatchError(isParam, context, v, nextWant, i) + } + } + return nil +} + +// typeMismatchError returns an error similar to go compiler's error on type mismatch. +func typeMismatchError(isParam bool, context string, have ValueType, want ValueType, i int) error { + var ret strings.Builder + ret.WriteString("cannot use ") + ret.WriteString(ValueTypeName(have)) + if context != "" { + ret.WriteString(" in ") + ret.WriteString(context) + ret.WriteString(" block") + } + if isParam { + ret.WriteString(" as param") + } else { + ret.WriteString(" as result") + } + ret.WriteString("[") + ret.WriteString(strconv.Itoa(i)) + ret.WriteString("] type ") + ret.WriteString(ValueTypeName(want)) + return errors.New(ret.String()) +} + +// typeCountError returns an error similar to go compiler's error on type count mismatch. +func typeCountError(isParam bool, context string, have []ValueType, want []ValueType) error { + var ret strings.Builder + if len(have) > len(want) { + ret.WriteString("too many ") + } else { + ret.WriteString("not enough ") + } + if isParam { + ret.WriteString("params") + } else { + ret.WriteString("results") + } + if context != "" { + if isParam { + ret.WriteString(" for ") + } else { + ret.WriteString(" in ") + } + ret.WriteString(context) + ret.WriteString(" block") + } + ret.WriteString("\n\thave (") + writeValueTypes(have, &ret) + ret.WriteString(")\n\twant (") + writeValueTypes(want, &ret) + ret.WriteByte(')') + return errors.New(ret.String()) +} + +func writeValueTypes(vts []ValueType, ret *strings.Builder) { + switch len(vts) { + case 0: + case 1: + ret.WriteString(ValueTypeName(vts[0])) + default: + ret.WriteString(ValueTypeName(vts[0])) + for _, vt := range vts[1:] { + ret.WriteString(", ") + ret.WriteString(ValueTypeName(vt)) + } + } +} + +func (s *valueTypeStack) String() string { + var typeStrs, limits []string + for _, v := range s.stack { + var str string + if v == valueTypeUnknown { + str = "unknown" + } else { + str = ValueTypeName(v) + } + typeStrs = append(typeStrs, str) + } + for _, d := range s.stackLimits { + limits = append(limits, fmt.Sprintf("%d", d)) + } + return fmt.Sprintf("{stack: [%s], limits: [%s]}", + strings.Join(typeStrs, ", "), strings.Join(limits, ",")) +} + +type controlBlock struct { + startAt, elseAt, endAt uint64 + blockType *FunctionType + blockTypeBytes uint64 + // op is zero when the outermost block + op Opcode +} + +// DecodeBlockType decodes the type index from a positive 33-bit signed integer. Negative numbers indicate up to one +// WebAssembly 1.0 (20191205) compatible result type. Positive numbers are decoded when `enabledFeatures` include +// CoreFeatureMultiValue and include an index in the Module.TypeSection. +// +// See https://www.w3.org/TR/2019/REC-wasm-core-1-20191205/#binary-blocktype +// See https://github.com/WebAssembly/spec/blob/wg-2.0.draft1/proposals/multi-value/Overview.md +func DecodeBlockType(types []FunctionType, r *bytes.Reader, enabledFeatures api.CoreFeatures) (*FunctionType, uint64, error) { + raw, num, err := leb128.DecodeInt33AsInt64(r) + if err != nil { + return nil, 0, fmt.Errorf("decode int33: %w", err) + } + + var ret *FunctionType + switch raw { + case -64: // 0x40 in original byte = nil + ret = blockType_v_v + case -1: // 0x7f in original byte = i32 + ret = blockType_v_i32 + case -2: // 0x7e in original byte = i64 + ret = blockType_v_i64 + case -3: // 0x7d in original byte = f32 + ret = blockType_v_f32 + case -4: // 0x7c in original byte = f64 + ret = blockType_v_f64 + case -5: // 0x7b in original byte = v128 + ret = blockType_v_v128 + case -16: // 0x70 in original byte = funcref + ret = blockType_v_funcref + case -17: // 0x6f in original byte = externref + ret = blockType_v_externref + default: + if err = enabledFeatures.RequireEnabled(api.CoreFeatureMultiValue); err != nil { + return nil, num, fmt.Errorf("block with function type return invalid as %v", err) + } + if raw < 0 || (raw >= int64(len(types))) { + return nil, 0, fmt.Errorf("type index out of range: %d", raw) + } + ret = &types[raw] + } + return ret, num, err +} + +// These block types are defined as globals in order to avoid allocations in DecodeBlockType. +var ( + blockType_v_v = &FunctionType{} + blockType_v_i32 = &FunctionType{Results: []ValueType{ValueTypeI32}, ResultNumInUint64: 1} + blockType_v_i64 = &FunctionType{Results: []ValueType{ValueTypeI64}, ResultNumInUint64: 1} + blockType_v_f32 = &FunctionType{Results: []ValueType{ValueTypeF32}, ResultNumInUint64: 1} + blockType_v_f64 = &FunctionType{Results: []ValueType{ValueTypeF64}, ResultNumInUint64: 1} + blockType_v_v128 = &FunctionType{Results: []ValueType{ValueTypeV128}, ResultNumInUint64: 2} + blockType_v_funcref = &FunctionType{Results: []ValueType{ValueTypeFuncref}, ResultNumInUint64: 1} + blockType_v_externref = &FunctionType{Results: []ValueType{ValueTypeExternref}, ResultNumInUint64: 1} +) + +// SplitCallStack returns the input stack resliced to the count of params and +// results, or errors if it isn't long enough for either. +func SplitCallStack(ft *FunctionType, stack []uint64) (params []uint64, results []uint64, err error) { + stackLen := len(stack) + if n := ft.ParamNumInUint64; n > stackLen { + return nil, nil, fmt.Errorf("need %d params, but stack size is %d", n, stackLen) + } else if n > 0 { + params = stack[:n] + } + if n := ft.ResultNumInUint64; n > stackLen { + return nil, nil, fmt.Errorf("need %d results, but stack size is %d", n, stackLen) + } else if n > 0 { + results = stack[:n] + } + return +} diff --git a/vendor/github.com/tetratelabs/wazero/internal/wasm/function_definition.go b/vendor/github.com/tetratelabs/wazero/internal/wasm/function_definition.go new file mode 100644 index 000000000..c5f6e9121 --- /dev/null +++ b/vendor/github.com/tetratelabs/wazero/internal/wasm/function_definition.go @@ -0,0 +1,188 @@ +package wasm + +import ( + "github.com/tetratelabs/wazero/api" + "github.com/tetratelabs/wazero/internal/internalapi" + "github.com/tetratelabs/wazero/internal/wasmdebug" +) + +// ImportedFunctions returns the definitions of each imported function. +// +// Note: Unlike ExportedFunctions, there is no unique constraint on imports. +func (m *Module) ImportedFunctions() (ret []api.FunctionDefinition) { + for i := uint32(0); i < m.ImportFunctionCount; i++ { + ret = append(ret, m.FunctionDefinition(i)) + } + return +} + +// ExportedFunctions returns the definitions of each exported function. +func (m *Module) ExportedFunctions() map[string]api.FunctionDefinition { + ret := map[string]api.FunctionDefinition{} + for i := range m.ExportSection { + exp := &m.ExportSection[i] + if exp.Type == ExternTypeFunc { + d := m.FunctionDefinition(exp.Index) + ret[exp.Name] = d + } + } + return ret +} + +// FunctionDefinition returns the FunctionDefinition for the given `index`. +func (m *Module) FunctionDefinition(index Index) *FunctionDefinition { + // TODO: function initialization is lazy, but bulk. Make it per function. + m.buildFunctionDefinitions() + return &m.FunctionDefinitionSection[index] +} + +// buildFunctionDefinitions generates function metadata that can be parsed from +// the module. This must be called after all validation. +func (m *Module) buildFunctionDefinitions() { + m.functionDefinitionSectionInitOnce.Do(m.buildFunctionDefinitionsOnce) +} + +func (m *Module) buildFunctionDefinitionsOnce() { + var moduleName string + var functionNames NameMap + var localNames, resultNames IndirectNameMap + if m.NameSection != nil { + moduleName = m.NameSection.ModuleName + functionNames = m.NameSection.FunctionNames + localNames = m.NameSection.LocalNames + resultNames = m.NameSection.ResultNames + } + + importCount := m.ImportFunctionCount + m.FunctionDefinitionSection = make([]FunctionDefinition, importCount+uint32(len(m.FunctionSection))) + + importFuncIdx := Index(0) + for i := range m.ImportSection { + imp := &m.ImportSection[i] + if imp.Type != ExternTypeFunc { + continue + } + + def := &m.FunctionDefinitionSection[importFuncIdx] + def.importDesc = imp + def.index = importFuncIdx + def.Functype = &m.TypeSection[imp.DescFunc] + importFuncIdx++ + } + + for codeIndex, typeIndex := range m.FunctionSection { + code := &m.CodeSection[codeIndex] + idx := importFuncIdx + Index(codeIndex) + def := &m.FunctionDefinitionSection[idx] + def.index = idx + def.Functype = &m.TypeSection[typeIndex] + def.goFunc = code.GoFunc + } + + n, nLen := 0, len(functionNames) + for i := range m.FunctionDefinitionSection { + d := &m.FunctionDefinitionSection[i] + // The function name section begins with imports, but can be sparse. + // This keeps track of how far in the name section we've searched. + funcIdx := d.index + var funcName string + for ; n < nLen; n++ { + next := &functionNames[n] + if next.Index > funcIdx { + break // we have function names, but starting at a later index. + } else if next.Index == funcIdx { + funcName = next.Name + break + } + } + + d.moduleName = moduleName + d.name = funcName + d.Debugname = wasmdebug.FuncName(moduleName, funcName, funcIdx) + d.paramNames = paramNames(localNames, funcIdx, len(d.Functype.Params)) + d.resultNames = paramNames(resultNames, funcIdx, len(d.Functype.Results)) + + for i := range m.ExportSection { + e := &m.ExportSection[i] + if e.Type == ExternTypeFunc && e.Index == funcIdx { + d.exportNames = append(d.exportNames, e.Name) + } + } + } +} + +// FunctionDefinition implements api.FunctionDefinition +type FunctionDefinition struct { + internalapi.WazeroOnlyType + moduleName string + index Index + name string + // Debugname is exported for testing purpose. + Debugname string + goFunc interface{} + // Functype is exported for testing purpose. + Functype *FunctionType + importDesc *Import + exportNames []string + paramNames []string + resultNames []string +} + +// ModuleName implements the same method as documented on api.FunctionDefinition. +func (f *FunctionDefinition) ModuleName() string { + return f.moduleName +} + +// Index implements the same method as documented on api.FunctionDefinition. +func (f *FunctionDefinition) Index() uint32 { + return f.index +} + +// Name implements the same method as documented on api.FunctionDefinition. +func (f *FunctionDefinition) Name() string { + return f.name +} + +// DebugName implements the same method as documented on api.FunctionDefinition. +func (f *FunctionDefinition) DebugName() string { + return f.Debugname +} + +// Import implements the same method as documented on api.FunctionDefinition. +func (f *FunctionDefinition) Import() (moduleName, name string, isImport bool) { + if f.importDesc != nil { + importDesc := f.importDesc + moduleName, name, isImport = importDesc.Module, importDesc.Name, true + } + return +} + +// ExportNames implements the same method as documented on api.FunctionDefinition. +func (f *FunctionDefinition) ExportNames() []string { + return f.exportNames +} + +// GoFunction implements the same method as documented on api.FunctionDefinition. +func (f *FunctionDefinition) GoFunction() interface{} { + return f.goFunc +} + +// ParamTypes implements api.FunctionDefinition ParamTypes. +func (f *FunctionDefinition) ParamTypes() []ValueType { + return f.Functype.Params +} + +// ParamNames implements the same method as documented on api.FunctionDefinition. +func (f *FunctionDefinition) ParamNames() []string { + return f.paramNames +} + +// ResultTypes implements api.FunctionDefinition ResultTypes. +func (f *FunctionDefinition) ResultTypes() []ValueType { + return f.Functype.Results +} + +// ResultNames implements the same method as documented on api.FunctionDefinition. +func (f *FunctionDefinition) ResultNames() []string { + return f.resultNames +} diff --git a/vendor/github.com/tetratelabs/wazero/internal/wasm/global.go b/vendor/github.com/tetratelabs/wazero/internal/wasm/global.go new file mode 100644 index 000000000..abaa2d1f9 --- /dev/null +++ b/vendor/github.com/tetratelabs/wazero/internal/wasm/global.go @@ -0,0 +1,55 @@ +package wasm + +import ( + "github.com/tetratelabs/wazero/api" + "github.com/tetratelabs/wazero/internal/internalapi" +) + +// constantGlobal wraps GlobalInstance to implement api.Global. +type constantGlobal struct { + internalapi.WazeroOnlyType + g *GlobalInstance +} + +// Type implements api.Global. +func (g constantGlobal) Type() api.ValueType { + return g.g.Type.ValType +} + +// Get implements api.Global. +func (g constantGlobal) Get() uint64 { + ret, _ := g.g.Value() + return ret +} + +// String implements api.Global. +func (g constantGlobal) String() string { + return g.g.String() +} + +// mutableGlobal extends constantGlobal to allow updates. +type mutableGlobal struct { + internalapi.WazeroOnlyType + g *GlobalInstance +} + +// Type implements api.Global. +func (g mutableGlobal) Type() api.ValueType { + return g.g.Type.ValType +} + +// Get implements api.Global. +func (g mutableGlobal) Get() uint64 { + ret, _ := g.g.Value() + return ret +} + +// String implements api.Global. +func (g mutableGlobal) String() string { + return g.g.String() +} + +// Set implements the same method as documented on api.MutableGlobal. +func (g mutableGlobal) Set(v uint64) { + g.g.SetValue(v, 0) +} diff --git a/vendor/github.com/tetratelabs/wazero/internal/wasm/gofunc.go b/vendor/github.com/tetratelabs/wazero/internal/wasm/gofunc.go new file mode 100644 index 000000000..9510c2588 --- /dev/null +++ b/vendor/github.com/tetratelabs/wazero/internal/wasm/gofunc.go @@ -0,0 +1,279 @@ +package wasm + +import ( + "bytes" + "context" + "errors" + "fmt" + "math" + "reflect" + + "github.com/tetratelabs/wazero/api" +) + +type paramsKind byte + +const ( + paramsKindNoContext paramsKind = iota + paramsKindContext + paramsKindContextModule +) + +// Below are reflection code to get the interface type used to parse functions and set values. + +var ( + moduleType = reflect.TypeOf((*api.Module)(nil)).Elem() + goContextType = reflect.TypeOf((*context.Context)(nil)).Elem() + errorType = reflect.TypeOf((*error)(nil)).Elem() +) + +// compile-time check to ensure reflectGoModuleFunction implements +// api.GoModuleFunction. +var _ api.GoModuleFunction = (*reflectGoModuleFunction)(nil) + +type reflectGoModuleFunction struct { + fn *reflect.Value + params, results []ValueType +} + +// Call implements the same method as documented on api.GoModuleFunction. +func (f *reflectGoModuleFunction) Call(ctx context.Context, mod api.Module, stack []uint64) { + callGoFunc(ctx, mod, f.fn, stack) +} + +// EqualTo is exposed for testing. +func (f *reflectGoModuleFunction) EqualTo(that interface{}) bool { + if f2, ok := that.(*reflectGoModuleFunction); !ok { + return false + } else { + // TODO compare reflect pointers + return bytes.Equal(f.params, f2.params) && bytes.Equal(f.results, f2.results) + } +} + +// compile-time check to ensure reflectGoFunction implements api.GoFunction. +var _ api.GoFunction = (*reflectGoFunction)(nil) + +type reflectGoFunction struct { + fn *reflect.Value + pk paramsKind + params, results []ValueType +} + +// EqualTo is exposed for testing. +func (f *reflectGoFunction) EqualTo(that interface{}) bool { + if f2, ok := that.(*reflectGoFunction); !ok { + return false + } else { + // TODO compare reflect pointers + return f.pk == f2.pk && + bytes.Equal(f.params, f2.params) && bytes.Equal(f.results, f2.results) + } +} + +// Call implements the same method as documented on api.GoFunction. +func (f *reflectGoFunction) Call(ctx context.Context, stack []uint64) { + if f.pk == paramsKindNoContext { + ctx = nil + } + callGoFunc(ctx, nil, f.fn, stack) +} + +// callGoFunc executes the reflective function by converting params to Go +// types. The results of the function call are converted back to api.ValueType. +func callGoFunc(ctx context.Context, mod api.Module, fn *reflect.Value, stack []uint64) { + tp := fn.Type() + + var in []reflect.Value + pLen := tp.NumIn() + if pLen != 0 { + in = make([]reflect.Value, pLen) + + i := 0 + if ctx != nil { + in[0] = newContextVal(ctx) + i++ + } + if mod != nil { + in[1] = newModuleVal(mod) + i++ + } + + for j := 0; i < pLen; i++ { + next := tp.In(i) + val := reflect.New(next).Elem() + k := next.Kind() + raw := stack[j] + j++ + + switch k { + case reflect.Float32: + val.SetFloat(float64(math.Float32frombits(uint32(raw)))) + case reflect.Float64: + val.SetFloat(math.Float64frombits(raw)) + case reflect.Uint32, reflect.Uint64, reflect.Uintptr: + val.SetUint(raw) + case reflect.Int32, reflect.Int64: + val.SetInt(int64(raw)) + default: + panic(fmt.Errorf("BUG: param[%d] has an invalid type: %v", i, k)) + } + in[i] = val + } + } + + // Execute the host function and push back the call result onto the stack. + for i, ret := range fn.Call(in) { + switch ret.Kind() { + case reflect.Float32: + stack[i] = uint64(math.Float32bits(float32(ret.Float()))) + case reflect.Float64: + stack[i] = math.Float64bits(ret.Float()) + case reflect.Uint32, reflect.Uint64, reflect.Uintptr: + stack[i] = ret.Uint() + case reflect.Int32, reflect.Int64: + stack[i] = uint64(ret.Int()) + default: + panic(fmt.Errorf("BUG: result[%d] has an invalid type: %v", i, ret.Kind())) + } + } +} + +func newContextVal(ctx context.Context) reflect.Value { + val := reflect.New(goContextType).Elem() + val.Set(reflect.ValueOf(ctx)) + return val +} + +func newModuleVal(m api.Module) reflect.Value { + val := reflect.New(moduleType).Elem() + val.Set(reflect.ValueOf(m)) + return val +} + +// MustParseGoReflectFuncCode parses Code from the go function or panics. +// +// Exposing this simplifies FunctionDefinition of host functions in built-in host +// modules and tests. +func MustParseGoReflectFuncCode(fn interface{}) Code { + _, _, code, err := parseGoReflectFunc(fn) + if err != nil { + panic(err) + } + return code +} + +func parseGoReflectFunc(fn interface{}) (params, results []ValueType, code Code, err error) { + fnV := reflect.ValueOf(fn) + p := fnV.Type() + + if fnV.Kind() != reflect.Func { + err = fmt.Errorf("kind != func: %s", fnV.Kind().String()) + return + } + + pk, kindErr := kind(p) + if kindErr != nil { + err = kindErr + return + } + + pOffset := 0 + switch pk { + case paramsKindNoContext: + case paramsKindContext: + pOffset = 1 + case paramsKindContextModule: + pOffset = 2 + } + + pCount := p.NumIn() - pOffset + if pCount > 0 { + params = make([]ValueType, pCount) + } + for i := 0; i < len(params); i++ { + pI := p.In(i + pOffset) + if t, ok := getTypeOf(pI.Kind()); ok { + params[i] = t + continue + } + + // Now, we will definitely err, decide which message is best + var arg0Type reflect.Type + if hc := pI.Implements(moduleType); hc { + arg0Type = moduleType + } else if gc := pI.Implements(goContextType); gc { + arg0Type = goContextType + } + + if arg0Type != nil { + err = fmt.Errorf("param[%d] is a %s, which may be defined only once as param[0]", i+pOffset, arg0Type) + } else { + err = fmt.Errorf("param[%d] is unsupported: %s", i+pOffset, pI.Kind()) + } + return + } + + rCount := p.NumOut() + if rCount > 0 { + results = make([]ValueType, rCount) + } + for i := 0; i < len(results); i++ { + rI := p.Out(i) + if t, ok := getTypeOf(rI.Kind()); ok { + results[i] = t + continue + } + + // Now, we will definitely err, decide which message is best + if rI.Implements(errorType) { + err = fmt.Errorf("result[%d] is an error, which is unsupported", i) + } else { + err = fmt.Errorf("result[%d] is unsupported: %s", i, rI.Kind()) + } + return + } + + code = Code{} + if pk == paramsKindContextModule { + code.GoFunc = &reflectGoModuleFunction{fn: &fnV, params: params, results: results} + } else { + code.GoFunc = &reflectGoFunction{pk: pk, fn: &fnV, params: params, results: results} + } + return +} + +func kind(p reflect.Type) (paramsKind, error) { + pCount := p.NumIn() + if pCount > 0 && p.In(0).Kind() == reflect.Interface { + p0 := p.In(0) + if p0.Implements(moduleType) { + return 0, errors.New("invalid signature: api.Module parameter must be preceded by context.Context") + } else if p0.Implements(goContextType) { + if pCount >= 2 && p.In(1).Implements(moduleType) { + return paramsKindContextModule, nil + } + return paramsKindContext, nil + } + } + // Without context param allows portability with reflective runtimes. + // This allows people to more easily port to wazero. + return paramsKindNoContext, nil +} + +func getTypeOf(kind reflect.Kind) (ValueType, bool) { + switch kind { + case reflect.Float64: + return ValueTypeF64, true + case reflect.Float32: + return ValueTypeF32, true + case reflect.Int32, reflect.Uint32: + return ValueTypeI32, true + case reflect.Int64, reflect.Uint64: + return ValueTypeI64, true + case reflect.Uintptr: + return ValueTypeExternref, true + default: + return 0x00, false + } +} diff --git a/vendor/github.com/tetratelabs/wazero/internal/wasm/host.go b/vendor/github.com/tetratelabs/wazero/internal/wasm/host.go new file mode 100644 index 000000000..bca686d1d --- /dev/null +++ b/vendor/github.com/tetratelabs/wazero/internal/wasm/host.go @@ -0,0 +1,179 @@ +package wasm + +import ( + "errors" + "fmt" + + "github.com/tetratelabs/wazero/api" + "github.com/tetratelabs/wazero/internal/wasmdebug" +) + +type HostFuncExporter interface { + ExportHostFunc(*HostFunc) +} + +// HostFunc is a function with an inlined type, used for NewHostModule. +// Any corresponding FunctionType will be reused or added to the Module. +type HostFunc struct { + // ExportName is the only value returned by api.FunctionDefinition. + ExportName string + + // Name is equivalent to the same method on api.FunctionDefinition. + Name string + + // ParamTypes is equivalent to the same method on api.FunctionDefinition. + ParamTypes []ValueType + + // ParamNames is equivalent to the same method on api.FunctionDefinition. + ParamNames []string + + // ResultTypes is equivalent to the same method on api.FunctionDefinition. + ResultTypes []ValueType + + // ResultNames is equivalent to the same method on api.FunctionDefinition. + ResultNames []string + + // Code is the equivalent function in the SectionIDCode. + Code Code +} + +// WithGoModuleFunc returns a copy of the function, replacing its Code.GoFunc. +func (f *HostFunc) WithGoModuleFunc(fn api.GoModuleFunc) *HostFunc { + ret := *f + ret.Code.GoFunc = fn + return &ret +} + +// NewHostModule is defined internally for use in WASI tests and to keep the code size in the root directory small. +func NewHostModule( + moduleName string, + exportNames []string, + nameToHostFunc map[string]*HostFunc, + enabledFeatures api.CoreFeatures, +) (m *Module, err error) { + if moduleName != "" { + m = &Module{NameSection: &NameSection{ModuleName: moduleName}} + } else { + return nil, errors.New("a module name must not be empty") + } + + if exportCount := uint32(len(nameToHostFunc)); exportCount > 0 { + m.ExportSection = make([]Export, 0, exportCount) + m.Exports = make(map[string]*Export, exportCount) + if err = addFuncs(m, exportNames, nameToHostFunc, enabledFeatures); err != nil { + return + } + } + + m.IsHostModule = true + // Uses the address of *wasm.Module as the module ID so that host functions can have each state per compilation. + // Downside of this is that compilation cache on host functions (trampoline codes for Go functions and + // Wasm codes for Wasm-implemented host functions) are not available and compiles each time. On the other hand, + // compilation of host modules is not costly as it's merely small trampolines vs the real-world native Wasm binary. + // TODO: refactor engines so that we can properly cache compiled machine codes for host modules. + m.AssignModuleID([]byte(fmt.Sprintf("@@@@@@@@%p", m)), // @@@@@@@@ = any 8 bytes different from Wasm header. + nil, false) + return +} + +func addFuncs( + m *Module, + exportNames []string, + nameToHostFunc map[string]*HostFunc, + enabledFeatures api.CoreFeatures, +) (err error) { + if m.NameSection == nil { + m.NameSection = &NameSection{} + } + moduleName := m.NameSection.ModuleName + + for _, k := range exportNames { + hf := nameToHostFunc[k] + if hf.Name == "" { + hf.Name = k // default name to export name + } + switch hf.Code.GoFunc.(type) { + case api.GoModuleFunction, api.GoFunction: + continue // already parsed + } + + // Resolve the code using reflection + hf.ParamTypes, hf.ResultTypes, hf.Code, err = parseGoReflectFunc(hf.Code.GoFunc) + if err != nil { + return fmt.Errorf("func[%s.%s] %w", moduleName, k, err) + } + + // Assign names to the function, if they exist. + params := hf.ParamTypes + if paramNames := hf.ParamNames; paramNames != nil { + if paramNamesLen := len(paramNames); paramNamesLen != len(params) { + return fmt.Errorf("func[%s.%s] has %d params, but %d params names", moduleName, k, paramNamesLen, len(params)) + } + } + + results := hf.ResultTypes + if resultNames := hf.ResultNames; resultNames != nil { + if resultNamesLen := len(resultNames); resultNamesLen != len(results) { + return fmt.Errorf("func[%s.%s] has %d results, but %d results names", moduleName, k, resultNamesLen, len(results)) + } + } + } + + funcCount := uint32(len(exportNames)) + m.NameSection.FunctionNames = make([]NameAssoc, 0, funcCount) + m.FunctionSection = make([]Index, 0, funcCount) + m.CodeSection = make([]Code, 0, funcCount) + + idx := Index(0) + for _, name := range exportNames { + hf := nameToHostFunc[name] + debugName := wasmdebug.FuncName(moduleName, name, idx) + typeIdx, typeErr := m.maybeAddType(hf.ParamTypes, hf.ResultTypes, enabledFeatures) + if typeErr != nil { + return fmt.Errorf("func[%s] %v", debugName, typeErr) + } + m.FunctionSection = append(m.FunctionSection, typeIdx) + m.CodeSection = append(m.CodeSection, hf.Code) + + export := hf.ExportName + m.ExportSection = append(m.ExportSection, Export{Type: ExternTypeFunc, Name: export, Index: idx}) + m.Exports[export] = &m.ExportSection[len(m.ExportSection)-1] + m.NameSection.FunctionNames = append(m.NameSection.FunctionNames, NameAssoc{Index: idx, Name: hf.Name}) + + if len(hf.ParamNames) > 0 { + localNames := NameMapAssoc{Index: idx} + for i, n := range hf.ParamNames { + localNames.NameMap = append(localNames.NameMap, NameAssoc{Index: Index(i), Name: n}) + } + m.NameSection.LocalNames = append(m.NameSection.LocalNames, localNames) + } + if len(hf.ResultNames) > 0 { + resultNames := NameMapAssoc{Index: idx} + for i, n := range hf.ResultNames { + resultNames.NameMap = append(resultNames.NameMap, NameAssoc{Index: Index(i), Name: n}) + } + m.NameSection.ResultNames = append(m.NameSection.ResultNames, resultNames) + } + idx++ + } + return nil +} + +func (m *Module) maybeAddType(params, results []ValueType, enabledFeatures api.CoreFeatures) (Index, error) { + if len(results) > 1 { + // Guard >1.0 feature multi-value + if err := enabledFeatures.RequireEnabled(api.CoreFeatureMultiValue); err != nil { + return 0, fmt.Errorf("multiple result types invalid as %v", err) + } + } + for i := range m.TypeSection { + t := &m.TypeSection[i] + if t.EqualsSignature(params, results) { + return Index(i), nil + } + } + + result := m.SectionElementCount(SectionIDType) + m.TypeSection = append(m.TypeSection, FunctionType{Params: params, Results: results}) + return result, nil +} diff --git a/vendor/github.com/tetratelabs/wazero/internal/wasm/instruction.go b/vendor/github.com/tetratelabs/wazero/internal/wasm/instruction.go new file mode 100644 index 000000000..67f196b8b --- /dev/null +++ b/vendor/github.com/tetratelabs/wazero/internal/wasm/instruction.go @@ -0,0 +1,1866 @@ +package wasm + +// Opcode is the binary Opcode of an instruction. See also InstructionName +type Opcode = byte + +const ( + // OpcodeUnreachable causes an unconditional trap. + OpcodeUnreachable Opcode = 0x00 + // OpcodeNop does nothing + OpcodeNop Opcode = 0x01 + // OpcodeBlock brackets a sequence of instructions. A branch instruction on an if label breaks out to after its + // OpcodeEnd. + OpcodeBlock Opcode = 0x02 + // OpcodeLoop brackets a sequence of instructions. A branch instruction on a loop label will jump back to the + // beginning of its block. + OpcodeLoop Opcode = 0x03 + // OpcodeIf brackets a sequence of instructions. When the top of the stack evaluates to 1, the block is executed. + // Zero jumps to the optional OpcodeElse. A branch instruction on an if label breaks out to after its OpcodeEnd. + OpcodeIf Opcode = 0x04 + // OpcodeElse brackets a sequence of instructions enclosed by an OpcodeIf. A branch instruction on a then label + // breaks out to after the OpcodeEnd on the enclosing OpcodeIf. + OpcodeElse Opcode = 0x05 + // OpcodeEnd terminates a control instruction OpcodeBlock, OpcodeLoop or OpcodeIf. + OpcodeEnd Opcode = 0x0b + + // OpcodeBr is a stack-polymorphic opcode that performs an unconditional branch. How the stack is modified depends + // on whether the "br" is enclosed by a loop, and if CoreFeatureMultiValue is enabled. + // + // Here are the rules in pseudocode about how the stack is modified based on the "br" operand L (label): + // if L is loop: append(L.originalStackWithoutInputs, N-values popped from the stack) where N == L.inputs + // else: append(L.originalStackWithoutInputs, N-values popped from the stack) where N == L.results + // + // In WebAssembly 1.0 (20191205), N can be zero or one. When CoreFeatureMultiValue is enabled, N can be more than one, + // depending on the type use of the label L. + // + // See https://www.w3.org/TR/2019/REC-wasm-core-1-20191205/#-hrefsyntax-instr-controlmathsfbrl + OpcodeBr Opcode = 0x0c + + OpcodeBrIf Opcode = 0x0d + OpcodeBrTable Opcode = 0x0e + OpcodeReturn Opcode = 0x0f + OpcodeCall Opcode = 0x10 + OpcodeCallIndirect Opcode = 0x11 + + // parametric instructions + + OpcodeDrop Opcode = 0x1a + OpcodeSelect Opcode = 0x1b + OpcodeTypedSelect Opcode = 0x1c + + // variable instructions + + OpcodeLocalGet Opcode = 0x20 + OpcodeLocalSet Opcode = 0x21 + OpcodeLocalTee Opcode = 0x22 + OpcodeGlobalGet Opcode = 0x23 + OpcodeGlobalSet Opcode = 0x24 + + // Below are toggled with CoreFeatureReferenceTypes + + OpcodeTableGet Opcode = 0x25 + OpcodeTableSet Opcode = 0x26 + + // memory instructions + + OpcodeI32Load Opcode = 0x28 + OpcodeI64Load Opcode = 0x29 + OpcodeF32Load Opcode = 0x2a + OpcodeF64Load Opcode = 0x2b + OpcodeI32Load8S Opcode = 0x2c + OpcodeI32Load8U Opcode = 0x2d + OpcodeI32Load16S Opcode = 0x2e + OpcodeI32Load16U Opcode = 0x2f + OpcodeI64Load8S Opcode = 0x30 + OpcodeI64Load8U Opcode = 0x31 + OpcodeI64Load16S Opcode = 0x32 + OpcodeI64Load16U Opcode = 0x33 + OpcodeI64Load32S Opcode = 0x34 + OpcodeI64Load32U Opcode = 0x35 + OpcodeI32Store Opcode = 0x36 + OpcodeI64Store Opcode = 0x37 + OpcodeF32Store Opcode = 0x38 + OpcodeF64Store Opcode = 0x39 + OpcodeI32Store8 Opcode = 0x3a + OpcodeI32Store16 Opcode = 0x3b + OpcodeI64Store8 Opcode = 0x3c + OpcodeI64Store16 Opcode = 0x3d + OpcodeI64Store32 Opcode = 0x3e + OpcodeMemorySize Opcode = 0x3f + OpcodeMemoryGrow Opcode = 0x40 + + // const instructions + + OpcodeI32Const Opcode = 0x41 + OpcodeI64Const Opcode = 0x42 + OpcodeF32Const Opcode = 0x43 + OpcodeF64Const Opcode = 0x44 + + // numeric instructions + + OpcodeI32Eqz Opcode = 0x45 + OpcodeI32Eq Opcode = 0x46 + OpcodeI32Ne Opcode = 0x47 + OpcodeI32LtS Opcode = 0x48 + OpcodeI32LtU Opcode = 0x49 + OpcodeI32GtS Opcode = 0x4a + OpcodeI32GtU Opcode = 0x4b + OpcodeI32LeS Opcode = 0x4c + OpcodeI32LeU Opcode = 0x4d + OpcodeI32GeS Opcode = 0x4e + OpcodeI32GeU Opcode = 0x4f + + OpcodeI64Eqz Opcode = 0x50 + OpcodeI64Eq Opcode = 0x51 + OpcodeI64Ne Opcode = 0x52 + OpcodeI64LtS Opcode = 0x53 + OpcodeI64LtU Opcode = 0x54 + OpcodeI64GtS Opcode = 0x55 + OpcodeI64GtU Opcode = 0x56 + OpcodeI64LeS Opcode = 0x57 + OpcodeI64LeU Opcode = 0x58 + OpcodeI64GeS Opcode = 0x59 + OpcodeI64GeU Opcode = 0x5a + + OpcodeF32Eq Opcode = 0x5b + OpcodeF32Ne Opcode = 0x5c + OpcodeF32Lt Opcode = 0x5d + OpcodeF32Gt Opcode = 0x5e + OpcodeF32Le Opcode = 0x5f + OpcodeF32Ge Opcode = 0x60 + + OpcodeF64Eq Opcode = 0x61 + OpcodeF64Ne Opcode = 0x62 + OpcodeF64Lt Opcode = 0x63 + OpcodeF64Gt Opcode = 0x64 + OpcodeF64Le Opcode = 0x65 + OpcodeF64Ge Opcode = 0x66 + + OpcodeI32Clz Opcode = 0x67 + OpcodeI32Ctz Opcode = 0x68 + OpcodeI32Popcnt Opcode = 0x69 + OpcodeI32Add Opcode = 0x6a + OpcodeI32Sub Opcode = 0x6b + OpcodeI32Mul Opcode = 0x6c + OpcodeI32DivS Opcode = 0x6d + OpcodeI32DivU Opcode = 0x6e + OpcodeI32RemS Opcode = 0x6f + OpcodeI32RemU Opcode = 0x70 + OpcodeI32And Opcode = 0x71 + OpcodeI32Or Opcode = 0x72 + OpcodeI32Xor Opcode = 0x73 + OpcodeI32Shl Opcode = 0x74 + OpcodeI32ShrS Opcode = 0x75 + OpcodeI32ShrU Opcode = 0x76 + OpcodeI32Rotl Opcode = 0x77 + OpcodeI32Rotr Opcode = 0x78 + + OpcodeI64Clz Opcode = 0x79 + OpcodeI64Ctz Opcode = 0x7a + OpcodeI64Popcnt Opcode = 0x7b + OpcodeI64Add Opcode = 0x7c + OpcodeI64Sub Opcode = 0x7d + OpcodeI64Mul Opcode = 0x7e + OpcodeI64DivS Opcode = 0x7f + OpcodeI64DivU Opcode = 0x80 + OpcodeI64RemS Opcode = 0x81 + OpcodeI64RemU Opcode = 0x82 + OpcodeI64And Opcode = 0x83 + OpcodeI64Or Opcode = 0x84 + OpcodeI64Xor Opcode = 0x85 + OpcodeI64Shl Opcode = 0x86 + OpcodeI64ShrS Opcode = 0x87 + OpcodeI64ShrU Opcode = 0x88 + OpcodeI64Rotl Opcode = 0x89 + OpcodeI64Rotr Opcode = 0x8a + + OpcodeF32Abs Opcode = 0x8b + OpcodeF32Neg Opcode = 0x8c + OpcodeF32Ceil Opcode = 0x8d + OpcodeF32Floor Opcode = 0x8e + OpcodeF32Trunc Opcode = 0x8f + OpcodeF32Nearest Opcode = 0x90 + OpcodeF32Sqrt Opcode = 0x91 + OpcodeF32Add Opcode = 0x92 + OpcodeF32Sub Opcode = 0x93 + OpcodeF32Mul Opcode = 0x94 + OpcodeF32Div Opcode = 0x95 + OpcodeF32Min Opcode = 0x96 + OpcodeF32Max Opcode = 0x97 + OpcodeF32Copysign Opcode = 0x98 + + OpcodeF64Abs Opcode = 0x99 + OpcodeF64Neg Opcode = 0x9a + OpcodeF64Ceil Opcode = 0x9b + OpcodeF64Floor Opcode = 0x9c + OpcodeF64Trunc Opcode = 0x9d + OpcodeF64Nearest Opcode = 0x9e + OpcodeF64Sqrt Opcode = 0x9f + OpcodeF64Add Opcode = 0xa0 + OpcodeF64Sub Opcode = 0xa1 + OpcodeF64Mul Opcode = 0xa2 + OpcodeF64Div Opcode = 0xa3 + OpcodeF64Min Opcode = 0xa4 + OpcodeF64Max Opcode = 0xa5 + OpcodeF64Copysign Opcode = 0xa6 + + OpcodeI32WrapI64 Opcode = 0xa7 + OpcodeI32TruncF32S Opcode = 0xa8 + OpcodeI32TruncF32U Opcode = 0xa9 + OpcodeI32TruncF64S Opcode = 0xaa + OpcodeI32TruncF64U Opcode = 0xab + + OpcodeI64ExtendI32S Opcode = 0xac + OpcodeI64ExtendI32U Opcode = 0xad + OpcodeI64TruncF32S Opcode = 0xae + OpcodeI64TruncF32U Opcode = 0xaf + OpcodeI64TruncF64S Opcode = 0xb0 + OpcodeI64TruncF64U Opcode = 0xb1 + + OpcodeF32ConvertI32S Opcode = 0xb2 + OpcodeF32ConvertI32U Opcode = 0xb3 + OpcodeF32ConvertI64S Opcode = 0xb4 + OpcodeF32ConvertI64U Opcode = 0xb5 + OpcodeF32DemoteF64 Opcode = 0xb6 + + OpcodeF64ConvertI32S Opcode = 0xb7 + OpcodeF64ConvertI32U Opcode = 0xb8 + OpcodeF64ConvertI64S Opcode = 0xb9 + OpcodeF64ConvertI64U Opcode = 0xba + OpcodeF64PromoteF32 Opcode = 0xbb + + OpcodeI32ReinterpretF32 Opcode = 0xbc + OpcodeI64ReinterpretF64 Opcode = 0xbd + OpcodeF32ReinterpretI32 Opcode = 0xbe + OpcodeF64ReinterpretI64 Opcode = 0xbf + + // OpcodeRefNull pushes a null reference value whose type is specified by immediate to this opcode. + // This is defined in the reference-types proposal, but necessary for CoreFeatureBulkMemoryOperations as well. + // + // Currently only supported in the constant expression in element segments. + OpcodeRefNull = 0xd0 + // OpcodeRefIsNull pops a reference value, and pushes 1 if it is null, 0 otherwise. + // This is defined in the reference-types proposal, but necessary for CoreFeatureBulkMemoryOperations as well. + // + // Currently not supported. + OpcodeRefIsNull = 0xd1 + // OpcodeRefFunc pushes a funcref value whose index equals the immediate to this opcode. + // This is defined in the reference-types proposal, but necessary for CoreFeatureBulkMemoryOperations as well. + // + // Currently, this is only supported in the constant expression in element segments. + OpcodeRefFunc = 0xd2 + + // Below are toggled with CoreFeatureSignExtensionOps + + // OpcodeI32Extend8S extends a signed 8-bit integer to a 32-bit integer. + // Note: This is dependent on the flag CoreFeatureSignExtensionOps + OpcodeI32Extend8S Opcode = 0xc0 + + // OpcodeI32Extend16S extends a signed 16-bit integer to a 32-bit integer. + // Note: This is dependent on the flag CoreFeatureSignExtensionOps + OpcodeI32Extend16S Opcode = 0xc1 + + // OpcodeI64Extend8S extends a signed 8-bit integer to a 64-bit integer. + // Note: This is dependent on the flag CoreFeatureSignExtensionOps + OpcodeI64Extend8S Opcode = 0xc2 + + // OpcodeI64Extend16S extends a signed 16-bit integer to a 64-bit integer. + // Note: This is dependent on the flag CoreFeatureSignExtensionOps + OpcodeI64Extend16S Opcode = 0xc3 + + // OpcodeI64Extend32S extends a signed 32-bit integer to a 64-bit integer. + // Note: This is dependent on the flag CoreFeatureSignExtensionOps + OpcodeI64Extend32S Opcode = 0xc4 + + // OpcodeMiscPrefix is the prefix of various multi-byte opcodes. + // Introduced in CoreFeatureNonTrappingFloatToIntConversion, but used in other + // features, such as CoreFeatureBulkMemoryOperations. + OpcodeMiscPrefix Opcode = 0xfc + + // OpcodeVecPrefix is the prefix of all vector isntructions introduced in + // CoreFeatureSIMD. + OpcodeVecPrefix Opcode = 0xfd + + // OpcodeAtomicPrefix is the prefix of all atomic instructions introduced in + // CoreFeatureThreads. + OpcodeAtomicPrefix Opcode = 0xfe +) + +// OpcodeMisc represents opcodes of the miscellaneous operations. +// Such an operations has multi-byte encoding which is prefixed by OpcodeMiscPrefix. +type OpcodeMisc = byte + +const ( + // Below are toggled with CoreFeatureNonTrappingFloatToIntConversion. + // https://github.com/WebAssembly/spec/blob/ce4b6c4d47eb06098cc7ab2e81f24748da822f20/proposals/nontrapping-float-to-int-conversion/Overview.md + + OpcodeMiscI32TruncSatF32S OpcodeMisc = 0x00 + OpcodeMiscI32TruncSatF32U OpcodeMisc = 0x01 + OpcodeMiscI32TruncSatF64S OpcodeMisc = 0x02 + OpcodeMiscI32TruncSatF64U OpcodeMisc = 0x03 + OpcodeMiscI64TruncSatF32S OpcodeMisc = 0x04 + OpcodeMiscI64TruncSatF32U OpcodeMisc = 0x05 + OpcodeMiscI64TruncSatF64S OpcodeMisc = 0x06 + OpcodeMiscI64TruncSatF64U OpcodeMisc = 0x07 + + // Below are toggled with CoreFeatureBulkMemoryOperations. + // Opcodes are those new in document/core/appendix/index-instructions.rst (the commit that merged the feature). + // See https://github.com/WebAssembly/spec/commit/7fa2f20a6df4cf1c114582c8cb60f5bfcdbf1be1 + // See https://www.w3.org/TR/2022/WD-wasm-core-2-20220419/appendix/changes.html#bulk-memory-and-table-instructions + + OpcodeMiscMemoryInit OpcodeMisc = 0x08 + OpcodeMiscDataDrop OpcodeMisc = 0x09 + OpcodeMiscMemoryCopy OpcodeMisc = 0x0a + OpcodeMiscMemoryFill OpcodeMisc = 0x0b + OpcodeMiscTableInit OpcodeMisc = 0x0c + OpcodeMiscElemDrop OpcodeMisc = 0x0d + OpcodeMiscTableCopy OpcodeMisc = 0x0e + + // Below are toggled with CoreFeatureReferenceTypes + + OpcodeMiscTableGrow OpcodeMisc = 0x0f + OpcodeMiscTableSize OpcodeMisc = 0x10 + OpcodeMiscTableFill OpcodeMisc = 0x11 +) + +// OpcodeVec represents an opcode of a vector instructions which has +// multi-byte encoding and is prefixed by OpcodeMiscPrefix. +// +// These opcodes are toggled with CoreFeatureSIMD. +type OpcodeVec = byte + +const ( + // Loads and stores. + + OpcodeVecV128Load OpcodeVec = 0x00 + OpcodeVecV128Load8x8s OpcodeVec = 0x01 + OpcodeVecV128Load8x8u OpcodeVec = 0x02 + OpcodeVecV128Load16x4s OpcodeVec = 0x03 + OpcodeVecV128Load16x4u OpcodeVec = 0x04 + OpcodeVecV128Load32x2s OpcodeVec = 0x05 + OpcodeVecV128Load32x2u OpcodeVec = 0x06 + OpcodeVecV128Load8Splat OpcodeVec = 0x07 + OpcodeVecV128Load16Splat OpcodeVec = 0x08 + OpcodeVecV128Load32Splat OpcodeVec = 0x09 + OpcodeVecV128Load64Splat OpcodeVec = 0x0a + + OpcodeVecV128Load32zero OpcodeVec = 0x5c + OpcodeVecV128Load64zero OpcodeVec = 0x5d + + OpcodeVecV128Store OpcodeVec = 0x0b + OpcodeVecV128Load8Lane OpcodeVec = 0x54 + OpcodeVecV128Load16Lane OpcodeVec = 0x55 + OpcodeVecV128Load32Lane OpcodeVec = 0x56 + OpcodeVecV128Load64Lane OpcodeVec = 0x57 + OpcodeVecV128Store8Lane OpcodeVec = 0x58 + OpcodeVecV128Store16Lane OpcodeVec = 0x59 + OpcodeVecV128Store32Lane OpcodeVec = 0x5a + OpcodeVecV128Store64Lane OpcodeVec = 0x5b + + // OpcodeVecV128Const is the vector const instruction. + OpcodeVecV128Const OpcodeVec = 0x0c + + // OpcodeVecV128i8x16Shuffle is the vector shuffle instruction. + OpcodeVecV128i8x16Shuffle OpcodeVec = 0x0d + + // Extrac and replaces. + + OpcodeVecI8x16ExtractLaneS OpcodeVec = 0x15 + OpcodeVecI8x16ExtractLaneU OpcodeVec = 0x16 + OpcodeVecI8x16ReplaceLane OpcodeVec = 0x17 + OpcodeVecI16x8ExtractLaneS OpcodeVec = 0x18 + OpcodeVecI16x8ExtractLaneU OpcodeVec = 0x19 + OpcodeVecI16x8ReplaceLane OpcodeVec = 0x1a + OpcodeVecI32x4ExtractLane OpcodeVec = 0x1b + OpcodeVecI32x4ReplaceLane OpcodeVec = 0x1c + OpcodeVecI64x2ExtractLane OpcodeVec = 0x1d + OpcodeVecI64x2ReplaceLane OpcodeVec = 0x1e + OpcodeVecF32x4ExtractLane OpcodeVec = 0x1f + OpcodeVecF32x4ReplaceLane OpcodeVec = 0x20 + OpcodeVecF64x2ExtractLane OpcodeVec = 0x21 + OpcodeVecF64x2ReplaceLane OpcodeVec = 0x22 + + // Splat and swizzle. + + OpcodeVecI8x16Swizzle OpcodeVec = 0x0e + OpcodeVecI8x16Splat OpcodeVec = 0x0f + OpcodeVecI16x8Splat OpcodeVec = 0x10 + OpcodeVecI32x4Splat OpcodeVec = 0x11 + OpcodeVecI64x2Splat OpcodeVec = 0x12 + OpcodeVecF32x4Splat OpcodeVec = 0x13 + OpcodeVecF64x2Splat OpcodeVec = 0x14 + + // i8 comparisons. + + OpcodeVecI8x16Eq OpcodeVec = 0x23 + OpcodeVecI8x16Ne OpcodeVec = 0x24 + OpcodeVecI8x16LtS OpcodeVec = 0x25 + OpcodeVecI8x16LtU OpcodeVec = 0x26 + OpcodeVecI8x16GtS OpcodeVec = 0x27 + OpcodeVecI8x16GtU OpcodeVec = 0x28 + OpcodeVecI8x16LeS OpcodeVec = 0x29 + OpcodeVecI8x16LeU OpcodeVec = 0x2a + OpcodeVecI8x16GeS OpcodeVec = 0x2b + OpcodeVecI8x16GeU OpcodeVec = 0x2c + + // i16 comparisons. + + OpcodeVecI16x8Eq OpcodeVec = 0x2d + OpcodeVecI16x8Ne OpcodeVec = 0x2e + OpcodeVecI16x8LtS OpcodeVec = 0x2f + OpcodeVecI16x8LtU OpcodeVec = 0x30 + OpcodeVecI16x8GtS OpcodeVec = 0x31 + OpcodeVecI16x8GtU OpcodeVec = 0x32 + OpcodeVecI16x8LeS OpcodeVec = 0x33 + OpcodeVecI16x8LeU OpcodeVec = 0x34 + OpcodeVecI16x8GeS OpcodeVec = 0x35 + OpcodeVecI16x8GeU OpcodeVec = 0x36 + + // i32 comparisons. + + OpcodeVecI32x4Eq OpcodeVec = 0x37 + OpcodeVecI32x4Ne OpcodeVec = 0x38 + OpcodeVecI32x4LtS OpcodeVec = 0x39 + OpcodeVecI32x4LtU OpcodeVec = 0x3a + OpcodeVecI32x4GtS OpcodeVec = 0x3b + OpcodeVecI32x4GtU OpcodeVec = 0x3c + OpcodeVecI32x4LeS OpcodeVec = 0x3d + OpcodeVecI32x4LeU OpcodeVec = 0x3e + OpcodeVecI32x4GeS OpcodeVec = 0x3f + OpcodeVecI32x4GeU OpcodeVec = 0x40 + + // i64 comparisons. + + OpcodeVecI64x2Eq OpcodeVec = 0xd6 + OpcodeVecI64x2Ne OpcodeVec = 0xd7 + OpcodeVecI64x2LtS OpcodeVec = 0xd8 + OpcodeVecI64x2GtS OpcodeVec = 0xd9 + OpcodeVecI64x2LeS OpcodeVec = 0xda + OpcodeVecI64x2GeS OpcodeVec = 0xdb + + // f32 comparisons. + + OpcodeVecF32x4Eq OpcodeVec = 0x41 + OpcodeVecF32x4Ne OpcodeVec = 0x42 + OpcodeVecF32x4Lt OpcodeVec = 0x43 + OpcodeVecF32x4Gt OpcodeVec = 0x44 + OpcodeVecF32x4Le OpcodeVec = 0x45 + OpcodeVecF32x4Ge OpcodeVec = 0x46 + + // f64 comparisons. + + OpcodeVecF64x2Eq OpcodeVec = 0x47 + OpcodeVecF64x2Ne OpcodeVec = 0x48 + OpcodeVecF64x2Lt OpcodeVec = 0x49 + OpcodeVecF64x2Gt OpcodeVec = 0x4a + OpcodeVecF64x2Le OpcodeVec = 0x4b + OpcodeVecF64x2Ge OpcodeVec = 0x4c + + // v128 logical instructions. + + OpcodeVecV128Not OpcodeVec = 0x4d + OpcodeVecV128And OpcodeVec = 0x4e + OpcodeVecV128AndNot OpcodeVec = 0x4f + OpcodeVecV128Or OpcodeVec = 0x50 + OpcodeVecV128Xor OpcodeVec = 0x51 + OpcodeVecV128Bitselect OpcodeVec = 0x52 + OpcodeVecV128AnyTrue OpcodeVec = 0x53 + + // i8 misc. + + OpcodeVecI8x16Abs OpcodeVec = 0x60 + OpcodeVecI8x16Neg OpcodeVec = 0x61 + OpcodeVecI8x16Popcnt OpcodeVec = 0x62 + OpcodeVecI8x16AllTrue OpcodeVec = 0x63 + OpcodeVecI8x16BitMask OpcodeVec = 0x64 + OpcodeVecI8x16NarrowI16x8S OpcodeVec = 0x65 + OpcodeVecI8x16NarrowI16x8U OpcodeVec = 0x66 + + OpcodeVecI8x16Shl OpcodeVec = 0x6b + OpcodeVecI8x16ShrS OpcodeVec = 0x6c + OpcodeVecI8x16ShrU OpcodeVec = 0x6d + OpcodeVecI8x16Add OpcodeVec = 0x6e + OpcodeVecI8x16AddSatS OpcodeVec = 0x6f + + OpcodeVecI8x16AddSatU OpcodeVec = 0x70 + OpcodeVecI8x16Sub OpcodeVec = 0x71 + OpcodeVecI8x16SubSatS OpcodeVec = 0x72 + OpcodeVecI8x16SubSatU OpcodeVec = 0x73 + OpcodeVecI8x16MinS OpcodeVec = 0x76 + OpcodeVecI8x16MinU OpcodeVec = 0x77 + OpcodeVecI8x16MaxS OpcodeVec = 0x78 + OpcodeVecI8x16MaxU OpcodeVec = 0x79 + OpcodeVecI8x16AvgrU OpcodeVec = 0x7b + + // i16 misc. + + OpcodeVecI16x8ExtaddPairwiseI8x16S OpcodeVec = 0x7c + OpcodeVecI16x8ExtaddPairwiseI8x16U OpcodeVec = 0x7d + OpcodeVecI16x8Abs OpcodeVec = 0x80 + OpcodeVecI16x8Neg OpcodeVec = 0x81 + OpcodeVecI16x8Q15mulrSatS OpcodeVec = 0x82 + OpcodeVecI16x8AllTrue OpcodeVec = 0x83 + OpcodeVecI16x8BitMask OpcodeVec = 0x84 + OpcodeVecI16x8NarrowI32x4S OpcodeVec = 0x85 + OpcodeVecI16x8NarrowI32x4U OpcodeVec = 0x86 + OpcodeVecI16x8ExtendLowI8x16S OpcodeVec = 0x87 + OpcodeVecI16x8ExtendHighI8x16S OpcodeVec = 0x88 + OpcodeVecI16x8ExtendLowI8x16U OpcodeVec = 0x89 + OpcodeVecI16x8ExtendHighI8x16U OpcodeVec = 0x8a + OpcodeVecI16x8Shl OpcodeVec = 0x8b + OpcodeVecI16x8ShrS OpcodeVec = 0x8c + OpcodeVecI16x8ShrU OpcodeVec = 0x8d + OpcodeVecI16x8Add OpcodeVec = 0x8e + OpcodeVecI16x8AddSatS OpcodeVec = 0x8f + OpcodeVecI16x8AddSatU OpcodeVec = 0x90 + OpcodeVecI16x8Sub OpcodeVec = 0x91 + OpcodeVecI16x8SubSatS OpcodeVec = 0x92 + OpcodeVecI16x8SubSatU OpcodeVec = 0x93 + OpcodeVecI16x8Mul OpcodeVec = 0x95 + OpcodeVecI16x8MinS OpcodeVec = 0x96 + OpcodeVecI16x8MinU OpcodeVec = 0x97 + OpcodeVecI16x8MaxS OpcodeVec = 0x98 + OpcodeVecI16x8MaxU OpcodeVec = 0x99 + OpcodeVecI16x8AvgrU OpcodeVec = 0x9b + OpcodeVecI16x8ExtMulLowI8x16S OpcodeVec = 0x9c + OpcodeVecI16x8ExtMulHighI8x16S OpcodeVec = 0x9d + OpcodeVecI16x8ExtMulLowI8x16U OpcodeVec = 0x9e + OpcodeVecI16x8ExtMulHighI8x16U OpcodeVec = 0x9f + + // i32 misc. + + OpcodeVecI32x4ExtaddPairwiseI16x8S OpcodeVec = 0x7e + OpcodeVecI32x4ExtaddPairwiseI16x8U OpcodeVec = 0x7f + OpcodeVecI32x4Abs OpcodeVec = 0xa0 + OpcodeVecI32x4Neg OpcodeVec = 0xa1 + OpcodeVecI32x4AllTrue OpcodeVec = 0xa3 + OpcodeVecI32x4BitMask OpcodeVec = 0xa4 + OpcodeVecI32x4ExtendLowI16x8S OpcodeVec = 0xa7 + OpcodeVecI32x4ExtendHighI16x8S OpcodeVec = 0xa8 + OpcodeVecI32x4ExtendLowI16x8U OpcodeVec = 0xa9 + OpcodeVecI32x4ExtendHighI16x8U OpcodeVec = 0xaa + OpcodeVecI32x4Shl OpcodeVec = 0xab + OpcodeVecI32x4ShrS OpcodeVec = 0xac + OpcodeVecI32x4ShrU OpcodeVec = 0xad + OpcodeVecI32x4Add OpcodeVec = 0xae + OpcodeVecI32x4Sub OpcodeVec = 0xb1 + OpcodeVecI32x4Mul OpcodeVec = 0xb5 + OpcodeVecI32x4MinS OpcodeVec = 0xb6 + OpcodeVecI32x4MinU OpcodeVec = 0xb7 + OpcodeVecI32x4MaxS OpcodeVec = 0xb8 + OpcodeVecI32x4MaxU OpcodeVec = 0xb9 + OpcodeVecI32x4DotI16x8S OpcodeVec = 0xba + OpcodeVecI32x4ExtMulLowI16x8S OpcodeVec = 0xbc + OpcodeVecI32x4ExtMulHighI16x8S OpcodeVec = 0xbd + OpcodeVecI32x4ExtMulLowI16x8U OpcodeVec = 0xbe + OpcodeVecI32x4ExtMulHighI16x8U OpcodeVec = 0xbf + + // i64 misc. + + OpcodeVecI64x2Abs OpcodeVec = 0xc0 + OpcodeVecI64x2Neg OpcodeVec = 0xc1 + OpcodeVecI64x2AllTrue OpcodeVec = 0xc3 + OpcodeVecI64x2BitMask OpcodeVec = 0xc4 + OpcodeVecI64x2ExtendLowI32x4S OpcodeVec = 0xc7 + OpcodeVecI64x2ExtendHighI32x4S OpcodeVec = 0xc8 + OpcodeVecI64x2ExtendLowI32x4U OpcodeVec = 0xc9 + OpcodeVecI64x2ExtendHighI32x4U OpcodeVec = 0xca + OpcodeVecI64x2Shl OpcodeVec = 0xcb + OpcodeVecI64x2ShrS OpcodeVec = 0xcc + OpcodeVecI64x2ShrU OpcodeVec = 0xcd + OpcodeVecI64x2Add OpcodeVec = 0xce + OpcodeVecI64x2Sub OpcodeVec = 0xd1 + OpcodeVecI64x2Mul OpcodeVec = 0xd5 + OpcodeVecI64x2ExtMulLowI32x4S OpcodeVec = 0xdc + OpcodeVecI64x2ExtMulHighI32x4S OpcodeVec = 0xdd + OpcodeVecI64x2ExtMulLowI32x4U OpcodeVec = 0xde + OpcodeVecI64x2ExtMulHighI32x4U OpcodeVec = 0xdf + + // f32 misc. + + OpcodeVecF32x4Ceil OpcodeVec = 0x67 + OpcodeVecF32x4Floor OpcodeVec = 0x68 + OpcodeVecF32x4Trunc OpcodeVec = 0x69 + OpcodeVecF32x4Nearest OpcodeVec = 0x6a + OpcodeVecF32x4Abs OpcodeVec = 0xe0 + OpcodeVecF32x4Neg OpcodeVec = 0xe1 + OpcodeVecF32x4Sqrt OpcodeVec = 0xe3 + OpcodeVecF32x4Add OpcodeVec = 0xe4 + OpcodeVecF32x4Sub OpcodeVec = 0xe5 + OpcodeVecF32x4Mul OpcodeVec = 0xe6 + OpcodeVecF32x4Div OpcodeVec = 0xe7 + OpcodeVecF32x4Min OpcodeVec = 0xe8 + OpcodeVecF32x4Max OpcodeVec = 0xe9 + OpcodeVecF32x4Pmin OpcodeVec = 0xea + OpcodeVecF32x4Pmax OpcodeVec = 0xeb + + // f64 misc. + + OpcodeVecF64x2Ceil OpcodeVec = 0x74 + OpcodeVecF64x2Floor OpcodeVec = 0x75 + OpcodeVecF64x2Trunc OpcodeVec = 0x7a + OpcodeVecF64x2Nearest OpcodeVec = 0x94 + OpcodeVecF64x2Abs OpcodeVec = 0xec + OpcodeVecF64x2Neg OpcodeVec = 0xed + OpcodeVecF64x2Sqrt OpcodeVec = 0xef + OpcodeVecF64x2Add OpcodeVec = 0xf0 + OpcodeVecF64x2Sub OpcodeVec = 0xf1 + OpcodeVecF64x2Mul OpcodeVec = 0xf2 + OpcodeVecF64x2Div OpcodeVec = 0xf3 + OpcodeVecF64x2Min OpcodeVec = 0xf4 + OpcodeVecF64x2Max OpcodeVec = 0xf5 + OpcodeVecF64x2Pmin OpcodeVec = 0xf6 + OpcodeVecF64x2Pmax OpcodeVec = 0xf7 + + // conversions. + + OpcodeVecI32x4TruncSatF32x4S OpcodeVec = 0xf8 + OpcodeVecI32x4TruncSatF32x4U OpcodeVec = 0xf9 + OpcodeVecF32x4ConvertI32x4S OpcodeVec = 0xfa + OpcodeVecF32x4ConvertI32x4U OpcodeVec = 0xfb + OpcodeVecI32x4TruncSatF64x2SZero OpcodeVec = 0xfc + OpcodeVecI32x4TruncSatF64x2UZero OpcodeVec = 0xfd + OpcodeVecF64x2ConvertLowI32x4S OpcodeVec = 0xfe + OpcodeVecF64x2ConvertLowI32x4U OpcodeVec = 0xff + OpcodeVecF32x4DemoteF64x2Zero OpcodeVec = 0x5e + OpcodeVecF64x2PromoteLowF32x4Zero OpcodeVec = 0x5f +) + +// OpcodeAtomic represents an opcode of atomic instructions which has +// multi-byte encoding and is prefixed by OpcodeAtomicPrefix. +// +// These opcodes are toggled with CoreFeaturesThreads. +type OpcodeAtomic = byte + +const ( + // OpcodeAtomicMemoryNotify represents the instruction memory.atomic.notify. + OpcodeAtomicMemoryNotify OpcodeAtomic = 0x00 + // OpcodeAtomicMemoryWait32 represents the instruction memory.atomic.wait32. + OpcodeAtomicMemoryWait32 OpcodeAtomic = 0x01 + // OpcodeAtomicMemoryWait64 represents the instruction memory.atomic.wait64. + OpcodeAtomicMemoryWait64 OpcodeAtomic = 0x02 + // OpcodeAtomicFence represents the instruction atomic.fence. + OpcodeAtomicFence OpcodeAtomic = 0x03 + + // OpcodeAtomicI32Load represents the instruction i32.atomic.load. + OpcodeAtomicI32Load OpcodeAtomic = 0x10 + // OpcodeAtomicI64Load represents the instruction i64.atomic.load. + OpcodeAtomicI64Load OpcodeAtomic = 0x11 + // OpcodeAtomicI32Load8U represents the instruction i32.atomic.load8_u. + OpcodeAtomicI32Load8U OpcodeAtomic = 0x12 + // OpcodeAtomicI32Load16U represents the instruction i32.atomic.load16_u. + OpcodeAtomicI32Load16U OpcodeAtomic = 0x13 + // OpcodeAtomicI64Load8U represents the instruction i64.atomic.load8_u. + OpcodeAtomicI64Load8U OpcodeAtomic = 0x14 + // OpcodeAtomicI64Load16U represents the instruction i64.atomic.load16_u. + OpcodeAtomicI64Load16U OpcodeAtomic = 0x15 + // OpcodeAtomicI64Load32U represents the instruction i64.atomic.load32_u. + OpcodeAtomicI64Load32U OpcodeAtomic = 0x16 + // OpcodeAtomicI32Store represents the instruction i32.atomic.store. + OpcodeAtomicI32Store OpcodeAtomic = 0x17 + // OpcodeAtomicI64Store represents the instruction i64.atomic.store. + OpcodeAtomicI64Store OpcodeAtomic = 0x18 + // OpcodeAtomicI32Store8 represents the instruction i32.atomic.store8. + OpcodeAtomicI32Store8 OpcodeAtomic = 0x19 + // OpcodeAtomicI32Store16 represents the instruction i32.atomic.store16. + OpcodeAtomicI32Store16 OpcodeAtomic = 0x1a + // OpcodeAtomicI64Store8 represents the instruction i64.atomic.store8. + OpcodeAtomicI64Store8 OpcodeAtomic = 0x1b + // OpcodeAtomicI64Store16 represents the instruction i64.atomic.store16. + OpcodeAtomicI64Store16 OpcodeAtomic = 0x1c + // OpcodeAtomicI64Store32 represents the instruction i64.atomic.store32. + OpcodeAtomicI64Store32 OpcodeAtomic = 0x1d + + // OpcodeAtomicI32RmwAdd represents the instruction i32.atomic.rmw.add. + OpcodeAtomicI32RmwAdd OpcodeAtomic = 0x1e + // OpcodeAtomicI64RmwAdd represents the instruction i64.atomic.rmw.add. + OpcodeAtomicI64RmwAdd OpcodeAtomic = 0x1f + // OpcodeAtomicI32Rmw8AddU represents the instruction i32.atomic.rmw8.add_u. + OpcodeAtomicI32Rmw8AddU OpcodeAtomic = 0x20 + // OpcodeAtomicI32Rmw16AddU represents the instruction i32.atomic.rmw16.add_u. + OpcodeAtomicI32Rmw16AddU OpcodeAtomic = 0x21 + // OpcodeAtomicI64Rmw8AddU represents the instruction i64.atomic.rmw8.add_u. + OpcodeAtomicI64Rmw8AddU OpcodeAtomic = 0x22 + // OpcodeAtomicI64Rmw16AddU represents the instruction i64.atomic.rmw16.add_u. + OpcodeAtomicI64Rmw16AddU OpcodeAtomic = 0x23 + // OpcodeAtomicI64Rmw32AddU represents the instruction i64.atomic.rmw32.add_u. + OpcodeAtomicI64Rmw32AddU OpcodeAtomic = 0x24 + + // OpcodeAtomicI32RmwSub represents the instruction i32.atomic.rmw.sub. + OpcodeAtomicI32RmwSub OpcodeAtomic = 0x25 + // OpcodeAtomicI64RmwSub represents the instruction i64.atomic.rmw.sub. + OpcodeAtomicI64RmwSub OpcodeAtomic = 0x26 + // OpcodeAtomicI32Rmw8SubU represents the instruction i32.atomic.rmw8.sub_u. + OpcodeAtomicI32Rmw8SubU OpcodeAtomic = 0x27 + // OpcodeAtomicI32Rmw16SubU represents the instruction i32.atomic.rmw16.sub_u. + OpcodeAtomicI32Rmw16SubU OpcodeAtomic = 0x28 + // OpcodeAtomicI64Rmw8SubU represents the instruction i64.atomic.rmw8.sub_u. + OpcodeAtomicI64Rmw8SubU OpcodeAtomic = 0x29 + // OpcodeAtomicI64Rmw16SubU represents the instruction i64.atomic.rmw16.sub_u. + OpcodeAtomicI64Rmw16SubU OpcodeAtomic = 0x2a + // OpcodeAtomicI64Rmw32SubU represents the instruction i64.atomic.rmw32.sub_u. + OpcodeAtomicI64Rmw32SubU OpcodeAtomic = 0x2b + + // OpcodeAtomicI32RmwAnd represents the instruction i32.atomic.rmw.and. + OpcodeAtomicI32RmwAnd OpcodeAtomic = 0x2c + // OpcodeAtomicI64RmwAnd represents the instruction i64.atomic.rmw.and. + OpcodeAtomicI64RmwAnd OpcodeAtomic = 0x2d + // OpcodeAtomicI32Rmw8AndU represents the instruction i32.atomic.rmw8.and_u. + OpcodeAtomicI32Rmw8AndU OpcodeAtomic = 0x2e + // OpcodeAtomicI32Rmw16AndU represents the instruction i32.atomic.rmw16.and_u. + OpcodeAtomicI32Rmw16AndU OpcodeAtomic = 0x2f + // OpcodeAtomicI64Rmw8AndU represents the instruction i64.atomic.rmw8.and_u. + OpcodeAtomicI64Rmw8AndU OpcodeAtomic = 0x30 + // OpcodeAtomicI64Rmw16AndU represents the instruction i64.atomic.rmw16.and_u. + OpcodeAtomicI64Rmw16AndU OpcodeAtomic = 0x31 + // OpcodeAtomicI64Rmw32AndU represents the instruction i64.atomic.rmw32.and_u. + OpcodeAtomicI64Rmw32AndU OpcodeAtomic = 0x32 + + // OpcodeAtomicI32RmwOr represents the instruction i32.atomic.rmw.or. + OpcodeAtomicI32RmwOr OpcodeAtomic = 0x33 + // OpcodeAtomicI64RmwOr represents the instruction i64.atomic.rmw.or. + OpcodeAtomicI64RmwOr OpcodeAtomic = 0x34 + // OpcodeAtomicI32Rmw8OrU represents the instruction i32.atomic.rmw8.or_u. + OpcodeAtomicI32Rmw8OrU OpcodeAtomic = 0x35 + // OpcodeAtomicI32Rmw16OrU represents the instruction i32.atomic.rmw16.or_u. + OpcodeAtomicI32Rmw16OrU OpcodeAtomic = 0x36 + // OpcodeAtomicI64Rmw8OrU represents the instruction i64.atomic.rmw8.or_u. + OpcodeAtomicI64Rmw8OrU OpcodeAtomic = 0x37 + // OpcodeAtomicI64Rmw16OrU represents the instruction i64.atomic.rmw16.or_u. + OpcodeAtomicI64Rmw16OrU OpcodeAtomic = 0x38 + // OpcodeAtomicI64Rmw32OrU represents the instruction i64.atomic.rmw32.or_u. + OpcodeAtomicI64Rmw32OrU OpcodeAtomic = 0x39 + + // OpcodeAtomicI32RmwXor represents the instruction i32.atomic.rmw.xor. + OpcodeAtomicI32RmwXor OpcodeAtomic = 0x3a + // OpcodeAtomicI64RmwXor represents the instruction i64.atomic.rmw.xor. + OpcodeAtomicI64RmwXor OpcodeAtomic = 0x3b + // OpcodeAtomicI32Rmw8XorU represents the instruction i32.atomic.rmw8.xor_u. + OpcodeAtomicI32Rmw8XorU OpcodeAtomic = 0x3c + // OpcodeAtomicI32Rmw16XorU represents the instruction i32.atomic.rmw16.xor_u. + OpcodeAtomicI32Rmw16XorU OpcodeAtomic = 0x3d + // OpcodeAtomicI64Rmw8XorU represents the instruction i64.atomic.rmw8.xor_u. + OpcodeAtomicI64Rmw8XorU OpcodeAtomic = 0x3e + // OpcodeAtomicI64Rmw16XorU represents the instruction i64.atomic.rmw16.xor_u. + OpcodeAtomicI64Rmw16XorU OpcodeAtomic = 0x3f + // OpcodeAtomicI64Rmw32XorU represents the instruction i64.atomic.rmw32.xor_u. + OpcodeAtomicI64Rmw32XorU OpcodeAtomic = 0x40 + + // OpcodeAtomicI32RmwXchg represents the instruction i32.atomic.rmw.xchg. + OpcodeAtomicI32RmwXchg OpcodeAtomic = 0x41 + // OpcodeAtomicI64RmwXchg represents the instruction i64.atomic.rmw.xchg. + OpcodeAtomicI64RmwXchg OpcodeAtomic = 0x42 + // OpcodeAtomicI32Rmw8XchgU represents the instruction i32.atomic.rmw8.xchg_u. + OpcodeAtomicI32Rmw8XchgU OpcodeAtomic = 0x43 + // OpcodeAtomicI32Rmw16XchgU represents the instruction i32.atomic.rmw16.xchg_u. + OpcodeAtomicI32Rmw16XchgU OpcodeAtomic = 0x44 + // OpcodeAtomicI64Rmw8XchgU represents the instruction i64.atomic.rmw8.xchg_u. + OpcodeAtomicI64Rmw8XchgU OpcodeAtomic = 0x45 + // OpcodeAtomicI64Rmw16XchgU represents the instruction i64.atomic.rmw16.xchg_u. + OpcodeAtomicI64Rmw16XchgU OpcodeAtomic = 0x46 + // OpcodeAtomicI64Rmw32XchgU represents the instruction i64.atomic.rmw32.xchg_u. + OpcodeAtomicI64Rmw32XchgU OpcodeAtomic = 0x47 + + // OpcodeAtomicI32RmwCmpxchg represents the instruction i32.atomic.rmw.cmpxchg. + OpcodeAtomicI32RmwCmpxchg OpcodeAtomic = 0x48 + // OpcodeAtomicI64RmwCmpxchg represents the instruction i64.atomic.rmw.cmpxchg. + OpcodeAtomicI64RmwCmpxchg OpcodeAtomic = 0x49 + // OpcodeAtomicI32Rmw8CmpxchgU represents the instruction i32.atomic.rmw8.cmpxchg_u. + OpcodeAtomicI32Rmw8CmpxchgU OpcodeAtomic = 0x4a + // OpcodeAtomicI32Rmw16CmpxchgU represents the instruction i32.atomic.rmw16.cmpxchg_u. + OpcodeAtomicI32Rmw16CmpxchgU OpcodeAtomic = 0x4b + // OpcodeAtomicI64Rmw8CmpxchgU represents the instruction i64.atomic.rmw8.cmpxchg_u. + OpcodeAtomicI64Rmw8CmpxchgU OpcodeAtomic = 0x4c + // OpcodeAtomicI64Rmw16CmpxchgU represents the instruction i64.atomic.rmw16.cmpxchg_u. + OpcodeAtomicI64Rmw16CmpxchgU OpcodeAtomic = 0x4d + // OpcodeAtomicI64Rmw32CmpxchgU represents the instruction i64.atomic.rmw32.cmpxchg_u. + OpcodeAtomicI64Rmw32CmpxchgU OpcodeAtomic = 0x4e +) + +const ( + OpcodeUnreachableName = "unreachable" + OpcodeNopName = "nop" + OpcodeBlockName = "block" + OpcodeLoopName = "loop" + OpcodeIfName = "if" + OpcodeElseName = "else" + OpcodeEndName = "end" + OpcodeBrName = "br" + OpcodeBrIfName = "br_if" + OpcodeBrTableName = "br_table" + OpcodeReturnName = "return" + OpcodeCallName = "call" + OpcodeCallIndirectName = "call_indirect" + OpcodeDropName = "drop" + OpcodeSelectName = "select" + OpcodeTypedSelectName = "typed_select" + OpcodeLocalGetName = "local.get" + OpcodeLocalSetName = "local.set" + OpcodeLocalTeeName = "local.tee" + OpcodeGlobalGetName = "global.get" + OpcodeGlobalSetName = "global.set" + OpcodeI32LoadName = "i32.load" + OpcodeI64LoadName = "i64.load" + OpcodeF32LoadName = "f32.load" + OpcodeF64LoadName = "f64.load" + OpcodeI32Load8SName = "i32.load8_s" + OpcodeI32Load8UName = "i32.load8_u" + OpcodeI32Load16SName = "i32.load16_s" + OpcodeI32Load16UName = "i32.load16_u" + OpcodeI64Load8SName = "i64.load8_s" + OpcodeI64Load8UName = "i64.load8_u" + OpcodeI64Load16SName = "i64.load16_s" + OpcodeI64Load16UName = "i64.load16_u" + OpcodeI64Load32SName = "i64.load32_s" + OpcodeI64Load32UName = "i64.load32_u" + OpcodeI32StoreName = "i32.store" + OpcodeI64StoreName = "i64.store" + OpcodeF32StoreName = "f32.store" + OpcodeF64StoreName = "f64.store" + OpcodeI32Store8Name = "i32.store8" + OpcodeI32Store16Name = "i32.store16" + OpcodeI64Store8Name = "i64.store8" + OpcodeI64Store16Name = "i64.store16" + OpcodeI64Store32Name = "i64.store32" + OpcodeMemorySizeName = "memory.size" + OpcodeMemoryGrowName = "memory.grow" + OpcodeI32ConstName = "i32.const" + OpcodeI64ConstName = "i64.const" + OpcodeF32ConstName = "f32.const" + OpcodeF64ConstName = "f64.const" + OpcodeI32EqzName = "i32.eqz" + OpcodeI32EqName = "i32.eq" + OpcodeI32NeName = "i32.ne" + OpcodeI32LtSName = "i32.lt_s" + OpcodeI32LtUName = "i32.lt_u" + OpcodeI32GtSName = "i32.gt_s" + OpcodeI32GtUName = "i32.gt_u" + OpcodeI32LeSName = "i32.le_s" + OpcodeI32LeUName = "i32.le_u" + OpcodeI32GeSName = "i32.ge_s" + OpcodeI32GeUName = "i32.ge_u" + OpcodeI64EqzName = "i64.eqz" + OpcodeI64EqName = "i64.eq" + OpcodeI64NeName = "i64.ne" + OpcodeI64LtSName = "i64.lt_s" + OpcodeI64LtUName = "i64.lt_u" + OpcodeI64GtSName = "i64.gt_s" + OpcodeI64GtUName = "i64.gt_u" + OpcodeI64LeSName = "i64.le_s" + OpcodeI64LeUName = "i64.le_u" + OpcodeI64GeSName = "i64.ge_s" + OpcodeI64GeUName = "i64.ge_u" + OpcodeF32EqName = "f32.eq" + OpcodeF32NeName = "f32.ne" + OpcodeF32LtName = "f32.lt" + OpcodeF32GtName = "f32.gt" + OpcodeF32LeName = "f32.le" + OpcodeF32GeName = "f32.ge" + OpcodeF64EqName = "f64.eq" + OpcodeF64NeName = "f64.ne" + OpcodeF64LtName = "f64.lt" + OpcodeF64GtName = "f64.gt" + OpcodeF64LeName = "f64.le" + OpcodeF64GeName = "f64.ge" + OpcodeI32ClzName = "i32.clz" + OpcodeI32CtzName = "i32.ctz" + OpcodeI32PopcntName = "i32.popcnt" + OpcodeI32AddName = "i32.add" + OpcodeI32SubName = "i32.sub" + OpcodeI32MulName = "i32.mul" + OpcodeI32DivSName = "i32.div_s" + OpcodeI32DivUName = "i32.div_u" + OpcodeI32RemSName = "i32.rem_s" + OpcodeI32RemUName = "i32.rem_u" + OpcodeI32AndName = "i32.and" + OpcodeI32OrName = "i32.or" + OpcodeI32XorName = "i32.xor" + OpcodeI32ShlName = "i32.shl" + OpcodeI32ShrSName = "i32.shr_s" + OpcodeI32ShrUName = "i32.shr_u" + OpcodeI32RotlName = "i32.rotl" + OpcodeI32RotrName = "i32.rotr" + OpcodeI64ClzName = "i64.clz" + OpcodeI64CtzName = "i64.ctz" + OpcodeI64PopcntName = "i64.popcnt" + OpcodeI64AddName = "i64.add" + OpcodeI64SubName = "i64.sub" + OpcodeI64MulName = "i64.mul" + OpcodeI64DivSName = "i64.div_s" + OpcodeI64DivUName = "i64.div_u" + OpcodeI64RemSName = "i64.rem_s" + OpcodeI64RemUName = "i64.rem_u" + OpcodeI64AndName = "i64.and" + OpcodeI64OrName = "i64.or" + OpcodeI64XorName = "i64.xor" + OpcodeI64ShlName = "i64.shl" + OpcodeI64ShrSName = "i64.shr_s" + OpcodeI64ShrUName = "i64.shr_u" + OpcodeI64RotlName = "i64.rotl" + OpcodeI64RotrName = "i64.rotr" + OpcodeF32AbsName = "f32.abs" + OpcodeF32NegName = "f32.neg" + OpcodeF32CeilName = "f32.ceil" + OpcodeF32FloorName = "f32.floor" + OpcodeF32TruncName = "f32.trunc" + OpcodeF32NearestName = "f32.nearest" + OpcodeF32SqrtName = "f32.sqrt" + OpcodeF32AddName = "f32.add" + OpcodeF32SubName = "f32.sub" + OpcodeF32MulName = "f32.mul" + OpcodeF32DivName = "f32.div" + OpcodeF32MinName = "f32.min" + OpcodeF32MaxName = "f32.max" + OpcodeF32CopysignName = "f32.copysign" + OpcodeF64AbsName = "f64.abs" + OpcodeF64NegName = "f64.neg" + OpcodeF64CeilName = "f64.ceil" + OpcodeF64FloorName = "f64.floor" + OpcodeF64TruncName = "f64.trunc" + OpcodeF64NearestName = "f64.nearest" + OpcodeF64SqrtName = "f64.sqrt" + OpcodeF64AddName = "f64.add" + OpcodeF64SubName = "f64.sub" + OpcodeF64MulName = "f64.mul" + OpcodeF64DivName = "f64.div" + OpcodeF64MinName = "f64.min" + OpcodeF64MaxName = "f64.max" + OpcodeF64CopysignName = "f64.copysign" + OpcodeI32WrapI64Name = "i32.wrap_i64" + OpcodeI32TruncF32SName = "i32.trunc_f32_s" + OpcodeI32TruncF32UName = "i32.trunc_f32_u" + OpcodeI32TruncF64SName = "i32.trunc_f64_s" + OpcodeI32TruncF64UName = "i32.trunc_f64_u" + OpcodeI64ExtendI32SName = "i64.extend_i32_s" + OpcodeI64ExtendI32UName = "i64.extend_i32_u" + OpcodeI64TruncF32SName = "i64.trunc_f32_s" + OpcodeI64TruncF32UName = "i64.trunc_f32_u" + OpcodeI64TruncF64SName = "i64.trunc_f64_s" + OpcodeI64TruncF64UName = "i64.trunc_f64_u" + OpcodeF32ConvertI32SName = "f32.convert_i32_s" + OpcodeF32ConvertI32UName = "f32.convert_i32_u" + OpcodeF32ConvertI64SName = "f32.convert_i64_s" + OpcodeF32ConvertI64UName = "f32.convert_i64u" + OpcodeF32DemoteF64Name = "f32.demote_f64" + OpcodeF64ConvertI32SName = "f64.convert_i32_s" + OpcodeF64ConvertI32UName = "f64.convert_i32_u" + OpcodeF64ConvertI64SName = "f64.convert_i64_s" + OpcodeF64ConvertI64UName = "f64.convert_i64_u" + OpcodeF64PromoteF32Name = "f64.promote_f32" + OpcodeI32ReinterpretF32Name = "i32.reinterpret_f32" + OpcodeI64ReinterpretF64Name = "i64.reinterpret_f64" + OpcodeF32ReinterpretI32Name = "f32.reinterpret_i32" + OpcodeF64ReinterpretI64Name = "f64.reinterpret_i64" + + OpcodeRefNullName = "ref.null" + OpcodeRefIsNullName = "ref.is_null" + OpcodeRefFuncName = "ref.func" + + OpcodeTableGetName = "table.get" + OpcodeTableSetName = "table.set" + + // Below are toggled with CoreFeatureSignExtensionOps + + OpcodeI32Extend8SName = "i32.extend8_s" + OpcodeI32Extend16SName = "i32.extend16_s" + OpcodeI64Extend8SName = "i64.extend8_s" + OpcodeI64Extend16SName = "i64.extend16_s" + OpcodeI64Extend32SName = "i64.extend32_s" + + OpcodeMiscPrefixName = "misc_prefix" + OpcodeVecPrefixName = "vector_prefix" + OpcodeAtomicPrefixName = "atomic_prefix" +) + +var instructionNames = [256]string{ + OpcodeUnreachable: OpcodeUnreachableName, + OpcodeNop: OpcodeNopName, + OpcodeBlock: OpcodeBlockName, + OpcodeLoop: OpcodeLoopName, + OpcodeIf: OpcodeIfName, + OpcodeElse: OpcodeElseName, + OpcodeEnd: OpcodeEndName, + OpcodeBr: OpcodeBrName, + OpcodeBrIf: OpcodeBrIfName, + OpcodeBrTable: OpcodeBrTableName, + OpcodeReturn: OpcodeReturnName, + OpcodeCall: OpcodeCallName, + OpcodeCallIndirect: OpcodeCallIndirectName, + OpcodeDrop: OpcodeDropName, + OpcodeSelect: OpcodeSelectName, + OpcodeTypedSelect: OpcodeTypedSelectName, + OpcodeLocalGet: OpcodeLocalGetName, + OpcodeLocalSet: OpcodeLocalSetName, + OpcodeLocalTee: OpcodeLocalTeeName, + OpcodeGlobalGet: OpcodeGlobalGetName, + OpcodeGlobalSet: OpcodeGlobalSetName, + OpcodeI32Load: OpcodeI32LoadName, + OpcodeI64Load: OpcodeI64LoadName, + OpcodeF32Load: OpcodeF32LoadName, + OpcodeF64Load: OpcodeF64LoadName, + OpcodeI32Load8S: OpcodeI32Load8SName, + OpcodeI32Load8U: OpcodeI32Load8UName, + OpcodeI32Load16S: OpcodeI32Load16SName, + OpcodeI32Load16U: OpcodeI32Load16UName, + OpcodeI64Load8S: OpcodeI64Load8SName, + OpcodeI64Load8U: OpcodeI64Load8UName, + OpcodeI64Load16S: OpcodeI64Load16SName, + OpcodeI64Load16U: OpcodeI64Load16UName, + OpcodeI64Load32S: OpcodeI64Load32SName, + OpcodeI64Load32U: OpcodeI64Load32UName, + OpcodeI32Store: OpcodeI32StoreName, + OpcodeI64Store: OpcodeI64StoreName, + OpcodeF32Store: OpcodeF32StoreName, + OpcodeF64Store: OpcodeF64StoreName, + OpcodeI32Store8: OpcodeI32Store8Name, + OpcodeI32Store16: OpcodeI32Store16Name, + OpcodeI64Store8: OpcodeI64Store8Name, + OpcodeI64Store16: OpcodeI64Store16Name, + OpcodeI64Store32: OpcodeI64Store32Name, + OpcodeMemorySize: OpcodeMemorySizeName, + OpcodeMemoryGrow: OpcodeMemoryGrowName, + OpcodeI32Const: OpcodeI32ConstName, + OpcodeI64Const: OpcodeI64ConstName, + OpcodeF32Const: OpcodeF32ConstName, + OpcodeF64Const: OpcodeF64ConstName, + OpcodeI32Eqz: OpcodeI32EqzName, + OpcodeI32Eq: OpcodeI32EqName, + OpcodeI32Ne: OpcodeI32NeName, + OpcodeI32LtS: OpcodeI32LtSName, + OpcodeI32LtU: OpcodeI32LtUName, + OpcodeI32GtS: OpcodeI32GtSName, + OpcodeI32GtU: OpcodeI32GtUName, + OpcodeI32LeS: OpcodeI32LeSName, + OpcodeI32LeU: OpcodeI32LeUName, + OpcodeI32GeS: OpcodeI32GeSName, + OpcodeI32GeU: OpcodeI32GeUName, + OpcodeI64Eqz: OpcodeI64EqzName, + OpcodeI64Eq: OpcodeI64EqName, + OpcodeI64Ne: OpcodeI64NeName, + OpcodeI64LtS: OpcodeI64LtSName, + OpcodeI64LtU: OpcodeI64LtUName, + OpcodeI64GtS: OpcodeI64GtSName, + OpcodeI64GtU: OpcodeI64GtUName, + OpcodeI64LeS: OpcodeI64LeSName, + OpcodeI64LeU: OpcodeI64LeUName, + OpcodeI64GeS: OpcodeI64GeSName, + OpcodeI64GeU: OpcodeI64GeUName, + OpcodeF32Eq: OpcodeF32EqName, + OpcodeF32Ne: OpcodeF32NeName, + OpcodeF32Lt: OpcodeF32LtName, + OpcodeF32Gt: OpcodeF32GtName, + OpcodeF32Le: OpcodeF32LeName, + OpcodeF32Ge: OpcodeF32GeName, + OpcodeF64Eq: OpcodeF64EqName, + OpcodeF64Ne: OpcodeF64NeName, + OpcodeF64Lt: OpcodeF64LtName, + OpcodeF64Gt: OpcodeF64GtName, + OpcodeF64Le: OpcodeF64LeName, + OpcodeF64Ge: OpcodeF64GeName, + OpcodeI32Clz: OpcodeI32ClzName, + OpcodeI32Ctz: OpcodeI32CtzName, + OpcodeI32Popcnt: OpcodeI32PopcntName, + OpcodeI32Add: OpcodeI32AddName, + OpcodeI32Sub: OpcodeI32SubName, + OpcodeI32Mul: OpcodeI32MulName, + OpcodeI32DivS: OpcodeI32DivSName, + OpcodeI32DivU: OpcodeI32DivUName, + OpcodeI32RemS: OpcodeI32RemSName, + OpcodeI32RemU: OpcodeI32RemUName, + OpcodeI32And: OpcodeI32AndName, + OpcodeI32Or: OpcodeI32OrName, + OpcodeI32Xor: OpcodeI32XorName, + OpcodeI32Shl: OpcodeI32ShlName, + OpcodeI32ShrS: OpcodeI32ShrSName, + OpcodeI32ShrU: OpcodeI32ShrUName, + OpcodeI32Rotl: OpcodeI32RotlName, + OpcodeI32Rotr: OpcodeI32RotrName, + OpcodeI64Clz: OpcodeI64ClzName, + OpcodeI64Ctz: OpcodeI64CtzName, + OpcodeI64Popcnt: OpcodeI64PopcntName, + OpcodeI64Add: OpcodeI64AddName, + OpcodeI64Sub: OpcodeI64SubName, + OpcodeI64Mul: OpcodeI64MulName, + OpcodeI64DivS: OpcodeI64DivSName, + OpcodeI64DivU: OpcodeI64DivUName, + OpcodeI64RemS: OpcodeI64RemSName, + OpcodeI64RemU: OpcodeI64RemUName, + OpcodeI64And: OpcodeI64AndName, + OpcodeI64Or: OpcodeI64OrName, + OpcodeI64Xor: OpcodeI64XorName, + OpcodeI64Shl: OpcodeI64ShlName, + OpcodeI64ShrS: OpcodeI64ShrSName, + OpcodeI64ShrU: OpcodeI64ShrUName, + OpcodeI64Rotl: OpcodeI64RotlName, + OpcodeI64Rotr: OpcodeI64RotrName, + OpcodeF32Abs: OpcodeF32AbsName, + OpcodeF32Neg: OpcodeF32NegName, + OpcodeF32Ceil: OpcodeF32CeilName, + OpcodeF32Floor: OpcodeF32FloorName, + OpcodeF32Trunc: OpcodeF32TruncName, + OpcodeF32Nearest: OpcodeF32NearestName, + OpcodeF32Sqrt: OpcodeF32SqrtName, + OpcodeF32Add: OpcodeF32AddName, + OpcodeF32Sub: OpcodeF32SubName, + OpcodeF32Mul: OpcodeF32MulName, + OpcodeF32Div: OpcodeF32DivName, + OpcodeF32Min: OpcodeF32MinName, + OpcodeF32Max: OpcodeF32MaxName, + OpcodeF32Copysign: OpcodeF32CopysignName, + OpcodeF64Abs: OpcodeF64AbsName, + OpcodeF64Neg: OpcodeF64NegName, + OpcodeF64Ceil: OpcodeF64CeilName, + OpcodeF64Floor: OpcodeF64FloorName, + OpcodeF64Trunc: OpcodeF64TruncName, + OpcodeF64Nearest: OpcodeF64NearestName, + OpcodeF64Sqrt: OpcodeF64SqrtName, + OpcodeF64Add: OpcodeF64AddName, + OpcodeF64Sub: OpcodeF64SubName, + OpcodeF64Mul: OpcodeF64MulName, + OpcodeF64Div: OpcodeF64DivName, + OpcodeF64Min: OpcodeF64MinName, + OpcodeF64Max: OpcodeF64MaxName, + OpcodeF64Copysign: OpcodeF64CopysignName, + OpcodeI32WrapI64: OpcodeI32WrapI64Name, + OpcodeI32TruncF32S: OpcodeI32TruncF32SName, + OpcodeI32TruncF32U: OpcodeI32TruncF32UName, + OpcodeI32TruncF64S: OpcodeI32TruncF64SName, + OpcodeI32TruncF64U: OpcodeI32TruncF64UName, + OpcodeI64ExtendI32S: OpcodeI64ExtendI32SName, + OpcodeI64ExtendI32U: OpcodeI64ExtendI32UName, + OpcodeI64TruncF32S: OpcodeI64TruncF32SName, + OpcodeI64TruncF32U: OpcodeI64TruncF32UName, + OpcodeI64TruncF64S: OpcodeI64TruncF64SName, + OpcodeI64TruncF64U: OpcodeI64TruncF64UName, + OpcodeF32ConvertI32S: OpcodeF32ConvertI32SName, + OpcodeF32ConvertI32U: OpcodeF32ConvertI32UName, + OpcodeF32ConvertI64S: OpcodeF32ConvertI64SName, + OpcodeF32ConvertI64U: OpcodeF32ConvertI64UName, + OpcodeF32DemoteF64: OpcodeF32DemoteF64Name, + OpcodeF64ConvertI32S: OpcodeF64ConvertI32SName, + OpcodeF64ConvertI32U: OpcodeF64ConvertI32UName, + OpcodeF64ConvertI64S: OpcodeF64ConvertI64SName, + OpcodeF64ConvertI64U: OpcodeF64ConvertI64UName, + OpcodeF64PromoteF32: OpcodeF64PromoteF32Name, + OpcodeI32ReinterpretF32: OpcodeI32ReinterpretF32Name, + OpcodeI64ReinterpretF64: OpcodeI64ReinterpretF64Name, + OpcodeF32ReinterpretI32: OpcodeF32ReinterpretI32Name, + OpcodeF64ReinterpretI64: OpcodeF64ReinterpretI64Name, + + OpcodeRefNull: OpcodeRefNullName, + OpcodeRefIsNull: OpcodeRefIsNullName, + OpcodeRefFunc: OpcodeRefFuncName, + + OpcodeTableGet: OpcodeTableGetName, + OpcodeTableSet: OpcodeTableSetName, + + // Below are toggled with CoreFeatureSignExtensionOps + + OpcodeI32Extend8S: OpcodeI32Extend8SName, + OpcodeI32Extend16S: OpcodeI32Extend16SName, + OpcodeI64Extend8S: OpcodeI64Extend8SName, + OpcodeI64Extend16S: OpcodeI64Extend16SName, + OpcodeI64Extend32S: OpcodeI64Extend32SName, + + OpcodeMiscPrefix: OpcodeMiscPrefixName, + OpcodeVecPrefix: OpcodeVecPrefixName, +} + +// InstructionName returns the instruction corresponding to this binary Opcode. +// See https://www.w3.org/TR/2019/REC-wasm-core-1-20191205/#a7-index-of-instructions +func InstructionName(oc Opcode) string { + return instructionNames[oc] +} + +const ( + OpcodeI32TruncSatF32SName = "i32.trunc_sat_f32_s" + OpcodeI32TruncSatF32UName = "i32.trunc_sat_f32_u" + OpcodeI32TruncSatF64SName = "i32.trunc_sat_f64_s" + OpcodeI32TruncSatF64UName = "i32.trunc_sat_f64_u" + OpcodeI64TruncSatF32SName = "i64.trunc_sat_f32_s" + OpcodeI64TruncSatF32UName = "i64.trunc_sat_f32_u" + OpcodeI64TruncSatF64SName = "i64.trunc_sat_f64_s" + OpcodeI64TruncSatF64UName = "i64.trunc_sat_f64_u" + + OpcodeMemoryInitName = "memory.init" + OpcodeDataDropName = "data.drop" + OpcodeMemoryCopyName = "memory.copy" + OpcodeMemoryFillName = "memory.fill" + OpcodeTableInitName = "table.init" + OpcodeElemDropName = "elem.drop" + OpcodeTableCopyName = "table.copy" + OpcodeTableGrowName = "table.grow" + OpcodeTableSizeName = "table.size" + OpcodeTableFillName = "table.fill" +) + +var miscInstructionNames = [256]string{ + OpcodeMiscI32TruncSatF32S: OpcodeI32TruncSatF32SName, + OpcodeMiscI32TruncSatF32U: OpcodeI32TruncSatF32UName, + OpcodeMiscI32TruncSatF64S: OpcodeI32TruncSatF64SName, + OpcodeMiscI32TruncSatF64U: OpcodeI32TruncSatF64UName, + OpcodeMiscI64TruncSatF32S: OpcodeI64TruncSatF32SName, + OpcodeMiscI64TruncSatF32U: OpcodeI64TruncSatF32UName, + OpcodeMiscI64TruncSatF64S: OpcodeI64TruncSatF64SName, + OpcodeMiscI64TruncSatF64U: OpcodeI64TruncSatF64UName, + + OpcodeMiscMemoryInit: OpcodeMemoryInitName, + OpcodeMiscDataDrop: OpcodeDataDropName, + OpcodeMiscMemoryCopy: OpcodeMemoryCopyName, + OpcodeMiscMemoryFill: OpcodeMemoryFillName, + OpcodeMiscTableInit: OpcodeTableInitName, + OpcodeMiscElemDrop: OpcodeElemDropName, + OpcodeMiscTableCopy: OpcodeTableCopyName, + OpcodeMiscTableGrow: OpcodeTableGrowName, + OpcodeMiscTableSize: OpcodeTableSizeName, + OpcodeMiscTableFill: OpcodeTableFillName, +} + +// MiscInstructionName returns the instruction corresponding to this miscellaneous Opcode. +func MiscInstructionName(oc OpcodeMisc) string { + return miscInstructionNames[oc] +} + +const ( + OpcodeVecV128LoadName = "v128.load" + OpcodeVecV128Load8x8SName = "v128.load8x8_s" + OpcodeVecV128Load8x8UName = "v128.load8x8_u" + OpcodeVecV128Load16x4SName = "v128.load16x4_s" + OpcodeVecV128Load16x4UName = "v128.load16x4_u" + OpcodeVecV128Load32x2SName = "v128.load32x2_s" + OpcodeVecV128Load32x2UName = "v128.load32x2_u" + OpcodeVecV128Load8SplatName = "v128.load8_splat" + OpcodeVecV128Load16SplatName = "v128.load16_splat" + OpcodeVecV128Load32SplatName = "v128.load32_splat" + OpcodeVecV128Load64SplatName = "v128.load64_splat" + OpcodeVecV128Load32zeroName = "v128.load32_zero" + OpcodeVecV128Load64zeroName = "v128.load64_zero" + OpcodeVecV128StoreName = "v128.store" + OpcodeVecV128Load8LaneName = "v128.load8_lane" + OpcodeVecV128Load16LaneName = "v128.load16_lane" + OpcodeVecV128Load32LaneName = "v128.load32_lane" + OpcodeVecV128Load64LaneName = "v128.load64_lane" + OpcodeVecV128Store8LaneName = "v128.store8_lane" + OpcodeVecV128Store16LaneName = "v128.store16_lane" + OpcodeVecV128Store32LaneName = "v128.store32_lane" + OpcodeVecV128Store64LaneName = "v128.store64_lane" + OpcodeVecV128ConstName = "v128.const" + OpcodeVecV128i8x16ShuffleName = "v128.shuffle" + OpcodeVecI8x16ExtractLaneSName = "i8x16.extract_lane_s" + OpcodeVecI8x16ExtractLaneUName = "i8x16.extract_lane_u" + OpcodeVecI8x16ReplaceLaneName = "i8x16.replace_lane" + OpcodeVecI16x8ExtractLaneSName = "i16x8.extract_lane_s" + OpcodeVecI16x8ExtractLaneUName = "i16x8.extract_lane_u" + OpcodeVecI16x8ReplaceLaneName = "i16x8.replace_lane" + OpcodeVecI32x4ExtractLaneName = "i32x4.extract_lane" + OpcodeVecI32x4ReplaceLaneName = "i32x4.replace_lane" + OpcodeVecI64x2ExtractLaneName = "i64x2.extract_lane" + OpcodeVecI64x2ReplaceLaneName = "i64x2.replace_lane" + OpcodeVecF32x4ExtractLaneName = "f32x4.extract_lane" + OpcodeVecF32x4ReplaceLaneName = "f32x4.replace_lane" + OpcodeVecF64x2ExtractLaneName = "f64x2.extract_lane" + OpcodeVecF64x2ReplaceLaneName = "f64x2.replace_lane" + OpcodeVecI8x16SwizzleName = "i8x16.swizzle" + OpcodeVecI8x16SplatName = "i8x16.splat" + OpcodeVecI16x8SplatName = "i16x8.splat" + OpcodeVecI32x4SplatName = "i32x4.splat" + OpcodeVecI64x2SplatName = "i64x2.splat" + OpcodeVecF32x4SplatName = "f32x4.splat" + OpcodeVecF64x2SplatName = "f64x2.splat" + OpcodeVecI8x16EqName = "i8x16.eq" + OpcodeVecI8x16NeName = "i8x16.ne" + OpcodeVecI8x16LtSName = "i8x16.lt_s" + OpcodeVecI8x16LtUName = "i8x16.lt_u" + OpcodeVecI8x16GtSName = "i8x16.gt_s" + OpcodeVecI8x16GtUName = "i8x16.gt_u" + OpcodeVecI8x16LeSName = "i8x16.le_s" + OpcodeVecI8x16LeUName = "i8x16.le_u" + OpcodeVecI8x16GeSName = "i8x16.ge_s" + OpcodeVecI8x16GeUName = "i8x16.ge_u" + OpcodeVecI16x8EqName = "i16x8.eq" + OpcodeVecI16x8NeName = "i16x8.ne" + OpcodeVecI16x8LtSName = "i16x8.lt_s" + OpcodeVecI16x8LtUName = "i16x8.lt_u" + OpcodeVecI16x8GtSName = "i16x8.gt_s" + OpcodeVecI16x8GtUName = "i16x8.gt_u" + OpcodeVecI16x8LeSName = "i16x8.le_s" + OpcodeVecI16x8LeUName = "i16x8.le_u" + OpcodeVecI16x8GeSName = "i16x8.ge_s" + OpcodeVecI16x8GeUName = "i16x8.ge_u" + OpcodeVecI32x4EqName = "i32x4.eq" + OpcodeVecI32x4NeName = "i32x4.ne" + OpcodeVecI32x4LtSName = "i32x4.lt_s" + OpcodeVecI32x4LtUName = "i32x4.lt_u" + OpcodeVecI32x4GtSName = "i32x4.gt_s" + OpcodeVecI32x4GtUName = "i32x4.gt_u" + OpcodeVecI32x4LeSName = "i32x4.le_s" + OpcodeVecI32x4LeUName = "i32x4.le_u" + OpcodeVecI32x4GeSName = "i32x4.ge_s" + OpcodeVecI32x4GeUName = "i32x4.ge_u" + OpcodeVecI64x2EqName = "i64x2.eq" + OpcodeVecI64x2NeName = "i64x2.ne" + OpcodeVecI64x2LtSName = "i64x2.lt" + OpcodeVecI64x2GtSName = "i64x2.gt" + OpcodeVecI64x2LeSName = "i64x2.le" + OpcodeVecI64x2GeSName = "i64x2.ge" + OpcodeVecF32x4EqName = "f32x4.eq" + OpcodeVecF32x4NeName = "f32x4.ne" + OpcodeVecF32x4LtName = "f32x4.lt" + OpcodeVecF32x4GtName = "f32x4.gt" + OpcodeVecF32x4LeName = "f32x4.le" + OpcodeVecF32x4GeName = "f32x4.ge" + OpcodeVecF64x2EqName = "f64x2.eq" + OpcodeVecF64x2NeName = "f64x2.ne" + OpcodeVecF64x2LtName = "f64x2.lt" + OpcodeVecF64x2GtName = "f64x2.gt" + OpcodeVecF64x2LeName = "f64x2.le" + OpcodeVecF64x2GeName = "f64x2.ge" + OpcodeVecV128NotName = "v128.not" + OpcodeVecV128AndName = "v128.and" + OpcodeVecV128AndNotName = "v128.andnot" + OpcodeVecV128OrName = "v128.or" + OpcodeVecV128XorName = "v128.xor" + OpcodeVecV128BitselectName = "v128.bitselect" + OpcodeVecV128AnyTrueName = "v128.any_true" + OpcodeVecI8x16AbsName = "i8x16.abs" + OpcodeVecI8x16NegName = "i8x16.neg" + OpcodeVecI8x16PopcntName = "i8x16.popcnt" + OpcodeVecI8x16AllTrueName = "i8x16.all_true" + OpcodeVecI8x16BitMaskName = "i8x16.bitmask" + OpcodeVecI8x16NarrowI16x8SName = "i8x16.narrow_i16x8_s" + OpcodeVecI8x16NarrowI16x8UName = "i8x16.narrow_i16x8_u" + OpcodeVecI8x16ShlName = "i8x16.shl" + OpcodeVecI8x16ShrSName = "i8x16.shr_s" + OpcodeVecI8x16ShrUName = "i8x16.shr_u" + OpcodeVecI8x16AddName = "i8x16.add" + OpcodeVecI8x16AddSatSName = "i8x16.add_sat_s" + OpcodeVecI8x16AddSatUName = "i8x16.add_sat_u" + OpcodeVecI8x16SubName = "i8x16.sub" + OpcodeVecI8x16SubSatSName = "i8x16.sub_s" + OpcodeVecI8x16SubSatUName = "i8x16.sub_u" + OpcodeVecI8x16MinSName = "i8x16.min_s" + OpcodeVecI8x16MinUName = "i8x16.min_u" + OpcodeVecI8x16MaxSName = "i8x16.max_s" + OpcodeVecI8x16MaxUName = "i8x16.max_u" + OpcodeVecI8x16AvgrUName = "i8x16.avgr_u" + OpcodeVecI16x8ExtaddPairwiseI8x16SName = "i16x8.extadd_pairwise_i8x16_s" + OpcodeVecI16x8ExtaddPairwiseI8x16UName = "i16x8.extadd_pairwise_i8x16_u" + OpcodeVecI16x8AbsName = "i16x8.abs" + OpcodeVecI16x8NegName = "i16x8.neg" + OpcodeVecI16x8Q15mulrSatSName = "i16x8.q15mulr_sat_s" + OpcodeVecI16x8AllTrueName = "i16x8.all_true" + OpcodeVecI16x8BitMaskName = "i16x8.bitmask" + OpcodeVecI16x8NarrowI32x4SName = "i16x8.narrow_i32x4_s" + OpcodeVecI16x8NarrowI32x4UName = "i16x8.narrow_i32x4_u" + OpcodeVecI16x8ExtendLowI8x16SName = "i16x8.extend_low_i8x16_s" + OpcodeVecI16x8ExtendHighI8x16SName = "i16x8.extend_high_i8x16_s" + OpcodeVecI16x8ExtendLowI8x16UName = "i16x8.extend_low_i8x16_u" + OpcodeVecI16x8ExtendHighI8x16UName = "i16x8.extend_high_i8x16_u" + OpcodeVecI16x8ShlName = "i16x8.shl" + OpcodeVecI16x8ShrSName = "i16x8.shr_s" + OpcodeVecI16x8ShrUName = "i16x8.shr_u" + OpcodeVecI16x8AddName = "i16x8.add" + OpcodeVecI16x8AddSatSName = "i16x8.add_sat_s" + OpcodeVecI16x8AddSatUName = "i16x8.add_sat_u" + OpcodeVecI16x8SubName = "i16x8.sub" + OpcodeVecI16x8SubSatSName = "i16x8.sub_sat_s" + OpcodeVecI16x8SubSatUName = "i16x8.sub_sat_u" + OpcodeVecI16x8MulName = "i16x8.mul" + OpcodeVecI16x8MinSName = "i16x8.min_s" + OpcodeVecI16x8MinUName = "i16x8.min_u" + OpcodeVecI16x8MaxSName = "i16x8.max_s" + OpcodeVecI16x8MaxUName = "i16x8.max_u" + OpcodeVecI16x8AvgrUName = "i16x8.avgr_u" + OpcodeVecI16x8ExtMulLowI8x16SName = "i16x8.extmul_low_i8x16_s" + OpcodeVecI16x8ExtMulHighI8x16SName = "i16x8.extmul_high_i8x16_s" + OpcodeVecI16x8ExtMulLowI8x16UName = "i16x8.extmul_low_i8x16_u" + OpcodeVecI16x8ExtMulHighI8x16UName = "i16x8.extmul_high_i8x16_u" + OpcodeVecI32x4ExtaddPairwiseI16x8SName = "i32x4.extadd_pairwise_i16x8_s" + OpcodeVecI32x4ExtaddPairwiseI16x8UName = "i32x4.extadd_pairwise_i16x8_u" + OpcodeVecI32x4AbsName = "i32x4.abs" + OpcodeVecI32x4NegName = "i32x4.neg" + OpcodeVecI32x4AllTrueName = "i32x4.all_true" + OpcodeVecI32x4BitMaskName = "i32x4.bitmask" + OpcodeVecI32x4ExtendLowI16x8SName = "i32x4.extend_low_i16x8_s" + OpcodeVecI32x4ExtendHighI16x8SName = "i32x4.extend_high_i16x8_s" + OpcodeVecI32x4ExtendLowI16x8UName = "i32x4.extend_low_i16x8_u" + OpcodeVecI32x4ExtendHighI16x8UName = "i32x4.extend_high_i16x8_u" + OpcodeVecI32x4ShlName = "i32x4.shl" + OpcodeVecI32x4ShrSName = "i32x4.shr_s" + OpcodeVecI32x4ShrUName = "i32x4.shr_u" + OpcodeVecI32x4AddName = "i32x4.add" + OpcodeVecI32x4SubName = "i32x4.sub" + OpcodeVecI32x4MulName = "i32x4.mul" + OpcodeVecI32x4MinSName = "i32x4.min_s" + OpcodeVecI32x4MinUName = "i32x4.min_u" + OpcodeVecI32x4MaxSName = "i32x4.max_s" + OpcodeVecI32x4MaxUName = "i32x4.max_u" + OpcodeVecI32x4DotI16x8SName = "i32x4.dot_i16x8_s" + OpcodeVecI32x4ExtMulLowI16x8SName = "i32x4.extmul_low_i16x8_s" + OpcodeVecI32x4ExtMulHighI16x8SName = "i32x4.extmul_high_i16x8_s" + OpcodeVecI32x4ExtMulLowI16x8UName = "i32x4.extmul_low_i16x8_u" + OpcodeVecI32x4ExtMulHighI16x8UName = "i32x4.extmul_high_i16x8_u" + OpcodeVecI64x2AbsName = "i64x2.abs" + OpcodeVecI64x2NegName = "i64x2.neg" + OpcodeVecI64x2AllTrueName = "i64x2.all_true" + OpcodeVecI64x2BitMaskName = "i64x2.bitmask" + OpcodeVecI64x2ExtendLowI32x4SName = "i64x2.extend_low_i32x4_s" + OpcodeVecI64x2ExtendHighI32x4SName = "i64x2.extend_high_i32x4_s" + OpcodeVecI64x2ExtendLowI32x4UName = "i64x2.extend_low_i32x4_u" + OpcodeVecI64x2ExtendHighI32x4UName = "i64x2.extend_high_i32x4_u" + OpcodeVecI64x2ShlName = "i64x2.shl" + OpcodeVecI64x2ShrSName = "i64x2.shr_s" + OpcodeVecI64x2ShrUName = "i64x2.shr_u" + OpcodeVecI64x2AddName = "i64x2.add" + OpcodeVecI64x2SubName = "i64x2.sub" + OpcodeVecI64x2MulName = "i64x2.mul" + OpcodeVecI64x2ExtMulLowI32x4SName = "i64x2.extmul_low_i32x4_s" + OpcodeVecI64x2ExtMulHighI32x4SName = "i64x2.extmul_high_i32x4_s" + OpcodeVecI64x2ExtMulLowI32x4UName = "i64x2.extmul_low_i32x4_u" + OpcodeVecI64x2ExtMulHighI32x4UName = "i64x2.extmul_high_i32x4_u" + OpcodeVecF32x4CeilName = "f32x4.ceil" + OpcodeVecF32x4FloorName = "f32x4.floor" + OpcodeVecF32x4TruncName = "f32x4.trunc" + OpcodeVecF32x4NearestName = "f32x4.nearest" + OpcodeVecF32x4AbsName = "f32x4.abs" + OpcodeVecF32x4NegName = "f32x4.neg" + OpcodeVecF32x4SqrtName = "f32x4.sqrt" + OpcodeVecF32x4AddName = "f32x4.add" + OpcodeVecF32x4SubName = "f32x4.sub" + OpcodeVecF32x4MulName = "f32x4.mul" + OpcodeVecF32x4DivName = "f32x4.div" + OpcodeVecF32x4MinName = "f32x4.min" + OpcodeVecF32x4MaxName = "f32x4.max" + OpcodeVecF32x4PminName = "f32x4.pmin" + OpcodeVecF32x4PmaxName = "f32x4.pmax" + OpcodeVecF64x2CeilName = "f64x2.ceil" + OpcodeVecF64x2FloorName = "f64x2.floor" + OpcodeVecF64x2TruncName = "f64x2.trunc" + OpcodeVecF64x2NearestName = "f64x2.nearest" + OpcodeVecF64x2AbsName = "f64x2.abs" + OpcodeVecF64x2NegName = "f64x2.neg" + OpcodeVecF64x2SqrtName = "f64x2.sqrt" + OpcodeVecF64x2AddName = "f64x2.add" + OpcodeVecF64x2SubName = "f64x2.sub" + OpcodeVecF64x2MulName = "f64x2.mul" + OpcodeVecF64x2DivName = "f64x2.div" + OpcodeVecF64x2MinName = "f64x2.min" + OpcodeVecF64x2MaxName = "f64x2.max" + OpcodeVecF64x2PminName = "f64x2.pmin" + OpcodeVecF64x2PmaxName = "f64x2.pmax" + OpcodeVecI32x4TruncSatF32x4SName = "i32x4.trunc_sat_f32x4_s" + OpcodeVecI32x4TruncSatF32x4UName = "i32x4.trunc_sat_f32x4_u" + OpcodeVecF32x4ConvertI32x4SName = "f32x4.convert_i32x4_s" + OpcodeVecF32x4ConvertI32x4UName = "f32x4.convert_i32x4_u" + OpcodeVecI32x4TruncSatF64x2SZeroName = "i32x4.trunc_sat_f64x2_s_zero" + OpcodeVecI32x4TruncSatF64x2UZeroName = "i32x4.trunc_sat_f64x2_u_zero" + OpcodeVecF64x2ConvertLowI32x4SName = "f64x2.convert_low_i32x4_s" + OpcodeVecF64x2ConvertLowI32x4UName = "f64x2.convert_low_i32x4_u" + OpcodeVecF32x4DemoteF64x2ZeroName = "f32x4.demote_f64x2_zero" + OpcodeVecF64x2PromoteLowF32x4ZeroName = "f64x2.promote_low_f32x4" +) + +var vectorInstructionName = map[OpcodeVec]string{ + OpcodeVecV128Load: OpcodeVecV128LoadName, + OpcodeVecV128Load8x8s: OpcodeVecV128Load8x8SName, + OpcodeVecV128Load8x8u: OpcodeVecV128Load8x8UName, + OpcodeVecV128Load16x4s: OpcodeVecV128Load16x4SName, + OpcodeVecV128Load16x4u: OpcodeVecV128Load16x4UName, + OpcodeVecV128Load32x2s: OpcodeVecV128Load32x2SName, + OpcodeVecV128Load32x2u: OpcodeVecV128Load32x2UName, + OpcodeVecV128Load8Splat: OpcodeVecV128Load8SplatName, + OpcodeVecV128Load16Splat: OpcodeVecV128Load16SplatName, + OpcodeVecV128Load32Splat: OpcodeVecV128Load32SplatName, + OpcodeVecV128Load64Splat: OpcodeVecV128Load64SplatName, + OpcodeVecV128Load32zero: OpcodeVecV128Load32zeroName, + OpcodeVecV128Load64zero: OpcodeVecV128Load64zeroName, + OpcodeVecV128Store: OpcodeVecV128StoreName, + OpcodeVecV128Load8Lane: OpcodeVecV128Load8LaneName, + OpcodeVecV128Load16Lane: OpcodeVecV128Load16LaneName, + OpcodeVecV128Load32Lane: OpcodeVecV128Load32LaneName, + OpcodeVecV128Load64Lane: OpcodeVecV128Load64LaneName, + OpcodeVecV128Store8Lane: OpcodeVecV128Store8LaneName, + OpcodeVecV128Store16Lane: OpcodeVecV128Store16LaneName, + OpcodeVecV128Store32Lane: OpcodeVecV128Store32LaneName, + OpcodeVecV128Store64Lane: OpcodeVecV128Store64LaneName, + OpcodeVecV128Const: OpcodeVecV128ConstName, + OpcodeVecV128i8x16Shuffle: OpcodeVecV128i8x16ShuffleName, + OpcodeVecI8x16ExtractLaneS: OpcodeVecI8x16ExtractLaneSName, + OpcodeVecI8x16ExtractLaneU: OpcodeVecI8x16ExtractLaneUName, + OpcodeVecI8x16ReplaceLane: OpcodeVecI8x16ReplaceLaneName, + OpcodeVecI16x8ExtractLaneS: OpcodeVecI16x8ExtractLaneSName, + OpcodeVecI16x8ExtractLaneU: OpcodeVecI16x8ExtractLaneUName, + OpcodeVecI16x8ReplaceLane: OpcodeVecI16x8ReplaceLaneName, + OpcodeVecI32x4ExtractLane: OpcodeVecI32x4ExtractLaneName, + OpcodeVecI32x4ReplaceLane: OpcodeVecI32x4ReplaceLaneName, + OpcodeVecI64x2ExtractLane: OpcodeVecI64x2ExtractLaneName, + OpcodeVecI64x2ReplaceLane: OpcodeVecI64x2ReplaceLaneName, + OpcodeVecF32x4ExtractLane: OpcodeVecF32x4ExtractLaneName, + OpcodeVecF32x4ReplaceLane: OpcodeVecF32x4ReplaceLaneName, + OpcodeVecF64x2ExtractLane: OpcodeVecF64x2ExtractLaneName, + OpcodeVecF64x2ReplaceLane: OpcodeVecF64x2ReplaceLaneName, + OpcodeVecI8x16Swizzle: OpcodeVecI8x16SwizzleName, + OpcodeVecI8x16Splat: OpcodeVecI8x16SplatName, + OpcodeVecI16x8Splat: OpcodeVecI16x8SplatName, + OpcodeVecI32x4Splat: OpcodeVecI32x4SplatName, + OpcodeVecI64x2Splat: OpcodeVecI64x2SplatName, + OpcodeVecF32x4Splat: OpcodeVecF32x4SplatName, + OpcodeVecF64x2Splat: OpcodeVecF64x2SplatName, + OpcodeVecI8x16Eq: OpcodeVecI8x16EqName, + OpcodeVecI8x16Ne: OpcodeVecI8x16NeName, + OpcodeVecI8x16LtS: OpcodeVecI8x16LtSName, + OpcodeVecI8x16LtU: OpcodeVecI8x16LtUName, + OpcodeVecI8x16GtS: OpcodeVecI8x16GtSName, + OpcodeVecI8x16GtU: OpcodeVecI8x16GtUName, + OpcodeVecI8x16LeS: OpcodeVecI8x16LeSName, + OpcodeVecI8x16LeU: OpcodeVecI8x16LeUName, + OpcodeVecI8x16GeS: OpcodeVecI8x16GeSName, + OpcodeVecI8x16GeU: OpcodeVecI8x16GeUName, + OpcodeVecI16x8Eq: OpcodeVecI16x8EqName, + OpcodeVecI16x8Ne: OpcodeVecI16x8NeName, + OpcodeVecI16x8LtS: OpcodeVecI16x8LtSName, + OpcodeVecI16x8LtU: OpcodeVecI16x8LtUName, + OpcodeVecI16x8GtS: OpcodeVecI16x8GtSName, + OpcodeVecI16x8GtU: OpcodeVecI16x8GtUName, + OpcodeVecI16x8LeS: OpcodeVecI16x8LeSName, + OpcodeVecI16x8LeU: OpcodeVecI16x8LeUName, + OpcodeVecI16x8GeS: OpcodeVecI16x8GeSName, + OpcodeVecI16x8GeU: OpcodeVecI16x8GeUName, + OpcodeVecI32x4Eq: OpcodeVecI32x4EqName, + OpcodeVecI32x4Ne: OpcodeVecI32x4NeName, + OpcodeVecI32x4LtS: OpcodeVecI32x4LtSName, + OpcodeVecI32x4LtU: OpcodeVecI32x4LtUName, + OpcodeVecI32x4GtS: OpcodeVecI32x4GtSName, + OpcodeVecI32x4GtU: OpcodeVecI32x4GtUName, + OpcodeVecI32x4LeS: OpcodeVecI32x4LeSName, + OpcodeVecI32x4LeU: OpcodeVecI32x4LeUName, + OpcodeVecI32x4GeS: OpcodeVecI32x4GeSName, + OpcodeVecI32x4GeU: OpcodeVecI32x4GeUName, + OpcodeVecI64x2Eq: OpcodeVecI64x2EqName, + OpcodeVecI64x2Ne: OpcodeVecI64x2NeName, + OpcodeVecI64x2LtS: OpcodeVecI64x2LtSName, + OpcodeVecI64x2GtS: OpcodeVecI64x2GtSName, + OpcodeVecI64x2LeS: OpcodeVecI64x2LeSName, + OpcodeVecI64x2GeS: OpcodeVecI64x2GeSName, + OpcodeVecF32x4Eq: OpcodeVecF32x4EqName, + OpcodeVecF32x4Ne: OpcodeVecF32x4NeName, + OpcodeVecF32x4Lt: OpcodeVecF32x4LtName, + OpcodeVecF32x4Gt: OpcodeVecF32x4GtName, + OpcodeVecF32x4Le: OpcodeVecF32x4LeName, + OpcodeVecF32x4Ge: OpcodeVecF32x4GeName, + OpcodeVecF64x2Eq: OpcodeVecF64x2EqName, + OpcodeVecF64x2Ne: OpcodeVecF64x2NeName, + OpcodeVecF64x2Lt: OpcodeVecF64x2LtName, + OpcodeVecF64x2Gt: OpcodeVecF64x2GtName, + OpcodeVecF64x2Le: OpcodeVecF64x2LeName, + OpcodeVecF64x2Ge: OpcodeVecF64x2GeName, + OpcodeVecV128Not: OpcodeVecV128NotName, + OpcodeVecV128And: OpcodeVecV128AndName, + OpcodeVecV128AndNot: OpcodeVecV128AndNotName, + OpcodeVecV128Or: OpcodeVecV128OrName, + OpcodeVecV128Xor: OpcodeVecV128XorName, + OpcodeVecV128Bitselect: OpcodeVecV128BitselectName, + OpcodeVecV128AnyTrue: OpcodeVecV128AnyTrueName, + OpcodeVecI8x16Abs: OpcodeVecI8x16AbsName, + OpcodeVecI8x16Neg: OpcodeVecI8x16NegName, + OpcodeVecI8x16Popcnt: OpcodeVecI8x16PopcntName, + OpcodeVecI8x16AllTrue: OpcodeVecI8x16AllTrueName, + OpcodeVecI8x16BitMask: OpcodeVecI8x16BitMaskName, + OpcodeVecI8x16NarrowI16x8S: OpcodeVecI8x16NarrowI16x8SName, + OpcodeVecI8x16NarrowI16x8U: OpcodeVecI8x16NarrowI16x8UName, + OpcodeVecI8x16Shl: OpcodeVecI8x16ShlName, + OpcodeVecI8x16ShrS: OpcodeVecI8x16ShrSName, + OpcodeVecI8x16ShrU: OpcodeVecI8x16ShrUName, + OpcodeVecI8x16Add: OpcodeVecI8x16AddName, + OpcodeVecI8x16AddSatS: OpcodeVecI8x16AddSatSName, + OpcodeVecI8x16AddSatU: OpcodeVecI8x16AddSatUName, + OpcodeVecI8x16Sub: OpcodeVecI8x16SubName, + OpcodeVecI8x16SubSatS: OpcodeVecI8x16SubSatSName, + OpcodeVecI8x16SubSatU: OpcodeVecI8x16SubSatUName, + OpcodeVecI8x16MinS: OpcodeVecI8x16MinSName, + OpcodeVecI8x16MinU: OpcodeVecI8x16MinUName, + OpcodeVecI8x16MaxS: OpcodeVecI8x16MaxSName, + OpcodeVecI8x16MaxU: OpcodeVecI8x16MaxUName, + OpcodeVecI8x16AvgrU: OpcodeVecI8x16AvgrUName, + OpcodeVecI16x8ExtaddPairwiseI8x16S: OpcodeVecI16x8ExtaddPairwiseI8x16SName, + OpcodeVecI16x8ExtaddPairwiseI8x16U: OpcodeVecI16x8ExtaddPairwiseI8x16UName, + OpcodeVecI16x8Abs: OpcodeVecI16x8AbsName, + OpcodeVecI16x8Neg: OpcodeVecI16x8NegName, + OpcodeVecI16x8Q15mulrSatS: OpcodeVecI16x8Q15mulrSatSName, + OpcodeVecI16x8AllTrue: OpcodeVecI16x8AllTrueName, + OpcodeVecI16x8BitMask: OpcodeVecI16x8BitMaskName, + OpcodeVecI16x8NarrowI32x4S: OpcodeVecI16x8NarrowI32x4SName, + OpcodeVecI16x8NarrowI32x4U: OpcodeVecI16x8NarrowI32x4UName, + OpcodeVecI16x8ExtendLowI8x16S: OpcodeVecI16x8ExtendLowI8x16SName, + OpcodeVecI16x8ExtendHighI8x16S: OpcodeVecI16x8ExtendHighI8x16SName, + OpcodeVecI16x8ExtendLowI8x16U: OpcodeVecI16x8ExtendLowI8x16UName, + OpcodeVecI16x8ExtendHighI8x16U: OpcodeVecI16x8ExtendHighI8x16UName, + OpcodeVecI16x8Shl: OpcodeVecI16x8ShlName, + OpcodeVecI16x8ShrS: OpcodeVecI16x8ShrSName, + OpcodeVecI16x8ShrU: OpcodeVecI16x8ShrUName, + OpcodeVecI16x8Add: OpcodeVecI16x8AddName, + OpcodeVecI16x8AddSatS: OpcodeVecI16x8AddSatSName, + OpcodeVecI16x8AddSatU: OpcodeVecI16x8AddSatUName, + OpcodeVecI16x8Sub: OpcodeVecI16x8SubName, + OpcodeVecI16x8SubSatS: OpcodeVecI16x8SubSatSName, + OpcodeVecI16x8SubSatU: OpcodeVecI16x8SubSatUName, + OpcodeVecI16x8Mul: OpcodeVecI16x8MulName, + OpcodeVecI16x8MinS: OpcodeVecI16x8MinSName, + OpcodeVecI16x8MinU: OpcodeVecI16x8MinUName, + OpcodeVecI16x8MaxS: OpcodeVecI16x8MaxSName, + OpcodeVecI16x8MaxU: OpcodeVecI16x8MaxUName, + OpcodeVecI16x8AvgrU: OpcodeVecI16x8AvgrUName, + OpcodeVecI16x8ExtMulLowI8x16S: OpcodeVecI16x8ExtMulLowI8x16SName, + OpcodeVecI16x8ExtMulHighI8x16S: OpcodeVecI16x8ExtMulHighI8x16SName, + OpcodeVecI16x8ExtMulLowI8x16U: OpcodeVecI16x8ExtMulLowI8x16UName, + OpcodeVecI16x8ExtMulHighI8x16U: OpcodeVecI16x8ExtMulHighI8x16UName, + OpcodeVecI32x4ExtaddPairwiseI16x8S: OpcodeVecI32x4ExtaddPairwiseI16x8SName, + OpcodeVecI32x4ExtaddPairwiseI16x8U: OpcodeVecI32x4ExtaddPairwiseI16x8UName, + OpcodeVecI32x4Abs: OpcodeVecI32x4AbsName, + OpcodeVecI32x4Neg: OpcodeVecI32x4NegName, + OpcodeVecI32x4AllTrue: OpcodeVecI32x4AllTrueName, + OpcodeVecI32x4BitMask: OpcodeVecI32x4BitMaskName, + OpcodeVecI32x4ExtendLowI16x8S: OpcodeVecI32x4ExtendLowI16x8SName, + OpcodeVecI32x4ExtendHighI16x8S: OpcodeVecI32x4ExtendHighI16x8SName, + OpcodeVecI32x4ExtendLowI16x8U: OpcodeVecI32x4ExtendLowI16x8UName, + OpcodeVecI32x4ExtendHighI16x8U: OpcodeVecI32x4ExtendHighI16x8UName, + OpcodeVecI32x4Shl: OpcodeVecI32x4ShlName, + OpcodeVecI32x4ShrS: OpcodeVecI32x4ShrSName, + OpcodeVecI32x4ShrU: OpcodeVecI32x4ShrUName, + OpcodeVecI32x4Add: OpcodeVecI32x4AddName, + OpcodeVecI32x4Sub: OpcodeVecI32x4SubName, + OpcodeVecI32x4Mul: OpcodeVecI32x4MulName, + OpcodeVecI32x4MinS: OpcodeVecI32x4MinSName, + OpcodeVecI32x4MinU: OpcodeVecI32x4MinUName, + OpcodeVecI32x4MaxS: OpcodeVecI32x4MaxSName, + OpcodeVecI32x4MaxU: OpcodeVecI32x4MaxUName, + OpcodeVecI32x4DotI16x8S: OpcodeVecI32x4DotI16x8SName, + OpcodeVecI32x4ExtMulLowI16x8S: OpcodeVecI32x4ExtMulLowI16x8SName, + OpcodeVecI32x4ExtMulHighI16x8S: OpcodeVecI32x4ExtMulHighI16x8SName, + OpcodeVecI32x4ExtMulLowI16x8U: OpcodeVecI32x4ExtMulLowI16x8UName, + OpcodeVecI32x4ExtMulHighI16x8U: OpcodeVecI32x4ExtMulHighI16x8UName, + OpcodeVecI64x2Abs: OpcodeVecI64x2AbsName, + OpcodeVecI64x2Neg: OpcodeVecI64x2NegName, + OpcodeVecI64x2AllTrue: OpcodeVecI64x2AllTrueName, + OpcodeVecI64x2BitMask: OpcodeVecI64x2BitMaskName, + OpcodeVecI64x2ExtendLowI32x4S: OpcodeVecI64x2ExtendLowI32x4SName, + OpcodeVecI64x2ExtendHighI32x4S: OpcodeVecI64x2ExtendHighI32x4SName, + OpcodeVecI64x2ExtendLowI32x4U: OpcodeVecI64x2ExtendLowI32x4UName, + OpcodeVecI64x2ExtendHighI32x4U: OpcodeVecI64x2ExtendHighI32x4UName, + OpcodeVecI64x2Shl: OpcodeVecI64x2ShlName, + OpcodeVecI64x2ShrS: OpcodeVecI64x2ShrSName, + OpcodeVecI64x2ShrU: OpcodeVecI64x2ShrUName, + OpcodeVecI64x2Add: OpcodeVecI64x2AddName, + OpcodeVecI64x2Sub: OpcodeVecI64x2SubName, + OpcodeVecI64x2Mul: OpcodeVecI64x2MulName, + OpcodeVecI64x2ExtMulLowI32x4S: OpcodeVecI64x2ExtMulLowI32x4SName, + OpcodeVecI64x2ExtMulHighI32x4S: OpcodeVecI64x2ExtMulHighI32x4SName, + OpcodeVecI64x2ExtMulLowI32x4U: OpcodeVecI64x2ExtMulLowI32x4UName, + OpcodeVecI64x2ExtMulHighI32x4U: OpcodeVecI64x2ExtMulHighI32x4UName, + OpcodeVecF32x4Ceil: OpcodeVecF32x4CeilName, + OpcodeVecF32x4Floor: OpcodeVecF32x4FloorName, + OpcodeVecF32x4Trunc: OpcodeVecF32x4TruncName, + OpcodeVecF32x4Nearest: OpcodeVecF32x4NearestName, + OpcodeVecF32x4Abs: OpcodeVecF32x4AbsName, + OpcodeVecF32x4Neg: OpcodeVecF32x4NegName, + OpcodeVecF32x4Sqrt: OpcodeVecF32x4SqrtName, + OpcodeVecF32x4Add: OpcodeVecF32x4AddName, + OpcodeVecF32x4Sub: OpcodeVecF32x4SubName, + OpcodeVecF32x4Mul: OpcodeVecF32x4MulName, + OpcodeVecF32x4Div: OpcodeVecF32x4DivName, + OpcodeVecF32x4Min: OpcodeVecF32x4MinName, + OpcodeVecF32x4Max: OpcodeVecF32x4MaxName, + OpcodeVecF32x4Pmin: OpcodeVecF32x4PminName, + OpcodeVecF32x4Pmax: OpcodeVecF32x4PmaxName, + OpcodeVecF64x2Ceil: OpcodeVecF64x2CeilName, + OpcodeVecF64x2Floor: OpcodeVecF64x2FloorName, + OpcodeVecF64x2Trunc: OpcodeVecF64x2TruncName, + OpcodeVecF64x2Nearest: OpcodeVecF64x2NearestName, + OpcodeVecF64x2Abs: OpcodeVecF64x2AbsName, + OpcodeVecF64x2Neg: OpcodeVecF64x2NegName, + OpcodeVecF64x2Sqrt: OpcodeVecF64x2SqrtName, + OpcodeVecF64x2Add: OpcodeVecF64x2AddName, + OpcodeVecF64x2Sub: OpcodeVecF64x2SubName, + OpcodeVecF64x2Mul: OpcodeVecF64x2MulName, + OpcodeVecF64x2Div: OpcodeVecF64x2DivName, + OpcodeVecF64x2Min: OpcodeVecF64x2MinName, + OpcodeVecF64x2Max: OpcodeVecF64x2MaxName, + OpcodeVecF64x2Pmin: OpcodeVecF64x2PminName, + OpcodeVecF64x2Pmax: OpcodeVecF64x2PmaxName, + OpcodeVecI32x4TruncSatF32x4S: OpcodeVecI32x4TruncSatF32x4SName, + OpcodeVecI32x4TruncSatF32x4U: OpcodeVecI32x4TruncSatF32x4UName, + OpcodeVecF32x4ConvertI32x4S: OpcodeVecF32x4ConvertI32x4SName, + OpcodeVecF32x4ConvertI32x4U: OpcodeVecF32x4ConvertI32x4UName, + OpcodeVecI32x4TruncSatF64x2SZero: OpcodeVecI32x4TruncSatF64x2SZeroName, + OpcodeVecI32x4TruncSatF64x2UZero: OpcodeVecI32x4TruncSatF64x2UZeroName, + OpcodeVecF64x2ConvertLowI32x4S: OpcodeVecF64x2ConvertLowI32x4SName, + OpcodeVecF64x2ConvertLowI32x4U: OpcodeVecF64x2ConvertLowI32x4UName, + OpcodeVecF32x4DemoteF64x2Zero: OpcodeVecF32x4DemoteF64x2ZeroName, + OpcodeVecF64x2PromoteLowF32x4Zero: OpcodeVecF64x2PromoteLowF32x4ZeroName, +} + +// VectorInstructionName returns the instruction name corresponding to the vector Opcode. +func VectorInstructionName(oc OpcodeVec) (ret string) { + return vectorInstructionName[oc] +} + +const ( + OpcodeAtomicMemoryNotifyName = "memory.atomic.notify" + OpcodeAtomicMemoryWait32Name = "memory.atomic.wait32" + OpcodeAtomicMemoryWait64Name = "memory.atomic.wait64" + OpcodeAtomicFenceName = "atomic.fence" + + OpcodeAtomicI32LoadName = "i32.atomic.load" + OpcodeAtomicI64LoadName = "i64.atomic.load" + OpcodeAtomicI32Load8UName = "i32.atomic.load8_u" + OpcodeAtomicI32Load16UName = "i32.atomic.load16_u" + OpcodeAtomicI64Load8UName = "i64.atomic.load8_u" + OpcodeAtomicI64Load16UName = "i64.atomic.load16_u" + OpcodeAtomicI64Load32UName = "i64.atomic.load32_u" + OpcodeAtomicI32StoreName = "i32.atomic.store" + OpcodeAtomicI64StoreName = "i64.atomic.store" + OpcodeAtomicI32Store8Name = "i32.atomic.store8" + OpcodeAtomicI32Store16Name = "i32.atomic.store16" + OpcodeAtomicI64Store8Name = "i64.atomic.store8" + OpcodeAtomicI64Store16Name = "i64.atomic.store16" + OpcodeAtomicI64Store32Name = "i64.atomic.store32" + + OpcodeAtomicI32RmwAddName = "i32.atomic.rmw.add" + OpcodeAtomicI64RmwAddName = "i64.atomic.rmw.add" + OpcodeAtomicI32Rmw8AddUName = "i32.atomic.rmw8.add_u" + OpcodeAtomicI32Rmw16AddUName = "i32.atomic.rmw16.add_u" + OpcodeAtomicI64Rmw8AddUName = "i64.atomic.rmw8.add_u" + OpcodeAtomicI64Rmw16AddUName = "i64.atomic.rmw16.add_u" + OpcodeAtomicI64Rmw32AddUName = "i64.atomic.rmw32.add_u" + + OpcodeAtomicI32RmwSubName = "i32.atomic.rmw.sub" + OpcodeAtomicI64RmwSubName = "i64.atomic.rmw.sub" + OpcodeAtomicI32Rmw8SubUName = "i32.atomic.rmw8.sub_u" + OpcodeAtomicI32Rmw16SubUName = "i32.atomic.rmw16.sub_u" + OpcodeAtomicI64Rmw8SubUName = "i64.atomic.rmw8.sub_u" + OpcodeAtomicI64Rmw16SubUName = "i64.atomic.rmw16.sub_u" + OpcodeAtomicI64Rmw32SubUName = "i64.atomic.rmw32.sub_u" + + OpcodeAtomicI32RmwAndName = "i32.atomic.rmw.and" + OpcodeAtomicI64RmwAndName = "i64.atomic.rmw.and" + OpcodeAtomicI32Rmw8AndUName = "i32.atomic.rmw8.and_u" + OpcodeAtomicI32Rmw16AndUName = "i32.atomic.rmw16.and_u" + OpcodeAtomicI64Rmw8AndUName = "i64.atomic.rmw8.and_u" + OpcodeAtomicI64Rmw16AndUName = "i64.atomic.rmw16.and_u" + OpcodeAtomicI64Rmw32AndUName = "i64.atomic.rmw32.and_u" + + OpcodeAtomicI32RmwOrName = "i32.atomic.rmw.or" + OpcodeAtomicI64RmwOrName = "i64.atomic.rmw.or" + OpcodeAtomicI32Rmw8OrUName = "i32.atomic.rmw8.or_u" + OpcodeAtomicI32Rmw16OrUName = "i32.atomic.rmw16.or_u" + OpcodeAtomicI64Rmw8OrUName = "i64.atomic.rmw8.or_u" + OpcodeAtomicI64Rmw16OrUName = "i64.atomic.rmw16.or_u" + OpcodeAtomicI64Rmw32OrUName = "i64.atomic.rmw32.or_u" + + OpcodeAtomicI32RmwXorName = "i32.atomic.rmw.xor" + OpcodeAtomicI64RmwXorName = "i64.atomic.rmw.xor" + OpcodeAtomicI32Rmw8XorUName = "i32.atomic.rmw8.xor_u" + OpcodeAtomicI32Rmw16XorUName = "i32.atomic.rmw16.xor_u" + OpcodeAtomicI64Rmw8XorUName = "i64.atomic.rmw8.xor_u" + OpcodeAtomicI64Rmw16XorUName = "i64.atomic.rmw16.xor_u" + OpcodeAtomicI64Rmw32XorUName = "i64.atomic.rmw32.xor_u" + + OpcodeAtomicI32RmwXchgName = "i32.atomic.rmw.xchg" + OpcodeAtomicI64RmwXchgName = "i64.atomic.rmw.xchg" + OpcodeAtomicI32Rmw8XchgUName = "i32.atomic.rmw8.xchg_u" + OpcodeAtomicI32Rmw16XchgUName = "i32.atomic.rmw16.xchg_u" + OpcodeAtomicI64Rmw8XchgUName = "i64.atomic.rmw8.xchg_u" + OpcodeAtomicI64Rmw16XchgUName = "i64.atomic.rmw16.xchg_u" + OpcodeAtomicI64Rmw32XchgUName = "i64.atomic.rmw32.xchg_u" + + OpcodeAtomicI32RmwCmpxchgName = "i32.atomic.rmw.cmpxchg" + OpcodeAtomicI64RmwCmpxchgName = "i64.atomic.rmw.cmpxchg" + OpcodeAtomicI32Rmw8CmpxchgUName = "i32.atomic.rmw8.cmpxchg_u" + OpcodeAtomicI32Rmw16CmpxchgUName = "i32.atomic.rmw16.cmpxchg_u" + OpcodeAtomicI64Rmw8CmpxchgUName = "i64.atomic.rmw8.cmpxchg_u" + OpcodeAtomicI64Rmw16CmpxchgUName = "i64.atomic.rmw16.cmpxchg_u" + OpcodeAtomicI64Rmw32CmpxchgUName = "i64.atomic.rmw32.cmpxchg_u" +) + +var atomicInstructionName = map[OpcodeAtomic]string{ + OpcodeAtomicMemoryNotify: OpcodeAtomicMemoryNotifyName, + OpcodeAtomicMemoryWait32: OpcodeAtomicMemoryWait32Name, + OpcodeAtomicMemoryWait64: OpcodeAtomicMemoryWait64Name, + OpcodeAtomicFence: OpcodeAtomicFenceName, + + OpcodeAtomicI32Load: OpcodeAtomicI32LoadName, + OpcodeAtomicI64Load: OpcodeAtomicI64LoadName, + OpcodeAtomicI32Load8U: OpcodeAtomicI32Load8UName, + OpcodeAtomicI32Load16U: OpcodeAtomicI32Load16UName, + OpcodeAtomicI64Load8U: OpcodeAtomicI64Load8UName, + OpcodeAtomicI64Load16U: OpcodeAtomicI64Load16UName, + OpcodeAtomicI64Load32U: OpcodeAtomicI64Load32UName, + OpcodeAtomicI32Store: OpcodeAtomicI32StoreName, + OpcodeAtomicI64Store: OpcodeAtomicI64StoreName, + OpcodeAtomicI32Store8: OpcodeAtomicI32Store8Name, + OpcodeAtomicI32Store16: OpcodeAtomicI32Store16Name, + OpcodeAtomicI64Store8: OpcodeAtomicI64Store8Name, + OpcodeAtomicI64Store16: OpcodeAtomicI64Store16Name, + OpcodeAtomicI64Store32: OpcodeAtomicI64Store32Name, + + OpcodeAtomicI32RmwAdd: OpcodeAtomicI32RmwAddName, + OpcodeAtomicI64RmwAdd: OpcodeAtomicI64RmwAddName, + OpcodeAtomicI32Rmw8AddU: OpcodeAtomicI32Rmw8AddUName, + OpcodeAtomicI32Rmw16AddU: OpcodeAtomicI32Rmw16AddUName, + OpcodeAtomicI64Rmw8AddU: OpcodeAtomicI64Rmw8AddUName, + OpcodeAtomicI64Rmw16AddU: OpcodeAtomicI64Rmw16AddUName, + OpcodeAtomicI64Rmw32AddU: OpcodeAtomicI64Rmw32AddUName, + + OpcodeAtomicI32RmwSub: OpcodeAtomicI32RmwSubName, + OpcodeAtomicI64RmwSub: OpcodeAtomicI64RmwSubName, + OpcodeAtomicI32Rmw8SubU: OpcodeAtomicI32Rmw8SubUName, + OpcodeAtomicI32Rmw16SubU: OpcodeAtomicI32Rmw16SubUName, + OpcodeAtomicI64Rmw8SubU: OpcodeAtomicI64Rmw8SubUName, + OpcodeAtomicI64Rmw16SubU: OpcodeAtomicI64Rmw16SubUName, + OpcodeAtomicI64Rmw32SubU: OpcodeAtomicI64Rmw32SubUName, + + OpcodeAtomicI32RmwAnd: OpcodeAtomicI32RmwAndName, + OpcodeAtomicI64RmwAnd: OpcodeAtomicI64RmwAndName, + OpcodeAtomicI32Rmw8AndU: OpcodeAtomicI32Rmw8AndUName, + OpcodeAtomicI32Rmw16AndU: OpcodeAtomicI32Rmw16AndUName, + OpcodeAtomicI64Rmw8AndU: OpcodeAtomicI64Rmw8AndUName, + OpcodeAtomicI64Rmw16AndU: OpcodeAtomicI64Rmw16AndUName, + OpcodeAtomicI64Rmw32AndU: OpcodeAtomicI64Rmw32AndUName, + + OpcodeAtomicI32RmwOr: OpcodeAtomicI32RmwOrName, + OpcodeAtomicI64RmwOr: OpcodeAtomicI64RmwOrName, + OpcodeAtomicI32Rmw8OrU: OpcodeAtomicI32Rmw8OrUName, + OpcodeAtomicI32Rmw16OrU: OpcodeAtomicI32Rmw16OrUName, + OpcodeAtomicI64Rmw8OrU: OpcodeAtomicI64Rmw8OrUName, + OpcodeAtomicI64Rmw16OrU: OpcodeAtomicI64Rmw16OrUName, + OpcodeAtomicI64Rmw32OrU: OpcodeAtomicI64Rmw32OrUName, + + OpcodeAtomicI32RmwXor: OpcodeAtomicI32RmwXorName, + OpcodeAtomicI64RmwXor: OpcodeAtomicI64RmwXorName, + OpcodeAtomicI32Rmw8XorU: OpcodeAtomicI32Rmw8XorUName, + OpcodeAtomicI32Rmw16XorU: OpcodeAtomicI32Rmw16XorUName, + OpcodeAtomicI64Rmw8XorU: OpcodeAtomicI64Rmw8XorUName, + OpcodeAtomicI64Rmw16XorU: OpcodeAtomicI64Rmw16XorUName, + OpcodeAtomicI64Rmw32XorU: OpcodeAtomicI64Rmw32XorUName, + + OpcodeAtomicI32RmwXchg: OpcodeAtomicI32RmwXchgName, + OpcodeAtomicI64RmwXchg: OpcodeAtomicI64RmwXchgName, + OpcodeAtomicI32Rmw8XchgU: OpcodeAtomicI32Rmw8XchgUName, + OpcodeAtomicI32Rmw16XchgU: OpcodeAtomicI32Rmw16XchgUName, + OpcodeAtomicI64Rmw8XchgU: OpcodeAtomicI64Rmw8XchgUName, + OpcodeAtomicI64Rmw16XchgU: OpcodeAtomicI64Rmw16XchgUName, + OpcodeAtomicI64Rmw32XchgU: OpcodeAtomicI64Rmw32XchgUName, + + OpcodeAtomicI32RmwCmpxchg: OpcodeAtomicI32RmwCmpxchgName, + OpcodeAtomicI64RmwCmpxchg: OpcodeAtomicI64RmwCmpxchgName, + OpcodeAtomicI32Rmw8CmpxchgU: OpcodeAtomicI32Rmw8CmpxchgUName, + OpcodeAtomicI32Rmw16CmpxchgU: OpcodeAtomicI32Rmw16CmpxchgUName, + OpcodeAtomicI64Rmw8CmpxchgU: OpcodeAtomicI64Rmw8CmpxchgUName, + OpcodeAtomicI64Rmw16CmpxchgU: OpcodeAtomicI64Rmw16CmpxchgUName, + OpcodeAtomicI64Rmw32CmpxchgU: OpcodeAtomicI64Rmw32CmpxchgUName, +} + +// AtomicInstructionName returns the instruction name corresponding to the atomic Opcode. +func AtomicInstructionName(oc OpcodeAtomic) (ret string) { + return atomicInstructionName[oc] +} diff --git a/vendor/github.com/tetratelabs/wazero/internal/wasm/memory.go b/vendor/github.com/tetratelabs/wazero/internal/wasm/memory.go new file mode 100644 index 000000000..5cc5012da --- /dev/null +++ b/vendor/github.com/tetratelabs/wazero/internal/wasm/memory.go @@ -0,0 +1,461 @@ +package wasm + +import ( + "container/list" + "encoding/binary" + "fmt" + "math" + "reflect" + "sync" + "sync/atomic" + "time" + "unsafe" + + "github.com/tetratelabs/wazero/api" + "github.com/tetratelabs/wazero/experimental" + "github.com/tetratelabs/wazero/internal/internalapi" + "github.com/tetratelabs/wazero/internal/wasmruntime" +) + +const ( + // MemoryPageSize is the unit of memory length in WebAssembly, + // and is defined as 2^16 = 65536. + // See https://www.w3.org/TR/2019/REC-wasm-core-1-20191205/#memory-instances%E2%91%A0 + MemoryPageSize = uint32(65536) + // MemoryLimitPages is maximum number of pages defined (2^16). + // See https://www.w3.org/TR/2019/REC-wasm-core-1-20191205/#grow-mem + MemoryLimitPages = uint32(65536) + // MemoryPageSizeInBits satisfies the relation: "1 << MemoryPageSizeInBits == MemoryPageSize". + MemoryPageSizeInBits = 16 +) + +// compile-time check to ensure MemoryInstance implements api.Memory +var _ api.Memory = &MemoryInstance{} + +type waiters struct { + mux sync.Mutex + l *list.List +} + +// MemoryInstance represents a memory instance in a store, and implements api.Memory. +// +// Note: In WebAssembly 1.0 (20191205), there may be up to one Memory per store, which means the precise memory is always +// wasm.Store Memories index zero: `store.Memories[0]` +// See https://www.w3.org/TR/2019/REC-wasm-core-1-20191205/#memory-instances%E2%91%A0. +type MemoryInstance struct { + internalapi.WazeroOnlyType + + Buffer []byte + Min, Cap, Max uint32 + Shared bool + // definition is known at compile time. + definition api.MemoryDefinition + + // Mux is used in interpreter mode to prevent overlapping calls to atomic instructions, + // introduced with WebAssembly threads proposal. + Mux sync.Mutex + + // waiters implements atomic wait and notify. It is implemented similarly to golang.org/x/sync/semaphore, + // with a fixed weight of 1 and no spurious notifications. + waiters sync.Map + + expBuffer experimental.LinearMemory +} + +// NewMemoryInstance creates a new instance based on the parameters in the SectionIDMemory. +func NewMemoryInstance(memSec *Memory, allocator experimental.MemoryAllocator) *MemoryInstance { + minBytes := MemoryPagesToBytesNum(memSec.Min) + capBytes := MemoryPagesToBytesNum(memSec.Cap) + maxBytes := MemoryPagesToBytesNum(memSec.Max) + + var buffer []byte + var expBuffer experimental.LinearMemory + if allocator != nil { + expBuffer = allocator.Allocate(capBytes, maxBytes) + buffer = expBuffer.Reallocate(minBytes) + } else if memSec.IsShared { + // Shared memory needs a fixed buffer, so allocate with the maximum size. + // + // The rationale as to why we can simply use make([]byte) to a fixed buffer is that Go's GC is non-relocating. + // That is not a part of Go spec, but is well-known thing in Go community (wazero's compiler heavily relies on it!) + // * https://github.com/go4org/unsafe-assume-no-moving-gc + // + // Also, allocating Max here isn't harmful as the Go runtime uses mmap for large allocations, therefore, + // the memory buffer allocation here is virtual and doesn't consume physical memory until it's used. + // * https://github.com/golang/go/blob/8121604559035734c9677d5281bbdac8b1c17a1e/src/runtime/malloc.go#L1059 + // * https://github.com/golang/go/blob/8121604559035734c9677d5281bbdac8b1c17a1e/src/runtime/malloc.go#L1165 + buffer = make([]byte, minBytes, maxBytes) + } else { + buffer = make([]byte, minBytes, capBytes) + } + return &MemoryInstance{ + Buffer: buffer, + Min: memSec.Min, + Cap: memoryBytesNumToPages(uint64(cap(buffer))), + Max: memSec.Max, + Shared: memSec.IsShared, + expBuffer: expBuffer, + } +} + +// Definition implements the same method as documented on api.Memory. +func (m *MemoryInstance) Definition() api.MemoryDefinition { + return m.definition +} + +// Size implements the same method as documented on api.Memory. +func (m *MemoryInstance) Size() uint32 { + return uint32(len(m.Buffer)) +} + +// ReadByte implements the same method as documented on api.Memory. +func (m *MemoryInstance) ReadByte(offset uint32) (byte, bool) { + if !m.hasSize(offset, 1) { + return 0, false + } + return m.Buffer[offset], true +} + +// ReadUint16Le implements the same method as documented on api.Memory. +func (m *MemoryInstance) ReadUint16Le(offset uint32) (uint16, bool) { + if !m.hasSize(offset, 2) { + return 0, false + } + return binary.LittleEndian.Uint16(m.Buffer[offset : offset+2]), true +} + +// ReadUint32Le implements the same method as documented on api.Memory. +func (m *MemoryInstance) ReadUint32Le(offset uint32) (uint32, bool) { + return m.readUint32Le(offset) +} + +// ReadFloat32Le implements the same method as documented on api.Memory. +func (m *MemoryInstance) ReadFloat32Le(offset uint32) (float32, bool) { + v, ok := m.readUint32Le(offset) + if !ok { + return 0, false + } + return math.Float32frombits(v), true +} + +// ReadUint64Le implements the same method as documented on api.Memory. +func (m *MemoryInstance) ReadUint64Le(offset uint32) (uint64, bool) { + return m.readUint64Le(offset) +} + +// ReadFloat64Le implements the same method as documented on api.Memory. +func (m *MemoryInstance) ReadFloat64Le(offset uint32) (float64, bool) { + v, ok := m.readUint64Le(offset) + if !ok { + return 0, false + } + return math.Float64frombits(v), true +} + +// Read implements the same method as documented on api.Memory. +func (m *MemoryInstance) Read(offset, byteCount uint32) ([]byte, bool) { + if !m.hasSize(offset, uint64(byteCount)) { + return nil, false + } + return m.Buffer[offset : offset+byteCount : offset+byteCount], true +} + +// WriteByte implements the same method as documented on api.Memory. +func (m *MemoryInstance) WriteByte(offset uint32, v byte) bool { + if !m.hasSize(offset, 1) { + return false + } + m.Buffer[offset] = v + return true +} + +// WriteUint16Le implements the same method as documented on api.Memory. +func (m *MemoryInstance) WriteUint16Le(offset uint32, v uint16) bool { + if !m.hasSize(offset, 2) { + return false + } + binary.LittleEndian.PutUint16(m.Buffer[offset:], v) + return true +} + +// WriteUint32Le implements the same method as documented on api.Memory. +func (m *MemoryInstance) WriteUint32Le(offset, v uint32) bool { + return m.writeUint32Le(offset, v) +} + +// WriteFloat32Le implements the same method as documented on api.Memory. +func (m *MemoryInstance) WriteFloat32Le(offset uint32, v float32) bool { + return m.writeUint32Le(offset, math.Float32bits(v)) +} + +// WriteUint64Le implements the same method as documented on api.Memory. +func (m *MemoryInstance) WriteUint64Le(offset uint32, v uint64) bool { + return m.writeUint64Le(offset, v) +} + +// WriteFloat64Le implements the same method as documented on api.Memory. +func (m *MemoryInstance) WriteFloat64Le(offset uint32, v float64) bool { + return m.writeUint64Le(offset, math.Float64bits(v)) +} + +// Write implements the same method as documented on api.Memory. +func (m *MemoryInstance) Write(offset uint32, val []byte) bool { + if !m.hasSize(offset, uint64(len(val))) { + return false + } + copy(m.Buffer[offset:], val) + return true +} + +// WriteString implements the same method as documented on api.Memory. +func (m *MemoryInstance) WriteString(offset uint32, val string) bool { + if !m.hasSize(offset, uint64(len(val))) { + return false + } + copy(m.Buffer[offset:], val) + return true +} + +// MemoryPagesToBytesNum converts the given pages into the number of bytes contained in these pages. +func MemoryPagesToBytesNum(pages uint32) (bytesNum uint64) { + return uint64(pages) << MemoryPageSizeInBits +} + +// Grow implements the same method as documented on api.Memory. +func (m *MemoryInstance) Grow(delta uint32) (result uint32, ok bool) { + currentPages := m.Pages() + if delta == 0 { + return currentPages, true + } + + // If exceeds the max of memory size, we push -1 according to the spec. + newPages := currentPages + delta + if newPages > m.Max || int32(delta) < 0 { + return 0, false + } else if m.expBuffer != nil { + buffer := m.expBuffer.Reallocate(MemoryPagesToBytesNum(newPages)) + if m.Shared { + if unsafe.SliceData(buffer) != unsafe.SliceData(m.Buffer) { + panic("shared memory cannot move, this is a bug in the memory allocator") + } + // We assume grow is called under a guest lock. + // But the memory length is accessed elsewhere, + // so use atomic to make the new length visible across threads. + atomicStoreLengthAndCap(&m.Buffer, uintptr(len(buffer)), uintptr(cap(buffer))) + m.Cap = memoryBytesNumToPages(uint64(cap(buffer))) + } else { + m.Buffer = buffer + m.Cap = newPages + } + return currentPages, true + } else if newPages > m.Cap { // grow the memory. + if m.Shared { + panic("shared memory cannot be grown, this is a bug in wazero") + } + m.Buffer = append(m.Buffer, make([]byte, MemoryPagesToBytesNum(delta))...) + m.Cap = newPages + return currentPages, true + } else { // We already have the capacity we need. + if m.Shared { + // We assume grow is called under a guest lock. + // But the memory length is accessed elsewhere, + // so use atomic to make the new length visible across threads. + atomicStoreLength(&m.Buffer, uintptr(MemoryPagesToBytesNum(newPages))) + } else { + m.Buffer = m.Buffer[:MemoryPagesToBytesNum(newPages)] + } + return currentPages, true + } +} + +// Pages implements the same method as documented on api.Memory. +func (m *MemoryInstance) Pages() (result uint32) { + return memoryBytesNumToPages(uint64(len(m.Buffer))) +} + +// PagesToUnitOfBytes converts the pages to a human-readable form similar to what's specified. e.g. 1 -> "64Ki" +// +// See https://www.w3.org/TR/2019/REC-wasm-core-1-20191205/#memory-instances%E2%91%A0 +func PagesToUnitOfBytes(pages uint32) string { + k := pages * 64 + if k < 1024 { + return fmt.Sprintf("%d Ki", k) + } + m := k / 1024 + if m < 1024 { + return fmt.Sprintf("%d Mi", m) + } + g := m / 1024 + if g < 1024 { + return fmt.Sprintf("%d Gi", g) + } + return fmt.Sprintf("%d Ti", g/1024) +} + +// Below are raw functions used to implement the api.Memory API: + +// Uses atomic write to update the length of a slice. +func atomicStoreLengthAndCap(slice *[]byte, length uintptr, cap uintptr) { + slicePtr := (*reflect.SliceHeader)(unsafe.Pointer(slice)) + capPtr := (*uintptr)(unsafe.Pointer(&slicePtr.Cap)) + atomic.StoreUintptr(capPtr, cap) + lenPtr := (*uintptr)(unsafe.Pointer(&slicePtr.Len)) + atomic.StoreUintptr(lenPtr, length) +} + +// Uses atomic write to update the length of a slice. +func atomicStoreLength(slice *[]byte, length uintptr) { + slicePtr := (*reflect.SliceHeader)(unsafe.Pointer(slice)) + lenPtr := (*uintptr)(unsafe.Pointer(&slicePtr.Len)) + atomic.StoreUintptr(lenPtr, length) +} + +// memoryBytesNumToPages converts the given number of bytes into the number of pages. +func memoryBytesNumToPages(bytesNum uint64) (pages uint32) { + return uint32(bytesNum >> MemoryPageSizeInBits) +} + +// hasSize returns true if Len is sufficient for byteCount at the given offset. +// +// Note: This is always fine, because memory can grow, but never shrink. +func (m *MemoryInstance) hasSize(offset uint32, byteCount uint64) bool { + return uint64(offset)+byteCount <= uint64(len(m.Buffer)) // uint64 prevents overflow on add +} + +// readUint32Le implements ReadUint32Le without using a context. This is extracted as both ints and floats are stored in +// memory as uint32le. +func (m *MemoryInstance) readUint32Le(offset uint32) (uint32, bool) { + if !m.hasSize(offset, 4) { + return 0, false + } + return binary.LittleEndian.Uint32(m.Buffer[offset : offset+4]), true +} + +// readUint64Le implements ReadUint64Le without using a context. This is extracted as both ints and floats are stored in +// memory as uint64le. +func (m *MemoryInstance) readUint64Le(offset uint32) (uint64, bool) { + if !m.hasSize(offset, 8) { + return 0, false + } + return binary.LittleEndian.Uint64(m.Buffer[offset : offset+8]), true +} + +// writeUint32Le implements WriteUint32Le without using a context. This is extracted as both ints and floats are stored +// in memory as uint32le. +func (m *MemoryInstance) writeUint32Le(offset uint32, v uint32) bool { + if !m.hasSize(offset, 4) { + return false + } + binary.LittleEndian.PutUint32(m.Buffer[offset:], v) + return true +} + +// writeUint64Le implements WriteUint64Le without using a context. This is extracted as both ints and floats are stored +// in memory as uint64le. +func (m *MemoryInstance) writeUint64Le(offset uint32, v uint64) bool { + if !m.hasSize(offset, 8) { + return false + } + binary.LittleEndian.PutUint64(m.Buffer[offset:], v) + return true +} + +// Wait32 suspends the caller until the offset is notified by a different agent. +func (m *MemoryInstance) Wait32(offset uint32, exp uint32, timeout int64, reader func(mem *MemoryInstance, offset uint32) uint32) uint64 { + w := m.getWaiters(offset) + w.mux.Lock() + + cur := reader(m, offset) + if cur != exp { + w.mux.Unlock() + return 1 + } + + return m.wait(w, timeout) +} + +// Wait64 suspends the caller until the offset is notified by a different agent. +func (m *MemoryInstance) Wait64(offset uint32, exp uint64, timeout int64, reader func(mem *MemoryInstance, offset uint32) uint64) uint64 { + w := m.getWaiters(offset) + w.mux.Lock() + + cur := reader(m, offset) + if cur != exp { + w.mux.Unlock() + return 1 + } + + return m.wait(w, timeout) +} + +func (m *MemoryInstance) wait(w *waiters, timeout int64) uint64 { + if w.l == nil { + w.l = list.New() + } + + // The specification requires a trap if the number of existing waiters + 1 == 2^32, so we add a check here. + // In practice, it is unlikely the application would ever accumulate such a large number of waiters as it + // indicates several GB of RAM used just for the list of waiters. + // https://github.com/WebAssembly/threads/blob/main/proposals/threads/Overview.md#wait + if uint64(w.l.Len()+1) == 1<<32 { + w.mux.Unlock() + panic(wasmruntime.ErrRuntimeTooManyWaiters) + } + + ready := make(chan struct{}) + elem := w.l.PushBack(ready) + w.mux.Unlock() + + if timeout < 0 { + <-ready + return 0 + } else { + select { + case <-ready: + return 0 + case <-time.After(time.Duration(timeout)): + // While we could see if the channel completed by now and ignore the timeout, similar to x/sync/semaphore, + // the Wasm spec doesn't specify this behavior, so we keep things simple by prioritizing the timeout. + w.mux.Lock() + w.l.Remove(elem) + w.mux.Unlock() + return 2 + } + } +} + +func (m *MemoryInstance) getWaiters(offset uint32) *waiters { + wAny, ok := m.waiters.Load(offset) + if !ok { + // The first time an address is waited on, simultaneous waits will cause extra allocations. + // Further operations will be loaded above, which is also the general pattern of usage with + // mutexes. + wAny, _ = m.waiters.LoadOrStore(offset, &waiters{}) + } + + return wAny.(*waiters) +} + +// Notify wakes up at most count waiters at the given offset. +func (m *MemoryInstance) Notify(offset uint32, count uint32) uint32 { + wAny, ok := m.waiters.Load(offset) + if !ok { + return 0 + } + w := wAny.(*waiters) + + w.mux.Lock() + defer w.mux.Unlock() + if w.l == nil { + return 0 + } + + res := uint32(0) + for num := w.l.Len(); num > 0 && res < count; num = w.l.Len() { + w := w.l.Remove(w.l.Front()).(chan struct{}) + close(w) + res++ + } + + return res +} diff --git a/vendor/github.com/tetratelabs/wazero/internal/wasm/memory_definition.go b/vendor/github.com/tetratelabs/wazero/internal/wasm/memory_definition.go new file mode 100644 index 000000000..03d6fd303 --- /dev/null +++ b/vendor/github.com/tetratelabs/wazero/internal/wasm/memory_definition.go @@ -0,0 +1,128 @@ +package wasm + +import ( + "github.com/tetratelabs/wazero/api" + "github.com/tetratelabs/wazero/internal/internalapi" +) + +// ImportedMemories implements the same method as documented on wazero.CompiledModule. +func (m *Module) ImportedMemories() (ret []api.MemoryDefinition) { + for i := range m.MemoryDefinitionSection { + d := &m.MemoryDefinitionSection[i] + if d.importDesc != nil { + ret = append(ret, d) + } + } + return +} + +// ExportedMemories implements the same method as documented on wazero.CompiledModule. +func (m *Module) ExportedMemories() map[string]api.MemoryDefinition { + ret := map[string]api.MemoryDefinition{} + for i := range m.MemoryDefinitionSection { + d := &m.MemoryDefinitionSection[i] + for _, e := range d.exportNames { + ret[e] = d + } + } + return ret +} + +// BuildMemoryDefinitions generates memory metadata that can be parsed from +// the module. This must be called after all validation. +// +// Note: This is exported for wazero.Runtime `CompileModule`. +func (m *Module) BuildMemoryDefinitions() { + var moduleName string + if m.NameSection != nil { + moduleName = m.NameSection.ModuleName + } + + memoryCount := m.ImportMemoryCount + if m.MemorySection != nil { + memoryCount++ + } + + if memoryCount == 0 { + return + } + + m.MemoryDefinitionSection = make([]MemoryDefinition, 0, memoryCount) + importMemIdx := Index(0) + for i := range m.ImportSection { + imp := &m.ImportSection[i] + if imp.Type != ExternTypeMemory { + continue + } + + m.MemoryDefinitionSection = append(m.MemoryDefinitionSection, MemoryDefinition{ + importDesc: &[2]string{imp.Module, imp.Name}, + index: importMemIdx, + memory: imp.DescMem, + }) + importMemIdx++ + } + + if m.MemorySection != nil { + m.MemoryDefinitionSection = append(m.MemoryDefinitionSection, MemoryDefinition{ + index: importMemIdx, + memory: m.MemorySection, + }) + } + + for i := range m.MemoryDefinitionSection { + d := &m.MemoryDefinitionSection[i] + d.moduleName = moduleName + for i := range m.ExportSection { + e := &m.ExportSection[i] + if e.Type == ExternTypeMemory && e.Index == d.index { + d.exportNames = append(d.exportNames, e.Name) + } + } + } +} + +// MemoryDefinition implements api.MemoryDefinition +type MemoryDefinition struct { + internalapi.WazeroOnlyType + moduleName string + index Index + importDesc *[2]string + exportNames []string + memory *Memory +} + +// ModuleName implements the same method as documented on api.MemoryDefinition. +func (f *MemoryDefinition) ModuleName() string { + return f.moduleName +} + +// Index implements the same method as documented on api.MemoryDefinition. +func (f *MemoryDefinition) Index() uint32 { + return f.index +} + +// Import implements the same method as documented on api.MemoryDefinition. +func (f *MemoryDefinition) Import() (moduleName, name string, isImport bool) { + if importDesc := f.importDesc; importDesc != nil { + moduleName, name, isImport = importDesc[0], importDesc[1], true + } + return +} + +// ExportNames implements the same method as documented on api.MemoryDefinition. +func (f *MemoryDefinition) ExportNames() []string { + return f.exportNames +} + +// Min implements the same method as documented on api.MemoryDefinition. +func (f *MemoryDefinition) Min() uint32 { + return f.memory.Min +} + +// Max implements the same method as documented on api.MemoryDefinition. +func (f *MemoryDefinition) Max() (max uint32, encoded bool) { + max = f.memory.Max + encoded = f.memory.IsMaxEncoded + return +} diff --git a/vendor/github.com/tetratelabs/wazero/internal/wasm/module.go b/vendor/github.com/tetratelabs/wazero/internal/wasm/module.go new file mode 100644 index 000000000..68573b918 --- /dev/null +++ b/vendor/github.com/tetratelabs/wazero/internal/wasm/module.go @@ -0,0 +1,1083 @@ +package wasm + +import ( + "bytes" + "crypto/sha256" + "encoding/binary" + "errors" + "fmt" + "io" + "sort" + "strings" + "sync" + + "github.com/tetratelabs/wazero/api" + "github.com/tetratelabs/wazero/experimental" + "github.com/tetratelabs/wazero/internal/ieee754" + "github.com/tetratelabs/wazero/internal/leb128" + "github.com/tetratelabs/wazero/internal/wasmdebug" +) + +// Module is a WebAssembly binary representation. +// See https://www.w3.org/TR/2019/REC-wasm-core-1-20191205/#modules%E2%91%A8 +// +// Differences from the specification: +// * NameSection is the only key ("name") decoded from the SectionIDCustom. +// * ExportSection is represented as a map for lookup convenience. +// * Code.GoFunc is contains any go `func`. It may be present when Code.Body is not. +type Module struct { + // TypeSection contains the unique FunctionType of functions imported or defined in this module. + // + // Note: Currently, there is no type ambiguity in the index as WebAssembly 1.0 only defines function type. + // In the future, other types may be introduced to support CoreFeatures such as module linking. + // + // Note: In the Binary Format, this is SectionIDType. + // + // See https://www.w3.org/TR/2019/REC-wasm-core-1-20191205/#types%E2%91%A0%E2%91%A0 + TypeSection []FunctionType + + // ImportSection contains imported functions, tables, memories or globals required for instantiation + // (Store.Instantiate). + // + // Note: there are no unique constraints relating to the two-level namespace of Import.Module and Import.Name. + // + // Note: In the Binary Format, this is SectionIDImport. + // + // See https://www.w3.org/TR/2019/REC-wasm-core-1-20191205/#import-section%E2%91%A0 + ImportSection []Import + // ImportFunctionCount ImportGlobalCount ImportMemoryCount, and ImportTableCount are + // the cached import count per ExternType set during decoding. + ImportFunctionCount, + ImportGlobalCount, + ImportMemoryCount, + ImportTableCount Index + // ImportPerModule maps a module name to the list of Import to be imported from the module. + // This is used to do fast import resolution during instantiation. + ImportPerModule map[string][]*Import + + // FunctionSection contains the index in TypeSection of each function defined in this module. + // + // Note: The function Index space begins with imported functions and ends with those defined in this module. + // For example, if there are two imported functions and one defined in this module, the function Index 3 is defined + // in this module at FunctionSection[0]. + // + // Note: FunctionSection is index correlated with the CodeSection. If given the same position, e.g. 2, a function + // type is at TypeSection[FunctionSection[2]], while its locals and body are at CodeSection[2]. + // + // Note: In the Binary Format, this is SectionIDFunction. + // + // See https://www.w3.org/TR/2019/REC-wasm-core-1-20191205/#function-section%E2%91%A0 + FunctionSection []Index + + // TableSection contains each table defined in this module. + // + // Note: The table Index space begins with imported tables and ends with those defined in this module. + // For example, if there are two imported tables and one defined in this module, the table Index 3 is defined in + // this module at TableSection[0]. + // + // Note: Version 1.0 (20191205) of the WebAssembly spec allows at most one table definition per module, so the + // length of the TableSection can be zero or one, and can only be one if there is no imported table. + // + // Note: In the Binary Format, this is SectionIDTable. + // + // See https://www.w3.org/TR/2019/REC-wasm-core-1-20191205/#table-section%E2%91%A0 + TableSection []Table + + // MemorySection contains each memory defined in this module. + // + // Note: The memory Index space begins with imported memories and ends with those defined in this module. + // For example, if there are two imported memories and one defined in this module, the memory Index 3 is defined in + // this module at TableSection[0]. + // + // Note: Version 1.0 (20191205) of the WebAssembly spec allows at most one memory definition per module, so the + // length of the MemorySection can be zero or one, and can only be one if there is no imported memory. + // + // Note: In the Binary Format, this is SectionIDMemory. + // + // See https://www.w3.org/TR/2019/REC-wasm-core-1-20191205/#memory-section%E2%91%A0 + MemorySection *Memory + + // GlobalSection contains each global defined in this module. + // + // Global indexes are offset by any imported globals because the global index begins with imports, followed by + // ones defined in this module. For example, if there are two imported globals and three defined in this module, the + // global at index 3 is defined in this module at GlobalSection[0]. + // + // Note: In the Binary Format, this is SectionIDGlobal. + // + // See https://www.w3.org/TR/2019/REC-wasm-core-1-20191205/#global-section%E2%91%A0 + GlobalSection []Global + + // ExportSection contains each export defined in this module. + // + // Note: In the Binary Format, this is SectionIDExport. + // + // See https://www.w3.org/TR/2019/REC-wasm-core-1-20191205/#exports%E2%91%A0 + ExportSection []Export + // Exports maps a name to Export, and is convenient for fast look up of exported instances at runtime. + // Each item of this map points to an element of ExportSection. + Exports map[string]*Export + + // StartSection is the index of a function to call before returning from Store.Instantiate. + // + // Note: The index here is not the position in the FunctionSection, rather in the function index, which + // begins with imported functions. + // + // Note: In the Binary Format, this is SectionIDStart. + // + // See https://www.w3.org/TR/2019/REC-wasm-core-1-20191205/#start-section%E2%91%A0 + StartSection *Index + + // Note: In the Binary Format, this is SectionIDElement. + ElementSection []ElementSegment + + // CodeSection is index-correlated with FunctionSection and contains each + // function's locals and body. + // + // When present, the HostFunctionSection of the same index must be nil. + // + // Note: In the Binary Format, this is SectionIDCode. + // + // See https://www.w3.org/TR/2019/REC-wasm-core-1-20191205/#code-section%E2%91%A0 + CodeSection []Code + + // Note: In the Binary Format, this is SectionIDData. + DataSection []DataSegment + + // NameSection is set when the SectionIDCustom "name" was successfully decoded from the binary format. + // + // Note: This is the only SectionIDCustom defined in the WebAssembly 1.0 (20191205) Binary Format. + // Others are skipped as they are not used in wazero. + // + // See https://www.w3.org/TR/2019/REC-wasm-core-1-20191205/#name-section%E2%91%A0 + // See https://www.w3.org/TR/2019/REC-wasm-core-1-20191205/#custom-section%E2%91%A0 + NameSection *NameSection + + // CustomSections are set when the SectionIDCustom other than "name" were successfully decoded from the binary format. + // + // See https://www.w3.org/TR/2019/REC-wasm-core-1-20191205/#custom-section%E2%91%A0 + CustomSections []*CustomSection + + // DataCountSection is the optional section and holds the number of data segments in the data section. + // + // Note: This may exist in WebAssembly 2.0 or WebAssembly 1.0 with CoreFeatureBulkMemoryOperations. + // See https://www.w3.org/TR/2022/WD-wasm-core-2-20220419/binary/modules.html#data-count-section + // See https://www.w3.org/TR/2022/WD-wasm-core-2-20220419/appendix/changes.html#bulk-memory-and-table-instructions + DataCountSection *uint32 + + // ID is the sha256 value of the source wasm plus the configurations which affect the runtime representation of + // Wasm binary. This is only used for caching. + ID ModuleID + + // IsHostModule true if this is the host module, false otherwise. + IsHostModule bool + + // functionDefinitionSectionInitOnce guards FunctionDefinitionSection so that it is initialized exactly once. + functionDefinitionSectionInitOnce sync.Once + + // FunctionDefinitionSection is a wazero-specific section. + FunctionDefinitionSection []FunctionDefinition + + // MemoryDefinitionSection is a wazero-specific section. + MemoryDefinitionSection []MemoryDefinition + + // DWARFLines is used to emit DWARF based stack trace. This is created from the multiple custom sections + // as described in https://yurydelendik.github.io/webassembly-dwarf/, though it is not specified in the Wasm + // specification: https://github.com/WebAssembly/debugging/issues/1 + DWARFLines *wasmdebug.DWARFLines + + // NonStaticLocals collects the local indexes that will change its value through either local.get or local.tee. + NonStaticLocals []map[Index]struct{} +} + +// ModuleID represents sha256 hash value uniquely assigned to Module. +type ModuleID = [sha256.Size]byte + +// The wazero specific limitation described at RATIONALE.md. +// TL;DR; We multiply by 8 (to get offsets in bytes) and the multiplication result must be less than 32bit max +const ( + MaximumGlobals = uint32(1 << 27) + MaximumFunctionIndex = uint32(1 << 27) + MaximumTableIndex = uint32(1 << 27) +) + +// AssignModuleID calculates a sha256 checksum on `wasm` and other args, and set Module.ID to the result. +// See the doc on Module.ID on what it's used for. +func (m *Module) AssignModuleID(wasm []byte, listeners []experimental.FunctionListener, withEnsureTermination bool) { + h := sha256.New() + h.Write(wasm) + // Use the pre-allocated space backed by m.ID below. + + // Write the existence of listeners to the checksum per function. + for i, l := range listeners { + binary.LittleEndian.PutUint32(m.ID[:], uint32(i)) + m.ID[4] = boolToByte(l != nil) + h.Write(m.ID[:5]) + } + // Write the flag of ensureTermination to the checksum. + m.ID[0] = boolToByte(withEnsureTermination) + h.Write(m.ID[:1]) + // Get checksum by passing the slice underlying m.ID. + h.Sum(m.ID[:0]) +} + +func boolToByte(b bool) (ret byte) { + if b { + ret = 1 + } + return +} + +// typeOfFunction returns the wasm.FunctionType for the given function space index or nil. +func (m *Module) typeOfFunction(funcIdx Index) *FunctionType { + typeSectionLength, importedFunctionCount := uint32(len(m.TypeSection)), m.ImportFunctionCount + if funcIdx < importedFunctionCount { + // Imports are not exclusively functions. This is the current function index in the loop. + cur := Index(0) + for i := range m.ImportSection { + imp := &m.ImportSection[i] + if imp.Type != ExternTypeFunc { + continue + } + if funcIdx == cur { + if imp.DescFunc >= typeSectionLength { + return nil + } + return &m.TypeSection[imp.DescFunc] + } + cur++ + } + } + + funcSectionIdx := funcIdx - m.ImportFunctionCount + if funcSectionIdx >= uint32(len(m.FunctionSection)) { + return nil + } + typeIdx := m.FunctionSection[funcSectionIdx] + if typeIdx >= typeSectionLength { + return nil + } + return &m.TypeSection[typeIdx] +} + +func (m *Module) Validate(enabledFeatures api.CoreFeatures) error { + for i := range m.TypeSection { + tp := &m.TypeSection[i] + tp.CacheNumInUint64() + } + + if err := m.validateStartSection(); err != nil { + return err + } + + functions, globals, memory, tables, err := m.AllDeclarations() + if err != nil { + return err + } + + if err = m.validateImports(enabledFeatures); err != nil { + return err + } + + if err = m.validateGlobals(globals, uint32(len(functions)), MaximumGlobals); err != nil { + return err + } + + if err = m.validateMemory(memory, globals, enabledFeatures); err != nil { + return err + } + + if err = m.validateExports(enabledFeatures, functions, globals, memory, tables); err != nil { + return err + } + + if m.CodeSection != nil { + if err = m.validateFunctions(enabledFeatures, functions, globals, memory, tables, MaximumFunctionIndex); err != nil { + return err + } + } // No need to validate host functions as NewHostModule validates + + if err = m.validateTable(enabledFeatures, tables, MaximumTableIndex); err != nil { + return err + } + + if err = m.validateDataCountSection(); err != nil { + return err + } + return nil +} + +func (m *Module) validateStartSection() error { + // Check the start function is valid. + // TODO: this should be verified during decode so that errors have the correct source positions + if m.StartSection != nil { + startIndex := *m.StartSection + ft := m.typeOfFunction(startIndex) + if ft == nil { // TODO: move this check to decoder so that a module can never be decoded invalidly + return fmt.Errorf("invalid start function: func[%d] has an invalid type", startIndex) + } + if len(ft.Params) > 0 || len(ft.Results) > 0 { + return fmt.Errorf("invalid start function: func[%d] must have an empty (nullary) signature: %s", startIndex, ft) + } + } + return nil +} + +func (m *Module) validateGlobals(globals []GlobalType, numFuncts, maxGlobals uint32) error { + if uint32(len(globals)) > maxGlobals { + return fmt.Errorf("too many globals in a module") + } + + // Global initialization constant expression can only reference the imported globals. + // See the note on https://www.w3.org/TR/2019/REC-wasm-core-1-20191205/#constant-expressions%E2%91%A0 + importedGlobals := globals[:m.ImportGlobalCount] + for i := range m.GlobalSection { + g := &m.GlobalSection[i] + if err := validateConstExpression(importedGlobals, numFuncts, &g.Init, g.Type.ValType); err != nil { + return err + } + } + return nil +} + +func (m *Module) validateFunctions(enabledFeatures api.CoreFeatures, functions []Index, globals []GlobalType, memory *Memory, tables []Table, maximumFunctionIndex uint32) error { + if uint32(len(functions)) > maximumFunctionIndex { + return fmt.Errorf("too many functions (%d) in a module", len(functions)) + } + + functionCount := m.SectionElementCount(SectionIDFunction) + codeCount := m.SectionElementCount(SectionIDCode) + if functionCount == 0 && codeCount == 0 { + return nil + } + + typeCount := m.SectionElementCount(SectionIDType) + if codeCount != functionCount { + return fmt.Errorf("code count (%d) != function count (%d)", codeCount, functionCount) + } + + declaredFuncIndexes, err := m.declaredFunctionIndexes() + if err != nil { + return err + } + + // Create bytes.Reader once as it causes allocation, and + // we frequently need it (e.g. on every If instruction). + br := bytes.NewReader(nil) + // Also, we reuse the stacks across multiple function validations to reduce allocations. + vs := &stacks{} + // Non-static locals are gathered during validation and used in the down-stream compilation. + m.NonStaticLocals = make([]map[Index]struct{}, len(m.FunctionSection)) + for idx, typeIndex := range m.FunctionSection { + if typeIndex >= typeCount { + return fmt.Errorf("invalid %s: type section index %d out of range", m.funcDesc(SectionIDFunction, Index(idx)), typeIndex) + } + c := &m.CodeSection[idx] + if c.GoFunc != nil { + continue + } + if err = m.validateFunction(vs, enabledFeatures, Index(idx), functions, globals, memory, tables, declaredFuncIndexes, br); err != nil { + return fmt.Errorf("invalid %s: %w", m.funcDesc(SectionIDFunction, Index(idx)), err) + } + } + return nil +} + +// declaredFunctionIndexes returns a set of function indexes that can be used as an immediate for OpcodeRefFunc instruction. +// +// The criteria for which function indexes can be available for that instruction is vague in the spec: +// +// - "References: the list of function indices that occur in the module outside functions and can hence be used to form references inside them." +// - https://www.w3.org/TR/2022/WD-wasm-core-2-20220419/valid/conventions.html#contexts +// - "Ref is the set funcidx(module with functions=ε, start=ε) , i.e., the set of function indices occurring in the module, except in its functions or start function." +// - https://www.w3.org/TR/2022/WD-wasm-core-2-20220419/valid/modules.html#valid-module +// +// To clarify, we reverse-engineer logic required to pass the WebAssembly Core specification 2.0 test suite: +// https://github.com/WebAssembly/spec/blob/d39195773112a22b245ffbe864bab6d1182ccb06/test/core/ref_func.wast#L78-L115 +// +// To summarize, the function indexes OpcodeRefFunc can refer include: +// - existing in an element section regardless of its mode (active, passive, declarative). +// - defined as globals whose value type is ValueRefFunc. +// - used as an exported function. +// +// See https://github.com/WebAssembly/reference-types/issues/31 +// See https://github.com/WebAssembly/reference-types/issues/76 +func (m *Module) declaredFunctionIndexes() (ret map[Index]struct{}, err error) { + ret = map[uint32]struct{}{} + + for i := range m.ExportSection { + exp := &m.ExportSection[i] + if exp.Type == ExternTypeFunc { + ret[exp.Index] = struct{}{} + } + } + + for i := range m.GlobalSection { + g := &m.GlobalSection[i] + if g.Init.Opcode == OpcodeRefFunc { + var index uint32 + index, _, err = leb128.LoadUint32(g.Init.Data) + if err != nil { + err = fmt.Errorf("%s[%d] failed to initialize: %w", SectionIDName(SectionIDGlobal), i, err) + return + } + ret[index] = struct{}{} + } + } + + for i := range m.ElementSection { + elem := &m.ElementSection[i] + for _, index := range elem.Init { + if index != ElementInitNullReference { + ret[index] = struct{}{} + } + } + } + return +} + +func (m *Module) funcDesc(sectionID SectionID, sectionIndex Index) string { + // Try to improve the error message by collecting any exports: + var exportNames []string + funcIdx := sectionIndex + m.ImportFunctionCount + for i := range m.ExportSection { + exp := &m.ExportSection[i] + if exp.Index == funcIdx && exp.Type == ExternTypeFunc { + exportNames = append(exportNames, fmt.Sprintf("%q", exp.Name)) + } + } + sectionIDName := SectionIDName(sectionID) + if exportNames == nil { + return fmt.Sprintf("%s[%d]", sectionIDName, sectionIndex) + } + sort.Strings(exportNames) // go map keys do not iterate consistently + return fmt.Sprintf("%s[%d] export[%s]", sectionIDName, sectionIndex, strings.Join(exportNames, ",")) +} + +func (m *Module) validateMemory(memory *Memory, globals []GlobalType, _ api.CoreFeatures) error { + var activeElementCount int + for i := range m.DataSection { + d := &m.DataSection[i] + if !d.IsPassive() { + activeElementCount++ + } + } + if activeElementCount > 0 && memory == nil { + return fmt.Errorf("unknown memory") + } + + // Constant expression can only reference imported globals. + // https://github.com/WebAssembly/spec/blob/5900d839f38641989a9d8df2df4aee0513365d39/test/core/data.wast#L84-L91 + importedGlobals := globals[:m.ImportGlobalCount] + for i := range m.DataSection { + d := &m.DataSection[i] + if !d.IsPassive() { + if err := validateConstExpression(importedGlobals, 0, &d.OffsetExpression, ValueTypeI32); err != nil { + return fmt.Errorf("calculate offset: %w", err) + } + } + } + return nil +} + +func (m *Module) validateImports(enabledFeatures api.CoreFeatures) error { + for i := range m.ImportSection { + imp := &m.ImportSection[i] + if imp.Module == "" { + return fmt.Errorf("import[%d] has an empty module name", i) + } + switch imp.Type { + case ExternTypeFunc: + if int(imp.DescFunc) >= len(m.TypeSection) { + return fmt.Errorf("invalid import[%q.%q] function: type index out of range", imp.Module, imp.Name) + } + case ExternTypeGlobal: + if !imp.DescGlobal.Mutable { + continue + } + if err := enabledFeatures.RequireEnabled(api.CoreFeatureMutableGlobal); err != nil { + return fmt.Errorf("invalid import[%q.%q] global: %w", imp.Module, imp.Name, err) + } + } + } + return nil +} + +func (m *Module) validateExports(enabledFeatures api.CoreFeatures, functions []Index, globals []GlobalType, memory *Memory, tables []Table) error { + for i := range m.ExportSection { + exp := &m.ExportSection[i] + index := exp.Index + switch exp.Type { + case ExternTypeFunc: + if index >= uint32(len(functions)) { + return fmt.Errorf("unknown function for export[%q]", exp.Name) + } + case ExternTypeGlobal: + if index >= uint32(len(globals)) { + return fmt.Errorf("unknown global for export[%q]", exp.Name) + } + if !globals[index].Mutable { + continue + } + if err := enabledFeatures.RequireEnabled(api.CoreFeatureMutableGlobal); err != nil { + return fmt.Errorf("invalid export[%q] global[%d]: %w", exp.Name, index, err) + } + case ExternTypeMemory: + if index > 0 || memory == nil { + return fmt.Errorf("memory for export[%q] out of range", exp.Name) + } + case ExternTypeTable: + if index >= uint32(len(tables)) { + return fmt.Errorf("table for export[%q] out of range", exp.Name) + } + } + } + return nil +} + +func validateConstExpression(globals []GlobalType, numFuncs uint32, expr *ConstantExpression, expectedType ValueType) (err error) { + var actualType ValueType + switch expr.Opcode { + case OpcodeI32Const: + // Treat constants as signed as their interpretation is not yet known per /RATIONALE.md + _, _, err = leb128.LoadInt32(expr.Data) + if err != nil { + return fmt.Errorf("read i32: %w", err) + } + actualType = ValueTypeI32 + case OpcodeI64Const: + // Treat constants as signed as their interpretation is not yet known per /RATIONALE.md + _, _, err = leb128.LoadInt64(expr.Data) + if err != nil { + return fmt.Errorf("read i64: %w", err) + } + actualType = ValueTypeI64 + case OpcodeF32Const: + _, err = ieee754.DecodeFloat32(expr.Data) + if err != nil { + return fmt.Errorf("read f32: %w", err) + } + actualType = ValueTypeF32 + case OpcodeF64Const: + _, err = ieee754.DecodeFloat64(expr.Data) + if err != nil { + return fmt.Errorf("read f64: %w", err) + } + actualType = ValueTypeF64 + case OpcodeGlobalGet: + id, _, err := leb128.LoadUint32(expr.Data) + if err != nil { + return fmt.Errorf("read index of global: %w", err) + } + if uint32(len(globals)) <= id { + return fmt.Errorf("global index out of range") + } + actualType = globals[id].ValType + case OpcodeRefNull: + if len(expr.Data) == 0 { + return fmt.Errorf("read reference type for ref.null: %w", io.ErrShortBuffer) + } + reftype := expr.Data[0] + if reftype != RefTypeFuncref && reftype != RefTypeExternref { + return fmt.Errorf("invalid type for ref.null: 0x%x", reftype) + } + actualType = reftype + case OpcodeRefFunc: + index, _, err := leb128.LoadUint32(expr.Data) + if err != nil { + return fmt.Errorf("read i32: %w", err) + } else if index >= numFuncs { + return fmt.Errorf("ref.func index out of range [%d] with length %d", index, numFuncs-1) + } + actualType = ValueTypeFuncref + case OpcodeVecV128Const: + if len(expr.Data) != 16 { + return fmt.Errorf("%s needs 16 bytes but was %d bytes", OpcodeVecV128ConstName, len(expr.Data)) + } + actualType = ValueTypeV128 + default: + return fmt.Errorf("invalid opcode for const expression: 0x%x", expr.Opcode) + } + + if actualType != expectedType { + return fmt.Errorf("const expression type mismatch expected %s but got %s", + ValueTypeName(expectedType), ValueTypeName(actualType)) + } + return nil +} + +func (m *Module) validateDataCountSection() (err error) { + if m.DataCountSection != nil && int(*m.DataCountSection) != len(m.DataSection) { + err = fmt.Errorf("data count section (%d) doesn't match the length of data section (%d)", + *m.DataCountSection, len(m.DataSection)) + } + return +} + +func (m *ModuleInstance) buildGlobals(module *Module, funcRefResolver func(funcIndex Index) Reference) { + importedGlobals := m.Globals[:module.ImportGlobalCount] + + me := m.Engine + engineOwnGlobal := me.OwnsGlobals() + for i := Index(0); i < Index(len(module.GlobalSection)); i++ { + gs := &module.GlobalSection[i] + g := &GlobalInstance{} + if engineOwnGlobal { + g.Me = me + g.Index = i + module.ImportGlobalCount + } + m.Globals[i+module.ImportGlobalCount] = g + g.Type = gs.Type + g.initialize(importedGlobals, &gs.Init, funcRefResolver) + } +} + +func paramNames(localNames IndirectNameMap, funcIdx uint32, paramLen int) []string { + for i := range localNames { + nm := &localNames[i] + // Only build parameter names if we have one for each. + if nm.Index != funcIdx || len(nm.NameMap) < paramLen { + continue + } + + ret := make([]string, paramLen) + for j := range nm.NameMap { + p := &nm.NameMap[j] + if int(p.Index) < paramLen { + ret[p.Index] = p.Name + } + } + return ret + } + return nil +} + +func (m *ModuleInstance) buildMemory(module *Module, allocator experimental.MemoryAllocator) { + memSec := module.MemorySection + if memSec != nil { + m.MemoryInstance = NewMemoryInstance(memSec, allocator) + m.MemoryInstance.definition = &module.MemoryDefinitionSection[0] + } +} + +// Index is the offset in an index, not necessarily an absolute position in a Module section. This is because +// indexs are often preceded by a corresponding type in the Module.ImportSection. +// +// For example, the function index starts with any ExternTypeFunc in the Module.ImportSection followed by +// the Module.FunctionSection +// +// See https://www.w3.org/TR/2019/REC-wasm-core-1-20191205/#binary-index +type Index = uint32 + +// FunctionType is a possibly empty function signature. +// +// See https://www.w3.org/TR/2019/REC-wasm-core-1-20191205/#function-types%E2%91%A0 +type FunctionType struct { + // Params are the possibly empty sequence of value types accepted by a function with this signature. + Params []ValueType + + // Results are the possibly empty sequence of value types returned by a function with this signature. + // + // Note: In WebAssembly 1.0 (20191205), there can be at most one result. + // See https://www.w3.org/TR/2019/REC-wasm-core-1-20191205/#result-types%E2%91%A0 + Results []ValueType + + // string is cached as it is used both for String and key + string string + + // ParamNumInUint64 is the number of uint64 values requires to represent the Wasm param type. + ParamNumInUint64 int + + // ResultsNumInUint64 is the number of uint64 values requires to represent the Wasm result type. + ResultNumInUint64 int +} + +func (f *FunctionType) CacheNumInUint64() { + if f.ParamNumInUint64 == 0 { + for _, tp := range f.Params { + f.ParamNumInUint64++ + if tp == ValueTypeV128 { + f.ParamNumInUint64++ + } + } + } + + if f.ResultNumInUint64 == 0 { + for _, tp := range f.Results { + f.ResultNumInUint64++ + if tp == ValueTypeV128 { + f.ResultNumInUint64++ + } + } + } +} + +// EqualsSignature returns true if the function type has the same parameters and results. +func (f *FunctionType) EqualsSignature(params []ValueType, results []ValueType) bool { + return bytes.Equal(f.Params, params) && bytes.Equal(f.Results, results) +} + +// key gets or generates the key for Store.typeIDs. e.g. "i32_v" for one i32 parameter and no (void) result. +func (f *FunctionType) key() string { + if f.string != "" { + return f.string + } + var ret string + for _, b := range f.Params { + ret += ValueTypeName(b) + } + if len(f.Params) == 0 { + ret += "v_" + } else { + ret += "_" + } + for _, b := range f.Results { + ret += ValueTypeName(b) + } + if len(f.Results) == 0 { + ret += "v" + } + f.string = ret + return ret +} + +// String implements fmt.Stringer. +func (f *FunctionType) String() string { + return f.key() +} + +// Import is the binary representation of an import indicated by Type +// See https://www.w3.org/TR/2019/REC-wasm-core-1-20191205/#binary-import +type Import struct { + Type ExternType + // Module is the possibly empty primary namespace of this import + Module string + // Module is the possibly empty secondary namespace of this import + Name string + // DescFunc is the index in Module.TypeSection when Type equals ExternTypeFunc + DescFunc Index + // DescTable is the inlined Table when Type equals ExternTypeTable + DescTable Table + // DescMem is the inlined Memory when Type equals ExternTypeMemory + DescMem *Memory + // DescGlobal is the inlined GlobalType when Type equals ExternTypeGlobal + DescGlobal GlobalType + // IndexPerType has the index of this import per ExternType. + IndexPerType Index +} + +// Memory describes the limits of pages (64KB) in a memory. +type Memory struct { + Min, Cap, Max uint32 + // IsMaxEncoded true if the Max is encoded in the original binary. + IsMaxEncoded bool + // IsShared true if the memory is shared for access from multiple agents. + IsShared bool +} + +// Validate ensures values assigned to Min, Cap and Max are within valid thresholds. +func (m *Memory) Validate(memoryLimitPages uint32) error { + min, capacity, max := m.Min, m.Cap, m.Max + + if max > memoryLimitPages { + return fmt.Errorf("max %d pages (%s) over limit of %d pages (%s)", + max, PagesToUnitOfBytes(max), memoryLimitPages, PagesToUnitOfBytes(memoryLimitPages)) + } else if min > memoryLimitPages { + return fmt.Errorf("min %d pages (%s) over limit of %d pages (%s)", + min, PagesToUnitOfBytes(min), memoryLimitPages, PagesToUnitOfBytes(memoryLimitPages)) + } else if min > max { + return fmt.Errorf("min %d pages (%s) > max %d pages (%s)", + min, PagesToUnitOfBytes(min), max, PagesToUnitOfBytes(max)) + } else if capacity < min { + return fmt.Errorf("capacity %d pages (%s) less than minimum %d pages (%s)", + capacity, PagesToUnitOfBytes(capacity), min, PagesToUnitOfBytes(min)) + } else if capacity > memoryLimitPages { + return fmt.Errorf("capacity %d pages (%s) over limit of %d pages (%s)", + capacity, PagesToUnitOfBytes(capacity), memoryLimitPages, PagesToUnitOfBytes(memoryLimitPages)) + } + return nil +} + +type GlobalType struct { + ValType ValueType + Mutable bool +} + +type Global struct { + Type GlobalType + Init ConstantExpression +} + +type ConstantExpression struct { + Opcode Opcode + Data []byte +} + +// Export is the binary representation of an export indicated by Type +// See https://www.w3.org/TR/2019/REC-wasm-core-1-20191205/#binary-export +type Export struct { + Type ExternType + + // Name is what the host refers to this definition as. + Name string + + // Index is the index of the definition to export, the index is by Type + // e.g. If ExternTypeFunc, this is a position in the function index. + Index Index +} + +// Code is an entry in the Module.CodeSection containing the locals and body of the function. +// See https://www.w3.org/TR/2019/REC-wasm-core-1-20191205/#binary-code +type Code struct { + // LocalTypes are any function-scoped variables in insertion order. + // See https://www.w3.org/TR/2019/REC-wasm-core-1-20191205/#binary-local + LocalTypes []ValueType + + // Body is a sequence of expressions ending in OpcodeEnd + // See https://www.w3.org/TR/2019/REC-wasm-core-1-20191205/#binary-expr + Body []byte + + // GoFunc is non-nil when IsHostFunction and defined in go, either + // api.GoFunction or api.GoModuleFunction. When present, LocalTypes and Body must + // be nil. + // + // Note: This has no serialization format, so is not encodable. + // See https://www.w3.org/TR/2019/REC-wasm-core-1-20191205/#host-functions%E2%91%A2 + GoFunc interface{} + + // BodyOffsetInCodeSection is the offset of the beginning of the body in the code section. + // This is used for DWARF based stack trace where a program counter represents an offset in code section. + BodyOffsetInCodeSection uint64 +} + +type DataSegment struct { + OffsetExpression ConstantExpression + Init []byte + Passive bool +} + +// IsPassive returns true if this data segment is "passive" in the sense that memory offset and +// index is determined at runtime and used by OpcodeMemoryInitName instruction in the bulk memory +// operations proposal. +// +// See https://www.w3.org/TR/2022/WD-wasm-core-2-20220419/appendix/changes.html#bulk-memory-and-table-instructions +func (d *DataSegment) IsPassive() bool { + return d.Passive +} + +// NameSection represent the known custom name subsections defined in the WebAssembly Binary Format +// +// Note: This can be nil if no names were decoded for any reason including configuration. +// See https://www.w3.org/TR/2019/REC-wasm-core-1-20191205/#name-section%E2%91%A0 +type NameSection struct { + // ModuleName is the symbolic identifier for a module. e.g. math + // + // Note: This can be empty for any reason including configuration. + ModuleName string + + // FunctionNames is an association of a function index to its symbolic identifier. e.g. add + // + // * the key (idx) is in the function index, where module defined functions are preceded by imported ones. + // See https://www.w3.org/TR/2019/REC-wasm-core-1-20191205/#functions%E2%91%A7 + // + // For example, assuming the below text format is the second import, you would expect FunctionNames[1] = "mul" + // (import "Math" "Mul" (func $mul (param $x f32) (param $y f32) (result f32))) + // + // Note: FunctionNames are only used for debugging. At runtime, functions are called based on raw numeric index. + // Note: This can be nil for any reason including configuration. + FunctionNames NameMap + + // LocalNames contains symbolic names for function parameters or locals that have one. + // + // Note: In the Text Format, function local names can inherit parameter + // names from their type. Here are some examples: + // * (module (import (func (param $x i32) (param i32))) (func (type 0))) = [{0, {x,0}}] + // * (module (import (func (param i32) (param $y i32))) (func (type 0) (local $z i32))) = [0, [{y,1},{z,2}]] + // * (module (func (param $x i32) (local $y i32) (local $z i32))) = [{x,0},{y,1},{z,2}] + // + // Note: LocalNames are only used for debugging. At runtime, locals are called based on raw numeric index. + // Note: This can be nil for any reason including configuration. + LocalNames IndirectNameMap + + // ResultNames is a wazero-specific mechanism to store result names. + ResultNames IndirectNameMap +} + +// CustomSection contains the name and raw data of a custom section. +type CustomSection struct { + Name string + Data []byte +} + +// NameMap associates an index with any associated names. +// +// Note: Often the index bridges multiple sections. For example, the function index starts with any +// ExternTypeFunc in the Module.ImportSection followed by the Module.FunctionSection +// +// Note: NameMap is unique by NameAssoc.Index, but NameAssoc.Name needn't be unique. +// Note: When encoding in the Binary format, this must be ordered by NameAssoc.Index +// See https://www.w3.org/TR/2019/REC-wasm-core-1-20191205/#binary-namemap +type NameMap []NameAssoc + +type NameAssoc struct { + Index Index + Name string +} + +// IndirectNameMap associates an index with an association of names. +// +// Note: IndirectNameMap is unique by NameMapAssoc.Index, but NameMapAssoc.NameMap needn't be unique. +// Note: When encoding in the Binary format, this must be ordered by NameMapAssoc.Index +// https://www.w3.org/TR/2019/REC-wasm-core-1-20191205/#binary-indirectnamemap +type IndirectNameMap []NameMapAssoc + +type NameMapAssoc struct { + Index Index + NameMap NameMap +} + +// AllDeclarations returns all declarations for functions, globals, memories and tables in a module including imported ones. +func (m *Module) AllDeclarations() (functions []Index, globals []GlobalType, memory *Memory, tables []Table, err error) { + for i := range m.ImportSection { + imp := &m.ImportSection[i] + switch imp.Type { + case ExternTypeFunc: + functions = append(functions, imp.DescFunc) + case ExternTypeGlobal: + globals = append(globals, imp.DescGlobal) + case ExternTypeMemory: + memory = imp.DescMem + case ExternTypeTable: + tables = append(tables, imp.DescTable) + } + } + + functions = append(functions, m.FunctionSection...) + for i := range m.GlobalSection { + g := &m.GlobalSection[i] + globals = append(globals, g.Type) + } + if m.MemorySection != nil { + if memory != nil { // shouldn't be possible due to Validate + err = errors.New("at most one table allowed in module") + return + } + memory = m.MemorySection + } + if m.TableSection != nil { + tables = append(tables, m.TableSection...) + } + return +} + +// SectionID identifies the sections of a Module in the WebAssembly 1.0 (20191205) Binary Format. +// +// Note: these are defined in the wasm package, instead of the binary package, as a key per section is needed regardless +// of format, and deferring to the binary type avoids confusion. +// +// See https://www.w3.org/TR/2019/REC-wasm-core-1-20191205/#sections%E2%91%A0 +type SectionID = byte + +const ( + // SectionIDCustom includes the standard defined NameSection and possibly others not defined in the standard. + SectionIDCustom SectionID = iota // don't add anything not in https://www.w3.org/TR/2019/REC-wasm-core-1-20191205/#sections%E2%91%A0 + SectionIDType + SectionIDImport + SectionIDFunction + SectionIDTable + SectionIDMemory + SectionIDGlobal + SectionIDExport + SectionIDStart + SectionIDElement + SectionIDCode + SectionIDData + + // SectionIDDataCount may exist in WebAssembly 2.0 or WebAssembly 1.0 with CoreFeatureBulkMemoryOperations enabled. + // + // See https://www.w3.org/TR/2022/WD-wasm-core-2-20220419/binary/modules.html#data-count-section + // See https://www.w3.org/TR/2022/WD-wasm-core-2-20220419/appendix/changes.html#bulk-memory-and-table-instructions + SectionIDDataCount +) + +// SectionIDName returns the canonical name of a module section. +// https://www.w3.org/TR/2019/REC-wasm-core-1-20191205/#sections%E2%91%A0 +func SectionIDName(sectionID SectionID) string { + switch sectionID { + case SectionIDCustom: + return "custom" + case SectionIDType: + return "type" + case SectionIDImport: + return "import" + case SectionIDFunction: + return "function" + case SectionIDTable: + return "table" + case SectionIDMemory: + return "memory" + case SectionIDGlobal: + return "global" + case SectionIDExport: + return "export" + case SectionIDStart: + return "start" + case SectionIDElement: + return "element" + case SectionIDCode: + return "code" + case SectionIDData: + return "data" + case SectionIDDataCount: + return "data_count" + } + return "unknown" +} + +// ValueType is an alias of api.ValueType defined to simplify imports. +type ValueType = api.ValueType + +const ( + ValueTypeI32 = api.ValueTypeI32 + ValueTypeI64 = api.ValueTypeI64 + ValueTypeF32 = api.ValueTypeF32 + ValueTypeF64 = api.ValueTypeF64 + // TODO: ValueTypeV128 is not exposed in the api pkg yet. + ValueTypeV128 ValueType = 0x7b + // TODO: ValueTypeFuncref is not exposed in the api pkg yet. + ValueTypeFuncref ValueType = 0x70 + ValueTypeExternref = api.ValueTypeExternref +) + +// ValueTypeName is an alias of api.ValueTypeName defined to simplify imports. +func ValueTypeName(t ValueType) string { + if t == ValueTypeFuncref { + return "funcref" + } else if t == ValueTypeV128 { + return "v128" + } + return api.ValueTypeName(t) +} + +func isReferenceValueType(vt ValueType) bool { + return vt == ValueTypeExternref || vt == ValueTypeFuncref +} + +// ExternType is an alias of api.ExternType defined to simplify imports. +type ExternType = api.ExternType + +const ( + ExternTypeFunc = api.ExternTypeFunc + ExternTypeFuncName = api.ExternTypeFuncName + ExternTypeTable = api.ExternTypeTable + ExternTypeTableName = api.ExternTypeTableName + ExternTypeMemory = api.ExternTypeMemory + ExternTypeMemoryName = api.ExternTypeMemoryName + ExternTypeGlobal = api.ExternTypeGlobal + ExternTypeGlobalName = api.ExternTypeGlobalName +) + +// ExternTypeName is an alias of api.ExternTypeName defined to simplify imports. +func ExternTypeName(t ValueType) string { + return api.ExternTypeName(t) +} diff --git a/vendor/github.com/tetratelabs/wazero/internal/wasm/module_instance.go b/vendor/github.com/tetratelabs/wazero/internal/wasm/module_instance.go new file mode 100644 index 000000000..20c733e6f --- /dev/null +++ b/vendor/github.com/tetratelabs/wazero/internal/wasm/module_instance.go @@ -0,0 +1,251 @@ +package wasm + +import ( + "context" + "errors" + "fmt" + + "github.com/tetratelabs/wazero/api" + "github.com/tetratelabs/wazero/sys" +) + +// FailIfClosed returns a sys.ExitError if CloseWithExitCode was called. +func (m *ModuleInstance) FailIfClosed() (err error) { + if closed := m.Closed.Load(); closed != 0 { + switch closed & exitCodeFlagMask { + case exitCodeFlagResourceClosed: + case exitCodeFlagResourceNotClosed: + // This happens when this module is closed asynchronously in CloseModuleOnCanceledOrTimeout, + // and the closure of resources have been deferred here. + _ = m.ensureResourcesClosed(context.Background()) + } + return sys.NewExitError(uint32(closed >> 32)) // Unpack the high order bits as the exit code. + } + return nil +} + +// CloseModuleOnCanceledOrTimeout take a context `ctx`, which might be a Cancel or Timeout context, +// and spawns the Goroutine to check the context is canceled ot deadline exceeded. If it reaches +// one of the conditions, it sets the appropriate exit code. +// +// Callers of this function must invoke the returned context.CancelFunc to release the spawned Goroutine. +func (m *ModuleInstance) CloseModuleOnCanceledOrTimeout(ctx context.Context) context.CancelFunc { + // Creating an empty channel in this case is a bit more efficient than + // creating a context.Context and canceling it with the same effect. We + // really just need to be notified when to stop listening to the users + // context. Closing the channel will unblock the select in the goroutine + // causing it to return an stop listening to ctx.Done(). + cancelChan := make(chan struct{}) + go m.closeModuleOnCanceledOrTimeout(ctx, cancelChan) + return func() { close(cancelChan) } +} + +// closeModuleOnCanceledOrTimeout is extracted from CloseModuleOnCanceledOrTimeout for testing. +func (m *ModuleInstance) closeModuleOnCanceledOrTimeout(ctx context.Context, cancelChan <-chan struct{}) { + select { + case <-ctx.Done(): + select { + case <-cancelChan: + // In some cases by the time this goroutine is scheduled, the caller + // has already closed both the context and the cancelChan. In this + // case go will randomize which branch of the outer select to enter + // and we don't want to close the module. + default: + // This is the same logic as CloseWithCtxErr except this calls closeWithExitCodeWithoutClosingResource + // so that we can defer the resource closure in FailIfClosed. + switch { + case errors.Is(ctx.Err(), context.Canceled): + // TODO: figure out how to report error here. + _ = m.closeWithExitCodeWithoutClosingResource(sys.ExitCodeContextCanceled) + case errors.Is(ctx.Err(), context.DeadlineExceeded): + // TODO: figure out how to report error here. + _ = m.closeWithExitCodeWithoutClosingResource(sys.ExitCodeDeadlineExceeded) + } + } + case <-cancelChan: + } +} + +// CloseWithCtxErr closes the module with an exit code based on the type of +// error reported by the context. +// +// If the context's error is unknown or nil, the module does not close. +func (m *ModuleInstance) CloseWithCtxErr(ctx context.Context) { + switch { + case errors.Is(ctx.Err(), context.Canceled): + // TODO: figure out how to report error here. + _ = m.CloseWithExitCode(ctx, sys.ExitCodeContextCanceled) + case errors.Is(ctx.Err(), context.DeadlineExceeded): + // TODO: figure out how to report error here. + _ = m.CloseWithExitCode(ctx, sys.ExitCodeDeadlineExceeded) + } +} + +// Name implements the same method as documented on api.Module +func (m *ModuleInstance) Name() string { + return m.ModuleName +} + +// String implements the same method as documented on api.Module +func (m *ModuleInstance) String() string { + return fmt.Sprintf("Module[%s]", m.Name()) +} + +// Close implements the same method as documented on api.Module. +func (m *ModuleInstance) Close(ctx context.Context) (err error) { + return m.CloseWithExitCode(ctx, 0) +} + +// CloseWithExitCode implements the same method as documented on api.Module. +func (m *ModuleInstance) CloseWithExitCode(ctx context.Context, exitCode uint32) (err error) { + if !m.setExitCode(exitCode, exitCodeFlagResourceClosed) { + return nil // not an error to have already closed + } + _ = m.s.deleteModule(m) + return m.ensureResourcesClosed(ctx) +} + +// IsClosed implements the same method as documented on api.Module. +func (m *ModuleInstance) IsClosed() bool { + return m.Closed.Load() != 0 +} + +func (m *ModuleInstance) closeWithExitCodeWithoutClosingResource(exitCode uint32) (err error) { + if !m.setExitCode(exitCode, exitCodeFlagResourceNotClosed) { + return nil // not an error to have already closed + } + _ = m.s.deleteModule(m) + return nil +} + +// closeWithExitCode is the same as CloseWithExitCode besides this doesn't delete it from Store.moduleList. +func (m *ModuleInstance) closeWithExitCode(ctx context.Context, exitCode uint32) (err error) { + if !m.setExitCode(exitCode, exitCodeFlagResourceClosed) { + return nil // not an error to have already closed + } + return m.ensureResourcesClosed(ctx) +} + +type exitCodeFlag = uint64 + +const exitCodeFlagMask = 0xff + +const ( + // exitCodeFlagResourceClosed indicates that the module was closed and resources were already closed. + exitCodeFlagResourceClosed = 1 << iota + // exitCodeFlagResourceNotClosed indicates that the module was closed while resources are not closed yet. + exitCodeFlagResourceNotClosed +) + +func (m *ModuleInstance) setExitCode(exitCode uint32, flag exitCodeFlag) bool { + closed := flag | uint64(exitCode)<<32 // Store exitCode as high-order bits. + return m.Closed.CompareAndSwap(0, closed) +} + +// ensureResourcesClosed ensures that resources assigned to ModuleInstance is released. +// Only one call will happen per module, due to external atomic guards on Closed. +func (m *ModuleInstance) ensureResourcesClosed(ctx context.Context) (err error) { + if closeNotifier := m.CloseNotifier; closeNotifier != nil { // experimental + closeNotifier.CloseNotify(ctx, uint32(m.Closed.Load()>>32)) + m.CloseNotifier = nil + } + + if sysCtx := m.Sys; sysCtx != nil { // nil if from HostModuleBuilder + err = sysCtx.FS().Close() + m.Sys = nil + } + + if mem := m.MemoryInstance; mem != nil { + if mem.expBuffer != nil { + mem.expBuffer.Free() + mem.expBuffer = nil + } + } + + if m.CodeCloser != nil { + if e := m.CodeCloser.Close(ctx); err == nil { + err = e + } + m.CodeCloser = nil + } + return err +} + +// Memory implements the same method as documented on api.Module. +func (m *ModuleInstance) Memory() api.Memory { + return m.MemoryInstance +} + +// ExportedMemory implements the same method as documented on api.Module. +func (m *ModuleInstance) ExportedMemory(name string) api.Memory { + _, err := m.getExport(name, ExternTypeMemory) + if err != nil { + return nil + } + // We Assume that we have at most one memory. + return m.MemoryInstance +} + +// ExportedMemoryDefinitions implements the same method as documented on +// api.Module. +func (m *ModuleInstance) ExportedMemoryDefinitions() map[string]api.MemoryDefinition { + // Special case as we currently only support one memory. + if mem := m.MemoryInstance; mem != nil { + // Now, find out if it is exported + for name, exp := range m.Exports { + if exp.Type == ExternTypeMemory { + return map[string]api.MemoryDefinition{name: mem.definition} + } + } + } + return map[string]api.MemoryDefinition{} +} + +// ExportedFunction implements the same method as documented on api.Module. +func (m *ModuleInstance) ExportedFunction(name string) api.Function { + exp, err := m.getExport(name, ExternTypeFunc) + if err != nil { + return nil + } + return m.Engine.NewFunction(exp.Index) +} + +// ExportedFunctionDefinitions implements the same method as documented on +// api.Module. +func (m *ModuleInstance) ExportedFunctionDefinitions() map[string]api.FunctionDefinition { + result := map[string]api.FunctionDefinition{} + for name, exp := range m.Exports { + if exp.Type == ExternTypeFunc { + result[name] = m.Source.FunctionDefinition(exp.Index) + } + } + return result +} + +// GlobalVal is an internal hack to get the lower 64 bits of a global. +func (m *ModuleInstance) GlobalVal(idx Index) uint64 { + return m.Globals[idx].Val +} + +// ExportedGlobal implements the same method as documented on api.Module. +func (m *ModuleInstance) ExportedGlobal(name string) api.Global { + exp, err := m.getExport(name, ExternTypeGlobal) + if err != nil { + return nil + } + g := m.Globals[exp.Index] + if g.Type.Mutable { + return mutableGlobal{g: g} + } + return constantGlobal{g: g} +} + +// NumGlobal implements experimental.InternalModule. +func (m *ModuleInstance) NumGlobal() int { + return len(m.Globals) +} + +// Global implements experimental.InternalModule. +func (m *ModuleInstance) Global(idx int) api.Global { + return constantGlobal{g: m.Globals[idx]} +} diff --git a/vendor/github.com/tetratelabs/wazero/internal/wasm/module_instance_lookup.go b/vendor/github.com/tetratelabs/wazero/internal/wasm/module_instance_lookup.go new file mode 100644 index 000000000..442d26a22 --- /dev/null +++ b/vendor/github.com/tetratelabs/wazero/internal/wasm/module_instance_lookup.go @@ -0,0 +1,73 @@ +package wasm + +import ( + "context" + "fmt" + + "github.com/tetratelabs/wazero/api" + "github.com/tetratelabs/wazero/internal/internalapi" +) + +// LookupFunction looks up the table by the given index, and returns the api.Function implementation if found, +// otherwise this panics according to the same semantics as call_indirect instruction. +// Currently, this is only used by emscripten which needs to do call_indirect-like operation in the host function. +func (m *ModuleInstance) LookupFunction(t *TableInstance, typeId FunctionTypeID, tableOffset Index) api.Function { + fm, index := m.Engine.LookupFunction(t, typeId, tableOffset) + if source := fm.Source; source.IsHostModule { + // This case, the found function is a host function stored in the table. Generally, Engine.NewFunction are only + // responsible for calling Wasm-defined functions (not designed for calling Go functions!). Hence we need to wrap + // the host function as a special case. + def := &source.FunctionDefinitionSection[index] + goF := source.CodeSection[index].GoFunc + switch typed := goF.(type) { + case api.GoFunction: + // GoFunction doesn't need looked up module. + return &lookedUpGoFunction{def: def, g: goFunctionAsGoModuleFunction(typed)} + case api.GoModuleFunction: + return &lookedUpGoFunction{def: def, lookedUpModule: m, g: typed} + default: + panic(fmt.Sprintf("unexpected GoFunc type: %T", goF)) + } + } else { + return fm.Engine.NewFunction(index) + } +} + +// lookedUpGoFunction implements lookedUpGoModuleFunction. +type lookedUpGoFunction struct { + internalapi.WazeroOnly + def *FunctionDefinition + // lookedUpModule is the *ModuleInstance from which this Go function is looked up, i.e. owner of the table. + lookedUpModule *ModuleInstance + g api.GoModuleFunction +} + +// goFunctionAsGoModuleFunction converts api.GoFunction to api.GoModuleFunction which ignores the api.Module argument. +func goFunctionAsGoModuleFunction(g api.GoFunction) api.GoModuleFunction { + return api.GoModuleFunc(func(ctx context.Context, _ api.Module, stack []uint64) { + g.Call(ctx, stack) + }) +} + +// Definition implements api.Function. +func (l *lookedUpGoFunction) Definition() api.FunctionDefinition { return l.def } + +// Call implements api.Function. +func (l *lookedUpGoFunction) Call(ctx context.Context, params ...uint64) ([]uint64, error) { + typ := l.def.Functype + stackSize := typ.ParamNumInUint64 + rn := typ.ResultNumInUint64 + if rn > stackSize { + stackSize = rn + } + stack := make([]uint64, stackSize) + copy(stack, params) + return stack[:rn], l.CallWithStack(ctx, stack) +} + +// CallWithStack implements api.Function. +func (l *lookedUpGoFunction) CallWithStack(ctx context.Context, stack []uint64) error { + // The Go host function always needs to access caller's module, in this case the one holding the table. + l.g.Call(ctx, l.lookedUpModule, stack) + return nil +} diff --git a/vendor/github.com/tetratelabs/wazero/internal/wasm/store.go b/vendor/github.com/tetratelabs/wazero/internal/wasm/store.go new file mode 100644 index 000000000..1db661e85 --- /dev/null +++ b/vendor/github.com/tetratelabs/wazero/internal/wasm/store.go @@ -0,0 +1,668 @@ +package wasm + +import ( + "context" + "encoding/binary" + "fmt" + "sync" + "sync/atomic" + + "github.com/tetratelabs/wazero/api" + "github.com/tetratelabs/wazero/experimental" + "github.com/tetratelabs/wazero/internal/expctxkeys" + "github.com/tetratelabs/wazero/internal/internalapi" + "github.com/tetratelabs/wazero/internal/leb128" + internalsys "github.com/tetratelabs/wazero/internal/sys" + "github.com/tetratelabs/wazero/sys" +) + +// nameToModuleShrinkThreshold is the size the nameToModule map can grow to +// before it starts to be monitored for shrinking. +// The capacity will never be smaller than this once the threshold is met. +const nameToModuleShrinkThreshold = 100 + +type ( + // Store is the runtime representation of "instantiated" Wasm module and objects. + // Multiple modules can be instantiated within a single store, and each instance, + // (e.g. function instance) can be referenced by other module instances in a Store via Module.ImportSection. + // + // Every type whose name ends with "Instance" suffix belongs to exactly one store. + // + // Note that store is not thread (concurrency) safe, meaning that using single Store + // via multiple goroutines might result in race conditions. In that case, the invocation + // and access to any methods and field of Store must be guarded by mutex. + // + // See https://www.w3.org/TR/2019/REC-wasm-core-1-20191205/#store%E2%91%A0 + Store struct { + // moduleList ensures modules are closed in reverse initialization order. + moduleList *ModuleInstance // guarded by mux + + // nameToModule holds the instantiated Wasm modules by module name from Instantiate. + // It ensures no race conditions instantiating two modules of the same name. + nameToModule map[string]*ModuleInstance // guarded by mux + + // nameToModuleCap tracks the growth of the nameToModule map in order to + // track when to shrink it. + nameToModuleCap int // guarded by mux + + // EnabledFeatures are read-only to allow optimizations. + EnabledFeatures api.CoreFeatures + + // Engine is a global context for a Store which is in responsible for compilation and execution of Wasm modules. + Engine Engine + + // typeIDs maps each FunctionType.String() to a unique FunctionTypeID. This is used at runtime to + // do type-checks on indirect function calls. + typeIDs map[string]FunctionTypeID + + // functionMaxTypes represents the limit on the number of function types in a store. + // Note: this is fixed to 2^27 but have this a field for testability. + functionMaxTypes uint32 + + // mux is used to guard the fields from concurrent access. + mux sync.RWMutex + } + + // ModuleInstance represents instantiated wasm module. + // The difference from the spec is that in wazero, a ModuleInstance holds pointers + // to the instances, rather than "addresses" (i.e. index to Store.Functions, Globals, etc) for convenience. + // + // See https://www.w3.org/TR/2019/REC-wasm-core-1-20191205/#syntax-moduleinst + // + // This implements api.Module. + ModuleInstance struct { + internalapi.WazeroOnlyType + + ModuleName string + Exports map[string]*Export + Globals []*GlobalInstance + MemoryInstance *MemoryInstance + Tables []*TableInstance + + // Engine implements function calls for this module. + Engine ModuleEngine + + // TypeIDs is index-correlated with types and holds typeIDs which is uniquely assigned to a type by store. + // This is necessary to achieve fast runtime type checking for indirect function calls at runtime. + TypeIDs []FunctionTypeID + + // DataInstances holds data segments bytes of the module. + // This is only used by bulk memory operations. + // + // https://www.w3.org/TR/2022/WD-wasm-core-2-20220419/exec/runtime.html#data-instances + DataInstances []DataInstance + + // ElementInstances holds the element instance, and each holds the references to either functions + // or external objects (unimplemented). + ElementInstances []ElementInstance + + // Sys is exposed for use in special imports such as WASI, assemblyscript. + // + // # Notes + // + // - This is a part of ModuleInstance so that scope and Close is coherent. + // - This is not exposed outside this repository (as a host function + // parameter) because we haven't thought through capabilities based + // security implications. + Sys *internalsys.Context + + // Closed is used both to guard moduleEngine.CloseWithExitCode and to store the exit code. + // + // The update value is closedType + exitCode << 32. This ensures an exit code of zero isn't mistaken for never closed. + // + // Note: Exclusively reading and updating this with atomics guarantees cross-goroutine observations. + // See /RATIONALE.md + Closed atomic.Uint64 + + // CodeCloser is non-nil when the code should be closed after this module. + CodeCloser api.Closer + + // s is the Store on which this module is instantiated. + s *Store + // prev and next hold the nodes in the linked list of ModuleInstance held by Store. + prev, next *ModuleInstance + // Source is a pointer to the Module from which this ModuleInstance derives. + Source *Module + + // CloseNotifier is an experimental hook called once on close. + CloseNotifier experimental.CloseNotifier + } + + // DataInstance holds bytes corresponding to the data segment in a module. + // + // https://www.w3.org/TR/2022/WD-wasm-core-2-20220419/exec/runtime.html#data-instances + DataInstance = []byte + + // GlobalInstance represents a global instance in a store. + // See https://www.w3.org/TR/2019/REC-wasm-core-1-20191205/#global-instances%E2%91%A0 + GlobalInstance struct { + Type GlobalType + // Val holds a 64-bit representation of the actual value. + // If me is non-nil, the value will not be updated and the current value is stored in the module engine. + Val uint64 + // ValHi is only used for vector type globals, and holds the higher bits of the vector. + // If me is non-nil, the value will not be updated and the current value is stored in the module engine. + ValHi uint64 + // Me is the module engine that owns this global instance. + // The .Val and .ValHi fields are only valid when me is nil. + // If me is non-nil, the value is stored in the module engine. + Me ModuleEngine + Index Index + } + + // FunctionTypeID is a uniquely assigned integer for a function type. + // This is wazero specific runtime object and specific to a store, + // and used at runtime to do type-checks on indirect function calls. + FunctionTypeID uint32 +) + +// The wazero specific limitations described at RATIONALE.md. +const maximumFunctionTypes = 1 << 27 + +// GetFunctionTypeID is used by emscripten. +func (m *ModuleInstance) GetFunctionTypeID(t *FunctionType) FunctionTypeID { + id, err := m.s.GetFunctionTypeID(t) + if err != nil { + // This is not recoverable in practice since the only error GetFunctionTypeID returns is + // when there's too many function types in the store. + panic(err) + } + return id +} + +func (m *ModuleInstance) buildElementInstances(elements []ElementSegment) { + m.ElementInstances = make([][]Reference, len(elements)) + for i, elm := range elements { + if elm.Type == RefTypeFuncref && elm.Mode == ElementModePassive { + // Only passive elements can be access as element instances. + // See https://www.w3.org/TR/2022/WD-wasm-core-2-20220419/syntax/modules.html#element-segments + inits := elm.Init + inst := make([]Reference, len(inits)) + m.ElementInstances[i] = inst + for j, idx := range inits { + if index, ok := unwrapElementInitGlobalReference(idx); ok { + global := m.Globals[index] + inst[j] = Reference(global.Val) + } else { + if idx != ElementInitNullReference { + inst[j] = m.Engine.FunctionInstanceReference(idx) + } + } + } + } + } +} + +func (m *ModuleInstance) applyElements(elems []ElementSegment) { + for elemI := range elems { + elem := &elems[elemI] + if !elem.IsActive() || + // Per https://github.com/WebAssembly/spec/issues/1427 init can be no-op. + len(elem.Init) == 0 { + continue + } + var offset uint32 + if elem.OffsetExpr.Opcode == OpcodeGlobalGet { + // Ignore error as it's already validated. + globalIdx, _, _ := leb128.LoadUint32(elem.OffsetExpr.Data) + global := m.Globals[globalIdx] + offset = uint32(global.Val) + } else { + // Ignore error as it's already validated. + o, _, _ := leb128.LoadInt32(elem.OffsetExpr.Data) + offset = uint32(o) + } + + table := m.Tables[elem.TableIndex] + references := table.References + if int(offset)+len(elem.Init) > len(references) { + // ErrElementOffsetOutOfBounds is the error raised when the active element offset exceeds the table length. + // Before CoreFeatureReferenceTypes, this was checked statically before instantiation, after the proposal, + // this must be raised as runtime error (as in assert_trap in spectest), not even an instantiation error. + // https://github.com/WebAssembly/spec/blob/d39195773112a22b245ffbe864bab6d1182ccb06/test/core/linking.wast#L264-L274 + // + // In wazero, we ignore it since in any way, the instantiated module and engines are fine and can be used + // for function invocations. + return + } + + if table.Type == RefTypeExternref { + for i := 0; i < len(elem.Init); i++ { + references[offset+uint32(i)] = Reference(0) + } + } else { + for i, init := range elem.Init { + if init == ElementInitNullReference { + continue + } + + var ref Reference + if index, ok := unwrapElementInitGlobalReference(init); ok { + global := m.Globals[index] + ref = Reference(global.Val) + } else { + ref = m.Engine.FunctionInstanceReference(index) + } + references[offset+uint32(i)] = ref + } + } + } +} + +// validateData ensures that data segments are valid in terms of memory boundary. +// Note: this is used only when bulk-memory/reference type feature is disabled. +func (m *ModuleInstance) validateData(data []DataSegment) (err error) { + for i := range data { + d := &data[i] + if !d.IsPassive() { + offset := int(executeConstExpressionI32(m.Globals, &d.OffsetExpression)) + ceil := offset + len(d.Init) + if offset < 0 || ceil > len(m.MemoryInstance.Buffer) { + return fmt.Errorf("%s[%d]: out of bounds memory access", SectionIDName(SectionIDData), i) + } + } + } + return +} + +// applyData uses the given data segments and mutate the memory according to the initial contents on it +// and populate the `DataInstances`. This is called after all the validation phase passes and out of +// bounds memory access error here is not a validation error, but rather a runtime error. +func (m *ModuleInstance) applyData(data []DataSegment) error { + m.DataInstances = make([][]byte, len(data)) + for i := range data { + d := &data[i] + m.DataInstances[i] = d.Init + if !d.IsPassive() { + offset := executeConstExpressionI32(m.Globals, &d.OffsetExpression) + if offset < 0 || int(offset)+len(d.Init) > len(m.MemoryInstance.Buffer) { + return fmt.Errorf("%s[%d]: out of bounds memory access", SectionIDName(SectionIDData), i) + } + copy(m.MemoryInstance.Buffer[offset:], d.Init) + } + } + return nil +} + +// GetExport returns an export of the given name and type or errs if not exported or the wrong type. +func (m *ModuleInstance) getExport(name string, et ExternType) (*Export, error) { + exp, ok := m.Exports[name] + if !ok { + return nil, fmt.Errorf("%q is not exported in module %q", name, m.ModuleName) + } + if exp.Type != et { + return nil, fmt.Errorf("export %q in module %q is a %s, not a %s", name, m.ModuleName, ExternTypeName(exp.Type), ExternTypeName(et)) + } + return exp, nil +} + +func NewStore(enabledFeatures api.CoreFeatures, engine Engine) *Store { + return &Store{ + nameToModule: map[string]*ModuleInstance{}, + nameToModuleCap: nameToModuleShrinkThreshold, + EnabledFeatures: enabledFeatures, + Engine: engine, + typeIDs: map[string]FunctionTypeID{}, + functionMaxTypes: maximumFunctionTypes, + } +} + +// Instantiate uses name instead of the Module.NameSection ModuleName as it allows instantiating the same module under +// different names safely and concurrently. +// +// * ctx: the default context used for function calls. +// * name: the name of the module. +// * sys: the system context, which will be closed (SysContext.Close) on ModuleInstance.Close. +// +// Note: Module.Validate must be called prior to instantiation. +func (s *Store) Instantiate( + ctx context.Context, + module *Module, + name string, + sys *internalsys.Context, + typeIDs []FunctionTypeID, +) (*ModuleInstance, error) { + // Instantiate the module and add it to the store so that other modules can import it. + m, err := s.instantiate(ctx, module, name, sys, typeIDs) + if err != nil { + return nil, err + } + + // Now that the instantiation is complete without error, add it. + if err = s.registerModule(m); err != nil { + _ = m.Close(ctx) + return nil, err + } + return m, nil +} + +func (s *Store) instantiate( + ctx context.Context, + module *Module, + name string, + sysCtx *internalsys.Context, + typeIDs []FunctionTypeID, +) (m *ModuleInstance, err error) { + m = &ModuleInstance{ModuleName: name, TypeIDs: typeIDs, Sys: sysCtx, s: s, Source: module} + + m.Tables = make([]*TableInstance, int(module.ImportTableCount)+len(module.TableSection)) + m.Globals = make([]*GlobalInstance, int(module.ImportGlobalCount)+len(module.GlobalSection)) + m.Engine, err = s.Engine.NewModuleEngine(module, m) + if err != nil { + return nil, err + } + + if err = m.resolveImports(module); err != nil { + return nil, err + } + + err = m.buildTables(module, + // As of reference-types proposal, boundary check must be done after instantiation. + s.EnabledFeatures.IsEnabled(api.CoreFeatureReferenceTypes)) + if err != nil { + return nil, err + } + + allocator, _ := ctx.Value(expctxkeys.MemoryAllocatorKey{}).(experimental.MemoryAllocator) + + m.buildGlobals(module, m.Engine.FunctionInstanceReference) + m.buildMemory(module, allocator) + m.Exports = module.Exports + for _, exp := range m.Exports { + if exp.Type == ExternTypeTable { + t := m.Tables[exp.Index] + t.involvingModuleInstances = append(t.involvingModuleInstances, m) + } + } + + // As of reference types proposal, data segment validation must happen after instantiation, + // and the side effect must persist even if there's out of bounds error after instantiation. + // https://github.com/WebAssembly/spec/blob/d39195773112a22b245ffbe864bab6d1182ccb06/test/core/linking.wast#L395-L405 + if !s.EnabledFeatures.IsEnabled(api.CoreFeatureReferenceTypes) { + if err = m.validateData(module.DataSection); err != nil { + return nil, err + } + } + + // After engine creation, we can create the funcref element instances and initialize funcref type globals. + m.buildElementInstances(module.ElementSection) + + // Now all the validation passes, we are safe to mutate memory instances (possibly imported ones). + if err = m.applyData(module.DataSection); err != nil { + return nil, err + } + + m.applyElements(module.ElementSection) + + m.Engine.DoneInstantiation() + + // Execute the start function. + if module.StartSection != nil { + funcIdx := *module.StartSection + ce := m.Engine.NewFunction(funcIdx) + _, err = ce.Call(ctx) + if exitErr, ok := err.(*sys.ExitError); ok { // Don't wrap an exit error! + return nil, exitErr + } else if err != nil { + return nil, fmt.Errorf("start %s failed: %w", module.funcDesc(SectionIDFunction, funcIdx), err) + } + } + return +} + +func (m *ModuleInstance) resolveImports(module *Module) (err error) { + for moduleName, imports := range module.ImportPerModule { + var importedModule *ModuleInstance + importedModule, err = m.s.module(moduleName) + if err != nil { + return err + } + + for _, i := range imports { + var imported *Export + imported, err = importedModule.getExport(i.Name, i.Type) + if err != nil { + return + } + + switch i.Type { + case ExternTypeFunc: + expectedType := &module.TypeSection[i.DescFunc] + src := importedModule.Source + actual := src.typeOfFunction(imported.Index) + if !actual.EqualsSignature(expectedType.Params, expectedType.Results) { + err = errorInvalidImport(i, fmt.Errorf("signature mismatch: %s != %s", expectedType, actual)) + return + } + + m.Engine.ResolveImportedFunction(i.IndexPerType, imported.Index, importedModule.Engine) + case ExternTypeTable: + expected := i.DescTable + importedTable := importedModule.Tables[imported.Index] + if expected.Type != importedTable.Type { + err = errorInvalidImport(i, fmt.Errorf("table type mismatch: %s != %s", + RefTypeName(expected.Type), RefTypeName(importedTable.Type))) + return + } + + if expected.Min > importedTable.Min { + err = errorMinSizeMismatch(i, expected.Min, importedTable.Min) + return + } + + if expected.Max != nil { + expectedMax := *expected.Max + if importedTable.Max == nil { + err = errorNoMax(i, expectedMax) + return + } else if expectedMax < *importedTable.Max { + err = errorMaxSizeMismatch(i, expectedMax, *importedTable.Max) + return + } + } + m.Tables[i.IndexPerType] = importedTable + importedTable.involvingModuleInstancesMutex.Lock() + if len(importedTable.involvingModuleInstances) == 0 { + panic("BUG: involvingModuleInstances must not be nil when it's imported") + } + importedTable.involvingModuleInstances = append(importedTable.involvingModuleInstances, m) + importedTable.involvingModuleInstancesMutex.Unlock() + case ExternTypeMemory: + expected := i.DescMem + importedMemory := importedModule.MemoryInstance + + if expected.Min > memoryBytesNumToPages(uint64(len(importedMemory.Buffer))) { + err = errorMinSizeMismatch(i, expected.Min, importedMemory.Min) + return + } + + if expected.Max < importedMemory.Max { + err = errorMaxSizeMismatch(i, expected.Max, importedMemory.Max) + return + } + m.MemoryInstance = importedMemory + m.Engine.ResolveImportedMemory(importedModule.Engine) + case ExternTypeGlobal: + expected := i.DescGlobal + importedGlobal := importedModule.Globals[imported.Index] + + if expected.Mutable != importedGlobal.Type.Mutable { + err = errorInvalidImport(i, fmt.Errorf("mutability mismatch: %t != %t", + expected.Mutable, importedGlobal.Type.Mutable)) + return + } + + if expected.ValType != importedGlobal.Type.ValType { + err = errorInvalidImport(i, fmt.Errorf("value type mismatch: %s != %s", + ValueTypeName(expected.ValType), ValueTypeName(importedGlobal.Type.ValType))) + return + } + m.Globals[i.IndexPerType] = importedGlobal + } + } + } + return +} + +func errorMinSizeMismatch(i *Import, expected, actual uint32) error { + return errorInvalidImport(i, fmt.Errorf("minimum size mismatch: %d > %d", expected, actual)) +} + +func errorNoMax(i *Import, expected uint32) error { + return errorInvalidImport(i, fmt.Errorf("maximum size mismatch: %d, but actual has no max", expected)) +} + +func errorMaxSizeMismatch(i *Import, expected, actual uint32) error { + return errorInvalidImport(i, fmt.Errorf("maximum size mismatch: %d < %d", expected, actual)) +} + +func errorInvalidImport(i *Import, err error) error { + return fmt.Errorf("import %s[%s.%s]: %w", ExternTypeName(i.Type), i.Module, i.Name, err) +} + +// executeConstExpressionI32 executes the ConstantExpression which returns ValueTypeI32. +// The validity of the expression is ensured when calling this function as this is only called +// during instantiation phrase, and the validation happens in compilation (validateConstExpression). +func executeConstExpressionI32(importedGlobals []*GlobalInstance, expr *ConstantExpression) (ret int32) { + switch expr.Opcode { + case OpcodeI32Const: + ret, _, _ = leb128.LoadInt32(expr.Data) + case OpcodeGlobalGet: + id, _, _ := leb128.LoadUint32(expr.Data) + g := importedGlobals[id] + ret = int32(g.Val) + } + return +} + +// initialize initializes the value of this global instance given the const expr and imported globals. +// funcRefResolver is called to get the actual funcref (engine specific) from the OpcodeRefFunc const expr. +// +// Global initialization constant expression can only reference the imported globals. +// See the note on https://www.w3.org/TR/2019/REC-wasm-core-1-20191205/#constant-expressions%E2%91%A0 +func (g *GlobalInstance) initialize(importedGlobals []*GlobalInstance, expr *ConstantExpression, funcRefResolver func(funcIndex Index) Reference) { + switch expr.Opcode { + case OpcodeI32Const: + // Treat constants as signed as their interpretation is not yet known per /RATIONALE.md + v, _, _ := leb128.LoadInt32(expr.Data) + g.Val = uint64(uint32(v)) + case OpcodeI64Const: + // Treat constants as signed as their interpretation is not yet known per /RATIONALE.md + v, _, _ := leb128.LoadInt64(expr.Data) + g.Val = uint64(v) + case OpcodeF32Const: + g.Val = uint64(binary.LittleEndian.Uint32(expr.Data)) + case OpcodeF64Const: + g.Val = binary.LittleEndian.Uint64(expr.Data) + case OpcodeGlobalGet: + id, _, _ := leb128.LoadUint32(expr.Data) + importedG := importedGlobals[id] + switch importedG.Type.ValType { + case ValueTypeI32: + g.Val = uint64(uint32(importedG.Val)) + case ValueTypeI64: + g.Val = importedG.Val + case ValueTypeF32: + g.Val = importedG.Val + case ValueTypeF64: + g.Val = importedG.Val + case ValueTypeV128: + g.Val, g.ValHi = importedG.Val, importedG.ValHi + case ValueTypeFuncref, ValueTypeExternref: + g.Val = importedG.Val + } + case OpcodeRefNull: + switch expr.Data[0] { + case ValueTypeExternref, ValueTypeFuncref: + g.Val = 0 // Reference types are opaque 64bit pointer at runtime. + } + case OpcodeRefFunc: + v, _, _ := leb128.LoadUint32(expr.Data) + g.Val = uint64(funcRefResolver(v)) + case OpcodeVecV128Const: + g.Val, g.ValHi = binary.LittleEndian.Uint64(expr.Data[0:8]), binary.LittleEndian.Uint64(expr.Data[8:16]) + } +} + +// String implements api.Global. +func (g *GlobalInstance) String() string { + switch g.Type.ValType { + case ValueTypeI32, ValueTypeI64: + return fmt.Sprintf("global(%d)", g.Val) + case ValueTypeF32: + return fmt.Sprintf("global(%f)", api.DecodeF32(g.Val)) + case ValueTypeF64: + return fmt.Sprintf("global(%f)", api.DecodeF64(g.Val)) + default: + panic(fmt.Errorf("BUG: unknown value type %X", g.Type.ValType)) + } +} + +func (g *GlobalInstance) Value() (uint64, uint64) { + if g.Me != nil { + return g.Me.GetGlobalValue(g.Index) + } + return g.Val, g.ValHi +} + +func (g *GlobalInstance) SetValue(lo, hi uint64) { + if g.Me != nil { + g.Me.SetGlobalValue(g.Index, lo, hi) + } else { + g.Val, g.ValHi = lo, hi + } +} + +func (s *Store) GetFunctionTypeIDs(ts []FunctionType) ([]FunctionTypeID, error) { + ret := make([]FunctionTypeID, len(ts)) + for i := range ts { + t := &ts[i] + inst, err := s.GetFunctionTypeID(t) + if err != nil { + return nil, err + } + ret[i] = inst + } + return ret, nil +} + +func (s *Store) GetFunctionTypeID(t *FunctionType) (FunctionTypeID, error) { + s.mux.RLock() + key := t.key() + id, ok := s.typeIDs[key] + s.mux.RUnlock() + if !ok { + s.mux.Lock() + defer s.mux.Unlock() + // Check again in case another goroutine has already added the type. + if id, ok = s.typeIDs[key]; ok { + return id, nil + } + l := len(s.typeIDs) + if uint32(l) >= s.functionMaxTypes { + return 0, fmt.Errorf("too many function types in a store") + } + id = FunctionTypeID(l) + s.typeIDs[key] = id + } + return id, nil +} + +// CloseWithExitCode implements the same method as documented on wazero.Runtime. +func (s *Store) CloseWithExitCode(ctx context.Context, exitCode uint32) (err error) { + s.mux.Lock() + defer s.mux.Unlock() + // Close modules in reverse initialization order. + for m := s.moduleList; m != nil; m = m.next { + // If closing this module errs, proceed anyway to close the others. + if e := m.closeWithExitCode(ctx, exitCode); e != nil && err == nil { + // TODO: use multiple errors handling in Go 1.20. + err = e // first error + } + } + s.moduleList = nil + s.nameToModule = nil + s.nameToModuleCap = 0 + s.typeIDs = nil + return +} diff --git a/vendor/github.com/tetratelabs/wazero/internal/wasm/store_module_list.go b/vendor/github.com/tetratelabs/wazero/internal/wasm/store_module_list.go new file mode 100644 index 000000000..17c63e38e --- /dev/null +++ b/vendor/github.com/tetratelabs/wazero/internal/wasm/store_module_list.go @@ -0,0 +1,97 @@ +package wasm + +import ( + "errors" + "fmt" + + "github.com/tetratelabs/wazero/api" +) + +// deleteModule makes the moduleName available for instantiation again. +func (s *Store) deleteModule(m *ModuleInstance) error { + s.mux.Lock() + defer s.mux.Unlock() + + // Remove this module name. + if m.prev != nil { + m.prev.next = m.next + } + if m.next != nil { + m.next.prev = m.prev + } + if s.moduleList == m { + s.moduleList = m.next + } + // Clear the m state so it does not enter any other branch + // on subsequent calls to deleteModule. + m.prev = nil + m.next = nil + + if m.ModuleName != "" { + delete(s.nameToModule, m.ModuleName) + + // Shrink the map if it's allocated more than twice the size of the list + newCap := len(s.nameToModule) + if newCap < nameToModuleShrinkThreshold { + newCap = nameToModuleShrinkThreshold + } + if newCap*2 <= s.nameToModuleCap { + nameToModule := make(map[string]*ModuleInstance, newCap) + for k, v := range s.nameToModule { + nameToModule[k] = v + } + s.nameToModule = nameToModule + s.nameToModuleCap = newCap + } + } + return nil +} + +// module returns the module of the given name or error if not in this store +func (s *Store) module(moduleName string) (*ModuleInstance, error) { + s.mux.RLock() + defer s.mux.RUnlock() + m, ok := s.nameToModule[moduleName] + if !ok { + return nil, fmt.Errorf("module[%s] not instantiated", moduleName) + } + return m, nil +} + +// registerModule registers a ModuleInstance into the store. +// This makes the ModuleInstance visible for import if it's not anonymous, and ensures it is closed when the store is. +func (s *Store) registerModule(m *ModuleInstance) error { + s.mux.Lock() + defer s.mux.Unlock() + + if s.nameToModule == nil { + return errors.New("already closed") + } + + if m.ModuleName != "" { + if _, ok := s.nameToModule[m.ModuleName]; ok { + return fmt.Errorf("module[%s] has already been instantiated", m.ModuleName) + } + s.nameToModule[m.ModuleName] = m + if len(s.nameToModule) > s.nameToModuleCap { + s.nameToModuleCap = len(s.nameToModule) + } + } + + // Add the newest node to the moduleNamesList as the head. + m.next = s.moduleList + if m.next != nil { + m.next.prev = m + } + s.moduleList = m + return nil +} + +// Module implements wazero.Runtime Module +func (s *Store) Module(moduleName string) api.Module { + m, err := s.module(moduleName) + if err != nil { + return nil + } + return m +} diff --git a/vendor/github.com/tetratelabs/wazero/internal/wasm/table.go b/vendor/github.com/tetratelabs/wazero/internal/wasm/table.go new file mode 100644 index 000000000..2123693c6 --- /dev/null +++ b/vendor/github.com/tetratelabs/wazero/internal/wasm/table.go @@ -0,0 +1,339 @@ +package wasm + +import ( + "fmt" + "math" + "sync" + + "github.com/tetratelabs/wazero/api" + "github.com/tetratelabs/wazero/internal/leb128" +) + +// Table describes the limits of elements and its type in a table. +type Table struct { + Min uint32 + Max *uint32 + Type RefType +} + +// RefType is either RefTypeFuncref or RefTypeExternref as of WebAssembly core 2.0. +type RefType = byte + +const ( + // RefTypeFuncref represents a reference to a function. + RefTypeFuncref = ValueTypeFuncref + // RefTypeExternref represents a reference to a host object, which is not currently supported in wazero. + RefTypeExternref = ValueTypeExternref +) + +func RefTypeName(t RefType) (ret string) { + switch t { + case RefTypeFuncref: + ret = "funcref" + case RefTypeExternref: + ret = "externref" + default: + ret = fmt.Sprintf("unknown(0x%x)", t) + } + return +} + +// ElementMode represents a mode of element segment which is either active, passive or declarative. +// +// https://www.w3.org/TR/2022/WD-wasm-core-2-20220419/syntax/modules.html#element-segments +type ElementMode = byte + +const ( + // ElementModeActive is the mode which requires the runtime to initialize table with the contents in .Init field combined with OffsetExpr. + ElementModeActive ElementMode = iota + // ElementModePassive is the mode which doesn't require the runtime to initialize table, and only used with OpcodeTableInitName. + ElementModePassive + // ElementModeDeclarative is introduced in reference-types proposal which can be used to declare function indexes used by OpcodeRefFunc. + ElementModeDeclarative +) + +// ElementSegment are initialization instructions for a TableInstance +// +// See https://www.w3.org/TR/2019/REC-wasm-core-1-20191205/#syntax-elem +type ElementSegment struct { + // OffsetExpr returns the table element offset to apply to Init indices. + // Note: This can be validated prior to instantiation unless it includes OpcodeGlobalGet (an imported global). + OffsetExpr ConstantExpression + + // TableIndex is the table's index to which this element segment is applied. + // Note: This is used if and only if the Mode is active. + TableIndex Index + + // Followings are set/used regardless of the Mode. + + // Init indices are (nullable) table elements where each index is the function index by which the module initialize the table. + Init []Index + + // Type holds the type of this element segment, which is the RefType in WebAssembly 2.0. + Type RefType + + // Mode is the mode of this element segment. + Mode ElementMode +} + +const ( + // ElementInitNullReference represents the null reference in ElementSegment's Init. + // In Wasm spec, an init item represents either Function's Index or null reference, + // and in wazero, we limit the maximum number of functions available in a module to + // MaximumFunctionIndex. Therefore, it is safe to use 1 << 31 to represent the null + // reference in Element segments. + ElementInitNullReference Index = 1 << 31 + // elementInitImportedGlobalReferenceType represents an init item which is resolved via an imported global constexpr. + // The actual function reference stored at Global is only known at instantiation-time, so we set this flag + // to items of ElementSegment.Init at binary decoding, and unwrap this flag at instantiation to resolve the value. + // + // This might collide the init element resolved via ref.func instruction which is resolved with the func index at decoding, + // but in practice, that is not allowed in wazero thanks to our limit MaximumFunctionIndex. Thus, it is safe to set this flag + // in init element to indicate as such. + elementInitImportedGlobalReferenceType Index = 1 << 30 +) + +// unwrapElementInitGlobalReference takes an item of the init vector of an ElementSegment, +// and returns the Global index if it is supposed to get generated from a global. +// ok is true if the given init item is as such. +func unwrapElementInitGlobalReference(init Index) (_ Index, ok bool) { + if init&elementInitImportedGlobalReferenceType == elementInitImportedGlobalReferenceType { + return init &^ elementInitImportedGlobalReferenceType, true + } + return init, false +} + +// WrapGlobalIndexAsElementInit wraps the given index as an init item which is resolved via an imported global value. +// See the comments on elementInitImportedGlobalReferenceType for more details. +func WrapGlobalIndexAsElementInit(init Index) Index { + return init | elementInitImportedGlobalReferenceType +} + +// IsActive returns true if the element segment is "active" mode which requires the runtime to initialize table +// with the contents in .Init field. +func (e *ElementSegment) IsActive() bool { + return e.Mode == ElementModeActive +} + +// TableInstance represents a table of (RefTypeFuncref) elements in a module. +// +// See https://www.w3.org/TR/2019/REC-wasm-core-1-20191205/#table-instances%E2%91%A0 +type TableInstance struct { + // References holds references whose type is either RefTypeFuncref or RefTypeExternref (unsupported). + // + // Currently, only function references are supported. + References []Reference + + // Min is the minimum (function) elements in this table and cannot grow to accommodate ElementSegment. + Min uint32 + + // Max if present is the maximum (function) elements in this table, or nil if unbounded. + Max *uint32 + + // Type is either RefTypeFuncref or RefTypeExternRef. + Type RefType + + // The following is only used when the table is exported. + + // involvingModuleInstances is a set of module instances which are involved in the table instance. + // This is critical for safety purpose because once a table is imported, it can hold any reference to + // any function in the owner and importing module instances. Therefore, these module instance, + // transitively the compiled modules, must be alive as long as the table instance is alive. + involvingModuleInstances []*ModuleInstance + // involvingModuleInstancesMutex is a mutex to protect involvingModuleInstances. + involvingModuleInstancesMutex sync.RWMutex +} + +// ElementInstance represents an element instance in a module. +// +// See https://www.w3.org/TR/2022/WD-wasm-core-2-20220419/exec/runtime.html#element-instances +type ElementInstance = []Reference + +// Reference is the runtime representation of RefType which is either RefTypeFuncref or RefTypeExternref. +type Reference = uintptr + +// validateTable ensures any ElementSegment is valid. This caches results via Module.validatedActiveElementSegments. +// Note: limitsType are validated by decoders, so not re-validated here. +func (m *Module) validateTable(enabledFeatures api.CoreFeatures, tables []Table, maximumTableIndex uint32) error { + if len(tables) > int(maximumTableIndex) { + return fmt.Errorf("too many tables in a module: %d given with limit %d", len(tables), maximumTableIndex) + } + + importedTableCount := m.ImportTableCount + + // Create bounds checks as these can err prior to instantiation + funcCount := m.ImportFunctionCount + m.SectionElementCount(SectionIDFunction) + globalsCount := m.ImportGlobalCount + m.SectionElementCount(SectionIDGlobal) + + // Now, we have to figure out which table elements can be resolved before instantiation and also fail early if there + // are any imported globals that are known to be invalid by their declarations. + for i := range m.ElementSection { + elem := &m.ElementSection[i] + idx := Index(i) + initCount := uint32(len(elem.Init)) + + // Any offset applied is to the element, not the function index: validate here if the funcidx is sound. + for ei, init := range elem.Init { + if init == ElementInitNullReference { + continue + } + index, ok := unwrapElementInitGlobalReference(init) + if ok { + if index >= globalsCount { + return fmt.Errorf("%s[%d].init[%d] global index %d out of range", SectionIDName(SectionIDElement), idx, ei, index) + } + } else { + if elem.Type == RefTypeExternref { + return fmt.Errorf("%s[%d].init[%d] must be ref.null but was %d", SectionIDName(SectionIDElement), idx, ei, init) + } + if index >= funcCount { + return fmt.Errorf("%s[%d].init[%d] func index %d out of range", SectionIDName(SectionIDElement), idx, ei, index) + } + } + } + + if elem.IsActive() { + if len(tables) <= int(elem.TableIndex) { + return fmt.Errorf("unknown table %d as active element target", elem.TableIndex) + } + + t := tables[elem.TableIndex] + if t.Type != elem.Type { + return fmt.Errorf("element type mismatch: table has %s but element has %s", + RefTypeName(t.Type), RefTypeName(elem.Type), + ) + } + + // global.get needs to be discovered during initialization + oc := elem.OffsetExpr.Opcode + if oc == OpcodeGlobalGet { + globalIdx, _, err := leb128.LoadUint32(elem.OffsetExpr.Data) + if err != nil { + return fmt.Errorf("%s[%d] couldn't read global.get parameter: %w", SectionIDName(SectionIDElement), idx, err) + } else if err = m.verifyImportGlobalI32(SectionIDElement, idx, globalIdx); err != nil { + return err + } + } else if oc == OpcodeI32Const { + // Per https://github.com/WebAssembly/spec/blob/wg-1.0/test/core/elem.wast#L117 we must pass if imported + // table has set its min=0. Per https://github.com/WebAssembly/spec/blob/wg-1.0/test/core/elem.wast#L142, we + // have to do fail if module-defined min=0. + if !enabledFeatures.IsEnabled(api.CoreFeatureReferenceTypes) && elem.TableIndex >= importedTableCount { + // Treat constants as signed as their interpretation is not yet known per /RATIONALE.md + o, _, err := leb128.LoadInt32(elem.OffsetExpr.Data) + if err != nil { + return fmt.Errorf("%s[%d] couldn't read i32.const parameter: %w", SectionIDName(SectionIDElement), idx, err) + } + offset := Index(o) + if err = checkSegmentBounds(t.Min, uint64(initCount)+uint64(offset), idx); err != nil { + return err + } + } + } else { + return fmt.Errorf("%s[%d] has an invalid const expression: %s", SectionIDName(SectionIDElement), idx, InstructionName(oc)) + } + } + } + return nil +} + +// buildTable returns TableInstances if the module defines or imports a table. +// - importedTables: returned as `tables` unmodified. +// - importedGlobals: include all instantiated, imported globals. +// +// If the result `init` is non-nil, it is the `tableInit` parameter of Engine.NewModuleEngine. +// +// Note: An error is only possible when an ElementSegment.OffsetExpr is out of range of the TableInstance.Min. +func (m *ModuleInstance) buildTables(module *Module, skipBoundCheck bool) (err error) { + idx := module.ImportTableCount + for i := range module.TableSection { + tsec := &module.TableSection[i] + // The module defining the table is the one that sets its Min/Max etc. + m.Tables[idx] = &TableInstance{ + References: make([]Reference, tsec.Min), Min: tsec.Min, Max: tsec.Max, + Type: tsec.Type, + } + idx++ + } + + if !skipBoundCheck { + for elemI := range module.ElementSection { // Do not loop over the value since elementSegments is a slice of value. + elem := &module.ElementSection[elemI] + table := m.Tables[elem.TableIndex] + var offset uint32 + if elem.OffsetExpr.Opcode == OpcodeGlobalGet { + // Ignore error as it's already validated. + globalIdx, _, _ := leb128.LoadUint32(elem.OffsetExpr.Data) + global := m.Globals[globalIdx] + offset = uint32(global.Val) + } else { // i32.const + // Ignore error as it's already validated. + o, _, _ := leb128.LoadInt32(elem.OffsetExpr.Data) + offset = uint32(o) + } + + // Check to see if we are out-of-bounds + initCount := uint64(len(elem.Init)) + if err = checkSegmentBounds(table.Min, uint64(offset)+initCount, Index(elemI)); err != nil { + return + } + } + } + return +} + +// checkSegmentBounds fails if the capacity needed for an ElementSegment.Init is larger than limitsType.Min +// +// WebAssembly 1.0 (20191205) doesn't forbid growing to accommodate element segments, and spectests are inconsistent. +// For example, the spectests enforce elements within Table limitsType.Min, but ignore Import.DescTable min. What this +// means is we have to delay offset checks on imported tables until we link to them. +// e.g. https://github.com/WebAssembly/spec/blob/wg-1.0/test/core/elem.wast#L117 wants pass on min=0 for import +// e.g. https://github.com/WebAssembly/spec/blob/wg-1.0/test/core/elem.wast#L142 wants fail on min=0 module-defined +func checkSegmentBounds(min uint32, requireMin uint64, idx Index) error { // uint64 in case offset was set to -1 + if requireMin > uint64(min) { + return fmt.Errorf("%s[%d].init exceeds min table size", SectionIDName(SectionIDElement), idx) + } + return nil +} + +func (m *Module) verifyImportGlobalI32(sectionID SectionID, sectionIdx Index, idx uint32) error { + ig := uint32(math.MaxUint32) // +1 == 0 + for i := range m.ImportSection { + imp := &m.ImportSection[i] + if imp.Type == ExternTypeGlobal { + ig++ + if ig == idx { + if imp.DescGlobal.ValType != ValueTypeI32 { + return fmt.Errorf("%s[%d] (global.get %d): import[%d].global.ValType != i32", SectionIDName(sectionID), sectionIdx, idx, i) + } + return nil + } + } + } + return fmt.Errorf("%s[%d] (global.get %d): out of range of imported globals", SectionIDName(sectionID), sectionIdx, idx) +} + +// Grow appends the `initialRef` by `delta` times into the References slice. +// Returns -1 if the operation is not valid, otherwise the old length of the table. +// +// https://www.w3.org/TR/2022/WD-wasm-core-2-20220419/exec/instructions.html#xref-syntax-instructions-syntax-instr-table-mathsf-table-grow-x +func (t *TableInstance) Grow(delta uint32, initialRef Reference) (currentLen uint32) { + currentLen = uint32(len(t.References)) + if delta == 0 { + return + } + + if newLen := int64(currentLen) + int64(delta); // adding as 64bit ints to avoid overflow. + newLen >= math.MaxUint32 || (t.Max != nil && newLen > int64(*t.Max)) { + return 0xffffffff // = -1 in signed 32-bit integer. + } + t.References = append(t.References, make([]uintptr, delta)...) + + // Uses the copy trick for faster filling the new region with the initial value. + // https://gist.github.com/taylorza/df2f89d5f9ab3ffd06865062a4cf015d + newRegion := t.References[currentLen:] + newRegion[0] = initialRef + for i := 1; i < len(newRegion); i *= 2 { + copy(newRegion[i:], newRegion[:i]) + } + return +} diff --git a/vendor/github.com/tetratelabs/wazero/internal/wasmdebug/debug.go b/vendor/github.com/tetratelabs/wazero/internal/wasmdebug/debug.go new file mode 100644 index 000000000..ff0e0cccc --- /dev/null +++ b/vendor/github.com/tetratelabs/wazero/internal/wasmdebug/debug.go @@ -0,0 +1,170 @@ +// Package wasmdebug contains utilities used to give consistent search keys between stack traces and error messages. +// Note: This is named wasmdebug to avoid conflicts with the normal go module. +// Note: This only imports "api" as importing "wasm" would create a cyclic dependency. +package wasmdebug + +import ( + "fmt" + "runtime" + "runtime/debug" + "strconv" + "strings" + + "github.com/tetratelabs/wazero/api" + "github.com/tetratelabs/wazero/internal/wasmruntime" + "github.com/tetratelabs/wazero/sys" +) + +// FuncName returns the naming convention of "moduleName.funcName". +// +// - moduleName is the possibly empty name the module was instantiated with. +// - funcName is the name in the Custom Name section. +// - funcIdx is the position in the function index, prefixed with +// imported functions. +// +// Note: "moduleName.$funcIdx" is used when the funcName is empty, as commonly +// the case in TinyGo. +func FuncName(moduleName, funcName string, funcIdx uint32) string { + var ret strings.Builder + + // Start module.function + ret.WriteString(moduleName) + ret.WriteByte('.') + if funcName == "" { + ret.WriteByte('$') + ret.WriteString(strconv.Itoa(int(funcIdx))) + } else { + ret.WriteString(funcName) + } + + return ret.String() +} + +// signature returns a formatted signature similar to how it is defined in Go. +// +// * paramTypes should be from wasm.FunctionType +// * resultTypes should be from wasm.FunctionType +// TODO: add paramNames +func signature(funcName string, paramTypes []api.ValueType, resultTypes []api.ValueType) string { + var ret strings.Builder + ret.WriteString(funcName) + + // Start params + ret.WriteByte('(') + paramCount := len(paramTypes) + switch paramCount { + case 0: + case 1: + ret.WriteString(api.ValueTypeName(paramTypes[0])) + default: + ret.WriteString(api.ValueTypeName(paramTypes[0])) + for _, vt := range paramTypes[1:] { + ret.WriteByte(',') + ret.WriteString(api.ValueTypeName(vt)) + } + } + ret.WriteByte(')') + + // Start results + resultCount := len(resultTypes) + switch resultCount { + case 0: + case 1: + ret.WriteByte(' ') + ret.WriteString(api.ValueTypeName(resultTypes[0])) + default: // As this is used for errors, don't panic if there are multiple returns, even if that's invalid! + ret.WriteByte(' ') + ret.WriteByte('(') + ret.WriteString(api.ValueTypeName(resultTypes[0])) + for _, vt := range resultTypes[1:] { + ret.WriteByte(',') + ret.WriteString(api.ValueTypeName(vt)) + } + ret.WriteByte(')') + } + + return ret.String() +} + +// ErrorBuilder helps build consistent errors, particularly adding a WASM stack trace. +// +// AddFrame should be called beginning at the frame that panicked until no more frames exist. Once done, call Format. +type ErrorBuilder interface { + // AddFrame adds the next frame. + // + // * funcName should be from FuncName + // * paramTypes should be from wasm.FunctionType + // * resultTypes should be from wasm.FunctionType + // * sources is the source code information for this frame and can be empty. + // + // Note: paramTypes and resultTypes are present because signature misunderstanding, mismatch or overflow are common. + AddFrame(funcName string, paramTypes, resultTypes []api.ValueType, sources []string) + + // FromRecovered returns an error with the wasm stack trace appended to it. + FromRecovered(recovered interface{}) error +} + +func NewErrorBuilder() ErrorBuilder { + return &stackTrace{} +} + +type stackTrace struct { + // frameCount is the number of stack frame currently pushed into lines. + frameCount int + // lines contains the stack trace and possibly the inlined source code information. + lines []string +} + +// GoRuntimeErrorTracePrefix is the prefix coming before the Go runtime stack trace included in the face of runtime.Error. +// This is exported for testing purpose. +const GoRuntimeErrorTracePrefix = "Go runtime stack trace:" + +func (s *stackTrace) FromRecovered(recovered interface{}) error { + if false { + debug.PrintStack() + } + + if exitErr, ok := recovered.(*sys.ExitError); ok { // Don't wrap an exit error! + return exitErr + } + + stack := strings.Join(s.lines, "\n\t") + + // If the error was internal, don't mention it was recovered. + if wasmErr, ok := recovered.(*wasmruntime.Error); ok { + return fmt.Errorf("wasm error: %w\nwasm stack trace:\n\t%s", wasmErr, stack) + } + + // If we have a runtime.Error, something severe happened which should include the stack trace. This could be + // a nil pointer from wazero or a user-defined function from HostModuleBuilder. + if runtimeErr, ok := recovered.(runtime.Error); ok { + return fmt.Errorf("%w (recovered by wazero)\nwasm stack trace:\n\t%s\n\n%s\n%s", + runtimeErr, stack, GoRuntimeErrorTracePrefix, debug.Stack()) + } + + // At this point we expect the error was from a function defined by HostModuleBuilder that intentionally called panic. + if runtimeErr, ok := recovered.(error); ok { // e.g. panic(errors.New("whoops")) + return fmt.Errorf("%w (recovered by wazero)\nwasm stack trace:\n\t%s", runtimeErr, stack) + } else { // e.g. panic("whoops") + return fmt.Errorf("%v (recovered by wazero)\nwasm stack trace:\n\t%s", recovered, stack) + } +} + +// MaxFrames is the maximum number of frames to include in the stack trace. +const MaxFrames = 30 + +// AddFrame implements ErrorBuilder.AddFrame +func (s *stackTrace) AddFrame(funcName string, paramTypes, resultTypes []api.ValueType, sources []string) { + if s.frameCount == MaxFrames { + return + } + s.frameCount++ + sig := signature(funcName, paramTypes, resultTypes) + s.lines = append(s.lines, sig) + for _, source := range sources { + s.lines = append(s.lines, "\t"+source) + } + if s.frameCount == MaxFrames { + s.lines = append(s.lines, "... maybe followed by omitted frames") + } +} diff --git a/vendor/github.com/tetratelabs/wazero/internal/wasmdebug/dwarf.go b/vendor/github.com/tetratelabs/wazero/internal/wasmdebug/dwarf.go new file mode 100644 index 000000000..3b0d3a7a6 --- /dev/null +++ b/vendor/github.com/tetratelabs/wazero/internal/wasmdebug/dwarf.go @@ -0,0 +1,226 @@ +package wasmdebug + +import ( + "debug/dwarf" + "errors" + "fmt" + "io" + "sort" + "strings" + "sync" +) + +// DWARFLines is used to retrieve source code line information from the DWARF data. +type DWARFLines struct { + // d is created by DWARF custom sections. + d *dwarf.Data + // linesPerEntry maps dwarf.Offset for dwarf.Entry to the list of lines contained by the entry. + // The value is sorted in the increasing order by the address. + linesPerEntry map[dwarf.Offset][]line + mux sync.Mutex +} + +type line struct { + addr uint64 + pos dwarf.LineReaderPos +} + +// NewDWARFLines returns DWARFLines for the given *dwarf.Data. +func NewDWARFLines(d *dwarf.Data) *DWARFLines { + if d == nil { + return nil + } + return &DWARFLines{d: d, linesPerEntry: map[dwarf.Offset][]line{}} +} + +// isTombstoneAddr returns true if the given address is invalid a.k.a tombstone address which was made no longer valid +// by linker. According to the DWARF spec[1], the value is encoded as 0xffffffff for Wasm (as 32-bit target), +// but some tools encode it either in -1, -2 [2] or 1<<32 (This might not be by tools, but by debug/dwarf package's bug). +// +// [1] https://dwarfstd.org/issues/200609.1.html +// [2] https://github.com/WebAssembly/binaryen/blob/97178d08d4a20d2a5e3a6be813fc6a7079ef86e1/src/wasm/wasm-debug.cpp#L651-L660 +// [3] https://reviews.llvm.org/D81784 +func isTombstoneAddr(addr uint64) bool { + addr32 := int32(addr) + return addr32 == -1 || addr32 == -2 || + addr32 == 0 // This covers 1 <<32. +} + +// Line returns the line information for the given instructionOffset which is an offset in +// the code section of the original Wasm binary. Returns empty string if the info is not found. +func (d *DWARFLines) Line(instructionOffset uint64) (ret []string) { + if d == nil { + return + } + + // DWARFLines is created per Wasm binary, so there's a possibility that multiple instances + // created from a same binary face runtime error at the same time, and that results in + // concurrent access to this function. + d.mux.Lock() + defer d.mux.Unlock() + + r := d.d.Reader() + + var inlinedRoutines []*dwarf.Entry + var cu *dwarf.Entry + var inlinedDone bool +entry: + for { + ent, err := r.Next() + if err != nil || ent == nil { + break + } + + // If we already found the compilation unit and relevant inlined routines, we can stop searching entries. + if cu != nil && inlinedDone { + break + } + + switch ent.Tag { + case dwarf.TagCompileUnit, dwarf.TagInlinedSubroutine: + default: + // Only CompileUnit and InlinedSubroutines are relevant. + continue + } + + // Check if the entry spans the range which contains the target instruction. + ranges, err := d.d.Ranges(ent) + if err != nil { + continue + } + for _, pcs := range ranges { + start, end := pcs[0], pcs[1] + if isTombstoneAddr(start) || isTombstoneAddr(end) { + continue + } + if start <= instructionOffset && instructionOffset < end { + switch ent.Tag { + case dwarf.TagCompileUnit: + cu = ent + case dwarf.TagInlinedSubroutine: + inlinedRoutines = append(inlinedRoutines, ent) + // Search inlined subroutines until all the children. + inlinedDone = !ent.Children + // Not that "children" in the DWARF spec is defined as the next entry to this entry. + // See "2.3 Relationship of Debugging Information Entries" in https://dwarfstd.org/doc/DWARF4.pdf + } + continue entry + } + } + } + + // If the relevant compilation unit is not found, nothing we can do with this DWARF info. + if cu == nil { + return + } + + lineReader, err := d.d.LineReader(cu) + if err != nil || lineReader == nil { + return + } + var lines []line + var ok bool + var le dwarf.LineEntry + // Get the lines inside the entry. + if lines, ok = d.linesPerEntry[cu.Offset]; !ok { + // If not found, we create the list of lines by reading all the LineEntries in the Entry. + // + // Note that the dwarf.LineEntry.SeekPC API shouldn't be used because the Go's dwarf package assumes that + // all the line entries in an Entry are sorted in increasing order which *might not* be true + // for some languages. Such order requirement is not a part of DWARF specification, + // and in fact Zig language tends to emit interleaved line information. + // + // Thus, here we read all line entries here, and sort them in the increasing order wrt addresses. + for { + pos := lineReader.Tell() + err = lineReader.Next(&le) + if errors.Is(err, io.EOF) { + break + } else if err != nil { + return + } + // TODO: Maybe we should ignore tombstone addresses by using isTombstoneAddr, + // but not sure if that would be an issue in practice. + lines = append(lines, line{addr: le.Address, pos: pos}) + } + sort.Slice(lines, func(i, j int) bool { return lines[i].addr < lines[j].addr }) + d.linesPerEntry[cu.Offset] = lines // Caches for the future inquiries for the same Entry. + } + + // Now we have the lines for this entry. We can find the corresponding source line for instructionOffset + // via binary search on the list. + n := len(lines) + index := sort.Search(n, func(i int) bool { return lines[i].addr >= instructionOffset }) + + if index == n { // This case the address is not found. See the doc sort.Search. + return + } + + ln := lines[index] + if ln.addr != instructionOffset { + // If the address doesn't match exactly, the previous entry is the one that contains the instruction. + // That can happen anytime as the DWARF spec allows it, and other tools can handle it in this way conventionally + // https://github.com/gimli-rs/addr2line/blob/3a2dbaf84551a06a429f26e9c96071bb409b371f/src/lib.rs#L236-L242 + // https://github.com/kateinoigakukun/wasminspect/blob/f29f052f1b03104da9f702508ac0c1bbc3530ae4/crates/debugger/src/dwarf/mod.rs#L453-L459 + if index-1 < 0 { + return + } + ln = lines[index-1] + } + + // Advance the line reader for the found position. + lineReader.Seek(ln.pos) + err = lineReader.Next(&le) + + if err != nil { + // If we reach this block, that means there's a bug in the []line creation logic above. + panic("BUG: stored dwarf.LineReaderPos is invalid") + } + + // In the inlined case, the line info is the innermost inlined function call. + inlined := len(inlinedRoutines) != 0 + prefix := fmt.Sprintf("%#x: ", instructionOffset) + ret = append(ret, formatLine(prefix, le.File.Name, int64(le.Line), int64(le.Column), inlined)) + + if inlined { + prefix = strings.Repeat(" ", len(prefix)) + files := lineReader.Files() + // inlinedRoutines contain the inlined call information in the reverse order (children is higher than parent), + // so we traverse the reverse order and emit the inlined calls. + for i := len(inlinedRoutines) - 1; i >= 0; i-- { + inlined := inlinedRoutines[i] + fileIndex, ok := inlined.Val(dwarf.AttrCallFile).(int64) + if !ok { + return + } else if fileIndex >= int64(len(files)) { + // This in theory shouldn't happen according to the spec, but guard against ill-formed DWARF info. + return + } + fileName := files[fileIndex] + line, _ := inlined.Val(dwarf.AttrCallLine).(int64) + col, _ := inlined.Val(dwarf.AttrCallColumn).(int64) + ret = append(ret, formatLine(prefix, fileName.Name, line, col, + // Last one is the origin of the inlined function calls. + i != 0)) + } + } + return +} + +func formatLine(prefix, fileName string, line, col int64, inlined bool) string { + builder := strings.Builder{} + builder.WriteString(prefix) + builder.WriteString(fileName) + + if line != 0 { + builder.WriteString(fmt.Sprintf(":%d", line)) + if col != 0 { + builder.WriteString(fmt.Sprintf(":%d", col)) + } + } + + if inlined { + builder.WriteString(" (inlined)") + } + return builder.String() +} diff --git a/vendor/github.com/tetratelabs/wazero/internal/wasmruntime/errors.go b/vendor/github.com/tetratelabs/wazero/internal/wasmruntime/errors.go new file mode 100644 index 000000000..556e5de82 --- /dev/null +++ b/vendor/github.com/tetratelabs/wazero/internal/wasmruntime/errors.go @@ -0,0 +1,50 @@ +// Package wasmruntime contains internal symbols shared between modules for error handling. +// Note: This is named wasmruntime to avoid conflicts with the normal go module. +// Note: This only imports "api" as importing "wasm" would create a cyclic dependency. +package wasmruntime + +var ( + // ErrRuntimeStackOverflow indicates that there are too many function calls, + // and the Engine terminated the execution. + ErrRuntimeStackOverflow = New("stack overflow") + // ErrRuntimeInvalidConversionToInteger indicates the Wasm function tries to + // convert NaN floating point value to integers during trunc variant instructions. + ErrRuntimeInvalidConversionToInteger = New("invalid conversion to integer") + // ErrRuntimeIntegerOverflow indicates that an integer arithmetic resulted in + // overflow value. For example, when the program tried to truncate a float value + // which doesn't fit in the range of target integer. + ErrRuntimeIntegerOverflow = New("integer overflow") + // ErrRuntimeIntegerDivideByZero indicates that an integer div or rem instructions + // was executed with 0 as the divisor. + ErrRuntimeIntegerDivideByZero = New("integer divide by zero") + // ErrRuntimeUnreachable means "unreachable" instruction was executed by the program. + ErrRuntimeUnreachable = New("unreachable") + // ErrRuntimeOutOfBoundsMemoryAccess indicates that the program tried to access the + // region beyond the linear memory. + ErrRuntimeOutOfBoundsMemoryAccess = New("out of bounds memory access") + // ErrRuntimeInvalidTableAccess means either offset to the table was out of bounds of table, or + // the target element in the table was uninitialized during call_indirect instruction. + ErrRuntimeInvalidTableAccess = New("invalid table access") + // ErrRuntimeIndirectCallTypeMismatch indicates that the type check failed during call_indirect. + ErrRuntimeIndirectCallTypeMismatch = New("indirect call type mismatch") + // ErrRuntimeUnalignedAtomic indicates that an atomic operation was made with incorrect memory alignment. + ErrRuntimeUnalignedAtomic = New("unaligned atomic") + // ErrRuntimeExpectedSharedMemory indicates that an operation was made against unshared memory when not allowed. + ErrRuntimeExpectedSharedMemory = New("expected shared memory") + // ErrRuntimeTooManyWaiters indicates that atomic.wait was called with too many waiters. + ErrRuntimeTooManyWaiters = New("too many waiters") +) + +// Error is returned by a wasm.Engine during the execution of Wasm functions, and they indicate that the Wasm runtime +// state is unrecoverable. +type Error struct { + s string +} + +func New(text string) *Error { + return &Error{s: text} +} + +func (e *Error) Error() string { + return e.s +} diff --git a/vendor/github.com/tetratelabs/wazero/netlify.toml b/vendor/github.com/tetratelabs/wazero/netlify.toml new file mode 100644 index 000000000..1ba638bfe --- /dev/null +++ b/vendor/github.com/tetratelabs/wazero/netlify.toml @@ -0,0 +1,15 @@ +[build] + base = "site" + publish = "public" + +[build.environment] + HUGO_VERSION = "0.115.2" + +[context.production] + command = "git submodule update --init && hugo --gc --minify" + +[context.deploy-preview] + command = "git submodule update --init && hugo --gc --minify -b $DEPLOY_PRIME_URL" + +[context.branch-deploy] + command = "git submodule update --init && hugo --gc --minify -b $DEPLOY_PRIME_URL" diff --git a/vendor/github.com/tetratelabs/wazero/runtime.go b/vendor/github.com/tetratelabs/wazero/runtime.go new file mode 100644 index 000000000..d1f0a1a31 --- /dev/null +++ b/vendor/github.com/tetratelabs/wazero/runtime.go @@ -0,0 +1,374 @@ +package wazero + +import ( + "context" + "fmt" + "sync/atomic" + + "github.com/tetratelabs/wazero/api" + experimentalapi "github.com/tetratelabs/wazero/experimental" + "github.com/tetratelabs/wazero/internal/expctxkeys" + internalsock "github.com/tetratelabs/wazero/internal/sock" + internalsys "github.com/tetratelabs/wazero/internal/sys" + "github.com/tetratelabs/wazero/internal/wasm" + binaryformat "github.com/tetratelabs/wazero/internal/wasm/binary" + "github.com/tetratelabs/wazero/sys" +) + +// Runtime allows embedding of WebAssembly modules. +// +// The below is an example of basic initialization: +// +// ctx := context.Background() +// r := wazero.NewRuntime(ctx) +// defer r.Close(ctx) // This closes everything this Runtime created. +// +// mod, _ := r.Instantiate(ctx, wasm) +// +// # Notes +// +// - This is an interface for decoupling, not third-party implementations. +// All implementations are in wazero. +// - Closing this closes any CompiledModule or Module it instantiated. +type Runtime interface { + // Instantiate instantiates a module from the WebAssembly binary (%.wasm) + // with default configuration, which notably calls the "_start" function, + // if it exists. + // + // Here's an example: + // ctx := context.Background() + // r := wazero.NewRuntime(ctx) + // defer r.Close(ctx) // This closes everything this Runtime created. + // + // mod, _ := r.Instantiate(ctx, wasm) + // + // # Notes + // + // - See notes on InstantiateModule for error scenarios. + // - See InstantiateWithConfig for configuration overrides. + Instantiate(ctx context.Context, source []byte) (api.Module, error) + + // InstantiateWithConfig instantiates a module from the WebAssembly binary + // (%.wasm) or errs for reasons including exit or validation. + // + // Here's an example: + // ctx := context.Background() + // r := wazero.NewRuntime(ctx) + // defer r.Close(ctx) // This closes everything this Runtime created. + // + // mod, _ := r.InstantiateWithConfig(ctx, wasm, + // wazero.NewModuleConfig().WithName("rotate")) + // + // # Notes + // + // - See notes on InstantiateModule for error scenarios. + // - If you aren't overriding defaults, use Instantiate. + // - This is a convenience utility that chains CompileModule with + // InstantiateModule. To instantiate the same source multiple times, + // use CompileModule as InstantiateModule avoids redundant decoding + // and/or compilation. + InstantiateWithConfig(ctx context.Context, source []byte, config ModuleConfig) (api.Module, error) + + // NewHostModuleBuilder lets you create modules out of functions defined in Go. + // + // Below defines and instantiates a module named "env" with one function: + // + // ctx := context.Background() + // hello := func() { + // fmt.Fprintln(stdout, "hello!") + // } + // _, err := r.NewHostModuleBuilder("env"). + // NewFunctionBuilder().WithFunc(hello).Export("hello"). + // Instantiate(ctx, r) + // + // Note: empty `moduleName` is not allowed. + NewHostModuleBuilder(moduleName string) HostModuleBuilder + + // CompileModule decodes the WebAssembly binary (%.wasm) or errs if invalid. + // Any pre-compilation done after decoding wasm is dependent on RuntimeConfig. + // + // There are two main reasons to use CompileModule instead of Instantiate: + // - Improve performance when the same module is instantiated multiple times under different names + // - Reduce the amount of errors that can occur during InstantiateModule. + // + // # Notes + // + // - The resulting module name defaults to what was binary from the custom name section. + // - Any pre-compilation done after decoding the source is dependent on RuntimeConfig. + // + // See https://www.w3.org/TR/2019/REC-wasm-core-1-20191205/#name-section%E2%91%A0 + CompileModule(ctx context.Context, binary []byte) (CompiledModule, error) + + // InstantiateModule instantiates the module or errs for reasons including + // exit or validation. + // + // Here's an example: + // mod, _ := n.InstantiateModule(ctx, compiled, wazero.NewModuleConfig(). + // WithName("prod")) + // + // # Errors + // + // While CompiledModule is pre-validated, there are a few situations which + // can cause an error: + // - The module name is already in use. + // - The module has a table element initializer that resolves to an index + // outside the Table minimum size. + // - The module has a start function, and it failed to execute. + // - The module was compiled to WASI and exited with a non-zero exit + // code, you'll receive a sys.ExitError. + // - RuntimeConfig.WithCloseOnContextDone was enabled and a context + // cancellation or deadline triggered before a start function returned. + InstantiateModule(ctx context.Context, compiled CompiledModule, config ModuleConfig) (api.Module, error) + + // CloseWithExitCode closes all the modules that have been initialized in this Runtime with the provided exit code. + // An error is returned if any module returns an error when closed. + // + // Here's an example: + // ctx := context.Background() + // r := wazero.NewRuntime(ctx) + // defer r.CloseWithExitCode(ctx, 2) // This closes everything this Runtime created. + // + // // Everything below here can be closed, but will anyway due to above. + // _, _ = wasi_snapshot_preview1.InstantiateSnapshotPreview1(ctx, r) + // mod, _ := r.Instantiate(ctx, wasm) + CloseWithExitCode(ctx context.Context, exitCode uint32) error + + // Module returns an instantiated module in this runtime or nil if there aren't any. + Module(moduleName string) api.Module + + // Closer closes all compiled code by delegating to CloseWithExitCode with an exit code of zero. + api.Closer +} + +// NewRuntime returns a runtime with a configuration assigned by NewRuntimeConfig. +func NewRuntime(ctx context.Context) Runtime { + return NewRuntimeWithConfig(ctx, NewRuntimeConfig()) +} + +// NewRuntimeWithConfig returns a runtime with the given configuration. +func NewRuntimeWithConfig(ctx context.Context, rConfig RuntimeConfig) Runtime { + config := rConfig.(*runtimeConfig) + var engine wasm.Engine + var cacheImpl *cache + if c := config.cache; c != nil { + // If the Cache is configured, we share the engine. + cacheImpl = c.(*cache) + engine = cacheImpl.initEngine(config.engineKind, config.newEngine, ctx, config.enabledFeatures) + } else { + // Otherwise, we create a new engine. + engine = config.newEngine(ctx, config.enabledFeatures, nil) + } + store := wasm.NewStore(config.enabledFeatures, engine) + return &runtime{ + cache: cacheImpl, + store: store, + enabledFeatures: config.enabledFeatures, + memoryLimitPages: config.memoryLimitPages, + memoryCapacityFromMax: config.memoryCapacityFromMax, + dwarfDisabled: config.dwarfDisabled, + storeCustomSections: config.storeCustomSections, + ensureTermination: config.ensureTermination, + } +} + +// runtime allows decoupling of public interfaces from internal representation. +type runtime struct { + store *wasm.Store + cache *cache + enabledFeatures api.CoreFeatures + memoryLimitPages uint32 + memoryCapacityFromMax bool + dwarfDisabled bool + storeCustomSections bool + + // closed is the pointer used both to guard moduleEngine.CloseWithExitCode and to store the exit code. + // + // The update value is 1 + exitCode << 32. This ensures an exit code of zero isn't mistaken for never closed. + // + // Note: Exclusively reading and updating this with atomics guarantees cross-goroutine observations. + // See /RATIONALE.md + closed atomic.Uint64 + + ensureTermination bool +} + +// Module implements Runtime.Module. +func (r *runtime) Module(moduleName string) api.Module { + if len(moduleName) == 0 { + return nil + } + return r.store.Module(moduleName) +} + +// CompileModule implements Runtime.CompileModule +func (r *runtime) CompileModule(ctx context.Context, binary []byte) (CompiledModule, error) { + if err := r.failIfClosed(); err != nil { + return nil, err + } + + internal, err := binaryformat.DecodeModule(binary, r.enabledFeatures, + r.memoryLimitPages, r.memoryCapacityFromMax, !r.dwarfDisabled, r.storeCustomSections) + if err != nil { + return nil, err + } else if err = internal.Validate(r.enabledFeatures); err != nil { + // TODO: decoders should validate before returning, as that allows + // them to err with the correct position in the wasm binary. + return nil, err + } + + // Now that the module is validated, cache the memory definitions. + // TODO: lazy initialization of memory definition. + internal.BuildMemoryDefinitions() + + c := &compiledModule{module: internal, compiledEngine: r.store.Engine} + + // typeIDs are static and compile-time known. + typeIDs, err := r.store.GetFunctionTypeIDs(internal.TypeSection) + if err != nil { + return nil, err + } + c.typeIDs = typeIDs + + listeners, err := buildFunctionListeners(ctx, internal) + if err != nil { + return nil, err + } + internal.AssignModuleID(binary, listeners, r.ensureTermination) + if err = r.store.Engine.CompileModule(ctx, internal, listeners, r.ensureTermination); err != nil { + return nil, err + } + return c, nil +} + +func buildFunctionListeners(ctx context.Context, internal *wasm.Module) ([]experimentalapi.FunctionListener, error) { + // Test to see if internal code are using an experimental feature. + fnlf := ctx.Value(expctxkeys.FunctionListenerFactoryKey{}) + if fnlf == nil { + return nil, nil + } + factory := fnlf.(experimentalapi.FunctionListenerFactory) + importCount := internal.ImportFunctionCount + listeners := make([]experimentalapi.FunctionListener, len(internal.FunctionSection)) + for i := 0; i < len(listeners); i++ { + listeners[i] = factory.NewFunctionListener(internal.FunctionDefinition(uint32(i) + importCount)) + } + return listeners, nil +} + +// failIfClosed returns an error if CloseWithExitCode was called implicitly (by Close) or explicitly. +func (r *runtime) failIfClosed() error { + if closed := r.closed.Load(); closed != 0 { + return fmt.Errorf("runtime closed with exit_code(%d)", uint32(closed>>32)) + } + return nil +} + +// Instantiate implements Runtime.Instantiate +func (r *runtime) Instantiate(ctx context.Context, binary []byte) (api.Module, error) { + return r.InstantiateWithConfig(ctx, binary, NewModuleConfig()) +} + +// InstantiateWithConfig implements Runtime.InstantiateWithConfig +func (r *runtime) InstantiateWithConfig(ctx context.Context, binary []byte, config ModuleConfig) (api.Module, error) { + if compiled, err := r.CompileModule(ctx, binary); err != nil { + return nil, err + } else { + compiled.(*compiledModule).closeWithModule = true + return r.InstantiateModule(ctx, compiled, config) + } +} + +// InstantiateModule implements Runtime.InstantiateModule. +func (r *runtime) InstantiateModule( + ctx context.Context, + compiled CompiledModule, + mConfig ModuleConfig, +) (mod api.Module, err error) { + if err = r.failIfClosed(); err != nil { + return nil, err + } + + code := compiled.(*compiledModule) + config := mConfig.(*moduleConfig) + + // Only add guest module configuration to guests. + if !code.module.IsHostModule { + if sockConfig, ok := ctx.Value(internalsock.ConfigKey{}).(*internalsock.Config); ok { + config.sockConfig = sockConfig + } + } + + var sysCtx *internalsys.Context + if sysCtx, err = config.toSysContext(); err != nil { + return + } + + name := config.name + if !config.nameSet && code.module.NameSection != nil && code.module.NameSection.ModuleName != "" { + name = code.module.NameSection.ModuleName + } + + // Instantiate the module. + mod, err = r.store.Instantiate(ctx, code.module, name, sysCtx, code.typeIDs) + if err != nil { + // If there was an error, don't leak the compiled module. + if code.closeWithModule { + _ = code.Close(ctx) // don't overwrite the error + } + return + } + + if closeNotifier, ok := ctx.Value(expctxkeys.CloseNotifierKey{}).(experimentalapi.CloseNotifier); ok { + mod.(*wasm.ModuleInstance).CloseNotifier = closeNotifier + } + + // Attach the code closer so that anything afterward closes the compiled + // code when closing the module. + if code.closeWithModule { + mod.(*wasm.ModuleInstance).CodeCloser = code + } + + // Now, invoke any start functions, failing at first error. + for _, fn := range config.startFunctions { + start := mod.ExportedFunction(fn) + if start == nil { + continue + } + if _, err = start.Call(ctx); err != nil { + _ = mod.Close(ctx) // Don't leak the module on error. + + if se, ok := err.(*sys.ExitError); ok { + if se.ExitCode() == 0 { // Don't err on success. + err = nil + } + return // Don't wrap an exit error + } + err = fmt.Errorf("module[%s] function[%s] failed: %w", name, fn, err) + return + } + } + return +} + +// Close implements api.Closer embedded in Runtime. +func (r *runtime) Close(ctx context.Context) error { + return r.CloseWithExitCode(ctx, 0) +} + +// CloseWithExitCode implements Runtime.CloseWithExitCode +// +// Note: it also marks the internal `closed` field +func (r *runtime) CloseWithExitCode(ctx context.Context, exitCode uint32) error { + closed := uint64(1) + uint64(exitCode)<<32 // Store exitCode as high-order bits. + if !r.closed.CompareAndSwap(0, closed) { + return nil + } + err := r.store.CloseWithExitCode(ctx, exitCode) + if r.cache == nil { + // Close the engine if the cache is not configured, which means that this engine is scoped in this runtime. + if errCloseEngine := r.store.Engine.Close(); errCloseEngine != nil { + return errCloseEngine + } + } + return err +} diff --git a/vendor/github.com/tetratelabs/wazero/sys/clock.go b/vendor/github.com/tetratelabs/wazero/sys/clock.go new file mode 100644 index 000000000..1c91ce246 --- /dev/null +++ b/vendor/github.com/tetratelabs/wazero/sys/clock.go @@ -0,0 +1,26 @@ +package sys + +// ClockResolution is a positive granularity of clock precision in +// nanoseconds. For example, if the resolution is 1us, this returns 1000. +// +// Note: Some implementations return arbitrary resolution because there's +// no perfect alternative. For example, according to the source in time.go, +// windows monotonic resolution can be 15ms. See /RATIONALE.md. +type ClockResolution uint32 + +// Walltime returns the current unix/epoch time, seconds since midnight UTC +// 1 January 1970, with a nanosecond fraction. +type Walltime func() (sec int64, nsec int32) + +// Nanotime returns nanoseconds since an arbitrary start point, used to measure +// elapsed time. This is sometimes referred to as a tick or monotonic time. +// +// Note: There are no constraints on the value return except that it +// increments. For example, -1 is a valid if the next value is >= 0. +type Nanotime func() int64 + +// Nanosleep puts the current goroutine to sleep for at least ns nanoseconds. +type Nanosleep func(ns int64) + +// Osyield yields the processor, typically to implement spin-wait loops. +type Osyield func() diff --git a/vendor/github.com/tetratelabs/wazero/sys/error.go b/vendor/github.com/tetratelabs/wazero/sys/error.go new file mode 100644 index 000000000..c3efbad96 --- /dev/null +++ b/vendor/github.com/tetratelabs/wazero/sys/error.go @@ -0,0 +1,83 @@ +// Package sys includes constants and types used by both public and internal APIs. +package sys + +import ( + "context" + "fmt" +) + +// These two special exit codes are reserved by wazero for context Cancel and Timeout integrations. +// The assumption here is that well-behaving Wasm programs won't use these two exit codes. +const ( + // ExitCodeContextCanceled corresponds to context.Canceled and returned by ExitError.ExitCode in that case. + ExitCodeContextCanceled uint32 = 0xffffffff + // ExitCodeDeadlineExceeded corresponds to context.DeadlineExceeded and returned by ExitError.ExitCode in that case. + ExitCodeDeadlineExceeded uint32 = 0xefffffff +) + +// ExitError is returned to a caller of api.Function when api.Module CloseWithExitCode was invoked, +// or context.Context passed to api.Function Call was canceled or reached the Timeout. +// +// ExitCode zero value means success while any other value is an error. +// +// Here's an example of how to get the exit code: +// +// main := module.ExportedFunction("main") +// if err := main(ctx); err != nil { +// if exitErr, ok := err.(*sys.ExitError); ok { +// // This means your module exited with non-zero code! +// } +// --snip-- +// +// Note: While possible the reason of this was "proc_exit" from "wasi_snapshot_preview1", it could be from other host +// functions, for example an AssemblyScript's abort handler, or any arbitrary caller of CloseWithExitCode. +// +// See https://github.com/WebAssembly/WASI/blob/main/phases/snapshot/docs.md#proc_exit and +// https://www.assemblyscript.org/concepts.html#special-imports +// +// Note: In the case of context cancellation or timeout, the api.Module from which the api.Function created is closed. +type ExitError struct { + // Note: this is a struct not a uint32 type as it was originally one and + // we don't want to break call-sites that cast into it. + exitCode uint32 +} + +var exitZero = &ExitError{} + +func NewExitError(exitCode uint32) *ExitError { + if exitCode == 0 { + return exitZero + } + return &ExitError{exitCode: exitCode} +} + +// ExitCode returns zero on success, and an arbitrary value otherwise. +func (e *ExitError) ExitCode() uint32 { + return e.exitCode +} + +// Error implements the error interface. +func (e *ExitError) Error() string { + switch e.exitCode { + case ExitCodeContextCanceled: + return fmt.Sprintf("module closed with %s", context.Canceled) + case ExitCodeDeadlineExceeded: + return fmt.Sprintf("module closed with %s", context.DeadlineExceeded) + default: + return fmt.Sprintf("module closed with exit_code(%d)", e.exitCode) + } +} + +// Is allows use via errors.Is +func (e *ExitError) Is(err error) bool { + if target, ok := err.(*ExitError); ok { + return e.exitCode == target.exitCode + } + if e.exitCode == ExitCodeContextCanceled && err == context.Canceled { + return true + } + if e.exitCode == ExitCodeDeadlineExceeded && err == context.DeadlineExceeded { + return true + } + return false +} diff --git a/vendor/github.com/tetratelabs/wazero/sys/stat.go b/vendor/github.com/tetratelabs/wazero/sys/stat.go new file mode 100644 index 000000000..bb7b9e5d3 --- /dev/null +++ b/vendor/github.com/tetratelabs/wazero/sys/stat.go @@ -0,0 +1,107 @@ +package sys + +import "io/fs" + +// Inode is the file serial number, or zero if unknown. +// +// Any constant value will invalidate functions that use this for +// equivalence, such as os.SameFile (Stat_t.Ino). +// +// When zero is returned by a `readdir`, some compilers will attempt to +// get a non-zero value with `lstat`. Those using this for darwin's definition +// of `getdirentries` conflate zero `d_fileno` with a deleted file, so skip the +// entry. See /RATIONALE.md for more on this. +type Inode = uint64 + +// ^-- Inode is a type alias to consolidate documentation and aid in reference +// searches. While only Stat_t is exposed publicly at the moment, this is used +// internally for Dirent and several function return values. + +// EpochNanos is a timestamp in epoch nanoseconds, or zero if unknown. +// +// This defines epoch time the same way as Walltime, except this value is +// packed into an int64. Common conversions are detailed in the examples. +type EpochNanos = int64 + +// Stat_t is similar to syscall.Stat_t, except available on all operating +// systems, including Windows. +// +// # Notes +// +// - This is used for WebAssembly ABI emulating the POSIX `stat` system call. +// Fields included are required for WebAssembly ABI including wasip1 +// (a.k.a. wasix) and wasi-filesystem (a.k.a. wasip2). See +// https://pubs.opengroup.org/onlinepubs/9699919799/functions/stat.html +// - Fields here are required for WebAssembly ABI including wasip1 +// (a.k.a. wasix) and wasi-filesystem (a.k.a. wasip2). +// - This isn't the same as syscall.Stat_t because wazero supports Windows, +// which doesn't have that type. runtime.GOOS that has this already also +// have inconsistent field lengths, which complicates wasm binding. +// - Use NewStat_t to create this from an existing fs.FileInfo. +// - For portability, numeric fields are 64-bit when at least one platform +// defines it that large. +type Stat_t struct { + // Dev is the device ID of device containing the file. + Dev uint64 + + // Ino is the file serial number, or zero if not available. See Inode for + // more details including impact returning a zero value. + Ino Inode + + // Mode is the same as Mode on fs.FileInfo containing bits to identify the + // type of the file (fs.ModeType) and its permissions (fs.ModePerm). + Mode fs.FileMode + + // Nlink is the number of hard links to the file. + // + // Note: This value is platform-specific and often at least one. Linux will + // return 1+N for a directory, where BSD (like Darwin) return 2+N, which + // includes the dot entry. + Nlink uint64 + + // Size is the length in bytes for regular files. For symbolic links, this + // is length in bytes of the pathname contained in the symbolic link. + Size int64 + + // Atim is the last data access timestamp in epoch nanoseconds. + Atim EpochNanos + + // Mtim is the last data modification timestamp in epoch nanoseconds. + Mtim EpochNanos + + // Ctim is the last file status change timestamp in epoch nanoseconds. + Ctim EpochNanos +} + +// NewStat_t fills a new Stat_t from `info`, including any runtime.GOOS-specific +// details from fs.FileInfo `Sys`. When `Sys` is already a *Stat_t, it is +// returned as-is. +// +// # Notes +// +// - When already in fs.FileInfo `Sys`, Stat_t must be a pointer. +// - When runtime.GOOS is "windows" Stat_t.Ino will be zero. +// - When fs.FileInfo `Sys` is nil or unknown, some fields not in fs.FileInfo +// are defaulted: Stat_t.Atim and Stat_t.Ctim are set to `ModTime`, and +// are set to ModTime and Stat_t.Nlink is set to 1. +func NewStat_t(info fs.FileInfo) Stat_t { + // Note: Pointer, not val, for parity with Go, which sets *syscall.Stat_t + if st, ok := info.Sys().(*Stat_t); ok { + return *st + } + return statFromFileInfo(info) +} + +func defaultStatFromFileInfo(info fs.FileInfo) Stat_t { + st := Stat_t{} + st.Ino = 0 + st.Dev = 0 + st.Mode = info.Mode() + st.Nlink = 1 + st.Size = info.Size() + mtim := info.ModTime().UnixNano() // Set all times to the mod time + st.Atim = mtim + st.Mtim = mtim + st.Ctim = mtim + return st +} diff --git a/vendor/github.com/tetratelabs/wazero/sys/stat_bsd.go b/vendor/github.com/tetratelabs/wazero/sys/stat_bsd.go new file mode 100644 index 000000000..3bf9b5d14 --- /dev/null +++ b/vendor/github.com/tetratelabs/wazero/sys/stat_bsd.go @@ -0,0 +1,29 @@ +//go:build (amd64 || arm64) && (darwin || freebsd) + +package sys + +import ( + "io/fs" + "syscall" +) + +const sysParseable = true + +func statFromFileInfo(info fs.FileInfo) Stat_t { + if d, ok := info.Sys().(*syscall.Stat_t); ok { + st := Stat_t{} + st.Dev = uint64(d.Dev) + st.Ino = d.Ino + st.Mode = info.Mode() + st.Nlink = uint64(d.Nlink) + st.Size = d.Size + atime := d.Atimespec + st.Atim = atime.Sec*1e9 + atime.Nsec + mtime := d.Mtimespec + st.Mtim = mtime.Sec*1e9 + mtime.Nsec + ctime := d.Ctimespec + st.Ctim = ctime.Sec*1e9 + ctime.Nsec + return st + } + return defaultStatFromFileInfo(info) +} diff --git a/vendor/github.com/tetratelabs/wazero/sys/stat_linux.go b/vendor/github.com/tetratelabs/wazero/sys/stat_linux.go new file mode 100644 index 000000000..9b5e20e8d --- /dev/null +++ b/vendor/github.com/tetratelabs/wazero/sys/stat_linux.go @@ -0,0 +1,32 @@ +//go:build (amd64 || arm64 || riscv64) && linux + +// Note: This expression is not the same as compiler support, even if it looks +// similar. Platform functions here are used in interpreter mode as well. + +package sys + +import ( + "io/fs" + "syscall" +) + +const sysParseable = true + +func statFromFileInfo(info fs.FileInfo) Stat_t { + if d, ok := info.Sys().(*syscall.Stat_t); ok { + st := Stat_t{} + st.Dev = uint64(d.Dev) + st.Ino = uint64(d.Ino) + st.Mode = info.Mode() + st.Nlink = uint64(d.Nlink) + st.Size = d.Size + atime := d.Atim + st.Atim = atime.Sec*1e9 + atime.Nsec + mtime := d.Mtim + st.Mtim = mtime.Sec*1e9 + mtime.Nsec + ctime := d.Ctim + st.Ctim = ctime.Sec*1e9 + ctime.Nsec + return st + } + return defaultStatFromFileInfo(info) +} diff --git a/vendor/github.com/tetratelabs/wazero/sys/stat_unsupported.go b/vendor/github.com/tetratelabs/wazero/sys/stat_unsupported.go new file mode 100644 index 000000000..583c2adb0 --- /dev/null +++ b/vendor/github.com/tetratelabs/wazero/sys/stat_unsupported.go @@ -0,0 +1,17 @@ +//go:build (!((amd64 || arm64 || riscv64) && linux) && !((amd64 || arm64) && (darwin || freebsd)) && !((amd64 || arm64) && windows)) || js + +package sys + +import "io/fs" + +// sysParseable is only used here as we define "supported" as being able to +// parse `info.Sys()`. The above `go:build` constraints exclude 32-bit until +// that's requested. +// +// TODO: When Go 1.21 is out, use the "unix" build constraint (as 1.21 makes +// our floor Go version 1.19. +const sysParseable = false + +func statFromFileInfo(info fs.FileInfo) Stat_t { + return defaultStatFromFileInfo(info) +} diff --git a/vendor/github.com/tetratelabs/wazero/sys/stat_windows.go b/vendor/github.com/tetratelabs/wazero/sys/stat_windows.go new file mode 100644 index 000000000..1a7070f48 --- /dev/null +++ b/vendor/github.com/tetratelabs/wazero/sys/stat_windows.go @@ -0,0 +1,26 @@ +//go:build (amd64 || arm64) && windows + +package sys + +import ( + "io/fs" + "syscall" +) + +const sysParseable = true + +func statFromFileInfo(info fs.FileInfo) Stat_t { + if d, ok := info.Sys().(*syscall.Win32FileAttributeData); ok { + st := Stat_t{} + st.Ino = 0 // not in Win32FileAttributeData + st.Dev = 0 // not in Win32FileAttributeData + st.Mode = info.Mode() + st.Nlink = 1 // not in Win32FileAttributeData + st.Size = info.Size() + st.Atim = d.LastAccessTime.Nanoseconds() + st.Mtim = d.LastWriteTime.Nanoseconds() + st.Ctim = d.CreationTime.Nanoseconds() + return st + } + return defaultStatFromFileInfo(info) +} diff --git a/vendor/modules.txt b/vendor/modules.txt index 6c766fe69..c9687d03e 100644 --- a/vendor/modules.txt +++ b/vendor/modules.txt @@ -517,9 +517,21 @@ github.com/modern-go/concurrent # github.com/modern-go/reflect2 v1.0.2 ## explicit; go 1.12 github.com/modern-go/reflect2 +# github.com/ncruces/go-sqlite3 v0.16.0 +## explicit; go 1.21 +github.com/ncruces/go-sqlite3 +github.com/ncruces/go-sqlite3/driver +github.com/ncruces/go-sqlite3/embed +github.com/ncruces/go-sqlite3/internal/util +github.com/ncruces/go-sqlite3/util/osutil +github.com/ncruces/go-sqlite3/vfs +github.com/ncruces/go-sqlite3/vfs/memdb # github.com/ncruces/go-strftime v0.1.9 ## explicit; go 1.17 github.com/ncruces/go-strftime +# github.com/ncruces/julianday v1.0.0 +## explicit; go 1.17 +github.com/ncruces/julianday # github.com/oklog/ulid v1.3.1 ## explicit github.com/oklog/ulid @@ -820,6 +832,41 @@ github.com/tdewolff/parse/v2/strconv # github.com/technologize/otel-go-contrib v1.1.1 ## explicit; go 1.17 github.com/technologize/otel-go-contrib/otelginmetrics +# github.com/tetratelabs/wazero v1.7.2 +## explicit; go 1.20 +github.com/tetratelabs/wazero +github.com/tetratelabs/wazero/api +github.com/tetratelabs/wazero/experimental +github.com/tetratelabs/wazero/experimental/sys +github.com/tetratelabs/wazero/internal/descriptor +github.com/tetratelabs/wazero/internal/engine/interpreter +github.com/tetratelabs/wazero/internal/engine/wazevo +github.com/tetratelabs/wazero/internal/engine/wazevo/backend +github.com/tetratelabs/wazero/internal/engine/wazevo/backend/isa/amd64 +github.com/tetratelabs/wazero/internal/engine/wazevo/backend/isa/arm64 +github.com/tetratelabs/wazero/internal/engine/wazevo/backend/regalloc +github.com/tetratelabs/wazero/internal/engine/wazevo/frontend +github.com/tetratelabs/wazero/internal/engine/wazevo/ssa +github.com/tetratelabs/wazero/internal/engine/wazevo/wazevoapi +github.com/tetratelabs/wazero/internal/expctxkeys +github.com/tetratelabs/wazero/internal/filecache +github.com/tetratelabs/wazero/internal/fsapi +github.com/tetratelabs/wazero/internal/ieee754 +github.com/tetratelabs/wazero/internal/internalapi +github.com/tetratelabs/wazero/internal/leb128 +github.com/tetratelabs/wazero/internal/moremath +github.com/tetratelabs/wazero/internal/platform +github.com/tetratelabs/wazero/internal/sock +github.com/tetratelabs/wazero/internal/sys +github.com/tetratelabs/wazero/internal/sysfs +github.com/tetratelabs/wazero/internal/u32 +github.com/tetratelabs/wazero/internal/u64 +github.com/tetratelabs/wazero/internal/version +github.com/tetratelabs/wazero/internal/wasm +github.com/tetratelabs/wazero/internal/wasm/binary +github.com/tetratelabs/wazero/internal/wasmdebug +github.com/tetratelabs/wazero/internal/wasmruntime +github.com/tetratelabs/wazero/sys # github.com/tmthrgd/go-hex v0.0.0-20190904060850-447a3041c3bc ## explicit github.com/tmthrgd/go-hex @@ -1270,7 +1317,7 @@ modernc.org/mathutil # modernc.org/memory v1.8.0 ## explicit; go 1.18 modernc.org/memory -# modernc.org/sqlite v1.29.8 => gitlab.com/NyaaaWhatsUpDoc/sqlite v1.29.9-concurrency-workaround +# modernc.org/sqlite v0.0.0-00010101000000-000000000000 => gitlab.com/NyaaaWhatsUpDoc/sqlite v1.29.9-concurrency-workaround ## explicit; go 1.20 modernc.org/sqlite modernc.org/sqlite/lib