Merge branch 'main' into 2fa

This commit is contained in:
tobi 2025-04-07 12:57:02 +02:00
commit c6c212fb81
271 changed files with 7226 additions and 11019 deletions

View file

@ -7,7 +7,6 @@ linters:
- dogsled
- dupl
- errcheck
- exportloopref
- exhaustive
- gochecknoinits
- goconst

View file

@ -2,6 +2,7 @@ package cors
import (
"net/http"
"regexp"
"strings"
"github.com/gin-gonic/gin"
@ -122,21 +123,32 @@ func (cors *cors) isOriginValid(c *gin.Context, origin string) bool {
return valid
}
var originRegex = regexp.MustCompile(`^/(.+)/[gimuy]?$`)
func (cors *cors) validateOrigin(origin string) bool {
if cors.allowAllOrigins {
return true
}
for _, value := range cors.allowOrigins {
if value == origin {
if !originRegex.MatchString(value) && value == origin {
return true
}
if originRegex.MatchString(value) &&
regexp.MustCompile(originRegex.FindStringSubmatch(value)[1]).MatchString(origin) {
return true
}
}
if len(cors.wildcardOrigins) > 0 && cors.validateWildcardOrigin(origin) {
return true
}
if cors.allowOriginFunc != nil {
return cors.allowOriginFunc(origin)
}
return false
}

View file

@ -3,6 +3,7 @@ package cors
import (
"errors"
"fmt"
"regexp"
"strings"
"time"
@ -103,8 +104,17 @@ func (c Config) getAllowedSchemas() []string {
return allowedSchemas
}
var regexpBasedOrigin = regexp.MustCompile(`^\/(.+)\/[gimuy]?$`)
func (c Config) validateAllowedSchemas(origin string) bool {
allowedSchemas := c.getAllowedSchemas()
if regexpBasedOrigin.MatchString(origin) {
// Normalize regexp-based origins
origin = regexpBasedOrigin.FindStringSubmatch(origin)[1]
origin = strings.Replace(origin, "?", "", 1)
}
for _, schema := range allowedSchemas {
if strings.HasPrefix(origin, schema) {
return true

View file

@ -1,5 +1,4 @@
# This is an example goreleaser.yaml file with some sane defaults.
# Make sure to check the documentation at http://goreleaser.com
version: 2
builds:
-
@ -27,16 +26,7 @@ builds:
archives:
-
id: cpuid
name_template: "cpuid-{{ .Os }}_{{ .Arch }}_{{ .Version }}"
replacements:
aix: AIX
darwin: OSX
linux: Linux
windows: Windows
386: i386
amd64: x86_64
freebsd: FreeBSD
netbsd: NetBSD
name_template: "cpuid-{{ .Os }}_{{ .Arch }}{{ if .Arm }}v{{ .Arm }}{{ end }}"
format_overrides:
- goos: windows
format: zip
@ -44,8 +34,6 @@ archives:
- LICENSE
checksum:
name_template: 'checksums.txt'
snapshot:
name_template: "{{ .Tag }}-next"
changelog:
sort: asc
filters:
@ -58,7 +46,7 @@ changelog:
nfpms:
-
file_name_template: "cpuid_package_{{ .Version }}_{{ .Os }}_{{ .Arch }}"
file_name_template: "cpuid_package_{{ .Os }}_{{ .Arch }}{{ if .Arm }}v{{ .Arm }}{{ end }}"
vendor: Klaus Post
homepage: https://github.com/klauspost/cpuid
maintainer: Klaus Post <klauspost@gmail.com>
@ -67,8 +55,3 @@ nfpms:
formats:
- deb
- rpm
replacements:
darwin: Darwin
linux: Linux
freebsd: FreeBSD
amd64: x86_64

View file

@ -282,7 +282,9 @@ Exit Code 1
| AMXINT8 | Tile computational operations on 8-bit integers |
| AMXFP16 | Tile computational operations on FP16 numbers |
| AMXFP8 | Tile computational operations on FP8 numbers |
| AMXCOMPLEX | Tile computational operations on complex numbers |
| AMXTILE | Tile architecture |
| AMXTF32 | Matrix Multiplication of TF32 Tiles into Packed Single Precision Tile |
| APX_F | Intel APX |
| AVX | AVX functions |
| AVX10 | If set the Intel AVX10 Converged Vector ISA is supported |
@ -480,12 +482,16 @@ Exit Code 1
| DCPOP | Data cache clean to Point of Persistence (DC CVAP) |
| EVTSTRM | Generic timer |
| FCMA | Floatin point complex number addition and multiplication |
| FHM | FMLAL and FMLSL instructions |
| FP | Single-precision and double-precision floating point |
| FPHP | Half-precision floating point |
| GPA | Generic Pointer Authentication |
| JSCVT | Javascript-style double->int convert (FJCVTZS) |
| LRCPC | Weaker release consistency (LDAPR, etc) |
| PMULL | Polynomial Multiply instructions (PMULL/PMULL2) |
| RNDR | Random Number instructions |
| TLB | Outer Shareable and TLB range maintenance instructions |
| TS | Flag manipulation instructions |
| SHA1 | SHA-1 instructions (SHA1C, etc) |
| SHA2 | SHA-2 instructions (SHA256H, etc) |
| SHA3 | SHA-3 instructions (EOR3, RAXI, XAR, BCAX) |

View file

@ -83,6 +83,8 @@ const (
AMXINT8 // Tile computational operations on 8-bit integers
AMXFP8 // Tile computational operations on FP8 numbers
AMXTILE // Tile architecture
AMXTF32 // Tile architecture
AMXCOMPLEX // Matrix Multiplication of TF32 Tiles into Packed Single Precision Tile
APX_F // Intel APX
AVX // AVX functions
AVX10 // If set the Intel AVX10 Converged Vector ISA is supported
@ -282,12 +284,16 @@ const (
DCPOP // Data cache clean to Point of Persistence (DC CVAP)
EVTSTRM // Generic timer
FCMA // Floatin point complex number addition and multiplication
FHM // FMLAL and FMLSL instructions
FP // Single-precision and double-precision floating point
FPHP // Half-precision floating point
GPA // Generic Pointer Authentication
JSCVT // Javascript-style double->int convert (FJCVTZS)
LRCPC // Weaker release consistency (LDAPR, etc)
PMULL // Polynomial Multiply instructions (PMULL/PMULL2)
RNDR // Random Number instructions
TLB // Outer Shareable and TLB range maintenance instructions
TS // Flag manipulation instructions
SHA1 // SHA-1 instructions (SHA1C, etc)
SHA2 // SHA-2 instructions (SHA256H, etc)
SHA3 // SHA-3 instructions (EOR3, RAXI, XAR, BCAX)
@ -532,7 +538,7 @@ func (c CPUInfo) Ia32TscAux() uint32 {
return ecx
}
// SveLengths returns arm SVE vector and predicate lengths.
// SveLengths returns arm SVE vector and predicate lengths in bits.
// Will return 0, 0 if SVE is not enabled or otherwise unable to detect.
func (c CPUInfo) SveLengths() (vl, pl uint64) {
if !c.Has(SVE) {
@ -1284,6 +1290,8 @@ func support() flagSet {
// CPUID.(EAX=7, ECX=1).EDX
fs.setIf(edx1&(1<<4) != 0, AVXVNNIINT8)
fs.setIf(edx1&(1<<5) != 0, AVXNECONVERT)
fs.setIf(edx1&(1<<7) != 0, AMXTF32)
fs.setIf(edx1&(1<<8) != 0, AMXCOMPLEX)
fs.setIf(edx1&(1<<10) != 0, AVXVNNIINT16)
fs.setIf(edx1&(1<<14) != 0, PREFETCHI)
fs.setIf(edx1&(1<<19) != 0, AVX10)

View file

@ -157,6 +157,10 @@ func addInfo(c *CPUInfo, safe bool) {
// x--------------------------------------------------x
// | Name | bits | visible |
// |--------------------------------------------------|
// | RNDR | [63-60] | y |
// |--------------------------------------------------|
// | TLB | [59-56] | y |
// |--------------------------------------------------|
// | TS | [55-52] | y |
// |--------------------------------------------------|
// | FHM | [51-48] | y |
@ -182,12 +186,10 @@ func addInfo(c *CPUInfo, safe bool) {
// | AES | [7-4] | y |
// x--------------------------------------------------x
// if instAttrReg0&(0xf<<52) != 0 {
// fmt.Println("TS")
// }
// if instAttrReg0&(0xf<<48) != 0 {
// fmt.Println("FHM")
// }
f.setIf(instAttrReg0&(0xf<<60) != 0, RNDR)
f.setIf(instAttrReg0&(0xf<<56) != 0, TLB)
f.setIf(instAttrReg0&(0xf<<52) != 0, TS)
f.setIf(instAttrReg0&(0xf<<48) != 0, FHM)
f.setIf(instAttrReg0&(0xf<<44) != 0, ASIMDDP)
f.setIf(instAttrReg0&(0xf<<40) != 0, SM4)
f.setIf(instAttrReg0&(0xf<<36) != 0, SM3)

View file

@ -17,223 +17,229 @@ func _() {
_ = x[AMXINT8-7]
_ = x[AMXFP8-8]
_ = x[AMXTILE-9]
_ = x[APX_F-10]
_ = x[AVX-11]
_ = x[AVX10-12]
_ = x[AVX10_128-13]
_ = x[AVX10_256-14]
_ = x[AVX10_512-15]
_ = x[AVX2-16]
_ = x[AVX512BF16-17]
_ = x[AVX512BITALG-18]
_ = x[AVX512BW-19]
_ = x[AVX512CD-20]
_ = x[AVX512DQ-21]
_ = x[AVX512ER-22]
_ = x[AVX512F-23]
_ = x[AVX512FP16-24]
_ = x[AVX512IFMA-25]
_ = x[AVX512PF-26]
_ = x[AVX512VBMI-27]
_ = x[AVX512VBMI2-28]
_ = x[AVX512VL-29]
_ = x[AVX512VNNI-30]
_ = x[AVX512VP2INTERSECT-31]
_ = x[AVX512VPOPCNTDQ-32]
_ = x[AVXIFMA-33]
_ = x[AVXNECONVERT-34]
_ = x[AVXSLOW-35]
_ = x[AVXVNNI-36]
_ = x[AVXVNNIINT8-37]
_ = x[AVXVNNIINT16-38]
_ = x[BHI_CTRL-39]
_ = x[BMI1-40]
_ = x[BMI2-41]
_ = x[CETIBT-42]
_ = x[CETSS-43]
_ = x[CLDEMOTE-44]
_ = x[CLMUL-45]
_ = x[CLZERO-46]
_ = x[CMOV-47]
_ = x[CMPCCXADD-48]
_ = x[CMPSB_SCADBS_SHORT-49]
_ = x[CMPXCHG8-50]
_ = x[CPBOOST-51]
_ = x[CPPC-52]
_ = x[CX16-53]
_ = x[EFER_LMSLE_UNS-54]
_ = x[ENQCMD-55]
_ = x[ERMS-56]
_ = x[F16C-57]
_ = x[FLUSH_L1D-58]
_ = x[FMA3-59]
_ = x[FMA4-60]
_ = x[FP128-61]
_ = x[FP256-62]
_ = x[FSRM-63]
_ = x[FXSR-64]
_ = x[FXSROPT-65]
_ = x[GFNI-66]
_ = x[HLE-67]
_ = x[HRESET-68]
_ = x[HTT-69]
_ = x[HWA-70]
_ = x[HYBRID_CPU-71]
_ = x[HYPERVISOR-72]
_ = x[IA32_ARCH_CAP-73]
_ = x[IA32_CORE_CAP-74]
_ = x[IBPB-75]
_ = x[IBPB_BRTYPE-76]
_ = x[IBRS-77]
_ = x[IBRS_PREFERRED-78]
_ = x[IBRS_PROVIDES_SMP-79]
_ = x[IBS-80]
_ = x[IBSBRNTRGT-81]
_ = x[IBSFETCHSAM-82]
_ = x[IBSFFV-83]
_ = x[IBSOPCNT-84]
_ = x[IBSOPCNTEXT-85]
_ = x[IBSOPSAM-86]
_ = x[IBSRDWROPCNT-87]
_ = x[IBSRIPINVALIDCHK-88]
_ = x[IBS_FETCH_CTLX-89]
_ = x[IBS_OPDATA4-90]
_ = x[IBS_OPFUSE-91]
_ = x[IBS_PREVENTHOST-92]
_ = x[IBS_ZEN4-93]
_ = x[IDPRED_CTRL-94]
_ = x[INT_WBINVD-95]
_ = x[INVLPGB-96]
_ = x[KEYLOCKER-97]
_ = x[KEYLOCKERW-98]
_ = x[LAHF-99]
_ = x[LAM-100]
_ = x[LBRVIRT-101]
_ = x[LZCNT-102]
_ = x[MCAOVERFLOW-103]
_ = x[MCDT_NO-104]
_ = x[MCOMMIT-105]
_ = x[MD_CLEAR-106]
_ = x[MMX-107]
_ = x[MMXEXT-108]
_ = x[MOVBE-109]
_ = x[MOVDIR64B-110]
_ = x[MOVDIRI-111]
_ = x[MOVSB_ZL-112]
_ = x[MOVU-113]
_ = x[MPX-114]
_ = x[MSRIRC-115]
_ = x[MSRLIST-116]
_ = x[MSR_PAGEFLUSH-117]
_ = x[NRIPS-118]
_ = x[NX-119]
_ = x[OSXSAVE-120]
_ = x[PCONFIG-121]
_ = x[POPCNT-122]
_ = x[PPIN-123]
_ = x[PREFETCHI-124]
_ = x[PSFD-125]
_ = x[RDPRU-126]
_ = x[RDRAND-127]
_ = x[RDSEED-128]
_ = x[RDTSCP-129]
_ = x[RRSBA_CTRL-130]
_ = x[RTM-131]
_ = x[RTM_ALWAYS_ABORT-132]
_ = x[SBPB-133]
_ = x[SERIALIZE-134]
_ = x[SEV-135]
_ = x[SEV_64BIT-136]
_ = x[SEV_ALTERNATIVE-137]
_ = x[SEV_DEBUGSWAP-138]
_ = x[SEV_ES-139]
_ = x[SEV_RESTRICTED-140]
_ = x[SEV_SNP-141]
_ = x[SGX-142]
_ = x[SGXLC-143]
_ = x[SHA-144]
_ = x[SME-145]
_ = x[SME_COHERENT-146]
_ = x[SPEC_CTRL_SSBD-147]
_ = x[SRBDS_CTRL-148]
_ = x[SRSO_MSR_FIX-149]
_ = x[SRSO_NO-150]
_ = x[SRSO_USER_KERNEL_NO-151]
_ = x[SSE-152]
_ = x[SSE2-153]
_ = x[SSE3-154]
_ = x[SSE4-155]
_ = x[SSE42-156]
_ = x[SSE4A-157]
_ = x[SSSE3-158]
_ = x[STIBP-159]
_ = x[STIBP_ALWAYSON-160]
_ = x[STOSB_SHORT-161]
_ = x[SUCCOR-162]
_ = x[SVM-163]
_ = x[SVMDA-164]
_ = x[SVMFBASID-165]
_ = x[SVML-166]
_ = x[SVMNP-167]
_ = x[SVMPF-168]
_ = x[SVMPFT-169]
_ = x[SYSCALL-170]
_ = x[SYSEE-171]
_ = x[TBM-172]
_ = x[TDX_GUEST-173]
_ = x[TLB_FLUSH_NESTED-174]
_ = x[TME-175]
_ = x[TOPEXT-176]
_ = x[TSCRATEMSR-177]
_ = x[TSXLDTRK-178]
_ = x[VAES-179]
_ = x[VMCBCLEAN-180]
_ = x[VMPL-181]
_ = x[VMSA_REGPROT-182]
_ = x[VMX-183]
_ = x[VPCLMULQDQ-184]
_ = x[VTE-185]
_ = x[WAITPKG-186]
_ = x[WBNOINVD-187]
_ = x[WRMSRNS-188]
_ = x[X87-189]
_ = x[XGETBV1-190]
_ = x[XOP-191]
_ = x[XSAVE-192]
_ = x[XSAVEC-193]
_ = x[XSAVEOPT-194]
_ = x[XSAVES-195]
_ = x[AESARM-196]
_ = x[ARMCPUID-197]
_ = x[ASIMD-198]
_ = x[ASIMDDP-199]
_ = x[ASIMDHP-200]
_ = x[ASIMDRDM-201]
_ = x[ATOMICS-202]
_ = x[CRC32-203]
_ = x[DCPOP-204]
_ = x[EVTSTRM-205]
_ = x[FCMA-206]
_ = x[FP-207]
_ = x[FPHP-208]
_ = x[GPA-209]
_ = x[JSCVT-210]
_ = x[LRCPC-211]
_ = x[PMULL-212]
_ = x[SHA1-213]
_ = x[SHA2-214]
_ = x[SHA3-215]
_ = x[SHA512-216]
_ = x[SM3-217]
_ = x[SM4-218]
_ = x[SVE-219]
_ = x[lastID-220]
_ = x[AMXTF32-10]
_ = x[AMXCOMPLEX-11]
_ = x[APX_F-12]
_ = x[AVX-13]
_ = x[AVX10-14]
_ = x[AVX10_128-15]
_ = x[AVX10_256-16]
_ = x[AVX10_512-17]
_ = x[AVX2-18]
_ = x[AVX512BF16-19]
_ = x[AVX512BITALG-20]
_ = x[AVX512BW-21]
_ = x[AVX512CD-22]
_ = x[AVX512DQ-23]
_ = x[AVX512ER-24]
_ = x[AVX512F-25]
_ = x[AVX512FP16-26]
_ = x[AVX512IFMA-27]
_ = x[AVX512PF-28]
_ = x[AVX512VBMI-29]
_ = x[AVX512VBMI2-30]
_ = x[AVX512VL-31]
_ = x[AVX512VNNI-32]
_ = x[AVX512VP2INTERSECT-33]
_ = x[AVX512VPOPCNTDQ-34]
_ = x[AVXIFMA-35]
_ = x[AVXNECONVERT-36]
_ = x[AVXSLOW-37]
_ = x[AVXVNNI-38]
_ = x[AVXVNNIINT8-39]
_ = x[AVXVNNIINT16-40]
_ = x[BHI_CTRL-41]
_ = x[BMI1-42]
_ = x[BMI2-43]
_ = x[CETIBT-44]
_ = x[CETSS-45]
_ = x[CLDEMOTE-46]
_ = x[CLMUL-47]
_ = x[CLZERO-48]
_ = x[CMOV-49]
_ = x[CMPCCXADD-50]
_ = x[CMPSB_SCADBS_SHORT-51]
_ = x[CMPXCHG8-52]
_ = x[CPBOOST-53]
_ = x[CPPC-54]
_ = x[CX16-55]
_ = x[EFER_LMSLE_UNS-56]
_ = x[ENQCMD-57]
_ = x[ERMS-58]
_ = x[F16C-59]
_ = x[FLUSH_L1D-60]
_ = x[FMA3-61]
_ = x[FMA4-62]
_ = x[FP128-63]
_ = x[FP256-64]
_ = x[FSRM-65]
_ = x[FXSR-66]
_ = x[FXSROPT-67]
_ = x[GFNI-68]
_ = x[HLE-69]
_ = x[HRESET-70]
_ = x[HTT-71]
_ = x[HWA-72]
_ = x[HYBRID_CPU-73]
_ = x[HYPERVISOR-74]
_ = x[IA32_ARCH_CAP-75]
_ = x[IA32_CORE_CAP-76]
_ = x[IBPB-77]
_ = x[IBPB_BRTYPE-78]
_ = x[IBRS-79]
_ = x[IBRS_PREFERRED-80]
_ = x[IBRS_PROVIDES_SMP-81]
_ = x[IBS-82]
_ = x[IBSBRNTRGT-83]
_ = x[IBSFETCHSAM-84]
_ = x[IBSFFV-85]
_ = x[IBSOPCNT-86]
_ = x[IBSOPCNTEXT-87]
_ = x[IBSOPSAM-88]
_ = x[IBSRDWROPCNT-89]
_ = x[IBSRIPINVALIDCHK-90]
_ = x[IBS_FETCH_CTLX-91]
_ = x[IBS_OPDATA4-92]
_ = x[IBS_OPFUSE-93]
_ = x[IBS_PREVENTHOST-94]
_ = x[IBS_ZEN4-95]
_ = x[IDPRED_CTRL-96]
_ = x[INT_WBINVD-97]
_ = x[INVLPGB-98]
_ = x[KEYLOCKER-99]
_ = x[KEYLOCKERW-100]
_ = x[LAHF-101]
_ = x[LAM-102]
_ = x[LBRVIRT-103]
_ = x[LZCNT-104]
_ = x[MCAOVERFLOW-105]
_ = x[MCDT_NO-106]
_ = x[MCOMMIT-107]
_ = x[MD_CLEAR-108]
_ = x[MMX-109]
_ = x[MMXEXT-110]
_ = x[MOVBE-111]
_ = x[MOVDIR64B-112]
_ = x[MOVDIRI-113]
_ = x[MOVSB_ZL-114]
_ = x[MOVU-115]
_ = x[MPX-116]
_ = x[MSRIRC-117]
_ = x[MSRLIST-118]
_ = x[MSR_PAGEFLUSH-119]
_ = x[NRIPS-120]
_ = x[NX-121]
_ = x[OSXSAVE-122]
_ = x[PCONFIG-123]
_ = x[POPCNT-124]
_ = x[PPIN-125]
_ = x[PREFETCHI-126]
_ = x[PSFD-127]
_ = x[RDPRU-128]
_ = x[RDRAND-129]
_ = x[RDSEED-130]
_ = x[RDTSCP-131]
_ = x[RRSBA_CTRL-132]
_ = x[RTM-133]
_ = x[RTM_ALWAYS_ABORT-134]
_ = x[SBPB-135]
_ = x[SERIALIZE-136]
_ = x[SEV-137]
_ = x[SEV_64BIT-138]
_ = x[SEV_ALTERNATIVE-139]
_ = x[SEV_DEBUGSWAP-140]
_ = x[SEV_ES-141]
_ = x[SEV_RESTRICTED-142]
_ = x[SEV_SNP-143]
_ = x[SGX-144]
_ = x[SGXLC-145]
_ = x[SHA-146]
_ = x[SME-147]
_ = x[SME_COHERENT-148]
_ = x[SPEC_CTRL_SSBD-149]
_ = x[SRBDS_CTRL-150]
_ = x[SRSO_MSR_FIX-151]
_ = x[SRSO_NO-152]
_ = x[SRSO_USER_KERNEL_NO-153]
_ = x[SSE-154]
_ = x[SSE2-155]
_ = x[SSE3-156]
_ = x[SSE4-157]
_ = x[SSE42-158]
_ = x[SSE4A-159]
_ = x[SSSE3-160]
_ = x[STIBP-161]
_ = x[STIBP_ALWAYSON-162]
_ = x[STOSB_SHORT-163]
_ = x[SUCCOR-164]
_ = x[SVM-165]
_ = x[SVMDA-166]
_ = x[SVMFBASID-167]
_ = x[SVML-168]
_ = x[SVMNP-169]
_ = x[SVMPF-170]
_ = x[SVMPFT-171]
_ = x[SYSCALL-172]
_ = x[SYSEE-173]
_ = x[TBM-174]
_ = x[TDX_GUEST-175]
_ = x[TLB_FLUSH_NESTED-176]
_ = x[TME-177]
_ = x[TOPEXT-178]
_ = x[TSCRATEMSR-179]
_ = x[TSXLDTRK-180]
_ = x[VAES-181]
_ = x[VMCBCLEAN-182]
_ = x[VMPL-183]
_ = x[VMSA_REGPROT-184]
_ = x[VMX-185]
_ = x[VPCLMULQDQ-186]
_ = x[VTE-187]
_ = x[WAITPKG-188]
_ = x[WBNOINVD-189]
_ = x[WRMSRNS-190]
_ = x[X87-191]
_ = x[XGETBV1-192]
_ = x[XOP-193]
_ = x[XSAVE-194]
_ = x[XSAVEC-195]
_ = x[XSAVEOPT-196]
_ = x[XSAVES-197]
_ = x[AESARM-198]
_ = x[ARMCPUID-199]
_ = x[ASIMD-200]
_ = x[ASIMDDP-201]
_ = x[ASIMDHP-202]
_ = x[ASIMDRDM-203]
_ = x[ATOMICS-204]
_ = x[CRC32-205]
_ = x[DCPOP-206]
_ = x[EVTSTRM-207]
_ = x[FCMA-208]
_ = x[FHM-209]
_ = x[FP-210]
_ = x[FPHP-211]
_ = x[GPA-212]
_ = x[JSCVT-213]
_ = x[LRCPC-214]
_ = x[PMULL-215]
_ = x[RNDR-216]
_ = x[TLB-217]
_ = x[TS-218]
_ = x[SHA1-219]
_ = x[SHA2-220]
_ = x[SHA3-221]
_ = x[SHA512-222]
_ = x[SM3-223]
_ = x[SM4-224]
_ = x[SVE-225]
_ = x[lastID-226]
_ = x[firstID-0]
}
const _FeatureID_name = "firstIDADXAESNIAMD3DNOWAMD3DNOWEXTAMXBF16AMXFP16AMXINT8AMXFP8AMXTILEAPX_FAVXAVX10AVX10_128AVX10_256AVX10_512AVX2AVX512BF16AVX512BITALGAVX512BWAVX512CDAVX512DQAVX512ERAVX512FAVX512FP16AVX512IFMAAVX512PFAVX512VBMIAVX512VBMI2AVX512VLAVX512VNNIAVX512VP2INTERSECTAVX512VPOPCNTDQAVXIFMAAVXNECONVERTAVXSLOWAVXVNNIAVXVNNIINT8AVXVNNIINT16BHI_CTRLBMI1BMI2CETIBTCETSSCLDEMOTECLMULCLZEROCMOVCMPCCXADDCMPSB_SCADBS_SHORTCMPXCHG8CPBOOSTCPPCCX16EFER_LMSLE_UNSENQCMDERMSF16CFLUSH_L1DFMA3FMA4FP128FP256FSRMFXSRFXSROPTGFNIHLEHRESETHTTHWAHYBRID_CPUHYPERVISORIA32_ARCH_CAPIA32_CORE_CAPIBPBIBPB_BRTYPEIBRSIBRS_PREFERREDIBRS_PROVIDES_SMPIBSIBSBRNTRGTIBSFETCHSAMIBSFFVIBSOPCNTIBSOPCNTEXTIBSOPSAMIBSRDWROPCNTIBSRIPINVALIDCHKIBS_FETCH_CTLXIBS_OPDATA4IBS_OPFUSEIBS_PREVENTHOSTIBS_ZEN4IDPRED_CTRLINT_WBINVDINVLPGBKEYLOCKERKEYLOCKERWLAHFLAMLBRVIRTLZCNTMCAOVERFLOWMCDT_NOMCOMMITMD_CLEARMMXMMXEXTMOVBEMOVDIR64BMOVDIRIMOVSB_ZLMOVUMPXMSRIRCMSRLISTMSR_PAGEFLUSHNRIPSNXOSXSAVEPCONFIGPOPCNTPPINPREFETCHIPSFDRDPRURDRANDRDSEEDRDTSCPRRSBA_CTRLRTMRTM_ALWAYS_ABORTSBPBSERIALIZESEVSEV_64BITSEV_ALTERNATIVESEV_DEBUGSWAPSEV_ESSEV_RESTRICTEDSEV_SNPSGXSGXLCSHASMESME_COHERENTSPEC_CTRL_SSBDSRBDS_CTRLSRSO_MSR_FIXSRSO_NOSRSO_USER_KERNEL_NOSSESSE2SSE3SSE4SSE42SSE4ASSSE3STIBPSTIBP_ALWAYSONSTOSB_SHORTSUCCORSVMSVMDASVMFBASIDSVMLSVMNPSVMPFSVMPFTSYSCALLSYSEETBMTDX_GUESTTLB_FLUSH_NESTEDTMETOPEXTTSCRATEMSRTSXLDTRKVAESVMCBCLEANVMPLVMSA_REGPROTVMXVPCLMULQDQVTEWAITPKGWBNOINVDWRMSRNSX87XGETBV1XOPXSAVEXSAVECXSAVEOPTXSAVESAESARMARMCPUIDASIMDASIMDDPASIMDHPASIMDRDMATOMICSCRC32DCPOPEVTSTRMFCMAFPFPHPGPAJSCVTLRCPCPMULLSHA1SHA2SHA3SHA512SM3SM4SVElastID"
const _FeatureID_name = "firstIDADXAESNIAMD3DNOWAMD3DNOWEXTAMXBF16AMXFP16AMXINT8AMXFP8AMXTILEAMXTF32AMXCOMPLEXAPX_FAVXAVX10AVX10_128AVX10_256AVX10_512AVX2AVX512BF16AVX512BITALGAVX512BWAVX512CDAVX512DQAVX512ERAVX512FAVX512FP16AVX512IFMAAVX512PFAVX512VBMIAVX512VBMI2AVX512VLAVX512VNNIAVX512VP2INTERSECTAVX512VPOPCNTDQAVXIFMAAVXNECONVERTAVXSLOWAVXVNNIAVXVNNIINT8AVXVNNIINT16BHI_CTRLBMI1BMI2CETIBTCETSSCLDEMOTECLMULCLZEROCMOVCMPCCXADDCMPSB_SCADBS_SHORTCMPXCHG8CPBOOSTCPPCCX16EFER_LMSLE_UNSENQCMDERMSF16CFLUSH_L1DFMA3FMA4FP128FP256FSRMFXSRFXSROPTGFNIHLEHRESETHTTHWAHYBRID_CPUHYPERVISORIA32_ARCH_CAPIA32_CORE_CAPIBPBIBPB_BRTYPEIBRSIBRS_PREFERREDIBRS_PROVIDES_SMPIBSIBSBRNTRGTIBSFETCHSAMIBSFFVIBSOPCNTIBSOPCNTEXTIBSOPSAMIBSRDWROPCNTIBSRIPINVALIDCHKIBS_FETCH_CTLXIBS_OPDATA4IBS_OPFUSEIBS_PREVENTHOSTIBS_ZEN4IDPRED_CTRLINT_WBINVDINVLPGBKEYLOCKERKEYLOCKERWLAHFLAMLBRVIRTLZCNTMCAOVERFLOWMCDT_NOMCOMMITMD_CLEARMMXMMXEXTMOVBEMOVDIR64BMOVDIRIMOVSB_ZLMOVUMPXMSRIRCMSRLISTMSR_PAGEFLUSHNRIPSNXOSXSAVEPCONFIGPOPCNTPPINPREFETCHIPSFDRDPRURDRANDRDSEEDRDTSCPRRSBA_CTRLRTMRTM_ALWAYS_ABORTSBPBSERIALIZESEVSEV_64BITSEV_ALTERNATIVESEV_DEBUGSWAPSEV_ESSEV_RESTRICTEDSEV_SNPSGXSGXLCSHASMESME_COHERENTSPEC_CTRL_SSBDSRBDS_CTRLSRSO_MSR_FIXSRSO_NOSRSO_USER_KERNEL_NOSSESSE2SSE3SSE4SSE42SSE4ASSSE3STIBPSTIBP_ALWAYSONSTOSB_SHORTSUCCORSVMSVMDASVMFBASIDSVMLSVMNPSVMPFSVMPFTSYSCALLSYSEETBMTDX_GUESTTLB_FLUSH_NESTEDTMETOPEXTTSCRATEMSRTSXLDTRKVAESVMCBCLEANVMPLVMSA_REGPROTVMXVPCLMULQDQVTEWAITPKGWBNOINVDWRMSRNSX87XGETBV1XOPXSAVEXSAVECXSAVEOPTXSAVESAESARMARMCPUIDASIMDASIMDDPASIMDHPASIMDRDMATOMICSCRC32DCPOPEVTSTRMFCMAFHMFPFPHPGPAJSCVTLRCPCPMULLRNDRTLBTSSHA1SHA2SHA3SHA512SM3SM4SVElastID"
var _FeatureID_index = [...]uint16{0, 7, 10, 15, 23, 34, 41, 48, 55, 61, 68, 73, 76, 81, 90, 99, 108, 112, 122, 134, 142, 150, 158, 166, 173, 183, 193, 201, 211, 222, 230, 240, 258, 273, 280, 292, 299, 306, 317, 329, 337, 341, 345, 351, 356, 364, 369, 375, 379, 388, 406, 414, 421, 425, 429, 443, 449, 453, 457, 466, 470, 474, 479, 484, 488, 492, 499, 503, 506, 512, 515, 518, 528, 538, 551, 564, 568, 579, 583, 597, 614, 617, 627, 638, 644, 652, 663, 671, 683, 699, 713, 724, 734, 749, 757, 768, 778, 785, 794, 804, 808, 811, 818, 823, 834, 841, 848, 856, 859, 865, 870, 879, 886, 894, 898, 901, 907, 914, 927, 932, 934, 941, 948, 954, 958, 967, 971, 976, 982, 988, 994, 1004, 1007, 1023, 1027, 1036, 1039, 1048, 1063, 1076, 1082, 1096, 1103, 1106, 1111, 1114, 1117, 1129, 1143, 1153, 1165, 1172, 1191, 1194, 1198, 1202, 1206, 1211, 1216, 1221, 1226, 1240, 1251, 1257, 1260, 1265, 1274, 1278, 1283, 1288, 1294, 1301, 1306, 1309, 1318, 1334, 1337, 1343, 1353, 1361, 1365, 1374, 1378, 1390, 1393, 1403, 1406, 1413, 1421, 1428, 1431, 1438, 1441, 1446, 1452, 1460, 1466, 1472, 1480, 1485, 1492, 1499, 1507, 1514, 1519, 1524, 1531, 1535, 1537, 1541, 1544, 1549, 1554, 1559, 1563, 1567, 1571, 1577, 1580, 1583, 1586, 1592}
var _FeatureID_index = [...]uint16{0, 7, 10, 15, 23, 34, 41, 48, 55, 61, 68, 75, 85, 90, 93, 98, 107, 116, 125, 129, 139, 151, 159, 167, 175, 183, 190, 200, 210, 218, 228, 239, 247, 257, 275, 290, 297, 309, 316, 323, 334, 346, 354, 358, 362, 368, 373, 381, 386, 392, 396, 405, 423, 431, 438, 442, 446, 460, 466, 470, 474, 483, 487, 491, 496, 501, 505, 509, 516, 520, 523, 529, 532, 535, 545, 555, 568, 581, 585, 596, 600, 614, 631, 634, 644, 655, 661, 669, 680, 688, 700, 716, 730, 741, 751, 766, 774, 785, 795, 802, 811, 821, 825, 828, 835, 840, 851, 858, 865, 873, 876, 882, 887, 896, 903, 911, 915, 918, 924, 931, 944, 949, 951, 958, 965, 971, 975, 984, 988, 993, 999, 1005, 1011, 1021, 1024, 1040, 1044, 1053, 1056, 1065, 1080, 1093, 1099, 1113, 1120, 1123, 1128, 1131, 1134, 1146, 1160, 1170, 1182, 1189, 1208, 1211, 1215, 1219, 1223, 1228, 1233, 1238, 1243, 1257, 1268, 1274, 1277, 1282, 1291, 1295, 1300, 1305, 1311, 1318, 1323, 1326, 1335, 1351, 1354, 1360, 1370, 1378, 1382, 1391, 1395, 1407, 1410, 1420, 1423, 1430, 1438, 1445, 1448, 1455, 1458, 1463, 1469, 1477, 1483, 1489, 1497, 1502, 1509, 1516, 1524, 1531, 1536, 1541, 1548, 1552, 1555, 1557, 1561, 1564, 1569, 1574, 1579, 1583, 1586, 1588, 1592, 1596, 1600, 1606, 1609, 1612, 1615, 1621}
func (i FeatureID) String() string {
if i < 0 || i >= FeatureID(len(_FeatureID_index)-1) {

View file

@ -96,9 +96,11 @@ func tryToFillCPUInfoFomSysctl(c *CPUInfo) {
setFeature(c, "hw.optional.arm.FEAT_DPB", DCPOP)
// setFeature(c, "", EVTSTRM)
setFeature(c, "hw.optional.arm.FEAT_FCMA", FCMA)
setFeature(c, "hw.optional.arm.FEAT_FHM", FHM)
setFeature(c, "hw.optional.arm.FEAT_FP", FP)
setFeature(c, "hw.optional.arm.FEAT_FP16", FPHP)
setFeature(c, "hw.optional.arm.FEAT_PAuth", GPA)
setFeature(c, "hw.optional.arm.FEAT_RNG", RNDR)
setFeature(c, "hw.optional.arm.FEAT_JSCVT", JSCVT)
setFeature(c, "hw.optional.arm.FEAT_LRCPC", LRCPC)
setFeature(c, "hw.optional.arm.FEAT_PMULL", PMULL)
@ -106,6 +108,10 @@ func tryToFillCPUInfoFomSysctl(c *CPUInfo) {
setFeature(c, "hw.optional.arm.FEAT_SHA256", SHA2)
setFeature(c, "hw.optional.arm.FEAT_SHA3", SHA3)
setFeature(c, "hw.optional.arm.FEAT_SHA512", SHA512)
setFeature(c, "hw.optional.arm.FEAT_TLBIOS", TLB)
setFeature(c, "hw.optional.arm.FEAT_TLBIRANGE", TLB)
setFeature(c, "hw.optional.arm.FEAT_FlagM", TS)
setFeature(c, "hw.optional.arm.FEAT_FlagM2", TS)
// setFeature(c, "", SM3)
// setFeature(c, "", SM4)
setFeature(c, "hw.optional.arm.FEAT_SVE", SVE)

View file

@ -39,6 +39,80 @@ const (
hwcap_SHA512 = 1 << 21
hwcap_SVE = 1 << 22
hwcap_ASIMDFHM = 1 << 23
hwcap_DIT = 1 << 24
hwcap_USCAT = 1 << 25
hwcap_ILRCPC = 1 << 26
hwcap_FLAGM = 1 << 27
hwcap_SSBS = 1 << 28
hwcap_SB = 1 << 29
hwcap_PACA = 1 << 30
hwcap_PACG = 1 << 31
hwcap_GCS = 1 << 32
hwcap2_DCPODP = 1 << 0
hwcap2_SVE2 = 1 << 1
hwcap2_SVEAES = 1 << 2
hwcap2_SVEPMULL = 1 << 3
hwcap2_SVEBITPERM = 1 << 4
hwcap2_SVESHA3 = 1 << 5
hwcap2_SVESM4 = 1 << 6
hwcap2_FLAGM2 = 1 << 7
hwcap2_FRINT = 1 << 8
hwcap2_SVEI8MM = 1 << 9
hwcap2_SVEF32MM = 1 << 10
hwcap2_SVEF64MM = 1 << 11
hwcap2_SVEBF16 = 1 << 12
hwcap2_I8MM = 1 << 13
hwcap2_BF16 = 1 << 14
hwcap2_DGH = 1 << 15
hwcap2_RNG = 1 << 16
hwcap2_BTI = 1 << 17
hwcap2_MTE = 1 << 18
hwcap2_ECV = 1 << 19
hwcap2_AFP = 1 << 20
hwcap2_RPRES = 1 << 21
hwcap2_MTE3 = 1 << 22
hwcap2_SME = 1 << 23
hwcap2_SME_I16I64 = 1 << 24
hwcap2_SME_F64F64 = 1 << 25
hwcap2_SME_I8I32 = 1 << 26
hwcap2_SME_F16F32 = 1 << 27
hwcap2_SME_B16F32 = 1 << 28
hwcap2_SME_F32F32 = 1 << 29
hwcap2_SME_FA64 = 1 << 30
hwcap2_WFXT = 1 << 31
hwcap2_EBF16 = 1 << 32
hwcap2_SVE_EBF16 = 1 << 33
hwcap2_CSSC = 1 << 34
hwcap2_RPRFM = 1 << 35
hwcap2_SVE2P1 = 1 << 36
hwcap2_SME2 = 1 << 37
hwcap2_SME2P1 = 1 << 38
hwcap2_SME_I16I32 = 1 << 39
hwcap2_SME_BI32I32 = 1 << 40
hwcap2_SME_B16B16 = 1 << 41
hwcap2_SME_F16F16 = 1 << 42
hwcap2_MOPS = 1 << 43
hwcap2_HBC = 1 << 44
hwcap2_SVE_B16B16 = 1 << 45
hwcap2_LRCPC3 = 1 << 46
hwcap2_LSE128 = 1 << 47
hwcap2_FPMR = 1 << 48
hwcap2_LUT = 1 << 49
hwcap2_FAMINMAX = 1 << 50
hwcap2_F8CVT = 1 << 51
hwcap2_F8FMA = 1 << 52
hwcap2_F8DP4 = 1 << 53
hwcap2_F8DP2 = 1 << 54
hwcap2_F8E4M3 = 1 << 55
hwcap2_F8E5M2 = 1 << 56
hwcap2_SME_LUTV2 = 1 << 57
hwcap2_SME_F8F16 = 1 << 58
hwcap2_SME_F8F32 = 1 << 59
hwcap2_SME_SF8FMA = 1 << 60
hwcap2_SME_SF8DP4 = 1 << 61
hwcap2_SME_SF8DP2 = 1 << 62
hwcap2_POE = 1 << 63
)
func detectOS(c *CPUInfo) bool {
@ -104,11 +178,15 @@ func detectOS(c *CPUInfo) bool {
c.featureSet.setIf(isSet(hwcap, hwcap_DCPOP), DCPOP)
c.featureSet.setIf(isSet(hwcap, hwcap_EVTSTRM), EVTSTRM)
c.featureSet.setIf(isSet(hwcap, hwcap_FCMA), FCMA)
c.featureSet.setIf(isSet(hwcap, hwcap_ASIMDFHM), FHM)
c.featureSet.setIf(isSet(hwcap, hwcap_FP), FP)
c.featureSet.setIf(isSet(hwcap, hwcap_FPHP), FPHP)
c.featureSet.setIf(isSet(hwcap, hwcap_JSCVT), JSCVT)
c.featureSet.setIf(isSet(hwcap, hwcap_LRCPC), LRCPC)
c.featureSet.setIf(isSet(hwcap, hwcap_PMULL), PMULL)
c.featureSet.setIf(isSet(hwcap, hwcap2_RNG), RNDR)
// c.featureSet.setIf(isSet(hwcap, hwcap_), TLB)
// c.featureSet.setIf(isSet(hwcap, hwcap_), TS)
c.featureSet.setIf(isSet(hwcap, hwcap_SHA1), SHA1)
c.featureSet.setIf(isSet(hwcap, hwcap_SHA2), SHA2)
c.featureSet.setIf(isSet(hwcap, hwcap_SHA3), SHA3)

202
vendor/github.com/minio/crc64nvme/LICENSE generated vendored Normal file
View file

@ -0,0 +1,202 @@
Apache License
Version 2.0, January 2004
http://www.apache.org/licenses/
TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION
1. Definitions.
"License" shall mean the terms and conditions for use, reproduction,
and distribution as defined by Sections 1 through 9 of this document.
"Licensor" shall mean the copyright owner or entity authorized by
the copyright owner that is granting the License.
"Legal Entity" shall mean the union of the acting entity and all
other entities that control, are controlled by, or are under common
control with that entity. For the purposes of this definition,
"control" means (i) the power, direct or indirect, to cause the
direction or management of such entity, whether by contract or
otherwise, or (ii) ownership of fifty percent (50%) or more of the
outstanding shares, or (iii) beneficial ownership of such entity.
"You" (or "Your") shall mean an individual or Legal Entity
exercising permissions granted by this License.
"Source" form shall mean the preferred form for making modifications,
including but not limited to software source code, documentation
source, and configuration files.
"Object" form shall mean any form resulting from mechanical
transformation or translation of a Source form, including but
not limited to compiled object code, generated documentation,
and conversions to other media types.
"Work" shall mean the work of authorship, whether in Source or
Object form, made available under the License, as indicated by a
copyright notice that is included in or attached to the work
(an example is provided in the Appendix below).
"Derivative Works" shall mean any work, whether in Source or Object
form, that is based on (or derived from) the Work and for which the
editorial revisions, annotations, elaborations, or other modifications
represent, as a whole, an original work of authorship. For the purposes
of this License, Derivative Works shall not include works that remain
separable from, or merely link (or bind by name) to the interfaces of,
the Work and Derivative Works thereof.
"Contribution" shall mean any work of authorship, including
the original version of the Work and any modifications or additions
to that Work or Derivative Works thereof, that is intentionally
submitted to Licensor for inclusion in the Work by the copyright owner
or by an individual or Legal Entity authorized to submit on behalf of
the copyright owner. For the purposes of this definition, "submitted"
means any form of electronic, verbal, or written communication sent
to the Licensor or its representatives, including but not limited to
communication on electronic mailing lists, source code control systems,
and issue tracking systems that are managed by, or on behalf of, the
Licensor for the purpose of discussing and improving the Work, but
excluding communication that is conspicuously marked or otherwise
designated in writing by the copyright owner as "Not a Contribution."
"Contributor" shall mean Licensor and any individual or Legal Entity
on behalf of whom a Contribution has been received by Licensor and
subsequently incorporated within the Work.
2. Grant of Copyright License. Subject to the terms and conditions of
this License, each Contributor hereby grants to You a perpetual,
worldwide, non-exclusive, no-charge, royalty-free, irrevocable
copyright license to reproduce, prepare Derivative Works of,
publicly display, publicly perform, sublicense, and distribute the
Work and such Derivative Works in Source or Object form.
3. Grant of Patent License. Subject to the terms and conditions of
this License, each Contributor hereby grants to You a perpetual,
worldwide, non-exclusive, no-charge, royalty-free, irrevocable
(except as stated in this section) patent license to make, have made,
use, offer to sell, sell, import, and otherwise transfer the Work,
where such license applies only to those patent claims licensable
by such Contributor that are necessarily infringed by their
Contribution(s) alone or by combination of their Contribution(s)
with the Work to which such Contribution(s) was submitted. If You
institute patent litigation against any entity (including a
cross-claim or counterclaim in a lawsuit) alleging that the Work
or a Contribution incorporated within the Work constitutes direct
or contributory patent infringement, then any patent licenses
granted to You under this License for that Work shall terminate
as of the date such litigation is filed.
4. Redistribution. You may reproduce and distribute copies of the
Work or Derivative Works thereof in any medium, with or without
modifications, and in Source or Object form, provided that You
meet the following conditions:
(a) You must give any other recipients of the Work or
Derivative Works a copy of this License; and
(b) You must cause any modified files to carry prominent notices
stating that You changed the files; and
(c) You must retain, in the Source form of any Derivative Works
that You distribute, all copyright, patent, trademark, and
attribution notices from the Source form of the Work,
excluding those notices that do not pertain to any part of
the Derivative Works; and
(d) If the Work includes a "NOTICE" text file as part of its
distribution, then any Derivative Works that You distribute must
include a readable copy of the attribution notices contained
within such NOTICE file, excluding those notices that do not
pertain to any part of the Derivative Works, in at least one
of the following places: within a NOTICE text file distributed
as part of the Derivative Works; within the Source form or
documentation, if provided along with the Derivative Works; or,
within a display generated by the Derivative Works, if and
wherever such third-party notices normally appear. The contents
of the NOTICE file are for informational purposes only and
do not modify the License. You may add Your own attribution
notices within Derivative Works that You distribute, alongside
or as an addendum to the NOTICE text from the Work, provided
that such additional attribution notices cannot be construed
as modifying the License.
You may add Your own copyright statement to Your modifications and
may provide additional or different license terms and conditions
for use, reproduction, or distribution of Your modifications, or
for any such Derivative Works as a whole, provided Your use,
reproduction, and distribution of the Work otherwise complies with
the conditions stated in this License.
5. Submission of Contributions. Unless You explicitly state otherwise,
any Contribution intentionally submitted for inclusion in the Work
by You to the Licensor shall be under the terms and conditions of
this License, without any additional terms or conditions.
Notwithstanding the above, nothing herein shall supersede or modify
the terms of any separate license agreement you may have executed
with Licensor regarding such Contributions.
6. Trademarks. This License does not grant permission to use the trade
names, trademarks, service marks, or product names of the Licensor,
except as required for reasonable and customary use in describing the
origin of the Work and reproducing the content of the NOTICE file.
7. Disclaimer of Warranty. Unless required by applicable law or
agreed to in writing, Licensor provides the Work (and each
Contributor provides its Contributions) on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
implied, including, without limitation, any warranties or conditions
of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A
PARTICULAR PURPOSE. You are solely responsible for determining the
appropriateness of using or redistributing the Work and assume any
risks associated with Your exercise of permissions under this License.
8. Limitation of Liability. In no event and under no legal theory,
whether in tort (including negligence), contract, or otherwise,
unless required by applicable law (such as deliberate and grossly
negligent acts) or agreed to in writing, shall any Contributor be
liable to You for damages, including any direct, indirect, special,
incidental, or consequential damages of any character arising as a
result of this License or out of the use or inability to use the
Work (including but not limited to damages for loss of goodwill,
work stoppage, computer failure or malfunction, or any and all
other commercial damages or losses), even if such Contributor
has been advised of the possibility of such damages.
9. Accepting Warranty or Additional Liability. While redistributing
the Work or Derivative Works thereof, You may choose to offer,
and charge a fee for, acceptance of support, warranty, indemnity,
or other liability obligations and/or rights consistent with this
License. However, in accepting such obligations, You may act only
on Your own behalf and on Your sole responsibility, not on behalf
of any other Contributor, and only if You agree to indemnify,
defend, and hold each Contributor harmless for any liability
incurred by, or claims asserted against, such Contributor by reason
of your accepting any such warranty or additional liability.
END OF TERMS AND CONDITIONS
APPENDIX: How to apply the Apache License to your work.
To apply the Apache License to your work, attach the following
boilerplate notice, with the fields enclosed by brackets "[]"
replaced with your own identifying information. (Don't include
the brackets!) The text should be enclosed in the appropriate
comment syntax for the file format. We also recommend that a
file or class name and description of purpose be included on the
same "printed page" as the copyright notice for easier
identification within third-party archives.
Copyright [yyyy] [name of copyright owner]
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.

20
vendor/github.com/minio/crc64nvme/README.md generated vendored Normal file
View file

@ -0,0 +1,20 @@
## crc64nvme
This Golang package calculates CRC64 checksums using carryless-multiplication accelerated with SIMD instructions for both ARM and x86. It is based on the NVME polynomial as specified in the [NVM Express® NVM Command Set Specification](https://nvmexpress.org/wp-content/uploads/NVM-Express-NVM-Command-Set-Specification-1.0d-2023.12.28-Ratified.pdf).
The code is based on the [crc64fast-nvme](https://github.com/awesomized/crc64fast-nvme.git) package in Rust and is released under the Apache 2.0 license.
For more background on the exact technique used, see this [Fast CRC Computation for Generic Polynomials Using PCLMULQDQ Instruction](https://web.archive.org/web/20131224125630/https://www.intel.com/content/dam/www/public/us/en/documents/white-papers/fast-crc-computation-generic-polynomials-pclmulqdq-paper.pdf) paper.
### Performance
To follow.
### Requirements
All Go versions >= 1.22 are supported.
### Contributing
Contributions are welcome, please send PRs for any enhancements.

180
vendor/github.com/minio/crc64nvme/crc64.go generated vendored Normal file
View file

@ -0,0 +1,180 @@
// Copyright (c) 2025 Minio Inc. All rights reserved.
// Use of this source code is governed by a license that can be
// found in the LICENSE file.
// Package crc64nvme implements the 64-bit cyclic redundancy check with NVME polynomial.
package crc64nvme
import (
"encoding/binary"
"errors"
"hash"
"sync"
"unsafe"
)
const (
// The size of a CRC-64 checksum in bytes.
Size = 8
// The NVME polynoimial (reversed, as used by Go)
NVME = 0x9a6c9329ac4bc9b5
)
var (
// precalculated table.
nvmeTable = makeTable(NVME)
)
// table is a 256-word table representing the polynomial for efficient processing.
type table [256]uint64
var (
slicing8TablesBuildOnce sync.Once
slicing8TableNVME *[8]table
)
func buildSlicing8TablesOnce() {
slicing8TablesBuildOnce.Do(buildSlicing8Tables)
}
func buildSlicing8Tables() {
slicing8TableNVME = makeSlicingBy8Table(makeTable(NVME))
}
func makeTable(poly uint64) *table {
t := new(table)
for i := 0; i < 256; i++ {
crc := uint64(i)
for j := 0; j < 8; j++ {
if crc&1 == 1 {
crc = (crc >> 1) ^ poly
} else {
crc >>= 1
}
}
t[i] = crc
}
return t
}
func makeSlicingBy8Table(t *table) *[8]table {
var helperTable [8]table
helperTable[0] = *t
for i := 0; i < 256; i++ {
crc := t[i]
for j := 1; j < 8; j++ {
crc = t[crc&0xff] ^ (crc >> 8)
helperTable[j][i] = crc
}
}
return &helperTable
}
// digest represents the partial evaluation of a checksum.
type digest struct {
crc uint64
}
// New creates a new hash.Hash64 computing the CRC-64 checksum using the
// NVME polynomial. Its Sum method will lay the
// value out in big-endian byte order. The returned Hash64 also
// implements [encoding.BinaryMarshaler] and [encoding.BinaryUnmarshaler] to
// marshal and unmarshal the internal state of the hash.
func New() hash.Hash64 { return &digest{0} }
func (d *digest) Size() int { return Size }
func (d *digest) BlockSize() int { return 1 }
func (d *digest) Reset() { d.crc = 0 }
const (
magic = "crc\x02"
marshaledSize = len(magic) + 8 + 8
)
func (d *digest) MarshalBinary() ([]byte, error) {
b := make([]byte, 0, marshaledSize)
b = append(b, magic...)
b = binary.BigEndian.AppendUint64(b, tableSum)
b = binary.BigEndian.AppendUint64(b, d.crc)
return b, nil
}
func (d *digest) UnmarshalBinary(b []byte) error {
if len(b) < len(magic) || string(b[:len(magic)]) != magic {
return errors.New("hash/crc64: invalid hash state identifier")
}
if len(b) != marshaledSize {
return errors.New("hash/crc64: invalid hash state size")
}
if tableSum != binary.BigEndian.Uint64(b[4:]) {
return errors.New("hash/crc64: tables do not match")
}
d.crc = binary.BigEndian.Uint64(b[12:])
return nil
}
func update(crc uint64, p []byte) uint64 {
if hasAsm && len(p) > 127 {
ptr := unsafe.Pointer(&p[0])
if align := (uintptr(ptr)+15)&^0xf - uintptr(ptr); align > 0 {
// Align to 16-byte boundary.
crc = update(crc, p[:align])
p = p[align:]
}
runs := len(p) / 128
crc = updateAsm(crc, p[:128*runs])
return update(crc, p[128*runs:])
}
buildSlicing8TablesOnce()
crc = ^crc
// table comparison is somewhat expensive, so avoid it for small sizes
for len(p) >= 64 {
var helperTable = slicing8TableNVME
// Update using slicing-by-8
for len(p) > 8 {
crc ^= binary.LittleEndian.Uint64(p)
crc = helperTable[7][crc&0xff] ^
helperTable[6][(crc>>8)&0xff] ^
helperTable[5][(crc>>16)&0xff] ^
helperTable[4][(crc>>24)&0xff] ^
helperTable[3][(crc>>32)&0xff] ^
helperTable[2][(crc>>40)&0xff] ^
helperTable[1][(crc>>48)&0xff] ^
helperTable[0][crc>>56]
p = p[8:]
}
}
// For reminders or small sizes
for _, v := range p {
crc = nvmeTable[byte(crc)^v] ^ (crc >> 8)
}
return ^crc
}
// Update returns the result of adding the bytes in p to the crc.
func Update(crc uint64, p []byte) uint64 {
return update(crc, p)
}
func (d *digest) Write(p []byte) (n int, err error) {
d.crc = update(d.crc, p)
return len(p), nil
}
func (d *digest) Sum64() uint64 { return d.crc }
func (d *digest) Sum(in []byte) []byte {
s := d.Sum64()
return append(in, byte(s>>56), byte(s>>48), byte(s>>40), byte(s>>32), byte(s>>24), byte(s>>16), byte(s>>8), byte(s))
}
// Checksum returns the CRC-64 checksum of data
// using the NVME polynomial.
func Checksum(data []byte) uint64 { return update(0, data) }
// ISO tablesum of NVME poly
const tableSum = 0x8ddd9ee4402c7163

15
vendor/github.com/minio/crc64nvme/crc64_amd64.go generated vendored Normal file
View file

@ -0,0 +1,15 @@
// Copyright (c) 2025 Minio Inc. All rights reserved.
// Use of this source code is governed by a license that can be
// found in the LICENSE file.
//go:build !noasm && !appengine && !gccgo
package crc64nvme
import (
"github.com/klauspost/cpuid/v2"
)
var hasAsm = cpuid.CPU.Supports(cpuid.SSE2, cpuid.CLMUL, cpuid.SSE4)
func updateAsm(crc uint64, p []byte) (checksum uint64)

157
vendor/github.com/minio/crc64nvme/crc64_amd64.s generated vendored Normal file
View file

@ -0,0 +1,157 @@
// Copyright (c) 2025 Minio Inc. All rights reserved.
// Use of this source code is governed by a license that can be
// found in the LICENSE file.
//go:build !noasm && !appengine && !gccgo
#include "textflag.h"
TEXT ·updateAsm(SB), $0-40
MOVQ crc+0(FP), AX // checksum
MOVQ p_base+8(FP), SI // start pointer
MOVQ p_len+16(FP), CX // length of buffer
NOTQ AX
SHRQ $7, CX
CMPQ CX, $1
JLT skip128
VMOVDQA 0x00(SI), X0
VMOVDQA 0x10(SI), X1
VMOVDQA 0x20(SI), X2
VMOVDQA 0x30(SI), X3
VMOVDQA 0x40(SI), X4
VMOVDQA 0x50(SI), X5
VMOVDQA 0x60(SI), X6
VMOVDQA 0x70(SI), X7
MOVQ AX, X8
PXOR X8, X0
CMPQ CX, $1
JE tail128
MOVQ $0xa1ca681e733f9c40, AX
MOVQ AX, X8
MOVQ $0x5f852fb61e8d92dc, AX
PINSRQ $0x1, AX, X9
loop128:
ADDQ $128, SI
SUBQ $1, CX
VMOVDQA X0, X10
PCLMULQDQ $0x00, X8, X10
PCLMULQDQ $0x11, X9, X0
PXOR X10, X0
PXOR 0(SI), X0
VMOVDQA X1, X10
PCLMULQDQ $0x00, X8, X10
PCLMULQDQ $0x11, X9, X1
PXOR X10, X1
PXOR 0x10(SI), X1
VMOVDQA X2, X10
PCLMULQDQ $0x00, X8, X10
PCLMULQDQ $0x11, X9, X2
PXOR X10, X2
PXOR 0x20(SI), X2
VMOVDQA X3, X10
PCLMULQDQ $0x00, X8, X10
PCLMULQDQ $0x11, X9, X3
PXOR X10, X3
PXOR 0x30(SI), X3
VMOVDQA X4, X10
PCLMULQDQ $0x00, X8, X10
PCLMULQDQ $0x11, X9, X4
PXOR X10, X4
PXOR 0x40(SI), X4
VMOVDQA X5, X10
PCLMULQDQ $0x00, X8, X10
PCLMULQDQ $0x11, X9, X5
PXOR X10, X5
PXOR 0x50(SI), X5
VMOVDQA X6, X10
PCLMULQDQ $0x00, X8, X10
PCLMULQDQ $0x11, X9, X6
PXOR X10, X6
PXOR 0x60(SI), X6
VMOVDQA X7, X10
PCLMULQDQ $0x00, X8, X10
PCLMULQDQ $0x11, X9, X7
PXOR X10, X7
PXOR 0x70(SI), X7
CMPQ CX, $1
JGT loop128
tail128:
MOVQ $0xd083dd594d96319d, AX
MOVQ AX, X11
PCLMULQDQ $0x00, X0, X11
MOVQ $0x946588403d4adcbc, AX
PINSRQ $0x1, AX, X12
PCLMULQDQ $0x11, X12, X0
PXOR X11, X7
PXOR X0, X7
MOVQ $0x3c255f5ebc414423, AX
MOVQ AX, X11
PCLMULQDQ $0x00, X1, X11
MOVQ $0x34f5a24e22d66e90, AX
PINSRQ $0x1, AX, X12
PCLMULQDQ $0x11, X12, X1
PXOR X11, X1
PXOR X7, X1
MOVQ $0x7b0ab10dd0f809fe, AX
MOVQ AX, X11
PCLMULQDQ $0x00, X2, X11
MOVQ $0x03363823e6e791e5, AX
PINSRQ $0x1, AX, X12
PCLMULQDQ $0x11, X12, X2
PXOR X11, X2
PXOR X1, X2
MOVQ $0x0c32cdb31e18a84a, AX
MOVQ AX, X11
PCLMULQDQ $0x00, X3, X11
MOVQ $0x62242240ace5045a, AX
PINSRQ $0x1, AX, X12
PCLMULQDQ $0x11, X12, X3
PXOR X11, X3
PXOR X2, X3
MOVQ $0xbdd7ac0ee1a4a0f0, AX
MOVQ AX, X11
PCLMULQDQ $0x00, X4, X11
MOVQ $0xa3ffdc1fe8e82a8b, AX
PINSRQ $0x1, AX, X12
PCLMULQDQ $0x11, X12, X4
PXOR X11, X4
PXOR X3, X4
MOVQ $0xb0bc2e589204f500, AX
MOVQ AX, X11
PCLMULQDQ $0x00, X5, X11
MOVQ $0xe1e0bb9d45d7a44c, AX
PINSRQ $0x1, AX, X12
PCLMULQDQ $0x11, X12, X5
PXOR X11, X5
PXOR X4, X5
MOVQ $0xeadc41fd2ba3d420, AX
MOVQ AX, X11
PCLMULQDQ $0x00, X6, X11
MOVQ $0x21e9761e252621ac, AX
PINSRQ $0x1, AX, X12
PCLMULQDQ $0x11, X12, X6
PXOR X11, X6
PXOR X5, X6
MOVQ AX, X5
PCLMULQDQ $0x00, X6, X5
PSHUFD $0xee, X6, X6
PXOR X5, X6
MOVQ $0x27ecfa329aef9f77, AX
MOVQ AX, X4
PCLMULQDQ $0x00, X4, X6
PEXTRQ $0, X6, BX
MOVQ $0x34d926535897936b, AX
MOVQ AX, X4
PCLMULQDQ $0x00, X4, X6
PXOR X5, X6
PEXTRQ $1, X6, AX
XORQ BX, AX
skip128:
NOTQ AX
MOVQ AX, checksum+32(FP)
RET

15
vendor/github.com/minio/crc64nvme/crc64_arm64.go generated vendored Normal file
View file

@ -0,0 +1,15 @@
// Copyright (c) 2025 Minio Inc. All rights reserved.
// Use of this source code is governed by a license that can be
// found in the LICENSE file.
//go:build !noasm && !appengine && !gccgo
package crc64nvme
import (
"github.com/klauspost/cpuid/v2"
)
var hasAsm = cpuid.CPU.Supports(cpuid.ASIMD) && cpuid.CPU.Supports(cpuid.PMULL)
func updateAsm(crc uint64, p []byte) (checksum uint64)

157
vendor/github.com/minio/crc64nvme/crc64_arm64.s generated vendored Normal file
View file

@ -0,0 +1,157 @@
// Copyright (c) 2025 Minio Inc. All rights reserved.
// Use of this source code is governed by a license that can be
// found in the LICENSE file.
//go:build !noasm && !appengine && !gccgo
#include "textflag.h"
TEXT ·updateAsm(SB), $0-40
MOVD crc+0(FP), R0 // checksum
MOVD p_base+8(FP), R1 // start pointer
MOVD p_len+16(FP), R2 // length of buffer
MOVD $·const(SB), R3 // constants
MVN R0, R0
LSR $7, R2, R2
CMP $1, R2
BLT skip128
FLDPQ (R1), (F0, F1)
FLDPQ 32(R1), (F2, F3)
FLDPQ 64(R1), (F4, F5)
FLDPQ 96(R1), (F6, F7)
FMOVD R0, F8
VMOVI $0, V9.B16
VMOV V9.D[0], V8.D[1]
VEOR V8.B16, V0.B16, V0.B16
CMP $1, R2
BEQ tail128
MOVD 112(R3), R4
MOVD 120(R3), R5
FMOVD R4, F8
VDUP R5, V9.D2
loop128:
ADD $128, R1, R1
SUB $1, R2, R2
VPMULL V0.D1, V8.D1, V10.Q1
VPMULL2 V0.D2, V9.D2, V0.Q1
FLDPQ (R1), (F11, F12)
VEOR3 V0.B16, V11.B16, V10.B16, V0.B16
VPMULL V1.D1, V8.D1, V10.Q1
VPMULL2 V1.D2, V9.D2, V1.Q1
VEOR3 V1.B16, V12.B16, V10.B16, V1.B16
VPMULL V2.D1, V8.D1, V10.Q1
VPMULL2 V2.D2, V9.D2, V2.Q1
FLDPQ 32(R1), (F11, F12)
VEOR3 V2.B16, V11.B16, V10.B16, V2.B16
VPMULL V3.D1, V8.D1, V10.Q1
VPMULL2 V3.D2, V9.D2, V3.Q1
VEOR3 V3.B16, V12.B16, V10.B16, V3.B16
VPMULL V4.D1, V8.D1, V10.Q1
VPMULL2 V4.D2, V9.D2, V4.Q1
FLDPQ 64(R1), (F11, F12)
VEOR3 V4.B16, V11.B16, V10.B16, V4.B16
VPMULL V5.D1, V8.D1, V10.Q1
VPMULL2 V5.D2, V9.D2, V5.Q1
VEOR3 V5.B16, V12.B16, V10.B16, V5.B16
VPMULL V6.D1, V8.D1, V10.Q1
VPMULL2 V6.D2, V9.D2, V6.Q1
FLDPQ 96(R1), (F11, F12)
VEOR3 V6.B16, V11.B16, V10.B16, V6.B16
VPMULL V7.D1, V8.D1, V10.Q1
VPMULL2 V7.D2, V9.D2, V7.Q1
VEOR3 V7.B16, V12.B16, V10.B16, V7.B16
CMP $1, R2
BHI loop128
tail128:
MOVD (R3), R4
FMOVD R4, F11
VPMULL V0.D1, V11.D1, V11.Q1
MOVD 8(R3), R4
VDUP R4, V12.D2
VPMULL2 V0.D2, V12.D2, V0.Q1
VEOR3 V0.B16, V7.B16, V11.B16, V7.B16
MOVD 16(R3), R4
FMOVD R4, F11
VPMULL V1.D1, V11.D1, V11.Q1
MOVD 24(R3), R4
VDUP R4, V12.D2
VPMULL2 V1.D2, V12.D2, V1.Q1
VEOR3 V1.B16, V11.B16, V7.B16, V1.B16
MOVD 32(R3), R4
FMOVD R4, F11
VPMULL V2.D1, V11.D1, V11.Q1
MOVD 40(R3), R4
VDUP R4, V12.D2
VPMULL2 V2.D2, V12.D2, V2.Q1
VEOR3 V2.B16, V11.B16, V1.B16, V2.B16
MOVD 48(R3), R4
FMOVD R4, F11
VPMULL V3.D1, V11.D1, V11.Q1
MOVD 56(R3), R4
VDUP R4, V12.D2
VPMULL2 V3.D2, V12.D2, V3.Q1
VEOR3 V3.B16, V11.B16, V2.B16, V3.B16
MOVD 64(R3), R4
FMOVD R4, F11
VPMULL V4.D1, V11.D1, V11.Q1
MOVD 72(R3), R4
VDUP R4, V12.D2
VPMULL2 V4.D2, V12.D2, V4.Q1
VEOR3 V4.B16, V11.B16, V3.B16, V4.B16
MOVD 80(R3), R4
FMOVD R4, F11
VPMULL V5.D1, V11.D1, V11.Q1
MOVD 88(R3), R4
VDUP R4, V12.D2
VPMULL2 V5.D2, V12.D2, V5.Q1
VEOR3 V5.B16, V11.B16, V4.B16, V5.B16
MOVD 96(R3), R4
FMOVD R4, F11
VPMULL V6.D1, V11.D1, V11.Q1
MOVD 104(R3), R4
VDUP R4, V12.D2
VPMULL2 V6.D2, V12.D2, V6.Q1
VEOR3 V6.B16, V11.B16, V5.B16, V6.B16
FMOVD R4, F5
VPMULL V6.D1, V5.D1, V5.Q1
VDUP V6.D[1], V6.D2
VEOR V5.B8, V6.B8, V6.B8
MOVD 128(R3), R4
FMOVD R4, F4
VPMULL V4.D1, V6.D1, V6.Q1
FMOVD F6, R4
MOVD 136(R3), R5
FMOVD R5, F4
VPMULL V4.D1, V6.D1, V6.Q1
VEOR V6.B16, V5.B16, V6.B16
VMOV V6.D[1], R5
EOR R4, R5, R0
skip128:
MVN R0, R0
MOVD R0, checksum+32(FP)
RET
DATA ·const+0x000(SB)/8, $0xd083dd594d96319d // K_959
DATA ·const+0x008(SB)/8, $0x946588403d4adcbc // K_895
DATA ·const+0x010(SB)/8, $0x3c255f5ebc414423 // K_831
DATA ·const+0x018(SB)/8, $0x34f5a24e22d66e90 // K_767
DATA ·const+0x020(SB)/8, $0x7b0ab10dd0f809fe // K_703
DATA ·const+0x028(SB)/8, $0x03363823e6e791e5 // K_639
DATA ·const+0x030(SB)/8, $0x0c32cdb31e18a84a // K_575
DATA ·const+0x038(SB)/8, $0x62242240ace5045a // K_511
DATA ·const+0x040(SB)/8, $0xbdd7ac0ee1a4a0f0 // K_447
DATA ·const+0x048(SB)/8, $0xa3ffdc1fe8e82a8b // K_383
DATA ·const+0x050(SB)/8, $0xb0bc2e589204f500 // K_319
DATA ·const+0x058(SB)/8, $0xe1e0bb9d45d7a44c // K_255
DATA ·const+0x060(SB)/8, $0xeadc41fd2ba3d420 // K_191
DATA ·const+0x068(SB)/8, $0x21e9761e252621ac // K_127
DATA ·const+0x070(SB)/8, $0xa1ca681e733f9c40 // K_1087
DATA ·const+0x078(SB)/8, $0x5f852fb61e8d92dc // K_1023
DATA ·const+0x080(SB)/8, $0x27ecfa329aef9f77 // MU
DATA ·const+0x088(SB)/8, $0x34d926535897936b // POLY
GLOBL ·const(SB), (NOPTR+RODATA), $144

11
vendor/github.com/minio/crc64nvme/crc64_other.go generated vendored Normal file
View file

@ -0,0 +1,11 @@
// Copyright (c) 2025 Minio Inc. All rights reserved.
// Use of this source code is governed by a license that can be
// found in the LICENSE file.
//go:build (!amd64 || noasm || appengine || gccgo) && (!arm64 || noasm || appengine || gccgo)
package crc64nvme
var hasAsm = false
func updateAsm(crc uint64, p []byte) (checksum uint64) { panic("should not be reached") }

View file

@ -1,27 +1,72 @@
linters-settings:
misspell:
locale: US
version: "2"
linters:
disable-all: true
enable:
- typecheck
- goimports
- misspell
- revive
- durationcheck
- gocritic
- gomodguard
- govet
- ineffassign
- gosimple
- misspell
- revive
- staticcheck
- unconvert
- unused
- gocritic
- usetesting
- whitespace
settings:
misspell:
locale: US
staticcheck:
checks:
- all
- -SA1008
- -SA1019
- -SA4000
- -SA9004
- -ST1000
- -ST1005
- -ST1016
- -ST1021
- -ST1020
- -U1000
exclusions:
generated: lax
rules:
- path: (.+)\.go$
text: "empty-block:"
- path: (.+)\.go$
text: "unused-parameter:"
- path: (.+)\.go$
text: "dot-imports:"
- path: (.+)\.go$
text: "singleCaseSwitch: should rewrite switch statement to if statement"
- path: (.+)\.go$
text: "unlambda: replace"
- path: (.+)\.go$
text: "captLocal:"
- path: (.+)\.go$
text: "should have a package comment"
- path: (.+)\.go$
text: "ifElseChain:"
- path: (.+)\.go$
text: "elseif:"
- path: (.+)\.go$
text: "Error return value of"
- path: (.+)\.go$
text: "unnecessary conversion"
- path: (.+)\.go$
text: "Error return value is not checked"
issues:
exclude-use-default: false
exclude:
# todo fix these when we get enough time.
- "singleCaseSwitch: should rewrite switch statement to if statement"
- "unlambda: replace"
- "captLocal:"
- "ifElseChain:"
- "elseif:"
- "should have a package comment"
max-issues-per-linter: 100
max-same-issues: 100
formatters:
enable:
- gofumpt
- goimports
exclusions:
generated: lax
paths:
- third_party$
- builtin$
- examples$

View file

@ -251,7 +251,6 @@ func (c *Client) ListenBucketNotification(ctx context.Context, bucketName, prefi
// Close current connection before looping further.
closeResponse(resp)
}
}(notificationInfoCh)

View file

@ -90,6 +90,7 @@ type BucketVersioningConfiguration struct {
// Requires versioning to be enabled
ExcludedPrefixes []ExcludedPrefix `xml:",omitempty"`
ExcludeFolders bool `xml:",omitempty"`
PurgeOnDelete string `xml:",omitempty"`
}
// Various supported states

View file

@ -30,6 +30,7 @@ import (
"github.com/google/uuid"
"github.com/minio/minio-go/v7/pkg/encrypt"
"github.com/minio/minio-go/v7/pkg/s3utils"
"github.com/minio/minio-go/v7/pkg/tags"
)
// CopyDestOptions represents options specified by user for CopyObject/ComposeObject APIs
@ -98,8 +99,8 @@ func (opts CopyDestOptions) Marshal(header http.Header) {
const replaceDirective = "REPLACE"
if opts.ReplaceTags {
header.Set(amzTaggingHeaderDirective, replaceDirective)
if tags := s3utils.TagEncode(opts.UserTags); tags != "" {
header.Set(amzTaggingHeader, tags)
if tags, _ := tags.NewTags(opts.UserTags, true); tags != nil {
header.Set(amzTaggingHeader, tags.String())
}
}
@ -236,7 +237,9 @@ func (c *Client) copyObjectDo(ctx context.Context, srcBucket, srcObject, destBuc
}
if len(dstOpts.UserTags) != 0 {
headers.Set(amzTaggingHeader, s3utils.TagEncode(dstOpts.UserTags))
if tags, _ := tags.NewTags(dstOpts.UserTags, true); tags != nil {
headers.Set(amzTaggingHeader, tags.String())
}
}
reqMetadata := requestMetadata{

View file

@ -212,6 +212,8 @@ type ObjectInfo struct {
// not to be confused with `Expires` HTTP header.
Expiration time.Time
ExpirationRuleID string
// NumVersions is the number of versions of the object.
NumVersions int
Restore *RestoreInfo

View file

@ -135,16 +135,16 @@ func getAmzGrantACL(aCPolicy *accessControlPolicy) map[string][]string {
res := map[string][]string{}
for _, g := range grants {
switch {
case g.Permission == "READ":
switch g.Permission {
case "READ":
res["X-Amz-Grant-Read"] = append(res["X-Amz-Grant-Read"], "id="+g.Grantee.ID)
case g.Permission == "WRITE":
case "WRITE":
res["X-Amz-Grant-Write"] = append(res["X-Amz-Grant-Write"], "id="+g.Grantee.ID)
case g.Permission == "READ_ACP":
case "READ_ACP":
res["X-Amz-Grant-Read-Acp"] = append(res["X-Amz-Grant-Read-Acp"], "id="+g.Grantee.ID)
case g.Permission == "WRITE_ACP":
case "WRITE_ACP":
res["X-Amz-Grant-Write-Acp"] = append(res["X-Amz-Grant-Write-Acp"], "id="+g.Grantee.ID)
case g.Permission == "FULL_CONTROL":
case "FULL_CONTROL":
res["X-Amz-Grant-Full-Control"] = append(res["X-Amz-Grant-Full-Control"], "id="+g.Grantee.ID)
}
}

View file

@ -22,6 +22,7 @@ import (
"fmt"
"net/http"
"net/url"
"slices"
"time"
"github.com/minio/minio-go/v7/pkg/s3utils"
@ -421,20 +422,17 @@ func (c *Client) listObjectVersions(ctx context.Context, bucketName string, opts
var (
keyMarker = ""
versionIDMarker = ""
preName = ""
preKey = ""
perVersions []Version
numVersions int
)
for {
// Get list of objects a maximum of 1000 per request.
result, err := c.listObjectVersionsQuery(ctx, bucketName, opts, keyMarker, versionIDMarker, delimiter)
if err != nil {
sendObjectInfo(ObjectInfo{
Err: err,
})
return
send := func(vers []Version) {
if opts.WithVersions && opts.ReverseVersions {
slices.Reverse(vers)
numVersions = len(vers)
}
// If contents are available loop through and send over channel.
for _, version := range result.Versions {
for _, version := range vers {
info := ObjectInfo{
ETag: trimEtag(version.ETag),
Key: version.Key,
@ -448,6 +446,7 @@ func (c *Client) listObjectVersions(ctx context.Context, bucketName string, opts
UserTags: version.UserTags,
UserMetadata: version.UserMetadata,
Internal: version.Internal,
NumVersions: numVersions,
}
select {
// Send object version info.
@ -457,6 +456,38 @@ func (c *Client) listObjectVersions(ctx context.Context, bucketName string, opts
return
}
}
}
for {
// Get list of objects a maximum of 1000 per request.
result, err := c.listObjectVersionsQuery(ctx, bucketName, opts, keyMarker, versionIDMarker, delimiter)
if err != nil {
sendObjectInfo(ObjectInfo{
Err: err,
})
return
}
if opts.WithVersions && opts.ReverseVersions {
for _, version := range result.Versions {
if preName == "" {
preName = result.Name
preKey = version.Key
}
if result.Name == preName && preKey == version.Key {
// If the current name is same as previous name,
// we need to append the version to the previous version.
perVersions = append(perVersions, version)
continue
}
// Send the file versions.
send(perVersions)
perVersions = perVersions[:0]
perVersions = append(perVersions, version)
preName = result.Name
preKey = version.Key
}
} else {
send(result.Versions)
}
// Send all common prefixes if any.
// NOTE: prefixes are only present if the request is delimited.
@ -480,8 +511,17 @@ func (c *Client) listObjectVersions(ctx context.Context, bucketName string, opts
versionIDMarker = result.NextVersionIDMarker
}
// If context is canceled, return here.
if contextCanceled(ctx) {
return
}
// Listing ends result is not truncated, return right here.
if !result.IsTruncated {
// sent the lasted file with versions
if opts.ReverseVersions && len(perVersions) > 0 {
send(perVersions)
}
return
}
}
@ -683,6 +723,8 @@ func (c *Client) listObjectsQuery(ctx context.Context, bucketName, objectPrefix,
// ListObjectsOptions holds all options of a list object request
type ListObjectsOptions struct {
// ReverseVersions - reverse the order of the object versions
ReverseVersions bool
// Include objects versions in the listing
WithVersions bool
// Include objects metadata in the listing

View file

@ -350,7 +350,6 @@ func (c *Client) putObjectMultipartStreamOptionalChecksum(ctx context.Context, b
// Part number always starts with '1'.
var partNumber int
for partNumber = 1; partNumber <= totalPartsCount; partNumber++ {
// Proceed to upload the part.
if partNumber == totalPartsCount {
partSize = lastPartSize

View file

@ -30,6 +30,7 @@ import (
"github.com/minio/minio-go/v7/pkg/encrypt"
"github.com/minio/minio-go/v7/pkg/s3utils"
"github.com/minio/minio-go/v7/pkg/tags"
"golang.org/x/net/http/httpguts"
)
@ -229,7 +230,9 @@ func (opts PutObjectOptions) Header() (header http.Header) {
}
if len(opts.UserTags) != 0 {
header.Set(amzTaggingHeader, s3utils.TagEncode(opts.UserTags))
if tags, _ := tags.NewTags(opts.UserTags, true); tags != nil {
header.Set(amzTaggingHeader, tags.String())
}
}
for k, v := range opts.UserMetadata {

View file

@ -392,10 +392,7 @@ func (c *Client) removeObjects(ctx context.Context, bucketName string, objectsCh
defer close(resultCh)
// Loop over entries by 1000 and call MultiDelete requests
for {
if finish {
break
}
for !finish {
count := 0
var batch []ObjectInfo

View file

@ -194,7 +194,6 @@ func (l *ListVersionsResult) UnmarshalXML(d *xml.Decoder, _ xml.StartElement) (e
default:
return errors.New("unrecognized option:" + tagName)
}
}
}
return nil

View file

@ -609,7 +609,6 @@ func (s *SelectResults) start(pipeWriter *io.PipeWriter) {
closeResponse(s.resp)
return
}
}
}()
}
@ -669,7 +668,6 @@ func extractHeader(body io.Reader, myHeaders http.Header) error {
}
myHeaders.Set(headerTypeName, headerValueName)
}
return nil
}

View file

@ -155,7 +155,7 @@ type Options struct {
// Global constants.
const (
libraryName = "minio-go"
libraryVersion = "v7.0.85"
libraryVersion = "v7.0.89"
)
// User Agent should always following the below style.
@ -598,7 +598,7 @@ func (c *Client) do(req *http.Request) (resp *http.Response, err error) {
// If trace is enabled, dump http request and response,
// except when the traceErrorsOnly enabled and the response's status code is ok
if c.isTraceEnabled && !(c.traceErrorsOnly && resp.StatusCode == http.StatusOK) {
if c.isTraceEnabled && (!c.traceErrorsOnly || resp.StatusCode != http.StatusOK) {
err = c.dumpHTTP(req, resp)
if err != nil {
return nil, err

View file

@ -30,6 +30,8 @@ import (
"math/bits"
"net/http"
"sort"
"github.com/minio/crc64nvme"
)
// ChecksumType contains information about the checksum type.
@ -152,9 +154,6 @@ func (c ChecksumType) RawByteLen() int {
const crc64NVMEPolynomial = 0xad93d23594c93659
// crc64 uses reversed polynomials.
var crc64Table = crc64.MakeTable(bits.Reverse64(crc64NVMEPolynomial))
// Hasher returns a hasher corresponding to the checksum type.
// Returns nil if no checksum.
func (c ChecksumType) Hasher() hash.Hash {
@ -168,7 +167,7 @@ func (c ChecksumType) Hasher() hash.Hash {
case ChecksumSHA256:
return sha256.New()
case ChecksumCRC64NVME:
return crc64.New(crc64Table)
return crc64nvme.New()
}
return nil
}

View file

@ -104,6 +104,8 @@ type STSAssumeRoleOptions struct {
RoleARN string
RoleSessionName string
ExternalID string
TokenRevokeType string // Optional, used for token revokation (MinIO only extension)
}
// NewSTSAssumeRole returns a pointer to a new
@ -161,6 +163,9 @@ func getAssumeRoleCredentials(clnt *http.Client, endpoint string, opts STSAssume
if opts.ExternalID != "" {
v.Set("ExternalId", opts.ExternalID)
}
if opts.TokenRevokeType != "" {
v.Set("TokenRevokeType", opts.TokenRevokeType)
}
u, err := url.Parse(endpoint)
if err != nil {

View file

@ -69,6 +69,9 @@ type CustomTokenIdentity struct {
// RequestedExpiry is to set the validity of the generated credentials
// (this value bounded by server).
RequestedExpiry time.Duration
// Optional, used for token revokation
TokenRevokeType string
}
// RetrieveWithCredContext with Retrieve optionally cred context
@ -98,6 +101,9 @@ func (c *CustomTokenIdentity) RetrieveWithCredContext(cc *CredContext) (value Va
if c.RequestedExpiry != 0 {
v.Set("DurationSeconds", fmt.Sprintf("%d", int(c.RequestedExpiry.Seconds())))
}
if c.TokenRevokeType != "" {
v.Set("TokenRevokeType", c.TokenRevokeType)
}
u.RawQuery = v.Encode()

View file

@ -73,6 +73,9 @@ type LDAPIdentity struct {
// RequestedExpiry is the configured expiry duration for credentials
// requested from LDAP.
RequestedExpiry time.Duration
// Optional, used for token revokation
TokenRevokeType string
}
// NewLDAPIdentity returns new credentials object that uses LDAP
@ -152,6 +155,9 @@ func (k *LDAPIdentity) RetrieveWithCredContext(cc *CredContext) (value Value, er
if k.RequestedExpiry != 0 {
v.Set("DurationSeconds", fmt.Sprintf("%d", int(k.RequestedExpiry.Seconds())))
}
if k.TokenRevokeType != "" {
v.Set("TokenRevokeType", k.TokenRevokeType)
}
req, err := http.NewRequest(http.MethodPost, u.String(), strings.NewReader(v.Encode()))
if err != nil {

View file

@ -80,6 +80,9 @@ type STSCertificateIdentity struct {
// Certificate is the client certificate that is used for
// STS authentication.
Certificate tls.Certificate
// Optional, used for token revokation
TokenRevokeType string
}
// NewSTSCertificateIdentity returns a STSCertificateIdentity that authenticates
@ -122,6 +125,9 @@ func (i *STSCertificateIdentity) RetrieveWithCredContext(cc *CredContext) (Value
queryValues := url.Values{}
queryValues.Set("Action", "AssumeRoleWithCertificate")
queryValues.Set("Version", STSVersion)
if i.TokenRevokeType != "" {
queryValues.Set("TokenRevokeType", i.TokenRevokeType)
}
endpointURL.RawQuery = queryValues.Encode()
req, err := http.NewRequest(http.MethodPost, endpointURL.String(), nil)

View file

@ -93,6 +93,9 @@ type STSWebIdentity struct {
// roleSessionName is the identifier for the assumed role session.
roleSessionName string
// Optional, used for token revokation
TokenRevokeType string
}
// NewSTSWebIdentity returns a pointer to a new
@ -135,7 +138,7 @@ func WithPolicy(policy string) func(*STSWebIdentity) {
}
func getWebIdentityCredentials(clnt *http.Client, endpoint, roleARN, roleSessionName string, policy string,
getWebIDTokenExpiry func() (*WebIdentityToken, error),
getWebIDTokenExpiry func() (*WebIdentityToken, error), tokenRevokeType string,
) (AssumeRoleWithWebIdentityResponse, error) {
idToken, err := getWebIDTokenExpiry()
if err != nil {
@ -168,6 +171,9 @@ func getWebIdentityCredentials(clnt *http.Client, endpoint, roleARN, roleSession
v.Set("Policy", policy)
}
v.Set("Version", STSVersion)
if tokenRevokeType != "" {
v.Set("TokenRevokeType", tokenRevokeType)
}
u, err := url.Parse(endpoint)
if err != nil {
@ -236,7 +242,7 @@ func (m *STSWebIdentity) RetrieveWithCredContext(cc *CredContext) (Value, error)
return Value{}, errors.New("STS endpoint unknown")
}
a, err := getWebIdentityCredentials(client, stsEndpoint, m.RoleARN, m.roleSessionName, m.Policy, m.GetWebIDTokenExpiry)
a, err := getWebIdentityCredentials(client, stsEndpoint, m.RoleARN, m.roleSessionName, m.Policy, m.GetWebIDTokenExpiry, m.TokenRevokeType)
if err != nil {
return Value{}, err
}

View file

@ -192,7 +192,7 @@ func (t Transition) IsDaysNull() bool {
// IsDateNull returns true if date field is null
func (t Transition) IsDateNull() bool {
return t.Date.Time.IsZero()
return t.Date.IsZero()
}
// IsNull returns true if no storage-class is set.
@ -323,7 +323,7 @@ type ExpirationDate struct {
// MarshalXML encodes expiration date if it is non-zero and encodes
// empty string otherwise
func (eDate ExpirationDate) MarshalXML(e *xml.Encoder, startElement xml.StartElement) error {
if eDate.Time.IsZero() {
if eDate.IsZero() {
return nil
}
return e.EncodeElement(eDate.Format(time.RFC3339), startElement)
@ -392,7 +392,7 @@ func (e Expiration) IsDaysNull() bool {
// IsDateNull returns true if date field is null
func (e Expiration) IsDateNull() bool {
return e.Date.Time.IsZero()
return e.Date.IsZero()
}
// IsDeleteMarkerExpirationEnabled returns true if the auto-expiration of delete marker is enabled

View file

@ -283,7 +283,6 @@ func (b *Configuration) AddTopic(topicConfig Config) bool {
for _, n := range b.TopicConfigs {
// If new config matches existing one
if n.Topic == newTopicConfig.Arn.String() && newTopicConfig.Filter == n.Filter {
existingConfig := set.NewStringSet()
for _, v := range n.Events {
existingConfig.Add(string(v))
@ -308,7 +307,6 @@ func (b *Configuration) AddQueue(queueConfig Config) bool {
newQueueConfig := QueueConfig{Config: queueConfig, Queue: queueConfig.Arn.String()}
for _, n := range b.QueueConfigs {
if n.Queue == newQueueConfig.Arn.String() && newQueueConfig.Filter == n.Filter {
existingConfig := set.NewStringSet()
for _, v := range n.Events {
existingConfig.Add(string(v))
@ -333,7 +331,6 @@ func (b *Configuration) AddLambda(lambdaConfig Config) bool {
newLambdaConfig := LambdaConfig{Config: lambdaConfig, Lambda: lambdaConfig.Arn.String()}
for _, n := range b.LambdaConfigs {
if n.Lambda == newLambdaConfig.Arn.String() && newLambdaConfig.Filter == n.Filter {
existingConfig := set.NewStringSet()
for _, v := range n.Events {
existingConfig.Add(string(v))
@ -372,7 +369,7 @@ func (b *Configuration) RemoveTopicByArnEventsPrefixSuffix(arn Arn, events []Eve
removeIndex := -1
for i, v := range b.TopicConfigs {
// if it matches events and filters, mark the index for deletion
if v.Topic == arn.String() && v.Config.Equal(events, prefix, suffix) {
if v.Topic == arn.String() && v.Equal(events, prefix, suffix) {
removeIndex = i
break // since we have at most one matching config
}
@ -400,7 +397,7 @@ func (b *Configuration) RemoveQueueByArnEventsPrefixSuffix(arn Arn, events []Eve
removeIndex := -1
for i, v := range b.QueueConfigs {
// if it matches events and filters, mark the index for deletion
if v.Queue == arn.String() && v.Config.Equal(events, prefix, suffix) {
if v.Queue == arn.String() && v.Equal(events, prefix, suffix) {
removeIndex = i
break // since we have at most one matching config
}
@ -428,7 +425,7 @@ func (b *Configuration) RemoveLambdaByArnEventsPrefixSuffix(arn Arn, events []Ev
removeIndex := -1
for i, v := range b.LambdaConfigs {
// if it matches events and filters, mark the index for deletion
if v.Lambda == arn.String() && v.Config.Equal(events, prefix, suffix) {
if v.Lambda == arn.String() && v.Equal(events, prefix, suffix) {
removeIndex = i
break // since we have at most one matching config
}

View file

@ -868,8 +868,20 @@ type ReplQNodeStats struct {
XferStats map[MetricName]XferStats `json:"transferSummary"`
TgtXferStats map[string]map[MetricName]XferStats `json:"tgtTransferStats"`
QStats InQueueMetric `json:"queueStats"`
MRFStats ReplMRFStats `json:"mrfStats"`
QStats InQueueMetric `json:"queueStats"`
MRFStats ReplMRFStats `json:"mrfStats"`
Retries CounterSummary `json:"retries"`
Errors CounterSummary `json:"errors"`
}
// CounterSummary denotes the stats counter summary
type CounterSummary struct {
// Counted last 1hr
Last1hr uint64 `json:"last1hr"`
// Counted last 1m
Last1m uint64 `json:"last1m"`
// Total counted since uptime
Total uint64 `json:"total"`
}
// ReplQueueStats holds stats for replication queue across nodes
@ -914,8 +926,10 @@ type ReplQStats struct {
XferStats map[MetricName]XferStats `json:"xferStats"`
TgtXferStats map[string]map[MetricName]XferStats `json:"tgtXferStats"`
QStats InQueueMetric `json:"qStats"`
MRFStats ReplMRFStats `json:"mrfStats"`
QStats InQueueMetric `json:"qStats"`
MRFStats ReplMRFStats `json:"mrfStats"`
Retries CounterSummary `json:"retries"`
Errors CounterSummary `json:"errors"`
}
// QStats returns cluster level stats for objects in replication queue
@ -958,6 +972,12 @@ func (q ReplQueueStats) QStats() (r ReplQStats) {
r.MRFStats.LastFailedCount += node.MRFStats.LastFailedCount
r.MRFStats.TotalDroppedCount += node.MRFStats.TotalDroppedCount
r.MRFStats.TotalDroppedBytes += node.MRFStats.TotalDroppedBytes
r.Retries.Last1hr += node.Retries.Last1hr
r.Retries.Last1m += node.Retries.Last1m
r.Retries.Total += node.Retries.Total
r.Errors.Last1hr += node.Errors.Last1hr
r.Errors.Last1m += node.Errors.Last1m
r.Errors.Total += node.Errors.Total
r.Uptime += node.Uptime
}
if len(q.Nodes) > 0 {
@ -968,7 +988,21 @@ func (q ReplQueueStats) QStats() (r ReplQStats) {
// MetricsV2 represents replication metrics for a bucket.
type MetricsV2 struct {
Uptime int64 `json:"uptime"`
CurrentStats Metrics `json:"currStats"`
QueueStats ReplQueueStats `json:"queueStats"`
Uptime int64 `json:"uptime"`
CurrentStats Metrics `json:"currStats"`
QueueStats ReplQueueStats `json:"queueStats"`
DowntimeInfo map[string]DowntimeInfo `json:"downtimeInfo"`
}
// DowntimeInfo represents the downtime info
type DowntimeInfo struct {
Duration Stat `json:"duration"`
Count Stat `json:"count"`
}
// Stat represents the aggregates
type Stat struct {
Total int64 `json:"total"`
Avg int64 `json:"avg"`
Max int64 `json:"max"`
}

View file

@ -218,7 +218,7 @@ func IsAmazonPrivateLinkEndpoint(endpointURL url.URL) bool {
if endpointURL == sentinelURL {
return false
}
return amazonS3HostPrivateLink.MatchString(endpointURL.Host)
return amazonS3HostPrivateLink.MatchString(endpointURL.Hostname())
}
// IsGoogleEndpoint - Match if it is exactly Google cloud storage endpoint.
@ -261,44 +261,6 @@ func QueryEncode(v url.Values) string {
return buf.String()
}
// TagDecode - decodes canonical tag into map of key and value.
func TagDecode(ctag string) map[string]string {
if ctag == "" {
return map[string]string{}
}
tags := strings.Split(ctag, "&")
tagMap := make(map[string]string, len(tags))
var err error
for _, tag := range tags {
kvs := strings.SplitN(tag, "=", 2)
if len(kvs) == 0 {
return map[string]string{}
}
if len(kvs) == 1 {
return map[string]string{}
}
tagMap[kvs[0]], err = url.PathUnescape(kvs[1])
if err != nil {
continue
}
}
return tagMap
}
// TagEncode - encodes tag values in their URL encoded form. In
// addition to the percent encoding performed by urlEncodePath() used
// here, it also percent encodes '/' (forward slash)
func TagEncode(tags map[string]string) string {
if tags == nil {
return ""
}
values := url.Values{}
for k, v := range tags {
values[k] = []string{v}
}
return QueryEncode(values)
}
// if object matches reserved string, no need to encode them
var reservedObjectNames = regexp.MustCompile("^[a-zA-Z0-9-_.~/]+$")

View file

@ -212,7 +212,6 @@ func (s *StreamingUSReader) Read(buf []byte) (int, error) {
}
return 0, err
}
}
}
return s.buf.Read(buf)

View file

@ -387,7 +387,6 @@ func (s *StreamingReader) Read(buf []byte) (int, error) {
}
return 0, err
}
}
}
return s.buf.Read(buf)

View file

@ -148,7 +148,7 @@ func SignV2(req http.Request, accessKeyID, secretAccessKey string, virtualHost b
// Prepare auth header.
authHeader := new(bytes.Buffer)
authHeader.WriteString(fmt.Sprintf("%s %s:", signV2Algorithm, accessKeyID))
fmt.Fprintf(authHeader, "%s %s:", signV2Algorithm, accessKeyID)
encoder := base64.NewEncoder(base64.StdEncoding, authHeader)
encoder.Write(hm.Sum(nil))
encoder.Close()

View file

@ -128,8 +128,8 @@ func getCanonicalHeaders(req http.Request, ignoredHeaders map[string]bool) strin
for _, k := range headers {
buf.WriteString(k)
buf.WriteByte(':')
switch {
case k == "host":
switch k {
case "host":
buf.WriteString(getHostAddr(&req))
buf.WriteByte('\n')
default:

View file

@ -41,6 +41,7 @@ import (
md5simd "github.com/minio/md5-simd"
"github.com/minio/minio-go/v7/pkg/s3utils"
"github.com/minio/minio-go/v7/pkg/tags"
)
func trimEtag(etag string) string {
@ -322,7 +323,13 @@ func ToObjectInfo(bucketName, objectName string, h http.Header) (ObjectInfo, err
userMetadata[strings.TrimPrefix(k, "X-Amz-Meta-")] = v[0]
}
}
userTags := s3utils.TagDecode(h.Get(amzTaggingHeader))
userTags, err := tags.ParseObjectTags(h.Get(amzTaggingHeader))
if err != nil {
return ObjectInfo{}, ErrorResponse{
Code: "InternalError",
}
}
var tagCount int
if count := h.Get(amzTaggingCount); count != "" {
@ -373,7 +380,7 @@ func ToObjectInfo(bucketName, objectName string, h http.Header) (ObjectInfo, err
// which are not part of object metadata.
Metadata: metadata,
UserMetadata: userMetadata,
UserTags: userTags,
UserTags: userTags.ToMap(),
UserTagCount: tagCount,
Restore: restore,

View file

@ -65,17 +65,20 @@ db.QueryRow(`SELECT sqlite_version()`).Scan(&version)
This module replaces the SQLite [OS Interface](https://sqlite.org/vfs.html)
(aka VFS) with a [pure Go](vfs/) implementation,
which has advantages and disadvantages.
Read more about the Go VFS design [here](vfs/README.md).
Because each database connection executes within a Wasm sandboxed environment,
memory usage will be higher than alternatives.
### Testing
This project aims for [high test coverage](https://github.com/ncruces/go-sqlite3/wiki/Test-coverage-report).
It also benefits greatly from [SQLite's](https://sqlite.org/testing.html) and
[wazero's](https://tetrate.io/blog/introducing-wazero-from-tetrate/#:~:text=Rock%2Dsolid%20test%20approach) thorough testing.
[wazero's](https://tetrate.io/blog/introducing-wazero-from-tetrate/#:~:text=Rock%2Dsolid%20test%20approach)
thorough testing.
Every commit is [tested](https://github.com/ncruces/go-sqlite3/wiki/Support-matrix) on
Linux (amd64/arm64/386/riscv64/ppc64le/s390x), macOS (amd64/arm64),
Linux (amd64/arm64/386/riscv64/ppc64le/s390x), macOS (arm64/amd64),
Windows (amd64), FreeBSD (amd64/arm64), OpenBSD (amd64), NetBSD (amd64/arm64),
DragonFly BSD (amd64), illumos (amd64), and Solaris (amd64).
@ -84,12 +87,21 @@ The Go VFS is tested by running SQLite's
### Performance
Perfomance of the [`database/sql`](https://pkg.go.dev/database/sql) driver is
Performance of the [`database/sql`](https://pkg.go.dev/database/sql) driver is
[competitive](https://github.com/cvilsmeier/go-sqlite-bench) with alternatives.
The Wasm and VFS layers are also tested by running SQLite's
The Wasm and VFS layers are also benchmarked by running SQLite's
[speedtest1](https://github.com/sqlite/sqlite/blob/master/test/speedtest1.c).
### Concurrency
This module behaves similarly to SQLite in [multi-thread](https://sqlite.org/threadsafe.html) mode:
it is goroutine-safe, provided that no single database connection, or object derived from it,
is used concurrently by multiple goroutines.
The [`database/sql`](https://pkg.go.dev/database/sql) API is safe to use concurrently,
according to its documentation.
### FAQ, issues, new features
For questions, please see [Discussions](https://github.com/ncruces/go-sqlite3/discussions/categories/q-a).

View file

@ -31,6 +31,10 @@ var _ io.ReadWriteSeeker = &Blob{}
//
// https://sqlite.org/c3ref/blob_open.html
func (c *Conn) OpenBlob(db, table, column string, row int64, write bool) (*Blob, error) {
if c.interrupt.Err() != nil {
return nil, INTERRUPT
}
defer c.arena.mark()()
blobPtr := c.arena.new(ptrlen)
dbPtr := c.arena.string(db)
@ -42,7 +46,6 @@ func (c *Conn) OpenBlob(db, table, column string, row int64, write bool) (*Blob,
flags = 1
}
c.checkInterrupt(c.handle)
rc := res_t(c.call("sqlite3_blob_open", stk_t(c.handle),
stk_t(dbPtr), stk_t(tablePtr), stk_t(columnPtr),
stk_t(row), stk_t(flags), stk_t(blobPtr)))
@ -253,7 +256,9 @@ func (b *Blob) Seek(offset int64, whence int) (int64, error) {
//
// https://sqlite.org/c3ref/blob_reopen.html
func (b *Blob) Reopen(row int64) error {
b.c.checkInterrupt(b.c.handle)
if b.c.interrupt.Err() != nil {
return INTERRUPT
}
err := b.c.error(res_t(b.c.call("sqlite3_blob_reopen", stk_t(b.handle), stk_t(row))))
b.bytes = int64(int32(b.c.call("sqlite3_blob_bytes", stk_t(b.handle))))
b.offset = 0

View file

@ -4,6 +4,7 @@ import (
"context"
"fmt"
"strconv"
"sync/atomic"
"github.com/tetratelabs/wazero/api"
@ -48,6 +49,15 @@ func (c *Conn) Config(op DBConfig, arg ...bool) (bool, error) {
return util.ReadBool(c.mod, argsPtr), c.error(rc)
}
var defaultLogger atomic.Pointer[func(code ExtendedErrorCode, msg string)]
// ConfigLog sets up the default error logging callback for new connections.
//
// https://sqlite.org/errlog.html
func ConfigLog(cb func(code ExtendedErrorCode, msg string)) {
defaultLogger.Store(&cb)
}
// ConfigLog sets up the error logging callback for the connection.
//
// https://sqlite.org/errlog.html
@ -265,6 +275,10 @@ func traceCallback(ctx context.Context, mod api.Module, evt TraceEvent, pDB, pAr
//
// https://sqlite.org/c3ref/wal_checkpoint_v2.html
func (c *Conn) WALCheckpoint(schema string, mode CheckpointMode) (nLog, nCkpt int, err error) {
if c.interrupt.Err() != nil {
return 0, 0, INTERRUPT
}
defer c.arena.mark()()
nLogPtr := c.arena.new(ptrlen)
nCkptPtr := c.arena.new(ptrlen)
@ -378,6 +392,6 @@ func (c *Conn) EnableChecksums(schema string) error {
}
// Checkpoint the WAL.
_, _, err = c.WALCheckpoint(schema, CHECKPOINT_RESTART)
_, _, err = c.WALCheckpoint(schema, CHECKPOINT_FULL)
return err
}

View file

@ -25,7 +25,6 @@ type Conn struct {
*sqlite
interrupt context.Context
pending *Stmt
stmts []*Stmt
busy func(context.Context, int) bool
log func(xErrorCode, string)
@ -41,6 +40,7 @@ type Conn struct {
busylst time.Time
arena arena
handle ptr_t
gosched uint8
}
// Open calls [OpenFlags] with [OPEN_READWRITE], [OPEN_CREATE] and [OPEN_URI].
@ -49,7 +49,7 @@ func Open(filename string) (*Conn, error) {
}
// OpenContext is like [Open] but includes a context,
// which is used to interrupt the process of opening the connectiton.
// which is used to interrupt the process of opening the connection.
func OpenContext(ctx context.Context, filename string) (*Conn, error) {
return newConn(ctx, filename, OPEN_READWRITE|OPEN_CREATE|OPEN_URI)
}
@ -92,6 +92,9 @@ func newConn(ctx context.Context, filename string, flags OpenFlag) (ret *Conn, _
}()
c.ctx = context.WithValue(c.ctx, connKey{}, c)
if logger := defaultLogger.Load(); logger != nil {
c.ConfigLog(*logger)
}
c.arena = c.newArena()
c.handle, err = c.openDB(filename, flags)
if err == nil {
@ -117,7 +120,7 @@ func (c *Conn) openDB(filename string, flags OpenFlag) (ptr_t, error) {
return 0, err
}
c.call("sqlite3_progress_handler_go", stk_t(handle), 100)
c.call("sqlite3_progress_handler_go", stk_t(handle), 1000)
if flags|OPEN_URI != 0 && strings.HasPrefix(filename, "file:") {
var pragmas strings.Builder
if _, after, ok := strings.Cut(filename, "?"); ok {
@ -129,7 +132,6 @@ func (c *Conn) openDB(filename string, flags OpenFlag) (ptr_t, error) {
}
}
if pragmas.Len() != 0 {
c.checkInterrupt(handle)
pragmaPtr := c.arena.string(pragmas.String())
rc := res_t(c.call("sqlite3_exec", stk_t(handle), stk_t(pragmaPtr), 0, 0, 0))
if err := c.sqlite.error(rc, handle, pragmas.String()); err != nil {
@ -163,9 +165,6 @@ func (c *Conn) Close() error {
return nil
}
c.pending.Close()
c.pending = nil
rc := res_t(c.call("sqlite3_close", stk_t(c.handle)))
if err := c.error(rc); err != nil {
return err
@ -180,11 +179,16 @@ func (c *Conn) Close() error {
//
// https://sqlite.org/c3ref/exec.html
func (c *Conn) Exec(sql string) error {
defer c.arena.mark()()
sqlPtr := c.arena.string(sql)
if c.interrupt.Err() != nil {
return INTERRUPT
}
return c.exec(sql)
}
c.checkInterrupt(c.handle)
rc := res_t(c.call("sqlite3_exec", stk_t(c.handle), stk_t(sqlPtr), 0, 0, 0))
func (c *Conn) exec(sql string) error {
defer c.arena.mark()()
textPtr := c.arena.string(sql)
rc := res_t(c.call("sqlite3_exec", stk_t(c.handle), stk_t(textPtr), 0, 0, 0))
return c.error(rc, sql)
}
@ -203,20 +207,22 @@ func (c *Conn) PrepareFlags(sql string, flags PrepareFlag) (stmt *Stmt, tail str
if len(sql) > _MAX_SQL_LENGTH {
return nil, "", TOOBIG
}
if c.interrupt.Err() != nil {
return nil, "", INTERRUPT
}
defer c.arena.mark()()
stmtPtr := c.arena.new(ptrlen)
tailPtr := c.arena.new(ptrlen)
sqlPtr := c.arena.string(sql)
textPtr := c.arena.string(sql)
c.checkInterrupt(c.handle)
rc := res_t(c.call("sqlite3_prepare_v3", stk_t(c.handle),
stk_t(sqlPtr), stk_t(len(sql)+1), stk_t(flags),
stk_t(textPtr), stk_t(len(sql)+1), stk_t(flags),
stk_t(stmtPtr), stk_t(tailPtr)))
stmt = &Stmt{c: c}
stmt = &Stmt{c: c, sql: sql}
stmt.handle = util.Read32[ptr_t](c.mod, stmtPtr)
if sql := sql[util.Read32[ptr_t](c.mod, tailPtr)-sqlPtr:]; sql != "" {
if sql := sql[util.Read32[ptr_t](c.mod, tailPtr)-textPtr:]; sql != "" {
tail = sql
}
@ -337,43 +343,17 @@ func (c *Conn) GetInterrupt() context.Context {
//
// https://sqlite.org/c3ref/interrupt.html
func (c *Conn) SetInterrupt(ctx context.Context) (old context.Context) {
if ctx == nil {
panic("nil Context")
}
old = c.interrupt
c.interrupt = ctx
if ctx == old || ctx.Done() == old.Done() {
return old
}
// A busy SQL statement prevents SQLite from ignoring an interrupt
// that comes before any other statements are started.
if c.pending == nil {
defer c.arena.mark()()
stmtPtr := c.arena.new(ptrlen)
loopPtr := c.arena.string(`WITH RECURSIVE c(x) AS (VALUES(0) UNION ALL SELECT x FROM c) SELECT x FROM c`)
c.call("sqlite3_prepare_v3", stk_t(c.handle), stk_t(loopPtr), math.MaxUint64,
stk_t(PREPARE_PERSISTENT), stk_t(stmtPtr), 0)
c.pending = &Stmt{c: c}
c.pending.handle = util.Read32[ptr_t](c.mod, stmtPtr)
}
if old.Done() != nil && ctx.Err() == nil {
c.pending.Reset()
}
if ctx.Done() != nil {
c.pending.Step()
}
return old
}
func (c *Conn) checkInterrupt(handle ptr_t) {
if c.interrupt.Err() != nil {
c.call("sqlite3_interrupt", stk_t(handle))
}
}
func progressCallback(ctx context.Context, mod api.Module, _ ptr_t) (interrupt int32) {
if c, ok := ctx.Value(connKey{}).(*Conn); ok {
if c.interrupt.Done() != nil {
if c.gosched++; c.gosched%16 == 0 {
runtime.Gosched()
}
if c.interrupt.Err() != nil {
@ -429,11 +409,8 @@ func (c *Conn) BusyHandler(cb func(ctx context.Context, count int) (retry bool))
func busyCallback(ctx context.Context, mod api.Module, pDB ptr_t, count int32) (retry int32) {
if c, ok := ctx.Value(connKey{}).(*Conn); ok && c.handle == pDB && c.busy != nil {
interrupt := c.interrupt
if interrupt == nil {
interrupt = context.Background()
}
if interrupt.Err() == nil && c.busy(interrupt, int(count)) {
if interrupt := c.interrupt; interrupt.Err() == nil &&
c.busy(interrupt, int(count)) {
retry = 1
}
}

View file

@ -11,10 +11,9 @@ const (
_ROW = 100 /* sqlite3_step() has another row ready */
_DONE = 101 /* sqlite3_step() has finished executing */
_MAX_NAME = 1e6 // Self-imposed limit for most NUL terminated strings.
_MAX_LENGTH = 1e9
_MAX_SQL_LENGTH = 1e9
_MAX_FUNCTION_ARG = 100
_MAX_NAME = 1e6 // Self-imposed limit for most NUL terminated strings.
_MAX_LENGTH = 1e9
_MAX_SQL_LENGTH = 1e9
ptrlen = util.PtrLen
intlen = util.IntLen

View file

@ -89,20 +89,26 @@ func (ctx Context) ResultText(value string) {
}
// ResultRawText sets the text result of the function to a []byte.
// Returning a nil slice is the same as calling [Context.ResultNull].
//
// https://sqlite.org/c3ref/result_blob.html
func (ctx Context) ResultRawText(value []byte) {
if len(value) == 0 {
ctx.ResultText("")
return
}
ptr := ctx.c.newBytes(value)
ctx.c.call("sqlite3_result_text_go",
stk_t(ctx.handle), stk_t(ptr), stk_t(len(value)))
}
// ResultBlob sets the result of the function to a []byte.
// Returning a nil slice is the same as calling [Context.ResultNull].
//
// https://sqlite.org/c3ref/result_blob.html
func (ctx Context) ResultBlob(value []byte) {
if len(value) == 0 {
ctx.ResultZeroBlob(0)
return
}
ptr := ctx.c.newBytes(value)
ctx.c.call("sqlite3_result_blob_go",
stk_t(ctx.handle), stk_t(ptr), stk_t(len(value)))

View file

@ -20,22 +20,45 @@
// - a [serializable] transaction is always "immediate";
// - a [read-only] transaction is always "deferred".
//
// # Datatypes In SQLite
//
// SQLite is dynamically typed.
// Columns can mostly hold any value regardless of their declared type.
// SQLite supports most [driver.Value] types out of the box,
// but bool and [time.Time] require special care.
//
// Booleans can be stored on any column type and scanned back to a *bool.
// However, if scanned to a *any, booleans may either become an
// int64, string or bool, depending on the declared type of the column.
// If you use BOOLEAN for your column type,
// 1 and 0 will always scan as true and false.
//
// # Working with time
//
// Time values can similarly be stored on any column type.
// The time encoding/decoding format can be specified using "_timefmt":
//
// sql.Open("sqlite3", "file:demo.db?_timefmt=sqlite")
//
// Possible values are: "auto" (the default), "sqlite", "rfc3339";
// Special values are: "auto" (the default), "sqlite", "rfc3339";
// - "auto" encodes as RFC 3339 and decodes any [format] supported by SQLite;
// - "sqlite" encodes as SQLite and decodes any [format] supported by SQLite;
// - "rfc3339" encodes and decodes RFC 3339 only.
//
// If you encode as RFC 3339 (the default),
// consider using the TIME [collating sequence] to produce a time-ordered sequence.
// You can also set "_timefmt" to an arbitrary [sqlite3.TimeFormat] or [time.Layout].
//
// To scan values in other formats, [sqlite3.TimeFormat.Scanner] may be helpful.
// To bind values in other formats, [sqlite3.TimeFormat.Encode] them before binding.
// If you encode as RFC 3339 (the default),
// consider using the TIME [collating sequence] to produce time-ordered sequences.
//
// If you encode as RFC 3339 (the default),
// time values will scan back to a *time.Time unless your column type is TEXT.
// Otherwise, if scanned to a *any, time values may either become an
// int64, float64 or string, depending on the time format and declared type of the column.
// If you use DATE, TIME, DATETIME, or TIMESTAMP for your column type,
// "_timefmt" will be used to decode values.
//
// To scan values in custom formats, [sqlite3.TimeFormat.Scanner] may be helpful.
// To bind values in custom formats, [sqlite3.TimeFormat.Encode] them before binding.
//
// When using a custom time struct, you'll have to implement
// [database/sql/driver.Valuer] and [database/sql.Scanner].
@ -48,7 +71,7 @@
// The Scan method needs to take into account that the value it receives can be of differing types.
// It can already be a [time.Time], if the driver decoded the value according to "_timefmt" rules.
// Or it can be a: string, int64, float64, []byte, or nil,
// depending on the column type and what whoever wrote the value.
// depending on the column type and whoever wrote the value.
// [sqlite3.TimeFormat.Decode] may help.
//
// # Setting PRAGMAs
@ -358,13 +381,10 @@ func (c *conn) Commit() error {
}
func (c *conn) Rollback() error {
err := c.Conn.Exec(`ROLLBACK` + c.txReset)
if errors.Is(err, sqlite3.INTERRUPT) {
old := c.Conn.SetInterrupt(context.Background())
defer c.Conn.SetInterrupt(old)
err = c.Conn.Exec(`ROLLBACK` + c.txReset)
}
return err
// ROLLBACK even if interrupted.
old := c.Conn.SetInterrupt(context.Background())
defer c.Conn.SetInterrupt(old)
return c.Conn.Exec(`ROLLBACK` + c.txReset)
}
func (c *conn) Prepare(query string) (driver.Stmt, error) {
@ -598,6 +618,28 @@ const (
_TIME
)
func scanFromDecl(decl string) scantype {
// These types are only used before we have rows,
// and otherwise as type hints.
// The first few ensure STRICT tables are strictly typed.
// The other two are type hints for booleans and time.
switch decl {
case "INT", "INTEGER":
return _INT
case "REAL":
return _REAL
case "TEXT":
return _TEXT
case "BLOB":
return _BLOB
case "BOOLEAN":
return _BOOL
case "DATE", "TIME", "DATETIME", "TIMESTAMP":
return _TIME
}
return _ANY
}
var (
// Ensure these interfaces are implemented:
_ driver.RowsColumnTypeDatabaseTypeName = &rows{}
@ -622,6 +664,18 @@ func (r *rows) Columns() []string {
return r.names
}
func (r *rows) scanType(index int) scantype {
if r.scans == nil {
count := r.Stmt.ColumnCount()
scans := make([]scantype, count)
for i := range scans {
scans[i] = scanFromDecl(strings.ToUpper(r.Stmt.ColumnDeclType(i)))
}
r.scans = scans
}
return r.scans[index]
}
func (r *rows) loadColumnMetadata() {
if r.nulls == nil {
count := r.Stmt.ColumnCount()
@ -635,24 +689,7 @@ func (r *rows) loadColumnMetadata() {
r.Stmt.ColumnTableName(i),
col)
types[i] = strings.ToUpper(types[i])
// These types are only used before we have rows,
// and otherwise as type hints.
// The first few ensure STRICT tables are strictly typed.
// The other two are type hints for booleans and time.
switch types[i] {
case "INT", "INTEGER":
scans[i] = _INT
case "REAL":
scans[i] = _REAL
case "TEXT":
scans[i] = _TEXT
case "BLOB":
scans[i] = _BLOB
case "BOOLEAN":
scans[i] = _BOOL
case "DATE", "TIME", "DATETIME", "TIMESTAMP":
scans[i] = _TIME
}
scans[i] = scanFromDecl(types[i])
}
}
r.nulls = nulls
@ -661,27 +698,15 @@ func (r *rows) loadColumnMetadata() {
}
}
func (r *rows) declType(index int) string {
if r.types == nil {
count := r.Stmt.ColumnCount()
types := make([]string, count)
for i := range types {
types[i] = strings.ToUpper(r.Stmt.ColumnDeclType(i))
}
r.types = types
}
return r.types[index]
}
func (r *rows) ColumnTypeDatabaseTypeName(index int) string {
r.loadColumnMetadata()
decltype := r.types[index]
if len := len(decltype); len > 0 && decltype[len-1] == ')' {
if i := strings.LastIndexByte(decltype, '('); i >= 0 {
decltype = decltype[:i]
decl := r.types[index]
if len := len(decl); len > 0 && decl[len-1] == ')' {
if i := strings.LastIndexByte(decl, '('); i >= 0 {
decl = decl[:i]
}
}
return strings.TrimSpace(decltype)
return strings.TrimSpace(decl)
}
func (r *rows) ColumnTypeNullable(index int) (nullable, ok bool) {
@ -748,36 +773,49 @@ func (r *rows) Next(dest []driver.Value) error {
}
data := unsafe.Slice((*any)(unsafe.SliceData(dest)), len(dest))
err := r.Stmt.Columns(data...)
if err := r.Stmt.ColumnsRaw(data...); err != nil {
return err
}
for i := range dest {
if t, ok := r.decodeTime(i, dest[i]); ok {
dest[i] = t
}
}
return err
}
func (r *rows) decodeTime(i int, v any) (_ time.Time, ok bool) {
switch v := v.(type) {
case int64, float64:
// could be a time value
case string:
if r.tmWrite != "" && r.tmWrite != time.RFC3339 && r.tmWrite != time.RFC3339Nano {
scan := r.scanType(i)
switch v := dest[i].(type) {
case int64:
if scan == _BOOL {
switch v {
case 1:
dest[i] = true
case 0:
dest[i] = false
}
continue
}
case []byte:
if len(v) == cap(v) { // a BLOB
continue
}
if scan != _TEXT {
switch r.tmWrite {
case "", time.RFC3339, time.RFC3339Nano:
t, ok := maybeTime(v)
if ok {
dest[i] = t
continue
}
}
}
dest[i] = string(v)
case float64:
break
default:
continue
}
t, ok := maybeTime(v)
if ok {
return t, true
if scan == _TIME {
t, err := r.tmRead.Decode(dest[i])
if err == nil {
dest[i] = t
continue
}
}
default:
return
}
switch r.declType(i) {
case "DATE", "TIME", "DATETIME", "TIMESTAMP":
// could be a time value
default:
return
}
t, err := r.tmRead.Decode(v)
return t, err == nil
return nil
}

View file

@ -1,12 +1,15 @@
package driver
import "time"
import (
"bytes"
"time"
)
// Convert a string in [time.RFC3339Nano] format into a [time.Time]
// if it roundtrips back to the same string.
// This way times can be persisted to, and recovered from, the database,
// but if a string is needed, [database/sql] will recover the same string.
func maybeTime(text string) (_ time.Time, _ bool) {
func maybeTime(text []byte) (_ time.Time, _ bool) {
// Weed out (some) values that can't possibly be
// [time.RFC3339Nano] timestamps.
if len(text) < len("2006-01-02T15:04:05Z") {
@ -21,8 +24,8 @@ func maybeTime(text string) (_ time.Time, _ bool) {
// Slow path.
var buf [len(time.RFC3339Nano)]byte
date, err := time.Parse(time.RFC3339Nano, text)
if err == nil && text == string(date.AppendFormat(buf[:0], time.RFC3339Nano)) {
date, err := time.Parse(time.RFC3339Nano, string(text))
if err == nil && bytes.Equal(text, date.AppendFormat(buf[:0], time.RFC3339Nano)) {
return date, true
}
return

View file

@ -2,7 +2,6 @@ package sqlite3
import (
"errors"
"strconv"
"strings"
"github.com/ncruces/go-sqlite3/internal/util"
@ -12,7 +11,6 @@ import (
//
// https://sqlite.org/c3ref/errcode.html
type Error struct {
str string
msg string
sql string
code res_t
@ -29,19 +27,13 @@ func (e *Error) Code() ErrorCode {
//
// https://sqlite.org/rescode.html
func (e *Error) ExtendedCode() ExtendedErrorCode {
return ExtendedErrorCode(e.code)
return xErrorCode(e.code)
}
// Error implements the error interface.
func (e *Error) Error() string {
var b strings.Builder
b.WriteString("sqlite3: ")
if e.str != "" {
b.WriteString(e.str)
} else {
b.WriteString(strconv.Itoa(int(e.code)))
}
b.WriteString(util.ErrorCodeString(uint32(e.code)))
if e.msg != "" {
b.WriteString(": ")
@ -103,12 +95,12 @@ func (e ErrorCode) Error() string {
// Temporary returns true for [BUSY] errors.
func (e ErrorCode) Temporary() bool {
return e == BUSY
return e == BUSY || e == INTERRUPT
}
// ExtendedCode returns the extended error code for this error.
func (e ErrorCode) ExtendedCode() ExtendedErrorCode {
return ExtendedErrorCode(e)
return xErrorCode(e)
}
// Error implements the error interface.
@ -133,7 +125,7 @@ func (e ExtendedErrorCode) As(err any) bool {
// Temporary returns true for [BUSY] errors.
func (e ExtendedErrorCode) Temporary() bool {
return ErrorCode(e) == BUSY
return ErrorCode(e) == BUSY || ErrorCode(e) == INTERRUPT
}
// Timeout returns true for [BUSY_TIMEOUT] errors.

View file

@ -3,7 +3,9 @@ package sqlite3
import (
"context"
"io"
"iter"
"sync"
"sync/atomic"
"github.com/tetratelabs/wazero/api"
@ -45,7 +47,7 @@ func (c Conn) AnyCollationNeeded() error {
// CreateCollation defines a new collating sequence.
//
// https://sqlite.org/c3ref/create_collation.html
func (c *Conn) CreateCollation(name string, fn func(a, b []byte) int) error {
func (c *Conn) CreateCollation(name string, fn CollatingFunction) error {
var funcPtr ptr_t
defer c.arena.mark()()
namePtr := c.arena.string(name)
@ -57,6 +59,10 @@ func (c *Conn) CreateCollation(name string, fn func(a, b []byte) int) error {
return c.error(rc)
}
// Collating function is the type of a collation callback.
// Implementations must not retain a or b.
type CollatingFunction func(a, b []byte) int
// CreateFunction defines a new scalar SQL function.
//
// https://sqlite.org/c3ref/create_function.html
@ -77,34 +83,67 @@ func (c *Conn) CreateFunction(name string, nArg int, flag FunctionFlag, fn Scala
// Implementations must not retain arg.
type ScalarFunction func(ctx Context, arg ...Value)
// CreateWindowFunction defines a new aggregate or aggregate window SQL function.
// If fn returns a [WindowFunction], then an aggregate window function is created.
// If fn returns an [io.Closer], it will be called to free resources.
// CreateAggregateFunction defines a new aggregate SQL function.
//
// https://sqlite.org/c3ref/create_function.html
func (c *Conn) CreateWindowFunction(name string, nArg int, flag FunctionFlag, fn func() AggregateFunction) error {
func (c *Conn) CreateAggregateFunction(name string, nArg int, flag FunctionFlag, fn AggregateSeqFunction) error {
var funcPtr ptr_t
defer c.arena.mark()()
namePtr := c.arena.string(name)
call := "sqlite3_create_aggregate_function_go"
if fn != nil {
agg := fn()
if c, ok := agg.(io.Closer); ok {
if err := c.Close(); err != nil {
return err
funcPtr = util.AddHandle(c.ctx, AggregateConstructor(func() AggregateFunction {
var a aggregateFunc
coro := func(yieldCoro func(struct{}) bool) {
seq := func(yieldSeq func([]Value) bool) {
for yieldSeq(a.arg) {
if !yieldCoro(struct{}{}) {
break
}
}
}
fn(&a.ctx, seq)
}
}
if _, ok := agg.(WindowFunction); ok {
call = "sqlite3_create_window_function_go"
}
funcPtr = util.AddHandle(c.ctx, fn)
a.next, a.stop = iter.Pull(coro)
return &a
}))
}
rc := res_t(c.call(call,
rc := res_t(c.call("sqlite3_create_aggregate_function_go",
stk_t(c.handle), stk_t(namePtr), stk_t(nArg),
stk_t(flag), stk_t(funcPtr)))
return c.error(rc)
}
// AggregateSeqFunction is the type of an aggregate SQL function.
// Implementations must not retain the slices yielded by seq.
type AggregateSeqFunction func(ctx *Context, seq iter.Seq[[]Value])
// CreateWindowFunction defines a new aggregate or aggregate window SQL function.
// If fn returns a [WindowFunction], an aggregate window function is created.
// If fn returns an [io.Closer], it will be called to free resources.
//
// https://sqlite.org/c3ref/create_function.html
func (c *Conn) CreateWindowFunction(name string, nArg int, flag FunctionFlag, fn AggregateConstructor) error {
var funcPtr ptr_t
defer c.arena.mark()()
namePtr := c.arena.string(name)
if fn != nil {
funcPtr = util.AddHandle(c.ctx, AggregateConstructor(func() AggregateFunction {
agg := fn()
if win, ok := agg.(WindowFunction); ok {
return win
}
return windowFunc{agg, name}
}))
}
rc := res_t(c.call("sqlite3_create_window_function_go",
stk_t(c.handle), stk_t(namePtr), stk_t(nArg),
stk_t(flag), stk_t(funcPtr)))
return c.error(rc)
}
// AggregateConstructor is a an [AggregateFunction] constructor.
type AggregateConstructor func() AggregateFunction
// AggregateFunction is the interface an aggregate function should implement.
//
// https://sqlite.org/appfunc.html
@ -153,26 +192,24 @@ func collationCallback(ctx context.Context, mod api.Module, pArg, pDB ptr_t, eTe
}
func compareCallback(ctx context.Context, mod api.Module, pApp ptr_t, nKey1 int32, pKey1 ptr_t, nKey2 int32, pKey2 ptr_t) uint32 {
fn := util.GetHandle(ctx, pApp).(func(a, b []byte) int)
fn := util.GetHandle(ctx, pApp).(CollatingFunction)
return uint32(fn(util.View(mod, pKey1, int64(nKey1)), util.View(mod, pKey2, int64(nKey2))))
}
func funcCallback(ctx context.Context, mod api.Module, pCtx, pApp ptr_t, nArg int32, pArg ptr_t) {
args := getFuncArgs()
defer putFuncArgs(args)
db := ctx.Value(connKey{}).(*Conn)
args := callbackArgs(db, nArg, pArg)
defer returnArgs(args)
fn := util.GetHandle(db.ctx, pApp).(ScalarFunction)
callbackArgs(db, args[:nArg], pArg)
fn(Context{db, pCtx}, args[:nArg]...)
fn(Context{db, pCtx}, *args...)
}
func stepCallback(ctx context.Context, mod api.Module, pCtx, pAgg, pApp ptr_t, nArg int32, pArg ptr_t) {
args := getFuncArgs()
defer putFuncArgs(args)
db := ctx.Value(connKey{}).(*Conn)
callbackArgs(db, args[:nArg], pArg)
args := callbackArgs(db, nArg, pArg)
defer returnArgs(args)
fn, _ := callbackAggregate(db, pAgg, pApp)
fn.Step(Context{db, pCtx}, args[:nArg]...)
fn.Step(Context{db, pCtx}, *args...)
}
func valueCallback(ctx context.Context, mod api.Module, pCtx, pAgg, pApp ptr_t, final int32) {
@ -196,12 +233,11 @@ func valueCallback(ctx context.Context, mod api.Module, pCtx, pAgg, pApp ptr_t,
}
func inverseCallback(ctx context.Context, mod api.Module, pCtx, pAgg ptr_t, nArg int32, pArg ptr_t) {
args := getFuncArgs()
defer putFuncArgs(args)
db := ctx.Value(connKey{}).(*Conn)
callbackArgs(db, args[:nArg], pArg)
args := callbackArgs(db, nArg, pArg)
defer returnArgs(args)
fn := util.GetHandle(db.ctx, pAgg).(WindowFunction)
fn.Inverse(Context{db, pCtx}, args[:nArg]...)
fn.Inverse(Context{db, pCtx}, *args...)
}
func callbackAggregate(db *Conn, pAgg, pApp ptr_t) (AggregateFunction, ptr_t) {
@ -211,7 +247,7 @@ func callbackAggregate(db *Conn, pAgg, pApp ptr_t) (AggregateFunction, ptr_t) {
}
// We need to create the aggregate.
fn := util.GetHandle(db.ctx, pApp).(func() AggregateFunction)()
fn := util.GetHandle(db.ctx, pApp).(AggregateConstructor)()
if pAgg != 0 {
handle := util.AddHandle(db.ctx, fn)
util.Write32(db.mod, pAgg, handle)
@ -220,25 +256,64 @@ func callbackAggregate(db *Conn, pAgg, pApp ptr_t) (AggregateFunction, ptr_t) {
return fn, 0
}
func callbackArgs(db *Conn, arg []Value, pArg ptr_t) {
for i := range arg {
arg[i] = Value{
var (
valueArgsPool sync.Pool
valueArgsLen atomic.Int32
)
func callbackArgs(db *Conn, nArg int32, pArg ptr_t) *[]Value {
arg, ok := valueArgsPool.Get().(*[]Value)
if !ok || cap(*arg) < int(nArg) {
max := valueArgsLen.Or(nArg) | nArg
lst := make([]Value, max)
arg = &lst
}
lst := (*arg)[:nArg]
for i := range lst {
lst[i] = Value{
c: db,
handle: util.Read32[ptr_t](db.mod, pArg+ptr_t(i)*ptrlen),
}
}
*arg = lst
return arg
}
var funcArgsPool sync.Pool
func putFuncArgs(p *[_MAX_FUNCTION_ARG]Value) {
funcArgsPool.Put(p)
func returnArgs(p *[]Value) {
valueArgsPool.Put(p)
}
func getFuncArgs() *[_MAX_FUNCTION_ARG]Value {
if p := funcArgsPool.Get(); p == nil {
return new([_MAX_FUNCTION_ARG]Value)
} else {
return p.(*[_MAX_FUNCTION_ARG]Value)
type aggregateFunc struct {
next func() (struct{}, bool)
stop func()
ctx Context
arg []Value
}
func (a *aggregateFunc) Step(ctx Context, arg ...Value) {
a.ctx = ctx
a.arg = append(a.arg[:0], arg...)
if _, more := a.next(); !more {
a.stop()
}
}
func (a *aggregateFunc) Value(ctx Context) {
a.ctx = ctx
a.stop()
}
func (a *aggregateFunc) Close() error {
a.stop()
return nil
}
type windowFunc struct {
AggregateFunction
name string
}
func (w windowFunc) Inverse(ctx Context, arg ...Value) {
// Implementing inverse allows certain queries that don't really need it to succeed.
ctx.ResultError(util.ErrorString(w.name + ": may not be used as a window function"))
}

View file

@ -75,7 +75,7 @@ func ErrorCodeString(rc uint32) string {
return "sqlite3: unable to open database file"
case PROTOCOL:
return "sqlite3: locking protocol"
case FORMAT:
case EMPTY:
break
case SCHEMA:
return "sqlite3: database schema has changed"
@ -91,7 +91,7 @@ func ErrorCodeString(rc uint32) string {
break
case AUTH:
return "sqlite3: authorization denied"
case EMPTY:
case FORMAT:
break
case RANGE:
return "sqlite3: column index out of range"

View file

@ -135,11 +135,10 @@ func ReadString(mod api.Module, ptr Ptr_t, maxlen int64) string {
panic(RangeErr)
}
}
if i := bytes.IndexByte(buf, 0); i < 0 {
panic(NoNulErr)
} else {
if i := bytes.IndexByte(buf, 0); i >= 0 {
return string(buf[:i])
}
panic(NoNulErr)
}
func WriteBytes(mod api.Module, ptr Ptr_t, b []byte) {

View file

@ -120,33 +120,33 @@ func (sqlt *sqlite) error(rc res_t, handle ptr_t, sql ...string) error {
return nil
}
err := Error{code: rc}
if err.Code() == NOMEM || err.ExtendedCode() == IOERR_NOMEM {
if ErrorCode(rc) == NOMEM || xErrorCode(rc) == IOERR_NOMEM {
panic(util.OOMErr)
}
if ptr := ptr_t(sqlt.call("sqlite3_errstr", stk_t(rc))); ptr != 0 {
err.str = util.ReadString(sqlt.mod, ptr, _MAX_NAME)
}
if handle != 0 {
var msg, query string
if ptr := ptr_t(sqlt.call("sqlite3_errmsg", stk_t(handle))); ptr != 0 {
err.msg = util.ReadString(sqlt.mod, ptr, _MAX_LENGTH)
msg = util.ReadString(sqlt.mod, ptr, _MAX_LENGTH)
switch {
case msg == "not an error":
msg = ""
case msg == util.ErrorCodeString(uint32(rc))[len("sqlite3: "):]:
msg = ""
}
}
if len(sql) != 0 {
if i := int32(sqlt.call("sqlite3_error_offset", stk_t(handle))); i != -1 {
err.sql = sql[0][i:]
query = sql[0][i:]
}
}
}
switch err.msg {
case err.str, "not an error":
err.msg = ""
if msg != "" || query != "" {
return &Error{code: rc, msg: msg, sql: query}
}
}
return &err
return xErrorCode(rc)
}
func (sqlt *sqlite) getfn(name string) api.Function {
@ -212,14 +212,10 @@ func (sqlt *sqlite) realloc(ptr ptr_t, size int64) ptr_t {
}
func (sqlt *sqlite) newBytes(b []byte) ptr_t {
if (*[0]byte)(b) == nil {
if len(b) == 0 {
return 0
}
size := len(b)
if size == 0 {
size = 1
}
ptr := sqlt.new(int64(size))
ptr := sqlt.new(int64(len(b)))
util.WriteBytes(sqlt.mod, ptr, b)
return ptr
}
@ -288,7 +284,7 @@ func (a *arena) new(size int64) ptr_t {
}
func (a *arena) bytes(b []byte) ptr_t {
if (*[0]byte)(b) == nil {
if len(b) == 0 {
return 0
}
ptr := a.new(int64(len(b)))

View file

@ -106,7 +106,14 @@ func (s *Stmt) Busy() bool {
//
// https://sqlite.org/c3ref/step.html
func (s *Stmt) Step() bool {
s.c.checkInterrupt(s.c.handle)
if s.c.interrupt.Err() != nil {
s.err = INTERRUPT
return false
}
return s.step()
}
func (s *Stmt) step() bool {
rc := res_t(s.c.call("sqlite3_step", stk_t(s.handle)))
switch rc {
case _ROW:
@ -131,7 +138,11 @@ func (s *Stmt) Err() error {
// Exec is a convenience function that repeatedly calls [Stmt.Step] until it returns false,
// then calls [Stmt.Reset] to reset the statement and get any error that occurred.
func (s *Stmt) Exec() error {
for s.Step() {
if s.c.interrupt.Err() != nil {
return INTERRUPT
}
// TODO: implement this in C.
for s.step() {
}
return s.Reset()
}
@ -254,13 +265,15 @@ func (s *Stmt) BindText(param int, value string) error {
// BindRawText binds a []byte to the prepared statement as text.
// The leftmost SQL parameter has an index of 1.
// Binding a nil slice is the same as calling [Stmt.BindNull].
//
// https://sqlite.org/c3ref/bind_blob.html
func (s *Stmt) BindRawText(param int, value []byte) error {
if len(value) > _MAX_LENGTH {
return TOOBIG
}
if len(value) == 0 {
return s.BindText(param, "")
}
ptr := s.c.newBytes(value)
rc := res_t(s.c.call("sqlite3_bind_text_go",
stk_t(s.handle), stk_t(param),
@ -270,13 +283,15 @@ func (s *Stmt) BindRawText(param int, value []byte) error {
// BindBlob binds a []byte to the prepared statement.
// The leftmost SQL parameter has an index of 1.
// Binding a nil slice is the same as calling [Stmt.BindNull].
//
// https://sqlite.org/c3ref/bind_blob.html
func (s *Stmt) BindBlob(param int, value []byte) error {
if len(value) > _MAX_LENGTH {
return TOOBIG
}
if len(value) == 0 {
return s.BindZeroBlob(param, 0)
}
ptr := s.c.newBytes(value)
rc := res_t(s.c.call("sqlite3_bind_blob_go",
stk_t(s.handle), stk_t(param),
@ -560,7 +575,7 @@ func (s *Stmt) ColumnBlob(col int, buf []byte) []byte {
func (s *Stmt) ColumnRawText(col int) []byte {
ptr := ptr_t(s.c.call("sqlite3_column_text",
stk_t(s.handle), stk_t(col)))
return s.columnRawBytes(col, ptr)
return s.columnRawBytes(col, ptr, 1)
}
// ColumnRawBlob returns the value of the result column as a []byte.
@ -572,10 +587,10 @@ func (s *Stmt) ColumnRawText(col int) []byte {
func (s *Stmt) ColumnRawBlob(col int) []byte {
ptr := ptr_t(s.c.call("sqlite3_column_blob",
stk_t(s.handle), stk_t(col)))
return s.columnRawBytes(col, ptr)
return s.columnRawBytes(col, ptr, 0)
}
func (s *Stmt) columnRawBytes(col int, ptr ptr_t) []byte {
func (s *Stmt) columnRawBytes(col int, ptr ptr_t, nul int32) []byte {
if ptr == 0 {
rc := res_t(s.c.call("sqlite3_errcode", stk_t(s.c.handle)))
if rc != _ROW && rc != _DONE {
@ -586,7 +601,7 @@ func (s *Stmt) columnRawBytes(col int, ptr ptr_t) []byte {
n := int32(s.c.call("sqlite3_column_bytes",
stk_t(s.handle), stk_t(col)))
return util.View(s.c.mod, ptr, int64(n))
return util.View(s.c.mod, ptr, int64(n+nul))[:n]
}
// ColumnJSON parses the JSON-encoded value of the result column
@ -633,22 +648,12 @@ func (s *Stmt) ColumnValue(col int) Value {
// [INTEGER] columns will be retrieved as int64 values,
// [FLOAT] as float64, [NULL] as nil,
// [TEXT] as string, and [BLOB] as []byte.
// Any []byte are owned by SQLite and may be invalidated by
// subsequent calls to [Stmt] methods.
func (s *Stmt) Columns(dest ...any) error {
defer s.c.arena.mark()()
count := int64(len(dest))
typePtr := s.c.arena.new(count)
dataPtr := s.c.arena.new(count * 8)
rc := res_t(s.c.call("sqlite3_columns_go",
stk_t(s.handle), stk_t(count), stk_t(typePtr), stk_t(dataPtr)))
if err := s.c.error(rc); err != nil {
types, ptr, err := s.columns(int64(len(dest)))
if err != nil {
return err
}
types := util.View(s.c.mod, typePtr, count)
// Avoid bounds checks on types below.
if len(types) != len(dest) {
panic(util.AssertErr())
@ -657,26 +662,95 @@ func (s *Stmt) Columns(dest ...any) error {
for i := range dest {
switch types[i] {
case byte(INTEGER):
dest[i] = util.Read64[int64](s.c.mod, dataPtr)
dest[i] = util.Read64[int64](s.c.mod, ptr)
case byte(FLOAT):
dest[i] = util.ReadFloat64(s.c.mod, dataPtr)
dest[i] = util.ReadFloat64(s.c.mod, ptr)
case byte(NULL):
dest[i] = nil
default:
ptr := util.Read32[ptr_t](s.c.mod, dataPtr+0)
if ptr == 0 {
dest[i] = []byte{}
continue
}
len := util.Read32[int32](s.c.mod, dataPtr+4)
buf := util.View(s.c.mod, ptr, int64(len))
if types[i] == byte(TEXT) {
case byte(TEXT):
len := util.Read32[int32](s.c.mod, ptr+4)
if len != 0 {
ptr := util.Read32[ptr_t](s.c.mod, ptr)
buf := util.View(s.c.mod, ptr, int64(len))
dest[i] = string(buf)
} else {
dest[i] = buf
dest[i] = ""
}
case byte(BLOB):
len := util.Read32[int32](s.c.mod, ptr+4)
if len != 0 {
ptr := util.Read32[ptr_t](s.c.mod, ptr)
buf := util.View(s.c.mod, ptr, int64(len))
tmp, _ := dest[i].([]byte)
dest[i] = append(tmp[:0], buf...)
} else {
dest[i], _ = dest[i].([]byte)
}
}
dataPtr += 8
ptr += 8
}
return nil
}
// ColumnsRaw populates result columns into the provided slice.
// The slice must have [Stmt.ColumnCount] length.
//
// [INTEGER] columns will be retrieved as int64 values,
// [FLOAT] as float64, [NULL] as nil,
// [TEXT] and [BLOB] as []byte.
// Any []byte are owned by SQLite and may be invalidated by
// subsequent calls to [Stmt] methods.
func (s *Stmt) ColumnsRaw(dest ...any) error {
types, ptr, err := s.columns(int64(len(dest)))
if err != nil {
return err
}
// Avoid bounds checks on types below.
if len(types) != len(dest) {
panic(util.AssertErr())
}
for i := range dest {
switch types[i] {
case byte(INTEGER):
dest[i] = util.Read64[int64](s.c.mod, ptr)
case byte(FLOAT):
dest[i] = util.ReadFloat64(s.c.mod, ptr)
case byte(NULL):
dest[i] = nil
default:
len := util.Read32[int32](s.c.mod, ptr+4)
if len == 0 && types[i] == byte(BLOB) {
dest[i] = []byte{}
} else {
cap := len
if types[i] == byte(TEXT) {
cap++
}
ptr := util.Read32[ptr_t](s.c.mod, ptr)
buf := util.View(s.c.mod, ptr, int64(cap))[:len]
dest[i] = buf
}
}
ptr += 8
}
return nil
}
func (s *Stmt) columns(count int64) ([]byte, ptr_t, error) {
defer s.c.arena.mark()()
typePtr := s.c.arena.new(count)
dataPtr := s.c.arena.new(count * 8)
rc := res_t(s.c.call("sqlite3_columns_go",
stk_t(s.handle), stk_t(count), stk_t(typePtr), stk_t(dataPtr)))
if rc == res_t(MISUSE) {
return nil, 0, MISUSE
}
if err := s.c.error(rc); err != nil {
return nil, 0, err
}
return util.View(s.c.mod, typePtr, count), dataPtr, nil
}

View file

@ -2,7 +2,6 @@ package sqlite3
import (
"context"
"errors"
"math/rand"
"runtime"
"strconv"
@ -21,11 +20,13 @@ type Txn struct {
}
// Begin starts a deferred transaction.
// It panics if a transaction is in-progress.
// For nested transactions, use [Conn.Savepoint].
//
// https://sqlite.org/lang_transaction.html
func (c *Conn) Begin() Txn {
// BEGIN even if interrupted.
err := c.txnExecInterrupted(`BEGIN DEFERRED`)
err := c.exec(`BEGIN DEFERRED`)
if err != nil {
panic(err)
}
@ -120,7 +121,8 @@ func (tx Txn) Commit() error {
//
// https://sqlite.org/lang_transaction.html
func (tx Txn) Rollback() error {
return tx.c.txnExecInterrupted(`ROLLBACK`)
// ROLLBACK even if interrupted.
return tx.c.exec(`ROLLBACK`)
}
// Savepoint is a marker within a transaction
@ -143,7 +145,7 @@ func (c *Conn) Savepoint() Savepoint {
// Names can be reused, but this makes catching bugs more likely.
name = QuoteIdentifier(name + "_" + strconv.Itoa(int(rand.Int31())))
err := c.txnExecInterrupted(`SAVEPOINT ` + name)
err := c.exec(`SAVEPOINT ` + name)
if err != nil {
panic(err)
}
@ -199,7 +201,7 @@ func (s Savepoint) Release(errp *error) {
return
}
// ROLLBACK and RELEASE even if interrupted.
err := s.c.txnExecInterrupted(`ROLLBACK TO ` + s.name + `; RELEASE ` + s.name)
err := s.c.exec(`ROLLBACK TO ` + s.name + `; RELEASE ` + s.name)
if err != nil {
panic(err)
}
@ -212,17 +214,7 @@ func (s Savepoint) Release(errp *error) {
// https://sqlite.org/lang_transaction.html
func (s Savepoint) Rollback() error {
// ROLLBACK even if interrupted.
return s.c.txnExecInterrupted(`ROLLBACK TO ` + s.name)
}
func (c *Conn) txnExecInterrupted(sql string) error {
err := c.Exec(sql)
if errors.Is(err, INTERRUPT) {
old := c.SetInterrupt(context.Background())
defer c.SetInterrupt(old)
err = c.Exec(sql)
}
return err
return s.c.exec(`ROLLBACK TO ` + s.name)
}
// TxnState determines the transaction state of a database.

View file

@ -1,16 +0,0 @@
//go:build !windows
package osutil
import (
"io/fs"
"os"
)
// OpenFile behaves the same as [os.OpenFile],
// except on Windows it sets [syscall.FILE_SHARE_DELETE].
//
// See: https://go.dev/issue/32088#issuecomment-502850674
func OpenFile(name string, flag int, perm fs.FileMode) (*os.File, error) {
return os.OpenFile(name, flag, perm)
}

View file

@ -1,115 +0,0 @@
package osutil
import (
"io/fs"
"os"
. "syscall"
"unsafe"
)
// OpenFile behaves the same as [os.OpenFile],
// except on Windows it sets [syscall.FILE_SHARE_DELETE].
//
// See: https://go.dev/issue/32088#issuecomment-502850674
func OpenFile(name string, flag int, perm fs.FileMode) (*os.File, error) {
if name == "" {
return nil, &os.PathError{Op: "open", Path: name, Err: ENOENT}
}
r, e := syscallOpen(name, flag|O_CLOEXEC, uint32(perm.Perm()))
if e != nil {
return nil, &os.PathError{Op: "open", Path: name, Err: e}
}
return os.NewFile(uintptr(r), name), nil
}
// syscallOpen is a copy of [syscall.Open]
// that uses [syscall.FILE_SHARE_DELETE].
//
// https://go.dev/src/syscall/syscall_windows.go
func syscallOpen(path string, mode int, perm uint32) (fd Handle, err error) {
if len(path) == 0 {
return InvalidHandle, ERROR_FILE_NOT_FOUND
}
pathp, err := UTF16PtrFromString(path)
if err != nil {
return InvalidHandle, err
}
var access uint32
switch mode & (O_RDONLY | O_WRONLY | O_RDWR) {
case O_RDONLY:
access = GENERIC_READ
case O_WRONLY:
access = GENERIC_WRITE
case O_RDWR:
access = GENERIC_READ | GENERIC_WRITE
}
if mode&O_CREAT != 0 {
access |= GENERIC_WRITE
}
if mode&O_APPEND != 0 {
access &^= GENERIC_WRITE
access |= FILE_APPEND_DATA
}
sharemode := uint32(FILE_SHARE_READ | FILE_SHARE_WRITE | FILE_SHARE_DELETE)
var sa *SecurityAttributes
if mode&O_CLOEXEC == 0 {
sa = makeInheritSa()
}
var createmode uint32
switch {
case mode&(O_CREAT|O_EXCL) == (O_CREAT | O_EXCL):
createmode = CREATE_NEW
case mode&(O_CREAT|O_TRUNC) == (O_CREAT | O_TRUNC):
createmode = CREATE_ALWAYS
case mode&O_CREAT == O_CREAT:
createmode = OPEN_ALWAYS
case mode&O_TRUNC == O_TRUNC:
createmode = TRUNCATE_EXISTING
default:
createmode = OPEN_EXISTING
}
var attrs uint32 = FILE_ATTRIBUTE_NORMAL
if perm&S_IWRITE == 0 {
attrs = FILE_ATTRIBUTE_READONLY
if createmode == CREATE_ALWAYS {
const _ERROR_BAD_NETPATH = Errno(53)
// We have been asked to create a read-only file.
// If the file already exists, the semantics of
// the Unix open system call is to preserve the
// existing permissions. If we pass CREATE_ALWAYS
// and FILE_ATTRIBUTE_READONLY to CreateFile,
// and the file already exists, CreateFile will
// change the file permissions.
// Avoid that to preserve the Unix semantics.
h, e := CreateFile(pathp, access, sharemode, sa, TRUNCATE_EXISTING, FILE_ATTRIBUTE_NORMAL, 0)
switch e {
case ERROR_FILE_NOT_FOUND, _ERROR_BAD_NETPATH, ERROR_PATH_NOT_FOUND:
// File does not exist. These are the same
// errors as Errno.Is checks for ErrNotExist.
// Carry on to create the file.
default:
// Success or some different error.
return h, e
}
}
}
if createmode == OPEN_EXISTING && access == GENERIC_READ {
// Necessary for opening directory handles.
attrs |= FILE_FLAG_BACKUP_SEMANTICS
}
if mode&O_SYNC != 0 {
const _FILE_FLAG_WRITE_THROUGH = 0x80000000
attrs |= _FILE_FLAG_WRITE_THROUGH
}
if mode&O_NONBLOCK != 0 {
attrs |= FILE_FLAG_OVERLAPPED
}
return CreateFile(pathp, access, sharemode, sa, createmode, attrs, 0)
}
func makeInheritSa() *SecurityAttributes {
var sa SecurityAttributes
sa.Length = uint32(unsafe.Sizeof(sa))
sa.InheritHandle = 1
return &sa
}

View file

@ -1,33 +0,0 @@
package osutil
import (
"io/fs"
"os"
)
// FS implements [fs.FS], [fs.StatFS], and [fs.ReadFileFS]
// using package [os].
//
// This filesystem does not respect [fs.ValidPath] rules,
// and fails [testing/fstest.TestFS]!
//
// Still, it can be a useful tool to unify implementations
// that can access either the [os] filesystem or an [fs.FS].
// It's OK to use this to open files, but you should avoid
// opening directories, resolving paths, or walking the file system.
type FS struct{}
// Open implements [fs.FS].
func (FS) Open(name string) (fs.File, error) {
return OpenFile(name, os.O_RDONLY, 0)
}
// ReadFileFS implements [fs.StatFS].
func (FS) Stat(name string) (fs.FileInfo, error) {
return os.Stat(name)
}
// ReadFile implements [fs.ReadFileFS].
func (FS) ReadFile(name string) ([]byte, error) {
return os.ReadFile(name)
}

View file

@ -1,2 +0,0 @@
// Package osutil implements operating system utilities.
package osutil

View file

@ -5,5 +5,5 @@ package sql3util
//
// https://sqlite.org/fileformat.html#pages
func ValidPageSize(s int) bool {
return 512 <= s && s <= 65536 && s&(s-1) == 0
return s&(s-1) == 0 && 512 <= s && s <= 65536
}

View file

@ -139,7 +139,7 @@ func (v Value) Blob(buf []byte) []byte {
// https://sqlite.org/c3ref/value_blob.html
func (v Value) RawText() []byte {
ptr := ptr_t(v.c.call("sqlite3_value_text", v.protected()))
return v.rawBytes(ptr)
return v.rawBytes(ptr, 1)
}
// RawBlob returns the value as a []byte.
@ -149,16 +149,16 @@ func (v Value) RawText() []byte {
// https://sqlite.org/c3ref/value_blob.html
func (v Value) RawBlob() []byte {
ptr := ptr_t(v.c.call("sqlite3_value_blob", v.protected()))
return v.rawBytes(ptr)
return v.rawBytes(ptr, 0)
}
func (v Value) rawBytes(ptr ptr_t) []byte {
func (v Value) rawBytes(ptr ptr_t, nul int32) []byte {
if ptr == 0 {
return nil
}
n := int32(v.c.call("sqlite3_value_bytes", v.protected()))
return util.View(v.c.mod, ptr, int64(n))
return util.View(v.c.mod, ptr, int64(n+nul))[:n]
}
// Pointer gets the pointer associated with this value,

View file

@ -6,22 +6,30 @@ It replaces the default SQLite VFS with a **pure Go** implementation,
and exposes [interfaces](https://pkg.go.dev/github.com/ncruces/go-sqlite3/vfs#VFS)
that should allow you to implement your own [custom VFSes](#custom-vfses).
Since it is a from scratch reimplementation,
there are naturally some ways it deviates from the original.
See the [support matrix](https://github.com/ncruces/go-sqlite3/wiki/Support-matrix)
for the list of supported OS and CPU architectures.
The main differences are [file locking](#file-locking) and [WAL mode](#write-ahead-logging) support.
Since this is a from scratch reimplementation,
there are naturally some ways it deviates from the original.
It's also not as battle tested as the original.
The main differences to be aware of are
[file locking](#file-locking) and
[WAL mode](#write-ahead-logging) support.
### File Locking
POSIX advisory locks, which SQLite uses on Unix, are
[broken by design](https://github.com/sqlite/sqlite/blob/b74eb0/src/os_unix.c#L1073-L1161).
POSIX advisory locks,
which SQLite uses on [Unix](https://github.com/sqlite/sqlite/blob/5d60f4/src/os_unix.c#L13-L14),
are [broken by design](https://github.com/sqlite/sqlite/blob/5d60f4/src/os_unix.c#L1074-L1162).
Instead, on Linux and macOS, this package uses
[OFD locks](https://www.gnu.org/software/libc/manual/html_node/Open-File-Description-Locks.html)
to synchronize access to database files.
This package can also use
[BSD locks](https://man.freebsd.org/cgi/man.cgi?query=flock&sektion=2),
albeit with reduced concurrency (`BEGIN IMMEDIATE` behaves like `BEGIN EXCLUSIVE`).
albeit with reduced concurrency (`BEGIN IMMEDIATE` behaves like `BEGIN EXCLUSIVE`,
[docs](https://sqlite.org/lang_transaction.html#immediate)).
BSD locks are the default on BSD and illumos,
but you can opt into them with the `sqlite3_flock` build tag.
@ -44,11 +52,11 @@ to check if your build supports file locking.
### Write-Ahead Logging
On Unix, this package may use `mmap` to implement
On Unix, this package uses `mmap` to implement
[shared-memory for the WAL-index](https://sqlite.org/wal.html#implementation_of_shared_memory_for_the_wal_index),
like SQLite.
On Windows, this package may use `MapViewOfFile`, like SQLite.
On Windows, this package uses `MapViewOfFile`, like SQLite.
You can also opt into a cross-platform, in-process, memory sharing implementation
with the `sqlite3_dotlk` build tag.
@ -63,6 +71,11 @@ you must disable connection pooling by calling
You can use [`vfs.SupportsSharedMemory`](https://pkg.go.dev/github.com/ncruces/go-sqlite3/vfs#SupportsSharedMemory)
to check if your build supports shared memory.
### Blocking Locks
On Windows and macOS, this package implements
[Wal-mode blocking locks](https://sqlite.org/src/doc/tip/doc/wal-lock.md).
### Batch-Atomic Write
On Linux, this package may support
@ -94,8 +107,10 @@ The VFS can be customized with a few build tags:
> [`unix-flock` VFS](https://sqlite.org/compile.html#enable_locking_style);
> `sqlite3_dotlk` builds are compatible with the
> [`unix-dotfile` VFS](https://sqlite.org/compile.html#enable_locking_style).
> If incompatible file locking is used, accessing databases concurrently with
> _other_ SQLite libraries will eventually corrupt data.
> [!CAUTION]
> Concurrently accessing databases using incompatible VFSes
> will eventually corrupt data.
### Custom VFSes

View file

@ -49,9 +49,7 @@ func (c cksmFile) ReadAt(p []byte, off int64) (n int, err error) {
n, err = c.File.ReadAt(p, off)
p = p[:n]
// SQLite is reading the header of a database file.
if c.isDB && off == 0 && len(p) >= 100 &&
bytes.HasPrefix(p, []byte("SQLite format 3\000")) {
if isHeader(c.isDB, p, off) {
c.init((*[100]byte)(p))
}
@ -67,9 +65,7 @@ func (c cksmFile) ReadAt(p []byte, off int64) (n int, err error) {
}
func (c cksmFile) WriteAt(p []byte, off int64) (n int, err error) {
// SQLite is writing the first page of a database file.
if c.isDB && off == 0 && len(p) >= 100 &&
bytes.HasPrefix(p, []byte("SQLite format 3\000")) {
if isHeader(c.isDB, p, off) {
c.init((*[100]byte)(p))
}
@ -116,9 +112,11 @@ func (c cksmFile) fileControl(ctx context.Context, mod api.Module, op _FcntlOpco
c.inCkpt = true
case _FCNTL_CKPT_DONE:
c.inCkpt = false
}
if rc := vfsFileControlImpl(ctx, mod, c, op, pArg); rc != _NOTFOUND {
return rc
case _FCNTL_PRAGMA:
rc := vfsFileControlImpl(ctx, mod, c, op, pArg)
if rc != _NOTFOUND {
return rc
}
}
return vfsFileControlImpl(ctx, mod, c.File, op, pArg)
}
@ -135,6 +133,14 @@ func (f *cksmFlags) init(header *[100]byte) {
}
}
func isHeader(isDB bool, p []byte, off int64) bool {
check := sql3util.ValidPageSize(len(p))
if isDB {
check = off == 0 && len(p) >= 100
}
return check && bytes.HasPrefix(p, []byte("SQLite format 3\000"))
}
func cksmCompute(a []byte) (cksm [8]byte) {
var s1, s2 uint32
for len(a) >= 8 {

View file

@ -6,9 +6,8 @@ import (
"io/fs"
"os"
"path/filepath"
"runtime"
"syscall"
"github.com/ncruces/go-sqlite3/util/osutil"
)
type vfsOS struct{}
@ -40,7 +39,7 @@ func (vfsOS) Delete(path string, syncDir bool) error {
if err != nil {
return err
}
if canSyncDirs && syncDir {
if isUnix && syncDir {
f, err := os.Open(filepath.Dir(path))
if err != nil {
return _OK
@ -96,7 +95,7 @@ func (vfsOS) OpenFilename(name *Filename, flags OpenFlag) (File, OpenFlag, error
if name == nil {
f, err = os.CreateTemp(os.Getenv("SQLITE_TMPDIR"), "*.db")
} else {
f, err = osutil.OpenFile(name.String(), oflags, 0666)
f, err = os.OpenFile(name.String(), oflags, 0666)
}
if err != nil {
if name == nil {
@ -118,15 +117,17 @@ func (vfsOS) OpenFilename(name *Filename, flags OpenFlag) (File, OpenFlag, error
return nil, flags, _IOERR_FSTAT
}
}
if flags&OPEN_DELETEONCLOSE != 0 {
if isUnix && flags&OPEN_DELETEONCLOSE != 0 {
os.Remove(f.Name())
}
file := vfsFile{
File: f,
psow: true,
atomic: osBatchAtomic(f),
readOnly: flags&OPEN_READONLY != 0,
syncDir: canSyncDirs && isCreate && isJournl,
syncDir: isUnix && isCreate && isJournl,
delete: !isUnix && flags&OPEN_DELETEONCLOSE != 0,
shm: NewSharedMemory(name.String()+"-shm", flags),
}
return &file, flags, nil
@ -139,6 +140,8 @@ type vfsFile struct {
readOnly bool
keepWAL bool
syncDir bool
atomic bool
delete bool
psow bool
}
@ -152,6 +155,9 @@ var (
)
func (f *vfsFile) Close() error {
if f.delete {
defer os.Remove(f.Name())
}
if f.shm != nil {
f.shm.Close()
}
@ -175,7 +181,7 @@ func (f *vfsFile) Sync(flags SyncFlag) error {
if err != nil {
return err
}
if canSyncDirs && f.syncDir {
if isUnix && f.syncDir {
f.syncDir = false
d, err := os.Open(filepath.Dir(f.File.Name()))
if err != nil {
@ -200,12 +206,15 @@ func (f *vfsFile) SectorSize() int {
func (f *vfsFile) DeviceCharacteristics() DeviceCharacteristic {
ret := IOCAP_SUBPAGE_READ
if osBatchAtomic(f.File) {
if f.atomic {
ret |= IOCAP_BATCH_ATOMIC
}
if f.psow {
ret |= IOCAP_POWERSAFE_OVERWRITE
}
if runtime.GOOS == "windows" {
ret |= IOCAP_UNDELETABLE_WHEN_OPEN
}
return ret
}
@ -214,6 +223,9 @@ func (f *vfsFile) SizeHint(size int64) error {
}
func (f *vfsFile) HasMoved() (bool, error) {
if runtime.GOOS == "windows" {
return false, nil
}
fi, err := f.Stat()
if err != nil {
return false, err

View file

@ -50,11 +50,15 @@ func osDowngradeLock(file *os.File, _ LockLevel) _ErrorCode {
}
func osReleaseLock(file *os.File, _ LockLevel) _ErrorCode {
err := unix.Flock(int(file.Fd()), unix.LOCK_UN)
if err != nil {
return _IOERR_UNLOCK
for {
err := unix.Flock(int(file.Fd()), unix.LOCK_UN)
if err == nil {
return _OK
}
if err != unix.EINTR {
return _IOERR_UNLOCK
}
}
return _OK
}
func osCheckReservedLock(file *os.File) (bool, _ErrorCode) {
@ -89,13 +93,18 @@ func osLock(file *os.File, typ int16, start, len int64, def _ErrorCode) _ErrorCo
}
func osUnlock(file *os.File, start, len int64) _ErrorCode {
err := unix.FcntlFlock(file.Fd(), unix.F_SETLK, &unix.Flock_t{
lock := unix.Flock_t{
Type: unix.F_UNLCK,
Start: start,
Len: len,
})
if err != nil {
return _IOERR_UNLOCK
}
return _OK
for {
err := unix.FcntlFlock(file.Fd(), unix.F_SETLK, &lock)
if err == nil {
return _OK
}
if err != unix.EINTR {
return _IOERR_UNLOCK
}
}
}

View file

@ -27,7 +27,12 @@ func osSync(file *os.File, fullsync, _ /*dataonly*/ bool) error {
if fullsync {
return file.Sync()
}
return unix.Fsync(int(file.Fd()))
for {
err := unix.Fsync(int(file.Fd()))
if err != unix.EINTR {
return err
}
}
}
func osAllocate(file *os.File, size int64) error {
@ -85,13 +90,18 @@ func osLock(file *os.File, typ int16, start, len int64, timeout time.Duration, d
}
func osUnlock(file *os.File, start, len int64) _ErrorCode {
err := unix.FcntlFlock(file.Fd(), _F_OFD_SETLK, &unix.Flock_t{
lock := unix.Flock_t{
Type: unix.F_UNLCK,
Start: start,
Len: len,
})
if err != nil {
return _IOERR_UNLOCK
}
return _OK
for {
err := unix.FcntlFlock(file.Fd(), _F_OFD_SETLK, &lock)
if err == nil {
return _OK
}
if err != unix.EINTR {
return _IOERR_UNLOCK
}
}
}

View file

@ -3,6 +3,7 @@
package vfs
import (
"io"
"os"
"time"
@ -11,14 +12,36 @@ import (
func osSync(file *os.File, _ /*fullsync*/, _ /*dataonly*/ bool) error {
// SQLite trusts Linux's fdatasync for all fsync's.
return unix.Fdatasync(int(file.Fd()))
for {
err := unix.Fdatasync(int(file.Fd()))
if err != unix.EINTR {
return err
}
}
}
func osAllocate(file *os.File, size int64) error {
if size == 0 {
return nil
}
return unix.Fallocate(int(file.Fd()), 0, 0, size)
for {
err := unix.Fallocate(int(file.Fd()), 0, 0, size)
if err == unix.EOPNOTSUPP {
break
}
if err != unix.EINTR {
return err
}
}
off, err := file.Seek(0, io.SeekEnd)
if err != nil {
return err
}
if size <= off {
return nil
}
return file.Truncate(size)
}
func osReadLock(file *os.File, start, len int64, timeout time.Duration) _ErrorCode {
@ -37,22 +60,27 @@ func osLock(file *os.File, typ int16, start, len int64, timeout time.Duration, d
}
var err error
switch {
case timeout < 0:
err = unix.FcntlFlock(file.Fd(), unix.F_OFD_SETLKW, &lock)
default:
err = unix.FcntlFlock(file.Fd(), unix.F_OFD_SETLK, &lock)
case timeout < 0:
err = unix.FcntlFlock(file.Fd(), unix.F_OFD_SETLKW, &lock)
}
return osLockErrorCode(err, def)
}
func osUnlock(file *os.File, start, len int64) _ErrorCode {
err := unix.FcntlFlock(file.Fd(), unix.F_OFD_SETLK, &unix.Flock_t{
lock := unix.Flock_t{
Type: unix.F_UNLCK,
Start: start,
Len: len,
})
if err != nil {
return _IOERR_UNLOCK
}
return _OK
for {
err := unix.FcntlFlock(file.Fd(), unix.F_OFD_SETLK, &lock)
if err == nil {
return _OK
}
if err != unix.EINTR {
return _IOERR_UNLOCK
}
}
}

View file

@ -8,8 +8,8 @@ import (
)
const (
isUnix = false
_O_NOFOLLOW = 0
canSyncDirs = false
)
func osAccess(path string, flags AccessFlag) error {

View file

@ -10,8 +10,8 @@ import (
)
const (
isUnix = true
_O_NOFOLLOW = unix.O_NOFOLLOW
canSyncDirs = true
)
func osAccess(path string, flags AccessFlag) error {
@ -65,10 +65,15 @@ func osTestLock(file *os.File, start, len int64) (int16, _ErrorCode) {
Start: start,
Len: len,
}
if unix.FcntlFlock(file.Fd(), unix.F_GETLK, &lock) != nil {
return 0, _IOERR_CHECKRESERVEDLOCK
for {
err := unix.FcntlFlock(file.Fd(), unix.F_GETLK, &lock)
if err == nil {
return lock.Type, _OK
}
if err != unix.EINTR {
return 0, _IOERR_CHECKRESERVEDLOCK
}
}
return lock.Type, _OK
}
func osLockErrorCode(err error, def _ErrorCode) _ErrorCode {

View file

@ -135,12 +135,10 @@ func osWriteLock(file *os.File, start, len uint32, timeout time.Duration) _Error
func osLock(file *os.File, flags, start, len uint32, timeout time.Duration, def _ErrorCode) _ErrorCode {
var err error
switch {
case timeout == 0:
default:
err = osLockEx(file, flags|windows.LOCKFILE_FAIL_IMMEDIATELY, start, len)
case timeout < 0:
err = osLockEx(file, flags, start, len)
default:
err = osLockExTimeout(file, flags, start, len, timeout)
}
return osLockErrorCode(err, def)
}
@ -162,37 +160,6 @@ func osLockEx(file *os.File, flags, start, len uint32) error {
0, len, 0, &windows.Overlapped{Offset: start})
}
func osLockExTimeout(file *os.File, flags, start, len uint32, timeout time.Duration) error {
event, err := windows.CreateEvent(nil, 1, 0, nil)
if err != nil {
return err
}
defer windows.CloseHandle(event)
fd := windows.Handle(file.Fd())
overlapped := &windows.Overlapped{
Offset: start,
HEvent: event,
}
err = windows.LockFileEx(fd, flags, 0, len, 0, overlapped)
if err != windows.ERROR_IO_PENDING {
return err
}
ms := (timeout + time.Millisecond - 1) / time.Millisecond
rc, err := windows.WaitForSingleObject(event, uint32(ms))
if rc == windows.WAIT_OBJECT_0 {
return nil
}
defer windows.CancelIoEx(fd, overlapped)
if err != nil {
return err
}
return windows.Errno(rc)
}
func osLockErrorCode(err error, def _ErrorCode) _ErrorCode {
if err == nil {
return _OK

View file

@ -68,16 +68,11 @@ func (s *vfsShm) Close() error {
panic(util.AssertErr())
}
func (s *vfsShm) shmOpen() _ErrorCode {
func (s *vfsShm) shmOpen() (rc _ErrorCode) {
if s.vfsShmParent != nil {
return _OK
}
var f *os.File
// Close file on error.
// Keep this here to avoid confusing checklocks.
defer func() { f.Close() }()
vfsShmListMtx.Lock()
defer vfsShmListMtx.Unlock()
@ -98,11 +93,16 @@ func (s *vfsShm) shmOpen() _ErrorCode {
}
// Always open file read-write, as it will be shared.
f, err = os.OpenFile(s.path,
f, err := os.OpenFile(s.path,
os.O_RDWR|os.O_CREATE|_O_NOFOLLOW, 0666)
if err != nil {
return _CANTOPEN
}
defer func() {
if rc != _OK {
f.Close()
}
}()
// Dead man's switch.
if lock, rc := osTestLock(f, _SHM_DMS, 1); rc != _OK {
@ -131,7 +131,6 @@ func (s *vfsShm) shmOpen() _ErrorCode {
File: f,
info: fi,
}
f = nil // Don't close the file.
for i, g := range vfsShmList {
if g == nil {
vfsShmList[i] = s.vfsShmParent

View file

@ -7,14 +7,11 @@ import (
"io"
"os"
"sync"
"syscall"
"time"
"github.com/tetratelabs/wazero/api"
"golang.org/x/sys/windows"
"github.com/ncruces/go-sqlite3/internal/util"
"github.com/ncruces/go-sqlite3/util/osutil"
)
type vfsShm struct {
@ -33,8 +30,6 @@ type vfsShm struct {
sync.Mutex
}
var _ blockingSharedMemory = &vfsShm{}
func (s *vfsShm) Close() error {
// Unmap regions.
for _, r := range s.regions {
@ -48,8 +43,7 @@ func (s *vfsShm) Close() error {
func (s *vfsShm) shmOpen() _ErrorCode {
if s.File == nil {
f, err := osutil.OpenFile(s.path,
os.O_RDWR|os.O_CREATE|syscall.O_NONBLOCK, 0666)
f, err := os.OpenFile(s.path, os.O_RDWR|os.O_CREATE, 0666)
if err != nil {
return _CANTOPEN
}
@ -67,7 +61,7 @@ func (s *vfsShm) shmOpen() _ErrorCode {
return _IOERR_SHMOPEN
}
}
rc := osReadLock(s.File, _SHM_DMS, 1, time.Millisecond)
rc := osReadLock(s.File, _SHM_DMS, 1, 0)
s.fileLock = rc == _OK
return rc
}
@ -135,11 +129,6 @@ func (s *vfsShm) shmMap(ctx context.Context, mod api.Module, id, size int32, ext
}
func (s *vfsShm) shmLock(offset, n int32, flags _ShmFlag) (rc _ErrorCode) {
var timeout time.Duration
if s.blocking {
timeout = time.Millisecond
}
switch {
case flags&_SHM_LOCK != 0:
defer s.shmAcquire(&rc)
@ -151,9 +140,9 @@ func (s *vfsShm) shmLock(offset, n int32, flags _ShmFlag) (rc _ErrorCode) {
case flags&_SHM_UNLOCK != 0:
return osUnlock(s.File, _SHM_BASE+uint32(offset), uint32(n))
case flags&_SHM_SHARED != 0:
return osReadLock(s.File, _SHM_BASE+uint32(offset), uint32(n), timeout)
return osReadLock(s.File, _SHM_BASE+uint32(offset), uint32(n), 0)
case flags&_SHM_EXCLUSIVE != 0:
return osWriteLock(s.File, _SHM_BASE+uint32(offset), uint32(n), timeout)
return osWriteLock(s.File, _SHM_BASE+uint32(offset), uint32(n), 0)
default:
panic(util.AssertErr())
}
@ -184,7 +173,3 @@ func (s *vfsShm) shmUnmap(delete bool) {
os.Remove(s.path)
}
}
func (s *vfsShm) shmEnableBlocking(block bool) {
s.blocking = block
}

View file

@ -79,9 +79,12 @@ func implements[T any](typ reflect.Type) bool {
//
// https://sqlite.org/c3ref/declare_vtab.html
func (c *Conn) DeclareVTab(sql string) error {
if c.interrupt.Err() != nil {
return INTERRUPT
}
defer c.arena.mark()()
sqlPtr := c.arena.string(sql)
rc := res_t(c.call("sqlite3_declare_vtab", stk_t(c.handle), stk_t(sqlPtr)))
textPtr := c.arena.string(sql)
rc := res_t(c.call("sqlite3_declare_vtab", stk_t(c.handle), stk_t(textPtr)))
return c.error(rc)
}
@ -162,6 +165,7 @@ type VTabDestroyer interface {
}
// A VTabUpdater allows a virtual table to be updated.
// Implementations must not retain arg.
type VTabUpdater interface {
VTab
// https://sqlite.org/vtab.html#xupdate
@ -241,6 +245,7 @@ type VTabSavepointer interface {
// to loop through the virtual table.
// A VTabCursor may optionally implement
// [io.Closer] to free resources.
// Implementations of Filter must not retain arg.
//
// https://sqlite.org/c3ref/vtab_cursor.html
type VTabCursor interface {
@ -489,12 +494,12 @@ func vtabBestIndexCallback(ctx context.Context, mod api.Module, pVTab, pIdxInfo
}
func vtabUpdateCallback(ctx context.Context, mod api.Module, pVTab ptr_t, nArg int32, pArg, pRowID ptr_t) res_t {
vtab := vtabGetHandle(ctx, mod, pVTab).(VTabUpdater)
db := ctx.Value(connKey{}).(*Conn)
args := make([]Value, nArg)
callbackArgs(db, args, pArg)
rowID, err := vtab.Update(args...)
args := callbackArgs(db, nArg, pArg)
defer returnArgs(args)
vtab := vtabGetHandle(ctx, mod, pVTab).(VTabUpdater)
rowID, err := vtab.Update(*args...)
if err == nil {
util.Write64(mod, pRowID, rowID)
}
@ -593,15 +598,17 @@ func cursorCloseCallback(ctx context.Context, mod api.Module, pCur ptr_t) res_t
}
func cursorFilterCallback(ctx context.Context, mod api.Module, pCur ptr_t, idxNum int32, idxStr ptr_t, nArg int32, pArg ptr_t) res_t {
cursor := vtabGetHandle(ctx, mod, pCur).(VTabCursor)
db := ctx.Value(connKey{}).(*Conn)
args := make([]Value, nArg)
callbackArgs(db, args, pArg)
args := callbackArgs(db, nArg, pArg)
defer returnArgs(args)
var idxName string
if idxStr != 0 {
idxName = util.ReadString(mod, idxStr, _MAX_LENGTH)
}
err := cursor.Filter(int(idxNum), idxName, args...)
cursor := vtabGetHandle(ctx, mod, pCur).(VTabCursor)
err := cursor.Filter(int(idxNum), idxName, *args...)
return vtabError(ctx, mod, pCur, _CURSOR_ERROR, err)
}

View file

@ -1535,27 +1535,29 @@ func (v *Viper) MergeInConfig() error {
func ReadConfig(in io.Reader) error { return v.ReadConfig(in) }
func (v *Viper) ReadConfig(in io.Reader) error {
if v.configType == "" {
return errors.New("cannot decode configuration: config type is not set")
config := make(map[string]any)
err := v.unmarshalReader(in, config)
if err != nil {
return err
}
v.config = make(map[string]any)
return v.unmarshalReader(in, v.config)
v.config = config
return nil
}
// MergeConfig merges a new configuration with an existing config.
func MergeConfig(in io.Reader) error { return v.MergeConfig(in) }
func (v *Viper) MergeConfig(in io.Reader) error {
if v.configType == "" {
return errors.New("cannot decode configuration: config type is not set")
}
config := make(map[string]any)
cfg := make(map[string]any)
if err := v.unmarshalReader(in, cfg); err != nil {
if err := v.unmarshalReader(in, config); err != nil {
return err
}
return v.MergeConfigMap(cfg)
return v.MergeConfigMap(config)
}
// MergeConfigMap merges the configuration from the map given with an existing config.
@ -1662,15 +1664,21 @@ func (v *Viper) writeConfig(filename string, force bool) error {
}
func (v *Viper) unmarshalReader(in io.Reader, c map[string]any) error {
format := strings.ToLower(v.getConfigType())
if format == "" {
return errors.New("cannot decode configuration: unable to determine config type")
}
buf := new(bytes.Buffer)
buf.ReadFrom(in)
format := strings.ToLower(v.getConfigType())
// TODO: remove this once SupportedExts is deprecated/removed
if !slices.Contains(SupportedExts, format) {
return UnsupportedConfigError(format)
}
// TODO: return [UnsupportedConfigError] if the registry does not contain the format
// TODO: consider deprecating this error type
decoder, err := v.decoderRegistry.Decoder(format)
if err != nil {
return ConfigParseError{err}

View file

@ -49,6 +49,7 @@ release:
fi; \
rm -rf dist/$$t; \
done
./publish.sh
clean:
echo "Cleaning dist/"

View file

@ -145,11 +145,11 @@ func (o *Minifier) Minify(m *minify.M, w io.Writer, r io.Reader, _ map[string]st
w.Write(t.Data)
}
omitSpace = false
case html.TemplateToken:
w.Write(t.Data)
omitSpace = false
case html.TextToken:
if t.HasTemplate {
w.Write(t.Data)
omitSpace = parse.IsWhitespace(t.Data[len(t.Data)-1])
} else if rawTagHash != 0 {
if rawTagHash != 0 && !t.HasTemplate {
if rawTagHash == Style || rawTagHash == Script || rawTagHash == Iframe {
var mimetype []byte
var params map[string]string
@ -196,7 +196,7 @@ func (o *Minifier) Minify(m *minify.M, w io.Writer, r io.Reader, _ map[string]st
t.Data = t.Data[:len(t.Data)-1]
omitSpace = false
break
} else if next.TokenType == html.TextToken && !parse.IsAllWhitespace(next.Data) {
} else if next.TokenType == html.TextToken && !parse.IsAllWhitespace(next.Data) || next.TokenType == html.TemplateToken {
// stop looking when text encountered
break
} else if next.TokenType == html.StartTagToken || next.TokenType == html.EndTagToken || next.TokenType == html.SvgToken || next.TokenType == html.MathToken {
@ -215,7 +215,6 @@ func (o *Minifier) Minify(m *minify.M, w io.Writer, r io.Reader, _ map[string]st
i++
}
}
w.Write(t.Data)
}
case html.StartTagToken, html.EndTagToken:

35
vendor/github.com/tdewolff/minify/v2/publish.sh generated vendored Normal file
View file

@ -0,0 +1,35 @@
#!/bin/sh
VERSION=`git describe --tags --abbrev=0 | cut -c 2-`
cd dist
echo ""
echo "Downloading v$VERSION.tar.gz..."
wget -q --show-progress https://github.com/tdewolff/minify/archive/v$VERSION.tar.gz
SHA256=`sha256sum v$VERSION.tar.gz`
SHA256=( $SHA256 )
echo ""
echo "Releasing for AUR..."
cd /home/taco/dev/aur/minify
sed -i "s/^pkgver=.*$/pkgver=$VERSION/" PKGBUILD
sed -i "s/^sha256sums=.*$/sha256sums=('$SHA256')/" PKGBUILD
./build.sh
git commit -am "Update to v$VERSION"
git push
cd -
echo ""
echo "Releasing for Homebrew..."
cd /home/taco/dev/brew/homebrew-tap/Formula
sed -i "s,^ url \".*\"$, url \"https://github.com/tdewolff/minify/archive/v$VERSION.tar.gz\"," minify.rb
sed -i "s/^ sha256 \".*\"$/ sha256 \"$SHA256\"/" minify.rb
git commit -am "Update to v$VERSION"
git push
cd -
#echo ""
#echo "Releasing Python bindings..."
#cd ../bindings/py
#make publish
#cd -

View file

@ -23,6 +23,7 @@ const (
TextToken
SvgToken
MathToken
TemplateToken
)
// String returns the string representation of a TokenType.
@ -185,18 +186,19 @@ func (l *Lexer) Next() (TokenType, []byte) {
} else if c == '!' {
l.r.Move(2)
return l.readMarkup()
} else if 0 < len(l.tmplBegin) && l.at(l.tmplBegin...) {
l.r.Move(len(l.tmplBegin))
l.moveTemplate()
l.hasTmpl = true
} else if c == '?' {
l.r.Move(1)
return CommentToken, l.shiftBogusComment()
}
} else if 0 < len(l.tmplBegin) && l.at(l.tmplBegin...) {
if 0 < l.r.Pos() {
l.text = l.r.Shift()
return TextToken, l.text
}
l.r.Move(len(l.tmplBegin))
l.moveTemplate()
l.hasTmpl = true
return TemplateToken, l.r.Shift()
} else if c == 0 && l.r.Err() != nil {
if 0 < l.r.Pos() {
l.text = l.r.Shift()