Merge branch 'main' into test/added-sqlean-extensions

Author: ManakRaj-7

.github/workflows/extensions-test.yml

@@ -68,6 +68,9 @@ jobs:
       - uses: hecrj/setup-rust-action@v2
       - name: Checkout repository
         uses: actions/checkout@v3
+      - name: install libs
+        run: |
+          sudo apt install libblas-dev liblapack-dev
       - name: build libsql and ffi bindings
         run: |
           cargo xtask build-bundled

.github/workflows/golang-bindings.yml

@@ -68,8 +68,7 @@ jobs:
         path: "go-libsql"
 
     - name: Copy sql-experimental library to go-libsql
-      run: cp target/release/libsql_experimental.a go-libsql/lib/darwin_amd64/
-           && cp target/release/libsql_experimental.a go-libsql/lib/darwin_arm64/
+      run: cp target/release/libsql_experimental.a go-libsql/lib/darwin_arm64/
            && cp target/release/libsql_experimental.a go-libsql/lib/linux_amd64/
            && cp target/release/libsql_experimental.a go-libsql/lib/linux_arm64/
            && cp bindings/c/include/libsql.h go-libsql/lib/include/

.github/workflows/libsql-server-release.yml

@@ -1,10 +1,12 @@
-# Copyright 2022-2024, axodotdev
+# This file was autogenerated by dist: https://github.com/astral-sh/cargo-dist
+#
+# Copyright 2025 Astral Software Inc.
 # SPDX-License-Identifier: MIT or Apache-2.0
 #
 # CI that:
 #
 # * checks for a Git Tag that looks like a release
-# * builds artifacts with cargo-dist (archives, installers, hashes)
+# * builds artifacts with dist (archives, installers, hashes)
 # * uploads those artifacts to temporary workflow zip
 # * on success, uploads the artifacts to a GitHub Release
 #
@@ -22,10 +24,10 @@ permissions:
 # must be a Cargo-style SemVer Version (must have at least major.minor.patch).
 #
 # If PACKAGE_NAME is specified, then the announcement will be for that
-# package (erroring out if it doesn't have the given version or isn't cargo-dist-able).
+# package (erroring out if it doesn't have the given version or isn't dist-able).
 #
 # If PACKAGE_NAME isn't specified, then the announcement will be for all
-# (cargo-dist-able) packages in the workspace with that version (this mode is
+# (dist-able) packages in the workspace with that version (this mode is
 # intended for workspaces with only one dist-able package, or with all dist-able
 # packages versioned/released in lockstep).
 #
@@ -43,9 +45,9 @@ on:
       - 'libsql-server**[0-9]+.[0-9]+.[0-9]+*'
 
 jobs:
-  # Run 'cargo dist plan' (or host) to determine what tasks we need to do
+  # Run 'dist plan' (or host) to determine what tasks we need to do
   plan:
-    runs-on: "ubuntu-20.04"
+    runs-on: "ubuntu-22.04"
     outputs:
       val: ${{ steps.plan.outputs.manifest }}
       tag: ${{ !github.event.pull_request && github.ref_name || '' }}
@@ -57,25 +59,25 @@ jobs:
       - uses: actions/checkout@v4
         with:
           submodules: recursive
-      - name: Install cargo-dist
+      - name: Install dist
         # we specify bash to get pipefail; it guards against the `curl` command
         # failing. otherwise `sh` won't catch that `curl` returned non-0
         shell: bash
-        run: "curl --proto '=https' --tlsv1.2 -LsSf https://github.com/axodotdev/cargo-dist/releases/download/v0.21.0/cargo-dist-installer.sh | sh"
-      - name: Cache cargo-dist
+        run: "curl --proto '=https' --tlsv1.2 -LsSf https://github.com/astral-sh/cargo-dist/releases/download/v0.28.3/cargo-dist-installer.sh | sh"
+      - name: Cache dist
         uses: actions/upload-artifact@v4
         with:
           name: cargo-dist-cache
-          path: ~/.cargo/bin/cargo-dist
+          path: ~/.cargo/bin/dist
       # sure would be cool if github gave us proper conditionals...
       # so here's a doubly-nested ternary-via-truthiness to try to provide the best possible
       # functionality based on whether this is a pull_request, and whether it's from a fork.
       # (PRs run on the *source* but secrets are usually on the *target* -- that's *good*
       # but also really annoying to build CI around when it needs secrets to work right.)
       - id: plan
         run: |
-          cargo dist ${{ (!github.event.pull_request && format('host --steps=create --tag={0}', github.ref_name)) || 'plan' }} --output-format=json > plan-dist-manifest.json
-          echo "cargo dist ran successfully"
+          dist ${{ (!github.event.pull_request && format('host --steps=create --tag={0}', github.ref_name)) || 'plan' }} --output-format=json > plan-dist-manifest.json
+          echo "dist ran successfully"
           cat plan-dist-manifest.json
           echo "manifest=$(jq -c "." plan-dist-manifest.json)" >> "$GITHUB_OUTPUT"
       - name: "Upload dist-manifest.json"
@@ -93,18 +95,19 @@ jobs:
     if: ${{ fromJson(needs.plan.outputs.val).ci.github.artifacts_matrix.include != null && (needs.plan.outputs.publishing == 'true' || fromJson(needs.plan.outputs.val).ci.github.pr_run_mode == 'upload') }}
     strategy:
       fail-fast: false
-      # Target platforms/runners are computed by cargo-dist in create-release.
+      # Target platforms/runners are computed by dist in create-release.
       # Each member of the matrix has the following arguments:
       #
       # - runner: the github runner
-      # - dist-args: cli flags to pass to cargo dist
-      # - install-dist: expression to run to install cargo-dist on the runner
+      # - dist-args: cli flags to pass to dist
+      # - install-dist: expression to run to install dist on the runner
       #
       # Typically there will be:
       # - 1 "global" task that builds universal installers
       # - N "local" tasks that build each platform's binaries and platform-specific installers
       matrix: ${{ fromJson(needs.plan.outputs.val).ci.github.artifacts_matrix }}
     runs-on: ${{ matrix.runner }}
+    container: ${{ matrix.container && matrix.container.image || null }}
     env:
       GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
       BUILD_MANIFEST_NAME: target/distrib/${{ join(matrix.targets, '-') }}-dist-manifest.json
@@ -115,10 +118,17 @@ jobs:
       - uses: actions/checkout@v4
         with:
           submodules: recursive
+      - name: Install Rust non-interactively if not already installed
+        if: ${{ matrix.container }}
+        run: |
+          if ! command -v cargo > /dev/null 2>&1; then
+            curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh -s -- -y
+            echo "$HOME/.cargo/bin" >> $GITHUB_PATH
+          fi
       - name: "Prepare env vars"
         run: "echo \"RUSTFLAGS=--cfg tokio_unstable\" >> $GITHUB_ENV"
-      - name: Install cargo-dist
-        run: ${{ matrix.install_dist }}
+      - name: Install dist
+        run: ${{ matrix.install_dist.run }}
       # Get the dist-manifest
       - name: Fetch local artifacts
         uses: actions/download-artifact@v4
@@ -132,8 +142,8 @@ jobs:
       - name: Build artifacts
         run: |
           # Actually do builds and make zips and whatnot
-          cargo dist build ${{ needs.plan.outputs.tag-flag }} --print=linkage --output-format=json ${{ matrix.dist_args }} > dist-manifest.json
-          echo "cargo dist ran successfully"
+          dist build ${{ needs.plan.outputs.tag-flag }} --print=linkage --output-format=json ${{ matrix.dist_args }} > dist-manifest.json
+          echo "dist ran successfully"
       - id: cargo-dist
         name: Post-build
         # We force bash here just because github makes it really hard to get values up
@@ -143,7 +153,7 @@ jobs:
         run: |
           # Parse out what we just built and upload it to scratch storage
           echo "paths<<EOF" >> "$GITHUB_OUTPUT"
-          jq --raw-output ".upload_files[]" dist-manifest.json >> "$GITHUB_OUTPUT"
+          dist print-upload-files-from-manifest --manifest dist-manifest.json >> "$GITHUB_OUTPUT"
           echo "EOF" >> "$GITHUB_OUTPUT"
 
           cp dist-manifest.json "$BUILD_MANIFEST_NAME"
@@ -160,20 +170,20 @@ jobs:
     needs:
       - plan
       - build-local-artifacts
-    runs-on: "ubuntu-20.04"
+    runs-on: "ubuntu-22.04"
     env:
       GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
       BUILD_MANIFEST_NAME: target/distrib/global-dist-manifest.json
     steps:
       - uses: actions/checkout@v4
         with:
           submodules: recursive
-      - name: Install cached cargo-dist
+      - name: Install cached dist
         uses: actions/download-artifact@v4
         with:
           name: cargo-dist-cache
           path: ~/.cargo/bin/
-      - run: chmod +x ~/.cargo/bin/cargo-dist
+      - run: chmod +x ~/.cargo/bin/dist
       # Get all the local artifacts for the global tasks to use (for e.g. checksums)
       - name: Fetch local artifacts
         uses: actions/download-artifact@v4
@@ -184,8 +194,8 @@ jobs:
       - id: cargo-dist
         shell: bash
         run: |
-          cargo dist build ${{ needs.plan.outputs.tag-flag }} --output-format=json "--artifacts=global" > dist-manifest.json
-          echo "cargo dist ran successfully"
+          dist build ${{ needs.plan.outputs.tag-flag }} --output-format=json "--artifacts=global" > dist-manifest.json
+          echo "dist ran successfully"
 
           # Parse out what we just built and upload it to scratch storage
           echo "paths<<EOF" >> "$GITHUB_OUTPUT"
@@ -210,19 +220,19 @@ jobs:
     if: ${{ always() && needs.plan.outputs.publishing == 'true' && (needs.build-global-artifacts.result == 'skipped' || needs.build-global-artifacts.result == 'success') && (needs.build-local-artifacts.result == 'skipped' || needs.build-local-artifacts.result == 'success') }}
     env:
       GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
-    runs-on: "ubuntu-20.04"
+    runs-on: "ubuntu-22.04"
     outputs:
       val: ${{ steps.host.outputs.manifest }}
     steps:
       - uses: actions/checkout@v4
         with:
           submodules: recursive
-      - name: Install cached cargo-dist
+      - name: Install cached dist
         uses: actions/download-artifact@v4
         with:
           name: cargo-dist-cache
           path: ~/.cargo/bin/
-      - run: chmod +x ~/.cargo/bin/cargo-dist
+      - run: chmod +x ~/.cargo/bin/dist
       # Fetch artifacts from scratch-storage
       - name: Fetch artifacts
         uses: actions/download-artifact@v4
@@ -233,7 +243,7 @@ jobs:
       - id: host
         shell: bash
         run: |
-          cargo dist host ${{ needs.plan.outputs.tag-flag }} --steps=upload --steps=release --output-format=json > dist-manifest.json
+          dist host ${{ needs.plan.outputs.tag-flag }} --steps=upload --steps=release --output-format=json > dist-manifest.json
           echo "artifacts uploaded and released successfully"
           cat dist-manifest.json
           echo "manifest=$(jq -c "." dist-manifest.json)" >> "$GITHUB_OUTPUT"
@@ -270,7 +280,7 @@ jobs:
     needs:
       - plan
       - host
-    runs-on: "ubuntu-20.04"
+    runs-on: "ubuntu-22.04"
     env:
       GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
       PLAN: ${{ needs.plan.outputs.val }}
@@ -301,6 +311,11 @@ jobs:
             name=$(echo "$filename" | sed "s/\.rb$//")
             version=$(echo "$release" | jq .app_version --raw-output)
 
+            export PATH="/home/linuxbrew/.linuxbrew/bin:$PATH"
+            brew update
+            # We avoid reformatting user-provided data such as the app description and homepage.
+            brew style --except-cops FormulaAudit/Homepage,FormulaAudit/Desc,FormulaAuditStrict --fix "Formula/${filename}" || true
+
             git add "Formula/${filename}"
             git commit -m "${name} ${version}"
           done
@@ -315,7 +330,7 @@ jobs:
     # still allowing individual publish jobs to skip themselves (for prereleases).
     # "host" however must run to completion, no skipping allowed!
     if: ${{ always() && needs.host.result == 'success' && (needs.publish-homebrew-formula.result == 'skipped' || needs.publish-homebrew-formula.result == 'success') }}
-    runs-on: "ubuntu-20.04"
+    runs-on: "ubuntu-22.04"
     env:
       GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
     steps:

.github/workflows/publish-server.yml

@@ -124,7 +124,7 @@ jobs:
 
   build-arm64:
     permissions: write-all
-    runs-on: buildjet-8vcpu-ubuntu-2204-arm
+    runs-on: ubuntu-24.04-arm
     env:
       platform: "linux/arm64"
     steps:

.github/workflows/rust.yml

@@ -23,6 +23,7 @@ jobs:
     runs-on: ubuntu-latest
     name: Run Checks
     env:
+      RUST_BACKTRACE: 1
       RUSTFLAGS: -D warnings --cfg tokio_unstable
     steps:
       - uses: hecrj/setup-rust-action@v2
@@ -130,12 +131,97 @@ jobs:
     - name: Run tests
       run: cargo nextest run
 
-    - name: Run tests with custom pager
-      env:
-        LIBSQL_EXPERIMENTAL_PAGER: 1000
-      run: cargo nextest run
-    - name: clean build dir
-      run: rm -rf libsql-ffi/bundled/SQLite3MultipleCiphers/build
+    - name: Run libsql encryption tests
+      run: cargo test --features encryption --color=always --test encryption test_encryption
+  # test-custom-pager:
+  #   runs-on: ubuntu-latest
+  #   name: Run Tests
+  #   env:
+  #     RUSTFLAGS: -D warnings --cfg tokio_unstable
+  #   steps:
+  #   - uses: hecrj/setup-rust-action@v2
+
+  #   # needed because we run out of disk space during tests
+  #   - name: Free Disk Space (Ubuntu)
+  #     uses: jlumbroso/free-disk-space@main
+  #     with:
+  #       # this might remove tools that are actually needed,
+  #       # when set to "true" but frees about 6 GB
+  #       tool-cache: true
+
+  #   - name: Install deps
+  #     run: sudo apt update && sudo apt install -y libclang-dev
+
+  #   - name: Install protoc
+  #     uses: taiki-e/install-action@v2
+  #     with:
+  #       tool: protoc@${{ env.PROTOC_VERSION }}
+
+  #   - uses: taiki-e/install-action@v2
+  #     with:
+  #       tool: nextest
+
+  #   - uses: actions/checkout@v3
+
+  #   - name: Set up cargo cache
+  #     uses: actions/cache@v3
+  #     continue-on-error: false
+  #     with:
+  #       path: |
+  #         ~/.cargo/bin/
+  #         ~/.cargo/registry/index/
+  #         ~/.cargo/registry/cache/
+  #         ~/.cargo/git/db/
+  #         target/
+  #       key: ${{ runner.os }}-cargo-${{ hashFiles('**/Cargo.lock') }}
+  #       restore-keys: ${{ runner.os }}-cargo-
+  #   - name: Run tests with custom pager
+  #     env:
+  #       LIBSQL_EXPERIMENTAL_PAGER: 1000
+  #     run: cargo nextest run
+  test-encryption:
+    runs-on: ubuntu-latest
+    name: Run Tests
+    env:
+      RUST_BACKTRACE: 1
+      RUSTFLAGS: -D warnings --cfg tokio_unstable
+    steps:
+    - uses: hecrj/setup-rust-action@v2
+
+    # needed because we run out of disk space during tests
+    - name: Free Disk Space (Ubuntu)
+      uses: jlumbroso/free-disk-space@main
+      with:
+        # this might remove tools that are actually needed,
+        # when set to "true" but frees about 6 GB
+        tool-cache: true
+
+    - name: Install deps
+      run: sudo apt update && sudo apt install -y libclang-dev
+
+    - name: Install protoc
+      uses: taiki-e/install-action@v2
+      with:
+        tool: protoc@${{ env.PROTOC_VERSION }}
+
+    - uses: taiki-e/install-action@v2
+      with:
+        tool: nextest
+
+    - uses: actions/checkout@v3
+
+    - name: Set up cargo cache
+      uses: actions/cache@v3
+      continue-on-error: false
+      with:
+        path: |
+          ~/.cargo/bin/
+          ~/.cargo/registry/index/
+          ~/.cargo/registry/cache/
+          ~/.cargo/git/db/
+          target/
+        key: ${{ runner.os }}-cargo-${{ hashFiles('**/Cargo.lock') }}
+        restore-keys: ${{ runner.os }}-cargo-
     - name: embedded replica encryption tests
       run: cargo xtask test-encryption
   windows:

.github/workflows/server-pr-images.yml

@@ -69,20 +69,6 @@ jobs:
           cache-from: type=gha
           cache-to: type=gha,mode=max
 
-      - name: Build and push Docker image with durable-wal
-        uses: docker/build-push-action@ad44023a93711e3deb337508980b4b5e9bcdc5dc
-        with:
-          context: .
-          push: true
-          tags: ${{ steps.meta.outputs.tags }}-durable-wal-${{ steps.get-short-sha.outputs.sha }}
-          labels: ${{ steps.meta.outputs.labels }}
-          cache-from: type=gha
-          cache-to: type=gha,mode=max
-          build-args: |
-            BUILD_DEBUG=true
-            ENABLE_FEATURES=durable-wal
-
       - name: Echo image name
         run: |
           echo "Pushed: ${{ steps.meta.outputs.tags }}-${{ steps.get-short-sha.outputs.sha }}"
-          echo "Pushed: ${{ steps.meta.outputs.tags }}-durable-wal-${{ steps.get-short-sha.outputs.sha }}"