feat(cogs): create inventory-tracker crate for storage oplog

Author: matt-codecov

.github/workflows/ci.yml

@@ -22,6 +22,13 @@ jobs:
     runs-on: ubuntu-latest
 
     steps:
+      # librdkafka is built from source and links against libcurl, whose headers are not
+      # on the runner by default. Needed by anything that builds the `kafka` feature.
+      - name: Install libcurl-dev
+        run: |
+          sudo apt-get update
+          sudo apt-get install -y libcurl4-openssl-dev
+
       - uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5.0.0
 
       - name: Install Rust Toolchain
@@ -75,6 +82,13 @@ jobs:
     runs-on: ubuntu-latest
 
     steps:
+      # librdkafka is built from source and links against libcurl, whose headers are not
+      # on the runner by default. Needed by anything that builds the `kafka` feature.
+      - name: Install libcurl-dev
+        run: |
+          sudo apt-get update
+          sudo apt-get install -y libcurl4-openssl-dev
+
       - uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5.0.0
 
       - name: Start Bigtable Emulator
@@ -181,6 +195,13 @@ jobs:
       id-token: write
 
     steps:
+      # librdkafka is built from source and links against libcurl, whose headers are not
+      # on the runner by default. Needed by anything that builds the `kafka` feature.
+      - name: Install libcurl-dev
+        run: |
+          sudo apt-get update
+          sudo apt-get install -y libcurl4-openssl-dev
+
       - uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5.0.0
 
       - name: Install Rust Toolchain

Cargo.lock

@@ -21,6 +21,7 @@ debug = true
 lto = "thin"
 
 [workspace.dependencies]
+objectstore-inventory-tracker = { path = "objectstore-inventory-tracker" }
 objectstore-client = { path = "clients/rust" }
 objectstore-log = { path = "objectstore-log" }
 objectstore-metrics = { path = "objectstore-metrics" }
@@ -42,6 +43,7 @@ async-trait = "0.1.89"
 axum = "0.8.9"
 base64 = "0.22.1"
 bigtable_rs = { git = "https://github.com/getsentry/bigtable_rs.git", rev = "4cb75bc5e5f87204363973f6302107768e64972e" }
+blake3 = "1.8.5"
 bytes = "1.12.0"
 bytesize = "2.4.0"
 chrono = "0.4.45"
@@ -81,6 +83,10 @@ rustls = { version = "0.23.40", default-features = false }
 secrecy = "0.10.3"
 sentry = "0.48.3"
 sentry-options = "1.2.4"
+# - `ssl` is required for SASL/SCRAM
+# - `librdkafka` must be built from source either way, and `cmake-build` is the
+#    recommended way to do it
+rdkafka = { version = "0.39.0", features = ["cmake-build", "ssl", "tracing"] }
 serde = { version = "1.0.228", features = ["derive"] }
 serde_json = "1.0.150"
 serde_yaml = "0.9.34-deprecated"