[ { "id": "datadog-when-failover-isn-t-safe-building-high-availability-postgresql-on-kubern", "title": "When failover isn't safe: Building high-availability PostgreSQL on Kubernetes", "company": "Datadog", "date": "2026-06-04", "url": "https://www.datadoghq.com/blog/engineering/postgresql-ha-kubernetes/", "stack": [ "PostgreSQL", "Kubernetes", "Patroni", "ZooKeeper" ], "scale": [ "+53% avg write latency with remote_apply (pgbench TPC-B)", "-34% tps with remote_apply (pgbench TPC-B)" ], "techniques": [ "synchronous replication", "failover", "high availability", "chaos testing" ], "summary": "A gameday that injected network latency into one availability zone left several of Datadog's Kubernetes-based PostgreSQL clusters stuck: replicas lagged past Patroni's maximum_lag_on_failover, so no standby could be promoted safely. Datadog moved to a hybrid model: standbys in the leader pool take part in synchronous replication (one synchronous standby, synchronous_mode_strict, synchronous_commit=remote_apply) while read replicas stay asynchronous. pgbench showed remote_apply raised average latency by 53% and cut throughput by 34%, yet production clusters saw no measurable impact after a gradual rollout, and synchronous replication is now enabled in all of its clusters.", "verified": "2026-10-23" }, { "id": "grab-app-stability-at-scale-with-a-layered-defense", "title": "App stability at scale with a layered defense", "company": "Grab", "date": "2026-10-21", "url": "https://engineering.grab.com/app-stability-at-scale-with-a-layered-defense", "stack": [ "Android", "iOS", "Firebase Crashlytics", "BigQuery" ], "scale": [ "millions of lines of Android code", "100M+ Google Play downloads", "weekly release cycle" ], "techniques": [ "shift-left testing", "staged rollout", "feature flags", "crash triage" ], "summary": "With many engineers and AI-assisted code landing in a weekly release of a superapp that has millions of lines of Android code, Grab stops crashes in layers. Lint rules (including custom ones written after post-mortems) and tests block merges, a mandatory one-week Grab Early Access build catches crashes before production, and a staged rollout over a week is automatically paused when the new version's crash rate looks abnormal. In production, on-call engineers first flip feature flags, then try server-side fixes, then hotfix, helped by a flag-rollout history dashboard and Crashlytics data exported to BigQuery.", "verified": "2026-10-23" }, { "id": "canva-session-revocations-at-scale", "title": "Session revocations at scale", "company": "Canva", "date": "2026-07-22", "url": "https://www.canva.dev/blog/engineering/session-revocations-at-scale/", "stack": [ "MySQL", "Amazon S3", "ZooKeeper", "Java" ], "scale": [ "hundreds of millions of users", "hundreds of thousands of session checks/sec", "2,000+ revocations/sec write throughput", "87.5% smaller in-memory cache" ], "techniques": [ "caching", "object storage", "optimistic concurrency control", "binary encoding" ], "summary": "Canva keeps 12 hours of session revocations in memory on every gateway, but hundreds of pods each loading over a million revocations from MySQL on deploy caused a stampede on the database. A worker now packs revocations into sorted 16-byte records in 30-minute chunks on S3, using conditional PUTs for optimistic concurrency and ZooKeeper leader election to avoid conflicts, and reaches over 2,000 revocations per second. Gateways binary-search the downloaded bytes directly and poll with conditional GETs, which shrank the cache by 87.5% and cut the revocation database down to two read replicas kept for redundancy.", "verified": "2026-10-22" }, { "id": "netflix-timeseries-tiered-storage-journey-kafka-flink-streams-to-native-cassandr", "title": "TimeSeries Tiered Storage Journey: Kafka/Flink Streams to Native Cassandra Cold Reads", "company": "Netflix", "date": "2026-10-14", "url": "https://netflixtechblog.com/timeseries-tiered-storage-journey-kafka-flink-streams-to-native-cassandra-cold-reads-e59d597c9d60", "stack": [ "Cassandra", "Kafka", "Flink", "Amazon S3", "Parquet", "EVCache" ], "scale": [ "15+ PB compressed data in cold storage", "disk usage cut from 80% to 7%", "cold-read p99 under ~500 ms", "thousands of cold reads/sec per region" ], "techniques": [ "tiered storage", "compaction", "cost optimization", "backups as cold tier" ], "summary": "Netflix moved older data in its multi-petabyte TimeSeries Abstraction out of hot Cassandra storage in two steps. First, Kafka and Flink streamed events to S3 as Parquet files, which a daily job compacted into small, time-sorted row groups with Bloom filters cached in EVCache (p50 ~150 ms, p99 ~500 ms). The second design skips that streaming pipeline: a ReadOnlyCompactionStrategy aligns immutable SSTables to token ranges, and a custom pluggable Memtable (Cassandra 4.1) reads directly from existing Cassandra backups in S3. It now holds over 15 PB of compressed cold data, cut disk usage on affected clusters from 80% to 7%, and improved p90 latency by ~30%.", "verified": "2026-10-22" }, { "id": "github-the-bridge-to-agent-scale-git-infrastructure", "title": "The bridge to agent-scale Git infrastructure", "company": "GitHub", "date": "2026-10-21", "url": "https://github.blog/engineering/architecture-optimization/the-bridge-to-agent-scale-git-infrastructure/", "stack": [ "Git", "Spokes", "Azure", "Azure Blob Storage" ], "scale": [ "99.99999% of repositories copied to Azure", "Azure serves about half of Git traffic", "3-4x write throughput on different references (internal testing)", "100+ billion Spokes operations/day" ], "techniques": [ "per-reference quorum", "storage/compute separation", "cloud migration", "read replicas" ], "summary": "GitHub has copied 99.99999% of its Git repositories to Azure, where most now keep two of their five replicas and Azure serves about half of Git traffic. The busiest remaining repositories were blocked by a ~25ms cross-region round trip, because Spokes held a repository-wide lock for each write while waiting on the slowest healthy replica. Scoping the quorum to individual Git references lets writes to different branches proceed in parallel (3-4x higher throughput in internal testing), and a virtual write replica that streams writes into Azure Blob Storage lets stateless read replicas scale out on demand.", "verified": "2026-10-22" }, { "id": "meta-inside-prometheus-meta-s-journey-to-a-gigawatt-scale-supercluster", "title": "Inside Prometheus: Meta's Journey to a Gigawatt-Scale Supercluster", "company": "Meta", "date": "2026-10-19", "url": "https://engineering.fb.com/2026/10/19/data-center-engineering/inside-prometheus-meta-gigawatt-scale-supercluster/", "stack": [ "NVIDIA GB200 NVL72", "NVIDIA GB300 NVL72", "Ethernet back-end network", "liquid cooling" ], "scale": [ "gigawatt-scale supercluster", "hundreds of thousands of GPUs today, ~1 million planned", "85%+ pre-training goodput, 90%+ RL goodput", "MTBI raised from ~47K to 54K+ GPU hours" ], "techniques": [ "GPU clusters", "distributed training", "hardware reliability", "networking", "liquid cooling" ], "summary": "Meta turned its New Albany, Ohio data center site from web traffic into Prometheus, a gigawatt-scale AI supercluster spanning multiple interconnected buildings that behaves as one machine. It runs NVL72 racks (Catalina on GB200 with 800G of scale-out per GPU, Clemente on GB300 in a single rack) on a dedicated Ethernet back-end network for GPU-to-GPU traffic. Reliability work pushed pre-training goodput above 85% and mean time between interruptions past 54K GPU hours, and the cluster is growing toward roughly 1 million GPUs.", "verified": "2026-10-21" }, { "id": "meta-meta-s-generative-ads-model-gem-scaling-the-central-model-and-compute-engin", "title": "Meta's Generative Ads Model (GEM): Scaling the Central Model and Compute Engine for Frontier Ads AI", "company": "Meta", "date": "2026-10-13", "url": "https://engineering.fb.com/2026/10/13/ml-applications/generative-ads-model-gem-scaling-frontier-ads-ai/", "stack": [ "GPUs", "PyTorch", "custom low-precision kernels" ], "scale": [ "2.5x model complexity", "4x total training FLOPs", "20-25% MFU (training efficiency doubled)", "3.6 billion daily users across Meta apps" ], "techniques": [ "recommendation systems", "distributed training", "knowledge distillation", "model consolidation" ], "summary": "Meta replaced separate per-surface ad prediction systems with GEM, one central recommendation foundation model trained across all of its ad surfaces. A new LLM-style architecture (Kunlun) showed LLM-like scaling with 2.5x more model complexity, while custom low-precision kernels and 5D parallelism doubled end-to-end training efficiency to 20-25% MFU at 4x the training FLOPs across thousands of GPUs. Post-training transfer (LoopFM) and elastic pre-computation (Solaris) pass GEM's gains to real-time serving models without extra latency.", "verified": "2026-10-21" }, { "id": "discord-how-discord-automates-scylladb-clusters-at-scale", "title": "How Discord Automates ScyllaDB Clusters at Scale", "company": "Discord", "date": "2026-05-08", "url": "https://discord.com/blog/how-discord-automates-scylladb-clusters-at-scale", "stack": [ "ScyllaDB", "Rust", "YAML", "SQLite" ], "scale": [ "dozens of clusters, hundreds of database nodes", "7-person persistence team", "36-hour cluster operations cut to under 2 hours" ], "techniques": [ "database operations automation", "idempotent tasks", "zone-aware rollouts", "resumable jobs" ], "summary": "Discord's 7-person persistence team replaced fragile Python and bash scripts with the Scylla Control Plane, a Rust framework of idempotent tasks with declared preconditions, composed into YAML workflows. Jobs batch nodes by availability zone with concurrency limits to protect quorum, store their state in SQLite so they resume after interruptions, and send a webhook alert on unrecoverable errors. Operations like standing up shadow clusters and rolling upgrades went from a day and a half of babysitting to under two hours.", "verified": "2026-10-21" }, { "id": "zalando-client-side-load-balancing-at-a-million-requests-per-second", "title": "Client-Side Load Balancing at a Million Requests Per Second", "company": "Zalando", "date": "2026-06-23", "url": "https://engineering.zalando.com/posts/2026/06/client-side-load-balancing.html", "stack": [ "Java", "Kubernetes", "Skipper", "DynamoDB" ], "scale": [ "1 million+ requests/sec moved off shared ingress", "up to 100 downstream calls per batch", "Skipper pods cut from 50+ to 8", "$1,000+/day saved" ], "techniques": [ "load balancing", "consistent hashing", "caching", "fan-out", "load shedding" ], "summary": "Zalando moved the internal fan-out of its Product Read API off the shared Skipper ingress into an in-process JVM load balancer that rebuilds the same xxHash64 consistent-hash ring. They added N-ring fade-in so new pods warm only the products they will keep, bounded load based on occupancy (Little's Law) weighted by latency, and a capped ring walk. Pods ran more evenly, the autoscaler target rose from 50% to 65% CPU, and the fleet shrank by over 25%.", "verified": "2026-10-20" }, { "id": "github-building-git-infrastructure-for-agent-scale-development", "title": "Building Git infrastructure for agent-scale development", "company": "GitHub", "date": "2026-10-06", "url": "https://github.blog/engineering/architecture-optimization/building-git-infrastructure-for-agent-scale-development/", "stack": [ "Git", "Spokes", "Azure Blob Storage" ], "scale": [ "473.3 billion Git events/month", "7.38 billion commits in Sept 2026", "3.35 billion pushes/month", "up to 35x write throughput (internal benchmarks)" ], "techniques": [ "control plane separation", "read replicas", "caching", "workload isolation" ], "summary": "GitHub explains why agent workloads strain Spokes, which keeps five durable copies of each repo on fileserver disks, so every added read replica slows writes. The new design coordinates only the ref update, moves compaction and GC to separate workers, and separates storage from compute: Azure Blob Storage holds the authoritative data while lightweight caching workers serve reads.", "verified": "2026-10-20" }, { "id": "meta-shipping-media-over-quic-at-meta-scale-for-instagram-live-ingest", "title": "Shipping Media Over QUIC at Meta Scale for Instagram Live Ingest", "company": "Meta", "date": "2026-10-12", "url": "https://engineering.fb.com/2026/10/12/video-engineering/shipping-media-over-quic-at-meta-scale-for-instagram-live-ingest/", "stack": [ "QUIC", "MoQ", "moxygen", "mvfst" ], "scale": [], "techniques": [ "protocol migration", "edge computing", "fan-out", "zero downtime" ], "summary": "Instagram Live ingest moved from Meta's in-house RUSH protocol to the open IETF Media over QUIC standard on iOS and Android. Edge endpoints translate MoQ frames into the RUSH format, so the backend did not change. Server-side routing let the experiment resize in minutes, the transport falls back to TCP when UDP is blocked, and the launch matched RUSH on quality and reliability.", "verified": "2026-10-20" }, { "id": "openai-rapidly-scaling-online-storage-to-serve-over-1-billion-chatgpt-users", "title": "Rapidly scaling online storage to serve over 1 billion ChatGPT users", "company": "OpenAI", "date": "2026-09-11", "url": "https://openai.com/index/scaling-storage-one-billion-users-part-one", "stack": [ "Python", "Rust", "Cosmos DB", "Envoy", "Rockset" ], "scale": [ "70 million+ requests/sec", "1 billion+ weekly users", "500 PB+", "almost 40 regions" ], "techniques": [ "connection pooling", "load balancing", "rate limiting", "change data capture", "rewrite" ], "summary": "OpenAI's Habitat storage platform grew from a Python client library over Cosmos DB into a central service handling 70M+ requests per second. The team ran many low-concurrency Python processes to keep asyncio delay down, switched aiohttp connection reuse from LIFO to FIFO to stop a metastable failure, used Envoy for connection fan-in, kept a deliberately simple NoSQL API, then rewrote the service in Rust.", "verified": "2026-10-19" }, { "id": "meta-2026-world-cup-breaking-records-spiky-charts-and-reliability-at-scale", "title": "2026 World Cup: Breaking Records, Spiky Charts and Reliability at Scale", "company": "Meta", "date": "2026-10-15", "url": "https://engineering.fb.com/2026/10/15/production-engineering/2026-world-cup-record-traffic-reliability-at-scale/", "stack": [ "Taiji", "Katran", "Proxygen" ], "scale": [ "30 million+ WhatsApp messages/sec", "8.4 billion messages in 5 minutes", "49 million messages/min (Messenger, Threads, Instagram Direct)" ], "techniques": [ "capacity planning", "load testing", "load shedding", "load balancing" ], "summary": "How Meta absorbed the 2026 World Cup, when WhatsApp set a record of more than 30 million messages per second during the final. Pre-provisioned capacity buffers were the most effective lever, alongside country-level shedding of non-critical load, load-testing and resizing 100+ services, forecasting each region's peak separately, and no deploy freeze.", "verified": "2026-10-19" }, { "id": "shopify-we-replaced-redis-with-mysql-for-inventory-reservations-and-it-scaled", "title": "We replaced Redis with MySQL for inventory reservations\u2014and it scaled", "company": "Shopify", "date": "2026-05-12", "url": "https://shopify.engineering/scaling-inventory-reservations", "stack": [ "MySQL", "Redis", "ProxySQL" ], "scale": [ "$5.1 million sales/minute (Black Friday 2025 peak)" ], "techniques": [ "database migration", "dual writes", "query optimization", "connection pooling" ], "summary": "Shopify moved checkout inventory reservations from Redis to MySQL using one row per sellable unit, a bounded pool of 1,000 rows per item/location and SELECT ... FOR UPDATE SKIP LOCKED. The real ceiling was connections held by other checkout code: tagging SQL by business process and measuring hold time in ProxySQL found it, and the cleanup removed 50% of reads and 33% of transactions on the primary.", "verified": "2026-10-19" }, { "id": "openai-scaling-postgresql-to-power-800-million-chatgpt-users", "title": "Scaling PostgreSQL to power 800 million ChatGPT users", "company": "OpenAI", "date": "2026-01-22", "url": "https://openai.com/index/scaling-postgresql/", "stack": [ "PostgreSQL", "Azure", "PgBouncer", "Kubernetes", "Cosmos DB" ], "scale": [ "800 million users", "millions of QPS", "~50 read replicas" ], "techniques": [ "read replicas", "connection pooling", "caching", "rate limiting", "workload isolation", "query optimization" ], "summary": "OpenAI runs ChatGPT on a single-primary Postgres with nearly 50 global read replicas. Reads are offloaded, write-heavy shardable workloads move to Cosmos DB, and PgBouncer, cache leasing and multi-layer rate limits keep the primary safe.", "verified": "2026-10-15" }, { "id": "cloudflare-under-the-hood-off-the-hot-path-how-d1-scales-to-millions-of-database", "title": "Under the hood, off the hot path: how D1 scales to millions of databases", "company": "Cloudflare", "date": "2026-10-13", "url": "https://blog.cloudflare.com/d1-control-plane/", "stack": [ "SQLite", "Durable Objects", "Workers" ], "scale": [ "millions of databases" ], "techniques": [ "control plane separation", "multi-tenancy", "edge computing" ], "summary": "How Cloudflare's D1 keeps query execution at the edge while a separate control plane manages millions of SQLite databases.", "verified": "2026-10-15" }, { "id": "tiger-data-timescale-how-we-scaled-postgresql-to-350-tb", "title": "How We Scaled PostgreSQL to 350 TB+", "company": "Tiger Data (Timescale)", "date": "2023-11-02", "url": "https://www.tigerdata.com/blog/how-we-scaled-postgresql-to-350-tb-with-10b-new-records-day", "stack": [ "PostgreSQL", "TimescaleDB" ], "scale": [ "350 TB+", "10 billion new records/day" ], "techniques": [ "partitioning", "compression", "tiered storage" ], "summary": "Timescale dogfooded its own Postgres extension to build a database-observability product, using hypertables, columnar compression and tiered storage to keep a 350 TB+ dataset queryable.", "verified": "2026-10-15" }, { "id": "axiom-haydex-from-zero-to-178-600-000-000-rows-a-second-in-30-days", "title": "Haydex: From Zero to 178,600,000,000 rows a second in 30 days", "company": "Axiom", "date": "2025-09-24", "url": "https://axiom.co/blog/building-haydex", "stack": [ "Rust", "Object storage" ], "scale": [ "178.6 billion rows/sec" ], "techniques": [ "indexing", "filters", "query optimization" ], "summary": "Axiom turned a failed filter-index prototype into a production system for its event store, letting queries skip data and scan 178.6B rows per second.", "verified": "2026-10-15" }, { "id": "discord-how-discord-stores-trillions-of-messages", "title": "How Discord Stores Trillions of Messages", "company": "Discord", "date": "2023-03-06", "url": "https://discord.com/blog/how-discord-stores-trillions-of-messages", "stack": [ "ScyllaDB", "Cassandra", "Rust" ], "scale": [ "trillions of messages" ], "techniques": [ "database migration", "request coalescing", "data services layer" ], "summary": "Discord moved its message store from Cassandra to ScyllaDB and put Rust data services in front of it to coalesce hot-partition requests.", "verified": "2026-10-15" }, { "id": "discord-how-discord-stores-billions-of-messages", "title": "How Discord Stores Billions of Messages", "company": "Discord", "date": "2017-01-13", "url": "https://discord.com/blog/how-discord-stores-billions-of-messages", "stack": [ "Cassandra", "MongoDB" ], "scale": [ "billions of messages", "120 million messages/day" ], "techniques": [ "database migration", "data modeling", "partitioning" ], "summary": "Early story of Discord outgrowing MongoDB and moving messages to Cassandra with a channel+time-bucket partition key.", "verified": "2026-10-15" }, { "id": "discord-how-discord-indexes-billions-of-messages", "title": "How Discord Indexes Billions of Messages", "company": "Discord", "date": "2017-03-16", "url": "https://discord.com/blog/how-discord-indexes-billions-of-messages", "stack": [ "Elasticsearch", "Python" ], "scale": [ "billions of messages" ], "techniques": [ "sharding", "queues", "search indexing" ], "summary": "Discord built message search on many small Elasticsearch clusters, sharding by guild and indexing lazily through a queue.", "verified": "2026-10-15" }, { "id": "discord-how-discord-scaled-elixir-to-5-000-000-concurrent-users", "title": "How Discord Scaled Elixir to 5,000,000 Concurrent Users", "company": "Discord", "date": "2017-07-06", "url": "https://discord.com/blog/how-discord-scaled-elixir-to-5-000-000-concurrent-users", "stack": [ "Elixir", "Erlang/BEAM" ], "scale": [ "5,000,000 concurrent users" ], "techniques": [ "fan-out", "concurrency", "load shedding" ], "summary": "Discord tuned its Elixir guild processes and fan-out paths, building custom libraries to push events to millions of concurrent users.", "verified": "2026-10-15" }, { "id": "discord-why-discord-is-switching-from-go-to-rust", "title": "Why Discord is switching from Go to Rust", "company": "Discord", "date": "2020-02-04", "url": "https://discord.com/blog/why-discord-is-switching-from-go-to-rust", "stack": [ "Rust", "Go" ], "scale": [], "techniques": [ "rewrite", "caching", "latency reduction" ], "summary": "Discord rewrote its Read States service from Go to Rust to get rid of garbage-collection latency spikes in a large in-memory cache.", "verified": "2026-10-15" }, { "id": "figma-how-figma-s-databases-team-lived-to-tell-the-scale", "title": "How Figma's Databases Team Lived to Tell the Scale", "company": "Figma", "date": "2024-03-14", "url": "https://www.figma.com/blog/how-figmas-databases-team-lived-to-tell-the-scale/", "stack": [ "PostgreSQL", "AWS RDS" ], "scale": [], "techniques": [ "horizontal sharding", "query proxy", "online migration" ], "summary": "Figma's nine-month effort to horizontally shard its Postgres stack using a custom query proxy and logical sharding before physical splits.", "verified": "2026-10-15" }, { "id": "figma-the-growing-pains-of-database-architecture", "title": "The growing pains of database architecture", "company": "Figma", "date": "2023-04-04", "url": "https://www.figma.com/blog/how-figma-scaled-to-multiple-databases/", "stack": [ "PostgreSQL", "AWS RDS" ], "scale": [], "techniques": [ "vertical partitioning", "read replicas", "connection pooling" ], "summary": "How Figma moved from one Postgres instance to multiple databases by vertically partitioning table groups.", "verified": "2026-10-15" }, { "id": "figma-how-mozilla-s-rust-dramatically-improved-our-server-side-performance", "title": "How Mozilla's Rust dramatically improved our server-side performance", "company": "Figma", "date": "2018-05-02", "url": "https://www.figma.com/blog/rust-in-production-at-figma/", "stack": [ "Rust", "TypeScript" ], "scale": [], "techniques": [ "rewrite", "performance" ], "summary": "Figma rewrote its multiplayer server in Rust to cut latency and memory use as documents and users grew.", "verified": "2026-10-15" }, { "id": "figma-livegraph-real-time-data-fetching-at-figma", "title": "LiveGraph: real-time data fetching at Figma", "company": "Figma", "date": "2021-10-14", "url": "https://www.figma.com/blog/livegraph-real-time-data-fetching-at-figma/", "stack": [ "PostgreSQL", "GraphQL" ], "scale": [], "techniques": [ "change data capture", "real-time sync", "caching" ], "summary": "Figma built LiveGraph, which tails the Postgres replication log so its UI can subscribe to live query results.", "verified": "2026-10-15" }, { "id": "notion-herding-elephants-lessons-learned-from-sharding-postgres-at-notion", "title": "Herding elephants: Lessons learned from sharding Postgres at Notion", "company": "Notion", "date": "2021-10-06", "url": "https://www.notion.com/blog/sharding-postgres-at-notion", "stack": [ "PostgreSQL", "AWS RDS" ], "scale": [], "techniques": [ "horizontal sharding", "online migration", "double writes" ], "summary": "Notion split its Postgres monolith into a fleet of logical shards keyed by workspace, using double-writes and backfills to migrate.", "verified": "2026-10-15" }, { "id": "notion-the-great-re-shard-adding-postgres-capacity-again-with-zero-downtime", "title": "The Great Re-shard: adding Postgres capacity (again) with zero downtime", "company": "Notion", "date": "2023-07-17", "url": "https://www.notion.com/blog/the-great-re-shard", "stack": [ "PostgreSQL", "PgBouncer" ], "scale": [], "techniques": [ "resharding", "logical replication", "zero downtime" ], "summary": "Notion tripled its Postgres fleet by re-sharding with logical replication and careful PgBouncer cutovers, with no downtime.", "verified": "2026-10-15" }, { "id": "notion-building-and-scaling-notion-s-data-lake", "title": "Building and scaling Notion's data lake", "company": "Notion", "date": "2024-07-01", "url": "https://www.notion.com/blog/building-and-scaling-notions-data-lake", "stack": [ "Kafka", "Debezium", "Apache Hudi", "Spark", "S3" ], "scale": [], "techniques": [ "change data capture", "data lake", "incremental ingestion" ], "summary": "Notion replaced slow warehouse syncs with a CDC pipeline (Debezium to Kafka to Hudi on S3) processed with Spark.", "verified": "2026-10-15" }, { "id": "notion-how-we-sped-up-notion-in-the-browser-with-wasm-sqlite", "title": "How we sped up Notion in the browser with WASM SQLite", "company": "Notion", "date": "2024-07-10", "url": "https://www.notion.com/blog/how-we-sped-up-notion-in-the-browser-with-wasm-sqlite", "stack": [ "SQLite", "WebAssembly" ], "scale": [], "techniques": [ "client-side caching" ], "summary": "Notion cached data in the browser with WASM SQLite and OPFS, improving page navigation times.", "verified": "2026-10-15" }, { "id": "slack-scaling-datastores-at-slack-with-vitess", "title": "Scaling Datastores at Slack with Vitess", "company": "Slack", "date": "2020-12-01", "url": "https://slack.engineering/scaling-datastores-at-slack-with-vitess/", "stack": [ "MySQL", "Vitess" ], "scale": [ "2.3 million QPS" ], "techniques": [ "horizontal sharding", "database migration" ], "summary": "Slack's multi-year migration from active-active MySQL pairs to Vitess, so data could be sharded by any key, not only workspace.", "verified": "2026-10-15" }, { "id": "slack-flannel-an-application-level-edge-cache-to-make-slack-scale", "title": "Flannel: An Application-Level Edge Cache to Make Slack Scale", "company": "Slack", "date": "2017-05-31", "url": "https://slack.engineering/flannel-an-application-level-edge-cache-to-make-slack-scale/", "stack": [ "Go", "WebSockets" ], "scale": [], "techniques": [ "edge caching", "lazy loading" ], "summary": "Slack added Flannel, an edge cache that serves user and channel data lazily so huge teams no longer need giant boot payloads.", "verified": "2026-10-15" }, { "id": "slack-real-time-messaging", "title": "Real-time Messaging", "company": "Slack", "date": "2023-04-11", "url": "https://slack.engineering/real-time-messaging/", "stack": [ "Java", "WebSockets" ], "scale": [], "techniques": [ "pub/sub", "consistent hashing", "fan-out" ], "summary": "An overview of Slack's channel and gateway servers that fan messages out to millions of connected clients in real time.", "verified": "2026-10-15" }, { "id": "github-partitioning-github-s-relational-databases-to-handle-scale", "title": "Partitioning GitHub's relational databases to handle scale", "company": "GitHub", "date": "2021-09-27", "url": "https://github.blog/engineering/infrastructure/partitioning-githubs-relational-databases-scale/", "stack": [ "MySQL", "Ruby on Rails", "Vitess" ], "scale": [], "techniques": [ "vertical partitioning", "schema domains", "online migration" ], "summary": "GitHub defined schema domains and linted cross-domain queries so it could move table groups onto separate MySQL clusters.", "verified": "2026-10-15" }, { "id": "github-upgrading-github-com-to-mysql-8-0", "title": "Upgrading GitHub.com to MySQL 8.0", "company": "GitHub", "date": "2023-12-07", "url": "https://github.blog/engineering/infrastructure/upgrading-github-com-to-mysql-8-0/", "stack": [ "MySQL" ], "scale": [ "5.5 million QPS", "300+ TB", "50+ database clusters" ], "techniques": [ "fleet upgrade", "replication", "rollback planning" ], "summary": "How GitHub upgraded its whole MySQL fleet to 8.0 with staged replicas and a working rollback path.", "verified": "2026-10-15" }, { "id": "github-how-we-improved-push-processing-on-github", "title": "How we improved push processing on GitHub", "company": "GitHub", "date": "2024-06-11", "url": "https://github.blog/engineering/architecture-optimization/how-we-improved-push-processing-on-github/", "stack": [ "Ruby on Rails", "Kafka" ], "scale": [], "techniques": [ "queues", "job decomposition" ], "summary": "GitHub broke one huge push-processing job into smaller, independently retried jobs driven by Kafka events.", "verified": "2026-10-15" }, { "id": "shopify-horizontally-scaling-the-rails-backend-of-shop-app-with-vitess", "title": "Horizontally scaling the Rails backend of Shop app with Vitess", "company": "Shopify", "date": "2024-01-17", "url": "https://shopify.engineering/horizontally-scaling-the-rails-backend-of-shop-app-with-vitess", "stack": [ "MySQL", "Vitess", "Ruby on Rails" ], "scale": [], "techniques": [ "horizontal sharding", "database migration" ], "summary": "Shopify's Shop app adopted Vitess to shard its Rails backend, covering keyspace design and the migration plan.", "verified": "2026-10-15" }, { "id": "shopify-shard-balancing-moving-shops-confidently-with-zero-downtime-at-terabyte-", "title": "Shard Balancing: Moving Shops Confidently with Zero-Downtime at Terabyte-scale", "company": "Shopify", "date": "2021-09-24", "url": "https://shopify.engineering/mysql-database-shard-balancing-terabyte-scale", "stack": [ "MySQL", "Ghostferry" ], "scale": [ "terabyte scale" ], "techniques": [ "shard rebalancing", "online migration", "zero downtime" ], "summary": "Shopify rebalances MySQL shards by moving individual shops online with its Ghostferry tool.", "verified": "2026-10-15" }, { "id": "shopify-read-consistency-with-database-replicas", "title": "Read Consistency with Database Replicas", "company": "Shopify", "date": "2021-02-22", "url": "https://shopify.engineering/read-consistency-database-replicas", "stack": [ "MySQL" ], "scale": [], "techniques": [ "read replicas", "consistency" ], "summary": "Shopify routes reads to MySQL replicas while giving apps consistency guarantees such as monotonic reads.", "verified": "2026-10-15" }, { "id": "shopify-capacity-planning-at-scale", "title": "Capacity Planning at Scale", "company": "Shopify", "date": "2020-12-03", "url": "https://shopify.engineering/capacity-planning-shopify", "stack": [ "Kubernetes", "GCP" ], "scale": [], "techniques": [ "capacity planning", "load testing" ], "summary": "How Shopify plans and checks capacity for traffic peaks like BFCM using forecasts and scalability tests.", "verified": "2026-10-15" }, { "id": "meta-zippydb-facebook-s-key-value-store", "title": "ZippyDB: Facebook's key value store", "company": "Meta", "date": "2021-08-06", "url": "https://engineering.fb.com/2021/08/06/core-infra/zippydb/", "stack": [ "RocksDB", "Paxos" ], "scale": [], "techniques": [ "replication", "sharding", "tunable consistency" ], "summary": "Meta's general-purpose distributed key-value store built on RocksDB, with shard management and configurable consistency.", "verified": "2026-10-15" }, { "id": "meta-tao-the-power-of-the-graph", "title": "TAO: The power of the graph", "company": "Meta", "date": "2013-06-25", "url": "https://engineering.fb.com/2013/06/25/core-infra/tao-the-power-of-the-graph/", "stack": [ "MySQL", "Memcache" ], "scale": [ "over 1 billion reads/sec", "millions of writes/sec" ], "techniques": [ "caching", "geo-distribution", "graph data model" ], "summary": "TAO is Facebook's geo-distributed graph cache over sharded MySQL, serving the social graph at more than a billion reads per second.", "verified": "2026-10-15" }, { "id": "dropbox-scaling-to-exabytes-and-beyond", "title": "Scaling to exabytes and beyond", "company": "Dropbox", "date": "", "url": "https://dropbox.tech/infrastructure/magic-pocket-infrastructure", "stack": [ "Custom storage", "Rust", "Go" ], "scale": [ "500 PB" ], "techniques": [ "custom storage", "migration off cloud", "erasure coding" ], "summary": "Dropbox built Magic Pocket, its own exabyte-class block store, and moved user data off third-party cloud storage.", "verified": "2026-10-15" }, { "id": "dropbox-how-we-migrated-dropbox-from-nginx-to-envoy", "title": "How we migrated Dropbox from Nginx to Envoy", "company": "Dropbox", "date": "", "url": "https://dropbox.tech/infrastructure/how-we-migrated-dropbox-from-nginx-to-envoy", "stack": [ "Envoy", "Nginx" ], "scale": [], "techniques": [ "proxy migration", "load balancing" ], "summary": "Dropbox moved its edge traffic proxy from Nginx to Envoy for better observability and extensibility.", "verified": "2026-10-15" }, { "id": "dropbox-rewriting-the-heart-of-our-sync-engine", "title": "Rewriting the heart of our sync engine", "company": "Dropbox", "date": "", "url": "https://dropbox.tech/infrastructure/rewriting-the-heart-of-our-sync-engine", "stack": [ "Rust", "Python" ], "scale": [], "techniques": [ "rewrite", "testing" ], "summary": "Dropbox rewrote its desktop sync engine in Rust with a new data model built for correctness and testability.", "verified": "2026-10-15" }, { "id": "dropbox-how-we-optimized-magic-pocket-for-cold-storage", "title": "How we optimized Magic Pocket for cold storage", "company": "Dropbox", "date": "2019-05-06", "url": "https://dropbox.tech/infrastructure/how-we-optimized-magic-pocket-for-cold-storage", "stack": [ "Custom storage" ], "scale": [], "techniques": [ "tiered storage", "erasure coding" ], "summary": "Dropbox added a cold tier to Magic Pocket that cuts replication overhead for rarely read data.", "verified": "2026-10-15" }, { "id": "stripe-online-migrations-at-scale", "title": "Online migrations at scale", "company": "Stripe", "date": "2017-02-02", "url": "https://stripe.com/blog/online-migrations", "stack": [ "MongoDB", "Ruby" ], "scale": [], "techniques": [ "online migration", "dual writes" ], "summary": "Stripe's four-step dual-write playbook for migrating data models without downtime.", "verified": "2026-10-15" }, { "id": "stripe-how-stripe-s-document-databases-supported-99-999-uptime-with-zero-downtim", "title": "How Stripe's document databases supported 99.999% uptime with zero-downtime data migrations", "company": "Stripe", "date": "2024-06-06", "url": "https://stripe.com/blog/how-stripes-document-databases-supported-99.999-uptime-with-zero-downtime-data-migrations", "stack": [ "MongoDB", "DocDB" ], "scale": [ "1.5 PB", "99.999% uptime" ], "techniques": [ "sharding", "shard migration", "zero downtime" ], "summary": "Stripe's DocDB and its Data Movement Platform move data between shards without downtime.", "verified": "2026-10-15" }, { "id": "stripe-scaling-your-api-with-rate-limiters", "title": "Scaling your API with rate limiters", "company": "Stripe", "date": "2017-03-30", "url": "https://stripe.com/blog/rate-limiters", "stack": [ "Redis" ], "scale": [], "techniques": [ "rate limiting", "load shedding" ], "summary": "The four kinds of rate limiters and load shedders Stripe uses to protect its API.", "verified": "2026-10-15" }, { "id": "linkedin-running-kafka-at-scale", "title": "Running Kafka At Scale", "company": "LinkedIn", "date": "2015-03-20", "url": "https://engineering.linkedin.com/kafka/running-kafka-scale", "stack": [ "Kafka" ], "scale": [ "800 billion messages/day", "175 TB/day" ], "techniques": [ "queues", "streaming", "multi-datacenter" ], "summary": "How LinkedIn runs Kafka as its central data pipeline, including cluster tiers and mirroring between datacenters.", "verified": "2026-10-15" }, { "id": "linkedin-followfeed-linkedin-s-feed-made-faster-and-smarter", "title": "FollowFeed: LinkedIn's Feed Made Faster and Smarter", "company": "LinkedIn", "date": "2016-03-31", "url": "https://engineering.linkedin.com/blog/2016/03/followfeed--linkedin-s-feed-made-faster-and-smarter", "stack": [ "Java", "RocksDB" ], "scale": [], "techniques": [ "fan-out on read", "partitioning", "ranking" ], "summary": "LinkedIn moved its feed to a fan-out-on-read system that keeps member timelines in memory and on RocksDB.", "verified": "2026-10-15" }, { "id": "cloudflare-how-we-built-pingora-the-proxy-that-connects-cloudflare-to-the-intern", "title": "How we built Pingora, the proxy that connects Cloudflare to the Internet", "company": "Cloudflare", "date": "2022-09-14", "url": "https://blog.cloudflare.com/how-we-built-pingora-the-proxy-that-connects-cloudflare-to-the-internet/", "stack": [ "Rust", "Nginx" ], "scale": [ "1 trillion requests/day" ], "techniques": [ "rewrite", "connection reuse" ], "summary": "Cloudflare replaced Nginx with Pingora, a Rust proxy that reuses connections better and needs less CPU and memory.", "verified": "2026-10-15" }, { "id": "cloudflare-http-analytics-for-6m-requests-per-second-using-clickhouse", "title": "HTTP Analytics for 6M requests per second using ClickHouse", "company": "Cloudflare", "date": "2018-03-06", "url": "https://blog.cloudflare.com/http-analytics-for-6m-requests-per-second-using-clickhouse/", "stack": [ "ClickHouse", "Kafka", "Go" ], "scale": [ "6M requests/sec" ], "techniques": [ "columnar storage", "aggregation", "pipeline redesign" ], "summary": "Cloudflare rebuilt its HTTP analytics pipeline on ClickHouse, replacing a Postgres/Citus setup.", "verified": "2026-10-15" }, { "id": "cloudflare-log-analytics-using-clickhouse", "title": "Log analytics using ClickHouse", "company": "Cloudflare", "date": "2022-09-02", "url": "https://blog.cloudflare.com/log-analytics-using-clickhouse/", "stack": [ "ClickHouse" ], "scale": [], "techniques": [ "columnar storage", "schema design" ], "summary": "Cloudflare moved its error-log pipeline to ClickHouse and shares schema and query lessons.", "verified": "2026-10-15" }, { "id": "cloudflare-introducing-quicksilver-configuration-distribution-at-internet-scale", "title": "Introducing Quicksilver: Configuration Distribution at Internet Scale", "company": "Cloudflare", "date": "2020-03-30", "url": "https://blog.cloudflare.com/introducing-quicksilver-configuration-distribution-at-internet-scale/", "stack": [ "LMDB", "Go" ], "scale": [ "14 million HTTP requests/sec (network peak)" ], "techniques": [ "replication", "config distribution" ], "summary": "Quicksilver replicates Cloudflare's config key-value store to every edge server in seconds, built on LMDB.", "verified": "2026-10-15" }, { "id": "segment-twilio-goodbye-microservices", "title": "Goodbye Microservices", "company": "Segment (Twilio)", "date": "2018-07-09", "url": "https://segment.com/blog/goodbye-microservices/", "stack": [ "Node.js" ], "scale": [], "techniques": [ "monolith consolidation" ], "summary": "Segment folded more than 100 destination microservices back into a monolith to cut operational overhead.", "verified": "2026-10-15" }, { "id": "datadog-introducing-husky-datadog-s-third-generation-event-store", "title": "Introducing Husky, Datadog's third-generation event store", "company": "Datadog", "date": "2022-05-17", "url": "https://www.datadoghq.com/blog/engineering/introducing-husky/", "stack": [ "Go", "Object storage" ], "scale": [], "techniques": [ "columnar storage", "storage/compute separation", "multi-tenancy" ], "summary": "Datadog's Husky is a distributed, schemaless column store that keeps event data on object storage.", "verified": "2026-10-15" }, { "id": "datadog-husky-exactly-once-ingestion-and-multi-tenancy-at-scale", "title": "Husky: Exactly-once ingestion and multi-tenancy at scale", "company": "Datadog", "date": "2023-02-22", "url": "https://www.datadoghq.com/blog/engineering/husky-deep-dive/", "stack": [ "Go", "Kafka", "FoundationDB" ], "scale": [], "techniques": [ "exactly-once ingestion", "sharding", "multi-tenancy" ], "summary": "A deep dive on how Husky routes, shards and ingests events exactly once across tenants.", "verified": "2026-10-15" }, { "id": "datadog-timeseries-indexing-at-scale", "title": "Timeseries indexing at scale", "company": "Datadog", "date": "2024-06-28", "url": "https://www.datadoghq.com/blog/engineering/timeseries-indexing-at-scale/", "stack": [ "Rust", "RocksDB" ], "scale": [], "techniques": [ "indexing", "rewrite" ], "summary": "Datadog redesigned its timeseries indexing to handle a large jump in ingested data.", "verified": "2026-10-15" }, { "id": "atlassian-migrating-the-jira-database-platform-to-aws-aurora", "title": "Migrating the Jira Database Platform to AWS Aurora", "company": "Atlassian", "date": "2025-07-02", "url": "https://www.atlassian.com/blog/atlassian-engineering/migrating-jira-database-platform-to-aws-aurora", "stack": [ "PostgreSQL", "AWS Aurora", "AWS RDS" ], "scale": [ "4 million databases" ], "techniques": [ "database migration", "multi-tenancy", "cost optimization" ], "summary": "Atlassian migrated about four million Jira databases from RDS Postgres to Aurora, balancing reliability, downtime and cost.", "verified": "2026-10-15" }, { "id": "gitlab-we-are-splitting-our-database-into-main-and-ci", "title": "We are splitting our database into Main and CI", "company": "GitLab", "date": "2022-06-02", "url": "https://about.gitlab.com/blog/2022/06/02/splitting-database-into-main-and-ci/", "stack": [ "PostgreSQL", "Ruby on Rails" ], "scale": [], "techniques": [ "vertical partitioning", "decomposition" ], "summary": "GitLab.com split CI tables into their own Postgres database to relieve its main database.", "verified": "2026-10-15" }, { "id": "gitlab-why-we-spent-the-last-month-eliminating-postgresql-subtransactions", "title": "Why we spent the last month eliminating PostgreSQL subtransactions", "company": "GitLab", "date": "2021-09-29", "url": "https://about.gitlab.com/blog/2021/09/29/why-we-spent-the-last-month-eliminating-postgresql-subtransactions/", "stack": [ "PostgreSQL", "Ruby on Rails" ], "scale": [], "techniques": [ "query optimization", "read replicas" ], "summary": "GitLab traced replica stalls to Postgres subtransaction overflow and removed subtransactions from its code.", "verified": "2026-10-15" }, { "id": "planetscale-one-million-queries-per-second-with-mysql", "title": "One million queries per second with MySQL", "company": "PlanetScale", "date": "2022-09-01", "url": "https://planetscale.com/blog/one-million-queries-per-second-with-mysql", "stack": [ "MySQL", "Vitess" ], "scale": [ "1 million QPS" ], "techniques": [ "horizontal sharding", "benchmarking" ], "summary": "PlanetScale benchmarked a horizontally sharded Vitess cluster up to one million QPS.", "verified": "2026-10-15" }, { "id": "supabase-supavisor-scaling-postgres-to-1-million-connections", "title": "Supavisor: Scaling Postgres to 1 Million Connections", "company": "Supabase", "date": "2023-08-11", "url": "https://supabase.com/blog/supavisor-1-million", "stack": [ "PostgreSQL", "Elixir" ], "scale": [ "1 million connections", "20,000 QPS" ], "techniques": [ "connection pooling", "benchmarking" ], "summary": "Supabase load-tested Supavisor, its Elixir Postgres pooler, with a million client connections.", "verified": "2026-10-15" }, { "id": "tailscale-switching-from-etcd-to-sqlite-a-database-migration-story", "title": "Switching from etcd to SQLite: A Database Migration Story", "company": "Tailscale", "date": "2022-04-01", "url": "https://tailscale.com/blog/database-for-2022", "stack": [ "SQLite", "Litestream", "etcd" ], "scale": [], "techniques": [ "simplification", "database migration", "replication" ], "summary": "Tailscale replaced etcd with SQLite plus Litestream replication for a simpler control-plane datastore.", "verified": "2026-10-15" }, { "id": "stack-overflow-stack-overflow-the-architecture-2016-edition", "title": "Stack Overflow: The Architecture - 2016 Edition", "company": "Stack Overflow", "date": "2016-02-17", "url": "https://stackoverflow.blog/2016/02/17/stack-overflow-the-architecture-2016-edition/", "stack": [ "SQL Server", ".NET", "Redis", "Elasticsearch" ], "scale": [], "techniques": [ "vertical scaling", "caching" ], "summary": "A tour of how Stack Overflow serves a huge site from a small number of powerful servers with heavy caching.", "verified": "2026-10-15" }, { "id": "meta-how-meta-trains-large-language-models-at-scale", "title": "How Meta trains large language models at scale", "company": "Meta", "date": "2024-06-12", "url": "https://engineering.fb.com/2024/06/12/data-infrastructure/training-large-language-models-at-scale-meta/", "stack": [ "GPUs", "PyTorch" ], "scale": [], "techniques": [ "distributed training", "hardware reliability", "networking" ], "summary": "Meta covers the hardware, networking and reliability work needed to train LLMs across very large GPU clusters.", "verified": "2026-10-15" }, { "id": "openai-rapidly-scaling-online-storage-to-serve-over-1-billion-chatgpt-users-part", "title": "Rapidly scaling online storage to serve over 1 billion ChatGPT users (Part II)", "company": "OpenAI", "date": "2026-10-23", "url": "https://openai.com/index/scaling-storage-one-billion-users-part-two", "stack": [ "Azure Cosmos DB", "Valkey", "Kafka", "Rust", "Python" ], "scale": [ "1B+ users across 40 regions", "500+ PB served by Habitat", "largest Cosmos DB account: 10s of PB, 20M+ QPS", "global read p50 ~120 ms to ~10 ms (12x)", "millions of CDC changes/sec" ], "techniques": [ "multi-tenancy", "hot key throttling", "CDC cache invalidation", "geo-replication", "Rust rewrite" ], "summary": "OpenAI's Habitat storage platform went from one shared Azure Cosmos DB container, to dedicated single-tenant containers that pushed capacity planning onto every product team, and back to a shared container by default once Azure added hot-key throttling, request-priority QoS and a new serverless mode where each physical partition scales to 10K RU/s and splits on RU usage. After failover and failback outages in late 2024, OpenAI and Azure now co-own testing and monitoring. Replicating small latency-critical datasets widely, a Valkey lookaside cache invalidated through the Cosmos DB change feed, and a Rust rewrite of the proxy cut global read p50 from about 120 ms to about 10 ms.", "verified": "2026-10-26" }, { "id": "datadog-scaling-real-time-file-monitoring-with-ebpf-how-we-filtered-billions-of-", "title": "Scaling real-time file monitoring with eBPF: How we filtered billions of kernel events per minute", "company": "Datadog", "date": "2025-11-18", "url": "https://www.datadoghq.com/blog/engineering/workload-protection-ebpf-fim/", "stack": [ "eBPF", "Linux", "Datadog Agent" ], "scale": [ "10B+ file events/min across Datadog's infrastructure", "~5 KB per serialized event", "up to 94% of events filtered in-kernel", "~1M events/min sent upstream" ], "techniques": [ "in-kernel filtering", "edge filtering", "approvers and discarders", "LRU maps" ], "summary": "Datadog's eBPF-based file integrity monitoring sees more than 10 billion file events per minute, and at about 5 KB each, shipping them all would mean terabytes per second. Agent-side rules cut what goes upstream to roughly one million events per minute, and moving evaluation into the kernel stops the ring buffer from overflowing: static approvers compiled from detection rules let matching events through, while runtime discarders stored in LRU eBPF maps drop values no rule can match. Together they pre-filter up to 94% of events in the kernel with no dropped events.", "verified": "2026-10-26" }, { "id": "datadog-breaking-up-a-monolith-how-we-re-unwinding-a-shared-database-at-scale", "title": "Breaking up a monolith: How we're unwinding a shared database at scale", "company": "Datadog", "date": "2025-06-17", "url": "https://www.datadoghq.com/blog/engineering/unwinding-shared-database/", "stack": [ "PostgreSQL", "PgBouncer", "PG Proxy", "OrgStore" ], "scale": [ "30+ teams consulted", "~30 domain schemas created", "326 to 257 tables (Dec 2023 to Apr 2025)", "70+ dead tables removed", "<5 s downtime at write cutover" ], "techniques": [ "database decomposition", "logical separation", "proxy-based cutover", "migration automation" ], "summary": "Datadog is splitting its large shared Postgres database into team-owned instances in three phases. Logical separation moves tables into about 30 private domain schemas with explicit ACLs, using query samples and interviews with 30+ teams. Access reduction routes services through per-team PG Proxies so ACLs can be narrowed safely, and physical separation replicates each schema to a managed OrgStore instance, moves reads first and then writes with under five seconds of downtime, and replicates back for rollback. Two workloads have fully migrated and more than 70 dead tables were deleted along the way.", "verified": "2026-10-26" }, { "id": "shopify-cross-store-product-clustering-across-millions-of-stores", "title": "Cross-store product clustering across millions of stores", "company": "Shopify", "date": "2026-10-20", "url": "https://shopify.engineering/catalog-clustering-part-ii", "stack": [ "GBDT", "Multilingual E5 cross-encoder", "LLM judge", "GPUs", "Vector ANN search" ], "scale": [ "billions of product listings across millions of stores", "trillions of candidate pairs scored by L1", "clusters rebuilt daily" ], "techniques": [ "embedding retrieval", "tiered model cascade", "bounded union-find graph partitioning", "hierarchical agglomerative clustering", "LLM-labeled training data", "score caching" ], "summary": "Shopify groups the same real-world product sold by different merchants under one Universal Product Identifier. Embedding nearest-neighbor search finds plausible pairs across billions of listings, then three scorers of rising cost judge them: a CPU GBDT (L1) over trillions of pairs, a fine-tuned multilingual E5 cross-encoder (L2), and an image-aware reranker (L3) fed from prioritized GPU queues. A bounded union-find splits the graph into single-machine components, and HAC with a learned GBDT merge rule builds high-precision clusters daily, with an LLM judge validated against human labels supplying training data and cached scores reused between builds.", "verified": "2026-10-27" }, { "id": "planetscale-118-million-queries-per-second-on-neki", "title": "118 million queries per second on Neki", "company": "PlanetScale", "date": "2026-09-11", "url": "https://planetscale.com/blog/118-million-queries-per-second-on-neki", "stack": [ "PostgreSQL", "Neki", "AWS r8g.16xlarge" ], "scale": [ "118.5M QPS sustained for 16 minutes", "512 shards, 480 routers", "1.22 PiB of data", "p99 6.06 ms at router, 13.95 ms at client", "15.8M read IOPS, 2+ Tb/s network" ], "techniques": [ "horizontal sharding", "benchmarking", "linear scalability" ], "summary": "PlanetScale benchmarked Neki, its sharded Postgres, with single-shard primary-key point selects. Targeting about 200K QPS per shard, it went from 5 shards (about 1M QPS) to 50 (9.9M QPS, per-shard rate within 0.8%) to 512 shards and 480 routers, sustaining 118.5M QPS for 16 minutes over 1.22 PiB with a p99 of 6.06 ms at the router and about one error per 1.8 million queries. PlanetScale notes the run was read-only, primary-only with no replicas, and had no failover during the measured window.", "verified": "2026-10-27" }, { "id": "netflix-ontology-driven-observability-building-an-end-to-end-knowledge-graph-at-", "title": "Ontology-Driven Observability: Building an End-to-End Knowledge Graph at Netflix Scale", "company": "Netflix", "date": "2026-10-16", "url": "https://netflixtechblog.com/ontology-driven-observability-building-an-end-to-end-knowledge-graph-at-netflix-scale-421392cf5adf", "stack": [ "RDF", "OWL", "SHACL", "SPARQL", "QuipuDB", "redb", "Rust", "LLMs" ], "scale": [ "~65M concurrent streams (Paul vs. Tyson, Nov 2024)", "2B+ requests/day", "38M+ real-time logging events/sec", "~49,000-entry software catalog", "13 signal types in the correlation engine" ], "techniques": [ "knowledge graphs", "observability", "root cause analysis", "AI agents" ], "summary": "Netflix found a typical regression investigation took about four hours, mostly spent reconstructing relationships between systems like Atlas, Radar, Spinnaker, PagerDuty, incident.io and Slack that each named the same thing differently. It built an Observability Ontology on the W3C stack (RDF, OWL, SHACL, SPARQL), stored as a knowledge graph in QuipuDB, its homegrown distributed quad store built on the Rust key-value store redb. Enrichers, some LLM-based, turn noisy inputs into typed triples, every application is validated against a catalog of about 49,000 entries, and dependency edges are capped. A correlation engine scores chains across 13 signal types within a ten-minute window, discarding anything under 0.6 confidence, and an agentic AutoRCA system reads the graph to hand engineers ranked, evidence-backed hypotheses.", "verified": "2026-10-28" }, { "id": "cloudflare-origin-range-requests-serving-large-files-without-range-request-tunin", "title": "Origin Range Requests: Serving large files without range-request tuning", "company": "Cloudflare", "date": "2026-10-23", "url": "https://blog.cloudflare.com/origin-range-requests", "stack": [ "Pingora", "R2", "Tiered Cache" ], "scale": [ "R2 p99 time to first byte roughly halved (hits and misses)", "~6% less tiered cache traffic for R2-like workloads, petabytes/day", "old configured range sizes from tens of KB to tens of GB", "R2 previously used 64 MiB ranges (up to 128 MiB per small read)" ], "techniques": [ "caching", "request coalescing", "tiered storage" ], "summary": "Cloudflare's old range-request caching sliced files into a per-zone configured range size, so a small video seek could pull 128 MiB from origin for R2, while small ranges caused floods of tiny origin requests and memory pressure, and concurrent requests in the same range blocked each other. Moving from nginx to its Pingora-based cache, it reused Pingora's fixed-size on-disk chunks as the unit: a coordinator serves cached bytes, attaches to in-flight downloads, fetches only missing chunk-aligned ranges from origin, and validates size and ETag before composing one 206 response. For R2, p99 time to first byte dropped by about half, and letting one response combine disk and upstream bytes cut tiered cache traffic about 6% week over week.", "verified": "2026-10-28" }, { "id": "meta-catching-bugs-before-they-land-jittesting-s-human-on-the-loop-approach-at-f", "title": "Catching Bugs Before They Land: JiTTesting's Human-on-the-Loop Approach at Full Scale", "company": "Meta", "date": "2026-10-29", "url": "https://engineering.fb.com/2026/10/29/security/catching-bugs-before-they-land-jittestings-human-on-the-loop-approach-at-full-scale/", "stack": [ "LLMs", "LLM-as-Judge ensemble", "mutation testing", "RubFake" ], "scale": [ "hundreds of millions of lines of code", "589 engineers engaged (64% of outreach)", "60% true-positive rate", "74 SEV privacy bugs caught at diff stage in H1 2026", "privacy SEVs down 20%" ], "techniques": [ "LLM test generation", "mutation testing", "false-positive filtering", "human-on-the-loop" ], "summary": "Meta's Catching Just-in-Time Tests (JiTTests) are now production infrastructure. Overnight, on spare capacity, the system picks risky diffs from the previous day and generates tests designed to fail: a \"dodgy diff\" workflow treats each change as a mutant, while an intent-aware workflow infers what the diff is meant to do, lists the risks, and builds mutants from them. An ensemble of LLM judges plus a deterministic rule-based assessor (RubFake) ranks the failing \"weak catches\", and engineers are only contacted for the strongest ones. 589 engineers have engaged, reporting 60% true positives, and in the first half of 2026 it helped catch 74 serious privacy bugs before landing, cutting privacy SEVs by 20%.", "verified": "2026-10-29" }, { "id": "planetscale-shard-design-doesn-t-start-from-your-tables", "title": "Shard design doesn't start from your tables", "company": "PlanetScale", "date": "2026-10-23", "url": "https://planetscale.com/blog/shard-groups-the-unit-of-co-location", "stack": [ "PostgreSQL", "Neki", "xxhash" ], "scale": [ "64-bit hashed keyspace split across shards", "cross-shard transactions commit per shard, no atomic commit" ], "techniques": [ "shard groups", "co-location", "reference tables", "global secondary indexes", "resharding" ], "summary": "In Neki, PlanetScale's sharded Postgres, a shard group is the unit of co-location: tables in the same group that hash their shard key the same way (xxhash into a 64-bit keyspace split by key ranges) always land on the same shard. Sharding customers, orders, items and addresses by customer_id keeps single-customer queries and transactions on one shard, which matters because a transaction across two shards gets no atomic commit or shared snapshot. Lookup tables become reference tables copied to every shard, catalog tables get their own group keyed on lower(sku), range and modulo indexes support legacy schemes, and a global secondary index routes queries like orders by id without fanning out. Shard groups can be changed later with the Reshard workflow.", "verified": "2026-10-29" }, { "id": "planetscale-designing-a-planned-switchover-for-minimum-disruption", "title": "Designing a planned switchover for minimum disruption", "company": "PlanetScale", "date": "2026-10-29", "url": "https://planetscale.com/blog/designing-a-planned-switchover-for-minimum-disruption", "stack": [ "PostgreSQL", "Neki", "Kubernetes" ], "scale": [ "one primary and multiple replicas per shard", "synchronous replication ANY 1 across eligible replicas" ], "techniques": [ "planned switchover", "query buffering", "synchronous replication", "preflight validation", "rollback" ], "summary": "When a Neki shard's primary has to step aside for a pod replacement, kernel upgrade or cell move, PlanetScale does a planned switchover instead of stopping Postgres the way Patroni does, so a failed switchover can quickly fall back to the old primary. Admin, Neki's control plane, first validates shard state, the target replica, logical failover slots and Postgres settings. Routers then proactively buffer primary-bound queries, the primary defaults new transactions to read-only, and the other replicas are disconnected to pin a final WAL position the target must replay. If the target can't catch up before a deadline, a config reload makes the old primary writable again. Buffered queries are replanned against the new primary, so apps see added latency instead of errors, and every acknowledged commit is on the target when it is promoted.", "verified": "2026-10-30" }, { "id": "gitlab-scaling-git-for-the-agentic-software-factory", "title": "Scaling Git for the agentic software factory", "company": "GitLab", "date": "2026-10-13", "url": "https://about.gitlab.com/blog/scaling-git-for-the-agentic-software-factory", "stack": [ "Git", "Object storage", "S3-compatible storage" ], "scale": [ "single agent invocation can need 5-10 GB of transfer and 30+ s of setup", "up to 50x faster, 1,000x less network traffic in early internal tests", "40% more CI/CD pipelines and 50% more code pushes on GitLab.com in a year" ], "techniques": [ "stateless compute", "storage/compute separation", "partial fetch", "agent-native interfaces" ], "summary": "GitLab is building next-gen source control for thousands of concurrent AI agents working on large repositories. With conventional Git, a single agent invocation can mean 5 to 10 GB of data transfer and more than 30 seconds of setup just to answer one question. Next-gen SCM keeps the Git protocol but runs stateless compute that scales horizontally against shared object storage, including S3-compatible storage customers control, and lets agents request only the files and history they need instead of cloning whole repos. In early internal tests, workloads ran up to 50x faster with up to 1,000x less network traffic, which GitLab calls peak results it is still validating. It also notes CI is the next bottleneck: GitLab.com customers created 40% more pipelines and pushed 50% more code over the past year.", "verified": "2026-10-30" }, { "id": "netflix-single-tenant-titus-vm-level-isolation-on-a-multi-tenant-container-platf", "title": "Single-tenant Titus: VM-level isolation on a multi-tenant container platform", "company": "Netflix", "date": "2026-10-22", "url": "https://netflixtechblog.com/single-tenant-titus-vm-level-isolation-on-a-multi-tenant-container-platform-6d8b1909d163", "stack": [ "Titus", "Kubernetes", "AWS EC2", "cgroups", "Flink" ], "scale": [ "millions of containers launched per week", "tens of thousands of EC2 instances across multiple AWS regions", "fixed host overhead: tens of percent of a small dedicated node, ~2% of the largest" ], "techniques": [ "workload isolation", "multi-tenancy", "autoscaling", "right-sizing", "bin-packing" ], "summary": "Titus, Netflix's Kubernetes-based container platform, launches millions of containers a week on tens of thousands of EC2 instances, packing many teams onto shared hosts. cgroups can't fence last-level cache, memory bandwidth or the global lock in HTB network shaping, so a small-packet flood from one container could starve its neighbors, and stateful streaming jobs like Flink would fall behind and need minutes to catch up. After ruling out microVMs (Firecracker, Kata), gVisor and moving teams to plain EC2, Netflix added an allowlisted single-tenant mode: the federation layer tags eligible tasks, the autoscaler launches the smallest instance that fits each one just in time, and anti-affinity, taints, no preemption and no repacking keep it alone on its host, all behind the same Titus API. Flink, streaming connectors and telemetry pipelines now report stable operation, and the shared fleet packs more densely because sensitive workloads no longer need anti-affinity spreading. The cost is slower launches, since every task waits for its own instance, and fixed host overhead that can take tens of percent of a small node.", "verified": "2026-11-02" }, { "id": "netflix-behind-the-scenes-evolving-netflix-s-ads-event-pipeline-for-live-part-ii", "title": "Behind the Scenes: Evolving Netflix's Ads Event Pipeline for Live, Part II", "company": "Netflix", "date": "2026-10-14", "url": "https://netflixtechblog.com/behind-the-scenes-evolving-netflixs-ads-event-pipeline-for-live-part-ii-826ebf9ad9fb", "stack": [ "Flink", "Kafka", "Spark", "Maestro" ], "scale": [ "hundreds of thousands of events/sec for one ad break in a single AWS region (load test)", "~99.999% of events joined within a 60-minute window", "old and new pipelines agreed on well over 99.99% of events before cutover", "25,000+ Flink jobs run at Netflix" ], "techniques": [ "streaming joins", "deduplication", "shadow traffic", "staged rollout", "zero downtime" ], "summary": "Netflix's Ads Event Publisher used to enrich each client ad event by looking up metadata the ad server had written to a globally replicated key-value registry. For live events with tens of millions of concurrent viewers, that meant must-succeed database writes on the ad-serving critical path, a store that couldn't safely be scaled mid-event, and late events whenever lookups slowed. Netflix replaced it with an Apache Flink job: the ad server logs its responses to Kafka, and a custom KeyedCoProcessFunction joins them to client events over a 60-minute window, which catches roughly 99.999% of events. An hourly Spark job orchestrated by Maestro recovers the rest from the data warehouse. Revenue-bearing events are de-duplicated in local Flink state keyed by a hashed event ID. Both pipelines ran side by side with automated audits, and a stable per-event key moved publishing to Flink a slice at a time, region by region. When they agreed on well over 99.99% of events, Netflix retired the legacy service and the registry. In a load test, one AWS region handled an ad break at hundreds of thousands of events per second.", "verified": "2026-11-02" }, { "id": "cloudflare-inside-hyperdrive-how-we-solved-a-rare-hyperdrive-mysql-caching-bug", "title": "Inside Hyperdrive: How we solved a rare Hyperdrive MySQL caching bug", "company": "Cloudflare", "date": "2026-10-14", "url": "https://blog.cloudflare.com/hyperdrive-mysql", "stack": [ "MySQL", "Workers", "Hyperdrive", "mysql2" ], "scale": [ "99.99% production availability target", "bug hit ~0.001% of queries", "Client connection success rate reached 99.99% after the fix" ], "techniques": [ "connection pooling", "caching", "protocol translation" ], "summary": "Hyperdrive, Cloudflare's distributed connection pooler for Workers, keeps warm database connections on its network and can cache common query results in the CDN cache. Taking its MySQL support from beta to GA meant finding bugs like utf8 being a 3-byte alias (fixed with utf8mb4) and COM_RESET_CONNECTION silently resetting the charset to the server default. The hardest was an empty-result bug seen by the highest-volume MySQL customer in about 0.001% of queries, only with caching on. Each Hyperdrive instance has a Client node next to the Worker and an Endpoint node near the database, and they share a cache when they run in the same data center. Because drivers like Node's mysql2 don't support CLIENT_DEPRECATE_EOF, the Client rewrites results into the Worker's capabilities, so the two nodes were caching different bytes for the same query and could read each other's entries. Caching one canonical encoding no matter which node stores it fixed the bug, and Client connection success reached 99.99%.", "verified": "2026-11-02" }, { "id": "openai-scaling-git-at-openai", "title": "Scaling Git at OpenAI", "company": "OpenAI", "date": "2026-11-02", "url": "https://openai.com/index/scaling-git-at-openai", "stack": [ "Git", "Pando", "Rust", "blob storage" ], "scale": [ "nearly a petabyte of Git data/day", "median push latency cut nearly 5x (~80%)", "57+ TB of Git packs in a single hour", "packfile-uri cache hit rates over 99%", "saturates 10 Gbps clone/fetch links on standard VMs" ], "techniques": [ "per-ref causal consistency", "reusable packfiles", "storage/compute separation", "LSM-style compaction", "zero downtime migration" ], "summary": "OpenAI built Pando, a Git-compatible backend for its monorepo after Codex pushed concurrent branches, merges and fetches past the old system (reliability sometimes below 90%). Pando stores each push as an immutable pack in blob storage and records per-branch ref versions in a horizontally scalable metadata store, giving serializability per ref and causal consistency across the repo so unrelated branches do not contend. Instead of building unique packs per client, it reuses stored packs and serves them via packfile-uri through regional Express caches (over 99% hit rate), with LSM-style background compaction so fetches plan in logarithmic time. Deployed first as a read/write-through proxy, then cut over with a Git CLI wrapper; it now powers all monorepo development, cut median push latency by nearly 5x, and can deliver nearly a petabyte of Git data per day (57+ TB in a single hour).", "verified": "2026-11-03" }, { "id": "zalando-from-homegrown-to-flink-migrating-a-stateful-ad-event-join-at-scale", "title": "From Homegrown to Flink: Migrating a Stateful Ad Event Join at Scale", "company": "Zalando", "date": "2026-07-24", "url": "https://engineering.zalando.com/posts/2026/07/migrating-ad-event-processing-to-flink.html", "stack": [ "Apache Flink", "RocksDB", "Kubernetes", "Nakadi", "Kinesis", "Java" ], "scale": [ "up to 200 MB/s on the old pipeline", "average pods cut from a constant 20 to ~5 (range 1-8)", "join-operator subtasks 30-60 down to 10 max", "memory ~320 GB to ~100 GB; EC2 ~\u20ac80 to ~\u20ac30/day", "shadow match rate 99.9% vs production; +0.5% enrichment from two-way state" ], "techniques": [ "stream joins", "checkpointing", "keyed state", "shadow traffic", "autoscaling" ], "summary": "Zalando Marketing Services replaced a 7-year-old homegrown in-memory join that matched Sponsored Products auction bids to clicks/views within 15 minutes. The old Java service read up to 200 MB/s, could not partition both streams cleanly, and lost up to 15 minutes of state on every deploy or rescale. After Interval Join burned 15-30% CPU on RocksDB seeks and held a ~70M-entry timer queue, they rewrote the match with KeyedCoProcessFunction (point lookups, RocksDB TTL expiry), incremental 3-minute checkpoints, and two-way state. A four-week shadow pipeline hit 99.9% enrichment parity, then a one-week A/B test; average pods fell from 20 to about 5, memory from ~320 GB to ~100 GB, and EC2 cost from about \u20ac80 to \u20ac30 per day, with enrichment up ~0.5%.", "verified": "2026-11-03" }, { "id": "datadog-when-upserts-don-t-update-but-still-write-debugging-postgres-performance", "title": "When upserts don't update but still write: Debugging Postgres performance at scale", "company": "Datadog", "date": "2026-03-23", "url": "https://www.datadoghq.com/blog/engineering/debugging-postgres-performance/", "stack": [ "PostgreSQL", "pg_walinspect" ], "scale": [ "25,000 upserts/sec target in largest DCs", "500 upserts/sec doubled write IOPS and ~4x WAL syncs in a small DC rollout", "gp3 EBS ~1,000 WAL syncs/sec ceiling via pg_test_fsync" ], "techniques": [ "query optimization", "HOT updates", "WAL inspection", "write amplification" ], "summary": "Datadog tracks last-seen time for millions of ephemeral hosts so inactive ones can be deleted after 7 days. An ON CONFLICT DO UPDATE upsert meant to touch each host at most once a day still locked the conflicting row and wrote Heap LOCK plus COMMIT WAL records even when the WHERE clause skipped the update\u2014at 500 upserts/sec that roughly doubled write IOPS and quadrupled WAL syncs, and at 25,000/sec it would overwhelm a single writer. Using pg_walinspect and a debugger on heap_lock_tuple, they replaced it with an ON CONFLICT DO NOTHING CTE plus a conditional UPDATE that generates no WAL on true no-ops, keeps HOT updates when a day has passed, and preserves correctness for coarse host tracking.", "verified": "2026-11-03" }, { "id": "meta-f3-lightning-how-meta-accelerates-ads-ml-features-to-production-at-scale", "title": "F3 Lightning: How Meta Accelerates Ads ML Features to Production at Scale", "company": "Meta", "date": "2026-11-03", "url": "https://engineering.fb.com/2026/11/03/ml-applications/f3-lightning-how-meta-accelerates-ads-ml-features-to-production/", "stack": [ "F3", "DAG feature graphs", "Ads ML" ], "scale": [ "20,000+ features per ads ranking request", "~3.6x faster feature releases to production", "no increase in infrastructure compute cost" ], "techniques": [ "per-feature release", "shared execution plan", "stable computation identifiers", "shared memory buffer", "dense index mapping" ], "summary": "Meta's Facebook Feature Framework (F3) compiles ads ranking features into one merged DAG so shared work runs once, but that graph was also the release unit, so one bad definition blocked every other feature as the catalog grew past 20,000 features per request. F3 Lightning makes each feature an independently compiled, validated, deployed, and rolled-back configuration, then rebuilds shared execution at service startup: stable identifiers derived from each computation's meaning, one flattened execution plan that deduplicates common nodes, a single per-request memory buffer, and dense integer slot mapping so request-path reads are direct array indexes. Comparable extraction features now reach production about 3.6x faster than traditional F3 with no increase in infrastructure compute cost.", "verified": "2026-11-04" }, { "id": "datadog-evolving-our-real-time-timeseries-storage-again-built-in-rust-for-perfor", "title": "Evolving our real-time timeseries storage again: Built in Rust for performance at scale", "company": "Datadog", "date": "2025-08-04", "url": "https://www.datadoghq.com/blog/engineering/rust-timeseries-engine/", "stack": [ "Rust", "Tokio", "LSM tree", "Kafka", "DDSketch", "Monocle" ], "scale": [ "60x increase in ingestion performance", "5x faster queries at peak scale", "extreme query outliers from minutes to seconds" ], "techniques": [ "shard-per-core", "LSM tree", "tiered compaction", "series cache", "query throttling" ], "summary": "Datadog rebuilt its real-time timeseries database (RTDB) storage engine in Rust as Monocle after outgrowing Cassandra, Redis, MDBM, and a Go B+ tree plus a separate RocksDB path for DDSketch distributions. The design shards the keyspace early with a shard-per-core model (for example, 24 storage shards on a 32-core machine), ingesting from Kafka partitions over MPSC channels into per-shard LSM trees on single-threaded Tokio schedulers, with tiered compaction, a unified series cache, and permit- and cost-based query throttlers (including CoDel). At peak scale the rewrite delivered a 60x increase in ingestion performance and 5x faster queries, turning extreme outliers that used to take minutes into results that return in seconds.", "verified": "2026-11-04" }, { "id": "grab-agent-platform-part-1-how-we-help-grab-build-and-run-ai-agents-at-scale", "title": "Agent platform (Part 1): How we help Grab build and run AI agents at scale", "company": "Grab", "date": "2026-07-24", "url": "https://engineering.grab.com/how-grab-builds-and-runs-ai-agents-at-scale", "stack": [ "LLM-Kit", "GrabGPT Gateway", "Model Context Protocol", "Go" ], "scale": [ "500+ services on the internal agent framework", "50+ remote MCP servers", "billions of tokens/month through one LLM gateway", "millions of merchants, drivers, and consumers" ], "techniques": [ "agent framework", "LLM gateway", "remote MCP", "tool plane" ], "summary": "Grab's internal AI agent work started as a Tech Infra support bot and grew into a company-wide platform: more than 500 services run on its agent framework (LLM-Kit), over 50 Model Context Protocol servers are registered on its remote MCP framework, and a single LLM gateway fronts every model call across the company, handling billions of tokens each month for production agents used daily by millions of merchants, drivers, and consumers. Part 1 covers how the original Go bot's reasoning plane and tool plane became reusable framework building blocks, and how problems that appeared around 500 agents\u2014shared model control, safe tool reuse across teams, and evals after prompt changes\u2014pushed the team toward the GrabGPT Gateway, remote MCP, and an evals platform covered in later parts.", "verified": "2026-11-04" }, { "id": "figma-how-figma-manages-sandboxes-at-scale", "title": "How Figma manages sandboxes at scale", "company": "Figma", "date": "2026-11-04", "url": "https://www.figma.com/blog/how-figma-manages-sandboxes-at-scale", "stack": [ "Nimbus", "Postgres", "Redis", "multi-cloud" ], "scale": [ "tens of thousands of sandboxes across products", "warm pools cut provisioning latency ~90%", "provisioning availability 99% to 99.95%", "target: provisioned sandbox in under 1 second" ], "techniques": [ "warm pools", "multi-provider failover", "network policy isolation", "control plane" ], "summary": "Figma's AI products needed secure places to run untrusted code, so the team built Nimbus, a centralized sandbox platform that product teams call through a simple namespace config and GetOrCreate/Get/Delete APIs. Nimbus picks a cloud provider, applies networking and security policies (trusted-host allowlists plus read-only GET/HEAD/OPTIONS to the rest of the internet, credentials decrypted only on the way out to their intended destination), and runs sandboxd inside each box for process supervision and metrics. Warm pools of already-running sandboxes cut provisioning latency by about 90% versus cold starts, raised availability from 99% to 99.95%, and absorb minutes of provider outages while Postgres claim transactions and a Redis health-check queue keep the pool correctly sized and singly assigned.", "verified": "2026-11-05" }, { "id": "datadog-achieving-relentless-kafka-reliability-at-scale-with-the-streaming-platf", "title": "Achieving relentless Kafka reliability at scale with the Streaming Platform", "company": "Datadog", "date": "2025-02-19", "url": "https://www.datadoghq.com/blog/engineering/streaming-platform-kafka-custom-abstractions/", "stack": [ "Apache Kafka", "Kubernetes", "Rust", "libstreaming" ], "scale": [ "hundreds of trillions of observability events/day", "hundreds of Kafka clusters, thousands of topics, millions of partitions", "terabytes of traffic/sec", "petabyte-scale pipelines" ], "techniques": [ "multi-cluster failover", "custom coordinator", "commit log enrichment", "QoS lanes", "dead-letter queue" ], "summary": "Datadog treats Kafka as commodity hardware behind a Streaming Platform control layer. Streams span multiple Kafka clusters, AZs, and Kubernetes clusters so producers and consumers talk to stable identifiers while the control plane redirects traffic to healthier topics in seconds\u2014for incidents, rebalancing, decommissioning, or repartitioning\u2014without app redeploys. A custom Assigner replaces Kafka's group coordinator for multi-cluster rebalancing in seconds; Stream lanes isolate QoS and park poison pills in a DLQ; an enriched commit log tracks multiple offset ranges plus timestamps so consumers can process live traffic while replaying backlogs and the control plane can compute wall-clock lag across millions of partitions. After zstd \"bombs\" crashed stock clients, Datadog wrote libstreaming in Rust (with Java/Go/Python bindings) for unified, high-throughput access without a proxy in the TB/s data path.", "verified": "2026-11-05" }, { "id": "zalando-contributing-to-debezium-fixing-logical-replication-at-scale", "title": "Contributing to Debezium: Fixing Logical Replication at Scale", "company": "Zalando", "date": "2025-12-19", "url": "https://engineering.zalando.com/posts/2025/12/contributing-to-debezium.html", "stack": [ "PostgreSQL", "Debezium", "Kubernetes", "Patroni", "Fabric Event Streams" ], "scale": [ "hundreds of event streams from Postgres", "hundreds of thousands of events/sec at peak", "100+ Kubernetes clusters", "billions of events since late 2018" ], "techniques": [ "logical replication", "WAL management", "open source contribution", "replication slots" ], "summary": "Zalando's Fabric Event Streams provisions Debezium-in-embedded micro-apps so teams declare Postgres-sourced Kafka streams on Kubernetes; at peak the connectors handle hundreds of thousands of events per second across 100+ clusters, and have processed billions of events since 2018. Uncontrolled WAL growth on low-activity databases was the top operational issue until a pgjdbc keepalive flush advanced replication slots without table traffic. When newer Debezium hard-disabled that flush, Zalando contributed lsn.flush.mode (including connector_and_driver) and offset.mismatch.strategy (including trust_slot / trust_greater_lsn) so operators can opt into keepalive flushes while choosing whether the slot or the offset store wins on restart\u2014both shipped in Debezium 3.4.0.Final.", "verified": "2026-11-05" }, { "id": "pinterest-scaling-model-evaluation-at-pinterest-with-ray-part-4", "title": "Scaling Model Evaluation at Pinterest with Ray (Part 4)", "company": "Pinterest", "date": "2026-11-04", "url": "https://medium.com/pinterest-engineering/scaling-model-evaluation-at-pinterest-with-ray-part-4-4a96cd12cc11", "stack": [ "Ray", "Apache Spark", "S3" ], "scale": [ "up to 32% faster end-to-end evaluation", "Large eval set \u22483M rows (also \u224830K / \u2248300K)", "at 100 experiments, fusion cuts wall-clock 25\u201332%", "fused Large runtime ~43 min fixed + ~41s/experiment as experiments scale to 300" ], "techniques": [ "fused inference and metrics", "reducible metrics", "batch inference", "feature importance" ], "summary": "Pinterest's feature-importance and multi-experiment evaluation used to run Ray inference, write predictions to S3, then start a separate metrics job that reread everything. They defined a ReducibleMetric contract (dataset_preprocessor, compute_partials, reduce_partials) so MAE, threshold-based ROC-AUC, and group-wise Precision@K can all compute from batches or segment-preserving buckets, then fused metric reduction into the Ray inference pass via the shared object store. On a Pin Ranking Model, fusion cut end-to-end wall-clock time by up to 32% (25\u201332% at 100 experiments across \u224830K\u20133M row sets) and kept Large-set runtime roughly linear (~43 min fixed plus ~41s per experiment up to 300), absorbing a separate ~11s-per-experiment metrics stage.", "verified": "2026-11-06" }, { "id": "openai-how-openai-delivers-low-latency-voice-ai-at-scale", "title": "How OpenAI delivers low-latency voice AI at scale", "company": "OpenAI", "date": "2026-05-04", "url": "https://openai.com/index/delivering-low-latency-voice-ai-at-scale", "stack": [ "WebRTC", "Pion", "Go", "Kubernetes", "Redis", "Cloudflare" ], "scale": [ "900 million+ weekly active users" ], "techniques": [ "WebRTC relay + transceiver", "ICE ufrag routing", "geo-steered signaling", "single-port UDP demux" ], "summary": "OpenAI rebuilt its WebRTC stack for ChatGPT voice and the Realtime API after one-port-per-session media termination collided with Kubernetes: large public UDP ranges, sticky ICE/DTLS ownership, and global first-hop latency. A lightweight Go relay exposes a small fixed UDP surface, reads ICE ufrag routing hints from the first STUN packet, and forwards opaque media to the transceiver that owns ICE, DTLS, and SRTP state (with Redis-backed recovery). Cloudflare geo and proximity steering place signaling near users while Global Relay shortens the first media hop, preserving standard WebRTC for clients without making inference backends WebRTC peers.", "verified": "2026-11-06" }, { "id": "neon-inside-branch-based-restores-in-lakebase-postgres", "title": "Inside branch-based restores in Lakebase Postgres", "company": "Neon", "date": "2026-09-28", "url": "https://neon.com/blog/inside-branch-based-restores-in-lakebase-postgres", "stack": [ "PostgreSQL", "Lakebase", "object storage", "pageserver", "safekeepers" ], "scale": [ "restores in seconds even at 100 TB", "99.99999% of reads served from local caches" ], "techniques": [ "storage/compute separation", "branch-based PITR", "WAL to pages", "copy-on-write branches" ], "summary": "Traditional managed Postgres PITR provisions a new compute+disk monolith, pulls a snapshot out of object storage, and replays WAL\u2014hours at large size\u2014while a survey Neon cites of 50 production Postgres developers found 30% down 3+ hours after a critical failure. Lakebase splits compute from durable history: safekeepers quorum-ack WAL, a pageserver materializes page versions, and object storage keeps immutable history, with 99.99999% of reads served from local caches. A restore is then a metadata branch at a timestamp rather than a copy-and-replay, so time-to-restore drops to seconds even for a 100 TB database.", "verified": "2026-11-06" } ]