diff --git a/.woodpecker.yml b/.woodpecker.yml index fc0ff08c9..de874ae5d 100644 --- a/.woodpecker.yml +++ b/.woodpecker.yml @@ -135,6 +135,18 @@ steps: - diesel migration redo when: *slow_check_paths + check_db_perf_tool: + image: *rust_image + environment: + LEMMY_DATABASE_URL: postgres://lemmy:password@database:5432/lemmy + RUST_BACKTRACE: "1" + CARGO_HOME: .cargo_home + commands: + # same as scripts/db_perf.sh but without creating a new database server + - export LEMMY_CONFIG_LOCATION=config/config.hjson + - cargo run --package lemmy_db_perf -- --posts 10 --read-post-pages 1 + when: *slow_check_paths + cargo_clippy: image: *rust_image environment: diff --git a/Cargo.lock b/Cargo.lock index 4b54af59d..e56b26b27 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -17,7 +17,7 @@ dependencies = [ "activitystreams-kinds", "actix-web", "async-trait", - "base64 0.21.5", + "base64 0.21.7", "bytes", "chrono", "derive_builder", @@ -90,9 +90,9 @@ dependencies = [ [[package]] name = "actix-form-data" -version = "0.7.0-beta.5" +version = "0.7.0-beta.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "02564e87c7a34fe7877d6b60259413ea04412b91abbabd3c2c71610e24800d0d" +checksum = "a0588d156cb871d8c237d55ce398e2a65727370fb98352ba5b65c15a2f834b0f" dependencies = [ "actix-multipart", "actix-web", @@ -116,8 +116,8 @@ dependencies = [ "actix-tls", "actix-utils", "ahash", - "base64 0.21.5", - "bitflags 2.4.1", + "base64 0.21.7", + "bitflags 2.4.2", "brotli", "bytes", "bytestring", @@ -179,9 +179,9 @@ dependencies = [ [[package]] name = "actix-router" -version = "0.5.1" +version = "0.5.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d66ff4d247d2b160861fa2866457e85706833527840e4133f8f49aa423a38799" +checksum = "d22475596539443685426b6bdadb926ad0ecaefdfc5fb05e5e3441f15463c511" dependencies = [ "bytestring", "http", @@ -212,7 +212,7 @@ dependencies = [ "futures-core", "futures-util", "mio", - "socket2 0.5.5", + "socket2", "tokio", "tracing", ] @@ -230,9 +230,9 @@ dependencies = [ [[package]] name = "actix-tls" -version = "3.1.1" +version = "3.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "72616e7fbec0aa99c6f3164677fa48ff5a60036d0799c98cab894a44f3e0efc3" +checksum = "929e47cc23865cdb856e59673cfba2d28f00b3bbd060dfc80e33a00a3cea8317" dependencies = [ "actix-rt", "actix-service", @@ -240,8 +240,6 @@ dependencies = [ "futures-core", "impl-more", "pin-project-lite", - "rustls 0.21.10", - "rustls-webpki", "tokio", "tokio-rustls 0.23.4", "tokio-util", @@ -295,7 +293,7 @@ dependencies = [ "serde_json", "serde_urlencoded", "smallvec", - "socket2 0.5.5", + "socket2", "time", "url", ] @@ -320,7 +318,7 @@ checksum = "1d613edf08a42ccc6864c941d30fe14e1b676a77d16f1dbadc1174d065a0a775" dependencies = [ "actix-utils", "actix-web", - "base64 0.21.5", + "base64 0.21.7", "futures-core", "futures-util", "log", @@ -357,9 +355,9 @@ checksum = "f26201604c87b1e01bd3d98f8d5d9a8fcbb815e8cedb41ffccbeb4bf593a35fe" [[package]] name = "ahash" -version = "0.8.6" +version = "0.8.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "91429305e9f0a25f6205c5b8e0d2db09e0708a7a6df0f42212bb56c32c8ac97a" +checksum = "77c3a9648d43b9cd48db467b3f87fdd6e146bcc88ab0180006cef2179fe11d01" dependencies = [ "cfg-if", "getrandom", @@ -415,9 +413,9 @@ dependencies = [ [[package]] name = "anstream" -version = "0.6.5" +version = "0.6.11" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d664a92ecae85fd0a7392615844904654d1d5f5514837f471ddef4a057aba1b6" +checksum = "6e2e1ebcb11de5c03c67de28a7df593d32191b44939c482e97702baaaa6ab6a5" dependencies = [ "anstyle", "anstyle-parse", @@ -463,9 +461,9 @@ dependencies = [ [[package]] name = "anyhow" -version = "1.0.75" +version = "1.0.79" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a4668cab20f66d8d020e1fbc0ebe47217433c1b6c8f2040faf858554e394ace6" +checksum = "080e9890a082662b09c1ad45f567faeeb47f22b5fb23895fbe1e651e718e25ca" dependencies = [ "backtrace", ] @@ -488,9 +486,9 @@ dependencies = [ [[package]] name = "async-compression" -version = "0.4.5" +version = "0.4.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bc2d0cfb2a7388d34f590e76686704c494ed7aaceed62ee1ba35cbf363abc2a5" +checksum = "a116f46a969224200a0a97f29cfd4c50e7534e4b4826bd23ea2c3c533039c82c" dependencies = [ "flate2", "futures-core", @@ -646,9 +644,9 @@ checksum = "9e1b586273c5702936fe7b7d6896644d8be71e6314cfe09d3167c95f712589e8" [[package]] name = "base64" -version = "0.21.5" +version = "0.21.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "35636a1494ede3b646cc98f74f8e62c773a38a659ebc777a2cf26b9b74171df9" +checksum = "9d297deb1925b89f2ccc13d7635fa0714f12c87adce1c75356b39ca9b7178567" [[package]] name = "bcrypt" @@ -656,7 +654,7 @@ version = "0.15.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "28d1c9c15093eb224f0baa400f38fcd713fc1391a6f1c389d886beef146d60a3" dependencies = [ - "base64 0.21.5", + "base64 0.21.7", "blowfish", "getrandom", "subtle", @@ -695,9 +693,9 @@ checksum = "bef38d45163c2f1dde094a7dfd33ccf595c92905c8f8f4fdc18d06fb1037718a" [[package]] name = "bitflags" -version = "2.4.1" +version = "2.4.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "327762f6e5a765692301e5bb513e0d9fef63be86bbc14528052b1cd3e6f03e07" +checksum = "ed570934406eb16438a4e976b1b4500774099c13b8cb96eec99f620f05090ddf" [[package]] name = "block-buffer" @@ -803,9 +801,9 @@ dependencies = [ [[package]] name = "cargo-platform" -version = "0.1.5" +version = "0.1.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e34637b3140142bdf929fb439e8aa4ebad7651ebf7b1080b3930aa16ac1459ff" +checksum = "ceed8ef69d8518a5dda55c07425450b58a4e1946f4951eab6d7191ee86c2443d" dependencies = [ "serde", ] @@ -847,9 +845,9 @@ checksum = "baf1de4339761588bc0619e3cbc0120ee582ebb74b53b4efbf79117bd2da40fd" [[package]] name = "chrono" -version = "0.4.31" +version = "0.4.32" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7f2c685bad3eb3d45a01354cedb7d5faa66194d1d58ba6e267a8de788f79db38" +checksum = "41daef31d7a747c5c847246f36de49ced6f7403b4cdabc807a97b5cc184cda7a" dependencies = [ "android-tzdata", "iana-time-zone", @@ -857,7 +855,7 @@ dependencies = [ "num-traits", "serde", "wasm-bindgen", - "windows-targets 0.48.5", + "windows-targets 0.52.0", ] [[package]] @@ -882,9 +880,9 @@ dependencies = [ [[package]] name = "clap" -version = "4.4.11" +version = "4.4.18" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bfaff671f6b22ca62406885ece523383b9b64022e341e53e009a62ebc47a45f2" +checksum = "1e578d6ec4194633722ccf9544794b71b1385c3c027efe0c55db226fc880865c" dependencies = [ "clap_builder", "clap_derive", @@ -892,9 +890,9 @@ dependencies = [ [[package]] name = "clap_builder" -version = "4.4.11" +version = "4.4.18" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a216b506622bb1d316cd51328dce24e07bdff4a6128a47c7e7fad11878d5adbb" +checksum = "4df4df40ec50c46000231c914968278b1eb05098cf8f1b3a518a95030e71d1c7" dependencies = [ "anstream", "anstyle", @@ -1129,9 +1127,9 @@ checksum = "06ea2b9bc92be3c2baa9334a323ebca2d6f074ff852cd1d7b11064035cd3868f" [[package]] name = "cpufeatures" -version = "0.2.11" +version = "0.2.12" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ce420fe07aecd3e67c5f910618fe65e94158f6dcc0adf44e00d69ce2bdfe0fd0" +checksum = "53fe5e26ff1b7aef8bca9c6080520cfb8d9333c7568e1829cef191a9723e5504" dependencies = [ "libc", ] @@ -1147,35 +1145,27 @@ dependencies = [ [[package]] name = "crossbeam-channel" -version = "0.5.8" +version = "0.5.11" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a33c2bf77f2df06183c3aa30d1e96c0695a313d4f9c453cc3762a6db39f99200" +checksum = "176dc175b78f56c0f321911d9c8eb2b77a78a4860b9c19db83835fea1a46649b" dependencies = [ - "cfg-if", "crossbeam-utils", ] [[package]] name = "crossbeam-epoch" -version = "0.9.15" +version = "0.9.18" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ae211234986c545741a7dc064309f67ee1e5ad243d0e48335adc0484d960bcc7" +checksum = "5b82ac4a3c2ca9c3460964f020e1402edd5753411d7737aa39c3714ad1b5420e" dependencies = [ - "autocfg", - "cfg-if", "crossbeam-utils", - "memoffset", - "scopeguard", ] [[package]] name = "crossbeam-utils" -version = "0.8.16" +version = "0.8.19" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5a22b2d63d4d1dc0b7f1b6b2747dd0088008a9be28b6ddf0b1e7d335e3037294" -dependencies = [ - "cfg-if", -] +checksum = "248e3bacc7dc6baa3b21e405ee045c3047101a49145e7e9eca583ab4c2ca5345" [[package]] name = "crypto-common" @@ -1341,9 +1331,9 @@ dependencies = [ [[package]] name = "deranged" -version = "0.3.10" +version = "0.3.11" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8eb30d70a07a3b04884d2677f06bec33509dc67ca60d92949e5535352d3191dc" +checksum = "b42b6fa04a440b495c8b04d0e71b707c585f83cb9cb28cf8cd0d976c315e31b4" dependencies = [ "powerfmt", "serde", @@ -1419,7 +1409,7 @@ version = "2.1.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "62c6fcf842f17f8c78ecf7c81d75c5ce84436b41ee07e03f490fbb5f5a8731d8" dependencies = [ - "bitflags 2.4.1", + "bitflags 2.4.2", "byteorder", "chrono", "diesel_derives", @@ -1482,9 +1472,9 @@ dependencies = [ [[package]] name = "diesel_ltree" -version = "0.3.0" +version = "0.3.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "92d9f0b872d6c87b68a71f105802b941a7262788bf69d1bcd05654669cdbd55d" +checksum = "9f5884ffa287a93dce7bd7e5263241c4db5ba7418863fe754d6b731c7e5e06f2" dependencies = [ "byteorder", "diesel", @@ -1593,7 +1583,7 @@ version = "0.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "dbfb21b9878cf7a348dcb8559109aabc0ec40d69924bd706fa5149846c4fef75" dependencies = [ - "base64 0.21.5", + "base64 0.21.7", "memchr", ] @@ -1759,9 +1749,9 @@ checksum = "0206175f82b8d6bf6652ff7d71a1e27fd2e4efde587fd368662814d6ec1d9ce0" [[package]] name = "eyre" -version = "0.6.10" +version = "0.6.11" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8bbb8258be8305fb0237d7b295f47bb24ff1b136a535f473baf40e70468515aa" +checksum = "b6267a1fa6f59179ea4afc8e50fd8612a3cc60bc858f786ff877a4a8cb042799" dependencies = [ "indenter", "once_cell", @@ -1800,9 +1790,9 @@ checksum = "25cbce373ec4653f1a01a31e8a5e5ec0c622dc27ff9c4e6606eefef5cbbed4a5" [[package]] name = "fdeflate" -version = "0.3.1" +version = "0.3.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "64d6dafc854908ff5da46ff3f8f473c6984119a2876a383a860246dd7841a868" +checksum = "4f9bfee30e4dedf0ab8b422f03af778d9612b63f502710fc500a334ebe2de645" dependencies = [ "simd-adler32", ] @@ -1995,9 +1985,9 @@ dependencies = [ [[package]] name = "getrandom" -version = "0.2.11" +version = "0.2.12" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fe9006bed769170c11f845cf00c7c1e9092aeb3f268e007c3e760ac68008070f" +checksum = "190092ea657667030ac6a35e305e62fc4dd69fd98ac98631e5d3a2b1575a12b5" dependencies = [ "cfg-if", "js-sys", @@ -2020,9 +2010,9 @@ checksum = "d2fabcfbdc87f4758337ca535fb41a6d701b65693ce38287d856d1674551ec9b" [[package]] name = "h2" -version = "0.3.22" +version = "0.3.24" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4d6250322ef6e60f93f9a2162799302cd6f68f79f6e5d85c8c16f14d1d958178" +checksum = "bb2c4422095b67ee78da96fbb51a4cc413b3b25883c7717ff7ca1ab31022c9c9" dependencies = [ "bytes", "fnv", @@ -2068,7 +2058,7 @@ version = "7.5.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "765c9198f173dd59ce26ff9f95ef0aafd0a0fe01fb9d72841bc5066a4c06511d" dependencies = [ - "base64 0.21.5", + "base64 0.21.7", "byteorder", "flate2", "nom", @@ -2083,9 +2073,9 @@ checksum = "95505c38b4572b2d910cecb0281560f54b440a19336cbbcb27bf6ce6adc6f5a8" [[package]] name = "hermit-abi" -version = "0.3.3" +version = "0.3.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d77f7ec81a6d05a3abb01ab6eb7590f6083d08449fe5a1c8b1e620283546ccb7" +checksum = "5d3d0e0f38255e7fa3cf31335b3a56f05febd18025f4db5ef7a0cfb4f8da651f" [[package]] name = "hex" @@ -2237,9 +2227,9 @@ checksum = "9a3a5bfb195931eeb336b2a7b4d761daec841b97f947d34394601737a7bba5e4" [[package]] name = "hyper" -version = "0.14.27" +version = "0.14.28" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ffb1cfd654a8219eaef89881fdb3bb3b1cdc5fa75ded05d6933b2b382e395468" +checksum = "bf96e135eb83a2a8ddf766e426a841d8ddd7449d5f00d34ea02b41d2f19eef80" dependencies = [ "bytes", "futures-channel", @@ -2252,7 +2242,7 @@ dependencies = [ "httpdate", "itoa", "pin-project-lite", - "socket2 0.4.10", + "socket2", "tokio", "tower-service", "tracing", @@ -2299,10 +2289,30 @@ dependencies = [ ] [[package]] -name = "iana-time-zone" -version = "0.1.58" +name = "i-love-jesus" +version = "0.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8326b86b6cff230b97d0d312a6c40a60726df3332e721f72a1b035f451663b20" +checksum = "39fa60e3281e1529cc56d96cca925215f51f9b39a96bc677982fbfdf2663cc84" +dependencies = [ + "diesel", + "i-love-jesus-macros", +] + +[[package]] +name = "i-love-jesus-macros" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8215279f83f9b829403812f845aa2d0dd5966332aa2fd0334a375256f3dd0322" +dependencies = [ + "quote", + "syn 2.0.48", +] + +[[package]] +name = "iana-time-zone" +version = "0.1.59" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b6a67363e2aa4443928ce15e57ebae94fd8949958fd1223c4cfc0cd473ad7539" dependencies = [ "android_system_properties", "core-foundation-sys", @@ -2349,14 +2359,13 @@ dependencies = [ [[package]] name = "image" -version = "0.24.7" +version = "0.24.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6f3dfdbdd72063086ff443e297b61695500514b1e41095b6fb9a5ab48a70a711" +checksum = "034bbe799d1909622a74d1193aa50147769440040ff36cb2baa947609b0a4e23" dependencies = [ "bytemuck", "byteorder", "color_quant", - "num-rational", "num-traits", "png", ] @@ -2494,9 +2503,9 @@ dependencies = [ [[package]] name = "js-sys" -version = "0.3.66" +version = "0.3.67" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cee9c64da59eae3b50095c18d3e74f8b73c0b86d2792824ff01bbce68ba229ca" +checksum = "9a1d36f1235bc969acba30b7f5990b864423a6068a10f7c90ae8f0112e3a59d1" dependencies = [ "wasm-bindgen", ] @@ -2507,7 +2516,7 @@ version = "8.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6971da4d9c3aa03c3d8f3ff0f4155b534aad021292003895a469716b2a230378" dependencies = [ - "base64 0.21.5", + "base64 0.21.7", "pem", "ring 0.16.20", "serde", @@ -2529,13 +2538,13 @@ checksum = "e2abad23fbc42b3700f2f279844dc832adb2b2eb069b2df918f455c4e18cc646" [[package]] name = "lemmy_api" -version = "0.19.3-rc.1" +version = "0.19.3" dependencies = [ "activitypub_federation", "actix-web", "actix-web-httpauth", "anyhow", - "base64 0.21.5", + "base64 0.21.7", "bcrypt", "captcha", "chrono", @@ -2558,7 +2567,7 @@ dependencies = [ [[package]] name = "lemmy_api_common" -version = "0.19.3-rc.1" +version = "0.19.3" dependencies = [ "activitypub_federation", "actix-web", @@ -2593,7 +2602,7 @@ dependencies = [ [[package]] name = "lemmy_api_crud" -version = "0.19.3-rc.1" +version = "0.19.3" dependencies = [ "activitypub_federation", "actix-web", @@ -2615,7 +2624,7 @@ dependencies = [ [[package]] name = "lemmy_apub" -version = "0.19.3-rc.1" +version = "0.19.3" dependencies = [ "activitypub_federation", "actix-web", @@ -2653,11 +2662,26 @@ dependencies = [ "uuid", ] +[[package]] +name = "lemmy_db_perf" +version = "0.19.3" +dependencies = [ + "anyhow", + "clap", + "diesel", + "diesel-async", + "lemmy_db_schema", + "lemmy_db_views", + "lemmy_utils", + "tokio", +] + [[package]] name = "lemmy_db_schema" -version = "0.19.3-rc.1" +version = "0.19.3" dependencies = [ "activitypub_federation", + "anyhow", "async-trait", "bcrypt", "chrono", @@ -2669,6 +2693,7 @@ dependencies = [ "diesel_ltree", "diesel_migrations", "futures-util", + "i-love-jesus", "lemmy_utils", "once_cell", "pretty_assertions", @@ -2692,13 +2717,14 @@ dependencies = [ [[package]] name = "lemmy_db_views" -version = "0.19.3-rc.1" +version = "0.19.3" dependencies = [ "actix-web", "chrono", "diesel", "diesel-async", "diesel_ltree", + "i-love-jesus", "lemmy_db_schema", "lemmy_utils", "pretty_assertions", @@ -2708,12 +2734,11 @@ dependencies = [ "tokio", "tracing", "ts-rs", - "url", ] [[package]] name = "lemmy_db_views_actor" -version = "0.19.3-rc.1" +version = "0.19.3" dependencies = [ "chrono", "diesel", @@ -2731,7 +2756,7 @@ dependencies = [ [[package]] name = "lemmy_db_views_moderator" -version = "0.19.3-rc.1" +version = "0.19.3" dependencies = [ "diesel", "diesel-async", @@ -2743,7 +2768,7 @@ dependencies = [ [[package]] name = "lemmy_federate" -version = "0.19.3-rc.1" +version = "0.19.3" dependencies = [ "activitypub_federation", "anyhow", @@ -2766,7 +2791,7 @@ dependencies = [ [[package]] name = "lemmy_routes" -version = "0.19.3-rc.1" +version = "0.19.3" dependencies = [ "activitypub_federation", "actix-web", @@ -2790,7 +2815,7 @@ dependencies = [ [[package]] name = "lemmy_server" -version = "0.19.3-rc.1" +version = "0.19.3" dependencies = [ "activitypub_federation", "actix-cors", @@ -2833,7 +2858,7 @@ dependencies = [ [[package]] name = "lemmy_utils" -version = "0.19.3-rc.1" +version = "0.19.3" dependencies = [ "actix-web", "anyhow", @@ -2871,12 +2896,12 @@ dependencies = [ [[package]] name = "lettre" -version = "0.11.2" +version = "0.11.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a48c2e9831b370bc2d7233c2620298c45f3a158ed6b4b8d7416b2ada5a268fd8" +checksum = "f5aaf628956b6b0852e12ac3505d20d7a12ecc1e32d5ea921f002af4a74036a5" dependencies = [ "async-trait", - "base64 0.21.5", + "base64 0.21.7", "chumsky", "email-encoding", "email_address", @@ -2889,9 +2914,8 @@ dependencies = [ "mime", "native-tls", "nom", - "once_cell", "quoted_printable", - "socket2 0.5.5", + "socket2", "tokio", "tokio-native-tls", "url", @@ -2899,9 +2923,9 @@ dependencies = [ [[package]] name = "libc" -version = "0.2.151" +version = "0.2.152" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "302d7ab3130588088d277783b1e2d2e10c9e9e4a16dd9050e6ec93fb3e7048f4" +checksum = "13e3bf6590cbc649f4d1a3eefc9d5d6eb746f5200ffb04e5e142700b8faa56e7" [[package]] name = "line-wrap" @@ -2935,9 +2959,9 @@ checksum = "f051f77a7c8e6957c0696eac88f26b0117e54f52d3fc682ab19397a8812846a4" [[package]] name = "linux-raw-sys" -version = "0.4.12" +version = "0.4.13" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c4cd1a83af159aa67994778be9070f0ae1bd732942279cabb14f86f986a21456" +checksum = "01cda141df6706de531b6c46c3a33ecca755538219bd484262fa09410c13539c" [[package]] name = "local-channel" @@ -2968,9 +2992,9 @@ dependencies = [ [[package]] name = "lodepng" -version = "3.9.2" +version = "3.10.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b00f56ff9bcd5721ab172b73eac8a7d4e9439f47a98581e666178dbe7df97e13" +checksum = "a42d298694b14401847de29abd44adf278b42e989e516deac7b72018400002d8" dependencies = [ "crc32fast", "fallible_collections", @@ -2991,15 +3015,6 @@ version = "0.1.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c41e0c4fef86961ac6d6f8a82609f55f31b05e4fce149ac5710e439df7619ba4" -[[package]] -name = "mach2" -version = "0.4.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6d0d1830bcd151a6fc4aea1369af235b36c1528fe976b8ff678683c9995eade8" -dependencies = [ - "libc", -] - [[package]] name = "markdown-it" version = "0.6.0" @@ -3093,37 +3108,27 @@ dependencies = [ [[package]] name = "memchr" -version = "2.6.4" +version = "2.7.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f665ee40bc4a3c5590afb1e9677db74a508659dfd71e126420da8274909a0167" - -[[package]] -name = "memoffset" -version = "0.9.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5a634b1c61a95585bd15607c6ab0c4e5b226e695ff2800ba0cdccddf208c406c" -dependencies = [ - "autocfg", -] +checksum = "523dc4f511e55ab87b694dc30d0f820d60906ef06413f93d4d7a1385599cc149" [[package]] name = "metrics" -version = "0.21.1" +version = "0.22.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fde3af1a009ed76a778cb84fdef9e7dbbdf5775ae3e4cc1f434a6a307f6f76c5" +checksum = "77b9e10a211c839210fd7f99954bda26e5f8e26ec686ad68da6a32df7c80e782" dependencies = [ "ahash", - "metrics-macros", "portable-atomic", ] [[package]] name = "metrics-exporter-prometheus" -version = "0.12.1" +version = "0.13.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8a4964177ddfdab1e3a2b37aec7cf320e14169abb0ed73999f558136409178d5" +checksum = "83a4c4718a371ddfb7806378f23617876eea8b82e5ff1324516bcd283249d9ea" dependencies = [ - "base64 0.21.5", + "base64 0.21.7", "hyper", "indexmap 1.9.3", "ipnet", @@ -3134,22 +3139,11 @@ dependencies = [ "tokio", ] -[[package]] -name = "metrics-macros" -version = "0.7.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ddece26afd34c31585c74a4db0630c376df271c285d682d1e55012197830b6df" -dependencies = [ - "proc-macro2", - "quote", - "syn 2.0.48", -] - [[package]] name = "metrics-util" -version = "0.15.1" +version = "0.16.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4de2ed6e491ed114b40b732e4d1659a9d53992ebd87490c44a6ffe23739d973e" +checksum = "2670b8badcc285d486261e2e9f1615b506baff91427b61bd336a472b65bbf5ed" dependencies = [ "crossbeam-epoch", "crossbeam-utils", @@ -3227,9 +3221,9 @@ dependencies = [ [[package]] name = "moka" -version = "0.12.2" +version = "0.12.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f353abec74660d4b8533c2516c86eb062f1ec8ca49a2758f4f2b1b60b06b0c6e" +checksum = "ad9dc9808102655926a6086abd0b9965ebefd4a39ef0d184f074c34ba5049ec6" dependencies = [ "async-lock", "async-trait", @@ -3335,17 +3329,6 @@ dependencies = [ "num-traits", ] -[[package]] -name = "num-rational" -version = "0.4.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0638a1c9d0a3c0914158145bc76cff373a75a627e6ecbfb71cbe6f453a5a19b0" -dependencies = [ - "autocfg", - "num-integer", - "num-traits", -] - [[package]] name = "num-traits" version = "0.2.17" @@ -3367,9 +3350,9 @@ dependencies = [ [[package]] name = "object" -version = "0.32.1" +version = "0.32.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9cf5f9dd3933bd50a9e1f149ec995f39ae2c496d31fd772c1fd45ebc27e902b0" +checksum = "a6a622008b6e321afc04970976f62ee297fdbaa6f95318ca343e3eebb9648441" dependencies = [ "memchr", ] @@ -3382,11 +3365,11 @@ checksum = "3fdb12b2476b595f9358c5161aa467c2438859caa136dec86c26fdd2efe17b92" [[package]] name = "openssl" -version = "0.10.62" +version = "0.10.63" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8cde4d2d9200ad5909f8dac647e29482e07c3a35de8a13fce7c9c7747ad9f671" +checksum = "15c9d69dd87a29568d4d017cfe8ec518706046a05184e5aea92d0af890b803c8" dependencies = [ - "bitflags 2.4.1", + "bitflags 2.4.2", "cfg-if", "foreign-types", "libc", @@ -3414,9 +3397,9 @@ checksum = "ff011a302c396a5197692431fc1948019154afc178baf7d8e37367442a4601cf" [[package]] name = "openssl-sys" -version = "0.9.98" +version = "0.9.99" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c1665caf8ab2dc9aef43d1c0023bd904633a6a05cb30b0ad59bec2ae986e57a7" +checksum = "22e1bf214306098e4832460f797824c05d25aacdf896f64a985fb0fd992454ae" dependencies = [ "cc", "libc", @@ -3497,7 +3480,7 @@ dependencies = [ "opentelemetry 0.21.0", "opentelemetry-proto 0.4.0", "opentelemetry-semantic-conventions", - "opentelemetry_sdk 0.21.1", + "opentelemetry_sdk 0.21.2", "prost 0.11.9", "thiserror", "tokio", @@ -3524,7 +3507,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a2e155ce5cc812ea3d1dffbd1539aed653de4bf4882d60e6e04dcf0901d674e1" dependencies = [ "opentelemetry 0.21.0", - "opentelemetry_sdk 0.21.1", + "opentelemetry_sdk 0.21.2", "prost 0.11.9", "tonic 0.9.2", ] @@ -3578,9 +3561,9 @@ dependencies = [ [[package]] name = "opentelemetry_sdk" -version = "0.21.1" +version = "0.21.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "968ba3f2ca03e90e5187f5e4f46c791ef7f2c163ae87789c8ce5f5ca3b7b7de5" +checksum = "2f16aec8a98a457a52664d69e0091bac3a0abd18ead9b641cb00202ba4e0efe4" dependencies = [ "async-trait", "crossbeam-channel", @@ -3752,16 +3735,15 @@ dependencies = [ [[package]] name = "pict-rs" -version = "0.5.0-rc.2" +version = "0.5.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f24eed67dd30cd04e67959b7c9a2b14c8e7c9d9a67b095eb56c70477a6eef212" +checksum = "ca72ffda2e229b6be6aeda550b7069aa8dc72589711da63c59914df89bf91b02" dependencies = [ "actix-form-data", "actix-web", - "anyhow", "async-trait", "barrel", - "base64 0.21.5", + "base64 0.21.7", "clap", "color-eyre", "config", @@ -3780,7 +3762,7 @@ dependencies = [ "mime", "opentelemetry 0.21.0", "opentelemetry-otlp 0.14.0", - "opentelemetry_sdk 0.21.1", + "opentelemetry_sdk 0.21.2", "pin-project-lite", "refinery", "reqwest", @@ -3846,9 +3828,9 @@ checksum = "8b870d8c151b6f2fb93e84a13146138f05d02ed11c7e7c54f8826aaaf7c9f184" [[package]] name = "pkg-config" -version = "0.3.27" +version = "0.3.29" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "26072860ba924cbfa98ea39c8c19b4dd6a4a25423dbdf219c1eca91aa0cf6964" +checksum = "2900ede94e305130c13ddd391e0ab7cbaeb783945ae07a279c268cb05109c6cb" [[package]] name = "plist" @@ -3856,7 +3838,7 @@ version = "1.6.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e5699cc8a63d1aa2b1ee8e12b9ad70ac790d65788cd36101fa37f87ea46c4cef" dependencies = [ - "base64 0.21.5", + "base64 0.21.7", "indexmap 2.1.0", "line-wrap", "quick-xml 0.31.0", @@ -3866,9 +3848,9 @@ dependencies = [ [[package]] name = "png" -version = "0.17.10" +version = "0.17.11" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "dd75bf2d8dd3702b9707cdbc56a5b9ef42cec752eb8b3bafc01234558442aa64" +checksum = "1f6c3c3e617595665b8ea2ff95a86066be38fb121ff920a9c0eb282abcd1da5a" dependencies = [ "bitflags 1.3.2", "crc32fast", @@ -3903,7 +3885,7 @@ version = "0.6.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "49b6c5ef183cd3ab4ba005f1ca64c21e8bd97ce4699cfea9e8d9a2c4958ca520" dependencies = [ - "base64 0.21.5", + "base64 0.21.7", "byteorder", "bytes", "fallible-iterator", @@ -3969,9 +3951,9 @@ dependencies = [ [[package]] name = "proc-macro2" -version = "1.0.75" +version = "1.0.78" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "907a61bd0f64c2f29cd1cf1dc34d05176426a3f504a78010f08416ddb7b13708" +checksum = "e2422ad645d89c99f8f3e6b88a9fdeca7fabeac836b1002371c4367c8f984aae" dependencies = [ "unicode-ident", ] @@ -4098,13 +4080,12 @@ dependencies = [ [[package]] name = "quanta" -version = "0.11.1" +version = "0.12.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a17e662a7a8291a865152364c20c7abc5e60486ab2001e8ec10b24862de0b9ab" +checksum = "9ca0b7bac0b97248c40bb77288fc52029cf1459c0461ea1b05ee32ccf011de2c" dependencies = [ "crossbeam-utils", "libc", - "mach2", "once_cell", "raw-cpuid", "wasi", @@ -4179,18 +4160,18 @@ dependencies = [ [[package]] name = "raw-cpuid" -version = "10.7.0" +version = "11.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6c297679cb867470fa8c9f67dbba74a78d78e3e98d7cf2b08d6d71540f797332" +checksum = "9d86a7c4638d42c44551f4791a20e687dbb4c3de1f33c43dd71e355cd429def1" dependencies = [ - "bitflags 1.3.2", + "bitflags 2.4.2", ] [[package]] name = "readonly" -version = "0.2.11" +version = "0.2.12" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b8f439da1766942fe069954da6058b2e6c1760eb878bae76f5be9fc29f56f574" +checksum = "a25d631e41bfb5fdcde1d4e2215f62f7f0afa3ff11e26563765bd6ea1d229aeb" dependencies = [ "proc-macro2", "quote", @@ -4217,9 +4198,9 @@ dependencies = [ [[package]] name = "refinery" -version = "0.8.11" +version = "0.8.12" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "529664dbccc0a296947615c997a857912d72d1c44be1fafb7bae54ecfa7a8c24" +checksum = "a2783724569d96af53464d0711dff635cab7a4934df5e22e9fbc9e181523b83e" dependencies = [ "refinery-core", "refinery-macros", @@ -4227,13 +4208,12 @@ dependencies = [ [[package]] name = "refinery-core" -version = "0.8.11" +version = "0.8.12" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e895cb870cf06e92318cbbeb701f274d022d5ca87a16fa8244e291cd035ef954" +checksum = "08d6c80329c0455510a8d42fce286ecb4b6bcd8c57e1816d9f2d6bd7379c2cc8" dependencies = [ "async-trait", "cfg-if", - "lazy_static", "log", "postgres", "regex", @@ -4243,16 +4223,16 @@ dependencies = [ "time", "tokio", "tokio-postgres", - "toml 0.7.8", + "toml 0.8.8", "url", "walkdir", ] [[package]] name = "refinery-macros" -version = "0.8.11" +version = "0.8.12" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "123e8b80f8010c3ae38330c81e76938fc7adf6cdbfbaad20295bb8c22718b4f1" +checksum = "6ab6e31e166a49d55cb09b62639e5ab9ba2e73f2f124336b06f6c321dc602779" dependencies = [ "proc-macro2", "quote", @@ -4263,13 +4243,13 @@ dependencies = [ [[package]] name = "regex" -version = "1.10.2" +version = "1.10.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "380b951a9c5e80ddfd6136919eef32310721aa4aacd4889a8d39124b026ab343" +checksum = "b62dbe01f0b06f9d8dc7d49e05a0785f153b00b2c227856282f671e0318c9b15" dependencies = [ "aho-corasick", "memchr", - "regex-automata 0.4.3", + "regex-automata 0.4.4", "regex-syntax 0.8.2", ] @@ -4284,9 +4264,9 @@ dependencies = [ [[package]] name = "regex-automata" -version = "0.4.3" +version = "0.4.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5f804c7828047e88b2d32e2d7fe5a105da8ee3264f01902f796c8e067dc2483f" +checksum = "3b7fa1134405e2ec9353fd416b17f8dacd46c473d7d3fd1cf202706a14eb792a" dependencies = [ "aho-corasick", "memchr", @@ -4318,7 +4298,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "37b1ae8d9ac08420c66222fb9096fc5de435c3c48542bc5336c51892cffafb41" dependencies = [ "async-compression", - "base64 0.21.5", + "base64 0.21.7", "bytes", "encoding_rs", "futures-core", @@ -4375,9 +4355,9 @@ dependencies = [ [[package]] name = "reqwest-tracing" -version = "0.4.6" +version = "0.4.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "14b1e66540e0cac90acadaf7109bf99c90d95abcc94b4c096bfa16a2d7aa7a71" +checksum = "5a0152176687dd5cfe7f507ac1cb1a491c679cfe483afd133a7db7aaea818bb3" dependencies = [ "anyhow", "async-trait", @@ -4474,9 +4454,9 @@ checksum = "2f8c01b9158de3aa5a7ac041a41c0e854d7adc3e473e7d7e2143eb5432bc5ba2" [[package]] name = "rss" -version = "2.0.6" +version = "2.0.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7e6c0ea0e621c2a3aa34850ebd711526f0ac7385921f57d2430a47cecc7b9cbc" +checksum = "f7b2c77eb4450d7d5f98df52c381cd6c4e19b75dad9209a9530b85a44510219a" dependencies = [ "atom_syndication", "derive_builder", @@ -4515,14 +4495,14 @@ dependencies = [ [[package]] name = "rustix" -version = "0.38.28" +version = "0.38.30" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "72e572a5e8ca657d7366229cdde4bd14c4eb5499a9573d4d366fe1b599daa316" +checksum = "322394588aaf33c24007e8bb3238ee3e4c5c09c084ab32bc73890b99ff326bca" dependencies = [ - "bitflags 2.4.1", + "bitflags 2.4.2", "errno", "libc", - "linux-raw-sys 0.4.12", + "linux-raw-sys 0.4.13", "windows-sys 0.52.0", ] @@ -4556,7 +4536,7 @@ version = "1.0.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1c74cae0a4cf6ccbbf5f359f08efdf8ee7e1dc532573bf0db71968cb56b1448c" dependencies = [ - "base64 0.21.5", + "base64 0.21.7", ] [[package]] @@ -4581,7 +4561,7 @@ version = "0.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "31aa883f1b986a5249641e574ca0e11ac4fb9970b009c6fbb96fedaf4fa78db8" dependencies = [ - "base64 0.21.5", + "base64 0.21.7", "hmac", "md-5", "percent-encoding", @@ -4617,11 +4597,11 @@ dependencies = [ [[package]] name = "schannel" -version = "0.1.22" +version = "0.1.23" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0c3733bf4cf7ea0880754e19cb5a462007c4a8c1914bff372ccc95b464f1df88" +checksum = "fbc91545643bcf3a0bbb6569265615222618bdf33ce4ffbbd13c4bbd4c093534" dependencies = [ - "windows-sys 0.48.0", + "windows-sys 0.52.0", ] [[package]] @@ -4686,18 +4666,18 @@ dependencies = [ [[package]] name = "semver" -version = "1.0.20" +version = "1.0.21" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "836fa6a3e1e547f9a2c4040802ec865b5d85f4014efe00555d7090a3dcaa1090" +checksum = "b97ed7a9823b74f99c7742f5336af7be5ecd3eeafcb1507d1fa93347b1d589b0" dependencies = [ "serde", ] [[package]] name = "serde" -version = "1.0.194" +version = "1.0.195" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0b114498256798c94a0689e1a15fec6005dee8ac1f41de56404b67afc2a4b773" +checksum = "63261df402c67811e9ac6def069e4786148c4563f4b50fd4bf30aa370d626b02" dependencies = [ "serde_derive", ] @@ -4713,9 +4693,9 @@ dependencies = [ [[package]] name = "serde_derive" -version = "1.0.194" +version = "1.0.195" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a3385e45322e8f9931410f01b3031ec534c3947d0e94c18049af4d9f9907d4e0" +checksum = "46fe8f8603d81ba86327b23a2e9cdf49e1255fb94a4c5f297f6ee0547178ea2c" dependencies = [ "proc-macro2", "quote", @@ -4745,9 +4725,9 @@ dependencies = [ [[package]] name = "serde_spanned" -version = "0.6.4" +version = "0.6.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "12022b835073e5b11e90a14f86838ceb1c8fb0325b72416845c487ac0fa95e80" +checksum = "eb3622f419d1296904700073ea6cc23ad690adbd66f13ea683df73298736f0c1" dependencies = [ "serde", ] @@ -4766,11 +4746,11 @@ dependencies = [ [[package]] name = "serde_with" -version = "3.4.0" +version = "3.5.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "64cd236ccc1b7a29e7e2739f27c0b2dd199804abc4290e32f59f3b68d6405c23" +checksum = "f5c9fdb6b00a489875b22efd4b78fe2b363b72265cc5f6eb2e2b9ee270e6140c" dependencies = [ - "base64 0.21.5", + "base64 0.21.7", "chrono", "hex", "indexmap 1.9.3", @@ -4783,9 +4763,9 @@ dependencies = [ [[package]] name = "serde_with_macros" -version = "3.4.0" +version = "3.5.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "93634eb5f75a2323b16de4748022ac4297f9e76b6dced2be287a099f41b5e788" +checksum = "dbff351eb4b33600a2e138dfa0b10b65a238ea8ff8fb2387c422c5022a3e8298" dependencies = [ "darling 0.20.3", "proc-macro2", @@ -4946,9 +4926,9 @@ dependencies = [ [[package]] name = "smallvec" -version = "1.11.2" +version = "1.13.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4dccd0940a2dcdf68d092b8cbab7dc0ad8fa938bf95787e1b916b0e3d0e8e970" +checksum = "e6ecd384b10a64542d77071bd64bd7b231f4ed5940fba55e98c3de13824cf3d7" [[package]] name = "smart-default" @@ -4961,16 +4941,6 @@ dependencies = [ "syn 2.0.48", ] -[[package]] -name = "socket2" -version = "0.4.10" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9f7916fc008ca5542385b89a3d3ce689953c143e9304a9bf8beec1de48994c0d" -dependencies = [ - "libc", - "winapi", -] - [[package]] name = "socket2" version = "0.5.5" @@ -5186,15 +5156,15 @@ dependencies = [ [[package]] name = "tempfile" -version = "3.8.1" +version = "3.9.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7ef1adac450ad7f4b3c28589471ade84f25f731a7a0fe30d71dfa9f60fd808e5" +checksum = "01ce4141aa927a6d1bd34a041795abd0db1cccba5d5f24b009f694bdf3a1f3fa" dependencies = [ "cfg-if", "fastrand", "redox_syscall 0.4.1", - "rustix 0.38.28", - "windows-sys 0.48.0", + "rustix 0.38.30", + "windows-sys 0.52.0", ] [[package]] @@ -5210,9 +5180,9 @@ dependencies = [ [[package]] name = "termcolor" -version = "1.4.0" +version = "1.4.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ff1bc3d3f05aff0403e8ac0d92ced918ec05b666a43f83297ccef5bea8a3d449" +checksum = "06794f8f6c5c898b3275aebefa6b8a1cb24cd2c6c79397ab15774837a0bc5755" dependencies = [ "winapi-util", ] @@ -5249,9 +5219,9 @@ dependencies = [ [[package]] name = "time" -version = "0.3.30" +version = "0.3.31" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c4a34ab300f2dee6e562c10a046fc05e358b29f9bf92277f30c3c8d82275f6f5" +checksum = "f657ba42c3f86e7680e53c8cd3af8abbe56b5491790b46e22e19c0d57463583e" dependencies = [ "deranged", "itoa", @@ -5269,9 +5239,9 @@ checksum = "ef927ca75afb808a4d64dd374f00a2adf8d0fcff8e7b184af886c3c87ec4a3f3" [[package]] name = "time-macros" -version = "0.2.15" +version = "0.2.16" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4ad70d68dba9e1f8aceda7aa6711965dfec1cac869f311a51bd08b3a2ccbce20" +checksum = "26197e33420244aeb70c3e8c78376ca46571bc4e701e4791c2cd9f57dcb3a43f" dependencies = [ "time-core", ] @@ -5311,7 +5281,7 @@ dependencies = [ "parking_lot 0.12.1", "pin-project-lite", "signal-hook-registry", - "socket2 0.5.5", + "socket2", "tokio-macros", "tracing", "windows-sys 0.48.0", @@ -5368,7 +5338,7 @@ dependencies = [ "postgres-protocol", "postgres-types", "rand", - "socket2 0.5.5", + "socket2", "tokio", "tokio-util", "whoami", @@ -5542,7 +5512,7 @@ checksum = "3082666a3a6433f7f511c7192923fa1fe07c69332d3c6a2e6bb040b569199d5a" dependencies = [ "async-trait", "axum", - "base64 0.21.5", + "base64 0.21.7", "bytes", "futures-core", "futures-util", @@ -5571,7 +5541,7 @@ dependencies = [ "async-stream", "async-trait", "axum", - "base64 0.21.5", + "base64 0.21.7", "bytes", "h2", "http", @@ -5591,9 +5561,9 @@ dependencies = [ [[package]] name = "totp-rs" -version = "5.4.0" +version = "5.5.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "df3504f96adf86d28e7eb16fa236a7951ec72c15ee100d1b5318e225944bc8cb" +checksum = "6c4ae9724c5888c0417d2396037ed3b60665925624766416e3e342b6ba5dbd3f" dependencies = [ "base32", "constant_time_eq", @@ -5763,7 +5733,7 @@ dependencies = [ "js-sys", "once_cell", "opentelemetry 0.21.0", - "opentelemetry_sdk 0.21.1", + "opentelemetry_sdk 0.21.2", "smallvec", "tracing", "tracing-core", @@ -5817,9 +5787,9 @@ checksum = "e421abadd41a4225275504ea4d6566923418b7f05506fbc9c0fe86ba7396114b" [[package]] name = "ts-rs" -version = "7.0.0" +version = "7.1.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e1ff1f8c90369bc172200013ac17ae86e7b5def580687df4e6127883454ff2b0" +checksum = "fc2cae1fc5d05d47aa24b64f9a4f7cba24cdc9187a2084dd97ac57bef5eccae6" dependencies = [ "chrono", "thiserror", @@ -5828,9 +5798,9 @@ dependencies = [ [[package]] name = "ts-rs-macros" -version = "7.0.0" +version = "7.1.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a6f41cc0aeb7a4a55730188e147d3795a7349b501f8334697fd37629b896cdc2" +checksum = "73f7f9b821696963053a89a7bd8b292dc34420aea8294d7b225274d488f3ec92" dependencies = [ "Inflector", "proc-macro2", @@ -5841,18 +5811,18 @@ dependencies = [ [[package]] name = "typed-builder" -version = "0.18.0" +version = "0.18.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e47c0496149861b7c95198088cbf36645016b1a0734cf350c50e2a38e070f38a" +checksum = "444d8748011b93cb168770e8092458cb0f8854f931ff82fdf6ddfbd72a9c933e" dependencies = [ "typed-builder-macro", ] [[package]] name = "typed-builder-macro" -version = "0.18.0" +version = "0.18.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "982ee4197351b5c9782847ef5ec1fdcaf50503fb19d68f9771adae314e72b492" +checksum = "563b3b88238ec95680aef36bdece66896eaa7ce3c0f1b4f39d38fb2435261352" dependencies = [ "proc-macro2", "quote", @@ -5876,9 +5846,9 @@ dependencies = [ [[package]] name = "unicode-bidi" -version = "0.3.14" +version = "0.3.15" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6f2528f27a9eb2b21e69c95319b30bd0efd85d09c379741b0f78ea1d86be2416" +checksum = "08f95100a766bf4f8f28f90d77e0a5461bbdb219042e7679bebe79004fed8d75" [[package]] name = "unicode-general-category" @@ -5963,9 +5933,9 @@ checksum = "711b9620af191e0cdc7468a8d14e709c3dcdb115b36f838e601583af800a370a" [[package]] name = "uuid" -version = "1.6.1" +version = "1.7.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5e395fcf16a7a3d8127ec99782007af141946b4795001f876d54fb0d55978560" +checksum = "f00cc9702ca12d3c81455259621e676d0f7251cec66a21e98fe2e9a37db93b2a" dependencies = [ "atomic", "getrandom", @@ -6017,9 +5987,9 @@ checksum = "9c8d87e72b64a3b4db28d11ce29237c246188f4f51057d65a7eab63b7987e423" [[package]] name = "wasm-bindgen" -version = "0.2.89" +version = "0.2.90" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0ed0d4f68a3015cc185aff4db9506a015f4b96f95303897bfa23f846db54064e" +checksum = "b1223296a201415c7fad14792dbefaace9bd52b62d33453ade1c5b5f07555406" dependencies = [ "cfg-if", "wasm-bindgen-macro", @@ -6027,9 +5997,9 @@ dependencies = [ [[package]] name = "wasm-bindgen-backend" -version = "0.2.89" +version = "0.2.90" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1b56f625e64f3a1084ded111c4d5f477df9f8c92df113852fa5a374dbda78826" +checksum = "fcdc935b63408d58a32f8cc9738a0bffd8f05cc7c002086c6ef20b7312ad9dcd" dependencies = [ "bumpalo", "log", @@ -6042,9 +6012,9 @@ dependencies = [ [[package]] name = "wasm-bindgen-futures" -version = "0.4.39" +version = "0.4.40" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ac36a15a220124ac510204aec1c3e5db8a22ab06fd6706d881dc6149f8ed9a12" +checksum = "bde2032aeb86bdfaecc8b261eef3cba735cc426c1f3a3416d1e0791be95fc461" dependencies = [ "cfg-if", "js-sys", @@ -6054,9 +6024,9 @@ dependencies = [ [[package]] name = "wasm-bindgen-macro" -version = "0.2.89" +version = "0.2.90" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0162dbf37223cd2afce98f3d0785506dcb8d266223983e4b5b525859e6e182b2" +checksum = "3e4c238561b2d428924c49815533a8b9121c664599558a5d9ec51f8a1740a999" dependencies = [ "quote", "wasm-bindgen-macro-support", @@ -6064,9 +6034,9 @@ dependencies = [ [[package]] name = "wasm-bindgen-macro-support" -version = "0.2.89" +version = "0.2.90" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f0eb82fcb7930ae6219a7ecfd55b217f5f0893484b7a13022ebb2b2bf20b5283" +checksum = "bae1abb6806dc1ad9e560ed242107c0f6c84335f1749dd4e8ddb012ebd5e25a7" dependencies = [ "proc-macro2", "quote", @@ -6077,9 +6047,9 @@ dependencies = [ [[package]] name = "wasm-bindgen-shared" -version = "0.2.89" +version = "0.2.90" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7ab9b36309365056cd639da3134bf87fa8f3d86008abf99e612384a6eecd459f" +checksum = "4d91413b1c31d7539ba5ef2451af3f0b833a005eb27a631cec32bc0635a8602b" [[package]] name = "wasm-streams" @@ -6105,9 +6075,9 @@ dependencies = [ [[package]] name = "web-sys" -version = "0.3.66" +version = "0.3.67" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "50c24a44ec86bb68fbecd1b3efed7e85ea5621b39b35ef2766b66cd984f8010f" +checksum = "58cd2333b6e0be7a39605f0e255892fd7418a682d8da8fe042fe25128794d2ed" dependencies = [ "js-sys", "wasm-bindgen", @@ -6115,9 +6085,9 @@ dependencies = [ [[package]] name = "web-time" -version = "0.2.3" +version = "0.2.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "57099a701fb3a8043f993e8228dc24229c7b942e2b009a1b962e54489ba1d3bf" +checksum = "aa30049b1c872b72c89866d458eae9f20380ab280ffd1b1e18df2d3e2d98cfe0" dependencies = [ "js-sys", "wasm-bindgen", @@ -6218,11 +6188,11 @@ checksum = "712e227841d057c1ee1cd2fb22fa7e5a5461ae8e48fa2ca79ec42cfc1931183f" [[package]] name = "windows-core" -version = "0.51.1" +version = "0.52.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f1f8cf84f35d2db49a46868f947758c7a1138116f7fac3bc844f43ade1292e64" +checksum = "33ab640c8d7e35bf8ba19b884ba838ceb4fba93a4e8c65a9059d08afcfc683d9" dependencies = [ - "windows-targets 0.48.5", + "windows-targets 0.52.0", ] [[package]] @@ -6425,9 +6395,9 @@ checksum = "dff9641d1cd4be8d1a070daf9e3773c5f67e78b4d9d42263020c057706765c04" [[package]] name = "winnow" -version = "0.5.28" +version = "0.5.34" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6c830786f7720c2fd27a1a0e27a709dbd3c4d009b56d098fc742d4f4eab91fe2" +checksum = "b7cf47b659b318dccbd69cc4797a39ae128f533dce7902a1096044d1967b9c16" dependencies = [ "memchr", ] @@ -6476,18 +6446,18 @@ checksum = "09041cd90cf85f7f8b2df60c646f853b7f535ce68f85244eb6731cf89fa498ec" [[package]] name = "zerocopy" -version = "0.7.30" +version = "0.7.32" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "306dca4455518f1f31635ec308b6b3e4eb1b11758cefafc782827d0aa7acb5c7" +checksum = "74d4d3961e53fa4c9a25a8637fc2bfaf2595b3d3ae34875568a5cf64787716be" dependencies = [ "zerocopy-derive", ] [[package]] name = "zerocopy-derive" -version = "0.7.30" +version = "0.7.32" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "be912bf68235a88fbefd1b73415cb218405958d1655b2ece9035a19920bdf6ba" +checksum = "9ce1b18ccd8e73a9321186f97e46f9f04b778851177567b1975109d26a08d2a6" dependencies = [ "proc-macro2", "quote", diff --git a/Cargo.toml b/Cargo.toml index ca5813335..4ff7eb2be 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -1,5 +1,5 @@ [workspace.package] -version = "0.19.3-rc.1" +version = "0.19.3" edition = "2021" description = "A link aggregator for the fediverse" license = "AGPL-3.0" @@ -54,6 +54,7 @@ members = [ "crates/api_common", "crates/apub", "crates/utils", + "crates/db_perf", "crates/db_schema", "crates/db_views", "crates/db_views_actor", @@ -85,25 +86,25 @@ unused_self = "deny" unwrap_used = "deny" [workspace.dependencies] -lemmy_api = { version = "=0.19.3-rc.1", path = "./crates/api" } -lemmy_api_crud = { version = "=0.19.3-rc.1", path = "./crates/api_crud" } -lemmy_apub = { version = "=0.19.3-rc.1", path = "./crates/apub" } -lemmy_utils = { version = "=0.19.3-rc.1", path = "./crates/utils" } -lemmy_db_schema = { version = "=0.19.3-rc.1", path = "./crates/db_schema" } -lemmy_api_common = { version = "=0.19.3-rc.1", path = "./crates/api_common" } -lemmy_routes = { version = "=0.19.3-rc.1", path = "./crates/routes" } -lemmy_db_views = { version = "=0.19.3-rc.1", path = "./crates/db_views" } -lemmy_db_views_actor = { version = "=0.19.3-rc.1", path = "./crates/db_views_actor" } -lemmy_db_views_moderator = { version = "=0.19.3-rc.1", path = "./crates/db_views_moderator" } +lemmy_api = { version = "=0.19.3", path = "./crates/api" } +lemmy_api_crud = { version = "=0.19.3", path = "./crates/api_crud" } +lemmy_apub = { version = "=0.19.3", path = "./crates/apub" } +lemmy_utils = { version = "=0.19.3", path = "./crates/utils" } +lemmy_db_schema = { version = "=0.19.3", path = "./crates/db_schema" } +lemmy_api_common = { version = "=0.19.3", path = "./crates/api_common" } +lemmy_routes = { version = "=0.19.3", path = "./crates/routes" } +lemmy_db_views = { version = "=0.19.3", path = "./crates/db_views" } +lemmy_db_views_actor = { version = "=0.19.3", path = "./crates/db_views_actor" } +lemmy_db_views_moderator = { version = "=0.19.3", path = "./crates/db_views_moderator" } activitypub_federation = { version = "0.5.1-beta.1", default-features = false, features = [ "actix-web", ] } diesel = "2.1.4" diesel_migrations = "2.1.0" diesel-async = "0.4.1" -serde = { version = "1.0.193", features = ["derive"] } -serde_with = "3.4.0" -actix-web = { version = "4.4.0", default-features = false, features = [ +serde = { version = "1.0.195", features = ["derive"] } +serde_with = "3.5.1" +actix-web = { version = "4.4.1", default-features = false, features = [ "macros", "rustls", "compress-brotli", @@ -117,45 +118,47 @@ tracing-error = "0.2.0" tracing-log = "0.2.0" tracing-subscriber = { version = "0.3.18", features = ["env-filter"] } url = { version = "2.5.0", features = ["serde"] } -reqwest = { version = "0.11.22", features = ["json", "blocking", "gzip"] } +reqwest = { version = "0.11.23", features = ["json", "blocking", "gzip"] } reqwest-middleware = "0.2.4" -reqwest-tracing = "0.4.6" +reqwest-tracing = "0.4.7" clokwerk = "0.4.0" doku = { version = "0.21.1", features = ["url-2"] } bcrypt = "0.15.0" -chrono = { version = "0.4.31", features = ["serde"], default-features = false } -serde_json = { version = "1.0.108", features = ["preserve_order"] } -base64 = "0.21.5" -uuid = { version = "1.6.1", features = ["serde", "v4"] } -async-trait = "0.1.74" +chrono = { version = "0.4.32", features = ["serde"], default-features = false } +serde_json = { version = "1.0.111", features = ["preserve_order"] } +base64 = "0.21.7" +uuid = { version = "1.7.0", features = ["serde", "v4"] } +async-trait = "0.1.77" captcha = "0.0.9" -anyhow = { version = "1.0.75", features = [ +anyhow = { version = "1.0.79", features = [ "backtrace", ] } # backtrace is on by default on nightly, but not stable rust -diesel_ltree = "0.3.0" -typed-builder = "0.18.0" +diesel_ltree = "0.3.1" +typed-builder = "0.18.1" serial_test = "2.0.0" -tokio = { version = "1.35.0", features = ["full"] } -regex = "1.10.2" +tokio = { version = "1.35.1", features = ["full"] } +regex = "1.10.3" once_cell = "1.19.0" diesel-derive-newtype = "2.1.0" diesel-derive-enum = { version = "2.1.0", features = ["postgres"] } strum = "0.25.0" strum_macros = "0.25.3" itertools = "0.12.0" -futures = "0.3.29" +futures = "0.3.30" http = "0.2.11" percent-encoding = "2.3.1" rosetta-i18n = "0.1.3" opentelemetry = { version = "0.19.0", features = ["rt-tokio"] } tracing-opentelemetry = { version = "0.19.0" } -ts-rs = { version = "7.0.0", features = ["serde-compat", "chrono-impl"] } +ts-rs = { version = "7.1.1", features = ["serde-compat", "chrono-impl"] } rustls = { version = "0.21.10", features = ["dangerous_configuration"] } -futures-util = "0.3.29" +futures-util = "0.3.30" tokio-postgres = "0.7.10" tokio-postgres-rustls = "0.10.0" enum-map = "2.7" -moka = { version = "0.12.1", features = ["future"] } +moka = { version = "0.12.4", features = ["future"] } +i-love-jesus = { version = "0.1.0" } +clap = { version = "4.4.18", features = ["derive"] } pretty_assertions = "1.4.0" [dependencies] @@ -166,7 +169,7 @@ lemmy_utils = { workspace = true } lemmy_db_schema = { workspace = true } lemmy_api_common = { workspace = true } lemmy_routes = { workspace = true } -lemmy_federate = { version = "0.19.3-rc.1", path = "crates/federate" } +lemmy_federate = { version = "0.19.3", path = "crates/federate" } activitypub_federation = { workspace = true } diesel = { workspace = true } diesel-async = { workspace = true } @@ -186,14 +189,14 @@ tracing-opentelemetry = { workspace = true, optional = true } opentelemetry = { workspace = true, optional = true } console-subscriber = { version = "0.1.10", optional = true } opentelemetry-otlp = { version = "0.12.0", optional = true } -pict-rs = { version = "0.5.0-rc.2", optional = true } +pict-rs = { version = "0.5.1", optional = true } tokio.workspace = true actix-cors = "0.6.5" futures-util = { workspace = true } chrono = { workspace = true } prometheus = { version = "0.13.3", features = ["process"] } serial_test = { workspace = true } -clap = { version = "4.4.11", features = ["derive"] } +clap = { workspace = true } actix-web-prom = "0.7.0" [dev-dependencies] diff --git a/api_tests/package.json b/api_tests/package.json index 4d3e57d21..94cb2df9c 100644 --- a/api_tests/package.json +++ b/api_tests/package.json @@ -27,7 +27,7 @@ "eslint": "^8.55.0", "eslint-plugin-prettier": "^5.0.1", "jest": "^29.5.0", - "lemmy-js-client": "0.19.0", + "lemmy-js-client": "0.19.2-alpha.2", "prettier": "^3.1.1", "ts-jest": "^29.1.0", "typescript": "^5.3.3" diff --git a/api_tests/src/follow.spec.ts b/api_tests/src/follow.spec.ts index 314b45eaf..0187e3ee1 100644 --- a/api_tests/src/follow.spec.ts +++ b/api_tests/src/follow.spec.ts @@ -24,21 +24,32 @@ test("Follow local community", async () => { let community = (await resolveBetaCommunity(user)).community!; expect(community.counts.subscribers).toBe(1); + expect(community.counts.subscribers_local).toBe(1); let follow = await followCommunity(user, true, community.community.id); // Make sure the follow response went through expect(follow.community_view.community.local).toBe(true); expect(follow.community_view.subscribed).toBe("Subscribed"); expect(follow.community_view.counts.subscribers).toBe(2); + expect(follow.community_view.counts.subscribers_local).toBe(2); // Test an unfollow let unfollow = await followCommunity(user, false, community.community.id); expect(unfollow.community_view.subscribed).toBe("NotSubscribed"); expect(unfollow.community_view.counts.subscribers).toBe(1); + expect(unfollow.community_view.counts.subscribers_local).toBe(1); }); test("Follow federated community", async () => { - let betaCommunity = (await resolveBetaCommunity(alpha)).community; + // It takes about 1 second for the community aggregates to federate + let betaCommunity = ( + await waitUntil( + () => resolveBetaCommunity(alpha), + c => + c.community?.counts.subscribers === 1 && + c.community.counts.subscribers_local === 0, + ) + ).community; if (!betaCommunity) { throw "Missing beta community"; } @@ -55,10 +66,12 @@ test("Follow federated community", async () => { expect(betaCommunity?.community.local).toBe(false); expect(betaCommunity?.community.name).toBe("main"); expect(betaCommunity?.subscribed).toBe("Subscribed"); + expect(betaCommunity?.counts.subscribers_local).toBe(1); // check that unfollow was federated let communityOnBeta1 = await resolveBetaCommunity(beta); expect(communityOnBeta1.community?.counts.subscribers).toBe(2); + expect(communityOnBeta1.community?.counts.subscribers_local).toBe(1); // Check it from local let site = await getSite(alpha); @@ -83,4 +96,5 @@ test("Follow federated community", async () => { // check that unfollow was federated let communityOnBeta2 = await resolveBetaCommunity(beta); expect(communityOnBeta2.community?.counts.subscribers).toBe(1); + expect(communityOnBeta2.community?.counts.subscribers_local).toBe(1); }); diff --git a/api_tests/yarn.lock b/api_tests/yarn.lock index d2cf2ab26..6769952cc 100644 --- a/api_tests/yarn.lock +++ b/api_tests/yarn.lock @@ -2286,10 +2286,10 @@ kleur@^3.0.3: resolved "https://registry.yarnpkg.com/kleur/-/kleur-3.0.3.tgz#a79c9ecc86ee1ce3fa6206d1216c501f147fc07e" integrity sha512-eTIzlVOSUR+JxdDFepEYcBMtZ9Qqdef+rnzWdRZuMbOywu5tO2w2N7rqjoANZ5k9vywhL6Br1VRjUIgTQx4E8w== -lemmy-js-client@0.19.0: - version "0.19.0" - resolved "https://registry.yarnpkg.com/lemmy-js-client/-/lemmy-js-client-0.19.0.tgz#50098183264fa176784857f45665b06994b31e18" - integrity sha512-h+E8wC9RKjlToWw9+kuGFAzk4Fiaf61KqAwzvoCDAfj2L1r+YNt5EDMOggGCoRx5PlqLuIVr7BNEU46KxJfmHA== +lemmy-js-client@0.19.2-alpha.2: + version "0.19.2-alpha.2" + resolved "https://registry.yarnpkg.com/lemmy-js-client/-/lemmy-js-client-0.19.2-alpha.2.tgz#09956df6392fa7df437343d1f1576b6297537113" + integrity sha512-/RztLo4EIDQeEN51awYJfx8JcNCHecOPrM14sSJ6/qLOOxQTPFsDrd7a2WplHpj7Wf8xci2UNfW26PmnVMOPaQ== dependencies: cross-fetch "^3.1.5" form-data "^4.0.0" diff --git a/crates/api/Cargo.toml b/crates/api/Cargo.toml index 2f39166c7..66dcea19b 100644 --- a/crates/api/Cargo.toml +++ b/crates/api/Cargo.toml @@ -35,7 +35,7 @@ chrono = { workspace = true } url = { workspace = true } wav = "1.0.0" sitemap-rs = "0.2.0" -totp-rs = { version = "5.4.0", features = ["gen_secret", "otpauth"] } +totp-rs = { version = "5.5.1", features = ["gen_secret", "otpauth"] } actix-web-httpauth = "0.8.1" [dev-dependencies] diff --git a/crates/api/src/comment/list_comment_likes.rs b/crates/api/src/comment/list_comment_likes.rs index cb487c9f1..b442cce23 100644 --- a/crates/api/src/comment/list_comment_likes.rs +++ b/crates/api/src/comment/list_comment_likes.rs @@ -2,9 +2,9 @@ use actix_web::web::{Data, Json, Query}; use lemmy_api_common::{ comment::{ListCommentLikes, ListCommentLikesResponse}, context::LemmyContext, - utils::is_admin, + utils::is_mod_or_admin, }; -use lemmy_db_views::structs::{LocalUserView, VoteView}; +use lemmy_db_views::structs::{CommentView, LocalUserView, VoteView}; use lemmy_utils::error::LemmyError; /// Lists likes for a comment @@ -14,8 +14,18 @@ pub async fn list_comment_likes( context: Data, local_user_view: LocalUserView, ) -> Result, LemmyError> { - // Make sure user is an admin - is_admin(&local_user_view)?; + let comment_view = CommentView::read( + &mut context.pool(), + data.comment_id, + Some(local_user_view.person.id), + ) + .await?; + is_mod_or_admin( + &mut context.pool(), + &local_user_view.person, + comment_view.community.id, + ) + .await?; let comment_likes = VoteView::list_for_comment(&mut context.pool(), data.comment_id, data.page, data.limit).await?; diff --git a/crates/api/src/post/list_post_likes.rs b/crates/api/src/post/list_post_likes.rs index 0e52052df..84690a41b 100644 --- a/crates/api/src/post/list_post_likes.rs +++ b/crates/api/src/post/list_post_likes.rs @@ -2,8 +2,9 @@ use actix_web::web::{Data, Json, Query}; use lemmy_api_common::{ context::LemmyContext, post::{ListPostLikes, ListPostLikesResponse}, - utils::is_admin, + utils::is_mod_or_admin, }; +use lemmy_db_schema::{source::post::Post, traits::Crud}; use lemmy_db_views::structs::{LocalUserView, VoteView}; use lemmy_utils::error::LemmyError; @@ -14,8 +15,13 @@ pub async fn list_post_likes( context: Data, local_user_view: LocalUserView, ) -> Result, LemmyError> { - // Make sure user is an admin - is_admin(&local_user_view)?; + let post = Post::read(&mut context.pool(), data.post_id).await?; + is_mod_or_admin( + &mut context.pool(), + &local_user_view.person, + post.community_id, + ) + .await?; let post_likes = VoteView::list_for_post(&mut context.pool(), data.post_id, data.page, data.limit).await?; diff --git a/crates/api_common/Cargo.toml b/crates/api_common/Cargo.toml index 0e1ea6fb4..a9db23f86 100644 --- a/crates/api_common/Cargo.toml +++ b/crates/api_common/Cargo.toml @@ -68,7 +68,7 @@ once_cell = { workspace = true, optional = true } actix-web = { workspace = true, optional = true } jsonwebtoken = { version = "8.3.0", optional = true } # necessary for wasmt compilation -getrandom = { version = "0.2.11", features = ["js"] } +getrandom = { version = "0.2.12", features = ["js"] } enum-map = { workspace = true } [package.metadata.cargo-machete] diff --git a/crates/api_common/src/person.rs b/crates/api_common/src/person.rs index 145e00e7a..328674e1f 100644 --- a/crates/api_common/src/person.rs +++ b/crates/api_common/src/person.rs @@ -1,6 +1,7 @@ use crate::sensitive::Sensitive; use lemmy_db_schema::{ newtypes::{CommentReplyId, CommunityId, LanguageId, PersonId, PersonMentionId}, + source::site::Site, CommentSortType, ListingType, PostListingMode, @@ -172,12 +173,14 @@ pub struct GetPersonDetails { pub saved_only: Option, } +#[skip_serializing_none] #[derive(Debug, Serialize, Deserialize, Clone)] #[cfg_attr(feature = "full", derive(TS))] #[cfg_attr(feature = "full", ts(export))] /// A person's details response. pub struct GetPersonDetailsResponse { pub person_view: PersonView, + pub site: Option, pub comments: Vec, pub posts: Vec, pub moderates: Vec, diff --git a/crates/api_common/src/utils.rs b/crates/api_common/src/utils.rs index 7cc96174b..ec8e7863e 100644 --- a/crates/api_common/src/utils.rs +++ b/crates/api_common/src/utils.rs @@ -20,6 +20,7 @@ use lemmy_db_schema::{ person::{Person, PersonUpdateForm}, person_block::PersonBlock, post::{Post, PostRead}, + site::Site, }, traits::Crud, utils::DbPool, @@ -547,6 +548,18 @@ pub fn check_private_instance_and_federation_enabled( } } +/// Read the site for an actor_id. +/// +/// Used for GetCommunityResponse and GetPersonDetails +pub async fn read_site_for_actor( + actor_id: DbUrl, + context: &LemmyContext, +) -> Result, LemmyError> { + let site_id = Site::instance_actor_id_from_url(actor_id.clone().into()); + let site = Site::read_from_apub_id(&mut context.pool(), &site_id.into()).await?; + Ok(site) +} + pub async fn purge_image_posts_for_person( banned_person_id: PersonId, context: &LemmyContext, diff --git a/crates/apub/src/api/read_community.rs b/crates/apub/src/api/read_community.rs index afa6fb829..a41deb32c 100644 --- a/crates/apub/src/api/read_community.rs +++ b/crates/apub/src/api/read_community.rs @@ -4,13 +4,12 @@ use actix_web::web::{Json, Query}; use lemmy_api_common::{ community::{GetCommunity, GetCommunityResponse}, context::LemmyContext, - utils::{check_private_instance, is_mod_or_admin_opt}, + utils::{check_private_instance, is_mod_or_admin_opt, read_site_for_actor}, }; use lemmy_db_schema::source::{ actor_language::CommunityLanguage, community::Community, local_site::LocalSite, - site::Site, }; use lemmy_db_views::structs::LocalUserView; use lemmy_db_views_actor::structs::{CommunityModeratorView, CommunityView}; @@ -64,15 +63,7 @@ pub async fn get_community( .await .with_lemmy_type(LemmyErrorType::CouldntFindCommunity)?; - let site_id = Site::instance_actor_id_from_url(community_view.community.actor_id.clone().into()); - let mut site = Site::read_from_apub_id(&mut context.pool(), &site_id.into()).await?; - // no need to include metadata for local site (its already available through other endpoints). - // this also prevents us from leaking the federation private key. - if let Some(s) = &site { - if s.actor_id.domain() == Some(context.settings().hostname.as_ref()) { - site = None; - } - } + let site = read_site_for_actor(community_view.community.actor_id.clone(), &context).await?; let community_id = community_view.community.id; let discussion_languages = CommunityLanguage::read(&mut context.pool(), community_id).await?; diff --git a/crates/apub/src/api/read_person.rs b/crates/apub/src/api/read_person.rs index 107212e89..52ef1a357 100644 --- a/crates/apub/src/api/read_person.rs +++ b/crates/apub/src/api/read_person.rs @@ -4,7 +4,7 @@ use actix_web::web::{Json, Query}; use lemmy_api_common::{ context::LemmyContext, person::{GetPersonDetails, GetPersonDetailsResponse}, - utils::check_private_instance, + utils::{check_private_instance, read_site_for_actor}, }; use lemmy_db_schema::{source::person::Person, utils::post_to_comment_sort_type}; use lemmy_db_views::{ @@ -90,9 +90,12 @@ pub async fn read_person( let moderates = CommunityModeratorView::for_person(&mut context.pool(), person_details_id).await?; + let site = read_site_for_actor(person_view.person.actor_id.clone(), &context).await?; + // Return the jwt Ok(Json(GetPersonDetailsResponse { person_view, + site, moderates, comments, posts, diff --git a/crates/apub/src/api/user_settings_backup.rs b/crates/apub/src/api/user_settings_backup.rs index bbc71b52a..e42e74d30 100644 --- a/crates/apub/src/api/user_settings_backup.rs +++ b/crates/apub/src/api/user_settings_backup.rs @@ -106,10 +106,10 @@ pub async fn import_settings( let local_user_form = LocalUserUpdateForm { show_nsfw: data.settings.as_ref().map(|s| s.show_nsfw), - theme: data.settings.as_ref().map(|s| s.theme.clone()), + theme: data.settings.clone().map(|s| s.theme.clone()), default_sort_type: data.settings.as_ref().map(|s| s.default_sort_type), default_listing_type: data.settings.as_ref().map(|s| s.default_listing_type), - interface_language: data.settings.as_ref().map(|s| s.interface_language.clone()), + interface_language: data.settings.clone().map(|s| s.interface_language), show_avatars: data.settings.as_ref().map(|s| s.show_avatars), send_notifications_to_email: data .settings diff --git a/crates/apub/src/collections/community_featured.rs b/crates/apub/src/collections/community_featured.rs index b3ee54db6..b106e3d9a 100644 --- a/crates/apub/src/collections/community_featured.rs +++ b/crates/apub/src/collections/community_featured.rs @@ -15,7 +15,7 @@ use lemmy_utils::error::LemmyError; use url::Url; #[derive(Clone, Debug)] -pub(crate) struct ApubCommunityFeatured(Vec); +pub(crate) struct ApubCommunityFeatured(()); #[async_trait::async_trait] impl Collection for ApubCommunityFeatured { @@ -86,6 +86,6 @@ impl Collection for ApubCommunityFeatured { .await; // This return value is unused, so just set an empty vec - Ok(ApubCommunityFeatured(Vec::new())) + Ok(ApubCommunityFeatured(())) } } diff --git a/crates/apub/src/collections/community_follower.rs b/crates/apub/src/collections/community_follower.rs index da0e52069..a4f5debbc 100644 --- a/crates/apub/src/collections/community_follower.rs +++ b/crates/apub/src/collections/community_follower.rs @@ -15,7 +15,7 @@ use lemmy_utils::error::LemmyError; use url::Url; #[derive(Clone, Debug)] -pub(crate) struct ApubCommunityFollower(Vec<()>); +pub(crate) struct ApubCommunityFollower(()); #[async_trait::async_trait] impl Collection for ApubCommunityFollower { @@ -61,6 +61,6 @@ impl Collection for ApubCommunityFollower { ) .await?; - Ok(ApubCommunityFollower(Vec::new())) + Ok(ApubCommunityFollower(())) } } diff --git a/crates/apub/src/collections/community_moderators.rs b/crates/apub/src/collections/community_moderators.rs index fc4d09a02..4ad499ea2 100644 --- a/crates/apub/src/collections/community_moderators.rs +++ b/crates/apub/src/collections/community_moderators.rs @@ -19,7 +19,7 @@ use lemmy_utils::error::LemmyError; use url::Url; #[derive(Clone, Debug)] -pub(crate) struct ApubCommunityModerators(pub(crate) Vec); +pub(crate) struct ApubCommunityModerators(()); #[async_trait::async_trait] impl Collection for ApubCommunityModerators { @@ -96,7 +96,7 @@ impl Collection for ApubCommunityModerators { } // This return value is unused, so just set an empty vec - Ok(ApubCommunityModerators(Vec::new())) + Ok(ApubCommunityModerators(())) } } diff --git a/crates/apub/src/collections/community_outbox.rs b/crates/apub/src/collections/community_outbox.rs index 8e319403d..0799db789 100644 --- a/crates/apub/src/collections/community_outbox.rs +++ b/crates/apub/src/collections/community_outbox.rs @@ -27,7 +27,7 @@ use lemmy_utils::error::LemmyError; use url::Url; #[derive(Clone, Debug)] -pub(crate) struct ApubCommunityOutbox(Vec); +pub(crate) struct ApubCommunityOutbox(()); #[async_trait::async_trait] impl Collection for ApubCommunityOutbox { @@ -111,6 +111,6 @@ impl Collection for ApubCommunityOutbox { .await; // This return value is unused, so just set an empty vec - Ok(ApubCommunityOutbox(Vec::new())) + Ok(ApubCommunityOutbox(())) } } diff --git a/crates/db_perf/Cargo.toml b/crates/db_perf/Cargo.toml new file mode 100644 index 000000000..87d2a58ac --- /dev/null +++ b/crates/db_perf/Cargo.toml @@ -0,0 +1,23 @@ +[package] +name = "lemmy_db_perf" +version.workspace = true +edition.workspace = true +description.workspace = true +license.workspace = true +homepage.workspace = true +documentation.workspace = true +repository.workspace = true + + +[lints] +workspace = true + +[dependencies] +anyhow = { workspace = true } +clap = { workspace = true } +diesel = { workspace = true } +diesel-async = { workspace = true } +lemmy_db_schema = { workspace = true } +lemmy_db_views = { workspace = true, features = ["full"] } +lemmy_utils = { workspace = true } +tokio = { workspace = true } diff --git a/crates/db_perf/src/main.rs b/crates/db_perf/src/main.rs new file mode 100644 index 000000000..60bd30c82 --- /dev/null +++ b/crates/db_perf/src/main.rs @@ -0,0 +1,179 @@ +mod series; + +use crate::series::ValuesFromSeries; +use anyhow::Context; +use clap::Parser; +use diesel::{ + dsl::{self, sql}, + sql_types, + ExpressionMethods, + IntoSql, +}; +use diesel_async::{RunQueryDsl, SimpleAsyncConnection}; +use lemmy_db_schema::{ + schema::post, + source::{ + community::{Community, CommunityInsertForm}, + instance::Instance, + person::{Person, PersonInsertForm}, + }, + traits::Crud, + utils::{build_db_pool, get_conn, now}, + SortType, +}; +use lemmy_db_views::{post_view::PostQuery, structs::PaginationCursor}; +use lemmy_utils::error::{LemmyErrorExt2, LemmyResult}; +use std::num::NonZeroU32; + +#[derive(Parser, Debug)] +struct CmdArgs { + #[arg(long, default_value_t = 3.try_into().unwrap())] + communities: NonZeroU32, + #[arg(long, default_value_t = 3.try_into().unwrap())] + people: NonZeroU32, + #[arg(long, default_value_t = 100000.try_into().unwrap())] + posts: NonZeroU32, + #[arg(long, default_value_t = 0)] + read_post_pages: u32, + #[arg(long)] + explain_insertions: bool, +} + +#[tokio::main] +async fn main() -> anyhow::Result<()> { + let mut result = try_main().await.into_anyhow(); + if let Ok(path) = std::env::var("PGDATA") { + result = result.with_context(|| { + format!("Failed to run lemmy_db_perf (more details might be available in {path}/log)") + }); + } + result +} + +async fn try_main() -> LemmyResult<()> { + let args = CmdArgs::parse(); + let pool = &build_db_pool().await?; + let pool = &mut pool.into(); + let conn = &mut get_conn(pool).await?; + + if args.explain_insertions { + // log_nested_statements is enabled to log trigger execution + conn + .batch_execute( + "SET auto_explain.log_min_duration = 0; SET auto_explain.log_nested_statements = on;", + ) + .await?; + } + + let instance = Instance::read_or_create(&mut conn.into(), "reddit.com".to_owned()).await?; + + println!("🫃 creating {} people", args.people); + let mut person_ids = vec![]; + for i in 0..args.people.get() { + let form = PersonInsertForm::builder() + .name(format!("p{i}")) + .public_key("pubkey".to_owned()) + .instance_id(instance.id) + .build(); + person_ids.push(Person::create(&mut conn.into(), &form).await?.id); + } + + println!("🌍 creating {} communities", args.communities); + let mut community_ids = vec![]; + for i in 0..args.communities.get() { + let form = CommunityInsertForm::builder() + .name(format!("c{i}")) + .title(i.to_string()) + .instance_id(instance.id) + .build(); + community_ids.push(Community::create(&mut conn.into(), &form).await?.id); + } + + let post_batches = args.people.get() * args.communities.get(); + let posts_per_batch = args.posts.get() / post_batches; + let num_posts = post_batches * posts_per_batch; + println!( + "📜 creating {} posts ({} featured in community)", + num_posts, post_batches + ); + let mut num_inserted_posts = 0; + // TODO: progress bar + for person_id in &person_ids { + for community_id in &community_ids { + let n = dsl::insert_into(post::table) + .values(ValuesFromSeries { + start: 1, + stop: posts_per_batch.into(), + selection: ( + "AAAAAAAAAAA".into_sql::(), + person_id.into_sql::(), + community_id.into_sql::(), + series::current_value.eq(1), + now() + - sql::("make_interval(secs => ") + .bind::(series::current_value) + .sql(")"), + ), + }) + .into_columns(( + post::name, + post::creator_id, + post::community_id, + post::featured_community, + post::published, + )) + .execute(conn) + .await?; + num_inserted_posts += n; + } + } + // Make sure the println above shows the correct amount + assert_eq!(num_inserted_posts, num_posts as usize); + + // Enable auto_explain + conn + .batch_execute( + "SET auto_explain.log_min_duration = 0; SET auto_explain.log_nested_statements = off;", + ) + .await?; + + // TODO: show execution duration stats + let mut page_after = None; + for page_num in 1..=args.read_post_pages { + println!( + "👀 getting page {page_num} of posts (pagination cursor used: {})", + page_after.is_some() + ); + + // TODO: include local_user + let post_views = PostQuery { + community_id: community_ids.as_slice().first().cloned(), + sort: Some(SortType::New), + limit: Some(20), + page_after, + ..Default::default() + } + .list(&mut conn.into()) + .await?; + + if let Some(post_view) = post_views.into_iter().last() { + println!("👀 getting pagination cursor data for next page"); + let cursor_data = PaginationCursor::after_post(&post_view) + .read(&mut conn.into()) + .await?; + page_after = Some(cursor_data); + } else { + println!("👀 reached empty page"); + break; + } + } + + // Delete everything, which might prevent problems if this is not run using scripts/db_perf.sh + Instance::delete(&mut conn.into(), instance.id).await?; + + if let Ok(path) = std::env::var("PGDATA") { + println!("🪵 query plans written in {path}/log"); + } + + Ok(()) +} diff --git a/crates/db_perf/src/series.rs b/crates/db_perf/src/series.rs new file mode 100644 index 000000000..0dcddbd9f --- /dev/null +++ b/crates/db_perf/src/series.rs @@ -0,0 +1,98 @@ +use diesel::{ + dsl, + expression::{is_aggregate, ValidGrouping}, + pg::Pg, + query_builder::{AsQuery, AstPass, QueryFragment}, + result::Error, + sql_types, + AppearsOnTable, + Expression, + Insertable, + QueryId, + SelectableExpression, +}; + +/// Gererates a series of rows for insertion. +/// +/// An inclusive range is created from `start` and `stop`. A row for each number is generated using `selection`, which can be a tuple. +/// [`current_value`] is an expression that gets the current value. +/// +/// For example, if there's a `numbers` table with a `number` column, this inserts all numbers from 1 to 10 in a single statement: +/// +/// ``` +/// dsl::insert_into(numbers::table) +/// .values(ValuesFromSeries { +/// start: 1, +/// stop: 10, +/// selection: series::current_value, +/// }) +/// .into_columns(numbers::number) +/// ``` +#[derive(QueryId)] +pub struct ValuesFromSeries { + pub start: i64, + pub stop: i64, + pub selection: S, +} + +impl> QueryFragment for ValuesFromSeries { + fn walk_ast<'b>(&'b self, mut out: AstPass<'_, 'b, Pg>) -> Result<(), Error> { + self.selection.walk_ast(out.reborrow())?; + out.push_sql(" FROM generate_series("); + out.push_bind_param::(&self.start)?; + out.push_sql(", "); + out.push_bind_param::(&self.stop)?; + out.push_sql(")"); + + Ok(()) + } +} + +impl Expression for ValuesFromSeries { + type SqlType = S::SqlType; +} + +impl> AppearsOnTable for ValuesFromSeries {} + +impl> SelectableExpression for ValuesFromSeries {} + +impl> Insertable for ValuesFromSeries +where + dsl::BareSelect: AsQuery + Insertable, +{ + type Values = as Insertable>::Values; + + fn values(self) -> Self::Values { + dsl::select(self).values() + } +} + +impl> ValidGrouping<()> + for ValuesFromSeries +{ + type IsAggregate = is_aggregate::No; +} + +#[allow(non_camel_case_types)] +#[derive(QueryId, Clone, Copy, Debug)] +pub struct current_value; + +impl QueryFragment for current_value { + fn walk_ast<'b>(&'b self, mut out: AstPass<'_, 'b, Pg>) -> Result<(), Error> { + out.push_identifier("generate_series")?; + + Ok(()) + } +} + +impl Expression for current_value { + type SqlType = sql_types::BigInt; +} + +impl AppearsOnTable for current_value {} + +impl SelectableExpression for current_value {} + +impl ValidGrouping<()> for current_value { + type IsAggregate = is_aggregate::No; +} diff --git a/crates/db_schema/Cargo.toml b/crates/db_schema/Cargo.toml index 377af58b9..91ff39bfe 100644 --- a/crates/db_schema/Cargo.toml +++ b/crates/db_schema/Cargo.toml @@ -36,6 +36,7 @@ full = [ "tokio-postgres", "tokio-postgres-rustls", "rustls", + "i-love-jesus", ] [dependencies] @@ -76,6 +77,8 @@ tokio-postgres = { workspace = true, optional = true } tokio-postgres-rustls = { workspace = true, optional = true } rustls = { workspace = true, optional = true } uuid = { workspace = true, features = ["v4"] } +i-love-jesus = { workspace = true, optional = true } +anyhow = { workspace = true } [dev-dependencies] serial_test = { workspace = true } diff --git a/crates/db_schema/src/aggregates/community_aggregates.rs b/crates/db_schema/src/aggregates/community_aggregates.rs index f4202738d..334688b97 100644 --- a/crates/db_schema/src/aggregates/community_aggregates.rs +++ b/crates/db_schema/src/aggregates/community_aggregates.rs @@ -156,6 +156,7 @@ mod tests { .unwrap(); assert_eq!(2, community_aggregates_before_delete.subscribers); + assert_eq!(2, community_aggregates_before_delete.subscribers_local); assert_eq!(1, community_aggregates_before_delete.posts); assert_eq!(2, community_aggregates_before_delete.comments); @@ -164,6 +165,7 @@ mod tests { .await .unwrap(); assert_eq!(1, another_community_aggs.subscribers); + assert_eq!(1, another_community_aggs.subscribers_local); assert_eq!(0, another_community_aggs.posts); assert_eq!(0, another_community_aggs.comments); @@ -175,6 +177,7 @@ mod tests { .await .unwrap(); assert_eq!(1, after_unfollow.subscribers); + assert_eq!(1, after_unfollow.subscribers_local); // Follow again just for the later tests CommunityFollower::follow(pool, &second_person_follow) @@ -184,6 +187,7 @@ mod tests { .await .unwrap(); assert_eq!(2, after_follow_again.subscribers); + assert_eq!(2, after_follow_again.subscribers_local); // Remove a parent post (the comment count should also be 0) Post::delete(pool, inserted_post.id).await.unwrap(); @@ -201,6 +205,7 @@ mod tests { .await .unwrap(); assert_eq!(1, after_person_delete.subscribers); + assert_eq!(1, after_person_delete.subscribers_local); // This should delete all the associated rows, and fire triggers let person_num_deleted = Person::delete(pool, inserted_person.id).await.unwrap(); diff --git a/crates/db_schema/src/aggregates/structs.rs b/crates/db_schema/src/aggregates/structs.rs index 0020ecab0..45a43adf8 100644 --- a/crates/db_schema/src/aggregates/structs.rs +++ b/crates/db_schema/src/aggregates/structs.rs @@ -9,6 +9,8 @@ use crate::schema::{ site_aggregates, }; use chrono::{DateTime, Utc}; +#[cfg(feature = "full")] +use i_love_jesus::CursorKeysModule; use serde::{Deserialize, Serialize}; #[cfg(feature = "full")] use ts_rs::TS; @@ -66,6 +68,7 @@ pub struct CommunityAggregates { pub users_active_half_year: i64, #[serde(skip)] pub hot_rank: f64, + pub subscribers_local: i64, } #[derive(PartialEq, Eq, Debug, Serialize, Deserialize, Clone, Default)] @@ -92,13 +95,21 @@ pub struct PersonAggregates { #[derive(PartialEq, Debug, Serialize, Deserialize, Clone)] #[cfg_attr( feature = "full", - derive(Queryable, Selectable, Associations, Identifiable, TS) + derive( + Queryable, + Selectable, + Associations, + Identifiable, + TS, + CursorKeysModule + ) )] #[cfg_attr(feature = "full", diesel(table_name = post_aggregates))] #[cfg_attr(feature = "full", diesel(belongs_to(crate::source::post::Post)))] #[cfg_attr(feature = "full", diesel(primary_key(post_id)))] #[cfg_attr(feature = "full", diesel(check_for_backend(diesel::pg::Pg)))] #[cfg_attr(feature = "full", ts(export))] +#[cfg_attr(feature = "full", cursor_keys_module(name = post_aggregates_keys))] /// Aggregate data for a post. pub struct PostAggregates { pub post_id: PostId, diff --git a/crates/db_schema/src/impls/actor_language.rs b/crates/db_schema/src/impls/actor_language.rs index 08ebcbd05..1f8fa2e91 100644 --- a/crates/db_schema/src/impls/actor_language.rs +++ b/crates/db_schema/src/impls/actor_language.rs @@ -96,16 +96,18 @@ impl LocalUserLanguage { .execute(conn) .await?; - for l in lang_ids { - let form = LocalUserLanguageForm { + let forms = lang_ids + .into_iter() + .map(|l| LocalUserLanguageForm { local_user_id: for_local_user_id, language_id: l, - }; - insert_into(local_user_language) - .values(form) - .get_result::(conn) - .await?; - } + }) + .collect::>(); + + insert_into(local_user_language) + .values(forms) + .execute(conn) + .await?; Ok(()) }) as _ }) @@ -164,16 +166,18 @@ impl SiteLanguage { .execute(conn) .await?; - for l in lang_ids { - let form = SiteLanguageForm { + let forms = lang_ids + .into_iter() + .map(|l| SiteLanguageForm { site_id: for_site_id, language_id: l, - }; - insert_into(site_language) - .values(form) - .get_result::(conn) - .await?; - } + }) + .collect::>(); + + insert_into(site_language) + .values(forms) + .get_result::(conn) + .await?; CommunityLanguage::limit_languages(conn, instance_id).await?; diff --git a/crates/db_schema/src/schema.rs b/crates/db_schema/src/schema.rs index 990b38d6f..0ffd62302 100644 --- a/crates/db_schema/src/schema.rs +++ b/crates/db_schema/src/schema.rs @@ -199,6 +199,7 @@ diesel::table! { users_active_month -> Int8, users_active_half_year -> Int8, hot_rank -> Float8, + subscribers_local -> Int8, } } diff --git a/crates/db_schema/src/utils.rs b/crates/db_schema/src/utils.rs index d7af17544..945a00647 100644 --- a/crates/db_schema/src/utils.rs +++ b/crates/db_schema/src/utils.rs @@ -6,6 +6,7 @@ use crate::{ SortType, }; use activitypub_federation::{fetch::object_id::ObjectId, traits::Object}; +use anyhow::Context; use chrono::{DateTime, Utc}; use deadpool::Runtime; use diesel::{ @@ -13,9 +14,11 @@ use diesel::{ deserialize::FromSql, helper_types::AsExprOf, pg::Pg, + query_builder::{Query, QueryFragment}, + query_dsl::methods::LimitDsl, result::{ConnectionError, ConnectionResult, Error as DieselError, Error::QueryBuilderError}, serialize::{Output, ToSql}, - sql_types::{Text, Timestamptz}, + sql_types::{self, Text, Timestamptz}, IntoSql, PgConnection, }; @@ -29,6 +32,7 @@ use diesel_async::{ }; use diesel_migrations::EmbeddedMigrations; use futures_util::{future::BoxFuture, Future, FutureExt}; +use i_love_jesus::CursorKey; use lemmy_utils::{ error::{LemmyError, LemmyErrorExt, LemmyErrorType}, settings::SETTINGS, @@ -150,6 +154,86 @@ macro_rules! try_join_with_pool { }}; } +pub struct ReverseTimestampKey(pub K); + +impl CursorKey for ReverseTimestampKey +where + K: CursorKey, +{ + type SqlType = sql_types::BigInt; + type CursorValue = functions::reverse_timestamp_sort::HelperType; + type SqlValue = functions::reverse_timestamp_sort::HelperType; + + fn get_cursor_value(cursor: &C) -> Self::CursorValue { + functions::reverse_timestamp_sort(K::get_cursor_value(cursor)) + } + + fn get_sql_value() -> Self::SqlValue { + functions::reverse_timestamp_sort(K::get_sql_value()) + } +} + +/// Includes an SQL comment before `T`, which can be used to label auto_explain output +#[derive(QueryId)] +pub struct Commented { + comment: String, + inner: T, +} + +impl Commented { + pub fn new(inner: T) -> Self { + Commented { + comment: String::new(), + inner, + } + } + + /// Adds `text` to the comment if `condition` is true + pub fn text_if(mut self, text: &str, condition: bool) -> Self { + if condition { + if !self.comment.is_empty() { + self.comment.push_str(", "); + } + self.comment.push_str(text); + } + self + } + + /// Adds `text` to the comment + pub fn text(self, text: &str) -> Self { + self.text_if(text, true) + } +} + +impl Query for Commented { + type SqlType = T::SqlType; +} + +impl> QueryFragment for Commented { + fn walk_ast<'b>( + &'b self, + mut out: diesel::query_builder::AstPass<'_, 'b, Pg>, + ) -> Result<(), DieselError> { + for line in self.comment.lines() { + out.push_sql("\n-- "); + out.push_sql(line); + } + out.push_sql("\n"); + self.inner.walk_ast(out.reborrow()) + } +} + +impl LimitDsl for Commented { + type Output = Commented; + + fn limit(self, limit: i64) -> Self::Output { + Commented { + comment: self.comment, + inner: self.inner.limit(limit), + } + } +} + pub fn fuzzy_search(q: &str) -> String { let replaced = q.replace('%', "\\%").replace('_', "\\_").replace(' ', "%"); format!("%{replaced}%") @@ -275,15 +359,18 @@ impl ServerCertVerifier for NoCertVerifier { pub const MIGRATIONS: EmbeddedMigrations = embed_migrations!(); -fn run_migrations(db_url: &str) { +fn run_migrations(db_url: &str) -> Result<(), LemmyError> { // Needs to be a sync connection let mut conn = - PgConnection::establish(db_url).unwrap_or_else(|e| panic!("Error connecting to {db_url}: {e}")); + PgConnection::establish(db_url).with_context(|| format!("Error connecting to {db_url}"))?; + info!("Running Database migrations (This may take a long time)..."); - let _ = &mut conn + conn .run_pending_migrations(MIGRATIONS) - .unwrap_or_else(|e| panic!("Couldn't run DB Migrations: {e}")); + .map_err(|e| anyhow::anyhow!("Couldn't run DB Migrations: {e}"))?; info!("Database migrations complete."); + + Ok(()) } pub async fn build_db_pool() -> Result { @@ -304,7 +391,7 @@ pub async fn build_db_pool() -> Result { .runtime(Runtime::Tokio1) .build()?; - run_migrations(&db_url); + run_migrations(&db_url)?; Ok(pool) } @@ -357,6 +444,8 @@ pub mod functions { fn controversy_rank(upvotes: BigInt, downvotes: BigInt, score: BigInt) -> Double; } + sql_function!(fn reverse_timestamp_sort(time: Timestamptz) -> BigInt); + sql_function!(fn lower(x: Text) -> Text); // really this function is variadic, this just adds the two-argument version diff --git a/crates/db_views/Cargo.toml b/crates/db_views/Cargo.toml index 8bce6de33..3f0ba5aff 100644 --- a/crates/db_views/Cargo.toml +++ b/crates/db_views/Cargo.toml @@ -23,6 +23,7 @@ full = [ "tracing", "ts-rs", "actix-web", + "i-love-jesus", "lemmy_db_schema/full", ] @@ -37,7 +38,7 @@ serde_with = { workspace = true } tracing = { workspace = true, optional = true } ts-rs = { workspace = true, optional = true } actix-web = { workspace = true, optional = true } -url = { workspace = true } +i-love-jesus = { workspace = true, optional = true } [dev-dependencies] serial_test = { workspace = true } diff --git a/crates/db_views/src/post_view.rs b/crates/db_views/src/post_view.rs index 29e262cee..be78043da 100644 --- a/crates/db_views/src/post_view.rs +++ b/crates/db_views/src/post_view.rs @@ -3,6 +3,7 @@ use diesel::{ debug_query, dsl::{exists, not, IntervalDsl}, pg::Pg, + query_builder::AsQuery, result::Error, sql_types, BoolExpressionMethods, @@ -16,8 +17,9 @@ use diesel::{ QueryDsl, }; use diesel_async::RunQueryDsl; +use i_love_jesus::PaginatedQueryBuilder; use lemmy_db_schema::{ - aggregates::structs::PostAggregates, + aggregates::structs::{post_aggregates_keys as key, PostAggregates}, newtypes::{CommunityId, LocalUserId, PersonId, PostId}, schema::{ community, @@ -44,45 +46,19 @@ use lemmy_db_schema::{ get_conn, limit_and_offset, now, + Commented, DbConn, DbPool, ListFn, Queries, ReadFn, + ReverseTimestampKey, }, ListingType, SortType, }; use tracing::debug; -#[derive(Clone, Copy, Debug, PartialEq, Eq)] -enum Ord { - Desc, - Asc, -} - -struct PaginationCursorField { - then_order_by_desc: fn(Q) -> Q, - then_order_by_asc: fn(Q) -> Q, - le: fn(&PostAggregates) -> Box>, - ge: fn(&PostAggregates) -> Box>, - ne: fn(&PostAggregates) -> Box>, -} - -/// Returns `PaginationCursorField<_, _>` for the given name -macro_rules! field { - ($name:ident) => { - // Type inference doesn't work if normal method call syntax is used - PaginationCursorField { - then_order_by_desc: |query| QueryDsl::then_order_by(query, post_aggregates::$name.desc()), - then_order_by_asc: |query| QueryDsl::then_order_by(query, post_aggregates::$name.asc()), - le: |e| Box::new(post_aggregates::$name.le(e.$name)), - ge: |e| Box::new(post_aggregates::$name.ge(e.$name)), - ne: |e| Box::new(post_aggregates::$name.ne(e.$name)), - } - }; -} - fn queries<'a>() -> Queries< impl ReadFn<'a, PostView, (PostId, Option, bool)>, impl ListFn<'a, PostView, (PostQuery<'a>, &'a Site)>, @@ -283,7 +259,10 @@ fn queries<'a>() -> Queries< ); } - query.first::(&mut conn).await + Commented::new(query) + .text("PostView::read") + .first::(&mut conn) + .await }; let list = move |mut conn: DbConn<'a>, (options, site): (PostQuery<'a>, &'a Site)| async move { @@ -461,107 +440,81 @@ fn queries<'a>() -> Queries< query = query.filter(not(is_creator_blocked(person_id))); } - let featured_field = if options.community_id.is_none() || options.community_id_just_for_prefetch - { - field!(featured_local) - } else { - field!(featured_community) - }; - - let (main_sort, top_sort_interval) = match options.sort.unwrap_or(SortType::Hot) { - SortType::Active => ((Ord::Desc, field!(hot_rank_active)), None), - SortType::Hot => ((Ord::Desc, field!(hot_rank)), None), - SortType::Scaled => ((Ord::Desc, field!(scaled_rank)), None), - SortType::Controversial => ((Ord::Desc, field!(controversy_rank)), None), - SortType::New => ((Ord::Desc, field!(published)), None), - SortType::Old => ((Ord::Asc, field!(published)), None), - SortType::NewComments => ((Ord::Desc, field!(newest_comment_time)), None), - SortType::MostComments => ((Ord::Desc, field!(comments)), None), - SortType::TopAll => ((Ord::Desc, field!(score)), None), - SortType::TopYear => ((Ord::Desc, field!(score)), Some(1.years())), - SortType::TopMonth => ((Ord::Desc, field!(score)), Some(1.months())), - SortType::TopWeek => ((Ord::Desc, field!(score)), Some(1.weeks())), - SortType::TopDay => ((Ord::Desc, field!(score)), Some(1.days())), - SortType::TopHour => ((Ord::Desc, field!(score)), Some(1.hours())), - SortType::TopSixHour => ((Ord::Desc, field!(score)), Some(6.hours())), - SortType::TopTwelveHour => ((Ord::Desc, field!(score)), Some(12.hours())), - SortType::TopThreeMonths => ((Ord::Desc, field!(score)), Some(3.months())), - SortType::TopSixMonths => ((Ord::Desc, field!(score)), Some(6.months())), - SortType::TopNineMonths => ((Ord::Desc, field!(score)), Some(9.months())), - }; - - if let Some(interval) = top_sort_interval { - query = query.filter(post_aggregates::published.gt(now() - interval)); - } - - let sorts = [ - Some((Ord::Desc, featured_field)), - Some(main_sort), - Some((Ord::Desc, field!(post_id))), - ]; - let sorts_iter = sorts.iter().flatten(); - - // This loop does almost the same thing as sorting by and comparing tuples. If the rows were - // only sorted by 1 field called `foo` in descending order, then it would be like this: - // - // ``` - // query = query.then_order_by(foo.desc()); - // if let Some(first) = &options.page_after { - // query = query.filter(foo.le(first.foo)); - // } - // if let Some(last) = &page_before_or_equal { - // query = query.filter(foo.ge(last.foo)); - // } - // ``` - // - // If multiple rows have the same value for a sorted field, then they are - // grouped together, and the rows in that group are sorted by the next fields. - // When checking if a row is within the range determined by the cursors, a field - // that's sorted after other fields is only compared if the row and the cursor - // are in the same group created by the previous sort, which is checked by using - // `or` to skip the comparison if any previously sorted field is not equal. - for (i, (order, field)) in sorts_iter.clone().enumerate() { - // Both cursors are treated as inclusive here. `page_after` is made exclusive - // by adding `1` to the offset. - let (then_order_by_field, compare_first, compare_last) = match order { - Ord::Desc => (field.then_order_by_desc, field.le, field.ge), - Ord::Asc => (field.then_order_by_asc, field.ge, field.le), - }; - - query = then_order_by_field(query); - - for (cursor_data, compare) in [ - (&options.page_after, compare_first), - (&options.page_before_or_equal, compare_last), - ] { - let Some(cursor_data) = cursor_data else { - continue; - }; - let mut condition: Box> = - Box::new(compare(&cursor_data.0)); - - // For each field that was sorted before the current one, skip the filter by changing - // `condition` to `true` if the row's value doesn't equal the cursor's value. - for (_, other_field) in sorts_iter.clone().take(i) { - condition = Box::new(condition.or((other_field.ne)(&cursor_data.0))); - } - - query = query.filter(condition); - } - } - - let (limit, mut offset) = limit_and_offset(options.page, options.limit)?; - if options.page_after.is_some() { - // always skip exactly one post because that's the last post of the previous page - // fixing the where clause is more difficult because we'd have to change only the last order-by-where clause - // e.g. WHERE (featured_local<=, hot_rank<=, published<=) to WHERE (<=, <=, <) - offset = 1; - } + let (limit, offset) = limit_and_offset(options.page, options.limit)?; query = query.limit(limit).offset(offset); + let mut query = PaginatedQueryBuilder::new(query); + + let page_after = options.page_after.map(|c| c.0); + let page_before_or_equal = options.page_before_or_equal.map(|c| c.0); + + if options.page_back { + query = query + .before(page_after) + .after_or_equal(page_before_or_equal) + .limit_and_offset_from_end(); + } else { + query = query + .after(page_after) + .before_or_equal(page_before_or_equal); + } + + // featured posts first + query = if options.community_id.is_none() || options.community_id_just_for_prefetch { + query.then_desc(key::featured_local) + } else { + query.then_desc(key::featured_community) + }; + + let time = |interval| post_aggregates::published.gt(now() - interval); + + // then use the main sort + query = match options.sort.unwrap_or(SortType::Hot) { + SortType::Active => query.then_desc(key::hot_rank_active), + SortType::Hot => query.then_desc(key::hot_rank), + SortType::Scaled => query.then_desc(key::scaled_rank), + SortType::Controversial => query.then_desc(key::controversy_rank), + SortType::New => query.then_desc(key::published), + SortType::Old => query.then_desc(ReverseTimestampKey(key::published)), + SortType::NewComments => query.then_desc(key::newest_comment_time), + SortType::MostComments => query.then_desc(key::comments), + SortType::TopAll => query.then_desc(key::score), + SortType::TopYear => query.then_desc(key::score).filter(time(1.years())), + SortType::TopMonth => query.then_desc(key::score).filter(time(1.months())), + SortType::TopWeek => query.then_desc(key::score).filter(time(1.weeks())), + SortType::TopDay => query.then_desc(key::score).filter(time(1.days())), + SortType::TopHour => query.then_desc(key::score).filter(time(1.hours())), + SortType::TopSixHour => query.then_desc(key::score).filter(time(6.hours())), + SortType::TopTwelveHour => query.then_desc(key::score).filter(time(12.hours())), + SortType::TopThreeMonths => query.then_desc(key::score).filter(time(3.months())), + SortType::TopSixMonths => query.then_desc(key::score).filter(time(6.months())), + SortType::TopNineMonths => query.then_desc(key::score).filter(time(9.months())), + }; + + // use publish as fallback. especially useful for hot rank which reaches zero after some days. + // necessary because old posts can be fetched over federation and inserted with high post id + query = match options.sort.unwrap_or(SortType::Hot) { + // A second time-based sort would not be very useful + SortType::New | SortType::Old | SortType::NewComments => query, + _ => query.then_desc(key::published), + }; + + // finally use unique post id as tie breaker + query = query.then_desc(key::post_id); + + // Not done by debug_query + let query = query.as_query(); + debug!("Post View Query: {:?}", debug_query::(&query)); - query.load::(&mut conn).await + Commented::new(query) + .text("PostQuery::list") + .text_if( + "getting upper bound for next query", + options.community_id_just_for_prefetch, + ) + .load::(&mut conn) + .await }; Queries::new(read, list) @@ -628,6 +581,7 @@ pub struct PostQuery<'a> { pub limit: Option, pub page_after: Option, pub page_before_or_equal: Option, + pub page_back: bool, } impl<'a> PostQuery<'a> { @@ -699,9 +653,15 @@ impl<'a> PostQuery<'a> { if (v.len() as i64) < limit { Ok(Some(self.clone())) } else { - let page_before_or_equal = Some(PaginationCursorData(v.pop().expect("else case").counts)); + let item = if self.page_back { + // for backward pagination, get first element instead + v.into_iter().next() + } else { + v.pop() + }; + let limit_cursor = Some(PaginationCursorData(item.expect("else case").counts)); Ok(Some(PostQuery { - page_before_or_equal, + page_before_or_equal: limit_cursor, ..self.clone() })) } @@ -1403,15 +1363,19 @@ mod tests { } } + let options = PostQuery { + community_id: Some(inserted_community.id), + sort: Some(SortType::MostComments), + limit: Some(10), + ..Default::default() + }; + let mut listed_post_ids = vec![]; let mut page_after = None; loop { let post_listings = PostQuery { - community_id: Some(inserted_community.id), - sort: Some(SortType::MostComments), - limit: Some(10), page_after, - ..Default::default() + ..options.clone() } .list(&data.site, pool) .await?; @@ -1425,6 +1389,34 @@ mod tests { } } + // Check that backward pagination matches forward pagination + let mut listed_post_ids_forward = listed_post_ids.clone(); + let mut page_before = None; + loop { + let post_listings = PostQuery { + page_after: page_before, + page_back: true, + ..options.clone() + } + .list(pool) + .await?; + + let listed_post_ids = post_listings.iter().map(|p| p.post.id).collect::>(); + + let index = listed_post_ids_forward.len() - listed_post_ids.len(); + assert_eq!( + listed_post_ids_forward.get(index..), + listed_post_ids.get(..) + ); + listed_post_ids_forward.truncate(index); + + if let Some(p) = post_listings.into_iter().next() { + page_before = Some(PaginationCursorData(p.counts)); + } else { + break; + } + } + inserted_post_ids.sort_unstable_by_key(|id| id.0); listed_post_ids.sort_unstable_by_key(|id| id.0); diff --git a/crates/db_views/src/structs.rs b/crates/db_views/src/structs.rs index a68001a52..6ffef5fe0 100644 --- a/crates/db_views/src/structs.rs +++ b/crates/db_views/src/structs.rs @@ -101,7 +101,7 @@ pub struct PostReportView { #[derive(Serialize, Deserialize, Debug, Clone)] #[cfg_attr(feature = "full", derive(ts_rs::TS))] #[cfg_attr(feature = "full", ts(export))] -pub struct PaginationCursor(pub(crate) String); +pub struct PaginationCursor(pub String); #[skip_serializing_none] #[derive(Debug, PartialEq, Serialize, Deserialize, Clone)] diff --git a/crates/federate/src/worker.rs b/crates/federate/src/worker.rs index b57c5e8ae..0155ecd9b 100644 --- a/crates/federate/src/worker.rs +++ b/crates/federate/src/worker.rs @@ -296,7 +296,7 @@ impl InstanceWorker { } if let Some(t) = &activity.send_community_followers_of { if let Some(urls) = self.followed_communities.get(t) { - inbox_urls.extend(urls.iter().map(std::clone::Clone::clone)); + inbox_urls.extend(urls.iter().cloned()); } } inbox_urls.extend( diff --git a/crates/routes/Cargo.toml b/crates/routes/Cargo.toml index 82c786576..365bdb7a0 100644 --- a/crates/routes/Cargo.toml +++ b/crates/routes/Cargo.toml @@ -33,4 +33,4 @@ url = { workspace = true } once_cell = { workspace = true } tracing = { workspace = true } tokio = { workspace = true } -rss = "2.0.6" +rss = "2.0.7" diff --git a/crates/utils/Cargo.toml b/crates/utils/Cargo.toml index b6120026e..539219149 100644 --- a/crates/utils/Cargo.toml +++ b/crates/utils/Cargo.toml @@ -41,11 +41,11 @@ uuid = { workspace = true, features = ["serde", "v4"] } rosetta-i18n = { workspace = true } percent-encoding = { workspace = true } tokio = { workspace = true } -openssl = "0.10.61" +openssl = "0.10.63" html2text = "0.6.0" deser-hjson = "2.2.4" smart-default = "0.7.1" -lettre = { version = "0.11.2", features = ["tokio1", "tokio1-native-tls"] } +lettre = { version = "0.11.3", features = ["tokio1", "tokio1-native-tls"] } markdown-it = "0.6.0" ts-rs = { workspace = true, optional = true } enum-map = { workspace = true } diff --git a/docker/docker-compose.yml b/docker/docker-compose.yml index db0c9f41e..7b22c2420 100644 --- a/docker/docker-compose.yml +++ b/docker/docker-compose.yml @@ -25,7 +25,7 @@ services: lemmy: # use "image" to pull down an already compiled lemmy. make sure to comment out "build". - # image: dessalines/lemmy:0.19.2 + # image: dessalines/lemmy:0.19.3 # platform: linux/x86_64 # no arm64 support. uncomment platform if using m1. # use "build" to build your local lemmy server image for development. make sure to comment out "image". # run: docker compose up --build @@ -55,7 +55,7 @@ services: lemmy-ui: # use "image" to pull down an already compiled lemmy-ui. make sure to comment out "build". - image: dessalines/lemmy-ui:0.19.2 + image: dessalines/lemmy-ui:0.19.3 # platform: linux/x86_64 # no arm64 support. uncomment platform if using m1. # use "build" to build your local lemmy ui image for development. make sure to comment out "image". # run: docker compose up --build diff --git a/docker/federation/docker-compose.yml b/docker/federation/docker-compose.yml index 1f38e2b5f..814d43119 100644 --- a/docker/federation/docker-compose.yml +++ b/docker/federation/docker-compose.yml @@ -2,7 +2,7 @@ version: "3.7" x-ui-default: &ui-default init: true - image: dessalines/lemmy-ui:0.19.2 + image: dessalines/lemmy-ui:0.19.3 # assuming lemmy-ui is cloned besides lemmy directory # build: # context: ../../../lemmy-ui diff --git a/migrations/2023-12-19-210053_tolerable-batch-insert-speed/down.sql b/migrations/2023-12-19-210053_tolerable-batch-insert-speed/down.sql new file mode 100644 index 000000000..3b08a616b --- /dev/null +++ b/migrations/2023-12-19-210053_tolerable-batch-insert-speed/down.sql @@ -0,0 +1,88 @@ +CREATE OR REPLACE FUNCTION post_aggregates_post () + RETURNS TRIGGER + LANGUAGE plpgsql + AS $$ +BEGIN + IF (TG_OP = 'INSERT') THEN + INSERT INTO post_aggregates (post_id, published, newest_comment_time, newest_comment_time_necro, community_id, creator_id, instance_id) + SELECT + NEW.id, + NEW.published, + NEW.published, + NEW.published, + NEW.community_id, + NEW.creator_id, + community.instance_id + FROM + community + WHERE + NEW.community_id = community.id; + ELSIF (TG_OP = 'DELETE') THEN + DELETE FROM post_aggregates + WHERE post_id = OLD.id; + END IF; + RETURN NULL; +END +$$; + +CREATE OR REPLACE TRIGGER post_aggregates_post + AFTER INSERT OR DELETE ON post + FOR EACH ROW + EXECUTE PROCEDURE post_aggregates_post (); + +CREATE OR REPLACE TRIGGER community_aggregates_post_count + AFTER INSERT OR DELETE OR UPDATE OF removed, + deleted ON post + FOR EACH ROW + EXECUTE PROCEDURE community_aggregates_post_count (); + +DROP FUNCTION IF EXISTS community_aggregates_post_count_insert CASCADE; + +DROP FUNCTION IF EXISTS community_aggregates_post_update CASCADE; + +DROP FUNCTION IF EXISTS site_aggregates_post_update CASCADE; + +DROP FUNCTION IF EXISTS person_aggregates_post_insert CASCADE; + +CREATE OR REPLACE FUNCTION site_aggregates_post_insert () + RETURNS TRIGGER + LANGUAGE plpgsql + AS $$ +BEGIN + IF (was_restored_or_created (TG_OP, OLD, NEW)) THEN + UPDATE + site_aggregates sa + SET + posts = posts + 1 + FROM + site s + WHERE + sa.site_id = s.id; + END IF; + RETURN NULL; +END +$$; + +CREATE OR REPLACE TRIGGER site_aggregates_post_insert + AFTER INSERT OR UPDATE OF removed, + deleted ON post + FOR EACH ROW + WHEN (NEW.local = TRUE) + EXECUTE PROCEDURE site_aggregates_post_insert (); + +CREATE OR REPLACE FUNCTION generate_unique_changeme () + RETURNS text + LANGUAGE sql + AS $$ + SELECT + 'http://changeme.invalid/' || substr(md5(random()::text), 0, 25); +$$; + +CREATE OR REPLACE TRIGGER person_aggregates_post_count + AFTER INSERT OR DELETE OR UPDATE OF removed, + deleted ON post + FOR EACH ROW + EXECUTE PROCEDURE person_aggregates_post_count (); + +DROP SEQUENCE IF EXISTS changeme_seq; + diff --git a/migrations/2023-12-19-210053_tolerable-batch-insert-speed/up.sql b/migrations/2023-12-19-210053_tolerable-batch-insert-speed/up.sql new file mode 100644 index 000000000..aaa866d3a --- /dev/null +++ b/migrations/2023-12-19-210053_tolerable-batch-insert-speed/up.sql @@ -0,0 +1,166 @@ +-- Change triggers to run once per statement instead of once per row +-- post_aggregates_post trigger doesn't need to handle deletion because the post_id column has ON DELETE CASCADE +CREATE OR REPLACE FUNCTION post_aggregates_post () + RETURNS TRIGGER + LANGUAGE plpgsql + AS $$ +BEGIN + INSERT INTO post_aggregates (post_id, published, newest_comment_time, newest_comment_time_necro, community_id, creator_id, instance_id) + SELECT + id, + published, + published, + published, + community_id, + creator_id, + ( + SELECT + community.instance_id + FROM + community + WHERE + community.id = community_id + LIMIT 1) +FROM + new_post; + RETURN NULL; +END +$$; + +CREATE OR REPLACE FUNCTION community_aggregates_post_count_insert () + RETURNS TRIGGER + LANGUAGE plpgsql + AS $$ +BEGIN + UPDATE + community_aggregates + SET + posts = posts + post_group.count + FROM ( + SELECT + community_id, + count(*) + FROM + new_post + GROUP BY + community_id) post_group +WHERE + community_aggregates.community_id = post_group.community_id; + RETURN NULL; +END +$$; + +CREATE OR REPLACE FUNCTION person_aggregates_post_insert () + RETURNS TRIGGER + LANGUAGE plpgsql + AS $$ +BEGIN + UPDATE + person_aggregates + SET + post_count = post_count + post_group.count + FROM ( + SELECT + creator_id, + count(*) + FROM + new_post + GROUP BY + creator_id) post_group +WHERE + person_aggregates.person_id = post_group.creator_id; + RETURN NULL; +END +$$; + +CREATE OR REPLACE TRIGGER post_aggregates_post + AFTER INSERT ON post REFERENCING NEW TABLE AS new_post + FOR EACH STATEMENT + EXECUTE PROCEDURE post_aggregates_post (); + +-- Don't run old trigger for insert +CREATE OR REPLACE TRIGGER community_aggregates_post_count + AFTER DELETE OR UPDATE OF removed, + deleted ON post + FOR EACH ROW + EXECUTE PROCEDURE community_aggregates_post_count (); + +CREATE OR REPLACE TRIGGER community_aggregates_post_count_insert + AFTER INSERT ON post REFERENCING NEW TABLE AS new_post + FOR EACH STATEMENT + EXECUTE PROCEDURE community_aggregates_post_count_insert (); + +CREATE OR REPLACE FUNCTION site_aggregates_post_update () + RETURNS TRIGGER + LANGUAGE plpgsql + AS $$ +BEGIN + IF (was_restored_or_created (TG_OP, OLD, NEW)) THEN + UPDATE + site_aggregates sa + SET + posts = posts + 1 + FROM + site s + WHERE + sa.site_id = s.id; + END IF; + RETURN NULL; +END +$$; + +CREATE OR REPLACE FUNCTION site_aggregates_post_insert () + RETURNS TRIGGER + LANGUAGE plpgsql + AS $$ +BEGIN + UPDATE + site_aggregates sa + SET + posts = posts + ( + SELECT + count(*) + FROM + new_post) + FROM + site s + WHERE + sa.site_id = s.id; + RETURN NULL; +END +$$; + +CREATE OR REPLACE TRIGGER site_aggregates_post_update + AFTER UPDATE OF removed, + deleted ON post + FOR EACH ROW + WHEN (NEW.local = TRUE) + EXECUTE PROCEDURE site_aggregates_post_update (); + +CREATE OR REPLACE TRIGGER site_aggregates_post_insert + AFTER INSERT ON post REFERENCING NEW TABLE AS new_post + FOR EACH STATEMENT + EXECUTE PROCEDURE site_aggregates_post_insert (); + +CREATE OR REPLACE TRIGGER person_aggregates_post_count + AFTER DELETE OR UPDATE OF removed, + deleted ON post + FOR EACH ROW + EXECUTE PROCEDURE person_aggregates_post_count (); + +CREATE OR REPLACE TRIGGER person_aggregates_post_insert + AFTER INSERT ON post REFERENCING NEW TABLE AS new_post + FOR EACH STATEMENT + EXECUTE PROCEDURE person_aggregates_post_insert (); + +-- Avoid running hash function and random number generation for default ap_id +CREATE SEQUENCE IF NOT EXISTS changeme_seq AS bigint CYCLE; + +CREATE OR REPLACE FUNCTION generate_unique_changeme () + RETURNS text + LANGUAGE sql + AS $$ + SELECT + 'http://changeme.invalid/seq/' || nextval('changeme_seq')::text; +$$; + diff --git a/migrations/2023-12-22-040137_make-mixed-sorting-directions-work-with-tuple-comparison/down.sql b/migrations/2023-12-22-040137_make-mixed-sorting-directions-work-with-tuple-comparison/down.sql new file mode 100644 index 000000000..0b84b2ba1 --- /dev/null +++ b/migrations/2023-12-22-040137_make-mixed-sorting-directions-work-with-tuple-comparison/down.sql @@ -0,0 +1,4 @@ +DROP INDEX idx_post_aggregates_community_published_asc, idx_post_aggregates_featured_community_published_asc, idx_post_aggregates_featured_local_published_asc, idx_post_aggregates_published_asc; + +DROP FUNCTION reverse_timestamp_sort (t timestamp with time zone); + diff --git a/migrations/2023-12-22-040137_make-mixed-sorting-directions-work-with-tuple-comparison/up.sql b/migrations/2023-12-22-040137_make-mixed-sorting-directions-work-with-tuple-comparison/up.sql new file mode 100644 index 000000000..450a6b582 --- /dev/null +++ b/migrations/2023-12-22-040137_make-mixed-sorting-directions-work-with-tuple-comparison/up.sql @@ -0,0 +1,18 @@ +CREATE FUNCTION reverse_timestamp_sort (t timestamp with time zone) + RETURNS bigint + AS $$ +BEGIN + RETURN (-1000000 * EXTRACT(EPOCH FROM t))::bigint; +END; +$$ +LANGUAGE plpgsql +IMMUTABLE PARALLEL SAFE; + +CREATE INDEX idx_post_aggregates_community_published_asc ON public.post_aggregates USING btree (community_id, featured_local DESC, reverse_timestamp_sort (published) DESC); + +CREATE INDEX idx_post_aggregates_featured_community_published_asc ON public.post_aggregates USING btree (community_id, featured_community DESC, reverse_timestamp_sort (published) DESC); + +CREATE INDEX idx_post_aggregates_featured_local_published_asc ON public.post_aggregates USING btree (featured_local DESC, reverse_timestamp_sort (published) DESC); + +CREATE INDEX idx_post_aggregates_published_asc ON public.post_aggregates USING btree (reverse_timestamp_sort (published) DESC); + diff --git a/migrations/2024-01-05-213000_community_aggregates_add_local_subscribers/down.sql b/migrations/2024-01-05-213000_community_aggregates_add_local_subscribers/down.sql new file mode 100644 index 000000000..43f92d461 --- /dev/null +++ b/migrations/2024-01-05-213000_community_aggregates_add_local_subscribers/down.sql @@ -0,0 +1,42 @@ +ALTER TABLE community_aggregates + DROP COLUMN subscribers_local; + +-- old function from migrations/2023-10-02-145002_community_followers_count_federated/up.sql +-- The subscriber count should only be updated for local communities. For remote +-- communities it is read over federation from the origin instance. +CREATE OR REPLACE FUNCTION community_aggregates_subscriber_count () + RETURNS TRIGGER + LANGUAGE plpgsql + AS $$ +BEGIN + IF (TG_OP = 'INSERT') THEN + UPDATE + community_aggregates + SET + subscribers = subscribers + 1 + FROM + community + WHERE + community.id = community_id + AND community.local + AND community_id = NEW.community_id; + ELSIF (TG_OP = 'DELETE') THEN + UPDATE + community_aggregates + SET + subscribers = subscribers - 1 + FROM + community + WHERE + community.id = community_id + AND community.local + AND community_id = OLD.community_id; + END IF; + RETURN NULL; +END +$$; + +DROP TRIGGER IF EXISTS delete_follow_before_person ON person; + +DROP FUNCTION IF EXISTS delete_follow_before_person; + diff --git a/migrations/2024-01-05-213000_community_aggregates_add_local_subscribers/up.sql b/migrations/2024-01-05-213000_community_aggregates_add_local_subscribers/up.sql new file mode 100644 index 000000000..2ed68ea58 --- /dev/null +++ b/migrations/2024-01-05-213000_community_aggregates_add_local_subscribers/up.sql @@ -0,0 +1,81 @@ +-- Couldn't find a way to put subscribers_local right after subscribers except recreating the table. +ALTER TABLE community_aggregates + ADD COLUMN subscribers_local bigint NOT NULL DEFAULT 0; + +-- update initial value +-- update by counting local persons who follow communities. +WITH follower_counts AS ( + SELECT + community_id, + count(*) AS local_sub_count + FROM + community_follower cf + JOIN person p ON p.id = cf.person_id + WHERE + p.local = TRUE + GROUP BY + community_id) +UPDATE + community_aggregates ca +SET + subscribers_local = local_sub_count +FROM + follower_counts +WHERE + ca.community_id = follower_counts.community_id; + +-- subscribers should be updated only when a local community is followed by a local or remote person +-- subscribers_local should be updated only when a local person follows a local or remote community +CREATE OR REPLACE FUNCTION community_aggregates_subscriber_count () + RETURNS TRIGGER + LANGUAGE plpgsql + AS $$ +BEGIN + IF (TG_OP = 'INSERT') THEN + UPDATE + community_aggregates ca + SET + subscribers = subscribers + community.local::int, + subscribers_local = subscribers_local + person.local::int + FROM + community + LEFT JOIN person ON person.id = NEW.person_id + WHERE + community.id = NEW.community_id + AND community.id = ca.community_id + AND person.local IS NOT NULL; + ELSIF (TG_OP = 'DELETE') THEN + UPDATE + community_aggregates ca + SET + subscribers = subscribers - community.local::int, + subscribers_local = subscribers_local - person.local::int + FROM + community + LEFT JOIN person ON person.id = OLD.person_id + WHERE + community.id = OLD.community_id + AND community.id = ca.community_id + AND person.local IS NOT NULL; + END IF; + RETURN NULL; +END +$$; + +-- to be able to join person on the trigger above, we need to run it before the person is deleted: https://github.com/LemmyNet/lemmy/pull/4166#issuecomment-1874095856 +CREATE FUNCTION delete_follow_before_person () + RETURNS TRIGGER + LANGUAGE plpgsql + AS $$ +BEGIN + DELETE FROM community_follower AS c + WHERE c.person_id = OLD.id; + RETURN OLD; +END; +$$; + +CREATE TRIGGER delete_follow_before_person + BEFORE DELETE ON person + FOR EACH ROW + EXECUTE FUNCTION delete_follow_before_person (); + diff --git a/scripts/db_perf.sh b/scripts/db_perf.sh new file mode 100755 index 000000000..ef4b2751a --- /dev/null +++ b/scripts/db_perf.sh @@ -0,0 +1,20 @@ +#!/usr/bin/env bash + +# This script runs crates/lemmy_db_perf/src/main.rs, which lets you see info related to database query performance, such as query plans. + +set -e + +CWD="$(cd -P -- "$(dirname -- "${BASH_SOURCE[0]}")" && pwd -P)" + +cd $CWD/../ + +source scripts/start_dev_db.sh + +export LEMMY_CONFIG_LOCATION=config/config.hjson +export RUST_BACKTRACE=1 + +cargo run --package lemmy_db_perf -- "$@" + +pg_ctl stop --silent + +# $PGDATA directory is kept so log can be seen diff --git a/scripts/start_dev_db.sh b/scripts/start_dev_db.sh index f192defa6..8ea4a294e 100644 --- a/scripts/start_dev_db.sh +++ b/scripts/start_dev_db.sh @@ -2,23 +2,47 @@ export PGDATA="$PWD/dev_pgdata" export PGHOST=$PWD -export LEMMY_DATABASE_URL="postgresql://lemmy:password@/lemmy?host=$PWD" +export DATABASE_URL="postgresql://lemmy:password@/lemmy?host=$PWD" +export LEMMY_DATABASE_URL=$DATABASE_URL # If cluster exists, stop the server and delete the cluster -if [ -d $PGDATA ] +if [[ -d $PGDATA ]] then - # Prevent `stop` from failing if server already stopped - pg_ctl restart > /dev/null - pg_ctl stop + # Only stop server if it is running + pg_status_exit_code=0 + (pg_ctl status > /dev/null) || pg_status_exit_code=$? + if [[ ${pg_status_exit_code} -ne 3 ]] + then + pg_ctl stop --silent + fi + rm -rf $PGDATA fi -# Create cluster -initdb --username=postgres --auth=trust --no-instructions +config_args=( + # Only listen to socket in current directory + -c listen_addresses= + -c unix_socket_directories=$PWD -# Start server that only listens to socket in current directory -pg_ctl start --options="-c listen_addresses= -c unix_socket_directories=$PWD" > /dev/null + # Write logs to a file in $PGDATA/log + -c logging_collector=on + + # Allow auto_explain to be turned on + -c session_preload_libraries=auto_explain + + # Include actual row amounts and run times for query plan nodes + -c auto_explain.log_analyze=on + + # Don't log parameter values + -c auto_explain.log_parameter_max_length=0 +) + +# Create cluster +pg_ctl init --silent --options="--username=postgres --auth=trust --no-instructions" + +# Start server +pg_ctl start --silent --options="${config_args[*]}" # Setup database -psql -c "CREATE USER lemmy WITH PASSWORD 'password' SUPERUSER;" -U postgres -psql -c "CREATE DATABASE lemmy WITH OWNER lemmy;" -U postgres +psql --quiet -c "CREATE USER lemmy WITH PASSWORD 'password' SUPERUSER;" -U postgres +psql --quiet -c "CREATE DATABASE lemmy WITH OWNER lemmy;" -U postgres diff --git a/scripts/test.sh b/scripts/test.sh index cdfbf7611..efe9b1513 100755 --- a/scripts/test.sh +++ b/scripts/test.sh @@ -27,5 +27,5 @@ cargo test -p lemmy_utils --all-features --no-fail-fast # Add this to do printlns: -- --nocapture -pg_ctl stop +pg_ctl stop --silent rm -rf $PGDATA