diff --git a/.gitignore b/.gitignore index 3b8fac7..870bfa9 100644 --- a/.gitignore +++ b/.gitignore @@ -1,2 +1,3 @@ /target mastodon-data.toml +http-cacache diff --git a/Cargo.lock b/Cargo.lock index 2cf3572..00d4996 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -9,7 +9,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "5c192eb8f11fc081b0fe4259ba5af04217d4e0faddd02417310a927911abd7c8" dependencies = [ "crypto-common", - "generic-array", + "generic-array 0.14.6", ] [[package]] @@ -46,6 +46,127 @@ dependencies = [ "libc", ] +[[package]] +name = "anyhow" +version = "1.0.66" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "216261ddc8289130e551ddcd5ce8a064710c0d064a4d2895c67151c92b5443f6" + +[[package]] +name = "async-channel" +version = "1.7.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e14485364214912d3b19cc3435dde4df66065127f05fa0d75c712f36f12c2f28" +dependencies = [ + "concurrent-queue", + "event-listener", + "futures-core", +] + +[[package]] +name = "async-executor" +version = "1.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "871f9bb5e0a22eeb7e8cf16641feb87c9dc67032ccf8ff49e772eb9941d3a965" +dependencies = [ + "async-task", + "concurrent-queue", + "fastrand", + "futures-lite", + "once_cell", + "slab", +] + +[[package]] +name = "async-global-executor" +version = "2.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f1b6f5d7df27bd294849f8eec66ecfc63d11814df7a4f5d74168a2394467b776" +dependencies = [ + "async-channel", + "async-executor", + "async-io", + "async-lock", + "blocking", + "futures-lite", + "once_cell", +] + +[[package]] +name = "async-io" +version = "1.10.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e8121296a9f05be7f34aa4196b1747243b3b62e048bb7906f644f3fbfc490cf7" +dependencies = [ + "async-lock", + "autocfg", + "concurrent-queue", + "futures-lite", + "libc", + "log", + "parking", + "polling", + "slab", + "socket2", + "waker-fn", + "winapi", +] + +[[package]] +name = "async-lock" +version = "2.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c8101efe8695a6c17e02911402145357e718ac92d3ff88ae8419e84b1707b685" +dependencies = [ + "event-listener", + "futures-lite", +] + +[[package]] +name = "async-process" +version = "1.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "02111fd8655a613c25069ea89fc8d9bb89331fa77486eb3bc059ee757cfa481c" +dependencies = [ + "async-io", + "autocfg", + "blocking", + "cfg-if", + "event-listener", + "futures-lite", + "libc", + "once_cell", + "signal-hook", + "winapi", +] + +[[package]] +name = "async-std" +version = "1.12.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "62565bb4402e926b29953c785397c6dc0391b7b446e45008b0049eb43cec6f5d" +dependencies = [ + "async-channel", + "async-global-executor", + "async-io", + "async-lock", + "async-process", + "crossbeam-utils", + "futures-channel", + "futures-core", + "futures-io", + "futures-lite", + "gloo-timers", + "kv-log-macro", + "log", + "memchr", + "once_cell", + "pin-project-lite", + "pin-utils", + "slab", + "wasm-bindgen-futures", +] + [[package]] name = "async-stream" version = "0.3.3" @@ -67,6 +188,12 @@ dependencies = [ "syn", ] +[[package]] +name = "async-task" +version = "4.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7a40729d2133846d9ed0ea60a8b9541bccddab49cd30f0715a1da672fe9a2524" + [[package]] name = "async-trait" version = "0.1.58" @@ -78,6 +205,12 @@ dependencies = [ "syn", ] +[[package]] +name = "async_once" +version = "0.2.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2ce4f10ea3abcd6617873bae9f91d1c5332b4a778bd9ce34d0cd517474c1de82" + [[package]] name = "atom_syndication" version = "0.11.0" @@ -100,6 +233,12 @@ dependencies = [ "autocfg", ] +[[package]] +name = "atomic-waker" +version = "1.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "065374052e7df7ee4047b1160cca5e1467a12351a40b3da123c870ba0b8eda2a" + [[package]] name = "atty" version = "0.2.14" @@ -117,6 +256,15 @@ version = "1.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d468802bab17cbc0cc575e9b053f41e72aa36bfa6b7f55e3529ffa43161b97fa" +[[package]] +name = "base64" +version = "0.10.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0b25d992356d2eb0ed82172f5248873db5560c4721f564b13cb5193bda5e668e" +dependencies = [ + "byteorder", +] + [[package]] name = "base64" version = "0.13.1" @@ -129,19 +277,72 @@ version = "0.1.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "383d29d513d8764dcdc42ea295d979eb99c3c9f00607b3692cf68a431f7dca72" +[[package]] +name = "bincode" +version = "1.3.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b1f45e9417d87227c7a56d22e471c6206462cba514c7590c09aff4cf6d1ddcad" +dependencies = [ + "serde", +] + [[package]] name = "bitflags" version = "1.3.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "bef38d45163c2f1dde094a7dfd33ccf595c92905c8f8f4fdc18d06fb1037718a" +[[package]] +name = "block-buffer" +version = "0.7.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c0940dc441f31689269e10ac70eb1002a3a1d3ad1390e030043662eb7fe4688b" +dependencies = [ + "block-padding", + "byte-tools", + "byteorder", + "generic-array 0.12.4", +] + +[[package]] +name = "block-buffer" +version = "0.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4152116fd6e9dadb291ae18fc1ec3575ed6d84c29642d97890f4b4a3417297e4" +dependencies = [ + "generic-array 0.14.6", +] + [[package]] name = "block-buffer" version = "0.10.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "69cce20737498f97b993470a6e536b8523f0af7892a4f928cceb1ac5e52ebe7e" dependencies = [ - "generic-array", + "generic-array 0.14.6", +] + +[[package]] +name = "block-padding" +version = "0.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fa79dedbb091f449f1f39e53edf88d5dbe95f895dae6135a8d7b881fb5af73f5" +dependencies = [ + "byte-tools", +] + +[[package]] +name = "blocking" +version = "1.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c6ccb65d468978a086b69884437ded69a90faab3bbe6e67f242173ea728acccc" +dependencies = [ + "async-channel", + "async-task", + "atomic-waker", + "fastrand", + "futures-lite", + "once_cell", ] [[package]] @@ -150,12 +351,90 @@ version = "3.11.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "572f695136211188308f16ad2ca5c851a712c464060ae6974944458eb83880ba" +[[package]] +name = "byte-tools" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e3b5ca7a04898ad4bcd41c90c5285445ff5b791899bb1b0abdd2a2aa791211d7" + +[[package]] +name = "byteorder" +version = "1.4.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "14c189c53d098945499cdfa7ecc63567cf3886b3332b312a5b4585d8d3a6a610" + [[package]] name = "bytes" version = "1.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ec8a7b6a70fde80372154c65702f00a0f56f3e1c36abbc6c440484be248856db" +[[package]] +name = "cacache" +version = "10.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5c13caedf5b624de6448b78e980320a27266557350198dc67cf64bc8561c27e8" +dependencies = [ + "async-std", + "digest 0.9.0", + "either", + "futures", + "hex 0.4.3", + "memmap2", + "serde", + "serde_derive", + "serde_json", + "sha-1 0.9.8", + "sha2 0.9.9", + "ssri", + "tempfile", + "thiserror", + "walkdir", +] + +[[package]] +name = "cache-padded" +version = "1.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c1db59621ec70f09c5e9b597b220c7a2b43611f4710dc03ceb8748637775692c" + +[[package]] +name = "cached" +version = "0.40.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "72b4147cd94d5fbdc2ab71b11d50a2f45493625576b3bb70257f59eedea69f3d" +dependencies = [ + "async-trait", + "async_once", + "cached_proc_macro", + "cached_proc_macro_types", + "futures", + "hashbrown", + "instant", + "lazy_static", + "once_cell", + "thiserror", + "tokio", +] + +[[package]] +name = "cached_proc_macro" +version = "0.15.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "751f7f4e7a091545e7f6c65bacc404eaee7e87bfb1f9ece234a1caa173dc16f2" +dependencies = [ + "cached_proc_macro_types", + "darling 0.13.4", + "quote", + "syn", +] + +[[package]] +name = "cached_proc_macro_types" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3a4f925191b4367301851c6d99b09890311d74b0d43f274c0b34c86d308a3663" + [[package]] name = "cc" version = "1.0.73" @@ -241,6 +520,15 @@ dependencies = [ "unicode-width", ] +[[package]] +name = "concurrent-queue" +version = "1.2.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "af4780a44ab5696ea9e28294517f1fffb421a83a25af521333c838635509db9c" +dependencies = [ + "cache-padded", +] + [[package]] name = "cookie" version = "0.16.1" @@ -248,12 +536,12 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "344adc371239ef32293cb1c4fe519592fcf21206c79c02854320afcdf3ab4917" dependencies = [ "aes-gcm", - "base64", + "base64 0.13.1", "hkdf", "hmac", "percent-encoding", "rand", - "sha2", + "sha2 0.10.6", "subtle", "time 0.3.16", "version_check", @@ -296,12 +584,15 @@ name = "corobel" version = "0.4.0" dependencies = [ "atom_syndication", + "cached", "chrono", "clap", "eggbug", + "http-cache-reqwest", "once_cell", "pulldown-cmark", "reqwest", + "reqwest-middleware", "rocket", "rss", "serde", @@ -318,17 +609,36 @@ dependencies = [ "libc", ] +[[package]] +name = "crossbeam-utils" +version = "0.8.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "edbafec5fa1f196ca66527c1b12c2ec4745ca14b50f1ad8f9f6f720b55d11fac" +dependencies = [ + "cfg-if", +] + [[package]] name = "crypto-common" version = "0.1.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1bfb12502f3fc46cca1bb51ac28df9d618d813cdc3d2f25b9fe775a34af26bb3" dependencies = [ - "generic-array", + "generic-array 0.14.6", "rand_core", "typenum", ] +[[package]] +name = "ctor" +version = "0.1.26" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6d2301688392eb071b0bf1a37be05c469d3cc4dbbd95df672fe28ab021e6a096" +dependencies = [ + "quote", + "syn", +] + [[package]] name = "ctr" version = "0.9.2" @@ -388,8 +698,18 @@ version = "0.12.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "5f2c43f534ea4b0b049015d00269734195e6d3f0f6635cb692251aca6f9f8b3c" dependencies = [ - "darling_core", - "darling_macro", + "darling_core 0.12.4", + "darling_macro 0.12.4", +] + +[[package]] +name = "darling" +version = "0.13.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a01d95850c592940db9b8194bc39f4bc0e89dee5c4265e4b1807c34a9aba453c" +dependencies = [ + "darling_core 0.13.4", + "darling_macro 0.13.4", ] [[package]] @@ -406,13 +726,38 @@ dependencies = [ "syn", ] +[[package]] +name = "darling_core" +version = "0.13.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "859d65a907b6852c9361e3185c862aae7fafd2887876799fa55f5f99dc40d610" +dependencies = [ + "fnv", + "ident_case", + "proc-macro2", + "quote", + "strsim", + "syn", +] + [[package]] name = "darling_macro" version = "0.12.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "29b5acf0dea37a7f66f7b25d2c5e93fd46f8f6968b1a5d7a3e02e97768afc95a" dependencies = [ - "darling_core", + "darling_core 0.12.4", + "quote", + "syn", +] + +[[package]] +name = "darling_macro" +version = "0.13.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9c972679f83bdf9c42bd905396b6c3588a843a17f0f16dfcfa3e2c5d57441835" +dependencies = [ + "darling_core 0.13.4", "quote", "syn", ] @@ -432,7 +777,7 @@ version = "0.10.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "66e616858f6187ed828df7c64a6d71720d83767a7f19740b2d1b6fe6327b36e5" dependencies = [ - "darling", + "darling 0.12.4", "proc-macro2", "quote", "syn", @@ -492,13 +837,31 @@ dependencies = [ "syn", ] +[[package]] +name = "digest" +version = "0.8.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f3d0c8c8752312f9713efd397ff63acb9f85585afbf179282e720e7704954dd5" +dependencies = [ + "generic-array 0.12.4", +] + +[[package]] +name = "digest" +version = "0.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d3dd60d1080a57a05ab032377049e0591415d2b31afd7028356dbf3cc6dcb066" +dependencies = [ + "generic-array 0.14.6", +] + [[package]] name = "digest" version = "0.10.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "adfbc57365a37acbd2ebf2b64d7e69bb766e2fea813521ed536f5d0520dcf86c" dependencies = [ - "block-buffer", + "block-buffer 0.10.3", "crypto-common", "subtle", ] @@ -518,7 +881,7 @@ version = "0.1.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "4004d4ec28b9e2c564617596bebe2fe586d2c2baef0b34311c358ccf875530d4" dependencies = [ - "base64", + "base64 0.13.1", "bytes", "derive_more", "futures", @@ -527,7 +890,7 @@ dependencies = [ "reqwest", "serde", "serde_json", - "sha2", + "sha2 0.10.6", "thiserror", "tokio", "tokio-util", @@ -550,6 +913,18 @@ dependencies = [ "cfg-if", ] +[[package]] +name = "event-listener" +version = "2.5.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0206175f82b8d6bf6652ff7d71a1e27fd2e4efde587fd368662814d6ec1d9ce0" + +[[package]] +name = "fake-simd" +version = "0.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e88a8acf291dafb59c2d96e8f59828f3838bb1a70398823ade51a84de6a6deed" + [[package]] name = "fastrand" version = "1.8.0" @@ -611,6 +986,7 @@ checksum = "38390104763dc37a5145a53c29c63c1290b5d316d6086ec32c293f6736051bb0" dependencies = [ "futures-channel", "futures-core", + "futures-executor", "futures-io", "futures-sink", "futures-task", @@ -633,12 +1009,49 @@ version = "0.3.25" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "04909a7a7e4633ae6c4a9ab280aeb86da1236243a77b694a49eacd659a4bd3ac" +[[package]] +name = "futures-executor" +version = "0.3.25" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7acc85df6714c176ab5edf386123fafe217be88c0840ec11f199441134a074e2" +dependencies = [ + "futures-core", + "futures-task", + "futures-util", +] + [[package]] name = "futures-io" version = "0.3.25" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "00f5fb52a06bdcadeb54e8d3671f8888a39697dcb0b81b23b55174030427f4eb" +[[package]] +name = "futures-lite" +version = "1.12.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7694489acd39452c77daa48516b894c153f192c3578d5a839b62c58099fcbf48" +dependencies = [ + "fastrand", + "futures-core", + "futures-io", + "memchr", + "parking", + "pin-project-lite", + "waker-fn", +] + +[[package]] +name = "futures-macro" +version = "0.3.25" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bdfb8ce053d86b91919aad980c220b1fb8401a9394410e1c289ed7e66b61835d" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + [[package]] name = "futures-sink" version = "0.3.25" @@ -660,6 +1073,7 @@ dependencies = [ "futures-channel", "futures-core", "futures-io", + "futures-macro", "futures-sink", "futures-task", "memchr", @@ -681,6 +1095,15 @@ dependencies = [ "windows", ] +[[package]] +name = "generic-array" +version = "0.12.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ffdf9f34f1447443d37393cc6c2b8313aebddcd96906caf34e54c68d8e57d7bd" +dependencies = [ + "typenum", +] + [[package]] name = "generic-array" version = "0.14.6" @@ -717,7 +1140,7 @@ version = "0.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d930750de5717d2dd0b8c0d42c076c0e884c81a73e6cab859bbd2339c71e3e40" dependencies = [ - "opaque-debug", + "opaque-debug 0.3.0", "polyval", ] @@ -727,6 +1150,18 @@ version = "0.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9b919933a397b79c37e33b77bb2aa3dc8eb6e165ad809e58ff75bc7db2e34574" +[[package]] +name = "gloo-timers" +version = "0.2.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5fb7d06c1c8cc2a29bee7ec961009a0b2caa0793ee4900c2ffb348734ba1c8f9" +dependencies = [ + "futures-channel", + "futures-core", + "js-sys", + "wasm-bindgen", +] + [[package]] name = "h2" version = "0.3.15" @@ -767,6 +1202,18 @@ dependencies = [ "libc", ] +[[package]] +name = "hex" +version = "0.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "805026a5d0141ffc30abb3be3173848ad46a1b1664fe632428479619a3644d77" + +[[package]] +name = "hex" +version = "0.4.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7f24254aa9a54b5c858eaee2f5bccdb46aaf0e486a595ed5fd8f86ba55232a70" + [[package]] name = "hkdf" version = "0.12.3" @@ -782,7 +1229,7 @@ version = "0.12.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6c49c37c09c17a53d937dfbb742eb3a961d65a994e6bcdcf37e7399d0cc8ab5e" dependencies = [ - "digest", + "digest 0.10.5", ] [[package]] @@ -807,6 +1254,65 @@ dependencies = [ "pin-project-lite", ] +[[package]] +name = "http-cache" +version = "0.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "48834b3de285fb372571cfd0cb875a8d274ca18414fe7e91e2102919203c8789" +dependencies = [ + "anyhow", + "async-trait", + "bincode", + "cacache", + "http", + "http-cache-semantics", + "httpdate", + "miette", + "serde", + "thiserror", + "url", +] + +[[package]] +name = "http-cache-reqwest" +version = "0.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d1dac55f0fc31c4ee72789a5dc489d9ea3348f2babdd79a7c0f7b2d1b34c7a38" +dependencies = [ + "anyhow", + "async-trait", + "http", + "http-cache", + "http-cache-semantics", + "reqwest", + "reqwest-middleware", + "serde", + "task-local-extensions", + "url", +] + +[[package]] +name = "http-cache-semantics" +version = "1.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "14246388577086faaaa56fb59f0b94e288800fecfff75918a237813297cdda17" +dependencies = [ + "http", + "http-serde", + "serde", + "time 0.3.16", +] + +[[package]] +name = "http-serde" +version = "1.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0e272971f774ba29341db2f686255ff8a979365a26fb9e4277f6b6d9ec0cdd5e" +dependencies = [ + "http", + "serde", +] + [[package]] name = "httparse" version = "1.8.0" @@ -930,7 +1436,7 @@ version = "0.1.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a0c10553d664a4d0bcff9f4215d0aac67a639cc68ef660840afe309b807bc9f5" dependencies = [ - "generic-array", + "generic-array 0.14.6", ] [[package]] @@ -963,6 +1469,15 @@ dependencies = [ "wasm-bindgen", ] +[[package]] +name = "kv-log-macro" +version = "1.0.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0de8b303297635ad57c9f5059fd9cee7a47f8e8daa09df0fcd07dd39fb22977f" +dependencies = [ + "log", +] + [[package]] name = "lazy_static" version = "1.4.0" @@ -1001,6 +1516,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "abb12e687cfb44aa40f41fc3978ef76448f9b6038cad6aef4259d3c095a2382e" dependencies = [ "cfg-if", + "value-bag", ] [[package]] @@ -1039,6 +1555,38 @@ version = "2.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "2dffe52ecf27772e601905b7522cb4ef790d2cc203488bbd0e2fe85fcb74566d" +[[package]] +name = "memmap2" +version = "0.5.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "95af15f345b17af2efc8ead6080fb8bc376f8cec1b35277b935637595fe77498" +dependencies = [ + "libc", +] + +[[package]] +name = "miette" +version = "4.7.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1c90329e44f9208b55f45711f9558cec15d7ef8295cc65ecd6d4188ae8edc58c" +dependencies = [ + "miette-derive", + "once_cell", + "thiserror", + "unicode-width", +] + +[[package]] +name = "miette-derive" +version = "4.7.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6b5bc45b761bcf1b5e6e6c4128cd93b84c218721a8d9b894aa0aff4ed180174c" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + [[package]] name = "mime" version = "0.3.16" @@ -1165,6 +1713,12 @@ version = "1.16.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "86f0b0d4bf799edbc74508c1e8bf170ff5f41238e5f8225603ca7caaae2b7860" +[[package]] +name = "opaque-debug" +version = "0.2.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2839e79665f131bdb5782e51f2c6c9599c133c6098982a54c794358bf432529c" + [[package]] name = "opaque-debug" version = "0.3.0" @@ -1228,6 +1782,12 @@ version = "0.1.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b15813163c1d831bf4a13c3610c05c0d03b39feb07f7e09fa234dac9b15aaf39" +[[package]] +name = "parking" +version = "2.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "427c3892f9e783d91cc128285287e70a59e206ca452770ece88a76f7a3eddd72" + [[package]] name = "parking_lot" version = "0.12.1" @@ -1257,7 +1817,7 @@ version = "0.11.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "83a0692ec44e4cf1ef28ca317f14f8f07da2d95ec3fa01f86e4467b725e60917" dependencies = [ - "digest", + "digest 0.10.5", ] [[package]] @@ -1307,6 +1867,20 @@ version = "0.3.26" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6ac9a59f73473f1b8d852421e59e64809f025994837ef743615c6d0c5b305160" +[[package]] +name = "polling" +version = "2.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ab4609a838d88b73d8238967b60dd115cc08d38e2bbaf51ee1e4b695f89122e2" +dependencies = [ + "autocfg", + "cfg-if", + "libc", + "log", + "wepoll-ffi", + "winapi", +] + [[package]] name = "polyval" version = "0.6.0" @@ -1315,7 +1889,7 @@ checksum = "7ef234e08c11dfcb2e56f79fd70f6f2eb7f025c0ce2333e82f4f0518ecad30c6" dependencies = [ "cfg-if", "cpufeatures", - "opaque-debug", + "opaque-debug 0.3.0", "universal-hash", ] @@ -1522,7 +2096,7 @@ version = "0.11.12" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "431949c384f4e2ae07605ccaa56d1d9d2ecdb5cadd4f9577ccfab29f2e5149fc" dependencies = [ - "base64", + "base64 0.13.1", "bytes", "cookie", "cookie_store", @@ -1558,6 +2132,22 @@ dependencies = [ "winreg", ] +[[package]] +name = "reqwest-middleware" +version = "0.1.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "69539cea4148dce683bec9dc95be3f0397a9bb2c248a49c8296a9d21659a8cdd" +dependencies = [ + "anyhow", + "async-trait", + "futures", + "http", + "reqwest", + "serde", + "task-local-extensions", + "thiserror", +] + [[package]] name = "rocket" version = "0.5.0-rc.2" @@ -1663,6 +2253,15 @@ version = "1.0.11" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "4501abdff3ae82a1c1b477a17252eb69cee9e66eb915c1abaa4f44d873df9f09" +[[package]] +name = "same-file" +version = "1.0.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "93fc1dc3aaa9bfed95e02e6eadabb4baf7e3078b0bd1b4d7b6b0b68378900502" +dependencies = [ + "winapi-util", +] + [[package]] name = "schannel" version = "0.1.20" @@ -1757,6 +2356,56 @@ dependencies = [ "serde", ] +[[package]] +name = "sha-1" +version = "0.8.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f7d94d0bede923b3cea61f3f1ff57ff8cdfd77b400fb8f9998949e0cf04163df" +dependencies = [ + "block-buffer 0.7.3", + "digest 0.8.1", + "fake-simd", + "opaque-debug 0.2.3", +] + +[[package]] +name = "sha-1" +version = "0.9.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "99cd6713db3cf16b6c84e06321e049a9b9f699826e16096d23bbcc44d15d51a6" +dependencies = [ + "block-buffer 0.9.0", + "cfg-if", + "cpufeatures", + "digest 0.9.0", + "opaque-debug 0.3.0", +] + +[[package]] +name = "sha2" +version = "0.8.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a256f46ea78a0c0d9ff00077504903ac881a1dafdc20da66545699e7776b3e69" +dependencies = [ + "block-buffer 0.7.3", + "digest 0.8.1", + "fake-simd", + "opaque-debug 0.2.3", +] + +[[package]] +name = "sha2" +version = "0.9.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4d58a1e1bf39749807d89cf2d98ac2dfa0ff1cb3faa38fbb64dd88ac8013d800" +dependencies = [ + "block-buffer 0.9.0", + "cfg-if", + "cpufeatures", + "digest 0.9.0", + "opaque-debug 0.3.0", +] + [[package]] name = "sha2" version = "0.10.6" @@ -1765,7 +2414,7 @@ checksum = "82e6b795fe2e3b1e845bafcb27aa35405c4d47cdfc92af5fc8d3002f76cebdc0" dependencies = [ "cfg-if", "cpufeatures", - "digest", + "digest 0.10.5", ] [[package]] @@ -1777,6 +2426,16 @@ dependencies = [ "lazy_static", ] +[[package]] +name = "signal-hook" +version = "0.3.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a253b5e89e2698464fc26b545c9edceb338e18a89effeeecfea192c3025be29d" +dependencies = [ + "libc", + "signal-hook-registry", +] + [[package]] name = "signal-hook-registry" version = "1.4.0" @@ -1817,6 +2476,21 @@ version = "0.9.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7f6002a767bff9e83f8eeecf883ecb8011875a21ae8da43bffb817a57e78cc09" +[[package]] +name = "ssri" +version = "7.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a9cec0d388f39fbe79d7aa600e8d38053bf97b1bc8d350da7c0ba800d0f423f2" +dependencies = [ + "base64 0.10.1", + "digest 0.8.1", + "hex 0.3.2", + "serde", + "sha-1 0.8.2", + "sha2 0.8.2", + "thiserror", +] + [[package]] name = "stable-pattern" version = "0.1.0" @@ -1858,6 +2532,15 @@ dependencies = [ "unicode-ident", ] +[[package]] +name = "task-local-extensions" +version = "0.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4167afbec18ae012de40f8cf1b9bf48420abb390678c34821caa07d924941cc4" +dependencies = [ + "tokio", +] + [[package]] name = "tempfile" version = "3.3.0" @@ -2200,6 +2883,7 @@ dependencies = [ "form_urlencoded", "idna 0.3.0", "percent-encoding", + "serde", ] [[package]] @@ -2217,6 +2901,16 @@ version = "0.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "830b7e5d4d90034032940e4ace0d9a9a057e7a45cd94e6c007832e39edb82f6d" +[[package]] +name = "value-bag" +version = "1.0.0-alpha.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2209b78d1249f7e6f3293657c9779fe31ced465df091bbd433a1cf88e916ec55" +dependencies = [ + "ctor", + "version_check", +] + [[package]] name = "vcpkg" version = "0.2.15" @@ -2229,6 +2923,23 @@ version = "0.9.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "49874b5167b65d7193b8aba1567f5c7d93d001cafc34600cee003eda787e483f" +[[package]] +name = "waker-fn" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9d5b2c62b4012a3e1eca5a7e077d13b3bf498c4073e33ccd58626607748ceeca" + +[[package]] +name = "walkdir" +version = "2.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "808cf2735cd4b6866113f648b791c6adc5714537bc222d9347bb203386ffda56" +dependencies = [ + "same-file", + "winapi", + "winapi-util", +] + [[package]] name = "want" version = "0.3.0" @@ -2327,6 +3038,15 @@ dependencies = [ "wasm-bindgen", ] +[[package]] +name = "wepoll-ffi" +version = "0.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d743fdedc5c64377b5fc2bc036b01c7fd642205a0d96356034ae3404d49eb7fb" +dependencies = [ + "cc", +] + [[package]] name = "winapi" version = "0.3.9" diff --git a/Cargo.toml b/Cargo.toml index 2490136..b840944 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -18,3 +18,6 @@ chrono = { version = "0.4.22", features = [ "serde" ] } rss = { path = "./rss/", version = "2.0.1-atom-link-fix", features = [ "builders", "atom", "chrono" ] } pulldown-cmark = "0.9.2" atom_syndication = "0.11.0" +http-cache-reqwest = "0.5.0" +reqwest-middleware = "0.1.6" +cached = "0.40.0" diff --git a/README.md b/README.md index baaea59..6f7844a 100644 --- a/README.md +++ b/README.md @@ -19,7 +19,9 @@ ports to use for development and deployment. - [ ] More robust parsing (defaults for all!) - [ ] RSS feeds for tags - [x] Atom Extension pagination support -- [x] Disable pagination and just go for it lmao +- [x] Disable pagination + - [x] HTTP Cacheing + - [x] Data cacheing - [ ] Read More support - [ ] Dublin Core support - [ ] Media Envelope support diff --git a/src/cohost_account.rs b/src/cohost_account.rs index 9ca57a8..3ba25a2 100644 --- a/src/cohost_account.rs +++ b/src/cohost_account.rs @@ -3,7 +3,7 @@ use serde::Deserialize; /// The API URL from whence Cohost serves JSON project definitions pub const COHOST_ACCOUNT_API_URL: &str = "https://cohost.org/api/v1/project/"; -#[derive(Debug, Deserialize, PartialEq, Eq)] +#[derive(Debug, Clone, Deserialize, PartialEq, Eq)] pub struct CohostAccount { #[serde(rename = "projectId")] pub project_id: u64, diff --git a/src/cohost_posts.rs b/src/cohost_posts.rs index 769bc8f..8a36317 100644 --- a/src/cohost_posts.rs +++ b/src/cohost_posts.rs @@ -11,7 +11,7 @@ pub fn cohost_posts_api_url(project: impl AsRef, page: u64) -> String { // Cohost doesn't give us Next links ("rel: next") for further pages, so we'll have to ALWAYS populate the rel=next field -#[derive(Debug, Deserialize)] +#[derive(Debug, Clone, Deserialize)] pub struct CohostPostsPage { #[serde(rename = "nItems")] pub number_items: usize, @@ -22,7 +22,7 @@ pub struct CohostPostsPage { pub links: Vec, } -#[derive(Debug, Deserialize)] +#[derive(Debug, Clone, Deserialize)] pub struct CohostPost { #[serde(rename = "postId")] pub id: u64, @@ -50,7 +50,7 @@ pub struct CohostPost { pub share_tree: Vec, } -#[derive(Debug, Deserialize)] +#[derive(Debug, Clone, Deserialize)] pub struct CohostPostingProject { #[serde(rename = "projectId")] pub id: u64, @@ -70,7 +70,7 @@ pub struct CohostPostingProject { pub pronouns: String, } -#[derive(Debug, Deserialize)] +#[derive(Debug, Clone, Deserialize)] pub struct CohostPostLink { #[serde(deserialize_with = "deserialize_null_default", default)] pub href: String, diff --git a/src/main.rs b/src/main.rs index 3359029..2bae8be 100644 --- a/src/main.rs +++ b/src/main.rs @@ -3,7 +3,10 @@ use std::collections::HashMap; use std::error::Error; #[macro_use] extern crate rocket; -use reqwest::{Client, StatusCode}; +use cached::proc_macro::cached; +use http_cache_reqwest::{CACacheManager, Cache, CacheMode, HttpCache}; +use reqwest::StatusCode; +use reqwest_middleware::{ClientBuilder, ClientWithMiddleware}; use rocket::response::content::RawHtml; use rocket::serde::json::Json; @@ -40,6 +43,20 @@ fn user_agent() -> String { } static ARGS: once_cell::sync::Lazy = once_cell::sync::Lazy::new(|| Args::parse()); +static CLIENT: once_cell::sync::Lazy = once_cell::sync::Lazy::new(|| { + ClientBuilder::new( + reqwest::Client::builder() + .user_agent(user_agent()) + .build() + .unwrap(), + ) + .with(Cache(HttpCache { + mode: CacheMode::Default, + manager: CACacheManager::default(), + options: None, + })) + .build() +}); #[get("/")] fn index() -> RawHtml<&'static str> { @@ -52,13 +69,13 @@ struct MdResponse { inner: String, } -#[derive(Responder)] +#[derive(Debug, Clone, Responder)] #[response(content_type = "application/rss+xml")] struct RssResponse { inner: String, } -#[derive(Responder)] +#[derive(Debug, Responder)] #[response(content_type = "text/plain")] enum ErrorResponse { #[response(status = 404)] @@ -67,14 +84,11 @@ enum ErrorResponse { InternalError(String), } -async fn get_post_from_page( - client: &mut Client, - project_id: &str, - post_id: u64, -) -> Result { +#[cached(time = 60, result)] +async fn get_post_from_page(project_id: String, post_id: u64) -> Result { let mut page = 0; loop { - let new_page = get_page_data(client, project_id, page).await?; + let new_page = get_page_data(project_id.clone(), page).await?; if new_page.items.is_empty() { // Once there are no posts, we're done. return Err(ErrorResponse::NotFound( @@ -89,14 +103,12 @@ async fn get_post_from_page( } } -async fn get_full_post_data( - client: &mut Client, - project_id: &str, -) -> Result { +#[cached(time = 120, result)] +async fn get_full_post_data(project_id: String) -> Result { let mut page = 0; - let mut merged_page = get_page_data(client, project_id, page).await?; + let mut merged_page = get_page_data(project_id.clone(), page).await?; loop { - let mut new_page = get_page_data(client, project_id, page).await?; + let mut new_page = get_page_data(project_id.clone(), page).await?; if new_page.items.is_empty() { // Once there are no posts, we're done. break; @@ -109,14 +121,11 @@ async fn get_full_post_data( Ok(merged_page) } -async fn get_page_data( - client: &mut Client, - project_id: &str, - page: u64, -) -> Result { - let posts_url = cohost_posts_api_url(project_id, page); +// Not cached because it's never used individually. +async fn get_page_data(project_id: String, page: u64) -> Result { + let posts_url = cohost_posts_api_url(&project_id, page); eprintln!("making request to {}", posts_url); - match client.get(posts_url).send().await { + match CLIENT.get(posts_url).send().await { Ok(v) => match v.status() { StatusCode::OK => match v.json::().await { Ok(page_data) => Ok(page_data), @@ -147,35 +156,11 @@ async fn get_page_data( } } -#[get("//feed.rss")] -async fn syndication_rss_route(project: &str) -> Result { - let mut client = get_client()?; - - let project_data = get_project_data(&mut client, project).await?; - let page_data = get_full_post_data(&mut client, project).await?; - Ok(RssResponse { - inner: syndication::channel_for_posts_page(project, project_data, page_data).to_string(), - }) -} - -#[get("//")] -async fn post_md_route(project: &str, id: u64) -> Result { - let mut client = get_client()?; - - let _project_data = get_project_data(&mut client, project).await?; - let post_data = get_post_from_page(&mut client, project, id).await?; - Ok(MdResponse { - inner: post_data.plain_body, - }) -} - -async fn get_project_data( - client: &mut Client, - project_id: &str, -) -> Result { +#[cached(time = 60, result)] +async fn get_project_data(project_id: String) -> Result { let project_url = format!("{}{}", COHOST_ACCOUNT_API_URL, project_id); eprintln!("making request to {}", project_url); - match client.get(project_url).send().await { + match CLIENT.get(project_url).send().await { Ok(v) => match v.status() { StatusCode::OK => match v.json::().await { Ok(a) => Ok(a), @@ -209,15 +194,23 @@ async fn get_project_data( } } -fn get_client() -> Result { - match Client::builder().user_agent(user_agent()).build() { - Ok(v) => Ok(v), - Err(e) => { - let err = format!("Couldn't build a reqwest client: {:?}", e); - eprintln!("{}", err); - Err(ErrorResponse::InternalError(err)) - } - } +#[get("//feed.rss")] +async fn syndication_rss_route(project: String) -> Result { + let project_data = get_project_data(project.clone()).await?; + let page_data = get_full_post_data(project.clone()).await?; + Ok(RssResponse { + inner: syndication::channel_for_posts_page(project.clone(), project_data, page_data) + .to_string(), + }) +} + +#[get("//")] +async fn post_md_route(project: String, id: u64) -> Result { + let _project_data = get_project_data(project.clone()).await?; + let post_data = get_post_from_page(project.clone(), id).await?; + Ok(MdResponse { + inner: post_data.plain_body, + }) } #[get("/.well-known/webfinger?")] @@ -232,9 +225,8 @@ async fn webfinger_route( eprintln!("{}", err); return Err(ErrorResponse::InternalError(err)); } - let mut client = get_client()?; if let Some(param) = params.iter().next() { - let _project_data = get_project_data(&mut client, param.0.as_str()).await?; + let _project_data = get_project_data(param.0.clone()).await?; Ok(Json(CohostWebfingerResource::new( param.0.as_str(), &ARGS.domain,