From 45bed71c368fe3c72a182025505d6e1ca49bfdd9 Mon Sep 17 00:00:00 2001 From: Nutomic Date: Wed, 25 Oct 2023 12:54:58 +0200 Subject: [PATCH 1/6] Include prometheus in default build, remove build feature (fixes #3558) (#4071) Co-authored-by: Dessalines --- Cargo.lock | 5 +- Cargo.toml | 7 ++- crates/apub/src/protocol/mod.rs | 2 +- crates/utils/src/settings/structs.rs | 8 ++-- src/lib.rs | 13 ++---- src/prometheus_metrics.rs | 68 +++++++++------------------- 6 files changed, 37 insertions(+), 66 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index af86b65d6..1d9b2a690 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -10,8 +10,9 @@ checksum = "fe438c63458706e03479442743baae6c88256498e6431708f6dfc520a26515d3" [[package]] name = "activitypub_federation" -version = "0.5.0-beta.3" -source = "git+https://github.com/LemmyNet/activitypub-federation-rust.git?branch=webfinger-alphabets#071218396b2b1254e12ad061362befe0f17e76c9" +version = "0.5.0-beta.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9a122cf2c2adf45b164134946bc069659cd93083fab294839a3f1d794b707c17" dependencies = [ "activitystreams-kinds", "actix-web", diff --git a/Cargo.toml b/Cargo.toml index 0a01644e7..ec3be45f9 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -41,7 +41,6 @@ console = [ "reqwest-tracing/opentelemetry_0_16", ] json-log = ["tracing-subscriber/json"] -prometheus-metrics = ["prometheus", "actix-web-prom"] default = [] [workspace] @@ -70,7 +69,7 @@ lemmy_routes = { version = "=0.19.0-rc.3", path = "./crates/routes" } lemmy_db_views = { version = "=0.19.0-rc.3", path = "./crates/db_views" } lemmy_db_views_actor = { version = "=0.19.0-rc.3", path = "./crates/db_views_actor" } lemmy_db_views_moderator = { version = "=0.19.0-rc.3", path = "./crates/db_views_moderator" } -activitypub_federation = { git = "https://github.com/LemmyNet/activitypub-federation-rust.git", branch = "webfinger-alphabets", default-features = false, features = [ +activitypub_federation = { version = "0.5.0-beta.4", default-features = false, features = [ "actix-web", ] } diesel = "2.1.3" @@ -169,8 +168,8 @@ futures-util = { workspace = true } tokio-postgres = { workspace = true } tokio-postgres-rustls = { workspace = true } chrono = { workspace = true } -prometheus = { version = "0.13.3", features = ["process"], optional = true } -actix-web-prom = { version = "0.6.0", optional = true } +prometheus = { version = "0.13.3", features = ["process"] } +actix-web-prom = { version = "0.6.0" } serial_test = { workspace = true } clap = { version = "4.4.7", features = ["derive"] } actix-web-httpauth = "0.8.1" diff --git a/crates/apub/src/protocol/mod.rs b/crates/apub/src/protocol/mod.rs index dba21f99d..bcf949f02 100644 --- a/crates/apub/src/protocol/mod.rs +++ b/crates/apub/src/protocol/mod.rs @@ -74,7 +74,7 @@ impl IdOrNestedObject { pub(crate) async fn object(self, context: &Data) -> Result { match self { // TODO: move IdOrNestedObject struct to library and make fetch_object_http private - IdOrNestedObject::Id(i) => Ok(fetch_object_http(&i, context).await?), + IdOrNestedObject::Id(i) => Ok(fetch_object_http(&i, context).await?.object), IdOrNestedObject::NestedObject(o) => Ok(o), } } diff --git a/crates/utils/src/settings/structs.rs b/crates/utils/src/settings/structs.rs index aa3f852ce..c7d0cd3e6 100644 --- a/crates/utils/src/settings/structs.rs +++ b/crates/utils/src/settings/structs.rs @@ -170,11 +170,11 @@ pub struct SetupConfig { #[serde(deny_unknown_fields)] pub struct PrometheusConfig { // Address that the Prometheus metrics will be served on. - #[default(Some(IpAddr::V4(Ipv4Addr::new(127, 0, 0, 1))))] + #[default(IpAddr::V4(Ipv4Addr::new(127, 0, 0, 1)))] #[doku(example = "127.0.0.1")] - pub bind: Option, + pub bind: IpAddr, // Port that the Prometheus metrics will be served on. - #[default(Some(10002))] + #[default(10002)] #[doku(example = "10002")] - pub port: Option, + pub port: i32, } diff --git a/src/lib.rs b/src/lib.rs index 64bfa802d..774857b95 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -1,6 +1,5 @@ pub mod api_routes_http; pub mod code_migrations; -#[cfg(feature = "prometheus-metrics")] pub mod prometheus_metrics; pub mod root_span_builder; pub mod scheduled_tasks; @@ -52,6 +51,7 @@ use lemmy_utils::{ response::jsonify_plain_text_errors, settings::{structs::Settings, SETTINGS}, }; +use prometheus_metrics::serve_prometheus; use reqwest_middleware::{ClientBuilder, ClientWithMiddleware}; use reqwest_tracing::TracingMiddleware; use serde_json::json; @@ -63,12 +63,6 @@ use tracing_error::ErrorLayer; use tracing_log::LogTracer; use tracing_subscriber::{filter::Targets, layer::SubscriberExt, Layer, Registry}; use url::Url; -#[cfg(feature = "prometheus-metrics")] -use { - actix_web_prom::PrometheusMetricsBuilder, - prometheus::default_registry, - prometheus_metrics::serve_prometheus, -}; #[derive(Parser, Debug)] #[command( @@ -173,8 +167,9 @@ pub async fn start_lemmy_server(args: CmdArgs) -> Result<(), LemmyError> { let _scheduled_tasks = tokio::task::spawn(scheduled_tasks::setup(context.clone())); } - #[cfg(feature = "prometheus-metrics")] - serve_prometheus(SETTINGS.prometheus.as_ref(), context.clone()); + if let Some(prometheus) = SETTINGS.prometheus.clone() { + serve_prometheus(prometheus, context.clone())?; + } let federation_config = FederationConfig::builder() .domain(SETTINGS.hostname.clone()) diff --git a/src/prometheus_metrics.rs b/src/prometheus_metrics.rs index ad964263b..c4ab204e7 100644 --- a/src/prometheus_metrics.rs +++ b/src/prometheus_metrics.rs @@ -1,14 +1,9 @@ -// TODO: should really not unwrap everywhere here.... -#![allow(clippy::unwrap_used)] -use actix_web::{rt::System, web, App, HttpResponse, HttpServer, Responder}; +use actix_web::{rt::System, web, App, HttpServer}; use lemmy_api_common::context::LemmyContext; -use lemmy_utils::settings::structs::PrometheusConfig; +use lemmy_utils::{error::LemmyResult, settings::structs::PrometheusConfig}; use prometheus::{default_registry, Encoder, Gauge, Opts, TextEncoder}; -use std::{ - net::{IpAddr, Ipv4Addr}, - sync::Arc, - thread, -}; +use std::{sync::Arc, thread}; +use tracing::error; struct PromContext { lemmy: LemmyContext, @@ -21,23 +16,12 @@ struct DbPoolMetrics { available: Gauge, } -static DEFAULT_BIND: IpAddr = IpAddr::V4(Ipv4Addr::new(127, 0, 0, 1)); -static DEFAULT_PORT: i32 = 10002; - -pub fn serve_prometheus(config: Option<&PrometheusConfig>, lemmy_context: LemmyContext) { +pub fn serve_prometheus(config: PrometheusConfig, lemmy_context: LemmyContext) -> LemmyResult<()> { let context = Arc::new(PromContext { lemmy: lemmy_context, - db_pool_metrics: create_db_pool_metrics(), + db_pool_metrics: create_db_pool_metrics()?, }); - let (bind, port) = match config { - Some(config) => ( - config.bind.unwrap_or(DEFAULT_BIND), - config.port.unwrap_or(DEFAULT_PORT), - ), - None => (DEFAULT_BIND, DEFAULT_PORT), - }; - // spawn thread that blocks on handling requests // only mapping /metrics to a handler thread::spawn(move || { @@ -48,19 +32,20 @@ pub fn serve_prometheus(config: Option<&PrometheusConfig>, lemmy_context: LemmyC .app_data(web::Data::new(Arc::clone(&context))) .route("/metrics", web::get().to(metrics)) }) - .bind((bind, port as u16)) - .unwrap_or_else(|_| panic!("Cannot bind to {}:{}", bind, port)) + .bind((config.bind, config.port as u16)) + .unwrap_or_else(|e| panic!("Cannot bind to {}:{}: {e}", config.bind, config.port)) .run(); if let Err(err) = server.await { - eprintln!("Prometheus server error: {}", err); + error!("Prometheus server error: {err}"); } }) }); + Ok(()) } // handler for the /metrics path -async fn metrics(context: web::Data>) -> impl Responder { +async fn metrics(context: web::Data>) -> LemmyResult { // collect metrics collect_db_pool_metrics(&context).await; @@ -69,43 +54,34 @@ async fn metrics(context: web::Data>) -> impl Responder { // gather metrics from registry and encode in prometheus format let metric_families = prometheus::gather(); - encoder.encode(&metric_families, &mut buffer).unwrap(); - let output = String::from_utf8(buffer).unwrap(); + encoder.encode(&metric_families, &mut buffer)?; + let output = String::from_utf8(buffer)?; - HttpResponse::Ok().body(output) + Ok(output) } // create lemmy_db_pool_* metrics and register them with the default registry -fn create_db_pool_metrics() -> DbPoolMetrics { +fn create_db_pool_metrics() -> LemmyResult { let metrics = DbPoolMetrics { max_size: Gauge::with_opts(Opts::new( "lemmy_db_pool_max_connections", "Maximum number of connections in the pool", - )) - .unwrap(), + ))?, size: Gauge::with_opts(Opts::new( "lemmy_db_pool_connections", "Current number of connections in the pool", - )) - .unwrap(), + ))?, available: Gauge::with_opts(Opts::new( "lemmy_db_pool_available_connections", "Number of available connections in the pool", - )) - .unwrap(), + ))?, }; - default_registry() - .register(Box::new(metrics.max_size.clone())) - .unwrap(); - default_registry() - .register(Box::new(metrics.size.clone())) - .unwrap(); - default_registry() - .register(Box::new(metrics.available.clone())) - .unwrap(); + default_registry().register(Box::new(metrics.max_size.clone()))?; + default_registry().register(Box::new(metrics.size.clone()))?; + default_registry().register(Box::new(metrics.available.clone()))?; - metrics + Ok(metrics) } async fn collect_db_pool_metrics(context: &PromContext) { From 568233b06243d58e5ceaa3c9f8487bc0c19d5463 Mon Sep 17 00:00:00 2001 From: Nutomic Date: Wed, 25 Oct 2023 13:14:59 +0200 Subject: [PATCH 2/6] Resolve federated objects from other instances via redirect (fixes #3129) (#4073) * Resolve federated objects from other instances via redirect (fixes #3129) * restore domain check using library change * add test case, update apub lib --------- Co-authored-by: Dessalines --- api_tests/src/post.spec.ts | 25 +++++++++++++++++++- crates/apub/src/http/comment.rs | 4 ++-- crates/apub/src/http/mod.rs | 12 ++++++---- crates/apub/src/http/post.rs | 4 ++-- docker/federation/docker-compose.yml | 2 +- docker/federation/start-local-instances.bash | 2 +- 6 files changed, 37 insertions(+), 12 deletions(-) diff --git a/api_tests/src/post.spec.ts b/api_tests/src/post.spec.ts index 8c1f22226..1d6e90c72 100644 --- a/api_tests/src/post.spec.ts +++ b/api_tests/src/post.spec.ts @@ -39,7 +39,7 @@ import { loginUser, } from "./shared"; import { PostView } from "lemmy-js-client/dist/types/PostView"; -import { LemmyHttp } from "lemmy-js-client"; +import { LemmyHttp, ResolveObject } from "lemmy-js-client"; let betaCommunity: CommunityView | undefined; @@ -556,3 +556,26 @@ test("Report a post", async () => { expect(betaReport.original_post_body).toBe(alphaReport.original_post_body); expect(betaReport.reason).toBe(alphaReport.reason); }); + +test("Fetch post via redirect", async () => { + let alphaPost = await createPost(alpha, betaCommunity!.community.id); + expect(alphaPost.post_view.post).toBeDefined(); + // Make sure that post is liked on beta + const betaPost = await waitForPost( + beta, + alphaPost.post_view.post, + res => res?.counts.score === 1, + ); + + expect(betaPost).toBeDefined(); + expect(betaPost.post?.ap_id).toBe(alphaPost.post_view.post.ap_id); + + // Fetch post from url on beta instance instead of ap_id + let q = `http://lemmy-beta:8551/post/${betaPost.post.id}`; + let form: ResolveObject = { + q, + }; + let gammaPost = await gamma.resolveObject(form); + expect(gammaPost).toBeDefined(); + expect(gammaPost.post?.post.ap_id).toBe(alphaPost.post_view.post.ap_id); +}); diff --git a/crates/apub/src/http/comment.rs b/crates/apub/src/http/comment.rs index 931caaee4..1704066aa 100644 --- a/crates/apub/src/http/comment.rs +++ b/crates/apub/src/http/comment.rs @@ -1,5 +1,5 @@ use crate::{ - http::{create_apub_response, create_apub_tombstone_response, err_object_not_local}, + http::{create_apub_response, create_apub_tombstone_response, redirect_remote_object}, objects::comment::ApubComment, }; use activitypub_federation::{config::Data, traits::Object}; @@ -23,7 +23,7 @@ pub(crate) async fn get_apub_comment( let id = CommentId(info.comment_id.parse::()?); let comment: ApubComment = Comment::read(&mut context.pool(), id).await?.into(); if !comment.local { - Err(err_object_not_local()) + Ok(redirect_remote_object(&comment.ap_id)) } else if !comment.deleted && !comment.removed { create_apub_response(&comment.into_json(&context).await?) } else { diff --git a/crates/apub/src/http/mod.rs b/crates/apub/src/http/mod.rs index c261d9e49..7c1d8529a 100644 --- a/crates/apub/src/http/mod.rs +++ b/crates/apub/src/http/mod.rs @@ -11,10 +11,10 @@ use activitypub_federation::{ FEDERATION_CONTENT_TYPE, }; use actix_web::{web, web::Bytes, HttpRequest, HttpResponse}; -use http::StatusCode; +use http::{header::LOCATION, StatusCode}; use lemmy_api_common::context::LemmyContext; -use lemmy_db_schema::source::activity::SentActivity; -use lemmy_utils::error::{LemmyError, LemmyErrorType, LemmyResult}; +use lemmy_db_schema::{newtypes::DbUrl, source::activity::SentActivity}; +use lemmy_utils::error::{LemmyError, LemmyResult}; use serde::{Deserialize, Serialize}; use std::ops::Deref; use url::Url; @@ -64,8 +64,10 @@ fn create_apub_tombstone_response>(id: T) -> LemmyResult LemmyError { - LemmyErrorType::ObjectNotLocal.into() +fn redirect_remote_object(url: &DbUrl) -> HttpResponse { + let mut res = HttpResponse::PermanentRedirect(); + res.insert_header((LOCATION, url.as_str())); + res.finish() } #[derive(Deserialize)] diff --git a/crates/apub/src/http/post.rs b/crates/apub/src/http/post.rs index f65968f15..084637c5b 100644 --- a/crates/apub/src/http/post.rs +++ b/crates/apub/src/http/post.rs @@ -1,5 +1,5 @@ use crate::{ - http::{create_apub_response, create_apub_tombstone_response, err_object_not_local}, + http::{create_apub_response, create_apub_tombstone_response, redirect_remote_object}, objects::post::ApubPost, }; use activitypub_federation::{config::Data, traits::Object}; @@ -23,7 +23,7 @@ pub(crate) async fn get_apub_post( let id = PostId(info.post_id.parse::()?); let post: ApubPost = Post::read(&mut context.pool(), id).await?.into(); if !post.local { - Err(err_object_not_local()) + Ok(redirect_remote_object(&post.ap_id)) } else if !post.deleted && !post.removed { create_apub_response(&post.into_json(&context).await?) } else { diff --git a/docker/federation/docker-compose.yml b/docker/federation/docker-compose.yml index 634a97449..142267596 100644 --- a/docker/federation/docker-compose.yml +++ b/docker/federation/docker-compose.yml @@ -2,7 +2,7 @@ version: "3.7" x-ui-default: &ui-default init: true - image: dessalines/lemmy-ui:0.18.4 + image: dessalines/lemmy-ui:0.19.0-rc.3 # assuming lemmy-ui is cloned besides lemmy directory # build: # context: ../../../lemmy-ui diff --git a/docker/federation/start-local-instances.bash b/docker/federation/start-local-instances.bash index 716f4d45c..615790764 100755 --- a/docker/federation/start-local-instances.bash +++ b/docker/federation/start-local-instances.bash @@ -8,4 +8,4 @@ for Item in alpha beta gamma delta epsilon ; do sudo chown -R 991:991 volumes/pictrs_$Item done -sudo docker compose up +sudo docker compose up --build From 64b00ee850ce6da92eb3b35a0079b7e03ab71885 Mon Sep 17 00:00:00 2001 From: Dessalines Date: Wed, 25 Oct 2023 10:14:12 -0400 Subject: [PATCH 3/6] Dont allow bots to vote. Fixes #3940 (#4100) * Dont allow bots to vote. Fixes #3940 * Removing pointless function. --- crates/api/src/comment/like.rs | 3 ++- crates/api/src/post/like.rs | 8 +++++++- crates/api_common/src/utils.rs | 10 ++++++++++ crates/apub/src/activities/voting/vote.rs | 5 ++++- crates/utils/src/error.rs | 1 + 5 files changed, 24 insertions(+), 3 deletions(-) diff --git a/crates/api/src/comment/like.rs b/crates/api/src/comment/like.rs index e11a3e155..514c571ca 100644 --- a/crates/api/src/comment/like.rs +++ b/crates/api/src/comment/like.rs @@ -5,7 +5,7 @@ use lemmy_api_common::{ comment::{CommentResponse, CreateCommentLike}, context::LemmyContext, send_activity::{ActivityChannel, SendActivityData}, - utils::{check_community_user_action, check_downvotes_enabled}, + utils::{check_bot_account, check_community_user_action, check_downvotes_enabled}, }; use lemmy_db_schema::{ newtypes::LocalUserId, @@ -32,6 +32,7 @@ pub async fn like_comment( // Don't do a downvote if site has downvotes disabled check_downvotes_enabled(data.score, &local_site)?; + check_bot_account(&local_user_view.person)?; let comment_id = data.comment_id; let orig_comment = CommentView::read(&mut context.pool(), comment_id, None).await?; diff --git a/crates/api/src/post/like.rs b/crates/api/src/post/like.rs index 751d1b9e5..176eaae16 100644 --- a/crates/api/src/post/like.rs +++ b/crates/api/src/post/like.rs @@ -5,7 +5,12 @@ use lemmy_api_common::{ context::LemmyContext, post::{CreatePostLike, PostResponse}, send_activity::{ActivityChannel, SendActivityData}, - utils::{check_community_user_action, check_downvotes_enabled, mark_post_as_read}, + utils::{ + check_bot_account, + check_community_user_action, + check_downvotes_enabled, + mark_post_as_read, + }, }; use lemmy_db_schema::{ source::{ @@ -29,6 +34,7 @@ pub async fn like_post( // Don't do a downvote if site has downvotes disabled check_downvotes_enabled(data.score, &local_site)?; + check_bot_account(&local_user_view.person)?; // Check for a community ban let post_id = data.post_id; diff --git a/crates/api_common/src/utils.rs b/crates/api_common/src/utils.rs index 5ba9a34c3..5060f2983 100644 --- a/crates/api_common/src/utils.rs +++ b/crates/api_common/src/utils.rs @@ -247,6 +247,16 @@ pub fn check_downvotes_enabled(score: i16, local_site: &LocalSite) -> Result<(), } } +/// Dont allow bots to do certain actions, like voting +#[tracing::instrument(skip_all)] +pub fn check_bot_account(person: &Person) -> Result<(), LemmyError> { + if person.bot_account { + Err(LemmyErrorType::InvalidBotAction)? + } else { + Ok(()) + } +} + #[tracing::instrument(skip_all)] pub fn check_private_instance( local_user_view: &Option, diff --git a/crates/apub/src/activities/voting/vote.rs b/crates/apub/src/activities/voting/vote.rs index 926c29302..3dfd46fbf 100644 --- a/crates/apub/src/activities/voting/vote.rs +++ b/crates/apub/src/activities/voting/vote.rs @@ -18,7 +18,7 @@ use activitypub_federation::{ traits::{ActivityHandler, Actor}, }; use anyhow::anyhow; -use lemmy_api_common::context::LemmyContext; +use lemmy_api_common::{context::LemmyContext, utils::check_bot_account}; use lemmy_db_schema::source::local_site::LocalSite; use lemmy_utils::error::LemmyError; use url::Url; @@ -74,6 +74,9 @@ impl ActivityHandler for Vote { async fn receive(self, context: &Data) -> Result<(), LemmyError> { let actor = self.actor.dereference(context).await?; let object = self.object.dereference(context).await?; + + check_bot_account(&actor.0)?; + match object { PostOrComment::Post(p) => vote_post(&self.kind, actor, &p, context).await, PostOrComment::Comment(c) => vote_comment(&self.kind, actor, &c, context).await, diff --git a/crates/utils/src/error.rs b/crates/utils/src/error.rs index 9fa6bc508..9eddb67e8 100644 --- a/crates/utils/src/error.rs +++ b/crates/utils/src/error.rs @@ -226,6 +226,7 @@ pub enum LemmyErrorType { CommunityHasNoFollowers, BanExpirationInPast, InvalidUnixTime, + InvalidBotAction, Unknown(String), } From 1b751a8cacb8175388adbb52eac3e45c36094622 Mon Sep 17 00:00:00 2001 From: Dessalines Date: Wed, 25 Oct 2023 10:46:34 -0400 Subject: [PATCH 4/6] Adding cors_origin to settings. Fixes #3665 (#4095) * Adding cors_origin to settings. Fixes #3665 * Fix result to option. * Forgot to update config defaults. * Setting a cors origin doku default. * Adding comments for CORS. --- config/defaults.hjson | 3 +++ crates/utils/src/settings/structs.rs | 5 +++++ src/lib.rs | 7 +++++-- 3 files changed, 13 insertions(+), 2 deletions(-) diff --git a/config/defaults.hjson b/config/defaults.hjson index 4b616f677..46e4e0a41 100644 --- a/config/defaults.hjson +++ b/config/defaults.hjson @@ -86,4 +86,7 @@ bind: "127.0.0.1" port: 10002 } + # Sets a response Access-Control-Allow-Origin CORS header + # https://developer.mozilla.org/en-US/docs/Web/HTTP/Headers/Access-Control-Allow-Origin + cors_origin: "*" } diff --git a/crates/utils/src/settings/structs.rs b/crates/utils/src/settings/structs.rs index c7d0cd3e6..4c39e08aa 100644 --- a/crates/utils/src/settings/structs.rs +++ b/crates/utils/src/settings/structs.rs @@ -49,6 +49,11 @@ pub struct Settings { #[default(None)] #[doku(example = "Some(Default::default())")] pub prometheus: Option, + /// Sets a response Access-Control-Allow-Origin CORS header + /// https://developer.mozilla.org/en-US/docs/Web/HTTP/Headers/Access-Control-Allow-Origin + #[default(None)] + #[doku(example = "*")] + pub cors_origin: Option, } #[derive(Debug, Deserialize, Serialize, Clone, SmartDefault, Document)] diff --git a/src/lib.rs b/src/lib.rs index 774857b95..6e62a6803 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -282,11 +282,14 @@ fn create_http_server( let context: LemmyContext = federation_config.deref().clone(); let rate_limit_cell = federation_config.rate_limit_cell().clone(); let self_origin = settings.get_protocol_and_hostname(); + let cors_origin_setting = settings.cors_origin; // Create Http server with websocket support let server = HttpServer::new(move || { - let cors_origin = env::var("LEMMY_CORS_ORIGIN"); + let cors_origin = env::var("LEMMY_CORS_ORIGIN") + .ok() + .or(cors_origin_setting.clone()); let cors_config = match (cors_origin, cfg!(debug_assertions)) { - (Ok(origin), false) => Cors::default() + (Some(origin), false) => Cors::default() .allowed_origin(&origin) .allowed_origin(&self_origin), _ => Cors::default() From b63836b31bf349bb524be8d312825252e277ab7f Mon Sep 17 00:00:00 2001 From: Dessalines Date: Wed, 25 Oct 2023 10:50:56 -0400 Subject: [PATCH 5/6] Add link to githubs new issue button for security advisories. Fixes #3734 (#4107) --- SECURITY.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/SECURITY.md b/SECURITY.md index 0bb85174e..1e1750489 100644 --- a/SECURITY.md +++ b/SECURITY.md @@ -2,4 +2,4 @@ ## Reporting a Vulnerability -Message contact at join-lemmy.org for any security-related issues. +Use [Github's security advisory issue system](https://github.com/LemmyNet/lemmy/security/advisories/new). From 08739e2925762eb7152f48032e2b170c061b5da0 Mon Sep 17 00:00:00 2001 From: Nutomic Date: Wed, 25 Oct 2023 17:34:38 +0200 Subject: [PATCH 6/6] Move usage of env::var to lemmy_utils, simplify db init (ref #4095) (#4108) --- crates/db_schema/src/utils.rs | 81 +++++++++------------------- crates/utils/src/email.rs | 5 +- crates/utils/src/settings/mod.rs | 3 ++ crates/utils/src/settings/structs.rs | 31 ++++++++--- src/lib.rs | 20 ++----- 5 files changed, 60 insertions(+), 80 deletions(-) diff --git a/crates/db_schema/src/utils.rs b/crates/db_schema/src/utils.rs index 7e83569a7..1ebdd36e2 100644 --- a/crates/db_schema/src/utils.rs +++ b/crates/db_schema/src/utils.rs @@ -30,7 +30,7 @@ use diesel_migrations::EmbeddedMigrations; use futures_util::{future::BoxFuture, Future, FutureExt}; use lemmy_utils::{ error::{LemmyError, LemmyErrorExt, LemmyErrorType}, - settings::structs::Settings, + settings::SETTINGS, }; use once_cell::sync::Lazy; use regex::Regex; @@ -39,8 +39,6 @@ use rustls::{ ServerName, }; use std::{ - env, - env::VarError, ops::{Deref, DerefMut}, sync::Arc, time::{Duration, SystemTime}, @@ -149,10 +147,6 @@ macro_rules! try_join_with_pool { }}; } -pub fn get_database_url_from_env() -> Result { - env::var("LEMMY_DATABASE_URL") -} - pub fn fuzzy_search(q: &str) -> String { let replaced = q.replace('%', "\\%").replace('_', "\\_").replace(' ', "%"); format!("%{replaced}%") @@ -238,36 +232,6 @@ pub fn diesel_option_overwrite_to_url_create( } } -async fn build_db_pool_settings_opt( - settings: Option<&Settings>, -) -> Result { - let db_url = get_database_url(settings); - let pool_size = settings.map(|s| s.database.pool_size).unwrap_or(5); - // We only support TLS with sslmode=require currently - let tls_enabled = db_url.contains("sslmode=require"); - let manager = if tls_enabled { - // diesel-async does not support any TLS connections out of the box, so we need to manually - // provide a setup function which handles creating the connection - AsyncDieselConnectionManager::::new_with_setup(&db_url, establish_connection) - } else { - AsyncDieselConnectionManager::::new(&db_url) - }; - let pool = Pool::builder(manager) - .max_size(pool_size) - .wait_timeout(POOL_TIMEOUT) - .create_timeout(POOL_TIMEOUT) - .recycle_timeout(POOL_TIMEOUT) - .runtime(Runtime::Tokio1) - .build()?; - - // If there's no settings, that means its a unit test, and migrations need to be run - if settings.is_none() { - run_migrations(&db_url); - } - - Ok(pool) -} - fn establish_connection(config: &str) -> BoxFuture> { let fut = async { let rustls_config = rustls::ClientConfig::builder() @@ -308,7 +272,7 @@ impl ServerCertVerifier for NoCertVerifier { pub const MIGRATIONS: EmbeddedMigrations = embed_migrations!(); -pub fn run_migrations(db_url: &str) { +fn run_migrations(db_url: &str) { // Needs to be a sync connection let mut conn = PgConnection::establish(db_url).unwrap_or_else(|e| panic!("Error connecting to {db_url}: {e}")); @@ -319,29 +283,36 @@ pub fn run_migrations(db_url: &str) { info!("Database migrations complete."); } -pub async fn build_db_pool(settings: &Settings) -> Result { - build_db_pool_settings_opt(Some(settings)).await +pub async fn build_db_pool() -> Result { + let db_url = SETTINGS.get_database_url(); + // We only support TLS with sslmode=require currently + let tls_enabled = db_url.contains("sslmode=require"); + let manager = if tls_enabled { + // diesel-async does not support any TLS connections out of the box, so we need to manually + // provide a setup function which handles creating the connection + AsyncDieselConnectionManager::::new_with_setup(&db_url, establish_connection) + } else { + AsyncDieselConnectionManager::::new(&db_url) + }; + let pool = Pool::builder(manager) + .max_size(SETTINGS.database.pool_size) + .wait_timeout(POOL_TIMEOUT) + .create_timeout(POOL_TIMEOUT) + .recycle_timeout(POOL_TIMEOUT) + .runtime(Runtime::Tokio1) + .build()?; + + run_migrations(&db_url); + + Ok(pool) } pub async fn build_db_pool_for_tests() -> ActualDbPool { - build_db_pool_settings_opt(None) - .await - .expect("db pool missing") -} - -pub fn get_database_url(settings: Option<&Settings>) -> String { - // The env var should override anything in the settings config - match get_database_url_from_env() { - Ok(url) => url, - Err(e) => match settings { - Some(settings) => settings.get_database_url(), - None => panic!("Failed to read database URL from env var LEMMY_DATABASE_URL: {e}"), - }, - } + build_db_pool().await.expect("db pool missing") } pub fn naive_now() -> DateTime { - chrono::prelude::Utc::now() + Utc::now() } pub fn post_to_comment_sort_type(sort: SortType) -> CommentSortType { diff --git a/crates/utils/src/email.rs b/crates/utils/src/email.rs index fdff19033..1a786b0ef 100644 --- a/crates/utils/src/email.rs +++ b/crates/utils/src/email.rs @@ -75,10 +75,7 @@ pub async fn send_email( }; // Set the creds if they exist - let smtp_password = std::env::var("LEMMY_SMTP_PASSWORD") - .ok() - .or(email_config.smtp_password); - + let smtp_password = email_config.smtp_password(); if let (Some(username), Some(password)) = (email_config.smtp_login, smtp_password) { builder = builder.credentials(Credentials::new(username, password)); } diff --git a/crates/utils/src/settings/mod.rs b/crates/utils/src/settings/mod.rs index 6b8982a11..25aa7206d 100644 --- a/crates/utils/src/settings/mod.rs +++ b/crates/utils/src/settings/mod.rs @@ -45,6 +45,9 @@ impl Settings { } pub fn get_database_url(&self) -> String { + if let Ok(url) = env::var("LEMMY_DATABASE_URL") { + return url; + } match &self.database.connection { DatabaseConnection::Uri { uri } => uri.clone(), DatabaseConnection::Parts(parts) => { diff --git a/crates/utils/src/settings/structs.rs b/crates/utils/src/settings/structs.rs index 4c39e08aa..a31b3605e 100644 --- a/crates/utils/src/settings/structs.rs +++ b/crates/utils/src/settings/structs.rs @@ -1,6 +1,9 @@ use doku::Document; use serde::{Deserialize, Serialize}; -use std::net::{IpAddr, Ipv4Addr}; +use std::{ + env, + net::{IpAddr, Ipv4Addr}, +}; use url::Url; #[derive(Debug, Deserialize, Serialize, Clone, SmartDefault, Document)] @@ -53,7 +56,15 @@ pub struct Settings { /// https://developer.mozilla.org/en-US/docs/Web/HTTP/Headers/Access-Control-Allow-Origin #[default(None)] #[doku(example = "*")] - pub cors_origin: Option, + cors_origin: Option, +} + +impl Settings { + pub fn cors_origin(&self) -> Option { + env::var("LEMMY_CORS_ORIGIN") + .ok() + .or(self.cors_origin.clone()) + } } #[derive(Debug, Deserialize, Serialize, Clone, SmartDefault, Document)] @@ -77,7 +88,7 @@ pub struct PictrsConfig { #[serde(default)] pub struct DatabaseConfig { #[serde(flatten, default)] - pub connection: DatabaseConnection, + pub(crate) connection: DatabaseConnection, /// Maximum number of active sql connections #[default(95)] @@ -122,10 +133,10 @@ pub struct DatabaseConnectionParts { pub(super) user: String, /// Password to connect to postgres #[default("password")] - pub password: String, + pub(super) password: String, #[default("localhost")] /// Host where postgres is running - pub host: String, + pub(super) host: String, /// Port where postgres can be accessed #[default(5432)] pub(super) port: i32, @@ -143,7 +154,7 @@ pub struct EmailConfig { /// Login name for smtp server pub smtp_login: Option, /// Password to login to the smtp server - pub smtp_password: Option, + smtp_password: Option, #[doku(example = "noreply@example.com")] /// Address to send emails from, eg "noreply@your-instance.com" pub smtp_from_address: String, @@ -153,6 +164,14 @@ pub struct EmailConfig { pub tls_type: String, } +impl EmailConfig { + pub fn smtp_password(&self) -> Option { + std::env::var("LEMMY_SMTP_PASSWORD") + .ok() + .or(self.smtp_password.clone()) + } +} + #[derive(Debug, Deserialize, Serialize, Clone, SmartDefault, Document)] #[serde(deny_unknown_fields)] pub struct SetupConfig { diff --git a/src/lib.rs b/src/lib.rs index 6e62a6803..ec2a8fdae 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -39,10 +39,7 @@ use lemmy_apub::{ VerifyUrlData, FEDERATION_HTTP_FETCH_LIMIT, }; -use lemmy_db_schema::{ - source::secret::Secret, - utils::{build_db_pool, get_database_url, run_migrations}, -}; +use lemmy_db_schema::{source::secret::Secret, utils::build_db_pool}; use lemmy_federate::{start_stop_federation_workers_cancellable, Opts}; use lemmy_routes::{feeds, images, nodeinfo, webfinger}; use lemmy_utils::{ @@ -114,12 +111,8 @@ pub async fn start_lemmy_server(args: CmdArgs) -> Result<(), LemmyError> { startup_server_handle = Some(create_startup_server()?); } - // Run the DB migrations - let db_url = get_database_url(Some(&SETTINGS)); - run_migrations(&db_url); - // Set up the connection pool - let pool = build_db_pool(&SETTINGS).await?; + let pool = build_db_pool().await?; // Run the Code-required migrations run_advanced_migrations(&mut (&pool).into(), &SETTINGS).await?; @@ -282,13 +275,10 @@ fn create_http_server( let context: LemmyContext = federation_config.deref().clone(); let rate_limit_cell = federation_config.rate_limit_cell().clone(); let self_origin = settings.get_protocol_and_hostname(); - let cors_origin_setting = settings.cors_origin; + let cors_origin_setting = settings.cors_origin(); // Create Http server with websocket support let server = HttpServer::new(move || { - let cors_origin = env::var("LEMMY_CORS_ORIGIN") - .ok() - .or(cors_origin_setting.clone()); - let cors_config = match (cors_origin, cfg!(debug_assertions)) { + let cors_config = match (cors_origin_setting.clone(), cfg!(debug_assertions)) { (Some(origin), false) => Cors::default() .allowed_origin(&origin) .allowed_origin(&self_origin), @@ -341,7 +331,7 @@ fn create_http_server( pub fn init_logging(opentelemetry_url: &Option) -> Result<(), LemmyError> { LogTracer::init()?; - let log_description = std::env::var("RUST_LOG").unwrap_or_else(|_| "info".into()); + let log_description = env::var("RUST_LOG").unwrap_or_else(|_| "info".into()); let targets = log_description .trim()