gsathecat
/
lemmy
mirror of https://github.com/LemmyNet/lemmy.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Merge remote-tracking branch 'origin/main' into federation-send-parallel

federation-send-parallel
phiresky 2024-06-24 16:35:57 +02:00
parent 2dd7b71590 d09854a722
commit 2e2345e6f7
96 changed files with 2003 additions and 1407 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

763
Cargo.lock generated
View File

File diff suppressed because it is too large Load Diff

41
Cargo.toml
View File

@ -1,5 +1,5 @@
[workspace.package]
version = "0.19.4-rc.5"
version = "0.19.5"
edition = "2021"
description = "A link aggregator for the fediverse"
license = "AGPL-3.0"
@ -88,24 +88,24 @@ unused_self = "deny"
unwrap_used = "deny"
[workspace.dependencies]
lemmy_api = { version = "=0.19.4-rc.5", path = "./crates/api" }
lemmy_api_crud = { version = "=0.19.4-rc.5", path = "./crates/api_crud" }
lemmy_apub = { version = "=0.19.4-rc.5", path = "./crates/apub" }
lemmy_utils = { version = "=0.19.4-rc.5", path = "./crates/utils", default-features = false }
lemmy_db_schema = { version = "=0.19.4-rc.5", path = "./crates/db_schema" }
lemmy_api_common = { version = "=0.19.4-rc.5", path = "./crates/api_common" }
lemmy_routes = { version = "=0.19.4-rc.5", path = "./crates/routes" }
lemmy_db_views = { version = "=0.19.4-rc.5", path = "./crates/db_views" }
lemmy_db_views_actor = { version = "=0.19.4-rc.5", path = "./crates/db_views_actor" }
lemmy_db_views_moderator = { version = "=0.19.4-rc.5", path = "./crates/db_views_moderator" }
lemmy_federate = { version = "=0.19.4-rc.5", path = "./crates/federate" }
lemmy_api = { version = "=0.19.5", path = "./crates/api" }
lemmy_api_crud = { version = "=0.19.5", path = "./crates/api_crud" }
lemmy_apub = { version = "=0.19.5", path = "./crates/apub" }
lemmy_utils = { version = "=0.19.5", path = "./crates/utils", default-features = false }
lemmy_db_schema = { version = "=0.19.5", path = "./crates/db_schema" }
lemmy_api_common = { version = "=0.19.5", path = "./crates/api_common" }
lemmy_routes = { version = "=0.19.5", path = "./crates/routes" }
lemmy_db_views = { version = "=0.19.5", path = "./crates/db_views" }
lemmy_db_views_actor = { version = "=0.19.5", path = "./crates/db_views_actor" }
lemmy_db_views_moderator = { version = "=0.19.5", path = "./crates/db_views_moderator" }
lemmy_federate = { version = "=0.19.5", path = "./crates/federate" }
activitypub_federation = { version = "0.5.6", default-features = false, features = [
"actix-web",
] }
diesel = "2.1.6"
diesel_migrations = "2.1.0"
diesel-async = "0.4.1"
serde = { version = "1.0.202", features = ["derive"] }
serde = { version = "1.0.203", features = ["derive"] }
serde_with = "3.8.1"
actix-web = { version = "4.6.0", default-features = false, features = [
"macros",
@ -116,7 +116,7 @@ actix-web = { version = "4.6.0", default-features = false, features = [
"cookies",
] }
tracing = "0.1.40"
tracing-actix-web = { version = "0.7.10", default-features = false }
tracing-actix-web = { version = "0.7.11", default-features = false }
tracing-error = "0.2.0"
tracing-log = "0.2.0"
tracing-subscriber = { version = "0.3.18", features = ["env-filter"] }
@ -139,13 +139,13 @@ anyhow = { version = "1.0.86", features = [
diesel_ltree = "0.3.1"
typed-builder = "0.18.2"
serial_test = "3.1.1"
tokio = { version = "1.37.0", features = ["full"] }
tokio = { version = "1.38.0", features = ["full"] }
regex = "1.10.4"
once_cell = "1.19.0"
diesel-derive-newtype = "2.1.2"
diesel-derive-enum = { version = "2.1.0", features = ["postgres"] }
strum = "0.26.2"
strum_macros = "0.26.2"
strum_macros = "0.26.4"
itertools = "0.13.0"
futures = "0.3.30"
http = "0.2.12"
@ -157,7 +157,7 @@ ts-rs = { version = "7.1.1", features = [
"chrono-impl",
"no-serde-warnings",
] }
rustls = { version = "0.23.8", features = ["ring"] }
rustls = { version = "0.23.9", features = ["ring"] }
futures-util = "0.3.30"
tokio-postgres = "0.7.10"
tokio-postgres-rustls = "0.12.0"
@ -165,8 +165,9 @@ urlencoding = "2.1.3"
enum-map = "2.7"
moka = { version = "0.12.7", features = ["future"] }
i-love-jesus = { version = "0.1.0" }
clap = { version = "4.5.4", features = ["derive", "env"] }
clap = { version = "4.5.6", features = ["derive", "env"] }
pretty_assertions = "1.4.0"
derive-new = "0.6.0"
[dependencies]
lemmy_api = { workspace = true }
@ -194,9 +195,9 @@ clokwerk = { workspace = true }
serde_json = { workspace = true }
tracing-opentelemetry = { workspace = true, optional = true }
opentelemetry = { workspace = true, optional = true }
console-subscriber = { version = "0.2.0", optional = true }
console-subscriber = { version = "0.3.0", optional = true }
opentelemetry-otlp = { version = "0.12.0", optional = true }
pict-rs = { version = "0.5.14", optional = true }
pict-rs = { version = "0.5.15", optional = true }
tokio.workspace = true
actix-cors = "0.7.0"
futures-util = { workspace = true }

42
api_tests/.eslintrc.json
View File

@ -1,42 +0,0 @@
{
"root": true,
"env": {
"browser": true
},
"plugins": ["@typescript-eslint"],
"extends": ["eslint:recommended", "plugin:@typescript-eslint/recommended"],
"parser": "@typescript-eslint/parser",
"parserOptions": {
"project": "./tsconfig.json",
"warnOnUnsupportedTypeScriptVersion": false
},
"rules": {
"@typescript-eslint/ban-ts-comment": 0,
"@typescript-eslint/no-explicit-any": 0,
"@typescript-eslint/explicit-module-boundary-types": 0,
"@typescript-eslint/no-var-requires": 0,
"arrow-body-style": 0,
"curly": 0,
"eol-last": 0,
"eqeqeq": 0,
"func-style": 0,
"import/no-duplicates": 0,
"max-statements": 0,
"max-params": 0,
"new-cap": 0,
"no-console": 0,
"no-duplicate-imports": 0,
"no-extra-parens": 0,
"no-return-assign": 0,
"no-throw-literal": 0,
"no-trailing-spaces": 0,
"no-unused-expressions": 0,
"no-useless-constructor": 0,
"no-useless-escape": 0,
"no-var": 0,
"prefer-const": 0,
"prefer-rest-params": 0,
"quote-props": 0,
"unicorn/filename-case": 0
}
}

56
api_tests/eslint.config.mjs 100644
View File

@ -0,0 +1,56 @@
import pluginJs from "@eslint/js";
import tseslint from "typescript-eslint";
export default [
pluginJs.configs.recommended,
...tseslint.configs.recommended,
{
languageOptions: {
parser: tseslint.parser,
},
},
// For some reason this has to be in its own block
{
ignores: [
"putTypesInIndex.js",
"dist/*",
"docs/*",
".yalc",
"jest.config.js",
],
},
{
files: ["src/**/*"],
rules: {
"@typescript-eslint/no-empty-interface": 0,
"@typescript-eslint/no-empty-function": 0,
"@typescript-eslint/ban-ts-comment": 0,
"@typescript-eslint/no-explicit-any": 0,
"@typescript-eslint/explicit-module-boundary-types": 0,
"@typescript-eslint/no-var-requires": 0,
"arrow-body-style": 0,
curly: 0,
"eol-last": 0,
eqeqeq: 0,
"func-style": 0,
"import/no-duplicates": 0,
"max-statements": 0,
"max-params": 0,
"new-cap": 0,
"no-console": 0,
"no-duplicate-imports": 0,
"no-extra-parens": 0,
"no-return-assign": 0,
"no-throw-literal": 0,
"no-trailing-spaces": 0,
"no-unused-expressions": 0,
"no-useless-constructor": 0,
"no-useless-escape": 0,
"no-var": 0,
"prefer-const": 0,
"prefer-rest-params": 0,
"quote-props": 0,
"unicorn/filename-case": 0,
},
},
];

11
api_tests/package.json
View File

@ -6,9 +6,9 @@
"repository": "https://github.com/LemmyNet/lemmy",
"author": "Dessalines",
"license": "AGPL-3.0",
"packageManager": "pnpm@9.1.4",
"packageManager": "pnpm@9.4.0",
"scripts": {
"lint": "tsc --noEmit && eslint --report-unused-disable-directives --ext .js,.ts,.tsx src && prettier --check 'src/**/*.ts'",
"lint": "tsc --noEmit && eslint --report-unused-disable-directives && prettier --check 'src/**/*.ts'",
"fix": "prettier --write src && eslint --fix src",
"api-test": "jest -i follow.spec.ts && jest -i image.spec.ts && jest -i user.spec.ts && jest -i private_message.spec.ts && jest -i community.spec.ts && jest -i post.spec.ts && jest -i comment.spec.ts ",
"api-test-follow": "jest -i follow.spec.ts",
@ -25,12 +25,13 @@
"@typescript-eslint/eslint-plugin": "^7.5.0",
"@typescript-eslint/parser": "^7.5.0",
"download-file-sync": "^1.0.4",
"eslint": "^8.57.0",
"eslint": "^9.0.0",
"eslint-plugin-prettier": "^5.1.3",
"jest": "^29.5.0",
"lemmy-js-client": "0.19.4-alpha.18",
"lemmy-js-client": "0.19.5-alpha.1",
"prettier": "^3.2.5",
"ts-jest": "^29.1.0",
"typescript": "^5.4.4"
"typescript": "^5.4.4",
"typescript-eslint": "^7.13.0"
}
}

656
api_tests/pnpm-lock.yaml
View File

File diff suppressed because it is too large Load Diff

2
api_tests/prepare-drone-federation-test.sh
View File

@ -15,7 +15,7 @@ export LEMMY_TEST_FAST_FEDERATION=1 # by default, the persistent federation queu
# pictrs setup
if [ ! -f "api_tests/pict-rs" ]; then
curl "https://git.asonix.dog/asonix/pict-rs/releases/download/v0.5.13/pict-rs-linux-amd64" -o api_tests/pict-rs
curl "https://git.asonix.dog/asonix/pict-rs/releases/download/v0.5.16/pict-rs-linux-amd64" -o api_tests/pict-rs
chmod +x api_tests/pict-rs
fi
./api_tests/pict-rs \

4
api_tests/src/image.spec.ts
View File

@ -160,6 +160,7 @@ test("Purge post, linked image removed", async () => {
upload.url,
);
expect(post.post_view.post.url).toBe(upload.url);
expect(post.post_view.image_details).toBeDefined();
// purge post
const purgeForm: PurgePost = {
@ -184,6 +185,9 @@ test("Images in remote image post are proxied if setting enabled", async () => {
const post = postRes.post_view.post;
expect(post).toBeDefined();
// Make sure it fetched the image details
expect(postRes.post_view.image_details).toBeDefined();
// remote image gets proxied after upload
expect(
post.thumbnail_url?.startsWith(

2
api_tests/src/post.spec.ts
View File

@ -512,7 +512,7 @@ test("Enforce site ban federation for local user", async () => {
}
let newAlphaUserJwt = await loginUser(alpha, alphaUserPerson.name);
alphaUserHttp.setHeaders({
Authorization: "Bearer " + newAlphaUserJwt.jwt ?? "",
Authorization: "Bearer " + newAlphaUserJwt.jwt,
});
// alpha makes new post in beta community, it federates
let postRes2 = await createPost(alphaUserHttp, betaCommunity!.community.id);

19
api_tests/src/user.spec.ts
View File

@ -21,6 +21,7 @@ import {
fetchFunction,
alphaImage,
unfollows,
saveUserSettingsBio,
} from "./shared";
import { LemmyHttp, SaveUserSettings, UploadImage } from "lemmy-js-client";
import { GetPosts } from "lemmy-js-client/dist/types/GetPosts";
@ -190,10 +191,26 @@ test("Set a new avatar, old avatar is deleted", async () => {
expect(upload2.url).toBeDefined();
let form2 = {
avatar: upload1.url,
avatar: upload2.url,
};
await saveUserSettings(alpha, form2);
// make sure only the new avatar is kept
const listMediaRes2 = await alphaImage.listMedia();
expect(listMediaRes2.images.length).toBe(1);
// Upload that same form2 avatar, make sure it isn't replaced / deleted
await saveUserSettings(alpha, form2);
// make sure only the new avatar is kept
const listMediaRes3 = await alphaImage.listMedia();
expect(listMediaRes3.images.length).toBe(1);
// Now try to save a user settings, with the icon missing,
// and make sure it doesn't clear the data, or delete the image
await saveUserSettingsBio(alpha);
let site = await getSite(alpha);
expect(site.my_user?.local_user_view.person.avatar).toBe(upload2.url);
// make sure only the new avatar is kept
const listMediaRes4 = await alphaImage.listMedia();
expect(listMediaRes4.images.length).toBe(1);
});

3
cliff.toml
View File

@ -26,6 +26,7 @@ body = """
{%- endif %}
{%- endfor -%}
{%- if github -%}
{% if github.contributors | filter(attribute="is_first_time", value=true) | length != 0 %}
{% raw %}\n{% endraw -%}
## New Contributors
@ -36,6 +37,7 @@ body = """
[#{{ contributor.pr_number }}]({{ self::remote_url() }}/pull/{{ contributor.pr_number }}) \
{%- endif %}
{%- endfor -%}
{%- endif -%}
{% if version %}
{% if previous.version %}
@ -70,6 +72,7 @@ commit_preprocessors = [
# remove issue numbers from commits
{ pattern = '\((\w+\s)?#([0-9]+)\)', replace = "" },
]
commit_parsers = [{ field = "author.name", pattern = "renovate", skip = true }]
# protect breaking changes from being skipped due to matching a skipping commit_parser
protect_breaking_commits = false
# filter out the commits that are not matched by commit parsers

2
crates/api/Cargo.toml
View File

@ -33,7 +33,7 @@ anyhow = { workspace = true }
tracing = { workspace = true }
chrono = { workspace = true }
url = { workspace = true }
wav = "1.0.1"
hound = "3.5.1"
sitemap-rs = "0.2.1"
totp-rs = { version = "5.5.1", features = ["gen_secret", "otpauth"] }
actix-web-httpauth = "0.8.1"

5
crates/api/src/community/ban.rs
View File

@ -43,7 +43,10 @@ pub async fn ban_from_community(
&mut context.pool(),
)
.await?;
is_valid_body_field(&data.reason, false)?;
if let Some(reason) = &data.reason {
is_valid_body_field(reason, false)?;
}
let community_user_ban_form = CommunityPersonBanForm {
community_id: data.community_id,

35
crates/api/src/lib.rs
View File

@ -44,33 +44,38 @@ pub mod site;
pub mod sitemap;
/// Converts the captcha to a base64 encoded wav audio file
#[allow(deprecated)]
pub(crate) fn captcha_as_wav_base64(captcha: &Captcha) -> LemmyResult<String> {
let letters = captcha.as_wav();
// Decode each wav file, concatenate the samples
let mut concat_samples: Vec<i16> = Vec::new();
let mut any_header: Option<wav::Header> = None;
let mut any_header: Option<hound::WavSpec> = None;
for letter in letters {
let mut cursor = Cursor::new(letter.unwrap_or_default());
let (header, samples) = wav::read(&mut cursor)?;
any_header = Some(header);
if let Some(samples16) = samples.as_sixteen() {
concat_samples.extend(samples16);
} else {
Err(LemmyErrorType::CouldntCreateAudioCaptcha)?
}
let reader = hound::WavReader::new(&mut cursor)?;
any_header = Some(reader.spec());
let samples16 = reader
.into_samples::<i16>()
.collect::<Result<Vec<_>, _>>()
.with_lemmy_type(LemmyErrorType::CouldntCreateAudioCaptcha)?;
concat_samples.extend(samples16);
}
// Encode the concatenated result as a wav file
let mut output_buffer = Cursor::new(vec![]);
if let Some(header) = any_header {
wav::write(
header,
&wav::BitDepth::Sixteen(concat_samples),
&mut output_buffer,
)
.with_lemmy_type(LemmyErrorType::CouldntCreateAudioCaptcha)?;
let mut writer = hound::WavWriter::new(&mut output_buffer, header)
.with_lemmy_type(LemmyErrorType::CouldntCreateAudioCaptcha)?;
let mut writer16 = writer.get_i16_writer(concat_samples.len() as u32);
for sample in concat_samples {
writer16.write_sample(sample);
}
writer16
.flush()
.with_lemmy_type(LemmyErrorType::CouldntCreateAudioCaptcha)?;
writer
.finalize()
.with_lemmy_type(LemmyErrorType::CouldntCreateAudioCaptcha)?;
Ok(base64.encode(output_buffer.into_inner()))
} else {

4
crates/api/src/local_user/ban_person.rs
View File

@ -31,7 +31,9 @@ pub async fn ban_from_site(
// Make sure user is an admin
is_admin(&local_user_view)?;
is_valid_body_field(&data.reason, false)?;
if let Some(reason) = &data.reason {
is_valid_body_field(reason, false)?;
}
let expires = check_expire_time(data.expires)?;

26
crates/api/src/local_user/save_settings.rs
View File

@ -21,7 +21,7 @@ use lemmy_db_schema::{
person::{Person, PersonUpdateForm},
},
traits::Crud,
utils::diesel_option_overwrite,
utils::{diesel_string_update, diesel_url_update},
};
use lemmy_db_views::structs::{LocalUserView, SiteView};
use lemmy_utils::{
@ -42,18 +42,24 @@ pub async fn save_user_settings(
let slur_regex = local_site_to_slur_regex(&site_view.local_site);
let url_blocklist = get_url_blocklist(&context).await?;
let bio = diesel_option_overwrite(
process_markdown_opt(&data.bio, &slur_regex, &url_blocklist, &context).await?,
let bio = diesel_string_update(
process_markdown_opt(&data.bio, &slur_regex, &url_blocklist, &context)
.await?
.as_deref(),
);
replace_image(&data.avatar, &local_user_view.person.avatar, &context).await?;
replace_image(&data.banner, &local_user_view.person.banner, &context).await?;
let avatar = proxy_image_link_opt_api(&data.avatar, &context).await?;
let banner = proxy_image_link_opt_api(&data.banner, &context).await?;
let display_name = diesel_option_overwrite(data.display_name.clone());
let matrix_user_id = diesel_option_overwrite(data.matrix_user_id.clone());
let avatar = diesel_url_update(data.avatar.as_deref())?;
replace_image(&avatar, &local_user_view.person.avatar, &context).await?;
let avatar = proxy_image_link_opt_api(avatar, &context).await?;
let banner = diesel_url_update(data.banner.as_deref())?;
replace_image(&banner, &local_user_view.person.banner, &context).await?;
let banner = proxy_image_link_opt_api(banner, &context).await?;
let display_name = diesel_string_update(data.display_name.as_deref());
let matrix_user_id = diesel_string_update(data.matrix_user_id.as_deref());
let email_deref = data.email.as_deref().map(str::to_lowercase);
let email = diesel_option_overwrite(email_deref.clone());
let email = diesel_string_update(email_deref.as_deref());
if let Some(Some(email)) = &email {
let previous_email = local_user_view.local_user.email.clone().unwrap_or_default();

9
crates/api/src/post/get_link_metadata.rs
View File

@ -4,14 +4,19 @@ use lemmy_api_common::{
post::{GetSiteMetadata, GetSiteMetadataResponse},
request::fetch_link_metadata,
};
use lemmy_utils::error::LemmyResult;
use lemmy_utils::{
error::{LemmyErrorExt, LemmyResult},
LemmyErrorType,
};
use url::Url;
#[tracing::instrument(skip(context))]
pub async fn get_link_metadata(
data: Query<GetSiteMetadata>,
context: Data<LemmyContext>,
) -> LemmyResult<Json<GetSiteMetadataResponse>> {
let metadata = fetch_link_metadata(&data.url, &context).await?;
let url = Url::parse(&data.url).with_lemmy_type(LemmyErrorType::InvalidUrl)?;
let metadata = fetch_link_metadata(&url, &context).await?;
Ok(Json(GetSiteMetadataResponse { metadata }))
}

4
crates/api/src/site/registration_applications/approve.rs
View File

@ -10,7 +10,7 @@ use lemmy_db_schema::{
registration_application::{RegistrationApplication, RegistrationApplicationUpdateForm},
},
traits::Crud,
utils::diesel_option_overwrite,
utils::diesel_string_update,
};
use lemmy_db_views::structs::{LocalUserView, RegistrationApplicationView};
use lemmy_utils::{error::LemmyResult, LemmyErrorType};
@ -26,7 +26,7 @@ pub async fn approve_registration_application(
is_admin(&local_user_view)?;
// Update the registration with reason, admin_id
let deny_reason = diesel_option_overwrite(data.deny_reason.clone());
let deny_reason = diesel_string_update(data.deny_reason.as_deref());
let app_form = RegistrationApplicationUpdateForm {
admin_id: Some(Some(local_user_view.person.id)),
deny_reason,

6
crates/api_common/src/claims.rs
View File

@ -112,11 +112,7 @@ mod tests {
.await
.unwrap();
let new_person = PersonInsertForm::builder()
.name("Gerry9812".into())
.public_key("pubkey".to_string())
.instance_id(inserted_instance.id)
.build();
let new_person = PersonInsertForm::test_form(inserted_instance.id, "Gerry9812");
let inserted_person = Person::create(pool, &new_person).await.unwrap();

18
crates/api_common/src/post.rs
View File

@ -10,7 +10,6 @@ use serde::{Deserialize, Serialize};
use serde_with::skip_serializing_none;
#[cfg(feature = "full")]
use ts_rs::TS;
use url::Url;
#[skip_serializing_none]
#[derive(Debug, Serialize, Deserialize, Clone, Default, PartialEq, Eq, Hash)]
@ -20,8 +19,7 @@ use url::Url;
pub struct CreatePost {
pub name: String,
pub community_id: CommunityId,
#[cfg_attr(feature = "full", ts(type = "string"))]
pub url: Option<Url>,
pub url: Option<String>,
/// An optional body for the post in markdown.
pub body: Option<String>,
/// An optional alt_text, usable for image posts.
@ -30,9 +28,8 @@ pub struct CreatePost {
pub honeypot: Option<String>,
pub nsfw: Option<bool>,
pub language_id: Option<LanguageId>,
#[cfg_attr(feature = "full", ts(type = "string"))]
/// Instead of fetching a thumbnail, use a custom one.
pub custom_thumbnail: Option<Url>,
pub custom_thumbnail: Option<String>,
}
#[derive(Debug, Serialize, Deserialize, Clone)]
@ -82,6 +79,8 @@ pub struct GetPosts {
pub liked_only: Option<bool>,
pub disliked_only: Option<bool>,
pub show_hidden: Option<bool>,
/// If true, then show the read posts (even if your user setting is to hide them)
pub show_read: Option<bool>,
pub page_cursor: Option<PaginationCursor>,
}
@ -114,17 +113,15 @@ pub struct CreatePostLike {
pub struct EditPost {
pub post_id: PostId,
pub name: Option<String>,
#[cfg_attr(feature = "full", ts(type = "string"))]
pub url: Option<Url>,
pub url: Option<String>,
/// An optional body for the post in markdown.
pub body: Option<String>,
/// An optional alt_text, usable for image posts.
pub alt_text: Option<String>,
pub nsfw: Option<bool>,
pub language_id: Option<LanguageId>,
#[cfg_attr(feature = "full", ts(type = "string"))]
/// Instead of fetching a thumbnail, use a custom one.
pub custom_thumbnail: Option<Url>,
pub custom_thumbnail: Option<String>,
}
#[derive(Debug, Serialize, Deserialize, Clone, Copy, Default, PartialEq, Eq, Hash)]
@ -249,8 +246,7 @@ pub struct ListPostReportsResponse {
#[cfg_attr(feature = "full", ts(export))]
/// Get metadata for a given site.
pub struct GetSiteMetadata {
#[cfg_attr(feature = "full", ts(type = "string"))]
pub url: Url,
pub url: String,
}
#[derive(Debug, Serialize, Deserialize, Clone)]

73
crates/api_common/src/request.rs
View File

@ -11,7 +11,7 @@ use encoding_rs::{Encoding, UTF_8};
use lemmy_db_schema::{
newtypes::DbUrl,
source::{
images::{LocalImage, LocalImageForm},
images::{ImageDetailsForm, LocalImage, LocalImageForm},
local_site::LocalSite,
post::{Post, PostUpdateForm},
},
@ -209,6 +209,19 @@ pub struct PictrsFileDetails {
pub created_at: DateTime<Utc>,
}
impl PictrsFileDetails {
/// Builds the image form. This should always use the thumbnail_url,
/// Because the post_view joins to it
pub fn build_image_details_form(&self, thumbnail_url: &Url) -> ImageDetailsForm {
ImageDetailsForm {
link: thumbnail_url.clone().into(),
width: self.width.into(),
height: self.height.into(),
content_type: self.content_type.clone(),
}
}
}
#[derive(Deserialize, Serialize, Debug)]
struct PictrsPurgeResponse {
msg: String,
@ -316,11 +329,52 @@ async fn generate_pictrs_thumbnail(image_url: &Url, context: &LemmyContext) -> L
let protocol_and_hostname = context.settings().get_protocol_and_hostname();
let thumbnail_url = image.thumbnail_url(&protocol_and_hostname)?;
LocalImage::create(&mut context.pool(), &form).await?;
// Also store the details for the image
let details_form = image.details.build_image_details_form(&thumbnail_url);
LocalImage::create(&mut context.pool(), &form, &details_form).await?;
Ok(thumbnail_url)
}
/// Fetches the image details for pictrs proxied images
///
/// We don't need to check for image mode, as that's already been done
#[tracing::instrument(skip_all)]
pub async fn fetch_pictrs_proxied_image_details(
image_url: &Url,
context: &LemmyContext,
) -> LemmyResult<PictrsFileDetails> {
let pictrs_url = context.settings().pictrs_config()?.url;
let encoded_image_url = encode(image_url.as_str());
// Pictrs needs you to fetch the proxied image before you can fetch the details
let proxy_url = format!("{pictrs_url}image/original?proxy={encoded_image_url}");
let res = context
.client()
.get(&proxy_url)
.timeout(REQWEST_TIMEOUT)
.send()
.await?
.status();
if !res.is_success() {
Err(LemmyErrorType::NotAnImageType)?
}
let details_url = format!("{pictrs_url}image/details/original?proxy={encoded_image_url}");
let res = context
.client()
.get(&details_url)
.timeout(REQWEST_TIMEOUT)
.send()
.await?
.json()
.await?;
Ok(res)
}
// TODO: get rid of this by reading content type from db
#[tracing::instrument(skip_all)]
async fn is_image_content_type(client: &ClientWithMiddleware, url: &Url) -> LemmyResult<()> {
@ -338,16 +392,19 @@ async fn is_image_content_type(client: &ClientWithMiddleware, url: &Url) -> Lemm
}
}
/// When adding a new avatar or similar image, delete the old one.
/// When adding a new avatar, banner or similar image, delete the old one.
pub async fn replace_image(
new_image: &Option<String>,
new_image: &Option<Option<DbUrl>>,
old_image: &Option<DbUrl>,
context: &Data<LemmyContext>,
) -> LemmyResult<()> {
if new_image.is_some() {
// Ignore errors because image may be stored externally.
if let Some(avatar) = &old_image {
let image = LocalImage::delete_by_url(&mut context.pool(), avatar)
if let (Some(Some(new_image)), Some(old_image)) = (new_image, old_image) {
// Note: Oftentimes front ends will include the current image in the form.
// In this case, deleting `old_image` would also be deletion of `new_image`,
// so the deletion must be skipped for the image to be kept.
if new_image != old_image {
// Ignore errors because image may be stored externally.
let image = LocalImage::delete_by_url(&mut context.pool(), old_image)
.await
.ok();
if let Some(image) = image {

80
crates/api_common/src/utils.rs
View File

@ -1,6 +1,10 @@
use crate::{
context::LemmyContext,
request::{delete_image_from_pictrs, purge_image_from_pictrs},
request::{
delete_image_from_pictrs,
fetch_pictrs_proxied_image_details,
purge_image_from_pictrs,
},
site::{FederatedInstances, InstanceWithFederationState},
};
use chrono::{DateTime, Days, Local, TimeZone, Utc};
@ -949,7 +953,18 @@ pub async fn process_markdown(
if context.settings().pictrs_config()?.image_mode() == PictrsImageMode::ProxyAllImages {
let (text, links) = markdown_rewrite_image_links(text);
RemoteImage::create(&mut context.pool(), links).await?;
// Create images and image detail rows
for link in links {
// Insert image details for the remote image
let details_res = fetch_pictrs_proxied_image_details(&link, context).await;
if let Ok(details) = details_res {
let proxied =
build_proxied_image_url(&link, &context.settings().get_protocol_and_hostname())?;
let details_form = details.build_image_details_form(&proxied);
RemoteImage::create(&mut context.pool(), &details_form).await?;
}
}
Ok(text)
} else {
Ok(text)
@ -984,8 +999,14 @@ async fn proxy_image_link_internal(
Ok(link.into())
} else if image_mode == PictrsImageMode::ProxyAllImages {
let proxied = build_proxied_image_url(&link, &context.settings().get_protocol_and_hostname())?;
// This should fail softly, since pictrs might not even be running
let details_res = fetch_pictrs_proxied_image_details(&link, context).await;
if let Ok(details) = details_res {
let details_form = details.build_image_details_form(&proxied);
RemoteImage::create(&mut context.pool(), &details_form).await?;
};
RemoteImage::create(&mut context.pool(), vec![link]).await?;
Ok(proxied.into())
} else {
Ok(link.into())
@ -1004,26 +1025,25 @@ pub(crate) async fn proxy_image_link(link: Url, context: &LemmyContext) -> Lemmy
}
pub async fn proxy_image_link_opt_api(
link: &Option<String>,
link: Option<Option<DbUrl>>,
context: &LemmyContext,
) -> LemmyResult<Option<Option<DbUrl>>> {
proxy_image_link_api(link, context).await.map(Some)
if let Some(Some(link)) = link {
proxy_image_link(link.into(), context)
.await
.map(Some)
.map(Some)
} else {
Ok(link)
}
}
pub async fn proxy_image_link_api(
link: &Option<String>,
link: Option<DbUrl>,
context: &LemmyContext,
) -> LemmyResult<Option<DbUrl>> {
let link: Option<DbUrl> = match link.as_ref().map(String::as_str) {
// An empty string is an erase
Some("") => None,
Some(str_url) => Url::parse(str_url)
.map(|u| Some(u.into()))
.with_lemmy_type(LemmyErrorType::InvalidUrl)?,
None => None,
};
if let Some(l) = link {
proxy_image_link(l.into(), context).await.map(Some)
if let Some(link) = link {
proxy_image_link(link.into(), context).await.map(Some)
} else {
Ok(link)
}
@ -1124,35 +1144,13 @@ mod tests {
"https://lemmy-alpha/api/v3/image_proxy?url=http%3A%2F%2Flemmy-beta%2Fimage.png",
proxied.as_str()
);
// This fails, because the details can't be fetched without pictrs running,
// And a remote image won't be inserted.
assert!(
RemoteImage::validate(&mut context.pool(), remote_image.into())
.await
.is_ok()
);
}
#[tokio::test]
#[serial]
async fn test_diesel_option_overwrite_to_url() {
let context = LemmyContext::init_test_context().await;
assert!(matches!(
proxy_image_link_api(&None, &context).await,
Ok(None)
));
assert!(matches!(
proxy_image_link_opt_api(&Some(String::new()), &context).await,
Ok(Some(None))
));
assert!(
proxy_image_link_opt_api(&Some("invalid_url".to_string()), &context)
.await
.is_err()
);
let example_url = "https://lemmy-alpha/image.png";
assert!(matches!(
proxy_image_link_opt_api(&Some(example_url.to_string()), &context).await,
Ok(Some(Some(url))) if url == Url::parse(example_url).unwrap().into()
));
}
}

2
crates/api_crud/src/comment/create.rs
View File

@ -47,7 +47,7 @@ pub async fn create_comment(
let slur_regex = local_site_to_slur_regex(&local_site);
let url_blocklist = get_url_blocklist(&context).await?;
let content = process_markdown(&data.content, &slur_regex, &url_blocklist, &context).await?;
is_valid_body_field(&Some(content.clone()), false)?;
is_valid_body_field(&content, false)?;
// Check for a community ban
let post_id = data.post_id;

4
crates/api_crud/src/comment/update.rs
View File

@ -63,7 +63,9 @@ pub async fn update_comment(
let slur_regex = local_site_to_slur_regex(&local_site);
let url_blocklist = get_url_blocklist(&context).await?;
let content = process_markdown_opt(&data.content, &slur_regex, &url_blocklist, &context).await?;
is_valid_body_field(&content, false)?;
if let Some(content) = &content {
is_valid_body_field(content, false)?;
}
let comment_id = data.comment_id;
let form = CommentUpdateForm {

14
crates/api_crud/src/community/create.rs
View File

@ -30,6 +30,7 @@ use lemmy_db_schema::{
},
},
traits::{ApubActor, Crud, Followable, Joinable},
utils::diesel_url_create,
};
use lemmy_db_views::structs::{LocalUserView, SiteView};
use lemmy_utils::{
@ -61,11 +62,18 @@ pub async fn create_community(
check_slurs(&data.title, &slur_regex)?;
let description =
process_markdown_opt(&data.description, &slur_regex, &url_blocklist, &context).await?;
let icon = proxy_image_link_api(&data.icon, &context).await?;
let banner = proxy_image_link_api(&data.banner, &context).await?;
let icon = diesel_url_create(data.icon.as_deref())?;
let icon = proxy_image_link_api(icon, &context).await?;
let banner = diesel_url_create(data.banner.as_deref())?;
let banner = proxy_image_link_api(banner, &context).await?;
is_valid_actor_name(&data.name, local_site.actor_name_max_length as usize)?;
is_valid_body_field(&data.description, false)?;
if let Some(desc) = &data.description {
is_valid_body_field(desc, false)?;
}
// Double check for duplicate community actor_ids
let community_actor_id = generate_local_apub_endpoint(

28
crates/api_crud/src/community/update.rs
View File

@ -21,7 +21,7 @@ use lemmy_db_schema::{
local_site::LocalSite,
},
traits::Crud,
utils::{diesel_option_overwrite, naive_now},
utils::{diesel_string_update, diesel_url_update, naive_now},
};
use lemmy_db_views::structs::LocalUserView;
use lemmy_utils::{
@ -40,18 +40,28 @@ pub async fn update_community(
let slur_regex = local_site_to_slur_regex(&local_site);
let url_blocklist = get_url_blocklist(&context).await?;
check_slurs_opt(&data.title, &slur_regex)?;
let description =
process_markdown_opt(&data.description, &slur_regex, &url_blocklist, &context).await?;
is_valid_body_field(&data.description, false)?;
let description = diesel_string_update(
process_markdown_opt(&data.description, &slur_regex, &url_blocklist, &context)
.await?
.as_deref(),
);
if let Some(Some(desc)) = &description {
is_valid_body_field(desc, false)?;
}
let old_community = Community::read(&mut context.pool(), data.community_id)
.await?
.ok_or(LemmyErrorType::CouldntFindCommunity)?;
replace_image(&data.icon, &old_community.icon, &context).await?;
replace_image(&data.banner, &old_community.banner, &context).await?;
let description = diesel_option_overwrite(description);
let icon = proxy_image_link_opt_api(&data.icon, &context).await?;
let banner = proxy_image_link_opt_api(&data.banner, &context).await?;
let icon = diesel_url_update(data.icon.as_deref())?;
replace_image(&icon, &old_community.icon, &context).await?;
let icon = proxy_image_link_opt_api(icon, &context).await?;
let banner = diesel_url_update(data.banner.as_deref())?;
replace_image(&banner, &old_community.banner, &context).await?;
let banner = proxy_image_link_opt_api(banner, &context).await?;
// Verify its a mod (only mods can edit it)
check_community_mod_action(

31
crates/api_crud/src/post/create.rs
View File

@ -26,6 +26,7 @@ use lemmy_db_schema::{
post::{Post, PostInsertForm, PostLike, PostLikeForm, PostUpdateForm},
},
traits::{Crud, Likeable},
utils::diesel_url_create,
CommunityVisibility,
};
use lemmy_db_views::structs::LocalUserView;
@ -37,7 +38,6 @@ use lemmy_utils::{
slurs::check_slurs,
validation::{
check_url_scheme,
clean_url_params,
is_url_blocked,
is_valid_alt_text_field,
is_valid_body_field,
@ -64,16 +64,27 @@ pub async fn create_post(
let url_blocklist = get_url_blocklist(&context).await?;
let body = process_markdown_opt(&data.body, &slur_regex, &url_blocklist, &context).await?;
let data_url = data.url.as_ref();
let url = data_url.map(clean_url_params); // TODO no good way to handle a "clear"
let custom_thumbnail = data.custom_thumbnail.as_ref().map(clean_url_params);
let url = diesel_url_create(data.url.as_deref())?;
let custom_thumbnail = diesel_url_create(data.custom_thumbnail.as_deref())?;
is_valid_post_title(&data.name)?;
is_valid_body_field(&body, true)?;
is_valid_alt_text_field(&data.alt_text)?;
is_url_blocked(&url, &url_blocklist)?;
check_url_scheme(&url)?;
check_url_scheme(&custom_thumbnail)?;
if let Some(url) = &url {
is_url_blocked(url, &url_blocklist)?;
check_url_scheme(url)?;
}
if let Some(custom_thumbnail) = &custom_thumbnail {
check_url_scheme(custom_thumbnail)?;
}
if let Some(alt_text) = &data.alt_text {
is_valid_alt_text_field(alt_text)?;
}
if let Some(body) = &body {
is_valid_body_field(body, true)?;
}
check_community_user_action(
&local_user_view.person,
@ -156,7 +167,7 @@ pub async fn create_post(
generate_post_link_metadata(
updated_post.clone(),
custom_thumbnail,
custom_thumbnail.map(Into::into),
|post| Some(SendActivityData::CreatePost(post)),
Some(local_site),
context.reset_request_count(),

2
crates/api_crud/src/post/read.rs
View File

@ -83,11 +83,13 @@ pub async fn get_post(
.ok_or(LemmyErrorType::CouldntFindCommunity)?;
let moderators = CommunityModeratorView::for_community(&mut context.pool(), community_id).await?;
let local_user = local_user_view.as_ref().map(|u| &u.local_user);
// Fetch the cross_posts
let cross_posts = if let Some(url) = &post_view.post.url {
let mut x_posts = PostQuery {
url_search: Some(url.inner().as_str().into()),
local_user,
..Default::default()
}
.list(&local_site.site, &mut context.pool())

52
crates/api_crud/src/post/update.rs
View File

@ -20,16 +20,15 @@ use lemmy_db_schema::{
post::{Post, PostUpdateForm},
},
traits::Crud,
utils::{diesel_option_overwrite, naive_now},
utils::{diesel_string_update, diesel_url_update, naive_now},
};
use lemmy_db_views::structs::LocalUserView;
use lemmy_utils::{
error::{LemmyErrorExt, LemmyErrorType, LemmyResult},
utils::{
slurs::check_slurs_opt,
slurs::check_slurs,
validation::{
check_url_scheme,
clean_url_params,
is_url_blocked,
is_valid_alt_text_field,
is_valid_body_field,
@ -47,26 +46,43 @@ pub async fn update_post(
) -> LemmyResult<Json<PostResponse>> {
let local_site = LocalSite::read(&mut context.pool()).await?;
// TODO No good way to handle a clear.
// Issue link: https://github.com/LemmyNet/lemmy/issues/2287
let url = data.url.as_ref().map(clean_url_params);
let custom_thumbnail = data.custom_thumbnail.as_ref().map(clean_url_params);
let url = diesel_url_update(data.url.as_deref())?;
let custom_thumbnail = diesel_url_update(data.custom_thumbnail.as_deref())?;
let url_blocklist = get_url_blocklist(&context).await?;
let slur_regex = local_site_to_slur_regex(&local_site);
check_slurs_opt(&data.name, &slur_regex)?;
let body = process_markdown_opt(&data.body, &slur_regex, &url_blocklist, &context).await?;
let body = diesel_string_update(
process_markdown_opt(&data.body, &slur_regex, &url_blocklist, &context)
.await?
.as_deref(),
);
let alt_text = diesel_string_update(data.alt_text.as_deref());
if let Some(name) = &data.name {
is_valid_post_title(name)?;
check_slurs(name, &slur_regex)?;
}
is_valid_body_field(&body, true)?;
is_valid_alt_text_field(&data.alt_text)?;
is_url_blocked(&url, &url_blocklist)?;
check_url_scheme(&url)?;
check_url_scheme(&custom_thumbnail)?;
if let Some(Some(body)) = &body {
is_valid_body_field(body, true)?;
}
if let Some(Some(alt_text)) = &alt_text {
is_valid_alt_text_field(alt_text)?;
}
if let Some(Some(url)) = &url {
is_url_blocked(url, &url_blocklist)?;
check_url_scheme(url)?;
}
if let Some(Some(custom_thumbnail)) = &custom_thumbnail {
check_url_scheme(custom_thumbnail)?;
}
let post_id = data.post_id;
let orig_post = Post::read(&mut context.pool(), post_id)
@ -95,9 +111,9 @@ pub async fn update_post(
let post_form = PostUpdateForm {
name: data.name.clone(),
url: Some(url.map(Into::into)),
body: diesel_option_overwrite(body),
alt_text: diesel_option_overwrite(data.alt_text.clone()),
url,
body,
alt_text,
nsfw: data.nsfw,
language_id: data.language_id,
updated: Some(Some(naive_now())),
@ -111,7 +127,7 @@ pub async fn update_post(
generate_post_link_metadata(
updated_post.clone(),
custom_thumbnail,
custom_thumbnail.flatten().map(Into::into),
|post| Some(SendActivityData::UpdatePost(post)),
Some(local_site),
context.reset_request_count(),

2
crates/api_crud/src/private_message/create.rs
View File

@ -39,7 +39,7 @@ pub async fn create_private_message(
let slur_regex = local_site_to_slur_regex(&local_site);
let url_blocklist = get_url_blocklist(&context).await?;
let content = process_markdown(&data.content, &slur_regex, &url_blocklist, &context).await?;
is_valid_body_field(&Some(content.clone()), false)?;
is_valid_body_field(&content, false)?;
check_person_block(
local_user_view.person.id,

2
crates/api_crud/src/private_message/update.rs
View File

@ -41,7 +41,7 @@ pub async fn update_private_message(
let slur_regex = local_site_to_slur_regex(&local_site);
let url_blocklist = get_url_blocklist(&context).await?;
let content = process_markdown(&data.content, &slur_regex, &url_blocklist, &context).await?;
is_valid_body_field(&Some(content.clone()), false)?;
is_valid_body_field(&content, false)?;
let private_message_id = data.private_message_id;
PrivateMessage::update(

32
crates/api_crud/src/site/create.rs
View File

@ -11,7 +11,7 @@ use lemmy_api_common::{
local_site_rate_limit_to_rate_limit_config,
local_site_to_slur_regex,
process_markdown_opt,
proxy_image_link_opt_api,
proxy_image_link_api,
},
};
use lemmy_db_schema::{
@ -23,7 +23,7 @@ use lemmy_db_schema::{
tagline::Tagline,
},
traits::Crud,
utils::{diesel_option_overwrite, naive_now},
utils::{diesel_string_update, diesel_url_create, naive_now},
};
use lemmy_db_views::structs::{LocalUserView, SiteView};
use lemmy_utils::{
@ -61,21 +61,25 @@ pub async fn create_site(
let slur_regex = local_site_to_slur_regex(&local_site);
let url_blocklist = get_url_blocklist(&context).await?;
let sidebar = process_markdown_opt(&data.sidebar, &slur_regex, &url_blocklist, &context).await?;
let icon = proxy_image_link_opt_api(&data.icon, &context).await?;
let banner = proxy_image_link_opt_api(&data.banner, &context).await?;
let icon = diesel_url_create(data.icon.as_deref())?;
let icon = proxy_image_link_api(icon, &context).await?;
let banner = diesel_url_create(data.banner.as_deref())?;
let banner = proxy_image_link_api(banner, &context).await?;
let site_form = SiteUpdateForm {
name: Some(data.name.clone()),
sidebar: diesel_option_overwrite(sidebar),
description: diesel_option_overwrite(data.description.clone()),
icon,
banner,
sidebar: diesel_string_update(sidebar.as_deref()),
description: diesel_string_update(data.description.as_deref()),
icon: Some(icon),
banner: Some(banner),
actor_id: Some(actor_id),
last_refreshed_at: Some(naive_now()),
inbox_url,
private_key: Some(Some(keypair.private_key)),
public_key: Some(keypair.public_key),
content_warning: diesel_option_overwrite(data.content_warning.clone()),
content_warning: diesel_string_update(data.content_warning.as_deref()),
..Default::default()
};
@ -91,16 +95,16 @@ pub async fn create_site(
enable_nsfw: data.enable_nsfw,
community_creation_admin_only: data.community_creation_admin_only,
require_email_verification: data.require_email_verification,
application_question: diesel_option_overwrite(data.application_question.clone()),
application_question: diesel_string_update(data.application_question.as_deref()),
private_instance: data.private_instance,
default_theme: data.default_theme.clone(),
default_post_listing_type: data.default_post_listing_type,
default_sort_type: data.default_sort_type,
legal_information: diesel_option_overwrite(data.legal_information.clone()),
legal_information: diesel_string_update(data.legal_information.as_deref()),
application_email_admins: data.application_email_admins,
hide_modlog_mod_names: data.hide_modlog_mod_names,
updated: Some(Some(naive_now())),
slur_filter_regex: diesel_option_overwrite(data.slur_filter_regex.clone()),
slur_filter_regex: diesel_string_update(data.slur_filter_regex.as_deref()),
actor_name_max_length: data.actor_name_max_length,
federation_enabled: data.federation_enabled,
captcha_enabled: data.captcha_enabled,
@ -179,7 +183,9 @@ fn validate_create_payload(local_site: &LocalSite, create_site: &CreateSite) ->
)?;
// Ensure that the sidebar has fewer than the max num characters...
is_valid_body_field(&create_site.sidebar, false)?;
if let Some(body) = &create_site.sidebar {
is_valid_body_field(body, false)?;
}
application_question_check(
&local_site.application_question,

37
crates/api_crud/src/site/update.rs
View File

@ -27,7 +27,7 @@ use lemmy_db_schema::{
tagline::Tagline,
},
traits::Crud,
utils::{diesel_option_overwrite, naive_now},
utils::{diesel_string_update, diesel_url_update, naive_now},
RegistrationMode,
};
use lemmy_db_views::structs::{LocalUserView, SiteView};
@ -67,22 +67,29 @@ pub async fn update_site(
SiteLanguage::update(&mut context.pool(), discussion_languages.clone(), &site).await?;
}
replace_image(&data.icon, &site.icon, &context).await?;
replace_image(&data.banner, &site.banner, &context).await?;
let slur_regex = local_site_to_slur_regex(&local_site);
let url_blocklist = get_url_blocklist(&context).await?;
let sidebar = process_markdown_opt(&data.sidebar, &slur_regex, &url_blocklist, &context).await?;
let icon = proxy_image_link_opt_api(&data.icon, &context).await?;
let banner = proxy_image_link_opt_api(&data.banner, &context).await?;
let sidebar = diesel_string_update(
process_markdown_opt(&data.sidebar, &slur_regex, &url_blocklist, &context)
.await?
.as_deref(),
);
let icon = diesel_url_update(data.icon.as_deref())?;
replace_image(&icon, &site.icon, &context).await?;
let icon = proxy_image_link_opt_api(icon, &context).await?;
let banner = diesel_url_update(data.banner.as_deref())?;
replace_image(&banner, &site.banner, &context).await?;
let banner = proxy_image_link_opt_api(banner, &context).await?;
let site_form = SiteUpdateForm {
name: data.name.clone(),
sidebar: diesel_option_overwrite(sidebar),
description: diesel_option_overwrite(data.description.clone()),
sidebar,
description: diesel_string_update(data.description.as_deref()),
icon,
banner,
content_warning: diesel_option_overwrite(data.content_warning.clone()),
content_warning: diesel_string_update(data.content_warning.as_deref()),
updated: Some(Some(naive_now())),
..Default::default()
};
@ -99,16 +106,16 @@ pub async fn update_site(
enable_nsfw: data.enable_nsfw,
community_creation_admin_only: data.community_creation_admin_only,
require_email_verification: data.require_email_verification,
application_question: diesel_option_overwrite(data.application_question.clone()),
application_question: diesel_string_update(data.application_question.as_deref()),
private_instance: data.private_instance,
default_theme: data.default_theme.clone(),
default_post_listing_type: data.default_post_listing_type,
default_sort_type: data.default_sort_type,
legal_information: diesel_option_overwrite(data.legal_information.clone()),
legal_information: diesel_string_update(data.legal_information.as_deref()),
application_email_admins: data.application_email_admins,
hide_modlog_mod_names: data.hide_modlog_mod_names,
updated: Some(Some(naive_now())),
slur_filter_regex: diesel_option_overwrite(data.slur_filter_regex.clone()),
slur_filter_regex: diesel_string_update(data.slur_filter_regex.as_deref()),
actor_name_max_length: data.actor_name_max_length,
federation_enabled: data.federation_enabled,
captcha_enabled: data.captcha_enabled,
@ -229,7 +236,9 @@ fn validate_update_payload(local_site: &LocalSite, edit_site: &EditSite) -> Lemm
)?;
// Ensure that the sidebar has fewer than the max num characters...
is_valid_body_field(&edit_site.sidebar, false)?;
if let Some(body) = &edit_site.sidebar {
is_valid_body_field(body, false)?;
}
application_question_check(
&local_site.application_question,

20
crates/api_crud/src/user/create.rs
View File

@ -112,15 +112,17 @@ pub async fn register(
// We have to create both a person, and local_user
// Register the new person
let person_form = PersonInsertForm::builder()
.name(data.username.clone())
.actor_id(Some(actor_id.clone()))
.private_key(Some(actor_keypair.private_key))
.public_key(actor_keypair.public_key)
.inbox_url(Some(generate_inbox_url(&actor_id)?))
.shared_inbox_url(Some(generate_shared_inbox_url(context.settings())?))
.instance_id(site_view.site.instance_id)
.build();
let person_form = PersonInsertForm {
actor_id: Some(actor_id.clone()),
inbox_url: Some(generate_inbox_url(&actor_id)?),
shared_inbox_url: Some(generate_shared_inbox_url(context.settings())?),
private_key: Some(actor_keypair.private_key),
..PersonInsertForm::new(
data.username.clone(),
actor_keypair.public_key,
site_view.site.instance_id,
)
};
// insert the person
let inserted_person = Person::create(&mut context.pool(), &person_form)

12
crates/apub/src/api/list_comments.rs
View File

@ -37,11 +37,11 @@ pub async fn list_comments(
};
let sort = data.sort;
let max_depth = data.max_depth;
let saved_only = data.saved_only.unwrap_or_default();
let saved_only = data.saved_only;
let liked_only = data.liked_only.unwrap_or_default();
let disliked_only = data.disliked_only.unwrap_or_default();
if liked_only && disliked_only {
let liked_only = data.liked_only;
let disliked_only = data.disliked_only;
if liked_only.unwrap_or_default() && disliked_only.unwrap_or_default() {
return Err(LemmyError::from(LemmyErrorType::ContradictingFilters));
}
@ -70,6 +70,8 @@ pub async fn list_comments(
let parent_path_cloned = parent_path.clone();
let post_id = data.post_id;
let local_user = local_user_view.as_ref().map(|l| &l.local_user);
let comments = CommentQuery {
listing_type,
sort,
@ -80,7 +82,7 @@ pub async fn list_comments(
community_id,
parent_path: parent_path_cloned,
post_id,
local_user: local_user_view.as_ref(),
local_user,
page,
limit,
..Default::default()

20
crates/apub/src/api/list_posts.rs
View File

@ -40,26 +40,27 @@ pub async fn list_posts(
} else {
data.community_id
};
let saved_only = data.saved_only.unwrap_or_default();
let show_hidden = data.show_hidden.unwrap_or_default();
let saved_only = data.saved_only;
let show_hidden = data.show_hidden;
let show_read = data.show_read;
let liked_only = data.liked_only.unwrap_or_default();
let disliked_only = data.disliked_only.unwrap_or_default();
if liked_only && disliked_only {
let liked_only = data.liked_only;
let disliked_only = data.disliked_only;
if liked_only.unwrap_or_default() && disliked_only.unwrap_or_default() {
return Err(LemmyError::from(LemmyErrorType::ContradictingFilters));
}
let local_user_ref = local_user_view.as_ref().map(|u| &u.local_user);
let local_user = local_user_view.as_ref().map(|u| &u.local_user);
let listing_type = Some(listing_type_with_default(
data.type_,
local_user_ref,
local_user,
&local_site.local_site,
community_id,
));
let sort = Some(sort_type_with_default(
data.sort,
local_user_ref,
local_user,
&local_site.local_site,
));
@ -71,7 +72,7 @@ pub async fn list_posts(
};
let posts = PostQuery {
local_user: local_user_view.as_ref(),
local_user,
listing_type,
sort,
community_id,
@ -82,6 +83,7 @@ pub async fn list_posts(
page_after,
limit,
show_hidden,
show_read,
..Default::default()
}
.list(&local_site.site, &mut context.pool())

10
crates/apub/src/api/read_person.rs
View File

@ -55,20 +55,22 @@ pub async fn read_person(
let sort = data.sort;
let page = data.page;
let limit = data.limit;
let saved_only = data.saved_only.unwrap_or_default();
let saved_only = data.saved_only;
let community_id = data.community_id;
// If its saved only, you don't care what creator it was
// Or, if its not saved, then you only want it for that specific creator
let creator_id = if !saved_only {
let creator_id = if !saved_only.unwrap_or_default() {
Some(person_details_id)
} else {
None
};
let local_user = local_user_view.as_ref().map(|l| &l.local_user);
let posts = PostQuery {
sort,
saved_only,
local_user: local_user_view.as_ref(),
local_user,
community_id,
page,
limit,
@ -79,7 +81,7 @@ pub async fn read_person(
.await?;
let comments = CommentQuery {
local_user: local_user_view.as_ref(),
local_user,
sort: sort.map(post_to_comment_sort_type),
saved_only,
community_id,

11
crates/apub/src/api/search.rs
View File

@ -55,7 +55,7 @@ pub async fn search(
data.community_id
};
let creator_id = data.creator_id;
let local_user = local_user_view.as_ref().map(|luv| &luv.local_user);
let local_user = local_user_view.as_ref().map(|l| &l.local_user);
match search_type {
SearchType::Posts => {
@ -64,7 +64,7 @@ pub async fn search(
listing_type: (listing_type),
community_id: (community_id),
creator_id: (creator_id),
local_user: (local_user_view.as_ref()),
local_user,
search_term: (Some(q)),
page: (page),
limit: (limit),
@ -80,7 +80,7 @@ pub async fn search(
search_term: (Some(q)),
community_id: (community_id),
creator_id: (creator_id),
local_user: (local_user_view.as_ref()),
local_user,
page: (page),
limit: (limit),
..Default::default()
@ -125,7 +125,7 @@ pub async fn search(
listing_type: (listing_type),
community_id: (community_id),
creator_id: (creator_id),
local_user: (local_user_view.as_ref()),
local_user,
search_term: (Some(q)),
page: (page),
limit: (limit),
@ -142,7 +142,7 @@ pub async fn search(
search_term: (Some(q)),
community_id: (community_id),
creator_id: (creator_id),
local_user: (local_user_view.as_ref()),
local_user,
page: (page),
limit: (limit),
..Default::default()
@ -192,6 +192,7 @@ pub async fn search(
community_id: (community_id),
creator_id: (creator_id),
url_search: (Some(q)),
local_user,
page: (page),
limit: (limit),
..Default::default()

12
crates/apub/src/api/user_settings_backup.rs
View File

@ -338,13 +338,11 @@ mod tests {
context: &Data<LemmyContext>,
) -> LemmyResult<LocalUserView> {
let instance = Instance::read_or_create(&mut context.pool(), "example.com".to_string()).await?;
let person_form = PersonInsertForm::builder()
.name(name.clone())
.display_name(Some(name.clone()))
.bio(bio)
.public_key("asd".to_string())
.instance_id(instance.id)
.build();
let person_form = PersonInsertForm {
display_name: Some(name.clone()),
bio,
..PersonInsertForm::test_form(instance.id, &name)
};
let person = Person::create(&mut context.pool(), &person_form).await?;
let user_form = LocalUserInsertForm::builder()

6
crates/apub/src/collections/community_moderators.rs
View File

@ -129,11 +129,7 @@ mod tests {
let inserted_instance =
Instance::read_or_create(&mut context.pool(), "my_domain.tld".to_string()).await?;
let old_mod = PersonInsertForm::builder()
.name("holly".into())
.public_key("pubkey".to_string())
.instance_id(inserted_instance.id)
.build();
let old_mod = PersonInsertForm::test_form(inserted_instance.id, "holly");
let old_mod = Person::create(&mut context.pool(), &old_mod).await?;
let community_moderator_form = CommunityModeratorForm {

5
crates/apub/src/objects/post.rs
View File

@ -219,7 +219,10 @@ impl Object for ApubPost {
} else {
None
};
check_url_scheme(&url)?;
if let Some(url) = &url {
check_url_scheme(url)?;
}
let alt_text = first_attachment.cloned().and_then(Attachment::alt_text);

6
crates/db_perf/src/main.rs
View File

@ -72,11 +72,7 @@ async fn try_main() -> LemmyResult<()> {
println!("🫃 creating {} people", args.people);
let mut person_ids = vec![];
for i in 0..args.people.get() {
let form = PersonInsertForm::builder()
.name(format!("p{i}"))
.public_key("pubkey".to_owned())
.instance_id(instance.id)
.build();
let form = PersonInsertForm::test_form(instance.id, &format!("p{i}"));
person_ids.push(Person::create(&mut conn.into(), &form).await?.id);
}

1
crates/db_schema/Cargo.toml
View File

@ -81,6 +81,7 @@ uuid = { workspace = true, features = ["v4"] }
i-love-jesus = { workspace = true, optional = true }
anyhow = { workspace = true }
moka.workspace = true
derive-new.workspace = true
[dev-dependencies]
serial_test = { workspace = true }

19
crates/db_schema/replaceable_schema/triggers.sql
View File

@ -5,12 +5,17 @@
-- (even if only other columns are updated) because triggers can run after the deletion of referenced rows and
-- before the automatic deletion of the row that references it. This is not a problem for insert or delete.
--
-- After a row update begins, a concurrent update on the same row can't begin until the whole
-- transaction that contains the first update is finished. To reduce this locking, statements in
-- triggers should be ordered based on the likelihood of concurrent writers. For example, updating
-- site_aggregates should be done last because the same row is updated for all local stuff. If
-- it were not last, then the locking period for concurrent writers would extend to include the
-- time consumed by statements that come after.
-- Triggers that update multiple tables should use this order: person_aggregates, comment_aggregates,
-- post_aggregates, community_aggregates, site_aggregates
-- * The order matters because the updated rows are locked until the end of the transaction, and statements
-- in a trigger don't use separate transactions. This means that updates closer to the beginning cause
-- longer locks because the duration of each update extends the durations of the locks caused by previous
-- updates. Long locks are worse on rows that have more concurrent transactions trying to update them. The
-- listed order starts with tables that are less likely to have such rows.
-- https://www.postgresql.org/docs/16/transaction-iso.html#XACT-READ-COMMITTED
-- * Using the same order in every trigger matters because a deadlock is possible if multiple transactions
-- update the same rows in a different order.
-- https://www.postgresql.org/docs/current/explicit-locking.html#LOCKING-DEADLOCKS
--
--
-- Create triggers for both post and comments
@ -481,7 +486,7 @@ BEGIN
INNER JOIN old_post ON old_post.id = new_post.id
AND (old_post.featured_community,
old_post.featured_local) != (new_post.featured_community,
old_post.featured_local)
new_post.featured_local)
WHERE
post_aggregates.post_id = new_post.id;
RETURN NULL;

12
crates/db_schema/src/aggregates/comment_aggregates.rs
View File

@ -64,19 +64,11 @@ mod tests {
.await
.unwrap();
let new_person = PersonInsertForm::builder()
.name("thommy_comment_agg".into())
.public_key("pubkey".into())
.instance_id(inserted_instance.id)
.build();
let new_person = PersonInsertForm::test_form(inserted_instance.id, "thommy_comment_agg");
let inserted_person = Person::create(pool, &new_person).await.unwrap();
let another_person = PersonInsertForm::builder()
.name("jerry_comment_agg".into())
.public_key("pubkey".into())
.instance_id(inserted_instance.id)
.build();
let another_person = PersonInsertForm::test_form(inserted_instance.id, "jerry_comment_agg");
let another_inserted_person = Person::create(pool, &another_person).await.unwrap();

12
crates/db_schema/src/aggregates/community_aggregates.rs
View File

@ -65,19 +65,11 @@ mod tests {
.await
.unwrap();
let new_person = PersonInsertForm::builder()
.name("thommy_community_agg".into())
.public_key("pubkey".to_string())
.instance_id(inserted_instance.id)
.build();
let new_person = PersonInsertForm::test_form(inserted_instance.id, "thommy_community_agg");
let inserted_person = Person::create(pool, &new_person).await.unwrap();
let another_person = PersonInsertForm::builder()
.name("jerry_community_agg".into())
.public_key("pubkey".to_string())
.instance_id(inserted_instance.id)
.build();
let another_person = PersonInsertForm::test_form(inserted_instance.id, "jerry_community_agg");
let another_inserted_person = Person::create(pool, &another_person).await.unwrap();

12
crates/db_schema/src/aggregates/person_aggregates.rs
View File

@ -49,19 +49,11 @@ mod tests {
.await
.unwrap();
let new_person = PersonInsertForm::builder()
.name("thommy_user_agg".into())
.public_key("pubkey".to_string())
.instance_id(inserted_instance.id)
.build();
let new_person = PersonInsertForm::test_form(inserted_instance.id, "thommy_user_agg");
let inserted_person = Person::create(pool, &new_person).await.unwrap();
let another_person = PersonInsertForm::builder()
.name("jerry_user_agg".into())
.public_key("pubkey".to_string())
.instance_id(inserted_instance.id)
.build();
let another_person = PersonInsertForm::test_form(inserted_instance.id, "jerry_user_agg");
let another_inserted_person = Person::create(pool, &another_person).await.unwrap();

18
crates/db_schema/src/aggregates/post_aggregates.rs
View File

@ -83,19 +83,11 @@ mod tests {
.await
.unwrap();
let new_person = PersonInsertForm::builder()
.name("thommy_community_agg".into())
.public_key("pubkey".to_string())
.instance_id(inserted_instance.id)
.build();
let new_person = PersonInsertForm::test_form(inserted_instance.id, "thommy_community_agg");
let inserted_person = Person::create(pool, &new_person).await.unwrap();
let another_person = PersonInsertForm::builder()
.name("jerry_community_agg".into())
.public_key("pubkey".to_string())
.instance_id(inserted_instance.id)
.build();
let another_person = PersonInsertForm::test_form(inserted_instance.id, "jerry_community_agg");
let another_inserted_person = Person::create(pool, &another_person).await.unwrap();
@ -229,11 +221,7 @@ mod tests {
.await
.unwrap();
let new_person = PersonInsertForm::builder()
.name("thommy_community_agg".into())
.public_key("pubkey".to_string())
.instance_id(inserted_instance.id)
.build();
let new_person = PersonInsertForm::test_form(inserted_instance.id, "thommy_community_agg");
let inserted_person = Person::create(pool, &new_person).await.unwrap();

6
crates/db_schema/src/aggregates/site_aggregates.rs
View File

@ -42,11 +42,7 @@ mod tests {
.await
.unwrap();
let new_person = PersonInsertForm::builder()
.name("thommy_site_agg".into())
.public_key("pubkey".to_string())
.instance_id(inserted_instance.id)
.build();
let new_person = PersonInsertForm::test_form(inserted_instance.id, "thommy_site_agg");
let inserted_person = Person::create(pool, &new_person).await.unwrap();

12
crates/db_schema/src/impls/actor_language.rs
View File

@ -531,11 +531,7 @@ mod tests {
let (site, instance) = create_test_site(pool).await;
let person_form = PersonInsertForm::builder()
.name("my test person".to_string())
.public_key("pubkey".to_string())
.instance_id(instance.id)
.build();
let person_form = PersonInsertForm::test_form(instance.id, "my test person");
let person = Person::create(pool, &person_form).await.unwrap();
let local_user_form = LocalUserInsertForm::builder()
.person_id(person.id)
@ -647,11 +643,7 @@ mod tests {
.await
.unwrap();
let person_form = PersonInsertForm::builder()
.name("my test person".to_string())
.public_key("pubkey".to_string())
.instance_id(instance.id)
.build();
let person_form = PersonInsertForm::test_form(instance.id, "my test person");
let person = Person::create(pool, &person_form).await.unwrap();
let local_user_form = LocalUserInsertForm::builder()
.person_id(person.id)

11
crates/db_schema/src/impls/comment.rs
View File

@ -118,8 +118,9 @@ impl Crud for Comment {
type IdType = CommentId;
/// This is unimplemented, use [[Comment::create]]
async fn create(_pool: &mut DbPool<'_>, _comment_form: &Self::InsertForm) -> Result<Self, Error> {
unimplemented!();
async fn create(pool: &mut DbPool<'_>, comment_form: &Self::InsertForm) -> Result<Self, Error> {
debug_assert!(false);
Comment::create(pool, comment_form, None).await
}
async fn update(
@ -233,11 +234,7 @@ mod tests {
.await
.unwrap();
let new_person = PersonInsertForm::builder()
.name("terry".into())
.public_key("pubkey".to_string())
.instance_id(inserted_instance.id)
.build();
let new_person = PersonInsertForm::test_form(inserted_instance.id, "terry");
let inserted_person = Person::create(pool, &new_person).await.unwrap();

6
crates/db_schema/src/impls/community.rs
View File

@ -434,11 +434,7 @@ mod tests {
.await
.unwrap();
let new_person = PersonInsertForm::builder()
.name("bobbee".into())
.public_key("pubkey".to_string())
.instance_id(inserted_instance.id)
.build();
let new_person = PersonInsertForm::test_form(inserted_instance.id, "bobbee");
let inserted_person = Person::create(pool, &new_person).await.unwrap();

78
crates/db_schema/src/impls/images.rs
View File

@ -1,7 +1,14 @@
use crate::{
newtypes::DbUrl,
schema::{local_image, remote_image},
source::images::{LocalImage, LocalImageForm, RemoteImage, RemoteImageForm},
schema::{image_details, local_image, remote_image},
source::images::{
ImageDetails,
ImageDetailsForm,
LocalImage,
LocalImageForm,
RemoteImage,
RemoteImageForm,
},
utils::{get_conn, DbPool},
};
use diesel::{
@ -13,15 +20,29 @@ use diesel::{
NotFound,
QueryDsl,
};
use diesel_async::RunQueryDsl;
use url::Url;
use diesel_async::{AsyncPgConnection, RunQueryDsl};
impl LocalImage {
pub async fn create(pool: &mut DbPool<'_>, form: &LocalImageForm) -> Result<Self, Error> {
pub async fn create(
pool: &mut DbPool<'_>,
form: &LocalImageForm,
image_details_form: &ImageDetailsForm,
) -> Result<Self, Error> {
let conn = &mut get_conn(pool).await?;
insert_into(local_image::table)
.values(form)
.get_result::<Self>(conn)
conn
.build_transaction()
.run(|conn| {
Box::pin(async move {
let local_insert = insert_into(local_image::table)
.values(form)
.get_result::<Self>(conn)
.await;
ImageDetails::create(conn, image_details_form).await?;
local_insert
}) as _
})
.await
}
@ -39,16 +60,26 @@ impl LocalImage {
}
impl RemoteImage {
pub async fn create(pool: &mut DbPool<'_>, links: Vec<Url>) -> Result<usize, Error> {
pub async fn create(pool: &mut DbPool<'_>, form: &ImageDetailsForm) -> Result<usize, Error> {
let conn = &mut get_conn(pool).await?;
let forms = links
.into_iter()
.map(|url| RemoteImageForm { link: url.into() })
.collect::<Vec<_>>();
insert_into(remote_image::table)
.values(forms)
.on_conflict_do_nothing()
.execute(conn)
conn
.build_transaction()
.run(|conn| {
Box::pin(async move {
let remote_image_form = RemoteImageForm {
link: form.link.clone(),
};
let remote_insert = insert_into(remote_image::table)
.values(remote_image_form)
.on_conflict_do_nothing()
.execute(conn)
.await;
ImageDetails::create(conn, form).await?;
remote_insert
}) as _
})
.await
}
@ -67,3 +98,16 @@ impl RemoteImage {
}
}
}
impl ImageDetails {
pub(crate) async fn create(
conn: &mut AsyncPgConnection,
form: &ImageDetailsForm,
) -> Result<usize, Error> {
insert_into(image_details::table)
.values(form)
.on_conflict_do_nothing()
.execute(conn)
.await
}
}

39
crates/db_schema/src/impls/local_user.rs
View File

@ -5,6 +5,7 @@ use crate::{
actor_language::LocalUserLanguage,
local_user::{LocalUser, LocalUserInsertForm, LocalUserUpdateForm},
local_user_vote_display_mode::{LocalUserVoteDisplayMode, LocalUserVoteDisplayModeInsertForm},
site::Site,
},
utils::{
functions::{coalesce, lower},
@ -216,6 +217,44 @@ impl LocalUser {
}
}
/// Adds some helper functions for an optional LocalUser
pub trait LocalUserOptionHelper {
fn person_id(&self) -> Option<PersonId>;
fn local_user_id(&self) -> Option<LocalUserId>;
fn show_bot_accounts(&self) -> bool;
fn show_read_posts(&self) -> bool;
fn is_admin(&self) -> bool;
fn show_nsfw(&self, site: &Site) -> bool;
}
impl LocalUserOptionHelper for Option<&LocalUser> {
fn person_id(&self) -> Option<PersonId> {
self.map(|l| l.person_id)
}
fn local_user_id(&self) -> Option<LocalUserId> {
self.map(|l| l.id)
}
fn show_bot_accounts(&self) -> bool {
self.map(|l| l.show_bot_accounts).unwrap_or(true)
}
fn show_read_posts(&self) -> bool {
self.map(|l| l.show_read_posts).unwrap_or(true)
}
fn is_admin(&self) -> bool {
self.map(|l| l.admin).unwrap_or(false)
}
fn show_nsfw(&self, site: &Site) -> bool {
self
.map(|l| l.show_nsfw)
.unwrap_or(site.content_warning.is_some())
}
}
impl LocalUserInsertForm {
pub fn test_form(person_id: PersonId) -> Self {
Self::builder()

12
crates/db_schema/src/impls/moderator.rs
View File

@ -513,19 +513,11 @@ mod tests {
.await
.unwrap();
let new_mod = PersonInsertForm::builder()
.name("the mod".into())
.public_key("pubkey".to_string())
.instance_id(inserted_instance.id)
.build();
let new_mod = PersonInsertForm::test_form(inserted_instance.id, "the mod");
let inserted_mod = Person::create(pool, &new_mod).await.unwrap();
let new_person = PersonInsertForm::builder()
.name("jim2".into())
.public_key("pubkey".to_string())
.instance_id(inserted_instance.id)
.build();
let new_person = PersonInsertForm::test_form(inserted_instance.id, "jim2");
let inserted_person = Person::create(pool, &new_person).await.unwrap();

6
crates/db_schema/src/impls/password_reset_request.rs
View File

@ -70,11 +70,7 @@ mod tests {
// Setup
let inserted_instance = Instance::read_or_create(pool, "my_domain.tld".to_string()).await?;
let new_person = PersonInsertForm::builder()
.name("thommy prw".into())
.public_key("pubkey".to_string())
.instance_id(inserted_instance.id)
.build();
let new_person = PersonInsertForm::test_form(inserted_instance.id, "thommy prw");
let inserted_person = Person::create(pool, &new_person).await?;
let new_local_user = LocalUserInsertForm::builder()
.person_id(inserted_person.id)

37
crates/db_schema/src/impls/person.rs
View File

@ -12,7 +12,14 @@ use crate::{
traits::{ApubActor, Crud, Followable},
utils::{functions::lower, get_conn, naive_now, DbPool},
};
use diesel::{dsl::insert_into, result::Error, CombineDsl, ExpressionMethods, JoinOnDsl, QueryDsl};
use diesel::{
dsl::{insert_into, not},
result::Error,
CombineDsl,
ExpressionMethods,
JoinOnDsl,
QueryDsl,
};
use diesel_async::RunQueryDsl;
#[async_trait]
@ -100,6 +107,8 @@ impl Person {
.inner_join(post::table)
.inner_join(community::table.on(post::community_id.eq(community::id)))
.filter(community::local.eq(true))
.filter(not(community::deleted))
.filter(not(community::removed))
.filter(comment::creator_id.eq(for_creator_id))
.select(community::id)
.union(
@ -116,11 +125,7 @@ impl Person {
impl PersonInsertForm {
pub fn test_form(instance_id: InstanceId, name: &str) -> Self {
Self::builder()
.name(name.to_owned())
.public_key("pubkey".to_string())
.instance_id(instance_id)
.build()
Self::new(name.to_owned(), "pubkey".to_string(), instance_id)
}
}
@ -240,11 +245,7 @@ mod tests {
.await
.unwrap();
let new_person = PersonInsertForm::builder()
.name("holly".into())
.public_key("nada".to_owned())
.instance_id(inserted_instance.id)
.build();
let new_person = PersonInsertForm::test_form(inserted_instance.id, "holly");
let inserted_person = Person::create(pool, &new_person).await.unwrap();
@ -263,7 +264,7 @@ mod tests {
local: true,
bot_account: false,
private_key: None,
public_key: "nada".to_owned(),
public_key: "pubkey".to_owned(),
last_refreshed_at: inserted_person.published,
inbox_url: inserted_person.inbox_url.clone(),
shared_inbox_url: None,
@ -303,17 +304,9 @@ mod tests {
.await
.unwrap();
let person_form_1 = PersonInsertForm::builder()
.name("erich".into())
.public_key("pubkey".to_string())
.instance_id(inserted_instance.id)
.build();
let person_form_1 = PersonInsertForm::test_form(inserted_instance.id, "erich");
let person_1 = Person::create(pool, &person_form_1).await.unwrap();
let person_form_2 = PersonInsertForm::builder()
.name("michele".into())
.public_key("pubkey".to_string())
.instance_id(inserted_instance.id)
.build();
let person_form_2 = PersonInsertForm::test_form(inserted_instance.id, "michele");
let person_2 = Person::create(pool, &person_form_2).await.unwrap();
let follow_form = PersonFollowerForm {

6
crates/db_schema/src/impls/post.rs
View File

@ -401,11 +401,7 @@ mod tests {
.await
.unwrap();
let new_person = PersonInsertForm::builder()
.name("jim".into())
.public_key("pubkey".to_string())
.instance_id(inserted_instance.id)
.build();
let new_person = PersonInsertForm::test_form(inserted_instance.id, "jim");
let inserted_person = Person::create(pool, &new_person).await.unwrap();

6
crates/db_schema/src/impls/post_report.rs
View File

@ -101,11 +101,7 @@ mod tests {
let inserted_instance = Instance::read_or_create(pool, "my_domain.tld".to_string())
.await
.unwrap();
let person_form = PersonInsertForm::builder()
.name("jim".into())
.public_key("pubkey".to_string())
.instance_id(inserted_instance.id)
.build();
let person_form = PersonInsertForm::test_form(inserted_instance.id, "jim");
let person = Person::create(pool, &person_form).await.unwrap();
let community_form = CommunityInsertForm::builder()

12
crates/db_schema/src/impls/private_message.rs
View File

@ -111,19 +111,11 @@ mod tests {
.await
.unwrap();
let creator_form = PersonInsertForm::builder()
.name("creator_pm".into())
.public_key("pubkey".to_string())
.instance_id(inserted_instance.id)
.build();
let creator_form = PersonInsertForm::test_form(inserted_instance.id, "creator_pm");
let inserted_creator = Person::create(pool, &creator_form).await.unwrap();
let recipient_form = PersonInsertForm::builder()
.name("recipient_pm".into())
.public_key("pubkey".to_string())
.instance_id(inserted_instance.id)
.build();
let recipient_form = PersonInsertForm::test_form(inserted_instance.id, "recipient_pm");
let inserted_recipient = Person::create(pool, &recipient_form).await.unwrap();

13
crates/db_schema/src/schema.rs
View File

@ -309,6 +309,15 @@ diesel::table! {
}
}
diesel::table! {
image_details (link) {
link -> Text,
width -> Int4,
height -> Int4,
content_type -> Text,
}
}
diesel::table! {
instance (id) {
id -> Int4,
@ -849,8 +858,7 @@ diesel::table! {
}
diesel::table! {
remote_image (id) {
id -> Int4,
remote_image (link) {
link -> Text,
published -> Timestamptz,
}
@ -1055,6 +1063,7 @@ diesel::allow_tables_to_appear_in_same_query!(
federation_allowlist,
federation_blocklist,
federation_queue_state,
image_details,
instance,
instance_block,
language,

33
crates/db_schema/src/source/images.rs
View File

@ -1,13 +1,12 @@
use crate::newtypes::{DbUrl, LocalUserId};
#[cfg(feature = "full")]
use crate::schema::{local_image, remote_image};
use crate::schema::{image_details, local_image, remote_image};
use chrono::{DateTime, Utc};
use serde::{Deserialize, Serialize};
use serde_with::skip_serializing_none;
use std::fmt::Debug;
#[cfg(feature = "full")]
use ts_rs::TS;
use typed_builder::TypedBuilder;
#[skip_serializing_none]
#[derive(Clone, PartialEq, Eq, Debug, Serialize, Deserialize)]
@ -30,7 +29,7 @@ pub struct LocalImage {
pub published: DateTime<Utc>,
}
#[derive(Debug, Clone, TypedBuilder)]
#[derive(Debug, Clone)]
#[cfg_attr(feature = "full", derive(Insertable, AsChangeset))]
#[cfg_attr(feature = "full", diesel(table_name = local_image))]
pub struct LocalImageForm {
@ -46,15 +45,39 @@ pub struct LocalImageForm {
#[cfg_attr(feature = "full", derive(Queryable, Selectable, Identifiable))]
#[cfg_attr(feature = "full", diesel(table_name = remote_image))]
#[cfg_attr(feature = "full", diesel(check_for_backend(diesel::pg::Pg)))]
#[cfg_attr(feature = "full", diesel(primary_key(link)))]
pub struct RemoteImage {
pub id: i32,
pub link: DbUrl,
pub published: DateTime<Utc>,
}
#[derive(Debug, Clone, TypedBuilder)]
#[derive(Debug, Clone)]
#[cfg_attr(feature = "full", derive(Insertable, AsChangeset))]
#[cfg_attr(feature = "full", diesel(table_name = remote_image))]
pub struct RemoteImageForm {
pub link: DbUrl,
}
#[skip_serializing_none]
#[derive(Clone, PartialEq, Eq, Debug, Serialize, Deserialize)]
#[cfg_attr(feature = "full", derive(Queryable, Selectable, Identifiable, TS))]
#[cfg_attr(feature = "full", ts(export))]
#[cfg_attr(feature = "full", diesel(table_name = image_details))]
#[cfg_attr(feature = "full", diesel(check_for_backend(diesel::pg::Pg)))]
#[cfg_attr(feature = "full", diesel(primary_key(link)))]
pub struct ImageDetails {
pub link: DbUrl,
pub width: i32,
pub height: i32,
pub content_type: String,
}
#[derive(Debug, Clone)]
#[cfg_attr(feature = "full", derive(Insertable, AsChangeset))]
#[cfg_attr(feature = "full", diesel(table_name = image_details))]
pub struct ImageDetailsForm {
pub link: DbUrl,
pub width: i32,
pub height: i32,
pub content_type: String,
}

24
crates/db_schema/src/source/person.rs
View File

@ -10,7 +10,6 @@ use serde::{Deserialize, Serialize};
use serde_with::skip_serializing_none;
#[cfg(feature = "full")]
use ts_rs::TS;
use typed_builder::TypedBuilder;
#[skip_serializing_none]
#[derive(Clone, PartialEq, Eq, Debug, Serialize, Deserialize)]
@ -60,33 +59,46 @@ pub struct Person {
pub instance_id: InstanceId,
}
#[derive(Clone, TypedBuilder)]
#[builder(field_defaults(default))]
#[derive(Clone, derive_new::new)]
#[cfg_attr(feature = "full", derive(Insertable, AsChangeset))]
#[cfg_attr(feature = "full", diesel(table_name = person))]
pub struct PersonInsertForm {
#[builder(!default)]
pub name: String,
#[builder(!default)]
pub public_key: String,
#[builder(!default)]
pub instance_id: InstanceId,
#[new(default)]
pub display_name: Option<String>,
#[new(default)]
pub avatar: Option<DbUrl>,
#[new(default)]
pub banned: Option<bool>,
#[new(default)]
pub published: Option<DateTime<Utc>>,
#[new(default)]
pub updated: Option<DateTime<Utc>>,
#[new(default)]
pub actor_id: Option<DbUrl>,
#[new(default)]
pub bio: Option<String>,
#[new(default)]
pub local: Option<bool>,
#[new(default)]
pub private_key: Option<String>,
#[new(default)]
pub last_refreshed_at: Option<DateTime<Utc>>,
#[new(default)]
pub banner: Option<DbUrl>,
#[new(default)]
pub deleted: Option<bool>,
#[new(default)]
pub inbox_url: Option<DbUrl>,
#[new(default)]
pub shared_inbox_url: Option<DbUrl>,
#[new(default)]
pub matrix_user_id: Option<String>,
#[new(default)]
pub bot_account: Option<bool>,
#[new(default)]
pub ban_expires: Option<DateTime<Utc>>,
}

82
crates/db_schema/src/utils.rs
View File

@ -1,7 +1,15 @@
use crate::{newtypes::DbUrl, CommentSortType, SortType};
use crate::{
diesel::ExpressionMethods,
newtypes::{DbUrl, PersonId},
schema::community,
CommentSortType,
CommunityVisibility,
SortType,
};
use chrono::{DateTime, TimeDelta, Utc};
use deadpool::Runtime;
use diesel::{
dsl,
helper_types::AsExprOf,
pg::Pg,
query_builder::{Query, QueryFragment},
@ -29,6 +37,7 @@ use i_love_jesus::CursorKey;
use lemmy_utils::{
error::{LemmyErrorExt, LemmyErrorType, LemmyResult},
settings::SETTINGS,
utils::validation::clean_url_params,
};
use once_cell::sync::Lazy;
use regex::Regex;
@ -287,37 +296,35 @@ pub fn is_email_regex(test: &str) -> bool {
EMAIL_REGEX.is_match(test)
}
pub fn diesel_option_overwrite(opt: Option<String>) -> Option<Option<String>> {
/// Takes an API text input, and converts it to an optional diesel DB update.
pub fn diesel_string_update(opt: Option<&str>) -> Option<Option<String>> {
match opt {
// An empty string is an erase
Some(unwrapped) => {
if !unwrapped.eq("") {
Some(Some(unwrapped))
} else {
Some(None)
}
}
Some("") => Some(None),
Some(str) => Some(Some(str.into())),
None => None,
}
}
pub fn diesel_option_overwrite_to_url(opt: &Option<String>) -> LemmyResult<Option<Option<DbUrl>>> {
match opt.as_ref().map(String::as_str) {
/// Takes an optional API URL-type input, and converts it to an optional diesel DB update.
/// Also cleans the url params.
pub fn diesel_url_update(opt: Option<&str>) -> LemmyResult<Option<Option<DbUrl>>> {
match opt {
// An empty string is an erase
Some("") => Ok(Some(None)),
Some(str_url) => Url::parse(str_url)
.map(|u| Some(Some(u.into())))
.map(|u| Some(Some(clean_url_params(&u).into())))
.with_lemmy_type(LemmyErrorType::InvalidUrl),
None => Ok(None),
}
}
pub fn diesel_option_overwrite_to_url_create(opt: &Option<String>) -> LemmyResult<Option<DbUrl>> {
match opt.as_ref().map(String::as_str) {
// An empty string is nothing
Some("") => Ok(None),
/// Takes an optional API URL-type input, and converts it to an optional diesel DB create.
/// Also cleans the url params.
pub fn diesel_url_create(opt: Option<&str>) -> LemmyResult<Option<DbUrl>> {
match opt {
Some(str_url) => Url::parse(str_url)
.map(|u| Some(u.into()))
.map(|u| Some(clean_url_params(&u).into()))
.with_lemmy_type(LemmyErrorType::InvalidUrl),
None => Ok(None),
}
@ -325,6 +332,10 @@ pub fn diesel_option_overwrite_to_url_create(opt: &Option<String>) -> LemmyResul
fn establish_connection(config: &str) -> BoxFuture<ConnectionResult<AsyncPgConnection>> {
let fut = async {
rustls::crypto::ring::default_provider()
.install_default()
.expect("Failed to install rustls crypto provider");
let rustls_config = DangerousClientConfigBuilder {
cfg: ClientConfig::builder(),
}
@ -568,8 +579,21 @@ impl<RF, LF> Queries<RF, LF> {
}
}
pub fn visible_communities_only<Q>(my_person_id: Option<PersonId>, query: Q) -> Q
where
Q: diesel::query_dsl::methods::FilterDsl<
dsl::Eq<community::visibility, CommunityVisibility>,
Output = Q,
>,
{
if my_person_id.is_none() {
query.filter(community::visibility.eq(CommunityVisibility::Public))
} else {
query
}
}
#[cfg(test)]
#[allow(clippy::unwrap_used)]
#[allow(clippy::indexing_slicing)]
mod tests {
@ -593,26 +617,24 @@ mod tests {
#[test]
fn test_diesel_option_overwrite() {
assert_eq!(diesel_option_overwrite(None), None);
assert_eq!(diesel_option_overwrite(Some(String::new())), Some(None));
assert_eq!(diesel_string_update(None), None);
assert_eq!(diesel_string_update(Some("")), Some(None));
assert_eq!(
diesel_option_overwrite(Some("test".to_string())),
diesel_string_update(Some("test")),
Some(Some("test".to_string()))
);
}
#[test]
fn test_diesel_option_overwrite_to_url() {
assert!(matches!(diesel_option_overwrite_to_url(&None), Ok(None)));
assert!(matches!(
diesel_option_overwrite_to_url(&Some(String::new())),
Ok(Some(None))
));
assert!(diesel_option_overwrite_to_url(&Some("invalid_url".to_string())).is_err());
fn test_diesel_option_overwrite_to_url() -> LemmyResult<()> {
assert!(matches!(diesel_url_update(None), Ok(None)));
assert!(matches!(diesel_url_update(Some("")), Ok(Some(None))));
assert!(diesel_url_update(Some("invalid_url")).is_err());
let example_url = "https://example.com";
assert!(matches!(
diesel_option_overwrite_to_url(&Some(example_url.to_string())),
Ok(Some(Some(url))) if url == Url::parse(example_url).unwrap().into()
diesel_url_update(Some(example_url)),
Ok(Some(Some(url))) if url == Url::parse(example_url)?.into()
));
Ok(())
}
}

18
crates/db_views/src/comment_report_view.rs
View File

@ -297,11 +297,7 @@ mod tests {
.await
.unwrap();
let new_person = PersonInsertForm::builder()
.name("timmy_crv".into())
.public_key("pubkey".to_string())
.instance_id(inserted_instance.id)
.build();
let new_person = PersonInsertForm::test_form(inserted_instance.id, "timmy_crv");
let inserted_timmy = Person::create(pool, &new_person).await.unwrap();
@ -319,20 +315,12 @@ mod tests {
counts: Default::default(),
};
let new_person_2 = PersonInsertForm::builder()
.name("sara_crv".into())
.public_key("pubkey".to_string())
.instance_id(inserted_instance.id)
.build();
let new_person_2 = PersonInsertForm::test_form(inserted_instance.id, "sara_crv");
let inserted_sara = Person::create(pool, &new_person_2).await.unwrap();
// Add a third person, since new ppl can only report something once.
let new_person_3 = PersonInsertForm::builder()
.name("jessica_crv".into())
.public_key("pubkey".to_string())
.instance_id(inserted_instance.id)
.build();
let new_person_3 = PersonInsertForm::test_form(inserted_instance.id, "jessica_crv");
let inserted_jessica = Person::create(pool, &new_person_3).await.unwrap();

94
crates/db_views/src/comment_view.rs
View File

@ -1,4 +1,4 @@
use crate::structs::{CommentView, LocalUserView};
use crate::structs::CommentView;
use diesel::{
dsl::{exists, not},
pg::Pg,
@ -16,6 +16,7 @@ use diesel::{
use diesel_async::RunQueryDsl;
use diesel_ltree::{nlevel, subpath, Ltree, LtreeExtensions};
use lemmy_db_schema::{
impls::local_user::LocalUserOptionHelper,
newtypes::{CommentId, CommunityId, LocalUserId, PersonId, PostId},
schema::{
comment,
@ -34,9 +35,18 @@ use lemmy_db_schema::{
person_block,
post,
},
utils::{fuzzy_search, limit_and_offset, DbConn, DbPool, ListFn, Queries, ReadFn},
source::local_user::LocalUser,
utils::{
fuzzy_search,
limit_and_offset,
visible_communities_only,
DbConn,
DbPool,
ListFn,
Queries,
ReadFn,
},
CommentSortType,
CommunityVisibility,
ListingType,
};
@ -174,22 +184,19 @@ fn queries<'a>() -> Queries<
let read = move |mut conn: DbConn<'a>,
(comment_id, my_person_id): (CommentId, Option<PersonId>)| async move {
let mut query = all_joins(comment::table.find(comment_id).into_boxed(), my_person_id);
// Hide local only communities from unauthenticated users
if my_person_id.is_none() {
query = query.filter(community::visibility.eq(CommunityVisibility::Public));
}
query = visible_communities_only(my_person_id, query);
query.first(&mut conn).await
};
let list = move |mut conn: DbConn<'a>, options: CommentQuery<'a>| async move {
let my_person_id = options.local_user.map(|l| l.person.id);
let my_local_user_id = options.local_user.map(|l| l.local_user.id);
// The left join below will return None in this case
let person_id_join = my_person_id.unwrap_or(PersonId(-1));
let local_user_id_join = my_local_user_id.unwrap_or(LocalUserId(-1));
let person_id_join = options.local_user.person_id().unwrap_or(PersonId(-1));
let local_user_id_join = options
.local_user
.local_user_id()
.unwrap_or(LocalUserId(-1));
let mut query = all_joins(comment::table.into_boxed(), my_person_id);
let mut query = all_joins(comment::table.into_boxed(), options.local_user.person_id());
if let Some(creator_id) = options.creator_id {
query = query.filter(comment::creator_id.eq(creator_id));
@ -245,26 +252,22 @@ fn queries<'a>() -> Queries<
}
// If its saved only, then filter, and order by the saved time, not the comment creation time.
if options.saved_only {
if options.saved_only.unwrap_or_default() {
query = query
.filter(comment_saved::person_id.is_not_null())
.then_order_by(comment_saved::published.desc());
}
if let Some(my_id) = my_person_id {
if let Some(my_id) = options.local_user.person_id() {
let not_creator_filter = comment::creator_id.ne(my_id);
if options.liked_only {
if options.liked_only.unwrap_or_default() {
query = query.filter(not_creator_filter).filter(score(my_id).eq(1));
} else if options.disliked_only {
} else if options.disliked_only.unwrap_or_default() {
query = query.filter(not_creator_filter).filter(score(my_id).eq(-1));
}
}
if !options
.local_user
.map(|l| l.local_user.show_bot_accounts)
.unwrap_or(true)
{
if !options.local_user.show_bot_accounts() {
query = query.filter(person::bot_account.eq(false));
};
@ -298,10 +301,7 @@ fn queries<'a>() -> Queries<
query = query.filter(not(is_creator_blocked(person_id_join)));
};
// Hide comments in local only communities from unauthenticated users
if options.local_user.is_none() {
query = query.filter(community::visibility.eq(CommunityVisibility::Public));
}
query = visible_communities_only(options.local_user.person_id(), query);
// A Max depth given means its a tree fetch
let (limit, offset) = if let Some(max_depth) = options.max_depth {
@ -396,11 +396,11 @@ pub struct CommentQuery<'a> {
pub post_id: Option<PostId>,
pub parent_path: Option<Ltree>,
pub creator_id: Option<PersonId>,
pub local_user: Option<&'a LocalUserView>,
pub local_user: Option<&'a LocalUser>,
pub search_term: Option<String>,
pub saved_only: bool,
pub liked_only: bool,
pub disliked_only: bool,
pub saved_only: Option<bool>,
pub liked_only: Option<bool>,
pub disliked_only: Option<bool>,
pub page: Option<i64>,
pub limit: Option<i64>,
pub max_depth: Option<i32>,
@ -488,11 +488,7 @@ mod tests {
async fn init_data(pool: &mut DbPool<'_>) -> LemmyResult<Data> {
let inserted_instance = Instance::read_or_create(pool, "my_domain.tld".to_string()).await?;
let timmy_person_form = PersonInsertForm::builder()
.name("timmy".into())
.public_key("pubkey".to_string())
.instance_id(inserted_instance.id)
.build();
let timmy_person_form = PersonInsertForm::test_form(inserted_instance.id, "timmy");
let inserted_timmy_person = Person::create(pool, &timmy_person_form).await?;
let timmy_local_user_form = LocalUserInsertForm::builder()
.person_id(inserted_timmy_person.id)
@ -501,11 +497,7 @@ mod tests {
.build();
let inserted_timmy_local_user = LocalUser::create(pool, &timmy_local_user_form, vec![]).await?;
let sara_person_form = PersonInsertForm::builder()
.name("sara".into())
.public_key("pubkey".to_string())
.instance_id(inserted_instance.id)
.build();
let sara_person_form = PersonInsertForm::test_form(inserted_instance.id, "sara");
let inserted_sara_person = Person::create(pool, &sara_person_form).await?;
let new_community = CommunityInsertForm::builder()
@ -667,7 +659,7 @@ mod tests {
let read_comment_views_with_person = CommentQuery {
sort: (Some(CommentSortType::Old)),
post_id: (Some(data.inserted_post.id)),
local_user: (Some(&data.timmy_local_user_view)),
local_user: (Some(&data.timmy_local_user_view.local_user)),
..Default::default()
}
.list(pool)
@ -719,8 +711,8 @@ mod tests {
CommentLike::like(pool, &comment_like_form).await?;
let read_liked_comment_views = CommentQuery {
local_user: (Some(&data.timmy_local_user_view)),
liked_only: (true),
local_user: Some(&data.timmy_local_user_view.local_user),
liked_only: Some(true),
..Default::default()
}
.list(pool)
@ -735,8 +727,8 @@ mod tests {
assert_length!(1, read_liked_comment_views);
let read_disliked_comment_views: Vec<CommentView> = CommentQuery {
local_user: (Some(&data.timmy_local_user_view)),
disliked_only: (true),
local_user: Some(&data.timmy_local_user_view.local_user),
disliked_only: Some(true),
..Default::default()
}
.list(pool)
@ -830,7 +822,7 @@ mod tests {
// by default, user has all languages enabled and should see all comments
// (except from blocked user)
let all_languages = CommentQuery {
local_user: (Some(&data.timmy_local_user_view)),
local_user: (Some(&data.timmy_local_user_view.local_user)),
..Default::default()
}
.list(pool)
@ -848,7 +840,7 @@ mod tests {
)
.await?;
let finnish_comments = CommentQuery {
local_user: (Some(&data.timmy_local_user_view)),
local_user: (Some(&data.timmy_local_user_view.local_user)),
..Default::default()
}
.list(pool)
@ -874,7 +866,7 @@ mod tests {
)
.await?;
let undetermined_comment = CommentQuery {
local_user: (Some(&data.timmy_local_user_view)),
local_user: (Some(&data.timmy_local_user_view.local_user)),
..Default::default()
}
.list(pool)
@ -987,8 +979,8 @@ mod tests {
// Fetch the saved comments
let comments = CommentQuery {
local_user: Some(&data.timmy_local_user_view),
saved_only: true,
local_user: Some(&data.timmy_local_user_view.local_user),
saved_only: Some(true),
..Default::default()
}
.list(pool)
@ -1166,7 +1158,7 @@ mod tests {
assert_eq!(0, unauthenticated_query.len());
let authenticated_query = CommentQuery {
local_user: Some(&data.timmy_local_user_view),
local_user: Some(&data.timmy_local_user_view.local_user),
..Default::default()
}
.list(pool)

18
crates/db_views/src/post_report_view.rs
View File

@ -319,11 +319,7 @@ mod tests {
.await
.unwrap();
let new_person = PersonInsertForm::builder()
.name("timmy_prv".into())
.public_key("pubkey".to_string())
.instance_id(inserted_instance.id)
.build();
let new_person = PersonInsertForm::test_form(inserted_instance.id, "timmy_prv");
let inserted_timmy = Person::create(pool, &new_person).await.unwrap();
@ -341,20 +337,12 @@ mod tests {
counts: Default::default(),
};
let new_person_2 = PersonInsertForm::builder()
.name("sara_prv".into())
.public_key("pubkey".to_string())
.instance_id(inserted_instance.id)
.build();
let new_person_2 = PersonInsertForm::test_form(inserted_instance.id, "sara_prv");
let inserted_sara = Person::create(pool, &new_person_2).await.unwrap();
// Add a third person, since new ppl can only report something once.
let new_person_3 = PersonInsertForm::builder()
.name("jessica_prv".into())
.public_key("pubkey".to_string())
.instance_id(inserted_instance.id)
.build();
let new_person_3 = PersonInsertForm::test_form(inserted_instance.id, "jessica_prv");
let inserted_jessica = Person::create(pool, &new_person_3).await.unwrap();

142
crates/db_views/src/post_view.rs
View File

@ -1,4 +1,4 @@
use crate::structs::{LocalUserView, PaginationCursor, PostView};
use crate::structs::{PaginationCursor, PostView};
use diesel::{
debug_query,
dsl::{exists, not, IntervalDsl},
@ -20,6 +20,7 @@ use diesel_async::RunQueryDsl;
use i_love_jesus::PaginatedQueryBuilder;
use lemmy_db_schema::{
aggregates::structs::{post_aggregates_keys as key, PostAggregates},
impls::local_user::LocalUserOptionHelper,
newtypes::{CommunityId, LocalUserId, PersonId, PostId},
schema::{
community,
@ -27,6 +28,7 @@ use lemmy_db_schema::{
community_follower,
community_moderator,
community_person_ban,
image_details,
instance_block,
local_user,
local_user_language,
@ -40,13 +42,14 @@ use lemmy_db_schema::{
post_read,
post_saved,
},
source::site::Site,
source::{local_user::LocalUser, site::Site},
utils::{
functions::coalesce,
fuzzy_search,
get_conn,
limit_and_offset,
now,
visible_communities_only,
Commented,
DbConn,
DbPool,
@ -55,7 +58,6 @@ use lemmy_db_schema::{
ReadFn,
ReverseTimestampKey,
},
CommunityVisibility,
ListingType,
SortType,
};
@ -217,6 +219,7 @@ fn queries<'a>() -> Queries<
.inner_join(person::table)
.inner_join(community::table)
.inner_join(post::table)
.left_join(image_details::table.on(post::thumbnail_url.eq(image_details::link.nullable())))
.left_join(
post_saved::table.on(
post_aggregates::post_id
@ -228,6 +231,7 @@ fn queries<'a>() -> Queries<
post::all_columns,
person::all_columns,
community::all_columns,
image_details::all_columns.nullable(),
is_creator_banned_from_community,
is_local_user_banned_from_community_selection,
creator_is_moderator,
@ -285,10 +289,7 @@ fn queries<'a>() -> Queries<
);
}
// Hide posts in local only communities from unauthenticated users
if my_person_id.is_none() {
query = query.filter(community::visibility.eq(CommunityVisibility::Public));
}
query = visible_communities_only(my_person_id, query);
Commented::new(query)
.text("PostView::read")
@ -297,31 +298,30 @@ fn queries<'a>() -> Queries<
};
let list = move |mut conn: DbConn<'a>, (options, site): (PostQuery<'a>, &'a Site)| async move {
let my_person_id = options.local_user.map(|l| l.person.id);
let my_local_user_id = options.local_user.map(|l| l.local_user.id);
// The left join below will return None in this case
let person_id_join = my_person_id.unwrap_or(PersonId(-1));
let local_user_id_join = my_local_user_id.unwrap_or(LocalUserId(-1));
let person_id_join = options.local_user.person_id().unwrap_or(PersonId(-1));
let local_user_id_join = options
.local_user
.local_user_id()
.unwrap_or(LocalUserId(-1));
let mut query = all_joins(post_aggregates::table.into_boxed(), my_person_id);
let mut query = all_joins(
post_aggregates::table.into_boxed(),
options.local_user.person_id(),
);
// hide posts from deleted communities
query = query.filter(community::deleted.eq(false));
// only show deleted posts to creator
if let Some(person_id) = my_person_id {
if let Some(person_id) = options.local_user.person_id() {
query = query.filter(post::deleted.eq(false).or(post::creator_id.eq(person_id)));
} else {
query = query.filter(post::deleted.eq(false));
}
let is_admin = options
.local_user
.map(|l| l.local_user.admin)
.unwrap_or(false);
// only show removed posts to admin when viewing user profile
if !(options.creator_id.is_some() && is_admin) {
if !(options.creator_id.is_some() && options.local_user.is_admin()) {
query = query
.filter(community::removed.eq(false))
.filter(post::removed.eq(false));
@ -335,7 +335,7 @@ fn queries<'a>() -> Queries<
}
if let Some(listing_type) = options.listing_type {
if let Some(person_id) = my_person_id {
if let Some(person_id) = options.local_user.person_id() {
let is_subscribed = exists(
community_follower::table.filter(
post_aggregates::community_id
@ -392,28 +392,18 @@ fn queries<'a>() -> Queries<
.filter(not(post::removed.or(post::deleted)));
}
// If there is a content warning, show nsfw content by default.
let has_content_warning = site.content_warning.is_some();
if !options
.local_user
.map(|l| l.local_user.show_nsfw)
.unwrap_or(has_content_warning)
{
if !options.local_user.show_nsfw(site) {
query = query
.filter(post::nsfw.eq(false))
.filter(community::nsfw.eq(false));
};
if !options
.local_user
.map(|l| l.local_user.show_bot_accounts)
.unwrap_or(true)
{
if !options.local_user.show_bot_accounts() {
query = query.filter(person::bot_account.eq(false));
};
// If its saved only, then filter, and order by the saved time, not the comment creation time.
if let (true, Some(_person_id)) = (options.saved_only, my_person_id) {
if options.saved_only.unwrap_or_default() {
query = query
.filter(post_saved::person_id.is_not_null())
.then_order_by(post_saved::published.desc());
@ -421,41 +411,37 @@ fn queries<'a>() -> Queries<
// Only hide the read posts, if the saved_only is false. Otherwise ppl with the hide_read
// setting wont be able to see saved posts.
else if !options
.local_user
.map(|l| l.local_user.show_read_posts)
.unwrap_or(true)
.show_read
.unwrap_or(options.local_user.show_read_posts())
{
// Do not hide read posts when it is a user profile view
// Or, only hide read posts on non-profile views
if let (None, Some(person_id)) = (options.creator_id, my_person_id) {
if let (None, Some(person_id)) = (options.creator_id, options.local_user.person_id()) {
query = query.filter(not(is_read(person_id)));
}
}
if !options.show_hidden {
if !options.show_hidden.unwrap_or_default() {
// If a creator id isn't given (IE its on home or community pages), hide the hidden posts
if let (None, Some(person_id)) = (options.creator_id, my_person_id) {
if let (None, Some(person_id)) = (options.creator_id, options.local_user.person_id()) {
query = query.filter(not(is_hidden(person_id)));
}
}
if let Some(my_id) = my_person_id {
if let Some(my_id) = options.local_user.person_id() {
let not_creator_filter = post_aggregates::creator_id.ne(my_id);
if options.liked_only {
if options.liked_only.unwrap_or_default() {
query = query.filter(not_creator_filter).filter(score(my_id).eq(1));
} else if options.disliked_only {
} else if options.disliked_only.unwrap_or_default() {
query = query.filter(not_creator_filter).filter(score(my_id).eq(-1));
}
};
// Hide posts in local only communities from unauthenticated users
if options.local_user.is_none() {
query = query.filter(community::visibility.eq(CommunityVisibility::Public));
}
query = visible_communities_only(options.local_user.person_id(), query);
// Dont filter blocks or missing languages for moderator view type
if let (Some(person_id), false) = (
my_person_id,
options.local_user.person_id(),
options.listing_type.unwrap_or_default() == ListingType::ModeratorView,
) {
// Filter out the rows with missing languages
@ -493,7 +479,7 @@ fn queries<'a>() -> Queries<
let page_after = options.page_after.map(|c| c.0);
let page_before_or_equal = options.page_before_or_equal.map(|c| c.0);
if options.page_back {
if options.page_back.unwrap_or_default() {
query = query
.before(page_after)
.after_or_equal(page_before_or_equal)
@ -618,18 +604,19 @@ pub struct PostQuery<'a> {
// if true, the query should be handled as if community_id was not given except adding the
// literal filter
pub community_id_just_for_prefetch: bool,
pub local_user: Option<&'a LocalUserView>,
pub local_user: Option<&'a LocalUser>,
pub search_term: Option<String>,
pub url_search: Option<String>,
pub saved_only: bool,
pub liked_only: bool,
pub disliked_only: bool,
pub saved_only: Option<bool>,
pub liked_only: Option<bool>,
pub disliked_only: Option<bool>,
pub page: Option<i64>,
pub limit: Option<i64>,
pub page_after: Option<PaginationCursorData>,
pub page_before_or_equal: Option<PaginationCursorData>,
pub page_back: bool,
pub show_hidden: bool,
pub page_back: Option<bool>,
pub show_hidden: Option<bool>,
pub show_read: Option<bool>,
}
impl<'a> PostQuery<'a> {
@ -663,11 +650,7 @@ impl<'a> PostQuery<'a> {
"legacy pagination cannot be combined with v2 pagination".into(),
));
}
let self_person_id = self
.local_user
.expect("part of the above if")
.local_user
.person_id;
let self_person_id = self.local_user.expect("part of the above if").person_id;
let largest_subscribed = {
let conn = &mut get_conn(pool).await?;
community_follower
@ -704,7 +687,7 @@ impl<'a> PostQuery<'a> {
if (v.len() as i64) < limit {
Ok(Some(self.clone()))
} else {
let item = if self.page_back {
let item = if self.page_back.unwrap_or_default() {
// for backward pagination, get first element instead
v.into_iter().next()
} else {
@ -807,7 +790,7 @@ mod tests {
fn default_post_query(&self) -> PostQuery<'_> {
PostQuery {
sort: Some(SortType::New),
local_user: Some(&self.local_user_view),
local_user: Some(&self.local_user_view.local_user),
..Default::default()
}
}
@ -1143,7 +1126,7 @@ mod tests {
// Read the liked only
let read_liked_post_listing = PostQuery {
community_id: Some(data.inserted_community.id),
liked_only: true,
liked_only: Some(true),
..data.default_post_query()
}
.list(&data.site, pool)
@ -1154,7 +1137,7 @@ mod tests {
let read_disliked_post_listing = PostQuery {
community_id: Some(data.inserted_community.id),
disliked_only: true,
disliked_only: Some(true),
..data.default_post_query()
}
.list(&data.site, pool)
@ -1324,8 +1307,8 @@ mod tests {
// Deleted post is only shown to creator
for (local_user, expect_contains_deleted) in [
(None, false),
(Some(&data.blocked_local_user_view), false),
(Some(&data.local_user_view), true),
(Some(&data.blocked_local_user_view.local_user), false),
(Some(&data.local_user_view.local_user), true),
] {
let contains_deleted = PostQuery {
local_user,
@ -1480,7 +1463,7 @@ mod tests {
loop {
let post_listings = PostQuery {
page_after: page_before,
page_back: true,
page_back: Some(true),
..options.clone()
}
.list(&data.site, pool)
@ -1538,6 +1521,26 @@ mod tests {
let post_listings_hide_read = data.default_post_query().list(&data.site, pool).await?;
assert_eq!(vec![POST], names(&post_listings_hide_read));
// Test with the show_read override as true
let post_listings_show_read_true = PostQuery {
show_read: Some(true),
..data.default_post_query()
}
.list(&data.site, pool)
.await?;
assert_eq!(
vec![POST_BY_BOT, POST],
names(&post_listings_show_read_true)
);
// Test with the show_read override as false
let post_listings_show_read_false = PostQuery {
show_read: Some(false),
..data.default_post_query()
}
.list(&data.site, pool)
.await?;
assert_eq!(vec![POST], names(&post_listings_show_read_false));
cleanup(data, pool).await
}
@ -1563,8 +1566,8 @@ mod tests {
// Make sure it does come back with the show_hidden option
let post_listings_show_hidden = PostQuery {
sort: Some(SortType::New),
local_user: Some(&data.local_user_view),
show_hidden: true,
local_user: Some(&data.local_user_view.local_user),
show_hidden: Some(true),
..Default::default()
}
.list(&data.site, pool)
@ -1655,6 +1658,7 @@ mod tests {
public_key: inserted_person.public_key.clone(),
last_refreshed_at: inserted_person.last_refreshed_at,
},
image_details: None,
creator_banned_from_community: false,
banned_from_community: false,
creator_is_moderator: false,
@ -1738,7 +1742,7 @@ mod tests {
assert_eq!(0, unauthenticated_query.len());
let authenticated_query = PostQuery {
local_user: Some(&data.local_user_view),
local_user: Some(&data.local_user_view.local_user),
..Default::default()
}
.list(&data.site, pool)

18
crates/db_views/src/private_message_report_view.rs
View File

@ -140,18 +140,10 @@ mod tests {
.await
.unwrap();
let new_person_1 = PersonInsertForm::builder()
.name("timmy_mrv".into())
.public_key("pubkey".to_string())
.instance_id(inserted_instance.id)
.build();
let new_person_1 = PersonInsertForm::test_form(inserted_instance.id, "timmy_mrv");
let inserted_timmy = Person::create(pool, &new_person_1).await.unwrap();
let new_person_2 = PersonInsertForm::builder()
.name("jessica_mrv".into())
.public_key("pubkey".to_string())
.instance_id(inserted_instance.id)
.build();
let new_person_2 = PersonInsertForm::test_form(inserted_instance.id, "jessica_mrv");
let inserted_jessica = Person::create(pool, &new_person_2).await.unwrap();
// timmy sends private message to jessica
@ -184,11 +176,7 @@ mod tests {
assert_eq!(pm_report.reason, reports[0].private_message_report.reason);
assert_eq!(pm.content, reports[0].private_message.content);
let new_person_3 = PersonInsertForm::builder()
.name("admin_mrv".into())
.public_key("pubkey".to_string())
.instance_id(inserted_instance.id)
.build();
let new_person_3 = PersonInsertForm::test_form(inserted_instance.id, "admin_mrv");
let inserted_admin = Person::create(pool, &new_person_3).await.unwrap();
// admin resolves the report (after taking appropriate action)

18
crates/db_views/src/private_message_view.rs
View File

@ -209,27 +209,15 @@ mod tests {
.await
.unwrap();
let timmy_form = PersonInsertForm::builder()
.name("timmy_rav".into())
.public_key("pubkey".to_string())
.instance_id(instance.id)
.build();
let timmy_form = PersonInsertForm::test_form(instance.id, "timmy_rav");
let timmy = Person::create(pool, &timmy_form).await.unwrap();
let sara_form = PersonInsertForm::builder()
.name("sara_rav".into())
.public_key("pubkey".to_string())
.instance_id(instance.id)
.build();
let sara_form = PersonInsertForm::test_form(instance.id, "sara_rav");
let sara = Person::create(pool, &sara_form).await.unwrap();
let jess_form = PersonInsertForm::builder()
.name("jess_rav".into())
.public_key("pubkey".to_string())
.instance_id(instance.id)
.build();
let jess_form = PersonInsertForm::test_form(instance.id, "jess_rav");
let jess = Person::create(pool, &jess_form).await.unwrap();

18
crates/db_views/src/registration_application_view.rs
View File

@ -163,11 +163,7 @@ mod tests {
.await
.unwrap();
let timmy_person_form = PersonInsertForm::builder()
.name("timmy_rav".into())
.public_key("pubkey".to_string())
.instance_id(inserted_instance.id)
.build();
let timmy_person_form = PersonInsertForm::test_form(inserted_instance.id, "timmy_rav");
let inserted_timmy_person = Person::create(pool, &timmy_person_form).await.unwrap();
@ -181,11 +177,7 @@ mod tests {
.await
.unwrap();
let sara_person_form = PersonInsertForm::builder()
.name("sara_rav".into())
.public_key("pubkey".to_string())
.instance_id(inserted_instance.id)
.build();
let sara_person_form = PersonInsertForm::test_form(inserted_instance.id, "sara_rav");
let inserted_sara_person = Person::create(pool, &sara_person_form).await.unwrap();
@ -213,11 +205,7 @@ mod tests {
.unwrap()
.unwrap();
let jess_person_form = PersonInsertForm::builder()
.name("jess_rav".into())
.public_key("pubkey".to_string())
.instance_id(inserted_instance.id)
.build();
let jess_person_form = PersonInsertForm::test_form(inserted_instance.id, "jess_rav");
let inserted_jess_person = Person::create(pool, &jess_person_form).await.unwrap();

3
crates/db_views/src/structs.rs
View File

@ -8,7 +8,7 @@ use lemmy_db_schema::{
community::Community,
custom_emoji::CustomEmoji,
custom_emoji_keyword::CustomEmojiKeyword,
images::LocalImage,
images::{ImageDetails, LocalImage},
local_site::LocalSite,
local_site_rate_limit::LocalSiteRateLimit,
local_user::LocalUser,
@ -131,6 +131,7 @@ pub struct PostView {
pub post: Post,
pub creator: Person,
pub community: Community,
pub image_details: Option<ImageDetails>,
pub creator_banned_from_community: bool,
pub banned_from_community: bool,
pub creator_is_moderator: bool,

12
crates/db_views/src/vote_view.rs
View File

@ -112,19 +112,11 @@ mod tests {
.await
.unwrap();
let new_person = PersonInsertForm::builder()
.name("timmy_vv".into())
.public_key("pubkey".to_string())
.instance_id(inserted_instance.id)
.build();
let new_person = PersonInsertForm::test_form(inserted_instance.id, "timmy_vv");
let inserted_timmy = Person::create(pool, &new_person).await.unwrap();
let new_person_2 = PersonInsertForm::builder()
.name("sara_vv".into())
.public_key("pubkey".to_string())
.instance_id(inserted_instance.id)
.build();
let new_person_2 = PersonInsertForm::test_form(inserted_instance.id, "sara_vv");
let inserted_sara = Person::create(pool, &new_person_2).await.unwrap();

16
crates/db_views_actor/src/comment_reply_view.rs
View File

@ -334,19 +334,13 @@ mod tests {
let inserted_instance = Instance::read_or_create(pool, "my_domain.tld".to_string()).await?;
let terry_form = PersonInsertForm::builder()
.name("terrylake".into())
.public_key("pubkey".to_string())
.instance_id(inserted_instance.id)
.build();
let terry_form = PersonInsertForm::test_form(inserted_instance.id, "terrylake");
let inserted_terry = Person::create(pool, &terry_form).await?;
let recipient_form = PersonInsertForm::builder()
.name("terrylakes recipient".into())
.public_key("pubkey".to_string())
.instance_id(inserted_instance.id)
.local(Some(true))
.build();
let recipient_form = PersonInsertForm {
local: Some(true),
..PersonInsertForm::test_form(inserted_instance.id, "terrylakes recipient")
};
let inserted_recipient = Person::create(pool, &recipient_form).await?;
let recipient_id = inserted_recipient.id;

2
crates/db_views_actor/src/community_moderator_view.rs
View File

@ -90,7 +90,7 @@ impl CommunityModeratorView {
.distinct_on(community_moderator::community_id)
.order_by((
community_moderator::community_id,
community_moderator::person_id,
community_moderator::published,
))
.load::<CommunityModeratorView>(conn)
.await

56
crates/db_views_actor/src/community_view.rs
View File

@ -11,6 +11,7 @@ use diesel::{
};
use diesel_async::RunQueryDsl;
use lemmy_db_schema::{
impls::local_user::LocalUserOptionHelper,
newtypes::{CommunityId, PersonId},
schema::{
community,
@ -19,11 +20,18 @@ use lemmy_db_schema::{
community_follower,
community_person_ban,
instance_block,
local_user,
},
source::{community::CommunityFollower, local_user::LocalUser, site::Site},
utils::{fuzzy_search, limit_and_offset, DbConn, DbPool, ListFn, Queries, ReadFn},
CommunityVisibility,
utils::{
fuzzy_search,
limit_and_offset,
visible_communities_only,
DbConn,
DbPool,
ListFn,
Queries,
ReadFn,
},
ListingType,
SortType,
};
@ -97,10 +105,7 @@ fn queries<'a>() -> Queries<
query = query.filter(not_removed_or_deleted);
}
// Hide local only communities from unauthenticated users
if my_person_id.is_none() {
query = query.filter(community::visibility.eq(CommunityVisibility::Public));
}
query = visible_communities_only(my_person_id, query);
query.first(&mut conn).await
};
@ -108,14 +113,14 @@ fn queries<'a>() -> Queries<
let list = move |mut conn: DbConn<'a>, (options, site): (CommunityQuery<'a>, &'a Site)| async move {
use SortType::*;
let my_person_id = options.local_user.map(|l| l.person_id);
// The left join below will return None in this case
let person_id_join = my_person_id.unwrap_or(PersonId(-1));
let person_id_join = options.local_user.person_id().unwrap_or(PersonId(-1));
let mut query = all_joins(community::table.into_boxed(), my_person_id)
.left_join(local_user::table.on(local_user::person_id.eq(person_id_join)))
.select(selection);
let mut query = all_joins(
community::table.into_boxed(),
options.local_user.person_id(),
)
.select(selection);
if let Some(search_term) = options.search_term {
let searcher = fuzzy_search(&search_term);
@ -162,21 +167,14 @@ fn queries<'a>() -> Queries<
// Don't show blocked communities and communities on blocked instances. nsfw communities are
// also hidden (based on profile setting)
if options.local_user.is_some() {
query = query.filter(instance_block::person_id.is_null());
query = query.filter(community_block::person_id.is_null());
query = query.filter(community::nsfw.eq(false).or(local_user::show_nsfw.eq(true)));
} else {
// No person in request, only show nsfw communities if show_nsfw is passed into request or if
// site has content warning.
let has_content_warning = site.content_warning.is_some();
if !options.show_nsfw && !has_content_warning {
query = query.filter(community::nsfw.eq(false));
}
// Hide local only communities from unauthenticated users
query = query.filter(community::visibility.eq(CommunityVisibility::Public));
query = query.filter(instance_block::person_id.is_null());
query = query.filter(community_block::person_id.is_null());
if !(options.local_user.show_nsfw(site) || options.show_nsfw) {
query = query.filter(community::nsfw.eq(false));
}
query = visible_communities_only(options.local_user.person_id(), query);
let (limit, offset) = limit_and_offset(options.page, options.limit)?;
query
.limit(limit)
@ -286,11 +284,7 @@ mod tests {
let person_name = "tegan".to_string();
let new_person = PersonInsertForm::builder()
.name(person_name.clone())
.public_key("pubkey".to_string())
.instance_id(inserted_instance.id)
.build();
let new_person = PersonInsertForm::test_form(inserted_instance.id, &person_name);
let inserted_person = Person::create(pool, &new_person).await.unwrap();

12
crates/db_views_actor/src/person_mention_view.rs
View File

@ -334,19 +334,11 @@ mod tests {
let inserted_instance = Instance::read_or_create(pool, "my_domain.tld".to_string()).await?;
let new_person = PersonInsertForm::builder()
.name("terrylake".into())
.public_key("pubkey".to_string())
.instance_id(inserted_instance.id)
.build();
let new_person = PersonInsertForm::test_form(inserted_instance.id, "terrylake");
let inserted_person = Person::create(pool, &new_person).await?;
let recipient_form = PersonInsertForm::builder()
.name("terrylakes recipient".into())
.public_key("pubkey".to_string())
.instance_id(inserted_instance.id)
.build();
let recipient_form = PersonInsertForm::test_form(inserted_instance.id, "terrylakes recipient");
let inserted_recipient = Person::create(pool, &recipient_form).await?;
let recipient_id = inserted_recipient.id;

22
crates/db_views_actor/src/person_view.rs
View File

@ -191,12 +191,10 @@ mod tests {
async fn init_data(pool: &mut DbPool<'_>) -> LemmyResult<Data> {
let inserted_instance = Instance::read_or_create(pool, "my_domain.tld".to_string()).await?;
let alice_form = PersonInsertForm::builder()
.name("alice".to_string())
.public_key("pubkey".to_string())
.instance_id(inserted_instance.id)
.local(Some(true))
.build();
let alice_form = PersonInsertForm {
local: Some(true),
..PersonInsertForm::test_form(inserted_instance.id, "alice")
};
let alice = Person::create(pool, &alice_form).await?;
let alice_local_user_form = LocalUserInsertForm::builder()
.person_id(alice.id)
@ -204,13 +202,11 @@ mod tests {
.build();
let alice_local_user = LocalUser::create(pool, &alice_local_user_form, vec![]).await?;
let bob_form = PersonInsertForm::builder()
.name("bob".to_string())
.bot_account(Some(true))
.public_key("pubkey".to_string())
.instance_id(inserted_instance.id)
.local(Some(false))
.build();
let bob_form = PersonInsertForm {
bot_account: Some(true),
local: Some(false),
..PersonInsertForm::test_form(inserted_instance.id, "bob")
};
let bob = Person::create(pool, &bob_form).await?;
let bob_local_user_form = LocalUserInsertForm::builder()
.person_id(bob.id)

8
crates/federate/src/worker.rs
View File

@ -136,6 +136,14 @@ impl InstanceWorker {
// lazily fetch latest id only if we have cought up
newest_id = self.get_latest_ids().await?.1;
if next_id_to_send > newest_id {
if next_id_to_send > ActivityId(newest_id.0 + 1) {
tracing::error!(
"{}: next send id {} is higher than latest id {}+1 in database (did the db get cleared?)",
self.instance.domain,
next_id_to_send.0,
newest_id.0
);
}
// no more work to be done, wait before rechecking
tokio::select! {
() = sleep(*WORK_FINISHED_RECHECK_DELAY) => {},

2
crates/routes/src/feeds.rs
View File

@ -355,7 +355,7 @@ async fn get_feed_front(
let posts = PostQuery {
listing_type: (Some(ListingType::Subscribed)),
local_user: (Some(&local_user)),
local_user: (Some(&local_user.local_user)),
sort: (Some(*sort_type)),
limit: (Some(*limit)),
page: (Some(*page)),

8
crates/routes/src/images.rs
View File

@ -103,7 +103,13 @@ async fn upload(
pictrs_alias: image.file.to_string(),
pictrs_delete_token: image.delete_token.to_string(),
};
LocalImage::create(&mut context.pool(), &form).await?;
let protocol_and_hostname = context.settings().get_protocol_and_hostname();
let thumbnail_url = image.thumbnail_url(&protocol_and_hostname)?;
// Also store the details for the image
let details_form = image.details.build_image_details_form(&thumbnail_url);
LocalImage::create(&mut context.pool(), &form, &details_form).await?;
}
}

137
crates/utils/src/utils/validation.rs
View File

@ -11,8 +11,10 @@ static VALID_MATRIX_ID_REGEX: Lazy<Regex> = Lazy::new(|| {
});
// taken from https://en.wikipedia.org/wiki/UTM_parameters
static CLEAN_URL_PARAMS_REGEX: Lazy<Regex> = Lazy::new(|| {
Regex::new(r"^utm_source|utm_medium|utm_campaign|utm_term|utm_content|gclid|gclsrc|dclid|fbclid$")
.expect("compile regex")
Regex::new(
r"^(utm_source|utm_medium|utm_campaign|utm_term|utm_content|gclid|gclsrc|dclid|fbclid)=",
)
.expect("compile regex")
});
const ALLOWED_POST_URL_SCHEMES: [&str; 3] = ["http", "https", "magnet"];
@ -158,14 +160,12 @@ pub fn is_valid_post_title(title: &str) -> LemmyResult<()> {
}
/// This could be post bodies, comments, or any description field
pub fn is_valid_body_field(body: &Option<String>, post: bool) -> LemmyResult<()> {
if let Some(body) = body {
if post {
max_length_check(body, POST_BODY_MAX_LENGTH, LemmyErrorType::InvalidBodyField)?;
} else {
max_length_check(body, BODY_MAX_LENGTH, LemmyErrorType::InvalidBodyField)?;
};
}
pub fn is_valid_body_field(body: &str, post: bool) -> LemmyResult<()> {
if post {
max_length_check(body, POST_BODY_MAX_LENGTH, LemmyErrorType::InvalidBodyField)?;
} else {
max_length_check(body, BODY_MAX_LENGTH, LemmyErrorType::InvalidBodyField)?;
};
Ok(())
}
@ -173,16 +173,14 @@ pub fn is_valid_bio_field(bio: &str) -> LemmyResult<()> {
max_length_check(bio, BIO_MAX_LENGTH, LemmyErrorType::BioLengthOverflow)
}
pub fn is_valid_alt_text_field(alt_text: &Option<String>) -> LemmyResult<()> {
if let Some(alt_text) = alt_text {
max_length_check(
alt_text,
ALT_TEXT_MAX_LENGTH,
LemmyErrorType::AltTextLengthOverflow,
)
} else {
Ok(())
}
pub fn is_valid_alt_text_field(alt_text: &str) -> LemmyResult<()> {
max_length_check(
alt_text,
ALT_TEXT_MAX_LENGTH,
LemmyErrorType::AltTextLengthOverflow,
)?;
Ok(())
}
/// Checks the site name length, the limit as defined in the DB.
@ -260,12 +258,11 @@ pub fn build_and_check_regex(regex_str_opt: &Option<&str>) -> LemmyResult<Option
pub fn clean_url_params(url: &Url) -> Url {
let mut url_out = url.clone();
if url.query().is_some() {
let new_query = url
.query_pairs()
.filter(|q| !CLEAN_URL_PARAMS_REGEX.is_match(&q.0))
.map(|q| format!("{}={}", q.0, q.1))
.join("&");
if let Some(query) = url.query() {
let new_query = query
.split_inclusive('&')
.filter(|q| !CLEAN_URL_PARAMS_REGEX.is_match(q))
.collect::<String>();
url_out.set_query(Some(&new_query));
}
url_out
@ -287,23 +284,17 @@ pub fn check_site_visibility_valid(
}
}
pub fn check_url_scheme(url: &Option<Url>) -> LemmyResult<()> {
if let Some(url) = url {
if !ALLOWED_POST_URL_SCHEMES.contains(&url.scheme()) {
Err(LemmyErrorType::InvalidUrlScheme.into())
} else {
Ok(())
}
} else {
Ok(())
pub fn check_url_scheme(url: &Url) -> LemmyResult<()> {
if !ALLOWED_POST_URL_SCHEMES.contains(&url.scheme()) {
Err(LemmyErrorType::InvalidUrlScheme)?
}
Ok(())
}
pub fn is_url_blocked(url: &Option<Url>, blocklist: &RegexSet) -> LemmyResult<()> {
if let Some(url) = url {
if blocklist.is_match(url.as_str()) {
Err(LemmyErrorType::BlockedUrl)?
}
pub fn is_url_blocked(url: &Url, blocklist: &RegexSet) -> LemmyResult<()> {
if blocklist.is_match(url.as_str()) {
Err(LemmyErrorType::BlockedUrl)?
}
Ok(())
@ -350,12 +341,11 @@ pub fn build_url_str_without_scheme(url_str: &str) -> LemmyResult<String> {
}
#[cfg(test)]
#[allow(clippy::unwrap_used)]
#[allow(clippy::indexing_slicing)]
mod tests {
use crate::{
error::LemmyErrorType,
error::{LemmyErrorType, LemmyResult},
utils::validation::{
build_and_check_regex,
check_site_visibility_valid,
@ -379,15 +369,17 @@ mod tests {
use url::Url;
#[test]
fn test_clean_url_params() {
let url = Url::parse("https://example.com/path/123?utm_content=buffercf3b2&utm_medium=social&username=randomuser&id=123").unwrap();
fn test_clean_url_params() -> LemmyResult<()> {
let url = Url::parse("https://example.com/path/123?utm_content=buffercf3b2&utm_medium=social&user+name=random+user%20&id=123")?;
let cleaned = clean_url_params(&url);
let expected = Url::parse("https://example.com/path/123?username=randomuser&id=123").unwrap();
let expected = Url::parse("https://example.com/path/123?user+name=random+user%20&id=123")?;
assert_eq!(expected.to_string(), cleaned.to_string());
let url = Url::parse("https://example.com/path/123").unwrap();
let url = Url::parse("https://example.com/path/123")?;
let cleaned = clean_url_params(&url);
assert_eq!(url.to_string(), cleaned.to_string());
Ok(())
}
#[test]
@ -465,7 +457,7 @@ mod tests {
}
#[test]
fn test_valid_site_name() {
fn test_valid_site_name() -> LemmyResult<()> {
let valid_names = [
(0..SITE_NAME_MAX_LENGTH).map(|_| 'A').collect::<String>(),
String::from("A"),
@ -496,12 +488,13 @@ mod tests {
assert!(result.is_err());
assert!(
result.unwrap_err().error_type.eq(&expected_err.clone()),
result.is_err_and(|e| e.error_type.eq(&expected_err.clone())),
"Testing {}, expected error {}",
invalid_name,
expected_err
);
});
Ok(())
}
#[test]
@ -513,10 +506,7 @@ mod tests {
assert!(
invalid_result.is_err()
&& invalid_result
.unwrap_err()
.error_type
.eq(&LemmyErrorType::BioLengthOverflow)
&& invalid_result.is_err_and(|e| e.error_type.eq(&LemmyErrorType::BioLengthOverflow))
);
}
@ -537,10 +527,9 @@ mod tests {
assert!(
invalid_result.is_err()
&& invalid_result
.unwrap_err()
&& invalid_result.is_err_and(|e| e
.error_type
.eq(&LemmyErrorType::SiteDescriptionLengthOverflow)
.eq(&LemmyErrorType::SiteDescriptionLengthOverflow))
);
}
@ -570,7 +559,7 @@ mod tests {
assert!(result.is_err());
assert!(
result.unwrap_err().error_type.eq(&expected_err.clone()),
result.is_err_and(|e| e.error_type.eq(&expected_err.clone())),
"Testing regex {:?}, expected error {}",
regex_str,
expected_err
@ -591,38 +580,38 @@ mod tests {
}
#[test]
fn test_check_url_scheme() {
assert!(check_url_scheme(&None).is_ok());
assert!(check_url_scheme(&Some(Url::parse("http://example.com").unwrap())).is_ok());
assert!(check_url_scheme(&Some(Url::parse("https://example.com").unwrap())).is_ok());
assert!(check_url_scheme(&Some(Url::parse("https://example.com").unwrap())).is_ok());
assert!(check_url_scheme(&Some(Url::parse("ftp://example.com").unwrap())).is_err());
assert!(check_url_scheme(&Some(Url::parse("javascript:void").unwrap())).is_err());
fn test_check_url_scheme() -> LemmyResult<()> {
assert!(check_url_scheme(&Url::parse("http://example.com")?).is_ok());
assert!(check_url_scheme(&Url::parse("https://example.com")?).is_ok());
assert!(check_url_scheme(&Url::parse("https://example.com")?).is_ok());
assert!(check_url_scheme(&Url::parse("ftp://example.com")?).is_err());
assert!(check_url_scheme(&Url::parse("javascript:void")?).is_err());
let magnet_link="magnet:?xt=urn:btih:4b390af3891e323778959d5abfff4b726510f14c&dn=Ravel%20Complete%20Piano%20Sheet%20Music%20-%20Public%20Domain&tr=udp%3A%2F%2Fopen.tracker.cl%3A1337%2Fannounce";
assert!(check_url_scheme(&Some(Url::parse(magnet_link).unwrap())).is_ok());
assert!(check_url_scheme(&Url::parse(magnet_link)?).is_ok());
Ok(())
}
#[test]
fn test_url_block() {
fn test_url_block() -> LemmyResult<()> {
let set = regex::RegexSet::new(vec![
r"(https://)?example\.org/page/to/article",
r"(https://)?example\.net/?",
r"(https://)?example\.com/?",
])
.unwrap();
])?;
assert!(is_url_blocked(&Some(Url::parse("https://example.blog").unwrap()), &set).is_ok());
assert!(is_url_blocked(&Url::parse("https://example.blog")?, &set).is_ok());
assert!(is_url_blocked(&Some(Url::parse("https://example.org").unwrap()), &set).is_ok());
assert!(is_url_blocked(&Url::parse("https://example.org")?, &set).is_ok());
assert!(is_url_blocked(&None, &set).is_ok());
assert!(is_url_blocked(&Url::parse("https://example.com")?, &set).is_err());
assert!(is_url_blocked(&Some(Url::parse("https://example.com").unwrap()), &set).is_err());
Ok(())
}
#[test]
fn test_url_parsed() {
fn test_url_parsed() -> LemmyResult<()> {
// Make sure the scheme is removed, and uniques also
assert_eq!(
&check_urls_are_valid(&vec![
@ -630,8 +619,7 @@ mod tests {
"http://example.com".to_string(),
"https://example.com".to_string(),
"https://example.com/test?q=test2&q2=test3#test4".to_string(),
])
.unwrap(),
])?,
&vec![
"example.com".to_string(),
"example.com/test?q=test2&q2=test3#test4".to_string()
@ -639,5 +627,6 @@ mod tests {
);
assert!(check_urls_are_valid(&vec!["https://example .com".to_string()]).is_err());
Ok(())
}
}

2
crates/utils/translations

@ -1 +1 @@
Subproject commit dca5d8ca0309d7cb07eaca821140e4525e75ee57
Subproject commit ee2cffac809ad466644f061ad79ac577b6c2e4fd

2
docker/Dockerfile
View File

@ -1,4 +1,4 @@
# syntax=docker/dockerfile:1.7
# syntax=docker/dockerfile:1.8
ARG RUST_VERSION=1.78
ARG CARGO_BUILD_FEATURES=default
ARG RUST_RELEASE_MODE=debug

6
docker/docker-compose.yml
View File

@ -23,7 +23,7 @@ services:
lemmy:
# use "image" to pull down an already compiled lemmy. make sure to comment out "build".
# image: dessalines/lemmy:0.19.3
# image: dessalines/lemmy:0.19.5
# platform: linux/x86_64 # no arm64 support. uncomment platform if using m1.
# use "build" to build your local lemmy server image for development. make sure to comment out "image".
# run: docker compose up --build
@ -53,7 +53,7 @@ services:
lemmy-ui:
# use "image" to pull down an already compiled lemmy-ui. make sure to comment out "build".
image: dessalines/lemmy-ui:0.19.4-rc.3
image: dessalines/lemmy-ui:0.19.5
# platform: linux/x86_64 # no arm64 support. uncomment platform if using m1.
# use "build" to build your local lemmy ui image for development. make sure to comment out "image".
# run: docker compose up --build
@ -75,7 +75,7 @@ services:
init: true
pictrs:
image: asonix/pictrs:0.5.14
image: asonix/pictrs:0.5.16
# this needs to match the pictrs url in lemmy.hjson
hostname: pictrs
# we can set options to pictrs like this, here we set max. image size and forced format for conversion

4
docker/federation/docker-compose.yml
View File

@ -2,7 +2,7 @@ version: "3.7"
x-ui-default: &ui-default
init: true
image: dessalines/lemmy-ui:0.19.3
image: dessalines/lemmy-ui:0.19.5
# assuming lemmy-ui is cloned besides lemmy directory
# build:
# context: ../../../lemmy-ui
@ -49,7 +49,7 @@ services:
pictrs:
restart: always
image: asonix/pictrs:0.5.14
image: asonix/pictrs:0.5.16
user: 991:991
volumes:
- ./volumes/pictrs_alpha:/mnt:Z

7
migrations/2024-05-05-162540_add_image_detail_table/down.sql 100644
View File

@ -0,0 +1,7 @@
ALTER TABLE remote_image
ADD UNIQUE (link),
DROP CONSTRAINT remote_image_pkey,
ADD COLUMN id serial PRIMARY KEY;
DROP TABLE image_details;

15
migrations/2024-05-05-162540_add_image_detail_table/up.sql 100644
View File

@ -0,0 +1,15 @@
-- Drop the id column from the remote_image table, just use link
ALTER TABLE remote_image
DROP COLUMN id,
ADD PRIMARY KEY (link),
DROP CONSTRAINT remote_image_link_key;
-- No good way to do references here unfortunately, unless we combine the images tables
-- The link should be the URL, not the pictrs_alias, to allow joining from post.thumbnail_url
CREATE TABLE image_details (
link text PRIMARY KEY,
width integer NOT NULL,
height integer NOT NULL,
content_type text NOT NULL
);

3
migrations/2024-06-17-160323_fix_post_aggregates_featured_local/down.sql 100644
View File

@ -0,0 +1,3 @@
SELECT
;

11
migrations/2024-06-17-160323_fix_post_aggregates_featured_local/up.sql 100644
View File

@ -0,0 +1,11 @@
-- Fix rows that were not updated because of the old incorrect trigger
UPDATE
post_aggregates
SET
featured_local = post.featured_local
FROM
post
WHERE
post.id = post_aggregates.post_id
AND post.featured_local != post_aggregates.featured_local;

2
scripts/postgres_15_to_16_upgrade.sh
View File

@ -24,7 +24,7 @@ echo "Removing the old postgres folder"
sudo rm -rf volumes/postgres
echo "Updating docker compose to use postgres version 16."
sudo sed -i "s/image: .*postgres:.*/image: docker.io/postgres:16-alpine/" ./docker-compose.yml
sudo sed -i "s/image: .*postgres:.*/image: docker.io\/postgres:16-alpine/" ./docker-compose.yml
echo "Starting up new postgres..."
sudo docker compose up -d postgres

10
scripts/release.sh
View File

@ -10,7 +10,7 @@ third_semver=$(echo $new_tag | cut -d "." -f 3)
CWD="$(cd -P -- "$(dirname -- "${BASH_SOURCE[0]}")" && pwd -P)"
cd $CWD/../
# The ansible and docker installs should only update for non release-candidates
# The docker installs should only update for non release-candidates
# IE, when the third semver is a number, not '2-rc'
if [ ! -z "${third_semver##*[!0-9]*}" ]; then
pushd docker
@ -20,14 +20,6 @@ if [ ! -z "${third_semver##*[!0-9]*}" ]; then
git add docker-compose.yml
git add federation/docker-compose.yml
popd
# Setting the version for Ansible
pushd ../lemmy-ansible
echo $new_tag > "VERSION"
git add "VERSION"
git commit -m"Updating VERSION"
git push
popd
fi
# Update crate versions

3
scripts/upgrade_deps.sh
View File

@ -5,6 +5,9 @@ pushd ../
# Check unused deps
cargo udeps --all-targets
# Update deps first
cargo update
# Upgrade deps
cargo upgrade

20
src/code_migrations.rs
View File

@ -455,15 +455,17 @@ async fn initialize_local_site_2022_10_10(
)?;
// Register the user if there's a site setup
let person_form = PersonInsertForm::builder()
.name(setup.admin_username.clone())
.instance_id(instance.id)
.actor_id(Some(person_actor_id.clone()))
.private_key(Some(person_keypair.private_key))
.public_key(person_keypair.public_key)
.inbox_url(Some(generate_inbox_url(&person_actor_id)?))
.shared_inbox_url(Some(generate_shared_inbox_url(settings)?))
.build();
let person_form = PersonInsertForm {
actor_id: Some(person_actor_id.clone()),
inbox_url: Some(generate_inbox_url(&person_actor_id)?),
shared_inbox_url: Some(generate_shared_inbox_url(settings)?),
private_key: Some(person_keypair.private_key),
..PersonInsertForm::new(
setup.admin_username.clone(),
person_keypair.public_key,
instance.id,
)
};
let person_inserted = Person::create(pool, &person_form).await?;
let local_user_form = LocalUserInsertForm::builder()

89
src/scheduled_tasks.rs
View File

@ -484,9 +484,6 @@ async fn update_instance_software(
/// This builds an instance update form, for a given domain.
/// If the instance sends a response, but doesn't have a well-known or nodeinfo,
/// Then return a default form with only the updated field.
///
/// TODO This function is a bit of a nightmare with its embedded matches, but the only other way
/// would be to extract the fetches into functions which return the default_form on errors.
async fn build_update_instance_form(
domain: &str,
client: &ClientWithMiddleware,
@ -504,55 +501,51 @@ async fn build_update_instance_form(
// First, fetch their /.well-known/nodeinfo, then extract the correct nodeinfo link from it
let well_known_url = format!("https://{}/.well-known/nodeinfo", domain);
match client.get(&well_known_url).send().await {
Ok(res) if res.status().is_client_error() => {
// Instance doesn't have well-known but sent a response, consider it alive
Some(instance_form)
}
Ok(res) => match res.json::<NodeInfoWellKnown>().await {
Ok(well_known) => {
// Find the first link where the rel contains the allowed rels above
match well_known.links.into_iter().find(|links| {
links
.rel
.as_str()
.starts_with("http://nodeinfo.diaspora.software/ns/schema/2.")
}) {
Some(well_known_link) => {
let node_info_url = well_known_link.href;
let Ok(res) = client.get(&well_known_url).send().await else {
// This is the only kind of error that means the instance is dead
return None;
};
// Fetch the node_info from the well known href
match client.get(node_info_url).send().await {
Ok(node_info_res) => match node_info_res.json::<NodeInfo>().await {
Ok(node_info) => {
// Instance sent valid nodeinfo, write it to db
// Set the instance form fields.
if let Some(software) = node_info.software.as_ref() {
instance_form.software.clone_from(&software.name);
instance_form.version.clone_from(&software.version);
}
Some(instance_form)
}
Err(_) => Some(instance_form),
},
Err(_) => Some(instance_form),
}
}
// If none is found, use the default form above
None => Some(instance_form),
}
}
Err(_) => {
// No valid nodeinfo but valid HTTP response, consider instance alive
Some(instance_form)
}
},
Err(_) => {
// dead instance, do nothing
None
// In this block, returning `None` is ignored, and only means not writing nodeinfo to db
async {
if res.status().is_client_error() {
return None;
}
let node_info_url = res
.json::<NodeInfoWellKnown>()
.await
.ok()?
.links
.into_iter()
.find(|links| {
links
.rel
.as_str()
.starts_with("http://nodeinfo.diaspora.software/ns/schema/2.")
})?
.href;
let software = client
.get(node_info_url)
.send()
.await
.ok()?
.json::<NodeInfo>()
.await
.ok()?
.software?;
instance_form.software = software.name;
instance_form.version = software.version;
Some(())
}
.await;
Some(instance_form)
}
#[cfg(test)]
#[allow(clippy::indexing_slicing)]
mod tests {

6
src/session_middleware.rs
View File

@ -142,11 +142,7 @@ mod tests {
.await
.unwrap();
let new_person = PersonInsertForm::builder()
.name("Gerry9812".into())
.public_key("pubkey".to_string())
.instance_id(inserted_instance.id)
.build();
let new_person = PersonInsertForm::test_form(inserted_instance.id, "Gerry9812");
let inserted_person = Person::create(pool, &new_person).await.unwrap();