From 55b25a777bd937ff90b257abe7b200070bcb62ef Mon Sep 17 00:00:00 2001 From: Essem Date: Sun, 14 Jan 2024 01:22:06 -0600 Subject: [PATCH] Initial bubble timeline support --- .../api/v1/timelines/public_controller.rb | 3 +- .../flavours/glitch/actions/streaming.js | 9 ++ .../flavours/glitch/actions/timelines.js | 2 + .../glitch/features/firehose/index.jsx | 99 +++++++++++++------ .../flavours/glitch/features/ui/index.jsx | 1 + .../flavours/glitch/locales/en.json | 3 + app/models/bubble_domain.rb | 43 ++++++++ app/models/public_feed.rb | 14 ++- app/models/status.rb | 2 + app/models/tag_feed.rb | 1 + config/routes.rb | 1 + .../20240114042123_create_bubble_domains.rb | 13 +++ db/schema.rb | 9 +- lib/mastodon/cli/bubble_domains.rb | 87 ++++++++++++++++ lib/mastodon/cli/main.rb | 4 + streaming/index.js | 18 ++++ 16 files changed, 275 insertions(+), 34 deletions(-) create mode 100644 app/models/bubble_domain.rb create mode 100644 db/migrate/20240114042123_create_bubble_domains.rb create mode 100644 lib/mastodon/cli/bubble_domains.rb diff --git a/app/controllers/api/v1/timelines/public_controller.rb b/app/controllers/api/v1/timelines/public_controller.rb index 5bc8de8334..9556963b74 100644 --- a/app/controllers/api/v1/timelines/public_controller.rb +++ b/app/controllers/api/v1/timelines/public_controller.rb @@ -3,7 +3,7 @@ class Api::V1::Timelines::PublicController < Api::V1::Timelines::BaseController before_action :require_user!, only: [:show], if: :require_auth? - PERMITTED_PARAMS = %i(local remote limit only_media allow_local_only).freeze + PERMITTED_PARAMS = %i(local remote bubble limit only_media allow_local_only).freeze def show cache_if_unauthenticated! @@ -38,6 +38,7 @@ class Api::V1::Timelines::PublicController < Api::V1::Timelines::BaseController PublicFeed.new( current_account, local: truthy_param?(:local), + bubble: truthy_param?(:bubble), remote: truthy_param?(:remote), only_media: truthy_param?(:only_media), allow_local_only: truthy_param?(:allow_local_only), diff --git a/app/javascript/flavours/glitch/actions/streaming.js b/app/javascript/flavours/glitch/actions/streaming.js index d8341a5c16..c46e9ea5ba 100644 --- a/app/javascript/flavours/glitch/actions/streaming.js +++ b/app/javascript/flavours/glitch/actions/streaming.js @@ -22,6 +22,7 @@ import { fillPublicTimelineGaps, fillCommunityTimelineGaps, fillListTimelineGaps, + fillBubbleTimelineGaps, } from './timelines'; /** @@ -150,6 +151,14 @@ export const connectUserStream = () => export const connectCommunityStream = ({ onlyMedia } = {}) => connectTimelineStream(`community${onlyMedia ? ':media' : ''}`, `public:local${onlyMedia ? ':media' : ''}`, {}, { fillGaps: () => (fillCommunityTimelineGaps({ onlyMedia })) }); +/** + * @param {Object} options + * @param {boolean} [options.onlyMedia] + * @returns {function(): void} + */ +export const connectBubbleStream = ({ onlyMedia } = {}) => + connectTimelineStream(`bubble${onlyMedia ? ':media' : ''}`, `public:bubble${onlyMedia ? ':media' : ''}`, {}, { fillGaps: () => (fillBubbleTimelineGaps({ onlyMedia })) }); + /** * @param {Object} options * @param {boolean} [options.onlyMedia] diff --git a/app/javascript/flavours/glitch/actions/timelines.js b/app/javascript/flavours/glitch/actions/timelines.js index fa69bca985..2d7e0449f1 100644 --- a/app/javascript/flavours/glitch/actions/timelines.js +++ b/app/javascript/flavours/glitch/actions/timelines.js @@ -156,6 +156,7 @@ export function fillTimelineGaps(timelineId, path, params = {}, done = noOp) { export const expandHomeTimeline = ({ maxId } = {}, done = noOp) => expandTimeline('home', '/api/v1/timelines/home', { max_id: maxId }, done); export const expandPublicTimeline = ({ maxId, onlyMedia, onlyRemote, allowLocalOnly } = {}, done = noOp) => expandTimeline(`public${onlyRemote ? ':remote' : (allowLocalOnly ? ':allow_local_only' : '')}${onlyMedia ? ':media' : ''}`, '/api/v1/timelines/public', { remote: !!onlyRemote, allow_local_only: !!allowLocalOnly, max_id: maxId, only_media: !!onlyMedia }, done); export const expandCommunityTimeline = ({ maxId, onlyMedia } = {}, done = noOp) => expandTimeline(`community${onlyMedia ? ':media' : ''}`, '/api/v1/timelines/public', { local: true, max_id: maxId, only_media: !!onlyMedia }, done); +export const expandBubbleTimeline = ({ maxId, onlyMedia } = {}, done = noOp) => expandTimeline(`bubble${onlyMedia ? ':media' : ''}`, '/api/v1/timelines/public', { bubble: true, max_id: maxId, only_media: !!onlyMedia }, done); export const expandDirectTimeline = ({ maxId } = {}, done = noOp) => expandTimeline('direct', '/api/v1/timelines/direct', { max_id: maxId }, done); export const expandAccountTimeline = (accountId, { maxId, withReplies, tagged } = {}) => expandTimeline(`account:${accountId}${withReplies ? ':with_replies' : ''}${tagged ? `:${tagged}` : ''}`, `/api/v1/accounts/${accountId}/statuses`, { exclude_replies: !withReplies, exclude_reblogs: withReplies, tagged, max_id: maxId }); export const expandAccountFeaturedTimeline = (accountId, { tagged } = {}) => expandTimeline(`account:${accountId}:pinned`, `/api/v1/accounts/${accountId}/statuses`, { pinned: true, tagged }); @@ -174,6 +175,7 @@ export const expandHashtagTimeline = (hashtag, { maxId, tags, local } = export const fillHomeTimelineGaps = (done = noOp) => fillTimelineGaps('home', '/api/v1/timelines/home', {}, done); export const fillPublicTimelineGaps = ({ onlyMedia, onlyRemote, allowLocalOnly } = {}, done = noOp) => fillTimelineGaps(`public${onlyRemote ? ':remote' : (allowLocalOnly ? ':allow_local_only' : '')}${onlyMedia ? ':media' : ''}`, '/api/v1/timelines/public', { remote: !!onlyRemote, only_media: !!onlyMedia, allow_local_only: !!allowLocalOnly }, done); export const fillCommunityTimelineGaps = ({ onlyMedia } = {}, done = noOp) => fillTimelineGaps(`community${onlyMedia ? ':media' : ''}`, '/api/v1/timelines/public', { local: true, only_media: !!onlyMedia }, done); +export const fillBubbleTimelineGaps = ({ onlyMedia } = {}, done = noOp) => fillTimelineGaps(`bubble${onlyMedia ? ':media' : ''}`, '/api/v1/timelines/public', { bubble: true, only_media: !!onlyMedia }, done); export const fillListTimelineGaps = (id, done = noOp) => fillTimelineGaps(`list:${id}`, `/api/v1/timelines/list/${id}`, {}, done); export function expandTimelineRequest(timeline, isLoadingMore) { diff --git a/app/javascript/flavours/glitch/features/firehose/index.jsx b/app/javascript/flavours/glitch/features/firehose/index.jsx index c01c767b86..9e02748f3e 100644 --- a/app/javascript/flavours/glitch/features/firehose/index.jsx +++ b/app/javascript/flavours/glitch/features/firehose/index.jsx @@ -8,8 +8,8 @@ import { NavLink } from 'react-router-dom'; import { addColumn } from 'flavours/glitch/actions/columns'; import { changeSetting } from 'flavours/glitch/actions/settings'; -import { connectPublicStream, connectCommunityStream } from 'flavours/glitch/actions/streaming'; -import { expandPublicTimeline, expandCommunityTimeline } from 'flavours/glitch/actions/timelines'; +import { connectPublicStream, connectCommunityStream, connectBubbleStream } from 'flavours/glitch/actions/streaming'; +import { expandPublicTimeline, expandCommunityTimeline, expandBubbleTimeline } from 'flavours/glitch/actions/timelines'; import { DismissableBanner } from 'flavours/glitch/components/dismissable_banner'; import SettingText from 'flavours/glitch/components/setting_text'; import initialState, { domain } from 'flavours/glitch/initial_state'; @@ -91,6 +91,9 @@ const Firehose = ({ feedType, multiColumn }) => { case 'public': dispatch(addColumn('PUBLIC', { other: { onlyMedia, allowLocalOnly }, regex: { body: regex } })); break; + case 'bubble': + dispatch(addColumn('BUBBLE', { other: { onlyMedia }, regex: { body: regex } })); + break; case 'public:remote': dispatch(addColumn('REMOTE', { other: { onlyMedia, onlyRemote: true }, regex: { body: regex } })); break; @@ -105,6 +108,9 @@ const Firehose = ({ feedType, multiColumn }) => { case 'community': dispatch(expandCommunityTimeline({ maxId, onlyMedia })); break; + case 'bubble': + dispatch(expandBubbleTimeline({ maxId, onlyMedia })); + break; case 'public': dispatch(expandPublicTimeline({ maxId, onlyMedia, allowLocalOnly })); break; @@ -128,6 +134,12 @@ const Firehose = ({ feedType, multiColumn }) => { disconnect = dispatch(connectCommunityStream({ onlyMedia })); } break; + case 'bubble': + dispatch(expandBubbleTimeline({ onlyMedia })); + if (signedIn) { + disconnect = dispatch(connectBubbleStream({ onlyMedia })); + } + break; case 'public': dispatch(expandPublicTimeline({ onlyMedia, allowLocalOnly })); if (signedIn) { @@ -145,35 +157,58 @@ const Firehose = ({ feedType, multiColumn }) => { return () => disconnect?.(); }, [dispatch, signedIn, feedType, onlyMedia, allowLocalOnly]); - const prependBanner = feedType === 'community' ? ( - - - - ) : ( - - - - ); + let prependBanner; + let emptyMessage; - const emptyMessage = feedType === 'community' ? ( - - ) : ( - - ); + if (feedType === 'community') { + prependBanner = ( + + + + ); + emptyMessage = ( + + ); + } else if (feedType === 'bubble') { + prependBanner = ( + + + + ); + emptyMessage = ( + + ); + } else { + prependBanner = ( + + + + ); + emptyMessage = ( + + ); + } return ( @@ -193,6 +228,10 @@ const Firehose = ({ feedType, multiColumn }) => { + + + + diff --git a/app/javascript/flavours/glitch/features/ui/index.jsx b/app/javascript/flavours/glitch/features/ui/index.jsx index fa943f579f..c4ced17099 100644 --- a/app/javascript/flavours/glitch/features/ui/index.jsx +++ b/app/javascript/flavours/glitch/features/ui/index.jsx @@ -208,6 +208,7 @@ class SwitchingColumnsArea extends PureComponent { + diff --git a/app/javascript/flavours/glitch/locales/en.json b/app/javascript/flavours/glitch/locales/en.json index 4b2418b5ed..8fce6777e4 100644 --- a/app/javascript/flavours/glitch/locales/en.json +++ b/app/javascript/flavours/glitch/locales/en.json @@ -43,8 +43,11 @@ "confirmations.unfilter.filters": "Matching {count, plural, one {filter} other {filters}}", "content-type.change": "Content type", "direct.group_by_conversations": "Group by conversation", + "dismissable_banner.bubble_timeline": "These are the most recent public posts from people on the social web whose accounts are on other servers selected by {domain}.", + "empty_column.bubble": "The bubble timeline is currently empty, but something might show up here soon!", "endorsed_accounts_editor.endorsed_accounts": "Featured accounts", "favourite_modal.combo": "You can press {combo} to skip this next time", + "firehose.bubble": "Bubble servers", "firehose.column_settings.allow_local_only": "Show local-only posts in \"All\"", "home.column_settings.advanced": "Advanced", "home.column_settings.filter_regex": "Filter out by regular expressions", diff --git a/app/models/bubble_domain.rb b/app/models/bubble_domain.rb new file mode 100644 index 0000000000..ba47e6dd7b --- /dev/null +++ b/app/models/bubble_domain.rb @@ -0,0 +1,43 @@ +# frozen_string_literal: true + +# == Schema Information +# +# Table name: bubble_domains +# +# id :bigint(8) not null, primary key +# domain :string default(""), not null +# created_at :datetime not null +# updated_at :datetime not null +# + +class BubbleDomain < ApplicationRecord + include Paginable + include DomainNormalizable + include DomainMaterializable + + validates :domain, presence: true, uniqueness: true, domain: true + + scope :matches_domain, ->(value) { where(arel_table[:domain].matches("%#{value}%")) } + + def to_log_human_identifier + domain + end + + class << self + def in_bubble?(domain) + !rule_for(domain).nil? + end + + def bubble_domains + select(:domain) + end + + def rule_for(domain) + return if domain.blank? + + uri = Addressable::URI.new.tap { |u| u.host = domain.delete('/') } + + find_by(domain: uri.normalized_host) + end + end +end diff --git a/app/models/public_feed.rb b/app/models/public_feed.rb index 8a7c3a4512..1d835ad1f9 100644 --- a/app/models/public_feed.rb +++ b/app/models/public_feed.rb @@ -6,6 +6,7 @@ class PublicFeed # @option [Boolean] :with_replies # @option [Boolean] :with_reblogs # @option [Boolean] :local + # @option [Boolean] :bubble # @option [Boolean] :remote # @option [Boolean] :only_media # @option [Boolean] :allow_local_only @@ -26,6 +27,7 @@ class PublicFeed scope.merge!(without_replies_scope) unless with_replies? scope.merge!(without_reblogs_scope) unless with_reblogs? scope.merge!(local_only_scope) if local_only? + scope.merge!(bubble_only_scope) if bubble_only? scope.merge!(remote_only_scope) if remote_only? scope.merge!(account_filters_scope) if account? scope.merge!(media_only_scope) if media_only? @@ -51,11 +53,15 @@ class PublicFeed end def local_only? - options[:local] && !options[:remote] + options[:local] && !options[:remote] && !options[:bubble] + end + + def bubble_only? + options[:bubble] && !options[:local] && !options[:remote] end def remote_only? - options[:remote] && !options[:local] + options[:remote] && !options[:local] && !options[:bubble] end def account? @@ -78,6 +84,10 @@ class PublicFeed Status.local end + def bubble_only_scope + Status.bubble + end + def remote_only_scope Status.remote end diff --git a/app/models/status.rb b/app/models/status.rb index 653bea1d35..7796d08815 100644 --- a/app/models/status.rb +++ b/app/models/status.rb @@ -127,6 +127,8 @@ class Status < ApplicationRecord scope :not_local_only, -> { where(local_only: [false, nil]) } + scope :bubble, -> { left_outer_joins(:account).where(accounts: { domain: BubbleDomain.bubble_domains }) } + after_create_commit :trigger_create_webhooks after_update_commit :trigger_update_webhooks diff --git a/app/models/tag_feed.rb b/app/models/tag_feed.rb index fbbdbaae27..1b6c5707f3 100644 --- a/app/models/tag_feed.rb +++ b/app/models/tag_feed.rb @@ -30,6 +30,7 @@ class TagFeed < PublicFeed scope.merge!(tagged_with_all_scope) scope.merge!(tagged_with_none_scope) scope.merge!(local_only_scope) if local_only? + scope.merge!(bubble_only_scope) if bubble_only? scope.merge!(remote_only_scope) if remote_only? scope.merge!(account_filters_scope) if account? scope.merge!(media_only_scope) if media_only? diff --git a/config/routes.rb b/config/routes.rb index d4cd27a491..4c2ee3ddaf 100644 --- a/config/routes.rb +++ b/config/routes.rb @@ -25,6 +25,7 @@ Rails.application.routes.draw do /home /public /public/local + /public/bubble /public/remote /conversations /lists/(*any) diff --git a/db/migrate/20240114042123_create_bubble_domains.rb b/db/migrate/20240114042123_create_bubble_domains.rb new file mode 100644 index 0000000000..83c87956bf --- /dev/null +++ b/db/migrate/20240114042123_create_bubble_domains.rb @@ -0,0 +1,13 @@ +# frozen_string_literal: true + +class CreateBubbleDomains < ActiveRecord::Migration[7.1] + def change + create_table :bubble_domains do |t| + t.string :domain, default: '', null: false + + t.timestamps + end + + add_index :bubble_domains, :domain, name: :index_bubble_domains_on_domain, unique: true + end +end diff --git a/db/schema.rb b/db/schema.rb index da2be71318..a704e50640 100644 --- a/db/schema.rb +++ b/db/schema.rb @@ -10,7 +10,7 @@ # # It's strongly recommended that you check this file into your version control system. -ActiveRecord::Schema[7.1].define(version: 2024_01_09_103012) do +ActiveRecord::Schema[7.1].define(version: 2024_01_14_042123) do # These are extensions that must be enabled in order to support this database enable_extension "plpgsql" @@ -296,6 +296,13 @@ ActiveRecord::Schema[7.1].define(version: 2024_01_09_103012) do t.index ["status_id"], name: "index_bookmarks_on_status_id" end + create_table "bubble_domains", force: :cascade do |t| + t.string "domain", default: "", null: false + t.datetime "created_at", null: false + t.datetime "updated_at", null: false + t.index ["domain"], name: "index_bubble_domains_on_domain", unique: true + end + create_table "bulk_import_rows", force: :cascade do |t| t.bigint "bulk_import_id", null: false t.jsonb "data" diff --git a/lib/mastodon/cli/bubble_domains.rb b/lib/mastodon/cli/bubble_domains.rb new file mode 100644 index 0000000000..63a38bc372 --- /dev/null +++ b/lib/mastodon/cli/bubble_domains.rb @@ -0,0 +1,87 @@ +# frozen_string_literal: true + +require 'concurrent' +require_relative 'base' + +module Mastodon::CLI + class BubbleDomains < Base + desc 'list', 'List domains in the bubble' + def list + BubbleDomain.find_each do |entry| + say(entry.domain.to_s, :white) + end + end + + desc 'add [DOMAIN...]', 'Add domains to the bubble' + def add(*domains) + if domains.empty? + say('No domain(s) given', :red) + exit(1) + end + + domains = domains.map { |domain| TagManager.instance.normalize_domain(domain) } + + skipped = 0 + processed = 0 + + domains.each do |domain| + if BubbleDomain.exists?(domain: domain) + say("#{domain} is already in the bubble.", :yellow) + skipped += 1 + next + end + + bubble_domain = BubbleDomain.new(domain: domain) + bubble_domain.save! + processed += 1 + end + + say("Added #{processed}, skipped #{skipped}", color(processed, 0)) + end + + desc 'remove DOMAIN...', 'Remove domain from the bubble' + def remove(*domains) + if domains.empty? + say('No domain(s) given', :red) + exit(1) + end + + skipped = 0 + processed = 0 + failed = 0 + + domains.each do |domain| + entry = BubbleDomain.find_by(domain: domain) + + if entry.nil? + say("#{domain} is not in the bubble.", :yellow) + skipped += 1 + next + end + + result = entry.destroy + + if result + processed += 1 + else + say("#{domain} could not be removed.", :red) + failed += 1 + end + end + + say("Removed #{processed}, skipped #{skipped}, failed #{failed}", color(processed, failed)) + end + + private + + def color(processed, failed) + if !processed.zero? && failed.zero? + :green + elsif failed.zero? + :yellow + else + :red + end + end + end +end diff --git a/lib/mastodon/cli/main.rb b/lib/mastodon/cli/main.rb index ef40b81f33..7c49a6a228 100644 --- a/lib/mastodon/cli/main.rb +++ b/lib/mastodon/cli/main.rb @@ -3,6 +3,7 @@ require_relative 'base' require_relative 'accounts' +require_relative 'bubble_domains' require_relative 'cache' require_relative 'canonical_email_blocks' require_relative 'domains' @@ -63,6 +64,9 @@ module Mastodon::CLI desc 'canonical_email_blocks SUBCOMMAND ...ARGS', 'Manage canonical e-mail blocks' subcommand 'canonical_email_blocks', CanonicalEmailBlocks + desc 'bubble_domains SUBCOMMAND ...ARGS', 'Manage bubble domains' + subcommand 'bubble_domains', BubbleDomains + desc 'maintenance SUBCOMMAND ...ARGS', 'Various maintenance utilities' subcommand 'maintenance', Maintenance diff --git a/streaming/index.js b/streaming/index.js index 9ed24dc02d..c62ab1e12f 100644 --- a/streaming/index.js +++ b/streaming/index.js @@ -164,6 +164,8 @@ const PUBLIC_CHANNELS = [ 'public:media', 'public:local', 'public:local:media', + 'public:bubble', + 'public:bubble:media', 'public:remote', 'public:remote:media', 'hashtag', @@ -438,6 +440,8 @@ const startServer = async () => { return onlyMedia ? 'public:media' : 'public'; case '/api/v1/streaming/public/local': return onlyMedia ? 'public:local:media' : 'public:local'; + case '/api/v1/streaming/public/bubble': + return onlyMedia ? 'public:bubble:media' : 'public:bubble'; case '/api/v1/streaming/public/remote': return onlyMedia ? 'public:remote:media' : 'public:remote'; case '/api/v1/streaming/hashtag': @@ -1096,6 +1100,13 @@ const startServer = async () => { options: { needsFiltering: true, allowLocalOnly: true }, }); + break; + case 'public:bubble': + resolve({ + channelIds: ['timeline:public:bubble'], + options: { needsFiltering: true, allowLocalOnly: false }, + }); + break; case 'public:remote': resolve({ @@ -1124,6 +1135,13 @@ const startServer = async () => { options: { needsFiltering: true, allowLocalOnly: true }, }); + break; + case 'public:bubble:media': + resolve({ + channelIds: ['timeline:public:bubble:media'], + options: { needsFiltering: true, allowLocalOnly: false }, + }); + break; case 'public:remote:media': resolve({