diff --git a/.env.production.sample b/.env.production.sample
index 7bcce0f7e5..a6f7b9c5a6 100644
--- a/.env.production.sample
+++ b/.env.production.sample
@@ -301,3 +301,17 @@ MAX_POLL_OPTION_CHARS=100
# -----------------------
IP_RETENTION_PERIOD=31556952
SESSION_RETENTION_PERIOD=31556952
+
+# Scope of full-text status searches (previously named SEARCH_SCOPE):
+# - discoverable: search any status with public visibility that was created by a discoverable, non-silenced account
+# - Note that statuses are not automatically re-indexed when the creating account's flags change.
+# - public: search any status with public visibility
+# - public_or_unlisted: search any status with public or unlisted visibility
+# - classic: searches only a user's own statuses, favs, bookmarks, and mentions
+STATUS_SEARCH_SCOPE=discoverable
+
+# Scope of full-text account searches:
+# - discoverable: search any discoverable, non-silenced account's profile by full-text search on bio text and fields
+# - all: search any profile by full-text search on bio text and fields
+# - classic: search only account names, display names, and hashtags, do not use full-text indexes
+ACCOUNT_SEARCH_SCOPE=discoverable
diff --git a/app/chewy/accounts_index.rb b/app/chewy/accounts_index.rb
index e38e14a106..bbff796fd5 100644
--- a/app/chewy/accounts_index.rb
+++ b/app/chewy/accounts_index.rb
@@ -2,6 +2,21 @@
class AccountsIndex < Chewy::Index
settings index: { refresh_interval: '30s' }, analysis: {
+ filter: {
+ english_stop: {
+ type: 'stop',
+ stopwords: '_english_',
+ },
+ english_stemmer: {
+ type: 'stemmer',
+ language: 'english',
+ },
+ english_possessive_stemmer: {
+ type: 'stemmer',
+ language: 'possessive_english',
+ },
+ },
+
analyzer: {
content: {
tokenizer: 'whitespace',
@@ -12,6 +27,18 @@ class AccountsIndex < Chewy::Index
tokenizer: 'edge_ngram',
filter: %w(lowercase asciifolding cjk_width),
},
+
+ text: {
+ tokenizer: 'uax_url_email',
+ filter: %w(
+ english_possessive_stemmer
+ lowercase
+ asciifolding
+ cjk_width
+ english_stop
+ english_stemmer
+ ),
+ },
},
tokenizer: {
@@ -21,6 +48,17 @@ class AccountsIndex < Chewy::Index
max_gram: 15,
},
},
+
+ normalizer: {
+ tag: {
+ type: 'custom',
+ filter: %w(
+ lowercase
+ asciifolding
+ cjk_width
+ ),
+ },
+ },
}
index_scope ::Account.searchable.includes(:account_stat)
@@ -38,6 +76,17 @@ class AccountsIndex < Chewy::Index
field :following_count, type: 'long', value: ->(account) { account.following_count }
field :followers_count, type: 'long', value: ->(account) { account.followers_count }
+ field :created_at, type: 'date'
field :last_status_at, type: 'date', value: ->(account) { account.last_status_at || account.created_at }
+ field :discoverable, type: 'boolean'
+ field :silenced, type: 'boolean', value: ->(account) { account.silenced? }
+ field :domain, type: 'keyword', value: ->(account) { account.domain }
+ field :is, type: 'keyword', value: ->(account) { account.searchable_is }
+ field :emojis, type: 'keyword', value: ->(account) { account.searchable_emojis }
+ field :tags, type: 'keyword', normalizer: 'tag', value: ->(account) { account.searchable_tags }
+
+ field :text, type: 'text', value: ->(account) { account.searchable_text } do
+ field :stemmed, type: 'text', analyzer: 'text'
+ end
end
end
diff --git a/app/chewy/statuses_index.rb b/app/chewy/statuses_index.rb
index 6dd4fb18b0..f94b9e69aa 100644
--- a/app/chewy/statuses_index.rb
+++ b/app/chewy/statuses_index.rb
@@ -18,6 +18,7 @@ class StatusesIndex < Chewy::Index
language: 'possessive_english',
},
},
+
analyzer: {
content: {
tokenizer: 'uax_url_email',
@@ -31,6 +32,17 @@ class StatusesIndex < Chewy::Index
),
},
},
+
+ normalizer: {
+ tag: {
+ type: 'custom',
+ filter: %w(
+ lowercase
+ asciifolding
+ cjk_width
+ ),
+ },
+ },
}
# We do not use delete_if option here because it would call a method that we
@@ -65,6 +77,17 @@ class StatusesIndex < Chewy::Index
root date_detection: false do
field :id, type: 'long'
field :account_id, type: 'long'
+ field :created_at, type: 'date'
+ field :visibility, type: 'keyword'
+ field :discoverable, type: 'boolean', value: ->(status) { status.account.discoverable }
+ field :silenced, type: 'boolean', value: ->(status) { status.account.silenced? }
+ field :domain, type: 'keyword', value: ->(status) { status.account.domain }
+ field :lang, type: 'keyword', value: ->(status) { status.language }
+ field :is, type: 'keyword', value: ->(status) { status.searchable_is }
+ field :has, type: 'keyword', value: ->(status) { status.searchable_has }
+ field :emojis, type: 'keyword', value: ->(status) { status.searchable_emojis }
+ field :tags, type: 'keyword', normalizer: 'tag', value: ->(status) { status.searchable_tags }
+ field :mentions_ids, type: 'long', value: ->(status) { status.searchable_mentions_ids }
field :text, type: 'text', value: ->(status) { status.searchable_text } do
field :stemmed, type: 'text', analyzer: 'content'
diff --git a/app/javascript/mastodon/features/search_reference/index.jsx b/app/javascript/mastodon/features/search_reference/index.jsx
new file mode 100644
index 0000000000..b30a4aef4a
--- /dev/null
+++ b/app/javascript/mastodon/features/search_reference/index.jsx
@@ -0,0 +1,202 @@
+import React from 'react';
+import Column from 'mastodon/components/column';
+import { defineMessages, injectIntl, FormattedMessage } from 'react-intl';
+import PropTypes from 'prop-types';
+import ImmutablePureComponent from 'react-immutable-pure-component';
+import ColumnHeader from 'mastodon/components/column_header';
+import { Helmet } from 'react-helmet';
+import { searchEnabled } from '../../initial_state';
+
+const messages = defineMessages({
+ heading: { id: 'search_reference.heading', defaultMessage: 'Search Reference' },
+});
+
+class SearchReference extends ImmutablePureComponent {
+
+ static propTypes = {
+ intl: PropTypes.object.isRequired,
+ multiColumn: PropTypes.bool,
+ };
+
+ render () {
+ const { intl, multiColumn } = this.props;
+
+ return (
+ <Column>
+ <ColumnHeader
+ title={intl.formatMessage(messages.heading)}
+ icon='question'
+ multiColumn={multiColumn}
+ />
+
+ <div className='search-reference scrollable optionally-scrollable'>
+
+ <p>
+ { searchEnabled
+ ? <FormattedMessage id='search_popout.tips.full_text' defaultMessage='Simple text returns statuses you have written, favourited, boosted, or have been mentioned in, as well as matching usernames, display names, and hashtags.' />
+ : <FormattedMessage id='search_popout.tips.text' defaultMessage='Simple text returns matching display names, usernames and hashtags' />
+ }
+ </p>
+
+ <table>
+ <thead>
+ <tr>
+ <th><FormattedMessage id='search_reference.operator' defaultMessage='Operator' /></th>
+ <th><FormattedMessage id='search_reference.description' defaultMessage='Description' /></th>
+ </tr>
+ </thead>
+
+ <tbody>
+ <tr>
+ <th colSpan='2'><FormattedMessage id='search_reference.search_operators.sections.lookups' defaultMessage='Lookups' /></th>
+ </tr>
+ <tr>
+ <td><kbd>#example</kbd></td>
+ <td><FormattedMessage id='search_popout.tips.hashtag' defaultMessage='hashtag' /></td>
+ </tr>
+ <tr>
+ <td><kbd>@username@domain</kbd></td>
+ <td><FormattedMessage id='search_popout.tips.user' defaultMessage='user' /></td>
+ </tr>
+ <tr>
+ <td><kbd>URL</kbd></td>
+ <td><FormattedMessage id='search_popout.tips.user' defaultMessage='user' /></td>
+ </tr>
+ <tr>
+ <td><kbd>URL</kbd></td>
+ <td><FormattedMessage id='search_popout.tips.status' defaultMessage='status' /></td>
+ </tr>
+ </tbody>
+
+ { searchEnabled &&
+ <tbody>
+ <tr>
+ <th colSpan='2'><FormattedMessage id='search_reference.search_operators.sections.advanced_syntax' defaultMessage='Advanced syntax' /></th>
+ </tr>
+ <tr>
+ <td><kbd>+term</kbd></td>
+ <td><FormattedMessage id='search_reference.search_operators.include' defaultMessage='require term in results' /></td>
+ </tr>
+ <tr>
+ <td><kbd>-term</kbd></td>
+ <td><FormattedMessage id='search_reference.search_operators.exclude' defaultMessage='exclude results containing term' /></td>
+ </tr>
+ <tr>
+ <td><kbd>"John Mastodon"</kbd></td>
+ <td><FormattedMessage id='search_reference.search_operators.phrase' defaultMessage='search for an entire phrase instead of a single word' /></td>
+ </tr>
+ <tr>
+ <td><kbd>cat has:media</kbd></td>
+ <td><FormattedMessage id='search_reference.search_operators.combinations' defaultMessage='operators can be combined with search terms or each other' /></td>
+ </tr>
+ <tr>
+ <td><kbd>-is:bot</kbd></td>
+ <td><FormattedMessage id='search_reference.search_operators.negation' defaultMessage='most operators can be negated' /></td>
+ </tr>
+ </tbody>
+ }
+
+ { searchEnabled &&
+ <tbody>
+ <tr>
+ <th colSpan='2'><FormattedMessage id='search_reference.search_operators.sections.accounts_and_posts' defaultMessage='Account and post operators' /></th>
+ </tr>
+ <tr>
+ <td><kbd>is:bot</kbd></td>
+ <td><FormattedMessage id='search_reference.search_operators.is.bot' defaultMessage='automated accounts and posts from them' /></td>
+ </tr>
+ <tr>
+ <td><kbd>domain:example.org</kbd></td>
+ <td><FormattedMessage id='search_reference.search_operators.domain' defaultMessage='limit search to users and posts from a given domain' /></td>
+ </tr>
+ <tr>
+ <td><kbd>scope:following</kbd></td>
+ <td><FormattedMessage id='search_reference.search_operators.scope.following' defaultMessage='limit search to users that you follow and posts from them' /></td>
+ </tr>
+ </tbody>
+ }
+
+ { searchEnabled &&
+ <tbody>
+ <tr>
+ <th colSpan='2'><FormattedMessage id='search_reference.search_operators.sections.accounts' defaultMessage='Account operators' /></th>
+ </tr>
+ <tr>
+ <td><kbd>is:group</kbd></td>
+ <td><FormattedMessage id='search_reference.search_operators.is.group' defaultMessage='accounts which represent groups' /></td>
+ </tr>
+ </tbody>
+ }
+
+ { searchEnabled &&
+ <tbody>
+ <tr>
+ <th colspan='2'><FormattedMessage id='search_reference.search_operators.sections.posts' defaultMessage='Post operators' /></th>
+ </tr>
+ <tr>
+ <td><kbd>from:@username@domain</kbd></td>
+ <td><FormattedMessage id='search_reference.search_operators.from' defaultMessage='posts authored by a given user' /></td>
+ </tr>
+ <tr>
+ <td><kbd>mentions:@username@domain</kbd></td>
+ <td><FormattedMessage id='search_reference.search_operators.mentions' defaultMessage='posts mentioning a given user' /></td>
+ </tr>
+ <tr>
+ <td><kbd>is:reply</kbd></td>
+ <td><FormattedMessage id='search_reference.search_operators.is.reply' defaultMessage='posts that are replies to another post' /></td>
+ </tr>
+ <tr>
+ <td><kbd>is:sensitive</kbd></td>
+ <td><FormattedMessage id='search_reference.search_operators.is.sensitive' defaultMessage='posts that include sensitive media' /></td>
+ </tr>
+ <tr>
+ <td><kbd>lang:es</kbd></td>
+ <td><FormattedMessage id='search_reference.search_operators.lang' defaultMessage='posts in the given language' /></td>
+ </tr>
+ <tr>
+ <td><kbd>has:link</kbd></td>
+ <td><FormattedMessage id='search_reference.search_operators.has.link' defaultMessage='posts that contain links' /></td>
+ </tr>
+ <tr>
+ <td><kbd>has:media</kbd></td>
+ <td><FormattedMessage id='search_reference.search_operators.has.media' defaultMessage='posts that include media of some kind' /></td>
+ </tr>
+ <tr>
+ <td><kbd>has:poll</kbd></td>
+ <td><FormattedMessage id='search_reference.search_operators.has.poll' defaultMessage='posts that include a poll' /></td>
+ </tr>
+ <tr>
+ <td><kbd>has:warning</kbd></td>
+ <td><FormattedMessage id='search_reference.search_operators.has.warning' defaultMessage='posts that have a content warning' /></td>
+ </tr>
+ <tr>
+ <td><kbd>before:2022-12-17</kbd></td>
+ <td><FormattedMessage id='search_reference.search_operators.date.before' defaultMessage='search before a given date' /></td>
+ </tr>
+ <tr>
+ <td><kbd>after:2022-12-17</kbd></td>
+ <td><FormattedMessage id='search_reference.search_operators.date.after' defaultMessage='search after a given date' /></td>
+ </tr>
+ <tr>
+ <td><kbd>sort:newest</kbd></td>
+ <td><FormattedMessage id='search_reference.search_operators.sort.newest' defaultMessage='show newest results first' /></td>
+ </tr>
+ <tr>
+ <td><kbd>sort:oldest</kbd></td>
+ <td><FormattedMessage id='search_reference.search_operators.sort.oldest' defaultMessage='show oldest results first' /></td>
+ </tr>
+ </tbody>
+ }
+ </table>
+ </div>
+
+ <Helmet>
+ <meta name='robots' content='noindex' />
+ </Helmet>
+ </Column>
+ );
+ }
+
+}
+
+export default injectIntl(SearchReference);
diff --git a/app/javascript/mastodon/features/ui/components/link_footer.jsx b/app/javascript/mastodon/features/ui/components/link_footer.jsx
index 68ef015ab9..9e17b05fca 100644
--- a/app/javascript/mastodon/features/ui/components/link_footer.jsx
+++ b/app/javascript/mastodon/features/ui/components/link_footer.jsx
@@ -89,6 +89,8 @@ class LinkFooter extends React.PureComponent {
{DividingCircle}
<Link to='/keyboard-shortcuts'><FormattedMessage id='footer.keyboard_shortcuts' defaultMessage='Keyboard shortcuts' /></Link>
{DividingCircle}
+ <Link to='/search-reference'><FormattedMessage id='footer.search_reference' defaultMessage='Search reference' /></Link>
+ {DividingCircle}
<a href={source_url} rel='noopener noreferrer' target='_blank'><FormattedMessage id='footer.source_code' defaultMessage='View source code' /></a>
{DividingCircle}
v{version}
diff --git a/app/javascript/mastodon/features/ui/index.jsx b/app/javascript/mastodon/features/ui/index.jsx
index 0e73f4c096..72d98cc2d2 100644
--- a/app/javascript/mastodon/features/ui/index.jsx
+++ b/app/javascript/mastodon/features/ui/index.jsx
@@ -27,6 +27,7 @@ import {
Status,
GettingStarted,
KeyboardShortcuts,
+ SearchReference,
PublicTimeline,
CommunityTimeline,
AccountTimeline,
@@ -177,6 +178,7 @@ class SwitchingColumnsArea extends React.PureComponent {
<WrappedRoute path='/getting-started' component={GettingStarted} content={children} />
<WrappedRoute path='/keyboard-shortcuts' component={KeyboardShortcuts} content={children} />
+ <WrappedRoute path='/search-reference' component={SearchReference} content={children} />
<WrappedRoute path='/about' component={About} content={children} />
<WrappedRoute path='/privacy-policy' component={PrivacyPolicy} content={children} />
@@ -484,7 +486,7 @@ class UI extends React.PureComponent {
};
handleHotkeyToggleHelp = () => {
- if (this.props.location.pathname === '/keyboard-shortcuts') {
+ if (this.props.location.pathname === '/keyboard-shortcuts' || this.props.location.pathname === '/search-reference') {
this.context.router.history.goBack();
} else {
this.context.router.history.push('/keyboard-shortcuts');
diff --git a/app/javascript/mastodon/features/ui/util/async-components.js b/app/javascript/mastodon/features/ui/util/async-components.js
index 1cf07f6453..a856177674 100644
--- a/app/javascript/mastodon/features/ui/util/async-components.js
+++ b/app/javascript/mastodon/features/ui/util/async-components.js
@@ -50,6 +50,10 @@ export function KeyboardShortcuts () {
return import(/* webpackChunkName: "features/keyboard_shortcuts" */'../../keyboard_shortcuts');
}
+export function SearchReference () {
+ return import(/* webpackChunkName: "features/search_reference" */'../../search_reference');
+}
+
export function PinnedStatuses () {
return import(/* webpackChunkName: "features/pinned_statuses" */'../../pinned_statuses');
}
diff --git a/app/javascript/styles/mastodon/components.scss b/app/javascript/styles/mastodon/components.scss
index 8622527817..fc9a3f0404 100644
--- a/app/javascript/styles/mastodon/components.scss
+++ b/app/javascript/styles/mastodon/components.scss
@@ -3320,10 +3320,15 @@ $ui-header-height: 55px;
}
}
-.keyboard-shortcuts {
+.keyboard-shortcuts,
+.search-reference {
padding: 8px 0 0;
overflow: hidden;
+ p {
+ padding: 0 10px 8px;
+ }
+
thead {
position: absolute;
inset-inline-start: -9999px;
@@ -3333,6 +3338,12 @@ $ui-header-height: 55px;
padding: 0 10px 8px;
}
+ th {
+ padding: 8px 10px;
+ text-align: left;
+ font-weight: bold;
+ }
+
kbd {
display: inline-block;
padding: 3px 5px;
diff --git a/app/lib/importer/statuses_index_importer.rb b/app/lib/importer/statuses_index_importer.rb
index b0721c2e02..4aa31161c1 100644
--- a/app/lib/importer/statuses_index_importer.rb
+++ b/app/lib/importer/statuses_index_importer.rb
@@ -24,7 +24,7 @@ class Importer::StatusesIndexImporter < Importer::BaseImporter
# is called before rendering the data and we need to filter based
# on the results of the filter, so this filtering happens here instead
bulk.map! do |entry|
- new_entry = if entry[:index] && entry.dig(:index, :data, 'searchable_by').blank?
+ new_entry = if entry[:index] && entry.dig(:index, :data, 'searchable_by').blank? && Rails.configuration.x.status_search_scope == :classic
{ delete: entry[:index].except(:data) }
else
entry
@@ -56,13 +56,23 @@ class Importer::StatusesIndexImporter < Importer::BaseImporter
end
def scopes
- [
+ classic_scopes = [
local_statuses_scope,
local_mentions_scope,
local_favourites_scope,
local_votes_scope,
local_bookmarks_scope,
]
+ case Rails.configuration.x.status_search_scope
+ when :discoverable
+ classic_scopes + [discoverable_scope]
+ when :public
+ classic_scopes + [public_scope]
+ when :public_or_unlisted
+ classic_scopes + [public_or_unlisted_scope]
+ else
+ classic_scopes
+ end
end
def local_mentions_scope
@@ -84,4 +94,16 @@ class Importer::StatusesIndexImporter < Importer::BaseImporter
def local_statuses_scope
Status.local.select('"statuses"."id", COALESCE("statuses"."reblog_of_id", "statuses"."id") AS status_id')
end
+
+ def discoverable_scope
+ Status.with_public_visibility.where(account: Account.discoverable).select('"statuses"."id", "statuses"."id" AS status_id')
+ end
+
+ def public_scope
+ Status.with_public_visibility.select('"statuses"."id", "statuses"."id" AS status_id')
+ end
+
+ def public_or_unlisted_scope
+ Status.with_public_or_unlisted_visibility.select('"statuses"."id", "statuses"."id" AS status_id')
+ end
end
diff --git a/app/lib/search_query_parser.rb b/app/lib/search_query_parser.rb
index 15956d4cfd..9512d47eee 100644
--- a/app/lib/search_query_parser.rb
+++ b/app/lib/search_query_parser.rb
@@ -1,15 +1,20 @@
# frozen_string_literal: true
class SearchQueryParser < Parslet::Parser
- rule(:term) { match('[^\s":]').repeat(1).as(:term) }
+ rule(:term) { match('[^\s"]').repeat(1).as(:term) }
rule(:quote) { str('"') }
rule(:colon) { str(':') }
+ rule(:hash) { str('#') }
rule(:space) { match('\s').repeat(1) }
rule(:operator) { (str('+') | str('-')).as(:operator) }
- rule(:prefix) { (term >> colon).as(:prefix) }
- rule(:shortcode) { (colon >> term >> colon.maybe).as(:shortcode) }
- rule(:phrase) { (quote >> (term >> space.maybe).repeat >> quote).as(:phrase) }
- rule(:clause) { (prefix.maybe >> operator.maybe >> (phrase | term | shortcode)).as(:clause) }
+ # See SearchQueryTransformer::PrefixClause::initialize for list of legal prefix operators.
+ # These are explictly enumerated here so they don't get mistaken for URLs.
+ rule(:prefix) { ((str('domain') | str('is') | str('has') | str('lang') | str('before') | str('after') | str('from') | str('mentions') | str('to') | str('scope') | str('sort')).as(:prefix) >> colon) }
+ # See CustomEmoji::SHORTCODE_RE_FRAGMENT and SCAN_RE for emoji grammar.
+ rule(:shortcode) { (colon >> match('[a-zA-Z0-9_]').repeat(2).as(:shortcode) >> colon) }
+ rule(:hashtag) { (hash >> match('[^\s#]').repeat(1).as(:hashtag)) }
+ rule(:phrase) { (quote >> match('[^"]').repeat(1).as(:phrase) >> quote) }
+ rule(:clause) { (operator.maybe >> prefix.maybe >> (phrase | shortcode | hashtag | term)).as(:clause) }
rule(:query) { (clause >> space.maybe).repeat.as(:query) }
root(:query)
end
diff --git a/app/lib/search_query_transformer.rb b/app/lib/search_query_transformer.rb
index aef05e9d9d..acb5f2bc6b 100644
--- a/app/lib/search_query_transformer.rb
+++ b/app/lib/search_query_transformer.rb
@@ -2,43 +2,150 @@
class SearchQueryTransformer < Parslet::Transform
class Query
- attr_reader :should_clauses, :must_not_clauses, :must_clauses, :filter_clauses
+ attr_reader :should_clauses, :must_not_clauses, :must_clauses, :filter_clauses, :order_clauses
def initialize(clauses)
- grouped = clauses.chunk(&:operator).to_h
+ grouped = clauses.group_by(&:operator).to_h
@should_clauses = grouped.fetch(:should, [])
@must_not_clauses = grouped.fetch(:must_not, [])
@must_clauses = grouped.fetch(:must, [])
@filter_clauses = grouped.fetch(:filter, [])
+ @order_clauses = grouped.fetch(:order, [])
end
- def apply(search)
- should_clauses.each { |clause| search = search.query.should(clause_to_query(clause)) }
- must_clauses.each { |clause| search = search.query.must(clause_to_query(clause)) }
- must_not_clauses.each { |clause| search = search.query.must_not(clause_to_query(clause)) }
- filter_clauses.each { |clause| search = search.filter(**clause_to_filter(clause)) }
+ # Return account IDs that must not block the searching account for the search to be legal.
+ def statuses_required_account_ids
+ ids = Set.new
+ positive_clauses = [should_clauses, must_clauses, filter_clauses]
+ positive_clauses.each do |clauses|
+ clauses.each do |clause|
+ id = clause_to_account_id(clause)
+ ids << id unless id.nil?
+ end
+ end
+ ids
+ end
+
+ # Modifies a statuses search to include clauses from this query.
+ def statuses_apply(search, account_id, following_ids)
+ search_type = :statuses
+ check_search_type(search_type)
+
+ search_fields = %w(text text.stemmed)
+ should_clauses.each { |clause| search = search.query.should(clause_to_query(clause, search_type, search_fields, account_id: account_id, following_ids: following_ids)) }
+ must_clauses.each { |clause| search = search.query.must(clause_to_query(clause, search_type, search_fields, account_id: account_id, following_ids: following_ids)) }
+ must_not_clauses.each { |clause| search = search.query.must_not(clause_to_query(clause, search_type, search_fields, account_id: account_id, following_ids: following_ids)) }
+ filter_clauses.each { |clause| search = search.filter(clause_to_query(clause, search_type, search_fields, account_id: account_id, following_ids: following_ids)) }
+ if order_clauses.empty?
+ # Default to most recent results first.
+ search = search.order(created_at: :desc)
+ else
+ order_clauses.each { |clause| search = search.order(clause_to_order(clause)) }
+ end
search.query.minimum_should_match(1)
end
+ # Generates the core query used for an accounts search.
+ def accounts_query(likely_acct, search_scope, account_exists, following, following_ids)
+ search_type = :accounts
+ check_search_type(search_type)
+
+ full_text_enabled = account_exists && search_scope != :classic
+
+ search_fields = %w(acct.edge_ngram acct)
+ search_fields += %w(display_name.edge_ngram display_name) unless likely_acct
+ search_fields += %w(text.stemmed text) if full_text_enabled
+
+ params = {
+ must: must_clauses.map { |clause| clause_to_query(clause, search_type, search_fields, following_ids: following_ids) },
+ must_not: must_not_clauses.map { |clause| clause_to_query(clause, search_type, search_fields, following_ids: following_ids) },
+ should: should_clauses.map { |clause| clause_to_query(clause, search_type, search_fields, following_ids: following_ids) },
+ filter: filter_clauses.map { |clause| clause_to_query(clause, search_type, search_fields, following_ids: following_ids) },
+ }
+
+ if account_exists
+ if following
+ params[:filter] << { terms: { id: following_ids } }
+ elsif following_ids.any?
+ params[:should] << { terms: { id: following_ids, boost: 0.5 } }
+ end
+ end
+
+ if full_text_enabled && search_scope == :discoverable
+ params[:filter] << { term: { discoverable: true } }
+ params[:filter] << { term: { silenced: false } }
+ end
+
+ { bool: params }
+ end
+
private
- def clause_to_query(clause)
- case clause
- when TermClause
- { multi_match: { type: 'most_fields', query: clause.term, fields: ['text', 'text.stemmed'] } }
- when PhraseClause
- { match_phrase: { text: { query: clause.phrase } } }
- else
- raise "Unexpected clause type: #{clause}"
+ # Raise an exception if there are clauses that don't work with this search type.
+ def check_search_type(search_type)
+ [
+ @should_clauses,
+ @must_not_clauses,
+ @must_clauses,
+ @filter_clauses,
+ @order_clauses,
+ ].each do |clauses|
+ clauses.each do |clause|
+ raise Mastodon::SyntaxError, "Unexpected clause for search type #{search_type}" if clause.respond_to?(:search_types) && clause.search_types.exclude?(search_type)
+ end
end
end
- def clause_to_filter(clause)
+ # Return any account ID related to a clause.
+ def clause_to_account_id(clause)
+ clause.term if clause.is_a?(PrefixClause) && %i(account_id mentions_ids).include?(clause.filter.to_sym)
+ end
+
+ def clause_to_query(clause, search_type, search_fields, account_id: nil, following_ids: nil)
+ case clause
+ when TermClause
+ { multi_match: { type: 'most_fields', query: clause.term, fields: search_fields, operator: 'and' } }
+ when PhraseClause
+ { match_phrase: { text: { query: clause.phrase } } }
+ when PrefixClause
+ # Some prefix clauses yield queries that depend on the search type or account.
+ filter = case clause.filter
+ when :account_id_filter_placeholder
+ case search_type
+ when :accounts
+ 'id'
+ when :statuses
+ 'account_id'
+ else
+ raise Mastodon::SyntaxError, "Unexpected search type for query: #{search_type}"
+ end
+ else
+ clause.filter
+ end
+ term = case clause.term
+ when :account_id_placeholder
+ account_id
+ when :following_ids_placeholder
+ following_ids
+ else
+ clause.term
+ end
+ { clause.query => { filter => term } }
+ when EmojiClause
+ { term: { emojis: clause.shortcode } }
+ when TagClause
+ { term: { tags: clause.tag } }
+ else
+ raise Mastodon::SyntaxError, "Unexpected clause type for query: #{clause}"
+ end
+ end
+
+ def clause_to_order(clause)
case clause
when PrefixClause
- { term: { clause.filter => clause.term } }
+ { clause.term => clause.order }
else
- raise "Unexpected clause type: #{clause}"
+ raise Mastodon::SyntaxError, "Unexpected clause type for filter: #{clause}"
end
end
end
@@ -54,7 +161,18 @@ class SearchQueryTransformer < Parslet::Transform
when nil
:should
else
- raise "Unknown operator: #{str}"
+ raise Mastodon::SyntaxError, "Unknown operator: #{str}"
+ end
+ end
+
+ def filter_context_symbol(str)
+ case str
+ when '+', nil
+ :filter
+ when '-'
+ :must_not
+ else
+ raise Mastodon::SyntaxError, "Unknown operator: #{str}"
end
end
end
@@ -80,40 +198,176 @@ class SearchQueryTransformer < Parslet::Transform
end
end
+ class EmojiClause
+ attr_reader :prefix, :operator, :shortcode
+
+ def initialize(prefix, operator, shortcode)
+ @prefix = prefix
+ @operator = Operator.filter_context_symbol(operator)
+ @shortcode = shortcode
+ end
+ end
+
+ class TagClause
+ attr_reader :prefix, :operator, :tag
+
+ def initialize(prefix, operator, tag)
+ @prefix = prefix
+ @operator = Operator.filter_context_symbol(operator)
+ @tag = tag
+ end
+ end
+
+ # If you add a new prefix here, make sure to add it to SearchQueryParser as well.
class PrefixClause
- attr_reader :filter, :operator, :term
+ attr_reader :filter, :operator, :term, :order, :query, :search_types
+
+ def initialize(prefix, operator, term)
+ # These defaults may be modified by prefix-operator-specific initializers below.
+ @query = :term
+ @filter = prefix
+ @term = term
+ # Some prefixes don't apply to all search types.
+ @search_types = %i(accounts statuses)
+ @operator = Operator.filter_context_symbol(operator)
- def initialize(prefix, term)
- @operator = :filter
case prefix
- when 'from'
- @filter = :account_id
+ when 'domain'
+ initialize_is_local if TagManager.instance.local_domain?(term)
- username, domain = term.gsub(/\A@/, '').split('@')
- domain = nil if TagManager.instance.local_domain?(domain)
- account = Account.find_remote!(username, domain)
+ when 'is'
+ initialize_is(term)
+
+ when 'has'
+ initialize_has(term)
+
+ when 'lang'
+ @search_types = %i(statuses)
+
+ when 'before', 'after'
+ initialize_date_range(prefix, operator, term)
+
+ when 'from', 'mentions', 'to'
+ initialize_account(prefix, term)
+
+ when 'scope'
+ initialize_scope(operator, term)
+
+ when 'sort'
+ initialize_sort(operator, term)
- @term = account.id
else
raise Mastodon::SyntaxError
end
end
+
+ private
+
+ def initialize_is(term)
+ case term
+ when 'bot', 'group'
+ # These apply to all search types. No action required.
+ when 'local'
+ initialize_is_local
+ when 'local_only', 'reply', 'sensitive'
+ @search_types = %i(statuses)
+ else
+ raise Mastodon::SyntaxError, "Unknown keyword for is: prefix: #{term}"
+ end
+ end
+
+ # We can identify local objects by querying for objects that don't have a domain field.
+ def initialize_is_local
+ @operator = @operator == :filter ? :must_not : :filter
+ @query = :exists
+ @filter = :field
+ @term = 'domain'
+ end
+
+ def initialize_has(term)
+ @search_types = %i(statuses)
+ case term
+ when 'link', 'media', 'poll', 'warning', *(MediaAttachment.types.keys.reject { |t| t == 'unknown' })
+ # Pass all of these through.
+ when 'cw', 'spoiler'
+ @term = 'warning'
+ else
+ raise Mastodon::SyntaxError, "Unknown keyword for has: prefix: #{term}"
+ end
+ end
+
+ def initialize_date_range(prefix, operator, term)
+ raise Mastodon::SyntaxError, 'Operator not allowed for date range' unless operator.nil?
+
+ @query = :range
+ @filter = 'created_at'
+ @term = {
+ before: { lt: term },
+ after: { gt: term },
+ }[prefix.to_sym] or raise Mastodon::SyntaxError, "Unknown date range prefix: #{prefix}"
+ end
+
+ def initialize_account(prefix, term)
+ @search_types = %i(statuses)
+ @filter = {
+ from: :account_id,
+ mentions: :mentions_ids,
+ to: :mentions_ids,
+ }[prefix.to_sym] or raise Mastodon::SyntaxError, "Unknown account filter prefix: #{prefix}"
+
+ username, domain = term.gsub(/\A@/, '').split('@')
+ domain = nil if TagManager.instance.local_domain?(domain)
+ account = Account.find_remote!(username, domain)
+
+ @term = account.id
+ end
+
+ def initialize_scope(operator, term)
+ raise Mastodon::SyntaxError, 'Operator not allowed for scope: prefix' unless operator.nil?
+
+ case term
+ when 'classic'
+ @search_types = %i(statuses)
+ @filter = 'searchable_by'
+ @term = :account_id_placeholder
+ when 'following'
+ @query = :terms
+ # This scope queries different fields depending on search context.
+ @filter = :account_id_filter_placeholder
+ @term = :following_ids_placeholder
+ else
+ raise Mastodon::SyntaxError, "Unknown scope: #{term}"
+ end
+ end
+
+ def initialize_sort(operator, term)
+ raise Mastodon::SyntaxError, 'Operator not allowed for sort: prefix' unless operator.nil?
+
+ @operator = :order
+ @term = :created_at
+ @order = {
+ oldest: :asc,
+ newest: :desc,
+ }[term.to_sym] or raise Mastodon::SyntaxError, "Unknown sort: #{term}"
+ end
end
rule(clause: subtree(:clause)) do
- prefix = clause[:prefix][:term].to_s if clause[:prefix]
+ prefix = clause[:prefix]&.to_s
operator = clause[:operator]&.to_s
if clause[:prefix]
- PrefixClause.new(prefix, clause[:term].to_s)
+ PrefixClause.new(prefix, operator, clause[:term].to_s)
elsif clause[:term]
TermClause.new(prefix, operator, clause[:term].to_s)
elsif clause[:shortcode]
- TermClause.new(prefix, operator, ":#{clause[:term]}:")
+ EmojiClause.new(prefix, operator, clause[:shortcode].to_s)
+ elsif clause[:hashtag]
+ TagClause.new(prefix, operator, clause[:hashtag].to_s)
elsif clause[:phrase]
- PhraseClause.new(prefix, operator, clause[:phrase].is_a?(Array) ? clause[:phrase].map { |p| p[:term].to_s }.join(' ') : clause[:phrase].to_s)
+ PhraseClause.new(prefix, operator, clause[:phrase].to_s)
else
- raise "Unexpected clause type: #{clause}"
+ raise Mastodon::SyntaxError, "Unexpected clause type: #{clause}"
end
end
diff --git a/app/models/account.rb b/app/models/account.rb
index 4fc7b9d08b..6e8ba3547b 100644
--- a/app/models/account.rb
+++ b/app/models/account.rb
@@ -545,6 +545,39 @@ class Account < ApplicationRecord
save!
end
+ def searchable_text
+ [
+ PlainTextFormatter.new(note, local?).to_s,
+ fields.map do |field|
+ [
+ field.name,
+ PlainTextFormatter.new(field.value, local?).to_s,
+ ].join(' ')
+ end,
+ ].join("\n\n")
+ end
+
+ def searchable_emojis
+ CustomEmoji.from_text(display_name, domain).reject(&:disabled).pluck(:shortcode)
+ end
+
+ def searchable_tags
+ return [] unless discoverable
+
+ tags = []
+ tags += Extractor.extract_hashtags(PlainTextFormatter.new(note, local?).to_s)
+ tags += featured_tags.pluck(:name)
+ tags.uniq
+ end
+
+ def searchable_is
+ keywords = []
+ keywords << :bot if bot?
+ keywords << :group if group?
+ keywords << :local if local?
+ keywords
+ end
+
private
def prepare_contents
diff --git a/app/models/status.rb b/app/models/status.rb
index 8a58e5d685..6de8b98e13 100644
--- a/app/models/status.rb
+++ b/app/models/status.rb
@@ -100,6 +100,7 @@ class Status < ApplicationRecord
scope :without_replies, -> { where('statuses.reply = FALSE OR statuses.in_reply_to_account_id = statuses.account_id') }
scope :without_reblogs, -> { where(statuses: { reblog_of_id: nil }) }
scope :with_public_visibility, -> { where(visibility: :public) }
+ scope :with_public_or_unlisted_visibility, -> { where(visibility: [:public, :unlisted]) }
scope :tagged_with, ->(tag_ids) { joins(:statuses_tags).where(statuses_tags: { tag_id: tag_ids }) }
scope :excluding_silenced_accounts, -> { left_outer_joins(:account).where(accounts: { silenced_at: nil }) }
scope :including_silenced_accounts, -> { left_outer_joins(:account).where.not(accounts: { silenced_at: nil }) }
@@ -171,6 +172,10 @@ class Status < ApplicationRecord
ids.uniq
end
+ def searchable_mentions_ids
+ mentions.joins(:account).active.pluck(:account_id)
+ end
+
def searchable_text
[
spoiler_text,
@@ -180,6 +185,38 @@ class Status < ApplicationRecord
].compact.join("\n\n")
end
+ def searchable_emojis
+ emojis.reject(&:disabled).pluck(:shortcode)
+ end
+
+ def searchable_tags
+ Extractor.extract_hashtags(FormattingHelper.extract_status_plain_text(self))
+ end
+
+ def searchable_is
+ keywords = []
+ keywords << :bot if account.bot?
+ keywords << :group if account.group?
+ # Glitch and Hometown have local-only posts. Vanilla Mastodon doesn't.
+ keywords << :local_only if self.class.method_defined?(:local_only?) && local_only?
+ keywords << :reply if reply?
+ keywords << :sensitive if sensitive?
+ keywords
+ end
+
+ def searchable_has
+ keywords = []
+ keywords << :warning if spoiler_text?
+ keywords << :link if FetchLinkCardService.new.link?(self)
+ keywords << :poll if preloadable_poll.present?
+
+ media_types = media_attachments.pluck(:type).map(&:to_sym).uniq
+ keywords << :media if media_types.present?
+ keywords += media_types.reject { |t| t == :unknown }
+
+ keywords
+ end
+
def to_log_human_identifier
account.acct
end
diff --git a/app/services/account_search_service.rb b/app/services/account_search_service.rb
index dfc3a45f8f..cbafdedae7 100644
--- a/app/services/account_search_service.rb
+++ b/app/services/account_search_service.rb
@@ -72,20 +72,18 @@ class AccountSearchService < BaseService
end
def from_elasticsearch
- must_clauses = [{ multi_match: { query: terms_for_query, fields: likely_acct? ? %w(acct.edge_ngram acct) : %w(acct.edge_ngram acct display_name.edge_ngram display_name), type: 'most_fields', operator: 'and' } }]
- should_clauses = []
+ return [] if account && options[:following] && following_ids.empty?
- if account
- return [] if options[:following] && following_ids.empty?
-
- if options[:following]
- must_clauses << { terms: { id: following_ids } }
- elsif following_ids.any?
- should_clauses << { terms: { id: following_ids, boost: 100 } }
- end
- end
-
- query = { bool: { must: must_clauses, should: should_clauses } }
+ query = SearchQueryTransformer
+ .new
+ .apply(SearchQueryParser.new.parse(@query))
+ .accounts_query(
+ likely_acct?,
+ Rails.configuration.x.account_search_scope,
+ !account.nil?,
+ options[:following],
+ following_ids
+ )
functions = [reputation_score_function, followers_score_function, time_distance_function]
records = AccountsIndex.query(function_score: { query: query, functions: functions, boost_mode: 'multiply', score_mode: 'avg' })
diff --git a/app/services/fetch_link_card_service.rb b/app/services/fetch_link_card_service.rb
index 8d07958b73..ba7cd29709 100644
--- a/app/services/fetch_link_card_service.rb
+++ b/app/services/fetch_link_card_service.rb
@@ -34,6 +34,15 @@ class FetchLinkCardService < BaseService
nil
end
+ # Detect whether the status has at least one link.
+ def link?(status)
+ @status = status
+ @original_url = parse_urls
+ !@original_url.nil?
+ rescue Addressable::URI::InvalidURIError
+ false
+ end
+
private
def process_url
diff --git a/app/services/search_service.rb b/app/services/search_service.rb
index b1ce5453fb..768504beb6 100644
--- a/app/services/search_service.rb
+++ b/app/services/search_service.rb
@@ -15,9 +15,39 @@ class SearchService < BaseService
if url_query?
results.merge!(url_resource_results) unless url_resource.nil? || @offset.positive? || (@options[:type].present? && url_resource_symbol != @options[:type].to_sym)
elsif @query.present?
- results[:accounts] = perform_accounts_search! if account_searchable?
- results[:statuses] = perform_statuses_search! if full_text_searchable?
- results[:hashtags] = perform_hashtags_search! if hashtag_searchable?
+ # Account and status searches use different sets of prefix operators.
+ # Throw a syntax error only if the syntax is invalid in all search contexts.
+ search_succeeded = false
+ syntax_error = nil
+
+ if account_searchable?
+ begin
+ results[:accounts] = perform_accounts_search!
+ search_succeeded = true
+ rescue Mastodon::SyntaxError => e
+ syntax_error = e
+ end
+ end
+
+ if status_searchable?
+ begin
+ results[:statuses] = perform_statuses_search!
+ search_succeeded = true
+ rescue Mastodon::SyntaxError => e
+ syntax_error = e
+ end
+ end
+
+ if hashtag_searchable?
+ begin
+ results[:hashtags] = perform_hashtags_search!
+ search_succeeded = true
+ rescue Mastodon::SyntaxError => e
+ syntax_error = e
+ end
+ end
+
+ raise syntax_error unless syntax_error.nil? || search_succeeded
end
end
end
@@ -35,7 +65,34 @@ class SearchService < BaseService
end
def perform_statuses_search!
- definition = parsed_query.apply(StatusesIndex.filter(term: { searchable_by: @account.id }))
+ required_account_ids = parsed_query.statuses_required_account_ids
+ return [] if @account.blocked_by.exists?(id: required_account_ids)
+
+ statuses_index = StatusesIndex.filter(term: { searchable_by: @account.id })
+
+ status_search_scope = Rails.configuration.x.status_search_scope
+ case status_search_scope
+ when :discoverable
+ statuses_index = statuses_index.filter.or(
+ bool: {
+ filter: [
+ { term: { visibility: 'public' } },
+ { term: { discoverable: true } },
+ { term: { silenced: false } },
+ ],
+ }
+ )
+ when :public
+ statuses_index = statuses_index.filter.or(term: { visibility: 'public' })
+ when :public_or_unlisted
+ statuses_index = statuses_index.filter.or(terms: { visibility: %w(public unlisted) })
+ when :classic
+ # No alternate filter queries.
+ else
+ raise InvalidParameterError, "Unexpected status search scope: #{status_search_scope}"
+ end
+
+ definition = parsed_query.statuses_apply(statuses_index, @account.id, following_ids)
definition = definition.filter(term: { account_id: @options[:account_id] }) if @options[:account_id].present?
@@ -85,14 +142,14 @@ class SearchService < BaseService
url_resource.class.name.downcase.pluralize.to_sym
end
- def full_text_searchable?
+ def status_searchable?
return false unless Chewy.enabled?
- statuses_search? && !@account.nil? && !((@query.start_with?('#') || @query.include?('@')) && !@query.include?(' '))
+ statuses_search? && !@account.nil?
end
def account_searchable?
- account_search? && !(@query.start_with?('#') || (@query.include?('@') && @query.include?(' ')))
+ account_search?
end
def hashtag_searchable?
@@ -114,4 +171,8 @@ class SearchService < BaseService
def parsed_query
SearchQueryTransformer.new.apply(SearchQueryParser.new.parse(@query))
end
+
+ def following_ids
+ @following_ids ||= @account.active_relationships.pluck(:target_account_id) + [@account.id]
+ end
end
diff --git a/config/initializers/search_scope.rb b/config/initializers/search_scope.rb
new file mode 100644
index 0000000000..218246fa28
--- /dev/null
+++ b/config/initializers/search_scope.rb
@@ -0,0 +1,24 @@
+# frozen_string_literal: true
+
+Rails.application.configure do
+ # STATUS_SEARCH_SCOPE was previously known as SEARCH_SCOPE and only covered statuses.
+ config.x.status_search_scope = case
+ when (ENV['STATUS_SEARCH_SCOPE'] || ENV['SEARCH_SCOPE']) == 'discoverable'
+ :discoverable
+ when (ENV['STATUS_SEARCH_SCOPE'] || ENV['SEARCH_SCOPE']) == 'public'
+ :public
+ when (ENV['STATUS_SEARCH_SCOPE'] || ENV['SEARCH_SCOPE']) == 'public_or_unlisted'
+ :public_or_unlisted
+ else
+ :classic
+ end
+
+ config.x.account_search_scope = case
+ when ENV['ACCOUNT_SEARCH_SCOPE'] == 'all'
+ :all
+ when ENV['ACCOUNT_SEARCH_SCOPE'] == 'discoverable'
+ :discoverable
+ else
+ :classic
+ end
+end
diff --git a/config/routes.rb b/config/routes.rb
index 0999331165..dd396b6884 100644
--- a/config/routes.rb
+++ b/config/routes.rb
@@ -10,6 +10,7 @@ Rails.application.routes.draw do
/getting-started
/getting-started-misc
/keyboard-shortcuts
+ /search-reference
/home
/public
/public/local
diff --git a/spec/lib/search_query_parser_spec.rb b/spec/lib/search_query_parser_spec.rb
new file mode 100644
index 0000000000..ee3287d971
--- /dev/null
+++ b/spec/lib/search_query_parser_spec.rb
@@ -0,0 +1,239 @@
+# frozen_string_literal: true
+
+require 'rails_helper'
+
+describe SearchQueryParser do
+ subject(:parser) { described_class.new }
+
+ describe '#parse' do
+ context 'when given a simple text query' do
+ let(:query) { 'text' }
+
+ it 'parses the query' do
+ parsed_query = parser.parse(query)
+ expect(parsed_query).to match(
+ query: [
+ {
+ clause: {
+ term: 'text',
+ },
+ },
+ ]
+ )
+ end
+ end
+
+ context 'when given a query with a remote account name' do
+ let(:query) { 'user@domain.tld' }
+
+ it 'parses the account name as a term' do
+ parsed_query = parser.parse(query)
+ expect(parsed_query).to match(
+ query: [
+ {
+ clause: {
+ term: 'user@domain.tld',
+ },
+ },
+ ]
+ )
+ end
+ end
+
+ context 'when given a query with an @-prefixed remote account name' do
+ let(:query) { '@user@domain.tld' }
+
+ it 'parses the account name as a term' do
+ parsed_query = parser.parse(query)
+ expect(parsed_query).to match(
+ query: [
+ {
+ clause: {
+ term: '@user@domain.tld',
+ },
+ },
+ ]
+ )
+ end
+ end
+
+ context 'when given a query with an @-prefixed local account name' do
+ let(:query) { '@user' }
+
+ it 'parses the account name as a term' do
+ parsed_query = parser.parse(query)
+ expect(parsed_query).to match(
+ query: [
+ {
+ clause: {
+ term: '@user',
+ },
+ },
+ ]
+ )
+ end
+ end
+
+ context 'when given a quoted phrase query' do
+ let(:query) { '"a phrase"' }
+
+ it 'parses the query' do
+ parsed_query = parser.parse(query)
+ expect(parsed_query).to match(
+ query: [
+ {
+ clause: {
+ phrase: 'a phrase',
+ },
+ },
+ ]
+ )
+ end
+ end
+
+ context 'when given a malformed quoted phrase query' do
+ let(:query) { '"a phrase' }
+
+ it 'raises a Parslet exception' do
+ expect { parser.parse(query) }.to raise_exception Parslet::ParseFailed
+ end
+ end
+
+ context 'when given a text query with an operator' do
+ let(:query) { '+text' }
+
+ it 'parses the query' do
+ parsed_query = parser.parse(query)
+ expect(parsed_query).to match(
+ query: [
+ {
+ clause: {
+ operator: '+',
+ term: 'text',
+ },
+ },
+ ]
+ )
+ end
+ end
+
+ context 'when given a prefix query' do
+ let(:query) { 'from:user' }
+
+ it 'parses the query' do
+ parsed_query = parser.parse(query)
+ expect(parsed_query).to match(
+ query: [
+ {
+ clause: {
+ prefix: 'from',
+ term: 'user',
+ },
+ },
+ ]
+ )
+ end
+ end
+
+ context 'when given a query containing a prefix with nothing after it' do
+ let(:query) { 'from:' }
+
+ it 'raises a Parslet exception' do
+ expect { parser.parse(query) }.to raise_exception Parslet::ParseFailed
+ end
+ end
+
+ context 'when given a prefix query with an operator' do
+ let(:query) { '-from:user' }
+
+ it 'parses the query' do
+ parsed_query = parser.parse(query)
+ expect(parsed_query).to match(
+ query: [
+ {
+ clause: {
+ operator: '-',
+ prefix: 'from',
+ term: 'user',
+ },
+ },
+ ]
+ )
+ end
+ end
+
+ context 'when given a prefix query with a remote account name' do
+ let(:query) { 'from:user@domain.tld' }
+
+ it 'parses the query' do
+ parsed_query = parser.parse(query)
+ expect(parsed_query).to match(
+ query: [
+ {
+ clause: {
+ prefix: 'from',
+ term: 'user@domain.tld',
+ },
+ },
+ ]
+ )
+ end
+ end
+
+ context 'when given a mixed text and hashtag query' do
+ let(:query) { 'text #hashtag' }
+
+ it 'parses the query' do
+ parsed_query = parser.parse(query)
+ expect(parsed_query).to match(
+ query: [
+ {
+ clause: {
+ term: 'text',
+ },
+ },
+ {
+ clause: {
+ hashtag: 'hashtag',
+ },
+ },
+ ]
+ )
+ end
+ end
+
+ context 'when given a bare URL query' do
+ let(:query) { 'https://example.org/' }
+
+ it 'parses the URL as a term' do
+ parsed_query = parser.parse(query)
+ expect(parsed_query).to match(
+ query: [
+ {
+ clause: {
+ term: 'https://example.org/',
+ },
+ },
+ ]
+ )
+ end
+ end
+
+ context 'when given a quoted URL query' do
+ let(:query) { '"https://example.org/"' }
+
+ it 'parses the URL as a phrase' do
+ parsed_query = parser.parse(query)
+ expect(parsed_query).to match(
+ query: [
+ {
+ clause: {
+ phrase: 'https://example.org/',
+ },
+ },
+ ]
+ )
+ end
+ end
+ end
+end
diff --git a/spec/lib/search_query_transformer_spec.rb b/spec/lib/search_query_transformer_spec.rb
index 1095334695..b608b6e0dd 100644
--- a/spec/lib/search_query_transformer_spec.rb
+++ b/spec/lib/search_query_transformer_spec.rb
@@ -3,16 +3,252 @@
require 'rails_helper'
describe SearchQueryTransformer do
- describe 'initialization' do
- let(:parser) { SearchQueryParser.new.parse('query') }
+ subject(:transformer) { described_class.new.apply(SearchQueryParser.new.parse(query)) }
- it 'sets attributes' do
- transformer = described_class.new.apply(parser)
+ describe '#initialize' do
+ context 'when given a query' do
+ let(:query) { 'query' }
- expect(transformer.should_clauses.first).to be_a(SearchQueryTransformer::TermClause)
- expect(transformer.must_clauses.first).to be_nil
- expect(transformer.must_not_clauses.first).to be_nil
- expect(transformer.filter_clauses.first).to be_nil
+ it 'sets attributes' do
+ expect(transformer.should_clauses.first).to be_a(SearchQueryTransformer::TermClause)
+ expect(transformer.must_clauses.first).to be_nil
+ expect(transformer.must_not_clauses.first).to be_nil
+ expect(transformer.filter_clauses.first).to be_nil
+ expect(transformer.order_clauses.first).to be_nil
+ end
+ end
+
+ context 'when given a domain: query for the test domain' do
+ let(:query) { 'domain:cb6e6126.ngrok.io' }
+
+ it 'generates a does-not-exist query on the domain field' do
+ expect(transformer.must_not_clauses.length).to eq(1)
+ expect(transformer.filter_clauses).to be_empty
+
+ clause = transformer.must_not_clauses[0]
+ expect(clause).to be_a(described_class::PrefixClause)
+ expect(clause.query).to eq(:exists)
+ expect(clause.filter).to eq(:field)
+ expect(clause.term).to eq('domain')
+ end
+ end
+
+ context 'when given a domain: query for a remote domain' do
+ let(:query) { 'domain:example.org' }
+
+ it 'generates a match query on the domain field' do
+ expect(transformer.must_not_clauses).to be_empty
+ expect(transformer.filter_clauses.length).to eq(1)
+
+ clause = transformer.filter_clauses[0]
+ expect(clause).to be_a(described_class::PrefixClause)
+ expect(clause.query).to eq(:term)
+ expect(clause.filter).to eq('domain')
+ expect(clause.term).to eq('example.org')
+ end
+ end
+
+ context 'when given an is:local query' do
+ let(:query) { 'is:local' }
+
+ it 'generates a does-not-exist query on the domain field' do
+ expect(transformer.must_not_clauses.length).to eq(1)
+ expect(transformer.filter_clauses).to be_empty
+
+ clause = transformer.must_not_clauses[0]
+ expect(clause).to be_a(described_class::PrefixClause)
+ expect(clause.query).to eq(:exists)
+ expect(clause.filter).to eq(:field)
+ expect(clause.term).to eq('domain')
+ end
+ end
+
+ context 'when given a -is:local query' do
+ let(:query) { '-is:local' }
+
+ it 'generates an exists query on the domain field' do
+ expect(transformer.must_not_clauses).to be_empty
+ expect(transformer.filter_clauses.length).to eq(1)
+
+ clause = transformer.filter_clauses[0]
+ expect(clause).to be_a(described_class::PrefixClause)
+ expect(clause.query).to eq(:exists)
+ expect(clause.filter).to eq(:field)
+ expect(clause.term).to eq('domain')
+ end
+ end
+
+ context 'when given an is:sensitive query' do
+ let(:query) { 'is:sensitive' }
+
+ it 'generates a term query on the is field' do
+ expect(transformer.must_not_clauses).to be_empty
+ expect(transformer.filter_clauses.length).to eq(1)
+
+ clause = transformer.filter_clauses[0]
+ expect(clause).to be_a(described_class::PrefixClause)
+ expect(clause.query).to eq(:term)
+ expect(clause.filter).to eq('is')
+ expect(clause.term).to eq('sensitive')
+ end
+ end
+
+ context 'when given an -is:sensitive query' do
+ let(:query) { '-is:sensitive' }
+
+ it 'generates a term query on the is field' do
+ expect(transformer.must_not_clauses.length).to eq(1)
+ expect(transformer.filter_clauses).to be_empty
+
+ clause = transformer.must_not_clauses[0]
+ expect(clause).to be_a(described_class::PrefixClause)
+ expect(clause.query).to eq(:term)
+ expect(clause.filter).to eq('is')
+ expect(clause.term).to eq('sensitive')
+ end
+ end
+ end
+
+ describe '#statuses_required_account_ids' do
+ let(:blocking_account) { Fabricate(:account, domain: nil, username: 'blocksyou') }
+
+ before do
+ blocking_account.save!
+ end
+
+ context 'when given a from: query' do
+ let(:query) { 'from:blocksyou' }
+
+ it "returns the mentioned account's ID" do
+ required_account_ids = transformer.statuses_required_account_ids
+ expect(required_account_ids).to include(blocking_account.id)
+ end
+ end
+
+ context 'when given a -from: query' do
+ let(:query) { '-from:blocksyou' }
+
+ it "does not return the mentioned account's ID" do
+ required_account_ids = transformer.statuses_required_account_ids
+ expect(required_account_ids).to_not include(blocking_account.id)
+ end
+ end
+
+ context 'when given a mentions: query' do
+ let(:query) { 'mentions:blocksyou' }
+
+ it "returns the mentioned account's ID" do
+ required_account_ids = transformer.statuses_required_account_ids
+ expect(required_account_ids).to include(blocking_account.id)
+ end
+ end
+
+ context 'when given a -mentions: query' do
+ let(:query) { '-mentions:blocksyou' }
+
+ it "does not return the mentioned account's ID" do
+ required_account_ids = transformer.statuses_required_account_ids
+ expect(required_account_ids).to_not include(blocking_account.id)
+ end
+ end
+ end
+
+ describe '#statuses_apply' do
+ let(:account) { Fabricate(:account, domain: nil, username: 'you') }
+ let(:following_ids) { [] }
+
+ context 'when given a simple text query' do
+ let(:query) { 'text' }
+
+ it 'modifies the search based on the query' do
+ search = Chewy::Search::Request.new
+ search = transformer.statuses_apply(search, account.id, following_ids)
+ expect(search.render).to match(
+ hash_including(
+ body: hash_including(
+ query: hash_including(
+ bool: hash_including(
+ :should
+ )
+ )
+ )
+ )
+ )
+ end
+ end
+
+ context 'when given a query with universal operators' do
+ let(:query) { '-is:bot' }
+
+ it 'modifies the search based on the query' do
+ search = Chewy::Search::Request.new
+ search = transformer.statuses_apply(search, account.id, following_ids)
+ expect(search.render).to match(
+ hash_including(
+ body: hash_including(
+ query: hash_including(
+ bool: hash_including(
+ :must_not
+ )
+ )
+ )
+ )
+ )
+ end
+ end
+
+ context 'when given a query with status-specific operators' do
+ let(:query) { 'is:reply' }
+
+ it 'modifies the search based on the query' do
+ search = Chewy::Search::Request.new
+ search = transformer.statuses_apply(search, account.id, following_ids)
+ expect(search.render).to match(
+ hash_including(
+ body: hash_including(
+ query: hash_including(
+ bool: hash_including(
+ :filter
+ )
+ )
+ )
+ )
+ )
+ end
+ end
+ end
+
+ describe '#accounts_query' do
+ let(:likely_acct) { false }
+ let(:search_scope) { :discoverable }
+ let(:account_exists) { true }
+ let(:following) { false }
+ let(:following_ids) { [] }
+
+ context 'when given a simple text query' do
+ let(:query) { 'text' }
+
+ it 'returns an ES query hash' do
+ es_query = transformer.accounts_query(likely_acct, search_scope, account_exists, following, following_ids)
+ expect(es_query).to be_a(Hash)
+ end
+ end
+
+ context 'when given a query with universal operators' do
+ let(:query) { '-is:bot' }
+
+ it 'returns an ES query hash' do
+ es_query = transformer.accounts_query(likely_acct, search_scope, account_exists, following, following_ids)
+ expect(es_query).to be_a(Hash)
+ end
+ end
+
+ context 'when given a query with status-specific operators' do
+ let(:query) { 'is:reply' }
+
+ it 'throws a syntax error' do
+ expect { transformer.accounts_query(likely_acct, search_scope, account_exists, following, following_ids) }.to raise_exception Mastodon::SyntaxError
+ end
end
end
end
diff --git a/spec/services/search_service_spec.rb b/spec/services/search_service_spec.rb
index 1ad0efe0af..050df1a769 100644
--- a/spec/services/search_service_spec.rb
+++ b/spec/services/search_service_spec.rb
@@ -92,15 +92,6 @@ describe SearchService, type: :service do
expect(Tag).to_not have_received(:search_for)
expect(results).to eq empty_results
end
-
- it 'does not include account when starts with # character' do
- query = '#tag'
- allow(AccountSearchService).to receive(:new)
-
- results = subject.call(query, nil, 10)
- expect(AccountSearchService).to_not have_received(:new)
- expect(results).to eq empty_results
- end
end
end
end