Renaming Philomena and scraping-related classes directory

src/lib/philomena/scraping/ScrapedAPI.ts

@@ -0,0 +1,52 @@
+import PostParser from "$lib/philomena/scraping/parsing/PostParser";
+
+type UpdaterFunction = (tags: Set<string>) => Set<string>;
+
+export default class ScrapedAPI {
+  /**
+   * Update the tags of the image using callback.
+   * @param imageId ID of the image.
+   * @param callback Callback to call to change the content.
+   * @return Updated tags and aliases list for updating internal cached state.
+   */
+  async updateImageTags(imageId: number, callback: UpdaterFunction): Promise<Map<string, string> | null> {
+    const postParser = new PostParser(imageId);
+    const formData = await postParser.resolveTagEditorFormData();
+    const tagsFieldValue = formData.get(PostParser.tagsInputName);
+
+    if (typeof tagsFieldValue !== 'string') {
+      throw new Error('Missing tags field!');
+    }
+
+    const tagsList = new Set(
+      tagsFieldValue
+        .split(',')
+        .map(tagName => tagName.trim())
+    );
+
+    const updateTagsList = callback(tagsList);
+
+    if (!(updateTagsList instanceof Set)) {
+      throw new Error("Return value is not a set!");
+    }
+
+    formData.set(
+      PostParser.tagsInputName,
+      Array.from(updateTagsList).join(', ')
+    );
+
+    await fetch(`/images/${imageId}/tags`, {
+      method: 'POST',
+      body: formData,
+    });
+
+    // We need to remove stored version of the document to request an updated version.
+    postParser.clear();
+
+    // Additional request to re-fetch the new list of tags and aliases. I couldn't find the way to request this list
+    // using official API.
+    // TODO Maybe it will be better to resolve aliases on the extension side somehow, maybe by requesting and caching
+    //      aliases in storage.
+    return await postParser.resolveTagsAndAliases();
+  }
+}

src/lib/philomena/scraping/parsing/PageParser.ts

@@ -0,0 +1,47 @@
+export default class PageParser {
+  readonly #url: string;
+  #fragment: DocumentFragment | null = null;
+
+  constructor(url: string) {
+    this.#url = url;
+  }
+
+  async resolveFragment(): Promise<DocumentFragment> {
+    if (this.#fragment) {
+      return this.#fragment;
+    }
+
+    const response = await fetch(this.#url);
+
+    if (!response.ok) {
+      throw new Error(`Failed to load page from ${this.#url}`);
+    }
+
+    this.#fragment = await PageParser.resolveFragmentFromResponse(response);
+
+    return this.#fragment;
+  }
+
+  clear() {
+    this.#fragment = null;
+  }
+
+  /**
+   * Create a document fragment from the following response.
+   *
+   * @param response Response to create a fragment from. Note, that this response will be used. If you need to use the
+   * same response somewhere else, then you need to pass a cloned version of the response.
+   *
+   * @return Resulting document fragment ready for processing.
+   */
+  static async resolveFragmentFromResponse(response: Response): Promise<DocumentFragment> {
+    const documentFragment = document.createDocumentFragment();
+    const template = document.createElement('template');
+    template.innerHTML = await response.text();
+
+    documentFragment.append(...template.content.childNodes);
+
+    return documentFragment;
+  }
+}
+

src/lib/philomena/scraping/parsing/PostParser.ts

@@ -0,0 +1,82 @@
+import PageParser from "$lib/philomena/scraping/parsing/PageParser";
+import { buildTagsAndAliasesMap } from "$lib/philomena/tag-utils";
+
+export default class PostParser extends PageParser {
+  #tagEditorForm: HTMLFormElement | null = null;
+
+  constructor(imageId: number) {
+    super(`/images/${imageId}`);
+  }
+
+  async resolveTagEditorForm(): Promise<HTMLFormElement> {
+    if (this.#tagEditorForm) {
+      return this.#tagEditorForm;
+    }
+
+    const documentFragment = await this.resolveFragment();
+    const tagsFormElement = documentFragment.querySelector<HTMLFormElement>("#tags-form");
+
+    if (!tagsFormElement) {
+      throw new Error("Failed to find the tag editor form");
+    }
+
+    this.#tagEditorForm = tagsFormElement;
+
+    return tagsFormElement;
+  }
+
+  async resolveTagEditorFormData() {
+    return new FormData(
+      await this.resolveTagEditorForm()
+    );
+  }
+
+  /**
+   * Resolve the tags and aliases mapping from the post page.
+   */
+  async resolveTagsAndAliases(): Promise<Map<string, string> | null> {
+    return PostParser.resolveTagsAndAliasesFromPost(
+      await this.resolveFragment()
+    );
+  }
+
+  /**
+   * Resolve the list of tags and aliases from the post content.
+   *
+   * @param documentFragment Real content to parse the data from.
+   *
+   * @return Tags and aliases or null if failed to parse.
+   */
+  static resolveTagsAndAliasesFromPost(documentFragment: DocumentFragment): Map<string, string> | null {
+    const imageShowContainer = documentFragment.querySelector<HTMLElement>('.image-show-container');
+    const tagsForm = documentFragment.querySelector<HTMLFormElement>('#tags-form');
+
+    if (!imageShowContainer || !tagsForm) {
+      return null;
+    }
+
+    const tagsFormData = new FormData(tagsForm);
+    const tagsAndAliasesValue = imageShowContainer.dataset.imageTagAliases;
+    const tagsValue = tagsFormData.get(this.tagsInputName);
+
+    if (!tagsAndAliasesValue || !tagsValue || typeof tagsValue !== 'string') {
+      console.warn('Failed to locate tags & aliases!');
+      return null;
+    }
+
+    const tagsAndAliasesList = tagsAndAliasesValue
+      .split(',')
+      .map(tagName => tagName.trim());
+
+    const actualTagsList = tagsValue
+      .split(',')
+      .map(tagName => tagName.trim());
+
+    return buildTagsAndAliasesMap(
+      tagsAndAliasesList,
+      actualTagsList,
+    );
+  }
+
+  static tagsInputName = 'image[tag_input]';
+}

src/lib/philomena/search/QueryLexer.ts

@@ -0,0 +1,248 @@
+export class Token {
+  readonly index: number;
+  readonly value: string;
+
+  constructor(index: number, value: string) {
+    this.index = index;
+    this.value = value;
+  }
+}
+
+
+export class AndToken extends Token {
+}
+
+export class NotToken extends Token {
+}
+
+export class OrToken extends Token {
+}
+
+export class GroupStartToken extends Token {
+}
+
+export class GroupEndToken extends Token {
+}
+
+export class BoostToken extends Token {
+}
+
+export class QuotedTermToken extends Token {
+  readonly #quotedValue: string;
+
+  constructor(index: number, value: string, quotedValue: string) {
+    super(index, value);
+
+    this.#quotedValue = quotedValue;
+  }
+
+  get decodedValue() {
+    return QuotedTermToken.decode(this.#quotedValue);
+  }
+
+  static decode(value: string): string {
+    return value.replace(/\\([\\"])/g, "$1");
+  }
+
+  static encode(value: string): string {
+    return value.replace(/[\\"]/g, "\\$&");
+  }
+}
+
+export class TermToken extends Token {
+}
+
+type MatchResultCarry = {
+  match?: RegExpMatchArray | null
+}
+
+/**
+ * Search query tokenizer. Should mostly work for the cases of parsing and finding the selected term for
+ * auto-completion. Follows the rules described in the Philomena booru engine.
+ */
+export class QueryLexer {
+  /**
+   * The original value to be parsed.
+   */
+  readonly #value: string;
+
+  /**
+   * Current position of the parser in the value.
+   */
+  #index: number = 0;
+
+  constructor(value: string) {
+    this.#value = value;
+  }
+
+  /**
+   * Parse the query and get the list of tokens.
+   *
+   * @return List of tokens.
+   */
+  parse(): Token[] {
+    const tokens: Token[] = [];
+    const result: MatchResultCarry = {};
+
+    let dirtyText: string;
+
+    while (this.#index < this.#value.length) {
+      if (this.#value[this.#index] === QueryLexer.#commaCharacter) {
+        tokens.push(new AndToken(this.#index, this.#value[this.#index]));
+        this.#index++;
+        continue;
+      }
+
+      if (this.#match(QueryLexer.#negotiationOperator, result)) {
+        tokens.push(new NotToken(this.#index, result.match![0]));
+        this.#index += result.match![0].length;
+        continue;
+      }
+
+      if (this.#match(QueryLexer.#andOperator, result)) {
+        tokens.push(new AndToken(this.#index, result.match![0]));
+        this.#index += result.match![0].length;
+        continue;
+      }
+
+      if (this.#match(QueryLexer.#orOperator, result)) {
+        tokens.push(new OrToken(this.#index, result.match![0]));
+        this.#index += result.match![0].length;
+        continue;
+      }
+
+      if (this.#match(QueryLexer.#notOperator, result)) {
+        tokens.push(new NotToken(this.#index, result.match![0]));
+        this.#index += result.match![0].length;
+        continue;
+      }
+
+      if (this.#value[this.#index] === QueryLexer.#bracketsOpenCharacter) {
+        tokens.push(new GroupStartToken(this.#index, this.#value[this.#index]));
+        this.#index++;
+        continue;
+      }
+
+      if (this.#value[this.#index] === QueryLexer.#bracketsCloseCharacter) {
+        tokens.push(new GroupEndToken(this.#index, this.#value[this.#index]));
+        this.#index++;
+        continue;
+      }
+
+      if (this.#match(QueryLexer.#boostOperator, result)) {
+        tokens.push(new BoostToken(this.#index, result.match![0]));
+        this.#index += result.match![0].length;
+        continue;
+      }
+
+      if (this.#match(QueryLexer.#whitespaces, result)) {
+        this.#index += result.match![0].length;
+        continue;
+      }
+
+      if (this.#match(QueryLexer.#quotedText, result)) {
+        tokens.push(new QuotedTermToken(this.#index, result.match![0], result.match![1]));
+        this.#index += result.match![0].length;
+        continue;
+      }
+
+      dirtyText = this.#parseDirtyText(this.#index);
+
+      if (dirtyText) {
+        tokens.push(new TermToken(this.#index, dirtyText));
+        this.#index += dirtyText.length;
+        continue;
+      }
+
+      break;
+    }
+
+    return tokens;
+  }
+
+  /**
+   * Match the provided regular expression on the string with the current parser position.
+   *
+   * @param targetRegExp Target RegExp to parse with.
+   * @param [resultCarrier] Object for passing the results into.
+   *
+   * @return Is there a match?
+   */
+  #match(targetRegExp: RegExp, resultCarrier: MatchResultCarry = {}): boolean {
+    return this.#matchAt(targetRegExp, this.#index, resultCarrier);
+  }
+
+  /**
+   * Match the provided regular expression in the string with the specific index.
+   *
+   * @param targetRegExp Target RegExp to parse with.
+   * @param index Index to match the expression from.
+   * @param [resultCarrier] Object for passing the results into.
+   *
+   * @return Is there a match?
+   */
+  #matchAt(targetRegExp: RegExp, index: number, resultCarrier: MatchResultCarry = {}): boolean {
+    targetRegExp.lastIndex = index;
+    resultCarrier.match = this.#value.match(targetRegExp);
+
+    return resultCarrier.match !== null;
+  }
+
+  /**
+   * Parse the dirty text.
+   *
+   * @param {number} index Index to start the parsing from.
+   *
+   * @return {string} Matched text.
+   */
+  #parseDirtyText(index: number): string {
+    let resultValue: string = '';
+
+    const result: MatchResultCarry = {match: null};
+
+    // Loop over
+    while (index < this.#value.length) {
+      // If the stop word found then return the value.
+      if (this.#matchAt(QueryLexer.#dirtyTextStopWords, index)) {
+        break;
+      }
+
+      if (this.#matchAt(QueryLexer.#dirtyTextContent, index, result)) {
+        resultValue += result.match![0];
+        index += result.match![0].length;
+        continue;
+      }
+
+      if (this.#value[index] === QueryLexer.#bracketsOpenCharacter) {
+        let bracketsContent = QueryLexer.#bracketsOpenCharacter + this.#parseDirtyText(index + 1);
+
+        if (this.#value[index + bracketsContent.length + 1] === QueryLexer.#bracketsCloseCharacter) {
+          bracketsContent += QueryLexer.#bracketsCloseCharacter;
+        }
+
+        // There could be an error about brackets not being open
+
+        resultValue += bracketsContent;
+        index += bracketsContent.length;
+        continue;
+      }
+
+      break;
+    }
+
+    return resultValue;
+  }
+
+  static #commaCharacter = ',';
+  static #negotiationOperator = /[!-]/y;
+  static #andOperator = /\s+(?:AND|&&)\s+/y;
+  static #orOperator = /\s+(?:OR|\|\|)\s+/y;
+  static #notOperator = /NOT\s+/y;
+  static #bracketsOpenCharacter = "(";
+  static #bracketsCloseCharacter = ")";
+  static #boostOperator = /\^[+-]?\d+(?:\.\d+)?/y;
+  static #whitespaces = /\s+/y;
+  static #quotedText = /"((?:\\.|[^\\"])+)"/y;
+  static #dirtyTextStopWords = /,|\s+(?:AND|&&|OR|\|\|)\s+|\s+(?:\)|\^[+-]?\d+(?:\.\d+)?)/y;
+  static #dirtyTextContent = /\\.|[^()]/y;
+}

src/lib/philomena/tag-utils.ts

@@ -0,0 +1,118 @@
+import { namespaceCategories } from "$config/tags";
+import { QueryLexer, QuotedTermToken, TermToken } from "$lib/philomena/search/QueryLexer";
+
+/**
+ * Build the map containing both real tags and their aliases.
+ *
+ * @param realAndAliasedTags List combining aliases and tag names.
+ * @param realTags List of actual tag names, excluding aliases.
+ *
+ * @return Map where key is a tag or alias and value is an actual tag name.
+ */
+export function buildTagsAndAliasesMap(realAndAliasedTags: string[], realTags: string[]): Map<string, string> {
+  const tagsAndAliasesMap: Map<string, string> = new Map();
+
+  for (const tagName of realTags) {
+    tagsAndAliasesMap.set(tagName, tagName);
+  }
+
+  let realTagName: string | null = null;
+
+  for (const tagNameOrAlias of realAndAliasedTags) {
+    if (tagsAndAliasesMap.has(tagNameOrAlias)) {
+      realTagName = tagNameOrAlias;
+      continue;
+    }
+
+    if (!realTagName) {
+      console.warn('No real tag found for the alias:', tagNameOrAlias);
+      continue;
+    }
+
+    tagsAndAliasesMap.set(tagNameOrAlias, realTagName);
+  }
+
+  return tagsAndAliasesMap;
+}
+
+const tagLinkRegExp = /\/tags\/(?<encodedTagName>[^/?#]+)/;
+
+/**
+ * List of encoded characters from Philomena.
+ *
+ * @see https://github.com/philomena-dev/philomena/blob/6086757b654da8792ae52adb2a2f501ea6c30d12/lib/philomena/slug.ex#L52-L57
+ */
+const slugEncodedCharacters: Map<string, string> = new Map([
+  ['-dash-', '-'],
+  ['-fwslash-', '/'],
+  ['-bwslash-', '\\'],
+  ['-colon-', ':'],
+  ['-dot-', '.'],
+  ['-plus-', '+'],
+]);
+
+/**
+ * Try to parse the tag name from the search query URL. It uses the same tokenizer as the booru. It only returns the
+ * tag name if query contains only one single tag without any additional conditions.
+ *
+ * @param searchLink Link with search query.
+ *
+ * @return Tag name or NULL if query contains more than 1 tag or doesn't have any tags at all.
+ */
+function parseTagNameFromSearchQuery(searchLink: URL): string | null {
+  const lexer = new QueryLexer(searchLink.searchParams.get('q') || '');
+  const parsedQuery = lexer.parse();
+
+  if (parsedQuery.length !== 1) {
+    return null;
+  }
+
+  const [token] = parsedQuery;
+
+  switch (true) {
+    case token instanceof TermToken:
+      return token.value;
+    case token instanceof QuotedTermToken:
+      return token.decodedValue;
+  }
+
+  return null;
+}
+
+/**
+ * Decode the tag name from the following link.
+ *
+ * @param tagLink Search link or link to the tag to parse the tag name from.
+ *
+ * @return Tag name or NULL if function is failed to parse the name of the tag.
+ */
+export function resolveTagNameFromLink(tagLink: URL): string | null {
+  if (tagLink.pathname.startsWith('/search') && tagLink.searchParams.has('q')) {
+    return parseTagNameFromSearchQuery(tagLink);
+  }
+
+  tagLinkRegExp.lastIndex = 0;
+
+  const result = tagLinkRegExp.exec(tagLink.pathname);
+  const encodedTagName = result?.groups?.encodedTagName;
+
+  if (!encodedTagName) {
+    return null;
+  }
+
+  return decodeURIComponent(encodedTagName)
+    .replaceAll(/-[a-z]+-/gi, match => slugEncodedCharacters.get(match) ?? match)
+    .replaceAll('-', ' ')
+    .replaceAll('+', ' ');
+}
+
+/**
+ * Try to resolve the category from the tag name.
+ *
+ * @param tagName Name of the tag.
+ */
+export function resolveTagCategoryFromTagName(tagName: string): string | null {
+  const namespace = tagName.split(':')[0];
+
+  return namespaceCategories.get(namespace) ?? null;
+}