[do not merge] test-run

add-on/src/lib/ipfs-request.js

@@ -188,7 +188,7 @@ export function createRequestModifier (getState, dnslinkResolver, ipfsPathValida
       }
       // poor-mans protocol handlers - https://github.com/ipfs/ipfs-companion/issues/164#issuecomment-328374052
       if (state.catchUnhandledProtocols && mayContainUnhandledIpfsProtocol(request)) {
-        const fix = normalizedUnhandledIpfsProtocol(request, state.pubGwURLString)
+        const fix = await normalizedUnhandledIpfsProtocol(request, state.pubGwURLString)
         if (fix) {
           return fix
         }
@@ -485,15 +485,15 @@ export function createRequestModifier (getState, dnslinkResolver, ipfsPathValida
  * @param {object} input contains originUrl and redirectUrl.
  * @returns
  */
-function handleRedirection ({ originUrl, redirectUrl, request }) {
+async function handleRedirection ({ originUrl, redirectUrl, request }) {
   if (redirectUrl !== '' && originUrl !== '' && redirectUrl !== originUrl) {
     resolvedRequestTracker.track(request)
-    if (supportsBlock) {
+    if (supportsBlock()) {
       return { redirectUrl }
     }
 
     // Let browser handle redirection MV3 style.
-    addRuleToDynamicRuleSet({ originUrl, redirectUrl })
+    await addRuleToDynamicRuleSet({ originUrl, redirectUrl })
   }
 }
 

add-on/src/lib/redirect-handler/baseRegexFilter.ts

@@ -0,0 +1,104 @@
+export interface IRegexFilter {
+  originUrl: string
+  redirectUrl: string
+}
+
+export interface IFilter {
+  regexFilter: string
+  regexSubstitution: string
+}
+
+/**
+ * Base class for all regex filters.
+ */
+export class RegexFilter {
+  readonly _redirectUrl!: string
+  readonly _originUrl!: string
+  readonly originURL: URL
+  readonly redirectURL: URL
+  readonly originNS: string
+  readonly redirectNS: string
+  // by default we cannot handle the request.
+  _canHandle = false
+  regexFilter!: string
+  regexSubstitution!: string
+
+  constructor ({ originUrl, redirectUrl }: IRegexFilter) {
+    this._originUrl = originUrl
+    this._redirectUrl = redirectUrl
+    this.originURL = new URL(this._originUrl)
+    this.redirectURL = new URL(this._redirectUrl)
+    this.redirectNS = this.computeNamespaceFromUrl(this.redirectURL)
+    this.originNS = this.computeNamespaceFromUrl(this.originURL)
+    this.computeFilter()
+    this.normalizeRegexFilter()
+  }
+
+  /**
+   * Getter for the originUrl provided at construction.
+   */
+  get originUrl (): string {
+    return this._originUrl
+  }
+
+  /**
+   * Getter for the redirectUrl provided at construction.
+   */
+  get redirectUrl (): string {
+    return this._redirectUrl
+  }
+
+  /**
+   * Getter for the canHandle flag.
+   */
+  get canHandle (): boolean {
+    return this._canHandle
+  }
+
+  /**
+   * Setter for the canHandle flag.
+   */
+  set canHandle (value: boolean) {
+    this._canHandle = value
+  }
+
+  /**
+   * Getter for the filter. This is the regex filter and substitution.
+   */
+  get filter (): IFilter {
+    if (!this.canHandle) {
+      throw new Error('Cannot handle this request')
+    }
+
+    return {
+      regexFilter: this.regexFilter,
+      regexSubstitution: this.regexSubstitution
+    }
+  }
+
+  /**
+   * Compute the regex filter and substitution.
+   * This is the main method that needs to be implemented by subclasses.
+   */
+  computeFilter (): void {
+    throw new Error('Method not implemented.')
+  }
+
+  /**
+   * Normalize the regex filter. This is a helper method that can be used by subclasses.
+   */
+  normalizeRegexFilter (): void {
+    this.regexFilter = this.regexFilter.replace(/https?\??/ig, 'https?')
+  }
+
+  /**
+   * Compute the namespace from the URL. This finds the first path segment.
+   * e.g. http://<gateway>/<namespace>/path/to/file/or/cid
+   *
+   * @param url URL
+   */
+  computeNamespaceFromUrl ({ pathname }: URL): string {
+    // regex to match the first path segment.
+    return (/\/([^/]+)\//i.exec(pathname)?.[1] ?? '').toLowerCase()
+  }
+}

add-on/src/lib/redirect-handler/blockOrObserve.ts

@@ -1,16 +1,24 @@
 import debug from 'debug'
 import browser from 'webextension-polyfill'
 import { CompanionState } from '../../types/companion.js'
+import { SubdomainRedirectRegexFilter } from './subdomainRedirectRegexFilter.js'
+import { NamespaceRedirectRegexFilter } from './namespaceRedirectRegexFilter.js'
+import { IFilter, IRegexFilter, RegexFilter } from './baseRegexFilter.js'
+import { CommonPatterRedirectRegexFilter } from './commonPatternRedirectRegexFilter.js'
 
 // this won't work in webworker context. Needs to be enabled manually
 // https://github.com/debug-js/debug/issues/916
 const log = debug('ipfs-companion:redirect-handler:blockOrObserve')
 log.error = debug('ipfs-companion:redirect-handler:blockOrObserve:error')
 
+export const DEFAULT_NAMESPACES = new Set(['ipfs', 'ipns'])
+
 export const GLOBAL_STATE_CHANGE = 'GLOBAL_STATE_CHANGE'
 export const GLOBAL_STATE_OPTION_CHANGE = 'GLOBAL_STATE_OPTION_CHANGE'
 export const DELETE_RULE_REQUEST = 'DELETE_RULE_REQUEST'
 export const DELETE_RULE_REQUEST_SUCCESS = 'DELETE_RULE_REQUEST_SUCCESS'
+
+// We need to match the rest of the URL, so we can use a wildcard.
 export const RULE_REGEX_ENDING = '((?:[^\\.]|$).*)$'
 
 interface regexFilterMap {
@@ -21,6 +29,7 @@ interface regexFilterMap {
 interface redirectHandlerInput {
   originUrl: string
   redirectUrl: string
+  getPort: (state: CompanionState) => string
 }
 
 type messageToSelfType = typeof GLOBAL_STATE_CHANGE | typeof GLOBAL_STATE_OPTION_CHANGE | typeof DELETE_RULE_REQUEST
@@ -29,10 +38,16 @@ interface messageToSelf {
   value?: string | Record<string, unknown>
 }
 
+export const defaultNSRegexStr = `(${[...DEFAULT_NAMESPACES].join('|')})`
+
 // We need to check if the browser supports the declarativeNetRequest API.
 // TODO: replace with check for `Blocking` in `chrome.webRequest.OnBeforeRequestOptions`
 // which is currently a bug https://bugs.chromium.org/p/chromium/issues/detail?id=1427952
-export const supportsBlock = !(browser.declarativeNetRequest?.MAX_NUMBER_OF_DYNAMIC_AND_SESSION_RULES === 5000)
+// this needs to be a function call, because in tests we mock browser.declarativeNetRequest
+// the way sinon ends up stubbing it, it's not directly available in the global scope on import
+// rather it gets replaced dynamically when the module is imported. Which means, we can't
+// just check for the existence of the property, we need to call the browser instance at that point.
+export const supportsBlock = (): boolean => !(browser.declarativeNetRequest?.MAX_NUMBER_OF_DYNAMIC_AND_SESSION_RULES > 0)
 
 /**
  * Notify self about state change.
@@ -61,21 +76,28 @@ export async function notifyDeleteRule (id: number): Promise<void> {
  */
 async function sendMessageToSelf (msg: messageToSelfType, value?: any): Promise<void> {
   // this check ensures we don't send messages to ourselves if blocking mode is enabled.
-  if (!supportsBlock) {
-    const message: messageToSelf = { type: msg, value }
-    await browser.runtime.sendMessage({ message })
+  if (!supportsBlock()) {
+    const message: messageToSelf = { type: msg }
+    await browser.runtime.sendMessage(message)
   }
 }
 
 const savedRegexFilters: Map<string, regexFilterMap> = new Map()
 const DEFAULT_LOCAL_RULES: redirectHandlerInput[] = [
   {
     originUrl: 'http://127.0.0.1',
-    redirectUrl: 'http://localhost'
+    redirectUrl: 'http://localhost',
+    getPort: ({ gwURLString }): string => new URL(gwURLString).port
   },
   {
     originUrl: 'http://[::1]',
-    redirectUrl: 'http://localhost'
+    redirectUrl: 'http://localhost',
+    getPort: ({ gwURLString }): string => new URL(gwURLString).port
+  },
+  {
+    originUrl: 'http://localhost',
+    redirectUrl: 'http://127.0.0.1',
+    getPort: ({ apiURL }): string => new URL(apiURL).port
   }
 ]
 
@@ -97,10 +119,10 @@ export function isLocalHost (url: string): boolean {
  * @param str URL string to escape
  * @returns
  */
-function escapeURLRegex (str: string): string {
+export function escapeURLRegex (str: string): string {
   // these characters are allowed in the URL, but not in the regex.
   // eslint-disable-next-line no-useless-escape
-  const ALLOWED_CHARS_URL_REGEX = /([:\/\?#\[\]@!$&'\(\ )\*\+,;=-_\.~])/g
+  const ALLOWED_CHARS_URL_REGEX = /([:\/\?#\[\]@!$&'\(\ )\*\+,;=\-_\.~])/g
   return str.replace(ALLOWED_CHARS_URL_REGEX, '\\$1')
 }
 
@@ -111,43 +133,29 @@ function escapeURLRegex (str: string): string {
  * @param redirectUrl
  * @returns
  */
-function constructRegexFilter ({ originUrl, redirectUrl }: redirectHandlerInput): {
-  regexSubstitution: string
-  regexFilter: string
-} {
-  // We can traverse the URL from the end, and find the first character that is different.
-  let commonIdx = 1
-  while (commonIdx < Math.min(originUrl.length, redirectUrl.length)) {
-    if (originUrl[originUrl.length - commonIdx] !== redirectUrl[redirectUrl.length - commonIdx]) {
-      break
+function constructRegexFilter ({ originUrl, redirectUrl }: IRegexFilter): IFilter {
+  // the order is very important here, because we want to match the best possible filter.
+  const filtersToTryInOrder: Array<typeof RegexFilter> = [
+    SubdomainRedirectRegexFilter,
+    NamespaceRedirectRegexFilter,
+    CommonPatterRedirectRegexFilter
+  ]
+
+  for (const Filter of filtersToTryInOrder) {
+    const filter = new Filter({ originUrl, redirectUrl })
+    if (filter.canHandle) {
+      return filter.filter
     }
-    commonIdx += 1
-  }
-
-  // We can now construct the regex filter and substitution.
-  let regexSubstitution = redirectUrl.slice(0, redirectUrl.length - commonIdx + 1) + '\\1'
-  // We need to escape the characters that are allowed in the URL, but not in the regex.
-  const regexFilterFirst = escapeURLRegex(originUrl.slice(0, originUrl.length - commonIdx + 1))
-  // We need to match the rest of the URL, so we can use a wildcard.
-  const RULE_REGEX_ENDING = '((?:[^\\.]|$).*)$'
-  let regexFilter = `^${regexFilterFirst}${RULE_REGEX_ENDING}`.replace(/https?/ig, 'https?')
-
-  // This method does not parse:
-  // originUrl: "https://awesome.ipfs.io/"
-  // redirectUrl: "http://localhost:8081/ipns/awesome.ipfs.io/"
-  // that ends up with capturing all urls which we do not want.
-  if (regexFilter === `^https?\\:\\/${RULE_REGEX_ENDING}`) {
-    const subdomain = new URL(originUrl).hostname
-    regexFilter = `^https?\\:\\/\\/${escapeURLRegex(subdomain)}${RULE_REGEX_ENDING}`
-    regexSubstitution = regexSubstitution.replace('\\1', `/${subdomain}\\1`)
   }
 
-  return { regexSubstitution, regexFilter }
+  // this is just to satisfy the compiler, this should never happen. Because CommonPatterRedirectRegexFilter can always
+  // handle.
+  return new CommonPatterRedirectRegexFilter({ originUrl, redirectUrl }).filter
 }
 
 // If the browser supports the declarativeNetRequest API, we can block the request.
 export function getExtraInfoSpec<T> (additionalParams: T[] = []): T[] {
-  if (supportsBlock) {
+  if (supportsBlock()) {
     return ['blocking' as T, ...additionalParams]
   }
   return additionalParams
@@ -239,15 +247,15 @@ async function reconcileRulesAndRemoveOld (state: CompanionState): Promise<void>
     if (rules.length === 0) {
       // we need to populate old rules.
       for (const [regexFilter, { regexSubstitution, id }] of savedRegexFilters.entries()) {
-        addRules.push(generateRule(id, regexFilter, regexSubstitution))
+        addRules.push(generateAddRule(id, regexFilter, regexSubstitution))
       }
     }
 
     // make sure that the default rules are added.
-    for (const { originUrl, redirectUrl } of DEFAULT_LOCAL_RULES) {
-      const { port } = new URL(state.gwURLString)
-      const regexFilter = `^${escapeURLRegex(`${originUrl}:${port}`)}(.*)$`
-      const regexSubstitution = `${redirectUrl}:${port}\\1`
+    for (const { originUrl, redirectUrl, getPort } of DEFAULT_LOCAL_RULES) {
+      const port = getPort(state)
+      const regexFilter = `^${escapeURLRegex(`${originUrl}:${port}`)}\\/${defaultNSRegexStr}\\/${RULE_REGEX_ENDING}`
+      const regexSubstitution = `${redirectUrl}:${port}/\\1/\\2`
 
       if (!savedRegexFilters.has(regexFilter)) {
         // We need to add the new rule.
@@ -276,7 +284,7 @@ function saveAndGenerateRule (
   const id = Math.floor(Math.random() * 29999)
   // We need to save the regex filter and ID to check if the rule already exists later.
   savedRegexFilters.set(regexFilter, { id, regexSubstitution })
-  return generateRule(id, regexFilter, regexSubstitution, excludedInitiatorDomains)
+  return generateAddRule(id, regexFilter, regexSubstitution, excludedInitiatorDomains)
 }
 
 /**
@@ -287,7 +295,7 @@ function saveAndGenerateRule (
  * @param excludedInitiatorDomains - The domains that are excluded from the rule.
  * @returns
  */
-function generateRule (
+export function generateAddRule (
   id: number,
   regexFilter: string,
   regexSubstitution: string,

add-on/src/lib/redirect-handler/commonPatternRedirectRegexFilter.ts

@@ -0,0 +1,40 @@
+import { IRegexFilter, RegexFilter } from './baseRegexFilter.js'
+import { RULE_REGEX_ENDING, escapeURLRegex } from './blockOrObserve.js'
+
+export class CommonPatterRedirectRegexFilter extends RegexFilter {
+  constructor ({ originUrl, redirectUrl }: IRegexFilter) {
+    super({ originUrl, redirectUrl })
+
+    // this filter is the worst case scenario, we can handle any redirect.
+    this._canHandle = true
+  }
+
+  computeFilter (): void {
+    // We can traverse the URL from the end, and find the first character that is different.
+    let commonIdx = 1
+    while (commonIdx < Math.min(this.originUrl.length, this.redirectUrl.length)) {
+      if (this.originUrl[this.originUrl.length - commonIdx] !== this.redirectUrl[this.redirectUrl.length - commonIdx]) {
+        break
+      }
+      commonIdx += 1
+    }
+
+    // We can now construct the regex filter and substitution.
+    this.regexSubstitution = this.redirectUrl.slice(0, this.redirectUrl.length - commonIdx + 1) + '\\1'
+    // We need to escape the characters that are allowed in the URL, but not in the regex.
+    const regexFilterFirst = escapeURLRegex(this.originUrl.slice(0, this.originUrl.length - commonIdx + 1))
+    this.regexFilter = `^${regexFilterFirst}${RULE_REGEX_ENDING}`
+    // calling normalize should add the protocol in the regexFilter.
+    this.normalizeRegexFilter()
+
+    // This method does not parse:
+    // originUrl: "https://awesome.ipfs.io/"
+    // redirectUrl: "http://localhost:8081/ipns/awesome.ipfs.io/"
+    // that ends up with capturing all urls which we do not want.
+    if (this.regexFilter === `^https?\\:\\/${RULE_REGEX_ENDING}`) {
+      const subdomain = new URL(this.originUrl).hostname
+      this.regexFilter = `^https?\\:\\/\\/${escapeURLRegex(subdomain)}${RULE_REGEX_ENDING}`
+      this.regexSubstitution = this.regexSubstitution.replace('\\1', `/${subdomain}\\1`)
+    }
+  }
+}

add-on/src/lib/redirect-handler/namespaceRedirectRegexFilter.ts

@@ -0,0 +1,24 @@
+import { DEFAULT_NAMESPACES, RULE_REGEX_ENDING, defaultNSRegexStr, escapeURLRegex } from './blockOrObserve.js'
+import { IRegexFilter, RegexFilter } from './baseRegexFilter.js'
+
+export class NamespaceRedirectRegexFilter extends RegexFilter {
+  constructor ({ originUrl, redirectUrl }: IRegexFilter) {
+    super({ originUrl, redirectUrl })
+    this._canHandle = DEFAULT_NAMESPACES.has(this.originNS) &&
+      DEFAULT_NAMESPACES.has(this.redirectNS) &&
+      this.originNS === this.redirectNS &&
+      this.originURL.searchParams.get('uri') == null
+  }
+
+  computeFilter (): void {
+    // if the namespaces are the same, we can generate simpler regex.
+    // The only value that needs special handling is the `uri` param.
+    // A redirect like
+    // https://ipfs.io/ipfs/QmZMxU -> http://localhost:8080/ipfs/QmZMxU
+    const [originFirst, originLast] = this.originUrl.split(`/${this.originNS}/`)
+    this.regexFilter = `^${escapeURLRegex(originFirst)}\\/${defaultNSRegexStr}\\/${RULE_REGEX_ENDING}`
+    this.regexSubstitution = this.redirectUrl
+      .replace(`/${this.redirectNS}/`, '/\\1/')
+      .replace(originLast, '\\2')
+  }
+}