Bug 2067852 - Migrate rst docs to markdown and host inside of bugzilla itself

.github/workflows/deploy.yml

@@ -32,7 +32,7 @@ jobs:
           docker cp push_data:/app/version.json build_info/version.json
           docker rm push_data
       - name: Upload artifacts
-        uses: actions/upload-artifact@v4
+        uses: actions/upload-artifact@v7
         with:
           name: Build Info
           path: build_info

.github/workflows/perl-slim.yml

@@ -28,7 +28,7 @@ jobs:
         docker run -v $(pwd):/app/result bmo-cpanfile cp cpanfile cpanfile.snapshot /app/result
         cp cpanfile cpanfile.snapshot build_info
     - name: Upload artifacts
-      uses: actions/upload-artifact@v4
+      uses: actions/upload-artifact@v7
       with:
         name: Build Info
         path: build_info

.vscode/settings.json

@@ -17,6 +17,5 @@
     },
     "search.exclude": {
         "**/local": true
-    },
-    "esbonio.sphinx.confDir": "${workspaceFolder}/docs/en/rst/conf.py"
+    }
 }

Bugzilla.pm

@@ -13,7 +13,7 @@ use warnings;
 
 use Bugzilla::Logging;
 
-our $VERSION = '20260805.1';
+our $VERSION = '20260825.1';
 
 use Bugzilla::Auth;
 use Bugzilla::Auth::Persist::Cookie;

Bugzilla/App/Controller/Docs.pm

@@ -0,0 +1,186 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+#
+# This Source Code Form is "Incompatible With Secondary Licenses", as
+# defined by the Mozilla Public License, v. 2.0.
+
+package Bugzilla::App::Controller::Docs;
+
+use 5.10.1;
+use Mojo::Base 'Mojolicious::Controller';
+
+use Bugzilla::Constants;
+use Cwd qw(realpath);
+use Encode qw(decode);
+use File::Basename qw(basename);
+use Mojo::DOM;
+use Mojo::File ();
+
+# The Markdown documentation lives in docs/en/md and its images in
+# docs/en/images, so the /docs/en URL space mirrors docs/en on disk. That
+# way the relative links inside the converted files (../using/index.md,
+# ../../images/foo.png) resolve in the browser without any rewriting.
+sub _docs_root { realpath(bz_locations()->{libpath} . '/docs/en') }
+
+use constant IMAGE_TYPES => {
+  gif  => 'image/gif',
+  jpeg => 'image/jpeg',
+  jpg  => 'image/jpeg',
+  png  => 'image/png',
+  svg  => 'image/svg+xml',
+};
+
+use constant ALERT_TITLES => {
+  caution   => 'Caution',
+  important => 'Important',
+  note      => 'Note',
+  tip       => 'Tip',
+  warning   => 'Warning',
+};
+
+sub setup_routes {
+  my ($class, $r) = @_;
+  $r->get('/docs')->to('Docs#index')->name('docs_index');
+  $r->get('/docs/en')->to('Docs#index');
+  $r->get('/docs/en/*docs_path')->to('Docs#show')->name('docs_show');
+}
+
+sub index {    ## no critic (ProhibitBuiltinHomonyms)
+  my ($self) = @_;
+  return $self->redirect_to($self->url_for('docs_show', docs_path => 'md/index.md'));
+}
+
+sub show {
+  my ($self) = @_;
+  Bugzilla->usage_mode(USAGE_MODE_MOJO);
+  $self->bugzilla->login || return undef;
+
+  my $path = $self->stash('docs_path') // '';
+  $path =~ s{/+$}{};
+  return $self->index if $path eq '';
+
+  # Be careful not to allow directory traversal.
+  if ($path =~ /\.\./ || $path !~ m{^[\w\-./]+$}) {
+    return $self->_not_found($path);
+  }
+
+  my $root = _docs_root();
+  my $file = realpath("$root/$path");
+  unless (defined $file && CORE::index($file, "$root/") == 0 && (-f $file || -d $file)) {
+    # The docs were once built to Sphinx HTML and served externally
+    # (docs_urlbase), so old links and bookmarks use .html paths; send
+    # those to the Markdown page with the same name.
+    if ($path =~ m{^md/.+\.html$}) {
+      (my $md_path = $path) =~ s/\.html$/.md/;
+      my $md_file = realpath("$root/$md_path");
+      if (defined $md_file && CORE::index($md_file, "$root/") == 0 && -f $md_file) {
+        return $self->redirect_to(
+          $self->url_for('docs_show', docs_path => $md_path));
+      }
+    }
+    return $self->_not_found($path);
+  }
+
+  # Directory URLs (e.g. /docs/en/md, /docs/en/md/using) go to the
+  # section's index page.
+  if (-d $file) {
+    return $self->_not_found($path) unless -f "$file/index.md";
+    return $self->redirect_to(
+      $self->url_for('docs_show', docs_path => "$path/index.md"));
+  }
+
+  if ($path =~ m{^md/.+\.md$}) {
+    return $self->_render_markdown($file, $path);
+  }
+
+  if ($path =~ m{^images/.+\.(\w+)$} && IMAGE_TYPES->{lc $1}) {
+    $self->res->headers->content_type(IMAGE_TYPES->{lc $1});
+    return $self->reply->file($file);
+  }
+
+  return $self->_not_found($path);
+}
+
+sub _not_found {
+  my ($self, $path) = @_;
+  return $self->user_error('docs_page_not_found', {path => $path},
+    {status => 404, skip_exception_page => 1});
+}
+
+sub _render_markdown {
+  my ($self, $file, $path) = @_;
+
+  require Bugzilla::Markdown::GFM;
+  require Bugzilla::Markdown::GFM::Parser;
+
+  my $markdown = Mojo::File->new($file)->slurp;
+
+  # The documentation is trusted content shipped in the repository, so raw
+  # HTML (the API reference tables, the <a id> anchors kept for deep links)
+  # is allowed through; tagfilter still neutralizes script-capable tags.
+  my $parser = Bugzilla::Markdown::GFM::Parser->new({
+    unsafe        => 1,
+    validate_utf8 => 1,
+    extensions    => [qw( autolink tagfilter table strikethrough )],
+  });
+
+  my $dom = Mojo::DOM->new(decode('UTF-8', $parser->render_html($markdown)));
+  _add_heading_ids($dom);
+  _convert_alerts($dom);
+
+  my $h1 = $dom->at('h1');
+  my $title = $h1 ? $h1->all_text : basename($file, '.md');
+
+  $self->stash(
+    doc_html  => $dom->to_string,
+    doc_title => $title,
+    doc_path  => $path,
+  );
+  return $self->render(template => 'pages/doc_viewer', handler => 'bugzilla',
+    format => 'html');
+}
+
+# cmark-gfm does not add ids to headings; GitHub does that in a separate
+# pass. The docs link to GitHub-style heading slugs (lowercase; keep
+# alphanumerics, "_" and "-"; spaces become "-"; everything else is
+# dropped; duplicates get -1, -2, ...), so reproduce that algorithm or
+# in-page anchors would dangle.
+sub _add_heading_ids {
+  my ($dom) = @_;
+  my %seen;
+  $dom->find('h1, h2, h3, h4, h5, h6')->each(sub {
+    my ($h) = @_;
+    return if defined $h->attr('id');
+    my $slug = lc $h->all_text;
+    $slug =~ s/^\s+|\s+$//g;
+    $slug =~ s/[^\w\- \t]//g;
+    $slug =~ s/[ \t]/-/g;
+    my $count = $seen{$slug}++;
+    $slug .= "-$count" if $count;
+    $h->attr(id => $slug);
+  });
+}
+
+# GitHub renders "> [!NOTE]" blockquotes as styled callouts; cmark-gfm
+# leaves the marker as literal text, so turn those blockquotes into styled
+# alert boxes here.
+sub _convert_alerts {
+  my ($dom) = @_;
+  $dom->find('blockquote')->each(sub {
+    my ($bq) = @_;
+    my $p = $bq->at('p') or return;
+    my $first = $p->child_nodes->first;
+    return unless $first && $first->type eq 'text';
+    my $text = $first->content;
+    return unless $text =~ s/^\s*\[!(NOTE|TIP|IMPORTANT|WARNING|CAUTION)\]\s*//;
+    my $kind = lc $1;
+    $first->content($text);
+    $p->remove unless $p->all_text =~ /\S/ || $p->children->size;
+    $bq->attr(class => "docs-alert docs-alert-$kind");
+    $bq->prepend_content(
+      qq{<p class="docs-alert-title">${\ ALERT_TITLES->{$kind}}</p>});
+  });
+}
+
+1;

Bugzilla/App/Controller/MFA/Duo.pm

@@ -63,8 +63,10 @@ sub callback {
 
   # Retrieve the event data from the mfa token
   my $provider = Bugzilla::MFA->new_from($user, 'Duo');
-  my $event
-    = $provider->verify_token($mfa_cookie, {no_redirect => 1, no_delete => 1});
+  # provider_callback skips the duo_verified gate: we are the request that is
+  # about to establish it.
+  my $event = $provider->verify_token($mfa_cookie,
+    {no_redirect => 1, no_delete => 1, provider_callback => 1});
   if (!$event) {
     return $self->code_error('duo_client_error',
       {reason => ERR_INVALID_MFA_COOKIE});

Bugzilla/App/Plugin/Error.pm

@@ -90,7 +90,7 @@ sub _render_error {
       error         => 1,
       code          => $code,
       message       => $message,
-      documentation => 'https://bmo.readthedocs.io/en/latest/api/',
+      documentation => Bugzilla->localconfig->urlbase . 'docs/en/md/api/index.md',
     };
 
     $c->render(json => $error, status => $status_code);

Bugzilla/Config/General.pm

@@ -31,13 +31,6 @@ use constant get_param_list => (
     checker  => \&check_email
   },
 
-  {
-    name    => 'docs_urlbase',
-    type    => 't',
-    default => 'docs/%lang%/html/',
-    checker => \&check_url
-  },
-
   {
     name    => 'utf8',
     type    => 's',

Bugzilla/Install/DB.pm

@@ -847,6 +847,9 @@ sub update_table_definitions {
   # Bug 1806896 - xavier.lhour@gmail.com
   _migrate_flag_state_activity();
 
+  # Bug 2060356 - dkl@mozilla.com
+  _remove_duo_recovery_codes();
+
   ################################################################
   # New --TABLE-- changes should go *** A B O V E *** this point #
   ################################################################
@@ -4538,6 +4541,19 @@ sub _migrate_flag_state_activity {
   $dbh->bz_drop_table('flag_state_activity');
 }
 
+sub _remove_duo_recovery_codes {
+  my $dbh = Bugzilla->dbh;
+
+  # Duo users were able to generate recovery codes but never had a form in
+  # which to enter one, so these rows are unusable secrets. Recovery for Duo
+  # is handled by Duo Security itself.
+  $dbh->do(
+    "DELETE FROM profile_mfa
+      WHERE name LIKE 'recovery.%'
+            AND user_id IN (SELECT userid FROM profiles WHERE mfa = 'Duo')"
+  );
+}
+
 1;
 
 __END__

Bugzilla/Install/Filesystem.pm

@@ -185,8 +185,6 @@ sub FILESYSTEM {
     'cvs-update.log'               => {perms => WS_SERVE},
     'scripts/sendunsentbugmail.pl' => {perms => WS_EXECUTE},
     'docs/bugzilla.ent'            => {perms => OWNER_WRITE},
-    'docs/makedocs.pl'             => {perms => OWNER_EXECUTE},
-    'docs/style.css'               => {perms => WS_SERVE},
     'docs/*/rel_notes.txt'         => {perms => WS_SERVE},
     'docs/*/README.docs'           => {perms => OWNER_WRITE},
     "$datadir/params.old"          => {perms => CGI_WRITE},
@@ -240,9 +238,6 @@ sub FILESYSTEM {
     js                     => {files => WS_SERVE, dirs => DIR_WS_SERVE},
     static                 => {files => WS_SERVE, dirs => DIR_WS_SERVE},
     $skinsdir              => {files => WS_SERVE, dirs => DIR_WS_SERVE},
-    'docs/*/html'          => {files => WS_SERVE, dirs => DIR_WS_SERVE},
-    'docs/*/pdf'           => {files => WS_SERVE, dirs => DIR_WS_SERVE},
-    'docs/*/txt'           => {files => WS_SERVE, dirs => DIR_WS_SERVE},
     'docs/*/images'        => {files => WS_SERVE, dirs => DIR_WS_SERVE},
     "$extensionsdir/*/web" => {files => WS_SERVE, dirs => DIR_WS_SERVE},
     $confdir               => {files => WS_SERVE, dirs => DIR_WS_SERVE,},
@@ -255,7 +250,6 @@ sub FILESYSTEM {
     # Directories only for the owner, not for the webserver.
     t            => {files => OWNER_WRITE,   dirs => DIR_OWNER_WRITE},
     xt           => {files => OWNER_WRITE,   dirs => DIR_OWNER_WRITE},
-    'docs/lib'   => {files => OWNER_WRITE,   dirs => DIR_OWNER_WRITE},
     'docs/*/xml' => {files => OWNER_WRITE,   dirs => DIR_OWNER_WRITE},
     'contrib'    => {files => OWNER_EXECUTE, dirs => DIR_OWNER_WRITE,},
     'scripts'    => {files => OWNER_EXECUTE, dirs => DIR_OWNER_WRITE,},

Bugzilla/Install/Requirements.pm

@@ -83,7 +83,6 @@ use constant FEATURE_FILES => (
   moving        => ['importxml.pl'],
   auth_ldap     => ['Bugzilla/Auth/Verify/LDAP.pm'],
   auth_radius   => ['Bugzilla/Auth/Verify/RADIUS.pm'],
-  documentation => ['docs/makedocs.pl'],
   inbound_email => ['email_in.pl'],
   jobqueue      => [
     'Bugzilla/Job/*',      'Bugzilla/JobQueue.pm',