Fix redirects during partial Learn ingest (#2985)

Author: ktsaou

ingest/autogenerateRedirects.py

@@ -138,13 +138,27 @@ def readLegacyLearnDocMap(pathToFile):
 		return ({key.replace("https://learn.netdata.cloud", ""): value for key, value in json.load(json_file).items()})
 
 
-def UpdateGHLinksBasedOnMap(mapMatrix, inputDictionary):
+def _github_repository_identity(url):
+	parts = urlsplit(url)
+	if parts.scheme not in {"http", "https"} or parts.netloc.lower() != "github.com":
+		return None
+	segments = [segment for segment in parts.path.split("/") if segment]
+	if len(segments) < 2:
+		return None
+	return f"{segments[0]}/{segments[1]}".lower()
+
+
+def UpdateGHLinksBasedOnMap(
+	mapMatrix, inputDictionary, ignored_github_repositories=()
+):
+	ignored = {repository.lower() for repository in ignored_github_repositories}
+	updated = {}
 	for k, v in inputDictionary.items():
 		if v in mapMatrix.keys():
-			inputDictionary[k] = mapMatrix[v]
-		else:
-			pass
-	return (inputDictionary)
+			updated[k] = mapMatrix[v]
+		elif _github_repository_identity(v) not in ignored:
+			updated[k] = v
+	return (updated)
 
 
 def addMovedRedirects(mapping):
@@ -367,22 +381,23 @@ def refresh_current_netlify_config():
 	return redirects
 
 
-def main(GHLinksCorrelation):
+def main(GHLinksCorrelation, ignored_github_repositories=()):
 
 	mapping = reductTonew_learn_pathFromGHLinksCorrelation(GHLinksCorrelation)
 	append_entries_to_json(addMovedRedirects(mapping))
 	# print(GHLinksCorrelation)
 	oldLearn = readLegacyLearnDocMap("LegacyLearnCorrelateLinksWithGHURLs.json")
 	# print(oldLearn)
-	oldLearn_redirects = UpdateGHLinksBasedOnMap(mapping, oldLearn)
+	oldLearn_redirects = UpdateGHLinksBasedOnMap(
+		mapping,
+		oldLearn,
+		ignored_github_repositories=ignored_github_repositories,
+	)
 	# print(mapping)
 
 	# print(oldLearn)
-	try:
-		finalDict = combineDictsOverwrite(readRedirectsFromFile("netlify.toml"), oldLearn_redirects)
-		# print(finalDict)
-	except Exception as e:
-		print(f"An exception occurred: {e}")
+	finalDict = combineDictsOverwrite(readRedirectsFromFile("netlify.toml"), oldLearn_redirects)
+	# print(finalDict)
 
 	active_routes = set(mapping.values())
 	finalDict = clean_redirects(finalDict, active_routes)

ingest/ingest.py

@@ -3994,7 +3994,13 @@ def regenerate_grids_only(
     for md_file in to_publish:
         convert_github_links(file_dict[md_file]["learnPath"], file_dict)
 
-    genRedirects.main(file_dict)
+    ignored_redirect_repositories = set()
+    if IGNORE_ON_PREM_REPO:
+        ignored_redirect_repositories.add("netdata/netdata-cloud-onprem")
+    genRedirects.main(
+        file_dict,
+        ignored_github_repositories=ignored_redirect_repositories,
+    )
     print(
         "Done.",
         "Uncorrelated links (links from our github repos that the files are not in Learn):",

ingest/test_redirect_cleanup.py

@@ -1,4 +1,5 @@
 import unittest
+from unittest import mock
 
 import autogenerateRedirects as redirects
 
@@ -76,5 +77,89 @@ def test_merge_preserves_external_host_identity(self):
                 {"/old/": "https://example.net/release"},
             )
 
+    def test_ignored_repository_removes_only_its_unresolved_legacy_redirect(self):
+        legacy = {
+            "/on-prem": (
+                "https://github.com/netdata/netdata-cloud-onprem/"
+                "blob/master/docs/page.md"
+            ),
+            "/agent": "https://github.com/netdata/netdata/blob/master/docs/page.md",
+        }
+        self.assertEqual(
+            redirects.UpdateGHLinksBasedOnMap(
+                {},
+                legacy,
+                ignored_github_repositories={"netdata/netdata-cloud-onprem"},
+            ),
+            {"/agent": "https://github.com/netdata/netdata/blob/master/docs/page.md"},
+        )
+
+    def test_non_ignored_unresolved_legacy_redirect_is_preserved(self):
+        legacy = {
+            "/on-prem": (
+                "https://github.com/netdata/netdata-cloud-onprem/"
+                "blob/master/docs/page.md"
+            )
+        }
+        self.assertEqual(redirects.UpdateGHLinksBasedOnMap({}, legacy), legacy)
+
+    def test_ignored_on_prem_mapping_preserves_the_tracked_internal_redirect(self):
+        source = "/docs/netdata-cloud-on-prem/light-poc-deployment"
+        tracked = {source: "/docs/netdata-cloud-on-prem/poc-without-k8s"}
+        legacy = {
+            source: (
+                "https://github.com/netdata/netdata-cloud-onprem/"
+                "blob/master/docs/learn.netdata.cloud/poc-without-k8s.md"
+            )
+        }
+        filtered = redirects.UpdateGHLinksBasedOnMap(
+            {},
+            legacy,
+            ignored_github_repositories={"netdata/netdata-cloud-onprem"},
+        )
+        self.assertEqual(redirects.combineDictsOverwrite(tracked, filtered), tracked)
+        with self.assertRaisesRegex(ValueError, "Conflicting redirect identity"):
+            redirects.combineDictsOverwrite(
+                tracked,
+                redirects.UpdateGHLinksBasedOnMap({}, legacy),
+            )
+
+    def test_current_mapping_wins_even_when_its_repository_is_ignored(self):
+        github_url = (
+            "https://github.com/netdata/netdata-cloud-onprem/"
+            "blob/master/docs/page.md"
+        )
+        self.assertEqual(
+            redirects.UpdateGHLinksBasedOnMap(
+                {github_url: "/docs/current"},
+                {"/old": github_url},
+                ignored_github_repositories={"netdata/netdata-cloud-onprem"},
+            ),
+            {"/old": "/docs/current"},
+        )
+
+    def test_main_does_not_hide_redirect_conflicts(self):
+        with (
+            mock.patch.object(
+                redirects,
+                "reductTonew_learn_pathFromGHLinksCorrelation",
+                return_value={},
+            ),
+            mock.patch.object(redirects, "addMovedRedirects", return_value={}),
+            mock.patch.object(redirects, "append_entries_to_json"),
+            mock.patch.object(
+                redirects,
+                "readLegacyLearnDocMap",
+                return_value={"/old": "/second"},
+            ),
+            mock.patch.object(
+                redirects,
+                "readRedirectsFromFile",
+                return_value={"/old": "/first"},
+            ),
+        ):
+            with self.assertRaisesRegex(ValueError, "Conflicting redirect identity /old"):
+                redirects.main({})
+
 if __name__ == "__main__":
     unittest.main()