Introduce nodeplugins/markdown and nodeplugins/downloader

cmd/app/exec.go

@@ -11,14 +11,14 @@ import (
 	"strings"
 	"sync"
 
+	"github.com/gardener/docforge/pkg/core"
 	"github.com/gardener/docforge/pkg/manifest"
+	"github.com/gardener/docforge/pkg/nodeplugins"
+	"github.com/gardener/docforge/pkg/nodeplugins/downloader"
+	"github.com/gardener/docforge/pkg/nodeplugins/markdown"
 	"github.com/gardener/docforge/pkg/osfakes/osshim"
 	"github.com/gardener/docforge/pkg/registry"
 	"github.com/gardener/docforge/pkg/registry/repositoryhost"
-	"github.com/gardener/docforge/pkg/workers/document"
-	"github.com/gardener/docforge/pkg/workers/githubinfo"
-	"github.com/gardener/docforge/pkg/workers/linkvalidator"
-	"github.com/gardener/docforge/pkg/workers/taskqueue"
 	"github.com/spf13/viper"
 	"k8s.io/klog/v2"
 )
@@ -52,10 +52,6 @@ func exec(ctx context.Context, vip *viper.Viper) error {
 
 	config := getReactorConfig(options.Options, options.Hugo, rhs)
 	manifestURL := options.ManifestPath
-	var (
-		ghInfo      githubinfo.GitHubInfo
-		ghInfoTasks taskqueue.QueueController
-	)
 	reactorWG := &sync.WaitGroup{}
 
 	rhRegistry := registry.NewRegistry(append(localRH, config.RepositoryHosts...)...)
@@ -67,36 +63,15 @@ func exec(ctx context.Context, vip *viper.Viper) error {
 		fmt.Println(documentNodes[0])
 	}
 
-	v, validatorTasks, err := linkvalidator.New(config.ValidationWorkersCount, config.FailFast, reactorWG, rhRegistry, config.HostsToReport)
+	mdPlugin, mdTasks, err := markdown.NewPlugin(config.DocumentWorkersCount, config.FailFast, reactorWG, documentNodes, rhRegistry, config.Hugo, config.Writer, config.SkipLinkValidation, config.ValidationWorkersCount, config.HostsToReport, config.ResourceDownloadWorkersCount, config.GitInfoWriter)
 	if err != nil {
 		return err
 	}
-	docProcessor, docTasks, err := document.New(config.DocumentWorkersCount, config.FailFast, reactorWG, documentNodes, v, rhRegistry, config.Hugo, config.Writer, config.SkipLinkValidation)
+	dPlugin, downloadTasks, err := downloader.NewPlugin(config.ResourceDownloadWorkersCount, config.FailFast, reactorWG, rhRegistry, config.Writer)
 	if err != nil {
 		return err
 	}
-
-	qcc := taskqueue.NewQueueControllerCollection(reactorWG, validatorTasks, docTasks)
-
-	if config.GitInfoWriter != nil {
-		ghInfo, ghInfoTasks, err = githubinfo.New(config.ResourceDownloadWorkersCount, config.FailFast, reactorWG, rhRegistry, config.GitInfoWriter)
-		if err != nil {
-			return err
-		}
-		for _, node := range documentNodes {
-			ghInfo.WriteGitHubInfo(node)
-		}
-		qcc.Add(ghInfoTasks)
-	}
-
-	for _, node := range documentNodes {
-		docProcessor.ProcessNode(node)
-	}
-
-	qcc.Start(ctx)
-	qcc.Wait()
-	qcc.Stop()
-	qcc.LogTaskProcessed()
+	err = core.Run(ctx, documentNodes, reactorWG, []nodeplugins.Interface{mdPlugin, dPlugin}, append(mdTasks, downloadTasks))
 	rhRegistry.LogRateLimits(ctx)
-	return qcc.GetErrorList().ErrorOrNil()
+	return err
 }

pkg/core/run.go

@@ -0,0 +1,41 @@
+package core
+
+import (
+	"context"
+	"fmt"
+	"sync"
+
+	"github.com/gardener/docforge/pkg/manifest"
+	"github.com/gardener/docforge/pkg/nodeplugins"
+	"github.com/gardener/docforge/pkg/workers/taskqueue"
+)
+
+// Run is the method that constructs the website bundle
+func Run(ctx context.Context, nodes []*manifest.Node, reactorWG *sync.WaitGroup, plugins []nodeplugins.Interface, pluginQC []taskqueue.QueueController) error {
+	qcc := taskqueue.NewQueueControllerCollection(reactorWG, pluginQC...)
+
+	processorToPlugin := map[string]nodeplugins.Interface{}
+	for _, plugin := range plugins {
+		processorToPlugin[plugin.Processor()] = plugin
+
+	}
+	for _, node := range nodes {
+		if node.Type != "file" {
+			continue
+		}
+		if processor, ok := processorToPlugin[node.Processor]; ok {
+			if err := processor.Process(node); err != nil {
+				return fmt.Errorf("processor %s failed processing node \n%s\n: %w", processor.Processor(), node, err)
+			}
+		} else {
+			// TODO may be undesired if we expect multiple core.Run calls
+			return fmt.Errorf("node \n%s\n did not have a processor", node)
+		}
+	}
+
+	qcc.Start(ctx)
+	qcc.Wait()
+	qcc.Stop()
+	qcc.LogTaskProcessed()
+	return qcc.GetErrorList().ErrorOrNil()
+}

pkg/manifest/manifest.go

@@ -423,6 +423,20 @@ func calculateAliases(node *Node, parent *Node, _ *Node, _ registry.Interface, _
 	return nil
 }
 
+func setMarkdownProcessor(node *Node, parent *Node, _ *Node, _ registry.Interface, _ []string) error {
+	if node.Type == "file" && strings.HasSuffix(node.File, ".md") {
+		node.Processor = "markdown"
+	}
+	return nil
+}
+
+func setDefaultProcessor(node *Node, parent *Node, _ *Node, _ registry.Interface, _ []string) error {
+	if node.Type == "file" {
+		node.Processor = "downloader"
+	}
+	return nil
+}
+
 // ResolveManifest collects files in FileCollector from a given url and resourcehandlers.FileSource
 func ResolveManifest(url string, r registry.Interface, contentFileFormats []string) ([]*Node, error) {
 	manifest := Node{
@@ -446,6 +460,8 @@ func ResolveManifest(url string, r registry.Interface, contentFileFormats []stri
 		mergeFolders,
 		calculatePath,
 		setParent,
+		setDefaultProcessor,
+		setMarkdownProcessor,
 		propagateFrontmatter,
 		propagateSkipValidation,
 		calculateAliases,

pkg/manifest/node.go

@@ -22,6 +22,8 @@ type Node struct {
 
 	FilesTreeType `yaml:",inline"`
 
+	// Proccessor determines which node processor to use
+	Processor string `yaml:"processor,omitempty"`
 	// Properties of the node
 	SkipValidation bool `yaml:"skipValidation,omitempty"`
 	// Frontmatter of the node

pkg/manifest/tests/results/aliases.yaml

@@ -1,4 +1,5 @@
 - file: README.md
+  processor: markdown
   type: file
   source: https://github.com/gardener/docforge/blob/master/contents/README.md
   path: root/topic/new_section_1
@@ -7,6 +8,7 @@
     - "/root2/rebase/README/"
     - "/root3/rebase_slash/README/"
 - file: _index.md
+  processor: markdown
   type: file
   source: https://github.com/gardener/docforge/blob/master/contents/website/blog/2024/_index.md
   path: root/topic/new_section_1/subsection
@@ -15,6 +17,7 @@
     - "/root2/rebase/subsection/"
     - "/root3/rebase_slash/subsection/"
 - file: foo.md
+  processor: markdown
   type: file
   source: https://github.com/gardener/docforge/blob/master/contents/blogs/2024/foo.md
   path: root/topic/new_section_1/subsection
@@ -24,20 +27,23 @@
     - "/root2/rebase/subsection/foo/"
     - "/root3/rebase_slash/subsection/foo/"
 - file: two.md
+  processor: markdown
   type: file
   source: https://github.com/gardener/docforge/blob/master/contents/blogs/2024/two.md
   path: root
   frontmatter:
     aliases:
     - "/root4/normal/alias/"
 - file: _index.md
+  processor: markdown
   type: file
   source: https://github.com/gardener/docforge/blob/master/contents/docs/architecture/_index.md
   path: root/architecture
   frontmatter:
     aliases:
     - "/root/"
 - file: concept.md
+  processor: markdown
   type: file
   source: https://github.com/gardener/docforge/blob/master/contents/docs/architecture/concept.md
   path: root/architecture

pkg/manifest/tests/results/fileTree_filtering.yaml

@@ -1,9 +1,11 @@
 
 - file: two.md
+  processor: markdown
   type: file
   source: https://github.com/gardener/docforge/blob/master/contents/blogs/2024/two.md
   path: .
 - file: foo.md
+  processor: markdown
   type: file
   source: https://github.com/gardener/docforge/blob/master/contents/blogs/2024/foo.md
   path: .

pkg/manifest/tests/results/index_md_with_properties.yaml

@@ -1,24 +1,29 @@
 - file: _index.md
+  processor: markdown
   type: file
   properties:
     title: Blogs
     aggregate: true
   path: foo/bar/one
 - file: _index.md
+  processor: markdown
   type: file
   source: https://github.com/gardener/docforge/blob/master/contents/website/blog/2024/_index.md
   properties:
     aggregate: false
   path: foo/bar/two
 - file: _index.md
+  processor: markdown
   type: file
   source: https://github.com/gardener/docforge/blob/master/contents/website/blog/2024/_index.md
   path: foo/bar/three
 - file: renamed.md
+  processor: markdown
   type: file
   source: https://github.com/gardener/docforge/blob/master/contents/website/blog/2024/_index.md
   path: foo/bar
 - file: _index.md
+  processor: markdown
   type: file
   source: https://github.com/gardener/docforge/blob/master/contents/website/blog/2024/_index.md
   path: foo/bar/blog/2024

pkg/manifest/tests/results/manifest.yaml

@@ -1,40 +1,49 @@
 - file: _index.md
+  processor: markdown
   type: file
   source: https://github.com/gardener/docforge/blob/master/contents/website/blog/2024/_index.md
   path: blog/2024
 - file: foo.md
+  processor: markdown
   type: file
   source: https://github.com/gardener/docforge/blob/master/contents/blogs/2024/foo.md
   path: blog/2024
 - file: two.md
+  processor: markdown
   type: file
   source: https://github.com/gardener/docforge/blob/master/contents/blogs/2024/two.md
   path: blog/2024
 - file: foo.md
+  processor: markdown
   type: file
   source: https://github.com/gardener/docforge/blob/master/contents/blogs/2024/foo.md
   path: blog
 - file: _index.md
+  processor: markdown
   type: file
   properties:
     title: Blogs
     aggregate: true
   path: foo/bar/one
 - file: _index.md
+  processor: markdown
   type: file
   source: https://github.com/gardener/docforge/blob/master/contents/website/blog/2024/_index.md
   properties:
     aggregate: false
   path: foo/bar/two
 - file: _index.md
+  processor: markdown
   type: file
   source: https://github.com/gardener/docforge/blob/master/contents/website/blog/2024/_index.md
   path: foo/bar/three
 - file: renamed.md
+  processor: markdown
   type: file
   source: https://github.com/gardener/docforge/blob/master/contents/website/blog/2024/_index.md
   path: foo/bar
 - file: _index.md
+  processor: markdown
   type: file
   source: https://github.com/gardener/docforge/blob/master/contents/website/blog/2024/_index.md
   path: foo/bar/blog/2024

pkg/manifest/tests/results/merging.yaml

@@ -1,16 +1,20 @@
 - file: _index.md
+  processor: markdown
   type: file
   source: https://github.com/gardener/docforge/blob/master/contents/website/blog/2024/_index.md
   path: blog/2024
 - file: foo.md
+  processor: markdown
   type: file
   source: https://github.com/gardener/docforge/blob/master/contents/blogs/2024/foo.md
   path: blog/2024
 - file: two.md
+  processor: markdown
   type: file
   source: https://github.com/gardener/docforge/blob/master/contents/blogs/2024/two.md
   path: blog/2024
 - file: foo.md
+  processor: markdown
   type: file
   source: https://github.com/gardener/docforge/blob/master/contents/blogs/2024/foo.md
   path: blog

pkg/manifest/tests/results/multisource.yaml

@@ -1,4 +1,5 @@
 - file: testo.md
+  processor: markdown
   type: file
   multiSource:
   - https://github.com/gardener/docforge/blob/master/contents/blogs/2024/foo.md