v5 v6: ao_composite, nested includes, .ssi (insert) can now include .ssi & .sst

* .ssm can contain/include .sst or .ssi
  .sst files can be processed independently, they do not contain other files
  .ssi files have been only insertions of text & are not processed independently
  but as part of an ssm
  change proposed:
  .ssm unchanged,
  .sst unchanged,
  .ssi as before, but can include .sst or .ssi, so:
  files are only insertions & are not processed independently but as part of an
  .ssm or another .ssi
  i.e. as before with the addition that it like a .ssm can contain/include
  .sst or .ssi
* requested feature, (Closes: #744408)
  User beware. In previous versions there has been a deliberate attempt to keep
  it easy to follow documents & not have to dig through different levels of
  includes, as this is more likely to lead to errors in markup.
  The idea of nested includes has not been attractive (messy for a document),
  however, this is now made possible, use with care
  * it is not as easy to see document structure at a glance, and structural
    errors may be introduced and will need to be taken care of (requiring
    document markup debugging)
  * allowing .ssi to also include other .ssi or .sst could lead to infinite
    recursion if an .ssi includes another which includes itself; stopping after
    an additional level of includes seems arbitrary, and possibly prone to error
    if you are dealing with many documents
* requires testing
* version bump, new behavior added to .ssi one of the 3 sisu filetypes

2 files changed, 156 insertions, 130 deletions

diff --git a/lib/sisu/v5/ao_composite.rb b/lib/sisu/v5/ao_composite.rb
index b35b2419..5cdfaa81 100644
--- a/lib/sisu/v5/ao_composite.rb
+++ b/lib/sisu/v5/ao_composite.rb
@@ -101,9 +101,17 @@ module SiSU_Assemble
       begin
         pwd=Dir.pwd
         Dir.chdir(@opt.f_pth[:pth])
-        fns_array=IO.readlines(@opt.fno,'')
-        assembled=insertions?(fns_array)
-        write(assembled)
+        if @opt.fno =~/\S+?\.ssm$/
+          SiSU_Screen::Ansi.new(
+            @opt.act[:color_state][:set],
+            'Composite Document',
+            "[#{@opt.f_pth[:lng_is]}] #{@opt.fno}",
+          ).grey_title_hi unless @opt.act[:quiet][:set]==:on
+          assembled=loadfile(@opt.fno)
+          #assembled=insertions?(fns_array)
+          write(assembled)
+          write(assembled)
+        end
         Dir.chdir(pwd)
       rescue
         SiSU_Errors::Rescued.new($!,$@,@opt.cmd,@opt.fns).location do
@@ -112,6 +120,73 @@ module SiSU_Assemble
       ensure
       end
     end
+    def insert?(para)
+      if para =~ /^<<\s+((?:https?|file):\/\/\S+?\.ss[it])$/ # and NetTest
+        url($1.strip)
+      elsif para =~/^<<\s+(\S+?\.ss[it])$/
+        loadfilename=$1.strip
+        insert_array=loadfile(loadfilename)
+        file=insertion(loadfilename,insert_array)
+        file[:prepared]
+      else para
+      end
+    end
+    def loadfile(loadfilename)
+      tuned_file=[]
+      begin
+        if FileTest.file?(loadfilename)
+          insert_array=IO.readlines(loadfilename,'')
+          if loadfilename =~/\S+?\.ss[im]$/
+            if (@opt.act[:verbose][:set]==:on \
+            || @opt.act[:verbose_plus][:set]==:on \
+            || @opt.act[:maintenance][:set]==:on)
+              SiSU_Screen::Ansi.new(
+                @opt.act[:color_state][:set],
+                'loading:',
+                loadfilename,
+              ).txt_grey
+            end
+            insert_array.each do |para|
+              tuned_file << insert?(para)
+            end
+          elsif loadfilename =~/\S+?\.sst$/
+            insert_array.each do |para|
+              tuned_file << para
+            end
+          end
+        end
+        tuned_file=tuned_file.flatten.compact
+      rescue
+        SiSU_Errors::Rescued.new($!,$@,@opt.cmd,@opt.fns).location do
+          __LINE__.to_s + ':' + __FILE__
+        end
+      ensure
+      end
+    end
+    def url(loadfilename)
+      if loadfilename =~ /((?:https?|file):\/\/\S+?\.ss[it])$/ # and NetTest
+        loadfilename=$1
+        imagedir = /((?:https?|file):\/\/\S+?)\/[^\/]+?\.ss[it]$/.
+          match(loadfilename).captures.join +
+          '/_sisu/image' #watch
+        begin
+          require 'uri'
+          require 'open-uri'
+          require 'pp'
+        rescue LoadError
+          SiSU_Utils::CodeMarker.new(__LINE__,__FILE__,:fuchsia).
+            error('uri, open-uri or pp NOT FOUND (LoadError)')
+        end
+        image_uri=URI.parse(imagedir)
+        insert=open(loadfilename)
+        insert_array=insert.dup
+        insert.close
+        file=insertion(loadfilename,insert_array)
+        @@imager[image_uri] ||=[]
+        @@imager[image_uri] << file[:images]
+        file[:prepared]
+      end
+    end
     def write(assembled)
       assembled_file=File.new("#{@env.processing_path.composite_file}/#{@opt.fnb}.ssm.sst",'w+')
       assembled.each {|a| assembled_file << a }
@@ -165,68 +240,6 @@ module SiSU_Assemble
       end
       file
     end
-    def insertions?(fns_array)
-      tuned_file,imagedir=[],[]
-      SiSU_Screen::Ansi.new(
-        @opt.act[:color_state][:set],
-        'Composite Document',
-        "[#{@opt.f_pth[:lng_is]}] #{@opt.fno}",
-      ).grey_title_hi unless @opt.act[:quiet][:set]==:on
-      fns_array.each do |para|
-        if para =~/^<<\s+(\S+?\.ss[it])$/
-          loadfilename=$1.strip
-          if (@opt.act[:verbose][:set]==:on \
-          || @opt.act[:verbose_plus][:set]==:on \
-          || @opt.act[:maintenance][:set]==:on)
-            SiSU_Screen::Ansi.new(
-              @opt.act[:color_state][:set],
-              'loading:',
-              loadfilename,
-            ).txt_grey
-          end
-          tuned_file << if loadfilename =~ /(?:https?|file):\/\/\S+?\.ss[it]$/ # and NetTest
-            imagedir = /((?:https?|file):\/\/\S+?)\/[^\/]+?\.ss[it]$/.
-              match(loadfilename).captures.join +
-              '/_sisu/image' #watch
-            begin
-              require 'uri'
-              require 'open-uri'
-              require 'pp'
-            rescue LoadError
-              SiSU_Utils::CodeMarker.new(__LINE__,__FILE__,:fuchsia).
-                error('uri, open-uri or pp NOT FOUND (LoadError)')
-            end
-            image_uri=URI.parse(imagedir)
-            insert=open(loadfilename)
-            insert_array=insert.dup
-            insert.close
-            file=insertion(loadfilename,insert_array)
-            @@imager[image_uri] ||=[]
-            @@imager[image_uri] << file[:images]
-            file[:prepared]
-          elsif loadfilename =~ /\.ss[it]$/ \
-          and FileTest.file?(loadfilename)
-            insert_array=IO.readlines(loadfilename,'')
-            file=insertion(loadfilename,insert_array)
-            file[:prepared]
-          else
-            STDERR.puts %{SKIPPED processing file: [#{@opt.lng}] "#{@opt.fns}" it requires an invalid or non-existent file: "#{loadfilename}"}
-            $process_document = :skip; break #remove this line to continue processing documents that have missing include files
-            para
-          end
-        else tuned_file << para
-        end
-        tuned_file=tuned_file.flatten.compact
-      end
-      if @@imager.length >0
-        @@imager.each do |d,i|
-          i=i.flatten.uniq
-          image_info=d + i
-          download_images(image_info.flatten)
-        end
-      end
-      tuned_file
-    end
   end
   class CompositeFileList
     @@imager={}
diff --git a/lib/sisu/v6/ao_composite.rb b/lib/sisu/v6/ao_composite.rb
index efe3815f..078e8ece 100644
--- a/lib/sisu/v6/ao_composite.rb
+++ b/lib/sisu/v6/ao_composite.rb
@@ -101,9 +101,17 @@ module SiSU_Assemble
       begin
         pwd=Dir.pwd
         Dir.chdir(@opt.f_pth[:pth])
-        fns_array=IO.readlines(@opt.fno,'')
-        assembled=insertions?(fns_array)
-        write(assembled)
+        if @opt.fno =~/\S+?\.ssm$/
+          SiSU_Screen::Ansi.new(
+            @opt.act[:color_state][:set],
+            'Composite Document',
+            "[#{@opt.f_pth[:lng_is]}] #{@opt.fno}",
+          ).grey_title_hi unless @opt.act[:quiet][:set]==:on
+          assembled=loadfile(@opt.fno)
+          #assembled=insertions?(fns_array)
+          write(assembled)
+          write(assembled)
+        end
         Dir.chdir(pwd)
       rescue
         SiSU_Errors::Rescued.new($!,$@,@opt.cmd,@opt.fns).location do
@@ -112,6 +120,73 @@ module SiSU_Assemble
       ensure
       end
     end
+    def insert?(para)
+      if para =~ /^<<\s+((?:https?|file):\/\/\S+?\.ss[it])$/ # and NetTest
+        url($1.strip)
+      elsif para =~/^<<\s+(\S+?\.ss[it])$/
+        loadfilename=$1.strip
+        insert_array=loadfile(loadfilename)
+        file=insertion(loadfilename,insert_array)
+        file[:prepared]
+      else para
+      end
+    end
+    def loadfile(loadfilename)
+      tuned_file=[]
+      begin
+        if FileTest.file?(loadfilename)
+          insert_array=IO.readlines(loadfilename,'')
+          if loadfilename =~/\S+?\.ss[im]$/
+            if (@opt.act[:verbose][:set]==:on \
+            || @opt.act[:verbose_plus][:set]==:on \
+            || @opt.act[:maintenance][:set]==:on)
+              SiSU_Screen::Ansi.new(
+                @opt.act[:color_state][:set],
+                'loading:',
+                loadfilename,
+              ).txt_grey
+            end
+            insert_array.each do |para|
+              tuned_file << insert?(para)
+            end
+          elsif loadfilename =~/\S+?\.sst$/
+            insert_array.each do |para|
+              tuned_file << para
+            end
+          end
+        end
+        tuned_file=tuned_file.flatten.compact
+      rescue
+        SiSU_Errors::Rescued.new($!,$@,@opt.cmd,@opt.fns).location do
+          __LINE__.to_s + ':' + __FILE__
+        end
+      ensure
+      end
+    end
+    def url(loadfilename)
+      if loadfilename =~ /((?:https?|file):\/\/\S+?\.ss[it])$/ # and NetTest
+        loadfilename=$1
+        imagedir = /((?:https?|file):\/\/\S+?)\/[^\/]+?\.ss[it]$/.
+          match(loadfilename).captures.join +
+          '/_sisu/image' #watch
+        begin
+          require 'uri'
+          require 'open-uri'
+          require 'pp'
+        rescue LoadError
+          SiSU_Utils::CodeMarker.new(__LINE__,__FILE__,:fuchsia).
+            error('uri, open-uri or pp NOT FOUND (LoadError)')
+        end
+        image_uri=URI.parse(imagedir)
+        insert=open(loadfilename)
+        insert_array=insert.dup
+        insert.close
+        file=insertion(loadfilename,insert_array)
+        @@imager[image_uri] ||=[]
+        @@imager[image_uri] << file[:images]
+        file[:prepared]
+      end
+    end
     def write(assembled)
       assembled_file=File.new("#{@env.processing_path.composite_file}/#{@opt.fnb}.ssm.sst",'w+')
       assembled.each {|a| assembled_file << a }
@@ -165,68 +240,6 @@ module SiSU_Assemble
       end
       file
     end
-    def insertions?(fns_array)
-      tuned_file,imagedir=[],[]
-      SiSU_Screen::Ansi.new(
-        @opt.act[:color_state][:set],
-        'Composite Document',
-        "[#{@opt.f_pth[:lng_is]}] #{@opt.fno}",
-      ).grey_title_hi unless @opt.act[:quiet][:set]==:on
-      fns_array.each do |para|
-        if para =~/^<<\s+(\S+?\.ss[it])$/
-          loadfilename=$1.strip
-          if (@opt.act[:verbose][:set]==:on \
-          || @opt.act[:verbose_plus][:set]==:on \
-          || @opt.act[:maintenance][:set]==:on)
-            SiSU_Screen::Ansi.new(
-              @opt.act[:color_state][:set],
-              'loading:',
-              loadfilename,
-            ).txt_grey
-          end
-          tuned_file << if loadfilename =~ /(?:https?|file):\/\/\S+?\.ss[it]$/ # and NetTest
-            imagedir = /((?:https?|file):\/\/\S+?)\/[^\/]+?\.ss[it]$/.
-              match(loadfilename).captures.join +
-              '/_sisu/image' #watch
-            begin
-              require 'uri'
-              require 'open-uri'
-              require 'pp'
-            rescue LoadError
-              SiSU_Utils::CodeMarker.new(__LINE__,__FILE__,:fuchsia).
-                error('uri, open-uri or pp NOT FOUND (LoadError)')
-            end
-            image_uri=URI.parse(imagedir)
-            insert=open(loadfilename)
-            insert_array=insert.dup
-            insert.close
-            file=insertion(loadfilename,insert_array)
-            @@imager[image_uri] ||=[]
-            @@imager[image_uri] << file[:images]
-            file[:prepared]
-          elsif loadfilename =~ /\.ss[it]$/ \
-          and FileTest.file?(loadfilename)
-            insert_array=IO.readlines(loadfilename,'')
-            file=insertion(loadfilename,insert_array)
-            file[:prepared]
-          else
-            STDERR.puts %{SKIPPED processing file: [#{@opt.lng}] "#{@opt.fns}" it requires an invalid or non-existent file: "#{loadfilename}"}
-            $process_document = :skip; break #remove this line to continue processing documents that have missing include files
-            para
-          end
-        else tuned_file << para
-        end
-        tuned_file=tuned_file.flatten.compact
-      end
-      if @@imager.length >0
-        @@imager.each do |d,i|
-          i=i.flatten.uniq
-          image_info=d + i
-          download_images(image_info.flatten)
-        end
-      end
-      tuned_file
-    end
   end
   class CompositeFileList
     @@imager={}