Merge tag 'sisu_7.2.1' into debian

SiSU 7.2.1

1 files changed, 1719 insertions, 0 deletions

diff --git a/org/sst.org b/org/sst.org
new file mode 100644
index 00000000..93945585
--- /dev/null
+++ b/org/sst.org
@@ -0,0 +1,1719 @@
+-*- mode: org -*-
+#+TITLE:       sisu sst
+#+DESCRIPTION: documents - structuring, various output representations & search
+#+FILETAGS:    :sisu:sst:
+#+AUTHOR:      Ralph Amissah
+#+EMAIL:       [[mailto:ralph.amissah@gmail.com][ralph.amissah@gmail.com]]
+#+COPYRIGHT:   Copyright (C) 2015 - 2021 Ralph Amissah
+#+LANGUAGE:    en
+#+STARTUP:     content hideblocks hidestars noindent entitiespretty
+#+OPTIONS:     H:3 num:nil toc:t \n:nil @:t ::t |:t ^:nil _:nil -:t f:t *:t <:t
+#+PROPERTY:    header-args  :exports code
+#+PROPERTY:    header-args+ :noweb yes
+#+PROPERTY:    header-args+ :eval no
+#+PROPERTY:    header-args+ :results no
+#+PROPERTY:    header-args+ :cache no
+#+PROPERTY:    header-args+ :padline no
+#+PROPERTY:    header-args+ :mkdirp yes
+
+* sst_from_xml.rb
+
+#+HEADER: :tangle "../lib/sisu/sst_from_xml.rb"
+#+BEGIN_SRC ruby
+#<<sisu_document_header>>
+module SiSU_sstFromXML
+  require_relative 'se'                                 # se.rb
+  class Convert
+    begin
+      require 'rexml/document'
+        include REXML
+    rescue LoadError
+      SiSU_Utils::CodeMarker.new(__LINE__,__FILE__,:fuchsia).
+        error('rexml/document NOT FOUND (LoadError)')
+    end
+    def initialize(opt)
+      @opt=opt
+      @sisu,@sisu_base=[],[]
+      @ver=SiSU_Env::InfoVersion.instance.get_version
+    end
+    def tell(filename,type)
+      SiSU_Screen::Ansi.new(
+        @opt.act[:color_state][:set],
+        "XML #{type} to SiSU sst",
+        "#{filename} --> #{filename}.sst"
+      ).green_hi_blue
+    end
+    def read
+      xml_to_sisu
+    end
+    def markup_head(text)
+      text.strip!
+      text.gsub!(/(?:\s*\n|\s\s+)/,' ')
+      text.gsub!(/<header class=['"]\S+?['"]>(.+?)<\/header>/,'\1')
+      text.gsub!(/<(\w+)>(.+?)<\/\w+>/,'@\1: \2')
+      text.gsub!(/<header class=['"]\S+?['"]><(\w+)>(.+?)<\/\w+><\/header>/,'@\1: \2')
+      text.gsub!(/\s +/,' ')
+      text.strip!
+      text + "\n\n"
+    end
+    def markup(text)
+      text.strip!
+      text.gsub!(/(?:\s*\n|\s\s+)/,' ')
+      text.gsub!(/<text class='h1'>(.+?)<\/text>/,':A~ \1')
+      text.gsub!(/<text class='h2'>(.+?)<\/text>/,':B~ \1')
+      text.gsub!(/<text class='h3'>(.+?)<\/text>/,':C~ \1')
+      text.gsub!(/<text class='h4'>(.+?)<\/text>/,'1~ \1')
+      text.gsub!(/<text class='h5'>(.+?)<\/text>/,'2~ \1')
+      text.gsub!(/<text class='h6'>(.+?)<\/text>/,'3~ \1')
+      text.gsub!(/<text class='norm'>(.+?)<\/text>/,'\1')
+      text.gsub!(/<endnote symbol='norm'>(.+?)<\/endnote>/,'~{ \1 }~')
+      text.gsub!(/<br ?\/>/,'<br>')
+      text.gsub!(/<i>(.+?)<\/i>/,'/{\1}/')
+      text.gsub!(/<b>(.+?)<\/b>/,'*{\1}*')
+      text.gsub!(/<u>(.+?)<\/u>/,'_{\1}_')
+      text.gsub!(/<sem:([a-z_]+)\s+depth=['"]zero['"]>(\s*.+?\s*)<\/sem:\1>/,';{ \2 };\1')
+      text.gsub!(/<sem:([a-z_]+)\s+depth=['"]one['"]>(\s*.+?\s*)<\/sem:\1>/,':{ \2 }:\1')
+      text.gsub!(/<sem:([a-z_]+)\s+depth=['"]many['"]>(\s*.+?\s*)<\/sem:\1>/,'\1:{ \2 }:\1')
+      text.gsub!(/<sem:([a-z_]+)>(\s*.+?\s*)<\/sem:\1>/,'\1:{ \2 }:\1')
+      text.gsub!(/\s +/,' ')
+      text.strip!
+      text + "\n\n"
+    end
+    def sax
+      out_file=File.new(@output_file_name,'w')
+      head=@doc.root.get_elements('//head/header')
+      body=@doc.root.get_elements('//object/text')
+      out_file.puts "% SiSU text #{@ver.version} (generated from a SiSU XML markup representation)\n\n"
+      head.each do |x|
+        if x.name=='header'
+          head=markup_head(x.to_s)
+          out_file.puts head
+        end
+      end
+      body.each do |x|
+        if x.name=='text'
+          body=markup(x.to_s)
+          out_file.puts body
+        end
+      end
+    end
+    def node
+      sax
+    end
+    def dom
+      raise "#{__FILE__}::#{__LINE__} xml dom representation to sst not yet implemented (experimental simple xml representations sax and node to sst are in place)."
+    end
+    def xml_to_sisu
+      unless @opt.files.empty?
+        @opt.files.each do |xml|
+          @sisu_base=[]
+          if xml =~/\.sx[sdn]\.xml$/
+            begin
+              @doc_str=IO.readlines(xml,'').join("\n")
+              @output=File.new("#{xml}.sst",'w')
+              @doc=REXML::Document.new(@doc_str)
+              @output_file_name="#{Dir.pwd}/#{xml}.sst"
+              @el=[]
+            rescue REXML::ParseException
+            end
+          end
+          if xml =~/\.sxs\.xml$/
+            unless @opt.act[:quiet][:set]==:on
+              tell(xml,'sax')
+            end
+            sax
+          elsif xml =~/\.sxd\.xml$/
+            unless @opt.act[:quiet][:set]==:on
+              tell(xml,'dom')
+            end
+            dom
+          elsif xml =~/\.sxn\.xml$/
+            unless @opt.act[:quiet][:set]==:on
+              tell(xml,'node')
+            end
+            node
+          else puts "filename not recognised: << #{xml} >>"
+          end
+          @output << @sisu_base
+        end
+      else puts '.xml file for conversion to sisu expected'
+      end
+      puts @opt.files.inspect
+    end
+  end
+end
+__END__
+#+END_SRC
+
+* sst_to_s_xml_sax.rb
+
+#+HEADER: :tangle "../lib/sisu/sst_to_s_xml_sax.rb"
+#+BEGIN_SRC ruby
+#<<sisu_document_header>>
+module SiSU_SimpleXML_ModelSax
+  require_relative 'se_hub_particulars'                 # se_hub_particulars.rb
+    include SiSU_Particulars
+  require_relative 'dp'                                 # dp.rb
+    include SiSU_Param
+  require_relative 'se'                                 # se.rb
+    include SiSU_Env
+  require_relative 'ao_doc_str'                         # ao_doc_str.rb
+  require_relative 'xml_shared'                         # xml_shared.rb
+    include SiSU_XML_Munge
+  require_relative 'shared_sem'                         # shared_sem.rb
+  require_relative 'xml_format'                         # xml_format.rb
+    include SiSU_XML_Format
+  require_relative 'rexml'                              # rexml.rb
+    include SiSU_Rexml
+  @@alt_id_count=0
+  @@tablefoot=''
+  class Convert
+    @@fns=nil
+    def initialize(opt)
+      @opt=opt
+      @particulars=SiSU_Particulars::CombinedSingleton.instance.get_env_md(opt)
+    end
+    def read
+      begin
+        @md=@particulars.md #bug, relies on info persistence, assumes -m has previously been run
+        @env=@particulars.env
+        SiSU_Screen::Ansi.new(
+          @opt.act[:color_state][:set],
+          'invert',
+          'XML SAX',
+          "#{@md.fns} -> #{@md.fn[:sxs]}"
+        ).colorize unless @opt.act[:quiet][:set]==:on
+        if (@opt.act[:verbose_plus][:set]==:on \
+        || @opt.act[:maintenance][:set]==:on)
+          SiSU_Screen::Ansi.new(
+            @opt.act[:color_state][:set],
+            @opt.fns,
+            "#{Dir.pwd}/#{@md.fn[:sxs]}"
+          ).flow
+        end
+        unless @@fns==@opt.fns
+          @@fns=@opt.fns
+          @@fns_array=[]
+        end
+        @fns_array=if @@fns_array.empty?; read_fnm
+        else @@fns_array.dup #check
+        end
+        SiSU_SimpleXML_ModelSax::Convert::Songsheet.new(@fns_array,@particulars).songsheet
+      rescue
+        SiSU_Errors::Rescued.new($!,$@,@opt.cmd,@opt.fns).location do
+          __LINE__.to_s + ':' + __FILE__
+        end
+      ensure #file closed in songsheet
+      end
+    end
+    def read_fnm
+      ao=[]
+      if FileTest.file?("#{Dir.pwd}/#{@opt.fns}")
+        ao=IO.readlines("#{Dir.pwd}/#{@opt.fns}","\n\n")
+      else STDERR.puts 'Error'
+      end
+    end
+    private
+    class Songsheet
+      def initialize(data,particulars)
+        @data,@particulars,@env,@md=data,particulars,particulars.env,particulars.md
+      end
+      def songsheet
+        begin
+          SiSU_SimpleXML_ModelSax::Convert::Scroll.new(@data,@particulars).songsheet
+          if (@md.opt.act[:verbose][:set]==:on \
+          || @md.opt.act[:verbose_plus][:set]==:on \
+          || @md.opt.act[:maintenance][:set]==:on)
+            SiSU_SimpleXML_ModelSax::Convert::Tidy.new(@md,@env).xml # test wellformedness, comment out when not in use
+          end
+          SiSU_Rexml::Rexml.new(@md,@md.fn[:sxs]).xml if @md.opt.act[:maintenance][:set]==:on # test rexml parsing, comment out when not in use #debug
+        rescue
+          SiSU_Errors::Rescued.new($!,$@,@md.opt.cmd,@md.fns).location do
+            __LINE__.to_s + ':' + __FILE__
+          end
+        ensure
+        end
+      end
+    end
+    class Scroll
+      require_relative 'txt_shared'                     # txt_shared.rb
+      require_relative 'css'                            # css.rb
+        include SiSU_TextUtils
+      @@xml={ body: [], open: [], close: [], head: [] }
+      def initialize(data='',particulars='')
+        @data,@env,@md=data,particulars.env,particulars.md
+        @regx=/^(?:#{Mx[:mk_o]}:p[bn]#{Mx[:mk_c]}\s*)?(?:#{Mx[:lv_o]}[1-9]:(\S*)#{Mx[:lv_c]})?(.+)/
+        @tab="\t"
+        if @md
+          @trans=SiSU_XML_Munge::Trans.new(@md)
+        end
+        @sys=SiSU_Env::SystemCall.new
+      end
+      def songsheet
+        pre
+        markup
+        post
+        publish
+      end
+    protected
+      def embedded_endnotes(para='')
+        para.gsub!(/~\{(.+?)\}~/,'<endnote symbol="norm">\1</endnote> ')
+        para.gsub!(/~\[([*+])\s+(.+?)\]~/,'<endnote symbol="\1">\2</endnote> ')
+      end
+      def xml_head(meta)
+        txt=meta.text
+        txt.gsub!(/\/{(.+?)}\//,'<i>\1</i>')
+        txt.gsub!(/[*!]{(.+?)}[*!]/,'<b>\1</b>')
+        txt.gsub!(/_{(.+?)}_/,'<u>\1</u>')
+        txt.gsub!(/-{(.+?)}-/,'<del>\1</del>')
+        txt.gsub!(/<br(?: \/)?>/,'<br />')
+        txt.gsub!(/ & /,' and ')
+        @@xml[:head] <<<<WOK
+#{@tab}<header class="#{meta.attrib}">
+#{@tab*2}<#{meta.el}>
+#{@tab*3}#{txt}
+#{@tab*2}</#{meta.el}>
+#{@tab}</header>
+WOK
+      end
+      def xml_sc(md='')
+        sc=if @md.sc_info
+          <<WOK
+    <source_control>
+      <sc class="sourcefile">
+        #{@md.sc_filename}
+      </sc>
+      <sc class="number">
+        #{@md.sc_number}
+      </sc>
+      <sc class="date">
+        #{@md.sc_date}
+      </sc>
+    </source_control>
+WOK
+        else ''
+        end
+        @@xml[:sc]=sc
+      end
+      def xml_structure(para='',lv='',hname='') #extracted endnotes
+        lv=lv.to_i
+        lv=nil if lv==0
+        embedded_endnotes(para)
+        if para[@regx]
+          paragraph="#{para[@regx,2]}"
+          util=SiSU_TextUtils::Wrap.new(paragraph,70)
+          wrapped=util.line_wrap
+        end
+        @@xml[:body] << "#{@tab*0}<object>" if para[@regx]
+        @@xml[:body] << "#{@tab*1}" << "\n" if para[@regx]
+        @@xml[:body] << if lv; %{#{@tab*1}<text class="h#{lv}">\n#{@tab*2}#{wrapped}\n#{@tab*1}</text>\n} << "\n"
+        elsif wrapped =~/\A%%?\s+/; %{<!--\n#{@tab*1}<text class="comment">\n#{@tab*2}#{wrapped}\n#{@tab*1}</text>\n-->\n} # comments
+        else                        %{#{@tab*1}<text class="norm">\n#{@tab*2}#{wrapped}\n#{@tab*1}</text>\n} # main text, contents, body KEEP
+        end
+        @@xml[:body] << "#{@endnotes}" if @endnotes # main text, endnotes KEEP
+        @@xml[:body] << "#{@tab*0}</object>" << "\n" if para[@regx]
+        @endnotes=[]
+      end
+      def block_structure(para='')
+        para.gsub!(/<:block(?:-end)?>/,'')
+        para.strip!
+        @@xml[:body] << %{#{@tab*0}<object>}
+        @@xml[:body] << %{#{@tab*1}<text class="block">#{@tab*1}\n}
+        @@xml[:body] << %{#{@tab*2}#{para}#{@tab*1}\n}
+        @@xml[:body] << %{#{@tab*1}</text>\n}
+        @@xml[:body] << "#{@tab*0}</object>"
+      end
+      def group_structure(para='')
+        para.gsub!(/<:group(?:-end)?>/,'')
+        para.strip!
+        @@xml[:body] << %{#{@tab*0}<object>}
+        @@xml[:body] << %{#{@tab*1}<text class="group">#{@tab*1}\n}
+        @@xml[:body] << %{#{@tab*2}#{para}#{@tab*1}\n}
+        @@xml[:body] << %{#{@tab*1}</text>\n}
+        @@xml[:body] << "#{@tab*0}</object>"
+      end
+      def poem_structure(para='')
+        para.gsub!(/<:verse(?:-end)?>/,'')
+        para.strip!
+        @@xml[:body] << %{#{@tab*0}<object>}
+        @@xml[:body] << %{#{@tab*1}<text class="verse">#{@tab*1}\n}
+        @@xml[:body] << %{#{@tab*2}#{para}#{@tab*1}\n}
+        @@xml[:body] << %{#{@tab*1}</text>\n}
+        @@xml[:body] << "#{@tab*0}</object>" << "\n"
+      end
+      def code_structure(para='')
+        para.gsub!(/<:code(?:-end)?>/,'')
+        para.strip!
+        @@xml[:body] << %{#{@tab*0}<object>}
+        @@xml[:body] << %{#{@tab*1}<text class="code">#{@tab*1}\n}
+        @@xml[:body] << %{#{@tab*2}#{para}#{@tab*1}\n}
+        @@xml[:body] << %{#{@tab*1}</text>\n}
+        @@xml[:body] << "#{@tab*0}</object>" << "\n"
+      end
+      def table_structure(table='') #tables
+        @@xml[:body] << %{#{@tab*0}<object>}
+        @@xml[:body] << %{#{@tab*1}#{table}\n#{@tab*1}\n} # unless lv  # main text, contents, body KEEP
+        @@xml[:body] << "#{@tab*0}</object>" << "\n" #if para[@regx]
+        @endnotes=[]
+      end
+      def tidywords(wordlist)
+        wordlist.each do |x|
+          x.gsub!(/&/,'&amp;') unless x =~/&\S+;/
+        end
+      end
+      def xml_clean(para)
+        para.gsub!(/#{Mx[:gl_o]}[1-9]:\S*?#{Mx[:gl_c]}/,'') #Danger, watch
+        para
+      end
+      def markup
+        data=[]
+        xml_sc(@md)
+        @endnotes,@level,@cont,@copen,@xml_contents_close=[],[],[],[],[]
+        @rcdc=false
+        (0..6).each { |x| @cont[x]=@level[x]=false }
+        (4..6).each { |x| @xml_contents_close[x]='' }
+        @data.each do |para|
+          data << SiSU_AO_DocumentStructureExtract::Structure.new(@md,para).structure #takes on Mx marks
+        end
+        data.each do |para|
+          if para !~/^\s*(?:%+ |<:code>)/
+            if @md.sem_tag and para =~/[:;]\{|\}[:;]/
+              para=@trans.xml_semantic_tags(para)
+            end
+            if para =~/[:;]\{|\}[:;]/
+              para=SiSU_Sem::Tags.new(para,@md).rm.all
+            end
+          end
+          para=@trans.markup_light(para)
+          @trans.char_enc.utf8(para) if @sys.locale =~/utf-?8/i #% utf8
+          para.gsub!(/^@(\S+?):/,"#{Mx[:lv_o]}@\\1#{Mx[:lv_c]}")
+          if para =~/\A#{Mx[:lv_o]}@(\S+?)#{Mx[:lv_c]}\s*(.+?)\Z/m # for headers
+            d_meta=SiSU_TextUtils::HeaderScan.new(@md,para).meta
+            if d_meta; xml_head(d_meta)
+            end
+          end
+          para='' if para=~/#{Mx[:lv_o]}@\S+?#{Mx[:lv_c]}/
+          if @rcdc==false \
+          and (para =~/~metadata/ or para =~/^1~meta\s+Document Information/)
+            @rcdc=true
+          end
+          if para !~/(^@\S+?:|^\s*$|<ENDNOTES>|<EOF>)/
+            @sto=SiSU_text_parts::SplitTextObject.new(@md,para).lev_segname_para
+            unless @rcdc
+              SiSU_XML_Format::FormatScroll.new(@md,@sto.text) if @sto.format =~/i[1-9]|ordinary/
+              case @sto.format
+              when /^(1):(\S*)/
+                xml_clean(para)
+                xml_structure(para,$1,$2)
+                para=@sto.lev_para_ocn.heading_body1
+              when /^(2):(\S*)/
+                xml_clean(para)
+                xml_structure(para,$1,$2)
+                para=@sto.lev_para_ocn.heading_body2
+              when /^(3):(\S*)/
+                xml_clean(para)
+                xml_structure(para,$1,$2)
+                para=@sto.lev_para_ocn.heading_body3
+              when /^(4):(\S*)/ # work on see SplitTextObject
+                xml_clean(para)
+                xml_structure(para,$1,$2)
+                para=@sto.lev_para_ocn.heading_body4
+              when /^(5):(\S*)/
+                xml_clean(para)
+                xml_structure(para,$1,$2)
+                para=@sto.lev_para_ocn.heading_body5
+              when /^(6):(\S*)/
+                xml_clean(para)
+                xml_structure(para,$1,$2)
+                para=@sto.lev_para_ocn.heading_body6
+              else
+                if para =~ /<:verse>/
+                  para=poem_structure(para)
+                elsif para =~ /<:group>/
+                  para=group_structure(para)
+                elsif para =~ /<:code>/
+                  para.gsub!(/</,'&lt;')
+                  para.gsub!(/>/,'&gt;')
+                  para=code_structure(para)
+                elsif para =~/<!Th?.+/ # tables come as single block #work area 2005w13
+                  table=SiSU_Tables::TableXML.new(para)
+                  para=table.table_split
+                  para=table_structure(para)
+                else xml_structure(para,nil,nil)
+                end
+              end
+              if para =~/<a name="n\d+">/ \
+              and para =~/^(-\{{2}~\d+|<!e[:_]\d+!>)/ # -endnote
+                para=''
+              end
+              if para =~/.*<:#>.*$/
+                para=case para
+                when /<:i1>/
+                  format_text=FormatTextObject.new(para,'')
+                  format_text.scr_inden_ocn_e_no_paranum
+                when /<:i2>/
+                  format_text=FormatTextObject.new(para,'')
+                  format_text.scr_inden_ocn_e_no_paranum
+                end
+              end
+              if para =~/<:center>/
+                one,two=/(.*)<:center>(.*)/.match(para)[1,2]
+                format_text=FormatTextObject.new(one,two)
+                para=format_text.center
+              end
+            end
+            para.gsub!(/<:\S+?>/,'')
+            para.gsub!(/<!.+!>/,'') ## Clean Prepared Text #bugwatch reinstate
+            para
+          end
+          para
+        end
+        6.downto(4) do |x|
+          y=x - 1; v=x - 3
+          @@xml[:body] << "#{@tab*5}</content>\n#{@tab*y}</contents#{v}>\n" if @level[x]==true
+        end
+        3.downto(1) do |x|
+          y=x - 1
+          @@xml[:body] << "#{@tab*y}</heading#{x}>\n" if @level[x]==true
+        end
+      end
+      def pre
+        rdf=SiSU_XML_Tags::RDF.new(@md)
+        dir=SiSU_Env::InfoEnv.new
+        @@xml[:head],@@xml[:body]=[],[]
+        css=SiSU_Env::CSS_Select.new(@md).xml_sax
+        encoding=if @sys.locale =~/utf-?8/i then '<?xml version="1.0" encoding="UTF-8" standalone="no"?>'
+        else                                     '<?xml version="1.0" encoding="ISO-8859-1" standalone="no"?>'
+        end
+        @@xml[:open] =<<WOK
+#{encoding}
+<?xml-stylesheet type="text/css" href="../#{dir.path.style}/#{css}"?>
+#{rdf.comment_xml_sax}
+<document>
+WOK
+        @@xml[:head] << "<head>\n"
+        @@xml[:body] << "<body>\n"
+      end
+      def post
+        @@xml[:head] << @@xml[:sc]
+        @@xml[:head] << "</head>\n"
+        @@xml[:body] << "</body>\n"
+        @@xml[:close] = "</document>\n"
+      end
+      def publish
+        content=[]
+        content << @@xml[:open] << @@xml[:head] << @@xml[:body] << @@xml[:metadata]
+        content << @@xml[:owner_details] if @md.stmp =~/\w\w/
+        content << @@xml[:tail] << @@xml[:close]
+        Output.new(content.join,@md).xml
+        @@xml={}
+      end
+    end
+    class Output
+      def initialize(data,md)
+        @data,@md=data,md
+      end
+      def xml
+        @sisu=[]
+        @data.each do |para|
+          para.gsub!(/<:\S+?>/,'')
+          para.gsub!(/<!.+?!>/,'')
+          para="#{para}\n" unless para.empty?
+          @sisu << para
+        end
+        new_file_data=@sisu.join
+        @sisu=new_file_data.scan(/.+/)
+        SiSU_Env::FileOp.new(@md).mkdir
+        filename_sxm=SiSU_Env::FileOp.new(@md,@md.fn[:sxs]).mkfile_pwd
+        if filename_sxm.is_a?(File)
+          @sisu.each {|para| filename_sxm.puts para}
+          filename_sxm.close
+        else puts 'file not created, is directory writable?'
+        end
+      end
+    end
+    class Tidy
+      def initialize(md,dir)
+        @md,@env=md,dir
+        @prog=SiSU_Env::InfoProgram.new
+      end
+      def xml
+        if @prog.tidy !=false #note values can be other than true
+          if (@md.opt.act[:verbose_plus][:set]==:on \
+          || @md.opt.act[:maintenance][:set]==:on)
+            SiSU_Screen::Ansi.new(
+              @md.opt.act[:color_state][:set],
+              'invert',
+              'Using XML Tidy',
+              'check document structure'
+            ).colorize unless @md.opt.act[:quiet][:set]==:on
+            SiSU_Screen::Ansi.new(
+              @md.opt.act[:color_state][:set],
+              '',
+              '',
+              'check document structure'
+            )
+            tell.grey_open unless @md.opt.act[:quiet][:set]==:on
+            tidyfile='/dev/null' #don't want one or screen output, check for alternative flags
+            tidy =SiSU_Env::SystemCall.new("#{Dir.pwd}/#{@md.fn[:sxs]}",tidyfile)
+            tidy.well_formed?
+            tell.p_off unless @md.opt.act[:quiet][:set]==:on
+          end
+        end
+      end
+    end
+  end
+end
+__END__
+#+END_SRC
+
+* sst_identify_markup.rb
+
+#+HEADER: :tangle "../lib/sisu/sst_identify_markup.rb"
+#+BEGIN_SRC ruby
+#<<sisu_document_header>>
+module SiSU_Markup
+  class MarkupInform
+    attr_accessor :version
+    def initialize(version,message,declared_markup='',declared_type='')
+      @version,@message,@declared_markup,@declared_type=version,message,declared_markup,declared_type
+    end
+    def version
+      @version
+    end
+    def message
+      @message
+    end
+    def declared_version
+      @declared_markup
+    end
+    def declared_type
+      @declared_type
+    end
+    def history
+      MarkupHistory.new(@version).query
+    end
+  end
+  class MarkupIdentify
+    def initialize(opt)
+      @opt=opt
+      @description='This is a script attempts to identify the version of markup used in SiSU (and provides information on changes in markup)'
+    end
+    def help
+    print <<WOK
+
+#{@description}
+
+WOK
+      exit
+    end
+    def identify
+      f=@opt.fns
+      if f =~/(?:\.sst|\.ssm|\.ssi|\.s[123])$/ \
+      and File.exist?(f)
+        file=File.open(f,'r')
+        cont=file.readlines
+        file.close
+        links,oldlinks='',''
+        markup=nil
+        @declared_type,@declared_markup='[text?]',''
+        if cont[0] =~ /^(?:%\s+)?SiSU\s+(text|master|insert)\s+([0-9](?:\.[0-9]+){1,2})/ \
+        or cont[0] =~ /^(?:%\s+)?sisu-([0-9](?:\.[0-9]+){1,2})/
+          @declared_type,@declared_markup=$1,$2
+        elsif cont[0] =~ /^(?:%\s+)?SiSU\s+([0-9](?:\.[0-9]+){1,2})/ \
+        or cont[0] =~ /^(?:%\s+)?sisu-([0-9](?:\.[0-9]+){1,2})/
+          @declared_markup=$1
+        end
+        @flag_2_0,@flag_1_0,@flag_69,@flag_66,@flag_57,@flag_38=false,false,false,false,false,false
+        cont.each_with_index do |y,i|
+          if y =~/^(?:0\{?~links?|@links?:)\s/ \
+          and f =~/(?:\.sst|\.ssm|\.ssi|\.s[123])/
+            links=unless y =~/\{.+?\}\S+/; oldlinks=' (pre 0.20.4 header links)'
+            else ' (post 0.20.4 header links)'
+            end
+          end
+          if @flag_2_0 \
+          or y =~/^@make:|^@classify|^\s\s?:[a-z_-]+?:\s+\S/
+            version=2.0.to_f
+            markup=MarkupInform.new(version,'2.0' + oldlinks,@declared_markup,@declared_type)
+            @flag_2_0=true
+            break
+          end
+          unless @flag_38
+            if (y =~/^:?A~/ and f =~/(?:\.sst|\.ssm|\.ssi)/)
+              version='0.38'
+              markup=MarkupInform.new(version,'0.38' + oldlinks,@declared_markup,@declared_type)
+              @flag_38=true
+            end
+          end
+          if @flag_38
+            if @flag_1_0 \
+            or y =~/^=\{.+?\}\s*$/
+              version='0.69'
+              markup=MarkupInform.new(version,'0.69' + oldlinks,@declared_markup,@declared_type)
+              @flag_1_0=true
+              break
+            end
+            if @flag_66 \
+            or y =~/[a-z+][:;]\{.+?\}[:;][a-z+]/
+              version='0.66'
+              markup=MarkupInform.new(version,'0.66' + oldlinks,@declared_markup,@declared_type)
+              @flag_66=true
+              break
+            end
+          end
+        end
+        unless @flag_2_0 \
+        or @flag_1_0 \
+        or @flag_66
+          cont.each_with_index do |y,i|
+            if y =~/^(?:0\{?~links?|@links?:)\s/ \
+            and f =~/(?:\.sst|\.ssm|\.ssi|\.s[123])/
+              links=unless y =~/\{.+?\}\S+/; oldlinks=' (pre 0.20.4 header links)'
+              else ' (post 0.20.4 header links)'
+              end
+            end
+            if @flag_57 \
+            or (y =~/^:?A~\?? @title/ and f =~/(?:\.sst|\.ssm|\.ssi)/)
+              version='0.57'
+              markup=MarkupInform.new(version,'0.57' + oldlinks,@declared_markup,@declared_type)
+              @flag_57=true
+              break
+            end
+            if @flag_38 \
+            or (y =~/^:?A~/ and f =~/(?:\.sst|\.ssm|\.ssi)/)
+              version='0.38'
+              markup=MarkupInform.new(version,'0.38' + oldlinks,@declared_markup,@declared_type)
+              @flag_38=true
+              break if i >= 200
+              if y =~ /(?:~{\*+|~\[\*|~\[\+)\s/
+                version='0.42'
+                markup=MarkupInform.new(version,'0.42' + oldlinks,@declared_markup,@declared_type)
+                break
+              end
+            end
+            if (y =~/^1~/ and f =~/(?:\.sst|\.ssm|\.ssi)/) \
+            and not @flag_38
+              version='0.37'
+              markup=MarkupInform.new(version,'0.37 is substantially 0.16 - 0.36 markup with new file-extension' + oldlinks,@declared_markup,@declared_type)
+              break
+            end
+            if y =~/^1~/ \
+            and f =~/\.([rs])([123])/ \
+            and not @flag_38
+              t,n=$1,$2
+              version='0.16'
+              instruct=if t =~/r/
+                " (change file extension from .#{t}#{n} to .ssm)"
+              else " (change file extension from .#{t}#{n} to .sst)"
+              end
+              markup=MarkupInform.new(version,'0.16 - 0.36' + instruct + links,@declared_markup,@declared_type)
+              break
+            end
+            if y =~/^0\{~/ \
+            and not @flag_38
+              version='0.1'
+              markup=MarkupInform.new(version,'0.1 - 0.15',@declared_markup,@declared_type)
+              break
+            end
+            if y =~/^0\{{3}/ \
+            and not @flag_38
+              markup=MarkupInform.new('circa. 1997','old, check date',@declared_markup,@declared_type)
+              break
+            end
+            markup='Not a recognised file type '
+          end
+        end
+        markup
+      else MarkupHistory.new(@opt).help_query
+      end
+    end
+    def determine_markup_version
+      if @opt.fns.nil? \
+      or @opt.fns.empty?
+        MarkupHistory.new(@opt).help_identify
+      end
+      if File.exist?(@opt.fns)
+        if @opt.fns =~/\.(?:sst|ssm|ssi|s[123i]|r[123])/
+          markup=identify #(@opt.fns)
+          if defined? markup.version
+            unless @opt.act[:quiet][:set]==:on
+              message=unless markup.declared_version.empty?
+                "#{@opt.fns}\n  markup Type Declared as SiSU #{markup.declared_version} #{markup.declared_type}\n  appears to be SiSU #{markup.version}"
+              else
+                "Markup Type Appears to be SiSU #{markup.version}\n  in file #{@opt.fns}"
+              end
+              puts message
+              puts %{"sisu --query-#{markup.version}" for a brief description of markup type}
+            end
+          end
+        else puts 'file-type not recognised: ' + @opt.fns
+        end
+      else puts 'file not found: ' + @opt.fns
+      end
+      (defined? markup.version) \
+      ? markup.version
+      : 'markup type/version not determined'
+    end
+    def markup_version?
+      if @opt.fns.empty?
+        @opt.files.each do |fns|
+          @opt.fns=fns
+          determine_markup_version
+        end
+      else determine_markup_version
+      end
+    end
+  end
+  class MarkupHistory
+    def initialize(opt)
+      @opt=opt
+    end
+    def sisu_3_0
+      <<WOK
+  SiSU 3.0 same as 2.0, apart from change to headers
+
+    see document markup samples, and sisu --help headers
+
+WOK
+    end
+    def sisu_2_0
+      <<WOK
+  SiSU 2.0 same as 1.0, apart from the changing of headers and the addition of a monospace tag
+    related headers now grouped, e.g.
+
+    @title:
+     :subtitle:
+
+    @creator:
+     :author:
+     :translator:
+     :illustrator:
+
+    see document markup samples, and sisu --help headers
+
+    the monospace tag takes the form of a has '#' \#{ this enclosed text would be monospaced }#
+
+WOK
+    end
+    def sisu_1_0
+      <<WOK
+  SiSU 1.0 same as 0.69
+
+WOK
+    end
+    def sisu_0_69
+      <<WOK
+  SiSU 0.69 (same as 1.0) as previous (0.57) with the addition of book index tags
+    /^=\{.+?\}$/
+    e.g. appended to a paragraph, on a new-line (without a blank line in between)
+    logical structure produced assuming this is the first text "object"
+    ={GNU/Linux community distribution:Debian+2|Fedora|Gentoo;Free Software Foundation+5}
+
+   Free Software Foundation, 1-6
+   GNU/Linux community distribution, 1
+       Debian, 1-3
+       Fedora, 1
+       Gentoo,
+
+WOK
+    end
+    def sisu_0_66
+      <<WOK
+  SiSU 0.66 same as previous, adds semantic tags
+    /[:;]\{.+?\}[:;][a-z+]/
+    e.g. :{ Ralph last;{Amissah};last }:author
+
+WOK
+    end
+    def sisu_0_65
+      <<WOK
+  SiSU 0.65 same as previous, adds semantic tags
+    /[a-z+][:;]\{.+?\}[:;][a-z+]/
+    e.g. author:{ Ralph last;{Amissah};last }:author
+
+WOK
+    end
+    def sisu_0_57
+      <<WOK
+
+  SiSU 0.57 (a subset of 1.0) is the same as 0.42 with the introduction of some
+  a shortcut to use the headers @title and @creator in the first heading
+  [expanded using the contents of the headers @title: and @author:]
+
+  :A~ @title by @author
+
+WOK
+    end
+    def sisu_0_42
+      <<WOK
+  SiSU 0.42 (a subset of 1.0) is the same as 0.38 with the introduction of some additional endnote types,
+
+  Introduces some varations on endnotes, in particular the use of the asterisk
+  ~{* for example for describing an author }~ and ~{** for describing a second author }~
+
+  * for example for describing an author
+
+  ** for describing a second author
+
+  and ~[* my note ]~ or ~[+ another note ]~ which numerically increments an
+  asterisk and plus respectively
+
+  *1 my note
+  +1 another note
+
+WOK
+    end
+    def sisu_0_38
+      <<WOK
+
+  SiSU 0.38 (a subset of 1.0) introduced alternative experimental header and heading/structure markers,
+
+  @headername: and headers :A~ :B~ :C~ 1~ 2~ 3~
+
+  as the equivalent of (the superceded)
+
+  0~headername and headers 1~ 2~ 3~ 4~ 5~ 6~
+
+  The internal document markup of SiSU 0.16 remains valid and standard
+  Though note that SiSU 0.37 introduced a new file naming convention
+
+  SiSU has in effect two sets of levels to be considered, using 0.38 notation
+  A-C headings/levels, pre-ordinary paragraphs /pre-substantive text, and
+  1-3 headings/levels, levels which are followed by ordinary text.
+    This may be conceptualised as levels A,B,C, 1,2,3, and using such letter
+    number notation, in effect:
+      A must exist, optional B and C may follow in sequence (not strict)
+      1 must exist, optional 2 and 3 may follow in sequence
+    i.e. there are two independent heading level sequences A,B,C and 1,2,3
+      (using the 0.16 standard notation 1,2,3 and 4,5,6)
+    on the positive side:
+      * the 0.38 A,B,C,1,2,3 alternative makes explicit an aspect of structuring
+        documents in SiSU that is not otherwise obvious to the newcomer (though
+        it appears more complicated, is more in your face and likely to be
+        understood fairly quickly)
+      * the substantive text follows levels 1,2,3 and it is 'nice' to do
+        most work in those levels
+WOK
+    end
+    def sisu_0_37
+      <<WOK
+
+  SiSU 0.37 introduced the file naming convention, that remains in use in SiSU
+  v1 and v2, using the file extensions .sst .ssm and .ssi
+  to replace .s1 .s2 .s3 .r1 .r2 .r3  and .si
+
+  this is captured by the following file 'rename' instruction:
+
+  rename 's/\.s[123]$/\.sst/' *.s{1,2,3}
+  rename 's/\.r[123]$/\.ssm/' *.r{1,2,3}
+  rename 's/\.si$/\.ssi/' *.si
+
+  The internal document markup remains unchanged, from SiSU 0.16
+WOK
+    end
+    def sisu_0_16
+      <<WOK
+
+  SiSU 0.16 (0.15 development branch) introduced the use of
+
+  the header 0~ and headings/structure 1~ 2~ 3~ 4~ 5~ 6~
+
+  in place of the 0.1 header, heading/structure notation
+WOK
+    end
+    def sisu_0_1
+      <<WOK
+
+  SiSU 0.1 headers and headings structure represented by
+  header 0{~ and headings/structure  1{ 2{ 3{ 4{~ 5{ 6{
+WOK
+    end
+    def help_query
+      <<WOK
+
+  sisu --query=[sisu version [0.38] or 'history]
+  provides a short history of changes to SiSU markup
+
+WOK
+    end
+    def help_identify
+      <<WOK
+
+  sisu --identify [filename]
+  attempts to identify the SiSU markup used in a file
+
+WOK
+    end
+    def query
+      tell=if @opt.selections.str =~/--query/
+        tell=case @opt.selections.str
+        when /history/
+          "#{sisu_3_0}#{sisu_2_0}#{sisu_1_0}#{sisu_0_69}#{sisu_0_66}#{sisu_0_57}#{sisu_0_42}#{sisu_0_38}\n#{sisu_0_37}\n#{sisu_0_16}\n#{sisu_0_1}"
+        when /3.0/
+          "#{sisu_3_0}#{sisu_2_0}#{sisu_1_0}#{sisu_0_69}#{sisu_0_66}#{sisu_0_57}#{sisu_0_42}#{sisu_0_38}#{sisu_0_16}"
+        when /2.0/
+          "#{sisu_2_0}#{sisu_1_0}#{sisu_0_69}#{sisu_0_66}#{sisu_0_57}#{sisu_0_42}#{sisu_0_38}#{sisu_0_16}"
+        when /1.0/
+          "#{sisu_1_0}#{sisu_0_69}#{sisu_0_66}#{sisu_0_57}#{sisu_0_42}#{sisu_0_38}#{sisu_0_16}"
+        when /0.69/
+          "#{sisu_0_69}#{sisu_0_66}#{sisu_0_57}#{sisu_0_42}#{sisu_0_38}#{sisu_0_16}"
+        when /0.66/
+          "#{sisu_0_66}#{sisu_0_57}#{sisu_0_42}#{sisu_0_38}#{sisu_0_16}"
+        when /0.65/
+          "#{sisu_0_65}#{sisu_0_57}#{sisu_0_42}#{sisu_0_38}#{sisu_0_16}"
+        when /0.57/
+          "#{sisu_0_57}#{sisu_0_42}#{sisu_0_38}#{sisu_0_16}"
+        when /0.42/
+          "#{sisu_0_42}#{sisu_0_38}#{sisu_0_16}"
+        when /0.38/
+          "#{sisu_0_38}#{sisu_0_16}"
+        when /0.37/
+          "#{sisu_0_37}\n#{sisu_0_16}"
+        when /0.1[6-9]|0.2[0-9]|0.3[0-6]/
+          "#{sisu_0_16}\n#{sisu_0_1}"
+        when /0.[1-9]|0.1[1-4]/
+          sisu_0_1
+        else puts "NOT RECOGNISED: #{@opt.selections.str}"
+          help_query
+        end
+        tell
+      else help_query
+      end
+    end
+  end
+end
+__END__
+#%% to use as independent program ------------------------->
+f=$*
+cf=f[0].to_s
+f.shift
+match_and_replace=[]
+unless f.length > 0; f=Dir.glob("[a-z]*.ss?")  #restricted to sisu type files, it need not be
+end
+puts "SiSU files:"
+puts f
+f.each do |x|
+  SiSU_Markup::MarkupIdentify.new(x).markup_version?
+end
+#+END_SRC
+
+* sst_do_inline_footnotes.rb
+
+#+HEADER: :tangle "../lib/sisu/sst_do_inline_footnotes.rb"
+#+BEGIN_SRC ruby
+#<<sisu_document_header>>
+module SiSU_ConvertFootnotes
+  require_relative 'se'                                 # se.rb
+    include SiSU_Env
+  require_relative 'dp'                                 # dp.rb
+    include SiSU_Param
+  require_relative 'ao_syntax'                          # ao_syntax.rb
+    include SiSU_AO_Syntax
+  require_relative 'i18n'                               # i18n.rb
+  class Instantiate < SiSU_Param::Parameters::Instructions
+     @@flag={} #Beware!!
+    def initialize
+      @@flag['table_to']=false
+      @@counter=@@column=@@columns=@@flag_vocab=0
+      @@endnote={}
+      @@endnote_array=@@word_mode=[]
+      @@endnote_call_counter=1
+      @@line_mode=''
+    end
+  end
+  class Source <Instantiate
+    @@ao_array=[]
+    @@fns=nil
+    def initialize(opt)
+      @opt=opt
+      @@fns||@opt.fns
+      @my_make=SiSU_Env::CreateFile.new(@opt.fns)
+      @fnm=SiSU_Env::InfoFile.new(@opt.fns).marshal.ao_content
+    end
+    def read                                                                     #creates ao
+      begin
+        @@ao_array=[]
+        @@fns=@opt.fns
+        create_ao
+      rescue
+        SiSU_Errors::Rescued.new($!,$@,@opt.cmd,@opt.fns).location do
+          __LINE__.to_s + ':' + __FILE__
+        end
+      ensure
+        Instantiate.new
+      end
+    end
+    def get                                                                      #reads ao, unless does not exist then creates first
+      begin
+        ao=[]
+        unless @@fns==@opt.fns
+          @@fns=@opt.fns
+          @@ao_array=[]
+        end
+        ao=(@@ao_array.empty?) \
+        ? read_fnm
+        : @@ao_array.dup #check
+      rescue
+        SiSU_Errors::Rescued.new($!,$@,@opt.cmd,@opt.fns).location do
+          __LINE__.to_s + ':' + __FILE__
+        end
+      ensure
+        Instantiate.new
+      end
+    end
+  protected
+    def create_ao
+      ao_array=[]
+      SiSU_Screen::Ansi.new(
+        @opt.act[:color_state][:set],
+        'convert footnotes'
+      ).green_title_hi unless @opt.act[:quiet][:set]==:on
+      file_array=IO.readlines(@opt.fns,'')
+      file_array.each do |l|
+        if l =~/\r\n/ then l.gsub!(/\r\n/,"\n")
+        end
+      end
+      meta=file_array.dup
+      meta=meta.join.split("\n\n") #check whether can be eliminated, some of these are large objects to have twice
+      @md=SiSU_Param::Parameters::Instructions.new(meta,@opt).extract
+      if @md.en[:mismatch]==0 \
+      or @md.opt.selections.str =~/=footnotes-force/
+        meta=nil
+        ao=SiSU_ConvertFootnotes::Make.new(@md,file_array).song
+        SiSU_Screen::Ansi.new(
+          @opt.act[:color_state][:set],
+          @opt.fns,
+          "#{@md.fns}.fn"
+        ).output if @md.opt.act[:verbose][:set]==:on
+        SiSU_Screen::Ansi.new(
+          @opt.act[:color_state][:set],
+          "#{@md.fns}.fn -> #{@md.fns}.fn"
+        ).txt_red unless @md.opt.act[:quiet][:set]==:on
+        ao.each {|s| ao_array << "#{s.strip}\n\n" unless s.strip.empty?}
+        ao_array
+      else
+        SiSU_Screen::Ansi.new(
+          @md.opt.act[:color_state][:set],
+          '*WARN* no footnote conversion done, problem with source file',
+          'to override use --convert=footnote-force (this is not advised)'
+        ).warn unless @md.opt.act[:quiet][:set]==:on
+        ''
+      end
+    end
+    def read_fnm
+      ao=[]
+      ao=(FileTest.file?(@fnm)) \
+      ? (File.open(@fnm){ |f| ao=Marshal.load(f)})
+      : (SiSU_ConvertFootnotes::Source.new(@opt).create_ao) #watch
+    end
+  end
+  class Output
+    def initialize(md,data)
+      @md,@data=md,data
+      @my_make=SiSU_Env::CreateFile.new(@md.fns)
+      SiSU_Env::InfoEnv.new(@md.fns)
+      @hard="#{Dir.pwd}/#{@md.fns}.fn"
+    end
+    def hard_output
+      filename_note=@my_make.file_note
+      @data.each {|s| filename_note.puts s.strip + "\n\n" unless s.strip.empty?}
+    end
+  end
+  class Make
+    @@endnote={}
+    @@endnote_array=@@word_mode=[]
+    @@endnote_call_counter=1
+    @@comment='%'
+    @@flag={ ['table_to']=>false }
+    def initialize(md,data)
+      @md,@data=md,data
+      @@word_mode=[]
+      @env=SiSU_Env::InfoEnv.new(@md.fns)
+      l=SiSU_Env::StandardiseLanguage.new(@md.opt.lng).language
+      @language=l[:n]
+      @translate=SiSU_Translate::Source.new(@md,@language)
+    end
+    def reset
+      @@counter=@@column=@@columns=@@flag_vocab=0
+      @@endnote={}
+      @@endnote_array=@@word_mode=[]
+      @@endnote_call_counter=1
+      @@line_mode=''
+    end
+    def song
+      reset
+      data=@data
+      @metafile="#{@env.processing_path.ao}/#{@md.fns}.meta"
+      SiSU_Env::CreateFile.new(@md.fns)
+      data=data.join.split("\n\n")
+      data_new=[]
+      data.each do |x|
+        data_new << (x =~ /\n\n/m) \
+        ? (x.split(/\n\n+/))
+        : x
+      end
+      data=data_new.flatten
+      data=SiSU_ConvertFootnotes::Make.new(@md,data).character_check
+      data=SiSU_ConvertFootnotes::Make.new(@md,data).endnotes
+      SiSU_ConvertFootnotes::Output.new(@md,data).hard_output
+      reset
+      data
+    end
+  protected
+    def vocabulary
+      data=@data
+      tuned_file,vocab_insert=[],[]
+      data.each do |para|
+        if para =~/^1~/ \
+        and @@flag_vocab==0
+          vocab_insert << '@vocabulary: lex' << "\n\n" << para
+          tuned_file << vocab_insert unless para.nil?
+          @@flag_vocab=1
+        else tuned_file << para unless para.nil?
+        end
+      end
+      tuned_file
+    end
+    def character_check
+      reset
+      data=@data
+      @tuned_file=[]
+      endnote_no=1
+      data.each do |para|
+        para.strip!
+        para.gsub!(/^[{~}]\s*$/,'')
+        para.gsub!(/^#{@@comment}.*/,'')                                       #remove comment and divider #%
+        para.gsub!(/<~#>|~#\s*/,'~#')
+        para.gsub!(/-#\s*/,'-#')
+        para.gsub!(/(~\{ )\s+/,'\1')
+        para.gsub!(/ \/\//,'<br />')                                           #added 2004w29
+        para.gsub!(/<br>/,'<br />')                                            #needed by xml, xhtml etc.
+        para.gsub!(/`/,"'")
+        para.gsub!(/\342\200\231/,"'") #if para =~/’/  #Avoid #&lsquo; &rsquo; #&ldquo; &rdquo;
+        para.gsub!(/\t/,' ')
+        para.gsub!(/�/,' ') #watch, replace with char code
+        para.gsub!(/[“”]/,'""')
+        para.gsub!(/[­–—]/,'-')   #— – chk
+        para.gsub!(/·/,'*')
+        para.gsub!(/\\copy(?:right)?\b/,'&#169;')
+        para.gsub!(/\\trademark\b|\\tm\b/,'&#174;')
+        para.gsub!(/\44/,'&#36;') #$ watch
+        para=para + "\n"
+        case para
+        when /\^~/                                                             # endnotes
+                                                                               #% Note must do this first (earlier loop) and then enter gathered data into ~^\d+
+          sub_para=para.dup
+          @@endnote_array << sub_para.gsub!(/\n/,'').gsub!(/\^~\s+(.+)\s*/,'~{ \1 }~').strip
+           endnote_no+=1
+          para=nil if para =~/\^~ .+/ #removes 'binary' endnote now in endnote array for later insertion
+        end
+        @tuned_file << para unless para.nil?
+      end
+      @tuned_file
+    end
+    def name_endnote_seg
+      data=@data
+      @tuned_file=[]
+      data.each do |para|
+        para.gsub!(/<:3>\s*<:ee>/,
+          "#{@@endnote['special_align']} <p /><br />\r " +
+          "#{@@endnote['seg_name_3']} <p /> " +
+          "#{@@endnote['special_align_close']}")
+        para.gsub!(/<:2>\s*<:ee>/,
+          "#{@@endnote['special_align']} <p /><br />\r " +
+          "#{@@endnote['seg_name_2']} <p />" +
+          "#{@@endnote['special_align_close']}")
+        para.gsub!(/<:1>\s*<:ee>/,
+          "#{@@endnote['special_align']} <p /><br />\r " +
+          "#{@@endnote['seg_name_1']} <p /> " +
+          "#{@@endnote['special_align_close']}")
+        @tuned_file << para
+      end
+      if @md.flag_auto_endnotes \
+      and @md.flag_separate_endnotes_make
+        @tuned_file << "\n1~endnotes Endnotes" #prob numbering, revisit
+      end
+      @tuned_file << "\n<ENDNOTES>"
+      @tuned_file
+    end
+    def owner_details_seg
+      data << '1~owner.details Owner Details'
+    end
+    def number_sub_heading(para,num,title_no)
+      case para
+      when /#{num}~- /    then para.gsub!(/#{num}~- /,"#{title_no} ")
+      when /^#{num}~#\s*/ then para.gsub!(/^#{num}~#\s*/,"#{title_no} ")
+      when /^#{num}~[a-z_\.]+ /
+        para.gsub!(/^#{num}~([a-z_\.]+)\s+(.+)/i,%{#{num}~\\1 #{title_no} \\2  <:name##{title_no}>})
+      else para.gsub!(/^#{num}~ /,"#{num}~#{title_no} #{title_no} ") #main
+      end
+      if @md.toc_lev_limit \
+      and @md.toc_lev_limit < num
+        para.gsub!(/^[2-6]~(?:~\S+)?\s*/,'!_ ')
+      end
+      para
+    end
+    def set_heading_top                                                        #% make sure no false positives
+      unless @md.set_heading_top
+        if (@md.opt.act[:verbose_plus][:set]==:on \
+        || @md.opt.act[:maintenance][:set]==:on)
+          puts "\tdocument contains no top level heading, (will have to manufacture one)"
+        end
+        data=@data
+        @tuned_file=[]
+        data.each do |para|
+          unless @md.set_heading_top
+            if para !~/^(?:@\S+:|0~\S+)\s/m \
+            and para !~/\A\s*\Z/m
+              @md.set_heading_top=true
+              head=(@md.title.full) \
+              ? (":A~ #{@md.title.full}")
+              : (':A~ [no title provided]')
+              @tuned_file << head
+            end
+          end
+          @tuned_file << para
+        end
+        @tuned_file
+      end
+    end
+    def set_heading_seg                                                        #% make sure no false positives
+      unless @md.set_heading_seg
+        if (@md.opt.act[:verbose_plus][:set]==:on \
+        || @md.opt.act[:maintenance][:set]==:on)
+          puts "\tdocument contains no segment level, (will have to manufacture one)"
+        end
+        data=@data
+        @tuned_file=[]
+        data.each do |para|
+          unless @md.set_heading_seg
+            if para !~/^(?:@\S+:|0~\S+|:[ABC]~)/m \
+            and para !~/\A\s*\Z/m \
+            and para !~/<:p[bn]>/
+              @md.set_heading_seg=true
+              head=(@md.title.full) \
+              ? ("1~seg [#{@md.title.full}]")
+              : ('1~seg [segment]')
+              @tuned_file << head
+            end
+          end
+          @tuned_file << para
+        end
+        @tuned_file
+      end
+    end
+    def set_header_title                                                       #% make sure no false positives
+      unless @md.set_header_title
+        if (@md.opt.act[:verbose_plus][:set]==:on \
+        || @md.opt.act[:maintenance][:set]==:on)
+          puts "\t no document title provided, (will have to manufacture one)"
+        end
+        data=@data
+        @tuned_file=[]
+        data.each do |para|
+          unless @md.set_header_title
+            if para !~/^%{1,2}\s/m \
+            and para !~/\A\s*\Z/m
+              @tuned_file << "0~title #{@md.heading_seg_first}"
+              @md.title.full=@md.heading_seg_first
+              @md.set_header_title=true
+            end
+          end
+          @tuned_file << para
+        end
+        @tuned_file
+      end
+    end
+    def endnotes                                                                         #% endnote work zone
+      data=@data
+      @tuned_file=[]
+      endnote_ref=1
+      data.each do |para|
+        case para                                                                               # manually numbered endnotes <!e(\d)!> <!e_(\d)!> -->
+        when /~\{\s+.+?\}~/                                                                               # auto-numbered endnotes <!e!> <!e_!> -->
+          para.gsub!(/\s*\}~/,' }~')                                           # required 2003w31
+          @word_mode=para.scan(/\S+/)
+          word_mode=SiSU_ConvertFootnotes::Make.new(@md,@word_mode).endnote_call_number
+          para=word_mode.join(' ')
+          endnote_ref+=1
+        when /~\^(?:\s|$)|<:e>/                                                #%Note inserts endnotes previously gathered from /^(<!e[:_]!>|[-~]\{{3})/ (in earlier loop)
+          word_mode=para.scan(/\S+/)
+          word_mode=SiSU_ConvertFootnotes::Make.new(@md,word_mode).endnote_call_number
+          para=word_mode.join(' ')
+          endnote_ref+=1
+        end
+        @tuned_file << para
+      end
+      @tuned_file
+    end
+    def endnote_call_number
+      data=@data
+      data.each do |word|
+        case word
+        when /~\{/
+          unless word =~/~\{\*+/
+            @@endnote_call_counter+=1
+          end
+        when /~\^|<:e>/
+          word.gsub!(/~\^|<:e>/,"#{@@endnote_array[@@endnote_call_counter-1]}")
+          @@endnote_call_counter+=1
+        end
+      end
+    end
+    def strip_clean_extra_spaces(s)                                            # ao output tuned
+      s=s.dup
+      s=s.gsub(/[ ]+([,.;:?](?:$|\s))/,'\1')
+      s=s.gsub(/ [ ]+/,' ')
+      s=s.gsub(/^ [ ]+/,'')
+      s=s.gsub(/ [ ]+$/,'')
+      s=s.gsub(/(<\/[bi]>')[ ]+(s )/,'\1\2')
+    end
+    def strip_clean_of_markup(s)                                               # used for digest, define rules, make same as in db clean
+      s=s.dup
+      s=s.gsub(/(?:<\/?[ib]>|^:[A-C]~\S+|^[1-6]~\S+|~\{\d+\s.+?\}~)/,'') # markup and endnotes removed
+                                                                               #% same as db clean -->
+      s=s.gsub(/<del>(.+?)<\/del>/,'DELETED(\1)')                           # deletions
+      s=s.gsub(/<sup>(\d+)<\/sup>/,'[\1]')
+      s=s.gsub(/(?:&nbsp\\;|#{Mx[:nbsp]})+/,' ')  #checking source Mx not necessary
+      s=s.gsub(/\{.+?\.(?:png|jpg|gif).+?\}(?:https?|file|ftp)\\\:\S+ /,' [image] ')             # else image names found in search
+      s=s.gsub(/#{Mx[:lnk_o]}.+?\.(?:png|jpg|gif).+?#{Mx[:lnk_c]}#{Mx[:url_o]}\S+?#{Mx[:url_c]}/,' [image] ')             # else image names found in search, re-check
+      s=s.gsub(/\s\s+/,' ')
+      s=s.strip
+    end
+  end
+end
+__END__
+@particulars=SiSU_Particulars::CombinedSingleton.instance.get_all(opt)
+ao_array=@particulars.ao_array # ao file drawn here
+#+END_SRC
+
+* sst_convert_markup.rb
+
+#+HEADER: :tangle "../lib/sisu/sst_convert_markup.rb"
+#+BEGIN_SRC ruby
+#<<sisu_document_header>>
+module SiSU_Modify
+  require_relative 'sst_identify_markup'                # sst_identify_markup.rb
+  require_relative 'sst_from_xml'                       # sst_from_xml.rb
+  require_relative 'utils_response'                     # utils_response.rb
+  class ConvertMarkup
+    include SiSU_Response
+    def initialize(opt)
+      @opt=opt
+      @description='This is a script that contains canned text conversions for reuse'
+      @warn='WARNING, PROCEED AT YOUR OWN RISK, will make file changes.'
+    end
+    def current_match_and_replace
+      convert_37_to_38
+    end
+    def message(text)
+      response=''
+      unless @opt.cmd=~/QQ/ \
+      or @opt.act[:quiet][:set]==:on
+        response=response?(%{#{  text}\nProceed? })
+      end
+    end
+    def help
+    print <<WOK
+
+#{@description}
+
+sisu --convert --to38 [filename/wildcard]
+  converts pre 0.37 sisu markup to 0.38 experimental
+  [--37to38]
+
+sisu --convert --to37 [filename/wildcard]
+  converts pre 0.37 sisu markup to 0.38 experimental
+  [--38to37]
+
+sisu --convert --36to37 [filename/wildcard]
+  converts pre 0.36 file-name, to 0.37 file-name
+  [--36to37]
+
+sisu --identify [filename]
+  attempts to identify markup version used in file
+
+sisu --query [version number]
+  gives short summary of distinguishing characteristic
+  of that version of markup
+
+WOK
+      exit
+    end
+    #%% substitutions to be made
+    def convert_37_to_38
+      message("#{@warn}\nConvert sisu markup from 0.37 to 0.38")
+      [
+        [/^0~(\S+?)([+-])\s+/,    '@\1:\2 ',               //],
+        [/^0~(\S+)\s+/,           '@\1: ',                 //],
+        [/^@toc:\s+/,             '@structure: ',          //],
+        [/^1~/,                   ':A~',                   //],
+        [/^2~/,                   ':B~',                   //],
+        [/^3~/,                   ':C~',                   //],
+        [/^4~/,                   '1~',                    //],
+        [/^5~/,                   '2~',                    //],
+        [/^6~/,                   '3~',                    //],
+        [/^7~/,                   '4~',                    //],
+        [/^8~/,                   '5~',                    //],
+        [/^9~/,                   '6~',                    //],
+        [/1/,                     ':A',                    /^@(?:level|markup):\s/],
+        [/2/,                     ':B',                    /^@(?:level|markup):\s/],
+        [/3/,                     ':C',                    /^@(?:level|markup):\s/],
+        [/4/,                     '1',                     /^@(?:level|markup):\s/],
+        [/5/,                     '2',                     /^@(?:level|markup):\s/],
+        [/6/,                     '3',                     /^@(?:level|markup):\s/]
+      ]
+    end
+    def convert_38_to_37
+      message("#{@warn}\nConvert sisu markup from 0.38 to 0.37")
+      [
+        [/^@(\S+?):([+-])\s+/,    '0~\1\2 ',               //],
+        [/^@(\S+?):\s+/,          '0~\1 ',                 //],
+        [/^0~structure\s+/,       '0~toc ',                //],
+        [/^1~/,                   '4~',                    //],
+        [/^2~/,                   '5~',                    //],
+        [/^3~/,                   '6~',                    //],
+        [/^4~/,                   '7~',                    //],
+        [/^5~/,                   '8~',                    //],
+        [/^6~/,                   '9~',                    //],
+        [/^:?A~/,                 '1~',                    //],
+        [/^:?B~/,                 '2~',                    //],
+        [/^:?C~/,                 '3~',                    //],
+        [/1/,                     '4',                     /^0~(?:level|markup)\s/],
+        [/2/,                     '5',                     /^0~(?:level|markup)\s/],
+        [/3/,                     '6',                     /^0~(?:level|markup)\s/],
+        [/:?A/,                   '1',                     /^0~(?:level|markup)\s/],
+        [/:?B/,                   '2',                     /^0~(?:level|markup)\s/],
+        [/:?C/,                   '3',                     /^0~(?:level|markup)\s/]
+      ]
+    end
+    def convert_filename_36_to_37
+      @opt.files.each do |f|
+        s=case f
+        when /(\.s[1-3])$/ then f.sub($1,'.sst')
+        when /(\.r[1-3])$/ then f.sub($1,'.ssm')
+        when /(\.ri)$/     then f.sub($1,'.ssi')
+        else f
+        end
+        pwd=Dir.pwd
+        unless f==s
+          unless File.exist?("#{pwd}/#{s}")
+            puts "./#{f} -> ./#{s}"
+            FileUtils::cp("#{pwd}/#{f}","#{pwd}/#{s}")
+          else "File already exists, < #{s} >  will not overwrite"
+          end
+        end
+      end
+    end
+    def convert_to_simple_xml_model_sax
+      SiSU_SimpleXML_ModelSax::Convert.new(@opt).read
+    end
+    def convert_to_simple_xml_model_dom
+      SiSU_simple_xml_model_dom::Convert.new(@opt).read
+    end
+    def convert_to_simple_xml_model_node
+      SiSU_simple_xml_model_node::Convert.new(@opt).read
+    end
+    def convert_kdi_to_sst
+      SiSU_Kdissert::Convert.new(@opt).read
+    end
+    def convert_s_xml_to_sst
+      SiSU_sstFromXML::Convert.new(@opt).read
+    end
+    def convert_footnotes
+      require_relative 'sst_do_inline_footnotes'
+      SiSU_ConvertFootnotes::Source.new(@opt).read
+    end
+    def conversion
+      #%% do it                                   -------------------------->
+      if @opt.files \
+      and @opt.files.length > 0
+        mr=nil
+        #%% changes to make m match, r replace      -------------------------->
+        if @opt.selections.str =~/--help/ then help
+        elsif @opt.selections.str =~/(?:convert|to)[=-](?:xml |sxs|sax|sxd|dom|sxn|node)/
+          ext=case @opt.selections.str
+          when /(?:convert|to)[=-](?:xml|sxs|sax)/ then '.sxs.xml'
+          when /(?:convert|to)[=-](?:sxd|dom)/     then '.sxd.xml'
+          when /(?:convert|to)[=-](?:sxn|node)/    then '.sxn.xml'
+          end
+          message("#{@opt.files.inspect}\n\nWARNING, PROCEED AT YOUR OWN RISK,\noverwriting any equivalent file with the extension #{ext}")
+          mr=case @opt.selections.str
+          when /(?:convert|to)[=-](?:sxs|sax|xml )/ then convert_to_simple_xml_model_sax
+          when /(?:convert|to)[=-](?:sxd|dom)/      then convert_to_simple_xml_model_dom
+          when /(?:convert|to)[=-](?:sxn|node)/     then convert_to_simple_xml_model_node
+          else help
+          end
+        else
+          mr=case @opt.selections.str
+          when /(?:(?:37)?to-?38|--(?:convert|to)[=-](?:current|0.38))/           then convert_37_to_38
+          when /(?:(?:38)?to-?37|--(?:convert|to)[=-](?:0.37))/                   then convert_38_to_37
+          when /(?:36to37)/                                                       then convert_filename_36_to_37
+          when /(?:convert|from)[=-]kdi/                                          then convert_kdi_to_sst
+          when /(?:(?:convert|from)[=-])?(?:xml_to_sst|xml2sst|sxml|sxs|sxd|sxd)/ then convert_s_xml_to_sst
+          when /(?:convert|to)[=-]footnotes/                                      then convert_footnotes
+          when /convert|default/                                                  then current_match_and_replace
+          else help
+          end
+        end
+        unless @opt.selections.str =~/kdi/
+          match_and_replace=mr
+          #start_processing =/not used in this example/i
+          end_processing =/END\s+OF\s+FILE/
+          i=@opt.fns
+          if i =~/(?:\.sst|\.ssm|\.ssi)$/
+            @new,@matched,@flag_start,@flag_end,@empty1,@empty2=true,false,false,false,false,false
+            o="#{i}.bk" #o is for old
+            markup_version=SiSU_Markup::MarkupIdentify.new(@opt).markup_version?
+            if (@opt.selections.str=~/37/ and markup_version=~/0.38/) \
+            or (@opt.selections.str=~/current|38/ and markup_version=~/0.37/)
+              puts "#{i} #{markup_version}"
+              file=File.open(i,'r')
+              cont=file.readlines
+              file.close
+              cont.each do |y|
+                match_and_replace.each do |m,r,w|
+                  if y =~m \
+                  and y =~w
+                    if @new
+                      @new=false
+                      File.unlink(o) if File.exist?(o)
+                      File.rename(i,o)
+                      File.unlink(i) if File.exist?(i)
+                      @file=File.new(i,'w')
+                      @matched=true
+                      break
+                    end
+                  end
+                end
+              end
+              if @matched
+                puts "conversion match in #{i}" unless @opt.act[:quiet][:set]==:on
+                @flag_start=true
+                cont.each do |y|
+                  if y =~end_processing
+                    @flag_end=true
+                  end
+                  if @flag_start \
+                  and not @flag_end
+                    match_and_replace.each do |m,r,w|
+                      if y =~m \
+                      and y =~w
+                        puts m.inspect + ' -> ' + r unless @opt.act[:quiet][:set]==:on
+                        if (@opt.act[:verbose][:set]==:on \
+                        || @opt.act[:verbose_plus][:set]==:on \
+                        || @opt.act[:maintenance][:set]==:on)
+                          puts "in:  #{y}"
+                        end
+                        y.gsub!(m,r) if m and r
+                        if (@opt.act[:verbose][:set]==:on \
+                        || @opt.act[:verbose_plus][:set]==:on \
+                        || @opt.act[:maintenance][:set]==:on)
+                          puts "out: #{y}"
+                        end
+                      end
+                    end
+                  end
+                  @empty1=(y=~/^\s*$/) \
+                  ? true
+                  : false
+                  @file.puts y unless (@empty1==true and @empty2==true)
+                  @empty2=(y=~/^\s*$/) \
+                  ? true
+                  : false
+                end
+                @file.close
+              else puts "NO conversion match in #{i}" unless @opt.act[:quiet][:set]==:on
+              end
+            else
+              if (@opt.act[:verbose][:set]==:on \
+              || @opt.act[:verbose_plus][:set]==:on \
+              || @opt.act[:maintenance][:set]==:on)
+                puts "Requested conversion #{@opt.selections.str} markup #{markup_version} identified in #{i}"
+              end
+            end
+          end
+        end
+      else puts 'this routine makes permanent changes to the contents of the files matched, as instructed within [no matches]'
+      end
+    end
+  end
+end
+#%% files to match for this conversion set  ------------------------->
+require_relative 'hub_options'                          # hub_options.rb
+argv=$*
+base_path=Dir.pwd
+@opt=SiSU_Commandline::Options.new(argv,base_path)
+case @opt.selections.str
+when /=kdi/
+  SiSU_Modify::ConvertMarkup.new(@opt).conversion
+when /(?:36|37|38)?to-?(?:37|38)|--convert|--to|--from|default/
+@opt.files.each do |fns|
+  @opt.fns=fns
+  SiSU_Modify::ConvertMarkup.new(@opt).conversion
+end
+else
+  @opt.selections.str='--help'
+ SiSU_Modify::ConvertMarkup.new(@opt).help
+end
+__END__
+#+END_SRC
+
+* document header
+
+#+NAME: sisu_document_header
+#+BEGIN_SRC text
+encoding: utf-8
+- Name: SiSU
+
+  - Description: documents, structuring, processing, publishing, search
+    sst
+
+  - Author: Ralph Amissah
+    <ralph.amissah@gmail.com>
+
+  - Copyright: (C) 1997, 1998, 1999, 2000, 2001, 2002, 2003, 2004, 2005, 2006,
+    2007, 2008, 2009, 2010, 2011, 2012, 2013, 2014, 2015, 2016, 2017, 2019,
+    2020, 2021, Ralph Amissah,
+    All Rights Reserved.
+
+  - License: GPL 3 or later:
+
+    SiSU, a framework for document structuring, publishing and search
+
+    Copyright (C) Ralph Amissah
+
+    This program is free software: you can redistribute it and/or modify it
+    under the terms of the GNU General Public License as published by the Free
+    Software Foundation, either version 3 of the License, or (at your option)
+    any later version.
+
+    This program is distributed in the hope that it will be useful, but WITHOUT
+    ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+    FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
+    more details.
+
+    You should have received a copy of the GNU General Public License along with
+    this program. If not, see <http://www.gnu.org/licenses/>.
+
+    If you have Internet connection, the latest version of the GPL should be
+    available at these locations:
+    <http://www.fsf.org/licensing/licenses/gpl.html>
+    <http://www.gnu.org/licenses/gpl.html>
+
+    <http://www.sisudoc.org/sisu/en/manifest/gpl.fsf.html>
+
+  - SiSU uses:
+    - Standard SiSU markup syntax,
+    - Standard SiSU meta-markup syntax, and the
+    - Standard SiSU object citation numbering and system
+
+  - Homepages:
+    <http://www.sisudoc.org>
+
+  - Git
+    <https://git.sisudoc.org/projects/>
+    <https://git.sisudoc.org/projects/?p=software/sisu.git;a=summary>
+    <https://git.sisudoc.org/projects/?p=markup/sisu-markup-samples.git;a=summary>
+#+END_SRC