# encoding: utf-8
=begin

* Name: SiSU

** Description: documents, structuring, processing, publishing, search
*** Syntax for markup, input markup syntaxes, determined here

** Author: Ralph Amissah
  <ralph@amissah.com>
  <ralph.amissah@gmail.com>

** Copyright: (C) 1997, 1998, 1999, 2000, 2001, 2002, 2003, 2004, 2005, 2006,
  2007, 2008, 2009, 2010, 2011, 2012, 2013, 2014, 2015 Ralph Amissah,
  All Rights Reserved.

** License: GPL 3 or later:

  SiSU, a framework for document structuring, publishing and search

  Copyright (C) Ralph Amissah

  This program is free software: you can redistribute it and/or modify it
  under the terms of the GNU General Public License as published by the Free
  Software Foundation, either version 3 of the License, or (at your option)
  any later version.

  This program is distributed in the hope that it will be useful, but WITHOUT
  ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
  FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
  more details.

  You should have received a copy of the GNU General Public License along with
  this program. If not, see <http://www.gnu.org/licenses/>.

  If you have Internet connection, the latest version of the GPL should be
  available at these locations:
  <http://www.fsf.org/licensing/licenses/gpl.html>
  <http://www.gnu.org/licenses/gpl.html>

  <http://www.sisudoc.org/sisu/en/manifest/gpl.fsf.html>

** SiSU uses:
  * Standard SiSU markup syntax,
  * Standard SiSU meta-markup syntax, and the
  * Standard SiSU object citation numbering and system

** Hompages:
  <http://www.jus.uio.no/sisu>
  <http://www.sisudoc.org>

** Git
  <http://git.sisudoc.org/gitweb/?p=code/sisu.git;a=summary>
  <http://git.sisudoc.org/gitweb/?p=code/sisu.git;a=blob;f=lib/sisu/develop/ao_syntax.rb;hb=HEAD>

=end
module SiSU_AO_Syntax
  class Words
    def initialize(line,md,mkp)
      @line,@md,@mkp=line,md,mkp
    end
  end
  class Markup
    def initialize(md='',data='',biblio=[])
      @md,@data,@bibliography=md,data,biblio
      @data_new=[]
      url_and_stub=SiSU_Env::InfoEnv.new.url
      @output_url="#{url_and_stub.remote}"
      @env=SiSU_Env::InfoEnv.new
      emph_set=if defined? @md.emphasis_set_to \
      and not @md.emphasis_set_to.nil?
        @md.emphasis_set_to
      else @env.markup_emphasis
      end
      @emph=case emph_set
      when /bold/
        emph_italics=false
        { o: Mx[:fa_bold_o], c: Mx[:fa_bold_c] }
      when /italics/
        emph_italics=true
        { o: Mx[:fa_italics_o], c: Mx[:fa_italics_c] }
      when /underscore/
        emph_italics=false
        { o: Mx[:fa_underscore_o], c: Mx[:fa_underscore_c] }
      else p __LINE__.to_s + '::' + __FILE__
      end
      @http_m=%r{\{.+?\}https?://\S+|https?:\S+|:\S+|\.\.\/\S+|#\S+|\S+?\.png\b|[*]~\S+|^#{Mx[:meta_o]}.+|#{Mx[:gr_o]}(?:code(?:\.[a-z][0-9a-z_]+)?|box(?:\.[a-z_]+)?|block|group|alt|verse)(?:-end)?#{Mx[:gr_c]}|#{Mx[:fa_o]}:br#{Mx[:fa_c]}}
      @manmkp_ital=emph_italics \
      ? '[i/*]\\{.+?\\}[i/*]'
      : '[i/]\\{.+?\\}[i/]'
      tail_m_ital=%q{(?:\s|'s\b|[.,;:?!'")]|~\^|~\\\{\s|$)}
      tail_m_bold=%{(?:(?:#{Mx[:fa_italics_c]})?(?:\s|'s\b|[.,;:?!'")]|~\^|~\\\{\s|$))?}
      bold_line=%{^!_\s.+?(?:#{Mx[:br_line]}|\n|$)}
      #ital_line=%{^/_\s.+?(?:#{Mx[:br_line]}|\n|$)} #not implemented
      @line_scan_ital=if defined? @md.italics_match_list[:str]
        /#{@http_m}|#{bold_line}|#{@manmkp_ital}#{tail_m_ital}|#{@md.italics_match_list[:str]}#{tail_m_ital}|\S+|\n/i
      end
      @manmkp_bold=emph_italics \
      ? '^!_\s.+?(?:\n|$)|[!b]\\{.+?\\}[*!b]|[*!][a-zA-Z0-9\-_]+[!]'
      : '^!_\s.+?(?:\n|$)|[*!b]\\{.+?\\}[*!b]|[*!][a-zA-Z0-9\-_]+[*!]'
      @line_scan_bold=if defined? @md.bold_match_list[:str] \
      and @md.bold_match_list[:str]
        /#{@http_m}|#{bold_line}|(?:#{@manmkp_bold}|#{@md.bold_match_list[:str]})#{tail_m_bold}|\S+|\n/i
      end
    end
    def songsheet
      @data=@data.compact
      @data.each do |dob|
        dob=breaks(dob)
        dob=if @md.sem_tag then sem(dob) else dob end #revisit
        dob=line_actions(dob)
        dob=paragraph_set(dob)
        dob=substitutions(dob)
        dob=wordlist_italics(dob)
        dob=wordlist_bold(dob)
        dob=bodymarkup(dob)
        @data_new << dob unless dob.nil?
      end
      @data_new
    end
    def sem(dob) #revisit
      dob=SiSU_Sem::Tags.new(dob,@md).rm.all
    end
    def breaks(dob)
      if dob.is !=:meta \
      && dob.is !=:comment \
      && dob.is !=:code \
      && dob.is !=:table
        dob.obj=dob.obj.
          gsub(/^-\\\\-\s*$/,"#{Mx[:br_page]}").
          gsub(/^=\\\\=\s*$/,"#{Mx[:br_page_new]}").
          gsub(/ \\\\(?: |$)/,"#{Mx[:br_line]}").
          gsub(/(?:<:?pb>)/,"#{Mx[:br_page]}").                         # depreciated
          gsub(/(?:<:?pn>)/,"#{Mx[:br_page_new]}").                     # depreciated
          gsub(/(?:<:?br>|<br \/>)/,"#{Mx[:br_line]}").                 # depreciated
          gsub(/(?:^-\.\.-\s*$)/,"#{Mx[:br_page_line]}")
      end
      dob
    end
    def wordlist_italics(dob)
      dob=dob.dup
      if (defined? @md.italics_match_list[:str] \
      and @md.italics_match_list[:str])
        dob.obj=if dob.is !=:meta \
        && dob.is !=:heading \
        && dob.is !=:heading_insert \
        && dob.is !=:code \
        && dob.is !=:layout \
        && dob.is !=:comment
          word=dob.obj.scan(@line_scan_ital)
          word=word.flatten.compact
          line_array=[]
          word.each do |w|
            unless /#{@manmkp_ital}|#{@http_m}/.match(w)
              if defined? @md.italics_match_list[:regx] \
              and @md.italics_match_list[:regx]
                w=w.gsub(@md.italics_match_list[:regx],
                  "#{Mx[:fa_italics_o]}\\1#{Mx[:fa_italics_c]}")
              else w
              end
            end
            line_array << w
          end
          line_array.join(' ')
        else dob.obj
        end
      end
      dob
    end
    def embolden(given)
      given=given.
        gsub(/^!_\s+((?:\{|#{Mx[:lnk_o]})(?:~^ )?.+?(?:\}|#{Mx[:lnk_o]})https?:\/\/\S+.*?)([#{Mx[:br_line]}#{Mx[:br_paragraph]}#{Mx[:br_nl]}])/,
          "#{Mx[:fa_bold_o]} \\1 #{Mx[:fa_bold_c]}\\2").
        gsub(/^!_\s+((?:\{|#{Mx[:lnk_o]})(?:~^ )?.+?(?:\}|#{Mx[:lnk_o]})https?:\/\/\S+.*)/,
          "#{Mx[:fa_bold_o]} \\1 #{Mx[:fa_bold_c]}").
        gsub(/(?:^!_|^#{Mx[:lv_o]}[7-9]:\S*?#{Mx[:lv_c]})\s*(.+?)([#{Mx[:br_line]}#{Mx[:br_paragraph]}#{Mx[:br_nl]}])/,
          "#{Mx[:fa_bold_o]}\\1#{Mx[:fa_bold_c]}\\2").
        gsub(/(?:^!_|^#{Mx[:lv_o]}[7-9]:\S*?#{Mx[:lv_c]})\s*(.+?)\s+((?:[*]~\S+\s*)+)/,
          "#{Mx[:fa_bold_o]}\\1#{Mx[:fa_bold_c]}\\2").
        gsub(/(?:^!_|^#{Mx[:lv_o]}[7-9]:\S*?#{Mx[:lv_c]})\s*(.+?)\s*([~-]#)$/,
          "#{Mx[:fa_bold_o]}\\1#{Mx[:fa_bold_c]}\\2").
        gsub(/(?:^!_\s+|^#{Mx[:lv_o]}[7-9]:\S*?#{Mx[:lv_c]}\s*)(.*)?\s*$/,
          "#{Mx[:fa_bold_o]}\\1#{Mx[:fa_bold_c]}")
    end
    def italicise(given)
      given=given.
        gsub(/^\/_\s*(.+?)([#{Mx[:br_line]}#{Mx[:br_paragraph]}#{Mx[:br_nl]}])/,
          "#{Mx[:fa_italics_o]}\\1#{Mx[:fa_italics_c]}\\2").
        gsub(/^\/_\s*(.+?)\s+((?:[*]~\S+\s*)+)/,
          "#{Mx[:fa_italics_o]}\\1#{Mx[:fa_italics_c]}\\2").
        gsub(/^\/_\s*(.+?)\s*([~-]#)$/,
          "#{Mx[:fa_italics_o]}\\1#{Mx[:fa_italics_c]}\\2").
        gsub(/^\/_\s+(.*)?\s*$/,
          "#{Mx[:fa_italics_o]}\\1#{Mx[:fa_italics_c]}")
    end
    def line_actions(dob)
      dob.obj=if (dob.is !=:heading \
      && dob.is !=:heading_insert \
      && dob.is !=:comment \
      && dob.is !=:meta) \
      and dob.obj =~ /^!_\s+/
        embolden(dob.obj)
      elsif dob.obj =~ /^\/_\s+/
        italicise(dob.obj)
      else dob.obj
      end
      dob
    end
    def paragraph_set(dob)
      dob.obj=if dob.is !=:meta \
      && dob.is !=:heading \
      && dob.is !=:heading_insert \
      && dob.is !=:code \
      && dob.is !=:comment \
      && dob.is !=:table
        dob.obj.gsub(/\n/m,' ').
          gsub(/ \s+/m,' ')
      else dob.obj
      end
      dob
    end
    def substitutions(dob)
      dob=dob.dup
      dob=if @md.flag_auto_biblio \
      and @bibliography.length > 0
        dob=if dob.is !=:meta \
        && dob.is !=:heading_insert \
        && dob.is !=:code \
        && dob.is !=:comment \
        && dob.is !=:table
          @bibliography.each do |c|
            if c[:id] and not c[:id].nil? and not c[:id].empty?
              dob.obj=dob.obj.gsub(/#{c[:id]}/mi,c[:short_name])
            end
          end
          dob
        else dob
        end
        dob
      else dob
      end
      dob=if defined? @md.substitution_match_list[:match_and_replace] \
      and @md.substitution_match_list[:match_and_replace].is_a?(Array)
        dob=if dob.is !=:meta \
        && dob.is !=:heading_insert \
        && dob.is !=:code \
        && dob.is !=:comment \
        && dob.is !=:table
          if dob.obj =~/#{@md.substitution_match_list[:matches]}/
            @md.substitution_match_list[:match_and_replace].each do |x|
              dob.obj=if x[:case_s]==:i
                dob.obj.gsub(/#{x[:match]}/mi,x[:replace])
              else
                dob.obj.gsub(/#{x[:match]}/m,x[:replace])
              end
            end
          end
          dob
        else dob
        end
        dob
      else dob
      end
    end
    def wordlist_bold(dob)
      dob=dob.dup
      if (defined? @md.bold_match_list[:str] \
      and @md.bold_match_list[:str])
        dob.obj=if dob.is !=:meta \
        && dob.is !=:heading \
        && dob.is !=:heading_insert \
        && dob.is !=:code \
        && dob.is !=:comment \
        && dob.is !=:table
          line_array=[]
          word=dob.obj.scan(@line_scan_bold)
          word=word.flatten.compact
          word.each do |w|
            unless /#{@manmkp_bold}|#{@http_m}/.match(w)
              if defined? @md.bold_match_list[:regx] \
              and @md.bold_match_list[:regx]                                             #document header: @bold: [bold word list]
                w=w.gsub(@md.bold_match_list[:regx],
                  "#{Mx[:fa_bold_o]}\\1#{Mx[:fa_bold_c]}")
              end
            else
              w=if w =~ /(?:^!_|^#{Mx[:lv_o]}[7-9]:\S*?#{Mx[:lv_c]})\s+/
                embolden(w)      #bold paragraph/emphasize #may wish to remove think about 7{ 8{ conversion not satisfactory, as information is lost!
              elsif w =~/^\/_\s+/
                italicise(w)
              else w
              end
            end
            line_array << w
          end
          line_array.join(' ')
        else dob.obj
        end
      else
        dob.obj=if dob.is==:heading \
        and dob.ln.to_s =~/[7-9]/
          embolden(dob.obj)
        else dob.obj
        end
      end
      dob
    end
    def fontface_lines(dob,leader)
      while (dob.obj =~/#{Mx[:br_nl]}/ \
      and dob.obj =~/(?:#{leader})([*!\/_#])\{(.+?)\}\1/m) \
      and $2 =~/#{Mx[:br_nl]}/
        dob=if dob.obj =~/#{Mx[:br_nl]}/ \
        and dob.obj =~/(#{leader})([*!\/_#])\{(.+?)\}\2/m
          lead,fce,txt=$1,$2,$3
          dob=if txt =~/#{Mx[:br_nl]}/
            lead_break=if dob.obj =~/^#{Mx[:br_nl]}/
              dob.obj=dob.obj.sub(/^#{Mx[:br_nl]}/,'')
              Mx[:br_nl]
            else ''
            end
            txt="#{lead_break}#{fce}\{" + txt.split(Mx[:br_nl]).join("\}#{fce}#{Mx[:br_nl]}#{fce}\{") + "\}#{fce}"
            dob.obj=dob.obj.
              sub(/(?:^|#{Mx[:gl_c]}|\s+|['"]|[#{Mx[:nbsp]}#{Mx[:fa_o_c]}#{Mx[:fa_c]}#{Mx[:lnk_o]}#{Mx[:br_nl]}#{Mx[:br_line]}#{Mx[:br_paragraph]}#{Mx[:tc_c]}#{Mx[:tc_p]}]|[\(\[\{]|\>)([*!\/_#])\{.+?\}\1/m,
                "#{lead}#{txt}")
            dob
          else dob
          end
        end
        dob
      end
      dob
    end
    def fontface(dob)
      leader=/^|#{Mx[:gl_c]}|\s+|['"]|[#{Mx[:nbsp]}#{Mx[:fa_o_c]}#{Mx[:fa_c]}#{Mx[:lnk_o]}#{Mx[:br_nl]}#{Mx[:br_line]}#{Mx[:br_paragraph]}#{Mx[:tc_c]}#{Mx[:tc_p]}]|[\(\[\{]|[、。「‹«¿¡]|\>/
      dob=fontface_lines(dob,leader)
      dob.obj=dob.obj.
        gsub(/(#{leader})\*\{(.+?)\}\*/m,
          "\\1#{@emph[:o]}\\2#{@emph[:c]}").                                                                                                                             #emphasis
        gsub(/(#{leader})!\{(.+?)\}!/m,
          "\\1#{Mx[:fa_bold_o]}\\2#{Mx[:fa_bold_c]}").                                                                                                                   #bold
        gsub(/(#{leader})\/\{(.+?)\}\//m,
          "\\1#{Mx[:fa_italics_o]}\\2#{Mx[:fa_italics_c]}").                                                                                                             #italics
        gsub(/(#{leader})_\{(.+?)\}_/m,
          "\\1#{Mx[:fa_underscore_o]}\\2#{Mx[:fa_underscore_c]}").                                                                                                       #underscore
        gsub(/(#{leader})#\{(.+?)\}#/m,
          "\\1#{Mx[:fa_monospace_o]}\\2#{Mx[:fa_monospace_c]}").                                                                                                         #monospace
        gsub(/(^|#{Mx[:gl_c]}|\s+|['"]|[#{Mx[:nbsp]}#{Mx[:fa_o_c]}#{Mx[:fa_c]}]|\(|\>)\"\{(.+?)\}\"/m,
          "\\1#{Mx[:fa_cite_o]}\\2#{Mx[:fa_c_o]}cite#{Mx[:fa_c]}").                                                                                                      #cite /blockquote?
        gsub(/(^|[^\\])\^\{(.+?)\}\^/m,
          "\\1#{Mx[:fa_superscript_o]}\\2#{Mx[:fa_superscript_c]}").                                                                                                     #superscript
        gsub(/(^|[^\\]),\{(.+?)\},/m,
          "\\1#{Mx[:fa_subscript_o]}\\2#{Mx[:fa_subscript_c]}").                                                                                                         #subscript
        gsub(/(^|#{Mx[:gl_c]}|\s+|['"]|#{Mx[:nbsp]}|#{Mx[:fa_o_c]}|#{Mx[:fa_c]}|\(|\>)\+\{(.+?)\}\+/m,
          "\\1#{Mx[:fa_insert_o]}\\2#{Mx[:fa_insert_c]}").                                                                                                               #inserted text
        gsub(/(^|#{Mx[:gl_c]}|\s+|['"]|#{Mx[:nbsp]}|#{Mx[:fa_o_c]}|#{Mx[:fa_c]}|\(|\>)-\{(.+?)\}-/m,
          "\\1#{Mx[:fa_strike_o]}\\2#{Mx[:fa_strike_c]}").                                                                                                               #strikethrough - deleted text
        gsub(/(^|#{Mx[:gl_c]}|\s+|['"]|#{Mx[:nbsp]}|#{Mx[:fa_o_c]}|#{Mx[:fa_c]}|\(|\>|\d+)\^(\S+?)\^/,
          "\\1#{Mx[:fa_superscript_o]}\\2#{Mx[:fa_superscript_c]}")                                                                                                      #superscript single word, watch digit added
      dob
    end
    def bodymarkup(dob)
      # << http://www.jus.uio.no/sisu/sisu_markup_table/markup >>
      # See: data/sisu/sample/document_samples_sisu_markup/
      ## fontface
      # *{emphasis}*        e{emphasis}e       <strong>emphasis</strong>
      # !{bold text}!       b{bold}b           <b>bold text</b>
      # _{underline}_       u{underline}u      <u>underline</u>
      # /{italics}/         i{italics}i        <i>italics</i>
      # "{citation}"        c{citation}c       <cite>citation</cite> #blockquote?
      # ^{superscript}^                        <sup>superscript</sup>
      # ,{subscript},                          <sub>subscript</sub>
      # +{inserted text}+                      <ins>inserted text</ins>
      # -{deleted text}-                       <del>deleted text</del>
      # #{monospace text}#
      #
      # {url address}:url
      # {image.png}imageurl
      # {image.png}png
      # ~{endnote}~
      # !_                                    #bold/emphasise paragraph
      # _"                                    #blockquote paragraph
      # _1                  <:i1>            #indent paragraph 1 step
      # _2                  <:i2>            #indent paragraph 2 steps
      # _3                  <:i3>            #indent paragraph 3 steps
      # _4                  <:i4>            #indent paragraph 4 steps
      # _*                                    #bullet (list) ●
      # _1*                                   #bullet (list) indented
      # _1*                                   #bullet (list) indented
      # #                                     #numbered (list) level 1
      # _#                                    #numbered (list) level 2
      dob=dob.dup
      if dob.is !=:meta \
      && dob.is !=:comment \
      && dob.is !=:code \
      && dob.is !=:table
        line_array=[]
        word=dob.obj.scan(/\S+|\n/) #unless line =~/^(?:#{Mx[:meta_o]}|%+\s)/ #visit
        if word
          word.each do |w| # _ - / # | : ! ^ ~
            unless w =~/~\{|\}~|~\[|\]~|^\^~|~\^|\*~\S+|~#|\{t?~|\{table|https?:\/\/\S+/           # do something earlier about table!!
              w=w.gsub(/\\?~/,"#{Mx[:gl_o]}#126#{Mx[:gl_c]}")                                      #escaped special character
            end
            w=w.gsub(/^\<$/,
              "#{Mx[:gl_o]}#lt#{Mx[:gl_c]}").gsub(/^\>$/,"#{Mx[:gl_o]}#gt#{Mx[:gl_c]}")            #escaped special character
            line_array << w
          end
          dob.obj=line_array.join(' ')
          dob.obj=dob.obj.strip
        end
        dob.obj=dob.obj.
          gsub(/^([*#.-]{1,12})$/,'\1 ~#').                                                        #ocn off for these paragraph separators
          gsub(/~\{(.+?)\}~/m,Mx[:en_a_o] + '\1' + Mx[:en_a_c]).
          gsub(/~\[([^*+].+?)\]~/m,Mx[:en_b_o] + '* \1' + Mx[:en_b_c]).                            #default if markup does not specify
          gsub(/~\[(.+?)\]~/m,Mx[:en_b_o] + '\1' + Mx[:en_b_c])
        if dob.is ==:heading \
        and dob.ln ==0
          dob.obj=dob.obj.gsub(/\s*@title\b/," #{@md.title.full}")
          dob.obj=if defined? @md.creator.author \
          and @md.creator.author
            dob.obj.gsub(/\s+(?:@creator|@author)/,",#{Mx[:br_line]}#{@md.creator.author}")
          else dob.obj.gsub(/\s+(?:@creator|@author)/,'')
          end
        end
        if defined? @md.title \
        and @md.title \
        and defined? @md.title.full \
        and defined? @md.creator \
        and @md.creator
          if dob.is ==:heading
            dob.obj=dob.obj.gsub(/^\s*@title\s*$/,@md.title.full) if dob.lv =~/1/
            dob.obj=if dob.lv =~/[23]/ \
            and defined? @md.creator.author \
            and @md.creator.author
              dob.obj.
                gsub(/^\s*(?:(by\s+)?(?:@creator|@author))\s*$/,
                  "\\1#{@md.creator.author}")
            else dob.obj.gsub(/^\s*(?:(by\s+)?(?:@creator|@author))\s*$/,'\1')
            end
          end
        end
        dob.obj=dob.obj.gsub(/<(https?:\/\/\S+?)>/,'< \1 >').                     #catch problem markup
          gsub(/<:=(\S+?)>/,'{ c_\1.png 14x14 }image').
          gsub(/<!(\S+)!>/,'<:\1>').                                              #escaped special character
          gsub(/&nbsp;/,"#{Mx[:nbsp]}").                                          #escaped special character
          gsub(/\\~/,"#{Mx[:gl_o]}#126#{Mx[:gl_c]}").                             #escaped special character
          gsub(/\\\{/,"#{Mx[:gl_o]}#123#{Mx[:gl_c]}").                            #escaped special character
          gsub(/\\\}/,"#{Mx[:gl_o]}#125#{Mx[:gl_c]}").                            #escaped special character
          gsub(/\\\<</,"#{Mx[:gl_o]}#lt#{Mx[:gl_c]}#{Mx[:gl_o]}#lt#{Mx[:gl_c]}"). #escaped special character
          gsub(/\\\>>/,"#{Mx[:gl_o]}#gt#{Mx[:gl_c]}#{Mx[:gl_o]}#gt#{Mx[:gl_c]}"). #escaped special character
          gsub(/\\\</,"#{Mx[:gl_o]}#lt#{Mx[:gl_c]}").                             #escaped special character
          gsub(/\\\>/,"#{Mx[:gl_o]}#gt#{Mx[:gl_c]}").                             #escaped special character
          gsub(/\\\_/,"#{Mx[:gl_o]}#095#{Mx[:gl_c]}").                            #escaped special character
          gsub(/\\\-/,"#{Mx[:gl_o]}#045#{Mx[:gl_c]}").                            #escaped special character
          gsub(/\\\+/,"#{Mx[:gl_o]}#043#{Mx[:gl_c]}").                            #escaped special character
          gsub(/\\\//,"#{Mx[:gl_o]}#047#{Mx[:gl_c]}").                            #escaped special character
          gsub(/\\\#/,"#{Mx[:gl_o]}#035#{Mx[:gl_c]}").                            #escaped special character
          gsub(/\\\&/,"#{Mx[:gl_o]}#038#{Mx[:gl_c]}").                            #&amp; #escaped special character
          gsub(/\\\|/,"#{Mx[:gl_o]}#124#{Mx[:gl_c]}").                            #not really a sisu special character but made available as possibility
          gsub(/\\\:/,"#{Mx[:gl_o]}#058#{Mx[:gl_c]}").                            #not really a sisu special character but made available as possibility
          gsub(/\\\!/,"#{Mx[:gl_o]}#033#{Mx[:gl_c]}").                            #not really a sisu special character but made available as possibility
          gsub(/\\\^/,"#{Mx[:gl_o]}#094#{Mx[:gl_c]}").                            #not really a sisu special character but made available as possibility
          gsub(/\\\,/,"#{Mx[:gl_o]}#044#{Mx[:gl_c]}").                            #not really a sisu special character but made available as possibility
          gsub(/\\\\/,"#{Mx[:gl_o]}#092#{Mx[:gl_c]}").                            #escaped special character
          gsub(/\\\*/,"#{Mx[:gl_o]}#042#{Mx[:gl_c]}").                            #escaped special character
          gsub(/\\\!/,"#{Mx[:gl_o]}#033#{Mx[:gl_c]}")                             #escaped special character
        if dob.obj=~/(?:https?:|ftp:|\{([^{}]+?)\}(?:#|:|[.]{1,2}\/))\S+/m
          if dob.obj=~/(?:^|[#{Mx[:gl_c]}#{Mx[:nbsp]} ])\{~\^ (?:.+?)\s*\}(?:(?:https?:|ftp:|:|[.]{1,2}\/)\S+?)\s*#{Mx[:en_a_o]}(.+?)#{Mx[:en_a_c]}/m
            dob.obj=dob.obj.
              gsub(/(^|[#{Mx[:gl_c]}#{Mx[:nbsp]} ])\{~\^ ([^}]+?)\s*\}((?:https?:|ftp:|:|[.]{1,2}\/)\S+?)\s*#{Mx[:en_a_o]}(.+?)#{Mx[:en_a_c]}/m,
                "\\1#{Mx[:lnk_o]}\\2#{Mx[:lnk_c]}\\3 #{Mx[:en_a_o]}\\3 \\4#{Mx[:en_a_c]}") # watch
          end
          if dob.obj=~/(?:^|[#{Mx[:gl_c]}#{Mx[:nbsp]} ])\{~\^ (?:.+?)\s*\}(?:(?:https?:|ftp:|:|[.]{1,2}\/)\S+?)([;,.]?)(?=\s|[#{Mx[:br_line]}#{Mx[:br_paragraph]}#{Mx[:br_nl]}]|$)/m
            dob.obj=dob.obj.
              gsub(/(^|[#{Mx[:gl_c]}#{Mx[:nbsp]} ])\{~\^ (.+?)\s*\}((?:https?:|ftp:|:|[.]{1,2}\/)\S+?)([;,.]?)(?=\s|[#{Mx[:br_line]}#{Mx[:br_paragraph]}#{Mx[:br_nl]}]|$)/m,
                "\\1#{Mx[:lnk_o]}\\2#{Mx[:lnk_c]}\\3\\4 #{Mx[:en_a_o]}\\3#{Mx[:en_a_c]} ")
          end
          dob.obj=dob.obj.
              gsub(/(^|[^#])\{\s*([^{}]+?)\s*\}((?:https?:|:|[.]{2}\/|#)\S+?)(?=\s|[#{Mx[:br_line]}#{Mx[:br_paragraph]}#{Mx[:br_nl]}#{Mx[:en_a_o]}#{Mx[:en_b_o]}]|$)/,
                "\\1#{Mx[:lnk_o]}\\2#{Mx[:lnk_c]}\\3").                                                                                                                    #linked (text or image, however text cannot include modified face, e.g. bold, ital, underline)
            gsub(/(^|[#{Mx[:gl_c]}#{Mx[:lnk_c]}#{Mx[:en_a_o]}#{Mx[:en_b_o]}(\s])((?:https?|ftp):\/\/\S+?\.[^>< ]+?)([,.;'"]?)(?=[\s#{Mx[:en_a_c]}#{Mx[:en_b_c]}#{Mx[:br_line]}#{Mx[:br_paragraph]}#{Mx[:br_nl]}]|$)/m,
              %{\\1#{Mx[:url_o]}\\2#{Mx[:url_c]}\\3}).
            gsub(/#{Mx[:lnk_c]}#(\S+?[^>< ]+?)([()\[\]]*[,.;:!?'"]{0,2})(?=[\s#{Mx[:en_a_c]}#{Mx[:en_b_c]}#{Mx[:br_line]}#{Mx[:br_paragraph]}#{Mx[:br_nl]}]|$)/m,
              %{#{Mx[:lnk_c]}#{Mx[:rel_o]}\\1#{Mx[:rel_c]}\\2}).
            gsub(/#{Mx[:lnk_c]}:(\S+?[^>< ]+?)([()\[\]]*[,.;:!?'"]{0,2})(?=[\s#{Mx[:en_a_c]}#{Mx[:en_b_c]}#{Mx[:br_line]}#{Mx[:br_paragraph]}#{Mx[:br_nl]}]|$)/m,
              %{#{Mx[:lnk_c]}#{Mx[:rel_o]}:\\1#{Mx[:rel_c]}\\2}).
            gsub(/#{Mx[:lnk_c]}[.]{2}\/(\S+?[^>< ]+?)([()\[\]]*[,.;:!?'"]{0,2})(?=[\s#{Mx[:en_a_c]}#{Mx[:en_b_c]}#{Mx[:br_line]}#{Mx[:br_paragraph]}#{Mx[:br_nl]}]|$)/m,
              %{#{Mx[:lnk_c]}#{Mx[:rel_o]}:\\1#{Mx[:rel_c]}\\2})
        end
        if dob.obj=~/_(?:https?|ftp):\S+/m           # _http://url #CHECK
          dob.obj=dob.obj.gsub(/(^|[#{Mx[:gl_c]}#{Mx[:lnk_c]}#{Mx[:en_a_o]}#{Mx[:en_b_o]}(\s])(_(?:https?|ftp):\/\/\S+?\.[^>< ]+?)([,.;'"]?)(?=[\s#{Mx[:en_a_c]}#{Mx[:en_b_c]}#{Mx[:br_line]}#{Mx[:br_paragraph]}#{Mx[:br_nl]}]|$)/m,
            %{\\1#{Mx[:url_o]}\\2#{Mx[:url_c]}\\3})
        end
        dob=fontface(dob)
        dob.obj=dob.obj.
          gsub(/<[:e]\s+(.+?)!?>/,
            "#{Mx[:en_a_o]}\\1#{Mx[:en_a_c]}").                                                                                                                             #not tested
          gsub(/(^|#{Mx[:br_nl]})\s*_\*\s*/,
            "\\1#{Mx[:gl_bullet]}").                                                                                                                                        #bullets, shortcut
          gsub(/=\{(.+?)\}/,
            "#{Mx[:idx_o]}\\1#{Mx[:idx_c]}").
          gsub(/^\s*_([1-9])\*\s*/,
            "#{Mx[:pa_o]}:i\\1:\\1#{Mx[:pa_c]}#{Mx[:gl_bullet]}").                                                                                                          #bullets, shortcut
          gsub(/^\s*_([1-9])\s+/,
            "#{Mx[:pa_o]}:i\\1:\\1#{Mx[:pa_c]}").                                                                                                                           #indent
          gsub(/^\s*_([1-9])!\s+(.+?)\s*$/,
            "#{Mx[:pa_o]}:i\\1:\\1#{Mx[:pa_c]}#{Mx[:fa_bold_o]}\\2#{Mx[:fa_bold_c]} ").                                                                                     #indent bold
          gsub(/^\s*__([1-9])\s+/,
            "#{Mx[:pa_o]}:i0:\\1#{Mx[:pa_c]}").                                                                                                                             #hang
          gsub(/^\s*__([1-9])!\s+(.+?)\s*$/,
            "#{Mx[:pa_o]}:i0:\\1#{Mx[:pa_c]}#{Mx[:fa_bold_o]}\\2#{Mx[:fa_bold_c]} ").                                                                                       #hangdef
          gsub(/^\s*_([0-9])_([0-9])\s+/,
            "#{Mx[:pa_o]}:i\\1:\\2#{Mx[:pa_c]}").                                                                                                                           #hang
          gsub(/^\s*_([0-9])_([0-9])!\s+(.+?)\s*$/,
            "#{Mx[:pa_o]}:i\\1:\\2#{Mx[:pa_c]}#{Mx[:fa_bold_o]}\\3#{Mx[:fa_bold_c]} ").                                                                                     #hangdef
          gsub(/<:hi>/,"#{Mx[:fa_hilite_o]}").                                                                                                                              #'<span style="background-color: rgb(255,240,196)">'). # bright yellow rgb(255,255,0) pale yellow rgb(255,255,200)
          gsub(/<:\/hi>/,"#{Mx[:fa_hilite_c]}"). #'</span>').
          gsub(/(#{Mx[:gr_o]}verse#{Mx[:gr_c]}.+)/m,"\\1\n").
          gsub(/[ ]+($)/,'\1').
          gsub(/\{\s*(.+?)\s*\}(https?:\S+?)([;,.]?)(?=\s|[#{Mx[:br_line]}#{Mx[:br_paragraph]}#{Mx[:br_nl]}#{Mx[:en_a_o]}#{Mx[:en_b_o]}]|$)/,
            "#{Mx[:lnk_o]}\\1#{Mx[:lnk_c]}#{Mx[:url_o]}\\2#{Mx[:url_c]}\\3").                                                                                               #any remaining linked text or image
          gsub(/\{\s*(.+?)\s*\}(#{Mx[:url_o]}\S+?#{Mx[:url_c]})/,
            "#{Mx[:lnk_o]}\\1#{Mx[:lnk_c]}\\2").                                                                                                                            #any remaining linked text or image
          gsub(/(^|\s)([a-zA-Z0-9._-]+\@\S+?\.[a-zA-Z0-9._-]+)/,"\\1#{Mx[:url_o]}\\2#{Mx[:url_c]}").
          gsub(/(^|[ ])\{\s*(.+?)\s*\}(\S+?)([;,.]?)(?=\s|[#{Mx[:br_line]}#{Mx[:br_paragraph]}#{Mx[:br_nl]}#{Mx[:en_a_o]}#{Mx[:en_b_o]}]|$)/,
            "\\1#{Mx[:lnk_o]}\\2#{Mx[:lnk_c]}\\3\\4").                                                                                                                      #any remaining linked text or image
          gsub(/\{\s*(.+?)\s*\}#([a-zA-Z0-9][a-zA-Z0-9_-]*)([;,.]?)(?=\s|[#{Mx[:br_line]}#{Mx[:br_paragraph]}#{Mx[:br_nl]}#{Mx[:en_a_o]}#{Mx[:en_b_o]}]|$)/,
            "#{Mx[:lnk_o]}\\1#{Mx[:lnk_c]}#{Mx[:rel_o]}\\2#{Mx[:rel_c]}\\3").                                                                                               #any remaining linked text or image, check need
          gsub(/\{\s*(.+?)\s*\}(#{Mx[:rel_o]}\S+?#{Mx[:rel_c]})/,
            "#{Mx[:lnk_o]}\\1#{Mx[:lnk_c]}\\2").                                                                                                                            #any remaining linked text or image, check need
          gsub(/\{\s*(.+?)\s*\}(image)/,
            "#{Mx[:lnk_o]}\\1#{Mx[:lnk_c]}\\2")                                                                                                                             #linked image
      elsif dob.is==:table
        dob=fontface(dob)
      elsif dob.is ==:code
        dob.obj=dob.obj.
          gsub(/#{Mx[:meta_o]}(\S+?)#{Mx[:meta_c]}\s*/,'@\1: ').
          gsub(/(^|#{Mx[:gl_c]}|\s)&lt;(?:br(?: \/)?)&gt;([\s,.]|$)/,'\1<br>\2') #convert <br> <br /> back, clumsy
        if dob.number_
          codeline=[]
          ln=1
          dob.obj.split(/#{Mx[:gr_o]}codeline#{Mx[:gr_c]}|<br(?: \/)?>|\n/).each_with_index do |cl,i|
            unless i == 0
              cl=cl.gsub(Mx[:br_nl],'')
              w=3-ln.to_s.length
              cl = "#{ln}#{Mx[:nbsp]*w}#{Mx[:vline]}#{cl}#{Mx[:br_nl]}"
              ln +=1
            end
            codeline << cl
          end
          codeline= codeline.join("")
          dob.obj=codeline
        else
          dob.obj=dob.obj.gsub(/#{Mx[:gr_o]}codeline#{Mx[:gr_c]}/,"\n")
        end
        dob
      else # @\S+?:
      end
      dob
    end
    def tech                                                                       #script markup planned to be more strict for technical documents
      # *{emphasis}*        e{emphasis}e       <strong>emphasis</strong>
      # !{bold text}!       b{bold}b           <b>bold text</b>
      # _{underline}_       u{underline}u      <u>underline</u>
      # /{italics}/         i{italics}i        <i>italics</i>
      # "{citation}"        c{citation}c       <cite>citation</cite>
      # ^{superscript}^                        <sup>superscript</sup>
      # ,{subscript},                          <sub>subscript</sub>
      # +{inserted text}+                      <ins>inserted text</ins>
      # -{deleted text}-                       <del>deleted text</del>
      # #{monospace text}#
      # {url address}:url
      # {image.png}imageurl
      # {image.png}png
      # ~{endnote}~
      # +1                  <!i1!>
      # +2                  <!i2!>
      puts 'tech'
      @data.each do |line|
        line=line.
          gsub(/(^|\s+|['"]|#{Mx[:fa_o_c]}|#{Mx[:fa_c]}|[\(\[]|\>)e\{(.+?)\}e/,
            "\\1#{@emph[:o]}\\2#{@emph[:c]}").                                                        #emphasis
          gsub(/(^|\s+|['"]|#{Mx[:fa_o_c]}|#{Mx[:fa_c]}|[\(\[]|\>)b\{(.+?)\}b/,
            "\\1#{Mx[:fa_bold_o]}\\2#{Mx[:fa_bold_c]}").                                              #bold
          gsub(/(^|\s+|['"]|#{Mx[:fa_o_c]}|#{Mx[:fa_c]}|[\(\[]|\>)u\{(.+?)\}u/,
            "\\1#{Mx[:fa_underscore_o]}\\2#{Mx[:fa_underscore_c]}").                                  #underscore
          gsub(/(^|\s+|['"]|#{Mx[:fa_o_c]}|#{Mx[:fa_c]}|[\(\[]|\>)c\{(.+?)\}c/,
            "\\1#{Mx[:fa_cite_o]}\\2#{Mx[:fa_c_o]}cite#{Mx[:fa_c]}").                                 #cite
          gsub(/(^|\s+|['"]|#{Mx[:fa_o_c]}|#{Mx[:fa_c]}|[\(\[]|\>)i\{(.+?)\}i/,
            "\\1#{Mx[:fa_italics_o]}\\2#{Mx[:fa_italics_c]}").                                        #italics
          gsub(/(^|\s+|['"]|#{Mx[:fa_o_c]}|#{Mx[:fa_c]}|[\(\[]|\>)!\{(.+?)\}!/,
            "\\1#{Mx[:fa_bold_o]}\\2#{Mx[:fa_bold_c]}").                                              #bold
          gsub(/(^|\s+|['"]|#{Mx[:fa_o_c]}|#{Mx[:fa_c]}|[\(\[]|\>)\*\{(.+?)\}\*/,
            "\\1#{@emph[:o]}\\2#{@emph[:c]}").                                                        #emphasis
          gsub(/(^|\s+|['"]|#{Mx[:fa_o_c]}|#{Mx[:fa_c]}|[\(\[]|\>)_\{(.+?)\}_/,
            "\\1#{Mx[:fa_underscore_o]}\\2#{Mx[:fa_underscore_c]}").                                  #underscore
          gsub(/(^|\s+|['"]|#{Mx[:fa_o_c]}|#{Mx[:fa_c]}|[\(\[]|\(|\>)\/\{(.+?)\}\//,
            "\\1#{Mx[:fa_italics_o]}\\2#{Mx[:fa_italics_c]}").                                        #italics
          gsub(/(^|\s+|['"]|#{Mx[:fa_o_c]}|#{Mx[:fa_c]}|\(|\>)\"\{(.+?)\}\"/,
            "\\1#{Mx[:fa_cite_o]}\\2#{Mx[:fa_c_o]}cite#{Mx[:fa_c]}").
          gsub(/(^|\s+|['"]|#{Mx[:fa_o_c]}|#{Mx[:fa_c]}|\(|\>)\^\{(.+?)\}\^/,
            "\\1#{Mx[:fa_superscript_o]}\\2#{Mx[:fa_superscript_c]}").
          gsub(/(^|\s+|['"]|#{Mx[:fa_o_c]}|#{Mx[:fa_c]}|\(|\>)9\{(.+?)\}9/,
            "\\1#{Mx[:fa_superscript_o]}\\2#{Mx[:fa_superscript_c]}").
          gsub(/(^|\s+|['"]|#{Mx[:fa_o_c]}|#{Mx[:fa_c]}|\(|\>),\{(.+?)\},/,
            "\\1#{Mx[:fa_subscript_o]}\\2#{Mx[:fa_subscript_c]}").
          gsub(/(^|\s+|['"]|#{Mx[:fa_o_c]}|#{Mx[:fa_c]}|\(|\>)6\{(.+?)\}6/,
            "\\1#{Mx[:fa_subscript_o]}\\2#{Mx[:fa_subscript_c]}").
          gsub(/(^|\s+|['"]|#{Mx[:fa_o_c]}|#{Mx[:fa_c]}|\(|\>)\+\{(.+?)\}\+/,
            "\\1#{Mx[:fa_insert_o]}\\2#{Mx[:fa_insert_c]}").
          gsub(/(^|\s+|['"]|#{Mx[:fa_o_c]}|#{Mx[:fa_c]}|\(|\>)v\{(.+?)\}v/,
            "\\1#{Mx[:fa_insert_o]}\\2#{Mx[:fa_insert_c]}").
          gsub(/(^|\s+|['"]|#{Mx[:fa_o_c]}|#{Mx[:fa_c]}|\(|\>)-\{(.+?)\}-/,
            "\\1#{Mx[:fa_strike_o]}\\2#{Mx[:fa_strike_c]}").
          gsub(/(^|\s+|['"]|#{Mx[:fa_o_c]}|#{Mx[:fa_c]}|\(|\>)x\{(.+?)\}x/,
            "\\1#{Mx[:fa_strike_o]}\\2#{Mx[:fa_strike_c]}").
          gsub(/(^|\s+|['"]|#{Mx[:fa_o_c]}|#{Mx[:fa_c]}|\(|\>)\*(\S+?)\*/,
            "\\1#{@emph[:o]}\\2#{@emph[:c]}").                                                        #emphasise single word, watch
          gsub(/(^|\s+|['"]|#{Mx[:fa_o_c]}|#{Mx[:fa_c]}|\(|\>)\!(\S+?)\!/,
            "\\1#{Mx[:fa_bold_o]}\\2#{Mx[:fa_bold_c]}").                                              #bold single word, watch
          gsub(/(^|\s+|['"]|#{Mx[:fa_o_c]}|#{Mx[:fa_c]}|\(|\>)\/([\(\)a-zA-Z0-9']+?)\/([ ,.;:'"~$]|[^a-zA-Z0-9])/,
            "\\1#{Mx[:fa_italics_o]}\\2#{Mx[:fa_italics_c]}\\3").                                     #italics single word, watch
          gsub(/(^|\s+|['"]|#{Mx[:fa_o_c]}|#{Mx[:fa_c]}|\(|\>)_(\S+?)_/,
            "\\1#{Mx[:fa_underscore_o]}\\2#{Mx[:fa_underscore_c]}").                                  #underscore single word, watch
          gsub(/(^|\s+|['"]|#{Mx[:fa_o_c]}|#{Mx[:fa_c]}|\(|\>)\^(\S+?)\^/,
            "\\1#{Mx[:fa_superscript_o]}\\2#{Mx[:fa_superscript_c]}").                                #check  #superscript single word, watch digit added
          gsub(/^\s*_\([1-9]\)\(\*\+\)\s*/,
            "#{Mx[:pa_o]}:i\\1#{Mx[:pa_c]}#{Mx[:fa_o]}\\2#{Mx[:fa_c_o]}").                            #bullets, shortcut
          gsub(/^\s*_\([1-9]\)\s+/,
            "#{Mx[:pa_o]}:i\\1#{Mx[:pa_c]}"). #watch
          gsub(/^\s*__\([1-9]\)\s+/,
            "#{Mx[:pa_o]}:h\\1#{Mx[:pa_c]}"). #watch
          #line.gsub(/^\s*__\([1-9]\)!\s+/,
          #  "#{Mx[:pa_o]}:hd\\1#{Mx[:pa_c]}"). #watch
          gsub(/#{Mx[:br_line]}\s*_[12]\s+/,
            "#{Mx[:br_line]} ")                                                                      #indent used in endnotes, not implemented, replace when ready with: line.gsub(/(?:<br>|<br \/>)\s*_([12])\s+/,'<br><:i\1> ')
      end
      @data
    end
  end
end
__END__