Import SQLite 3.10.2.

third_party/sqlite/src/ext/fts3/unicode/mkunicode.tcl

 
-#
-# Parameter $zName must be a path to the file UnicodeData.txt. This command
-# reads the file and returns a list of mappings required to remove all
-# diacritical marks from a unicode string. Each mapping is itself a list
-# consisting of two elements - the unicode codepoint and the single ASCII
-# character that it should be replaced with, or an empty string if the 
-# codepoint should simply be removed from the input. Examples:
-#
-#   { 224 a  }     (replace codepoint 224 to "a")
-#   { 769 "" }     (remove codepoint 769 from input)
-#
-# Mappings are only returned for non-upper case codepoints. It is assumed
-# that the input has already been folded to lower case.
-#
-proc rd_load_unicodedata_text {zName} {
-  global tl_lookup_table
-
-  set fd [open $zName]
-  set lField {
-    code
-    character_name
-    general_category
-    canonical_combining_classes
-    bidirectional_category
-    character_decomposition_mapping
-    decimal_digit_value
-    digit_value
-    numeric_value
-    mirrored
-    unicode_1_name
-    iso10646_comment_field
-    uppercase_mapping
-    lowercase_mapping
-    titlecase_mapping
-  }
-  set lRet [list]
-
-  while { ![eof $fd] } {
-    set line [gets $fd]
-    if {$line == ""} continue
-
-    set fields [split $line ";"]
-    if {[llength $fields] != [llength $lField]} { error "parse error: $line" }
-    foreach $lField $fields {}
-    if { [llength $character_decomposition_mapping]!=2
-      || [string is xdigit [lindex $character_decomposition_mapping 0]]==0
-    } {
-      continue
-    }
-
-    set iCode  [expr "0x$code"]
-    set iAscii [expr "0x[lindex $character_decomposition_mapping 0]"]
-    set iDia   [expr "0x[lindex $character_decomposition_mapping 1]"]
-
-    if {[info exists tl_lookup_table($iCode)]} continue
-
-    if { ($iAscii >= 97 && $iAscii <= 122)
-      || ($iAscii >= 65 && $iAscii <= 90)
-    } {
-      lappend lRet [list $iCode [string tolower [format %c $iAscii]]]
-      set dia($iDia) 1
-    }
-  }
-
-  foreach d [array names dia] {
-    lappend lRet [list $d ""]
-  }
-  set lRet [lsort -integer -index 0 $lRet]
-
-  close $fd
-  set lRet
-}
-
+source [file join [file dirname [info script]] parseunicode.tcl]
 
 proc print_rd {map} {
   global tl_lookup_table
   set aChar [list]
   set lRange [list]
 
   set nRange 1
   set iFirst  [lindex $map 0 0]
   set cPrev   [lindex $map 0 1]
 
@@ skipping 25 matching lines @@
   lappend aChar $cPrev
 
   puts "/*"
   puts "** If the argument is a codepoint corresponding to a lowercase letter"
   puts "** in the ASCII range with a diacritic added, return the codepoint"
   puts "** of the ASCII letter only. For example, if passed 235 - \"LATIN"
   puts "** SMALL LETTER E WITH DIAERESIS\" - return 65 (\"LATIN SMALL LETTER"
   puts "** E\"). The resuls of passing a codepoint that corresponds to an"
   puts "** uppercase letter are undefined."
   puts "*/"
-  puts "static int remove_diacritic(int c)\{"
+  puts "static int ${::remove_diacritic}(int c)\{"
   puts "  unsigned short aDia\[\] = \{"
   puts -nonewline "        0, "
   set i 1
   foreach r $lRange {
     foreach {iCode nRange} $r {}
     if {($i % 8)==0} {puts "" ; puts -nonewline "    " }
     incr i
 
     puts -nonewline [format "%5d" [expr ($iCode<<3) + $nRange-1]]
     puts -nonewline ", "
@@ skipping 66 matching lines @@
   puts "  if( c<$iFirst || c>$iLast ) return 0;"
   puts "  return (c < $iFirst+32) ?"
   puts "      (mask0 & (1 << (c-$iFirst))) :"
   puts "      (mask1 & (1 << (c-$iFirst-32)));"
   puts "\}"
 }
 
 
 #-------------------------------------------------------------------------
 
-# Parameter $zName must be a path to the file UnicodeData.txt. This command
-# reads the file and returns a list of codepoints (integers). The list
-# contains all codepoints in the UnicodeData.txt assigned to any "General
-# Category" that is not a "Letter" or "Number".
-#
-proc an_load_unicodedata_text {zName} {
-  set fd [open $zName]
-  set lField {
-    code
-    character_name
-    general_category
-    canonical_combining_classes
-    bidirectional_category
-    character_decomposition_mapping
-    decimal_digit_value
-    digit_value
-    numeric_value
-    mirrored
-    unicode_1_name
-    iso10646_comment_field
-    uppercase_mapping
-    lowercase_mapping
-    titlecase_mapping
-  }
-  set lRet [list]
-
-  while { ![eof $fd] } {
-    set line [gets $fd]
-    if {$line == ""} continue
-
-    set fields [split $line ";"]
-    if {[llength $fields] != [llength $lField]} { error "parse error: $line" }
-    foreach $lField $fields {}
-
-    set iCode [expr "0x$code"]
-    set bAlnum [expr {
-         [lsearch {L N} [string range $general_category 0 0]] >= 0
-      || $general_category=="Co"
-    }]
-
-    if { !$bAlnum } { lappend lRet $iCode }
-  }
-
-  close $fd
-  set lRet
-}
-
 proc an_load_separator_ranges {} {
   global unicodedata.txt
   set lSep [an_load_unicodedata_text ${unicodedata.txt}]
   unset -nocomplain iFirst 
   unset -nocomplain nRange 
   set lRange [list]
   foreach sep $lSep {
     if {0==[info exists iFirst]} {
       set iFirst $sep
       set nRange 1
@@ skipping 169 matching lines @@
       return 1;
     }
   }
   }]
   puts "  return 0;"
   puts "\}"
 }
 
 #-------------------------------------------------------------------------
 
-proc tl_load_casefolding_txt {zName} {
-  global tl_lookup_table
-
-  set fd [open $zName]
-  while { ![eof $fd] } {
-    set line [gets $fd]
-    if {[string range $line 0 0] == "#"} continue
-    if {$line == ""} continue
-
-    foreach x {a b c d} {unset -nocomplain $x}
-    foreach {a b c d} [split $line ";"] {}
-
-    set a2 [list]
-    set c2 [list]
-    foreach elem $a { lappend a2 [expr "0x[string trim $elem]"] }
-    foreach elem $c { lappend c2 [expr "0x[string trim $elem]"] }
-    set b [string trim $b]
-    set d [string trim $d]
-
-    if {$b=="C" || $b=="S"} { set tl_lookup_table($a2) $c2 }
-  }
-}
-
 proc tl_create_records {} {
   global tl_lookup_table
 
   set iFirst ""
   set nOff 0
   set nRange 0
   set nIncr 0
 
   set lRecord [list]
   foreach code [lsort -integer [array names tl_lookup_table]] {
@@ skipping 143 matching lines @@
   set liOff [tl_generate_ioff_table $lRecord]
   tl_print_table_header
   foreach entry $lRecord { 
     if {[tl_print_table_entry toggle $entry $liOff]} { 
       lappend lHigh $entry 
     } 
   }
   tl_print_table_footer toggle
   tl_print_ioff_table $liOff
 
-  puts {
+  puts [subst -nocommands {
   int ret = c;
 
-  assert( c>=0 );
   assert( sizeof(unsigned short)==2 && sizeof(unsigned char)==1 );
 
   if( c<128 ){
     if( c>='A' && c<='Z' ) ret = c + ('a' - 'A');
   }else if( c<65536 ){
+    const struct TableEntry *p;
     int iHi = sizeof(aEntry)/sizeof(aEntry[0]) - 1;
     int iLo = 0;
     int iRes = -1;
 
+    assert( c>aEntry[0].iCode );
     while( iHi>=iLo ){
       int iTest = (iHi + iLo) / 2;
       int cmp = (c - aEntry[iTest].iCode);
       if( cmp>=0 ){
         iRes = iTest;
         iLo = iTest+1;
       }else{
         iHi = iTest-1;
       }
     }
-    assert( iRes<0 || c>=aEntry[iRes].iCode );
 
-    if( iRes>=0 ){
-      const struct TableEntry *p = &aEntry[iRes];
-      if( c<(p->iCode + p->nRange) && 0==(0x01 & p->flags & (p->iCode ^ c)) ){
-        ret = (c + (aiOff[p->flags>>1])) & 0x0000FFFF;
-        assert( ret>0 );
-      }
+    assert( iRes>=0 && c>=aEntry[iRes].iCode );
+    p = &aEntry[iRes];
+    if( c<(p->iCode + p->nRange) && 0==(0x01 & p->flags & (p->iCode ^ c)) ){
+      ret = (c + (aiOff[p->flags>>1])) & 0x0000FFFF;
+      assert( ret>0 );
     }
 
-    if( bRemoveDiacritic ) ret = remove_diacritic(ret);
+    if( bRemoveDiacritic ) ret = ${::remove_diacritic}(ret);
   }
-  }
+  }]
 
   foreach entry $lHigh {
     tl_print_if_entry $entry
   }
 
   puts ""
   puts "  return ret;"
   puts "\}"
 }
 
@@ skipping 50 matching lines @@
 **    May you share freely, never taking more than you give.
 **
 ******************************************************************************
 */
 
 /*
 ** DO NOT EDIT THIS MACHINE GENERATED FILE.
 */
   }]
   puts ""
-  puts "#ifndef SQLITE_DISABLE_FTS3_UNICODE"
-  puts "#if defined(SQLITE_ENABLE_FTS3) || defined(SQLITE_ENABLE_FTS4)"
+  if {$::generate_fts5_code} {
+    # no-op
+  } else {
+    puts "#ifndef SQLITE_DISABLE_FTS3_UNICODE"
+    puts "#if defined(SQLITE_ENABLE_FTS3) || defined(SQLITE_ENABLE_FTS4)"
+  }
   puts ""
   puts "#include <assert.h>"
   puts ""
 }
 
 proc print_test_main {} {
   puts ""
   puts "#include <stdio.h>"
   puts ""
   puts "int main(int argc, char **argv)\{"
   puts "  int r1, r2;"
   puts "  int code;"
   puts "  r1 = isalnum_test(&code);"
   puts "  if( r1 ) printf(\"isalnum(): Problem with code %d\\n\",code);"
   puts "  else printf(\"isalnum(): test passed\\n\");"
   puts "  r2 = fold_test(&code);"
   puts "  if( r2 ) printf(\"fold(): Problem with code %d\\n\",code);"
   puts "  else printf(\"fold(): test passed\\n\");"
   puts "  return (r1 || r2);"
   puts "\}"
 }
 
 # Proces the command line arguments. Exit early if they are not to
 # our liking.
 #
 proc usage {} {
-  puts -nonewline stderr "Usage: $::argv0 ?-test? "
+  puts -nonewline stderr "Usage: $::argv0 ?-test? ?-fts5? "
   puts            stderr "<CaseFolding.txt file> <UnicodeData.txt file>"
   exit 1
 }
-if {[llength $argv]!=2 && [llength $argv]!=3} usage
-if {[llength $argv]==3 && [lindex $argv 0]!="-test"} usage
+if {[llength $argv]<2} usage
 set unicodedata.txt [lindex $argv end]
 set casefolding.txt [lindex $argv end-1]
-set generate_test_code [expr {[llength $argv]==3}]
+
+set remove_diacritic remove_diacritic
+set generate_test_code 0
+set generate_fts5_code 0
+set function_prefix "sqlite3Fts"
+for {set i 0} {$i < [llength $argv]-2} {incr i} {
+  switch -- [lindex $argv $i] {
+    -test {
+      set generate_test_code 1
+    }
+    -fts5 {
+      set function_prefix sqlite3Fts5
+      set generate_fts5_code 1
+      set remove_diacritic fts5_remove_diacritic
+    }
+    default {
+      usage
+    }
+  }
+}
 
 print_fileheader
 
 # Print the isalnum() function to stdout.
 #
 set lRange [an_load_separator_ranges]
-print_isalnum sqlite3FtsUnicodeIsalnum $lRange
+print_isalnum ${function_prefix}UnicodeIsalnum $lRange
 
 # Leave a gap between the two generated C functions.
 #
 puts ""
 puts ""
 
 # Load the fold data. This is used by the [rd_XXX] commands
 # as well as [print_fold].
 tl_load_casefolding_txt ${casefolding.txt}
 
 set mappings [rd_load_unicodedata_text ${unicodedata.txt}]
 print_rd $mappings
 puts ""
 puts ""
-print_isdiacritic sqlite3FtsUnicodeIsdiacritic $mappings
+print_isdiacritic ${function_prefix}UnicodeIsdiacritic $mappings
 puts ""
 puts ""
 
 # Print the fold() function to stdout.
 #
-print_fold sqlite3FtsUnicodeFold
+print_fold ${function_prefix}UnicodeFold
 
 # Print the test routines and main() function to stdout, if -test 
 # was specified.
 #
 if {$::generate_test_code} {
-  print_test_isalnum sqlite3FtsUnicodeIsalnum $lRange
-  print_fold_test sqlite3FtsUnicodeFold $mappings
+  print_test_isalnum ${function_prefix}UnicodeIsalnum $lRange
+  print_fold_test ${function_prefix}UnicodeFold $mappings
   print_test_main 
 }
 
-puts "#endif /* defined(SQLITE_ENABLE_FTS3) || defined(SQLITE_ENABLE_FTS4) */"
-puts "#endif /* !defined(SQLITE_DISABLE_FTS3_UNICODE) */"
+if {$generate_fts5_code} {
+  # no-op
+} else {
+  puts "#endif /* defined(SQLITE_ENABLE_FTS3) || defined(SQLITE_ENABLE_FTS4) */"
+  puts "#endif /* !defined(SQLITE_DISABLE_FTS3_UNICODE) */"
+}