Source file src/pkg/vendor/golang.org/x/text/unicode/norm/iter.go

     1	// Copyright 2011 The Go Authors. All rights reserved.
     2	// Use of this source code is governed by a BSD-style
     3	// license that can be found in the LICENSE file.
     4	
     5	package norm
     6	
     7	import (
     8		"fmt"
     9		"unicode/utf8"
    10	)
    11	
    12	// MaxSegmentSize is the maximum size of a byte buffer needed to consider any
    13	// sequence of starter and non-starter runes for the purpose of normalization.
    14	const MaxSegmentSize = maxByteBufferSize
    15	
    16	// An Iter iterates over a string or byte slice, while normalizing it
    17	// to a given Form.
    18	type Iter struct {
    19		rb     reorderBuffer
    20		buf    [maxByteBufferSize]byte
    21		info   Properties // first character saved from previous iteration
    22		next   iterFunc   // implementation of next depends on form
    23		asciiF iterFunc
    24	
    25		p        int    // current position in input source
    26		multiSeg []byte // remainder of multi-segment decomposition
    27	}
    28	
    29	type iterFunc func(*Iter) []byte
    30	
    31	// Init initializes i to iterate over src after normalizing it to Form f.
    32	func (i *Iter) Init(f Form, src []byte) {
    33		i.p = 0
    34		if len(src) == 0 {
    35			i.setDone()
    36			i.rb.nsrc = 0
    37			return
    38		}
    39		i.multiSeg = nil
    40		i.rb.init(f, src)
    41		i.next = i.rb.f.nextMain
    42		i.asciiF = nextASCIIBytes
    43		i.info = i.rb.f.info(i.rb.src, i.p)
    44		i.rb.ss.first(i.info)
    45	}
    46	
    47	// InitString initializes i to iterate over src after normalizing it to Form f.
    48	func (i *Iter) InitString(f Form, src string) {
    49		i.p = 0
    50		if len(src) == 0 {
    51			i.setDone()
    52			i.rb.nsrc = 0
    53			return
    54		}
    55		i.multiSeg = nil
    56		i.rb.initString(f, src)
    57		i.next = i.rb.f.nextMain
    58		i.asciiF = nextASCIIString
    59		i.info = i.rb.f.info(i.rb.src, i.p)
    60		i.rb.ss.first(i.info)
    61	}
    62	
    63	// Seek sets the segment to be returned by the next call to Next to start
    64	// at position p.  It is the responsibility of the caller to set p to the
    65	// start of a segment.
    66	func (i *Iter) Seek(offset int64, whence int) (int64, error) {
    67		var abs int64
    68		switch whence {
    69		case 0:
    70			abs = offset
    71		case 1:
    72			abs = int64(i.p) + offset
    73		case 2:
    74			abs = int64(i.rb.nsrc) + offset
    75		default:
    76			return 0, fmt.Errorf("norm: invalid whence")
    77		}
    78		if abs < 0 {
    79			return 0, fmt.Errorf("norm: negative position")
    80		}
    81		if int(abs) >= i.rb.nsrc {
    82			i.setDone()
    83			return int64(i.p), nil
    84		}
    85		i.p = int(abs)
    86		i.multiSeg = nil
    87		i.next = i.rb.f.nextMain
    88		i.info = i.rb.f.info(i.rb.src, i.p)
    89		i.rb.ss.first(i.info)
    90		return abs, nil
    91	}
    92	
    93	// returnSlice returns a slice of the underlying input type as a byte slice.
    94	// If the underlying is of type []byte, it will simply return a slice.
    95	// If the underlying is of type string, it will copy the slice to the buffer
    96	// and return that.
    97	func (i *Iter) returnSlice(a, b int) []byte {
    98		if i.rb.src.bytes == nil {
    99			return i.buf[:copy(i.buf[:], i.rb.src.str[a:b])]
   100		}
   101		return i.rb.src.bytes[a:b]
   102	}
   103	
   104	// Pos returns the byte position at which the next call to Next will commence processing.
   105	func (i *Iter) Pos() int {
   106		return i.p
   107	}
   108	
   109	func (i *Iter) setDone() {
   110		i.next = nextDone
   111		i.p = i.rb.nsrc
   112	}
   113	
   114	// Done returns true if there is no more input to process.
   115	func (i *Iter) Done() bool {
   116		return i.p >= i.rb.nsrc
   117	}
   118	
   119	// Next returns f(i.input[i.Pos():n]), where n is a boundary of i.input.
   120	// For any input a and b for which f(a) == f(b), subsequent calls
   121	// to Next will return the same segments.
   122	// Modifying runes are grouped together with the preceding starter, if such a starter exists.
   123	// Although not guaranteed, n will typically be the smallest possible n.
   124	func (i *Iter) Next() []byte {
   125		return i.next(i)
   126	}
   127	
   128	func nextASCIIBytes(i *Iter) []byte {
   129		p := i.p + 1
   130		if p >= i.rb.nsrc {
   131			p0 := i.p
   132			i.setDone()
   133			return i.rb.src.bytes[p0:p]
   134		}
   135		if i.rb.src.bytes[p] < utf8.RuneSelf {
   136			p0 := i.p
   137			i.p = p
   138			return i.rb.src.bytes[p0:p]
   139		}
   140		i.info = i.rb.f.info(i.rb.src, i.p)
   141		i.next = i.rb.f.nextMain
   142		return i.next(i)
   143	}
   144	
   145	func nextASCIIString(i *Iter) []byte {
   146		p := i.p + 1
   147		if p >= i.rb.nsrc {
   148			i.buf[0] = i.rb.src.str[i.p]
   149			i.setDone()
   150			return i.buf[:1]
   151		}
   152		if i.rb.src.str[p] < utf8.RuneSelf {
   153			i.buf[0] = i.rb.src.str[i.p]
   154			i.p = p
   155			return i.buf[:1]
   156		}
   157		i.info = i.rb.f.info(i.rb.src, i.p)
   158		i.next = i.rb.f.nextMain
   159		return i.next(i)
   160	}
   161	
   162	func nextHangul(i *Iter) []byte {
   163		p := i.p
   164		next := p + hangulUTF8Size
   165		if next >= i.rb.nsrc {
   166			i.setDone()
   167		} else if i.rb.src.hangul(next) == 0 {
   168			i.rb.ss.next(i.info)
   169			i.info = i.rb.f.info(i.rb.src, i.p)
   170			i.next = i.rb.f.nextMain
   171			return i.next(i)
   172		}
   173		i.p = next
   174		return i.buf[:decomposeHangul(i.buf[:], i.rb.src.hangul(p))]
   175	}
   176	
   177	func nextDone(i *Iter) []byte {
   178		return nil
   179	}
   180	
   181	// nextMulti is used for iterating over multi-segment decompositions
   182	// for decomposing normal forms.
   183	func nextMulti(i *Iter) []byte {
   184		j := 0
   185		d := i.multiSeg
   186		// skip first rune
   187		for j = 1; j < len(d) && !utf8.RuneStart(d[j]); j++ {
   188		}
   189		for j < len(d) {
   190			info := i.rb.f.info(input{bytes: d}, j)
   191			if info.BoundaryBefore() {
   192				i.multiSeg = d[j:]
   193				return d[:j]
   194			}
   195			j += int(info.size)
   196		}
   197		// treat last segment as normal decomposition
   198		i.next = i.rb.f.nextMain
   199		return i.next(i)
   200	}
   201	
   202	// nextMultiNorm is used for iterating over multi-segment decompositions
   203	// for composing normal forms.
   204	func nextMultiNorm(i *Iter) []byte {
   205		j := 0
   206		d := i.multiSeg
   207		for j < len(d) {
   208			info := i.rb.f.info(input{bytes: d}, j)
   209			if info.BoundaryBefore() {
   210				i.rb.compose()
   211				seg := i.buf[:i.rb.flushCopy(i.buf[:])]
   212				i.rb.insertUnsafe(input{bytes: d}, j, info)
   213				i.multiSeg = d[j+int(info.size):]
   214				return seg
   215			}
   216			i.rb.insertUnsafe(input{bytes: d}, j, info)
   217			j += int(info.size)
   218		}
   219		i.multiSeg = nil
   220		i.next = nextComposed
   221		return doNormComposed(i)
   222	}
   223	
   224	// nextDecomposed is the implementation of Next for forms NFD and NFKD.
   225	func nextDecomposed(i *Iter) (next []byte) {
   226		outp := 0
   227		inCopyStart, outCopyStart := i.p, 0
   228		for {
   229			if sz := int(i.info.size); sz <= 1 {
   230				i.rb.ss = 0
   231				p := i.p
   232				i.p++ // ASCII or illegal byte.  Either way, advance by 1.
   233				if i.p >= i.rb.nsrc {
   234					i.setDone()
   235					return i.returnSlice(p, i.p)
   236				} else if i.rb.src._byte(i.p) < utf8.RuneSelf {
   237					i.next = i.asciiF
   238					return i.returnSlice(p, i.p)
   239				}
   240				outp++
   241			} else if d := i.info.Decomposition(); d != nil {
   242				// Note: If leading CCC != 0, then len(d) == 2 and last is also non-zero.
   243				// Case 1: there is a leftover to copy.  In this case the decomposition
   244				// must begin with a modifier and should always be appended.
   245				// Case 2: no leftover. Simply return d if followed by a ccc == 0 value.
   246				p := outp + len(d)
   247				if outp > 0 {
   248					i.rb.src.copySlice(i.buf[outCopyStart:], inCopyStart, i.p)
   249					// TODO: this condition should not be possible, but we leave it
   250					// in for defensive purposes.
   251					if p > len(i.buf) {
   252						return i.buf[:outp]
   253					}
   254				} else if i.info.multiSegment() {
   255					// outp must be 0 as multi-segment decompositions always
   256					// start a new segment.
   257					if i.multiSeg == nil {
   258						i.multiSeg = d
   259						i.next = nextMulti
   260						return nextMulti(i)
   261					}
   262					// We are in the last segment.  Treat as normal decomposition.
   263					d = i.multiSeg
   264					i.multiSeg = nil
   265					p = len(d)
   266				}
   267				prevCC := i.info.tccc
   268				if i.p += sz; i.p >= i.rb.nsrc {
   269					i.setDone()
   270					i.info = Properties{} // Force BoundaryBefore to succeed.
   271				} else {
   272					i.info = i.rb.f.info(i.rb.src, i.p)
   273				}
   274				switch i.rb.ss.next(i.info) {
   275				case ssOverflow:
   276					i.next = nextCGJDecompose
   277					fallthrough
   278				case ssStarter:
   279					if outp > 0 {
   280						copy(i.buf[outp:], d)
   281						return i.buf[:p]
   282					}
   283					return d
   284				}
   285				copy(i.buf[outp:], d)
   286				outp = p
   287				inCopyStart, outCopyStart = i.p, outp
   288				if i.info.ccc < prevCC {
   289					goto doNorm
   290				}
   291				continue
   292			} else if r := i.rb.src.hangul(i.p); r != 0 {
   293				outp = decomposeHangul(i.buf[:], r)
   294				i.p += hangulUTF8Size
   295				inCopyStart, outCopyStart = i.p, outp
   296				if i.p >= i.rb.nsrc {
   297					i.setDone()
   298					break
   299				} else if i.rb.src.hangul(i.p) != 0 {
   300					i.next = nextHangul
   301					return i.buf[:outp]
   302				}
   303			} else {
   304				p := outp + sz
   305				if p > len(i.buf) {
   306					break
   307				}
   308				outp = p
   309				i.p += sz
   310			}
   311			if i.p >= i.rb.nsrc {
   312				i.setDone()
   313				break
   314			}
   315			prevCC := i.info.tccc
   316			i.info = i.rb.f.info(i.rb.src, i.p)
   317			if v := i.rb.ss.next(i.info); v == ssStarter {
   318				break
   319			} else if v == ssOverflow {
   320				i.next = nextCGJDecompose
   321				break
   322			}
   323			if i.info.ccc < prevCC {
   324				goto doNorm
   325			}
   326		}
   327		if outCopyStart == 0 {
   328			return i.returnSlice(inCopyStart, i.p)
   329		} else if inCopyStart < i.p {
   330			i.rb.src.copySlice(i.buf[outCopyStart:], inCopyStart, i.p)
   331		}
   332		return i.buf[:outp]
   333	doNorm:
   334		// Insert what we have decomposed so far in the reorderBuffer.
   335		// As we will only reorder, there will always be enough room.
   336		i.rb.src.copySlice(i.buf[outCopyStart:], inCopyStart, i.p)
   337		i.rb.insertDecomposed(i.buf[0:outp])
   338		return doNormDecomposed(i)
   339	}
   340	
   341	func doNormDecomposed(i *Iter) []byte {
   342		for {
   343			i.rb.insertUnsafe(i.rb.src, i.p, i.info)
   344			if i.p += int(i.info.size); i.p >= i.rb.nsrc {
   345				i.setDone()
   346				break
   347			}
   348			i.info = i.rb.f.info(i.rb.src, i.p)
   349			if i.info.ccc == 0 {
   350				break
   351			}
   352			if s := i.rb.ss.next(i.info); s == ssOverflow {
   353				i.next = nextCGJDecompose
   354				break
   355			}
   356		}
   357		// new segment or too many combining characters: exit normalization
   358		return i.buf[:i.rb.flushCopy(i.buf[:])]
   359	}
   360	
   361	func nextCGJDecompose(i *Iter) []byte {
   362		i.rb.ss = 0
   363		i.rb.insertCGJ()
   364		i.next = nextDecomposed
   365		i.rb.ss.first(i.info)
   366		buf := doNormDecomposed(i)
   367		return buf
   368	}
   369	
   370	// nextComposed is the implementation of Next for forms NFC and NFKC.
   371	func nextComposed(i *Iter) []byte {
   372		outp, startp := 0, i.p
   373		var prevCC uint8
   374		for {
   375			if !i.info.isYesC() {
   376				goto doNorm
   377			}
   378			prevCC = i.info.tccc
   379			sz := int(i.info.size)
   380			if sz == 0 {
   381				sz = 1 // illegal rune: copy byte-by-byte
   382			}
   383			p := outp + sz
   384			if p > len(i.buf) {
   385				break
   386			}
   387			outp = p
   388			i.p += sz
   389			if i.p >= i.rb.nsrc {
   390				i.setDone()
   391				break
   392			} else if i.rb.src._byte(i.p) < utf8.RuneSelf {
   393				i.rb.ss = 0
   394				i.next = i.asciiF
   395				break
   396			}
   397			i.info = i.rb.f.info(i.rb.src, i.p)
   398			if v := i.rb.ss.next(i.info); v == ssStarter {
   399				break
   400			} else if v == ssOverflow {
   401				i.next = nextCGJCompose
   402				break
   403			}
   404			if i.info.ccc < prevCC {
   405				goto doNorm
   406			}
   407		}
   408		return i.returnSlice(startp, i.p)
   409	doNorm:
   410		// reset to start position
   411		i.p = startp
   412		i.info = i.rb.f.info(i.rb.src, i.p)
   413		i.rb.ss.first(i.info)
   414		if i.info.multiSegment() {
   415			d := i.info.Decomposition()
   416			info := i.rb.f.info(input{bytes: d}, 0)
   417			i.rb.insertUnsafe(input{bytes: d}, 0, info)
   418			i.multiSeg = d[int(info.size):]
   419			i.next = nextMultiNorm
   420			return nextMultiNorm(i)
   421		}
   422		i.rb.ss.first(i.info)
   423		i.rb.insertUnsafe(i.rb.src, i.p, i.info)
   424		return doNormComposed(i)
   425	}
   426	
   427	func doNormComposed(i *Iter) []byte {
   428		// First rune should already be inserted.
   429		for {
   430			if i.p += int(i.info.size); i.p >= i.rb.nsrc {
   431				i.setDone()
   432				break
   433			}
   434			i.info = i.rb.f.info(i.rb.src, i.p)
   435			if s := i.rb.ss.next(i.info); s == ssStarter {
   436				break
   437			} else if s == ssOverflow {
   438				i.next = nextCGJCompose
   439				break
   440			}
   441			i.rb.insertUnsafe(i.rb.src, i.p, i.info)
   442		}
   443		i.rb.compose()
   444		seg := i.buf[:i.rb.flushCopy(i.buf[:])]
   445		return seg
   446	}
   447	
   448	func nextCGJCompose(i *Iter) []byte {
   449		i.rb.ss = 0 // instead of first
   450		i.rb.insertCGJ()
   451		i.next = nextComposed
   452		// Note that we treat any rune with nLeadingNonStarters > 0 as a non-starter,
   453		// even if they are not. This is particularly dubious for U+FF9E and UFF9A.
   454		// If we ever change that, insert a check here.
   455		i.rb.ss.first(i.info)
   456		i.rb.insertUnsafe(i.rb.src, i.p, i.info)
   457		return doNormComposed(i)
   458	}
   459	

View as plain text