slm/python/pinyin_seg.py

#!/usr/bin/python

# DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS HEADER.
#
# Copyright (c) 2009 Sun Microsystems, Inc. All Rights Reserved.
#
# The contents of this file are subject to the terms of either the GNU Lesser
# General Public License Version 2.1 only ("LGPL") or the Common Development and
# Distribution License ("CDDL")(collectively, the "License"). You may not use this
# file except in compliance with the License. You can obtain a copy of the CDDL at
# http://www.opensource.org/licenses/cddl1.php and a copy of the LGPLv2.1 at
# http://www.opensource.org/licenses/lgpl-license.php. See the License for the
# specific language governing permissions and limitations under the License. When
# distributing the software, include this License Header Notice in each file and
# include the full text of the License in the License file as well as the
# following notice:
#
# NOTICE PURSUANT TO SECTION 9 OF THE COMMON DEVELOPMENT AND DISTRIBUTION LICENSE
# (CDDL)
# For Covered Software in this distribution, this License shall be governed by the
# laws of the State of California (excluding conflict-of-law provisions).
# Any litigation relating to this License shall be subject to the jurisdiction of
# the Federal Courts of the Northern District of California and the state courts
# of the State of California, with venue lying in Santa Clara County, California.
#
# Contributor(s):
#
# If you wish your version of this file to be governed by only the CDDL or only
# the LGPL Version 2.1, indicate your decision by adding "[Contributor]" elects to
# include this software in this distribution under the [CDDL or LGPL Version 2.1]
# license." If you don't indicate a single choice of license, a recipient has the
# option to distribute your version of this file under either the CDDL or the LGPL
# Version 2.1, or to extend the choice of license to its licensees as provided
# above. However, if you add LGPL Version 2.1 code and therefore, elected the LGPL
# Version 2 license, then the option applies only if the new code is made subject
# to such option by the copyright holder.

from trie import DATrie, match_longest

class IPySegmentor:
    class CSegment:
        def __init__ (self, value, length):
            self.value = value
            self.length = length

    def get_segments (self): pass
    def updated_from (self): pass
    def push (self, ch): pass
    def pop (self): pass
    def insert (self, ch, idx): pass
    def delete (self, idx, backward=True): pass
    def clear (self): pass

class QuanPinSegmentor (IPySegmentor):
    def __init__ (self, fname):
        self.clear ()
        self.load (fname)

    def clear (self):
        self.__updated_from = 0
        self.__segs = []
        self.__pystr = []

    def load (self, fname):
        self.pytrie = DATrie ()

        try:
            self.pytrie.load (fname)
        except:
            from trie import Trie
            from pinyin_data import valid_syllables

            trie = Trie ()
            for s in valid_syllables:
                trie.add (s[::-1], valid_syllables[s])

            self.pytrie.construct_from_trie (trie)
            self.pytrie.save (fname)

    def updated_from (self):
        return self.__updated_from

    def get_segments (self):
        return self.__segs[::-1]

    def __locate_indices (self, idx):
        'return the indices of pystr and segs'
        i = 0; j = -1
        while i+self.__segs[j].length < idx:
            i += self.__segs[j].length
            j -= 1
        j += len(self.__segs)
        i = len(self.__pystr) - i
        return i, j

    def insert (self, ch, idx):
        i, j = self.__locate_indices (idx)
        self.__updated_from = len(self.__pystr) - i

        if (idx == 0): self.__pystr.append(ch)
        else:          self.__pystr.insert(-idx, ch)

        newpystr = self.__pystr[:i+1]
        self.__pystr = self.__pystr[i+1:]
        self.__segs = self.__segs[j+1:]

        for ch in newpystr[::-1]:
            self.__push (ch.lower())

    def delete (self, idx, backward=True):
        if not backward: idx += 1
        i, j = self.__locate_indices (idx)
        self.__updated_from = len(self.__pystr) - i

        self.__pystr.pop(-idx)

        newpystr = self.__pystr[:i-1]
        self.__pystr = self.__pystr[i-1:]
        self.__segs = self.__segs[j+1:]

        for ch in newpystr[::-1]:
            self.__push (ch.lower())

    def __push (self, ch):
        self.__pystr.insert(0, ch)
        v, l = match_longest (self.pytrie, self.__pystr)

        if l == 0:
            self.__segs.insert(0, IPySegmentor.CSegment(0, 1))
            return

        if l == 1:
            if len(self.__pystr)>1 and self.__pystr[1].isupper():
                self.__pystr[1] = self.__pystr[1].lower()
                vv, ll = match_longest (self.pytrie, self.__pystr)
                if ll == self.__segs[0].length + 1:
                    self.__segs[0].length += 1
                    self.__segs[0].value = vv
                    return
                self.__pystr[1] = self.__pystr[1].upper()

            self.__segs.insert(0, IPySegmentor.CSegment(v, l))
            return

        if l == self.__segs[0].length + 1:
            self.__segs[0].length += 1
            self.__segs[0].value = v
            return

        i, isum = 0, self.__segs[0].length + 1
        lsum, new_segs = l, [IPySegmentor.CSegment(v, l)]
        if isum < lsum: self.__pystr[0] = self.__pystr[0].upper()
        while isum != lsum:
            if lsum < isum:
                v, l = match_longest (self.pytrie, self.__pystr[lsum:])
                new_segs.append(IPySegmentor.CSegment(v, l))
                lsum += l
            if isum < lsum:
                i += 1
                isum += self.__segs[i].length
        self.__segs = new_segs + self.__segs[i+1:]

    def push (self, ch):
        self.__push (ch)
        self.__updated_from = len(self.__pystr) - self.__segs[0].length

    def __pop (self):
        if not self.__pystr:
            return

        i = self.__segs.pop(0).length
        if i == 1:
            self.__pystr.pop(0)
            return

        newpystr, self.__pystr = self.__pystr[1:i], self.__pystr[i:]

        for ch in newpystr[::-1]:
            self.__push (ch.lower())

    def pop (self):
        self.__updated_from = len(self.__pystr) - self.__segs[0].length
        self.__pop ()

def test ():
    from pinyin_data import valid_syllables
    r_valid_syllables = dict (zip (valid_syllables.values(), valid_syllables.keys()))

    pyseg = QuanPinSegmentor ('pinyin_suffix.dat')
    pystr = "min'ganceshizhongdierlongenfengniao"
    print pystr

    def print_segs ():
        for s in pyseg.get_segments ():
            print "[0x%x:%d:%s]" % (s.value, s.length, r_valid_syllables[s.value] if s.value else '?'),
        print 'UPDATED_FROM:', pyseg.updated_from()

    for ch in pystr:
        pyseg.push (ch)
        print_segs ()

    pyseg.pop ()
    print_segs ()

    pyseg.pop ()
    print_segs ()

    pyseg.insert ('i', 2)
    print_segs ()

    pyseg.delete (2, True)
    print_segs ()

if __name__ == "__main__":
    test ()