数据清洗demo 1

from urllib import urlopen

from BeautifulSoup import BeautifulSoup as BS

import re

def ngrams(input,n):

    input=input.replace("\n","")

    input=input.split(' ')

    output=[]

    for i in range(len(input)-n+1):

        output.append(input[i:i+1])

    return output

html=urlopen("https://en.wikipedia.org/wiki/Python_(programming_language)%22)")

bsObj=BS(html)

content=bsObj.find("div",{"id":"mw-content-text"})

ngrams=ngrams(str(content),2)

print ngrams

print "2-ngrams count is:" + str(len(ngrams))