Parse local html file with python and beautifulsoup

from BeautifulSoup import BeautifulSoup

f = open("test.html")      # simplified for the example (no urllib)
soup = BeautifulSoup(f)
f.close()

titleLst = soup.findAll("h1", {"class": "title"})
print titleLst             # the list with single

title = titleLst[0].contents[0]
print title                # the string from the h1 element

g_a = soup.findAll("div", {"id" : "a"})  # the elements from inside the div a element
alst = []                                # the future result list
for x in g_a:
    for elem in x.findAll('div'):        # the divs inside the div a
        print elem.contents              # just to see what is inside
        alst.append(elem.contents[0].strip('\n\t ,'))  # collect the wanted info
        
print alst               # wanted result in a technical form            
print ', '.join(alst)    # wanted result as a string (items separated by comma and space

Select allOpen in new window

from BeautifulSoup import BeautifulSoup
import re

f = open('test.html')      # simplified for the example (no urllib)
soup = BeautifulSoup(f)
f.close()

h1_title = soup.find('h1', {'class': 'title'})
print h1_title             # the header element

title = h1_title.contents[0]
print title                # the string from the h1 element

rowLst = [ title ]
div_a = soup.find('div', {'id' : 'a'})   # the div a element
lst_a = []                               # the future result list
for elem in div_a.findAll('div'):        # the divs inside the div a
    print elem.contents                  # just to see what is inside
    lst_a.append(elem.contents[0].strip('\n\t ,'))  # collect the wanted info
        
print lst_a              # wanted result in a technical form
cat = ', '.join(lst_a)   # wanted result as a string (items separated by comma and space            
print cat                # (I do not know what meaning has 'cat' for you.)
rowLst.append(cat)

div_b = soup.find('div', {'id' : 'b'})
rexClean = re.compile(r'^\s*(?P<id>\w+:)\s*(?P<text>.*?)\s*$', re.MULTILINE)
p = div_b.find('p')      # The p element from div b
lst_b = []
for x in p.contents:
    if getattr(x, 'name', None) == 'br':
        continue                          # ignote the br elements
    elif getattr(x, 'name', None) == 'a':
        lst_b.append(x['href'])           # href attribute here (not the enclosed string)
    else:    
        lst_b.append(rexClean.sub(r'\2', x))  # only the info without whitespaces
print lst_b    

# Remove the empty strings and newline strings, leave out the 'mailto:'.
lst_b2 = []
for s in lst_b:
    if s == u'' or s == u'\n':
        continue
    else:
        lst_b2.append(s.replace('mailto:', ''))
print lst_b2    

# Append all elements from lst_b2 to the column list.
rowLst.extend(lst_b2)

# Extraction from the div c.
div_c = soup.find('div', {'id' : 'c'})
cu1 = div_c.find('a', {'href': 'c.pl?u=1'})
print cu1
rowLst.append(cu1.contents[0])

cu2 = div_c.find('a', {'href': 'c.pl?u=2'})
print cu2
rowLst.append(cu2.contents[0])

# Show the rowLst -- this should be the Excel line in future.
print rowLst

Select allOpen in new window

from BeautifulSoup import BeautifulSoup
import csv
import glob
import re

def getRowFromFilename(fname):
    f = open(fname)              # simplified for the example (no urllib)
    soup = BeautifulSoup(f)
    f.close()

    # Extract the title.
    h1_title = soup.find('h1', {'class': 'title'})  # the header element
    title = h1_title.contents[0]             # the string from the h1 element

    rowLst = [ title ]                       # first column in the result list

    # Extract the div a info as one string.
    div_a = soup.find('div', {'id' : 'a'})   # the div a element
    lst_a = []                               # the future result list
    for elem in div_a.findAll('div'):        # the divs inside the div a
        lst_a.append(elem.contents[0].strip('\n\t ,'))  # collect the wanted info
    cat = ', '.join(lst_a)                   # wanted result as a string
    rowLst.append(cat)

    # Extract the div b info as several strings.
    div_b = soup.find('div', {'id' : 'b'})
    rexClean = re.compile(r'^\s*(?P<id>\w+:)\s*(?P<text>.*?)\s*$', re.MULTILINE)
    p = div_b.find('p')      # The p element from div b
    lst_b = []
    for x in p.contents:
        if getattr(x, 'name', None) == 'br':
            continue                          # ignote the br elements
        elif getattr(x, 'name', None) == 'a':
            lst_b.append(x['href'])           # href attribute here
        else:    
            lst_b.append(rexClean.sub(r'\2', x))  # only the info without whitespaces

    # Remove the empty strings and newline strings, leave out the 'mailto:'.
    lst_b2 = []
    for s in lst_b:
        if s == u'' or s == u'\n':
            continue
        else:
            lst_b2.append(s.replace('mailto:', ''))

    # Append all elements from lst_b2 to the column list.
    rowLst.extend(lst_b2)

    # Extraction from the div c.
    div_c = soup.find('div', {'id' : 'c'})
    cu1 = div_c.find('a', {'href': 'c.pl?u=1'})
    rowLst.append(cu1.contents[0])

    cu2 = div_c.find('a', {'href': 'c.pl?u=2'})
    rowLst.append(cu2.contents[0])
    
    # Return the resulting row.
    return rowLst


#-------------------------------------------------------------------------
# Get the sequence of the processed filenames, extract the row from each
# of them, and write it to the csv file for Excel.  Beware the Excel.  The dialect
# may not be enough for your purpose.  Some experiments may be neccessary.

f = open('result.csv', 'wb')                  # must be in binary mode
writer = csv.writer(f, dialect='excel')       # csv writer wrap around the f

for fname in sorted(glob.glob('test*.html')): # can be modified to fit your needs
    print fname                               # just to see what is processed
    rowLst = getRowFromFilename(fname)        # extract the info from one file
    writer.writerow(rowLst)                   # write it to one row of the result

f.close()            # do not forget to close the result file

Select allOpen in new window

for n in range(1, 122):
    url = 'http://www.test.com/%d.html' % n
    print url

Select allOpen in new window

if cu1 is not None:
        rowLst.append(cu1.contents[0])
    else:
        rowLst.append('')             # i.e. empty string

Select allOpen in new window