# Last update: 2003/12/07
# 
# HTML Table to CSV
#
# (p) Jerry Nagasaki
#
#
# Converts "simple" n*m HTML tables (w/o merged cells or nested
# tables) to a csv file.
#
# INPUT:  Extracted HTML table (<table> ... </table>) named "table.html"
# OUTPUT: Generated CSV File (;) named "table.csv"
#
# (optional 2nd output: "Plain Table" named "table2.html")
#
#
# Notes: 
# - New-line breaks in <td>-texts are NOT converted to a "simple space"
# - Empty <td> entries should be filled with a character in the source file
#

open(FILE, "<table.html") || die "Could not open table.html\n";
my @infile = <FILE>;	
close(FILE);


@dummy = "";
@dummy2 = "";
@table2 = "";
$newelem = "";

foreach (@infile)
{
	$_ =~ s/\n//g;			# remove \n of line
	$newelem = $newelem.$_;
}
push (@dummy, $newelem);


$i = 0;

foreach (@dummy)
{
	print "$i $_";
	$i++;
	$_ =~ s/\&nbsp;/ /g;		# turn &nbsp to "simple space"
	$_ =~ tr/ //s;			# remove multiple spaces
	$_ =~ s/\t//g;			# remove tabs
	$_ =~ s/<table.+?>/<table>/gi;	# remove additional table arguments
	$_ =~ s/<tr.+?>/<tr>/gi;	# remove additional tr arguments
	$_ =~ s/<th.+?>/<th>/gi;	# remove additional th arguments
	$_ =~ s/<td.+?>/<td>/gi;	# remove additional td arguments
	$_ =~ s/<font.+?>//gi;		# remove <font*>
	$_ =~ s/<\/font>//gi;		# remove </font>
	$_ =~ s/<img.+?>//g;		# remove <img*>
	$_ =~ s/<!--.+?-->//gi;		# remove comments
	$_ =~ s/<a href=.+?>//gi;	# remove links
	$_ =~ s/<\/a>/ /gi;		# remove /links
	$_ =~ s/\s*>/>/g;		# remove spaces in tags	
	$_ =~ s/>\s/>/g;		# remove space after tags
	$_ =~ s/<center>//g;		# remove <center>
	$_ =~ s/<\/center>//g;		# remove </center>	
	$_ =~ s/<br>//gi;		# remove line break tags
	$_ =~ s/<code>//gi;		# remove style tags
	$_ =~ s/<\/code>//gi;
	$_ =~ s/<var>//gi;		# remove <var>
	$_ =~ s/<\/var>/ /gi;		# remove <var>
	$_ =~ s/<i>//gi;		# remove intalic tags
	$_ =~ s/<\/i>//gi;
	$_ =~ s/<p>//gi;		# remove paragraph tags
	$_ =~ s/<\/p>//gi;
	$_ =~ s/<b>//gi;		# remove bold tags
	$_ =~ s/nowrap//g;		# remove nowrap
	$_ =~ s/<\/b>//gi;
	$_ =~ s/<th>/<td>/gi;		# trun <th> to <td>
	$_ =~ s/<\/th>/<\/td>/gi;	# trun </th> to </td>
	
	$_ =~ s/\n//gi;			# remove all \n
	$_ =~ s/<table>/<table>\n/gi;	# set \n after <table>
	$_ =~ s/<tr>/<tr>\n/gi;		# set \n after <tr>
	$_ =~ s/<\/tr>/<\/tr>\n/gi;	# set \n after </tr>
	$_ =~ s/<\/td>/<\/td>\n/gi;	# set \n after </td>	
	$_ =~ s/\s</</g;		# remove leading space before tag
	
	# at this point table is still html valid but completely unformated
	push (@table2, $_);
	
	$_ =~ s/\&Auml;/Ä/g;		# convert special characters ...
	$_ =~ s/\&auml;/ä/g;
	$_ =~ s/\&Ouml;/Ö/g;
	$_ =~ s/\&ouml;/ö/g;
	$_ =~ s/\&Uuml;/Ü/g;
	$_ =~ s/\&uuml;/ü/g;
	$_ =~ s/\&szlig;/ß/g;
	$_ =~ s/\&quot;/\"/g;
	$_ =~ s/\&\#132;/\"/g;		# Anfuerungsstrich unten (,,) zu "
	$_ =~ s/\&\#147;/\"/g;		# Anfuerungsstrich oben (´´) zu "
	
	# make csv ";" format
	
	$_ =~ s/;/,/g;			# turn ; to , to preserve csv format
	$_ =~ s/<table>//gi;		# remove <table>
	$_ =~ s/<\/table>//gi;		# remove </table>
	$_ =~ s/<tr>//gi;		# remove <tr>
	$_ =~ s/<td>//gi;		# remove <td>
	$_ =~ s/<\/td>\n/;/gi;		# turn </td> to ;
	$_ =~ s/\n//g;			# remove all \n
	$_ =~ s/<\/tr>/\n/gi;		# turn <tr> to \n
		
	push (@dummy2, $_);
}

# make all elements of @dummy-$_ stream until \n to ONE element
#  (for finding last ";")

@outfile = "";
$newelem = "";

foreach (@dummy2)
{
	$newelem = $newelem.$_;
	
	if ($_ =~ /\n/)			# make new element if \n found
	{
		$newelem =~ s/;\n/\n/g;	# remove last ; of line
		push (@outfile, $newelem);
		$newelem = "";
	}
}

open(FILE, ">./table.csv");
print FILE @outfile;	
close(FILE);

# un-comment the following lines, if plain text table should
#  be generated:

#open(FILE, ">./table2.html");
#print FILE @table2;	
#close(FILE);