OnLinux: extract data from html table using HTML::TableExtract

in this tutorial you will get to learn :

1) How to extract data from a html table

2 ) regex

3) appending a hash to a hash

4) for loops

----------------------------------------------------------------------------------------------

HTML file

----------------------------------------------------------------------------------------------

<table class="table1" border="1" cellspacing="1" cellpadding="1" style="width: 1144px; height: 883px;">

<tr>

<td> Dom0 

</td><td> DomU 

</td><td> ipaddress 

</td><td> server name 

</td><td> application 

</td><td> website 

</td><td> assigned to 

</td></tr>

<tr>

<td> S1 (1.51) 

</td><td> yourwebsitenews-db 10GB ram 100 GB HDD   

</td><td> 1.0.0.101 

</td><td> yourwebsitenews-db

</td><td> win2k8 ,sql

</td><td> MSSQL Server

</td><td> yourwebsitenews

</td></tr>

<tr>

<td> 

</td><td> yourwebsitetvweb3  4GB ram 48 GB HDD 

</td><td> 9.9.02.132 

</td><td> yourwebsitetvweb3 

</td><td> win2k8,IIS 

</td><td> yourwebsite TV 

</td><td> yourwebsitetv 

</td></tr>

<tr>

<td> 

</td><td> yourwebsitetvweb4  4GB ram 30 GB HDD 

</td><td> 9.9.202.146 

</td><td> yourwebsitetvweb4 

</td><td> win2k8,IIS 

</td><td> yourwebsite TV 

</td><td> yourwebsitetv 

</td></tr>

<tr>

<td> 

</td><td> 

</td></tr>

<tr>

<td> 

</td><td> 

</td></tr>

<tr>

<td> S2 (1.52) 

</td><td> web1 5.5GB ram 50GB hdd 

</td><td> 1.0.0.106 

</td><td> web1 

</td><td> apache,php,memcache 

</td><td> someweb,cndas,lms.someweb.com 

</td><td> IWPL

</td></tr>

<tr>

<td> 

</td><td> web4 5.5GB ram 50GB hdd 

</td><td> 1.0.0.109 

</td><td> web4 

</td><td> apache,php,memcache 

</td><td> b.in,gs.in.com

</td><td> IWPL

</td></tr>

<tr>

<td> 

</td><td> 

</td></tr>

<tr>

<td> S3 (1.53) 

</td><td> DB1 11GB ram 100GB hdd 

</td><td> 1.0.0.111 

</td><td> db1 

</td><td> mysql 

</td><td> all commons (auth ),someweb 

</td><td> IL

</td></tr>

<tr>

<td> 

<tr>

<td> 

</td><td> 

</td></tr>

<tr>

<td> S4 (1.54) 

</td><td> DB3 10GB ram 200GB hdd 

</td><td> 1.0.0.116 

</td><td> db3 

</td><td> mysql 

</td><td> ors_db,ls_ting

</td><td> IWPL

</td></tr>

<tr>

<td> 

</td><td> yourwebsitetvmemcach-1 10 Gb ram 24 Gb hdd 

</td><td> 9.9.202.134 

</td><td> yourwebsitetvmem-1 

</td><td> yourwebsite Tv Memcached Server-1 

</td><td> yourwebsitetv

</td></tr>

<tr>

</td><td> 

</td></tr>

<tr>

<td> S5 (1.55) 

</td><td> 1 systems ops 8GB 100GB 

</td><td> 1.0.0.121 

</td><td> ops 

</td><td> mysql,php,apache 

</td><td> odw opsview reports runtime 

</td><td> IWPL

</td></tr>

<tr>

<td> 

</td><td> cache 4GB ram 129GB hdd 

</td><td> 9.9.202.209 

</td><td> linux-yourwebsite 

</td><td> php,apache 

</td><td> yourwebsite 

</td><td> yourwebsitetv

</td></tr>

<tr>

<td> 

</td><td> cache 6GB ram 20GB hdd 

</td><td> 1.0.0.125 

</td><td> cache-9 

</td><td> backup server

</td><td> IL

</td></tr>

<tr>

<td> 

</td><td> web16- 5 GB RAM 46GB HDD

</td><td> 1.0.0.103

</td><td> web16

</td><td> php,apache

</td><td> voiofa.com,solr instance

</td><td> Il

</td></tr>

<tr>

<td> 

</td><td> 

</td></tr>

<tr>

<td> S6 (1.56) 

</td><td> web9 5GBram 

</td><td> 1.0.0.126 

</td><td> web9 

</td><td> apache,php,memcache 

</td><td> www.ia.com,dia.com,dia.com,bon.ia.com 

</td><td> Il

</td></tr>

<tr>

<td> 

</td><td> web10 

</td><td> 1.0.0.127 

</td><td> web10 

</td><td> apache,php,memcache 

</td><td> hea.ia.com,esults.ia.com

</td><td> Il

</td></tr>

<tr>

<td> 

</td><td> r1s6v1web13 5-GB ram - 50-GB hdd 

</td><td> 1.0.0.129 

</td><td> web13 

</td><td> apache,php,memcache

</td><td> critry,m.try,admin.ccry.com

</td><td> IL

</td></tr></table>

==================================================================

PERL script

==================================================================

#!/usr/bin/perl

use strict;
use warnings;
use Data::Dumper;
use LWP::Simple;
use HTML::TableExtract;

#host your table on a local site
my $url = 'http://site2.int.com/table_tmp1.html';

my @data = get_data($url) ;

shift(@data);
my %serverdata;
my %serverdata_sub;

my $DOM0;

my @aeey;

my $re1='((?:[a-z][a-z]*[0-9]+[a-z0-9]*))'; # Alphanum 1
my $re2='(\\s+)'; # White Space 1
my $re3='\$(.*)\$'; # Round Braces 1

foreach my $y (@data) {
next unless ($y->[2]);
$DOM0 = ($y->[0])? $y->[0] : $DOM0;

my $re=$re1.$re2.$re3;
($DOM0 = $DOM0) =~ s/$re/172.16.$3/isg; # S1 (1.51)

# Dom0 DomU ipaddress server name application website assigned to
# "$DOM0 -----> $y->[2] ========== > $y->[1], $y->[3], $y->[4] $y->[5] $y->[6] ";

push (@{$serverdata{$DOM0}{$y->[2]}}, { 'serverdetails' => $y->[1] , 'servername' => $y->[3], 'application' => $y->[4] , 'websites' => $y->[5] } ) ;

if ( $y->[0] ) {
$DOM0= "$y->[0]";
}else {
$DOM0 = $DOM0;
}
}

foreach my $keysi ( keys %serverdata) {

print "================== $keysi ===============\n";
print Dumper ($serverdata{$keysi}) ;
print "================== ===============\n";

}

sub get_data {
my $sub_url = $_[0];
my $html = get "$sub_url";

my @arry;
my $te = new HTML::TableExtract( depth => 0, count => 0, attribs => { 'class' => "table1" } );

$te->parse($html);
foreach my $ts ($te->tables) {
foreach my $row ($ts->rows) {
# next unless $row->[0] =~ /\w/; # skip garbage rows
@{$row} = grep(s/\n\n|(^\s+|\s+$)//g, @{$row});

push( @arry , [@{$row}] ) ;
# print "$row->[0] , $row->[1],$row->[2],$row->[3]\n";
}
}
return @arry ;
}

Pages

extract data from html table using HTML::TableExtract

No comments:

Other Articles