Difference between revisions of "Bugzilla Administration/BugzillaFetch"

#!/usr/bin/perl
#
# Fetch bugzilla data to migrate to new bugzilla!
#
# Created by:  Derek Atkins <derek@ihtfp.com>
#

use JSON;
use URI::Escape;
use LWP::Simple;
use List::MoreUtils qw(natatime);

$URL="https://bugzilla.gnome.org/jsonrpc.cgi";
$FILEBASE="/root/Bugzilla/data";

my $debug_RPC;

# RPC(method, params)
#  method:  method name (e.g. Products.get or Bug.search
#  params:  an arrayref of a hashref of parameters for the API
#
# Returns the json hashref of the result
sub RPC(@@) {
    my ($method, $params) = @_;
    my $url = "$URL?method=$method&params=" . uri_escape(encode_json($params));
    print "URL: $url\n" if ($debug_RPC);
    my $res = decode_json(get($url));
    my $json = JSON->new;
    die $json->pretty->encode($res) if ($res->{error} != null || $res->{result} == null);
    return $res->{result};
}

# GetProductByName(name)
# Returns a hashref of the product "name"
sub GetProductByName(@) {
    my ($name) = @_;
    my $params = [{names => [ $name ]}];
    my $json = RPC("Product.get", $params);
    return $json->{products}[0];
}

# GetBugsForProduct(name)
# Returns an arrayref of each bug-id, where each bug-id is a
#  hashref with fields id and last_change_time
sub GetBugsForProduct(@) {
    my ($name) = @_;
    my $params = [{product => [$name],
		   include_fields => ["id", "last_change_time"]}];
    #$debug_RPC = 1;
    my $json = RPC("Bug.search", $params);
    undef $debug_RPC;
    #my @ids = ();
    #foreach $id (@{$json->{bugs}}) {
	#push @ids, $id->{id};
    #}
    #return \@ids;
    return $json->{bugs};
}

# Check bug's last_change_time to see if it needs to be refreshed
#  Argument is a hashref with fields id and last_change_time
#  Returns null or the bug reference
sub CheckBug(@) {
    my ($b_id) = @_;
    my $filename = "$FILEBASE/bugs/bug_" . $b_id->{id} . ".json";
    #print "Checking bug " . $b_id->{id} . ", last_change_time=" . $b_id->{last_change_time} . "...";
    if (! -f $filename) {
	#print "No file found\n";
	return null;
    }

    open FH, $filename || die "Can't open $filename: $!";
    my $bug = decode_json <FH>;
    close FH;

    #print "Found file...";
    if ($b_id->{last_change_time} eq $bug->{last_change_time}) {
	#print "No changes.\n";
	return $bug;
    }

    #print "Refresh.\n";
    return null;
}

# GetBugById(id)
# Return a hashref of the Bug with comments, history, and attachments
sub GetBugById(@) {
    my ($id) = @_;
    my $params = [{ids => [$id]}];
    my $json = RPC("Bug.get", $params);
    my $bug = $json->{bugs}[0];

    # Pull down the comments for this bug
    # comment #0 is the bug description (sorted by count)
    $json = RPC("Bug.comments", $params);
    $bug->{comments} = $json->{bugs}->{$id}->{comments};
    #@comments = sort { a->{count} <=> b->{count} } @comments;

    $json = RPC("Bug.history", $params);
    $bug->{history} = $json->{bugs}[0]->{history};
    $json = RPC("Bug.attachments", $params);
    $bug->{attachments} = $json->{bugs}->{$id};

    return $bug;
}

# AddUsernames(hash, names..)
#  Add the names to the hashref
sub AddUsernames(@@) {
    my ($hash, @names) = @_;
    foreach my $n (@names) {
	next if ($n =~ /^$/);
	$hash->{$n} = 1;
    }
}

# SortUsernames(hash)
#  Return an arrayref of the sorted (unique) names (i.e. keys of hashref)
sub SortUsernames(@) {
    my ($hash) = @_;
    my @names = sort keys(%{$hash});
    return \@names;
}

# GetUsernamesFromBug(bug)
#  Given a hashref of a bug, return an arrayref of the list of usernames
#  associated with the bug.  These names include the submitter, CCs,
#  commenters, assignee, etc.
sub GetUsernamesFromBug(@) {
    my ($bug) = @_;
    my %names;
#    my @top_levels_keys = ("qa_contact", "creator", "assigned_to"
#			   "cc" (array)
#			   comments -> "creator", "author"
#			   history -> "who"
#			   attachments -> "attacher", "creator");

    # Push top-level ids (creator, qa_contact, assigned_to, cc):
    #print "Creator: " . $bug->{creator} . "\n";
    $names{$bug->{creator}} = 1;
    #print "QA Contact: " . $bug->{qa_contact} . "\n";
    $names{$bug->{qa_contact}} = 1;
    #print "Assigned To: " . $bug->{assigned_to} . "\n";
    $names{$bug->{assigned_to}} = 1;
    AddUsernames(\%names, @{$bug->{cc}});

    # Pull data from comments:
    foreach my $c (@{$bug->{comments}}) {
	#print "Comment Creator: " . $c->{creator} . "\n";
	$names{$c->{creator}} = 1;
	#print "Comment Author: " . $c->{author} . "\n";
	$names{$c->{author}} = 1;
    }

    # Pull data from history:
    foreach my $h (@{$bug->{history}}) {
	#print "Comment History Who: " . $h->{who} . "\n";
	$names{$h->{who}} = 1;
    }

    # Pull data from attachments:
    foreach my $a (@{$bug->{attachments}}) {
	#print "Attachment Creator: " . $a->{creator} . "\n";
	$names{$a->{creator}} = 1;
	#print "Attachment Attacher: " . $a->{attacher} . "\n";
	$names{$a->{attacher}} = 1;
    }

    # Reduce down to unique names
    return SortUsernames(\%names);
}

# GetUsersByNames(names)
#  names:  an arrayref to a list of names to get (email addresses)
# Returns an arrayref of the user data for all the names in the input array
sub GetUsersByNames(@) {
    my ($names) = @_;
    print "Trying to download " . scalar(@{$names}) . " users\n";
    my @users;
    my $iterator = natatime 100, @{$names};
    #$debug_RPC = true;
    my $j_ob = JSON->new;
    while (my @items = $iterator->() ) {
	#print "Downloading " . scalar(@items) . " users ...\n";
	my $params = [{names => \@items}];
	#print $j_ob->pretty->encode($params);
	my $json = RPC("User.get", $params);
	push @users, @{$json->{users}};
    }
    undef $debug_RPC;
    return \@users;
}

# SaveObjectToFile(obj, filename)
#   encodes the object to JSON and saves it to the file
sub SaveObjectToFile(@@) {
    my ($obj, $filename) = @_;
    open FH, ">$filename" || die "Failed to open $filename";
    print FH encode_json($obj);
    close FH;
}

# DownloadBugzilla(projectName)
#   download all the data for the product and save it to the data location
sub DownloadBugzilla(@) {
    my ($prodname) = @_;
    my %usernames;

    # Ensure storage exists
    if (! -d "$FILEBASE/bugs") {
	`mkdir -p $FILEBASE/bugs`;
    }

    # Download product
    print "Downloading product $prodname...\n";
    my $prod = GetProductByName($prodname);
    SaveObjectToFile([$prod], "$FILEBASE/products.json");
    foreach my $comp (@{$prod->{components}}) {
	AddUsernames(\%usernames, ($comp->{default_qa_contact}, $comp->{default_assigned_to} ));
    }

    # Download all the bugs for the product and extract the users
    print "Extracting bug list...\n";
    my @buglist = @{GetBugsForProduct($prodname)};
    print "Found " . $#buglist . " bugs to extract.\n";

    # Download the bugs
    foreach my $b_id (@buglist) {
	my $bug = CheckBug($b_id);
	if ($bug == null) {
	    print "Downloading bug #" . $b_id->{id} . "...\n";
	    $bug = GetBugById($b_id->{id});
	    SaveObjectToFile($bug, "$FILEBASE/bugs/bug_" . $bug->{id} . ".json");
	}
	AddUsernames(\%usernames, @{GetUsernamesFromBug($bug)});
    }

    # Download the users
    print "Downloading users...\n";
    my $users = GetUsersByNames(SortUsernames(\%usernames));
    SaveObjectToFile($users, "$FILEBASE/users.json");

    # Download the available Bug Fields
    print "Downloading bug fields...\n";
    my $json = RPC("Bug.fields", []);
    SaveObjectToFile($json->{fields}, "$FILEBASE/bug_fields.json");

    print "Done!\n";
}

DownloadBugzilla("gnucash");