add new enhanced rpm --fetch command

openpkg/openpkg.spec

@@ -39,8 +39,8 @@
 #   o any cc(1)
 
 #   the package version and release
-%define       V_openpkg 20020404
-%define       R_openpkg 20020404
+%define       V_openpkg 20020408
+%define       R_openpkg 20020408
 
 #   the used software versions
 %define       V_rpm     4.0.2

openpkg/rpmmacros

@@ -192,6 +192,28 @@
 %ifdef()                 %if %{expand:%%{?%{1}:1}%%{!?%{1}:0}}
 %ifndef()                %if %{expand:%%{?%{1}:0}%%{!?%{1}:1}}
 
+#   default list of vendor source fetching locations
+%l_fetch_mirror_0        -
+%l_fetch_mirror_1        -
+%l_fetch_mirror_2        -
+%l_fetch_mirror_3        -
+%l_fetch_mirror_4        -
+%l_fetch_mirror_5        -
+%l_fetch_mirror_6        -
+%l_fetch_mirror_7        -
+%l_fetch_mirror_8        -
+%l_fetch_mirror_9        -
+%l_fetch_backup_0        -
+%l_fetch_backup_1        -
+%l_fetch_backup_2        -
+%l_fetch_backup_3        -
+%l_fetch_backup_4        -
+%l_fetch_backup_5        -
+%l_fetch_backup_6        -
+%l_fetch_backup_7        -
+%l_fetch_backup_8        -
+%l_fetch_backup_9        ftp://ftp.openpkg.org/sources/DST/%{name}/
+
 #   sane build environment
 %l_build_path            %{l_prefix}/bin:@l_build_path@
 %l_build_ldlp            %{l_prefix}/lib:@l_build_ldlp@

openpkg/rpmx.pl

@@ -73,9 +73,10 @@ foreach $var (qw(
     _dbpath _rpmdir _srcrpmdir _tmppath
     _target
     l_prefix
-    l_repo_cache
-    l_repo_0 l_repo_1 l_repo_2 l_repo_3 l_repo_4
-    l_repo_5 l_repo_6 l_repo_7 l_repo_8 l_repo_9
+    l_fetch_mirror_0 l_fetch_mirror_1 l_fetch_mirror_2 l_fetch_mirror_3 l_fetch_mirror_4 
+    l_fetch_mirror_5 l_fetch_mirror_6 l_fetch_mirror_7 l_fetch_mirror_8 l_fetch_mirror_9 
+    l_fetch_backup_0 l_fetch_backup_1 l_fetch_backup_2 l_fetch_backup_3 l_fetch_backup_4 
+    l_fetch_backup_5 l_fetch_backup_6 l_fetch_backup_7 l_fetch_backup_8 l_fetch_backup_9 
 )) {
     $vars .= "${var}=\%{${var}};";
 }
@@ -230,13 +231,15 @@ sub fetch_url {
     unlink("$dst.hdr");
     $rc = system($CFG->{PRG}->{"curl"}.
                  " --location" .
-                 " --max-time 120" . 
-                 " --stderr $dst.err" .
+                 " --max-time 1800" .
+                 " --connect-timeout 20" .
                  " --dump-header $dst.hdr" .
-                 " --output $dst '$src'");
+                 " --output '$dst' '$src' 2>&1 | tee $dst.err");
+    #            " --stderr $dst.err" .
 
     #   check whether command failed
     if ($rc != 0) {
+        unlink($dst);
         open(FP, "<$dst.err");
         my $err = <FP>;
         close(FP);
@@ -252,6 +255,7 @@ sub fetch_url {
         if ($response =~ m|^HTTP/[\d.]+\s+(\d+)|) {
             if ($1 ne 200) {
                 $response =~ s|\n$||;
+                unlink($dst);
                 return $response;
             }
         }
@@ -494,19 +498,19 @@ sub op_fetch {
     my $name = $spec;
     $name =~ s|\.[^.]+$||;
     $name =~ s|^.+/([^/]+)$|$1|;
-    my $srcdir = `$CFG->{PRG}->{"rpm"} --define 'name $name' --eval '%_sourcedir'`;
-    $srcdir =~ s|\n+$||s;
-    my $specdir = `$CFG->{PRG}->{"rpm"} --define 'name $name' --eval '%_specdir'`;
-    $specdir =~ s|\n+$||s;
+    my $local_srcdir = `$CFG->{PRG}->{"rpm"} --define 'name $name' --eval '%_sourcedir'`;
+    $local_srcdir =~ s|\n+$||s;
+    my $local_specdir = `$CFG->{PRG}->{"rpm"} --define 'name $name' --eval '%_specdir'`;
+    $local_specdir =~ s|\n+$||s;
 
     #   make sure source and spec directory actually exists
-    if (not -d $srcdir) {
-        print(STDERR "rpm: Creating directory $srcdir\n");
-        system("mkdir $srcdir");
+    if (not -d $local_srcdir) {
+        print(STDERR "rpm: Creating directory $local_srcdir\n");
+        system("mkdir $local_srcdir");
     }
-    if (not -d $specdir) {
-        print(STDERR "rpm: Creating directory $specdir\n");
-        system("mkdir $specdir");
+    if (not -d $local_specdir) {
+        print(STDERR "rpm: Creating directory $local_specdir\n");
+        system("mkdir $local_specdir");
     }
 
     #   parse spec file
@@ -521,15 +525,25 @@ sub op_fetch {
         if (m/^%define\s+([a-zA-Z_][a-zA-Z0-9_]*)\s+(.+?)\s*$/) {
             $DEF->{lc($1)} = $2;
         }
-        if (m/^((Source|Patch)[0-9]+?):\s*(.+)\s*$/i) {
-            my ($srcid, $srcurl) = ($1, $3);
-            1 while ($srcurl =~ s|%{?([a-zA-Z_][a-zA-Z0-9_]*)}?|$DEF->{lc($1)}|sge);
-            my $srcfile = $srcurl;
-            $srcfile =~ s|^.*/([^/]+)$|$1|;
-            my $file = $srcfile;
+        if (m/^((Source|Patch)[0-9]+?):\s*(.+?)\s*$/i) {
+            my ($remote_srcid, $remote_srcurl) = ($1, $3);
+
+            #   determine expanded remote source location
+            1 while ($remote_srcurl =~ s|%{?([a-zA-Z_][a-zA-Z0-9_]*)}?|$DEF->{lc($1)}|sge);
+            my $remote_srcdir  = '';
+            my $remote_srcfile = $remote_srcurl;
+            if ($remote_srcfile =~ m|^(.+)/([^/]+)$|) {
+                $remote_srcdir  = $1;
+                $remote_srcfile = $2;
+            }
+
+            #   display remote source location
+            my $file = $remote_srcfile;
             $file = substr($file, 0, 40) if (length($file) > 40);
-            printf(STDERR "%-9s %-40s ", "$srcid:", $file);
-            if (-f "$srcdir/$srcfile" or -f "$specdir/$srcfile") {
+            printf(STDERR "%-9s %-40s ", "$remote_srcid:", $file);
+
+            if (-f "$local_srcdir/$remote_srcfile" or -f "$local_specdir/$remote_srcfile") {
+                #   source already on local filesystem
                 my $size;
                 if (-f "$srcdir/$srcfile") {
                    $size = (stat("$srcdir/$srcfile"))[7];
@@ -549,16 +563,38 @@ sub op_fetch {
                 print STDERR "...OK [$size]\n";
             }
             else {
+                #   fetch source onto local filesystem
                 print STDERR "...MISSING\n";
-                print STDERR "rpm: Fetching $srcurl\n";
-                my $cmd = $CFG->{PRG}->{"curl"}.
-                          " --location" .
-                          " --output $srcdir/$srcfile" .
-                          " $srcurl";
-                my $rc = system($cmd);
-                print STDOUT "\n";
-                if ($rc != 0) {
-                    print STDERR "rpm: Failed to fetch source file `$srcfile'\n";
+
+                #   determine prioritized list of possible source locations
+                my @fetch_srcdir = ();
+                for ($i = 0; $i <= 9; $i++) {
+                    my $url = $CFG->{RC}->{"l_fetch_mirror_$i"};
+                    next if (not defined($url) or $url eq '' or $url eq '-');
+                    1 while ($url =~ s|%{?([a-zA-Z_][a-zA-Z0-9_]*)}?|$DEF->{lc($1)}|sge);
+                    push(@fetch_srcdir, $url);
+                }
+                push(@fetch_srcdir, $remote_srcdir);
+                for ($i = 0; $i <= 9; $i++) {
+                    my $url = $CFG->{RC}->{"l_fetch_backup_$i"};
+                    next if (not defined($url) or $url eq '' or $url eq '-');
+                    1 while ($url =~ s|%{?([a-zA-Z_][a-zA-Z0-9_]*)}?|$DEF->{lc($1)}|sge);
+                    push(@fetch_srcdir, $url);
+                }
+
+                #   try to fetch from all possible locations
+                foreach $fetch_srcdir (@fetch_srcdir) {
+                    $fetch_srcdir =~ s|/+$||s;
+                    print STDERR "rpm: Fetching from $fetch_srcdir/\n";
+                    if (($err = &fetch_url($CFG, "$fetch_srcdir/$remote_srcfile", "$local_srcdir/$remote_srcfile"))) {
+                        $err = substr($err, 0, 37)."..." if (length($err) > 40);
+                        print STDOUT "rpm: Error: $err\n";
+                        next;
+                    }
+                    last;
+                }
+                if (not -f "$local_srcdir/$remote_srcfile") {
+                    print STDERR "rpm: Failed to fetch source file `$remote_srcfile'\n";
                     return 1;
                 }
             }