emdebian-grip 3.1.0 / usr / sbin / apt-grip

#!/usr/bin/perl

# Copyright (C) 2009-2013  Neil Williams <codehelp@debian.org>
#
# This package is free software; you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation; either version 3 of the License, or
# (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program.  If not, see <http://www.gnu.org/licenses/>.

use IO::File;
use File::Basename;
use POSIX qw(locale_h);
use Locale::gettext;
use strict;
use warnings;
use vars qw/ $host $mirror $suite $config_str $dir %suites $arch @touch
 $dpkgdir $etcdir $sourcesname $ourversion $progname @files $msg $noauth
 @dirs @source_list $vendor $preserve $clean $build $ignore_status $components /;

setlocale(LC_MESSAGES, "");
textdomain("emdebian-grip");

$progname = basename($0);
$ourversion = &scripts_version();
$dir = "/var/lib/apt-grip/";
$mirror = "http://ftp.uk.debian.org/debian/";
%suites  = ();
$etcdir = "etc/apt-grip/"; # sources
# use the real system status information.
# we'd only have to copy it otherwise.
$dpkgdir = "/var/lib/dpkg/";        # state

while( @ARGV ) {
	$_= shift( @ARGV );
	last if m/^--$/;
	if (!/^-/) {
		unshift(@ARGV,$_);
		last;
	} elsif (/^(-\?|-h|--help|--version)$/) {
	&usageversion();
		exit 0;
	} elsif (/^(-M|--mirror)$/) {
		$mirror = shift(@ARGV);
	} elsif (/^(-S|--suite)$/) {
		$suite = shift(@ARGV);
	} elsif (/^(-V|--vendor)$/) {
		$vendor = shift(@ARGV);
	} elsif (/^(-a|--arch)$/) {
		$arch = shift (@ARGV);
	} elsif (/^(--no-auth)$/) {
		$noauth = " -o Apt::Get::AllowUnauthenticated=true";
	} elsif (/^(--ignore-status)$/) {
		$ignore_status++;
	} elsif (/^(-k|--keep-cache)$/) {
		$preserve = 1;
	} elsif (/^(-c|--clean-cache)$/) {
		$clean = 1;
	} elsif (/^(-b|--build-only)$/) {
		$build = 1;
	} else {
		die "$progname: "._g("Unknown option")." $_.\n";
	}
}

# imply -k if -b in use.
$preserve = 1 if (defined $build);

# no point building if clean is also set.
if ((defined $build) and (defined $clean)) {
	printf( _g("%s: Illogical options set.\n"), $progname);
	printf( _g("%s: --build-only cannot be used with --clean-cache\n"),
		$progname);
	exit 4;
}

if (defined $clean) {
	printf( _g("%s: Cleaning %s* \n"), $progname, $dir);
	system ("rm -rf ${dir}*");
	printf( _g("%s: Done.\n\n"), $progname);
	exit 0;
}

@files = @ARGV;

if (scalar @files == 0) {
	my $msg = sprintf(_g("ERROR: Please specify some packages for %s to convert.\n"), $progname);
	warn ($msg);
	&usageversion;
	exit (0);
}

$host = `dpkg-architecture -qDEB_BUILD_ARCH 2>/dev/null`;
chomp ($host);
$arch = $host if (not defined $arch);
$vendor = $ENV{"DEB_VENDOR"}
	if (defined $ENV{"DEB_VENDOR"} and (not defined $vendor));
# check the default has been setup.
my $check = `dpkg-vendor --vendor emdebian-grip --query vendor`;
chomp ($check);
die (_g("ERROR: Unable to find default emdebian-grip dpkg-vendor configuration!\n"))
	if ((not defined $vendor) and ($check !~ /^emdebian-grip$/));
$vendor = $check
	if ((not defined $vendor) and ($check =~ /^emdebian-grip$/));
my $msg = sprintf(_g("ERROR: %s: misconfiguration, '%s' missing.\n"),
	$progname, $dir);
die ($msg) if (not -d "$dir");
system ("mkdir -p ${dir}${etcdir}sources.list.d/");
system ("mkdir -p ${dir}${etcdir}preferences.d/");
unlink ("${dir}${etcdir}sources.list") if (-f "${dir}${etcdir}sources.list");
system ("rm -f ${dir}${etcdir}sources.list.d/*");
mkdir "$dir/lists" if (not -d "$dir/lists");
mkdir "$dir/lists/partial" if (not -d "$dir/lists/partial");
mkdir "$dir/archives" if (not -d "$dir/archives");
mkdir "$dir/output" if (not -d "$dir/output");
mkdir "$dir/archives/partial" if (not -d "$dir/archives/partial");
@dirs = qw/ alternatives info parts updates/;
@touch = qw/ available diversions statoverride status lock/;

if (defined $suite) {
	unlink "${dir}${etcdir}sources.list.d/apt-grip.list";
	open (SOURCES, ">${dir}${etcdir}sources.list.d/apt-grip.list")
		or die _g("Cannot open sources list")." $!";
	print SOURCES "deb $mirror $suite $components\n";
	close SOURCES;
} else {
	&prepare_sources_list;
}
$config_str = '';
$config_str .= " -o Apt::Get::Download-Only=true";
if (($arch ne $host) or (defined $ignore_status)) {
	$dpkgdir = "${dir}/${arch}/dpkg/";
	mkdir "$dir/$arch";
	mkdir "$dir/$arch/dpkg";
	system ("touch ${dir}/${arch}/dpkg/status");
	$config_str .= " -y -o Apt::Architecture=$arch";
} else {
	$config_str .= " -y --reinstall -o Dir=$dir";
}
$config_str .= " -o Apt::Install-Recommends=false";
$config_str .= " -o Dir::Etc=${dir}${etcdir}";
$config_str .= " -o Dir::Etc::TrustedParts=/etc/apt/trusted.gpg.d";
$config_str .= $noauth if (defined $noauth);
$config_str .= " -o Dir::Etc::Trusted=/etc/apt/trusted.gpg"; 
$config_str .= " -o Dir::Etc::SourceList=${dir}${etcdir}sources.list";
$config_str .= " -o Dir::Etc::SourceParts=${dir}${etcdir}sources.list.d/";
$config_str .= " -o Dir::State=${dir}";
$config_str .= " -o Dir::State::Status=${dpkgdir}status";
$config_str .= " -o Dir::Cache=${dir}";

print "apt-get $config_str update\n";
system ("apt-get $config_str update 2>/dev/null");
my $str = (defined $suite) ? join ("/$suite ", @files) : join (" ", @files);
print "apt-get $config_str install $str\n";
system ("apt-get $config_str install $str");
$msg = _g("Cannot read");
opendir (DEBS, "${dir}archives/") or die ("$msg ${dir}archives/ : $!\n");
my @list = grep(/\.deb$/, readdir DEBS);
closedir (DEBS);

foreach my $pkg (@list) {
	system ("emgrip --vendor $vendor -o ${dir}output/ ${dir}archives/$pkg");
	unlink ("${dir}archives/$pkg");
}

system ("dpkg -i ${dir}output/*.deb")
	if ((scalar @list > 0) and (not defined $build) and ($host eq $arch));

system ("rm -rf ${dir}*") if ((not defined $preserve) and ($host eq $arch));

exit 0;

sub prepare_sources_list {
	# copy existing sources into our directories
	if (-e "/etc/apt/sources.list") {
		open (SOURCES, "/etc/apt/sources.list")
			or die _g("cannot open apt sources list.")." $!";
		@source_list = <SOURCES>;
		close (SOURCES);
		open (SOURCES, ">${dir}${etcdir}sources.list")
			or die _g("cannot open apt sources list.")." $!";
		print SOURCES @source_list;
		close (SOURCES);
	}
	@source_list=();
	if (-d "/etc/apt/sources.list.d/") {
		opendir (FILES, "/etc/apt/sources.list.d/") or
			die "cannot open apt sources.list directory $!";
		my @files = grep(!/^\.\.?$/, readdir FILES);
		foreach my $f (@files) {
			# just skip some obvious backups
			next if ($f =~ /\.ucf-old$/);
			next if ($f =~ /.*~/);
			open (SOURCES, "/etc/apt/sources.list.d/$f") or
				die _g("cannot open apt sources list.")." $!";
			@source_list=<SOURCES>;
			close (SOURCES);
			open (SOURCES, ">${dir}${etcdir}sources.list.d/$f") or
				die _g("cannot open apt sources list.")." $!";
			print SOURCES @source_list;
			close (SOURCES);
		}
		closedir (FILES);
	}
	@source_list=();
}

sub usageversion {
	printf STDERR (_g("
%s version %s

Usage:
 %s [-M|--mirror] [-V|--vendor] [-S|--suite] [-k|--keep-cache] PACKAGES ...
 %s -c|--clean-cache
 %s -?|-h|--help|--version

Commands:
 -c|--clean-cache:        Remove any downloaded cache files and exit.

Options:
 -b|--build-only:         Get and process the packages, do not install
                           (implies -k)
 -M|--mirror:             A Debian mirror with the requested package(s)
 -S|--suite:              Which Debian suite to use for the package(s)
 -V|--vendor:             Alternative to setting DEB_VENDOR
 -k|--keep-cache:         Preserve the downloaded cache files to use again.
 -a|--arch ARCHITECTURE:  Download selected architecture only, not install.
    --ignore-status:      Ignore currently installed packages

Emdebian Grip has a limited number of packages in the main repository,
principally to reduce the size of the apt cache data. On occassion,
individual packages from standard Debian need to be added to a single
machine running Emdebian Grip. apt-grip has been written with that
purpose in mind.

apt-grip requires DEB_VENDOR support in dpkg to determine how the
package should be gripped. The default Vendor is 'emdebian-grip'
but note the section in the apt-grip manpage on the limitations if
you use apt-grip on an unmodified Debian system.

The usual case is that the system has already been upgraded to Emdebian
Grip before apt-grip is used.

apt-grip tidies up after itself by removing all temporary data and
packages after installation, unless the --keep-cache option is used.
(Converted packages are not preserved.)

If the device running Grip has insufficient space to download and
process the package(s), run apt-grip on a different machine of the
same architecture using both the --build-only and --keep-cache
options. The processed archives will be in /var/lib/apt-grip/archives/
and can be copied from there onto the device directly or by including
the packages into a locally accessible repository. Once installed on
the Grip device use apt-grip -c on the build machine to clear the
cache. Using --build-only implies --keep-cache.

%s also supports use with Emdebian Baked to prepare packages of foreign
architectures.

"), $progname, $ourversion, $progname, $progname, $progname, $progname);
}

sub scripts_version {
	my $query = `dpkg-query -W -f='\${Version}' emdebian-grip`;
	(defined $query) ? return $query : return "";
}

sub _g {
	return gettext(shift);
}

=pod

=head1 NAME

apt-grip - extend Emdebian Grip to add Debian packages on-the-fly

=head1 Synopsis

 $ sudo apt-grip foo bar baz

 $ sudo apt-grip -M http://ftp.fr.debian.org/debian/ foo bar baz
 
 $ sudo apt-grip --clean-cache

=head1 Description

Emdebian Grip has a limited number of packages in the main repository,
principally to reduce the size of the apt cache data. On occassion,
individual packages from standard Debian need to be added to a single
machine running Emdebian Grip. F<apt-grip> has been written with that
purpose in mind.

F<apt-grip> requires DEB_VENDOR support in dpkg to determine how the
package should be C<gripped>. The default Vendor is C<emdebian-grip>
but note the next section on the limitations of apt-grip if
you plan to use it on an unmodified Debian system.

The usual case is that the system has already been upgraded to Emdebian
Grip before F<apt-grip> is used.

Contact the debian-embedded@lists.debian.org mailing list for requests
to add packages to Emdebian Grip repositories directly.

Downloading the Packages files can take a reasonable amount of time, so
to grip a number of packages, either specify all packages in one command
or use the C<--keep-cache> option for each run and use the
C<--clean-cache> option at the end.

If the device running Grip has insufficient space to download and
process the package(s), run apt-grip on a different machine of the
using the C<--build-only> and C<--arch> options.

The processed archives will be in F</var/lib/apt-grip/output/>
and can be copied from there onto the device directly or by including
the packages into a locally accessible repository. Once installed on
the Grip device use C<apt-grip -c> on the build machine to clear the
cache. C<--keep-cache> is implied when C<--build-only> is set.

Note also that, in common with the rest of Emdebian processing,
Install-Recommends is always turned off, so if you need a package that
is only recommended by packages in the list given to C<apt-grip>, that
package will need to be added to the list explicitly.

=head1 Limitations

Installing any package from repositories outside the normal apt sources
(especially if those packages are subsequently modified as in emgrip)
will list those packages as "local or obsolete" in package managers.
Converted packages cannot be upgraded without repeating the call to 
C<apt-grip> because C<apt-get> does not know about the temporary
mirror that C<apt-grip> used to download the packages. This can cause
problems if dependencies of such packages need to be upgraded. It is
possible that the main system C<apt> will try to remove these local
packages in order to proceed with the main system upgrade.

The best option is to seek addition of such packages to the repository
you use for your main apt sources. (Use the F<emdebian-grip-server>
package to create your own repository.)

C<Gripping> a package means making a new version (with the em1 version
suffix) with less files in the new package. See emgrip (1) for detailed
information on that process. Changing the version string means that some
dependencies need to be changed - particularly strict dependencies. This
means that F<apt-grip> could fail with some combinations of packages.

C<apt-grip> uses the C<--reinstall> option to F<apt-get> to cope with
some of these situations.

=head1 Strict dependency issues

If a source package builds multiple binary packages, where at least one
package has a strict dependency on one of the other binary packages and
one of those binary packages is already installed from Debian,
it will be necessary to install Grip versions of both the binary
packages involved so that the strict dependency can be satisfied.

 Source: foo

 Package: foo
 Depends: bar (= 0.0.1)

 Package: bar

 $ sudo apt-grip foo bar

In the example above, F<foo_0.0.1_amd64.deb> will become 
F<foo_0.0.1em1_amd64.deb> and will be given a strict dependency on
C<bar (= 0.0.1em1)> by F<emgrip>.

=head1 Default mirror

F<apt-grip> uses C<http://ftp.uk.debian.org/debian/> as the default
Debian mirror. Use the -M|--mirrror option to change it.

=head1 SecureApt

Use the C<--no-auth> option to allow the use of unauthenticated
repositories. Note that this disables authentication against all
available repositories.

=head1 Upgrading to Grip

Change your sources list from debian mirrors to the emdebian grip
mirror.

e.g. for unstable:

 deb http://www.emdebian.org/grip/ unstable main

(Emdebian Grip only supports Debian suites: unstable, sid, testing,
squeeze, stable, lenny, lenny-proposed-updates.)

Then update and upgrade:

 $ sudo apt-get update
 $ sudo apt-get clean
 $ sudo apt-get upgrade
 $ sudo apt-get dist-upgrade

(Note that F<apt-get> will usually report the ability to free several
hundred megabytes on your system when some of your Debian packages
are upgraded to Emdebian Grip but you still need enough room to actually
download and unpack the new packages.) Use C<apt-get clean> after the
upgrade to recover more free space.

=head1 Grip components

Grip divides the archive further by creating new areas of the archive
for packages related to development, debugging, documentation or java,
again to reduce the final size of the cached package data on the system.
If you want to use C<apt-grip> on your Emdebian Grip system, ensure that
you add the F<dev> component to your apt sources:

 deb http://www.emdebian.org/grip/ unstable main dev

Similarly for F<doc>, F<debug> and F<java>. Other components may be
added from time to time, so check the Emdebian website:

 http://www.emdebian.org/grip/search.php

Add more components as required:

 deb http://www.emdebian.org/grip/ unstable main dev debug java

=head1 Emdebian Baked

apt-grip has been extended to help generate packages for use with
Emdebian Baked. This has meant adding foreign architecture support as
well as support for ignoring the status of currently installed packages
on the device running apt-grip.

The difficult part of this process is managing updated packages and
changed dependencies. apt-grip can only help with Debian stable.

emdebian-grip has not been backported to Lenny, so your development
machine must be running Debian Squeeze or Sid (testing or unstable) to
develop a Baked root filesystem based on Lenny.

=head1 Example apt-grip command for Baked

 $ sudo apt-grip -a mipsel -V emdebian-baked -S stable dash snmpd

Baked packages will then exist in /var/lib/apt-grip/output/, including
all dependencies of the specified packages. These can then be included
into an existing reprepro repository:

 $ reprepro includedeb stable /var/lib/apt-grip/output/*.deb

If your repository is to support more than one architecture, ensure that
you remove or ignore the Architecture: all packages which have already
been processed by apt-grip:

 $ sudo rm /var/lib/apt-grip/output*_all.deb

Then use the -A option to reprepro to only include the added
architecture:

 $ reprepro -A armel includedeb stable /var/lib/apt-grip/output/*_armel.deb

For more information, see the emdebian website: http://www.emdebian.org/baked/

=cut