head	1.113;
access;
symbols;
locks; strict;
comment	@# @;


1.113
date	2007.08.26.18.20.05;	author debdev;	state Exp;
branches;
next	1.112;

1.112
date	2007.08.26.18.10.48;	author debdev;	state Exp;
branches;
next	1.111;

1.111
date	2007.07.19.12.05.55;	author debdev;	state Exp;
branches;
next	1.110;

1.110
date	2007.07.18.12.20.59;	author debdev;	state Exp;
branches;
next	1.109;

1.109
date	2007.07.17.08.09.07;	author debdev;	state Exp;
branches;
next	1.108;

1.108
date	2007.07.16.07.41.39;	author debdev;	state Exp;
branches;
next	1.107;

1.107
date	2007.04.13.09.57.31;	author debdev;	state Exp;
branches;
next	1.106;

1.106
date	2007.04.13.09.49.03;	author debdev;	state Exp;
branches;
next	1.105;

1.105
date	2006.12.21.13.02.11;	author debdev;	state Exp;
branches;
next	1.104;

1.104
date	2006.09.07.19.02.57;	author debdev;	state Exp;
branches;
next	1.103;

1.103
date	2006.09.07.18.47.01;	author debdev;	state Exp;
branches;
next	1.102;

1.102
date	2006.07.23.09.15.04;	author debdev;	state Exp;
branches;
next	1.101;

1.101
date	2006.07.12.20.14.36;	author debdev;	state Exp;
branches;
next	1.100;

1.100
date	2006.07.12.15.10.13;	author debdev;	state Exp;
branches;
next	1.99;

1.99
date	2006.07.09.17.02.02;	author debdev;	state Exp;
branches;
next	1.98;

1.98
date	2006.07.09.08.42.36;	author debdev;	state Exp;
branches;
next	1.97;

1.97
date	2006.07.09.08.21.46;	author debdev;	state Exp;
branches;
next	1.96;

1.96
date	2006.07.09.07.22.34;	author debdev;	state Exp;
branches;
next	1.95;

1.95
date	2006.07.07.20.42.36;	author debdev;	state Exp;
branches;
next	1.94;

1.94
date	2006.07.07.20.24.25;	author debdev;	state Exp;
branches;
next	1.93;

1.93
date	2006.07.07.20.21.05;	author debdev;	state Exp;
branches;
next	1.92;

1.92
date	2006.07.07.20.12.56;	author debdev;	state Exp;
branches;
next	1.91;

1.91
date	2006.07.07.20.08.16;	author debdev;	state Exp;
branches;
next	1.90;

1.90
date	2006.07.07.20.03.14;	author debdev;	state Exp;
branches;
next	1.89;

1.89
date	2006.07.07.12.41.53;	author debdev;	state Exp;
branches;
next	1.88;

1.88
date	2006.07.07.12.04.27;	author debdev;	state Exp;
branches;
next	1.87;

1.87
date	2006.07.07.12.01.56;	author debdev;	state Exp;
branches;
next	1.86;

1.86
date	2006.07.07.11.59.07;	author debdev;	state Exp;
branches;
next	1.85;

1.85
date	2006.07.07.11.57.01;	author debdev;	state Exp;
branches;
next	1.84;

1.84
date	2006.07.07.11.56.06;	author debdev;	state Exp;
branches;
next	1.83;

1.83
date	2006.07.07.11.55.35;	author debdev;	state Exp;
branches;
next	1.82;

1.82
date	2006.07.06.16.57.36;	author debdev;	state Exp;
branches;
next	1.81;

1.81
date	2006.07.06.16.52.03;	author debdev;	state Exp;
branches;
next	1.80;

1.80
date	2006.06.30.14.20.40;	author debdev;	state Exp;
branches;
next	1.79;

1.79
date	2006.06.30.14.13.12;	author debdev;	state Exp;
branches;
next	1.78;

1.78
date	2006.06.29.08.07.13;	author debdev;	state Exp;
branches;
next	1.77;

1.77
date	2006.06.27.20.21.22;	author debdev;	state Exp;
branches;
next	1.76;

1.76
date	2006.06.23.15.04.52;	author debdev;	state Exp;
branches;
next	1.75;

1.75
date	2006.06.23.14.52.50;	author debdev;	state Exp;
branches;
next	1.74;

1.74
date	2006.06.23.11.26.14;	author debdev;	state Exp;
branches;
next	1.73;

1.73
date	2006.06.23.10.02.43;	author debdev;	state Exp;
branches;
next	1.72;

1.72
date	2006.06.23.09.57.09;	author debdev;	state Exp;
branches;
next	1.71;

1.71
date	2006.06.23.09.39.04;	author debdev;	state Exp;
branches;
next	1.70;

1.70
date	2006.06.23.09.27.47;	author debdev;	state Exp;
branches;
next	1.69;

1.69
date	2006.06.23.09.17.06;	author debdev;	state Exp;
branches;
next	1.68;

1.68
date	2006.06.23.08.16.35;	author debdev;	state Exp;
branches;
next	1.67;

1.67
date	2006.06.23.07.27.20;	author debdev;	state Exp;
branches;
next	1.66;

1.66
date	2006.06.22.16.00.34;	author debdev;	state Exp;
branches;
next	1.65;

1.65
date	2006.06.22.12.11.14;	author mennucci;	state Exp;
branches;
next	1.64;

1.64
date	2006.06.21.10.12.53;	author mennucci;	state Exp;
branches;
next	1.63;

1.63
date	2006.06.20.10.11.26;	author mennucci;	state Exp;
branches;
next	1.62;

1.62
date	2006.06.20.06.08.12;	author debdev;	state Exp;
branches;
next	1.61;

1.61
date	2006.06.20.06.05.28;	author debdev;	state Exp;
branches;
next	1.60;

1.60
date	2006.06.19.21.11.28;	author debdev;	state Exp;
branches;
next	1.59;

1.59
date	2006.06.19.12.45.01;	author mennucci;	state Exp;
branches;
next	1.58;

1.58
date	2006.06.19.10.13.09;	author mennucci;	state Exp;
branches;
next	1.57;

1.57
date	2006.06.19.09.03.51;	author mennucci;	state Exp;
branches;
next	1.56;

1.56
date	2006.06.19.08.46.14;	author mennucci;	state Exp;
branches;
next	1.55;

1.55
date	2006.06.18.17.26.04;	author debdev;	state Exp;
branches;
next	1.54;

1.54
date	2006.06.18.16.08.31;	author debdev;	state Exp;
branches;
next	1.53;

1.53
date	2006.06.16.16.46.54;	author debdev;	state Exp;
branches;
next	1.52;

1.52
date	2006.06.16.16.37.51;	author debdev;	state Exp;
branches;
next	1.51;

1.51
date	2006.06.16.16.26.59;	author debdev;	state Exp;
branches;
next	1.50;

1.50
date	2006.06.15.19.35.04;	author debdev;	state Exp;
branches;
next	1.49;

1.49
date	2006.06.15.11.52.55;	author debdev;	state Exp;
branches;
next	1.48;

1.48
date	2006.06.15.11.25.13;	author debdev;	state Exp;
branches;
next	1.47;

1.47
date	2006.06.15.10.39.39;	author debdev;	state Exp;
branches;
next	1.46;

1.46
date	2006.06.15.10.12.07;	author debdev;	state Exp;
branches;
next	1.45;

1.45
date	2006.06.15.07.16.25;	author debdev;	state Exp;
branches;
next	1.44;

1.44
date	2006.06.15.06.52.23;	author debdev;	state Exp;
branches;
next	1.43;

1.43
date	2006.06.14.08.00.51;	author mennucci;	state Exp;
branches;
next	1.42;

1.42
date	2006.06.13.10.14.50;	author debdev;	state Exp;
branches;
next	1.41;

1.41
date	2006.06.13.07.54.39;	author debdev;	state Exp;
branches;
next	1.40;

1.40
date	2006.06.13.07.39.23;	author debdev;	state Exp;
branches;
next	1.39;

1.39
date	2006.06.13.07.27.38;	author debdev;	state Exp;
branches;
next	1.38;

1.38
date	2006.06.13.07.08.30;	author debdev;	state Exp;
branches;
next	1.37;

1.37
date	2006.06.12.17.56.32;	author debdev;	state Exp;
branches;
next	1.36;

1.36
date	2006.06.12.17.41.33;	author debdev;	state Exp;
branches;
next	1.35;

1.35
date	2006.06.12.09.53.29;	author debdev;	state Exp;
branches;
next	1.34;

1.34
date	2006.06.12.09.25.21;	author debdev;	state Exp;
branches;
next	1.33;

1.33
date	2006.06.11.19.01.14;	author debdev;	state Exp;
branches;
next	1.32;

1.32
date	2006.06.11.16.13.50;	author debdev;	state Exp;
branches;
next	1.31;

1.31
date	2006.06.11.10.16.20;	author debdev;	state Exp;
branches;
next	1.30;

1.30
date	2006.06.11.09.18.16;	author debdev;	state Exp;
branches;
next	1.29;

1.29
date	2006.06.11.06.42.36;	author debdev;	state Exp;
branches;
next	1.28;

1.28
date	2006.06.10.11.12.20;	author debdev;	state Exp;
branches;
next	1.27;

1.27
date	2006.06.07.14.05.18;	author debdev;	state Exp;
branches;
next	1.26;

1.26
date	2006.06.01.21.10.06;	author debdev;	state Exp;
branches;
next	1.25;

1.25
date	2006.05.31.14.14.16;	author debdev;	state Exp;
branches;
next	1.24;

1.24
date	2006.05.31.13.13.39;	author debdev;	state Exp;
branches;
next	1.23;

1.23
date	2006.05.31.12.04.41;	author debdev;	state Exp;
branches;
next	1.22;

1.22
date	2006.05.30.19.37.21;	author debdev;	state Exp;
branches;
next	1.21;

1.21
date	2006.05.30.14.12.54;	author debdev;	state Exp;
branches;
next	1.20;

1.20
date	2006.05.30.10.06.52;	author debdev;	state Exp;
branches;
next	1.19;

1.19
date	2006.05.30.08.16.42;	author debdev;	state Exp;
branches;
next	1.18;

1.18
date	2006.05.30.07.51.09;	author debdev;	state Exp;
branches;
next	1.17;

1.17
date	2006.05.30.06.56.38;	author debdev;	state Exp;
branches;
next	1.16;

1.16
date	2006.05.29.14.15.28;	author debdev;	state Exp;
branches;
next	1.15;

1.15
date	2006.05.28.12.41.18;	author debdev;	state Exp;
branches;
next	1.14;

1.14
date	2006.05.27.10.39.53;	author debdev;	state Exp;
branches;
next	1.13;

1.13
date	2006.05.26.16.03.47;	author debdev;	state Exp;
branches;
next	1.12;

1.12
date	2006.05.23.16.18.10;	author debdev;	state Exp;
branches;
next	1.11;

1.11
date	2006.05.23.16.10.55;	author debdev;	state Exp;
branches;
next	1.10;

1.10
date	2006.05.23.13.56.24;	author debdev;	state Exp;
branches;
next	1.9;

1.9
date	2006.05.23.13.27.39;	author debdev;	state Exp;
branches;
next	1.8;

1.8
date	2006.05.23.11.25.10;	author debdev;	state Exp;
branches;
next	1.7;

1.7
date	2006.05.22.14.30.30;	author debdev;	state Exp;
branches;
next	1.6;

1.6
date	2006.05.22.08.35.55;	author debdev;	state Exp;
branches;
next	1.5;

1.5
date	2006.05.20.15.34.38;	author debdev;	state Exp;
branches;
next	1.4;

1.4
date	2006.05.20.11.05.43;	author debdev;	state Exp;
branches;
next	1.3;

1.3
date	2006.05.19.19.36.52;	author debdev;	state Exp;
branches;
next	1.2;

1.2
date	2006.05.19.12.25.14;	author debdev;	state Exp;
branches;
next	1.1;

1.1
date	2006.05.18.19.19.25;	author debdev;	state Exp;
branches;
next	;


desc
@@


1.113
log
@document --delta-algo ; and reformat help
@
text
@#!/usr/bin/python

doc={}
doc['delta']="""\
Usage: debdelta [ option...  ] fromfile tofile patchout
  Computes a delta from fromfile to tofile and writes it to patchout

Options:
--no-md5    do not include MD5 info in debdelta
--needsold  create a patch that can only be used if the old .deb is available
 -M Mb      maximum memory  to use (for 'bsdiff' or 'xdelta')
--delta-algo ALGO
            use a specific backend for computing binary diffs;
            possible values are: xdelta xdelta-bzip xdelta3 bsdiff
"""


doc['deltas']="""\
Usage: debdeltas [ option...  ]  [deb_files and dirs]
  Computes all missing deltas for Debian files.
  It orders by version number and produce deltas to the newest version

Options:
--dir DIR   force saving of deltas in this DIR
            (otherwise they go in the dir of the newer deb_file)

--alt DIR   for any cmdline argument, search for debs also in this dir 

            if DIR ends in // , then the dirname of the cmdline argument
            will be appended to DIR, as well (useful when creating archives)
            
 -n N       how many deltas to produce for each package (default 1)
--no-md5    do not include MD5 info in debdelta
--needsold  create a patch that can only be used if the old .deb is available
--delta-algo ALGO
            use a specific backend for computing binary diffs;
            possible values are: xdelta xdelta-bzip xdelta3 bsdiff
 -M Mb      maximum memory to use (for 'bsdiff' or 'xdelta')
--clean-deltas     delete deltas if newer deb is not in archive
--clean-alt        delete debs in --alt if too old (see -n )
"""

## implement : --search    search in the directory of the above debs for older versions

doc['patch']="""\
Usage: debpatch [ option...  ] patchin  fromfile  tofile 
  Applies patchin to fromfile and produces a reconstructed  version of tofile.

(When using 'debpatch' and the old .deb is not available,
  use '/' for the fromfile.)

Usage: debpatch --info  patch
  Write info on patch.

Options:
--no-md5   do not verify MD5 (if found in info in debdelta)
"""

doc['delta-upgrade']="""\
Usage: debdelta-upgrade
  Downloads all deltas that may be used to 'apt-get upgrade', and apply them

Options:
--dir DIR   directory where to save results
            (default: /var/cache/apt/archives for root,
              /tmp/archive for non-root users)
"""


doc_common="""\
 -v         verbose (can be added multiple times)
--no-act    do not do that (whatever it is!)
 -d         add extra debugging checks
 -k         keep temporary files (use for debugging)
"""

minigzip='/usr/lib/debdelta/minigzip'


####################################################################

import sys , os , tempfile , string ,getopt , tarfile , shutil , time, md5, traceback

from stat    import ST_SIZE, ST_MTIME, ST_MODE, S_IMODE, S_IRUSR, S_IWUSR, S_IXUSR 
from os.path import abspath
from copy    import copy

from types import StringType, FunctionType, TupleType, ListType, DictType

import shutil

################################################# main program, read options

#target of: maximum memory that bsdiff will use
MAXMEMORY = 1024 * 1024 * 50

#this is +-10% , depending on the package size
MAX_DELTA_PERCENT = 70

#min size of .deb that debdelta will consider
#very small packages cannot be effectively delta-ed
MIN_DEB_SIZE = 10 * 1024


N_DELTAS= 1

f=os.popen('grep bogomips /proc/cpuinfo')
BOGOMIPS=float(f.read().split(':')[-1])
f.close()

f=os.popen('hostname -f')
HOSTID=md5.new( f.read() ).hexdigest()
f.close()


USE_DELTA_ALGO  = 'bsdiff'

DEBUG   = 0
VERBOSE = 0
KEEP    = False
INFO    = False
NEEDSOLD= False
DIR     = None
ALT     = None
AVOID   = None
ACT     = True
DO_MD5  = True

CLEAN_DELTAS = False
CLEAN_ALT    = False


RCS_VERSION="$Id: debdelta,v 1.112 2007/08/26 18:10:48 debdev Exp debdev $"

HTTP_USER_AGENT={'User-Agent': ('Debian debdelta-upgrade' ) }

if os.path.dirname(sys.argv[0]) == '/usr/lib/apt/methods' :
  action = None
else:
  action=(os.path.basename(sys.argv[0]))[3:]
  actions =  ('delta','patch','deltas','delta-upgrade')
  
  if action not in actions:
    print 'wrong filename: should be "deb" + '+repr(actions)
    raise SystemExit(0)

  __doc__ = doc[action] + doc_common

  try: 
    ( opts, argv ) = getopt.getopt(sys.argv[1:], 'vkhdM:n:' ,
                 ('help','info','needsold','dir=','no-act','alt=','avoid=','delta-algo=','max-percent=','clean-deltas','clean-alt','no-md5','debug') )
  except getopt.GetoptError,a:
      sys.stderr.write(sys.argv[0] +': '+ str(a)+'\n')
      raise SystemExit(2)

  for  o , v  in  opts :
    if o == '-v' : VERBOSE += 1
    elif o == '-d' or o == '--debug' : DEBUG += 1
    elif o == '-k' : KEEP = True
    elif o == '--no-act': ACT=False
    elif o == '--no-md5': DO_MD5=False
    elif o == '--clean-deltas' : CLEAN_DELTAS = True
    elif o == '--clean-alt' : CLEAN_ALT = True
    elif o == '--needsold' :  NEEDSOLD = True
    elif o == '--delta-algo': USE_DELTA_ALGO=v
    elif o == '--max-percent': MAX_DELTA_PERCENT=int(v)
    elif o == '-M' :
      if int(v) <= 1:
        print 'Error: "-M ',int(v),'" is too small.'
        raise SystemExit(1)
      if int(v) <= 12:
        print 'Warning: "-M ',int(v),'" is quite small.'
      MAXMEMORY = 1024 * 1024 * int(v)
    elif o == '-n' :
      N_DELTAS = int(v)
      if N_DELTAS <= 0:
        print 'Error: -n ',v,' is negative or zero.'
        raise SystemExit(3) 
    elif o == '--info' and action == 'patch' : INFO = True
    elif o == '--avoid'  :
      AVOID = v
      if not os.path.isfile(AVOID):
        print 'Error: --avoid ',AVOID,' does not exist.'
        raise SystemExit(3)
    elif o == '--dir'  :
      DIR = v
      if not os.path.isdir(DIR):
        print 'Error: --dir ',DIR,' does not exist.'
        raise SystemExit(3)
    elif o == '--alt'  :
      ALT = v
      if not os.path.isdir(ALT):
        print 'Error: --alt ',ALT,' does not exist.'
        raise SystemExit(3)
    elif o ==  '--help' or o ==  '-h':
      print __doc__
      raise SystemExit(0)
    else:
      print ' option ',o,'is unknown, try --help'
      raise SystemExit(1)

def dummy(): #otherwise the python mode for emacs fails to index my routines
  pass

TMPDIR = ( os.getenv('TMPDIR') or '/tmp' ).rstrip('/')

if KEEP:
  def unlink(a):
    if VERBOSE > 4: print ' would unlink ',a
  def rmdir(a):
    if VERBOSE > 4: print ' would rmdir ',a
  def rmtree(a):
    if VERBOSE > 4: print ' would rm -r ',a
else:
  def __wrap__(a,cmd):
    c=cmd.__name__+"("+a+")"
    if a[ : len(TMPDIR)+9 ] != TMPDIR+'/debdelta' :
      raise DebDeltaError,'Internal error! refuse to  '+c
    try:
      cmd(a)
    except OSError,s:
      print ' Warning! when trying to ',repr(c),'got OSError',repr(str(s))
      if DEBUG > 2 : raise

  def unlink(a):
    return __wrap__(a,os.unlink)
  def rmdir(a):
    return __wrap__(a,os.rmdir)
  def rmtree(a):
    return __wrap__(a,shutil.rmtree)

#################################################### various routines

def freespace(w):
  assert(os.path.exists(w))
  try:
    a=os.statvfs(w)
    freespace= a[0] * a[4]
  except:
    if VERBOSE : print ' statvfs error ',a
    freespace=None
  return freespace

dpkg_keeps_controls = (
  'conffiles','config','list','md5sums','postinst',
  'postrm','preinst','prerm','shlibs','templates')

def parse_dist(f,d):
  a=f.readline()
  p={}
  while a:
    if a[:4] in ('Pack','Vers','Arch','Stat','Inst','File','Size','MD5s'):
      a=de_n(a)
      i=a.index(':')
      assert(a[i:i+2] == ': ')
      p[a[:i]] = a[i+2:]
    elif a == '\n':
      d[p['Package']] = p
      p={}
    a=f.readline()


def scan_control(p,params=None,prefix=None,info=None):
  if prefix == None:
    prefix = ''
  else:
    prefix += '/'
  a=p.readline()
  while a:
    a=de_n(a)
    if a[:4] in ('Pack','Vers','Arch','Stat','Inst','File'):
      if info != None :
        info.append(prefix+a)
      if params != None:
        i=a.index(':')
        assert(a[i:i+2] == ': ')
        params[prefix+a[:i]] = a[i+2:]
    a=p.readline()

def append_info(delta,info):
  #new style : special info file
  TD = abspath(tempfile.mkdtemp(prefix='debdelta',dir=TMPDIR))
  infofile=open(TD+'/info','w')
  for i in info:
    infofile.write(i+'\n')
  infofile.close()
  system(['ar','rSi','0',delta, 'info'],  TD)
  rmtree(TD)

def make_parents(f):
  assert(f[0] == '/')
  s=f.split('/')
  d=''
  for a in s[:-1] :
    if a:
      d=d+'/'+a
      if not os.path.exists(d):
        os.mkdir(d)
  d=d+'/'+s[-1]
  return d

def de_n(a):
  if a and a[-1] ==  '\n' :
    a = a[:-1]
  return a

def de_bar(a):
  if a and a[:2] == './' :
    a=a[2:]
  if a and a[0] == '/' :
    a=a[1:]
  return a

def list_ar(f):
  assert(os.path.exists(f))
  ar_list = []
  p=os.popen('ar t '+f,'r')
  while 1:
    a=p.readline()
    if not a : break
    a=de_n(a)
    ar_list.append(a)    
  p.close()
  return ar_list

def list_tar(f):
  assert(os.path.exists(f))
  ar_list = []
  p=os.popen('tar t '+f,'r')
  while 1:
    a=p.readline()
    if not a : break
    a=de_n(a)
    ar_list.append(a)    
  p.close()
  return ar_list

#####################################################################

ALLOWED = '<>()[]{}.,;:!_-+/ abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ'

ECHO_TEST = r"""c='\0151\0141'
echo='echo -ne'
if test c`$echo 'i'"$c" `o = ciiao  ; then
 :
else
 echo='echo -n'
 if test c`$echo  'i'"$c" `o = ciiao  ; then 
  :
 else
  #echo WARNING : BUILTIN echo DOES NOT WORK OK
  echo='/bin/echo -ne'
  test c`$echo  'i'"$c" `o = ciiao  
 fi
fi
"""

def prepare_for_echo__(s):
  assert ( type (s) == StringType )
  r=''
  shortquoted=False
  for a in s:
    if a in ALLOWED :
      r += a
      shortquoted = False
    elif a in '0123456789' :
      if shortquoted :
        a = "\\" + ('000' +oct(ord(a)))[-4:]
      shortquoted = False
      r += a
    else:
      a = "\\" + oct(ord(a))
      r += a
      shortquoted = len(a) < 5
  return r

def apply_prepare_for_echo(shell,repres):
    a=ECHO_TEST  + " $echo '" + repres +  "' \n exit "
    o,i=os.popen2(shell)
    o.write(a)
    o.close()
    a=i.read()
    i.close()
    return a

#ack! I wanted to use 'dash' as preferred shell, but bug 379227 stopped me
SHELL = '/bin/bash'
#check my code
s='\x00'+'1ciao88\n77\r566'+'\x00'+'99\n'
r=prepare_for_echo__(s)
a=apply_prepare_for_echo(SHELL,r)
if a != s :
    print 'string='+repr(s)
    print 'repres='+repr(r)
    print 'shell='+SHELL
    print 'output='+repr(a)
    print 'Errror in prepare_for_echo.'
    raise SystemExit(2)

###

def prepare_for_echo(s):
    r=prepare_for_echo__(s)
    if DEBUG > 1 :
        a=apply_prepare_for_echo(SHELL,r)
        if a != s:
            print 'Errror in prepare_for_echo.'
            if DEBUG or VERBOSE > 2 :
                print 'string='+repr(s)
                print 'repres='+repr(r)
                print 'shell='+SHELL
                print 'output='+repr(a)
            raise SystemExit(2)
    return r

#####################################################################

from string import join

def version_mangle(v):
  if  ':' in v :
    return join(v.split(':'),'%3a')
  else:
    return v
  
def version_demangle(v):
  if  '%' in v :
    return join(v.split('%3a'),':')
  else:
    return v
  
def tempo():
  TD = abspath(tempfile.mkdtemp(prefix='debdelta',dir=TMPDIR))
  for i in 'OLD','NEW','PATCH' :
    os.mkdir(TD+'/'+i)
  if  VERBOSE > 2 or KEEP :  print 'Temporary in '+TD
  return TD

##########


class DebDeltaError(Exception):  #should derive from (Exception):http://docs.python.org/dev/whatsnew/pep-352.html
  # Subclasses that define an __init__ must call Exception.__init__
  # or define self.args.  Otherwise, str() will fail.
  def __init__(self,s,retriable=False):
    self.args = s
    self.retriable = retriable
  def __str__(self):
    if DEBUG:
      if self.retriable:
        return self.args + ' (retriable) '
      else:
        return self.args + ' (non retriable) '
    else:
      return self.args

def die(s):
  #if s : sys.stderr.write(s+'\n')
  raise DebDeltaError,s
  
def system(a,TD):
  if type(a) != StringType :
    a=string.join(a,' ')
  if VERBOSE and TD[: (len(TMPDIR)+9) ] != TMPDIR+'/debdelta' :
    print 'Warning "system()" in ',TD,' for ',a
  ret = os.system("cd '" +TD +"' ; "+a)  
  if ret == 2:
    raise KeyboardInterrupt
  if  ret != 0 and ( ret != 256 or a[:6] != 'xdelta') :
    die('Error , non zero return status '+str(ret)+' for command "'+a+'"')

def check_deb(f):
  if not  os.path.isfile(f) :
    die('Error: '+f + ' does not exist.')
  p=open(f)
  if p.read(21) != "!<arch>\ndebian-binary" :
    die('Error: '+f+ ' does not seem to be a Debian package ')
  p.close()

def check_is_delta(f):
  if not  os.path.isfile(f) :
    die('Error: '+f + ' does not exist.')
  p=open(f)
  if p.read(8) != "!<arch>\n" :
    die('Error: '+f+ ' does not seem to be a Debian delta ')
  p.close()

def puke(s,e=''):
  (typ, value, trace)=sys.exc_info()
  if VERBOSE or e == '':    print s,' : ',e,str(typ),str(value)
  else: print s,' : ',e
  if DEBUG : print traceback.print_tb(trace)

#################################################################### apply patch

########### info auxiliary routines

def _delta_info_unzip_(TD):
  if os.path.exists(TD+'PATCH/info.gz'):
    system('gunzip PATCH/info.gz',TD)
  if os.path.exists(TD+'PATCH/patch.sh.gz'):
    system('gunzip PATCH/patch.sh.gz',TD)
  elif os.path.exists(TD+'PATCH/patch.sh.bz2'):
    system('bunzip2 PATCH/patch.sh.bz2',TD)  

def get_info_slow(delta,T=None):
  if T:
    TD=T
  else:
    TD=tempo()
  if TD[-1] != '/':
    TD = TD + '/'
  delta=abspath(delta)
  system('ar x  '+delta+' info info.gz patch.sh patch.sh.gz patch.sh.bz2 2> /dev/null', \
         TD+'/PATCH')
  _delta_info_unzip_(TD)
  info = _scan_delta_info_(TD)
  if T == None:
    rmtree(TD)
  return info

def get_info_fast(delta):
  f=open(delta)
  s=f.readline()
  if  "!<arch>\n" != s :
    raise DebdeltaError('This is not a debdelta file: '+delta)
  s = f.read(60)
  if len(s) != 60 :
    print '(Warning, cannot get info from  truncated: '+delta+' )'
    return None
  if s[:4] != 'info':
    #old style debdelta, with info in patch.sh
    if VERBOSE > 1 :
      print '(Warning, cannot get info from old style: '+delta+' )'
    return None
  ##parse ar segment
  ## see /usr/include/ar.h
  if s[-2:] != '`\n' :
    print '(Warning, cannot get info from  '+delta+' , format not known)'
    return None
  l=int(s[ -12:-2 ])
  s=f.read(l)
  if len(s) != l :
    print '(Warning, cannot get info from truncated: '+delta+' )'
    return None
  info= s.split('\n')
  f.close()
  return info

def get_info(delta,TD=None):
  info=get_info_fast(delta)
  if info == None:
    info=get_info_slow(delta,TD)
  return info

def _scan_delta_info_(TD):
    info=[]
    if os.path.isfile(TD+'PATCH/info'):
      #new style debdelta, with info file
      p=open(TD+'PATCH/info')
      info=p.read().split('\n')
      p.close()
      if info[-1] == '': info.pop()
    else:
      #old style debdelta, with info in patch.sh
      p=open(TD+'PATCH/patch.sh')
      s=p.readline()
      s=p.readline()
      while s:
        if s[0] == '#' :
          s=de_n(s)
          info.append(s[1:])
        s=p.readline()
      p.close()
    return info

def info_2_db(info):
  params={}
  for s in info:
    if ':' in s:
      i=s.index(':')  
      params[s[:i]] = s[i+2:]
    elif s:
      params[s] = True
  return params

########### other auxiliary routines

def patch_check_tmp_space(params,olddeb):
  if type(params) != DictType:
    params=info_2_db(params)
  if 'NEW/Installed-Size' not in params or 'OLD/Installed-Size' not in params:
    print '(Warning... Installed size unknown...)'
    return True
  free=freespace(TMPDIR)
  if free == None : return True
  free = free / 1024
  if olddeb == '/':
    instsize=int(params['NEW/Installed-Size'])
    #the last action of the script is to gzip the data.tar, so
    if 'NEW/Size' in params :
      instsize += int(params['NEW/Size']) / 1024
    else:
      instsize = instsize * 1.8
  else:
    instsize=int(params['NEW/Installed-Size'])+int(params['OLD/Installed-Size'])
  instsize +=  2**13
  if free <  instsize :
    return 'not enough disk space (%dkB) in %s for applying delta (needs %dkB).' % \
        ( int(free) , TMPDIR, instsize )
  else:
    return True


def scan_diversions():
  f=open('/var/lib/dpkg/diversions')
  d={}
  a=1
  while 1:
    a=f.readline()
    if not a: break
    a=de_n(a)
    b=de_n(f.readline())
    p=de_n(f.readline())
    d[a]=(b,p)
  f.close()
  return d

############ do_patch

def do_patch(delta,olddeb,newdeb, info=None, diversions=None):
  try:
    T=tempo()
    r=do_patch_(delta,olddeb,newdeb,TD=T, info=info, diversions=diversions)
  except:
    rmtree(T)
    if newdeb and os.path.exists(newdeb):
      os.unlink(newdeb)
    raise
  rmtree(T)
  return r

def do_patch_(delta,olddeb,newdeb, TD, info=None, diversions=None):
  if TD[-1] != '/':
    TD = TD + '/'
  
  delta=abspath(delta)
  if newdeb:
    newdeb=abspath(newdeb)
  if olddeb != '/':
    olddeb=abspath(olddeb)
    
  start_sec = time.time()
  
  check_is_delta(delta)

  if olddeb != '/':
      check_deb(olddeb)
  if DEBUG and  newdeb and os.path.exists(newdeb) and os.path.getsize(newdeb) > 0 :
      die("Don't want to overwrite: "+newdeb)
  
  system('ar xo '+delta,  TD+'/PATCH')

  _delta_info_unzip_(TD)

  if not os.path.isfile(TD+'PATCH/patch.sh'):
    die('Error. File '+delta+' is not a debdelta file.')

  os.symlink(minigzip,TD+'minigzip')
  
  #lets scan parameters, to see what it does and what it requires
  if info == None :
      info=_scan_delta_info_(TD)
  params=info_2_db(info)
  
  ###
  s=patch_check_tmp_space(params,olddeb)
  if s != True:
    raise DebDeltaError('Sorry, '+s, True )

  if olddeb != '/':
      os.symlink(olddeb,TD+'/OLD.file')
      #unpack the old control structure, if available
      os.mkdir(TD+'/OLD/CONTROL')
      #unpack control.tar.gz
      system('ar p '+TD+'OLD.file control.tar.gz | tar -x -z -p -f - -C '+TD+'OLD/CONTROL',TD)
  #then we check for the conformance
  if olddeb != '/' and 'OLD/Size' in params:
    olddebsize = os.stat(olddeb)[ST_SIZE]
    if olddebsize != int(params['OLD/Size']):
      raise DebDeltaError('Old deb size is '+str(olddebsize)+' instead of '+params['OLD/Size'])
  
  if  DEBUG or olddeb != '/':
      #this is currently disabled, since  'dpkg -s' is vey slow (~ 1.6 sec)
      dpkg_params={}
      b=params['OLD/Package']
      if olddeb == '/' :
        p=os.popen('env -i dpkg -s '+b)
      else:        
        p=open(TD+'OLD/CONTROL/control')
      scan_control(p,params=dpkg_params,prefix='OLD')
      p.close()
      if  olddeb == '/' :
        if 'OLD/Status' not in dpkg_params:
          die('Error: package %s is not known to dpkg.' % b)
        if  dpkg_params['OLD/Status'] != 'install ok installed' :
          die('Error: package %s is not installed, status is %s.'
            % ( b , dpkg_params['OLD/Status'] ) )
      for a in  params:
        if a[:3] == 'OLD' and a != 'OLD/Installed-Size' and a != 'OLD/Size':
          if a not in dpkg_params:
            die('Error parsing old control file , parameter %s not found' % a)
          elif  params[a] != dpkg_params[a] :
            die( 'Error : in debdelta , '+a+' = ' +params[a] +\
                 '\nin old/installed deb, '+a+' = ' +dpkg_params[a])

  ### some auxiliary routines, separated to make code more readable

  def dpkg_L_faster(pa,diversions):
    " 'diversions' must be prepared by scan_diversions() "
    s=[]
    f=open('/var/lib/dpkg/info/'+pa+'.list')
    while 1:
      a=f.readline()
      if not a: break
      a=de_n(a)
      if a in diversions:
        b,p= diversions[a]
        if p != pa:    s.append((a,b))
        else:     s.append((a,a))
      else: s.append((a,a))
    f.close()
    return s

  def dpkg_L(pa):
    s=[]
    p=os.popen('env -i dpkg -L '+pa)
    a=p.readline()
    while a:
      a=de_n(a)
      #support diversions
      if a[:26] == 'package diverts others to:':
        continue
      if s and a[:11] == 'diverted by' or  a[:20] == 'locally diverted to:':
        orig,divert=s.pop()
        i = a.index(':')
        divert = a[i+2:]
        s.append( (orig,divert) )
      else:
        s.append( (a,a) )
      a=p.readline()
    p.close()
    return s

  def _symlink_data_tree(pa,TD,diversions):
    if diversions:
      s=dpkg_L_faster(pa,diversions)
    else:
      s=dpkg_L(pa)
    for orig,divert in s:          
      if os.path.isfile(divert) and not os.path.islink(divert) :            
        a=make_parents(TD+'/OLD/DATA'+orig)
        if VERBOSE > 3 : print '   symlinking ',divert,' to ',a
        os.symlink(divert, a)
      else:
        if VERBOSE > 3 : print '    not symlinking ',divert,' to ',orig


  def chmod_add(n,m):
    "same as 'chmod ...+...  n '"
    om=S_IMODE(os.stat(n)[ST_MODE])
    nm=om | m
    if nm != om :
      if VERBOSE > 1 : print ' Performing chmod ',n,oct(om),oct(nm)
      os.chmod(n,nm)
  
  def _fix_data_tree_(TD):
    for (dirpath, dirnames, filenames) in os.walk(TD+'OLD/DATA'):
      chmod_add(dirpath,  S_IRUSR | S_IWUSR| S_IXUSR  )
      for i in filenames:
        i=os.path.join(dirpath,i)
        if os.path.isfile(i):
          chmod_add(i,  S_IRUSR |  S_IWUSR )
      for i in dirnames:
        i=os.path.join(dirpath,i)
        chmod_add(i,  S_IRUSR | S_IWUSR| S_IXUSR  )


  ###see into parameters: the patch may need extra info and data
  for a in params:
    if 'unpack-old' == a:
      if olddeb == '/':
        die('This patch needs the old version Debian package')
      unpack ('OLD',olddeb,TD)
    elif 'needs-old' == a and olddeb == '/':
      die('This patch needs the old version Debian package')
    elif 'old-data-tree' == a :
      os.mkdir(TD+'/OLD/DATA')
      if olddeb == '/':
        pa=params['OLD/Package']
        _symlink_data_tree(pa,TD,diversions)
      else:
        system('ar p '+TD+'OLD.file data.tar.gz | tar -x -z -p -f - -C '+TD+'OLD/DATA', TD)
        _fix_data_tree_(TD)
    elif 'old-control-tree' == a:
        if olddeb == '/':
          if not os.path.isdir(TD+'OLD/CONTROL'):
            os.mkdir(TD+'OLD/CONTROL')
          p=params['OLD/Package']
          for  b in dpkg_keeps_controls :
            a='/var/lib/dpkg/info/' + p +'.'+b
            if os.path.exists(a ):
              os.symlink(a,TD+'OLD/CONTROL/'+b)
        #else... we always unpack the control of a .deb
    elif params[a] == True:
        print  'WARNING patch says "'+a+'" and this is unsupported. Get a newer debdelta.'
  ##then , really execute the patch
  a=''
  if VERBOSE > 3 : a = '-v'
  script_time = - time.time()
  system(SHELL+' -e '+a+' PATCH/patch.sh', TD)
  script_time += time.time()

  #then we check for the conformance
  if  'NEW/Size' in params:
    newdebsize = os.stat(TD+'NEW.file')[ST_SIZE]
    if newdebsize != int(params['NEW/Size']):
      raise DebDeltaError('New deb size is '+str(newdebsize)+' instead of '+params['NEW/Size'])

  if DO_MD5:
    if 'NEW/MD5sum' in params:
      if VERBOSE > 1 : print '  verifying MD5  for ',os.path.basename(newdeb or delta)
      system('echo "'+params['NEW/MD5sum']+'  NEW.file" | md5sum -c > /dev/null', TD)
    else: print ' Warning! no MD5 was verified for ',os.path.basename(newdeb or delta)

  if newdeb:
      shutil.move(TD+'NEW.file',newdeb)

  end_sec = time.time()
  elaps=(end_sec - start_sec)

  if VERBOSE :
      if newdeb:
        debsize = os.stat(newdeb)[ST_SIZE]
      else:
        debsize = os.stat(olddeb)[ST_SIZE]
      a=''
      if newdeb != None:
        a='result: '+os.path.basename(newdeb)
      print ' Patching done, time: %.2fsec, speed: %dkB/sec %s (script time %.2fsec ) ' % \
            (elaps,(debsize / 1024 /  (elaps+.001)),a , script_time)
  return (newdeb,elaps)

##################################################### compute delta
def do_delta(olddeb,newdeb,delta):
  try:
    T=tempo()
    r=do_delta_(olddeb,newdeb,delta,TD=T)
  except:
    if delta and os.path.exists(delta):
      os.unlink(delta)
    rmtree(T)
    raise
  rmtree(T)
  return r

def do_delta_(olddeb,newdeb,delta,TD):
  if TD[-1] != '/':
    TD = TD + '/'
  
  start_sec = time.time()
  #I do not like global variables but I do not know of another solution
  global bsdiff_time, bsdiff_datasize
  bsdiff_time = 0
  bsdiff_datasize = 0
  
  olddeb=abspath(olddeb)
  check_deb(olddeb)
  os.symlink(olddeb,TD+'/OLD.file')
  olddebsize = os.stat(olddeb)[ST_SIZE]
  
  newdeb=abspath(newdeb)
  check_deb(newdeb)
  os.symlink(newdeb,TD+'/NEW.file')
  newdebsize = os.stat(newdeb)[ST_SIZE]
  
  free=freespace(TD)
  if free and free < newdebsize :
    raise DebDeltaError('Error: not enough disk space in '+TD, True)

  delta=abspath(delta)
  if  os.path.exists(delta) :
    os.rename(delta,delta+'~')
  
  #generater for numbered files
  def a_numb_file_gen():    
    deltacount = 0
    while 1:
      yield str(deltacount)
      deltacount+=1      
  a_numb_file=a_numb_file_gen()
  
  #start writing script 
  script=open(TD+'PATCH/patch.sh','w')
  script.write('#!/bin/sh -e\n')
    
  ##### unpack control.tar.gz, scan control, write  parameters
  info=[]
  for o in 'OLD', 'NEW' :
      os.mkdir(TD+o+'/CONTROL')
      #unpack control.tar.gz
      system('ar p '+TD+o+'.file control.tar.gz | tar -x -z -f - -C '+TD+o+'/CONTROL',TD)
      ## scan control
      p=open(TD+'/'+o+'/CONTROL/control')
      s=[]
      scan_control(p,params=None,prefix=o,info=s)
      p.close()
      if  VERBOSE  :
        sys.stdout.write(o+': '+join([o[4:] for o in  s],' ')+'\n')
      info = info + s
      del s,p
  info.append('OLD/Size: '+str(olddebsize))
  info.append('NEW/Size: '+str(newdebsize))
  params=info_2_db(info)
  
  if DO_MD5 :
    # compute a MD5 of NEW deb
    p=os.popen('md5sum '+TD+'NEW.file')
    a=p.readline()
    p.read()
    p.close
    newdeb_md5sum=a[:32]
    info.append('NEW/MD5sum: '+ newdeb_md5sum[:32])
  else:
    newdeb_md5sum=None

  if NEEDSOLD :
    #this delta needs the old deb 
    info.append('needs-old')
  else:
    info.append('old-data-tree')
    info.append('old-control-tree')

  info.append('needs-'+USE_DELTA_ALGO)

  #backward compatibility
  for i in info:
    script.write('#'+i+'\n')

  #### check for disk space
  if 'NEW/Installed-Size' in params and 'OLD/Installed-Size' in params:
    free=freespace(TD)  
    instsize=int(params['NEW/Installed-Size']) + int(params['OLD/Installed-Size'])
    if free and free < ( instsize * 1024 + + 2**23 + MAXMEMORY / 6 ) :
      raise DebDeltaError(' Not enough disk space (%dkB) for creating delta (needs %dkB).' % \
          ( int(free/1024) , instsize ) , True )

    
  ############# check for conffiles 
  a=TD+'/OLD/CONTROL/conffiles'
  if os.path.exists(a):
    p=open(a)
    old_conffiles=[ de_bar(a) for a in p.read().split('\n') ]
    p.close()
  else:
    old_conffiles=()

  def shell_not_allowed(name):
    "Strings that I do not trust to inject into the shell script; maybe I am a tad too paranoid..."
    #FIXME should use it , by properly quoting for the shell script
    return '"' in name or "'" in name or '\\' in name or '`' in name 

  # uses MD5 to detect identical files (even when renamed)
  def scan_md5(n):
    md5={}
    f=open(n)
    a=de_n(f.readline())
    while a:
      m , n = a[:32] ,  de_bar( a[34:] )
      md5[n]=m
      a=de_n(f.readline())
    f.close()
    return md5


  new_md5=None
  if os.path.exists(TD+'/NEW/CONTROL/md5sums'):
    new_md5=scan_md5(TD+'/NEW/CONTROL/md5sums')
    
  old_md5=None
  if os.path.exists(TD+'/OLD/CONTROL/md5sums') :
    old_md5=scan_md5(TD+'/OLD/CONTROL/md5sums')

  ############### some routines  to prepare delta of two files

  def script_md5_check_file(n,md5=None):
    if md5==None:
      assert(os.path.isfile(TD+n))
      pm=os.popen('md5sum '+TD+n)
      a=pm.readline()
      pm.read()
      pm.close
      md5=a[:32]
    print "    adding extra MD5 for ",n
    script.write('echo "'+md5+'  '+n+'" | md5sum -c > /dev/null\n')

  def patch_append(f):
    if VERBOSE > 1 :
      a=os.stat(TD+'PATCH/'+f)[ST_SIZE]
      print '   appending ',f,' of size ', a,' to debdelta, %3.2f'  % ( a * 100. /  newdebsize ) , '% of new .deb'
    system(['ar','qSc', delta,f],  TD+'/PATCH')
    unlink(TD+'PATCH/'+f)

  def verbatim(f):
    pp=a_numb_file.next()
    p = 'PATCH/'+pp
    if VERBOSE > 1 : print '  including "',name,'" verbatim in patch'
    os.rename(TD+f,TD+p)
    patch_append(pp)
    return p
      
  def unzip(f, in_script_as_well = None):
    c=''
    if f[-3:] == '.gz' :
      system('gunzip '+f,TD)
      if in_script_as_well or ( in_script_as_well == None and f[:3] != 'NEW' ):
        script.write('gunzip '+f+'\n')
      f=f[:-3]
      c='.gz'
    elif  f[-3:] == '.bz2' :
      print 'WARNING ! ',f,' is in BZIP2 format ! please fixme !'
    return (f,c)

  def script_zip(n,cn,newhead):
    if cn == '.gz' :
      s=prepare_for_echo(newhead)
      script.write("$echo  '"+ s +"' >> "+n+cn +' && ./minigzip -9 < '+n+' | tail -c +'+str(len(newhead)+1)+' >> '+n+cn+' && rm '+n+' \n')
    elif  cn == '.bz2' :
      print 'WARNING ! ',n,' is in BZIP2 format ! please fixme !'

  def delta_files__(o,n,p,algo='bsdiff'):
    #bdiff
    #http://www.webalice.it/g_pochini/bdiff/
    if algo == 'bdiff':
      system('~/debdelta/bdiff-1.0.5/bdiff -q -nooldmd5 -nonewmd5 -d  '+o+' '+n+' '+p,TD)
      script.write('~/debdelta/bdiff-1.0.5/bdiff -p '+o+' '+p+' '+n+' ; rm '+p+'\n')    
    #zdelta
    #http://cis.poly.edu/zdelta/
    elif algo == 'zdelta':
      system('~/debdelta/zdelta-2.1/zdc  '+o+' '+n+' '+p,TD)
      script.write('~/debdelta/zdelta-2.1/zdu '+o+' '+p+' '+n+' ; rm '+p+'\n')
    #bdelta 
    #http://deltup.sf.net
    elif algo == 'bdelta':
      system('~/debdelta/bdelta-0.1.0/bdelta  '+o+' '+n+' '+p,TD)
      script.write('~/debdelta/bdelta-0.1.0/bpatch '+o+' '+n+' '+p+' ; rm '+p+'\n')
    #diffball
    #http://developer.berlios.de/projects/diffball/
    elif algo == 'diffball':
      system('~/debdelta/diffball-0.7.2/differ  '+o+' '+n+' '+p,TD)
      script.write('~/debdelta/diffball-0.7.2/patcher '+o+' '+p+' '+n+' ; rm '+p+'\n')
    #rdiff
    elif algo == 'rdiff':
      system('rdiff signature '+o+' sign_file.tmp  ',TD)
      system('rdiff delta  sign_file.tmp  '+n+' '+p,TD)
      script.write('rdiff patch '+o+' '+p+' '+n+' ; rm '+p+'\n')
    #xdelta3
    elif algo == 'xdelta3' :
      system('/usr/bin/xdelta3 -9 -R -D -n -S djw -s  '+o+' '+n+' '+p,TD)
      script.write('/usr/bin/xdelta3 -d -s '+o+' '+p+' '+n+' ; rm '+p+'\n')
    ## according to the man page,
    ## bsdiff uses memory equal to 17 times the size of oldfile
    ## but , in my experiments, this number is more like 12.
    ##But bsdiff is sooooo slow!
    elif algo == 'bsdiff' : # not ALLOW_XDELTA or ( osize < (MAXMEMORY / 12)):    
      system('bsdiff  '+o+' '+n+' '+p,TD)
      script.write('bspatch '+o+' '+n+' '+p+'; rm '+p+'\n')
    #seems that 'xdelta' is buggy on 64bit and different-endian machines
    #xdelta does not deal with different endianness!
    elif algo == 'xdelta-bzip' :
      system('xdelta delta --pristine --noverify -0 -m'+str(int(MAXMEMORY/1024))+'k '+o+' '+n+' '+p,TD)
      system('bzip2 -9 '+p,TD)
      script.write('bunzip2 '+p+'.bz2 ; xdelta patch '+p+' '+o+' '+n+' ; rm '+p+'\n')
      p  += '.bz2'
    elif algo == 'xdelta' :
      system('xdelta delta --pristine --noverify -9 -m'+str(int(MAXMEMORY/1024))+'k '+o+' '+n+' '+p,TD)
      script.write('xdelta patch '+p+' '+o+' '+n+' ; rm '+p+'\n')
    elif algo == 'jojodiff' :
      system('~/debdelta/jdiff06/src/jdiff -b '+o+' '+n+' '+p,TD)
      script.write('~/debdelta/jdiff06/src/jpatch '+o+' '+p+' '+n+' ; rm '+p+'\n')
    else: raise
    return p

  def delta_files(o,n):
    " compute delta of two files , and prepare the script consequently"
    nsize = os.path.getsize(TD+n)
    osize = os.path.getsize(TD+o)
    if VERBOSE > 1 : print '  compute delta for %s (%dkB) and %s (%dkB)' % \
       (o,osize/1024,n,nsize/1024)
    #
    p = 'PATCH/'+a_numb_file.next()
    tim = -time.time()
    #
    if DEBUG > 3 :  script_md5_check_file(o)
    #
    if USE_DELTA_ALGO == 'bsdiff' and osize > ( 1.1 * (MAXMEMORY / 12))  and VERBOSE  :
      print '  Warning, memory usage by bsdiff on the order of %dMb' % (12 * osize / 2**20)
    #
    p = delta_files__(o,n,p,USE_DELTA_ALGO)
    #script.write(s)
    #
    if DEBUG > 2 :  script_md5_check_file(n)
    #
    tim += time.time()      
    #
    global bsdiff_time, bsdiff_datasize
    bsdiff_time += tim
    bsdiff_datasize += nsize
    #
    script.write('rm '+o+'\n')
    ## how did we fare ?
    deltasize = os.path.getsize(TD+p)
    if VERBOSE > 1 :
      print '   delta is %3.2f%% of %s, speed: %dkB /sec'  % \
            ( ( deltasize * 100. /  nsize ) , n, (nsize / 1024. / ( tim + 0.001 )))
    #save it
    patch_append(p[6:])
    #clean up
    unlink(TD+o)

  def cmp_gz(o,n):
    "compare gzip files, ignoring header; returns first different byte (+-10), or True if equal"
    of=open(TD+o)
    nf=open(TD+n)
    oa=of.read(10)
    na=nf.read(10)
    if na[:3] != '\037\213\010' :
      print ' Warning: was not created with gzip: ',n
      nf.close() ; of.close() 
      return 0
    if oa[:3] != '\037\213\010' :
      print ' Warning: was not created with gzip: ',o
      nf.close() ; of.close() 
      return 0
    oflag=ord(oa[3])
    if oflag & 0xf7:
      print ' Warning: unsupported  .gz flags: ',oct(oflag),o
    if oflag & 8 : #skip orig name
      oa=of.read(1)
      while ord(oa) != 0:
        oa=of.read(1)
    l=10
    nflag=ord(na[3])
    if nflag & 0xf7:
      print ' Warning: unsupported  .gz flags: ',oct(nflag),n
    if nflag & 8 : #skip orig name
      na=nf.read(1)
      s=na
      while ord(na) != 0:
        na=nf.read(1)
        s+=na
      l+=len(s)
      #print repr(s)
    while oa and na:
      oa=of.read(2)
      na=nf.read(2)
      if oa != na:
        return l
      l+=2
    if oa or na: return l
    return True
    
  def delta_gzipped_files(o,n):
    "delta o and n, replace o with n"
    assert(o[-3:] == '.gz' and n[-3:] == '.gz')
    before=cmp_gz(o,n)
    if before == True:
      if VERBOSE > 3: print '    equal but for header: ',n
      return
    #compare the cost of leaving as is , VS the minimum cost of delta
    newsize=os.path.getsize(TD+n)
    if ( newsize - before + 10 ) < 200 :
      if VERBOSE > 3: print '    not worthwhile gunzipping: ',n
      return
    f=open(TD+n)
    a=f.read(10)
    f.close()
    if a[:3] != '\037\213\010' :
      print ' Warning: was not created with gzip: ',n
      return
    flag=ord(a[3]) # mostly ignored  :->
    orig_name='-n'
    if flag & 8:
      orig_name='-N'
    if flag & 0xf7:
      print ' Warning: unsupported  .gz flags: ',oct(flag),n
    #a[4:8] #mtime ! ignored ! FIXME will be changed... 
    #from deflate.c in gzip source code
    format=ord(a[8])
    FAST=4
    SLOW=2 #unfortunately intermediate steps are lost....
    pack_level=6
    if format ==  0 :
      pass
    if format ==  FAST :
      pack_level == 1
    if format ==  SLOW :
      pack_level == 9
    else:
      print ' Warning: unsupported compression .gz format: ',oct(format),n
      return
    if a[9] != '\003' :
      if VERBOSE: print ' Warning: unknown OS in .gz format: ',oct(ord(a[9])),n
    p='PATCH/tmp_gzip'
    #save new file and unzip
    shutil.copy2(TD+n,TD+p+'.new.gz')
    system("gunzip '"+n+"'",TD)
    shutil.copy2(TD+n[:-3],TD+p+'.new')
    #test our ability of recompressing
    l=[1,2,3,4,5,6,7,8,9]
    del l[pack_level]
    l.append(pack_level)
    l.reverse()
    for i in l:
      #force -n  ... no problem with timestamps
      gzip_flags="-n -"+str(i)      
      system("gzip -c "+gzip_flags+" '"+n[:-3]+"' > "+p+'.faked.gz',TD)
      r=cmp_gz(p+'.new.gz',p+'.faked.gz')
      if r == True:
        break
      if i == pack_level and VERBOSE > 3:
        print '    warning: wrong guess to re-gzip to equal file: ',gzip_flags,r,n
    if r != True:
      if VERBOSE > 2: print '   warning: cannot re-gzip to equal file: ',r,n
      os.unlink(TD+p+".new") ; os.unlink(TD+p+'.new.gz') ; os.unlink(TD+p+'.faked.gz') 
      return
    #actual delta of decompressed files
    system("zcat '"+o+"' > "+p+'.old',TD)
    script.write("zcat '"+o+"' > "+p+".old ; rm '"+o+"' \n")
    if VERBOSE > 2 :
      print '   ',n[9:],'  (= to %d%%): ' % (100*before/newsize) ,
    delta_files(p+'.old',p+'.new')
    os.rename(TD+p+'.faked.gz',TD+o)
    script.write("mv "+p+".new '"+o[:-3]+"' ;  gzip "+gzip_flags+" '"+o[:-3]+"'\n")
    if DEBUG > 1 :  script_md5_check_file(o)
    os.unlink(TD+p+'.new.gz')
    
  ########### helper sh functions for script, for delta_tar()

  import difflib

  def file_similarity_premangle(oo):
    o=oo.split('/')
    (ob,oe)=os.path.splitext(o[-1])
    return o[:-1]+ ob.split('_')+[oe]
  
  def files_similarity_score__noext__(oo,nn):
    ln=len(nn)
    lo=len(oo)
    l=0
    while oo and nn:
      while oo and nn and oo[-1] == nn[-1]:
        oo=oo[:-1]
        nn=nn[:-1]
      if not oo or not nn: break
      while oo and nn and oo[0] == nn[0]:
        oo=oo[1:]
        nn=nn[1:]
      if not oo or not nn: break
      if len(nn) > 1 and oo[0] == nn[1]:
        l+=1
        nn=nn[1:]
      if len(oo) > 1 and oo[1] == nn[0]:
        l+=1
        oo=oo[1:]
      if not oo or not nn: break
      if  oo[-1] != nn[-1]:
        oo=oo[:-1]
        nn=nn[:-1]
        l+=2
      if not oo or not nn: break
      if oo[0] != nn[0]:
        oo=oo[1:]
        nn=nn[1:]
        l+=2
    return (l +len(oo) + len(nn)) * 2.0 / float(ln+lo)

  def files_similarity_score__(oo,nn):
    oo=copy(oo)
    nn=copy(nn)
    if oo.pop() != nn.pop() :
      penalty=0.2
      return 0.2 + files_similarity_score__noext__(oo,nn)
    else:
      return files_similarity_score__noext__(oo,nn)
  
  def files_similarity_score__difflib__(oo,nn):
    "compute similarity by difflib. Too slow."
    if oo == nn :
      return 0
    d=difflib.context_diff(oo,nn,'','','','',0,'')
    d=[a for a in tuple(d) if a and a[:3] != '---' and a[:3] != '***' ]
    if oo[-1] != nn[-1] : #penalty for wrong extension
      return 0.2+float(len(d)) * 2.0 / float(len(oo)+len(nn))
    else:
      return float(len(d)) * 2.0 / float(len(oo)+len(nn))
    
  def files_similarity_score(oo,nn):
    if oo == nn :
      return 0
    if type(oo) == StringType:
      oo=file_similarity_premangle(oo)
    if type(nn) == StringType:
      nn=file_similarity_premangle(nn)
    return files_similarity_score__(oo,nn)

  def fake_tar_header_2nd():
    " returns the second part of a tar header , for regular files and dirs"
    # The following code was contributed by Detlef Lannert.
    # into /usr/lib/python2.3/tarfile.py
    MAGIC      = "ustar"            # magic tar string
    VERSION    = "00"               # version number
    NUL        = "\0"               # the null character
    parts = []
    for value, fieldsize in (
      ("", 100),
      # unfortunately this is not what DPKG does
      #(MAGIC, 6),
      #(VERSION, 2),
      #  this is  what DPKG does
      ('ustar  \x00',8),
      ("root", 32),
      ("root", 32),
      ("%07o" % 0, 8),
      ("%07o" % 0, 8),
      ("", 155)
      ):
      l = len(value)
      parts.append(value + (fieldsize - l) * NUL)      
    buf = "".join(parts)
    return buf
  
  fake_tar_2nd=fake_tar_header_2nd()
  fake_tar_2nd_echo=prepare_for_echo(fake_tar_2nd)
  script.write("FTH='"+fake_tar_2nd_echo+"'\n")
  script.write(ECHO_TEST)
  
  script.write('CR () { cat "$1"  >> OLD/mega_cat ; rm "$1" ;}\n')
  
  global time_corr
  time_corr=0

  ####################  vvv     delta_tar    vvv ###########################
  def delta_tar(old_filename,new_filename,CWD,skip=(),old_md5={},new_md5={}, chunked_p=True):
    " compute delta of two tar files, and prepare the script consequently"
    assert( type(old_filename) == StringType or type(old_filename) == FunctionType )
    if os.path.exists(TD+'OLD/mega_cat'):
      print 'Warning!!! OLD/mega_cat  exists !!!!'
      # if -k is given, still we need to delete it...
      os.unlink(TD+'OLD/mega_cat')
      script.write('rm OLD/mega_cat || true \n')
    mega_cat=open(TD+'OLD/mega_cat','w')
    #helper function
    def _append_(w,rm=False):
      assert(os.path.isfile(TD+w))
      f=open(TD+w)
      a=f.read(1024)
      while a:
        try:
          mega_cat.write(a)
        except OSError,s :
           raise DebDeltaError(' OSError (at _a_) while writing: '+str(s), True)
        a=f.read(1024)
      f.close()
      if rm:
        script.write("CR '"+w+"'\n")
        unlink(TD+w)
      else:
        script.write("cat '"+w+"'  >> OLD/mega_cat\n")

    #### scan once for regular files
    if type(old_filename) == StringType :
      (old_filename,old_filename_ext) = unzip(old_filename,False)
      oldtar = tarfile.open(TD+old_filename, "r")
    else:
      old_filename_ext=None
      oldfileobj = old_filename()
      oldtar = tarfile.open(mode="r|", fileobj=oldfileobj)
    oldnames = []
    oldtarinfos = {}
    for oldtarinfo in oldtar:
      oldname = oldtarinfo.name
      if  (oldname in skip) or shell_not_allowed(oldname) or \
             not oldtarinfo.isreg() or oldtarinfo.size == 0:
        continue
      if VERBOSE > 3 and oldname != de_bar(oldname):
        print ' Filename in old tar has weird ./ in front: ' , oldname 
      oldname = de_bar(oldname)
      if oldname in skip:
        continue
      oldnames.append(oldname)
      oldtarinfos[oldname] = oldtarinfo
      oldtar.extract(oldtarinfo,TD+"OLD/"+CWD )
    oldtar.close()
    if type(old_filename) == StringType :
      unlink(TD+old_filename)
    else:
      while oldfileobj.read(512):
        pass
    #save header part of new_filename, since it changes in newer versions
    f=open(TD+new_filename)
    new_file_zip_head=f.read(20)
    f.close()
    (new_filename,new_filename_ext) = unzip(new_filename)
    assert(0 == (os.path.getsize(TD+new_filename)% 512))
    newtar = tarfile.open(TD+new_filename, "r")
    newnames = []
    newtarinfos = {}
    for newtarinfo in newtar:
      newname =  newtarinfo.name
      #just curious to know
      t=newtarinfo.type
      a=newtarinfo.mode
      if VERBOSE and (( t == '2' and a  != 0777 ) or \
                      ( t == '0' and ( (a & 0400 ) == 0 )) or \
                      ( t == '5' and ( (a & 0500 ) == 0 ))):
        print ' Weird permission: ',newname,oct(a),repr(newtarinfo.type)
      ###
      if   not newtarinfo.isreg():
        continue
      if VERBOSE > 3 and newname != de_bar(newname):
        print ' Filename in new tar has weird ./ in front: ' , newname 
      newname = de_bar(newname)
      newnames.append(newname)
      newtarinfos[newname] = newtarinfo
      
    old_used={}
    correspondence={}

    ##############################
    global time_corr
    time_corr=-time.time()

    if VERBOSE > 2 : print '  finding correspondences  ',n

    reverse_old_md5={}
    if old_md5:
      for o in old_md5:
        if o in oldnames:
          reverse_old_md5[old_md5[o]] = o
        else:
          #would you believe? many packages contain MD5 for files they do not ship...
          if VERBOSE and o not in skip: print '  Hmmm... there is a md5 but not a file: ',o

    oldnames_premangle={}
    for o in oldnames:
      a,b=os.path.splitext(o)
      if b not in oldnames_premangle:
        oldnames_premangle[b]={}
      oldnames_premangle[b][o]=file_similarity_premangle(a)

    for newname in newnames:
      newtarinfo=newtarinfos[newname]
      oldname=None
      #ignore empty files
      if newtarinfo.size == 0:
        continue
      #try correspondence by MD5
      if new_md5 and newname in new_md5:
        md5=new_md5[newname]        
        if md5 in reverse_old_md5:
          oldname=reverse_old_md5[md5]
          if VERBOSE > 2 :
            if oldname  == newname :
              print '   use identical old file: ',newname
            else:
              print '   use identical old file: ',oldname, newname
      #try correspondence by file name
      if oldname == None and newname in oldnames:
        oldname=newname
        if VERBOSE > 2 : print '   use same name old file: ',newname
      #try correspondence by file name and len similarity
      nb,ne=os.path.splitext(newname)
      if oldname == None and ne in oldnames_premangle:
        basescore=1.6
        nl=newtarinfo.size
        np=file_similarity_premangle(nb)
        for o in oldnames_premangle[ne]:
          op=oldnames_premangle[ne][o]
          l=oldtarinfos[o].size
          sfile=files_similarity_score__noext__(op,np)
          slen = abs(float(l - nl))/float(l+nl)
          s=slen+sfile
          if VERBOSE > 3 : print '    name/len diff %.2f+%.2f=%.2f ' % (slen,sfile,s), o
          if s < basescore:
              oldname=o
              basescore=s
        if oldname and VERBOSE > 2 : print '   best similar  ','%.3f' % basescore,newname,oldname
      if not oldname:
        if VERBOSE > 2 : print '   no correspondence for: ',newname
        continue
      #we have correspondence, lets store
      if oldname not in old_used:
        old_used[oldname]=[]
      old_used[oldname].append(newname)
      correspondence[newname]=oldname
      
    time_corr+=time.time()
    if VERBOSE > 1 : print '  time lost so far in finding correspondence %.2f' % time_corr
    
    ######### now do real scanning
    if VERBOSE > 2 : print '  scanning ',n

    #helper function
    def mega_cat_chunk(oldoffset,newoffset):
      p = a_numb_file.next()
      f=open(TD+new_filename)
      f.seek(oldoffset)
      of=open(TD+p,'w')
      l=oldoffset
      while l<newoffset:
        s=f.read(512)
        l+=len(s)
        assert(len(s))
        try:
          of.write(s)
        except OSError,s :
          raise DebDeltaError(' OSError (at MCK) while writing: '+str(s), True)
      f.close()
      of.close()
      #move to a temporary
      pt=a_numb_file.next()
      script.write('mv OLD/mega_cat '+pt+'\n')
      os.rename(TD+'OLD/mega_cat',TD+pt)
      #do delta, in background there
      script.write('wait ; ( ')
      delta_files(pt,p)
      script.write('cat '+p+' >> '+new_filename+'; rm '+p+' ; ) & \n')
      os.unlink(TD+p)

    #there may be files that have been renamed and edited...
    def some_old_file_gen():
      for oldname in oldnames :
        if (oldname in skip) or (oldname in old_used ) :
          continue
        if VERBOSE > 2 : print '   provide also old file ', oldname
        yield oldname
      while 1:
        yield None

    some_old_file=some_old_file_gen()
    one_old_file=some_old_file.next()

    max_chunk_size = MAXMEMORY / 12
    chunk_discount = 0.3

    progressive_new_offset=0

    for newtarinfo in newtar:
      ## for tracking strange bugs
      if DEBUG > 3 and mega_cat.tell() > 0 :
        script_md5_check_file("OLD/mega_cat")
      #progressive mega_cat
      a=mega_cat.tell()
      if chunked_p and ((a >=  max_chunk_size * chunk_discount) or \
         (a >= max_chunk_size * chunk_discount * 0.9 and one_old_file ) or \
         (a>0 and (a+newtarinfo.size) >= max_chunk_size * chunk_discount )):
        #provide some old unused files, if any
        while one_old_file:
          w="OLD/"+CWD+"/"+one_old_file
          if os.path.isfile(TD+w):
            _append_(w)
          else: print 'Warning!!! ',w,'does not exists ???'
          if mega_cat.tell() >=  max_chunk_size * chunk_discount :
            break
          one_old_file=some_old_file.next()
        mega_cat.close()
        mega_cat_chunk(progressive_new_offset, newtarinfo.offset )
        progressive_new_offset=newtarinfo.offset
        mega_cat=open(TD+'OLD/mega_cat','w')
        chunk_discount = min( 1. , chunk_discount * 1.2 )
      #
      name = de_bar( newtarinfo.name )
      #recreate also parts of the tar headers
      mega_cat.write(newtarinfo.name+fake_tar_2nd)
      s=prepare_for_echo(newtarinfo.name)
      script.write("$echo '"+ s +"'\"${FTH}\" >> OLD/mega_cat\n")

      if newtarinfo.isdir():
        if VERBOSE > 2 : print '   directory   in new : ', name
        continue

      if not newtarinfo.isreg():
        if VERBOSE > 2 : print '   not regular in new : ', name
        continue

      if newtarinfo.size == 0:
        if VERBOSE > 2 : print '   empty  new file    : ', name
        continue

      if name not in correspondence:
        if VERBOSE > 2: print '   no corresponding fil: ', name
        continue 
      oldname = correspondence[name]

      mul=len( old_used[oldname]) > 1 #multiple usage
      
      if not mul and oldname == name and oldname[-3:] == '.gz' and \
             newtarinfo.size > 120 and  \
        not ( new_md5 and name in new_md5 and old_md5 and name in old_md5 and \
           new_md5[name] == old_md5[name]):
        newtar.extract(newtarinfo,TD+"NEW/"+CWD )
        delta_gzipped_files("OLD/"+CWD+'/'+name,"NEW/"+CWD+'/'+name)

      if VERBOSE > 2 :  print '   adding reg file: ', oldname, mul and '(multiple)' or ''
      _append_( "OLD/"+CWD+"/"+oldname , not mul )
      old_used[oldname].pop()


    mega_cat.close()
    if os.path.exists(TD+'/OLD/'+CWD):
      rmtree(TD+'/OLD/'+CWD)
    if os.path.getsize(TD+'OLD/mega_cat') > 0 :
      if progressive_new_offset > 0 :
        assert(chunked_p)
        mega_cat_chunk(progressive_new_offset, os.path.getsize(TD+new_filename))
      else:
        delta_files('OLD/mega_cat',new_filename)
        unlink(TD+new_filename)
    else:
      p=verbatim(new_filename)
      script.write('mv '+p+' '+new_filename+ '\n')
    script.write('wait\n')
    script_zip(new_filename,new_filename_ext,new_file_zip_head)
  ####################  ^^^^    delta_tar    ^^^^ ###########################

  ############ start computing deltas  
  def append_NEW_file(s):
    'appends some data to NEW.file'
    s=prepare_for_echo(s)
    script.write("$echo '"+ s +"' >> NEW.file\n")
    
  #this following is actually
  #def delta_debs_using_old(old,new):

  ### start scanning the new deb  
  newdeb_file=open(newdeb)
  # pop the "!<arch>\n"
  s = newdeb_file.readline()
  assert( "!<arch>\n" == s)
  append_NEW_file(s)

  #process all contents of old vs new .deb
  ar_list_old= list_ar(TD+'OLD.file')
  ar_list_new= list_ar(TD+'NEW.file')

  def md5_ar(TD,n,name):
    "extra md5 check, for tracking strange bugs"
    pm=os.popen('cd '+TD+'; ar p OLD.file '+name+' | md5sum -')
    data_tar_md5=pm.readline()[:32]
    pm.read()
    pm.close()
    script_md5_check_file(n,data_tar_md5)

  for name in ar_list_new :
    n = 'NEW/'+name
    system('ar p '+TD+'NEW.file '+name+' >> '+TD+n,TD)

    newsize = os.stat(TD+n)[ST_SIZE]
    if VERBOSE > 1: print '  studying ' , name , ' of len %dkB' % (newsize/1024)
    #add 'ar' structure
    s = newdeb_file.read(60)
    if VERBOSE > 3: print '  ar line: ',repr(s)
    assert( s[:len(name)] == name and s[-2] == '`' and s[-1] == '\n' )
    append_NEW_file(s)
    #sometimes there is an extra \n, depending if the previous was odd length
    newdeb_file.seek(newsize  ,1)
    if newsize & 1 :
      extrachar = newdeb_file.read(1)
    else:
      extrachar = ''
    #add file to debdelta
    if newsize < 128:      #file is too short to compute a delta,
      p=open(TD+n)
      append_NEW_file( p.read(newsize))
      p.close()
      unlink(TD+n)
    elif not NEEDSOLD and name[:11] == 'control.tar' :
      #(mm this is almost useless, just saves a few bytes)
      o = 'OLD/'+name
      system('ar p OLD.file '+name+' >> '+o, TD)
      ##avoid using strange files that dpkg may not install in /var...info/
      skip=[]
      for a in os.listdir(TD+'OLD/CONTROL') :
        if a not in dpkg_keeps_controls:
          skip.append(a)
      #delta it
      #never chunked .. otherwise the first file in the ar will not be '0'!
      delta_tar(o,n,'CONTROL',skip, chunked_p=False)
      if DEBUG > 3 : md5_ar(TD,n,name)
      script.write('cat '+n+' >> NEW.file ;  rm '+n+'\n')
    elif not NEEDSOLD and name[:8] == 'data.tar'  :
      o = 'OLD/'+name
      #system('ar p OLD.file '+name+' >> '+o, TD)
      assert(name[-3:] == '.gz')#should add support for bz2 data.tar
      def x():
        return os.popen('cd '+TD+'; ar p OLD.file '+name+' | gzip -cd')
      delta_tar(x,n,'DATA',old_conffiles,old_md5,new_md5)
      if DEBUG > 3 : md5_ar(TD,n,name)
      script.write('cat '+n+' >> NEW.file ;  rm '+n+'\n')
    elif  not NEEDSOLD  or name not in ar_list_old :   #or it is not in old deb
      p=verbatim(n)
      script.write('cat '+p+' >> NEW.file ; rm '+p+'\n')
    elif  NEEDSOLD :
      #file is long, and has old version ; lets compute a delta
      o = 'OLD/'+name
      system('ar p OLD.file '+name+' >> '+o, TD)
      script.write('ar p OLD.file '+name+' >> '+o+'\n')
      (o,co) = unzip(o)
      (n,cn) = unzip(n)
      delta_files(o,n)
      script_zip(n,cn)
      script.write('cat '+n+cn+' >> NEW.file ;  rm '+n+'\n')
      unlink(TD+n)
    else:
      die('internal error j98')
    #pad new deb
    if extrachar :
      append_NEW_file(extrachar)
  # put in script any leftover
  s = newdeb_file.read()
  if s:
    if VERBOSE > 2: print '   ar leftover character: ',repr(s)
    append_NEW_file(s)

  #this is done already from the receiving end
  if DEBUG > 2 and newdeb_md5sum :
    script_md5_check_file("NEW.file",md5=newdeb_md5sum)
  
  #script is done
  script.close()

  patchsize = os.stat(TD+'PATCH/patch.sh')[ST_SIZE]
  v=''
  #if VERBOSE > 1 :v ='-v' #disabled... it does not look good inlogs
  system('bzip2 --keep -9  '+v+'  PATCH/patch.sh 2>&1', TD)
  system('gzip -9 -n '+v+' PATCH/patch.sh 2>&1', TD)  
  if  os.path.getsize(TD+'PATCH/patch.sh.gz') > os.path.getsize(TD+'PATCH/patch.sh.bz2') :
    if VERBOSE > 1 : print '  bzip2 wins on patch.sh  '
    patch_append('patch.sh.bz2')
  else:
    if VERBOSE > 1 : print '  gzip wins on patch.sh  '
    patch_append('patch.sh.gz')
  
  #OK, OK... this is not yet correct, since I will add the info file later on
  elaps =  time.time() - start_sec
  info.append('DeltaTime: %.2f' % elaps)
  deltasize = os.stat(delta)[ST_SIZE] + 60 + sum(map(len,info))
  percent =  deltasize * 100. /  newdebsize
  info.append('Ratio: %.4f' % (float(deltasize) / float(newdebsize)) )

  if VERBOSE:
    print ' deb delta is  %3.1f%% of deb; that is, %dkB are saved, on a total of %dkB.' \
          % ( percent , (( newdebsize -deltasize ) / 1024),( newdebsize/ 1024))
    print ' delta time: %.2f sec, speed: %dkB /sec, (%s time: %.2fsec speed  %dkB /sec) (corr %.2f sec)' %  \
          (elaps, newdebsize / 1024. / (elaps+0.001), \
           USE_DELTA_ALGO,bsdiff_time, bsdiff_datasize / 1024. / (bsdiff_time + 0.001) , time_corr )
  return (delta, percent, elaps, info)


##################################################### compute many deltas

def do_deltas(debs):
  original_cwd = os.getcwd()
  start_time = time.time()
  import warnings
  warnings.simplefilter("ignore",FutureWarning)
  try:
    from apt import VersionCompare
  except ImportError:
    import apt_pkg
    apt_pkg.InitSystem()
    from apt_pkg import VersionCompare

  if AVOID and type(AVOID) == StringType:
    import shelve
    if VERBOSE: print ' Using avoid dict ',AVOID
    avoid_pack = shelve.open(AVOID,'r')
  else:
    avoid_pack = {}
  
  info_by_pack_arch={}
  info_by_file={}
  deb_dir_visited=[]

  def scan_deb_dir(f, pack_filter, label):
      "pack filter may be a function that matches by basename"
      assert( os.path.isdir(f))
      if f not in deb_dir_visited:
        if pack_filter == None :
          deb_dir_visited.append(f)
        for d in os.listdir(f):
          dt=os.path.join(f,d)
          if os.path.isfile(dt) and d[-4:] == '.deb' and \
                 ( pack_filter == None or  pack_filter(d) ) :
            scan_deb( dt , label )
            
  def scan_deb(of, label):
      assert( os.path.isfile(of) )
      f=abspath(of)
      if f in info_by_file:
        #just (in case) promote to status of CMDLINE package
        if label == 'CMDLINE' :
          #this changes also the entry in info_by_pack_arch (magic python)
          info_by_file[f]['Label']=label
        return
      p=open(f)
      if p.read(21) != "!<arch>\ndebian-binary" :
	  p.close()
	  if os.path.getsize(f) == 0 :
	      print ('Warning: '+f+ ' is an empty file; removing it. ')
	      os.unlink(f)
	  else:  
	      print ('Error: '+f+ ' does not seem to be a Debian package ')
	  return
      p.close()
      info_by_file[f]={}
      p=os.popen('ar p '+f+' control.tar.gz | tar -x -z -f - -O ./control')
      scan_control(p,params=info_by_file[f])
      p.close()
      info_by_file[f]['File'] = of
      pa=info_by_file[f]['Package']
      ar=info_by_file[f]['Architecture']
      ve=info_by_file[f]['Version']
      info_by_file[f]['Label'] = label
      if pa in avoid_pack and ( avoid_pack[pa]['Version'] == ve ):
        #note that 'f' is in  info_by_file and not in info_by_pack_arch
        if VERBOSE > 1 :     print 'Avoid: ', new['File']
        return
      if  (pa,ar) not in  info_by_pack_arch :
         info_by_pack_arch[ (pa,ar) ]=[]
      info_by_pack_arch[ (pa,ar) ].append( info_by_file[f] )

  # contains list of triples (filename,oldversion,newversion)
  old_deltas_by_pack_arch={}
  old_deltas_dir_visited=[]
  def scan_delta_dir(f,pack_filter=None):
    assert( os.path.isdir(f) )
    if f not in old_deltas_dir_visited:
      if pack_filter == None :
        old_deltas_dir_visited.append(f)
      for d in os.listdir(f):
        dt=os.path.join(f,d)
        if os.path.isfile(dt) and ( pack_filter == None or  pack_filter(d) ):
          scan_delta( dt )
  
  def scan_delta(f):
    assert( os.path.isfile(f) )
    if f[-9:] == '.debdelta' :
      a=f[:-9]
    elif f[-17:] == '.debdelta-too-big' :
      a=f[:-17]
    elif f[-15:] == '.debdelta-fails' :
      a=f[:-15]
    else: return
    a=os.path.basename(a)
    a=a.split('_')
    pa=a[0]
    ar=a[3]
    if  (pa,ar) not in old_deltas_by_pack_arch:
      old_deltas_by_pack_arch[ (pa,ar) ]=[]
    ov=version_demangle(a[1])
    nv=version_demangle(a[2])
    if (f,ov,nv) not in old_deltas_by_pack_arch[ (pa,ar) ]:
      old_deltas_by_pack_arch[ (pa,ar) ].append( (f, ov, nv ) )

  def delta_dirname(f,altdir):
    "compute augmented dirname"
    if os.path.isfile(f):
      f=os.path.dirname(f) or '.'
    assert(os.path.isdir(f))
    if altdir:
      if altdir[-2:] == '//' :
        a=altdir+f
        return make_parents(abspath(a)+'/')
      else:
        return altdir
    else:
      return abspath(f)

  def __name_filter__(n):
    "returns a function that filters by package name"
    n=os.path.basename(n)
    n=n.split('_')[0] + '_'
    l=len(n)
    return lambda x : x[:l] == n

  #scan cmdline arguments and prepare list of debs and deltas
  for f in debs:
    if os.path.isfile(f):
      if f[-4: ] != '.deb' :
        print 'Warning: skipping cmd line argument: ',f
        continue
      scan_deb(f, 'CMDLINE')
      di=os.path.dirname(f) or '.'
      scan_deb_dir(di, __name_filter__(f), 'SAMEDIR' )
      if ALT:        
        scan_deb_dir(delta_dirname(f,ALT), __name_filter__(f), 'ALT' )
      if CLEAN_DELTAS:
        scan_delta_dir(delta_dirname(f,DIR), __name_filter__(f) )
    elif  os.path.isdir(f) :
      scan_deb_dir(f, None, 'CMDLINE')
      if ALT:
        scan_deb_dir(delta_dirname(f,ALT), None, 'ALT')
      if CLEAN_DELTAS:
        scan_delta_dir(delta_dirname(f,DIR))
    else:
      print 'Warning: '+f+' is not a regular file or a directory.'
  
  def order_by_version(a,b):
    return VersionCompare( a['Version'] , b['Version']  )
  
  for pa,ar in info_by_pack_arch :
    info_pack=info_by_pack_arch[ (pa,ar) ]
    info_pack.sort(order_by_version)

    versions = [ o['Version'] for o in info_pack ]

    versions_not_alt = [ o['Version'] for o in info_pack if o['Label'] != "ALT" ]

    #delete deltas that are useless
    if CLEAN_DELTAS and (pa,ar) in old_deltas_by_pack_arch :
      for f_d,o_d,n_d in old_deltas_by_pack_arch[ (pa,ar) ] :
        if n_d not in versions_not_alt :
          if os.path.exists(f_d):
            if VERBOSE: print 'Removing: ',f_d          
            if ACT: os.unlink(f_d)
    
    how_many= len( info_pack  )
    if VERBOSE>2:
      print 'Package: ',pa,' Versions:',versions
    if how_many <= 1 :
      continue
    
    newest = how_many -1
    while newest >= 0 :
      new=info_pack[newest]
      if new['Label'] != 'CMDLINE' :
        if VERBOSE > 1 :
          print 'Newest version deb was not in cmdline, skip down one: ', new['File']
      else:
        break
      newest -= 1

    if newest <= 0 :
      continue

    newdebsize=os.path.getsize(new['File'])
    #very small packages cannot be effectively delta-ed
    if newdebsize <= MIN_DEB_SIZE :
      if VERBOSE > 1:     print '  Skip , too small: ', new['File']
      continue

    deltadirname=delta_dirname(new['File'],DIR)
    free=freespace(deltadirname)
    if free and (free < (newdebsize /2 + 2**15)) :
      if VERBOSE : print 'Not enough disk space for storing ',delta
      continue

    l = newest
    while (l>0) and (l > newest - N_DELTAS):
        l -= 1
        old=info_pack[l]
        
        if  old['Version'] == new['Version'] :
          continue
                
        assert( old['Package'] == pa and pa == new['Package'] )
        deltabasename = pa +'_'+  version_mangle(old['Version']) +\
                        '_'+ version_mangle(new['Version']) +'_'+ar+'.debdelta'

        make_parents(abspath(deltadirname)+'/')
        delta=os.path.join(deltadirname,deltabasename)
        
        if os.path.exists(delta):
          if VERBOSE > 1:     print '  Skip , already exists: ',delta
          continue
        
        if os.path.exists(delta+'-too-big'):
          if VERBOSE > 1:     print '  Skip , tried and too big: ',delta
          continue

        if os.path.exists(delta+'-fails'):
          if VERBOSE > 1:     print '  Skip , tried and fails: ',delta
          continue

        if not ACT:
          print 'Would create:',delta
          continue
        
        if VERBOSE: print 'Creating :',delta
        ret= None
        try:
          ret=do_delta(old['File'],new['File'], delta)
        except DebDeltaError,s:
          if not VERBOSE: print 'Creating: ',delta
          print ' Creation of delta failed, reason: ',str(s)
          if not s.retriable :
            p=open(delta+'-fails','w')
            p.close()
        except KeyboardInterrupt:
          raise
        except:
          puke( " *** Error while creating delta  "+delta)

        if ret == None:
          continue
        
        (delta_, percent, elaps, info_delta) = ret
        assert(delta == delta_)
        info_delta.append('ServerID: '+HOSTID)
        info_delta.append('ServerBogomips: '+str(BOGOMIPS))
        
        if MAX_DELTA_PERCENT and  percent > MAX_DELTA_PERCENT:
            os.unlink(delta)
            if VERBOSE : print ' Warning, too big!'
            p=open(delta+'-too-big','w')
            p.close()
            continue

        if DEBUG :
          pret=None
          try:
            pret=do_patch(delta,old['File'],None , info=info_delta)
          except DebDeltaError,s:
            print ' Error: testing of delta failed: ',str(s)
            if not  s.retriable :
              p=open(delta+'-fails','w')
              p.close()
              if os.path.exists(delta):
                os.unlink(delta)
          except KeyboardInterrupt:
            raise
          except:
            puke(" *** Error while testing delta  "+delta)
            if os.path.exists(delta):
              os.unlink(delta)
          
          if pret == None:
            continue
          
          (newdeb_,p_elaps)=pret
          info_delta.append('PatchTime: %.2f' % p_elaps)
        append_info(delta,info_delta)
    #delete debs in --alt that are too old
    if CLEAN_ALT:
      while l>=0:
        old=info_pack[l]
        if old['Label'] == 'ALT':
          f=old['File']
          if os.path.exists(f):
            if VERBOSE: print 'Removing alt deb: ',f
            if ACT: os.unlink(f)
        l-=1

  if VERBOSE: print 'Total running time: %.1f ' % ( -start_time + time.time())

################################################# main program, do stuff

if action == 'patch':
  if INFO  :
    if  len(argv) > 1 and VERBOSE :
      print '(printing info - extra arguments are ignored)'
    elif  len(argv) == 0  :
      print ' need a  filename ;  try --help'
      raise SystemExit(1)
    try:
        delta=abspath(argv[0])
        check_is_delta(delta)
        info=get_info(delta)
        for s in info:
          if s:
            print ' info: ',s
    except (KeyboardInterrupt, SystemExit):
        if DEBUG : puke('debpatch exited')
    except DebDeltaError,s:
        print  str(s)
        raise SystemExit(1)
    except :
        puke( "Unexpected error" )
        raise SystemExit(1)
    raise SystemExit(0)
  #really patch
  if len(argv) != 3 :
    print ' need 3 filenames ;  try --help'
    raise SystemExit(1)


  newdeb=abspath(argv[2])
  if newdeb == '/dev/null':
      newdeb = None

  try:
    do_patch(abspath(argv[0]), abspath(argv[1]), newdeb)
  except (KeyboardInterrupt, SystemExit):
    if DEBUG : puke('debpatch exited')
  except Exception,s:
    puke( 'debpatch failed',s)
    raise SystemExit(2)
  
elif action == 'delta' :
  if len(argv) != 3 :  
    print ' need 3 filenames ;  try --help'
    raise SystemExit(1)
  
  delta=abspath(argv[2])
  try:
    r = do_delta(abspath(argv[0]), abspath(argv[1]), delta)
  except (KeyboardInterrupt, SystemExit):
    if DEBUG : puke('debdeltas exited')
  except DebDeltaError,s:
    puke('Failed: ',s)
    raise SystemExit(2)
  except:
    puke('debdelta failed' )
    raise SystemExit(3)
  else:
    (delta, percent, elaps, info) = r
    append_info(delta,info)
  
elif action == 'deltas' :
  try:
    do_deltas(argv)
  except (KeyboardInterrupt, SystemExit):
    if DEBUG : puke('debdeltas exited')
  except:
    puke( 'debdeltas failed')
    raise SystemExit(2)
  
  
##################################################### delta-upgrade

class Predictor:
  package_stats = None
  upgrade_stats = None
  def __init__(self):
    import shelve
    #self.shelve=shelve
    if os.getuid() == 0:
      basedir='/var/lib/debdelta'
    else:
      basedir=os.path.expanduser('~/.debdelta')
    s=os.path.join(basedir,'upgrade.db')
    if not os.path.exists(s):
      print 'Creating: ',s
    make_parents(s)
    self.upgrade_stats=shelve.open(s,flag='c')

    s=os.path.join(basedir,'packages_stats.db')
    
    if  os.path.exists(s) or DEBUG :
      if not os.path.exists(s):
        print 'Creating: ',s
      make_parents(s)
      self.package_stats=shelve.open(s,flag='c')

    self.patch_time_predictor=self.patch_time_predictor_math

  ##### predictor for patching time
  def patch_time_predictor_simple(self,p):
    if 'ServerBogomips' in p and 'PatchTime' in p:
      return (float(p[ 'PatchTime']) / BOGOMIPS * float(p['ServerBogomips']) )
    else:
      return None

  def update(self,p,t):
    #save delta info
    if self.package_stats != None :
      n=p['NEW/Package']
      d=copy(p)
      d['LocalDeltaTime']=t
      self.package_stats[n]=d
    
    s='ServerID'
    if s not in p :
      return
    s=s+':'+p[s]
    if s not in self.upgrade_stats:
      r=1
      if 'ServerBogomips' in p :
        r=   float(p['ServerBogomips']) / BOGOMIPS
      self.upgrade_stats[s]={ 'PatchSpeedRatio' : r }

    if 'PatchTime' not in p:
      return
    ut=float(p[ 'PatchTime'])

    r=self.upgrade_stats[s]['PatchSpeedRatio']
    
    nr =  0.95 * r + 0.05 * (  t / ut )
    a=self.upgrade_stats[s]
    a['PatchSpeedRatio'] = nr
    self.upgrade_stats[s]=a
    if VERBOSE > 1 :
      print ' Upstream ',ut,'PatchSpeedRatio from ',r,' to ',nr
      print self.upgrade_stats[s]['PatchSpeedRatio']
      
  def patch_time_predictor_math(self,p):
    "Predicts time to patch."
    if 'PatchTime' not in p:
      return None
    ut=float(p[ 'PatchTime'])
    #
    s='ServerID'
    if s not in p :
      return self.patch_time_predictor_simple(p)
    s=s+':'+p[s]
    if s not in self.upgrade_stats:
      return self.patch_time_predictor_simple(p)

    r=self.upgrade_stats[s]['PatchSpeedRatio']
    return r * ut

  
def delta_upgrade_():
  original_cwd = os.getcwd()

  import  thread , pickle, urllib, fcntl, atexit, signal, ConfigParser

  config=ConfigParser.SafeConfigParser()
  a=config.read(['/etc/debdelta/sources.conf', os.path.expanduser('~/.debdelta/sources.conf')  ])
  # FIXME this does not work as documented in Python
  #if VERBOSE > 1 : print 'Read config files: ',repr(a)
  
  import warnings
  warnings.simplefilter("ignore",FutureWarning)
  
  try:
    import  apt_pkg
  except ImportError:
    print 'ERROR!!! python module "apt_pkg" is missing. Please install python-apt'
    raise SystemExit
  
  try:
    import  apt
  except ImportError:
    print 'ERROR!!! python module "apt" is missing. Please install a newer version of python-apt (newer than 0.6.12)'
    raise SystemExit
  
  apt_pkg.init()

  from apt import SizeToStr

  cache=apt.Cache()
  cache.upgrade(True)

  diversions=scan_diversions()

  if DIR == None:
    if os.getuid() == 0:
      DEB_DIR='/var/cache/apt/archives'
    else:
      DEB_DIR='/tmp/archives'
  else:
    DEB_DIR=DIR
  if not os.path.exists(DEB_DIR):
    os.mkdir(DEB_DIR)
  if not os.path.exists(DEB_DIR+'/partial'):
    os.mkdir(DEB_DIR+'/partial')
    
  try:
    ##APT does (according to strace)
    #open("/var/cache/apt/archives/lock", O_RDWR|O_CREAT|O_TRUNC, 0640) = 17
    #fcntl64(17, F_SETFD, FD_CLOEXEC)        = 0
    #fcntl64(17, F_SETLK, {type=F_WRLCK, whence=SEEK_SET, start=0, len=0}) = 0
    ##so
    a=os.open(DEB_DIR+'/lock', os.O_RDWR | os.O_TRUNC | os.O_CREAT, 0640)
    fcntl.fcntl(a, fcntl.F_SETFD, fcntl.FD_CLOEXEC)
    # synopsis lockf(  	fd, operation, [length, [start, [whence]]])
    fcntl.lockf(a, fcntl.LOCK_EX | fcntl.LOCK_NB, 0,0,0)
  except IOError, s:
    if s.errno == 11 :
      a=' already locked!'
    else:
      a=str(s)
    if DEB_DIR == '/var/cache/apt/archives' :
      a=a+' (is APT running?)'
    print 'Could not lock dir: ',DEB_DIR, a
    raise SystemExit(1)
    
  print 'Recreated debs are saved in ',DEB_DIR

  #these are the packages that do not have a delta
  no_delta = []

  start_sec = time.time()
  len_deltas=0


  ##### predictor for patching time
  predictor = Predictor()

  #this is a dictonary (key is package name) of parameters of deltas
  #(to add some math in the future)
  params_of_delta={}
  
  (qout,qin)=os.pipe()
  thread_returns={}
  ######################## thread_do_patch
  def thread_do_patch(qout,threads,no_delta,returns):
      if VERBOSE > 1 : print ' Patching thread started. '
      debs_size=0
      debs_time=0
      while 1:
        s=os.read(qout,1)
        c=''
        while s != '\t' :
          c+=s
          s=os.read(qout,1)
        if c == '\t' or c == '': break
        (name, delta , newdeb, deb_uri) = pickle.loads(c)
        debs_time -= time.time()
        if not ACT:
          print 'Would create: ',newdeb,'   '
        else:
          if VERBOSE>=2 : print ' Now patching for: ',name
          try:
            ret=do_patch(delta,'/',newdeb , diversions=diversions)
            if VERBOSE == 0 : print 'Created ',newdeb,'   '
          except KeyboardInterrupt:
            thread.interrupt_main()
            return
          except DebDeltaError,s:
            print ' Error: applying of delta for ',name,'failed: ',str(s)
            no_delta.append( (deb_uri, newdeb) )
          except:
            puke( " *** Error while applying delta for "+name+": ")
            no_delta.append( (deb_uri, newdeb) )
          else:
            if name in params_of_delta :
              p= params_of_delta[name]
              name,elaps=ret
              predictor.update(p,elaps)
              if VERBOSE > 1 :
                t=predictor.patch_time_predictor(p)
                if t: print '   (Predicted %.3f sec )'  % t
            debs_size += os.path.getsize(newdeb)
            if os.path.exists(delta):
              os.unlink(delta)
        debs_time += time.time()
      threads.pop()
      if VERBOSE > 1 : print ' Patching thread ended , bye bye. '
      returns['debs_size']=debs_size
      returns['debs_time']=debs_time

  import socket, httplib
  from urlparse import urlparse

  #################### manage connections
  #keeps a cache of all connections, by URL
  http_conns={}
  
  def __host_by_url__(url):
    if url[:7] == 'http://' :
      url = urlparse(url)[1]
    return url

  def conn_by_url(url):
    url=__host_by_url__(url)
    if url not in http_conns:
      if (DEBUG or VERBOSE > 1) :
        print '-Opening connection to: ',url
      http_conns[url] = httplib.HTTPConnection(url)
    return http_conns[url]
  
  def conn_close(url,fatal=False):
    url=__host_by_url__(url)
    conn=http_conns.get(url)
    if fatal:
      http_conns[url] = None
    else:
      del http_conns[url]
    if conn != None :
      if (DEBUG or VERBOSE > 1)  :
        print '-Closing connection to: ',url
      conn.close()

  def delta_uri_from_config(**dictio):
    secs=config.sections()
    for s in secs:
      opt=config.options(s)
      if 'delta_uri' not in opt:
        print 'Error!! sources.conf section ',s,'does not contain delta_uri'
        raise SystemExit(1)
      match=True
      for a in dictio:
        #damn it, ConfigParser changes everything to lowercase !
        if ( a.lower() in opt ) and ( dictio[a] != config.get( s, a) ) :
          #print '!!',a, repr(dictio[a]) , ' != ',repr(config.get( s, a))
          match=False
          break
      if match:
        return  config.get( s, 'delta_uri' )
    if VERBOSE:
      print '(sources.conf does not provide a server for ', repr(dictio['PackageName']),')'
  
  
  ################################################# various HTTP facilities
  def _http_whine_(uri,r):
    a='Url'
    if uri[-9:] == '.debdelta':
      a='Debdelta'
    if  r.status == 200 or  r.status == 206:
      pass
    if r.status == 404:
      print a,' is not present: ',uri
    else:
      print a,' is not available (',repr(r.status), r.reason,'): ', uri


  def _parse_ContentRange(r):
    #bytes 0-1023/25328
    s=r.getheader('Content-Range')
    if s == None: return
    if s[:6] != "bytes " :
      print "Malformed Content-Range",s
      return
    a=s[6:].split('/')
    if len(a) != 2 :
      print "Malformed Content-Range",s
      return
    b=a[0].split('-')
    if len(b) != 2 :
      print "Malformed Content-Range",s
      return
    return int(b[0]),int(b[1]),int(a[1])
  ###################################### test_uri
  def test_uri(uri):
      conn=conn_by_url(uri)
      uri_p=urlparse(uri)
      assert(uri_p[0] == 'http')
      conn.request("HEAD", urllib.quote(uri_p[2]),headers=HTTP_USER_AGENT)
      r = conn.getresponse()
      _http_whine_(uri,r)
      r.read()
      r.close()
      if r.status == 200:
        return r
      if not VERBOSE: return False
      if uri[-9:] == '.debdelta':
        conn.request("HEAD", urllib.quote(uri_p[2]+'-too-big'))
        r2 = conn.getresponse()
        r2.read()
        r2.close()
        if r2.status == 200:
          print 'Too big: ',uri
          return False
      _http_whine_(uri,r)
      return False

  ###################################### download_1k_uri
  def download_1k_uri(uri,outname):
      uri_p=urlparse(uri)
      assert(uri_p[0] == 'http')
      re=copy(HTTP_USER_AGENT)
      re["Range"] =  "bytes=0-1023"
      try:
        conn=conn_by_url(uri)
        if conn == None : return
        conn.request("GET", urllib.quote(uri_p[2]),headers=re)
        r = conn.getresponse()
      except (httplib.HTTPException, socket.error),e:
        puke('Connection error: ',e)
        conn_close(uri)
        return e
      #print '1K Content-Range', r.getheader('Content-Range') #HACK
      if r.status == 206:
        outnametemp=os.path.join(os.path.dirname(outname),'partial',os.path.basename(outname))
      elif r.status == 200:
        outnametemp=outname
      else:
        _http_whine_(uri,r)
        r.read()
        r.close()
        return False
      if os.path.exists(outnametemp) and os.path.getsize(outnametemp) >= 1023 :
        r.read()
        r.close()
        return r, outnametemp
      out=open(outnametemp,'w')
      out.write(r.read())
      #print '1K OK', outnametemp, out.tell() #HACK
      out.close()
      r.close()
      return r, outnametemp

  ###################################### download_uri
  def download_uri(uri,outname,conn_time,len_downloaded):
      uri_p=urlparse(uri)
      assert(uri_p[0] == 'http')
      outnametemp=os.path.join(os.path.dirname(outname),'partial',os.path.basename(outname))
      re=copy(HTTP_USER_AGENT)
      #content range
      l=None
      if os.path.exists(outnametemp):
        #shamelessly adapted from APT, methods/http.cc
        s=os.stat(outnametemp)
        l=s[ST_SIZE]
        #t=s[ST_MTIME]
        ### unfortunately these do not yet work
        #thank god for http://docs.python.org/lib/module-time.html
        #actually APT does
        #t=time.strftime("%a, %d %b %Y %H:%M:%S GMT", time.gmtime(t))
        ##re["If-Range"] =  time.strftime("%a, %d %b %Y %H:%M:%S GMT", time.gmtime(t))
        ####re["If-Range"] =  time.strftime("%a, %d %b %Y %H:%M:%S +0000", time.gmtime(t))
        re["Range"] =  "bytes=%li-" % ( (long(l)-1) )
      try:
        conn=conn_by_url(uri)
        if conn == None : return
        conn.request("GET", urllib.quote(uri_p[2]),headers=re)
        r = conn.getresponse()
      except (httplib.HTTPException,socket.error),e:
        if DEBUG or VERBOSE > 1 : puke( 'Connection error (retrying): ',e)
        try:
          conn_close(uri)
          conn=conn_by_url(uri)
          if conn == None : return
          conn.request("GET", urllib.quote(uri_p[2]),headers=re)
          r = conn.getresponse()
        except (httplib.HTTPException,socket.error),e:
          puke( 'Connection error (fatal): ',e)
          try:
            conn_close(uri,fatal=True)
          except: pass
          return e
      if not ( r.status == 200 or ( r.status == 206 and l != None ) ):
        if VERBOSE : _http_whine_(uri,r)
        r.read()
        r.close()
        return None
      #print 'ooK Content-Range', r.getheader('Content-Range') #HACK
      if l and r.status == 200 :
        print ' Hmmm... our HTTP range request failed, ',repr(re),r.status,r.reason
      assert( r.length == int(r.getheader('content-length')) )
      free=freespace(os.path.dirname(outname))
      if free and (free + 2**14 ) < r.length  :
        print 'Not enough disk space to download: ',os.path.basename(uri)
        r.read()
        r.close()
        return None
      if r.status == 200 :
        out=open(outnametemp,'w')
        total_len = r.length
      elif r.status == 206 :
        #APT does scanf of    "bytes %lu-%*u/%lu",&StartPos,&Size
        #first-byte-pos "-" last-byte-pos "/" instance-length
        a,b,total_len =_parse_ContentRange(r)
        out=open(outnametemp,'a')
        out.seek(a)
        out.truncate()
      a=time.time()
      conn_time-=a
      j=out.tell()
      s=r.read(1024)
      while s and j < total_len :
        j+=len(s)
        out.write(s)
        if not DEBUG and a + 0.5 < time.time() :
          a=time.time()
          sys.stderr.write("%d%% (%4s/s) %s \r" % \
                           (100*j / total_len,
                            SizeToStr((j+len_downloaded)/(a+conn_time)),\
                            os.path.basename(uri)[:50] ))
        s=r.read(1024)
      out.close()
      r.close()
      conn_time+=time.time()
      if DEBUG:
        a = time.time() - a
        print "Downloaded, time: %.2fsec speed: %4s/sec uri: %s " % (a , SizeToStr(total_len / (a+0.001)) , uri)
      else:
        sys.stderr.write("Downloaded:  %s \n" % os.path.basename(uri) )
      os.rename(outnametemp,outname)
      #FIXME this is incorrect by 1024 bytes
      return  conn_time , (j+len_downloaded)      

  ###################################### end of HTTP stuff
  
  deltas_down_size=0
  deltas_down_time=0

  #this is a list of tuples of .....
  available_deltas=[]

  ## first merry-go-round, use package cache to fill available_deltas
  for p in cache :
    if p.isInstalled and  p.markedUpgrade :
      #thanks a lot to Michael Vogt
      p._lookupRecord(True)
      dpkg_params = apt_pkg.ParseSection(p._records.Record)
      cand = p._depcache.GetCandidateVer(p._pkg)
      deb_path=dpkg_params['Filename']      
      for (packagefile,i) in cand.FileList:
        indexfile = cache._list.FindIndex(packagefile)
        if indexfile:
          deb_uri=indexfile.ArchiveURI(deb_path)
          break
      
      arch=dpkg_params['Architecture']      
      
      #newdeb=p.name+'_'+version_mangle(p.candidateVersion)+'_'+arch+'.deb'
      newdeb=os.path.basename(deb_uri)
      if os.path.exists(DEB_DIR+'/'+newdeb) or \
             os.path.exists('/var/cache/apt/archives/'+newdeb):
        if VERBOSE > 1 : print  'Already downloaded: ',newdeb
        continue
      newdeb = DEB_DIR+'/'+newdeb

      if VERBOSE > 1:
        print 'Looking for a delta for %s from %s to %s ' % \
              ( p.name, p.installedVersion, p.candidateVersion )
      delta_uri_base=delta_uri_from_config(Origin=p.candidateOrigin[0].origin,
                                           Label=p.candidateOrigin[0].label,
                                           Site=p.candidateOrigin[0].site,
                                           Archive=p.candidateOrigin[0].archive,
                                           PackageName=p.name)
      if delta_uri_base == None:
        no_delta.append( (deb_uri, newdeb) )
        continue

      a=urlparse(delta_uri_base)
      assert(a[0] == 'http')

      #delta name
      delta_name=p.name+'_'+version_mangle(p.installedVersion)+\
                  '_'+ version_mangle(p.candidateVersion)+'_'+\
                  arch+'.debdelta'

      uri=delta_uri_base+'/'+os.path.dirname(deb_path)+'/'+delta_name
      
      #download first part of delta
      abs_delta_name= DEB_DIR+'/'+delta_name
      if os.path.exists(abs_delta_name):
        l=os.path.getsize(abs_delta_name)
        if VERBOSE > 1 : print 'Already here: ',abs_delta_name
        s=get_info_fast(abs_delta_name)
        if s:
          params_of_delta[p.name]=info_2_db(s)
        available_deltas.append( (l, p.name, uri, abs_delta_name , newdeb, deb_uri, abs_delta_name )  )
        continue
      r = download_1k_uri(uri,abs_delta_name)

      if  r == None or isinstance(r, httplib.HTTPException) or isinstance(r, socket.error) :
        if VERBOSE : print ' You may wish to rerun, to get also: ',uri
        continue
      
      if not r:
        no_delta.append( (deb_uri, newdeb) )
        continue
      
      r,tempname = r
      
      if r.status == 206:
        a,b,l = _parse_ContentRange(r)
      else:
        l=int(r.getheader('content-length'))
      s=get_info_fast(tempname)
      if s:
        params_of_delta[p.name]=info_2_db(s)
        s=patch_check_tmp_space(params_of_delta[p.name],  '/')
        if s != True:
          print p.name,' : sorry '+s
          #neither download deb nor delta..
          #the user may wish to free space and retry
          continue
      #FIXME may check that parameters are conformant to what we expect

      available_deltas.append( (l, p.name, uri, abs_delta_name , newdeb, deb_uri, tempname  ) )
  ## end of first merry-go-round

  available_deltas.sort()

  threads=[]
  threads.append(thread.start_new_thread(thread_do_patch  , (qout,threads,no_delta, thread_returns) ) )
  
  ## second merry-go-round, try downloading available delta
  for delta_len, name, uri, abs_delta_name , newdeb, deb_uri, tempname  in available_deltas :
    if  not os.path.exists(abs_delta_name) and os.path.exists(tempname) and os.path.getsize(tempname) == delta_len:
      print 'just Rename ',name
      os.rename(tempname,abs_delta_name)

    if name in params_of_delta:
      s=patch_check_tmp_space(params_of_delta[name],  '/')
      if s != True:
        print name,' : sorry, '+s
        #argh, we ran out of space in meantime
        continue
    
    if not os.path.exists(abs_delta_name):
      r=download_uri(uri , abs_delta_name , deltas_down_time,deltas_down_size)
      if r == None or isinstance(r, httplib.HTTPException) :
        if VERBOSE : print ' You may wish to rerun,  to get also: ',uri
        continue
      else:
        deltas_down_time = r[0]
        deltas_down_size = r[1]

      #queue to apply delta
    if os.path.exists(abs_delta_name):
        #append to queue
        c=pickle.dumps(  (name, abs_delta_name  ,newdeb, deb_uri ) )
        os.write(qin, c + '\t' )
    else:
      no_delta.append( (deb_uri, newdeb) )
  ## end of second merry-go-round

  #terminate queue
  os.write(qin,'\t\t\t')
  if threads:
    time.sleep(0.2)
  
  #do something useful in the meantime
  debs_down_size=0
  debs_down_time=0
  if  threads and no_delta and VERBOSE > 1 :
    print ' Downloading deltas done, downloading debs while waiting for patching thread.'
  while threads:
    while no_delta:
      uri, newdeb  = no_delta.pop()
      r=download_uri(uri , newdeb, debs_down_time, debs_down_size )
      if isinstance(r, httplib.HTTPException) :
        if VERBOSE : print ' You may wish to rerun, to get also: ',uri
        continue
      if r:
        debs_down_time = r[0]
        debs_down_size = r[1]
    time.sleep(0.2)

  #save predictor...
  
  for i in http_conns:
    if http_conns[i] != None :
      http_conns[i].close()
  
  elaps =  time.time() - start_sec
  print 'Delta-upgrade statistics:'
  if VERBOSE:
    if deltas_down_time :
      a=float(deltas_down_size)
      t=deltas_down_time
      print ' download deltas size %s time %dsec speed %s/sec' %\
            ( SizeToStr(a) , int(t), SizeToStr(a / t ))
    if thread_returns['debs_time'] :
      a=float(thread_returns['debs_size'])
      t=thread_returns['debs_time']
      print ' patching to debs size %s time %dsec speed %s/sec' %\
            ( SizeToStr(a) , int(t), SizeToStr(a / t ))
    if debs_down_time :
      a=float(debs_down_size)
      t=debs_down_time
      print ' download debs size %s time %dsec speed %s/sec' %\
            ( SizeToStr(a) , int(t), SizeToStr(a / t ))
  if elaps:
    a=float(debs_down_size  + thread_returns['debs_size'])
    print ' total resulting debs size %s time %dsec virtual speed: %s/sec' %  \
          ( SizeToStr(a ), int(elaps), SizeToStr(a / elaps))


####

if action == 'delta-upgrade':
  import warnings
  warnings.simplefilter("ignore",FutureWarning)
  try:
    delta_upgrade_()
  except (KeyboardInterrupt, SystemExit):
    if DEBUG : puke('debdelta-upgrade exited')
  except:
    puke('delta-upgrade failed')
    raise SystemExit(2)
  raise SystemExit(0)


##################################################### apt method

### still work in progress
if  os.path.dirname(sys.argv[0]) == '/usr/lib/apt/methods' :
  import os,sys, select, fcntl, apt, thread, threading, time

  apt_cache=apt.Cache()
  
  log=open('/tmp/log','a')
  log.write('  --- here we go\n')
  
  ( hi, ho , he) = os.popen3('/usr/lib/apt/methods/http.distrib','b',2)

  nthreads=3

  class cheat_apt_gen:
    def __init__(self):
      self.uri=None
      self.filename=None
      self.acquire=False
    def process(self,cmd):
      if self.uri:
        self.filename=cmd[10:-1]
        log.write(' download %s for %s\n' % (repr(self.uri),repr(self.filename)))
        self.uri=None
        self.filename=None
        self.acquire=False
        return cmd
      elif self.acquire:
        self.uri=cmd[5:-1]
        return cmd
      elif cmd[:3] == '600' :
        self.acquire=True
      else:
        return cmd
  
  def copyin():
    bufin=''
    while 1:
      #print ' o'
      s=os.read(ho.fileno(),1)
      bufin += s
      if log and bufin and (s == '' or s == '\n') :
        log.write( ' meth ' +repr(bufin)+'\n' )
        bufin=''
      if s == '':
        thread.interrupt_main(   )
        global nthreads
        if nthreads:
          nthreads-=1
        #log.write( ' in closed \n' )
        #return
      os.write(1,s)


  def copyerr():
    buferr=''
    while 1:
      s=os.read(he.fileno(),1)
      buferr += s
      if log and buferr and (s == '' or s == '\n') :
        log.write( ' err ' +repr(buferr)+'\n' )
        buferr=''
      if s == '':
        thread.interrupt_main(   )
        global nthreads
        if nthreads:
          nthreads-=1
        log.write( ' err closed \n' )
        #return
      os.write(2,s)

  def copyout():
    gen=cheat_apt_gen()
    bufout=''
    while 1:
      s=os.read(0,1)
      bufout += s
      if log and bufout and (s == '' or s == '\n') :
        log.write( ' apt ' +repr(bufout)+'\n' )

        bufout=gen.process(bufout) 
        
        bufout=''
      if s == '':
        thread.interrupt_main()
        global nthreads
        if nthreads:
          nthreads-=1
        #log.write( ' out closed \n' )
        #return
      os.write(hi.fileno(),(s))

        
  tin=thread.start_new_thread(copyin,())
  tout=thread.start_new_thread(copyout,())
  terr=thread.start_new_thread(copyerr,())
  while nthreads>0 :
    log.write( ' nthreads %d \n' % nthreads )
    try:
      while nthreads>0 :
        time.sleep(1)      
    except KeyboardInterrupt:
      pass
  raise SystemExit(0)

@


1.112
log
@update path of xdelta3 (it is in Debian in /usr/bin/xdelta3)

save delta algo in header
@
text
@d11 4
a14 1
  -M Mb     maximum memory  to use (for 'bsdiff' or 'xdelta')
d35 4
a38 1
  -M Mb     maximum memory to use (for 'bsdiff' or 'xdelta')
d71 4
a74 4
  -v      verbose (can be added multiple times)
 --no-act do not do that (whatever it is!)
  -d      add extra debugging checks
  -k      keep temporary files (use for debugging)
d133 1
a133 1
RCS_VERSION="$Id: debdelta,v 1.111 2007/07/19 12:05:55 debdev Exp debdev $"
@


1.111
log
@backend xdelta3 now refers to xdelta30q
@
text
@d127 1
a127 1
RCS_VERSION="$Id: debdelta,v 1.110 2007/07/18 12:20:59 debdev Exp debdev $"
d938 2
a1061 3
    #http://code.google.com/p/xdelta/downloads/detail?name=xdelta30m.tar.gz
    #used to crash
    #http://sourceforge.net/tracker/index.php?func=detail&aid=1506523&group_id=6966&atid=106966
d1063 2
a1064 2
      system(' ~/debdelta/xdelta30q/xdelta3 -9 -R -D -n -S djw -s  '+o+' '+n+' '+p,TD)
      script.write(' ~/debdelta/xdelta30q/xdelta3 -d -s '+o+' '+p+' '+n+' ; rm '+p+'\n')
@


1.110
log
@(again) corrected files_similarity_score__noext__() so that it does not eat lists
@
text
@d127 1
a127 1
RCS_VERSION="$Id: debdelta,v 1.109 2007/07/17 08:09:07 debdev Exp debdev $"
d1064 2
a1065 2
      system(' ~/debdelta/xdelta30m/xdelta3 -9 -s  '+o+' '+n+' '+p,TD)
      script.write(' ~/debdelta/xdelta30m/xdelta3 -d -s '+o+' '+p+' '+n+' ; rm '+p+'\n')
@


1.109
log
@save header of minigzip output in patch script, since it changes in newer versions of zlib
@
text
@d127 1
a127 1
RCS_VERSION="$Id: debdelta,v 1.108 2007/07/16 07:41:39 debdev Exp debdev $"
d1273 2
a1274 2
        oo.pop()
        nn.pop()
@


1.108
log
@corrected files_similarity_score__noext__() so that it does not eat lists ;
and reviewed correspondence code
@
text
@d127 1
a127 1
RCS_VERSION="$Id: debdelta,v 1.107 2007/04/13 09:57:31 debdev Exp debdev $"
a988 1
    assert(os.path.isfile(TD+n))
d990 1
d996 1
d1026 1
a1026 1
  def script_zip(n,cn):
d1028 2
a1029 1
      script.write('./minigzip -9 '+n+'\n')
d1405 4
d1629 1
a1629 1
    script_zip(new_filename,new_filename_ext)
d1632 1
a1632 2
  ############ start computing deltas
    
d1652 8
d1695 1
d1704 1
@


1.107
log
@option --debug was incorrectly passed to getopt
@
text
@d127 1
a127 1
RCS_VERSION="$Id: debdelta,v 1.106 2007/04/13 09:49:03 debdev Exp debdev $"
a1250 1
    "warning: destroys the input"
d1256 2
a1257 2
        oo.pop()
        nn.pop()
d1474 1
a1474 1
        basescore=0.6
d1476 1
a1476 1
        np=file_similarity_premangle(nb)        
d1480 4
a1483 2
          s=files_similarity_score__noext__(op,np) + abs(float(l - nl))/float(l+nl)
          #print ' diff ',s,o
d1487 1
a1487 1
        if oldname and VERBOSE > 2 : print '   best similar  ',int(100*basescore),newname,oldname
@


1.106
log
@code to test  xdelta 30m  and  jojodiff (= jdiff06)
@
text
@d127 1
a127 1
RCS_VERSION="$Id: debdelta,v 1.105 2006/12/21 13:02:11 debdev Exp debdev $"
d145 1
a145 1
                 ('help','info','needsold','dir=','no-act','alt=','avoid=','delta-algo=','max-percent=','clean-deltas','clean-alt','no-md5','--debug') )
@


1.105
log
@add --debug option
@
text
@d127 1
a127 1
RCS_VERSION="$Id: debdelta,v 1.104 2006/09/07 19:02:57 debdev Exp debdev $"
d1058 2
a1059 1
    #crashes!
d1062 2
a1063 2
      system(' ~/debdelta/xdelta30e/xdelta3 -s  '+o+' '+n+' '+p,TD)
      script.write(' ~/debdelta/xdelta30e/xdelta3 -d -s '+o+' '+p+' '+n+' ; rm '+p+'\n')
d1081 3
@


1.104
log
@fixed stupid grave bug, command 'debdelta' was failing on
   append_info(delta,info,T)
@
text
@d127 1
a127 1
RCS_VERSION="$Id: debdelta,v 1.103 2006/09/07 18:47:01 debdev Exp debdev $"
d145 1
a145 1
                 ('help','info','needsold','dir=','no-act','alt=','avoid=','delta-algo=','max-percent=','clean-deltas','clean-alt','no-md5') )
d152 1
a152 1
    elif o == '-d' : DEBUG += 1
@


1.103
log
@strip / from TMPDIR

correct Predictor so that it really save stats

debdelta-upgrade : close and reopen connection on some errors
@
text
@d127 1
a127 1
RCS_VERSION="$Id: debdelta,v 1.102 2006/07/23 09:15:04 debdev Exp debdev $"
d2104 1
a2104 1
    append_info(delta,info,T)
@


1.102
log
@do_patch() : add ECHO_TEST in script, to test if using 'echo -ne' or 'echo -n'

prepare_for_echo() : shorter quoted strings

apply_prepare_for_echo() :  check that they work

SHELL is currently /bin/bash

do_patch() : use SHELL in invoking PATCH/patch.sh

----

reviewed exception handling, corrected one error
@
text
@d127 1
a127 1
RCS_VERSION="$Id: debdelta,v 1.101 2006/07/12 20:14:36 debdev Exp debdev $"
d199 1
a199 1
TMPDIR = os.getenv('TMPDIR') or '/tmp'
d2119 2
d2125 1
a2125 1
      self.dir='/var/lib/debdelta'
d2127 2
a2128 2
      self.dir=os.path.expanduser('~/.debdelta')
    s=os.path.join(self.dir,'upgrade.db')
d2132 1
a2132 1
    self.upgrade_stats=shelve.open(s)
d2134 1
a2134 1
    s=os.path.join(self.dir,'packages_stats.db')
d2136 1
a2136 3
    if not os.path.exists(s) and DEBUG == 0 :
      self.package_stats = None
    else:
d2140 1
a2140 1
      self.package_stats=shelve.open(s)
d2153 1
a2153 1
    if self.package_stats:
d2331 1
a2331 1
  import httplib
d2334 1
d2337 2
a2338 1
  def conn_by_url(url):
d2341 4
d2346 2
d2350 12
a2437 1
      conn=conn_by_url(uri)
d2443 2
d2447 1
a2447 1
      except httplib.HTTPException,e:
d2449 1
a2473 1
      conn=conn_by_url(uri)
d2493 2
d2497 14
a2510 3
      except httplib.HTTPException,e:
        puke( 'Connection error: ',e)
        return e
d2543 1
a2543 1
        if a + 0.5 < time.time() :
d2553 5
a2557 1
      sys.stderr.write("Downloaded:  %s \n" % os.path.basename(uri) )
d2560 1
a2560 1
      return  conn_time , (j+len_downloaded)
a2605 1
      deltas_conn=conn_by_url(delta_uri_base)
d2628 1
a2628 1
      if  isinstance(r, httplib.HTTPException) :
d2676 2
a2677 4
      if r == None or r == False:
        print 'Disappeared ?? ',uri
      elif isinstance(r, httplib.HTTPException) :
        if VERBOSE : print ' You may wish to rerun, to get also: ',uri
d2717 2
a2718 1
    http_conns[i].close()
@


1.101
log
@class Predictor()  : to predict patching time (still unused :-);
 and to store statistics of debdelta_upgrade


this is 'debdelta' package 0.16 in Debian
@
text
@d127 1
a127 1
RCS_VERSION="$Id: debdelta,v 1.100 2006/07/12 15:10:13 debdev Exp debdev $"
d332 1
d334 1
a334 1
ALLOWED = '<>()[]{}.,;:!_-+/ abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789'
d336 18
a353 1
def prepare_for_echo(s):
d355 2
a356 3
  while s:
    a=s[0]
    s=s[1:]
d359 6
d366 3
a368 1
      r += "\\" + ( '0000' +oct(ord(a)))[-4:]
d371 41
d451 1
a451 1
def die(s=None):
a453 1

d484 2
a485 2
  if VERBOSE or e == '':    print s,e,str(typ),str(value)
  else: print s,e
d815 1
a815 1
  system('/bin/sh -e '+a+' PATCH/patch.sh', TD)
d1336 1
d1571 1
a1571 1
      script.write("echo -n -e '"+ s +"'\"$FTH\" >> OLD/mega_cat\n")
d1626 1
a1626 1
    script.write("echo -n -e '"+ s +"' >> NEW.file\n")
d1701 1
a1701 1
      die('internal error')
d1987 1
a1987 2
          print 'KeyboardInterrupt'
          raise SystemExit(2)
d1989 1
a1989 1
          puke( " *** Error while creating delta  "+delta+": ")
d2018 1
a2018 2
            print 'KeyboardInterrupt'
            raise SystemExit(2)
d2020 1
a2020 1
            puke(" *** Error while testing delta  "+delta+": ")
d2059 2
d2065 1
a2065 1
        puke( " Unexpected error: " )
d2080 4
a2083 2
  except DebDeltaError,s:
    puke( 'Failed: ',s)
a2084 5
  except KeyboardInterrupt:
    raise SystemExit(0)
  except:
    puke( 'Failed: ',s)
    raise SystemExit(3)
d2094 2
a2098 2
  except KeyboardInterrupt:
    raise SystemExit(0)
d2100 5
a2104 3
    raise
  (delta, percent, elaps, info) = r
  append_info(delta,info,T)
d2109 4
a2112 2
  except e:
    puke( 'debdeltas failed: ',e)
d2114 1
a2114 1

d2123 1
a2123 1
      self.dir=os.path.expanduser('/var/lib/debdelta')
a2198 8

def delta_upgrade():
  try:
    delta_upgrade_()
  except:
    puke('Failed: ')
    raise SystemExit(2)
  raise SystemExit(0)
d2713 9
a2721 1
  delta_upgrade()
@


1.100
log
@append_info() do_patch()  do_delta() delta_upgrade() :
 wrap up so that they create the /tmp dir, and clean up on return or
 exception


DebDeltaError: define self.args
@
text
@d127 1
a127 1
RCS_VERSION="$Id: debdelta,v 1.99 2006/07/09 17:02:02 debdev Exp debdev $"
d2051 18
a2068 6
class predictor:
  def __init__():
    s=os.path.expanduser('~/.debdelta/upgrade.db')
    if not os.path.exists(s) and DEBUG < 2 :
      self.upgrade_stats=None
      self.patch_time_predictor=patch_time_predictor_simple
d2073 1
a2073 3
      import shelve
      self.upgrade_stats=shelve.open(s)
      self.patch_time_predictor=patch_time_predictor_math
d2075 10
a2084 2
  def patch_time_predictor_math(p,t=None):
    "Predicts time to patch. If t is given, adapts."
d2086 32
a2117 5
    n=p['NEW/Package']
    d=copy(p)
    d['LocalDeltaTime']=t
    upgrade_stats['Package:'+n]=d
    #
d2124 1
a2124 1
      return patch_time_predictor_simple(p,t)
d2126 4
a2129 15
    if s not in upgrade_stats:
      r=1
      if 'ServerBogomips' in p :
        r=   float(p['ServerBogomips']) / BOGOMIPS
      upgrade_stats[s]={ 'PatchSpeedRatio' : r }
    r=upgrade_stats[s]['PatchSpeedRatio']
    if t:
      nr =  0.8 * r + 0.2 * (  t / ut )
      a=upgrade_stats[s]
      a['PatchSpeedRatio'] = nr
      upgrade_stats[s]=a
      #upgrade_stats.sync()
      #if VERBOSE > 1 :
      #  print ' Upstream ',ut,'PatchSpeedRatio from ',r,' to ',nr
      #  print upgrade_stats[s]['PatchSpeedRatio']
d2215 1
d2217 1
a2217 5
  def patch_time_predictor_simple(p,t=None):
    if 'ServerBogomips' in p and 'PatchTime' in p:
      return (float(p[ 'PatchTime']) / BOGOMIPS * float(p['ServerBogomips']) )
    else:
      return None
d2258 5
a2262 2
              p=patch_time_predictor_simple(p)
              if p and VERBOSE > 1 : print '   (Predicted %.3f sec )'  % p
@


1.99
log
@added some predictor code

other minor changes
@
text
@d127 1
a127 1
RCS_VERSION="$Id: debdelta,v 1.98 2006/07/09 08:42:36 debdev Exp debdev $"
d274 1
a274 1
def append_info(delta,info,TD):
d276 2
a277 1
  infofile=open(TD+'/PATCH/info','w')
d281 2
a282 1
  system(['ar','rSi','0',delta, 'info'],  TD+'/PATCH')
d371 2
d374 1
a374 1
    self.__str = s
d377 7
a383 1
    return self.__str
d560 13
a572 1
def do_patch(delta,olddeb,newdeb,TD, info=None, diversions=None):
d604 1
d784 11
d796 1
a796 1
def do_delta(olddeb,newdeb,delta,TD):
a1911 1
        T=tempo()          
d1913 1
a1913 1
          ret=do_delta(old['File'],new['File'], delta, T)
a1914 2
          if os.path.exists(delta):
            os.unlink(delta)
a1920 3
          if os.path.exists(delta):
            os.unlink(delta)
          rmtree(T)
d1925 1
a1925 3
          if os.path.exists(delta):
            os.unlink(delta)
        
a1926 1
          rmtree(T)
a1938 1
            rmtree(T)
a1941 2
          rmtree(T)
          T=tempo()
d1944 1
a1944 1
            pret=do_patch(delta,old['File'],None ,T, info=info_delta)
d1949 1
a1949 1
              p.close()              
a1952 1
            rmtree(T)
a1960 1
            rmtree(T)
d1965 1
a1965 2
        append_info(delta,info_delta,T)
        rmtree(T)
a2011 1
  T=tempo()
d2013 1
a2013 1
    do_patch(abspath(argv[0]), abspath(argv[1]), newdeb ,T)
d2015 1
a2015 4
    print 'Failed: ',str(s)
    if newdeb and os.path.exists(newdeb):
      os.unlink(newdeb)
    rmtree(T)
a2017 3
    if newdeb and os.path.exists(newdeb):
      os.unlink(newdeb)
    rmtree(T)
d2020 2
a2021 4
    if newdeb and os.path.exists(newdeb):
      os.unlink(newdeb)
    rmtree(T)
    raise
d2027 1
a2027 2
    
  T=tempo()
d2030 1
a2030 1
    r = do_delta(abspath(argv[0]), abspath(argv[1]), delta ,T)  
d2032 1
a2032 4
    print 'Failed: ',str(s)
    rmtree(T)
    if os.path.exists(delta):
      os.unlink(delta)
d2035 1
a2035 2
    if os.path.exists(delta):
      os.unlink(delta)
a2036 2
    if os.path.exists(delta):
      os.unlink(delta)
a2039 1
  rmtree(T)
d2044 2
a2045 2
  except DebDeltaError,s:
    print 'Failed: ',str(s)
a2098 1
    
d2100 8
d2187 1
a2187 1
    
a2211 1
          T=tempo()
d2213 1
a2213 1
            ret=do_patch(delta,'/',newdeb ,T, diversions=diversions)
a2214 4
            if name in params_of_delta :
              p= params_of_delta[name]
              p=patch_time_predictor_simple(p)
              if p and VERBOSE > 1 : print '   (Predicted %.3f sec )'  % p
a2215 2
            if os.path.exists(newdeb):
              os.unlink(newdeb)
a2216 1
            rmtree(T)
a2219 2
            if os.path.exists(newdeb):
              os.unlink(newdeb)
a2222 2
            if os.path.exists(newdeb):
              os.unlink(newdeb)
d2225 4
a2231 2
          if os.path.exists(T):
            rmtree(T)
@


1.98
log
@ OLD/Size and NEW/Size are the sizes of the .debs (in bytes) ;
 are  saved in .debdelta , and used to check for tmp space and for
 patch results

changed scan_control()
@
text
@d127 1
a127 1
RCS_VERSION="$Id: debdelta,v 1.97 2006/07/09 08:21:46 debdev Exp debdev $"
d2048 49
d2128 1
a2128 1
  cache.upgrade()
d2427 1
a2427 1
      sys.stderr.write("Downloaded:  %s\n" % os.path.basename(uri) )
d2525 1
a2525 1

d2563 1
d2586 2
@


1.97
log
@do_delta() : never divide control.tar in chunks .. otherwise the first file in the ar will not be '0'
@
text
@d127 1
a127 1
RCS_VERSION="$Id: debdelta,v 1.96 2006/07/09 07:22:34 debdev Exp debdev $"
d257 1
a257 1
def scan_control(p,params,prefix=None,info=None):
d268 4
a271 3
      i=a.index(':')
      assert(a[i:i+2] == ': ')
      params[prefix+a[:i]] = a[i+2:]
d515 2
d521 1
a521 1
      instsize += int(params['NEW/Size'])
d527 1
a527 1
  if free and free < ( instsize * 1024  ) :
d529 1
a529 1
        ( int(free/1024) , TMPDIR, instsize )
d594 5
d607 1
a607 1
      scan_control(p,dpkg_params,'OLD')
d616 1
a616 1
        if a[:3] == 'OLD' and a != 'OLD/Installed-Size':
d730 6
d775 2
a776 1

a803 1
  params={}
d811 1
a811 1
      scan_control(p,params,o,s)
d817 4
a820 1

d1704 1
a1704 1
      scan_control(p,info_by_file[f])
@


1.96
log
@added some auxiliary routines to get infor from delta
 get_info_slow() get_info_fast() get_info() info_2_db()

then use the info:
  patch_check_tmp_space()
to avoid downloading deltas that cannot be then applied

a first rudimentary
  patch_time_predictor_simple()
@
text
@d127 1
a127 1
RCS_VERSION="$Id: debdelta,v 1.95 2006/07/07 20:42:36 debdev Exp debdev $"
d591 1
a591 1
  if  DEBUG:
d1227 1
a1227 1
  def delta_tar(old_filename,new_filename,CWD,skip=(),old_md5={},new_md5={}):
d1432 1
a1432 1
      if (a >=  max_chunk_size * chunk_discount) or \
d1434 1
a1434 1
         (a>0 and (a+newtarinfo.size) >= max_chunk_size * chunk_discount ):
d1492 1
d1558 2
a1559 1
      delta_tar(o,n,'CONTROL',skip)
@


1.95
log
@do_patch() :
 -  some auxiliary routines were separated to make code more readable
do_patch() and debdelta-upgrade :
 - do not use 'dpkg -L' that is too slow, scan diversions once and reuse it
@
text
@d82 1
a82 1
from types import StringType, FunctionType, TupleType, ListType
d127 1
a127 1
RCS_VERSION="$Id: debdelta,v 1.94 2006/07/07 20:24:25 debdev Exp debdev $"
d414 2
d424 5
a428 1
def print_delta_info(delta,TD):
d435 39
a473 4
  info=_scan_delta_info_(TD)
  for s in info:
      print ' info: ',s
      
d495 35
d545 2
d578 1
a578 7
  params={}
  for s in info:
    if ':' in s:
      i=s.index(':')  
      params[s[:i]] = s[i+2:]
    else:
      params[s] = True
d580 3
a582 10
  if 'NEW/Installed-Size' in params and 'OLD/Installed-Size' in params:
    free=freespace(TD)
    if olddeb == '/':
      instsize=int(params['NEW/Installed-Size'])
    else:
      instsize=int(params['NEW/Installed-Size'])+int(params['OLD/Installed-Size'])
    instsize +=  2**14
    if free and free < ( instsize * 1024  ) :
      raise DebDeltaError(' Not enough disk space (%dkB) for applying delta (needs %dkB).' % \
          ( int(free/1024) , instsize ), True )
a1949 1
    T=tempo()
d1953 4
a1956 1
        print_delta_info(delta,T)
a1958 1
        rmtree(T)
a1961 1
        rmtree(T)
a1962 1
    rmtree(T)
d2103 12
a2114 1
      
d2139 4
d2423 4
a2426 1
        print 'Already here: ',abs_delta_name
d2445 11
d2469 8
a2476 1
      
@


1.94
log
@no_delta is now a list of (uri, newdeb)
@
text
@d127 1
a127 1
RCS_VERSION="$Id: debdelta,v 1.93 2006/07/07 12:04:27 debdev Exp debdev $"
d454 16
a469 1
def do_patch(delta,olddeb,newdeb,TD, info=None):
d550 72
d634 1
a634 24
        s=[]
        p=os.popen('env -i dpkg -L '+pa)
        a=p.readline()
        while a:
          a=de_n(a)
          #support diversions
          if a[:26] == 'package diverts others to:':
            continue
          if s and a[:11] == 'diverted by' or  a[:20] == 'locally diverted to:':
            orig,divert=s.pop()            
            i = a.index(':')
            divert = a[i+2:]
            s.append( (orig,divert) )
          else:
            s.append( (a,a) )
          a=p.readline()
        p.close()        
        for orig,divert in s:          
          if os.path.isfile(divert) and not os.path.islink(divert) :            
            a=make_parents(TD+'/OLD/DATA'+orig)
            if VERBOSE > 3 : print '   symlinking ',divert,' to ',a
            os.symlink(divert, a)
          else:
            if VERBOSE > 3 : print '    not symlinking ',divert,' to ',orig
d637 1
a637 15
        def chmod_add(n,m):
          om=S_IMODE(os.stat(n)[ST_MODE])
          nm=om | m
          if nm != om:
            if VERBOSE > 1 : print ' Performing chmod ',n,oct(om),oct(nm)
            os.chmod(n,nm)
        for (dirpath, dirnames, filenames) in os.walk(TD+'OLD/DATA'):
          chmod_add(dirpath,  S_IRUSR | S_IWUSR| S_IXUSR  )
          for i in filenames:
            i=os.path.join(dirpath,i)
            if os.path.isfile(i):
              chmod_add(i,  S_IRUSR |  S_IWUSR )
          for i in dirnames:
            i=os.path.join(dirpath,i)
            chmod_add(i,  S_IRUSR | S_IWUSR| S_IXUSR  )
d1838 1
a1838 1
            pret=do_patch(delta,old['File'],None ,T, info_delta)
d1998 1
d2062 1
a2062 1
            ret=do_patch(delta,'/',newdeb ,T)
@


1.93
log
@revert a patch that should not have been there
@
text
@d127 1
a127 1
RCS_VERSION="$Id: debdelta,v 1.92 2006/07/07 20:12:56 debdev Exp debdev $"
d2023 1
a2023 1
            no_delta.append(deb_uri)
d2028 1
a2028 1
            no_delta.append(deb_uri)
d2275 1
d2303 1
a2303 1
        no_delta.append(deb_uri)
d2343 1
a2343 1
      no_delta.append(deb_uri)
d2357 2
a2358 2
      uri = no_delta.pop()
      r=download_uri(uri , DEB_DIR+'/'+os.path.basename(uri), debs_down_time, debs_down_size )
@


1.92
log
@always use
  newdeb=os.path.basename(deb_uri)
and not this and other times
  newdeb=p.name+'_'+version_mangle(p.candidateVersion)+'_'+arch+'.deb'
@
text
@d127 1
a127 1
RCS_VERSION="$Id: debdelta,v 1.91 2006/07/07 20:08:16 debdev Exp debdev $"
d2302 1
a2302 1
        no_delta.append( (deb_uri, newdeb) )
@


1.91
log
@puke() is the standard reporting of exceptions

httplib.HTTPException are caught (hopefully)
@
text
@d127 1
a127 1
RCS_VERSION="$Id: debdelta,v 1.90 2006/07/07 20:03:14 debdev Exp debdev $"
d2258 2
a2259 1
      newdeb=p.name+'_'+version_mangle(p.candidateVersion)+'_'+arch+'.deb'
@


1.90
log
@delta_uri_from_config() : bug!  was not working
@
text
@d127 1
a127 1
RCS_VERSION="$Id: debdelta,v 1.89 2006/07/07 12:41:53 debdev Exp $"
d406 6
d1762 1
a1762 4
          (typ, value, trace)=sys.exc_info()
          print " *** Error while creating delta  ",delta,": ",str(typ),str(value)
          if DEBUG :
            print traceback.print_tb(trace)
d1801 1
a1801 4
            (typ, value, trace)=sys.exc_info()
            print " *** Error while testing delta  ",delta,": ",str(typ),str(value)
            if DEBUG > 1:
              print traceback.print_tb(trace)
d1845 1
a1845 1
        print " Unexpected error:",  sys.exc_info()[0]
d2025 1
a2025 4
            (typ, value, trace)=sys.exc_info()
            print " *** Error while applying delta for ",name,": ",str(typ),str(value)
            if DEBUG > 1:
              print traceback.print_tb(trace)
d2133 6
a2138 2
      conn.request("GET", urllib.quote(uri_p[2]),headers=re)
      r = conn.getresponse()
d2148 1
a2148 1
        return False, False
d2181 6
a2186 2
      conn.request("GET", urllib.quote(uri_p[2]),headers=re)
      r = conn.getresponse()
d2294 14
a2307 8
      r,tempname = download_1k_uri(uri,abs_delta_name)
      #print '1K down? ',r and r.status,uri
      if r:
        if r.status == 206:
          a,b,l = _parse_ContentRange(r)
        else:
          l=int(r.getheader('content-length'))
        available_deltas.append( (l, p.name, uri, abs_delta_name , newdeb, deb_uri, tempname  ) )
d2309 2
a2310 1
        no_delta.append(deb_uri)
d2326 1
a2326 1
      if r == None:
d2328 3
d2357 3
@


1.89
log
@do_patch() :  newdeb may be None, cannot do os.path.basename(newdeb)
@
text
@d127 1
a127 1
RCS_VERSION="$Id: debdelta,v 1.88 2006/07/07 12:04:27 debdev Exp debdev $"
d2063 1
d2065 4
a2068 1
        if a not in opt or dictio[a] != config.get( s, a) :
d2070 2
a2071 1
      return  config.get( s, 'delta_uri' )
d2073 1
a2073 1
      print 'Warning: sources.conf does not provide a server for', repr(dictio)
@


1.88
log
@properly report problems with sources.conf
@
text
@d127 1
a127 1
RCS_VERSION="$Id: debdelta,v 1.87 2006/07/07 12:01:56 debdev Exp debdev $"
d603 1
a603 1
      if VERBOSE > 1 : print ' verifying MD5  for ',os.path.basename(newdeb)
d605 1
a605 1
    else: print ' Warning! no MD5 was verified for ',os.path.basename(newdeb)
@


1.87
log
@correctly get KeyboardInterrup and exit
@
text
@d127 1
a127 1
RCS_VERSION="$Id: debdelta,v 1.86 2006/07/07 11:59:07 debdev Exp debdev $"
d2061 1
a2061 1
        print 'Error!! config file section ',s,'does not contain delta_uri'
d2068 1
a2068 1
      print 'Warning: no configured source for', repr(dictio)
@


1.86
log
@do_patch() : warn if MD5 was not verified (when requested)
@
text
@d127 1
a127 1
RCS_VERSION="$Id: debdelta,v 1.85 2006/07/07 11:57:01 debdev Exp debdev $"
d1749 6
d1793 4
d2013 6
@


1.85
log
@rename   check_diff()  to   check_is_delta()
@
text
@d127 1
a127 1
RCS_VERSION="$Id: debdelta,v 1.84 2006/07/07 11:56:06 debdev Exp debdev $"
d601 3
a603 2
  if DO_MD5 and 'NEW/MD5sum' in params:
      if VERBOSE > 1 : print '  verifying MD5  for ',os.path.basename(newdeb or delta)
d605 1
@


1.84
log
@do_patch() : newdeb may be None
@
text
@d127 1
a127 1
RCS_VERSION="$Id: debdelta,v 1.83 2006/07/07 11:55:35 debdev Exp debdev $"
d398 1
a398 1
def check_diff(f):
d460 1
a460 1
  check_diff(delta)
d1826 1
a1826 1
        check_diff(delta)
@


1.83
log
@TMPDIR is /tmp or env TMPDIR

now temporary directories are of the form  ${TMPDIR}/debdeltaXXXXXX
@
text
@d127 1
a127 1
RCS_VERSION="$Id: debdelta,v 1.82 2006/07/06 16:57:36 debdev Exp $"
d602 1
a602 1
      if VERBOSE > 1 : print ' verifying MD5  for ',os.path.basename(newdeb)
@


1.82
log
@'return KeyboardInterrupt' ?? is 'raise KeyboardInterrupt'
@
text
@d127 1
a127 1
RCS_VERSION="$Id: debdelta,v 1.81 2006/07/06 16:52:03 debdev Exp debdev $"
d199 2
a209 3
    t=os.getenv('TMPDIR')
    if t == None:
      t='/tmp'
d211 1
a211 1
    if a[ : len(t)+4 ] != t+'/tmp' :
d358 1
a358 9
  TD = abspath(tempfile.mkdtemp())
  t=os.getenv('TMPDIR')
  if t == None:
    t='/tmp'
  #this is fascist but still I do not trust my code
  # and if this fails, then __wrap_ fails as well
  if TD[ : len(t) ] != t :
    raise DebDeltaError; ('Sorry I do not like the temp dir "%s"' % TD)
  #
d382 1
a382 4
  t=os.getenv('TMPDIR')
  if t == None:
    t='/tmp'
  if VERBOSE and TD[: (len(t)+4) ] != t+'/tmp' :
@


1.81
log
@DEBUG is now 0

md5 generation and verification is disabled by new option  --no-md5
@
text
@d127 1
a127 1
RCS_VERSION="$Id: debdelta,v 1.80 2006/06/30 14:20:40 debdev Exp debdev $"
d398 1
a398 1
    return KeyboardInterrupt
@


1.80
log
@debdeltas : ignore (due to disk full) corrupted .debs
 and remove empty ones

debdeltas : forgot a 'old=info_pack[l]' in --clean-alt
@
text
@d9 1
d30 1
d50 1
d66 3
a68 1
  -k      keep temporary files
a70 3
## currently this is always true:
## -d      debug : add md5sums, check that  versions do match

d112 1
a112 1
DEBUG   = 1
d121 2
d127 1
a127 1
RCS_VERSION="$Id: debdelta,v 1.79 2006/06/30 14:13:12 debdev Exp debdev $"
d145 1
a145 1
                 ('help','info','needsold','dir=','no-act','alt=','avoid=','delta-algo=','max-percent=','clean-deltas','clean-alt') )
d155 1
d505 2
a506 1
    if free and free < ( instsize * 1024 + 2**23 + MAXMEMORY / 6 ) :
d518 1
d609 1
d611 1
d613 2
a614 2
  if DEBUG and 'NEW/MD5sum' in params:
      if VERBOSE > 1 : print ' verifying MD5 ', params['NEW/MD5sum']
d631 2
a632 2
      print ' Patching done, time: %.2fsec, speed: %dkB/sec %s' % \
            (elaps,(debsize / 1024 /  (elaps+.001)),a)
d693 1
a693 1
  if DEBUG:
d1007 1
a1007 1
    if DEBUG > 2 :  script_md5_check_file(o)
d1481 2
a1482 1
  if DEBUG > 1 and newdeb_md5sum :
d1484 1
a1484 3



d1762 1
a1762 1
          if DEBUG>1:
d1784 1
a1784 1
        if DEBUG > 1:
d1800 1
a1800 1
            if DEBUG>1:
d2021 1
a2021 1
            if DEBUG>1:
d2205 1
a2205 1
          sys.stderr.write("%d%% (%4s/s) for ...%s \r" % \
d2208 1
a2208 1
                            uri[-50:]))
d2213 1
a2213 1
      sys.stderr.write("Downloaded: ...%s \n" % uri[-60:])
d2275 1
a2275 1
        print 'Present ',uri,r.status
@


1.79
log
@debdelta-upgrade :
 download_1k_uri()  downloads a small piece of each .debdelta
 then they are sorted by size
 then download_uri()  downloads the rest of  each .debdelta
 (FIXME test_uri is unused now)
@
text
@d123 1
a123 1
RCS_VERSION="$Id$"
d1554 10
d1808 1
@


1.78
log
@debdelta-upgrade : use conf files
 '/etc/debdelta/sources.conf' '~/.debdelta/sources.conf'
 to deduce the source of deltas

debdelta-upgrade :
 use an unique  cache of http connections; changed test_uri() and download_uri()

debdelta-upgrade : since python-apt is only recommended, bail out with a
 nice error if it is missing

Use 'raise SystemExit' instead of sys.exit .

This was uploaded into Debian , package version 0.15.
@
text
@d76 1
a76 1
from stat    import ST_SIZE, ST_MODE, S_IMODE, S_IRUSR, S_IWUSR, S_IXUSR 
d123 4
a2018 3
  threads=[]
  threads.append(thread.start_new_thread(thread_do_patch  , (qout,threads,no_delta, thread_returns) ) )

d2046 29
d2080 1
a2080 5
      a='Url'
      if uri[-9:] == '.debdelta':
        a='Debdelta'
      s=uri[-60:]
      conn.request("HEAD", urllib.quote(uri_p[2]))
d2082 1
a2085 1
        if VERBOSE > 2: print a,' is present: ',s
d2087 1
a2087 1
      if not VERBOSE: return None        
d2089 1
a2089 1
        conn.request("HEAD", urllib.quote(uri+'-too-big'))
d2094 19
a2112 4
          print a,' is too big: ',s
          return None
      if r.status == 404:
        print a,'is not present: ',s
d2114 15
a2128 2
        print a,'is not available (',repr(r.status), r.reason,'): ', s
      return None
d2135 16
a2150 2
      #should implement Content-Range
      conn.request("GET", urllib.quote(uri_p[2]))
d2152 2
a2153 2
      if r.status != 200:
        if VERBOSE: print 'Not present: ...',uri
d2157 3
a2159 1
      length=r.length
d2162 1
a2162 1
      if free and (free + 2**14 ) < length  :
d2164 2
d2167 10
a2176 2

      out=open(outnametemp,'w')
d2178 4
a2181 4
      conn_time-=a      
      j=0
      s=r.read(min(1024,r.length))
      while s and j < length:
d2187 1
a2187 1
                           (100*j / length,
d2190 1
a2190 1
        s=r.read(min(1024,r.length))
d2196 1
d2199 2
a2200 1

d2204 4
d2252 12
a2263 4
      #download delta
      if not os.path.exists(DEB_DIR+'/'+delta_name):
        if VERBOSE:
          r=test_uri(uri)
d2265 24
a2288 8
          r=True
        if r:
          r=download_uri(uri, DEB_DIR+'/'+delta_name,deltas_down_time,deltas_down_size)
        if r == None:
          no_delta.append(deb_uri)
        else:
          deltas_down_time = r[0]
          deltas_down_size = r[1]
d2291 1
a2291 1
      if os.path.exists(DEB_DIR+'/'+delta_name):
d2293 1
a2293 1
        c=pickle.dumps(  (p.name, DEB_DIR+'/'+delta_name  ,newdeb, deb_uri ) )
d2295 2
@


1.77
log
@bugs corrected in 'debdeltas' :
 do not use variable named 'type'
 do not use 'info' for two different variables
@
text
@a85 3

DELTA_URL="http://tonelli.sns.it/mirror/debian-deltas"

d131 1
a131 1
    sys.exit(0)
d140 1
a140 1
      sys.exit(2)
d155 1
a155 1
        sys.exit(1)
d163 1
a163 1
        sys.exit(3) 
d169 1
a169 1
        sys.exit(3)
d174 1
a174 1
        sys.exit(3)
d179 1
a179 1
        sys.exit(3)
d182 1
a182 1
      sys.exit(0)
d185 1
a185 1
      sys.exit(1)
d367 1
a367 1
class DebDeltaError:
d1811 1
a1811 1
      sys.exit(1)
d1820 1
a1820 1
        sys.exit(1)
d1824 1
a1824 1
        sys.exit(1)
d1826 1
a1826 1
    sys.exit(0)
d1830 1
a1830 1
    sys.exit(1)
d1845 1
a1845 1
    sys.exit(2)
d1850 1
a1850 1
    sys.exit(0)
d1860 1
a1860 1
    sys.exit(1)
d1871 1
a1871 1
    sys.exit(2)
d1888 1
a1888 1
    sys.exit(2)
d1896 1
a1896 1
  import  thread , pickle, urllib, fcntl, atexit, signal
d1898 5
d1905 13
a1917 2

  import  apt, apt_pkg
d1925 1
d1956 1
a1956 1
    sys.exit(1)
a1959 1
  time.sleep(3)
d2021 29
a2049 7
  a=urlparse(DELTA_URL)
  assert(a[0] == 'http')
  deltas_conn=httplib.HTTPConnection(a[1])
  delta_http_base=a[2]

  ###################################### download_uri
  def test_uri(conn,uri):
d2054 1
a2054 1
      conn.request("HEAD", urllib.quote(uri))
d2075 5
a2079 2
  ###################################### test_uri
  def download_uri(conn,uri,outname,conn_time,len_downloaded):
d2082 1
a2082 1
      conn.request("GET", urllib.quote(uri))
d2093 1
a2093 1
        print 'Not enough disk space do download: ',os.path.basename(uri)
d2123 1
a2123 3
    if p.isInstalled and  p.markedUpgrade \
           and p.candidateOrigin[0].origin == 'Debian':

d2133 1
a2133 1
          break      
d2142 1
d2147 11
a2157 2
      
      newdeb = DEB_DIR+'/'+newdeb
d2164 2
a2165 2
      uri=delta_http_base+'/'+os.path.dirname(deb_path)+'/'+delta_name

d2169 1
a2169 1
          r=test_uri(deltas_conn,uri)
d2173 1
a2173 1
          r=download_uri(deltas_conn,uri, DEB_DIR+'/'+delta_name,deltas_down_time,deltas_down_size)
a2185 2
  deltas_conn.close()

d2194 6
a2199 12
  deb_conns={}
  if threads and no_delta:
    if VERBOSE > 1 :
      print ' Downloading deltas done, downloading debs while waiting for patching thread.'
    while threads and no_delta:
      a = no_delta.pop()
      a=urlparse(a)
      assert(a[0] == 'http')
      if a[1] not in deb_conns:        
        deb_conns[a[1]] = httplib.HTTPConnection(a[1])
      r=download_uri(deb_conns[a[1]], a[2] , \
                   DEB_DIR+'/'+os.path.basename(a[2]),debs_down_time, debs_down_size )
a2202 5
    for i in deb_conns:
      deb_conns[i].close()
    
  if VERBOSE > 1 : print ' Downloading done, waiting for patching thread. '
  while threads:
d2204 4
a2207 1

d2343 1
a2343 1
  sys.exit(0)
@


1.76
log
@debdeltas:  better tracebacks in case of errors
 do not exit in case of errors (so, go on creating other deltas)
@
text
@d383 1
a383 1
  if type(a) != type('') :
d1064 1
a1064 1
    if type(oo) == type(''):
d1066 1
a1066 1
    if type(nn) == type(''):
d1521 1
a1521 1
  if AVOID and type(AVOID) == type(''):
d1650 2
a1651 2
    info=info_by_pack_arch[ (pa,ar) ]
    info.sort(order_by_version)
d1653 1
a1653 1
    versions = [ o['Version'] for o in info ]
d1655 1
a1655 1
    versions_not_alt = [ o['Version'] for o in info if o['Label'] != "ALT" ]
d1665 1
a1665 1
    how_many= len( info  )
d1673 1
a1673 1
      new=info[newest]
d1699 1
a1699 1
        old=info[l]
d1741 2
a1742 2
          (type, value, trace)=sys.exc_info()
          print " *** Error while creating delta  ",delta,": ",str(type),str(value)
d1752 1
a1752 1
        (delta_, percent, elaps, info) = ret
d1754 2
a1755 2
        info.append('ServerID: '+HOSTID)
        info.append('ServerBogomips: '+str(BOGOMIPS))
d1770 1
a1770 1
            pret=do_patch(delta,old['File'],None ,T, info)
d1779 2
a1780 2
            (type, value, trace)=sys.exc_info()
            print " *** Error while testing delta  ",delta,": ",str(type),str(value)
d1791 2
a1792 2
          info.append('PatchTime: %.2f' % p_elaps)
        append_info(delta,info,T)
a1796 1
        old=info[l]
d1983 2
a1984 2
            (type, value, trace)=sys.exc_info()
            print " *** Error while applying delta for ",name,": ",str(type),str(value)
@


1.75
log
@debdelta-upgrade: better error message when locking
@
text
@d1101 1
a1101 1

a1739 2
            rmtree(T)
            continue
d1741 4
d1747 1
a1747 3
          rmtree(T)
          raise

a1777 2
              rmtree(T)
              continue
d1779 4
a1782 1
            print " Unexpected error while testing delta:", sys.exc_info()[0]
d1785 2
d1788 2
a1789 1
            raise
d1985 1
d1987 1
a1987 3
              print " Error while applying delta for ",name,": ",str(type),str(value),traceback.print_tb(trace)
            else:
              print " Error while applying delta for ",name,": ",str(type),str(value)
@


1.74
log
@debdelta-upgrade : save in /var/cache/apt/archives if root,
 (and lock it as APT does)
@
text
@a1922 3
    #open("/var/lib/dpkg/lock", O_RDWR|O_CREAT|O_TRUNC, 0640) = 4
    #fcntl64(4, F_SETFD, FD_CLOEXEC)         = 0
    #fcntl64(4, F_SETLK, {type=F_WRLCK, whence=SEEK_SET, start=0, len=0}) = 0
d1926 1
d1932 2
a1933 3
    if s.errno == 11:
      if DEB_DIR == '/var/cache/apt/archives' :
        print 'Could not lock dir: ',DEB_DIR,  ' (is APT running?)'
d1935 4
a1938 1
      print 'Could not lock dir: ',DEB_DIR, str(s)
d1942 2
a1943 1
  
@


1.73
log
@debdelta : do not complain 'Hmmm... there is a md5 but not a file' for
 conf files
@
text
@d55 3
a57 1
--dir DIR   directory where to save results (default: /tmp/archive)
d1896 1
a1896 1
  import  thread , pickle, urllib
d1910 4
a1913 1
    DEB_DIR='/tmp/archives'
d1916 1
a1916 1
  if not os.path.exists(DEB_DIR):    
d1918 24
a1941 1

a1945 1
  print 'Recreated debs are saved in ',DEB_DIR
d2039 2
d2054 2
a2055 1
      out=open(outname+'.temp','w')
d2074 1
a2074 1
      os.rename(outname+'.temp',outname)
@


1.72
log
@do_patch : print also newdeb (as it did before)
@
text
@d1195 1
a1195 4
        if o not in oldnames:
          #would you believe? 'sql-ledger' contains MD5 for files it does not ship...
          if VERBOSE: print 'Hmmm... there is a md5 but not a file: ',o
        elif o not in skip:
d1197 3
@


1.71
log
@do_patch() : forgot to mkdir OLD/CONTROL
@
text
@d616 5
a620 2
      print ' Patching done, time: %.2fsec, speed: %dkB/sec' % \
            (elaps,(debsize / 1024 /  (elaps+.001)))
@


1.70
log
@debpatch was refactored a bit; now "debpatch --info" works again ;
morover do_patch() will complain if it finds a parameter that is unknown
@
text
@d72 1
a72 1
import sys , os , tempfile , string ,getopt , tarfile , shutil , time, md5
d585 8
a592 6
        if olddeb == '/':      
            p=params['OLD/Package']
            for  b in dpkg_keeps_controls :
                a='/var/lib/dpkg/info/' + p +'.'+b
                if os.path.exists(a ):
                    os.symlink(a,TD+'OLD/CONTROL/'+b)
d1949 5
a1953 1
            print " Error while applying delta for ",name,":",sys.exc_info()
@


1.69
log
@debdeltas: one stupid abspath was damaging the --dir /...// behaviour
@
text
@d412 40
d466 1
a466 5
  if INFO:
    system('ar x  '+delta+' patch.sh patch.sh.gz patch.sh.bz2 2> /dev/null', \
           TD+'/PATCH')
  else:    
    if olddeb != '/':
d468 4
a471 5
    if  newdeb and os.path.exists(newdeb) :
      os.rename(newdeb,newdeb+'~')
    system('ar xo '+delta,  TD+'/PATCH')
    
  os.symlink(minigzip,TD+'minigzip')
d473 1
a473 4
  if os.path.exists(TD+'PATCH/patch.sh.gz'):
    system('gunzip PATCH/patch.sh.gz',TD)
  elif os.path.exists(TD+'PATCH/patch.sh.bz2'):
    system('bunzip2 PATCH/patch.sh.bz2',TD)  
d478 2
d482 1
a482 17
    if os.path.isfile(TD+'PATCH/info'):
      #new style debdelta, with info file
      p=open(TD+'PATCH/info')
      info=p.read().split('\n')
      p.close()
      if info[-1] == '': info.pop()
    else:
      #old style debdelta, with info in patch.sh
      p=open(TD+'PATCH/patch.sh')
      s=p.readline()
      s=p.readline()
      while s:
        if s[0] == '#' :
          s=de_n(s)
          info.append(s[1:])
        s=p.readline()
      p.close()
a489 2
    if  INFO :
      print ' info: ',s
d501 1
a501 5
  ## really we will apply the patch
  if not INFO:
    #unpack the old control structure, if available
    os.mkdir(TD+'/OLD/CONTROL')
    if olddeb != '/' :
d503 2
d506 3
a508 4
      system('ar p '+TD+'OLD.file control.tar.gz | tar -x -z -p -f - -C '+TD+'OLD/CONTROL',\
             TD)
    #then we check for the conformance
    if  DEBUG:
d531 3
a533 3
    ###see into parameters: the patch may need extra info
    #as a whole unpack of 'ar' of the old deb
    if 'unpack-old' in params:
d537 1
a537 2

    if 'needs-old' in params and olddeb == '/':
d539 1
a539 2

    if 'old-data-tree' in params :
d584 14
d599 1
a599 13
    if 'old-control-tree' in params and olddeb == '/':      
      p=params['OLD/Package']
      for  b in dpkg_keeps_controls :
        a='/var/lib/dpkg/info/' + p +'.'+b
        if os.path.exists(a ):
          os.symlink(a,TD+'OLD/CONTROL/'+b)

    ##then , really execute the patch
    a=''
    if VERBOSE > 3 : a = '-v'
    system('/bin/sh -e '+a+' PATCH/patch.sh', TD)

    if DEBUG and 'NEW/MD5sum' in params:
d603 1
a603 1
    if newdeb:
d606 2
a607 2
    end_sec = time.time()
    elaps=(end_sec - start_sec)
d609 1
a609 1
    if VERBOSE :
d614 3
a616 3
      print ' Patching done, time: %.2fsec, speed: %dkB/sec, result: %s' % \
            (elaps,(debsize / 1024 /  (elaps+.001)), (newdeb))
    return (newdeb,elaps)
d1805 17
a1821 1
  elif len(argv) != 3 :  
d1824 2
a1825 1
  
d1827 3
d1835 1
a1835 1
    if os.path.exists(newdeb):
d1840 1
a1840 1
    if os.path.exists(newdeb):
d1842 2
d1845 1
a1845 1
    if os.path.exists(newdeb):
d1847 1
a1848 1
  rmtree(T)
@


1.68
log
@.debdeltas start with an info file containing all parameters and also
speed info
@
text
@d1524 3
a1526 3
  def scan_deb(f, label):
      assert( os.path.isfile(f) )
      f=abspath(f)
d1537 1
a1537 1
      info_by_file[f]['File'] = f
@


1.67
log
@debdelta: delta_tar() : do not use  MD5 of files that are really not
  shipped (yes it happens)
@
text
@d72 1
a72 1
import sys , os , tempfile , string ,getopt , tarfile , shutil , time
d100 8
d250 1
a250 1
def scan_control(p,params,prefix=None,script=None,stdout = None):
d259 2
a260 2
      if script : script.write('#'+prefix+a+'\n')
      if stdout : stdout.write(' ' + a)
d266 8
d412 1
a412 1
def do_patch(delta,olddeb,newdeb,TD):
a442 2
  #lets scan parameters, to see what it does and what it requires
  params={}
a444 18
  p=open(TD+'PATCH/patch.sh')
  s=p.readline()
  #skip #!/bin/sh
  if s[:2] == '#!' :
    s=p.readline()
  while s :
    if s[0] == '#' :
      s=de_n(s)[1:]
      if VERBOSE > 1 or (VERBOSE and action != 'deltas' and \
                         action != 'delta-upgrade' ) or INFO :
        print ' info: ',s
      if ':' in s:
        i=s.index(':')  
        params[s[:i]] = s[i+2:]
      else:
        params[s] = True
    s=p.readline()
  p.close()
d446 29
d593 3
a600 2
      end_sec = time.time()
      a=(end_sec - start_sec)
d602 2
a603 1
            (a,(debsize / 1024 /  (a+.001)), (newdeb))
d647 1
d654 4
d659 2
a660 8
        s=sys.stdout
        s.write(o+': ')
      else:
        s=None
      p=open(TD+'/'+o+'/CONTROL/control') 
      scan_control(p,params,o,script,s)
      p.close()
      if s: print
d670 1
a670 1
    script.write('#NEW/MD5sum: '+ newdeb_md5sum[:32]+'\n')
d676 1
a676 1
    script.write('#needs-old\n')
d678 6
a683 2
    script.write('#old-data-tree\n')
    script.write('#old-control-tree\n')
d1179 1
a1179 1
          if VERBOSE > 1 : print '  Hmmm... there is a md5 but not a file: ',o
d1454 2
d1458 1
a1469 2

  deltasize = os.stat(delta)[ST_SIZE]
d1471 1
d1473 5
a1477 1
  percent =  deltasize * 100. /  newdebsize 
d1484 1
a1484 1
  return (delta, percent, elaps)
d1711 1
a1711 1
          ret=do_delta(old['File'],new['File'], delta,T)
a1712 1
          #os.chdir(original_cwd)
d1720 2
a1722 1
          #os.chdir(original_cwd)
a1726 2
        #os.chdir(original_cwd)
        rmtree(T)
d1728 10
a1737 6
        if ret and MAX_DELTA_PERCENT:
          deltasize=os.stat(delta)[ST_SIZE]
          if ( ret[1] > (MAX_DELTA_PERCENT+10)  ) or \
            ( ret[1] > (MAX_DELTA_PERCENT+5) and  deltasize >= 150*1024)  or \
            ( ret[1] > (MAX_DELTA_PERCENT-5) and  deltasize >= 1500*1024) or \
            ( ret[1] > (MAX_DELTA_PERCENT-10) and  deltasize >= 15000*1024):
d1742 2
a1743 1
            ret = None
d1745 2
a1746 1
        if DEBUG > 1 and ret :
d1748 1
d1750 1
a1750 1
            do_patch(delta,old['File'],None ,T)
d1758 2
a1761 1
            #os.chdir(original_cwd)
d1766 4
a1769 3
          #os.chdir(original_cwd)
          if os.path.exists(T):
            rmtree(T)
d1823 1
a1823 1
    do_delta(abspath(argv[0]), abspath(argv[1]), delta ,T)  
d1837 2
a1919 1
          #os.chdir(original_cwd)
@


1.66
log
@debdeltas: --clean becomes two options --clean-alt and --clean-deltas

debdeltas: better code
@
text
@d682 1
a682 1
    a=1
a683 1
      a=de_n(f.readline())
d686 1
d1147 4
a1150 1
        if o not in skip:
@


1.65
log
@use     os.path.dirname(f) or '.
since dirname may be empty

print 'Filename in old tar has weird ./ in front'
 instead of  'Weird filename in old tar'   and only with -vvvv

debdeltas: scan also directory where a deb is ;
 distinguish between CMDLINE  debs and non cmdline ;
 create only deltas to newest cmdline deb (from older debs, of course, up -n)
@
text
@d28 1
a28 1
 -n N       how many deltas to produce for each deb package (default 1)
d31 2
a32 1
--clean     delete deltas if newer deb is not in archive
d93 5
a110 1
CLEAN   = False
d112 3
d130 1
a130 1
                 ('help','info','needsold','dir=','no-act','alt=','avoid=','delta-algo=','max-percent=','clean') )
d140 2
a141 1
    elif o == '--clean' : CLEAN = True
d1501 1
d1503 4
d1565 1
d1576 1
a1576 1
      if CLEAN:
d1582 1
a1582 1
      if CLEAN:
d1593 1
d1595 5
a1599 1
    if CLEAN and (pa,ar) in old_deltas_by_pack_arch :
d1601 1
a1601 1
        if n_d not in versions:
d1606 1
a1606 1
    how_many= len( info  )    
a1608 1
      
d1612 6
a1617 6
    while how_many > 0 :
      new=info[how_many - 1]
      if pa in avoid_pack and ( avoid_pack[pa]['Version'] == new['Version']) :
        if VERBOSE > 1 :   print '(Due to new version) avoid: ', new['File']        
      elif new['Label'] != 'CMDLINE' :
        if VERBOSE > 1 :   print 'Newest version deb was not in cmdline: ', new['File']
d1620 10
a1629 1
      how_many -= 1
d1631 4
a1634 1
    if how_many <= 1 :
d1636 3
a1638 3
    
    l = how_many - 1
    while (l>0) and (l >= how_many - N_DELTAS):
a1640 4

        if pa in avoid_pack and ( avoid_pack[pa]['Version'] == old['Version']):
          if VERBOSE > 1 :     print '(Due to old version) avoid: ', new['File']
          continue
d1644 1
a1644 8
        
        newdebsize=os.path.getsize(new['File'])
        #very small packages cannot be effectively delta-ed
        if newdebsize <= 4 * 1024 :
          #this actually affects 1 every ~60 packages in the archives
          if VERBOSE > 1:     print '  Skip , too small: ', new['File']
          break
        
a1648 1
        deltadirname=delta_dirname(new['File'],DIR)
a1663 5
        free=freespace(deltadirname)
        if free and (free < (newdebsize /2 + 2**15)) :
          if VERBOSE : print 'Not enough disk space for storing ',delta
          break

d1724 11
@


1.64
log
@tested bdelta (=deltup) and  diffball
@
text
@d1089 2
a1090 2
      if VERBOSE > 2 and oldname != de_bar(oldname):
        print ' Weird filename in old tar: ' , oldname 
d1120 2
a1121 2
      if VERBOSE > 2 and newname != de_bar(newname):
        print ' Weird filename in new tar: ' , newname 
d1464 1
a1464 1
  def scan_deb_dir(f,pack_filter=None):
d1474 1
a1474 1
            scan_deb( dt )
d1476 1
a1476 1
  def scan_deb(f):
d1478 1
d1480 4
d1492 1
d1533 1
a1533 1
      f=os.path.dirname(f)
d1556 3
a1558 1
      scan_deb(f)
d1560 1
a1560 1
        scan_deb_dir(delta_dirname(f,ALT), __name_filter__(f) )
d1564 1
a1564 1
      scan_deb_dir(f)
d1566 1
a1566 1
        scan_deb_dir(delta_dirname(f,ALT))
d1571 1
a1571 2

      
d1577 1
d1585 1
a1585 1
      
d1590 18
a1607 5
    if how_many > 1 :
      info.sort(order_by_version)
      l = how_many - 1
      while (l>0) and (l >= how_many - N_DELTAS):
        #os.chdir(original_cwd)
a1609 1
        new=info[how_many - 1]
d1614 3
a1616 4

        if pa in avoid_pack and ( avoid_pack[pa]['Version'] == new['Version']) :
          if VERBOSE > 1 :     print '(Due to new version) avoid: ', new['File']
          break        
@


1.63
log
@debdeltas and debdelta-upgrade : added option --no-act

debdeltas : smarter scanning of --alt dir (faster)
 corrected bug (if abspath() of --dir, the // syntax cannot work!)
@
text
@d736 1
a736 1
    #I   tested bdiff
a737 1
    #It is extremely slow!! and it performs much worse than other choices
d740 2
a741 3
      script.write('~/debdelta/bdiff-1.0.5/bdiff -p '+o+' '+p+' '+n+' ; rm '+p+'\n')
    #
    #I tested zdelta
a742 1
    #it is outperformed by xdelta
d746 11
a756 2
    #
    #I am also testing rdiff
d761 2
a762 2
    #xdelta3 IS buggy and crashes on some inputs
    #xdelta3 crashes!
@


1.62
log
@use abspath also in delta_dirname
@
text
@d106 1
a106 1

d122 1
a122 1
                 ('help','info','needsold','dir=','alt=','avoid=','delta-algo=','max-percent=','clean') )
d131 1
d155 1
a155 1
      DIR = abspath(v)
d160 1
a160 1
      ALT = abspath(v)
d1435 1
a1435 1
def do_deltas(argv):
a1453 4
  debs=[]
  for i in argv:
    debs.append(i)

d1458 12
d1471 2
a1472 7
      if os.path.isdir(f):
        if f not in deb_dir_visited:
          deb_dir_visited.append(f)
          for d in os.listdir(f):
            d=os.path.join(f,d)
            if os.path.isfile(d) and d[-4:] == '.deb' :
              scan_deb( d )
d1488 10
d1499 1
a1499 8
    if os.path.isdir(f):
      if f not in old_deltas_dir_visited:
        old_deltas_dir_visited.append(f)
        for d in os.listdir(f):
          d=os.path.join(f,d)
          if os.path.isfile(d):
            scan_delta( d )
      return
d1526 1
a1526 1
        return make_parents(a)
d1532 7
d1545 2
a1546 2
      if ALT:
        scan_deb(delta_dirname(f,ALT))
d1548 1
a1548 1
        scan_delta(delta_dirname(f,DIR))
d1550 1
a1550 1
      scan_deb(f)
d1552 1
a1552 1
        scan_deb(delta_dirname(f,ALT))
d1554 1
a1554 1
        scan_delta(delta_dirname(f,DIR))
a1565 1

d1570 1
a1570 1
            os.unlink(f_d)
d1605 1
a1605 1
        make_parents(deltadirname+'/')
d1624 4
d1798 1
a1798 2
        (name, delta , newdeb, deb_uri) = pickle.loads(c)      
        if VERBOSE>=2 : print ' Now patching for: ',name
d1800 4
a1803 1
        if True:
@


1.61
log
@--dir and --alt are abspath-ed
@
text
@d1523 1
a1523 1
      return f
@


1.60
log
@debdelta: delta_tar(): only look for correspondence between files with same
 extension (faster)
@
text
@d154 1
a154 1
      DIR = v
d159 1
a159 1
      ALT = v
@


1.59
log
@debdelta(s) -vv : print time  lost in finding correspondences
@
text
@d943 2
a944 1
  def files_similarity_score__(oo,nn):
a946 2
    oo=copy(oo)
    nn=copy(nn)
a947 3
    penalty=0
    if oo.pop() != nn.pop() :
      penalty=0.2
d973 10
a982 1
    return penalty + (l +len(oo) + len(nn)) * 2.0 / float(ln+lo)
d1136 4
a1139 1
      oldnames_premangle[o]=file_similarity_premangle(o)
d1162 2
a1163 1
      if oldname == None:
d1166 3
a1168 2
        np=file_similarity_premangle(newname)
        for o in oldnames:
d1170 1
a1170 1
          s=files_similarity_score__(oldnames_premangle[o],np) + abs(float(l - nl))/float(l+nl)
@


1.58
log
@when printing times, use two digits (that is, centiseconds)
@
text
@d1031 3
d1118 3
d1122 1
d1174 4
a1177 1

d1416 1
a1416 1
    print ' delta time: %.2f sec, speed: %dkB /sec, (%s time: %.2fsec speed  %dkB /sec)' %  \
d1418 1
a1418 1
           USE_DELTA_ALGO,bsdiff_time, bsdiff_datasize / 1024. / (bsdiff_time + 0.001) )
@


1.57
log
@debdelta option --max-percent , useful when testing

option --delta-algo can be 'xdelta' or 'xdelta-bzip' (xdelta with bzip compression)
@
text
@d565 1
a565 1
      print ' Patching done, time: %dsec, speed: %dkB/sec, result: %s' % \
d1388 6
a1393 4
  if  (patchsize > newdebsize / 10 and patchsize > 512 ) or patchsize > 4*1024:
    v=''
    if VERBOSE > 1 : print '  patch.sh is quite large: using bzip2 ' ; v='-v'
    system('bzip2 -9  '+v+'  PATCH/patch.sh', TD)
d1396 1
a1396 3
    v=''
    if VERBOSE > 1 : print '  patch.sh is small: using gzip ' ; v='-v'
    system('gzip -9 -n '+v+' PATCH/patch.sh', TD)  
d1406 1
a1406 1
    print ' delta time: %.1f sec, speed: %dkB /sec, (%s time: %.1fsec speed  %dkB /sec)' %  \
@


1.56
log
@debdelta: better protection against 'disk out of space'  errors
@
text
@d122 1
a122 1
                 ('help','info','needsold','dir=','alt=','avoid=','delta-algo=','clean') )
d134 1
d769 2
a770 2
    elif algo == 'xdelta' :
      system('xdelta delta -n -0 -m'+str(int(MAXMEMORY/1024))+'k '+o+' '+n+' '+p,TD)
d774 3
@


1.55
log
@debdelta-upgrade: prints better statistics (in particular with -v )
@
text
@d445 1
a445 1
    if free and free < ( instsize * 1024 + 2**23) :
d589 1
a589 1
  if free and free < newdebsize * 2:
d648 1
a648 1
    if free and free < instsize * 1024  :
d1043 4
a1046 1
        mega_cat.write(a)
d1178 4
a1181 1
        of.write(s)
@


1.54
log
@--delta-algo : option to choose binary delta system (to ease my testings)
@
text
@a1735 1
  len_newdebs=0
d1738 1
d1740 1
a1740 1
  def thread_do_patch(qout,threads,no_delta):
d1742 2
d1753 2
a1754 1
        if True: 
d1770 1
d1776 1
d1779 2
a1780 1
      return len_newdebs
d1783 1
a1783 4
  threads.append(thread.start_new_thread(thread_do_patch  , (qout,threads,no_delta) ) )

  len_downloaded=0
  conn_time=0
d1857 4
a1860 1
  
d1904 1
a1904 1
          r=download_uri(deltas_conn,uri, DEB_DIR+'/'+delta_name,conn_time,len_downloaded)
d1908 2
a1909 2
          conn_time = r[0]
          len_downloaded = r[1]
a1915 2
        # compute virtual speed . FIXME this is incorrect if patching fails!
        len_newdebs += int( dpkg_params['Size'])
d1925 2
d1936 6
a1941 3
        deb_conns[a[1]] = httplib.HTTPConnection(a[1])      
      download_uri(deb_conns[a[1]], a[2] , \
                   DEB_DIR+'/'+os.path.basename(a[2]),conn_time,len_downloaded )
d1950 21
a1970 6
  if 1 or VERBOSE:
    if conn_time:
      print 'Delta-upgrade download time %dsec speed %s/sec' %\
          (int(conn_time), SizeToStr(len_downloaded / conn_time))
      print '              total time: %dsec; virtual speed: %s/sec.' %  \
          (elaps, SizeToStr(len_newdebs / elaps))
@


1.53
log
@debdeltas: bug! was not scanning cmdline directories anymore

debdelta: code to test bdiff
@
text
@d94 2
a95 4
#seems that 'xdelta' is buggy on 64bit and different-endian machines
USE_XDELTA  = False
#xdelta3 IS buggy and crashes on some inputs
USE_XDELTA3 = False
d122 1
a122 1
                 ('help','info','needsold','dir=','alt=','avoid=','clean') )
d133 1
d732 15
a746 7
      
  def delta_files(o,n):
    " compute delta of two files , and prepare the script consequently"
    nsize = os.path.getsize(TD+n)
    osize = os.path.getsize(TD+o)
    if VERBOSE > 1 : print '  compute delta for %s (%dkB) and %s (%dkB)' % \
       (o,osize/1024,n,nsize/1024)
a747 13
    pp=a_numb_file.next()
    p = 'PATCH/'+pp
    tim = -time.time()

    if DEBUG > 2 :  script_md5_check_file(o)


    #I am also testing bdiff
    if False:
      system(' ~/bin/bdiff -nooldmd5 -nonewmd5 -d  '+o+' '+n+' '+p,TD)
      script.write(' ~/bin/bdiff -p '+o+' '+p+' '+n+'\n')
      tim += time.time()

d749 1
a749 1
    elif False:
d752 2
a753 6

      script.write('rdiff patch '+o+' '+p+' '+n+'\n')
      script_md5_check_file(n)

      tim += time.time()
      
d756 1
a756 1
    elif USE_XDELTA3:
d758 1
a758 3
      script.write(' ~/debdelta/xdelta30e/xdelta3 -d -s '+o+' '+p+' '+n+'\n')
      tim += time.time()

d763 1
a763 4
    elif True: # not ALLOW_XDELTA or ( osize < (MAXMEMORY / 12)):    
      if osize > ( 1.1 * (MAXMEMORY / 12))  and VERBOSE  :
        print '  Warning, memory usage by bsdiff on the order of %dMb' % (12 * osize / 2**20)
      global bsdiff_time, bsdiff_datasize
d765 2
a766 4
      tim += time.time()
      bsdiff_time += tim
      bsdiff_datasize += nsize
      script.write('bspatch '+o+' '+n+' '+p+'\n')
d768 1
a768 3
    elif False:
      a=''
      if VERBOSE > 2 : print '    fallback on xdelta instead of bsdiff' ; a = '-v'
d770 2
a771 5
      system('bzip2 -9 '+a+' '+p,TD)
      tim += time.time()
      script.write('bunzip2 '+p+'.bz2\n')
      script.write('xdelta patch '+p+' '+o+' '+n+'\n')
      pp += '.bz2'
d774 1
d776 27
a802 1
    script.write('rm '+p+' '+o+'\n')
d809 1
a809 1
    patch_append(pp)
d1396 1
a1396 1
    print ' delta time: %.1f sec, speed: %dkB /sec, (bsdiff time: %.1fsec speed  %dkB /sec)' %  \
d1398 1
a1398 1
           bsdiff_time, bsdiff_datasize / 1024. / (bsdiff_time + 0.001) )
@


1.52
log
@debdeltas: changed some 'break' in 'continue' (otherwise, with -n > 1,
 it may not create some deltas)
@
text
@d747 7
d755 1
a755 1
    if False:
d1433 1
a1433 1
              scan_delta( d )
@


1.51
log
@debdeltas: added  '--alt DIR' option
@
text
@d1532 1
a1532 1
          break        
d1538 1
a1538 1
        newdebsize=os.stat(new['File'])[ST_SIZE]
d1555 2
a1556 1
          break
d1559 1
a1559 1
          break
d1563 1
a1563 1
          break
@


1.50
log
@delta_files() does not unzip files any more

 mega_cat_chunk():
  patching is done in background
  chunk size increases progressively
@
text
@d22 6
a27 2
            if DIR ends in // , recreate directory tree
            (i.e. the dirname of deb_file will be used as well)
d105 1
d124 1
a124 1
                 ('help','info','needsold','dir=','avoid=','clean') )
d158 5
d1417 1
d1420 8
d1449 1
a1449 1
            scan_delta( os.path.join(f,d))
d1469 8
a1476 5
  def delta_dirname(f):
    "compute delta dirname given deb path"
    if DIR:
      if DIR[-2:] == '//' :
        a=DIR+os.path.dirname(f)
d1479 1
a1479 1
        return DIR
d1481 1
a1481 1
      return os.path.dirname(f)
d1489 2
d1492 1
a1492 2
        di=delta_dirname(f)
        scan_delta(di)
d1494 5
a1498 6
      a=None
      for a in  filter( lambda a : a[-4:] == '.deb' ,os.listdir(f) ) :
        scan_deb(os.path.join(f,a))
      if CLEAN and a:
        di=delta_dirname(os.path.join(f,a))
        scan_delta(di)
d1549 1
a1549 1
        deltadirname=delta_dirname(new['File'])
@


1.49
log
@debdelta-upgrade: if a delta fails to apply, then queue the
 corresponding .deb for download
@
text
@d724 1
a724 1
  def delta_files(o,n,unzip_p=True):
d730 1
a730 6
    if unzip_p:
      (o,co) = unzip(o)
      (n,cn) = unzip(n)
    else:
      co=''
      cn=''
a732 7
    ## according to the man page,
    ## bsdiff uses memory equal to 17 times the size of oldfile
    ## but , in my experiments, this number is more like 12
    #free=freespace(TD)
    #if free == None :
    #  free = MAXMEMORY * 16
    #    #this interferes with chhnked  and (osize < (free / 8)) :
d754 4
a757 1
    #bsdiff is sooooo slow!
a780 1
    script_zip(n,cn)
d1005 1
a1005 1
  ##########################################################################
d1144 1
a1144 1
      p = 'PATCH/tmp_new_tar'
d1156 8
a1163 2
      delta_files('OLD/mega_cat',p)
      script.write('cat '+p+' >> '+new_filename+'; rm '+p+'\n')
d1179 3
d1190 3
a1192 3
      if (a >= MAXMEMORY / 12 ) or \
         (a >= MAXMEMORY / 13 and one_old_file ) or \
         (a>0 and (a+newtarinfo.size) >= MAXMEMORY / 12):
d1199 1
a1199 1
          if mega_cat.tell() >= MAXMEMORY / 12:
d1206 1
d1257 1
d1259 2
d1333 2
d1336 2
a1337 1
      script.write('cat '+n+' >> NEW.file ;  rm '+n+'\n')
@


1.48
log
@correction to similarity fun

debpatch: add write permission to files, otherwise 'rm ' in script
will choke
@
text
@d1687 1
d1689 3
d1699 1
a1699 1
  def thread_do_patch(qout,threads):
d1708 1
a1708 1
        (name, delta , newdeb) = pickle.loads(c)      
d1714 1
a1714 1
            if VERBOSE == 0 : print 'Created ',newdeb
d1719 1
d1724 1
d1736 1
a1736 1
  threads.append(thread.start_new_thread(thread_do_patch  , (qout,threads) ) )
a1748 4

  #these are the packages that do not have a delta
  no_delta = []

d1859 1
a1859 1
          no_delta.append(deb_uri   )
d1867 1
a1867 1
        c=pickle.dumps(  (p.name, DEB_DIR+'/'+delta_name  ,newdeb) )
@


1.47
log
@faster similarity engine (difflib is too slow)
@
text
@d524 1
a524 1
              chmod_add(i,  S_IRUSR   )
d937 1
a937 1
      while oo[-1] == nn[-1]:
d941 1
a941 1
      while oo[0] == nn[0]:
@


1.46
log
@correct --help on -M

option --allow-xdelta is no more (will put it in config file)

added some code to test xdelta3 : unusable! it is too buggy!

added some code to test rdiff

script_md5_check_file() to check MD5 of partial files while patching
and
debdelta -ddd will do that

properly remove files in delta_files()

solved bug: md5 correspondence was using old conf files

 all parameters are at the beginning of patch.sh
@
text
@d71 1
d926 1
d928 37
d973 1
@


1.45
log
@show speed (=slowness) of bsdiff
@
text
@d10 1
a10 1
  -M Mb   maximum memory (to decide if using 'bsdiff' or 'xdelta')
d26 1
a26 1
  -M Mb     maximum memory to use for 'bsdiff' or 'xdelta'
d54 1
a54 1
doc_common="""
d90 3
a92 1
ALLOW_XDELTA = False
d118 1
a118 1
                 ('help','info','needsold','allow-xdelta','dir=','avoid=','clean') )
a127 1
    elif o == '--allow-xdelta' :  ALLOW_XDELTA = True
d597 1
a597 12
  
  if DEBUG:
    # compute a MD5 of NEW deb
    p=os.popen('md5sum '+TD+'NEW.file')
    a=p.readline()
    p.read()
    p.close
    new_md5sum=a[:32]
    script.write('#NEW/MD5sum: '+ new_md5sum[:32]+'\n')
  else:
    new_md5sum=None
  
d616 17
d680 10
d745 22
a766 1
    if not ALLOW_XDELTA or ( osize < (MAXMEMORY / 12)):    
d775 2
a776 2
      script.write('rm '+p+' '+o+'\n')
    else:
a783 1
      script.write('rm '+p+' '+o+'\n')
d786 3
d914 1
d980 1
d983 1
d987 1
d1060 2
a1061 1
        reverse_old_md5[old_md5[o]] = o
d1146 3
d1156 4
a1159 1
          _append_("OLD/"+CWD+"/"+one_old_file)
d1219 1
a1219 1

a1224 6
  if NEEDSOLD :
    #this delta needs the old deb 
    script.write('#needs-old\n')
  else:
    script.write('#old-data-tree\n')
    script.write('#old-control-tree\n')
d1304 2
a1305 2
  if DEBUG > 1 and new_md5sum :
    script.write('echo "'+new_md5sum+'  NEW.file" | md5sum -c > /dev/null')
d1417 3
@


1.44
log
@really correct (!) bug that should have been fixed in previous version:
"debdeltas: create dir before checking for freespace in it!"
@
text
@d563 5
a567 1

d708 2
a709 2
    nsize = os.stat(TD+n)[ST_SIZE]
    osize = os.stat(TD+o)[ST_SIZE]
d727 1
d731 1
d733 3
d743 1
d751 1
a751 1
    deltasize = os.stat(TD+p)[ST_SIZE]
d753 2
a754 1
      print '   delta is %3.2f'  % ( deltasize * 100. /  nsize ) , '% of ',n
d1283 3
a1285 2
    print ' delta time: %dsec, speed: %dkB per second ' %  \
          (elaps, newdebsize / 1024 / (elaps+0.001))
d1293 1
d1449 1
a1449 1
        if free and (free < (newdebsize /2 + 1024)) :
d1509 1
@


1.43
log
@debdeltas: create dir before checking for freespace in it!
@
text
@d1421 1
a1421 1
        os.makedirs(deltadirname)
@


1.42
log
@debdeltas: can --clean unusable debdeltas
@
text
@d1420 3
a1422 1
        delta=os.path.join(delta_dirname(new['File']),deltabasename)        
d1435 1
a1435 1
        free=freespace(os.path.dirname(delta))
@


1.41
log
@less verbosity on delta of .gz files.
@
text
@d21 4
a24 4
            (otherwise they go in the dir of the newer deb)
            if DIR ends in // , then the dirname of deb_file
            the  will be used as well
 -n N       how many deltas to produce for each deb (default 1)
d26 2
a27 2
  -M Mb     maximum memory (to decide if using 'bsdiff' or 'xdelta',
            and how much memory to use for 'xdelta' )
d99 1
a99 1

d116 1
a116 1
                 ('help','info','needsold','allow-xdelta','dir=','avoid=') )
d125 1
d305 7
a311 1

d1316 41
d1360 3
d1364 1
d1366 4
a1369 1
        scan_deb(f+'/'+a)
d1372 1
d1378 11
a1388 1
    how_many= len( info_by_pack_arch[ (pa,ar) ] )    
d1390 1
a1390 1
      print '   I see: ',pa,[ o['Version'] for o in info_by_pack_arch[(pa,ar)]]
d1393 1
a1393 1
      info_by_pack_arch[ (pa,ar) ].sort(order_by_version)
d1398 2
a1399 2
        old=info_by_pack_arch[ (pa,ar) ][l]
        new=info_by_pack_arch[ (pa,ar) ][how_many - 1]
d1419 3
a1421 9
        if DIR:
          if DIR[-2:] == '//' :
            a=DIR+os.path.dirname(new['File'])+'/'+deltabasename
            delta=make_parents(a)
          else:
            delta = DIR+'/'+deltabasename
        else:
          delta = os.path.dirname(new['File']) + '/'+deltabasename
          
@


1.40
log
@do_delta(): report only weirdest permissions
@
text
@d665 1
a665 1
      print '  appending ',f,' of size ', a,' to debdelta, %3.2f'  % ( a * 100. /  newdebsize ) , '% of new .deb'
d789 1
d792 3
a794 2
    if ( os.path.getsize(TD+n) - before + 10 ) < 200 :
      if VERBOSE > 1: print '   Not worthwhile gunzipping: ',n
d842 2
a843 2
      if i == pack_level and VERBOSE > 2:
        print '   Warning: wrong guess to re-gzip to equal file: ',gzip_flags,r,n
d845 1
a845 1
      if VERBOSE > 1: print '  Warning: cannot re-gzip to equal file: ',r,n
d851 2
a852 2
    if VERBOSE > 1 :
      print '  ',n,': ',
@


1.39
log
@do_delta(): adjustments to treatment of gzipped files
@
text
@a972 1
      a=("%o" % newtarinfo.mode)
d974 5
a978 3
      if VERBOSE and (( t == '2' and a  != '777' ) or \
                      ((t == '5' or t == '0') and (a not in [ '755' , '644' ]))):
        print ' Weird permission: ',newname,a,repr(newtarinfo.type)
@


1.38
log
@corrected bug!  gunzip cannot be called on symlinks! patching failed
on debs from filesystem!
@
text
@a760 1
      s=oa
d763 1
a763 2
        s+=oa
      #print repr(s)
d773 1
a774 3
    l=10
    oa=of.read(2)
    na=nf.read(2)    
d790 4
d849 2
@


1.37
log
@do_delta(): move parameters in  patch.sh at beginning
@
text
@d847 3
a849 3
    system("gunzip '"+o+"'",TD)
    script.write("gunzip '"+o+"'\n")
    delta_files(o[:-3],p+'.new')
d851 1
a851 1
    script.write("mv "+p+".new '"+o[:-3]+"' ; gzip "+gzip_flags+" '"+o[:-3]+"'\n")
@


1.36
log
@do_delta(): new code to gunzip .gz files in data.tar before delta, and gzip after
@
text
@a736 1
      #redundant ', %3.2f'  % ( deltasize * 100. /  newdebsize ) , '% of new .deb'
d1154 3
a1156 1

d1194 1
a1194 1
    elif  name[:11] == 'control.tar' :
a1195 1
      script.write('#old-control-tree\n')
a1206 1
      script.write('#old-data-tree\n')
@


1.35
log
@do_debdelta(): use a generator to generate successive names for files
@
text
@d736 2
a737 2
      print '   delta is %3.2f'  % ( deltasize * 100. /  nsize ) , '% of ',n,\
            ', %3.2f'  % ( deltasize * 100. /  newdebsize ) , '% of new .deb'
a741 1
    unlink(TD+n)
d743 112
d1061 1
a1061 1
      os.unlink(p)
d1117 2
d1120 7
a1126 1
      mul=len( old_used[oldname]) > 1 #multiple usage
d1140 1
d1225 1
@


1.34
log
@do_debdelta(): delta_tar(): ignore empty files
@
text
@d574 8
a581 4
  #counter for numbered files  FIXME this will not work in threads....
  global deltacount
  deltacount = 0 

d670 1
a670 3
    global deltacount
    deltacount += 1
    pp=str(deltacount)
d707 1
a707 3
    global deltacount
    deltacount += 1
    pp=str(deltacount)
d936 1
a936 3
      global deltacount
      deltacount += 1
      p = 'PATCH/'+ str(deltacount)
d950 1
@


1.33
log
@debdelta_upgrade: thanks to Michael Vogt, now uses APT caches at best
@
text
@d837 2
a838 2
             not oldtarinfo.isreg():
        continue      
d892 3
d999 4
@


1.32
log
@do_delta(): delta_tar(): chunk tar , always use 'bsdiff'
@
text
@a79 1
DEB_URL="http://ftp.debian.org/debian"
a1391 25

  #this is slow but currently python-apt does not provide these info
  f=os.popen('apt-cache dumpavail')
  #f=open('/var/lib/dpkg/available')
  dpkg_avail={}
  parse_dist(f,dpkg_avail)
  f.close()

  ## FIXME : how do I get the URI and/or architecture out of python-apt ??
  def fake_uri(p):
    if p.sourcePackageName[:3] == 'lib':
      b=p.sourcePackageName[:4]
    else:
      b=p.sourcePackageName[0]
    c=p.candidateOrigin[0].component
    # rely on usual pool structure....
    return c+'/'+b+'/'+p.sourcePackageName
  def get_uri_from_dpkg(p):
    ##this sucks... it is soooo slow..., but it works
    dpkg_params={}
    pip=os.popen('env -i dpkg -p '+p.name)
    parse_dist(pip,dpkg_params)
    pip.close()
    assert(dpkg_params['Version'] == p.installedVersion )
    return dpkg_params
a1480 1
      shorturi=uri[-50:]
d1484 3
a1486 1
        if VERBOSE: print 'Not present: ...',s
d1490 4
d1507 1
a1507 1
                            shorturi))
d1512 1
a1512 1
      sys.stderr.write("Downloaded: ...%s \n" % shorturi)
d1520 11
a1530 5
      
      dpkg_params=dpkg_avail[p.name]
      if 'Filename' not in dpkg_params :
        if VERBOSE: print ' I am lost ! apt-cache dumpavail  does not give URI for ',p.name
        continue
a1532 1
      deb_uri=dpkg_params['Filename']
d1543 1
a1543 1

d1551 1
a1551 1
      uri=delta_http_base+'/'+os.path.dirname(deb_uri)+'/'+delta_name
d1573 1
a1573 4
        if 'Size' in dpkg_params:
          len_newdebs += int( dpkg_params['Size'])
        elif VERBOSE:
          print ' (Size of %s will not be counted in virtual speed ...)' % p.name
d1583 2
a1584 1
  if  threads and no_delta:
a1586 4
    a=urlparse(DEB_URL)
    assert(a[0] == 'http')
    debs_conn=httplib.HTTPConnection(a[1])
    debs_http_base=a[2]
d1588 9
a1596 5
      a=no_delta.pop()
      download_uri(debs_conn, debs_http_base+'/'+a , \
                   DEB_DIR+'/'+os.path.basename(a),conn_time,len_downloaded )
    
    debs_conn.close()
@


1.31
log
@do_patch(): add permission to files in old data.tar ('angband' does
 not have permissions for the user to read files)
@
text
@d82 1
d90 3
d117 1
a117 1
                                   ('help','info','needsold','dir=','avoid=') )
a121 1

d126 1
d128 7
a134 1
    elif o == '-M' :    MAXMEMORY = 1024 * 1024 * int(v)
d424 5
a428 2
    instsize=int(params['NEW/Installed-Size']) + int(params['OLD/Installed-Size'])
    if free and free < instsize * 1024  :
a695 1
    if VERBOSE > 1 : print '   compute delta for  ',o,' and ',n
d698 2
d713 7
a719 4
    free=freespace(TD)
    if free == None :
      free = MAXMEMORY * 16
    if ( osize < (MAXMEMORY / 12)) and (osize < (free / 8)) :
d851 3
a853 1
    
d855 1
d930 20
a949 1
    if VERBOSE > 2 : print '   scanning ',n
d952 13
a964 9
    for oldname in oldnames :
      if (oldname in skip) or (oldname in old_used ) :
        continue
      if VERBOSE > 2 : print '   provide also old file ', oldname
      #mega_cat.write(fake_tar_2nd)
      #script.write("echo -n -e \"$FTH\" >> OLD/mega_cat\n")
      _append_( "OLD/"+CWD+"/"+oldname )
      if mega_cat.tell() > 2**21 :
        break
d967 16
d1007 1
d1012 4
a1015 1
      delta_files('OLD/mega_cat',new_filename)
d1050 1
a1050 1
    if VERBOSE > 1: print '  studying ' , name , ' of len ' , newsize
@


1.30
log
@do_delta(): less disk space needed for computing delta
@
text
@d69 1
a69 1
from stat    import ST_SIZE
d426 1
a426 1
      system('ar p '+TD+'OLD.file control.tar.gz | tar -x -z -f - -C '+TD+'OLD/CONTROL',\
d491 16
a506 2
        system('ar p '+TD+'OLD.file data.tar.gz | tar -x -z -f - -C '+TD+'OLD/DATA', TD)

@


1.29
log
@debpatch: support for local diversions
@
text
@d71 3
a574 1
        print o
d576 1
d589 1
a589 1
    instsize=int(params['NEW/Installed-Size']) + 2 * int(params['OLD/Installed-Size'])
d772 1
d775 1
a787 1
      unlink(TD+w)
d790 1
a792 3
    
    (old_filename,old_filename_ext) = unzip(old_filename,False)
    (new_filename,new_filename_ext) = unzip(new_filename)
d795 7
a801 1
    oldtar = tarfile.open(TD+old_filename, "r")
d809 1
a809 1
      if VERBOSE and oldname != de_bar(oldname):
d816 6
a821 1

d836 1
a836 1
      if VERBOSE and newname != de_bar(newname):
a843 1
    multiple={}
d890 1
a890 3
      if oldname in old_used:
        multiple[oldname]=True
      else:
d897 12
d929 2
a930 3
      mul=multiple.get(oldname)
      if VERBOSE > 2 :  print '   adding reg file: ', oldname, mul
      oldtar.extract(oldtarinfos[oldname],TD+"OLD/"+CWD )
d932 1
a932 8

    #there may be files that have been renamed and edited...
    for oldname in oldnames :
      if (oldname in skip) or (oldname in old_used ) :
        continue
      if VERBOSE > 2 : print '   provide also old file ', oldname
      oldtar.extract(oldtarinfos[oldname],TD+"OLD/"+CWD )
      _append_( "OLD/"+CWD+"/"+oldname )
d935 1
a935 1
    if os.path.exists(TD+'OLD/mega_cat'):#now, always
d937 1
a941 1
    unlink(TD+old_filename)
d1007 5
a1011 2
      system('ar p OLD.file '+name+' >> '+o, TD)
      delta_tar(o,n,'DATA',old_conffiles,old_md5,new_md5)
@


1.28
log
@MAX_DELTA_PERCENT = 70

--needsold in debdeltas

def tempo(): fascist check

class DebDeltaError:  some errors (e.g. not enought disk space) are retriable

do_delta(): first simply scan for md5 ;
 delta_tar(): then in a double pass, use them, otherwise filename
   similarity, AND properly check for multiple use of old files (!)

 def shell_not_allowed(name): encodes my paranoia for shell quoting

 def file_similarity(): compute filename similarity

debdelta_upgrade(): test_uri() before downloading
  show % of download
@
text
@d468 5
a472 3
          if s and a[:11] == 'diverted by':
            orig,divert=s.pop()
            #support diversions
@


1.27
log
@MAX_DELTA_PERCENT = 60

avoid FutureWarning

Readjusted verbosity  :-> .
Now -vvv is really verbose, gives per-file info.

append(s) is renamed to append_NEW_file(s)

fake_tar_header_2nd() and delta_tar() :
  write some part of the tar header into mega_cat :
  really improves compressibility with xdelta, for .deb with many
  small files

delta_tar() : do not use system('cat')

debdelta-upgrade : do not fork , use real threads.

debdelta-upgrade : while the thread is patching,
 also download some .debs for which deltas are not available.
@
text
@d82 1
a82 1
MAX_DELTA_PERCENT = 60
d120 1
a120 1
    elif o == '--needsold' and action == 'delta' :  NEEDSOLD = True
d167 1
d295 8
d312 1
a312 1
  def __init__(self,s):
d314 1
d413 2
a414 2
      die(' Not enough disk space (%dkB) for applying delta (needs %dkB).' % \
          ( int(free/1024) , instsize ) )
d537 1
a537 1
    die('Error: not enough disk space in '+TD)
d543 1
a543 1
  #counter for numbered files
d581 1
a581 36

  # uses MD5 to detect identical files (even when renamed)
  data_identical={}
  if os.path.exists(TD+'/NEW/CONTROL/md5sums') and \
     os.path.exists(TD+'/OLD/CONTROL/md5sums') :
    f=open(TD+'/OLD/CONTROL/md5sums')
    do={}
    a=de_n(f.readline())
    while a:
      m , n = a[:32] ,  de_bar( a[34:] )
      do[ m ] = n
      a=de_n(f.readline())
    f.close()
    f=open(TD+'/NEW/CONTROL/md5sums')
    dn={}
    mul={}
    us={}
    r=None
    a=de_n(f.readline())
    while a:
      m , n = a[:32] ,  de_bar( a[34:] )
      if m in do:
         #sometimes there are multiple occurences of the same file...
         # ( as in    tetex-doc_3.0-18_all.deb    )
         r=do[m]
         if r in us:
           mul[r] = True
         us[ r ] = True
         dn[ n ] = r
      a=de_n(f.readline())
    f.close()
    for n in dn:
      data_identical[ n ] = ( dn[ n ], mul.get( dn[ n ] ) )
    del f,dn,do,a,mul,n,m,r,us

    
d584 1
a584 1
    instsize=int(params['NEW/Installed-Size']) + int(params['OLD/Installed-Size'])
d586 2
a587 2
      die(' Not enough disk space (%dkB) for creating delta (needs %dkB).' % \
          ( int(free/1024) , instsize ) )
d589 2
a590 1
  ## check for conffiles 
d598 27
d711 24
d767 1
a767 1
  def delta_tar(o,n,w,skip=(),renames={}):
d787 23
a809 7
    (o,co) = unzip(o,False)
    (n,cn) = unzip(n)
    oldtar = tarfile.open(TD+o, "r")
    oldnames = oldtar.getnames() #map( de_bar , oldtar.getnames() )
    oldused={}
    if VERBOSE > 2 : print '   scanning ',n
    newtar = tarfile.open(TD+n, "r")
d811 2
a812 2
      name = de_bar( newtarinfo.name )

d817 65
a881 1
        print ' weird permission: ',name,a,repr(newtarinfo.type)
d883 4
a890 2
      #mega_cat.write(newtarinfo.name)
      #script.write("echo -n -e '"+ s +"' >> OLD/mega_cat\n")
d900 4
a903 14
      multiple = False
      if name in renames:
        ( oldname , multiple ) = renames[name]
      else:
        oldname = name

      if   oldname in skip :
        if VERBOSE > 2 : print '   skip using old file ', name
        continue

      if '"' in oldname or "'" in oldname or '\\' in oldname or '`' in oldname :
        #FIXME should use it , by properly quoting for the shell script
        if VERBOSE  : print ' weird chars in old file ', oldname
        continue
d905 4
a908 13
      if oldname in oldnames  :
        oldtarinfo = oldtar.getmember(oldname)        
        assert( oldtarinfo.name == oldname )
        if oldtarinfo.isreg() :
          if VERBOSE > 2 :
            if name in renames : print '   use identical old file ', oldname
            else:  print '   use same name old file ', oldname
          oldused[oldname] = name
          del name
          oldtar.extract(oldname,TD+"OLD/"+w )
          _append_( "OLD/"+w+"/"+oldname , not multiple)
        elif VERBOSE > 2 : print '   not regular in old : ', name
      elif VERBOSE > 2 : print '   not present in old : ', name
d912 5
a916 7
      if (oldname not in oldused) and  (oldname not in skip) :
        oldtarinfo = oldtar.getmember(oldname)
        assert( oldtarinfo.name == oldname )
        if oldtarinfo.isreg() :
          if VERBOSE > 2 : print '   provide also old file ', oldname
          oldtar.extract(oldname,TD+"OLD/"+w )
          _append_( "OLD/"+w+"/"+oldname )
d920 2
a921 2
      rmtree(TD+'/OLD/'+w)
      delta_files('OLD/mega_cat',n)
d923 4
a926 4
      p=verbatim(n)
      script.write('mv '+p+' '+n+ '\n')
    unlink(TD+o)
    script_zip(n,cn)
d992 1
a992 1
      delta_tar(o,n,'DATA',old_conffiles,data_identical)
d1107 1
a1107 1
        new=info_by_pack_arch[ (pa,ar) ][l+1]
d1143 4
a1156 6
        except KeyboardInterrupt:
          #os.chdir(original_cwd)
          if os.path.exists(delta):
            os.unlink(delta)
          rmtree(T)
          raise
d1163 3
d1170 1
d1177 1
a1177 1
          if ( ret[1] > (MAX_DELTA_PERCENT+10) and  deltasize >= 15*1024 ) or \
a1190 6
          except KeyboardInterrupt:
            #os.chdir(original_cwd)
            if os.path.exists(delta):
              os.unlink(delta)
            rmtree(T)
            raise
d1193 5
a1197 3
            #os.chdir(original_cwd)
            if os.path.exists(delta):
              os.unlink(delta)
d1278 1
a1278 1
  import  thread , pickle
a1366 1
  global conn_time
a1377 4
  a=urlparse(DEB_URL)
  assert(a[0] == 'http')
  debs_conn=httplib.HTTPConnection(a[1])
  debs_http_base=a[2]
d1383 30
a1412 4
  def download_uri(conn,uri,outname):
      if VERBOSE > 2: print ' uri ',uri
      conn.connect()
      conn.request("GET", uri)
d1415 1
a1415 3
        if VERBOSE: print '  Uri is not available',uri,repr(r.status), r.reason
        data1 = r.read()
        #conn_time+=time.time()
d1417 2
a1418 1
      ext=string.split(outname,'.')[-1]
a1420 1
      global conn_time
d1423 2
a1424 2
      s=r.read(1024)
      while s:
d1429 2
a1430 2
          sys.stderr.write("%s %s ( %s/sec ) for %s \r" % \
                           (ext,SizeToStr(j),
d1432 2
a1433 2
                            delta_name[:50]))
        s=r.read(1024)
d1435 1
a1435 1
      conn.close()
d1437 1
a1437 1
      sys.stderr.write(" " * 70 + '\r')
d1439 1
a1439 1
      return j
d1457 1
a1457 1
        if VERBOSE: print  'Already downloaded: ',newdeb
d1460 1
a1460 1
      if VERBOSE:
d1474 7
a1480 2
      if not os.path.exists(DEB_DIR+'/'+delta_name):        
        r=download_uri(deltas_conn,uri, DEB_DIR+'/'+delta_name)
d1484 2
a1485 1
          len_downloaded += r
d1492 5
d1498 1
a1498 4
      if 'Size' in dpkg_params:
        len_newdebs += int( dpkg_params['Size'])
      elif VERBOSE:
        print ' (Size of %s will not be counted in virtual speed ...)' % p.name
d1506 14
a1519 5
  while threads and no_delta:
    a=no_delta.pop()
    download_uri(debs_conn, debs_http_base+'/'+a , \
                 DEB_DIR+'/'+os.path.basename(a) )

@


1.26
log
@safer code: rmtree, rmdir and unlink check that they are working in
temp dir

safer code: avoid inserting into shell code filenames with ' or " or \

code does not rely on CWD, so it is thread safe now.

shorter patch.sh, by using CR macro, and bzip

MD5 is now added as a parameter, so it is up to 'debpatch' to check it

corrected buggy 'break' when 'I am lost' in debdeltas
@
text
@d76 3
d82 1
a82 1
MAX_DELTA_PERCENT = 50
d94 2
d150 1
a150 1
    if VERBOSE : print ' would unlink ',a
d152 1
a152 1
    if VERBOSE : print ' would rmdir ',a
d154 1
a154 1
    if VERBOSE : print ' would rm -r ',a
d320 1
a320 1
    print 'Warning system in ',TD,' for ',a
d332 1
a332 1
    die('Error: '+f+ 'does not seem to be a Debian package ')
a359 1
    #os.chdir(TD+'/PATCH')
d388 1
a388 1
      if VERBOSE > 2 or (VERBOSE and action != 'deltas' and \
d474 1
a474 1
            if VERBOSE > 4 : print '    not symlinking ',divert,' to ',orig
d484 1
a484 1
          os.symlink(a,'OLD/CONTROL/'+b)
d488 1
a488 1
    if VERBOSE > 4 : a = '-v'
d492 1
a492 1
      if VERBOSE: print ' veryfing MD5 ', params['NEW/MD5sum']
a551 5
  def append(s):
    'appends some data to NEW.file'
    s=prepare_for_echo(s)
    script.write("echo -n -e '"+ s +"' >> NEW.file\n")

d559 1
a559 1
      if  VERBOSE > 1 :
a570 2
  ## helper sh function for script, for delta_tar()
  script.write('CR () { cat "$@@"  >> OLD/mega_cat ; rm "$@@" ;}\n')
d625 3
a627 2
    if VERBOSE > 3 :
      print '   appending ',f,' of size ', os.stat(TD+'PATCH/'+f)[ST_SIZE]
d636 1
a636 1
    if VERBOSE > 3 : print '   including "',name,'" verbatim in patch'
d659 1
a659 1
  def delta_files(o,n):
d661 1
a661 1
    if VERBOSE > 3 : print '   compute delta for  ',o,' and ',n
d664 6
a669 2
    (o,co) = unzip(o)
    (n,cn) = unzip(n)
d683 1
d685 5
a689 2
      if VERBOSE > 3 : print '   fallback on xdelta instead of bsdiff' 
      system('xdelta delta -n -9 -m'+str(MAXMEMORY)+'M '+o+' '+n+' '+p,TD)
d691 3
a693 2
    ## clean up
    script.write('rm '+p+' '+o+'\n')
d697 3
a699 2
    if VERBOSE > 2 :
      print '  delta is  %3.4f'  % ( deltasize * 100. /  nsize ) , '% of ',n
d705 32
d738 2
d745 2
a746 1
    #helper fun
d748 6
a753 1
      system("cat '"+w+"' >>  OLD/mega_cat", TD)
d765 1
a765 1
    if VERBOSE > 3 : print '   scanning ',n
d769 18
d788 1
a788 1
        if VERBOSE > 4 : print '  not regular in new : ', name
d790 1
d798 1
a798 1
        if VERBOSE > 4 : print '   skip using old file ', name
d801 3
a803 2
      if '"' in oldname or "'" in oldname or '\\' in oldname :
        if VERBOSE > 3 : print '   weird chars in old file ', oldname
d810 3
a812 3
          if VERBOSE > 4 :
            if name in renames : print '  use identical old file ', oldname
            else:  print '  use same name old file ', oldname
d817 2
a818 2
        elif VERBOSE > 4 : print '  not regular in old : ', name
      elif VERBOSE > 4 : print '  not present in old : ', name
d826 1
a826 1
          if VERBOSE > 4 : print ' provide also old file ', oldname
d829 3
a831 2
    
    if os.path.exists(TD+'OLD/mega_cat'):
d841 4
d858 1
a858 1
  append(s)
d869 1
a869 1
    if VERBOSE > 2: print ' studying ' , name , ' of len ' , newsize
d872 1
a872 1
    if VERBOSE > 4: print '  ar line: ',repr(s)
d874 1
a874 1
    append(s)
d884 1
a884 1
      append( p.read(newsize))
d920 1
a920 1
      append(extrachar)
d924 2
a925 2
    if VERBOSE > 2: print '  ar leftover character: ',repr(s)
    append(s)
a927 1
    if VERBOSE > 2 : print '   ',a
d934 3
a936 2
    if VERBOSE > 2 : print '  patch.sh is quite large: using bzip2 '
    system('bzip2 -9  PATCH/patch.sh', TD)
d939 3
a941 1
    system('gzip -9 -n PATCH/patch.sh', TD)  
d951 2
a952 3
    if VERBOSE :
      print '  delta time: %dsec, speed: %dkB per second ' %  \
            (elaps, newdebsize / 1024 / (elaps+0.001))
d960 2
d1009 2
a1010 2
    if VERBOSE>3:
      print ' I see: ',pa,[ o['Version'] for o in info_by_pack_arch[(pa,ar)]]
d1016 1
a1016 1
        os.chdir(original_cwd)
d1022 1
a1022 1
          if VERBOSE :     print '(Due to old version) avoid: ', new['File']
d1026 1
a1026 1
          if VERBOSE :     print '(Due to new version) avoid: ', new['File']
d1031 3
a1033 3
        if newdebsize <= 8 * 1024 :
          #this actually affects 1 every ~30 packages in the archives
          if VERBOSE > 1:     print 'Skip , too small: ', new['File']
d1049 1
a1049 1
          if VERBOSE > 2:     print 'Skip , already exists: ',delta
d1052 1
a1052 1
          if VERBOSE > 2:     print 'Skip , tried and too big: ',delta
d1066 1
a1066 1
          os.chdir(original_cwd)
d1072 1
a1072 1
          os.chdir(original_cwd)
d1078 1
a1078 1
          os.chdir(original_cwd)
d1082 1
a1082 1
        os.chdir(original_cwd)
d1102 1
a1102 1
            os.chdir(original_cwd)
d1109 1
a1109 1
            os.chdir(original_cwd)
d1114 1
a1114 1
            os.chdir(original_cwd)
d1119 1
a1119 1
          os.chdir(original_cwd)
d1190 2
a1193 5
  original_cwd = os.getcwd() 
  import httplib
  conn=httplib.HTTPConnection("tonelli.sns.it")
  delta_http_base='/mirror/debian-deltas'

d1213 2
a1214 1
  f=open('/var/lib/dpkg/available')
d1243 1
a1243 1
  #qoutf=os.fdopen(qout)
d1245 1
a1245 1
      if VERBOSE>=2 : print ' Patching thread started. '
d1253 1
a1253 1
        (name,tmp_delta , newdeb) = pickle.loads(c)      
d1255 1
a1255 3
        # my fault...  do_patch() wants its own CWD!
        pid=os.fork()
        if pid == 0:
d1258 1
a1258 1
            ret=do_patch(tmp_delta,'/',newdeb ,T)
d1268 4
a1271 3
          if os.path.exists(tmp_delta):
            unlink(tmp_delta)
          os.chdir(original_cwd)
a1273 4
          #exit fork
          return
        else:
          os.waitpid(pid,0)
d1275 1
a1275 1
      if VERBOSE>=2 : print ' Patching thread ended , bye bye. '
d1281 1
d1284 52
d1343 1
a1343 1
        if VERBOSE: print ' I am lost ! "dpkg -p ',p.name,'" does not give URI!'
d1368 7
a1374 30
      if VERBOSE > 2: print ' uri ',uri

      conn.connect()
      conn.request("GET", uri)
      r = conn.getresponse()
      if r.status != 200:
        if VERBOSE: print '  delta is not available',repr(r.status), r.reason
        data1 = r.read()
        #conn_time+=time.time()
        continue
      a=time.time()
      conn_time-=a
      (delta_fd, tmp_delta) = tempfile.mkstemp()
      j=0
      s=r.read(1024)
      while s:
        len_downloaded += len(s)
        j+=len(s)
        os.write(delta_fd,s)
        if a + 0.5 < time.time() :
          a=time.time()
          sys.stderr.write(" %s ( %s/sec ) for %s \r" % \
                           (SizeToStr(j),
                            SizeToStr(len_downloaded/(a+conn_time)),\
                            delta_name))
        s=r.read(1024)
      os.close(delta_fd)
      conn.close()
      conn_time+=time.time()
      sys.stderr.write(" " * 70 + '\r')
d1376 5
a1380 3
      #append to queue
      c=pickle.dumps(  (p.name,tmp_delta,newdeb) )
      os.write(qin, c + '\t' )
d1389 10
a1398 1
  if VERBOSE>=2 : print ' Downloading done, waiting for patching thread. '
d1409 4
a1412 2
##################################################### apt method
### still work in progress
d1417 4
a1420 1
      
@


1.25
log
@debdeltas: '-n N' to decide how many deltas to compute for each package.
@
text
@d26 2
a27 1
  -M Mb     maximum memory (to decide if using 'bsdiff' or 'xdelta')
a70 1
from os      import unlink, rmdir
d151 7
a157 1
  def rmtree(a):
d159 1
a159 1
      shutil.rmtree(a)
d161 8
a168 1
      print ' Warning! when trying to remove ',repr(a),'got OSError',repr(str(s))
d173 1
a265 9
def unpack(d,f,T):
  "unpacks 'ar' file f in directory d"
  assert(os.path.exists(f))
  cwd = os.getcwd()
  os.chdir(T+'/'+d)
  system('ar xo '+f)
  os.chdir(cwd)


d308 1
a308 1
def system(a):
d311 6
a316 1
  ret = os.system(a)
d340 4
a343 1
def do_patch(delta,olddeb,newdeb,TD):  
d355 3
a357 2
    os.chdir(TD+'/PATCH')
    system('ar x  '+delta+' patch.sh patch.sh.gz patch.sh.bz2 2> /dev/null')
d363 1
a363 3
    unpack ('PATCH',delta,TD)
  #from here on, we live in the temp dir
  os.chdir(TD)
d365 1
a365 1
  os.symlink(minigzip,'minigzip')
d367 4
a370 4
  if os.path.exists('PATCH/patch.sh.gz'):
    system('gunzip PATCH/patch.sh.gz')
  elif os.path.exists('PATCH/patch.sh.bz2'):
    system('bunzip2 PATCH/patch.sh.bz2')  
d374 1
a374 1
  if not os.path.isfile('PATCH/patch.sh'):
d376 1
a376 1
  p=open('PATCH/patch.sh')
d409 2
a410 1
      system('ar p OLD.file control.tar.gz | tar -x -z -f - -C OLD/CONTROL')
d418 1
a418 1
        p=open('OLD/CONTROL/control')
d472 1
a472 1
        system('ar p OLD.file data.tar.gz | tar -x -z -f - -C OLD/DATA')
d484 7
a490 2
    if VERBOSE >= 4 : a = '-v'
    system('/bin/sh -e '+a+' PATCH/patch.sh')
d492 1
a492 1
      shutil.move('NEW.file',newdeb)
d507 3
d520 1
a520 1

a528 3
  #from here on, we live in the temp dir
  os.chdir(TD)

d534 1
a534 1
  script=open('PATCH/patch.sh','w')
d536 12
a547 2


d556 1
a556 1
      os.mkdir(TD+'/'+o+'/CONTROL')
d558 1
a558 1
      system('ar p '+o+'.file control.tar.gz | tar -x -z -f - -C '+o+'/CONTROL')
d571 4
d628 4
a631 4
    os.chdir(TD+'/PATCH')
    system(['ar','qSc', delta,f])
    unlink(f)
    os.chdir(TD)
d639 1
a639 1
    os.rename(f,p)
d646 1
a646 1
      system('gunzip '+f)
d664 2
a665 2
    nsize = os.stat(n)[ST_SIZE]
    osize = os.stat(o)[ST_SIZE]
d679 1
a679 1
      system('bsdiff  '+o+' '+n+' '+p)
d683 1
a683 1
      system('xdelta delta -n -9 '+o+' '+n+' '+p)
d689 1
a689 1
    deltasize = os.stat(p)[ST_SIZE]
d695 3
a697 5
    unlink(o)
    unlink(n)
    if DEBUG:
      pass #implement MD5

d700 1
a700 1
    if os.path.exists('OLD/mega_cat'):
d702 2
a703 2
      os.unlink('OLD/mega_cat')
    
d705 2
a706 3
      system("cat '"+w+"' >> OLD/mega_cat")
      unlink(w)          
      script.write("cat '"+w+"'  >> OLD/mega_cat\n")
d708 3
a710 1
        script.write("rm '"+w+"'\n")
d714 1
a714 1
    oldtar = tarfile.open(o, "r")
d718 1
a718 1
    newtar = tarfile.open(n, "r")
a726 1
        if VERBOSE > 4 : print '   identical!  ', oldname, name, multiple
d733 4
d742 3
a744 1
          if VERBOSE > 4 : print '  use old file ', oldname
d747 1
a747 1
          oldtar.extract(oldname,"OLD/"+w )
d759 1
a759 1
          oldtar.extract(oldname,"OLD/"+w )
d761 3
a763 3
          
    if os.path.exists('OLD/mega_cat'):
      rmtree('OLD/'+w)
d768 1
a768 1
    unlink(o)
d788 2
a789 2
  ar_list_old= list_ar('OLD.file')
  ar_list_new= list_ar('NEW.file')
d793 1
a793 1
    system('ar p NEW.file '+name+' >> '+n)
d795 1
a795 1
    newsize = os.stat(n)[ST_SIZE]
d810 1
a810 1
      p=open(n)
d813 1
a813 1
      unlink(n)
d818 1
a818 1
      system('ar p OLD.file '+name+' >> '+o)
d821 1
a821 1
      for a in os.listdir('OLD/CONTROL') :
d830 1
a830 1
      system('ar p OLD.file '+name+' >> '+o)
d839 1
a839 1
      system('ar p OLD.file '+name+' >> '+o)
d854 1
a854 7
  if DEBUG:
    # add a MD5 check to script
    p=os.popen('md5sum NEW.file')
    a=p.readline()
    p.read()
    p.close
    a=de_n(a)
d856 1
a856 1
    script.write('echo "'+a+'" | md5sum -c > /dev/null')
d860 2
a861 2
  patchsize = os.stat('PATCH/patch.sh')[ST_SIZE]
  if  patchsize > newdebsize / 5 and patchsize > 512 :
d863 1
a863 1
    system('bzip2 -9  PATCH/patch.sh')
d866 1
a866 1
    system('gzip -9 -n PATCH/patch.sh')  
d1121 3
d1222 2
a1223 1
        break
@


1.24
log
@Rewrite code  delta_tar()  , use  _append_() .

Properly exit from forks!
@
text
@d15 3
a17 2
Usage: debdeltas [ option...  ] deb_files
  Computes all missing deltas for Debian files deb_files
d24 1
d81 2
d104 1
a104 1
    ( opts, argv ) = getopt.getopt(sys.argv[1:], 'vkhdM:' ,
d117 5
d696 1
a696 1
        if VERBOSE >3 : print '   identical!  ', oldname, name, multiple
d701 1
a701 1
        if VERBOSE > 3 : print '   skip using old file ', name
d902 5
a906 3
    l= len( info_by_pack_arch[ (pa,ar) ] )

    if l > 1 :
d908 2
a909 3

      l -= 1
      while l>0:
d914 1
@


1.23
log
@Allow +- 10% on MAX_DELTA_PERCENT , depending on delta size.

debdeltas: '--avoid file' option, to avoid packages from a dist.

debdelta : really skip conf files !

Better verbosity yet.
@
text
@d474 2
a475 1
      print ' patching time: %dsec, speed:  %dkB per second ' % (a,(debsize / 1024 /  (a+.001)))
d661 11
d703 1
a703 5
          system("cat 'OLD/"+w+"/"+oldname+"' >> OLD/mega_cat")
          unlink('OLD/'+w+'/'+oldname)          
          script.write("cat 'OLD/"+w+"/"+oldname+"'  >> OLD/mega_cat\n")
          if not multiple:
            script.write("rm 'OLD/"+w+"/"+oldname+"'\n")
d715 2
a716 4
          system("cat 'OLD/"+w+"/"+oldname+"' >> OLD/mega_cat")
          unlink('OLD/'+w+'/'+oldname)
          script.write("cat 'OLD/"+w+"/"+oldname+"'  >> OLD/mega_cat ; rm 'OLD/"+w+"/"+oldname+"'\n")
    
a719 3
      if os.path.exists('OLD/mega_cat'):
        # if -k is given, still we need to delete it...
        os.unlink('OLD/mega_cat')
d1143 1
a1143 2
            if VERBOSE : print '   for ',name
            else: print 'Created ',newdeb
d1157 2
@


1.22
log
@debdelta' can use MD5 to exploit identical files that were renamed.
This can express the difference between tetex-doc 3.0-17 and 3.0-18
into  260kB , even though all the directory tree was moved around !

This debdelta is in Debian package version 0.8
@
text
@d76 1
d85 1
d100 2
a101 2
      ( opts, argv ) = getopt.getopt(sys.argv[1:], 'vkhdM:' ,
                                     ('help','info','needsold','dir=') )
d114 5
d163 15
a355 1
  s=p.readline()#skip #!/bin/sh
d357 3
d474 1
a474 1
      print ' patching time: %dsec, speed:  %dkB per second ' % (a,(debsize / 1024 /  (a+1)))
d579 1
a579 1
    old_conffiles=p.read().split('\n')
d832 2
a833 2
    print ' deb delta is  %3.1f%% of deb ; that is, %dkB would be saved' \
          % ( percent , (( newdebsize -deltasize ) / 1024) )
d836 1
a836 1
            (elaps, newdebsize / 1024 / (elaps+1))
d850 8
d901 8
a908 1

d911 2
a912 2
        if newdebsize <= 4 * 1024 :
          #this actually affects 1 every 60 packages in the archives
d921 1
a921 1
            a=DIR+'/'+os.path.dirname(new['File'])+'/'+deltabasename
d927 1
d936 1
a936 1
        if free and free < newdebsize /2 + 1024 :
d966 5
a970 1
          if ret[1] > MAX_DELTA_PERCENT and os.stat(delta)[ST_SIZE] >= 4*1024 :
d1069 1
a1069 1
def delta_upgrade(DIR):
d1080 2
d1086 31
a1116 5
    DIR='/tmp/archives'
  if not os.path.exists(DIR):    
    os.mkdir(DIR)

  print 'Recreated debs are saved in ',DIR
d1118 1
d1120 1
a1120 2
  len_deltas=0

d1141 1
a1156 3
        global len_newdebs
        if len_newdebs != None and os.path.exists(newdeb):
          len_newdebs += os.stat(newdeb)[ST_SIZE]
d1159 1
a1159 1
      return
d1164 3
d1170 6
a1175 15

      ## FIXME : how do I get the URI and/or architecture out of python-apt ??
      #if p.sourcePackageName[:3] == 'lib':
      #  b=p.sourcePackageName[:4]
      #else:
      #  b=p.sourcePackageName[0]
      #c=p.candidateOrigin[0].component
      ## rely on usual pool structure....
      #u=c+'/'+b+'/'+p.sourcePackageName
      ##this sucks... it is so slow..., but it works
      dpkg_params={}
      pip=os.popen('env -i dpkg -p '+p.name)
      scan_control(pip,dpkg_params)
      pip.close()
      arch=dpkg_params['Architecture']
d1177 1
a1177 3
      assert(dpkg_params['Version'] == p.installedVersion )


d1179 1
a1179 1
      if os.path.exists(DIR+'/'+newdeb) or \
d1188 1
a1188 1
      newdeb = DIR+'/'+newdeb
d1205 1
d1207 2
d1213 2
d1216 7
a1222 3
        j += 1
        sys.stderr.write(" %5d kB for %s \r" % ( j , delta_name))
        s=r.read(1024)        
d1225 1
d1232 5
d1244 6
a1249 3
  if VERBOSE:
    print 'Delta-upgrade download and patch time: %dsec; virtual speed: %dkB per second.' %  \
          (elaps, len_newdebs / 1024 / (elaps+1)) 
d1255 1
a1255 1
  delta_upgrade(DIR)
@


1.21
log
@delta_upgrade : is a function ;
 " : accepts --dir option
 " : patching queue ending was not managed OK

started preparing delta_tar in do_delta for md5 .
@
text
@d69 1
a69 1
from shutil  import rmtree
d134 6
d185 1
a185 1
  if a[-1] ==  '\n' :
d189 7
d507 37
a543 1
  
d639 2
a640 1
    oldnames = oldtar.getnames()
d644 5
a648 1
      name = newtarinfo.name
d650 2
a651 1
        oldname = renames[name]
d654 2
a655 1
      if  (('/'+name) in  skip ) or ( name in skip ):
d657 5
a661 2
      elif name in oldnames and  newtarinfo.isreg() :
        oldtarinfo = oldtar.getmember(newtarinfo.name)
d663 24
a686 7
          if VERBOSE > 4 : print '  use old file ', name
          oldtar.extract(oldtarinfo.name,"OLD/"+w )
          system("cat 'OLD/"+w+"/"+name+"' >> OLD/mega_cat")
          unlink('OLD/'+w+'/'+name)
          script.write("cat 'OLD/"+w+"/"+name+"'  >> OLD/mega_cat ; rm 'OLD/"+w+"/"+name+"'\n")
      elif VERBOSE > 4 : print '  not diffable from old : ', name

d690 3
d759 1
a759 1
      delta_tar(o,n,'DATA',old_conffiles)
d956 2
a957 1
          rmtree(T)
@


1.20
log
@debdelta-upgrade: use a separare thread (and fork :-( ) for the patching.

Again adjustments to VERBOSE... lets say that -vvv is a reasonable output.

debdelta-upgrade: use 'arch' from 'dpkg -p' when creating package file name.
@
text
@d112 1
a112 1
    elif o == '--dir'  and action == 'deltas' :
d124 2
a125 1

a177 4
#def symlink_w_parents(f,d):
#  d=make_parents(fd)
#  os.symlink(f,d)

a182 1

d585 2
a586 1
  def delta_tar(o,n,w,skip=()):
d595 4
d942 1
d945 1
a945 1
elif action == 'delta-upgrade' :
d967 1
d971 1
a971 1
  def thread_do_patch(qout,qin):
d982 1
a982 1
        # my fault... echo do_patch() wants its own CWD!
d1007 1
a1007 3
      if VERBOSE>=2 : print ' Patching thread loop ended. '
      s=os.write(qin,'\t')
      s=os.read(qout,2)
d1011 2
a1012 1
  thread_patch=thread.start_new_thread(thread_do_patch  ,(qout,qin))
d1083 2
a1084 2
  os.read(qout,1)
  #while thread_patch:    time.sleep(1)
d1092 4
d1097 1
a1097 1
elif  os.path.dirname(sys.argv[0]) == '/usr/lib/apt/methods' :
@


1.19
log
@Allow -M in debdeltas.

Bug: do not remove non-existent file.
@
text
@d429 1
a429 1
    if VERBOSE > 2 : a = '-v'
d944 1
a945 4
  p=os.popen('dpkg --print-architecture')
  arch=de_n(p.read())
  p.close()
  
d950 1
a950 1
  
d966 46
a1015 12
      newdeb=p.name+'_'+version_mangle(p.candidateVersion)+'_'+arch+'.deb'
      if os.path.exists(DIR+'/'+newdeb) or \
             os.path.exists('/var/cache/apt/archives/'+newdeb):
        if VERBOSE: print  'Already downloaded: ',newdeb
        continue

      if VERBOSE:
        print 'Looking for a delta for %s from %s to %s ' % \
              ( p.name, p.installedVersion, p.candidateVersion )

      newdeb = DIR+'/'+newdeb

d1029 2
a1030 2
      a=dpkg_params['Architecture']
      u=dpkg_params['Filename']
d1032 14
d1049 1
a1049 1
                  a+'.debdelta'
d1051 1
a1051 1
      uri=delta_http_base+'/'+os.path.dirname(u)+'/'+delta_name
d1073 11
a1083 11
      T=tempo()
      try:
        ret=do_patch(tmp_delta,'/',newdeb ,T)
        len_newdebs += os.stat(newdeb)[ST_SIZE]
      except DebDeltaError,s:
        print ' Error: applying of delta failed: ',str(s)
        if os.path.exists(newdeb):
          os.unlink(newdeb)
      unlink(tmp_delta)
      os.chdir(original_cwd)
      rmtree(T)
d1087 1
a1087 1
          (elaps, len_newdebs / 1024 / (elaps+1))
@


1.18
log
@On keyboard interrupt, clean up tmp files.

The test for xdelta VS bsdiff was not working (wrong parentheses?).
Moreover, my test shows that bsdiff uses 12 times the memory not 17.

Quote filenames in patch.sh.
@
text
@d110 1
a110 1
    elif o == '-M' and action == 'delta' :    MAXMEMORY = 1024 * 1024 * int(v)
d330 3
a332 1
      if VERBOSE > 1 or (VERBOSE and action != 'deltas' ) or INFO : print ' info: ',s
d1034 2
a1035 1
        os.unlink(newdeb)
@


1.17
log
@Recover upstream uri and architecture from 'dpkg -p' (very slow but it
works).

Adjustments to verbosity.

die() does not print anything; DebDeltaError ships the error message.
@
text
@d559 1
d561 1
d565 1
a565 1
    if osize < MAXMEMORY / 17 and osize * 8 < free  :
d569 1
a569 1
      if VERBOSE > 4 : print '  fallback on xdelta instead of bsdiff' 
d603 1
a603 1
          system('cat OLD/'+w+'/'+name+' >> OLD/mega_cat')
d605 1
a605 1
          script.write('cat OLD/'+w+'/'+name+'  >> OLD/mega_cat ; rm OLD/'+w+'/'+name+'\n')
d792 1
a792 1
          if VERBOSE :     print 'Skip , too small: ', new['File']
d824 4
d856 4
d867 1
d869 3
a871 2
            print " Unexpected error while testing delta:", sys.exc_info()[0]
            os.unlink(delta)
d899 3
d923 3
@


1.16
log
@Implements 'debdelta-upgrade' .

Start writing some kind of APT method

Change ':' to '%3a' in file names.
@
text
@d157 1
a157 1
    if a[:4] in ('Pack','Vers','Arch','Stat','Inst'):
d255 2
a256 2
  def __str__(self,s):
    return __str
d259 1
a259 1
  if s : sys.stderr.write(s+'\n')
d330 1
a330 1
      if VERBOSE or INFO : print ' info: ',s
d432 1
a432 1
    if VERBOSE > 1:
d726 1
a726 1
    if VERBOSE > 1:
d813 1
a813 1
          if VERBOSE : print ' Not enough disk space for',delta
d823 1
a823 1
        except DebDeltaError:
d827 2
a828 1
          print ' Creation of ',delta,' failed.'
d838 1
a838 1
          if ret[1] > MAX_DELTA_PERCENT:
d840 1
a840 1
            if VERBOSE : print ' Error, too big!'
d851 2
a852 2
          except DebDeltaError:
            print ' Error: testing of delta failed: ',delta
d881 2
a882 1
  except DebDeltaError:
d902 2
a903 1
  except DebDeltaError:
d917 2
a918 1
  except DebDeltaError:
d932 1
a932 1
  delta_http_base='/mirror/debian-deltas/pool/'
d946 3
a948 1
  
d964 17
a980 1
      
d983 5
a987 9
                  '_'+ version_mangle(p.candidateVersion)+'_i386.debdelta'
      ## FIXME : how do I get the URI out of python-apt ??
      if p.sourcePackageName[:3] == 'lib':
        b=p.sourcePackageName[:4]
      else:
        b=p.sourcePackageName[0]
      c=p.candidateOrigin[0].component
      ## rely on usual pool structure....
      uri=delta_http_base+c+'/'+b+'/'+p.sourcePackageName +'/'+delta_name
d1011 3
a1013 2
      except DebDeltaError:
        print ' Error: applying of delta failed: ',delta
d1018 4
a1021 2

  
@


1.15
log
@Catch getopt exception.

1/60th of packages is of size less than 4kB ; and those produce "large" deltas.
@
text
@d15 2
a16 2
Usage: debdeltas [ option...  ] debs
  Computes all missing deltas for debs
d21 2
a22 2
            if DIR ends in // , then the dirname of the arguments will be used as well
--search    search in the directory of the above debs for older versions
d27 2
d42 9
a73 16

action=(os.path.basename(sys.argv[0]))[3:]
actions =  ('delta','patch','deltas')
if action not in actions:
  print 'wrong filename: should be "deb" + '+repr(actions)
  sys.exit(0)

__doc__ = doc[action] + doc_common

try: 
  ( opts, argv ) = getopt.getopt(sys.argv[1:], 'vkhdM:' ,
                                 ('help','info','needsold','dir=') )
except getopt.GetoptError,a:
  sys.stderr.write(sys.argv[0] +': '+ str(a)+'\n')
  sys.exit(2)

d85 8
a92 14
for  o , v  in  opts :
  if o == '-v' : VERBOSE += 1
  elif o == '-d' : DEBUG += 1
  elif o == '-k' : KEEP = True
  elif o == '--needsold' and action == 'delta' :  NEEDSOLD = True
  elif o == '-M' and action == 'delta' :    MAXMEMORY = 1024 * 1024 * int(v)
  elif o == '--info' and action == 'patch' : INFO = True
  elif o == '--dir'  and action == 'deltas' :
    DIR = v
    if not os.path.isdir(DIR):
      print 'Error: --dir ',DIR,' does not exist.'
      sys.exit(3)
  elif o ==  '--help' or o ==  '-h':
    print __doc__
a93 3
  else:
    print ' option ',o,'is unknown, try --help'
    sys.exit(1)
d95 29
a123 9
if INFO  :
  if  len(argv) > 1 and VERBOSE :
    print '(printing info - extra arguments are ignored)'
  elif  len(argv) == 0  :
    print ' need a  filename ;  try --help'
    sys.exit(1)
elif action != 'deltas' and len(argv) != 3 :  
  print ' need 3 filenames ;  try --help'
  sys.exit(1)
d234 1
d236 5
d794 2
a795 1
        deltabasename = pa +'_'+  old['Version'] +'_'+ new['Version'] +'_'+ar+'.debdelta'
d839 1
a839 1
            if VERBOSE : print ' Error, too big:',delta
d866 10
d891 5
a895 1
elif action == 'delta' :  
d917 89
d1007 99
a1105 1
    
@


1.14
log
@Corrected some bugs.

Some checks for disk space.
@
text
@d72 6
a77 4
( opts, argv ) = getopt.getopt(sys.argv[1:], 'vkhdM:' ,
                               ('help','info','needsold','dir=') )


d775 3
a777 1
        if newdebsize <= 20 * 1024 :
@


1.13
log
@'debdeltas' to scan archive and create many deltas

'debdelta' deals with dpkg diversions

main operations are now functions

errors are reported using exceptions

reviewed verbosity
@
text
@d78 2
a79 1
MAX_DELTA_PERCENT = 40
d95 5
a99 1
  elif o == '--dir'    and action == 'deltas' : DIR = v
a125 2


d149 1
a149 1
    if a[:3] in ('Pac','Ver','Arc','Sta'):
d160 1
a160 1
  d='/'
a237 1

d239 4
a242 1
  pass
d325 6
d357 1
a357 1
        if a[:3] == 'OLD':
d382 1
d388 1
a388 1
            s.append(orig,divert)
d393 2
a394 2
        for orig,divert in s:
          if os.path.isfile(divert) and not os.path.islink(divert) :
d396 1
d398 2
d482 7
d740 1
a740 1
      info_by_file[f]['File'] = abspath(f)
d772 3
a774 3

        if os.stat(new['File'])[ST_SIZE] < 16 * 1024 :
          if VERBOSE > 2:     print 'Skip , too small: ', new['File']
d781 2
a782 2
            a=os.path.dirname(new['File'])+'/'+deltabasename
            delta=make_parents(a,DIR)
a793 1

d795 1
a795 1
        if free and free < 2 ** 20 :
d804 3
d808 3
a810 3
        except DebDeltaError:
          os.unlink(delta)
          print 'Creation of ',delta,' failed.'
d812 3
a814 1
          os.unlink(delta)
d816 2
a817 1
          rmtree(T)
d822 1
a822 1
            if VERBOSE : print 'Error, too big:',delta
d831 4
d836 2
d839 2
a840 2
            print "Unexpected error:", sys.exc_info()[0]
            print 'Error: applying of delta failed: ',delta
d843 1
a844 1
        
a845 2

  
d854 2
a855 1
    os.unlink(newdeb)
d859 2
a860 1
    os.unlink(newdeb)
d871 2
a872 1
    os.unlink(delta)
d875 2
a876 1
    os.unlink(delta)
@


1.12
log
@Typo.
@
text
@d3 2
a4 1
"""\
d8 20
d29 1
a29 1
  Applies patchin to fromfile and produces  a  reconstructed  version of tofile.
d37 2
a38 6
Options for debdelta:
--needsold  create a patch that can only be used if the old .deb is available
  -M Mb   maximum memory (to decide if using 'bsdiff' or 'xdelta')

Options for debpatch:
 --info  print info on two Debian files, and exists
d40 1
a40 1
Options for both:
d60 1
a60 1
####################################################################
a61 1
start_sec = time.time()
d65 1
a65 1
actions =  ('delta','patch')
d70 2
d73 2
a74 1
                               ('help','info','needsold') )
a75 1
original_cwd = os.getcwd()
d78 2
d85 1
d94 1
d108 1
a108 1
elif len(argv) != 3  or ( len(argv) != 3   ):  
d112 21
a132 1
######################################################################
d138 5
a142 1
def scan_control(p,params,prefix,script=None,stdout = None):
d146 2
a147 2
    if a[:3] in ('Pac','Ver','Arc'):
      if script : script.write('#'+prefix+'/'+a+'\n')
d151 1
a151 1
      params[prefix+'/'+a[:i]] = a[i+2:]
d154 4
a157 2
def symlink_w_parents(f,d):
  s=f.split('/') 
d160 1
a160 1
      d = d + '/' + a
d162 7
a168 4
        os.mkdir(d)      
  d += '/'+s[-1]
  os.symlink(f,d)

d201 1
a201 1
def unpack(d,f):
d205 1
a205 1
  os.chdir(TD+'/'+d)
a222 8
####################################################################
if KEEP:
  def unlink(a):
    if VERBOSE : print ' would unlink ',a
  def rmdir(a):
    if VERBOSE : print ' would rmdir ',a
  def rmtree(a):
    if VERBOSE : print ' would rm -r ',a
a223 4
TD = abspath(tempfile.mkdtemp())
for i in 'OLD','NEW','PATCH' :
  os.mkdir(TD+'/'+i)
if  VERBOSE > 1 or KEEP :  print 'temporary in '+TD
d225 6
d234 5
d241 1
a241 2
  rmtree(TD)
  sys.exit(2)
d248 2
d271 9
a279 2
if action == 'patch':  
  delta = abspath(argv[0])
a285 1
    olddeb = abspath(argv[1])
d288 1
a288 3

    newdeb = abspath(argv[2])
    if  os.path.exists(newdeb) :
d290 1
a290 2
    
    unpack ('PATCH',delta)
d334 1
a334 1
        p=os.popen('dpkg -s '+b)
d339 6
d358 1
a358 1
      unpack ('OLD',olddeb)
d364 1
d366 19
a384 11
        a=params['OLD/Package']
        b='/var/lib/dpkg/info/' + a +'.list'
        if not os.path.exists(b ):
          die('Package "'+a+'" is not installed ??')
        p=open(b)
        s=p.read().split('\n')
        p.close()
        os.mkdir(TD+'/OLD/DATA')
        for a in s:
          if os.path.isfile(a) and not os.path.islink(a) :
            symlink_w_parents(a, TD+'/OLD/DATA')
a385 1
        os.mkdir(TD+'/OLD/DATA')
d400 2
a401 1
    shutil.move('NEW.file',newdeb)
d404 4
a407 1
      newdebsize = os.stat(newdeb)[ST_SIZE]
d410 1
a410 1
      print ' time: %dsec, speed:  %dkB per second ' % (a,(newdebsize / 1024 /  (a+1)))
d413 5
a417 2
elif action == 'delta' :
  olddeb = abspath(argv[0])
d420 2
a421 2
  
  newdeb = abspath(argv[1])
d425 6
a430 2
  
  delta = abspath(argv[2])
d483 11
a493 1
    
d524 4
a527 7
    try:
      a=os.statvfs(TD)
      freespace= a[0] * a[4]
    except a:
      if VERBOSE : print ' statvfs error ',a
      freespace = MAXMEMORY * 16
    if osize < MAXMEMORY / 17 and osize * 8 < freespace   :
d531 1
a531 1
      if VERBOSE > 2 : print '  fallback on xdelta instead of bsdiff' 
d539 1
a539 1
    if VERBOSE > 1 :
d554 1
a554 1
    
d563 1
d568 8
a575 2
    delta_files('OLD/mega_cat',n)
    #clean up
a576 1
    rmtree('OLD/'+w)
d604 1
a604 1
    if VERBOSE > 1: print ' studying ' , name , ' of len ' , newsize
d607 1
a607 1
    if VERBOSE > 2: print '  ar line: ',repr(s)
d642 2
a643 7
      deltacount += 1
      pp=str(deltacount)
      p = 'PATCH/'+pp
      if VERBOSE > 3 : print '   including "',name,'" verbatim in patch'
      os.rename(n,p)
      patch_append(pp)
      script.write('cat '+p+' >> NEW.file ; rm '+p+'\n')      
d684 3
a686 1

d688 2
a689 3
    print ' deb delta is  %3.1f'  % \
          ( deltasize * 100. /  newdebsize ) ,    '% of deb'
    print '  that is, %dkB would be saved ' % (( newdebsize -deltasize ) / 1024)
d691 48
a738 3
      end_sec = time.time()
      a=(end_sec - start_sec)
      print '  time: %dsec, speed: %dkB per second ' % (a,newdebsize / 1024 / (a+1))
d740 2
a741 4
####################################
else:
  #unimplemented action
  assert(0)
d743 69
a811 3
#cleanup
os.chdir(original_cwd)
rmtree(TD)
d814 2
a815 3
##   a='ar qSc result.deb '
##   for o in arlist['NEW'] :
##     a=a+ ' ' + o + ' '
d817 13
a829 3
##     '!<arch>\n'

##   a='fakeroot sh -c "chown root.root * ; ' + a + ' " '
d831 13
a843 1
##   s.write()
d845 5
a849 1
##   S(' cd NEW ; ' + a)
a850 47
##   ret=os.system('cmp NEW/result.deb '+newdeb )

##   if ret:
##     S('xdelta delta -n -9 '+newdeb+' NEW/result.deb ')
##   #S(['ar','qSc', 'temp.deb',]+ deltaparts)
##   #s.write('xdelta patch '+o+'.xdelta'+' ../OLD/'+o+' '+o+'\n')
  
##   def a(p,k,v):
##     if  p == None:
##       p = {}
##     if len(k) > 1  :
##       p[k[0]] = a(p.get(k[0]) , k[1:] ,v  )
##     else:
##       p[k[0]] = v
##       return p    

##     if '/' in s:
##       s=s.split('/')
##       if (len (s) == 2) :
##         ( a,  v ) = s
##         if '/' in s:
##           ....
##         params[ a  ] = v

##       elif (len (s) == 3) :
##         ( a, b, v ) = s
##         if a not in params :
##           params[a] ={}
##         params[ a ][b] = v
##       else:
##         print 'internal error on parm ', repr(s)


##     if False and DEBUG:
##       a=params['OLD/Package']
##       b='/var/lib/dpkg/info/' + a +'.list'
##       if os.path.exists(b ):
##         p=open(b)
##         s=p.read()
##         s=s.split('\n')
##         p.close()
##         for b in s :
##           if not ( b[1:] in oldnames ) :
##             print ' CASINO ',b
##         for b in oldnames : assert( '/'+b in s )
##       else:
##         print ' (package is not installed )',b
d852 1
@


1.11
log
@Added yet another die()
@
text
@d204 1
a204 1
    die('Error: '+f + ' does not exists.')
d212 1
a212 1
    die('Error: '+f + ' does not exists ')
@


1.10
log
@If -vv , display time to compute, and kB per second.

Use die() when files are not debs , or are not existant.
@
text
@d212 1
a212 1
    print f , ' does not exists '
@


1.9
log
@Option '--fs' is no more; new option '--needsold' (that is the opposite).

When error, invoke 'die()' that prints error and cleans up.

Treat 'control.tar.gz' as we treat 'data.tar.gz'.

In patch.sh, call './minigzip' and not 'minigzip'.
@
text
@d36 1
a36 1
import sys , os , tempfile , string ,getopt , tarfile , shutil
d45 3
a117 17
def check_deb(f):
  if not  os.path.isfile(f) :
    print f , ' does not exists '
  p=open(f)
  if p.read(21) != "!<arch>\ndebian-binary" :
    print f , ' does not seem to be a Debian package '
    sys.exit(1)
  p.close()

def check_diff(f):
  if not  os.path.isfile(f) :
    print f , ' does not exists '
  p=open(f)
  if p.read(8) != "!<arch>\n" :
    print f , ' does not seem to be a Debian delta '
    sys.exit(1)
  p.close()
d202 16
d227 5
a231 1
  else:
a235 4
    olddeb = abspath(argv[1])
    if olddeb != '/':
      check_deb(olddeb)
    
d333 6
a338 1
    
d601 6
a606 1
    print ' that is, %dkB would be saved ' % (( newdebsize -deltasize  ) / 1024)
@


1.8
log
@Can create deltas that can be used to recreate a new .deb using the
the installed of the old .deb.

--info is now only a 'debpatch' option ;
 debdelta always include the info.
@
text
@d10 3
d17 1
a17 3
  --fs    create a patch that can be used to recreate the new .deb
          from the old deb that is installed in the host.
          In this case, when using 'debpatch', use '/' for fromfile.
d19 1
d22 1
a23 1
  -d      debug : add md5sums, check installed version for --fs
d28 3
d41 1
d52 1
a52 1
                               ('help','info','fs') )
d54 1
a54 1
cwd = os.getcwd()
d61 1
a61 1
FS      = False
d67 1
a67 1
  elif o == '--fs' and action == 'delta' : FS = True
a114 8
def system(a):
  if type(a) != type('') :
    a=string.join(a,' ')
  ret = os.system(a)
  if  ret != 0 and ( ret != 256 or a[:6] != 'xdelta') :
    print ' error , non zero return status ',ret,' for ',a
    sys.exit(2)

d192 2
a193 1

d198 4
a201 1
if  VERBOSE > 1 :  print 'temporary in '+TD
d203 4
d209 7
d245 1
a245 1
  #lets see what it does and what it requires
d247 2
d265 1
d267 3
a269 1
    if olddeb != '/':
d271 4
a274 2

    if olddeb == '/' and DEBUG:
d276 5
a280 1
      p=os.popen('dpkg -s '+params['OLD/Package'])      
d283 10
a292 6
      for a in  dpkg_params:
        if  params[a] != dpkg_params[a] :
          print 'Error : in installed version , '+a+' = ' +dpkg_params[a]
          print '         in debdelta version , '+a+' = ' +params[a]          
          sys.exit(2)
      
d295 1
a295 1
        raise 'needs old version Debian package'
d299 1
a299 1
        raise 'needs old version Debian package'
d306 1
a306 1
          raise ' package "'+a+'" is not installed ??'
d318 9
d330 2
a331 13

    if 'old-data-tree' in params :
      shutil.rmtree('OLD/DATA')

    os.rename('NEW.file',newdeb)

  for o in  'PATCH/patch.sh','PATCH.file','minigzip','OLD.file':
    if os.path.exists(o):
      if VERBOSE > 5 : print ' deleting ',o
      unlink(o)
  for o in os.listdir('OLD'):
    if VERBOSE > 5 : print ' deleting OLD/',o
    unlink('OLD/'+o)
d412 1
a412 1
      script.write('minigzip -9 '+n+'\n')
d456 1
a456 1
  def delta_data(o,n):
d465 3
a467 3
      if  ('/'+name) in  old_conffiles :
        if VERBOSE > 3 : print '   skip conffile ', name
      elif name in oldnames and  newtarinfo.isreg()  :
d470 5
a474 5
          oldtar.extract(oldtarinfo.name,"OLD/DATA" )
          system('cat OLD/DATA/'+name+' >> OLD/data_mega_cat')
          unlink('OLD/DATA/'+name)
          script.write('cat OLD/DATA/'+name+'  >> OLD/data_mega_cat ; rm OLD/DATA/'+name+'\n')
    delta_files('OLD/data_mega_cat',n)
d477 1
a477 1
    shutil.rmtree('OLD/DATA')
d482 1
a482 1
  if not  FS:
d523 2
a524 2
    elif False and name[:11] == 'control.tar' :
      #TODO
d526 11
a536 1
    elif FS and name[:8] == 'data.tar'  :
d540 1
a540 1
      delta_data(o,n)
d542 1
a542 1
    elif  FS or name not in ar_list_old :       #or it is not in old deb
d550 1
a550 1
    elif not FS:
d558 1
a558 1
      raise
a588 3
  shutil.rmtree('OLD/CONTROL')
  shutil.rmtree('NEW/CONTROL')
    
a589 2
  unlink('NEW.file')
  unlink('OLD.file')
d601 2
a602 5
os.chdir(TD)
rmdir('PATCH')
rmdir('OLD')
rmdir('NEW')
rmdir(TD)
d664 2
@


1.7
log
@'debpatch --info' accepts only 1 argument.

When old ar component was missing, must 'patch_append(p)' it.
@
text
@d14 1
a14 1
  --fs    create a (larger) patch that can be used to recreate the new .deb
d17 1
a17 3
  -d      debug : add md5sums to patch
 --noinfo do not insert in patch the info on two Debian files
  -M Mb   maximum memory (decides between using 'bsdiff' or 'xdelta')
d21 2
a22 1
  -v      verbose (can be added multiple times
d31 1
a31 1
import sys , os , tempfile , string ,getopt
d51 1
a51 1
DEBUG   = 0
d54 1
a54 1
INFO    = action == 'delta'
d59 1
a59 1
  elif o == '-d' and action == 'delta' : DEBUG += 1
d63 1
a63 2
  elif o == '--info' : INFO = True
  elif o == '--noinfo' : INFO = False 
d71 1
a71 1
if INFO and action == 'patch' :
d82 27
d153 12
d199 1
a199 1
if DEBUG or VERBOSE > 1 :  print 'temporary in '+TD
d237 9
a245 8
  while s and s[0] == '#':
    s=de_n(s)[1:]
    if VERBOSE or INFO : print ' info: ',s
    if ':' in s:
      i=s.index(':')  
      params[s[:i]] = s[i+1:]
    else:
      params[s] = True
d251 14
d269 5
a273 3
      os.symlink(olddeb,TD+'/OLD.file')
    
    if 'needs-old' in params:
d275 14
a288 2
        raise 'needs old version Debian package'
      os.symlink(olddeb,TD+'/OLD.file')
d294 3
a332 15
  ##### write parameters
  if DEBUG or INFO:
    for o in 'OLD', 'NEW' :
      if INFO : print o
      system('ar p  '+o+'.file control.tar.gz | tar xzf - ./control')
      p=open('control')
      a=p.readline()
      while a:
        a=de_n(a)
        if a[:3] in ('Pac','Ver','Arc'):
          if DEBUG : script.write('#'+o+'/'+a+'\n')
          if INFO : print ' ' , a
        a=p.readline()
      p.close()
    unlink('control')
d335 1
d339 16
d356 8
a363 5
  #this delta needs the old deb , unpacked in 'OLD'
  #script.write('#unpack-old\n')
  #this delta needs the old deb 
  script.write('#needs-old\n')

d372 1
a372 1
  def unzip(f):
d376 1
a376 1
      if f[:3] != 'NEW' :
a421 1

d430 30
a459 1
  ############# start scanning the new deb
d463 1
a470 1

d497 10
a506 1
    elif  name not in ar_list_old :       #or it is not in old deb
d508 3
a510 1
      p = 'PATCH/'+str(deltacount)
d512 3
a514 4
      patch_append(p)
      script.write('echo PATCH/'+o+' >> NEW.file')
      if DEBUG: script.write('rm PATCH/'+o+'\n')
    else:
d521 2
d543 12
a554 4
  script.close()  
  system('gzip -9 -n PATCH/patch.sh')
  
  patch_append('patch.sh.gz')
d620 16
@


1.6
log
@Use bsdiff when memory does not exceed 50Mb, and free disk space is enough;

' debpatch --info  patch' to just know info on a patch

Shipped in packagde 0.3
@
text
@d14 1
a14 1
  --fs    TODO: create a (larger) patch that can be used to recreate the new .deb
d73 7
a79 1
if len(argv) != 3  :  
d81 1
a81 1
  sys.exit(0)
d232 1
d235 1
a235 1
    if DEBUG: print ' deleting OLD/',o
a252 3
  #unpack('OLD',olddeb)
  #unpack('NEW',newdeb)

a255 2
  #components of this patch
  deltaparts=['patch.sh.gz']
a326 1
    #deltaparts.append(str(deltacount))
d399 1
a399 1
      deltaparts.append(p)
@


1.5
log
@This version uses bspatch / bsdiff ... but it uses too much memory.
@
text
@d3 2
a4 1
"""debdelta [ option...  ] fromfile tofile patchout
d7 1
a7 1
debpatch [ option...  ] patchin  fromfile  tofile 
d10 5
a14 3
Options for debpatch:
  --fs    TODO
          create a (larger) patch that can be used to recreate the new .deb
d18 4
d23 1
a23 1
  -v      verbose
a24 1
  --info  print info on two Debian files
d46 1
a46 1
( opts, argv ) = getopt.getopt(sys.argv[1:], 'vkhd' ,
d51 1
d53 1
a53 1
VERBOSE = 1
d55 1
a55 1
INFO    = False
d62 4
a65 2
  elif o == '--fs' and action == 'delta' : FS = True  
  elif o == '--info' : INFO = True  
a72 3

#should use getopt.gnu_getopt

d87 2
d96 2
d109 2
a110 2
def unpack(d,f):
  "unpacks 'ar' file f in directory d"
d112 1
a112 4

  os.symlink(f,TD+'/'+d+'.file')
  
  arlist[d] = []
d118 1
a118 1
    arlist[d].append(a)    
d120 2
d123 3
d158 1
a158 1
arlist = {}
d162 1
a162 5
if action == 'patch':
  newdeb = abspath(argv[2])
  if  os.path.exists(newdeb) :
    os.rename(newdeb,newdeb+'~')
  
a164 5
  
  olddeb = abspath(argv[1])
  check_deb(olddeb)

  unpack ('PATCH',delta)
d166 13
d181 1
a181 1

d184 1
a184 1
  if 'patch.sh.gz' in  arlist['PATCH']:
d186 2
a187 2
  elif 'patch.sh.bz2' in  arlist['PATCH']:
    system('bunzip2 PATCH/patch.sh.bz2')
d192 1
a192 1
  s=p.readline()
d195 2
a196 1
    s=de_n(s)[1:]    
d198 2
a199 3
      (a , b) = s.split(':')
      b = b[1:] 
      params[a] = b      
a204 1
  if VERBOSE : print ' info:',repr(params)
d206 15
a220 4
  if 'unpack-old' in params:
    unpack ('OLD',olddeb)
  
  system('/bin/sh -e PATCH/patch.sh')
d222 1
a222 1
  os.rename('NEW.file',newdeb)
d235 2
a236 1

d239 1
d246 2
a247 2
  unpack('OLD',olddeb)
  unpack('NEW',newdeb)
d255 1
d266 1
a266 1
      system('tar xzf '+o+'/control.tar.gz ./control')
d272 1
a272 1
          if DEBUG : script.write('#'+o+':'+a+'\n')
d284 72
a355 2
  script.write('#unpack-old\n')
  
d357 3
a359 1
    
d367 10
a376 4
    
  for o in arlist['NEW'] :
    oldsize = os.stat('NEW/'+o)[ST_SIZE]
    if VERBOSE > 1: print 'studying ',o,' of len ',oldsize
d379 2
a380 2
    if VERBOSE > 2: print 'ar line: ',repr(s)
    assert( s[:len(o)] == o and s[-2] == '`' and s[-1] == '\n' )
d383 2
a384 2
    newdeb_file.seek(oldsize  ,1)
    if oldsize & 1 :
d389 3
a391 3
    if oldsize < 128:      #file is too short to compute a delta,
      p=open('NEW/'+o)
      append( p.read(oldsize))
d393 6
a398 6
      unlink('NEW/'+o)
      if o in arlist['OLD'] :
        unlink('OLD/'+o)
    elif  o not in arlist['OLD'] :       #or it is not in old deb
      os.rename('NEW/'+o,'PATCH/'+o)
      deltaparts.append(o)
d403 6
a408 30
      c=''
      if o[-3:] == '.gz' :
        #cannot gunzip if there is a link ! os.link('NEW/'+o,'tmp_n')
        o=o[:-3]
        system('gunzip  NEW/'+o+'.gz')
        system('gunzip  OLD/'+o+'.gz')
        c='.gz'
      elif  o[-3:] == '.bz2' :
        print 'WARNING ! ',o,' is in BZIP2 format ! please fixme !'
      system('bsdiff  OLD/'+o+' NEW/'+o+' PATCH/'+o+'.bsdiff')
      deltaparts.append(o+'.bsdiff')
      unlink('NEW/'+o)
      unlink('OLD/'+o)
      ## how did we fare ?
      deltasize = os.stat('PATCH/'+o+'.bsdiff')[ST_SIZE]
      if VERBOSE > 1 :
        print ' delta is  %3.4f'  % ( deltasize * 100. /  oldsize ) , '% of ',o
      elif  (deltasize > oldsize  and DEBUG): 
        print 'this sucks: deltasize ',deltasize,' > oldsize ',oldsize
      ## and prepare the script consequently
      if c == '.gz':
        script.write('gunzip OLD/'+o+'.gz\n')  
      script.write('bspatch OLD/'+o+' NEW/'+o+' PATCH/'+o+'.bsdiff\n')
      script.write('rm PATCH/'+o+'.bsdiff OLD/'+o+'\n')
      if c == '.gz' :
        script.write('minigzip -9 NEW/'+o+'\n')
      if DEBUG:
        pass #implement MD5
      script.write('cat NEW/'+o+c+' >> NEW.file\n')
      script.write('rm NEW/'+o+c+'\n')
a412 1
  if VERBOSE > 2: print ' ar leftover character: ',repr(s)
d414 1
d418 8
a425 1
    pass #implement MD5
d430 2
a431 7
  #create final debdelta
  os.chdir(TD+'/PATCH')  
  system(['ar','qSc', delta,]+ deltaparts)
  for o in deltaparts:
    unlink(o)

  os.chdir(TD)
d473 24
@


1.4
log
@Added getopt support.
Reorganized code.
Be careful of cwd when using os.path.abspath.
Add parameters to patch.sh , in particular,
 support for '#unpack-old' keyword
Graduated verbosity.
@
text
@d144 1
a144 1
if DEBUG :  print 'temporary in '+TD
d177 1
d181 3
a183 2
      i=s.index(':')
      params[s[:i]] = s[i+1:]
d226 2
d305 2
a306 4
      s= '-n'
      if DEBUG: s=''
      system('xdelta delta '+s+' -9 OLD/'+o+' NEW/'+o+' PATCH/'+o+'.xdelta')
      deltaparts.append(o+'.xdelta')
d310 1
a310 1
      deltasize = os.stat('PATCH/'+o+'.xdelta')[ST_SIZE]
d318 2
a319 2
      script.write('xdelta patch PATCH/'+o+'.xdelta OLD/'+o+' NEW/'+o+'\n')
      script.write('rm PATCH/'+o+'.xdelta OLD/'+o+'\n')
@


1.3
log
@can build diff of two .debs and patch one to get the other
(debian version 0.1)
@
text
@d3 2
a4 3
__doc__ = """
   debelta
       The debdelta command has the following synopsis:
d6 2
a7 1
       debdelta [ option...  ] fromfile tofile patchout
d9 10
a18 9
       Computes a delta from fromfile to tofile and writes it to patchout

   debpatch
       The debpatch command has the following synopsis:

       debpatch [ option...  ] patchin  fromfile  tofile 

       Applies patchin to fromfile and produces  a  reconstructed  version  of
       tofile.
a22 5
DEBUG   = True
VERBOSE = 1
KEEP    = False 

actions =  ('delta','patch')
d26 1
a26 1
import sys , os , tempfile , string
d34 38
d113 2
a114 1
  
d117 1
a117 1
  os.chdir(TD)
d120 1
a120 1
ALLOWED = '<>()[]{}.,;:_-+/ abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789'
a139 12
#should use getopt.gnu_getopt

if len(sys.argv) <= 1 or sys.argv[1] == '--help' or sys.argv[1] == '-h' :
  print __doc__
  sys.exit(0)

action=(os.path.basename(sys.argv[0]))[3:]

if action not in actions:
  print 'wrong action: may be '+repr(actions)

argv=sys.argv[1:]
a156 2

  unpack ('PATCH',delta)
d161 2
d230 1
a230 1
  if DEBUG:
d232 1
d239 2
a240 1
          script.write('#'+o+a+'\n')
a244 4
  #this delta needs the old deb , unpacked in 'OLD'
  script.write('#unpack-old\n')

  
d249 4
d265 1
a265 1
    if VERBOSE: print 'studying ',o,' of len ',oldsize
d268 1
a268 1
    if VERBOSE > 1: print 'ar line: ',repr(s)
d309 1
a309 1
      if VERBOSE :
d328 1
a328 1
  if VERBOSE: print 'leftover: ',repr(s)
d351 1
a351 1
    print ' deb delta is  %3.4f'  % \
d353 1
@


1.2
log
@this works, it creates deltas that can rebuild the exact .deb
@
text
@d20 1
a20 1
minigzip='/home/andrea/bin/minigzip'
d60 26
a85 1
ALLOWED = '. abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789'
d111 1
a111 1
action=sys.argv[0][-5:]
d118 4
d123 1
a123 17
if action == 'delta' :
  olddeb = abspath(argv[0])
  check_deb(olddeb)

  newdeb = abspath(argv[1])
  check_deb(newdeb)
  newdebsize = os.stat(newdeb)[ST_SIZE]
  
  delta = abspath(argv[2])
  if  os.path.exists(delta) :
    os.rename(delta,delta+'~')
  
  pairs = [ ('OLD',olddeb) , ('NEW',newdeb) ]
  
elif action == 'patch':
  olddeb = abspath(argv[1])
  check_deb(olddeb)
d125 1
d127 1
a133 3
  pairs = [ ('OLD',olddeb) , ('PATCH',delta) ]
else:
  assert(0)
d135 4
a138 4
TD = abspath(tempfile.mkdtemp())
if DEBUG :  print 'temporary in '+TD
#from here on, we live in the temp dir
os.chdir(TD)
d140 2
a141 1
arlist = {}
d143 1
a143 3
##################
for (d,f) in pairs :
  assert(os.path.exists(f))
d145 17
a161 10
  #os.symlink(f,TD+'/'+d+'.file')
  
  arlist[d] = []
  p=os.popen('ar t '+f,'r')
  while 1:
    a=p.readline()
    if not a : break
    if a[-1] ==  '\n' :
      a = a[:-1]
    arlist[d].append(a)    
d164 1
a164 6
  os.mkdir(d)  
  os.chdir(d)
  system('ar xo '+f)
  os.chdir(TD)
  
################# compute patch
d166 2
a167 4
if action == 'patch':
  os.symlink(minigzip,'minigzip')
  
  os.mkdir(TD+'/NEW')
a168 1
  system('gunzip PATCH/patch.sh.gz')
d173 8
a180 1
############## compute delta
d182 17
a198 1
  os.mkdir(TD+'/PATCH')
d206 18
d228 2
d264 1
d314 1
a314 1
  os.chdir(TD+'/PATCH')
d318 1
d320 1
a320 5
  rmdir('PATCH')
  rmdir('OLD')
  rmdir('NEW')
  rmdir(TD)
  
d322 10
a331 3
  
  print ' deb delta is  %3.4f'  % ( deltasize * 100. /  newdebsize ) , '% of deb'
  
d333 6
@


1.1
log
@Initial revision
@
text
@d4 2
a5 2
   Delta
       The delta subcommand has the following synopsis:
d7 1
a7 1
       debdelta delta [ option...  ] fromfile tofile patchout
d11 2
a12 2
   Patch
       The patch subcommand has the following synopsis:
d14 1
a14 1
       debdelta patch [ option...  ] patchin [ fromfile [ tofile ]]
d20 6
a25 1
minigzip='~/bin/minigzip'
d28 3
a30 1
import sys,os,tempfile , string
d36 1
a36 1
DEBUG =True
d38 41
a78 4
#def unlink(a):
#  print ' unlink ',a
#def rmdir(a):
#  print ' rmdir ',a
d82 1
a82 1
if len(sys.argv) <= 1 or sys.argv[1] == '--help' :
d86 1
a86 1
action=sys.argv[1]
d91 1
a91 1
argv=sys.argv[2:]
a92 3
TD = tempfile.mkdtemp()

if DEBUG :  print 'temporary in '+TD
d96 2
d99 1
d101 1
a105 1
  os.mkdir(TD+'/PATCH')
d107 1
d110 3
d114 5
a118 1
  delta = abspath(argv[0]) 
d123 3
a125 8
def S(a):
  if type(a) != type('') :
    a=string.join(a,' ')
  ret = os.system(a)
  if  ret != 0 and ( ret != 256 or a[:6] != 'xdelta') :
    print ' error , non zero return status ',ret,' for ',a
    sys.exit(2)
  
d129 2
a130 1
  
a131 3
  os.chdir(TD)
  os.mkdir(d)
  os.chdir(d)
d133 3
d143 1
a143 2
    arlist[d].append(a)
    
d145 4
a148 1
  S('ar xo '+f)
d151 1
a151 1
os.chdir(TD)
d154 8
a161 4
  os.chdir(TD+'/PATCH')
  S('gunzip patch.sh.gz')
  S('/bin/sh patch.sh')
  os.rename('result.deb',newdeb)
d163 1
d165 1
d168 1
d170 12
a181 2
  s=open('PATCH/patch.sh','w')
  s.write('#!/bin/sh -e\n')
d184 1
a184 1
  
d187 21
a207 1
    if o not in arlist['OLD'] or oldsize < 128:
a208 1
      unlink('OLD/'+o)
d210 1
d212 1
d217 2
a218 2
        S('gunzip -cv NEW/'+o+'.gz > ' + 'NEW/'+o)
        S('gunzip -cv OLD/'+o+'.gz > ' + 'OLD/'+o)
d220 6
a225 14
      S('xdelta delta -n -V -9 OLD/'+o+' NEW/'+o+' PATCH/'+o+'.xdelta')      
      deltasize = os.stat('PATCH/'+o+'.xdelta')[ST_SIZE]
      if deltasize > oldsize  :
        print 'bello schifo ',deltasize,' > ',oldsize
      if 1:
        if c == '.gz':
          s.write('gunzip ../OLD/'+o+'.gz\n')  
        deltaparts.append(o+'.xdelta')
        s.write('xdelta patch '+o+'.xdelta'+' ../OLD/'+o+' '+o+'\n')
        s.write('rm '+o+'.xdelta ../OLD/'+o+'\n')
        if c == '.gz' :
          s.write(minigzip+' -9 '+o+'\n')
          if DEBUG:
            pass
d228 32
a259 23
      if c:
        unlink('NEW/'+o+c)
        unlink('OLD/'+o+c)

  a='ar qSc result.deb '
  for o in arlist['NEW'] :
    a=a+ ' ' + o + ' '

  a='fakeroot sh -c "chown root.root * ; ' + a + ' " '
  
  s.write()
  
  S(' cd NEW ; ' + a)

  ret=os.system('cmp NEW/result.deb '+newdeb )

  if ret:
    S('xdelta delta -n -9 '+newdeb+' NEW/result.deb ')
  #S(['ar','qSc', 'temp.deb',]+ deltaparts)
  #s.write('xdelta patch '+o+'.xdelta'+' ../OLD/'+o+' '+o+'\n')
  
  s.close()
  S('gzip -9 PATCH/patch.sh')
d261 1
a261 1
  S(['ar','qSc', delta,]+ deltaparts)
d274 22
@
