Source file src/pkg/archive/tar/format.go

     1	// Copyright 2016 The Go Authors. All rights reserved.
     2	// Use of this source code is governed by a BSD-style
     3	// license that can be found in the LICENSE file.
     4	
     5	package tar
     6	
     7	import "strings"
     8	
     9	// Format represents the tar archive format.
    10	//
    11	// The original tar format was introduced in Unix V7.
    12	// Since then, there have been multiple competing formats attempting to
    13	// standardize or extend the V7 format to overcome its limitations.
    14	// The most common formats are the USTAR, PAX, and GNU formats,
    15	// each with their own advantages and limitations.
    16	//
    17	// The following table captures the capabilities of each format:
    18	//
    19	//	                  |  USTAR |       PAX |       GNU
    20	//	------------------+--------+-----------+----------
    21	//	Name              |   256B | unlimited | unlimited
    22	//	Linkname          |   100B | unlimited | unlimited
    23	//	Size              | uint33 | unlimited |    uint89
    24	//	Mode              | uint21 |    uint21 |    uint57
    25	//	Uid/Gid           | uint21 | unlimited |    uint57
    26	//	Uname/Gname       |    32B | unlimited |       32B
    27	//	ModTime           | uint33 | unlimited |     int89
    28	//	AccessTime        |    n/a | unlimited |     int89
    29	//	ChangeTime        |    n/a | unlimited |     int89
    30	//	Devmajor/Devminor | uint21 |    uint21 |    uint57
    31	//	------------------+--------+-----------+----------
    32	//	string encoding   |  ASCII |     UTF-8 |    binary
    33	//	sub-second times  |     no |       yes |        no
    34	//	sparse files      |     no |       yes |       yes
    35	//
    36	// The table's upper portion shows the Header fields, where each format reports
    37	// the maximum number of bytes allowed for each string field and
    38	// the integer type used to store each numeric field
    39	// (where timestamps are stored as the number of seconds since the Unix epoch).
    40	//
    41	// The table's lower portion shows specialized features of each format,
    42	// such as supported string encodings, support for sub-second timestamps,
    43	// or support for sparse files.
    44	//
    45	// The Writer currently provides no support for sparse files.
    46	type Format int
    47	
    48	// Constants to identify various tar formats.
    49	const (
    50		// Deliberately hide the meaning of constants from public API.
    51		_ Format = (1 << iota) / 4 // Sequence of 0, 0, 1, 2, 4, 8, etc...
    52	
    53		// FormatUnknown indicates that the format is unknown.
    54		FormatUnknown
    55	
    56		// The format of the original Unix V7 tar tool prior to standardization.
    57		formatV7
    58	
    59		// FormatUSTAR represents the USTAR header format defined in POSIX.1-1988.
    60		//
    61		// While this format is compatible with most tar readers,
    62		// the format has several limitations making it unsuitable for some usages.
    63		// Most notably, it cannot support sparse files, files larger than 8GiB,
    64		// filenames larger than 256 characters, and non-ASCII filenames.
    65		//
    66		// Reference:
    67		//	http://pubs.opengroup.org/onlinepubs/9699919799/utilities/pax.html#tag_20_92_13_06
    68		FormatUSTAR
    69	
    70		// FormatPAX represents the PAX header format defined in POSIX.1-2001.
    71		//
    72		// PAX extends USTAR by writing a special file with Typeflag TypeXHeader
    73		// preceding the original header. This file contains a set of key-value
    74		// records, which are used to overcome USTAR's shortcomings, in addition to
    75		// providing the ability to have sub-second resolution for timestamps.
    76		//
    77		// Some newer formats add their own extensions to PAX by defining their
    78		// own keys and assigning certain semantic meaning to the associated values.
    79		// For example, sparse file support in PAX is implemented using keys
    80		// defined by the GNU manual (e.g., "GNU.sparse.map").
    81		//
    82		// Reference:
    83		//	http://pubs.opengroup.org/onlinepubs/009695399/utilities/pax.html
    84		FormatPAX
    85	
    86		// FormatGNU represents the GNU header format.
    87		//
    88		// The GNU header format is older than the USTAR and PAX standards and
    89		// is not compatible with them. The GNU format supports
    90		// arbitrary file sizes, filenames of arbitrary encoding and length,
    91		// sparse files, and other features.
    92		//
    93		// It is recommended that PAX be chosen over GNU unless the target
    94		// application can only parse GNU formatted archives.
    95		//
    96		// Reference:
    97		//	https://www.gnu.org/software/tar/manual/html_node/Standard.html
    98		FormatGNU
    99	
   100		// Schily's tar format, which is incompatible with USTAR.
   101		// This does not cover STAR extensions to the PAX format; these fall under
   102		// the PAX format.
   103		formatSTAR
   104	
   105		formatMax
   106	)
   107	
   108	func (f Format) has(f2 Format) bool   { return f&f2 != 0 }
   109	func (f *Format) mayBe(f2 Format)     { *f |= f2 }
   110	func (f *Format) mayOnlyBe(f2 Format) { *f &= f2 }
   111	func (f *Format) mustNotBe(f2 Format) { *f &^= f2 }
   112	
   113	var formatNames = map[Format]string{
   114		formatV7: "V7", FormatUSTAR: "USTAR", FormatPAX: "PAX", FormatGNU: "GNU", formatSTAR: "STAR",
   115	}
   116	
   117	func (f Format) String() string {
   118		var ss []string
   119		for f2 := Format(1); f2 < formatMax; f2 <<= 1 {
   120			if f.has(f2) {
   121				ss = append(ss, formatNames[f2])
   122			}
   123		}
   124		switch len(ss) {
   125		case 0:
   126			return "<unknown>"
   127		case 1:
   128			return ss[0]
   129		default:
   130			return "(" + strings.Join(ss, " | ") + ")"
   131		}
   132	}
   133	
   134	// Magics used to identify various formats.
   135	const (
   136		magicGNU, versionGNU     = "ustar ", " \x00"
   137		magicUSTAR, versionUSTAR = "ustar\x00", "00"
   138		trailerSTAR              = "tar\x00"
   139	)
   140	
   141	// Size constants from various tar specifications.
   142	const (
   143		blockSize  = 512 // Size of each block in a tar stream
   144		nameSize   = 100 // Max length of the name field in USTAR format
   145		prefixSize = 155 // Max length of the prefix field in USTAR format
   146	)
   147	
   148	// blockPadding computes the number of bytes needed to pad offset up to the
   149	// nearest block edge where 0 <= n < blockSize.
   150	func blockPadding(offset int64) (n int64) {
   151		return -offset & (blockSize - 1)
   152	}
   153	
   154	var zeroBlock block
   155	
   156	type block [blockSize]byte
   157	
   158	// Convert block to any number of formats.
   159	func (b *block) V7() *headerV7       { return (*headerV7)(b) }
   160	func (b *block) GNU() *headerGNU     { return (*headerGNU)(b) }
   161	func (b *block) STAR() *headerSTAR   { return (*headerSTAR)(b) }
   162	func (b *block) USTAR() *headerUSTAR { return (*headerUSTAR)(b) }
   163	func (b *block) Sparse() sparseArray { return sparseArray(b[:]) }
   164	
   165	// GetFormat checks that the block is a valid tar header based on the checksum.
   166	// It then attempts to guess the specific format based on magic values.
   167	// If the checksum fails, then FormatUnknown is returned.
   168	func (b *block) GetFormat() Format {
   169		// Verify checksum.
   170		var p parser
   171		value := p.parseOctal(b.V7().Chksum())
   172		chksum1, chksum2 := b.ComputeChecksum()
   173		if p.err != nil || (value != chksum1 && value != chksum2) {
   174			return FormatUnknown
   175		}
   176	
   177		// Guess the magic values.
   178		magic := string(b.USTAR().Magic())
   179		version := string(b.USTAR().Version())
   180		trailer := string(b.STAR().Trailer())
   181		switch {
   182		case magic == magicUSTAR && trailer == trailerSTAR:
   183			return formatSTAR
   184		case magic == magicUSTAR:
   185			return FormatUSTAR | FormatPAX
   186		case magic == magicGNU && version == versionGNU:
   187			return FormatGNU
   188		default:
   189			return formatV7
   190		}
   191	}
   192	
   193	// SetFormat writes the magic values necessary for specified format
   194	// and then updates the checksum accordingly.
   195	func (b *block) SetFormat(format Format) {
   196		// Set the magic values.
   197		switch {
   198		case format.has(formatV7):
   199			// Do nothing.
   200		case format.has(FormatGNU):
   201			copy(b.GNU().Magic(), magicGNU)
   202			copy(b.GNU().Version(), versionGNU)
   203		case format.has(formatSTAR):
   204			copy(b.STAR().Magic(), magicUSTAR)
   205			copy(b.STAR().Version(), versionUSTAR)
   206			copy(b.STAR().Trailer(), trailerSTAR)
   207		case format.has(FormatUSTAR | FormatPAX):
   208			copy(b.USTAR().Magic(), magicUSTAR)
   209			copy(b.USTAR().Version(), versionUSTAR)
   210		default:
   211			panic("invalid format")
   212		}
   213	
   214		// Update checksum.
   215		// This field is special in that it is terminated by a NULL then space.
   216		var f formatter
   217		field := b.V7().Chksum()
   218		chksum, _ := b.ComputeChecksum() // Possible values are 256..128776
   219		f.formatOctal(field[:7], chksum) // Never fails since 128776 < 262143
   220		field[7] = ' '
   221	}
   222	
   223	// ComputeChecksum computes the checksum for the header block.
   224	// POSIX specifies a sum of the unsigned byte values, but the Sun tar used
   225	// signed byte values.
   226	// We compute and return both.
   227	func (b *block) ComputeChecksum() (unsigned, signed int64) {
   228		for i, c := range b {
   229			if 148 <= i && i < 156 {
   230				c = ' ' // Treat the checksum field itself as all spaces.
   231			}
   232			unsigned += int64(c)
   233			signed += int64(int8(c))
   234		}
   235		return unsigned, signed
   236	}
   237	
   238	// Reset clears the block with all zeros.
   239	func (b *block) Reset() {
   240		*b = block{}
   241	}
   242	
   243	type headerV7 [blockSize]byte
   244	
   245	func (h *headerV7) Name() []byte     { return h[000:][:100] }
   246	func (h *headerV7) Mode() []byte     { return h[100:][:8] }
   247	func (h *headerV7) UID() []byte      { return h[108:][:8] }
   248	func (h *headerV7) GID() []byte      { return h[116:][:8] }
   249	func (h *headerV7) Size() []byte     { return h[124:][:12] }
   250	func (h *headerV7) ModTime() []byte  { return h[136:][:12] }
   251	func (h *headerV7) Chksum() []byte   { return h[148:][:8] }
   252	func (h *headerV7) TypeFlag() []byte { return h[156:][:1] }
   253	func (h *headerV7) LinkName() []byte { return h[157:][:100] }
   254	
   255	type headerGNU [blockSize]byte
   256	
   257	func (h *headerGNU) V7() *headerV7       { return (*headerV7)(h) }
   258	func (h *headerGNU) Magic() []byte       { return h[257:][:6] }
   259	func (h *headerGNU) Version() []byte     { return h[263:][:2] }
   260	func (h *headerGNU) UserName() []byte    { return h[265:][:32] }
   261	func (h *headerGNU) GroupName() []byte   { return h[297:][:32] }
   262	func (h *headerGNU) DevMajor() []byte    { return h[329:][:8] }
   263	func (h *headerGNU) DevMinor() []byte    { return h[337:][:8] }
   264	func (h *headerGNU) AccessTime() []byte  { return h[345:][:12] }
   265	func (h *headerGNU) ChangeTime() []byte  { return h[357:][:12] }
   266	func (h *headerGNU) Sparse() sparseArray { return sparseArray(h[386:][:24*4+1]) }
   267	func (h *headerGNU) RealSize() []byte    { return h[483:][:12] }
   268	
   269	type headerSTAR [blockSize]byte
   270	
   271	func (h *headerSTAR) V7() *headerV7      { return (*headerV7)(h) }
   272	func (h *headerSTAR) Magic() []byte      { return h[257:][:6] }
   273	func (h *headerSTAR) Version() []byte    { return h[263:][:2] }
   274	func (h *headerSTAR) UserName() []byte   { return h[265:][:32] }
   275	func (h *headerSTAR) GroupName() []byte  { return h[297:][:32] }
   276	func (h *headerSTAR) DevMajor() []byte   { return h[329:][:8] }
   277	func (h *headerSTAR) DevMinor() []byte   { return h[337:][:8] }
   278	func (h *headerSTAR) Prefix() []byte     { return h[345:][:131] }
   279	func (h *headerSTAR) AccessTime() []byte { return h[476:][:12] }
   280	func (h *headerSTAR) ChangeTime() []byte { return h[488:][:12] }
   281	func (h *headerSTAR) Trailer() []byte    { return h[508:][:4] }
   282	
   283	type headerUSTAR [blockSize]byte
   284	
   285	func (h *headerUSTAR) V7() *headerV7     { return (*headerV7)(h) }
   286	func (h *headerUSTAR) Magic() []byte     { return h[257:][:6] }
   287	func (h *headerUSTAR) Version() []byte   { return h[263:][:2] }
   288	func (h *headerUSTAR) UserName() []byte  { return h[265:][:32] }
   289	func (h *headerUSTAR) GroupName() []byte { return h[297:][:32] }
   290	func (h *headerUSTAR) DevMajor() []byte  { return h[329:][:8] }
   291	func (h *headerUSTAR) DevMinor() []byte  { return h[337:][:8] }
   292	func (h *headerUSTAR) Prefix() []byte    { return h[345:][:155] }
   293	
   294	type sparseArray []byte
   295	
   296	func (s sparseArray) Entry(i int) sparseElem { return sparseElem(s[i*24:]) }
   297	func (s sparseArray) IsExtended() []byte     { return s[24*s.MaxEntries():][:1] }
   298	func (s sparseArray) MaxEntries() int        { return len(s) / 24 }
   299	
   300	type sparseElem []byte
   301	
   302	func (s sparseElem) Offset() []byte { return s[00:][:12] }
   303	func (s sparseElem) Length() []byte { return s[12:][:12] }
   304	

View as plain text