Tag: PDS

Reading NASA’s PDS files for conversion purposes

Posted by GuShH on April 5, 2020

Looking at the original raw images from Mars Global Surveyor among others using NASA’s image viewer (nasaview-3.18.0_win) I wondered about the file format, specifically how the data was encoded. Turns out it is a binary dump with a fixed ascii header!

I found several utilities including http://bjj.mmedia.is/utils/img2png/ (IMG2PNG from Björn Jonsson), some better than others but I couldn’t find the specifics about each format version, namely each format definition.

At first I thought the header was fixed but sometimes a field may or may not be present, at the moment I think I got most of them on the HEADER structure, at least all of the test images worked fine.

This code simply interprets the header and then reads the data. It can output the image into a BMP, PNG or JPEG.

I believe there is a bug regarding the final image dimensions, maybe someone could chip in a solution.

; NASA PDS FILE READER FOR VERSION 3
; This code is for learning purposes, it is considered incomplete and merely a starting point.
; GuShH - Gustavo Fiorenza @ 05/apr/2020
; info@gushh.net

EnableExplicit
#HEADER_MAXSIZE = 2048

Structure HEADER
	PDS_VERSION_ID.s
	FILE_NAME.s
	RECORD_TYPE.s
	RECORD_BYTES.i
	FILE_RECORDS.i
	LABEL_RECORDS.i
	IMAGE.i
	SPACECRAFT_NAME.s
	MISSION_PHASE_NAME.s
	TARGET_NAME.s
	INSTRUMENT_ID.s
	PRODUCER_ID.s
	DATA_SET_ID.s
	PRODUCT_CREATION_TIME.s
	SOFTWARE_NAME.s
	UPLOAD_ID.s
	PRODUCT_ID.s
	START_TIME.s
	IMAGE_TIME.s
	STOP_TIME.s
	SPACECRAFT_CLOCK_START_COUNT.s
	SPACECRAFT_CLOCK_STOP_COUNT.s
	FOCAL_PLANE_TEMPERATURE.s
	GAIN_MODE_ID.s
	OFFSET_MODE_ID.s
	LINE_EXPOSURE_DURATION.s
	DOWNTRACK_SUMMING.s
	CROSSTRACK_SUMMING.s
	EDIT_MODE_ID.s
	RATIONALE_DESC.s
	OBJECT.s
	LINES.i
	LINE_SAMPLES.i
	LINE_PREFIX_BYTES.i
	LINE_SUFFIX_BYTES.i
	SAMPLE_TYPE.s
	SAMPLE_BITS.i
	SAMPLE_BIT_MASK.s
	CHECKSUM.s
	END_OBJECT.s
	ENDSTR.s
EndStructure

Structure PDS
	filename.s
	filePointer.i
	header.HEADER
	readamount.i
	header_size.i
	contentDataAddress.i
	bitmap.i
	width.i
	height.i
	bits.i
	size.i
EndStructure

Procedure.s pds_readheaderparam_str( token.s )
	ProcedureReturn UCase(Trim( StringField( token, 2, "=" ) ))
EndProcedure

Procedure.i pds_readheaderparam_int( token.s )
	ProcedureReturn Val(pds_readheaderparam_str(token))
EndProcedure

Procedure.i pds_readheader( *this.PDS )
	
	Define.s header 				= ReadString(*this\filePointer, #PB_Ascii | #PB_File_IgnoreEOL)	
	Define.i lines 					= CountString( header, #LF$ )
	Define.s token 					= ""
	Define.i current_line
	
	For current_line = 1 To lines
		
		token = UCase(Trim( StringField( header, current_line, #LF$ ) ))
		token = ReplaceString( token, #CR$, #Null$, #PB_String_NoCase ) ;sanitize possible left over carriage returns from makepds version 1.3
		token = ReplaceString( token, #LF$, #Null$, #PB_String_NoCase ) ;and LF just for good measure, no need though.
		
		If token
			Debug token
			
			Select UCase(Trim( StringField( token, 1, "=" ) ))
					
				Case "PDS_VERSION_ID"
					*this\header\PDS_VERSION_ID 								= pds_readheaderparam_str( token )
				Case "FILE_NAME"
					*this\header\FILE_NAME 											= pds_readheaderparam_str( token )
				Case "RECORD_TYPE"
					*this\header\RECORD_TYPE										= pds_readheaderparam_str( token )
				Case "RECORD_BYTES"
					*this\header\RECORD_BYTES										= pds_readheaderparam_int( token )
				Case "FILE_RECORDS"
					*this\header\FILE_RECORDS										= pds_readheaderparam_int( token )
				Case "LABEL_RECORDS"
					*this\header\LABEL_RECORDS									= pds_readheaderparam_int( token )
				Case "IMAGE"
					*this\header\IMAGE													= pds_readheaderparam_int( token )
				Case "SPACECRAFT_NAME"
					*this\header\SPACECRAFT_NAME								= pds_readheaderparam_str( token )
				Case "MISSION_PHASE_NAME"
					*this\header\MISSION_PHASE_NAME							= pds_readheaderparam_str( token )
				Case "TARGET_NAME"
					*this\header\TARGET_NAME										= pds_readheaderparam_str( token )
				Case "INSTRUMENT_ID"
					*this\header\INSTRUMENT_ID									= pds_readheaderparam_str( token )
				Case "PRODUCER_ID"
					*this\header\PRODUCER_ID										= pds_readheaderparam_str( token )
				Case "DATA_SET_ID"
					*this\header\DATA_SET_ID										= pds_readheaderparam_str( token )
				Case "PRODUCT_CREATION_TIME"
					*this\header\PRODUCT_CREATION_TIME					= pds_readheaderparam_str( token )
				Case "SOFTWARE_NAME"
					*this\header\SOFTWARE_NAME									= pds_readheaderparam_str( token )
				Case "UPLOAD_ID"
					*this\header\UPLOAD_ID											= pds_readheaderparam_str( token )
				Case "PRODUCT_ID"
					*this\header\PRODUCT_ID											= pds_readheaderparam_str( token )
				Case "START_TIME"
					*this\header\START_TIME											= pds_readheaderparam_str( token )
				Case "IMAGE_TIME"
					*this\header\IMAGE_TIME											= pds_readheaderparam_str( token )
				Case "STOP_TIME"
					*this\header\STOP_TIME											= pds_readheaderparam_str( token )
				Case "SPACECRAFT_CLOCK_START_COUNT"
					*this\header\SPACECRAFT_CLOCK_START_COUNT		= pds_readheaderparam_str( token )
				Case "SPACECRAFT_CLOCK_STOP_COUNT"
					*this\header\SPACECRAFT_CLOCK_STOP_COUNT		= pds_readheaderparam_str( token )
				Case "FOCAL_PLANE_TEMPERATURE"
					*this\header\FOCAL_PLANE_TEMPERATURE				= pds_readheaderparam_str( token )
				Case "GAIN_MODE_ID"
					*this\header\GAIN_MODE_ID										= pds_readheaderparam_str( token )
				Case "OFFSET_MODE_ID"
					*this\header\OFFSET_MODE_ID									= pds_readheaderparam_str( token )
				Case "LINE_EXPOSURE_DURATION"
					*this\header\LINE_EXPOSURE_DURATION					= pds_readheaderparam_str( token )
				Case "DOWNTRACK_SUMMING"
					*this\header\DOWNTRACK_SUMMING							= pds_readheaderparam_str( token )
				Case "CROSSTRACK_SUMMING"
					*this\header\CROSSTRACK_SUMMING							= pds_readheaderparam_str( token )
				Case "EDIT_MODE_ID"
					*this\header\EDIT_MODE_ID										= pds_readheaderparam_str( token )
				Case "RATIONALE_DESC"
					*this\header\RATIONALE_DESC									= pds_readheaderparam_str( token )
				Case "OBJECT"
					*this\header\OBJECT													= pds_readheaderparam_str( token )
				Case "LINES"
					*this\header\LINES													= pds_readheaderparam_int( token )
				Case "LINE_SAMPLES"
					*this\header\LINE_SAMPLES										= pds_readheaderparam_int( token )
				Case "LINE_PREFIX_BYTES"
					*this\header\LINE_PREFIX_BYTES							= pds_readheaderparam_int( token )
				Case "LINE_SUFFIX_BYTES"
					*this\header\LINE_SUFFIX_BYTES							= pds_readheaderparam_int( token )
				Case "SAMPLE_TYPE"
					*this\header\SAMPLE_TYPE										= pds_readheaderparam_str( token )
				Case "SAMPLE_BITS"
					*this\header\SAMPLE_BITS										= pds_readheaderparam_int( token )
				Case "SAMPLE_BIT_MASK"
					*this\header\SAMPLE_BIT_MASK								= pds_readheaderparam_str( token )
				Case "CHECKSUM"
					*this\header\CHECKSUM												= pds_readheaderparam_str( token )
				Case "END_OBJECT"
					*this\header\END_OBJECT											= pds_readheaderparam_str( token )
				Case "END"
					*this\header\ENDSTR													= pds_readheaderparam_str( token )
			EndSelect
		EndIf
		
	Next
	
	; find a better way to do this!!
	FileSeek( *this\filePointer, FindString( header, "END_OBJECT = IMAGE")+Len("END_OBJECT = IMAGE") + 3 )
	
	Define.i byte = 0
	Repeat
		byte = ReadByte(*this\filePointer)
	Until byte = $0d
	
	Repeat
		byte = ReadByte(*this\filePointer)
	Until byte = $0a
	
	Repeat
		byte = ReadByte(*this\filePointer)
	Until byte <> 20 And byte <> 32
	
	ProcedureReturn #True
	
EndProcedure

Procedure.i pds_load( *this.PDS, filename.s )
	
	If *this
		If filename
			
			*this\filename 			= filename
			*this\filePointer 	= ReadFile(#PB_Any, *this\filename )
			
			If IsFile( *this\filePointer )
				If pds_readheader( *this )
					If FindString( *this\header\PDS_VERSION_ID, "PDS3" )
						
						*this\header_size 	= Loc(*this\filePointer)
						*this\width 				= *this\header\LINE_SAMPLES
						*this\height 				= *this\header\LINES
						*this\bits					= *this\header\SAMPLE_BITS
						*this\size 					= (*this\width * *this\height) * *this\bits
						
						If *this\size <= 0
							ProcedureReturn  #False
						EndIf
						
						*this\contentDataAddress = AllocateMemory( *this\size )
						*this\readamount = ReadData( *this\filePointer, *this\contentDataAddress, *this\size )
						
					Else
						Debug "error file version mismatch"
						ProcedureReturn 0
					EndIf
					
					CloseFile(*this\filePointer)
				EndIf
			EndIf
			
			ProcedureReturn *this\readamount
			
		EndIf
	EndIf
	
EndProcedure

Structure Pixel
	Pixel.l
EndStructure

Procedure.i pds_plot( *this.PDS )
	
	If *this
		If *this\header_size
			
			If *this\width > 0 And *this\height > 0
				
				*this\bitmap = CreateImage( #PB_Any, *this\width, *this\height, 32 )
				Define.byte *mem
				
				If StartDrawing( ImageOutput(*this\bitmap) )
					
					Define.i offset 			= 0
					Define.i buffer 			= DrawingBuffer()
					Define.i pitch       	= DrawingBufferPitch()
					Define.i pixelFormat 	= DrawingBufferPixelFormat()
					Define.i real_bpp 		= 0
					Define.i inversion_y 	= pixelformat ! #PB_PixelFormat_ReversedY   
					
					Select pixelformat ! #PB_PixelFormat_ReversedY
						Case #PB_PixelFormat_8Bits
							real_bpp = 1
						Case #PB_PixelFormat_15Bits
							real_bpp = 2
						Case #PB_PixelFormat_16Bits
							real_bpp = 2
						Case #PB_PixelFormat_24Bits_RGB
							real_bpp = 3
						Case #PB_PixelFormat_24Bits_BGR
							real_bpp = 3
						Case #PB_PixelFormat_32Bits_RGB
							real_bpp = 4
						Case #PB_PixelFormat_32Bits_BGR
							real_bpp = 4
					EndSelect
					
					If buffer
						
						Define.i inv_height = 0
						If inversion_y <> 0
							inv_height = *this\height
						EndIf
						
						Define.integer *buf = buffer
						Define.i x, y
						For y = 0 To *this\height - 1
							For x = 0 To *this\width - 1
								
								*mem 		= *this\contentDataAddress + *this\header_size + (( ((inv_height-y) * *this\width) + x) )
								*buf 		= buffer + (( y * pitch) + x * real_bpp)
								*buf\i 	= RGB(*mem\b, *mem\b, *mem\b )
								
							Next
						Next
					EndIf
					
					StopDrawing()
				EndIf
				
			Else
				Debug "error, image dimensions couldn't be loaded"
			EndIf
			
		EndIf
	EndIf
	
EndProcedure

Procedure.i pds_save( *this.PDS, OutputFileName.s, Format.i = #PB_ImagePlugin_BMP, Quality.i = 7 )
	
	If *this
		If Len(OutputFileName) > 0
			
			If IsImage(*this\bitmap)
				ProcedureReturn SaveImage( *this\bitmap, OutputFileName, Format, Quality )
			EndIf
			
		EndIf
	EndIf
	
EndProcedure

Procedure.i pds_create( )
	
	Define.PDS *this = AllocateMemory( SizeOf(PDS) )
	
	If *this
		InitializeStructure(*this, PDS)
		ProcedureReturn *this
	EndIf
	
EndProcedure

Procedure.i pds_destroy( *this.PDS )
	
	If *this
		If *this\contentDataAddress
			
			If IsImage(*this\bitmap)
				FreeImage(*this\bitmap)
			EndIf
			
			FreeMemory(*this\contentDataAddress)
			*this\contentDataAddress = #Null
			ClearStructure( *this, PDS )
			ProcedureReturn #True
			
		EndIf
	EndIf
	
EndProcedure

Testing images can be found at the NASA repository, a sample image is included here for studying purposes (Property of NASA).

Here is an example of how to load and convert an image:

	Define.PDS *file = pds_create()
	If *file
		pds_load( *file, "ab100402.img" )
		pds_plot( *file )
		
		If pds_save( *file, "output.bmp")
			RunProgram("output.bmp")
		EndIf
		
		pds_destroy(*file)
	EndIf

Have fun and stay safe!