Trimble Business Center — Produce final deliverables in one workspace. Start free trial.

AI Assistant

Notifications

Clear all

Python Script for Converting FieldGenius RAW files to TBC importable RW5 files...

Summarize Topic

Software, CAD & Mapping

Last Post by field-dog 1 year ago

9 Posts

4 Users

2 Reactions

2,384 Views

RSS

Dan Steely

(@dan-steely)

Posts: 53

Free

Topic starter

Translate ▼

IT'S ALIVE! ChatGPT and I spent all day getting the vectors and covariances from a FieldGenius RAW file into a TBC importable RW5 (TDS style). Copy and paste code into an ascii file; RAW2RW5.py, to run: "python RAW2RW5.py "input.raw" "output.rw5". It is definitely not perfect, and has issues separating different base setups and base coordinates. The vectors come in correctly though and that's all I care about (for right now). If you want to test it out and help me refine it, be my guest!

#!/usr/bin/env python3
import re, math, sys
from pathlib import Path

USFT2M = 0.3048006096012192
A_WGS84 = 6378137.0
F_INV = 298.257223563
F = 1.0 / F_INV
E2 = F*(2-F)

def geodetic_to_ecef(lat_deg, lon_deg, h_m):
lat = math.radians(lat_deg); lon = math.radians(lon_deg)
sl, cl = math.sin(lat), math.cos(lat)
sb, cb = math.sin(lon), math.cos(lon)
N = A_WGS84 / math.sqrt(1.0 - E2 * sl * sl)
X = (N + h_m) * cl * cb
Y = (N + h_m) * cl * sb
Z = (N*(1 - E2) + h_m) * sl
return (X, Y, Z)

def ecef_up_unit(lat_deg, lon_deg):
lat = math.radians(lat_deg); lon = math.radians(lon_deg)
sl, cl = math.sin(lat), math.cos(lat)
sb, cb = math.sin(lon), math.cos(lon)
return (cl*cb, cl*sb, sl)

def parse_units(mo_line: str):
# UN1: meters, UN2: US survey feet; UN3 (rare): international feet
m = re.search(r'(UN\d)', mo_line or '')
un = m.group(1) if m else 'UN1'
if un == 'UN1': return un, 1.0
if un == 'UN2': return un, USFT2M
if un == 'UN3': return un, 0.3048
return un, 1.0

def last_AH_before(idx, AH_list):
last = None
for ii,MA,RA in AH_list:
if ii < idx: last = (ii,MA,RA)
else: break
return last

def main(inp: str, outp: str):
raw = Path(inp).read_text(errors='ignore').splitlines()

mo_line = next((ln for ln in raw if ln.startswith('MO,')), None)
un_tag, to_m = parse_units(mo_line or '')
mo_fallback = f"MO,AD0,{un_tag},SF1.000000,EC0,EO0.0" # numeric-enum fallback (TBC-friendly)

jb = next((ln for ln in raw if ln.startswith('JB,')), None)
job_date = None
if jb:
mdt = re.match(r'JB,[^,]*,DT(\d{2}-\d{2}-\d{4}),TM(\d{2}:\d{2}:\d{2})', jb)
if mdt: job_date = mdt.group(1).replace('-', '/')

# Parse AH records (idx, MA, RA) for base (DC1) and rover (DC2)
AH_base, AH_rover = [], []
for i, ln in enumerate(raw):
if ln.startswith('AH,'):
m = re.match(r'AH,DC(\d),MA([-0-9\.]+),ME\d+,RA([-0-9\.]+)', ln)
if m:
dc = m.group(1); MA=float(m.group(2)); RA=float(m.group(3))
(AH_base if dc=='1' else AH_rover).append((i,MA,RA))

# Bases (BP) with ECEF(APC) and attached base AH prior to BP
BPs = []
for i, ln in enumerate(raw):
if ln.startswith('BP,PN'):
m = re.match(r'BP,PN([^,]+),LA([-0-9\.]+),LN([-0-9\.]+),HT([-0-9\.]+)', ln)
if m:
pn = m.group(1); la=float(m.group(2)); lnw=float(m.group(3)); ht=float(m.group(4))
ecef_apc = geodetic_to_ecef(la, lnw, ht*to_m)
ahb = last_AH_before(i, AH_base)
BPs.append({'idx': i, 'pn': pn, 'la': la, 'ln': lnw, 'ht_apc': ht,
'ecef_apc': ecef_apc, 'AH': ahb})

if not BPs:
raise SystemExit('No BP records found.')

def last_bp_before(idx):
last = None
for b in BPs:
if b['idx'] < idx: last = b
else: break
return last or BPs[-1]

# Helper to capture CV right after BL
def next_CV_after_idx(idx):
j = idx + 1
while j < len(raw):
s = raw[j]
if s.startswith('BL,') or s.startswith('EP,'):
return None
if s.startswith('CV,'):
mcv = re.search(r'XX([-0-9\.Ee]+),XY([-0-9\.Ee]+),XZ([-0-9\.Ee]+),YY([-0-9\.Ee]+),YZ([-0-9\.Ee]+),ZZ([-0-9\.Ee]+)', s)
msc = re.search(r'SC([-0-9\.Ee]+)', s)
if mcv:
xx,xy,xz,yy,yz,zz = map(float, mcv.groups())
sc = float(msc.group(1)) if msc else 1.0
u2 = (USFT2M**2) if un_tag in ('UN2','UN3') else 1.0
s2 = u2 * (sc*sc)
return [[xx*s2, xy*s2, xz*s2],
[xy*s2, yy*s2, yz*s2],
[xz*s2, yz*s2, zz*s2]]
j += 1
return None

out = []
if jb: out.append(jb)
out.append(mo_line if mo_line else mo_fallback)

# Emit BP with AG=RA so HI shows in TBC (EL left as recorded APC ellipsoid)
for b in BPs:
AG_RA = (b['AH'][2] if b['AH'] else 0.0)
out.append(f"BP,PN{b['pn']},LA{b['la']:.10f},LN{b['ln']:.10f},EL{b['ht_apc']:.4f},AG{AG_RA:.3f},PA0.0000,SRBASE,--")

ep_re = re.compile(r'EP,TM(\d{2}:\d{2}:\d{2}(?:\.\d+)?),\s*LA([-0-9\.]+),LN([-0-9\.]+),HT([-0-9\.]+)')
bl_re = re.compile(r'BL,(?:[^,]*,)*PN([^,]+),')

i = 0
while i < len(raw):
if raw[i].startswith('EP,'):
mep = ep_re.match(raw[i])
if not mep: i += 1; continue
tm = mep.group(1); r_la=float(mep.group(2)); r_ln=float(mep.group(3)); r_ht_apc=float(mep.group(4))
rover_ecef_apc = geodetic_to_ecef(r_la, r_ln, r_ht_apc*to_m)
u_up = ecef_up_unit(r_la, r_ln)

base = last_bp_before(i)
base_code = base['pn']
base_ecef_apc = base['ecef_apc']
base_AG_RA = (base['AH'][2] if base['AH'] else 0.0)

j = i + 1
while j < len(raw) and not raw[j].startswith('EP,'):
s = raw[j]
if s.startswith('BL,'):
mbl = bl_re.match(s)
if mbl:
pn = mbl.group(1)
# rover HI from last AH before EP/BL (same "good" behavior)
ahr = last_AH_before(i, AH_rover)
rover_HR_RA = ahr[2] if ahr else 0.0

dx0 = rover_ecef_apc[0] - base_ecef_apc[0]
dy0 = rover_ecef_apc[1] - base_ecef_apc[1]
dz0 = rover_ecef_apc[2] - base_ecef_apc[2]

# pre-comp +(HR-AG) along Up at rover
delta_up_m = (rover_HR_RA - base_AG_RA) * (USFT2M if un_tag in ('UN2','UN3') else 1.0)
dx = dx0 + delta_up_m * u_up[0]
dy = dy0 + delta_up_m * u_up[1]
dz = dz0 + delta_up_m * u_up[2]

g0 = f"G0,{job_date or '2025/01/01'} {tm},Base ID read at rover: {base_code}"
out.append(f"LS,HR{rover_HR_RA:.3f}")
out.append(f"GPS,PN{pn},LA{r_la:.10f},LN{r_ln:.10f},EL{r_ht_apc:.4f},--")
out.append(g0)
out.append(f"G1,BP{base_code},PN{pn},DX{dx:.6f},DY{dy:.6f},DZ{dz:.6f}")

C = next_CV_after_idx(j)
if C:
out.append(f"G2,VX{C[0][0]:.11f},VY{C[1][1]:.11f},VZ{C[2][2]:.11f}")
out.append(f"G3,XY{C[0][1]:.11f},XZ{C[0][2]:.11f},YZ{C[1][2]:.11f}")
out.append("G4,BVAPC,RVAPC")
j += 1
i = j
else:
i += 1

Path(outp).write_text('\n'.join(out))
print(f'Wrote {outp}')

if __name__ == '__main__':
if len(sys.argv) < 3:
print('Usage: python fg_raw_to_rw5_precomp.py input.raw output.rw5')
sys.exit(2)
main(sys.argv[1], sys.argv[2])

Posted : September 10, 2025 11:00 pm

hpalmer

(@hpalmer)

Posts: 513

Member P&R, Founder

Translate ▼

surveying has always been on the forefront of technology, some of the first to use distance meters, electronic theodolites, data collectors, gps/gnss, field data collection programs, cad programs. We are in the next phase to include coding to make this stuff work better across different platforms, proprietary manufactures, and different languages. Thanks for posting and I hope someone takes you up on your efforts. Just fyi, can you ask chat to compile a py script to (using the code already written for TBC) convert fieldgenius raw to Microstation or OpenRoads and post same?

Posted : September 11, 2025 4:12 pm

field-dog

(@field-dog)

Posts: 1582

Member

Translate ▼

Posted by: @hpalmer

↑

We are in the next phase to include coding to make this stuff work better across different platforms, proprietary manufactures, and different languages.

I have always been an advocate of this and have dabbled in it for years.

Posted : September 13, 2025 4:01 am

field-dog

(@field-dog)

Posts: 1582

Member

Translate ▼

Posted by: @dan-steely

↑

ChatGPT and I spent all day getting the vectors and covariances from a FieldGenius RAW file into a TBC importable RW5 (TDS style).

Can you post a small piece of a FieldGenius .raw file?

Posted : September 14, 2025 2:48 pm

Landbutcher464MHz

(@landbutcher464mhz)

Posts: 263

Member P&R

Translate ▼

Can you post a small piece of a FieldGenius .raw file?

Here is a piece of a GPS file.

--FieldGenius v 10.4.1.2 (2020-08-13)
JB,NM4th Ave Bdy 03,DT06-20-2024,TM22:12:17
MO,AD2,UN2,SF1.0000000000,EC1,EO0.0,AU0
CS,CO3,ZGState Planes NAD83,ZNCA83-VI,DN
VA,PV3,N 0.0000,E 0.0000,LZ0.0000,SO0.0000000000000,SA0.0000000000000,GN3:g2018u5.bin
--Projection Parameters: Standard Parallel 1=N33°53'00.00000",Standard Parallel 2=N32°47'00.00000",Central Meridian=W116°15'00.00000",Latitude of Origin=N32°10'00.00000",False Easting (m)=2000000.000000,False Northing (m)=500000.000000
--Datum Parameters: Delta X (m)=0.000000,Delta Y (m)=0.000000,Delta Z (m)=0.000000
--Ellipsoid Parameters: Equatorial Radius (a) (m)=6378137.000000,Polar Radius (b) (m)=6356752.314140,Inverse Flattening (1/f)=298.257222
SP,PN41,N 1855258.5411,E 6281496.4501,EL0.0000,--OFST
--7x7'o/sCR.10505
SP,PN54,N 1854601.7186,E 6281495.6873,EL0.0000,--OFST
--7'o/sCR.28099
--FieldGenius v 11.4.0.524 (2022-10-06)
JB,NM4th Ave Bdy 03,DT06-22-2024,TM13:14:24
MO,AD2,UN2,SF1.0000000000,EC1,EO0.0,AU0
--Instrument Selected: Type=GNSS,Profile=S850A NET,Model=S850A
--GNSS Profile Tolerance RT: Obs=24,Solution=RTK Fixed,Elev=10,PDOP=2.2,SVs=8,RefID=Any,StdDevH=0.2000',StdDevV=0.2500'
--GNSS Profile Tolerance PP: Not Active
--GNSS Statistics RT: Obs=24,Solution=RTK Fixed,PDOPMax=1.10,SVMin=10,StdDevH=0.0032',StdDevV=0.0071',RefID=0001,RefLat=N32°53'21.14663",RefLon=W117°06'16.86185",RefHgt=454.6654',RefFormat=,MountPoint=P472_RTCM3,Tilt=0.68,TiltCompensated=No
--GNSS Statistics PP: Not Active
--PP Time: StartWeek=2319,StartSec=591431.0,StopWeek=2319,StopSec=591459.0
--Antenna: Desc=S850A,True=8.2290',Meas=8.0000',ARP_V=0.0mm,ARP_H=0.0mm,NGS_ID="STXS700A NONE",NGS_L1=69.8mm,NGS_L2=63.3mm
AH,DC2,MA8.000,ME2,RA8.229
JB,NM4th Ave Bdy 03,DT06-22-2024,TM13:18:08
EP,TM20:17:39.0000,LA32.75196370299,LN-117.16125987667,HT172.3641,RN0.0025,RE0.0019,RV0.0071,DH0.6,DV1.0,GM4,CL1
BL,DCROVER,PN154,DX-27704.4105,DY-15949.3651,DZ-42118.3793,--FD,GM4,CL1,HP0.003,VP0.007
CV,DCROVER,SV10,SC1,XX0.00018480817,XY0.00023359379,XZ-0.00011880925,YY0.00043900254,YZ-0.00018636429,ZZ0.00016556704
GS,PN154,N 1854603.9950,E 6281493.2792,EL286.7774,--FD
--L+D
--GNSS Statistics RT: Obs=24,Solution=RTK Fixed,PDOPMax=1.00,SVMin=13,StdDevH=0.0023',StdDevV=0.0043',RefID=0001,RefLat=N32°53'21.14663",RefLon=W117°06'16.86185",RefHgt=454.6654',RefFormat=,MountPoint=P472_RTCM3,Tilt=0.50,TiltCompensated=No
--GNSS Statistics PP: Not Active
--PP Time: StartWeek=2319,StartSec=591809.0,StopWeek=2319,StopSec=591837.0
JB,NM4th Ave Bdy 03,DT06-22-2024,TM13:24:36
EP,TM20:23:57.0000,LA32.75377003200,LN-117.16127646726,HT182.2308,RN0.0018,RE0.0015,RV0.0043,DH0.5,DV0.8,GM4,CL1
BL,DCROVER,PN141,DX-27550.4181,DY-15638.0591,DZ-41560.3041,--FD,GM4,CL1,HP0.002,VP0.004
CV,DCROVER,SV13,SC1,XX0.00005728671,XY0.00006513254,XZ-0.00004141000,YY0.00015796092,YZ-0.00007198954,ZZ0.00008863287
GS,PN141,N 1855261.2410,E 6281493.9225,EL296.6204,--FD
--L+D

Posted : September 14, 2025 5:22 pm

Landbutcher464MHz

(@landbutcher464mhz)

Posts: 263

Member P&R

Translate ▼

Can you post a small piece of a FieldGenius .raw file?

Here is a piece of a total station (robotic) raw file.

--FieldGenius v 10.4.1.2 (2020-08-13)
JB,NM11th St Bdy 02,DT07-25-2024,TM16:54:31
MO,AD2,UN2,SF1.0000000000,EC1,EO0.0,AU0
CS,CO3,ZGState Planes NAD83,ZNCA83-VI,DN
VA,PV3,N 0.0000,E 0.0000,LZ0.0000,SO0.0000000000000,SA0.0000000000000,GN3:g2018u5.bin
--Projection Parameters: Standard Parallel 1=N33°53'00.00000",Standard Parallel 2=N32°47'00.00000",Central Meridian=W116°15'00.00000",Latitude of Origin=N32°10'00.00000",False Easting (m)=2000000.000000,False Northing (m)=500000.000000
--Datum Parameters: Delta X (m)=0.000000,Delta Y (m)=0.000000,Delta Z (m)=0.000000
--Ellipsoid Parameters: Equatorial Radius (a) (m)=6378137.000000,Polar Radius (b) (m)=6356752.314140,Inverse Flattening (1/f)=298.257222
SP,PN600,N 1790973.8549,E 6296571.2140,EL16.3530,--ROBOT
--BLUE"X" SE.PCR
SP,PN613,N 1790975.7272,E 6296720.6982,EL16.8304,--CN
--FieldGenius v 11.4.0.524 (2022-10-06)
JB,NM11th St Bdy 02,DT08-21-2024,TM10:47:47
MO,AD2,UN2,SF1.0000000000,EC1,EO0.0,AU0
--Instrument Selected: Type=Total Station,Profile=TS15,Model=Robotic
--EDM Mode: IR Standard
--Target selected: Leica Mini Prism Leica Constant: 17.5mm
--Orientation
LS,HI5.039,HR4.900
OC,OP613,N 1790975.7272,E 6296720.6982,EL16.8304,--CN
--setup_time,DT08-21-2024,TM10:59:31
BK,OP613,BP600,BS269.16567,BC0.00000
BR,OP613,BP600,AR0.00000,ZE90.13322,SD149.4940
--Orientation Notes
-- Observed Values: HA 0°00'00" VA 90°13'32" SD 149.4940' HD 149.4928' HR 4.9000'
-- Observed Reference: Point
-- Distance Calculated: 149.4959'
-- Distance Error: -0.0031'
-- BS Elevation: 16.3811'
-- BS Elevation Error: 0.0281'
JB,NMMay 11th St 1021 Bdy 02,DT08-21-2024,TM11:44:32
SS,OP613,FP301,AR91.42187,ZE88.56062,SD561.9898,--TOPO
--GAS.GV
JB,NMMay 11th St 1021 Bdy 02,DT08-21-2024,TM11:47:00
SS,OP613,FP302,AR93.04192,ZE89.07557,SD311.5153,--TOPO
--SMH.RIM

Posted : September 14, 2025 5:25 pm

field-dog

(@field-dog)

Posts: 1582

Member

Translate ▼

Posted by: @landbutcher464mhz

↑

PN154,DX-27704.4105,DY-15949.3651,DZ-42118.3793

PN141,DX-27550.4181,DY-15638.0591,DZ-41560.3041

Vector data for point numbers 154 and 141?

Posted : September 14, 2025 7:32 pm

Landbutcher464MHz

(@landbutcher464mhz)

Posts: 263

Member P&R

Translate ▼

Posted by: @field-dog

↑

Posted by: @landbutcher464mhz

↑

PN154,DX-27704.4105,DY-15949.3651,DZ-42118.3793

PN141,DX-27550.4181,DY-15638.0591,DZ-41560.3041

Vector data for point numbers 154 and 141?