Refactor everything

Huge dump of refactored code. Still in the middle of the changes that
are to be squashed later in a one huge monster commit, because there is
no value in anything atomic here.

rottlib/tests/common.rs

@@ -0,0 +1,63 @@
+use std::path::{Path, PathBuf};
+
+use rottlib::lexer::{Token, TokenData, TokenPosition, TokenizedFile};
+
+pub fn fixture_path(name: &str) -> PathBuf {
+    Path::new(env!("CARGO_MANIFEST_DIR"))
+        .join("tests")
+        .join("fixtures")
+        .join(name)
+}
+
+pub fn read_fixture(name: &str) -> String {
+    let path = fixture_path(name);
+    std::fs::read_to_string(&path)
+        .unwrap_or_else(|e| panic!("failed to read fixture {}: {e}", path.display()))
+}
+
+pub fn with_fixture(name: &str, f: impl for<'src> FnOnce(&'src str, TokenizedFile<'src>)) {
+    let source = read_fixture(name);
+    let file = TokenizedFile::tokenize(&source);
+    f(&source, file);
+}
+
+pub fn line_lexemes<'file, 'src>(file: &'file TokenizedFile<'src>, line: usize) -> Vec<&'src str> {
+    file.line_tokens(line).map(|(_, t)| t.lexeme).collect()
+}
+
+pub fn line_tokens<'src>(file: &TokenizedFile<'src>, line: usize) -> Vec<Token> {
+    file.line_tokens(line).map(|(_, t)| t.token).collect()
+}
+
+pub fn line_positions<'src>(file: &TokenizedFile<'src>, line: usize) -> Vec<TokenPosition> {
+    file.line_tokens(line).map(|(pos, _)| pos).collect()
+}
+
+pub fn line_pairs<'file, 'src>(
+    file: &'file TokenizedFile<'src>,
+    line: usize,
+) -> Vec<(Token, &'src str)> {
+    file.line_tokens(line)
+        .map(|(_, t)| (t.token, t.lexeme))
+        .collect()
+}
+
+pub fn all_lexemes<'file, 'src>(file: &'file TokenizedFile<'src>) -> Vec<&'src str> {
+    file.iter().map(|(_, t)| t.lexeme).collect()
+}
+
+pub fn all_tokens<'src>(file: &TokenizedFile<'src>) -> Vec<Token> {
+    file.iter().map(|(_, t)| t.token).collect()
+}
+
+pub fn token_at<'src>(file: &TokenizedFile<'src>, index: usize) -> Option<TokenData<'src>> {
+    file.token_at(TokenPosition(index))
+}
+
+pub fn reconstruct_source<'file, 'src>(file: &'file TokenizedFile<'src>) -> String {
+    file.iter().map(|(_, t)| t.lexeme).collect()
+}
+
+pub fn find_line<'src>(file: &TokenizedFile<'src>, needle: &str) -> Option<usize> {
+    (0..file.line_count()).find(|&line| file.line_text(line).as_deref() == Some(needle))
+}

rottlib/tests/fixtures/KVehicle.uc

@@ -0,0 +1,326 @@
+// Generic 'Karma Vehicle' base class that can be controlled by a Pawn.
+
+class KVehicle extends Vehicle
+    native
+    abstract;
+
+cpptext
+{
+#ifdef WITH_KARMA
+	virtual void PostNetReceive();
+    virtual void PostEditChange();
+	virtual void setPhysics(BYTE NewPhysics, AActor *NewFloor, FVector NewFloorV);
+	virtual void TickSimulated( FLOAT DeltaSeconds );
+	virtual void TickAuthoritative( FLOAT DeltaSeconds );
+#endif
+
+}
+
+// Effect spawned when vehicle is destroyed
+var (KVehicle) class<Actor>	DestroyEffectClass;
+
+// Simple 'driving-in-rings' logic.
+var (KVehicle) bool		bAutoDrive;
+
+// The factory that created this vehicle.
+//var			   KVehicleFactory	ParentFactory;
+
+// Weapon system
+var				bool	bVehicleIsFiring, bVehicleIsAltFiring;
+
+const					FilterFrames = 5;
+var				vector	CameraHistory[FilterFrames];
+var				int		NextHistorySlot;
+var				bool	bHistoryWarmup;
+
+// Useful function for plotting data to real-time graph on screen.
+native final function GraphData(string DataName, float DataValue);
+
+// if _RO_
+function TakeDamage(int Damage, Pawn instigatedBy, Vector hitlocation,
+						vector momentum, class<DamageType> damageType, optional int HitIndex)
+// else UT
+//function TakeDamage(int Damage, Pawn instigatedBy, Vector hitlocation,
+//						vector momentum, class<DamageType> damageType)
+{
+	Super.TakeDamage(Damage,instigatedBy,HitLocation,Momentum,DamageType);
+}
+
+// You got some new info from the server (ie. VehicleState has some new info).
+event VehicleStateReceived();
+
+// Called when a parameter of the overall articulated actor has changed (like PostEditChange)
+// The script must then call KUpdateConstraintParams or Actor Karma mutators as appropriate.
+simulated event KVehicleUpdateParams();
+
+// The pawn Driver has tried to take control of this vehicle
+function bool TryToDrive(Pawn P)
+{
+	if ( P.bIsCrouched || (P.Controller == None) || (Driver != None) || !P.Controller.bIsPlayer )
+		return false;
+
+    if ( !P.IsHumanControlled() || !P.Controller.IsInState('PlayerDriving') )
+	{
+		KDriverEnter(P);
+		return true;
+	}
+
+	return false;
+}
+
+// Events called on driver entering/leaving vehicle
+
+simulated function ClientKDriverEnter(PlayerController pc)
+{
+	pc.myHUD.bCrosshairShow = false;
+	pc.myHUD.bShowWeaponInfo = false;
+	pc.myHUD.bShowPoints = false;
+
+	pc.bBehindView = true;
+	pc.bFreeCamera = true;
+
+    pc.SetRotation(rotator( vect(-1, 0, 0) >> Rotation ));
+}
+
+function KDriverEnter(Pawn P)
+{
+	local PlayerController PC;
+	local Controller C;
+
+	// Set pawns current controller to control the vehicle pawn instead
+	Driver = P;
+
+	// Move the driver into position, and attach to car.
+	Driver.SetCollision(false, false);
+	Driver.bCollideWorld = false;
+	Driver.bPhysicsAnimUpdate = false;
+	Driver.Velocity = vect(0,0,0);
+	Driver.SetPhysics(PHYS_None);
+	Driver.SetBase(self);
+
+	// Disconnect PlayerController from Driver and connect to KVehicle.
+	C = P.Controller;
+	p.Controller.Unpossess();
+	Driver.SetOwner(C); // This keeps the driver relevant.
+	C.Possess(self);
+
+	PC = PlayerController(C);
+	if ( PC != None )
+	{
+		PC.ClientSetViewTarget(self); // Set playercontroller to view the vehicle
+
+		// Change controller state to driver
+		PC.GotoState('PlayerDriving');
+
+		ClientKDriverEnter(PC);
+	}
+}
+
+simulated function ClientKDriverLeave(PlayerController pc)
+{
+	pc.bBehindView = false;
+	pc.bFreeCamera = false;
+	// This removes any 'roll' from the look direction.
+	//exitLookDir = Vector(pc.Rotation);
+	//pc.SetRotation(Rotator(exitLookDir));
+
+    pc.myHUD.bCrosshairShow = pc.myHUD.default.bCrosshairShow;
+	pc.myHUD.bShowWeaponInfo = pc.myHUD.default.bShowWeaponInfo;
+	pc.myHUD.bShowPoints = pc.myHUD.default.bShowPoints;
+
+	// Reset the view-smoothing
+	NextHistorySlot = 0;
+	bHistoryWarmup = true;
+}
+
+// Called from the PlayerController when player wants to get out.
+function bool KDriverLeave(bool bForceLeave)
+{
+	local PlayerController pc;
+	local int i;
+	local bool havePlaced;
+	local vector HitLocation, HitNormal, tryPlace;
+
+	// Do nothing if we're not being driven
+	if(Driver == None)
+		return false;
+
+	// Before we can exit, we need to find a place to put the driver.
+	// Iterate over array of possible exit locations.
+
+	if (!bRemoteControlled)
+    {
+
+	    Driver.bCollideWorld = true;
+	    Driver.SetCollision(true, true);
+
+	    havePlaced = false;
+	    for(i=0; i < ExitPositions.Length && havePlaced == false; i++)
+	    {
+	        //Log("Trying Exit:"$i);
+
+	        tryPlace = Location + (ExitPositions[i] >> Rotation);
+
+	        // First, do a line check (stops us passing through things on exit).
+	        if( Trace(HitLocation, HitNormal, tryPlace, Location, false) != None )
+	            continue;
+
+	        // Then see if we can place the player there.
+	        if( !Driver.SetLocation(tryPlace) )
+	            continue;
+
+	        havePlaced = true;
+	    }
+
+	    // If we could not find a place to put the driver, leave driver inside as before.
+	    if(!havePlaced && !bForceLeave)
+	    {
+	        Log("Could not place driver.");
+
+	        Driver.bCollideWorld = false;
+	        Driver.SetCollision(false, false);
+
+	        return false;
+	    }
+	}
+
+	pc = PlayerController(Controller);
+	ClientKDriverLeave(pc);
+
+	// Reconnect PlayerController to Driver.
+	pc.Unpossess();
+	pc.Possess(Driver);
+
+	pc.ClientSetViewTarget(Driver); // Set playercontroller to view the persone that got out
+
+	Controller = None;
+
+	Driver.PlayWaiting();
+	Driver.bPhysicsAnimUpdate = Driver.Default.bPhysicsAnimUpdate;
+
+	// Do stuff on client
+	//pc.ClientSetBehindView(false);
+	//pc.ClientSetFixedCamera(true);
+
+	if (!bRemoteControlled)
+    {
+
+	    Driver.Acceleration = vect(0, 0, 24000);
+		Driver.SetPhysics(PHYS_Falling);
+		Driver.SetBase(None);
+	}
+
+	// Car now has no driver
+	Driver = None;
+
+	// Put brakes on before you get out :)
+    Throttle=0;
+    Steering=0;
+
+	// Stop firing when you get out!
+	bVehicleIsFiring = false;
+	bVehicleIsAltFiring = false;
+
+    return true;
+}
+
+// Special calc-view for vehicles
+simulated function bool SpecialCalcView(out actor ViewActor, out vector CameraLocation, out rotator CameraRotation )
+{
+	local vector CamLookAt, HitLocation, HitNormal;
+	local PlayerController pc;
+	local int i, averageOver;
+
+	pc = PlayerController(Controller);
+
+	// Only do this mode we have a playercontroller viewing this vehicle
+	if(pc == None || pc.ViewTarget != self)
+		return false;
+
+	ViewActor = self;
+	CamLookAt = Location + (vect(-100, 0, 100) >> Rotation);
+
+	//////////////////////////////////////////////////////
+	// Smooth lookat position over a few frames.
+	CameraHistory[NextHistorySlot] = CamLookAt;
+	NextHistorySlot++;
+
+	if(bHistoryWarmup)
+		averageOver = NextHistorySlot;
+	else
+		averageOver = FilterFrames;
+
+	CamLookAt = vect(0, 0, 0);
+	for(i=0; i<averageOver; i++)
+		CamLookAt += CameraHistory[i];
+
+	CamLookAt /= float(averageOver);
+
+	if(NextHistorySlot == FilterFrames)
+	{
+		NextHistorySlot = 0;
+		bHistoryWarmup=false;
+	}
+	//////////////////////////////////////////////////////
+
+	CameraLocation = CamLookAt + (vect(-600, 0, 0) >> CameraRotation);
+
+	if( Trace( HitLocation, HitNormal, CameraLocation, CamLookAt, false, vect(10, 10, 10) ) != None )
+	{
+		CameraLocation = HitLocation;
+	}
+
+	return true;
+}
+
+simulated function Destroyed()
+{
+	// If there was a driver in the vehicle, destroy him too
+	if ( Driver != None )
+		Driver.Destroy();
+
+	// Trigger any effects for destruction
+	if(DestroyEffectClass != None)
+		spawn(DestroyEffectClass, , , Location, Rotation);
+
+	Super.Destroyed();
+}
+
+simulated event Tick(float deltaSeconds)
+{
+}
+
+// Includes properties from KActor
+defaultproperties
+{
+    Steering=0
+    Throttle=0
+
+	ExitPositions(0)=(X=0,Y=0,Z=0)
+
+	DrivePos=(X=0,Y=0,Z=0)
+	DriveRot=()
+
+	bHistoryWarmup = true;
+
+
+    Physics=PHYS_Karma
+	bEdShouldSnap=True
+	bStatic=False
+	bShadowCast=False
+	bCollideActors=True
+	bCollideWorld=False
+    bProjTarget=True
+	bBlockActors=True
+	bBlockNonZeroExtentTraces=True
+	bBlockZeroExtentTraces=True
+	bWorldGeometry=False
+	bBlockKarma=True
+	bAcceptsProjectors=True
+	bCanBeBaseForPawns=True
+	bAlwaysRelevant=True
+	RemoteRole=ROLE_SimulatedProxy
+	bNetInitialRotation=True
+	bSpecialCalcView=True
+	//bSpecialHUD=true
+}

rottlib/tests/fixtures_tokenization.rs

@@ -0,0 +1,135 @@
+use std::{fs, path::PathBuf};
+
+use rottlib::lexer::{Keyword, Token, TokenizedFile};
+
+/// Returns the path to a fixture file in `tests/fixtures/`.
+fn fixture_file_path(name: &str) -> PathBuf {
+    PathBuf::from(env!("CARGO_MANIFEST_DIR"))
+        .join("tests")
+        .join("fixtures")
+        .join(name)
+}
+
+/// Loads a fixture source file as UTF-8 text.
+fn read_fixture_source(name: &str) -> String {
+    fs::read_to_string(fixture_file_path(name))
+        .unwrap_or_else(|e| panic!("failed to read fixture {name}: {e}"))
+}
+
+/// Returns the token at the given token index on a physical line.
+///
+/// Here `line` is 1-based, to match human line numbers in fixture files.
+/// `token_index` is 0-based within `TokenizedFile::line_tokens`.
+fn token_on_line(file: &TokenizedFile<'_>, line: usize, token_index: usize) -> Option<Token> {
+    file.line_tokens(line - 1)
+        .nth(token_index)
+        .map(|(_, token_data)| token_data.token)
+}
+
+/// Returns reconstructed visible text for a physical line.
+///
+/// Here `line` is 1-based, to match human line numbers in fixture files.
+fn line_text(file: &TokenizedFile<'_>, line: usize) -> Option<String> {
+    file.line_text(line - 1)
+}
+
+#[test]
+fn command_api_fixture_queries() {
+    let source = read_fixture_source("CommandAPI.uc");
+    let file = TokenizedFile::tokenize(&source);
+    assert_eq!(file.line_count(), 1578);
+
+    assert_eq!(
+        line_text(&file, 704).as_deref(),
+        Some(
+            "public final function CommandConfigInfo ResolveCommandForUserID(BaseText itemName, UserID id) {"
+        )
+    );
+    assert_eq!(
+        line_text(&file, 806).as_deref(),
+        Some("        _.memory.Free(wrapper);")
+    );
+    assert_eq!(
+        line_text(&file, 1274).as_deref(),
+        Some("/// Method must be called after [`Voting`] with a given name is added.")
+    );
+    assert_eq!(
+        line_text(&file, 14).as_deref(),
+        Some(" * Acedia is distributed in the hope that it will be useful,")
+    );
+
+    let token = token_on_line(&file, 22, 0).unwrap();
+    assert_eq!(token, Token::Keyword(Keyword::Class));
+
+    let token = token_on_line(&file, 1577, 0).unwrap();
+    assert_eq!(token, Token::Keyword(Keyword::DefaultProperties));
+
+    let token = token_on_line(&file, 649, 4).unwrap();
+    assert_eq!(token, Token::Whitespace);
+}
+
+#[test]
+fn dbrecord_fixture_queries() {
+    let source = read_fixture_source("DBRecord.uc");
+    let file = TokenizedFile::tokenize(&source);
+    assert_eq!(file.line_count(), 1199);
+
+    assert_eq!(
+        line_text(&file, 149).as_deref(),
+        Some(" *      However, JSON pointers are not convenient or efficient enough for that,")
+    );
+    assert_eq!(
+        line_text(&file, 787).as_deref(),
+        Some(" *      3. 'number'     -> either `IntBox` or `FloatBox`, depending on")
+    );
+    assert_eq!(
+        line_text(&file, 1023).as_deref(),
+        Some("    bool        makeMutable)")
+    );
+    assert_eq!(
+        line_text(&file, 29).as_deref(),
+        Some("    config(AcediaDB);")
+    );
+
+    let token = token_on_line(&file, 565, 0).unwrap();
+    assert_eq!(token, Token::BlockComment);
+
+    let token = token_on_line(&file, 467, 10).unwrap();
+    assert_eq!(token, Token::Identifier);
+
+    let token = token_on_line(&file, 467, 9).unwrap();
+    assert_eq!(token, Token::LeftParenthesis);
+}
+
+#[test]
+fn kvehicle_fixture_queries() {
+    let source = read_fixture_source("KVehicle.uc");
+    let file = TokenizedFile::tokenize(&source);
+    assert_eq!(file.line_count(), 326);
+
+    assert_eq!(
+        line_text(&file, 12).as_deref(),
+        Some("	virtual void setPhysics(BYTE NewPhysics, AActor *NewFloor, FVector NewFloorV);")
+    );
+    assert_eq!(
+        line_text(&file, 127).as_deref(),
+        Some("    pc.myHUD.bCrosshairShow = pc.myHUD.default.bCrosshairShow;")
+    );
+    assert_eq!(
+        line_text(&file, 264).as_deref(),
+        Some("	//////////////////////////////////////////////////////")
+    );
+    assert_eq!(
+        line_text(&file, 299).as_deref(),
+        Some("	ExitPositions(0)=(X=0,Y=0,Z=0)")
+    );
+
+    let token = token_on_line(&file, 17, 0).unwrap();
+    assert_eq!(token, Token::Newline);
+
+    let token = token_on_line(&file, 20, 7).unwrap();
+    assert_eq!(token, Token::Less);
+
+    let token = token_on_line(&file, 246, 2).unwrap();
+    assert_eq!(token, Token::Increment);
+}