Reconnection System

Automatic reconnection and recovery mechanisms for high availability music playback

Configuration

Basic Reconnection Setup

from salada import Salad

NODES = [{
    'host': 'localhost',
    'port': 2333,
    'auth': 'youshallnotpass',
    'ssl': False
}]

salad = Salad(client, NODES, {
    'enableReconnect': True,
    'stateSaveInterval': 5.0,
    'maxReconnectAttempts': 5,
    'infiniteReconnect': True,
    'baseReconnectDelay': 1.0,
    'maxReconnectDelay': 300.0
})

await salad.start(NODES, str(client.user.id))

Reconnection Configuration Options

Available Options

opts = {
    'enableReconnect': True,
    'stateSaveInterval': 5.0,
    'maxReconnectAttempts': 5,
    'infiniteReconnect': True,
    'baseReconnectDelay': 1.0,
    'maxReconnectDelay': 300.0
}

Option Descriptions

• enableReconnect: Enable automatic reconnection and state management
• stateSaveInterval: How often to save player states (seconds)
• maxReconnectAttempts: Maximum reconnection attempts before giving up
• infiniteReconnect: Keep trying to reconnect indefinitely
• baseReconnectDelay: Initial delay between reconnection attempts (seconds)
• maxReconnectDelay: Maximum delay between reconnection attempts (seconds)

Event Handling

Reconnection Events

@bot.salad.on('nodeConnect')
def on_node_connect(node):
    print(f'Node connected: {node.host}:{node.port}')

@bot.salad.on('nodeDisconnect')
def on_node_disconnect(node):
    print(f'Node disconnected: {node.host}:{node.port}')

@bot.salad.on('nodeReconnecting')
def on_node_reconnecting(node, attempt, delay):
    print(f'Node reconnecting (attempt {attempt}), waiting {delay}s')

@bot.salad.on('nodeReconnectExhausted')
def on_reconnect_exhausted(node, attempts):
    print(f'Node exhausted reconnection after {attempts} attempts')

@bot.salad.on('playersRestored')
def on_players_restored(node, count):
    print(f'Restored {count} players on {node.host}')

@bot.salad.on('nodeError')
def on_node_error(node, error):
    print(f'Node error on {node.host}: {error}')

@bot.salad.on('playerPositionUpdate')
def on_position_update(player, state):
    print(f'Player position: {state.get("position")}ms')

@bot.salad.on('nodeReady')
def on_node_ready(node, data):
    print(f'Node ready: {node.sessionId}')

@bot.salad.on('nodeStats')
def on_node_stats(node, stats):
    print(f'Node stats - Players: {stats.get("players")}')

Node Management

Check Node Status

def get_node_status():
    node = bot.salad.nodes[0]
    
    status = {
        'host': f'{node.host}:{node.port}',
        'connected': node.connected,
        'sessionId': node.sessionId,
        'players': len(node.players),
        'stats': node.stats
    }
    
    return status

print('Node Status:', get_node_status())

Check Node Connection

def is_node_available():
    node = bot.salad.nodes[0]
    return node.connected and node.sessionId is not None

if is_node_available():
    print('Node is ready for operations')
else:
    print('Node is not available')

Player State Preservation

Automatic State Management

Salada automatically saves and restores player states when the node disconnects and reconnects.

await bot.salad.save_player_states()

restored_count = await bot.salad.restore_players()
print(f'Restored {restored_count} players')

await bot.salad.clear_saved_states()

Manual State Operations

async def backup_player_state(guild_id):
    player = bot.salad.getPlayer(guild_id)
    if not player:
        return None
    
    state = {
        'guildId': guild_id,
        'voiceChannel': player.voiceChannel,
        'textChannel': player.textChannel,
        'queue': [track.track for track in player.queue._q],
        'current': player.current,
        'position': player.position,
        'volume': player.volume,
        'paused': player.paused,
        'loop': str(player.queue.loop)
    }
    
    return state

async def restore_player_state(state):
    if not state:
        return False
    
    try:
        player = await bot.salad.createConnection({
            'guildId': state['guildId'],
            'voiceChannel': state['voiceChannel'],
            'textChannel': state['textChannel']
        })
        
        if not player:
            return False
        
        for track_data in state['queue']:
            from salada import Track
            track = Track({'encoded': track_data}, None)
            player.queue.add(track)
        
        if state.get('current'):
            player.current = state['current']
            await player.play()
            await player.seek(state.get('position', 0))
        
        await player.setVolume(state.get('volume', 100))
        
        if state.get('paused'):
            await player.pause(True)
        
        return True
    except Exception as e:
        print(f'Failed to restore player state: {e}')
        return False

Connection Reliability

Circuit Breaker

The node implements a circuit breaker pattern to prevent overwhelming a failing server.

node = bot.salad.nodes[0]

circuit_status = {
    'failures': node._circuit_breaker_failures,
    'threshold': node._circuit_breaker_threshold,
    'open_until': node._circuit_open_until
}

print(f'Circuit breaker status: {circuit_status}')

Exponential Backoff

Reconnection attempts use exponential backoff with jitter to avoid thundering herd problems.

import asyncio

async def monitor_reconnection():
    node = bot.salad.nodes[0]
    
    while True:
        if node._reconnecting:
            print(f'Reconnecting... Attempt: {node._reconnect_attempts}')
        
        await asyncio.sleep(1)

Monitoring

Connection Metrics

class ConnectionMetrics:
    def __init__(self, salad):
        self.salad = salad
        self.metrics = {
            'total_disconnects': 0,
            'total_reconnects': 0,
            'failed_reconnects': 0,
            'last_disconnect': None,
            'players_restored': 0
        }
        
        self.setup_event_listeners()
    
    def setup_event_listeners(self):
        @self.salad.on('nodeDisconnect')
        def on_disconnect(node):
            self.record_disconnect()
        
        @self.salad.on('nodeConnect')
        def on_connect(node):
            self.record_reconnect()
        
        @self.salad.on('nodeReconnectExhausted')
        def on_exhausted(node, attempts):
            self.record_failed_reconnect()
        
        @self.salad.on('playersRestored')
        def on_restored(node, count):
            self.metrics['players_restored'] += count
    
    def record_disconnect(self):
        self.metrics['total_disconnects'] += 1
        self.metrics['last_disconnect'] = asyncio.get_event_loop().time()
    
    def record_reconnect(self):
        self.metrics['total_reconnects'] += 1
    
    def record_failed_reconnect(self):
        self.metrics['failed_reconnects'] += 1
    
    def get_report(self):
        success_rate = 0
        if self.metrics['total_disconnects'] > 0:
            success_rate = (
                self.metrics['total_reconnects'] / 
                self.metrics['total_disconnects']
            ) * 100
        
        return {
            **self.metrics,
            'success_rate': f'{success_rate:.2f}%'
        }

metrics = ConnectionMetrics(bot.salad)

Health Monitoring

import asyncio

async def monitor_node_health():
    while True:
        node = bot.salad.nodes[0]
        
        health = {
            'connected': node.connected,
            'session_id': node.sessionId,
            'players': len(node.players),
            'reconnecting': node._reconnecting,
            'reconnect_attempts': node._reconnect_attempts,
            'stats': node.stats
        }
        
        print(f'Node Health: {health}')
        await asyncio.sleep(30)

asyncio.create_task(monitor_node_health())

Testing

Simulate Node Failure

async def simulate_node_failure():
    node = bot.salad.nodes[0]
    
    print(f'Simulating failure for node: {node.host}:{node.port}')
    
    affected_players = len(node.players)
    print(f'{affected_players} players will be affected')
    
    if node.ws and not node.ws.closed:
        await node.ws.close(1006, 'Simulating node failure')
    
    await asyncio.sleep(10)
    
    print('Node should be attempting to reconnect...')

Reconnection Test

async def test_reconnection():
    print('Starting reconnection test...')
    
    node = bot.salad.nodes[0]
    initial_status = {
        'connected': node.connected,
        'players': len(node.players)
    }
    
    print(f'Initial status: {initial_status}')
    
    await simulate_node_failure()
    
    await asyncio.sleep(15)
    
    final_status = {
        'connected': node.connected,
        'players': len(node.players)
    }
    
    print(f'Final status: {final_status}')
    print(f'Active players after reconnection: {len(bot.salad.players)}')

Error Handling

Connection Error Recovery

@bot.salad.on('nodeError')
async def handle_node_error(node, error):
    print(f'Node {node.host}:{node.port} encountered error: {error}')
    
    if not node.connected:
        print('Node is disconnected, automatic reconnection will be attempted')
        
        if len(bot.salad.players) > 0:
            await notify_users('Music service temporarily unavailable')

async def notify_users(message):
    print(f'NOTIFICATION: {message}')

Best Practices

Optimal Configuration

salad = Salad(client, NODES, {
    'enableReconnect': True,
    'maxReconnectAttempts': 5,
    'infiniteReconnect': True,
    'baseReconnectDelay': 1.5,
    'maxReconnectDelay': 300.0,
    'stateSaveInterval': 5.0
})

Monitoring Setup

import asyncio

async def setup_monitoring():
    metrics = ConnectionMetrics(bot.salad)
    
    while True:
        report = metrics.get_report()
        print(f'Connection Report: {report}')
        
        node_status = get_node_status()
        print(f'Node Status: {node_status}')
        
        await asyncio.sleep(30)

asyncio.create_task(setup_monitoring())