Edit File: spark_driver.cpython-37.pyc
B Z~�_! � @ s� d dl mZ d dlmZ d dlmZ d dlmZ d dlm Z e rhd dl m Z d dl m Z d dlm Z mZ G d d � d e�Zd d � Zd d� Zdd� ZG dd� de�ZG dd� de�ZdS )� )�configure_scope)�Hub)� Integration)�capture_internal_exceptions)�MYPY)�Any)�Optional)�Event�Hintc @ s e Zd ZdZedd� �ZdS )�SparkIntegrationZsparkc C s t � d S )N)�patch_spark_context_init� r r �[/opt/alt/python37/lib/python3.7/site-packages/sentry_sdk/integrations/spark/spark_driver.py� setup_once s zSparkIntegration.setup_onceN)�__name__� __module__� __qualname__� identifier� staticmethodr r r r r r s r c C s6 ddl m} | j}|r2|�d|j� |�d|j� dS )z� Set properties in driver that propagate to worker processes, allowing for workers to have access to those properties. This allows worker integration to have access to app_name and application_id. r )� SparkContextZsentry_app_nameZsentry_application_idN)�pysparkr Z_active_spark_contextZsetLocalProperty�appName� applicationId)r Z spark_contextr r r �_set_app_properties s r c C s4 ddl m} | j}||� t� }| j�� �|� dS )zA Start java gateway server to add custom `SparkListener` r )� ensure_callback_server_startedN)Zpyspark.java_gatewayr Z_gateway�SentryListenerZ_jsc�scZaddSparkListener)r r ZgwZlistenerr r r �_start_sentry_listener( s r c s( ddl m} | j� � fdd�}|| _d S )Nr )r c sZ �� f|�|�}t j�t�d kr$|S t� � t� t� �}|j� fdd��}W d Q R X |S )Nc s t � �� tj�t�d kr | S | �di ��d� �� � | �di ��d� j�d�� | d �d� j�d�� | d �d� j�d �� | d �d � j�d �� | d �d � j � | d �d � j � | d �d� j � | d �d� j � | d �d� j � | �di ��d� j� W d Q R X | S )N�user�idZtagsz executor.idzspark.executor.idzspark-submit.deployModezspark.submit.deployModez driver.hostzspark.driver.hostz driver.portzspark.driver.portZ spark_versionZapp_nameZapplication_id�masterZ spark_home�extraZweb_url)r r �current�get_integrationr � setdefaultZ sparkUserZ_conf�get�versionr r r Z sparkHomeZuiWebUrl)�eventZhint)�selfr r � process_eventG s( z[patch_spark_context_init.<locals>._sentry_patched_spark_context_init.<locals>.process_event)r r" r# r r r r Zadd_event_processor)r( �args�kwargs�initZscoper) )�spark_context_init)r( r �"_sentry_patched_spark_context_init; s zDpatch_spark_context_init.<locals>._sentry_patched_spark_context_init)r r Z_do_init)r r. r )r- r r 5 s .r c @ s� e Zd Zdd� Zdd� Zdd� Zdd� Zd d � Zd d � Zd d� Z dd� Z dd� Z dd� Z dd� Z dd� Zdd� Zdd � Zd d � Zd d � Zd!d"� Zd#d$� Zd%d&� Zd'd(� Zd)d*� Zd+d,� Zd-d.� Zd/d0� ZG d1d2� d2�Zd3S )4� SparkListenerc C s d S )Nr )r( ZapplicationEndr r r �onApplicationEndm s z SparkListener.onApplicationEndc C s d S )Nr )r( ZapplicationStartr r r �onApplicationStartq s z SparkListener.onApplicationStartc C s d S )Nr )r( ZblockManagerAddedr r r �onBlockManagerAddedu s z!SparkListener.onBlockManagerAddedc C s d S )Nr )r( ZblockManagerRemovedr r r �onBlockManagerRemovedy s z#SparkListener.onBlockManagerRemovedc C s d S )Nr )r( Z blockUpdatedr r r �onBlockUpdated} s z SparkListener.onBlockUpdatedc C s d S )Nr )r( ZenvironmentUpdater r r �onEnvironmentUpdate� s z!SparkListener.onEnvironmentUpdatec C s d S )Nr )r( Z executorAddedr r r �onExecutorAdded� s z SparkListener.onExecutorAddedc C s d S )Nr )r( ZexecutorBlacklistedr r r �onExecutorBlacklisted� s z#SparkListener.onExecutorBlacklistedc C s d S )Nr )r( ZexecutorBlacklistedForStager r r � onExecutorBlacklistedForStage� s z+SparkListener.onExecutorBlacklistedForStagec C s d S )Nr )r( ZexecutorMetricsUpdater r r �onExecutorMetricsUpdate� s z%SparkListener.onExecutorMetricsUpdatec C s d S )Nr )r( ZexecutorRemovedr r r �onExecutorRemoved� s z SparkListener.onExecutorRemovedc C s d S )Nr )r( �jobEndr r r �onJobEnd� s zSparkListener.onJobEndc C s d S )Nr )r( �jobStartr r r � onJobStart� s zSparkListener.onJobStartc C s d S )Nr )r( ZnodeBlacklistedr r r �onNodeBlacklisted� s z SparkListener.onNodeBlacklistedc C s d S )Nr )r( ZnodeBlacklistedForStager r r �onNodeBlacklistedForStage� s z'SparkListener.onNodeBlacklistedForStagec C s d S )Nr )r( ZnodeUnblacklistedr r r �onNodeUnblacklisted� s z!SparkListener.onNodeUnblacklistedc C s d S )Nr )r( r' r r r � onOtherEvent� s zSparkListener.onOtherEventc C s d S )Nr )r( ZspeculativeTaskr r r �onSpeculativeTaskSubmitted� s z(SparkListener.onSpeculativeTaskSubmittedc C s d S )Nr )r( �stageCompletedr r r �onStageCompleted� s z SparkListener.onStageCompletedc C s d S )Nr )r( �stageSubmittedr r r �onStageSubmitted� s z SparkListener.onStageSubmittedc C s d S )Nr )r( ZtaskEndr r r � onTaskEnd� s zSparkListener.onTaskEndc C s d S )Nr )r( ZtaskGettingResultr r r �onTaskGettingResult� s z!SparkListener.onTaskGettingResultc C s d S )Nr )r( Z taskStartr r r � onTaskStart� s zSparkListener.onTaskStartc C s d S )Nr )r( Z unpersistRDDr r r �onUnpersistRDD� s z SparkListener.onUnpersistRDDc @ s e Zd ZdgZdS )zSparkListener.Javaz1org.apache.spark.scheduler.SparkListenerInterfaceN)r r r Z implementsr r r r �Java� s rL N) r r r r0 r1 r2 r3 r4 r5 r6 r7 r8 r9 r: r< r> r? r@ rA rB rC rE rG rH rI rJ rK rL r r r r r/ l s2 r/ c @ s4 e Zd Zdd� Zdd� Zdd� Zdd� Zd d � Zd S ) r c C s t j| _d S )N)r r" �hub)r( r r r �__init__� s zSentryListener.__init__c C s( d� |�� �}| jjd|d� t� d S )NzJob {} Started�info)�level�message)�format�jobIdrM �add_breadcrumbr )r( r= rQ r r r r> � s zSentryListener.onJobStartc C sd d}d}d|� � �� i}|� � �� dkr<d}d�|�� �}nd}d�|�� �}| jj|||d� d S ) N� �resultZ JobSucceededrO z Job {} Ended�warningz Job {} Failed)rP rQ �data)Z jobResult�toStringrR rS rM rT )r( r; rP rQ rX r r r r< � s zSentryListener.onJobEndc C sD |� � }d�|�� �}|�� |�� d�}| jjd||d� t� d S )NzStage {} Submitted)� attemptId�namerO )rP rQ rX )� stageInforR �stageIdrZ r[ rM rT r )r( rF � stage_inforQ rX r r r rG � s z SentryListener.onStageSubmittedc C s� ddl m} |�� }d}d}|�� |�� d�}y&|�� �� |d< d�|�� �}d}W n& |k rz d�|�� �}d }Y nX | j j |||d � d S ) Nr )� Py4JJavaErrorrU )rZ r[ �reasonzStage {} FailedrW zStage {} CompletedrO )rP rQ rX ) Z py4j.protocolr_ r\ rZ r[ Z failureReasonr% rR r] rM rT )r( rD r_ r^ rQ rP rX r r r rE � s z SentryListener.onStageCompletedN)r r r rN r> r< rG rE r r r r r � s r N)� sentry_sdkr Zsentry_sdk.hubr Zsentry_sdk.integrationsr Zsentry_sdk.utilsr Zsentry_sdk._typesr �typingr r r r r r r r �objectr/ r r r r r �<module> s 7g